 Caffe2 - Python API A deep learning, cross platform ML framework
linear.py
1 import math
2
3 import torch
4 from torch.nn.parameter import Parameter
5 from .. import functional as F
6 from .. import init
7 from .module import Module
8 from ..._jit_internal import weak_module, weak_script_method
9
10
11 @weak_module
12 class Linear(Module):
13  r"""Applies a linear transformation to the incoming data: :math:y = xA^T + b
14
15  Args:
16  in_features: size of each input sample
17  out_features: size of each output sample
18  bias: If set to False, the layer will not learn an additive bias.
19  Default: True
20
21  Shape:
22  - Input: :math:(N, *, H_{in}) where :math:* means any number of
23  additional dimensions and :math:H_{in} = \text{in\_features}
24  - Output: :math:(N, *, H_{out}) where all but the last dimension
25  are the same shape as the input and :math:H_{out} = \text{out\_features}.
26
27  Attributes:
28  weight: the learnable weights of the module of shape
29  :math:(\text{out\_features}, \text{in\_features}). The values are
30  initialized from :math:\mathcal{U}(-\sqrt{k}, \sqrt{k}), where
31  :math:k = \frac{1}{\text{in\_features}}
32  bias: the learnable bias of the module of shape :math:(\text{out\_features}).
33  If :attr:bias is True, the values are initialized from
34  :math:\mathcal{U}(-\sqrt{k}, \sqrt{k}) where
35  :math:k = \frac{1}{\text{in\_features}}
36
37  Examples::
38
39  >>> m = nn.Linear(20, 30)
40  >>> input = torch.randn(128, 20)
41  >>> output = m(input)
42  >>> print(output.size())
43  torch.Size([128, 30])
44  """
45  __constants__ = ['bias']
46
47  def __init__(self, in_features, out_features, bias=True):
48  super(Linear, self).__init__()
49  self.in_features = in_features
50  self.out_features = out_features
51  self.weight = Parameter(torch.Tensor(out_features, in_features))
52  if bias:
53  self.bias = Parameter(torch.Tensor(out_features))
54  else:
55  self.register_parameter('bias', None)
56  self.reset_parameters()
57
58  def reset_parameters(self):
59  init.kaiming_uniform_(self.weight, a=math.sqrt(5))
60  if self.bias is not None:
61  fan_in, _ = init._calculate_fan_in_and_fan_out(self.weight)
62  bound = 1 / math.sqrt(fan_in)
63  init.uniform_(self.bias, -bound, bound)
64
65  @weak_script_method
66  def forward(self, input):
67  return F.linear(input, self.weight, self.bias)
68
69  def extra_repr(self):
70  return 'in_features={}, out_features={}, bias={}'.format(
71  self.in_features, self.out_features, self.bias is not None
72  )
73
74
75 @weak_module
76 class Bilinear(Module):
77  r"""Applies a bilinear transformation to the incoming data:
78  :math:y = x_1 A x_2 + b
79
80  Args:
81  in1_features: size of each first input sample
82  in2_features: size of each second input sample
83  out_features: size of each output sample
84  bias: If set to False, the layer will not learn an additive bias.
85  Default: True
86
87  Shape:
88  - Input1: :math:(N, *, H_{in1}) where :math:H_{in1}=\text{in1\_features} and
89  :math:* means any number of additional dimensions. All but the last dimension
90  of the inputs should be the same.
91  - Input2: :math:(N, *, H_{in2}) where :math:H_{in2}=\text{in2\_features}.
92  - Output: :math:(N, *, H_{out}) where :math:H_{out}=\text{out\_features}
93  and all but the last dimension are the same shape as the input.
94
95  Attributes:
96  weight: the learnable weights of the module of shape
97  :math:(\text{out\_features}, \text{in1\_features}, \text{in2\_features}).
98  The values are initialized from :math:\mathcal{U}(-\sqrt{k}, \sqrt{k}), where
99  :math:k = \frac{1}{\text{in1\_features}}
100  bias: the learnable bias of the module of shape :math:(\text{out\_features}).
101  If :attr:bias is True, the values are initialized from
102  :math:\mathcal{U}(-\sqrt{k}, \sqrt{k}), where
103  :math:k = \frac{1}{\text{in1\_features}}
104
105  Examples::
106
107  >>> m = nn.Bilinear(20, 30, 40)
108  >>> input1 = torch.randn(128, 20)
109  >>> input2 = torch.randn(128, 30)
110  >>> output = m(input1, input2)
111  >>> print(output.size())
112  torch.Size([128, 40])
113  """
114  __constants__ = ['in1_features', 'in2_features', 'out_features', 'bias']
115
116  def __init__(self, in1_features, in2_features, out_features, bias=True):
117  super(Bilinear, self).__init__()
118  self.in1_features = in1_features
119  self.in2_features = in2_features
120  self.out_features = out_features
121  self.weight = Parameter(torch.Tensor(out_features, in1_features, in2_features))
122
123  if bias:
124  self.bias = Parameter(torch.Tensor(out_features))
125  else:
126  self.register_parameter('bias', None)
127  self.reset_parameters()
128
129  def reset_parameters(self):
130  bound = 1 / math.sqrt(self.weight.size(1))
131  init.uniform_(self.weight, -bound, bound)
132  if self.bias is not None:
133  init.uniform_(self.bias, -bound, bound)
134
135  @weak_script_method
136  def forward(self, input1, input2):
137  return F.bilinear(input1, input2, self.weight, self.bias)
138
139  def extra_repr(self):
140  return 'in1_features={}, in2_features={}, out_features={}, bias={}'.format(
141  self.in1_features, self.in2_features, self.out_features, self.bias is not None
142  )
143
144 # TODO: PartialLinear - maybe in sparse?