Caffe2 - Python API
A deep learning, cross platform ML framework
activation.py
1 import warnings
2 import torch
3 from torch.nn.parameter import Parameter
4 
5 from .module import Module
6 from .. import functional as F
7 from ..._jit_internal import weak_module, weak_script_method
8 
9 
10 @weak_module
11 class Threshold(Module):
12  r"""Thresholds each element of the input Tensor.
13 
14  Threshold is defined as:
15 
16  .. math::
17  y =
18  \begin{cases}
19  x, &\text{ if } x > \text{threshold} \\
20  \text{value}, &\text{ otherwise }
21  \end{cases}
22 
23  Args:
24  threshold: The value to threshold at
25  value: The value to replace with
26  inplace: can optionally do the operation in-place. Default: ``False``
27 
28  Shape:
29  - Input: :math:`(N, *)` where `*` means, any number of additional
30  dimensions
31  - Output: :math:`(N, *)`, same shape as the input
32 
33  Examples::
34 
35  >>> m = nn.Threshold(0.1, 20)
36  >>> input = torch.randn(2)
37  >>> output = m(input)
38  """
39  __constants__ = ['threshold', 'value', 'inplace']
40 
41  def __init__(self, threshold, value, inplace=False):
42  super(Threshold, self).__init__()
43  self.threshold = threshold
44  self.value = value
45  self.inplace = inplace
46  # TODO: check in THNN (if inplace == True, then assert value <= threshold)
47 
48  @weak_script_method
49  def forward(self, input):
50  return F.threshold(input, self.threshold, self.value, self.inplace)
51 
52  def extra_repr(self):
53  inplace_str = ', inplace' if self.inplace else ''
54  return 'threshold={}, value={}{}'.format(
55  self.threshold, self.value, inplace_str
56  )
57 
58 
59 @weak_module
60 class ReLU(Threshold):
61  r"""Applies the rectified linear unit function element-wise:
62 
63  :math:`\text{ReLU}(x)= \max(0, x)`
64 
65  Args:
66  inplace: can optionally do the operation in-place. Default: ``False``
67 
68  Shape:
69  - Input: :math:`(N, *)` where `*` means, any number of additional
70  dimensions
71  - Output: :math:`(N, *)`, same shape as the input
72 
73  .. image:: scripts/activation_images/ReLU.png
74 
75  Examples::
76 
77  >>> m = nn.ReLU()
78  >>> input = torch.randn(2)
79  >>> output = m(input)
80 
81 
82  An implementation of CReLU - https://arxiv.org/abs/1603.05201
83 
84  >>> m = nn.ReLU()
85  >>> input = torch.randn(2).unsqueeze(0)
86  >>> output = torch.cat((m(input),m(-input)))
87  """
88 
89  def __init__(self, inplace=False):
90  super(ReLU, self).__init__(0., 0., inplace)
91 
92  def extra_repr(self):
93  inplace_str = 'inplace' if self.inplace else ''
94  return inplace_str
95 
96 
97 @weak_module
98 class RReLU(Module):
99  r"""Applies the randomized leaky rectified liner unit function, element-wise,
100  as described in the paper:
101 
102  `Empirical Evaluation of Rectified Activations in Convolutional Network`_.
103 
104  The function is defined as:
105 
106  .. math::
107  \text{RReLU}(x) =
108  \begin{cases}
109  x & \text{if } x \geq 0 \\
110  ax & \text{ otherwise }
111  \end{cases}
112 
113  where :math:`a` is randomly sampled from uniform distribution
114  :math:`\mathcal{U}(\text{lower}, \text{upper})`.
115 
116  See: https://arxiv.org/pdf/1505.00853.pdf
117 
118  Args:
119  lower: lower bound of the uniform distribution. Default: :math:`\frac{1}{8}`
120  upper: upper bound of the uniform distribution. Default: :math:`\frac{1}{3}`
121  inplace: can optionally do the operation in-place. Default: ``False``
122 
123  Shape:
124  - Input: :math:`(N, *)` where `*` means, any number of additional
125  dimensions
126  - Output: :math:`(N, *)`, same shape as the input
127 
128  Examples::
129 
130  >>> m = nn.RReLU(0.1, 0.3)
131  >>> input = torch.randn(2)
132  >>> output = m(input)
133 
134  .. _`Empirical Evaluation of Rectified Activations in Convolutional Network`:
135  https://arxiv.org/abs/1505.00853
136  """
137  __constants__ = ['lower', 'upper', 'inplace']
138 
139  def __init__(self, lower=1. / 8, upper=1. / 3, inplace=False):
140  super(RReLU, self).__init__()
141  self.lower = lower
142  self.upper = upper
143  self.inplace = inplace
144 
145  @weak_script_method
146  def forward(self, input):
147  return F.rrelu(input, self.lower, self.upper, self.training, self.inplace)
148 
149  def extra_repr(self):
150  inplace_str = ', inplace' if self.inplace else ''
151  return 'lower={}, upper={}{}'.format(self.lower, self.upper, inplace_str)
152 
153 
154 @weak_module
155 class Hardtanh(Module):
156  r"""Applies the HardTanh function element-wise
157 
158  HardTanh is defined as:
159 
160  .. math::
161  \text{HardTanh}(x) = \begin{cases}
162  1 & \text{ if } x > 1 \\
163  -1 & \text{ if } x < -1 \\
164  x & \text{ otherwise } \\
165  \end{cases}
166 
167  The range of the linear region :math:`[-1, 1]` can be adjusted using
168  :attr:`min_val` and :attr:`max_val`.
169 
170  Args:
171  min_val: minimum value of the linear region range. Default: -1
172  max_val: maximum value of the linear region range. Default: 1
173  inplace: can optionally do the operation in-place. Default: ``False``
174 
175  Keyword arguments :attr:`min_value` and :attr:`max_value`
176  have been deprecated in favor of :attr:`min_val` and :attr:`max_val`.
177 
178  Shape:
179  - Input: :math:`(N, *)` where `*` means, any number of additional
180  dimensions
181  - Output: :math:`(N, *)`, same shape as the input
182 
183  .. image:: scripts/activation_images/Hardtanh.png
184 
185  Examples::
186 
187  >>> m = nn.Hardtanh(-2, 2)
188  >>> input = torch.randn(2)
189  >>> output = m(input)
190  """
191  __constants__ = ['min_val', 'max_val', 'inplace']
192 
193  def __init__(self, min_val=-1., max_val=1., inplace=False, min_value=None, max_value=None):
194  super(Hardtanh, self).__init__()
195  if min_value is not None:
196  warnings.warn("keyword argument min_value is deprecated and renamed to min_val")
197  min_val = min_value
198  if max_value is not None:
199  warnings.warn("keyword argument max_value is deprecated and renamed to max_val")
200  max_val = max_value
201 
202  self.min_val = min_val
203  self.max_val = max_val
204  self.inplace = inplace
205  assert self.max_val > self.min_val
206 
207  @weak_script_method
208  def forward(self, input):
209  return F.hardtanh(input, self.min_val, self.max_val, self.inplace)
210 
211  def extra_repr(self):
212  inplace_str = ', inplace' if self.inplace else ''
213  return 'min_val={}, max_val={}{}'.format(
214  self.min_val, self.max_val, inplace_str
215  )
216 
217 
218 @weak_module
220  r"""Applies the element-wise function:
221 
222  .. math::
223  \text{ReLU6}(x) = \min(\max(0,x), 6)
224 
225  Args:
226  inplace: can optionally do the operation in-place. Default: ``False``
227 
228  Shape:
229  - Input: :math:`(N, *)` where `*` means, any number of additional
230  dimensions
231  - Output: :math:`(N, *)`, same shape as the input
232 
233  .. image:: scripts/activation_images/ReLU6.png
234 
235  Examples::
236 
237  >>> m = nn.ReLU6()
238  >>> input = torch.randn(2)
239  >>> output = m(input)
240  """
241 
242  def __init__(self, inplace=False):
243  super(ReLU6, self).__init__(0., 6., inplace)
244 
245  def extra_repr(self):
246  inplace_str = 'inplace' if self.inplace else ''
247  return inplace_str
248 
249 
250 @weak_module
251 class Sigmoid(Module):
252  r"""Applies the element-wise function:
253 
254  .. math::
255  \text{Sigmoid}(x) = \frac{1}{1 + \exp(-x)}
256 
257 
258  Shape:
259  - Input: :math:`(N, *)` where `*` means, any number of additional
260  dimensions
261  - Output: :math:`(N, *)`, same shape as the input
262 
263  .. image:: scripts/activation_images/Sigmoid.png
264 
265  Examples::
266 
267  >>> m = nn.Sigmoid()
268  >>> input = torch.randn(2)
269  >>> output = m(input)
270  """
271 
272  @weak_script_method
273  def forward(self, input):
274  return torch.sigmoid(input)
275 
276 
277 @weak_module
278 class Tanh(Module):
279  r"""Applies the element-wise function:
280 
281  .. math::
282  \text{Tanh}(x) = \tanh(x) = \frac{e^x - e^{-x}} {e^x + e^{-x}}
283 
284  Shape:
285  - Input: :math:`(N, *)` where `*` means, any number of additional
286  dimensions
287  - Output: :math:`(N, *)`, same shape as the input
288 
289  .. image:: scripts/activation_images/Tanh.png
290 
291  Examples::
292 
293  >>> m = nn.Tanh()
294  >>> input = torch.randn(2)
295  >>> output = m(input)
296  """
297 
298  @weak_script_method
299  def forward(self, input):
300  return torch.tanh(input)
301 
302 
303 @weak_module
304 class ELU(Module):
305  r"""Applies the element-wise function:
306 
307  .. math::
308  \text{ELU}(x) = \max(0,x) + \min(0, \alpha * (\exp(x) - 1))
309 
310  Args:
311  alpha: the :math:`\alpha` value for the ELU formulation. Default: 1.0
312  inplace: can optionally do the operation in-place. Default: ``False``
313 
314  Shape:
315  - Input: :math:`(N, *)` where `*` means, any number of additional
316  dimensions
317  - Output: :math:`(N, *)`, same shape as the input
318 
319  .. image:: scripts/activation_images/ELU.png
320 
321  Examples::
322 
323  >>> m = nn.ELU()
324  >>> input = torch.randn(2)
325  >>> output = m(input)
326  """
327  __constants__ = ['alpha', 'inplace']
328 
329  def __init__(self, alpha=1., inplace=False):
330  super(ELU, self).__init__()
331  self.alpha = alpha
332  self.inplace = inplace
333 
334  @weak_script_method
335  def forward(self, input):
336  return F.elu(input, self.alpha, self.inplace)
337 
338  def extra_repr(self):
339  inplace_str = ', inplace' if self.inplace else ''
340  return 'alpha={}{}'.format(self.alpha, inplace_str)
341 
342 
343 @weak_module
344 class CELU(Module):
345  r"""Applies the element-wise function:
346 
347  .. math::
348  \text{CELU}(x) = \max(0,x) + \min(0, \alpha * (\exp(x/\alpha) - 1))
349 
350  More details can be found in the paper `Continuously Differentiable Exponential Linear Units`_ .
351 
352  Args:
353  alpha: the :math:`\alpha` value for the CELU formulation. Default: 1.0
354  inplace: can optionally do the operation in-place. Default: ``False``
355 
356  Shape:
357  - Input: :math:`(N, *)` where `*` means, any number of additional
358  dimensions
359  - Output: :math:`(N, *)`, same shape as the input
360 
361  .. image:: scripts/activation_images/CELU.png
362 
363  Examples::
364 
365  >>> m = nn.CELU()
366  >>> input = torch.randn(2)
367  >>> output = m(input)
368 
369  .. _`Continuously Differentiable Exponential Linear Units`:
370  https://arxiv.org/abs/1704.07483
371  """
372  __constants__ = ['alpha', 'inplace']
373 
374  def __init__(self, alpha=1., inplace=False):
375  super(CELU, self).__init__()
376  self.alpha = alpha
377  self.inplace = inplace
378 
379  @weak_script_method
380  def forward(self, input):
381  return F.celu(input, self.alpha, self.inplace)
382 
383  def extra_repr(self):
384  inplace_str = ', inplace' if self.inplace else ''
385  return 'alpha={}{}'.format(self.alpha, inplace_str)
386 
387 
388 @weak_module
389 class SELU(Module):
390  r"""Applied element-wise, as:
391 
392  .. math::
393  \text{SELU}(x) = \text{scale} * (\max(0,x) + \min(0, \alpha * (\exp(x) - 1)))
394 
395  with :math:`\alpha = 1.6732632423543772848170429916717` and
396  :math:`\text{scale} = 1.0507009873554804934193349852946`.
397 
398  More details can be found in the paper `Self-Normalizing Neural Networks`_ .
399 
400  Args:
401  inplace (bool, optional): can optionally do the operation in-place. Default: ``False``
402 
403  Shape:
404  - Input: :math:`(N, *)` where `*` means, any number of additional
405  dimensions
406  - Output: :math:`(N, *)`, same shape as the input
407 
408  .. image:: scripts/activation_images/SELU.png
409 
410  Examples::
411 
412  >>> m = nn.SELU()
413  >>> input = torch.randn(2)
414  >>> output = m(input)
415 
416  .. _Self-Normalizing Neural Networks: https://arxiv.org/abs/1706.02515
417  """
418  __constants__ = ['inplace']
419 
420  def __init__(self, inplace=False):
421  super(SELU, self).__init__()
422  self.inplace = inplace
423 
424  @weak_script_method
425  def forward(self, input):
426  return F.selu(input, self.inplace)
427 
428  def extra_repr(self):
429  inplace_str = 'inplace' if self.inplace else ''
430  return inplace_str
431 
432 
433 @weak_module
434 class GLU(Module):
435  r"""Applies the gated linear unit function
436  :math:`{GLU}(a, b)= a \otimes \sigma(b)` where :math:`a` is the first half
437  of the input matrices and :math:`b` is the second half.
438 
439  Args:
440  dim (int): the dimension on which to split the input. Default: -1
441 
442  Shape:
443  - Input: :math:`(\ast_1, N, \ast_2)` where `*` means, any number of additional
444  dimensions
445  - Output: :math:`(\ast_1, M, \ast_2)` where :math:`M=N/2`
446 
447  Examples::
448 
449  >>> m = nn.GLU()
450  >>> input = torch.randn(4, 2)
451  >>> output = m(input)
452  """
453  __constants__ = ['dim']
454 
455  def __init__(self, dim=-1):
456  super(GLU, self).__init__()
457  self.dim = dim
458 
459  @weak_script_method
460  def forward(self, input):
461  return F.glu(input, self.dim)
462 
463  def extra_repr(self):
464  return 'dim={}'.format(self.dim)
465 
466 
467 @weak_module
468 class Hardshrink(Module):
469  r"""Applies the hard shrinkage function element-wise:
470 
471  .. math::
472  \text{HardShrink}(x) =
473  \begin{cases}
474  x, & \text{ if } x > \lambda \\
475  x, & \text{ if } x < -\lambda \\
476  0, & \text{ otherwise }
477  \end{cases}
478 
479  Args:
480  lambd: the :math:`\lambda` value for the Hardshrink formulation. Default: 0.5
481 
482  Shape:
483  - Input: :math:`(N, *)` where `*` means, any number of additional
484  dimensions
485  - Output: :math:`(N, *)`, same shape as the input
486 
487  .. image:: scripts/activation_images/Hardshrink.png
488 
489  Examples::
490 
491  >>> m = nn.Hardshrink()
492  >>> input = torch.randn(2)
493  >>> output = m(input)
494  """
495  __constants__ = ['lambd']
496 
497  def __init__(self, lambd=0.5):
498  super(Hardshrink, self).__init__()
499  self.lambd = lambd
500 
501  @weak_script_method
502  def forward(self, input):
503  return F.hardshrink(input, self.lambd)
504 
505  def extra_repr(self):
506  return '{}'.format(self.lambd)
507 
508 
509 @weak_module
510 class LeakyReLU(Module):
511  r"""Applies the element-wise function:
512 
513  .. math::
514  \text{LeakyReLU}(x) = \max(0, x) + \text{negative\_slope} * \min(0, x)
515 
516 
517  or
518 
519  .. math::
520  \text{LeakyRELU}(x) =
521  \begin{cases}
522  x, & \text{ if } x \geq 0 \\
523  \text{negative\_slope} \times x, & \text{ otherwise }
524  \end{cases}
525 
526  Args:
527  negative_slope: Controls the angle of the negative slope. Default: 1e-2
528  inplace: can optionally do the operation in-place. Default: ``False``
529 
530  Shape:
531  - Input: :math:`(N, *)` where `*` means, any number of additional
532  dimensions
533  - Output: :math:`(N, *)`, same shape as the input
534 
535  .. image:: scripts/activation_images/LeakyReLU.png
536 
537  Examples::
538 
539  >>> m = nn.LeakyReLU(0.1)
540  >>> input = torch.randn(2)
541  >>> output = m(input)
542  """
543  __constants__ = ['inplace', 'negative_slope']
544 
545  def __init__(self, negative_slope=1e-2, inplace=False):
546  super(LeakyReLU, self).__init__()
547  self.negative_slope = negative_slope
548  self.inplace = inplace
549 
550  @weak_script_method
551  def forward(self, input):
552  return F.leaky_relu(input, self.negative_slope, self.inplace)
553 
554  def extra_repr(self):
555  inplace_str = ', inplace' if self.inplace else ''
556  return 'negative_slope={}{}'.format(self.negative_slope, inplace_str)
557 
558 
559 @weak_module
560 class LogSigmoid(Module):
561  r"""Applies the element-wise function:
562 
563  .. math::
564  \text{LogSigmoid}(x) = \log\left(\frac{ 1 }{ 1 + \exp(-x)}\right)
565 
566  Shape:
567  - Input: :math:`(N, *)` where `*` means, any number of additional
568  dimensions
569  - Output: :math:`(N, *)`, same shape as the input
570 
571  .. image:: scripts/activation_images/LogSigmoid.png
572 
573  Examples::
574 
575  >>> m = nn.LogSigmoid()
576  >>> input = torch.randn(2)
577  >>> output = m(input)
578  """
579 
580  @weak_script_method
581  def forward(self, input):
582  return F.logsigmoid(input)
583 
584 
585 @weak_module
586 class Softplus(Module):
587  r"""Applies the element-wise function:
588 
589  .. math::
590  \text{Softplus}(x) = \frac{1}{\beta} * \log(1 + \exp(\beta * x))
591 
592  SoftPlus is a smooth approximation to the ReLU function and can be used
593  to constrain the output of a machine to always be positive.
594 
595  For numerical stability the implementation reverts to the linear function
596  for inputs above a certain value.
597 
598  Args:
599  beta: the :math:`\beta` value for the Softplus formulation. Default: 1
600  threshold: values above this revert to a linear function. Default: 20
601 
602  Shape:
603  - Input: :math:`(N, *)` where `*` means, any number of additional
604  dimensions
605  - Output: :math:`(N, *)`, same shape as the input
606 
607  .. image:: scripts/activation_images/Softplus.png
608 
609  Examples::
610 
611  >>> m = nn.Softplus()
612  >>> input = torch.randn(2)
613  >>> output = m(input)
614  """
615  __constants__ = ['beta', 'threshold']
616 
617  def __init__(self, beta=1, threshold=20):
618  super(Softplus, self).__init__()
619  self.beta = beta
620  self.threshold = threshold
621 
622  @weak_script_method
623  def forward(self, input):
624  return F.softplus(input, self.beta, self.threshold)
625 
626  def extra_repr(self):
627  return 'beta={}, threshold={}'.format(self.beta, self.threshold)
628 
629 
630 @weak_module
631 class Softshrink(Module):
632  r"""Applies the soft shrinkage function elementwise:
633 
634  .. math::
635  \text{SoftShrinkage}(x) =
636  \begin{cases}
637  x - \lambda, & \text{ if } x > \lambda \\
638  x + \lambda, & \text{ if } x < -\lambda \\
639  0, & \text{ otherwise }
640  \end{cases}
641 
642  Args:
643  lambd: the :math:`\lambda` value for the Softshrink formulation. Default: 0.5
644 
645  Shape:
646  - Input: :math:`(N, *)` where `*` means, any number of additional
647  dimensions
648  - Output: :math:`(N, *)`, same shape as the input
649 
650  .. image:: scripts/activation_images/Softshrink.png
651 
652  Examples::
653 
654  >>> m = nn.Softshrink()
655  >>> input = torch.randn(2)
656  >>> output = m(input)
657  """
658  __constants__ = ['lambd']
659 
660  def __init__(self, lambd=0.5):
661  super(Softshrink, self).__init__()
662  self.lambd = lambd
663 
664  @weak_script_method
665  def forward(self, input):
666  return F.softshrink(input, self.lambd)
667 
668  def extra_repr(self):
669  return str(self.lambd)
670 
671 
672 @weak_module
673 class PReLU(Module):
674  r"""Applies the element-wise function:
675 
676  .. math::
677  \text{PReLU}(x) = \max(0,x) + a * \min(0,x)
678 
679  or
680 
681  .. math::
682  \text{PReLU}(x) =
683  \begin{cases}
684  x, & \text{ if } x \geq 0 \\
685  ax, & \text{ otherwise }
686  \end{cases}
687 
688  Here :math:`a` is a learnable parameter. When called without arguments, `nn.PReLU()` uses a single
689  parameter :math:`a` across all input channels. If called with `nn.PReLU(nChannels)`,
690  a separate :math:`a` is used for each input channel.
691 
692 
693  .. note::
694  weight decay should not be used when learning :math:`a` for good performance.
695 
696  .. note::
697  Channel dim is the 2nd dim of input. When input has dims < 2, then there is
698  no channel dim and the number of channels = 1.
699 
700  Args:
701  num_parameters (int): number of :math:`a` to learn.
702  Although it takes an int as input, there is only two values are legitimate:
703  1, or the number of channels at input. Default: 1
704  init (float): the initial value of :math:`a`. Default: 0.25
705 
706  Shape:
707  - Input: :math:`(N, *)` where `*` means, any number of additional
708  dimensions
709  - Output: :math:`(N, *)`, same shape as the input
710 
711  Attributes:
712  weight (Tensor): the learnable weights of shape (:attr:`num_parameters`).
713 
714  .. image:: scripts/activation_images/PReLU.png
715 
716  Examples::
717 
718  >>> m = nn.PReLU()
719  >>> input = torch.randn(2)
720  >>> output = m(input)
721  """
722 
723  def __init__(self, num_parameters=1, init=0.25):
724  self.num_parameters = num_parameters
725  super(PReLU, self).__init__()
726  self.weight = Parameter(torch.Tensor(num_parameters).fill_(init))
727 
728  @weak_script_method
729  def forward(self, input):
730  return F.prelu(input, self.weight)
731 
732  def extra_repr(self):
733  return 'num_parameters={}'.format(self.num_parameters)
734 
735 
736 @weak_module
737 class Softsign(Module):
738  r"""Applies the element-wise function:
739 
740  .. math::
741  \text{SoftSign}(x) = \frac{x}{ 1 + |x|}
742 
743  Shape:
744  - Input: :math:`(N, *)` where `*` means, any number of additional
745  dimensions
746  - Output: :math:`(N, *)`, same shape as the input
747 
748  .. image:: scripts/activation_images/Softsign.png
749 
750  Examples::
751 
752  >>> m = nn.Softsign()
753  >>> input = torch.randn(2)
754  >>> output = m(input)
755  """
756 
757  @weak_script_method
758  def forward(self, input):
759  return F.softsign(input)
760 
761 
762 @weak_module
763 class Tanhshrink(Module):
764  r"""Applies the element-wise function:
765 
766  .. math::
767  \text{Tanhshrink}(x) = x - \text{Tanh}(x)
768 
769  Shape:
770  - Input: :math:`(N, *)` where `*` means, any number of additional
771  dimensions
772  - Output: :math:`(N, *)`, same shape as the input
773 
774  .. image:: scripts/activation_images/Tanhshrink.png
775 
776  Examples::
777 
778  >>> m = nn.Tanhshrink()
779  >>> input = torch.randn(2)
780  >>> output = m(input)
781  """
782 
783  @weak_script_method
784  def forward(self, input):
785  return F.tanhshrink(input)
786 
787 
788 @weak_module
789 class Softmin(Module):
790  r"""Applies the Softmin function to an n-dimensional input Tensor
791  rescaling them so that the elements of the n-dimensional output Tensor
792  lie in the range `[0, 1]` and sum to 1.
793 
794  Softmin is defined as:
795 
796  .. math::
797  \text{Softmin}(x_{i}) = \frac{\exp(-x_i)}{\sum_j \exp(-x_j)}
798 
799  Shape:
800  - Input: :math:`(*)` where `*` means, any number of additional
801  dimensions
802  - Output: :math:`(*)`, same shape as the input
803 
804  Arguments:
805  dim (int): A dimension along which Softmin will be computed (so every slice
806  along dim will sum to 1).
807 
808  Returns:
809  a Tensor of the same dimension and shape as the input, with
810  values in the range [0, 1]
811 
812  Examples::
813 
814  >>> m = nn.Softmin()
815  >>> input = torch.randn(2, 3)
816  >>> output = m(input)
817  """
818  __constants__ = ['dim']
819 
820  def __init__(self, dim=None):
821  super(Softmin, self).__init__()
822  self.dim = dim
823 
824  @weak_script_method
825  def forward(self, input):
826  return F.softmin(input, self.dim, _stacklevel=5)
827 
828 
829 @weak_module
830 class Softmax(Module):
831  r"""Applies the Softmax function to an n-dimensional input Tensor
832  rescaling them so that the elements of the n-dimensional output Tensor
833  lie in the range [0,1] and sum to 1.
834 
835  Softmax is defined as:
836 
837  .. math::
838  \text{Softmax}(x_{i}) = \frac{\exp(x_i)}{\sum_j \exp(x_j)}
839 
840  Shape:
841  - Input: :math:`(*)` where `*` means, any number of additional
842  dimensions
843  - Output: :math:`(*)`, same shape as the input
844 
845  Returns:
846  a Tensor of the same dimension and shape as the input with
847  values in the range [0, 1]
848 
849  Arguments:
850  dim (int): A dimension along which Softmax will be computed (so every slice
851  along dim will sum to 1).
852 
853  .. note::
854  This module doesn't work directly with NLLLoss,
855  which expects the Log to be computed between the Softmax and itself.
856  Use `LogSoftmax` instead (it's faster and has better numerical properties).
857 
858  Examples::
859 
860  >>> m = nn.Softmax()
861  >>> input = torch.randn(2, 3)
862  >>> output = m(input)
863  """
864  __constants__ = ['dim']
865 
866  def __init__(self, dim=None):
867  super(Softmax, self).__init__()
868  self.dim = dim
869 
870  def __setstate__(self, state):
871  self.__dict__.update(state)
872  if not hasattr(self, 'dim'):
873  self.dim = None
874 
875  @weak_script_method
876  def forward(self, input):
877  return F.softmax(input, self.dim, _stacklevel=5)
878 
879 
880 @weak_module
881 class Softmax2d(Module):
882  r"""Applies SoftMax over features to each spatial location.
883 
884  When given an image of ``Channels x Height x Width``, it will
885  apply `Softmax` to each location :math:`(Channels, h_i, w_j)`
886 
887  Shape:
888  - Input: :math:`(N, C, H, W)`
889  - Output: :math:`(N, C, H, W)` (same shape as input)
890 
891  Returns:
892  a Tensor of the same dimension and shape as the input with
893  values in the range [0, 1]
894 
895  Examples::
896 
897  >>> m = nn.Softmax2d()
898  >>> # you softmax over the 2nd dimension
899  >>> input = torch.randn(2, 3, 12, 13)
900  >>> output = m(input)
901  """
902 
903  @weak_script_method
904  def forward(self, input):
905  assert input.dim() == 4, 'Softmax2d requires a 4D tensor as input'
906  return F.softmax(input, 1, _stacklevel=5)
907 
908 
909 @weak_module
910 class LogSoftmax(Module):
911  r"""Applies the :math:`\log(\text{Softmax}(x))` function to an n-dimensional
912  input Tensor. The LogSoftmax formulation can be simplified as:
913 
914  .. math::
915  \text{LogSoftmax}(x_{i}) = \log\left(\frac{\exp(x_i) }{ \sum_j \exp(x_j)} \right)
916 
917  Shape:
918  - Input: :math:`(*)` where `*` means, any number of additional
919  dimensions
920  - Output: :math:`(*)`, same shape as the input
921 
922  Arguments:
923  dim (int): A dimension along which LogSoftmax will be computed.
924 
925  Returns:
926  a Tensor of the same dimension and shape as the input with
927  values in the range [-inf, 0)
928 
929  Examples::
930 
931  >>> m = nn.LogSoftmax()
932  >>> input = torch.randn(2, 3)
933  >>> output = m(input)
934  """
935  __constants__ = ['dim']
936 
937  def __init__(self, dim=None):
938  super(LogSoftmax, self).__init__()
939  self.dim = dim
940 
941  def __setstate__(self, state):
942  self.__dict__.update(state)
943  if not hasattr(self, 'dim'):
944  self.dim = None
945 
946  @weak_script_method
947  def forward(self, input):
948  return F.log_softmax(input, self.dim, _stacklevel=5)