Caffe2 - Python API
A deep learning, cross platform ML framework
tensor.py
1 import sys
2 import torch
3 import torch._C as _C
4 from collections import OrderedDict
5 import torch.utils.hooks as hooks
6 import warnings
7 import weakref
8 from torch._six import imap
9 from torch._C import _add_docstr
10 from numbers import Number
11 
12 
13 # NB: If you subclass Tensor, and want to share the subclassed class
14 # across processes, you must also update torch/multiprocessing/reductions.py
15 # to define a ForkingPickler serialization mode for the class.
16 #
17 # NB: If you add a new method to Tensor, you must update
18 # torch/__init__.py.in to add a type annotation for your method;
19 # otherwise, it will not show up in autocomplete.
20 class Tensor(torch._C._TensorBase):
21  def __deepcopy__(self, memo):
22  if not self.is_leaf:
23  raise RuntimeError("Only Tensors created explicitly by the user "
24  "(graph leaves) support the deepcopy protocol at the moment")
25  if id(self) in memo:
26  return memo[id(self)]
27  with torch.no_grad():
28  if self.is_sparse:
29  new_tensor = self.clone()
30  else:
31  new_storage = self.storage().__deepcopy__(memo)
32  new_tensor = self.new()
33  new_tensor.set_(new_storage, self.storage_offset(), self.size(), self.stride())
34  memo[id(self)] = new_tensor
35  new_tensor.requires_grad = self.requires_grad
36  return new_tensor
37 
38  def __reduce_ex__(self, proto):
39  # See Note [Don't serialize hooks]
41  args = (self.storage(),
42  self.storage_offset(),
43  tuple(self.size()),
44  self.stride(),
45  self.requires_grad,
46  OrderedDict()) # previously was self._backward_hooks
47  return (torch._utils._rebuild_tensor_v2, args)
48 
49  def __setstate__(self, state):
50  # Warning: this method is NOT called when you torch.load() a tensor;
51  # that is managed by _rebuild_tensor_v2
52  if not self.is_leaf:
53  raise RuntimeError('__setstate__ can be only called on leaf Tensors')
54  if len(state) == 4:
55  # legacy serialization of Tensor
56  self.set_(*state)
57  return
58  elif len(state) == 5:
59  # legacy serialization of Variable
60  self.data = state[0]
61  state = (state[3], state[4], state[2])
62  # The setting of _backward_hooks is expected to be a no-op.
63  # See Note [Don't serialize hooks]
64  self.requires_grad, _, self._backward_hooks = state
65 
66  def __repr__(self):
67  # All strings are unicode in Python 3, while we have to encode unicode
68  # strings in Python2. If we can't, let python decide the best
69  # characters to replace unicode characters with.
70  if sys.version_info > (3,):
71  return torch._tensor_str._str(self)
72  else:
73  if hasattr(sys.stdout, 'encoding'):
74  return torch._tensor_str._str(self).encode(
75  sys.stdout.encoding or 'UTF-8', 'replace')
76  else:
77  return torch._tensor_str._str(self).encode('UTF-8', 'replace')
78 
79  def backward(self, gradient=None, retain_graph=None, create_graph=False):
80  r"""Computes the gradient of current tensor w.r.t. graph leaves.
81 
82  The graph is differentiated using the chain rule. If the tensor is
83  non-scalar (i.e. its data has more than one element) and requires
84  gradient, the function additionally requires specifying ``gradient``.
85  It should be a tensor of matching type and location, that contains
86  the gradient of the differentiated function w.r.t. ``self``.
87 
88  This function accumulates gradients in the leaves - you might need to
89  zero them before calling it.
90 
91  Arguments:
92  gradient (Tensor or None): Gradient w.r.t. the
93  tensor. If it is a tensor, it will be automatically converted
94  to a Tensor that does not require grad unless ``create_graph`` is True.
95  None values can be specified for scalar Tensors or ones that
96  don't require grad. If a None value would be acceptable then
97  this argument is optional.
98  retain_graph (bool, optional): If ``False``, the graph used to compute
99  the grads will be freed. Note that in nearly all cases setting
100  this option to True is not needed and often can be worked around
101  in a much more efficient way. Defaults to the value of
102  ``create_graph``.
103  create_graph (bool, optional): If ``True``, graph of the derivative will
104  be constructed, allowing to compute higher order derivative
105  products. Defaults to ``False``.
106  """
107  torch.autograd.backward(self, gradient, retain_graph, create_graph)
108 
109  def register_hook(self, hook):
110  r"""Registers a backward hook.
111 
112  The hook will be called every time a gradient with respect to the
113  Tensor is computed. The hook should have the following signature::
114 
115  hook(grad) -> Tensor or None
116 
117 
118  The hook should not modify its argument, but it can optionally return
119  a new gradient which will be used in place of :attr:`grad`.
120 
121  This function returns a handle with a method ``handle.remove()``
122  that removes the hook from the module.
123 
124  Example::
125 
126  >>> v = torch.tensor([0., 0., 0.], requires_grad=True)
127  >>> h = v.register_hook(lambda grad: grad * 2) # double the gradient
128  >>> v.backward(torch.tensor([1., 2., 3.]))
129  >>> v.grad
130 
131  2
132  4
133  6
134  [torch.FloatTensor of size (3,)]
135 
136  >>> h.remove() # removes the hook
137  """
138  if not self.requires_grad:
139  raise RuntimeError("cannot register a hook on a tensor that "
140  "doesn't require gradient")
141  if self._backward_hooks is None:
142  self._backward_hooks = OrderedDict()
143  if self.grad_fn is not None:
144  self.grad_fn._register_hook_dict(self)
145  handle = hooks.RemovableHandle(self._backward_hooks)
146  self._backward_hooks[handle.id] = hook
147  return handle
148 
149  def reinforce(self, reward):
150  def trim(str):
151  return '\n'.join([line.strip() for line in str.split('\n')])
152 
153  raise RuntimeError(trim(r"""reinforce() was removed.
154  Use torch.distributions instead.
155  See https://pytorch.org/docs/master/distributions.html
156 
157  Instead of:
158 
159  probs = policy_network(state)
160  action = probs.multinomial()
161  next_state, reward = env.step(action)
162  action.reinforce(reward)
163  action.backward()
164 
165  Use:
166 
167  probs = policy_network(state)
168  # NOTE: categorical is equivalent to what used to be called multinomial
169  m = torch.distributions.Categorical(probs)
170  action = m.sample()
171  next_state, reward = env.step(action)
172  loss = -m.log_prob(action) * reward
173  loss.backward()
174  """))
175 
176  detach = _add_docstr(_C._TensorBase.detach, r"""
177  Returns a new Tensor, detached from the current graph.
178 
179  The result will never require gradient.
180 
181  .. note::
182 
183  Returned Tensor shares the same storage with the original one.
184  In-place modifications on either of them will be seen, and may trigger
185  errors in correctness checks.
186  IMPORTANT NOTE: Previously, in-place size / stride / storage changes
187  (such as `resize_` / `resize_as_` / `set_` / `transpose_`) to the returned tensor
188  also update the original tensor. Now, these in-place changes will not update the
189  original tensor anymore, and will instead trigger an error.
190  For sparse tensors:
191  In-place indices / values changes (such as `zero_` / `copy_` / `add_`) to the
192  returned tensor will not update the original tensor anymore, and will instead
193  trigger an error.
194  """)
195 
196  detach_ = _add_docstr(_C._TensorBase.detach_, r"""
197  Detaches the Tensor from the graph that created it, making it a leaf.
198  Views cannot be detached in-place.
199  """)
200 
201  def retain_grad(self):
202  r"""Enables .grad attribute for non-leaf Tensors."""
203  if self.grad_fn is None: # no-op for leaves
204  return
205  if not self.requires_grad:
206  raise RuntimeError("can't retain_grad on Tensor that has requires_grad=False")
207  if hasattr(self, 'retains_grad'):
208  return
209  weak_self = weakref.ref(self)
210 
211  def retain_grad_hook(grad):
212  var = weak_self()
213  if var is None:
214  return
215  if var._grad is None:
216  var._grad = grad.clone()
217  else:
218  var._grad = var._grad + grad
219 
220  self.register_hook(retain_grad_hook)
221  self.retains_grad = True
222 
223  def is_pinned(self):
224  r"""Returns true if this tensor resides in pinned memory"""
225  storage = self.storage()
226  return storage.is_pinned() if storage else False
227 
228  def is_shared(self):
229  r"""Checks if tensor is in shared memory.
230 
231  This is always ``True`` for CUDA tensors.
232  """
233  return self.storage().is_shared()
234 
235  def share_memory_(self):
236  r"""Moves the underlying storage to shared memory.
237 
238  This is a no-op if the underlying storage is already in shared memory
239  and for CUDA tensors. Tensors in shared memory cannot be resized.
240  """
241  self.storage().share_memory_()
242  return self
243 
244  def __reversed__(self):
245  r"""Reverses the tensor along dimension 0."""
246  if self.dim() == 0:
247  return self
248  else:
249  return self.flip(0)
250 
251  def norm(self, p="fro", dim=None, keepdim=False, dtype=None):
252  r"""See :func:`torch.norm`"""
253  return torch.norm(self, p, dim, keepdim, dtype=dtype)
254 
255  def potrf(self, upper=True):
256  r"""See :func:`torch.cholesky`"""
257  warnings.warn("torch.potrf is deprecated in favour of torch.cholesky and will be removed "
258  "in the next release. Please use torch.cholesky instead and note that the "
259  ":attr:`upper` argument in torch.cholesky defaults to ``False``.", stacklevel=2)
260  return super(Tensor, self).cholesky(upper=upper)
261 
262  def pstrf(self, upper=True):
263  r"""See :func:`torch.pstrf`"""
264  warnings.warn("torch.pstrf is deprecated in favour of torch.cholesky and will be removed "
265  "in the next release.", stacklevel=2)
266  return super(Tensor, self).pstrf(upper=upper)
267 
268  def potrs(self, u, upper=True):
269  r"""See :func:`torch.cholesky_solve`"""
270  warnings.warn("torch.potrs is deprecated in favour of torch.cholesky_solve and "
271  "will be removed in the next release. Please use torch.cholesky_solve instead "
272  "and note that the :attr:`upper` argument in torch.cholesky_solve defaults "
273  "to ``False``.", stacklevel=2)
274  return super(Tensor, self).cholesky_solve(u, upper=upper)
275 
276  def gesv(self, A):
277  r"""See :func:`torch.solve`"""
278  warnings.warn("torch.gesv is deprecated in favour of torch.solve and will be removed in the "
279  "next release. Please use torch.solve instead.", stacklevel=2)
280  return super(Tensor, self).solve(A)
281 
282  def stft(self, n_fft, hop_length=None, win_length=None, window=None,
283  center=True, pad_mode='reflect', normalized=False, onesided=True):
284  r"""See :func:`torch.stft`
285 
286  .. warning::
287  This function changed signature at version 0.4.1. Calling with
288  the previous signature may cause error or return incorrect result.
289  """
290  return torch.stft(self, n_fft, hop_length, win_length, window, center,
291  pad_mode, normalized, onesided)
292 
293  def resize(self, *sizes):
294  warnings.warn("non-inplace resize is deprecated")
295  from torch.autograd._functions import Resize
296  return Resize.apply(self, sizes)
297 
298  def resize_as(self, tensor):
299  warnings.warn("non-inplace resize_as is deprecated")
300  from torch.autograd._functions import Resize
301  return Resize.apply(self, tensor.size())
302 
303  def split(self, split_size, dim=0):
304  r"""See :func:`torch.split`
305  """
306  if isinstance(split_size, int):
307  return super(Tensor, self).split(split_size, dim)
308  else:
309  return super(Tensor, self).split_with_sizes(split_size, dim)
310 
311  def unique(self, sorted=True, return_inverse=False, dim=None):
312  r"""Returns the unique scalar elements of the tensor as a 1-D tensor.
313 
314  See :func:`torch.unique`
315  """
316  if dim is not None:
317  output, inverse_indices = torch._unique_dim(
318  self,
319  sorted=sorted,
320  return_inverse=return_inverse,
321  dim=dim
322  )
323  else:
324  output, inverse_indices = torch._unique(
325  self,
326  sorted=sorted,
327  return_inverse=return_inverse
328  )
329  if return_inverse:
330  return output, inverse_indices
331  else:
332  return output
333 
334  def __rsub__(self, other):
335  return _C._VariableFunctions.rsub(self, other)
336 
337  def __rdiv__(self, other):
338  if self.dtype.is_floating_point:
339  return self.reciprocal() * other
340  else:
341  return (self.double().reciprocal() * other).type_as(self)
342 
343  __rtruediv__ = __rdiv__
344  __itruediv__ = _C._TensorBase.__idiv__
345 
346  __pow__ = _C._TensorBase.pow
347 
348  def __format__(self, format_spec):
349  if self.dim() == 0:
350  return self.item().__format__(format_spec)
351  return object.__format__(self, format_spec)
352 
353  def __ipow__(self, other):
354  raise NotImplementedError("in-place pow not implemented")
355 
356  def __rpow__(self, other):
357  return self.new_tensor(other) ** self
358 
359  def __floordiv__(self, other):
360  result = self / other
361  if result.dtype.is_floating_point:
362  result = result.trunc()
363  return result
364 
365  def __rfloordiv__(self, other):
366  result = other / self
367  if result.dtype.is_floating_point:
368  result = result.trunc()
369  return result
370 
371  __neg__ = _C._TensorBase.neg
372 
373  __eq__ = _C._TensorBase.eq
374  __ne__ = _C._TensorBase.ne
375  __lt__ = _C._TensorBase.lt
376  __le__ = _C._TensorBase.le
377  __gt__ = _C._TensorBase.gt
378  __ge__ = _C._TensorBase.ge
379  __abs__ = _C._TensorBase.abs
380 
381  def __len__(self):
382  if self.dim() == 0:
383  raise TypeError("len() of a 0-d tensor")
384  return self.shape[0]
385 
386  def __iter__(self):
387  # NB: we use 'imap' and not 'map' here, so that in Python 2 we get a
388  # generator and don't eagerly perform all the indexes. This could
389  # save us work, and also helps keep trace ordering deterministic
390  # (e.g., if you zip(*hiddens), the eager map will force all the
391  # indexes of hiddens[0] before hiddens[1], while the generator
392  # map will interleave them.)
393  if self.dim() == 0:
394  raise TypeError('iteration over a 0-d tensor')
395  if torch._C._get_tracing_state():
396  warnings.warn('Iterating over a tensor might cause the trace to be incorrect. '
397  'Passing a tensor of different shape won\'t change the number of '
398  'iterations executed (and might lead to errors or silently give '
399  'incorrect results).', category=RuntimeWarning)
400  return iter(imap(lambda i: self[i], range(self.size(0))))
401 
402  def __hash__(self):
403  return id(self)
404 
405  def __dir__(self):
406  tensor_methods = dir(self.__class__)
407  tensor_methods.remove('volatile') # deprecated
408  attrs = list(self.__dict__.keys())
409  keys = tensor_methods + attrs
410 
411  # property only available dense, cuda tensors
412  if (not self.is_cuda) or self.is_sparse:
413  keys.remove("__cuda_array_interface__")
414 
415  return sorted(keys)
416 
417  # Numpy array interface, to support `numpy.asarray(tensor) -> ndarray`
418  __array_priority__ = 1000 # prefer Tensor ops over numpy ones
419 
420  def __array__(self, dtype=None):
421  if dtype is None:
422  return self.numpy()
423  else:
424  return self.numpy().astype(dtype, copy=False)
425 
426  # Wrap Numpy array again in a suitable tensor when done, to support e.g.
427  # `numpy.sin(tensor) -> tensor` or `numpy.greater(tensor, 0) -> ByteTensor`
428  def __array_wrap__(self, array):
429  if array.dtype == bool:
430  # Workaround, torch has no built-in bool tensor
431  array = array.astype('uint8')
432  return torch.from_numpy(array)
433 
434  def __contains__(self, element):
435  r"""Check if `element` is present in tensor
436 
437  Arguments:
438  element (Tensor or scalar): element to be checked
439  for presence in current tensor"
440  """
441  if isinstance(element, (torch.Tensor, Number)):
442  return (element == self).any().item()
443  return NotImplemented
444 
445  @property
447  """Array view description for cuda tensors.
448 
449  See:
450  https://numba.pydata.org/numba-doc/latest/cuda/cuda_array_interface.html
451  """
452 
453  # raise AttributeError for unsupported tensors, so that
454  # hasattr(cpu_tensor, "__cuda_array_interface__") is False.
455  if not self.is_cuda:
456  raise AttributeError(
457  "Can't get __cuda_array_interface__ on non-CUDA tensor type: %s "
458  "If CUDA data is required use tensor.cuda() to copy tensor to device memory." %
459  self.type()
460  )
461 
462  if self.is_sparse:
463  raise AttributeError(
464  "Can't get __cuda_array_interface__ on sparse type: %s "
465  "Use Tensor.to_dense() to convert to a dense tensor first." %
466  self.type()
467  )
468 
469  # RuntimeError, matching tensor.__array__() behavior.
470  if self.requires_grad:
471  raise RuntimeError(
472  "Can't get __cuda_array_interface__ on Variable that requires grad. "
473  "If gradients aren't required, use var.detach() to get Variable that doesn't require grad."
474  )
475 
476  # CUDA devices are little-endian and tensors are stored in native byte
477  # order. 1-byte entries are endian-agnostic.
478  typestr = {
479  torch.float16: "<f2",
480  torch.float32: "<f4",
481  torch.float64: "<f8",
482  torch.uint8: "|u1",
483  torch.int8: "|i1",
484  torch.int16: "<i2",
485  torch.int32: "<i4",
486  torch.int64: "<i8",
487  }[self.dtype]
488 
489  itemsize = self.storage().element_size()
490 
491  shape = self.shape
492  strides = tuple(s * itemsize for s in self.stride())
493  data = (self.data_ptr(), False) # read-only is false
494 
495  return dict(typestr=typestr, shape=shape, strides=strides, data=data, version=0)
496 
497  __module__ = 'torch'
Module caffe2.python.layers.split.
def backward(tensors, grad_tensors=None, retain_graph=None, create_graph=False, grad_variables=None)
Definition: __init__.py:38
def register_hook(self, hook)
Definition: tensor.py:109
def __cuda_array_interface__(self)
Definition: tensor.py:446
def warn_if_has_hooks(tensor)
Definition: hooks.py:51