Caffe2 - Python API A deep learning, cross platform ML framework
homotopy_weight.py
1 # @package homotopy_weight
2 # Module caffe2.fb.python.layers.homotopy_weight
3
4 from __future__ import absolute_import
5 from __future__ import division
6 from __future__ import print_function
7 from __future__ import unicode_literals
8
9 from caffe2.python import core, schema
10 from caffe2.python.layers.layers import ModelLayer
11 import numpy as np
12 import logging
13 logger = logging.getLogger(__name__)
14 '''
15 Homotopy Weighting between two weights x, y by doing:
16  alpha x + beta y
17 where alpha is a decreasing scalar parameter ranging from [min, max] (default,
18 [0, 1]), and alpha + beta = max + min, which means that beta is increasing in
19 the range [min, max];
20
21 Homotopy methods first solves an "easy" problem (one to which the solution is
22 well known), and is gradually transformed into the target problem
23 '''
24
25
27  def __init__(
28  self,
29  model,
30  input_record,
31  name='homotopy_weight',
32  min_weight=0.,
33  max_weight=1.,
34  half_life=1e6,
36  atomic_iter=None,
37  **kwargs
38  ):
39  super(HomotopyWeight,
40  self).__init__(model, name, input_record, **kwargs)
42  np.float32, self.get_next_blob_reference('homotopy_weight')
43  )
44  data = self.input_record.field_blobs()
45  assert len(data) == 2
46  self.x = data[0]
47  self.y = data[1]
48  # TODO: currently model building does not have access to iter counter or
49  # learning rate; it's added at optimization time;
50  self.use_external_iter = (atomic_iter is not None)
51  self.atomic_iter = (
52  atomic_iter if self.use_external_iter else self.create_atomic_iter()
53  )
54  # to map lr to [min, max]; alpha = scale * lr + offset
55  assert max_weight > min_weight
56  self.scale = float(max_weight - min_weight)
58  '%s_offset_1dfloat' % self.name, float(min_weight)
59  )
60  self.gamma, self.power = self.solve_inv_lr_params(half_life, quad_life)
61
63  # ensure that the gamma, power is solvable
64  assert half_life > 0
65  # convex monotonically decreasing
66  assert quad_life > 2 * half_life
67  t = float(quad_life) / float(half_life)
68  x = t * (1.0 + np.sqrt(2.0)) / 2.0 - np.sqrt(2.0)
69  gamma = (x - 1.0) / float(half_life)
70  power = np.log(2.0) / np.log(x)
71  logger.info(
72  'homotopy_weighting: found lr param: gamma=%g, power=%g' %
73  (gamma, power)
74  )
75  return gamma, power
76
77  def create_atomic_iter(self):
78  self.mutex = self.create_param(
79  param_name=('%s_mutex' % self.name),
80  shape=None,
81  initializer=('CreateMutex', ),
82  optimizer=self.model.NoOptim,
83  )
84  self.atomic_iter = self.create_param(
85  param_name=('%s_atomic_iter' % self.name),
86  shape=[1],
87  initializer=(
88  'ConstantFill', {
89  'value': 0,
90  'dtype': core.DataType.INT64
91  }
92  ),
93  optimizer=self.model.NoOptim,
94  )
95  return self.atomic_iter
96
97  def update_weight(self, net):
98  alpha = net.NextScopedBlob('alpha')
99  beta = net.NextScopedBlob('beta')
100  lr = net.NextScopedBlob('lr')
101  comp_lr = net.NextScopedBlob('complementary_lr')
102  scaled_lr = net.NextScopedBlob('scaled_lr')
103  scaled_comp_lr = net.NextScopedBlob('scaled_complementary_lr')
104  if not self.use_external_iter:
105  net.AtomicIter([self.mutex, self.atomic_iter], [self.atomic_iter])
106  net.LearningRate(
107  [self.atomic_iter],
108  [lr],
109  policy='inv',
110  gamma=self.gamma,
111  power=self.power,
112  base_lr=1.0,
113  )
114  net.Sub([self.model.global_constants['ONE'], lr], [comp_lr])
115  net.Scale([lr], [scaled_lr], scale=self.scale)
116  net.Scale([comp_lr], [scaled_comp_lr], scale=self.scale)