Caffe2 - Python API
A deep learning, cross platform ML framework
fc.py
1 # Copyright (c) 2016-present, Facebook, Inc.
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 # http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14 ##############################################################################
15 
16 ## @package fc
17 # Module caffe2.python.helpers.fc
18 from __future__ import absolute_import
19 from __future__ import division
20 from __future__ import print_function
21 from __future__ import unicode_literals
22 
23 from caffe2.python import core
24 from caffe2.python.modeling import initializers
25 from caffe2.python.modeling.parameter_info import ParameterTags
26 
27 
28 def _FC_or_packed_FC(
29  model, op_call, blob_in, blob_out, dim_in, dim_out, weight_init=None,
30  bias_init=None, WeightInitializer=None, BiasInitializer=None,
31  enable_tensor_core=False, float16_compute=False, **kwargs
32 ):
33  WeightInitializer = initializers.update_initializer(
34  WeightInitializer, weight_init, ("XavierFill", {})
35  )
36  BiasInitializer = initializers.update_initializer(
37  BiasInitializer, bias_init, ("ConstantFill", {})
38  )
39  if not model.init_params:
40  WeightInitializer = initializers.ExternalInitializer()
41  BiasInitializer = initializers.ExternalInitializer()
42 
43  blob_out = blob_out or model.net.NextName()
44  bias_tags = [ParameterTags.BIAS]
45  if 'freeze_bias' in kwargs:
46  bias_tags.append(ParameterTags.COMPUTED_PARAM)
47 
48  weight = model.create_param(
49  param_name=blob_out + '_w',
50  shape=[dim_out, dim_in],
51  initializer=WeightInitializer,
52  tags=ParameterTags.WEIGHT
53  )
54  bias = model.create_param(
55  param_name=blob_out + '_b',
56  shape=[dim_out, ],
57  initializer=BiasInitializer,
58  tags=bias_tags
59  )
60 
61  # enable TensorCore by setting appropriate engine
62  if enable_tensor_core:
63  kwargs['engine'] = 'TENSORCORE'
64 
65  # Enable float 16 compute kernel (relevant for CUDA)
66  if float16_compute:
67  kwargs['float16_compute'] = True
68 
69  return op_call([blob_in, weight, bias], blob_out, **kwargs)
70 
71 
72 def fc(model, *args, **kwargs):
73  return _FC_or_packed_FC(model, model.net.FC, *args, **kwargs)
74 
75 
76 def packed_fc(model, *args, **kwargs):
77  return _FC_or_packed_FC(model, model.net.PackedFC, *args, **kwargs)
78 
79 
80 def fc_decomp(
81  model, blob_in, blob_out, dim_in, dim_out,
82  rank_approx=5, weight_init=None, bias_init=None,
83  WeightInitializer=None, BiasInitializer=None, **kwargs
84 ):
85  """FC_Decomp version
86  Here we assume that the rank of original input is bigger than 5.
87  """
88  WeightInitializer = initializers.update_initializer(
89  WeightInitializer, weight_init, ("XavierFill", {})
90  )
91  BiasInitializer = initializers.update_initializer(
92  BiasInitializer, bias_init, ("ConstantFill", {})
93  )
94  blob_out = blob_out or model.net.NextName()
95  u = model.create_param(
96  param_name=blob_out + '_u',
97  shape=[dim_out, rank_approx],
98  initializer=WeightInitializer,
99  )
100  v = model.create_param(
101  param_name=blob_out + '_v',
102  shape=[dim_in, rank_approx],
103  initializer=WeightInitializer,
104  )
105  bias = model.create_param(
106  param_name=blob_out + '_b',
107  shape=[dim_out, ],
108  initializer=BiasInitializer,
109  )
110  return model.net.FC_Decomp([blob_in, u, v, bias], blob_out, **kwargs)
111 
112 
113 def fc_prune(
114  model, blob_in, blob_out, dim_in, dim_out,
115  weight_init=None, bias_init=None, mask_init=None,
116  threshold=0.00001, need_compress_rate=False,
117  comp_lb=0.05,
118  **kwargs
119 ):
120  """FC_Prune version
121  Runnable so far. Great!:)
122  """
123  weight_init = weight_init if weight_init else ('XavierFill', {})
124  bias_init = bias_init if bias_init else ('ConstantFill', {})
125  mask_init = mask_init if mask_init else ('ConstantFill', {})
126  blob_out = blob_out or model.net.NextName()
127  compress_rate = blob_out + '_compress_rate'
128  if model.init_params:
129  compress_lb = model.param_init_net.ConstantFill(
130  [],
131  blob_out + '_lb',
132  shape=[1],
133  value=comp_lb
134  )
135  weight = model.param_init_net.__getattr__(weight_init[0])(
136  [],
137  blob_out + '_w',
138  shape=[dim_out, dim_in],
139  **weight_init[1]
140  )
141  mask = model.param_init_net.ConstantFill(
142  [],
143  blob_out + '_m',
144  shape=[dim_out, dim_in],
145  value=1.0
146  )
147  ag_dw = model.param_init_net.__getattr__(mask_init[0])(
148  [],
149  blob_out + '_ag_dw',
150  shape=[dim_out, dim_in],
151  **mask_init[1]
152  )
153  bias = model.param_init_net.__getattr__(bias_init[0])(
154  [],
155  blob_out + '_b',
156  shape=[dim_out, ],
157  **bias_init[1]
158  )
159  mask_seq = model.param_init_net.__getattr__(mask_init[0])(
160  [],
161  blob_out + '_mask_seq',
162  shape=[dim_out, dim_in],
163  **mask_init[1]
164  )
165  thres = model.param_init_net.ConstantFill(
166  [],
167  blob_out + '_thres',
168  shape=[1],
169  value=threshold
170  )
171  else:
172  compress_lb = core.ScopedBlobReference(
173  blob_out + '_lb', model.param_init_net)
174  weight = core.ScopedBlobReference(
175  blob_out + '_w', model.param_init_net)
176  bias = core.ScopedBlobReference(
177  blob_out + '_b', model.param_init_net)
178  mask = core.ScopedBlobReference(
179  blob_out + '_m', model.param_init_net)
180  ag_dw = core.ScopedBlobReference(
181  blob_out + '_ag_dw', model.param_init_net)
182  mask_seq = core.ScopedBlobReference(
183  blob_out + '_mask_seq', model.param_init_net)
184  thres = core.ScopedBlobReference(
185  blob_out + '_thres', model.param_init_net)
186 
187  model.AddParameter(weight)
188  model.AddParameter(bias)
189  if need_compress_rate:
190  return model.net.FC_Prune([blob_in, weight, mask, bias, ag_dw, mask_seq,
191  thres, compress_lb],
192  [blob_out, compress_rate], **kwargs)
193  else:
194  return model.net.FC_Prune([blob_in, weight, mask,
195  bias, ag_dw, mask_seq,
196  thres, compress_lb],
197  blob_out, **kwargs)
198 
199 
200 def fc_sparse(
201  model, blob_in, blob_out, w_csr, iw, jw, bias,
202  **kwargs
203 ):
204  """FC_Sparse: Only takes in alocated weights"""
205  if not (w_csr and iw and jw and bias):
206  print("Warning...")
207  model.AddParameter(w_csr)
208  model.AddParameter(iw)
209  model.AddParameter(jw)
210  model.AddParameter(bias)
211  return model.net.FC_Sparse([blob_in, w_csr, iw, jw, bias],
212  blob_out, **kwargs)
Module caffe2.python.helpers.fc.