Caffe2 - C++ API
A deep learning, cross platform ML framework
sparse_normalize_op.cc
1 
17 #include "caffe2/operators/sparse_normalize_op.h"
18 #include "caffe2/core/tensor.h"
19 
20 namespace caffe2 {
21 
22 template <>
23 template <typename SIndex>
24 bool SparseNormalizeOp<float, CPUContext>::DoRunWithType() {
25  const auto* indices = Input(INDICES).template data<SIndex>();
26  const auto* paramIn = Input(PARAM).template data<float>();
27  auto* paramOut = Output(OUTPUT_PARAM)->template mutable_data<float>();
28  const float kEps = 1e-12f;
29 
30  // n: number of sparse embeddings to be normalized
31  auto n = Input(INDICES).size();
32  if (n == 0) {
33  return true;
34  }
35 
36  // embedding length, e.g. 32, 64, 128
37  auto block_size = Input(GRAD).size() / n;
38  for (int i = 0; i < n; ++i) {
39  auto idx = indices[i];
40  auto offsetIdx = idx * block_size;
41  ConstEigenVectorMap<float> xVec(paramIn + offsetIdx, block_size);
42  auto norm = xVec.template lpNorm<2>() + kEps;
43 
44  if (use_max_norm_ && norm < norm_) {
45  continue;
46  }
47 
48  for (int j = 0; j < block_size; j++) {
49  paramOut[offsetIdx + j] = paramOut[offsetIdx + j] * (norm_ / norm);
50  }
51  }
52  return true;
53 }
54 
55 REGISTER_CPU_OPERATOR(SparseNormalize, SparseNormalizeOp<float, CPUContext>);
56 OPERATOR_SCHEMA(SparseNormalize)
57  .NumInputs(3)
58  .NumOutputs(1)
59  .Input(0, "param", "Parameters to be normalized")
60  .Input(1, "indices", "Sparse indices")
61  .Input(2, "grad", "Gradient computed")
62  .Output(0, "output_param", "Normalized parameters")
63  .EnforceOneToOneInplace()
64  .Arg(
65  "use_max_norm",
66  "A bool variable to control whether to use max norm \
67  or constant norm. When use_max_norm = false, constant norm is used so that \
68  all the embedding vectors are scaled to have a L2 norm equals to A \
69  (see blow arugment norm=A). If use_max_norm = true, \
70  max norm is used so that embedding is scaled so that its l2 norm is no larger \
71  than A. If an embedding's norm is less than A originally, \
72  the embedding is left unchanged.\
73  The default is True.")
74  .Arg("norm", "L2 norm of the embedding. The default is 1.0.")
75  .SetDoc(R"DOC(
76 Given a sparse matrix, apply max_norm or constant_norm sparse regularization.
77 )DOC");
78 
79 SHOULD_NOT_DO_GRADIENT(SparseNormalize);
80 } // namespace caffe2
Copyright (c) 2016-present, Facebook, Inc.