Caffe2 - C++ API
A deep learning, cross platform ML framework
perf_observer.cc
1 #include "caffe2/share/contrib/observers/perf_observer.h"
2 #include "caffe2/share/contrib/observers/observer_config.h"
3 
4 #include <random>
5 #include "caffe2/core/common.h"
6 #include "caffe2/core/init.h"
7 #include "caffe2/core/operator.h"
8 
9 namespace caffe2 {
10 namespace {
11 
12 bool registerGlobalPerfNetObserverCreator(int* /*pargc*/, char*** /*pargv*/) {
13  SetGlobalNetObserverCreator([](NetBase* subject) {
14  return caffe2::make_unique<PerfNetObserver>(subject);
15  });
16  return true;
17 }
18 } // namespace
19 
20 REGISTER_CAFFE2_EARLY_INIT_FUNCTION(
21  registerGlobalPerfNetObserverCreator,
22  &registerGlobalPerfNetObserverCreator,
23  "Caffe2 net global observer creator");
24 
25 PerfNetObserver::PerfNetObserver(NetBase* subject_)
26  : NetObserver(subject_), numRuns_(0) {}
27 
28 PerfNetObserver::~PerfNetObserver() {}
29 
30 void PerfNetObserver::Start() {
31  static int visitCount = 0;
32  // Select whether to log the operator or the net.
33  // We have one sample rate for the entire app.
34  int netInitSampleRate = ObserverConfig::getNetInitSampleRate();
35  int netFollowupSampleRate = ObserverConfig::getNetFollowupSampleRate();
36  int netFollowupSampleCount = ObserverConfig::getNetFollowupSampleCount();
37  int operatorNetSampleRatio = ObserverConfig::getOpoeratorNetSampleRatio();
38  int skipIters = ObserverConfig::getSkipIters();
39  int sampleRate = visitCount > 0 ? netFollowupSampleRate : netInitSampleRate;
40  if (skipIters <= numRuns_ && sampleRate > 0 && rand() % sampleRate == 0) {
41  visitCount++;
42  if (visitCount == netFollowupSampleCount) {
43  visitCount = 0;
44  }
45  if (operatorNetSampleRatio > 0 && rand() % operatorNetSampleRatio == 0) {
46  logType_ = PerfNetObserver::OPERATOR_DELAY;
47  } else {
48  logType_ = PerfNetObserver::NET_DELAY;
49  }
50  } else {
51  logType_ = PerfNetObserver::NONE;
52  }
53  numRuns_++;
54 
55  if (logType_ == PerfNetObserver::OPERATOR_DELAY) {
56  /* Always recreate new operator observers
57  whenever we measure operator delay */
58  const auto& operators = subject_->GetOperators();
59  for (auto* op : operators) {
60  observerMap_[op] = op->AttachObserver(
61  caffe2::make_unique<PerfOperatorObserver>(op, this));
62  }
63  }
64 
65  if (logType_ != PerfNetObserver::NONE) {
66  /* Only start timer when we need to */
67  timer_.Start();
68  }
69 }
70 
71 void PerfNetObserver::Stop() {
72  if (logType_ == PerfNetObserver::NONE) {
73  return;
74  }
75  auto currentRunTime = timer_.MilliSeconds();
76  std::map<std::string, double> delays;
77  delays.insert({"NET_DELAY", currentRunTime});
78  if (logType_ == PerfNetObserver::OPERATOR_DELAY) {
79  const auto& operators = subject_->GetOperators();
80  for (int idx = 0; idx < operators.size(); ++idx) {
81  const auto* op = operators[idx];
82  auto name = getObserverName(op, idx);
83  double delay = static_cast<const PerfOperatorObserver*>(observerMap_[op])
84  ->getMilliseconds();
85  delays.insert({name, delay});
86  }
87  /* clear all operator delay after use so that we don't spent time
88  collecting the operator delay info in later runs */
89  for (auto* op : operators) {
90  op->DetachObserver(observerMap_[op]);
91  }
92  observerMap_.clear();
93  }
94  ObserverConfig::getReporter()->reportDelay(subject_, delays, "ms");
95 }
96 
97 caffe2::string PerfNetObserver::getObserverName(const OperatorBase* op, int idx)
98  const {
99  string opType = op->has_debug_def() ? op->debug_def().type() : "NO_TYPE";
100  string displayName =
101  (op->has_debug_def() ? op->debug_def().name().size()
102  ? op->debug_def().name()
103  : (op->debug_def().output_size() ? op->debug_def().output(0)
104  : "NO_OUTPUT")
105  : "NO_DEF");
106  caffe2::string name =
107  "ID_" + caffe2::to_string(idx) + "_" + opType + "_" + displayName;
108  return name;
109 }
110 
111 PerfOperatorObserver::PerfOperatorObserver(
112  OperatorBase* op,
113  PerfNetObserver* netObserver)
114  : ObserverBase<OperatorBase>(op),
115  netObserver_(netObserver),
116  milliseconds_(0) {
117  CAFFE_ENFORCE(netObserver_, "Observers can't operate outside of the net");
118 }
119 
120 PerfOperatorObserver::~PerfOperatorObserver() {}
121 
122 void PerfOperatorObserver::Start() {
123  /* Get the time from the start of the net minus the time spent
124  in previous invocations. It is the time spent on other operators.
125  This way, when the operator finishes, the time from the start of the net
126  minus the time spent in all other operators is the total time on this
127  operator. This is done to avoid saving a timer in each operator */
128  milliseconds_ = netObserver_->getTimer().MilliSeconds() - milliseconds_;
129 }
130 
131 void PerfOperatorObserver::Stop() {
132  /* Time from the start of the net minus the time spent on all other
133  operators is the time spent on this operator */
134  milliseconds_ = netObserver_->getTimer().MilliSeconds() - milliseconds_;
135 }
136 
137 double PerfOperatorObserver::getMilliseconds() const {
138  return milliseconds_;
139 }
140 
141 std::unique_ptr<ObserverBase<OperatorBase>> PerfOperatorObserver::copy(
142  OperatorBase* subject) {
143  return std::unique_ptr<ObserverBase<OperatorBase>>(
144  new PerfOperatorObserver(subject, netObserver_));
145 }
146 
147 } // namespace caffe2
Copyright (c) 2016-present, Facebook, Inc.