Caffe2 - Python API
A deep learning, cross platform ML framework
generator.py
1 # Copyright (c) 2016-present, Facebook, Inc.
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 # http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14 ##############################################################################
15 
16 ## @package generator
17 # Module caffe2.python.docs.generator
18 from __future__ import absolute_import
19 from __future__ import division
20 from __future__ import print_function
21 from __future__ import unicode_literals
22 import argparse
23 import os
24 from caffe2.python import core, workspace
25 from caffe2.python.docs.formatter import Markdown
26 from future.utils import viewitems, viewvalues
27 
28 OpSchema = workspace.C.OpSchema
29 
30 
31 class DocUploader(object):
32  def __init__(self):
33  pass
34 
35  def upload(self, text):
36  pass
37 
38 
39 class DocGenerator(object):
40  def __init__(self, formatter, uploader):
41  self.formatter = formatter
42  self.uploader = uploader
43  self.content_body = ""
44 
45  def create_body(self):
46  pass
47 
48  def update(self):
49  self.uploader.upload(self.content_body)
50 
51 
53  def getOperatorDoc(self, name, schema, priority):
54  return OperatorDoc(name, schema, priority)
55 
56  def getOperatorEngine(self, name):
57  return OperatorEngine(name)
58 
59  def getOperators(self):
60  # map: op_name -> operator
61  self.operators = {}
62  # map: op_name -> [engine, engine]
63  self.engines = {}
64 
65  def filePriority(x):
66  if x == "caffe2/caffe2/operators":
67  return 0
68  if 'contrib' in x.split('/'):
69  return 2
70  if 'experiments' in x.split('/'):
71  return 3
72  return 1
73 
74  for name in core._GetRegisteredOperators():
75  schema = OpSchema.get(name)
76  if schema:
77  priority = filePriority(os.path.dirname(schema.file))
78  operator = self.getOperatorDoc(name, schema, priority)
79  self.operators[name] = operator
80 
81  # Engine
82  elif name.find("_ENGINE_") != -1:
83  engine = self.getOperatorEngine(name)
84  if engine.base_op_name in self.engines:
85  self.engines[engine.base_op_name].append(engine)
86  else:
87  self.engines[engine.base_op_name] = [engine]
88 
89  # No schema
90  else:
91  priority = 4
92  self.operators[name] = self.getOperatorDoc(name, schema, priority)
93 
94  for name, engines in viewitems(self.engines):
95  if name in self.operators:
96  self.operators[name].addEngines(engines)
97 
98  # Generate a sorted list of operators
99  return sorted(
100  viewvalues(self.operators),
101  key=lambda op: (op.priority, op.name)
102  )
103 
104  def createBody(self):
105  operators = self.getOperators()
106 
107  for operator in operators:
108  operator.generateSchema(self.formatter)
109 
110  self.content_body += self.formatter.dump()
111 
112 
113 class OperatorEngine(object):
114  def __init__(self, name):
115  self.op_name = name
116  self.base_op_name, self.engine = name.split("_ENGINE_", 1)
117 
118  def getDeviceImpl(self):
119  deviceImplList = []
120  for device, impl in [('CPU', OpSchema.get_cpu_impl(self.op_name)),
121  ('CUDA', OpSchema.get_cuda_impl(self.op_name))]:
122  if not impl:
123  continue
124  deviceImplList.append((device, impl))
125  return deviceImplList
126 
127  def generateDoc(self, formatter):
128  for device, impl in self.getDeviceImpl():
129  formatter.addLine(
130  '{engine} on {device}: {impl}'.format(engine=self.engine,
131  device=device,
132  impl=impl))
133 
134 
135 class OperatorDoc(object):
136  def __init__(self, name, schema, priority):
137  self.name = name
138  self.schema = schema
139  self.priority = priority
140  print("Gathering docs for {}...".format(self.name))
141  self.engines = []
142 
143  def addEngines(self, engines):
144  self.engines = engines
145 
146  def generateDoc(self, formatter):
147  if self.schema.doc:
148  formatter.parseAndAdd(self.schema.doc)
149  formatter.addLinebreak()
150  else:
151  formatter.addLine("No documentation yet.")
152 
153  def generateTable(self, formatter, tuples, title_row, title):
154  if tuples:
155  if title:
156  formatter.addHeader(title, 3)
157  table = []
158  if title_row:
159  table = [title_row]
160  for name, doc in tuples:
161  table.append([name, doc or ''])
162  formatter.addTable(table, (table == []))
163 
164  def generateInterface(self, formatter):
165  def makeDesc(title, args):
166  f = formatter.clone()
167  f.addEmphasis(title, 1)
168  out = [(f.dump(), '')]
169  for arg in args:
170  f = formatter.clone()
171  if isinstance(arg, tuple):
172  name = arg[0]
173  if len(arg) > 1:
174  description = arg[1] or ''
175  else:
176  description = ''
177  else:
178  name = arg.name
179  description = arg.description or ''
180  f.addCode(name, inline=True)
181  out.append((f.dump(), description or ''))
182  return out
183 
184  tuples = []
185 
186  if self.schema.args:
187  tuples += makeDesc('Arguments', self.schema.args)
188 
189  if self.schema.input_desc:
190  tuples += makeDesc('Inputs', self.schema.input_desc)
191 
192  if self.schema.output_desc:
193  tuples += makeDesc('Outputs', self.schema.output_desc)
194 
195  self.generateTable(formatter, tuples, None, 'Interface')
196  print("Generated interface for {}".format(self.name))
197 
198  def generateCodeLink(self, formatter):
199  formatter.addHeader("Code", 3)
200  formatter.addLinebreak()
201  formatter.addCodeLink(self.schema.file)
202 
203  def getInfo(self, formatter, name, impl):
204  pass
205 
206  def generateDevices(self, formatter):
207  formatter.addHeader("Devices", 3)
208  devices = [
209  self.getInfo(formatter,
210  'CPU', OpSchema.get_cpu_impl(self.name)),
211  self.getInfo(formatter,
212  'GPU', OpSchema.get_cuda_impl(self.name)),
213  ]
214  formatter.addList([i for i in devices if i])
215 
216  def generateEngines(self, formatter):
217  if not len(self.engines):
218  return
219  formatter.addHeader("Engines", 3)
220  for engine in self.engines:
221  engine.generateDoc(formatter)
222 
223  def generateSchema(self, formatter):
224  formatter.addHeader(self.name, 2)
225  if self.schema:
226  self.generateDoc(formatter)
227  self.generateInterface(formatter)
228  self.generateCodeLink(formatter)
229  self.generateDevices(formatter)
230  self.generateEngines(formatter)
231  formatter.addBreak()
232  else:
233  formatter.addLine("No schema documented yet.")
234  self.generateDevices(formatter)
235 
236 
237 if __name__ == "__main__":
238  parser = argparse.ArgumentParser(description="Operators catalog generator.")
239  parser.add_argument('catalog_path', type=str,
240  help='operators-catalogue.md to write out to')
241  args = parser.parse_args()
242 
243  with open(args.catalog_path, 'w') as fp:
244  ops = OpDocGenerator(Markdown(), DocUploader())
245  ops.createBody()
246  fp.write(ops.content_body)
def getOperatorDoc(self, name, schema, priority)
Definition: generator.py:53
def getOperatorEngine(self, name)
Definition: generator.py:56
def generateTable(self, formatter, tuples, title_row, title)
Definition: generator.py:153