Caffe2 - Python API
A deep learning, cross platform ML framework
clang_tidy.py
1 #!/usr/bin/env python
2 """
3 A driver script to run clang-tidy on changes detected via git.
4 
5 By default, clang-tidy runs on all files you point it at. This means that even
6 if you changed only parts of that file, you will get warnings for the whole
7 file. This script has the ability to ask git for the exact lines that have
8 changed since a particular git revision, and makes clang-tidy only lint those.
9 This makes it much less overhead to integrate in CI and much more relevant to
10 developers. This git-enabled mode is optional, and full scans of a directory
11 tree are also possible. In both cases, the script allows filtering files via
12 glob or regular expressions.
13 """
14 
15 import argparse
16 import collections
17 import fnmatch
18 import json
19 import os.path
20 import re
21 import shlex
22 import subprocess
23 import sys
24 import tempfile
25 
26 try:
27  from shlex import quote
28 except ImportError:
29  from pipes import quote
30 
31 Patterns = collections.namedtuple("Patterns", "positive, negative")
32 
33 
34 # NOTE: Clang-tidy cannot lint headers directly, because headers are not
35 # compiled -- translation units are, of which there is one per implementation
36 # (c/cc/cpp) file.
37 DEFAULT_FILE_PATTERN = re.compile(r".*\.c(c|pp)?")
38 
39 # @@ -start,count +start,count @@
40 CHUNK_PATTERN = r"^@@\s+-\d+,\d+\s+\+(\d+)(?:,(\d+))?\[email protected]@"
41 
42 # Set from command line arguments in main().
43 VERBOSE = False
44 
45 
46 def run_shell_command(arguments):
47  """Executes a shell command."""
48  if VERBOSE:
49  print(" ".join(arguments))
50  try:
51  output = subprocess.check_output(arguments).decode().strip()
52  except subprocess.CalledProcessError:
53  _, error, _ = sys.exc_info()
54  error_output = error.output.decode().strip()
55  raise RuntimeError("Error executing {}: {}".format(" ".join(arguments), error_output))
56 
57  return output
58 
59 
61  """Separates negative patterns (that start with a dash) from positive patterns"""
62  positive, negative = [], []
63  for pattern in patterns:
64  if pattern.startswith("-"):
65  negative.append(pattern[1:])
66  else:
67  positive.append(pattern)
68 
69  return Patterns(positive, negative)
70 
71 
72 def get_file_patterns(globs, regexes):
73  """Returns a list of compiled regex objects from globs and regex pattern strings."""
74  # fnmatch.translate converts a glob into a regular expression.
75  # https://docs.python.org/2/library/fnmatch.html#fnmatch.translate
77  regexes = split_negative_from_positive_patterns(regexes)
78 
79  positive_regexes = regexes.positive + [fnmatch.translate(g) for g in glob.positive]
80  negative_regexes = regexes.negative + [fnmatch.translate(g) for g in glob.negative]
81 
82  positive_patterns = [re.compile(regex) for regex in positive_regexes] or [
83  DEFAULT_FILE_PATTERN
84  ]
85  negative_patterns = [re.compile(regex) for regex in negative_regexes]
86 
87  return Patterns(positive_patterns, negative_patterns)
88 
89 
90 def filter_files(files, file_patterns):
91  """Returns all files that match any of the patterns."""
92  if VERBOSE:
93  print("Filtering with these file patterns: {}".format(file_patterns))
94  for file in files:
95  if not any(n.match(file) for n in file_patterns.negative):
96  if any(p.match(file) for p in file_patterns.positive):
97  yield file
98  continue
99  if VERBOSE:
100  print("{} ommitted due to file filters".format(file))
101 
102 
103 def get_changed_files(revision, paths):
104  """Runs git diff to get the paths of all changed files."""
105  # --diff-filter AMU gets us files that are (A)dded, (M)odified or (U)nmerged (in the working copy).
106  # --name-only makes git diff return only the file paths, without any of the source changes.
107  command = "git diff-index --diff-filter=AMU --ignore-all-space --name-only"
108  output = run_shell_command(shlex.split(command) + [revision] + paths)
109  return output.split("\n")
110 
111 
112 def get_all_files(paths):
113  """Returns all files that are tracked by git in the given paths."""
114  output = run_shell_command(["git", "ls-files"] + paths)
115  return output.split("\n")
116 
117 
118 def get_changed_lines(revision, filename):
119  """Runs git diff to get the line ranges of all file changes."""
120  command = shlex.split("git diff-index --unified=0") + [revision, filename]
121  output = run_shell_command(command)
122  changed_lines = []
123  for chunk in re.finditer(CHUNK_PATTERN, output, re.MULTILINE):
124  start = int(chunk.group(1))
125  count = int(chunk.group(2) or 1)
126  changed_lines.append([start, start + count])
127 
128  return {"name": filename, "lines": changed_lines}
129 
130 ninja_template = """
131 rule do_cmd
132  command = $cmd
133  description = Running clang-tidy
134 
135 {build_rules}
136 """
137 
138 build_template = """
139 build {i}: do_cmd
140  cmd = {cmd}
141 """
142 
143 
145  """runs all the commands in parallel with ninja, commands is a List[List[str]]"""
146  build_entries = [build_template.format(i=i, cmd=' '.join([quote(s) for s in command]))
147  for i, command in enumerate(commands)]
148 
149  file_contents = ninja_template.format(build_rules='\n'.join(build_entries)).encode()
150  f = tempfile.NamedTemporaryFile(delete=False)
151  try:
152  f.write(file_contents)
153  f.close()
154  return run_shell_command(['ninja', '-f', f.name])
155  finally:
156  os.unlink(f.name)
157 
158 
159 def run_clang_tidy(options, line_filters, files):
160  """Executes the actual clang-tidy command in the shell."""
161  command = [options.clang_tidy_exe, "-p", options.compile_commands_dir]
162  if not options.config_file and os.path.exists(".clang-tidy"):
163  options.config_file = ".clang-tidy"
164  if options.config_file:
165  import yaml
166 
167  with open(options.config_file) as config:
168  # Here we convert the YAML config file to a JSON blob.
169  command += ["-config", json.dumps(yaml.load(config))]
170  command += options.extra_args
171 
172  if line_filters:
173  command += ["-line-filter", json.dumps(line_filters)]
174 
175  if options.parallel:
176  commands = [list(command) + [f] for f in files]
177  output = run_shell_commands_in_parallel(commands)
178  else:
179  command += files
180  if options.dry_run:
181  command = [re.sub(r"^([{[].*[]}])$", r"'\1'", arg) for arg in command]
182  return " ".join(command)
183 
184  output = run_shell_command(command)
185 
186  if not options.keep_going and "[clang-diagnostic-error]" in output:
187  message = "Found clang-diagnostic-errors in clang-tidy output: {}"
188  raise RuntimeError(message.format(output))
189 
190  return output
191 
192 
194  """Parses the command line options."""
195  parser = argparse.ArgumentParser(description="Run Clang-Tidy (on your Git changes)")
196  parser.add_argument(
197  "-e",
198  "--clang-tidy-exe",
199  default="clang-tidy",
200  help="Path to clang-tidy executable",
201  )
202  parser.add_argument(
203  "-g",
204  "--glob",
205  action="append",
206  default=[],
207  help="Only lint files that match these glob patterns "
208  "(see documentation for `fnmatch` for supported syntax)."
209  "If a pattern starts with a - the search is negated for that pattern.",
210  )
211  parser.add_argument(
212  "-x",
213  "--regex",
214  action="append",
215  default=[],
216  help="Only lint files that match these regular expressions (from the start of the filename). "
217  "If a pattern starts with a - the search is negated for that pattern.",
218  )
219  parser.add_argument(
220  "-c",
221  "--compile-commands-dir",
222  default="build",
223  help="Path to the folder containing compile_commands.json",
224  )
225  parser.add_argument(
226  "-d", "--diff", help="Git revision to diff against to get changes"
227  )
228  parser.add_argument(
229  "-p",
230  "--paths",
231  nargs="+",
232  default=["."],
233  help="Lint only the given paths (recursively)",
234  )
235  parser.add_argument(
236  "-n",
237  "--dry-run",
238  action="store_true",
239  help="Only show the command to be executed, without running it",
240  )
241  parser.add_argument("-v", "--verbose", action="store_true", help="Verbose output")
242  parser.add_argument(
243  "--config-file",
244  help="Path to a clang-tidy config file. Defaults to '.clang-tidy'.",
245  )
246  parser.add_argument(
247  "-k",
248  "--keep-going",
249  action="store_true",
250  help="Don't error on compiler errors (clang-diagnostic-error)",
251  )
252  parser.add_argument(
253  "-j",
254  "--parallel",
255  action="store_true",
256  help="Run clang tidy in parallel per-file (requires ninja to be installed).",
257  )
258  parser.add_argument(
259  "extra_args", nargs="*", help="Extra arguments to forward to clang-tidy"
260  )
261  return parser.parse_args()
262 
263 
264 def main():
265  options = parse_options()
266 
267  # This flag is pervasive enough to set it globally. It makes the code
268  # cleaner compared to threading it through every single function.
269  global VERBOSE
270  VERBOSE = options.verbose
271 
272  # Normalize the paths first.
273  paths = [path.rstrip("/") for path in options.paths]
274  if options.diff:
275  files = get_changed_files(options.diff, paths)
276  else:
277  files = get_all_files(paths)
278  file_patterns = get_file_patterns(options.glob, options.regex)
279  files = list(filter_files(files, file_patterns))
280 
281  # clang-tidy error's when it does not get input files.
282  if not files:
283  print("No files detected.")
284  sys.exit()
285 
286  line_filters = []
287  if options.diff:
288  line_filters = [get_changed_lines(options.diff, f) for f in files]
289 
290  print(run_clang_tidy(options, line_filters, files))
291 
292 
293 if __name__ == "__main__":
294  main()
def get_changed_files(revision, paths)
Definition: clang_tidy.py:103
def split_negative_from_positive_patterns(patterns)
Definition: clang_tidy.py:60
def get_file_patterns(globs, regexes)
Definition: clang_tidy.py:72
def filter_files(files, file_patterns)
Definition: clang_tidy.py:90
def get_changed_lines(revision, filename)
Definition: clang_tidy.py:118
def run_clang_tidy(options, line_filters, files)
Definition: clang_tidy.py:159
def get_all_files(paths)
Definition: clang_tidy.py:112
def run_shell_command(arguments)
Definition: clang_tidy.py:46
def run_shell_commands_in_parallel(commands)
Definition: clang_tidy.py:144