presubmit.py: Add several new presubmit checks
- Add presubmit checks for:
* clang/gcc/arm GN builds
* clang-format
* clang-tidy
* yapf
* mypy
* copyright header
Many of these checks are not yet passing on the whole repo.
- Define --clean option for deleting cached .presubmiit files.
- Define --skip_init option for whether to skip the init step.
Change-Id: I223964c9fe076bfb8c882f4a358e3f8c6de6806b
diff --git a/presubmit.py b/presubmit.py
index 3ad6a1c..163450b 100755
--- a/presubmit.py
+++ b/presubmit.py
@@ -3,29 +3,35 @@
# Copyright 2019 The Pigweed Authors
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
-# use this file except in compliance with the License. You may obtain a copy
-# of the License at
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
+# License for the specific language governing permissions and limitations under
+# the License.
+"""Runs the Pigweed local presubmit checks."""
+import argparse
import os
+import re
import shutil
-import subprocess
+import sys
-from pw_presubmit.presubmit_tools import *
+from pw_presubmit.presubmit_tools import call, filter_paths, PresubmitFailure
+from pw_presubmit import format_cc, presubmit_tools
def _init_cipd():
cipd = os.path.abspath('.presubmit/cipd')
call(sys.executable, 'env_setup/cipd/update.py', '--install-dir', cipd)
os.environ['PATH'] = os.pathsep.join((
- cipd, os.path.join(cipd, 'bin'), os.environ['PATH'],
+ cipd,
+ os.path.join(cipd, 'bin'),
+ os.environ['PATH'],
))
print('PATH', os.environ['PATH'])
@@ -38,7 +44,7 @@
os.environ['PATH'] = os.pathsep.join((
os.path.join(venv, 'bin'),
os.environ['PATH'],
- )) # yapf: disable
+ ))
call('python', '-m', 'pip', 'install', '--upgrade', 'pip')
call('python', '-m', 'pip', 'install',
@@ -51,18 +57,17 @@
_init_virtualenv()
-def gn_test():
- """Test with gn."""
- out = '.presubmit/gn'
- call('gn', 'gen', '--check', out)
- call('ninja', '-C', out)
+presubmit_dir = lambda *paths: os.path.join('.presubmit', *paths)
-def bazel_test():
- """Test with bazel."""
- prefix = '.presubmit/bazel-'
- call('bazel', 'build', '//...', '--symlink_prefix', prefix)
- call('bazel', 'test', '//...', '--symlink_prefix', prefix)
+#
+# GN presubmit checks
+#
+def gn_args(**kwargs):
+ return '--args=' + ' '.join(f'{arg}={val}' for arg, val in kwargs.items())
+
+
+GN_GEN = 'gn', 'gen', '--color=always', '--check'
@filter_paths(endswith=['.gn', '.gni'])
@@ -70,21 +75,236 @@
call('gn', 'format', '--dry-run', *paths)
-@filter_paths(endswith='.py')
-def pylint(paths):
- call(sys.executable, '-m', 'pylint', '-E', *paths)
+def gn_clang_build():
+ call(
+ *GN_GEN, '--export-compile-commands', presubmit_dir('clang'),
+ gn_args(pw_target_config='"//targets/host/host.gni"',
+ pw_target_toolchain='"//pw_toolchain:host_clang_os"'))
+ call('ninja', '-C', presubmit_dir('clang'))
-PRESUBMIT_PROGRAM = (
- init,
- pragma_once,
- gn_format,
- # pylint, # TODO(hepler): enable pylint when it passes
- bazel_test,
- gn_test,
+def gn_gcc_build():
+ call(
+ *GN_GEN, presubmit_dir('gcc'),
+ gn_args(pw_target_config='"//targets/host/host.gni"',
+ pw_target_toolchain='"//pw_toolchain:host_gcc_os"'))
+ call('ninja', '-C', presubmit_dir('gcc'))
+
+
+def gn_arm_build():
+ call(
+ *GN_GEN, presubmit_dir('arm'),
+ gn_args(
+ pw_target_config='"//targets/stm32f429i-disc1/target_config.gni"'))
+ call('ninja', '-C', presubmit_dir('arm'))
+
+
+GN = (
+ gn_format,
+ gn_clang_build,
+ gn_gcc_build,
+ gn_arm_build,
)
+#
+# C++ presubmit checks
+#
+@filter_paths(endswith=format_cc.SOURCE_EXTENSIONS)
+def clang_format(paths):
+ if format_cc.check_format(paths):
+ raise PresubmitFailure
+
+
+@filter_paths(endswith=format_cc.SOURCE_EXTENSIONS)
+def clang_tidy(paths):
+ if not os.path.exists(presubmit_dir('clang', 'compile_commands.json')):
+ raise PresubmitFailure('clang_tidy MUST be run after generating '
+ 'compile_commands.json in a clang build!')
+
+ call('clang-tidy', f'-p={presubmit_dir("clang")}', *paths)
+
+
+CC = (
+ presubmit_tools.pragma_once,
+ clang_format,
+ # TODO(hepler): Enable clang-tidy when it passes.
+ # clang_tidy,
+)
+
+
+#
+# Python presubmit checks
+#
+@filter_paths(endswith='.py')
+def pylint_errors(paths):
+ call(sys.executable, '-m', 'pylint', '-E', *paths)
+
+
+@filter_paths(endswith='.py')
+def yapf(paths):
+ from yapf.yapflib.yapf_api import FormatFile
+
+ errors = []
+
+ for path in paths:
+ diff, _, changed = FormatFile(path, print_diff=True, in_place=False)
+ if changed:
+ errors.append(path)
+ print(format_cc.colorize_diff(diff))
+
+ if errors:
+ print(f'--> Files with formatting errors: {len(errors)}')
+ print(' ', '\n '.join(errors))
+ raise PresubmitFailure
+
+
+@filter_paths(endswith='.py', exclude=r'(?:.+/)?setup\.py')
+def mypy(paths):
+ import mypy.api as mypy_api
+
+ report, errors, exit_status = mypy_api.run(paths)
+ if exit_status:
+ print(errors)
+ print(report)
+ raise PresubmitFailure
+
+
+PYTHON = (
+ # TODO(hepler): Enable yapf, mypy, and pylint when they pass.
+ # pylint_errors,
+ # yapf,
+ # mypy,
+)
+
+
+#
+# Bazel presubmit checks
+#
+@filter_paths(endswith=format_cc.SOURCE_EXTENSIONS)
+def bazel_test(unused_paths):
+ prefix = '.presubmit/bazel-'
+ call('bazel', 'build', '//...', '--symlink_prefix', prefix)
+ call('bazel', 'test', '//...', '--symlink_prefix', prefix)
+
+
+BAZEL = (bazel_test, )
+
+#
+# General presubmit checks
+#
+COPYRIGHT_FIRST_LINE = re.compile(
+ r'^(#|//| \*) Copyright 20\d\d The Pigweed Authors$')
+
+COPYRIGHT_LINES = tuple("""\
+
+ Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ use this file except in compliance with the License. You may obtain a copy of
+ the License at
+
+ https://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ License for the specific language governing permissions and limitations under
+ the License.
+""".splitlines(True))
+
+_EXCLUDE_FROM_COPYRIGHT_NOTICE = (
+ r'(?:.+/)?\..+',
+ r'AUTHORS',
+ r'LICENSE',
+ r'.*\.md',
+ r'.*\.rst',
+ r'(?:.+/)?requirements.txt',
+ r'(?:.+/)?requirements.in',
+)
+
+
+@filter_paths(exclude=_EXCLUDE_FROM_COPYRIGHT_NOTICE)
+def copyright_notice(paths):
+ """Checks that the copyright notice is present."""
+
+ errors = []
+
+ for path in paths:
+ with open(path) as file:
+ # Skip shebang and blank lines
+ line = file.readline()
+ while line.startswith(('#!', '/*')) or not line.strip():
+ line = file.readline()
+
+ first_line = COPYRIGHT_FIRST_LINE.match(line)
+ if not first_line:
+ errors.append(path)
+ continue
+
+ comment = first_line.group(1)
+
+ for expected, actual in zip(COPYRIGHT_LINES, file):
+ if comment + expected != actual:
+ errors.append(path)
+ break
+
+ if errors:
+ print('-->', presubmit_tools.plural(errors, 'file'),
+ 'with a missing or incorrect copyright notice:')
+ print(' ', '\n '.join(errors))
+ raise PresubmitFailure
+
+
+GENERAL = (copyright_notice, )
+
+#
+# Presubmit check programs
+#
+QUICK_PRESUBMIT = (
+ *GENERAL,
+ *PYTHON,
+ gn_format,
+ gn_clang_build,
+ presubmit_tools.pragma_once,
+ clang_format,
+)
+
+PROGRAMS = {
+ 'full': GN + CC + PYTHON + BAZEL + GENERAL,
+ 'quick': QUICK_PRESUBMIT,
+}
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument(
+ '--clean',
+ action='store_true',
+ help='Deletes the .presubmit directory before starting')
+ parser.add_argument(
+ '--skip_init',
+ action='store_true',
+ help='Clone the buildtools to prior to running the checks')
+ parser.add_argument('-p',
+ '--program',
+ choices=PROGRAMS,
+ default='full',
+ help='Which presubmit program to run')
+
+ presubmit_tools.add_parser_arguments(parser)
+
+ args = parser.parse_args()
+
+ if args.clean and os.path.exists(presubmit_dir()):
+ shutil.rmtree(presubmit_dir())
+
+ init_step = () if args.skip_init else (init,)
+ program = init_step + PROGRAMS[args.program]
+
+ # Remove custom arguments so we can use args to call run_presubmit.
+ del args.clean, args.program, args.skip_init
+
+ return 0 if presubmit_tools.run_presubmit(program, **vars(args)) else 1
+
if __name__ == '__main__':
- sys.exit(0 if parse_args_and_run_presubmit(PRESUBMIT_PROGRAM) else 1)
+ sys.exit(main())
