blob: 7a993251aaf426cbfd34e29f036625ca20c1bfe0 [file] [log] [blame]
#!/usr/bin/env python3
# Copyright 2019 The Pigweed Authors
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.
"""Runs the local presubmit checks for the Pigweed repository."""
import argparse
import glob
import logging
import os
import re
import shutil
import subprocess
import sys
from typing import Dict, Sequence
try:
import pw_presubmit
except ImportError:
# Append the pw_presubmit package path to the module search path to allow
# running this module without installing the pw_presubmit package.
sys.path.append(os.path.dirname(os.path.dirname(
os.path.abspath(__file__))))
import pw_presubmit
from pw_presubmit import format_code, PresubmitContext
from pw_presubmit.install_hook import install_hook
from pw_presubmit import call, filter_paths, log_run, plural, PresubmitFailure
_LOG: logging.Logger = logging.getLogger(__name__)
PRESUBMIT_PREFIX = '.presubmit'
def presubmit_dir(*paths):
"""Returns a relative path from the presubmit output directory."""
return os.path.join(PRESUBMIT_PREFIX, *paths)
def run_python_module(*args, **kwargs):
return call('python', '-m', *args, **kwargs)
#
# Initialization
#
def init_cipd(unused_ctx: PresubmitContext):
cipd = os.path.abspath('.presubmit/cipd')
call(sys.executable, 'env_setup/cipd/update.py', '--install-dir', cipd)
paths = [cipd, os.path.join(cipd, 'bin')]
for base in glob.glob(os.path.join(cipd, '*')):
paths.append(base)
paths.append(os.path.join(base, 'bin'))
paths.append(os.environ['PATH'])
os.environ['PATH'] = os.pathsep.join(paths)
_LOG.debug('PATH %s', os.environ['PATH'])
def init_virtualenv(unused_ctx: PresubmitContext):
"""Set up virtualenv, assumes recent Python 3 is already installed."""
venv = os.path.abspath('.presubmit/venv')
# For speed, don't build the venv if it exists. Use --clean to recreate it.
if not os.path.isdir(venv):
call(
'python3',
'env_setup/virtualenv/init.py',
f'--venv_path={venv}',
'--requirements=env_setup/virtualenv/requirements.txt',
)
os.environ['PATH'] = os.pathsep.join((
os.path.join(venv, 'bin'),
os.environ['PATH'],
))
INIT = (
init_cipd,
init_virtualenv,
)
#
# GN presubmit checks
#
def gn_args(**kwargs):
return '--args=' + ' '.join(f'{arg}={val}' for arg, val in kwargs.items())
def gn_gen(*args):
call('gn', 'gen', '--color=always', '--check', *args)
_CLANG_GEN_ARGS = (presubmit_dir('clang'),
gn_args(
pw_target_config='"//targets/host/host.gni"',
pw_target_toolchain='"//pw_toolchain:host_clang_os"'))
def gn_clang_build(unused_ctx: PresubmitContext):
gn_gen('--export-compile-commands', *_CLANG_GEN_ARGS)
call('ninja', '-C', presubmit_dir('clang'))
@filter_paths(endswith=format_code.C_FORMAT.extensions)
def gn_gcc_build(unused_ctx: PresubmitContext):
gn_gen(
presubmit_dir('gcc'),
gn_args(pw_target_config='"//targets/host/host.gni"',
pw_target_toolchain='"//pw_toolchain:host_gcc_os"'))
call('ninja', '-C', presubmit_dir('gcc'))
_ARM_GEN_ARGS = (
presubmit_dir('arm'),
gn_args(pw_target_config='"//targets/stm32f429i-disc1/target_config.gni"'))
@filter_paths(endswith=format_code.C_FORMAT.extensions)
def gn_arm_build(unused_ctx: PresubmitContext):
gn_gen(*_ARM_GEN_ARGS)
call('ninja', '-C', presubmit_dir('arm'))
GN = (
gn_clang_build,
gn_gcc_build,
gn_arm_build,
)
#
# C++ presubmit checks
#
@filter_paths(endswith=format_code.C_FORMAT.extensions)
def clang_tidy(ctx: PresubmitContext):
if not os.path.exists(presubmit_dir('clang', 'compile_commands.json')):
raise PresubmitFailure('clang_tidy MUST be run after generating '
'compile_commands.json in a clang build!')
call('clang-tidy', f'-p={presubmit_dir("clang")}', *ctx.paths)
CC = (
pw_presubmit.pragma_once,
# TODO(hepler): Enable clang-tidy when it passes.
# clang_tidy,
)
#
# Python presubmit checks
#
@filter_paths(endswith='.py')
def test_python_packages(ctx: PresubmitContext):
packages = pw_presubmit.find_python_packages(ctx.paths)
if not packages:
_LOG.info('No Python packages were found.')
return
for package in packages:
call('python', os.path.join(package, 'setup.py'), 'test', cwd=package)
@filter_paths(endswith='.py')
def pylint(ctx: PresubmitContext):
disable_checkers = [
# BUG(pwbug/22): Hanging indent check conflicts with YAPF 0.29. For
# now, use YAPF's version even if Pylint is doing the correct thing
# just to keep operations simpler. When YAPF upstream fixes the issue,
# delete this code.
#
# See also: https://github.com/google/yapf/issues/781
'bad-continuation',
]
run_python_module(
'pylint',
'--jobs=0',
f'--disable={",".join(disable_checkers)}',
*ctx.paths,
)
@filter_paths(endswith='.py', exclude=r'(?:.+/)?setup\.py')
def mypy(ctx: PresubmitContext):
run_python_module('mypy', *ctx.paths)
PYTHON = (
test_python_packages,
pylint,
# TODO(hepler): Enable mypy when it passes.
# mypy,
)
#
# Bazel presubmit checks
#
@filter_paths(endswith=format_code.C_FORMAT.extensions)
def bazel_test(unused_paths):
prefix = '.presubmit/bazel-'
call('bazel', 'build', '//...', '--symlink_prefix', prefix)
call('bazel', 'test', '//...', '--symlink_prefix', prefix)
BAZEL = (bazel_test, )
#
# Code format presubmit checks
#
COPYRIGHT_FIRST_LINE = re.compile(
r'^(#|//| \*) Copyright 20\d\d The Pigweed Authors$')
COPYRIGHT_LINES = tuple("""\
Licensed under the Apache License, Version 2.0 (the "License"); you may not
use this file except in compliance with the License. You may obtain a copy of
the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations under
the License.
""".splitlines(True))
_EXCLUDE_FROM_COPYRIGHT_NOTICE: Sequence[str] = (
r'(?:.+/)?\..+',
r'AUTHORS',
r'LICENSE',
r'.*\.md',
r'.*\.rst',
r'(?:.+/)?requirements.txt',
r'(.+/)?go.(mod|sum)',
)
@filter_paths(exclude=_EXCLUDE_FROM_COPYRIGHT_NOTICE)
def copyright_notice(ctx: PresubmitContext):
"""Checks that the copyright notice is present."""
errors = []
for path in ctx.paths:
with open(path) as file:
# Skip shebang and blank lines
line = file.readline()
while line.startswith(('#!', '/*')) or not line.strip():
line = file.readline()
first_line = COPYRIGHT_FIRST_LINE.match(line)
if not first_line:
errors.append(path)
continue
comment = first_line.group(1)
for expected, actual in zip(COPYRIGHT_LINES, file):
if comment + expected != actual:
errors.append(path)
break
if errors:
_LOG.warning('%s with a missing or incorrect copyright notice:\n%s',
pw_presubmit.plural(errors, 'file'), '\n'.join(errors))
raise PresubmitFailure
CODE_FORMAT = (copyright_notice, *format_code.PRESUBMIT_CHECKS)
#
# General presubmit checks
#
def _get_paths_from_command(*args):
"""Runs a command and reads Bazel or GN //-style paths from it."""
process = log_run(*args, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
files = set()
for line in process.stdout.splitlines():
path = line.strip().lstrip(b'/').replace(b':', b'/').decode()
if os.path.isfile(path):
files.add(path)
return files
@filter_paths(endswith=('.rst', *format_code.C_FORMAT.extensions))
def source_is_in_build_files(ctx: PresubmitContext):
"""Checks that source files are in the GN and Bazel builds."""
# Collect all paths in the Bazel build.
build = _get_paths_from_command('bazel', 'query',
'kind("source file", //...:*)')
# Collect all paths in the ARM and Clang GN builds.
gn_gen(*_ARM_GEN_ARGS)
build_gn = _get_paths_from_command('gn', 'desc', presubmit_dir('arm'), '*')
gn_gen(*_CLANG_GEN_ARGS)
build_gn.update(
_get_paths_from_command('gn', 'desc', presubmit_dir('clang'), '*'))
missing_bazel = []
missing_gn = []
for path in ctx.paths:
if not path.endswith('.rst') and path not in build:
missing_bazel.append(path)
if path not in build_gn:
missing_gn.append(path)
if missing_bazel or missing_gn:
for build, files in [('Bazel', missing_bazel), ('GN', missing_gn)]:
if files:
_LOG.warning('%s are missing from the %s build:\n%s',
plural(files, 'file'), build, '\n'.join(files))
_LOG.warning(
'All source files must appear in BUILD and BUILD.gn files')
raise PresubmitFailure
GENERAL = (source_is_in_build_files, )
#
# Presubmit check programs
#
QUICK_PRESUBMIT: Sequence = (
*INIT,
*PYTHON,
gn_clang_build,
pw_presubmit.pragma_once,
*CODE_FORMAT,
*GENERAL,
)
PROGRAMS: Dict[str, Sequence] = {
'full': INIT + GN + CC + PYTHON + BAZEL + CODE_FORMAT + GENERAL,
'quick': QUICK_PRESUBMIT,
}
def argument_parser(parser=None) -> argparse.ArgumentParser:
"""Create argument parser."""
if parser is None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
'--clean',
action='store_true',
help='Delete the entire .presubmit directory before starting.')
parser.add_argument(
'--clean-py',
action='store_true',
help='Delete the Python virtualenv at .presubmit/venv before starting.'
)
exclusive = parser.add_mutually_exclusive_group()
exclusive.add_argument(
'--install',
action='store_true',
help='Install the presubmit as a Git pre-push hook and exits.')
exclusive.add_argument('-p',
'--program',
dest='program_name',
choices=PROGRAMS,
default='full',
help='Which presubmit program to run')
exclusive.add_argument(
'--step',
choices=[x.__name__ for x in PROGRAMS['full']],
action='append',
help='Provide explicit steps instead of running a predefined program.',
)
pw_presubmit.add_arguments(parser)
return parser
def main(
program_name: str,
clean: bool,
clean_py: bool,
install: bool,
directory: str,
step: Sequence[str],
**presubmit_args,
) -> int:
"""Entry point for presubmit."""
environment = pw_presubmit.git_repo_path(PRESUBMIT_PREFIX, repo=directory)
_LOG.debug('Using environment at %s', environment)
if clean and environment.exists():
shutil.rmtree(environment)
elif clean_py and environment.joinpath('venv').exists():
shutil.rmtree(environment.joinpath('venv'))
if install:
install_hook(__file__, 'pre-push', ['--base', 'origin/master..HEAD'],
directory)
return 0
program = PROGRAMS[program_name]
if step:
program = [x for x in PROGRAMS['full'] if x.__name__ in step]
if pw_presubmit.run_presubmit(program,
directory=directory,
**presubmit_args):
return 0
return 1
if __name__ == '__main__':
# By default, display log messages like a simple print statement.
logging.basicConfig(format='%(message)s', level=logging.INFO)
sys.exit(main(**vars(argument_parser().parse_args())))