pw_presubmit: Add banned words check Add check for non-inclusive language. Change-Id: Iea485b4e2a40f9db56ef84b97930848eeb9114cc Bug: 386 Reviewed-on: https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/46360 Pigweed-Auto-Submit: Rob Mohr <mohrr@google.com> Commit-Queue: Auto-Submit <auto-submit@pigweed.google.com.iam.gserviceaccount.com> Reviewed-by: Keir Mierle <keir@google.com>
diff --git a/pw_presubmit/docs.rst b/pw_presubmit/docs.rst index 2173f6e..e53238e 100644 --- a/pw_presubmit/docs.rst +++ b/pw_presubmit/docs.rst
@@ -127,8 +127,9 @@ sys.exit(2) import pw_presubmit - from pw_presubmit import build, cli, environment, format_code, git_repo - from pw_presubmit import python_checks, filter_paths, PresubmitContext + from pw_presubmit import banned_words, build, cli, environment, format_code + from pw_presubmit import git_repo, python_checks, filter_paths + from pw_presubmit import PresubmitContext from pw_presubmit.install_hook import install_hook # Set up variables for key project paths. @@ -184,14 +185,13 @@ # Presubmit check programs # QUICK = ( - # Initialize an environment for running presubmit checks. - init_cipd, - init_virtualenv, # List some presubmit checks to run pragma_once, host_tests, # Use the upstream formatting checks, with custom path filters applied. format_code.presubmit_checks(exclude=PATH_EXCLUSIONS), + # Include the upstream inclusive language check. + banned_words.banned_words, ) FULL = ( @@ -238,4 +238,4 @@ --------------------- The ``pw_presubmit.format_code`` module formats supported source files using external code format tools. The file ``format_code.py`` can be invoked directly -from the command line or from ``pw`` as ``pw format``. \ No newline at end of file +from the command line or from ``pw`` as ``pw format``.
diff --git a/pw_presubmit/py/BUILD.gn b/pw_presubmit/py/BUILD.gn index cfaa150..444ab0f 100644 --- a/pw_presubmit/py/BUILD.gn +++ b/pw_presubmit/py/BUILD.gn
@@ -20,6 +20,7 @@ setup = [ "setup.py" ] sources = [ "pw_presubmit/__init__.py", + "pw_presubmit/banned_words.py", "pw_presubmit/build.py", "pw_presubmit/cli.py", "pw_presubmit/environment.py",
diff --git a/pw_presubmit/py/pw_presubmit/banned_words.py b/pw_presubmit/py/pw_presubmit/banned_words.py new file mode 100644 index 0000000..eae763b --- /dev/null +++ b/pw_presubmit/py/pw_presubmit/banned_words.py
@@ -0,0 +1,141 @@ +# Copyright 2021 The Pigweed Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +"""Banned words presubmit check.""" + +import collections +from pathlib import Path +import re +from typing import Dict, List + +from . import presubmit + +# List borrowed from Android: +# https://source.android.com/setup/contribute/respectful-code +# banned-words: disable +BANNED_WORDS = [ + r'master', + r'slave', + r'(white|gr[ae]y|black)\s*(list|hat)', + r'craz(y|ie)', + r'insane', + r'crip+led?', + r'sanity', + r'sane', + r'dummy', + r'grandfather', + r's?he', + r'his', + r'her', + r'm[ae]n[-\s]*in[-\s]*the[-\s]*middle', + r'mitm', +] +# banned-words: enable + +# Test: master # banned-words: ignore +# Test: master + + +def _process_banned_words(*words): + """Turn banned-words list into one big regex with common inflections.""" + + if not words: + words = tuple(BANNED_WORDS) + + all_words = [] + for entry in words: + if isinstance(entry, str): + all_words.append(entry) + elif isinstance(entry, (list, tuple)): + all_words.extend(entry) + all_words.extend(x for x in words) + all_words = tuple(all_words) + + # Confirm each individual word compiles as a valid regex. + for word in all_words: + _ = re.compile(word) + + return re.compile( + r"\b({})(\b|e?[sd]\b)".format('|'.join(all_words)), + re.IGNORECASE, + ) + + +BANNED_WORDS_REGEX = _process_banned_words() + +# If seen, ignore this line and the next. +_IGNORE = 'banned-words: ignore' + +# Ignore a whole section. Please do not change the order of these lines. +_DISABLE = 'banned-words: disable' +_ENABLE = 'banned-words: enable' + + +def banned_words( + ctx: presubmit.PresubmitContext, + words_regex=BANNED_WORDS_REGEX, +): + """Presubmit check that ensures files do not contain banned words.""" + + Match = collections.namedtuple('Match', 'line word') + found_words: Dict[Path, List[Match]] = {} + + for path in ctx.paths: + try: + with open(path, 'r') as ins: + enabled = True + prev = '' + for i, line in enumerate(ins, start=1): + if _DISABLE in line: + enabled = False + if _ENABLE in line: + enabled = True + + # If we see the ignore line on this or the previous line we + # ignore any bad words on this line. + ignored = _IGNORE in prev or _IGNORE in line + + if enabled and not ignored: + match = words_regex.search(line) + + if match: + found_words.setdefault(path, []) + found_words[path].append(Match(i, match.group(0))) + + # Not using 'continue' so this line always executes. + prev = line + + except UnicodeDecodeError: + # File is not text, like a gif. + pass + + for path, matches in found_words.items(): + print('=' * 40) + print(path) + for match in matches: + print(f'Found banned word "{match.word}" on line {match.line}') + + if found_words: + raise presubmit.PresubmitFailure + + +def banned_words_checker(*words): + """Create banned words checker for the given list of banned words.""" + + regex = _process_banned_words(*words) + + def banned_words( # pylint: disable=redefined-outer-name + ctx: presubmit.PresubmitContext): + globals()['banned_words'](ctx, regex) + + return banned_words
diff --git a/pw_presubmit/py/pw_presubmit/pigweed_presubmit.py b/pw_presubmit/py/pw_presubmit/pigweed_presubmit.py index 4e6676b..d2e0040 100755 --- a/pw_presubmit/py/pw_presubmit/pigweed_presubmit.py +++ b/pw_presubmit/py/pw_presubmit/pigweed_presubmit.py
@@ -35,7 +35,7 @@ import pw_package.pigweed_packages -from pw_presubmit import build, cli, format_code, git_repo +from pw_presubmit import banned_words, build, cli, format_code, git_repo from pw_presubmit import call, filter_paths, plural, PresubmitContext from pw_presubmit import PresubmitFailure, Programs from pw_presubmit.install_hook import install_hook @@ -621,6 +621,7 @@ # OTHER_CHECKS = ( + banned_words.banned_words, # TODO(pwbug/45): Remove clang-tidy from OTHER_CHECKS when it passes. clang_tidy, # Build that attempts to duplicate the build OSS-Fuzz does. Currently