[ci] Move code for quick lint job into bash scripts
This shouldn't cause any changes to behaviour, but I have done a bit
of cleaning up (using the magic pathspec arguments to git diff to
simplify some things, and avoiding creating files in $PWD, to make
this nicer to run locally).
The idea is that a developer can now run
ci/jobs/quick-lint.sh
to get roughly the same tests as are run by the quick lint step for a
pull request in CI.
Signed-off-by: Rupert Swarbrick <rswarbrick@lowrisc.org>
diff --git a/ci/jobs/quick-lint.sh b/ci/jobs/quick-lint.sh
new file mode 100755
index 0000000..97e99ea
--- /dev/null
+++ b/ci/jobs/quick-lint.sh
@@ -0,0 +1,63 @@
+#!/bin/bash
+# Copyright lowRISC contributors.
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+
+# A wrapper that duplicates the code for the quick lint job in
+# azure-pipelines.yml. The two should be kept in sync.
+#
+# This doesn't install dependencies, but should otherwise behave the
+# same as what CI would do on a pull request.
+
+set -e
+
+case $# in
+ 0)
+ tgt_branch=master
+ ;;
+ 1)
+ tgt_branch="$1"
+ shift
+ ;;
+ *)
+ echo >&2 "Usage: quick-lint.sh [<tgt-branch>]"
+ exit 1
+esac
+
+echo "### Display environment information"
+ci/scripts/show-env.sh
+
+echo -e "\n### Check commit metadata"
+ci/scripts/lint-commits.sh $tgt_branch
+
+echo -e "\n### Check Licence Headers"
+ci/scripts/check-licence-headers.sh $tgt_branch
+
+echo -e "\n### Run Python lint"
+ci/scripts/python-lint.sh $tgt_branch || {
+ echo "(ignoring python lint errors)"
+}
+
+echo -e "\n### Ensure all generated files are clean and up-to-date"
+ci/scripts/check-generated.sh
+
+echo -e "\n### Use clang-format to check C/C++ coding style"
+ci/scripts/clang-format.sh $tgt_branch
+
+echo -e "\n### Check formatting on header guards"
+ci/scripts/include-guard.sh $tgt_branch
+
+echo -e "\n### Style-Lint RTL Verilog source files with Verible"
+ci/scripts/verible-lint.sh rtl
+
+echo -e "\n### Style-Lint DV Verilog source files with Verible"
+ci/scripts/verible-lint.sh dv
+
+echo -e "\n### Render documentation"
+ci/scripts/build-docs.sh
+
+echo -e "\n### Render landing site"
+ci/scripts/build-site.sh
+
+echo -e "\n### Check what kinds of changes the PR contains"
+ci/scripts/get-build-type.sh $tgt_branch PullRequest
diff --git a/ci/scripts/README.md b/ci/scripts/README.md
new file mode 100644
index 0000000..21ef48d
--- /dev/null
+++ b/ci/scripts/README.md
@@ -0,0 +1,4 @@
+# CI scripts
+
+These scripts are run as part of CI. The top-level orchestration is
+contained in azure-pipelines.yml.
diff --git a/ci/scripts/build-docs.sh b/ci/scripts/build-docs.sh
new file mode 100755
index 0000000..eabbb85
--- /dev/null
+++ b/ci/scripts/build-docs.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+# Copyright lowRISC contributors.
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+
+# Build docs and tell the Azure runner to upload any doxygen warnings
+
+set -e
+util/build_docs.py || {
+ echo -n "##vso[task.logissue type=error]"
+ echo "Documentation build failed."
+ exit 1
+}
+
+# Upload Doxygen Warnings if Present
+if [ -f "build/docs-generated/sw/doxygen_warnings.log" ]; then
+ echo -n "##vso[task.uploadfile]"
+ echo "${PWD}/build/docs-generated/sw/doxygen_warnings.log"
+ # Doxygen currently generates lots of warnings.
+ # echo -n "##vso[task.issue type=warning]"
+ # echo "Doxygen generated warnings. Use 'util/build_docs.py' to generate warning logfile."
+fi
diff --git a/ci/scripts/build-site.sh b/ci/scripts/build-site.sh
new file mode 100755
index 0000000..2353f4f
--- /dev/null
+++ b/ci/scripts/build-site.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+# Copyright lowRISC contributors.
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+
+# Build the landing site
+
+set -e
+cd site/landing
+../../build/docs-hugo/hugo
diff --git a/ci/scripts/check-generated.sh b/ci/scripts/check-generated.sh
new file mode 100755
index 0000000..43929bf
--- /dev/null
+++ b/ci/scripts/check-generated.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+# Copyright lowRISC contributors.
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+
+# Check generated files are up to date
+
+make -C hw regs && git diff --exit-code
+if [[ $? != 0 ]]; then
+ echo -n "##vso[task.logissue type=error]"
+ echo "Register headers not up-to-date. Regenerate them with 'make -C hw regs'."
+ exit 1
+fi
+
+make -C hw top && git diff --exit-code
+if [[ $? != 0 ]]; then
+ echo -n "##vso[task.logissue type=error]"
+ echo "Autogenerated tops not up-to-date. Regenerate with 'make -C hw top'."
+ exit 1
+fi
+
+make -C hw otp-mmap && git diff --exit-code
+if [[ $? != 0 ]]; then
+ echo -n "##vso[task.logissue type=error]"
+ echo "Autogenerated OTP memory map files not up-to-date. Regenerate with 'make -C hw otp-mmap'."
+ exit 1
+fi
+
+make -C hw lc-state-enc && git diff --exit-code
+if [[ $? != 0 ]]; then
+ echo -n "##vso[task.logissue type=error]"
+ echo "Autogenerated LC state not up-to-date. Regenerate with 'make -C hw lc-state-enc'."
+ exit 1
+fi
diff --git a/ci/scripts/check-licence-headers.sh b/ci/scripts/check-licence-headers.sh
new file mode 100755
index 0000000..23e8aa8
--- /dev/null
+++ b/ci/scripts/check-licence-headers.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+# Copyright lowRISC contributors.
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+
+# A wrapper around licence-checker.py, used for CI.
+#
+# Expects a single argument, which is the pull request's target branch
+# (usually "master").
+
+set -e
+
+if [ $# != 1 ]; then
+ echo >&2 "Usage: check-licence-headers.sh <tgt-branch>"
+ exit 1
+fi
+tgt_branch="$1"
+
+merge_base="$(git merge-base --fork-point origin/$tgt_branch)" || {
+ echo >&2 "Failed to find fork point for origin/$tgt_branch."
+ exit 1
+}
+echo "Checking licence headers on files changed since $merge_base"
+
+lc_script=util/lowrisc_misc-linters/licence-checker/licence-checker.py
+lc_cmd="$lc_script --config util/licence-checker.hjson"
+
+# Ask git for a list of null-separated names of changed files and pipe
+# those through to the licence checker using xargs. Setting pipefail
+# ensures that we'll see an error if the git diff command fails for
+# some reason.
+set -o pipefail
+git diff -z --name-only --diff-filter=ACMRTUXB "$merge_base" | \
+ xargs -r -0 $lc_cmd || {
+
+ echo >&2 -n "##vso[task.logissue type=error]"
+ echo >&2 "Licence header check failed."
+ exit 1
+}
diff --git a/ci/scripts/clang-format.sh b/ci/scripts/clang-format.sh
new file mode 100755
index 0000000..bc89fc9
--- /dev/null
+++ b/ci/scripts/clang-format.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+# Copyright lowRISC contributors.
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+
+# A wrapper around clang-format, used for CI.
+#
+# Expects a single argument, which is the pull request's target branch
+# (usually "master").
+
+set -e
+
+if [ $# != 1 ]; then
+ echo >&2 "Usage: clang-format.sh <tgt-branch>"
+ exit 1
+fi
+tgt_branch="$1"
+
+merge_base="$(git merge-base --fork-point origin/$tgt_branch)" || {
+ echo >&2 "Failed to find fork point for origin/$tgt_branch."
+ exit 1
+}
+echo "Running C/C++ lint checks on files changed since $merge_base"
+
+TMPFILE="$(mktemp)" || {
+ echo >&2 "Failed to create temporary file"
+ exit 1
+}
+trap 'rm -f "$TMPFILE"' EXIT
+
+set -o pipefail
+git diff -U0 "$merge_base" -- "*.cpp" "*.cc" "*.c" "*.h" ':!*/vendor/*' | \
+ clang-format-diff -p1 | \
+ tee "$TMPFILE"
+if [ -s "$TMPFILE" ]; then
+ echo -n "##vso[task.logissue type=error]"
+ echo "C/C++ lint failed. Use 'git clang-format' with appropriate options to reformat the changed code."
+ exit 1
+fi
diff --git a/ci/scripts/get-build-type.sh b/ci/scripts/get-build-type.sh
new file mode 100755
index 0000000..43dd300
--- /dev/null
+++ b/ci/scripts/get-build-type.sh
@@ -0,0 +1,63 @@
+#!/bin/bash
+# Copyright lowRISC contributors.
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+
+# Set various Azure Pipelines variables to figure out what the build
+# type should be. These are used to control what downstream CI tasks
+# get run.
+
+set -e
+
+if [ $# != 2 ]; then
+ echo >&2 "Usage: get-build-type.sh <tgt-branch> <build-reason>"
+ exit 1
+fi
+tgt_branch="$1"
+build_reason="$2"
+
+only_doc_changes=0
+only_dv_changes=0
+has_otbn_changes=1
+if [[ "$build_reason" = "PullRequest" ]]; then
+ # Conservative way of checking for documentation-only and OTBN changes.
+ # Only relevant for pipelines triggered from pull requests
+ merge_base="$(git merge-base --fork-point origin/$tgt_branch)" || {
+ echo >&2 "Failed to find fork point for origin/$tgt_branch."
+ exit 1
+ }
+ echo "Considering changes since $merge_base"
+
+ echo "Checking for changes in this PR other than to .md files"
+ # If git diff fails, this means that there were some changes to
+ # files other than documentation. Otherwise, there are not.
+ git diff --quiet "$merge_base" -- ':!*.md' && only_doc_changes=1 || true
+ if [[ $only_doc_changes -eq 1 ]]; then
+ echo "PR is only doc changes"
+ else
+ echo "PR contains non doc changes"
+ fi
+
+ echo "Checking for non-DV changes in this PR"
+ git diff --quiet "$merge_base" -- ':!*/dv/*' && only_dv_changes=1 || true
+ if [[ $only_dv_changes -eq 1 ]]; then
+ echo "PR is only DV changes"
+ else
+ echo "PR contains non-DV changes"
+ fi
+
+ # Check if any changes were made to OTBN-related files (hardware,
+ # software or tooling). This command is "the other way around"
+ # from the docs and DV commands above. It fails if there are any
+ # OTBN changes and succeeds otherwise.
+ echo "Checking if any OTBN changes are in this pull request"
+ git diff --quiet "$merge_base" -- '*/otbn/*' && has_otbn_changes=0
+ if [[ $has_otbn_changes -eq 1 ]]; then
+ echo "PR contains OTBN changes"
+ else
+ echo "PR doesn't contain OTBN changes"
+ fi
+fi
+echo "##vso[task.setvariable variable=onlyDocChanges;isOutput=true]${only_doc_changes}"
+echo "##vso[task.setvariable variable=onlyDvChanges;isOutput=true]${only_dv_changes}"
+echo "##vso[task.setvariable variable=hasOTBNChanges;isOutput=true]${has_otbn_changes}"
diff --git a/ci/scripts/include-guard.sh b/ci/scripts/include-guard.sh
new file mode 100755
index 0000000..e7314fc
--- /dev/null
+++ b/ci/scripts/include-guard.sh
@@ -0,0 +1,40 @@
+#!/bin/bash
+# Copyright lowRISC contributors.
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+
+# A wrapper around fix_include_guard.py, used for CI.
+#
+# Expects a single argument, which is the pull request's target branch
+# (usually "master").
+
+set -e
+
+if [ $# != 1 ]; then
+ echo >&2 "Usage: include-guard.sh <tgt-branch>"
+ exit 1
+fi
+tgt_branch="$1"
+
+merge_base="$(git merge-base --fork-point origin/$tgt_branch)" || {
+ echo >&2 "Failed to find fork point for origin/$tgt_branch."
+ exit 1
+}
+echo "Checking header guards on headers changed since $merge_base"
+
+TMPFILE="$(mktemp)" || {
+ echo >&2 "Failed to create temporary file"
+ exit 1
+}
+trap 'rm -f "$TMPFILE"' EXIT
+
+set -o pipefail
+git diff --name-only --diff-filter=ACMRTUXB "$merge_base" -- \
+ "*.h" ':!*/vendor/*' | \
+ xargs -r util/fix_include_guard.py --dry-run | \
+ tee "$TMPFILE"
+if [ -s "$TMPFILE" ]; then
+ echo -n "##vso[task.logissue type=error]"
+ echo "Include guard check failed. Please run util/fix_include_guard.py on the above files."
+ exit 1
+fi
diff --git a/ci/scripts/lint-commits.sh b/ci/scripts/lint-commits.sh
new file mode 100755
index 0000000..83bf01f
--- /dev/null
+++ b/ci/scripts/lint-commits.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+# Copyright lowRISC contributors.
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+
+# A wrapper around lint_commits.py, used for CI.
+#
+# Expects a single argument, which is the pull request's target branch
+# (usually "master").
+
+if [ $# != 1 ]; then
+ echo >&2 "Usage: lint-commits.sh <tgt-branch>"
+ exit 1
+fi
+tgt_branch="$1"
+
+merge_base="$(git merge-base --fork-point origin/$tgt_branch)" || {
+ echo >&2 "Failed to find fork point for origin/$tgt_branch."
+ exit 1
+}
+echo "Checking commit messages since $merge_base"
+
+# Notes:
+# * Merge commits are not checked. We always use rebases instead of
+# merges to keep a linear history, which makes merge commits disappear
+# ultimately, making them only a CI artifact which should not be
+# checked.
+# * 'type=error' is used even for warnings. Only "errors" are shown in
+# the GitHub checks API. However, warnings don't return a non-zero
+# error code so don't fail the build step.
+util/lint_commits.py \
+ --no-merges \
+ --error-msg-prefix="##vso[task.logissue type=error]" \
+ --warning-msg-prefix="##vso[task.logissue type=error]" \
+ "$merge_base"..HEAD
diff --git a/ci/scripts/python-lint.sh b/ci/scripts/python-lint.sh
new file mode 100755
index 0000000..109180a
--- /dev/null
+++ b/ci/scripts/python-lint.sh
@@ -0,0 +1,51 @@
+#!/bin/bash
+# Copyright lowRISC contributors.
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+
+# A wrapper around lintpy.py, used for CI.
+#
+# Expects a single argument, which is the pull request's target branch
+# (usually "master").
+
+set -e
+
+if [ $# != 1 ]; then
+ echo >&2 "Usage: python-lint.sh <tgt-branch>"
+ exit 1
+fi
+tgt_branch="$1"
+
+merge_base="$(git merge-base --fork-point origin/$tgt_branch)" || {
+ echo >&2 "Failed to find fork point for origin/$tgt_branch."
+ exit 1
+}
+echo "Linting python code changed since $merge_base"
+
+ignored_subtrees=(
+ "*/vendor/"
+ util/lowrisc_misc-linters
+)
+
+pathspec_args=("*.py")
+for st in "${ignored_subtrees[@]}"; do
+ # This generates an argument like :!/FOO*, that tells git to
+ # ignore every path matching the glob /FOO*. See the description
+ # of pathspecs in gitglossary(7) for more information.
+ pathspec_args+=(':!/'"$st"'*')
+done
+
+lintpy_cmd="util/lintpy.py --tools flake8 -f"
+
+# Ask git for a list of null-separated names of changed files that
+# don't appear in an ignored directory. Pipe those through to the
+# licence checker using xargs. Setting pipefail ensures that we'll see
+# an error if the git diff command fails for some reason.
+set -o pipefail
+git diff -z --name-only --diff-filter=ACMRTUXB "$merge_base" -- \
+ "${pathspec_args[@]}" | \
+ xargs -0 -r $lintpy_cmd || {
+ echo -n "##vso[task.logissue type=error]"
+ echo "Python lint failed."
+ exit 1
+}
diff --git a/ci/scripts/show-env.sh b/ci/scripts/show-env.sh
new file mode 100755
index 0000000..e537ea4
--- /dev/null
+++ b/ci/scripts/show-env.sh
@@ -0,0 +1,37 @@
+#!/bin/bash
+# Copyright lowRISC contributors.
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+
+# A script that dumps the current environment, plus the versions of
+# various tools to stdout.
+
+set -e
+
+tools=(
+ git
+ python3
+ yapf
+ isort
+ clang-format
+ flake8
+ ninja
+ meson
+ doxygen
+ verible-verilog-lint
+)
+
+for tool in "${tools[@]}"; do
+ set -x
+ $tool --version
+ { set +x; } 2>/dev/null
+ echo
+done
+
+set -x
+echo "PATH=$PATH"
+{ set +x; } 2>/dev/null
+echo
+
+set -x
+printenv
diff --git a/ci/scripts/verible-lint.sh b/ci/scripts/verible-lint.sh
new file mode 100755
index 0000000..0fbadd6
--- /dev/null
+++ b/ci/scripts/verible-lint.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+# Copyright lowRISC contributors.
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+
+# A wrapper around Verible lint, used for CI.
+#
+# Expects a single argument, which is the pull request's target branch
+# (usually "master").
+
+set -e
+
+if [ $# != 1 ]; then
+ echo >&2 "Usage: verible-lint.sh <flavour>"
+ exit 1
+fi
+flavour="$1"
+
+case "$flavour" in
+ rtl)
+ human_desc=design
+ dvsim_cfg=hw/top_earlgrey/lint/top_earlgrey_lint_cfgs.hjson
+ ;;
+
+ dv)
+ human_desc=DV
+ dvsim_cfg=hw/top_earlgrey/lint/top_earlgrey_dv_lint_cfgs.hjson
+ ;;
+
+ *)
+ echo >&2 "Unknown lint flavour: $flavour"
+ exit 1
+esac
+
+util/dvsim/dvsim.py --tool=veriblelint "$dvsim_cfg" || {
+ echo -n "##vso[task.logissue type=error]"
+ echo "Verilog style lint of $human_desc sources with Verible failed. Run 'util/dvsim/dvsim.py -t veriblelint $dvsim_cfg' and fix all errors."
+ exit 1
+}
