[ci] Be explicit about files in the partial BIN_DIR
$BIN_DIR is the designated directory for "final" build artifacts. Build
jobs place their final outputs there. From this directory subsequent
build jobs can also take artifacts and process them further. For
example, the FPGA bitstream build step takes the compiled boot rom
binary from $BIN_DIR, synthesizes it into a bitstream, and places the
final bitstream file back into $BIN_DIR.
This concept assumes that every file in $BIN_DIR is unambiguously
associated with a build rule/job. Unfortunaely, this assumption was not
enforced before, and hence was violated (two steps produced the same
file). To decrease the likelihood of this happening again we now
explicitly state which files in $BIN_DIR are expected to be outputs from
a CI job. We also remove all files which were produced by
"upstream"/"dependent" jobs.
To finally ensure that files in partial $BIN_DIR archives don't override
each other is performed by `tar` when extracting partial archives: it
bails out of such an overwrite is about to happen.
Signed-off-by: Philipp Wagner <phw@lowrisc.org>
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index e0b1a2b..bf767a2 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -135,7 +135,8 @@
displayName: Delete all Nexys Video build artifacts
- template: ci/upload-artifacts-template.yml
parameters:
- artifact: sw_build
+ includePatterns:
+ - "/sw/***"
# Software targeting the Nexys Video board is produced by patching the source
# tree before building. This produces a full sw build tree, however only the
@@ -174,7 +175,8 @@
displayName: Run unit tests
- template: ci/upload-artifacts-template.yml
parameters:
- artifact: sw_build_nexysvideo
+ includePatterns:
+ - "/sw/device/**/*nexysvideo*"
# We continue building with GCC, despite defaulting to Clang. This is a copy of
# `sw_build` with `meson_init.sh` configured with the GCC toolchain, instead of
@@ -238,7 +240,8 @@
displayName: Build simulation with Verilator
- template: ci/upload-artifacts-template.yml
parameters:
- artifact: top_earlgrey_verilator
+ includePatterns:
+ - "/hw/top_earlgrey/Vtop_earlgrey_verilator"
- job: top_englishbreakfast_verilator
displayName: Build Verilator simulation of the English Breakfast toplevel design
@@ -273,7 +276,8 @@
displayName: Build simulation with Verilator
- template: ci/upload-artifacts-template.yml
parameters:
- artifact: top_englishbreakfast_verilator
+ includePatterns:
+ - "/hw/top_englishbreakfast/Vtop_englishbreakfast_verilator"
- job: execute_verilated_tests
displayName: Execute tests on the Verilated system
@@ -394,7 +398,8 @@
displayName: Display synthesis and implementation logs
- template: ci/upload-artifacts-template.yml
parameters:
- artifact: top_earlgrey_nexysvideo
+ includePatterns:
+ - "/hw/top_earlgrey/lowrisc_systems_top_earlgrey_nexysvideo_0.1.bit"
- job: top_englishbreakfast_cw305
displayName: Build CW305 variant of the English Breakfast toplevel design using Vivado
@@ -438,7 +443,8 @@
displayName: Build bitstream with Vivado
- template: ci/upload-artifacts-template.yml
parameters:
- artifact: top_englishbreakfast_cw305
+ includePatterns:
+ - "/hw/top_englishbreakfast/lowrisc_systems_top_englishbreakfast_cw305_0.1.bit"
- job: execute_fpga_tests
displayName: Execute tests on FPGA
diff --git a/ci/download-artifacts-template.yml b/ci/download-artifacts-template.yml
index d5b9e74..abd0d49 100644
--- a/ci/download-artifacts-template.yml
+++ b/ci/download-artifacts-template.yml
@@ -35,6 +35,12 @@
set -e
test -n "$BUILD_ROOT"
. util/build_consts.sh
+
+ test -f "$BUILD_ROOT/upstream_bin_dir_contents.txt" && {
+ echo The download-artifacts-template.yml template can be called only once per job.
+ exit 1
+ }
+
mkdir -p "$BIN_DIR"
echo 'Extracting partial BIN_DIRs:'
@@ -43,6 +49,14 @@
-exec \
tar -C "$BIN_DIR" \
--strip-components=1 \
- --overwrite \
-xvf {} \;
+
+ # Remember all files which were present in the upstream $BIN_DIRs.
+ find "$BIN_DIR" -type f -fprintf "$BUILD_ROOT/upstream_bin_dir_contents.txt" '%P\n'
+
+ echo
+ echo Upstream BIN_DIR contents:
+ echo vvvvvvvvvvvvvvvvvv
+ cat "$BUILD_ROOT/upstream_bin_dir_contents.txt"
+ echo ^^^^^^^^^^^^^^^^^^
displayName: Unpack upstream outputs
diff --git a/ci/install-package-dependencies.yml b/ci/install-package-dependencies.yml
index 5194bff..bc94770 100644
--- a/ci/install-package-dependencies.yml
+++ b/ci/install-package-dependencies.yml
@@ -43,6 +43,7 @@
cp apt-requirements.txt apt-requirements-ci.txt
echo "verilator-$(VERILATOR_VERSION)" >> apt-requirements-ci.txt
+ echo rsync >> apt-requirements-ci.txt
cat apt-requirements-ci.txt
# Ensure apt package index is up-to-date.
diff --git a/ci/upload-artifacts-template.yml b/ci/upload-artifacts-template.yml
index d2ff0d6..986675b 100644
--- a/ci/upload-artifacts-template.yml
+++ b/ci/upload-artifacts-template.yml
@@ -5,22 +5,68 @@
# Azure template for archiving pipeline step outputs and uploading them.
#
# This template will archive all of $BIN_DIR, and upload it for use by
-# downstream jobs using download_artifacts_template.yml.
+# downstream jobs using download-artifacts-template.yml.
#
# This template expects that a variable $BUILD_ROOT is set. See
# util/build_consts.sh for more information.
+
+parameters:
+ # Rsync-style file patterns to include in the partial BIN_DIR output.
+ - name: includePatterns
+ type: object
+ default: []
+
steps:
- bash: |
set -e
test -n "$BUILD_ROOT"
. util/build_consts.sh
+ # Write all include patterns to a file used by rsync.
+ echo "${{ join('\n', parameters.includePatterns) }}" > "$BUILD_ROOT/include_patterns.txt"
+
+ echo
+ echo Files matching these patterns will be included in the binary build artifact for this job:
+ echo vvvvvvvvvvvvvvvvvv
+ cat "$BUILD_ROOT/include_patterns.txt"
+ echo ^^^^^^^^^^^^^^^^^^
+
+ # The file upstream_bin_dir_contents.txt lists all files which were part
+ # of an "upstream" BIN_DIR which got downloaded at the beginning of this
+ # job. Ensure that this file exists, even if no upstream BIN_DIR was
+ # downloaded.
+ touch "$BUILD_ROOT/upstream_bin_dir_contents.txt"
+
+ BIN_DIR_FULL="${BIN_DIR}.full"
+ mv "$BIN_DIR" "$BIN_DIR_FULL"
+ mkdir -p "$BIN_DIR"
+
+ echo
+ echo Copying files into the output archive:
+ rsync \
+ --archive \
+ --verbose \
+ --remove-source-files \
+ --prune-empty-dirs \
+ --exclude-from="$BUILD_ROOT/upstream_bin_dir_contents.txt" \
+ --include="*/" \
+ --include-from="$BUILD_ROOT/include_patterns.txt" \
+ --exclude="*" \
+ "${BIN_DIR_FULL}/" "${BIN_DIR}/"
+
+ echo
+ echo 'Files in $BIN_DIR not considered outputs of this job:'
+ echo vvvvvvvvvvvvvvvvvv
+ find "$BIN_DIR_FULL"
+ echo ^^^^^^^^^^^^^^^^^^
+
tar -C "$BUILD_ROOT" \
-cvf "$BUILD_ROOT/build-bin.tar" \
"${BIN_DIR#"$BUILD_ROOT/"}"
- displayName: 'Archive step outputs'
+ displayName: Archive step outputs
- publish: "$(Build.ArtifactStagingDirectory)/build-bin.tar"
- artifact: ${{ parameters.artifact }}-build-bin
- displayName: 'Upload step outputs'
-
+ # The PhaseName is the string after the "job" key in the build description,
+ # e.g. "job: my_phase_name".
+ artifact: partial-build-bin-$(System.PhaseName)
+ displayName: Upload step outputs
