Drop build-dir outputs from build_all CI job. (#16333)
Progress on https://github.com/openxla/iree/issues/16203
All remaining uses of the `build-dir` outputs have been removed, so we
can skip generating and uploading those files. The `install-dir` is half
the size, and we may be able to shrink it even further (I think we still
want debug symbols and assertions for jobs that test using packages, but
that would be the largest lever to flip).
diff --git a/.github/workflows/ARTIFACT_SUMMARY_TEMPLATE.md b/.github/workflows/ARTIFACT_SUMMARY_TEMPLATE.md
index c8c8df4..e3060d1 100644
--- a/.github/workflows/ARTIFACT_SUMMARY_TEMPLATE.md
+++ b/.github/workflows/ARTIFACT_SUMMARY_TEMPLATE.md
@@ -2,7 +2,7 @@
```sh
# Compiler and host tools archive:
-export BUILD_DIR_URL="${BUILD_DIR_GCS_ARTIFACT}"
+export INSTALL_DIR_URL="${INSTALL_DIR_GCS_ARTIFACT}"
# Directory of e2e test artifacts:
export E2E_TEST_ARTIFACTS_DIR_URL="${E2E_TEST_ARTIFACTS_GCS_ARTIFACT_DIR}"
# Directory of benchmark tool binaries:
diff --git a/.github/workflows/build_all.yml b/.github/workflows/build_all.yml
index 3ed499b..d443a81 100644
--- a/.github/workflows/build_all.yml
+++ b/.github/workflows/build_all.yml
@@ -22,15 +22,6 @@
required: true
type: string
outputs:
- build-dir:
- description: Local build directory path.
- value: ${{ jobs.build_all.outputs.build-dir }}
- build-dir-archive:
- description: Name of the zipped build directory.
- value: ${{ jobs.build_all.outputs.build-dir-archive }}
- build-dir-gcs-artifact:
- description: GCS path to the uploaded build archive.
- value: ${{ jobs.build_all.outputs.build-dir-gcs-artifact }}
install-dir:
description: Local install directory path.
value: ${{ jobs.build_all.outputs.install-dir }}
@@ -61,7 +52,6 @@
BUILD_DIR: full-build-dir
INSTALL_DIR: full-build-dir/install
outputs:
- build-dir: ${{ env.BUILD_DIR }}
install-dir: ${{ env.INSTALL_DIR }}
# Pass the install directory as output for other jobs to use.
# TODO(#16203): replace this with iree-dist packages
@@ -70,10 +60,6 @@
# an intermediate step towards a fully package-based CI setup.
install-dir-archive: ${{ steps.install-archive.outputs.install-dir-archive }}
install-dir-gcs-artifact: ${{ steps.install-upload.outputs.install-dir-gcs-artifact }}
- # Pass the build directory as output for other jobs to use.
- # DEPRECATED: prefer to use install-* instead.
- build-dir-archive: ${{ steps.archive.outputs.build-dir-archive }}
- build-dir-gcs-artifact: ${{ steps.upload.outputs.build-dir-gcs-artifact }}
steps:
- name: "Checking out repository"
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
@@ -102,15 +88,6 @@
gcr.io/iree-oss/base@sha256:61e9aae211007dbad95e1f429e9e5121fd5968c204791038424979c21146cf75 \
./build_tools/cmake/test_iree_dialects.sh "${BUILD_DIR}"
- # Remove unused files to save disk space.
- # TODO(#16203): Switch to package-based CI
- # This could also be an allow-list instead of a deny-list, but packages
- # are even more explicit.
- - name: "Removing unused .a files"
- run: find "${BUILD_DIR}" -type f -name "*.a" -print -delete
- - name: "Removing unused .o files"
- run: find "${BUILD_DIR}" -type f -name "*.o" -print -delete
-
# TODO(#16203): If this (or iree-dist) is small enough, just use GitHub
# artifacts instead of `tar` commands and GCS
- name: "Creating install dir archive"
@@ -129,27 +106,3 @@
run: |
gcloud storage cp "${INSTALL_DIR_ARCHIVE}" "${INSTALL_DIR_GCS_ARTIFACT}"
echo "install-dir-gcs-artifact=${INSTALL_DIR_GCS_ARTIFACT}" >> "${GITHUB_OUTPUT}"
-
- # Things get more complicated here than when we're just building the
- # runtime. The build directory is way bigger. We're also using on our own
- # runners on GCE. So uploading to GitHub actions artifact storage hosted
- # on Azure is dirt slow. We drop static libraries and object files, which
- # aren't needed for testing. Then we do some minimal compression locally
- # *in parallel* and upload to GCS. This can be further optimized.
- # Especially decompression is still pretty slow. See #9881.
- - name: "Creating build dir archive"
- id: archive
- env:
- BUILD_DIR_ARCHIVE: ${{ env.BUILD_DIR }}.tar.zst
- run: |
- tar -I 'zstd -T0' \
- -cf ${BUILD_DIR_ARCHIVE} ${BUILD_DIR}
- echo "build-dir-archive=${BUILD_DIR_ARCHIVE}" >> "${GITHUB_OUTPUT}"
- - name: "Uploading build dir archive"
- id: upload
- env:
- BUILD_DIR_ARCHIVE: ${{ steps.archive.outputs.build-dir-archive }}
- BUILD_DIR_GCS_ARTIFACT: ${{ env.GCS_DIR }}/${{ steps.archive.outputs.build-dir-archive }}
- run: |
- gcloud storage cp "${BUILD_DIR_ARCHIVE}" "${BUILD_DIR_GCS_ARTIFACT}"
- echo "build-dir-gcs-artifact=${BUILD_DIR_GCS_ARTIFACT}" >> "${GITHUB_OUTPUT}"
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 341d8b8..5183d6f 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -980,7 +980,7 @@
is-pr: ${{ fromJson(needs.setup.outputs.is-pr) }}
install-dir: ${{ needs.build_all.outputs.install-dir }}
install-dir-archive: ${{ needs.build_all.outputs.install-dir-archive }}
- install-dir-gcs-artifact: ${{ needs.build_all.outputs.build-dir-gcs-artifact }}
+ install-dir-gcs-artifact: ${{ needs.build_all.outputs.install-dir-gcs-artifact }}
test_benchmark_suites:
needs: [setup, build_all, build_e2e_test_artifacts]
@@ -1122,7 +1122,7 @@
if: always()
env:
# If the job of an artifact is skipped or failed, we show "NOT_PRESENT".
- BUILD_DIR_GCS_ARTIFACT: ${{ needs.build_all.outputs.build-dir-gcs-artifact || 'NOT_PRESENT' }}
+ INSTALL_DIR_GCS_ARTIFACT: ${{ needs.build_all.outputs.install-dir-gcs-artifact || 'NOT_PRESENT' }}
E2E_TEST_ARTIFACTS_GCS_ARTIFACT_DIR: ${{ needs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-gcs-artifact-dir || 'NOT_PRESENT' }}
BENCHMARK_TOOLS_GCS_ARTIFACT_DIR: ${{ needs.build_benchmark_tools.outputs.benchmark-tools-gcs-artifact-dir || 'NOT_PRESENT' }}
EXECUTION_BENCHMARK_RESULTS_GCS_ARTIFACT_DIR: ${{ needs.execution_benchmarks.outputs.benchmark-results-gcs-artifact-dir || 'NOT_PRESENT' }}
