| # Copyright 2022 The IREE Authors | |
| # | |
| # Licensed under the Apache License v2.0 with LLVM Exceptions. | |
| # See https://llvm.org/LICENSE.txt for license information. | |
| # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |
| # | |
| # Workflow for building E2E test artifacts. | |
| # It is designed to be called from a parent workflow. | |
| # The concurrency of this workflow is controlled by the caller's job. | |
| name: Build E2E Test Artifacts | |
| on: | |
| workflow_call: | |
| inputs: | |
| runner-group: | |
| required: true | |
| type: string | |
| runner-env: | |
| required: true | |
| type: string | |
| install-dir: | |
| required: true | |
| type: string | |
| install-dir-archive: | |
| required: true | |
| type: string | |
| install-dir-gcs-artifact: | |
| required: true | |
| type: string | |
| benchmark-presets: | |
| description: | | |
| A comma-separated string of benchmark presets to build. | |
| required: true | |
| type: string | |
| build-default-benchmark-suites: | |
| description: | | |
| Build default benchmark suites for sanity check and e2e model testing | |
| if set to 1. | |
| default: 1 | |
| type: number | |
| shard-count: | |
| description: | | |
| A device-name to integer mapping as a comma separated list. | |
| Allows to assign a distinct shard count for each target device. | |
| The reserved keyword `default` assigns a shard count to all target devices | |
| that are not explicitly listed. | |
| default: default=1 | |
| type: string | |
| outputs: | |
| e2e-test-artifacts-dir: | |
| description: | | |
| Local path that stores compiled test artifacts. | |
| value: ${{ jobs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-dir }} | |
| e2e-test-artifacts-gcs-artifact-dir: | |
| description: | | |
| GCS path to the uploaded test artifacts. | |
| value: ${{ jobs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-gcs-artifact-dir }} | |
| e2e-test-artifacts-build-log: | |
| description: | | |
| Local path that contains the build log. | |
| value: ${{ jobs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-build-log }} | |
| e2e-test-artifacts-build-log-gcs-artifact: | |
| description: | | |
| GCS path to the uploaded build log. | |
| value: ${{ jobs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-build-log-gcs-artifact }} | |
| env: | |
| # This duplicates the variable from ci.yml. The variable needs to be in env | |
| # instead of the outputs of setup because it contains the run attempt and we | |
| # want that to be the current attempt, not whatever attempt the setup step | |
| # last ran in. It therefore can't be passed in via inputs because the env | |
| # context isn't available there. | |
| GCS_DIR: gs://iree-github-actions-${{ github.event_name == 'pull_request' && 'presubmit' || 'postsubmit' }}-artifacts/${{ github.run_id }}/${{ github.run_attempt }} | |
| jobs: | |
| build_e2e_test_artifacts: | |
| runs-on: | |
| - self-hosted # must come first | |
| - runner-group=${{ inputs.runner-group }} | |
| - environment=${{ inputs.runner-env }} | |
| - cpu | |
| - os-family=Linux | |
| env: | |
| INSTALL_DIR: ${{ inputs.install-dir }} | |
| INSTALL_DIR_ARCHIVE: ${{ inputs.install-dir-archive }} | |
| INSTALL_DIR_GCS_ARTIFACT: ${{ inputs.install-dir-gcs-artifact }} | |
| outputs: | |
| e2e-test-artifacts-dir: ${{ steps.build.outputs.e2e-test-artifacts-dir }} | |
| e2e-test-artifacts-gcs-artifact-dir: ${{ steps.upload.outputs.e2e-test-artifacts-gcs-artifact-dir }} | |
| e2e-test-artifacts-build-log: ${{ steps.build.outputs.e2e-test-artifacts-build-log }} | |
| e2e-test-artifacts-build-log-gcs-artifact: ${{ steps.upload.outputs.e2e-test-artifacts-build-log-gcs-artifact }} | |
| steps: | |
| - name: "Checking out repository" | |
| uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 | |
| - name: "Checking out runtime submodules" | |
| run: ./build_tools/scripts/git/update_runtime_submodules.sh | |
| - name: "Downloading install dir archive" | |
| run: gcloud storage cp "${INSTALL_DIR_GCS_ARTIFACT}" "${INSTALL_DIR_ARCHIVE}" | |
| - name: "Extracting install directory" | |
| run: tar -xf "${INSTALL_DIR_ARCHIVE}" | |
| - name: "Building e2e test artifacts" | |
| id: build | |
| env: | |
| IREE_BENCHMARK_PRESETS: ${{ inputs.benchmark-presets }} | |
| IREE_BUILD_DEFAULT_BENCHMARK_SUITES: ${{ inputs.build-default-benchmark-suites }} | |
| BUILD_E2E_TEST_ARTIFACTS_DIR: build-e2e-test-artifacts | |
| IREE_SHARD_COUNT: ${{ inputs.shard-count }} | |
| run: | | |
| echo "e2e-test-artifacts-dir=${BUILD_E2E_TEST_ARTIFACTS_DIR}/e2e_test_artifacts" >> "${GITHUB_OUTPUT}" | |
| echo "e2e-test-artifacts-build-log=${BUILD_E2E_TEST_ARTIFACTS_DIR}/.ninja_log" >> "${GITHUB_OUTPUT}" | |
| build_tools/github_actions/docker_run.sh \ | |
| --env "IREE_HOST_BIN_DIR=${INSTALL_DIR}/bin" \ | |
| --env "IREE_BENCHMARK_PRESETS=${IREE_BENCHMARK_PRESETS}" \ | |
| --env "IREE_BUILD_DEFAULT_BENCHMARK_SUITES=${IREE_BUILD_DEFAULT_BENCHMARK_SUITES}" \ | |
| --env "IREE_SHARD_COUNT=${IREE_SHARD_COUNT}" \ | |
| gcr.io/iree-oss/base@sha256:dc314b4fe30fc1315742512891357bffed4d1b62ffcb46258b1e0761c737b446 \ | |
| build_tools/cmake/build_e2e_test_artifacts.sh \ | |
| "${BUILD_E2E_TEST_ARTIFACTS_DIR}" | |
| - name: "Uploading e2e test artifacts" | |
| id: upload | |
| if: ${{ !cancelled() }} | |
| env: | |
| E2E_TEST_ARTIFACTS_DIR: ${{ steps.build.outputs.e2e-test-artifacts-dir }} | |
| E2E_TEST_ARTIFACTS_GCS_ARTIFACT_DIR: ${{ env.GCS_DIR }}/e2e-test-artifacts | |
| E2E_TEST_ARTIFACTS_BUILD_LOG: ${{ steps.build.outputs.e2e-test-artifacts-build-log }} | |
| E2E_TEST_ARTIFACTS_BUILD_LOG_GCS_ARTIFACT: ${{ env.GCS_DIR }}/e2e-test-artifacts/ninja_log | |
| run: | | |
| # TODO(#13677): MLIRs of the versioned MLIR source models are fetched | |
| # from GCS and they won't change. We can upload links instead of files | |
| # to save space. | |
| # Uploads all artifacts except for the `*.tflite`, which are the | |
| # source models we imported into MLIR, and empty directories (GCS | |
| # doesn't accept them). | |
| # Not archiving the directory to allow fetching each file as needed | |
| # separately. `find` depth is limited at 1 to exclude the directory | |
| # itself and list all files and subdirectories at the first level. Let | |
| # gcloud cli handle the recursive file traversal. | |
| find "${E2E_TEST_ARTIFACTS_DIR}" -mindepth 1 -maxdepth 1 ! -name "*.tflite" ! -empty \ | |
| | gcloud storage cp --read-paths-from-stdin -r \ | |
| "${E2E_TEST_ARTIFACTS_GCS_ARTIFACT_DIR}" | |
| gcloud storage cp "${E2E_TEST_ARTIFACTS_BUILD_LOG}" \ | |
| "${E2E_TEST_ARTIFACTS_BUILD_LOG_GCS_ARTIFACT}" | |
| echo "e2e-test-artifacts-gcs-artifact-dir=${E2E_TEST_ARTIFACTS_GCS_ARTIFACT_DIR}" >> "${GITHUB_OUTPUT}" | |
| echo "e2e-test-artifacts-build-log-gcs-artifact=${E2E_TEST_ARTIFACTS_BUILD_LOG_GCS_ARTIFACT}" >> "${GITHUB_OUTPUT}" |