| # Copyright 2024 The IREE Authors | |
| # | |
| # Licensed under the Apache License v2.0 with LLVM Exceptions. | |
| # See https://llvm.org/LICENSE.txt for license information. | |
| # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |
| name: Benchmark | |
| on: | |
| workflow_dispatch: | |
| pull_request: | |
| push: | |
| branches: | |
| - main | |
| concurrency: | |
| # A PR number if a pull request and otherwise the commit hash. This cancels | |
| # queued and in-progress runs for the same PR (presubmit) or commit | |
| # (postsubmit). The workflow name is prepended to avoid conflicts between | |
| # different workflows. | |
| group: ${{ github.workflow }}-${{ github.event.number || github.sha }} | |
| cancel-in-progress: true | |
| env: | |
| # This needs to be in env instead of the outputs of setup because it contains | |
| # the run attempt and we want that to be the current attempt, not whatever | |
| # attempt the setup step last ran in. | |
| GCS_DIR: gs://iree-github-actions-${{ github.event_name == 'pull_request' && 'presubmit' || 'postsubmit' }}-artifacts/${{ github.run_id }}/${{ github.run_attempt }} | |
| jobs: | |
| setup: | |
| uses: ./.github/workflows/setup.yml | |
| build_for_benchmarks: | |
| needs: setup | |
| if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'build_for_benchmarks') && needs.setup.outputs.benchmark-presets != '' | |
| uses: ./.github/workflows/build_all.yml | |
| with: | |
| runner-group: ${{ needs.setup.outputs.runner-group }} | |
| runner-env: ${{ needs.setup.outputs.runner-env }} | |
| write-caches: ${{ needs.setup.outputs.write-caches }} | |
| run-tests: false | |
| build_benchmark_tools: | |
| needs: [setup, build_for_benchmarks] | |
| if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'build_benchmark_tools') && needs.setup.outputs.benchmark-presets != '' | |
| uses: ./.github/workflows/build_benchmark_tools.yml | |
| with: | |
| runner-group: ${{ needs.setup.outputs.runner-group }} | |
| runner-env: ${{ needs.setup.outputs.runner-env }} | |
| install-dir: ${{ needs.build_for_benchmarks.outputs.install-dir }} | |
| install-dir-archive: ${{ needs.build_for_benchmarks.outputs.install-dir-archive }} | |
| install-dir-gcs-artifact: ${{ needs.build_for_benchmarks.outputs.install-dir-gcs-artifact }} | |
| build_e2e_test_artifacts: | |
| needs: [setup, build_for_benchmarks] | |
| if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'build_e2e_test_artifacts') && needs.setup.outputs.benchmark-presets != '' | |
| uses: ./.github/workflows/build_e2e_test_artifacts.yml | |
| with: | |
| runner-group: ${{ needs.setup.outputs.runner-group }} | |
| runner-env: ${{ needs.setup.outputs.runner-env }} | |
| install-dir: ${{ needs.build_for_benchmarks.outputs.install-dir }} | |
| install-dir-archive: ${{ needs.build_for_benchmarks.outputs.install-dir-archive }} | |
| install-dir-gcs-artifact: ${{ needs.build_for_benchmarks.outputs.install-dir-gcs-artifact }} | |
| benchmark-presets: ${{ needs.setup.outputs.benchmark-presets }} | |
| shard-count: "c2-standard-60=2,default=1" | |
| test_benchmark_suites: | |
| needs: [setup, build_for_benchmarks, build_e2e_test_artifacts] | |
| if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'test_benchmark_suites') && needs.setup.outputs.benchmark-presets != '' | |
| strategy: | |
| matrix: | |
| target: | |
| - platform: linux | |
| arch: riscv_64 | |
| docker_image: "gcr.io/iree-oss/riscv@sha256:62e87bad3405d691ddba6f9be0ef44eeb60461a467c8d86f0842c81a1f97da79" | |
| run_scripts: "./build_tools/cmake/build_riscv.sh && ./build_tools/cmake/test_riscv.sh" | |
| - platform: linux | |
| arch: riscv_32 | |
| docker_image: "gcr.io/iree-oss/riscv@sha256:62e87bad3405d691ddba6f9be0ef44eeb60461a467c8d86f0842c81a1f97da79" | |
| run_scripts: "./build_tools/cmake/build_riscv.sh && ./build_tools/cmake/test_riscv.sh" | |
| - platform: linux | |
| arch: x86_64 | |
| docker_image: "gcr.io/iree-oss/base@sha256:dc314b4fe30fc1315742512891357bffed4d1b62ffcb46258b1e0761c737b446" | |
| run_scripts: "./build_tools/cmake/test_benchmark_suites_on_linux.sh" | |
| # Requires Intel CascadeLake CPU. | |
| host_machine: c2s601t | |
| runs-on: | |
| - self-hosted # must come first | |
| - runner-group=${{ needs.setup.outputs.runner-group }} | |
| - environment=${{ needs.setup.outputs.runner-env }} | |
| - ${{ matrix.target.host_machine || 'cpu' }} # Default to generic x86_64 VM. | |
| env: | |
| PLATFORM: ${{ matrix.target.platform }} | |
| ARCH: ${{ matrix.target.arch }} | |
| DOCKER_IMAGE: ${{ matrix.target.docker_image }} | |
| RUN_SCRIPTS: ${{ matrix.target.run_scripts }} | |
| INSTALL_DIR: ${{ needs.build_for_benchmarks.outputs.install-dir }} | |
| INSTALL_DIR_ARCHIVE: ${{ needs.build_for_benchmarks.outputs.install-dir-archive }} | |
| INSTALL_DIR_GCS_ARTIFACT: ${{ needs.build_for_benchmarks.outputs.install-dir-gcs-artifact }} | |
| TARGET_BUILD_DIR: build-${{ matrix.target.platform }}-${{ matrix.target.arch }} | |
| E2E_TEST_ARTIFACTS_DIR: ${{ needs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-dir }} | |
| E2E_TEST_ARTIFACTS_GCS_ARTIFACT_DIR: ${{ needs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-gcs-artifact-dir }} | |
| steps: | |
| - name: "Checking out repository" | |
| uses: actions/checkout@v4.1.7 | |
| - name: "Checking out runtime submodules" | |
| run: ./build_tools/scripts/git/update_runtime_submodules.sh | |
| - name: "Downloading install dir archive" | |
| run: gcloud storage cp "${INSTALL_DIR_GCS_ARTIFACT}" "${INSTALL_DIR_ARCHIVE}" | |
| - name: "Extracting install directory" | |
| run: tar -xf "${INSTALL_DIR_ARCHIVE}" | |
| # TODO(#11136): Only download the needed artifacts instead of everything. | |
| - name: "Downloading e2e test artifacts" | |
| run: | | |
| mkdir -p ${E2E_TEST_ARTIFACTS_DIR} | |
| gcloud storage cp -r "${E2E_TEST_ARTIFACTS_GCS_ARTIFACT_DIR}/*" "${E2E_TEST_ARTIFACTS_DIR}" | |
| - name: "Build iree-run-module and test benchmark suite modules" | |
| run: | | |
| ./build_tools/github_actions/docker_run.sh \ | |
| --env "IREE_TARGET_PLATFORM=${PLATFORM}" \ | |
| --env "IREE_TARGET_ARCH=${ARCH}" \ | |
| --env "IREE_TARGET_BUILD_DIR=${TARGET_BUILD_DIR}" \ | |
| --env "BUILD_PRESET=benchmark-suite-test" \ | |
| --env "IREE_HOST_BIN_DIR=${INSTALL_DIR}/bin" \ | |
| --env "E2E_TEST_ARTIFACTS_DIR=${E2E_TEST_ARTIFACTS_DIR}" \ | |
| "${DOCKER_IMAGE}" \ | |
| bash -euo pipefail -c \ | |
| "${RUN_SCRIPTS}" | |
| compilation_benchmarks: | |
| needs: [setup, build_e2e_test_artifacts] | |
| if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'compilation_benchmarks') && needs.setup.outputs.benchmark-presets != '' | |
| uses: ./.github/workflows/benchmark_compilation.yml | |
| with: | |
| runner-group: ${{ needs.setup.outputs.runner-group }} | |
| runner-env: ${{ needs.setup.outputs.runner-env }} | |
| e2e-test-artifacts-dir: ${{ needs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-dir }} | |
| e2e-test-artifacts-gcs-artifact-dir: ${{ needs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-gcs-artifact-dir }} | |
| e2e-test-artifacts-build-log: ${{ needs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-build-log }} | |
| e2e-test-artifacts-build-log-gcs-artifact: ${{ needs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-build-log-gcs-artifact }} | |
| execution_benchmarks: | |
| needs: [setup, build_benchmark_tools, build_e2e_test_artifacts] | |
| if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'execution_benchmarks') && needs.setup.outputs.benchmark-presets != '' | |
| uses: ./.github/workflows/benchmark_execution.yml | |
| with: | |
| runner-group: ${{ needs.setup.outputs.runner-group }} | |
| runner-env: ${{ needs.setup.outputs.runner-env }} | |
| e2e-test-artifacts-dir: ${{ needs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-dir }} | |
| e2e-test-artifacts-gcs-artifact-dir: ${{ needs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-gcs-artifact-dir }} | |
| benchmark-tools-gcs-artifact-dir: ${{ needs.build_benchmark_tools.outputs.benchmark-tools-gcs-artifact-dir }} | |
| process_benchmark_results: | |
| needs: [setup, compilation_benchmarks, execution_benchmarks] | |
| if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'process_benchmark_results') && needs.setup.outputs.benchmark-presets != '' | |
| runs-on: | |
| - self-hosted # must come first | |
| - runner-group=${{ needs.setup.outputs.runner-group }} | |
| - environment=${{ needs.setup.outputs.runner-env }} | |
| - cpu | |
| - os-family=Linux | |
| env: | |
| COMPILE_STATS_RESULTS: ${{ needs.compilation_benchmarks.outputs.compile-stats-results }} | |
| COMPILE_STATS_RESULTS_GCS_ARTIFACT: ${{ needs.compilation_benchmarks.outputs.compile-stats-results-gcs-artifact }} | |
| # Empty if no execution benchmark runs. | |
| EXECUTION_BENCHMARK_RESULTS_DIR: ${{ needs.execution_benchmarks.outputs.benchmark-results-dir }} | |
| # Empty if no execution benchmark runs. | |
| EXECUTION_BENCHMARK_RESULTS_GCS_ARTIFACT_DIR: ${{ needs.execution_benchmarks.outputs.benchmark-results-gcs-artifact-dir }} | |
| steps: | |
| - name: "Checking out repository" | |
| uses: actions/checkout@v4.1.7 | |
| with: | |
| # We need the full history (and main branch) to generate the report. | |
| fetch-depth: 0 | |
| - name: Downloading compilation benchmark results | |
| run: | | |
| gcloud storage cp \ | |
| "${COMPILE_STATS_RESULTS_GCS_ARTIFACT}" \ | |
| "${COMPILE_STATS_RESULTS}" | |
| - name: Downloading execution benchmark results | |
| id: download-execution-results | |
| # Skip the download if there is no execution benchmark results (e.g. no | |
| # benchmark matches the preset/filter). In such case, no benchmark job | |
| # is run in benchmark_execution.yml and the output variables are empty. | |
| if: env.EXECUTION_BENCHMARK_RESULTS_GCS_ARTIFACT_DIR != '' | |
| run: | | |
| gcloud storage cp -r \ | |
| "${EXECUTION_BENCHMARK_RESULTS_GCS_ARTIFACT_DIR}/benchmark-results-*.json" \ | |
| "${EXECUTION_BENCHMARK_RESULTS_DIR}" | |
| echo "execution-benchmark-results-pattern=${EXECUTION_BENCHMARK_RESULTS_DIR}/benchmark-results-*.json" >> "${GITHUB_OUTPUT}" | |
| - name: Generating comment | |
| if: fromJson(needs.setup.outputs.is-pr) | |
| id: generate-comment | |
| env: | |
| # Wildcard pattern to match all execution benchmark results. Empty if | |
| # execution_benchmarks is skipped, which results in no match. | |
| EXECUTION_BENCHMARK_RESULTS_PATTERN: ${{ steps.download-execution-results.outputs.execution-benchmark-results-pattern }} | |
| IREE_BUILD_URL: https://github.com/iree-org/iree/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }} | |
| PR_NUMBER: ${{ github.event.pull_request.number }} | |
| BENCHMARK_COMMENT_ARTIFACT: benchmark-comment.json | |
| run: | | |
| build_tools/github_actions/docker_run.sh \ | |
| gcr.io/iree-oss/benchmark-report@sha256:7498c6f32f63f13faf085463cc38656d4297519c824e63e1c99c8c258147f6ff \ | |
| ./build_tools/benchmarks/generate_benchmark_comment.py \ | |
| --verbose \ | |
| --pr_number="${PR_NUMBER}" \ | |
| --pr_committish="${GITHUB_SHA}" \ | |
| --pr_base_branch="origin/${GITHUB_BASE_REF}" \ | |
| --comment_type="benchmark-summary" \ | |
| --build_url="${IREE_BUILD_URL}" \ | |
| --benchmark_files="${EXECUTION_BENCHMARK_RESULTS_PATTERN}" \ | |
| --compile_stats_files="${COMPILE_STATS_RESULTS}" \ | |
| --output="${BENCHMARK_COMMENT_ARTIFACT}" | |
| echo "benchmark-comment-artifact=${BENCHMARK_COMMENT_ARTIFACT}" >> "${GITHUB_OUTPUT}" | |
| - name: Uploading comment artifact | |
| # Due to security reasons, instead of posting the comment to PR, we only | |
| # upload the comment data in presubmit workflow and trigger the posting | |
| # workflow on the main branch. See post_benchmark_comment.yaml | |
| if: fromJson(needs.setup.outputs.is-pr) | |
| env: | |
| BENCHMARK_COMMENT_ARTIFACT: ${{ steps.generate-comment.outputs.benchmark-comment-artifact }} | |
| BENCHMARK_COMMENT_GCS_ARTIFACT: ${{ env.GCS_DIR }}/${{ steps.generate-comment.outputs.benchmark-comment-artifact }} | |
| run: | | |
| gcloud storage cp \ | |
| "${BENCHMARK_COMMENT_ARTIFACT}" \ | |
| "${BENCHMARK_COMMENT_GCS_ARTIFACT}" | |
| - name: Uploading results to dashboard | |
| if: github.ref_name == 'main' | |
| env: | |
| EXECUTION_BENCHMARK_RESULTS_PATTERN: ${{ steps.download-execution-results.outputs.execution-benchmark-results-pattern }} | |
| IREE_DASHBOARD_API_TOKEN: ${{ secrets.IREE_DASHBOARD_API_TOKEN }} | |
| run: | | |
| build_tools/github_actions/docker_run.sh \ | |
| --env "IREE_DASHBOARD_API_TOKEN=${IREE_DASHBOARD_API_TOKEN}" \ | |
| gcr.io/iree-oss/benchmark-report@sha256:7498c6f32f63f13faf085463cc38656d4297519c824e63e1c99c8c258147f6ff \ | |
| ./build_tools/benchmarks/upload_benchmarks_to_dashboard.py \ | |
| --verbose \ | |
| --benchmark_files="${EXECUTION_BENCHMARK_RESULTS_PATTERN}" \ | |
| --compile_stats_files="${COMPILE_STATS_RESULTS}" | |
| ############################################################################## | |
| # Depends on all the other jobs to provide a single anchor that indicates the | |
| # final status. Status reporting will become more sophisticated in the future | |
| # and we can hopefully avoid the need to explicitly list every single job... | |
| benchmark_summary: | |
| # Even if you have an explicit if condition, you still need to override | |
| # GitHub's default behavior of not running if any dependencies failed. | |
| if: always() | |
| runs-on: ubuntu-20.04 | |
| needs: | |
| - setup | |
| - build_for_benchmarks | |
| # Benchmark pipeline | |
| - build_benchmark_tools | |
| - build_e2e_test_artifacts | |
| - test_benchmark_suites | |
| - compilation_benchmarks | |
| - execution_benchmarks | |
| - process_benchmark_results | |
| steps: | |
| - name: "Checking out repository" | |
| uses: actions/checkout@v4.1.7 | |
| - name: Getting failed jobs | |
| id: failed_jobs | |
| run: | | |
| echo '${{ toJson(needs) }}' | |
| FAILED_JOBS="$(echo '${{ toJson(needs) }}' \ | |
| | jq --raw-output \ | |
| 'map_values(select(.result!="success" and .result!="skipped")) | keys | join(",")' \ | |
| )" | |
| echo "failed-jobs=${FAILED_JOBS}" >> $GITHUB_OUTPUT | |
| if [[ "${FAILED_JOBS}" != "" ]]; then | |
| echo "The following jobs failed: ${FAILED_JOBS}" | |
| exit 1 | |
| fi | |
| - name: Show useful artifact links | |
| if: always() | |
| env: | |
| # If the job of an artifact is skipped or failed, we show "NOT_PRESENT". | |
| INSTALL_DIR_GCS_ARTIFACT: ${{ needs.build_for_benchmarks.outputs.install-dir-gcs-artifact || 'NOT_PRESENT' }} | |
| E2E_TEST_ARTIFACTS_GCS_ARTIFACT_DIR: ${{ needs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-gcs-artifact-dir || 'NOT_PRESENT' }} | |
| BENCHMARK_TOOLS_GCS_ARTIFACT_DIR: ${{ needs.build_benchmark_tools.outputs.benchmark-tools-gcs-artifact-dir || 'NOT_PRESENT' }} | |
| EXECUTION_BENCHMARK_RESULTS_GCS_ARTIFACT_DIR: ${{ needs.execution_benchmarks.outputs.benchmark-results-gcs-artifact-dir || 'NOT_PRESENT' }} | |
| COMPILATION_BENCHMARK_RESULTS_GCS_ARTIFACT: ${{ needs.compilation_benchmarks.outputs.compile-stats-results-gcs-artifact || 'NOT_PRESENT' }} | |
| run: | | |
| envsubst < ./.github/workflows/ARTIFACT_SUMMARY_TEMPLATE.md >> "${GITHUB_STEP_SUMMARY}" | |
| - name: Posting to Discord | |
| uses: sarisia/actions-status-discord@v1.14.3 | |
| if: failure() && github.ref_name == 'main' | |
| with: | |
| webhook: ${{ secrets.DISCORD_WEBHOOK }} | |
| description: "The following jobs failed: ${{ steps.failed_jobs.outputs.failed-jobs }}" | |
| url: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}" |