Move benchmark config generation to build_e2e_test_artifacts (#13291)
When working on https://github.com/openxla/iree/pull/13273, I noticed
that benchmark configs and modules are uploaded in 3 different jobs.
These artifacts can be uploaded to different GCS dirs when some of them
are re-run due to failures (because re-run jobs will create [GCS_DIR
with new run
attempts](https://github.com/openxla/iree/blob/main/.github/workflows/benchmark_compilation.yml#L44-L50)).
As a result, users might not be able to download all benchmark artifacts
from a single GCS dir URL, which can be confusing.
This PR changes the workflow to generate all benchmark modules and
configs in `build_e2e_test_artifacts` to avoid such issue. All files are
uploaded to `${GCS_URL}/e2e-test-artifacts`, a single path to download
all benchmark artifacts.
Besides the reason above, I think during the CMake build we should
generate the benchmark configs under
`${IREE_BUILD_DIR}/e2e_test_artifacts` instead of calling
`export_benchmark_config.py` separately. There are some questions about
how to pass the benchmark presets/filters through CMake configuration,
so I decided to defer that as the next step.diff --git a/.github/workflows/benchmark_compilation.yml b/.github/workflows/benchmark_compilation.yml
index aaa2e06..616f945 100644
--- a/.github/workflows/benchmark_compilation.yml
+++ b/.github/workflows/benchmark_compilation.yml
@@ -68,54 +68,43 @@
steps:
- name: "Checking out repository"
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
- - name: "Exporting configs"
- id: "export"
- env:
- COMPILATION_CONFIG: "compilation-config.json"
- run: |
- ./build_tools/benchmarks/export_benchmark_config.py \
- compilation \
- --output="${COMPILATION_CONFIG}"
- echo "compilation-config=${COMPILATION_CONFIG}" >> "${GITHUB_OUTPUT}"
- name: "Downloading assets"
id: "download-assets"
env:
- COMPILATION_CONFIG: ${{ steps.export.outputs.compilation-config }}
+ BENCHMARK_CONFIG: ${{ env.E2E_TEST_ARTIFACTS_DIR }}/compilation-benchmark-config.json
+ BENCHMARK_CONFIG_GCS_ARTIFACT: ${{ env.E2E_TEST_ARTIFACTS_GCS_ARTIFACT_DIR }}/compilation-benchmark-config.json
run: |
+ mkdir -p "${E2E_TEST_ARTIFACTS_DIR}"
+ gcloud storage cp "${BENCHMARK_CONFIG_GCS_ARTIFACT}" "${BENCHMARK_CONFIG}"
gcloud storage cp \
"${E2E_TEST_ARTIFACTS_BUILD_LOG_GCS_ARTIFACT}" \
"${E2E_TEST_ARTIFACTS_BUILD_LOG}"
- mkdir -p "${E2E_TEST_ARTIFACTS_DIR}"
jq -r \
--arg GCS_ARTIFACT_DIR "${E2E_TEST_ARTIFACTS_GCS_ARTIFACT_DIR}" \
'.module_dir_paths | map("\($GCS_ARTIFACT_DIR)/\(.)") | join("\n")' \
- "${COMPILATION_CONFIG}" | \
+ "${BENCHMARK_CONFIG}" | \
gcloud storage cp -r --read-paths-from-stdin \
"${E2E_TEST_ARTIFACTS_DIR}"
+ echo "benchmark-config=${BENCHMARK_CONFIG}" >> "${GITHUB_OUTPUT}"
- name: "Collecting compilation statistics"
id: collect
env:
- COMPILATION_CONFIG: ${{ steps.export.outputs.compilation-config }}
- GENERATION_CONFIG: generation-config.json
+ BENCHMARK_CONFIG: ${{ steps.download-assets.outputs.benchmark-config }}
COMPILE_STATS_RESULTS: benchmark-results/compile-stats-results.json
run: |
mkdir -p benchmark-results
./build_tools/benchmarks/collect_compilation_statistics.py alpha \
--e2e_test_artifacts_dir="${E2E_TEST_ARTIFACTS_DIR}" \
--build_log="${E2E_TEST_ARTIFACTS_BUILD_LOG}" \
- --compilation_benchmark_config="${COMPILATION_CONFIG}" \
+ --compilation_benchmark_config="${BENCHMARK_CONFIG}" \
--output="${COMPILE_STATS_RESULTS}"
echo "compile-stats-results=${COMPILE_STATS_RESULTS}" >> "${GITHUB_OUTPUT}"
- name: "Uploading benchmark results"
id: upload
env:
- COMPILATION_CONFIG: ${{ steps.export.outputs.compilation-config }}
COMPILE_STATS_RESULTS: ${{ steps.collect.outputs.compile-stats-results }}
COMPILE_STATS_RESULTS_GCS_ARTIFACT: ${{ env.GCS_DIR }}/${{ steps.collect.outputs.compile-stats-results }}
run: |
- # Upload files with two commands since they go into different GCS
- # directories.
- gcloud storage cp "${COMPILATION_CONFIG}" "${GCS_DIR}"
gcloud storage cp \
"${COMPILE_STATS_RESULTS}" \
"${COMPILE_STATS_RESULTS_GCS_ARTIFACT}"
diff --git a/.github/workflows/benchmark_execution.yml b/.github/workflows/benchmark_execution.yml
index 7d60beb..2ee1645 100644
--- a/.github/workflows/benchmark_execution.yml
+++ b/.github/workflows/benchmark_execution.yml
@@ -28,9 +28,6 @@
benchmark-tools-gcs-artifact-dir:
required: true
type: string
- benchmark-presets:
- required: true
- type: string
outputs:
benchmark-results-dir:
description: |
@@ -50,9 +47,11 @@
# last ran in. It therefore can't be passed in via inputs because the env
# context isn't available there.
GCS_DIR: gs://iree-github-actions-${{ github.event_name == 'pull_request' && 'presubmit' || 'postsubmit' }}-artifacts/${{ github.run_id }}/${{ github.run_attempt }}
+ BENCHMARK_CONFIG: ${{ inputs.e2e-test-artifacts-dir }}/execution-benchmark-config.json
+ BENCHMARK_CONFIG_GCS_ARTIFACT: ${{ inputs.e2e-test-artifacts-gcs-artifact-dir }}/execution-benchmark-config.json
jobs:
- export_benchmark_config:
+ generate_matrix:
runs-on:
- self-hosted # must come first
- runner-group=${{ inputs.runner-group }}
@@ -60,51 +59,31 @@
- cpu
- os-family=Linux
outputs:
- benchmark-matrix: ${{ steps.export.outputs.benchmark-matrix }}
- benchmark-config: ${{ steps.export.outputs.benchmark-config }}
- benchmark-config-gcs-artifact: ${{ steps.upload.outputs.benchmark-config-gcs-artifact }}
+ benchmark-matrix: ${{ steps.generate.outputs.benchmark-matrix }}
steps:
- name: "Checking out repository"
uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
- name: "Checking out runtime submodules"
run: ./build_tools/scripts/git/update_runtime_submodules.sh
- - name: "Exporting benchmark run config"
- id: export
- env:
- BENCHMARK_CONFIG: "benchmark-config.json"
- BENCHMARK_PRESETS: ${{ inputs.benchmark-presets }}
+ - name: "Generating benchmark matrix"
+ id: generate
run: |
- ./build_tools/benchmarks/export_benchmark_config.py \
- execution \
- --benchmark_presets="${BENCHMARK_PRESETS}" \
- --output="${BENCHMARK_CONFIG}"
- echo "benchmark-config=${BENCHMARK_CONFIG}" >> "${GITHUB_OUTPUT}"
+ gcloud storage cp "${BENCHMARK_CONFIG_GCS_ARTIFACT}" "${BENCHMARK_CONFIG}"
echo benchmark-matrix=$(jq \
'to_entries | map({"device_name": .key, "host_environment": .value.host_environment})' \
"${BENCHMARK_CONFIG}") \
>> "${GITHUB_OUTPUT}"
- - name: "Uploading benchmark config"
- id: upload
- if: steps.export.outputs.benchmark-matrix != '[]'
- env:
- BENCHMARK_CONFIG: ${{ steps.export.outputs.benchmark-config }}
- BENCHMARK_CONFIG_GCS_ARTIFACT: ${{ env.GCS_DIR }}/${{ steps.export.outputs.benchmark-config }}
- run: |
- gcloud storage cp \
- "${BENCHMARK_CONFIG}" \
- "${BENCHMARK_CONFIG_GCS_ARTIFACT}"
- echo "benchmark-config-gcs-artifact=${BENCHMARK_CONFIG_GCS_ARTIFACT}" >> "${GITHUB_OUTPUT}"
run_benchmarks:
- needs: [export_benchmark_config]
- if: needs.export_benchmark_config.outputs.benchmark-matrix != '[]'
+ needs: [generate_matrix]
+ if: needs.generate_matrix.outputs.benchmark-matrix != '[]'
strategy:
- # Matrix is dynamically generated by the job export_benchmark_config. So
- # we only runs the benchmarks specified in inputs.benchmark-presets.
+ # Matrix is dynamically generated by the job generate_matrix. So we only
+ # runs the benchmarks specified in ${BENCHMARK_CONFIG}.
# All tasks in matrix are seen as a single job in Github CI and the job
# can output a single set of values.
matrix:
- benchmark: ${{ fromJSON(needs.export_benchmark_config.outputs.benchmark-matrix) }}
+ benchmark: ${{ fromJSON(needs.generate_matrix.outputs.benchmark-matrix) }}
runs-on:
- self-hosted # must come first
- runner-group=${{ inputs.runner-group }}
@@ -138,9 +117,6 @@
echo "benchmark-tools-archive=${BENCHMARK_TOOLS_ARCHIVE}" >> "${GITHUB_OUTPUT}"
- name: "Downloading benchmark assets"
id: download-assets
- env:
- BENCHMARK_CONFIG: ${{ needs.export_benchmark_config.outputs.benchmark-config }}
- BENCHMARK_CONFIG_GCS_ARTIFACT: ${{ needs.export_benchmark_config.outputs.benchmark-config-gcs-artifact }}
run: |
gcloud storage cp "${BENCHMARK_CONFIG_GCS_ARTIFACT}" "${BENCHMARK_CONFIG}"
mkdir -p "${E2E_TEST_ARTIFACTS_DIR}"
diff --git a/.github/workflows/benchmark_long.yml b/.github/workflows/benchmark_long.yml
index 9a713e2..5c39384 100644
--- a/.github/workflows/benchmark_long.yml
+++ b/.github/workflows/benchmark_long.yml
@@ -55,6 +55,7 @@
build-dir: ${{ needs.build_all.outputs.build-dir }}
build-dir-archive: ${{ needs.build_all.outputs.build-dir-archive }}
build-dir-gcs-artifact: ${{ needs.build_all.outputs.build-dir-gcs-artifact }}
+ benchmark-presets: cuda-long
compilation_benchmarks:
needs: [setup, build_e2e_test_artifacts]
@@ -80,7 +81,6 @@
e2e-test-artifacts-dir: ${{ needs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-dir }}
e2e-test-artifacts-gcs-artifact-dir: ${{ needs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-gcs-artifact-dir }}
benchmark-tools-gcs-artifact-dir: ${{ needs.build_benchmark_tools.outputs.benchmark-tools-gcs-artifact-dir }}
- benchmark-presets: cuda-long
summary:
# Even if you have an explicit if condition, you still need to override
diff --git a/.github/workflows/build_e2e_test_artifacts.yml b/.github/workflows/build_e2e_test_artifacts.yml
index ec78e07..7fbe6c7 100644
--- a/.github/workflows/build_e2e_test_artifacts.yml
+++ b/.github/workflows/build_e2e_test_artifacts.yml
@@ -28,6 +28,9 @@
build-dir-gcs-artifact:
required: true
type: string
+ benchmark-presets:
+ required: true
+ type: string
outputs:
e2e-test-artifacts-dir:
description: |
@@ -92,6 +95,21 @@
"${BUILD_E2E_TEST_ARTIFACTS_DIR}"
echo "e2e-test-artifacts-dir=${BUILD_E2E_TEST_ARTIFACTS_DIR}/e2e_test_artifacts" >> "${GITHUB_OUTPUT}"
echo "e2e-test-artifacts-build-log=${BUILD_E2E_TEST_ARTIFACTS_DIR}/.ninja_log" >> "${GITHUB_OUTPUT}"
+ - name: "Exporting benchmark configs"
+ id: export
+ if: inputs.benchmark-presets != ''
+ env:
+ COMPILATION_BENCHMARK_CONFIG: ${{ steps.build.outputs.e2e-test-artifacts-dir }}/compilation-benchmark-config.json
+ EXECUTION_BENCHMARK_CONFIG: ${{ steps.build.outputs.e2e-test-artifacts-dir }}/execution-benchmark-config.json
+ BENCHMARK_PRESETS: ${{ inputs.benchmark-presets }}
+ run: |
+ ./build_tools/benchmarks/export_benchmark_config.py \
+ compilation \
+ --output="${COMPILATION_BENCHMARK_CONFIG}"
+ ./build_tools/benchmarks/export_benchmark_config.py \
+ execution \
+ --benchmark_presets="${BENCHMARK_PRESETS}" \
+ --output="${EXECUTION_BENCHMARK_CONFIG}"
- name: "Uploading e2e test artifacts"
id: upload
env:
@@ -105,7 +123,7 @@
# Not archiving the directory to allow fetching each file as needed
# separately.
find "${E2E_TEST_ARTIFACTS_DIR}" -maxdepth 1 \
- -name "iree_*" -o -name "model_*.mlir" | \
+ -name "iree_*" -o -name "model_*.mlir" -o -name "*.json" | \
gcloud storage cp --read-paths-from-stdin -r \
"${E2E_TEST_ARTIFACTS_GCS_ARTIFACT_DIR}"
gcloud storage cp "${E2E_TEST_ARTIFACTS_BUILD_LOG}" \
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 2ff017a..feeb756 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -720,6 +720,7 @@
build-dir: ${{ needs.build_all.outputs.build-dir }}
build-dir-archive: ${{ needs.build_all.outputs.build-dir-archive }}
build-dir-gcs-artifact: ${{ needs.build_all.outputs.build-dir-gcs-artifact }}
+ benchmark-presets: ${{ needs.setup.outputs.benchmark-presets }}
compilation_benchmarks:
needs: [setup, build_e2e_test_artifacts]
@@ -745,7 +746,6 @@
e2e-test-artifacts-dir: ${{ needs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-dir }}
e2e-test-artifacts-gcs-artifact-dir: ${{ needs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-gcs-artifact-dir }}
benchmark-tools-gcs-artifact-dir: ${{ needs.build_benchmark_tools.outputs.benchmark-tools-gcs-artifact-dir }}
- benchmark-presets: ${{ needs.setup.outputs.benchmark-presets }}
process_benchmark_results:
needs: [setup, compilation_benchmarks, execution_benchmarks]
diff --git a/docs/developers/developing_iree/benchmark_suites.md b/docs/developers/developing_iree/benchmark_suites.md
index ddbea94..b17721b 100644
--- a/docs/developers/developing_iree/benchmark_suites.md
+++ b/docs/developers/developing_iree/benchmark_suites.md
@@ -158,18 +158,14 @@
# Download all artifacts
mkdir "${E2E_TEST_ARTIFACTS_DIR?}"
gcloud storage cp -r "${GCS_URL?}/e2e-test-artifacts" "${E2E_TEST_ARTIFACTS_DIR?}"
-
-# Download benchmark configs
-gcloud storage cp "${GCS_URL?}/benchmark-config.json" "${E2E_TEST_ARTIFACTS_DIR?}/exec_config.json"
-gcloud storage cp "${GCS_URL?}/compilation-config.json" "${E2E_TEST_ARTIFACTS_DIR?}/comp_config.json"
```
Run the helper tool to dump benchmark commands from benchmark configs:
```sh
build_tools/benchmarks/benchmark_helper.py dump-cmds \
- --execution_benchmark_config="${E2E_TEST_ARTIFACTS_DIR?}/exec_config.json" \
- --compilation_benchmark_config="${E2E_TEST_ARTIFACTS_DIR?}/comp_config.json" \
+ --execution_benchmark_config="${E2E_TEST_ARTIFACTS_DIR?}/execution-benchmark-config.json" \
+ --compilation_benchmark_config="${E2E_TEST_ARTIFACTS_DIR?}/compilation-benchmark-config.json" \
--e2e_test_artifacts_dir="${E2E_TEST_ARTIFACTS_DIR?}" \
--benchmark_id="<benchmark_id>"
```
@@ -239,10 +235,14 @@
```sh
# Execution benchmark config:
-gcloud storage cp "${GCS_URL?}/benchmark-config.json" "${E2E_TEST_ARTIFACTS_DIR?}/exec_config.json"
+gcloud storage cp \
+ "${GCS_URL?}/e2e-test-artifacts/execution-benchmark-config.json" \
+ "${E2E_TEST_ARTIFACTS_DIR?}/exec_config.json"
# Compilation benchmark config:
-gcloud storage cp "${GCS_URL?}/compilation-config.json" "${E2E_TEST_ARTIFACTS_DIR?}/comp_config.json"
+gcloud storage cp \
+ "${GCS_URL?}/e2e-test-artifacts/compilation-benchmark-config.json" \
+ "${E2E_TEST_ARTIFACTS_DIR?}/comp_config.json"
```
Benchmark raw results and traces can be downloaded at:
