Only build required benchmark targets in CI (#13665)
Make `build_e2e_test_artifacts.yml` only build the required benchmark
targets depending on the benchmark presets. This avoids building large
benchmark suites in presubmit and postsubmit, which takes lots of time
and GCS space.
diff --git a/build_tools/benchmarks/export_benchmark_config.py b/build_tools/benchmarks/export_benchmark_config.py
index 0deedc7..9808ed7 100755
--- a/build_tools/benchmarks/export_benchmark_config.py
+++ b/build_tools/benchmarks/export_benchmark_config.py
@@ -68,11 +68,6 @@
(config.target_device_spec.architecture.type == common_definitions.
ArchitectureType.GPU and config.target_device_spec.host_environment.
platform == "android"),
- # Not a preset for execution benchmarks.
- "comp-stats":
- lambda _: False,
- "comp-stats-long":
- lambda _: False,
}
COMPILATION_BENCHMARK_PRESET_MATCHERS: Dict[str, PresetMatcher] = {
diff --git a/build_tools/cmake/build_e2e_test_artifacts.sh b/build_tools/cmake/build_e2e_test_artifacts.sh
index 8ae8bf1..c4d8348 100755
--- a/build_tools/cmake/build_e2e_test_artifacts.sh
+++ b/build_tools/cmake/build_e2e_test_artifacts.sh
@@ -13,7 +13,11 @@
# migrated.
#
# The required IREE_HOST_BIN_DIR environment variable indicates the location
-# of the precompiled IREE binaries.
+# of the precompiled IREE binaries. The IREE_BENCHMARK_PRESETS environment
+# variable can be set to build required artifacts for the comma-separated
+# benchmark presets. By default `iree-benchmark-suites` is built for sanity
+# check and e2e model testing. It can be disabled with the environment variable
+# `IREE_BUILD_DEFAULT_BENCHMARK_SUITES=0`.
#
# Designed for CI, but can be run locally. The desired build directory can be
# passed as the first argument. Otherwise, it uses the environment variable
@@ -26,10 +30,53 @@
BUILD_DIR="${1:-${IREE_BUILD_E2E_TEST_ARTIFACTS_DIR:-build-e2e-test-artifacts}}"
IREE_HOST_BIN_DIR="$(realpath ${IREE_HOST_BIN_DIR})"
+BENCHMARK_PRESETS="${IREE_BENCHMARK_PRESETS:-}"
+BUILD_DEFAULT_BENCHMARK_SUITES="${IREE_BUILD_DEFAULT_BENCHMARK_SUITES:-1}"
source build_tools/cmake/setup_build.sh
source build_tools/cmake/setup_tf_python.sh
+declare -a BUILD_TARGETS
+
+if (( "${BUILD_DEFAULT_BENCHMARK_SUITES}" == 1 )); then
+ BUILD_TARGETS+=("iree-benchmark-suites")
+fi
+
+# Separate the presets into the execution and compilation benchmark presets to
+# export different configs with export_benchmark_config.py.
+COMPILATION_PRESETS=""
+EXECUTION_PRESETS=""
+if [[ -n "${BENCHMARK_PRESETS}" ]]; then
+ IFS=, read -r -a PRESET_ARRAY <<< "${BENCHMARK_PRESETS}"
+ for PRESET in "${PRESET_ARRAY[@]}"; do
+ case "${PRESET}" in
+ comp-stats)
+ BUILD_TARGETS+=(iree-e2e-compile-stats-suites)
+ COMPILATION_PRESETS="${COMPILATION_PRESETS},${PRESET}"
+ ;;
+ comp-stats-long)
+ BUILD_TARGETS+=(iree-e2e-compile-stats-suites-long)
+ COMPILATION_PRESETS="${COMPILATION_PRESETS},${PRESET}"
+ ;;
+ *-long)
+ BUILD_TARGETS+=(iree-benchmark-suites-long)
+ EXECUTION_PRESETS="${EXECUTION_PRESETS},${PRESET}"
+ ;;
+ *)
+ # Build target of the default preset has been added above.
+ EXECUTION_PRESETS="${EXECUTION_PRESETS},${PRESET}"
+ ;;
+ esac
+ done
+ COMPILATION_PRESETS="${COMPILATION_PRESETS#,}"
+ EXECUTION_PRESETS="${EXECUTION_PRESETS#,}"
+fi
+
+if (( "${#BUILD_TARGETS[@]}" == 0 )); then
+ echo "No target to build."
+ exit 1
+fi
+
echo "Configuring to build e2e test artifacts"
"${CMAKE_BIN}" -B "${BUILD_DIR}" \
-G Ninja \
@@ -44,5 +91,22 @@
echo "Building e2e test artifacts"
"${CMAKE_BIN}" \
--build "${BUILD_DIR}" \
- --target iree-e2e-test-artifacts \
+ --target "${BUILD_TARGETS[@]}" \
-- -k 0
+
+E2E_TEST_ARTIFACTS_DIR="${BUILD_DIR}/e2e_test_artifacts"
+COMPILATION_CONFIG="${E2E_TEST_ARTIFACTS_DIR}/compilation-benchmark-config.json"
+EXECUTION_CONFIG="${E2E_TEST_ARTIFACTS_DIR}/execution-benchmark-config.json"
+FLAG_DUMP="${E2E_TEST_ARTIFACTS_DIR}/benchmark-flag-dump.txt"
+./build_tools/benchmarks/export_benchmark_config.py \
+ compilation \
+ --benchmark_presets="${COMPILATION_PRESETS}" \
+ --output="${COMPILATION_CONFIG}"
+./build_tools/benchmarks/export_benchmark_config.py \
+ execution \
+ --benchmark_presets="${EXECUTION_PRESETS}" \
+ --output="${EXECUTION_CONFIG}"
+./build_tools/benchmarks/benchmark_helper.py dump-cmds \
+ --execution_benchmark_config="${EXECUTION_CONFIG}" \
+ --compilation_benchmark_config="${COMPILATION_CONFIG}" \
+ > "${FLAG_DUMP}"
diff --git a/build_tools/github_actions/configure_ci.py b/build_tools/github_actions/configure_ci.py
index 3f5b6c2..3e68f92 100755
--- a/build_tools/github_actions/configure_ci.py
+++ b/build_tools/github_actions/configure_ci.py
@@ -302,6 +302,11 @@
preset_options.remove(DEFAULT_BENCHMARK_PRESET)
preset_options.update(DEFAULT_BENCHMARK_PRESET_GROUP)
+ if preset_options.intersection(DEFAULT_BENCHMARK_PRESET_GROUP):
+ # The is a sugar to run the compilation benchmarks when any default
+ # benchmark preset is present.
+ preset_options.add("comp-stats")
+
preset_options = sorted(preset_options)
for preset_option in preset_options:
if preset_option not in BENCHMARK_PRESET_OPTIONS:
diff --git a/build_tools/github_actions/configure_ci_test.py b/build_tools/github_actions/configure_ci_test.py
index 2f77166..5b33ba6 100644
--- a/build_tools/github_actions/configure_ci_test.py
+++ b/build_tools/github_actions/configure_ci_test.py
@@ -32,7 +32,7 @@
is_pr=True,
is_llvm_integrate_pr=False)
- self.assertEqual(presets_str, "cuda,x86_64")
+ self.assertEqual(presets_str, "comp-stats,cuda,x86_64")
def test_get_benchmark_presets_from_trailers_and_labels(self):
presets_str = configure_ci.get_benchmark_presets(
@@ -41,7 +41,8 @@
is_pr=True,
is_llvm_integrate_pr=False)
- self.assertEqual(presets_str, "android-cpu,android-gpu,cuda,x86_64")
+ self.assertEqual(presets_str,
+ "android-cpu,android-gpu,comp-stats,cuda,x86_64")
def test_get_benchmark_presets_from_default_group(self):
presets_str = configure_ci.get_benchmark_presets(
