Benchmark on c2-standard-60 in background (#15467)
Run x86_64 benchmarks on c2-standard-60 VM but don't publish to the dashboard.
#14661
diff --git a/.github/workflows/benchmark_execution.yml b/.github/workflows/benchmark_execution.yml
index d97150a..3b8ae6d 100644
--- a/.github/workflows/benchmark_execution.yml
+++ b/.github/workflows/benchmark_execution.yml
@@ -185,6 +185,11 @@
run: |
mkdir -p ${BENCHMARK_RESULTS_DIR}
export IREE_E2E_TEST_ARTIFACTS_DIR="${E2E_TEST_ARTIFACTS_GCS_ARTIFACT_DIR/gs:\/\//https://storage.googleapis.com/}"
+ # TODO(#14661): Annotate the c2-standard-60 resutls as experimental
+ # and avoid uploading to the dashboard (excluded from the pattern).
+ if [[ "${IREE_TARGET_DEVICE_NAME}" == "c2-standard-60" ]]; then
+ export IREE_BENCHMARK_RESULTS="${IREE_BENCHMARK_RESULTS/benchmark-results-/experimental-benchmark-results-}"
+ fi
./build_tools/benchmarks/run_benchmarks.sh
- name: "Uploading benchmark results"
run: gcloud storage cp -r "${BENCHMARK_RESULTS_DIR}" "${GCS_DIR}/"
diff --git a/build_tools/benchmarks/run_benchmarks.sh b/build_tools/benchmarks/run_benchmarks.sh
index e92f137..36df6f6 100755
--- a/build_tools/benchmarks/run_benchmarks.sh
+++ b/build_tools/benchmarks/run_benchmarks.sh
@@ -51,7 +51,7 @@
--shard_index="${SHARD_INDEX}" \
--output="${BENCHMARK_RESULTS}" \
--verbose
-elif [[ "${TARGET_DEVICE_NAME}" == "c2-standard-16" ]]; then
+elif [[ "${TARGET_DEVICE_NAME}" =~ ^(c2-standard-16|c2-standard-60)$ ]]; then
${DOCKER_WRAPPER} \
gcr.io/iree-oss/base-bleeding-edge@sha256:14200dacca3a0f3a66f8aa87c6f64729b83a2eeb403b689c24204074ad157418 \
./build_tools/benchmarks/run_benchmarks_on_linux.py \
@@ -64,7 +64,7 @@
--target_device_name="${TARGET_DEVICE_NAME}" \
--shard_index="${SHARD_INDEX}" \
--output="${BENCHMARK_RESULTS}" \
- --device_model=GCP-c2-standard-16 \
+ --device_model="GCP-${TARGET_DEVICE_NAME}" \
--cpu_uarch=CascadeLake \
--verbose
elif [[ "${TARGET_DEVICE_NAME}" =~ ^(pixel-4|pixel-6-pro|moto-edge-x30)$ ]]; then
diff --git a/build_tools/python/e2e_test_framework/device_specs/device_collections.py b/build_tools/python/e2e_test_framework/device_specs/device_collections.py
index 9b9206e..9031cea 100644
--- a/build_tools/python/e2e_test_framework/device_specs/device_collections.py
+++ b/build_tools/python/e2e_test_framework/device_specs/device_collections.py
@@ -61,6 +61,7 @@
moto_edge_x30_specs.GPU,
# GCP machines
gcp_specs.GCP_C2_STANDARD_16,
+ gcp_specs.GCP_C2_STANDARD_60,
gcp_specs.GCP_A2_HIGHGPU_1G,
# RISCV emulators
riscv_specs.EMULATOR_RISCV_32,
diff --git a/build_tools/python/e2e_test_framework/device_specs/gcp_specs.py b/build_tools/python/e2e_test_framework/device_specs/gcp_specs.py
index 9ca92f8..97ffb27 100644
--- a/build_tools/python/e2e_test_framework/device_specs/gcp_specs.py
+++ b/build_tools/python/e2e_test_framework/device_specs/gcp_specs.py
@@ -18,6 +18,15 @@
tags=["cpu"],
)
+GCP_C2_STANDARD_60 = common_definitions.DeviceSpec.build(
+ id=unique_ids.DEVICE_SPEC_GCP_C2_STANDARD_60,
+ device_name="c2-standard-60",
+ host_environment=common_definitions.HostEnvironment.LINUX_X86_64,
+ architecture=common_definitions.DeviceArchitecture.X86_64_CASCADELAKE,
+ device_parameters=[device_parameters.ALL_CORES],
+ tags=["cpu"],
+)
+
GCP_A2_HIGHGPU_1G = common_definitions.DeviceSpec.build(
id=unique_ids.DEVICE_SPEC_GCP_A2_HIGHGPU_1G,
device_name="a2-highgpu-1g",
diff --git a/build_tools/python/e2e_test_framework/unique_ids.py b/build_tools/python/e2e_test_framework/unique_ids.py
index 0a9b6fe..b426336 100644
--- a/build_tools/python/e2e_test_framework/unique_ids.py
+++ b/build_tools/python/e2e_test_framework/unique_ids.py
@@ -136,6 +136,7 @@
# Devices
DEVICE_SPEC_GCP_C2_STANDARD_16 = "9a4804f1-b1b9-46cd-b251-7f16a655f782"
+DEVICE_SPEC_GCP_C2_STANDARD_60 = "2ebc0ede-eac2-4b77-b17d-eb3b14c17f89"
DEVICE_SPEC_GCP_A2_HIGHGPU_1G = "78c56b95-2d7d-44b5-b5fd-8e47aa961108"
DEVICE_SPEC_MOBILE_PIXEL_4 = "fc901efc-ddf8-44c0-b009-8eecb8286521"
DEVICE_SPEC_MOBILE_PIXEL_6_PRO = "0f624778-dc50-43eb-8060-7f4aea9634e1"
