Fix benchmark presets for CUDA and vulkan-nvidia benchmarks (#13416)
diff --git a/build_tools/benchmarks/common/benchmark_definition.py b/build_tools/benchmarks/common/benchmark_definition.py
index dffde66..0d0b5d5 100644
--- a/build_tools/benchmarks/common/benchmark_definition.py
+++ b/build_tools/benchmarks/common/benchmark_definition.py
@@ -50,15 +50,24 @@
# A map from GPU name to IREE's benchmark target architecture.
GPU_NAME_TO_TARGET_ARCH_MAP = {
- "adreno-640": common_definitions.DeviceArchitecture.QUALCOMM_ADRENO,
- "adreno-650": common_definitions.DeviceArchitecture.QUALCOMM_ADRENO,
- "adreno-660": common_definitions.DeviceArchitecture.QUALCOMM_ADRENO,
- "adreno-730": common_definitions.DeviceArchitecture.QUALCOMM_ADRENO,
- "mali-g77": common_definitions.DeviceArchitecture.ARM_VALHALL,
- "mali-g78": common_definitions.DeviceArchitecture.ARM_VALHALL,
- "tesla-v100-sxm2-16gb": common_definitions.DeviceArchitecture.CUDA_SM70,
- "nvidia-a100-sxm4-40gb": common_definitions.DeviceArchitecture.CUDA_SM80,
- "nvidia-geforce-rtx-3090": common_definitions.DeviceArchitecture.CUDA_SM80,
+ "adreno-640":
+ common_definitions.DeviceArchitecture.QUALCOMM_ADRENO,
+ "adreno-650":
+ common_definitions.DeviceArchitecture.QUALCOMM_ADRENO,
+ "adreno-660":
+ common_definitions.DeviceArchitecture.QUALCOMM_ADRENO,
+ "adreno-730":
+ common_definitions.DeviceArchitecture.QUALCOMM_ADRENO,
+ "mali-g77":
+ common_definitions.DeviceArchitecture.ARM_VALHALL,
+ "mali-g78":
+ common_definitions.DeviceArchitecture.ARM_VALHALL,
+ "tesla-v100-sxm2-16gb":
+ common_definitions.DeviceArchitecture.NVIDIA_PASCAL,
+ "nvidia-a100-sxm4-40gb":
+ common_definitions.DeviceArchitecture.NVIDIA_AMPERE,
+ "nvidia-geforce-rtx-3090":
+ common_definitions.DeviceArchitecture.NVIDIA_AMPERE,
}
diff --git a/build_tools/benchmarks/export_benchmark_config.py b/build_tools/benchmarks/export_benchmark_config.py
index 33b32fc..c4c091c 100755
--- a/build_tools/benchmarks/export_benchmark_config.py
+++ b/build_tools/benchmarks/export_benchmark_config.py
@@ -50,12 +50,12 @@
"x86_64":
lambda config: config.target_device_spec.architecture.architecture ==
"x86_64",
- "cuda":
- lambda config: config.target_device_spec.architecture.architecture ==
- "cuda" and "long-running" not in config.tags,
+ "cuda": (lambda config: "cuda" in config.tags and "long-running" not in
+ config.tags),
"cuda-long":
- lambda config: config.target_device_spec.architecture.architecture ==
- "cuda" and "long-running" in config.tags,
+ lambda config: "cuda" in config.tags and "long-running" in config.tags,
+ "vulkan-nvidia":
+ lambda config: "vulkan-nvidia" in config.tags,
# TODO(#9855): Enable benchmarks on Pixel-6-Pro and XT2201-2.
"experimental-android-cpu":
lambda config:
diff --git a/build_tools/github_actions/configure_ci.py b/build_tools/github_actions/configure_ci.py
index 51f91fa..07dbc42 100755
--- a/build_tools/github_actions/configure_ci.py
+++ b/build_tools/github_actions/configure_ci.py
@@ -77,7 +77,7 @@
RUNNER_ENV_DEFAULT = "prod"
RUNNER_ENV_OPTIONS = [RUNNER_ENV_DEFAULT, "testing"]
-DEFAULT_BENCHMARK_PRESETS = ["cuda", "x86_64", "comp-stats"]
+DEFAULT_BENCHMARK_PRESETS = ["cuda", "x86_64", "vulkan-nvidia", "comp-stats"]
BENCHMARK_PRESET_OPTIONS = DEFAULT_BENCHMARK_PRESETS + [
"experimental-android-cpu"
]
diff --git a/build_tools/python/benchmark_suites/iree/cuda_benchmarks.py b/build_tools/python/benchmark_suites/iree/cuda_benchmarks.py
index 2aa7b89..08c1cb4 100644
--- a/build_tools/python/benchmark_suites/iree/cuda_benchmarks.py
+++ b/build_tools/python/benchmark_suites/iree/cuda_benchmarks.py
@@ -73,18 +73,21 @@
"""Generates IREE compile and run configs."""
gen_configs, run_configs = self._generate_configs(model_groups.CUDA_MODELS,
self.SM_80_COMPILE_CONFIG)
+ # The `cuda` tag is required to put them into the CUDA benchmark preset.
ubench_gen_configs, ubench_run_configs = self._generate_configs(
model_groups.MICRO_MATMUL,
self.SM_80_UBENCH_MATMUL_COMPILE_CONFIG,
- execution_config=module_execution_configs.CUDA_BATCH_SIZE_100_CONFIG)
+ execution_config=module_execution_configs.CUDA_BATCH_SIZE_100_CONFIG,
+ run_tags=["cuda"])
ubench_splitk_gen_configs, ubench_splitk_run_configs = self._generate_configs(
model_groups.MICRO_MATMUL_SPLITK,
self.SM_80_UBENCH_MATMUL_SPLITK_COMPILE_CONFIG,
- execution_config=module_execution_configs.CUDA_BATCH_SIZE_100_CONFIG)
+ execution_config=module_execution_configs.CUDA_BATCH_SIZE_100_CONFIG,
+ run_tags=["cuda"])
long_running_gen_configs, long_running_module_configs = self._generate_configs(
model_groups.CUDA_MODELS_LONG,
self.SM_80_COMPILE_CONFIG,
- run_tags=["long-running"])
+ run_tags=["cuda", "long-running"])
return (gen_configs + ubench_gen_configs + ubench_splitk_gen_configs +
long_running_gen_configs, run_configs + ubench_run_configs +
ubench_splitk_run_configs + long_running_module_configs)
diff --git a/build_tools/python/benchmark_suites/iree/vulkan_nvidia_benchmarks.py b/build_tools/python/benchmark_suites/iree/vulkan_nvidia_benchmarks.py
index e1a3fc6..fc9c764 100644
--- a/build_tools/python/benchmark_suites/iree/vulkan_nvidia_benchmarks.py
+++ b/build_tools/python/benchmark_suites/iree/vulkan_nvidia_benchmarks.py
@@ -88,9 +88,15 @@
) -> Tuple[List[iree_definitions.ModuleGenerationConfig],
List[iree_definitions.E2EModelRunConfig]]:
"""Generates IREE compile and run configs."""
+ # The `vulkan-nvidia` tag is required to put them into the Vulkan NVIDIA
+ # benchmark preset.
tensorcore_gen_configs, tensorcore_run_configs = self._generate_configs(
- model_groups.VULKAN_MODELS, self.TENSORCORE_COMPILE_CONFIG)
+ model_groups.VULKAN_MODELS,
+ self.TENSORCORE_COMPILE_CONFIG,
+ run_tags=["vulkan-nvidia"])
simt_gen_configs, simt_run_configs = self._generate_configs(
- model_groups.VULKAN_MODELS, self.SIMT_COMPILE_CONFIG)
+ model_groups.VULKAN_MODELS,
+ self.SIMT_COMPILE_CONFIG,
+ run_tags=["vulkan-nvidia"])
return (tensorcore_gen_configs + simt_gen_configs,
tensorcore_run_configs + simt_run_configs)
