[spirv][vulkan] Run e2 i8 matmul tests in CI (#13312)
Only enable those on machines with Nvidia GPU. This is guarded by a new
`ctest_all.sh` env var setting: `IREE_NVIDIA_GPU_TESTS_DISABLE`, which
defaults to skipping tests that require nvidia gpu.
Fixes: https://github.com/openxla/iree/issues/13123
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index feeb756..51c6ac5 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -286,6 +286,7 @@
./build_tools/github_actions/docker_run.sh \
--env IREE_VULKAN_F16_DISABLE=0 \
--env IREE_CUDA_DISABLE=0 \
+ --env IREE_NVIDIA_GPU_TESTS_DISABLE=0 \
--env CTEST_PARALLEL_LEVEL=2 \
--gpus all \
--env NVIDIA_DRIVER_CAPABILITIES=all \
@@ -439,6 +440,7 @@
run: |
./build_tools/github_actions/docker_run.sh \
--env IREE_LLVM_CPU_DISABLE=1 \
+ --env IREE_NVIDIA_GPU_TESTS_DISABLE=0 \
--gpus all \
--env NVIDIA_DRIVER_CAPABILITIES=all \
gcr.io/iree-oss/frontends-nvidia@sha256:b0df86dff8bfcf5f43eedb70d178703b4538f31a2bf45e3b2eb035724a7f6a42 \
diff --git a/build_tools/bazel/build_core.sh b/build_tools/bazel/build_core.sh
index 85f0e60..08a961a 100755
--- a/build_tools/bazel/build_core.sh
+++ b/build_tools/bazel/build_core.sh
@@ -35,11 +35,15 @@
if ! [[ -v IREE_VULKAN_DISABLE ]]; then
IREE_VULKAN_DISABLE=0
fi
+if ! [[ -v IREE_NVIDIA_GPU_TESTS_DISABLE ]]; then
+ IREE_NVIDIA_GPU_TESTS_DISABLE=1
+fi
declare -a test_env_args=(
--test_env="LD_PRELOAD=libvulkan.so.1"
--test_env="VK_ICD_FILENAMES=${VK_ICD_FILENAMES}"
--test_env=IREE_VULKAN_DISABLE="${IREE_VULKAN_DISABLE}"
+ --test_env=IREE_NVIDIA_GPU_TESTS_DISABLE="${IREE_NVIDIA_GPU_TESTS_DISABLE}"
)
if ! [[ -n IREE_LLVM_SYSTEM_LINKER_PATH ]]; then
@@ -62,6 +66,9 @@
if [[ "${IREE_VULKAN_DISABLE?}" == 1 ]]; then
default_test_tag_filters+=("-driver=vulkan")
fi
+if [[ "${IREE_NVIDIA_GPU_TESTS_DISABLE?}" == 1 ]]; then
+ default_test_tag_filters+=("-requires-gpu-nvidia")
+fi
# Use user-environment variables if set, otherwise use CI-friendly defaults.
if ! [[ -v BUILD_TAG_FILTERS ]]; then
diff --git a/build_tools/cmake/build_and_test_asan.sh b/build_tools/cmake/build_and_test_asan.sh
index edd56d6..aa387ca 100755
--- a/build_tools/cmake/build_and_test_asan.sh
+++ b/build_tools/cmake/build_and_test_asan.sh
@@ -68,6 +68,8 @@
# The VK_KHR_shader_float16_int8 extension is optional prior to Vulkan 1.2.
# We test on SwiftShader as a baseline, which does not support this extension.
export IREE_VULKAN_F16_DISABLE=${IREE_VULKAN_F16_DISABLE:-1}
+# Respect the user setting, but default to skipping tests that require Nvidia GPU.
+export IREE_NVIDIA_GPU_TESTS_DISABLE=${IREE_NVIDIA_GPU_TESTS_DISABLE:-1}
# Tests to exclude by label. In addition to any custom labels (which are carried
# over from Bazel tags), every test should be labeled with its directory.
@@ -95,10 +97,12 @@
if [[ "${IREE_CUDA_DISABLE?}" == 1 ]]; then
label_exclude_args+=("^driver=cuda$")
fi
-
if [[ "${IREE_VULKAN_F16_DISABLE?}" == 1 ]]; then
label_exclude_args+=("^vulkan_uses_vk_khr_shader_float16_int8$")
fi
+if [[ "${IREE_NVIDIA_GPU_TESTS_DISABLE}" == 1 ]]; then
+ label_exclude_args+=("^requires-gpu-nvidia$")
+fi
label_exclude_regex="($(IFS="|" ; echo "${label_exclude_args[*]?}"))"
diff --git a/build_tools/cmake/ctest_all.sh b/build_tools/cmake/ctest_all.sh
index 86d5c3e..be9336e 100755
--- a/build_tools/cmake/ctest_all.sh
+++ b/build_tools/cmake/ctest_all.sh
@@ -33,6 +33,8 @@
# The VK_KHR_shader_float16_int8 extension is optional prior to Vulkan 1.2.
# We test on SwiftShader as a baseline, which does not support this extension.
export IREE_VULKAN_F16_DISABLE=${IREE_VULKAN_F16_DISABLE:-1}
+# Respect the user setting, but default to skipping tests that require Nvidia GPU.
+export IREE_NVIDIA_GPU_TESTS_DISABLE=${IREE_NVIDIA_GPU_TESTS_DISABLE:-1}
# Respect the user setting, default to no --repeat-until-fail.
export IREE_CTEST_REPEAT_UNTIL_FAIL_COUNT=${IREE_CTEST_REPEAT_UNTIL_FAIL_COUNT:-}
# Respect the user setting, default to no --tests-regex.
@@ -69,6 +71,9 @@
if [[ "${IREE_VULKAN_F16_DISABLE}" == 1 ]]; then
label_exclude_args+=("^vulkan_uses_vk_khr_shader_float16_int8$")
fi
+if [[ "${IREE_NVIDIA_GPU_TESTS_DISABLE}" == 1 ]]; then
+ label_exclude_args+=("^requires-gpu-nvidia$")
+fi
IFS=',' read -ra extra_label_exclude_args <<< "${IREE_EXTRA_COMMA_SEPARATED_CTEST_LABELS_TO_EXCLUDE:-}"
label_exclude_args+=(${extra_label_exclude_args[@]})
diff --git a/tests/e2e/matmul/BUILD.bazel b/tests/e2e/matmul/BUILD.bazel
index e3725d3..4916257 100644
--- a/tests/e2e/matmul/BUILD.bazel
+++ b/tests/e2e/matmul/BUILD.bazel
@@ -327,13 +327,16 @@
]]
[iree_generated_trace_runner_test(
- name = "e2e_matmul_direct_f32_gpu_large_%s" % vulkan_target_and_pipeline[0],
+ name = "e2e_matmul_direct_{0}_gpu_large_{1}".format(
+ lhs_rhs_type,
+ vulkan_target_and_pipeline[0],
+ ),
compiler_flags = [
"--iree-vulkan-target-triple=%s" % vulkan_target_and_pipeline[0],
],
generator = ":generate_e2e_matmul_tests",
generator_args = [
- "--lhs_rhs_type=f32",
+ "--lhs_rhs_type=%s" % lhs_rhs_type,
"--shapes=gpu_large",
"--compilation_info=%s" % vulkan_target_and_pipeline[1],
],
@@ -347,6 +350,9 @@
) for vulkan_target_and_pipeline in [
("valhall-unknown-android31", "SPIRVVectorizeMali"),
("ampere-unknown-linux", "SPIRVVectorizeNVIDIA"),
+] for lhs_rhs_type in [
+ "i8",
+ "f32",
]]
# Check tests
diff --git a/tests/e2e/matmul/CMakeLists.txt b/tests/e2e/matmul/CMakeLists.txt
index c17f45f..81af47a 100644
--- a/tests/e2e/matmul/CMakeLists.txt
+++ b/tests/e2e/matmul/CMakeLists.txt
@@ -400,6 +400,27 @@
iree_generated_trace_runner_test(
NAME
+ e2e_matmul_direct_i8_gpu_large_valhall-unknown-android31
+ GENERATOR
+ "generate_e2e_matmul_tests.py"
+ GENERATOR_ARGS
+ "--lhs_rhs_type=i8"
+ "--shapes=gpu_large"
+ "--compilation_info=SPIRVVectorizeMali"
+ TRACE_RUNNER
+ iree-e2e-matmul-test
+ TARGET_BACKENDS
+ "vulkan-spirv"
+ DRIVERS
+ "vulkan"
+ COMPILER_FLAGS
+ "--iree-vulkan-target-triple=valhall-unknown-android31"
+ LABELS
+ "requires-gpu-nvidia"
+)
+
+iree_generated_trace_runner_test(
+ NAME
e2e_matmul_direct_f32_gpu_large_valhall-unknown-android31
GENERATOR
"generate_e2e_matmul_tests.py"
@@ -421,6 +442,27 @@
iree_generated_trace_runner_test(
NAME
+ e2e_matmul_direct_i8_gpu_large_ampere-unknown-linux
+ GENERATOR
+ "generate_e2e_matmul_tests.py"
+ GENERATOR_ARGS
+ "--lhs_rhs_type=i8"
+ "--shapes=gpu_large"
+ "--compilation_info=SPIRVVectorizeNVIDIA"
+ TRACE_RUNNER
+ iree-e2e-matmul-test
+ TARGET_BACKENDS
+ "vulkan-spirv"
+ DRIVERS
+ "vulkan"
+ COMPILER_FLAGS
+ "--iree-vulkan-target-triple=ampere-unknown-linux"
+ LABELS
+ "requires-gpu-nvidia"
+)
+
+iree_generated_trace_runner_test(
+ NAME
e2e_matmul_direct_f32_gpu_large_ampere-unknown-linux
GENERATOR
"generate_e2e_matmul_tests.py"
