Add vmvx integration tests to CI (#13584)
Add a few integration tests running vmvx backend to CI so we can
track/catch the numerical issues (#10514).
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 845e555..8612aef 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -386,6 +386,7 @@
- name: "Running TF integrations tests"
run: |
./build_tools/github_actions/docker_run.sh \
+ --env IREE_VMVX_DISABLE=0 \
gcr.io/iree-oss/frontends-swiftshader@sha256:da14cc93637d3bfad469a670d4d7a49982df5d107b775331965e3bacb981d4cf \
build_tools/cmake/run_tf_tests.sh \
"${BUILD_DIR}"
diff --git a/build_tools/cmake/run_tf_tests.sh b/build_tools/cmake/run_tf_tests.sh
index e0852b9..fad778d 100755
--- a/build_tools/cmake/run_tf_tests.sh
+++ b/build_tools/cmake/run_tf_tests.sh
@@ -15,6 +15,10 @@
IREE_VULKAN_DISABLE="${IREE_VULKAN_DISABLE:-0}"
IREE_LLVM_CPU_DISABLE="${IREE_LLVM_CPU_DISABLE:-0}"
+# VMVX codegen is for reference and less optimized than other target backends.
+# Disable the tests by default to reduce the test time.
+IREE_VMVX_DISABLE="${IREE_VMVX_DISABLE:-1}"
+
source "${BUILD_DIR}/.env" && export PYTHONPATH
source build_tools/cmake/setup_tf_python.sh
@@ -30,8 +34,19 @@
--max-time 1800
)
+declare -a TARGET_BACKENDS=()
+
if (( ${IREE_VULKAN_DISABLE} != 1 )); then
- CMD+=(-D FEATURES=vulkan)
+ TARGET_BACKENDS+=(vulkan)
+fi
+
+if (( ${IREE_VMVX_DISABLE} != 1 )); then
+ TARGET_BACKENDS+=(vmvx)
+fi
+
+if [[ -n "${TARGET_BACKENDS[*]}" ]]; then
+ TARGET_BACKENDS_STR="$(IFS="," ; echo "${TARGET_BACKENDS[*]}")"
+ CMD+=(-D FEATURES=${TARGET_BACKENDS_STR})
fi
if (( ${IREE_LLVM_CPU_DISABLE} == 1 )); then
diff --git a/integrations/tensorflow/test/iree_tfl_tests/vmvx_mobilebert_tf2_quant.run b/integrations/tensorflow/test/iree_tfl_tests/vmvx_mobilebert_tf2_quant.run
new file mode 100644
index 0000000..63c8663
--- /dev/null
+++ b/integrations/tensorflow/test/iree_tfl_tests/vmvx_mobilebert_tf2_quant.run
@@ -0,0 +1,2 @@
+# REQUIRES: vmvx
+# RUN: %PYTHON -m iree_tfl_tests.mobilebert_tf2_quant_test --target_backend=vmvx --artifacts_dir=%t
diff --git a/integrations/tensorflow/test/iree_tfl_tests/vmvx_mobilenet_v3-large_uint8.run b/integrations/tensorflow/test/iree_tfl_tests/vmvx_mobilenet_v3-large_uint8.run
new file mode 100644
index 0000000..eb452ff
--- /dev/null
+++ b/integrations/tensorflow/test/iree_tfl_tests/vmvx_mobilenet_v3-large_uint8.run
@@ -0,0 +1,4 @@
+# REQUIRES: vmvx
+# RUN: %PYTHON -m iree_tfl_tests.mobilenet_v3-large_uint8_test --target_backend=vmvx --artifacts_dir=%t
+# TODO(#10514): Address vmvx numerical mismatch issue.
+# XFAIL: vmvx
diff --git a/integrations/tensorflow/test/iree_tfl_tests/vmvx_person_detect.run b/integrations/tensorflow/test/iree_tfl_tests/vmvx_person_detect.run
new file mode 100644
index 0000000..9ef1adb
--- /dev/null
+++ b/integrations/tensorflow/test/iree_tfl_tests/vmvx_person_detect.run
@@ -0,0 +1,2 @@
+# REQUIRES: vmvx
+# RUN: %PYTHON -m iree_tfl_tests.person_detect_test --target_backend=vmvx --artifacts_dir=%t
diff --git a/integrations/tensorflow/test/iree_tfl_tests/vmvx_resnet_50_int8.run b/integrations/tensorflow/test/iree_tfl_tests/vmvx_resnet_50_int8.run
new file mode 100644
index 0000000..f972d7d
--- /dev/null
+++ b/integrations/tensorflow/test/iree_tfl_tests/vmvx_resnet_50_int8.run
@@ -0,0 +1,4 @@
+# REQUIRES: vmvx
+# RUN: %PYTHON -m iree_tfl_tests.resnet_50_int8_test --target_backend=vmvx -artifacts_dir=%t
+# TODO(#10514): Address vmvx numerical mismatch issue.
+# XFAIL: vmvx
diff --git a/integrations/tensorflow/test/python/iree_tfl_tests/test_util.py b/integrations/tensorflow/test/python/iree_tfl_tests/test_util.py
index 7df4ce3..dc5321b 100644
--- a/integrations/tensorflow/test/python/iree_tfl_tests/test_util.py
+++ b/integrations/tensorflow/test/python/iree_tfl_tests/test_util.py
@@ -19,11 +19,13 @@
targets = {
'llvmcpu': 'llvm-cpu',
+ 'vmvx': 'vmvx',
'vulkan': 'vulkan-spirv',
}
configs = {
'llvmcpu': 'local-task',
+ 'vmvx': 'local-task',
'vulkan': 'vulkan',
}
