Add 15 minute test timeout to all ctest invocations (#6617)
Hanging forever is extremely unhelpful. Setting test-specific timeouts,
which could be derived from the Bazel timeouts as part of Bazel->CMake
would be helpful, but in the meantime, just setting a reasonable upper
bound should get us a lot of mileage. This matches the Bazel timeout for
"long" tests, which is the maximum timeout we have for all of our tests
that we run in our CI.
Part of https://github.com/google/iree/issues/5121
diff --git a/build_tools/buildkite/cmake/android/arm64-v8a/pipeline.yml b/build_tools/buildkite/cmake/android/arm64-v8a/pipeline.yml
index 1c62998..e10fb90 100644
--- a/build_tools/buildkite/cmake/android/arm64-v8a/pipeline.yml
+++ b/build_tools/buildkite/cmake/android/arm64-v8a/pipeline.yml
@@ -23,7 +23,7 @@
- "tar xzf build-artifacts.tgz"
- "find build-android/ -name '*.cmake' -exec sed -i \"s!\\$IREE_DOCKER_WORKDIR/!\\$PWD/!g\" {} \\;"
- "cd build-android/"
- - "ctest --output-on-failure"
+ - "ctest --timeout 900 --output-on-failure"
agents:
- "android-soc=exynos-990"
- "queue=test-android"
@@ -37,7 +37,7 @@
- "tar xzf build-artifacts.tgz"
- "find build-android/ -name '*.cmake' -exec sed -i \"s!\\$IREE_DOCKER_WORKDIR/!\\$PWD/!g\" {} \\;"
- "cd build-android/"
- - "ctest --output-on-failure"
+ - "ctest --timeout 900 --output-on-failure"
agents:
- "android-soc=exynos-9820"
- "queue=test-android"
@@ -53,7 +53,7 @@
- "cd build-android/"
# vulkan tests using khr_shader_float16_int8 are failing on pixel4.
# Disabling it until we identify the root cause.
- - "ctest --output-on-failure --label-exclude \"^vulkan_uses_vk_khr_shader_float16_int8\\$\""
+ - "ctest --timeout 900 --output-on-failure --label-exclude \"^vulkan_uses_vk_khr_shader_float16_int8\\$\""
agents:
- "android-soc=snapdragon-855"
- "queue=test-android"
diff --git a/build_tools/cmake/test.sh b/build_tools/cmake/test.sh
index da602bb..120d391 100755
--- a/build_tools/cmake/test.sh
+++ b/build_tools/cmake/test.sh
@@ -67,4 +67,4 @@
label_exclude_regex="($(IFS="|" ; echo "${label_exclude_args[*]?}"))"
cd "$BUILD_DIR"
-ctest --output-on-failure --label-exclude "${label_exclude_regex?}"
+ctest --timeout 900 --output-on-failure --label-exclude "${label_exclude_regex?}"
diff --git a/build_tools/kokoro/gcp_ubuntu/cmake-bazel/linux/x86-swiftshader/build.sh b/build_tools/kokoro/gcp_ubuntu/cmake-bazel/linux/x86-swiftshader/build.sh
index f74194c..5966d91 100755
--- a/build_tools/kokoro/gcp_ubuntu/cmake-bazel/linux/x86-swiftshader/build.sh
+++ b/build_tools/kokoro/gcp_ubuntu/cmake-bazel/linux/x86-swiftshader/build.sh
@@ -64,6 +64,6 @@
export CTEST_PARALLEL_LEVEL=${CTEST_PARALLEL_LEVEL:-$(nproc)}
echo "Testing with CTest"
-ctest --output-on-failure \
+ctest --timeout 900 --output-on-failure \
--tests-regex "^integrations/tensorflow/|^bindings/python/" \
--label-exclude "^nokokoro$|^vulkan_uses_vk_khr_shader_float16_int8$"
diff --git a/build_tools/kokoro/gcp_ubuntu/cmake-bazel/linux/x86-turing/build.sh b/build_tools/kokoro/gcp_ubuntu/cmake-bazel/linux/x86-turing/build.sh
index a5d1d99..eb92860 100755
--- a/build_tools/kokoro/gcp_ubuntu/cmake-bazel/linux/x86-turing/build.sh
+++ b/build_tools/kokoro/gcp_ubuntu/cmake-bazel/linux/x86-turing/build.sh
@@ -68,7 +68,7 @@
# Only test drivers that use the GPU, since we run all tests on non-GPU machines
# as well.
echo "Testing with CTest"
-ctest --output-on-failure \
+ctest --timeout 900 --output-on-failure \
--tests-regex "^integrations/tensorflow/|^bindings/python/" \
--label-regex "^driver=vulkan$|^driver=cuda$" \
--label-exclude "^nokokoro$"
diff --git a/build_tools/kokoro/gcp_ubuntu/cmake/linux/x86-swiftshader-asan/build.sh b/build_tools/kokoro/gcp_ubuntu/cmake/linux/x86-swiftshader-asan/build.sh
index 1f5b793..3c6a615 100755
--- a/build_tools/kokoro/gcp_ubuntu/cmake/linux/x86-swiftshader-asan/build.sh
+++ b/build_tools/kokoro/gcp_ubuntu/cmake/linux/x86-swiftshader-asan/build.sh
@@ -125,6 +125,6 @@
cd ${CMAKE_BUILD_DIR?}
echo "Testing with ctest"
-ctest --output-on-failure \
+ctest --timeout 900 --output-on-failure \
--label-exclude "^driver=cuda$|^driver=vulkan$" \
--exclude-regex "${excluded_tests_regex?}"
