Added timeout to testing infrastructure (#9356)
Included a couple of changes:
using bazel's timeout arg instead of size
update bazel_to_cmake_converter.py to to set TIMEOUT on cmake files
update the appropriate CMAKE files for above
updated cmake files to plumb TIMEOUT down to setting CTEST's timeout property
set default TIMEOUT value on tests to 60 (shortest bazel timeout)
Co-authored-by: Scott Todd <scotttodd@google.com>
diff --git a/build_tools/bazel_to_cmake/bazel_to_cmake_converter.py b/build_tools/bazel_to_cmake/bazel_to_cmake_converter.py
index 3ee6f8a..a0bef57 100644
--- a/build_tools/bazel_to_cmake/bazel_to_cmake_converter.py
+++ b/build_tools/bazel_to_cmake/bazel_to_cmake_converter.py
@@ -39,6 +39,23 @@
return f" {name}\n {value}\n"
+# Match Bazel's timeout values
+# https://docs.bazel.build/versions/main/test-encyclopedia.html
+timeout_map = {
+ "short": 60,
+ "moderate": 300,
+ "long": 900,
+ "eternal": 3600,
+}
+
+
+def _convert_timeout_arg_block(name, value):
+ if value is None:
+ return ""
+ value = timeout_map[value]
+ return f" {name}\n {value}\n"
+
+
def _convert_string_list_block(name, values, quote=True, sort=False):
# Note this deliberately distinguishes between an empty list (argument
# explicitly specified) and None (argument left as default).
@@ -323,6 +340,7 @@
defines=None,
data=None,
deps=None,
+ timeout=None,
args=None,
tags=None,
**kwargs):
@@ -335,6 +353,7 @@
deps_block = _convert_target_list_block("DEPS", deps)
args_block = _convert_string_list_block("ARGS", args)
labels_block = _convert_string_list_block("LABELS", tags)
+ timeout_block = _convert_timeout_arg_block("TIMEOUT", timeout)
self.converter.body += (f"iree_cc_test(\n"
f"{name_block}"
@@ -346,6 +365,7 @@
f"{deps_block}"
f"{args_block}"
f"{labels_block}"
+ f"{timeout_block}"
f")\n\n")
def iree_runtime_cc_test(self, deps=[], **kwargs):
@@ -509,12 +529,14 @@
tools=None,
data=None,
tags=None,
+ timeout=None,
**kwargs):
name_block = _convert_string_arg_block("NAME", name, quote=False)
srcs_block = _convert_srcs_block(srcs)
tools_block = _convert_target_list_block("TOOLS", tools)
data_block = _convert_target_list_block("DATA", data)
labels_block = _convert_string_list_block("LABELS", tags)
+ timeout_block = _convert_timeout_arg_block("TIMEOUT", timeout)
self.converter.body += (f"iree_lit_test_suite(\n"
f"{name_block}"
@@ -522,6 +544,7 @@
f"{tools_block}"
f"{data_block}"
f"{labels_block}"
+ f"{timeout_block}"
f")\n\n")
def iree_check_single_backend_test_suite(self,
@@ -534,6 +557,7 @@
runner_args=None,
tags=None,
target_cpu_features=None,
+ timeout=None,
**kwargs):
name_block = _convert_string_arg_block("NAME", name, quote=False)
srcs_block = _convert_srcs_block(srcs)
@@ -546,6 +570,7 @@
labels_block = _convert_string_list_block("LABELS", tags)
target_cpu_features_block = _convert_string_arg_block(
"TARGET_CPU_FEATURES", target_cpu_features)
+ timeout_block = _convert_timeout_arg_block("TIMEOUT", timeout)
self.converter.body += (f"iree_check_single_backend_test_suite(\n"
f"{name_block}"
@@ -556,6 +581,7 @@
f"{runner_args_block}"
f"{labels_block}"
f"{target_cpu_features_block}"
+ f"{timeout_block}"
f")\n\n")
def iree_check_test_suite(self,
@@ -566,6 +592,7 @@
runner_args=None,
tags=None,
target_cpu_features_variants=None,
+ timeout=None,
**kwargs):
target_backends = None
drivers = None
@@ -584,6 +611,7 @@
labels_block = _convert_string_list_block("LABELS", tags)
target_cpu_features_variants_block = _convert_string_list_block(
"TARGET_CPU_FEATURES_VARIANTS", target_cpu_features_variants)
+ timeout_block = _convert_timeout_arg_block("TIMEOUT", timeout)
self.converter.body += (f"iree_check_test_suite(\n"
f"{name_block}"
@@ -594,6 +622,7 @@
f"{runner_args_block}"
f"{labels_block}"
f"{target_cpu_features_variants_block}"
+ f"{timeout_block}"
f")\n\n")
def iree_generated_trace_runner_test(self,
@@ -646,7 +675,13 @@
f"{target_cpu_features_variants_block}"
f")\n\n")
- def native_test(self, name, src, args=None, data=None, tags=None):
+ def native_test(self,
+ name,
+ src,
+ args=None,
+ data=None,
+ tags=None,
+ timeout=None):
if data is not None:
self._convert_unimplemented_function("native_test", name + " has data")
@@ -654,6 +689,7 @@
test_binary_block = _convert_single_target_block("SRC", src)
args_block = _convert_string_list_block("ARGS", args)
labels_block = _convert_string_list_block("LABELS", tags)
+ timeout_block = _convert_timeout_arg_block("TIMEOUT", timeout)
self.converter.body += (f"iree_native_test(\n"
f"{name_block}"
diff --git a/build_tools/cmake/iree_cc_test.cmake b/build_tools/cmake/iree_cc_test.cmake
index 82311e4..9335169 100644
--- a/build_tools/cmake/iree_cc_test.cmake
+++ b/build_tools/cmake/iree_cc_test.cmake
@@ -60,7 +60,7 @@
_RULE
""
"NAME"
- "ARGS;SRCS;COPTS;DEFINES;LINKOPTS;DATA;DEPS;LABELS"
+ "ARGS;SRCS;COPTS;DEFINES;LINKOPTS;DATA;DEPS;LABELS;TIMEOUT"
${ARGN}
)
@@ -175,6 +175,11 @@
iree_add_test_environment_properties(${_NAME_PATH})
endif(ANDROID)
+ if (NOT DEFINED _RULE_TIMEOUT)
+ set(_RULE_TIMEOUT 60)
+ endif()
+
list(APPEND _RULE_LABELS "${_PACKAGE_PATH}")
set_property(TEST ${_NAME_PATH} PROPERTY LABELS "${_RULE_LABELS}")
+ set_property(TEST ${_NAME_PATH} PROPERTY TIMEOUT ${_RULE_TIMEOUT})
endfunction()
diff --git a/build_tools/cmake/iree_check_test.cmake b/build_tools/cmake/iree_check_test.cmake
index b778ea2..e685bcd 100644
--- a/build_tools/cmake/iree_check_test.cmake
+++ b/build_tools/cmake/iree_check_test.cmake
@@ -115,7 +115,7 @@
_RULE
""
"NAME;SRC;TARGET_BACKEND;DRIVER;MODULE_FILE_NAME"
- "COMPILER_FLAGS;RUNNER_ARGS;LABELS;TARGET_CPU_FEATURES"
+ "COMPILER_FLAGS;RUNNER_ARGS;LABELS;TARGET_CPU_FEATURES;TIMEOUT"
${ARGN}
)
@@ -192,6 +192,8 @@
LABELS
${_RULE_LABELS}
${_RULE_TARGET_CPU_FEATURES}
+ TIMEOUT
+ ${_RULE_TIMEOUT}
)
endfunction()
@@ -231,7 +233,7 @@
_RULE
""
"NAME;TARGET_BACKEND;DRIVER"
- "SRCS;COMPILER_FLAGS;RUNNER_ARGS;LABELS;TARGET_CPU_FEATURES"
+ "SRCS;COMPILER_FLAGS;RUNNER_ARGS;LABELS;TARGET_CPU_FEATURES;TIMEOUT"
${ARGN}
)
@@ -286,6 +288,8 @@
${_RULE_LABELS}
TARGET_CPU_FEATURES
${_RULE_TARGET_CPU_FEATURES}
+ TIMEOUT
+ ${_RULE_TIMEOUT}
)
endforeach()
endfunction()
@@ -406,7 +410,7 @@
_RULE
""
"NAME"
- "SRCS;TARGET_BACKENDS;DRIVERS;RUNNER_ARGS;LABELS;TARGET_CPU_FEATURES_VARIANTS"
+ "SRCS;TARGET_BACKENDS;DRIVERS;RUNNER_ARGS;LABELS;TARGET_CPU_FEATURES_VARIANTS;TIMEOUT"
${ARGN}
)
@@ -456,6 +460,8 @@
${_RULE_LABELS}
TARGET_CPU_FEATURES
${_TARGET_CPU_FEATURES}
+ TIMEOUT
+ ${_RULE_TIMEOUT}
)
endforeach()
endforeach()
diff --git a/build_tools/cmake/iree_lit_test.cmake b/build_tools/cmake/iree_lit_test.cmake
index b983800..9b5aa9a 100644
--- a/build_tools/cmake/iree_lit_test.cmake
+++ b/build_tools/cmake/iree_lit_test.cmake
@@ -35,7 +35,7 @@
_RULE
""
"NAME;TEST_FILE"
- "DATA;TOOLS;LABELS"
+ "DATA;TOOLS;LABELS;TIMEOUT"
${ARGN}
)
@@ -84,6 +84,7 @@
"TEST_TMPDIR=${IREE_BINARY_DIR}/tmp/${_NAME}_test_tmpdir"
"LIT_OPTS=-v"
"FILECHECK_OPTS=--enable-var-scope")
+ set_property(TEST ${_NAME_PATH} PROPERTY TIMEOUT ${_RULE_TIMEOUT})
iree_add_test_environment_properties(${_NAME_PATH})
# TODO(gcmn): Figure out how to indicate a dependency on _RULE_DATA being built
@@ -116,10 +117,14 @@
_RULE
""
"NAME"
- "SRCS;DATA;TOOLS;LABELS"
+ "SRCS;DATA;TOOLS;LABELS;TIMEOUT"
${ARGN}
)
+ if (NOT DEFINED _RULE_TIMEOUT)
+ set(_RULE_TIMEOUT 60)
+ endif()
+
foreach(_TEST_FILE ${_RULE_SRCS})
get_filename_component(_TEST_BASENAME ${_TEST_FILE} NAME)
iree_lit_test(
@@ -133,6 +138,8 @@
"${_RULE_TOOLS}"
LABELS
"${_RULE_LABELS}"
+ TIMEOUT
+ ${_RULE_TIMEOUT}
)
endforeach()
endfunction()
diff --git a/build_tools/cmake/iree_native_test.cmake b/build_tools/cmake/iree_native_test.cmake
index dd1d872..2655c2a 100644
--- a/build_tools/cmake/iree_native_test.cmake
+++ b/build_tools/cmake/iree_native_test.cmake
@@ -56,7 +56,7 @@
_RULE
""
"NAME;SRC;DRIVER;TEST_INPUT_FILE_ARG"
- "ARGS;LABELS;DATA"
+ "ARGS;LABELS;DATA;TIMEOUT"
${ARGN}
)
@@ -126,7 +126,12 @@
iree_add_test_environment_properties(${_TEST_NAME})
endif()
+ if (NOT DEFINED _RULE_TIMEOUT)
+ set(_RULE_TIMEOUT 60)
+ endif()
+
list(APPEND _RULE_LABELS "${_PACKAGE_PATH}")
set_property(TEST ${_TEST_NAME} PROPERTY LABELS "${_RULE_LABELS}")
set_property(TEST "${_TEST_NAME}" PROPERTY REQUIRED_FILES "${_RULE_DATA}")
+ set_property(TEST ${_TEST_NAME} PROPERTY TIMEOUT ${_RULE_ARGS})
endfunction()
diff --git a/build_tools/cmake/iree_python.cmake b/build_tools/cmake/iree_python.cmake
index a2967ad..f1afd53 100644
--- a/build_tools/cmake/iree_python.cmake
+++ b/build_tools/cmake/iree_python.cmake
@@ -208,7 +208,7 @@
_RULE
"GENERATED_IN_BINARY_DIR"
"NAME;SRC"
- "ARGS;LABELS;PACKAGE_DIRS"
+ "ARGS;LABELS;PACKAGE_DIRS;TIMEOUT"
${ARGN}
)
@@ -238,7 +238,12 @@
list(APPEND _RULE_PACKAGE_DIRS "$ENV{PYTHONPATH}")
string(JOIN ":" _PYTHONPATH ${_RULE_PACKAGE_DIRS})
+ if (NOT DEFINED _RULE_TIMEOUT)
+ set(_RULE_TIMEOUT 60)
+ endif()
+
set_property(TEST ${_NAME_PATH} PROPERTY LABELS "${_RULE_LABELS}")
+ set_property(TEST ${_NAME_PATH} PROPERTY TIMEOUT ${_RULE_ARGS})
set_property(TEST ${_NAME_PATH} PROPERTY ENVIRONMENT
"PYTHONPATH=${_PYTHONPATH}"
"TEST_TMPDIR=${IREE_BINARY_DIR}/tmp/${_NAME}_test_tmpdir"
@@ -266,7 +271,7 @@
_RULE
"GENERATED_IN_BINARY_DIR"
"NAME;SRCS"
- "ARGS;LABELS"
+ "ARGS;LABELS;TIMEOUT"
${ARGN}
)
@@ -284,5 +289,7 @@
"${IREE_BINARY_DIR}/runtime/bindings/python"
GENERATED_IN_BINARY_DIR
"${_RULE_GENERATED_IN_BINARY_DIR}"
+ TIMEOUT
+ ${_RULE_TIMEOUT}
)
endfunction()
