CTest support for integration tests (#5101)
Add CTest support for integration tests. This is our interim solution
for integration testing support while we find a long-term solution. It
results from tests that were previously Bazel-only (because TF) and now
are CMake-only (because maintaining support for Python in Bazel was
determined to be not worth the effort).
The approach is to open-source our internal BUILD files for Blaze
(Google's internal version of Bazel, which happens to do some rather
different things with Python). These rely on a Starlark rule for
expanding a test matrix that is pretty painful to implement in CMake.
The author apologizes.
This enables 1317 new test targets.
Incidentally, it makes the tests run in parallel, which should speed
things up quite a bit...
diff --git a/build_tools/bazel_to_cmake/bazel_to_cmake_converter.py b/build_tools/bazel_to_cmake/bazel_to_cmake_converter.py
index 6a5e852..72ce425 100644
--- a/build_tools/bazel_to_cmake/bazel_to_cmake_converter.py
+++ b/build_tools/bazel_to_cmake/bazel_to_cmake_converter.py
@@ -190,6 +190,28 @@
return _convert_string_list_block(list_name, targets, sort=True, quote=False)
+# Copied from integrations/tensorflow/e2e/iree_e2e_cartesian_product_test_suite.bzl
+def _normalize_dictionary(dictionary):
+ """Wraps every value of dictionary in a list if it isn't one already."""
+ for key, value in dictionary.items():
+ if type(value) != type([]):
+ dictionary[key] = [value]
+ return dictionary
+
+
+def _dictionary_product(dictionary):
+ """Returns a named cartesian product of dictionary's values."""
+
+ # Converts {'a': [1, 2], 'b': [3, 4]} into
+ # [{'a': 1, 'b': 3}, {'a': 1, 'b': 4}, {'a': 2, 'b': 3}, {'a': 2, 'b': 4}]
+ product = [[]]
+ for values in dictionary.values():
+ # Iteratively grow the elements of the product.
+ product = [element + [value] for element in product for value in values]
+ dicts = [{k: v for k, v in zip(dictionary, element)} for element in product]
+ return dicts
+
+
class BuildFileFunctions(object):
"""Object passed to `exec` that has handlers for BUILD file functions."""
@@ -235,6 +257,11 @@
def exports_files(self, *args, **kwargs):
pass
+ # Technically we could do something with a CMake equivalent but we have no use
+ # case.
+ def py_binary(self, *args, **kwargs):
+ pass
+
def filegroup(self, name, **kwargs):
# Not implemented yet. Might be a no-op, or may want to evaluate the srcs
# attribute and pass them along to any targets that depend on the filegroup.
@@ -581,6 +608,64 @@
f"{labels_block}"
f")\n\n")
+ def iree_e2e_cartesian_product_test_suite(self,
+ name,
+ matrix,
+ failing_configurations=None,
+ tags=None,
+ data=None,
+ **kwargs):
+ # Note kwargs deps, size, python_version are unused
+ if data is not None:
+ self._convert_unimplemented_function(
+ "iree_e2e_cartesian_product_test_suite", name + " has data")
+
+ matrix_keys = matrix.keys()
+
+ name_block = _convert_string_arg_block("NAME", name, quote=False)
+ matrix_keys_block = _convert_string_list_block("MATRIX_KEYS", matrix_keys)
+ labels_block = _convert_string_list_block("LABELS", tags)
+
+ value_strings = []
+ for key in matrix_keys:
+ # ensure matching order
+ values = matrix[key]
+ if not isinstance(values, list):
+ values = [values]
+ if not values:
+ self._convert_unimplemented_function(
+ "iree_e2e_cartesian_product_test_suite",
+ name + f" has empty list for matrix key {key}")
+ value_strings.append(";".join(str(value) for value in values))
+ matrix_values_block = _convert_string_list_block("MATRIX_VALUES",
+ value_strings)
+
+ # Copied from integrations/tensorflow/e2e/iree_e2e_cartesian_product_test_suite.bzl
+ failing_configurations_block = ""
+ if failing_configurations is not None:
+ failing_matrix_configurations = []
+ for failing_configuration in failing_configurations:
+ failing_configuration = _normalize_dictionary(failing_configuration)
+
+ failing_matrix_configurations.extend(
+ _dictionary_product(failing_configuration))
+
+ failing_configuration_strings = []
+ for failing_configuration in failing_matrix_configurations:
+ failing_config_string = ",".join(
+ str(failing_configuration.get(key, "")) for key in matrix_keys)
+ failing_configuration_strings.append(failing_config_string)
+ failing_configurations_block = _convert_string_list_block(
+ "FAILING_CONFIGURATIONS", failing_configuration_strings)
+
+ self.converter.body += (f"iree_e2e_cartesian_product_test_suite(\n"
+ f"{name_block}"
+ f"{matrix_keys_block}"
+ f"{matrix_values_block}"
+ f"{failing_configurations_block}"
+ f"{labels_block}"
+ f")\n\n")
+
def run_binary_test(self, name, test_binary, args=None, data=None):
if data is not None:
self._convert_unimplemented_function("iree_run_binary_test",
diff --git a/build_tools/cmake/iree_python.cmake b/build_tools/cmake/iree_python.cmake
index 55767f7..71d39d3 100644
--- a/build_tools/cmake/iree_python.cmake
+++ b/build_tools/cmake/iree_python.cmake
@@ -243,7 +243,7 @@
# Parameters:
# NAME: name of test
# SRCS: Test source file
-# DEPS: List of deps the test requires
+# ARGS: Command line arguments to the Python source file.
# LABELS: Additional labels to apply to the test. The package path is added
# automatically.
function(iree_py_test)
@@ -255,7 +255,7 @@
_RULE
""
"NAME;SRCS"
- "DEPS;LABELS"
+ "ARGS;LABELS"
${ARGN}
)
@@ -276,6 +276,7 @@
"${CMAKE_SOURCE_DIR}/build_tools/cmake/run_test.${IREE_HOST_SCRIPT_EXT}"
"${Python3_EXECUTABLE}"
"${CMAKE_CURRENT_SOURCE_DIR}/${_RULE_SRCS}"
+ ${_RULE_ARGS}
INSTALLED_COMMAND
python
"${_PACKAGE_PATH}/${_RULE_SRCS}"
diff --git a/build_tools/kokoro/gcp_ubuntu/cmake-bazel/linux/x86-swiftshader/build.sh b/build_tools/kokoro/gcp_ubuntu/cmake-bazel/linux/x86-swiftshader/build.sh
index 97305f9..ec73635 100755
--- a/build_tools/kokoro/gcp_ubuntu/cmake-bazel/linux/x86-swiftshader/build.sh
+++ b/build_tools/kokoro/gcp_ubuntu/cmake-bazel/linux/x86-swiftshader/build.sh
@@ -67,6 +67,7 @@
cd "${CMAKE_BUILD_DIR?}"
ninja
+export CTEST_PARALLEL_LEVEL=${CTEST_PARALLEL_LEVEL:-$(nproc)}
+
echo "Testing with CTest"
-ctest -R 'tensorflow_e2e|bindings/python|integrations/tensorflow/' \
- --output-on-failure
+ctest --output-on-failure -L 'integrations/tensorflow' --label-exclude "^nokokoro$"
diff --git a/build_tools/kokoro/gcp_ubuntu/cmake-bazel/linux/x86-turing/build.sh b/build_tools/kokoro/gcp_ubuntu/cmake-bazel/linux/x86-turing/build.sh
index 85f7e14..a7d5973 100755
--- a/build_tools/kokoro/gcp_ubuntu/cmake-bazel/linux/x86-turing/build.sh
+++ b/build_tools/kokoro/gcp_ubuntu/cmake-bazel/linux/x86-turing/build.sh
@@ -70,6 +70,7 @@
cd "${CMAKE_BUILD_DIR?}"
ninja
+export CTEST_PARALLEL_LEVEL=${CTEST_PARALLEL_LEVEL:-$(nproc)}
+
echo "Testing with CTest"
-ctest -R 'tensorflow_e2e|bindings/python|integrations/tensorflow/' \
- --output-on-failure
+ctest --output-on-failure -L 'integrations/tensorflow' --label-exclude "^nokokoro$"
