Filter CMake tests by label
Exclude CMake tests by label instead of name. This allows us to do label-based exclusions like "driver=vulkan". Includes fixing the labels and test names for python test to match what we have in core and using bazel_to_cmake to propagate bazel tags as labels for tests. Technically you can label targets in CMake too, but I haven't seen any actual use case for that.
Needed for https://github.com/google/iree/issues/1308
Closes https://github.com/google/iree/pull/1332
COPYBARA_INTEGRATE_REVIEW=https://github.com/google/iree/pull/1332 from GMNGeoffrey:cmake-tags-labels 0e8815f9de6b3356a65794dbd42e85e62e8b8289
PiperOrigin-RevId: 304235613
diff --git a/bindings/python/build_tools/cmake/iree_py_test.cmake b/bindings/python/build_tools/cmake/iree_py_test.cmake
index 78e1ccd..f6e484e 100644
--- a/bindings/python/build_tools/cmake/iree_py_test.cmake
+++ b/bindings/python/build_tools/cmake/iree_py_test.cmake
@@ -22,6 +22,8 @@
# NAME: name of test
# SRCS: List of source file
# DEPS: List of deps the test requires
+# LABELS: Additional labels to apply to the test. The package path is added
+# automatically.
function(iree_py_test)
if(NOT IREE_BUILD_TESTS)
@@ -32,19 +34,25 @@
_RULE
""
"NAME"
- "SRCS;DEPS"
+ "SRCS;DEPS;LABELS"
${ARGN}
)
iree_package_name(_PACKAGE_NAME)
set(_NAME "${_PACKAGE_NAME}_${_RULE_NAME}")
+ string(REPLACE "_" "/" _PACKAGE_PATH ${_PACKAGE_NAME})
+ set(_NAME_PATH "${_PACKAGE_PATH}:${_RULE_NAME}")
+
add_test(
- NAME ${_NAME}
+ NAME ${_NAME_PATH}
COMMAND ${CMAKE_SOURCE_DIR}/build_tools/cmake/run_test.sh ${PYTHON_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/${_RULE_SRCS}"
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
)
- set_property(TEST ${_NAME} PROPERTY ENVIRONMENT "PYTHONPATH=${CMAKE_BINARY_DIR}/bindings/python:$ENV{PYTHONPATH};TEST_TMPDIR=${_NAME}_test_tmpdir")
+
+ list(APPEND _RULE_LABELS "${_PACKAGE_PATH}")
+ set_property(TEST ${_NAME_PATH} PROPERTY LABELS "${_RULE_LABELS}")
+ set_property(TEST ${_NAME_PATH} PROPERTY ENVIRONMENT "PYTHONPATH=${CMAKE_BINARY_DIR}/bindings/python:$ENV{PYTHONPATH};TEST_TMPDIR=${_NAME}_test_tmpdir")
# TODO(marbre): Find out how to add deps to tests.
# Similar to _RULE_DATA in iree_lit_test().
diff --git a/build_tools/bazel_to_cmake/bazel_to_cmake_converter.py b/build_tools/bazel_to_cmake/bazel_to_cmake_converter.py
index 9bea5ae..9aeca7e 100644
--- a/build_tools/bazel_to_cmake/bazel_to_cmake_converter.py
+++ b/build_tools/bazel_to_cmake/bazel_to_cmake_converter.py
@@ -83,9 +83,6 @@
values_list = "\n".join([f' "{v}"' for v in values])
return f" {name}\n{values_list}\n"
- def _convert_flags_block(self, flags):
- return self._convert_string_list_block("FLAGS", flags)
-
def _convert_translate_tool_block(self, translate_tool):
if translate_tool and translate_tool != "//iree/tools:iree-translate":
# Bazel `//iree/base` -> CMake `iree::base`
@@ -329,12 +326,20 @@
f"{testonly_block}"
f" PUBLIC\n)\n\n")
- def cc_test(self, name, hdrs=None, srcs=None, data=None, deps=None, **kwargs):
+ def cc_test(self,
+ name,
+ hdrs=None,
+ srcs=None,
+ data=None,
+ deps=None,
+ tags=None,
+ **kwargs):
name_block = self._convert_name_block(name)
hdrs_block = self._convert_hdrs_block(hdrs)
srcs_block = self._convert_srcs_block(srcs)
data_block = self._convert_data_block(data)
deps_block = self._convert_deps_block(deps)
+ labels_block = self._convert_string_list_block("LABELS", tags)
self.converter.body += (f"iree_cc_test(\n"
f"{name_block}"
@@ -342,6 +347,7 @@
f"{srcs_block}"
f"{data_block}"
f"{deps_block}"
+ f"{labels_block}"
f")\n\n")
def cc_binary(self,
@@ -418,7 +424,7 @@
src_block = self._convert_src_block(src)
namespace_block = self._convert_cc_namespace_block(cc_namespace)
translate_tool_block = self._convert_translate_tool_block(translate_tool)
- flags_block = self._convert_flags_block(flags)
+ flags_block = self._convert_string_list_block("FLAGS", flags)
self.converter.body += (f"iree_bytecode_module(\n"
f"{name_block}"
@@ -480,15 +486,17 @@
f"{tblgen_block}"
f")\n\n")
- def iree_lit_test_suite(self, name, srcs, data, **kwargs):
+ def iree_lit_test_suite(self, name, srcs, data, tags=None, **kwargs):
name_block = self._convert_name_block(name)
srcs_block = self._convert_srcs_block(srcs)
data_block = self._convert_data_block(data)
+ labels_block = self._convert_string_list_block("LABELS", tags)
self.converter.body += (f"iree_lit_test_suite(\n"
f"{name_block}"
f"{srcs_block}"
f"{data_block}"
+ f"{labels_block}"
f")\n\n")
def iree_check_test_suite(self,
@@ -496,6 +504,7 @@
srcs=None,
target_backends_and_drivers=None,
args=None,
+ tags=None,
**kwargs):
name_block = self._convert_name_block(name)
srcs_block = self._convert_srcs_block(srcs)
@@ -508,6 +517,7 @@
"TARGET_BACKENDS", target_backends)
drivers_block = self._convert_string_list_block("DRIVERS", drivers)
args_block = self._convert_string_list_block("ARGS", args)
+ labels_block = self._convert_string_list_block("LABELS", tags)
self.converter.body += (f"iree_check_test_suite(\n"
f"{name_block}"
@@ -515,6 +525,7 @@
f"{target_backends_block}"
f"{drivers_block}"
f"{args_block}"
+ f"{labels_block}"
f")\n\n")
diff --git a/build_tools/cmake/iree_cc_test.cmake b/build_tools/cmake/iree_cc_test.cmake
index b41a57e..75a60af 100644
--- a/build_tools/cmake/iree_cc_test.cmake
+++ b/build_tools/cmake/iree_cc_test.cmake
@@ -26,6 +26,8 @@
# COPTS: List of private compile options
# DEFINES: List of public defines
# LINKOPTS: List of link options
+# LABELS: Additional labels to apply to the test. The package path is added
+# automatically.
#
# Note:
# By default, iree_cc_test will always create a binary named iree_${NAME}.
@@ -60,7 +62,7 @@
_RULE
""
"NAME"
- "SRCS;COPTS;DEFINES;LINKOPTS;DATA;DEPS"
+ "SRCS;COPTS;DEFINES;LINKOPTS;DATA;DEPS;LABELS"
${ARGN}
)
@@ -117,6 +119,7 @@
WORKING_DIRECTORY
"${CMAKE_BINARY_DIR}"
)
+ list(APPEND _RULE_LABELS "${_PACKAGE_PATH}")
set_property(TEST ${_NAME_PATH} PROPERTY ENVIRONMENT "TEST_TMPDIR=${_NAME}_test_tmpdir")
- set_property(TEST ${_NAME_PATH} PROPERTY LABELS "${_PACKAGE_PATH}")
+ set_property(TEST ${_NAME_PATH} PROPERTY LABELS "${_RULE_LABELS}")
endfunction()
diff --git a/build_tools/cmake/iree_check_test.cmake b/build_tools/cmake/iree_check_test.cmake
index eaf64b9..c6afa4e 100644
--- a/build_tools/cmake/iree_check_test.cmake
+++ b/build_tools/cmake/iree_check_test.cmake
@@ -21,17 +21,19 @@
# Mirrors the bzl rule of the same name.
#
# Parameters:
-# NAME: Name of the target
-# SRC: mlir source file to be compiled to an IREE module.
-# TARGET_BACKEND: target backend to compile for.
-# DRIVER: driver to run the module with.
-# ARGS: additional args to pass to iree-check-module. The driver and input
+# NAME: Name of the target
+# SRC: mlir source file to be compiled to an IREE module.
+# TARGET_BACKEND: target backend to compile for.
+# DRIVER: driver to run the module with.
+# ARGS: additional args to pass to iree-check-module. The driver and input
# file are passed automatically.
+# LABELS: Additional labels to apply to the test. The package path and
+# "driver=${DRIVER}" are added automatically.
function(iree_check_test)
cmake_parse_arguments(
_RULE
""
- "NAME;SRC;TARGET_BACKEND;DRIVER"
+ "NAME;SRC;TARGET_BACKEND;DRIVER;LABELS"
"ARGS"
${ARGN}
)
@@ -93,9 +95,10 @@
${_RULE_ARGS}
)
+ list(APPEND _RULE_LABELS "${_PACKAGE_PATH}" "driver=${_RULE_DRIVER}")
set_property(TEST "${_NAME_PATH}" PROPERTY REQUIRED_FILES "${_MODULE_FILE_NAME}")
set_property(TEST "${_NAME_PATH}" PROPERTY ENVIRONMENT "TEST_TMPDIR=${_NAME}_test_tmpdir")
- set_property(TEST "${_NAME_PATH}" PROPERTY LABELS "${_PACKAGE_PATH}")
+ set_property(TEST "${_NAME_PATH}" PROPERTY LABELS "${_RULE_LABELS}")
endfunction()
@@ -120,12 +123,14 @@
# ARGS: additional args to pass to the underlying iree-check-module tests. The
# driver and input file are passed automatically. To use different args per
# test, create a separate suite or iree_check_test.
+# LABELS: Additional labels to apply to the generated tests. The package path is
+# added automatically.
function(iree_check_test_suite)
cmake_parse_arguments(
_RULE
""
"NAME"
- "SRCS;TARGET_BACKENDS;DRIVERS;ARGS"
+ "SRCS;TARGET_BACKENDS;DRIVERS;ARGS;LABELS"
${ARGN}
)
if(NOT IREE_BUILD_TESTS)
@@ -162,6 +167,8 @@
${_DRIVER}
ARGS
${_RULE_ARGS}
+ LABELS
+ ${_RULE_LABELS}
)
endforeach()
endforeach()
diff --git a/build_tools/cmake/iree_lit_test.cmake b/build_tools/cmake/iree_lit_test.cmake
index 88a3a72..bdcde49 100644
--- a/build_tools/cmake/iree_lit_test.cmake
+++ b/build_tools/cmake/iree_lit_test.cmake
@@ -25,6 +25,8 @@
# TEST_FILE: Test file to run with the lit runner.
# DATA: Additional data dependencies invoked by the test (e.g. binaries
# called in the RUN line)
+# LABELS: Additional labels to apply to the test. The package path is added
+# automatically.
#
# TODO(gcmn): allow using alternative driver
# A driver other than the default iree/tools/run_lit.sh is not currently supported.
@@ -33,7 +35,7 @@
_RULE
""
"NAME;TEST_FILE"
- "DATA"
+ "DATA;LABELS"
${ARGN}
)
if(NOT IREE_BUILD_TESTS)
@@ -70,8 +72,10 @@
WORKING_DIRECTORY
"${CMAKE_BINARY_DIR}"
)
+
+ list(APPEND _RULE_LABELS "${_PACKAGE_PATH}")
set_property(TEST ${_NAME_PATH} PROPERTY ENVIRONMENT "TEST_TMPDIR=${_NAME}_test_tmpdir")
- set_property(TEST ${_NAME_PATH} PROPERTY LABELS "${_PACKAGE_PATH}")
+ set_property(TEST ${_NAME_PATH} PROPERTY LABELS "${_RULE_LABELS}")
set_property(TEST ${_NAME_PATH} PROPERTY REQUIRED_FILES "${_TEST_FILE_PATH}")
# TODO(gcmn): Figure out how to indicate a dependency on _RULE_DATA being built
@@ -89,6 +93,8 @@
# SRCS: List of test files to run with the lit runner. Creates one test per source.
# DATA: Additional data dependencies invoked by the test (e.g. binaries
# called in the RUN line)
+# LABELS: Additional labels to apply to the generated tests. The package path is
+# added automatically.
#
# TODO(gcmn): allow using alternative driver
# A driver other than the default iree/tools/run_lit.sh is not currently supported.
@@ -97,7 +103,7 @@
_RULE
""
"NAME"
- "SRCS;DATA"
+ "SRCS;DATA;LABELS"
${ARGN}
)
IF(NOT IREE_BUILD_TESTS)
@@ -113,6 +119,8 @@
"${_TEST_FILE}"
DATA
"${_RULE_DATA}"
+ LABELS
+ "${_RULE_LABELS}"
)
endforeach()
endfunction()
diff --git a/build_tools/cmake/test.sh b/build_tools/cmake/test.sh
index 7c57700..83d2176 100755
--- a/build_tools/cmake/test.sh
+++ b/build_tools/cmake/test.sh
@@ -27,23 +27,33 @@
export IREE_LLVMJIT_DISABLE=${IREE_LLVMJIT_DISABLE:-1}
export IREE_VULKAN_DISABLE=${IREE_VULKAN_DISABLE:-1}
-EXCLUDED_TESTS=(
- iree/compiler/Translation/SPIRV/LinalgToSPIRV/test:pw_add.mlir.test
- iree/hal/vulkan:dynamic_symbols_test
- iree/test/e2e/xla:rem.mlir.test
- bindings_python_pyiree_rt_function_abi_test
- bindings_python_pyiree_rt_system_api_test
- bindings_python_pyiree_rt_vm_test
- bindings_python_pyiree_rt_hal_test # TODO: Enable after the VM is fixed
- bindings_python_pyiree_compiler_compiler_test # TODO: Enable after the VM is fixed
- # TODO(b/146898896) get label-based exclusions working
- iree/modules/check/test:check_success.mlir_vulkan-spirv_vulkan
- iree/modules/check/test:check_failure_failure.mlir_vulkan-spirv_vulkan
+# Tests to exclude by label. In addition to any custom labels (which are carried
+# over from Bazel tags), every test should be labeled with the directory it is in.
+declare -a label_exclude_args=(
+ # Exclude specific labels.
+ # Put the whole label with anchors for exact matches.
+ # For example:
+ # ^driver=vulkan$
+ ^driver=vulkan$
+ ^nokokoro$
+
+ # TODO(b/151445957) Enable the python tests when the Kokoro VMs support them.
+ # See also, https://github.com/google/iree/issues/1346.
+ # Exclude all tests in a directory.
+ # Put the whole directory with anchors for exact matches.
+ # For example:
+ # ^bindings/python/pyiree/rt$
+ ^bindings$
+ # Exclude all tests in some subdirectories.
+ # Put the whole parent directory with only a starting anchor.
+ # Use a trailing slash to avoid prefix collisions.
+ # For example:
+ # ^bindings/
+ ^bindings/
)
-# Join with | and add anchors
-EXCLUDED_REGEX="^($(IFS="|" ; echo "${EXCLUDED_TESTS[*]?}"))$"
+# Join on "|"
+label_exclude_regex="($(IFS="|" ; echo "${label_exclude_args[*]?}"))"
cd ${ROOT_DIR?}/build
-ctest -E "${EXCLUDED_REGEX?}"
-
+ctest --label-exclude "${label_exclude_regex?}"
diff --git a/iree/hal/vulkan/CMakeLists.txt b/iree/hal/vulkan/CMakeLists.txt
index 5a5881f..5ab49cb 100644
--- a/iree/hal/vulkan/CMakeLists.txt
+++ b/iree/hal/vulkan/CMakeLists.txt
@@ -195,6 +195,8 @@
iree::hal::vulkan::status_util
iree::hal::vulkan::dynamic_symbols
iree::testing::gtest_main
+ LABELS
+ "nokokoro"
)
iree_cc_library(
diff --git a/iree/modules/check/test/CMakeLists.txt b/iree/modules/check/test/CMakeLists.txt
index 671e989..6368d05 100644
--- a/iree/modules/check/test/CMakeLists.txt
+++ b/iree/modules/check/test/CMakeLists.txt
@@ -24,6 +24,8 @@
iree::modules::check::iree-check-module
iree::tools::IreeFileCheck
iree::tools::iree-translate
+ LABELS
+ "hostonly"
)
iree_check_test_suite(
diff --git a/iree/samples/simple_embedding/CMakeLists.txt b/iree/samples/simple_embedding/CMakeLists.txt
index 5dd80be..9026553 100644
--- a/iree/samples/simple_embedding/CMakeLists.txt
+++ b/iree/samples/simple_embedding/CMakeLists.txt
@@ -47,4 +47,7 @@
iree::testing::gtest_main
iree::vm
iree::vm::bytecode_module
+ LABELS
+ "noga"
+ "nokokoro"
)
diff --git a/iree/tools/test/CMakeLists.txt b/iree/tools/test/CMakeLists.txt
index 40e84f8..c4a9d23 100644
--- a/iree/tools/test/CMakeLists.txt
+++ b/iree/tools/test/CMakeLists.txt
@@ -26,4 +26,6 @@
iree::tools::iree-run-mlir
iree::tools::iree-run-module
iree::tools::iree-translate
+ LABELS
+ "hostonly"
)
