[CMake] Propagate WILL_FAIL through test rules (#24707)
This adds `WILL_FAIL` support to the CMake test wrappers that did not
expose it and forwards the value to the generated CTest entries.
For example, a backend-specific Python test for a known failing path can
now be declared directly:
```cmake
iree_py_test(
NAME
known_failure_cuda
SRCS
"known_failure_test.py"
ARGS
"--target_backend=cuda"
"--driver=cuda"
WILL_FAIL
TRUE
)
```
If the test exits with a non-zero status, CTest reports it as passing.
If it unexpectedly exits successfully, CTest reports a failure.
Previously, callers had to find the generated CTest name and set the
property manually with `set_tests_properties`. Suite wrappers also pass
`WILL_FAIL` to each test they generate.
Fixes: https://github.com/iree-org/iree/issues/13007
Assisted-by: OpenAI
Signed-off-by: zhangpan <98025960+zhangpan2001@users.noreply.github.com>diff --git a/build_tools/cmake/iree_cc_binary_benchmark.cmake b/build_tools/cmake/iree_cc_binary_benchmark.cmake
index a0d6564..923b7ec 100644
--- a/build_tools/cmake/iree_cc_binary_benchmark.cmake
+++ b/build_tools/cmake/iree_cc_binary_benchmark.cmake
@@ -28,6 +28,7 @@
# cross-compiling
# LABELS: Additional labels to apply to the test. The package path is added
# automatically.
+# WILL_FAIL: The test will run, but its pass/fail status will be inverted.
#
# )
@@ -36,7 +37,7 @@
cmake_parse_arguments(
_RULE
"HOSTONLY;TESTONLY"
- "NAME"
+ "NAME;WILL_FAIL"
"SRCS;COPTS;DEFINES;LINKOPTS;DATA;DEPS;LABELS"
${ARGN}
)
@@ -78,5 +79,7 @@
::${_RULE_NAME}
LABELS
${_RULE_LABELS}
+ WILL_FAIL
+ ${_RULE_WILL_FAIL}
)
endfunction()
diff --git a/build_tools/cmake/iree_cc_test.cmake b/build_tools/cmake/iree_cc_test.cmake
index 2ce0514..caed124 100644
--- a/build_tools/cmake/iree_cc_test.cmake
+++ b/build_tools/cmake/iree_cc_test.cmake
@@ -24,6 +24,7 @@
# GROUP: Optional test group to add the target to.
# LABELS: Additional labels to apply to the test. The package path is added
# automatically.
+# WILL_FAIL: The test will run, but its pass/fail status will be inverted.
# RESOURCE_GROUP: If set, tests sharing the same RESOURCE_GROUP name will not
# run concurrently under CTest. Maps to CTest's RESOURCE_LOCK property.
# Use for tests that compete for shared system resources (e.g.,
@@ -62,7 +63,7 @@
cmake_parse_arguments(
_RULE
""
- "NAME;RESOURCE_GROUP"
+ "NAME;RESOURCE_GROUP;WILL_FAIL"
"ARGS;SRCS;COPTS;DEFINES;LINKOPTS;DATA;DEPS;LABELS;GROUP;TIMEOUT"
${ARGN}
)
@@ -247,6 +248,9 @@
list(APPEND _RULE_LABELS "${_PACKAGE_PATH}")
set_property(TEST ${_NAME_PATH} PROPERTY LABELS "${_RULE_LABELS}")
set_property(TEST ${_NAME_PATH} PROPERTY TIMEOUT ${_RULE_TIMEOUT})
+ if(_RULE_WILL_FAIL)
+ set_property(TEST ${_NAME_PATH} PROPERTY WILL_FAIL ${_RULE_WILL_FAIL})
+ endif()
if(_RULE_RESOURCE_GROUP)
set_property(TEST ${_NAME_PATH} PROPERTY RESOURCE_LOCK "${_RULE_RESOURCE_GROUP}")
diff --git a/build_tools/cmake/iree_check_test.cmake b/build_tools/cmake/iree_check_test.cmake
index 72582ad..4b7b547 100644
--- a/build_tools/cmake/iree_check_test.cmake
+++ b/build_tools/cmake/iree_check_test.cmake
@@ -37,6 +37,7 @@
# and input file are passed automatically.
# LABELS: Additional labels to apply to the test. The package path and
# "driver=${DRIVER}" are added automatically.
+# WILL_FAIL: The test will run, but its pass/fail status will be inverted.
# MODULE_FILE_NAME: Optional, specifies the absolute path to the filename
# to use for the generated IREE module (.vmfb).
# DEPENDS: Optional. Additional dependencies beyond SRC and the tools.
@@ -50,7 +51,7 @@
cmake_parse_arguments(
_RULE
""
- "NAME;SRC;TARGET_BACKEND;DRIVER;MODULE_FILE_NAME;INPUT_TYPE"
+ "NAME;SRC;TARGET_BACKEND;DRIVER;MODULE_FILE_NAME;INPUT_TYPE;WILL_FAIL"
"COMPILER_FLAGS;RUNNER_ARGS;LABELS;DEPENDS;TIMEOUT"
${ARGN}
)
@@ -213,6 +214,8 @@
${_RULE_LABELS}
TIMEOUT
${_RULE_TIMEOUT}
+ WILL_FAIL
+ ${_RULE_WILL_FAIL}
DISABLED
${_TEST_DISABLED}
)
@@ -240,6 +243,8 @@
# different args per test, create a separate suite or iree_check_test.
# LABELS: Additional labels to apply to the generated tests. The package path
# is added automatically.
+# WILL_FAIL: The generated tests will run, but their pass/fail status will be
+# inverted.
# DEPENDS: Optional. Additional dependencies beyond SRC and the tools.
# INPUT_TYPE: The value for the --iree-input-type= flag. Also disables tests
# if no compiled support for that configuration.
@@ -251,7 +256,7 @@
cmake_parse_arguments(
_RULE
""
- "NAME;TARGET_BACKEND;DRIVER;INPUT_TYPE"
+ "NAME;TARGET_BACKEND;DRIVER;INPUT_TYPE;WILL_FAIL"
"SRCS;COMPILER_FLAGS;RUNNER_ARGS;LABELS;DEPENDS;TIMEOUT"
${ARGN}
)
@@ -282,6 +287,7 @@
LABELS ${_RULE_LABELS}
DEPENDS ${_RULE_DEPENDS}
TIMEOUT ${_RULE_TIMEOUT}
+ WILL_FAIL ${_RULE_WILL_FAIL}
)
endif()
@@ -306,6 +312,7 @@
LABELS ${_RULE_LABELS}
DEPENDS ${_RULE_DEPENDS}
TIMEOUT ${_RULE_TIMEOUT}
+ WILL_FAIL ${_RULE_WILL_FAIL}
)
endif()
@@ -324,6 +331,7 @@
LABELS ${_RULE_LABELS}
DEPENDS ${_RULE_DEPENDS}
TIMEOUT ${_RULE_TIMEOUT}
+ WILL_FAIL ${_RULE_WILL_FAIL}
)
endif()
endif()
@@ -408,6 +416,8 @@
# test, create a separate suite or iree_check_test.
# LABELS: Additional labels to apply to the generated tests. The package path is
# added automatically.
+# WILL_FAIL: The generated tests will run, but their pass/fail status will be
+# inverted.
# TARGET_CPU_FEATURES_VARIANTS: list of target cpu features variants. Each
# entry is either "generic" for the architecture defaults, or "host" for
# the host CPU, or a colon-separated triple "arch:name:cpu_features" where "arch" filters
@@ -426,7 +436,7 @@
cmake_parse_arguments(
_RULE
""
- "NAME;INPUT_TYPE"
+ "NAME;INPUT_TYPE;WILL_FAIL"
"SRCS;TARGET_BACKENDS;DRIVERS;RUNNER_ARGS;LABELS;TARGET_CPU_FEATURES_VARIANTS;TIMEOUT"
${ARGN}
)
@@ -493,6 +503,8 @@
${_LABELS}
TIMEOUT
${_RULE_TIMEOUT}
+ WILL_FAIL
+ ${_RULE_WILL_FAIL}
INPUT_TYPE
${_RULE_INPUT_TYPE}
)
diff --git a/build_tools/cmake/iree_e2e_generated_runner_test.cmake b/build_tools/cmake/iree_e2e_generated_runner_test.cmake
index dd850ca..d46ed60 100644
--- a/build_tools/cmake/iree_e2e_generated_runner_test.cmake
+++ b/build_tools/cmake/iree_e2e_generated_runner_test.cmake
@@ -30,6 +30,7 @@
# TEST_DEFINED: Whether to define a test target.
# TEST_DISABLED: The test target will be skipped and its status will be
# 'Not Run'.
+# WILL_FAIL: The test will run, but its pass/fail status will be inverted.
# TIMEOUT: Test timeout in seconds. Defaults to the iree_native_test default.
function(iree_e2e_runner_test)
if(NOT IREE_BUILD_TESTS)
@@ -44,7 +45,7 @@
cmake_parse_arguments(
_RULE
""
- "NAME;TEST_TYPE;VARIANT_NAME;TESTS_SRC;TESTS_VMFB;CALLS_SRC;CALLS_VMFB;TRACE;TARGET_BACKEND;DRIVER;TEST_RUNNER;TEST_DEFINED;TEST_DISABLED;TIMEOUT"
+ "NAME;TEST_TYPE;VARIANT_NAME;TESTS_SRC;TESTS_VMFB;CALLS_SRC;CALLS_VMFB;TRACE;TARGET_BACKEND;DRIVER;TEST_RUNNER;TEST_DEFINED;TEST_DISABLED;TIMEOUT;WILL_FAIL"
"COMPILER_FLAGS;RUNNER_ARGS;LABELS"
${ARGN}
)
@@ -127,6 +128,8 @@
${_RULE_LABELS}
DISABLED
${_RULE_TEST_DISABLED}
+ WILL_FAIL
+ ${_RULE_WILL_FAIL}
TIMEOUT
${_RULE_TIMEOUT}
)
@@ -155,6 +158,8 @@
# LABELS: Additional labels to apply to the test. The package path and
# "driver=${DRIVER}" are added automatically.
# TEST_RUNNER: trace-runner program to run.
+# WILL_FAIL: The generated tests will run, but their pass/fail status will be
+# inverted.
# TIMEOUT: Test timeout in seconds. Defaults to the iree_native_test default.
function(iree_single_backend_e2e_runner_test)
if(NOT IREE_BUILD_TESTS)
@@ -168,7 +173,7 @@
cmake_parse_arguments(
_RULE
""
- "NAME;TEST_TYPE;GENERATOR;TARGET_BACKEND;DRIVER;TEST_RUNNER;TIMEOUT"
+ "NAME;TEST_TYPE;GENERATOR;TARGET_BACKEND;DRIVER;TEST_RUNNER;TIMEOUT;WILL_FAIL"
"GENERATOR_ARGS;COMPILER_FLAGS;RUNNER_ARGS;LABELS"
${ARGN}
)
@@ -317,6 +322,7 @@
LABELS ${_RULE_LABELS}
TEST_DEFINED ${_TEST_DEFINED}
TEST_DISABLED ${_TEST_DISABLED}
+ WILL_FAIL ${_RULE_WILL_FAIL}
TIMEOUT ${_RULE_TIMEOUT}
)
# Note we are relying on the fact that the target created by
@@ -351,6 +357,7 @@
LABELS ${_RULE_LABELS}
TEST_DEFINED ${_TEST_DEFINED}
TEST_DISABLED ${_TEST_DISABLED}
+ WILL_FAIL ${_RULE_WILL_FAIL}
TIMEOUT ${_RULE_TIMEOUT}
)
# Note we are relying on the fact that the target created by
@@ -378,6 +385,7 @@
LABELS ${_RULE_LABELS}
TEST_DEFINED ${_TEST_DEFINED}
TEST_DISABLED ${_TEST_DISABLED}
+ WILL_FAIL ${_RULE_WILL_FAIL}
TIMEOUT ${_RULE_TIMEOUT}
)
# Note we are relying on the fact that the target created by
@@ -421,6 +429,8 @@
# LABELS: Additional labels to apply to the test. The package path and
# "driver=${DRIVER}" are added automatically.
# TEST_RUNNER: trace-runner program to run.
+# WILL_FAIL: The generated tests will run, but their pass/fail status will be
+# inverted.
# TARGET_CPU_FEATURES_VARIANTS: list of target cpu features variants. Each
# entry is either "generic" for the architecture defaults, or "host" for
# the host CPU, or a colon-separated triple "arch:name:cpu_features" where "arch" filters
@@ -438,7 +448,7 @@
cmake_parse_arguments(
_RULE
""
- "NAME;TEST_TYPE;GENERATOR;TEST_RUNNER;TIMEOUT"
+ "NAME;TEST_TYPE;GENERATOR;TEST_RUNNER;TIMEOUT;WILL_FAIL"
"TARGET_BACKENDS;DRIVERS;GENERATOR_ARGS;COMPILER_FLAGS;RUNNER_ARGS;LABELS;TARGET_CPU_FEATURES_VARIANTS"
${ARGN}
)
@@ -511,6 +521,8 @@
${_LABELS}
TIMEOUT
${_RULE_TIMEOUT}
+ WILL_FAIL
+ ${_RULE_WILL_FAIL}
)
endforeach()
endforeach()
diff --git a/build_tools/cmake/iree_hal_cts_test_suite.cmake b/build_tools/cmake/iree_hal_cts_test_suite.cmake
index 0403b2a..cf721df 100644
--- a/build_tools/cmake/iree_hal_cts_test_suite.cmake
+++ b/build_tools/cmake/iree_hal_cts_test_suite.cmake
@@ -158,11 +158,12 @@
# LABELS: Test labels for filtering.
# RESOURCE_GROUP: Optional shared resource group. Tests sharing the same
# resource group do not run concurrently under CTest.
+# WILL_FAIL: The tests will run, but their pass/fail status will be inverted.
function(iree_hal_cts_test_suite)
cmake_parse_arguments(
_RULE
""
- "BACKENDS_LIB;NAME;RESOURCE_GROUP"
+ "BACKENDS_LIB;NAME;RESOURCE_GROUP;WILL_FAIL"
"TESTDATA_LIBS;ARGS;LABELS"
${ARGN}
)
@@ -208,6 +209,11 @@
set(_RESOURCE_GROUP_BLOCK RESOURCE_GROUP "${_RULE_RESOURCE_GROUP}")
endif()
+ set(_WILL_FAIL_BLOCK "")
+ if(_RULE_WILL_FAIL)
+ set(_WILL_FAIL_BLOCK WILL_FAIL "${_RULE_WILL_FAIL}")
+ endif()
+
# Non-executable test categories.
foreach(_CATEGORY buffer command_buffer core file queue)
iree_cc_test(
@@ -219,6 +225,7 @@
${_ARGS_BLOCK}
${_LABELS_BLOCK}
${_RESOURCE_GROUP_BLOCK}
+ ${_WILL_FAIL_BLOCK}
)
endforeach()
@@ -257,6 +264,7 @@
${_ARGS_BLOCK}
${_LABELS_BLOCK}
${_RESOURCE_GROUP_BLOCK}
+ ${_WILL_FAIL_BLOCK}
)
endforeach()
endif()
diff --git a/build_tools/cmake/iree_lit_test.cmake b/build_tools/cmake/iree_lit_test.cmake
index 99dbfaf..e562d46 100644
--- a/build_tools/cmake/iree_lit_test.cmake
+++ b/build_tools/cmake/iree_lit_test.cmake
@@ -18,6 +18,7 @@
# called in the RUN line)
# LABELS: Additional labels to apply to the test. Package path and
# "test-type=lit-test" labels are added automatically.
+# WILL_FAIL: The test will run, but its pass/fail status will be inverted.
function(iree_lit_test)
if(NOT IREE_BUILD_TESTS)
return()
@@ -25,7 +26,7 @@
cmake_parse_arguments(
_RULE
""
- "NAME;TEST_FILE"
+ "NAME;TEST_FILE;WILL_FAIL"
"DATA;TOOLS;LABELS;TIMEOUT"
${ARGN}
)
@@ -80,6 +81,9 @@
"LIT_OPTS=string_prepend:-v "
"FILECHECK_OPTS=string_prepend:--enable-var-scope ")
set_property(TEST ${_NAME_PATH} PROPERTY TIMEOUT ${_RULE_TIMEOUT})
+ if(_RULE_WILL_FAIL)
+ set_property(TEST ${_NAME_PATH} PROPERTY WILL_FAIL ${_RULE_WILL_FAIL})
+ endif()
iree_configure_test(${_NAME_PATH})
# TODO(gcmn): Figure out how to indicate a dependency on _RULE_DATA being built
@@ -99,6 +103,8 @@
# DATA: Additional data dependencies used by the test
# LABELS: Additional labels to apply to the generated tests. The package path is
# added automatically.
+# WILL_FAIL: The generated tests will run, but their pass/fail status will be
+# inverted.
function(iree_lit_test_suite)
if(NOT IREE_BUILD_TESTS)
return()
@@ -111,7 +117,7 @@
cmake_parse_arguments(
_RULE
""
- "NAME"
+ "NAME;WILL_FAIL"
"SRCS;DATA;TOOLS;LABELS;TIMEOUT"
${ARGN}
)
@@ -133,6 +139,8 @@
"${_RULE_TOOLS}"
LABELS
"${_RULE_LABELS}"
+ WILL_FAIL
+ ${_RULE_WILL_FAIL}
TIMEOUT
${_RULE_TIMEOUT}
)
diff --git a/build_tools/cmake/iree_python.cmake b/build_tools/cmake/iree_python.cmake
index 41a0635..abec009 100644
--- a/build_tools/cmake/iree_python.cmake
+++ b/build_tools/cmake/iree_python.cmake
@@ -72,6 +72,7 @@
# ARGS: Command line arguments to the Python source file.
# LABELS: Additional labels to apply to the test. The package path is added
# automatically.
+# WILL_FAIL: The test will run, but its pass/fail status will be inverted.
# GENERATED_IN_BINARY_DIR: If present, indicates that the srcs have been
# in the CMAKE_CURRENT_BINARY_DIR.
# PACKAGE_DIRS: Python package paths to be added to PYTHONPATH.
@@ -83,7 +84,7 @@
cmake_parse_arguments(
_RULE
"GENERATED_IN_BINARY_DIR"
- "NAME;SRC"
+ "NAME;SRC;WILL_FAIL"
"ARGS;LABELS;PACKAGE_DIRS;TIMEOUT"
${ARGN}
)
@@ -115,6 +116,9 @@
set_property(TEST ${_NAME_PATH} PROPERTY LABELS "${_RULE_LABELS}")
set_property(TEST ${_NAME_PATH} PROPERTY TIMEOUT ${_RULE_TIMEOUT})
+ if(_RULE_WILL_FAIL)
+ set_property(TEST ${_NAME_PATH} PROPERTY WILL_FAIL ${_RULE_WILL_FAIL})
+ endif()
# Extend the PYTHONPATH environment variable with _RULE_PACKAGE_DIRS.
list(APPEND _RULE_PACKAGE_DIRS "$ENV{PYTHONPATH}")
@@ -145,13 +149,14 @@
# ARGS: Command line arguments to the Python source file.
# LABELS: Additional labels to apply to the test. The package path is added
# automatically.
+# WILL_FAIL: The test will run, but its pass/fail status will be inverted.
# GENERATED_IN_BINARY_DIR: If present, indicates that the srcs have been
# in the CMAKE_CURRENT_BINARY_DIR.
function(iree_py_test)
cmake_parse_arguments(
_RULE
"GENERATED_IN_BINARY_DIR"
- "NAME;SRCS"
+ "NAME;SRCS;WILL_FAIL"
"ARGS;LABELS;TIMEOUT"
${ARGN}
)
@@ -168,6 +173,8 @@
${_RULE_ARGS}
LABELS
${_RULE_LABELS}
+ WILL_FAIL
+ ${_RULE_WILL_FAIL}
PACKAGE_DIRS
"${IREE_BINARY_DIR}/compiler/bindings/python"
"${IREE_BINARY_DIR}/runtime/bindings/python"
diff --git a/build_tools/cmake/iree_static_linker_test.cmake b/build_tools/cmake/iree_static_linker_test.cmake
index 08b64f0..c4685f9 100644
--- a/build_tools/cmake/iree_static_linker_test.cmake
+++ b/build_tools/cmake/iree_static_linker_test.cmake
@@ -24,6 +24,7 @@
# FUNCTION_INPUTS: Module input in the format of mxnxcx<datatype>.
# LABELS: Additional labels to apply to the test. The package path and
# "driver=local-sync" are added automatically.
+# WILL_FAIL: The test will run, but its pass/fail status will be inverted.
# TARGET_CPU_FEATURES: If specified, a string passed as argument to
# --iree-llvmcpu-target-cpu-features.
#
@@ -60,7 +61,7 @@
cmake_parse_arguments(
_RULE
"EMITC"
- "NAME;SRC;DRIVER;STATIC_LIB_PREFIX;ENTRY_FUNCTION;INPUT_TYPE;HAL_TYPE"
+ "NAME;SRC;DRIVER;STATIC_LIB_PREFIX;ENTRY_FUNCTION;INPUT_TYPE;HAL_TYPE;WILL_FAIL"
"COMPILER_FLAGS;LABELS;TARGET_CPU_FEATURES;FUNCTION_INPUTS"
${ARGN}
)
@@ -270,5 +271,7 @@
LABELS
${_RULE_LABELS}
${_RULE_TARGET_CPU_FEATURES}
+ WILL_FAIL
+ ${_RULE_WILL_FAIL}
)
endfunction()
diff --git a/build_tools/cmake/iree_wasm_library.cmake b/build_tools/cmake/iree_wasm_library.cmake
index d49d525..bcf3a49 100644
--- a/build_tools/cmake/iree_wasm_library.cmake
+++ b/build_tools/cmake/iree_wasm_library.cmake
@@ -399,18 +399,32 @@
# Same as iree_wasm_cc_binary, plus a ctest test that runs the bundle via
# Node.js. The test passes if the .mjs entry point exits with code 0.
#
-# Parameters: same as iree_wasm_cc_binary.
+# Parameters: same as iree_wasm_cc_binary, plus:
+# WILL_FAIL: The test will run, but its pass/fail status will be inverted.
function(iree_wasm_cc_test)
cmake_parse_arguments(
_RULE
"TESTONLY"
- "PACKAGE;NAME;MAIN"
+ "PACKAGE;NAME;MAIN;WILL_FAIL"
"SRCS;DEPS;COPTS;DEFINES"
${ARGN}
)
- # Build the bundle using iree_wasm_cc_binary.
- iree_wasm_cc_binary(${ARGN})
+ # Build the bundle using iree_wasm_cc_binary without test-only arguments.
+ set(_TESTONLY_ARG "")
+ if(_RULE_TESTONLY)
+ set(_TESTONLY_ARG TESTONLY)
+ endif()
+ iree_wasm_cc_binary(
+ PACKAGE ${_RULE_PACKAGE}
+ NAME ${_RULE_NAME}
+ MAIN ${_RULE_MAIN}
+ SRCS ${_RULE_SRCS}
+ DEPS ${_RULE_DEPS}
+ COPTS ${_RULE_COPTS}
+ DEFINES ${_RULE_DEFINES}
+ ${_TESTONLY_ARG}
+ )
# Derive the output .mjs path (must match iree_wasm_cc_binary's output).
set(_OUTPUT_MJS "${CMAKE_CURRENT_BINARY_DIR}/${_RULE_NAME}.mjs")
@@ -430,4 +444,7 @@
NAME "${_TEST_NAME}"
COMMAND "${_NODE_EXECUTABLE}" "${_OUTPUT_MJS}"
)
+ if(_RULE_WILL_FAIL)
+ set_property(TEST ${_TEST_NAME} PROPERTY WILL_FAIL ${_RULE_WILL_FAIL})
+ endif()
endfunction()
