Initial gen_test_matrix commit. (#6126)
This uses a new test configuration file and corresponding generator program to produce a matrix of test runners for all possible combinations. These are then globbed in to the build system and run in bulk.
For CMake, we generate the tests at configure time. For internal bazel, we will just generate them when we import into the tree, since Bazel is too inflexible to do anything like this.
This gives us:
* A file that I can run for each permutation and will let me iterate on the test locally.
* XFAIL semantics: tests fail if they start passing but were marked XFAIL.
I re-worked the TF math directory for this and true'd it up to what our current support matrix is.
Landing as a POC for further iteration.
Co-authored-by: Scott Todd <scotttodd@google.com>
diff --git a/build_tools/cmake/iree_python.cmake b/build_tools/cmake/iree_python.cmake
index 355f5a3..5940e0a 100644
--- a/build_tools/cmake/iree_python.cmake
+++ b/build_tools/cmake/iree_python.cmake
@@ -242,6 +242,8 @@
# ARGS: Command line arguments to the Python source file.
# LABELS: Additional labels to apply to the test. The package path is added
# automatically.
+# GENERATED_IN_BINARY_DIR: If present, indicates that the srcs have been
+# in the CMAKE_CURRENT_BINARY_DIR.
function(iree_py_test)
if(NOT IREE_BUILD_TESTS)
return()
@@ -249,12 +251,18 @@
cmake_parse_arguments(
_RULE
- ""
+ "GENERATED_IN_BINARY_DIR"
"NAME;SRCS"
"ARGS;LABELS"
${ARGN}
)
+ # Switch between source and generated tests.
+ set(_SRC_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
+ if(_RULE_GENERATED_IN_BINARY_DIR)
+ set(_SRC_DIR "${CMAKE_CURRENT_BINARY_DIR}")
+ endif()
+
iree_package_name(_PACKAGE_NAME)
set(_NAME "${_PACKAGE_NAME}_${_RULE_NAME}")
@@ -271,7 +279,7 @@
COMMAND
"${CMAKE_SOURCE_DIR}/build_tools/cmake/run_test.${IREE_HOST_SCRIPT_EXT}"
"${Python3_EXECUTABLE}"
- "${CMAKE_CURRENT_SOURCE_DIR}/${_RULE_SRCS}"
+ "${_SRC_DIR}/${_RULE_SRCS}"
${_RULE_ARGS}
INSTALLED_COMMAND
python
diff --git a/build_tools/cmake/iree_test_matrix.cmake b/build_tools/cmake/iree_test_matrix.cmake
new file mode 100644
index 0000000..9a8aba4
--- /dev/null
+++ b/build_tools/cmake/iree_test_matrix.cmake
@@ -0,0 +1,75 @@
+# Copyright 2021 The IREE Authors
+#
+# Licensed under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+include(CMakeParseArguments)
+
+# Generates a matrix of permuted test cases from a description in the
+# test_matrix.yaml file in the current source directory.
+# The resulting test files will be written to:
+# ${CMAKE_CURRENT_BINARY_DIR}/generated
+# Typically this should be followed by:
+# iree_test_matrix_glob_py_tests
+#
+# (type of tests to add are based on the runners defined in the
+# generation script)
+# TODO: Add a `generate` step conditioned on the test_matrix.yaml file, so that
+# reconfigure is automatic.
+function(iree_test_matrix_gen)
+ set(_test_matrix_yaml "${CMAKE_CURRENT_SOURCE_DIR}/test_matrix.yaml")
+ set(_generator_script "${PROJECT_SOURCE_DIR}/build_tools/testing/gen_test_matrix.py")
+ message(STATUS "Generating tests for ${CMAKE_CURRENT_SOURCE_DIR}")
+ execute_process(
+ COMMAND "${Python3_EXECUTABLE}"
+ "${_generator_script}"
+ --dir "${CMAKE_CURRENT_SOURCE_DIR}"
+ --output_dir "${CMAKE_CURRENT_BINARY_DIR}"
+ WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}"
+ RESULTS_VARIABLE _gen_result)
+
+ if(_gen_result)
+ message(SEND_ERROR "Error generating tests for ${CMAKE_CURRENT_SOURCE_DIR}")
+ endif()
+endfunction()
+
+# Adds individual py_tests for each generated *.py file from an
+# iree_test_matrix_gen invocation.
+#
+# Parameters:
+# GLOB: The glob expression to use (defaults to *.py)
+# LABELS: Optional labels to add
+function(iree_test_matrix_glob_py_tests)
+ cmake_parse_arguments(ARG
+ ""
+ "GLOB"
+ "LABELS"
+ ${ARGN})
+ set(_glob "*.py")
+ if(ARG_GLOB)
+ set(_glob "${ARG_GLOB}")
+ endif()
+
+ file(GLOB
+ _found_test_files
+ LIST_DIRECTORIES NO
+ RELATIVE "${CMAKE_CURRENT_BINARY_DIR}/generated"
+ "${CMAKE_CURRENT_BINARY_DIR}/generated/${_glob}")
+
+ if(NOT _found_test_files)
+ message(SEND_ERROR
+ "No generated tests (${_glob}) found under ${CMAKE_CURRENT_BINARY_DIR}/generated")
+ endif()
+
+ foreach(test_file ${_found_test_files})
+ iree_py_test(
+ NAME "generated/${test_file}"
+ GENERATED_IN_BINARY_DIR
+ SRCS "generated/${test_file}"
+ LABELS
+ generated
+ ${ARG_LABELS}
+ )
+ endforeach()
+endfunction()
diff --git a/build_tools/cmake/run_test.sh b/build_tools/cmake/run_test.sh
index ad6b972..191cbbe 100755
--- a/build_tools/cmake/run_test.sh
+++ b/build_tools/cmake/run_test.sh
@@ -19,7 +19,7 @@
echo "Creating test environment"
rm -rf "${TEST_TMPDIR?}" # In case this wasn't cleaned up previously
-mkdir "${TEST_TMPDIR?}"
+mkdir -p "${TEST_TMPDIR?}"
trap cleanup EXIT
# Execute whatever we were passed.
"$@"
