Switch abs test to check framework This encompasses a lot of weirdness on different backend support and includes updating the build rules to allow this. I opted for a new function for the single backed/driver case even though you could just pass a one-element list to iree_check_test_suite because I wanted to avoid the case where the generated names become super verbose as the same information is added multiple times. PiperOrigin-RevId: 305913199
diff --git a/build_tools/bazel/iree_check_test.bzl b/build_tools/bazel/iree_check_test.bzl index ff1fc21..fc9c955 100644 --- a/build_tools/bazel/iree_check_test.bzl +++ b/build_tools/bazel/iree_check_test.bzl
@@ -23,7 +23,15 @@ ("llvm-ir", "llvm"), ] -def iree_check_test(name, src, target_backend, driver, args = [], tags = [], **kwargs): +def iree_check_test( + name, + src, + target_backend, + driver, + compiler_flags = [], + runner_args = [], + tags = [], + **kwargs): """Creates an iree-check-module test for the specified source file. Args: @@ -31,10 +39,12 @@ src: source mlir file containing the module. target_backend: target backend to compile for. driver: driver to run the module with. - args: additional args to pass to iree-check-module. The driver and input file are passed - automatically. + compiler_flags: additional flags to pass to the compiler. Bytecode translation and backend + flags are passed automatically. + runner_args: additional runner_args to pass to iree-check-module. The driver and input file + are passed automatically. tags: additional tags to apply to the generated test. A tag "driver=DRIVER" is added - automatically. + automatically. **kwargs: any additional attributes to pass to the underlying run_binary_test. """ bytecode_module_name = name + "_bytecode_module" @@ -44,7 +54,7 @@ flags = [ "-iree-mlir-to-vm-bytecode-module", "--iree-hal-target-backends=%s" % target_backend, - ], + ] + compiler_flags, visibility = ["//visibility:private"], ) @@ -53,52 +63,108 @@ args = [ "--driver=%s" % driver, "$(location :%s)" % bytecode_module_name, - ] + args, + ] + runner_args, data = [":%s" % bytecode_module_name], test_binary = "//iree/modules/check:iree-check-module", tags = tags + ["driver=%s" % driver], **kwargs ) +def iree_check_single_backend_test_suite( + name, + srcs, + target_backend, + driver, + compiler_flags = [], + runner_args = [], + tags = [], + **kwargs): + """Creates a test suite of iree-check-module tests for a single backend/driver pair. + + One test is generated per source file. + + Args: + name: name of the generated test suite. + srcs: source mlir files containing the module. + target_backend: target backend to compile for. + driver: driver to run the module with. + compiler_flags: additional flags to pass to the compiler. Bytecode translation and backend + flags are passed automatically. + runner_args: additional runner_args to pass to the underlying iree-check-module tests. The + driver and input file are passed automatically. To use different runner_args per test, + create a separate suite or iree_check_test. + tags: tags to apply to the generated tests. Note that as in standard test suites, manual + is treated specially and will also apply to the test suite itself. + **kwargs: any additional attributes to pass to the underlying tests and test suite. + """ + tests = [] + for src in srcs: + test_name = "_".join([name, src]) + iree_check_test( + name = test_name, + src = src, + target_backend = target_backend, + driver = driver, + compiler_flags = compiler_flags, + runner_args = runner_args, + tags = tags, + **kwargs + ) + tests.append(test_name) + native.test_suite( + name = name, + tests = tests, + # Note that only the manual tag really has any effect here. Others are + # used for test suite filtering, but all tests are passed the same tags. + tags = tags, + # If there are kwargs that need to be passed here which only apply to + # the generated tests and not to test_suite, they should be extracted + # into separate named arguments. + **kwargs + ) + def iree_check_test_suite( name, srcs, target_backends_and_drivers = ALL_TARGET_BACKENDS_AND_DRIVERS, - args = [], + compiler_flags = [], + runner_args = [], tags = [], **kwargs): """Creates a test suite of iree-check-module tests. - One test is generated per source and driver. + One test is generated per source file and backend/driver. Args: name: name of the generated test suite. srcs: source mlir files containing the module. - target_backends_and_drivers: backend, driver pairs to compile and run the module, respectively. - args: additional args to pass to the underlying iree-check-module tests. The driver and - input file are passed automatically. To use different args per test, create a - separate suite or iree_check_test. + target_backends_and_drivers: backend/driver pairs to compile and run the module, respectively. + compiler_flags: additional flags to pass to the compiler. Bytecode translation and backend + flags are passed automatically. + runner_args: additional runner_args to pass to the underlying iree-check-module tests. The + driver and input file are passed automatically. To use different runner_args per test, + create a separate suite or iree_check_test. tags: tags to apply to the generated tests. Note that as in standard test suites, manual - is treated specially and will also apply to the test suite itself. + is treated specially and will also apply to the test suite itself. **kwargs: any additional attributes to pass to the underlying tests and test suite. """ - # We could have complicated argument override logic for args and such, or... we could just - # create a separate target. The latter seems simpler and more readable. + # We could have complicated argument override logic for runner_args and such, or... the client + # could just create a test suite. The latter seems simpler and more readable. tests = [] for backend, driver in target_backends_and_drivers: - for src in srcs: - test_name = "_".join([name, src, backend, driver]) - iree_check_test( - name = test_name, - src = src, - driver = driver, - target_backend = backend, - args = args, - tags = tags, - **kwargs - ) - tests.append(test_name) + suite_name = "_".join([name, backend, driver]) + iree_check_single_backend_test_suite( + name = suite_name, + srcs = srcs, + driver = driver, + target_backend = backend, + compiler_flags = compiler_flags, + runner_args = runner_args, + tags = tags, + **kwargs + ) + tests.append(suite_name) native.test_suite( name = name, tests = tests,
diff --git a/build_tools/bazel_to_cmake/bazel_to_cmake_converter.py b/build_tools/bazel_to_cmake/bazel_to_cmake_converter.py index b73354f..87140f3 100644 --- a/build_tools/bazel_to_cmake/bazel_to_cmake_converter.py +++ b/build_tools/bazel_to_cmake/bazel_to_cmake_converter.py
@@ -51,6 +51,11 @@ # between similar rule conversions (e.g. cc_library and cc_binary). # # ------------------------------------------------------------------------- # + def _convert_string_arg_block(self, name, value): + # NAME + # value + return f" {name}\n {value}\n" + def _convert_name_block(self, name): # NAME # rule_name @@ -513,24 +518,60 @@ f"{labels_block}" f")\n\n") - def iree_check_test_suite(self, - name, - srcs=None, - target_backends_and_drivers=None, - args=None, - tags=None, - **kwargs): + def iree_check_single_backend_test_suite(self, + name, + srcs, + target_backend, + driver, + compiler_flags=None, + target_backends_and_drivers=None, + runner_args=None, + tags=None, + **kwargs): name_block = self._convert_name_block(name) srcs_block = self._convert_srcs_block(srcs) + target_backend_block = self._convert_string_arg_block( + "TARGET_BACKEND", target_backend) + driver_block = self._convert_string_arg_block("DRIVER", driver) + compiler_flags_block = self._convert_string_list_block( + "COMPILER_FLAGS", compiler_flags) + runner_args_block = self._convert_string_list_block("RUNNER_ARGS", + runner_args) + labels_block = self._convert_string_list_block("LABELS", tags) + + self.converter.body += (f"iree_check_single_backend_test_suite(\n" + f"{name_block}" + f"{srcs_block}" + f"{target_backend_block}" + f"{driver_block}" + f"{compiler_flags_block}" + f"{runner_args_block}" + f"{labels_block}" + f")\n\n") + + def iree_check_test_suite(self, + name, + srcs, + target_backends_and_drivers=None, + compiler_flags=None, + runner_args=None, + tags=None, + **kwargs): target_backends = None drivers = None if target_backends_and_drivers is not None: target_backends = [it[0] for it in target_backends_and_drivers] drivers = [it[1] for it in target_backends_and_drivers] + + name_block = self._convert_name_block(name) + srcs_block = self._convert_srcs_block(srcs) target_backends_block = self._convert_string_list_block( "TARGET_BACKENDS", target_backends) drivers_block = self._convert_string_list_block("DRIVERS", drivers) - args_block = self._convert_string_list_block("ARGS", args) + compiler_flags_block = self._convert_string_list_block( + "COMPILER_FLAGS", compiler_flags) + runner_args_block = self._convert_string_list_block("RUNNER_ARGS", + runner_args) labels_block = self._convert_string_list_block("LABELS", tags) self.converter.body += (f"iree_check_test_suite(\n" @@ -538,7 +579,8 @@ f"{srcs_block}" f"{target_backends_block}" f"{drivers_block}" - f"{args_block}" + f"{compiler_flags_block}" + f"{runner_args_block}" f"{labels_block}" f")\n\n")
diff --git a/build_tools/cmake/iree_check_test.cmake b/build_tools/cmake/iree_check_test.cmake index 54c42b4..5adf71d 100644 --- a/build_tools/cmake/iree_check_test.cmake +++ b/build_tools/cmake/iree_check_test.cmake
@@ -25,16 +25,18 @@ # SRC: mlir source file to be compiled to an IREE module. # TARGET_BACKEND: target backend to compile for. # DRIVER: driver to run the module with. -# ARGS: additional args to pass to iree-check-module. The driver and input -# file are passed automatically. +# COMPILER_FLAGS: additional flags to pass to the compiler. Bytecode +# translation and backend flags are passed automatically. +# RUNNER_ARGS: additional args to pass to iree-check-module. The driver +# and input file are passed automatically. # LABELS: Additional labels to apply to the test. The package path and # "driver=${DRIVER}" are added automatically. function(iree_check_test) cmake_parse_arguments( _RULE "" - "NAME;SRC;TARGET_BACKEND;DRIVER;LABELS" - "ARGS" + "NAME;SRC;TARGET_BACKEND;DRIVER" + "COMPILER_FLAGS;RUNNER_ARGS;LABELS" ${ARGN} ) if(NOT IREE_BUILD_TESTS) @@ -54,6 +56,7 @@ FLAGS "-iree-mlir-to-vm-bytecode-module" "--iree-hal-target-backends=${_RULE_TARGET_BACKEND}" + ${_RULE_COMPILER_FLAGS} TESTONLY ) @@ -93,7 +96,7 @@ "$<TARGET_FILE:iree_modules_check_iree-check-module>" "--driver=${_RULE_DRIVER}" "${CMAKE_CURRENT_BINARY_DIR}/${_MODULE_FILE_NAME}" - ${_RULE_ARGS} + ${_RULE_RUNNER_ARGS} ) list(APPEND _RULE_LABELS "${_PACKAGE_PATH}" "driver=${_RULE_DRIVER}") @@ -103,6 +106,59 @@ endfunction() +# iree_check_single_backend_test_suite() +# +# Creates a test suite of iree-check-module tests for a single backend/driver pair. +# +# Mirrors the bzl rule of the same name. +# +# One test is generated per source file. +# Parameters: +# NAME: name of the generated test suite. +# SRCS: source mlir files containing the module. +# TARGET_BACKEND: target backend to compile for. +# DRIVER: driver to run the module with. +# COMPILER_FLAGS: additional flags to pass to the compiler. Bytecode +# translation and backend flags are passed automatically. +# RUNNER_ARGS: additional args to pass to the underlying iree-check-module +# tests. The driver and input file are passed automatically. To use +# different args per test, create a separate suite or iree_check_test. +# LABELS: Additional labels to apply to the generated tests. The package path is +# added automatically. +function(iree_check_single_backend_test_suite) + cmake_parse_arguments( + _RULE + "" + "NAME;TARGET_BACKEND;DRIVER" + "SRCS;COMPILER_FLAGS;RUNNER_ARGS;LABELS" + ${ARGN} + ) + if(NOT IREE_BUILD_TESTS) + return() + endif() + + foreach(_SRC IN LISTS _RULE_SRCS) + set(_TEST_NAME "${_RULE_NAME}_${_SRC}") + iree_check_test( + NAME + ${_TEST_NAME} + SRC + ${_SRC} + TARGET_BACKEND + ${_RULE_TARGET_BACKEND} + DRIVER + ${_RULE_DRIVER} + COMPILER_FLAGS + ${_RULE_COMPILER_FLAGS} + RUNNER_ARGS + ${_RULE_RUNNER_ARGS} + LABELS + ${_RULE_LABELS} + ) + endforeach() +endfunction() + + # iree_check_test_suite() # # Creates a test suite of iree-check-module tests. @@ -121,7 +177,7 @@ # TARGET_BACKENDS argument (due to cmake limitations they are separate list # arguments). The lengths must exactly match. If no backends or drivers are # specified, a test will be generated for every supported pair. -# ARGS: additional args to pass to the underlying iree-check-module tests. The +# RUNNER_ARGS: additional args to pass to the underlying iree-check-module tests. The # driver and input file are passed automatically. To use different args per # test, create a separate suite or iree_check_test. # LABELS: Additional labels to apply to the generated tests. The package path is @@ -131,7 +187,7 @@ _RULE "" "NAME" - "SRCS;TARGET_BACKENDS;DRIVERS;ARGS;LABELS" + "SRCS;TARGET_BACKENDS;DRIVERS;RUNNER_ARGS;LABELS" ${ARGN} ) if(NOT IREE_BUILD_TESTS) @@ -155,22 +211,22 @@ foreach(_INDEX RANGE "${_MAX_INDEX}") list(GET _RULE_TARGET_BACKENDS ${_INDEX} _TARGET_BACKEND) list(GET _RULE_DRIVERS ${_INDEX} _DRIVER) - foreach(_SRC IN LISTS _RULE_SRCS) - set(_TEST_NAME "${_RULE_NAME}_${_SRC}_${_TARGET_BACKEND}_${_DRIVER}") - iree_check_test( - NAME - ${_TEST_NAME} - SRC - ${_SRC} - TARGET_BACKEND - ${_TARGET_BACKEND} - DRIVER - ${_DRIVER} - ARGS - ${_RULE_ARGS} - LABELS - ${_RULE_LABELS} - ) - endforeach() + set(_SUITE_NAME "${_RULE_NAME}_${_TARGET_BACKEND}_${_DRIVER}") + iree_check_single_backend_test_suite( + NAME + ${_SUITE_NAME} + SRCS + ${_RULE_SRCS} + TARGET_BACKEND + ${_TARGET_BACKEND} + DRIVER + ${_DRIVER} + COMPILER_FLAGS + ${_RULE_COMPILER_FLAGS} + RUNNER_ARGS + ${_RULE_RUNNER_ARGS} + LABELS + ${_RULE_LABELS} + ) endforeach() endfunction()
diff --git a/iree/modules/check/test/BUILD b/iree/modules/check/test/BUILD index 0c56779..9cdc2af 100644 --- a/iree/modules/check/test/BUILD +++ b/iree/modules/check/test/BUILD
@@ -39,5 +39,5 @@ iree_check_test_suite( name = "check_failure", srcs = ["failure.mlir"], - args = ["--expect_failure"], + runner_args = ["--expect_failure"], )
diff --git a/iree/modules/check/test/CMakeLists.txt b/iree/modules/check/test/CMakeLists.txt index 6368d05..38ccc33 100644 --- a/iree/modules/check/test/CMakeLists.txt +++ b/iree/modules/check/test/CMakeLists.txt
@@ -40,6 +40,6 @@ check_failure SRCS "failure.mlir" - ARGS + RUNNER_ARGS "--expect_failure" )
diff --git a/iree/test/e2e/xla/BUILD b/iree/test/e2e/xla/BUILD index ca87829..b6efe89 100644 --- a/iree/test/e2e/xla/BUILD +++ b/iree/test/e2e/xla/BUILD
@@ -14,7 +14,7 @@ # Tests for end-to-end IREE support starting from the XLA HLO dialect. -load("//build_tools/bazel:iree_check_test.bzl", "iree_check_test_suite") +load("//build_tools/bazel:iree_check_test.bzl", "iree_check_single_backend_test_suite") load("//iree:lit_test.bzl", "iree_lit_test_suite") package( @@ -22,11 +22,31 @@ licenses = ["notice"], # Apache 2.0 ) -CHECK_TESTS = ["reverse.mlir"] +VMLA_CHECK_TESTS = [ + "abs.mlir", + "reverse.mlir", +] + +VULKAN_CHECK_TESTS = [ + "abs.mlir", + "reverse.mlir", +] + +LINALG_TO_SPIRV_VULKAN_CHECK_TESTS = [ + "abs.mlir", +] + +LLVMJIT_CHECK_TESTS = [ + "abs.mlir", +] LEGACY_LIT_TESTS = glob( ["*.mlir"], - exclude = CHECK_TESTS, + exclude = + VMLA_CHECK_TESTS + + VULKAN_CHECK_TESTS + + LINALG_TO_SPIRV_VULKAN_CHECK_TESTS + + LLVMJIT_CHECK_TESTS, ) iree_lit_test_suite( @@ -38,11 +58,31 @@ ], ) -iree_check_test_suite( - name = "check", - srcs = CHECK_TESTS, - target_backends_and_drivers = [ - ("vmla", "vmla"), - ("vulkan-spirv", "vulkan"), - ], +iree_check_single_backend_test_suite( + name = "check_vmla_vmla", + srcs = VMLA_CHECK_TESTS, + driver = "vmla", + target_backend = "vmla", +) + +iree_check_single_backend_test_suite( + name = "check_vulkan-spirv_vulkan", + srcs = VULKAN_CHECK_TESTS, + driver = "vulkan", + target_backend = "vulkan-spirv", +) + +iree_check_single_backend_test_suite( + name = "check_linalg-to-spirv_vulkan", + srcs = LINALG_TO_SPIRV_VULKAN_CHECK_TESTS, + compiler_flags = ["-iree-use-linalg-to-spirv-path"], + driver = "vulkan", + target_backend = "vulkan-spirv", +) + +iree_check_single_backend_test_suite( + name = "check_llvm-ir_llvm", + srcs = LLVMJIT_CHECK_TESTS, + driver = "llvm", + target_backend = "llvm-ir", )
diff --git a/iree/test/e2e/xla/CMakeLists.txt b/iree/test/e2e/xla/CMakeLists.txt index f8cdc2c..f4e2312 100644 --- a/iree/test/e2e/xla/CMakeLists.txt +++ b/iree/test/e2e/xla/CMakeLists.txt
@@ -15,8 +15,18 @@ iree_add_all_subdirs() file(GLOB _GLOB_X_MLIR CONFIGURE_DEPENDS *.mlir) +file(GLOB _GLOB_ABS_MLIR CONFIGURE_DEPENDS abs.mlir) +list(REMOVE_ITEM _GLOB_X_MLIR ${_GLOB_ABS_MLIR}) file(GLOB _GLOB_REVERSE_MLIR CONFIGURE_DEPENDS reverse.mlir) list(REMOVE_ITEM _GLOB_X_MLIR ${_GLOB_REVERSE_MLIR}) +file(GLOB _GLOB_ABS_MLIR CONFIGURE_DEPENDS abs.mlir) +list(REMOVE_ITEM _GLOB_X_MLIR ${_GLOB_ABS_MLIR}) +file(GLOB _GLOB_REVERSE_MLIR CONFIGURE_DEPENDS reverse.mlir) +list(REMOVE_ITEM _GLOB_X_MLIR ${_GLOB_REVERSE_MLIR}) +file(GLOB _GLOB_ABS_MLIR CONFIGURE_DEPENDS abs.mlir) +list(REMOVE_ITEM _GLOB_X_MLIR ${_GLOB_ABS_MLIR}) +file(GLOB _GLOB_ABS_MLIR CONFIGURE_DEPENDS abs.mlir) +list(REMOVE_ITEM _GLOB_X_MLIR ${_GLOB_ABS_MLIR}) iree_lit_test_suite( NAME lit @@ -27,15 +37,50 @@ iree::tools::iree-run-mlir ) -iree_check_test_suite( +iree_check_single_backend_test_suite( NAME - check + check_vmla_vmla SRCS + "abs.mlir" "reverse.mlir" - TARGET_BACKENDS - "vmla" - "vulkan-spirv" - DRIVERS - "vmla" - "vulkan" + TARGET_BACKEND + vmla + DRIVER + vmla +) + +iree_check_single_backend_test_suite( + NAME + check_vulkan-spirv_vulkan + SRCS + "abs.mlir" + "reverse.mlir" + TARGET_BACKEND + vulkan-spirv + DRIVER + vulkan +) + +iree_check_single_backend_test_suite( + NAME + check_linalg-to-spirv_vulkan + SRCS + "abs.mlir" + TARGET_BACKEND + vulkan-spirv + DRIVER + vulkan + COMPILER_FLAGS + "-iree-use-linalg-to-spirv-path" +) + +iree_check_single_backend_test_suite( + NAME + check_llvm-ir_llvm + SRCS + "abs.mlir" + TARGET_BACKEND + llvm-ir + DRIVER + llvm )
diff --git a/iree/test/e2e/xla/abs.mlir b/iree/test/e2e/xla/abs.mlir index 477fc93..61afd90 100644 --- a/iree/test/e2e/xla/abs.mlir +++ b/iree/test/e2e/xla/abs.mlir
@@ -1,22 +1,13 @@ -// RUN: iree-run-mlir -iree-hal-target-backends=vmla %s | IreeFileCheck %s -// RUN: [[ $IREE_LLVMJIT_DISABLE == 1 ]] || (iree-run-mlir -iree-hal-target-backends=llvm-ir %s | IreeFileCheck %s) -// RUN: [[ $IREE_VULKAN_DISABLE == 1 ]] || (iree-run-mlir -iree-hal-target-backends=vulkan-spirv %s | IreeFileCheck %s) -// RUN: [[ $IREE_VULKAN_DISABLE == 1 ]] || (iree-run-mlir -iree-hal-target-backends=vulkan-spirv -iree-use-linalg-to-spirv-path %s | IreeFileCheck %s) - -// CHECK-LABEL: EXEC @tensor -func @tensor() -> tensor<4xf32> { +func @tensor() attributes { iree.module.export } { %input = iree.unfoldable_constant dense<[-1.0, -2.0, 3.0, 4.0]> : tensor<4xf32> %result = "xla_hlo.abs"(%input) : (tensor<4xf32>) -> tensor<4xf32> - return %result : tensor<4xf32> + check.expect_almost_eq_const(%result, dense<[1.0, 2.0, 3.0, 4.0]> : tensor<4xf32>) : tensor<4xf32> + return } -// CHECK: 4xf32=1 2 3 4 -// ----- - -// CHECK-LABEL: EXEC @scalar -func @scalar() -> tensor<f32> { +func @scalar() attributes { iree.module.export } { %input = iree.unfoldable_constant dense<-4.0> : tensor<f32> %result = "xla_hlo.abs"(%input) : (tensor<f32>) -> tensor<f32> - return %result : tensor<f32> + check.expect_almost_eq_const(%result, dense<4.0> : tensor<f32>) : tensor<f32> + return } -// CHECK: f32=4