CTest support for integration tests (#5101)
Add CTest support for integration tests. This is our interim solution
for integration testing support while we find a long-term solution. It
results from tests that were previously Bazel-only (because TF) and now
are CMake-only (because maintaining support for Python in Bazel was
determined to be not worth the effort).
The approach is to open-source our internal BUILD files for Blaze
(Google's internal version of Bazel, which happens to do some rather
different things with Python). These rely on a Starlark rule for
expanding a test matrix that is pretty painful to implement in CMake.
The author apologizes.
This enables 1317 new test targets.
Incidentally, it makes the tests run in parallel, which should speed
things up quite a bit...
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
index 06ec183..106bde0 100644
--- a/.github/workflows/lint.yml
+++ b/.github/workflows/lint.yml
@@ -22,8 +22,10 @@
steps:
- name: Checking out repository
uses: actions/checkout@v2
- - name: Running bazel_to_cmake
+ - name: Running bazel_to_cmake for IREE core
run: ./build_tools/bazel_to_cmake/bazel_to_cmake.py --verbosity=2
+ - name: Running bazel_to_cmake for IREE TF Integration Tests
+ run: ./build_tools/bazel_to_cmake/bazel_to_cmake.py --root_dir=integrations/tensorflow/e2e --verbosity=2
- name: Checking Diff
run: git diff --exit-code
diff --git a/build_tools/bazel_to_cmake/bazel_to_cmake_converter.py b/build_tools/bazel_to_cmake/bazel_to_cmake_converter.py
index 6a5e852..72ce425 100644
--- a/build_tools/bazel_to_cmake/bazel_to_cmake_converter.py
+++ b/build_tools/bazel_to_cmake/bazel_to_cmake_converter.py
@@ -190,6 +190,28 @@
return _convert_string_list_block(list_name, targets, sort=True, quote=False)
+# Copied from integrations/tensorflow/e2e/iree_e2e_cartesian_product_test_suite.bzl
+def _normalize_dictionary(dictionary):
+ """Wraps every value of dictionary in a list if it isn't one already."""
+ for key, value in dictionary.items():
+ if type(value) != type([]):
+ dictionary[key] = [value]
+ return dictionary
+
+
+def _dictionary_product(dictionary):
+ """Returns a named cartesian product of dictionary's values."""
+
+ # Converts {'a': [1, 2], 'b': [3, 4]} into
+ # [{'a': 1, 'b': 3}, {'a': 1, 'b': 4}, {'a': 2, 'b': 3}, {'a': 2, 'b': 4}]
+ product = [[]]
+ for values in dictionary.values():
+ # Iteratively grow the elements of the product.
+ product = [element + [value] for element in product for value in values]
+ dicts = [{k: v for k, v in zip(dictionary, element)} for element in product]
+ return dicts
+
+
class BuildFileFunctions(object):
"""Object passed to `exec` that has handlers for BUILD file functions."""
@@ -235,6 +257,11 @@
def exports_files(self, *args, **kwargs):
pass
+ # Technically we could do something with a CMake equivalent but we have no use
+ # case.
+ def py_binary(self, *args, **kwargs):
+ pass
+
def filegroup(self, name, **kwargs):
# Not implemented yet. Might be a no-op, or may want to evaluate the srcs
# attribute and pass them along to any targets that depend on the filegroup.
@@ -581,6 +608,64 @@
f"{labels_block}"
f")\n\n")
+ def iree_e2e_cartesian_product_test_suite(self,
+ name,
+ matrix,
+ failing_configurations=None,
+ tags=None,
+ data=None,
+ **kwargs):
+ # Note kwargs deps, size, python_version are unused
+ if data is not None:
+ self._convert_unimplemented_function(
+ "iree_e2e_cartesian_product_test_suite", name + " has data")
+
+ matrix_keys = matrix.keys()
+
+ name_block = _convert_string_arg_block("NAME", name, quote=False)
+ matrix_keys_block = _convert_string_list_block("MATRIX_KEYS", matrix_keys)
+ labels_block = _convert_string_list_block("LABELS", tags)
+
+ value_strings = []
+ for key in matrix_keys:
+ # ensure matching order
+ values = matrix[key]
+ if not isinstance(values, list):
+ values = [values]
+ if not values:
+ self._convert_unimplemented_function(
+ "iree_e2e_cartesian_product_test_suite",
+ name + f" has empty list for matrix key {key}")
+ value_strings.append(";".join(str(value) for value in values))
+ matrix_values_block = _convert_string_list_block("MATRIX_VALUES",
+ value_strings)
+
+ # Copied from integrations/tensorflow/e2e/iree_e2e_cartesian_product_test_suite.bzl
+ failing_configurations_block = ""
+ if failing_configurations is not None:
+ failing_matrix_configurations = []
+ for failing_configuration in failing_configurations:
+ failing_configuration = _normalize_dictionary(failing_configuration)
+
+ failing_matrix_configurations.extend(
+ _dictionary_product(failing_configuration))
+
+ failing_configuration_strings = []
+ for failing_configuration in failing_matrix_configurations:
+ failing_config_string = ",".join(
+ str(failing_configuration.get(key, "")) for key in matrix_keys)
+ failing_configuration_strings.append(failing_config_string)
+ failing_configurations_block = _convert_string_list_block(
+ "FAILING_CONFIGURATIONS", failing_configuration_strings)
+
+ self.converter.body += (f"iree_e2e_cartesian_product_test_suite(\n"
+ f"{name_block}"
+ f"{matrix_keys_block}"
+ f"{matrix_values_block}"
+ f"{failing_configurations_block}"
+ f"{labels_block}"
+ f")\n\n")
+
def run_binary_test(self, name, test_binary, args=None, data=None):
if data is not None:
self._convert_unimplemented_function("iree_run_binary_test",
diff --git a/build_tools/cmake/iree_python.cmake b/build_tools/cmake/iree_python.cmake
index 55767f7..71d39d3 100644
--- a/build_tools/cmake/iree_python.cmake
+++ b/build_tools/cmake/iree_python.cmake
@@ -243,7 +243,7 @@
# Parameters:
# NAME: name of test
# SRCS: Test source file
-# DEPS: List of deps the test requires
+# ARGS: Command line arguments to the Python source file.
# LABELS: Additional labels to apply to the test. The package path is added
# automatically.
function(iree_py_test)
@@ -255,7 +255,7 @@
_RULE
""
"NAME;SRCS"
- "DEPS;LABELS"
+ "ARGS;LABELS"
${ARGN}
)
@@ -276,6 +276,7 @@
"${CMAKE_SOURCE_DIR}/build_tools/cmake/run_test.${IREE_HOST_SCRIPT_EXT}"
"${Python3_EXECUTABLE}"
"${CMAKE_CURRENT_SOURCE_DIR}/${_RULE_SRCS}"
+ ${_RULE_ARGS}
INSTALLED_COMMAND
python
"${_PACKAGE_PATH}/${_RULE_SRCS}"
diff --git a/build_tools/kokoro/gcp_ubuntu/cmake-bazel/linux/x86-swiftshader/build.sh b/build_tools/kokoro/gcp_ubuntu/cmake-bazel/linux/x86-swiftshader/build.sh
index 97305f9..ec73635 100755
--- a/build_tools/kokoro/gcp_ubuntu/cmake-bazel/linux/x86-swiftshader/build.sh
+++ b/build_tools/kokoro/gcp_ubuntu/cmake-bazel/linux/x86-swiftshader/build.sh
@@ -67,6 +67,7 @@
cd "${CMAKE_BUILD_DIR?}"
ninja
+export CTEST_PARALLEL_LEVEL=${CTEST_PARALLEL_LEVEL:-$(nproc)}
+
echo "Testing with CTest"
-ctest -R 'tensorflow_e2e|bindings/python|integrations/tensorflow/' \
- --output-on-failure
+ctest --output-on-failure -L 'integrations/tensorflow' --label-exclude "^nokokoro$"
diff --git a/build_tools/kokoro/gcp_ubuntu/cmake-bazel/linux/x86-turing/build.sh b/build_tools/kokoro/gcp_ubuntu/cmake-bazel/linux/x86-turing/build.sh
index 85f7e14..a7d5973 100755
--- a/build_tools/kokoro/gcp_ubuntu/cmake-bazel/linux/x86-turing/build.sh
+++ b/build_tools/kokoro/gcp_ubuntu/cmake-bazel/linux/x86-turing/build.sh
@@ -70,6 +70,7 @@
cd "${CMAKE_BUILD_DIR?}"
ninja
+export CTEST_PARALLEL_LEVEL=${CTEST_PARALLEL_LEVEL:-$(nproc)}
+
echo "Testing with CTest"
-ctest -R 'tensorflow_e2e|bindings/python|integrations/tensorflow/' \
- --output-on-failure
+ctest --output-on-failure -L 'integrations/tensorflow' --label-exclude "^nokokoro$"
diff --git a/integrations/tensorflow/CMakeLists.txt b/integrations/tensorflow/CMakeLists.txt
index abb6071..7a8e5fd 100644
--- a/integrations/tensorflow/CMakeLists.txt
+++ b/integrations/tensorflow/CMakeLists.txt
@@ -73,5 +73,7 @@
endif()
if(${IREE_BUILD_TESTS} AND ${IREE_BUILD_PYTHON_BINDINGS})
+ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/e2e/")
+ include(iree_e2e_cartesian_product_test_suite)
add_subdirectory(e2e)
endif()
diff --git a/integrations/tensorflow/e2e/BUILD b/integrations/tensorflow/e2e/BUILD
index dec4fb7..2771730 100644
--- a/integrations/tensorflow/e2e/BUILD
+++ b/integrations/tensorflow/e2e/BUILD
@@ -48,6 +48,43 @@
for src in glob(["*_test.py"])
]
+ALL_SRCS = [
+ "batch_norm_test.py",
+ "batch_to_space_nd_test.py",
+ "broadcast_to_test.py",
+ "broadcasting_test.py",
+ "concat_test.py",
+ "control_flow_test.py",
+ "conv_test.py",
+ "conv_transpose_test.py",
+ "depth_conv_test.py",
+ "dynamic_mlp_relu_test.py",
+ "dynamic_mlp_test.py",
+ "einsum_dynamic_test.py",
+ "einsum_static_test.py",
+ "einsum_vector_test.py",
+ "fft_test.py",
+ "fill_test.py",
+ "gather_test.py",
+ "image_resize_test.py",
+ "linspace_test.py",
+ "mandelbrot_test.py",
+ "matrix_ops_dynamic_test.py",
+ "matrix_ops_static_test.py",
+ "quantization_dyn_test.py",
+ "quantization_test.py",
+ "range_test.py",
+ "resource_ops_test.py",
+ "ring_buffer_test.py",
+ "scatter_update_test.py",
+ "simple_arithmetic_test.py",
+ "simple_stateful_test.py",
+ "sliding_window_test.py",
+ "space_to_batch_nd_test.py",
+ "strings_test.py",
+ "tensorlist_test.py",
+]
+
# keep sorted
TFLITE_FAILING = [
"concat_test.py",
@@ -145,8 +182,9 @@
},
],
matrix = {
- "src": glob(
- ["*_test.py"],
+ "src": enforce_glob(
+ ALL_SRCS,
+ include = ["*_test.py"],
exclude = ["mobile_bert_squad_test.py"],
),
"target_backends": [
diff --git a/integrations/tensorflow/e2e/CMakeLists.txt b/integrations/tensorflow/e2e/CMakeLists.txt
index 3790193..d7b0287 100644
--- a/integrations/tensorflow/e2e/CMakeLists.txt
+++ b/integrations/tensorflow/e2e/CMakeLists.txt
@@ -1,142 +1,108 @@
-# Copyright 2020 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+################################################################################
+# Autogenerated by build_tools/bazel_to_cmake/bazel_to_cmake.py from #
+# integrations/tensorflow/e2e/BUILD #
+# #
+# Use iree_cmake_extra_content from iree/build_defs.oss.bzl to add arbitrary #
+# CMake-only content. #
+# #
+# To disable autogeneration for this file entirely, delete this header. #
+################################################################################
-add_subdirectory(keras)
+iree_add_all_subdirs()
-# Special cases to exclude from automatically expanding targets for all
-# backends.
-set(SPECIAL_CASES
+file(GLOB _GLOB_X_TEST_PY LIST_DIRECTORIES false RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} CONFIGURE_DEPENDS *_test.py)
+iree_e2e_cartesian_product_test_suite(
+ NAME
+ e2e_tests
+ MATRIX_KEYS
+ "src"
+ "target_backends"
+ "reference_backend"
+ MATRIX_VALUES
+ "batch_norm_test.py;batch_to_space_nd_test.py;broadcast_to_test.py;broadcasting_test.py;concat_test.py;control_flow_test.py;conv_test.py;conv_transpose_test.py;depth_conv_test.py;dynamic_mlp_relu_test.py;dynamic_mlp_test.py;einsum_dynamic_test.py;einsum_static_test.py;einsum_vector_test.py;fft_test.py;fill_test.py;gather_test.py;image_resize_test.py;linspace_test.py;mandelbrot_test.py;matrix_ops_dynamic_test.py;matrix_ops_static_test.py;quantization_dyn_test.py;quantization_test.py;range_test.py;resource_ops_test.py;ring_buffer_test.py;scatter_update_test.py;simple_arithmetic_test.py;simple_stateful_test.py;sliding_window_test.py;space_to_batch_nd_test.py;strings_test.py;tensorlist_test.py"
+ "tf;tflite;iree_vmla;iree_llvmaot;iree_vulkan"
+ "tf"
+ FAILING_CONFIGURATIONS
+ "concat_test.py,tflite,"
+ "einsum_dynamic_test.py,tflite,"
+ "einsum_static_test.py,tflite,"
+ "einsum_vector_test.py,tflite,"
+ "fft_test.py,tflite,"
+ "gather_test.py,tflite,"
+ "image_resize_test.py,tflite,"
+ "mandelbrot_test.py,tflite,"
+ "resource_ops_test.py,tflite,"
+ "ring_buffer_test.py,tflite,"
+ "scatter_update_test.py,tflite,"
+ "simple_stateful_test.py,tflite,"
+ "sliding_window_test.py,tflite,"
+ "strings_test.py,tflite,"
+ "einsum_dynamic_test.py,iree_vmla,"
+ "einsum_static_test.py,iree_vmla,"
+ "einsum_vector_test.py,iree_vmla,"
+ "fft_test.py,iree_vmla,"
+ "mandelbrot_test.py,iree_vmla,"
+ "ring_buffer_test.py,iree_vmla,"
+ "strings_test.py,iree_vmla,"
+ "tensorlist_test.py,iree_vmla,"
+ "broadcast_to_test.py,iree_llvmaot,"
+ "broadcasting_test.py,iree_llvmaot,"
+ "conv_transpose_test.py,iree_llvmaot,"
+ "dynamic_mlp_relu_test.py,iree_llvmaot,"
+ "dynamic_mlp_test.py,iree_llvmaot,"
+ "einsum_dynamic_test.py,iree_llvmaot,"
+ "einsum_static_test.py,iree_llvmaot,"
+ "einsum_vector_test.py,iree_llvmaot,"
+ "fft_test.py,iree_llvmaot,"
+ "fill_test.py,iree_llvmaot,"
+ "linspace_test.py,iree_llvmaot,"
+ "mandelbrot_test.py,iree_llvmaot,"
+ "matrix_ops_dynamic_test.py,iree_llvmaot,"
+ "quantization_dyn_test.py,iree_llvmaot,"
+ "range_test.py,iree_llvmaot,"
+ "ring_buffer_test.py,iree_llvmaot,"
+ "scatter_update_test.py,iree_llvmaot,"
+ "strings_test.py,iree_llvmaot,"
+ "tensorlist_test.py,iree_llvmaot,"
+ "broadcast_to_test.py,iree_vulkan,"
+ "broadcasting_test.py,iree_vulkan,"
+ "conv_transpose_test.py,iree_vulkan,"
+ "dynamic_mlp_relu_test.py,iree_vulkan,"
+ "dynamic_mlp_test.py,iree_vulkan,"
+ "einsum_dynamic_test.py,iree_vulkan,"
+ "einsum_static_test.py,iree_vulkan,"
+ "einsum_vector_test.py,iree_vulkan,"
+ "fft_test.py,iree_vulkan,"
+ "fill_test.py,iree_vulkan,"
+ "linspace_test.py,iree_vulkan,"
+ "mandelbrot_test.py,iree_vulkan,"
+ "matrix_ops_dynamic_test.py,iree_vulkan,"
+ "quantization_dyn_test.py,iree_vulkan,"
+ "range_test.py,iree_vulkan,"
+ "ring_buffer_test.py,iree_vulkan,"
+ "scatter_update_test.py,iree_vulkan,"
+ "strings_test.py,iree_vulkan,"
+ "tensorlist_test.py,iree_vulkan,"
+)
+
+iree_e2e_cartesian_product_test_suite(
+ NAME
+ mobile_bert_squad_tests
+ MATRIX_KEYS
+ "src"
+ "reference_backend"
+ "target_backends"
+ MATRIX_VALUES
"mobile_bert_squad_test.py"
+ "tf"
+ "tf;tflite;iree_vmla;iree_llvmaot;iree_vulkan"
+ LABELS
+ "external"
+ "guitar"
+ "manual"
+ "no-remote"
+ "nokokoro"
+ "notap"
)
-set(TFLITE_FAILING
- "concat_test.py"
- "einsum_dynamic_test.py"
- "einsum_static_test.py"
- "einsum_vector_test.py"
- "fft_test.py"
- "gather_test.py"
- "image_resize_test.py"
- "mandelbrot_test.py"
- "resource_ops_test.py"
- "ring_buffer_test.py"
- "scatter_update_test.py"
- "simple_stateful_test.py"
- "sliding_window_test.py"
- "strings_test.py"
-)
-
-set(VMLA_FAILING
- "einsum_dynamic_test.py"
- "einsum_static_test.py"
- "einsum_vector_test.py"
- "mandelbrot_test.py" # TODO(silvasean): Get this working on IREE.
- "ring_buffer_test.py" # TODO(b/148747011)
- "strings_test.py"
- "tensorlist_test.py" # TODO(suderman): Re-enable once dependencies resolved
-)
-
-set(LLVM_FAILING
- "broadcast_to_test.py"
- "broadcasting_test.py"
- "conv_transpose_test.py"
- "dynamic_mlp_relu_test.py"
- "dynamic_mlp_test.py"
- "einsum_dynamic_test.py"
- "einsum_static_test.py"
- "einsum_vector_test.py"
- "fft_test.py" # TODO(natashaknk): Get this working after kernel is in.
- "fill_test.py" # TODO(jennik): Get this test working on IREE.
- "linspace_test.py" # TODO(https://github.com/google/iree/issues/1521)
- "mandelbrot_test.py" # TODO(silvasean): Get this working on IREE.
- "matrix_ops_dynamic_test.py"
- "quantization_dyn_test.py"
- "range_test.py"
- "ring_buffer_test.py" # TODO(b/148747011)
- "scatter_update_test.py"
- "strings_test.py"
- "tensorlist_test.py" # TODO(suderman): Re-enable once dependencies resolved
-)
-
-set(VULKAN_FAILING
- "broadcast_to_test.py"
- "broadcasting_test.py"
- "conv_transpose_test.py"
- "dynamic_mlp_relu_test.py"
- "dynamic_mlp_test.py"
- "einsum_dynamic_test.py"
- "einsum_static_test.py"
- "einsum_vector_test.py"
- "fft_test.py" # TODO(natashaknk): Get this working after kernel is in.
- "fill_test.py" # TODO(jennik): Get this test working on IREE.
- "linspace_test.py" # TODO(https://github.com/google/iree/issues/1521)
- "mandelbrot_test.py" # TODO(silvasean): Get this working on IREE.
- "matrix_ops_dynamic_test.py"
- "quantization_dyn_test.py"
- "range_test.py"
- "ring_buffer_test.py" # TODO(b/148747011)
- "scatter_update_test.py"
- "strings_test.py"
- "tensorlist_test.py" # TODO(suderman): Re-enable once dependencies resolved
-)
-
-set(REFERENCE_BACKEND tf)
-
-function(add_e2e_test_backend filename target_backend labels)
- set(_name "tensorflow_e2e__${filename}__${target_backend}")
- add_test(
- NAME
- ${_name}
- WORKING_DIRECTORY
- "${CMAKE_CURRENT_BINARY_DIR}"
- COMMAND
- "${Python3_EXECUTABLE}" -B
- "${CMAKE_CURRENT_SOURCE_DIR}/${filename}"
- "--reference_backend=${REFERENCE_BACKEND}"
- "--target_backends=${target_backend}"
- )
- set_property(TEST ${_name} PROPERTY LABELS "${labels}")
- set_property(TEST ${_name} PROPERTY ENVIRONMENT
- "PYTHONPATH=${CMAKE_BINARY_DIR}/bindings/python")
-endfunction()
-
-function(add_e2e_test filename)
- # Exclude special files.
- if("${filename}" IN_LIST SPECIAL_CASES)
- return()
- endif()
-
- # Build target_backends
- if(NOT "${filename}" IN_LIST VMLA_FAILING)
- add_e2e_test_backend("${filename}" iree_vmla "")
- endif()
- if(NOT "${filename}" IN_LIST LLVM_FAILING)
- add_e2e_test_backend("${filename}" iree_llvmaot "driver=dylib")
- endif()
- if(NOT "${filename}" IN_LIST VULKAN_FAILING)
- add_e2e_test_backend("${filename}" iree_vulkan "driver=vulkan")
- endif()
- if(NOT "${filename}" IN_LIST TFLITE_FAILING)
- add_e2e_test_backend("${filename}" tflite "")
- endif()
-endfunction()
-
-function(add_all_e2e_tests pattern)
- file(GLOB _all_files RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "${pattern}")
- foreach(_filename ${_all_files})
- add_e2e_test("${_filename}")
- endforeach()
-endfunction()
-
-add_all_e2e_tests("*_test.py")
+### BAZEL_TO_CMAKE_PRESERVES_ALL_CONTENT_BELOW_THIS_LINE ###
diff --git a/integrations/tensorflow/e2e/iree_e2e_cartesian_product_test_suite.bzl b/integrations/tensorflow/e2e/iree_e2e_cartesian_product_test_suite.bzl
index 3b76c3d..3a1cc47 100644
--- a/integrations/tensorflow/e2e/iree_e2e_cartesian_product_test_suite.bzl
+++ b/integrations/tensorflow/e2e/iree_e2e_cartesian_product_test_suite.bzl
@@ -199,13 +199,6 @@
# For all other flags, append their key and value if the value isn't
# always the same.
for k, v in flags.items():
- # For bazel->cmake support. These characters are reserved.
- if ";" in str(k):
- fail("Illegal character `;` in matrix key".format(k))
- if ";" in str(v):
- fail("Illegal character `;` in matrix value".format(v))
- if "," in str(v):
- fail("Illegal character `,` in matrix value".format(v))
if len(matrix[k]) > 1:
test_name.append(k)
test_name.append(str(v))
diff --git a/integrations/tensorflow/e2e/iree_e2e_cartesian_product_test_suite.cmake b/integrations/tensorflow/e2e/iree_e2e_cartesian_product_test_suite.cmake
new file mode 100644
index 0000000..e7dc648
--- /dev/null
+++ b/integrations/tensorflow/e2e/iree_e2e_cartesian_product_test_suite.cmake
@@ -0,0 +1,282 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# iree_e2e_cartesian_product_test_suite()
+#
+# Expands a testing matrix to create python test targets.
+#
+# Mirrors the bzl rule of the same name.
+#
+# This rule uses CMake in a way that it was never meant to be used. In
+# particular it uses custom hackery to enable nested data structures. The
+# original Starlark rule was written without the expectation that it would ever
+# need to be ported to CMake and so has some complex matrix expansion logic.
+# The author apologizes for what you are about to read.
+#
+# Parameters:
+# NAME: Base name of the test targets.
+# MATRIX_KEYS: Keys for the different dimensions of the matrix to be expanded.
+# One of the keys must be `src` and is expanded to the source files for
+# each test. One key must be `target_backends` and is expanded to the
+# target backends for the IREE integration test. It is also used to derive
+# a driver label to attach to the generated tests. All other keys are
+# assumed to correspond to flag names that should be passed to the
+# generated tests (with the corresponding values from MATRIX_VALUES).
+# MATRIX_VALUES: Lists of values for each key of the matrix. Each element of
+# this argument is interpreted as a list and the order of arguments must
+# match MATRIX_KEYS.
+# FAILING_CONFIGURATIONS: Configurations in the matrix expansion that are
+# expected to fail. Each element of this argument is a *comma*-separated
+# list of matrix values in the same order as MATRIX_KEYS (and having the
+# same number of elements. An empty element is interpreted to correspond
+# to all values for that key in the matrix.
+# LABELS: Additional labels to apply to the test. The package path and
+# "driver=${DRIVER}" based on the target_backends matrix key are added
+# automatically.
+#
+#
+# Example:
+# iree_e2e_cartesian_product_test_suite(
+# NAME
+# my_tests
+# MATRIX_KEYS
+# "src"
+# "target_backend"
+# "reference_backend"
+# "magic_flag"
+# MATRIX_VALUES
+# "concat_test.py;range_test.py"
+# "tf;tflite;iree_vmla;iree_llvmaot;iree_vulkan"
+# "tf"
+# "true;false"
+# FAILING_CONFIGURATIONS
+# "concat_test.py,,,true"
+# "range_test.py,iree_vulkan,,"
+# )
+#
+# Would expand to the following tests:
+# DISABLED: python concat_test.py --target_backend=tf --reference_backend=tf --magic_flag=true
+# DISABLED: python concat_test.py --target_backend=tflite --reference_backend=tf --magic_flag=true
+# DISABLED: python concat_test.py --target_backend=iree_vmla --reference_backend=tf --magic_flag=true
+# DISABLED: python concat_test.py --target_backend=iree_llvmaot --reference_backend=tf --magic_flag=true
+# DISABLED: python concat_test.py --target_backend=iree_vulkan --reference_backend=tf --magic_flag=true
+# python range_test.py --target_backend=tf --reference_backend=tf --magic_flag=true
+# python range_test.py --target_backend=tflite --reference_backend=tf --magic_flag=true
+# python range_test.py --target_backend=iree_vmla --reference_backend=tf --magic_flag=true
+# python range_test.py --target_backend=iree_llvmaot --reference_backend=tf --magic_flag=true
+# DISABLED: python range_test.py --target_backend=iree_vulkan --reference_backend=tf --magic_flag=true
+# python concat_test.py --target_backend=tf --reference_backend=tf --magic_flag=false
+# python concat_test.py --target_backend=tflite --reference_backend=tf --magic_flag=false
+# python concat_test.py --target_backend=iree_vmla --reference_backend=tf --magic_flag=false
+# python concat_test.py --target_backend=iree_llvmaot --reference_backend=tf --magic_flag=false
+# python concat_test.py --target_backend=iree_vulkan --reference_backend=tf --magic_flag=false
+# python range_test.py --target_backend=tf --reference_backend=tf --magic_flag=false
+# python range_test.py --target_backend=tflite --reference_backend=tf --magic_flag=false
+# python range_test.py --target_backend=iree_vmla --reference_backend=tf --magic_flag=false
+# python range_test.py --target_backend=iree_llvmaot --reference_backend=tf --magic_flag=false
+# DISABLED: python range_test.py --target_backend=iree_vulkan --reference_backend=tf --magic_flag=false
+#
+function(iree_e2e_cartesian_product_test_suite)
+ if(NOT IREE_BUILD_TESTS)
+ return()
+ endif()
+
+ cmake_parse_arguments(
+ PARSE_ARGV 0
+ _RULE
+ ""
+ "NAME"
+ "MATRIX_KEYS;MATRIX_VALUES;FAILING_CONFIGURATIONS;LABELS"
+ )
+
+ list(LENGTH _RULE_MATRIX_KEYS _MATRIX_KEYS_COUNT)
+ list(LENGTH _RULE_MATRIX_VALUES _MATRIX_VALUES_COUNT)
+
+ if(NOT _MATRIX_KEYS_COUNT EQUAL _MATRIX_VALUES_COUNT)
+ message(
+ SEND_ERROR
+ "MATRIX_KEYS count ${_MATRIX_KEYS_COUNT} does not match MATRIX_VALUES"
+ " count ${_MATRIX_VALUES_COUNT}"
+ )
+ endif()
+ list(FIND _RULE_MATRIX_KEYS "src" _SRC_KEY_INDEX)
+ if(_SRC_KEY_INDEX EQUAL -1)
+ message(
+ SEND_ERROR
+ "Did not find key `src` in MATRIX_KEYS: ${_RULE_MATRIX_KEYS}"
+ )
+ endif()
+
+ list(FIND _RULE_MATRIX_KEYS "target_backends" _TARGET_BACKENDS_KEY_INDEX)
+ if(_TARGET_BACKENDS_KEY_INDEX EQUAL -1)
+ message(
+ SEND_ERROR
+ "Did not find key `target_backends` in MATRIX_KEYS: ${_RULE_MATRIX_KEYS}"
+ )
+ endif()
+ math(EXPR _MAX_INDEX "${_MATRIX_KEYS_COUNT} - 1")
+
+ # Process failing configurations, expanding empty entries to be all entries
+ # for that key.
+ set(_FAILING_CONFIGURATIONS "")
+ foreach(_FAILING_CONFIGURATION IN LISTS _RULE_FAILING_CONFIGURATIONS)
+ # Convert comma-delimited string into a list. Lists in CMake are just
+ # semicolon-delimited 🤢
+ string(REPLACE "," ";" _CONFIGURATION_LIST "${_FAILING_CONFIGURATION}")
+ list(LENGTH _CONFIGURATION_LIST _CONFIGURATION_KEY_COUNT)
+ if(NOT _CONFIGURATION_KEY_COUNT EQUAL _MATRIX_KEYS_COUNT)
+ message(
+ SEND_ERROR
+ "Failing configuration ${_FAILING_CONFIGURATION} does not have same"
+ " number of entries (${_MATRIX_KEY_COUNT}) as MATRIX_KEYS")
+ endif()
+
+ # If the first value in this config entry is empty, it expands to all values
+ # for the corresponding key. We have to seed the first entry to start off
+ # the list because it is impossible to create a list in CMake where the only
+ # entry is the empty string.
+ list(GET _CONFIGURATION_LIST 0 _FIRST_CONFIG_VALUE)
+ if("${_FIRST_CONFIG_VALUE}" STREQUAL "")
+ list(GET _RULE_MATRIX_VALUES 0 _EXPANDED_CONFIGURATIONS)
+ else()
+ set(_EXPANDED_CONFIGURATIONS "${_FIRST_CONFIG_VALUE}")
+ endif()
+
+ # For the remaining entries, append them to configurations that are already
+ # expanded.
+ foreach(_INDEX RANGE 1 "${_MAX_INDEX}")
+ list(GET _CONFIGURATION_LIST ${_INDEX} _CONFIG_VALUE)
+ if("${_CONFIG_VALUE}" STREQUAL "")
+ # If the value for this key is unset, it represents all possible values
+ # for this key. For each such value, create a new list of configs that
+ # is the current list with the value appended (comma-delimited).
+ set(_KEY_CONFIGURATIONS "")
+ list(GET _RULE_MATRIX_VALUES ${_INDEX} _MATRIX_VALUES)
+ foreach(_MATRIX_VALUE IN LISTS _MATRIX_VALUES)
+ list(TRANSFORM
+ _EXPANDED_CONFIGURATIONS
+ APPEND ",${_MATRIX_VALUE}"
+ OUTPUT_VARIABLE _KEY_VALUE_CONFIGURATIONS)
+ list(APPEND _KEY_CONFIGURATIONS ${_KEY_VALUE_CONFIGURATIONS})
+ endforeach()
+ set(_EXPANDED_CONFIGURATIONS "${_KEY_CONFIGURATIONS}")
+ else()
+ # If set, append it (comma-delimited) to every entry already existing.
+ list(TRANSFORM
+ _EXPANDED_CONFIGURATIONS
+ APPEND ",${_CONFIG_VALUE}"
+ OUTPUT_VARIABLE _EXPANDED_CONFIGURATIONS)
+ endif()
+ endforeach()
+
+ # Add the configurations for this entry to the full list.
+ list(APPEND _FAILING_CONFIGURATIONS ${_EXPANDED_CONFIGURATIONS})
+ endforeach()
+ list(REMOVE_DUPLICATES _FAILING_CONFIGURATIONS)
+
+
+ # Build up all configurations, taking a cartesian product of the matrix
+ # values. This is much like the processing of failing configurations except
+ # that every key expands to all possible values for that key.
+
+ # Seed configurations with the first list of matrix values.
+ list(GET _RULE_MATRIX_VALUES 0 _ALL_CONFIGURATIONS)
+
+ # For each value in each subsequent list of values, create a new list of
+ # configs that is the current list with the new value appended
+ # (comma-delimited).
+ foreach(_INDEX RANGE 1 "${_MAX_INDEX}")
+ list(GET _RULE_MATRIX_VALUES ${_INDEX} _MATRIX_VALUES)
+ set(_KEY_CONFIGURATIONS "")
+ foreach(_MATRIX_VALUE IN LISTS _MATRIX_VALUES)
+ list(TRANSFORM
+ _ALL_CONFIGURATIONS
+ APPEND ",${_MATRIX_VALUE}"
+ OUTPUT_VARIABLE _KEY_VALUE_CONFIGURATIONS)
+ list(APPEND _KEY_CONFIGURATIONS ${_KEY_VALUE_CONFIGURATIONS})
+ endforeach()
+ set(_ALL_CONFIGURATIONS ${_KEY_CONFIGURATIONS})
+ endforeach()
+
+
+ # Now that we have all the configurations, iterate through them all, excluding
+ # the failing configurations.
+ foreach(_CONFIGURATION IN LISTS _ALL_CONFIGURATIONS)
+ # Skip this configuration if it's failing.
+ list(FIND _FAILING_CONFIGURATIONS "${_CONFIGURATION}" _FAILING_INDEX)
+ if (NOT _FAILING_INDEX EQUAL -1)
+ continue()
+ endif()
+
+ # Convert comma-delimited string into a list. Lists in CMake are just
+ # semicolon-delimited 🤢
+ string(REGEX MATCHALL "[^,]+" _CONFIGURATION_LIST "${_CONFIGURATION}")
+
+ # Extract the special keys.
+ list(GET _CONFIGURATION_LIST ${_SRC_KEY_INDEX} _TEST_SRC)
+ list(GET _CONFIGURATION_LIST ${_TARGET_BACKENDS_KEY_INDEX} _TEST_TARGET_BACKENDS)
+
+ # Construct the test name, which is the base name followed by the salient
+ # part of the source file name, the target backend name, and finally the
+ # other matrix keys and values if there is more than one value for a given
+ # key.
+ set(_TEST_NAME_LIST "${_RULE_NAME}")
+ string(REGEX REPLACE "\.py$" "" _STRIPPED_SRC_NAME "${_TEST_SRC}")
+ string(REGEX REPLACE "_test$" "" _STRIPPED_SRC_NAME "${_STRIPPED_SRC_NAME}")
+ list(APPEND _TEST_NAME_LIST "${_STRIPPED_SRC_NAME}" "${_TEST_TARGET_BACKENDS}")
+
+ # Append the key and value for all other matrix keys if there is more than
+ # one value for the given key.
+ foreach(_INDEX RANGE "${_MAX_INDEX}")
+ list(GET _RULE_MATRIX_VALUES ${_INDEX} _KEY_MATRIX_VALUES)
+ list(LENGTH _KEY_MATRIX_VALUES _KEY_MATRIX_VALUES_COUNT)
+ if (NOT _INDEX EQUAL _SRC_KEY_INDEX AND
+ NOT _INDEX EQUAL _TARGET_BACKENDS_KEY_INDEX AND
+ NOT _KEY_MATRIX_VALUES_COUNT EQUAL 1)
+ list(GET _RULE_MATRIX_KEYS ${_INDEX} _MATRIX_KEY)
+ list(GET _CONFIGURATION_LIST ${_INDEX} _MATRIX_VALUE)
+ list(APPEND _TEST_NAME_LIST "${_MATRIX_KEY}" "${_MATRIX_VALUE}")
+ endif()
+ endforeach()
+ list(JOIN _TEST_NAME_LIST "__" _TEST_NAME)
+
+ # Consruct the test args
+ set(_TEST_ARGS "")
+ foreach(_INDEX RANGE "${_MAX_INDEX}")
+ if (NOT _INDEX EQUAL _SRC_KEY_INDEX)
+ list(GET _RULE_MATRIX_KEYS ${_INDEX} _MATRIX_KEY)
+ list(GET _CONFIGURATION_LIST ${_INDEX} _MATRIX_VALUE)
+ list(APPEND _TEST_ARGS "--${_MATRIX_KEY}=${_MATRIX_VALUE}")
+ endif()
+ endforeach()
+
+ # Extract the driver label
+ # TODO(#2175): Get rid of this when we have a better specification for
+ # backends.
+ string(REPLACE "iree_" "" _DRIVER "${_TEST_TARGET_BACKENDS}")
+ set(_TEST_LABELS "${_RULE_LABELS}")
+ list(APPEND _TEST_LABELS "driver=${_DRIVER}")
+
+ iree_py_test(
+ NAME
+ ${_TEST_NAME}
+ SRCS
+ "${_TEST_SRC}"
+ ARGS
+ ${_TEST_ARGS}
+ LABELS
+ ${_TEST_LABELS}
+ )
+ endforeach()
+endfunction()
diff --git a/integrations/tensorflow/e2e/keras/BUILD b/integrations/tensorflow/e2e/keras/BUILD
index 111bf8f..0c2ea94 100644
--- a/integrations/tensorflow/e2e/keras/BUILD
+++ b/integrations/tensorflow/e2e/keras/BUILD
@@ -135,6 +135,10 @@
"iree_vulkan",
],
},
+ tags = [
+ # Kokoro doesn't have kws_streaming installed
+ "nokokoro",
+ ],
deps = [
"//third_party/google_research/google_research/kws_streaming/models:models_lib",
"//third_party/google_research/google_research/kws_streaming/train:train_lib",
@@ -172,6 +176,10 @@
"iree_vulkan",
],
},
+ tags = [
+ # Kokoro doesn't have kws_streaming installed
+ "nokokoro",
+ ],
deps = [
"//third_party/google_research/google_research/kws_streaming/models:models_lib",
"//third_party/google_research/google_research/kws_streaming/train:train_lib",
@@ -216,6 +224,10 @@
"iree_vulkan",
],
},
+ tags = [
+ # Kokoro doesn't have kws_streaming installed
+ "nokokoro",
+ ],
deps = [
"//third_party/google_research/google_research/kws_streaming/models:models_lib",
"//third_party/google_research/google_research/kws_streaming/train:train_lib",
diff --git a/integrations/tensorflow/e2e/keras/CMakeLists.txt b/integrations/tensorflow/e2e/keras/CMakeLists.txt
index 4be3e81..9a07928 100644
--- a/integrations/tensorflow/e2e/keras/CMakeLists.txt
+++ b/integrations/tensorflow/e2e/keras/CMakeLists.txt
@@ -1,15 +1,117 @@
-# Copyright 2021 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+################################################################################
+# Autogenerated by build_tools/bazel_to_cmake/bazel_to_cmake.py from #
+# integrations/tensorflow/e2e/keras/BUILD #
+# #
+# Use iree_cmake_extra_content from iree/build_defs.oss.bzl to add arbitrary #
+# CMake-only content. #
+# #
+# To disable autogeneration for this file entirely, delete this header. #
+################################################################################
-add_subdirectory(train)
+iree_add_all_subdirs()
+
+file(GLOB _GLOB_X_TEST_PY LIST_DIRECTORIES false RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} CONFIGURE_DEPENDS *_test.py)
+file(GLOB _GLOB_KEYWORD_SPOTTING_STREAMING_TEST_PY LIST_DIRECTORIES false RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} CONFIGURE_DEPENDS keyword_spotting_streaming_test.py)
+list(REMOVE_ITEM _GLOB_X_TEST_PY ${_GLOB_KEYWORD_SPOTTING_STREAMING_TEST_PY})
+iree_e2e_cartesian_product_test_suite(
+ NAME
+ keyword_spotting_tests
+ MATRIX_KEYS
+ "src"
+ "reference_backend"
+ "mode"
+ "model"
+ "target_backends"
+ MATRIX_VALUES
+ "keyword_spotting_streaming_test.py"
+ "tf"
+ "non_streaming"
+ "svdf;svdf_resnet;ds_cnn;gru;lstm;cnn_stride;cnn;tc_resnet;crnn;dnn;att_rnn;att_mh_rnn;mobilenet;mobilenet_v2;xception;inception;inception_resnet;ds_tc_resnet"
+ "tf;tflite;iree_vmla;iree_llvmaot;iree_vulkan"
+ FAILING_CONFIGURATIONS
+ ",,,att_mh_rnn,iree_vmla"
+ ",,,att_rnn,iree_vmla"
+ ",,,crnn,iree_vmla"
+ ",,,gru,iree_vmla"
+ ",,,lstm,iree_vmla"
+ ",,,att_mh_rnn,iree_vulkan"
+ ",,,att_rnn,iree_vulkan"
+ ",,,gru,iree_vulkan"
+ ",,,crnn,iree_vulkan"
+ ",,,lstm,iree_vulkan"
+ ",,,att_mh_rnn,iree_llvmaot"
+ ",,,att_rnn,iree_llvmaot"
+ ",,,crnn,iree_llvmaot"
+ ",,,gru,iree_llvmaot"
+ ",,,lstm,iree_llvmaot"
+ LABELS
+ "nokokoro"
+)
+
+iree_e2e_cartesian_product_test_suite(
+ NAME
+ keyword_spotting_internal_streaming_tests
+ MATRIX_KEYS
+ "src"
+ "reference_backend"
+ "mode"
+ "model"
+ "target_backends"
+ MATRIX_VALUES
+ "keyword_spotting_streaming_test.py"
+ "tf"
+ "internal_streaming"
+ "svdf;svdf_resnet;ds_cnn;gru;lstm;cnn_stride;cnn;tc_resnet;crnn;dnn;att_rnn;att_mh_rnn;mobilenet;mobilenet_v2;xception;inception;inception_resnet;ds_tc_resnet"
+ "tf;tflite;iree_vmla;iree_llvmaot;iree_vulkan"
+ FAILING_CONFIGURATIONS
+ ",,,,tflite"
+ ",,,att_mh_rnn,"
+ ",,,att_rnn,"
+ ",,,ds_cnn,"
+ ",,,inception,"
+ ",,,inception_resnet,"
+ ",,,mobilenet,"
+ ",,,mobilenet_v2,"
+ ",,,svdf_resnet,"
+ ",,,tc_resnet,"
+ ",,,xception,"
+ LABELS
+ "nokokoro"
+)
+
+iree_e2e_cartesian_product_test_suite(
+ NAME
+ keyword_spotting_external_streaming_tests
+ MATRIX_KEYS
+ "src"
+ "reference_backend"
+ "mode"
+ "model"
+ "target_backends"
+ MATRIX_VALUES
+ "keyword_spotting_streaming_test.py"
+ "tf"
+ "external_streaming"
+ "svdf;svdf_resnet;ds_cnn;gru;lstm;cnn_stride;cnn;tc_resnet;crnn;dnn;att_rnn;att_mh_rnn;mobilenet;mobilenet_v2;xception;inception;inception_resnet;ds_tc_resnet"
+ "tf;tflite;iree_vmla;iree_llvmaot;iree_vulkan"
+ FAILING_CONFIGURATIONS
+ ",,,,tf"
+ ",,,,tflite"
+ ",,,,iree_vmla"
+ ",,,,iree_llvmaot"
+ ",,,,iree_vulkan"
+ ",,,att_mh_rnn,"
+ ",,,att_rnn,"
+ ",,,ds_cnn,"
+ ",,,inception,"
+ ",,,inception_resnet,"
+ ",,,mobilenet,"
+ ",,,mobilenet_v2,"
+ ",,,svdf_resnet,"
+ ",,,tc_resnet,"
+ ",,,xception,"
+ LABELS
+ "nokokoro"
+)
+
+### BAZEL_TO_CMAKE_PRESERVES_ALL_CONTENT_BELOW_THIS_LINE ###
diff --git a/integrations/tensorflow/e2e/keras/applications/CMakeLists.txt b/integrations/tensorflow/e2e/keras/applications/CMakeLists.txt
new file mode 100644
index 0000000..d52c264
--- /dev/null
+++ b/integrations/tensorflow/e2e/keras/applications/CMakeLists.txt
@@ -0,0 +1,138 @@
+################################################################################
+# Autogenerated by build_tools/bazel_to_cmake/bazel_to_cmake.py from #
+# integrations/tensorflow/e2e/keras/applications/BUILD #
+# #
+# Use iree_cmake_extra_content from iree/build_defs.oss.bzl to add arbitrary #
+# CMake-only content. #
+# #
+# To disable autogeneration for this file entirely, delete this header. #
+################################################################################
+
+iree_add_all_subdirs()
+
+file(GLOB _GLOB_X_TEST_PY LIST_DIRECTORIES false RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} CONFIGURE_DEPENDS *_test.py)
+iree_e2e_cartesian_product_test_suite(
+ NAME
+ large_cifar10_tests
+ MATRIX_KEYS
+ "src"
+ "reference_backend"
+ "data"
+ "model"
+ "target_backends"
+ MATRIX_VALUES
+ "applications_test.py"
+ "tf"
+ "cifar10"
+ "MobileNet;MobileNetV2;ResNet50;VGG16;VGG19"
+ "tf;tflite;iree_vmla;iree_llvmaot;iree_vulkan"
+ LABELS
+ "manual"
+)
+
+iree_e2e_cartesian_product_test_suite(
+ NAME
+ enormous_cifar10_tests
+ MATRIX_KEYS
+ "src"
+ "reference_backend"
+ "data"
+ "model"
+ "target_backends"
+ MATRIX_VALUES
+ "applications_test.py"
+ "tf"
+ "cifar10"
+ "DenseNet121;DenseNet169;DenseNet201;NASNetLarge;NASNetMobile;ResNet50V2;ResNet101;ResNet101V2;ResNet152;ResNet152V2"
+ "tf;tflite;iree_vmla;iree_llvmaot;iree_vulkan"
+ FAILING_CONFIGURATIONS
+ ",,,NASNetLarge,iree_vmla"
+ ",,,NASNetMobile,iree_vmla"
+ ",,,NASNetLarge,iree_llvmaot"
+ ",,,NASNetLarge,iree_vulkan"
+ ",,,NASNetMobile,iree_llvmaot"
+ ",,,NASNetMobile,iree_vulkan"
+ ",,,ResNet50V2,iree_llvmaot"
+ ",,,ResNet50V2,iree_vulkan"
+ ",,,ResNet101V2,iree_llvmaot"
+ ",,,ResNet101V2,iree_vulkan"
+ ",,,ResNet152V2,iree_llvmaot"
+ ",,,ResNet152V2,iree_vulkan"
+ LABELS
+ "guitar"
+ "manual"
+ "nokokoro"
+ "notap"
+)
+
+iree_e2e_cartesian_product_test_suite(
+ NAME
+ cifar10_non_hermetic_tests
+ MATRIX_KEYS
+ "src"
+ "reference_backend"
+ "data"
+ "url"
+ "use_external_weights"
+ "model"
+ "target_backends"
+ MATRIX_VALUES
+ "applications_test.py"
+ "tf"
+ "cifar10"
+ "https://storage.googleapis.com/iree_models/"
+ "True"
+ "MobileNet;MobileNetV2;ResNet50"
+ "tf;tflite;iree_vmla;iree_llvmaot;iree_vulkan"
+ LABELS
+ "external"
+ "guitar"
+ "manual"
+ "no-remote"
+ "nokokoro"
+ "notap"
+)
+
+iree_e2e_cartesian_product_test_suite(
+ NAME
+ imagenet_non_hermetic_tests
+ MATRIX_KEYS
+ "src"
+ "reference_backend"
+ "data"
+ "use_external_weights"
+ "model"
+ "target_backends"
+ MATRIX_VALUES
+ "applications_test.py"
+ "tf"
+ "imagenet"
+ "True"
+ "DenseNet121;DenseNet169;DenseNet201;EfficientNetB0;EfficientNetB1;EfficientNetB2;EfficientNetB3;EfficientNetB4;EfficientNetB5;EfficientNetB6;EfficientNetB7;InceptionResNetV2;InceptionV3;MobileNet;MobileNetV2;MobileNetV3Large;MobileNetV3Small;NASNetLarge;NASNetMobile;ResNet101;ResNet101V2;ResNet152;ResNet152V2;ResNet50;ResNet50V2;VGG16;VGG19"
+ "tf;tflite;iree_vmla;iree_llvmaot;iree_vulkan"
+ FAILING_CONFIGURATIONS
+ ",,,,NASNetLarge,iree_vmla"
+ ",,,,NASNetMobile,iree_vmla"
+ ",,,,InceptionResNetV2,iree_vulkan"
+ ",,,,InceptionV3,iree_vulkan"
+ ",,,,NASNetLarge,iree_llvmaot"
+ ",,,,NASNetLarge,iree_vulkan"
+ ",,,,NASNetMobile,iree_llvmaot"
+ ",,,,NASNetMobile,iree_vulkan"
+ ",,,,ResNet50V2,iree_llvmaot"
+ ",,,,ResNet50V2,iree_vulkan"
+ ",,,,ResNet101V2,iree_llvmaot"
+ ",,,,ResNet101V2,iree_vulkan"
+ ",,,,ResNet152V2,iree_llvmaot"
+ ",,,,ResNet152V2,iree_vulkan"
+ ",,,,Xception,iree_llvmaot"
+ ",,,,Xception,iree_vulkan"
+ LABELS
+ "external"
+ "guitar"
+ "manual"
+ "nokokoro"
+ "notap"
+)
+
+### BAZEL_TO_CMAKE_PRESERVES_ALL_CONTENT_BELOW_THIS_LINE ###
diff --git a/integrations/tensorflow/e2e/keras/layers/BUILD b/integrations/tensorflow/e2e/keras/layers/BUILD
index b870eda..b18308c 100644
--- a/integrations/tensorflow/e2e/keras/layers/BUILD
+++ b/integrations/tensorflow/e2e/keras/layers/BUILD
@@ -44,10 +44,7 @@
"//util/debuginfo:signalsafe_addr2line_installer",
],
)
- for src in glob(
- ["*_test.py"],
- exclude = ["keyword_spotting_streaming_test.py"],
- )
+ for src in glob(["*_test.py"])
]
# These layers were selected by:
@@ -265,16 +262,19 @@
FAILING_FULL_API = [
{
# Failing on TFLite
+ # keep sorted
"layer": [
"AveragePooling3D",
+ "Conv2DTranspose",
"Conv3D",
"Conv3DTranspose",
"ConvLSTM2D",
"DepthwiseConv2D",
"GRU",
+ "LSTM",
"LocallyConnected1D",
"LocallyConnected2D",
- "LSTM",
+ "MaxPool1D",
"MaxPool3D",
"SeparableConv1D", # Failing on Kokoro.
"SeparableConv2D",
@@ -379,11 +379,13 @@
},
{
# Failing on IREE
+ # keep sorted
"layer": [
"AdditiveAttention",
"AveragePooling1D",
"AveragePooling2D",
"AveragePooling3D",
+ "BatchNormalization",
"Concatenate",
"Conv1D",
"Conv1DTranspose",
@@ -401,11 +403,11 @@
"ELU",
"Flatten",
"GRU",
+ "LSTM", # TODO(silvasean): Get this test working on IREE.
"LayerNormalization",
"LeakyReLU",
"LocallyConnected1D",
"LocallyConnected2D",
- "LSTM", # TODO(silvasean): Get this test working on IREE.
"Masking",
"MaxPool1D",
"MaxPool2D",
@@ -433,23 +435,23 @@
},
{
# Failing on LLVM and Vulkan
+ # keep sorted
"layer": [
"Activation",
"Add",
"Attention",
"Average",
- "BatchNormalization",
"GlobalAveragePooling1D",
"GlobalAveragePooling2D",
"GlobalAveragePooling3D",
"GlobalMaxPool1D",
"GlobalMaxPool2D",
"GlobalMaxPool3D",
- "Permute",
"Lambda",
"Maximum",
"Minimum",
"Multiply",
+ "Permute",
"ReLU",
"Softmax",
"Subtract",
@@ -530,15 +532,17 @@
},
{
# Failing on IREE
+ # keep sorted
"layer": [
"AdditiveAttention",
"AlphaDropout",
"Attention",
+ "BatchNormalization",
"ConvLSTM2D",
"Dropout",
+ "GRU",
"GaussianDropout",
"GaussianNoise",
- "GRU",
"LSTM",
"MultiHeadAttention",
"SimpleRNN",
diff --git a/integrations/tensorflow/e2e/keras/layers/CMakeLists.txt b/integrations/tensorflow/e2e/keras/layers/CMakeLists.txt
new file mode 100644
index 0000000..e2d2e0f
--- /dev/null
+++ b/integrations/tensorflow/e2e/keras/layers/CMakeLists.txt
@@ -0,0 +1,449 @@
+################################################################################
+# Autogenerated by build_tools/bazel_to_cmake/bazel_to_cmake.py from #
+# integrations/tensorflow/e2e/keras/layers/BUILD #
+# #
+# Use iree_cmake_extra_content from iree/build_defs.oss.bzl to add arbitrary #
+# CMake-only content. #
+# #
+# To disable autogeneration for this file entirely, delete this header. #
+################################################################################
+
+iree_add_all_subdirs()
+
+file(GLOB _GLOB_X_TEST_PY LIST_DIRECTORIES false RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} CONFIGURE_DEPENDS *_test.py)
+iree_e2e_cartesian_product_test_suite(
+ NAME
+ layers_tests
+ MATRIX_KEYS
+ "src"
+ "reference_backend"
+ "layer"
+ "dynamic_dims"
+ "training"
+ "test_default_kwargs_only"
+ "target_backends"
+ MATRIX_VALUES
+ "layers_test.py"
+ "tf"
+ "Activation;ActivityRegularization;Add;AdditiveAttention;AlphaDropout;Attention;Average;AveragePooling1D;AveragePooling2D;AveragePooling3D;BatchNormalization;Concatenate;Conv1D;Conv1DTranspose;Conv2D;Conv2DTranspose;Conv3D;Conv3DTranspose;Cropping1D;Cropping2D;Cropping3D;Dense;DepthwiseConv2D;Dot;Dropout;ELU;Embedding;Flatten;GRU;GaussianDropout;GaussianNoise;GlobalAveragePooling1D;GlobalAveragePooling2D;GlobalAveragePooling3D;GlobalMaxPool1D;GlobalMaxPool2D;GlobalMaxPool3D;InputLayer;LSTM;Lambda;LayerNormalization;LeakyReLU;LocallyConnected1D;LocallyConnected2D;Masking;MaxPool1D;MaxPool2D;MaxPool3D;Maximum;Minimum;MultiHeadAttention;Multiply;PReLU;Permute;ReLU;RepeatVector;Reshape;SeparableConv1D;SeparableConv2D;Softmax;SpatialDropout1D;SpatialDropout2D;SpatialDropout3D;Subtract;ThresholdedReLU;UpSampling1D;UpSampling2D;UpSampling3D;ZeroPadding1D;ZeroPadding2D;ZeroPadding3D"
+ "False"
+ "False"
+ "True"
+ "tf;tflite;iree_vmla;iree_llvmaot;iree_vulkan"
+ FAILING_CONFIGURATIONS
+ ",,AveragePooling3D,,,,tflite"
+ ",,Conv3DTranspose,,,,tflite"
+ ",,Conv3D,,,,tflite"
+ ",,ConvLSTM2D,,,,tflite"
+ ",,LayerNormalization,,,,tflite"
+ ",,Softmax,,,,tflite"
+ ",,MaxPool3D,,,,tflite"
+ ",,ZeroPadding3D,,,,tflite"
+ ",,ConvLSTM2D,,,,iree_vmla"
+ ",,ConvLSTM2D,,,,iree_llvmaot"
+ ",,ConvLSTM2D,,,,iree_vulkan"
+ ",,GRU,,,,iree_vmla"
+ ",,GRU,,,,iree_llvmaot"
+ ",,GRU,,,,iree_vulkan"
+ ",,LSTM,,,,iree_vmla"
+ ",,LSTM,,,,iree_llvmaot"
+ ",,LSTM,,,,iree_vulkan"
+ ",,LayerNormalization,,,,iree_vmla"
+ ",,LayerNormalization,,,,iree_llvmaot"
+ ",,LayerNormalization,,,,iree_vulkan"
+ ",,LeakyReLU,,,,iree_vmla"
+ ",,LeakyReLU,,,,iree_llvmaot"
+ ",,LeakyReLU,,,,iree_vulkan"
+ ",,LocallyConnected2D,,,,iree_vmla"
+ ",,LocallyConnected2D,,,,iree_llvmaot"
+ ",,LocallyConnected2D,,,,iree_vulkan"
+ ",,MultiHeadAttention,,,,iree_vmla"
+ ",,MultiHeadAttention,,,,iree_llvmaot"
+ ",,MultiHeadAttention,,,,iree_vulkan"
+ ",,UpSampling2D,,,,iree_vmla"
+ ",,UpSampling2D,,,,iree_llvmaot"
+ ",,UpSampling2D,,,,iree_vulkan"
+ ",,Conv3DTranspose,,,,iree_vmla"
+ ",,Conv3D,,,,iree_vmla"
+ ",,Lambda,,,,iree_llvmaot"
+ ",,Lambda,,,,iree_vulkan"
+ ",,Masking,,,,iree_llvmaot"
+ ",,Masking,,,,iree_vulkan"
+ ",,MaxPool1D,,,,iree_llvmaot"
+ ",,MaxPool1D,,,,iree_vulkan"
+ ",,MaxPool2D,,,,iree_llvmaot"
+ ",,MaxPool2D,,,,iree_vulkan"
+ ",,MaxPool3D,,,,iree_llvmaot"
+ ",,MaxPool3D,,,,iree_vulkan"
+ ",,AveragePooling3D,,,,iree_llvmaot"
+ ",,AveragePooling3D,,,,iree_vulkan"
+ ",,AveragePooling1D,,,,iree_vulkan"
+ ",,AveragePooling2D,,,,iree_vulkan"
+ ",,ThresholdedReLU,,,,iree_vulkan"
+)
+
+iree_e2e_cartesian_product_test_suite(
+ NAME
+ layers_full_api_tests
+ MATRIX_KEYS
+ "src"
+ "reference_backend"
+ "layer"
+ "dynamic_dims"
+ "training"
+ "test_default_kwargs_only"
+ "target_backends"
+ MATRIX_VALUES
+ "layers_test.py"
+ "tf"
+ "ActivityRegularization;AdditiveAttention;Attention;AveragePooling1D;AveragePooling2D;AveragePooling3D;BatchNormalization;Concatenate;Conv1D;Conv1DTranspose;Conv2D;Conv2DTranspose;Conv3D;Conv3DTranspose;Cropping1D;Cropping2D;Cropping3D;DepthwiseConv2D;GRU;LSTM;LocallyConnected1D;LocallyConnected2D;MaxPool1D;MaxPool2D;MaxPool3D;SeparableConv1D;SeparableConv2D;SimpleRNN"
+ "False"
+ "False"
+ "False"
+ "tf;tflite;iree_vmla;iree_llvmaot;iree_vulkan"
+ FAILING_CONFIGURATIONS
+ ",,AveragePooling3D,,,,tflite"
+ ",,Conv2DTranspose,,,,tflite"
+ ",,Conv3D,,,,tflite"
+ ",,Conv3DTranspose,,,,tflite"
+ ",,ConvLSTM2D,,,,tflite"
+ ",,DepthwiseConv2D,,,,tflite"
+ ",,GRU,,,,tflite"
+ ",,LSTM,,,,tflite"
+ ",,LocallyConnected1D,,,,tflite"
+ ",,LocallyConnected2D,,,,tflite"
+ ",,MaxPool1D,,,,tflite"
+ ",,MaxPool3D,,,,tflite"
+ ",,SeparableConv1D,,,,tflite"
+ ",,SeparableConv2D,,,,tflite"
+ ",,SimpleRNN,,,,tflite"
+ ",,Conv3DTranspose,,,,iree_vmla"
+ ",,Conv3DTranspose,,,,iree_llvmaot"
+ ",,Conv3DTranspose,,,,iree_vulkan"
+ ",,ConvLSTM2D,,,,iree_vmla"
+ ",,ConvLSTM2D,,,,iree_llvmaot"
+ ",,ConvLSTM2D,,,,iree_vulkan"
+ ",,GRU,,,,iree_vmla"
+ ",,GRU,,,,iree_llvmaot"
+ ",,GRU,,,,iree_vulkan"
+ ",,LocallyConnected1D,,,,iree_vmla"
+ ",,LocallyConnected1D,,,,iree_llvmaot"
+ ",,LocallyConnected1D,,,,iree_vulkan"
+ ",,LocallyConnected2D,,,,iree_vmla"
+ ",,LocallyConnected2D,,,,iree_llvmaot"
+ ",,LocallyConnected2D,,,,iree_vulkan"
+ ",,LSTM,,,,iree_vmla"
+ ",,LSTM,,,,iree_llvmaot"
+ ",,LSTM,,,,iree_vulkan"
+ ",,SimpleRNN,,,,iree_vmla"
+ ",,SimpleRNN,,,,iree_llvmaot"
+ ",,SimpleRNN,,,,iree_vulkan"
+ ",,Conv3D,,,,iree_vmla"
+ ",,AveragePooling1D,,,,iree_llvmaot"
+ ",,AveragePooling1D,,,,iree_vulkan"
+ ",,AveragePooling2D,,,,iree_llvmaot"
+ ",,AveragePooling2D,,,,iree_vulkan"
+ ",,AveragePooling3D,,,,iree_llvmaot"
+ ",,AveragePooling3D,,,,iree_vulkan"
+ ",,Conv1DTranspose,,,,iree_llvmaot"
+ ",,Conv1DTranspose,,,,iree_vulkan"
+ ",,Conv2DTranspose,,,,iree_llvmaot"
+ ",,Conv2DTranspose,,,,iree_vulkan"
+ ",,MaxPool1D,,,,iree_llvmaot"
+ ",,MaxPool1D,,,,iree_vulkan"
+ ",,MaxPool2D,,,,iree_llvmaot"
+ ",,MaxPool2D,,,,iree_vulkan"
+ ",,MaxPool3D,,,,iree_llvmaot"
+ ",,MaxPool3D,,,,iree_vulkan"
+)
+
+iree_e2e_cartesian_product_test_suite(
+ NAME
+ layers_dynamic_dims_tests
+ MATRIX_KEYS
+ "src"
+ "reference_backend"
+ "layer"
+ "dynamic_dims"
+ "training"
+ "test_default_kwargs_only"
+ "target_backends"
+ MATRIX_VALUES
+ "layers_test.py"
+ "tf"
+ "Activation;ActivityRegularization;Add;AdditiveAttention;AlphaDropout;Attention;Average;AveragePooling1D;AveragePooling2D;AveragePooling3D;BatchNormalization;Concatenate;Conv1D;Conv1DTranspose;Conv2D;Conv2DTranspose;Conv3D;Conv3DTranspose;Cropping1D;Cropping2D;Cropping3D;Dense;DepthwiseConv2D;Dot;Dropout;ELU;Embedding;Flatten;GRU;GaussianDropout;GaussianNoise;GlobalAveragePooling1D;GlobalAveragePooling2D;GlobalAveragePooling3D;GlobalMaxPool1D;GlobalMaxPool2D;GlobalMaxPool3D;InputLayer;LSTM;Lambda;LayerNormalization;LeakyReLU;LocallyConnected1D;LocallyConnected2D;Masking;MaxPool1D;MaxPool2D;MaxPool3D;Maximum;Minimum;MultiHeadAttention;Multiply;PReLU;Permute;ReLU;RepeatVector;Reshape;SeparableConv1D;SeparableConv2D;Softmax;SpatialDropout1D;SpatialDropout2D;SpatialDropout3D;Subtract;ThresholdedReLU;UpSampling1D;UpSampling2D;UpSampling3D;ZeroPadding1D;ZeroPadding2D;ZeroPadding3D"
+ "True"
+ "False"
+ "True"
+ "tf;tflite;iree_vmla;iree_llvmaot;iree_vulkan"
+ FAILING_CONFIGURATIONS
+ ",,Add,,,,tflite"
+ ",,Average,,,,tflite"
+ ",,AveragePooling3D,,,,tflite"
+ ",,BatchNormalization,,,,tflite"
+ ",,Conv3D,,,,tflite"
+ ",,Conv3DTranspose,,,,tflite"
+ ",,ConvLSTM2D,,,,tflite"
+ ",,Dense,,,,tflite"
+ ",,GRU,,,,tflite"
+ ",,LSTM,,,,tflite"
+ ",,Lambda,,,,tflite"
+ ",,LayerNormalization,,,,tflite"
+ ",,Masking,,,,tflite"
+ ",,MaxPool3D,,,,tflite"
+ ",,Multiply,,,,tflite"
+ ",,PReLU,,,,tflite"
+ ",,SimpleRNN,,,,tflite"
+ ",,Softmax,,,,tflite"
+ ",,ThresholdedReLU,,,,tflite"
+ ",,ZeroPadding3D,,,,tflite"
+ ",,AdditiveAttention,,,,iree_vmla"
+ ",,AdditiveAttention,,,,iree_llvmaot"
+ ",,AdditiveAttention,,,,iree_vulkan"
+ ",,AveragePooling1D,,,,iree_vmla"
+ ",,AveragePooling1D,,,,iree_llvmaot"
+ ",,AveragePooling1D,,,,iree_vulkan"
+ ",,AveragePooling2D,,,,iree_vmla"
+ ",,AveragePooling2D,,,,iree_llvmaot"
+ ",,AveragePooling2D,,,,iree_vulkan"
+ ",,AveragePooling3D,,,,iree_vmla"
+ ",,AveragePooling3D,,,,iree_llvmaot"
+ ",,AveragePooling3D,,,,iree_vulkan"
+ ",,BatchNormalization,,,,iree_vmla"
+ ",,BatchNormalization,,,,iree_llvmaot"
+ ",,BatchNormalization,,,,iree_vulkan"
+ ",,Concatenate,,,,iree_vmla"
+ ",,Concatenate,,,,iree_llvmaot"
+ ",,Concatenate,,,,iree_vulkan"
+ ",,Conv1D,,,,iree_vmla"
+ ",,Conv1D,,,,iree_llvmaot"
+ ",,Conv1D,,,,iree_vulkan"
+ ",,Conv1DTranspose,,,,iree_vmla"
+ ",,Conv1DTranspose,,,,iree_llvmaot"
+ ",,Conv1DTranspose,,,,iree_vulkan"
+ ",,Conv2D,,,,iree_vmla"
+ ",,Conv2D,,,,iree_llvmaot"
+ ",,Conv2D,,,,iree_vulkan"
+ ",,Conv2DTranspose,,,,iree_vmla"
+ ",,Conv2DTranspose,,,,iree_llvmaot"
+ ",,Conv2DTranspose,,,,iree_vulkan"
+ ",,Conv3D,,,,iree_vmla"
+ ",,Conv3D,,,,iree_llvmaot"
+ ",,Conv3D,,,,iree_vulkan"
+ ",,Conv3DTranspose,,,,iree_vmla"
+ ",,Conv3DTranspose,,,,iree_llvmaot"
+ ",,Conv3DTranspose,,,,iree_vulkan"
+ ",,ConvLSTM2D,,,,iree_vmla"
+ ",,ConvLSTM2D,,,,iree_llvmaot"
+ ",,ConvLSTM2D,,,,iree_vulkan"
+ ",,Cropping1D,,,,iree_vmla"
+ ",,Cropping1D,,,,iree_llvmaot"
+ ",,Cropping1D,,,,iree_vulkan"
+ ",,Cropping2D,,,,iree_vmla"
+ ",,Cropping2D,,,,iree_llvmaot"
+ ",,Cropping2D,,,,iree_vulkan"
+ ",,Cropping3D,,,,iree_vmla"
+ ",,Cropping3D,,,,iree_llvmaot"
+ ",,Cropping3D,,,,iree_vulkan"
+ ",,Dense,,,,iree_vmla"
+ ",,Dense,,,,iree_llvmaot"
+ ",,Dense,,,,iree_vulkan"
+ ",,DepthwiseConv2D,,,,iree_vmla"
+ ",,DepthwiseConv2D,,,,iree_llvmaot"
+ ",,DepthwiseConv2D,,,,iree_vulkan"
+ ",,Dot,,,,iree_vmla"
+ ",,Dot,,,,iree_llvmaot"
+ ",,Dot,,,,iree_vulkan"
+ ",,ELU,,,,iree_vmla"
+ ",,ELU,,,,iree_llvmaot"
+ ",,ELU,,,,iree_vulkan"
+ ",,Flatten,,,,iree_vmla"
+ ",,Flatten,,,,iree_llvmaot"
+ ",,Flatten,,,,iree_vulkan"
+ ",,GRU,,,,iree_vmla"
+ ",,GRU,,,,iree_llvmaot"
+ ",,GRU,,,,iree_vulkan"
+ ",,LSTM,,,,iree_vmla"
+ ",,LSTM,,,,iree_llvmaot"
+ ",,LSTM,,,,iree_vulkan"
+ ",,LayerNormalization,,,,iree_vmla"
+ ",,LayerNormalization,,,,iree_llvmaot"
+ ",,LayerNormalization,,,,iree_vulkan"
+ ",,LeakyReLU,,,,iree_vmla"
+ ",,LeakyReLU,,,,iree_llvmaot"
+ ",,LeakyReLU,,,,iree_vulkan"
+ ",,LocallyConnected1D,,,,iree_vmla"
+ ",,LocallyConnected1D,,,,iree_llvmaot"
+ ",,LocallyConnected1D,,,,iree_vulkan"
+ ",,LocallyConnected2D,,,,iree_vmla"
+ ",,LocallyConnected2D,,,,iree_llvmaot"
+ ",,LocallyConnected2D,,,,iree_vulkan"
+ ",,Masking,,,,iree_vmla"
+ ",,Masking,,,,iree_llvmaot"
+ ",,Masking,,,,iree_vulkan"
+ ",,MaxPool1D,,,,iree_vmla"
+ ",,MaxPool1D,,,,iree_llvmaot"
+ ",,MaxPool1D,,,,iree_vulkan"
+ ",,MaxPool2D,,,,iree_vmla"
+ ",,MaxPool2D,,,,iree_llvmaot"
+ ",,MaxPool2D,,,,iree_vulkan"
+ ",,MaxPool3D,,,,iree_vmla"
+ ",,MaxPool3D,,,,iree_llvmaot"
+ ",,MaxPool3D,,,,iree_vulkan"
+ ",,MultiHeadAttention,,,,iree_vmla"
+ ",,MultiHeadAttention,,,,iree_llvmaot"
+ ",,MultiHeadAttention,,,,iree_vulkan"
+ ",,PReLU,,,,iree_vmla"
+ ",,PReLU,,,,iree_llvmaot"
+ ",,PReLU,,,,iree_vulkan"
+ ",,RepeatVector,,,,iree_vmla"
+ ",,RepeatVector,,,,iree_llvmaot"
+ ",,RepeatVector,,,,iree_vulkan"
+ ",,Reshape,,,,iree_vmla"
+ ",,Reshape,,,,iree_llvmaot"
+ ",,Reshape,,,,iree_vulkan"
+ ",,SeparableConv1D,,,,iree_vmla"
+ ",,SeparableConv1D,,,,iree_llvmaot"
+ ",,SeparableConv1D,,,,iree_vulkan"
+ ",,SeparableConv2D,,,,iree_vmla"
+ ",,SeparableConv2D,,,,iree_llvmaot"
+ ",,SeparableConv2D,,,,iree_vulkan"
+ ",,SimpleRNN,,,,iree_vmla"
+ ",,SimpleRNN,,,,iree_llvmaot"
+ ",,SimpleRNN,,,,iree_vulkan"
+ ",,ThresholdedReLU,,,,iree_vmla"
+ ",,ThresholdedReLU,,,,iree_llvmaot"
+ ",,ThresholdedReLU,,,,iree_vulkan"
+ ",,UpSampling1D,,,,iree_vmla"
+ ",,UpSampling1D,,,,iree_llvmaot"
+ ",,UpSampling1D,,,,iree_vulkan"
+ ",,UpSampling2D,,,,iree_vmla"
+ ",,UpSampling2D,,,,iree_llvmaot"
+ ",,UpSampling2D,,,,iree_vulkan"
+ ",,UpSampling3D,,,,iree_vmla"
+ ",,UpSampling3D,,,,iree_llvmaot"
+ ",,UpSampling3D,,,,iree_vulkan"
+ ",,ZeroPadding1D,,,,iree_vmla"
+ ",,ZeroPadding1D,,,,iree_llvmaot"
+ ",,ZeroPadding1D,,,,iree_vulkan"
+ ",,ZeroPadding2D,,,,iree_vmla"
+ ",,ZeroPadding2D,,,,iree_llvmaot"
+ ",,ZeroPadding2D,,,,iree_vulkan"
+ ",,ZeroPadding3D,,,,iree_vmla"
+ ",,ZeroPadding3D,,,,iree_llvmaot"
+ ",,ZeroPadding3D,,,,iree_vulkan"
+ ",,Activation,,,,iree_llvmaot"
+ ",,Activation,,,,iree_vulkan"
+ ",,Add,,,,iree_llvmaot"
+ ",,Add,,,,iree_vulkan"
+ ",,Attention,,,,iree_llvmaot"
+ ",,Attention,,,,iree_vulkan"
+ ",,Average,,,,iree_llvmaot"
+ ",,Average,,,,iree_vulkan"
+ ",,GlobalAveragePooling1D,,,,iree_llvmaot"
+ ",,GlobalAveragePooling1D,,,,iree_vulkan"
+ ",,GlobalAveragePooling2D,,,,iree_llvmaot"
+ ",,GlobalAveragePooling2D,,,,iree_vulkan"
+ ",,GlobalAveragePooling3D,,,,iree_llvmaot"
+ ",,GlobalAveragePooling3D,,,,iree_vulkan"
+ ",,GlobalMaxPool1D,,,,iree_llvmaot"
+ ",,GlobalMaxPool1D,,,,iree_vulkan"
+ ",,GlobalMaxPool2D,,,,iree_llvmaot"
+ ",,GlobalMaxPool2D,,,,iree_vulkan"
+ ",,GlobalMaxPool3D,,,,iree_llvmaot"
+ ",,GlobalMaxPool3D,,,,iree_vulkan"
+ ",,Lambda,,,,iree_llvmaot"
+ ",,Lambda,,,,iree_vulkan"
+ ",,Maximum,,,,iree_llvmaot"
+ ",,Maximum,,,,iree_vulkan"
+ ",,Minimum,,,,iree_llvmaot"
+ ",,Minimum,,,,iree_vulkan"
+ ",,Multiply,,,,iree_llvmaot"
+ ",,Multiply,,,,iree_vulkan"
+ ",,Permute,,,,iree_llvmaot"
+ ",,Permute,,,,iree_vulkan"
+ ",,ReLU,,,,iree_llvmaot"
+ ",,ReLU,,,,iree_vulkan"
+ ",,Softmax,,,,iree_llvmaot"
+ ",,Softmax,,,,iree_vulkan"
+ ",,Subtract,,,,iree_llvmaot"
+ ",,Subtract,,,,iree_vulkan"
+ ",,Embedding,,,,iree_vulkan"
+)
+
+iree_e2e_cartesian_product_test_suite(
+ NAME
+ layers_training_tests
+ MATRIX_KEYS
+ "src"
+ "reference_backend"
+ "layer"
+ "dynamic_dims"
+ "training"
+ "test_default_kwargs_only"
+ "target_backends"
+ MATRIX_VALUES
+ "layers_test.py"
+ "tf"
+ "AdditiveAttention;AlphaDropout;Attention;BatchNormalization;Dropout;GRU;GaussianDropout;GaussianNoise;LSTM;MultiHeadAttention;SpatialDropout1D;SpatialDropout2D;SpatialDropout3D"
+ "False"
+ "True"
+ "True"
+ "tf;tflite;iree_vmla;iree_llvmaot;iree_vulkan"
+ FAILING_CONFIGURATIONS
+ ",,AlphaDropout,,,,tflite"
+ ",,BatchNormalization,,,,tflite"
+ ",,ConvLSTM2D,,,,tflite"
+ ",,GaussianDropout,,,,tflite"
+ ",,GaussianNoise,,,,tflite"
+ ",,GRU,,,,tflite"
+ ",,LSTM,,,,tflite"
+ ",,SimpleRNN,,,,tflite"
+ ",,AdditiveAttention,,,,iree_vmla"
+ ",,AdditiveAttention,,,,iree_llvmaot"
+ ",,AdditiveAttention,,,,iree_vulkan"
+ ",,AlphaDropout,,,,iree_vmla"
+ ",,AlphaDropout,,,,iree_llvmaot"
+ ",,AlphaDropout,,,,iree_vulkan"
+ ",,Attention,,,,iree_vmla"
+ ",,Attention,,,,iree_llvmaot"
+ ",,Attention,,,,iree_vulkan"
+ ",,BatchNormalization,,,,iree_vmla"
+ ",,BatchNormalization,,,,iree_llvmaot"
+ ",,BatchNormalization,,,,iree_vulkan"
+ ",,ConvLSTM2D,,,,iree_vmla"
+ ",,ConvLSTM2D,,,,iree_llvmaot"
+ ",,ConvLSTM2D,,,,iree_vulkan"
+ ",,Dropout,,,,iree_vmla"
+ ",,Dropout,,,,iree_llvmaot"
+ ",,Dropout,,,,iree_vulkan"
+ ",,GRU,,,,iree_vmla"
+ ",,GRU,,,,iree_llvmaot"
+ ",,GRU,,,,iree_vulkan"
+ ",,GaussianDropout,,,,iree_vmla"
+ ",,GaussianDropout,,,,iree_llvmaot"
+ ",,GaussianDropout,,,,iree_vulkan"
+ ",,GaussianNoise,,,,iree_vmla"
+ ",,GaussianNoise,,,,iree_llvmaot"
+ ",,GaussianNoise,,,,iree_vulkan"
+ ",,LSTM,,,,iree_vmla"
+ ",,LSTM,,,,iree_llvmaot"
+ ",,LSTM,,,,iree_vulkan"
+ ",,MultiHeadAttention,,,,iree_vmla"
+ ",,MultiHeadAttention,,,,iree_llvmaot"
+ ",,MultiHeadAttention,,,,iree_vulkan"
+ ",,SimpleRNN,,,,iree_vmla"
+ ",,SimpleRNN,,,,iree_llvmaot"
+ ",,SimpleRNN,,,,iree_vulkan"
+ ",,SpatialDropout1D,,,,iree_vmla"
+ ",,SpatialDropout1D,,,,iree_llvmaot"
+ ",,SpatialDropout1D,,,,iree_vulkan"
+ ",,SpatialDropout2D,,,,iree_vmla"
+ ",,SpatialDropout2D,,,,iree_llvmaot"
+ ",,SpatialDropout2D,,,,iree_vulkan"
+ ",,SpatialDropout3D,,,,iree_vmla"
+ ",,SpatialDropout3D,,,,iree_llvmaot"
+ ",,SpatialDropout3D,,,,iree_vulkan"
+)
+
+### BAZEL_TO_CMAKE_PRESERVES_ALL_CONTENT_BELOW_THIS_LINE ###
diff --git a/integrations/tensorflow/e2e/keras/train/CMakeLists.txt b/integrations/tensorflow/e2e/keras/train/CMakeLists.txt
index c4b9bb6..1906dd4 100644
--- a/integrations/tensorflow/e2e/keras/train/CMakeLists.txt
+++ b/integrations/tensorflow/e2e/keras/train/CMakeLists.txt
@@ -1,47 +1,34 @@
-# Copyright 2021 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+################################################################################
+# Autogenerated by build_tools/bazel_to_cmake/bazel_to_cmake.py from #
+# integrations/tensorflow/e2e/keras/train/BUILD #
+# #
+# Use iree_cmake_extra_content from iree/build_defs.oss.bzl to add arbitrary #
+# CMake-only content. #
+# #
+# To disable autogeneration for this file entirely, delete this header. #
+################################################################################
-set(REFERENCE_BACKEND tf)
+iree_add_all_subdirs()
-function(add_e2e_test filename target_backend labels)
- iree_package_ns(_PACKAGE_NS)
- string(REPLACE "::" "/" _PACKAGE_PATH ${_PACKAGE_NS})
- string(REPLACE ".py" "" _name ${filename})
+file(GLOB _GLOB_X_TEST_PY LIST_DIRECTORIES false RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} CONFIGURE_DEPENDS *_test.py)
+iree_e2e_cartesian_product_test_suite(
+ NAME
+ training_tests
+ MATRIX_KEYS
+ "src"
+ "reference_backend"
+ "optimizer"
+ "target_backends"
+ MATRIX_VALUES
+ "classification_training_test.py;regression_training_test.py"
+ "tf"
+ "adadelta;adagrad;adam;adamax;ftrl;nadam;rmsprop;sgd"
+ "tf;tflite;iree_vmla;iree_llvmaot;iree_vulkan"
+ FAILING_CONFIGURATIONS
+ ",,,tflite"
+ ",,,iree_vmla"
+ ",,,iree_llvmaot"
+ ",,,iree_vulkan"
+)
- set(_name "${_PACKAGE_PATH}/${_name}__${target_backend}")
- add_test(
- NAME
- ${_name}
- WORKING_DIRECTORY
- "${CMAKE_CURRENT_BINARY_DIR}"
- COMMAND
- "${Python3_EXECUTABLE}" -B
- "${CMAKE_CURRENT_SOURCE_DIR}/${filename}"
- "--reference_backend=${REFERENCE_BACKEND}"
- "--target_backends=${target_backend}"
- )
- set_property(TEST ${_name} PROPERTY LABELS "${labels}")
- set_property(TEST ${_name} PROPERTY ENVIRONMENT
- "PYTHONPATH=${CMAKE_BINARY_DIR}/bindings/python")
-endfunction()
-
-add_e2e_test(regression_training_test.py tf "")
-add_e2e_test(regression_training_test.py iree_vmla "")
-add_e2e_test(regression_training_test.py iree_llvmaot "driver=dylib")
-add_e2e_test(regression_training_test.py iree_vulkan "driver=vulkan")
-
-add_e2e_test(classification_training_test.py tf "")
-add_e2e_test(classification_training_test.py iree_vmla "")
-add_e2e_test(classification_training_test.py iree_llvmaot "driver=dylib")
-add_e2e_test(classification_training_test.py iree_vulkan "driver=vulkan")
+### BAZEL_TO_CMAKE_PRESERVES_ALL_CONTENT_BELOW_THIS_LINE ###
diff --git a/integrations/tensorflow/e2e/math/BUILD b/integrations/tensorflow/e2e/math/BUILD
index c7b19b6..9d76bfd 100644
--- a/integrations/tensorflow/e2e/math/BUILD
+++ b/integrations/tensorflow/e2e/math/BUILD
@@ -224,6 +224,9 @@
"reduce_all",
"reduce_euclidean_norm",
"reduce_logsumexp",
+ "reduce_mean",
+ "reduce_std",
+ "reduce_variance",
"rint",
"segment_max",
"segment_mean",
@@ -242,7 +245,9 @@
"unsorted_segment_prod",
"unsorted_segment_sqrt_n",
"unsorted_segment_sum",
+ "xdivy",
"xlog1py",
+ "xlogy",
"zeta",
]
@@ -406,6 +411,7 @@
"igammac",
"in_top_k",
"invert_permutation",
+ "is_finite",
"is_non_decreasing",
"is_strictly_increasing",
"l2_normalize",
@@ -645,7 +651,6 @@
"greater",
"greater_equal",
"imag",
- "is_finite",
"is_inf",
"is_nan",
"lbeta",
diff --git a/integrations/tensorflow/e2e/math/CMakeLists.txt b/integrations/tensorflow/e2e/math/CMakeLists.txt
new file mode 100644
index 0000000..33afdbb
--- /dev/null
+++ b/integrations/tensorflow/e2e/math/CMakeLists.txt
@@ -0,0 +1,1016 @@
+################################################################################
+# Autogenerated by build_tools/bazel_to_cmake/bazel_to_cmake.py from #
+# integrations/tensorflow/e2e/math/BUILD #
+# #
+# Use iree_cmake_extra_content from iree/build_defs.oss.bzl to add arbitrary #
+# CMake-only content. #
+# #
+# To disable autogeneration for this file entirely, delete this header. #
+################################################################################
+
+iree_add_all_subdirs()
+
+iree_e2e_cartesian_product_test_suite(
+ NAME
+ math_tests
+ MATRIX_KEYS
+ "src"
+ "reference_backend"
+ "functions"
+ "dynamic_dims"
+ "test_complex"
+ "target_backends"
+ MATRIX_VALUES
+ "math_test.py"
+ "tf"
+ "abs;accumulate_n;acos;acosh;add;add_n;angle;argmax;argmin;asin;asinh;atan;atan2;atanh;bessel_i0;bessel_i0e;bessel_i1;bessel_i1e;betainc;bincount;ceil;confusion_matrix;cos;cosh;count_nonzero;cumprod;cumsum;cumulative_logsumexp;digamma;divide;divide_no_nan;equal;erf;erfc;erfinv;exp;expm1;floor;floordiv;floormod;greater;greater_equal;igamma;igammac;imag;in_top_k;invert_permutation;is_finite;is_inf;is_nan;is_non_decreasing;is_strictly_increasing;lbeta;less;less_equal;lgamma;log;log1p;log_sigmoid;log_softmax;logical_and;logical_not;logical_or;logical_xor;maximum;minimum;mod;multiply;multiply_no_nan;ndtri;negative;nextafter;not_equal;polygamma;polyval;pow;real;reciprocal;reciprocal_no_nan;reduce_all;reduce_any;reduce_euclidean_norm;reduce_logsumexp;reduce_max;reduce_mean;reduce_min;reduce_prod;reduce_std;reduce_sum;reduce_variance;rint;round;rsqrt;scalar_mul;segment_max;segment_mean;segment_min;segment_prod;segment_sum;sigmoid;sign;sin;sinh;sobol_sample;softmax;softplus;softsign;sqrt;square;squared_difference;subtract;tan;tanh;truediv;unsorted_segment_max;unsorted_segment_mean;unsorted_segment_min;unsorted_segment_prod;unsorted_segment_sqrt_n;unsorted_segment_sum;xdivy;xlog1py;xlogy;zero_fraction;zeta"
+ "False"
+ "False"
+ "tf;tflite;iree_vmla;iree_llvmaot;iree_vulkan"
+ FAILING_CONFIGURATIONS
+ ",,abs,,,tflite"
+ ",,acos,,,tflite"
+ ",,acosh,,,tflite"
+ ",,asin,,,tflite"
+ ",,asinh,,,tflite"
+ ",,atan,,,tflite"
+ ",,atan2,,,tflite"
+ ",,atanh,,,tflite"
+ ",,bessel_i0,,,tflite"
+ ",,bessel_i0e,,,tflite"
+ ",,bessel_i1,,,tflite"
+ ",,bessel_i1e,,,tflite"
+ ",,betainc,,,tflite"
+ ",,bincount,,,tflite"
+ ",,confusion_matrix,,,tflite"
+ ",,conj,,,tflite"
+ ",,cosh,,,tflite"
+ ",,cumprod,,,tflite"
+ ",,cumulative_logsumexp,,,tflite"
+ ",,digamma,,,tflite"
+ ",,divide,,,tflite"
+ ",,erf,,,tflite"
+ ",,erfc,,,tflite"
+ ",,erfinv,,,tflite"
+ ",,expm1,,,tflite"
+ ",,igamma,,,tflite"
+ ",,igammac,,,tflite"
+ ",,in_top_k,,,tflite"
+ ",,invert_permutation,,,tflite"
+ ",,is_finite,,,tflite"
+ ",,is_non_decreasing,,,tflite"
+ ",,is_strictly_increasing,,,tflite"
+ ",,l2_normalize,,,tflite"
+ ",,lbeta,,,tflite"
+ ",,lgamma,,,tflite"
+ ",,log1p,,,tflite"
+ ",,log_sigmoid,,,tflite"
+ ",,ndtri,,,tflite"
+ ",,nextafter,,,tflite"
+ ",,polygamma,,,tflite"
+ ",,polyval,,,tflite"
+ ",,pow,,,tflite"
+ ",,reduce_all,,,tflite"
+ ",,reduce_euclidean_norm,,,tflite"
+ ",,reduce_logsumexp,,,tflite"
+ ",,reduce_mean,,,tflite"
+ ",,reduce_std,,,tflite"
+ ",,reduce_variance,,,tflite"
+ ",,rint,,,tflite"
+ ",,segment_max,,,tflite"
+ ",,segment_mean,,,tflite"
+ ",,segment_min,,,tflite"
+ ",,segment_prod,,,tflite"
+ ",,sign,,,tflite"
+ ",,sinh,,,tflite"
+ ",,sobol_sample,,,tflite"
+ ",,softmax,,,tflite"
+ ",,softplus,,,tflite"
+ ",,softsign,,,tflite"
+ ",,tan,,,tflite"
+ ",,unsorted_segment_max,,,tflite"
+ ",,unsorted_segment_mean,,,tflite"
+ ",,unsorted_segment_min,,,tflite"
+ ",,unsorted_segment_prod,,,tflite"
+ ",,unsorted_segment_sqrt_n,,,tflite"
+ ",,unsorted_segment_sum,,,tflite"
+ ",,xdivy,,,tflite"
+ ",,xlog1py,,,tflite"
+ ",,xlogy,,,tflite"
+ ",,zeta,,,tflite"
+ ",,acosh,,,iree_vmla"
+ ",,argmax,,,iree_vmla"
+ ",,argmin,,,iree_vmla"
+ ",,asin,,,iree_vmla"
+ ",,asinh,,,iree_vmla"
+ ",,atan2,,,iree_vmla"
+ ",,atanh,,,iree_vmla"
+ ",,bessel_i0,,,iree_vmla"
+ ",,bessel_i0e,,,iree_vmla"
+ ",,bessel_i1,,,iree_vmla"
+ ",,bessel_i1e,,,iree_vmla"
+ ",,betainc,,,iree_vmla"
+ ",,bincount,,,iree_vmla"
+ ",,confusion_matrix,,,iree_vmla"
+ ",,cosh,,,iree_vmla"
+ ",,count_nonzero,,,iree_vmla"
+ ",,cumprod,,,iree_vmla"
+ ",,cumulative_logsumexp,,,iree_vmla"
+ ",,digamma,,,iree_vmla"
+ ",,divide,,,iree_vmla"
+ ",,erf,,,iree_vmla"
+ ",,erfc,,,iree_vmla"
+ ",,erfinv,,,iree_vmla"
+ ",,expm1,,,iree_vmla"
+ ",,igamma,,,iree_vmla"
+ ",,igammac,,,iree_vmla"
+ ",,in_top_k,,,iree_vmla"
+ ",,ndtri,,,iree_vmla"
+ ",,nextafter,,,iree_vmla"
+ ",,polygamma,,,iree_vmla"
+ ",,pow,,,iree_vmla"
+ ",,reduce_euclidean_norm,,,iree_vmla"
+ ",,reduce_prod,,,iree_vmla"
+ ",,rint,,,iree_vmla"
+ ",,segment_max,,,iree_vmla"
+ ",,segment_mean,,,iree_vmla"
+ ",,segment_min,,,iree_vmla"
+ ",,segment_prod,,,iree_vmla"
+ ",,segment_sum,,,iree_vmla"
+ ",,sign,,,iree_vmla"
+ ",,sobol_sample,,,iree_vmla"
+ ",,softsign,,,iree_vmla"
+ ",,unsorted_segment_max,,,iree_vmla"
+ ",,unsorted_segment_mean,,,iree_vmla"
+ ",,unsorted_segment_min,,,iree_vmla"
+ ",,unsorted_segment_prod,,,iree_vmla"
+ ",,unsorted_segment_sqrt_n,,,iree_vmla"
+ ",,unsorted_segment_sum,,,iree_vmla"
+ ",,zeta,,,iree_vmla"
+ ",,acos,,,iree_llvmaot"
+ ",,acosh,,,iree_llvmaot"
+ ",,argmax,,,iree_llvmaot"
+ ",,argmin,,,iree_llvmaot"
+ ",,asin,,,iree_llvmaot"
+ ",,asinh,,,iree_llvmaot"
+ ",,atan,,,iree_llvmaot"
+ ",,atan2,,,iree_llvmaot"
+ ",,atanh,,,iree_llvmaot"
+ ",,bessel_i0,,,iree_llvmaot"
+ ",,bessel_i0e,,,iree_llvmaot"
+ ",,bessel_i1,,,iree_llvmaot"
+ ",,bessel_i1e,,,iree_llvmaot"
+ ",,betainc,,,iree_llvmaot"
+ ",,bincount,,,iree_llvmaot"
+ ",,confusion_matrix,,,iree_llvmaot"
+ ",,cosh,,,iree_llvmaot"
+ ",,count_nonzero,,,iree_llvmaot"
+ ",,cumprod,,,iree_llvmaot"
+ ",,cumsum,,,iree_llvmaot"
+ ",,cumulative_logsumexp,,,iree_llvmaot"
+ ",,digamma,,,iree_llvmaot"
+ ",,divide,,,iree_llvmaot"
+ ",,erf,,,iree_llvmaot"
+ ",,erfc,,,iree_llvmaot"
+ ",,erfinv,,,iree_llvmaot"
+ ",,expm1,,,iree_llvmaot"
+ ",,igamma,,,iree_llvmaot"
+ ",,igammac,,,iree_llvmaot"
+ ",,in_top_k,,,iree_llvmaot"
+ ",,invert_permutation,,,iree_llvmaot"
+ ",,is_non_decreasing,,,iree_llvmaot"
+ ",,is_strictly_increasing,,,iree_llvmaot"
+ ",,l2_normalize,,,iree_llvmaot"
+ ",,ndtri,,,iree_llvmaot"
+ ",,nextafter,,,iree_llvmaot"
+ ",,polygamma,,,iree_llvmaot"
+ ",,reduce_all,,,iree_llvmaot"
+ ",,reduce_any,,,iree_llvmaot"
+ ",,reduce_euclidean_norm,,,iree_llvmaot"
+ ",,reduce_logsumexp,,,iree_llvmaot"
+ ",,reduce_max,,,iree_llvmaot"
+ ",,reduce_mean,,,iree_llvmaot"
+ ",,reduce_min,,,iree_llvmaot"
+ ",,reduce_prod,,,iree_llvmaot"
+ ",,reduce_std,,,iree_llvmaot"
+ ",,reduce_sum,,,iree_llvmaot"
+ ",,reduce_variance,,,iree_llvmaot"
+ ",,rint,,,iree_llvmaot"
+ ",,segment_max,,,iree_llvmaot"
+ ",,segment_mean,,,iree_llvmaot"
+ ",,segment_min,,,iree_llvmaot"
+ ",,segment_prod,,,iree_llvmaot"
+ ",,segment_sum,,,iree_llvmaot"
+ ",,sobol_sample,,,iree_llvmaot"
+ ",,softsign,,,iree_llvmaot"
+ ",,unsorted_segment_max,,,iree_llvmaot"
+ ",,unsorted_segment_mean,,,iree_llvmaot"
+ ",,unsorted_segment_min,,,iree_llvmaot"
+ ",,unsorted_segment_prod,,,iree_llvmaot"
+ ",,unsorted_segment_sqrt_n,,,iree_llvmaot"
+ ",,unsorted_segment_sum,,,iree_llvmaot"
+ ",,zeta,,,iree_llvmaot"
+ ",,acos,,,iree_vulkan"
+ ",,acosh,,,iree_vulkan"
+ ",,argmax,,,iree_vulkan"
+ ",,argmin,,,iree_vulkan"
+ ",,asin,,,iree_vulkan"
+ ",,asinh,,,iree_vulkan"
+ ",,atan,,,iree_vulkan"
+ ",,atan2,,,iree_vulkan"
+ ",,atanh,,,iree_vulkan"
+ ",,bessel_i0,,,iree_vulkan"
+ ",,bessel_i0e,,,iree_vulkan"
+ ",,bessel_i1,,,iree_vulkan"
+ ",,bessel_i1e,,,iree_vulkan"
+ ",,betainc,,,iree_vulkan"
+ ",,bincount,,,iree_vulkan"
+ ",,confusion_matrix,,,iree_vulkan"
+ ",,cosh,,,iree_vulkan"
+ ",,count_nonzero,,,iree_vulkan"
+ ",,cumprod,,,iree_vulkan"
+ ",,cumsum,,,iree_vulkan"
+ ",,cumulative_logsumexp,,,iree_vulkan"
+ ",,digamma,,,iree_vulkan"
+ ",,divide,,,iree_vulkan"
+ ",,erf,,,iree_vulkan"
+ ",,erfc,,,iree_vulkan"
+ ",,erfinv,,,iree_vulkan"
+ ",,expm1,,,iree_vulkan"
+ ",,igamma,,,iree_vulkan"
+ ",,igammac,,,iree_vulkan"
+ ",,in_top_k,,,iree_vulkan"
+ ",,invert_permutation,,,iree_vulkan"
+ ",,is_finite,,,iree_vulkan"
+ ",,is_non_decreasing,,,iree_vulkan"
+ ",,is_strictly_increasing,,,iree_vulkan"
+ ",,l2_normalize,,,iree_vulkan"
+ ",,logical_and,,,iree_vulkan"
+ ",,logical_not,,,iree_vulkan"
+ ",,logical_or,,,iree_vulkan"
+ ",,logical_xor,,,iree_vulkan"
+ ",,mod,,,iree_vulkan"
+ ",,ndtri,,,iree_vulkan"
+ ",,nextafter,,,iree_vulkan"
+ ",,polygamma,,,iree_vulkan"
+ ",,pow,,,iree_vulkan"
+ ",,reduce_all,,,iree_vulkan"
+ ",,reduce_any,,,iree_vulkan"
+ ",,reduce_euclidean_norm,,,iree_vulkan"
+ ",,reduce_logsumexp,,,iree_vulkan"
+ ",,reduce_max,,,iree_vulkan"
+ ",,reduce_mean,,,iree_vulkan"
+ ",,reduce_min,,,iree_vulkan"
+ ",,reduce_prod,,,iree_vulkan"
+ ",,reduce_std,,,iree_vulkan"
+ ",,reduce_sum,,,iree_vulkan"
+ ",,reduce_variance,,,iree_vulkan"
+ ",,rint,,,iree_vulkan"
+ ",,segment_max,,,iree_vulkan"
+ ",,segment_mean,,,iree_vulkan"
+ ",,segment_min,,,iree_vulkan"
+ ",,segment_prod,,,iree_vulkan"
+ ",,segment_sum,,,iree_vulkan"
+ ",,sign,,,iree_vulkan"
+ ",,sobol_sample,,,iree_vulkan"
+ ",,softsign,,,iree_vulkan"
+ ",,unsorted_segment_max,,,iree_vulkan"
+ ",,unsorted_segment_mean,,,iree_vulkan"
+ ",,unsorted_segment_min,,,iree_vulkan"
+ ",,unsorted_segment_prod,,,iree_vulkan"
+ ",,unsorted_segment_sqrt_n,,,iree_vulkan"
+ ",,unsorted_segment_sum,,,iree_vulkan"
+ ",,zeta,,,iree_vulkan"
+)
+
+iree_e2e_cartesian_product_test_suite(
+ NAME
+ math_dynamic_dims_tests
+ MATRIX_KEYS
+ "src"
+ "reference_backend"
+ "functions"
+ "dynamic_dims"
+ "test_complex"
+ "target_backends"
+ MATRIX_VALUES
+ "math_test.py"
+ "tf"
+ "abs;accumulate_n;acos;acosh;add;add_n;angle;argmax;argmin;asin;asinh;atan;atan2;atanh;bessel_i0;bessel_i0e;bessel_i1;bessel_i1e;betainc;bincount;ceil;confusion_matrix;cos;cosh;count_nonzero;cumprod;cumsum;cumulative_logsumexp;digamma;divide;divide_no_nan;equal;erf;erfc;erfinv;exp;expm1;floor;floordiv;floormod;greater;greater_equal;igamma;igammac;imag;in_top_k;invert_permutation;is_finite;is_inf;is_nan;is_non_decreasing;is_strictly_increasing;lbeta;less;less_equal;lgamma;log;log1p;log_sigmoid;log_softmax;logical_and;logical_not;logical_or;logical_xor;maximum;minimum;mod;multiply;multiply_no_nan;ndtri;negative;nextafter;not_equal;polygamma;polyval;pow;real;reciprocal;reciprocal_no_nan;reduce_all;reduce_any;reduce_euclidean_norm;reduce_logsumexp;reduce_max;reduce_mean;reduce_min;reduce_prod;reduce_std;reduce_sum;reduce_variance;rint;round;rsqrt;scalar_mul;segment_max;segment_mean;segment_min;segment_prod;segment_sum;sigmoid;sign;sin;sinh;sobol_sample;softmax;softplus;softsign;sqrt;square;squared_difference;subtract;tan;tanh;truediv;unsorted_segment_max;unsorted_segment_mean;unsorted_segment_min;unsorted_segment_prod;unsorted_segment_sqrt_n;unsorted_segment_sum;xdivy;xlog1py;xlogy;zero_fraction;zeta"
+ "True"
+ "False"
+ "tf;tflite;iree_vmla;iree_llvmaot;iree_vulkan"
+ FAILING_CONFIGURATIONS
+ ",,abs,,,tflite"
+ ",,acos,,,tflite"
+ ",,acosh,,,tflite"
+ ",,asin,,,tflite"
+ ",,asinh,,,tflite"
+ ",,atan,,,tflite"
+ ",,atan2,,,tflite"
+ ",,atanh,,,tflite"
+ ",,bessel_i0,,,tflite"
+ ",,bessel_i0e,,,tflite"
+ ",,bessel_i1,,,tflite"
+ ",,bessel_i1e,,,tflite"
+ ",,betainc,,,tflite"
+ ",,bincount,,,tflite"
+ ",,confusion_matrix,,,tflite"
+ ",,conj,,,tflite"
+ ",,cosh,,,tflite"
+ ",,cumprod,,,tflite"
+ ",,cumulative_logsumexp,,,tflite"
+ ",,digamma,,,tflite"
+ ",,divide,,,tflite"
+ ",,erf,,,tflite"
+ ",,erfc,,,tflite"
+ ",,erfinv,,,tflite"
+ ",,expm1,,,tflite"
+ ",,igamma,,,tflite"
+ ",,igammac,,,tflite"
+ ",,in_top_k,,,tflite"
+ ",,invert_permutation,,,tflite"
+ ",,is_finite,,,tflite"
+ ",,is_non_decreasing,,,tflite"
+ ",,is_strictly_increasing,,,tflite"
+ ",,l2_normalize,,,tflite"
+ ",,lbeta,,,tflite"
+ ",,lgamma,,,tflite"
+ ",,log1p,,,tflite"
+ ",,log_sigmoid,,,tflite"
+ ",,ndtri,,,tflite"
+ ",,nextafter,,,tflite"
+ ",,polygamma,,,tflite"
+ ",,polyval,,,tflite"
+ ",,pow,,,tflite"
+ ",,reduce_all,,,tflite"
+ ",,reduce_euclidean_norm,,,tflite"
+ ",,reduce_logsumexp,,,tflite"
+ ",,reduce_mean,,,tflite"
+ ",,reduce_std,,,tflite"
+ ",,reduce_variance,,,tflite"
+ ",,rint,,,tflite"
+ ",,segment_max,,,tflite"
+ ",,segment_mean,,,tflite"
+ ",,segment_min,,,tflite"
+ ",,segment_prod,,,tflite"
+ ",,sign,,,tflite"
+ ",,sinh,,,tflite"
+ ",,sobol_sample,,,tflite"
+ ",,softmax,,,tflite"
+ ",,softplus,,,tflite"
+ ",,softsign,,,tflite"
+ ",,tan,,,tflite"
+ ",,unsorted_segment_max,,,tflite"
+ ",,unsorted_segment_mean,,,tflite"
+ ",,unsorted_segment_min,,,tflite"
+ ",,unsorted_segment_prod,,,tflite"
+ ",,unsorted_segment_sqrt_n,,,tflite"
+ ",,unsorted_segment_sum,,,tflite"
+ ",,xdivy,,,tflite"
+ ",,xlog1py,,,tflite"
+ ",,xlogy,,,tflite"
+ ",,zeta,,,tflite"
+ ",,add,,,tflite"
+ ",,angle,,,tflite"
+ ",,count_nonzero,,,tflite"
+ ",,divide_no_nan,,,tflite"
+ ",,equal,,,tflite"
+ ",,floordiv,,,tflite"
+ ",,floormod,,,tflite"
+ ",,greater,,,tflite"
+ ",,greater_equal,,,tflite"
+ ",,less,,,tflite"
+ ",,less_equal,,,tflite"
+ ",,maximum,,,tflite"
+ ",,minimum,,,tflite"
+ ",,mod,,,tflite"
+ ",,multiply,,,tflite"
+ ",,multiply_no_nan,,,tflite"
+ ",,not_equal,,,tflite"
+ ",,reciprocal,,,tflite"
+ ",,reciprocal_no_nan,,,tflite"
+ ",,reduce_std,,,tflite"
+ ",,reduce_variance,,,tflite"
+ ",,square,,,tflite"
+ ",,squared_difference,,,tflite"
+ ",,subtract,,,tflite"
+ ",,truediv,,,tflite"
+ ",,xdivy,,,tflite"
+ ",,xlog1py,,,tflite"
+ ",,xlogy,,,tflite"
+ ",,zero_fraction,,,tflite"
+ ",,acosh,,,iree_vmla"
+ ",,argmax,,,iree_vmla"
+ ",,argmin,,,iree_vmla"
+ ",,asin,,,iree_vmla"
+ ",,asinh,,,iree_vmla"
+ ",,atan2,,,iree_vmla"
+ ",,atanh,,,iree_vmla"
+ ",,bessel_i0,,,iree_vmla"
+ ",,bessel_i0e,,,iree_vmla"
+ ",,bessel_i1,,,iree_vmla"
+ ",,bessel_i1e,,,iree_vmla"
+ ",,betainc,,,iree_vmla"
+ ",,bincount,,,iree_vmla"
+ ",,confusion_matrix,,,iree_vmla"
+ ",,cosh,,,iree_vmla"
+ ",,count_nonzero,,,iree_vmla"
+ ",,cumprod,,,iree_vmla"
+ ",,cumulative_logsumexp,,,iree_vmla"
+ ",,digamma,,,iree_vmla"
+ ",,divide,,,iree_vmla"
+ ",,erf,,,iree_vmla"
+ ",,erfc,,,iree_vmla"
+ ",,erfinv,,,iree_vmla"
+ ",,expm1,,,iree_vmla"
+ ",,igamma,,,iree_vmla"
+ ",,igammac,,,iree_vmla"
+ ",,in_top_k,,,iree_vmla"
+ ",,ndtri,,,iree_vmla"
+ ",,nextafter,,,iree_vmla"
+ ",,polygamma,,,iree_vmla"
+ ",,pow,,,iree_vmla"
+ ",,reduce_euclidean_norm,,,iree_vmla"
+ ",,reduce_prod,,,iree_vmla"
+ ",,rint,,,iree_vmla"
+ ",,segment_max,,,iree_vmla"
+ ",,segment_mean,,,iree_vmla"
+ ",,segment_min,,,iree_vmla"
+ ",,segment_prod,,,iree_vmla"
+ ",,segment_sum,,,iree_vmla"
+ ",,sign,,,iree_vmla"
+ ",,sobol_sample,,,iree_vmla"
+ ",,softsign,,,iree_vmla"
+ ",,unsorted_segment_max,,,iree_vmla"
+ ",,unsorted_segment_mean,,,iree_vmla"
+ ",,unsorted_segment_min,,,iree_vmla"
+ ",,unsorted_segment_prod,,,iree_vmla"
+ ",,unsorted_segment_sqrt_n,,,iree_vmla"
+ ",,unsorted_segment_sum,,,iree_vmla"
+ ",,zeta,,,iree_vmla"
+ ",,acos,,,iree_vmla"
+ ",,angle,,,iree_vmla"
+ ",,cumsum,,,iree_vmla"
+ ",,divide_no_nan,,,iree_vmla"
+ ",,floordiv,,,iree_vmla"
+ ",,floormod,,,iree_vmla"
+ ",,invert_permutation,,,iree_vmla"
+ ",,is_non_decreasing,,,iree_vmla"
+ ",,is_strictly_increasing,,,iree_vmla"
+ ",,lbeta,,,iree_vmla"
+ ",,lgamma,,,iree_vmla"
+ ",,log1p,,,iree_vmla"
+ ",,log_sigmoid,,,iree_vmla"
+ ",,logical_and,,,iree_vmla"
+ ",,logical_not,,,iree_vmla"
+ ",,logical_or,,,iree_vmla"
+ ",,logical_xor,,,iree_vmla"
+ ",,mod,,,iree_vmla"
+ ",,multiply_no_nan,,,iree_vmla"
+ ",,reciprocal_no_nan,,,iree_vmla"
+ ",,reduce_all,,,iree_vmla"
+ ",,reduce_any,,,iree_vmla"
+ ",,reduce_logsumexp,,,iree_vmla"
+ ",,reduce_max,,,iree_vmla"
+ ",,reduce_mean,,,iree_vmla"
+ ",,reduce_min,,,iree_vmla"
+ ",,reduce_std,,,iree_vmla"
+ ",,reduce_sum,,,iree_vmla"
+ ",,reduce_variance,,,iree_vmla"
+ ",,round,,,iree_vmla"
+ ",,sinh,,,iree_vmla"
+ ",,softplus,,,iree_vmla"
+ ",,xdivy,,,iree_vmla"
+ ",,xlog1py,,,iree_vmla"
+ ",,xlogy,,,iree_vmla"
+ ",,zero_fraction,,,iree_vmla"
+ ",,acos,,,iree_llvmaot"
+ ",,acosh,,,iree_llvmaot"
+ ",,argmax,,,iree_llvmaot"
+ ",,argmin,,,iree_llvmaot"
+ ",,asin,,,iree_llvmaot"
+ ",,asinh,,,iree_llvmaot"
+ ",,atan,,,iree_llvmaot"
+ ",,atan2,,,iree_llvmaot"
+ ",,atanh,,,iree_llvmaot"
+ ",,bessel_i0,,,iree_llvmaot"
+ ",,bessel_i0e,,,iree_llvmaot"
+ ",,bessel_i1,,,iree_llvmaot"
+ ",,bessel_i1e,,,iree_llvmaot"
+ ",,betainc,,,iree_llvmaot"
+ ",,bincount,,,iree_llvmaot"
+ ",,confusion_matrix,,,iree_llvmaot"
+ ",,cosh,,,iree_llvmaot"
+ ",,count_nonzero,,,iree_llvmaot"
+ ",,cumprod,,,iree_llvmaot"
+ ",,cumsum,,,iree_llvmaot"
+ ",,cumulative_logsumexp,,,iree_llvmaot"
+ ",,digamma,,,iree_llvmaot"
+ ",,divide,,,iree_llvmaot"
+ ",,erf,,,iree_llvmaot"
+ ",,erfc,,,iree_llvmaot"
+ ",,erfinv,,,iree_llvmaot"
+ ",,expm1,,,iree_llvmaot"
+ ",,igamma,,,iree_llvmaot"
+ ",,igammac,,,iree_llvmaot"
+ ",,in_top_k,,,iree_llvmaot"
+ ",,invert_permutation,,,iree_llvmaot"
+ ",,is_non_decreasing,,,iree_llvmaot"
+ ",,is_strictly_increasing,,,iree_llvmaot"
+ ",,l2_normalize,,,iree_llvmaot"
+ ",,ndtri,,,iree_llvmaot"
+ ",,nextafter,,,iree_llvmaot"
+ ",,polygamma,,,iree_llvmaot"
+ ",,reduce_all,,,iree_llvmaot"
+ ",,reduce_any,,,iree_llvmaot"
+ ",,reduce_euclidean_norm,,,iree_llvmaot"
+ ",,reduce_logsumexp,,,iree_llvmaot"
+ ",,reduce_max,,,iree_llvmaot"
+ ",,reduce_mean,,,iree_llvmaot"
+ ",,reduce_min,,,iree_llvmaot"
+ ",,reduce_prod,,,iree_llvmaot"
+ ",,reduce_std,,,iree_llvmaot"
+ ",,reduce_sum,,,iree_llvmaot"
+ ",,reduce_variance,,,iree_llvmaot"
+ ",,rint,,,iree_llvmaot"
+ ",,segment_max,,,iree_llvmaot"
+ ",,segment_mean,,,iree_llvmaot"
+ ",,segment_min,,,iree_llvmaot"
+ ",,segment_prod,,,iree_llvmaot"
+ ",,segment_sum,,,iree_llvmaot"
+ ",,sobol_sample,,,iree_llvmaot"
+ ",,softsign,,,iree_llvmaot"
+ ",,unsorted_segment_max,,,iree_llvmaot"
+ ",,unsorted_segment_mean,,,iree_llvmaot"
+ ",,unsorted_segment_min,,,iree_llvmaot"
+ ",,unsorted_segment_prod,,,iree_llvmaot"
+ ",,unsorted_segment_sqrt_n,,,iree_llvmaot"
+ ",,unsorted_segment_sum,,,iree_llvmaot"
+ ",,zeta,,,iree_llvmaot"
+ ",,accumulate_n,,,iree_llvmaot"
+ ",,add,,,iree_llvmaot"
+ ",,add_n,,,iree_llvmaot"
+ ",,angle,,,iree_llvmaot"
+ ",,cumsum,,,iree_llvmaot"
+ ",,divide,,,iree_llvmaot"
+ ",,divide_no_nan,,,iree_llvmaot"
+ ",,equal,,,iree_llvmaot"
+ ",,floordiv,,,iree_llvmaot"
+ ",,floormod,,,iree_llvmaot"
+ ",,greater,,,iree_llvmaot"
+ ",,greater_equal,,,iree_llvmaot"
+ ",,imag,,,iree_llvmaot"
+ ",,is_finite,,,iree_llvmaot"
+ ",,is_inf,,,iree_llvmaot"
+ ",,is_nan,,,iree_llvmaot"
+ ",,lbeta,,,iree_llvmaot"
+ ",,less,,,iree_llvmaot"
+ ",,less_equal,,,iree_llvmaot"
+ ",,lgamma,,,iree_llvmaot"
+ ",,log1p,,,iree_llvmaot"
+ ",,log_sigmoid,,,iree_llvmaot"
+ ",,log_softmax,,,iree_llvmaot"
+ ",,logical_and,,,iree_llvmaot"
+ ",,logical_not,,,iree_llvmaot"
+ ",,logical_or,,,iree_llvmaot"
+ ",,logical_xor,,,iree_llvmaot"
+ ",,maximum,,,iree_llvmaot"
+ ",,minimum,,,iree_llvmaot"
+ ",,mod,,,iree_llvmaot"
+ ",,multiply,,,iree_llvmaot"
+ ",,multiply_no_nan,,,iree_llvmaot"
+ ",,not_equal,,,iree_llvmaot"
+ ",,polyval,,,iree_llvmaot"
+ ",,pow,,,iree_llvmaot"
+ ",,reciprocal,,,iree_llvmaot"
+ ",,reciprocal_no_nan,,,iree_llvmaot"
+ ",,reduce_mean,,,iree_llvmaot"
+ ",,round,,,iree_llvmaot"
+ ",,scalar_mul,,,iree_llvmaot"
+ ",,sigmoid,,,iree_llvmaot"
+ ",,sinh,,,iree_llvmaot"
+ ",,softmax,,,iree_llvmaot"
+ ",,softplus,,,iree_llvmaot"
+ ",,square,,,iree_llvmaot"
+ ",,squared_difference,,,iree_llvmaot"
+ ",,subtract,,,iree_llvmaot"
+ ",,tan,,,iree_llvmaot"
+ ",,truediv,,,iree_llvmaot"
+ ",,xdivy,,,iree_llvmaot"
+ ",,xlog1py,,,iree_llvmaot"
+ ",,xlogy,,,iree_llvmaot"
+ ",,zero_fraction,,,iree_llvmaot"
+ ",,acos,,,iree_vulkan"
+ ",,acosh,,,iree_vulkan"
+ ",,argmax,,,iree_vulkan"
+ ",,argmin,,,iree_vulkan"
+ ",,asin,,,iree_vulkan"
+ ",,asinh,,,iree_vulkan"
+ ",,atan,,,iree_vulkan"
+ ",,atan2,,,iree_vulkan"
+ ",,atanh,,,iree_vulkan"
+ ",,bessel_i0,,,iree_vulkan"
+ ",,bessel_i0e,,,iree_vulkan"
+ ",,bessel_i1,,,iree_vulkan"
+ ",,bessel_i1e,,,iree_vulkan"
+ ",,betainc,,,iree_vulkan"
+ ",,bincount,,,iree_vulkan"
+ ",,confusion_matrix,,,iree_vulkan"
+ ",,cosh,,,iree_vulkan"
+ ",,count_nonzero,,,iree_vulkan"
+ ",,cumprod,,,iree_vulkan"
+ ",,cumsum,,,iree_vulkan"
+ ",,cumulative_logsumexp,,,iree_vulkan"
+ ",,digamma,,,iree_vulkan"
+ ",,divide,,,iree_vulkan"
+ ",,erf,,,iree_vulkan"
+ ",,erfc,,,iree_vulkan"
+ ",,erfinv,,,iree_vulkan"
+ ",,expm1,,,iree_vulkan"
+ ",,igamma,,,iree_vulkan"
+ ",,igammac,,,iree_vulkan"
+ ",,in_top_k,,,iree_vulkan"
+ ",,invert_permutation,,,iree_vulkan"
+ ",,is_finite,,,iree_vulkan"
+ ",,is_non_decreasing,,,iree_vulkan"
+ ",,is_strictly_increasing,,,iree_vulkan"
+ ",,l2_normalize,,,iree_vulkan"
+ ",,logical_and,,,iree_vulkan"
+ ",,logical_not,,,iree_vulkan"
+ ",,logical_or,,,iree_vulkan"
+ ",,logical_xor,,,iree_vulkan"
+ ",,mod,,,iree_vulkan"
+ ",,ndtri,,,iree_vulkan"
+ ",,nextafter,,,iree_vulkan"
+ ",,polygamma,,,iree_vulkan"
+ ",,pow,,,iree_vulkan"
+ ",,reduce_all,,,iree_vulkan"
+ ",,reduce_any,,,iree_vulkan"
+ ",,reduce_euclidean_norm,,,iree_vulkan"
+ ",,reduce_logsumexp,,,iree_vulkan"
+ ",,reduce_max,,,iree_vulkan"
+ ",,reduce_mean,,,iree_vulkan"
+ ",,reduce_min,,,iree_vulkan"
+ ",,reduce_prod,,,iree_vulkan"
+ ",,reduce_std,,,iree_vulkan"
+ ",,reduce_sum,,,iree_vulkan"
+ ",,reduce_variance,,,iree_vulkan"
+ ",,rint,,,iree_vulkan"
+ ",,segment_max,,,iree_vulkan"
+ ",,segment_mean,,,iree_vulkan"
+ ",,segment_min,,,iree_vulkan"
+ ",,segment_prod,,,iree_vulkan"
+ ",,segment_sum,,,iree_vulkan"
+ ",,sign,,,iree_vulkan"
+ ",,sobol_sample,,,iree_vulkan"
+ ",,softsign,,,iree_vulkan"
+ ",,unsorted_segment_max,,,iree_vulkan"
+ ",,unsorted_segment_mean,,,iree_vulkan"
+ ",,unsorted_segment_min,,,iree_vulkan"
+ ",,unsorted_segment_prod,,,iree_vulkan"
+ ",,unsorted_segment_sqrt_n,,,iree_vulkan"
+ ",,unsorted_segment_sum,,,iree_vulkan"
+ ",,zeta,,,iree_vulkan"
+ ",,abs,,,iree_vulkan"
+ ",,accumulate_n,,,iree_vulkan"
+ ",,add,,,iree_vulkan"
+ ",,add_n,,,iree_vulkan"
+ ",,angle,,,iree_vulkan"
+ ",,ceil,,,iree_vulkan"
+ ",,cos,,,iree_vulkan"
+ ",,divide,,,iree_vulkan"
+ ",,divide_no_nan,,,iree_vulkan"
+ ",,equal,,,iree_vulkan"
+ ",,exp,,,iree_vulkan"
+ ",,floor,,,iree_vulkan"
+ ",,floordiv,,,iree_vulkan"
+ ",,floormod,,,iree_vulkan"
+ ",,greater,,,iree_vulkan"
+ ",,greater_equal,,,iree_vulkan"
+ ",,imag,,,iree_vulkan"
+ ",,is_inf,,,iree_vulkan"
+ ",,is_nan,,,iree_vulkan"
+ ",,lbeta,,,iree_vulkan"
+ ",,less,,,iree_vulkan"
+ ",,less_equal,,,iree_vulkan"
+ ",,lgamma,,,iree_vulkan"
+ ",,log,,,iree_vulkan"
+ ",,log1p,,,iree_vulkan"
+ ",,log_sigmoid,,,iree_vulkan"
+ ",,log_softmax,,,iree_vulkan"
+ ",,maximum,,,iree_vulkan"
+ ",,minimum,,,iree_vulkan"
+ ",,mod,,,iree_vulkan"
+ ",,multiply,,,iree_vulkan"
+ ",,multiply_no_nan,,,iree_vulkan"
+ ",,negative,,,iree_vulkan"
+ ",,not_equal,,,iree_vulkan"
+ ",,polyval,,,iree_vulkan"
+ ",,reciprocal,,,iree_vulkan"
+ ",,reciprocal_no_nan,,,iree_vulkan"
+ ",,reduce_max,,,iree_vulkan"
+ ",,reduce_mean,,,iree_vulkan"
+ ",,reduce_min,,,iree_vulkan"
+ ",,reduce_sum,,,iree_vulkan"
+ ",,round,,,iree_vulkan"
+ ",,rsqrt,,,iree_vulkan"
+ ",,scalar_mul,,,iree_vulkan"
+ ",,sigmoid,,,iree_vulkan"
+ ",,sin,,,iree_vulkan"
+ ",,sinh,,,iree_vulkan"
+ ",,softmax,,,iree_vulkan"
+ ",,softplus,,,iree_vulkan"
+ ",,sqrt,,,iree_vulkan"
+ ",,square,,,iree_vulkan"
+ ",,squared_difference,,,iree_vulkan"
+ ",,subtract,,,iree_vulkan"
+ ",,tan,,,iree_vulkan"
+ ",,tanh,,,iree_vulkan"
+ ",,truediv,,,iree_vulkan"
+ ",,xdivy,,,iree_vulkan"
+ ",,xlog1py,,,iree_vulkan"
+ ",,xlogy,,,iree_vulkan"
+ ",,zero_fraction,,,iree_vulkan"
+)
+
+iree_e2e_cartesian_product_test_suite(
+ NAME
+ math_complex_tests
+ MATRIX_KEYS
+ "src"
+ "reference_backend"
+ "functions"
+ "dynamic_dims"
+ "test_complex"
+ "target_backends"
+ MATRIX_VALUES
+ "math_test.py"
+ "tf"
+ "abs;add;angle;asinh;atanh;conj;cos;cosh;count_nonzero;cumprod;cumsum;divide;divide_no_nan;exp;expm1;imag;l2_normalize;log;log1p;multiply;multiply_no_nan;negative;pow;real;reciprocal;reciprocal_no_nan;reduce_euclidean_norm;reduce_std;reduce_variance;rsqrt;sigmoid;sign;sin;sinh;sqrt;square;squared_difference;subtract;tan;tanh;truediv;xdivy;xlog1py;xlogy;zero_fraction"
+ "False"
+ "True"
+ "tf;tflite;iree_vmla;iree_llvmaot;iree_vulkan"
+ FAILING_CONFIGURATIONS
+ ",,,,,tflite"
+ ",,acosh,,,iree_vmla"
+ ",,argmax,,,iree_vmla"
+ ",,argmin,,,iree_vmla"
+ ",,asin,,,iree_vmla"
+ ",,asinh,,,iree_vmla"
+ ",,atan2,,,iree_vmla"
+ ",,atanh,,,iree_vmla"
+ ",,bessel_i0,,,iree_vmla"
+ ",,bessel_i0e,,,iree_vmla"
+ ",,bessel_i1,,,iree_vmla"
+ ",,bessel_i1e,,,iree_vmla"
+ ",,betainc,,,iree_vmla"
+ ",,bincount,,,iree_vmla"
+ ",,confusion_matrix,,,iree_vmla"
+ ",,cosh,,,iree_vmla"
+ ",,count_nonzero,,,iree_vmla"
+ ",,cumprod,,,iree_vmla"
+ ",,cumulative_logsumexp,,,iree_vmla"
+ ",,digamma,,,iree_vmla"
+ ",,divide,,,iree_vmla"
+ ",,erf,,,iree_vmla"
+ ",,erfc,,,iree_vmla"
+ ",,erfinv,,,iree_vmla"
+ ",,expm1,,,iree_vmla"
+ ",,igamma,,,iree_vmla"
+ ",,igammac,,,iree_vmla"
+ ",,in_top_k,,,iree_vmla"
+ ",,ndtri,,,iree_vmla"
+ ",,nextafter,,,iree_vmla"
+ ",,polygamma,,,iree_vmla"
+ ",,pow,,,iree_vmla"
+ ",,reduce_euclidean_norm,,,iree_vmla"
+ ",,reduce_prod,,,iree_vmla"
+ ",,rint,,,iree_vmla"
+ ",,segment_max,,,iree_vmla"
+ ",,segment_mean,,,iree_vmla"
+ ",,segment_min,,,iree_vmla"
+ ",,segment_prod,,,iree_vmla"
+ ",,segment_sum,,,iree_vmla"
+ ",,sign,,,iree_vmla"
+ ",,sobol_sample,,,iree_vmla"
+ ",,softsign,,,iree_vmla"
+ ",,unsorted_segment_max,,,iree_vmla"
+ ",,unsorted_segment_mean,,,iree_vmla"
+ ",,unsorted_segment_min,,,iree_vmla"
+ ",,unsorted_segment_prod,,,iree_vmla"
+ ",,unsorted_segment_sqrt_n,,,iree_vmla"
+ ",,unsorted_segment_sum,,,iree_vmla"
+ ",,zeta,,,iree_vmla"
+ ",,angle,,,iree_vmla"
+ ",,cos,,,iree_vmla"
+ ",,cumsum,,,iree_vmla"
+ ",,divide_no_nan,,,iree_vmla"
+ ",,log,,,iree_vmla"
+ ",,log1p,,,iree_vmla"
+ ",,multiply_no_nan,,,iree_vmla"
+ ",,negative,,,iree_vmla"
+ ",,pow,,,iree_vmla"
+ ",,reciprocal,,,iree_vmla"
+ ",,reciprocal_no_nan,,,iree_vmla"
+ ",,reduce_std,,,iree_vmla"
+ ",,reduce_variance,,,iree_vmla"
+ ",,rsqrt,,,iree_vmla"
+ ",,sigmoid,,,iree_vmla"
+ ",,sign,,,iree_vmla"
+ ",,sin,,,iree_vmla"
+ ",,sinh,,,iree_vmla"
+ ",,sqrt,,,iree_vmla"
+ ",,tan,,,iree_vmla"
+ ",,tanh,,,iree_vmla"
+ ",,xdivy,,,iree_vmla"
+ ",,xlog1py,,,iree_vmla"
+ ",,xlogy,,,iree_vmla"
+ ",,zero_fraction,,,iree_vmla"
+ ",,acos,,,iree_llvmaot"
+ ",,acosh,,,iree_llvmaot"
+ ",,argmax,,,iree_llvmaot"
+ ",,argmin,,,iree_llvmaot"
+ ",,asin,,,iree_llvmaot"
+ ",,asinh,,,iree_llvmaot"
+ ",,atan,,,iree_llvmaot"
+ ",,atan2,,,iree_llvmaot"
+ ",,atanh,,,iree_llvmaot"
+ ",,bessel_i0,,,iree_llvmaot"
+ ",,bessel_i0e,,,iree_llvmaot"
+ ",,bessel_i1,,,iree_llvmaot"
+ ",,bessel_i1e,,,iree_llvmaot"
+ ",,betainc,,,iree_llvmaot"
+ ",,bincount,,,iree_llvmaot"
+ ",,confusion_matrix,,,iree_llvmaot"
+ ",,cosh,,,iree_llvmaot"
+ ",,count_nonzero,,,iree_llvmaot"
+ ",,cumprod,,,iree_llvmaot"
+ ",,cumsum,,,iree_llvmaot"
+ ",,cumulative_logsumexp,,,iree_llvmaot"
+ ",,digamma,,,iree_llvmaot"
+ ",,divide,,,iree_llvmaot"
+ ",,erf,,,iree_llvmaot"
+ ",,erfc,,,iree_llvmaot"
+ ",,erfinv,,,iree_llvmaot"
+ ",,expm1,,,iree_llvmaot"
+ ",,igamma,,,iree_llvmaot"
+ ",,igammac,,,iree_llvmaot"
+ ",,in_top_k,,,iree_llvmaot"
+ ",,invert_permutation,,,iree_llvmaot"
+ ",,is_non_decreasing,,,iree_llvmaot"
+ ",,is_strictly_increasing,,,iree_llvmaot"
+ ",,l2_normalize,,,iree_llvmaot"
+ ",,ndtri,,,iree_llvmaot"
+ ",,nextafter,,,iree_llvmaot"
+ ",,polygamma,,,iree_llvmaot"
+ ",,reduce_all,,,iree_llvmaot"
+ ",,reduce_any,,,iree_llvmaot"
+ ",,reduce_euclidean_norm,,,iree_llvmaot"
+ ",,reduce_logsumexp,,,iree_llvmaot"
+ ",,reduce_max,,,iree_llvmaot"
+ ",,reduce_mean,,,iree_llvmaot"
+ ",,reduce_min,,,iree_llvmaot"
+ ",,reduce_prod,,,iree_llvmaot"
+ ",,reduce_std,,,iree_llvmaot"
+ ",,reduce_sum,,,iree_llvmaot"
+ ",,reduce_variance,,,iree_llvmaot"
+ ",,rint,,,iree_llvmaot"
+ ",,segment_max,,,iree_llvmaot"
+ ",,segment_mean,,,iree_llvmaot"
+ ",,segment_min,,,iree_llvmaot"
+ ",,segment_prod,,,iree_llvmaot"
+ ",,segment_sum,,,iree_llvmaot"
+ ",,sobol_sample,,,iree_llvmaot"
+ ",,softsign,,,iree_llvmaot"
+ ",,unsorted_segment_max,,,iree_llvmaot"
+ ",,unsorted_segment_mean,,,iree_llvmaot"
+ ",,unsorted_segment_min,,,iree_llvmaot"
+ ",,unsorted_segment_prod,,,iree_llvmaot"
+ ",,unsorted_segment_sqrt_n,,,iree_llvmaot"
+ ",,unsorted_segment_sum,,,iree_llvmaot"
+ ",,zeta,,,iree_llvmaot"
+ ",,angle,,,iree_llvmaot"
+ ",,cos,,,iree_llvmaot"
+ ",,cumsum,,,iree_llvmaot"
+ ",,divide_no_nan,,,iree_llvmaot"
+ ",,log,,,iree_llvmaot"
+ ",,log1p,,,iree_llvmaot"
+ ",,multiply_no_nan,,,iree_llvmaot"
+ ",,negative,,,iree_llvmaot"
+ ",,pow,,,iree_llvmaot"
+ ",,reciprocal,,,iree_llvmaot"
+ ",,reciprocal_no_nan,,,iree_llvmaot"
+ ",,reduce_std,,,iree_llvmaot"
+ ",,reduce_variance,,,iree_llvmaot"
+ ",,rsqrt,,,iree_llvmaot"
+ ",,sigmoid,,,iree_llvmaot"
+ ",,sign,,,iree_llvmaot"
+ ",,sin,,,iree_llvmaot"
+ ",,sinh,,,iree_llvmaot"
+ ",,sqrt,,,iree_llvmaot"
+ ",,tan,,,iree_llvmaot"
+ ",,tanh,,,iree_llvmaot"
+ ",,xdivy,,,iree_llvmaot"
+ ",,xlog1py,,,iree_llvmaot"
+ ",,xlogy,,,iree_llvmaot"
+ ",,zero_fraction,,,iree_llvmaot"
+ ",,acos,,,iree_vulkan"
+ ",,acosh,,,iree_vulkan"
+ ",,argmax,,,iree_vulkan"
+ ",,argmin,,,iree_vulkan"
+ ",,asin,,,iree_vulkan"
+ ",,asinh,,,iree_vulkan"
+ ",,atan,,,iree_vulkan"
+ ",,atan2,,,iree_vulkan"
+ ",,atanh,,,iree_vulkan"
+ ",,bessel_i0,,,iree_vulkan"
+ ",,bessel_i0e,,,iree_vulkan"
+ ",,bessel_i1,,,iree_vulkan"
+ ",,bessel_i1e,,,iree_vulkan"
+ ",,betainc,,,iree_vulkan"
+ ",,bincount,,,iree_vulkan"
+ ",,confusion_matrix,,,iree_vulkan"
+ ",,cosh,,,iree_vulkan"
+ ",,count_nonzero,,,iree_vulkan"
+ ",,cumprod,,,iree_vulkan"
+ ",,cumsum,,,iree_vulkan"
+ ",,cumulative_logsumexp,,,iree_vulkan"
+ ",,digamma,,,iree_vulkan"
+ ",,divide,,,iree_vulkan"
+ ",,erf,,,iree_vulkan"
+ ",,erfc,,,iree_vulkan"
+ ",,erfinv,,,iree_vulkan"
+ ",,expm1,,,iree_vulkan"
+ ",,igamma,,,iree_vulkan"
+ ",,igammac,,,iree_vulkan"
+ ",,in_top_k,,,iree_vulkan"
+ ",,invert_permutation,,,iree_vulkan"
+ ",,is_finite,,,iree_vulkan"
+ ",,is_non_decreasing,,,iree_vulkan"
+ ",,is_strictly_increasing,,,iree_vulkan"
+ ",,l2_normalize,,,iree_vulkan"
+ ",,logical_and,,,iree_vulkan"
+ ",,logical_not,,,iree_vulkan"
+ ",,logical_or,,,iree_vulkan"
+ ",,logical_xor,,,iree_vulkan"
+ ",,mod,,,iree_vulkan"
+ ",,ndtri,,,iree_vulkan"
+ ",,nextafter,,,iree_vulkan"
+ ",,polygamma,,,iree_vulkan"
+ ",,pow,,,iree_vulkan"
+ ",,reduce_all,,,iree_vulkan"
+ ",,reduce_any,,,iree_vulkan"
+ ",,reduce_euclidean_norm,,,iree_vulkan"
+ ",,reduce_logsumexp,,,iree_vulkan"
+ ",,reduce_max,,,iree_vulkan"
+ ",,reduce_mean,,,iree_vulkan"
+ ",,reduce_min,,,iree_vulkan"
+ ",,reduce_prod,,,iree_vulkan"
+ ",,reduce_std,,,iree_vulkan"
+ ",,reduce_sum,,,iree_vulkan"
+ ",,reduce_variance,,,iree_vulkan"
+ ",,rint,,,iree_vulkan"
+ ",,segment_max,,,iree_vulkan"
+ ",,segment_mean,,,iree_vulkan"
+ ",,segment_min,,,iree_vulkan"
+ ",,segment_prod,,,iree_vulkan"
+ ",,segment_sum,,,iree_vulkan"
+ ",,sign,,,iree_vulkan"
+ ",,sobol_sample,,,iree_vulkan"
+ ",,softsign,,,iree_vulkan"
+ ",,unsorted_segment_max,,,iree_vulkan"
+ ",,unsorted_segment_mean,,,iree_vulkan"
+ ",,unsorted_segment_min,,,iree_vulkan"
+ ",,unsorted_segment_prod,,,iree_vulkan"
+ ",,unsorted_segment_sqrt_n,,,iree_vulkan"
+ ",,unsorted_segment_sum,,,iree_vulkan"
+ ",,zeta,,,iree_vulkan"
+ ",,angle,,,iree_vulkan"
+ ",,cos,,,iree_vulkan"
+ ",,cumsum,,,iree_vulkan"
+ ",,divide_no_nan,,,iree_vulkan"
+ ",,log,,,iree_vulkan"
+ ",,log1p,,,iree_vulkan"
+ ",,multiply_no_nan,,,iree_vulkan"
+ ",,negative,,,iree_vulkan"
+ ",,pow,,,iree_vulkan"
+ ",,reciprocal,,,iree_vulkan"
+ ",,reciprocal_no_nan,,,iree_vulkan"
+ ",,reduce_std,,,iree_vulkan"
+ ",,reduce_variance,,,iree_vulkan"
+ ",,rsqrt,,,iree_vulkan"
+ ",,sigmoid,,,iree_vulkan"
+ ",,sign,,,iree_vulkan"
+ ",,sin,,,iree_vulkan"
+ ",,sinh,,,iree_vulkan"
+ ",,sqrt,,,iree_vulkan"
+ ",,tan,,,iree_vulkan"
+ ",,tanh,,,iree_vulkan"
+ ",,xdivy,,,iree_vulkan"
+ ",,xlog1py,,,iree_vulkan"
+ ",,xlogy,,,iree_vulkan"
+ ",,zero_fraction,,,iree_vulkan"
+)
+
+### BAZEL_TO_CMAKE_PRESERVES_ALL_CONTENT_BELOW_THIS_LINE ###
diff --git a/integrations/tensorflow/e2e/slim_vision_models/CMakeLists.txt b/integrations/tensorflow/e2e/slim_vision_models/CMakeLists.txt
new file mode 100644
index 0000000..b72c985
--- /dev/null
+++ b/integrations/tensorflow/e2e/slim_vision_models/CMakeLists.txt
@@ -0,0 +1,51 @@
+################################################################################
+# Autogenerated by build_tools/bazel_to_cmake/bazel_to_cmake.py from #
+# integrations/tensorflow/e2e/slim_vision_models/BUILD #
+# #
+# Use iree_cmake_extra_content from iree/build_defs.oss.bzl to add arbitrary #
+# CMake-only content. #
+# #
+# To disable autogeneration for this file entirely, delete this header. #
+################################################################################
+
+iree_add_all_subdirs()
+
+iree_e2e_cartesian_product_test_suite(
+ NAME
+ slim_vision_tests
+ MATRIX_KEYS
+ "src"
+ "reference_backend"
+ "tf_hub_url"
+ "model"
+ "target_backends"
+ MATRIX_VALUES
+ "slim_vision_model_test.py"
+ "tf"
+ "https://tfhub.dev/google/imagenet/"
+ "amoebanet_a_n18_f448;inception_resnet_v2;inception_v1;inception_v2;inception_v3;mobilenet_v1_025_128;mobilenet_v1_025_160;mobilenet_v1_025_192;mobilenet_v1_025_224;mobilenet_v1_050_128;mobilenet_v1_050_160;mobilenet_v1_050_192;mobilenet_v1_050_224;mobilenet_v1_075_128;mobilenet_v1_075_160;mobilenet_v1_075_192;mobilenet_v1_075_224;mobilenet_v1_100_128;mobilenet_v1_100_160;mobilenet_v1_100_192;mobilenet_v1_100_224;mobilenet_v2_035_96;mobilenet_v2_035_128;mobilenet_v2_035_160;mobilenet_v2_035_192;mobilenet_v2_035_224;mobilenet_v2_050_96;mobilenet_v2_050_128;mobilenet_v2_050_160;mobilenet_v2_050_192;mobilenet_v2_050_224;mobilenet_v2_075_96;mobilenet_v2_075_128;mobilenet_v2_075_160;mobilenet_v2_075_192;mobilenet_v2_075_224;mobilenet_v2_100_96;mobilenet_v2_100_128;mobilenet_v2_100_160;mobilenet_v2_100_192;mobilenet_v2_100_224;mobilenet_v2_130_224;mobilenet_v2_140_224;nasnet_mobile;nasnet_large;pnasnet_large;resnet_v1_50;resnet_v1_101;resnet_v1_152;resnet_v2_50;resnet_v2_101;resnet_v2_152"
+ "tf;tflite;iree_vmla;iree_vulkan"
+ FAILING_CONFIGURATIONS
+ ",,,amoebanet_a_n18_f448,"
+ ",,,nasnet_large,iree_vmla"
+ ",,,nasnet_mobile,iree_vmla"
+ ",,,nasnet_mobile,iree_vulkan"
+ ",,,nasnet_large,iree_vulkan"
+ ",,,pnasnet_large,iree_vulkan"
+ ",,,resnet_v2_50,iree_vulkan"
+ ",,,resnet_v2_101,iree_vulkan"
+ ",,,resnet_v2_152,iree_vulkan"
+ ",,,inception_v1,iree_vulkan"
+ ",,,inception_v2,iree_vulkan"
+ ",,,inception_v3,iree_vulkan"
+ ",,,inception_resnet_v2,iree_vulkan"
+ LABELS
+ "external"
+ "guitar"
+ "manual"
+ "no-remote"
+ "nokokoro"
+ "notap"
+)
+
+### BAZEL_TO_CMAKE_PRESERVES_ALL_CONTENT_BELOW_THIS_LINE ###
