Disabling integrations/tensorflow/e2e LLVM JIT tests.
The LLVM JIT driver is deprecated and going away. These tests should
be using LLVM AOT instead (#2673), but this test infra is scary.
Ideally, this infra would barely be doing any of this per-backend;
there should be few - if any - things exclusively tested by this
path that aren't also covered in core tests that don't build/run
from python. Or at least, the infra should not be doing all of this
via build files (bazel is not a configuration language).
Unblocks #3757 and #3843.
diff --git a/integrations/tensorflow/bindings/python/pyiree/tf/support/BUILD b/integrations/tensorflow/bindings/python/pyiree/tf/support/BUILD
index fc67c96..9b1c869 100644
--- a/integrations/tensorflow/bindings/python/pyiree/tf/support/BUILD
+++ b/integrations/tensorflow/bindings/python/pyiree/tf/support/BUILD
@@ -60,7 +60,6 @@
],
python_version = "PY3",
tags = [
- "driver=llvm",
"driver=vmla",
],
deps = INTREE_TENSORFLOW_PY_DEPS + [
diff --git a/integrations/tensorflow/bindings/python/pyiree/tf/support/tf_utils.py b/integrations/tensorflow/bindings/python/pyiree/tf/support/tf_utils.py
index 687d78b..c1281dd 100644
--- a/integrations/tensorflow/bindings/python/pyiree/tf/support/tf_utils.py
+++ b/integrations/tensorflow/bindings/python/pyiree/tf/support/tf_utils.py
@@ -988,11 +988,6 @@
"driver": "vmla",
"compiler_targets": ["vmla"]
},
- "iree_llvmjit": {
- "compiled_module_class": IreeCompiledModule,
- "driver": "llvm",
- "compiler_targets": ["llvm-ir"]
- },
"iree_vulkan": {
"compiled_module_class": IreeCompiledModule,
"driver": "vulkan",
@@ -1005,13 +1000,13 @@
Args:
backend_name: a str specifying which backend to use. Should be one of
- 'tf', 'iree_vmla', 'iree_llvmjit', 'iree_vulkan'.
+ 'tf', 'iree_vmla', 'iree_vulkan'.
backend_id: an optional str specifying what name to use when saving
compiled artifacts. Must satisfy `backend_id.startswith(backend_name)`.
Raises:
- KeyError: if backend_name is not one of ['tf', 'iree_vmla',
- 'iree_llvmjit', 'iree_vulkan'].
+ KeyError: if backend_name is not one of
+ ['tf', 'iree_vmla', 'iree_vulkan'].
ValueError: if backend_id doesn't start with backend_name.
"""
if backend_name not in self._name_to_info:
diff --git a/integrations/tensorflow/e2e/README.md b/integrations/tensorflow/e2e/README.md
index 0ac1605..4b457b5 100644
--- a/integrations/tensorflow/e2e/README.md
+++ b/integrations/tensorflow/e2e/README.md
@@ -18,7 +18,7 @@
If you do not have your environment setup to use IREE with Vulkan (see
[this doc](https://google.github.io/iree/get-started/generic-vulkan-env-setup)),
then you can run the manual test targets with
-`--target_backends=tf,iree_vmla,iree_llvmjit` (that is, by omitting
+`--target_backends=tf,iree_vmla` (that is, by omitting
`iree_vulkan` from the list of backends to run the tests on).
The test suites can be run excluding Vulkan by specifying
@@ -234,8 +234,8 @@
to check numerical correctness against TensorFlow. Tests targets that pass are
placed into the `e2e_tests` test suite. Tests that fail on particular backends
are recorded in lists in the `BUILD` files. For example, if
-`experimental_new_test.py` fails on the `iree_llvmjit` and `iree_vulkan`
-backends then the following lines should be added to the `BUILD` file:
+`experimental_new_test.py` fails on the `iree_vulkan` backend then the following
+lines should be added to the `BUILD` file:
```build
LLVM_FAILING = [
diff --git a/integrations/tensorflow/e2e/iree_e2e_cartesian_product_test_suite.bzl b/integrations/tensorflow/e2e/iree_e2e_cartesian_product_test_suite.bzl
index ee6d4f1..df6b021 100644
--- a/integrations/tensorflow/e2e/iree_e2e_cartesian_product_test_suite.bzl
+++ b/integrations/tensorflow/e2e/iree_e2e_cartesian_product_test_suite.bzl
@@ -164,6 +164,13 @@
tests = []
for flags in all_flag_configurations:
+ if len(flags["target_backends"].split(",")) > 1:
+ fail("Multiple target backends cannot be specified at once, but " +
+ "got `{}`".format(flags["target_backends"]))
+ driver = get_driver(flags["target_backends"])
+ if not driver:
+ continue
+
# Check if this is a failing configuration.
failing = flags in failing_flag_configurations
@@ -180,12 +187,6 @@
tests.append(test_name)
args = ["--{}={}".format(k, v) for k, v in flags.items()]
-
- if len(flags["target_backends"].split(",")) > 1:
- fail("Multiple target backends cannot be specified at once, but " +
- "got `{}`".format(flags["target_backends"]))
-
- driver = get_driver(flags["target_backends"])
py_test_tags = ["driver={}".format(driver)]
if tags != None: # `is` is not supported.
py_test_tags += tags
diff --git a/integrations/tensorflow/e2e/iree_e2e_test_suite.bzl b/integrations/tensorflow/e2e/iree_e2e_test_suite.bzl
index 8fb35a4..2892e60 100644
--- a/integrations/tensorflow/e2e/iree_e2e_test_suite.bzl
+++ b/integrations/tensorflow/e2e/iree_e2e_test_suite.bzl
@@ -19,8 +19,9 @@
def get_driver(backend):
# TODO(#2175): Simplify this after backend names are standardized.
driver = backend.replace("iree_", "") # "iree_<driver>" --> "<driver>"
+ # TODO(#2673): enable LLVM AOT for these tests. JIT is deprecated.
if driver == "llvmjit":
- driver = "llvm"
+ driver = ""
return driver
def iree_e2e_test_suite(
@@ -70,6 +71,8 @@
]
driver = get_driver(backend)
+ if not driver:
+ continue
py_test_tags = ["driver={}".format(driver)]
if tags != None: # `is` is not supported.
py_test_tags += tags
diff --git a/integrations/tensorflow/e2e/keras/BUILD b/integrations/tensorflow/e2e/keras/BUILD
index d6e61ad..468eb79 100644
--- a/integrations/tensorflow/e2e/keras/BUILD
+++ b/integrations/tensorflow/e2e/keras/BUILD
@@ -44,13 +44,13 @@
vision_model_test_manual is for manual testing of all keras vision models.
Test will run only manually with all parameters specified manually, for example:
bazel run -c opt integrations/tensorflow/e2e/keras:vision_model_test_manual -- \
---target_backends=tf,iree_vmla,iree_llvmjit \
+--target_backends=tf,iree_vmla \
--data=imagenet \
--url=https://storage.googleapis.com/iree_models/ \
--model=ResNet50
Command arguments description:
---target_backends: can be combination of these: tf,iree_vmla,iree_llvmjit
+--target_backends: can be combination of these: tf,iree_vmla
--data: can be 'imagenet' or 'cifar10'.
imagenet - input image size (1, 224, 224, 3)
cifar10 - input image size (1, 32, 32, 3) - it is used for quick tests
diff --git a/integrations/tensorflow/e2e/slim_vision_models/BUILD b/integrations/tensorflow/e2e/slim_vision_models/BUILD
index 88f60b0..881aff3 100644
--- a/integrations/tensorflow/e2e/slim_vision_models/BUILD
+++ b/integrations/tensorflow/e2e/slim_vision_models/BUILD
@@ -76,7 +76,6 @@
"resnet_v2_152",
],
"target_backends": [
- "iree_llvmjit",
"iree_vulkan",
],
},
@@ -159,7 +158,6 @@
"tf",
"tflite",
"iree_vmla",
- "iree_llvmjit",
"iree_vulkan",
],
},
diff --git a/scripts/update_e2e_coverage.py b/scripts/update_e2e_coverage.py
index 325cb28..fed8c12 100755
--- a/scripts/update_e2e_coverage.py
+++ b/scripts/update_e2e_coverage.py
@@ -28,13 +28,12 @@
TENSORFLOW_COVERAGE_DIR = 'tensorflow_coverage'
REFERENCE_BACKEND = 'tf'
-# Assumes that tests are expanded for the tf, iree_vmla, iree_llvmjit and
+# Assumes that tests are expanded for the tf, iree_vmla, and
# iree_vulkan backends.
BACKENDS_TO_TITLES = collections.OrderedDict([
('tf', 'tensorflow'),
('tflite', 'tflite'),
('iree_vmla', 'vmla'),
- ('iree_llvmjit', 'llvm-ir'),
('iree_vulkan', 'vulkan-spirv'),
])
