Change IREE_DEFAULT_BACKENDS semantics to IREE_AVAILABLE_BACKENDS. IREE_AVAILABLE_BACKENDS is like IREE_DEFAULT_BACKENDS but has a slightly different behavior. Rather than being a fallback, it behaves as a "mask" on the backends that are otherwise selected. This is more suitable for CI and developer workflows where certain backends aren't available. Also, remove one test that was testing that a "disagreements" error was raised. If running with a reduced set of available backends, then it is possible to execute with just one backend, causing the test to fail since it isn't possible to compute the disagreements and raise the error. PiperOrigin-RevId: 306347874
diff --git a/integrations/tensorflow/bindings/python/pyiree/tf/support/tf_test_utils.py b/integrations/tensorflow/bindings/python/pyiree/tf/support/tf_test_utils.py index 29e9d3d..0459983 100644 --- a/integrations/tensorflow/bindings/python/pyiree/tf/support/tf_test_utils.py +++ b/integrations/tensorflow/bindings/python/pyiree/tf/support/tf_test_utils.py
@@ -516,9 +516,9 @@ return None -def get_default_backends(): - """Gets the BackendInfo instances to use by default.""" - backend_spec = os.environ.get("IREE_DEFAULT_BACKENDS") +def get_available_backends(): + """Gets the BackendInfo instances considered available for use.""" + backend_spec = os.environ.get("IREE_AVAILABLE_BACKENDS") if backend_spec is None: return BackendInfo.ALL.values() return _backend_spec_string_to_backends(backend_spec) @@ -581,8 +581,19 @@ if override_backends is not None: backends = override_backends elif backends is None: - backends = get_default_backends() + backends = list(BackendInfo.ALL.keys()) backends = [_resolve(backend) for backend in backends] + available_backends = get_available_backends() + backends = [ + backend for backend in backends if backend in available_backends + ] + if not backends: + # If no backends are available, then to avoid errors down the line, + # just use "tf", which should always be safe. + backends = [BackendInfo.ALL["tf"]] + logging.warning( + "Falling back to just `tf` backend because no other requested backends are available. Available backends '%s'", + [backend.name for backend in available_backends]) cls.compiled_modules[name] = dict([ (backend.name, CompiledModule.create(ctor, exported_names, backend))
diff --git a/integrations/tensorflow/e2e/README.md b/integrations/tensorflow/e2e/README.md index f12f654..12b5a45 100644 --- a/integrations/tensorflow/e2e/README.md +++ b/integrations/tensorflow/e2e/README.md
@@ -17,8 +17,8 @@ If you do not have your environment setup to use IREE with Vulkan (see [the doc](../../../docs/vulkan_and_spirv.md)), then you can run the tests with -`IREE_DEFAULT_BACKENDS=tf,iree_vmla` (that is, by omitting `iree_vulkan` from -the list of backends to use). +`IREE_AVAILABLE_BACKENDS=tf,iree_vmla` (that is, by omitting `iree_vulkan` from +the list of available backends). ## Running tests @@ -75,10 +75,24 @@ 1. The backends specified in `--override_backends`. -2. The backends specified in `IREE_OVERRIDE_BACKENDS`. +1. The backends specified in the `IREE_OVERRIDE_BACKENDS` environment variable. -3. The backends specified in the `tf_test_utils.compile_modules` decorator. +1. The backends specified in the `tf_test_utils.compile_modules` decorator. -4. The backends specified in `IREE_DEFAULT_BACKENDS`. +1. All known backends. -5. All known backends. +Additionally, the environment variable `IREE_AVAILABLE_BACKENDS` specifies which +backends should be considered available in a particular environment. Once the +list of backends above is formed, any backends not listed in +`IREE_AVAILABLE_BACKENDS` are removed. This is the final list of backends which +are run for the test. + +The default behavior if `IREE_AVAILABLE_BACKENDS` is not provided is that all +known backends are considered available. + +TODO(silvasean): `IREE_AVAILABLE_BACKENDS` is mainly to allow masking off the +Vulkan backend in environments where it is not a available. Currently, the +behavior when all backends get masked off is to emit a warning, which can result +in spuriously "passing" tests. This is only an issue for tests that currently +only run on Vulkan (which should decrease over time as e.g. VMLA gets more +coverage).
diff --git a/integrations/tensorflow/e2e/simple_arithmetic_test.py b/integrations/tensorflow/e2e/simple_arithmetic_test.py index 7cb55fd..a45c123 100644 --- a/integrations/tensorflow/e2e/simple_arithmetic_test.py +++ b/integrations/tensorflow/e2e/simple_arithmetic_test.py
@@ -75,16 +75,6 @@ r = self.modules.simple_arithmetic.all.simple_matmul(a, b) r.print().assert_all_close() - def test_disagreement(self): - a = np.array([1., 2., 3., 4.], dtype=np.float32) - b = np.array([400., 5., 6., 7.], dtype=np.float32) - - vmod = self.modules.simple_arithmetic.all - r = vmod.simple_mul(a, b) - # Using a negative atol will cause disagreement even if identical. - with self.assertRaisesRegex(AssertionError, "Disagreements"): - r.print().assert_all_close(atol=-3.0) - if __name__ == "__main__": if hasattr(tf, "enable_v2_behavior"):