Merge remote-tracking branch 'origin/master' into cross-cmake
diff --git a/.gitmodules b/.gitmodules
index 8733410..3ed8ebc 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -47,3 +47,6 @@
[submodule "third_party/tracy"]
path = third_party/tracy
url = https://github.com/wolfpld/tracy.git
+[submodule "third_party/mlir-emitc"]
+ path = third_party/mlir-emitc
+ url = https://github.com/iml130/mlir-emitc.git
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 1067fb8..76dc3d1 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -36,6 +36,7 @@
# LINT.IfChange(iree_options)
option(IREE_ENABLE_RUNTIME_TRACING "Enables instrumented runtime tracing." OFF)
option(IREE_ENABLE_LLVM "Enables LLVM dependencies." ON)
+option(IREE_ENABLE_EMITC "Enables MLIR EmitC dependencies." OFF)
option(IREE_BUILD_COMPILER "Builds the IREE compiler." ON)
option(IREE_BUILD_TESTS "Builds IREE unit tests." ON)
@@ -60,6 +61,10 @@
set(IREE_ENABLE_LLVM ON CACHE BOOL "Enable LLVM dependencies if the IREE compiler is build." FORCE)
endif()
+if(${IREE_ENABLE_EMITC} AND NOT ${IREE_ENABLE_LLVM})
+ message(FATAL_ERROR "Enabling EmitC requires setting IREE_ENABLE_LLVM to ON.")
+endif()
+
#-------------------------------------------------------------------------------
# Target and backend configuration
#-------------------------------------------------------------------------------
@@ -248,6 +253,9 @@
endif()
add_subdirectory(third_party/llvm-project/llvm EXCLUDE_FROM_ALL)
+ if(${IREE_ENABLE_EMITC})
+ add_subdirectory(third_party/mlir-emitc EXCLUDE_FROM_ALL)
+ endif()
# Reset CMAKE_BUILD_TYPE to its previous setting
set(CMAKE_BUILD_TYPE "${_CMAKE_BUILD_TYPE}" CACHE STRING "Build type (default ${DEFAULT_CMAKE_BUILD_TYPE})" FORCE)
diff --git a/SUBMODULE_VERSIONS b/SUBMODULE_VERSIONS
index 7b1a6a3..ed57676 100644
--- a/SUBMODULE_VERSIONS
+++ b/SUBMODULE_VERSIONS
@@ -3,13 +3,14 @@
4c13807b7d43ff0946b7ffea0ae3aee9e611d778 third_party/dear_imgui
a5d9d0f7d368054fd1691aedf1db4116efcc233e third_party/flatbuffers
f2fb48c3b3d79a75a88a99fba6576b25d42ec528 third_party/googletest
-a8ca0ec267050f9ded865a729d50c2c0eb078b7e third_party/llvm-project
+df06f4ff227bdcbbad01b418199876761f2a1ff0 third_party/llvm-project
+53265de20088f8b2af381c77173b22764cbfc038 third_party/mlir-emitc
80d452484c5409444b0ec19383faa84bb7a4d351 third_party/pybind11
9f53ba413e6fc879236dcaa3e008915973d67a4f third_party/ruy
b73f111094da3e380a1774b56b15f16c90ae8e23 third_party/sdl2
f8bf11a0253a32375c32cad92c841237b96696c0 third_party/spirv_headers
57eb48aed36160c4876bc8310d9ca84d42ee9e2a third_party/swiftshader
-f653ab8bb3910959c8189f163e5626df61ff0eab third_party/tensorflow
+0836e5e4ea18b81e43cae801307d642d82548b29 third_party/tensorflow
864d86e8b6d21449474db5e9313dbff90aa9c24f third_party/tracy
8a457f8552d8d47ce3a96ed80a714ff6396f8ad8 third_party/vulkan_extensionlayer
9bd3f561bcee3f01d22912de10bb07ce4e23d378 third_party/vulkan_headers
diff --git a/bindings/python/build_defs.oss.bzl b/bindings/python/build_defs.oss.bzl
index 7363c64..d8cf789 100644
--- a/bindings/python/build_defs.oss.bzl
+++ b/bindings/python/build_defs.oss.bzl
@@ -47,8 +47,7 @@
name,
copts = [],
features = [],
- deps = [
- ],
+ deps = [],
**kwargs):
"""Wrapper cc_library for deps that are part of the python bindings."""
cc_library(
diff --git a/build_tools/bazel/build.sh b/build_tools/bazel/build_bindings.sh
similarity index 94%
copy from build_tools/bazel/build.sh
copy to build_tools/bazel/build_bindings.sh
index df68412..56601d8 100755
--- a/build_tools/bazel/build.sh
+++ b/build_tools/bazel/build_bindings.sh
@@ -14,8 +14,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-# Build the IREE project with bazel. Designed for CI, but can be run manually.
-# Looks at environment variables and uses CI-friendly defaults if they are not set.
+# Build IREE's bindings (//bindings/...) with bazel. Designed for CI, but can be
+# run manually.
+
+# Looks at environment variables and uses CI-friendly defaults if they are not
+# set.
# IREE_LLVMJIT_DISABLE: Do not run tests that require LLVM-JIT. Default: 1
# IREE_VULKAN_DISABLE: Do not run tests that require Vulkan. Default: 1
# BUILD_TAG_FILTERS: Passed to bazel to filter targets to build.
@@ -66,7 +69,7 @@
# `bazel test //...` because the latter excludes targets tagged "manual". The
# "manual" tag allows targets to be excluded from human wildcard builds, but we
# want them built by CI unless they are excluded with "nokokoro".
-bazel query //iree/... + //bindings/... | \
+bazel query //bindings/... | \
xargs bazel test ${test_env_args[@]} \
--config=generic_clang \
--build_tag_filters="${BUILD_TAG_FILTERS?}" \
diff --git a/build_tools/bazel/build.sh b/build_tools/bazel/build_core.sh
similarity index 95%
rename from build_tools/bazel/build.sh
rename to build_tools/bazel/build_core.sh
index df68412..00e104e 100755
--- a/build_tools/bazel/build.sh
+++ b/build_tools/bazel/build_core.sh
@@ -14,8 +14,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-# Build the IREE project with bazel. Designed for CI, but can be run manually.
-# Looks at environment variables and uses CI-friendly defaults if they are not set.
+# Build IREE's core (//iree/...) with bazel. Designed for CI, but can be run
+# manually.
+
+# Looks at environment variables and uses CI-friendly defaults if they are not
+# set.
# IREE_LLVMJIT_DISABLE: Do not run tests that require LLVM-JIT. Default: 1
# IREE_VULKAN_DISABLE: Do not run tests that require Vulkan. Default: 1
# BUILD_TAG_FILTERS: Passed to bazel to filter targets to build.
@@ -66,7 +69,7 @@
# `bazel test //...` because the latter excludes targets tagged "manual". The
# "manual" tag allows targets to be excluded from human wildcard builds, but we
# want them built by CI unless they are excluded with "nokokoro".
-bazel query //iree/... + //bindings/... | \
+bazel query //iree/... | \
xargs bazel test ${test_env_args[@]} \
--config=generic_clang \
--build_tag_filters="${BUILD_TAG_FILTERS?}" \
diff --git a/build_tools/bazel/build.sh b/build_tools/bazel/build_tensorflow.sh
similarity index 89%
copy from build_tools/bazel/build.sh
copy to build_tools/bazel/build_tensorflow.sh
index df68412..cfffffe 100755
--- a/build_tools/bazel/build.sh
+++ b/build_tools/bazel/build_tensorflow.sh
@@ -14,8 +14,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-# Build the IREE project with bazel. Designed for CI, but can be run manually.
-# Looks at environment variables and uses CI-friendly defaults if they are not set.
+# Build IREE's integrations (//integrations/...) with bazel. Designed for CI,
+# but can be run manually.
+
+# Looks at environment variables and uses CI-friendly defaults if they are not
+# set.
# IREE_LLVMJIT_DISABLE: Do not run tests that require LLVM-JIT. Default: 1
# IREE_VULKAN_DISABLE: Do not run tests that require Vulkan. Default: 1
# BUILD_TAG_FILTERS: Passed to bazel to filter targets to build.
@@ -39,6 +42,7 @@
declare -a test_env_args=(
--test_env=IREE_LLVMJIT_DISABLE=$IREE_LLVMJIT_DISABLE
--test_env=IREE_VULKAN_DISABLE=$IREE_VULKAN_DISABLE
+ --test_env=IREE_AVAILABLE_BACKENDS="tf,iree_vmla"
)
declare -a default_build_tag_filters=("-nokokoro")
@@ -66,12 +70,13 @@
# `bazel test //...` because the latter excludes targets tagged "manual". The
# "manual" tag allows targets to be excluded from human wildcard builds, but we
# want them built by CI unless they are excluded with "nokokoro".
-bazel query //iree/... + //bindings/... | \
+bazel query '//integrations/...' | \
xargs bazel test ${test_env_args[@]} \
--config=generic_clang \
--build_tag_filters="${BUILD_TAG_FILTERS?}" \
--test_tag_filters="${TEST_TAG_FILTERS?}" \
- --keep_going \
--test_output=errors \
- --config=rs \
- --config=rbe
+ --keep_going
+ # TODO: Enable result store once the Kokoro VMs used for this test have the
+ # appropriate auth.
+ # --config=rs
diff --git a/build_tools/bazel/iree_py_test_suite.bzl b/build_tools/bazel/iree_py_test_suite.bzl
new file mode 100644
index 0000000..dab7055
--- /dev/null
+++ b/build_tools/bazel/iree_py_test_suite.bzl
@@ -0,0 +1,67 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Macro for building python test suites."""
+
+load("//bindings/python:build_defs.oss.bzl", "iree_py_test")
+
+def iree_py_test_suite(
+ name,
+ srcs,
+ deps = None,
+ tags = None,
+ size = None,
+ python_version = "PY3",
+ **kwargs):
+ """Creates one iree_py_test per source file and a test suite that bundles them.
+
+ Args:
+ name: name of the generated test suite.
+ srcs: test file sources.
+ deps: test dependencies.
+ tags: tags to apply to the test. Note that as in standard test suites,
+ manual is treated specially and will also apply to the test suite
+ itself.
+ size: size of the tests.
+ python_version: the python version to run the tests with. Uses python3
+ by default.
+ **kwargs: Any additional arguments that will be passed to the underlying
+ tests and test_suite.
+ """
+ tests = []
+ for src in srcs:
+ test_name = "{}_{}".format(name, src[:-3])
+ iree_py_test(
+ name = test_name,
+ main = src,
+ srcs = [src],
+ deps = deps,
+ tags = tags,
+ size = size,
+ python_version = python_version,
+ **kwargs
+ )
+ tests.append(test_name)
+
+ native.test_suite(
+ name = name,
+ tests = tests,
+ # Note that only the manual tag really has any effect here. Others are
+ # used for test suite filtering, but all tests are passed the same tags.
+ tags = tags,
+ # If there are kwargs that need to be passed here which only apply to
+ # the generated tests and not to test_suite, they should be extracted
+ # into separate named arguments.
+ **kwargs
+ )
diff --git a/docs/benchmarking.md b/docs/benchmarking.md
index 7037e45..1712fae 100644
--- a/docs/benchmarking.md
+++ b/docs/benchmarking.md
@@ -98,7 +98,7 @@
BM_RunModule/process_time/real_time 11416 ns 14202 ns 61654
```
-Remember to [restore cpu scaling](#cpu-configuration) when you're done.
+Remember to [restore CPU scaling](#cpu-configuration) when you're done.
## Microbenchmarks
@@ -129,12 +129,14 @@
Google benchmark provides some
[instructions](https://github.com/google/benchmark#disabling-cpu-frequency-scaling):
-Before benchmarking:
+Turn off CPU scaling before benchmarking:
```shell
$ sudo cpupower frequency-set --governor performance
```
+Restore CPU scaling after benchmarking:
+
```shell
$ sudo cpupower frequency-set --governor powersave
```
diff --git a/integrations/tensorflow/bindings/python/pyiree/tf/compiler/BUILD b/integrations/tensorflow/bindings/python/pyiree/tf/compiler/BUILD
index 6351fe3..89c8db1 100644
--- a/integrations/tensorflow/bindings/python/pyiree/tf/compiler/BUILD
+++ b/integrations/tensorflow/bindings/python/pyiree/tf/compiler/BUILD
@@ -121,3 +121,17 @@
"//integrations/tensorflow/bindings/python/pyiree/tf/compiler",
],
)
+
+iree_py_test(
+ name = "signature_def_saved_model_test",
+ srcs = ["signature_def_saved_model_test.py"],
+ python_version = "PY3",
+ tags = [
+ "noga",
+ "nokokoro",
+ ],
+ deps = INTREE_TENSORFLOW_PY_DEPS + NUMPY_DEPS + [
+ "@absl_py//absl/testing:absltest",
+ "//integrations/tensorflow/bindings/python/pyiree/tf/compiler",
+ ],
+)
diff --git a/integrations/tensorflow/bindings/python/pyiree/tf/compiler/saved_model_test.py b/integrations/tensorflow/bindings/python/pyiree/tf/compiler/saved_model_test.py
index 6f769dc..7fd7cee 100644
--- a/integrations/tensorflow/bindings/python/pyiree/tf/compiler/saved_model_test.py
+++ b/integrations/tensorflow/bindings/python/pyiree/tf/compiler/saved_model_test.py
@@ -71,32 +71,6 @@
print("XLA ASM:", xla_asm)
self.assertRegex(xla_asm, "xla_hlo.tanh")
- def testLoadSignatureDefSavedModel(self):
- """Tests loading a SignatureDef saved model with a single variable."""
-
- with tempfile.TemporaryDirectory() as temp_dir:
- sm_dir = os.path.join(temp_dir, "simple.sm")
- print("Saving to:", sm_dir)
-
- with tf.Graph().as_default() as graph:
- v = tf.Variable(10)
- result = v.read_value()
- tensor_info = tf.compat.v1.saved_model.utils.build_tensor_info(result)
- sig = tf.compat.v1.saved_model.signature_def_utils.build_signature_def(
- inputs={}, outputs={"result": tensor_info}, method_name="foo")
- builder = tf.compat.v1.saved_model.Builder(sm_dir)
- with tf.compat.v1.Session(graph=graph) as sess:
- sess.run(v.initializer)
- builder.add_meta_graph_and_variables(
- sess, ["bar"], {"baz": sig}, strip_default_attrs=True)
- builder.save()
-
- module = compiler.tf_load_signature_def_saved_model(
- sm_dir, tags=set(["bar"]), exported_names=["baz"])
-
- module_asm = module.to_asm(large_element_limit=100)
- self.assertRegexpMatches(module_asm, "flow.variable @[^ ]* dense<10>")
-
if __name__ == "__main__":
tf.test.main()
diff --git a/integrations/tensorflow/bindings/python/pyiree/tf/compiler/signature_def_saved_model_test.py b/integrations/tensorflow/bindings/python/pyiree/tf/compiler/signature_def_saved_model_test.py
new file mode 100644
index 0000000..da94f6f
--- /dev/null
+++ b/integrations/tensorflow/bindings/python/pyiree/tf/compiler/signature_def_saved_model_test.py
@@ -0,0 +1,70 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import importlib
+import os
+import sys
+import tempfile
+
+from pyiree.tf import compiler
+
+# Dynamically import tensorflow.
+try:
+ # Use a dynamic import so as to avoid hermetic dependency analysis
+ # (i.e. we only want the tensorflow from the environment).
+ tf = importlib.import_module("tensorflow")
+ # Just in case if linked against a pre-V2 defaulted version.
+ if hasattr(tf, "enable_v2_behavior"):
+ tf.enable_v2_behavior()
+ tf = tf.compat.v2
+except ImportError:
+ print("Not running tests because tensorflow is not available")
+ sys.exit(0)
+
+
+class RuntimeTest(tf.test.TestCase):
+
+ def testLoadSignatureDefSavedModel(self):
+ """Tests loading a SignatureDef saved model with a single variable."""
+
+ with tempfile.TemporaryDirectory() as temp_dir:
+ sm_dir = os.path.join(temp_dir, "simple.sm")
+ print("Saving to:", sm_dir)
+
+ with tf.Graph().as_default() as graph:
+ v = tf.Variable(10)
+ result = v.read_value()
+ tensor_info = tf.compat.v1.saved_model.utils.build_tensor_info(result)
+ sig = tf.compat.v1.saved_model.signature_def_utils.build_signature_def(
+ inputs={}, outputs={"result": tensor_info}, method_name="foo")
+ builder = tf.compat.v1.saved_model.Builder(sm_dir)
+ with tf.compat.v1.Session(graph=graph) as sess:
+ sess.run(v.initializer)
+ builder.add_meta_graph_and_variables(
+ sess, ["bar"], {"baz": sig}, strip_default_attrs=True)
+ builder.save()
+
+ module = compiler.tf_load_signature_def_saved_model(
+ sm_dir, tags=set(["bar"]), exported_names=["baz"])
+
+ module_asm = module.to_asm(large_element_limit=100)
+ self.assertRegexpMatches(module_asm, "flow.variable @[^ ]* dense<10>")
+
+
+if __name__ == "__main__":
+ tf.test.main()
diff --git a/integrations/tensorflow/bindings/python/pyiree/xla/compiler/BUILD b/integrations/tensorflow/bindings/python/pyiree/xla/compiler/BUILD
index f9cad82..8b70f4f 100644
--- a/integrations/tensorflow/bindings/python/pyiree/xla/compiler/BUILD
+++ b/integrations/tensorflow/bindings/python/pyiree/xla/compiler/BUILD
@@ -36,7 +36,10 @@
"__init__.py",
],
srcs_version = "PY3",
- tags = ["noga"],
+ tags = [
+ "noga",
+ "nokokoro",
+ ],
deps = [
":binding",
"//bindings/python:pathsetup", # build_cleaner: keep
@@ -52,7 +55,10 @@
copts = PYBIND_COPTS + PYBIND_EXTENSION_COPTS,
features = PYBIND_FEATURES,
linkstatic = 1,
- tags = ["noga"],
+ tags = [
+ "noga",
+ "nokokoro",
+ ],
win_def_file = "export.def",
deps = [
":compiler_library",
@@ -69,7 +75,10 @@
hdrs = [
"register_xla.h",
],
- tags = ["noga"],
+ tags = [
+ "noga",
+ "nokokoro",
+ ],
deps = [
"//bindings/python/pyiree/common",
"//bindings/python/pyiree/compiler:compiler_library",
@@ -84,7 +93,10 @@
name = "xla_module_proto_test",
srcs = ["xla_module_proto_test.py"],
python_version = "PY3",
- tags = ["noga"],
+ tags = [
+ "noga",
+ "nokokoro",
+ ],
deps = INTREE_TENSORFLOW_PY_DEPS + NUMPY_DEPS + [
"@absl_py//absl/testing:absltest",
"//integrations/tensorflow/bindings/python/pyiree/xla/compiler",
diff --git a/integrations/tensorflow/compiler/dialect/tf_strings/conversion/convert_tf_to_tf_strings.cc b/integrations/tensorflow/compiler/dialect/tf_strings/conversion/convert_tf_to_tf_strings.cc
index 53c47d0..b9e9755 100644
--- a/integrations/tensorflow/compiler/dialect/tf_strings/conversion/convert_tf_to_tf_strings.cc
+++ b/integrations/tensorflow/compiler/dialect/tf_strings/conversion/convert_tf_to_tf_strings.cc
@@ -65,13 +65,6 @@
return tf_strings::StringType::get(type.getContext());
});
}
-
- Operation *materializeConversion(PatternRewriter &rewriter, Type resultType,
- ArrayRef<Value> inputs,
- Location loc) override {
- llvm_unreachable("unhandled materialization");
- return nullptr;
- }
};
struct StringFormatOpLowering : public OpRewritePattern<TF::StringFormatOp> {
diff --git a/integrations/tensorflow/e2e/BUILD b/integrations/tensorflow/e2e/BUILD
index 8885741..67e4c08 100644
--- a/integrations/tensorflow/e2e/BUILD
+++ b/integrations/tensorflow/e2e/BUILD
@@ -18,205 +18,37 @@
"NUMPY_DEPS",
"iree_py_test",
)
+load(
+ "//build_tools/bazel:iree_py_test_suite.bzl",
+ "iree_py_test_suite",
+)
package(
default_visibility = ["//visibility:public"],
licenses = ["notice"], # Apache 2.0
)
-[
- iree_py_test(
- name = name,
- srcs = [name + ".py"],
- python_version = "PY3",
- deps = INTREE_TENSORFLOW_PY_DEPS + NUMPY_DEPS + [
- "//integrations/tensorflow/bindings/python/pyiree/tf/support",
- ],
- )
- for name in [
- "broadcasting_test",
- "batch_norm_test",
- "fill_test",
- "control_flow_test",
- "dynamic_mlp_test",
- "dynamic_mlp_relu_test",
- "depth_conv_test",
- "exported_names_test",
- "gather_test",
- "tensorlist_test",
- "keras_lstm_test",
- "mandelbrot_test",
- "matrix_ops_test",
- "resource_ops_test",
- "ring_buffer_test",
- "sliding_window_test",
- "simple_arithmetic_test",
- "simple_stateful_test",
- "strings_test",
- ]
-]
-
-[
- iree_py_test(
- name = "_".join([
- "keras_model_train",
- optimizer_name,
- backends,
- "test",
- ]),
- srcs = [
- "keras_model_train_test.py",
- ],
- args = [
- "--optimizer_name=%s" % optimizer_name,
- "--override_backends=%s" % backends,
- ],
- main = "keras_model_train_test.py",
- python_version = "PY3",
- tags = [
- "manual",
- "noga",
- "notap",
- ],
- deps = INTREE_TENSORFLOW_PY_DEPS + NUMPY_DEPS + [
- "//integrations/tensorflow/bindings/python/pyiree/tf/support",
- ],
- )
- for optimizer_name, backends in [
- ("sgd", "tf"),
- ("sgd", "tf,iree_vmla"),
- ("adam", "tf,iree_vmla"), # TODO(b/157581521)
- ]
-]
-
-[
- iree_py_test(
- name = "_".join([
- "keras_vision_model",
- data,
- "top",
- str(include_top),
- model_name,
- backends,
- "test",
- ]),
- size = "large",
- srcs = [
- "keras_vision_model_test.py",
- ],
- args = [
- "--model=%s" % model_name,
- "--override_backends=%s" % backends,
- "--data=%s" % data,
- "--include_top=%d" % include_top,
- ],
- main = "keras_vision_model_test.py",
- python_version = "PY3",
- tags = [
- "manual",
- ],
- deps = INTREE_TENSORFLOW_PY_DEPS + NUMPY_DEPS + [
- "//integrations/tensorflow/bindings/python/pyiree/tf/support",
- ],
- )
- # ResNet50 test is hermetic - does not access any extrnal urls
- # all other tests need real weights loaded from url
- for data, include_top, model_name, backends in [
- # "cifar10" has toy models with input 32x32, is good for debugging
- ("cifar10", 1, "ResNet50", "tf,iree_vmla"),
- ("cifar10", 1, "ResNet50", "tf,iree_llvmjit"),
- ("cifar10", 1, "ResNet50", "tf,iree_vulkan"),
- ]
-]
-
-# it requres access to external URL, so these tests will be run manually
-[
- iree_py_test(
- name = "_".join([
- "keras_vision_model",
- data,
- "top",
- str(include_top),
- model_name,
- backends,
- "test",
- ]),
- srcs = [
- "keras_vision_model_test.py",
- ],
- args = [
- "--model=%s" % model_name,
- "--override_backends=%s" % backends,
- "--data=%s" % data,
- "--include_top=%d" % include_top,
- "--url=https://storage.googleapis.com/iree_models/",
- ],
- main = "keras_vision_model_test.py",
- python_version = "PY3",
- tags = [
- "external",
- "large",
- "manual",
- "noga",
- "notap",
- ],
- deps = INTREE_TENSORFLOW_PY_DEPS + NUMPY_DEPS + [
- "//integrations/tensorflow/bindings/python/pyiree/tf/support",
- ],
- )
- # "cifar10" has toy models with input 32x32, is good for debugging
- # "imagenet" has real model weights for input 224x224
- for data, include_top, model_name, backends in [
- ("cifar10", 1, "MobileNet", "tf,iree_vmla"),
- ("cifar10", 1, "MobileNet", "tf,iree_vulkan"), # TODO(b/150244105)
- ("cifar10", 1, "MobileNet", "tf,iree_llvmjit"), # TODO(b/150244105)
- ("cifar10", 1, "MobileNetV2", "tf,iree_vmla"),
- ("cifar10", 1, "MobileNetV2", "tf,iree_vulkan"), # TODO(b/150244105)
- ("cifar10", 1, "MobileNetV2", "tf,iree_llvmjit"), # TODO(b/150244105)
- ("imagenet", 1, "ResNet50", "tf,iree_vmla"),
- ("imagenet", 1, "ResNet50", "tf,iree_vulkan"),
- ("imagenet", 1, "ResNet50", "tf,iree_llvmjit"),
- ("imagenet", 1, "MobileNet", "tf,iree_vmla"),
- ("imagenet", 1, "MobileNet", "tf,iree_vulkan"), # TODO(b/150244105)
- ("imagenet", 1, "MobileNet", "tf,iree_llvmjit"), # TODO(b/150244105)
- ("imagenet", 1, "MobileNetV2", "tf,iree_vmla"),
- ("imagenet", 1, "MobileNetV2", "tf,iree_vulkan"), # TODO(b/150244105)
- ("imagenet", 1, "MobileNetV2", "tf,iree_llvmjit"), # TODO(b/150244105)
- ]
-]
-
-[
- iree_py_test(
- name = name,
- srcs = [name + ".py"],
- python_version = "PY3",
- # TODO(b/145815906) Get this running in OSS CI.
- tags = ["noga"],
- deps = INTREE_TENSORFLOW_PY_DEPS + NUMPY_DEPS + [
- "//integrations/tensorflow/bindings/python/pyiree/tf/support",
- ],
- )
- for name in [
- "conv_test",
- "linspace_test",
- "math_test",
- # TODO(GH-1620): Re-enable this after fixing the failure on
- # GitHub Actions.
- "keras_lstm_static_test",
- ]
-]
-
-# It is used to produce weights for keras vision models with input image size 32x32.
-# These models are not optimized for accuracy or latency (they are for debugging only).
-# They have the same neural net topology
-# with keras vision models trained on imagenet data sets
-py_binary(
- name = "train_vision_models_on_cifar",
- srcs = [
- "train_vision_models_on_cifar.py",
- ],
+iree_py_test(
+ # TODO(GH-2082): `linspace_test.py` fails due to an unknown location error.
+ name = "linspace_test",
+ srcs = ["linspace_test.py"],
+ main = "linspace_test.py",
python_version = "PY3",
- srcs_version = "PY2AND3",
+ tags = [
+ "noga",
+ "nokokoro",
+ ],
+ deps = INTREE_TENSORFLOW_PY_DEPS + NUMPY_DEPS + [
+ "//integrations/tensorflow/bindings/python/pyiree/tf/support",
+ ],
+)
+
+iree_py_test_suite(
+ name = "e2e",
+ srcs = glob(
+ ["*_test.py"],
+ exclude = ["linspace_test.py"],
+ ),
deps = INTREE_TENSORFLOW_PY_DEPS + NUMPY_DEPS + [
"//integrations/tensorflow/bindings/python/pyiree/tf/support",
],
diff --git a/integrations/tensorflow/e2e/keras/BUILD b/integrations/tensorflow/e2e/keras/BUILD
new file mode 100644
index 0000000..5ea9116
--- /dev/null
+++ b/integrations/tensorflow/e2e/keras/BUILD
@@ -0,0 +1,195 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load(
+ "//bindings/python:build_defs.oss.bzl",
+ "INTREE_TENSORFLOW_PY_DEPS",
+ "NUMPY_DEPS",
+ "iree_py_test",
+)
+load(
+ "//build_tools/bazel:iree_py_test_suite.bzl",
+ "iree_py_test_suite",
+)
+load(
+ "//integrations/tensorflow/e2e/keras:iree_vision_test_suite.bzl",
+ "iree_vision_test_suite",
+)
+
+package(
+ default_visibility = ["//visibility:public"],
+ licenses = ["notice"], # Apache 2.0
+)
+
+iree_py_test(
+ # TODO(GH-1620): Include after fixing the failure on GitHub Actions.
+ name = "keras_lstm_static_test",
+ srcs = ["keras_lstm_static_test.py"],
+ main = "keras_lstm_static_test.py",
+ python_version = "PY3",
+ tags = [
+ "noga",
+ "nokokoro",
+ ],
+ deps = INTREE_TENSORFLOW_PY_DEPS + NUMPY_DEPS + [
+ "//integrations/tensorflow/bindings/python/pyiree/tf/support",
+ ],
+)
+
+# @unused
+DOC = """
+keras_vision_models_test is for manual testing of all keras vision models.
+Test will run only manually with all parameters specified manually, for example:
+bazel run -c opt integrations/tensorflow/e2e/keras/keras_vision_models_test -- \
+--override_backends=tf,iree_vmla,iree_llvmjit \
+--data=imagenet \
+--include_top=1 \
+--url=https://storage.googleapis.com/iree_models/ \
+--model=ResNet50
+
+Command arguments description:
+--override_backends: can be combination of these: tf,iree_vmla,iree_llvmjit
+--data: can be 'imagenet' or 'cifar10'.
+ imagenet - input image size (1, 224, 224, 3)
+ cifar10 - input image size (1, 32, 32, 3) - it is used for quick tests
+ and needs pretrained weights, we pretrained models: ResNet50, MobileNet, MobileNetV2
+--include_top: can be 1 or 0. Include top layer 1, not include top layer 0
+--url: we need it only for cifar10 models to load weights from https://storage.googleapis.com/iree_models/
+ imagenet pretrained weights url is specified by keras
+--model: supports ResNet50, MobileNet, MobileNetV2, ResNet101, ResNet152,
+ ResNet50V2, ResNet101V2, ResNet152V2, VGG16, VGG19, Xception,
+ InceptionV3, InceptionResNetV2, DenseNet121, DenseNet169,
+ DenseNet201, NASNetMobile, NASNetLarge
+ All above models works with 'imagenet' data sets.
+ ResNet50, MobileNet, MobileNetV2 work with both 'imagenet' and 'cifar10' data sets.
+"""
+
+iree_py_test(
+ name = "keras_vision_models_test",
+ srcs = ["keras_vision_model_test.py"],
+ main = "keras_vision_model_test.py",
+ python_version = "PY3",
+ tags = [
+ "external",
+ "manual",
+ "noga",
+ "nokokoro",
+ "notap",
+ ],
+ deps = INTREE_TENSORFLOW_PY_DEPS + NUMPY_DEPS + [
+ "//integrations/tensorflow/bindings/python/pyiree/tf/support",
+ ],
+)
+
+iree_py_test_suite(
+ name = "non_vision",
+ srcs = glob(
+ ["*_test.py"],
+ exclude = [
+ "keras_vision_model_test.py",
+ "keras_lstm_static_test.py",
+ ],
+ ),
+ deps = INTREE_TENSORFLOW_PY_DEPS + NUMPY_DEPS + [
+ "//integrations/tensorflow/bindings/python/pyiree/tf/support",
+ ],
+)
+
+iree_vision_test_suite(
+ name = "keras_vision_models",
+ configurations = [
+ # tuples of (dataset, include_top, model_name, backends)
+ # "cifar10" has toy models with input 32x32, is good for debugging
+ ("cifar10", 1, "ResNet50", "tf,iree_vmla"),
+ ("cifar10", 1, "ResNet50", "tf,iree_llvmjit"),
+ ("cifar10", 1, "ResNet50", "tf,iree_vulkan"),
+ ],
+ tags = ["manual"],
+ deps = INTREE_TENSORFLOW_PY_DEPS + NUMPY_DEPS + [
+ "//integrations/tensorflow/bindings/python/pyiree/tf/support",
+ ],
+)
+
+iree_vision_test_suite(
+ name = "keras_vision_models_external",
+ configurations = [
+ # tuples of (dataset, include_top, model_name, backends)
+ # "cifar10" has toy models with 32x32 input which are good for debugging
+ # "imagenet" has real model weights for input 224x224
+ ("cifar10", 1, "MobileNet", "tf,iree_vmla"),
+ ("cifar10", 1, "MobileNetV2", "tf,iree_vmla"),
+ ("imagenet", 1, "ResNet50", "tf,iree_vmla"),
+ ("imagenet", 1, "ResNet50", "tf,iree_vulkan"),
+ ("imagenet", 1, "ResNet50", "tf,iree_llvmjit"),
+ ("imagenet", 1, "MobileNet", "tf,iree_vmla"),
+ ("imagenet", 1, "MobileNetV2", "tf,iree_vmla"),
+ ],
+ external_weights = "https://storage.googleapis.com/iree_models/",
+ tags = [
+ "external",
+ "manual",
+ "noga",
+ "nokokoro",
+ "notap",
+ ],
+ deps = INTREE_TENSORFLOW_PY_DEPS + NUMPY_DEPS + [
+ "//integrations/tensorflow/bindings/python/pyiree/tf/support",
+ ],
+)
+
+iree_vision_test_suite(
+ name = "keras_vision_models_external_failing",
+ configurations = [
+ # tuples of (dataset, include_top, model_name, backends)
+ # "cifar10" has toy models with 32x32 input which are good for debugging
+ # "imagenet" has real model weights for input 224x224
+ # TODO(b/150244105): Compiling fails with commands targeting IREE
+ # interpreter and vulkan backends for these tests.
+ # TODO: Combine this suite with keras_vision_models_external once these
+ # tests pass.
+ ("cifar10", 1, "MobileNet", "tf,iree_vulkan"),
+ ("cifar10", 1, "MobileNet", "tf,iree_llvmjit"),
+ ("cifar10", 1, "MobileNetV2", "tf,iree_vulkan"),
+ ("cifar10", 1, "MobileNetV2", "tf,iree_llvmjit"),
+ ("imagenet", 1, "MobileNet", "tf,iree_vulkan"),
+ ("imagenet", 1, "MobileNet", "tf,iree_llvmjit"),
+ ("imagenet", 1, "MobileNetV2", "tf,iree_vulkan"),
+ ("imagenet", 1, "MobileNetV2", "tf,iree_llvmjit"),
+ ],
+ external_weights = "https://storage.googleapis.com/iree_models/",
+ tags = [
+ "external",
+ "manual",
+ "noga",
+ "nokokoro",
+ "notap",
+ ],
+ deps = INTREE_TENSORFLOW_PY_DEPS + NUMPY_DEPS + [
+ "//integrations/tensorflow/bindings/python/pyiree/tf/support",
+ ],
+)
+
+# It is used to produce weights for keras vision models with input image size
+# 32x32. These models are not optimized for accuracy or latency (they are for
+# debugging only). They have the same neural net topology with keras vision
+# models trained on imagenet data sets
+py_binary(
+ name = "train_vision_models_on_cifar",
+ srcs = ["train_vision_models_on_cifar.py"],
+ python_version = "PY3",
+ srcs_version = "PY2AND3",
+ deps = INTREE_TENSORFLOW_PY_DEPS + NUMPY_DEPS + [
+ "//integrations/tensorflow/bindings/python/pyiree/tf/support",
+ ],
+)
diff --git a/integrations/tensorflow/e2e/keras/iree_vision_test_suite.bzl b/integrations/tensorflow/e2e/keras/iree_vision_test_suite.bzl
new file mode 100644
index 0000000..16d6ff8
--- /dev/null
+++ b/integrations/tensorflow/e2e/keras/iree_vision_test_suite.bzl
@@ -0,0 +1,87 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Macro for building e2e keras vision model tests."""
+
+load("//bindings/python:build_defs.oss.bzl", "iree_py_test")
+
+def iree_vision_test_suite(
+ name,
+ configurations,
+ external_weights = None,
+ deps = None,
+ tags = None,
+ size = "large",
+ python_version = "PY3",
+ **kwargs):
+ """Creates one iree_py_test per configuration tuple and a test suite that bundles them.
+
+ Args:
+ name: name of the generated test suite.
+ configurations: a list of tuples of (dataset, include_top, model,
+ backends) that specifies which data, model and backends to
+ use for a given test.
+ external_weights: a base url to fetch trained model weights from.
+ tags: tags to apply to the test. Note that as in standard test suites,
+ manual is treated specially and will also apply to the test suite
+ itself.
+ size: size of the tests.
+ python_version: the python version to run the tests with. Uses python3
+ by default.
+ **kwargs: Any additional arguments that will be passed to the underlying
+ tests and test_suite.
+ """
+ tests = []
+ for dataset, include_top, model, backends in configurations:
+ test_name = "{}_{}_top_{}_{}_{}_test".format(
+ name,
+ dataset,
+ include_top,
+ model,
+ backends.replace(",", "_"),
+ )
+ tests.append(test_name)
+
+ args = [
+ "--data={}".format(dataset),
+ "--include_top={}".format(include_top),
+ "--model={}".format(model),
+ "--override_backends={}".format(backends),
+ ]
+ if external_weights:
+ args.append("--url={}".format(external_weights))
+
+ iree_py_test(
+ name = test_name,
+ main = "keras_vision_model_test.py",
+ srcs = ["keras_vision_model_test.py"],
+ args = args,
+ tags = tags,
+ deps = deps,
+ size = size,
+ python_version = python_version,
+ **kwargs
+ )
+
+ native.test_suite(
+ name = name,
+ tests = tests,
+ # Note that only the manual tag really has any effect here. Others are
+ # used for test suite filtering, but all tests are passed the same tags.
+ tags = tags,
+ # If there are kwargs that need to be passed here which only apply to
+ # the generated tests and not to test_suite, they should be extracted
+ # into separate named arguments.
+ **kwargs
+ )
diff --git a/integrations/tensorflow/e2e/keras_lstm_static_test.py b/integrations/tensorflow/e2e/keras/keras_lstm_static_test.py
similarity index 100%
rename from integrations/tensorflow/e2e/keras_lstm_static_test.py
rename to integrations/tensorflow/e2e/keras/keras_lstm_static_test.py
diff --git a/integrations/tensorflow/e2e/keras_lstm_test.py b/integrations/tensorflow/e2e/keras/keras_lstm_test.py
similarity index 100%
rename from integrations/tensorflow/e2e/keras_lstm_test.py
rename to integrations/tensorflow/e2e/keras/keras_lstm_test.py
diff --git a/integrations/tensorflow/e2e/keras_vision_model_test.py b/integrations/tensorflow/e2e/keras/keras_vision_model_test.py
similarity index 92%
rename from integrations/tensorflow/e2e/keras_vision_model_test.py
rename to integrations/tensorflow/e2e/keras/keras_vision_model_test.py
index 3105ef1..9164d9b 100644
--- a/integrations/tensorflow/e2e/keras_vision_model_test.py
+++ b/integrations/tensorflow/e2e/keras/keras_vision_model_test.py
@@ -71,9 +71,15 @@
}
-def get_input_shape(data):
+def get_input_shape(data, model):
if data == 'imagenet':
- return (1, 224, 224, 3)
+ if (model == 'InceptionV3' or model == 'Xception' or
+ model == 'InceptionResNetV2'):
+ return (1, 299, 299, 3)
+ elif model == 'NASNetLarge':
+ return (1, 331, 331, 3)
+ else:
+ return (1, 224, 224, 3)
elif data == 'cifar10':
return (1, 32, 32, 3)
else:
@@ -84,7 +90,7 @@
tf.keras.backend.set_learning_phase(False)
tf_test_utils.set_random_seed()
- input_shape = get_input_shape(FLAGS.data)
+ input_shape = get_input_shape(FLAGS.data, FLAGS.model)
# keras model receives images size as input,
# where batch size is not specified - by default it is dynamic
if FLAGS.model in APP_MODELS:
@@ -125,7 +131,7 @@
class AppTest(tf_test_utils.SavedModelTestCase):
def test_application(self):
- input_shape = get_input_shape(FLAGS.data)
+ input_shape = get_input_shape(FLAGS.data, FLAGS.model)
input_data = np.random.rand(np.prod(np.array(input_shape))).astype(
np.float32)
input_data = input_data.reshape(input_shape)
diff --git a/integrations/tensorflow/e2e/keras/train/BUILD b/integrations/tensorflow/e2e/keras/train/BUILD
new file mode 100644
index 0000000..58d3922
--- /dev/null
+++ b/integrations/tensorflow/e2e/keras/train/BUILD
@@ -0,0 +1,64 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load(
+ "//bindings/python:build_defs.oss.bzl",
+ "INTREE_TENSORFLOW_PY_DEPS",
+ "NUMPY_DEPS",
+)
+load(
+ "//integrations/tensorflow/e2e/keras/train:iree_train_test_suite.bzl",
+ "iree_train_test_suite",
+)
+
+package(
+ default_visibility = ["//visibility:public"],
+ licenses = ["notice"], # Apache 2.0
+)
+
+iree_train_test_suite(
+ name = "keras_model_train",
+ configurations = [
+ # tuples of (optimizer, backends)
+ ("sgd", "tf"),
+ ],
+ tags = [
+ "manual",
+ "noga",
+ "nokokoro",
+ "notap",
+ ],
+ deps = INTREE_TENSORFLOW_PY_DEPS + NUMPY_DEPS + [
+ "//integrations/tensorflow/bindings/python/pyiree/tf/support",
+ ],
+)
+
+iree_train_test_suite(
+ name = "keras_model_train_failing",
+ configurations = [
+ # tuples of (optimizer, backends)
+ # TODO: Combine this suite with keras_model_train once these tests pass.
+ ("sgd", "tf,iree_vmla"),
+ ("adam", "tf,iree_vmla"), # TODO(b/157581521)
+ ],
+ tags = [
+ "manual",
+ "noga",
+ "nokokoro",
+ "notap",
+ ],
+ deps = INTREE_TENSORFLOW_PY_DEPS + NUMPY_DEPS + [
+ "//integrations/tensorflow/bindings/python/pyiree/tf/support",
+ ],
+)
diff --git a/integrations/tensorflow/e2e/keras/train/iree_train_test_suite.bzl b/integrations/tensorflow/e2e/keras/train/iree_train_test_suite.bzl
new file mode 100644
index 0000000..1150db6
--- /dev/null
+++ b/integrations/tensorflow/e2e/keras/train/iree_train_test_suite.bzl
@@ -0,0 +1,73 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Macro for building e2e keras vision model tests."""
+
+load("//bindings/python:build_defs.oss.bzl", "iree_py_test")
+
+def iree_train_test_suite(
+ name,
+ configurations,
+ deps = None,
+ tags = None,
+ size = None,
+ python_version = "PY3",
+ **kwargs):
+ """Creates one iree_py_test per configuration tuple and a test suite that bundles them.
+
+ Args:
+ name: name of the generated test suite.
+ configurations: a list of tuples of (optimizer, backends).
+ tags: tags to apply to the test. Note that as in standard test suites,
+ manual is treated specially and will also apply to the test suite
+ itself.
+ size: size of the tests.
+ python_version: the python version to run the tests with. Uses python3
+ by default.
+ **kwargs: Any additional arguments that will be passed to the underlying
+ tests and test_suite.
+ """
+ tests = []
+ for optimizer, backends in configurations:
+ test_name = "{}_{}_{}_test".format(name, optimizer, backends)
+ tests.append(test_name)
+
+ args = [
+ "--optimizer_name={}".format(optimizer),
+ "--override_backends={}".format(backends),
+ ]
+
+ iree_py_test(
+ name = test_name,
+ main = "keras_model_train_test.py",
+ srcs = ["keras_model_train_test.py"],
+ args = args,
+ tags = tags,
+ deps = deps,
+ size = size,
+ python_version = python_version,
+ **kwargs
+ )
+
+ native.test_suite(
+ name = name,
+ tests = tests,
+ # Note that only the manual tag really has any effect here. Others are
+ # used for test suite filtering, but all tests are passed the same tags.
+ tags = tags,
+ # If there are kwargs that need to be passed here which only apply to
+ # the generated tests and not to test_suite, they should be extracted
+ # into separate named arguments.
+ **kwargs
+ )
diff --git a/integrations/tensorflow/e2e/keras_model_train_test.py b/integrations/tensorflow/e2e/keras/train/keras_model_train_test.py
similarity index 100%
rename from integrations/tensorflow/e2e/keras_model_train_test.py
rename to integrations/tensorflow/e2e/keras/train/keras_model_train_test.py
diff --git a/integrations/tensorflow/e2e/train_vision_models_on_cifar.py b/integrations/tensorflow/e2e/keras/train_vision_models_on_cifar.py
similarity index 100%
rename from integrations/tensorflow/e2e/train_vision_models_on_cifar.py
rename to integrations/tensorflow/e2e/keras/train_vision_models_on_cifar.py
diff --git a/iree/compiler/Dialect/Flow/Conversion/TypeConverter.cpp b/iree/compiler/Dialect/Flow/Conversion/TypeConverter.cpp
index 5ca37eb..9ba2da8 100644
--- a/iree/compiler/Dialect/Flow/Conversion/TypeConverter.cpp
+++ b/iree/compiler/Dialect/Flow/Conversion/TypeConverter.cpp
@@ -56,15 +56,7 @@
// here for certain cases (such as * on the LHS).
return Type();
});
-}
-
-Operation *FlowTypeConverter::materializeConversion(PatternRewriter &rewriter,
- Type resultType,
- ArrayRef<Value> inputs,
- Location loc) {
- // TODO(b/145876978): materialize conversion when this is called.
- llvm_unreachable("unhandled materialization");
- return nullptr;
+ // TODO(b/145876978): add conversion materializer
}
} // namespace iree_compiler
diff --git a/iree/compiler/Dialect/Flow/Conversion/TypeConverter.h b/iree/compiler/Dialect/Flow/Conversion/TypeConverter.h
index b117aea..a980a49 100644
--- a/iree/compiler/Dialect/Flow/Conversion/TypeConverter.h
+++ b/iree/compiler/Dialect/Flow/Conversion/TypeConverter.h
@@ -23,10 +23,6 @@
class FlowTypeConverter : public TypeConverter {
public:
FlowTypeConverter();
-
- Operation *materializeConversion(PatternRewriter &rewriter, Type resultType,
- ArrayRef<Value> inputs,
- Location loc) override;
};
} // namespace iree_compiler
diff --git a/iree/compiler/Dialect/VM/Conversion/TypeConverter.cpp b/iree/compiler/Dialect/VM/Conversion/TypeConverter.cpp
index c18ecdc..668442f 100644
--- a/iree/compiler/Dialect/VM/Conversion/TypeConverter.cpp
+++ b/iree/compiler/Dialect/VM/Conversion/TypeConverter.cpp
@@ -78,20 +78,16 @@
}
return success();
});
-}
-Operation *VMTypeConverter::materializeConversion(PatternRewriter &rewriter,
- Type resultType,
- ArrayRef<Value> inputs,
- Location loc) {
- LLVM_DEBUG(llvm::dbgs() << "MATERIALIZE CONVERSION: " << resultType << "\n");
- if (auto rsType = resultType.dyn_cast<Shape::RankedShapeType>()) {
- return rewriter.create<Shape::MakeRankedShapeOp>(loc, rsType, inputs);
- }
-
- // TODO(b/145876978): materialize conversion when this is called.
- llvm_unreachable("unhandled materialization");
- return nullptr;
+ // TODO(b/145876978): materialize conversion for other types
+ addMaterialization([](PatternRewriter &rewriter,
+ Shape::RankedShapeType resultType, ValueRange inputs,
+ Location loc) -> Optional<Value> {
+ LLVM_DEBUG(llvm::dbgs()
+ << "MATERIALIZE CONVERSION: " << resultType << "\n");
+ return rewriter.create<Shape::MakeRankedShapeOp>(loc, resultType, inputs)
+ .getResult();
+ });
}
} // namespace iree_compiler
diff --git a/iree/compiler/Dialect/VM/Conversion/TypeConverter.h b/iree/compiler/Dialect/VM/Conversion/TypeConverter.h
index 9e5d646..411ac33 100644
--- a/iree/compiler/Dialect/VM/Conversion/TypeConverter.h
+++ b/iree/compiler/Dialect/VM/Conversion/TypeConverter.h
@@ -23,10 +23,6 @@
class VMTypeConverter : public TypeConverter {
public:
VMTypeConverter();
-
- Operation *materializeConversion(PatternRewriter &rewriter, Type resultType,
- ArrayRef<Value> inputs,
- Location loc) override;
};
} // namespace iree_compiler
diff --git a/iree/tools/benchmark_module_main.cc b/iree/tools/benchmark_module_main.cc
index 3c7c4a1..616c78a 100644
--- a/iree/tools/benchmark_module_main.cc
+++ b/iree/tools/benchmark_module_main.cc
@@ -149,11 +149,20 @@
CHECK_OK(Run(state));
}
-// By default only the main thread is included in CPU time. Include all the
-// threads instead. To make single and multi-threaded benchmarks more
-// comparable, use the wall time to determine how many iterations to run.
-// See https://github.com/google/benchmark#cpu-timers,
-BENCHMARK(BM_RunModule)->MeasureProcessCPUTime()->UseRealTime();
+BENCHMARK(BM_RunModule)
+ // By default only the main thread is included in CPU time. Include all the
+ // threads instead.
+ ->MeasureProcessCPUTime()
+ // To make single and multi-threaded benchmarks more comparable, use the
+ // wall time to determine how many iterations to run.
+ // See https://github.com/google/benchmark#cpu-timers,
+ ->UseRealTime()
+ // Report timing in milliseconds, which is the general order of magnitude of
+ // model runs. The benchmark framework will print with precision between 0
+ // and 3 places after the decimal while aiming for three significant digits.
+ // If we end up wanting precision beyond microseconds, we can make this
+ // setting configurable with a custom command line flag.
+ ->Unit(benchmark::kMillisecond);
} // namespace
diff --git a/iree/tools/init_mlir_passes.h b/iree/tools/init_mlir_passes.h
index e2bb850..df3b037 100644
--- a/iree/tools/init_mlir_passes.h
+++ b/iree/tools/init_mlir_passes.h
@@ -124,11 +124,6 @@
createSymbolDCEPass();
createLocationSnapshotPass({});
- // GPU
- createGpuKernelOutliningPass();
- createSimpleSCFToGPUPass(0, 0);
- createLoopToGPUPass({}, {});
-
// Linalg
createLinalgFusionPass();
::mlir::registerPass("linalg-fusion-for-tensor-ops",
diff --git a/kokoro/gcp_ubuntu/bazel/bindings/build.sh b/kokoro/gcp_ubuntu/bazel/bindings/build.sh
new file mode 100755
index 0000000..93a12f3
--- /dev/null
+++ b/kokoro/gcp_ubuntu/bazel/bindings/build.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# For use within a IREE bazel-tensorflow docker image on a Kokoro VM.
+# Log some information about the environment, initialize the submodules and then
+# run the bazel integrations tests.
+
+set -e
+set -x
+
+# Print the UTC time when set -x is on
+export PS4='[$(date -u "+%T %Z")] '
+
+# Check these exist and print the versions for later debugging
+bazel --version
+"$CXX" --version
+"$CC" --version
+"$PYTHON_BIN" -V
+# TODO( #1875 ): Make PYTHON_BIN also control the runtime version
+python3 -V
+
+echo "Initializing submodules"
+./scripts/git/submodule_versions.py init
+
+echo "Building and testing with bazel"
+./build_tools/bazel/build_bindings.sh
diff --git a/kokoro/gcp_ubuntu/bazel/bindings/build_kokoro.sh b/kokoro/gcp_ubuntu/bazel/bindings/build_kokoro.sh
new file mode 100644
index 0000000..304a964
--- /dev/null
+++ b/kokoro/gcp_ubuntu/bazel/bindings/build_kokoro.sh
@@ -0,0 +1,41 @@
+#!/bin/bash
+
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Build and test the project within the gcr.io/iree-oss/bazel-tensorflow
+# image using Kokoro.
+
+set -e
+set -x
+
+# Print the UTC time when set -x is on
+export PS4='[$(date -u "+%T %Z")] '
+
+# Kokoro checks out the repository here.
+WORKDIR="${KOKORO_ARTIFACTS_DIR?}/github/iree"
+
+# Mount the checked out repository, make that the working directory and run the
+# tests in the bazel-tensorflow image.
+# TODO: Change image to gcr.io/iree-oss/bazel-bindings:prod once we create it.
+docker run \
+ --volume "${WORKDIR?}:${WORKDIR?}" \
+ --workdir="${WORKDIR?}" \
+ --rm \
+ gcr.io/iree-oss/bazel-tensorflow:prod \
+ kokoro/gcp_ubuntu/bazel/bindings/build.sh
+
+# Kokoro will rsync this entire directory back to the executor orchestrating the
+# build which takes forever and is totally useless.
+rm -rf "${KOKORO_ARTIFACTS_DIR?}/*"
diff --git a/kokoro/gcp_ubuntu/bazel/bindings/common.cfg b/kokoro/gcp_ubuntu/bazel/bindings/common.cfg
new file mode 100644
index 0000000..f53f602
--- /dev/null
+++ b/kokoro/gcp_ubuntu/bazel/bindings/common.cfg
@@ -0,0 +1,19 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Common configuration for Kokoro builds that run bazel on linux.
+
+build_file: "iree/kokoro/gcp_ubuntu/bazel/bindings/build_kokoro.sh"
diff --git a/kokoro/gcp_ubuntu/bazel/bindings/continuous.cfg b/kokoro/gcp_ubuntu/bazel/bindings/continuous.cfg
new file mode 100644
index 0000000..af84216
--- /dev/null
+++ b/kokoro/gcp_ubuntu/bazel/bindings/continuous.cfg
@@ -0,0 +1,19 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Deliberately blank as everything necessary is configured in common files, but
+# file must still exist to match corresponding (upstream only) job
+# configurations that trigger the builds.
diff --git a/kokoro/gcp_ubuntu/bazel/bindings/presubmit.cfg b/kokoro/gcp_ubuntu/bazel/bindings/presubmit.cfg
new file mode 100644
index 0000000..af84216
--- /dev/null
+++ b/kokoro/gcp_ubuntu/bazel/bindings/presubmit.cfg
@@ -0,0 +1,19 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Deliberately blank as everything necessary is configured in common files, but
+# file must still exist to match corresponding (upstream only) job
+# configurations that trigger the builds.
diff --git a/kokoro/gcp_ubuntu/bazel/build.sh b/kokoro/gcp_ubuntu/bazel/build.sh
index f3d5316..d8c8ece 100755
--- a/kokoro/gcp_ubuntu/bazel/build.sh
+++ b/kokoro/gcp_ubuntu/bazel/build.sh
@@ -36,4 +36,4 @@
./scripts/git/submodule_versions.py init
echo "Building and testing with bazel"
-./build_tools/bazel/build.sh
+./build_tools/bazel/build_core.sh
diff --git a/kokoro/gcp_ubuntu/bazel/integrations/build.sh b/kokoro/gcp_ubuntu/bazel/integrations/build.sh
new file mode 100755
index 0000000..3b438b5
--- /dev/null
+++ b/kokoro/gcp_ubuntu/bazel/integrations/build.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# For use within a IREE bazel-tensorflow docker image on a Kokoro VM.
+# Log some information about the environment, initialize the submodules and then
+# run the bazel integrations tests.
+
+set -e
+set -x
+
+# Print the UTC time when set -x is on
+export PS4='[$(date -u "+%T %Z")] '
+
+# Check these exist and print the versions for later debugging
+bazel --version
+"$CXX" --version
+"$CC" --version
+"$PYTHON_BIN" -V
+# TODO( #1875 ): Make PYTHON_BIN also control the runtime version
+python3 -V
+
+echo "Initializing submodules"
+./scripts/git/submodule_versions.py init
+
+echo "Building and testing with bazel"
+./build_tools/bazel/build_tensorflow.sh
diff --git a/kokoro/gcp_ubuntu/bazel/integrations/build_kokoro.sh b/kokoro/gcp_ubuntu/bazel/integrations/build_kokoro.sh
new file mode 100644
index 0000000..3010339
--- /dev/null
+++ b/kokoro/gcp_ubuntu/bazel/integrations/build_kokoro.sh
@@ -0,0 +1,40 @@
+#!/bin/bash
+
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Build and test the project within the gcr.io/iree-oss/bazel-tensorflow
+# image using Kokoro.
+
+set -e
+set -x
+
+# Print the UTC time when set -x is on
+export PS4='[$(date -u "+%T %Z")] '
+
+# Kokoro checks out the repository here.
+WORKDIR="${KOKORO_ARTIFACTS_DIR?}/github/iree"
+
+# Mount the checked out repository, make that the working directory and run the
+# tests in the bazel-tensorflow image.
+docker run \
+ --volume "${WORKDIR?}:${WORKDIR?}" \
+ --workdir="${WORKDIR?}" \
+ --rm \
+ gcr.io/iree-oss/bazel-tensorflow:prod \
+ kokoro/gcp_ubuntu/bazel/integrations/build.sh
+
+# Kokoro will rsync this entire directory back to the executor orchestrating the
+# build which takes forever and is totally useless.
+rm -rf "${KOKORO_ARTIFACTS_DIR?}/*"
diff --git a/kokoro/gcp_ubuntu/bazel/integrations/common.cfg b/kokoro/gcp_ubuntu/bazel/integrations/common.cfg
new file mode 100644
index 0000000..8594dc7
--- /dev/null
+++ b/kokoro/gcp_ubuntu/bazel/integrations/common.cfg
@@ -0,0 +1,19 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Common configuration for Kokoro builds that run bazel on linux.
+
+build_file: "iree/kokoro/gcp_ubuntu/bazel/integrations/build_kokoro.sh"
diff --git a/kokoro/gcp_ubuntu/bazel/integrations/continuous.cfg b/kokoro/gcp_ubuntu/bazel/integrations/continuous.cfg
new file mode 100644
index 0000000..af84216
--- /dev/null
+++ b/kokoro/gcp_ubuntu/bazel/integrations/continuous.cfg
@@ -0,0 +1,19 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Deliberately blank as everything necessary is configured in common files, but
+# file must still exist to match corresponding (upstream only) job
+# configurations that trigger the builds.
diff --git a/kokoro/gcp_ubuntu/bazel/integrations/presubmit.cfg b/kokoro/gcp_ubuntu/bazel/integrations/presubmit.cfg
new file mode 100644
index 0000000..af84216
--- /dev/null
+++ b/kokoro/gcp_ubuntu/bazel/integrations/presubmit.cfg
@@ -0,0 +1,19 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Deliberately blank as everything necessary is configured in common files, but
+# file must still exist to match corresponding (upstream only) job
+# configurations that trigger the builds.
diff --git a/scripts/git/update_tf_llvm_submodules.py b/scripts/git/update_tf_llvm_submodules.py
index 9fdd028..9ca4e84 100755
--- a/scripts/git/update_tf_llvm_submodules.py
+++ b/scripts/git/update_tf_llvm_submodules.py
@@ -63,7 +63,8 @@
default="TENSORFLOW")
parser.add_argument(
"--update_build_files",
- help="Updates the IREE LLVM build files from TensorFlow",
+ help=("Updates the IREE LLVM build files from TensorFlow."
+ "Defaults to True iff llvm_commit==TENSORFLOW"),
type=utils.str2bool,
nargs="?",
default=None)
diff --git a/third_party/llvm-project b/third_party/llvm-project
index a8ca0ec..df06f4f 160000
--- a/third_party/llvm-project
+++ b/third_party/llvm-project
@@ -1 +1 @@
-Subproject commit a8ca0ec267050f9ded865a729d50c2c0eb078b7e
+Subproject commit df06f4ff227bdcbbad01b418199876761f2a1ff0
diff --git a/third_party/mlir-emitc b/third_party/mlir-emitc
new file mode 160000
index 0000000..53265de
--- /dev/null
+++ b/third_party/mlir-emitc
@@ -0,0 +1 @@
+Subproject commit 53265de20088f8b2af381c77173b22764cbfc038
diff --git a/third_party/tensorflow b/third_party/tensorflow
index f653ab8..0836e5e 160000
--- a/third_party/tensorflow
+++ b/third_party/tensorflow
@@ -1 +1 @@
-Subproject commit f653ab8bb3910959c8189f163e5626df61ff0eab
+Subproject commit 0836e5e4ea18b81e43cae801307d642d82548b29
