Merge remote-tracking branch 'origin/main' into bazel-gpu
diff --git a/build_tools/docker/bazel_nvidia/Dockerfile b/build_tools/docker/bazel_nvidia/Dockerfile new file mode 100644 index 0000000..86305c4 --- /dev/null +++ b/build_tools/docker/bazel_nvidia/Dockerfile
@@ -0,0 +1,46 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# An image for building IREE's tensorflow integrations using bazel and vulkan. + +# Build using: +# docker build --tag gcr.io/iree-oss/bazel-nvidia \ +# build_tools/docker/bazel_nvidia/ + +# Run interactively using the following, where IREE_WORKDIR is the path to your +# local dev environment: +# docker run -it --rm --entrypoint bash --volume ${IREE_WORKDIR}:/usr/src/iree/ \ +# gcr.io/iree-oss/bazel-nvidia + +# Set up the image and working directory. +# We start from bazel-nvidia so this image can be used to testing TensorFlow +# integrations. +FROM gcr.io/iree-oss/bazel-tensorflow + +# Additionally, we need to install the Vulkan SDK and the NVIDIA Vulkan driver. + +ARG VULKAN_SDK_VERSION=1.2.141 + +# Disable apt-key parse waring. If someone knows how to do whatever the "proper" +# thing is then feel free. The warning complains about parsing apt-key output, +# which we're not even doing. +ARG APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 + +RUN wget -qO - http://packages.lunarg.com/lunarg-signing-key-pub.asc \ + | apt-key add - \ + && wget -qO \ + "/etc/apt/sources.list.d/lunarg-vulkan-${VULKAN_SDK_VERSION?}-bionic.list" \ + "http://packages.lunarg.com/vulkan/${VULKAN_SDK_VERSION?}/lunarg-vulkan-${VULKAN_SDK_VERSION?}-bionic.list" \ + && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y vulkan-sdk nvidia-driver-440
diff --git a/build_tools/docker/build_and_update_gcr.py b/build_tools/docker/build_and_update_gcr.py index cba4593..6de59a4 100755 --- a/build_tools/docker/build_and_update_gcr.py +++ b/build_tools/docker/build_and_update_gcr.py
@@ -33,6 +33,7 @@ 'bazel': [], 'bazel-bindings': ['bazel'], 'bazel-tensorflow': ['bazel-bindings'], + 'bazel-nvidia': ['bazel-tensorflow'], 'bazel-swiftshader': ['bazel-tensorflow'], 'cmake': [], 'cmake-android': ['cmake'],
diff --git a/build_tools/kokoro/gcp_ubuntu/bazel/linux/x86-turing/integrations/build.sh b/build_tools/kokoro/gcp_ubuntu/bazel/linux/x86-turing/integrations/build.sh new file mode 100755 index 0000000..51b491d --- /dev/null +++ b/build_tools/kokoro/gcp_ubuntu/bazel/linux/x86-turing/integrations/build.sh
@@ -0,0 +1,43 @@ +#!/bin/bash + +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# For use within a IREE bazel-nvidia docker image on a Kokoro VM. +# Log some information about the environment, initialize the submodules and then +# run the bazel integrations tests. + +set -e +set -x + +# Print the UTC time when set -x is on +export PS4='[$(date -u "+%T %Z")] ' + +# Check these exist and print the versions for later debugging +bazel --version +"$CXX" --version +"$CC" --version +"$PYTHON_BIN" -V +# TODO(#1875): Make PYTHON_BIN also control the runtime version +python3 -V + +# Print Vulkan related information: SDK version and GPU ICD version +vulkaninfo 2>/dev/null | grep "Vulkan Instance" || echo "Vulkan Instance not found!" +vulkaninfo 2>/dev/null | grep -A7 "VkPhysicalDeviceProperties" || echo "VkPhysicalDeviceProperties not found!" + +echo "Initializing submodules" +./scripts/git/submodule_versions.py init + +echo "Building and testing with bazel" +./build_tools/bazel/build_tensorflow.sh
diff --git a/build_tools/kokoro/gcp_ubuntu/bazel/linux/x86-turing/integrations/build_kokoro.sh b/build_tools/kokoro/gcp_ubuntu/bazel/linux/x86-turing/integrations/build_kokoro.sh new file mode 100755 index 0000000..0712a66 --- /dev/null +++ b/build_tools/kokoro/gcp_ubuntu/bazel/linux/x86-turing/integrations/build_kokoro.sh
@@ -0,0 +1,45 @@ +#!/bin/bash + +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Build and test IREE's integrations within the gcr.io/iree-oss/bazel-nvidia +# image using Kokoro. + +set -e +set -x + +# Print the UTC time when set -x is on +export PS4='[$(date -u "+%T %Z")] ' + +# Kokoro checks out the repository here. +WORKDIR="${KOKORO_ARTIFACTS_DIR?}/github/iree" + +# Mount the checked out repository, make that the working directory and run the +# tests in the bazel-tensorflow image. +docker run \ + --volume "${WORKDIR?}:${WORKDIR?}" \ + --workdir="${WORKDIR?}" \ + --rm \ + --env IREE_VULKAN_DISABLE=0 \ + --gpus all \ + gcr.io/iree-oss/bazel-nvidia:prod \ + build_tools/kokoro/gcp_ubuntu/bazel/linux/x86-turing/integrations/build.sh + +# Kokoro will rsync this entire directory back to the executor orchestrating the +# build which takes forever and is totally useless. +sudo rm -rf "${KOKORO_ARTIFACTS_DIR?}"/* + +# Print out artifacts dir contents after deleting them as a coherence check. +ls -1a "${KOKORO_ARTIFACTS_DIR?}/"
diff --git a/build_tools/kokoro/gcp_ubuntu/bazel/linux/x86-turing/integrations/common.cfg b/build_tools/kokoro/gcp_ubuntu/bazel/linux/x86-turing/integrations/common.cfg new file mode 100644 index 0000000..eb09ca3 --- /dev/null +++ b/build_tools/kokoro/gcp_ubuntu/bazel/linux/x86-turing/integrations/common.cfg
@@ -0,0 +1,17 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +build_file: "iree/build_tools/kokoro/gcp_ubuntu/bazel/linux/x86-turing/integrations/build_kokoro.sh"
diff --git a/build_tools/kokoro/gcp_ubuntu/bazel/linux/x86-turing/integrations/google.cfg b/build_tools/kokoro/gcp_ubuntu/bazel/linux/x86-turing/integrations/google.cfg new file mode 100644 index 0000000..50a7eed --- /dev/null +++ b/build_tools/kokoro/gcp_ubuntu/bazel/linux/x86-turing/integrations/google.cfg
@@ -0,0 +1,19 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Deliberately blank as everything necessary is configured in common files, but +# file must still exist to match corresponding (Google internal) job +# configurations that trigger the builds.
diff --git a/build_tools/kokoro/gcp_ubuntu/bazel/linux/x86-turing/integrations/main.cfg b/build_tools/kokoro/gcp_ubuntu/bazel/linux/x86-turing/integrations/main.cfg new file mode 100644 index 0000000..50a7eed --- /dev/null +++ b/build_tools/kokoro/gcp_ubuntu/bazel/linux/x86-turing/integrations/main.cfg
@@ -0,0 +1,19 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Deliberately blank as everything necessary is configured in common files, but +# file must still exist to match corresponding (Google internal) job +# configurations that trigger the builds.
diff --git a/build_tools/kokoro/gcp_ubuntu/bazel/linux/x86-turing/integrations/presubmit.cfg b/build_tools/kokoro/gcp_ubuntu/bazel/linux/x86-turing/integrations/presubmit.cfg new file mode 100644 index 0000000..50a7eed --- /dev/null +++ b/build_tools/kokoro/gcp_ubuntu/bazel/linux/x86-turing/integrations/presubmit.cfg
@@ -0,0 +1,19 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Deliberately blank as everything necessary is configured in common files, but +# file must still exist to match corresponding (Google internal) job +# configurations that trigger the builds.
diff --git a/integrations/tensorflow/e2e/BUILD b/integrations/tensorflow/e2e/BUILD index 2fd4545..d63b839 100644 --- a/integrations/tensorflow/e2e/BUILD +++ b/integrations/tensorflow/e2e/BUILD
@@ -93,6 +93,7 @@ "matrix_ops_test.py", "ring_buffer_test.py", # TODO(b/148747011) "scatter_update_test.py", + "sliding_window_test.py", # TODO(#2659) "strings_test.py", ]