Add cmake-bazel-tensorflow images for integrations (#4176)
- Add `cmake-bazel-tensorflow*` images to build and test the new tensorflow integrations.
- Tested that these build and test locally.
- Add `dry_run` arg to `get_repo_digests` to allow `dry_run` to work on non-pushed images.
diff --git a/build_tools/docker/README.md b/build_tools/docker/README.md
index 2bb7d4e..07ce7d8 100644
--- a/build_tools/docker/README.md
+++ b/build_tools/docker/README.md
@@ -74,7 +74,9 @@
1. Update the `Dockerfile` for the image that you want to modify or add. If
you're adding a new image, or updating the dependencies between images, be
- sure to update `IMAGES_TO_DEPENDENCIES` in `manage_images.py` as well.
+ sure to update `IMAGES_TO_DEPENDENCIES` in `manage_images.py` as well. If
+ you are adding new images, it is best add them via `git add` before
+ proceeding.
2. Build the image, push the image to GCR and update all references to the image
with the new GCR digest:
diff --git a/build_tools/docker/cmake-bazel-tensorflow-nvidia/Dockerfile b/build_tools/docker/cmake-bazel-tensorflow-nvidia/Dockerfile
new file mode 100644
index 0000000..b05548d
--- /dev/null
+++ b/build_tools/docker/cmake-bazel-tensorflow-nvidia/Dockerfile
@@ -0,0 +1,29 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# To use the host GPUs, `docker run` must be called with the `--gpus all` flag.
+
+# Set up the image and working directory by inheriting the vulkan CMake
+# configuration.
+# Note that we don't start from NVIDIA's docker base:
+# - nvidia/cuda (https://hub.docker.com/r/nvidia/cuda):
+# it's.. for CUDA.
+# - nvidia/vulkan (https://hub.docker.com/r/nvidia/vulkan):
+# does not support Ubuntu 18.04.
+# This allows to share configuration with base CMake, but it also means we need
+# to MATCH the driver version between the host machine and the docker image.
+FROM gcr.io/iree-oss/cmake-bazel-tensorflow-vulkan@sha256:f7ebc07ccfe03e6bb7de11e178583b43660f9fb152f4154a02b44fbb1b42cd4f AS final
+
+RUN apt-get update \
+ && DEBIAN_FRONTEND=noninteractive apt-get install -y libnvidia-gl-440=440.100-0ubuntu0.18.04.1
diff --git a/build_tools/docker/cmake-bazel-tensorflow-swiftshader/Dockerfile b/build_tools/docker/cmake-bazel-tensorflow-swiftshader/Dockerfile
new file mode 100644
index 0000000..fe52b89
--- /dev/null
+++ b/build_tools/docker/cmake-bazel-tensorflow-swiftshader/Dockerfile
@@ -0,0 +1,20 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM gcr.io/iree-oss/cmake-bazel-tensorflow-vulkan@sha256:f7ebc07ccfe03e6bb7de11e178583b43660f9fb152f4154a02b44fbb1b42cd4f AS final
+COPY --from=gcr.io/iree-oss/swiftshader@sha256:ccae32c83c89a31e8fc5542e480c29f28bbf4a3b3b80198c06b687a92c6813f3 \
+ /swiftshader /swiftshader
+
+# Set VK_ICD_FILENAMES so Vulkan loader can find the SwiftShader ICD.
+ENV VK_ICD_FILENAMES /swiftshader/vk_swiftshader_icd.json
diff --git a/build_tools/docker/cmake-bazel-tensorflow-vulkan/Dockerfile b/build_tools/docker/cmake-bazel-tensorflow-vulkan/Dockerfile
new file mode 100644
index 0000000..09b9559
--- /dev/null
+++ b/build_tools/docker/cmake-bazel-tensorflow-vulkan/Dockerfile
@@ -0,0 +1,33 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This image provides the Vulkan SDK. Requires a child image to provide
+# a Vulkan ICD.
+
+FROM gcr.io/iree-oss/cmake-bazel-tensorflow@sha256:41963cf49ecd8873d2349f9f9b3a0af86471614a4ba6916b3be5bbaff5e28b3b AS final
+
+ARG VULKAN_SDK_VERSION=1.2.154.0
+
+COPY --from=gcr.io/iree-oss/vulkan@sha256:5812ee64806a7f3df0739ccf0930c27cabce346901488eceb1ee66c9c0a5ae96 \
+ /opt/vulkan-sdk/ /opt/vulkan-sdk/
+
+ENV VULKAN_SDK="/opt/vulkan-sdk/${VULKAN_SDK_VERSION}/x86_64"
+
+ENV PATH="${VULKAN_SDK}/bin:$PATH"
+
+# Symlink the Vulkan loader to a system library directory. This is needed to
+# allow Vulkan applications to find the Vulkan loader. It also avoids using
+# LD_LIBRARY_PATH, which is not supported well by Docker.
+RUN ln -s "${VULKAN_SDK}/lib/libvulkan.so" /usr/lib/x86_64-linux-gnu/ \
+ && ln -s "${VULKAN_SDK}/lib/libvulkan.so.1" /usr/lib/x86_64-linux-gnu/
diff --git a/build_tools/docker/cmake-bazel-tensorflow/Dockerfile b/build_tools/docker/cmake-bazel-tensorflow/Dockerfile
new file mode 100644
index 0000000..209b196
--- /dev/null
+++ b/build_tools/docker/cmake-bazel-tensorflow/Dockerfile
@@ -0,0 +1,23 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM gcr.io/iree-oss/cmake-python@sha256:c1f05139fa478107bdd5889d6b3ca9f8a4bcf19e77e3e8058e2084367a1aa23d AS final
+
+# Install bazel.
+COPY --from=gcr.io/iree-oss/bazel@sha256:a5c4e189f48e503276c1ba208fee8365b20df503a1b201cde6608dee5eeebadd \
+ /usr/bin/bazel* \
+ /usr/bin/
+
+# Install tensorflow.
+RUN python3 -m pip install tf-nightly==2.5.0.dev20201116
diff --git a/build_tools/docker/cmake-python-nvidia/Dockerfile b/build_tools/docker/cmake-python-nvidia/Dockerfile
index e9559d4..f12045f 100644
--- a/build_tools/docker/cmake-python-nvidia/Dockerfile
+++ b/build_tools/docker/cmake-python-nvidia/Dockerfile
@@ -25,7 +25,7 @@
# does not support Ubuntu 18.04.
# This allows to share configuration with base CMake, but it also means we need
# to MATCH the driver version between the host machine and the docker image.
-FROM gcr.io/iree-oss/cmake-python-vulkan AS final
+FROM gcr.io/iree-oss/cmake-python-vulkan@sha256:6722f69c6300749f6bd4b141fc653244990381a6b0111f9c361061adcd65c07c AS final
RUN apt-get update \
&& DEBIAN_FRONTEND=noninteractive apt-get install -y libnvidia-gl-440=440.100-0ubuntu0.18.04.1
diff --git a/build_tools/docker/cmake-python-swiftshader/Dockerfile b/build_tools/docker/cmake-python-swiftshader/Dockerfile
index 9c652b5..691e9aa 100644
--- a/build_tools/docker/cmake-python-swiftshader/Dockerfile
+++ b/build_tools/docker/cmake-python-swiftshader/Dockerfile
@@ -15,8 +15,9 @@
# An image for building IREE using CMake and testing IREE with SwiftShader
# Vulkan implementation.
-FROM gcr.io/iree-oss/cmake-python-vulkan AS final
-COPY --from=gcr.io/iree-oss/swiftshader@sha256:ccae32c83c89a31e8fc5542e480c29f28bbf4a3b3b80198c06b687a92c6813f3 /swiftshader /swiftshader
+FROM gcr.io/iree-oss/cmake-python-vulkan@sha256:6722f69c6300749f6bd4b141fc653244990381a6b0111f9c361061adcd65c07c AS final
+COPY --from=gcr.io/iree-oss/swiftshader@sha256:ccae32c83c89a31e8fc5542e480c29f28bbf4a3b3b80198c06b687a92c6813f3 \
+ /swiftshader /swiftshader
# Set VK_ICD_FILENAMES so Vulkan loader can find the SwiftShader ICD.
ENV VK_ICD_FILENAMES /swiftshader/vk_swiftshader_icd.json
diff --git a/build_tools/docker/cmake-python-vulkan/Dockerfile b/build_tools/docker/cmake-python-vulkan/Dockerfile
index f8aa2d5..1d829e4 100644
--- a/build_tools/docker/cmake-python-vulkan/Dockerfile
+++ b/build_tools/docker/cmake-python-vulkan/Dockerfile
@@ -16,11 +16,12 @@
# This image provides the Vulkan SDK. Requires a child image to provide
# a Vulkan ICD.
-FROM gcr.io/iree-oss/cmake-python AS final
+FROM gcr.io/iree-oss/cmake-python@sha256:c1f05139fa478107bdd5889d6b3ca9f8a4bcf19e77e3e8058e2084367a1aa23d AS final
ARG VULKAN_SDK_VERSION=1.2.154.0
-COPY --from=gcr.io/iree-oss/vulkan@sha256:5812ee64806a7f3df0739ccf0930c27cabce346901488eceb1ee66c9c0a5ae96 /opt/vulkan-sdk/ /opt/vulkan-sdk/
+COPY --from=gcr.io/iree-oss/vulkan@sha256:5812ee64806a7f3df0739ccf0930c27cabce346901488eceb1ee66c9c0a5ae96 \
+ /opt/vulkan-sdk/ /opt/vulkan-sdk/
ENV VULKAN_SDK="/opt/vulkan-sdk/${VULKAN_SDK_VERSION}/x86_64"
diff --git a/build_tools/docker/manage_images.py b/build_tools/docker/manage_images.py
index 835033e..5a52c47 100755
--- a/build_tools/docker/manage_images.py
+++ b/build_tools/docker/manage_images.py
@@ -60,9 +60,15 @@
'cmake': ['base', 'util'],
'cmake-android': ['cmake', 'util'],
'cmake-python': ['cmake'],
- 'cmake-python-nvidia': ['cmake-python-vulkan'],
- 'cmake-python-swiftshader': ['cmake-python-vulkan', 'swiftshader'],
'cmake-python-vulkan': ['cmake-python', 'vulkan'],
+ 'cmake-python-swiftshader': ['cmake-python-vulkan', 'swiftshader'],
+ 'cmake-python-nvidia': ['cmake-python-vulkan'],
+ 'cmake-bazel-tensorflow': ['cmake-python', 'bazel'],
+ 'cmake-bazel-tensorflow-vulkan': ['cmake-bazel-tensorflow', 'vulkan'],
+ 'cmake-bazel-tensorflow-swiftshader': [
+ 'cmake-bazel-tensorflow-vulkan', 'swiftshader'
+ ],
+ 'cmake-bazel-tensorflow-nvidia': ['cmake-bazel-tensorflow-vulkan'],
'rbe-toolchain': ['vulkan'],
'swiftshader': ['cmake'],
'util': [],
@@ -134,7 +140,7 @@
return _dag_dfs(images, IMAGES_TO_DEPENDENCIES)
-def get_repo_digest(tagged_image_url: str) -> str:
+def get_repo_digest(tagged_image_url: str, dry_run: bool = False) -> str:
inspect_command = [
'docker',
'image',
@@ -150,8 +156,12 @@
capture_output=True,
timeout=10)
except subprocess.CalledProcessError as error:
- raise RuntimeError(f'Computing the repository digest for {tagged_image_url}'
- ' failed. Has it been pushed to GCR?') from error
+ if dry_run:
+ return ""
+ else:
+ raise RuntimeError(
+ f'Computing the repository digest for {tagged_image_url} failed. Has '
+ 'it been pushed to GCR?') from error
_, repo_digest = completed_process.stdout.strip().split('@')
return repo_digest
@@ -232,7 +242,7 @@
dry_run=args.dry_run)
for image in images_to_process:
- print(f'Processing image {image}')
+ print('\n' * 5 + f'Processing image {image}')
image_url = posixpath.join(IREE_GCR_URL, image)
tagged_image_url = f'{image_url}'
image_path = os.path.join(DOCKER_DIR, image)
@@ -244,7 +254,7 @@
utils.run_command(['docker', 'push', tagged_image_url],
dry_run=args.dry_run)
- digest = get_repo_digest(tagged_image_url)
+ digest = get_repo_digest(tagged_image_url, args.dry_run)
# Check that the image is in 'prod_digests.txt' and append it to the list
# in the file if it isn't.
diff --git a/build_tools/docker/prod_digests.txt b/build_tools/docker/prod_digests.txt
index 1945bec..91f3d19 100644
--- a/build_tools/docker/prod_digests.txt
+++ b/build_tools/docker/prod_digests.txt
@@ -2,16 +2,20 @@
gcr.io/iree-oss/util@sha256:40846b4aea5886af3250399d6adfdb3e1195a8b0177706bb0375e812d62dc49c
gcr.io/iree-oss/cmake@sha256:9d9953acf5ca0cf1ff3e8de32f10f24dfab1c4e8ec5d1fc047f556024ee4bed6
gcr.io/iree-oss/swiftshader@sha256:ccae32c83c89a31e8fc5542e480c29f28bbf4a3b3b80198c06b687a92c6813f3
-gcr.io/iree-oss/cmake-python@sha256:2777aaf49a41669c6f0567f25dd8e940d4058df64f8a7a78af0fdcb8a80eea4f
+gcr.io/iree-oss/cmake-python@sha256:c1f05139fa478107bdd5889d6b3ca9f8a4bcf19e77e3e8058e2084367a1aa23d
gcr.io/iree-oss/cmake-android@sha256:15d3266ae4865f7642a4ef4d76e5181f0dc3482a7cfba9021b6b55be524208ec
gcr.io/iree-oss/bazel@sha256:a5c4e189f48e503276c1ba208fee8365b20df503a1b201cde6608dee5eeebadd
gcr.io/iree-oss/bazel-python@sha256:6a1cee37fa2148a9c6c58273f6e02ca2ac89af0b4908962f1b8fe3ffbb6bd476
gcr.io/iree-oss/bazel-tensorflow@sha256:d0aa0d31b1c6cc61148e6520077bb725cfee238bfe268c77414c5baabf7608ac
gcr.io/iree-oss/vulkan@sha256:5812ee64806a7f3df0739ccf0930c27cabce346901488eceb1ee66c9c0a5ae96
gcr.io/iree-oss/rbe-toolchain@sha256:d69c260b98a97ad430d34c4591fb2399e00888750f5d47ede00c1e6f3e774e5a
-gcr.io/iree-oss/cmake-python-vulkan@sha256:f7695315d010a393f3669dace08c05d05735c6d8ce26d5fdda1795f338235f74
-gcr.io/iree-oss/cmake-python-swiftshader@sha256:68a757f54f8a494aee23d43305e3774344fc2607c6aafef33709a571d935bc11
-gcr.io/iree-oss/cmake-python-nvidia@sha256:3e29b42a0eb3bd32f71426b4b41068789a2848e1447467aa409af4109281f4cb
+gcr.io/iree-oss/cmake-python-vulkan@sha256:6722f69c6300749f6bd4b141fc653244990381a6b0111f9c361061adcd65c07c
+gcr.io/iree-oss/cmake-python-swiftshader@sha256:0be2b0c735a038365e7cad31f6b440805dd4e231e166a114ef22914a5469cbc8
+gcr.io/iree-oss/cmake-python-nvidia@sha256:b77508153d66d55cd89be80d54484972340ea9c49dbb1285403386215afdcdf7
gcr.io/iree-oss/bazel-tensorflow-vulkan@sha256:caa0c9699f4041406bf978ed4b1ce69b3b40af436d6e999fd3fac037cd4d4749
gcr.io/iree-oss/bazel-tensorflow-swiftshader@sha256:0f60ce244cc6a2caa89915905baa92d1fae4f806d8c12901ba4b97a56e803a75
gcr.io/iree-oss/bazel-tensorflow-nvidia@sha256:18f3c97bdac1c6705536efc5fb0f38eb541c723671fd451a257119feff2896b7
+gcr.io/iree-oss/cmake-bazel-tensorflow@sha256:41963cf49ecd8873d2349f9f9b3a0af86471614a4ba6916b3be5bbaff5e28b3b
+gcr.io/iree-oss/cmake-bazel-tensorflow-vulkan@sha256:f7ebc07ccfe03e6bb7de11e178583b43660f9fb152f4154a02b44fbb1b42cd4f
+gcr.io/iree-oss/cmake-bazel-tensorflow-nvidia@sha256:1351b5befd1bc4b89fc6f203754a05accff204579f122ec1123e86ed7e2bd319
+gcr.io/iree-oss/cmake-bazel-tensorflow-swiftshader@sha256:e4516f3ffadf40f2111c3f152453928f5af24951c7b5770ef3477f897709df0d
diff --git a/build_tools/kokoro/gcp_ubuntu/cmake/linux/x86-swiftshader/build_kokoro.sh b/build_tools/kokoro/gcp_ubuntu/cmake/linux/x86-swiftshader/build_kokoro.sh
index fc52b18..5f08e4e 100755
--- a/build_tools/kokoro/gcp_ubuntu/cmake/linux/x86-swiftshader/build_kokoro.sh
+++ b/build_tools/kokoro/gcp_ubuntu/cmake/linux/x86-swiftshader/build_kokoro.sh
@@ -32,7 +32,7 @@
docker_setup
docker run "${DOCKER_RUN_ARGS[@]?}" \
- gcr.io/iree-oss/cmake-python-swiftshader@sha256:68a757f54f8a494aee23d43305e3774344fc2607c6aafef33709a571d935bc11 \
+ gcr.io/iree-oss/cmake-python-swiftshader@sha256:0be2b0c735a038365e7cad31f6b440805dd4e231e166a114ef22914a5469cbc8 \
build_tools/kokoro/gcp_ubuntu/cmake/linux/x86-swiftshader/build.sh
# Kokoro will rsync this entire directory back to the executor orchestrating the
diff --git a/build_tools/kokoro/gcp_ubuntu/cmake/linux/x86-turing/build_kokoro.sh b/build_tools/kokoro/gcp_ubuntu/cmake/linux/x86-turing/build_kokoro.sh
index a919978..d55cc67 100755
--- a/build_tools/kokoro/gcp_ubuntu/cmake/linux/x86-turing/build_kokoro.sh
+++ b/build_tools/kokoro/gcp_ubuntu/cmake/linux/x86-turing/build_kokoro.sh
@@ -33,7 +33,7 @@
docker run "${DOCKER_RUN_ARGS[@]?}" \
--gpus all \
- gcr.io/iree-oss/cmake-python-nvidia@sha256:3e29b42a0eb3bd32f71426b4b41068789a2848e1447467aa409af4109281f4cb \
+ gcr.io/iree-oss/cmake-python-nvidia@sha256:b77508153d66d55cd89be80d54484972340ea9c49dbb1285403386215afdcdf7 \
build_tools/kokoro/gcp_ubuntu/cmake/linux/x86-turing/build.sh
# Kokoro will rsync this entire directory back to the executor orchestrating the
