Add cmake-bazel-tensorflow images for integrations (#4176) - Add `cmake-bazel-tensorflow*` images to build and test the new tensorflow integrations. - Tested that these build and test locally. - Add `dry_run` arg to `get_repo_digests` to allow `dry_run` to work on non-pushed images.
diff --git a/build_tools/docker/README.md b/build_tools/docker/README.md index 2bb7d4e..07ce7d8 100644 --- a/build_tools/docker/README.md +++ b/build_tools/docker/README.md
@@ -74,7 +74,9 @@ 1. Update the `Dockerfile` for the image that you want to modify or add. If you're adding a new image, or updating the dependencies between images, be - sure to update `IMAGES_TO_DEPENDENCIES` in `manage_images.py` as well. + sure to update `IMAGES_TO_DEPENDENCIES` in `manage_images.py` as well. If + you are adding new images, it is best add them via `git add` before + proceeding. 2. Build the image, push the image to GCR and update all references to the image with the new GCR digest:
diff --git a/build_tools/docker/cmake-bazel-tensorflow-nvidia/Dockerfile b/build_tools/docker/cmake-bazel-tensorflow-nvidia/Dockerfile new file mode 100644 index 0000000..b05548d --- /dev/null +++ b/build_tools/docker/cmake-bazel-tensorflow-nvidia/Dockerfile
@@ -0,0 +1,29 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# To use the host GPUs, `docker run` must be called with the `--gpus all` flag. + +# Set up the image and working directory by inheriting the vulkan CMake +# configuration. +# Note that we don't start from NVIDIA's docker base: +# - nvidia/cuda (https://hub.docker.com/r/nvidia/cuda): +# it's.. for CUDA. +# - nvidia/vulkan (https://hub.docker.com/r/nvidia/vulkan): +# does not support Ubuntu 18.04. +# This allows to share configuration with base CMake, but it also means we need +# to MATCH the driver version between the host machine and the docker image. +FROM gcr.io/iree-oss/cmake-bazel-tensorflow-vulkan@sha256:f7ebc07ccfe03e6bb7de11e178583b43660f9fb152f4154a02b44fbb1b42cd4f AS final + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y libnvidia-gl-440=440.100-0ubuntu0.18.04.1
diff --git a/build_tools/docker/cmake-bazel-tensorflow-swiftshader/Dockerfile b/build_tools/docker/cmake-bazel-tensorflow-swiftshader/Dockerfile new file mode 100644 index 0000000..fe52b89 --- /dev/null +++ b/build_tools/docker/cmake-bazel-tensorflow-swiftshader/Dockerfile
@@ -0,0 +1,20 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM gcr.io/iree-oss/cmake-bazel-tensorflow-vulkan@sha256:f7ebc07ccfe03e6bb7de11e178583b43660f9fb152f4154a02b44fbb1b42cd4f AS final +COPY --from=gcr.io/iree-oss/swiftshader@sha256:ccae32c83c89a31e8fc5542e480c29f28bbf4a3b3b80198c06b687a92c6813f3 \ + /swiftshader /swiftshader + +# Set VK_ICD_FILENAMES so Vulkan loader can find the SwiftShader ICD. +ENV VK_ICD_FILENAMES /swiftshader/vk_swiftshader_icd.json
diff --git a/build_tools/docker/cmake-bazel-tensorflow-vulkan/Dockerfile b/build_tools/docker/cmake-bazel-tensorflow-vulkan/Dockerfile new file mode 100644 index 0000000..09b9559 --- /dev/null +++ b/build_tools/docker/cmake-bazel-tensorflow-vulkan/Dockerfile
@@ -0,0 +1,33 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This image provides the Vulkan SDK. Requires a child image to provide +# a Vulkan ICD. + +FROM gcr.io/iree-oss/cmake-bazel-tensorflow@sha256:41963cf49ecd8873d2349f9f9b3a0af86471614a4ba6916b3be5bbaff5e28b3b AS final + +ARG VULKAN_SDK_VERSION=1.2.154.0 + +COPY --from=gcr.io/iree-oss/vulkan@sha256:5812ee64806a7f3df0739ccf0930c27cabce346901488eceb1ee66c9c0a5ae96 \ + /opt/vulkan-sdk/ /opt/vulkan-sdk/ + +ENV VULKAN_SDK="/opt/vulkan-sdk/${VULKAN_SDK_VERSION}/x86_64" + +ENV PATH="${VULKAN_SDK}/bin:$PATH" + +# Symlink the Vulkan loader to a system library directory. This is needed to +# allow Vulkan applications to find the Vulkan loader. It also avoids using +# LD_LIBRARY_PATH, which is not supported well by Docker. +RUN ln -s "${VULKAN_SDK}/lib/libvulkan.so" /usr/lib/x86_64-linux-gnu/ \ + && ln -s "${VULKAN_SDK}/lib/libvulkan.so.1" /usr/lib/x86_64-linux-gnu/
diff --git a/build_tools/docker/cmake-bazel-tensorflow/Dockerfile b/build_tools/docker/cmake-bazel-tensorflow/Dockerfile new file mode 100644 index 0000000..209b196 --- /dev/null +++ b/build_tools/docker/cmake-bazel-tensorflow/Dockerfile
@@ -0,0 +1,23 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM gcr.io/iree-oss/cmake-python@sha256:c1f05139fa478107bdd5889d6b3ca9f8a4bcf19e77e3e8058e2084367a1aa23d AS final + +# Install bazel. +COPY --from=gcr.io/iree-oss/bazel@sha256:a5c4e189f48e503276c1ba208fee8365b20df503a1b201cde6608dee5eeebadd \ + /usr/bin/bazel* \ + /usr/bin/ + +# Install tensorflow. +RUN python3 -m pip install tf-nightly==2.5.0.dev20201116
diff --git a/build_tools/docker/cmake-python-nvidia/Dockerfile b/build_tools/docker/cmake-python-nvidia/Dockerfile index e9559d4..f12045f 100644 --- a/build_tools/docker/cmake-python-nvidia/Dockerfile +++ b/build_tools/docker/cmake-python-nvidia/Dockerfile
@@ -25,7 +25,7 @@ # does not support Ubuntu 18.04. # This allows to share configuration with base CMake, but it also means we need # to MATCH the driver version between the host machine and the docker image. -FROM gcr.io/iree-oss/cmake-python-vulkan AS final +FROM gcr.io/iree-oss/cmake-python-vulkan@sha256:6722f69c6300749f6bd4b141fc653244990381a6b0111f9c361061adcd65c07c AS final RUN apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get install -y libnvidia-gl-440=440.100-0ubuntu0.18.04.1
diff --git a/build_tools/docker/cmake-python-swiftshader/Dockerfile b/build_tools/docker/cmake-python-swiftshader/Dockerfile index 9c652b5..691e9aa 100644 --- a/build_tools/docker/cmake-python-swiftshader/Dockerfile +++ b/build_tools/docker/cmake-python-swiftshader/Dockerfile
@@ -15,8 +15,9 @@ # An image for building IREE using CMake and testing IREE with SwiftShader # Vulkan implementation. -FROM gcr.io/iree-oss/cmake-python-vulkan AS final -COPY --from=gcr.io/iree-oss/swiftshader@sha256:ccae32c83c89a31e8fc5542e480c29f28bbf4a3b3b80198c06b687a92c6813f3 /swiftshader /swiftshader +FROM gcr.io/iree-oss/cmake-python-vulkan@sha256:6722f69c6300749f6bd4b141fc653244990381a6b0111f9c361061adcd65c07c AS final +COPY --from=gcr.io/iree-oss/swiftshader@sha256:ccae32c83c89a31e8fc5542e480c29f28bbf4a3b3b80198c06b687a92c6813f3 \ + /swiftshader /swiftshader # Set VK_ICD_FILENAMES so Vulkan loader can find the SwiftShader ICD. ENV VK_ICD_FILENAMES /swiftshader/vk_swiftshader_icd.json
diff --git a/build_tools/docker/cmake-python-vulkan/Dockerfile b/build_tools/docker/cmake-python-vulkan/Dockerfile index f8aa2d5..1d829e4 100644 --- a/build_tools/docker/cmake-python-vulkan/Dockerfile +++ b/build_tools/docker/cmake-python-vulkan/Dockerfile
@@ -16,11 +16,12 @@ # This image provides the Vulkan SDK. Requires a child image to provide # a Vulkan ICD. -FROM gcr.io/iree-oss/cmake-python AS final +FROM gcr.io/iree-oss/cmake-python@sha256:c1f05139fa478107bdd5889d6b3ca9f8a4bcf19e77e3e8058e2084367a1aa23d AS final ARG VULKAN_SDK_VERSION=1.2.154.0 -COPY --from=gcr.io/iree-oss/vulkan@sha256:5812ee64806a7f3df0739ccf0930c27cabce346901488eceb1ee66c9c0a5ae96 /opt/vulkan-sdk/ /opt/vulkan-sdk/ +COPY --from=gcr.io/iree-oss/vulkan@sha256:5812ee64806a7f3df0739ccf0930c27cabce346901488eceb1ee66c9c0a5ae96 \ + /opt/vulkan-sdk/ /opt/vulkan-sdk/ ENV VULKAN_SDK="/opt/vulkan-sdk/${VULKAN_SDK_VERSION}/x86_64"
diff --git a/build_tools/docker/manage_images.py b/build_tools/docker/manage_images.py index 835033e..5a52c47 100755 --- a/build_tools/docker/manage_images.py +++ b/build_tools/docker/manage_images.py
@@ -60,9 +60,15 @@ 'cmake': ['base', 'util'], 'cmake-android': ['cmake', 'util'], 'cmake-python': ['cmake'], - 'cmake-python-nvidia': ['cmake-python-vulkan'], - 'cmake-python-swiftshader': ['cmake-python-vulkan', 'swiftshader'], 'cmake-python-vulkan': ['cmake-python', 'vulkan'], + 'cmake-python-swiftshader': ['cmake-python-vulkan', 'swiftshader'], + 'cmake-python-nvidia': ['cmake-python-vulkan'], + 'cmake-bazel-tensorflow': ['cmake-python', 'bazel'], + 'cmake-bazel-tensorflow-vulkan': ['cmake-bazel-tensorflow', 'vulkan'], + 'cmake-bazel-tensorflow-swiftshader': [ + 'cmake-bazel-tensorflow-vulkan', 'swiftshader' + ], + 'cmake-bazel-tensorflow-nvidia': ['cmake-bazel-tensorflow-vulkan'], 'rbe-toolchain': ['vulkan'], 'swiftshader': ['cmake'], 'util': [], @@ -134,7 +140,7 @@ return _dag_dfs(images, IMAGES_TO_DEPENDENCIES) -def get_repo_digest(tagged_image_url: str) -> str: +def get_repo_digest(tagged_image_url: str, dry_run: bool = False) -> str: inspect_command = [ 'docker', 'image', @@ -150,8 +156,12 @@ capture_output=True, timeout=10) except subprocess.CalledProcessError as error: - raise RuntimeError(f'Computing the repository digest for {tagged_image_url}' - ' failed. Has it been pushed to GCR?') from error + if dry_run: + return "" + else: + raise RuntimeError( + f'Computing the repository digest for {tagged_image_url} failed. Has ' + 'it been pushed to GCR?') from error _, repo_digest = completed_process.stdout.strip().split('@') return repo_digest @@ -232,7 +242,7 @@ dry_run=args.dry_run) for image in images_to_process: - print(f'Processing image {image}') + print('\n' * 5 + f'Processing image {image}') image_url = posixpath.join(IREE_GCR_URL, image) tagged_image_url = f'{image_url}' image_path = os.path.join(DOCKER_DIR, image) @@ -244,7 +254,7 @@ utils.run_command(['docker', 'push', tagged_image_url], dry_run=args.dry_run) - digest = get_repo_digest(tagged_image_url) + digest = get_repo_digest(tagged_image_url, args.dry_run) # Check that the image is in 'prod_digests.txt' and append it to the list # in the file if it isn't.
diff --git a/build_tools/docker/prod_digests.txt b/build_tools/docker/prod_digests.txt index 1945bec..91f3d19 100644 --- a/build_tools/docker/prod_digests.txt +++ b/build_tools/docker/prod_digests.txt
@@ -2,16 +2,20 @@ gcr.io/iree-oss/util@sha256:40846b4aea5886af3250399d6adfdb3e1195a8b0177706bb0375e812d62dc49c gcr.io/iree-oss/cmake@sha256:9d9953acf5ca0cf1ff3e8de32f10f24dfab1c4e8ec5d1fc047f556024ee4bed6 gcr.io/iree-oss/swiftshader@sha256:ccae32c83c89a31e8fc5542e480c29f28bbf4a3b3b80198c06b687a92c6813f3 -gcr.io/iree-oss/cmake-python@sha256:2777aaf49a41669c6f0567f25dd8e940d4058df64f8a7a78af0fdcb8a80eea4f +gcr.io/iree-oss/cmake-python@sha256:c1f05139fa478107bdd5889d6b3ca9f8a4bcf19e77e3e8058e2084367a1aa23d gcr.io/iree-oss/cmake-android@sha256:15d3266ae4865f7642a4ef4d76e5181f0dc3482a7cfba9021b6b55be524208ec gcr.io/iree-oss/bazel@sha256:a5c4e189f48e503276c1ba208fee8365b20df503a1b201cde6608dee5eeebadd gcr.io/iree-oss/bazel-python@sha256:6a1cee37fa2148a9c6c58273f6e02ca2ac89af0b4908962f1b8fe3ffbb6bd476 gcr.io/iree-oss/bazel-tensorflow@sha256:d0aa0d31b1c6cc61148e6520077bb725cfee238bfe268c77414c5baabf7608ac gcr.io/iree-oss/vulkan@sha256:5812ee64806a7f3df0739ccf0930c27cabce346901488eceb1ee66c9c0a5ae96 gcr.io/iree-oss/rbe-toolchain@sha256:d69c260b98a97ad430d34c4591fb2399e00888750f5d47ede00c1e6f3e774e5a -gcr.io/iree-oss/cmake-python-vulkan@sha256:f7695315d010a393f3669dace08c05d05735c6d8ce26d5fdda1795f338235f74 -gcr.io/iree-oss/cmake-python-swiftshader@sha256:68a757f54f8a494aee23d43305e3774344fc2607c6aafef33709a571d935bc11 -gcr.io/iree-oss/cmake-python-nvidia@sha256:3e29b42a0eb3bd32f71426b4b41068789a2848e1447467aa409af4109281f4cb +gcr.io/iree-oss/cmake-python-vulkan@sha256:6722f69c6300749f6bd4b141fc653244990381a6b0111f9c361061adcd65c07c +gcr.io/iree-oss/cmake-python-swiftshader@sha256:0be2b0c735a038365e7cad31f6b440805dd4e231e166a114ef22914a5469cbc8 +gcr.io/iree-oss/cmake-python-nvidia@sha256:b77508153d66d55cd89be80d54484972340ea9c49dbb1285403386215afdcdf7 gcr.io/iree-oss/bazel-tensorflow-vulkan@sha256:caa0c9699f4041406bf978ed4b1ce69b3b40af436d6e999fd3fac037cd4d4749 gcr.io/iree-oss/bazel-tensorflow-swiftshader@sha256:0f60ce244cc6a2caa89915905baa92d1fae4f806d8c12901ba4b97a56e803a75 gcr.io/iree-oss/bazel-tensorflow-nvidia@sha256:18f3c97bdac1c6705536efc5fb0f38eb541c723671fd451a257119feff2896b7 +gcr.io/iree-oss/cmake-bazel-tensorflow@sha256:41963cf49ecd8873d2349f9f9b3a0af86471614a4ba6916b3be5bbaff5e28b3b +gcr.io/iree-oss/cmake-bazel-tensorflow-vulkan@sha256:f7ebc07ccfe03e6bb7de11e178583b43660f9fb152f4154a02b44fbb1b42cd4f +gcr.io/iree-oss/cmake-bazel-tensorflow-nvidia@sha256:1351b5befd1bc4b89fc6f203754a05accff204579f122ec1123e86ed7e2bd319 +gcr.io/iree-oss/cmake-bazel-tensorflow-swiftshader@sha256:e4516f3ffadf40f2111c3f152453928f5af24951c7b5770ef3477f897709df0d
diff --git a/build_tools/kokoro/gcp_ubuntu/cmake/linux/x86-swiftshader/build_kokoro.sh b/build_tools/kokoro/gcp_ubuntu/cmake/linux/x86-swiftshader/build_kokoro.sh index fc52b18..5f08e4e 100755 --- a/build_tools/kokoro/gcp_ubuntu/cmake/linux/x86-swiftshader/build_kokoro.sh +++ b/build_tools/kokoro/gcp_ubuntu/cmake/linux/x86-swiftshader/build_kokoro.sh
@@ -32,7 +32,7 @@ docker_setup docker run "${DOCKER_RUN_ARGS[@]?}" \ - gcr.io/iree-oss/cmake-python-swiftshader@sha256:68a757f54f8a494aee23d43305e3774344fc2607c6aafef33709a571d935bc11 \ + gcr.io/iree-oss/cmake-python-swiftshader@sha256:0be2b0c735a038365e7cad31f6b440805dd4e231e166a114ef22914a5469cbc8 \ build_tools/kokoro/gcp_ubuntu/cmake/linux/x86-swiftshader/build.sh # Kokoro will rsync this entire directory back to the executor orchestrating the
diff --git a/build_tools/kokoro/gcp_ubuntu/cmake/linux/x86-turing/build_kokoro.sh b/build_tools/kokoro/gcp_ubuntu/cmake/linux/x86-turing/build_kokoro.sh index a919978..d55cc67 100755 --- a/build_tools/kokoro/gcp_ubuntu/cmake/linux/x86-turing/build_kokoro.sh +++ b/build_tools/kokoro/gcp_ubuntu/cmake/linux/x86-turing/build_kokoro.sh
@@ -33,7 +33,7 @@ docker run "${DOCKER_RUN_ARGS[@]?}" \ --gpus all \ - gcr.io/iree-oss/cmake-python-nvidia@sha256:3e29b42a0eb3bd32f71426b4b41068789a2848e1447467aa409af4109281f4cb \ + gcr.io/iree-oss/cmake-python-nvidia@sha256:b77508153d66d55cd89be80d54484972340ea9c49dbb1285403386215afdcdf7 \ build_tools/kokoro/gcp_ubuntu/cmake/linux/x86-turing/build.sh # Kokoro will rsync this entire directory back to the executor orchestrating the