[ci] Configure docker for Vulkan on NVIDIA and use it in Kokoro (#2484)
This commit adds a Dockerfile for packing the NVIDIA driver and
Vulkan SDK inside a docker image and redirects the CMake tests
to use this docker image to enable testing Vulkan HAL backend.
Co-authored-by: Geoffrey Martin-Noble <gcmn@google.com>
diff --git a/build_tools/cmake/test.sh b/build_tools/cmake/test.sh
index 3dc7e6d..8cd9f2d 100755
--- a/build_tools/cmake/test.sh
+++ b/build_tools/cmake/test.sh
@@ -62,4 +62,4 @@
label_exclude_regex="($(IFS="|" ; echo "${label_exclude_args[*]?}"))"
cd ${ROOT_DIR?}/build
-ctest --label-exclude "${label_exclude_regex?}"
+ctest --output-on-failure --label-exclude "${label_exclude_regex?}"
diff --git a/build_tools/docker/build_and_update_gcr.py b/build_tools/docker/build_and_update_gcr.py
index e26d5d1..90c7454 100755
--- a/build_tools/docker/build_and_update_gcr.py
+++ b/build_tools/docker/build_and_update_gcr.py
@@ -27,8 +27,13 @@
DOCKER_DIR = 'build_tools/docker/'
IMAGES = [
- 'bazel', 'bazel-bindings', 'bazel-tensorflow', 'cmake', 'cmake-android',
- 'rbe-toolchain'
+ 'bazel',
+ 'bazel-bindings',
+ 'bazel-tensorflow',
+ 'cmake',
+ 'cmake-android',
+ 'cmake-nvidia',
+ 'rbe-toolchain',
]
IMAGES_HELP = [f'`{name}`' for name in IMAGES]
IMAGES_HELP = f'{", ".join(IMAGES_HELP[:-1])} or {IMAGES_HELP[-1]}'
@@ -36,7 +41,7 @@
# Map from image names to images that depend on them.
IMAGES_TO_DEPENDENT_IMAGES = {
'bazel': ['bazel-bindings', 'bazel-tensorflow'],
- 'cmake': ['cmake-android']
+ 'cmake': ['cmake-android', 'cmake-nvidia'],
}
RBE_MESSAGE = """
@@ -92,7 +97,8 @@
print(f'Updating image {image}')
image_url = os.path.join(IREE_GCR_URL, f'{image}:{args.tag}')
image_path = os.path.join(DOCKER_DIR, image.replace('-', '_'))
- subprocess.check_output(['docker', 'build', '--tag', image_url, image_path])
+ subprocess.check_output(
+ ['docker', 'build', '--tag', image_url, image_path])
if args.push:
subprocess.check_output(['docker', 'push', image_url])
diff --git a/build_tools/docker/cmake_nvidia/Dockerfile b/build_tools/docker/cmake_nvidia/Dockerfile
new file mode 100644
index 0000000..8679656
--- /dev/null
+++ b/build_tools/docker/cmake_nvidia/Dockerfile
@@ -0,0 +1,52 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# An image for building IREE using CMake and testing IREE with NVIDIA GPUs.
+
+# Build using:
+# docker build --tag gcr.io/iree-oss/cmake-nvidia \
+# build_tools/docker/cmake_nvidia/
+
+# Run interactively using the following, where IREE_WORKDIR is the path to your
+# local dev environment:
+# docker run -it --rm --entrypoint bash \
+# --volume "${IREE_WORKDIR?}:/usr/src/iree/" \
+# --gpus all \
+# gcr.io/iree-oss/cmake-nvidia
+
+# Set up the image and working directory by inheriting the base
+# CMake configuration.
+# Note that we don't start from NVIDIA's docker base:
+# - nvidia/cuda (https://hub.docker.com/r/nvidia/cuda):
+# it's.. for CUDA.
+# - nvidia/vulkan (https://hub.docker.com/r/nvidia/vulkan):
+# does not support Ubuntu 18.04.
+# This allows to share configuration with base CMake, but it also means we need
+# to MATCH the driver version between the host machine and the docker image.
+FROM gcr.io/iree-oss/cmake
+
+# Additionally, we need to install the Vulkan SDK and the NVIDIA Vulkan driver.
+
+ARG VULKAN_SDK_VERSION=1.2.141
+
+# Disable apt-key parse waring.
+ENV APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1
+
+# Disable prompt during keyboard configuration.
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN wget -qO - http://packages.lunarg.com/lunarg-signing-key-pub.asc | apt-key add - \
+ && wget -qO /etc/apt/sources.list.d/lunarg-vulkan-$VULKAN_SDK_VERSION-bionic.list http://packages.lunarg.com/vulkan/$VULKAN_SDK_VERSION/lunarg-vulkan-$VULKAN_SDK_VERSION-bionic.list \
+ && apt-get update \
+ && apt-get install -y vulkan-sdk nvidia-driver-440
diff --git a/kokoro/gcp_ubuntu/cmake/linux/x86-turing/build.sh b/kokoro/gcp_ubuntu/cmake/linux/x86-turing/build.sh
new file mode 100755
index 0000000..a56c847
--- /dev/null
+++ b/kokoro/gcp_ubuntu/cmake/linux/x86-turing/build.sh
@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Build the project with cmake using Kokoro.
+
+set -e
+set -x
+
+# Print the UTC time when set -x is on
+export PS4='[$(date -u "+%T %Z")] '
+
+# Check these exist and print the versions for later debugging
+export CMAKE_BIN="$(which cmake)"
+"$CMAKE_BIN" --version
+"$CC" --version
+"$CXX" --version
+python3 --version
+
+# Print Vulkan related information: SDK version and GPU ICD version
+vulkaninfo 2>/dev/null | grep "Vulkan Instance"
+vulkaninfo 2>/dev/null | grep -A7 "VkPhysicalDeviceProperties"
+
+echo "Initializing submodules"
+./scripts/git/submodule_versions.py init
+
+# TODO(gcmn): It would be nice to be able to build and test as much as possible,
+# so a build failure only prevents building/testing things that depend on it and
+# we can still run the other tests.
+echo "Building with cmake"
+./build_tools/cmake/clean_build.sh
+
+echo "Testing with ctest"
+./build_tools/cmake/test.sh
diff --git a/kokoro/gcp_ubuntu/cmake/linux/x86-turing/build_kokoro.sh b/kokoro/gcp_ubuntu/cmake/linux/x86-turing/build_kokoro.sh
new file mode 100755
index 0000000..051c0d0
--- /dev/null
+++ b/kokoro/gcp_ubuntu/cmake/linux/x86-turing/build_kokoro.sh
@@ -0,0 +1,45 @@
+#!/bin/bash
+
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Build and test the project within the gcr.io/iree-oss/cmake using Kokoro.
+
+set -e
+set -x
+
+# Print the UTC time when set -x is on
+export PS4='[$(date -u "+%T %Z")] '
+
+# Kokoro checks out the repository here.
+WORKDIR=${KOKORO_ARTIFACTS_DIR?}/github/iree
+
+# Mount the checked out repository, make that the working directory and run the
+# tests in the cmake image.
+docker run \
+ --volume "${WORKDIR?}:${WORKDIR?}" \
+ --workdir="${WORKDIR?}" \
+ --rm \
+ --env IREE_VULKAN_DISABLE=0 \
+ --gpus all \
+ gcr.io/iree-oss/cmake-nvidia:prod \
+ kokoro/gcp_ubuntu/cmake/linux/x86-turing/build.sh
+
+# Kokoro will rsync this entire directory back to the executor orchestrating the
+# build which takes forever and is totally useless.
+# TODO: enable this after making it work
+#sudo rm -rf "${KOKORO_ARTIFACTS_DIR?}"/*
+
+# Print out artifacts dir contents after deleting them as a coherence check.
+#ls -1a "${KOKORO_ARTIFACTS_DIR?}/"
diff --git a/kokoro/gcp_ubuntu/cmake/linux/x86-turing/common.cfg b/kokoro/gcp_ubuntu/cmake/linux/x86-turing/common.cfg
new file mode 100644
index 0000000..9e6847d
--- /dev/null
+++ b/kokoro/gcp_ubuntu/cmake/linux/x86-turing/common.cfg
@@ -0,0 +1,19 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Common configuration for Kokoro builds that run cmake on linux.
+
+build_file: "iree/kokoro/gcp_ubuntu/cmake/linux/x86-turing/build_kokoro.sh"
diff --git a/kokoro/gcp_ubuntu/cmake/linux/x86-turing/continuous.cfg b/kokoro/gcp_ubuntu/cmake/linux/x86-turing/continuous.cfg
new file mode 100644
index 0000000..e4cc270
--- /dev/null
+++ b/kokoro/gcp_ubuntu/cmake/linux/x86-turing/continuous.cfg
@@ -0,0 +1,19 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Deliberately blank as everything necessary is configured in common files, but
+# file must still exist to match corresponding (Google internal) job
+# configurations that trigger the builds.
diff --git a/kokoro/gcp_ubuntu/cmake/linux/x86-turing/google.cfg b/kokoro/gcp_ubuntu/cmake/linux/x86-turing/google.cfg
new file mode 100644
index 0000000..e4cc270
--- /dev/null
+++ b/kokoro/gcp_ubuntu/cmake/linux/x86-turing/google.cfg
@@ -0,0 +1,19 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Deliberately blank as everything necessary is configured in common files, but
+# file must still exist to match corresponding (Google internal) job
+# configurations that trigger the builds.
diff --git a/kokoro/gcp_ubuntu/cmake/linux/x86-turing/main.cfg b/kokoro/gcp_ubuntu/cmake/linux/x86-turing/main.cfg
new file mode 100644
index 0000000..e4cc270
--- /dev/null
+++ b/kokoro/gcp_ubuntu/cmake/linux/x86-turing/main.cfg
@@ -0,0 +1,19 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Deliberately blank as everything necessary is configured in common files, but
+# file must still exist to match corresponding (Google internal) job
+# configurations that trigger the builds.
diff --git a/kokoro/gcp_ubuntu/cmake/linux/x86-turing/presubmit.cfg b/kokoro/gcp_ubuntu/cmake/linux/x86-turing/presubmit.cfg
new file mode 100644
index 0000000..e4cc270
--- /dev/null
+++ b/kokoro/gcp_ubuntu/cmake/linux/x86-turing/presubmit.cfg
@@ -0,0 +1,19 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Deliberately blank as everything necessary is configured in common files, but
+# file must still exist to match corresponding (Google internal) job
+# configurations that trigger the builds.
