Avoid running docker builds as root (#2663)
Instead of mounting the user's existing Bazel cache, create a fake home
directory under the Kokoro root directory.
This is another attempt at https://github.com/google/iree/pull/2566
which had to be rolled back for the Bazel integrations build because it
made it three times slower. It turns out the root cause there is that
the docker container volumes are by default mounted in /tmpfs which on
Kokoro is a local-ssd. The home directory, on the other hand, is a
persistent SSD. The integrations job is running Bazel with 64 cores,
which means it's IO bound. The difference turns out to be substantial.
Fixes https://github.com/google/iree/issues/2653
Tested:
Ran simulate_kokoro on the core build locally to confirm this doesn't
break RBE cloud credentials.
The integrations presubmit on this PR takes 20 minutes, as before.
diff --git a/build_tools/kokoro/gcp_ubuntu/bazel/linux/x86/integrations/build_kokoro.sh b/build_tools/kokoro/gcp_ubuntu/bazel/linux/x86/integrations/build_kokoro.sh
index 19af965..c35d897 100755
--- a/build_tools/kokoro/gcp_ubuntu/bazel/linux/x86/integrations/build_kokoro.sh
+++ b/build_tools/kokoro/gcp_ubuntu/bazel/linux/x86/integrations/build_kokoro.sh
@@ -26,16 +26,10 @@
# Print the UTC time when set -x is on
export PS4='[$(date -u "+%T %Z")] '
-WORKDIR="${KOKORO_ARTIFACTS_DIR?}/github/iree"
+source "${KOKORO_ARTIFACTS_DIR?}/github/iree/build_tools/kokoro/gcp_ubuntu/docker_common.sh"
-# TODO(#2653): Don't run docker as root
-DOCKER_RUN_ARGS=(
- # Make the source repository available
- --volume="${WORKDIR?}:${WORKDIR?}"
- --workdir="${WORKDIR?}"
- # Delete the container after
- --rm
-)
+# Sets DOCKER_RUN_ARGS
+docker_setup
docker run "${DOCKER_RUN_ARGS[@]?}" \
gcr.io/iree-oss/bazel-tensorflow:prod \
@@ -43,8 +37,7 @@
# Kokoro will rsync this entire directory back to the executor orchestrating the
# build which takes forever and is totally useless.
-# TODO(#2653): Remove sudo when docker doesn't run as root
-sudo rm -rf "${KOKORO_ARTIFACTS_DIR?}"/*
+rm -rf "${KOKORO_ARTIFACTS_DIR?}"/*
# Print out artifacts dir contents after deleting them as a coherence check.
ls -1a "${KOKORO_ARTIFACTS_DIR?}/"
diff --git a/build_tools/kokoro/gcp_ubuntu/docker_common.sh b/build_tools/kokoro/gcp_ubuntu/docker_common.sh
index fbc896f..d0b15dd 100644
--- a/build_tools/kokoro/gcp_ubuntu/docker_common.sh
+++ b/build_tools/kokoro/gcp_ubuntu/docker_common.sh
@@ -22,12 +22,40 @@
# Sets the environment variable DOCKER_RUN_ARGS to be used by subsequent
# `docker run` invocations.
function docker_setup() {
- # Setup to run docker as the current user.
- # Provide a place to mount files that would normally be under /etc/
+ # Make the source repository available and launch containers in that
+ # directory.
+ local workdir="${KOKORO_ARTIFACTS_DIR?}/github/iree"
+ DOCKER_RUN_ARGS=(
+ --volume="${workdir?}:${workdir?}"
+ --workdir="${workdir?}"
+ )
+
+ # Delete the container after the run is complete.
+ DOCKER_RUN_ARGS+=(--rm)
+
+
+ # Run as the current user and group. If only it were this simple...
+ DOCKER_RUN_ARGS+=(--user="$(id -u):$(id -g)")
+
+
+ # The Docker container doesn't know about the users and groups of the host
+ # system. We have to tell it. This is just a mapping of IDs to names though.
+ # The thing that really matters is the IDs, so the key thing is that Docker
+ # writes files as the same ID as the current user, which we set above, but
+ # without the group and passwd file, lots of things get upset because they
+ # don't recognize the current user ID (e.g. `whoami` fails). Bazel in
+ # particular looks for a home directory and is not happy when it can't find
+ # one.
+ # So we make the container share the host mapping, which guarantees that the
+ # current user is mapped. If there was any user or group in the container
+ # that we cared about, this wouldn't necessarily work because the host and
+ # container don't necessarily map the ID to the same user. Luckily though,
+ # we don't.
# We don't just mount the real /etc/passwd and /etc/group because Google
# Linux workstations do some interesting stuff with user/group permissions
# such that they don't contain the information about normal users and we
# want these scripts to be runnable locally for debugging.
+ # Instead we dump the results of `getent` to some fake files.
local fake_etc_dir="${KOKORO_ROOT?}/fake_etc"
mkdir -p "${fake_etc_dir?}"
@@ -37,25 +65,35 @@
getent group > "${fake_group?}"
getent passwd > "${fake_passwd?}"
- local workdir="${KOKORO_ARTIFACTS_DIR?}/github/iree"
-
- DOCKER_RUN_ARGS=(
- # Run as the current user and group
- --user="$(id -u):$(id -g)"
- # Make the source repository available
- --volume="${workdir?}:${workdir?}"
- --workdir="${workdir?}"
- # Tell docker about the host users and groups. Bazel needs this
- # information, but it also makes some other things more pleasant.
+ DOCKER_RUN_ARGS+=(
--volume="${fake_group?}:/etc/group:ro"
--volume="${fake_passwd?}:/etc/passwd:ro"
- # Allow Bazel to write its special cache directories. This is the
- # default path Bazel will write to.
- --volume="${HOME?}/.cache/bazel:${HOME?}/.cache/bazel"
- # Make gcloud credentials available. This isn't necessary when running
- # in GCE but enables using this script locally with RBE.
+ )
+
+
+ # Bazel stores its cache in the user home directory by default. It's
+ # possible to override this, but that would require changing our Bazel
+ # startup options, which means polluting all our scripts and making them not
+ # runnable locally. Instead, we give it a special home directory to write
+ # into. We don't just mount the user home directory (or some subset thereof)
+ # for two reasons:
+ # 1. We probably don't want Docker to just write into the user's home
+ # directory when running locally.
+ # 2. When running with Kokoro, we mount a local scratch SSD to KOKORO_ROOT
+ # whereas the home directory is on the persistent SSD boot disk. It
+ # turns out that makes a huge difference in performance for Bazel
+ # running with local execution (not with RBE) because it is IO bound at
+ # 64 cores.
+ local fake_home_dir="${KOKORO_ROOT?}/fake_home"
+ mkdir -p "${fake_home_dir}"
+
+ DOCKER_RUN_ARGS+=(
+ --volume="${fake_home_dir?}:${HOME?}"
+ )
+
+ # Make gcloud credentials available. This isn't necessary when running in
+ # GCE but enables using this script locally with RBE.
+ DOCKER_RUN_ARGS+=(
--volume="${HOME?}/.config/gcloud:${HOME?}/.config/gcloud:ro"
- # Delete the container after
- --rm
)
}
