Separate the bindings and core Kokoro builds and build scripts.
- Forks and slightly modifies the `kokoro/gcp_ubuntu/bazel` configuration to run the bindings tests.
- Also moves `build_tools/bazel/build.sh` to `build_tools/bazel/build_core.sh` to match new organization semantics.
- Another option would be to move all of the build scripts into `build_tools/bazel/build_scripts/*` (e.g. `build_tools/bazel/build_scripts/core.sh`
This change was tested to work in PR #2100. These tests will be enabled for everyone in a later internal change.
Closes https://github.com/google/iree/pull/2104
COPYBARA_INTEGRATE_REVIEW=https://github.com/google/iree/pull/2104 from phoenix-meadowlark:kokoro-bazel-bindings c919513e144fc3db639623bdce9238fa22d30c19
PiperOrigin-RevId: 314611873
diff --git a/build_tools/bazel/build.sh b/build_tools/bazel/build_bindings.sh
similarity index 94%
copy from build_tools/bazel/build.sh
copy to build_tools/bazel/build_bindings.sh
index df68412..56601d8 100755
--- a/build_tools/bazel/build.sh
+++ b/build_tools/bazel/build_bindings.sh
@@ -14,8 +14,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-# Build the IREE project with bazel. Designed for CI, but can be run manually.
-# Looks at environment variables and uses CI-friendly defaults if they are not set.
+# Build IREE's bindings (//bindings/...) with bazel. Designed for CI, but can be
+# run manually.
+
+# Looks at environment variables and uses CI-friendly defaults if they are not
+# set.
# IREE_LLVMJIT_DISABLE: Do not run tests that require LLVM-JIT. Default: 1
# IREE_VULKAN_DISABLE: Do not run tests that require Vulkan. Default: 1
# BUILD_TAG_FILTERS: Passed to bazel to filter targets to build.
@@ -66,7 +69,7 @@
# `bazel test //...` because the latter excludes targets tagged "manual". The
# "manual" tag allows targets to be excluded from human wildcard builds, but we
# want them built by CI unless they are excluded with "nokokoro".
-bazel query //iree/... + //bindings/... | \
+bazel query //bindings/... | \
xargs bazel test ${test_env_args[@]} \
--config=generic_clang \
--build_tag_filters="${BUILD_TAG_FILTERS?}" \
diff --git a/build_tools/bazel/build.sh b/build_tools/bazel/build_core.sh
similarity index 95%
rename from build_tools/bazel/build.sh
rename to build_tools/bazel/build_core.sh
index df68412..00e104e 100755
--- a/build_tools/bazel/build.sh
+++ b/build_tools/bazel/build_core.sh
@@ -14,8 +14,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-# Build the IREE project with bazel. Designed for CI, but can be run manually.
-# Looks at environment variables and uses CI-friendly defaults if they are not set.
+# Build IREE's core (//iree/...) with bazel. Designed for CI, but can be run
+# manually.
+
+# Looks at environment variables and uses CI-friendly defaults if they are not
+# set.
# IREE_LLVMJIT_DISABLE: Do not run tests that require LLVM-JIT. Default: 1
# IREE_VULKAN_DISABLE: Do not run tests that require Vulkan. Default: 1
# BUILD_TAG_FILTERS: Passed to bazel to filter targets to build.
@@ -66,7 +69,7 @@
# `bazel test //...` because the latter excludes targets tagged "manual". The
# "manual" tag allows targets to be excluded from human wildcard builds, but we
# want them built by CI unless they are excluded with "nokokoro".
-bazel query //iree/... + //bindings/... | \
+bazel query //iree/... | \
xargs bazel test ${test_env_args[@]} \
--config=generic_clang \
--build_tag_filters="${BUILD_TAG_FILTERS?}" \
diff --git a/build_tools/bazel/build_tensorflow.sh b/build_tools/bazel/build_tensorflow.sh
index 672d246..cfffffe 100755
--- a/build_tools/bazel/build_tensorflow.sh
+++ b/build_tools/bazel/build_tensorflow.sh
@@ -14,9 +14,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-# Build the IREE integrations with bazel. Designed for CI, but can be run
-# manually. Looks at environment variables and uses CI-friendly defaults if they
-# are not set.
+# Build IREE's integrations (//integrations/...) with bazel. Designed for CI,
+# but can be run manually.
+
+# Looks at environment variables and uses CI-friendly defaults if they are not
+# set.
# IREE_LLVMJIT_DISABLE: Do not run tests that require LLVM-JIT. Default: 1
# IREE_VULKAN_DISABLE: Do not run tests that require Vulkan. Default: 1
# BUILD_TAG_FILTERS: Passed to bazel to filter targets to build.
diff --git a/kokoro/gcp_ubuntu/bazel/bindings/build.sh b/kokoro/gcp_ubuntu/bazel/bindings/build.sh
new file mode 100755
index 0000000..93a12f3
--- /dev/null
+++ b/kokoro/gcp_ubuntu/bazel/bindings/build.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# For use within a IREE bazel-tensorflow docker image on a Kokoro VM.
+# Log some information about the environment, initialize the submodules and then
+# run the bazel integrations tests.
+
+set -e
+set -x
+
+# Print the UTC time when set -x is on
+export PS4='[$(date -u "+%T %Z")] '
+
+# Check these exist and print the versions for later debugging
+bazel --version
+"$CXX" --version
+"$CC" --version
+"$PYTHON_BIN" -V
+# TODO( #1875 ): Make PYTHON_BIN also control the runtime version
+python3 -V
+
+echo "Initializing submodules"
+./scripts/git/submodule_versions.py init
+
+echo "Building and testing with bazel"
+./build_tools/bazel/build_bindings.sh
diff --git a/kokoro/gcp_ubuntu/bazel/bindings/build_kokoro.sh b/kokoro/gcp_ubuntu/bazel/bindings/build_kokoro.sh
new file mode 100644
index 0000000..304a964
--- /dev/null
+++ b/kokoro/gcp_ubuntu/bazel/bindings/build_kokoro.sh
@@ -0,0 +1,41 @@
+#!/bin/bash
+
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Build and test the project within the gcr.io/iree-oss/bazel-tensorflow
+# image using Kokoro.
+
+set -e
+set -x
+
+# Print the UTC time when set -x is on
+export PS4='[$(date -u "+%T %Z")] '
+
+# Kokoro checks out the repository here.
+WORKDIR="${KOKORO_ARTIFACTS_DIR?}/github/iree"
+
+# Mount the checked out repository, make that the working directory and run the
+# tests in the bazel-tensorflow image.
+# TODO: Change image to gcr.io/iree-oss/bazel-bindings:prod once we create it.
+docker run \
+ --volume "${WORKDIR?}:${WORKDIR?}" \
+ --workdir="${WORKDIR?}" \
+ --rm \
+ gcr.io/iree-oss/bazel-tensorflow:prod \
+ kokoro/gcp_ubuntu/bazel/bindings/build.sh
+
+# Kokoro will rsync this entire directory back to the executor orchestrating the
+# build which takes forever and is totally useless.
+rm -rf "${KOKORO_ARTIFACTS_DIR?}/*"
diff --git a/kokoro/gcp_ubuntu/bazel/bindings/common.cfg b/kokoro/gcp_ubuntu/bazel/bindings/common.cfg
new file mode 100644
index 0000000..f53f602
--- /dev/null
+++ b/kokoro/gcp_ubuntu/bazel/bindings/common.cfg
@@ -0,0 +1,19 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Common configuration for Kokoro builds that run bazel on linux.
+
+build_file: "iree/kokoro/gcp_ubuntu/bazel/bindings/build_kokoro.sh"
diff --git a/kokoro/gcp_ubuntu/bazel/bindings/continuous.cfg b/kokoro/gcp_ubuntu/bazel/bindings/continuous.cfg
new file mode 100644
index 0000000..af84216
--- /dev/null
+++ b/kokoro/gcp_ubuntu/bazel/bindings/continuous.cfg
@@ -0,0 +1,19 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Deliberately blank as everything necessary is configured in common files, but
+# file must still exist to match corresponding (upstream only) job
+# configurations that trigger the builds.
diff --git a/kokoro/gcp_ubuntu/bazel/bindings/presubmit.cfg b/kokoro/gcp_ubuntu/bazel/bindings/presubmit.cfg
new file mode 100644
index 0000000..af84216
--- /dev/null
+++ b/kokoro/gcp_ubuntu/bazel/bindings/presubmit.cfg
@@ -0,0 +1,19 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Deliberately blank as everything necessary is configured in common files, but
+# file must still exist to match corresponding (upstream only) job
+# configurations that trigger the builds.
diff --git a/kokoro/gcp_ubuntu/bazel/build.sh b/kokoro/gcp_ubuntu/bazel/build.sh
index f3d5316..d8c8ece 100755
--- a/kokoro/gcp_ubuntu/bazel/build.sh
+++ b/kokoro/gcp_ubuntu/bazel/build.sh
@@ -36,4 +36,4 @@
./scripts/git/submodule_versions.py init
echo "Building and testing with bazel"
-./build_tools/bazel/build.sh
+./build_tools/bazel/build_core.sh
