Add a pipeline to test the runtime (#9309)
This adds a pipeline to test the runtime with `ctest`. It depends on
the existing `cmake-build-runtime` pipeline, demonstrating how to
construct such a dependency. This was not quite as ergonomic as I would
like: in particular passing around all the right meta-data for fetching
artifacts. I'm going to fiddle with that some more.
diff --git a/build_tools/buildkite/pipelines/trusted/presubmit.yml b/build_tools/buildkite/pipelines/trusted/presubmit.yml
index f5d49e0..4eb9dc8 100644
--- a/build_tools/buildkite/pipelines/trusted/presubmit.yml
+++ b/build_tools/buildkite/pipelines/trusted/presubmit.yml
@@ -29,6 +29,7 @@
- wait
- label: "Executing build-runtime-cmake"
+ key: "build-runtime-cmake"
plugins:
- https://github.com/GMNGeoffrey/smooth-checkout-buildkite-plugin#24e54e7729:
repos:
@@ -42,3 +43,20 @@
./build_tools/buildkite/scripts/wait_for_pipeline_success.py \
--annotate \
build-runtime-cmake
+
+ - label: "Executing test-runtime-cmake"
+ key: "test-runtime-cmake"
+ depends_on: "build-runtime-cmake"
+ plugins:
+ - https://github.com/GMNGeoffrey/smooth-checkout-buildkite-plugin#24e54e7729:
+ repos:
+ - config:
+ - url: ${BUILDKITE_REPO}
+ ref: ${CONFIG_FETCH_REF}
+ commands: |
+ export BUILDKITE_ANNOTATION_CONTEXT="$${BUILDKITE_STEP_ID}"
+ export BUILDKITE_ACCESS_TOKEN="$(gcloud secrets versions access latest \
+ --secret=iree-buildkite-presubmit-pipelines)"
+ ./build_tools/buildkite/scripts/wait_for_pipeline_success.py \
+ --annotate \
+ test-runtime-cmake
diff --git a/build_tools/buildkite/pipelines/untrusted/build-runtime-cmake.yml b/build_tools/buildkite/pipelines/untrusted/build-runtime-cmake.yml
index 997ec8d..02e3afe 100644
--- a/build_tools/buildkite/pipelines/untrusted/build-runtime-cmake.yml
+++ b/build_tools/buildkite/pipelines/untrusted/build-runtime-cmake.yml
@@ -11,12 +11,18 @@
security: "untrusted"
env:
IREE_DOCKER_WORKDIR: "/usr/src/github/iree"
+ BUILD_DIR: "build-runtime-${BUILDKITE_BUILD_ID}"
commands: |
git submodule sync
git submodule update --init --jobs 8 --depth 1
+
docker run --user="$(id -u):$(id -g)" \
--volume="$$PWD:$$IREE_DOCKER_WORKDIR" \
--workdir="$$IREE_DOCKER_WORKDIR" \
--rm \
gcr.io/iree-oss/base@sha256:32a501b23abeae69eb33e5e78295312f5df28bacabc799dffe218764a66ed28e \
- ./build_tools/cmake/build_runtime.sh
+ ./build_tools/cmake/build_runtime.sh \
+ "$${BUILD_DIR}"
+
+ tar -czf "$${BUILD_DIR}.tgz" "$${BUILD_DIR}"
+ buildkite-agent artifact upload "$${BUILD_DIR}.tgz"
diff --git a/build_tools/buildkite/pipelines/untrusted/test-runtime-cmake.yml b/build_tools/buildkite/pipelines/untrusted/test-runtime-cmake.yml
new file mode 100644
index 0000000..d7862a8
--- /dev/null
+++ b/build_tools/buildkite/pipelines/untrusted/test-runtime-cmake.yml
@@ -0,0 +1,49 @@
+# Copyright 2022 The IREE Authors
+#
+# Licensed under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+steps:
+ - label: "Waiting for build-runtime-cmake"
+ key: build-runtime-cmake
+ agents:
+ queue: "orchestration"
+ security: "untrusted"
+ commands: |
+ export BUILDKITE_ACCESS_TOKEN="$(gcloud secrets versions access latest \
+ --secret=iree-buildkite-presubmit-pipelines)"
+ ./build_tools/buildkite/scripts/wait_for_pipeline_success.py \
+ --output-build-json=build.json \
+ build-runtime-cmake
+ # TODO: install jq on the buildkite agents to replace inline python.
+ buildkite-agent meta-data set \
+ "build-runtime-cmake-build-id" \
+ "$(cat build.json | python3 -c "import json,sys;print(json.load(sys.stdin)['id'])")"
+
+
+ # TODO: better emoji here
+ - label: ":hammer_and_wrench: Test the runtime only"
+ agents:
+ queue: "cpu"
+ security: "untrusted"
+ depends_on: "build-runtime-cmake"
+ env:
+ IREE_DOCKER_WORKDIR: "/usr/src/github/iree"
+ commands: |
+ BUILD_RUNTIME_CMAKE_BUILD_ID="$(buildkite-agent meta-data get "build-runtime-cmake-build-id")"
+ RUNTIME_BUILD_DIR="build-runtime-$${BUILD_RUNTIME_CMAKE_BUILD_ID}"
+ RUNTIME_BUILD_ARCHIVE="$${RUNTIME_BUILD_DIR}.tgz"
+ buildkite-agent artifact download \
+ --build "$${BUILD_RUNTIME_CMAKE_BUILD_ID}" \
+ "$${RUNTIME_BUILD_ARCHIVE}" \
+ ./
+ tar -xzf "$${RUNTIME_BUILD_ARCHIVE}"
+ docker run --user="$(id -u):$(id -g)" \
+ --volume="$$PWD:$$IREE_DOCKER_WORKDIR" \
+ --workdir="$$IREE_DOCKER_WORKDIR" \
+ --env IREE_VULKAN_DISABLE=1 \
+ --rm \
+ gcr.io/iree-oss/base@sha256:32a501b23abeae69eb33e5e78295312f5df28bacabc799dffe218764a66ed28e \
+ ./build_tools/cmake/ctest_all.sh \
+ "$${RUNTIME_BUILD_DIR}"
diff --git a/build_tools/buildkite/scripts/common/buildkite_pipeline_manager.py b/build_tools/buildkite/scripts/common/buildkite_pipeline_manager.py
index 796d31b..71fceb8 100644
--- a/build_tools/buildkite/scripts/common/buildkite_pipeline_manager.py
+++ b/build_tools/buildkite/scripts/common/buildkite_pipeline_manager.py
@@ -57,7 +57,7 @@
self._env = {
# Indicate the name of the pipeline to trigger. This is used by the
# 'unregistered' pipeline to upload the correct pipeline file.
- "IREE_BUILDKITE_IREE_BUILDKITE_REQUESTED_PIPELINE":
+ "IREE_BUILDKITE_REQUESTED_PIPELINE":
pipeline,
# Provide information about the pipeline doing the triggering.
"IREE_BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG":
diff --git a/build_tools/cmake/build_runtime.sh b/build_tools/cmake/build_runtime.sh
index 1300686..89b6023 100755
--- a/build_tools/cmake/build_runtime.sh
+++ b/build_tools/cmake/build_runtime.sh
@@ -5,30 +5,24 @@
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-# Build IREE's runtime using CMake. Designed for CI, but can be run manually.
-# This uses previously cached build results and does not clear build
-# directories.
+# Build IREE's runtime using CMake. The desired build directory can be passed as
+# the first argument. Otherwise, it uses the environment variable
+# IREE_RUNTIME_BUILD_DIR, defaulting to "build-runtime". Designed for CI, but
+# can be run manually. This reuses the build directory if it already exists.
-set -e
-set -x
+set -euo pipefail
-ROOT_DIR=$(git rev-parse --show-toplevel)
-cd ${ROOT_DIR?}
+SCRIPT_DIR="$(dirname -- "$( readlink -f -- "$0"; )")";
-CMAKE_BIN=${CMAKE_BIN:-$(which cmake)}
-"${CMAKE_BIN?}" --version
-ninja --version
+BUILD_DIR="${1:-}"
-if [ -d "build-runtime" ]
-then
- echo "build-runtime directory already exists. Will use cached results there."
-else
- echo "build-runtime directory does not already exist. Creating a new one."
- mkdir build-runtime
+if [[ -z "${BUILD_DIR}" ]]; then
+ BUILD_DIR="${IREE_RUNTIME_BUILD_DIR:-build-runtime}"
fi
-cd build-runtime
-"${CMAKE_BIN?}" -G Ninja .. \
+source "${SCRIPT_DIR}/setup_build.sh"
+
+"${CMAKE_BIN}" -B "${BUILD_DIR}" -G Ninja . \
-DCMAKE_BUILD_TYPE=RelWithDebInfo \
-DIREE_BUILD_COMPILER=OFF
-"${CMAKE_BIN?}" --build . -- -k 0
+"${CMAKE_BIN}" --build "${BUILD_DIR}" -- -k 0
diff --git a/build_tools/cmake/ctest_all.sh b/build_tools/cmake/ctest_all.sh
new file mode 100755
index 0000000..574de68
--- /dev/null
+++ b/build_tools/cmake/ctest_all.sh
@@ -0,0 +1,69 @@
+#!/bin/bash
+
+# Copyright 2020 The IREE Authors
+#
+# Licensed under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+# Run all ctest tests in a build directory, adding some automatic test filtering
+# based on environment variables and defaulting to running as many test jobs as
+# there are processes. The build directory to test is passed as the first
+# argument. Designed for CI, but can be run manually.
+
+set -euo pipefail
+
+BUILD_DIR="$1"
+
+# Respect the user setting, but default to as many jobs as we have cores.
+export CTEST_PARALLEL_LEVEL=${CTEST_PARALLEL_LEVEL:-$(nproc)}
+
+# Respect the user setting, but default to turning on Vulkan.
+export IREE_VULKAN_DISABLE=${IREE_VULKAN_DISABLE:-0}
+# Respect the user setting, but default to turning off CUDA.
+export IREE_CUDA_DISABLE=${IREE_CUDA_DISABLE:-1}
+# The VK_KHR_shader_float16_int8 extension is optional prior to Vulkan 1.2.
+# We test on SwiftShader as a baseline, which does not support this extension.
+export IREE_VULKAN_F16_DISABLE=${IREE_VULKAN_F16_DISABLE:-1}
+
+# Tests to exclude by label. In addition to any custom labels (which are carried
+# over from Bazel tags), every test should be labeled with its directory.
+declare -a label_exclude_args=(
+ # Exclude specific labels.
+ # Put the whole label with anchors for exact matches.
+ # For example:
+ # ^nokokoro$
+ # TODO: update label name as part of dropping Kokoro
+ ^nokokoro$
+
+ # Exclude all tests in a directory.
+ # Put the whole directory with anchors for exact matches.
+ # For example:
+ # ^bindings/python/iree/runtime$
+
+ # Exclude all tests in some subdirectories.
+ # Put the whole parent directory with only a starting anchor.
+ # Use a trailing slash to avoid prefix collisions.
+ # For example:
+ # ^bindings/
+)
+
+if [[ "${IREE_VULKAN_DISABLE}" == 1 ]]; then
+ label_exclude_args+=("^driver=vulkan$")
+fi
+if [[ "${IREE_CUDA_DISABLE}" == 1 ]]; then
+ label_exclude_args+=("^driver=cuda$")
+fi
+if [[ "${IREE_VULKAN_F16_DISABLE}" == 1 ]]; then
+ label_exclude_args+=("^vulkan_uses_vk_khr_shader_float16_int8$")
+fi
+
+# Join on "|"
+label_exclude_regex="($(IFS="|" ; echo "${label_exclude_args[*]}"))"
+
+echo "******************** Running main project ctests ************************"
+ctest\
+ --test-dir "${BUILD_DIR}" \
+ --timeout 900 \
+ --output-on-failure \
+ --label-exclude "${label_exclude_regex}"
diff --git a/build_tools/cmake/setup_build.sh b/build_tools/cmake/setup_build.sh
new file mode 100644
index 0000000..5906430
--- /dev/null
+++ b/build_tools/cmake/setup_build.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+# Copyright 2021 The IREE Authors
+#
+# Licensed under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+# Sets up a CMake build. Requires that BUILD_DIR be set in the environment and
+# expects to be *sourced* not invoked, since it sets environment variables.
+
+set -euo pipefail
+
+ROOT_DIR=$(git rev-parse --show-toplevel)
+cd "${ROOT_DIR}"
+
+CMAKE_BIN="${CMAKE_BIN:-$(which cmake)}"
+"${CMAKE_BIN}" --version
+ninja --version
+
+if [[ -d "${BUILD_DIR}" ]]; then
+ echo "'${BUILD_DIR}' directory already exists. Will use cached results there."
+else
+ echo "'${BUILD_DIR}' directory does not already exist. Creating a new one."
+ mkdir "${BUILD_DIR}"
+fi
