Create a runtime submodule subset (#10686)
Reduce CI build time by only updating a subset of submodules for the
non-compiler targets.
diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
index 4733f2d..a5d8b28 100644
--- a/.github/workflows/benchmarks.yml
+++ b/.github/workflows/benchmarks.yml
@@ -78,8 +78,8 @@
steps:
- name: "Checking out repository"
uses: actions/checkout@7884fcad6b5d53d10323aee724dc68d8b9096a2e # v2
- with:
- submodules: true
+ - name: "Checking out runtime submodules"
+ run: ./build_tools/scripts/git/update_runtime_submodules.sh
- name: "Downloading build dir archive"
run: gcloud alpha storage cp "${BUILD_DIR_GCS_ARTIFACT}" "${BUILD_DIR_ARCHIVE}"
- name: "Extracting install from build dir archive"
@@ -144,8 +144,8 @@
steps:
- name: "Checking out repository"
uses: actions/checkout@7884fcad6b5d53d10323aee724dc68d8b9096a2e # v2
- with:
- submodules: true
+ - name: "Checking out runtime submodules"
+ run: ./build_tools/scripts/git/update_runtime_submodules.sh
- name: "Downloading build dir archive"
run: gcloud alpha storage cp "${BUILD_DIR_GCS_ARTIFACT}" "${BUILD_DIR_ARCHIVE}"
- name: "Extracting host binaries"
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index a534ae3..e8b5590 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -264,8 +264,8 @@
steps:
- name: "Checking out repository"
uses: actions/checkout@7884fcad6b5d53d10323aee724dc68d8b9096a2e # v2
- with:
- submodules: true
+ - name: "Checking out runtime submodules"
+ run: ./build_tools/scripts/git/update_runtime_submodules.sh
- name: "Building runtime"
run: |
./build_tools/github_actions/docker_run.sh \
@@ -578,8 +578,8 @@
steps:
- name: "Checking out repository"
uses: actions/checkout@7884fcad6b5d53d10323aee724dc68d8b9096a2e # v2
- with:
- submodules: true
+ - name: "Checking out runtime submodules"
+ run: ./build_tools/scripts/git/update_runtime_submodules.sh
- name: "Downloading build dir archive"
run: gcloud alpha storage cp "${BUILD_DIR_GCS_ARTIFACT}" "${BUILD_DIR_ARCHIVE}"
- name: "Extracting install from build dir archive"
@@ -610,8 +610,8 @@
steps:
- name: "Checking out repository"
uses: actions/checkout@7884fcad6b5d53d10323aee724dc68d8b9096a2e # v2
- with:
- submodules: true
+ - name: "Checking out runtime submodules"
+ run: ./build_tools/scripts/git/update_runtime_submodules.sh
- name: "Downloading build dir archive"
run: gcloud alpha storage cp "${BUILD_DIR_GCS_ARTIFACT}" "${BUILD_DIR_ARCHIVE}"
- name: "Extracting install from build dir archive"
@@ -648,8 +648,8 @@
steps:
- name: "Checking out repository"
uses: actions/checkout@7884fcad6b5d53d10323aee724dc68d8b9096a2e # v2
- with:
- submodules: true
+ - name: "Checking out runtime submodules"
+ run: ./build_tools/scripts/git/update_runtime_submodules.sh
- name: "Downloading build dir archive"
run: gcloud alpha storage cp "${BUILD_DIR_GCS_ARTIFACT}" "${BUILD_DIR_ARCHIVE}"
- name: "Extracting build dir archive"
@@ -683,8 +683,8 @@
steps:
- name: "Checking out repository"
uses: actions/checkout@7884fcad6b5d53d10323aee724dc68d8b9096a2e # v2
- with:
- submodules: true
+ - name: "Checking out runtime submodules"
+ run: ./build_tools/scripts/git/update_runtime_submodules.sh
- name: "Downloading build dir archive"
run: gcloud alpha storage cp "${BUILD_DIR_GCS_ARTIFACT}" "${BUILD_DIR_ARCHIVE}"
- name: "Extracting install from build dir archive"
@@ -736,8 +736,8 @@
steps:
- name: "Checking out repository"
uses: actions/checkout@7884fcad6b5d53d10323aee724dc68d8b9096a2e # v2
- with:
- submodules: true
+ - name: "Checking out runtime submodules"
+ run: ./build_tools/scripts/git/update_runtime_submodules.sh
- name: "Downloading build dir archive"
run: gcloud alpha storage cp "${BUILD_DIR_GCS_ARTIFACT}" "${BUILD_DIR_ARCHIVE}"
- name: "Extracting build dir archive"
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 0893bdf..9596f68 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -462,8 +462,11 @@
OUTPUT_VARIABLE SHORT_HASH)
string(REGEX REPLACE "\n$" "" SHORT_HASH "${SHORT_HASH}")
if(SHORT_HASH_RESULT EQUAL "0" AND NOT "${IREE_GIT_SHORT_HASH}" STREQUAL "${SHORT_HASH}")
+ if(NOT IREE_BUILD_COMPILER)
+ set(CHECK_SUBMODULE_ARGS "--runtime_only")
+ endif()
execute_process(
- COMMAND ${Python3_EXECUTABLE} build_tools/scripts/git/check_submodule_init.py
+ COMMAND ${Python3_EXECUTABLE} build_tools/scripts/git/check_submodule_init.py ${CHECK_SUBMODULE_ARGS}
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
RESULT_VARIABLE SUBMODULE_INIT_RESULT
)
diff --git a/build_tools/scripts/git/check_submodule_init.py b/build_tools/scripts/git/check_submodule_init.py
index 219545c..611c32f 100644
--- a/build_tools/scripts/git/check_submodule_init.py
+++ b/build_tools/scripts/git/check_submodule_init.py
@@ -4,12 +4,22 @@
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+import argparse
import os
+import pathlib
import subprocess
import sys
def run():
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ "--runtime_only",
+ help=("Only check the initialization of the submodules for the"
+ "runtime-dependent submodules. Default: False"),
+ action="store_true",
+ default=False)
+ args = parser.parse_args()
# No-op if we're not in a git repository.
try:
subprocess.check_call(['git', 'rev-parse', '--is-inside-work-tree'],
@@ -21,11 +31,16 @@
output = os.popen("git submodule status")
submodules = output.readlines()
+ runtime_submodules = pathlib.Path(__file__).with_name(
+ "runtime_submodules.txt").read_text().split("\n")
+
for submodule in submodules:
- if (submodule.strip()[0] == "-"):
+ prefix = submodule.strip()[0]
+ name = submodule.split()[1]
+ if prefix == "-" and (not args.runtime_only or name in runtime_submodules):
print(
"The git submodule '%s' is not initialized. Please run `git submodule update --init`"
- % (submodule.split()[1]))
+ % (name))
sys.exit(1)
diff --git a/build_tools/scripts/git/runtime_submodules.txt b/build_tools/scripts/git/runtime_submodules.txt
new file mode 100644
index 0000000..c581b8b
--- /dev/null
+++ b/build_tools/scripts/git/runtime_submodules.txt
@@ -0,0 +1,13 @@
+third_party/benchmark
+third_party/cpuinfo
+third_party/flatcc
+third_party/googletest
+third_party/liburing
+third_party/libyaml
+third_party/musl
+third_party/spirv_cross
+third_party/spirv_headers
+third_party/tracy
+third_party/vulkan_headers
+third_party/vulkan_memory_allocator
+third_party/webgpu-headers
diff --git a/build_tools/scripts/git/update_runtime_submodules.sh b/build_tools/scripts/git/update_runtime_submodules.sh
new file mode 100755
index 0000000..a6e72f3
--- /dev/null
+++ b/build_tools/scripts/git/update_runtime_submodules.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# Copyright 2022 The IREE Authors
+#
+# Licensed under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# Initialize a subset of submodules for runtime only.
+
+set -xeuo pipefail
+
+SCRIPT_DIR="$(dirname -- "$( readlink -f -- "$0"; )")"
+
+readarray -t RUNTIME_SUBMODULES < "${SCRIPT_DIR}/runtime_submodules.txt"
+
+git submodule sync && git submodule update --init ${RUNTIME_SUBMODULES[@]}
