Test runtime Emscripten builds. (#10319)
Progress on https://github.com/iree-org/iree/issues/9855
* Update Emscripten SDK in our Dockerfile
* Delete the Buildkite wasm runtime job (and prerequisite host install
job)
* Add a new `emscripten` job to ci.yml (GitHub Actions post-submit for
now, pre-submit later)
This currently just builds the runtime (and `iree-test-deps`) using
Emscripten. Running tests or building the samples at
[`experimental/web`](https://github.com/iree-org/iree/tree/main/experimental/web)
will come later (likely as that code gets promoted to
[`runtime/bindings/`](https://github.com/iree-org/iree/tree/main/runtime/bindings)
in some way)
I also posted an RFC for this and future work: [Promoting IREE web
platform / Emscripten builds to
"stable"](https://groups.google.com/g/iree-discuss/c/2K5VJ9P8K8I/m/elidhfFMCAAJ)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index d7c9976..bfd1fa1 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -573,8 +573,7 @@
needs: [setup, build_all]
if: needs.setup.outputs.should-run == 'true'
runs-on:
- # Hacks, and order matters. See the comment at the top of the file.
- - self-hosted
+ - self-hosted # must come first
- runner-group=${{ needs.setup.outputs.runner-group }}
- environment=${{ needs.setup.outputs.runner-env }}
- cpu
@@ -683,6 +682,35 @@
bash -euo pipefail -c \
"./build_tools/cmake/build_riscv.sh && ./build_tools/cmake/test_riscv.sh"
+ emscripten:
+ needs: [setup, build_all]
+ if: needs.setup.outputs.should-run == 'true' && needs.setup.outputs.ci-stage == 'postsubmit'
+ runs-on:
+ - self-hosted # must come first
+ - runner-group=${{ needs.setup.outputs.runner-group }}
+ - environment=${{ needs.setup.outputs.runner-env }}
+ - cpu
+ - os-family=Linux
+ env:
+ BUILD_DIR: ${{ needs.build_all.outputs.build-dir }}
+ BUILD_DIR_ARCHIVE: ${{ needs.build_all.outputs.build-dir-archive }}
+ BUILD_DIR_GCS_ARTIFACT: ${{ needs.build_all.outputs.build-dir-gcs-artifact }}
+ steps:
+ - name: "Checking out repository"
+ uses: actions/checkout@7884fcad6b5d53d10323aee724dc68d8b9096a2e # v2
+ with:
+ submodules: true
+ - name: "Downloading build dir archive"
+ run: gcloud alpha storage cp "${BUILD_DIR_GCS_ARTIFACT}" "${BUILD_DIR_ARCHIVE}"
+ - name: "Extracting install from build dir archive"
+ run: tar -xf "${BUILD_DIR_ARCHIVE}" "${BUILD_DIR}/install"
+ - name: "Building the runtime for the web using Emscripten"
+ run: |
+ build_tools/github_actions/docker_run.sh \
+ --env "IREE_HOST_BINARY_ROOT=${BUILD_DIR}/install" \
+ gcr.io/iree-oss/emscripten@sha256:8ccc1c8de11919faf23aeaa585b13e5c5050952db76c101d6f61367280a3546f \
+ build_tools/cmake/build_runtime_emscripten.sh
+
##############################################################################
# Depends on all the other jobs to provide a single anchor that indicates the
@@ -720,6 +748,7 @@
- android_arm64
- baremetal_riscv32
- linux_riscv
+ - emscripten
# Benchmark pipeline
- benchmarks
diff --git a/build_tools/buildkite/cmake/build_configurations.yml b/build_tools/buildkite/cmake/build_configurations.yml
index 75eb6f8..72302e1 100644
--- a/build_tools/buildkite/cmake/build_configurations.yml
+++ b/build_tools/buildkite/cmake/build_configurations.yml
@@ -42,30 +42,6 @@
agents:
- "queue=build"
- - label: ":linux: Build host install"
- key: "build-host-install"
- commands:
- - "git submodule sync && git submodule update --init --jobs 8 --depth 1"
- - "docker run --user=$(id -u):$(id -g) --volume=\\$PWD:\\$IREE_DOCKER_WORKDIR --workdir=\\$IREE_DOCKER_WORKDIR --rm gcr.io/iree-oss/base@sha256:5d43683c6b50aebe1fca6c85f2012f3b0fa153bf4dd268e8767b619b1891423a ./build_tools/cmake/build_host_tools.sh"
- - "tar -czvf build-artifacts.tgz build-host/install"
- artifact_paths: "build-artifacts.tgz"
- env:
- IREE_DOCKER_WORKDIR: "/usr/src/github/iree"
- agents:
- - "queue=build"
-
- - label: ":webassembly: Build WebAssembly runtime with Emscripten"
- depends_on: "build-host-install"
- commands:
- - "buildkite-agent artifact download --step build-host-install build-artifacts.tgz ./"
- - "tar xzf build-artifacts.tgz"
- - "git submodule update --init --jobs 8 --depth 1"
- - "docker run --user=$(id -u):$(id -g) --volume=\\$PWD:\\$IREE_DOCKER_WORKDIR --workdir=\\$IREE_DOCKER_WORKDIR --rm gcr.io/iree-oss/emscripten@sha256:afa6aab07d753631c37a935695e69165d8f7598dec249b31d459b046593ccd56 ./build_tools/cmake/build_runtime_emscripten.sh"
- env:
- IREE_DOCKER_WORKDIR: "/usr/src/github/iree"
- agents:
- - "queue=build"
-
# Notify rotation on postsubmit failures.
notify:
- email: "bdi-build-cop+buildkite@grotations.appspotmail.com"
diff --git a/build_tools/buildkite/samples.yml b/build_tools/buildkite/samples.yml
index 9bc787f..bd3a103 100644
--- a/build_tools/buildkite/samples.yml
+++ b/build_tools/buildkite/samples.yml
@@ -43,7 +43,7 @@
- "buildkite-agent artifact download --step build-host-install build-artifacts.tgz ./"
- "tar xzf build-artifacts.tgz"
- "git submodule update --init --jobs 8 --depth 1"
- - "docker run --user=$(id -u):$(id -g) --volume=\\$PWD:\\$IREE_DOCKER_WORKDIR --workdir=\\$IREE_DOCKER_WORKDIR --rm gcr.io/iree-oss/emscripten@sha256:afa6aab07d753631c37a935695e69165d8f7598dec249b31d459b046593ccd56 ./experimental/web/test_samples.sh"
+ - "docker run --user=$(id -u):$(id -g) --volume=\\$PWD:\\$IREE_DOCKER_WORKDIR --workdir=\\$IREE_DOCKER_WORKDIR --rm gcr.io/iree-oss/emscripten@sha256:8ccc1c8de11919faf23aeaa585b13e5c5050952db76c101d6f61367280a3546f ./experimental/web/test_samples.sh"
env:
IREE_DOCKER_WORKDIR: "/usr/src/github/iree"
agents:
diff --git a/build_tools/cmake/build_runtime_emscripten.sh b/build_tools/cmake/build_runtime_emscripten.sh
index 8739b07..04f5a13 100755
--- a/build_tools/cmake/build_runtime_emscripten.sh
+++ b/build_tools/cmake/build_runtime_emscripten.sh
@@ -13,8 +13,7 @@
# ./build-host/install. Emscripten binaries (e.g. .wasm and .js files) will be
# built in ./build-emscripten/.
-set -x
-set -e
+set -xeuo pipefail
if ! command -v emcmake &> /dev/null
then
@@ -22,13 +21,12 @@
exit 1
fi
-CMAKE_BIN=${CMAKE_BIN:-$(which cmake)}
-"${CMAKE_BIN?}" --version
-ninja --version
-
ROOT_DIR=$(git rev-parse --show-toplevel)
cd ${ROOT_DIR?}
+CMAKE_BIN=${CMAKE_BIN:-$(which cmake)}
+IREE_HOST_BINARY_ROOT="$(realpath ${IREE_HOST_BINARY_ROOT})"
+
if [ -d "build-emscripten" ]
then
echo "build-emscripten directory already exists. Will use cached results there."
@@ -40,13 +38,11 @@
# Configure using Emscripten's CMake wrapper, then build.
emcmake "${CMAKE_BIN?}" -G Ninja .. \
- -DIREE_HOST_BINARY_ROOT=$PWD/../build-host/install \
+ -DIREE_HOST_BINARY_ROOT="${IREE_HOST_BINARY_ROOT}" \
-DIREE_BUILD_COMPILER=OFF \
-DIREE_HAL_DRIVER_DEFAULTS=OFF \
-DIREE_HAL_DRIVER_LOCAL_SYNC=ON \
-DIREE_HAL_DRIVER_LOCAL_TASK=ON \
- -DIREE_HAL_EXECUTABLE_LOADER_DEFAULTS=OFF \
- -DIREE_HAL_EXECUTABLE_LOADER_VMVX_MODULE=ON \
-DIREE_ENABLE_CPUINFO=OFF \
-DIREE_BUILD_TESTS=ON \
-DIREE_BUILD_SAMPLES=ON
diff --git a/build_tools/docker/emscripten/Dockerfile b/build_tools/docker/emscripten/Dockerfile
index 60a642b..e2a5ee9 100644
--- a/build_tools/docker/emscripten/Dockerfile
+++ b/build_tools/docker/emscripten/Dockerfile
@@ -12,8 +12,8 @@
# * https://github.com/emscripten-core/emsdk/blob/main/docker/Dockerfile
# * https://hub.docker.com/r/emscripten/emsdk
-ARG EMSDK_COMMIT=62d817b66dad481b150fd49cdc1e803725d23813
-ARG SDK_VERSION=3.1.12
+ARG EMSDK_COMMIT=21611d2a507fad73385120d89e05a794666070ae
+ARG SDK_VERSION=3.1.20
WORKDIR /
@@ -43,4 +43,4 @@
# Normally we'd run `source emsdk_env.sh`, but that doesn't integrate with
# Docker's environment properties model. Instead, we directly extend the path
# to include the directories suggested by `emsdk activate`.
-ENV PATH="${EMSDK?}:${EMSDK?}/node/14.15.5_64bit/bin:${EMSDK?}/upstream/emscripten:$PATH"
+ENV PATH="${EMSDK?}:${EMSDK?}/node/14.18.2_64bit/bin:${EMSDK?}/upstream/emscripten:$PATH"
diff --git a/build_tools/docker/prod_digests.txt b/build_tools/docker/prod_digests.txt
index e2f6633..a3b16ba 100644
--- a/build_tools/docker/prod_digests.txt
+++ b/build_tools/docker/prod_digests.txt
@@ -7,6 +7,6 @@
gcr.io/iree-oss/gradle-android@sha256:d9d0f880c3ac995b9e8a23bbf8079b80f6842851654016c5f362c747c09aaf93
gcr.io/iree-oss/riscv@sha256:d6f0e293a50faf5abbd564c1d1bb9dc6456d7ce93d07b131c381fa64c1daed62
gcr.io/iree-oss/nvidia@sha256:7c2f56db65e656c15e6c96b5812a8275dd53c82bf41221192f9ba8a451aad870
-gcr.io/iree-oss/emscripten@sha256:afa6aab07d753631c37a935695e69165d8f7598dec249b31d459b046593ccd56
+gcr.io/iree-oss/emscripten@sha256:8ccc1c8de11919faf23aeaa585b13e5c5050952db76c101d6f61367280a3546f
gcr.io/iree-oss/android@sha256:76c2a52dcd6d07601227b965ac87d021c1d2d5e2d01f46ad58da28c89267f2ab
gcr.io/iree-oss/manylinux2014_x86_64-release@sha256:b09c10868f846308bad2eab253a77d0a3f097816c40342bc289d8e62509bc5f9
