Add ConvPerf workflow (#11334)
Runs `convperf` (https://github.com/nod-ai/convperf) nightly and uploads
results to the convperf-benchmark-artifacts gcs bucket. Convperf
benchmarks convolution shapes and generates a report e.g.
https://storage.googleapis.com/convperf-benchmark-artifacts/cpu/latest/resnet50_thread1.png
diff --git a/.github/workflows/run_convperf.yml b/.github/workflows/run_convperf.yml
new file mode 100644
index 0000000..dc99c7e
--- /dev/null
+++ b/.github/workflows/run_convperf.yml
@@ -0,0 +1,87 @@
+# Copyright 2022 The IREE Authors
+#
+# Licensed under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# Workflow for running `convperf` (https://github.com/nod-ai/convperf).
+# `convperf` benchmarks convolution workloads on IREE and other backends such
+# as `libsxmm`, etc.
+#
+# The workflow runs benchmarks on CPU and uploads results to the
+# `convperf-benchmark-artifacts` GC bucket.
+
+name: convperf
+
+on:
+ schedule:
+ - cron: '0 16 * * *'
+ workflow_dispatch:
+
+env:
+ GCS_DIR: gs://iree-github-actions-${{ github.event_name == 'pull_request' && 'presubmit' || 'postsubmit' }}-artifacts/${{ github.run_id }}/${{ github.run_attempt }}
+
+jobs:
+ setup:
+ runs-on: ubuntu-20.04
+ env:
+ # The commit being checked out is the merge commit for the PR. Its first
+ # parent will be the tip of main.
+ BASE_REF: HEAD^
+ PR_TITLE: ${{ github.event.pull_request.title }}
+ PR_BODY: ${{ github.event.pull_request.body }}
+ IREE_SHA: ${{ github.sha }}
+ outputs:
+ artifact-upload-dir: ${{ steps.iree.outputs.artifact-upload-dir }}
+ should-run: ${{ steps.configure.outputs.should-run }}
+ runner-env: ${{ steps.configure.outputs.runner-env }}
+ runner-group: ${{ steps.configure.outputs.runner-group }}
+ steps:
+ - name: "Checking out repository"
+ uses: actions/checkout@7884fcad6b5d53d10323aee724dc68d8b9096a2e # v2
+ with:
+ # We need the parent commit to do a diff
+ fetch-depth: 2
+ - name: "Configuring CI options"
+ id: configure
+ run: |
+ # Just informative logging. There should only be two commits in the
+ # history here, but limiting the depth helps when copying from a local
+ # repo instead of using checkout, e.g. with
+ # https://github.com/nektos/act where there will be more.
+ git log --oneline --graph --max-count=3
+ ./build_tools/github_actions/configure_ci.py
+ - name: "Calculating version info"
+ id: iree
+ run: |
+ export GCS_ARTIFACT_DIR="$(date +'%Y-%m-%d').sha_${IREE_SHA}.timestamp_$(date +'%s')"
+ echo "artifact-upload-dir=${GCS_ARTIFACT_DIR}" >> $GITHUB_OUTPUT
+
+ build_and_benchmark_cpu:
+ needs: setup
+ if: needs.setup.outputs.should-run == 'true'
+ runs-on:
+ - self-hosted # must come first
+ - runner-group=${{ needs.setup.outputs.runner-group }}
+ - environment=${{ needs.setup.outputs.runner-env }}
+ - cpu
+ - os-family=Linux
+ env:
+ IREE_SHA: ${{ github.sha }}
+ BUILD_DIR: convperf-build
+ RESULTS_DIR: convperf-results
+ GCS_UPLOAD_PARENT_DIR: "gs://convperf-benchmark-artifacts/cpu"
+ GCS_UPLOAD_DIR_NAME: ${{ needs.setup.outputs.artifact-upload-dir }}
+ steps:
+ - name: "Checking out repository"
+ uses: actions/checkout@7884fcad6b5d53d10323aee724dc68d8b9096a2e # v2
+ - name: "Running convperf for CPU"
+ run: |
+ mkdir ${RESULTS_DIR}
+ ./build_tools/github_actions/docker_run.sh \
+ gcr.io/iree-oss/convperf@sha256:9c6a6cf89d4f667246382bcce96f8542f626872d9014184b3a8118750af8f878 \
+ ./build_tools/benchmarks/convperf/build_and_run_convperf.sh "${BUILD_DIR}" "${RESULTS_DIR}" "${IREE_SHA}"
+ - name: "Uploading results"
+ run: |
+ gcloud alpha storage cp "${RESULTS_DIR}/**" "${GCS_UPLOAD_PARENT_DIR}/${GCS_UPLOAD_DIR_NAME}/"
+ gcloud alpha storage cp "${RESULTS_DIR}/**" "${GCS_UPLOAD_PARENT_DIR}/latest/"
diff --git a/.github/workflows/run_mmperf.yml b/.github/workflows/run_mmperf.yml
index 5374f91..cd6eefa 100644
--- a/.github/workflows/run_mmperf.yml
+++ b/.github/workflows/run_mmperf.yml
@@ -78,13 +78,13 @@
- name: "Building mmperf for CPU"
run: |
./build_tools/github_actions/docker_run.sh \
- gcr.io/iree-oss/mmperf@sha256:bc74f3bbb0e3e8373ab0b35362b6f2a92b1d770432b6711f61403e73a1af076a \
+ gcr.io/iree-oss/mmperf@sha256:7b3bf8a1fafe3603428820865623b133aeecb409d8f79ac7f79b4400b391e207 \
./build_tools/benchmarks/mmperf/build_mmperf.sh "${BUILD_DIR}" "cpu" "${IREE_SHA}"
- name: "Running mmperf on CPU"
run: |
mkdir ${RESULTS_DIR}
./build_tools/github_actions/docker_run.sh \
- gcr.io/iree-oss/mmperf@sha256:bc74f3bbb0e3e8373ab0b35362b6f2a92b1d770432b6711f61403e73a1af076a \
+ gcr.io/iree-oss/mmperf@sha256:7b3bf8a1fafe3603428820865623b133aeecb409d8f79ac7f79b4400b391e207 \
./build_tools/benchmarks/mmperf/run_mmperf.sh "${BUILD_DIR}" "${RESULTS_DIR}"
- name: "Uploading results"
run: |
@@ -113,7 +113,7 @@
- name: "Building mmperf for CUDA"
run: |
./build_tools/github_actions/docker_run.sh \
- gcr.io/iree-oss/mmperf@sha256:bc74f3bbb0e3e8373ab0b35362b6f2a92b1d770432b6711f61403e73a1af076a \
+ gcr.io/iree-oss/mmperf@sha256:7b3bf8a1fafe3603428820865623b133aeecb409d8f79ac7f79b4400b391e207 \
./build_tools/benchmarks/mmperf/build_mmperf.sh "${BUILD_DIR}" "cuda" "${IREE_SHA}"
- name: "Removing unused files"
run: |
diff --git a/build_tools/benchmarks/convperf/build_and_run_convperf.sh b/build_tools/benchmarks/convperf/build_and_run_convperf.sh
new file mode 100755
index 0000000..2eff211
--- /dev/null
+++ b/build_tools/benchmarks/convperf/build_and_run_convperf.sh
@@ -0,0 +1,70 @@
+#!/bin/bash
+
+# Copyright 2022 The IREE Authors
+#
+# Licensed under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# Sets up `convperf` (https://github.com/nod-ai/convperf).
+#
+# `convperf` benchmarks convolution workloads on IREE and other backends such
+# as libxsmm. IREE is included as a submodule.
+#
+# Currently x86 CPU is supported.
+#
+# Usage:
+# ./build_and_run_convperf.sh \
+# <convperf build dir> \
+# <convperf results dir> \
+# <iree commit branch or sha> \
+# <convperf repo dir> (optional)
+
+set -xeuo pipefail
+
+export BUILD_DIR=$1
+export RESULTS_DIR=$2
+export IREE_COMMIT=${3:-"origin/main"}
+export REPO_DIR=${4:-"${CONVPERF_REPO_DIR}"}
+
+pushd ${REPO_DIR}
+source convperf.venv/bin/activate
+
+# Set all repos as a safe directory. Since this repo was created in the
+# Dockerfile under `root`, git will not run commands on this repo as a
+# non-root user unless it is marked safe.
+for i in $(find "${REPO_DIR}" -name '.git' | xargs dirname); do
+ git config --global --add safe.directory $i
+done
+
+# Update IREE.
+pushd external/iree
+git fetch https://github.com/iree-org/iree "${IREE_COMMIT}"
+git checkout "${IREE_COMMIT}"
+git submodule update --init --jobs 8 --depth 1
+popd # external/iree
+
+popd # ${REPO_DIR}
+
+# Build ConvPerf.
+cmake -GNinja -B "${BUILD_DIR}" "${REPO_DIR}"
+cmake --build "${BUILD_DIR}"
+
+# Run ConvPerf for several threading configurations.
+declare -a threads=( 1 2 4 8 16 )
+
+for i in "${threads[@]}"; do
+ python3 "${REPO_DIR}/convperf.py" \
+ --benchmark_tool "${BUILD_DIR}/tools/benchmark_conv" \
+ --runners iree,xsmm \
+ --benchmark_sizes \
+ "${REPO_DIR}/benchmark_sizes/resnet50.json" \
+ --num_threads="$i"
+
+ python "${REPO_DIR}/convperf.py" --visualize --runtimes_file runtimes.json
+ mv runtimes.json "${RESULTS_DIR}/resnet50_thread$i.json"
+ mv convs.png "${RESULTS_DIR}/resnet50_thread$i.png"
+done
+
+
+
diff --git a/build_tools/docker/context/setup_convperf.sh b/build_tools/docker/context/setup_convperf.sh
new file mode 100755
index 0000000..f6ec8aa
--- /dev/null
+++ b/build_tools/docker/context/setup_convperf.sh
@@ -0,0 +1,51 @@
+#!/bin/bash
+
+# Copyright 2022 The IREE Authors
+#
+# Licensed under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# Sets up `convperf` (https://github.com/nod-ai/convperf).
+#
+# `convperf` benchmarks convolution workloads on IREE and other backends such
+# as libxsmm. IREE is included as a submodule.
+#
+# Usage:
+# ./setup_convperf.sh \
+# <convperf repo dir> \
+# <convperf sha>
+
+set -xeuo pipefail
+
+export REPO_DIR="$1"
+export REPO_SHA="$2"
+
+pushd "${REPO_DIR}"
+
+mkdir convperf
+pushd convperf
+git init
+git fetch --depth 1 https://github.com/nod-ai/convperf.git "${REPO_SHA}"
+git checkout "${REPO_SHA}"
+git submodule update --init --recursive --jobs 8 --depth 1
+
+# Checkout a specific commit.
+git checkout "${REPO_SHA}"
+
+# Create virtual environment.
+python3 -m venv convperf.venv
+source convperf.venv/bin/activate
+pip install -r requirements.txt
+
+popd # convperf
+
+# Since the root user clones the convperf repo, we update permissions so that a
+# runner can access this repo, but we don't want to set the executable bit for
+# non-executables because git tracks this, so we then restore any git-tracked
+# changes.
+chmod -R 777 .
+git restore .
+git submodule foreach --recursive git restore .
+
+popd # "${REPO_DIR}"
diff --git a/build_tools/docker/docker_run.sh b/build_tools/docker/docker_run.sh
index df6f4b9..9c3a886 100755
--- a/build_tools/docker/docker_run.sh
+++ b/build_tools/docker/docker_run.sh
@@ -109,6 +109,13 @@
--env SANDBOX_BASE=/dev/shm
)
+ # Some scripts need elevated permissions to control system-level scheduling.
+ # Since we're not using Docker for sandboxing, it is fine to run in
+ # privileged mode.
+ DOCKER_RUN_ARGS+=(
+ --privileged
+ )
+
docker run "${DOCKER_RUN_ARGS[@]}" "$@"
}
diff --git a/build_tools/docker/dockerfiles/convperf.Dockerfile b/build_tools/docker/dockerfiles/convperf.Dockerfile
new file mode 100644
index 0000000..1407c59
--- /dev/null
+++ b/build_tools/docker/dockerfiles/convperf.Dockerfile
@@ -0,0 +1,29 @@
+# Copyright 2021 The IREE Authors
+#
+# Licensed under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+# An image for running `convperf`: https://github.com/nod-ai/convperf.
+#
+# `convperf` benchmarks convolution workloads on IREE and other backends such
+# as libxsmm. IREE is included as a submodule.
+
+FROM gcr.io/iree-oss/perf@sha256:d9ba4befb943c11671747a76bee8f9afd286d88254e9f5c9da905b00bd2a8404
+
+######## OpenMP ########
+RUN apt-get update \
+ && apt-get install -y libomp-14-dev
+##############
+
+######## ConvPerf ########
+COPY build_tools/docker/context/setup_convperf.sh /usr/local/bin
+
+ARG CONVPERF_SHA="ca181bf9f1b959477acf8043eb5b5579fc47a37f"
+
+# Generate a version of mmperf for CPU.
+RUN mkdir -p "/usr/local/src/convperf" \
+ && /usr/local/bin/setup_convperf.sh "/usr/local/src/convperf" "${CONVPERF_SHA}"
+
+ENV CONVPERF_REPO_DIR="/usr/local/src/convperf/convperf"
+############## \
diff --git a/build_tools/docker/dockerfiles/mmperf.Dockerfile b/build_tools/docker/dockerfiles/mmperf.Dockerfile
index 4a98b18..3237870 100644
--- a/build_tools/docker/dockerfiles/mmperf.Dockerfile
+++ b/build_tools/docker/dockerfiles/mmperf.Dockerfile
@@ -13,51 +13,7 @@
# mmperf repo. Later versions of Clang, LLVM, Python and Ubuntu are needed
# to satisfy the dependency requirements of the backends.
-# Ubuntu 22.04.
-FROM ubuntu@sha256:817cfe4672284dcbfee885b1a66094fd907630d610cab329114d036716be49ba
-
-######## Basic ########
-RUN apt-get update \
- && apt-get install -y \
- git \
- wget \
- cmake \
- curl \
- ninja-build
-##############
-
-######## Clang/LLVM ########
-RUN apt-get update \
- && apt-get install -y \
- llvm-14 \
- llvm-14-dev \
- clang-14 \
- clang-tools-14 \
- libclang-common-14-dev \
- libclang-14-dev \
- libclang1-14 \
- clang-format-14 \
- clangd-14 \
- clang-tidy-14 \
- lldb-14 \
- lld-14 \
- libmlir-14-dev \
- mlir-14-tools \
- && ln -s /usr/lib/llvm-14/bin/clang /usr/bin/clang \
- && ln -s /usr/lib/llvm-14/bin/clang++ /usr/bin/clang++
-##############
-
-######## Python ########
-WORKDIR /install-python
-
-ARG PYTHON_VERSION=3.10
-
-COPY runtime/bindings/python/iree/runtime/build_requirements.txt build_tools/docker/context/install_python_deps.sh ./
-RUN ./install_python_deps.sh "${PYTHON_VERSION}" \
- && rm -rf /install-python
-
-ENV PYTHON_BIN /usr/bin/python3
-##############
+FROM gcr.io/iree-oss/perf@sha256:d9ba4befb943c11671747a76bee8f9afd286d88254e9f5c9da905b00bd2a8404
######## CUDA ########
RUN apt-get update \
diff --git a/build_tools/docker/dockerfiles/perf.Dockerfile b/build_tools/docker/dockerfiles/perf.Dockerfile
new file mode 100644
index 0000000..dc2fea7
--- /dev/null
+++ b/build_tools/docker/dockerfiles/perf.Dockerfile
@@ -0,0 +1,55 @@
+# Copyright 2022 The IREE Authors
+#
+# Licensed under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+# An image that includes basic packages, compiler and Python support for
+# running perf-related workflows e.g. mmperf, convperf.
+
+# Ubuntu 22.04.
+FROM ubuntu@sha256:817cfe4672284dcbfee885b1a66094fd907630d610cab329114d036716be49ba
+
+######## Basic ########
+RUN apt-get update \
+ && apt-get install -y \
+ git \
+ wget \
+ cmake \
+ curl \
+ ninja-build \
+ numactl
+##############
+
+######## Clang/LLVM ########
+RUN apt-get update \
+ && apt-get install -y \
+ llvm-14 \
+ llvm-14-dev \
+ clang-14 \
+ clang-tools-14 \
+ libclang-common-14-dev \
+ libclang-14-dev \
+ libclang1-14 \
+ clang-format-14 \
+ clangd-14 \
+ clang-tidy-14 \
+ lldb-14 \
+ lld-14 \
+ libmlir-14-dev \
+ mlir-14-tools \
+ && ln -s /usr/lib/llvm-14/bin/clang /usr/bin/clang \
+ && ln -s /usr/lib/llvm-14/bin/clang++ /usr/bin/clang++
+##############
+
+######## Python ########
+WORKDIR /install-python
+
+ARG PYTHON_VERSION=3.10
+
+COPY runtime/bindings/python/iree/runtime/build_requirements.txt build_tools/docker/context/install_python_deps.sh ./
+RUN ./install_python_deps.sh "${PYTHON_VERSION}" \
+ && rm -rf /install-python
+
+ENV PYTHON_BIN /usr/bin/python3
+##############
diff --git a/build_tools/docker/manage_images.py b/build_tools/docker/manage_images.py
index efee09d..8be1b3d 100755
--- a/build_tools/docker/manage_images.py
+++ b/build_tools/docker/manage_images.py
@@ -50,7 +50,9 @@
"riscv": ["base"],
"gradle-android": ["base"],
"frontends": ["android"],
- "mmperf": [],
+ "perf": [],
+ "mmperf": ["perf"],
+ "convperf": ["perf"],
"shark": [],
"swiftshader": ["base"],
"samples": ["swiftshader"],
diff --git a/build_tools/docker/prod_digests.txt b/build_tools/docker/prod_digests.txt
index 693d98a..66655ee 100644
--- a/build_tools/docker/prod_digests.txt
+++ b/build_tools/docker/prod_digests.txt
@@ -13,4 +13,6 @@
gcr.io/iree-oss/shark@sha256:c72ef54dcb6ec485e8a96b0dfc43307875f4c4c7619f7fdc60bf5220a5672259
gcr.io/iree-oss/base-bleeding-edge@sha256:479eefb76447c865cf58c5be7ca9fe33f48584b474a1da3dfaa125aad2510463
gcr.io/iree-oss/swiftshader-bleeding-edge@sha256:c22afc61198e14a98e06e5261149de74c629acd2bc61b82e57ec90e5461b69be
-gcr.io/iree-oss/mmperf@sha256:bc74f3bbb0e3e8373ab0b35362b6f2a92b1d770432b6711f61403e73a1af076a
+gcr.io/iree-oss/mmperf@sha256:7b3bf8a1fafe3603428820865623b133aeecb409d8f79ac7f79b4400b391e207
+gcr.io/iree-oss/convperf@sha256:9c6a6cf89d4f667246382bcce96f8542f626872d9014184b3a8118750af8f878
+gcr.io/iree-oss/perf@sha256:d9ba4befb943c11671747a76bee8f9afd286d88254e9f5c9da905b00bd2a8404
