Add SHARK tank workflow (#10747)
Runs SHARK nightly. Example report: https://storage.googleapis.com/shark-benchmark-artifacts/2022-10-24.sha_1d33.timestamp_1666596893/summary.html
diff --git a/.github/workflows/run_shark_tank.yml b/.github/workflows/run_shark_tank.yml
new file mode 100644
index 0000000..2c8415c
--- /dev/null
+++ b/.github/workflows/run_shark_tank.yml
@@ -0,0 +1,139 @@
+name: SHARK
+
+on:
+ schedule:
+ - cron: '0 10 * * *'
+ workflow_dispatch:
+
+jobs:
+ setup:
+ runs-on: ubuntu-20.04
+ env:
+ # The commit being checked out is the merge commit for the PR. Its first
+ # parent will be the tip of main.
+ BASE_REF: HEAD^
+ PR_DESCRIPTION: ${{ github.event.pull_request.body }}
+ outputs:
+ should-run: ${{ steps.configure.outputs.should-run }}
+ ci-stage: ${{ steps.configure.outputs.ci-stage }}
+ runner-env: ${{ steps.configure.outputs.runner-env }}
+ runner-group: ${{ steps.configure.outputs.runner-group }}
+ write-caches: ${{ steps.configure.outputs.write-caches }}
+ shark-sha: ${{ steps.shark.outputs.shark-sha }}
+ artifact-upload-dir: ${{ steps.shark.outputs.artifact-upload-dir }}
+ steps:
+ - name: "Checking out repository"
+ uses: actions/checkout@7884fcad6b5d53d10323aee724dc68d8b9096a2e # v2
+ with:
+ # We need the parent commit to do a diff
+ fetch-depth: 2
+ - name: "Configuring CI options"
+ id: configure
+ run: |
+ # Just informative logging. There should only be two commits in the
+ # history here, but limiting the depth helps when copying from a local
+ # repo instead of using checkout, e.g. with
+ # https://github.com/nektos/act where there will be more.
+ git log --oneline --graph --max-count=3
+ ./build_tools/github_actions/configure_ci.py
+ - name: "Checking out SHARK tank"
+ uses: actions/checkout@7884fcad6b5d53d10323aee724dc68d8b9096a2e # v2
+ with:
+ repository: nod-ai/SHARK
+ path: ${{ github.workspace }}/SHARK
+ - name: "Calculating version info"
+ id: shark
+ run: |
+ cd ${{ github.workspace }}/SHARK
+ export SHARK_SHA=`git rev-parse --short=4 HEAD`
+ echo "shark-sha=${SHARK_SHA}" >> $GITHUB_OUTPUT
+ export DIR_NAME="$(date +'%Y-%m-%d').sha_${SHARK_SHA}.timestamp_$(date +'%s')"
+ export GCS_ARTIFACT_DIR="gs://shark-benchmark-artifacts/${DIR_NAME}"
+ echo "artifact-upload-dir=${GCS_ARTIFACT_DIR}" >> $GITHUB_OUTPUT
+
+ benchmark_cpu:
+ needs: setup
+ if: needs.setup.outputs.should-run == 'true'
+ runs-on:
+ - self-hosted # must come first
+ - runner-group=${{ needs.setup.outputs.runner-group }}
+ - environment=${{ needs.setup.outputs.runner-env }}
+ - cpu
+ - os-family=Linux
+ env:
+ SHARK_SHA: ${{ needs.setup.outputs.shark-sha }}
+ GCS_UPLOAD_DIR: ${{ needs.setup.outputs.artifact-upload-dir }}
+ SHARK_OUTPUT_DIR: shark-output-dir
+ steps:
+ - name: "Checking out repository"
+ uses: actions/checkout@7884fcad6b5d53d10323aee724dc68d8b9096a2e # v2
+ with:
+ submodules: true
+ - name: "Benchmarking SHARK tank on CPU"
+ # Only Tensorflow is working at the moment so limit benchmarking to TF models.
+ run: |
+ ./build_tools/github_actions/docker_run.sh \
+ gcr.io/iree-oss/shark@sha256:6ca44c4236370054283435d365fefc1a2825f6b92c712e6fa5037309e38c739e \
+ ./build_tools/benchmarks/shark/run_shark.sh "${SHARK_SHA}" "cpu" "cpu" "${SHARK_OUTPUT_DIR}"
+ - name: "Uploading artifacts"
+ run: |
+ gcloud alpha storage cp "${SHARK_OUTPUT_DIR}/**" "${GCS_UPLOAD_DIR}/"
+
+ benchmark_cuda:
+ needs: setup
+ if: needs.setup.outputs.should-run == 'true'
+ runs-on:
+ - self-hosted # must come first
+ - runner-group=${{ needs.setup.outputs.runner-group }}
+ - environment=${{ needs.setup.outputs.runner-env }}
+ - gpu
+ - os-family=Linux
+ env:
+ SHARK_SHA: ${{ needs.setup.outputs.shark-sha }}
+ GCS_UPLOAD_DIR: ${{ needs.setup.outputs.artifact-upload-dir }}
+ SHARK_OUTPUT_DIR: shark-output-dir
+ steps:
+ - name: "Checking out repository"
+ uses: actions/checkout@7884fcad6b5d53d10323aee724dc68d8b9096a2e # v2
+ with:
+ submodules: true
+ - name: "Benchmarking SHARK tank on CUDA"
+ # Only Tensorflow is working at the moment so limit benchmarking to TF models.
+ run: |
+ ./build_tools/github_actions/docker_run.sh \
+ --gpus all \
+ gcr.io/iree-oss/shark@sha256:6ca44c4236370054283435d365fefc1a2825f6b92c712e6fa5037309e38c739e \
+ ./build_tools/benchmarks/shark/run_shark.sh "${SHARK_SHA}" "cuda" "cuda" "${SHARK_OUTPUT_DIR}"
+ - name: "Uploading artifacts"
+ run: |
+ gcloud alpha storage cp "${SHARK_OUTPUT_DIR}/**" "${GCS_UPLOAD_DIR}/"
+
+ generate_report:
+ needs: [setup, benchmark_cpu, benchmark_cuda]
+ if: needs.setup.outputs.should-run == 'true'
+ runs-on:
+ - self-hosted # must come first
+ - runner-group=${{ needs.setup.outputs.runner-group }}
+ - environment=${{ needs.setup.outputs.runner-env }}
+ - cpu
+ - os-family=Linux
+ env:
+ GCS_UPLOAD_DIR: ${{ needs.setup.outputs.artifact-upload-dir }}
+ BENCHMARK_RESULTS_DIR: benchmark-results
+ steps:
+ - name: "Checking out repository"
+ uses: actions/checkout@7884fcad6b5d53d10323aee724dc68d8b9096a2e # v2
+ with:
+ submodules: true
+ - name: "Download benchmark results"
+ run: |
+ mkdir "${BENCHMARK_RESULTS_DIR}"
+ gcloud alpha storage cp "${GCS_UPLOAD_DIR}/**" "${BENCHMARK_RESULTS_DIR}/"
+ - name: "Generating report"
+ run: |
+ ./build_tools/github_actions/docker_run.sh \
+ gcr.io/iree-oss/shark@sha256:6ca44c4236370054283435d365fefc1a2825f6b92c712e6fa5037309e38c739e \
+ ./build_tools/benchmarks/shark/run_report.sh "${BENCHMARK_RESULTS_DIR}" "${BENCHMARK_RESULTS_DIR}/summary.html"
+ - name: "Uploading summary"
+ run: |
+ gcloud alpha storage cp "${BENCHMARK_RESULTS_DIR}/summary.html" "${GCS_UPLOAD_DIR}/"
diff --git a/build_tools/benchmarks/reporting/requirements.txt b/build_tools/benchmarks/reporting/requirements.txt
new file mode 100644
index 0000000..202a5e7
--- /dev/null
+++ b/build_tools/benchmarks/reporting/requirements.txt
@@ -0,0 +1,2 @@
+pandas==1.5.0
+jinja2==2.1.1
\ No newline at end of file
diff --git a/build_tools/benchmarks/shark/run_report.sh b/build_tools/benchmarks/shark/run_report.sh
new file mode 100755
index 0000000..7e25b26
--- /dev/null
+++ b/build_tools/benchmarks/shark/run_report.sh
@@ -0,0 +1,40 @@
+#!/bin/bash
+
+# Copyright 2022 The IREE Authors
+#
+# Licensed under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# Runs SHARK tank using both SHARK-Runtime and IREE-Runtime, producing benchmark
+# numbers.
+#
+# Usage:
+# ./run_report.sh \
+# <RAW_RESULTS_DIR> The directory consisting of the csv files to process. \
+# <SUMMARY_PATH> The path of the output HTML file.
+
+set -xeuo pipefail
+
+RAW_RESULTS_DIR=$1
+SUMMARY_PATH=$2
+
+python3 -m venv iree.venv
+source iree.venv/bin/activate
+pip install --upgrade pip
+pip install pandas
+pip install jinja2
+
+export SHARK_VERSION=`ls ${RAW_RESULTS_DIR} | grep "cpu_shark" | sed -e "s/^cpu_shark_\(.*\).csv$/\1/g"`
+export IREE_VERSION=`ls ${RAW_RESULTS_DIR} | grep "cpu_iree" | sed -e "s/^cpu_iree_\(.*\).csv$/\1/g"`
+
+python3 ./build_tools/benchmarks/reporting/parse_shark_benchmarks.py \
+ --shark_version="${SHARK_VERSION}" \
+ --iree_version="${IREE_VERSION}" \
+ --cpu_shark_csv="${RAW_RESULTS_DIR}/cpu_shark_${SHARK_VERSION}.csv" \
+ --cpu_iree_csv="${RAW_RESULTS_DIR}/cpu_iree_${IREE_VERSION}.csv" \
+ --cpu_baseline_csv="${RAW_RESULTS_DIR}/cpu_iree_${IREE_VERSION}.csv" \
+ --gpu_shark_csv="${RAW_RESULTS_DIR}/cuda_shark_${SHARK_VERSION}.csv" \
+ --gpu_iree_csv="${RAW_RESULTS_DIR}/cuda_iree_${IREE_VERSION}.csv" \
+ --gpu_baseline_csv="${RAW_RESULTS_DIR}/cuda_iree_${IREE_VERSION}.csv" \
+ --output_path="${SUMMARY_PATH}"
diff --git a/build_tools/benchmarks/shark/run_shark.sh b/build_tools/benchmarks/shark/run_shark.sh
new file mode 100755
index 0000000..b040760
--- /dev/null
+++ b/build_tools/benchmarks/shark/run_shark.sh
@@ -0,0 +1,60 @@
+#!/bin/bash
+
+# Copyright 2022 The IREE Authors
+#
+# Licensed under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# Runs SHARK tank using both SHARK-Runtime and IREE-Runtime, producing benchmark
+# numbers.
+#
+# Usage:
+# ./run_shark.sh \
+# <SHA of https://github.com/nod-ai/SHARK.git to pin to> \
+# <pytest regex> e.g. "cpu", "cuda", "cuda and torch".
+# <driver> e.g. "cpu", "cuda", "vulkan"
+# <output directory>
+
+set -xeuo pipefail
+
+export SHARK_SHA=$1
+export BENCHMARK_REGEX=$2
+export DRIVER=$3
+export SHARK_OUTPUT_DIR=`pwd`/$4
+
+mkdir "${SHARK_OUTPUT_DIR}"
+
+git clone https://github.com/nod-ai/SHARK.git
+cd SHARK
+git reset --hard ${SHARK_SHA}
+
+# Remove existing data.
+rm -rf ./shark_tmp
+rm -rf ~/.local/shark_tank
+
+# Run with SHARK-Runtime.
+PYTHON=python3.10 VENV_DIR=shark.venv BENCHMARK=1 IMPORTER=1 ./setup_venv.sh
+source shark.venv/bin/activate
+export SHARK_VERSION=`pip show iree-compiler | grep Version | sed -e "s/^Version: \(.*\)$/\1/g"`
+pytest --benchmark --maxfail=500 tank/test_models.py -k "${BENCHMARK_REGEX}" || true
+
+echo "######################################################"
+echo "Benchmarks for SHARK-Runtime Complete"
+cat bench_results.csv
+mv bench_results.csv "${SHARK_OUTPUT_DIR}/${DRIVER}_shark_${SHARK_VERSION}.csv"
+echo "######################################################"
+deactivate
+
+# Run with IREE.
+PYTHON=python3.10 VENV_DIR=iree.venv BENCHMARK=1 IMPORTER=1 USE_IREE=1 ./setup_venv.sh
+source iree.venv/bin/activate
+export IREE_VERSION=`pip show iree-compiler | grep Version | sed -e "s/^Version: \(.*\)$/\1/g"`
+pytest --benchmark --maxfail=500 tank/test_models.py -k "${BENCHMARK_REGEX}" || true
+
+echo "######################################################"
+echo "Benchmarks for IREE Complete"
+cat bench_results.csv
+mv bench_results.csv "${SHARK_OUTPUT_DIR}/${DRIVER}_iree_${IREE_VERSION}.csv"
+echo "######################################################"
+deactivate
diff --git a/build_tools/docker/manage_images.py b/build_tools/docker/manage_images.py
index bf55cf1..2939a9a 100755
--- a/build_tools/docker/manage_images.py
+++ b/build_tools/docker/manage_images.py
@@ -50,6 +50,7 @@
"riscv": ["base"],
"gradle-android": ["base"],
"frontends": ["android"],
+ "shark": [],
"swiftshader": ["base"],
"samples": ["swiftshader"],
"frontends-swiftshader": ["frontends", "swiftshader"],
diff --git a/build_tools/docker/prod_digests.txt b/build_tools/docker/prod_digests.txt
index a3b16ba..a31fd6b 100644
--- a/build_tools/docker/prod_digests.txt
+++ b/build_tools/docker/prod_digests.txt
@@ -10,3 +10,4 @@
gcr.io/iree-oss/emscripten@sha256:8ccc1c8de11919faf23aeaa585b13e5c5050952db76c101d6f61367280a3546f
gcr.io/iree-oss/android@sha256:76c2a52dcd6d07601227b965ac87d021c1d2d5e2d01f46ad58da28c89267f2ab
gcr.io/iree-oss/manylinux2014_x86_64-release@sha256:b09c10868f846308bad2eab253a77d0a3f097816c40342bc289d8e62509bc5f9
+gcr.io/iree-oss/shark@sha256:6ca44c4236370054283435d365fefc1a2825f6b92c712e6fa5037309e38c739e
diff --git a/build_tools/docker/shark/Dockerfile b/build_tools/docker/shark/Dockerfile
new file mode 100644
index 0000000..aaaf939
--- /dev/null
+++ b/build_tools/docker/shark/Dockerfile
@@ -0,0 +1,73 @@
+# Copyright 2021 The IREE Authors
+#
+# Licensed under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+# An image for running SHARK tank: https://github.com/nod-ai/SHARK.
+
+FROM ubuntu@sha256:fd25e706f3dea2a5ff705dbc3353cf37f08307798f3e360a13e9385840f73fb3
+
+######## Basic ########
+WORKDIR /base
+
+# Must set the timezone explicitly to avoid hanging when installing tzdata.
+# https://grigorkh.medium.com/fix-tzdata-hangs-docker-image-build-cdb52cc3360d
+ENV TZ=UTC
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+
+RUN apt-get update \
+ && apt-get install -y \
+ git \
+ wget
+
+WORKDIR /
+##############
+
+######## Python ########
+WORKDIR /install-python
+
+RUN apt-get update \
+ && apt-get install -y software-properties-common \
+ && add-apt-repository -y ppa:deadsnakes/ppa \
+ && apt-get update \
+ && apt-get install -y \
+ python3.10 \
+ python3.10-dev \
+ && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.10 1 \
+ && apt-get install -y \
+ python3-pip \
+ python3-setuptools \
+ python3-distutils \
+ python3-venv \
+ python3.10-venv
+
+ENV PYTHON_BIN /usr/bin/python3
+
+WORKDIR /
+##############
+
+######## Vulkan ########
+WORKDIR /install-vulkan
+ARG VULKAN_SDK_VERSION=1.2.154.0
+
+RUN wget -q \
+ # This file disappeared from the canonical source:
+ # "https://sdk.lunarg.com/sdk/download/${VULKAN_SDK_VERSION?}/linux/vulkansdk-linux-${VULKAN_SDK_VERSION?}.tar.gz"
+ "https://storage.googleapis.com/iree-shared-files/vulkansdk-linux-${VULKAN_SDK_VERSION?}.tar.gz" \
+ && mkdir -p /opt/vulkan-sdk \
+ && tar -xzf "vulkansdk-linux-${VULKAN_SDK_VERSION?}.tar.gz" -C /opt/vulkan-sdk \
+ && rm -rf /install-vulkan
+WORKDIR /
+
+ENV VULKAN_SDK="/opt/vulkan-sdk/${VULKAN_SDK_VERSION}/x86_64"
+
+ENV PATH="${VULKAN_SDK}/bin:$PATH"
+
+# Symlink the Vulkan loader to a system library directory. This is needed to
+# allow Vulkan applications to find the Vulkan loader. It also avoids using
+# LD_LIBRARY_PATH, which is not supported well by Docker.
+RUN ln -s "${VULKAN_SDK}/lib/libvulkan.so" /usr/lib/x86_64-linux-gnu/ \
+ && ln -s "${VULKAN_SDK}/lib/libvulkan.so.1" /usr/lib/x86_64-linux-gnu/
+
+############## \
