Drop convperf and mmperf Dockerfiles and workflows. (#14903)
These comparative benchmarking efforts are being migrated to
https://github.com/openxla/openxla-benchmark.
The workflows have also been silently skipped since
https://github.com/openxla/iree/commit/8ae0f520cbc9d52b795032dd687f11438f0cdfe0:
* https://github.com/openxla/iree/actions/workflows/run_mmperf.yml
* https://github.com/openxla/iree/actions/workflows/run_convperf.yml
(There have been some discussions about migrating Dockerfiles to a new
build structure and repository like
https://github.com/nod-ai/base-docker-images, so I'd like to prune
unused Dockerfiles ahead of that if possible)
diff --git a/.github/workflows/run_convperf.yml b/.github/workflows/run_convperf.yml
deleted file mode 100644
index 4694b86..0000000
--- a/.github/workflows/run_convperf.yml
+++ /dev/null
@@ -1,95 +0,0 @@
-# Copyright 2022 The IREE Authors
-#
-# Licensed under the Apache License v2.0 with LLVM Exceptions.
-# See https://llvm.org/LICENSE.txt for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#
-# Workflow for running `convperf` (https://github.com/nod-ai/convperf).
-# `convperf` benchmarks convolution workloads on IREE and other backends such
-# as `libsxmm`, etc.
-#
-# The workflow runs benchmarks on CPU and uploads results to the
-# `convperf-benchmark-artifacts` GC bucket.
-
-name: convperf
-
-on:
- schedule:
- - cron: '0 16 * * *'
- workflow_dispatch:
-
-concurrency:
- # A PR number if a pull request and otherwise the commit hash. This cancels
- # queued and in-progress runs for the same PR (presubmit) or commit
- # (postsubmit). The workflow name is prepended to avoid conflicts between
- # different workflows.
- group: ${{ github.workflow }}-${{ github.event.number || github.sha }}
- cancel-in-progress: true
-
-env:
- GCS_DIR: gs://iree-github-actions-${{ github.event_name == 'pull_request' && 'presubmit' || 'postsubmit' }}-artifacts/${{ github.run_id }}/${{ github.run_attempt }}
-
-jobs:
- setup:
- runs-on: ubuntu-20.04
- env:
- # The commit being checked out is the merge commit for the PR. Its first
- # parent will be the tip of main.
- BASE_REF: HEAD^
- PR_TITLE: ${{ github.event.pull_request.title }}
- PR_BODY: ${{ github.event.pull_request.body }}
- IREE_SHA: ${{ github.sha }}
- outputs:
- artifact-upload-dir: ${{ steps.iree.outputs.artifact-upload-dir }}
- should-run: ${{ steps.configure.outputs.should-run }}
- runner-env: ${{ steps.configure.outputs.runner-env }}
- runner-group: ${{ steps.configure.outputs.runner-group }}
- steps:
- - name: "Checking out repository"
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
- with:
- # We need the parent commit to do a diff
- fetch-depth: 2
- - name: "Configuring CI options"
- id: configure
- run: |
- # Just informative logging. There should only be two commits in the
- # history here, but limiting the depth helps when copying from a local
- # repo instead of using checkout, e.g. with
- # https://github.com/nektos/act where there will be more.
- git log --oneline --graph --max-count=3
- ./build_tools/github_actions/configure_ci.py
- - name: "Calculating version info"
- id: iree
- run: |
- export GCS_ARTIFACT_DIR="$(date +'%Y-%m-%d').sha_${IREE_SHA}.timestamp_$(date +'%s')"
- echo "artifact-upload-dir=${GCS_ARTIFACT_DIR}" >> $GITHUB_OUTPUT
-
- build_and_benchmark_cpu:
- needs: setup
- if: needs.setup.outputs.should-run == 'true'
- runs-on:
- - self-hosted # must come first
- - runner-group=${{ needs.setup.outputs.runner-group }}
- - environment=${{ needs.setup.outputs.runner-env }}
- - cpu
- - os-family=Linux
- env:
- IREE_SHA: ${{ github.sha }}
- BUILD_DIR: convperf-build
- RESULTS_DIR: convperf-results
- GCS_UPLOAD_PARENT_DIR: "gs://convperf-benchmark-artifacts/cpu"
- GCS_UPLOAD_DIR_NAME: ${{ needs.setup.outputs.artifact-upload-dir }}
- steps:
- - name: "Checking out repository"
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
- - name: "Running convperf for CPU"
- run: |
- mkdir ${RESULTS_DIR}
- ./build_tools/github_actions/docker_run.sh \
- gcr.io/iree-oss/convperf@sha256:8e242ad71896a791c9be53b0906b7c8e96a2d3bad3f7431622cc797a6f89ba0f \
- ./build_tools/benchmarks/convperf/build_and_run_convperf.sh "${BUILD_DIR}" "${RESULTS_DIR}" "${IREE_SHA}"
- - name: "Uploading results"
- run: |
- gcloud storage cp "${RESULTS_DIR}/**" "${GCS_UPLOAD_PARENT_DIR}/${GCS_UPLOAD_DIR_NAME}/"
- gcloud storage cp "${RESULTS_DIR}/**" "${GCS_UPLOAD_PARENT_DIR}/latest/"
diff --git a/.github/workflows/run_mmperf.yml b/.github/workflows/run_mmperf.yml
deleted file mode 100644
index 1e9efad..0000000
--- a/.github/workflows/run_mmperf.yml
+++ /dev/null
@@ -1,182 +0,0 @@
-# Copyright 2022 The IREE Authors
-#
-# Licensed under the Apache License v2.0 with LLVM Exceptions.
-# See https://llvm.org/LICENSE.txt for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#
-# Workflow for running `mmperf` (https://github.com/mmperf/mmperf).
-# `mmperf` benchmarks matrix-multiply workloads on IREE and other backends such
-# as RUY, TVM, Halide, CuBLAS, etc.
-#
-# The workflow runs benchmarks on CPU and GPU and uploads results to the
-# `mmperf-benchmark-artifacts` GC bucket.
-
-name: mmperf
-
-on:
- schedule:
- - cron: '0 13 * * *'
- workflow_dispatch:
-
-concurrency:
- # A PR number if a pull request and otherwise the commit hash. This cancels
- # queued and in-progress runs for the same PR (presubmit) or commit
- # (postsubmit). The workflow name is prepended to avoid conflicts between
- # different workflows.
- group: ${{ github.workflow }}-${{ github.event.number || github.sha }}
- cancel-in-progress: true
-
-env:
- GCS_DIR: gs://iree-github-actions-${{ github.event_name == 'pull_request' && 'presubmit' || 'postsubmit' }}-artifacts/${{ github.run_id }}/${{ github.run_attempt }}
-
-jobs:
- setup:
- runs-on: ubuntu-20.04
- env:
- # The commit being checked out is the merge commit for the PR. Its first
- # parent will be the tip of main.
- BASE_REF: HEAD^
- PR_TITLE: ${{ github.event.pull_request.title }}
- PR_BODY: ${{ github.event.pull_request.body }}
- IREE_SHA: ${{ github.sha }}
- outputs:
- artifact-upload-dir: ${{ steps.iree.outputs.artifact-upload-dir }}
- should-run: ${{ steps.configure.outputs.should-run }}
- runner-env: ${{ steps.configure.outputs.runner-env }}
- runner-group: ${{ steps.configure.outputs.runner-group }}
- steps:
- - name: "Checking out repository"
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
- with:
- # We need the parent commit to do a diff
- fetch-depth: 2
- - name: "Configuring CI options"
- id: configure
- run: |
- # Just informative logging. There should only be two commits in the
- # history here, but limiting the depth helps when copying from a local
- # repo instead of using checkout, e.g. with
- # https://github.com/nektos/act where there will be more.
- git log --oneline --graph --max-count=3
- ./build_tools/github_actions/configure_ci.py
- - name: "Calculating version info"
- id: iree
- run: |
- export GCS_ARTIFACT_DIR="$(date +'%Y-%m-%d').sha_${IREE_SHA}.timestamp_$(date +'%s')"
- echo "artifact-upload-dir=${GCS_ARTIFACT_DIR}" >> $GITHUB_OUTPUT
-
- build_and_benchmark_cpu:
- needs: setup
- if: needs.setup.outputs.should-run == 'true'
- runs-on:
- - self-hosted # must come first
- - runner-group=${{ needs.setup.outputs.runner-group }}
- - environment=${{ needs.setup.outputs.runner-env }}
- - cpu
- - os-family=Linux
- env:
- IREE_SHA: ${{ github.sha }}
- BUILD_DIR: mmperf-build-cpu
- RESULTS_DIR: mmperf-results-cpu
- GCS_UPLOAD_PARENT_DIR: "gs://mmperf-benchmark-artifacts/cpu"
- GCS_UPLOAD_DIR_NAME: ${{ needs.setup.outputs.artifact-upload-dir }}
- steps:
- - name: "Checking out repository"
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
- - name: "Building mmperf for CPU"
- run: |
- ./build_tools/github_actions/docker_run.sh \
- gcr.io/iree-oss/mmperf@sha256:dfeebcaf8f417275d44c7b8d9a1360c6a26d280b526779c9e64384798cfd9c92 \
- ./build_tools/benchmarks/mmperf/build_mmperf.sh "${BUILD_DIR}" "cpu" "${IREE_SHA}"
- - name: "Running mmperf on CPU"
- run: |
- mkdir ${RESULTS_DIR}
- ./build_tools/github_actions/docker_run.sh \
- gcr.io/iree-oss/mmperf@sha256:dfeebcaf8f417275d44c7b8d9a1360c6a26d280b526779c9e64384798cfd9c92 \
- ./build_tools/benchmarks/mmperf/run_mmperf.sh "${BUILD_DIR}" "${RESULTS_DIR}"
- - name: "Uploading results"
- run: |
- gcloud storage cp "${RESULTS_DIR}/latest/**" "${GCS_UPLOAD_PARENT_DIR}/${GCS_UPLOAD_DIR_NAME}/"
- gcloud storage cp "${RESULTS_DIR}/latest/matmul.png" "${GCS_UPLOAD_PARENT_DIR}/matmul.png"
-
- build_cuda:
- needs: setup
- if: needs.setup.outputs.should-run == 'true'
- runs-on:
- - self-hosted # must come first
- - runner-group=${{ needs.setup.outputs.runner-group }}
- - environment=${{ needs.setup.outputs.runner-env }}
- - cpu
- - os-family=Linux
- env:
- IREE_SHA: ${{ github.sha }}
- BUILD_DIR: mmperf-build-cuda
- outputs:
- build-dir: ${{ env.BUILD_DIR }}
- build-dir-archive: ${{ steps.archive.outputs.build-dir-archive }}
- build-dir-gcs-artifact: ${{ steps.upload.outputs.build-dir-gcs-artifact }}
- steps:
- - name: "Checking out repository"
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
- - name: "Building mmperf for CUDA"
- run: |
- ./build_tools/github_actions/docker_run.sh \
- gcr.io/iree-oss/mmperf@sha256:dfeebcaf8f417275d44c7b8d9a1360c6a26d280b526779c9e64384798cfd9c92 \
- ./build_tools/benchmarks/mmperf/build_mmperf.sh "${BUILD_DIR}" "cuda" "${IREE_SHA}"
- - name: "Removing unused files"
- run: |
- find "${BUILD_DIR}" -type f -name "*.a" -o -type f -name "*.o" \
- -print \
- -delete
- - name: "Creating build dir archive"
- id: archive
- env:
- BUILD_DIR_ARCHIVE: ${{ env.BUILD_DIR }}.tar.zst
- run: |
- tar -I 'zstd -T0' \
- -cf ${BUILD_DIR_ARCHIVE} ${BUILD_DIR}
- echo "build-dir-archive=${BUILD_DIR_ARCHIVE}" >> "${GITHUB_OUTPUT}"
- - name: "Uploading build dir archive"
- id: upload
- env:
- BUILD_DIR_ARCHIVE: ${{ steps.archive.outputs.build-dir-archive }}
- BUILD_DIR_GCS_ARTIFACT: ${{ env.GCS_DIR }}/${{ steps.archive.outputs.build-dir-archive }}
- run: |
- gcloud storage cp "${BUILD_DIR_ARCHIVE}" "${BUILD_DIR_GCS_ARTIFACT}"
- echo "build-dir-gcs-artifact=${BUILD_DIR_GCS_ARTIFACT}" >> "${GITHUB_OUTPUT}"
-
- benchmark_cuda:
- needs: [setup, build_cuda]
- if: needs.setup.outputs.should-run == 'true'
- runs-on:
- - self-hosted # must come first
- - runner-group=${{ needs.setup.outputs.runner-group }}
- - environment=${{ needs.setup.outputs.runner-env }}
- - machine-type=a2-highgpu-1g
- - os-family=Linux
- env:
- RESULTS_DIR: mmperf-results-cuda
- GCS_UPLOAD_PARENT_DIR: "gs://mmperf-benchmark-artifacts/cuda"
- GCS_UPLOAD_DIR_NAME: ${{ needs.setup.outputs.artifact-upload-dir }}
- BUILD_DIR: ${{ needs.build_cuda.outputs.build-dir }}
- BUILD_DIR_ARCHIVE: ${{ needs.build_cuda.outputs.build-dir-archive }}
- BUILD_DIR_GCS_ARTIFACT: ${{ needs.build_cuda.outputs.build-dir-gcs-artifact }}
- steps:
- - name: "Checking out repository"
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
- - name: "Downloading build dir archive"
- run: gcloud storage cp "${BUILD_DIR_GCS_ARTIFACT}" "${BUILD_DIR_ARCHIVE}"
- - name: "Extracting build dir archive"
- run: tar -xf "${BUILD_DIR_ARCHIVE}"
- - name: "Running mmperf on CUDA"
- run: |
- mkdir ${RESULTS_DIR}
- ./build_tools/github_actions/docker_run.sh \
- --gpus all \
- gcr.io/iree-oss/mmperf@sha256:dfeebcaf8f417275d44c7b8d9a1360c6a26d280b526779c9e64384798cfd9c92 \
- ./build_tools/benchmarks/mmperf/run_mmperf.sh "${BUILD_DIR}" "${RESULTS_DIR}"
- - name: "Uploading results"
- run: |
- export GCS_DIR_NAME="$(date +'%Y-%m-%d').$(date +'%s')"
- gcloud storage cp "${RESULTS_DIR}/latest/**" "${GCS_UPLOAD_PARENT_DIR}/${GCS_UPLOAD_DIR_NAME}/"
- gcloud storage cp "${RESULTS_DIR}/latest/matmul.png" "${GCS_UPLOAD_PARENT_DIR}/matmul.png"
diff --git a/build_tools/benchmarks/convperf/build_and_run_convperf.sh b/build_tools/benchmarks/convperf/build_and_run_convperf.sh
deleted file mode 100755
index efc6cd7..0000000
--- a/build_tools/benchmarks/convperf/build_and_run_convperf.sh
+++ /dev/null
@@ -1,70 +0,0 @@
-#!/bin/bash
-
-# Copyright 2022 The IREE Authors
-#
-# Licensed under the Apache License v2.0 with LLVM Exceptions.
-# See https://llvm.org/LICENSE.txt for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#
-# Sets up `convperf` (https://github.com/nod-ai/convperf).
-#
-# `convperf` benchmarks convolution workloads on IREE and other backends such
-# as libxsmm. IREE is included as a submodule.
-#
-# Currently x86 CPU is supported.
-#
-# Usage:
-# ./build_and_run_convperf.sh \
-# <convperf build dir> \
-# <convperf results dir> \
-# <iree commit branch or sha> \
-# <convperf repo dir> (optional)
-
-set -xeuo pipefail
-
-export BUILD_DIR=$1
-export RESULTS_DIR=$2
-export IREE_COMMIT=${3:-"origin/main"}
-export REPO_DIR=${4:-"${CONVPERF_REPO_DIR}"}
-
-pushd ${REPO_DIR}
-source convperf.venv/bin/activate
-
-# Set all repos as a safe directory. Since this repo was created in the
-# Dockerfile under `root`, git will not run commands on this repo as a
-# non-root user unless it is marked safe.
-for i in $(find "${REPO_DIR}" -name '.git' | xargs dirname); do
- git config --global --add safe.directory $i
-done
-
-# Update IREE.
-pushd external/iree
-git fetch https://github.com/openxla/iree "${IREE_COMMIT}"
-git checkout "${IREE_COMMIT}"
-git submodule update --init --jobs 8 --depth 1
-popd # external/iree
-
-popd # ${REPO_DIR}
-
-# Build ConvPerf.
-cmake -GNinja \
- -DCMAKE_C_COMPILER="${CC:-clang}" \
- -DCMAKE_CXX_COMPILER="${CXX:-clang++}" \
- -B "${BUILD_DIR}" "${REPO_DIR}"
-cmake --build "${BUILD_DIR}"
-
-# Run ConvPerf for several threading configurations.
-declare -a threads=( 1 2 4 8 16 )
-
-for i in "${threads[@]}"; do
- python3 "${REPO_DIR}/convperf.py" \
- --benchmark_tool "${BUILD_DIR}/tools/benchmark_conv" \
- --runners iree,xsmm \
- --benchmark_sizes \
- "${REPO_DIR}/benchmark_sizes/resnet50.json" \
- --num_threads="$i"
-
- python "${REPO_DIR}/convperf.py" --visualize --runtimes_file runtimes.json
- mv runtimes.json "${RESULTS_DIR}/resnet50_thread$i.json"
- mv convs.png "${RESULTS_DIR}/resnet50_thread$i.png"
-done
diff --git a/build_tools/benchmarks/mmperf/build_mmperf.sh b/build_tools/benchmarks/mmperf/build_mmperf.sh
deleted file mode 100755
index 1039fa5..0000000
--- a/build_tools/benchmarks/mmperf/build_mmperf.sh
+++ /dev/null
@@ -1,87 +0,0 @@
-#!/bin/bash
-
-# Copyright 2022 The IREE Authors
-#
-# Licensed under the Apache License v2.0 with LLVM Exceptions.
-# See https://llvm.org/LICENSE.txt for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#
-# Builds `mmperf` (https://github.com/mmperf/mmperf).
-#
-# `mmperf` benchmarks matrix-multiply workloads on IREE and other backends such
-# as RUY, TVM, Halide, CuBLAS, etc. Some backends are included as submodules
-# in the `mmperf` repo and built from source, and other backends are expected
-# to already be installed.
-#
-# Please refer to `build_tools/docker/dockerfiles/mmperf.Dockerfile` for commands on
-# installing various backends.
-#
-# Currently x86 CPU and CUDA are supported.
-#
-# Usage:
-# ./build_mmperf.sh \
-# <mmperf build dir> \
-# <backend> e.g. "cpu", "cuda". \
-# <iree sha> (optional) \
-# <mmperf repo dir> (optional)
-
-set -xeuo pipefail
-
-export BUILD_DIR=$1
-# Either `cpu` or `cuda`.
-export BACKEND=$2
-export IREE_SHA=${3:-"origin/main"}
-export REPO_DIR=${4:-${MMPERF_REPO_DIR}}
-
-pushd ${REPO_DIR}
-source mmperf.venv/bin/activate
-
-# Set all repos as a safe directory. Since this repo was created in the
-# DockerFile under `root`, git will not run commands on this repo as a
-# non-root user unless it is marked safe.
-for i in $(find ${REPO_DIR} -name '.git' | xargs dirname); do
- git config --global --add safe.directory $i
-done
-
-# Update IREE.
-pushd external/iree
-git fetch https://github.com/openxla/iree "${IREE_SHA}"
-git checkout "${IREE_SHA}"
-git submodule update --init --jobs 8 --depth 1
-popd # external/iree
-
-popd # ${REPO_DIR}
-
-# Build mmperf.
-declare -a args=(
- -GNinja
- -DCMAKE_CXX_COMPILER=/usr/bin/clang++
- -DCMAKE_C_COMPILER=/usr/bin/clang
- -DUSE_IREE=ON
-)
-
-if [[ ${BACKEND} == "cuda" ]]; then
- args+=(
- -DIREE_CUDA=ON
- -DUSE_CUBLAS=ON
- )
- cmake "${args[@]}" -B ${BUILD_DIR} ${REPO_DIR}
-elif [[ ${BACKEND} == "cpu" ]]; then
- args+=(
- -DMKL_DIR=/opt/intel/mkl
- -DBLIS_DIR=/opt/blis
- -DUSE_MKL=ON
- -DUSE_RUY=ON
- -DIREE_LLVMCPU=ON
- -DUSE_HALIDE=ON
- -DUSE_OPENBLAS=ON
- -DUSE_BLIS=ON
- -DUSE_TVM=ON
- )
- MKL_DIR=${MKL_DIR} BLIS_DIR=${BLIS_DIR} cmake "${args[@]}" -B ${BUILD_DIR} ${REPO_DIR}
-else
- echo "Error: Unsupported backend."
- exit 1
-fi
-
-cmake --build ${BUILD_DIR} --verbose
diff --git a/build_tools/benchmarks/mmperf/run_mmperf.sh b/build_tools/benchmarks/mmperf/run_mmperf.sh
deleted file mode 100755
index 43d3961..0000000
--- a/build_tools/benchmarks/mmperf/run_mmperf.sh
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/bin/bash
-
-# Copyright 2022 The IREE Authors
-#
-# Licensed under the Apache License v2.0 with LLVM Exceptions.
-# See https://llvm.org/LICENSE.txt for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#
-# Runs `mmperf` (https://github.com/mmperf/mmperf).
-#
-# `mmperf` benchmarks matrix-multiply workloads on IREE and other backends such
-# as RUY, TVM, Halide, CuBLAS, etc. Some backends are included as submodules
-# in the `mmperf` repo and built from source, and other backends are expected
-# to already be installed.
-#
-# Please refer to `build_tools/docker/mmperf/Dockerfile` for commands on
-# installing various backends.
-#
-# Usage:
-# ./run_mmperf.sh \
-# <mmperf build dir> \
-# <results directory> \
-# <mmperf repo dir> (optional)
-
-set -xeuo pipefail
-
-export BUILD_DIR=$1
-export REPORT_DIR=$2
-export REPO_DIR=${3:-${MMPERF_REPO_DIR}}
-
-source ${REPO_DIR}/mmperf.venv/bin/activate
-
-# Run benchmark.
-python3 ${REPO_DIR}/mmperf.py ${BUILD_DIR}/matmul/ ${REPORT_DIR}
diff --git a/build_tools/docker/context/setup_convperf.sh b/build_tools/docker/context/setup_convperf.sh
deleted file mode 100755
index bef9c84..0000000
--- a/build_tools/docker/context/setup_convperf.sh
+++ /dev/null
@@ -1,50 +0,0 @@
-#!/bin/bash
-
-# Copyright 2022 The IREE Authors
-#
-# Licensed under the Apache License v2.0 with LLVM Exceptions.
-# See https://llvm.org/LICENSE.txt for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#
-# Sets up `convperf` (https://github.com/nod-ai/convperf).
-#
-# `convperf` benchmarks convolution workloads on IREE and other backends such
-# as libxsmm. IREE is included as a submodule.
-#
-# Usage:
-# ./setup_convperf.sh \
-# <convperf repo dir> \
-# <convperf sha>
-
-set -xeuo pipefail
-
-export REPO_DIR="$1"
-export REPO_SHA="$2"
-
-pushd "${REPO_DIR}"
-
-mkdir convperf
-pushd convperf
-git init
-git fetch --depth 1 https://github.com/nod-ai/convperf.git "${REPO_SHA}"
-git checkout "${REPO_SHA}"
-git submodule update --init --recursive --jobs 8 --depth 1
-
-# Checkout a specific commit.
-git checkout "${REPO_SHA}"
-
-# Create virtual environment.
-python3 -m venv convperf.venv
-source convperf.venv/bin/activate
-pip install -r requirements.txt
-
-# Since the root user clones the convperf repo, we update permissions so that a
-# runner can access this repo, but we don't want to set the executable bit for
-# non-executables because git tracks this, so we then restore any git-tracked
-# changes.
-chmod -R 777 .
-git restore .
-git submodule foreach --recursive git restore .
-
-popd # convperf
-popd # "${REPO_DIR}"
diff --git a/build_tools/docker/context/setup_mmperf.sh b/build_tools/docker/context/setup_mmperf.sh
deleted file mode 100755
index fcf3652..0000000
--- a/build_tools/docker/context/setup_mmperf.sh
+++ /dev/null
@@ -1,54 +0,0 @@
-#!/bin/bash
-
-# Copyright 2022 The IREE Authors
-#
-# Licensed under the Apache License v2.0 with LLVM Exceptions.
-# See https://llvm.org/LICENSE.txt for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#
-# Sets up `mmperf` (https://github.com/mmperf/mmperf).
-#
-# `mmperf` benchmarks matrix-multiply workloads on IREE and other backends such
-# as RUY, TVM, Halide, CuBLAS, etc. Some backends are included as submodules
-# in the `mmperf` repo and built from source, and other backends are expected
-# to already be installed.
-#
-# Usage:
-# ./setup_mmperf.sh \
-# <mmperf repo dir> \
-# <mmperf sha>
-
-set -xeuo pipefail
-
-export REPO_DIR=$1
-export REPO_SHA=$2
-
-pushd ${REPO_DIR}
-
-mkdir mmperf
-pushd mmperf
-git init
-git fetch --depth 1 https://github.com/mmperf/mmperf.git "${REPO_SHA}"
-git checkout ${REPO_SHA}
-git submodule update --init --recursive --jobs 8 --depth 1
-
-# Create virtual environment.
-python3 -m venv mmperf.venv
-source mmperf.venv/bin/activate
-pip install -r requirements.txt
-pip install -r ./external/llvm-project/mlir/python/requirements.txt
-
-popd # mmperf
-
-# Since the root user clones the mmperf repo, we update permissions so that a
-# runner can access this repo.
-chmod -R 777 .
-
-# Make sure there are no local changes to the IREE submodule since the workflow
-# updates this at each run.
-pushd mmperf/external/iree
-git restore .
-git submodule foreach --recursive git restore .
-popd # mmperf/external/iree
-
-popd # ${REPO_DIR}
diff --git a/build_tools/docker/dockerfiles/convperf.Dockerfile b/build_tools/docker/dockerfiles/convperf.Dockerfile
deleted file mode 100644
index e94612e..0000000
--- a/build_tools/docker/dockerfiles/convperf.Dockerfile
+++ /dev/null
@@ -1,29 +0,0 @@
-# Copyright 2021 The IREE Authors
-#
-# Licensed under the Apache License v2.0 with LLVM Exceptions.
-# See https://llvm.org/LICENSE.txt for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
-# An image for running `convperf`: https://github.com/nod-ai/convperf.
-#
-# `convperf` benchmarks convolution workloads on IREE and other backends such
-# as libxsmm. IREE is included as a submodule.
-
-FROM gcr.io/iree-oss/perf@sha256:0e1c92dde6ec7312c7c92a8ee329892ad83bae4b7968e4f2c98f99ffad3faa48
-
-######## OpenMP ########
-RUN apt-get update \
- && apt-get install -y libomp-14-dev
-##############
-
-######## ConvPerf ########
-COPY build_tools/docker/context/setup_convperf.sh /usr/local/bin
-
-ARG CONVPERF_SHA="ca181bf9f1b959477acf8043eb5b5579fc47a37f"
-
-# Generate a version of mmperf for CPU.
-RUN mkdir -p "/usr/local/src/convperf" \
- && /usr/local/bin/setup_convperf.sh "/usr/local/src/convperf" "${CONVPERF_SHA}"
-
-ENV CONVPERF_REPO_DIR="/usr/local/src/convperf/convperf"
-############## \
diff --git a/build_tools/docker/dockerfiles/mmperf.Dockerfile b/build_tools/docker/dockerfiles/mmperf.Dockerfile
deleted file mode 100644
index 37535a9..0000000
--- a/build_tools/docker/dockerfiles/mmperf.Dockerfile
+++ /dev/null
@@ -1,71 +0,0 @@
-# Copyright 2022 The IREE Authors
-#
-# Licensed under the Apache License v2.0 with LLVM Exceptions.
-# See https://llvm.org/LICENSE.txt for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
-# An image for running mmperf: https://github.com/mmperf/mmperf.
-#
-# mmperf benchmarks matrix-multiplication workloads on IREE and various backends
-# such as OpenBLAS, MKL, TVM, Halide, CuBLAS, etc.
-#
-# These backends are included either in this image or as a submodule in the
-# mmperf repo. Later versions of Clang, LLVM, Python and Ubuntu are needed
-# to satisfy the dependency requirements of the backends.
-
-FROM gcr.io/iree-oss/perf@sha256:0e1c92dde6ec7312c7c92a8ee329892ad83bae4b7968e4f2c98f99ffad3faa48
-
-######## CUDA ########
-RUN apt-get update \
- && apt-get install -y \
- nvidia-cuda-toolkit \
- && mkdir -p "/usr/nvvm/libdevice" \
- && ln -s "/usr/lib/nvidia-cuda-toolkit/libdevice/libdevice.10.bc" "/usr/nvvm/libdevice/libdevice.10.bc"
-##############
-
-######## MKL ########
-WORKDIR /install-mkl
-
-RUN wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB \
- && apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB \
- && sh -c 'echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list' \
- && apt-get update \
- && apt-get install -y intel-mkl-64bit-2018.2-046 \
- && rm -rf /install-mkl
-
-WORKDIR /
-
-ENV MKL_DIR="/opt/intel/mkl"
-##############
-
-######## OPENBLAS ########
-RUN apt-get update \
- && apt-get install -y libopenblas-dev
-##############
-
-######## BLIS ########
-WORKDIR /install-blis
-
-RUN git clone --recurse-submodules https://github.com/flame/blis \
- && cd blis \
- && ./configure --prefix=/opt/blis --enable-cblas -c amd64 \
- && make -j 32 \
- && make install \
- && rm -rf /install-blis
-
-WORKDIR /
-
-ENV BLIS_DIR="/opt/blis"
-##############
-
-######## MMPERF ########
-COPY build_tools/docker/context/setup_mmperf.sh /usr/local/bin
-
-ARG MMPERF_SHA="1ebd6b9a72745e88b855c44d7320a92c83508f80"
-
-# Generate a version of mmperf for CPU.
-RUN mkdir -p "/usr/local/src/mmperf" \
- && /usr/local/bin/setup_mmperf.sh "/usr/local/src/mmperf" "${MMPERF_SHA}"
-
-ENV MMPERF_REPO_DIR="/usr/local/src/mmperf/mmperf"
-############## \
diff --git a/build_tools/docker/dockerfiles/perf.Dockerfile b/build_tools/docker/dockerfiles/perf.Dockerfile
deleted file mode 100644
index 63d9ed9..0000000
--- a/build_tools/docker/dockerfiles/perf.Dockerfile
+++ /dev/null
@@ -1,60 +0,0 @@
-# Copyright 2022 The IREE Authors
-#
-# Licensed under the Apache License v2.0 with LLVM Exceptions.
-# See https://llvm.org/LICENSE.txt for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
-# An image that includes basic packages, compiler and Python support for
-# running perf-related workflows e.g. mmperf, convperf.
-
-# Ubuntu 22.04.
-FROM ubuntu@sha256:817cfe4672284dcbfee885b1a66094fd907630d610cab329114d036716be49ba
-
-######## Basic ########
-RUN apt-get update \
- && apt-get install -y \
- git \
- wget \
- cmake \
- curl \
- ninja-build \
- numactl
-##############
-
-######## Clang/LLVM ########
-ARG LLVM_VERSION=14
-
-RUN apt-get update \
- && apt-get install -y \
- llvm-${LLVM_VERSION} \
- llvm-${LLVM_VERSION}-dev \
- clang-${LLVM_VERSION} \
- clang-tools-${LLVM_VERSION} \
- libclang-common-${LLVM_VERSION}-dev \
- libclang-${LLVM_VERSION}-dev \
- libclang1-${LLVM_VERSION} \
- clang-format-${LLVM_VERSION} \
- clangd-${LLVM_VERSION} \
- clang-tidy-${LLVM_VERSION} \
- lldb-${LLVM_VERSION} \
- lld-${LLVM_VERSION} \
- libmlir-${LLVM_VERSION}-dev \
- mlir-${LLVM_VERSION}-tools \
- && ln -s /usr/lib/llvm-${LLVM_VERSION}/bin/clang /usr/bin/clang \
- && ln -s /usr/lib/llvm-${LLVM_VERSION}/bin/clang++ /usr/bin/clang++
-
-ENV CC /usr/bin/clang
-ENV CXX /usr/bin/clang++
-##############
-
-######## Python ########
-WORKDIR /install-python
-
-ARG PYTHON_VERSION=3.10
-
-COPY runtime/bindings/python/iree/runtime/build_requirements.txt build_tools/docker/context/install_python_deps.sh ./
-RUN ./install_python_deps.sh "${PYTHON_VERSION}" \
- && rm -rf /install-python
-
-ENV PYTHON_BIN /usr/bin/python3
-##############
diff --git a/build_tools/docker/manage_images.py b/build_tools/docker/manage_images.py
index 25801e7..c8e7429 100755
--- a/build_tools/docker/manage_images.py
+++ b/build_tools/docker/manage_images.py
@@ -51,9 +51,6 @@
"riscv": ["base"],
"gradle-android": ["base"],
"frontends": ["android"],
- "perf": [],
- "mmperf": ["perf"],
- "convperf": ["perf"],
"shark": [],
"swiftshader": ["base"],
"samples": ["swiftshader"],
diff --git a/build_tools/docker/prod_digests.txt b/build_tools/docker/prod_digests.txt
index 4049999..ad0360b 100644
--- a/build_tools/docker/prod_digests.txt
+++ b/build_tools/docker/prod_digests.txt
@@ -12,8 +12,5 @@
gcr.io/iree-oss/shark@sha256:2b2e41dbf909194b598e226144a2fb1e74d31851d41fe519d0fcb6d071b77461
gcr.io/iree-oss/base-bleeding-edge@sha256:14200dacca3a0f3a66f8aa87c6f64729b83a2eeb403b689c24204074ad157418
gcr.io/iree-oss/swiftshader-bleeding-edge@sha256:d2483ac643184a16942489d8f709ef3714543ae795fa9ae8fd2e4d8bf68a24a0
-gcr.io/iree-oss/mmperf@sha256:dfeebcaf8f417275d44c7b8d9a1360c6a26d280b526779c9e64384798cfd9c92
-gcr.io/iree-oss/convperf@sha256:8e242ad71896a791c9be53b0906b7c8e96a2d3bad3f7431622cc797a6f89ba0f
-gcr.io/iree-oss/perf@sha256:0e1c92dde6ec7312c7c92a8ee329892ad83bae4b7968e4f2c98f99ffad3faa48
gcr.io/iree-oss/nvidia-bleeding-edge@sha256:522491c028ec3b4070f23910c70c8162fd9612e11d9cf062a13444df7e88ab70
gcr.io/iree-oss/base-arm64@sha256:31cdb58f556e415c3c65afae31ae9c54e43c1eb3dcd845f3d107abd9b368ff3d
