| # Copyright 2022 The IREE Authors |
| # |
| # Licensed under the Apache License v2.0 with LLVM Exceptions. |
| # See https://llvm.org/LICENSE.txt for license information. |
| # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| # |
| # Workflow for running `mmperf` (https://github.com/mmperf/mmperf). |
| # `mmperf` benchmarks matrix-multiply workloads on IREE and other backends such |
| # as RUY, TVM, Halide, CuBLAS, etc. |
| # |
| # The workflow runs benchmarks on CPU and GPU and uploads results to the |
| # `mmperf-benchmark-artifacts` GC bucket. |
| |
| name: mmperf |
| |
| on: |
| schedule: |
| - cron: '0 13 * * *' |
| workflow_dispatch: |
| |
| env: |
| GCS_DIR: gs://iree-github-actions-${{ github.event_name == 'pull_request' && 'presubmit' || 'postsubmit' }}-artifacts/${{ github.run_id }}/${{ github.run_attempt }} |
| |
| jobs: |
| setup: |
| runs-on: ubuntu-20.04 |
| env: |
| # The commit being checked out is the merge commit for the PR. Its first |
| # parent will be the tip of main. |
| BASE_REF: HEAD^ |
| PR_TITLE: ${{ github.event.pull_request.title }} |
| PR_BODY: ${{ github.event.pull_request.body }} |
| IREE_SHA: ${{ github.sha }} |
| outputs: |
| artifact-upload-dir: ${{ steps.iree.outputs.artifact-upload-dir }} |
| should-run: ${{ steps.configure.outputs.should-run }} |
| runner-env: ${{ steps.configure.outputs.runner-env }} |
| runner-group: ${{ steps.configure.outputs.runner-group }} |
| steps: |
| - name: "Checking out repository" |
| uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0 |
| with: |
| # We need the parent commit to do a diff |
| fetch-depth: 2 |
| - name: "Configuring CI options" |
| id: configure |
| run: | |
| # Just informative logging. There should only be two commits in the |
| # history here, but limiting the depth helps when copying from a local |
| # repo instead of using checkout, e.g. with |
| # https://github.com/nektos/act where there will be more. |
| git log --oneline --graph --max-count=3 |
| ./build_tools/github_actions/configure_ci.py |
| - name: "Calculating version info" |
| id: iree |
| run: | |
| export GCS_ARTIFACT_DIR="$(date +'%Y-%m-%d').sha_${IREE_SHA}.timestamp_$(date +'%s')" |
| echo "artifact-upload-dir=${GCS_ARTIFACT_DIR}" >> $GITHUB_OUTPUT |
| |
| build_and_benchmark_cpu: |
| needs: setup |
| if: needs.setup.outputs.should-run == 'true' |
| runs-on: |
| - self-hosted # must come first |
| - runner-group=${{ needs.setup.outputs.runner-group }} |
| - environment=${{ needs.setup.outputs.runner-env }} |
| - cpu |
| - os-family=Linux |
| env: |
| IREE_SHA: ${{ github.sha }} |
| BUILD_DIR: mmperf-build-cpu |
| RESULTS_DIR: mmperf-results-cpu |
| GCS_UPLOAD_PARENT_DIR: "gs://mmperf-benchmark-artifacts/cpu" |
| GCS_UPLOAD_DIR_NAME: ${{ needs.setup.outputs.artifact-upload-dir }} |
| steps: |
| - name: "Checking out repository" |
| uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0 |
| - name: "Building mmperf for CPU" |
| run: | |
| ./build_tools/github_actions/docker_run.sh \ |
| gcr.io/iree-oss/mmperf@sha256:fa4c29028413081636965bbaead40287e4a3dd403d8992b6b9269bd9d999de75 \ |
| ./build_tools/benchmarks/mmperf/build_mmperf.sh "${BUILD_DIR}" "cpu" "${IREE_SHA}" |
| - name: "Running mmperf on CPU" |
| run: | |
| mkdir ${RESULTS_DIR} |
| ./build_tools/github_actions/docker_run.sh \ |
| gcr.io/iree-oss/mmperf@sha256:fa4c29028413081636965bbaead40287e4a3dd403d8992b6b9269bd9d999de75 \ |
| ./build_tools/benchmarks/mmperf/run_mmperf.sh "${BUILD_DIR}" "${RESULTS_DIR}" |
| - name: "Uploading results" |
| run: | |
| gcloud storage cp "${RESULTS_DIR}/latest/**" "${GCS_UPLOAD_PARENT_DIR}/${GCS_UPLOAD_DIR_NAME}/" |
| gcloud storage cp "${RESULTS_DIR}/latest/matmul.png" "${GCS_UPLOAD_PARENT_DIR}/matmul.png" |
| |
| build_cuda: |
| needs: setup |
| if: needs.setup.outputs.should-run == 'true' |
| runs-on: |
| - self-hosted # must come first |
| - runner-group=${{ needs.setup.outputs.runner-group }} |
| - environment=${{ needs.setup.outputs.runner-env }} |
| - cpu |
| - os-family=Linux |
| env: |
| IREE_SHA: ${{ github.sha }} |
| BUILD_DIR: mmperf-build-cuda |
| outputs: |
| build-dir: ${{ env.BUILD_DIR }} |
| build-dir-archive: ${{ steps.archive.outputs.build-dir-archive }} |
| build-dir-gcs-artifact: ${{ steps.upload.outputs.build-dir-gcs-artifact }} |
| steps: |
| - name: "Checking out repository" |
| uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0 |
| - name: "Building mmperf for CUDA" |
| run: | |
| ./build_tools/github_actions/docker_run.sh \ |
| gcr.io/iree-oss/mmperf@sha256:fa4c29028413081636965bbaead40287e4a3dd403d8992b6b9269bd9d999de75 \ |
| ./build_tools/benchmarks/mmperf/build_mmperf.sh "${BUILD_DIR}" "cuda" "${IREE_SHA}" |
| - name: "Removing unused files" |
| run: | |
| find "${BUILD_DIR}" -type f -name "*.a" -o -type f -name "*.o" \ |
| -print \ |
| -delete |
| - name: "Creating build dir archive" |
| id: archive |
| env: |
| BUILD_DIR_ARCHIVE: ${{ env.BUILD_DIR }}.tar.zst |
| run: | |
| tar -I 'zstd -T0' \ |
| -cf ${BUILD_DIR_ARCHIVE} ${BUILD_DIR} |
| echo "build-dir-archive=${BUILD_DIR_ARCHIVE}" >> "${GITHUB_OUTPUT}" |
| - name: "Uploading build dir archive" |
| id: upload |
| env: |
| BUILD_DIR_ARCHIVE: ${{ steps.archive.outputs.build-dir-archive }} |
| BUILD_DIR_GCS_ARTIFACT: ${{ env.GCS_DIR }}/${{ steps.archive.outputs.build-dir-archive }} |
| run: | |
| gcloud storage cp "${BUILD_DIR_ARCHIVE}" "${BUILD_DIR_GCS_ARTIFACT}" |
| echo "build-dir-gcs-artifact=${BUILD_DIR_GCS_ARTIFACT}" >> "${GITHUB_OUTPUT}" |
| |
| benchmark_cuda: |
| needs: [setup, build_cuda] |
| if: needs.setup.outputs.should-run == 'true' |
| runs-on: |
| - self-hosted # must come first |
| - runner-group=${{ needs.setup.outputs.runner-group }} |
| - environment=${{ needs.setup.outputs.runner-env }} |
| - gpu |
| - os-family=Linux |
| env: |
| RESULTS_DIR: mmperf-results-cuda |
| GCS_UPLOAD_PARENT_DIR: "gs://mmperf-benchmark-artifacts/cuda" |
| GCS_UPLOAD_DIR_NAME: ${{ needs.setup.outputs.artifact-upload-dir }} |
| BUILD_DIR: ${{ needs.build_cuda.outputs.build-dir }} |
| BUILD_DIR_ARCHIVE: ${{ needs.build_cuda.outputs.build-dir-archive }} |
| BUILD_DIR_GCS_ARTIFACT: ${{ needs.build_cuda.outputs.build-dir-gcs-artifact }} |
| steps: |
| - name: "Checking out repository" |
| uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0 |
| - name: "Downloading build dir archive" |
| run: gcloud storage cp "${BUILD_DIR_GCS_ARTIFACT}" "${BUILD_DIR_ARCHIVE}" |
| - name: "Extracting build dir archive" |
| run: tar -xf "${BUILD_DIR_ARCHIVE}" |
| - name: "Running mmperf on CUDA" |
| run: | |
| mkdir ${RESULTS_DIR} |
| ./build_tools/github_actions/docker_run.sh \ |
| --gpus all \ |
| gcr.io/iree-oss/mmperf@sha256:fa4c29028413081636965bbaead40287e4a3dd403d8992b6b9269bd9d999de75 \ |
| ./build_tools/benchmarks/mmperf/run_mmperf.sh "${BUILD_DIR}" "${RESULTS_DIR}" |
| - name: "Uploading results" |
| run: | |
| export GCS_DIR_NAME="$(date +'%Y-%m-%d').$(date +'%s')" |
| gcloud storage cp "${RESULTS_DIR}/latest/**" "${GCS_UPLOAD_PARENT_DIR}/${GCS_UPLOAD_DIR_NAME}/" |
| gcloud storage cp "${RESULTS_DIR}/latest/matmul.png" "${GCS_UPLOAD_PARENT_DIR}/matmul.png" |
