.github/workflows/benchmarks.yml

# Copyright 2022 The IREE Authors
#
# Licensed under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
# Workflow for all benchmark-related jobs. It is designed to be called from
# the main workflow ci.yml. The concurrency of this workflow is controlled by
# the caller's job.

name: Benchmarks

on:
  workflow_call:
    inputs:
      runner-group:
        required: true
        type: string
      runner-env:
        required: true
        type: string
      build-dir:
        required: true
        type: string
      build-dir-archive:
        required: true
        type: string
      build-dir-gcs-artifact:
        required: true
        type: string
      tf-binaries-dir:
        required: true
        type: string
      tf-binaries-archive:
        required: true
        type: string
      tf-binaries-gcs-artifact:
        required: true
        type: string

env:
  # This duplicates the variable from ci.yml. The variable needs to be in env
  # instead of the outputs of setup because it contains the run attempt and we
  # want that to be the current attempt, not whatever attempt the setup step
  # last ran in. It therefore can't be passed in via inputs because the env
  # context isn't available there.
  GCS_DIR: gs://iree-github-actions-${{ github.event_name == 'pull_request' && 'presubmit' || 'postsubmit' }}-artifacts/${{ github.run_id }}/${{ github.run_attempt }}

jobs:
  # TODO(#11263): Drop this job once the IREE_BUILD_BENCHMARKS is removed.
  test_build_benchmark_suites:
    runs-on:
      - self-hosted # must come first
      - runner-group=${{ inputs.runner-group }}
      - environment=${{ inputs.runner-env }}
      - cpu
      - os-family=Linux
    env:
      BUILD_DIR: ${{ inputs.build-dir }}
      BUILD_DIR_ARCHIVE: ${{ inputs.build-dir-archive }}
      BUILD_DIR_GCS_ARTIFACT: ${{ inputs.build-dir-gcs-artifact }}
      TF_BINARIES_DIR: ${{ inputs.tf-binaries-dir }}
      TF_BINARIES_ARCHIVE: ${{ inputs.tf-binaries-archive }}
      TF_BINARIES_GCS_ARTIFACT: ${{ inputs.tf-binaries-gcs-artifact }}
    steps:
      - name: "Checking out repository"
        uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791 # v2.5.0
      - name: "Checking out runtime submodules"
        run: ./build_tools/scripts/git/update_runtime_submodules.sh
      - name: "Downloading build dir archive"
        run: gcloud alpha storage cp "${BUILD_DIR_GCS_ARTIFACT}" "${BUILD_DIR_ARCHIVE}"
      - name: "Extracting install from build dir archive"
        run: tar -xf "${BUILD_DIR_ARCHIVE}" "${BUILD_DIR}/install"
      - name: "Downloading TF binaries archive"
        run: gcloud alpha storage cp "${TF_BINARIES_GCS_ARTIFACT}" "${TF_BINARIES_ARCHIVE}"
      - name: "Extracting TF binaries archive"
        run: tar -xf "${TF_BINARIES_ARCHIVE}"
      - name: "Building benchmarks"
        id: build
        env:
          BENCHMARKS_OUTPUT_DIR: build-benchmarks
        run: |
          build_tools/github_actions/docker_run.sh \
            --env "IREE_TF_BINARIES_DIR=${TF_BINARIES_DIR}" \
            --env "IREE_HOST_BINARY_ROOT=${BUILD_DIR}/install" \
            --env "IREE_BUILD_BENCHMARKS_DIR=${BENCHMARKS_OUTPUT_DIR}" \
            gcr.io/iree-oss/base@sha256:605d86ccf4197e978a24867fabb7fc100334c926b067ee0518e46d0a4396e206 \
            build_tools/cmake/build_benchmarks.sh

  build_tools:
    runs-on:
      - self-hosted  # must come first
      - runner-group=${{ inputs.runner-group }}
      - environment=${{ inputs.runner-env }}
      - cpu
      - os-family=Linux
    strategy:
      matrix:
        target:
          - platform: "riscv"
            architecture: "rv64"
            docker_image: "gcr.io/iree-oss/riscv@sha256:5034a40068f14d91d5caf07d615f323561416caee8afda32ae622746a4dd8b04"
    outputs:
      benchmark-tools-gcs-artifacts: ${{ toJSON(steps.upload.outputs) }}
    env:
      PLATFORM: ${{ matrix.target.platform }}
      ARCHITECTURE: ${{ matrix.target.architecture }}
      DOCKER_IMAGE: ${{ matrix.target.docker_image }}
      BUILD_TOOLS_DIR: ${{ matrix.target.platform }}-${{ matrix.target.architecture }}-benchmark-tools-dir
      BUILD_DIR: ${{ inputs.build-dir }}
      BUILD_DIR_ARCHIVE: ${{ inputs.build-dir-archive }}
      BUILD_DIR_GCS_ARTIFACT: ${{ inputs.build-dir-gcs-artifact }}
    steps:
      - name: "Checking out repository"
        uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791 # v2.5.0
      - name: "Checking out runtime submodules"
        run: ./build_tools/scripts/git/update_runtime_submodules.sh
      - name: "Downloading build dir archive"
        run: gcloud alpha storage cp "${BUILD_DIR_GCS_ARTIFACT}" "${BUILD_DIR_ARCHIVE}"
      - name: "Extracting host binaries"
        run: tar -xf "${BUILD_DIR_ARCHIVE}" "${BUILD_DIR}/install"
      - name: "Compiling the benchmark tools"
        id: build
        run: |
          ./build_tools/github_actions/docker_run.sh \
            --env "${PLATFORM^^}_ARCH=${ARCHITECTURE}" \
            --env "BUILD_PRESET=benchmark" \
            --env "IREE_HOST_BINARY_ROOT=${BUILD_DIR}/install" \
            "${DOCKER_IMAGE}" \
            "./build_tools/cmake/build_${PLATFORM}.sh" \
            "${BUILD_TOOLS_DIR}/build"
      - name: "Compiling the benchmark tools with tracing"
        id: build-with-tracing
        run: |
          ./build_tools/github_actions/docker_run.sh \
            --env "${PLATFORM^^}_ARCH=${ARCHITECTURE}" \
            --env "BUILD_PRESET=benchmark-with-tracing" \
            --env "IREE_HOST_BINARY_ROOT=${BUILD_DIR}/install" \
            "${DOCKER_IMAGE}" \
            "./build_tools/cmake/build_${PLATFORM}.sh" \
            "${BUILD_TOOLS_DIR}/build-traced"
      - name: "Creating the benchmark tools archive"
        id: archive
        env:
          BENCHMARK_TOOLS_ARCHIVE: ${{ matrix.target.platform }}-${{ matrix.target.architecture }}-benchmarks-tools.tar
        run: |
          tar -cf "${BENCHMARK_TOOLS_ARCHIVE}" \
            "${BUILD_TOOLS_DIR}"/*/tools/iree-benchmark-module
          echo "benchmark-tools-archive=${BENCHMARK_TOOLS_ARCHIVE}" >> "${GITHUB_OUTPUT}"
      - name: "Uploading the benchmark tools archive"
        id: upload
        env:
          BENCHMARK_TOOLS_ARCHIVE: ${{ steps.archive.outputs.benchmark-tools-archive }}
          BENCHMARK_TOOLS_GCS_ARTIFACT: ${{ env.GCS_DIR }}/${{ steps.archive.outputs.benchmark-tools-archive }}
        run: |
          gcloud alpha storage cp "${BENCHMARK_TOOLS_ARCHIVE}" "${BENCHMARK_TOOLS_GCS_ARTIFACT}"
          echo "${PLATFORM}-${ARCHITECTURE}-benchmark-tools-gcs-artifact=${BENCHMARK_TOOLS_GCS_ARTIFACT}" >> "${GITHUB_OUTPUT}"