# Copyright 2022 The IREE Authors
#
# Licensed under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

name: CI

# A few notes:
#
# GitHub actions don't have variables or even support normal yaml anchors (they
# are specially disabled because...reasons?):
# See https://github.com/github-community/community/discussions/4501
# https://github.community/t/support-for-yaml-anchors/16128/92
# https://github.com/actions/runner/issues/1182
#
# Neither does it have any variables that are available everywhere. The
# top-level `env` field is available in many places, but not all
# See https://docs.github.com/en/actions/learn-github-actions/contexts#context-availability
# and https://github.com/community/community/discussions/27370
#
# So there are some odd patterns that are repeated in many places. To avoid
# repeating the comments, they are mentioned here:
#
# - self-hosted always has to be listed first in a runs-on block:
# https://docs.github.com/en/actions/hosting-your-own-runners/using-self-hosted-runners-in-a-workflow
#
# - We have to do a weird hack to get a pseudo-ternary operator. See
# https://github.com/actions/runner/issues/409, hence patterns like:
# runner-group=${{ github.event_name == 'pull_request' && 'presubmit' || 'postsubmit' }}

on:
  workflow_dispatch:
  pull_request:
  push:
    branches:
      - main

env:
  BUCKET: gs://iree-github-actions-${{ github.event_name == 'pull_request' && 'presubmit' || 'postsubmit' }}-artifacts

jobs:
  build_runtime:
    runs-on: ubuntu-20.04
    env:
      BUILD_DIR: build-runtime
    outputs:
      # Pass through the build directory as output so it's available to
      # dependent jobs.
      build-dir: ${{ env.BUILD_DIR }}
    steps:
      - name: "Checking out repository"
        uses: actions/checkout@7884fcad6b5d53d10323aee724dc68d8b9096a2e # v2
        with:
          submodules: true
      - name: "Building runtime"
        run: |
          ./build_tools/github_actions/docker_run.sh \
            gcr.io/iree-oss/base@sha256:5d43683c6b50aebe1fca6c85f2012f3b0fa153bf4dd268e8767b619b1891423a \
            ./build_tools/cmake/build_runtime.sh \
            "${BUILD_DIR}"
      # Using a tar archive is necessary to preserve file permissions. See
      # https://github.com/actions/upload-artifact#maintaining-file-permissions-and-case-sensitive-files
      # The upload action already does its own gzip compression, so no need to
      # do our own.
      - name: "Create build dir archive"
        run: tar -cf ${BUILD_DIR}.tar ${BUILD_DIR}
      - uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8 # v3.1.0
        with:
          name: "${{ env.BUILD_DIR }}.tar"
          path: "${{ env.BUILD_DIR }}.tar"

  test_runtime:
    needs: build_runtime
    runs-on: ubuntu-20.04
    env:
      BUILD_DIR: ${{ needs.build_runtime.outputs.build-dir }}
    steps:
      - name: "Checking out repository"
        uses: actions/checkout@7884fcad6b5d53d10323aee724dc68d8b9096a2e # v2
        with:
          submodules: true
      - name: "Downloading runtime build directory"
        uses: actions/download-artifact@fb598a63ae348fa914e94cd0ff38f362e927b741 # v3.0.0
        with:
          name: "${{ env.BUILD_DIR }}.tar"
      - name: "Extracting archive"
        run: tar -xf ${BUILD_DIR}.tar
      - name: "Testing runtime"
        run: |
          ./build_tools/github_actions/docker_run.sh \
            --env IREE_VULKAN_DISABLE=1 \
            gcr.io/iree-oss/base@sha256:5d43683c6b50aebe1fca6c85f2012f3b0fa153bf4dd268e8767b619b1891423a \
            ./build_tools/cmake/ctest_all.sh \
            "${BUILD_DIR}"

  build_all:
    runs-on:
      # Hacks, and order matters. See the comment at the top of the file.
      - self-hosted
      - runner-group=${{ github.event_name == 'pull_request' && 'presubmit' || 'postsubmit' }}
      - cpu
      - os-family=Linux
    env:
      BUILD_DIR: full-build-dir
    outputs:
      # Pass through the build directory as output so it's available to
      # dependent jobs.
      build-dir: ${{ env.BUILD_DIR }}
      build-dir-archive: ${{ steps.archive.outputs.build-dir-archive }}
      gcs-artifact: ${{ steps.upload.outputs.gcs-artifact }}
    steps:
      - name: "Checking out repository"
        uses: actions/checkout@7884fcad6b5d53d10323aee724dc68d8b9096a2e # v2
        with:
          submodules: true
      - name: "Building IREE"
        run: |
          ./build_tools/github_actions/docker_run.sh \
            gcr.io/iree-oss/base@sha256:5d43683c6b50aebe1fca6c85f2012f3b0fa153bf4dd268e8767b619b1891423a \
            ./build_tools/cmake/build_all.sh \
            "${BUILD_DIR}"
      # Things get more complicated here than when we're just building the
      # runtime. The build directory is way bigger. We're also using on our own
      # runners on GCE. So uploading to GitHub actions artifact storage hosted
      # on Azure is dirt slow. We drop static libraries and object files, which
      # aren't needed for testing. Then we do some minimal compression locally
      # *in parallel* and upload to GCS. This can be further optimized.
      # Especially decompression is still pretty slow. See #9881.
      - name: "Creating build dir archive"
        id: archive
        env:
          BUILD_DIR_ARCHIVE: ${{ env.BUILD_DIR }}.tar.zst
        run: |
          tar --exclude '*.a' --exclude '*.o' \
            -I 'zstd -T0' \
            -cf ${BUILD_DIR_ARCHIVE} ${BUILD_DIR}
          echo "::set-output name=build-dir-archive::${BUILD_DIR_ARCHIVE}"
      - name: "Uploading build dir archive"
        id: upload
        env:
          GCS_ARTIFACT: ${{ env.BUCKET }}/${{ github.sha }}/${{ steps.archive.outputs.build-dir-archive }}
          BUILD_DIR_ARCHIVE: ${{ steps.archive.outputs.build-dir-archive }}
        run: |
          gcloud alpha storage cp "${BUILD_DIR_ARCHIVE}" "${GCS_ARTIFACT}"
          echo "::set-output name=gcs-artifact::${GCS_ARTIFACT}"

  host_tools_assertions:
    uses: ./.github/workflows/host_tools.yml
    with:
      host-binary-root: host-tools-assertions
      enable-assertions: "ON"

  build_tf_integrations:
    runs-on:
      # Hacks, and order matters. See the comment at the top of the file.
      - self-hosted
      - runner-group=${{ github.event_name == 'pull_request' && 'presubmit' || 'postsubmit' }}
      - cpu
      - os-family=Linux
    outputs:
      binaries-dir: ${{ steps.build.outputs.binaries-dir }}
      binaries-archive: ${{ steps.archive.outputs.binaries-archive }}
      gcs-artifact: ${{ steps.upload.outputs.gcs-artifact }}
    steps:
      - name: "Checking out repository"
        uses: actions/checkout@7884fcad6b5d53d10323aee724dc68d8b9096a2e # v2
        with:
          submodules: true
      - name: "Building TF binaries"
        id: build
        env:
          IREE_TF_BINARIES_OUTPUT_DIR: iree-tf-binaries
        run: |
          ./build_tools/github_actions/docker_run.sh \
            --env "IREE_TF_BINARIES_OUTPUT_DIR=$(realpath "${IREE_TF_BINARIES_OUTPUT_DIR}")" \
            gcr.io/iree-oss/frontends-swiftshader@sha256:3090418a8d8a64c356d35eff285af32570a72f41127aa123209c1562f57abb01 \
            build_tools/cmake/build_tf_binaries.sh
          echo "::set-output name=binaries-dir::${IREE_TF_BINARIES_OUTPUT_DIR}"
      - name: "Creating archive of binaries"
        id: archive
        env:
          BINARIES_ARCHIVE: tf-binaries.tar
          BINARIES_DIR: ${{ steps.build.outputs.binaries-dir }}
        run: |
          tar -cf "${BINARIES_ARCHIVE}" "${BINARIES_DIR}"
          echo "::set-output name=binaries-archive::${BINARIES_ARCHIVE}"
      - name: "Uploading binaries archive"
        id: upload
        env:
          GCS_ARTIFACT: ${{ env.BUCKET }}/${{ github.sha }}/${{ steps.archive.outputs.binaries-archive }}
          BINARIES_ARCHIVE: ${{ steps.archive.outputs.binaries-archive }}
        run: |
          gcloud alpha storage cp "${BINARIES_ARCHIVE}" "${GCS_ARTIFACT}"
          echo "::set-output name=gcs-artifact::${GCS_ARTIFACT}"

  test_all:
    needs: build_all
    runs-on:
      # Hacks, and order matters. See the comment at the top of the file.
      - self-hosted
      - runner-group=${{ github.event_name == 'pull_request' && 'presubmit' || 'postsubmit' }}
      - cpu
      - os-family=Linux
    env:
      BUILD_DIR: ${{ needs.build_all.outputs.build-dir }}
      BUILD_DIR_ARCHIVE: ${{ needs.build_all.outputs.build-dir-archive }}
      GCS_ARTIFACT: ${{ needs.build_all.outputs.gcs-artifact }}
    steps:
      - name: "Checking out repository"
        uses: actions/checkout@7884fcad6b5d53d10323aee724dc68d8b9096a2e # v2
        with:
          submodules: true
      - name: "Downloading build dir archive"
        run: gcloud alpha storage cp "${GCS_ARTIFACT}" "${BUILD_DIR_ARCHIVE}"
      - name: "Extracting archive"
        run: tar -xf "${BUILD_DIR_ARCHIVE}"
      - name: "Testing all"
        run: |
          ./build_tools/github_actions/docker_run.sh \
            --env IREE_CUDA_DISABLE=1 \
            gcr.io/iree-oss/swiftshader@sha256:5027d56cdfee743d956bffd035668f7784166a486c48c74b42e5882cb0c289bf \
            ./build_tools/cmake/ctest_all.sh \
            "${BUILD_DIR}"

  test_gpu:
    if: github.event_name != 'pull_request'
    needs: build_all
    runs-on:
      # Hacks, and order matters. See the comment at the top of the file.
      - self-hosted
      - runner-group=${{ github.event_name == 'pull_request' && 'presubmit' || 'postsubmit' }}
      - gpu
      - os-family=Linux
    env:
      BUILD_DIR: ${{ needs.build_all.outputs.build-dir }}
      BUILD_DIR_ARCHIVE: ${{ needs.build_all.outputs.build-dir-archive }}
      GCS_ARTIFACT: ${{ needs.build_all.outputs.gcs-artifact }}
    steps:
      - name: "Checking out repository"
        uses: actions/checkout@7884fcad6b5d53d10323aee724dc68d8b9096a2e # v2
        with:
          submodules: true
      - name: Querying GPU information
        run: |
          ./build_tools/scripts/check_cuda.sh
          ./build_tools/scripts/check_vulkan.sh
      - name: "Downloading build dir archive"
        run: gcloud alpha storage cp "${GCS_ARTIFACT}" "${BUILD_DIR_ARCHIVE}"
      - name: "Extracting archive"
        run: tar -xf "${BUILD_DIR_ARCHIVE}"
      - name: "Testing with GPU"
        run: |
          ./build_tools/github_actions/docker_run.sh \
            --env IREE_VULKAN_F16_DISABLE=0 \
            --env IREE_CUDA_DISABLE=0 \
            --env CTEST_PARALLEL_LEVEL=2 \
            --gpus all \
            --env NVIDIA_DRIVER_CAPABILITIES=all \
            gcr.io/iree-oss/nvidia@sha256:7c2f56db65e656c15e6c96b5812a8275dd53c82bf41221192f9ba8a451aad870 \
            bash -c \
            "./build_tools/scripts/check_cuda.sh && ./build_tools/scripts/check_vulkan.sh && ./build_tools/cmake/ctest_all.sh ${BUILD_DIR}"

  test_tf_integrations:
    needs: [build_tf_integrations, build_all]
    runs-on:
      # Hacks, and order matters. See the comment at the top of the file.
      - self-hosted
      - runner-group=${{ github.event_name == 'pull_request' && 'presubmit' || 'postsubmit' }}
      - cpu
      - os-family=Linux
    env:
      BUILD_DIR: ${{ needs.build_all.outputs.build-dir }}
      BUILD_DIR_ARCHIVE: ${{ needs.build_all.outputs.build-dir-archive }}
      BUILD_DIR_GCS_ARTIFACT: ${{ needs.build_all.outputs.gcs-artifact }}
      TF_BINARIES_DIR: ${{ needs.build_tf_integrations.outputs.binaries-dir }}
      TF_BINARIES_ARCHIVE: ${{ needs.build_tf_integrations.outputs.binaries-archive }}
      TF_BINARIES_GCS_ARTIFACT: ${{ needs.build_tf_integrations.outputs.gcs-artifact }}
    steps:
      - name: "Checking out repository"
        uses: actions/checkout@7884fcad6b5d53d10323aee724dc68d8b9096a2e # v2
        with:
          submodules: true
      - name: "Downloading TF binaries archive"
        run: gcloud alpha storage cp "${TF_BINARIES_GCS_ARTIFACT}" "${TF_BINARIES_ARCHIVE}"
      - name: "Extracting TF binaries archive"
        run: tar -xf "${TF_BINARIES_ARCHIVE}"
      - name: "Symlinking TF binaries"
        run: |
          ./integrations/tensorflow/symlink_binaries.sh "$(realpath "${TF_BINARIES_DIR}")"
      - name: "Downloading build dir archive"
        run: gcloud alpha storage cp "${BUILD_DIR_GCS_ARTIFACT}" "${BUILD_DIR_ARCHIVE}"
      - name: "Extracting build dir archive"
        run: tar -xf "${BUILD_DIR_ARCHIVE}"
      - name: "Running TF integrations tests"
        run: |
          ./build_tools/github_actions/docker_run.sh \
            gcr.io/iree-oss/frontends-swiftshader@sha256:3090418a8d8a64c356d35eff285af32570a72f41127aa123209c1562f57abb01 \
            build_tools/cmake/run_tf_tests.sh \
            "${BUILD_DIR}"

  asan:
    runs-on:
      # Hacks, and order matters. See the comment at the top of the file.
      - self-hosted
      - runner-group=${{ github.event_name == 'pull_request' && 'presubmit' || 'postsubmit' }}
      - cpu
      - os-family=Linux
    steps:
      - name: "Checking out repository"
        uses: actions/checkout@7884fcad6b5d53d10323aee724dc68d8b9096a2e # v2
        with:
          submodules: true
      - name: "Building and testing with AddressSanitizer"
        run: |
          ./build_tools/github_actions/docker_run.sh \
            gcr.io/iree-oss/swiftshader@sha256:5027d56cdfee743d956bffd035668f7784166a486c48c74b42e5882cb0c289bf \
            ./build_tools/cmake/build_and_test_asan.sh

  tsan:
    runs-on:
      # Hacks, and order matters. See the comment at the top of the file.
      - self-hosted
      - runner-group=${{ github.event_name == 'pull_request' && 'presubmit' || 'postsubmit' }}
      - cpu
      - os-family=Linux
    steps:
      - name: "Checking out repository"
        uses: actions/checkout@7884fcad6b5d53d10323aee724dc68d8b9096a2e # v2
        with:
          submodules: true
      - name: "Building and testing with ThreadSanitizer"
        run: |
          ./build_tools/github_actions/docker_run.sh \
            gcr.io/iree-oss/base@sha256:5d43683c6b50aebe1fca6c85f2012f3b0fa153bf4dd268e8767b619b1891423a \
            ./build_tools/cmake/build_and_test_tsan.sh

  riscv32:
    needs: host_tools_assertions
    runs-on:
      # Hacks, and order matters. See the comment at the top of the file.
      - self-hosted
      - runner-group=${{ github.event_name == 'pull_request' && 'presubmit' || 'postsubmit' }}
      - cpu
      - os-family=Linux
    env:
      BUILD_RISCV_DIR: "build-riscv-rv32-baremetal"
      RISCV_CONFIG: "rv32-baremetal"
      HOST_BINARY_ROOT: ${{ needs.host_tools_assertions.outputs.host-binary-root }}
      HOST_BINARY_ARCHIVE: ${{ needs.host_tools_assertions.outputs.host-binary-root }}.tar
    steps:
      - name: "Checking out repository"
        uses: actions/checkout@7884fcad6b5d53d10323aee724dc68d8b9096a2e # v2
        with:
          submodules: true
      - name: "Downloading host tools"
        uses: actions/download-artifact@fb598a63ae348fa914e94cd0ff38f362e927b741 # v3.0.0
        with:
          name: "${{ env.HOST_BINARY_ARCHIVE }}"
      - name: "Extracting host tools archive"
        run: |
          tar -xvf ${HOST_BINARY_ARCHIVE}
      - name: "Cross-compiling and testing riscv32"
        run: |
          ./build_tools/github_actions/docker_run.sh \
            --env "RISCV_CONFIG=${RISCV_CONFIG}" \
            --env "BUILD_RISCV_DIR=${BUILD_RISCV_DIR}" \
            --env "IREE_HOST_BINARY_ROOT=${HOST_BINARY_ROOT}" \
            gcr.io/iree-oss/riscv@sha256:720bc0215d8462ea14352edc22710a6ce4c0c1daff581d179dd173885f1d8a35 \
            bash -c \
            "./build_tools/cmake/build_riscv.sh && tests/riscv32/smoke.sh"

  riscv64:
    needs: [build_all, host_tools_assertions, build_tf_integrations]
    runs-on:
      # Hacks, and order matters. See the comment at the top of the file.
      - self-hosted
      - runner-group=${{ github.event_name == 'pull_request' && 'presubmit' || 'postsubmit' }}
      - cpu
      - os-family=Linux
    env:
      BUILD_RISCV_DIR: "build-riscv-r64"
      RISCV_CONFIG: "rv64"
      HOST_BINARY_ROOT: ${{ needs.host_tools_assertions.outputs.host-binary-root }}
      HOST_BINARY_ARCHIVE: ${{ needs.host_tools_assertions.outputs.host-binary-root }}.tar
      BUILD_DIR: ${{ needs.build_all.outputs.build-dir }}
      BUILD_DIR_ARCHIVE: ${{ needs.build_all.outputs.build-dir-archive }}
      BUILD_DIR_GCS_ARTIFACT: ${{ needs.build_all.outputs.gcs-artifact }}
      TF_BINARIES_DIR: ${{ needs.build_tf_integrations.outputs.binaries-dir }}
      TF_BINARIES_ARCHIVE: ${{ needs.build_tf_integrations.outputs.binaries-archive }}
      TF_BINARIES_GCS_ARTIFACT: ${{ needs.build_tf_integrations.outputs.gcs-artifact }}
    steps:
      - name: "Checking out repository"
        uses: actions/checkout@7884fcad6b5d53d10323aee724dc68d8b9096a2e # v2
        with:
          submodules: true
      - name: "Downloading host tools"
        uses: actions/download-artifact@fb598a63ae348fa914e94cd0ff38f362e927b741 # v3.0.0
        with:
          name: "${{ env.HOST_BINARY_ARCHIVE }}"
      - name: "Extracting host tools archive"
        run: |
          tar -xvf ${HOST_BINARY_ARCHIVE}
      - name: "Downloading TF binaries archive"
        run: gcloud alpha storage cp "${TF_BINARIES_GCS_ARTIFACT}" "${TF_BINARIES_ARCHIVE}"
      - name: "Extracting TF binaries archive"
        run: tar -xf "${TF_BINARIES_ARCHIVE}"
        # FIXME: The riscv64 tests currently need the entire build dir archive
        # just to get FileCheck from LLVM's bin directory. This is silly and we
        # should make FileCheck available another way.
      - name: "Downloading build dir archive"
        run: gcloud alpha storage cp "${BUILD_DIR_GCS_ARTIFACT}" "${BUILD_DIR_ARCHIVE}"
      - name: "Extracting archive"
        run: tar -xf "${BUILD_DIR_ARCHIVE}"
      - name: "Cross-compiling and testing riscv64"
        run: |
          ./build_tools/github_actions/docker_run.sh \
            --env "RISCV_CONFIG=${RISCV_CONFIG}" \
            --env "BUILD_RISCV_DIR=${BUILD_RISCV_DIR}" \
            --env "IREE_HOST_BINARY_ROOT=${HOST_BINARY_ROOT}" \
            --env "IREE_IMPORT_TFLITE_BIN=${TF_BINARIES_DIR}/iree-import-tflite" \
            --env "LLVM_BIN_DIR=${BUILD_DIR}/third_party/llvm-project/llvm/bin" \
            gcr.io/iree-oss/riscv@sha256:720bc0215d8462ea14352edc22710a6ce4c0c1daff581d179dd173885f1d8a35 \
            bash -c \
              "./build_tools/cmake/build_riscv.sh && ./tests/riscv64/smoke.sh"


  summary:
    runs-on: ubuntu-20.04
    needs:
      - build_runtime
      - test_runtime
      - build_all
      - test_all
      - test_gpu
      - host_tools_assertions
      - asan
      - tsan
      - riscv32
      - riscv64
    if: always()
    steps:
      - name: Getting combined job status
        run: |
          echo '${{ toJson(needs.*.result) }}' \
            | jq --exit-status 'all(.=="success" or .=="skipped")' > /dev/null