.github/workflows/pkgci_regression_test_cpu.yml

# Copyright 2023 The IREE Authors
#
# Licensed under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

name: PkgCI Regression Test (CPU)
on:
  workflow_call:
    inputs:
      artifact_run_id:
        type: string
        default: ""
  workflow_dispatch:
    inputs:
      artifact_run_id:
        type: string
        default: ""

jobs:
  linux_x86_64:
    name: Linux (x86_64)
    runs-on: ubuntu-20.04-64core
    env:
      PACKAGE_DOWNLOAD_DIR: ${{ github.workspace }}/.packages
      IREERS_ARTIFACT_DIR: ${{ github.workspace }}/artifacts
      VENV_DIR: ${{ github.workspace }}/venv
    steps:
      - name: Checking out IREE repository
        uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
        with:
          submodules: false
      - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1  # v4.7.0
        with:
          # Must match the subset of versions built in pkgci_build_packages.
          python-version: '3.11'
      - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a  # v3.0.2
        with:
          name: linux_x86_64_release_packages
          path: ${{ env.PACKAGE_DOWNLOAD_DIR }}
      - name: Setup venv
        run: |
          ./build_tools/pkgci/setup_venv.py ${VENV_DIR} \
            --artifact-path=${PACKAGE_DOWNLOAD_DIR} \
            --fetch-gh-workflow=${{ inputs.artifact_run_id }}

      # In-tree tests
      - name: Run experimental/regression_suite tests
        run: |
          source ${VENV_DIR}/bin/activate
          pytest \
            -rA -s -m "plat_host_cpu and presubmit" \
            experimental/regression_suite

      # Out of tree tests
      - name: Check out external TestSuite repository
        uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
        with:
          repository: nod-ai/SHARK-TestSuite
          ref: d83c02dbecf7963320b5773ceb8ac3b3b7d90831
          path: SHARK-TestSuite
          submodules: false
          lfs: true
      - name: Install external TestSuite Python requirements
        run: |
          source ${VENV_DIR}/bin/activate
          python -m pip install -r SHARK-TestSuite/iree_tests/requirements.txt
      - name: Run external tests - ONNX test suite
        run: |
          source ${VENV_DIR}/bin/activate
          pytest SHARK-TestSuite/iree_tests/onnx/ \
              -n auto -rpfE --timeout=30 --retries=2 --durations=20 \
              --config-files=build_tools/pkgci/external_test_suite/onnx_cpu_llvm_sync.json \
              --no-skip-tests-missing-files

  # Note: this is a persistent runner with a local cache. Downloading all input
  # files (50GB+) without a cache can take 20+ minutes.
  linux_x86_64_models:
    name: Linux (x86_64) Model Testing
    runs-on: nodai-amdgpu-w7900-x86-64
    env:
      PACKAGE_DOWNLOAD_DIR: ${{ github.workspace }}/.packages
      IREERS_ARTIFACT_DIR: ${{ github.workspace }}/artifacts
      VENV_DIR: ${{ github.workspace }}/venv
      IREE_TEST_FILES: ~/iree_tests_cache
    steps:
      - name: Checking out IREE repository
        uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
        with:
          submodules: false
      - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1  # v4.7.0
        with:
          # Must match the subset of versions built in pkgci_build_packages.
          python-version: '3.11'
      - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a  # v3.0.2
        with:
          name: linux_x86_64_release_packages
          path: ${{ env.PACKAGE_DOWNLOAD_DIR }}
      - name: Setup venv
        run: |
          ./build_tools/pkgci/setup_venv.py ${VENV_DIR} \
            --artifact-path=${PACKAGE_DOWNLOAD_DIR} \
            --fetch-gh-workflow=${{ inputs.artifact_run_id }}

      # Out of tree tests
      - name: Check out external TestSuite repository
        uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
        with:
          repository: nod-ai/SHARK-TestSuite
          ref: d83c02dbecf7963320b5773ceb8ac3b3b7d90831
          path: SHARK-TestSuite
          submodules: false
          lfs: true
      - name: Install external TestSuite Python requirements
        run: |
          source ${VENV_DIR}/bin/activate
          python -m pip install -r SHARK-TestSuite/iree_tests/requirements.txt
      - name: Download remote files for real weight model tests
        run: |
          source ${VENV_DIR}/bin/activate
          python SHARK-TestSuite/iree_tests/download_remote_files.py --root-dir pytorch/models
      - name: Run external tests - Models with real weights
        run: |
          source ${VENV_DIR}/bin/activate
          pytest SHARK-TestSuite/iree_tests/pytorch/models -k real_weights \
               -rpfE --timeout=600 --durations=0 \
              --config-files=build_tools/pkgci/external_test_suite/pytorch_models_cpu_llvm_task.json \
              --no-skip-tests-missing-files