.github/workflows/pkgci_regression_test_cpu.yml

# Copyright 2023 The IREE Authors
#
# Licensed under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

name: PkgCI Regression Test (CPU)
on:
  workflow_call:
    inputs:
      artifact_run_id:
        type: string
        default: ""
  workflow_dispatch:
    inputs:
      artifact_run_id:
        type: string
        default: ""

jobs:
  linux_x86_64:
    name: Linux (x86_64)
    runs-on: ubuntu-20.04
    env:
      PACKAGE_DOWNLOAD_DIR: ${{ github.workspace }}/.packages
      IREERS_ARTIFACT_DIR: ${{ github.workspace }}/artifacts
      VENV_DIR: ${{ github.workspace }}/venv
    steps:
      - name: Checking out IREE repository
        uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
        with:
          submodules: false
      - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1  # v4.7.0
        with:
          # Must match the subset of versions built in pkgci_build_packages.
          python-version: '3.11'
      - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a  # v3.0.2
        with:
          name: linux_x86_64_release_packages
          path: ${{ env.PACKAGE_DOWNLOAD_DIR }}
      - name: Setup venv
        run: |
          ./build_tools/pkgci/setup_venv.py ${VENV_DIR} \
            --artifact-path=${PACKAGE_DOWNLOAD_DIR} \
            --fetch-gh-workflow=${{ inputs.artifact_run_id }}
      - name: Check out external TestSuite repository
        uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
        with:
          repository: nod-ai/SHARK-TestSuite
          ref: 337083616ae6f596c0206a9edd1c47e8afc0e400
          path: SHARK-TestSuite
          submodules: false
          lfs: true
      - name: Install external TestSuite Python requirements
        run: |
          source ${VENV_DIR}/bin/activate
          python -m pip install -r SHARK-TestSuite/iree_tests/requirements.txt
      - name: Run external tests - ONNX test suite
        run: |
          source ${VENV_DIR}/bin/activate
          pytest SHARK-TestSuite/iree_tests/onnx/ \
              -n auto -rpfE --timeout=30 --durations=20 \
              --config-files=build_tools/pkgci/external_test_suite/onnx_cpu_llvm_sync.json \
              --no-skip-tests-missing-files \
              --report-log=/tmp/iree_tests_onnx_cpu_llvm_sync_logs.json
      - name: "Updating config file with latest XFAIL lists"
        if: failure()
        run: |
          source ${VENV_DIR}/bin/activate
          python SHARK-TestSuite/iree_tests/update_config_xfails.py \
            --log-file=/tmp/iree_tests_onnx_cpu_llvm_sync_logs.json \
            --config-file=build_tools/pkgci/external_test_suite/onnx_cpu_llvm_sync.json
          cat build_tools/pkgci/external_test_suite/onnx_cpu_llvm_sync.json
      - name: "Uploading new config file"
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: "onnx_cpu_llvm_sync.json"
          path: "build_tools/pkgci/external_test_suite/onnx_cpu_llvm_sync.json"

  # TODO(#17370): re-enable when nodai-amdgpu-w7900-x86-64 runner is stable
  # # Note: this is a persistent runner with a local cache. Downloading all input
  # # files (50GB+) without a cache can take 20+ minutes.
  # linux_x86_64_models:
  #   name: Linux (x86_64) Model Testing
  #   runs-on: nodai-amdgpu-w7900-x86-64
  #   env:
  #     PACKAGE_DOWNLOAD_DIR: ${{ github.workspace }}/.packages
  #     IREERS_ARTIFACT_DIR: ${{ github.workspace }}/artifacts
  #     VENV_DIR: ${{ github.workspace }}/venv
  #     IREE_TEST_FILES: ~/iree_tests_cache
  #   steps:
  #     - name: Checking out IREE repository
  #       uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
  #       with:
  #         submodules: false
  #     - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1  # v4.7.0
  #       with:
  #         # Must match the subset of versions built in pkgci_build_packages.
  #         python-version: '3.11'
  #     - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a  # v3.0.2
  #       with:
  #         name: linux_x86_64_release_packages
  #         path: ${{ env.PACKAGE_DOWNLOAD_DIR }}
  #     - name: Setup venv
  #       run: |
  #         ./build_tools/pkgci/setup_venv.py ${VENV_DIR} \
  #           --artifact-path=${PACKAGE_DOWNLOAD_DIR} \
  #           --fetch-gh-workflow=${{ inputs.artifact_run_id }}

  #     # TODO(#17344): regenerate .mlirbc files
  #     # # In-tree tests
  #     # - name: Run experimental/regression_suite tests
  #     #   run: |
  #     #     source ${VENV_DIR}/bin/activate
  #     #     pytest \
  #     #       -rA -s -m "plat_host_cpu and presubmit" \
  #     #       experimental/regression_suite

  #     # Out of tree tests
  #     - name: Check out external TestSuite repository
  #       uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
  #       with:
  #         repository: nod-ai/SHARK-TestSuite
  #         ref: 337083616ae6f596c0206a9edd1c47e8afc0e400
  #         path: SHARK-TestSuite
  #         submodules: false
  #         lfs: true
  #     - name: Install external TestSuite Python requirements
  #       run: |
  #         source ${VENV_DIR}/bin/activate
  #         python -m pip install -r SHARK-TestSuite/iree_tests/requirements.txt
  #     - name: Download remote files for real weight model tests
  #       run: |
  #         source ${VENV_DIR}/bin/activate
  #         python SHARK-TestSuite/iree_tests/download_remote_files.py --root-dir pytorch/models
  #     - name: Run external tests - Models with real weights
  #       run: |
  #         source ${VENV_DIR}/bin/activate
  #         pytest SHARK-TestSuite/iree_tests/pytorch/models \
  #           -n 4 \
  #           -rpfE \
  #           -k real_weights \
  #           --no-skip-tests-missing-files \
  #           --capture=no \
  #           --timeout=1200 \
  #           --retries 2 \
  #           --retry-delay 5 \
  #           --durations=0 \
  #           --config-files=build_tools/pkgci/external_test_suite/pytorch_models_cpu_llvm_task.json
  #     - name: "Running real weight model tests scheduled unet cpu"
  #       run: |
  #         source ${VENV_DIR}/bin/activate
  #         pytest SHARK-TestSuite/iree_tests/pytorch/models/sdxl-scheduled-unet-3-tank \
  #           -rpfE \
  #           -k real_weights \
  #           --no-skip-tests-missing-files \
  #           --capture=no \
  #           --timeout=1200 \
  #           --retries 2 \
  #           --retry-delay 5 \
  #           --durations=0 \
  #           --config-files=build_tools/pkgci/external_test_suite/sdxl_scheduled_unet_cpu_llvm_task.json