.github/workflows/pkgci_regression_test_nvidiagpu_vulkan.yml

# Copyright 2024 The IREE Authors
#
# Licensed under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

name: PkgCI Regression Test (NVIDIA GPU Vulkan)
on:
  workflow_call:
    inputs:
      artifact_run_id:
        type: string
        default: ""
  workflow_dispatch:
    inputs:
      artifact_run_id:
        type: string
        default: ""

jobs:
  linux_x86_64:
    name: Linux (x86_64)
    runs-on:
      - self-hosted # must come first
      - runner-group=${{ github.event_name == 'pull_request' && 'presubmit' || 'postsubmit' }}
      - environment=prod
      - gpu  # TODO(scotttodd): qualify further with vendor/model
      - os-family=Linux
    env:
      PACKAGE_DOWNLOAD_DIR: ${{ github.workspace }}/.packages
      IREERS_ARTIFACT_DIR: ${{ github.workspace }}/artifacts
      VENV_DIR: ${{ github.workspace }}/venv
    steps:
      - name: Checking out IREE repository
        uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
        with:
          submodules: false
      - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1  # v4.7.0
        with:
          # Must match the subset of versions built in pkgci_build_packages.
          python-version: '3.11'
      - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a  # v3.0.2
        with:
          name: linux_x86_64_release_packages
          path: ${{ env.PACKAGE_DOWNLOAD_DIR }}
      - name: Setup venv
        run: |
          ./build_tools/pkgci/setup_venv.py ${VENV_DIR} \
            --artifact-path=${PACKAGE_DOWNLOAD_DIR} \
            --fetch-gh-workflow=${{ inputs.artifact_run_id }}

      # Note: in-tree tests are omitted from this file. We could run tests
      # marked for Vulkan and 'plat_nvidia_[t4,a100]' (matching the runner used)

      # Out of tree tests
      - name: Checking out external TestSuite repository
        uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
        with:
          repository: nod-ai/SHARK-TestSuite
          ref: 9e9774632d32674e9ffa44b0a2ffc054f6c590ad
          path: SHARK-TestSuite
          submodules: false
      - name: Installing external TestSuite Python requirements
        run: |
          source ${VENV_DIR}/bin/activate
          python -m pip install -r SHARK-TestSuite/iree_tests/requirements.txt
      - name: Run external tests - ONNX test suite
        run: |
          source ${VENV_DIR}/bin/activate
          pytest SHARK-TestSuite/iree_tests/onnx/ \
              -n 4 -rpfE --timeout=30 --retries=2 --durations=20 \
              --config-files=build_tools/pkgci/external_test_suite/onnx_gpu_vulkan.json \
              --no-skip-tests-missing-files \
              --report-log=/tmp/iree_tests_onnx_gpu_vulkan_logs.json
      - name: "Updating config file with latest XFAIL lists"
        if: failure()
        run: |
          source ${VENV_DIR}/bin/activate
          python SHARK-TestSuite/iree_tests/update_config_xfails.py \
            --log-file=/tmp/iree_tests_onnx_gpu_vulkan_logs.json \
            --config-file=build_tools/pkgci/external_test_suite/onnx_gpu_vulkan.json
          cat build_tools/pkgci/external_test_suite/onnx_gpu_vulkan.json
      - name: "Uploading new config file"
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: "onnx_gpu_vulkan.json"
          path: "build_tools/pkgci/external_test_suite/onnx_gpu_vulkan.json"