Run all framework sanity check tests and organize jobs. (#18420)
Fixes https://github.com/iree-org/iree/issues/16624 by running the
existing ONNX and PyTorch importer tests _with the packages they need
installed_.
Sample logs when a test fails:
https://github.com/iree-org/iree/actions/runs/10691656074/job/29638920091?pr=18420#step:9:19
```
Traceback (most recent call last):
File "/home/runner/work/iree/iree/compiler/bindings/python/test/extras/fx_importer_test.py", line 8, in <module>
from iree.compiler.extras import fx_importer
File "/home/runner/work/iree/iree/.venv/lib/python3.11/site-packages/iree/compiler/extras/fx_importer.py", line 138, in <module>
from .._mlir_libs._torchMlir import get_int64_max, get_int64_min
ModuleNotFoundError: No module named 'iree.compiler._mlir_libs._torchMlir'
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/home/runner/work/iree/iree/compiler/bindings/python/test/extras/fx_importer_test.py", line [19](https://github.com/iree-org/iree/actions/runs/10691656074/job/29638920091?pr=18420#step:9:20), in <module>
raise ModuleNotFoundError(
ModuleNotFoundError: Failed to import the fx_importer (for a reason other than torch not being found)
Error: Process completed with exit code 1.
```
---
I'm not really satisfied with how these tests are distributed across
jobs either before or after these changes, but I think this is a step in
a good direction at least.
* These tests depend on optional packages (torch, onnx, tensorflow) and
disable themselves if those optional packages are not present.
* The core project build (CMake/CTest, Python, packaging builds) strives
to be modular and not require the entire kitchen sink to function.
* Test workflows should make sense for both local development _and_ CI
usage. The local development flows here are relatively convoluted and
could use some work.diff --git a/.github/workflows/pkgci.yml b/.github/workflows/pkgci.yml
index f156dd8..b96cc70 100644
--- a/.github/workflows/pkgci.yml
+++ b/.github/workflows/pkgci.yml
@@ -37,18 +37,24 @@
with:
package_version: 0.dev1
+ # Package sanity checks.
unit_test:
name: Unit Test
needs: [setup, build_packages]
if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'unit_test')
uses: ./.github/workflows/pkgci_unit_test.yml
+ # Tests for large programs.
regression_test:
name: Regression Test
needs: [setup, build_packages]
if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'regression_test')
uses: ./.github/workflows/pkgci_regression_test.yml
+ ########################### Hardware/accelerators ###########################
+ # Jobs that run unit tests on special hardware platforms or accelerators
+ #############################################################################
+
test_amd_mi250:
name: Test AMD MI250
needs: [setup, build_packages]
@@ -73,13 +79,8 @@
if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'test_nvidia_t4')
uses: ./.github/workflows/pkgci_test_nvidia_t4.yml
- test_tensorflow_cpu:
- name: Test TensorFlow CPU
- needs: [setup, build_packages]
- if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'test_tensorflow_cpu')
- uses: ./.github/workflows/pkgci_test_tensorflow_cpu.yml
-
test_android:
+ name: Test Android
needs: [setup, build_packages]
if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'test_android')
uses: ./.github/workflows/pkgci_test_android.yml
@@ -87,8 +88,25 @@
write-caches: ${{ needs.setup.outputs.write-caches }}
test_riscv64:
+ name: Test RISC-V 64
needs: [setup, build_packages]
if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'test_riscv64')
uses: ./.github/workflows/pkgci_test_riscv64.yml
with:
write-caches: ${{ needs.setup.outputs.write-caches }}
+
+ ################################# Frameworks ################################
+ # Jobs that test machine learning frameworks or their Python APIs
+ #############################################################################
+
+ test_onnx:
+ name: Test ONNX
+ needs: [setup, build_packages]
+ if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'test_onnx')
+ uses: ./.github/workflows/pkgci_test_onnx.yml
+
+ test_tensorflow:
+ name: Test TensorFlow
+ needs: [setup, build_packages]
+ if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'test_tensorflow')
+ uses: ./.github/workflows/pkgci_test_tensorflow.yml
diff --git a/.github/workflows/pkgci_regression_test.yml b/.github/workflows/pkgci_regression_test.yml
index eb7a430..1ef6958 100644
--- a/.github/workflows/pkgci_regression_test.yml
+++ b/.github/workflows/pkgci_regression_test.yml
@@ -18,108 +18,6 @@
default: ""
jobs:
- test_onnx:
- name: "test_onnx :: ${{ matrix.name }}"
- runs-on: ${{ matrix.runs-on }}
- strategy:
- fail-fast: false
- matrix:
- include:
- # CPU
- - name: cpu_llvm_sync
- config-file: onnx_ops_cpu_llvm_sync.json
- numprocesses: auto
- runs-on: ubuntu-20.04
-
- # AMD GPU
- - name: amdgpu_rocm_rdna3
- numprocesses: 1
- config-file: onnx_ops_gpu_rocm_rdna3.json
- runs-on: nodai-amdgpu-w7900-x86-64
- - name: amdgpu_vulkan
- numprocesses: 4
- config-file: onnx_ops_gpu_vulkan.json
- runs-on: nodai-amdgpu-w7900-x86-64
-
- # NVIDIA GPU
- - name: nvidiagpu_cuda
- config-file: onnx_ops_gpu_cuda.json
- numprocesses: 4
- runs-on:
- - self-hosted # must come first
- - runner-group=${{ github.event_name == 'pull_request' && 'presubmit' || 'postsubmit' }}
- - environment=prod
- - gpu # TODO(scotttodd): qualify further with vendor/model
- - os-family=Linux
- - name: nvidiagpu_vulkan
- config-file: onnx_ops_gpu_vulkan.json
- numprocesses: 4
- runs-on:
- - self-hosted # must come first
- - runner-group=${{ github.event_name == 'pull_request' && 'presubmit' || 'postsubmit' }}
- - environment=prod
- - gpu # TODO(scotttodd): qualify further with vendor/model
- - os-family=Linux
- env:
- PACKAGE_DOWNLOAD_DIR: ${{ github.workspace }}/.packages
- CONFIG_FILE_PATH: tests/external/iree-test-suites/onnx_ops/${{ matrix.config-file }}
- NUMPROCESSES: ${{ matrix.numprocesses }}
- LOG_FILE_PATH: /tmp/test_onnx_ops_${{ matrix.name }}_logs.json
- VENV_DIR: ${{ github.workspace }}/venv
- steps:
- - name: Checking out IREE repository
- uses: actions/checkout@v4.1.7
- with:
- submodules: false
- - uses: actions/setup-python@v5.1.0
- with:
- # Must match the subset of versions built in pkgci_build_packages.
- python-version: "3.11"
- - uses: actions/download-artifact@v4.1.7
- with:
- name: linux_x86_64_release_packages
- path: ${{ env.PACKAGE_DOWNLOAD_DIR }}
- - name: Setup venv
- run: |
- ./build_tools/pkgci/setup_venv.py ${VENV_DIR} \
- --artifact-path=${PACKAGE_DOWNLOAD_DIR} \
- --fetch-gh-workflow=${{ inputs.artifact_run_id }}
-
- - name: Checkout test suites repository
- uses: actions/checkout@v4.1.7
- with:
- repository: iree-org/iree-test-suites
- ref: 9e921d0ea271a85f772eee22965585461c9b14c2
- path: iree-test-suites
- - name: Install ONNX ops test suite requirements
- run: |
- source ${VENV_DIR}/bin/activate
- python -m pip install -r iree-test-suites/onnx_ops/requirements.txt
- - name: Run ONNX ops test suite
- run: |
- source ${VENV_DIR}/bin/activate
- pytest iree-test-suites/onnx_ops/ \
- -rpfE \
- --numprocesses ${NUMPROCESSES} \
- --timeout=30 \
- --durations=20 \
- --config-files=${CONFIG_FILE_PATH} \
- --report-log=${LOG_FILE_PATH}
- - name: "Updating config file with latest XFAIL lists"
- if: failure()
- run: |
- source ${VENV_DIR}/bin/activate
- python iree-test-suites/onnx_ops/update_config_xfails.py \
- --log-file=${LOG_FILE_PATH} \
- --config-file=${CONFIG_FILE_PATH}
- cat ${CONFIG_FILE_PATH}
- - name: "Uploading new config file"
- if: failure()
- uses: actions/upload-artifact@v4.3.3
- with:
- name: ${{ matrix.config-file }}
- path: ${{ env.CONFIG_FILE_PATH }}
-
test_models:
name: "test_models :: ${{ matrix.name }}"
runs-on: ${{ matrix.runs-on }}
diff --git a/.github/workflows/pkgci_test_amd_mi250.yml b/.github/workflows/pkgci_test_amd_mi250.yml
index 3c9c1d0..903b93d 100644
--- a/.github/workflows/pkgci_test_amd_mi250.yml
+++ b/.github/workflows/pkgci_test_amd_mi250.yml
@@ -18,7 +18,7 @@
default: ""
jobs:
- test:
+ test_mi250:
runs-on: nodai-amdgpu-mi250-x86-64
env:
PACKAGE_DOWNLOAD_DIR: ${{ github.workspace }}/.packages
diff --git a/.github/workflows/pkgci_test_amd_mi300.yml b/.github/workflows/pkgci_test_amd_mi300.yml
index 0ec5ad1..495c19b 100644
--- a/.github/workflows/pkgci_test_amd_mi300.yml
+++ b/.github/workflows/pkgci_test_amd_mi300.yml
@@ -18,7 +18,7 @@
default: ""
jobs:
- test:
+ test_mi300:
runs-on: nodai-amdgpu-mi300-x86-64
env:
PACKAGE_DOWNLOAD_DIR: ${{ github.workspace }}/.packages
diff --git a/.github/workflows/pkgci_test_amd_w7900.yml b/.github/workflows/pkgci_test_amd_w7900.yml
index c409aa0..a2cb109 100644
--- a/.github/workflows/pkgci_test_amd_w7900.yml
+++ b/.github/workflows/pkgci_test_amd_w7900.yml
@@ -18,7 +18,7 @@
default: ""
jobs:
- test:
+ test_w7900:
runs-on: nodai-amdgpu-w7900-x86-64
env:
PACKAGE_DOWNLOAD_DIR: ${{ github.workspace }}/.packages
diff --git a/.github/workflows/pkgci_test_android.yml b/.github/workflows/pkgci_test_android.yml
index bb12582..7236e05 100644
--- a/.github/workflows/pkgci_test_android.yml
+++ b/.github/workflows/pkgci_test_android.yml
@@ -31,7 +31,7 @@
contents: read
jobs:
- cross_compile:
+ android_arm64:
runs-on: ubuntu-20.04
env:
PACKAGE_DOWNLOAD_DIR: ${{ github.workspace }}/.packages
diff --git a/.github/workflows/pkgci_test_nvidia_t4.yml b/.github/workflows/pkgci_test_nvidia_t4.yml
index 1e837cb..9fd4245 100644
--- a/.github/workflows/pkgci_test_nvidia_t4.yml
+++ b/.github/workflows/pkgci_test_nvidia_t4.yml
@@ -18,7 +18,7 @@
default: ""
jobs:
- test:
+ test_t4:
runs-on:
- self-hosted # must come first
- runner-group=${{ github.event_name == 'pull_request' && 'presubmit' || 'postsubmit' }}
diff --git a/.github/workflows/pkgci_test_onnx.yml b/.github/workflows/pkgci_test_onnx.yml
new file mode 100644
index 0000000..5f2191c
--- /dev/null
+++ b/.github/workflows/pkgci_test_onnx.yml
@@ -0,0 +1,121 @@
+# Copyright 2024 The IREE Authors
+#
+# Licensed under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+name: PkgCI Test ONNX
+on:
+ workflow_call:
+ inputs:
+ artifact_run_id:
+ type: string
+ default: ""
+ workflow_dispatch:
+ inputs:
+ artifact_run_id:
+ type: string
+ default: ""
+
+jobs:
+ test_onnx_ops:
+ name: "test_onnx :: ${{ matrix.name }}"
+ runs-on: ${{ matrix.runs-on }}
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ # CPU
+ - name: cpu_llvm_sync
+ config-file: onnx_ops_cpu_llvm_sync.json
+ numprocesses: auto
+ runs-on: ubuntu-20.04
+
+ # AMD GPU
+ - name: amdgpu_rocm_rdna3
+ numprocesses: 1
+ config-file: onnx_ops_gpu_rocm_rdna3.json
+ runs-on: nodai-amdgpu-w7900-x86-64
+ - name: amdgpu_vulkan
+ numprocesses: 4
+ config-file: onnx_ops_gpu_vulkan.json
+ runs-on: nodai-amdgpu-w7900-x86-64
+
+ # NVIDIA GPU
+ - name: nvidiagpu_cuda
+ config-file: onnx_ops_gpu_cuda.json
+ numprocesses: 4
+ runs-on:
+ - self-hosted # must come first
+ - runner-group=${{ github.event_name == 'pull_request' && 'presubmit' || 'postsubmit' }}
+ - environment=prod
+ - gpu # TODO(scotttodd): qualify further with vendor/model
+ - os-family=Linux
+ - name: nvidiagpu_vulkan
+ config-file: onnx_ops_gpu_vulkan.json
+ numprocesses: 4
+ runs-on:
+ - self-hosted # must come first
+ - runner-group=${{ github.event_name == 'pull_request' && 'presubmit' || 'postsubmit' }}
+ - environment=prod
+ - gpu # TODO(scotttodd): qualify further with vendor/model
+ - os-family=Linux
+ env:
+ PACKAGE_DOWNLOAD_DIR: ${{ github.workspace }}/.packages
+ CONFIG_FILE_PATH: tests/external/iree-test-suites/onnx_ops/${{ matrix.config-file }}
+ NUMPROCESSES: ${{ matrix.numprocesses }}
+ LOG_FILE_PATH: /tmp/test_onnx_ops_${{ matrix.name }}_logs.json
+ VENV_DIR: ${{ github.workspace }}/venv
+ steps:
+ - name: Checking out IREE repository
+ uses: actions/checkout@v4.1.7
+ with:
+ submodules: false
+ - uses: actions/setup-python@v5.1.0
+ with:
+ # Must match the subset of versions built in pkgci_build_packages.
+ python-version: "3.11"
+ - uses: actions/download-artifact@v4.1.7
+ with:
+ name: linux_x86_64_release_packages
+ path: ${{ env.PACKAGE_DOWNLOAD_DIR }}
+ - name: Setup venv
+ run: |
+ ./build_tools/pkgci/setup_venv.py ${VENV_DIR} \
+ --artifact-path=${PACKAGE_DOWNLOAD_DIR} \
+ --fetch-gh-workflow=${{ inputs.artifact_run_id }}
+
+ - name: Checkout test suites repository
+ uses: actions/checkout@v4.1.7
+ with:
+ repository: iree-org/iree-test-suites
+ ref: 9e921d0ea271a85f772eee22965585461c9b14c2
+ path: iree-test-suites
+ - name: Install ONNX ops test suite requirements
+ run: |
+ source ${VENV_DIR}/bin/activate
+ python -m pip install -r iree-test-suites/onnx_ops/requirements.txt
+ - name: Run ONNX ops test suite
+ run: |
+ source ${VENV_DIR}/bin/activate
+ pytest iree-test-suites/onnx_ops/ \
+ -rpfE \
+ --numprocesses ${NUMPROCESSES} \
+ --timeout=30 \
+ --durations=20 \
+ --config-files=${CONFIG_FILE_PATH} \
+ --report-log=${LOG_FILE_PATH}
+ - name: "Updating config file with latest XFAIL lists"
+ if: failure()
+ run: |
+ source ${VENV_DIR}/bin/activate
+ python iree-test-suites/onnx_ops/update_config_xfails.py \
+ --log-file=${LOG_FILE_PATH} \
+ --config-file=${CONFIG_FILE_PATH}
+ cat ${CONFIG_FILE_PATH}
+ - name: "Uploading new config file"
+ if: failure()
+ uses: actions/upload-artifact@v4.3.3
+ with:
+ name: ${{ matrix.config-file }}
+ path: ${{ env.CONFIG_FILE_PATH }}
diff --git a/.github/workflows/pkgci_test_riscv64.yml b/.github/workflows/pkgci_test_riscv64.yml
index 4ec6b47..82dd6f7 100644
--- a/.github/workflows/pkgci_test_riscv64.yml
+++ b/.github/workflows/pkgci_test_riscv64.yml
@@ -31,7 +31,7 @@
contents: read
jobs:
- cross_compile:
+ riscv64:
runs-on: ubuntu-20.04
env:
PACKAGE_DOWNLOAD_DIR: ${{ github.workspace }}/.packages
diff --git a/.github/workflows/pkgci_test_tensorflow_cpu.yml b/.github/workflows/pkgci_test_tensorflow.yml
similarity index 97%
rename from .github/workflows/pkgci_test_tensorflow_cpu.yml
rename to .github/workflows/pkgci_test_tensorflow.yml
index 221274a..3f9cfcd 100644
--- a/.github/workflows/pkgci_test_tensorflow_cpu.yml
+++ b/.github/workflows/pkgci_test_tensorflow.yml
@@ -4,7 +4,7 @@
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-name: PkgCI Test TensorFlow (CPU)
+name: PkgCI Test TensorFlow
on:
workflow_call:
inputs:
diff --git a/.github/workflows/pkgci_unit_test.yml b/.github/workflows/pkgci_unit_test.yml
index 27ed919..5ec0264 100644
--- a/.github/workflows/pkgci_unit_test.yml
+++ b/.github/workflows/pkgci_unit_test.yml
@@ -60,3 +60,22 @@
source ${VENV_DIR}/bin/activate
echo "Testing compiler package:"
python -m iree.compiler._package_test
+
+ # Test tools and importers that require additional deps.
+ - name: Test ONNX importer
+ run: |
+ source ${VENV_DIR}/bin/activate
+ python -m pip install onnx>=1.16.0
+ python compiler/bindings/python/test/tools/import_onnx_test.py
+ python compiler/bindings/python/test/extras/onnx_importer_test.py
+ - name: Test FX (PyTorch) importer
+ run: |
+ source ${VENV_DIR}/bin/activate
+ python -m pip install torch>=2.3.0
+ python compiler/bindings/python/test/extras/fx_importer_test.py
+ - name: Test TensorFlow importer
+ run: |
+ source ${VENV_DIR}/bin/activate
+ bash ./build_tools/scripts/setup_tf_python.sh
+ python compiler/bindings/python/test/tools/compiler_tflite_test.py
+ python compiler/bindings/python/test/tools/compiler_tf_test.py
diff --git a/build_tools/cmake/run_tf_tests.sh b/build_tools/cmake/run_tf_tests.sh
index 27380ce..2d5d8d8 100755
--- a/build_tools/cmake/run_tf_tests.sh
+++ b/build_tools/cmake/run_tf_tests.sh
@@ -62,9 +62,3 @@
echo "Some tests failed!!!"
exit 1
fi
-
-echo "***** Running TF and TFLite python api tests *****"
-
-TF_API_TEST_DIR="compiler/bindings/python/test/tools"
-
-pytest ${TF_API_TEST_DIR}/compiler_tflite_test.py ${TF_API_TEST_DIR}/compiler_tf_test.py
diff --git a/compiler/bindings/python/test/extras/fx_importer_test.py b/compiler/bindings/python/test/extras/fx_importer_test.py
index e6bfd13..7b9d42c 100644
--- a/compiler/bindings/python/test/extras/fx_importer_test.py
+++ b/compiler/bindings/python/test/extras/fx_importer_test.py
@@ -6,13 +6,16 @@
try:
from iree.compiler.extras import fx_importer
+
+ print("fx_importer imported successfully")
except ModuleNotFoundError as e:
+ e_orig = e
while e is not None:
if isinstance(e, ModuleNotFoundError) and e.name == "torch":
+ print("torch not found, skipping fx_importer_test")
break
e = e.__cause__
else:
raise ModuleNotFoundError(
- "Failed to import the fx_importer (for a reason other than torch "
- "not being found)"
- ) from e
+ f"Failed to import the fx_importer (for a reason other than torch not being found)"
+ ) from e_orig
diff --git a/compiler/bindings/python/test/extras/onnx_importer_test.py b/compiler/bindings/python/test/extras/onnx_importer_test.py
index f06c81a..8c5e732 100644
--- a/compiler/bindings/python/test/extras/onnx_importer_test.py
+++ b/compiler/bindings/python/test/extras/onnx_importer_test.py
@@ -6,13 +6,16 @@
try:
from iree.compiler.extras import onnx_importer
+
+ print("onnx_importer imported successfully")
except ModuleNotFoundError as e:
+ e_orig = e
while e is not None:
if isinstance(e, ModuleNotFoundError) and e.name == "onnx":
+ print("onnx not found, skipping onnx_importer_test")
break
e = e.__cause__
else:
raise ModuleNotFoundError(
- "Failed to import the fx_importer (for a reason other than onnx "
- "not being found)"
- ) from e
+ "Failed to import the onnx_importer (for a reason other than onnx not being found)"
+ ) from e_orig
