Bump test suite to include punet tests and fix xfail script. (#17897)
Highlights from this update:
*
https://github.com/nod-ai/SHARK-TestSuite/commit/fc9fd6224f8ecf35159c0984c418b2e5c4781e35
Fixing the `update_config_xfails.py` script
*
https://github.com/nod-ai/SHARK-TestSuite/commit/2c0a31242ba977f3f16952090a1de757f5a54f98
(and follow-ups) Adding the [Partitioned UNet
(punet)](https://github.com/nod-ai/sharktank/tree/main/sharktank/sharktank/models/punet)
model to the test suite, using data from
https://huggingface.co/amd-shark/sdxl-quant-models
This also changes the 'models' pytest command to include all output,
even for xfail'd tests. This will let us see the latest compile/run
errors for these programs using the tested flags.
ci-exactly: build_packages,regression_test
diff --git a/.github/workflows/pkgci_regression_test.yml b/.github/workflows/pkgci_regression_test.yml
index 1a61c76..d2d7c0f 100644
--- a/.github/workflows/pkgci_regression_test.yml
+++ b/.github/workflows/pkgci_regression_test.yml
@@ -90,7 +90,7 @@
uses: actions/checkout@v4.1.7
with:
repository: nod-ai/SHARK-TestSuite
- ref: a06e730ce325c12db40bb89b43e8e6e897052e96
+ ref: 1da89a12a8b00dc77506d702f61300803b6240b7
path: SHARK-TestSuite
submodules: false
lfs: false
@@ -198,7 +198,7 @@
uses: actions/checkout@v4.1.7
with:
repository: nod-ai/SHARK-TestSuite
- ref: a06e730ce325c12db40bb89b43e8e6e897052e96
+ ref: 1da89a12a8b00dc77506d702f61300803b6240b7
path: SHARK-TestSuite
submodules: false
lfs: true
@@ -220,7 +220,7 @@
pytest \
SHARK-TestSuite/iree_tests/pytorch/models \
SHARK-TestSuite/iree_tests/sharktank \
- -rpfE \
+ -rA \
-k real_weights \
--no-skip-tests-missing-files \
--capture=no \
diff --git a/build_tools/pkgci/external_test_suite/models_gpu_rocm_gfx90a.json b/build_tools/pkgci/external_test_suite/models_gpu_rocm_gfx90a.json
index 8b7f59c..1b15430 100644
--- a/build_tools/pkgci/external_test_suite/models_gpu_rocm_gfx90a.json
+++ b/build_tools/pkgci/external_test_suite/models_gpu_rocm_gfx90a.json
@@ -20,9 +20,11 @@
"pytorch/models/resnet50",
"pytorch/models/sdxl-vae-decode-tank",
- // error: 'builtin.module' op failed to run transform dialect passes
- // (transform spec file is specific to SDXL?)
- "sharktank/llama/open-llama-3b-v2-f16"
+ // TODO(#17874): error: a handle passed as operand #0 and consumed by this operation points to a payload entity more than once
+ "sharktank/llama/open-llama-3b-v2-f16",
+
+ "sharktank/punet/fp16",
+ "sharktank/punet/int8"
],
"expected_run_failures": []
}
diff --git a/build_tools/pkgci/external_test_suite/models_gpu_rocm_gfx942.json b/build_tools/pkgci/external_test_suite/models_gpu_rocm_gfx942.json
index 5d451f0..0cf808b 100644
--- a/build_tools/pkgci/external_test_suite/models_gpu_rocm_gfx942.json
+++ b/build_tools/pkgci/external_test_suite/models_gpu_rocm_gfx942.json
@@ -20,9 +20,11 @@
"pytorch/models/resnet50",
"pytorch/models/sdxl-vae-decode-tank",
- // error: 'builtin.module' op failed to run transform dialect passes
- // (transform spec file is specific to SDXL?)
- "sharktank/llama/open-llama-3b-v2-f16"
+ // TODO(#17874): error: a handle passed as operand #0 and consumed by this operation points to a payload entity more than once
+ "sharktank/llama/open-llama-3b-v2-f16",
+
+ "sharktank/punet/fp16",
+ "sharktank/punet/int8"
],
"expected_run_failures": []
}
diff --git a/build_tools/pkgci/external_test_suite/models_gpu_vulkan.json b/build_tools/pkgci/external_test_suite/models_gpu_vulkan.json
index d8af23a..7b40397 100644
--- a/build_tools/pkgci/external_test_suite/models_gpu_vulkan.json
+++ b/build_tools/pkgci/external_test_suite/models_gpu_vulkan.json
@@ -12,13 +12,16 @@
// TODO(#17344): need to regenerate .mlirbc
"pytorch/models/opt-125M",
"pytorch/models/resnet50",
+
"pytorch/models/sdxl-prompt-encoder-tank",
"pytorch/models/sdxl-scheduled-unet-3-tank",
"pytorch/models/sdxl-vae-decode-tank",
- // error: 'builtin.module' op failed to run transform dialect passes
- // (transform spec file is specific to SDXL?)
- "sharktank/llama/open-llama-3b-v2-f16"
+ // TODO(#17874): error: a handle passed as operand #0 and consumed by this operation points to a payload entity more than once
+ "sharktank/llama/open-llama-3b-v2-f16",
+
+ "sharktank/punet/fp16",
+ "sharktank/punet/int8"
],
"expected_run_failures": []
}
