Bump SHARK-TestSuite commit to include npy->bin output comparison. (#17281)
This fixes https://github.com/iree-org/iree/issues/16674 by switching
from comparing against .npy files to comparing against .bin files thanks
to https://github.com/nod-ai/SHARK-TestSuite/pull/212.
For example with
https://github.com/nod-ai/SHARK-TestSuite/tree/main/iree_tests/onnx/node/generated/test_add_uint8:
* model.mlir:
```mlir
module {
func.func @test_add_uint8(%arg0: !torch.vtensor<[3,4,5],ui8>, %arg1:
!torch.vtensor<[3,4,5],ui8>) -> !torch.vtensor<[3,4,5],ui8> attributes
{torch.onnx_meta.ir_version = 7 : si64, torch.onnx_meta.opset_version =
17 : si64, torch.onnx_meta.producer_name = "backend-test",
torch.onnx_meta.producer_version = ""} {
%none = torch.constant.none
%0 = torch.operator "onnx.Add"(%arg0, %arg1) :
(!torch.vtensor<[3,4,5],ui8>, !torch.vtensor<[3,4,5],ui8>) ->
!torch.vtensor<[3,4,5],ui8>
return %0 : !torch.vtensor<[3,4,5],ui8>
}
}
```
* `test_data_flags.txt` before:
```
--input=@input_0.npy
--input=@input_1.npy
--expected_output=@output_0.npy
```
with these errors:
```
[FAILED] result[0]: metadata is 3x4x5xi8; expected that the view matches
3x4x5xui8; expected that the view is equal to contents of a view of
3x4x5xui8
expected:
3x4x5xui8=[[18 24 22 22 20][14 38 44 14 27][30 14 20 12 20][36 25 31 6
8]][[28 30 19 7 18][26 33 23 34 19][29 39 14 32 44][26 25 18 28 24]][[24
36 16 28 25][16 23 22 20 39][27 20 34 19 26][16 17 8 16 11]]
actual:
3x4x5xi8=[[18 24 22 22 20][14 38 44 14 27][30 14 20 12 20][36 25 31 6
8]][[28 30 19 7 18][26 33 23 34 19][29 39 14 32 44][26 25 18 28 24]][[24
36 16 28 25][16 23 22 20 39][27 20 34 19 26][16 17 8 16 11]]
```
* `test_data_flags.txt` after:
```
--input=3x4x5xui8=@input_0.bin
--input=3x4x5xui8=@input_1.bin
--expected_output=3x4x5xi8=@output_0.bin
```
(that still seems a bit mismatched? it does pass the test though)diff --git a/.github/workflows/pkgci_regression_test_amdgpu_rocm.yml b/.github/workflows/pkgci_regression_test_amdgpu_rocm.yml
index 87719c5..79aae61 100644
--- a/.github/workflows/pkgci_regression_test_amdgpu_rocm.yml
+++ b/.github/workflows/pkgci_regression_test_amdgpu_rocm.yml
@@ -60,7 +60,7 @@
uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
with:
repository: nod-ai/SHARK-TestSuite
- ref: 072e8b7f3140b31669257e6042dc1f02f2a4e2cc
+ ref: 9e9774632d32674e9ffa44b0a2ffc054f6c590ad
path: SHARK-TestSuite
submodules: false
- name: Installing external TestSuite Python requirements
@@ -123,7 +123,7 @@
uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
with:
repository: nod-ai/SHARK-TestSuite
- ref: 072e8b7f3140b31669257e6042dc1f02f2a4e2cc
+ ref: 9e9774632d32674e9ffa44b0a2ffc054f6c590ad
path: SHARK-TestSuite
submodules: false
lfs: true
diff --git a/.github/workflows/pkgci_regression_test_amdgpu_vulkan.yml b/.github/workflows/pkgci_regression_test_amdgpu_vulkan.yml
index f0dbfd8..9d5ba18 100644
--- a/.github/workflows/pkgci_regression_test_amdgpu_vulkan.yml
+++ b/.github/workflows/pkgci_regression_test_amdgpu_vulkan.yml
@@ -57,7 +57,7 @@
uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
with:
repository: nod-ai/SHARK-TestSuite
- ref: 072e8b7f3140b31669257e6042dc1f02f2a4e2cc
+ ref: 9e9774632d32674e9ffa44b0a2ffc054f6c590ad
path: SHARK-TestSuite
submodules: false
- name: Installing external TestSuite Python requirements
@@ -106,7 +106,7 @@
uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
with:
repository: nod-ai/SHARK-TestSuite
- ref: 072e8b7f3140b31669257e6042dc1f02f2a4e2cc
+ ref: 9e9774632d32674e9ffa44b0a2ffc054f6c590ad
path: SHARK-TestSuite
submodules: false
lfs: true
diff --git a/.github/workflows/pkgci_regression_test_cpu.yml b/.github/workflows/pkgci_regression_test_cpu.yml
index 95daebc..19e2bb0 100644
--- a/.github/workflows/pkgci_regression_test_cpu.yml
+++ b/.github/workflows/pkgci_regression_test_cpu.yml
@@ -57,7 +57,7 @@
uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
with:
repository: nod-ai/SHARK-TestSuite
- ref: 072e8b7f3140b31669257e6042dc1f02f2a4e2cc
+ ref: 9e9774632d32674e9ffa44b0a2ffc054f6c590ad
path: SHARK-TestSuite
submodules: false
lfs: true
@@ -122,7 +122,7 @@
uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
with:
repository: nod-ai/SHARK-TestSuite
- ref: 072e8b7f3140b31669257e6042dc1f02f2a4e2cc
+ ref: 9e9774632d32674e9ffa44b0a2ffc054f6c590ad
path: SHARK-TestSuite
submodules: false
lfs: true
diff --git a/.github/workflows/pkgci_regression_test_nvidiagpu_cuda.yml b/.github/workflows/pkgci_regression_test_nvidiagpu_cuda.yml
index 2eeed0f..0d28a55 100644
--- a/.github/workflows/pkgci_regression_test_nvidiagpu_cuda.yml
+++ b/.github/workflows/pkgci_regression_test_nvidiagpu_cuda.yml
@@ -57,7 +57,7 @@
uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
with:
repository: nod-ai/SHARK-TestSuite
- ref: 072e8b7f3140b31669257e6042dc1f02f2a4e2cc
+ ref: 9e9774632d32674e9ffa44b0a2ffc054f6c590ad
path: SHARK-TestSuite
submodules: false
- name: Installing external TestSuite Python requirements
diff --git a/.github/workflows/pkgci_regression_test_nvidiagpu_vulkan.yml b/.github/workflows/pkgci_regression_test_nvidiagpu_vulkan.yml
index aba5e91..60dd4d5 100644
--- a/.github/workflows/pkgci_regression_test_nvidiagpu_vulkan.yml
+++ b/.github/workflows/pkgci_regression_test_nvidiagpu_vulkan.yml
@@ -57,7 +57,7 @@
uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
with:
repository: nod-ai/SHARK-TestSuite
- ref: 072e8b7f3140b31669257e6042dc1f02f2a4e2cc
+ ref: 9e9774632d32674e9ffa44b0a2ffc054f6c590ad
path: SHARK-TestSuite
submodules: false
- name: Installing external TestSuite Python requirements
diff --git a/build_tools/pkgci/external_test_suite/onnx_cpu_llvm_sync.json b/build_tools/pkgci/external_test_suite/onnx_cpu_llvm_sync.json
index 07b7b16..38ac383 100644
--- a/build_tools/pkgci/external_test_suite/onnx_cpu_llvm_sync.json
+++ b/build_tools/pkgci/external_test_suite/onnx_cpu_llvm_sync.json
@@ -603,66 +603,10 @@
"test_wrap_pad"
],
"expected_run_failures": [
- "test_add_uint8",
- "test_argmax_default_axis_example",
- "test_argmax_default_axis_example_select_last_index",
- "test_argmax_default_axis_random",
- "test_argmax_default_axis_random_select_last_index",
- "test_argmax_keepdims_example",
- "test_argmax_keepdims_example_select_last_index",
- "test_argmax_keepdims_random",
- "test_argmax_keepdims_random_select_last_index",
- "test_argmax_negative_axis_keepdims_example",
- "test_argmax_negative_axis_keepdims_example_select_last_index",
- "test_argmax_negative_axis_keepdims_random",
- "test_argmax_negative_axis_keepdims_random_select_last_index",
- "test_argmax_no_keepdims_example",
- "test_argmax_no_keepdims_example_select_last_index",
- "test_argmax_no_keepdims_random",
- "test_argmax_no_keepdims_random_select_last_index",
- "test_argmin_default_axis_example",
- "test_argmin_default_axis_example_select_last_index",
- "test_argmin_default_axis_random",
- "test_argmin_default_axis_random_select_last_index",
- "test_argmin_keepdims_example",
- "test_argmin_keepdims_example_select_last_index",
- "test_argmin_keepdims_random",
- "test_argmin_keepdims_random_select_last_index",
- "test_argmin_negative_axis_keepdims_example",
- "test_argmin_negative_axis_keepdims_example_select_last_index",
- "test_argmin_negative_axis_keepdims_random",
- "test_argmin_negative_axis_keepdims_random_select_last_index",
- "test_argmin_no_keepdims_example",
- "test_argmin_no_keepdims_example_select_last_index",
- "test_argmin_no_keepdims_random",
- "test_argmin_no_keepdims_random_select_last_index",
"test_averagepool_2d_ceil",
"test_bernoulli",
"test_bernoulli_double_expanded",
"test_bernoulli_expanded",
- "test_bitshift_left_uint16",
- "test_bitshift_left_uint32",
- "test_bitshift_left_uint64",
- "test_bitshift_left_uint8",
- "test_bitshift_right_uint16",
- "test_bitshift_right_uint32",
- "test_bitshift_right_uint64",
- "test_bitshift_right_uint8",
- "test_bitwise_and_i16_3d",
- "test_bitwise_and_i32_2d",
- "test_bitwise_and_ui64_bcast_3v1d",
- "test_bitwise_and_ui8_bcast_4v3d",
- "test_bitwise_not_2d",
- "test_bitwise_not_3d",
- "test_bitwise_not_4d",
- "test_bitwise_or_i16_4d",
- "test_bitwise_or_i32_2d",
- "test_bitwise_or_ui64_bcast_3v1d",
- "test_bitwise_or_ui8_bcast_4v3d",
- "test_bitwise_xor_i16_3d",
- "test_bitwise_xor_i32_2d",
- "test_bitwise_xor_ui64_bcast_3v1d",
- "test_bitwise_xor_ui8_bcast_4v3d",
"test_cast_BFLOAT16_to_FLOAT",
"test_cast_DOUBLE_to_FLOAT",
"test_cast_DOUBLE_to_FLOAT16",
@@ -681,64 +625,28 @@
"test_castlike_FLOAT_to_BFLOAT16_expanded",
"test_castlike_FLOAT_to_DOUBLE",
"test_castlike_FLOAT_to_DOUBLE_expanded",
- "test_clip_default_int8_max",
- "test_clip_default_int8_max_expanded",
- "test_clip_default_int8_min",
- "test_clip_default_int8_min_expanded",
+ "test_clip_default_int8_inbounds",
+ "test_clip_default_int8_inbounds_expanded",
"test_constant_pad",
"test_constantofshape_float_ones",
"test_constantofshape_int_shape_zero",
"test_constantofshape_int_zeros",
- "test_convinteger_with_padding",
- "test_convinteger_without_padding",
- "test_div_uint8",
"test_dropout_default_mask_ratio",
- "test_dynamicquantizelinear",
- "test_dynamicquantizelinear_expanded",
- "test_dynamicquantizelinear_max_adjusted",
- "test_dynamicquantizelinear_max_adjusted_expanded",
- "test_dynamicquantizelinear_min_adjusted",
- "test_dynamicquantizelinear_min_adjusted_expanded",
"test_einsum_transpose",
"test_elu_default",
"test_eyelike_with_dtype",
- "test_eyelike_without_dtype",
"test_gather_elements_negative_indices",
- "test_gathernd_example_int32",
"test_hardsigmoid",
"test_hardsigmoid_default",
"test_hardsigmoid_example",
"test_hardswish_expanded",
- "test_matmulinteger",
"test_max_float64",
- "test_max_int16",
- "test_max_int32",
- "test_max_int64",
- "test_max_int8",
- "test_max_uint16",
- "test_max_uint32",
- "test_max_uint64",
- "test_max_uint8",
"test_min_float64",
- "test_min_int16",
- "test_min_int32",
- "test_min_int64",
- "test_min_int8",
- "test_min_uint16",
- "test_min_uint32",
- "test_min_uint64",
- "test_min_uint8",
- "test_mod_broadcast",
- "test_mod_int64_fmod",
"test_mod_mixed_sign_int16",
"test_mod_mixed_sign_int32",
"test_mod_mixed_sign_int64",
"test_mod_mixed_sign_int8",
- "test_mod_uint16",
- "test_mod_uint32",
"test_mod_uint64",
- "test_mod_uint8",
- "test_mul_uint8",
"test_pow",
"test_pow_example",
"test_pow_types_float32_int32",
@@ -748,23 +656,14 @@
"test_qlinearconv",
"test_qlinearmatmul_2D_int8_float16",
"test_qlinearmatmul_2D_int8_float32",
- "test_qlinearmatmul_2D_uint8_float16",
- "test_qlinearmatmul_2D_uint8_float32",
"test_qlinearmatmul_3D_int8_float16",
"test_qlinearmatmul_3D_int8_float32",
"test_qlinearmatmul_3D_uint8_float16",
"test_qlinearmatmul_3D_uint8_float32",
- "test_quantizelinear",
- "test_range_int32_type_negative_delta",
"test_reduce_min_empty_set",
"test_reduce_sum_empty_set_non_reduced_axis_zero",
"test_scatter_elements_with_negative_indices",
- "test_shape",
- "test_shape_clip_end",
- "test_shape_clip_start",
"test_shape_end_1",
- "test_shape_end_negative_1",
- "test_shape_example",
"test_shape_start_1",
"test_shape_start_1_end_2",
"test_shape_start_1_end_negative_1",
@@ -773,19 +672,6 @@
"test_size_example",
"test_split_zero_size_splits_opset13",
"test_split_zero_size_splits_opset18",
- "test_sub_uint8",
- "test_top_k",
- "test_top_k_negative_axis",
- "test_top_k_smallest",
- "test_tril",
- "test_tril_neg",
- "test_tril_one_row_neg",
- "test_tril_out_neg",
- "test_tril_out_pos",
- "test_tril_pos",
- "test_tril_square",
- "test_tril_square_neg",
- "test_tril_zero",
- "test_where_long_example"
+ "test_tril_zero"
]
}
diff --git a/build_tools/pkgci/external_test_suite/onnx_gpu_cuda.json b/build_tools/pkgci/external_test_suite/onnx_gpu_cuda.json
index 56c95dc..b2a9fe9 100644
--- a/build_tools/pkgci/external_test_suite/onnx_gpu_cuda.json
+++ b/build_tools/pkgci/external_test_suite/onnx_gpu_cuda.json
@@ -604,66 +604,10 @@
"test_wrap_pad"
],
"expected_run_failures": [
- "test_add_uint8",
- "test_argmax_default_axis_example",
- "test_argmax_default_axis_example_select_last_index",
- "test_argmax_default_axis_random",
- "test_argmax_default_axis_random_select_last_index",
- "test_argmax_keepdims_example",
- "test_argmax_keepdims_example_select_last_index",
- "test_argmax_keepdims_random",
- "test_argmax_keepdims_random_select_last_index",
- "test_argmax_negative_axis_keepdims_example",
- "test_argmax_negative_axis_keepdims_example_select_last_index",
- "test_argmax_negative_axis_keepdims_random",
- "test_argmax_negative_axis_keepdims_random_select_last_index",
- "test_argmax_no_keepdims_example",
- "test_argmax_no_keepdims_example_select_last_index",
- "test_argmax_no_keepdims_random",
- "test_argmax_no_keepdims_random_select_last_index",
- "test_argmin_default_axis_example",
- "test_argmin_default_axis_example_select_last_index",
- "test_argmin_default_axis_random",
- "test_argmin_default_axis_random_select_last_index",
- "test_argmin_keepdims_example",
- "test_argmin_keepdims_example_select_last_index",
- "test_argmin_keepdims_random",
- "test_argmin_keepdims_random_select_last_index",
- "test_argmin_negative_axis_keepdims_example",
- "test_argmin_negative_axis_keepdims_example_select_last_index",
- "test_argmin_negative_axis_keepdims_random",
- "test_argmin_negative_axis_keepdims_random_select_last_index",
- "test_argmin_no_keepdims_example",
- "test_argmin_no_keepdims_example_select_last_index",
- "test_argmin_no_keepdims_random",
- "test_argmin_no_keepdims_random_select_last_index",
"test_averagepool_2d_ceil",
"test_bernoulli",
"test_bernoulli_double_expanded",
"test_bernoulli_expanded",
- "test_bitshift_left_uint16",
- "test_bitshift_left_uint32",
- "test_bitshift_left_uint64",
- "test_bitshift_left_uint8",
- "test_bitshift_right_uint16",
- "test_bitshift_right_uint32",
- "test_bitshift_right_uint64",
- "test_bitshift_right_uint8",
- "test_bitwise_and_i16_3d",
- "test_bitwise_and_i32_2d",
- "test_bitwise_and_ui64_bcast_3v1d",
- "test_bitwise_and_ui8_bcast_4v3d",
- "test_bitwise_not_2d",
- "test_bitwise_not_3d",
- "test_bitwise_not_4d",
- "test_bitwise_or_i16_4d",
- "test_bitwise_or_i32_2d",
- "test_bitwise_or_ui64_bcast_3v1d",
- "test_bitwise_or_ui8_bcast_4v3d",
- "test_bitwise_xor_i16_3d",
- "test_bitwise_xor_i32_2d",
- "test_bitwise_xor_ui64_bcast_3v1d",
- "test_bitwise_xor_ui8_bcast_4v3d",
"test_cast_BFLOAT16_to_FLOAT",
"test_cast_DOUBLE_to_FLOAT",
"test_cast_DOUBLE_to_FLOAT16",
@@ -682,64 +626,27 @@
"test_castlike_FLOAT_to_BFLOAT16_expanded",
"test_castlike_FLOAT_to_DOUBLE",
"test_castlike_FLOAT_to_DOUBLE_expanded",
- "test_clip_default_int8_max",
- "test_clip_default_int8_max_expanded",
- "test_clip_default_int8_min",
- "test_clip_default_int8_min_expanded",
+ "test_clip_default_int8_inbounds",
+ "test_clip_default_int8_inbounds_expanded",
"test_constant_pad",
"test_constantofshape_float_ones",
"test_constantofshape_int_shape_zero",
"test_constantofshape_int_zeros",
- "test_convinteger_with_padding",
- "test_convinteger_without_padding",
- "test_div_uint8",
"test_dropout_default_mask_ratio",
- "test_dynamicquantizelinear",
- "test_dynamicquantizelinear_expanded",
- "test_dynamicquantizelinear_max_adjusted",
- "test_dynamicquantizelinear_max_adjusted_expanded",
- "test_dynamicquantizelinear_min_adjusted",
- "test_dynamicquantizelinear_min_adjusted_expanded",
"test_einsum_transpose",
"test_elu_default",
"test_eyelike_with_dtype",
- "test_eyelike_without_dtype",
- "test_gathernd_example_int32",
"test_hardsigmoid",
"test_hardsigmoid_default",
"test_hardsigmoid_example",
"test_hardswish_expanded",
- "test_matmulinteger",
"test_max_float64",
- "test_max_int16",
- "test_max_int32",
- "test_max_int64",
- "test_max_int8",
- "test_max_uint16",
- "test_max_uint32",
- "test_max_uint64",
- "test_max_uint8",
"test_min_float64",
- "test_min_int16",
- "test_min_int32",
- "test_min_int64",
- "test_min_int8",
- "test_min_uint16",
- "test_min_uint32",
- "test_min_uint64",
- "test_min_uint8",
- "test_mod_broadcast",
- "test_mod_int64_fmod",
"test_mod_mixed_sign_float64",
"test_mod_mixed_sign_int16",
"test_mod_mixed_sign_int32",
"test_mod_mixed_sign_int64",
"test_mod_mixed_sign_int8",
- "test_mod_uint16",
- "test_mod_uint32",
- "test_mod_uint64",
- "test_mod_uint8",
- "test_mul_uint8",
"test_pow",
"test_pow_example",
"test_pow_types_float32_int32",
@@ -749,14 +656,10 @@
"test_qlinearconv",
"test_qlinearmatmul_2D_int8_float16",
"test_qlinearmatmul_2D_int8_float32",
- "test_qlinearmatmul_2D_uint8_float16",
- "test_qlinearmatmul_2D_uint8_float32",
"test_qlinearmatmul_3D_int8_float16",
"test_qlinearmatmul_3D_int8_float32",
"test_qlinearmatmul_3D_uint8_float16",
"test_qlinearmatmul_3D_uint8_float32",
- "test_quantizelinear",
- "test_range_int32_type_negative_delta",
"test_reduce_l1_default_axes_keepdims_example",
"test_reduce_l1_default_axes_keepdims_example_expanded",
"test_reduce_l1_default_axes_keepdims_random",
@@ -774,12 +677,7 @@
"test_reduce_sum_negative_axes_keepdims_random",
"test_reduce_sum_square_default_axes_keepdims_example_expanded",
"test_reduce_sum_square_default_axes_keepdims_random_expanded",
- "test_shape",
- "test_shape_clip_end",
- "test_shape_clip_start",
"test_shape_end_1",
- "test_shape_end_negative_1",
- "test_shape_example",
"test_shape_start_1",
"test_shape_start_1_end_2",
"test_shape_start_1_end_negative_1",
@@ -788,19 +686,6 @@
"test_size_example",
"test_split_zero_size_splits_opset13",
"test_split_zero_size_splits_opset18",
- "test_sub_uint8",
- "test_top_k",
- "test_top_k_negative_axis",
- "test_top_k_smallest",
- "test_tril",
- "test_tril_neg",
- "test_tril_one_row_neg",
- "test_tril_out_neg",
- "test_tril_out_pos",
- "test_tril_pos",
- "test_tril_square",
- "test_tril_square_neg",
- "test_tril_zero",
- "test_where_long_example"
+ "test_tril_zero"
]
}
diff --git a/build_tools/pkgci/external_test_suite/onnx_gpu_rocm_rdna3.json b/build_tools/pkgci/external_test_suite/onnx_gpu_rocm_rdna3.json
index 34e7930..b7287c6 100644
--- a/build_tools/pkgci/external_test_suite/onnx_gpu_rocm_rdna3.json
+++ b/build_tools/pkgci/external_test_suite/onnx_gpu_rocm_rdna3.json
@@ -601,66 +601,10 @@
"test_wrap_pad"
],
"expected_run_failures": [
- "test_add_uint8",
- "test_argmax_default_axis_example",
- "test_argmax_default_axis_example_select_last_index",
- "test_argmax_default_axis_random",
- "test_argmax_default_axis_random_select_last_index",
- "test_argmax_keepdims_example",
- "test_argmax_keepdims_example_select_last_index",
- "test_argmax_keepdims_random",
- "test_argmax_keepdims_random_select_last_index",
- "test_argmax_negative_axis_keepdims_example",
- "test_argmax_negative_axis_keepdims_example_select_last_index",
- "test_argmax_negative_axis_keepdims_random",
- "test_argmax_negative_axis_keepdims_random_select_last_index",
- "test_argmax_no_keepdims_example",
- "test_argmax_no_keepdims_example_select_last_index",
- "test_argmax_no_keepdims_random",
- "test_argmax_no_keepdims_random_select_last_index",
- "test_argmin_default_axis_example",
- "test_argmin_default_axis_example_select_last_index",
- "test_argmin_default_axis_random",
- "test_argmin_default_axis_random_select_last_index",
- "test_argmin_keepdims_example",
- "test_argmin_keepdims_example_select_last_index",
- "test_argmin_keepdims_random",
- "test_argmin_keepdims_random_select_last_index",
- "test_argmin_negative_axis_keepdims_example",
- "test_argmin_negative_axis_keepdims_example_select_last_index",
- "test_argmin_negative_axis_keepdims_random",
- "test_argmin_negative_axis_keepdims_random_select_last_index",
- "test_argmin_no_keepdims_example",
- "test_argmin_no_keepdims_example_select_last_index",
- "test_argmin_no_keepdims_random",
- "test_argmin_no_keepdims_random_select_last_index",
"test_averagepool_2d_ceil",
"test_bernoulli",
"test_bernoulli_double_expanded",
"test_bernoulli_expanded",
- "test_bitshift_left_uint16",
- "test_bitshift_left_uint32",
- "test_bitshift_left_uint64",
- "test_bitshift_left_uint8",
- "test_bitshift_right_uint16",
- "test_bitshift_right_uint32",
- "test_bitshift_right_uint64",
- "test_bitshift_right_uint8",
- "test_bitwise_and_i16_3d",
- "test_bitwise_and_i32_2d",
- "test_bitwise_and_ui64_bcast_3v1d",
- "test_bitwise_and_ui8_bcast_4v3d",
- "test_bitwise_not_2d",
- "test_bitwise_not_3d",
- "test_bitwise_not_4d",
- "test_bitwise_or_i16_4d",
- "test_bitwise_or_i32_2d",
- "test_bitwise_or_ui64_bcast_3v1d",
- "test_bitwise_or_ui8_bcast_4v3d",
- "test_bitwise_xor_i16_3d",
- "test_bitwise_xor_i32_2d",
- "test_bitwise_xor_ui64_bcast_3v1d",
- "test_bitwise_xor_ui8_bcast_4v3d",
"test_cast_BFLOAT16_to_FLOAT",
"test_cast_DOUBLE_to_FLOAT",
"test_cast_DOUBLE_to_FLOAT16",
@@ -683,24 +627,13 @@
"test_castlike_FLOAT_to_DOUBLE_expanded",
"test_castlike_FLOAT_to_FLOAT16",
"test_castlike_FLOAT_to_FLOAT16_expanded",
- "test_clip_default_int8_max",
- "test_clip_default_int8_max_expanded",
- "test_clip_default_int8_min",
- "test_clip_default_int8_min_expanded",
+ "test_clip_default_int8_inbounds",
+ "test_clip_default_int8_inbounds_expanded",
"test_constant_pad",
"test_constantofshape_float_ones",
"test_constantofshape_int_shape_zero",
"test_constantofshape_int_zeros",
- "test_convinteger_with_padding",
- "test_convinteger_without_padding",
- "test_div_uint8",
"test_dropout_default_mask_ratio",
- "test_dynamicquantizelinear",
- "test_dynamicquantizelinear_expanded",
- "test_dynamicquantizelinear_max_adjusted",
- "test_dynamicquantizelinear_max_adjusted_expanded",
- "test_dynamicquantizelinear_min_adjusted",
- "test_dynamicquantizelinear_min_adjusted_expanded",
"test_einsum_transpose",
"test_elu_default",
"test_expand_dim_changed",
@@ -709,43 +642,18 @@
"test_eyelike_with_dtype",
"test_eyelike_without_dtype",
"test_gather_elements_negative_indices",
- "test_gathernd_example_int32",
"test_gridsample_zeros_padding",
"test_hardsigmoid",
"test_hardsigmoid_default",
"test_hardsigmoid_example",
"test_hardswish_expanded",
- "test_matmulinteger",
"test_max_float64",
- "test_max_int16",
- "test_max_int32",
- "test_max_int64",
- "test_max_int8",
- "test_max_uint16",
- "test_max_uint32",
- "test_max_uint64",
- "test_max_uint8",
"test_min_float64",
- "test_min_int16",
- "test_min_int32",
- "test_min_int64",
- "test_min_int8",
- "test_min_uint16",
- "test_min_uint32",
- "test_min_uint64",
- "test_min_uint8",
- "test_mod_broadcast",
- "test_mod_int64_fmod",
"test_mod_mixed_sign_float64",
"test_mod_mixed_sign_int16",
"test_mod_mixed_sign_int32",
"test_mod_mixed_sign_int64",
"test_mod_mixed_sign_int8",
- "test_mod_uint16",
- "test_mod_uint32",
- "test_mod_uint64",
- "test_mod_uint8",
- "test_mul_uint8",
"test_pow",
"test_pow_example",
"test_pow_types_float32_int32",
@@ -755,13 +663,10 @@
"test_qlinearconv",
"test_qlinearmatmul_2D_int8_float16",
"test_qlinearmatmul_2D_int8_float32",
- "test_qlinearmatmul_2D_uint8_float16",
- "test_qlinearmatmul_2D_uint8_float32",
"test_qlinearmatmul_3D_int8_float16",
"test_qlinearmatmul_3D_int8_float32",
"test_qlinearmatmul_3D_uint8_float16",
"test_qlinearmatmul_3D_uint8_float32",
- "test_quantizelinear",
"test_range_float_type_positive_delta",
"test_range_int32_type_negative_delta",
"test_reduce_l1_default_axes_keepdims_example",
@@ -770,11 +675,11 @@
"test_reduce_l1_default_axes_keepdims_random_expanded",
"test_reduce_l2_default_axes_keepdims_example",
"test_reduce_l2_default_axes_keepdims_random",
- "test_reduce_mean_default_axes_keepdims_example",
"test_reduce_log_sum_asc_axes",
"test_reduce_log_sum_default",
"test_reduce_log_sum_desc_axes",
"test_reduce_log_sum_negative_axes",
+ "test_reduce_mean_default_axes_keepdims_example",
"test_reduce_mean_default_axes_keepdims_random",
"test_reduce_min_empty_set",
"test_reduce_sum_default_axes_keepdims_example",
@@ -799,19 +704,9 @@
"test_size_example",
"test_split_zero_size_splits_opset13",
"test_split_zero_size_splits_opset18",
- "test_sub_uint8",
"test_top_k",
"test_top_k_negative_axis",
"test_top_k_smallest",
- "test_tril",
- "test_tril_neg",
- "test_tril_one_row_neg",
- "test_tril_out_neg",
- "test_tril_out_pos",
- "test_tril_pos",
- "test_tril_square",
- "test_tril_square_neg",
- "test_tril_zero",
- "test_where_long_example"
+ "test_tril_zero"
]
}
diff --git a/build_tools/pkgci/external_test_suite/onnx_gpu_vulkan.json b/build_tools/pkgci/external_test_suite/onnx_gpu_vulkan.json
index 3f16ae8..ef20f42 100644
--- a/build_tools/pkgci/external_test_suite/onnx_gpu_vulkan.json
+++ b/build_tools/pkgci/external_test_suite/onnx_gpu_vulkan.json
@@ -632,53 +632,17 @@
"test_wrap_pad"
],
"expected_run_failures": [
- "test_add_uint8",
"test_and_bcast3v1d",
"test_and_bcast4v2d",
"test_and_bcast4v4d",
- "test_argmax_default_axis_example",
- "test_argmax_default_axis_random",
- "test_argmax_keepdims_example",
- "test_argmax_keepdims_random",
- "test_argmax_negative_axis_keepdims_example",
- "test_argmax_negative_axis_keepdims_random",
- "test_argmax_no_keepdims_example",
- "test_argmax_no_keepdims_random",
- "test_argmin_default_axis_example",
- "test_argmin_default_axis_random",
- "test_argmin_keepdims_example",
- "test_argmin_keepdims_random",
- "test_argmin_negative_axis_keepdims_example",
- "test_argmin_negative_axis_keepdims_random",
- "test_argmin_no_keepdims_example",
- "test_argmin_no_keepdims_random",
"test_averagepool_2d_ceil",
"test_bernoulli",
"test_bernoulli_double_expanded",
"test_bernoulli_expanded",
"test_bitshift_left_uint16",
- "test_bitshift_left_uint32",
- "test_bitshift_left_uint64",
"test_bitshift_left_uint8",
"test_bitshift_right_uint16",
- "test_bitshift_right_uint32",
- "test_bitshift_right_uint64",
"test_bitshift_right_uint8",
- "test_bitwise_and_i16_3d",
- "test_bitwise_and_i32_2d",
- "test_bitwise_and_ui64_bcast_3v1d",
- "test_bitwise_and_ui8_bcast_4v3d",
- "test_bitwise_not_2d",
- "test_bitwise_not_3d",
- "test_bitwise_not_4d",
- "test_bitwise_or_i16_4d",
- "test_bitwise_or_i32_2d",
- "test_bitwise_or_ui64_bcast_3v1d",
- "test_bitwise_or_ui8_bcast_4v3d",
- "test_bitwise_xor_i16_3d",
- "test_bitwise_xor_i32_2d",
- "test_bitwise_xor_ui64_bcast_3v1d",
- "test_bitwise_xor_ui8_bcast_4v3d",
"test_cast_BFLOAT16_to_FLOAT",
"test_cast_DOUBLE_to_FLOAT",
"test_cast_DOUBLE_to_FLOAT16",
@@ -715,20 +679,13 @@
"test_constantofshape_int_zeros",
"test_convinteger_with_padding",
"test_convinteger_without_padding",
- "test_div_uint8",
"test_dropout_default_mask_ratio",
"test_dynamicquantizelinear",
"test_dynamicquantizelinear_expanded",
- "test_dynamicquantizelinear_max_adjusted",
- "test_dynamicquantizelinear_max_adjusted_expanded",
- "test_dynamicquantizelinear_min_adjusted",
- "test_dynamicquantizelinear_min_adjusted_expanded",
"test_einsum_transpose",
"test_elu_default",
"test_eyelike_with_dtype",
- "test_eyelike_without_dtype",
"test_gather_elements_negative_indices",
- "test_gathernd_example_int32",
"test_hardsigmoid",
"test_hardsigmoid_default",
"test_hardsigmoid_example",
@@ -739,27 +696,16 @@
"test_max_float16",
"test_max_float64",
"test_max_int16",
- "test_max_int32",
- "test_max_int64",
"test_max_int8",
- "test_max_uint32",
- "test_max_uint64",
"test_min_float16",
"test_min_float64",
"test_min_int16",
- "test_min_int32",
- "test_min_int64",
"test_min_int8",
- "test_min_uint32",
- "test_min_uint64",
- "test_mod_broadcast",
"test_mod_mixed_sign_int16",
"test_mod_mixed_sign_int32",
"test_mod_mixed_sign_int8",
"test_mod_uint16",
- "test_mod_uint32",
"test_mod_uint8",
- "test_mul_uint8",
"test_or_bcast3v1d",
"test_or_bcast4v2d",
"test_or_bcast4v4d",
@@ -779,7 +725,6 @@
"test_qlinearmatmul_3D_uint8_float16",
"test_qlinearmatmul_3D_uint8_float32",
"test_quantizelinear",
- "test_range_int32_type_negative_delta",
"test_reduce_l1_default_axes_keepdims_example",
"test_reduce_l1_default_axes_keepdims_example_expanded",
"test_reduce_l1_default_axes_keepdims_random",
@@ -798,12 +743,7 @@
"test_reduce_sum_square_default_axes_keepdims_example_expanded",
"test_reduce_sum_square_default_axes_keepdims_random_expanded",
"test_scatter_elements_with_negative_indices",
- "test_shape",
- "test_shape_clip_end",
- "test_shape_clip_start",
"test_shape_end_1",
- "test_shape_end_negative_1",
- "test_shape_example",
"test_shape_start_1",
"test_shape_start_1_end_2",
"test_shape_start_1_end_negative_1",
@@ -812,20 +752,7 @@
"test_size_example",
"test_split_zero_size_splits_opset13",
"test_split_zero_size_splits_opset18",
- "test_sub_uint8",
- "test_top_k",
- "test_top_k_negative_axis",
- "test_top_k_smallest",
- "test_tril",
- "test_tril_neg",
- "test_tril_one_row_neg",
- "test_tril_out_neg",
- "test_tril_out_pos",
- "test_tril_pos",
- "test_tril_square",
- "test_tril_square_neg",
"test_tril_zero",
- "test_where_long_example",
"test_xor_bcast3v1d",
"test_xor_bcast4v2d",
"test_xor_bcast4v4d"
