################################################################################
# Autogenerated by build_tools/bazel_to_cmake/bazel_to_cmake.py from           #
# tests/e2e/matmul/BUILD.bazel                                                 #
#                                                                              #
# Use iree_cmake_extra_content from iree/build_defs.oss.bzl to add arbitrary   #
# CMake-only content.                                                          #
#                                                                              #
# To disable autogeneration for this file entirely, delete this header.        #
################################################################################

iree_add_all_subdirs()

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_arm_sme_nondt_f32_small_transpose_lhs_peel
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f32"
    "--acc_type=f32"
    "--shapes=small"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling=false"
    "--iree-llvmcpu-enable-scalable-vectorization"
    "--iree-llvmcpu-target-triple=aarch64-unknown-unknown"
    "--iree-preprocessing-pass-pipeline=builtin.module\(util.func\(iree-preprocessing-transpose-matmul-pass{input=lhs}\)\)"
    "--iree-llvmcpu-vector-pproc-strategy=peel"
  LABELS
    "requires-arm-sme"
  TARGET_CPU_FEATURES_VARIANTS
    "arm_64:sme:+sve,+sme"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_arm_sme_nondt_f32_small_transpose_lhs
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f32"
    "--acc_type=f32"
    "--shapes=small"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling=false"
    "--iree-llvmcpu-enable-scalable-vectorization"
    "--iree-llvmcpu-target-triple=aarch64-unknown-unknown"
    "--iree-preprocessing-pass-pipeline=builtin.module\(util.func\(iree-preprocessing-transpose-matmul-pass{input=lhs}\)\)"
  LABELS
    "requires-arm-sme"
  TARGET_CPU_FEATURES_VARIANTS
    "arm_64:sme:+sve,+sme"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_arm_sme_nondt_f32_small_peel
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f32"
    "--acc_type=f32"
    "--shapes=small"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling=false"
    "--iree-llvmcpu-enable-scalable-vectorization"
    "--iree-llvmcpu-target-triple=aarch64-unknown-unknown"
    "--iree-llvmcpu-vector-pproc-strategy=peel"
  LABELS
    "requires-arm-sme"
  TARGET_CPU_FEATURES_VARIANTS
    "arm_64:sme:+sve,+sme"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_arm_sme_nondt_f32_small
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f32"
    "--acc_type=f32"
    "--shapes=small"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling=false"
    "--iree-llvmcpu-enable-scalable-vectorization"
    "--iree-llvmcpu-target-triple=aarch64-unknown-unknown"
  LABELS
    "requires-arm-sme"
  TARGET_CPU_FEATURES_VARIANTS
    "arm_64:sme:+sve,+sme"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_arm_sme_nondt_f32_large_transpose_lhs_peel
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f32"
    "--acc_type=f32"
    "--shapes=large"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling=false"
    "--iree-llvmcpu-enable-scalable-vectorization"
    "--iree-llvmcpu-target-triple=aarch64-unknown-unknown"
    "--iree-preprocessing-pass-pipeline=builtin.module\(util.func\(iree-preprocessing-transpose-matmul-pass{input=lhs}\)\)"
    "--iree-llvmcpu-vector-pproc-strategy=peel"
  LABELS
    "requires-arm-sme"
  TARGET_CPU_FEATURES_VARIANTS
    "arm_64:sme:+sve,+sme"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_arm_sme_nondt_f32_large_transpose_lhs
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f32"
    "--acc_type=f32"
    "--shapes=large"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling=false"
    "--iree-llvmcpu-enable-scalable-vectorization"
    "--iree-llvmcpu-target-triple=aarch64-unknown-unknown"
    "--iree-preprocessing-pass-pipeline=builtin.module\(util.func\(iree-preprocessing-transpose-matmul-pass{input=lhs}\)\)"
  LABELS
    "requires-arm-sme"
  TARGET_CPU_FEATURES_VARIANTS
    "arm_64:sme:+sve,+sme"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_arm_sme_nondt_f32_large_peel
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f32"
    "--acc_type=f32"
    "--shapes=large"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling=false"
    "--iree-llvmcpu-enable-scalable-vectorization"
    "--iree-llvmcpu-target-triple=aarch64-unknown-unknown"
    "--iree-llvmcpu-vector-pproc-strategy=peel"
  LABELS
    "requires-arm-sme"
  TARGET_CPU_FEATURES_VARIANTS
    "arm_64:sme:+sve,+sme"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_arm_sme_nondt_f32_large
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f32"
    "--acc_type=f32"
    "--shapes=large"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling=false"
    "--iree-llvmcpu-enable-scalable-vectorization"
    "--iree-llvmcpu-target-triple=aarch64-unknown-unknown"
  LABELS
    "requires-arm-sme"
  TARGET_CPU_FEATURES_VARIANTS
    "arm_64:sme:+sve,+sme"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_dt_i8_i32_small
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=i8"
    "--acc_type=i32"
    "--shapes=small"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-llvmcpu-enable-ukernels=none"
  LABELS

  TARGET_CPU_FEATURES_VARIANTS
    "default"
    "arm_64:dotprod:+dotprod"
    "arm_64:i8mm:+i8mm"
    "x86_64:avx512vnni:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq,+avx512vnni"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_dt_i8_i32_large
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=i8"
    "--acc_type=i32"
    "--shapes=large"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-llvmcpu-enable-ukernels=none"
  LABELS
    "noasan"
    "notsan"
  TARGET_CPU_FEATURES_VARIANTS
    "default"
    "arm_64:dotprod:+dotprod"
    "arm_64:i8mm:+i8mm"
    "x86_64:avx512vnni:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq,+avx512vnni"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_dt_f32_f32_small
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f32"
    "--acc_type=f32"
    "--shapes=small"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-llvmcpu-enable-ukernels=none"
  LABELS

  TARGET_CPU_FEATURES_VARIANTS
    "default"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_dt_f32_f32_large
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f32"
    "--acc_type=f32"
    "--shapes=large"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-llvmcpu-enable-ukernels=none"
  LABELS
    "noasan"
    "notsan"
  TARGET_CPU_FEATURES_VARIANTS
    "default"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_dt_f16_f16_small
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f16"
    "--shapes=small"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-llvmcpu-enable-ukernels=none"
  LABELS
    "noriscv"
    "nowasm"
  TARGET_CPU_FEATURES_VARIANTS
    "default"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
    "arm_64:fullfp16:+fullfp16"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_dt_f16_f16_large
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f16"
    "--shapes=large"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-llvmcpu-enable-ukernels=none"
  LABELS
    "noasan"
    "notsan"
    "noriscv"
    "nowasm"
  TARGET_CPU_FEATURES_VARIANTS
    "default"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
    "arm_64:fullfp16:+fullfp16"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_dt_f16_f32_small
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f32"
    "--shapes=small"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-llvmcpu-enable-ukernels=none"
  LABELS
    "noriscv"
    "nowasm"
  TARGET_CPU_FEATURES_VARIANTS
    "default"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
    "arm_64:fp16fml:+fp16fml"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_dt_f16_f32_large
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f32"
    "--shapes=large"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-llvmcpu-enable-ukernels=none"
  LABELS
    "noasan"
    "notsan"
    "noriscv"
    "nowasm"
  TARGET_CPU_FEATURES_VARIANTS
    "default"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
    "arm_64:fp16fml:+fp16fml"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_dt_bf16_bf16_small
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=bf16"
    "--acc_type=bf16"
    "--shapes=small"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-llvmcpu-enable-ukernels=none"
  LABELS
    "noriscv"
    "nowasm"
  TARGET_CPU_FEATURES_VARIANTS
    "default"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
    "x86_64:avx512bf16:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq,+avx512bf16"
    "arm_64:bf16:+bf16"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_dt_bf16_bf16_large
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=bf16"
    "--acc_type=bf16"
    "--shapes=large"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-llvmcpu-enable-ukernels=none"
  LABELS
    "noasan"
    "notsan"
    "noriscv"
    "nowasm"
  TARGET_CPU_FEATURES_VARIANTS
    "default"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
    "x86_64:avx512bf16:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq,+avx512bf16"
    "arm_64:bf16:+bf16"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_dt_bf16_f32_small
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=bf16"
    "--acc_type=f32"
    "--shapes=small"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-llvmcpu-enable-ukernels=none"
  LABELS
    "noriscv"
    "nowasm"
  TARGET_CPU_FEATURES_VARIANTS
    "default"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
    "x86_64:avx512bf16:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq,+avx512bf16"
    "arm_64:bf16:+bf16"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_dt_bf16_f32_large
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=bf16"
    "--acc_type=f32"
    "--shapes=large"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-llvmcpu-enable-ukernels=none"
  LABELS
    "noasan"
    "notsan"
    "noriscv"
    "nowasm"
  TARGET_CPU_FEATURES_VARIANTS
    "default"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
    "x86_64:avx512bf16:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq,+avx512bf16"
    "arm_64:bf16:+bf16"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_dt_uk_i8_i32_small
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=i8"
    "--acc_type=i32"
    "--shapes=small"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-llvmcpu-enable-ukernels=all"
  LABELS

  TARGET_CPU_FEATURES_VARIANTS
    "default"
    "arm_64:dotprod:+dotprod"
    "arm_64:i8mm:+i8mm"
    "x86_64:avx512vnni:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq,+avx512vnni"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_dt_uk_i8_i32_large
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=i8"
    "--acc_type=i32"
    "--shapes=large"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-llvmcpu-enable-ukernels=all"
  LABELS
    "noasan"
    "notsan"
  TARGET_CPU_FEATURES_VARIANTS
    "default"
    "arm_64:dotprod:+dotprod"
    "arm_64:i8mm:+i8mm"
    "x86_64:avx512vnni:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq,+avx512vnni"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_dt_uk_f32_f32_small
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f32"
    "--acc_type=f32"
    "--shapes=small"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-llvmcpu-enable-ukernels=all"
  LABELS

  TARGET_CPU_FEATURES_VARIANTS
    "default"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_dt_uk_f32_f32_large
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f32"
    "--acc_type=f32"
    "--shapes=large"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-llvmcpu-enable-ukernels=all"
  LABELS
    "noasan"
    "notsan"
  TARGET_CPU_FEATURES_VARIANTS
    "default"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_dt_uk_f16_f16_small
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f16"
    "--shapes=small"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-llvmcpu-enable-ukernels=all"
  LABELS
    "noriscv"
    "nowasm"
  TARGET_CPU_FEATURES_VARIANTS
    "default"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
    "arm_64:fullfp16:+fullfp16"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_dt_uk_f16_f16_large
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f16"
    "--shapes=large"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-llvmcpu-enable-ukernels=all"
  LABELS
    "noasan"
    "notsan"
    "noriscv"
    "nowasm"
  TARGET_CPU_FEATURES_VARIANTS
    "default"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
    "arm_64:fullfp16:+fullfp16"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_dt_uk_f16_f32_small
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f32"
    "--shapes=small"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-llvmcpu-enable-ukernels=all"
  LABELS
    "noriscv"
    "nowasm"
  TARGET_CPU_FEATURES_VARIANTS
    "default"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
    "arm_64:fp16fml:+fp16fml"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_dt_uk_f16_f32_large
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f32"
    "--shapes=large"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-llvmcpu-enable-ukernels=all"
  LABELS
    "noasan"
    "notsan"
    "noriscv"
    "nowasm"
  TARGET_CPU_FEATURES_VARIANTS
    "default"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
    "arm_64:fp16fml:+fp16fml"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_dt_uk_bf16_bf16_small
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=bf16"
    "--acc_type=bf16"
    "--shapes=small"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-llvmcpu-enable-ukernels=all"
  LABELS
    "noriscv"
    "nowasm"
  TARGET_CPU_FEATURES_VARIANTS
    "default"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
    "x86_64:avx512bf16:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq,+avx512bf16"
    "arm_64:bf16:+bf16"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_dt_uk_bf16_bf16_large
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=bf16"
    "--acc_type=bf16"
    "--shapes=large"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-llvmcpu-enable-ukernels=all"
  LABELS
    "noasan"
    "notsan"
    "noriscv"
    "nowasm"
  TARGET_CPU_FEATURES_VARIANTS
    "default"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
    "x86_64:avx512bf16:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq,+avx512bf16"
    "arm_64:bf16:+bf16"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_dt_uk_bf16_f32_small
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=bf16"
    "--acc_type=f32"
    "--shapes=small"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-llvmcpu-enable-ukernels=all"
  LABELS
    "noriscv"
    "nowasm"
  TARGET_CPU_FEATURES_VARIANTS
    "default"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
    "x86_64:avx512bf16:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq,+avx512bf16"
    "arm_64:bf16:+bf16"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_dt_uk_bf16_f32_large
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=bf16"
    "--acc_type=f32"
    "--shapes=large"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-llvmcpu-enable-ukernels=all"
  LABELS
    "noasan"
    "notsan"
    "noriscv"
    "nowasm"
  TARGET_CPU_FEATURES_VARIANTS
    "default"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
    "x86_64:avx512bf16:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq,+avx512bf16"
    "arm_64:bf16:+bf16"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_experimental_dt_i8_i32_small
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=i8"
    "--acc_type=i32"
    "--shapes=small"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-global-opt-enable-early-materialization=false"
    "--iree-llvmcpu-enable-ukernels=none"
  LABELS

  TARGET_CPU_FEATURES_VARIANTS
    "default"
    "arm_64:dotprod:+dotprod"
    "arm_64:i8mm:+i8mm"
    "x86_64:avx512vnni:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq,+avx512vnni"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_experimental_dt_i8_i32_large
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=i8"
    "--acc_type=i32"
    "--shapes=large"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-global-opt-enable-early-materialization=false"
    "--iree-llvmcpu-enable-ukernels=none"
  LABELS
    "noasan"
    "notsan"
  TARGET_CPU_FEATURES_VARIANTS
    "default"
    "arm_64:dotprod:+dotprod"
    "arm_64:i8mm:+i8mm"
    "x86_64:avx512vnni:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq,+avx512vnni"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_experimental_dt_f32_f32_small
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f32"
    "--acc_type=f32"
    "--shapes=small"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-global-opt-enable-early-materialization=false"
    "--iree-llvmcpu-enable-ukernels=none"
  LABELS

  TARGET_CPU_FEATURES_VARIANTS
    "default"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_experimental_dt_f32_f32_large
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f32"
    "--acc_type=f32"
    "--shapes=large"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-global-opt-enable-early-materialization=false"
    "--iree-llvmcpu-enable-ukernels=none"
  LABELS
    "noasan"
    "notsan"
  TARGET_CPU_FEATURES_VARIANTS
    "default"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_experimental_dt_f16_f16_small
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f16"
    "--shapes=small"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-global-opt-enable-early-materialization=false"
    "--iree-llvmcpu-enable-ukernels=none"
  LABELS
    "noriscv"
    "nowasm"
  TARGET_CPU_FEATURES_VARIANTS
    "default"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
    "arm_64:fullfp16:+fullfp16"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_experimental_dt_f16_f16_large
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f16"
    "--shapes=large"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-global-opt-enable-early-materialization=false"
    "--iree-llvmcpu-enable-ukernels=none"
  LABELS
    "noasan"
    "notsan"
    "noriscv"
    "nowasm"
  TARGET_CPU_FEATURES_VARIANTS
    "default"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
    "arm_64:fullfp16:+fullfp16"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_experimental_dt_f16_f32_small
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f32"
    "--shapes=small"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-global-opt-enable-early-materialization=false"
    "--iree-llvmcpu-enable-ukernels=none"
  LABELS
    "noriscv"
    "nowasm"
  TARGET_CPU_FEATURES_VARIANTS
    "default"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
    "arm_64:fp16fml:+fp16fml"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_experimental_dt_f16_f32_large
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f32"
    "--shapes=large"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-global-opt-enable-early-materialization=false"
    "--iree-llvmcpu-enable-ukernels=none"
  LABELS
    "noasan"
    "notsan"
    "noriscv"
    "nowasm"
  TARGET_CPU_FEATURES_VARIANTS
    "default"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
    "arm_64:fp16fml:+fp16fml"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_experimental_dt_bf16_bf16_small
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=bf16"
    "--acc_type=bf16"
    "--shapes=small"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-global-opt-enable-early-materialization=false"
    "--iree-llvmcpu-enable-ukernels=none"
  LABELS
    "noriscv"
    "nowasm"
  TARGET_CPU_FEATURES_VARIANTS
    "default"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
    "x86_64:avx512bf16:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq,+avx512bf16"
    "arm_64:bf16:+bf16"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_experimental_dt_bf16_bf16_large
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=bf16"
    "--acc_type=bf16"
    "--shapes=large"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-global-opt-enable-early-materialization=false"
    "--iree-llvmcpu-enable-ukernels=none"
  LABELS
    "noasan"
    "notsan"
    "noriscv"
    "nowasm"
  TARGET_CPU_FEATURES_VARIANTS
    "default"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
    "x86_64:avx512bf16:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq,+avx512bf16"
    "arm_64:bf16:+bf16"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_experimental_dt_bf16_f32_small
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=bf16"
    "--acc_type=f32"
    "--shapes=small"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-global-opt-enable-early-materialization=false"
    "--iree-llvmcpu-enable-ukernels=none"
  LABELS
    "noriscv"
    "nowasm"
  TARGET_CPU_FEATURES_VARIANTS
    "default"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
    "x86_64:avx512bf16:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq,+avx512bf16"
    "arm_64:bf16:+bf16"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_experimental_dt_bf16_f32_large
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=bf16"
    "--acc_type=f32"
    "--shapes=large"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-global-opt-enable-early-materialization=false"
    "--iree-llvmcpu-enable-ukernels=none"
  LABELS
    "noasan"
    "notsan"
    "noriscv"
    "nowasm"
  TARGET_CPU_FEATURES_VARIANTS
    "default"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
    "x86_64:avx512bf16:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq,+avx512bf16"
    "arm_64:bf16:+bf16"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_experimental_dt_uk_i8_i32_small
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=i8"
    "--acc_type=i32"
    "--shapes=small"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-global-opt-enable-early-materialization=false"
    "--iree-llvmcpu-enable-ukernels=all"
  LABELS

  TARGET_CPU_FEATURES_VARIANTS
    "default"
    "arm_64:dotprod:+dotprod"
    "arm_64:i8mm:+i8mm"
    "x86_64:avx512vnni:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq,+avx512vnni"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_experimental_dt_uk_i8_i32_large
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=i8"
    "--acc_type=i32"
    "--shapes=large"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-global-opt-enable-early-materialization=false"
    "--iree-llvmcpu-enable-ukernels=all"
  LABELS
    "noasan"
    "notsan"
  TARGET_CPU_FEATURES_VARIANTS
    "default"
    "arm_64:dotprod:+dotprod"
    "arm_64:i8mm:+i8mm"
    "x86_64:avx512vnni:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq,+avx512vnni"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_experimental_dt_uk_f32_f32_small
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f32"
    "--acc_type=f32"
    "--shapes=small"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-global-opt-enable-early-materialization=false"
    "--iree-llvmcpu-enable-ukernels=all"
  LABELS

  TARGET_CPU_FEATURES_VARIANTS
    "default"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_experimental_dt_uk_f32_f32_large
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f32"
    "--acc_type=f32"
    "--shapes=large"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-global-opt-enable-early-materialization=false"
    "--iree-llvmcpu-enable-ukernels=all"
  LABELS
    "noasan"
    "notsan"
  TARGET_CPU_FEATURES_VARIANTS
    "default"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_experimental_dt_uk_f16_f16_small
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f16"
    "--shapes=small"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-global-opt-enable-early-materialization=false"
    "--iree-llvmcpu-enable-ukernels=all"
  LABELS
    "noriscv"
    "nowasm"
  TARGET_CPU_FEATURES_VARIANTS
    "default"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
    "arm_64:fullfp16:+fullfp16"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_experimental_dt_uk_f16_f16_large
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f16"
    "--shapes=large"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-global-opt-enable-early-materialization=false"
    "--iree-llvmcpu-enable-ukernels=all"
  LABELS
    "noasan"
    "notsan"
    "noriscv"
    "nowasm"
  TARGET_CPU_FEATURES_VARIANTS
    "default"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
    "arm_64:fullfp16:+fullfp16"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_experimental_dt_uk_f16_f32_small
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f32"
    "--shapes=small"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-global-opt-enable-early-materialization=false"
    "--iree-llvmcpu-enable-ukernels=all"
  LABELS
    "noriscv"
    "nowasm"
  TARGET_CPU_FEATURES_VARIANTS
    "default"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
    "arm_64:fp16fml:+fp16fml"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_experimental_dt_uk_f16_f32_large
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f32"
    "--shapes=large"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-global-opt-enable-early-materialization=false"
    "--iree-llvmcpu-enable-ukernels=all"
  LABELS
    "noasan"
    "notsan"
    "noriscv"
    "nowasm"
  TARGET_CPU_FEATURES_VARIANTS
    "default"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
    "arm_64:fp16fml:+fp16fml"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_experimental_dt_uk_bf16_bf16_small
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=bf16"
    "--acc_type=bf16"
    "--shapes=small"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-global-opt-enable-early-materialization=false"
    "--iree-llvmcpu-enable-ukernels=all"
  LABELS
    "noriscv"
    "nowasm"
  TARGET_CPU_FEATURES_VARIANTS
    "default"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
    "x86_64:avx512bf16:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq,+avx512bf16"
    "arm_64:bf16:+bf16"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_experimental_dt_uk_bf16_bf16_large
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=bf16"
    "--acc_type=bf16"
    "--shapes=large"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-global-opt-enable-early-materialization=false"
    "--iree-llvmcpu-enable-ukernels=all"
  LABELS
    "noasan"
    "notsan"
    "noriscv"
    "nowasm"
  TARGET_CPU_FEATURES_VARIANTS
    "default"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
    "x86_64:avx512bf16:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq,+avx512bf16"
    "arm_64:bf16:+bf16"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_experimental_dt_uk_bf16_f32_small
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=bf16"
    "--acc_type=f32"
    "--shapes=small"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-global-opt-enable-early-materialization=false"
    "--iree-llvmcpu-enable-ukernels=all"
  LABELS
    "noriscv"
    "nowasm"
  TARGET_CPU_FEATURES_VARIANTS
    "default"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
    "x86_64:avx512bf16:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq,+avx512bf16"
    "arm_64:bf16:+bf16"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_experimental_dt_uk_bf16_f32_large
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=bf16"
    "--acc_type=f32"
    "--shapes=large"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-global-opt-enable-early-materialization=false"
    "--iree-llvmcpu-enable-ukernels=all"
  LABELS
    "noasan"
    "notsan"
    "noriscv"
    "nowasm"
  TARGET_CPU_FEATURES_VARIANTS
    "default"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
    "x86_64:avx512bf16:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq,+avx512bf16"
    "arm_64:bf16:+bf16"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_vmvx_dt_uk_i8_small
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=i8"
    "--shapes=small"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "vmvx"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-vmvx-enable-microkernels"
    "--iree-opt-data-tiling"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_vmvx_dt_uk_f32_small
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f32"
    "--shapes=small"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "vmvx"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-vmvx-enable-microkernels"
    "--iree-opt-data-tiling"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cuda_f32_large_simt
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f32"
    "--shapes=gpu_large_aligned"
    "--compilation_info=LLVMGPUMatmulSimt"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "cuda"
  DRIVERS
    "cuda"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-nvidia"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cuda_f32_large_tensorcore
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f32"
    "--shapes=gpu_large_aligned"
    "--compilation_info=LLVMGPUMatmulTensorCore"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "cuda"
  DRIVERS
    "cuda"
  COMPILER_FLAGS
    "--iree-hal-cuda-llvm-target-arch=sm_80"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-sm80"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cuda_f32_large_unaligned
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f32"
    "--shapes=gpu_large"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "cuda"
  DRIVERS
    "cuda"
  COMPILER_FLAGS
    "--iree-hal-cuda-llvm-target-arch=sm_80"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-sm80"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cuda_f16_large_unaligned
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--shapes=gpu_large"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "cuda"
  DRIVERS
    "cuda"
  COMPILER_FLAGS
    "--iree-hal-cuda-llvm-target-arch=sm_80"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-sm80"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cuda_f32_large_mma_sync
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f32"
    "--shapes=gpu_large_aligned"
    "--compilation_info=LLVMGPUMatmulTensorCoreMmaSync"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "cuda"
  DRIVERS
    "cuda"
  COMPILER_FLAGS
    "--iree-hal-cuda-llvm-target-arch=sm_80"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-sm80"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cuda_f16_large_tensorcore
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--shapes=gpu_large_aligned"
    "--compilation_info=LLVMGPUMatmulTensorCore"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "cuda"
  DRIVERS
    "cuda"
  COMPILER_FLAGS
    "--iree-hal-cuda-llvm-target-arch=sm_80"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-sm80"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cuda_f16_large_mma_sync
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--shapes=gpu_large_aligned"
    "--compilation_info=LLVMGPUMatmulTensorCoreMmaSync"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "cuda"
  DRIVERS
    "cuda"
  COMPILER_FLAGS
    "--iree-hal-cuda-llvm-target-arch=sm_80"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-sm80"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cuda_f32_large_splitk
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f32"
    "--shapes=large"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "cuda"
  DRIVERS
    "cuda"
  COMPILER_FLAGS
    "--iree-flow-split-matmul-reduction=4"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-nvidia"
    "noriscv"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_vulkan_i8_large_valhall
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=i8"
    "--shapes=gpu_large_aligned"
    "--compilation_info=SPIRVVectorizeMali"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "vulkan-spirv"
  DRIVERS
    "vulkan"
  COMPILER_FLAGS
    "--iree-vulkan-target-triple=valhall-unknown-android31"
  LABELS
    "requires-gpu-nvidia"
    "vulkan_uses_vk_khr_shader_float16_int8"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_vulkan_f16_large_valhall
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--shapes=gpu_large_aligned"
    "--compilation_info=SPIRVVectorizeMali"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "vulkan-spirv"
  DRIVERS
    "vulkan"
  COMPILER_FLAGS
    "--iree-vulkan-target-triple=valhall-unknown-android31"
  LABELS
    "requires-gpu-nvidia"
    "vulkan_uses_vk_khr_shader_float16_int8"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_vulkan_f32_large_valhall
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f32"
    "--shapes=gpu_large_aligned"
    "--compilation_info=SPIRVVectorizeMali"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "vulkan-spirv"
  DRIVERS
    "vulkan"
  COMPILER_FLAGS
    "--iree-vulkan-target-triple=valhall-unknown-android31"
  LABELS
    "requires-gpu-nvidia"
    "vulkan_uses_vk_khr_shader_float16_int8"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_vulkan_i8_large_ampere
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=i8"
    "--shapes=gpu_large_aligned"
    "--compilation_info=SPIRVVectorizeNVIDIA"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "vulkan-spirv"
  DRIVERS
    "vulkan"
  COMPILER_FLAGS
    "--iree-vulkan-target-triple=ampere-unknown-linux"
  LABELS
    "requires-gpu-sm80"
    "vulkan_uses_vk_khr_shader_float16_int8"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_vulkan_f16_large_ampere
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--shapes=gpu_large_aligned"
    "--compilation_info=SPIRVVectorizeNVIDIA"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "vulkan-spirv"
  DRIVERS
    "vulkan"
  COMPILER_FLAGS
    "--iree-vulkan-target-triple=ampere-unknown-linux"
  LABELS
    "requires-gpu-sm80"
    "vulkan_uses_vk_khr_shader_float16_int8"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_vulkan_f32_large_ampere
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f32"
    "--shapes=gpu_large_aligned"
    "--compilation_info=SPIRVVectorizeNVIDIA"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "vulkan-spirv"
  DRIVERS
    "vulkan"
  COMPILER_FLAGS
    "--iree-vulkan-target-triple=ampere-unknown-linux"
  LABELS
    "requires-gpu-sm80"
    "vulkan_uses_vk_khr_shader_float16_int8"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_vulkan_f16_large_rdna3
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--shapes=gpu_large_aligned"
    "--compilation_info=SPIRVCooperativeMatrixVectorize"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "vulkan-spirv"
  DRIVERS
    "vulkan"
  COMPILER_FLAGS
    "--iree-vulkan-target-triple=rdna3-unknown-linux"
  RUNNER_ARGS
    "--require_exact_results=false"
  LABELS
    "requires-gpu"
    "requires-gpu-rdna3"
    "vulkan_uses_vk_khr_shader_float16_int8"
)

### BAZEL_TO_CMAKE_PRESERVES_ALL_CONTENT_BELOW_THIS_LINE ###

# To distinguish between CDNA(gfx9) and RDNA3(gfx11)
if(IREE_HIP_TEST_TARGET_CHIP MATCHES "^gfx9")

unset(IREE_HIP_TEST_COMPILER_FLAGS)
list(APPEND IREE_HIP_TEST_COMPILER_FLAGS
  "--iree-rocm-target-chip=${IREE_HIP_TEST_TARGET_CHIP}"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_rocm_f16_large_cdna3_mfma
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f32"
    "--shapes=gpu_large_aligned"
    "--compilation_info=LLVMGPUVectorDistributeMFMA"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-cdna3"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_rocm_f16_large_cdna3_mfma_tb
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f32"
    "--transpose_rhs"
    "--shapes=gpu_large_aligned"
    "--compilation_info=LLVMGPUVectorDistributeMFMA"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-cdna3"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_rocm_i8_large_cdna3_mfma_tb
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=i8"
    "--acc_type=i32"
    "--transpose_rhs"
    "--shapes=gpu_large_aligned"
    "--compilation_info=LLVMGPUVectorDistributeMFMA"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-cdna3"
)

elseif(IREE_HIP_TEST_TARGET_CHIP MATCHES "^gfx11")

unset(IREE_HIP_TEST_COMPILER_FLAGS)
list(APPEND IREE_HIP_TEST_COMPILER_FLAGS
  "--iree-rocm-target-chip=${IREE_HIP_TEST_TARGET_CHIP}"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_rocm_f16_large_rdna3_wmma
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f32"
    "--shapes=gpu_large_aligned"
    "--compilation_info=LLVMGPUVectorDistributeWMMA"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
  RUNNER_ARGS
    "--require_exact_results=false"
    "--acceptable_fp_delta=1e-04"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-rdna3"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_rocm_f16_large_rdna3_wmma_tb
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f32"
    "--transpose_rhs"
    "--shapes=gpu_large_aligned"
    "--compilation_info=LLVMGPUVectorDistributeWMMA"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
  RUNNER_ARGS
    "--require_exact_results=false"
    "--acceptable_fp_delta=1e-04"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-rdna3"
)
endif()