tests/e2e/matmul/CMakeLists.txt

################################################################################
# Autogenerated by build_tools/bazel_to_cmake/bazel_to_cmake.py from           #
# tests/e2e/matmul/BUILD.bazel                                                 #
#                                                                              #
# Use iree_cmake_extra_content from iree/build_defs.oss.bzl to add arbitrary   #
# CMake-only content.                                                          #
#                                                                              #
# To disable autogeneration for this file entirely, delete this header.        #
################################################################################

iree_add_all_subdirs()

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_arm_sme_nondt_f32__transpose_lhs_peel
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f32"
    "--acc_type=f32"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling=false"
    "--iree-llvmcpu-enable-scalable-vectorization"
    "--iree-llvmcpu-target-triple=aarch64-unknown-unknown"
    "--iree-preprocessing-pass-pipeline=builtin.module\(util.func\(iree-preprocessing-transpose-matmul-pass{input=lhs}\)\)"
    "--iree-llvmcpu-vector-pproc-strategy=peel"
  LABELS
    "requires-arm-sme"
  TARGET_CPU_FEATURES_VARIANTS
    "arm_64:sme:+sme"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_arm_sme_nondt_f32__transpose_lhs
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f32"
    "--acc_type=f32"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling=false"
    "--iree-llvmcpu-enable-scalable-vectorization"
    "--iree-llvmcpu-target-triple=aarch64-unknown-unknown"
    "--iree-preprocessing-pass-pipeline=builtin.module\(util.func\(iree-preprocessing-transpose-matmul-pass{input=lhs}\)\)"
  LABELS
    "requires-arm-sme"
  TARGET_CPU_FEATURES_VARIANTS
    "arm_64:sme:+sme"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_arm_sme_nondt_f32__peel
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f32"
    "--acc_type=f32"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling=false"
    "--iree-llvmcpu-enable-scalable-vectorization"
    "--iree-llvmcpu-target-triple=aarch64-unknown-unknown"
    "--iree-llvmcpu-vector-pproc-strategy=peel"
  LABELS
    "requires-arm-sme"
  TARGET_CPU_FEATURES_VARIANTS
    "arm_64:sme:+sme"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_arm_sme_nondt_f32_
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f32"
    "--acc_type=f32"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling=false"
    "--iree-llvmcpu-enable-scalable-vectorization"
    "--iree-llvmcpu-target-triple=aarch64-unknown-unknown"
  LABELS
    "requires-arm-sme"
  TARGET_CPU_FEATURES_VARIANTS
    "arm_64:sme:+sme"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_dt_i8_i32
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=i8"
    "--acc_type=i32"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-llvmcpu-enable-ukernels=none"
  LABELS
    "noriscv"
  TARGET_CPU_FEATURES_VARIANTS
    "generic"
    "arm_64:dotprod:+dotprod"
    "arm_64:i8mm:+i8mm"
    "x86_64:avx512vnni:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq,+avx512vnni"
    "riscv_64:ime:+v,+zvl256b,+xsmtvdot"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_dt_f32_f32
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f32"
    "--acc_type=f32"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-llvmcpu-enable-ukernels=none"
  LABELS
    "noriscv"
  TARGET_CPU_FEATURES_VARIANTS
    "generic"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_dt_f64_f64
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f64"
    "--acc_type=f64"
    "--shapes=small"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-llvmcpu-enable-ukernels=none"
    "--iree-input-demote-f64-to-f32=false"
  LABELS

  TARGET_CPU_FEATURES_VARIANTS
    "generic"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_dt_f16_f16
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f16"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-llvmcpu-enable-ukernels=none"
  LABELS
    "noriscv"
    "nowasm"
  TARGET_CPU_FEATURES_VARIANTS
    "generic"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
    "arm_64:fullfp16:+fullfp16"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_dt_f16_f32
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f32"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-llvmcpu-enable-ukernels=none"
  LABELS
    "noriscv"
    "nowasm"
  TARGET_CPU_FEATURES_VARIANTS
    "generic"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
    "arm_64:fp16fml:+fp16fml"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_dt_bf16_bf16
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=bf16"
    "--acc_type=bf16"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-llvmcpu-enable-ukernels=none"
  LABELS
    "noriscv"
    "nowasm"
  TARGET_CPU_FEATURES_VARIANTS
    "generic"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
    "x86_64:avx512bf16:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq,+avx512bf16"
    "arm_64:bf16:+bf16"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_dt_bf16_f32
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=bf16"
    "--acc_type=f32"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-llvmcpu-enable-ukernels=none"
  LABELS
    "noriscv"
    "nowasm"
  TARGET_CPU_FEATURES_VARIANTS
    "generic"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
    "x86_64:avx512bf16:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq,+avx512bf16"
    "arm_64:bf16:+bf16"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_dt_uk_i8_i32
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=i8"
    "--acc_type=i32"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-llvmcpu-enable-ukernels=all"
  LABELS

  TARGET_CPU_FEATURES_VARIANTS
    "generic"
    "arm_64:dotprod:+dotprod"
    "arm_64:i8mm:+i8mm"
    "x86_64:avx512vnni:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq,+avx512vnni"
    "riscv_64:ime:+v,+zvl256b,+xsmtvdot"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_dt_uk_f32_f32
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f32"
    "--acc_type=f32"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-llvmcpu-enable-ukernels=all"
  LABELS

  TARGET_CPU_FEATURES_VARIANTS
    "generic"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_dt_uk_f64_f64
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f64"
    "--acc_type=f64"
    "--shapes=small"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-llvmcpu-enable-ukernels=all"
    "--iree-input-demote-f64-to-f32=false"
  LABELS

  TARGET_CPU_FEATURES_VARIANTS
    "generic"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_dt_uk_f16_f16
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f16"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-llvmcpu-enable-ukernels=all"
  LABELS
    "noriscv"
    "nowasm"
  TARGET_CPU_FEATURES_VARIANTS
    "generic"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
    "arm_64:fullfp16:+fullfp16"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_dt_uk_f16_f32
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f32"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-llvmcpu-enable-ukernels=all"
  LABELS
    "noriscv"
    "nowasm"
  TARGET_CPU_FEATURES_VARIANTS
    "generic"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
    "arm_64:fp16fml:+fp16fml"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_dt_uk_bf16_bf16
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=bf16"
    "--acc_type=bf16"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-llvmcpu-enable-ukernels=all"
  LABELS
    "noriscv"
    "nowasm"
  TARGET_CPU_FEATURES_VARIANTS
    "generic"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
    "x86_64:avx512bf16:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq,+avx512bf16"
    "arm_64:bf16:+bf16"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_dt_uk_bf16_f32
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=bf16"
    "--acc_type=f32"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-llvmcpu-enable-ukernels=all"
  LABELS
    "noriscv"
    "nowasm"
  TARGET_CPU_FEATURES_VARIANTS
    "generic"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
    "x86_64:avx512bf16:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq,+avx512bf16"
    "arm_64:bf16:+bf16"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_dt_inner_tiled_i8_i32
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=i8"
    "--acc_type=i32"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-llvmcpu-enable-inner-tiled"
  LABELS

  TARGET_CPU_FEATURES_VARIANTS
    "generic"
    "x86_64:avx512vnni:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq,+avx512vnni"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_dt_inner_tiled_f32_f32
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f32"
    "--acc_type=f32"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-llvmcpu-enable-inner-tiled"
  LABELS

  TIMEOUT
    300
  TARGET_CPU_FEATURES_VARIANTS
    "generic"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_dt_inner_tiled_f64_f64
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f64"
    "--acc_type=f64"
    "--shapes=small"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-llvmcpu-enable-inner-tiled"
    "--iree-input-demote-f64-to-f32=false"
  LABELS

  TARGET_CPU_FEATURES_VARIANTS
    "generic"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_dt_inner_tiled_f16_f16
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f16"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-llvmcpu-enable-inner-tiled"
  LABELS
    "noriscv"
    "nowasm"
  TARGET_CPU_FEATURES_VARIANTS
    "generic"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_dt_inner_tiled_f16_f32
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f32"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-llvmcpu-enable-inner-tiled"
  LABELS
    "noriscv"
    "nowasm"
  TARGET_CPU_FEATURES_VARIANTS
    "generic"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_dt_inner_tiled_bf16_bf16
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=bf16"
    "--acc_type=bf16"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-llvmcpu-enable-inner-tiled"
  LABELS
    "noriscv"
    "nowasm"
  TARGET_CPU_FEATURES_VARIANTS
    "generic"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
    "x86_64:avx512bf16:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq,+avx512bf16"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_dt_inner_tiled_bf16_f32
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=bf16"
    "--acc_type=f32"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-llvmcpu-enable-inner-tiled"
  LABELS
    "noriscv"
    "nowasm"
  TARGET_CPU_FEATURES_VARIANTS
    "generic"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
    "x86_64:avx512bf16:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq,+avx512bf16"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_dt_inner_tiled_llvm_ukernel_bf16_f32
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=bf16"
    "--acc_type=f32"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-llvmcpu-enable-inner-tiled"
    "--iree-llvmcpu-enable-llvm-ukernels=inner_tiled"
  LABELS
    "noriscv"
    "nowasm"
  TARGET_CPU_FEATURES_VARIANTS
    "x86_64:avx512bf16:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq,+avx512bf16"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_dt_inner_tiled_llvm_ukernel_i8_i32
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=i8"
    "--acc_type=i32"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-llvmcpu-enable-inner-tiled"
    "--iree-llvmcpu-enable-llvm-ukernels=inner_tiled"
  TARGET_CPU_FEATURES_VARIANTS
    "x86_64:avx512vnni:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq,+avx512vnni"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_dt_inner_tiled_ui8_i8_i32
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs=ui8"
    "--rhs=i8"
    "--acc_type=i32"
    "--shapes=easy_large_static"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-llvmcpu-enable-inner-tiled"
  TARGET_CPU_FEATURES_VARIANTS
    "generic"
    "x86_64:avx512vnni:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq,+avx512vnni"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_experimental_dt_i8_i32
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=i8"
    "--acc_type=i32"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-global-opt-enable-early-materialization=false"
    "--iree-llvmcpu-enable-ukernels=none"
  LABELS
    "noriscv"
  TARGET_CPU_FEATURES_VARIANTS
    "generic"
    "arm_64:dotprod:+dotprod"
    "arm_64:i8mm:+i8mm"
    "x86_64:avx512vnni:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq,+avx512vnni"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_experimental_dt_f32_f32
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f32"
    "--acc_type=f32"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-global-opt-enable-early-materialization=false"
    "--iree-llvmcpu-enable-ukernels=none"
  LABELS
    "noriscv"
  TARGET_CPU_FEATURES_VARIANTS
    "generic"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_experimental_dt_f16_f16
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f16"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-global-opt-enable-early-materialization=false"
    "--iree-llvmcpu-enable-ukernels=none"
  LABELS
    "noriscv"
    "nowasm"
  TARGET_CPU_FEATURES_VARIANTS
    "generic"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
    "arm_64:fullfp16:+fullfp16"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_experimental_dt_f16_f32
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f32"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-global-opt-enable-early-materialization=false"
    "--iree-llvmcpu-enable-ukernels=none"
  LABELS
    "noriscv"
    "nowasm"
  TARGET_CPU_FEATURES_VARIANTS
    "generic"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
    "arm_64:fp16fml:+fp16fml"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_experimental_dt_bf16_bf16
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=bf16"
    "--acc_type=bf16"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-global-opt-enable-early-materialization=false"
    "--iree-llvmcpu-enable-ukernels=none"
  LABELS
    "noriscv"
    "nowasm"
  TARGET_CPU_FEATURES_VARIANTS
    "generic"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
    "x86_64:avx512bf16:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq,+avx512bf16"
    "arm_64:bf16:+bf16"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_experimental_dt_bf16_f32
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=bf16"
    "--acc_type=f32"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-global-opt-enable-early-materialization=false"
    "--iree-llvmcpu-enable-ukernels=none"
  LABELS
    "noriscv"
    "nowasm"
  TARGET_CPU_FEATURES_VARIANTS
    "generic"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
    "x86_64:avx512bf16:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq,+avx512bf16"
    "arm_64:bf16:+bf16"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_experimental_dt_uk_i8_i32
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=i8"
    "--acc_type=i32"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-global-opt-enable-early-materialization=false"
    "--iree-llvmcpu-enable-ukernels=all"
  LABELS

  TARGET_CPU_FEATURES_VARIANTS
    "generic"
    "arm_64:dotprod:+dotprod"
    "arm_64:i8mm:+i8mm"
    "x86_64:avx512vnni:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq,+avx512vnni"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_experimental_dt_uk_f32_f32
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f32"
    "--acc_type=f32"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-global-opt-enable-early-materialization=false"
    "--iree-llvmcpu-enable-ukernels=all"
  LABELS

  TARGET_CPU_FEATURES_VARIANTS
    "generic"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_experimental_dt_uk_f16_f16
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f16"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-global-opt-enable-early-materialization=false"
    "--iree-llvmcpu-enable-ukernels=all"
  LABELS
    "noriscv"
    "nowasm"
  TARGET_CPU_FEATURES_VARIANTS
    "generic"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
    "arm_64:fullfp16:+fullfp16"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_experimental_dt_uk_f16_f32
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f32"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-global-opt-enable-early-materialization=false"
    "--iree-llvmcpu-enable-ukernels=all"
  LABELS
    "noriscv"
    "nowasm"
  TARGET_CPU_FEATURES_VARIANTS
    "generic"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
    "arm_64:fp16fml:+fp16fml"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_experimental_dt_uk_bf16_bf16
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=bf16"
    "--acc_type=bf16"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-global-opt-enable-early-materialization=false"
    "--iree-llvmcpu-enable-ukernels=all"
  LABELS
    "noriscv"
    "nowasm"
  TARGET_CPU_FEATURES_VARIANTS
    "generic"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
    "x86_64:avx512bf16:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq,+avx512bf16"
    "arm_64:bf16:+bf16"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_experimental_dt_uk_bf16_f32
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=bf16"
    "--acc_type=f32"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-global-opt-enable-early-materialization=false"
    "--iree-llvmcpu-enable-ukernels=all"
  LABELS
    "noriscv"
    "nowasm"
  TARGET_CPU_FEATURES_VARIANTS
    "generic"
    "x86_64:avx2:+avx,+avx2,+fma,+f16c"
    "x86_64:avx512:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq"
    "x86_64:avx512bf16:+avx,+avx2,+fma,+f16c,+avx512f,+avx512vl,+avx512cd,+avx512bw,+avx512dq,+avx512bf16"
    "arm_64:bf16:+bf16"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_dt_riscv_small_i8_i32
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=i8"
    "--acc_type=i32"
    "--shapes=small"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-llvmcpu-enable-ukernels=none"
  TARGET_CPU_FEATURES_VARIANTS
    "riscv_64:rvv:+v,+zvl512b"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_dt_riscv_small_f32_f32
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f32"
    "--acc_type=f32"
    "--shapes=small"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-llvmcpu-enable-ukernels=none"
  TARGET_CPU_FEATURES_VARIANTS
    "riscv_64:rvv:+v,+zvl512b"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_experimental_dt_riscv_small_i8_i32
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=i8"
    "--acc_type=i32"
    "--shapes=small"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-llvmcpu-enable-ukernels=none"
    "--iree-global-opt-enable-early-materialization=false"
  TARGET_CPU_FEATURES_VARIANTS
    "riscv_64:rvv:+v,+zvl512b"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_experimental_dt_riscv_small_f32_f32
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f32"
    "--acc_type=f32"
    "--shapes=small"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-llvmcpu-enable-ukernels=none"
    "--iree-global-opt-enable-early-materialization=false"
  TARGET_CPU_FEATURES_VARIANTS
    "riscv_64:rvv:+v,+zvl512b"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_vmvx_experimental_dt_f32_f32
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f32"
    "--acc_type=f32"
    "--shapes=small"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "vmvx"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-global-opt-enable-early-materialization=false"
  LABELS

)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_vmvx_experimental_dt_uk_f32_f32
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f32"
    "--acc_type=f32"
    "--shapes=small"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "vmvx"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-global-opt-enable-early-materialization=false"
  LABELS

)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_spirv_experimental_dt_f32_f32
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f32"
    "--acc_type=f32"
    "--shapes=small"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "vulkan-spirv"
  DRIVERS
    "vulkan"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
    "--iree-global-opt-enable-early-materialization=false"
  LABELS

)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cpu_dt_mxfp4_f32
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f4E2M1FN"
    "--acc_type=f32"
    "--mx_scale_type=f8E8M0FNU"
    "--mx_block_size=32"
    "--shapes=easy_large_static"
    "--transpose_rhs"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "llvm-cpu"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-opt-data-tiling"
  TARGET_CPU_FEATURES_VARIANTS
    "generic"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_vmvx_dt_uk_i8_small
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=i8"
    "--acc_type=i32"
    "--shapes=small"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "vmvx"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-vmvx-enable-microkernels"
    "--iree-opt-data-tiling"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_vmvx_dt_uk_f32_small
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f32"
    "--acc_type=f32"
    "--shapes=small"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "vmvx"
  DRIVERS
    "local-task"
  COMPILER_FLAGS
    "--iree-vmvx-enable-microkernels"
    "--iree-opt-data-tiling"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_vulkan_i8_large_valhall
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=i8"
    "--acc_type=i32"
    "--shapes=easy_large_static"
    "--compilation_info=SPIRVVectorizeMali"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "vulkan-spirv"
  DRIVERS
    "vulkan"
  COMPILER_FLAGS
    "--iree-vulkan-target=valhall"
  LABELS
    "requires-gpu-nvidia"
    "vulkan_uses_vk_khr_shader_float16_int8"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_vulkan_f16_large_valhall
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f32"
    "--shapes=easy_large_static"
    "--compilation_info=SPIRVVectorizeMali"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "vulkan-spirv"
  DRIVERS
    "vulkan"
  COMPILER_FLAGS
    "--iree-vulkan-target=valhall"
  LABELS
    "requires-gpu-nvidia"
    "vulkan_uses_vk_khr_shader_float16_int8"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_vulkan_f32_large_valhall
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f32"
    "--acc_type=f32"
    "--shapes=easy_large_static"
    "--compilation_info=SPIRVVectorizeMali"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "vulkan-spirv"
  DRIVERS
    "vulkan"
  COMPILER_FLAGS
    "--iree-vulkan-target=valhall"
  LABELS
    "requires-gpu-nvidia"
    "vulkan_uses_vk_khr_shader_float16_int8"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_vulkan_i8_large_ampere
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=i8"
    "--acc_type=i32"
    "--shapes=easy_large_static"
    "--compilation_info=SPIRVVectorizeNVIDIA"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "vulkan-spirv"
  DRIVERS
    "vulkan"
  COMPILER_FLAGS
    "--iree-vulkan-target=ampere"
  LABELS
    "requires-gpu-sm80"
    "vulkan_uses_vk_khr_shader_float16_int8"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_vulkan_f16_large_ampere
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f32"
    "--shapes=easy_large_static"
    "--compilation_info=SPIRVVectorizeNVIDIA"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "vulkan-spirv"
  DRIVERS
    "vulkan"
  COMPILER_FLAGS
    "--iree-vulkan-target=ampere"
  LABELS
    "requires-gpu-sm80"
    "vulkan_uses_vk_khr_shader_float16_int8"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_vulkan_f32_large_ampere
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f32"
    "--acc_type=f32"
    "--shapes=easy_large_static"
    "--compilation_info=SPIRVVectorizeNVIDIA"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "vulkan-spirv"
  DRIVERS
    "vulkan"
  COMPILER_FLAGS
    "--iree-vulkan-target=ampere"
  LABELS
    "requires-gpu-sm80"
    "vulkan_uses_vk_khr_shader_float16_int8"
)

### BAZEL_TO_CMAKE_PRESERVES_ALL_CONTENT_BELOW_THIS_LINE ###

###########################################################################
##
## CUDA backend
##
###########################################################################

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cuda_vecdist_f16
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f32"
    "--shapes=easy_large_static"
    "--compilation_info=LLVMGPUVectorDistributeCUDA"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "cuda"
  DRIVERS
    "cuda"
  COMPILER_FLAGS
    "--iree-cuda-target=sm_80"
  LABELS
    "requires-gpu-sm80"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cuda_tileandfuse_f16
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f32"
    "--shapes=easy_large_static"
    "--compilation_info=LLVMGPUTileAndFuseCUDA"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "cuda"
  DRIVERS
    "cuda"
  COMPILER_FLAGS
    "--iree-cuda-target=sm_80"
  LABELS
    "requires-gpu-sm80"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cuda_vecdist_f16_f16
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f16"
    "--shapes=easy_large_static"
    "--compilation_info=LLVMGPUVectorDistributeCUDA"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "cuda"
  DRIVERS
    "cuda"
  COMPILER_FLAGS
    "--iree-cuda-target=sm_80"
  LABELS
    "requires-gpu-sm80"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cuda_tileandfuse_f16_f16
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f16"
    "--shapes=easy_large_static"
    "--compilation_info=LLVMGPUTileAndFuseCUDA"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "cuda"
  DRIVERS
    "cuda"
  COMPILER_FLAGS
    "--iree-cuda-target=sm_80"
  LABELS
    "requires-gpu-sm80"
)

###########################################################################
##
## HIP backend
##
###########################################################################

# To distinguish between CDNA(gfx9), RDNA3(gfx11), and RDNA4(gfx12)
if(IREE_ROCM_TEST_TARGET_CHIP MATCHES "^gfx9")

unset(IREE_HIP_TEST_COMPILER_FLAGS)
list(APPEND IREE_HIP_TEST_COMPILER_FLAGS
  "--iree-rocm-target=${IREE_ROCM_TEST_TARGET_CHIP}"
)

if(IREE_ROCM_TEST_TARGET_CHIP MATCHES "^gfx94")
  set(_CDNA_ARCH "cdna3")
  set(_F8E4M3_TYPE "f8E4M3FNUZ")
  set(_F8E5M2_TYPE "f8E5M2FNUZ")
elseif(IREE_ROCM_TEST_TARGET_CHIP MATCHES "^gfx95")
  set(_CDNA_ARCH "cdna4")
  set(_F8E4M3_TYPE "f8E4M3FN")
  set(_F8E5M2_TYPE "f8E5M2")
endif()

if(IREE_ROCM_TEST_TARGET_CHIP MATCHES "^gfx94" OR IREE_ROCM_TEST_TARGET_CHIP MATCHES "^gfx95")

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_${_CDNA_ARCH}_coalesced_dma_f32
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f32"
    "--acc_type=f32"
    "--shapes=custom_mnk"
    "--mnk=32,32,64"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
    "--iree-llvmgpu-use-direct-load"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-${_CDNA_ARCH}"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_${_CDNA_ARCH}_coalesced_dma_f32_larger
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f32"
    "--acc_type=f32"
    "--shapes=custom_mnk"
    "--mnk=32,64,64"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
    "--iree-llvmgpu-use-direct-load"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-${_CDNA_ARCH}"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_${_CDNA_ARCH}_coalesced_dma_f32_multi_transfer
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f32"
    "--acc_type=f32"
    "--shapes=custom_mnk"
    "--mnk=32,128,128"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
    "--iree-llvmgpu-use-direct-load"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-${_CDNA_ARCH}"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_${_CDNA_ARCH}_coalesced_dma_f32_large
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f32"
    "--acc_type=f32"
    "--shapes=custom_mnk"
    "--mnk=128,256,512"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
    "--iree-llvmgpu-use-direct-load"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-${_CDNA_ARCH}"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_${_CDNA_ARCH}_coalesced_dma_f32_1k
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f32"
    "--acc_type=f32"
    "--shapes=custom_mnk"
    "--mnk=1024,1024,1024"
    "--mnk_dynamicities=static,static,static"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
    "--iree-llvmgpu-use-direct-load"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-${_CDNA_ARCH}"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_${_CDNA_ARCH}_coalesced_dma_f32_2k
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f32"
    "--acc_type=f32"
    "--shapes=custom_mnk"
    "--mnk=2048,2048,2048"
    "--mnk_dynamicities=static,static,static"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
    "--iree-llvmgpu-use-direct-load"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-${_CDNA_ARCH}"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_${_CDNA_ARCH}_coalesced_dma_f32_4k
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f32"
    "--acc_type=f32"
    "--shapes=custom_mnk"
    "--mnk=4096,4096,4096"
    "--mnk_dynamicities=static,static,static"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
    "--iree-llvmgpu-use-direct-load"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-${_CDNA_ARCH}"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_${_CDNA_ARCH}_coalesced_dma_f32_8k
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f32"
    "--acc_type=f32"
    "--shapes=custom_mnk"
    "--mnk=8192,8192,8192"
    "--mnk_dynamicities=static,static,static"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
    "--iree-llvmgpu-use-direct-load"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-${_CDNA_ARCH}"
)

# Unaligned matmul tests for coalesced DMA with tensor.pad fusion
iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_${_CDNA_ARCH}_coalesced_dma_f32_unaligned_65x64x121
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f32"
    "--acc_type=f32"
    "--shapes=custom_mnk"
    "--mnk=65,64,121"
    "--mnk_dynamicities=static,static,static"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
    "--iree-llvmgpu-use-direct-load"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-${_CDNA_ARCH}"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_${_CDNA_ARCH}_coalesced_dma_f32_unaligned_133x97x65
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f32"
    "--acc_type=f32"
    "--shapes=custom_mnk"
    "--mnk=133,97,65"
    "--mnk_dynamicities=static,static,static"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
    "--iree-llvmgpu-use-direct-load"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-${_CDNA_ARCH}"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_${_CDNA_ARCH}_coalesced_dma_f32_unaligned_123x321x231
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f32"
    "--acc_type=f32"
    "--shapes=custom_mnk"
    "--mnk=123,321,231"
    "--mnk_dynamicities=static,static,static"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
    "--iree-llvmgpu-use-direct-load"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-${_CDNA_ARCH}"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_batch_matmul_${_CDNA_ARCH}_coalesced_dma_bf16
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_batch_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=bf16"
    "--acc_type=f32"
    "--shapes=block_static"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
    "--iree-llvmgpu-use-direct-load"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-${_CDNA_ARCH}"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_${_CDNA_ARCH}_vecdistmfma_f16
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f32"
    "--shapes=easy_large_static"
    "--compilation_info=LLVMGPUVectorDistributeMFMA"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-${_CDNA_ARCH}"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_${_CDNA_ARCH}_tileandfusemfma_f16
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f32"
    "--shapes=easy_large_static"
    "--compilation_info=LLVMGPUTileAndFuseMFMA"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-${_CDNA_ARCH}"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_${_CDNA_ARCH}_vecdistmfma_f32
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f32"
    "--acc_type=f32"
    "--shapes=easy_large_static"
    "--compilation_info=LLVMGPUVectorDistributeMFMA"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-${_CDNA_ARCH}"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_${_CDNA_ARCH}_tileandfusemfma_f32
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f32"
    "--acc_type=f32"
    "--shapes=easy_large_static"
    "--compilation_info=LLVMGPUTileAndFuseMFMA"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-${_CDNA_ARCH}"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_${_CDNA_ARCH}_vecdistmfma_tb_f16
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f32"
    "--transpose_rhs"
    "--shapes=easy_large_static"
    "--compilation_info=LLVMGPUVectorDistributeMFMA"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-${_CDNA_ARCH}"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_${_CDNA_ARCH}_tileandfusemfma_tb_f16
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f32"
    "--transpose_rhs"
    "--shapes=easy_large_static"
    "--compilation_info=LLVMGPUTileAndFuseMFMA"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-${_CDNA_ARCH}"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_${_CDNA_ARCH}_vecdistmfma_${_F8E4M3_TYPE}
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=${_F8E4M3_TYPE}"
    "--acc_type=f32"
    "--shapes=easy_large_static"
    "--compilation_info=LLVMGPUVectorDistributeMFMA"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-${_CDNA_ARCH}"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_${_CDNA_ARCH}_tileandfusemfma_${_F8E4M3_TYPE}
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=${_F8E4M3_TYPE}"
    "--acc_type=f32"
    "--shapes=easy_large_static"
    "--compilation_info=LLVMGPUTileAndFuseMFMA"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-${_CDNA_ARCH}"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_${_CDNA_ARCH}_vecdistmfma_i8
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=i8"
    "--acc_type=i32"
    "--transpose_rhs"
    "--shapes=easy_large_static"
    "--compilation_info=LLVMGPUVectorDistributeMFMA"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-${_CDNA_ARCH}"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_${_CDNA_ARCH}_tileandfusemfma_i8
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=i8"
    "--acc_type=i32"
    "--transpose_rhs"
    "--shapes=easy_large_static"
    "--compilation_info=LLVMGPUTileAndFuseMFMA"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-${_CDNA_ARCH}"
)

# Batch matmul tests for CDNA block intrinsics (F16, BF16, I8)
foreach(_BATCH_LHS_TYPE f16 bf16 i8)
  if(_BATCH_LHS_TYPE STREQUAL "i8")
    set(_BATCH_ACC_TYPE i32)
  else()
    set(_BATCH_ACC_TYPE f32)
  endif()

  iree_generated_e2e_runner_test(
    NAME
      e2e_batch_matmul_${_CDNA_ARCH}_vecdistmfma_block_${_BATCH_LHS_TYPE}
    TEST_TYPE
      matmul
    GENERATOR
      "generate_e2e_batch_matmul_tests.py"
    GENERATOR_ARGS
      "--lhs_rhs_type=${_BATCH_LHS_TYPE}"
      "--acc_type=${_BATCH_ACC_TYPE}"
      "--compilation_info=LLVMGPUVectorDistributeMFMABlockBatch"
      "--shapes=block_static"
    TEST_RUNNER
      iree_tools_testing_e2e_iree-e2e-matmul-test
    TARGET_BACKENDS
      "rocm"
    DRIVERS
      "hip"
    COMPILER_FLAGS
      ${IREE_HIP_TEST_COMPILER_FLAGS}
    LABELS
      "noasan"
      "nomsan"
      "notsan"
      "noubsan"
      "requires-gpu-${_CDNA_ARCH}"
  )

  iree_generated_e2e_runner_test(
    NAME
      e2e_batch_matmul_${_CDNA_ARCH}_tileandfusemfma_block_${_BATCH_LHS_TYPE}
    TEST_TYPE
      matmul
    GENERATOR
      "generate_e2e_batch_matmul_tests.py"
    GENERATOR_ARGS
      "--lhs_rhs_type=${_BATCH_LHS_TYPE}"
      "--acc_type=${_BATCH_ACC_TYPE}"
      "--compilation_info=LLVMGPUTileAndFuseMFMABlockBatch"
      "--shapes=block_static"
    TEST_RUNNER
      iree_tools_testing_e2e_iree-e2e-matmul-test
    TARGET_BACKENDS
      "rocm"
    DRIVERS
      "hip"
    COMPILER_FLAGS
      ${IREE_HIP_TEST_COMPILER_FLAGS}
    LABELS
      "noasan"
      "nomsan"
      "notsan"
      "noubsan"
      "requires-gpu-${_CDNA_ARCH}"
  )
endforeach()

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_${_CDNA_ARCH}_dt_f16
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f32"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
    "--iree-dispatch-creation-data-tiling"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-${_CDNA_ARCH}"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_${_CDNA_ARCH}_dt_bf16
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=bf16"
    "--acc_type=f32"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
    "--iree-dispatch-creation-data-tiling"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-${_CDNA_ARCH}"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_${_CDNA_ARCH}_dt_i8
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=i8"
    "--acc_type=i32"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
    "--iree-dispatch-creation-data-tiling"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-${_CDNA_ARCH}"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_${_CDNA_ARCH}_dt_uk_i8
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=i8"
    "--acc_type=i32"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
    "--iree-rocm-enable-ukernels=multi_mma"
    "--iree-dispatch-creation-data-tiling"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-${_CDNA_ARCH}"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_${_CDNA_ARCH}_dt_f32
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f32"
    "--acc_type=f32"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
    "--iree-dispatch-creation-data-tiling"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-${_CDNA_ARCH}"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_${_CDNA_ARCH}_dt_${_F8E5M2_TYPE}
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=${_F8E5M2_TYPE}"
    "--acc_type=f32"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
    "--iree-dispatch-creation-data-tiling"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-${_CDNA_ARCH}"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_${_CDNA_ARCH}_tensor_ukernel_${_F8E4M3_TYPE}_medium_expanded
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=${_F8E4M3_TYPE}"
    "--acc_type=f32"
    "--shapes=custom_mnk"
    "--mnk=1152,1024,1024"
    "--mnk_dynamicities=dynamic,static,static"
    "--transpose_rhs"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
    "--iree-rocm-enable-tensor-ukernels"
    "--iree-rocm-specialize-dispatches"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-${_CDNA_ARCH}"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_${_CDNA_ARCH}_tensor_ukernel_${_F8E4M3_TYPE}_large_expanded
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=${_F8E4M3_TYPE}"
    "--acc_type=f32"
    "--shapes=custom_mnk"
    "--mnk=2048,2048,2048"
    "--mnk_dynamicities=dynamic,static,static"
    "--transpose_rhs"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
    "--iree-rocm-enable-tensor-ukernels"
    "--iree-rocm-specialize-dispatches"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-${_CDNA_ARCH}"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_${_CDNA_ARCH}_dt_${_F8E4M3_TYPE}
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=${_F8E4M3_TYPE}"
    "--acc_type=f32"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
    "--iree-dispatch-creation-data-tiling"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-${_CDNA_ARCH}"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_${_CDNA_ARCH}_dt_${_F8E4M3_TYPE}_tensor_ukernel_medium
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=${_F8E4M3_TYPE}"
    "--acc_type=f32"
    "--shapes=custom_mnk"
    "--mnk=1024,1024,1024"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
    "--iree-dispatch-creation-data-tiling"
    "--iree-rocm-enable-tensor-ukernels"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-${_CDNA_ARCH}"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_${_CDNA_ARCH}_dt_${_F8E4M3_TYPE}_tensor_ukernel_large
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=${_F8E4M3_TYPE}"
    "--acc_type=f32"
    "--shapes=custom_mnk"
    "--mnk=2048,8192,2048"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
    "--iree-dispatch-creation-data-tiling"
    "--iree-rocm-enable-tensor-ukernels"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-${_CDNA_ARCH}"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_${_CDNA_ARCH}_dt_f64
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f64"
    "--acc_type=f64"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
    "--iree-dispatch-creation-data-tiling"
    "--iree-input-demote-f64-to-f32=false"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-${_CDNA_ARCH}"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_${_CDNA_ARCH}_pad_f32
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f32"
    "--acc_type=f32"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
    "--iree-dispatch-creation-data-tiling"
    "--iree-dispatch-creation-set-encoding-strategy=padding"
    "--iree-rocm-encoding-layout-resolver=pad"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-${_CDNA_ARCH}"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_${_CDNA_ARCH}_pad_i8
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=i8"
    "--acc_type=i32"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
    "--iree-dispatch-creation-data-tiling"
    "--iree-dispatch-creation-set-encoding-strategy=padding"
    "--iree-rocm-encoding-layout-resolver=pad"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-${_CDNA_ARCH}"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_${_CDNA_ARCH}_tensor_ukernel_f16f16f32_medium_expanded
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f32"
    "--shapes=custom_mnk"
    "--mnk=1152,1024,1024"
    "--mnk_dynamicities=dynamic,static,static"
    "--transpose_rhs"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
    "--iree-rocm-enable-tensor-ukernels"
    "--iree-rocm-specialize-dispatches"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-${_CDNA_ARCH}"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_${_CDNA_ARCH}_tensor_ukernel_f16f16f32_large
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f32"
    "--shapes=custom_mnk"
    "--mnk=1024,1024,1024"
    "--mnk_dynamicities=static,static,static"
    "--transpose_rhs"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
    "--iree-rocm-enable-tensor-ukernels"
    "--iree-rocm-specialize-dispatches"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-${_CDNA_ARCH}"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_${_CDNA_ARCH}_tensor_ukernel_f16f16f32_large_expanded
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f32"
    "--shapes=custom_mnk"
    "--mnk=2048,2048,4096"
    "--mnk_dynamicities=dynamic,static,static"
    "--transpose_rhs"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
    "--iree-rocm-enable-tensor-ukernels"
    "--iree-rocm-specialize-dispatches"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-${_CDNA_ARCH}"
)


iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_${_CDNA_ARCH}_dt_tensor_ukernel_f16f16f32_large
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f32"
    "--shapes=custom_mnk"
    "--mnk=1024,32832,1024"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
    "--iree-dispatch-creation-data-tiling"
    "--iree-rocm-enable-tensor-ukernels"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-${_CDNA_ARCH}"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_${_CDNA_ARCH}_tensor_ukernel_bf16bf16f32_medium_expanded
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=bf16"
    "--acc_type=f32"
    "--shapes=custom_mnk"
    "--mnk=1152,1024,1024"
    "--mnk_dynamicities=dynamic,static,static"
    "--transpose_rhs"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
    "--iree-rocm-enable-tensor-ukernels"
    "--iree-rocm-specialize-dispatches"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-${_CDNA_ARCH}"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_${_CDNA_ARCH}_tensor_ukernel_bf16bf16f32_large
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=bf16"
    "--acc_type=f32"
    "--shapes=custom_mnk"
    "--mnk=2048,2048,2048"
    "--mnk_dynamicities=static,static,static"
    "--transpose_rhs"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
    "--iree-rocm-enable-tensor-ukernels"
    "--iree-rocm-specialize-dispatches"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-${_CDNA_ARCH}"
)


iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_${_CDNA_ARCH}_tensor_ukernel_bf16bf16f32_large_expanded
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=bf16"
    "--acc_type=f32"
    "--shapes=custom_mnk"
    "--mnk=2048,2048,2048"
    "--mnk_dynamicities=dynamic,static,static"
    "--transpose_rhs"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
    "--iree-rocm-enable-tensor-ukernels"
    "--iree-rocm-specialize-dispatches"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-${_CDNA_ARCH}"
)

# VDMFMA (virtual dense MFMA) e2e tests for skinny matmuls.

if(IREE_ROCM_TEST_TARGET_CHIP MATCHES "^gfx94")

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cdna3_tileandfusevdmfma_f16
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f32"
    "--shapes=custom_mnk"
    "--mnk=8,128,512"
    "--mnk_dynamicities=static,static,static"
    "--compilation_info=LLVMGPUTileAndFuseVDMFMA_CDNA3"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-cdna3"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cdna3_tileandfusevdmfma_bf16
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=bf16"
    "--acc_type=f32"
    "--shapes=custom_mnk"
    "--mnk=8,128,512"
    "--mnk_dynamicities=static,static,static"
    "--compilation_info=LLVMGPUTileAndFuseVDMFMA_CDNA3"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-cdna3"
)

endif()

endif()

if(IREE_ROCM_TEST_TARGET_CHIP MATCHES "^gfx95")

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cdna4_mxfp4
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f4E2M1FN"
    "--acc_type=f32"
    "--mx_scale_type=f8E8M0FNU"
    "--mx_block_size=32"
    "--shapes=easy_large_static"
    "--transpose_rhs"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-cdna4"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cdna4_mxfp4_dt
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f4E2M1FN"
    "--acc_type=f32"
    "--mx_scale_type=f8E8M0FNU"
    "--mx_block_size=32"
    "--shapes=easy_large_static"
    "--transpose_rhs"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
    --iree-dispatch-creation-data-tiling=true
    --iree-rocm-encoding-layout-resolver=data-tiling
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-cdna4"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cdna4_mxfp4_dt_tensor_ukernel_medium
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f4E2M1FN"
    "--acc_type=f32"
    "--mx_scale_type=f8E8M0FNU"
    "--mx_block_size=32"
    "--shapes=custom_mnk"
    "--mnk=1024,1024,1024"
    "--mnk_dynamicities=static,static,static"
    "--transpose_rhs"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
    --iree-dispatch-creation-data-tiling=true
    --iree-rocm-encoding-layout-resolver=data-tiling
    --iree-rocm-enable-tensor-ukernels
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-cdna4"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cdna4_mxfp4_llama_0
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f4E2M1FN"
    "--acc_type=f32"
    "--mx_scale_type=f8E8M0FNU"
    "--mx_block_size=32"
    "--shapes=custom_mnk"
    "--mnk=10000,1024,16384"
    "--mnk_dynamicities=static,static,static"
    "--transpose_rhs"
    "--no-accumulate"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-cdna4"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cdna4_mxfp4_llama_1
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f4E2M1FN"
    "--acc_type=f32"
    "--mx_scale_type=f8E8M0FNU"
    "--mx_block_size=32"
    "--shapes=custom_mnk"
    "--mnk=10000,16384,16384"
    "--mnk_dynamicities=static,static,static"
    "--transpose_rhs"
    "--no-accumulate"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-cdna4"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cdna4_mxfp4_llama_2
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f4E2M1FN"
    "--acc_type=f32"
    "--mx_scale_type=f8E8M0FNU"
    "--mx_block_size=32"
    "--shapes=custom_mnk"
    "--mnk=10000,53248,16384"
    "--mnk_dynamicities=static,static,static"
    "--transpose_rhs"
    "--no-accumulate"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-cdna4"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_cdna4_mxfp4_llama_3
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f4E2M1FN"
    "--acc_type=f32"
    "--mx_scale_type=f8E8M0FNU"
    "--mx_block_size=32"
    "--shapes=custom_mnk"
    "--mnk=10000,53248,53248"
    "--mnk_dynamicities=static,static,static"
    "--transpose_rhs"
    "--no-accumulate"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-cdna4"
)

endif()


elseif(IREE_ROCM_TEST_TARGET_CHIP MATCHES "^gfx11")

unset(IREE_HIP_TEST_COMPILER_FLAGS)
list(APPEND IREE_HIP_TEST_COMPILER_FLAGS
  "--iree-rocm-target=${IREE_ROCM_TEST_TARGET_CHIP}"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_rocm_f16_large_rdna3_wmma
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f32"
    "--compilation_info=LLVMGPUVectorDistributeWMMAR3"
    "--shapes=easy_large_static"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
  RUNNER_ARGS
    "--require_exact_results=false"
    "--acceptable_fp_delta=1e-04"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-rdna3"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_rocm_f16_large_rdna3_tileandfusewmma
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f32"
    "--compilation_info=LLVMGPUTileAndFuseWMMAR3"
    "--shapes=easy_large_static"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
  RUNNER_ARGS
    "--require_exact_results=false"
    "--acceptable_fp_delta=1e-04"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-rdna3"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_rocm_f16_large_rdna3_wmma_tb
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f32"
    "--transpose_rhs"
    "--compilation_info=LLVMGPUVectorDistributeWMMAR3"
    "--shapes=easy_large_static"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
  RUNNER_ARGS
    "--require_exact_results=false"
    "--acceptable_fp_delta=1e-04"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-rdna3"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_rocm_f16_large_rdna3_tileandfusewmma_tb
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f32"
    "--transpose_rhs"
    "--compilation_info=LLVMGPUTileAndFuseWMMAR3"
    "--shapes=easy_large_static"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
  RUNNER_ARGS
    "--require_exact_results=false"
    "--acceptable_fp_delta=1e-04"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-rdna3"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_rocm_i8_large_rdna3_wmma_tb
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=i8"
    "--acc_type=i32"
    "--transpose_rhs"
    "--compilation_info=LLVMGPUVectorDistributeWMMAR3"
    "--shapes=easy_large_static"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-rdna3"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_rocm_i8_large_rdna3_tileandfusewmma_tb
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=i8"
    "--acc_type=i32"
    "--transpose_rhs"
    "--compilation_info=LLVMGPUTileAndFuseWMMAR3"
    "--shapes=easy_large_static"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-rdna3"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_rdna3_dt_f16
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f32"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
    "--iree-dispatch-creation-data-tiling"
  RUNNER_ARGS
    "--require_exact_results=false"
    "--acceptable_fp_delta=1e-04"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-rdna3"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_rdna3_dt_i8
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=i8"
    "--acc_type=i32"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
    "--iree-dispatch-creation-data-tiling"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-rdna3"
)

elseif(IREE_ROCM_TEST_TARGET_CHIP MATCHES "^gfx120[01]")

unset(IREE_HIP_TEST_COMPILER_FLAGS)
list(APPEND IREE_HIP_TEST_COMPILER_FLAGS
  "--iree-rocm-target=${IREE_ROCM_TEST_TARGET_CHIP}"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_rocm_f16_large_rdna4_wmma
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f32"
    "--compilation_info=LLVMGPUVectorDistributeWMMAR4"
    "--shapes=easy_large_static"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
  RUNNER_ARGS
    "--require_exact_results=false"
    "--acceptable_fp_delta=1e-04"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-rdna4"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_rocm_f16_large_rdna4_tileandfusewmma
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f32"
    "--compilation_info=LLVMGPUTileAndFuseWMMAR4"
    "--shapes=easy_large_static"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
  RUNNER_ARGS
    "--require_exact_results=false"
    "--acceptable_fp_delta=1e-04"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-rdna4"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_rocm_f16_large_rdna4_wmma_tb
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f32"
    "--transpose_rhs"
    "--compilation_info=LLVMGPUVectorDistributeWMMAR4"
    "--shapes=easy_large_static"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
  RUNNER_ARGS
    "--require_exact_results=false"
    "--acceptable_fp_delta=1e-04"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-rdna4"
  TIMEOUT
    120
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_rocm_f16_large_rdna4_tileandfusewmma_tb
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f32"
    "--transpose_rhs"
    "--compilation_info=LLVMGPUTileAndFuseWMMAR4"
    "--shapes=easy_large_static"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
  RUNNER_ARGS
    "--require_exact_results=false"
    "--acceptable_fp_delta=1e-04"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-rdna4"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_rocm_f8E4M3FN_large_rdna4_wmma
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f8E4M3FN"
    "--acc_type=f32"
    "--compilation_info=LLVMGPUVectorDistributeWMMAR4"
    "--shapes=easy_large_static"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
  RUNNER_ARGS
    "--require_exact_results=false"
    "--acceptable_fp_delta=1e-04"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-rdna4"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_rocm_f8E4M3FN_large_rdna4_tileandfusewmma
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f8E4M3FN"
    "--acc_type=f32"
    "--compilation_info=LLVMGPUTileAndFuseWMMAR4"
    "--shapes=easy_large_static"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
  RUNNER_ARGS
    "--require_exact_results=false"
    "--acceptable_fp_delta=1e-04"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-rdna4"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_rocm_f8e4M3FN_large_rdna4_wmma_tb
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f8E4M3FN"
    "--acc_type=f32"
    "--transpose_rhs"
    "--compilation_info=LLVMGPUVectorDistributeWMMAR4"
    "--shapes=easy_large_static"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
  RUNNER_ARGS
    "--require_exact_results=false"
    "--acceptable_fp_delta=1e-04"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-rdna4"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_rocm_f8e4M3FN_large_rdna4_tileandfusewmma_tb
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f8E4M3FN"
    "--acc_type=f32"
    "--transpose_rhs"
    "--compilation_info=LLVMGPUTileAndFuseWMMAR4"
    "--shapes=easy_large_static"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
  RUNNER_ARGS
    "--require_exact_results=false"
    "--acceptable_fp_delta=1e-04"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-rdna4"
  TIMEOUT
    120
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_rocm_i8_large_rdna4_wmma_tb
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=i8"
    "--acc_type=i32"
    "--transpose_rhs"
    "--compilation_info=LLVMGPUVectorDistributeWMMAR4"
    "--shapes=easy_large_static"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-rdna4"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_rocm_i8_large_rdna4_tileandfusewmma_tb
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=i8"
    "--acc_type=i32"
    "--transpose_rhs"
    "--compilation_info=LLVMGPUTileAndFuseWMMAR4"
    "--shapes=easy_large_static"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-rdna4"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_rdna4_dt_f16
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f32"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
    "--iree-dispatch-creation-data-tiling"
  RUNNER_ARGS
    "--require_exact_results=false"
    "--acceptable_fp_delta=1e-04"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-rdna4"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_rdna4_dt_f8E4M3FN
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f8E4M3FN"
    "--acc_type=f32"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
    "--iree-dispatch-creation-data-tiling"
  RUNNER_ARGS
    "--require_exact_results=false"
    "--acceptable_fp_delta=1e-04"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-rdna4"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_rdna4_dt_i8
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=i8"
    "--acc_type=i32"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
    "--iree-dispatch-creation-data-tiling"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-rdna4"
)

elseif(IREE_ROCM_TEST_TARGET_CHIP MATCHES "^gfx1250")

unset(IREE_HIP_TEST_COMPILER_FLAGS)
list(APPEND IREE_HIP_TEST_COMPILER_FLAGS
  "--iree-rocm-target=${IREE_ROCM_TEST_TARGET_CHIP}"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_gfx1250_wmma_f16
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f32"
    "--compilation_info=LLVMGPUVectorDistributeWMMA1250"
    "--shapes=easy_large_static"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
  RUNNER_ARGS
    "--require_exact_results=false"
    "--acceptable_fp_delta=1e-04"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-gfx1250"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_gfx1250_tileandfusewmma_f16
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f32"
    "--compilation_info=LLVMGPUTileAndFuseWMMA1250"
    "--shapes=easy_large_static"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
  RUNNER_ARGS
    "--require_exact_results=false"
    "--acceptable_fp_delta=1e-04"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-gfx1250"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_gfx1250_wmma_f16_tb
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f32"
    "--transpose_rhs"
    "--compilation_info=LLVMGPUVectorDistributeWMMA1250"
    "--shapes=easy_large_static"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
  RUNNER_ARGS
    "--require_exact_results=false"
    "--acceptable_fp_delta=1e-04"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-gfx1250"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_gfx1250_tileandfusewmma_f16_tb
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f32"
    "--transpose_rhs"
    "--compilation_info=LLVMGPUTileAndFuseWMMA1250"
    "--shapes=easy_large_static"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
  RUNNER_ARGS
    "--require_exact_results=false"
    "--acceptable_fp_delta=1e-04"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-gfx1250"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_gfx1250_wmma_f8E4M3FN
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f8E4M3FN"
    "--acc_type=f32"
    "--compilation_info=LLVMGPUVectorDistributeWMMA1250"
    "--shapes=easy_large_static"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
  RUNNER_ARGS
    "--require_exact_results=false"
    "--acceptable_fp_delta=1e-04"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-gfx1250"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_gfx1250_tileandfusewmma_f8E4M3FN
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f8E4M3FN"
    "--acc_type=f32"
    "--compilation_info=LLVMGPUTileAndFuseWMMA1250"
    "--shapes=easy_large_static"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
  RUNNER_ARGS
    "--require_exact_results=false"
    "--acceptable_fp_delta=1e-04"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-gfx1250"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_gfx1250_wmma_f8E4M3FN_tb
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f8E4M3FN"
    "--acc_type=f32"
    "--transpose_rhs"
    "--compilation_info=LLVMGPUVectorDistributeWMMA1250"
    "--shapes=easy_large_static"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
  RUNNER_ARGS
    "--require_exact_results=false"
    "--acceptable_fp_delta=1e-04"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-gfx1250"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_gfx1250_tileandfusewmma_f8E4M3FN_tb
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f8E4M3FN"
    "--acc_type=f32"
    "--transpose_rhs"
    "--compilation_info=LLVMGPUTileAndFuseWMMA1250"
    "--shapes=easy_large_static"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
  RUNNER_ARGS
    "--require_exact_results=false"
    "--acceptable_fp_delta=1e-04"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-gfx1250"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_gfx1250_wmma_i8_tb
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=i8"
    "--acc_type=i32"
    "--transpose_rhs"
    "--compilation_info=LLVMGPUVectorDistributeWMMA1250"
    "--shapes=easy_large_static"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-gfx1250"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_gfx1250_tileandfusewmma_i8_tb
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=i8"
    "--acc_type=i32"
    "--transpose_rhs"
    "--compilation_info=LLVMGPUTileAndFuseWMMA1250"
    "--shapes=easy_large_static"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-gfx1250"
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_gfx1250_dt_f16
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f16"
    "--acc_type=f32"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
    "--iree-dispatch-creation-data-tiling"
  RUNNER_ARGS
    "--require_exact_results=false"
    "--acceptable_fp_delta=1e-04"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-gfx1250"
  # TODO(kuhar): Drop the timeout after we switch to testing on the actual gfx1250 chip.
  TIMEOUT
    120
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_gfx1250_dt_f8E4M3FN
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=f8E4M3FN"
    "--acc_type=f32"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
    "--iree-dispatch-creation-data-tiling"
  RUNNER_ARGS
    "--require_exact_results=false"
    "--acceptable_fp_delta=1e-04"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-gfx1250"
  # TODO(kuhar): Drop the timeout after we switch to testing on the actual gfx1250 chip.
  TIMEOUT
    120
)

iree_generated_e2e_runner_test(
  NAME
    e2e_matmul_gfx1250_dt_i8
  TEST_TYPE
    matmul
  GENERATOR
    "generate_e2e_matmul_tests.py"
  GENERATOR_ARGS
    "--lhs_rhs_type=i8"
    "--acc_type=i32"
  TEST_RUNNER
    iree_tools_testing_e2e_iree-e2e-matmul-test
  TARGET_BACKENDS
    "rocm"
  DRIVERS
    "hip"
  COMPILER_FLAGS
    ${IREE_HIP_TEST_COMPILER_FLAGS}
    "--iree-dispatch-creation-data-tiling"
  LABELS
    "noasan"
    "nomsan"
    "notsan"
    "noubsan"
    "requires-gpu-gfx1250"
  # TODO(kuhar): Drop the timeout after we switch to testing on the actual gfx1250 chip.
  TIMEOUT
    120
)

endif()

# These complete quickly on hardware but take at least 20 seconds per test (or
# time out) when every GPU instruction is simulated. The RocJITsu CPU-only
# suite excludes this label; hardware suites continue to run them.
if(IREE_HAL_DRIVER_HIP AND IREE_ROCM_TEST_TARGET_CHIP MATCHES "^gfx94")
  foreach(_TEST IN ITEMS
    "iree/tests/e2e/matmul/e2e_matmul_cdna3_coalesced_dma_f32_2k_rocm_hip"
    "iree/tests/e2e/matmul/e2e_matmul_cdna3_coalesced_dma_f32_4k_rocm_hip"
    "iree/tests/e2e/matmul/e2e_matmul_cdna3_coalesced_dma_f32_8k_rocm_hip"
    "iree/tests/e2e/matmul/e2e_matmul_cdna3_coalesced_dma_f32_large_rocm_hip"
    "iree/tests/e2e/matmul/e2e_batch_matmul_cdna3_vecdistmfma_block_f16_rocm_hip"
    "iree/tests/e2e/matmul/e2e_batch_matmul_cdna3_tileandfusemfma_block_f16_rocm_hip"
    "iree/tests/e2e/matmul/e2e_batch_matmul_cdna3_vecdistmfma_block_bf16_rocm_hip"
    "iree/tests/e2e/matmul/e2e_batch_matmul_cdna3_tileandfusemfma_block_bf16_rocm_hip"
    "iree/tests/e2e/matmul/e2e_batch_matmul_cdna3_vecdistmfma_block_i8_rocm_hip"
    "iree/tests/e2e/matmul/e2e_batch_matmul_cdna3_tileandfusemfma_block_i8_rocm_hip"
    "iree/tests/e2e/matmul/e2e_matmul_cdna3_dt_f8E4M3FNUZ_tensor_ukernel_large_rocm_hip"
    "iree/tests/e2e/matmul/e2e_matmul_cdna3_tensor_ukernel_f8E4M3FNUZ_large_expanded_rocm_hip"
    "iree/tests/e2e/matmul/e2e_matmul_cdna3_dt_f32_rocm_hip"
    "iree/tests/e2e/matmul/e2e_matmul_cdna3_dt_f64_rocm_hip"
    "iree/tests/e2e/matmul/e2e_matmul_cdna3_pad_f32_rocm_hip"
    "iree/tests/e2e/matmul/e2e_matmul_cdna3_pad_i8_rocm_hip"
    "iree/tests/e2e/matmul/e2e_matmul_cdna3_tensor_ukernel_f16f16f32_large_expanded_rocm_hip"
    "iree/tests/e2e/matmul/e2e_matmul_cdna3_dt_tensor_ukernel_f16f16f32_large_rocm_hip"
    "iree/tests/e2e/matmul/e2e_matmul_cdna3_tensor_ukernel_bf16bf16f32_large_rocm_hip"
    "iree/tests/e2e/matmul/e2e_matmul_cdna3_tensor_ukernel_bf16bf16f32_large_expanded_rocm_hip"
  )
    if(TEST "${_TEST}")
      set_property(TEST "${_TEST}" APPEND PROPERTY LABELS "very-expensive")
    endif()
  endforeach()
elseif(IREE_HAL_DRIVER_HIP AND IREE_ROCM_TEST_TARGET_CHIP MATCHES "^gfx95")
  foreach(_TEST IN ITEMS
    "iree/tests/e2e/matmul/e2e_matmul_cdna4_coalesced_dma_f32_2k_rocm_hip"
    "iree/tests/e2e/matmul/e2e_matmul_cdna4_coalesced_dma_f32_4k_rocm_hip"
    "iree/tests/e2e/matmul/e2e_matmul_cdna4_coalesced_dma_f32_8k_rocm_hip"
    "iree/tests/e2e/matmul/e2e_matmul_cdna4_coalesced_dma_f32_large_rocm_hip"
    "iree/tests/e2e/matmul/e2e_batch_matmul_cdna4_vecdistmfma_block_f16_rocm_hip"
    "iree/tests/e2e/matmul/e2e_batch_matmul_cdna4_vecdistmfma_block_bf16_rocm_hip"
    "iree/tests/e2e/matmul/e2e_batch_matmul_cdna4_vecdistmfma_block_i8_rocm_hip"
    "iree/tests/e2e/matmul/e2e_batch_matmul_cdna4_tileandfusemfma_block_i8_rocm_hip"
    "iree/tests/e2e/matmul/e2e_matmul_cdna4_dt_f8E4M3FN_tensor_ukernel_large_rocm_hip"
    "iree/tests/e2e/matmul/e2e_matmul_cdna4_tensor_ukernel_f8E4M3FN_large_expanded_rocm_hip"
    "iree/tests/e2e/matmul/e2e_matmul_cdna4_dt_f64_rocm_hip"
    "iree/tests/e2e/matmul/e2e_matmul_cdna4_pad_f32_rocm_hip"
    "iree/tests/e2e/matmul/e2e_matmul_cdna4_pad_i8_rocm_hip"
    "iree/tests/e2e/matmul/e2e_matmul_cdna4_tensor_ukernel_f16f16f32_large_expanded_rocm_hip"
    "iree/tests/e2e/matmul/e2e_matmul_cdna4_dt_tensor_ukernel_f16f16f32_large_rocm_hip"
    "iree/tests/e2e/matmul/e2e_matmul_cdna4_tensor_ukernel_bf16bf16f32_large_rocm_hip"
    "iree/tests/e2e/matmul/e2e_matmul_cdna4_tensor_ukernel_bf16bf16f32_large_expanded_rocm_hip"
    "iree/tests/e2e/matmul/e2e_matmul_cdna4_mxfp4_llama_0_rocm_hip"
    "iree/tests/e2e/matmul/e2e_matmul_cdna4_mxfp4_llama_1_rocm_hip"
    "iree/tests/e2e/matmul/e2e_matmul_cdna4_mxfp4_llama_2_rocm_hip"
    "iree/tests/e2e/matmul/e2e_matmul_cdna4_mxfp4_llama_3_rocm_hip"
  )
    if(TEST "${_TEST}")
      set_property(TEST "${_TEST}" APPEND PROPERTY LABELS "very-expensive")
    endif()
  endforeach()
elseif(IREE_HAL_DRIVER_HIP AND IREE_ROCM_TEST_TARGET_CHIP MATCHES "^gfx1250")
  foreach(_TEST IN ITEMS
    "iree/tests/e2e/matmul/e2e_matmul_gfx1250_wmma_f16_rocm_hip"
    "iree/tests/e2e/matmul/e2e_matmul_gfx1250_tileandfusewmma_f16_rocm_hip"
    "iree/tests/e2e/matmul/e2e_matmul_gfx1250_wmma_f16_tb_rocm_hip"
    "iree/tests/e2e/matmul/e2e_matmul_gfx1250_tileandfusewmma_f16_tb_rocm_hip"
    "iree/tests/e2e/matmul/e2e_matmul_gfx1250_wmma_f8E4M3FN_rocm_hip"
    "iree/tests/e2e/matmul/e2e_matmul_gfx1250_tileandfusewmma_f8E4M3FN_rocm_hip"
    "iree/tests/e2e/matmul/e2e_matmul_gfx1250_wmma_f8E4M3FN_tb_rocm_hip"
    "iree/tests/e2e/matmul/e2e_matmul_gfx1250_tileandfusewmma_f8E4M3FN_tb_rocm_hip"
    "iree/tests/e2e/matmul/e2e_matmul_gfx1250_wmma_i8_tb_rocm_hip"
    "iree/tests/e2e/matmul/e2e_matmul_gfx1250_tileandfusewmma_i8_tb_rocm_hip"
    "iree/tests/e2e/matmul/e2e_matmul_gfx1250_dt_f16_rocm_hip"
    "iree/tests/e2e/matmul/e2e_matmul_gfx1250_dt_f8E4M3FN_rocm_hip"
    "iree/tests/e2e/matmul/e2e_matmul_gfx1250_dt_i8_rocm_hip"
  )
    if(TEST "${_TEST}")
      set_property(TEST "${_TEST}" APPEND PROPERTY LABELS "very-expensive")
    endif()
  endforeach()
endif()