[CPU] Vectorize parallel dims for non-AVX512 x86 matmul defaults (#24701)

See GitHub issue: https://github.com/iree-org/iree/issues/24700

`getDefaultMatmulVectorSizes` used `{1, 1, vectorSize}` for non-AVX512
x86, which vectorizes only the reduction (`K`) dim and leaves the
parallel/output dims scalar. This forces a horizontal reduction per
output element and is dramatically slower for f32 matmul_like /
batch_matmul kernels compared to v3.5 (~2.4x end-to-end regression on an
affected model; ~92x on an isolated `512x512x512` matmul at `x86-64-v3`,
`-O1`). Mirror the AArch64/RISC-V defaults by register-blocking the
matmul with `{8, vectorSize, vectorSize}`, vectorizing both parallel
dims (`M` unroll, `N` vector) in addition to `K`.

Numerics are unchanged. As a side effect, dynamic-shape accumulating
GEMMs may now hoist a couple of tiny fixed-size remainder scratch
buffers (`memref.alloca`) from register-blocked peeling; the accumulator
stays in registers and is written directly to the destination (no heap
alloc, no `linalg.generic` copy). The `pipeline_tests.mlir`
accumulating-GEMM guard is relaxed to forbid only heap allocation, and
the `select_x86_64` expected configs are updated `([1, 1, 0] -> [8, 4,
0])`.

---------

Signed-off-by: Paul Stark <paul.stark@cdprojektred.com>
Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>

compiler/src/iree/compiler/Codegen/LLVMCPU/KernelDispatch.cpp

diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/KernelDispatch.cpp b/compiler/src/iree/compiler/Codegen/LLVMCPU/KernelDispatch.cpp
index c075923..600ade9 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMCPU/KernelDispatch.cpp
+++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/KernelDispatch.cpp
@@ -1263,7 +1263,10 @@
     if (hasAVX512fFeature(targetAttr.getConfiguration())) {
       sizes.append({8, 32, 16});
     } else {
-      sizes.append({1, 1, vectorSize});
+      // For non-AVX512 x86, register-block the matmul by vectorizing
+      // both parallel dims (M unroll, N vector) in addition to the reduction
+      // dim.
+      sizes.append({8, vectorSize, vectorSize});
     }
     return;
   }

compiler/src/iree/compiler/Codegen/LLVMCPU/test/pipeline_tests.mlir

diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/test/pipeline_tests.mlir b/compiler/src/iree/compiler/Codegen/LLVMCPU/test/pipeline_tests.mlir
index edf6260..d0acc85 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMCPU/test/pipeline_tests.mlir
+++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/test/pipeline_tests.mlir
@@ -616,8 +616,8 @@
 
 // -----
 
-// Verify that stack buffer is not created in accumulating GEMMs dispatches;
-// it direct writes the result into the destination buffer.
+// Verify that accumulating GEMM dispatches write the result directly into the
+// destination buffer.
 
 #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu_features = "", native_vector_size = 16 : index, target_triple = "x86_64-none-elf"}>
 #pipeline_layout = #hal.pipeline.layout<constants = 3, bindings = [
@@ -648,7 +648,13 @@
   return
 }
 // CHECK-LABEL: func.func @matmul_accumulate_from_readonly(
-// CHECK-NOT:     memref.alloc
+// Register-blocked peeling emits small fixed-size stack buffers.
+// The memref.alloc and linalg.generic guards are repeated on both
+// sides of the alloca check so they cover the whole function.
+// CHECK-NOT:     memref.alloc(
+// CHECK-NOT:     linalg.generic
+// CHECK:         memref.alloca(
+// CHECK-NOT:     memref.alloc(
 // CHECK-NOT:     linalg.generic
 
 // -----

compiler/src/iree/compiler/Codegen/LLVMCPU/test/select_x86_64_lowering_strategy.mlir

diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/test/select_x86_64_lowering_strategy.mlir b/compiler/src/iree/compiler/Codegen/LLVMCPU/test/select_x86_64_lowering_strategy.mlir
index d73394e..a551c3c 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMCPU/test/select_x86_64_lowering_strategy.mlir
+++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/test/select_x86_64_lowering_strategy.mlir
@@ -556,7 +556,7 @@
   %7 = linalg.matmul ins(%3, %4 : tensor<128x384xi8>, tensor<384x1536xi8>) outs(%6 : tensor<128x1536xi32>) -> tensor<128x1536xi32>
   return %7 : tensor<128x1536xi32>
 }
-//  CHECK-DAG: #[[CONFIG:.+]] = #iree_cpu.lowering_config<cache_parallel = [64, 64, 0], distribution = [64, 64, 0], vector_common_parallel = [1, 1, 0], vector_reduction = [0, 0, 4]>
+//  CHECK-DAG: #[[CONFIG:.+]] = #iree_cpu.lowering_config<cache_parallel = [64, 64, 0], distribution = [64, 64, 0], vector_common_parallel = [8, 4, 0], vector_reduction = [0, 0, 4]>
 //  CHECK-DAG: #[[TRANSLATION:.+]] = #iree_codegen.translation_info<pipeline = #iree_cpu.pipeline<DoubleTilingExpert>, {{\{}}enable_loop_peeling}>
 //      CHECK: func.func @matmul_i8_i8_i32_static(
 // CHECK-SAME:     translation_info = #[[TRANSLATION]]
@@ -1036,7 +1036,7 @@
   } -> tensor<2304x144xi8>
   return %11 : tensor<2304x144xi8>
 }
-//  CHECK-DAG: #[[CONFIG:.+]] = #iree_cpu.lowering_config<cache_parallel = [64, 48, 0], distribution = [64, 48, 0], vector_common_parallel = [1, 1, 0], vector_reduction = [0, 0, 4]>
+//  CHECK-DAG: #[[CONFIG:.+]] = #iree_cpu.lowering_config<cache_parallel = [64, 48, 0], distribution = [64, 48, 0], vector_common_parallel = [8, 4, 0], vector_reduction = [0, 0, 4]>
 //  CHECK-DAG: #[[TRANSLATION:.+]] = #iree_codegen.translation_info<pipeline = #iree_cpu.pipeline<DoubleTilingExpert>, {{\{}}enable_loop_peeling}>
 //      CHECK: func.func @quant_model(
 // CHECK-SAME:     translation_info = #[[TRANSLATION]]