[GlobalOpt] Keep strided input maps when the producer can fuse (#24950)

Factoring `dim * cst` strides out of a contraction-like generic inserts
a strided `tensor.extract_slice` between the contraction and its input.
When the input has an elementwise producer (a dequantize feeding a 1x1
stride-2 convolution, say), the slice blocks both fusion paths:
elementwise fusion requires the producer to directly define the
consumer's operand, and tile-based producer fusion only supports
unit-stride slices. Codegen falls back to materializing the producer as
a whole-tensor stack allocation in every workgroup: a 3.2 MB alloca for
a 1x256x56x56xf32 dequantize, well over the 32 KB stack limit, and the
compile fails.

Skip the factoring only when the strided input's producer can fuse
elementwise into the contraction and into every other user of its
result, so the fusion actually eliminates the producer. Codegen then
merges the producer into the contraction, which reads the input directly
through the strided map with no intermediate tensor. Producers that are
materialized anyway (e.g. a result also returned from the function) keep
the factored projected-permutation input map.

Strided contractions with a fusable producer now lower through the
generic path instead of contraction selection such as AMDGPU MFMA, the
original motivation for the factoring, in exchange for eliminating the
intermediate tensor. Strided contractions on dispatch-boundary inputs
keep the factored form.

Fixes #24752

Tests: iree-opt --split-input-file --mlir-print-local-scope
-iree-global-opt-convert-strided-contraction-to-contraction
compiler/src/iree/compiler/GlobalOptimization/test/strided_contraction_to_contraction.mlir

Signed-off-by: zhangpan <98025960+zhangpan2001@users.noreply.github.com>
Co-authored-by: OpenAI <noreply@openai.com>
diff --git a/compiler/src/iree/compiler/GlobalOptimization/ConvertStridedContractionToContraction.cpp b/compiler/src/iree/compiler/GlobalOptimization/ConvertStridedContractionToContraction.cpp
index f973948..7034b77 100644
--- a/compiler/src/iree/compiler/GlobalOptimization/ConvertStridedContractionToContraction.cpp
+++ b/compiler/src/iree/compiler/GlobalOptimization/ConvertStridedContractionToContraction.cpp
@@ -105,6 +105,23 @@
       return failure();
     }
 
+    // Factoring the strides out separates the strided input from a producer
+    // that could otherwise fuse elementwise into this op: elementwise fusion
+    // requires the producer to directly define the operand, and tile-based
+    // producer fusion only supports unit-stride slices. Keep the strided map
+    // so that codegen can fuse the producer instead of materializing its
+    // whole result as a dispatch-local buffer. Only do so when every use of
+    // this producer result can fuse elementwise, allowing fusion to eliminate
+    // the intermediate tensor. Otherwise, keep the factored map since the
+    // producer result still needs to be materialized.
+    OpOperand *inputOperand = op.getDpsInputOperand(0);
+    if (linalg::areElementwiseOpsFusable(inputOperand) &&
+        llvm::all_of(inputOperand->get().getUses(), [&](OpOperand &use) {
+          return &use == inputOperand || linalg::areElementwiseOpsFusable(&use);
+        })) {
+      return failure();
+    }
+
     mapRange[inputPos] =
         AffineMap::get(inputMap.getNumDims(), inputMap.getNumSymbols(),
                        replacementExprs, op.getContext());
diff --git a/compiler/src/iree/compiler/GlobalOptimization/test/strided_contraction_to_contraction.mlir b/compiler/src/iree/compiler/GlobalOptimization/test/strided_contraction_to_contraction.mlir
index 780bfbf..0a69fe2 100644
--- a/compiler/src/iree/compiler/GlobalOptimization/test/strided_contraction_to_contraction.mlir
+++ b/compiler/src/iree/compiler/GlobalOptimization/test/strided_contraction_to_contraction.mlir
@@ -115,3 +115,143 @@
 // CHECK-SAME:      affine_map<(d0, d1, d2, d3) -> (d0, d1)>
 // CHECK-SAME:      ins(%[[SLICE]], %[[FILTER]]
 // CHECK: util.return %[[GEN]]
+
+// -----
+
+// The strided input is produced by an elementwise generic (dequantization
+// pattern). Factoring the strides into an extract_slice would separate the
+// producer from the contraction and block elementwise fusion during codegen,
+// which then materializes the producer's whole result as a dispatch-local
+// buffer (see https://github.com/iree-org/iree/issues/24752). The strided
+// indexing map is kept so that the producer can fuse.
+util.func public @strided_from_elementwise_producer(%input: tensor<2x118x182x448xi8>) -> tensor<2x59x91x896xf32> {
+  %cst = arith.constant 1.250000e-01 : f32
+  %cst_0 = arith.constant 0.000000e+00 : f32
+  %0 = tensor.empty() : tensor<2x118x182x448xf32>
+  %1 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2, d3) -> (d0, d1, d2, d3)>, affine_map<(d0, d1, d2, d3) -> (d0, d1, d2, d3)>], iterator_types = ["parallel", "parallel", "parallel", "parallel"]} ins(%input : tensor<2x118x182x448xi8>) outs(%0 : tensor<2x118x182x448xf32>) {
+  ^bb0(%in: i8, %out: f32):
+    %2 = arith.uitofp %in : i8 to f32
+    %3 = arith.mulf %2, %cst : f32
+    linalg.yield %3 : f32
+  } -> tensor<2x118x182x448xf32>
+  %filter = arith.constant dense<0.000000e+00> : tensor<896x448xf32>
+  %4 = tensor.empty() : tensor<2x59x91x896xf32>
+  %5 = linalg.fill ins(%cst_0 : f32) outs(%4 : tensor<2x59x91x896xf32>) -> tensor<2x59x91x896xf32>
+  %6 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2, d3, d4) -> (d0, d1 * 2, d2 * 2, d4)>, affine_map<(d0, d1, d2, d3, d4) -> (d3, d4)>, affine_map<(d0, d1, d2, d3, d4) -> (d0, d1, d2, d3)>], iterator_types = ["parallel", "parallel", "parallel", "parallel", "reduction"]} ins(%1, %filter : tensor<2x118x182x448xf32>, tensor<896x448xf32>) outs(%5 : tensor<2x59x91x896xf32>) {
+  ^bb0(%in: f32, %in_0: f32, %out: f32):
+    %7 = arith.mulf %in, %in_0 : f32
+    %8 = arith.addf %out, %7 : f32
+    linalg.yield %8 : f32
+  } -> tensor<2x59x91x896xf32>
+  util.return %6 : tensor<2x59x91x896xf32>
+}
+
+// CHECK-LABEL: @strided_from_elementwise_producer(
+// CHECK-NOT: tensor.extract_slice
+// CHECK: affine_map<(d0, d1, d2, d3, d4) -> (d0, d1 * 2, d2 * 2, d4)>
+// CHECK-NOT: tensor.extract_slice
+// CHECK: util.return
+
+// -----
+
+// A producer with a reduction loop cannot fuse elementwise into the
+// contraction, so the stride factoring still applies.
+util.func public @strided_from_reduction_producer(%input: tensor<2x64x118x182xf32>) -> tensor<2x59x91x896xf32> {
+  %cst_0 = arith.constant 0.000000e+00 : f32
+  %0 = tensor.empty() : tensor<2x118x182x448xf32>
+  %1 = linalg.fill ins(%cst_0 : f32) outs(%0 : tensor<2x118x182x448xf32>) -> tensor<2x118x182x448xf32>
+  %2 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2, d3, d4) -> (d0, d4, d1, d2)>, affine_map<(d0, d1, d2, d3, d4) -> (d0, d1, d2, d3)>], iterator_types = ["parallel", "parallel", "parallel", "parallel", "reduction"]} ins(%input : tensor<2x64x118x182xf32>) outs(%1 : tensor<2x118x182x448xf32>) {
+  ^bb0(%in: f32, %out: f32):
+    %3 = arith.addf %out, %in : f32
+    linalg.yield %3 : f32
+  } -> tensor<2x118x182x448xf32>
+  %filter = arith.constant dense<0.000000e+00> : tensor<896x448xf32>
+  %4 = tensor.empty() : tensor<2x59x91x896xf32>
+  %5 = linalg.fill ins(%cst_0 : f32) outs(%4 : tensor<2x59x91x896xf32>) -> tensor<2x59x91x896xf32>
+  %6 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2, d3, d4) -> (d0, d1 * 2, d2 * 2, d4)>, affine_map<(d0, d1, d2, d3, d4) -> (d3, d4)>, affine_map<(d0, d1, d2, d3, d4) -> (d0, d1, d2, d3)>], iterator_types = ["parallel", "parallel", "parallel", "parallel", "reduction"]} ins(%2, %filter : tensor<2x118x182x448xf32>, tensor<896x448xf32>) outs(%5 : tensor<2x59x91x896xf32>) {
+  ^bb0(%in: f32, %in_0: f32, %out: f32):
+    %7 = arith.mulf %in, %in_0 : f32
+    %8 = arith.addf %out, %7 : f32
+    linalg.yield %8 : f32
+  } -> tensor<2x59x91x896xf32>
+  util.return %6 : tensor<2x59x91x896xf32>
+}
+
+// CHECK-LABEL: @strided_from_reduction_producer(
+// CHECK: %[[SLICE:.*]] = tensor.extract_slice %[[PROD:.*]][0, 0, 0, 0] [2, 59, 91, 448] [1, 2, 2, 1]
+// CHECK: linalg.generic
+// CHECK-SAME: affine_map<(d0, d1, d2, d3, d4) -> (d0, d1, d2, d4)>
+// CHECK-SAME: ins(%[[SLICE]]
+// CHECK: util.return
+
+// -----
+
+// The producer result has a second use that cannot fuse elementwise (the
+// function return), so it is materialized regardless: the stride factoring
+// is kept to preserve the projected-permutation input map.
+util.func public @strided_from_multi_use_producer(%input: tensor<2x118x182x448xi8>) -> (tensor<2x59x91x896xf32>, tensor<2x118x182x448xf32>) {
+  %cst = arith.constant 1.250000e-01 : f32
+  %cst_0 = arith.constant 0.000000e+00 : f32
+  %0 = tensor.empty() : tensor<2x118x182x448xf32>
+  %1 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2, d3) -> (d0, d1, d2, d3)>, affine_map<(d0, d1, d2, d3) -> (d0, d1, d2, d3)>], iterator_types = ["parallel", "parallel", "parallel", "parallel"]} ins(%input : tensor<2x118x182x448xi8>) outs(%0 : tensor<2x118x182x448xf32>) {
+  ^bb0(%in: i8, %out: f32):
+    %2 = arith.uitofp %in : i8 to f32
+    %3 = arith.mulf %2, %cst : f32
+    linalg.yield %3 : f32
+  } -> tensor<2x118x182x448xf32>
+  %filter = arith.constant dense<0.000000e+00> : tensor<896x448xf32>
+  %4 = tensor.empty() : tensor<2x59x91x896xf32>
+  %5 = linalg.fill ins(%cst_0 : f32) outs(%4 : tensor<2x59x91x896xf32>) -> tensor<2x59x91x896xf32>
+  %6 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2, d3, d4) -> (d0, d1 * 2, d2 * 2, d4)>, affine_map<(d0, d1, d2, d3, d4) -> (d3, d4)>, affine_map<(d0, d1, d2, d3, d4) -> (d0, d1, d2, d3)>], iterator_types = ["parallel", "parallel", "parallel", "parallel", "reduction"]} ins(%1, %filter : tensor<2x118x182x448xf32>, tensor<896x448xf32>) outs(%5 : tensor<2x59x91x896xf32>) {
+  ^bb0(%in: f32, %in_0: f32, %out: f32):
+    %7 = arith.mulf %in, %in_0 : f32
+    %8 = arith.addf %out, %7 : f32
+    linalg.yield %8 : f32
+  } -> tensor<2x59x91x896xf32>
+  util.return %6, %1 : tensor<2x59x91x896xf32>, tensor<2x118x182x448xf32>
+}
+
+// CHECK-LABEL: @strided_from_multi_use_producer(
+// CHECK: %[[SLICE:.*]] = tensor.extract_slice %[[DEQ:.*]][0, 0, 0, 0] [2, 59, 91, 448] [1, 2, 2, 1]
+// CHECK: linalg.generic
+// CHECK-SAME: affine_map<(d0, d1, d2, d3, d4) -> (d0, d1, d2, d4)>
+// CHECK-SAME: ins(%[[SLICE]]
+// CHECK: util.return
+
+// -----
+
+// Keep the strided map when all uses of the producer result can fuse
+// elementwise, even when the result has multiple uses.
+util.func public @strided_from_multiple_fusable_uses(%input: tensor<2x118x182x448xi8>) -> (tensor<2x59x91x896xf32>, tensor<2x118x182x448xf32>) {
+  %cst = arith.constant 1.250000e-01 : f32
+  %cst_0 = arith.constant 0.000000e+00 : f32
+  %0 = tensor.empty() : tensor<2x118x182x448xf32>
+  %1 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2, d3) -> (d0, d1, d2, d3)>, affine_map<(d0, d1, d2, d3) -> (d0, d1, d2, d3)>], iterator_types = ["parallel", "parallel", "parallel", "parallel"]} ins(%input : tensor<2x118x182x448xi8>) outs(%0 : tensor<2x118x182x448xf32>) {
+  ^bb0(%in: i8, %out: f32):
+    %2 = arith.uitofp %in : i8 to f32
+    %3 = arith.mulf %2, %cst : f32
+    linalg.yield %3 : f32
+  } -> tensor<2x118x182x448xf32>
+  %filter = arith.constant dense<0.000000e+00> : tensor<896x448xf32>
+  %4 = tensor.empty() : tensor<2x59x91x896xf32>
+  %5 = linalg.fill ins(%cst_0 : f32) outs(%4 : tensor<2x59x91x896xf32>) -> tensor<2x59x91x896xf32>
+  %6 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2, d3, d4) -> (d0, d1 * 2, d2 * 2, d4)>, affine_map<(d0, d1, d2, d3, d4) -> (d3, d4)>, affine_map<(d0, d1, d2, d3, d4) -> (d0, d1, d2, d3)>], iterator_types = ["parallel", "parallel", "parallel", "parallel", "reduction"]} ins(%1, %filter : tensor<2x118x182x448xf32>, tensor<896x448xf32>) outs(%5 : tensor<2x59x91x896xf32>) {
+  ^bb0(%in: f32, %in_0: f32, %out: f32):
+    %7 = arith.mulf %in, %in_0 : f32
+    %8 = arith.addf %out, %7 : f32
+    linalg.yield %8 : f32
+  } -> tensor<2x59x91x896xf32>
+  %init2 = tensor.empty() : tensor<2x118x182x448xf32>
+  %9 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2, d3) -> (d0, d1, d2, d3)>, affine_map<(d0, d1, d2, d3) -> (d0, d1, d2, d3)>], iterator_types = ["parallel", "parallel", "parallel", "parallel"]} ins(%1 : tensor<2x118x182x448xf32>) outs(%init2 : tensor<2x118x182x448xf32>) {
+  ^bb0(%in: f32, %out: f32):
+    %10 = arith.negf %in : f32
+    linalg.yield %10 : f32
+  } -> tensor<2x118x182x448xf32>
+  util.return %6, %9 : tensor<2x59x91x896xf32>, tensor<2x118x182x448xf32>
+}
+
+// CHECK-LABEL: @strided_from_multiple_fusable_uses(
+// CHECK-NOT: tensor.extract_slice
+// CHECK: affine_map<(d0, d1, d2, d3, d4) -> (d0, d1 * 2, d2 * 2, d4)>
+// CHECK-NOT: tensor.extract_slice
+// CHECK: util.return