[DispatchCreation] Normalize contraction dimensions outside dispatches (#24965) Contractions with several contiguous M, N, K or batch dimensions can miss data-tiling encodings because dimension collapsing currently runs inside dispatches. Add a function pass that reuses the existing collapse rules outside dispatches and restores the original tensor shapes at the op boundaries. Leave existing dispatches (regions and workgroups) and ops with a preset compilation info or lowering config alone, since those configurations refer to the original loops. No pipeline runs the pass yet. #24969 adds it to the experimental early data-tiling route. This is one of five PRs adding an experimental CPU path that materializes data-tiling layouts before dispatch creation, so that `linalg.mmt4d` and data-tiled convolutions fuse with their elementwise epilogues and result unpacks. Assisted by AI.
diff --git a/compiler/src/iree/compiler/DispatchCreation/CollapseDimensions.cpp b/compiler/src/iree/compiler/DispatchCreation/CollapseDimensions.cpp index 7dcc2a1..73d6edb 100644 --- a/compiler/src/iree/compiler/DispatchCreation/CollapseDimensions.cpp +++ b/compiler/src/iree/compiler/DispatchCreation/CollapseDimensions.cpp
@@ -4,6 +4,7 @@ // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +#include "iree/compiler/Codegen/Dialect/Codegen/IR/IREECodegenAttrs.h" #include "iree/compiler/Dialect/Encoding/Utils/Utils.h" #include "iree/compiler/Dialect/Flow/IR/FlowOps.h" #include "iree/compiler/Dialect/Flow/Transforms/FormDispatchRegions.h" @@ -44,6 +45,7 @@ namespace mlir::iree_compiler::DispatchCreation { +#define GEN_PASS_DEF_COLLAPSECONTRACTIONDIMENSIONSPASS #define GEN_PASS_DEF_COLLAPSEDIMENSIONSPASS #include "iree/compiler/DispatchCreation/Passes.h.inc" @@ -56,6 +58,12 @@ using Base::Base; void runOnOperation() override; }; + +struct CollapseContractionDimensionsPass final + : impl::CollapseContractionDimensionsPassBase< + CollapseContractionDimensionsPass> { + void runOnOperation() override; +}; } // namespace //===---------------------------------------------------------------------===// @@ -1201,6 +1209,38 @@ // Passes //===---------------------------------------------------------------------===// +void CollapseContractionDimensionsPass::runOnOperation() { + SmallVector<linalg::GenericOp> contractions; + getOperation()->walk([&](linalg::GenericOp op) { + // Dispatch bodies already have their own normalization and shape-hoisting + // policy. Do not revisit explicitly scheduled workgroups either. A preset + // compilation info or lowering config refers to the original loops, so + // collapsing would invalidate it. + if (op->getParentOfType<IREE::Flow::DispatchRegionOp>() || + op->getParentOfType<IREE::Flow::DispatchWorkgroupsOp>() || + getCompilationInfo(op) || getLoweringConfig(op) || + !op.hasPureTensorSemantics() || !isEligibleForCollapse(op) || + !linalg::isaContractionOpInterface(op)) { + return; + } + contractions.push_back(op); + }); + + IRRewriter rewriter(&getContext()); + for (linalg::GenericOp op : contractions) { + SmallVector<ReassociationIndices> reassociation = getCollapsibleLoops(op); + if (reassociation.empty()) { + continue; + } + rewriter.setInsertionPoint(op); + FailureOr<linalg::CollapseResult> result = + linalg::collapseOpIterationDims(op, reassociation, rewriter); + if (succeeded(result)) { + rewriter.replaceOp(op, result->results); + } + } +} + void CollapseDimensionsPass::runOnOperation() { mlir::FunctionOpInterface funcOp = getOperation(); MLIRContext *context = funcOp->getContext();
diff --git a/compiler/src/iree/compiler/DispatchCreation/Passes.td b/compiler/src/iree/compiler/DispatchCreation/Passes.td index c2ae094..0b67715 100644 --- a/compiler/src/iree/compiler/DispatchCreation/Passes.td +++ b/compiler/src/iree/compiler/DispatchCreation/Passes.td
@@ -46,6 +46,26 @@ ]; } +def CollapseContractionDimensionsPass : + InterfacePass<"iree-dispatch-creation-collapse-contraction-dimensions", "mlir::FunctionOpInterface"> { + let summary = "Collapse contiguous dimensions of contractions outside dispatches."; + let description = [{ + Collapse contiguous dimensions of tensor contractions outside dispatches + using the same indexing-map and iterator legality as dispatch dimension + collapsing. Preserve operand and result shapes with tensor reshapes. This + makes collapsible multi-M/N/K/batch contractions eligible for encoding + without requiring dispatch formation first. Contractions inside dispatches + and contractions with a compilation info or lowering config are left + unchanged. + }]; + let dependentDialects = [ + "mlir::affine::AffineDialect", + "mlir::arith::ArithDialect", + "mlir::linalg::LinalgDialect", + "mlir::tensor::TensorDialect", + ]; +} + def InsertTensorBarriersPass : InterfacePass<"iree-dispatch-creation-insert-tensor-barriers", "mlir::FunctionOpInterface"> {
diff --git a/compiler/src/iree/compiler/DispatchCreation/test/BUILD.bazel b/compiler/src/iree/compiler/DispatchCreation/test/BUILD.bazel index 619a4b5..7e19e92 100644 --- a/compiler/src/iree/compiler/DispatchCreation/test/BUILD.bazel +++ b/compiler/src/iree/compiler/DispatchCreation/test/BUILD.bazel
@@ -22,6 +22,8 @@ "bubble_up_expand_shapes.mlir", "bubble_up_extract_slice.mlir", "clone_producers_into_dispatch_regions.mlir", + "collapse_contraction_dimensions.mlir", + "collapse_contraction_dimensions_scope.mlir", "collapse_dimensions.mlir", "collapse_linalg_generic_on_tensors.mlir", "convert_encoding_to_flow.mlir",
diff --git a/compiler/src/iree/compiler/DispatchCreation/test/CMakeLists.txt b/compiler/src/iree/compiler/DispatchCreation/test/CMakeLists.txt index 89044ac..e407742 100644 --- a/compiler/src/iree/compiler/DispatchCreation/test/CMakeLists.txt +++ b/compiler/src/iree/compiler/DispatchCreation/test/CMakeLists.txt
@@ -19,6 +19,8 @@ "bubble_up_expand_shapes.mlir" "bubble_up_extract_slice.mlir" "clone_producers_into_dispatch_regions.mlir" + "collapse_contraction_dimensions.mlir" + "collapse_contraction_dimensions_scope.mlir" "collapse_dimensions.mlir" "collapse_linalg_generic_on_tensors.mlir" "convert_encoding_to_flow.mlir"
diff --git a/compiler/src/iree/compiler/DispatchCreation/test/collapse_contraction_dimensions.mlir b/compiler/src/iree/compiler/DispatchCreation/test/collapse_contraction_dimensions.mlir new file mode 100644 index 0000000..cdd0ec2 --- /dev/null +++ b/compiler/src/iree/compiler/DispatchCreation/test/collapse_contraction_dimensions.mlir
@@ -0,0 +1,196 @@ +// RUN: iree-opt --split-input-file --pass-pipeline='builtin.module(util.func(iree-dispatch-creation-collapse-contraction-dimensions))' %s | FileCheck %s +// RUN: iree-opt --split-input-file --pass-pipeline='builtin.module(util.func(iree-dispatch-creation-collapse-contraction-dimensions,iree-dispatch-creation-annotate-data-tiling-hints,iree-dispatch-creation-set-encoding))' %s | FileCheck %s --check-prefix=ENCODE + +// Two contiguous M dimensions collapse into one, and the result is expanded +// back to the original shape. Encoding annotation rejects the uncollapsed form, +// so the ENCODE run checks that the collapsed contraction now gets encodings. +// CHECK-LABEL: @multi_m( +// CHECK-SAME: %[[A:[a-zA-Z0-9]+]]: tensor<2x3x8xf32> +// CHECK-SAME: %[[B:[a-zA-Z0-9]+]]: tensor<8x4xf32> +// CHECK-SAME: %[[INIT:[a-zA-Z0-9]+]]: tensor<2x3x4xf32> +// CHECK-DAG: %[[A_2D:.+]] = tensor.collapse_shape %[[A]] {{\[}}[0, 1], [2]] : tensor<2x3x8xf32> into tensor<6x8xf32> +// CHECK-DAG: %[[INIT_2D:.+]] = tensor.collapse_shape %[[INIT]] {{\[}}[0, 1], [2]] : tensor<2x3x4xf32> into tensor<6x4xf32> +// CHECK: %[[MATMUL:.+]] = linalg.generic +// CHECK-SAME: iterator_types = ["parallel", "parallel", "reduction"] +// CHECK-SAME: ins(%[[A_2D]], %[[B]] : tensor<6x8xf32>, tensor<8x4xf32>) +// CHECK-SAME: outs(%[[INIT_2D]] : tensor<6x4xf32>) +// CHECK: %[[RESULT:.+]] = tensor.expand_shape %[[MATMUL]] {{\[}}[0, 1], [2]] output_shape [2, 3, 4] : tensor<6x4xf32> into tensor<2x3x4xf32> +// CHECK: util.return %[[RESULT]] +// ENCODE-LABEL: @multi_m( +// ENCODE: %[[A_2D:.+]] = tensor.collapse_shape +// ENCODE: %[[LHS:.+]] = iree_encoding.set_encoding %[[A_2D]] : tensor<6x8xf32> -> tensor<6x8xf32, # +// ENCODE: %[[MATMUL:.+]] = linalg.generic +// ENCODE-SAME: ins(%[[LHS]], +// ENCODE: %[[UNSET:.+]] = iree_encoding.unset_encoding %[[MATMUL]] +// ENCODE: tensor.expand_shape %[[UNSET]] +util.func public @multi_m(%a: tensor<2x3x8xf32>, %b: tensor<8x4xf32>, %init: tensor<2x3x4xf32>) -> tensor<2x3x4xf32> { + %result = linalg.generic { + indexing_maps = [affine_map<(m0, m1, n, k) -> (m0, m1, k)>, + affine_map<(m0, m1, n, k) -> (k, n)>, + affine_map<(m0, m1, n, k) -> (m0, m1, n)>], + iterator_types = ["parallel", "parallel", "parallel", "reduction"]} + ins(%a, %b : tensor<2x3x8xf32>, tensor<8x4xf32>) + outs(%init : tensor<2x3x4xf32>) { + ^bb0(%lhs: f32, %rhs: f32, %acc: f32): + %mul = arith.mulf %lhs, %rhs : f32 + %sum = arith.addf %mul, %acc : f32 + linalg.yield %sum : f32 + } -> tensor<2x3x4xf32> + util.return %result : tensor<2x3x4xf32> +} + +// ----- + +// Only the K dimensions collapse; the result keeps its shape, so no expand is +// needed. +// CHECK-LABEL: @multi_k( +// CHECK-SAME: %[[A:[a-zA-Z0-9]+]]: tensor<256x64x2xf32> +// CHECK-SAME: %[[B:[a-zA-Z0-9]+]]: tensor<64x2x512xf32> +// CHECK-SAME: %[[INIT:[a-zA-Z0-9]+]]: tensor<256x512xf32> +// CHECK-DAG: %[[A_2D:.+]] = tensor.collapse_shape %[[A]] {{\[}}[0], [1, 2]] : tensor<256x64x2xf32> into tensor<256x128xf32> +// CHECK-DAG: %[[B_2D:.+]] = tensor.collapse_shape %[[B]] {{\[}}[0, 1], [2]] : tensor<64x2x512xf32> into tensor<128x512xf32> +// CHECK: %[[MATMUL:.+]] = linalg.generic +// CHECK-SAME: iterator_types = ["parallel", "parallel", "reduction"] +// CHECK-SAME: ins(%[[A_2D]], %[[B_2D]] : tensor<256x128xf32>, tensor<128x512xf32>) +// CHECK-SAME: outs(%[[INIT]] : tensor<256x512xf32>) +// CHECK-NOT: tensor.expand_shape +// CHECK: util.return %[[MATMUL]] +util.func public @multi_k(%a: tensor<256x64x2xf32>, %b: tensor<64x2x512xf32>, + %init: tensor<256x512xf32>) -> tensor<256x512xf32> { + %result = linalg.generic { + indexing_maps = [affine_map<(m, n, k0, k1) -> (m, k0, k1)>, + affine_map<(m, n, k0, k1) -> (k0, k1, n)>, + affine_map<(m, n, k0, k1) -> (m, n)>], + iterator_types = ["parallel", "parallel", "reduction", "reduction"]} + ins(%a, %b : tensor<256x64x2xf32>, tensor<64x2x512xf32>) + outs(%init : tensor<256x512xf32>) { + ^bb0(%lhs: f32, %rhs: f32, %acc: f32): + %mul = arith.mulf %lhs, %rhs : f32 + %sum = arith.addf %mul, %acc : f32 + linalg.yield %sum : f32 + } -> tensor<256x512xf32> + util.return %result : tensor<256x512xf32> +} + +// ----- + +// Two batch dimensions collapse across all three operands. +// CHECK-LABEL: @multi_batch( +// CHECK-SAME: %[[A:[a-zA-Z0-9]+]]: tensor<4x8x256x128xf32> +// CHECK-SAME: %[[B:[a-zA-Z0-9]+]]: tensor<4x8x128x512xf32> +// CHECK-SAME: %[[INIT:[a-zA-Z0-9]+]]: tensor<4x8x256x512xf32> +// CHECK-DAG: %[[A_3D:.+]] = tensor.collapse_shape %[[A]] {{\[}}[0, 1], [2], [3]] : tensor<4x8x256x128xf32> into tensor<32x256x128xf32> +// CHECK-DAG: %[[B_3D:.+]] = tensor.collapse_shape %[[B]] {{\[}}[0, 1], [2], [3]] : tensor<4x8x128x512xf32> into tensor<32x128x512xf32> +// CHECK-DAG: %[[INIT_3D:.+]] = tensor.collapse_shape %[[INIT]] {{\[}}[0, 1], [2], [3]] : tensor<4x8x256x512xf32> into tensor<32x256x512xf32> +// CHECK: %[[BATCH_MATMUL:.+]] = linalg.generic +// CHECK-SAME: iterator_types = ["parallel", "parallel", "parallel", "reduction"] +// CHECK-SAME: ins(%[[A_3D]], %[[B_3D]] : tensor<32x256x128xf32>, tensor<32x128x512xf32>) +// CHECK-SAME: outs(%[[INIT_3D]] : tensor<32x256x512xf32>) +// CHECK: %[[RESULT:.+]] = tensor.expand_shape %[[BATCH_MATMUL]] {{\[}}[0, 1], [2], [3]] output_shape [4, 8, 256, 512] +// CHECK: util.return %[[RESULT]] +util.func public @multi_batch(%a: tensor<4x8x256x128xf32>, %b: tensor<4x8x128x512xf32>, + %init: tensor<4x8x256x512xf32>) -> tensor<4x8x256x512xf32> { + %result = linalg.generic { + indexing_maps = [affine_map<(b0, b1, m, n, k) -> (b0, b1, m, k)>, + affine_map<(b0, b1, m, n, k) -> (b0, b1, k, n)>, + affine_map<(b0, b1, m, n, k) -> (b0, b1, m, n)>], + iterator_types = ["parallel", "parallel", "parallel", "parallel", "reduction"]} + ins(%a, %b : tensor<4x8x256x128xf32>, tensor<4x8x128x512xf32>) + outs(%init : tensor<4x8x256x512xf32>) { + ^bb0(%lhs: f32, %rhs: f32, %acc: f32): + %mul = arith.mulf %lhs, %rhs : f32 + %sum = arith.addf %mul, %acc : f32 + linalg.yield %sum : f32 + } -> tensor<4x8x256x512xf32> + util.return %result : tensor<4x8x256x512xf32> +} + +// ----- + +// A batch dimension that only the LHS and result carry is an M dimension; it +// collapses with the other M dimension. Extending bodies are preserved. +// CHECK-LABEL: @broadcast_rhs_batch_mmt( +// CHECK-SAME: %[[A:[a-zA-Z0-9]+]]: tensor<16x1024x1280xi8> +// CHECK-SAME: %[[B:[a-zA-Z0-9]+]]: tensor<10240x1280xi8> +// CHECK-SAME: %[[INIT:[a-zA-Z0-9]+]]: tensor<16x1024x10240xi32> +// CHECK-DAG: %[[A_2D:.+]] = tensor.collapse_shape %[[A]] {{\[}}[0, 1], [2]] : tensor<16x1024x1280xi8> into tensor<16384x1280xi8> +// CHECK-DAG: %[[INIT_2D:.+]] = tensor.collapse_shape %[[INIT]] {{\[}}[0, 1], [2]] : tensor<16x1024x10240xi32> into tensor<16384x10240xi32> +// CHECK: %[[MMT:.+]] = linalg.generic +// CHECK-SAME: ins(%[[A_2D]], %[[B]] : tensor<16384x1280xi8>, tensor<10240x1280xi8>) +// CHECK-SAME: outs(%[[INIT_2D]] : tensor<16384x10240xi32>) +// CHECK: arith.extsi +// CHECK: arith.extsi +// CHECK: arith.muli +// CHECK: arith.addi +// CHECK: %[[RESULT:.+]] = tensor.expand_shape %[[MMT]] {{\[}}[0, 1], [2]] output_shape [16, 1024, 10240] +// CHECK: util.return %[[RESULT]] +util.func public @broadcast_rhs_batch_mmt(%a: tensor<16x1024x1280xi8>, %b: tensor<10240x1280xi8>, + %init: tensor<16x1024x10240xi32>) -> tensor<16x1024x10240xi32> { + %result = linalg.generic { + indexing_maps = [affine_map<(m0, m1, n, k) -> (m0, m1, k)>, + affine_map<(m0, m1, n, k) -> (n, k)>, + affine_map<(m0, m1, n, k) -> (m0, m1, n)>], + iterator_types = ["parallel", "parallel", "parallel", "reduction"]} + ins(%a, %b : tensor<16x1024x1280xi8>, tensor<10240x1280xi8>) + outs(%init : tensor<16x1024x10240xi32>) { + ^bb0(%lhs: i8, %rhs: i8, %acc: i32): + %lhs_ext = arith.extsi %lhs : i8 to i32 + %rhs_ext = arith.extsi %rhs : i8 to i32 + %mul = arith.muli %lhs_ext, %rhs_ext : i32 + %sum = arith.addi %acc, %mul : i32 + linalg.yield %sum : i32 + } -> tensor<16x1024x10240xi32> + util.return %result : tensor<16x1024x10240xi32> +} + +// ----- + +// A dynamic outer M dimension is recovered from the input for the expand. +// CHECK-LABEL: @dynamic_multi_m( +// CHECK-SAME: %[[A:[a-zA-Z0-9]+]]: tensor<?x3x8xf32> +// CHECK: %[[M0:.+]] = tensor.dim %[[A]] +// CHECK: %[[A_2D:.+]] = tensor.collapse_shape %[[A]] {{\[}}[0, 1], [2]] : tensor<?x3x8xf32> into tensor<?x8xf32> +// CHECK: %[[MATMUL:.+]] = linalg.generic +// CHECK-SAME: ins(%[[A_2D]], +// CHECK: %[[RESULT:.+]] = tensor.expand_shape %[[MATMUL]] {{\[}}[0, 1], [2]] output_shape [%[[M0]], 3, 4] : tensor<?x4xf32> into tensor<?x3x4xf32> +// CHECK: util.return %[[RESULT]] +util.func public @dynamic_multi_m(%a: tensor<?x3x8xf32>, %b: tensor<8x4xf32>, %init: tensor<?x3x4xf32>) -> tensor<?x3x4xf32> { + %result = linalg.generic { + indexing_maps = [affine_map<(m0, m1, n, k) -> (m0, m1, k)>, + affine_map<(m0, m1, n, k) -> (k, n)>, + affine_map<(m0, m1, n, k) -> (m0, m1, n)>], + iterator_types = ["parallel", "parallel", "parallel", "reduction"]} + ins(%a, %b : tensor<?x3x8xf32>, tensor<8x4xf32>) + outs(%init : tensor<?x3x4xf32>) { + ^bb0(%lhs: f32, %rhs: f32, %acc: f32): + %mul = arith.mulf %lhs, %rhs : f32 + %sum = arith.addf %mul, %acc : f32 + linalg.yield %sum : f32 + } -> tensor<?x3x4xf32> + util.return %result : tensor<?x3x4xf32> +} + +// ----- + +// The M dimensions are not contiguous in the LHS, so nothing collapses. +// CHECK-LABEL: @no_collapse_noncontiguous_m( +// CHECK-NOT: tensor.collapse_shape +// CHECK: linalg.generic +// CHECK-SAME: iterator_types = ["parallel", "parallel", "parallel", "reduction"] +// CHECK-NOT: tensor.expand_shape +// CHECK: util.return +util.func public @no_collapse_noncontiguous_m(%a: tensor<2x8x3xf32>, %b: tensor<8x4xf32>, %init: tensor<2x3x4xf32>) -> tensor<2x3x4xf32> { + %result = linalg.generic { + indexing_maps = [affine_map<(m0, m1, n, k) -> (m0, k, m1)>, + affine_map<(m0, m1, n, k) -> (k, n)>, + affine_map<(m0, m1, n, k) -> (m0, m1, n)>], + iterator_types = ["parallel", "parallel", "parallel", "reduction"]} + ins(%a, %b : tensor<2x8x3xf32>, tensor<8x4xf32>) + outs(%init : tensor<2x3x4xf32>) { + ^bb0(%lhs: f32, %rhs: f32, %acc: f32): + %mul = arith.mulf %lhs, %rhs : f32 + %sum = arith.addf %mul, %acc : f32 + linalg.yield %sum : f32 + } -> tensor<2x3x4xf32> + util.return %result : tensor<2x3x4xf32> +}
diff --git a/compiler/src/iree/compiler/DispatchCreation/test/collapse_contraction_dimensions_scope.mlir b/compiler/src/iree/compiler/DispatchCreation/test/collapse_contraction_dimensions_scope.mlir new file mode 100644 index 0000000..879d4ad --- /dev/null +++ b/compiler/src/iree/compiler/DispatchCreation/test/collapse_contraction_dimensions_scope.mlir
@@ -0,0 +1,140 @@ +// RUN: iree-opt --split-input-file --pass-pipeline="builtin.module(util.func(iree-dispatch-creation-collapse-contraction-dimensions))" %s | FileCheck %s --implicit-check-not=tensor.collapse_shape --implicit-check-not=tensor.expand_shape + +#compilation = #iree_codegen.compilation_info< + lowering_config = #iree_codegen.lowering_config<tile_sizes = [[0, 0, 0, 0]]>, + translation_info = #iree_codegen.translation_info<pipeline = #iree_cpu.pipeline<Default>>> + +// A preset compilation info refers to the original loop dimensions. +// CHECK-LABEL: @no_collapse_compilation_info( +// CHECK: linalg.generic +// CHECK-SAME: iterator_types = ["parallel", "parallel", "parallel", "reduction"] +// CHECK-SAME: compilation_info = +// CHECK: util.return {{.*}} : tensor<2x3x4xf32> +util.func public @no_collapse_compilation_info(%a: tensor<2x3x8xf32>, %b: tensor<8x4xf32>, %init: tensor<2x3x4xf32>) -> tensor<2x3x4xf32> { + %result = linalg.generic {compilation_info = #compilation, + indexing_maps = [affine_map<(m0, m1, n, k) -> (m0, m1, k)>, + affine_map<(m0, m1, n, k) -> (k, n)>, + affine_map<(m0, m1, n, k) -> (m0, m1, n)>], + iterator_types = ["parallel", "parallel", "parallel", "reduction"]} + ins(%a, %b : tensor<2x3x8xf32>, tensor<8x4xf32>) + outs(%init : tensor<2x3x4xf32>) { + ^bb0(%lhs: f32, %rhs: f32, %acc: f32): + %mul = arith.mulf %lhs, %rhs : f32 + %sum = arith.addf %mul, %acc : f32 + linalg.yield %sum : f32 + } -> tensor<2x3x4xf32> + util.return %result : tensor<2x3x4xf32> +} + +// ----- + +// A lowering config without a translation info also refers to the original +// loop dimensions. +// CHECK-LABEL: @no_collapse_lowering_config( +// CHECK: linalg.generic +// CHECK-SAME: iterator_types = ["parallel", "parallel", "parallel", "reduction"] +// CHECK-SAME: lowering_config = +// CHECK: util.return {{.*}} : tensor<2x3x4xf32> +util.func public @no_collapse_lowering_config(%a: tensor<2x3x8xf32>, %b: tensor<8x4xf32>, %init: tensor<2x3x4xf32>) -> tensor<2x3x4xf32> { + %result = linalg.generic {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[0, 0, 0, 0]]>, + indexing_maps = [affine_map<(m0, m1, n, k) -> (m0, m1, k)>, + affine_map<(m0, m1, n, k) -> (k, n)>, + affine_map<(m0, m1, n, k) -> (m0, m1, n)>], + iterator_types = ["parallel", "parallel", "parallel", "reduction"]} + ins(%a, %b : tensor<2x3x8xf32>, tensor<8x4xf32>) + outs(%init : tensor<2x3x4xf32>) { + ^bb0(%lhs: f32, %rhs: f32, %acc: f32): + %mul = arith.mulf %lhs, %rhs : f32 + %sum = arith.addf %mul, %acc : f32 + linalg.yield %sum : f32 + } -> tensor<2x3x4xf32> + util.return %result : tensor<2x3x4xf32> +} + +// ----- + +// Dispatch formation already owns normalization inside this region. +// CHECK-LABEL: @no_collapse_in_dispatch_region( +// CHECK: flow.dispatch.region +// CHECK: linalg.generic +// CHECK-SAME: iterator_types = ["parallel", "parallel", "parallel", "reduction"] +// CHECK: flow.return {{.*}} : tensor<2x3x4xf32> +util.func public @no_collapse_in_dispatch_region(%a: tensor<2x3x8xf32>, %b: tensor<8x4xf32>, %init: tensor<2x3x4xf32>) -> tensor<2x3x4xf32> { + %dispatch = flow.dispatch.region -> (tensor<2x3x4xf32>) { + %result = linalg.generic { + indexing_maps = [affine_map<(m0, m1, n, k) -> (m0, m1, k)>, + affine_map<(m0, m1, n, k) -> (k, n)>, + affine_map<(m0, m1, n, k) -> (m0, m1, n)>], + iterator_types = ["parallel", "parallel", "parallel", "reduction"]} + ins(%a, %b : tensor<2x3x8xf32>, tensor<8x4xf32>) + outs(%init : tensor<2x3x4xf32>) { + ^bb0(%lhs: f32, %rhs: f32, %acc: f32): + %mul = arith.mulf %lhs, %rhs : f32 + %sum = arith.addf %mul, %acc : f32 + linalg.yield %sum : f32 + } -> tensor<2x3x4xf32> + flow.return %result : tensor<2x3x4xf32> + } + util.return %dispatch : tensor<2x3x4xf32> +} + +// ----- + +// Explicitly scheduled workgroups are not revisited either. +// CHECK-LABEL: @no_collapse_in_dispatch_workgroups( +// CHECK: flow.dispatch.workgroups +// CHECK: linalg.generic +// CHECK-SAME: iterator_types = ["parallel", "parallel", "parallel", "reduction"] +// CHECK: iree_tensor_ext.dispatch.tensor.store +util.func public @no_collapse_in_dispatch_workgroups(%a: tensor<2x3x8xf32>, %b: tensor<8x4xf32>, %init: tensor<2x3x4xf32>) -> tensor<2x3x4xf32> { + %dispatch = flow.dispatch.workgroups(%a, %b, %init) + : (tensor<2x3x8xf32>, tensor<8x4xf32>, tensor<2x3x4xf32>) -> tensor<2x3x4xf32> = ( + %a_binding: !iree_tensor_ext.dispatch.tensor<readonly:tensor<2x3x8xf32>>, + %b_binding: !iree_tensor_ext.dispatch.tensor<readonly:tensor<8x4xf32>>, + %init_binding: !iree_tensor_ext.dispatch.tensor<readonly:tensor<2x3x4xf32>>, + %result_binding: !iree_tensor_ext.dispatch.tensor<writeonly:tensor<2x3x4xf32>> + ) { + %a_tile = iree_tensor_ext.dispatch.tensor.load %a_binding, offsets = [0, 0, 0], sizes = [2, 3, 8], strides = [1, 1, 1] + : !iree_tensor_ext.dispatch.tensor<readonly:tensor<2x3x8xf32>> -> tensor<2x3x8xf32> + %b_tile = iree_tensor_ext.dispatch.tensor.load %b_binding, offsets = [0, 0], sizes = [8, 4], strides = [1, 1] + : !iree_tensor_ext.dispatch.tensor<readonly:tensor<8x4xf32>> -> tensor<8x4xf32> + %init_tile = iree_tensor_ext.dispatch.tensor.load %init_binding, offsets = [0, 0, 0], sizes = [2, 3, 4], strides = [1, 1, 1] + : !iree_tensor_ext.dispatch.tensor<readonly:tensor<2x3x4xf32>> -> tensor<2x3x4xf32> + %result = linalg.generic { + indexing_maps = [affine_map<(m0, m1, n, k) -> (m0, m1, k)>, + affine_map<(m0, m1, n, k) -> (k, n)>, + affine_map<(m0, m1, n, k) -> (m0, m1, n)>], + iterator_types = ["parallel", "parallel", "parallel", "reduction"]} + ins(%a_tile, %b_tile : tensor<2x3x8xf32>, tensor<8x4xf32>) + outs(%init_tile : tensor<2x3x4xf32>) { + ^bb0(%lhs: f32, %rhs: f32, %acc: f32): + %mul = arith.mulf %lhs, %rhs : f32 + %sum = arith.addf %mul, %acc : f32 + linalg.yield %sum : f32 + } -> tensor<2x3x4xf32> + iree_tensor_ext.dispatch.tensor.store %result, %result_binding, offsets = [0, 0, 0], sizes = [2, 3, 4], strides = [1, 1, 1] + : tensor<2x3x4xf32> -> !iree_tensor_ext.dispatch.tensor<writeonly:tensor<2x3x4xf32>> + flow.return + } + util.return %dispatch : tensor<2x3x4xf32> +} + +// ----- + +// This pass normalizes contractions, not unrelated elementwise operations. +// CHECK-LABEL: @no_collapse_pointwise( +// CHECK: linalg.generic +// CHECK-SAME: iterator_types = ["parallel", "parallel", "parallel"] +// CHECK: math.absf +// CHECK: util.return {{.*}} : tensor<2x3x4xf32> +util.func public @no_collapse_pointwise(%input: tensor<2x3x4xf32>, %init: tensor<2x3x4xf32>) -> tensor<2x3x4xf32> { + %result = linalg.generic { + indexing_maps = [affine_map<(i, j, k) -> (i, j, k)>, affine_map<(i, j, k) -> (i, j, k)>], + iterator_types = ["parallel", "parallel", "parallel"]} + ins(%input : tensor<2x3x4xf32>) outs(%init : tensor<2x3x4xf32>) { + ^bb0(%element: f32, %output: f32): + %abs = math.absf %element : f32 + linalg.yield %abs : f32 + } -> tensor<2x3x4xf32> + util.return %result : tensor<2x3x4xf32> +}