Adding alignment to vm.buffer.alloc and vm.buffer.clone. iree_vm_buffer_clone isn't as efficient today as there's no way to allocate uninitialized aligned memory but that's ok because we don't yet emit that call anyway. Fixes #9165.
diff --git a/compiler/src/iree/compiler/Dialect/HAL/Conversion/HALToVM/ConvertExecutableOps.cpp b/compiler/src/iree/compiler/Dialect/HAL/Conversion/HALToVM/ConvertExecutableOps.cpp index ad2d270..8034afd 100644 --- a/compiler/src/iree/compiler/Dialect/HAL/Conversion/HALToVM/ConvertExecutableOps.cpp +++ b/compiler/src/iree/compiler/Dialect/HAL/Conversion/HALToVM/ConvertExecutableOps.cpp
@@ -50,7 +50,8 @@ auto constantBuffer = builder.create<IREE::VM::BufferAllocOp>( constantBufferLoc, bufferRefType, builder.create<IREE::VM::ConstI64Op>(constantBufferLoc, - constantCount * sizeof(uint32_t))); + constantCount * sizeof(uint32_t)), + builder.create<IREE::VM::ConstI32Op>(constantBufferLoc, 16)); // Store each constant into it. // TODO(#8477): better ops for this pattern; this creates a lot of
diff --git a/compiler/src/iree/compiler/Dialect/HAL/Conversion/HALToVM/test/executable_ops.mlir b/compiler/src/iree/compiler/Dialect/HAL/Conversion/HALToVM/test/executable_ops.mlir index 89df592..b107b7b 100644 --- a/compiler/src/iree/compiler/Dialect/HAL/Conversion/HALToVM/test/executable_ops.mlir +++ b/compiler/src/iree/compiler/Dialect/HAL/Conversion/HALToVM/test/executable_ops.mlir
@@ -97,7 +97,7 @@ // CHECK-DAG: %[[FORMAT:.+]] = vm.rodata.inline "_utf8_format_ // CHECK-DAG: %[[BINARY:.+]] = vm.const.ref.rodata @exe_binary : !vm.buffer - // CHECK: %[[CONSTANTS:.+]] = vm.buffer.alloc %c12 : !vm.buffer + // CHECK: %[[CONSTANTS:.+]] = vm.buffer.alloc %c12, %c16 : !vm.buffer // CHECK-DAG: %[[INDEX0:.+]] = vm.const.i64 0 // CHECK-DAG: vm.buffer.store.i32 %[[CONSTANT0]], %[[CONSTANTS]][%[[INDEX0]]] : i32 -> !vm.buffer
diff --git a/compiler/src/iree/compiler/Dialect/Util/Conversion/MemRefToUtil/Patterns.cpp b/compiler/src/iree/compiler/Dialect/Util/Conversion/MemRefToUtil/Patterns.cpp index d51d9eb..dfeb8c5 100644 --- a/compiler/src/iree/compiler/Dialect/Util/Conversion/MemRefToUtil/Patterns.cpp +++ b/compiler/src/iree/compiler/Dialect/Util/Conversion/MemRefToUtil/Patterns.cpp
@@ -178,8 +178,10 @@ ConversionPatternRewriter &rewriter) const override { Location loc = allocaOp.getLoc(); auto allocationSize = getByteLength(rewriter, loc, allocaOp.getMemref()); + uint64_t alignment = allocaOp.getAlignment().value_or(0); rewriter.replaceOpWithNewOp<IREE::Util::BufferAllocOp>( - allocaOp, rewriter.getType<IREE::Util::BufferType>(), allocationSize); + allocaOp, rewriter.getType<IREE::Util::BufferType>(), allocationSize, + alignment ? rewriter.getIndexAttr(alignment) : IntegerAttr{}); return success(); } };
diff --git a/compiler/src/iree/compiler/Dialect/VM/Conversion/UtilToVM/ConvertBufferOps.cpp b/compiler/src/iree/compiler/Dialect/VM/Conversion/UtilToVM/ConvertBufferOps.cpp index 4e07f63..b42f327 100644 --- a/compiler/src/iree/compiler/Dialect/VM/Conversion/UtilToVM/ConvertBufferOps.cpp +++ b/compiler/src/iree/compiler/Dialect/VM/Conversion/UtilToVM/ConvertBufferOps.cpp
@@ -52,19 +52,26 @@ } }; +static Value getAlignment(Location loc, std::optional<APInt> alignment, + OpBuilder &builder) { + uint32_t alignmentValue = + alignment.has_value() + ? static_cast<int32_t>(alignment.value().getZExtValue()) + : 0; + return builder.create<IREE::VM::ConstI32Op>(loc, alignmentValue); +} + struct BufferAllocOpConversion : public OpConversionPattern<IREE::Util::BufferAllocOp> { using OpConversionPattern::OpConversionPattern; LogicalResult matchAndRewrite( IREE::Util::BufferAllocOp allocOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override { - // TODO(#9165): support alignment for vm.buffer.alloc. So far we ignore the - // alignment attribute when lowering the op to VM dialect. - (void)adaptor.getAlignment(); auto resultType = getTypeConverter()->convertType(allocOp.getResult().getType()); rewriter.replaceOpWithNewOp<IREE::VM::BufferAllocOp>( - allocOp, resultType, castToI64(adaptor.getStorageSize(), rewriter)); + allocOp, resultType, castToI64(adaptor.getStorageSize(), rewriter), + getAlignment(allocOp.getLoc(), adaptor.getAlignment(), rewriter)); return success(); } }; @@ -91,14 +98,12 @@ LogicalResult matchAndRewrite( IREE::Util::BufferSliceOp sliceOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override { - // TODO(#9165): support alignment for vm.buffer.alloc. So far we ignore the - // alignment attribute when lowering the op to VM dialect. - (void)adaptor.getAlignment(); auto resultType = getTypeConverter()->convertType(sliceOp.getResult().getType()); auto sliceLength = castToI64(adaptor.getResultSize(), rewriter); Value newBuffer = rewriter.create<IREE::VM::BufferAllocOp>( - sliceOp.getLoc(), resultType, sliceLength); + sliceOp.getLoc(), resultType, sliceLength, + getAlignment(sliceOp.getLoc(), adaptor.getAlignment(), rewriter)); Value zero = rewriter.create<IREE::VM::ConstI64ZeroOp>(sliceOp.getLoc()); rewriter.create<IREE::VM::BufferCopyOp>( sliceOp.getLoc(), adaptor.getSource(),
diff --git a/compiler/src/iree/compiler/Dialect/VM/Conversion/UtilToVM/test/buffer_ops.mlir b/compiler/src/iree/compiler/Dialect/VM/Conversion/UtilToVM/test/buffer_ops.mlir index 901ef16..ac7ef7e 100644 --- a/compiler/src/iree/compiler/Dialect/VM/Conversion/UtilToVM/test/buffer_ops.mlir +++ b/compiler/src/iree/compiler/Dialect/VM/Conversion/UtilToVM/test/buffer_ops.mlir
@@ -24,9 +24,21 @@ // CHECK-LABEL: @buffer_alloc func.func @buffer_alloc(%arg0: index) -> !util.buffer { // CHECK-32: %[[SIZE_64:.+]] = vm.ext.i32.i64.u %arg0 : i32 -> i64 - // CHECK-32: %[[BUFFER:.+]] = vm.buffer.alloc %[[SIZE_64]] : !vm.buffer - // CHECK-64: %[[BUFFER:.+]] = vm.buffer.alloc %arg0 : !vm.buffer - %0 = util.buffer.alloc uninitialized {alignment = 16 : index} : !util.buffer{%arg0} + // CHECK-32: %[[BUFFER:.+]] = vm.buffer.alloc %[[SIZE_64]], %zero : !vm.buffer + // CHECK-64: %[[BUFFER:.+]] = vm.buffer.alloc %arg0, %zero : !vm.buffer + %0 = util.buffer.alloc uninitialized : !util.buffer{%arg0} + // CHECK-32: return %[[BUFFER]] + return %0 : !util.buffer +} + +// ----- + +// CHECK-LABEL: @buffer_alloc_aligned +func.func @buffer_alloc_aligned(%arg0: index) -> !util.buffer { + // CHECK-32: %[[SIZE_64:.+]] = vm.ext.i32.i64.u %arg0 : i32 -> i64 + // CHECK-32: %[[BUFFER:.+]] = vm.buffer.alloc %[[SIZE_64]], %c32 : !vm.buffer + // CHECK-64: %[[BUFFER:.+]] = vm.buffer.alloc %arg0, %c32 : !vm.buffer + %0 = util.buffer.alloc uninitialized {alignment = 32 : index} : !util.buffer{%arg0} // CHECK-32: return %[[BUFFER]] return %0 : !util.buffer } @@ -47,11 +59,11 @@ // CHECK-LABEL: @buffer_slice func.func @buffer_slice(%arg0: !util.buffer, %arg1: index, %arg2: index, %arg3: index) -> !util.buffer { // CHECK-32: %[[SIZE_64:.+]] = vm.ext.i32.i64.u %arg3 : i32 -> i64 - // CHECK-32: %[[BUFFER:.+]] = vm.buffer.alloc %[[SIZE_64]] : !vm.buffer + // CHECK-32: %[[BUFFER:.+]] = vm.buffer.alloc %[[SIZE_64]], %c16 : !vm.buffer // CHECK-32-DAG: %[[ZERO:.+]] = vm.const.i64.zero // CHECK-32-DAG: %[[OFFSET_64:.+]] = vm.ext.i32.i64.u %arg1 : i32 -> i64 // CHECK-32: vm.buffer.copy %arg0, %[[OFFSET_64]], %[[BUFFER]], %[[ZERO]], %[[SIZE_64]] : !vm.buffer -> !vm.buffer - // CHECK-64-DAG: %[[BUFFER:.+]] = vm.buffer.alloc %arg3 : !vm.buffer + // CHECK-64-DAG: %[[BUFFER:.+]] = vm.buffer.alloc %arg3, %c16 : !vm.buffer // CHECK-64-DAG: %[[ZERO:.+]] = vm.const.i64.zero // CHECK-64: vm.buffer.copy %arg0, %arg1, %[[BUFFER]], %[[ZERO]], %arg3 : !vm.buffer -> !vm.buffer %0 = util.buffer.slice %arg0[%arg1] {alignment = 16 : index} : !util.buffer{%arg2} -> !util.buffer{%arg3}
diff --git a/compiler/src/iree/compiler/Dialect/VM/Conversion/VMToEmitC/ConvertVMToEmitC.cpp b/compiler/src/iree/compiler/Dialect/VM/Conversion/VMToEmitC/ConvertVMToEmitC.cpp index 2030dd8..2d204f2 100644 --- a/compiler/src/iree/compiler/Dialect/VM/Conversion/VMToEmitC/ConvertVMToEmitC.cpp +++ b/compiler/src/iree/compiler/Dialect/VM/Conversion/VMToEmitC/ConvertVMToEmitC.cpp
@@ -3781,9 +3781,11 @@ "IREE_VM_BUFFER_ACCESS_ORIGIN_GUEST")) .getResult(); Value length = adaptor.getOperands()[0]; + Value alignment = adaptor.getOperands()[1]; result.push_back(access); result.push_back(length); + result.push_back(alignment); result.push_back(allocator); result.push_back(containerPtr); @@ -3832,11 +3834,13 @@ Value offset = adaptor.getOperands()[1]; Value length = adaptor.getOperands()[2]; + Value alignment = adaptor.getOperands()[3]; result.push_back(access); result.push_back(source); result.push_back(offset); result.push_back(length); + result.push_back(alignment); result.push_back(allocator); result.push_back(containerPtr);
diff --git a/compiler/src/iree/compiler/Dialect/VM/Conversion/VMToEmitC/test/buffer_ops.mlir b/compiler/src/iree/compiler/Dialect/VM/Conversion/VMToEmitC/test/buffer_ops.mlir index 1159240..b1407e5 100644 --- a/compiler/src/iree/compiler/Dialect/VM/Conversion/VMToEmitC/test/buffer_ops.mlir +++ b/compiler/src/iree/compiler/Dialect/VM/Conversion/VMToEmitC/test/buffer_ops.mlir
@@ -4,17 +4,19 @@ vm.module @my_module { vm.func @buffer_alloc() { // CHECK: %[[SIZE:.+]] = "emitc.constant"() {value = 128 : i64} : () -> i64 - // CHECK-NEXT: %[[BUFFER:.+]] = "emitc.variable"() {value = #emitc.opaque<"NULL">} : () -> !emitc.ptr<!emitc.opaque<"iree_vm_buffer_t">> - // CHECK-NEXT: %[[BUFFER_PTR:.+]] = emitc.apply "&"(%[[BUFFER]]) : (!emitc.ptr<!emitc.opaque<"iree_vm_buffer_t">>) -> !emitc.ptr<!emitc.ptr<!emitc.opaque<"iree_vm_buffer_t">>> - // CHECK-NEXT: %[[ALLOCTOR:.+]] = emitc.call "EMITC_STRUCT_PTR_MEMBER"(%arg2) {args = [0 : index, #emitc.opaque<"allocator">]} : (!emitc.ptr<!emitc.opaque<"my_module_state_t">>) -> !emitc.opaque<"iree_allocator_t"> - // CHECK-NEXT: %[[BUFFER_ACCESS:.+]] = "emitc.constant"() {value = #emitc.opaque<"IREE_VM_BUFFER_ACCESS_MUTABLE | IREE_VM_BUFFER_ACCESS_ORIGIN_GUEST">} : () -> !emitc.opaque<"iree_vm_buffer_access_t"> - // CHECK-NEXT: %[[STATUS:.+]] = emitc.call "iree_vm_buffer_create"(%[[BUFFER_ACCESS]], %[[SIZE]], %[[ALLOCTOR]], %[[BUFFER_PTR]]) : (!emitc.opaque<"iree_vm_buffer_access_t">, i64, !emitc.opaque<"iree_allocator_t">, !emitc.ptr<!emitc.ptr<!emitc.opaque<"iree_vm_buffer_t">>>) -> !emitc.opaque<"iree_status_t"> + // CHECK-DAG: %[[ALIGNMENT:.+]] = "emitc.constant"() {value = 32 : i32} : () -> i32 + // CHECK-DAG: %[[BUFFER:.+]] = "emitc.variable"() {value = #emitc.opaque<"NULL">} : () -> !emitc.ptr<!emitc.opaque<"iree_vm_buffer_t">> + // CHECK-DAG: %[[BUFFER_PTR:.+]] = emitc.apply "&"(%[[BUFFER]]) : (!emitc.ptr<!emitc.opaque<"iree_vm_buffer_t">>) -> !emitc.ptr<!emitc.ptr<!emitc.opaque<"iree_vm_buffer_t">>> + // CHECK-DAG: %[[ALLOCTOR:.+]] = emitc.call "EMITC_STRUCT_PTR_MEMBER"(%arg2) {args = [0 : index, #emitc.opaque<"allocator">]} : (!emitc.ptr<!emitc.opaque<"my_module_state_t">>) -> !emitc.opaque<"iree_allocator_t"> + // CHECK-DAG: %[[BUFFER_ACCESS:.+]] = "emitc.constant"() {value = #emitc.opaque<"IREE_VM_BUFFER_ACCESS_MUTABLE | IREE_VM_BUFFER_ACCESS_ORIGIN_GUEST">} : () -> !emitc.opaque<"iree_vm_buffer_access_t"> + // CHECK-NEXT: %[[STATUS:.+]] = emitc.call "iree_vm_buffer_create"(%[[BUFFER_ACCESS]], %[[SIZE]], %[[ALIGNMENT]], %[[ALLOCTOR]], %[[BUFFER_PTR]]) : (!emitc.opaque<"iree_vm_buffer_access_t">, i64, i32, !emitc.opaque<"iree_allocator_t">, !emitc.ptr<!emitc.ptr<!emitc.opaque<"iree_vm_buffer_t">>>) -> !emitc.opaque<"iree_status_t"> // CHECK: %[[BUFFER_TYPE_ID:.+]] = emitc.call "iree_vm_buffer_type_id"() : () -> !emitc.opaque<"iree_vm_ref_type_t"> // CHECK-NEXT: %[[STATUS2:.+]] = emitc.call "iree_vm_ref_wrap_assign"(%[[BUFFER]], %[[BUFFER_TYPE_ID]], %1) : (!emitc.ptr<!emitc.opaque<"iree_vm_buffer_t">>, !emitc.opaque<"iree_vm_ref_type_t">, !emitc.ptr<!emitc.opaque<"iree_vm_ref_t">>) -> !emitc.opaque<"iree_status_t"> %c128 = vm.const.i64 128 - %buf = vm.buffer.alloc %c128 : !vm.buffer + %alignment = vm.const.i32 32 + %buf = vm.buffer.alloc %c128, %alignment : !vm.buffer vm.return } } @@ -24,20 +26,22 @@ // CHECK-LABEL: @my_module_buffer_clone vm.module @my_module { vm.func @buffer_clone(%buf : !vm.buffer) { - // CHECK: %[[C0:.+]] = "emitc.constant"() {value = 0 : i64} : () -> i64 - // CHECK-NEXT: %[[C32:.+]] = "emitc.constant"() {value = 32 : i64} : () -> i64 + // CHECK-DAG: %[[C0:.+]] = "emitc.constant"() {value = 0 : i64} : () -> i64 + // CHECK-DAG: %[[C32:.+]] = "emitc.constant"() {value = 32 : i64} : () -> i64 + // CHECK-DAG: %[[ALIGNMENT:.+]] = "emitc.constant"() {value = 64 : i32} : () -> i32 // CHECK: %[[BUFFER:.+]] = "emitc.variable"() {value = #emitc.opaque<"NULL">} : () -> !emitc.ptr<!emitc.opaque<"iree_vm_buffer_t">> - // CHECK-NEXT: %[[BUFFER_PTR:.+]] = emitc.apply "&"(%[[BUFFER]]) : (!emitc.ptr<!emitc.opaque<"iree_vm_buffer_t">>) -> !emitc.ptr<!emitc.ptr<!emitc.opaque<"iree_vm_buffer_t">>> - // CHECK-NEXT: %[[ALLOCATOR:.+]] = emitc.call "EMITC_STRUCT_PTR_MEMBER"(%arg2) {args = [0 : index, #emitc.opaque<"allocator">]} : (!emitc.ptr<!emitc.opaque<"my_module_state_t">>) -> !emitc.opaque<"iree_allocator_t"> - // CHECK-NEXT: %[[BUFFER_ACCESS:.+]] = "emitc.constant"() {value = #emitc.opaque<"IREE_VM_BUFFER_ACCESS_MUTABLE | IREE_VM_BUFFER_ACCESS_ORIGIN_GUEST">} : () -> !emitc.opaque<"iree_vm_buffer_access_t"> - // CHECK-NEXT: %[[BUFFER_REF2:.+]] = emitc.apply "*"(%arg3) : (!emitc.ptr<!emitc.opaque<"iree_vm_ref_t">>) -> !emitc.opaque<"iree_vm_ref_t"> - // CHECK-NEXT: %[[BUFFER_PTR2:.+]] = emitc.call "iree_vm_buffer_deref"(%[[BUFFER_REF2]]) : (!emitc.opaque<"iree_vm_ref_t">) -> !emitc.ptr<!emitc.opaque<"iree_vm_buffer_t">> + // CHECK-DAG: %[[BUFFER_PTR:.+]] = emitc.apply "&"(%[[BUFFER]]) : (!emitc.ptr<!emitc.opaque<"iree_vm_buffer_t">>) -> !emitc.ptr<!emitc.ptr<!emitc.opaque<"iree_vm_buffer_t">>> + // CHECK-DAG: %[[ALLOCATOR:.+]] = emitc.call "EMITC_STRUCT_PTR_MEMBER"(%arg2) {args = [0 : index, #emitc.opaque<"allocator">]} : (!emitc.ptr<!emitc.opaque<"my_module_state_t">>) -> !emitc.opaque<"iree_allocator_t"> + // CHECK-DAG: %[[BUFFER_ACCESS:.+]] = "emitc.constant"() {value = #emitc.opaque<"IREE_VM_BUFFER_ACCESS_MUTABLE | IREE_VM_BUFFER_ACCESS_ORIGIN_GUEST">} : () -> !emitc.opaque<"iree_vm_buffer_access_t"> + // CHECK-DAG: %[[BUFFER_REF2:.+]] = emitc.apply "*"(%arg3) : (!emitc.ptr<!emitc.opaque<"iree_vm_ref_t">>) -> !emitc.opaque<"iree_vm_ref_t"> + // CHECK-DAG: %[[BUFFER_PTR2:.+]] = emitc.call "iree_vm_buffer_deref"(%[[BUFFER_REF2]]) : (!emitc.opaque<"iree_vm_ref_t">) -> !emitc.ptr<!emitc.opaque<"iree_vm_buffer_t">> - // CHECK: %[[STATUS:.+]] = emitc.call "iree_vm_buffer_clone"(%[[BUFFER_ACCESS]], %[[BUFFER_PTR2]], %[[C0]], %[[C32]], %[[ALLOCATOR]], %[[BUFFER_PTR]]) : (!emitc.opaque<"iree_vm_buffer_access_t">, !emitc.ptr<!emitc.opaque<"iree_vm_buffer_t">>, i64, i64, !emitc.opaque<"iree_allocator_t">, !emitc.ptr<!emitc.ptr<!emitc.opaque<"iree_vm_buffer_t">>>) -> !emitc.opaque<"iree_status_t"> + // CHECK: %[[STATUS:.+]] = emitc.call "iree_vm_buffer_clone"(%[[BUFFER_ACCESS]], %[[BUFFER_PTR2]], %[[C0]], %[[C32]], %[[ALIGNMENT]], %[[ALLOCATOR]], %[[BUFFER_PTR]]) : (!emitc.opaque<"iree_vm_buffer_access_t">, !emitc.ptr<!emitc.opaque<"iree_vm_buffer_t">>, i64, i64, i32, !emitc.opaque<"iree_allocator_t">, !emitc.ptr<!emitc.ptr<!emitc.opaque<"iree_vm_buffer_t">>>) -> !emitc.opaque<"iree_status_t"> %c0 = vm.const.i64 0 %c32 = vm.const.i64 32 - %buf_clone = vm.buffer.clone %buf, %c0, %c32 : !vm.buffer -> !vm.buffer + %alignment = vm.const.i32 64 + %buf_clone = vm.buffer.clone %buf, %c0, %c32, %alignment : !vm.buffer -> !vm.buffer vm.return } } @@ -49,7 +53,7 @@ vm.func @buffer_length(%buf : !vm.buffer) { // CHECK: %[[BUFFER_REF:.+]] = emitc.apply "*"(%arg3) : (!emitc.ptr<!emitc.opaque<"iree_vm_ref_t">>) -> !emitc.opaque<"iree_vm_ref_t"> // CHECK-NEXT: %[[BUFFER_PTR:.+]] = emitc.call "iree_vm_buffer_deref"(%[[BUFFER_REF]]) : (!emitc.opaque<"iree_vm_ref_t">) -> !emitc.ptr<!emitc.opaque<"iree_vm_buffer_t">> - + // CHECK: %[[LENGTH:.+]] = emitc.call "iree_vm_buffer_length"(%[[BUFFER_PTR]]) : (!emitc.ptr<!emitc.opaque<"iree_vm_buffer_t">>) -> i64 %length = vm.buffer.length %buf : !vm.buffer -> i64 @@ -67,7 +71,7 @@ // CHECK: %[[BUFFER_REF:.+]] = emitc.apply "*"(%arg3) : (!emitc.ptr<!emitc.opaque<"iree_vm_ref_t">>) -> !emitc.opaque<"iree_vm_ref_t"> // CHECK-NEXT: %[[BUFFER_PTR:.+]] = emitc.call "iree_vm_buffer_deref"(%[[BUFFER_REF]]) : (!emitc.opaque<"iree_vm_ref_t">) -> !emitc.ptr<!emitc.opaque<"iree_vm_buffer_t">> - + // CHECK: %[[BUFFER_REF2:.+]] = emitc.apply "*"(%arg4) : (!emitc.ptr<!emitc.opaque<"iree_vm_ref_t">>) -> !emitc.opaque<"iree_vm_ref_t"> // CHECK-NEXT: %[[BUFFER_PTR2:.+]] = emitc.call "iree_vm_buffer_deref"(%[[BUFFER_REF2]]) : (!emitc.opaque<"iree_vm_ref_t">) -> !emitc.ptr<!emitc.opaque<"iree_vm_buffer_t">> @@ -91,7 +95,7 @@ // CHECK: %[[BUFFER_REF:.+]] = emitc.apply "*"(%arg3) : (!emitc.ptr<!emitc.opaque<"iree_vm_ref_t">>) -> !emitc.opaque<"iree_vm_ref_t"> // CHECK-NEXT: %[[BUFFER_PTR:.+]] = emitc.call "iree_vm_buffer_deref"(%[[BUFFER_REF]]) : (!emitc.opaque<"iree_vm_ref_t">) -> !emitc.ptr<!emitc.opaque<"iree_vm_buffer_t">> - + // CHECK: %[[BUFFER_REF2:.+]] = emitc.apply "*"(%arg4) : (!emitc.ptr<!emitc.opaque<"iree_vm_ref_t">>) -> !emitc.opaque<"iree_vm_ref_t"> // CHECK-NEXT: %[[BUFFER_PTR2:.+]] = emitc.call "iree_vm_buffer_deref"(%[[BUFFER_REF2]]) : (!emitc.opaque<"iree_vm_ref_t">) -> !emitc.ptr<!emitc.opaque<"iree_vm_buffer_t">> @@ -114,7 +118,7 @@ // CHECK: %[[BUFFER_REF:.+]] = emitc.apply "*"(%arg3) : (!emitc.ptr<!emitc.opaque<"iree_vm_ref_t">>) -> !emitc.opaque<"iree_vm_ref_t"> // CHECK-NEXT: %[[BUFFER_PTR:.+]] = emitc.call "iree_vm_buffer_deref"(%[[BUFFER_REF]]) : (!emitc.opaque<"iree_vm_ref_t">) -> !emitc.ptr<!emitc.opaque<"iree_vm_buffer_t">> - + // CHECK: %[[STATUS:.+]] = emitc.call "vm_buffer_fill_i8"(%[[BUFFER_PTR]], %[[C0]], %[[C16]], %[[C102]]) : (!emitc.ptr<!emitc.opaque<"iree_vm_buffer_t">>, i64, i64, i32) -> !emitc.opaque<"iree_status_t"> %c0 = vm.const.i64 0 %c16 = vm.const.i64 16 @@ -153,11 +157,11 @@ // CHECK: %[[BUFFER_REF:.+]] = emitc.apply "*"(%arg3) : (!emitc.ptr<!emitc.opaque<"iree_vm_ref_t">>) -> !emitc.opaque<"iree_vm_ref_t"> // CHECK-NEXT: %[[BUFFER_PTR:.+]] = emitc.call "iree_vm_buffer_deref"(%[[BUFFER_REF]]) : (!emitc.opaque<"iree_vm_ref_t">) -> !emitc.ptr<!emitc.opaque<"iree_vm_buffer_t">> - + // CHECK: %[[RESULT:.+]] = "emitc.variable"() {value = #emitc.opaque<"">} : () -> i32 // CHECK-NEXT: %[[RESULT_PTR:.+]] = emitc.apply "&"(%6) : (i32) -> !emitc.ptr<i32> // CHECK-NEXT: %[[STATUS:.+]] = emitc.call "vm_buffer_load_i8s"(%[[BUFFER_PTR]], %[[C0]], %[[RESULT_PTR]]) : (!emitc.ptr<!emitc.opaque<"iree_vm_buffer_t">>, i64, !emitc.ptr<i32>) -> !emitc.opaque<"iree_status_t"> - + %c0 = vm.const.i64 0 %v0 = vm.buffer.load.i8.s %buf[%c0] : !vm.buffer -> i32 vm.return @@ -206,7 +210,7 @@ // CHECK: %[[BUFFER_REF:.+]] = emitc.apply "*"(%arg3) : (!emitc.ptr<!emitc.opaque<"iree_vm_ref_t">>) -> !emitc.opaque<"iree_vm_ref_t"> // CHECK-NEXT: %[[BUFFER_PTR:.+]] = emitc.call "iree_vm_buffer_deref"(%[[BUFFER_REF]]) : (!emitc.opaque<"iree_vm_ref_t">) -> !emitc.ptr<!emitc.opaque<"iree_vm_buffer_t">> - + // CHECK: %[[STATUS:.+]] = emitc.call "vm_buffer_store_i8"(%[[BUFFER_PTR]], %[[C0]], %[[C102]]) : (!emitc.ptr<!emitc.opaque<"iree_vm_buffer_t">>, i64, i32) -> !emitc.opaque<"iree_status_t"> %c0 = vm.const.i64 0 %c102 = vm.const.i32 102
diff --git a/compiler/src/iree/compiler/Dialect/VM/IR/VMOps.td b/compiler/src/iree/compiler/Dialect/VM/IR/VMOps.td index f008f56..d7d8425 100644 --- a/compiler/src/iree/compiler/Dialect/VM/IR/VMOps.td +++ b/compiler/src/iree/compiler/Dialect/VM/IR/VMOps.td
@@ -1151,7 +1151,8 @@ }]; let arguments = (ins - VM_BufferIndex:$length + VM_BufferIndex:$length, + I32:$alignment ); let results = (outs VM_RefOf<VM_BufferType>:$result @@ -1164,6 +1165,7 @@ let encoding = [ VM_EncOpcode<VM_OPC_BufferAlloc>, VM_EncOperand<"length", 0>, + VM_EncOperand<"alignment", 1>, VM_EncResult<"result">, ]; } @@ -1183,7 +1185,8 @@ let arguments = (ins VM_RefOf<VM_BufferType>:$source_buffer, VM_BufferIndex:$source_offset, - VM_BufferIndex:$length + VM_BufferIndex:$length, + I32:$alignment ); let results = (outs VM_RefOf<VM_BufferType>:$result @@ -1198,6 +1201,7 @@ VM_EncOperand<"source_buffer", 0>, VM_EncOperand<"source_offset", 1>, VM_EncOperand<"length", 2>, + VM_EncOperand<"alignment", 3>, VM_EncResult<"result">, ]; }
diff --git a/runtime/bindings/python/tests/vm_types_test.py b/runtime/bindings/python/tests/vm_types_test.py index 891f647..782142e 100644 --- a/runtime/bindings/python/tests/vm_types_test.py +++ b/runtime/bindings/python/tests/vm_types_test.py
@@ -82,14 +82,14 @@ lst1.get_as_list(1) def test_vm_buffer(self): - b1 = rt.VmBuffer(10, mutable=True) + b1 = rt.VmBuffer(10, alignment=0, mutable=True) print(b1) contents = memoryview(b1) contents[0:] = b'0123456789' self.assertEqual(bytes(b1), b'0123456789') def test_vm_buffer_ro(self): - b1 = rt.VmBuffer(10, mutable=False) + b1 = rt.VmBuffer(10, alignment=16, mutable=False) contents = memoryview(b1) with self.assertRaises(TypeError): contents[0:] = b'0123456789'
diff --git a/runtime/bindings/python/vm.cc b/runtime/bindings/python/vm.cc index b550ce4..3e32df9 100644 --- a/runtime/bindings/python/vm.cc +++ b/runtime/bindings/python/vm.cc
@@ -532,19 +532,21 @@ iree_vm_buffer_retain_ref, iree_vm_buffer_deref, iree_vm_buffer_isa); vm_buffer - .def(py::init([](iree_host_size_t length, bool is_mutable) { + .def(py::init([](iree_host_size_t length, iree_host_size_t alignment, + bool is_mutable) { iree_vm_buffer_access_t access = 0; if (is_mutable) { access |= IREE_VM_BUFFER_ACCESS_MUTABLE; } iree_vm_buffer_t* raw_buffer; CheckApiStatus( - iree_vm_buffer_create(access, length, iree_allocator_system(), - &raw_buffer), + iree_vm_buffer_create(access, length, alignment, + iree_allocator_system(), &raw_buffer), "Error creating buffer"); return VmBuffer::StealFromRawPtr(raw_buffer); }), - py::arg("length"), py::arg("mutable") = true) + py::arg("length"), py::arg("alignment") = 0, + py::arg("mutable") = true) .def_buffer([](VmBuffer& self) -> py::buffer_info { return py::buffer_info( /*ptr=*/self.raw_ptr()->data.data,
diff --git a/runtime/src/iree/base/alignment.h b/runtime/src/iree/base/alignment.h index a1179f1..62e830c 100644 --- a/runtime/src/iree/base/alignment.h +++ b/runtime/src/iree/base/alignment.h
@@ -74,6 +74,11 @@ return (value + (alignment - 1)) & ~(alignment - 1); } +// Returns true if |value| is a power-of-two. +static inline bool iree_host_size_is_power_of_two(iree_host_size_t value) { + return (value != 0) && ((value & (value - 1)) == 0); +} + // Returns true if |value| matches the given minimum |alignment|. static inline bool iree_host_size_has_alignment(iree_host_size_t value, iree_host_size_t alignment) { @@ -87,6 +92,11 @@ return (value + (alignment - 1)) & ~(alignment - 1); } +// Returns true if |value| is a power-of-two. +static inline bool iree_device_size_is_power_of_two(iree_device_size_t value) { + return (value != 0) && ((value & (value - 1)) == 0); +} + // Returns true if |value| matches the given minimum |alignment|. static inline bool iree_device_size_has_alignment( iree_device_size_t value, iree_device_size_t alignment) {
diff --git a/runtime/src/iree/base/allocator.c b/runtime/src/iree/base/allocator.c index d409370..22b56cf 100644 --- a/runtime/src/iree/base/allocator.c +++ b/runtime/src/iree/base/allocator.c
@@ -137,12 +137,6 @@ // Aligned allocations via iree_allocator_t //===----------------------------------------------------------------------===// -// Returns true if |alignment| is a power of two (or 0). -static inline iree_host_size_t iree_alignment_is_pot( - iree_host_size_t alignment) { - return (alignment & (alignment - 1)) == 0; -} - // Returns a pointer into |unaligned_ptr| where |offset| matches |alignment|. static inline void* iree_aligned_ptr(void* unaligned_ptr, iree_host_size_t alignment, @@ -177,7 +171,7 @@ "allocations must be >0 bytes"); } const iree_host_size_t alignment = iree_max(min_alignment, iree_max_align_t); - if (IREE_UNLIKELY(!iree_alignment_is_pot(alignment))) { + if (IREE_UNLIKELY(!iree_host_size_is_power_of_two(alignment))) { return iree_make_status( IREE_STATUS_INVALID_ARGUMENT, "alignments must be powers of two (got %" PRIhsz ")", min_alignment); @@ -209,7 +203,7 @@ "allocations must be >0 bytes"); } const iree_host_size_t alignment = iree_min(min_alignment, iree_max_align_t); - if (IREE_UNLIKELY(!iree_alignment_is_pot(alignment))) { + if (IREE_UNLIKELY(!iree_host_size_is_power_of_two(alignment))) { return iree_make_status( IREE_STATUS_INVALID_ARGUMENT, "alignments must be powers of two (got %" PRIhsz ")", min_alignment);
diff --git a/runtime/src/iree/vm/buffer.c b/runtime/src/iree/vm/buffer.c index 0cb2974..5018331 100644 --- a/runtime/src/iree/vm/buffer.c +++ b/runtime/src/iree/vm/buffer.c
@@ -80,9 +80,10 @@ iree_allocator_free(buffer->allocator, buffer->data.data); } -IREE_API_EXPORT iree_status_t iree_vm_buffer_create( - iree_vm_buffer_access_t access, iree_host_size_t length, - iree_allocator_t allocator, iree_vm_buffer_t** out_buffer) { +IREE_API_EXPORT iree_status_t +iree_vm_buffer_create(iree_vm_buffer_access_t access, iree_host_size_t length, + iree_host_size_t alignment, iree_allocator_t allocator, + iree_vm_buffer_t** out_buffer) { IREE_ASSERT_ARGUMENT(out_buffer); *out_buffer = NULL; IREE_TRACE_ZONE_BEGIN(z0); @@ -95,7 +96,8 @@ // Allocate combined [prefix | buffer] memory. uint8_t* data_ptr = NULL; IREE_RETURN_AND_END_ZONE_IF_ERROR( - z0, iree_allocator_malloc(allocator, total_size, (void**)&data_ptr)); + z0, iree_allocator_malloc_aligned(allocator, total_size, alignment, + prefix_size, (void**)&data_ptr)); // Initialize the prefix buffer handle. iree_vm_buffer_t* buffer = (iree_vm_buffer_t*)data_ptr; @@ -115,7 +117,7 @@ // Buffers are stored as [prefix | data]; freeing the prefix is all we need // to do to free it all. iree_vm_buffer_t* buffer = (iree_vm_buffer_t*)ptr; - iree_allocator_free(buffer->allocator, buffer); + iree_allocator_free_aligned(buffer->allocator, buffer); IREE_TRACE_ZONE_END(z0); } @@ -131,7 +133,8 @@ IREE_API_EXPORT iree_status_t iree_vm_buffer_clone( iree_vm_buffer_access_t access, const iree_vm_buffer_t* source_buffer, iree_host_size_t source_offset, iree_host_size_t length, - iree_allocator_t allocator, iree_vm_buffer_t** out_buffer) { + iree_host_size_t alignment, iree_allocator_t allocator, + iree_vm_buffer_t** out_buffer) { IREE_ASSERT_ARGUMENT(source_buffer); IREE_ASSERT_ARGUMENT(out_buffer); *out_buffer = NULL; @@ -151,12 +154,11 @@ iree_host_size_t total_size = prefix_size + source_span.data_length; // Allocate combined [prefix | buffer] memory. - // NOTE: we are allocating without initialization here as we will be writing - // over all of it. + // TODO(benvanik): uninitialized allocation here to avoid the memset(0). uint8_t* data_ptr = NULL; IREE_RETURN_AND_END_ZONE_IF_ERROR( - z0, iree_allocator_malloc_uninitialized(allocator, total_size, - (void**)&data_ptr)); + z0, iree_allocator_malloc_aligned(allocator, total_size, alignment, + prefix_size, (void**)&data_ptr)); // Initialize the prefix buffer handle. iree_vm_buffer_t* buffer = (iree_vm_buffer_t*)data_ptr;
diff --git a/runtime/src/iree/vm/buffer.h b/runtime/src/iree/vm/buffer.h index 250eb65..4881ab8 100644 --- a/runtime/src/iree/vm/buffer.h +++ b/runtime/src/iree/vm/buffer.h
@@ -93,13 +93,14 @@ // Creates a new zero-initialized buffer of the given byte |length|. // The underlying storage buffer may be allocated larger to ensure alignment. -// The allocated data will be aligned to iree_max_align_t. +// The allocated data will be aligned to |alignment| or iree_max_align_t if 0. // // |access| can be used to control who (guest, host, etc) and how (read/write) // the buffer may be accessed. -IREE_API_EXPORT iree_status_t iree_vm_buffer_create( - iree_vm_buffer_access_t access, iree_host_size_t length, - iree_allocator_t allocator, iree_vm_buffer_t** out_buffer); +IREE_API_EXPORT iree_status_t +iree_vm_buffer_create(iree_vm_buffer_access_t access, iree_host_size_t length, + iree_host_size_t alignment, iree_allocator_t allocator, + iree_vm_buffer_t** out_buffer); // Retains the given |buffer| for the caller. IREE_API_EXPORT void iree_vm_buffer_retain(iree_vm_buffer_t* buffer); @@ -108,7 +109,7 @@ IREE_API_EXPORT void iree_vm_buffer_release(iree_vm_buffer_t* buffer); // Clones a range of bytes in |source| to a new buffer. -// The allocated data will be aligned to iree_max_align_t. +// The allocated data will be aligned to |alignment| or iree_max_align_t if 0. // // |access| can be used to control who (guest, host, etc) and how (read/write) // the buffer may be accessed. As this returns a newly allocated buffer the @@ -116,7 +117,8 @@ IREE_API_EXPORT iree_status_t iree_vm_buffer_clone( iree_vm_buffer_access_t access, const iree_vm_buffer_t* source_buffer, iree_host_size_t source_offset, iree_host_size_t length, - iree_allocator_t allocator, iree_vm_buffer_t** out_buffer); + iree_host_size_t alignment, iree_allocator_t allocator, + iree_vm_buffer_t** out_buffer); // Returns the user-visible length of the buffer in bytes. IREE_API_EXPORT iree_host_size_t
diff --git a/runtime/src/iree/vm/bytecode/disassembler.c b/runtime/src/iree/vm/bytecode/disassembler.c index 1c7bce8..3bdb3f4 100644 --- a/runtime/src/iree/vm/bytecode/disassembler.c +++ b/runtime/src/iree/vm/bytecode/disassembler.c
@@ -650,6 +650,7 @@ DISASM_OP(CORE, BufferAlloc) { uint16_t length_reg = VM_ParseOperandRegI64("length"); + uint16_t alignment_reg = VM_ParseOperandRegI32("alignment"); bool result_is_move; uint16_t result_reg = VM_ParseResultRegRef("result", &result_is_move); EMIT_REF_REG_NAME(result_reg); @@ -657,6 +658,9 @@ iree_string_builder_append_cstring(b, " = vm.buffer.alloc ")); EMIT_I64_REG_NAME(length_reg); EMIT_OPTIONAL_VALUE_I64(regs->i32[length_reg]); + IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", ")); + EMIT_I32_REG_NAME(alignment_reg); + EMIT_OPTIONAL_VALUE_I32(regs->i32[alignment_reg]); break; } @@ -665,6 +669,7 @@ uint16_t source_reg = VM_ParseOperandRegRef("source", &source_is_move); uint16_t offset_reg = VM_ParseOperandRegI64("offset"); uint16_t length_reg = VM_ParseOperandRegI64("length"); + uint16_t alignment_reg = VM_ParseOperandRegI32("alignment"); bool result_is_move; uint16_t result_reg = VM_ParseResultRegRef("result", &result_is_move); EMIT_REF_REG_NAME(result_reg); @@ -678,6 +683,9 @@ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", ")); EMIT_I64_REG_NAME(length_reg); EMIT_OPTIONAL_VALUE_I64(regs->i32[length_reg]); + IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", ")); + EMIT_I32_REG_NAME(alignment_reg); + EMIT_OPTIONAL_VALUE_I32(regs->i32[alignment_reg]); break; }
diff --git a/runtime/src/iree/vm/bytecode/dispatch.c b/runtime/src/iree/vm/bytecode/dispatch.c index ff1744c..63b4fac 100644 --- a/runtime/src/iree/vm/bytecode/dispatch.c +++ b/runtime/src/iree/vm/bytecode/dispatch.c
@@ -881,12 +881,13 @@ DISPATCH_OP(CORE, BufferAlloc, { iree_host_size_t length = VM_DecOperandRegI64HostSize("length"); + iree_host_size_t alignment = VM_DecOperandRegI32("alignment"); bool result_is_move; iree_vm_ref_t* result_ref = VM_DecResultRegRef("result", &result_is_move); iree_vm_buffer_t* buffer = NULL; IREE_RETURN_IF_ERROR(iree_vm_buffer_create( IREE_VM_BUFFER_ACCESS_MUTABLE | IREE_VM_BUFFER_ACCESS_ORIGIN_GUEST, - length, module_state->allocator, &buffer)); + length, alignment, module_state->allocator, &buffer)); IREE_RETURN_IF_ERROR(iree_vm_ref_wrap_assign( buffer, iree_vm_buffer_type_id(), result_ref)); }); @@ -901,12 +902,13 @@ } iree_host_size_t offset = VM_DecOperandRegI64HostSize("offset"); iree_host_size_t length = VM_DecOperandRegI64HostSize("length"); + iree_host_size_t alignment = VM_DecOperandRegI32("alignment"); bool result_is_move; iree_vm_ref_t* result_ref = VM_DecResultRegRef("result", &result_is_move); iree_vm_buffer_t* result = NULL; IREE_RETURN_IF_ERROR(iree_vm_buffer_clone( IREE_VM_BUFFER_ACCESS_MUTABLE | IREE_VM_BUFFER_ACCESS_ORIGIN_GUEST, - source, offset, length, module_state->allocator, &result)); + source, offset, length, alignment, module_state->allocator, &result)); IREE_RETURN_IF_ERROR(iree_vm_ref_wrap_assign( result, iree_vm_buffer_type_id(), result_ref)); });
diff --git a/runtime/src/iree/vm/bytecode/module_benchmark.mlir b/runtime/src/iree/vm/bytecode/module_benchmark.mlir index a2b8c85..39f122c 100644 --- a/runtime/src/iree/vm/bytecode/module_benchmark.mlir +++ b/runtime/src/iree/vm/bytecode/module_benchmark.mlir
@@ -87,7 +87,8 @@ %c4 = vm.const.i64 4 %count_i64 = vm.ext.i32.i64.u %count : i32 -> i64 %count_bytes = vm.mul.i64 %count_i64, %c4 : i64 - %buf = vm.buffer.alloc %count_bytes : !vm.buffer + %alignment = vm.const.i32 16 + %buf = vm.buffer.alloc %count_bytes, %alignment : !vm.buffer vm.buffer.fill.i32 %buf, %c0, %count_i64, %pattern : i32 -> !vm.buffer vm.br ^loop(%c0, %c0_i32 : i64, i32) ^loop(%i : i64, %sum : i32): @@ -110,7 +111,8 @@ %c4 = vm.const.i64 4 %count_i64 = vm.ext.i32.i64.u %count : i32 -> i64 %count_bytes = vm.mul.i64 %count_i64, %c4 : i64 - %buf = vm.buffer.alloc %count_bytes : !vm.buffer + %alignment = vm.const.i32 16 + %buf = vm.buffer.alloc %count_bytes, %alignment : !vm.buffer vm.buffer.fill.i32 %buf, %c0, %count_i64, %pattern : i32 -> !vm.buffer %sum_init = vm.const.i32.zero vm.br ^loop(%c0, %sum_init : i64, i32)
diff --git a/runtime/src/iree/vm/bytecode/verifier.c b/runtime/src/iree/vm/bytecode/verifier.c index dfcf9ff..26c7d15 100644 --- a/runtime/src/iree/vm/bytecode/verifier.c +++ b/runtime/src/iree/vm/bytecode/verifier.c
@@ -1158,6 +1158,7 @@ VERIFY_OP(CORE, BufferAlloc, { VM_VerifyOperandRegI64HostSize(length); + VM_VerifyOperandRegI32(alignment); VM_VerifyResultRegRef(result); }); @@ -1165,6 +1166,7 @@ VM_VerifyOperandRegRef(source); VM_VerifyOperandRegI64HostSize(offset); VM_VerifyOperandRegI64HostSize(length); + VM_VerifyOperandRegI32(alignment); VM_VerifyResultRegRef(result); });
diff --git a/runtime/src/iree/vm/test/buffer_ops.mlir b/runtime/src/iree/vm/test/buffer_ops.mlir index ddbf648..84a7517 100644 --- a/runtime/src/iree/vm/test/buffer_ops.mlir +++ b/runtime/src/iree/vm/test/buffer_ops.mlir
@@ -57,7 +57,8 @@ vm.export @test_alloc vm.func @test_alloc() { %c128 = vm.const.i64 128 - %buf = vm.buffer.alloc %c128 : !vm.buffer + %alignment = vm.const.i32 16 + %buf = vm.buffer.alloc %c128, %alignment : !vm.buffer %buf_dno = util.optimization_barrier %buf : !vm.buffer vm.check.nz %buf_dno, "!null" : !vm.buffer @@ -71,7 +72,8 @@ vm.export @test_alloc_empty vm.func @test_alloc_empty() { %c0 = vm.const.i64 0 - %buf = vm.buffer.alloc %c0 : !vm.buffer + %alignment = vm.const.i32 16 + %buf = vm.buffer.alloc %c0, %alignment : !vm.buffer %buf_dno = util.optimization_barrier %buf : !vm.buffer vm.check.nz %buf_dno, "!null" : !vm.buffer @@ -94,7 +96,8 @@ // Clone the last two 32-bit elements. %c4 = vm.const.i64 4 %c8 = vm.const.i64 8 - %buf = vm.buffer.clone %rodata, %c4, %c8 : !vm.buffer -> !vm.buffer + %alignment = vm.const.i32 16 + %buf = vm.buffer.clone %rodata, %c4, %c8, %alignment : !vm.buffer -> !vm.buffer %buf_dno = util.optimization_barrier %buf : !vm.buffer vm.check.nz %buf_dno, "!null" : !vm.buffer @@ -111,14 +114,15 @@ vm.func @test_clone_empty() { // Allocate source zero-length buffer. %c0 = vm.const.i64 0 - %buf0 = vm.buffer.alloc %c0 : !vm.buffer + %alignment = vm.const.i32 16 + %buf0 = vm.buffer.alloc %c0, %alignment : !vm.buffer %buf0_dno = util.optimization_barrier %buf0 : !vm.buffer vm.check.nz %buf0_dno, "!null" : !vm.buffer %buf0_length = vm.buffer.length %buf0_dno : !vm.buffer -> i64 vm.check.eq %c0, %buf0_length, "buffer length == 0" : i64 // Clone it all (or, clone nothing?). - %buf1 = vm.buffer.clone %buf0_dno, %c0, %c0 : !vm.buffer -> !vm.buffer + %buf1 = vm.buffer.clone %buf0_dno, %c0, %c0, %alignment : !vm.buffer -> !vm.buffer %buf1_dno = util.optimization_barrier %buf1 : !vm.buffer vm.check.nz %buf1_dno, "!null" : !vm.buffer %buf1_length = vm.buffer.length %buf1_dno : !vm.buffer -> i64 @@ -137,7 +141,8 @@ // Try to clone off the end of the buffer. %c8 = vm.const.i64 8 - %buf = vm.buffer.clone %rodata, %c8, %c8 : !vm.buffer -> !vm.buffer + %alignment = vm.const.i32 16 + %buf = vm.buffer.clone %rodata, %c8, %c8, %alignment : !vm.buffer -> !vm.buffer vm.return } @@ -155,7 +160,8 @@ vm.check.nz %rodata, "!null" : !vm.buffer // Allocate target buffer. - %buf = vm.buffer.alloc %rodata_length : !vm.buffer + %alignment = vm.const.i32 16 + %buf = vm.buffer.alloc %rodata_length, %alignment : !vm.buffer %buf_dno = util.optimization_barrier %buf : !vm.buffer vm.check.nz %buf_dno, "!null" : !vm.buffer @@ -177,7 +183,8 @@ vm.func @test_copy_partial() { // Allocate target buffer. %c4 = vm.const.i64 4 - %buf = vm.buffer.alloc %c4 : !vm.buffer + %alignment = vm.const.i32 16 + %buf = vm.buffer.alloc %c4, %alignment : !vm.buffer %buf_dno = util.optimization_barrier %buf : !vm.buffer vm.check.nz %buf_dno, "!null" : !vm.buffer @@ -199,7 +206,8 @@ vm.func @fail_copy_out_of_range_source_offset() { %rodata = vm.const.ref.rodata @rodata_3xi32 : !vm.buffer %c128 = vm.const.i64 128 - %buf = vm.buffer.alloc %c128 : !vm.buffer + %alignment = vm.const.i32 16 + %buf = vm.buffer.alloc %c128, %alignment : !vm.buffer %buf_dno = util.optimization_barrier %buf : !vm.buffer vm.check.nz %buf_dno, "!null" : !vm.buffer @@ -215,7 +223,8 @@ vm.func @fail_copy_out_of_range_source_length() { %rodata = vm.const.ref.rodata @rodata_3xi32 : !vm.buffer %c128 = vm.const.i64 128 - %buf = vm.buffer.alloc %c128 : !vm.buffer + %alignment = vm.const.i32 16 + %buf = vm.buffer.alloc %c128, %alignment : !vm.buffer %buf_dno = util.optimization_barrier %buf : !vm.buffer vm.check.nz %buf_dno, "!null" : !vm.buffer @@ -233,7 +242,8 @@ %rodata = vm.const.ref.rodata @rodata_3xi32 : !vm.buffer %rodata_length = vm.buffer.length %rodata : !vm.buffer -> i64 %c8 = vm.const.i64 8 - %buf = vm.buffer.alloc %c8 : !vm.buffer + %alignment = vm.const.i32 16 + %buf = vm.buffer.alloc %c8, %alignment : !vm.buffer %buf_dno = util.optimization_barrier %buf : !vm.buffer vm.check.nz %buf_dno, "!null" : !vm.buffer @@ -249,7 +259,8 @@ vm.func @fail_copy_out_of_range_target_length() { %rodata = vm.const.ref.rodata @rodata_3xi32 : !vm.buffer %c8 = vm.const.i64 8 - %buf = vm.buffer.alloc %c8 : !vm.buffer + %alignment = vm.const.i32 16 + %buf = vm.buffer.alloc %c8, %alignment : !vm.buffer %buf_dno = util.optimization_barrier %buf : !vm.buffer vm.check.nz %buf_dno, "!null" : !vm.buffer @@ -273,7 +284,8 @@ %element_size = vm.const.i64 4 %num_elements = vm.const.i64 4 %buffer_size = vm.mul.i64 %num_elements, %element_size : i64 - %buf = vm.buffer.alloc %buffer_size : !vm.buffer + %alignment = vm.const.i32 16 + %buf = vm.buffer.alloc %buffer_size, %alignment : !vm.buffer %buf_dno = util.optimization_barrier %buf : !vm.buffer vm.check.nz %buf_dno, "!null" : !vm.buffer @@ -301,7 +313,8 @@ %element_size = vm.const.i64 1 %num_elements = vm.const.i64 4 %buffer_size = vm.mul.i64 %num_elements, %element_size : i64 - %buf = vm.buffer.alloc %buffer_size : !vm.buffer + %alignment = vm.const.i32 16 + %buf = vm.buffer.alloc %buffer_size, %alignment : !vm.buffer %buf_dno = util.optimization_barrier %buf : !vm.buffer vm.check.nz %buf_dno, "!null" : !vm.buffer @@ -329,7 +342,8 @@ %element_size = vm.const.i64 2 %num_elements = vm.const.i64 4 %buffer_size = vm.mul.i64 %num_elements, %element_size : i64 - %buf = vm.buffer.alloc %buffer_size : !vm.buffer + %alignment = vm.const.i32 16 + %buf = vm.buffer.alloc %buffer_size, %alignment : !vm.buffer %buf_dno = util.optimization_barrier %buf : !vm.buffer vm.check.nz %buf_dno, "!null" : !vm.buffer @@ -357,7 +371,8 @@ %element_size = vm.const.i64 4 %num_elements = vm.const.i64 4 %buffer_size = vm.mul.i64 %num_elements, %element_size : i64 - %buf = vm.buffer.alloc %buffer_size : !vm.buffer + %alignment = vm.const.i32 16 + %buf = vm.buffer.alloc %buffer_size, %alignment : !vm.buffer %buf_dno = util.optimization_barrier %buf : !vm.buffer vm.check.nz %buf_dno, "!null" : !vm.buffer @@ -385,7 +400,8 @@ %element_size = vm.const.i64 8 %num_elements = vm.const.i64 4 %buffer_size = vm.mul.i64 %num_elements, %element_size : i64 - %buf = vm.buffer.alloc %buffer_size : !vm.buffer + %alignment = vm.const.i32 16 + %buf = vm.buffer.alloc %buffer_size, %alignment : !vm.buffer %buf_dno = util.optimization_barrier %buf : !vm.buffer vm.check.nz %buf_dno, "!null" : !vm.buffer @@ -570,7 +586,8 @@ %ref_dno = util.optimization_barrier %ref : !vm.buffer %ref_length = vm.buffer.length %ref_dno : !vm.buffer -> i64 - %buf = vm.buffer.alloc %ref_length : !vm.buffer + %alignment = vm.const.i32 16 + %buf = vm.buffer.alloc %ref_length, %alignment : !vm.buffer %buf_dno = util.optimization_barrier %buf : !vm.buffer %c0 = vm.const.i64 0 @@ -603,7 +620,8 @@ %ref_dno = util.optimization_barrier %ref : !vm.buffer %ref_length = vm.buffer.length %ref_dno : !vm.buffer -> i64 - %buf = vm.buffer.alloc %ref_length : !vm.buffer + %alignment = vm.const.i32 16 + %buf = vm.buffer.alloc %ref_length, %alignment : !vm.buffer %buf_dno = util.optimization_barrier %buf : !vm.buffer %c0 = vm.const.i64 0 @@ -636,7 +654,8 @@ %ref_dno = util.optimization_barrier %ref : !vm.buffer %ref_length = vm.buffer.length %ref_dno : !vm.buffer -> i64 - %buf = vm.buffer.alloc %ref_length : !vm.buffer + %alignment = vm.const.i32 16 + %buf = vm.buffer.alloc %ref_length, %alignment : !vm.buffer %buf_dno = util.optimization_barrier %buf : !vm.buffer %c0 = vm.const.i64 0