Fix the ukernels system build on older toolchains. (#14155)
In the process of adding the ukernels bitcode build, we dropped all
cmake configure-check for toolchain support for CPU-feature-enabling
flags, and configured headers. I didn't properly think through that:
that worked essentially because no one had tried building with an older
toolchain. On x86-64, that was OK because we didn't use any recent flag.
but on ARM that was more problematic with the `+i8mm` target feature.
@freddan80 ran into this on
https://github.com/openxla/iree/issues/12684.
So this brings back configure-checks and configured-headers, but only
where they are specifically needed and not interfering with the bitcode
build --- only in the arch/ subdirs and only in the system-build. Some
`#if defined(IREE_DEVICE_STANDALONE)` lets the bitcode build opt out of
including the configured headers.
This has a couple of side benefits. We get to drop the clumsy `#if`'s
trying to do version checks on compiler version tokens, and we get to
conditionally add those feature-specific dependencies instead of having
those `#if`'s around the entire files, which confuse syntax highlighting
when the feature token is not defined in the IDE build.
diff --git a/runtime/src/iree/builtins/ukernel/arch/arm_64/CMakeLists.txt b/runtime/src/iree/builtins/ukernel/arch/arm_64/CMakeLists.txt
index dba2b2e..138bece 100644
--- a/runtime/src/iree/builtins/ukernel/arch/arm_64/CMakeLists.txt
+++ b/runtime/src/iree/builtins/ukernel/arch/arm_64/CMakeLists.txt
@@ -77,6 +77,10 @@
"-march=armv8.2-a+i8mm"
)
+check_cxx_compiler_flag("${IREE_UK_COPTS_ARM_64_DOTPROD}" IREE_UK_BUILD_ARM_64_DOTPROD)
+check_cxx_compiler_flag("${IREE_UK_COPTS_ARM_64_I8MM}" IREE_UK_BUILD_ARM_64_I8MM)
+configure_file("config_arm_64.h.in" "config_arm_64.h")
+
iree_cc_library(
NAME
common_arm_64
@@ -87,6 +91,9 @@
iree::schemas::cpu_data
)
+set(IREE_UK_ARM_64_DEPS "")
+
+if(IREE_UK_BUILD_ARM_64_DOTPROD)
iree_cc_library(
NAME
arm_64_dotprod
@@ -97,7 +104,10 @@
DEPS
iree::builtins::ukernel::internal_headers
)
+list(APPEND IREE_UK_ARM_64_DEPS "::arm_64_dotprod")
+endif() # IREE_UK_BUILD_ARM_64_DOTPROD
+if(IREE_UK_BUILD_ARM_64_I8MM)
iree_cc_library(
NAME
arm_64_i8mm
@@ -108,6 +118,8 @@
DEPS
iree::builtins::ukernel::internal_headers
)
+list(APPEND IREE_UK_ARM_64_DEPS "::arm_64_i8mm")
+endif() # IREE_UK_BUILD_ARM_64_I8MM
iree_cc_library(
NAME
@@ -119,8 +131,6 @@
"unpack_arm_64.c"
DEPS
::common_arm_64
- ::arm_64_dotprod
- ::arm_64_i8mm
iree::base::core_headers
iree::schemas::cpu_data
iree::builtins::ukernel::internal_headers
diff --git a/runtime/src/iree/builtins/ukernel/arch/arm_64/common_arm_64.h b/runtime/src/iree/builtins/ukernel/arch/arm_64/common_arm_64.h
index 1bcb60a..55c3e72 100644
--- a/runtime/src/iree/builtins/ukernel/arch/arm_64/common_arm_64.h
+++ b/runtime/src/iree/builtins/ukernel/arch/arm_64/common_arm_64.h
@@ -12,22 +12,27 @@
#include "iree/builtins/ukernel/common.h"
#include "iree/schemas/cpu_data.h"
-#if IREE_UK_COMPILER_CLANG_VERSION_AT_LEAST(7, 0) || \
- IREE_UK_COMPILER_GCC_VERSION_AT_LEAST(8, 0)
+#if defined(IREE_DEVICE_STANDALONE)
+// Standalone builds (e.g. bitcode) use our own Clang, supporting everything.
#define IREE_UK_BUILD_ARM_64_DOTPROD
+#define IREE_UK_BUILD_ARM_64_I8MM
+#else
+// Compiling with the system toolchain. Include the configured header.
+#include "iree/builtins/ukernel/arch/arm_64/config_arm_64.h"
+#endif
+
+#if defined(IREE_UK_BUILD_ARM_64_DOTPROD)
static inline bool iree_uk_cpu_supports_dotprod(
const iree_uk_uint64_t* cpu_data) {
return iree_uk_all_bits_set(cpu_data[0], IREE_CPU_DATA0_ARM_64_DOTPROD);
}
-#endif
+#endif // IREE_UK_BUILD_ARM_64_DOTPROD
-#if IREE_UK_COMPILER_CLANG_VERSION_AT_LEAST(10, 0) || \
- IREE_UK_COMPILER_GCC_VERSION_AT_LEAST(10, 0)
-#define IREE_UK_BUILD_ARM_64_I8MM
+#if defined(IREE_UK_BUILD_ARM_64_I8MM)
static inline bool iree_uk_cpu_supports_i8mm(const iree_uk_uint64_t* cpu_data) {
return iree_uk_all_bits_set(cpu_data[0], IREE_CPU_DATA0_ARM_64_I8MM);
}
-#endif
+#endif // IREE_UK_BUILD_ARM_64_I8MM
static inline int8x16x2_t iree_uk_neon_load_8x4xi8_strided(
const iree_uk_int8_t* src, iree_uk_index_t stride) {
diff --git a/runtime/src/iree/builtins/ukernel/arch/arm_64/config_arm_64.h.in b/runtime/src/iree/builtins/ukernel/arch/arm_64/config_arm_64.h.in
new file mode 100644
index 0000000..cc1143b
--- /dev/null
+++ b/runtime/src/iree/builtins/ukernel/arch/arm_64/config_arm_64.h.in
@@ -0,0 +1,17 @@
+// Copyright 2023 The IREE Authors
+//
+// Licensed under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+// Source for configured header. Processed by CMake configure_file.
+// Only used in the system-toolchain build, not in standalone builds such as
+// bitcode where we use our own Clang.
+
+#ifndef IREE_BUILTINS_UKERNEL_ARCH_ARM_64_CONFIG_ARM_64_H_
+#define IREE_BUILTINS_UKERNEL_ARCH_ARM_64_CONFIG_ARM_64_H_
+
+#cmakedefine IREE_UK_BUILD_ARM_64_DOTPROD
+#cmakedefine IREE_UK_BUILD_ARM_64_I8MM
+
+#endif // IREE_BUILTINS_UKERNEL_ARCH_ARM_64_CONFIG_ARM_64_H_
diff --git a/runtime/src/iree/builtins/ukernel/arch/arm_64/mmt4d_arm_64_dotprod.c b/runtime/src/iree/builtins/ukernel/arch/arm_64/mmt4d_arm_64_dotprod.c
index 249c24e..e43f501 100644
--- a/runtime/src/iree/builtins/ukernel/arch/arm_64/mmt4d_arm_64_dotprod.c
+++ b/runtime/src/iree/builtins/ukernel/arch/arm_64/mmt4d_arm_64_dotprod.c
@@ -7,8 +7,6 @@
#include "iree/builtins/ukernel/arch/arm_64/common_arm_64.h"
#include "iree/builtins/ukernel/mmt4d_internal.h"
-#if defined(IREE_UK_BUILD_ARM_64_DOTPROD)
-
void iree_uk_mmt4d_tile_i8i8i32_8x8x4_arm_64_dotprod(
void* IREE_UK_RESTRICT out_tile, const void* IREE_UK_RESTRICT lhs_panel,
const void* IREE_UK_RESTRICT rhs_panel, iree_uk_int32_t K,
@@ -96,5 +94,3 @@
vst1q_s32(out_ptr + 4 * 14, acc14);
vst1q_s32(out_ptr + 4 * 15, acc15);
}
-
-#endif // defined(IREE_UK_BUILD_ARM_64_DOTPROD)
diff --git a/runtime/src/iree/builtins/ukernel/arch/arm_64/mmt4d_arm_64_i8mm.c b/runtime/src/iree/builtins/ukernel/arch/arm_64/mmt4d_arm_64_i8mm.c
index 67054f1..f31d9b9 100644
--- a/runtime/src/iree/builtins/ukernel/arch/arm_64/mmt4d_arm_64_i8mm.c
+++ b/runtime/src/iree/builtins/ukernel/arch/arm_64/mmt4d_arm_64_i8mm.c
@@ -7,8 +7,6 @@
#include "iree/builtins/ukernel/arch/arm_64/common_arm_64.h"
#include "iree/builtins/ukernel/mmt4d_internal.h"
-#if defined(IREE_UK_BUILD_ARM_64_I8MM)
-
static inline int32x4_t iree_uk_neon_zip1_s32_as_s64(int32x4_t a, int32x4_t b) {
return vreinterpretq_s32_s64(
vzip1q_s64(vreinterpretq_s64_s32(a), vreinterpretq_s64_s32(b)));
@@ -438,5 +436,3 @@
"v27", "v28", "v29", "v30", "v31");
}
#endif // defined(IREE_UK_ENABLE_INLINE_ASM)
-
-#endif // defined(IREE_UK_BUILD_ARM_64_I8MM)
diff --git a/runtime/src/iree/builtins/ukernel/arch/x86_64/CMakeLists.txt b/runtime/src/iree/builtins/ukernel/arch/x86_64/CMakeLists.txt
index 3de1dc4..867d67f 100644
--- a/runtime/src/iree/builtins/ukernel/arch/x86_64/CMakeLists.txt
+++ b/runtime/src/iree/builtins/ukernel/arch/x86_64/CMakeLists.txt
@@ -151,6 +151,11 @@
"${IREE_UK_COPTS_X86_64_AVX512_VNNI_RELATIVE}"
)
+check_cxx_compiler_flag("${IREE_UK_COPTS_X86_64_AVX2_FMA}" IREE_UK_BUILD_X86_64_AVX2_FMA)
+check_cxx_compiler_flag("${IREE_UK_COPTS_X86_64_AVX512_BASE}" IREE_UK_BUILD_X86_64_AVX512_BASE)
+check_cxx_compiler_flag("${IREE_UK_COPTS_X86_64_AVX512_VNNI}" IREE_UK_BUILD_X86_64_AVX512_VNNI)
+configure_file("config_x86_64.h.in" "config_x86_64.h")
+
iree_cc_library(
NAME
common_x86_64
@@ -161,6 +166,9 @@
iree::schemas::cpu_data
)
+set(IREE_UK_X86_64_DEPS "")
+
+if(IREE_UK_BUILD_X86_64_AVX2_FMA)
iree_cc_library(
NAME
x86_64_avx2_fma
@@ -173,7 +181,10 @@
DEPS
iree::builtins::ukernel::internal_headers
)
+list(APPEND IREE_UK_X86_64_DEPS "::x86_64_avx2_fma")
+endif() # IREE_UK_BUILD_X86_64_AVX2_FMA
+if(IREE_UK_BUILD_X86_64_AVX512_BASE)
iree_cc_library(
NAME
x86_64_avx512_base
@@ -186,7 +197,10 @@
DEPS
iree::builtins::ukernel::internal_headers
)
+list(APPEND IREE_UK_X86_64_DEPS "::x86_64_avx512_base")
+endif() # IREE_UK_BUILD_X86_64_AVX512_BASE
+if(IREE_UK_BUILD_X86_64_AVX512_VNNI)
iree_cc_library(
NAME
x86_64_avx512_vnni
@@ -197,6 +211,8 @@
DEPS
iree::builtins::ukernel::internal_headers
)
+list(APPEND IREE_UK_X86_64_DEPS "::x86_64_avx512_vnni")
+endif() # IREE_UK_BUILD_X86_64_AVX512_VNNI
iree_cc_library(
NAME
@@ -208,9 +224,6 @@
"unpack_x86_64.c"
DEPS
::common_x86_64
- ::x86_64_avx2_fma
- ::x86_64_avx512_base
- ::x86_64_avx512_vnni
iree::base::core_headers
iree::builtins::ukernel::internal_headers
${IREE_UK_X86_64_DEPS}
diff --git a/runtime/src/iree/builtins/ukernel/arch/x86_64/common_x86_64.h b/runtime/src/iree/builtins/ukernel/arch/x86_64/common_x86_64.h
index cebd52b..59a89c3 100644
--- a/runtime/src/iree/builtins/ukernel/arch/x86_64/common_x86_64.h
+++ b/runtime/src/iree/builtins/ukernel/arch/x86_64/common_x86_64.h
@@ -12,24 +12,22 @@
#include "iree/builtins/ukernel/common.h"
#include "iree/schemas/cpu_data.h"
-// We default to requiring Clang>=8 and GCC>=9 because that's what Ruy used to
-// do; in my memory it's not just about what's officially supported but also
-// about skipping over quirky support in earlier releases.
-#if IREE_UK_COMPILER_CLANG_VERSION_AT_LEAST(8, 0) || \
- IREE_UK_COMPILER_GCC_VERSION_AT_LEAST(9, 0) || \
- IREE_UK_COMPILER_MSVC_VERSION_AT_LEAST(1910) // MSVC 2017
+#if defined(IREE_DEVICE_STANDALONE)
+// Standalone builds (e.g. bitcode) use our own Clang, supporting everything.
#define IREE_UK_BUILD_X86_64_AVX2_FMA
+#define IREE_UK_BUILD_X86_64_AVX512_BASE
+#define IREE_UK_BUILD_X86_64_AVX512_VNNI
+#else // IREE_DEVICE_STANDALONE
+// Compiling with the system toolchain. Include the configured header.
+#include "iree/builtins/ukernel/arch/x86_64/config_x86_64.h"
+#endif // IREE_DEVICE_STANDALONE
+
static inline bool iree_uk_cpu_supports_avx2_fma(
const iree_uk_uint64_t* cpu_data) {
return iree_uk_all_bits_set(
cpu_data[0], IREE_CPU_DATA0_X86_64_AVX2 | IREE_CPU_DATA0_X86_64_FMA);
}
-#endif
-#if IREE_UK_COMPILER_CLANG_VERSION_AT_LEAST(8, 0) || \
- IREE_UK_COMPILER_GCC_VERSION_AT_LEAST(9, 0) || \
- IREE_UK_COMPILER_MSVC_VERSION_AT_LEAST(1920) // MSVC 2019
-#define IREE_UK_BUILD_X86_64_AVX512_BASE
static inline bool iree_uk_cpu_supports_avx512_base(
const iree_uk_uint64_t* cpu_data) {
return iree_uk_all_bits_set(cpu_data[0], IREE_CPU_DATA0_X86_64_AVX512F |
@@ -38,19 +36,12 @@
IREE_CPU_DATA0_X86_64_AVX512VL |
IREE_CPU_DATA0_X86_64_AVX512CD);
}
-#endif
-// GCC 9 introduced AVX512VNNI: https://gcc.gnu.org/gcc-9/changes.html
-#if IREE_UK_COMPILER_CLANG_VERSION_AT_LEAST(8, 0) || \
- IREE_UK_COMPILER_GCC_VERSION_AT_LEAST(9, 0) || \
- IREE_UK_COMPILER_MSVC_VERSION_AT_LEAST(1930) // MSVC 2022
-#define IREE_UK_BUILD_X86_64_AVX512_VNNI
static inline bool iree_uk_cpu_supports_avx512_vnni(
const iree_uk_uint64_t* cpu_data) {
return iree_uk_cpu_supports_avx512_base(cpu_data) &&
iree_uk_all_bits_set(cpu_data[0], IREE_CPU_DATA0_X86_64_AVX512VNNI);
}
-#endif
#if defined(__AVX2__)
diff --git a/runtime/src/iree/builtins/ukernel/arch/x86_64/config_x86_64.h.in b/runtime/src/iree/builtins/ukernel/arch/x86_64/config_x86_64.h.in
new file mode 100644
index 0000000..3740e33
--- /dev/null
+++ b/runtime/src/iree/builtins/ukernel/arch/x86_64/config_x86_64.h.in
@@ -0,0 +1,18 @@
+// Copyright 2023 The IREE Authors
+//
+// Licensed under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+// Source for configured header. Processed by CMake configure_file.
+// Only used in the system-toolchain built, not in standalone builds such as
+// bitcode where we use our own Clang.
+
+#ifndef IREE_BUILTINS_UKERNEL_ARCH_X86_64_CONFIG_ARM_64_H_
+#define IREE_BUILTINS_UKERNEL_ARCH_X86_64_CONFIG_ARM_64_H_
+
+#cmakedefine IREE_UK_BUILD_X86_64_AVX2_FMA
+#cmakedefine IREE_UK_BUILD_X86_64_AVX512_BASE
+#cmakedefine IREE_UK_BUILD_X86_64_AVX512_VNNI
+
+#endif // IREE_BUILTINS_UKERNEL_ARCH_X86_64_CONFIG_ARM_64_H_
diff --git a/runtime/src/iree/builtins/ukernel/arch/x86_64/mmt4d_x86_64.c b/runtime/src/iree/builtins/ukernel/arch/x86_64/mmt4d_x86_64.c
index 712c4fb..c565168 100644
--- a/runtime/src/iree/builtins/ukernel/arch/x86_64/mmt4d_x86_64.c
+++ b/runtime/src/iree/builtins/ukernel/arch/x86_64/mmt4d_x86_64.c
@@ -27,7 +27,7 @@
static iree_uk_mmt4d_tile_func_t
iree_uk_mmt4d_select_tile_func_x86_64_f32f32f32_8x8x1(
const iree_uk_mmt4d_params_t* params) {
-#ifdef IREE_UK_BUILD_X86_64_AVX2_FMA
+#if defined(IREE_UK_BUILD_X86_64_AVX2_FMA)
if (iree_uk_cpu_supports_avx2_fma(params->cpu_data)) {
return iree_uk_mmt4d_tile_f32f32f32_8x8x1_x86_64_avx2_fma;
}
@@ -38,7 +38,7 @@
static iree_uk_mmt4d_tile_func_t
iree_uk_mmt4d_select_tile_func_x86_64_f32f32f32_16x16x1(
const iree_uk_mmt4d_params_t* params) {
-#ifdef IREE_UK_BUILD_X86_64_AVX512_BASE
+#if defined(IREE_UK_BUILD_X86_64_AVX512_BASE)
if (iree_uk_cpu_supports_avx512_base(params->cpu_data)) {
return iree_uk_mmt4d_tile_f32f32f32_16x16x1_x86_64_avx512_base;
}
@@ -49,12 +49,12 @@
static iree_uk_mmt4d_tile_func_t
iree_uk_mmt4d_select_tile_func_x86_64_i8i8i32_16x16x2(
const iree_uk_mmt4d_params_t* params) {
-#ifdef IREE_UK_BUILD_X86_64_AVX512_VNNI
+#if defined(IREE_UK_BUILD_X86_64_AVX512_VNNI)
if (params->cpu_data[0] & (IREE_CPU_DATA0_X86_64_AVX512VNNI)) {
return iree_uk_mmt4d_tile_i8i8i32_16x16x2_x86_64_avx512_vnni;
}
#endif
-#ifdef IREE_UK_BUILD_X86_64_AVX512_BASE
+#if defined(IREE_UK_BUILD_X86_64_AVX512_BASE)
if (params->cpu_data[0] & (IREE_CPU_DATA0_X86_64_AVX512BW)) {
return iree_uk_mmt4d_tile_i8i8i32_16x16x2_x86_64_avx512_base;
}
@@ -65,7 +65,7 @@
static iree_uk_mmt4d_tile_func_t
iree_uk_mmt4d_select_tile_func_x86_64_i8i8i32_8x8x2(
const iree_uk_mmt4d_params_t* params) {
-#ifdef IREE_UK_BUILD_X86_64_AVX2_FMA
+#if defined(IREE_UK_BUILD_X86_64_AVX2_FMA)
if (iree_uk_cpu_supports_avx2_fma(params->cpu_data)) {
return iree_uk_mmt4d_tile_i8i8i32_8x8x2_x86_64_avx2_fma;
}
diff --git a/runtime/src/iree/builtins/ukernel/arch/x86_64/mmt4d_x86_64_avx2_fma.c b/runtime/src/iree/builtins/ukernel/arch/x86_64/mmt4d_x86_64_avx2_fma.c
index 0c630df..29c1fcc 100644
--- a/runtime/src/iree/builtins/ukernel/arch/x86_64/mmt4d_x86_64_avx2_fma.c
+++ b/runtime/src/iree/builtins/ukernel/arch/x86_64/mmt4d_x86_64_avx2_fma.c
@@ -7,8 +7,6 @@
#include "iree/builtins/ukernel/arch/x86_64/common_x86_64.h"
#include "iree/builtins/ukernel/mmt4d.h"
-#if defined(IREE_UK_BUILD_X86_64_AVX2_FMA)
-
void iree_uk_mmt4d_tile_f32f32f32_8x8x1_x86_64_avx2_fma(
void* IREE_UK_RESTRICT out_tile, const void* IREE_UK_RESTRICT lhs_panel,
const void* IREE_UK_RESTRICT rhs_panel, iree_uk_int32_t K,
@@ -158,5 +156,3 @@
iree_uk_avx_storeu_2x128((__m128i*)(out_ptr + 3 * 8 + 4),
(__m128i*)(out_ptr + 7 * 8 + 0), acc_3_4567_7_0123);
}
-
-#endif // defined(IREE_UK_BUILD_X86_64_AVX2_FMA)
diff --git a/runtime/src/iree/builtins/ukernel/arch/x86_64/mmt4d_x86_64_avx512_base.c b/runtime/src/iree/builtins/ukernel/arch/x86_64/mmt4d_x86_64_avx512_base.c
index 61262b1..5770f7c 100644
--- a/runtime/src/iree/builtins/ukernel/arch/x86_64/mmt4d_x86_64_avx512_base.c
+++ b/runtime/src/iree/builtins/ukernel/arch/x86_64/mmt4d_x86_64_avx512_base.c
@@ -7,8 +7,6 @@
#include "iree/builtins/ukernel/arch/x86_64/common_x86_64.h"
#include "iree/builtins/ukernel/mmt4d_internal.h"
-#if defined(IREE_UK_BUILD_X86_64_AVX512_BASE)
-
void iree_uk_mmt4d_tile_f32f32f32_16x16x1_x86_64_avx512_base(
void* IREE_UK_RESTRICT out_tile, const void* IREE_UK_RESTRICT lhs_panel,
const void* IREE_UK_RESTRICT rhs_panel, iree_uk_int32_t K,
@@ -291,5 +289,3 @@
iree_uk_avx512_storeu_4x128_to_16x16xi32(out_ptr, 3, 12, 7, 8, 11, 4, 15, 0,
acc_3_CDEF_7_89AB_B_4567_F_0123);
}
-
-#endif // defined(IREE_UK_BUILD_X86_64_AVX512_BASE)
diff --git a/runtime/src/iree/builtins/ukernel/arch/x86_64/mmt4d_x86_64_avx512_vnni.c b/runtime/src/iree/builtins/ukernel/arch/x86_64/mmt4d_x86_64_avx512_vnni.c
index 9006567..0066d8b 100644
--- a/runtime/src/iree/builtins/ukernel/arch/x86_64/mmt4d_x86_64_avx512_vnni.c
+++ b/runtime/src/iree/builtins/ukernel/arch/x86_64/mmt4d_x86_64_avx512_vnni.c
@@ -7,8 +7,6 @@
#include "iree/builtins/ukernel/arch/x86_64/common_x86_64.h"
#include "iree/builtins/ukernel/mmt4d_internal.h"
-#if defined(IREE_UK_BUILD_X86_64_AVX512_VNNI)
-
void iree_uk_mmt4d_tile_i8i8i32_16x16x2_x86_64_avx512_vnni(
void* IREE_UK_RESTRICT out_tile, const void* IREE_UK_RESTRICT lhs_panel,
const void* IREE_UK_RESTRICT rhs_panel, iree_uk_int32_t K,
@@ -199,5 +197,3 @@
iree_uk_avx512_storeu_4x128_to_16x16xi32(out_ptr, 3, 12, 7, 8, 11, 4, 15, 0,
acc_3_CDEF_7_89AB_B_4567_F_0123);
}
-
-#endif // defined(IREE_UK_BUILD_X86_64_AVX512_VNNI)
diff --git a/runtime/src/iree/builtins/ukernel/arch/x86_64/pack_x86_64.c b/runtime/src/iree/builtins/ukernel/arch/x86_64/pack_x86_64.c
index 6c06b88..74361c9 100644
--- a/runtime/src/iree/builtins/ukernel/arch/x86_64/pack_x86_64.c
+++ b/runtime/src/iree/builtins/ukernel/arch/x86_64/pack_x86_64.c
@@ -24,7 +24,7 @@
static iree_uk_pack_tile_func_t iree_uk_pack_select_tile_func_x86_64_8x8_x32(
const iree_uk_pack_params_t* params) {
-#ifdef IREE_UK_BUILD_X86_64_AVX2_FMA
+#if defined(IREE_UK_BUILD_X86_64_AVX2_FMA)
if (iree_uk_cpu_supports_avx2_fma(params->cpu_data)) {
bool transpose = params->flags & IREE_UK_FLAG_PACK_TRANSPOSE_INNER;
return transpose ? 0 : iree_uk_pack_tile_8x8_x32_x86_64_avx2_fma_direct;
@@ -35,7 +35,7 @@
static iree_uk_pack_tile_func_t iree_uk_pack_select_tile_func_x86_64_16x16_x32(
const iree_uk_pack_params_t* params) {
-#ifdef IREE_UK_BUILD_X86_64_AVX512_BASE
+#if defined(IREE_UK_BUILD_X86_64_AVX512_BASE)
if (iree_uk_cpu_supports_avx512_base(params->cpu_data)) {
bool transpose = params->flags & IREE_UK_FLAG_PACK_TRANSPOSE_INNER;
return transpose ? 0
@@ -47,7 +47,7 @@
static iree_uk_pack_tile_func_t iree_uk_pack_select_tile_func_x86_64_8x1_x32(
const iree_uk_pack_params_t* params) {
-#ifdef IREE_UK_BUILD_X86_64_AVX2_FMA
+#if defined(IREE_UK_BUILD_X86_64_AVX2_FMA)
if (iree_uk_cpu_supports_avx2_fma(params->cpu_data)) {
bool transpose = params->flags & IREE_UK_FLAG_PACK_TRANSPOSE_INNER;
return transpose ? iree_uk_pack_tile_8x1_x32_x86_64_avx2_fma_transpose
@@ -59,7 +59,7 @@
static iree_uk_pack_tile_func_t iree_uk_pack_select_tile_func_x86_64_16x1_x32(
const iree_uk_pack_params_t* params) {
-#ifdef IREE_UK_BUILD_X86_64_AVX512_BASE
+#if defined(IREE_UK_BUILD_X86_64_AVX512_BASE)
if (iree_uk_cpu_supports_avx512_base(params->cpu_data)) {
bool transpose = params->flags & IREE_UK_FLAG_PACK_TRANSPOSE_INNER;
return transpose ? iree_uk_pack_tile_16x1_x32_x86_64_avx512_base_transpose
@@ -71,7 +71,7 @@
static iree_uk_pack_tile_func_t iree_uk_pack_select_tile_func_x86_64_8x2_x8(
const iree_uk_pack_params_t* params) {
-#ifdef IREE_UK_BUILD_X86_64_AVX2_FMA
+#if defined(IREE_UK_BUILD_X86_64_AVX2_FMA)
if (iree_uk_cpu_supports_avx2_fma(params->cpu_data)) {
bool transpose = params->flags & IREE_UK_FLAG_PACK_TRANSPOSE_INNER;
return transpose ? iree_uk_pack_tile_8x2_x8_x86_64_avx2_fma_transpose
@@ -83,7 +83,7 @@
static iree_uk_pack_tile_func_t iree_uk_pack_select_tile_func_x86_64_16x2_x8(
const iree_uk_pack_params_t* params) {
-#ifdef IREE_UK_BUILD_X86_64_AVX512_BASE
+#if defined(IREE_UK_BUILD_X86_64_AVX512_BASE)
if (iree_uk_cpu_supports_avx512_base(params->cpu_data)) {
bool transpose = params->flags & IREE_UK_FLAG_PACK_TRANSPOSE_INNER;
return transpose ? iree_uk_pack_tile_16x2_x8_x86_64_avx512_base_transpose
diff --git a/runtime/src/iree/builtins/ukernel/arch/x86_64/pack_x86_64_avx2_fma.c b/runtime/src/iree/builtins/ukernel/arch/x86_64/pack_x86_64_avx2_fma.c
index 55e41cf..2ef8817 100644
--- a/runtime/src/iree/builtins/ukernel/arch/x86_64/pack_x86_64_avx2_fma.c
+++ b/runtime/src/iree/builtins/ukernel/arch/x86_64/pack_x86_64_avx2_fma.c
@@ -7,8 +7,6 @@
#include "iree/builtins/ukernel/arch/x86_64/common_x86_64.h"
#include "iree/builtins/ukernel/pack_internal.h"
-#if defined(IREE_UK_BUILD_X86_64_AVX2_FMA)
-
void iree_uk_pack_tile_8x8_x32_x86_64_avx2_fma_direct(
void* IREE_UK_RESTRICT out_tile_ptr,
const void* IREE_UK_RESTRICT in_tile_ptr, iree_uk_index_t outer_size1,
@@ -141,5 +139,3 @@
in_ptr += 8;
}
}
-
-#endif // defined(IREE_UK_BUILD_X86_64_AVX2_FMA)
diff --git a/runtime/src/iree/builtins/ukernel/arch/x86_64/pack_x86_64_avx512_base.c b/runtime/src/iree/builtins/ukernel/arch/x86_64/pack_x86_64_avx512_base.c
index 44a3dc3..29ad77e 100644
--- a/runtime/src/iree/builtins/ukernel/arch/x86_64/pack_x86_64_avx512_base.c
+++ b/runtime/src/iree/builtins/ukernel/arch/x86_64/pack_x86_64_avx512_base.c
@@ -7,8 +7,6 @@
#include "iree/builtins/ukernel/arch/x86_64/common_x86_64.h"
#include "iree/builtins/ukernel/pack_internal.h"
-#if defined(IREE_UK_BUILD_X86_64_AVX512_BASE)
-
void iree_uk_pack_tile_16x16_x32_x86_64_avx512_base_direct(
void* IREE_UK_RESTRICT out_tile_ptr,
const void* IREE_UK_RESTRICT in_tile_ptr, iree_uk_index_t outer_size1,
@@ -148,5 +146,3 @@
in_ptr += 16;
}
}
-
-#endif // defined(IREE_UK_BUILD_X86_64_AVX512_BASE)
diff --git a/runtime/src/iree/builtins/ukernel/arch/x86_64/query_tile_sizes_x86_64.c b/runtime/src/iree/builtins/ukernel/arch/x86_64/query_tile_sizes_x86_64.c
index 2efbc66..72b54e0 100644
--- a/runtime/src/iree/builtins/ukernel/arch/x86_64/query_tile_sizes_x86_64.c
+++ b/runtime/src/iree/builtins/ukernel/arch/x86_64/query_tile_sizes_x86_64.c
@@ -10,12 +10,12 @@
static iree_uk_matmul_tile_sizes_t
iree_uk_query_matmul_tile_sizes_x86_64_f32f32f32(
const iree_uk_query_tile_sizes_2d_params_t* params) {
-#ifdef IREE_UK_BUILD_X86_64_AVX512_BASE
+#if defined(IREE_UK_BUILD_X86_64_AVX512_BASE)
if (iree_uk_cpu_supports_avx512_base(params->cpu_data)) {
return (iree_uk_matmul_tile_sizes_t){.M = 16, .K = 1, .N = 16};
}
#endif
-#ifdef IREE_UK_BUILD_X86_64_AVX2_FMA
+#if defined(IREE_UK_BUILD_X86_64_AVX2_FMA)
if (iree_uk_cpu_supports_avx2_fma(params->cpu_data)) {
return (iree_uk_matmul_tile_sizes_t){.M = 8, .K = 1, .N = 8};
}
@@ -27,12 +27,12 @@
static iree_uk_matmul_tile_sizes_t
iree_uk_query_matmul_tile_sizes_x86_64_i8i8i32(
const iree_uk_query_tile_sizes_2d_params_t* params) {
-#ifdef IREE_UK_BUILD_X86_64_AVX512_VNNI
+#if defined(IREE_UK_BUILD_X86_64_AVX512_VNNI)
if (iree_uk_cpu_supports_avx512_vnni(params->cpu_data)) {
return (iree_uk_matmul_tile_sizes_t){.M = 16, .K = 2, .N = 16};
}
#endif
-#ifdef IREE_UK_BUILD_X86_64_AVX2_FMA
+#if defined(IREE_UK_BUILD_X86_64_AVX2_FMA)
if (iree_uk_cpu_supports_avx2_fma(params->cpu_data)) {
return (iree_uk_matmul_tile_sizes_t){.M = 8, .K = 2, .N = 8};
}
diff --git a/runtime/src/iree/builtins/ukernel/arch/x86_64/unpack_x86_64.c b/runtime/src/iree/builtins/ukernel/arch/x86_64/unpack_x86_64.c
index bc57b37..dd768b2 100644
--- a/runtime/src/iree/builtins/ukernel/arch/x86_64/unpack_x86_64.c
+++ b/runtime/src/iree/builtins/ukernel/arch/x86_64/unpack_x86_64.c
@@ -20,13 +20,13 @@
// Unpack is currently only used in practice with esize==4 and non-transpose.
if (esize != 4 || transpose) return 0;
if (params->in_size2 == 8 && params->in_size3 == 8) {
-#ifdef IREE_UK_BUILD_X86_64_AVX2_FMA
+#if defined(IREE_UK_BUILD_X86_64_AVX2_FMA)
if (iree_uk_cpu_supports_avx2_fma(params->cpu_data)) {
return iree_uk_unpack_tile_8x8_x32_x86_64_avx2_fma_direct;
}
#endif
} else if (params->in_size2 == 16 && params->in_size3 == 16) {
-#ifdef IREE_UK_BUILD_X86_64_AVX512_BASE
+#if defined(IREE_UK_BUILD_X86_64_AVX512_BASE)
if (iree_uk_cpu_supports_avx512_base(params->cpu_data)) {
return iree_uk_unpack_tile_16x16_x32_x86_64_avx512_base_direct;
}
diff --git a/runtime/src/iree/builtins/ukernel/arch/x86_64/unpack_x86_64_avx2_fma.c b/runtime/src/iree/builtins/ukernel/arch/x86_64/unpack_x86_64_avx2_fma.c
index cc2600d..1adcc07 100644
--- a/runtime/src/iree/builtins/ukernel/arch/x86_64/unpack_x86_64_avx2_fma.c
+++ b/runtime/src/iree/builtins/ukernel/arch/x86_64/unpack_x86_64_avx2_fma.c
@@ -7,8 +7,6 @@
#include "iree/builtins/ukernel/arch/x86_64/common_x86_64.h"
#include "iree/builtins/ukernel/unpack_internal.h"
-#if defined(IREE_UK_BUILD_X86_64_AVX2_FMA)
-
void iree_uk_unpack_tile_8x8_x32_x86_64_avx2_fma_direct(
void* IREE_UK_RESTRICT out_tile_ptr,
const void* IREE_UK_RESTRICT in_tile_ptr, iree_uk_index_t outer_size1,
@@ -27,5 +25,3 @@
in_ptr += 4 * in_stride1;
}
}
-
-#endif // defined(IREE_UK_BUILD_X86_64_AVX2_FMA)
diff --git a/runtime/src/iree/builtins/ukernel/arch/x86_64/unpack_x86_64_avx512_base.c b/runtime/src/iree/builtins/ukernel/arch/x86_64/unpack_x86_64_avx512_base.c
index 2c9229d..256cf0e 100644
--- a/runtime/src/iree/builtins/ukernel/arch/x86_64/unpack_x86_64_avx512_base.c
+++ b/runtime/src/iree/builtins/ukernel/arch/x86_64/unpack_x86_64_avx512_base.c
@@ -7,8 +7,6 @@
#include "iree/builtins/ukernel/arch/x86_64/common_x86_64.h"
#include "iree/builtins/ukernel/unpack_internal.h"
-#if defined(IREE_UK_BUILD_X86_64_AVX512_BASE)
-
void iree_uk_unpack_tile_16x16_x32_x86_64_avx512_base_direct(
void* IREE_UK_RESTRICT out_tile_ptr,
const void* IREE_UK_RESTRICT in_tile_ptr, iree_uk_index_t outer_size1,
@@ -27,5 +25,3 @@
in_ptr += 4 * in_stride1;
}
}
-
-#endif // defined(IREE_UK_BUILD_X86_64_AVX512_BASE)
