Use dwconv on input_depth == 1, 5x5
Change-Id: Icd523652329dc68ef3b73d6cf166cca952950d5d
diff --git a/tflm/opt/conv_s8.cc b/tflm/opt/conv_s8.cc
index d3529ef..3f77764 100644
--- a/tflm/opt/conv_s8.cc
+++ b/tflm/opt/conv_s8.cc
@@ -200,6 +200,11 @@
return; \
}
+ if (input_depth == 1 && filter_width == 5 && filter_height == 5 &&
+ output_depth == 24) {
+ RUN_KERNEL(kelvin::opt::ConvPerChannelD1OD24_5x5);
+ }
+
// special case of filter_depth = 4n
if (dilation_width_factor == 1 && dilation_height_factor == 1 &&
stride_width <= 2 && stride_height <= 2 && filter_depth % 4 == 0 &&
diff --git a/tflm/opt/conv_s8.h b/tflm/opt/conv_s8.h
index 91c535a..718d9ee 100644
--- a/tflm/opt/conv_s8.h
+++ b/tflm/opt/conv_s8.h
@@ -71,6 +71,15 @@
const int32_t* bias_data, const tflite::RuntimeShape& output_shape,
int8_t* output_data);
+// Input depth = 1, filter_width = 5, filter_height = 5, output_depth = 24
+void ConvPerChannelD1OD24_5x5(
+ const tflite::ConvParams& params, const int32_t* output_multiplier,
+ const int32_t* output_shift, const tflite::RuntimeShape& input_shape,
+ const int8_t* input_data, const tflite::RuntimeShape& filter_shape,
+ const int8_t* filter_data, const tflite::RuntimeShape& bias_shape,
+ const int32_t* bias_data, const tflite::RuntimeShape& output_shape,
+ int8_t* output_data);
+
} // namespace kelvin::opt
#endif // TFLM_OPT_CONV_S8_H_
diff --git a/tflm/opt/conv_s8_d1.cc b/tflm/opt/conv_s8_d1.cc
index b886f32..09f4110 100644
--- a/tflm/opt/conv_s8_d1.cc
+++ b/tflm/opt/conv_s8_d1.cc
@@ -64,8 +64,679 @@
output[24] = input[3];
}
}
+
} // namespace
+#define CALCULATE_IN_X(in_x_origin) \
+ { \
+ _Pragma("GCC unroll 5") for (int i = 0; i < 5; ++i) { \
+ in_x[i] = in_x_origin + (dilation_width_factor * i); \
+ } \
+ }
+
+#define CALCULATE_IN_Y(in_y_origin) \
+ { \
+ _Pragma("GCC unroll 5") for (int i = 0; i < 5; ++i) { \
+ in_y[i] = in_y_origin + (dilation_height_factor * i); \
+ } \
+ }
+
+#define PAD_ROW_0(input_offset) \
+ { \
+ vdup_b_x(v27, -input_offset); \
+ vdup_b_x(v28, -input_offset); \
+ vdup_b_x(v29, -input_offset); \
+ vdup_b_x(v30, -input_offset); \
+ vdup_b_x(v31, -input_offset); \
+ }
+#define PAD_ROW_1(input_offset) \
+ { \
+ vdup_b_x(v32, -input_offset); \
+ vdup_b_x(v33, -input_offset); \
+ vdup_b_x(v34, -input_offset); \
+ vdup_b_x(v35, -input_offset); \
+ vdup_b_x(v36, -input_offset); \
+ }
+#define PAD_ROW_2(input_offset) \
+ { \
+ vdup_b_x(v37, -input_offset); \
+ vdup_b_x(v38, -input_offset); \
+ vdup_b_x(v39, -input_offset); \
+ vdup_b_x(v40, -input_offset); \
+ vdup_b_x(v41, -input_offset); \
+ }
+#define PAD_ROW_3(input_offset) \
+ { \
+ vdup_b_x(v42, -input_offset); \
+ vdup_b_x(v43, -input_offset); \
+ vdup_b_x(v44, -input_offset); \
+ vdup_b_x(v45, -input_offset); \
+ vdup_b_x(v46, -input_offset); \
+ }
+#define PAD_ROW_4(input_offset) \
+ { \
+ vdup_b_x(v47, -input_offset); \
+ vdup_b_x(v48, -input_offset); \
+ vdup_b_x(v49, -input_offset); \
+ vdup_b_x(v50, -input_offset); \
+ vdup_b_x(v51, -input_offset); \
+ }
+
+#define LOAD_ROW_0(p_input, input_width, in_y, in_x) \
+ { \
+ const int8_t* p_row = p_input + (in_y[0] * input_width); \
+ vdup_b_x(v27, *(p_row + in_x[0])); \
+ vdup_b_x(v28, *(p_row + in_x[1])); \
+ vdup_b_x(v29, *(p_row + in_x[2])); \
+ vdup_b_x(v30, *(p_row + in_x[3])); \
+ vdup_b_x(v31, *(p_row + in_x[4])); \
+ }
+
+#define LOAD_ROW_1(p_input, input_width, in_y, in_x) \
+ { \
+ const int8_t* p_row = p_input + (in_y[1] * input_width); \
+ vdup_b_x(v32, *(p_row + in_x[0])); \
+ vdup_b_x(v33, *(p_row + in_x[1])); \
+ vdup_b_x(v34, *(p_row + in_x[2])); \
+ vdup_b_x(v35, *(p_row + in_x[3])); \
+ vdup_b_x(v36, *(p_row + in_x[4])); \
+ }
+
+#define LOAD_ROW_2(p_input, input_width, in_y, in_x) \
+ { \
+ const int8_t* p_row = p_input + (in_y[2] * input_width); \
+ vdup_b_x(v37, *(p_row + in_x[0])); \
+ vdup_b_x(v38, *(p_row + in_x[1])); \
+ vdup_b_x(v39, *(p_row + in_x[2])); \
+ vdup_b_x(v40, *(p_row + in_x[3])); \
+ vdup_b_x(v41, *(p_row + in_x[4])); \
+ }
+
+#define LOAD_ROW_3(p_input, input_width, in_y, in_x) \
+ { \
+ const int8_t* p_row = p_input + (in_y[3] * input_width); \
+ vdup_b_x(v42, *(p_row + in_x[0])); \
+ vdup_b_x(v43, *(p_row + in_x[1])); \
+ vdup_b_x(v44, *(p_row + in_x[2])); \
+ vdup_b_x(v45, *(p_row + in_x[3])); \
+ vdup_b_x(v46, *(p_row + in_x[4])); \
+ }
+
+#define LOAD_ROW_4(p_input, input_width, in_y, in_x) \
+ { \
+ const int8_t* p_row = p_input + (in_y[4] * input_width); \
+ vdup_b_x(v47, *(p_row + in_x[0])); \
+ vdup_b_x(v48, *(p_row + in_x[1])); \
+ vdup_b_x(v49, *(p_row + in_x[2])); \
+ vdup_b_x(v50, *(p_row + in_x[3])); \
+ vdup_b_x(v51, *(p_row + in_x[4])); \
+ }
+
+#define H_PAD_OR_LOAD_ROW_0(p_input, input_width, input_offset, in_y, in_x) \
+ if (in_x[0] >= 0 && in_x[4] < input_width) { \
+ LOAD_ROW_0(p_input, input_width, in_y, in_x); \
+ } else { \
+ const int8_t* p_row = p_input + (in_y[0] * input_width); \
+ if (in_x[0] < 0 || in_x[0] >= input_width) { \
+ vdup_b_x(v27, -input_offset); \
+ } else { \
+ vdup_b_x(v27, *(p_row + in_x[0])); \
+ } \
+ if (in_x[1] < 0 || in_x[1] >= input_width) { \
+ vdup_b_x(v28, -input_offset); \
+ } else { \
+ vdup_b_x(v28, *(p_row + in_x[1])); \
+ } \
+ if (in_x[2] < 0 || in_x[2] >= input_width) { \
+ vdup_b_x(v29, -input_offset); \
+ } else { \
+ vdup_b_x(v29, *(p_row + in_x[2])); \
+ } \
+ if (in_x[3] < 0 || in_x[3] >= input_width) { \
+ vdup_b_x(v30, -input_offset); \
+ } else { \
+ vdup_b_x(v30, *(p_row + in_x[3])); \
+ } \
+ if (in_x[4] < 0 || in_x[4] >= input_width) { \
+ vdup_b_x(v31, -input_offset); \
+ } else { \
+ vdup_b_x(v31, *(p_row + in_x[4])); \
+ } \
+ }
+
+#define H_PAD_OR_LOAD_ROW_1(p_input, input_width, input_offset, in_y, in_x) \
+ if (in_x[0] >= 0 && in_x[4] < input_width) { \
+ LOAD_ROW_1(p_input, input_width, in_y, in_x); \
+ } else { \
+ const int8_t* p_row = p_input + (in_y[1] * input_width); \
+ if (in_x[0] < 0 || in_x[0] >= input_width) { \
+ vdup_b_x(v32, -input_offset); \
+ } else { \
+ vdup_b_x(v32, *(p_row + in_x[0])); \
+ } \
+ if (in_x[1] < 0 || in_x[1] >= input_width) { \
+ vdup_b_x(v33, -input_offset); \
+ } else { \
+ vdup_b_x(v33, *(p_row + in_x[1])); \
+ } \
+ if (in_x[2] < 0 || in_x[2] >= input_width) { \
+ vdup_b_x(v34, -input_offset); \
+ } else { \
+ vdup_b_x(v34, *(p_row + in_x[2])); \
+ } \
+ if (in_x[3] < 0 || in_x[3] >= input_width) { \
+ vdup_b_x(v35, -input_offset); \
+ } else { \
+ vdup_b_x(v35, *(p_row + in_x[3])); \
+ } \
+ if (in_x[4] < 0 || in_x[4] >= input_width) { \
+ vdup_b_x(v36, -input_offset); \
+ } else { \
+ vdup_b_x(v36, *(p_row + in_x[4])); \
+ } \
+ }
+
+#define H_PAD_OR_LOAD_ROW_2(p_input, input_width, input_offset, in_y, in_x) \
+ if (in_x[0] >= 0 && in_x[4] < input_width) { \
+ LOAD_ROW_2(p_input, input_width, in_y, in_x); \
+ } else { \
+ const int8_t* p_row = p_input + (in_y[2] * input_width); \
+ if (in_x[0] < 0 || in_x[0] >= input_width) { \
+ vdup_b_x(v37, -input_offset); \
+ } else { \
+ vdup_b_x(v37, *(p_row + in_x[0])); \
+ } \
+ if (in_x[1] < 0 || in_x[1] >= input_width) { \
+ vdup_b_x(v38, -input_offset); \
+ } else { \
+ vdup_b_x(v38, *(p_row + in_x[1])); \
+ } \
+ if (in_x[2] < 0 || in_x[2] >= input_width) { \
+ vdup_b_x(v39, -input_offset); \
+ } else { \
+ vdup_b_x(v39, *(p_row + in_x[2])); \
+ } \
+ if (in_x[3] < 0 || in_x[3] >= input_width) { \
+ vdup_b_x(v40, -input_offset); \
+ } else { \
+ vdup_b_x(v40, *(p_row + in_x[3])); \
+ } \
+ if (in_x[4] < 0 || in_x[4] >= input_width) { \
+ vdup_b_x(v41, -input_offset); \
+ } else { \
+ vdup_b_x(v41, *(p_row + in_x[4])); \
+ } \
+ }
+
+#define H_PAD_OR_LOAD_ROW_3(p_input, input_width, input_offset, in_y, in_x) \
+ if (in_x[0] >= 0 && in_x[4] < input_width) { \
+ LOAD_ROW_3(p_input, input_width, in_y, in_x); \
+ } else { \
+ const int8_t* p_row = p_input + (in_y[3] * input_width); \
+ if (in_x[0] < 0 || in_x[0] >= input_width) { \
+ vdup_b_x(v42, -input_offset); \
+ } else { \
+ vdup_b_x(v42, *(p_row + in_x[0])); \
+ } \
+ if (in_x[1] < 0 || in_x[1] >= input_width) { \
+ vdup_b_x(v43, -input_offset); \
+ } else { \
+ vdup_b_x(v43, *(p_row + in_x[1])); \
+ } \
+ if (in_x[2] < 0 || in_x[2] >= input_width) { \
+ vdup_b_x(v44, -input_offset); \
+ } else { \
+ vdup_b_x(v44, *(p_row + in_x[2])); \
+ } \
+ if (in_x[3] < 0 || in_x[3] >= input_width) { \
+ vdup_b_x(v45, -input_offset); \
+ } else { \
+ vdup_b_x(v45, *(p_row + in_x[3])); \
+ } \
+ if (in_x[4] < 0 || in_x[4] >= input_width) { \
+ vdup_b_x(v46, -input_offset); \
+ } else { \
+ vdup_b_x(v46, *(p_row + in_x[4])); \
+ } \
+ }
+
+#define H_PAD_OR_LOAD_ROW_4(p_input, input_width, input_offset, in_y, in_x) \
+ if (in_x[0] >= 0 && in_x[4] < input_width) { \
+ LOAD_ROW_4(p_input, input_width, in_y, in_x); \
+ } else { \
+ const int8_t* p_row = p_input + (in_y[4] * input_width); \
+ if (in_x[0] < 0 || in_x[0] >= input_width) { \
+ vdup_b_x(v47, -input_offset); \
+ } else { \
+ vdup_b_x(v47, *(p_row + in_x[0])); \
+ } \
+ if (in_x[1] < 0 || in_x[1] >= input_width) { \
+ vdup_b_x(v48, -input_offset); \
+ } else { \
+ vdup_b_x(v48, *(p_row + in_x[1])); \
+ } \
+ if (in_x[2] < 0 || in_x[2] >= input_width) { \
+ vdup_b_x(v49, -input_offset); \
+ } else { \
+ vdup_b_x(v49, *(p_row + in_x[2])); \
+ } \
+ if (in_x[3] < 0 || in_x[3] >= input_width) { \
+ vdup_b_x(v50, -input_offset); \
+ } else { \
+ vdup_b_x(v50, *(p_row + in_x[3])); \
+ } \
+ if (in_x[4] < 0 || in_x[4] >= input_width) { \
+ vdup_b_x(v51, -input_offset); \
+ } else { \
+ vdup_b_x(v51, *(p_row + in_x[4])); \
+ } \
+ }
+
+#define _H_PAD_OR_LOAD_ROW(row, p_input, input_width, input_offset, in_y, \
+ in_x) \
+ H_PAD_OR_LOAD_ROW_##row(p_input, input_width, input_offset, in_y, in_x);
+
+#define _PAD_OR_LOAD_ROW(row, p_input, input_height, input_width, in_y, in_x, \
+ input_offset) \
+ { \
+ if (in_y[row] < 0 || in_y[row] >= input_height) { \
+ PAD_ROW_##row(input_offset); \
+ } else { \
+ _H_PAD_OR_LOAD_ROW(row, p_input, input_width, input_offset, in_y, in_x); \
+ } \
+ }
+
+#define PAD_OR_LOAD_ROW_0(p_input, input_height, input_width, in_y, in_x, \
+ input_offset) \
+ _PAD_OR_LOAD_ROW(0, p_input, input_height, input_width, in_y, in_x, \
+ input_offset);
+#define PAD_OR_LOAD_ROW_1(p_input, input_height, input_width, in_y, in_x, \
+ input_offset) \
+ _PAD_OR_LOAD_ROW(1, p_input, input_height, input_width, in_y, in_x, \
+ input_offset);
+#define PAD_OR_LOAD_ROW_2(p_input, input_height, input_width, in_y, in_x, \
+ input_offset) \
+ _PAD_OR_LOAD_ROW(2, p_input, input_height, input_width, in_y, in_x, \
+ input_offset);
+#define PAD_OR_LOAD_ROW_3(p_input, input_height, input_width, in_y, in_x, \
+ input_offset) \
+ _PAD_OR_LOAD_ROW(3, p_input, input_height, input_width, in_y, in_x, \
+ input_offset);
+#define PAD_OR_LOAD_ROW_4(p_input, input_height, input_width, in_y, in_x, \
+ input_offset) \
+ _PAD_OR_LOAD_ROW(4, p_input, input_height, input_width, in_y, in_x, \
+ input_offset);
+
+#define COMPUTE(cmds, swizzled_bias_data) \
+ { \
+ vld_w_x_m(v60, swizzled_bias_data); \
+ adwinit_v(v60, v60); \
+ adwconv_vxv(v60, v27, cmds, v0); \
+ adwconv_vxv(v60, v30, cmds, v3); \
+ adwconv_vxv(v60, v33, cmds, v6); \
+ adwconv_vxv(v60, v36, cmds, v9); \
+ adwconv_vxv(v60, v39, cmds, v12); \
+ adwconv_vxv(v60, v42, cmds, v15); \
+ adwconv_vxv(v60, v45, cmds, v18); \
+ adwconv_vxv(v60, v48, cmds, v21); \
+ vdwconv_vxv(v60, v51, cmds, v24); \
+ }
+
+#define OUTPUT(output_activation_min, output_activation_max, output_offset, \
+ local_output_data, n_channels) \
+ { \
+ INT32_TO_INT8_OUTPUT_PIPELINE_INPLACE( \
+ v60, v52, v56, output_activation_min, output_activation_max, \
+ output_offset); \
+ vsraqs_b_vx(v60, v60, 0); \
+ vst_b_l_xx(v60, local_output_data, n_channels); \
+ }
+
+// Estimated count of arithmetic ops: 58.297 M ops, equivalently 29.148 M MACs
+void ConvPerChannelD1OD24_5x5(
+ const tflite::ConvParams& params, const int32_t* output_multiplier,
+ const int32_t* output_shift, const tflite::RuntimeShape& input_shape,
+ const int8_t* input_data, const tflite::RuntimeShape& filter_shape,
+ const int8_t* filter_data, const tflite::RuntimeShape& bias_shape,
+ const int32_t* bias_data, const tflite::RuntimeShape& output_shape,
+ int8_t* output_data) {
+ // Get parameters.
+ const int32_t input_offset = params.input_offset; // r = s(q - Z)
+ const int stride_width = params.stride_width;
+ const int stride_height = params.stride_height;
+ const int dilation_width_factor = params.dilation_width_factor;
+ const int dilation_height_factor = params.dilation_height_factor;
+ const int pad_width = params.padding_values.width;
+ const int pad_height = params.padding_values.height;
+ const int32_t output_offset = params.output_offset;
+
+ // Set min and max value of the output.
+ const int32_t output_activation_min = params.quantized_activation_min;
+ const int32_t output_activation_max = params.quantized_activation_max;
+
+ // Consistency check.
+ TFLITE_DCHECK_LE(output_activation_min, output_activation_max);
+ TFLITE_DCHECK_EQ(input_shape.DimensionsCount(), 4);
+ TFLITE_DCHECK_EQ(filter_shape.DimensionsCount(), 4);
+ TFLITE_DCHECK_EQ(output_shape.DimensionsCount(), 4);
+ // const int batches = tflite::MatchingDim(input_shape, 0, output_shape, 0);
+ const int input_depth = input_shape.Dims(3);
+ const int output_depth =
+ tflite::MatchingDim(filter_shape, 0, output_shape, 3);
+ if (bias_data) {
+ TFLITE_DCHECK_EQ(bias_shape.FlatSize(), output_depth);
+ }
+
+ // Check dimensions of the tensors.
+ const int input_height = input_shape.Dims(1);
+ const int input_width = input_shape.Dims(2);
+ const int filter_height = filter_shape.Dims(1);
+ const int filter_width = filter_shape.Dims(2);
+ const int filter_input_depth = filter_shape.Dims(3);
+ const int groups = input_depth / filter_input_depth;
+ TFLITE_DCHECK_NE(groups, 0);
+ TFLITE_DCHECK_EQ(input_depth % filter_input_depth, 0);
+ const int filters_per_group = output_depth / groups;
+ TFLITE_DCHECK_NE(filters_per_group, 0);
+ const int output_height = output_shape.Dims(1);
+ const int output_width = output_shape.Dims(2);
+
+ // Scratch pads to juggle data
+ const size_t swizzled_filter_data_size = 24 * filter_height * filter_width;
+ std::unique_ptr<int8_t> swizzled_filter_data(reinterpret_cast<int8_t*>(
+ ::aligned_alloc(32, swizzled_filter_data_size)));
+ int32_t swizzled_bias_data[32];
+ int32_t swizzled_output_multiplier[32];
+ int32_t swizzled_output_shift[32];
+ // Transpose filter for easy loading
+ for (int filter_y = 0; filter_y < filter_height; ++filter_y) {
+ for (int filter_x = 0; filter_x < filter_width; ++filter_x) {
+ for (int i = 0; i < 24; i++) {
+ int filter_location =
+ (filter_y * filter_width * 24) + (filter_x * 24) + i;
+ swizzled_filter_data.get()[filter_location] =
+ filter_data[tflite::Offset(filter_shape, i, filter_y, filter_x, 0)];
+ }
+ }
+ }
+ const int8_t* p_flt_0 = swizzled_filter_data.get() + (0 * filter_width * 24);
+ const int8_t* p_flt_1 = swizzled_filter_data.get() + (1 * filter_width * 24);
+ const int8_t* p_flt_2 = swizzled_filter_data.get() + (2 * filter_width * 24);
+ const int8_t* p_flt_3 = swizzled_filter_data.get() + (3 * filter_width * 24);
+ const int8_t* p_flt_4 = swizzled_filter_data.get() + (4 * filter_width * 24);
+ vld_b_l_xx(v0, p_flt_0 + (0 * 24), 24);
+ vld_b_l_xx(v1, p_flt_0 + (1 * 24), 24);
+ vld_b_l_xx(v2, p_flt_0 + (2 * 24), 24);
+ vld_b_l_xx(v3, p_flt_0 + (3 * 24), 24);
+ vld_b_l_xx(v4, p_flt_0 + (4 * 24), 24);
+
+ vld_b_l_xx(v5, p_flt_1 + (0 * 24), 24);
+ vld_b_l_xx(v6, p_flt_1 + (1 * 24), 24);
+ vld_b_l_xx(v7, p_flt_1 + (2 * 24), 24);
+ vld_b_l_xx(v8, p_flt_1 + (3 * 24), 24);
+ vld_b_l_xx(v9, p_flt_1 + (4 * 24), 24);
+
+ vld_b_l_xx(v10, p_flt_2 + (0 * 24), 24);
+ vld_b_l_xx(v11, p_flt_2 + (1 * 24), 24);
+ vld_b_l_xx(v12, p_flt_2 + (2 * 24), 24);
+ vld_b_l_xx(v13, p_flt_2 + (3 * 24), 24);
+ vld_b_l_xx(v14, p_flt_2 + (4 * 24), 24);
+
+ vld_b_l_xx(v15, p_flt_3 + (0 * 24), 24);
+ vld_b_l_xx(v16, p_flt_3 + (1 * 24), 24);
+ vld_b_l_xx(v17, p_flt_3 + (2 * 24), 24);
+ vld_b_l_xx(v18, p_flt_3 + (3 * 24), 24);
+ vld_b_l_xx(v19, p_flt_3 + (4 * 24), 24);
+
+ vld_b_l_xx(v20, p_flt_4 + (0 * 24), 24);
+ vld_b_l_xx(v21, p_flt_4 + (1 * 24), 24);
+ vld_b_l_xx(v22, p_flt_4 + (2 * 24), 24);
+ vld_b_l_xx(v23, p_flt_4 + (3 * 24), 24);
+ vld_b_l_xx(v24, p_flt_4 + (4 * 24), 24);
+ vdup_b_x(v25, 0);
+ vdup_b_x(v26, 0);
+
+ union {
+ vdwconv_u8_t dwconv;
+ uint32_t raw;
+ } cmds;
+ cmds.raw = 0;
+ cmds.dwconv.sdata1 = true;
+ cmds.dwconv.sbias1 = input_offset;
+ cmds.dwconv.sdata2 = true;
+ cmds.dwconv.sbias2 = 0;
+ cmds.dwconv.mode = 0;
+ cmds.dwconv.sparsity = 0;
+ cmds.dwconv.regbase = 0;
+ int out_channel = 0;
+ int n_channels = 24;
+
+ memset(swizzled_bias_data, 0, 32 * sizeof(uint32_t));
+ JumptableSwizzle(bias_data + out_channel, swizzled_bias_data, n_channels);
+ memset(swizzled_output_multiplier, 0, 32 * sizeof(uint32_t));
+ JumptableSwizzle(output_multiplier + out_channel, swizzled_output_multiplier,
+ n_channels);
+ JumptableSwizzle(output_shift + out_channel, swizzled_output_shift,
+ n_channels);
+ vld_w_x_m(v52, swizzled_output_multiplier);
+ vld_w_x_m(v56, swizzled_output_shift);
+ vrsub_w_vx_m(v56, v56, 0);
+
+ int8_t* local_output_data = output_data + out_channel;
+ int in_y[5];
+ int in_x[5];
+ int out_y = 0;
+ const int8_t* p_input = input_data;
+ // Handle top row padding
+ for (; out_y < pad_height; ++out_y) {
+ int out_x = 0;
+ const int in_y_origin = (out_y * stride_height) - pad_height;
+ CALCULATE_IN_Y(in_y_origin);
+ // Left padding required
+ for (; out_x < pad_width; ++out_x) {
+ const int in_x_origin = (out_x * stride_width) - pad_width;
+ CALCULATE_IN_X(in_x_origin);
+ PAD_OR_LOAD_ROW_0(p_input, input_height, input_width, in_y, in_x,
+ input_offset);
+ PAD_OR_LOAD_ROW_1(p_input, input_height, input_width, in_y, in_x,
+ input_offset);
+ PAD_OR_LOAD_ROW_2(p_input, input_height, input_width, in_y, in_x,
+ input_offset);
+ PAD_OR_LOAD_ROW_3(p_input, input_height, input_width, in_y, in_x,
+ input_offset);
+ PAD_OR_LOAD_ROW_4(p_input, input_height, input_width, in_y, in_x,
+ input_offset);
+ COMPUTE(cmds, swizzled_bias_data);
+ OUTPUT(output_activation_min, output_activation_max, output_offset,
+ local_output_data, n_channels);
+ local_output_data += output_depth;
+ }
+ // No side padding
+ for (; out_x < (output_width - pad_width); ++out_x) {
+ const int in_x_origin = (out_x * stride_width) - pad_width;
+ CALCULATE_IN_X(in_x_origin);
+ PAD_OR_LOAD_ROW_0(p_input, input_height, input_width, in_y, in_x,
+ input_offset);
+ PAD_OR_LOAD_ROW_1(p_input, input_height, input_width, in_y, in_x,
+ input_offset);
+ PAD_OR_LOAD_ROW_2(p_input, input_height, input_width, in_y, in_x,
+ input_offset);
+ PAD_OR_LOAD_ROW_3(p_input, input_height, input_width, in_y, in_x,
+ input_offset);
+ PAD_OR_LOAD_ROW_4(p_input, input_height, input_width, in_y, in_x,
+ input_offset);
+ COMPUTE(cmds, swizzled_bias_data);
+ OUTPUT(output_activation_min, output_activation_max, output_offset,
+ local_output_data, n_channels);
+ local_output_data += output_depth;
+ }
+ // Right padding required
+ for (; out_x < output_width; ++out_x) {
+ const int in_x_origin = (out_x * stride_width) - pad_width;
+ CALCULATE_IN_X(in_x_origin);
+ PAD_OR_LOAD_ROW_0(p_input, input_height, input_width, in_y, in_x,
+ input_offset);
+ PAD_OR_LOAD_ROW_1(p_input, input_height, input_width, in_y, in_x,
+ input_offset);
+ PAD_OR_LOAD_ROW_2(p_input, input_height, input_width, in_y, in_x,
+ input_offset);
+ PAD_OR_LOAD_ROW_3(p_input, input_height, input_width, in_y, in_x,
+ input_offset);
+ PAD_OR_LOAD_ROW_4(p_input, input_height, input_width, in_y, in_x,
+ input_offset);
+ COMPUTE(cmds, swizzled_bias_data);
+ OUTPUT(output_activation_min, output_activation_max, output_offset,
+ local_output_data, n_channels);
+ local_output_data += output_depth;
+ }
+ }
+
+ // No height padding
+ for (; out_y < (output_height - pad_height); ++out_y) {
+ const int in_y_origin = (out_y * stride_height) - pad_height;
+ CALCULATE_IN_Y(in_y_origin);
+ // Left padding
+ int out_x = 0;
+ for (; out_x < pad_width; ++out_x) {
+ const int in_x_origin = (out_x * stride_width) - pad_width;
+ CALCULATE_IN_X(in_x_origin);
+ PAD_OR_LOAD_ROW_0(p_input, input_height, input_width, in_y, in_x,
+ input_offset);
+ PAD_OR_LOAD_ROW_1(p_input, input_height, input_width, in_y, in_x,
+ input_offset);
+ PAD_OR_LOAD_ROW_2(p_input, input_height, input_width, in_y, in_x,
+ input_offset);
+ PAD_OR_LOAD_ROW_3(p_input, input_height, input_width, in_y, in_x,
+ input_offset);
+ PAD_OR_LOAD_ROW_4(p_input, input_height, input_width, in_y, in_x,
+ input_offset);
+ COMPUTE(cmds, swizzled_bias_data);
+ OUTPUT(output_activation_min, output_activation_max, output_offset,
+ local_output_data, n_channels);
+ local_output_data += output_depth;
+ }
+ for (; out_x < (output_width - pad_width); ++out_x) {
+ const int in_x_origin = (out_x * stride_width) - pad_width;
+
+ CALCULATE_IN_X(in_x_origin);
+ LOAD_ROW_0(p_input, input_width, in_y, in_x);
+ LOAD_ROW_1(p_input, input_width, in_y, in_x);
+ LOAD_ROW_2(p_input, input_width, in_y, in_x);
+ LOAD_ROW_3(p_input, input_width, in_y, in_x);
+ LOAD_ROW_4(p_input, input_width, in_y, in_x);
+ COMPUTE(cmds, swizzled_bias_data);
+ OUTPUT(output_activation_min, output_activation_max, output_offset,
+ local_output_data, n_channels);
+ local_output_data += output_depth;
+ }
+ // Right padding
+ for (; out_x < output_width; ++out_x) {
+ const int in_x_origin = (out_x * stride_width) - pad_width;
+ CALCULATE_IN_X(in_x_origin);
+ PAD_OR_LOAD_ROW_0(p_input, input_height, input_width, in_y, in_x,
+ input_offset);
+ PAD_OR_LOAD_ROW_1(p_input, input_height, input_width, in_y, in_x,
+ input_offset);
+ PAD_OR_LOAD_ROW_2(p_input, input_height, input_width, in_y, in_x,
+ input_offset);
+ PAD_OR_LOAD_ROW_3(p_input, input_height, input_width, in_y, in_x,
+ input_offset);
+ PAD_OR_LOAD_ROW_4(p_input, input_height, input_width, in_y, in_x,
+ input_offset);
+ COMPUTE(cmds, swizzled_bias_data);
+ OUTPUT(output_activation_min, output_activation_max, output_offset,
+ local_output_data, n_channels);
+ local_output_data += output_depth;
+ }
+ }
+
+ // Handle bottom row padding
+ for (; out_y < output_height; ++out_y) {
+ const int in_y_origin = (out_y * stride_height) - pad_height;
+ CALCULATE_IN_Y(in_y_origin);
+ int out_x = 0;
+ for (; out_x < pad_width; ++out_x) {
+ const int in_x_origin = (out_x * stride_width) - pad_width;
+ CALCULATE_IN_X(in_x_origin);
+ PAD_OR_LOAD_ROW_0(p_input, input_height, input_width, in_y, in_x,
+ input_offset);
+ PAD_OR_LOAD_ROW_1(p_input, input_height, input_width, in_y, in_x,
+ input_offset);
+ PAD_OR_LOAD_ROW_2(p_input, input_height, input_width, in_y, in_x,
+ input_offset);
+ PAD_OR_LOAD_ROW_3(p_input, input_height, input_width, in_y, in_x,
+ input_offset);
+ PAD_OR_LOAD_ROW_4(p_input, input_height, input_width, in_y, in_x,
+ input_offset);
+ COMPUTE(cmds, swizzled_bias_data);
+ OUTPUT(output_activation_min, output_activation_max, output_offset,
+ local_output_data, n_channels);
+ local_output_data += output_depth;
+ }
+ for (; out_x < (output_width - pad_width); ++out_x) {
+ const int in_x_origin = (out_x * stride_width) - pad_width;
+ CALCULATE_IN_X(in_x_origin);
+ PAD_OR_LOAD_ROW_0(p_input, input_height, input_width, in_y, in_x,
+ input_offset);
+ PAD_OR_LOAD_ROW_1(p_input, input_height, input_width, in_y, in_x,
+ input_offset);
+ PAD_OR_LOAD_ROW_2(p_input, input_height, input_width, in_y, in_x,
+ input_offset);
+ PAD_OR_LOAD_ROW_3(p_input, input_height, input_width, in_y, in_x,
+ input_offset);
+ PAD_OR_LOAD_ROW_4(p_input, input_height, input_width, in_y, in_x,
+ input_offset);
+ COMPUTE(cmds, swizzled_bias_data);
+ OUTPUT(output_activation_min, output_activation_max, output_offset,
+ local_output_data, n_channels);
+ local_output_data += output_depth;
+ }
+ for (; out_x < output_width; ++out_x) {
+ const int in_x_origin = (out_x * stride_width) - pad_width;
+ CALCULATE_IN_X(in_x_origin);
+ PAD_OR_LOAD_ROW_0(p_input, input_height, input_width, in_y, in_x,
+ input_offset);
+ PAD_OR_LOAD_ROW_1(p_input, input_height, input_width, in_y, in_x,
+ input_offset);
+ PAD_OR_LOAD_ROW_2(p_input, input_height, input_width, in_y, in_x,
+ input_offset);
+ PAD_OR_LOAD_ROW_3(p_input, input_height, input_width, in_y, in_x,
+ input_offset);
+ PAD_OR_LOAD_ROW_4(p_input, input_height, input_width, in_y, in_x,
+ input_offset);
+ COMPUTE(cmds, swizzled_bias_data);
+ OUTPUT(output_activation_min, output_activation_max, output_offset,
+ local_output_data, n_channels);
+ local_output_data += output_depth;
+ }
+ }
+}
+
+#undef PAD_OR_LOAD_ROW_0
+#undef PAD_OR_LOAD_ROW_1
+#undef PAD_OR_LOAD_ROW_2
+#undef PAD_OR_LOAD_ROW_3
+#undef PAD_OR_LOAD_ROW_4
+#undef _PAD_OR_LOAD_ROW
+#undef _H_PAD_OR_LOAD_ROW
+#undef H_PAD_OR_LOAD_ROW_0
+#undef H_PAD_OR_LOAD_ROW_1
+#undef H_PAD_OR_LOAD_ROW_2
+#undef H_PAD_OR_LOAD_ROW_3
+#undef H_PAD_OR_LOAD_ROW_4
+#undef LOAD_ROW_0
+#undef LOAD_ROW_1
+#undef LOAD_ROW_2
+#undef LOAD_ROW_3
+#undef LOAD_ROW_4
+#undef PAD_ROW_0
+#undef PAD_ROW_1
+#undef PAD_ROW_2
+#undef PAD_ROW_3
+#undef PAD_ROW_4
+#undef CALCULATE_IN_X
+#undef CALCULATE_IN_Y
+
void ConvPerChannelD1(
const tflite::ConvParams& params, const int32_t* output_multiplier,
const int32_t* output_shift, const tflite::RuntimeShape& input_shape,
@@ -207,8 +878,8 @@
v48, v56, v60, output_activation_min, output_activation_max,
output_offset);
vsraqs_b_vx(v48, v48, 0);
- vst_b_l_xx(v48, output_data, n_channels);
- output_data += output_depth;
+ vst_b_l_xx(v48, local_output_data, n_channels);
+ local_output_data += output_depth;
}
}
}
diff --git a/tflm/opt/conv_s8_d4.cc b/tflm/opt/conv_s8_d4.cc
index 54c20bd..26f3885 100644
--- a/tflm/opt/conv_s8_d4.cc
+++ b/tflm/opt/conv_s8_d4.cc
@@ -26,6 +26,7 @@
#include "tensorflow/lite/kernels/internal/reference/integer_ops/conv.h"
#include "tensorflow/lite/kernels/internal/runtime_shape.h"
#include "tensorflow/lite/kernels/internal/types.h"
+#include "tflm/opt/conv_s8.h"
#define unlikely(x) (__builtin_expect(false || (x), false))
#define likely(x) (__builtin_expect(false || (x), true))
@@ -36,7 +37,7 @@
void Filter_N_H_W_M(const int8_t* input, int8_t* output, int N, int H, int W, int M) {
const int8_t(&in)[8][H][W][M] = *(int8_t(*)[8][H][W][M])input;
int8_t(&out)[H][W][M / 4][8][4] = *(int8_t(*)[H][W][M / 4][8][4]) output;
- assert(M >= 4);
+ // assert(M >= 4);
for (int zo = 0; zo < N; ++zo) {
for (int ky = 0; ky < H; ++ky) {
for (int kx = 0; kx < W; ++kx) {
@@ -52,7 +53,7 @@
for (int zo = N; zo < 8; ++zo) {
for (int ky = 0; ky < H; ++ky) {
for (int kx = 0; kx < W; ++kx) {
- for (int zi = 0; zi < M; ++zi) {
+ for (int zi = M; zi < 4; ++zi) {
const int zi_hi = zi >> 2; // div4
const int zi_lo = zi & 3; // rem4
out[ky][kx][zi_hi][zo][zi_lo] = 0;
