Implement DepthwiseConvS85x5D32 w/ adwconv - Use the specialized depthwise conv instructions instead of SIMD, ~10x performance improvement in simulation. Change-Id: I3234e44519c9549acc67cf6b10b6f6a808024bc6
diff --git a/tflm/opt/depthwise_conv_s8.cc b/tflm/opt/depthwise_conv_s8.cc index 4f9440c..a11c3d2 100644 --- a/tflm/opt/depthwise_conv_s8.cc +++ b/tflm/opt/depthwise_conv_s8.cc
@@ -1835,8 +1835,6 @@ const int input_height = input_shape.Dims(1); const int input_width = input_shape.Dims(2); const int input_depth = input_shape.Dims(3); - const int filter_height = filter_shape.Dims(1); - const int filter_width = filter_shape.Dims(2); const int output_height = output_shape.Dims(1); const int output_width = output_shape.Dims(2); const int output_depth = output_shape.Dims(3); @@ -1844,134 +1842,742 @@ int32_t swizzled_shift_multi[32]; int32_t swizzled_output_multi[32]; +#define FLT_0_0 v0 +#define FLT_0_1 v3 +#define FLT_0_2 v6 +#define FLT_0_3 v9 +#define FLT_0_4 v12 +#define FLT_1_0 v1 +#define FLT_1_1 v4 +#define FLT_1_2 v7 +#define FLT_1_3 v10 +#define FLT_1_4 v13 +#define FLT_2_0 v2 +#define FLT_2_1 v5 +#define FLT_2_2 v8 +#define FLT_2_3 v11 +#define FLT_2_4 v14 +#define FLT_3_0 v15 +#define FLT_3_1 v16 +#define FLT_3_2 v17 +#define FLT_3_3 v18 +#define FLT_3_4 v19 +#define FLT_HOLE v20 +#define FLT_4_0 v21 +#define FLT_4_1 v22 +#define FLT_4_2 v23 +#define FLT_4_3 v24 +#define FLT_4_4 v25 + +#define INPUT_0_0 v26 +#define INPUT_0_1 v29 +#define INPUT_0_2 v32 +#define INPUT_0_3 v35 +#define INPUT_0_4 v38 +#define INPUT_1_0 v27 +#define INPUT_1_1 v30 +#define INPUT_1_2 v33 +#define INPUT_1_3 v36 +#define INPUT_1_4 v39 +#define INPUT_2_0 v28 +#define INPUT_2_1 v31 +#define INPUT_2_2 v34 +#define INPUT_2_3 v37 +#define INPUT_2_4 v40 +#define INPUT_3_0 v41 +#define INPUT_3_1 v42 +#define INPUT_3_2 v43 +#define INPUT_3_3 v44 +#define INPUT_3_4 v45 +#define INPUT_4_0 v46 +#define INPUT_4_1 v47 +#define INPUT_4_2 v48 +#define INPUT_4_3 v49 +#define INPUT_4_4 v50 + for (int in_channel = 0; in_channel + 32 <= input_depth; in_channel += 32) { const int output_channel = in_channel; VectorSwizzle(bias_data + output_channel, swizzled_bias_data, 32); VectorSwizzle(output_multiplier + output_channel, swizzled_output_multi, 32); VectorSwizzle(output_shift + output_channel, swizzled_shift_multi, 32); - vld_w_x_m(v52, swizzled_bias_data); + union { + vdwconv_u8_t dwconv; + uint32_t raw; + } cmds; + cmds.raw = 0; + cmds.dwconv.sdata1 = true; + cmds.dwconv.sbias1 = input_offset; + cmds.dwconv.sdata2 = true; + cmds.dwconv.sbias2 = 0; + cmds.dwconv.mode = 0; + cmds.dwconv.sparsity = 0; + cmds.dwconv.regbase = 0; + vld_w_x_m(v56, swizzled_output_multi); vld_w_x_m(v60, swizzled_shift_multi); vrsub_w_vx_m(v60, v60, 0); - // Don't reorder me! const int8_t* p_flt = filter_data + in_channel; - vld_b_sp_xx(v6, p_flt, input_depth); - vld_b_sp_xx(v7, p_flt, input_depth); - vld_b_sp_xx_m(v8, p_flt, input_depth); - vld_b_sp_xx_m(v12, p_flt, input_depth); - vld_b_sp_xx_m(v16, p_flt, input_depth); - vld_b_sp_xx_m(v20, p_flt, input_depth); - vld_b_sp_xx_m(v24, p_flt, input_depth); - vld_b_sp_xx(v28, p_flt, input_depth); - vld_b_sp_xx(v29, p_flt, input_depth); - vld_b_sp_xx(v30, p_flt, input_depth); + vld_b_sp_xx(FLT_0_0, p_flt, input_depth); + vld_b_sp_xx(FLT_0_1, p_flt, input_depth); + vld_b_sp_xx(FLT_0_2, p_flt, input_depth); + vld_b_sp_xx(FLT_0_3, p_flt, input_depth); + vld_b_sp_xx(FLT_0_4, p_flt, input_depth); + vld_b_sp_xx(FLT_1_0, p_flt, input_depth); + vld_b_sp_xx(FLT_1_1, p_flt, input_depth); + vld_b_sp_xx(FLT_1_2, p_flt, input_depth); + vld_b_sp_xx(FLT_1_3, p_flt, input_depth); + vld_b_sp_xx(FLT_1_4, p_flt, input_depth); + + vld_b_sp_xx(FLT_2_0, p_flt, input_depth); + vld_b_sp_xx(FLT_2_1, p_flt, input_depth); + vld_b_sp_xx(FLT_2_2, p_flt, input_depth); + vld_b_sp_xx(FLT_2_3, p_flt, input_depth); + vld_b_sp_xx(FLT_2_4, p_flt, input_depth); + + vld_b_sp_xx(FLT_3_0, p_flt, input_depth); + vld_b_sp_xx(FLT_3_1, p_flt, input_depth); + vld_b_sp_xx(FLT_3_2, p_flt, input_depth); + vld_b_sp_xx(FLT_3_3, p_flt, input_depth); + vld_b_sp_xx(FLT_3_4, p_flt, input_depth); + + vld_b_sp_xx(FLT_4_0, p_flt, input_depth); + vld_b_sp_xx(FLT_4_1, p_flt, input_depth); + vld_b_sp_xx(FLT_4_2, p_flt, input_depth); + vld_b_sp_xx(FLT_4_3, p_flt, input_depth); + vld_b_sp_xx(FLT_4_4, p_flt, input_depth); + vdup_b_x(FLT_HOLE, 0); + +#define COMPUTE() \ + vld_w_x_m(v52, swizzled_bias_data); \ + adwinit_v(v52, v52); \ + adwconv_vxv(v52, INPUT_0_0, cmds, FLT_0_0); \ + adwconv_vxv(v52, INPUT_0_1, cmds, FLT_0_1); \ + adwconv_vxv(v52, INPUT_0_2, cmds, FLT_0_2); \ + adwconv_vxv(v52, INPUT_0_3, cmds, FLT_0_3); \ + adwconv_vxv(v52, INPUT_0_4, cmds, FLT_0_4); \ + adwconv_vxv(v52, INPUT_3_0, cmds, FLT_3_0); \ + adwconv_vxv(v52, INPUT_3_3, cmds, FLT_3_3); \ + adwconv_vxv(v52, INPUT_3_4, cmds, FLT_HOLE); \ + vdwconv_vxv(v52, INPUT_4_2, cmds, FLT_4_2); + +#define INPUT_PTRS() \ + const int8_t* p_input_0 = \ + input_data + (batch * input_height * input_width * input_depth) + \ + (in_y_origin * input_width * input_depth) + \ + (in_x_origin * input_depth) + in_channel; \ + const int8_t* p_input_1 = p_input_0 + (input_width * input_depth); \ + const int8_t* p_input_2 = p_input_1 + (input_width * input_depth); \ + const int8_t* p_input_3 = p_input_2 + (input_width * input_depth); \ + const int8_t* p_input_4 = p_input_3 + (input_width * input_depth); \ + (void)p_input_4; for (int batch = 0; batch < batches; ++batch) { - const int8_t* p_input = input_data + (batch * input_width * input_height * input_depth) + in_channel; - const int8_t* p_output = output_data + (batch * output_width * output_height * output_depth) + output_channel; - for (int out_y = 0; out_y < output_height; ++out_y) { - const int out_y_offset = (out_y * output_width * output_depth); - for (int out_x = 0; out_x < output_width; ++out_x) { + int out_y = 0; + int8_t* p_output = output_data + + (batch * output_height * output_width * output_depth) + + output_channel; + do { + const int in_y_origin = (out_y * stride_height) - pad_height; + if (in_y_origin >= 0) { + break; + } + int out_x = 0; + do { const int in_x_origin = (out_x * stride_width) - pad_width; - const int in_y_origin = (out_y * stride_height) - pad_height; + if (in_x_origin >= 0) { + break; + } + INPUT_PTRS(); +#define LOAD_INPUT(y, x) \ + if (in_y_origin + y < 0) { \ + vdup_b_x(INPUT_##y##_##x, -input_offset); \ + } else if (in_x_origin + x < 0) { \ + vdup_b_x(INPUT_##y##_##x, -input_offset); \ + } else { \ + vld_b_x(INPUT_##y##_##x, p_input_##y + (x * input_depth)); \ + } - // Initialize accumulators w/ bias_data - vmv_v_m(v48, v52); + LOAD_INPUT(0, 0); + LOAD_INPUT(0, 1); + LOAD_INPUT(0, 2); + LOAD_INPUT(0, 3); + LOAD_INPUT(0, 4); + LOAD_INPUT(1, 0); + LOAD_INPUT(1, 1); + LOAD_INPUT(1, 2); + LOAD_INPUT(1, 3); + LOAD_INPUT(1, 4); + LOAD_INPUT(2, 0); + LOAD_INPUT(2, 1); + LOAD_INPUT(2, 2); + LOAD_INPUT(2, 3); + LOAD_INPUT(2, 4); + LOAD_INPUT(3, 0); + LOAD_INPUT(3, 1); + LOAD_INPUT(3, 2); + LOAD_INPUT(3, 3); + LOAD_INPUT(3, 4); + LOAD_INPUT(4, 0); + LOAD_INPUT(4, 1); + LOAD_INPUT(4, 2); + LOAD_INPUT(4, 3); + LOAD_INPUT(4, 4); +#undef LOAD_INPUT + COMPUTE(); + INT32_TO_INT8_OUTPUT_PIPELINE_INPLACE( + v52, v56, v60, output_activation_min, output_activation_max, + output_offset); + vsraqs_b_vx(v52, v52, 0); + vst_b_x(v52, p_output); + p_output += output_depth; + ++out_x; + } while (out_x < output_width); + do { + const int in_x_origin = (out_x * stride_width) - pad_width; + if (in_x_origin + 4 >= input_width) { + break; + } + INPUT_PTRS(); + vdup_b_x(INPUT_0_0, -input_offset); + vdup_b_x(INPUT_0_1, -input_offset); + vdup_b_x(INPUT_0_2, -input_offset); + vdup_b_x(INPUT_0_3, -input_offset); + vdup_b_x(INPUT_0_4, -input_offset); + if (in_y_origin + 1 < 0) { + vdup_b_x(INPUT_1_0, -input_offset); + vdup_b_x(INPUT_1_1, -input_offset); + vdup_b_x(INPUT_1_2, -input_offset); + vdup_b_x(INPUT_1_3, -input_offset); + vdup_b_x(INPUT_1_4, -input_offset); + } else { + vld_b_sp_xx(INPUT_1_0, p_input_1, input_depth); + vld_b_sp_xx(INPUT_1_1, p_input_1, input_depth); + vld_b_sp_xx(INPUT_1_2, p_input_1, input_depth); + vld_b_sp_xx(INPUT_1_3, p_input_1, input_depth); + vld_b_sp_xx(INPUT_1_4, p_input_1, input_depth); + } + if (in_y_origin + 2 < 0) { + vdup_b_x(INPUT_2_0, -input_offset); + vdup_b_x(INPUT_2_1, -input_offset); + vdup_b_x(INPUT_2_2, -input_offset); + vdup_b_x(INPUT_2_3, -input_offset); + vdup_b_x(INPUT_2_4, -input_offset); + } else { + vld_b_sp_xx(INPUT_2_0, p_input_2, input_depth); + vld_b_sp_xx(INPUT_2_1, p_input_2, input_depth); + vld_b_sp_xx(INPUT_2_2, p_input_2, input_depth); + vld_b_sp_xx(INPUT_2_3, p_input_2, input_depth); + vld_b_sp_xx(INPUT_2_4, p_input_2, input_depth); + } + if (in_y_origin + 3 < 0) { + vdup_b_x(INPUT_3_0, -input_offset); + vdup_b_x(INPUT_3_1, -input_offset); + vdup_b_x(INPUT_3_2, -input_offset); + vdup_b_x(INPUT_3_3, -input_offset); + vdup_b_x(INPUT_3_4, -input_offset); + } else { + vld_b_sp_xx(INPUT_3_0, p_input_3, input_depth); + vld_b_sp_xx(INPUT_3_1, p_input_3, input_depth); + vld_b_sp_xx(INPUT_3_2, p_input_3, input_depth); + vld_b_sp_xx(INPUT_3_3, p_input_3, input_depth); + vld_b_sp_xx(INPUT_3_4, p_input_3, input_depth); + } + if (in_y_origin + 4 < 0) { + vdup_b_x(INPUT_4_0, -input_offset); + vdup_b_x(INPUT_4_1, -input_offset); + vdup_b_x(INPUT_4_2, -input_offset); + vdup_b_x(INPUT_4_3, -input_offset); + vdup_b_x(INPUT_4_4, -input_offset); + } else { + vld_b_sp_xx(INPUT_4_0, p_input_4, input_depth); + vld_b_sp_xx(INPUT_4_1, p_input_4, input_depth); + vld_b_sp_xx(INPUT_4_2, p_input_4, input_depth); + vld_b_sp_xx(INPUT_4_3, p_input_4, input_depth); + vld_b_sp_xx(INPUT_4_4, p_input_4, input_depth); + } + COMPUTE(); + INT32_TO_INT8_OUTPUT_PIPELINE_INPLACE( + v52, v56, v60, output_activation_min, output_activation_max, + output_offset); + vsraqs_b_vx(v52, v52, 0); + vst_b_x(v52, p_output); + p_output += output_depth; + ++out_x; + } while (out_x < output_width); + do { + const int in_x_origin = (out_x * stride_width) - pad_width; + INPUT_PTRS(); +#define LOAD_INPUT(y, x) \ + if (in_y_origin + y < 0) { \ + vdup_b_x(INPUT_##y##_##x, -input_offset); \ + } else if (in_x_origin + x >= input_width) { \ + vdup_b_x(INPUT_##y##_##x, -input_offset); \ + } else { \ + vld_b_x(INPUT_##y##_##x, p_input_##y + (x * input_depth)); \ + } - for (int filter_y = 0; filter_y < filter_height; ++filter_y) { - const int in_y = in_y_origin + filter_y; - if ((in_y < 0) || (in_y >= input_height)) { - continue; - } - switch (filter_y) { - case 0: - vaddw_h_vx(v31, v6, 0); - vaddw_h_vx(v33, v7, 0); - vaddw_h_vx(v35, v8, 0); - vaddw_h_vx(v37, v9, 0); - vaddw_h_vx(v39, v10, 0); - break; - case 1: - vaddw_h_vx(v31, v11, 0); - vaddw_h_vx(v33, v12, 0); - vaddw_h_vx(v35, v13, 0); - vaddw_h_vx(v37, v14, 0); - vaddw_h_vx(v39, v15, 0); - break; - case 2: - vaddw_h_vx(v31, v16, 0); - vaddw_h_vx(v33, v17, 0); - vaddw_h_vx(v35, v18, 0); - vaddw_h_vx(v37, v19, 0); - vaddw_h_vx(v39, v20, 0); - break; - case 3: - vaddw_h_vx(v31, v21, 0); - vaddw_h_vx(v33, v22, 0); - vaddw_h_vx(v35, v23, 0); - vaddw_h_vx(v37, v24, 0); - vaddw_h_vx(v39, v25, 0); - break; - case 4: - vaddw_h_vx(v31, v26, 0); - vaddw_h_vx(v33, v27, 0); - vaddw_h_vx(v35, v28, 0); - vaddw_h_vx(v37, v29, 0); - vaddw_h_vx(v39, v30, 0); - break; - } - const int in_y_offset = in_y * input_width * input_depth; - for (int filter_x = 0; filter_x < filter_width; ++filter_x) { - const int in_x = in_x_origin + filter_x; - if ((in_x < 0) || (in_x >= input_width)) { - continue; - } - - vld_b_x(v0, p_input + (in_x * input_depth) + in_y_offset); - - vaddw_h_vx(v0, v0, 0); - vadd_h_vx(v0, v0, static_cast<int16_t>(input_offset)); - vadd_h_vx(v1, v1, - static_cast<int16_t>(input_offset)); // v0 v1 input - switch (filter_x) { - case 0: - vmulw_w_vv(v2, v1, v32); - vmulw_w_vv(v0, v0, v31); - break; - case 1: - vmulw_w_vv(v2, v1, v34); - vmulw_w_vv(v0, v0, v33); - break; - case 2: - vmulw_w_vv(v2, v1, v36); - vmulw_w_vv(v0, v0, v35); - break; - case 3: - vmulw_w_vv(v2, v1, v38); - vmulw_w_vv(v0, v0, v37); - break; - case 4: - vmulw_w_vv(v2, v1, v40); - vmulw_w_vv(v0, v0, v39); - break; - } - vadd_w_vv_m(v48, v48, v0); - } + LOAD_INPUT(0, 0); + LOAD_INPUT(0, 1); + LOAD_INPUT(0, 2); + LOAD_INPUT(0, 3); + LOAD_INPUT(0, 4); + LOAD_INPUT(1, 0); + LOAD_INPUT(1, 1); + LOAD_INPUT(1, 2); + LOAD_INPUT(1, 3); + LOAD_INPUT(1, 4); + LOAD_INPUT(2, 0); + LOAD_INPUT(2, 1); + LOAD_INPUT(2, 2); + LOAD_INPUT(2, 3); + LOAD_INPUT(2, 4); + LOAD_INPUT(3, 0); + LOAD_INPUT(3, 1); + LOAD_INPUT(3, 2); + LOAD_INPUT(3, 3); + LOAD_INPUT(3, 4); + LOAD_INPUT(4, 0); + LOAD_INPUT(4, 1); + LOAD_INPUT(4, 2); + LOAD_INPUT(4, 3); + LOAD_INPUT(4, 4); +#undef LOAD_INPUT + COMPUTE(); + INT32_TO_INT8_OUTPUT_PIPELINE_INPLACE( + v52, v56, v60, output_activation_min, output_activation_max, + output_offset); + vsraqs_b_vx(v52, v52, 0); + vst_b_x(v52, p_output); + p_output += output_depth; + ++out_x; + } while (out_x < output_width); + ++out_y; + } while (out_y < output_height); + do { + const int in_y_origin = (out_y * stride_height) - pad_height; + if (in_y_origin + 4 >= input_height) { + break; + } + int out_x = 0; + do { + const int in_x_origin = (out_x * stride_width) - pad_width; + if (in_x_origin >= 0) { + break; + } + INPUT_PTRS(); + vdup_b_x(INPUT_0_0, -input_offset); + vdup_b_x(INPUT_1_0, -input_offset); + vdup_b_x(INPUT_2_0, -input_offset); + vdup_b_x(INPUT_3_0, -input_offset); + vdup_b_x(INPUT_4_0, -input_offset); + if (in_x_origin + 1 < 0) { + vdup_b_x(INPUT_0_1, -input_offset); + vdup_b_x(INPUT_1_1, -input_offset); + vdup_b_x(INPUT_2_1, -input_offset); + vdup_b_x(INPUT_3_1, -input_offset); + vdup_b_x(INPUT_4_1, -input_offset); + } else { + vld_b_x(INPUT_0_1, p_input_0 + (1 * input_depth)); + vld_b_x(INPUT_1_1, p_input_1 + (1 * input_depth)); + vld_b_x(INPUT_2_1, p_input_2 + (1 * input_depth)); + vld_b_x(INPUT_3_1, p_input_3 + (1 * input_depth)); + vld_b_x(INPUT_4_1, p_input_4 + (1 * input_depth)); + } + if (in_x_origin + 2 < 0) { + vdup_b_x(INPUT_0_2, -input_offset); + vdup_b_x(INPUT_1_2, -input_offset); + vdup_b_x(INPUT_2_2, -input_offset); + vdup_b_x(INPUT_3_2, -input_offset); + vdup_b_x(INPUT_4_2, -input_offset); + } else { + vld_b_x(INPUT_0_2, p_input_0 + (2 * input_depth)); + vld_b_x(INPUT_1_2, p_input_1 + (2 * input_depth)); + vld_b_x(INPUT_2_2, p_input_2 + (2 * input_depth)); + vld_b_x(INPUT_3_2, p_input_3 + (2 * input_depth)); + vld_b_x(INPUT_4_2, p_input_4 + (2 * input_depth)); + } + if (in_x_origin + 3 < 0) { + vdup_b_x(INPUT_0_3, -input_offset); + vdup_b_x(INPUT_1_3, -input_offset); + vdup_b_x(INPUT_2_3, -input_offset); + vdup_b_x(INPUT_3_3, -input_offset); + vdup_b_x(INPUT_4_3, -input_offset); + } else { + vld_b_x(INPUT_0_3, p_input_0 + (3 * input_depth)); + vld_b_x(INPUT_1_3, p_input_1 + (3 * input_depth)); + vld_b_x(INPUT_2_3, p_input_2 + (3 * input_depth)); + vld_b_x(INPUT_3_3, p_input_3 + (3 * input_depth)); + vld_b_x(INPUT_4_3, p_input_4 + (3 * input_depth)); + } + if (in_x_origin + 4 < 0) { + vdup_b_x(INPUT_0_4, -input_offset); + vdup_b_x(INPUT_1_4, -input_offset); + vdup_b_x(INPUT_2_4, -input_offset); + vdup_b_x(INPUT_3_4, -input_offset); + vdup_b_x(INPUT_4_4, -input_offset); + } else { + vld_b_x(INPUT_0_4, p_input_0 + (4 * input_depth)); + vld_b_x(INPUT_1_4, p_input_1 + (4 * input_depth)); + vld_b_x(INPUT_2_4, p_input_2 + (4 * input_depth)); + vld_b_x(INPUT_3_4, p_input_3 + (4 * input_depth)); + vld_b_x(INPUT_4_4, p_input_4 + (4 * input_depth)); } - vdmulh_w_rn_vv_m(v48, v48, v56); - vsha_w_r_vv_m(v48, v48, v60); - vadd_w_vx_m(v48, v48, output_offset); - vmax_w_vx_m(v48, v48, output_activation_min); - vmin_w_vx_m(v48, v48, output_activation_max); - vsraqs_b_vx(v48, v48, 0); - vst_b_x(v48, p_output + out_y_offset + (out_x * output_depth)); - } - } + COMPUTE(); + INT32_TO_INT8_OUTPUT_PIPELINE_INPLACE( + v52, v56, v60, output_activation_min, output_activation_max, + output_offset); + vsraqs_b_vx(v52, v52, 0); + vst_b_x(v52, p_output); + p_output += output_depth; + ++out_x; + } while (out_x < output_width); + do { + const int in_x_origin = (out_x * stride_width) - pad_width; + if (in_x_origin + 4 >= input_width) { + break; + } + INPUT_PTRS(); + vld_b_sp_xx(INPUT_0_0, p_input_0, input_depth); + vld_b_sp_xx(INPUT_0_1, p_input_0, input_depth); + vld_b_sp_xx(INPUT_0_2, p_input_0, input_depth); + vld_b_sp_xx(INPUT_0_3, p_input_0, input_depth); + vld_b_sp_xx(INPUT_0_4, p_input_0, input_depth); + vld_b_sp_xx(INPUT_1_0, p_input_1, input_depth); + vld_b_sp_xx(INPUT_1_1, p_input_1, input_depth); + vld_b_sp_xx(INPUT_1_2, p_input_1, input_depth); + vld_b_sp_xx(INPUT_1_3, p_input_1, input_depth); + vld_b_sp_xx(INPUT_1_4, p_input_1, input_depth); + vld_b_sp_xx(INPUT_2_0, p_input_2, input_depth); + vld_b_sp_xx(INPUT_2_1, p_input_2, input_depth); + vld_b_sp_xx(INPUT_2_2, p_input_2, input_depth); + vld_b_sp_xx(INPUT_2_3, p_input_2, input_depth); + vld_b_sp_xx(INPUT_2_4, p_input_2, input_depth); + vld_b_sp_xx(INPUT_3_0, p_input_3, input_depth); + vld_b_sp_xx(INPUT_3_1, p_input_3, input_depth); + vld_b_sp_xx(INPUT_3_2, p_input_3, input_depth); + vld_b_sp_xx(INPUT_3_3, p_input_3, input_depth); + vld_b_sp_xx(INPUT_3_4, p_input_3, input_depth); + vld_b_sp_xx(INPUT_4_0, p_input_4, input_depth); + vld_b_sp_xx(INPUT_4_1, p_input_4, input_depth); + vld_b_sp_xx(INPUT_4_2, p_input_4, input_depth); + vld_b_sp_xx(INPUT_4_3, p_input_4, input_depth); + vld_b_sp_xx(INPUT_4_4, p_input_4, input_depth); + + COMPUTE(); + INT32_TO_INT8_OUTPUT_PIPELINE_INPLACE( + v52, v56, v60, output_activation_min, output_activation_max, + output_offset); + vsraqs_b_vx(v52, v52, 0); + vst_b_x(v52, p_output); + p_output += output_depth; + ++out_x; + } while (out_x < output_width); + do { + const int in_x_origin = (out_x * stride_width) - pad_width; + INPUT_PTRS(); + if (in_x_origin >= input_width) { + vdup_b_x(INPUT_0_0, -input_offset); + vdup_b_x(INPUT_1_0, -input_offset); + vdup_b_x(INPUT_2_0, -input_offset); + vdup_b_x(INPUT_3_0, -input_offset); + vdup_b_x(INPUT_4_0, -input_offset); + } else { + vld_b_x(INPUT_0_0, p_input_0); + vld_b_x(INPUT_1_0, p_input_1); + vld_b_x(INPUT_2_0, p_input_2); + vld_b_x(INPUT_3_0, p_input_3); + vld_b_x(INPUT_4_0, p_input_4); + } + if (in_x_origin + 1 >= input_width) { + vdup_b_x(INPUT_0_1, -input_offset); + vdup_b_x(INPUT_1_1, -input_offset); + vdup_b_x(INPUT_2_1, -input_offset); + vdup_b_x(INPUT_3_1, -input_offset); + vdup_b_x(INPUT_4_1, -input_offset); + } else { + vld_b_x(INPUT_0_1, p_input_0 + (1 * input_depth)); + vld_b_x(INPUT_1_1, p_input_1 + (1 * input_depth)); + vld_b_x(INPUT_2_1, p_input_2 + (1 * input_depth)); + vld_b_x(INPUT_3_1, p_input_3 + (1 * input_depth)); + vld_b_x(INPUT_4_1, p_input_4 + (1 * input_depth)); + } + if (in_x_origin + 2 >= input_width) { + vdup_b_x(INPUT_0_2, -input_offset); + vdup_b_x(INPUT_1_2, -input_offset); + vdup_b_x(INPUT_2_2, -input_offset); + vdup_b_x(INPUT_3_2, -input_offset); + vdup_b_x(INPUT_4_2, -input_offset); + } else { + vld_b_x(INPUT_0_2, p_input_0 + (2 * input_depth)); + vld_b_x(INPUT_1_2, p_input_1 + (2 * input_depth)); + vld_b_x(INPUT_2_2, p_input_2 + (2 * input_depth)); + vld_b_x(INPUT_3_2, p_input_3 + (2 * input_depth)); + vld_b_x(INPUT_4_2, p_input_4 + (2 * input_depth)); + } + if (in_x_origin + 3 >= input_width) { + vdup_b_x(INPUT_0_3, -input_offset); + vdup_b_x(INPUT_1_3, -input_offset); + vdup_b_x(INPUT_2_3, -input_offset); + vdup_b_x(INPUT_3_3, -input_offset); + vdup_b_x(INPUT_4_3, -input_offset); + } else { + vld_b_x(INPUT_0_3, p_input_0 + (3 * input_depth)); + vld_b_x(INPUT_1_3, p_input_1 + (3 * input_depth)); + vld_b_x(INPUT_2_3, p_input_2 + (3 * input_depth)); + vld_b_x(INPUT_3_3, p_input_3 + (3 * input_depth)); + vld_b_x(INPUT_4_3, p_input_4 + (3 * input_depth)); + } + vdup_b_x(INPUT_0_4, -input_offset); + vdup_b_x(INPUT_1_4, -input_offset); + vdup_b_x(INPUT_2_4, -input_offset); + vdup_b_x(INPUT_3_4, -input_offset); + vdup_b_x(INPUT_4_4, -input_offset); + + COMPUTE(); + INT32_TO_INT8_OUTPUT_PIPELINE_INPLACE( + v52, v56, v60, output_activation_min, output_activation_max, + output_offset); + vsraqs_b_vx(v52, v52, 0); + vst_b_x(v52, p_output); + p_output += output_depth; + ++out_x; + } while (out_x < output_width); + ++out_y; + } while (out_y < output_height); + do { + const int in_y_origin = (out_y * stride_height) - pad_height; + int out_x = 0; + do { + const int in_x_origin = (out_x * stride_width) - pad_width; + if (in_x_origin >= 0) { + break; + } + INPUT_PTRS(); +#define LOAD_INPUT(y, x) \ + if (in_y_origin + y >= input_height) { \ + vdup_b_x(INPUT_##y##_##x, -input_offset); \ + } else if (in_x_origin + x < 0) { \ + vdup_b_x(INPUT_##y##_##x, -input_offset); \ + } else { \ + vld_b_x(INPUT_##y##_##x, p_input_##y + (x * input_depth)); \ + } + + LOAD_INPUT(0, 0); + LOAD_INPUT(0, 1); + LOAD_INPUT(0, 2); + LOAD_INPUT(0, 3); + LOAD_INPUT(0, 4); + LOAD_INPUT(1, 0); + LOAD_INPUT(1, 1); + LOAD_INPUT(1, 2); + LOAD_INPUT(1, 3); + LOAD_INPUT(1, 4); + LOAD_INPUT(2, 0); + LOAD_INPUT(2, 1); + LOAD_INPUT(2, 2); + LOAD_INPUT(2, 3); + LOAD_INPUT(2, 4); + LOAD_INPUT(3, 0); + LOAD_INPUT(3, 1); + LOAD_INPUT(3, 2); + LOAD_INPUT(3, 3); + LOAD_INPUT(3, 4); + LOAD_INPUT(4, 0); + LOAD_INPUT(4, 1); + LOAD_INPUT(4, 2); + LOAD_INPUT(4, 3); + LOAD_INPUT(4, 4); +#undef LOAD_INPUT + COMPUTE(); + INT32_TO_INT8_OUTPUT_PIPELINE_INPLACE( + v52, v56, v60, output_activation_min, output_activation_max, + output_offset); + vsraqs_b_vx(v52, v52, 0); + vst_b_x(v52, p_output); + p_output += output_depth; + ++out_x; + } while (out_x < output_width); + do { + const int in_x_origin = (out_x * stride_width) - pad_width; + if (in_x_origin + 4 >= input_width) { + break; + } + INPUT_PTRS(); + if (in_y_origin >= input_height) { + vdup_b_x(INPUT_0_0, -input_offset); + vdup_b_x(INPUT_0_1, -input_offset); + vdup_b_x(INPUT_0_2, -input_offset); + vdup_b_x(INPUT_0_3, -input_offset); + vdup_b_x(INPUT_0_4, -input_offset); + } else { + vld_b_sp_xx(INPUT_0_0, p_input_0, input_depth); + vld_b_sp_xx(INPUT_0_1, p_input_0, input_depth); + vld_b_sp_xx(INPUT_0_2, p_input_0, input_depth); + vld_b_sp_xx(INPUT_0_3, p_input_0, input_depth); + vld_b_sp_xx(INPUT_0_4, p_input_0, input_depth); + } + if (in_y_origin + 1 >= input_height) { + vdup_b_x(INPUT_1_0, -input_offset); + vdup_b_x(INPUT_1_1, -input_offset); + vdup_b_x(INPUT_1_2, -input_offset); + vdup_b_x(INPUT_1_3, -input_offset); + vdup_b_x(INPUT_1_4, -input_offset); + } else { + vld_b_sp_xx(INPUT_1_0, p_input_1, input_depth); + vld_b_sp_xx(INPUT_1_1, p_input_1, input_depth); + vld_b_sp_xx(INPUT_1_2, p_input_1, input_depth); + vld_b_sp_xx(INPUT_1_3, p_input_1, input_depth); + vld_b_sp_xx(INPUT_1_4, p_input_1, input_depth); + } + if (in_y_origin + 2 >= input_height) { + vdup_b_x(INPUT_2_0, -input_offset); + vdup_b_x(INPUT_2_1, -input_offset); + vdup_b_x(INPUT_2_2, -input_offset); + vdup_b_x(INPUT_2_3, -input_offset); + vdup_b_x(INPUT_2_4, -input_offset); + } else { + vld_b_sp_xx(INPUT_2_0, p_input_2, input_depth); + vld_b_sp_xx(INPUT_2_1, p_input_2, input_depth); + vld_b_sp_xx(INPUT_2_2, p_input_2, input_depth); + vld_b_sp_xx(INPUT_2_3, p_input_2, input_depth); + vld_b_sp_xx(INPUT_2_4, p_input_2, input_depth); + } + if (in_y_origin + 3 >= input_height) { + vdup_b_x(INPUT_3_0, -input_offset); + vdup_b_x(INPUT_3_1, -input_offset); + vdup_b_x(INPUT_3_2, -input_offset); + vdup_b_x(INPUT_3_3, -input_offset); + vdup_b_x(INPUT_3_4, -input_offset); + } else { + vld_b_sp_xx(INPUT_3_0, p_input_3, input_depth); + vld_b_sp_xx(INPUT_3_1, p_input_3, input_depth); + vld_b_sp_xx(INPUT_3_2, p_input_3, input_depth); + vld_b_sp_xx(INPUT_3_3, p_input_3, input_depth); + vld_b_sp_xx(INPUT_3_4, p_input_3, input_depth); + } + vdup_b_x(INPUT_4_0, -input_offset); + vdup_b_x(INPUT_4_1, -input_offset); + vdup_b_x(INPUT_4_2, -input_offset); + vdup_b_x(INPUT_4_3, -input_offset); + vdup_b_x(INPUT_4_4, -input_offset); + + COMPUTE(); + INT32_TO_INT8_OUTPUT_PIPELINE_INPLACE( + v52, v56, v60, output_activation_min, output_activation_max, + output_offset); + vsraqs_b_vx(v52, v52, 0); + vst_b_x(v52, p_output); + p_output += output_depth; + ++out_x; + } while (out_x < output_width); + do { + const int in_x_origin = (out_x * stride_width) - pad_width; + INPUT_PTRS(); +#define LOAD_INPUT(y, x) \ + if (in_y_origin + y >= input_height) { \ + vdup_b_x(INPUT_##y##_##x, -input_offset); \ + } else if (in_x_origin + x >= input_width) { \ + vdup_b_x(INPUT_##y##_##x, -input_offset); \ + } else { \ + vld_b_x(INPUT_##y##_##x, p_input_##y + (x * input_depth)); \ + } + + LOAD_INPUT(0, 0); + LOAD_INPUT(0, 1); + LOAD_INPUT(0, 2); + LOAD_INPUT(0, 3); + LOAD_INPUT(0, 4); + LOAD_INPUT(1, 0); + LOAD_INPUT(1, 1); + LOAD_INPUT(1, 2); + LOAD_INPUT(1, 3); + LOAD_INPUT(1, 4); + LOAD_INPUT(2, 0); + LOAD_INPUT(2, 1); + LOAD_INPUT(2, 2); + LOAD_INPUT(2, 3); + LOAD_INPUT(2, 4); + LOAD_INPUT(3, 0); + LOAD_INPUT(3, 1); + LOAD_INPUT(3, 2); + LOAD_INPUT(3, 3); + LOAD_INPUT(3, 4); + LOAD_INPUT(4, 0); + LOAD_INPUT(4, 1); + LOAD_INPUT(4, 2); + LOAD_INPUT(4, 3); + LOAD_INPUT(4, 4); +#undef LOAD_INPUT + COMPUTE(); + INT32_TO_INT8_OUTPUT_PIPELINE_INPLACE( + v52, v56, v60, output_activation_min, output_activation_max, + output_offset); + vsraqs_b_vx(v52, v52, 0); + vst_b_x(v52, p_output); + p_output += output_depth; + ++out_x; + } while (out_x < output_width); + ++out_y; + } while (out_y < output_height); } + +#undef COMPUTE +#undef INPUT_PTRS +#undef FLT_0_0 +#undef FLT_0_1 +#undef FLT_0_2 +#undef FLT_0_3 +#undef FLT_0_4 +#undef FLT_1_0 +#undef FLT_1_1 +#undef FLT_1_2 +#undef FLT_1_3 +#undef FLT_1_4 +#undef FLT_2_0 +#undef FLT_2_1 +#undef FLT_2_2 +#undef FLT_2_3 +#undef FLT_2_4 +#undef FLT_3_0 +#undef FLT_3_1 +#undef FLT_3_2 +#undef FLT_3_3 +#undef FLT_3_4 +#undef FLT_HOLE +#undef FLT_4_0 +#undef FLT_4_1 +#undef FLT_4_2 +#undef FLT_4_3 +#undef FLT_4_4 +#undef INPUT_0_0 +#undef INPUT_0_1 +#undef INPUT_0_2 +#undef INPUT_0_3 +#undef INPUT_0_4 +#undef INPUT_1_0 +#undef INPUT_1_1 +#undef INPUT_1_2 +#undef INPUT_1_3 +#undef INPUT_1_4 +#undef INPUT_2_0 +#undef INPUT_2_1 +#undef INPUT_2_2 +#undef INPUT_2_3 +#undef INPUT_2_4 +#undef INPUT_3_0 +#undef INPUT_3_1 +#undef INPUT_3_2 +#undef INPUT_3_3 +#undef INPUT_3_4 +#undef INPUT_4_0 +#undef INPUT_4_1 +#undef INPUT_4_2 +#undef INPUT_4_3 +#undef INPUT_4_4 } }