DepthwiseConv 3x3, 4 per iteration - Improve the readability of the 3x3 kernel a bit w/ names for registers - Process 4 outputs per iteration in the hot loop, and do a bit of instruction scheduling Change-Id: I8e024ddde830f7586347bd976116d7a48c32fb96
diff --git a/tflm/opt/depthwise_conv_s8.cc b/tflm/opt/depthwise_conv_s8.cc index b839c6f..aa7b6bb 100644 --- a/tflm/opt/depthwise_conv_s8.cc +++ b/tflm/opt/depthwise_conv_s8.cc
@@ -95,55 +95,88 @@ cmds.dwconv.sparsity = 0; cmds.dwconv.regbase = 0; +#define FLT_0_0 v0 +#define FLT_0_1 v3 +#define FLT_0_2 v6 +#define FLT_1_0 v1 +#define FLT_1_1 v4 +#define FLT_1_2 v7 +#define FLT_2_0 v2 +#define FLT_2_1 v5 +#define FLT_2_2 v8 + +#define INPUT_0_0 v9 +#define INPUT_0_1 v12 +#define INPUT_0_2 v15 +#define INPUT_0_3 v18 +#define INPUT_0_4 v21 +#define INPUT_0_5 v24 +#define INPUT_1_0 v10 +#define INPUT_1_1 v13 +#define INPUT_1_2 v16 +#define INPUT_1_3 v19 +#define INPUT_1_4 v22 +#define INPUT_1_5 v25 +#define INPUT_2_0 v11 +#define INPUT_2_1 v14 +#define INPUT_2_2 v17 +#define INPUT_2_3 v20 +#define INPUT_2_4 v23 +#define INPUT_2_5 v26 + +#define INPUT_PTRS(_strides) \ + const int in_y_origin = (out_y * stride_height) - pad_height; \ + const int in_x_origin = (out_x * stride_width) - pad_width; \ + const int8_t* p_in_0 = input_data + \ + (batch * input_height * input_width * input_depth) + \ + (in_y_origin * input_width * input_depth) + \ + ((in_x_origin + _strides) * input_depth) + \ + in_channel; \ + const int8_t* p_in_1 = p_in_0 + (input_width * input_depth); \ + const int8_t* p_in_2 = p_in_1 + (input_width * input_depth); \ + (void)p_in_2; + +#define COMPUTE() \ + adwinit_v(v48, v48); \ + adwconv_vxv(v48, INPUT_0_0, cmds, FLT_0_0); \ + adwconv_vxv(v48, INPUT_0_1, cmds, FLT_0_1); \ + vdwconv_vxv(v48, INPUT_0_2, cmds, FLT_0_2); + // Don't reorder me, otherwise data will not be // loaded in the correct order // (we can reuse the p_flt* due to the `p` vld variant). const int8_t* p_flt0 = filter_data + in_channel; const int8_t* p_flt1 = p_flt0 + input_depth; const int32_t stride = 2 * input_depth; - vld_b_sp_xx(v6, p_flt0, stride); - vld_b_sp_xx(v7, p_flt1, stride); - vld_b_sp_xx(v8, p_flt0, stride); - vld_b_sp_xx(v9, p_flt1, stride); - vld_b_sp_xx(v10, p_flt0, stride); - vld_b_sp_xx(v11, p_flt1, stride); - vld_b_sp_xx(v12, p_flt0, stride); - vld_b_sp_xx(v13, p_flt1, stride); - vld_b_sp_xx(v14, p_flt0, stride); + vld_b_sp_xx(FLT_0_0, p_flt0, stride); + vld_b_sp_xx(FLT_0_1, p_flt1, stride); + vld_b_sp_xx(FLT_0_2, p_flt0, stride); + vld_b_sp_xx(FLT_1_0, p_flt1, stride); + vld_b_sp_xx(FLT_1_1, p_flt0, stride); + vld_b_sp_xx(FLT_1_2, p_flt1, stride); + vld_b_sp_xx(FLT_2_0, p_flt0, stride); + vld_b_sp_xx(FLT_2_1, p_flt1, stride); + vld_b_sp_xx(FLT_2_2, p_flt0, stride); for (int batch = 0; batch < batches; ++batch) { int out_y = 0; for (; out_y < pad_height; ++out_y) { int out_x = 0; - const int in_y_origin = (out_y * stride_height) - pad_height; - assert(in_y_origin < 0); - vdup_b_x(v15, -input_offset); - vdup_b_x(v16, -input_offset); - vdup_b_x(v17, -input_offset); - const int8_t* p_in_0 = input_data + - (batch * input_height * input_width * input_depth) + - (in_y_origin * input_width * input_depth) + - (((out_x * stride_width) - pad_width) * input_depth) + - in_channel; - const int8_t* p_in_1 = p_in_0 + (input_width * input_depth); - const int8_t* p_in_2 = p_in_1 + (input_width * input_depth); + vdup_b_x(INPUT_0_0, -input_offset); + vdup_b_x(INPUT_0_1, -input_offset); + vdup_b_x(INPUT_0_2, -input_offset); for (; out_x < pad_width; ++out_x) { + INPUT_PTRS(1); vmv_v_m(v48, v52); - vdup_b_x(v18, -input_offset); - p_in_1 += input_depth; - vld_b_sp_xx(v19, p_in_1, input_depth); - vld_b_sp_xx(v20, p_in_1, input_depth); - vdup_b_x(v21, -input_offset); - p_in_2 += input_depth; - vld_b_sp_xx(v22, p_in_2, input_depth); - vld_b_sp_xx(v23, p_in_2, input_depth); + vdup_b_x(INPUT_1_0, -input_offset); + vld_b_sp_xx(INPUT_1_1, p_in_1, input_depth); + vld_b_sp_xx(INPUT_1_2, p_in_1, input_depth); + vdup_b_x(INPUT_2_0, -input_offset); + vld_b_sp_xx(INPUT_2_1, p_in_2, input_depth); + vld_b_sp_xx(INPUT_2_2, p_in_2, input_depth); - adwinit_v(v48, v48); - adwconv_vxv(v48, v15, cmds, v6); - adwconv_vxv(v48, v18, cmds, v9); - vdwconv_vxv(v48, v21, cmds, v12); - + COMPUTE(); INT32_TO_INT8_OUTPUT_PIPELINE_INPLACE( v48, v56, v60, output_activation_min, @@ -153,24 +186,18 @@ vst_b_x(v48, p_output); p_output += output_depth; - p_in_0 -= (2 * stride_width * input_depth); - p_in_1 -= (2 * stride_width * input_depth); - p_in_2 -= (2 * stride_width * input_depth); } for (; out_x < output_width - pad_width; ++out_x) { + INPUT_PTRS(0); vmv_v_m(v48, v52); - vld_b_sp_xx(v18, p_in_1, input_depth); - vld_b_sp_xx(v19, p_in_1, input_depth); - vld_b_sp_xx(v20, p_in_1, input_depth); - vld_b_sp_xx(v21, p_in_2, input_depth); - vld_b_sp_xx(v22, p_in_2, input_depth); - vld_b_sp_xx(v23, p_in_2, input_depth); + vld_b_sp_xx(INPUT_1_0, p_in_1, input_depth); + vld_b_sp_xx(INPUT_1_1, p_in_1, input_depth); + vld_b_sp_xx(INPUT_1_2, p_in_1, input_depth); + vld_b_sp_xx(INPUT_2_0, p_in_2, input_depth); + vld_b_sp_xx(INPUT_2_1, p_in_2, input_depth); + vld_b_sp_xx(INPUT_2_2, p_in_2, input_depth); - adwinit_v(v48, v48); - adwconv_vxv(v48, v15, cmds, v6); - adwconv_vxv(v48, v18, cmds, v9); - vdwconv_vxv(v48, v21, cmds, v12); - + COMPUTE(); INT32_TO_INT8_OUTPUT_PIPELINE_INPLACE( v48, v56, v60, output_activation_min, @@ -179,25 +206,19 @@ vsraqs_b_vx(v48, v48, 0); vst_b_x(v48, p_output); p_output += output_depth; - p_in_0 -= (2 * stride_width * input_depth); - p_in_1 -= (2 * stride_width * input_depth); - p_in_2 -= (2 * stride_width * input_depth); } for (; out_x < output_width; ++out_x) { + INPUT_PTRS(0); vmv_v_m(v48, v52); - vld_b_sp_xx(v18, p_in_1, input_depth); - vld_b_sp_xx(v19, p_in_1, input_depth); - vdup_b_x(v20, -input_offset); - vld_b_sp_xx(v21, p_in_2, input_depth); - vld_b_sp_xx(v22, p_in_2, input_depth); - vdup_b_x(v23, -input_offset); + vld_b_sp_xx(INPUT_1_0, p_in_1, input_depth); + vld_b_sp_xx(INPUT_1_1, p_in_1, input_depth); + vdup_b_x(INPUT_1_2, -input_offset); + vld_b_sp_xx(INPUT_2_0, p_in_2, input_depth); + vld_b_sp_xx(INPUT_2_1, p_in_2, input_depth); + vdup_b_x(INPUT_2_2, -input_offset); - adwinit_v(v48, v48); - adwconv_vxv(v48, v15, cmds, v6); - adwconv_vxv(v48, v18, cmds, v9); - vdwconv_vxv(v48, v21, cmds, v12); - + COMPUTE(); INT32_TO_INT8_OUTPUT_PIPELINE_INPLACE( v48, v56, v60, output_activation_min, @@ -207,41 +228,25 @@ vst_b_x(v48, p_output); p_output += output_depth; - p_in_0 -= (2 * stride_width * input_depth); - p_in_1 -= (2 * stride_width * input_depth); - p_in_2 -= (2 * stride_width * input_depth); } } for (; out_y < output_height - pad_height; ++out_y) { - const int in_y_origin = (out_y * stride_height) - pad_height; int out_x = 0; - const int8_t* p_in_0 = input_data + - (batch * input_height * input_width * input_depth) + - (in_y_origin * input_width * input_depth) + - (((out_x * stride_width) - pad_width) * input_depth) + - in_channel; - const int8_t* p_in_1 = p_in_0 + (input_width * input_depth); - const int8_t* p_in_2 = p_in_1 + (input_width * input_depth); for (; out_x < pad_width; ++out_x) { + INPUT_PTRS(1); vmv_v_m(v48, v52); - vdup_b_x(v15, -input_offset); - vdup_b_x(v18, -input_offset); - vdup_b_x(v21, -input_offset); - p_in_0 += input_depth; - p_in_1 += input_depth; - p_in_2 += input_depth; - vld_b_sp_xx(v16, p_in_0, input_depth); - vld_b_sp_xx(v17, p_in_0, input_depth); - vld_b_sp_xx(v19, p_in_1, input_depth); - vld_b_sp_xx(v20, p_in_1, input_depth); - vld_b_sp_xx(v22, p_in_2, input_depth); - vld_b_sp_xx(v23, p_in_2, input_depth); + vdup_b_x(INPUT_0_0, -input_offset); + vdup_b_x(INPUT_1_0, -input_offset); + vdup_b_x(INPUT_2_0, -input_offset); + vld_b_sp_xx(INPUT_0_1, p_in_0, input_depth); + vld_b_sp_xx(INPUT_0_2, p_in_0, input_depth); + vld_b_sp_xx(INPUT_1_1, p_in_1, input_depth); + vld_b_sp_xx(INPUT_1_2, p_in_1, input_depth); + vld_b_sp_xx(INPUT_2_1, p_in_2, input_depth); + vld_b_sp_xx(INPUT_2_2, p_in_2, input_depth); - adwinit_v(v48, v48); - adwconv_vxv(v48, v15, cmds, v6); - adwconv_vxv(v48, v18, cmds, v9); - vdwconv_vxv(v48, v21, cmds, v12); + COMPUTE(); INT32_TO_INT8_OUTPUT_PIPELINE_INPLACE( v48, v56, v60, output_activation_min, @@ -251,77 +256,108 @@ vst_b_x(v48, p_output); p_output += output_depth; - p_in_0 -= (2 * stride_width * input_depth); - p_in_1 -= (2 * stride_width * input_depth); - p_in_2 -= (2 * stride_width * input_depth); } - for (; out_x + 2 <= output_width - pad_width; out_x += 2) { + for (; out_x + 4 <= output_width - pad_width; out_x += 4) { + INPUT_PTRS(0); // Initialize accumulators w/ bias data. + vmv_v_m(v36, v52); + vmv_v_m(v40, v52); vmv_v_m(v44, v52); vmv_v_m(v48, v52); - vld_b_sp_xx(v15, p_in_0, stride_width * input_depth); - vld_b_sp_xx(v16, p_in_0, stride_width * input_depth); - vld_b_sp_xx(v17, p_in_0, stride_width * input_depth); - vld_b_sp_xx(v18, p_in_0, stride_width * input_depth); - vld_b_sp_xx(v19, p_in_1, stride_width * input_depth); - vld_b_sp_xx(v20, p_in_1, stride_width * input_depth); - vld_b_sp_xx(v21, p_in_1, stride_width * input_depth); - vld_b_sp_xx(v22, p_in_1, stride_width * input_depth); - vld_b_sp_xx(v23, p_in_2, stride_width * input_depth); - vld_b_sp_xx(v24, p_in_2, stride_width * input_depth); - vld_b_sp_xx(v25, p_in_2, stride_width * input_depth); - vld_b_sp_xx(v26, p_in_2, stride_width * input_depth); + vld_b_sp_xx(INPUT_0_0, p_in_0, stride_width * input_depth); + vld_b_sp_xx(INPUT_1_0, p_in_1, stride_width * input_depth); + vld_b_sp_xx(INPUT_2_0, p_in_2, stride_width * input_depth); + vld_b_sp_xx(INPUT_0_1, p_in_0, stride_width * input_depth); + vld_b_sp_xx(INPUT_1_1, p_in_1, stride_width * input_depth); + vld_b_sp_xx(INPUT_2_1, p_in_2, stride_width * input_depth); + vld_b_sp_xx(INPUT_0_2, p_in_0, stride_width * input_depth); + vld_b_sp_xx(INPUT_1_2, p_in_1, stride_width * input_depth); + vld_b_sp_xx(INPUT_2_2, p_in_2, stride_width * input_depth); adwinit_v(v48, v48); - adwconv_vxv(v48, v15, cmds, v6); - adwconv_vxv(v48, v19, cmds, v9); - vdwconv_vxv(v48, v23, cmds, v12); + adwconv_vxv(v48, INPUT_0_0, cmds, FLT_0_0); + adwconv_vxv(v48, INPUT_0_1, cmds, FLT_0_1); + vdwconv_vxv(v48, INPUT_0_2, cmds, FLT_0_2); + + vld_b_sp_xx(INPUT_0_3, p_in_0, stride_width * input_depth); + vld_b_sp_xx(INPUT_1_3, p_in_1, stride_width * input_depth); + vld_b_sp_xx(INPUT_2_3, p_in_2, stride_width * input_depth); adwinit_v(v44, v44); - adwconv_vxv(v44, v16, cmds, v6); - adwconv_vxv(v44, v20, cmds, v9); - vdwconv_vxv(v44, v24, cmds, v12); + adwconv_vxv(v44, INPUT_0_1, cmds, FLT_0_0); + adwconv_vxv(v44, INPUT_0_2, cmds, FLT_0_1); + vdwconv_vxv(v44, INPUT_0_3, cmds, FLT_0_2); + + vld_b_sp_xx(INPUT_0_4, p_in_0, stride_width * input_depth); + vld_b_sp_xx(INPUT_1_4, p_in_1, stride_width * input_depth); + vld_b_sp_xx(INPUT_2_4, p_in_2, stride_width * input_depth); + + adwinit_v(v40, v40); + adwconv_vxv(v40, INPUT_0_2, cmds, FLT_0_0); + adwconv_vxv(v40, INPUT_0_3, cmds, FLT_0_1); + vdwconv_vxv(v40, INPUT_0_4, cmds, FLT_0_2); + + vld_b_sp_xx(INPUT_0_5, p_in_0, stride_width * input_depth); + vld_b_sp_xx(INPUT_1_5, p_in_1, stride_width * input_depth); + vld_b_sp_xx(INPUT_2_5, p_in_2, stride_width * input_depth); + + adwinit_v(v36, v36); + adwconv_vxv(v36, INPUT_0_3, cmds, FLT_0_0); + adwconv_vxv(v36, INPUT_0_4, cmds, FLT_0_1); + vdwconv_vxv(v36, INPUT_0_5, cmds, FLT_0_2); + + INT32_TO_INT8_OUTPUT_PIPELINE_INPLACE( + v48, v56, v60, + output_activation_min, + output_activation_max, + output_offset); + vsraqs_b_vx(v48, v48, 0); + vst_b_x(v48, p_output); + p_output += output_depth; INT32_TO_INT8_OUTPUT_PIPELINE_INPLACE( v44, v56, v60, output_activation_min, output_activation_max, output_offset); - INT32_TO_INT8_OUTPUT_PIPELINE_INPLACE( - v48, v56, v60, - output_activation_min, - output_activation_max, - output_offset); - vsraqs_b_vx(v48, v48, 0); vsraqs_b_vx(v44, v44, 0); - vst_b_x(v48, p_output); - p_output += output_depth; vst_b_x(v44, p_output); p_output += output_depth; - p_in_0 -= (2 * stride_width * input_depth); - p_in_1 -= (2 * stride_width * input_depth); - p_in_2 -= (2 * stride_width * input_depth); + INT32_TO_INT8_OUTPUT_PIPELINE_INPLACE( + v40, v56, v60, + output_activation_min, + output_activation_max, + output_offset); + vsraqs_b_vx(v40, v40, 0); + vst_b_x(v40, p_output); + p_output += output_depth; + + INT32_TO_INT8_OUTPUT_PIPELINE_INPLACE( + v36, v56, v60, + output_activation_min, + output_activation_max, + output_offset); + vsraqs_b_vx(v36, v36, 0); + vst_b_x(v36, p_output); + p_output += output_depth; } for (; out_x < output_width - pad_width; ++out_x) { + INPUT_PTRS(0); vmv_v_m(v48, v52); - vld_b_sp_xx(v15, p_in_0, input_depth); - vld_b_sp_xx(v16, p_in_0, input_depth); - vld_b_sp_xx(v17, p_in_0, input_depth); - vld_b_sp_xx(v18, p_in_1, input_depth); - vld_b_sp_xx(v19, p_in_1, input_depth); - vld_b_sp_xx(v20, p_in_1, input_depth); - vld_b_sp_xx(v21, p_in_2, input_depth); - vld_b_sp_xx(v22, p_in_2, input_depth); - vld_b_sp_xx(v23, p_in_2, input_depth); + vld_b_sp_xx(INPUT_0_0, p_in_0, input_depth); + vld_b_sp_xx(INPUT_0_1, p_in_0, input_depth); + vld_b_sp_xx(INPUT_0_2, p_in_0, input_depth); + vld_b_sp_xx(INPUT_1_0, p_in_1, input_depth); + vld_b_sp_xx(INPUT_1_1, p_in_1, input_depth); + vld_b_sp_xx(INPUT_1_2, p_in_1, input_depth); + vld_b_sp_xx(INPUT_2_0, p_in_2, input_depth); + vld_b_sp_xx(INPUT_2_1, p_in_2, input_depth); + vld_b_sp_xx(INPUT_2_2, p_in_2, input_depth); - adwinit_v(v48, v48); - adwconv_vxv(v48, v15, cmds, v6); - adwconv_vxv(v48, v18, cmds, v9); - vdwconv_vxv(v48, v21, cmds, v12); - + COMPUTE(); INT32_TO_INT8_OUTPUT_PIPELINE_INPLACE( v48, v56, v60, output_activation_min, @@ -330,27 +366,22 @@ vsraqs_b_vx(v48, v48, 0); vst_b_x(v48, p_output); p_output += output_depth; - p_in_0 -= (2 * stride_width * input_depth); - p_in_1 -= (2 * stride_width * input_depth); - p_in_2 -= (2 * stride_width * input_depth); } for (; out_x < output_width; ++out_x) { + INPUT_PTRS(0); vmv_v_m(v48, v52); - vdup_b_x(v17, -input_offset); - vdup_b_x(v20, -input_offset); - vdup_b_x(v23, -input_offset); - vld_b_sp_xx(v15, p_in_0, input_depth); - vld_b_sp_xx(v16, p_in_0, input_depth); - vld_b_sp_xx(v18, p_in_1, input_depth); - vld_b_sp_xx(v19, p_in_1, input_depth); - vld_b_sp_xx(v21, p_in_2, input_depth); - vld_b_sp_xx(v22, p_in_2, input_depth); + vdup_b_x(INPUT_0_2, -input_offset); + vdup_b_x(INPUT_1_2, -input_offset); + vdup_b_x(INPUT_2_2, -input_offset); + vld_b_sp_xx(INPUT_0_0, p_in_0, input_depth); + vld_b_sp_xx(INPUT_0_1, p_in_0, input_depth); + vld_b_sp_xx(INPUT_1_0, p_in_1, input_depth); + vld_b_sp_xx(INPUT_1_1, p_in_1, input_depth); + vld_b_sp_xx(INPUT_2_0, p_in_2, input_depth); + vld_b_sp_xx(INPUT_2_1, p_in_2, input_depth); - adwinit_v(v48, v48); - adwconv_vxv(v48, v15, cmds, v6); - adwconv_vxv(v48, v18, cmds, v9); - vdwconv_vxv(v48, v21, cmds, v12); + COMPUTE(); INT32_TO_INT8_OUTPUT_PIPELINE_INPLACE( v48, v56, v60, output_activation_min, @@ -360,41 +391,25 @@ vst_b_x(v48, p_output); p_output += output_depth; - p_in_0 -= (2 * stride_width * input_depth); - p_in_1 -= (2 * stride_width * input_depth); - p_in_2 -= (2 * stride_width * input_depth); } } for (; out_y < output_height; ++out_y) { - const int in_y_origin = (out_y * stride_height) - pad_height; - assert(in_y_origin + 2 >= input_height); - vdup_b_x(v21, -input_offset); - vdup_b_x(v22, -input_offset); - vdup_b_x(v23, -input_offset); + vdup_b_x(INPUT_2_0, -input_offset); + vdup_b_x(INPUT_2_1, -input_offset); + vdup_b_x(INPUT_2_2, -input_offset); int out_x = 0; - const int8_t* p_in_0 = input_data + - (batch * input_height * input_width * input_depth) + - (in_y_origin * input_width * input_depth) + - (((out_x * stride_width) - pad_width) * input_depth) + - in_channel; - const int8_t* p_in_1 = p_in_0 + (input_width * input_depth); for (; out_x < pad_width; ++out_x) { + INPUT_PTRS(1); vmv_v_m(v48, v52); - vdup_b_x(v15, -input_offset); - p_in_0 += input_depth; - vld_b_sp_xx(v16, p_in_0, input_depth); - vld_b_sp_xx(v17, p_in_0, input_depth); - vdup_b_x(v18, -input_offset); - p_in_1 += input_depth; - vld_b_sp_xx(v19, p_in_1, input_depth); - vld_b_sp_xx(v20, p_in_1, input_depth); + vdup_b_x(INPUT_0_0, -input_offset); + vld_b_sp_xx(INPUT_0_1, p_in_0, input_depth); + vld_b_sp_xx(INPUT_0_2, p_in_0, input_depth); + vdup_b_x(INPUT_1_0, -input_offset); + vld_b_sp_xx(INPUT_1_1, p_in_1, input_depth); + vld_b_sp_xx(INPUT_1_2, p_in_1, input_depth); - adwinit_v(v48, v48); - adwconv_vxv(v48, v15, cmds, v6); - adwconv_vxv(v48, v18, cmds, v9); - vdwconv_vxv(v48, v21, cmds, v12); - + COMPUTE(); INT32_TO_INT8_OUTPUT_PIPELINE_INPLACE( v48, v56, v60, output_activation_min, @@ -403,23 +418,18 @@ vsraqs_b_vx(v48, v48, 0); vst_b_x(v48, p_output); p_output += output_depth; - p_in_0 -= (2 * stride_width * input_depth); - p_in_1 -= (2 * stride_width * input_depth); } for (; out_x < output_width - pad_width; ++out_x) { + INPUT_PTRS(0); vmv_v_m(v48, v52); - vld_b_sp_xx(v15, p_in_0, input_depth); - vld_b_sp_xx(v16, p_in_0, input_depth); - vld_b_sp_xx(v17, p_in_0, input_depth); - vld_b_sp_xx(v18, p_in_1, input_depth); - vld_b_sp_xx(v19, p_in_1, input_depth); - vld_b_sp_xx(v20, p_in_1, input_depth); + vld_b_sp_xx(INPUT_0_0, p_in_0, input_depth); + vld_b_sp_xx(INPUT_0_1, p_in_0, input_depth); + vld_b_sp_xx(INPUT_0_2, p_in_0, input_depth); + vld_b_sp_xx(INPUT_1_0, p_in_1, input_depth); + vld_b_sp_xx(INPUT_1_1, p_in_1, input_depth); + vld_b_sp_xx(INPUT_1_2, p_in_1, input_depth); - adwinit_v(v48, v48); - adwconv_vxv(v48, v15, cmds, v6); - adwconv_vxv(v48, v18, cmds, v9); - vdwconv_vxv(v48, v21, cmds, v12); - + COMPUTE(); INT32_TO_INT8_OUTPUT_PIPELINE_INPLACE( v48, v56, v60, output_activation_min, @@ -428,24 +438,19 @@ vsraqs_b_vx(v48, v48, 0); vst_b_x(v48, p_output); p_output += output_depth; - p_in_0 -= (2 * stride_width * input_depth); - p_in_1 -= (2 * stride_width * input_depth); } for (; out_x < output_width; ++out_x) { + INPUT_PTRS(0); vmv_v_m(v48, v52); - vld_b_sp_xx(v15, p_in_0, input_depth); - vld_b_sp_xx(v16, p_in_0, input_depth); - vdup_b_x(v17, -input_offset); - vld_b_sp_xx(v18, p_in_1, input_depth); - vld_b_sp_xx(v19, p_in_1, input_depth); - vdup_b_x(v20, -input_offset); + vld_b_sp_xx(INPUT_0_0, p_in_0, input_depth); + vld_b_sp_xx(INPUT_0_1, p_in_0, input_depth); + vdup_b_x(INPUT_0_2, -input_offset); + vld_b_sp_xx(INPUT_1_0, p_in_1, input_depth); + vld_b_sp_xx(INPUT_1_1, p_in_1, input_depth); + vdup_b_x(INPUT_1_2, -input_offset); - adwinit_v(v48, v48); - adwconv_vxv(v48, v15, cmds, v6); - adwconv_vxv(v48, v18, cmds, v9); - vdwconv_vxv(v48, v21, cmds, v12); - + COMPUTE(); INT32_TO_INT8_OUTPUT_PIPELINE_INPLACE( v48, v56, v60, output_activation_min, @@ -454,12 +459,20 @@ vsraqs_b_vx(v48, v48, 0); vst_b_x(v48, p_output); p_output += output_depth; - p_in_0 -= (2 * stride_width * input_depth); - p_in_1 -= (2 * stride_width * input_depth); } } } } +#undef FLT_0_0 +#undef FLT_0_1 +#undef FLT_0_2 +#undef FLT_1_0 +#undef FLT_1_1 +#undef FLT_1_2 +#undef FLT_2_0 +#undef FLT_2_1 +#undef FLT_2_2 +#undef COMPUTE } // special case of input depth = 32n, filter shape of 3x3