| /* | |
| * Copyright 2024 Google LLC | |
| * | |
| * Licensed under the Apache License, Version 2.0 (the "License"); | |
| * you may not use this file except in compliance with the License. | |
| * You may obtain a copy of the License at | |
| * | |
| * http://www.apache.org/licenses/LICENSE-2.0 | |
| * | |
| * Unless required by applicable law or agreed to in writing, software | |
| * distributed under the License is distributed on an "AS IS" BASIS, | |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| * See the License for the specific language governing permissions and | |
| * limitations under the License. | |
| */ | |
| #ifndef TFLM_OPT_CONV_UTIL_H_ | |
| #define TFLM_OPT_CONV_UTIL_H_ | |
| #include <cassert> | |
| #include <memory> | |
| #include "crt/kelvin.h" | |
| #include "tensorflow/lite/kernels/internal/common.h" | |
| #include "tensorflow/lite/kernels/internal/runtime_shape.h" | |
| #include "tensorflow/lite/kernels/internal/types.h" | |
| #include "tflm/opt/util.h" | |
| namespace kelvin::opt { | |
| /* clang-format off */ | |
| constexpr const int swizzle[16] = { | |
| 0, 4, 8, 12, | |
| 2, 6, 10, 14, | |
| 1, 5, 9, 13, | |
| 3, 7, 11, 15, | |
| }; | |
| /* clang-format on */ | |
| constexpr int kFilterHeightIndex = 1; | |
| constexpr int kFilterWidthIndex = 2; | |
| constexpr int kFilterInputChannelIndex = 3; | |
| constexpr int kInputChannelIndex = 3; | |
| constexpr int kOutputChannelIndex = 3; | |
| #define INA0 v0 | |
| #define FLTA0 v8 | |
| #define FLTA1 v9 | |
| #define FLTA2 v10 | |
| #define FLTA3 v11 | |
| #define FLTA4 v12 | |
| #define FLTA5 v13 | |
| #define FLTA6 v14 | |
| #define FLTA7 v15 | |
| #define ACC v48 | |
| #define ACC0 v48 | |
| #define OUT0 v56 | |
| // H,W ( height and width of filter) N -number of inputs, M -number of outputs | |
| template <int N> | |
| inline void Filter_N_H_W_M(const int8_t* input, int8_t* output, int H, int W, | |
| int M) { | |
| // Convert: input [zo][ky][kx][zi] (N,3,1,M) | |
| // output [zo.hi=N/8][ky][kx][zi_hi=M/4][zo.lo=8][zi_lo=4] | |
| const int8_t(&in)[N][H][W][M] = *(int8_t(*)[N][H][W][M])input; | |
| int8_t(&out)[N / 8][H][W][M / 4][8][4] = | |
| *(int8_t(*)[N / 8][H][W][M / 4][8][4]) output; | |
| assert(N >= 4 && M >= 4); | |
| for (int zo = 0; zo < N; ++zo) { | |
| for (int ky = 0; ky < H; ++ky) { | |
| for (int kx = 0; kx < W; ++kx) { | |
| for (int zi = 0; zi < M; ++zi) { | |
| const int zo_hi = zo >> 3; // div8 | |
| const int zo_lo = zo & 7; // rem8 | |
| const int zi_hi = zi >> 2; // div4 | |
| const int zi_lo = zi & 3; // rem4 | |
| out[zo_hi][ky][kx][zi_hi][zo_lo][zi_lo] = in[zo][ky][kx][zi]; | |
| } | |
| } | |
| } | |
| } | |
| } | |
| // Swizzle values, and duplicate 4 times for stripmining. | |
| inline void Swizzle(const int32_t* input, int32_t* output, int N, | |
| bool negate = false) { | |
| const int32_t(&in)[N] = *(int32_t(*)[N])input; | |
| int32_t(&out)[N * 4] = *(int32_t(*)[N * 4]) output; | |
| // Convert to accumulator swizzle pattern. | |
| for (int i = 0; i < N / 8; ++i) { | |
| int32_t* out0 = out + i * 32 + 0; | |
| int32_t* out1 = out + i * 32 + 16; | |
| int32_t* out2 = out + i * 32 + 8; | |
| int32_t* out3 = out + i * 32 + 24; | |
| for (int j = 0; j < 4; ++j) { | |
| const int32_t* p_in = in + i * 8; | |
| for (int k = 0; k < 2; ++k) { | |
| *out0++ = *p_in++; | |
| *out1++ = *p_in++; | |
| *out2++ = *p_in++; | |
| *out3++ = *p_in++; | |
| } | |
| } | |
| } | |
| if (negate) { | |
| for (int i = 0; i < N * 4; ++i) { | |
| out[i] = -out[i]; | |
| } | |
| } | |
| } | |
| // Runs strip-mined output pipeline (without bias addition) in place on | |
| // registers. | |
| #define INT32_TO_INT8_OUTPUT_PIPELINE_INPLACE(result, mult, shft, output_min, \ | |
| output_max, output_offset) \ | |
| { \ | |
| vdmulh_w_rn_vv_m(result, result, mult); \ | |
| vsha_w_r_vv_m(result, result, shft); \ | |
| vadd_w_vx_m(result, result, output_offset); \ | |
| vmax_w_vx_m(result, result, output_activation_min); \ | |
| vmin_w_vx_m(result, result, output_activation_max); \ | |
| } | |
| // Run output pipeline on int32 accumulators in [v48-v55] and store results | |
| // in v48 and v52. Clobbers [v48-v55]. | |
| #define INT32_TO_INT8_OUTPUT_PIPELINE(bias, mult, shft, output_min, \ | |
| output_max, output_offset, bias_reg, \ | |
| mult_reg, shift_reg) \ | |
| { \ | |
| vcget(v48); \ | |
| vld_w_x_m(bias_reg, bias); \ | |
| vld_w_x_m(mult_reg, mult); \ | |
| vld_w_x_m(shift_reg, shft); \ | |
| vadd_w_vv_m(v48, v48, bias_reg); \ | |
| vadd_w_vv_m(v52, v52, bias_reg); \ | |
| vmin_w_vx_m(v48, v48, output_max); \ | |
| vmax_w_vx_m(v52, v52, output_min); \ | |
| vdmulh_w_r_vv_m(v48, v48, mult_reg); \ | |
| vdmulh_w_r_vv_m(v52, v52, mult_reg); \ | |
| vsha_w_r_vv_m(v48, v48, shift_reg); \ | |
| vsha_w_r_vv_m(v52, v52, shift_reg); \ | |
| vadd_w_vx_m(v48, v48, output_offset); \ | |
| vadd_w_vx_m(v52, v52, output_offset); \ | |
| vsraqs_b_vx(v48, v48, 0); \ | |
| vsraqs_b_vx(v52, v52, 0); \ | |
| } | |
| } // namespace kelvin::opt | |
| #endif // TFLM_OPT_CONV_UTIL_H_ |