Add the option to optimize RISP4ML using RVV
This is the comprehensive code change for adding the option of using vector instructions to reduce # of instructions for the RISP4ML toolchain.
Summary:
- All 6 blocks (BLC, DG, WBG, Demosaic, Gamma and Downscaler) have been optimized.
- Numerical correctness has been verified. All unit tests passed (after
necessary rewriting).
- Significant saving: 40% -> 1%
Change-Id: I6abc8f253f5a74044985ab64797e5de5a76364a2
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 05fe4cc..c227663 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -31,6 +31,7 @@
add_link_options("LINKER:--defsym=__stack_size__=${STACK_SIZE}")
set(SPRINGBOK_LINKER_SCRIPT "$ENV{ROOTDIR}/sw/vec/springbok/springbok.ld" CACHE PATH "Springbok linker script path (default: springbok.ld)")
set(BUILD_WITH_SPRINGBOK ON CACHE BOOL "Build the target with springbok BSP (default: ON)")
+set(BUILD_ISP_WITH_RVV ON CACHE BOOL "Build the ISP pipeline with RVV (default: ON)")
#-------------------------------------------------------------------------------
# IREE-specific settings
diff --git a/samples/risp4ml/isp_stages/CMakeLists.txt b/samples/risp4ml/isp_stages/CMakeLists.txt
index bc2c798..7d47d93 100644
--- a/samples/risp4ml/isp_stages/CMakeLists.txt
+++ b/samples/risp4ml/isp_stages/CMakeLists.txt
@@ -1,10 +1,36 @@
+#-------------------------------------------------------------------------------
+# Build libraries and unit tests for RISP4ML blocks.
+# Use different source files for RVV and non-RVV versions
+#-------------------------------------------------------------------------------
+
+set(BLC_SRC "blc.c")
+set(DEMOSAIC_SRC "demosaic.c")
+set(DG_SRC "dg.c")
+set(DOWNSCALE_SRC "downscale.c")
+set(GAMMA_SRC "gamma.c")
+set(WBG_SRC "wbg.c")
+set(DOWNSCALE_TEST_SRC "downscale_test.cc")
+set(GAMMA_TEST_SRC "gamma_test.cc")
+
+if (${BUILD_ISP_WITH_RVV})
+ set(BLC_SRC "blc_rvv.c")
+ set(DEMOSAIC_SRC "demosaic_rvv.c")
+ set(DG_SRC "dg_rvv.c")
+ set(DOWNSCALE_SRC "downscale_rvv.c")
+ set(GAMMA_SRC "gamma_rvv.c")
+ set(WBG_SRC "wbg_rvv.c")
+ set(DOWNSCALE_TEST_SRC "downscale_rvv_test.cc")
+ set(GAMMA_TEST_SRC "gamma_rvv_test.cc")
+ add_definitions(-DISP_WITH_RVV)
+endif()
+
iree_cc_library(
NAME
blc
HDRS
"blc.h"
SRCS
- "blc.c"
+ ${BLC_SRC}
DEPS
samples::risp4ml::common::image
samples::risp4ml::common::utils
@@ -16,7 +42,7 @@
HDRS
"demosaic.h"
SRCS
- "demosaic.c"
+ ${DEMOSAIC_SRC}
DEPS
samples::risp4ml::common::image
samples::risp4ml::common::utils
@@ -28,7 +54,7 @@
HDRS
"dg.h"
SRCS
- "dg.c"
+ ${DG_SRC}
DEPS
samples::risp4ml::common::image
samples::risp4ml::common::utils
@@ -40,7 +66,7 @@
HDRS
"downscale.h"
SRCS
- "downscale.c"
+ ${DOWNSCALE_SRC}
DEPS
samples::risp4ml::common::image
samples::risp4ml::common::utils
@@ -52,7 +78,7 @@
HDRS
"gamma.h"
SRCS
- "gamma.c"
+ ${GAMMA_SRC}
DEPS
samples::risp4ml::common::image
samples::risp4ml::common::utils
@@ -64,7 +90,7 @@
HDRS
"wbg.h"
SRCS
- "wbg.c"
+ ${WBG_SRC}
DEPS
samples::risp4ml::common::image
samples::risp4ml::common::utils
@@ -100,7 +126,7 @@
NAME
downscale_test
SRCS
- "downscale_test.cc"
+ ${DOWNSCALE_TEST_SRC}
DEPS
::downscale
pw_unit_test
@@ -112,7 +138,7 @@
NAME
gamma_test
SRCS
- "gamma_test.cc"
+ ${GAMMA_TEST_SRC}
DEPS
::gamma
pw_unit_test
diff --git a/samples/risp4ml/isp_stages/blc_rvv.c b/samples/risp4ml/isp_stages/blc_rvv.c
new file mode 100644
index 0000000..21c53fe
--- /dev/null
+++ b/samples/risp4ml/isp_stages/blc_rvv.c
@@ -0,0 +1,42 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <riscv_vector.h>
+
+#include "samples/risp4ml/common/utils.h"
+#include "samples/risp4ml/isp_stages/blc.h"
+
+static BlcParams blc_params = {.enable = true,
+ .offsets = {2048, 2048, 2048, 2048}};
+
+void set_blc_params(BlcParams* params) { blc_params = *params; }
+
+void blc_process(Image* img) {
+ if (!blc_params.enable) return;
+
+ size_t vl;
+ size_t n = img->height * img->width * img->num_channels;
+ uint16_t offset = blc_params.offsets[0];
+ vuint16m8_t vx; // auxiliary variable
+ uint16_t* x;
+ for (size_t i = 0; i < n; i += vl) {
+ x = img->data + i;
+ vl = vsetvl_e16m8(n - i);
+ vx = vle16_v_u16m8(x, vl); // load
+ vx = vssubu(vx, offset, vl); // subtract
+ vse16(x, vx, vl); // save
+ }
+}
diff --git a/samples/risp4ml/isp_stages/demosaic_rvv.c b/samples/risp4ml/isp_stages/demosaic_rvv.c
new file mode 100644
index 0000000..2ed9d25
--- /dev/null
+++ b/samples/risp4ml/isp_stages/demosaic_rvv.c
@@ -0,0 +1,147 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <riscv_vector.h>
+
+#include "samples/risp4ml/common/utils.h"
+#include "samples/risp4ml/isp_stages/demosaic.h"
+
+#define kRgbColorChannels 3
+
+static DemosaicParams demosaic_params = {.enable = true};
+
+void set_demosaic_params(DemosaicParams* params) { demosaic_params = *params; }
+
+// Basic bilinear demosaic
+void demosaic_process(Image* input, Image* output) {
+ if (!demosaic_params.enable) {
+ return;
+ }
+
+ const pixel_type_t* line_in[kRgbColorChannels];
+ pixel_type_t* line_out[kRgbColorChannels];
+ int x_offset[kRgbColorChannels];
+ const uint16_t boundary[kRgbColorChannels] = {0, input->width - 2,
+ input->width - 1};
+
+ size_t vl;
+ // auxiliary variables
+ vuint16m4_t vx, vy;
+ vuint32m8_t vz;
+
+ for (uint16_t y = 0; y < input->height; ++y) {
+ line_in[0] = (y) ? image_row(input, 0, y - 1) : image_row(input, 0, 1);
+ line_in[1] = image_row(input, 0, y);
+ line_in[2] = (y < input->height - 1)
+ ? image_row(input, 0, y + 1)
+ : image_row(input, 0, input->height - 2);
+
+ line_out[1] = image_row(output, 1, y);
+ if ((y & 0x1) == 0) {
+ line_out[0] = image_row(output, 0, y);
+ line_out[2] = image_row(output, 2, y);
+ } else {
+ line_out[0] = image_row(output, 2, y);
+ line_out[2] = image_row(output, 0, y);
+ }
+
+ // x at boundary
+ for (uint8_t i = 0; i < 3; ++i) {
+ uint16_t x = boundary[i];
+ for (uint16_t c = 0; c < kRgbColorChannels; ++c) {
+ x_offset[c] = BayerMirrorBoundary(x - 1 + c, input->width);
+ }
+
+ BayerIndex bayer_index = GetBayerIndex(kBayerType, x, y);
+ switch (bayer_index) {
+ case (kB):
+ case (kR): {
+ line_out[0][x] = line_in[1][x_offset[1]];
+ line_out[1][x] = (line_in[0][x_offset[1]] + line_in[2][x_offset[1]] +
+ line_in[1][x_offset[0]] + line_in[1][x_offset[2]]) /
+ 4;
+ line_out[2][x] = (line_in[0][x_offset[0]] + line_in[0][x_offset[2]] +
+ line_in[2][x_offset[0]] + line_in[2][x_offset[2]]) /
+ 4;
+ }; break;
+ case (kGb):
+ case (kGr): {
+ line_out[0][x] =
+ (line_in[1][x_offset[0]] + line_in[1][x_offset[2]]) / 2;
+ line_out[1][x] = line_in[1][x_offset[1]];
+ line_out[2][x] =
+ (line_in[0][x_offset[1]] + line_in[2][x_offset[1]]) / 2;
+ }; break;
+ default:
+ break;
+ }
+ }
+
+ // x not at boundary: vector instructions
+ for (uint8_t n = 1; n <= 2; n++) {
+ for (uint16_t x = n; x < input->width - 2; x += 2 * vl) {
+ x_offset[0] = x - 1;
+ x_offset[1] = x;
+ x_offset[2] = x + 1;
+ ptrdiff_t stride = 2 * sizeof(uint16_t);
+ size_t avl = (input->width - 1 - x) / 2;
+ vl = vsetvl_e16m4(avl);
+
+ if (n + (y & 0x1) == 2) { // kR or kB
+ // ch0
+ vx = vlse16_v_u16m4(line_in[1] + x_offset[1], stride, vl); // load
+ vsse16(line_out[0] + x, stride, vx, vl); // save
+ // ch1
+ vx = vlse16_v_u16m4(line_in[0] + x_offset[1], stride, vl); // load
+ vy = vlse16_v_u16m4(line_in[2] + x_offset[1], stride, vl); // load
+ vz = vwaddu_vv(vx, vy, vl); // add
+ vy = vlse16_v_u16m4(line_in[1] + x_offset[0], stride, vl); // load
+ vz = vwaddu_wv(vz, vy, vl); // add
+ vy = vlse16_v_u16m4(line_in[1] + x_offset[2], stride, vl); // load
+ vz = vwaddu_wv(vz, vy, vl); // add
+ vx = vnsrl(vz, 2, vl); // 1/4
+ vsse16(line_out[1] + x, stride, vx, vl); // save
+ // ch2
+ vx = vlse16_v_u16m4(line_in[0] + x_offset[0], stride, vl); // load
+ vy = vlse16_v_u16m4(line_in[0] + x_offset[2], stride, vl); // load
+ vz = vwaddu_vv(vx, vy, vl); // add
+ vy = vlse16_v_u16m4(line_in[2] + x_offset[0], stride, vl); // load
+ vz = vwaddu_wv(vz, vy, vl); // add
+ vy = vlse16_v_u16m4(line_in[2] + x_offset[2], stride, vl); // load
+ vz = vwaddu_wv(vz, vy, vl); // add
+ vx = vnsrl(vz, 2, vl); // 1/4
+ vsse16(line_out[2] + x, stride, vx, vl); // save
+ } else { // kGr or kRb
+ // ch0
+ vx = vlse16_v_u16m4(line_in[1] + x_offset[0], stride, vl); // load
+ vy = vlse16_v_u16m4(line_in[1] + x_offset[2], stride, vl); // load
+ vz = vwaddu_vv(vx, vy, vl); // add
+ vx = vnsrl(vz, 1, vl); // 1/2
+ vsse16(line_out[0] + x, stride, vx, vl); // save
+ // ch1
+ vx = vlse16_v_u16m4(line_in[1] + x_offset[1], stride, vl); // load
+ vsse16(line_out[1] + x, stride, vx, vl); // save
+ // ch2
+ vx = vlse16_v_u16m4(line_in[0] + x_offset[1], stride, vl); // load
+ vy = vlse16_v_u16m4(line_in[2] + x_offset[1], stride, vl); // load
+ vz = vwaddu_vv(vx, vy, vl); // add
+ vx = vnsrl(vz, 1, vl); // 1/2
+ vsse16(line_out[2] + x, stride, vx, vl); // save
+ }
+ }
+ }
+ }
+}
diff --git a/samples/risp4ml/isp_stages/dg_rvv.c b/samples/risp4ml/isp_stages/dg_rvv.c
new file mode 100644
index 0000000..8cfa816
--- /dev/null
+++ b/samples/risp4ml/isp_stages/dg_rvv.c
@@ -0,0 +1,51 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include <riscv_vector.h>
+
+#include "samples/risp4ml/common/utils.h"
+#include "samples/risp4ml/isp_stages/dg.h"
+
+static const uint16_t kDgFractional = kRawPipelineFraction;
+static const uint16_t kDgUnityGain = 1 << kDgFractional;
+static DgParams dg_params = {
+ .enable = true,
+ .gains = {kDgUnityGain, kDgUnityGain, kDgUnityGain, kDgUnityGain}};
+
+void set_dg_params(DgParams* params) { dg_params = *params; }
+
+void dg_process(Image* img) {
+ if (!dg_params.enable) return;
+
+ size_t vl;
+ size_t n = img->height * img->width * img->num_channels;
+ uint16_t gain = dg_params.gains[0];
+ uint32_t offset = 1 << (kDgFractional - 1);
+ uint32_t max_val = kRawPipelineMaxVal << kDgFractional;
+ // auxiliary variables
+ vuint16m4_t vx;
+ vuint32m8_t vy;
+ uint16_t* x;
+ for (size_t i = 0; i < n; i += vl) {
+ x = img->data + i;
+ vl = vsetvl_e16m4(n - i);
+ vx = vle16_v_u16m4(x, vl); // load
+ vy = vwmulu(vx, gain, vl); // multiply
+ vy = vadd(vy, offset, vl); // add
+ vy = vminu(vy, max_val, vl); // clamp
+ vx = vnsrl(vy, kDgFractional, vl); // bit shift
+ vse16(x, vx, vl); // save
+ }
+}
diff --git a/samples/risp4ml/isp_stages/downscale.h b/samples/risp4ml/isp_stages/downscale.h
index d166610..0ca46bd 100644
--- a/samples/risp4ml/isp_stages/downscale.h
+++ b/samples/risp4ml/isp_stages/downscale.h
@@ -28,13 +28,14 @@
// scale_precision is the number of fractional bits used for scale factors and
// initial offsets
uint32_t scale_precision;
+ uint32_t scale_fixed_one;
+#ifndef ISP_WITH_RVV
// interpolate_precision is the number of fractional bits used for
// interpolation weights
uint32_t interpolate_precision;
// interpolate_shift is the shift for pixel value before interpolation to
// avoid rounding error.
uint32_t interpolate_shift;
- uint32_t scale_fixed_one;
uint32_t scale_fraction_mask;
uint32_t weight_shift;
@@ -51,6 +52,7 @@
// output pixel from the first input pixel in each direction respectively
uint32_t ver_initial_offset;
uint32_t hor_initial_offset;
+#endif // ! ISP_WITH_RVV
} DownscaleParams;
void set_downscale_param(DownscaleParams* params);
diff --git a/samples/risp4ml/isp_stages/downscale_rvv.c b/samples/risp4ml/isp_stages/downscale_rvv.c
new file mode 100644
index 0000000..454bd6c
--- /dev/null
+++ b/samples/risp4ml/isp_stages/downscale_rvv.c
@@ -0,0 +1,156 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <riscv_vector.h>
+
+#include "samples/risp4ml/common/utils.h"
+#include "samples/risp4ml/isp_stages/downscale.h"
+
+static const uint16_t kScalePrecision = 10;
+static const uint32_t kScaleFixedOne = (1 << kScalePrecision);
+
+static DownscaleParams params = {
+ .enable = true,
+ .scale_precision = kScalePrecision,
+ .scale_fixed_one = kScaleFixedOne,
+};
+
+void set_downscale_param(DownscaleParams* in_params) { params = *in_params; }
+void set_downscale_factor(Image* input, ImageU8* output) { return; }
+
+// Basic bilinear downscale
+// Implementation based on:
+// https://chao-ji.github.io/jekyll/update/2018/07/19/BilinearResize.html
+// Resamples image using bilinear interpolation.
+// 'output' is modified by this function to store the output image.
+void downscale_process(Image* input, ImageU8* output) {
+ if (!params.enable) {
+ return;
+ }
+
+ uint32_t input_width = input->width;
+ uint32_t input_w_1 = input->width - 1;
+ uint32_t input_h_1 = input->height - 1;
+ uint32_t w_1 = output->width - 1;
+ uint32_t h_1 = output->height - 1;
+
+ size_t vl;
+ size_t n = output->height * output->width;
+ // auxiliary variables
+ vuint32m8_t vx, vy, vz, vid, vp, vq;
+ // neiboring x & y coordinates
+ vuint32m8_t vx_l, vy_l, vx_h, vy_h;
+ // weights of neighbors
+ vuint32m8_t vx_weight, vy_weight, vx_weight_1minus, vy_weight_1minus;
+ // neighboring data points
+ vuint32m8_t va, vb, vc, vd;
+ vuint32m8_t vo; // 32bit output
+ vuint16m4_t vo_16b; // 16bit output
+ vuint8m2_t vo_8b; // 8bit output
+
+ for (uint16_t c = 0; c < output->num_channels; ++c) {
+ pixel_type_t* in = image_row(input, c, 0);
+ uint8_t* out = output->data + c;
+
+ for (size_t i = 0; i < n; i += vl) {
+ vl = vsetvl_e16m4(n - i);
+ vid = vid_v_u32m8(vl);
+ vid = vadd(vid, i, vl);
+
+ vy = vdivu(vid, output->width, vl);
+ vx = vremu(vid, output->width, vl);
+
+ // find neighbors
+ vx_l = vmul(vx, input_w_1, vl);
+ vx_l = vdivu(vx_l, w_1, vl);
+ vx_h = vadd(vx_l, 1, vl);
+ vx_h = vminu(vx_h, input_w_1, vl); // clamp
+
+ vy_l = vmul(vy, input_h_1, vl);
+ vy_l = vdivu(vy_l, h_1, vl);
+ vy_h = vadd(vy_l, 1, vl);
+ vy_h = vminu(vy_h, input_h_1, vl); // clamp
+
+ // load a, b, c, d
+ vz = vmul(vy_l, input_width, vl);
+ vz = vadd(vz, vx_l, vl);
+ vz = vsll(vz, 1, vl); // *2
+ vo_16b = vluxei32(in, vz, vl);
+ va = vwaddu_vx(vo_16b, 0, vl);
+
+ vz = vmul(vy_l, input_width, vl);
+ vz = vadd(vz, vx_h, vl);
+ vz = vsll(vz, 1, vl); // *2
+ vo_16b = vluxei32(in, vz, vl);
+ vb = vwaddu_vx(vo_16b, 0, vl);
+
+ vz = vmul(vy_h, input_width, vl);
+ vz = vadd(vz, vx_l, vl);
+ vz = vsll(vz, 1, vl); // *2
+ vo_16b = vluxei32(in, vz, vl);
+ vc = vwaddu_vx(vo_16b, 0, vl);
+
+ vz = vmul(vy_h, input_width, vl);
+ vz = vadd(vz, vx_h, vl);
+ vz = vsll(vz, 1, vl); // *2
+ vo_16b = vluxei32(in, vz, vl);
+ vd = vwaddu_vx(vo_16b, 0, vl);
+
+ // weights
+ vp = vmul(vx, input_w_1, vl);
+ vq = vmul(vx_l, w_1, vl);
+ vp = vssubu(vp, vq, vl);
+ vp = vsll(vp, params.scale_precision, vl);
+ vx_weight = vdivu(vp, w_1, vl);
+
+ vp = vmul(vy, input_h_1, vl);
+ vq = vmul(vy_l, h_1, vl);
+ vp = vssubu(vp, vq, vl);
+ vp = vsll(vp, params.scale_precision, vl);
+ vy_weight = vdivu(vp, h_1, vl);
+
+ vx_weight_1minus = vrsub(vx_weight, params.scale_fixed_one, vl);
+ vy_weight_1minus = vrsub(vy_weight, params.scale_fixed_one, vl);
+
+ // resized
+ vo = vmul(va, vx_weight_1minus, vl);
+ vo = vsrl(vo, params.scale_precision, vl);
+ vo = vmul(vo, vy_weight_1minus, vl);
+
+ vp = vmul(vb, vx_weight, vl);
+ vp = vsrl(vp, params.scale_precision, vl);
+ vp = vmul(vp, vy_weight_1minus, vl);
+ vo = vadd(vo, vp, vl);
+
+ vp = vmul(vc, vx_weight_1minus, vl);
+ vp = vsrl(vp, params.scale_precision, vl);
+ vp = vmul(vp, vy_weight, vl);
+ vo = vadd(vo, vp, vl);
+
+ vp = vmul(vd, vx_weight, vl);
+ vp = vsrl(vp, params.scale_precision, vl);
+ vp = vmul(vp, vy_weight, vl);
+ vo = vadd(vo, vp, vl);
+
+ // bit shift
+ vo_16b = vnsrl(vo, params.scale_precision, vl);
+ vo_8b = vnsrl(vo_16b, kRawPipelineBpp - kPipeOutputBpp, vl);
+
+ // save
+ vsse8(out + i * output->num_channels, output->num_channels, vo_8b, vl);
+ }
+ }
+}
diff --git a/samples/risp4ml/isp_stages/downscale_rvv_test.cc b/samples/risp4ml/isp_stages/downscale_rvv_test.cc
new file mode 100644
index 0000000..f38e8b8
--- /dev/null
+++ b/samples/risp4ml/isp_stages/downscale_rvv_test.cc
@@ -0,0 +1,210 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Use separate test for RVV as algorithm is implemented differently
+// Implementation based on:
+// https://chao-ji.github.io/jekyll/update/2018/07/19/BilinearResize.html
+
+#include <climits>
+#include <cmath>
+
+#include "pw_unit_test/framework.h"
+#include "samples/risp4ml/common/constants.h"
+#include "samples/risp4ml/common/test_utils.h"
+#include "samples/risp4ml/isp_stages/downscale.h"
+
+static constexpr uint16_t kScalePrecision = 10;
+static constexpr uint32_t kScaleFixedOne = (1 << kScalePrecision);
+static constexpr float kOutBitsShift = 1 << (kRawPipelineBpp - kPipeOutputBpp);
+
+class DownscaleRvvTest : public ::testing::Test {
+ protected:
+ void setup(uint16_t in_ch, uint16_t in_height, uint16_t in_width,
+ uint16_t out_ch, uint16_t out_height, uint16_t out_width) {
+ in_ = image_new(in_ch, in_height, in_width);
+ out_ = imageu8_new(out_ch, out_height, out_width);
+ params_.enable = true;
+ params_.scale_precision = kScalePrecision;
+ params_.scale_fixed_one = kScaleFixedOne;
+ }
+ void TearDown() override {
+ image_delete(in_);
+ imageu8_delete(out_);
+ }
+ ImageU8* imageu8_new(uint16_t num_channels, uint16_t height, uint16_t width);
+ void imageu8_delete(ImageU8* image) {
+ if (image) {
+ if (image->data) free(image->data);
+ free(image);
+ }
+ }
+ pixel_type_t imageu8_pixel_val(ImageU8* image, uint16_t c, uint16_t y,
+ uint16_t x) {
+ const uint32_t stride_c = 1;
+ const uint16_t stride_y = image->num_channels * image->width;
+ const uint16_t stride_x = image->num_channels;
+
+ return *(image->data + c * stride_c + y * stride_y + x * stride_x);
+ }
+ float ExpectedOut(uint16_t y, uint16_t x);
+ void ScaleRampImageTest(uint16_t output_width, uint16_t output_height,
+ uint16_t input_width = 640,
+ uint16_t input_height = 480);
+ struct BilinearScaleTestValues {
+ uint16_t output_width;
+ uint16_t output_height;
+ uint16_t input_width;
+ uint16_t input_height;
+ };
+
+ Image* in_;
+ ImageU8* out_;
+ DownscaleParams params_;
+};
+
+ImageU8* DownscaleRvvTest::imageu8_new(uint16_t num_channels, uint16_t height,
+ uint16_t width) {
+ ImageU8* image = (ImageU8*)malloc(sizeof(ImageU8));
+ if (image) {
+ image->num_channels = num_channels;
+ image->height = height;
+ image->width = width;
+ uint32_t num_pixels = width * height * num_channels;
+ image->data = (uint8_t*)malloc(num_pixels * sizeof(uint8_t));
+ }
+ return image;
+}
+
+float DownscaleRvvTest::ExpectedOut(uint16_t y, uint16_t x) {
+ float x_ratio = ((float)in_->width - 1) / (out_->width - 1);
+ float y_ratio = ((float)in_->height - 1) / (out_->height - 1);
+
+ uint32_t x_l = (uint32_t)(x_ratio * x);
+ uint32_t x_h = (x_l == in_->width - 1) ? x_l : x_l + 1;
+ uint32_t y_l = y_ratio * y;
+ uint32_t y_h = y_l == in_->height - 1 ? y_l : y_l + 1;
+ float x_weight = (x_ratio * x) - x_l;
+ float y_weight = (y_ratio * y) - y_l;
+
+ pixel_type_t a = image_pixel_val(in_, 0, y_l, x_l);
+ pixel_type_t b = image_pixel_val(in_, 0, y_l, x_h);
+ pixel_type_t c = image_pixel_val(in_, 0, y_h, x_l);
+ pixel_type_t d = image_pixel_val(in_, 0, y_h, x_h);
+
+ float expected_out = a * (1 - x_weight) * (1 - y_weight) +
+ b * x_weight * (1 - y_weight) +
+ c * y_weight * (1 - x_weight) + d * x_weight * y_weight;
+
+ expected_out = floorf(expected_out / kOutBitsShift);
+
+ return expected_out;
+}
+
+// Helper function for 2D ramp tests. image is downscaled successfully.
+void DownscaleRvvTest::ScaleRampImageTest(uint16_t output_width,
+ uint16_t output_height,
+ uint16_t input_width,
+ uint16_t input_height) {
+ constexpr int kTolerance = 1; // Tolerance for rounding error.
+ setup(1, input_height, input_width, 1, output_height, output_width);
+
+ // Fill in_ images as 2D ramp whose values are increased from the
+ // top-left corner to the bottom-right corner.
+ for (uint16_t y = 0; y < input_height; ++y) {
+ for (uint16_t x = 0; x < input_width; ++x) {
+ *image_pixel(in_, 0, y, x) = (y * input_width + x) % (1024);
+ }
+ }
+
+ set_downscale_param(¶ms_);
+ downscale_process(in_, out_);
+
+ for (uint16_t y = 0; y < output_height; ++y) {
+ for (uint16_t x = 0; x < output_width; ++x) {
+ float expected_out = ExpectedOut(y, x);
+ float diff =
+ std::abs((float)imageu8_pixel_val(out_, 0, y, x) - expected_out);
+ ASSERT_LE(diff, kTolerance);
+ }
+ }
+}
+
+TEST_F(DownscaleRvvTest, NoScaleTest) {
+ constexpr uint16_t kOutputWidth = 128;
+ constexpr uint16_t kInputHeight = 96;
+ setup(1, kInputHeight, kOutputWidth, 1, kInputHeight, kOutputWidth);
+
+ // Generate random image.
+ InitImageRandom(in_, 0, USHRT_MAX);
+
+ set_downscale_param(¶ms_);
+ downscale_process(in_, out_);
+
+ // Verify the out_ image is identical to the in_ image.
+ for (uint16_t y = 0; y < kInputHeight; ++y) {
+ for (uint16_t x = 0; x < kOutputWidth; ++x) {
+ ASSERT_EQ(imageu8_pixel_val(out_, 0, y, x),
+ static_cast<pixel_type_t>(
+ floorf(image_pixel_val(in_, 0, y, x) >>
+ (kRawPipelineBpp - kPipeOutputBpp))));
+ }
+ }
+}
+
+TEST_F(DownscaleRvvTest, DownscaleRvvTest) {
+ std::vector<BilinearScaleTestValues> tests = {
+ {8, 12, 64, 64}, {320, 240, 640, 480}, {80, 60, 640, 480},
+ {220, 95, 640, 480}, {415, 125, 640, 480}, {122, 13, 200, 100}};
+
+ for (const auto& test : tests) {
+ ScaleRampImageTest(test.output_width, test.output_height, test.input_width,
+ test.input_height);
+ }
+}
+
+TEST_F(DownscaleRvvTest, Trivial3DTest) {
+ constexpr uint16_t kChannels = 3;
+ constexpr uint16_t kInputHeight = 5;
+ constexpr uint16_t kInputWidth = 5;
+
+ constexpr uint16_t kVerScale = 2;
+ constexpr uint16_t kHorScale = 2;
+
+ constexpr uint16_t kOutputHeight = (kInputHeight - 1) / kVerScale + 1;
+ constexpr uint16_t kOutputWidth = (kInputWidth - 1) / kHorScale + 1;
+
+ setup(kChannels, kInputHeight, kInputWidth, kChannels, kOutputHeight,
+ kOutputWidth);
+
+ for (uint16_t c = 0; c < kChannels; ++c) {
+ for (uint16_t y = 0; y < kInputHeight; ++y) {
+ for (uint16_t x = 0; x < kInputWidth; ++x) {
+ *image_pixel(in_, c, y, x) = ((y * kInputWidth + x) * 10 + c) << 8;
+ }
+ }
+ }
+
+ set_downscale_param(¶ms_);
+ downscale_process(in_, out_);
+
+ // for exact integer ratios out_ is just downsampled in_
+ for (uint16_t c = 0; c < kChannels; ++c) {
+ for (uint16_t y = 0; y < kOutputHeight; ++y) {
+ for (uint16_t x = 0; x < kOutputWidth; ++x) {
+ ASSERT_EQ(imageu8_pixel_val(out_, c, y, x),
+ image_pixel_val(in_, c, y * kVerScale, x * kHorScale) >> 8);
+ }
+ }
+ }
+}
diff --git a/samples/risp4ml/isp_stages/gamma.h b/samples/risp4ml/isp_stages/gamma.h
index c3044ac..24b42cd 100644
--- a/samples/risp4ml/isp_stages/gamma.h
+++ b/samples/risp4ml/isp_stages/gamma.h
@@ -23,7 +23,11 @@
extern "C" {
#endif // __cplusplus
+#ifndef ISP_WITH_RVV
#define kGammaNumberPoints 81
+#else
+#define kGammaNumberPoints 2049
+#endif
typedef struct {
bool enable;
diff --git a/samples/risp4ml/isp_stages/gamma_rvv.c b/samples/risp4ml/isp_stages/gamma_rvv.c
new file mode 100644
index 0000000..a0f5503
--- /dev/null
+++ b/samples/risp4ml/isp_stages/gamma_rvv.c
@@ -0,0 +1,271 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include <riscv_vector.h>
+
+#include "samples/risp4ml/common/utils.h"
+#include "samples/risp4ml/isp_stages/gamma.h"
+
+#define kRgbColorChannels 3
+
+static const uint16_t kRgbPipelineBpp = 16;
+static const uint16_t kRgbPipelineMaxVal = (1 << kRgbPipelineBpp) - 1;
+static const uint16_t kGammaShiftBits = 5;
+static const uint16_t kGammaSpacing = (1 << kGammaShiftBits);
+static GammaParams gamma_params = {
+ .enable = true,
+ .lut = {
+ 0, 413, 827, 1240, 1654, 2067, 2481, 2884, 3255, 3600,
+ 3923, 4228, 4518, 4793, 5056, 5309, 5552, 5786, 6013, 6232,
+ 6444, 6651, 6851, 7047, 7237, 7423, 7605, 7783, 7956, 8127,
+ 8294, 8457, 8618, 8776, 8931, 9083, 9233, 9380, 9525, 9668,
+ 9809, 9948, 10084, 10219, 10352, 10484, 10613, 10741, 10868, 10992,
+ 11116, 11238, 11358, 11478, 11595, 11712, 11828, 11942, 12055, 12167,
+ 12278, 12387, 12496, 12604, 12711, 12816, 12921, 13025, 13128, 13230,
+ 13331, 13432, 13531, 13630, 13728, 13825, 13922, 14017, 14112, 14207,
+ 14300, 14393, 14485, 14577, 14668, 14758, 14848, 14937, 15026, 15113,
+ 15201, 15288, 15374, 15459, 15545, 15629, 15713, 15797, 15880, 15963,
+ 16045, 16126, 16208, 16288, 16368, 16448, 16528, 16607, 16685, 16763,
+ 16841, 16918, 16995, 17071, 17147, 17223, 17298, 17373, 17448, 17522,
+ 17596, 17669, 17742, 17815, 17887, 17959, 18031, 18102, 18173, 18244,
+ 18315, 18385, 18454, 18524, 18593, 18662, 18731, 18799, 18867, 18935,
+ 19002, 19069, 19136, 19203, 19269, 19335, 19401, 19466, 19531, 19596,
+ 19661, 19726, 19790, 19854, 19918, 19981, 20045, 20108, 20170, 20233,
+ 20295, 20357, 20419, 20481, 20543, 20604, 20665, 20726, 20786, 20847,
+ 20907, 20967, 21027, 21086, 21145, 21205, 21264, 21322, 21381, 21439,
+ 21498, 21556, 21613, 21671, 21728, 21786, 21843, 21900, 21956, 22013,
+ 22069, 22126, 22182, 22238, 22293, 22349, 22404, 22459, 22514, 22569,
+ 22624, 22679, 22733, 22787, 22841, 22895, 22949, 23003, 23056, 23110,
+ 23163, 23216, 23269, 23321, 23374, 23426, 23479, 23531, 23583, 23635,
+ 23687, 23738, 23790, 23841, 23892, 23943, 23994, 24045, 24096, 24146,
+ 24197, 24247, 24297, 24347, 24397, 24447, 24497, 24546, 24596, 24645,
+ 24694, 24743, 24792, 24841, 24890, 24938, 24987, 25035, 25083, 25132,
+ 25180, 25228, 25275, 25323, 25371, 25418, 25466, 25513, 25560, 25607,
+ 25654, 25701, 25747, 25794, 25841, 25887, 25933, 25980, 26026, 26072,
+ 26118, 26163, 26209, 26255, 26300, 26346, 26391, 26436, 26481, 26527,
+ 26571, 26616, 26661, 26706, 26750, 26795, 26839, 26884, 26928, 26972,
+ 27016, 27060, 27104, 27148, 27191, 27235, 27278, 27322, 27365, 27408,
+ 27452, 27495, 27538, 27581, 27623, 27666, 27709, 27751, 27794, 27836,
+ 27879, 27921, 27963, 28005, 28047, 28089, 28131, 28173, 28215, 28256,
+ 28298, 28339, 28381, 28422, 28464, 28505, 28546, 28587, 28628, 28669,
+ 28710, 28750, 28791, 28832, 28872, 28913, 28953, 28994, 29034, 29074,
+ 29114, 29154, 29194, 29234, 29274, 29314, 29353, 29393, 29433, 29472,
+ 29512, 29551, 29590, 29630, 29669, 29708, 29747, 29786, 29825, 29864,
+ 29903, 29941, 29980, 30019, 30057, 30096, 30134, 30173, 30211, 30249,
+ 30287, 30326, 30364, 30402, 30440, 30477, 30515, 30553, 30591, 30628,
+ 30666, 30704, 30741, 30779, 30816, 30853, 30891, 30928, 30965, 31002,
+ 31039, 31076, 31113, 31150, 31187, 31223, 31260, 31297, 31333, 31370,
+ 31406, 31443, 31479, 31516, 31552, 31588, 31624, 31660, 31696, 31733,
+ 31768, 31804, 31840, 31876, 31912, 31948, 31983, 32019, 32054, 32090,
+ 32125, 32161, 32196, 32232, 32267, 32302, 32337, 32372, 32407, 32442,
+ 32477, 32512, 32547, 32582, 32617, 32652, 32686, 32721, 32756, 32790,
+ 32825, 32859, 32894, 32928, 32962, 32997, 33031, 33065, 33099, 33133,
+ 33167, 33202, 33235, 33269, 33303, 33337, 33371, 33405, 33438, 33472,
+ 33506, 33539, 33573, 33606, 33640, 33673, 33707, 33740, 33773, 33807,
+ 33840, 33873, 33906, 33939, 33972, 34005, 34038, 34071, 34104, 34137,
+ 34170, 34203, 34235, 34268, 34301, 34333, 34366, 34398, 34431, 34463,
+ 34496, 34528, 34560, 34593, 34625, 34657, 34689, 34722, 34754, 34786,
+ 34818, 34850, 34882, 34914, 34946, 34977, 35009, 35041, 35073, 35104,
+ 35136, 35168, 35199, 35231, 35262, 35294, 35325, 35357, 35388, 35419,
+ 35451, 35482, 35513, 35545, 35576, 35607, 35638, 35669, 35700, 35731,
+ 35762, 35793, 35824, 35855, 35886, 35916, 35947, 35978, 36008, 36039,
+ 36070, 36100, 36131, 36161, 36192, 36222, 36253, 36283, 36314, 36344,
+ 36374, 36405, 36435, 36465, 36495, 36525, 36555, 36586, 36616, 36646,
+ 36676, 36706, 36735, 36765, 36795, 36825, 36855, 36885, 36914, 36944,
+ 36974, 37003, 37033, 37063, 37092, 37122, 37151, 37181, 37210, 37239,
+ 37269, 37298, 37328, 37357, 37386, 37415, 37444, 37474, 37503, 37532,
+ 37561, 37590, 37619, 37648, 37677, 37706, 37735, 37764, 37793, 37821,
+ 37850, 37879, 37908, 37937, 37965, 37994, 38023, 38051, 38080, 38108,
+ 38137, 38165, 38194, 38222, 38251, 38279, 38307, 38336, 38364, 38392,
+ 38421, 38449, 38477, 38505, 38533, 38561, 38590, 38618, 38646, 38674,
+ 38702, 38730, 38758, 38785, 38813, 38841, 38869, 38897, 38925, 38952,
+ 38980, 39008, 39036, 39063, 39091, 39118, 39146, 39174, 39201, 39229,
+ 39256, 39284, 39311, 39338, 39366, 39393, 39421, 39448, 39475, 39502,
+ 39530, 39557, 39584, 39611, 39638, 39666, 39693, 39720, 39747, 39774,
+ 39801, 39828, 39855, 39882, 39909, 39935, 39962, 39989, 40016, 40043,
+ 40070, 40096, 40123, 40150, 40176, 40203, 40230, 40256, 40283, 40310,
+ 40336, 40363, 40389, 40416, 40442, 40468, 40495, 40521, 40548, 40574,
+ 40600, 40627, 40653, 40679, 40705, 40732, 40758, 40784, 40810, 40836,
+ 40862, 40888, 40914, 40941, 40967, 40993, 41019, 41044, 41070, 41096,
+ 41122, 41148, 41174, 41200, 41226, 41251, 41277, 41303, 41329, 41354,
+ 41380, 41406, 41431, 41457, 41483, 41508, 41534, 41559, 41585, 41610,
+ 41636, 41661, 41687, 41712, 41737, 41763, 41788, 41814, 41839, 41864,
+ 41889, 41915, 41940, 41965, 41990, 42016, 42041, 42066, 42091, 42116,
+ 42141, 42166, 42191, 42216, 42241, 42266, 42291, 42316, 42341, 42366,
+ 42391, 42416, 42441, 42466, 42490, 42515, 42540, 42565, 42590, 42614,
+ 42639, 42664, 42688, 42713, 42738, 42762, 42787, 42812, 42836, 42861,
+ 42885, 42910, 42934, 42959, 42983, 43008, 43032, 43056, 43081, 43105,
+ 43129, 43154, 43178, 43202, 43227, 43251, 43275, 43299, 43324, 43348,
+ 43372, 43396, 43420, 43444, 43469, 43493, 43517, 43541, 43565, 43589,
+ 43613, 43637, 43661, 43685, 43709, 43733, 43756, 43780, 43804, 43828,
+ 43852, 43876, 43899, 43923, 43947, 43971, 43994, 44018, 44042, 44066,
+ 44089, 44113, 44137, 44160, 44184, 44207, 44231, 44254, 44278, 44302,
+ 44325, 44348, 44372, 44395, 44419, 44442, 44466, 44489, 44512, 44536,
+ 44559, 44582, 44606, 44629, 44652, 44676, 44699, 44722, 44745, 44769,
+ 44792, 44815, 44838, 44861, 44884, 44907, 44931, 44954, 44977, 45000,
+ 45023, 45046, 45069, 45092, 45115, 45138, 45161, 45184, 45206, 45229,
+ 45252, 45275, 45298, 45321, 45344, 45366, 45389, 45412, 45435, 45457,
+ 45480, 45503, 45526, 45548, 45571, 45594, 45616, 45639, 45662, 45684,
+ 45707, 45729, 45752, 45774, 45797, 45819, 45842, 45864, 45887, 45909,
+ 45932, 45954, 45977, 45999, 46021, 46044, 46066, 46088, 46111, 46133,
+ 46155, 46178, 46200, 46222, 46244, 46267, 46289, 46311, 46333, 46355,
+ 46378, 46400, 46422, 46444, 46466, 46488, 46510, 46532, 46554, 46576,
+ 46599, 46621, 46643, 46665, 46686, 46708, 46730, 46752, 46774, 46796,
+ 46818, 46840, 46862, 46884, 46905, 46927, 46949, 46971, 46993, 47014,
+ 47036, 47058, 47080, 47101, 47123, 47145, 47167, 47188, 47210, 47231,
+ 47253, 47275, 47296, 47318, 47339, 47361, 47383, 47404, 47426, 47447,
+ 47469, 47490, 47512, 47533, 47555, 47576, 47597, 47619, 47640, 47662,
+ 47683, 47704, 47726, 47747, 47768, 47790, 47811, 47832, 47854, 47875,
+ 47896, 47917, 47939, 47960, 47981, 48002, 48023, 48045, 48066, 48087,
+ 48108, 48129, 48150, 48171, 48192, 48213, 48234, 48256, 48277, 48298,
+ 48319, 48340, 48361, 48382, 48403, 48423, 48444, 48465, 48486, 48507,
+ 48528, 48549, 48570, 48591, 48612, 48632, 48653, 48674, 48695, 48716,
+ 48736, 48757, 48778, 48799, 48819, 48840, 48861, 48881, 48902, 48923,
+ 48943, 48964, 48985, 49005, 49026, 49047, 49067, 49088, 49108, 49129,
+ 49149, 49170, 49191, 49211, 49232, 49252, 49273, 49293, 49313, 49334,
+ 49354, 49375, 49395, 49416, 49436, 49456, 49477, 49497, 49517, 49538,
+ 49558, 49578, 49599, 49619, 49639, 49660, 49680, 49700, 49720, 49741,
+ 49761, 49781, 49801, 49821, 49842, 49862, 49882, 49902, 49922, 49942,
+ 49962, 49982, 50003, 50023, 50043, 50063, 50083, 50103, 50123, 50143,
+ 50163, 50183, 50203, 50223, 50243, 50263, 50283, 50303, 50323, 50343,
+ 50362, 50382, 50402, 50422, 50442, 50462, 50482, 50501, 50521, 50541,
+ 50561, 50581, 50600, 50620, 50640, 50660, 50680, 50699, 50719, 50739,
+ 50758, 50778, 50798, 50817, 50837, 50857, 50876, 50896, 50916, 50935,
+ 50955, 50974, 50994, 51014, 51033, 51053, 51072, 51092, 51111, 51131,
+ 51150, 51170, 51189, 51209, 51228, 51248, 51267, 51287, 51306, 51325,
+ 51345, 51364, 51384, 51403, 51422, 51442, 51461, 51480, 51500, 51519,
+ 51538, 51558, 51577, 51596, 51616, 51635, 51654, 51673, 51693, 51712,
+ 51731, 51750, 51769, 51789, 51808, 51827, 51846, 51865, 51884, 51904,
+ 51923, 51942, 51961, 51980, 51999, 52018, 52037, 52056, 52075, 52094,
+ 52113, 52132, 52151, 52170, 52189, 52208, 52227, 52246, 52265, 52284,
+ 52303, 52322, 52341, 52360, 52379, 52398, 52417, 52436, 52454, 52473,
+ 52492, 52511, 52530, 52549, 52568, 52586, 52605, 52624, 52643, 52662,
+ 52680, 52699, 52718, 52737, 52755, 52774, 52793, 52811, 52830, 52849,
+ 52867, 52886, 52905, 52923, 52942, 52961, 52979, 52998, 53017, 53035,
+ 53054, 53072, 53091, 53109, 53128, 53147, 53165, 53184, 53202, 53221,
+ 53239, 53258, 53276, 53295, 53313, 53332, 53350, 53369, 53387, 53405,
+ 53424, 53442, 53461, 53479, 53497, 53516, 53534, 53553, 53571, 53589,
+ 53608, 53626, 53644, 53663, 53681, 53699, 53718, 53736, 53754, 53772,
+ 53791, 53809, 53827, 53845, 53864, 53882, 53900, 53918, 53936, 53955,
+ 53973, 53991, 54009, 54027, 54045, 54064, 54082, 54100, 54118, 54136,
+ 54154, 54172, 54190, 54208, 54226, 54244, 54263, 54281, 54299, 54317,
+ 54335, 54353, 54371, 54389, 54407, 54425, 54443, 54461, 54479, 54496,
+ 54514, 54532, 54550, 54568, 54586, 54604, 54622, 54640, 54658, 54676,
+ 54693, 54711, 54729, 54747, 54765, 54783, 54800, 54818, 54836, 54854,
+ 54872, 54889, 54907, 54925, 54943, 54960, 54978, 54996, 55014, 55031,
+ 55049, 55067, 55084, 55102, 55120, 55138, 55155, 55173, 55191, 55208,
+ 55226, 55243, 55261, 55279, 55296, 55314, 55331, 55349, 55367, 55384,
+ 55402, 55419, 55437, 55454, 55472, 55490, 55507, 55525, 55542, 55560,
+ 55577, 55595, 55612, 55629, 55647, 55664, 55682, 55699, 55717, 55734,
+ 55752, 55769, 55786, 55804, 55821, 55839, 55856, 55873, 55891, 55908,
+ 55925, 55943, 55960, 55977, 55995, 56012, 56029, 56047, 56064, 56081,
+ 56098, 56116, 56133, 56150, 56168, 56185, 56202, 56219, 56236, 56254,
+ 56271, 56288, 56305, 56322, 56340, 56357, 56374, 56391, 56408, 56425,
+ 56443, 56460, 56477, 56494, 56511, 56528, 56545, 56562, 56579, 56597,
+ 56614, 56631, 56648, 56665, 56682, 56699, 56716, 56733, 56750, 56767,
+ 56784, 56801, 56818, 56835, 56852, 56869, 56886, 56903, 56920, 56937,
+ 56954, 56971, 56988, 57004, 57021, 57038, 57055, 57072, 57089, 57106,
+ 57123, 57140, 57156, 57173, 57190, 57207, 57224, 57241, 57257, 57274,
+ 57291, 57308, 57325, 57341, 57358, 57375, 57392, 57408, 57425, 57442,
+ 57459, 57475, 57492, 57509, 57526, 57542, 57559, 57576, 57592, 57609,
+ 57626, 57642, 57659, 57676, 57692, 57709, 57726, 57742, 57759, 57776,
+ 57792, 57809, 57825, 57842, 57859, 57875, 57892, 57908, 57925, 57941,
+ 57958, 57975, 57991, 58008, 58024, 58041, 58057, 58074, 58090, 58107,
+ 58123, 58140, 58156, 58173, 58189, 58206, 58222, 58238, 58255, 58271,
+ 58288, 58304, 58321, 58337, 58353, 58370, 58386, 58403, 58419, 58435,
+ 58452, 58468, 58484, 58501, 58517, 58533, 58550, 58566, 58582, 58599,
+ 58615, 58631, 58648, 58664, 58680, 58697, 58713, 58729, 58745, 58762,
+ 58778, 58794, 58810, 58827, 58843, 58859, 58875, 58891, 58908, 58924,
+ 58940, 58956, 58972, 58989, 59005, 59021, 59037, 59053, 59069, 59085,
+ 59102, 59118, 59134, 59150, 59166, 59182, 59198, 59214, 59230, 59246,
+ 59263, 59279, 59295, 59311, 59327, 59343, 59359, 59375, 59391, 59407,
+ 59423, 59439, 59455, 59471, 59487, 59503, 59519, 59535, 59551, 59567,
+ 59583, 59599, 59615, 59631, 59647, 59663, 59678, 59694, 59710, 59726,
+ 59742, 59758, 59774, 59790, 59806, 59822, 59837, 59853, 59869, 59885,
+ 59901, 59917, 59933, 59948, 59964, 59980, 59996, 60012, 60027, 60043,
+ 60059, 60075, 60091, 60106, 60122, 60138, 60154, 60169, 60185, 60201,
+ 60217, 60232, 60248, 60264, 60280, 60295, 60311, 60327, 60342, 60358,
+ 60374, 60389, 60405, 60421, 60436, 60452, 60468, 60483, 60499, 60515,
+ 60530, 60546, 60561, 60577, 60593, 60608, 60624, 60640, 60655, 60671,
+ 60686, 60702, 60717, 60733, 60749, 60764, 60780, 60795, 60811, 60826,
+ 60842, 60857, 60873, 60888, 60904, 60919, 60935, 60950, 60966, 60981,
+ 60997, 61012, 61028, 61043, 61059, 61074, 61089, 61105, 61120, 61136,
+ 61151, 61167, 61182, 61197, 61213, 61228, 61244, 61259, 61274, 61290,
+ 61305, 61320, 61336, 61351, 61366, 61382, 61397, 61412, 61428, 61443,
+ 61458, 61474, 61489, 61504, 61520, 61535, 61550, 61566, 61581, 61596,
+ 61611, 61627, 61642, 61657, 61672, 61688, 61703, 61718, 61733, 61749,
+ 61764, 61779, 61794, 61809, 61825, 61840, 61855, 61870, 61885, 61901,
+ 61916, 61931, 61946, 61961, 61976, 61991, 62007, 62022, 62037, 62052,
+ 62067, 62082, 62097, 62112, 62128, 62143, 62158, 62173, 62188, 62203,
+ 62218, 62233, 62248, 62263, 62278, 62293, 62308, 62323, 62338, 62353,
+ 62369, 62384, 62399, 62414, 62429, 62444, 62459, 62474, 62489, 62504,
+ 62518, 62533, 62548, 62563, 62578, 62593, 62608, 62623, 62638, 62653,
+ 62668, 62683, 62698, 62713, 62728, 62743, 62757, 62772, 62787, 62802,
+ 62817, 62832, 62847, 62862, 62876, 62891, 62906, 62921, 62936, 62951,
+ 62966, 62980, 62995, 63010, 63025, 63040, 63054, 63069, 63084, 63099,
+ 63114, 63128, 63143, 63158, 63173, 63188, 63202, 63217, 63232, 63247,
+ 63261, 63276, 63291, 63306, 63320, 63335, 63350, 63364, 63379, 63394,
+ 63408, 63423, 63438, 63453, 63467, 63482, 63497, 63511, 63526, 63541,
+ 63555, 63570, 63585, 63599, 63614, 63628, 63643, 63658, 63672, 63687,
+ 63702, 63716, 63731, 63745, 63760, 63774, 63789, 63804, 63818, 63833,
+ 63847, 63862, 63876, 63891, 63906, 63920, 63935, 63949, 63964, 63978,
+ 63993, 64007, 64022, 64036, 64051, 64065, 64080, 64094, 64109, 64123,
+ 64138, 64152, 64167, 64181, 64196, 64210, 64224, 64239, 64253, 64268,
+ 64282, 64297, 64311, 64326, 64340, 64354, 64369, 64383, 64398, 64412,
+ 64426, 64441, 64455, 64470, 64484, 64498, 64513, 64527, 64541, 64556,
+ 64570, 64584, 64599, 64613, 64627, 64642, 64656, 64670, 64685, 64699,
+ 64713, 64728, 64742, 64756, 64770, 64785, 64799, 64813, 64828, 64842,
+ 64856, 64870, 64885, 64899, 64913, 64927, 64942, 64956, 64970, 64984,
+ 64999, 65013, 65027, 65041, 65055, 65070, 65084, 65098, 65112, 65126,
+ 65141, 65155, 65169, 65183, 65197, 65211, 65226, 65240, 65254, 65268,
+ 65282, 65296, 65310, 65325, 65339, 65353, 65367, 65381, 65395, 65409,
+ 65423, 65437, 65452, 65466, 65480, 65494, 65508, 65522, 65535}};
+
+void set_gamma_params(GammaParams* params) { gamma_params = *params; }
+
+void gamma_process(Image* img) {
+ if (!gamma_params.enable) return;
+
+ size_t vl;
+ size_t n = img->height * img->width * img->num_channels;
+ // auxiliary variables
+ vuint16m8_t vx, vy;
+ vuint16m8_t v_offset;
+ vbool2_t mask;
+ uint16_t* x;
+
+ for (size_t i = 0; i < n; i += vl) {
+ x = img->data + i;
+
+ vl = vsetvl_e16m8(n - i);
+ vx = vle16_v_u16m8(x, vl); // load
+ mask = vmsne(vx, kRgbPipelineMaxVal,
+ vl); // mask to exclude kRgbPipelineMaxVal
+
+ vy = vsrl(vx, kGammaShiftBits, vl); // 1/32
+ vy = vsll(vy, 1, vl); // *2
+ v_offset = vand(vx, kGammaSpacing - 1, vl); // offset within bin
+
+ vx = vluxei16(gamma_params.lut, vy, vl); // left
+ vy = vluxei16(gamma_params.lut + 1, vy, vl); // right
+
+ vy = vsub(vy, vx, vl); // right - left
+ vy = vmul(vy, v_offset, vl); // multiply offset_within_bin
+ vy = vsrl(vy, kGammaShiftBits, vl); // 1/32
+ vx = vadd(vx, vy, vl); // add
+
+ vse16(mask, x, vx, vl); // save
+ }
+}
diff --git a/samples/risp4ml/isp_stages/gamma_rvv_test.cc b/samples/risp4ml/isp_stages/gamma_rvv_test.cc
new file mode 100644
index 0000000..e379e6d
--- /dev/null
+++ b/samples/risp4ml/isp_stages/gamma_rvv_test.cc
@@ -0,0 +1,141 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Use separate test for RVV as gamma LUT is generated differently
+#include <cmath>
+
+#include "pw_unit_test/framework.h"
+#include "samples/risp4ml/isp_stages/gamma.h"
+
+static constexpr uint16_t kRgbPipelineBpp = 16;
+static constexpr uint16_t kPostGammaPipelineBpp = 16;
+static constexpr uint16_t kRgbPipelineMaxVal = (1 << kRgbPipelineBpp) - 1;
+static constexpr uint16_t kGammaShiftBits = 5;
+static constexpr uint16_t kGammaSpacing = (1 << kGammaShiftBits);
+
+class GammaRvvTest : public ::testing::Test {
+ protected:
+ void setup(uint16_t width) {
+ in_ = image_new(3, 2, width);
+ for (uint16_t c = 0; c < in_->num_channels; ++c) {
+ for (uint16_t y = 0; y < in_->height; ++y) {
+ for (uint16_t x = 0; x < in_->width; ++x) {
+ *image_pixel(in_, c, y, x) = x;
+ }
+ }
+ }
+ out_ = image_new(3, 2, width);
+ const uint32_t num_bytes =
+ in_->num_channels * in_->height * in_->width * sizeof(pixel_type_t);
+ memcpy(out_->data, in_->data, num_bytes);
+ }
+ void TearDown() override {
+ image_delete(in_);
+ image_delete(out_);
+ }
+ void CreateLinearGamma(GammaParams* params) {
+ params->enable = true;
+ for (int n = 0; n <= kRgbPipelineMaxVal; n += kGammaSpacing) {
+ params->lut[n / kGammaSpacing] = n;
+ }
+ params->lut[kGammaNumberPoints - 1] = kRgbPipelineMaxVal;
+ }
+ float sRgb_gamma(float in_) {
+ return (in_ < 0.0031308f) ? 12.92f * in_
+ : 1.055f * std::pow(in_, 1.0f / 2.4f) - 0.055f;
+ }
+ void CreateRgbGamma(GammaParams* params) {
+ params->enable = true;
+ params->lut[0] = 0;
+
+ for (int n = 0; n <= kRgbPipelineMaxVal; n += kGammaSpacing) {
+ params->lut[n / kGammaSpacing] =
+ (1 << kRgbPipelineBpp) *
+ sRgb_gamma((float)n / (1 << kRgbPipelineBpp));
+ }
+ params->lut[kGammaNumberPoints - 1] = kRgbPipelineMaxVal;
+ }
+
+ Image* in_;
+ Image* out_;
+};
+
+TEST_F(GammaRvvTest, Bypass) {
+ setup((1 << 15) - 1);
+
+ GammaParams params;
+ CreateRgbGamma(¶ms);
+ params.enable = false;
+
+ set_gamma_params(¶ms);
+
+ gamma_process(out_);
+
+ for (uint16_t c = 0; c < in_->num_channels; ++c) {
+ for (uint16_t y = 0; y < in_->height; ++y) {
+ for (uint16_t x = 0; x < in_->width; ++x) {
+ pixel_type_t expected_val =
+ x >> (kRgbPipelineBpp - kPostGammaPipelineBpp);
+ ASSERT_EQ(expected_val, image_pixel_val(out_, c, y, x));
+ }
+ }
+ }
+}
+
+TEST_F(GammaRvvTest, Linear) {
+ setup((1 << 15) - 2);
+
+ GammaParams params;
+ CreateLinearGamma(¶ms);
+
+ set_gamma_params(¶ms);
+
+ gamma_process(out_);
+
+ for (uint16_t c = 0; c < in_->num_channels; ++c) {
+ for (uint16_t y = 0; y < in_->height; ++y) {
+ for (uint16_t x = 0; x < in_->width; ++x) {
+ ASSERT_EQ(image_pixel_val(out_, 0, y, x),
+ image_pixel_val(in_, 0, y, x));
+ }
+ }
+ }
+}
+
+TEST_F(GammaRvvTest, sRgbLUT) {
+ setup((1 << 15) - 1);
+
+ constexpr float kToleranceRatio = 0.03;
+
+ GammaParams params;
+ CreateRgbGamma(¶ms);
+ set_gamma_params(¶ms);
+
+ gamma_process(out_);
+
+ for (uint16_t c = 0; c < in_->num_channels; ++c) {
+ for (uint16_t y = 0; y < in_->height; ++y) {
+ for (uint16_t x = 0; x < in_->width; ++x) {
+ pixel_type_t expected_val =
+ (pixel_type_t)((1 << kRgbPipelineBpp) *
+ sRgb_gamma(static_cast<float>(x) /
+ (1 << kRgbPipelineBpp)));
+ float tolerance = ceilf(kToleranceRatio * expected_val);
+ float diff = std::abs((float)expected_val -
+ (float)image_pixel_val(out_, c, y, x));
+ ASSERT_LE(diff, tolerance);
+ }
+ }
+ }
+}
diff --git a/samples/risp4ml/isp_stages/wbg_rvv.c b/samples/risp4ml/isp_stages/wbg_rvv.c
new file mode 100644
index 0000000..a628f66
--- /dev/null
+++ b/samples/risp4ml/isp_stages/wbg_rvv.c
@@ -0,0 +1,122 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <riscv_vector.h>
+
+#include "samples/risp4ml/common/utils.h"
+#include "samples/risp4ml/isp_stages/wbg.h"
+
+#define kBayerColorChannels 4
+#define MAX(a, b) (((a) > (b)) ? (a) : (b))
+
+static const uint16_t kWbgFractional = kRawPipelineFraction;
+static const uint16_t kWbgUnityGain = 1 << kWbgFractional;
+static WbgParams wbg_params = {
+ .enable = true,
+ .fixed = false,
+ .gains = {kWbgUnityGain, kWbgUnityGain, kWbgUnityGain, kWbgUnityGain}};
+
+void set_wbg_params(WbgParams* params) { wbg_params = *params; }
+
+static void compute_wbg_gain(Image* img) {
+ // Calculate the white-balance gain values using the "gray world" algorithm
+ uint32_t size = img->num_channels * img->height * img->width;
+ uint64_t sum_of_reds = 0;
+ // will use only one of the greens for scaling, since the difference between
+ // the two green sensor pixels is negligible
+ uint64_t sum_of_greens = 0;
+ uint64_t sum_of_blues = 0;
+ size_t vl;
+ // auxiliary variables
+ vuint16m8_t vx;
+ vuint32m1_t vy;
+
+ for (uint16_t y = 0; y < img->height; ++y) {
+ pixel_type_t* line = image_row(img, 0, y);
+ for (uint8_t n = 0; n < 2; n++) {
+ for (uint16_t x = n; x < img->width; x += 2 * vl) {
+ size_t avl = (img->width + 1 - x) / 2;
+ vl = vsetvl_e16m8(avl);
+ vx = vlse16_v_u16m8(line + x, 2 * sizeof(uint16_t), vl); // load
+
+ vy = vmv_v_x_u32m1(0, vl); // init
+ vy = vwredsumu(vy, vx, vy, vl); // sum
+ uint32_t sum = vmv_x(vy);
+ if ((y & 0x1) == 0 && n == 0) {
+ sum_of_reds += sum;
+ } else if ((y & 0x1) == 1 && n == 1) {
+ sum_of_blues += sum;
+ } else {
+ sum_of_greens += sum;
+ }
+ }
+ }
+ }
+
+ // scale values to green channel
+ float average_red = 4.0 * sum_of_reds / size;
+ float average_green = 2.0 * sum_of_greens / size;
+ float average_blue = 4.0 * sum_of_blues / size;
+
+ float max_average = MAX(MAX(average_red, average_green), average_blue);
+
+ // Convert the float value to fixed point representation, i.e. 0xFF.FF
+ uint32_t red_wb = FloatToFixedPoint(max_average / average_red,
+ kRawPipelineInteger, kRawPipelineFraction,
+ /*bool is_signed*/ false);
+ uint32_t green_wb = FloatToFixedPoint(
+ max_average / average_green, kRawPipelineInteger, kRawPipelineFraction,
+ /*bool is_signed*/ false);
+ uint32_t blue_wb = FloatToFixedPoint(
+ max_average / average_blue, kRawPipelineInteger, kRawPipelineFraction,
+ /*bool is_signed*/ false);
+
+ wbg_params.gains[0] = red_wb;
+ wbg_params.gains[1] = green_wb;
+ wbg_params.gains[2] = green_wb;
+ wbg_params.gains[3] = blue_wb;
+}
+
+void wbg_process(Image* img) {
+ if (!wbg_params.enable) return;
+ if (!wbg_params.fixed) {
+ compute_wbg_gain(img);
+ }
+
+ size_t vl;
+ uint32_t offset = 1 << (kWbgFractional - 1);
+ uint32_t max_val = kRawPipelineMaxVal << kWbgFractional;
+ uint16_t gain;
+ // auxiliary variables
+ vuint16m4_t vx;
+ vuint32m8_t vy;
+ for (uint16_t y = 0; y < img->height; ++y) {
+ pixel_type_t* line = image_row(img, 0, y);
+ for (uint8_t n = 0; n < 2; n++) {
+ gain = (y & 0x1) ? wbg_params.gains[2 + n] : wbg_params.gains[n];
+ for (uint16_t x = n; x < img->width; x += 2 * vl) {
+ size_t avl = (img->width + 1 - x) / 2;
+ vl = vsetvl_e16m4(avl);
+ vx = vlse16_v_u16m4(line + x, 2 * sizeof(uint16_t), vl); // load
+ vy = vwmulu(vx, gain, vl); // multiply
+ vy = vadd(vy, offset, vl); // add
+ vy = vminu(vy, max_val, vl); // clamp
+ vx = vnsrl(vy, kWbgFractional, vl); // bit shift
+ vsse16(line + x, 2 * sizeof(uint16_t), vx, vl); // save
+ }
+ }
+ }
+}
