Rewrite RISP4ML in plain-C for Shodan This CL is Part 1 of the comprehensive 3-part changes of integrating RISP4ML into Shodan ML toolchain. This change focuses on re-writing RISP4ML in plain-C. Part 2 and 3 will be submitted after this CL is merged (codes can be seen at Patch Set #13). Details of the 3-part changes: (Part 1) Completely re-wrote RISP4ML in plain C. - The original RISP4ML on google3 was written in C++. The memory usage with C++ RISP4ML was significantly larger. Thus, I have completely re-written RISP4ML in plain C. The memory increase with the plain-C RISP4ML is much smaller. - Made the input/output shape to be configurable arguments instead of constants. - Made low-level optimizations and simplifications. (Part 2) Integrated plain-C RISP4ML into Shodan ML toolchain - Use fssd_25_8bit_v2 as an example to load the binary (raw bayer) file and go through RISP4ML toolchain, and feed the RISP4ML output into IREE flow. - The fssd_25_8bit_v2 example is up and running, and the correctness of output has been verified (compare output@plain-C risp4ml against output@google3). (Part 3) Added unit tests for plain-C RISP4ML - Added unit tests based on pw_unit_test - Build unit tests in iree_cc_binary - Run and check unit tests via lit_test Change-Id: Id17dd01fd3848d9705b8b39ea2866bde3334613c
diff --git a/samples/risp4ml/CMakeLists.txt b/samples/risp4ml/CMakeLists.txt new file mode 100644 index 0000000..0e9c88b --- /dev/null +++ b/samples/risp4ml/CMakeLists.txt
@@ -0,0 +1 @@ +iree_add_all_subdirs()
diff --git a/samples/risp4ml/common/CMakeLists.txt b/samples/risp4ml/common/CMakeLists.txt new file mode 100644 index 0000000..49346c1 --- /dev/null +++ b/samples/risp4ml/common/CMakeLists.txt
@@ -0,0 +1,26 @@ +iree_cc_library( + NAME + image + HDRS + "image.h" + SRCS + "image.c" +) + +iree_cc_library( + NAME + test_utils + HDRS + "test_utils.h" + DEPS + ::image +) + +iree_cc_library( + NAME + utils + HDRS + "utils.h" + SRCS + "utils.c" +)
diff --git a/samples/risp4ml/common/constants.h b/samples/risp4ml/common/constants.h new file mode 100644 index 0000000..5d3e658 --- /dev/null +++ b/samples/risp4ml/common/constants.h
@@ -0,0 +1,49 @@ +#ifndef SAMPLES_RISP4ML_COMMON_CONSTANTS_H_ +#define SAMPLES_RISP4ML_COMMON_CONSTANTS_H_ + +#include <stdint.h> + +// Input and output are expected to be 8 bits per pixel +static const uint16_t kPipeInputBpp = 8; +static const uint16_t kPipeOutputBpp = 8; + +// TODO(b/149969920): modify sign,bitwidths and internal precision for risp4ml +// Assume 8.8 format +static const uint16_t kRawPipelineBpp = 16; +static const uint16_t kRawPipelineInteger = 8; +static const uint16_t kRawPipelineFraction = 8; + +// max = 0xFF.FF = 65535 +// represnting 255 + 255/256 +static const uint16_t kRawPipelineMaxVal = 0xFFFF; +static const uint16_t kRawPipelineMinVal = 0; +// min fraction = 0x00.01 = 1/256 +static const uint16_t kRawPipelineMinFraction = 1; + +// BayerIndex defines the order in 2x2 normal Bayer quad. +// +---+---+ +// | R | Gr| +// +---+---+ +// | Gb| B | +// +---+---+ +typedef enum { + kR = 0, + kGr = 1, + kGb = 2, + kB = 3, +} BayerIndex; + +typedef enum { + kRggb = 0, + kGrbg = 1, + kGbrg = 2, + kBggr = 3, +} BayerPattern; + +#define kNumBayerPatterns 4 + +// TODO(alexkaplan): Add a way to update this based on the image +// or to make sure the BayerType corresponds to the loaded image +static const BayerPattern kBayerType = kRggb; + +#endif // SAMPLES_RISP4ML_COMMON_CONSTANTS_H_
diff --git a/samples/risp4ml/common/image.c b/samples/risp4ml/common/image.c new file mode 100644 index 0000000..5c77445 --- /dev/null +++ b/samples/risp4ml/common/image.c
@@ -0,0 +1,35 @@ +#include <stdlib.h> + +#include "samples/risp4ml/common/image.h" + +Image* image_new(uint16_t num_channels, uint16_t height, uint16_t width) { + Image* image = (Image*)malloc(sizeof(Image)); + if (image) { + image->num_channels = num_channels; + image->height = height; + image->width = width; + uint32_t num_pixels = width * height * num_channels; + image->data = (pixel_type_t*)malloc(num_pixels * sizeof(pixel_type_t)); + } + return image; +} + +void image_delete(Image* image) { + if (image) { + if (image->data) free(image->data); + free(image); + } +} + +pixel_type_t* image_pixel(Image* image, uint16_t c, uint16_t y, uint16_t x) { + const uint32_t stride_c = image->width * image->height; + const uint16_t stride_y = image->width; + const uint16_t stride_x = 1; + return (image->data + c * stride_c + y * stride_y + x * stride_x); +} + +pixel_type_t* image_row(Image* image, uint16_t c, uint16_t y) { + const uint32_t stride_c = image->width * image->height; + const uint16_t stride_y = image->width; + return (image->data + c * stride_c + y * stride_y); +}
diff --git a/samples/risp4ml/common/image.h b/samples/risp4ml/common/image.h new file mode 100644 index 0000000..12c1fb4 --- /dev/null +++ b/samples/risp4ml/common/image.h
@@ -0,0 +1,43 @@ +#ifndef SAMPLES_RISP4ML_COMMON_IMAGE_H_ +#define SAMPLES_RISP4ML_COMMON_IMAGE_H_ + +#include <stdint.h> + +#ifdef __cplusplus +extern "C" { +#endif // __cplusplus + +typedef uint16_t pixel_type_t; + +typedef struct { + uint16_t num_channels; + uint16_t height; + uint16_t width; + pixel_type_t* data; +} Image; + +typedef struct { + uint16_t num_channels; + uint16_t height; + uint16_t width; + uint8_t* data; +} ImageU8; + +Image* image_new(uint16_t num_channels, uint16_t height, uint16_t width); + +void image_delete(Image* image); + +pixel_type_t* image_pixel(Image* image, uint16_t c, uint16_t y, uint16_t x); + +inline pixel_type_t image_pixel_val(Image* image, uint16_t c, uint16_t y, + uint16_t x) { + return *image_pixel(image, c, y, x); +} + +pixel_type_t* image_row(Image* image, uint16_t c, uint16_t y); + +#ifdef __cplusplus +} // extern "C" +#endif // __cplusplus + +#endif // SAMPLES_RISP4ML_COMMON_IMAGE_H_
diff --git a/samples/risp4ml/common/test_utils.h b/samples/risp4ml/common/test_utils.h new file mode 100644 index 0000000..616b3da --- /dev/null +++ b/samples/risp4ml/common/test_utils.h
@@ -0,0 +1,42 @@ +#include <stdio.h> +#include <stdlib.h> + +#include "samples/risp4ml/common/image.h" + +inline pixel_type_t Pattern(uint16_t c, uint16_t y, uint16_t x) { + return (pixel_type_t)(x + y * 100 + c * 10000); +} + +inline void FillImage(Image* img) { + for (uint16_t c = 0; c < img->num_channels; ++c) { + for (uint16_t y = 0; y < img->height; ++y) { + for (uint16_t x = 0; x < img->width; ++x) { + *image_pixel(img, c, y, x) = Pattern(c, y, x); + } + } + } +} + +// Initializes raw image to random value within min and max range +inline void InitImageRandom(Image* image, pixel_type_t min_val, + pixel_type_t max_val) { + pixel_type_t range = max_val + 1 - min_val; + for (uint16_t c = 0; c < image->num_channels; ++c) { + for (uint16_t y = 0; y < image->height; ++y) { + for (uint16_t x = 0; x < image->width; ++x) { + *image_pixel(image, 0, y, x) = (pixel_type_t)(rand() % range + min_val); + } + } + } +} + +// Initializes raw image to raw pixel value +inline void InitImage(Image* image, pixel_type_t val) { + for (uint16_t c = 0; c < image->num_channels; ++c) { + for (uint16_t y = 0; y < image->height; ++y) { + for (uint16_t x = 0; x < image->width; ++x) { + *image_pixel(image, 0, y, x) = val; + } + } + } +}
diff --git a/samples/risp4ml/common/utils.c b/samples/risp4ml/common/utils.c new file mode 100644 index 0000000..83386b1 --- /dev/null +++ b/samples/risp4ml/common/utils.c
@@ -0,0 +1,37 @@ +#include "samples/risp4ml/common/utils.h" + +// int GetBayerIndex(int x, int y) { +// // The Bayer pattern code defines which color is top left in the quad: +// // +---+---+ +// // | R | Gr| +// // +---+---+ +// // | Gb| B | +// // +---+---+ +// return ((x & 1) + 2 * (y & 1)); +// } + +BayerIndex GetBayerIndex(BayerPattern bayerType, uint16_t x, uint16_t y) { + // The Bayer pattern code defines which color is top left in the quad: + // 0: +---+---+ 1: +---+---+ 2: +---+---+ 3: +---+---+ + // | R | Gr| | Gr| R | | Gb| B | | B | Gb| + // +---+---+ +---+---+ +---+---+ +---+---+ + // | Gb| B | | B | Gb| | R | Gr| | Gr| R | + // +---+---+ +---+---+ +---+---+ +---+---+ + // pattern 0 is base pattern and other patterns are shifted versions of the + // base + + // Patterns 1 and 3 shift in the x + uint16_t x_shift = (uint16_t)(bayerType == kGrbg || bayerType == kBggr); + // Patterns 2 and 3 shift in the y + uint16_t y_shift = (uint16_t)(bayerType == kGbrg || bayerType == kBggr); + return (BayerIndex)(((x + x_shift) & 1) + 2 * ((y + y_shift) & 1)); +} + +int BayerMirrorBoundary(int x, int size) { + if (x < 0) + return (-x + 2 * (-x & 0x1) - 2); + else if (x < size) + return x; + else + return 2 * size - x - 2 * ((x - size + 1) & 0x1); +}
diff --git a/samples/risp4ml/common/utils.h b/samples/risp4ml/common/utils.h new file mode 100644 index 0000000..bc29ea5 --- /dev/null +++ b/samples/risp4ml/common/utils.h
@@ -0,0 +1,86 @@ +#ifndef SAMPLES_RISP4ML_COMMON_UTILS_H_ +#define SAMPLES_RISP4ML_COMMON_UTILS_H_ + +#include <math.h> +#include <stdbool.h> +#include <stdio.h> +#include <stdlib.h> + +#include "samples/risp4ml/common/constants.h" + +// Return the RAW color channel index at position (x, y) for a given Bayer +// pattern +// The Bayer pattern code defines which color is top left in the quad: +// 0: +---+---+ 1: +---+---+ 2: +---+---+ 3: +---+---+ +// | R | Gr| | Gr| R | | Gb| B | | B | Gb| +// +---+---+ +---+---+ +---+---+ +---+---+ +// | Gb| B | | B | Gb| | R | Gr| | Gr| R | +// +---+---+ +---+---+ +---+---+ +---+---+ +// pattern 0 is base pattern and other patterns are shifted versions of the +// base +BayerIndex GetBayerIndex(BayerPattern bayerType, uint16_t x, uint16_t y); + +// Get the corresponding index of x in bayer images for when the index is out +// of bounds and mirrored across the boundary. +int BayerMirrorBoundary(int x, int size); + +inline uint32_t Clamp(uint32_t value, uint32_t low, uint32_t high) { + return value < low ? low : (value > high ? high : value); +} + +inline uint16_t SubUnsignedZeroClamp(uint16_t lhs, uint16_t rhs) { + return rhs < lhs ? lhs - rhs : 0; +} + +// Count the number of consecutive zeros from LHS in N msbs of the number +// represented using BPP bits +inline int ClzMsb(int in, int BPP, int N) { + int lz = 0; + while (lz < N && (in & (1 << (BPP - lz - 1))) == 0) { + ++lz; + } + return lz; +} + +inline float Roundf(float x) { + int d = x < 0 ? x - 0.5 : x + 0.5; + return (float)d; +} + +// This function converts floating point value `x` to fixed point with the +// specified `integer_bit`, `frac_bit`, and `is_signed` flag. +// TODO(alexkaplan): Detect overflow/underflow. +inline int FloatToFixedPoint(float x, int integer_bit, int frac_bit, + bool is_signed) { + float output_as_float = Roundf(x * (1 << frac_bit)); + float min_value = 0; + float max_value = (1 << (frac_bit + integer_bit)) - 1; + + if (is_signed) { + min_value = -(1 << (frac_bit + integer_bit - 1)); + max_value = (1 << (frac_bit + integer_bit - 1)) - 1; + } + + // Clamp to the allowed range. + if (output_as_float < min_value) { + return (int)min_value; + } else if (output_as_float > max_value) { + return (int)max_value; + } + return (int)output_as_float; +} + +// Helper function for fixed point rounding of values. +inline int Round(int value, int right_shift) { + int carry = right_shift == 0 ? 0 : (value >> (right_shift - 1)) & 1; + return (value >> right_shift) + carry; +} + +// Helper function for linearly interpolating 2 values. When weight equals 0, +// output = val0. When weight equals 1.0 (when represented in floating point), +// output = val1. +inline int Lerp(int val0, int val1, int weight, int weight_precision) { + return val0 + Round((val1 - val0) * weight, weight_precision); +} + +#endif // SAMPLES_RISP4ML_COMMON_UTILS_H_
diff --git a/samples/risp4ml/isp_stages/CMakeLists.txt b/samples/risp4ml/isp_stages/CMakeLists.txt new file mode 100644 index 0000000..54022b7 --- /dev/null +++ b/samples/risp4ml/isp_stages/CMakeLists.txt
@@ -0,0 +1,71 @@ +iree_cc_library( + NAME + blc + HDRS + "blc.h" + SRCS + "blc.c" + DEPS + samples::risp4ml::common::image + samples::risp4ml::common::utils +) + +iree_cc_library( + NAME + demosaic + HDRS + "demosaic.h" + SRCS + "demosaic.c" + DEPS + samples::risp4ml::common::image + samples::risp4ml::common::utils +) + +iree_cc_library( + NAME + dg + HDRS + "dg.h" + SRCS + "dg.c" + DEPS + samples::risp4ml::common::image + samples::risp4ml::common::utils +) + +iree_cc_library( + NAME + downscale + HDRS + "downscale.h" + SRCS + "downscale.c" + DEPS + samples::risp4ml::common::image + samples::risp4ml::common::utils +) + +iree_cc_library( + NAME + gamma + HDRS + "gamma.h" + SRCS + "gamma.c" + DEPS + samples::risp4ml::common::image + samples::risp4ml::common::utils +) + +iree_cc_library( + NAME + wbg + HDRS + "wbg.h" + SRCS + "wbg.c" + DEPS + samples::risp4ml::common::image + samples::risp4ml::common::utils +)
diff --git a/samples/risp4ml/isp_stages/blc.c b/samples/risp4ml/isp_stages/blc.c new file mode 100644 index 0000000..26e36b5 --- /dev/null +++ b/samples/risp4ml/isp_stages/blc.c
@@ -0,0 +1,28 @@ +#include "samples/risp4ml/common/utils.h" +#include "samples/risp4ml/isp_stages/blc.h" + +static BlcParams blc_params = {.enable = true, + .offsets = {2048, 2048, 2048, 2048}}; + +void set_blc_params(BlcParams* params) { blc_params = *params; } + +void blc_process(Image* input, Image* output) { + if (!blc_params.enable) { + *output = *input; + return; + } + + uint16_t height = input->height; + uint16_t width = input->width; + + for (uint16_t y = 0; y < height; ++y) { + const pixel_type_t* in_line = image_row(input, 0, y); + pixel_type_t* out_line = image_row(output, 0, y); + + for (uint16_t x = 0; x < width; ++x) { + BayerIndex bayer_index = GetBayerIndex(kBayerType, x, y); + out_line[x] = + SubUnsignedZeroClamp(in_line[x], blc_params.offsets[bayer_index]); + } + } +}
diff --git a/samples/risp4ml/isp_stages/blc.h b/samples/risp4ml/isp_stages/blc.h new file mode 100644 index 0000000..4ad1466 --- /dev/null +++ b/samples/risp4ml/isp_stages/blc.h
@@ -0,0 +1,23 @@ +#ifndef SAMPLES_RISP4ML_ISP_STAGES_BLC_H_ +#define SAMPLES_RISP4ML_ISP_STAGES_BLC_H_ + +#include "samples/risp4ml/common/image.h" + +#ifdef __cplusplus +extern "C" { +#endif // __cplusplus + +typedef struct { + bool enable; + pixel_type_t offsets[kNumBayerPatterns]; +} BlcParams; + +void set_blc_params(BlcParams* params); + +void blc_process(Image* input, Image* output); + +#ifdef __cplusplus +} // extern "C" +#endif // __cplusplus + +#endif // SAMPLES_RISP4ML_ISP_STAGES_BLC_H_
diff --git a/samples/risp4ml/isp_stages/demosaic.c b/samples/risp4ml/isp_stages/demosaic.c new file mode 100644 index 0000000..55a784b --- /dev/null +++ b/samples/risp4ml/isp_stages/demosaic.c
@@ -0,0 +1,78 @@ +#include <assert.h> + +#include "samples/risp4ml/common/utils.h" +#include "samples/risp4ml/isp_stages/demosaic.h" + +#define kRgbColorChannels 3 + +static DemosaicParams demosaic_params = {.enable = true}; + +void set_demosaic_params(DemosaicParams* params) { demosaic_params = *params; } + +// Basic bilinear demosaic +void demosaic_process(Image* input, Image* output) { + if (!demosaic_params.enable) { + return; + } + uint16_t height = input->height; + uint16_t width = input->width; + + const pixel_type_t* line_buffers[kRgbColorChannels]; + int x_offset[kRgbColorChannels]; + + for (uint16_t y = 0; y < height; ++y) { + line_buffers[0] = (y) ? image_row(input, 0, y - 1) : image_row(input, 0, 1); + line_buffers[1] = image_row(input, 0, y); + line_buffers[2] = (y < height - 1) ? image_row(input, 0, y + 1) + : image_row(input, 0, height - 2); + + for (uint16_t x = 0; x < width; ++x) { + for (uint16_t c = 0; c < kRgbColorChannels; ++c) { + x_offset[c] = BayerMirrorBoundary(x - 1 + c, width); + } + + BayerIndex bayer_index = GetBayerIndex(kBayerType, x, y); + switch (bayer_index) { + case (kR): { + *image_pixel(output, 0, y, x) = line_buffers[1][x_offset[1]]; + *image_pixel(output, 1, y, x) = + (line_buffers[0][x_offset[1]] + line_buffers[2][x_offset[1]] + + line_buffers[1][x_offset[0]] + line_buffers[1][x_offset[2]]) / + 4; + *image_pixel(output, 2, y, x) = + (line_buffers[0][x_offset[0]] + line_buffers[0][x_offset[2]] + + line_buffers[2][x_offset[0]] + line_buffers[2][x_offset[2]]) / + 4; + }; break; + case (kGr): { + *image_pixel(output, 0, y, x) = + (line_buffers[1][x_offset[0]] + line_buffers[1][x_offset[2]]) / 2; + *image_pixel(output, 1, y, x) = line_buffers[1][x_offset[1]]; + *image_pixel(output, 2, y, x) = + (line_buffers[0][x_offset[1]] + line_buffers[2][x_offset[1]]) / 2; + }; break; + case (kGb): { + *image_pixel(output, 0, y, x) = + (line_buffers[0][x_offset[1]] + line_buffers[2][x_offset[1]]) / 2; + *image_pixel(output, 1, y, x) = line_buffers[1][x_offset[1]]; + *image_pixel(output, 2, y, x) = + (line_buffers[1][x_offset[0]] + line_buffers[1][x_offset[2]]) / 2; + }; break; + case (kB): { + *image_pixel(output, 0, y, x) = + (line_buffers[0][x_offset[0]] + line_buffers[0][x_offset[2]] + + line_buffers[2][x_offset[0]] + line_buffers[2][x_offset[2]]) / + 4; + *image_pixel(output, 1, y, x) = + (line_buffers[0][x_offset[1]] + line_buffers[2][x_offset[1]] + + line_buffers[1][x_offset[0]] + line_buffers[1][x_offset[2]]) / + 4; + *image_pixel(output, 2, y, x) = line_buffers[1][x_offset[1]]; + }; break; + default: { + assert(0 && "Unexpected channel index"); + } + } + } + } +}
diff --git a/samples/risp4ml/isp_stages/demosaic.h b/samples/risp4ml/isp_stages/demosaic.h new file mode 100644 index 0000000..d32ae98 --- /dev/null +++ b/samples/risp4ml/isp_stages/demosaic.h
@@ -0,0 +1,22 @@ +#ifndef SAMPLES_RISP4ML_ISP_STAGES_DEMOSAIC_H_ +#define SAMPLES_RISP4ML_ISP_STAGES_DEMOSAIC_H_ + +#include "samples/risp4ml/common/image.h" + +#ifdef __cplusplus +extern "C" { +#endif // __cplusplus + +typedef struct { + bool enable; +} DemosaicParams; + +void set_demosaic_params(DemosaicParams* params); + +void demosaic_process(Image* input, Image* output); + +#ifdef __cplusplus +} // extern "C" +#endif // __cplusplus + +#endif // SAMPLES_RISP4ML_ISP_STAGES_DEMOSAIC_H_
diff --git a/samples/risp4ml/isp_stages/dg.c b/samples/risp4ml/isp_stages/dg.c new file mode 100644 index 0000000..73f7f46 --- /dev/null +++ b/samples/risp4ml/isp_stages/dg.c
@@ -0,0 +1,36 @@ +#include "samples/risp4ml/common/utils.h" +#include "samples/risp4ml/isp_stages/dg.h" + +static const uint16_t kDgFractional = kRawPipelineFraction; +static const uint16_t kDgUnityGain = 1 << kDgFractional; +static DgParams dg_params = { + .enable = true, + .gains = {kDgUnityGain, kDgUnityGain, kDgUnityGain, kDgUnityGain}}; + +void set_dg_params(DgParams* params) { dg_params = *params; } + +void dg_process(Image* input, Image* output) { + if (!dg_params.enable) { + *output = *input; + return; + } + + uint16_t height = input->height; + uint16_t width = input->width; + + for (uint16_t y = 0; y < height; ++y) { + const pixel_type_t* in_line = image_row(input, 0, y); + pixel_type_t* out_line = image_row(output, 0, y); + + for (uint16_t x = 0; x < width; ++x) { + BayerIndex bayer_index = GetBayerIndex(kBayerType, x, y); + uint16_t input_val = in_line[x]; + // + (1 << (kDgFractional -1)) adds 0.5 for more accurate rounding + uint32_t scaled_pixel = + input_val * dg_params.gains[bayer_index] + (1 << (kDgFractional - 1)); + + out_line[x] = (pixel_type_t)Clamp(scaled_pixel >> kDgFractional, 0, + kRawPipelineMaxVal); + } + } +}
diff --git a/samples/risp4ml/isp_stages/dg.h b/samples/risp4ml/isp_stages/dg.h new file mode 100644 index 0000000..1a0bc1b --- /dev/null +++ b/samples/risp4ml/isp_stages/dg.h
@@ -0,0 +1,23 @@ +#ifndef SAMPLES_RISP4ML_ISP_STAGES_DG_H_ +#define SAMPLES_RISP4ML_ISP_STAGES_DG_H_ + +#include "samples/risp4ml/common/image.h" + +#ifdef __cplusplus +extern "C" { +#endif // __cplusplus + +typedef struct { + bool enable; + pixel_type_t gains[kNumBayerPatterns]; +} DgParams; + +void set_dg_params(DgParams* params); + +void dg_process(Image* input, Image* output); + +#ifdef __cplusplus +} // extern "C" +#endif // __cplusplus + +#endif // SAMPLES_RISP4ML_ISP_STAGES_DG_H_
diff --git a/samples/risp4ml/isp_stages/downscale.c b/samples/risp4ml/isp_stages/downscale.c new file mode 100644 index 0000000..ac8e6f5 --- /dev/null +++ b/samples/risp4ml/isp_stages/downscale.c
@@ -0,0 +1,122 @@ +#include <assert.h> + +#include "samples/risp4ml/common/utils.h" +#include "samples/risp4ml/isp_stages/downscale.h" + +static const uint16_t kScalePrecision = 8; +static const uint16_t kScaleFixedOne = (1 << kScalePrecision); + +static DownscaleParams params = { + .enable = true, + .scale_precision = kScalePrecision, + .interpolate_precision = 8, + .interpolate_shift = 2, + .scale_fixed_one = kScaleFixedOne, + .scale_fraction_mask = kScaleFixedOne - 1, + .weight_shift = 0, + .hor_scale_factor = kScaleFixedOne, + .ver_scale_factor = kScaleFixedOne, + .ver_initial_offset = 0, + .hor_initial_offset = 0, +}; + +void set_downscale_param(DownscaleParams* in_params) { params = *in_params; } + +void set_downscale_factor(Image* input, ImageU8* output) { + params.hor_scale_factor = + (kScaleFixedOne * input->width - params.hor_initial_offset) / + output->width; + params.ver_scale_factor = + (kScaleFixedOne * input->height - params.ver_initial_offset) / + output->height; +} + +// Basic bilinear downscale + +// Resamples image using bilinear interpolation. +// 'output' is modified by this function to store the output image. +void downscale_process(Image* input, ImageU8* output) { + assert(input->num_channels == output->num_channels); + if (!params.enable) { + return; + } + for (uint16_t channel = 0; channel < output->num_channels; ++channel) { + // Each output pixel at (x, y) is sampled at (X, Y) in the input image + // coordinate according to the following formula: + // Y = (y * ver_scale_factor) + ver_initial_offset + // X = (x * hor_scale_factor) + hor_initial_offset + // `accumulated_pos_*` is the location in the inout image calculated by + // repeated addition of *_scale_factor + // `integer_pos_*` is the nearest integer index at top/left side. + uint32_t accumulated_pos_y = params.ver_initial_offset; + for (uint16_t y = 0; y < output->height; ++y) { + uint32_t integer_pos_y = accumulated_pos_y >> params.scale_precision; + uint32_t y0 = Clamp(integer_pos_y, 0, input->height - 1); + uint32_t y1 = Clamp(integer_pos_y + 1, 0, input->height - 1); + // The fractional part of the accumulated position gives us the weight but + // in scale_precision, since we want the weights to be in + // interpolate_precision we shift by their bitwidth difference which is + // weight_shift + uint32_t weight_y = Round(accumulated_pos_y & params.scale_fraction_mask, + params.weight_shift); + uint32_t accumulated_pos_x = params.hor_initial_offset; + + for (uint16_t x = 0; x < output->width; ++x) { + uint32_t integer_pos_x = accumulated_pos_x >> params.scale_precision; + uint32_t x0 = Clamp(integer_pos_x, 0, input->width - 1); + uint32_t x1 = Clamp(integer_pos_x + 1, 0, input->width - 1); + uint32_t weight_x = + Round(accumulated_pos_x & params.scale_fraction_mask, + params.weight_shift); + + // Perform vertical interpolation first to get p0 and p1, + // then horizontal interpolation to get output value. + // `interpolate_shift` is for preserving floating point for interpolated + // values to avoid incremented quantization errors. + // (x0, y0) (x1, y0) + // | | + // v v + // p0 -> output <- p1 + // ^ ^ + // | | + // (x0, y1) (x1, y1) + + uint32_t p0 = Lerp( + image_pixel_val(input, channel, y0, x0) << params.interpolate_shift, + image_pixel_val(input, channel, y1, x0) << params.interpolate_shift, + weight_y, params.interpolate_precision); + uint32_t p1 = Lerp( + image_pixel_val(input, channel, y0, x1) << params.interpolate_shift, + image_pixel_val(input, channel, y1, x1) << params.interpolate_shift, + weight_y, params.interpolate_precision); + // To normalize the output we shift right back by interpolate shift + uint32_t tmp_interpolated = + Round(Lerp(p0, p1, weight_x, params.interpolate_precision), + params.interpolate_shift); + + // As this is the final stage and we'd like to have the output image in + // an interleaved RGBRGBRGB format. + // The accessor (*output)(channel, y, x) + // is assuming planar layout of RGB channels - uses stride_c_ which is + // set in the constructor to stride_c_ = width * height. + // Hence image data is accessed and manipulated directly. + // Also should avoid using stride_c, stride_x and stride_y as they are + // defined in the image class with the planar layout assumption. + const uint16_t interleaved_stride_c = 1; + const uint16_t interleaved_stride_x = output->num_channels; + const uint32_t interleaved_stride_y = + output->num_channels * output->width; + + // Shift interpolated result to output bitwidth. Not rounding to avoid + // overflow to 256. + output->data[y * interleaved_stride_y + x * interleaved_stride_x + + channel * interleaved_stride_c] = + (uint8_t)(tmp_interpolated >> + (kRawPipelineBpp - kPipeOutputBpp)); // 16 - 8 + + accumulated_pos_x += params.hor_scale_factor; + } + accumulated_pos_y += params.ver_scale_factor; + } + } +}
diff --git a/samples/risp4ml/isp_stages/downscale.h b/samples/risp4ml/isp_stages/downscale.h new file mode 100644 index 0000000..c28f697 --- /dev/null +++ b/samples/risp4ml/isp_stages/downscale.h
@@ -0,0 +1,50 @@ +#ifndef SAMPLES_RISP4ML_ISP_STAGES_DOWNSCALE_H_ +#define SAMPLES_RISP4ML_ISP_STAGES_DOWNSCALE_H_ + +#include "samples/risp4ml/common/image.h" + +#ifdef __cplusplus +extern "C" { +#endif // __cplusplus + +typedef struct { + bool enable; + // scale_precision is the number of fractional bits used for scale factors and + // initial offsets + uint32_t scale_precision; + // interpolate_precision is the number of fractional bits used for + // interpolation weights + uint32_t interpolate_precision; + // interpolate_shift is the shift for pixel value before interpolation to + // avoid rounding error. + uint32_t interpolate_shift; + uint32_t scale_fixed_one; + uint32_t scale_fraction_mask; + uint32_t weight_shift; + + // hor_scale_factor and ver_scale_factor are the downscaling ratios between + // input size and output size + // Example: Running the scaler on 4x4 image with hor_scale=2 and ver_scale=2 + // will result in 2x2 image + // TODO(b/179302796): convert scaling factors to be floats. + // Use FloatToFixedPoint() to facilitate this. + uint32_t hor_scale_factor; + uint32_t ver_scale_factor; + + // hor_initial_offset and ver_initial_offset are the offset of the first + // output pixel from the first input pixel in each direction respectively + uint32_t ver_initial_offset; + uint32_t hor_initial_offset; +} DownscaleParams; + +void set_downscale_param(DownscaleParams* params); + +void set_downscale_factor(Image* input, ImageU8* output); + +void downscale_process(Image* input, ImageU8* output); + +#ifdef __cplusplus +} // extern "C" +#endif // __cplusplus + +#endif // SAMPLES_RISP4ML_ISP_STAGES_DOWNSCALE_H_
diff --git a/samples/risp4ml/isp_stages/gamma.c b/samples/risp4ml/isp_stages/gamma.c new file mode 100644 index 0000000..f0e9f51 --- /dev/null +++ b/samples/risp4ml/isp_stages/gamma.c
@@ -0,0 +1,87 @@ +#include "samples/risp4ml/common/utils.h" +#include "samples/risp4ml/isp_stages/gamma.h" + +#define kRgbColorChannels 3 + +static const uint16_t kRgbPipelineBpp = 16; +static const uint32_t kRgbPipelineMaxVal = (1 << kRgbPipelineBpp) - 1; + +// Fixed HW Parameters +static const uint8_t kGammaNumberSegments = 4; +static const uint8_t kGammaLogSegmentOffsets[] = {0, 3, 2, 1}; +static const uint8_t kGammaLogSegmentSpacing[] = {8, 9, 10, 11}; +static const uint16_t kGammaLogNumberPoints[] = {5, 4, 4, 4}; +static const uint16_t kGammaSegmentLutOffset[] = {0, 32, 48, 64}; + +static GammaParams gamma_params = { + .enable = true, + .lut = {0, 3255, 5552, 7237, 8618, 9809, 10868, 11828, 12710, + 13531, 14300, 15026, 15713, 16368, 16995, 17596, 18173, 18731, + 19269, 19790, 20295, 20786, 21264, 21728, 22182, 22624, 23056, + 23479, 23892, 24297, 24694, 25083, 25466, 26209, 26928, 27623, + 28298, 28953, 29590, 30211, 30816, 31406, 31983, 32547, 33099, + 33640, 34170, 34689, 35199, 36192, 37151, 38080, 38980, 39855, + 40705, 41534, 42341, 43129, 43899, 44652, 45389, 46111, 46818, + 47512, 48192, 49517, 50798, 52037, 53239, 54407, 55542, 56648, + 57726, 58778, 59806, 60811, 61794, 62757, 63702, 64627, 65535}}; + +void set_gamma_params(GammaParams* params) { gamma_params = *params; } + +void gamma_process(Image* input, Image* output) { + uint16_t height = input->height; + uint16_t width = input->width; + + const pixel_type_t* in_line[kRgbColorChannels]; + pixel_type_t* out_line[kRgbColorChannels]; + + for (uint16_t y = 0; y < height; ++y) { + for (uint16_t c = 0; c < kRgbColorChannels; ++c) { + in_line[c] = image_row(input, c, y); + out_line[c] = image_row(output, c, y); + } + + for (uint16_t x = 0; x < width; ++x) { + for (uint16_t c = 0; c < kRgbColorChannels; ++c) { + if (!gamma_params.enable) { + out_line[c][x] = in_line[c][x]; + } else { + pixel_type_t pixel_val = + (pixel_type_t)Clamp(in_line[c][x], 0, kRgbPipelineMaxVal); + + // Determine segment + int segment_index = + (kGammaNumberSegments - 1) - + ClzMsb(pixel_val, kRgbPipelineBpp, kGammaNumberSegments - 1); + uint16_t segment_left = + segment_index ? 1 << (kRgbPipelineBpp - + kGammaLogSegmentOffsets[segment_index]) + : 0; + + // Bin index + int bin_index = ((pixel_val - segment_left) >> + kGammaLogSegmentSpacing[segment_index]) + + kGammaSegmentLutOffset[segment_index]; + + int offset_within_bin = + (pixel_val - segment_left) & + ((1 << kGammaLogSegmentSpacing[segment_index]) - 1); + + uint16_t l_val = gamma_params.lut[bin_index]; + uint16_t r_val = gamma_params.lut[bin_index + 1]; + + uint16_t bin_size = 1 << kGammaLogSegmentSpacing[segment_index]; + + uint32_t lerp_val = (l_val * (bin_size - offset_within_bin) + + r_val * offset_within_bin + (bin_size >> 1)) >> + kGammaLogSegmentSpacing[segment_index]; + + // Clamping is not requied + // TODO(alexkaplan): The comment above is from gChips source. + // this calc needs to be checked carefully: + // + out_line[c][x] = (pixel_type_t)lerp_val; + } + } + } + } +}
diff --git a/samples/risp4ml/isp_stages/gamma.h b/samples/risp4ml/isp_stages/gamma.h new file mode 100644 index 0000000..ad6d61e --- /dev/null +++ b/samples/risp4ml/isp_stages/gamma.h
@@ -0,0 +1,25 @@ +#ifndef SAMPLES_RISP4ML_ISP_STAGES_GAMMA_H_ +#define SAMPLES_RISP4ML_ISP_STAGES_GAMMA_H_ + +#include "samples/risp4ml/common/image.h" + +#ifdef __cplusplus +extern "C" { +#endif // __cplusplus + +#define kGammaNumberPoints 81 + +typedef struct { + bool enable; + pixel_type_t lut[kGammaNumberPoints]; +} GammaParams; + +void set_gamma_params(GammaParams* params); + +void gamma_process(Image* input, Image* output); + +#ifdef __cplusplus +} // extern "C" +#endif // __cplusplus + +#endif // SAMPLES_RISP4ML_ISP_STAGES_GAMMA_H_
diff --git a/samples/risp4ml/isp_stages/wbg.c b/samples/risp4ml/isp_stages/wbg.c new file mode 100644 index 0000000..3e84fb2 --- /dev/null +++ b/samples/risp4ml/isp_stages/wbg.c
@@ -0,0 +1,116 @@ +#include <assert.h> + +#include "samples/risp4ml/common/utils.h" +#include "samples/risp4ml/isp_stages/wbg.h" + +#define kBayerColorChannels 4 +#define MAX(a, b) (((a) > (b)) ? (a) : (b)) + +static const uint16_t kWbgFractional = kRawPipelineFraction; +static const uint16_t kWbgUnityGain = 1 << kWbgFractional; +static WbgParams wbg_params = { + .enable = true, + .fixed = false, + .gains = {kWbgUnityGain, kWbgUnityGain, kWbgUnityGain, kWbgUnityGain}}; + +void set_wbg_params(WbgParams* params) { wbg_params = *params; } + +static void compute_wbg_gain(Image* input) { + // Calculate the white-balance gain values using the "gray world" algorithm + uint16_t height = input->height; + uint16_t width = input->width; + + pixel_type_t* in_line; + + int64_t sum_of_reds = 0; + uint32_t num_of_reds = 0; + // will use only one of the greens for scaling, since the difference between + // the two green sensor pixels is negligible + int64_t sum_of_greens = 0; + uint32_t num_of_greens = 0; + int64_t sum_of_blues = 0; + uint32_t num_of_blues = 0; + + for (uint16_t y = 0; y < height; ++y) { + in_line = image_row(input, 0, y); + for (uint16_t x = 0; x < width; ++x) { + BayerIndex bayer_index = GetBayerIndex(kBayerType, x, y); + switch (bayer_index) { + case (kR): { + sum_of_reds += in_line[x]; + num_of_reds++; + }; break; + case (kGr): { + sum_of_greens += in_line[x]; + num_of_greens++; + }; break; + case (kGb): { + sum_of_greens += in_line[x]; + num_of_greens++; + }; break; + case (kB): { + sum_of_blues += in_line[x]; + num_of_blues++; + }; break; + default: { + assert(0 && "Unexpected channel index"); + } + } + } + } + + // scale values to green channel + float average_red = 1.0 * sum_of_reds / num_of_reds; + float average_green = 1.0 * sum_of_greens / num_of_greens; + float average_blue = 1.0 * sum_of_blues / num_of_blues; + + float max_average = MAX(MAX(average_red, average_green), average_blue); + + // Convert the float value to fixed point representation, i.e. 0xFF.FF + uint32_t red_wb = FloatToFixedPoint(max_average / average_red, + kRawPipelineInteger, kRawPipelineFraction, + /*bool is_signed*/ false); + uint32_t green_wb = FloatToFixedPoint( + max_average / average_green, kRawPipelineInteger, kRawPipelineFraction, + /*bool is_signed*/ false); + uint32_t blue_wb = FloatToFixedPoint( + max_average / average_blue, kRawPipelineInteger, kRawPipelineFraction, + /*bool is_signed*/ false); + + wbg_params.gains[0] = red_wb; + wbg_params.gains[1] = green_wb; + wbg_params.gains[2] = green_wb; + wbg_params.gains[3] = blue_wb; +} + +void wbg_process(Image* input, Image* output) { + if (!wbg_params.enable) { + *output = *input; + return; + } + + uint16_t height = input->height; + uint16_t width = input->width; + + const pixel_type_t* in_line; + pixel_type_t* out_line; + + if (!wbg_params.fixed) { + compute_wbg_gain(input); + } + + for (uint16_t y = 0; y < height; ++y) { + in_line = image_row(input, 0, y); + out_line = image_row(output, 0, y); + + for (uint16_t x = 0; x < width; ++x) { + BayerIndex bayer_index = GetBayerIndex(kBayerType, x, y); + uint32_t input_val = in_line[x]; + uint32_t scaled_pixel = (input_val * wbg_params.gains[bayer_index] + + (1 << (kWbgFractional - 1))) >> + kWbgFractional; + out_line[x] = (pixel_type_t)Clamp(scaled_pixel, kRawPipelineMinVal, + kRawPipelineMaxVal); + } + } +}
diff --git a/samples/risp4ml/isp_stages/wbg.h b/samples/risp4ml/isp_stages/wbg.h new file mode 100644 index 0000000..2af1b84 --- /dev/null +++ b/samples/risp4ml/isp_stages/wbg.h
@@ -0,0 +1,24 @@ +#ifndef SAMPLES_RISP4ML_ISP_STAGES_WBG_H_ +#define SAMPLES_RISP4ML_ISP_STAGES_WBG_H_ + +#include "samples/risp4ml/common/image.h" + +#ifdef __cplusplus +extern "C" { +#endif // __cplusplus + +typedef struct { + bool enable; + bool fixed; + uint32_t gains[kNumBayerPatterns]; +} WbgParams; + +void set_wbg_params(WbgParams* params); + +void wbg_process(Image* input, Image* output); + +#ifdef __cplusplus +} // extern "C" +#endif // __cplusplus + +#endif // SAMPLES_RISP4ML_ISP_STAGES_WBG_H_
diff --git a/samples/risp4ml/pipeline/CMakeLists.txt b/samples/risp4ml/pipeline/CMakeLists.txt new file mode 100644 index 0000000..e6447c6 --- /dev/null +++ b/samples/risp4ml/pipeline/CMakeLists.txt
@@ -0,0 +1,16 @@ +iree_cc_library( + NAME + pipeline + HDRS + "pipeline.h" + SRCS + "pipeline.c" + DEPS + samples::risp4ml::common::image + samples::risp4ml::isp_stages::blc + samples::risp4ml::isp_stages::demosaic + samples::risp4ml::isp_stages::dg + samples::risp4ml::isp_stages::downscale + samples::risp4ml::isp_stages::gamma + samples::risp4ml::isp_stages::wbg +)
diff --git a/samples/risp4ml/pipeline/pipeline.c b/samples/risp4ml/pipeline/pipeline.c new file mode 100644 index 0000000..ec561b2 --- /dev/null +++ b/samples/risp4ml/pipeline/pipeline.c
@@ -0,0 +1,43 @@ +#include "samples/risp4ml/common/utils.h" +#include "samples/risp4ml/isp_stages/blc.h" +#include "samples/risp4ml/isp_stages/demosaic.h" +#include "samples/risp4ml/isp_stages/dg.h" +#include "samples/risp4ml/isp_stages/downscale.h" +#include "samples/risp4ml/isp_stages/gamma.h" +#include "samples/risp4ml/isp_stages/wbg.h" +#include "samples/risp4ml/pipeline/pipeline.h" + +void isp_pipeline(ImageU8 *input, ImageU8 *output) { + Image *input_image = + image_new(input->num_channels, input->height, input->width); + // shift the 8bits wide input to 16bits (the processing pipeline bitwidth) + uint32_t input_dimensions = + input->num_channels * input->height * input->width; + for (uint32_t i = 0; i < input_dimensions; ++i) { + input_image->data[i] = input->data[i] << kRawPipelineFraction; // 8 + } + + Image *image1 = image_new(input->num_channels, input->height, input->width); + blc_process(input_image, image1); + image_delete(input_image); + + Image *image2 = image_new(input->num_channels, input->height, input->width); + dg_process(image1, image2); + image_delete(image1); + + Image *image3 = image_new(input->num_channels, input->height, input->width); + wbg_process(image2, image3); + image_delete(image2); + + Image *image4 = image_new(output->num_channels, input->height, input->width); + demosaic_process(image3, image4); + image_delete(image3); + + Image *image5 = image_new(output->num_channels, input->height, input->width); + gamma_process(image4, image5); + image_delete(image4); + + set_downscale_factor(image5, output); + downscale_process(image5, output); + image_delete(image5); +}
diff --git a/samples/risp4ml/pipeline/pipeline.h b/samples/risp4ml/pipeline/pipeline.h new file mode 100644 index 0000000..244a41b --- /dev/null +++ b/samples/risp4ml/pipeline/pipeline.h
@@ -0,0 +1,8 @@ +#ifndef SAMPLES_RISP4ML_PIPELINE_PIPELINE_H_ +#define SAMPLES_RISP4ML_PIPELINE_PIPELINE_H_ + +#include "samples/risp4ml/common/image.h" + +void isp_pipeline(ImageU8* input, ImageU8* output); + +#endif // SAMPLES_RISP4ML_PIPELINE_PIPELINE_H_
diff --git a/samples/risp4ml/test_data/faces_480x640_uint8_numpy_bayer.bin b/samples/risp4ml/test_data/faces_480x640_uint8_numpy_bayer.bin new file mode 100644 index 0000000..3772000 --- /dev/null +++ b/samples/risp4ml/test_data/faces_480x640_uint8_numpy_bayer.bin Binary files differ