Add support for audio pre-processing (MFCC extraction)
This is the complete code change for adding audio pre-processing block (MFCC feature extraction).
The code was re-written in C based on http://google3/audio/dsp/mfcc/mfcc_mel.py, with necessary simplifications.
We also integrated audio pre-processing into daredevil, to allow for end-to-end simulations with .wav audio input.
Numerical correctness has been validated: extracted MFCC matches with
the reference (golden_whistle_spectrogram) and returned output is
correct (Whistling). Unit tests passed.
Profiling result: https://docs.google.com/document/d/1vAgRgZeVyOwIniiIp8RrmEvg3CB_p0PFgB2X9NLN4Fo
The pre-processing block adds a relatively constant number of extra instructions. So the percentage depends on the computational complexity of audio models (for daredevil: 65%)
Change-Id: I662914e68170e5e28384960ad345c3366f9fb627
diff --git a/build_tools/gen_mlmodel_input.py b/build_tools/gen_mlmodel_input.py
index cc9a352..213d508 100755
--- a/build_tools/gen_mlmodel_input.py
+++ b/build_tools/gen_mlmodel_input.py
@@ -21,6 +21,7 @@
import numpy as np
from PIL import Image
+from scipy.io import wavfile
parser = argparse.ArgumentParser(
@@ -38,10 +39,12 @@
args = parser.parse_args()
-def write_binary_file(file_path, input, is_quant):
+def write_binary_file(file_path, input, is_quant, is_audio):
with open(file_path, "wb+") as file:
for d in input:
- if is_quant:
+ if is_audio:
+ file.write(struct.pack("<h", d))
+ elif is_quant:
file.write(struct.pack("<B", d))
else:
file.write(struct.pack("<f", d))
@@ -53,10 +56,15 @@
if len(input_shape) < 3:
raise ValueError("Input shape < 3 dimensions")
input_ext = os.path.splitext(input_name)[1]
+ is_audio = False
if (not input_ext) or (input_ext == '.bin'):
with open(input_name, mode='rb') as f:
input = np.fromfile(f, dtype=np.uint8 if is_quant else np.float32)
input = input[:np.prod(input_shape)].reshape(np.prod(input_shape))
+ elif (input_ext == '.wav'):
+ is_audio = True
+ _, input = wavfile.read(input_name)
+ input = input[:np.prod(input_shape)].reshape(np.prod(input_shape))
else:
resized_img = Image.open(input_name).resize(
(input_shape[1], input_shape[2]))
@@ -65,7 +73,7 @@
low = np.min(float_input_range)
high = np.max(float_input_range)
input = (high - low) * input / 255.0 + low
- write_binary_file(output_file, input, is_quant)
+ write_binary_file(output_file, input, is_quant, is_audio)
if __name__ == '__main__':
diff --git a/samples/audio_prep/CMakeLists.txt b/samples/audio_prep/CMakeLists.txt
new file mode 100644
index 0000000..1991ad3
--- /dev/null
+++ b/samples/audio_prep/CMakeLists.txt
@@ -0,0 +1,33 @@
+iree_cc_library(
+ NAME
+ util
+ HDRS
+ "util.h"
+ SRCS
+ "util.c"
+ DEPS
+ "m"
+)
+
+iree_cc_library(
+ NAME
+ mfcc
+ HDRS
+ "mfcc.h"
+ SRCS
+ "mfcc.c"
+ DEPS
+ ::util
+)
+
+iree_cc_binary(
+ NAME
+ mfcc_test
+ SRCS
+ "mfcc_test.cc"
+ DEPS
+ ::mfcc
+ pw_unit_test
+ pw_unit_test.main
+ pw_assert_basic
+)
diff --git a/samples/audio_prep/mfcc.c b/samples/audio_prep/mfcc.c
new file mode 100644
index 0000000..bc01d8b
--- /dev/null
+++ b/samples/audio_prep/mfcc.c
@@ -0,0 +1,193 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Audio preprocessing: MLCC feature extraction
+
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "samples/audio_prep/mfcc.h"
+#include "samples/audio_prep/util.h"
+
+// config struct
+typedef struct {
+ MfccParams params;
+ int win_len;
+ int hop_len;
+ int fft_order;
+ int fft_len;
+ int num_spectra_bins;
+} MfccConfig;
+
+static MfccConfig config = {.params.num_frames = 96,
+ .params.num_mel_bins = 64,
+ .params.audio_samp_rate = 16000,
+ .params.low_edge_hz = 125,
+ .params.upper_edge_hz = 7500,
+ .params.win_len_sec = 0.025,
+ .params.hop_len_sec = 0.010,
+ .params.log_floor = 0.01,
+ .params.log_scaler = 20,
+ .win_len = 400,
+ .hop_len = 160,
+ .fft_order = 9,
+ .fft_len = 512,
+ .num_spectra_bins = 257};
+
+// set mfcc parameters
+void set_mfcc_params(MfccParams* in_params) {
+ config.params = *in_params;
+ config.win_len =
+ (int)(config.params.audio_samp_rate * config.params.win_len_sec + 0.5);
+ config.hop_len =
+ (int)(config.params.audio_samp_rate * config.params.hop_len_sec + 0.5);
+ config.fft_order = ceilf(log2f(config.win_len));
+ config.fft_len = 1 << config.fft_order; // 512
+ config.num_spectra_bins = config.fft_len / 2 + 1;
+}
+
+// Convert frequencies to mel scale using HTK formula
+static float hz_to_mel(float freq_hz) {
+ const float kMelBreakFreqHz = 700.0;
+ const float kMelHighFreqQ = 1127.0;
+ return kMelHighFreqQ * logf(1.0 + (freq_hz / kMelBreakFreqHz));
+}
+
+// Compute Hanning window coefficients
+static void hanning(float* window) {
+ for (int j = 0; j < config.win_len; j++) {
+ window[j] = 0.5 - 0.5 * cosf(2 * M_PI * j / config.win_len);
+ }
+}
+
+// Calculate short-time Fourier transform magnitude for one frame
+// output shape: num_spectra_bins
+static void stft_magnitude(float* in, float* window, float* out) {
+ float* frame = (float*)malloc(config.fft_len * sizeof(float));
+ memset(frame, 0, config.fft_len * sizeof(float));
+ memcpy(frame, in, config.win_len * sizeof(float));
+
+ // apply hanning window
+ for (int j = 0; j < config.win_len; j++) {
+ frame[j] *= window[j];
+ }
+
+ // real-valued FFT
+ rfft(frame, config.fft_order);
+
+ // compute STFT magnitude
+ out[0] = frame[0] > 0 ? frame[0] : -frame[0];
+ out[config.fft_len / 2] = frame[config.fft_len / 2] > 0
+ ? frame[config.fft_len / 2]
+ : -frame[config.fft_len / 2];
+ for (int j = 1; j < config.fft_len / 2; j++) {
+ out[j] = sqrtf(frame[j] * frame[j] +
+ frame[config.fft_len - j] * frame[config.fft_len - j]);
+ }
+
+ free(frame);
+}
+
+// Return a matrix that can post-multiply spectrogram rows to make mel
+// output shape: params.num_mel_bins * num_spectra_bins
+static void spectra_to_mel_matrix(float* weights) {
+ MfccParams* params = &config.params;
+ float nyquist_hz = params->audio_samp_rate / 2;
+ float* spectra_bins = (float*)malloc(config.num_spectra_bins * sizeof(float));
+ linspace(spectra_bins, 0.0, nyquist_hz, config.num_spectra_bins);
+ for (int i = 0; i < config.num_spectra_bins; i++) {
+ spectra_bins[i] = hz_to_mel(spectra_bins[i]);
+ }
+
+ float* band_edges =
+ (float*)malloc((params->num_mel_bins + 2) * sizeof(float));
+ linspace(band_edges, hz_to_mel(params->low_edge_hz),
+ hz_to_mel(params->upper_edge_hz), params->num_mel_bins + 2);
+
+ float lower = 0.0, center = 0.0, upper = 0.0;
+ float lower_slope = 0.0, upper_slope = 0.0;
+ for (int i = 0; i < params->num_mel_bins; i++) {
+ // spectrogram DC bin
+ weights[i * config.num_spectra_bins] = 0.0;
+
+ lower = band_edges[i];
+ center = band_edges[i + 1];
+ upper = band_edges[i + 2];
+ for (int j = 1; j < config.num_spectra_bins; j++) {
+ lower_slope = (spectra_bins[j] - lower) / (center - lower);
+ upper_slope = (upper - spectra_bins[j]) / (upper - center);
+ float clamp = (lower_slope < upper_slope) ? lower_slope : upper_slope;
+ clamp = (clamp < 0) ? 0 : clamp;
+ weights[i * config.num_spectra_bins + j] = clamp;
+ }
+ }
+
+ free(band_edges);
+ free(spectra_bins);
+}
+
+// Convert waveform to a log magnitude mel-frequency spectrogram
+// input: audio samples (int16) with params.num_frames * hop_len samples
+// zero pre-padding win_len - hop_len samples
+// output shape: params.num_frames * params.num_mel_bins (uint8)
+void extract_mfcc(int16_t* in, uint8_t* out, int in_len) {
+ MfccParams* params = &config.params;
+ // Calculate a "periodic" Hann window
+ float* window = (float*)malloc(config.win_len * sizeof(float));
+ hanning(window);
+
+ // Compute weights
+ float* weights = (float*)malloc(params->num_mel_bins *
+ config.num_spectra_bins * sizeof(float));
+ spectra_to_mel_matrix(weights);
+
+ float* frame = (float*)malloc(config.win_len * sizeof(float));
+ memset(frame, 0, config.win_len * sizeof(float));
+ float* spectra = (float*)malloc(config.num_spectra_bins * sizeof(float));
+
+ for (int i = 0; i < params->num_frames; i++) {
+ // update buffer
+ for (int j = 0; j < config.win_len - config.hop_len; j++) {
+ frame[j] = frame[j + config.hop_len];
+ }
+ // feed in new samples
+ for (int j = 0; j < config.hop_len; j++) {
+ int idx = i * config.hop_len + j;
+ frame[config.win_len - config.hop_len + j] =
+ idx < in_len ? (float)in[idx] : 0.0;
+ }
+
+ // compute STFT magnitude
+ stft_magnitude(frame, window, spectra);
+
+ // compute MFCC
+ for (int j = 0; j < params->num_mel_bins; j++) {
+ float temp = dot_product(spectra, weights + j * config.num_spectra_bins,
+ config.num_spectra_bins);
+ if (temp < params->log_floor) temp = params->log_floor;
+ temp = params->log_scaler * logf(temp);
+ temp = temp < 0.0 ? 0.0 : (temp > 255.0 ? 255.0 : temp);
+ out[i * params->num_mel_bins + j] = (uint8_t)temp;
+ }
+ }
+
+ free(window);
+ free(weights);
+ free(spectra);
+ free(frame);
+}
diff --git a/samples/audio_prep/mfcc.h b/samples/audio_prep/mfcc.h
new file mode 100644
index 0000000..97cbab6
--- /dev/null
+++ b/samples/audio_prep/mfcc.h
@@ -0,0 +1,44 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef SAMPLES_AUDIO_PREP_MFCC_H_
+#define SAMPLES_AUDIO_PREP_MFCC_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif // __cplusplus
+
+typedef struct {
+ int num_frames;
+ int num_mel_bins;
+ float audio_samp_rate;
+ float low_edge_hz;
+ float upper_edge_hz;
+ float win_len_sec;
+ float hop_len_sec;
+ float log_floor;
+ float log_scaler;
+} MfccParams;
+
+void set_mfcc_params(MfccParams* params);
+
+void extract_mfcc(int16_t* in, uint8_t* out, int in_len);
+
+#ifdef __cplusplus
+} // extern "C"
+#endif // __cplusplus
+
+#endif // SAMPLES_AUDIO_PREP_MFCC_H_
diff --git a/samples/audio_prep/mfcc_test.cc b/samples/audio_prep/mfcc_test.cc
new file mode 100644
index 0000000..659e90b
--- /dev/null
+++ b/samples/audio_prep/mfcc_test.cc
@@ -0,0 +1,120 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "pw_unit_test/framework.h"
+#include "samples/audio_prep/mfcc.h"
+
+static MfccParams golden_params = {.num_frames = 3,
+ .num_mel_bins = 10,
+ .audio_samp_rate = 16000,
+ .low_edge_hz = 125,
+ .upper_edge_hz = 7600,
+ .win_len_sec = 0.000625,
+ .hop_len_sec = 0.0003125,
+ .log_floor = 0.01,
+ .log_scaler = 20};
+static int16_t golden_input[] = {7954, 10085, 8733, 10844, 29949,
+ -549, 20833, 30345, 18086, 11375,
+ -27309, 12323, -22891, -23360, 11958};
+static uint8_t golden_output[] = {0, 0, 191, 187, 174, 186, 179, 175, 179, 173,
+ 0, 0, 209, 205, 181, 193, 182, 199, 209, 216,
+ 0, 0, 198, 194, 192, 204, 196, 201, 212, 223};
+
+// Golden value is derived from
+// http://google3/knowledge/cerebra/parrot/port/android/platform/
+// inference/daredevil/audio_prep/spectrogram_processor_test.cc
+TEST(MfccTest, AgreesWithGoldenValues) {
+ // set parameters
+ set_mfcc_params(&golden_params);
+ int out_len = golden_params.num_frames * golden_params.num_mel_bins;
+ uint8_t* out = reinterpret_cast<uint8_t*>(malloc(out_len * sizeof(uint8_t)));
+ // extract MFCC
+ extract_mfcc(golden_input, out, sizeof(golden_input) / sizeof(int16_t));
+
+ for (int i = 0; i < out_len; i++) {
+ ASSERT_EQ(out[i], golden_output[i]);
+ }
+ free(out);
+}
+
+TEST(MfccTest, DcInputSaneResult) {
+ MfccParams params = {.num_frames = 10,
+ .num_mel_bins = 64,
+ .audio_samp_rate = 16000,
+ .low_edge_hz = 125,
+ .upper_edge_hz = 7500,
+ .win_len_sec = 0.032,
+ .hop_len_sec = 0.016,
+ .log_floor = 0.01,
+ .log_scaler = 20};
+
+ // set parameters
+ set_mfcc_params(¶ms);
+
+ int hop_len = static_cast<int>(params.audio_samp_rate * params.hop_len_sec);
+ int in_len = hop_len * params.num_frames;
+ int16_t* in = reinterpret_cast<int16_t*>(
+ malloc(params.num_frames * hop_len * sizeof(int16_t)));
+ int out_len = params.num_frames * params.num_mel_bins;
+ uint8_t* out = reinterpret_cast<uint8_t*>(malloc(out_len * sizeof(uint8_t)));
+
+ // DC Input
+ memset(in, 255, in_len * sizeof(int16_t));
+ // extract MFCC
+ extract_mfcc(in, out, in_len);
+ // ignore the 1st frame due to pre zero-padding
+ // expect zero outputs
+ for (int i = params.num_mel_bins; i < out_len; i++) {
+ ASSERT_EQ(out[i], 0);
+ }
+
+ free(in);
+ free(out);
+}
+
+TEST(MfccTest, NyquistFreqInputSaneResult) {
+ MfccParams params = {.num_frames = 15,
+ .num_mel_bins = 32,
+ .audio_samp_rate = 16000,
+ .low_edge_hz = 125,
+ .upper_edge_hz = 7500,
+ .win_len_sec = 0.016,
+ .hop_len_sec = 0.008,
+ .log_floor = 0.01,
+ .log_scaler = 20};
+
+ // set parameters
+ set_mfcc_params(¶ms);
+
+ int hop_len = static_cast<int>(params.audio_samp_rate * params.hop_len_sec);
+ int in_len = hop_len * params.num_frames;
+ int16_t* in = reinterpret_cast<int16_t*>(
+ malloc(params.num_frames * hop_len * sizeof(int16_t)));
+ int out_len = params.num_frames * params.num_mel_bins;
+ uint8_t* out = reinterpret_cast<uint8_t*>(malloc(out_len * sizeof(uint8_t)));
+
+ // High (Nyquest) frequency Input
+ memset(in, 255, in_len * sizeof(int16_t));
+ for (int i = 1; i < in_len; i += 2) in[i] *= -1;
+ // extract MFCC
+ extract_mfcc(in, out, in_len);
+ // ignore the 1st frame due to pre zero-padding
+ // expect zero outputs
+ for (int i = params.num_mel_bins; i < out_len; i++) {
+ ASSERT_EQ(out[i], 0);
+ }
+
+ free(in);
+ free(out);
+}
diff --git a/samples/audio_prep/mfcc_test.txt b/samples/audio_prep/mfcc_test.txt
new file mode 100644
index 0000000..6b595fd
--- /dev/null
+++ b/samples/audio_prep/mfcc_test.txt
@@ -0,0 +1,3 @@
+// RUN: ${TEST_RUNNER_CMD} ${OUT}/springbok_iree/samples/audio_prep/mfcc_test 2>&1 | tee %t
+// RUN: cat %t | FileCheck %s
+// CHECK: {{PASSED}}
diff --git a/samples/audio_prep/util.c b/samples/audio_prep/util.c
new file mode 100644
index 0000000..0b3ffad
--- /dev/null
+++ b/samples/audio_prep/util.c
@@ -0,0 +1,200 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <math.h>
+
+#include "samples/audio_prep/util.h"
+
+#ifndef M_SQRT2
+#define M_SQRT2 1.41421356237309504880
+#endif
+
+// Evenly linear spaced array
+void linspace(float* x, float start, float end, int n) {
+ float step = (end - start) / (n - 1);
+ for (int i = 0; i < n; i++) {
+ x[i] = start + i * step;
+ }
+}
+
+// Calculate the dot product of two vectors
+float dot_product(float* v, float* u, int n) {
+ float result = 0.0;
+ for (int i = 0; i < n; i++) {
+ result += v[i] * u[i];
+ }
+ return result;
+}
+
+/*---------------------------------------------------------------------------
+ * FUNCTION NAME: rfft
+ *
+ * PURPOSE: Real valued, in-place split-radix FFT
+ *
+ * INPUT:
+ * x Pointer to input and output array
+ * m 2^m = n is the Length of FFT
+ *
+ * OUTPUT Output order
+ * Re(0), Re(1), ..., Re(n/2), Im(N/2-1), ..., Im(1)
+ *
+ * RETURN VALUE
+ * none
+ *
+ * DESIGN REFERENCE:
+ * IEEE Transactions on Acoustic, Speech, and Signal Processing,
+ * Vol. ASSP-35. No. 6, June 1987, pp. 849-863.
+ *
+ * Subroutine adapted from fortran routine pp. 858-859.
+ * Note corrected printing errors on page 859:
+ * SS1 = SIN(A3) -> should be SS1 = SIN(A);
+ * CC3 = COS(3) -> should be CC3 = COS(A3)
+ *
+ *---------------------------------------------------------------------------*/
+
+void rfft(float* x, int m) {
+ int n = 1 << m;
+ int j, i, k, is, id;
+ int i0, i1, i2, i3, i4, i5, i6, i7, i8;
+ int n2, n4, n8;
+ float xt, a0, e, a, a3;
+ float t1, t2, t3, t4, t5, t6;
+ float cc1, ss1, cc3, ss3;
+ float* r0;
+
+ /* Digit reverse counter */
+
+ j = 0;
+ r0 = x;
+
+ for (i = 0; i < n - 1; i++) {
+ if (i < j) {
+ xt = x[j];
+ x[j] = *r0;
+ *r0 = xt;
+ }
+ r0++;
+
+ k = n >> 1;
+
+ while (k <= j) {
+ j = j - k;
+ k >>= 1;
+ }
+ j += k;
+ }
+
+ /* Length two butterflies */
+ is = 0;
+ id = 4;
+
+ while (is < n - 1) {
+ for (i0 = is; i0 < n; i0 += id) {
+ i1 = i0 + 1;
+ a0 = x[i0];
+ x[i0] += x[i1];
+ x[i1] = a0 - x[i1];
+ }
+
+ is = (id << 1) - 2;
+ id <<= 2;
+ }
+
+ /* L shaped butterflies */
+ n2 = 2;
+ for (k = 1; k < m; k++) {
+ n2 <<= 1;
+ n4 = n2 >> 2;
+ n8 = n2 >> 3;
+ e = (M_PI * 2) / n2;
+ is = 0;
+ id = n2 << 1;
+ while (is < n) {
+ for (i = is; i <= n - 1; i += id) {
+ i1 = i;
+ i2 = i1 + n4;
+ i3 = i2 + n4;
+ i4 = i3 + n4;
+ t1 = x[i4] + x[i3];
+ x[i4] -= x[i3];
+ x[i3] = x[i1] - t1;
+ x[i1] += t1;
+
+ if (n4 != 1) {
+ i1 += n8;
+ i2 += n8;
+ i3 += n8;
+ i4 += n8;
+ t1 = (x[i3] + x[i4]) / M_SQRT2;
+ t2 = (x[i3] - x[i4]) / M_SQRT2;
+ x[i4] = x[i2] - t1;
+ x[i3] = -x[i2] - t1;
+ x[i2] = x[i1] - t2;
+ x[i1] = x[i1] + t2;
+ }
+ }
+ is = (id << 1) - n2;
+ id <<= 2;
+ }
+
+ for (j = 1; j < n8; j++) {
+ a = j * e;
+ a3 = 3 * a;
+ cc1 = cosf(a);
+ ss1 = sinf(a);
+ cc3 = cosf(a3);
+ ss3 = sinf(a3);
+
+ is = 0;
+ id = n2 << 1;
+
+ while (is < n) {
+ for (i = is; i <= n - 1; i += id) {
+ i1 = i + j;
+ i2 = i1 + n4;
+ i3 = i2 + n4;
+ i4 = i3 + n4;
+ i5 = i + n4 - j;
+ i6 = i5 + n4;
+ i7 = i6 + n4;
+ i8 = i7 + n4;
+ t1 = x[i3] * cc1 + x[i7] * ss1;
+ t2 = x[i7] * cc1 - x[i3] * ss1;
+ t3 = x[i4] * cc3 + x[i8] * ss3;
+ t4 = x[i8] * cc3 - x[i4] * ss3;
+ t5 = t1 + t3;
+ t6 = t2 + t4;
+ t3 = t1 - t3;
+ t4 = t2 - t4;
+ t2 = x[i6] + t6;
+ x[i3] = t6 - x[i6];
+ x[i8] = t2;
+ t2 = x[i2] - t3;
+ x[i7] = -x[i2] - t3;
+ x[i4] = t2;
+ t1 = x[i1] + t5;
+ x[i6] = x[i1] - t5;
+ x[i1] = t1;
+ t1 = x[i5] + t4;
+ x[i5] = x[i5] - t4;
+ x[i2] = t1;
+ }
+ is = (id << 1) - n2;
+ id <<= 2;
+ }
+ }
+ }
+}
diff --git a/samples/audio_prep/util.h b/samples/audio_prep/util.h
new file mode 100644
index 0000000..ade1610
--- /dev/null
+++ b/samples/audio_prep/util.h
@@ -0,0 +1,48 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef SAMPLES_AUDIO_PREP_UTIL_H_
+#define SAMPLES_AUDIO_PREP_UTIL_H_
+
+#ifndef M_PI
+#define M_PI 3.14159265358979323846
+#endif
+
+// Evenly linear spaced array
+void linspace(float* x, float start, float end, int n);
+
+// Calculate the dot product of two vectors
+float dot_product(float* v, float* u, int n);
+
+/*---------------------------------------------------------------------------
+ * FUNCTION NAME: rfft
+ *
+ * PURPOSE: Real valued, in-place split-radix FFT
+ *
+ * INPUT:
+ * x Pointer to input and output array
+ * m 2^m = n is the Length of FFT
+ *
+ * OUTPUT Output order
+ * Re(0), Re(1), ..., Re(n/2), Im(N/2-1), ..., Im(1)
+ *
+ * RETURN VALUE
+ * none
+ *
+ *---------------------------------------------------------------------------*/
+void rfft(float* x, int m);
+
+#endif // SAMPLES_AUDIO_PREP_UTIL_H_
diff --git a/samples/quant_model/CMakeLists.txt b/samples/quant_model/CMakeLists.txt
index 6fd6855..56d3b1e 100644
--- a/samples/quant_model/CMakeLists.txt
+++ b/samples/quant_model/CMakeLists.txt
@@ -165,9 +165,9 @@
NAME
daredevil_quant_input
SHAPE
- "1, 96, 64"
+ "1, 15360, 1"
SRC
- "$ENV{ROOTDIR}/ml/ml-models/test_data/golden_whistle_spectrogram"
+ "$ENV{ROOTDIR}/ml/ml-models/test_data/golden_whistle.wav"
QUANT
)
@@ -340,6 +340,7 @@
::daredevil_bytecode_module_static_c
::daredevil_quant_input_c
iree::vm::bytecode_module
+ samples::audio_prep::mfcc
samples::util::util
LINKOPTS
"LINKER:--defsym=__itcm_length__=1M"
@@ -355,6 +356,7 @@
::daredevil_c_module_static_c
::daredevil_c_module_static_emitc
::daredevil_quant_input_c
+ samples::audio_prep::mfcc
samples::util::util
LINKOPTS
"LINKER:--defsym=__itcm_length__=1M"
diff --git a/samples/quant_model/daredevil.c b/samples/quant_model/daredevil.c
index df78077..5601ebe 100644
--- a/samples/quant_model/daredevil.c
+++ b/samples/quant_model/daredevil.c
@@ -23,6 +23,7 @@
#include "daredevil.h"
#include "iree/base/api.h"
#include "iree/hal/api.h"
+#include "samples/audio_prep/mfcc.h"
#include "samples/util/util.h"
// Compiled module embedded here to avoid file IO:
@@ -73,12 +74,16 @@
iree_status_t load_input_data(const MlModel *model, void **buffer,
iree_const_byte_span_t **byte_span) {
+ iree_status_t result = alloc_input_buffer(model, buffer);
+ int16_t *input = (int16_t *)daredevil_quant_input;
+ uint8_t *output = (uint8_t *)buffer[0];
+ extract_mfcc(input, output, sizeof(daredevil_quant_input) / sizeof(int16_t));
+
byte_span[0] = malloc(sizeof(iree_const_byte_span_t));
*byte_span[0] = iree_make_const_byte_span(
- daredevil_quant_input,
- model->input_size_bytes[0] * model->input_length[0]);
+ buffer[0], model->input_size_bytes[0] * model->input_length[0]);
- return iree_ok_status();
+ return result;
}
iree_status_t process_output(const MlModel *model,
