Add support for using test data in model benchmarks
Supports providing a raw file containing test data. If no test data
is provided, then the input will be memset to zero.
Tested:
soundstream q16 streaming encoder with/without test data
Change-Id: I1323f17d5c2f70389aaad266b78ec0ea3054851b
diff --git a/benchmarks/benchmark_kelvin.cc b/benchmarks/benchmark_kelvin.cc
index 6a84038..6b9e22e 100644
--- a/benchmarks/benchmark_kelvin.cc
+++ b/benchmarks/benchmark_kelvin.cc
@@ -39,6 +39,12 @@
#define MODEL_HEADER STR(MODEL_HEADER_DIRECTORY BENCHMARK_NAME MODEL_HEADER_TYPE)
#include MODEL_HEADER
+#if (TEST_DATA == 1)
+#define TEST_DATA_HEADER_TYPE _test_data.h
+#define TEST_DATA_HEADER STR(MODEL_HEADER_DIRECTORY BENCHMARK_NAME TEST_DATA_HEADER_TYPE)
+#include TEST_DATA_HEADER
+#endif
+
namespace {
constexpr int kTensorArenaSize = 1024 * 1024;
uint8_t g_tensor_arena[kTensorArenaSize] __attribute__((aligned(64)));
@@ -118,9 +124,12 @@
}
TfLiteTensor* input = interpreter->input(0);
- // Set input tensor to zero for first inference, subsequent runs
- // will run on output tensor data (since the memory is shared).
+#if (TEST_DATA == 1)
+ memcpy(tflite::GetTensorData<uint8_t>(input), g_benchmark_test_data, input->bytes);
+#else
memset(tflite::GetTensorData<uint8_t>(input), 0, input->bytes);
+#endif
+
if (interpreter->Invoke() != kTfLiteOk) {
return kInvokeFailed;
}
@@ -130,6 +139,11 @@
// TODO(michaelbrooks): Possibly set/verify test data?
for (int i = 0; i < iterations; ++i) {
+#if (TEST_DATA == 1)
+ memcpy(tflite::GetTensorData<uint8_t>(input), g_benchmark_test_data, input->bytes);
+#else
+ memset(tflite::GetTensorData<uint8_t>(input), 0, input->bytes);
+#endif
interpreter->Invoke();
}
uint64_t end = mcycle_read();
diff --git a/benchmarks/benchmarks.bzl b/benchmarks/benchmarks.bzl
index 318269d..e28ed6a 100644
--- a/benchmarks/benchmarks.bzl
+++ b/benchmarks/benchmarks.bzl
@@ -21,6 +21,7 @@
name,
model,
iterations,
+ test_data = None,
profile = False,
hw_test_size = "medium",
hw_test_tags = [],
@@ -28,20 +29,33 @@
iss_test_tags = [],
**kwargs):
+ kelvin_headers = ["@kelvin_sw//benchmarks:benchmark.h"]
+ model_header_name = "{}_model".format(name)
bin_to_c_file(
- name = "{}_model".format(name),
+ name = model_header_name,
srcs = [model],
var_name = "g_benchmark_model_data",
)
+ kelvin_headers.append(model_header_name)
+
+ if test_data:
+ test_data_header_name = "{}_test_data".format(name)
+ bin_to_c_file(
+ name = test_data_header_name,
+ srcs = [test_data],
+ var_name = "g_benchmark_test_data",
+ )
+ kelvin_headers.append(test_data_header_name)
# Test to run in simulator and MPACT.
kelvin_test(
name = "{}".format(name),
srcs = ["@kelvin_sw//benchmarks:benchmark_kelvin.cc"],
- hdrs = ["@kelvin_sw//benchmarks:benchmark.h", "{}_model.h".format(name)],
+ hdrs = kelvin_headers,
copts = [
"-DITERATIONS={}".format(iterations),
"-DBENCHMARK_NAME={}".format(name),
+ "-DTEST_DATA={}".format(1 if test_data else 0),
"-DPROFILE={}".format(1 if profile else 0),
],
deps = [
@@ -61,6 +75,7 @@
name,
model,
iterations,
+ test_data = None,
profile = False,
**kwargs):
_kelvin_benchmark_device(
@@ -68,6 +83,7 @@
model = model,
device_type = "fpga_nexus",
iterations = iterations,
+ test_data = test_data,
profile = profile,
**kwargs,
)
@@ -76,6 +92,7 @@
name,
model,
iterations,
+ test_data = None,
profile = False,
**kwargs):
@@ -84,6 +101,7 @@
model = model,
device_type = "asic",
iterations = iterations,
+ test_data = test_data,
profile = profile,
**kwargs,
)
@@ -92,6 +110,7 @@
name,
model,
iterations,
+ test_data = None,
profile = False,
**kwargs):
@@ -99,6 +118,7 @@
name = "{}_asic".format(name),
model = model,
iterations = iterations,
+ test_data = test_data,
profile = profile,
**kwargs,
)
@@ -107,6 +127,7 @@
name = "{}_fpga".format(name),
model = model,
iterations = iterations,
+ test_data = test_data,
profile = profile,
**kwargs,
)
@@ -126,15 +147,10 @@
model,
device_type,
iterations,
+ test_data = None,
profile = False,
**kwargs):
- bin_to_c_file(
- name = "{}_model".format(name),
- srcs = [model],
- var_name = "g_benchmark_model_data",
- )
-
# Creation of binaries for running on FPGA
smc_flash_binary(
name = "{}_smc".format(name),
@@ -185,6 +201,24 @@
],
)
+ kelvin_headers = ["@kelvin_sw//benchmarks:benchmark.h"]
+ model_header_name = "{}_model".format(name)
+ bin_to_c_file(
+ name = "{}_model".format(name),
+ srcs = [model],
+ var_name = "g_benchmark_model_data",
+ )
+ kelvin_headers.append(model_header_name)
+
+ if test_data:
+ test_data_header_name = "{}_test_data".format(name)
+ bin_to_c_file(
+ name = test_data_header_name,
+ srcs = [test_data],
+ var_name = "g_benchmark_test_data",
+ )
+ kelvin_headers.append(test_data_header_name)
+
kelvin_binary(
name = "{}_kelvin".format(name),
srcs = [
@@ -193,12 +227,10 @@
copts = [
"-DITERATIONS={}".format(iterations),
"-DBENCHMARK_NAME={}".format(name),
+ "-DTEST_DATA={}".format(1 if test_data else 0),
"-DPROFILE={}".format(1 if profile else 0),
],
- hdrs = [
- "@kelvin_sw//benchmarks:benchmark.h",
- "{}_model.h".format(name),
- ],
+ hdrs = kelvin_headers,
deps = [
"@kelvin_sw//benchmarks:benchmark_header",
"@kelvin_sw//benchmarks:cycle_count",
