sw:vec_iree: Add lit test for the model executions
Use lit infrastructure we can check the expected output of the model, as
well as setting the XFAIL to track the codegen regression.
Run with `lit --path /usr/lib/llvm-11/bin -a sw/vec_iree/samples`
(Add -D RUNNER=qemu to run the qemu tests. Add -D FEATURES=internal to
enable the internal model tests)
See example result in https://bit.googleplex.com/#/hcindyl/6092963308371968
Change-Id: I7b3a94d79d8ca47c5cdb5a4cb115ba54ef795098
diff --git a/samples/float_model/mnist_bytecode_static_test.txt b/samples/float_model/mnist_bytecode_static_test.txt
new file mode 100644
index 0000000..2933e04
--- /dev/null
+++ b/samples/float_model/mnist_bytecode_static_test.txt
@@ -0,0 +1,3 @@
+// RUN: ${TEST_RUNNER_CMD} ${OUT}/springbok_iree/samples/float_model/mnist_bytecode_static 2>&1 | tee %t
+// RUN: cat %t | FileCheck %s
+// CHECK: {{digit: 4}}
diff --git a/samples/float_model/mnist_emitc_static_test.txt b/samples/float_model/mnist_emitc_static_test.txt
new file mode 100644
index 0000000..62f3b04
--- /dev/null
+++ b/samples/float_model/mnist_emitc_static_test.txt
@@ -0,0 +1,3 @@
+// RUN: ${TEST_RUNNER_CMD} ${OUT}/springbok_iree/samples/float_model/mnist_emitc_static 2>&1 | tee %t
+// RUN: cat %t | FileCheck %s
+// CHECK: {{digit: 4}}
diff --git a/samples/float_model/mobilenet_v1_bytecode_static_test.txt b/samples/float_model/mobilenet_v1_bytecode_static_test.txt
new file mode 100644
index 0000000..791e8be
--- /dev/null
+++ b/samples/float_model/mobilenet_v1_bytecode_static_test.txt
@@ -0,0 +1,3 @@
+// RUN: ${TEST_RUNNER_CMD} ${OUT}/springbok_iree/samples/float_model/mobilenet_v1_bytecode_static 2>&1 | tee %t
+// RUN: cat %t | FileCheck %s
+// CHECK: {{Image prediction result is: id: 178}}
diff --git a/samples/float_model/mobilenet_v1_emitc_static_test.txt b/samples/float_model/mobilenet_v1_emitc_static_test.txt
new file mode 100644
index 0000000..101926e
--- /dev/null
+++ b/samples/float_model/mobilenet_v1_emitc_static_test.txt
@@ -0,0 +1,3 @@
+// RUN: ${TEST_RUNNER_CMD} ${OUT}/springbok_iree/samples/float_model/mobilenet_v1_emitc_static 2>&1 | tee %t
+// RUN: cat %t | FileCheck %s
+// CHECK: {{Image prediction result is: id: 178}}
diff --git a/samples/lit.cfg.py b/samples/lit.cfg.py
new file mode 100644
index 0000000..fb3d112
--- /dev/null
+++ b/samples/lit.cfg.py
@@ -0,0 +1,43 @@
+import os
+import sys
+
+import lit.formats
+import lit.llvm
+
+# Configuration file for the 'lit' test runner.
+lit.llvm.initialize(lit_config, config)
+
+config.name = "Model tests"
+config.test_format = lit.formats.ShTest(True)
+
+config.suffixes = [".txt"]
+config.excludes = [
+ "CMakeLists.txt"
+]
+dir_path = os.path.dirname(os.path.realpath(__file__))
+config.environment["ROOTDIR"] = dir_path + "/../../.."
+config.environment["OUT"] = config.environment["ROOTDIR"] + "/out"
+
+qemu_cmd = (
+ "%s/sw/vec/scripts/test_runner.py"
+ " qemu --qemu-path %s/host/qemu/qemu-system-riscv32"
+ % (config.environment["ROOTDIR"], config.environment["OUT"]))
+renode_cmd = (
+ "%s/sw/vec/scripts/test_runner.py"
+ " renode --renode-path %s/host/renode/renode.sh"
+ % (config.environment["ROOTDIR"], config.environment["OUT"]))
+
+config.test_exec_root = config.environment["OUT"] + "/springbok_iree/tests"
+
+# Enable features based on -D FEATURES=internal syntax. FEATURE is used in the
+# REQUIRES field in the lit test. Can add multiple features with comma delimiter.
+features_param = lit_config.params.get("FEATURES")
+if features_param:
+ config.available_features.update(features_param.split(','))
+
+runner = lit_config.params.get("RUNNER")
+
+if runner and runner == "qemu":
+ config.environment["TEST_RUNNER_CMD"] = qemu_cmd
+else:
+ config.environment["TEST_RUNNER_CMD"] = renode_cmd
diff --git a/samples/quant_model/barcode_test.txt b/samples/quant_model/barcode_test.txt
new file mode 100644
index 0000000..362a871
--- /dev/null
+++ b/samples/quant_model/barcode_test.txt
@@ -0,0 +1,2 @@
+// RUN: ${TEST_RUNNER_CMD} ${OUT}/springbok_iree/samples/quant_model/barcode_bytecode_static
+// REQUIRES: internal
diff --git a/samples/quant_model/daredevil_test.txt b/samples/quant_model/daredevil_test.txt
new file mode 100644
index 0000000..b710964
--- /dev/null
+++ b/samples/quant_model/daredevil_test.txt
@@ -0,0 +1,3 @@
+// RUN: ${TEST_RUNNER_CMD} ${OUT}/springbok_iree/samples/quant_model/daredevil_bytecode_static
+// RUN: ${TEST_RUNNER_CMD} ${OUT}/springbok_iree/samples/quant_model/daredevil_emitc_static
+// REQUIRES: internal
diff --git a/samples/quant_model/fssd_test.txt b/samples/quant_model/fssd_test.txt
new file mode 100644
index 0000000..ecad831
--- /dev/null
+++ b/samples/quant_model/fssd_test.txt
@@ -0,0 +1,2 @@
+// RUN: ${TEST_RUNNER_CMD} ${OUT}/springbok_iree/samples/quant_model/fssd_25_8bit_v2_bytecode_static
+// REQUIRES: internal
diff --git a/samples/quant_model/mobilenet_v1_bytecode_static_test.txt b/samples/quant_model/mobilenet_v1_bytecode_static_test.txt
new file mode 100644
index 0000000..7393c88
--- /dev/null
+++ b/samples/quant_model/mobilenet_v1_bytecode_static_test.txt
@@ -0,0 +1,5 @@
+// RUN: ${TEST_RUNNER_CMD} ${OUT}/springbok_iree/samples/quant_model/mobilenet_v1_bytecode_static 2>&1 | tee %t
+// RUN: cat %t | FileCheck %s
+// TODO(hcindyl): Figure out ubuntu20.04 vs debian testing discrepancy.
+// CHECK: {{Image prediction result is: id: 17[5-8]}}
+// XFAIL: *
diff --git a/samples/quant_model/mobilenet_v1_emitc_static_test.txt b/samples/quant_model/mobilenet_v1_emitc_static_test.txt
new file mode 100644
index 0000000..54845c7
--- /dev/null
+++ b/samples/quant_model/mobilenet_v1_emitc_static_test.txt
@@ -0,0 +1,5 @@
+// RUN: ${TEST_RUNNER_CMD} ${OUT}/springbok_iree/samples/quant_model/mobilenet_v1_emitc_static 2>&1 | tee %t
+// RUN: cat %t | FileCheck %s
+// TODO(hcindyl): Figure out ubuntu20.04 vs debian testing discrepancy
+// CHECK: {{Image prediction result is: id: 17[5-8]}}
+// XFAIL: *
diff --git a/samples/quant_model/mobilenet_v2_bytecode_static_test.txt b/samples/quant_model/mobilenet_v2_bytecode_static_test.txt
new file mode 100644
index 0000000..a1179ef
--- /dev/null
+++ b/samples/quant_model/mobilenet_v2_bytecode_static_test.txt
@@ -0,0 +1,4 @@
+// RUN: ${TEST_RUNNER_CMD} ${OUT}/springbok_iree/samples/quant_model/mobilenet_v2_bytecode_static 2>&1 | tee %t
+// RUN: cat %t | FileCheck %s
+// CHECK: {{Image prediction result is: id: 210}}
+// XFAIL: *
diff --git a/samples/quant_model/mobilenet_v2_emitc_static_text.txt b/samples/quant_model/mobilenet_v2_emitc_static_text.txt
new file mode 100644
index 0000000..668458d
--- /dev/null
+++ b/samples/quant_model/mobilenet_v2_emitc_static_text.txt
@@ -0,0 +1,4 @@
+// RUN: ${TEST_RUNNER_CMD} ${OUT}/springbok_iree/samples/quant_model/mobilenet_v2_emitc_static 2>&1 | tee %t
+// RUN: cat %t | FileCheck %s
+// CHECK: {{Image prediction result is: id: 210}}
+// XFAIL: *
diff --git a/samples/quant_model/person_detection_bytecode_static_test.txt b/samples/quant_model/person_detection_bytecode_static_test.txt
new file mode 100644
index 0000000..31e94ef
--- /dev/null
+++ b/samples/quant_model/person_detection_bytecode_static_test.txt
@@ -0,0 +1,4 @@
+// RUN: ${TEST_RUNNER_CMD} ${OUT}/springbok_iree/samples/quant_model/person_detection_bytecode_static 2>&1 | tee %t
+// RUN: cat %t | FileCheck %s
+// CHECK: {{Non-person Score: -113; Person Score: 113}}
+// XFAIL: *
diff --git a/samples/quant_model/person_detection_emitc_static_test.txt b/samples/quant_model/person_detection_emitc_static_test.txt
new file mode 100644
index 0000000..28c8946
--- /dev/null
+++ b/samples/quant_model/person_detection_emitc_static_test.txt
@@ -0,0 +1,4 @@
+// RUN: ${TEST_RUNNER_CMD} ${OUT}/springbok_iree/samples/quant_model/person_detection_emitc_static 2>&1 | tee %t
+// RUN: cat %t | FileCheck %s
+// CHECK: {{Non-person Score: -113; Person Score: 113}}
+// XFAIL: *
diff --git a/samples/quant_model/scenenet_test.txt b/samples/quant_model/scenenet_test.txt
new file mode 100644
index 0000000..1be1d7f
--- /dev/null
+++ b/samples/quant_model/scenenet_test.txt
@@ -0,0 +1,3 @@
+// RUN: ${TEST_RUNNER_CMD} ${OUT}/springbok_iree/samples/quant_model/scenenet_v2_bytecode_static
+// RUN: ${TEST_RUNNER_CMD} ${OUT}/springbok_iree/samples/quant_model/scenenet_v2_emitc_static
+// REQUIRES: internal
diff --git a/samples/quant_model/semantic_lift_test.txt b/samples/quant_model/semantic_lift_test.txt
new file mode 100644
index 0000000..ed9e0bc
--- /dev/null
+++ b/samples/quant_model/semantic_lift_test.txt
@@ -0,0 +1,3 @@
+// RUN: ${TEST_RUNNER_CMD} --timeout 200 ${OUT}/springbok_iree/samples/quant_model/semantic_lift_bytecode_static
+// REQUIRES: internal
+// XFAIL: *
diff --git a/samples/quant_model/voice_commands_test.txt b/samples/quant_model/voice_commands_test.txt
new file mode 100644
index 0000000..2eaa078
--- /dev/null
+++ b/samples/quant_model/voice_commands_test.txt
@@ -0,0 +1,3 @@
+// RUN: ${TEST_RUNNER_CMD} ${OUT}/springbok_iree/samples/quant_model/voice_commands_bytecode_static
+// RUN: ${TEST_RUNNER_CMD} ${OUT}/springbok_iree/samples/quant_model/voice_commands_emitc_static
+// REQUIRES: internal
diff --git a/samples/simple_vec_mul/simple_test.txt b/samples/simple_vec_mul/simple_test.txt
new file mode 100644
index 0000000..132507c
--- /dev/null
+++ b/samples/simple_vec_mul/simple_test.txt
@@ -0,0 +1,4 @@
+// RUN: ${TEST_RUNNER_CMD} ${OUT}/springbok_iree/samples/simple_vec_mul/simple_int_vec_mul_bytecode_static
+// RUN: ${TEST_RUNNER_CMD} ${OUT}/springbok_iree/samples/simple_vec_mul/simple_int_vec_mul_emitc_static
+// RUN: ${TEST_RUNNER_CMD} ${OUT}/springbok_iree/samples/simple_vec_mul/simple_float_vec_mul_bytecode_static
+// RUN: ${TEST_RUNNER_CMD} ${OUT}/springbok_iree/samples/simple_vec_mul/simple_float_vec_mul_emitc_static
