sw:vec_iree: update lit test Move the lit test to be done at the build folder. The lit run files are copied to the build folder before building the executable (springbok_test). Update the run file extension to .run Change-Id: I5ea231aa99252e28fa82dbc6eadfa4df311b555b
diff --git a/CMakeLists.txt b/CMakeLists.txt index b8a650a..b54cb7e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt
@@ -84,6 +84,7 @@ include(springbok_c_module) include(springbok_modules) include(iree_model_input) +include(springbok_test) # softmax op (and mfcc) requires floorf implementation in libm. Use the nano # version.
diff --git a/README.md b/README.md index 0816101..557653f 100644 --- a/README.md +++ b/README.md
@@ -53,11 +53,11 @@ ## Test the executables In Shodan, this project utilizes LLVM `lit` and `FileCheck` to test the ML -executable performance. The tests are defined in the *_test.txt files under -`samples`. To run the test, at the Shodan top level directory +executable performance. The tests are defined in the *_test.run files under +`<output dir>`. To run the test, at the Shodan top level directory ```bash -lit --path <Filecheck dir> -a sw/vec_iree/samples +lit --path <Filecheck dir> -a <output dir> ``` Filecheck in Debian testing is under `/usr/lib/llvm-11/bin`
diff --git a/cmake/springbok_test.cmake b/cmake/springbok_test.cmake new file mode 100644 index 0000000..5a42826 --- /dev/null +++ b/cmake/springbok_test.cmake
@@ -0,0 +1,72 @@ +include(CMakeParseArguments) + +# springbok_test() +# +# A wrapper for the iree_cc_binary to attach lit run files to an executable +# Parameters: +# NAME: Name of target. +# SRCS: List of source files for the binary +# DEPS: List of other libraries to be linked in to the binary targets +# COPTS: List of public defines +# LINKOPTS: List of link options +# TESTFILES: List of lit test run files +# +# Examples: +# springbok_test( +# NAME +# simple_int_vec_mul_emitc_static +# SRCS +# "int_vec.c" +# DEPS +# ::simple_int_mul_c_module_static_c +# ::simple_int_mul_c_module_static_emitc +# samples::util::util +# COPTS +# "-DBUILD_EMITC" +# TESTFILES +# "simple_test.run" +# ) +function(springbok_test) + cmake_parse_arguments( + _RULE + "" + "NAME" + "SRCS;COPTS;LINKOPTS;DEPS;TESTFILES" + ${ARGN} + ) + + iree_cc_binary( + NAME + ${_RULE_NAME} + SRCS + ${_RULE_SRCS} + DEPS + ${_RULE_DEPS} + COPTS + ${_RULE_COPTS} + LINKOPTS + ${_RULE_LINKOPTS} + ) + if(NOT _RULE_TESTFILES) + return() + endif() + + iree_package_name(_PACKAGE_NAME) + foreach(_FILE ${_RULE_TESTFILES}) + get_filename_component(_FILE_PATH ${_FILE} REALPATH) + get_filename_component(_FILE_NAME ${_FILE_PATH} NAME) + set(_FILE_TARGET ${_PACKAGE_NAME}_${_FILE_NAME}) + if(NOT TARGET ${_FILE_TARGET}) + add_custom_target( + ${_FILE_TARGET} + COMMAND ${CMAKE_COMMAND} -E copy ${_FILE_PATH} + ${CMAKE_CURRENT_BINARY_DIR}/${_FILE_NAME} + BYPRODUCTS ${CMAKE_CURRENT_BINARY_DIR}/${_FILE_NAME} + COMMENT "Copy lit run file ${_FILE_NAME}" + VERBATIM + ) + endif() + + add_dependencies(${_PACKAGE_NAME}_${_RULE_NAME} ${_FILE_TARGET}) + endforeach() +endfunction()
diff --git a/samples/CMakeLists.txt b/samples/CMakeLists.txt index 0e9c88b..51f0af9 100644 --- a/samples/CMakeLists.txt +++ b/samples/CMakeLists.txt
@@ -1 +1,16 @@ iree_add_all_subdirs() + +# Add lit config file. +# Note the top level lit.cfg.py is also copied here for once in the build. +add_custom_target( + lit.cfg.py + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py + ${CMAKE_BINARY_DIR}/lit.cfg.py + BYPRODUCTS ${CMAKE_BINARY_DIR}/lit.cfg.py + COMMENT "Copy lit config" + VERBATIM +) + +add_dependencies(samples_simple_vec_mul_simple_int_vec_mul_emitc_static + lit.cfg.py +)
diff --git a/samples/audio_prep/CMakeLists.txt b/samples/audio_prep/CMakeLists.txt index 1991ad3..6f3467b 100644 --- a/samples/audio_prep/CMakeLists.txt +++ b/samples/audio_prep/CMakeLists.txt
@@ -20,7 +20,7 @@ ::util ) -iree_cc_binary( +springbok_test( NAME mfcc_test SRCS @@ -30,4 +30,6 @@ pw_unit_test pw_unit_test.main pw_assert_basic + TESTFILES + "mfcc_test.run" )
diff --git a/samples/audio_prep/mfcc_test.run b/samples/audio_prep/mfcc_test.run new file mode 100644 index 0000000..95e774d --- /dev/null +++ b/samples/audio_prep/mfcc_test.run
@@ -0,0 +1,3 @@ +// RUN: ${TEST_RUNNER_CMD} %S/mfcc_test 2>&1 | tee %t +// RUN: cat %t | FileCheck %s +// CHECK: PASSED
diff --git a/samples/audio_prep/mfcc_test.txt b/samples/audio_prep/mfcc_test.txt deleted file mode 100644 index 6b595fd..0000000 --- a/samples/audio_prep/mfcc_test.txt +++ /dev/null
@@ -1,3 +0,0 @@ -// RUN: ${TEST_RUNNER_CMD} ${OUT}/springbok_iree/samples/audio_prep/mfcc_test 2>&1 | tee %t -// RUN: cat %t | FileCheck %s -// CHECK: {{PASSED}}
diff --git a/samples/lit.cfg.py b/samples/lit.cfg.py index ac6b86f..f83b488 100644 --- a/samples/lit.cfg.py +++ b/samples/lit.cfg.py
@@ -19,32 +19,33 @@ import lit.llvm # Configuration file for the 'lit' test runner. +# pylint: disable=undefined-variable lit.llvm.initialize(lit_config, config) config.name = "Model tests" config.test_format = lit.formats.ShTest(True) -config.suffixes = [".txt"] -config.excludes = [ - "CMakeLists.txt" -] -dir_path = os.path.dirname(os.path.realpath(__file__)) -config.environment["ROOTDIR"] = dir_path + "/../../.." +config.suffixes = [".run"] +config.excludes = ["CMakeLists.txt"] + +if os.getenv("ROOTDIR") is None: + sys.exit("Error: Need to run source build/setup.sh first") + +config.environment["ROOTDIR"] = os.getenv("ROOTDIR") config.environment["OUT"] = config.environment["ROOTDIR"] + "/out" -qemu_cmd = ( - "%s/sw/vec/scripts/test_runner.py" - " qemu --qemu-path %s/host/qemu/qemu-system-riscv32" - % (config.environment["ROOTDIR"], config.environment["OUT"])) -renode_cmd = ( - "%s/sw/vec/scripts/test_runner.py" - " renode --renode-path %s/host/renode/renode" - % (config.environment["ROOTDIR"], config.environment["OUT"])) +qemu_cmd = (f"{config.environment['ROOTDIR']}/sw/vec/scripts/test_runner.py " + "qemu --qemu-path " + f"{config.environment['OUT']}/host/qemu/qemu-system-riscv32") +renode_cmd = (f"{config.environment['ROOTDIR']}/sw/vec/scripts/test_runner.py " + "renode --renode-path " + f"{config.environment['OUT']}/host/renode/renode") -config.test_exec_root = config.environment["OUT"] + "/springbok_iree/tests" +config.test_exec_root = os.path.dirname(__file__) + "/tests" # Enable features based on -D FEATURES=internal syntax. FEATURE is used in the -# REQUIRES field in the lit test. Can add multiple features with comma delimiter. +# REQUIRES field in the lit test. Can add multiple features with comma +# delimiter. features_param = lit_config.params.get("FEATURES") if features_param: config.available_features.update(features_param.split(','))
diff --git a/samples/risp4ml/common/CMakeLists.txt b/samples/risp4ml/common/CMakeLists.txt index 793f933..65bb9b1 100644 --- a/samples/risp4ml/common/CMakeLists.txt +++ b/samples/risp4ml/common/CMakeLists.txt
@@ -25,7 +25,7 @@ "utils.c" ) -iree_cc_binary( +springbok_test( NAME image_test SRCS @@ -36,4 +36,6 @@ pw_unit_test pw_unit_test.main pw_assert_basic + TESTFILES + "image_test.run" )
diff --git a/samples/risp4ml/common/image_test.run b/samples/risp4ml/common/image_test.run new file mode 100644 index 0000000..d8bf7c4 --- /dev/null +++ b/samples/risp4ml/common/image_test.run
@@ -0,0 +1,3 @@ +// RUN: ${TEST_RUNNER_CMD} %S/image_test 2>&1 | tee %t +// RUN: cat %t | FileCheck %s +// CHECK: PASSED
diff --git a/samples/risp4ml/common/image_test.txt b/samples/risp4ml/common/image_test.txt deleted file mode 100644 index 49f5c20..0000000 --- a/samples/risp4ml/common/image_test.txt +++ /dev/null
@@ -1,3 +0,0 @@ -// RUN: ${TEST_RUNNER_CMD} ${OUT}/springbok_iree/samples/risp4ml/common/image_test 2>&1 | tee %t -// RUN: cat %t | FileCheck %s -// CHECK: {{PASSED}}
diff --git a/samples/risp4ml/isp_stages/CMakeLists.txt b/samples/risp4ml/isp_stages/CMakeLists.txt index 7d47d93..960fbc9 100644 --- a/samples/risp4ml/isp_stages/CMakeLists.txt +++ b/samples/risp4ml/isp_stages/CMakeLists.txt
@@ -96,7 +96,7 @@ samples::risp4ml::common::utils ) -iree_cc_binary( +springbok_test( NAME blc_test SRCS @@ -107,9 +107,11 @@ pw_unit_test pw_unit_test.main pw_assert_basic + TESTFILES + "blc_test.run" ) -iree_cc_binary( +springbok_test( NAME dg_test SRCS @@ -120,9 +122,11 @@ pw_unit_test pw_unit_test.main pw_assert_basic + TESTFILES + "dg_test.run" ) -iree_cc_binary( +springbok_test( NAME downscale_test SRCS @@ -132,9 +136,11 @@ pw_unit_test pw_unit_test.main pw_assert_basic + TESTFILES + "downscale_test.run" ) -iree_cc_binary( +springbok_test( NAME gamma_test SRCS @@ -144,9 +150,11 @@ pw_unit_test pw_unit_test.main pw_assert_basic + TESTFILES + "gamma_test.run" ) -iree_cc_binary( +springbok_test( NAME wbg_test SRCS @@ -156,4 +164,6 @@ pw_unit_test pw_unit_test.main pw_assert_basic + TESTFILES + "wbg_test.run" )
diff --git a/samples/risp4ml/isp_stages/blc_test.run b/samples/risp4ml/isp_stages/blc_test.run new file mode 100644 index 0000000..830fef0 --- /dev/null +++ b/samples/risp4ml/isp_stages/blc_test.run
@@ -0,0 +1,3 @@ +// RUN: ${TEST_RUNNER_CMD} %S/blc_test 2>&1 | tee %t +// RUN: cat %t | FileCheck %s +// CHECK: PASSED
diff --git a/samples/risp4ml/isp_stages/blc_test.txt b/samples/risp4ml/isp_stages/blc_test.txt deleted file mode 100644 index c60e26a..0000000 --- a/samples/risp4ml/isp_stages/blc_test.txt +++ /dev/null
@@ -1,3 +0,0 @@ -// RUN: ${TEST_RUNNER_CMD} ${OUT}/springbok_iree/samples/risp4ml/isp_stages/blc_test 2>&1 | tee %t -// RUN: cat %t | FileCheck %s -// CHECK: {{PASSED}}
diff --git a/samples/risp4ml/isp_stages/dg_test.run b/samples/risp4ml/isp_stages/dg_test.run new file mode 100644 index 0000000..86724e1 --- /dev/null +++ b/samples/risp4ml/isp_stages/dg_test.run
@@ -0,0 +1,3 @@ +// RUN: ${TEST_RUNNER_CMD} %S/dg_test 2>&1 | tee %t +// RUN: cat %t | FileCheck %s +// CHECK: PASSED
diff --git a/samples/risp4ml/isp_stages/dg_test.txt b/samples/risp4ml/isp_stages/dg_test.txt deleted file mode 100644 index a782df9..0000000 --- a/samples/risp4ml/isp_stages/dg_test.txt +++ /dev/null
@@ -1,3 +0,0 @@ -// RUN: ${TEST_RUNNER_CMD} ${OUT}/springbok_iree/samples/risp4ml/isp_stages/dg_test 2>&1 | tee %t -// RUN: cat %t | FileCheck %s -// CHECK: {{PASSED}}
diff --git a/samples/risp4ml/isp_stages/downscale_test.run b/samples/risp4ml/isp_stages/downscale_test.run new file mode 100644 index 0000000..7686af8 --- /dev/null +++ b/samples/risp4ml/isp_stages/downscale_test.run
@@ -0,0 +1,3 @@ +// RUN: ${TEST_RUNNER_CMD} %S/downscale_test 2>&1 | tee %t +// RUN: cat %t | FileCheck %s +// CHECK: PASSED
diff --git a/samples/risp4ml/isp_stages/downscale_test.txt b/samples/risp4ml/isp_stages/downscale_test.txt deleted file mode 100644 index 2c46f81..0000000 --- a/samples/risp4ml/isp_stages/downscale_test.txt +++ /dev/null
@@ -1,3 +0,0 @@ -// RUN: ${TEST_RUNNER_CMD} ${OUT}/springbok_iree/samples/risp4ml/isp_stages/downscale_test 2>&1 | tee %t -// RUN: cat %t | FileCheck %s -// CHECK: {{PASSED}}
diff --git a/samples/risp4ml/isp_stages/gamma_test.run b/samples/risp4ml/isp_stages/gamma_test.run new file mode 100644 index 0000000..e5951e3 --- /dev/null +++ b/samples/risp4ml/isp_stages/gamma_test.run
@@ -0,0 +1,3 @@ +// RUN: ${TEST_RUNNER_CMD} %S/gamma_test 2>&1 | tee %t +// RUN: cat %t | FileCheck %s +// CHECK: PASSED
diff --git a/samples/risp4ml/isp_stages/gamma_test.txt b/samples/risp4ml/isp_stages/gamma_test.txt deleted file mode 100644 index c5dd21b..0000000 --- a/samples/risp4ml/isp_stages/gamma_test.txt +++ /dev/null
@@ -1,3 +0,0 @@ -// RUN: ${TEST_RUNNER_CMD} ${OUT}/springbok_iree/samples/risp4ml/isp_stages/gamma_test 2>&1 | tee %t -// RUN: cat %t | FileCheck %s -// CHECK: {{PASSED}}
diff --git a/samples/risp4ml/isp_stages/wbg_test.run b/samples/risp4ml/isp_stages/wbg_test.run new file mode 100644 index 0000000..761aefb --- /dev/null +++ b/samples/risp4ml/isp_stages/wbg_test.run
@@ -0,0 +1,3 @@ +// RUN: ${TEST_RUNNER_CMD} %S/wbg_test 2>&1 | tee %t +// RUN: cat %t | FileCheck %s +// CHECK: PASSED
diff --git a/samples/risp4ml/isp_stages/wbg_test.txt b/samples/risp4ml/isp_stages/wbg_test.txt deleted file mode 100644 index a971d84..0000000 --- a/samples/risp4ml/isp_stages/wbg_test.txt +++ /dev/null
@@ -1,3 +0,0 @@ -// RUN: ${TEST_RUNNER_CMD} ${OUT}/springbok_iree/samples/risp4ml/isp_stages/wbg_test 2>&1 | tee %t -// RUN: cat %t | FileCheck %s -// CHECK: {{PASSED}}
diff --git a/samples/simple_vec_mul/CMakeLists.txt b/samples/simple_vec_mul/CMakeLists.txt index 17bbc2f..341b6bd 100644 --- a/samples/simple_vec_mul/CMakeLists.txt +++ b/samples/simple_vec_mul/CMakeLists.txt
@@ -87,7 +87,7 @@ "LINKER:--defsym=__stack_size__=20k" ) -iree_cc_binary( +springbok_test( NAME simple_int_vec_mul_emitc_static SRCS @@ -100,4 +100,6 @@ "LINKER:--defsym=__stack_size__=20k" COPTS "-DBUILD_EMITC" + TESTFILES + "simple_test.run" )
diff --git a/samples/simple_vec_mul/simple_test.run b/samples/simple_vec_mul/simple_test.run new file mode 100644 index 0000000..fbb18bf --- /dev/null +++ b/samples/simple_vec_mul/simple_test.run
@@ -0,0 +1,4 @@ +// RUN: ${TEST_RUNNER_CMD} %S/simple_int_vec_mul_bytecode_static +// RUN: ${TEST_RUNNER_CMD} %S/simple_int_vec_mul_emitc_static +// RUN: ${TEST_RUNNER_CMD} %S/simple_float_vec_mul_bytecode_static +// RUN: ${TEST_RUNNER_CMD} %S/simple_float_vec_mul_emitc_static
diff --git a/samples/simple_vec_mul/simple_test.txt b/samples/simple_vec_mul/simple_test.txt deleted file mode 100644 index 132507c..0000000 --- a/samples/simple_vec_mul/simple_test.txt +++ /dev/null
@@ -1,4 +0,0 @@ -// RUN: ${TEST_RUNNER_CMD} ${OUT}/springbok_iree/samples/simple_vec_mul/simple_int_vec_mul_bytecode_static -// RUN: ${TEST_RUNNER_CMD} ${OUT}/springbok_iree/samples/simple_vec_mul/simple_int_vec_mul_emitc_static -// RUN: ${TEST_RUNNER_CMD} ${OUT}/springbok_iree/samples/simple_vec_mul/simple_float_vec_mul_bytecode_static -// RUN: ${TEST_RUNNER_CMD} ${OUT}/springbok_iree/samples/simple_vec_mul/simple_float_vec_mul_emitc_static