Merge #158 158: Create test runner to run integration tests in qemu r=torfmaster a=torfmaster This PR proposes a test runner written in rust. The goal is to have the following improvements over the previous solution using single a command line call: * it should time out transparently and signal that it has timed out * it should note that tests failed * it shouldn't be too fragile (e.g. changing output from the kernel should not break the test) * it should note that test were not executed Currently the code is a PoC and for example the tests in the example crate and in this crate could be connected so that the information which tests exist isn't duplicated (and tests are not forgotten). I'm just gathering hints what a robust test runner could look like so any feedback is welcome! Co-authored-by: torfmaster <briefe@kebes.de>
diff --git a/.travis.yml b/.travis.yml index 7b6bf87..8efe4b7 100644 --- a/.travis.yml +++ b/.travis.yml
@@ -16,10 +16,10 @@ # Once Travis supports a version of Ubuntu Disco or newer we can apt install QEMU for RISC-V # Until then we need to build it ourselves before_install: -# - sudo apt-get -y install qemu-system-misc -# addons: -# apt: -# update: true + # - sudo apt-get -y install qemu-system-misc + # addons: + # apt: + # update: true - wget https://download.qemu.org/qemu-4.2.0.tar.xz - tar xJf qemu-4.2.0.tar.xz - pushd qemu-4.2.0 @@ -42,6 +42,7 @@ script: - make test # Run a QEMU instance of the HiFive1 app - - make flash-hifive1 EXAMPLE=hello_world - - timeout --foreground 10s qemu-system-riscv32 -M sifive_e -kernel ../tock/boards/hifive1/target/riscv32imac-unknown-none-elf/release/hifive1 -device loader,file=./target/riscv32imac-unknown-none-elf/tab/hifive1/hello_world/rv32imac.tbf,addr=0x20430000 -nographic | tee serial - - grep "Hello Tock World" serial + - PLATFORM=hifive1 cargo rrv32imac --example libtock_test --features=alloc --features=__internal_disable_gpio_in_integration_test + - pushd test-runner + - cargo run + - popd
diff --git a/Cargo.toml b/Cargo.toml index 5641d3c..eb56daa 100644 --- a/Cargo.toml +++ b/Cargo.toml
@@ -9,6 +9,7 @@ alloc = ["libtock-core/alloc"] custom_panic_handler = ["libtock-core/custom_panic_handler"] custom_alloc_error_handler = ["libtock-core/custom_alloc_error_handler"] +__internal_disable_gpio_in_integration_test = [] [dependencies] core = { package = "async-support", path = "async-support" } @@ -59,5 +60,6 @@ members = [ "async-support", "codegen", - "core" + "core", + "test-runner" ]
diff --git a/examples-features/libtock_test.rs b/examples-features/libtock_test.rs index 26cccb4..5fe9e59 100644 --- a/examples-features/libtock_test.rs +++ b/examples-features/libtock_test.rs
@@ -2,7 +2,6 @@ // Requires P0.03 and P0.04 to be connected (on a nRF52 DK). #![no_std] - extern crate alloc; use alloc::string::String; @@ -36,6 +35,10 @@ } } +#[cfg_attr( + feature = "__internal_disable_gpio_in_integration_test", + allow(unused_variables) +)] async fn libtock_test( test: &mut LibtockTest, timer: &mut DriverContext, @@ -48,6 +51,7 @@ test.heap()?; test.drivers_only_instantiable_once()?; test.callbacks(timer).await?; + #[cfg(not(feature = "__internal_disable_gpio_in_integration_test"))] test.gpio(gpio)?; Ok(()) } @@ -113,7 +117,7 @@ let mut with_callback = timer_context.with_callback(|_, _| callback_hit = true); let mut timer = with_callback.init()?; - timer.set_alarm(Duration::from_ms(50))?; + timer.set_alarm(Duration::from_ms(1000))?; AlternatingFuture { yielded: false }.await; @@ -122,6 +126,10 @@ self.check_if_true(callback_hit, "Callbacks") } + #[cfg_attr( + feature = "__internal_disable_gpio_in_integration_test", + allow(dead_code) + )] fn gpio(&mut self, gpio: &mut GpioDriverFactory) -> TockResult<()> { let mut gpio_driver = gpio.init_driver().ok().unwrap();
diff --git a/test-runner/Cargo.toml b/test-runner/Cargo.toml new file mode 100644 index 0000000..b9769b9 --- /dev/null +++ b/test-runner/Cargo.toml
@@ -0,0 +1,19 @@ +[package] +name = "test-runner" +version = "0.1.0" +authors = ["torfmaster <briefe@kebes.de>"] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +structopt = { version = "0.3", default-features = false } +futures = "0.3.4" + +[dependencies.async-std] +version = "1.5.0" +features = ["attributes"] + +[dependencies.tokio] +version = "0.2.12" +features = ["process", "rt-threaded", "macros", "io-util", "time"]
diff --git a/test-runner/src/main.rs b/test-runner/src/main.rs new file mode 100644 index 0000000..0ee3455 --- /dev/null +++ b/test-runner/src/main.rs
@@ -0,0 +1,129 @@ +use std::fmt; +use std::process::Stdio; +use std::time::Duration; +use tokio::io::AsyncBufReadExt; +use tokio::io::BufReader; +use tokio::process::Command; +use tokio::time::timeout; + +#[tokio::main] +async fn main() -> Result<(), Box<dyn std::error::Error>> { + timeout(Duration::from_secs(10), perform_tests()).await? +} + +async fn perform_tests() -> Result<(), Box<dyn std::error::Error>> { + let mut failed_tests = Vec::new(); + + let tests = Command::new("qemu-system-riscv32") + .arg("-M") + .arg("sifive_e") + .arg("-kernel") + .arg("../../tock/boards/hifive1/target/riscv32imac-unknown-none-elf/release/hifive1") + .arg("-device") + .arg("loader,file=./../target/riscv32imac-unknown-none-elf/tab/hifive1/libtock_test/rv32imac.tbf,addr=0x20430000") + .arg("-nographic") + .stdout(Stdio::piped()) + .kill_on_drop(true) + .spawn()?; + + let stdout = tests.stdout.unwrap(); + + let stdout_reader = BufReader::new(stdout); + let mut stdout_lines = stdout_reader.lines(); + + while let Some(line) = stdout_lines.next_line().await? { + println!("UART: {}", line); + let test_result = test_succeeded(line, &mut failed_tests); + if let Some(true) = test_result { + return Ok(()); + } + if let Some(false) = test_result { + return Err(Box::new(TestError::TestFailure(failed_tests))); + } + } + Err(Box::new(TestError::QemuExit)) +} + +fn test_succeeded(input: String, failed_tests: &mut Vec<String>) -> Option<bool> { + let success = input.find("[ OK ]").is_some(); + let failure = input.find("[ FAILURE ]").is_some(); + let input = input.replace("[ OK ]", ""); + let input = input.replace("[ FAILURE ]", ""); + let input = input.trim(); + if input == "Test suite finished with state SUCCESS" && success { + return Some(true); + } else if input == "Test suite finished with state FAILURE" && !success { + return Some(false); + } else if failure { + failed_tests.push(input.to_string()); + } + None +} + +#[derive(Debug)] +enum TestError { + TestFailure(Vec<String>), + QemuExit, +} + +impl fmt::Display for TestError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + TestError::TestFailure(failures) => { + writeln!(f, "A test failure occured. Failed Tests")?; + for test in failures { + writeln!(f, "Test failed\"{}\"", test)?; + } + Ok(()) + } + TestError::QemuExit => write!(f, "Qemu exited unexpectedly."), + } + } +} + +impl std::error::Error for TestError {} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + pub fn detects_success_of_test_suite() { + let mut test_results = Vec::new(); + assert_eq!( + test_succeeded( + "[ OK ] Test suite finished with state SUCCESS".into(), + &mut test_results + ), + Some(true) + ); + } + + #[test] + pub fn detects_failure_of_test_suite() { + let mut test_results = Vec::new(); + assert_eq!( + test_succeeded( + "[ FAILURE ] Test suite finished with state FAILURE".into(), + &mut test_results + ), + Some(false) + ); + } + + #[test] + pub fn detects_test_failures() { + let mut test_results = Vec::new(); + test_succeeded("[ FAILURE ] Another test".into(), &mut test_results); + assert_eq!(test_results, vec!["Another test"]); + } + + #[test] + pub fn ignores_other_tests() { + let mut test_results = Vec::new(); + assert_eq!( + test_succeeded("[ SUCCESS ] Another test".into(), &mut test_results), + None + ); + } +}