Simplifiy running tests

.travis.yml

diff --git a/.travis.yml b/.travis.yml
index 162f863..8efe4b7 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -42,7 +42,7 @@
 script:
   - make test
   # Run a QEMU instance of the HiFive1 app
-  - PLATFORM=hifive1 cargo rrv32imac --example libtock_test --features=alloc
+  - PLATFORM=hifive1 cargo rrv32imac --example libtock_test --features=alloc --features=__internal_disable_gpio_in_integration_test
   - pushd test-runner
   - cargo run
   - popd

Cargo.toml

diff --git a/Cargo.toml b/Cargo.toml
index c1b9714..eb56daa 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -9,6 +9,7 @@
 alloc = ["libtock-core/alloc"]
 custom_panic_handler = ["libtock-core/custom_panic_handler"]
 custom_alloc_error_handler = ["libtock-core/custom_alloc_error_handler"]
+__internal_disable_gpio_in_integration_test = []
 
 [dependencies]
 core = { package = "async-support", path = "async-support" }

examples-features/libtock_test.rs

diff --git a/examples-features/libtock_test.rs b/examples-features/libtock_test.rs
index 26cccb4..5fe9e59 100644
--- a/examples-features/libtock_test.rs
+++ b/examples-features/libtock_test.rs
@@ -2,7 +2,6 @@
 // Requires P0.03 and P0.04 to be connected (on a nRF52 DK).
 
 #![no_std]
-
 extern crate alloc;
 
 use alloc::string::String;
@@ -36,6 +35,10 @@
     }
 }
 
+#[cfg_attr(
+    feature = "__internal_disable_gpio_in_integration_test",
+    allow(unused_variables)
+)]
 async fn libtock_test(
     test: &mut LibtockTest,
     timer: &mut DriverContext,
@@ -48,6 +51,7 @@
     test.heap()?;
     test.drivers_only_instantiable_once()?;
     test.callbacks(timer).await?;
+    #[cfg(not(feature = "__internal_disable_gpio_in_integration_test"))]
     test.gpio(gpio)?;
     Ok(())
 }
@@ -113,7 +117,7 @@
         let mut with_callback = timer_context.with_callback(|_, _| callback_hit = true);
         let mut timer = with_callback.init()?;
 
-        timer.set_alarm(Duration::from_ms(50))?;
+        timer.set_alarm(Duration::from_ms(1000))?;
 
         AlternatingFuture { yielded: false }.await;
 
@@ -122,6 +126,10 @@
         self.check_if_true(callback_hit, "Callbacks")
     }
 
+    #[cfg_attr(
+        feature = "__internal_disable_gpio_in_integration_test",
+        allow(dead_code)
+    )]
     fn gpio(&mut self, gpio: &mut GpioDriverFactory) -> TockResult<()> {
         let mut gpio_driver = gpio.init_driver().ok().unwrap();
 

test-runner/src/main.rs

diff --git a/test-runner/src/main.rs b/test-runner/src/main.rs
index e8c7dcb..0ee3455 100644
--- a/test-runner/src/main.rs
+++ b/test-runner/src/main.rs
@@ -1,25 +1,19 @@
-use std::collections::HashMap;
+use std::fmt;
 use std::process::Stdio;
 use std::time::Duration;
-use structopt::StructOpt;
 use tokio::io::AsyncBufReadExt;
 use tokio::io::BufReader;
 use tokio::process::Command;
 use tokio::time::timeout;
 
-#[derive(Debug, StructOpt)]
-#[structopt(name = "libtock test runner", about = "run libtock tests")]
-struct Arguments {
-    #[structopt(short, long)]
-    timeout_ms: usize,
-}
-
 #[tokio::main]
 async fn main() -> Result<(), Box<dyn std::error::Error>> {
     timeout(Duration::from_secs(10), perform_tests()).await?
 }
 
 async fn perform_tests() -> Result<(), Box<dyn std::error::Error>> {
+    let mut failed_tests = Vec::new();
+
     let  tests = Command::new("qemu-system-riscv32")
         .arg("-M")
         .arg("sifive_e")
@@ -36,70 +30,100 @@
 
     let stdout_reader = BufReader::new(stdout);
     let mut stdout_lines = stdout_reader.lines();
-    let mut result_map = HashMap::<TockTest, bool>::new();
 
     while let Some(line) = stdout_lines.next_line().await? {
         println!("UART: {}", line);
-        if let Some(entry) = parse_line(line) {
-            result_map.insert(entry.0, entry.1);
+        let test_result = test_succeeded(line, &mut failed_tests);
+        if let Some(true) = test_result {
+            return Ok(());
         }
-        if result_map.keys().len() >= NUMBER_OF_TESTS {
-            if result_map.values().all(|x| *x) {
-                return Ok(());
-            } else {
-                panic!();
-            }
+        if let Some(false) = test_result {
+            return Err(Box::new(TestError::TestFailure(failed_tests)));
         }
     }
-    panic!("qemu exited unexpectedly")
+    Err(Box::new(TestError::QemuExit))
 }
 
-fn parse_line(input: String) -> Option<(TockTest, bool)> {
-    let result = input.find("[      OK ]").is_some();
+fn test_succeeded(input: String, failed_tests: &mut Vec<String>) -> Option<bool> {
+    let success = input.find("[      OK ]").is_some();
+    let failure = input.find("[ FAILURE ]").is_some();
     let input = input.replace("[      OK ]", "");
     let input = input.replace("[ FAILURE ]", "");
     let input = input.trim();
-    let tock_test = label_to_tocktest(input.to_string());
-    tock_test.map(|test| (test, result))
+    if input == "Test suite finished with state SUCCESS" && success {
+        return Some(true);
+    } else if input == "Test suite finished with state FAILURE" && !success {
+        return Some(false);
+    } else if failure {
+        failed_tests.push(input.to_string());
+    }
+    None
 }
 
-const NUMBER_OF_TESTS: usize = 6;
-#[derive(PartialEq, Debug, Hash, Eq)]
-enum TockTest {
-    Console,
-    StaticMut,
-    DynamicDispatch,
-    Formatting,
-    Heap,
-    DriversOnlyInstantiableOnce,
+#[derive(Debug)]
+enum TestError {
+    TestFailure(Vec<String>),
+    QemuExit,
 }
 
-fn label_to_tocktest(label: String) -> Option<TockTest> {
-    match label.as_str() {
-        "Console" => Some(TockTest::Console),
-        "static mut" => Some(TockTest::StaticMut),
-        "Dynamic dispatch" => Some(TockTest::DynamicDispatch),
-        "Formatting" => Some(TockTest::Formatting),
-        "Heap" => Some(TockTest::Heap),
-        "Drivers only instantiable once" => Some(TockTest::DriversOnlyInstantiableOnce),
-        _ => None,
+impl fmt::Display for TestError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            TestError::TestFailure(failures) => {
+                writeln!(f, "A test failure occured. Failed Tests")?;
+                for test in failures {
+                    writeln!(f, "Test failed\"{}\"", test)?;
+                }
+                Ok(())
+            }
+            TestError::QemuExit => write!(f, "Qemu exited unexpectedly."),
+        }
     }
 }
 
+impl std::error::Error for TestError {}
+
 #[cfg(test)]
 mod test {
     use super::*;
 
     #[test]
-    pub fn works() {
+    pub fn detects_success_of_test_suite() {
+        let mut test_results = Vec::new();
         assert_eq!(
-            parse_line("[      OK ] Console".into()),
-            Some((TockTest::Console, true))
+            test_succeeded(
+                "[      OK ] Test suite finished with state SUCCESS".into(),
+                &mut test_results
+            ),
+            Some(true)
         );
     }
 
     #[test]
-    pub fn fails() {
-        assert_eq!(parse_line("[      OK ]  No Console".into()), None);
+    pub fn detects_failure_of_test_suite() {
+        let mut test_results = Vec::new();
+        assert_eq!(
+            test_succeeded(
+                "[ FAILURE ] Test suite finished with state FAILURE".into(),
+                &mut test_results
+            ),
+            Some(false)
+        );
+    }
+
+    #[test]
+    pub fn detects_test_failures() {
+        let mut test_results = Vec::new();
+        test_succeeded("[ FAILURE ]  Another test".into(), &mut test_results);
+        assert_eq!(test_results, vec!["Another test"]);
+    }
+
+    #[test]
+    pub fn ignores_other_tests() {
+        let mut test_results = Vec::new();
+        assert_eq!(
+            test_succeeded("[ SUCCESS ]  Another test".into(), &mut test_results),
+            None
+        );
     }
 }