[test] Add functional verilator tests.
|functional_verilator_test.py| takes a flag pointing to an executable,
which is then executed under a Verilator build. The test listens
over UART, and passes or fails the test depending on whether it prints
PASS! or FAIL!; see the file comment for details.
diff --git a/test/systemtest/conftest.py b/test/systemtest/conftest.py
index b2504ba..19a3afa 100644
--- a/test/systemtest/conftest.py
+++ b/test/systemtest/conftest.py
@@ -19,6 +19,7 @@
parser.addoption("--rom_bin", action="store", default="")
parser.addoption("--verilator_model", action="store", default="")
parser.addoption("--openocd", action="store", default="openocd")
+ parser.addoption("--uart_timeout", action="store", default="60")
@pytest.hookimpl(tryfirst=True)
@@ -93,3 +94,8 @@
def openocd(pytestconfig):
"""Return path to OpenOCD executable."""
return pytestconfig.getoption('openocd')
+
+@pytest.fixture(scope="session")
+def uart_timeout(pytestconfig):
+ """Return the timeout in seconds for UART to print PASS."""
+ return int(pytestconfig.getoption('uart_timeout'))
diff --git a/test/systemtest/functional_verilator_test.py b/test/systemtest/functional_verilator_test.py
new file mode 100644
index 0000000..baddfe9
--- /dev/null
+++ b/test/systemtest/functional_verilator_test.py
@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+# Copyright lowRISC contributors.
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+r"""Runs a binary on the verilated system, which is expected to write
+one of "PASS!\r\n" or "FAIL!\r\n" to UART to determine success or failure.
+Failing to write either will result in a timeout.
+
+This test requires some configuration options. Use the following steps to
+run the test manually after building Verilator and the sw/boot_rom and
+sw/examples/hello_world targets.
+
+$ cd ${REPO_TOP}
+$ pytest -s -v test/systemtest/functional_verilator_test.py \
+ --test_bin sw/tests/hmac/sw.vmem \
+ --rom_bin sw/boot_rom/rom.vmem \
+ --verilator_model build/lowrisc_systems_top_earlgrey_verilator_0.1/sim-verilator/Vtop_earlgrey_verilator
+"""
+
+import logging
+import os
+from pathlib import Path
+import re
+import select
+import subprocess
+import time
+
+import pytest
+
+import test_utils
+
+logging.basicConfig(level=logging.DEBUG)
+
+
+class TestFunctionalVerilator:
+ """
+ Execute a test binary in a Verilator-simulated hardware build, using UART
+ output to validate test success or failure.
+ """
+ @pytest.fixture
+ def sim_top_earlgrey(self, tmp_path, sim_top_build, sw_test_bin, rom_bin):
+ assert Path(sw_test_bin).is_file()
+ assert Path(rom_bin).is_file()
+ assert Path(sim_top_build).is_file()
+
+ cmd_sim = [
+ str(Path(sim_top_build).resolve()), '--flashinit',
+ str(Path(sw_test_bin).resolve()), '--rominit',
+ str(Path(rom_bin).resolve())
+ ]
+ p_sim = test_utils.Process(cmd_sim,
+ logdir=str(tmp_path),
+ cwd=str(tmp_path),
+ startup_done_expect='Simulation running',
+ startup_timeout=10)
+ p_sim.run()
+
+ yield p_sim
+
+ p_sim.terminate()
+
+ def test_execute_binary(self, sim_top_earlgrey, uart_timeout):
+ """
+ Executes the binary and inspects its UART for "PASS!\r\n" or "FAIL!\r\n".
+ """
+
+ logger = logging.getLogger(__name__)
+
+ # Verilator will print the string "UART: created /dev/pts/#" to
+ # indicate which pseudoterminal the UART port is bound to.
+ uart_match = sim_top_earlgrey.find_in_output(
+ re.compile('UART: Created (/dev/pts/\\d+)'), 5)
+
+ assert uart_match != None
+ uart_path = uart_match.group(1)
+ logger.info("Found UART port at %s." % uart_path)
+
+ # Now, open the UART device and read line by line until we pass or
+ # fail.
+ with open(uart_path, 'rb') as uart_device:
+ uart_fd = uart_device.fileno()
+ pattern = re.compile('.*?(PASS!\r\n|FAIL!\r\n)')
+ match = test_utils.stream_fd_to_log(uart_fd, logger, pattern,
+ uart_timeout)
+ if match == None:
+ pytest.fail('Deadline exceeded: did not see PASS! or FAIL! within %ds.', uart_timeout)
+
+ if match.group(1) == 'PASS!\r\n':
+ logger.debug('Got PASS! from binary.')
+ else:
+ pytest.fail('Got FAIL! from binary.')
diff --git a/test/systemtest/test_utils.py b/test/systemtest/test_utils.py
index 87f0565..8640bf2 100644
--- a/test/systemtest/test_utils.py
+++ b/test/systemtest/test_utils.py
@@ -9,6 +9,7 @@
import logging
import os
import re
+import select
import shlex
import signal
import subprocess
@@ -17,7 +18,6 @@
class Process:
"""Utility class used to spawn an interact with processs.s"""
-
def __init__(self,
cmd,
logdir,
@@ -80,14 +80,13 @@
self._f_stdout = open(logfile_stdout, 'w')
self._f_stderr = open(logfile_stderr, 'w')
- self.proc = subprocess.Popen(
- cmd,
- cwd=self.cwd,
- universal_newlines=True,
- bufsize=1,
- stdin=subprocess.PIPE,
- stdout=self._f_stdout,
- stderr=self._f_stderr)
+ self.proc = subprocess.Popen(cmd,
+ cwd=self.cwd,
+ universal_newlines=True,
+ bufsize=1,
+ stdin=subprocess.PIPE,
+ stdout=self._f_stdout,
+ stderr=self._f_stderr)
self._f_stdout_r = open(logfile_stdout, 'r')
self._f_stderr_r = open(logfile_stderr, 'r')
@@ -98,8 +97,8 @@
# check if the string indicating a successful startup appears in the
# the program output (STDOUT or STDERR)
- init_done = self._find_in_output(
- pattern=self.startup_done_expect, timeout=self.startup_timeout)
+ init_done = self.find_in_output(pattern=self.startup_done_expect,
+ timeout=self.startup_timeout)
if init_done == None:
raise subprocess.TimeoutExpired
@@ -144,9 +143,9 @@
if pattern == None:
return True
- return self._find_in_output(pattern, timeout) != None
+ return self.find_in_output(pattern, timeout) != None
- def _find_in_output(self, pattern, timeout):
+ def find_in_output(self, pattern, timeout):
"""Read STDOUT and STDERR to find an expected pattern.
Both streams are reset to the start of the stream before searching.
@@ -169,7 +168,6 @@
if timeout != None:
t_end = time.time() + timeout
-
# reset streams
self._f_stdout_r.seek(0)
self._f_stderr_r.seek(0)
@@ -217,3 +215,51 @@
pass
return None
+
+
+def stream_fd_to_log(fd, logger, pattern, timeout=None):
+ """
+ Streams lines from the given fd to log until pattern matches.
+
+ Returns the match object derived from pattern.match(), or None if
+ the timeout expires.
+ """
+
+ deadline = None
+ if timeout != None:
+ deadline = time.monotonic() + timeout
+
+ os.set_blocking(fd, False)
+ line_of_output = b''
+ while True:
+ if deadline != None and time.monotonic() > deadline:
+ return None
+
+ if line_of_output.endswith(b'\n'):
+ line_of_output = b''
+
+ # select() on the fd so that we don't waste time reading when
+ # we wouldn't get anything out of it.
+ if deadline != None:
+ rlist, _, _ = select.select([fd], [], [],
+ deadline - time.monotonic())
+ else:
+ rlist, _, _ = select.select([fd], [], [])
+
+ if len(rlist) == 0:
+ continue
+
+ raw_bytes = os.read(fd, 1024)
+ lines = raw_bytes.splitlines(True)
+
+ for line in lines:
+ line_of_output += line
+ if not line_of_output.endswith(b'\n'):
+ break
+
+ logger.debug('fd#%d: %s' % (fd, line_of_output))
+ match = pattern.match(line_of_output.decode('utf-8'))
+ if match != None:
+ return match
+
+ line_of_output = b''
