[test] Add functional verilator tests. |functional_verilator_test.py| takes a flag pointing to an executable, which is then executed under a Verilator build. The test listens over UART, and passes or fails the test depending on whether it prints PASS! or FAIL!; see the file comment for details.
diff --git a/test/systemtest/conftest.py b/test/systemtest/conftest.py index b2504ba..19a3afa 100644 --- a/test/systemtest/conftest.py +++ b/test/systemtest/conftest.py
@@ -19,6 +19,7 @@ parser.addoption("--rom_bin", action="store", default="") parser.addoption("--verilator_model", action="store", default="") parser.addoption("--openocd", action="store", default="openocd") + parser.addoption("--uart_timeout", action="store", default="60") @pytest.hookimpl(tryfirst=True) @@ -93,3 +94,8 @@ def openocd(pytestconfig): """Return path to OpenOCD executable.""" return pytestconfig.getoption('openocd') + +@pytest.fixture(scope="session") +def uart_timeout(pytestconfig): + """Return the timeout in seconds for UART to print PASS.""" + return int(pytestconfig.getoption('uart_timeout'))
diff --git a/test/systemtest/functional_verilator_test.py b/test/systemtest/functional_verilator_test.py new file mode 100644 index 0000000..baddfe9 --- /dev/null +++ b/test/systemtest/functional_verilator_test.py
@@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +# Copyright lowRISC contributors. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 +r"""Runs a binary on the verilated system, which is expected to write +one of "PASS!\r\n" or "FAIL!\r\n" to UART to determine success or failure. +Failing to write either will result in a timeout. + +This test requires some configuration options. Use the following steps to +run the test manually after building Verilator and the sw/boot_rom and +sw/examples/hello_world targets. + +$ cd ${REPO_TOP} +$ pytest -s -v test/systemtest/functional_verilator_test.py \ + --test_bin sw/tests/hmac/sw.vmem \ + --rom_bin sw/boot_rom/rom.vmem \ + --verilator_model build/lowrisc_systems_top_earlgrey_verilator_0.1/sim-verilator/Vtop_earlgrey_verilator +""" + +import logging +import os +from pathlib import Path +import re +import select +import subprocess +import time + +import pytest + +import test_utils + +logging.basicConfig(level=logging.DEBUG) + + +class TestFunctionalVerilator: + """ + Execute a test binary in a Verilator-simulated hardware build, using UART + output to validate test success or failure. + """ + @pytest.fixture + def sim_top_earlgrey(self, tmp_path, sim_top_build, sw_test_bin, rom_bin): + assert Path(sw_test_bin).is_file() + assert Path(rom_bin).is_file() + assert Path(sim_top_build).is_file() + + cmd_sim = [ + str(Path(sim_top_build).resolve()), '--flashinit', + str(Path(sw_test_bin).resolve()), '--rominit', + str(Path(rom_bin).resolve()) + ] + p_sim = test_utils.Process(cmd_sim, + logdir=str(tmp_path), + cwd=str(tmp_path), + startup_done_expect='Simulation running', + startup_timeout=10) + p_sim.run() + + yield p_sim + + p_sim.terminate() + + def test_execute_binary(self, sim_top_earlgrey, uart_timeout): + """ + Executes the binary and inspects its UART for "PASS!\r\n" or "FAIL!\r\n". + """ + + logger = logging.getLogger(__name__) + + # Verilator will print the string "UART: created /dev/pts/#" to + # indicate which pseudoterminal the UART port is bound to. + uart_match = sim_top_earlgrey.find_in_output( + re.compile('UART: Created (/dev/pts/\\d+)'), 5) + + assert uart_match != None + uart_path = uart_match.group(1) + logger.info("Found UART port at %s." % uart_path) + + # Now, open the UART device and read line by line until we pass or + # fail. + with open(uart_path, 'rb') as uart_device: + uart_fd = uart_device.fileno() + pattern = re.compile('.*?(PASS!\r\n|FAIL!\r\n)') + match = test_utils.stream_fd_to_log(uart_fd, logger, pattern, + uart_timeout) + if match == None: + pytest.fail('Deadline exceeded: did not see PASS! or FAIL! within %ds.', uart_timeout) + + if match.group(1) == 'PASS!\r\n': + logger.debug('Got PASS! from binary.') + else: + pytest.fail('Got FAIL! from binary.')
diff --git a/test/systemtest/test_utils.py b/test/systemtest/test_utils.py index 87f0565..8640bf2 100644 --- a/test/systemtest/test_utils.py +++ b/test/systemtest/test_utils.py
@@ -9,6 +9,7 @@ import logging import os import re +import select import shlex import signal import subprocess @@ -17,7 +18,6 @@ class Process: """Utility class used to spawn an interact with processs.s""" - def __init__(self, cmd, logdir, @@ -80,14 +80,13 @@ self._f_stdout = open(logfile_stdout, 'w') self._f_stderr = open(logfile_stderr, 'w') - self.proc = subprocess.Popen( - cmd, - cwd=self.cwd, - universal_newlines=True, - bufsize=1, - stdin=subprocess.PIPE, - stdout=self._f_stdout, - stderr=self._f_stderr) + self.proc = subprocess.Popen(cmd, + cwd=self.cwd, + universal_newlines=True, + bufsize=1, + stdin=subprocess.PIPE, + stdout=self._f_stdout, + stderr=self._f_stderr) self._f_stdout_r = open(logfile_stdout, 'r') self._f_stderr_r = open(logfile_stderr, 'r') @@ -98,8 +97,8 @@ # check if the string indicating a successful startup appears in the # the program output (STDOUT or STDERR) - init_done = self._find_in_output( - pattern=self.startup_done_expect, timeout=self.startup_timeout) + init_done = self.find_in_output(pattern=self.startup_done_expect, + timeout=self.startup_timeout) if init_done == None: raise subprocess.TimeoutExpired @@ -144,9 +143,9 @@ if pattern == None: return True - return self._find_in_output(pattern, timeout) != None + return self.find_in_output(pattern, timeout) != None - def _find_in_output(self, pattern, timeout): + def find_in_output(self, pattern, timeout): """Read STDOUT and STDERR to find an expected pattern. Both streams are reset to the start of the stream before searching. @@ -169,7 +168,6 @@ if timeout != None: t_end = time.time() + timeout - # reset streams self._f_stdout_r.seek(0) self._f_stderr_r.seek(0) @@ -217,3 +215,51 @@ pass return None + + +def stream_fd_to_log(fd, logger, pattern, timeout=None): + """ + Streams lines from the given fd to log until pattern matches. + + Returns the match object derived from pattern.match(), or None if + the timeout expires. + """ + + deadline = None + if timeout != None: + deadline = time.monotonic() + timeout + + os.set_blocking(fd, False) + line_of_output = b'' + while True: + if deadline != None and time.monotonic() > deadline: + return None + + if line_of_output.endswith(b'\n'): + line_of_output = b'' + + # select() on the fd so that we don't waste time reading when + # we wouldn't get anything out of it. + if deadline != None: + rlist, _, _ = select.select([fd], [], [], + deadline - time.monotonic()) + else: + rlist, _, _ = select.select([fd], [], []) + + if len(rlist) == 0: + continue + + raw_bytes = os.read(fd, 1024) + lines = raw_bytes.splitlines(True) + + for line in lines: + line_of_output += line + if not line_of_output.endswith(b'\n'): + break + + logger.debug('fd#%d: %s' % (fd, line_of_output)) + match = pattern.match(line_of_output.decode('utf-8')) + if match != None: + return match + + line_of_output = b''