[test] Add OpenOCD compliance test
Based on @imphil's CI system tests infrastructure. This commit adds a
test that runs the OpenOCD compliance test against a Verilator target.
The test takes in external parameters to make it easy to integrate with
CI.
Sample run (takes almost 2m to complete):
```console
$ pytest -s -v test/systemtest/openocd_verilator_test.py \
--test_bin=sw/build/sw.vmem \
--rom_bin=sw/build/rom.vmem \
--verilator_model=build/lowrisc_systems_top_earlgrey_verilator_0.1/sim-verilator/Vtop_earlgrey_verilator
```
Integration with CI is blocked until issue #413 is resolved.
diff --git a/test/systemtest/.gitignore b/test/systemtest/.gitignore
new file mode 100644
index 0000000..16d3c4d
--- /dev/null
+++ b/test/systemtest/.gitignore
@@ -0,0 +1 @@
+.cache
diff --git a/test/systemtest/conftest.py b/test/systemtest/conftest.py
new file mode 100644
index 0000000..b2504ba
--- /dev/null
+++ b/test/systemtest/conftest.py
@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+# Copyright lowRISC contributors.
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+
+import logging
+import os
+import shutil
+import subprocess
+from pathlib import Path
+
+import pytest
+import yaml
+
+
+def pytest_addoption(parser):
+ """Test harness configuration options."""
+ parser.addoption("--test_bin", action="store", default="")
+ parser.addoption("--rom_bin", action="store", default="")
+ parser.addoption("--verilator_model", action="store", default="")
+ parser.addoption("--openocd", action="store", default="openocd")
+
+
+@pytest.hookimpl(tryfirst=True)
+def pytest_exception_interact(node, call, report):
+ """Dump all log files in case of a test failure."""
+ try:
+ if not report.failed:
+ return
+ if not 'tmpdir' in node.funcargs:
+ return
+ except:
+ return
+
+ tmpdir = str(node.funcargs['tmpdir'])
+ print("\n\n")
+ print("================= DUMP OF ALL TEMPORARY FILES =================")
+
+ for f in os.listdir(tmpdir):
+ f_abs = os.path.join(tmpdir, f)
+ if not os.path.isfile(f_abs):
+ continue
+ print("vvvvvvvvvvvvvvvvvvvv {} vvvvvvvvvvvvvvvvvvvv".format(f))
+ with open(f_abs, 'r') as fp:
+ print(fp.read())
+ print("^^^^^^^^^^^^^^^^^^^^ {} ^^^^^^^^^^^^^^^^^^^^\n\n".format(f))
+
+
+@pytest.fixture(scope="session")
+def localconf(request):
+ """Host-local configuration."""
+ if os.getenv('OPENTITAN_TEST_LOCALCONF') and os.path.isfile(
+ os.environ['OPENTITAN_TEST_LOCALCONF']):
+ localconf_yaml_file = os.environ['OPENTITAN_TEST_LOCALCONF']
+ else:
+ XDG_CONFIG_HOME = os.getenv(
+ 'XDG_CONFIG_HOME', os.path.join(os.environ['HOME'], '.config'))
+ localconf_yaml_file = os.path.join(XDG_CONFIG_HOME, 'opentitan',
+ 'test-localconf.yaml')
+ logging.getLogger(__name__).info('Reading configuration from ' +
+ localconf_yaml_file)
+
+ with open(str(localconf_yaml_file), 'r') as fp:
+ return yaml.load(fp)
+
+
+@pytest.fixture(scope="session")
+def topsrcdir(request):
+ """Return the top-level source directory as Path object."""
+ # TODO: Consider making this configurable using a pytest arg.
+ return str(Path(os.path.dirname(__file__)) / '..' / '..')
+
+
+@pytest.fixture(scope="session")
+def sw_test_bin(pytestconfig):
+ """Return path to software test binary."""
+ return pytestconfig.getoption('test_bin')
+
+
+@pytest.fixture(scope="session")
+def rom_bin(pytestconfig):
+ """Return path to boot_rom binary."""
+ return pytestconfig.getoption('rom_bin')
+
+
+@pytest.fixture(scope="session")
+def sim_top_build(pytestconfig):
+ """Return path to Verilator sim model."""
+ return pytestconfig.getoption('verilator_model')
+
+
+@pytest.fixture(scope="session")
+def openocd(pytestconfig):
+ """Return path to OpenOCD executable."""
+ return pytestconfig.getoption('openocd')
diff --git a/test/systemtest/openocd_verilator_test.py b/test/systemtest/openocd_verilator_test.py
new file mode 100644
index 0000000..e912484
--- /dev/null
+++ b/test/systemtest/openocd_verilator_test.py
@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+# Copyright lowRISC contributors.
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+r"""Runs OpenOCD compliance test against Verilator target.
+
+This test requires some configuration options. Use the following steps to
+run the test manually after building Verilator and the sw/boot_rom and
+sw/examples/hello_world targets.
+
+$ cd ${REPO_TOP}
+$ pytest -s -v test/systemtest/openocd_verilator_test.py \
+ --test_bin sw/examples/hello_world/sw.vmem \
+ --rom_bin sw/boot_rom/rom.vmem \
+ --verilator_model build/lowrisc_systems_top_earlgrey_verilator_0.1/sim-verilator/Vtop_earlgrey_verilator
+
+In some cases the pytest environment may not be able to find the openocd binary.
+To work around this issue, run the test with an additional configuration option:
+
+ --openocd /tools/openocd/bin/openocd
+"""
+
+import logging
+import os
+from pathlib import Path
+import re
+import subprocess
+import time
+
+import pytest
+
+import test_utils
+
+logging.basicConfig(level=logging.DEBUG)
+
+
+class TestCoreVerilator:
+ """Test core functionality in a Verilator-simulated hardware build."""
+
+ @pytest.fixture
+ def sim_top_earlgrey(self, tmp_path, sim_top_build, sw_test_bin, rom_bin):
+ assert Path(sw_test_bin).is_file()
+ assert Path(rom_bin).is_file()
+ assert Path(sim_top_build).is_file()
+
+ cmd_sim = [
+ str(Path(sim_top_build).resolve()),
+ '--meminit', str(Path(sw_test_bin).resolve()),
+ '--rominit', str(Path(rom_bin).resolve())
+ ]
+ p_sim = test_utils.Process(
+ cmd_sim,
+ logdir=str(tmp_path),
+ cwd=str(tmp_path),
+ startup_done_expect='Simulation running',
+ startup_timeout=10)
+ p_sim.run()
+
+ yield p_sim
+
+ p_sim.terminate()
+
+ def test_openocd_riscv_compliancetest(self, tmp_path, sim_top_earlgrey,
+ topsrcdir, openocd):
+ """Run RISC-V Debug compliance test built into OpenOCD."""
+ assert subprocess.call([openocd, '--version']) == 0
+ cmd_openocd = [
+ openocd,
+ '-s', str(Path(topsrcdir) / 'util' / 'openocd'),
+ '-f', 'board/lowrisc-earlgrey-verilator.cfg',
+ '-c', 'init; riscv test_compliance; shutdown'
+ ]
+ p_openocd = test_utils.Process(
+ cmd_openocd, logdir=str(tmp_path), cwd=str(tmp_path))
+ p_openocd.run()
+
+ logging.getLogger(__name__).info(
+ "OpenOCD should terminate itself; give it up to 5 minutes")
+ p_openocd.proc.wait(timeout=300)
+ assert p_openocd.proc.returncode == 0
+
+ logging.getLogger(__name__).info(
+ "Now wait 60 seconds until the program has finished execution")
+ time.sleep(60)
+
+ try:
+ p_openocd.terminate()
+ except ProcessLookupError:
+ # process is already dead
+ pass
diff --git a/test/systemtest/test_utils.py b/test/systemtest/test_utils.py
new file mode 100644
index 0000000..87f0565
--- /dev/null
+++ b/test/systemtest/test_utils.py
@@ -0,0 +1,219 @@
+#!/usr/bin/env python3
+# Copyright lowRISC contributors.
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+
+import difflib
+import filecmp
+import inspect
+import logging
+import os
+import re
+import shlex
+import signal
+import subprocess
+import time
+
+
+class Process:
+ """Utility class used to spawn an interact with processs.s"""
+
+ def __init__(self,
+ cmd,
+ logdir,
+ cwd=None,
+ startup_done_expect=None,
+ startup_timeout=None):
+ """Creates Process instance.
+
+ Args:
+ cmd: Command line argument used to spawn process.
+ logdir: Directory used to store STDOUT and STDERR output.
+ cwd: Working directory used to spawn process.
+ startup_done_expect: Pattern used to check at process startup time.
+ startup_timeout: Timeout in seconds for |startup_done_expect| check.
+ """
+ if isinstance(cmd, str):
+ self.cmd = shlex.split(cmd)
+ else:
+ self.cmd = cmd
+ self.logdir = logdir
+ self.cwd = cwd
+ self.startup_done_expect = startup_done_expect
+ self.startup_timeout = startup_timeout
+ self.proc = None
+ self.logger = logging.getLogger(__name__)
+
+ self._f_stdout = None
+ self._f_stderr = None
+ self._f_stdout_r = None
+ self._f_stderr_r = None
+
+ def __del__(self):
+ try:
+ self.proc.kill()
+ self._f_stdout.close()
+ self._f_stderr.close()
+ self._f_stdout_r.close()
+ self._f_stderr_r.close()
+ except:
+ pass
+
+ def run(self):
+ """Start process with command line configured in constructor."""
+ cmd_name = os.path.basename(self.cmd[0])
+
+ # Enforce line-buffered STDOUT even when sending STDOUT/STDERR to file.
+ # If applications don't fflush() STDOUT manually, STDOUT goes through
+ # a 4kB buffer before we see any output, which prevents searching for
+ # the string indicating a successful startup.
+ # see discussion at http://www.pixelbeat.org/programming/stdio_buffering/
+ cmd = ['stdbuf', '-oL'] + self.cmd
+ self.logger.info("Running command " + ' '.join(cmd))
+
+ logfile_stdout = os.path.join(self.logdir,
+ "{}.stdout".format(cmd_name))
+ logfile_stderr = os.path.join(self.logdir,
+ "{}.stderr".format(cmd_name))
+ self.logger.debug("Capturing STDOUT at " + logfile_stdout)
+ self.logger.debug("Capturing STDERR at " + logfile_stderr)
+
+ self._f_stdout = open(logfile_stdout, 'w')
+ self._f_stderr = open(logfile_stderr, 'w')
+ self.proc = subprocess.Popen(
+ cmd,
+ cwd=self.cwd,
+ universal_newlines=True,
+ bufsize=1,
+ stdin=subprocess.PIPE,
+ stdout=self._f_stdout,
+ stderr=self._f_stderr)
+
+ self._f_stdout_r = open(logfile_stdout, 'r')
+ self._f_stderr_r = open(logfile_stderr, 'r')
+
+ # no startup match pattern given => startup done!
+ if self.startup_done_expect == None:
+ return True
+
+ # check if the string indicating a successful startup appears in the
+ # the program output (STDOUT or STDERR)
+ init_done = self._find_in_output(
+ pattern=self.startup_done_expect, timeout=self.startup_timeout)
+
+ if init_done == None:
+ raise subprocess.TimeoutExpired
+
+ self.logger.info("Startup sequence matched, startup done.")
+
+ return True
+
+ def terminate(self):
+ """Terminates process started by run call."""
+ if not self.proc:
+ return
+ self.proc.terminate()
+
+ def send_ctrl_c(self):
+ """Sends SIGINT to process started by run call."""
+ if not self.proc:
+ return
+ self.proc.send_signal(signal.SIGINT)
+
+ def expect(self, stdin_data=None, pattern=None, timeout=None):
+ """Write send to STDIN and check if the output is as expected.
+
+ Args:
+ stdin_data: Data to send to STDIN.
+ pattern: Pattern to search for after sending |stdin_data|.
+ timeout: Timeout in seconds for |pattern| check.
+ Returns:
+ True if |pattern| found, False otherwise.
+ """
+ # We don't get STDOUT/STDERR from subprocess.communicate() as it's
+ # redirected to file. We need to read the files instead.
+
+ # XXX: races (false positives) can happen here if output is generated
+ # before the input is sent to the process.
+ if pattern == None:
+ self._f_stdout_r.seek(0, 2)
+
+ self.proc.stdin.write(stdin_data)
+ self.proc.stdin.flush()
+
+ if pattern == None:
+ return True
+
+ return self._find_in_output(pattern, timeout) != None
+
+ def _find_in_output(self, pattern, timeout):
+ """Read STDOUT and STDERR to find an expected pattern.
+
+ Both streams are reset to the start of the stream before searching.
+ pattern can be of two types:
+ 1. A regular expression (a re object). In this case, the pattern is
+ matched against all lines and the result of re.match(pattern) is
+ returned. Multi-line matches are not supported.
+ 2. A string. In this case pattern is compared to each line with
+ startswith(), and the full matching line is returned on a match.
+ If no match is found None is returned.
+ timeout is given in seconds.
+
+ Args:
+ pattern: Pattern to search for in STDOUT or STDERR.
+ timeout: Timeout in seconds for |pattern| check.
+ Returns:
+ String containing |pattern|, None otherwise.
+ """
+
+ if timeout != None:
+ t_end = time.time() + timeout
+
+
+ # reset streams
+ self._f_stdout_r.seek(0)
+ self._f_stderr_r.seek(0)
+
+ while True:
+ # check STDOUT as long as there is one
+ i = 0
+ for line in self._f_stdout_r:
+ i += 1
+ if hasattr(pattern, "match"):
+ m = pattern.match(line)
+ if m:
+ return m
+ else:
+ if line.startswith(pattern):
+ return line
+
+ # Check if we exceed the timeout while reading from STDOUT
+ # do so only every 100 lines to reduce the performance impact.
+ if timeout != None and i % 100 == 99 and time.time() >= t_end:
+ break
+
+ # check STDERR as long as there is one
+ i = 0
+ for line in self._f_stderr_r:
+ i += 1
+ if hasattr(pattern, "match"):
+ m = pattern.match(line)
+ if m:
+ return m
+ else:
+ if line.startswith(pattern):
+ return line
+
+ # Check if we exceed the timeout while reading from STDERR
+ # do so only every 100 lines to reduce the performance impact.
+ if timeout != None and i % 100 == 99 and time.time() >= t_end:
+ break
+
+ # wait for 200ms for new output
+ if timeout != None:
+ try:
+ self.proc.wait(timeout=0.2)
+ except subprocess.TimeoutExpired:
+ pass
+
+ return None
