blob: 77ac8f5da6160cddf2ebfee791dfb55f07ded6af [file] [log] [blame]
# Copyright lowRISC contributors.
# Licensed under the Apache License, Version 2.0, see LICENSE for details.
# SPDX-License-Identifier: Apache-2.0
load("@//rules:opentitan.bzl", "opentitan_flash_binary", "opentitan_rom_binary")
load("@bazel_skylib//lib:shell.bzl", "shell")
load("@bazel_skylib//lib:collections.bzl", "collections")
VALID_TARGETS = ["dv", "verilator", "cw310_rom", "cw310_test_rom"]
OTTF_SUCCESS_MSG = r"PASS.*\n"
OTTF_FAILURE_MSG = r"(FAIL|FAULT).*\n"
ROM_BOOT_FAILURE_MSG = "BFV:[0-9a-f]{8}"
# These are defined for positive test cases and should be flipped for negative
# test cases, i.e., when a test failure is the expected outcome.
DEFAULT_TEST_SUCCESS_MSG = OTTF_SUCCESS_MSG
DEFAULT_TEST_FAILURE_MSG = "({})|({})".format(
OTTF_FAILURE_MSG,
ROM_BOOT_FAILURE_MSG,
)
# This constant holds a dictionary of slot-specific linker script dependencies
# that determine how an `opentitan_flash_binary` is built.
_FLASH_SLOTS = {
"silicon_creator_a": ["@//sw/device/lib/testing/test_framework:ottf_ld_silicon_creator_slot_a"],
"silicon_creator_b": ["@//sw/device/lib/testing/test_framework:ottf_ld_silicon_creator_slot_b"],
"silicon_creator_virtual": ["@//sw/device/lib/testing/test_framework:ottf_ld_silicon_creator_slot_virtual"],
}
_BASE_PARAMS = {
"args": [], # Passed to test runner as arguments.
"data": [],
"local": True,
"otp": "@//hw/ip/otp_ctrl/data:img_rma",
"rom": "@//sw/device/lib/testing/test_rom",
"tags": [],
"test_runner": "@//util:opentitan_functest_runner.sh",
"test_cmds": [], # Passed to test_runner via TEST_CMDS env var.
"timeout": "moderate", # 5 minutes
"exit_success": DEFAULT_TEST_SUCCESS_MSG,
"exit_failure": DEFAULT_TEST_FAILURE_MSG,
}
def dv_params(
# Base Parameters
args = _BASE_PARAMS["args"] + [
"$(location {dvsim_config})",
],
data = _BASE_PARAMS["data"],
local = _BASE_PARAMS["local"],
otp = _BASE_PARAMS["otp"],
rom = _BASE_PARAMS["rom"],
tags = _BASE_PARAMS["tags"],
timeout = _BASE_PARAMS["timeout"],
test_runner = "@//util:dvsim_test_runner.sh",
test_cmds = [],
# DV-specific Parameters
dvsim_config = "@//hw/top_earlgrey/dv:chip_sim_cfg.hjson",
**kwargs):
"""A macro to create DV sim parameters for OpenTitan functional tests.
This macro emits a dictionary of parameters which are pasted into the DV
simulation specific test rule.
Parameters:
@param args: Extra arguments to pass to the test runner (`dvsim.py`).
@param data: Data dependencies of the test.
@param local: Whether the test should be run locally without sandboxing.
@param otp: The OTP image to use.
@param rom: The ROM image to use.
@param tags: The test tags to apply to the test rule.
@param timeout: The timeout to apply to the test rule.
@param test_cmds: A list of required commands and args that make up the
immutable portion of the harness invocation.
@param dvsim_config: The dvsim.py Hjson config file for the toplevel.
"""
required_test_cmds = [
"-i",
"chip_sw_{name}",
"--",
]
required_data = [
dvsim_config,
"@//util/dvsim",
"@//hw:all_files",
"@//hw:fusesoc_ignore",
]
required_tags = ["dv"]
kwargs.update(
args = args,
data = required_data + data,
local = local,
otp = otp,
rom = rom,
tags = required_tags + tags,
test_runner = test_runner,
test_cmds = required_test_cmds + test_cmds,
timeout = timeout,
dvsim_config = dvsim_config,
)
return kwargs
def verilator_params(
# Base Parameters
args = _BASE_PARAMS["args"],
data = _BASE_PARAMS["data"],
exit_success = _BASE_PARAMS["exit_success"],
exit_failure = _BASE_PARAMS["exit_failure"],
local = _BASE_PARAMS["local"],
otp = _BASE_PARAMS["otp"],
rom = _BASE_PARAMS["rom"],
tags = _BASE_PARAMS["tags"],
timeout = _BASE_PARAMS["timeout"],
test_runner = _BASE_PARAMS["test_runner"],
test_cmds = _BASE_PARAMS["test_cmds"] + [
"console",
"--exit-success={exit_success}",
"--exit-failure={exit_failure}",
],
# Verilator-specific Parameters
# None
**kwargs):
"""A macro to create Verilator sim parameters for OpenTitan functional tests.
This macro emits a dictionary of parameters which are pasted into the
Verilator simulation specific test rule.
Parameters:
@param args: Extra arguments to pass to the test runner (`opentitantool`).
@param data: Data dependencies of the test.
@param local: Whether the test should be run locally without sandboxing.
@param otp: The OTP image to use.
@param rom: The ROM image to use.
@param tags: The test tags to apply to the test rule.
@param timeout: The timeout to apply to the test rule.
@param test_cmds: A list of required commands and args that make up the
immutable portion of the harness invocation.
"""
default_args = [
"--rcfile=",
"--logging={logging}",
]
required_test_cmds = [
"--interface=verilator",
"--verilator-bin=$(location @//hw:verilator)",
"--verilator-rom=$(location {rom})",
"--verilator-flash=$(location {flash})",
"--verilator-otp=$(location {otp})",
]
required_data = [
"@//hw:verilator",
"@//hw:fusesoc_ignore",
]
required_tags = ["verilator", "cpu:5"]
kwargs.update(
args = default_args + args,
data = required_data + data,
exit_success = exit_success,
exit_failure = exit_failure,
local = local,
otp = otp,
rom = rom,
tags = required_tags + tags,
test_runner = test_runner,
test_cmds = required_test_cmds + test_cmds,
timeout = timeout,
)
return kwargs
def cw310_params(
# Base Parameters
args = _BASE_PARAMS["args"],
data = _BASE_PARAMS["data"] + ["{bitstream}"],
exit_success = _BASE_PARAMS["exit_success"],
exit_failure = _BASE_PARAMS["exit_failure"],
local = _BASE_PARAMS["local"],
otp = _BASE_PARAMS["otp"],
tags = _BASE_PARAMS["tags"],
test_runner = _BASE_PARAMS["test_runner"],
test_cmds = [
"--exec=\"fpga load-bitstream --rom-kind={rom_kind} $(location {bitstream})\"",
"--exec=\"bootstrap --clear-uart=true $(location {flash})\"",
"--exec=\"console --exit-success={exit_success} --exit-failure={exit_failure}\"",
"{clear_bitstream}",
],
# CW310-specific Parameters
bitstream = None, # A bitstream value of None will cause the default bitstream values to be used
rom_kind = None,
clear_bitstream = False,
# None
timeout = "short",
interface = "cw310",
**kwargs):
"""A macro to create CW310 parameters for OpenTitan functional tests.
This macro emits a dictionary of parameters which are pasted into the
ChipWhisperer-310 FPGA specific test rule.
Parameters:
@param args: Extra arguments to pass the test runner `opentitantool`.
@param data: Data dependencies of the test.
@param local: Whether the test should be run locally without sandboxing.
@param otp: The OTP image to use.
@param tags: The test tags to apply to the test rule.
@param test_cmds: A list of required commands and args that make up the
immutable portion of the harness invocation.
@param timeout: The timeout to apply to the test rule.
@param bitstream: The bitstream to load into the FPGA (this specifies
the ROM image that is also used, since the ROM is
baked into the bitstream).
@param rom_kind: The ROM type ('testrom' or 'rom') that is baked into the
bitstream that is loaded into the FPGA.
@param clear_bitstream: Clear FPGA bitstream at the end of the test.
"""
default_args = [
"--rcfile=",
"--logging={logging}",
]
required_test_cmds = [
"--interface={}".format(interface),
]
required_tags = [
"cw310",
"exclusive",
]
kwargs.update(
args = default_args + args,
data = data,
exit_success = exit_success,
exit_failure = exit_failure,
local = local,
otp = otp,
tags = required_tags + tags,
test_runner = test_runner,
test_cmds = required_test_cmds + test_cmds,
timeout = timeout,
bitstream = bitstream,
rom_kind = rom_kind,
clear_bitstream = clear_bitstream,
)
return kwargs
def opentitan_functest(
name,
targets = VALID_TARGETS,
args = [],
data = [],
test_in_rom = False,
ot_flash_binary = None,
signed = True,
manifest = "@//sw/device/silicon_creator/rom_ext:manifest_standard",
slot = "silicon_creator_a",
test_harness = "@//sw/host/opentitantool",
key = "fake_prod_key_0",
logging = "info",
dv = None,
verilator = None,
cw310 = None,
**kwargs):
"""A helper macro for generating OpenTitan functional tests.
This macro is mostly a wrapper around `opentitan_flash_binary`, but creates
testing artifacts for each of the hardware targets in `targets`. The testing
artifacts are then given to an `sh_test` rule which dispatches the test to
the corresponding hardware target via `opentitantool`.
Parameters:
@param name: The name of this rule.
@param targets: A list of hardware targets on which to dispatch tests.
@param args: Extra arguments (in addition to those defined in the target-
specific parameter dictionary) to pass to the test harness
(e.g., `opentitantool`).
@param data: Extra data dependencies (in addition to those defined in the
target-specific parameter dictionary) needed while executing
the test.
@param test_in_rom: Whether to run the test from ROM, runs from flash by
default.
@param ot_flash_binary: Use the named `opentitan_flash_binary` as the
flash image for the test rather than building one
from srcs/deps.
@param signed: Whether to sign the test image. Signed by default.
@param manifest: Partially populated manifest to set boot stage/slot configs.
@param slot: What slot to build the image for.
@param test_harness: The binary on the host side that runs the test.
@param key: Which signed test image (by key) to use.
@param dv: DV test parameters.
@param verilator: Verilator test parameters.
@param cw310: CW310 test parameters.
@param **kwargs: Arguments to forward to `opentitan_flash_binary`.
This macro emits the following rules:
if `test_in_rom`:
opentitan_rom_binary named: {name}_rom_prog (and all emitted rules).
opentitan_flash_binary named: {name}_prog (and all emitted rules).
sh_test named: {name}_sim_dv
sh_test named: {name}_sim_verilator
sh_test named: {name}_fpga_cw310
test_suite named: {name}
"""
deps = kwargs.pop("deps", [])
all_tests = []
target_params = {}
# Only build SW for devices we are running tests on.
devices_to_build_for = []
for target in targets:
if target == "dv":
devices_to_build_for.append("sim_dv")
target_params["sim_dv"] = dv_params() if not dv else dv
elif target == "verilator":
devices_to_build_for.append("sim_verilator")
target_params["sim_verilator"] = verilator_params() if not verilator else verilator
elif target in ["cw310_rom", "cw310_test_rom"]:
devices_to_build_for.append("fpga_cw310")
# Copy `cw310` for each `target`. This is not a deep copy, thus we
# also copy tags below.
cw310_ = cw310_params() if not cw310 else dict(cw310.items())
cw310_["tags"] = [t for t in cw310_["tags"]]
# If the cw310 parameter was not provided or was provided without
# the bitstream field, determine the bitstream argument based on
# the target. Otherwise, use the provided bitstream argument.
DEFAULT_BITSTREAM = {
"cw310_test_rom": "@//hw/bitstream:test_rom",
"cw310_rom": "@//hw/bitstream:rom",
}
if (cw310 == None) or (cw310.get("bitstream") == None):
cw310_["bitstream"] = DEFAULT_BITSTREAM[target]
# If the bitstream field was provided, it will already have been copied into cw310_
# Fill in the remaining bitstream arguments
if target == "cw310_test_rom":
cw310_["rom_kind"] = "testrom"
cw310_["tags"].append("cw310_test_rom")
target_params["fpga_cw310_test_rom"] = cw310_
elif target == "cw310_rom":
cw310_["rom_kind"] = "rom"
cw310_["tags"].append("cw310_rom")
target_params["fpga_cw310_rom"] = cw310_
else:
fail("Expected `cw310_test_rom` or `cw310_rom` as the target name")
else:
fail("Invalid target. Target must be in {}".format(VALID_TARGETS))
devices_to_build_for = collections.uniq(devices_to_build_for)
# Handle the special case were the test is run at the ROM stage.
if test_in_rom:
if ot_flash_binary:
fail("Tests that run in ROM stage cannot use pre-built flash binary.")
if any(["cw310" in t for t in targets]):
fail("Tests that run in ROM stage cannot run on FPGA.")
ot_flash_binary = name + "_rom_prog"
opentitan_rom_binary(
name = ot_flash_binary,
deps = deps,
devices = devices_to_build_for,
testonly = True,
**kwargs
)
# Generate SW artifacts for the tests.
if not ot_flash_binary:
# Set the linker script for the specified slot.
if slot not in _FLASH_SLOTS:
fail("Invalid slot: {}. Valid slots are: silicon_creator_{a,b,virtual}".format(slot))
deps += _FLASH_SLOTS[slot]
ot_flash_binary = name + "_prog"
opentitan_flash_binary(
name = ot_flash_binary,
signed = signed,
deps = deps,
devices = devices_to_build_for,
manifest = manifest,
testonly = True,
**kwargs
)
for target, params in target_params.items():
# Set test name.
test_name = "{}_{}".format(name, target)
all_tests.append(test_name)
# The args/data lists will hold all of the test arguments/artifacts that
# need to be present at runtime.
target_args = args
target_data = data + [test_harness]
target_test_cmds = []
########################################################################
# Retrieve host-side test components.
########################################################################
# Environment variables to pass to the `sh_test`.
env = {}
# Set the test runner (i.e., the script that invokes the "test harness").
test_runner = params.pop("test_runner")
# Set the test harness (i.e., the host-side test component, or
# `opentitantool` in most cases).
env["TEST_HARNESS"] = "$(location {})".format(test_harness)
########################################################################
# Retrieve device-side test components.
########################################################################
# Set ROM image.
# Note: FPGA targets will not specify a ROM image as the ROM imaged is
# specified via the `bitstream` parameter (since the ROM is baked into
# the bitstream).
rom = params.pop("rom", None)
if rom:
if test_in_rom:
rom_filegroup = "{}_rom_prog_{}".format(name, target)
rom = "{}_scr_vmem".format(rom_filegroup)
else:
rom_label = Label(rom)
rom_filegroup = "@{}//{}:{}_{}".format(
rom_label.workspace_name,
rom_label.package,
rom_label.name,
target,
)
rom = "{}_scr_vmem".format(rom_filegroup)
target_data.append(rom)
target_data.append(rom_filegroup)
# Set flash image.
if target in ["sim_dv", "sim_verilator"]:
flash = "{}_{}_scr_vmem64".format(ot_flash_binary, target)
elif target in ["fpga_cw310_rom", "fpga_cw310_test_rom"]:
flash = "{}_fpga_cw310_bin".format(ot_flash_binary)
else:
fail("Unexpected target: {}".format(target))
if signed:
flash += "_signed"
# Multislot flash binaries could have different slots / stages
# signed with different keys. Therefore, the key name will not be
# part of the target name for such images.
if key != "multislot":
flash += "_{}".format(key)
# If test is to be run in ROM we load the same image into flash as a
# as a placeholder (since execution will never reach flash). Moreover,
# there is no need to update the data dependencies list as the ROM
# targets were already added above.
if test_in_rom:
flash = rom
else:
target_data.append(flash)
if target in ["fpga_cw310_rom", "fpga_cw310_test_rom"]:
target_data.append("{}_fpga_cw310".format(ot_flash_binary))
else:
target_data.append("{}_{}".format(ot_flash_binary, target))
# Set OTP image.
otp = params.pop("otp")
target_data.append(otp)
########################################################################
# Retrieve hardware-target-specific parameters.
########################################################################
# Set Bitstream (for FPGA targets).
bitstream = params.pop("bitstream", None)
rom_kind = params.pop("rom_kind", None)
if bitstream and not rom_kind:
if "test_rom" in bitstream:
rom_kind = "testrom"
elif "rom" in bitstream:
rom_kind = "rom"
else:
fail("Unknown bitstream type. Expected the bitstream label to contain the string 'test_rom' or 'rom'.")
clear_bitstream = "no-op"
if params.pop("clear_bitstream", False):
clear_bitstream = "fpga clear-bitstream"
# Set success/failure strings for target platforms that print test
# results over the UART (e.g., Verilator and FPGA).
exit_strings_kwargs = {}
if target in ["fpga_cw310_rom", "fpga_cw310_test_rom", "sim_verilator"]:
exit_strings_kwargs = {
"exit_success": shell.quote(params.pop("exit_success")),
"exit_failure": shell.quote(params.pop("exit_failure")),
}
# Set dvsim configuration file (for DV simulation target).
dvsim_config = params.pop("dvsim_config", None)
########################################################################
# Format arguments to send to the test running scripts/tools.
########################################################################
# Concatenate args / data passed into the opentitan_functest macro with
# args / data from device-specific params.
target_args = target_args + params.pop("args")
target_data = target_data + params.pop("data")
target_test_cmds = target_test_cmds + params.pop("test_cmds")
# Fill placeholders in arg/data lists.
format_dict = {
"name": name,
"flash": flash,
"rom": rom,
"otp": otp,
"dvsim_config": dvsim_config,
"clear_bitstream": clear_bitstream,
"bitstream": bitstream,
"rom_kind": rom_kind,
"logging": params.pop("logging", logging),
}
format_dict.update(exit_strings_kwargs)
target_args = [a.format(**format_dict) for a in target_args]
target_data = [d.format(**format_dict) for d in target_data]
target_test_cmds = [s.format(**format_dict) for s in target_test_cmds]
env["TEST_CMDS"] = " ".join(target_test_cmds)
if target in ["fpga_cw310_rom", "fpga_cw310_test_rom"]:
# We attach the UART configuration to the front of the command line
# so that they'll be parsed as global options rather than
# command-specific options.
target_args = select({
"@//ci:lowrisc_fpga_cw310": ["--cw310-uarts=/dev/ttyACM_CW310_1,/dev/ttyACM_CW310_0"],
"//conditions:default": [],
}) + target_args
########################################################################
# Instantiate the test rule.
########################################################################
native.sh_test(
name = test_name,
srcs = [test_runner],
args = target_args,
data = target_data,
env = env,
**params
)
# Guarantee that the test suite will only run tests in `all_tests`. This
# check is necessary because `test_suite()` defaults to all the tests in the
# caller's BUILD file when its `tests` attribute is the empty list.
if all_tests == []:
fail("Zero tests generated for :{}, test_suite() would run the wrong tests.".format(name))
############################################################################
# Instantiate a suite of tests to run the same test on each hardware target.
############################################################################
native.test_suite(
name = name,
tests = all_tests,
# In test_suites, tags perform a filtering function and will select
# matching tags internally instead of allowing filters to select
# test_suites.
# There are exceptions: "small", "medium", "large", "enourmous", "manual"
tags = [
# The manual tag is a special case and is applied to the test suite
# it prevents it from being included in wildcards so that
# --build_tag_filters=-verilator works as expected and excludes
# building verilator tests so the verilator build wont be invoked.
"manual",
# For more see https://bazel.build/reference/be/general#test_suite.tags
],
)
def _manual_test_impl(ctx):
executable = ctx.actions.declare_file("manual_test_wrapper")
ctx.actions.write(
output = executable,
content = "{runner} {testplan}".format(
runner = ctx.executable._runner.short_path,
testplan = ctx.file.testplan.short_path,
),
)
return [
DefaultInfo(
runfiles = ctx.runfiles(
files = [
ctx.executable._runner,
ctx.file.testplan,
],
).merge(ctx.attr._runner[DefaultInfo].default_runfiles),
executable = executable,
),
]
manual_test = rule(
_manual_test_impl,
attrs = {
"testplan": attr.label(
allow_single_file = [".hjson"],
doc = "Testplan with manual testpoints",
mandatory = True,
),
"_runner": attr.label(
default = "//util:run_manual_tests",
executable = True,
cfg = "exec",
),
},
doc = "Walks through the manual testpoints in a testplan",
test = True,
)