[bazel/ci] schedule and run cw310 tests with bazel
Marked presently broken tests with a broken tag and recorded issue
Prefer broken tag to other tags that steer CI execution
Copy bitstream to a bazel cache
Run bazel which should grab the bitstream from the cache and run all the
tests we expect to pass
extended timeout to 16 minutes to keep it from getting flaky. (There are
currently more tests than before and the cw310 for our CI isn't a
limiting factor
Signed-off-by: Drew Macrae <drewmacrae@google.com>
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 8e50d1d..3d378fd 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -425,7 +425,7 @@
- job: execute_fpga_tests_cw310
displayName: Execute tests on ChipWhisperer CW310 FPGA board
pool: FPGA
- timeoutInMinutes: 8
+ timeoutInMinutes: 16
dependsOn:
- chip_earlgrey_cw310
- sw_build
@@ -438,16 +438,19 @@
- sw_build
- bash: |
set -e
-
- # Install an additional pytest dependency for result upload.
- pip3 install pytest-azurepipelines
-
+ set -x
. util/build_consts.sh
- pytest --version
- pytest test/systemtest/earlgrey/test_fpga_cw310.py \
- --log-cli-level=DEBUG \
- --test-run-title="Run system tests on ChipWhisperer CW310 FPGA board" \
- --napoleon-docstrings
+ SHA=$(git rev-parse HEAD)
+ BITCACHE_DIR=~/.cache/opentitan-bitstreams/cache/${SHA}
+ mkdir -p $BITCACHE_DIR
+ BITCACHE_FILE=$BITCACHE_DIR/lowrisc_systems_chip_earlgrey_cw310_0.1.bit.orig
+ cp $BIN_DIR/hw/top_earlgrey/lowrisc_systems_chip_earlgrey_cw310_0.1.bit ${BITCACHE_FILE}
+ echo "" >> ${BITCACHE_DIR}/lowrisc_systems_chip_earlgrey_cw310_0.1.bit.splice
+ BITSTREAM="${SHA} --offline" ci/bazelisk.sh test \
+ --define DISABLE_VERILATOR_BUILD=true \
+ --test_tag_filters=cw310,-broken \
+ --build_tests_only \
+ //sw/...
displayName: Execute tests
- job: deploy_release_artifacts
diff --git a/sw/device/silicon_creator/lib/drivers/BUILD b/sw/device/silicon_creator/lib/drivers/BUILD
index b6e938a..572d8fc 100644
--- a/sw/device/silicon_creator/lib/drivers/BUILD
+++ b/sw/device/silicon_creator/lib/drivers/BUILD
@@ -2,7 +2,7 @@
# Licensed under the Apache License, Version 2.0, see LICENSE for details.
# SPDX-License-Identifier: Apache-2.0
-load("//rules:opentitan_test.bzl", "opentitan_functest", "verilator_params")
+load("//rules:opentitan_test.bzl", "cw310_params", "opentitan_functest", "verilator_params")
load("//rules:cross_platform.bzl", "dual_cc_device_library_of", "dual_cc_library", "dual_inputs")
package(default_visibility = ["//visibility:public"])
@@ -593,6 +593,10 @@
opentitan_functest(
name = "watchdog_functest",
srcs = ["watchdog_functest.c"],
+ cw310 = cw310_params(
+ # FIXME #12486 [bazel] targets in sw/device/tests failing on cw310 and verilator when built by bazel
+ tags = ["broken"],
+ ),
verilator = verilator_params(
timeout = "long",
),
diff --git a/sw/device/silicon_creator/mask_rom/BUILD b/sw/device/silicon_creator/mask_rom/BUILD
index 2fbd85e..8455c88 100644
--- a/sw/device/silicon_creator/mask_rom/BUILD
+++ b/sw/device/silicon_creator/mask_rom/BUILD
@@ -306,8 +306,8 @@
verilator = verilator_params(
rom = ":mask_rom_sim_verilator_scr_vmem",
tags = [
+ "broken",
"cpu:4",
- "failing_verilator",
],
),
deps = [
diff --git a/sw/device/tests/BUILD b/sw/device/tests/BUILD
index faff26a..8798a08 100644
--- a/sw/device/tests/BUILD
+++ b/sw/device/tests/BUILD
@@ -3,15 +3,19 @@
# SPDX-License-Identifier: Apache-2.0
load("//rules:opentitan.bzl", "OPENTITAN_CPU")
-load("//rules:opentitan_test.bzl", "opentitan_functest", "verilator_params")
+load("//rules:opentitan_test.bzl", "cw310_params", "opentitan_functest", "verilator_params")
opentitan_functest(
name = "aes_entropy_test",
srcs = ["aes_entropy_test.c"],
+ cw310 = cw310_params(
+ # FIXME #12486 [bazel] targets in sw/device/tests failing on cw310 and verilator when built by bazel
+ tags = ["broken"],
+ ),
verilator = verilator_params(
tags = [
+ "broken",
"cpu:4",
- "failing_verilator",
],
),
deps = [
@@ -29,10 +33,14 @@
opentitan_functest(
name = "aes_idle_test",
srcs = ["aes_idle_test.c"],
+ cw310 = cw310_params(
+ # FIXME #12486 [bazel] targets in sw/device/tests failing on cw310 and verilator when built by bazel
+ tags = ["broken"],
+ ),
verilator = verilator_params(
tags = [
+ "broken",
"cpu:4",
- "failing_verilator",
],
),
deps = [
@@ -91,6 +99,10 @@
opentitan_functest(
name = "aon_timer_irq_test",
srcs = ["aon_timer_irq_test.c"],
+ cw310 = cw310_params(
+ # FIXME #12486 [bazel] targets in sw/device/tests failing on cw310 and verilator when built by bazel
+ tags = ["broken"],
+ ),
deps = [
"//hw/top_earlgrey/sw/autogen:top_earlgrey",
"//sw/device/lib:irq",
@@ -123,10 +135,14 @@
opentitan_functest(
name = "aon_timer_wdog_bite_reset_test",
srcs = ["aon_timer_wdog_bite_reset_test.c"],
+ cw310 = cw310_params(
+ # FIXME #12486 [bazel] targets in sw/device/tests failing on cw310 and verilator when built by bazel
+ tags = ["broken"],
+ ),
verilator = verilator_params(
tags = [
+ "broken",
"cpu:4",
- "failing_verilator",
],
),
deps = [
@@ -146,6 +162,10 @@
opentitan_functest(
name = "aon_timer_wdog_lc_escalate_test",
srcs = ["aon_timer_wdog_lc_escalate_test.c"],
+ cw310 = cw310_params(
+ # FIXME #12486 [bazel] targets in sw/device/tests failing on cw310 and verilator when built by bazel
+ tags = ["broken"],
+ ),
deps = [
"//hw/top_earlgrey/sw/autogen:top_earlgrey",
"//sw/device/lib:irq",
@@ -186,10 +206,14 @@
opentitan_functest(
name = "clkmgr_external_clk_src_for_sw_fast_test",
srcs = ["clkmgr_external_clk_src_for_sw_fast_test.c"],
+ cw310 = cw310_params(
+ # FIXME #12486 [bazel] targets in sw/device/tests failing on cw310 and verilator when built by bazel
+ tags = ["broken"],
+ ),
verilator = verilator_params(
tags = [
+ "broken",
"cpu:4",
- "failing_verilator",
],
),
deps = [
@@ -201,10 +225,14 @@
opentitan_functest(
name = "clkmgr_external_clk_src_for_sw_slow_test",
srcs = ["clkmgr_external_clk_src_for_sw_slow_test.c"],
+ cw310 = cw310_params(
+ # FIXME #12486 [bazel] targets in sw/device/tests failing on cw310 and verilator when built by bazel
+ tags = ["broken"],
+ ),
verilator = verilator_params(
tags = [
+ "broken",
"cpu:4",
- "failing_verilator",
],
),
deps = [
@@ -289,10 +317,14 @@
opentitan_functest(
name = "csrng_smoketest",
srcs = ["csrng_smoketest.c"],
+ cw310 = cw310_params(
+ # FIXME #12486 [bazel] targets in sw/device/tests failing on cw310 and verilator when built by bazel
+ tags = ["broken"],
+ ),
verilator = verilator_params(
tags = [
+ "broken",
"cpu:4",
- "failing_verilator",
],
),
deps = [
@@ -307,10 +339,14 @@
opentitan_functest(
name = "entropy_src_fw_ovr_test",
srcs = ["entropy_src_fw_ovr_test.c"],
+ cw310 = cw310_params(
+ # FIXME #12486 [bazel] targets in sw/device/tests failing on cw310 and verilator when built by bazel
+ tags = ["broken"],
+ ),
verilator = verilator_params(
tags = [
+ "broken",
"cpu:4",
- "failing_verilator",
],
),
deps = [
@@ -326,6 +362,10 @@
opentitan_functest(
name = "entropy_src_smoketest",
srcs = ["entropy_src_smoketest.c"],
+ cw310 = cw310_params(
+ # FIXME #12486 [bazel] targets in sw/device/tests failing on cw310 and verilator when built by bazel
+ tags = ["broken"],
+ ),
verilator = verilator_params(
timeout = "long",
tags = ["failing_verilator"],
@@ -380,10 +420,14 @@
opentitan_functest(
name = "flash_ctrl_idle_low_power_test",
srcs = ["flash_ctrl_idle_low_power_test.c"],
+ cw310 = cw310_params(
+ # FIXME #12486 [bazel] targets in sw/device/tests failing on cw310 and verilator when built by bazel
+ tags = ["broken"],
+ ),
verilator = verilator_params(
tags = [
+ "broken",
"cpu:4",
- "failing_verilator",
],
),
deps = [
@@ -408,6 +452,10 @@
opentitan_functest(
name = "flash_ctrl_ops_test",
srcs = ["flash_ctrl_ops_test.c"],
+ cw310 = cw310_params(
+ # FIXME #12486 [bazel] targets in sw/device/tests failing on cw310 and verilator when built by bazel
+ tags = ["broken"],
+ ),
deps = [
"//hw/top_earlgrey/sw/autogen:top_earlgrey",
"//sw/device/lib:irq",
@@ -425,6 +473,10 @@
opentitan_functest(
name = "flash_ctrl_clock_freqs_test",
srcs = ["flash_ctrl_clock_freqs_test.c"],
+ cw310 = cw310_params(
+ # FIXME #12486 [bazel] targets in sw/device/tests failing on cw310 and verilator when built by bazel
+ tags = ["broken"],
+ ),
deps = [
"//sw/device/lib/base:mmio",
"//sw/device/lib/dif:clkmgr",
@@ -464,8 +516,8 @@
"--exit-success=PASS",
],
tags = [
+ "broken",
"cpu:4",
- "failing_verilator",
],
),
deps = [
@@ -477,10 +529,14 @@
opentitan_functest(
name = "hmac_enc_irq_test",
srcs = ["hmac_enc_irq_test.c"],
+ cw310 = cw310_params(
+ # FIXME #12486 [bazel] targets in sw/device/tests failing on cw310 and verilator when built by bazel
+ tags = ["broken"],
+ ),
verilator = verilator_params(
tags = [
+ "broken",
"cpu:4",
- "failing_verilator",
],
),
deps = [
@@ -648,7 +704,11 @@
name = "otbn_mem_scramble_test",
srcs = ["otbn_mem_scramble_test.c"],
verilator = verilator_params(
- tags = ["failing_verilator"],
+ tags = ["broken"],
+ ),
+ cw310 = cw310_params(
+ # FIXME #12486 [bazel] targets in sw/device/tests failing on cw310 and verilator when built by bazel
+ tags = ["broken"],
),
deps = [
"//hw/top_earlgrey/sw/autogen:top_earlgrey",
@@ -734,10 +794,15 @@
opentitan_functest(
name = "pmp_smoketest_napot",
srcs = ["pmp_smoketest_napot.c"],
+ cw310 = cw310_params(
+ # FIXME #12486 [bazel] targets in sw/device/tests failing on cw310 and verilator when built by bazel
+ tags = ["broken"],
+ ),
verilator = verilator_params(
+ # FIXME #12486 [bazel] targets in sw/device/tests failing on cw310 and verilator when built by bazel
tags = [
+ "broken",
"cpu:4",
- "failing_verilator",
],
),
deps = [
@@ -753,10 +818,15 @@
opentitan_functest(
name = "pmp_smoketest_tor",
srcs = ["pmp_smoketest_tor.c"],
+ cw310 = cw310_params(
+ # FIXME #12486 [bazel] pmp_smoketest_tor failing on cw310 and verilator when built by bazel
+ tags = ["broken"],
+ ),
verilator = verilator_params(
+ # FIXME #12486 [bazel] pmp_smoketest_tor failing on cw310 and verilator when built by bazel
tags = [
+ "broken",
"cpu:4",
- "failing_verilator",
],
),
deps = [
@@ -790,7 +860,11 @@
name = "pwrmgr_sleep_disabled_test",
srcs = ["pwrmgr_sleep_disabled_test.c"],
verilator = verilator_params(
- tags = ["failing_verilator"],
+ tags = ["broken"],
+ ),
+ cw310 = cw310_params(
+ # FIXME #12486 [bazel] targets in sw/device/tests failing on cw310 and verilator when built by bazel
+ tags = ["broken"],
),
deps = [
"//sw/device/lib:irq",
@@ -818,6 +892,9 @@
opentitan_functest(
name = "rstmgr_sw_req_test",
srcs = ["rstmgr_sw_req_test.c"],
+ cw310 = cw310_params(
+ tags = ["broken"],
+ ),
deps = [
"//hw/top_earlgrey/sw/autogen:top_earlgrey",
"//sw/device/lib/base:mmio",
@@ -831,7 +908,10 @@
name = "rstmgr_sw_rst_ctrl_test",
srcs = ["rstmgr_sw_rst_ctrl_test.c"],
verilator = verilator_params(
- tags = ["failing_verilator"],
+ tags = ["broken"],
+ ),
+ cw310 = cw310_params(
+ tags = ["broken"],
),
deps = [
"//hw/ip/i2c/data:i2c_regs",
@@ -906,6 +986,9 @@
verilator = verilator_params(
timeout = "eternal",
),
+ cw310 = cw310_params(
+ tags = ["broken"],
+ ),
deps = [
"//hw/top_earlgrey/ip/sensor_ctrl/data:sensor_ctrl_regs",
"//hw/top_earlgrey/sw/autogen:top_earlgrey",
@@ -934,6 +1017,10 @@
verilator = verilator_params(
timeout = "long",
),
+ cw310 = cw310_params(
+ # FIXME #12486 [bazel] targets in sw/device/tests failing on cw310 and verilator when built by bazel
+ tags = ["broken"],
+ ),
deps = [
"//hw/top_earlgrey/ip/sensor_ctrl/data:sensor_ctrl_regs",
"//hw/top_earlgrey/sw/autogen:top_earlgrey",
@@ -1038,6 +1125,7 @@
opentitan_functest(
name = "usbdev_test",
srcs = ["usbdev_test.c"],
+ targets = ["verilator"],
deps = [
"//hw/top_earlgrey/sw/autogen:top_earlgrey",
"//sw/device/lib:usb",
diff --git a/test/systemtest/earlgrey/test_fpga_cw310.py b/test/systemtest/earlgrey/test_fpga_cw310.py
deleted file mode 100644
index 449fa3f..0000000
--- a/test/systemtest/earlgrey/test_fpga_cw310.py
+++ /dev/null
@@ -1,103 +0,0 @@
-# Copyright lowRISC contributors.
-# Licensed under the Apache License, Version 2.0, see LICENSE for details.
-# SPDX-License-Identifier: Apache-2.0
-
-import logging
-import re
-
-import pytest
-
-from . import config
-from .. import utils
-
-log = logging.getLogger(__name__)
-
-
-@pytest.fixture(scope="module")
-def localconf_board(localconf):
- assert 'boards' in localconf
- assert 'cw310' in localconf['boards']
- boardconf = localconf['boards']['cw310']
-
- assert 'uart_device' in boardconf
- assert 'uart_speed' in boardconf
- return boardconf
-
-
-@pytest.fixture(scope="module")
-def board_earlgrey(tmp_path_factory, topsrcdir, bin_dir, localconf_board):
- """ A ChipWhispere CW310 board flashed with an Earl Grey bitstream """
-
- bitstream = bin_dir / "hw/top_earlgrey/lowrisc_systems_chip_earlgrey_cw310_0.1.bit"
- assert bitstream.is_file(), ("Bitstream not found at %s." % str(bitstream))
-
- cmd_pgm = [
- topsrcdir / 'util/fpga/cw310_loader.py',
- '--bitstream',
- bitstream,
- ]
-
- log.debug("Flashing ChipWhisperer CW310 board with bitstream {}".format(
- str(bitstream)))
- tmp_path = tmp_path_factory.mktemp('cw310_earlgrey')
- p_pgm = utils.Process(cmd_pgm, logdir=tmp_path, cwd=tmp_path)
- p_pgm.run()
- p_pgm.proc.wait(timeout=300)
- assert p_pgm.proc.returncode == 0
-
-
-@pytest.fixture(params=config.TEST_APPS_SELFCHECKING,
- ids=lambda param: param['name'])
-def app_selfchecking_bin(request, bin_dir):
- app_config = request.param
-
- if 'name' not in app_config:
- raise RuntimeError("Key 'name' not found in TEST_APPS_SELFCHECKING")
-
- if 'targets' in app_config and 'fpga_cw310' not in app_config[
- 'targets']:
- pytest.skip("Test %s skipped on ChipWhisperer CW310 board." % app_config['name'])
-
- if 'binary_name' in app_config:
- binary_name = app_config['binary_name']
- else:
- binary_name = app_config['name']
-
- # Allow tests to optionally specify their subdir within the project.
- test_dir = app_config.get('test_dir', 'sw/device/tests')
-
- test_filename = binary_name + '_fpga_cw310.bin'
- bin_path = bin_dir / test_dir / test_filename
- assert bin_path.is_file()
- return bin_path
-
-
-def test_apps_selfchecking(tmp_path, localconf_board, board_earlgrey,
- app_selfchecking_bin, topsrcdir, bin_dir, uart):
- """ Load a simple application, and check its output. """
-
- loader_path = topsrcdir / 'util/fpga/cw310_loader.py'
- utils.load_sw_over_spi(tmp_path,
- loader_path,
- app_selfchecking_bin,
- timeout=30)
-
- # We need to wait for this message to ensure we are asserting on the output
- # of the newly flashed software, not the software which might be already on
- # the chip (which might get re-executed when running the loading tool).
- bootstrap_done_exp = b'Bootstrap: DONE!'
- assert uart.find_in_output(bootstrap_done_exp, timeout=60)
-
- bootmsg_exp = b'Test ROM complete, jumping to flash!'
- assert uart.find_in_output(bootmsg_exp,
- timeout=10), "End-of-bootrom string not found."
-
- log.debug("Waiting for pass string from device test")
-
- result_match = uart.find_in_output(re.compile(br'^(PASS|FAIL)!$'),
- timeout=60)
- assert result_match is not None, "PASS/FAIL indication not found in test output."
-
- result_msg = result_match.group(1)
- log.info("Test ended with {}".format(result_msg))
- assert result_msg == b'PASS'
diff --git a/test/systemtest/earlgrey/test_fpga_nexysvideo.py b/test/systemtest/earlgrey/test_fpga_nexysvideo.py
deleted file mode 100644
index 64e791d..0000000
--- a/test/systemtest/earlgrey/test_fpga_nexysvideo.py
+++ /dev/null
@@ -1,110 +0,0 @@
-# Copyright lowRISC contributors.
-# Licensed under the Apache License, Version 2.0, see LICENSE for details.
-# SPDX-License-Identifier: Apache-2.0
-
-import logging
-import re
-
-import pytest
-
-from . import config
-from .. import utils
-
-log = logging.getLogger(__name__)
-
-
-@pytest.fixture(scope="module")
-def localconf_board(localconf):
- assert 'boards' in localconf
- assert 'nexysvideo' in localconf['boards']
- boardconf = localconf['boards']['nexysvideo']
-
- assert 'uart_device' in boardconf
- assert 'uart_speed' in boardconf
- return boardconf
-
-
-@pytest.fixture(scope="module")
-def board_earlgrey(tmp_path_factory, topsrcdir, bin_dir, localconf_board):
- """ A Nexys Video board flashed with an Earl Grey bitstream """
-
- bitstream = bin_dir / "hw/top_earlgrey/lowrisc_systems_chip_earlgrey_nexysvideo_0.1.bit"
- assert bitstream.is_file(), ("Bitstream not found at %s." % str(bitstream))
-
- cmd_pgm = [
- topsrcdir / 'util/opentitan-pgm-fpga/opentitan-pgm-fpga',
- 'xc7a200tsbg484-1',
- bitstream,
- ]
-
- # Explicitly use a certain hardware server (if set)
- if 'hw_server_url' in localconf_board:
- cmd_pgm += [
- '--hw-server-url',
- localconf_board['hw_server_url'],
- ]
-
- log.debug("Flashing Nexys Video board with bitstream {}".format(
- str(bitstream)))
- tmp_path = tmp_path_factory.mktemp('nexysvideo_earlgrey')
- p_pgm = utils.Process(cmd_pgm, logdir=tmp_path, cwd=tmp_path)
- p_pgm.run()
- p_pgm.proc.wait(timeout=300)
- assert p_pgm.proc.returncode == 0
-
-
-@pytest.fixture(params=config.TEST_APPS_SELFCHECKING,
- ids=lambda param: param['name'])
-def app_selfchecking_bin(request, bin_dir):
- app_config = request.param
-
- if 'name' not in app_config:
- raise RuntimeError("Key 'name' not found in TEST_APPS_SELFCHECKING")
-
- if 'targets' in app_config and 'fpga_nexysvideo' not in app_config[
- 'targets']:
- pytest.skip("Test %s skipped on Nexys Video board." % app_config['name'])
-
- if 'binary_name' in app_config:
- binary_name = app_config['binary_name']
- else:
- binary_name = app_config['name']
-
- # Allow tests to optionally specify their subdir within the project.
- test_dir = app_config.get('test_dir', 'sw/device/tests')
-
- test_filename = binary_name + '_fpga_nexysvideo.bin'
- bin_path = bin_dir / test_dir / test_filename
- assert bin_path.is_file()
- return bin_path
-
-
-def test_apps_selfchecking(tmp_path, localconf_board, board_earlgrey,
- app_selfchecking_bin, bin_dir, uart):
- """ Load a simple application, and check its output. """
-
- loader_path = bin_dir / 'sw/host/spiflash/spiflash'
- utils.load_sw_over_spi(tmp_path,
- loader_path,
- app_selfchecking_bin,
- timeout=30)
-
- # We need to wait for this message to ensure we are asserting on the output
- # of the newly flashed software, not the software which might be already on
- # the chip (which might get re-executed when running the loading tool).
- bootstrap_done_exp = b'Bootstrap: DONE!'
- assert uart.find_in_output(bootstrap_done_exp, timeout=60)
-
- bootmsg_exp = b'Test ROM complete, jumping to flash!'
- assert uart.find_in_output(bootmsg_exp,
- timeout=10), "End-of-bootrom string not found."
-
- log.debug("Waiting for pass string from device test")
-
- result_match = uart.find_in_output(re.compile(br'^(PASS|FAIL)!$'),
- timeout=60)
- assert result_match is not None, "PASS/FAIL indication not found in test output."
-
- result_msg = result_match.group(1)
- log.info("Test ended with {}".format(result_msg))
- assert result_msg == b'PASS'
