[dv] Add simulation result summary page
Here is the report that will be generated
https://reports.opentitan.org/hw/top_earlgrey/dv/summarys.html
Also fix xbar coverage error
Signed-off-by: Weicai Yang <weicai@google.com>
diff --git a/util/dvsim.py b/util/dvsim.py
index ee516a7..258d93b 100755
--- a/util/dvsim.py
+++ b/util/dvsim.py
@@ -458,10 +458,11 @@
cfg.deploy_objects()
# Generate results.
- results = cfg.gen_results()
+ cfg.gen_results()
# Publish results
if args.publish: cfg.publish_results()
+
else:
log.info("No items specified to be run.")
diff --git a/util/dvsim/FlowCfg.py b/util/dvsim/FlowCfg.py
index ad02c06..53838f7 100644
--- a/util/dvsim/FlowCfg.py
+++ b/util/dvsim/FlowCfg.py
@@ -75,6 +75,8 @@
# Full results in md text.
self.results_md = ""
+ # Summary results in md text.
+ self.results_summary_md = ""
def __post_init__(self):
# Run some post init checks
@@ -343,7 +345,51 @@
result = item._gen_results()
print(result)
results.append(result)
- return results
+
+ if self.is_master_cfg: self.gen_results_summary()
+
+ def gen_results_summary(self):
+ '''Public facing API to generate summary results for each IP/cfg file
+ '''
+ return
+
+ def publish_results_summary(self):
+ '''Public facing API for publishing md format results to the opentitan web server.
+ '''
+ results_html_file = 'summary.html'
+ # master cfg doesn't have server info, instead, get it from cfgs[0]
+ path = self.cfgs[0].results_server_prefix + self.cfgs[0].results_server + '/' + \
+ self.rel_path
+ results_page = path + '/' + results_html_file
+ results_page_url = results_page.replace(
+ self.cfgs[0].results_server_prefix,
+ self.cfgs[0].results_server_url_prefix)
+
+ # Assume that a 'style.css' is available at root path
+ css_path = (
+ (len(self.rel_path.split("/")) + 2) * "../") + "css/style.css"
+
+ # Publish the results page.
+ # First, write the results html file temporarily to the scratch area.
+ f = open(results_html_file, 'w')
+ f.write(
+ md_results_to_html(self.results_title, css_path,
+ self.results_summary_md))
+ f.close()
+ rm_cmd = "rm -rf " + results_html_file + "; "
+
+ log.info("Publishing results summary to %s", results_page_url)
+ cmd = self.cfgs[0].results_server_cmd + " cp " + results_html_file + " " + \
+ results_page + "; " + rm_cmd
+ log.log(VERBOSE, cmd)
+ try:
+ cmd_output = subprocess.run(args=cmd,
+ shell=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT)
+ log.log(VERBOSE, cmd_output.stdout.decode("utf-8"))
+ except Exception as e:
+ log.error("%s: Failed to publish results:\n\"%s\"", e, str(cmd))
def _publish_results(self):
'''Publish results to the opentitan web server.
@@ -492,3 +538,5 @@
'''
for item in self.cfgs:
item._publish_results()
+
+ if self.is_master_cfg: self.publish_results_summary()
diff --git a/util/dvsim/SimCfg.py b/util/dvsim/SimCfg.py
index 20aa1a4..93c8401 100644
--- a/util/dvsim/SimCfg.py
+++ b/util/dvsim/SimCfg.py
@@ -104,6 +104,7 @@
# Parse the cfg_file file tree
self.parse_flow_cfg(flow_cfg_file)
+ self.final_total = []
# Stop here if this is a master cfg list
if self.is_master_cfg: return
@@ -463,6 +464,9 @@
regr_results=regr_results,
map_full_testplan=self.map_full_testplan)
results_str += "\n"
+ self.final_total = self.testplan.final_total
+ # append link
+ self.final_total.append(self.append_result_link("Link"))
# Append coverage results of coverage was enabled.
if self.cov and self.cov_report_deploy.status == "P":
@@ -483,6 +487,27 @@
# Return only the tables
return results_str
+ def gen_results_summary(self):
+
+ # sim summary result has 4 columns provided by Testplan::results_table
+ table = [["Name", "Passing", "Total", "Pass Rate", "Detail"]]
+ colalign = ("center", ) * 5
+ for item in self.cfgs:
+ table.append(item.final_total)
+ self.results_summary_md = "## Simulation Summary Results\n"
+ self.results_summary_md += tabulate(table,
+ headers="firstrow",
+ tablefmt="pipe",
+ colalign=colalign)
+ print(self.results_summary_md)
+ return self.results_summary_md
+
+ def append_result_link(self, link_name):
+ results_page = self.results_server_dir + '/' + 'results.html'
+ results_page_url = results_page.replace(self.results_server_prefix,
+ self.results_server_url_prefix)
+ return "[%s](%s)" % (link_name, results_page_url)
+
def _cov_analyze(self):
'''Use the last regression coverage data to open up the GUI tool to
analyze the coverage.
diff --git a/util/testplanner/class_defs.py b/util/testplanner/class_defs.py
index 05694d9..56b4177 100644
--- a/util/testplanner/class_defs.py
+++ b/util/testplanner/class_defs.py
@@ -162,6 +162,9 @@
def __init__(self, name):
self.name = name
self.entries = []
+ self.final_total = []
+ self.results = ""
+
if name == "":
print("Error: testplan name cannot be empty")
sys.exit(1)
@@ -324,7 +327,13 @@
])
milestone = ""
entry_name = ""
- return tabulate(table,
- headers="firstrow",
- tablefmt="pipe",
- colalign=colalign)
+ if entry.milestone == "N.A." and entry.name == "N.A.":
+ self.final_total = [
+ self.name.upper(), test["passing"], test["total"],
+ pass_rate
+ ]
+ self.results = tabulate(table,
+ headers="firstrow",
+ tablefmt="pipe",
+ colalign=colalign)
+ return self.results
diff --git a/util/tlgen/generate.py b/util/tlgen/generate.py
index 6320783..3b39ab0 100644
--- a/util/tlgen/generate.py
+++ b/util/tlgen/generate.py
@@ -3,6 +3,7 @@
# SPDX-License-Identifier: Apache-2.0
import logging as log
+
from mako import exceptions
from mako.template import Template
from pkg_resources import resource_filename
@@ -11,7 +12,6 @@
from .xbar import Xbar
-
def generate(xbar): #xbar: Xbar -> str
"""generate uses elaborated model then creates top level Xbar module
with prefix.
diff --git a/util/tlgen/generate_tb.py b/util/tlgen/generate_tb.py
index 446f5f4..64b0d6b 100644
--- a/util/tlgen/generate_tb.py
+++ b/util/tlgen/generate_tb.py
@@ -16,7 +16,7 @@
# list all the generate files for TB
tb_files = [
"xbar_env_pkg__params.sv", "tb__xbar_connect.sv", "xbar.sim.core",
- "xbar.bind.sv", "Makefile", "xbar.sim_cfg.hjson"
+ "xbar.bind.sv", "Makefile", "xbar.sim_cfg.hjson", "xbar.testplan.hjson"
]
for fname in tb_files:
@@ -29,8 +29,15 @@
fname = "xbar_%s_bind.sv" % (xbar.name)
elif fname == "xbar.sim_cfg.hjson":
fname = "xbar_%s_sim_cfg.hjson" % (xbar.name)
+ elif fname == "xbar.testplan.hjson":
+ fname = "xbar_%s_testplan.hjson" % (xbar.name)
- dv_filepath = dv_path / fname
+ # save testplan at data directory
+ if fname == "xbar_%s_testplan.hjson" % (xbar.name):
+ dv_filepath = dv_path / '../../data/autogen' / fname
+ else:
+ dv_filepath = dv_path / fname
+
with dv_filepath.open(mode='w', encoding='UTF-8') as fout:
try:
fout.write(tpl.render(xbar=xbar))
diff --git a/util/tlgen/xbar.sim_cfg.hjson.tpl b/util/tlgen/xbar.sim_cfg.hjson.tpl
index decb93e..4ff37b2 100644
--- a/util/tlgen/xbar.sim_cfg.hjson.tpl
+++ b/util/tlgen/xbar.sim_cfg.hjson.tpl
@@ -7,12 +7,6 @@
// Top level dut name (sv module).
dut: xbar_${xbar.name}
- // Testplan hjson file.
- testplan: "{proj_root}/hw/top_earlgrey/ip/xbar_${xbar.name}/data/autogen/xbar_${xbar.name}_testplan.hjson"
-
- // Add xbar_${xbar.name} specific exclusion files.
- vcs_cov_excl_files: ["{proj_root}/hw/top_earlgrey/ip/xbar_${xbar.name}/dv/cov/xbar_cov_excl.el"]
-
// Import additional common sim cfg files.
import_cfgs: [// xbar common sim cfg file
"{proj_root}/hw/ip/tlul/generic_dv/xbar_sim_cfg.hjson"]
diff --git a/util/tlgen/xbar.testplan.hjson.tpl b/util/tlgen/xbar.testplan.hjson.tpl
new file mode 100644
index 0000000..11e5cc8
--- /dev/null
+++ b/util/tlgen/xbar.testplan.hjson.tpl
@@ -0,0 +1,9 @@
+// Copyright lowRISC contributors.
+// Licensed under the Apache License, Version 2.0, see LICENSE for details.
+// SPDX-License-Identifier: Apache-2.0
+//
+// xbar_${xbar.name}_testplan.hjson file generated by `tlgen.py` tool
+{
+ name: "xbar_${xbar.name}"
+ import_testplans: ["hw/ip/tlul/data/tlul_testplan.hjson"]
+}
