[dvsim] Enable coverage collection with Xcelium
- Enabled coverage collection with Xcelium
- Added Xcelium too opts
- Made fixes in SimCfg class to add support for it
- Fixed coverage 'bug' caught by Xceluim in `tl_agent_cov`
Publishing reports to the web server is disabled.
Signed-off-by: Srikrishna Iyer <sriyer@google.com>
diff --git a/util/dvsim/Deploy.py b/util/dvsim/Deploy.py
index e412c40..d7e2a68 100644
--- a/util/dvsim/Deploy.py
+++ b/util/dvsim/Deploy.py
@@ -17,6 +17,7 @@
import hjson
from tabulate import tabulate
+from sim_utils import *
from utils import *
@@ -170,6 +171,11 @@
# If renew_odir flag is True - then move it.
if self.renew_odir: self.odir_limiter(odir=self.odir)
os.system("mkdir -p " + self.odir)
+ # Dump all env variables for ease of debug.
+ with open(self.odir + "/env_vars", "w") as f:
+ for var in sorted(self.exports.keys()):
+ f.write("{}={}\n".format(var, self.exports[var]))
+ f.close()
os.system("ln -s " + self.odir + " " + self.sim_cfg.links['D'] +
'/' + self.odir_ln)
f = open(self.log, "w")
@@ -744,6 +750,9 @@
if self.sim_cfg.cov_merge_previous:
self.cov_db_dirs += prev_cov_db_dirs
+ # Append cov_db_dirs to the list of exports.
+ self.exports["cov_db_dirs"] = "\"{}\"".format(self.cov_db_dirs)
+
# Call base class __post_init__ to do checks and substitutions
super().__post_init__()
@@ -772,7 +781,7 @@
self.mandatory_misc_attrs = {
"cov_report_dir": False,
"cov_merge_db_dir": False,
- "cov_report_dashboard": False
+ "cov_report_txt": False
}
# Initialize
@@ -791,47 +800,21 @@
super().get_status()
# Once passed, extract the cov results summary from the dashboard.
if self.status == "P":
- try:
- with open(self.cov_report_dashboard, 'r') as f:
- for line in f:
- match = re.match("total coverage summary", line,
- re.IGNORECASE)
- if match:
- results = []
- # Metrics on the next line.
- line = f.readline().strip()
- results.append(line.split())
- # Values on the next.
- line = f.readline().strip()
- # Pretty up the values - add % sign for ease of post
- # processing.
- values = []
- for val in line.split():
- val += " %"
- values.append(val)
- # first row is coverage total
- self.cov_total = values[0]
- results.append(values)
- colalign = (("center", ) * len(values))
- self.cov_results = tabulate(results,
- headers="firstrow",
- tablefmt="pipe",
- colalign=colalign)
- break
+ results, self.cov_total, ex_msg = get_cov_summary_table(
+ self.cov_report_txt, self.sim_cfg.tool)
- except Exception as e:
- ex_msg = "Failed to parse \"{}\":\n{}".format(
- self.cov_report_dashboard, str(e))
+ if not ex_msg:
+ # Succeeded in obtaining the coverage data.
+ colalign = (("center", ) * len(results[0]))
+ self.cov_results = tabulate(results,
+ headers="firstrow",
+ tablefmt="pipe",
+ colalign=colalign)
+ else:
self.fail_msg += ex_msg
log.error(ex_msg)
self.status = "F"
- if self.cov_results == "":
- nf_msg = "Coverage summary not found in the reports dashboard!"
- self.fail_msg += nf_msg
- log.error(nf_msg)
- self.status = "F"
-
if self.status == "P":
# Delete the cov report - not needed.
os.system("rm -rf " + self.log)
@@ -851,6 +834,9 @@
self.fail_patterns = []
self.mandatory_cmd_attrs = {
+ # tool srcs
+ "tool_srcs": False,
+ "tool_srcs_dir": False,
"cov_analyze_cmd": False,
"cov_analyze_opts": False
}
diff --git a/util/dvsim/SimCfg.py b/util/dvsim/SimCfg.py
index 91562ba..55157c2 100644
--- a/util/dvsim/SimCfg.py
+++ b/util/dvsim/SimCfg.py
@@ -53,6 +53,9 @@
self.dry_run = args.dry_run
self.map_full_testplan = args.map_full_testplan
+ # Disable cov if --build-only is passed.
+ if self.build_only: self.cov = False
+
# Set default sim modes for unpacking
if self.waves is True: self.en_build_modes.append("waves")
if self.cov is True: self.en_build_modes.append("cov")
@@ -145,12 +148,6 @@
# Create objects from raw dicts - build_modes, sim_modes, run_modes,
# tests and regressions, only if not a master cfg obj
- # TODO: hack to prevent coverage collection if tool != vcs
- if self.cov and self.tool != "vcs":
- self.cov = False
- log.warning(
- "Coverage collection with tool \"%s\" is not supported yet",
- self.tool)
self._create_objects()
# Post init checks
@@ -405,14 +402,7 @@
analyze the coverage.
'''
cov_analyze_deploy = CovAnalyze(self)
- try:
- proc = subprocess.Popen(args=cov_analyze_deploy.cmd,
- shell=True,
- close_fds=True)
- except Exception as e:
- log.fatal("Failed to run coverage analysis cmd:\n\"%s\"\n%s",
- cov_analyze_deploy.cmd, e)
- sys.exit(1)
+ self.deploy = [cov_analyze_deploy]
def cov_analyze(self):
'''Public facing API for analyzing coverage.
@@ -500,12 +490,17 @@
if self.cov:
if self.cov_report_deploy.status == "P":
results_str += "\n## Coverage Results\n"
- results_str += "\n### [Coverage Dashboard]"
- results_str += "(cov_report/dashboard.html)\n\n"
+ # Link the dashboard page using "cov_report_page" value.
+ # TODO: hack to only link VCS generated results.
+ if self.tool == "vcs" and hasattr(self, "cov_report_page"):
+ results_str += "\n### [Coverage Dashboard]"
+ results_str += "({})\n\n".format(
+ getattr(self, "cov_report_page"))
results_str += self.cov_report_deploy.cov_results
self.results_summary[
"Coverage"] = self.cov_report_deploy.cov_total
- self.results_summary["Coverage"] = "--"
+ else:
+ self.results_summary["Coverage"] = "--"
# append link of detail result to block name
self.results_summary["Name"] = self._get_results_page_link(
@@ -552,6 +547,8 @@
super()._publish_results()
if self.cov:
+ # TODO: hack to only allow VCS coverage data to be uploaded.
+ if self.tool != "vcs": return
results_server_dir_url = self.results_server_dir.replace(
self.results_server_prefix, self.results_server_url_prefix)
diff --git a/util/dvsim/dvsim.py b/util/dvsim/dvsim.py
index b832fc8..93d8082 100755
--- a/util/dvsim/dvsim.py
+++ b/util/dvsim/dvsim.py
@@ -491,6 +491,7 @@
# tool.
if args.cov_analyze:
cfg.cov_analyze()
+ cfg.deploy_objects()
sys.exit(0)
# Purge the scratch path if --purge option is set.
diff --git a/util/dvsim/sim_utils.py b/util/dvsim/sim_utils.py
new file mode 100644
index 0000000..3a50536
--- /dev/null
+++ b/util/dvsim/sim_utils.py
@@ -0,0 +1,105 @@
+#!/usr/bin/env python3
+# Copyright lowRISC contributors.
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+"""
+This script provides common DV simulation specific utilities.
+"""
+
+import re
+from collections import OrderedDict
+
+
+# Capture the summary results as a list of lists.
+# The text coverage report is passed as input to the function, in addition to
+# the tool used. The tool returns a 2D list if the coverage report file was read
+# and the coverage was extracted successfully. It returns a tuple of:
+# List of metrics and values
+# Final coverage total
+# Error message, if failed
+def get_cov_summary_table(cov_report_txt, tool):
+ try:
+ with open(cov_report_txt, 'r') as f:
+ if tool == 'xcelium':
+ return xcelium_cov_summary_table(f)
+ if tool == 'vcs':
+ return vcs_cov_summary_table(f)
+
+ err_msg = "Unsupported tool for cov extraction: {}".format(tool)
+ return None, None, err_msg
+
+ except Exception as e:
+ err_msg = "Exception occurred: {}".format(str(e))
+ return None, None, err_msg
+
+
+# Same desc as above, but specific to Xcelium and takes an opened input stream.
+def xcelium_cov_summary_table(buf):
+ for line in buf:
+ if "name" in line:
+ # Strip the line and remove the unwanted "* Covered" string.
+ metrics = line.strip().replace("* Covered", "").split()
+ # Change first item to 'Score'.
+ metrics[0] = 'Score'
+
+ # metric.
+ items = OrderedDict()
+ for metric in metrics:
+ items[metric] = {}
+ items[metric]['covered'] = 0
+ items[metric]['total'] = 0
+ # Next line is a separator.
+ line = buf.readline()
+ # Subsequent lines are coverage items to be aggregated.
+ for line in buf:
+ line = re.sub(r"%\s+\(", "%(", line)
+ values = line.strip().split()
+ for i, value in enumerate(values):
+ value = value.strip()
+ m = re.search(r"\((\d+)/(\d+)\)", value)
+ if m:
+ items[metrics[i]]['covered'] += int(m.group(1))
+ items[metrics[i]]['total'] += int(m.group(2))
+ items['Score']['covered'] += int(m.group(1))
+ items['Score']['total'] += int(m.group(2))
+ # Capture the percentages and the aggregate.
+ values = []
+ cov_total = None
+ for metric in items.keys():
+ if items[metric]['total'] == 0: values.append("-- %")
+ else:
+ value = items[metric]['covered'] / items[metric][
+ 'total'] * 100
+ value = "{0:.2f} %".format(round(value, 2))
+ values.append(value)
+ if metric == 'Score': cov_total = value
+ return [metrics, values], cov_total, None
+
+ # If we reached here, then we were unable to extract the coverage.
+ err_msg = "ParseError: coverage data not found!"
+ return None, None, err_msg
+
+
+# Same desc as above, but specific to VCS and takes an opened input stream.
+def vcs_cov_summary_table(buf):
+ for line in buf:
+ match = re.match("total coverage summary", line, re.IGNORECASE)
+ if match:
+ # Metrics on the next line.
+ line = buf.readline().strip()
+ metrics = line.split()
+ # Values on the next.
+ line = buf.readline().strip()
+ # Pretty up the values - add % sign for ease of post
+ # processing.
+ values = []
+ for val in line.split():
+ val += " %"
+ values.append(val)
+ # first row is coverage total
+ cov_total = values[0]
+ return [metrics, values], cov_total, None
+
+ # If we reached here, then we were unable to extract the coverage.
+ err_msg = "ParseError: coverage data not found!"
+ return None, None, err_msg
