[dvsim tool] "use_cfgs" usecase enhancement - expanded "use_cfgs" to list hjson file targets or complete hjson dicts inline - this will allow specification of simpler, repeatitive cfgs such as lint a lot more concisely Signed-off-by: Srikrishna Iyer <sriyer@google.com>
diff --git a/util/dvsim/Deploy.py b/util/dvsim/Deploy.py index 84fb53c..95a4230 100644 --- a/util/dvsim/Deploy.py +++ b/util/dvsim/Deploy.py
@@ -674,13 +674,13 @@ except Exception as e: ex_msg = "Failed to parse \"{}\":\n{}".format( self.cov_report_dashboard, str(e)) - log.fail_msg += ex_msg + self.fail_msg += ex_msg log.error(ex_msg) self.status = "F" if self.cov_results == "": nf_msg = "Coverage summary not found in the reports dashboard!" - log.fail_msg += nf_msg + self.fail_msg += nf_msg log.error(nf_msg) self.status = "F"
diff --git a/util/dvsim/FlowCfg.py b/util/dvsim/FlowCfg.py index 309be96..306683c 100644 --- a/util/dvsim/FlowCfg.py +++ b/util/dvsim/FlowCfg.py
@@ -11,6 +11,8 @@ import pprint from shutil import which +import hjson + from .Deploy import * from .utils import * @@ -36,6 +38,10 @@ self.branch = args.branch self.job_prefix = args.job_prefix + # Options set from hjson cfg. + self.project = "" + self.scratch_path = "" + # Imported cfg files using 'import_cfgs' keyword self.imported_cfg_files = [] self.imported_cfg_files.append(flow_cfg_file) @@ -55,10 +61,6 @@ # a special key 'use_cfgs' within the hjson cfg. self.is_master_cfg = False - # Set the partial path to the IP's DV area. - self.rel_path = os.path.dirname(flow_cfg_file).replace( - self.proj_root + '/', '') - # Timestamp self.ts_format_long = args.ts_format_long self.timestamp_long = args.timestamp_long @@ -66,16 +68,21 @@ self.timestamp = args.timestamp # Results + self.rel_path = "" self.results_title = "" self.results_server_prefix = "" self.results_server_url_prefix = "" self.results_server_cmd = "" + self.results_server_css_path = "" self.results_server_path = "" self.results_server_dir = "" + self.results_server_html = "" + self.results_server_page = "" + self.results_summary_server_html = "" + self.results_summary_server_page = "" - # Full results in md text. + # Full and summary results in md text. self.results_md = "" - # Summary results in md text. self.results_summary_md = "" def __post_init__(self): @@ -117,12 +124,19 @@ # Resolve the raw hjson dict to build this object self.resolve_hjson_raw(hjson_dict) + def _post_parse_flow_cfg(self): + '''Hook to set some defaults not found in the flow cfg hjson files. + This function has to be called manually after calling the parse_flow_cfg(). + ''' + if self.rel_path == "": + self.rel_path = os.path.dirname(self.flow_cfg_file).replace( + self.proj_root + '/', '') + def check_if_master_cfg(self, hjson_dict): # This is a master cfg only if it has a single key called "use_cfgs" # which contains a list of actual flow cfgs. hjson_cfg_dict_keys = hjson_dict.keys() - return (len(hjson_cfg_dict_keys) == 1 and \ - "use_cfgs" in hjson_cfg_dict_keys and \ + return ("use_cfgs" in hjson_cfg_dict_keys and \ type(hjson_dict["use_cfgs"]) is list) def resolve_hjson_raw(self, hjson_dict): @@ -221,12 +235,74 @@ # Parse master cfg files if self.is_master_cfg: - for cfg_file in use_cfgs: - # Substitute wildcards in cfg_file files since we need to process - # them right away. - cfg_file = subst_wildcards(cfg_file, self.__dict__) - self.cfgs.append( - self.create_instance(cfg_file, self.proj_root, self.args)) + for entry in use_cfgs: + if type(entry) is str: + # Treat this as a file entry + # Substitute wildcards in cfg_file files since we need to process + # them right away. + cfg_file = subst_wildcards(entry, + self.__dict__, + ignore_error=True) + self.cfgs.append( + self.create_instance(cfg_file, self.proj_root, + self.args)) + + elif type(entry) is dict: + # Treat this as a cfg expanded in-line + temp_cfg_file = self._conv_inline_cfg_to_hjson(entry) + if not temp_cfg_file: continue + self.cfgs.append( + self.create_instance(temp_cfg_file, self.proj_root, + self.args)) + + # Delete the temp_cfg_file once the instance is created + try: + log.log(VERBOSE, "Deleting temp cfg file:\n%s", + temp_cfg_file) + os.system("/bin/rm -rf " + temp_cfg_file) + except IOError: + log.error("Failed to remove temp cfg file:\n%s", + temp_cfg_file) + + else: + log.error( + "Type of entry \"%s\" in the \"use_cfgs\" key is invalid: %s", + entry, str(type(entry))) + sys.exit(1) + + def _conv_inline_cfg_to_hjson(self, idict): + '''Dump a temp hjson file in the scratch space from input dict. + This method is to be called only by a master cfg''' + + if not self.is_master_cfg: + log.fatal("This method can only be called by a master cfg") + sys.exit(1) + + name = idict["name"] if "name" in idict.keys() else None + if not name: + log.error( + "In-line entry in use_cfgs list does not contain " + \ + "a \"name\" key (will be skipped!):\n%s", idict) + return None + + # Check if temp cfg file already exists + temp_cfg_file = self.scratch_root + "/." + self.branch + "__" + \ + name + "_cfg.hjson" + + # Create the file and dump the dict as hjson + log.log(VERBOSE, "Dumping inline cfg \"%s\" in hjson to:\n%s", name, + temp_cfg_file) + try: + with open(temp_cfg_file, "w") as f: + f.write(hjson.dumps(idict, for_json=True)) + except Exception as e: + log.error( + "Failed to hjson-dump temp cfg file\"%s\" for \"%s\"" + \ + "(will be skipped!) due to:\n%s", temp_cfg_file, name, e) + return None + + # Return the temp cfg file created + return temp_cfg_file def _process_overrides(self): # Look through the dict and find available overrides. @@ -256,7 +332,7 @@ ov_name, overrides_dict[ov_name], ov_value) sys.exit(1) else: - log.error("\"overrides\" is is a list of dicts with {\"name\": <name>, " \ + log.error("\"overrides\" is a list of dicts with {\"name\": <name>, " + \ "\"value\": <value>} pairs. Found this instead:\n%s", str(item)) sys.exit(1) @@ -364,47 +440,14 @@ ''' return - def publish_results_summary(self): - '''Public facing API for publishing md format results to the opentitan web server. - ''' - results_html_file = 'summary.html' - # master cfg doesn't have server info, instead, get it from cfgs[0] - path = self.cfgs[0].results_server_prefix + self.cfgs[0].results_server + '/' + \ - self.rel_path - results_page = path + '/' + results_html_file - results_page_url = results_page.replace( - self.cfgs[0].results_server_prefix, - self.cfgs[0].results_server_url_prefix) - - # Assume that a 'style.css' is available at root path - css_path = ( - (len(self.rel_path.split("/")) + 2) * "../") + "css/style.css" - - # Publish the results page. - # First, write the results html file temporarily to the scratch area. - f = open(results_html_file, 'w') - f.write( - md_results_to_html(self.results_title, css_path, - self.results_summary_md)) - f.close() - rm_cmd = "rm -rf " + results_html_file + "; " - - log.info("Publishing results summary to %s", results_page_url) - cmd = self.cfgs[0].results_server_cmd + " cp " + results_html_file + " " + \ - results_page + "; " + rm_cmd - log.log(VERBOSE, cmd) - try: - cmd_output = subprocess.run(args=cmd, - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - log.log(VERBOSE, cmd_output.stdout.decode("utf-8")) - except Exception as e: - log.error("%s: Failed to publish results:\n\"%s\"", e, str(cmd)) + def _get_results_page_link(self, link_text): + results_page_url = self.results_server_page.replace( + self.results_server_prefix, self.results_server_url_prefix) + return "[%s](%s)" % (link_text, results_page_url) def _publish_results(self): '''Publish results to the opentitan web server. - Results are uploaded to {results_server}/{rel_path}/latest/results. + Results are uploaded to {results_server_path}/latest/results. If the 'latest' directory exists, then it is renamed to its 'timestamp' directory. If the list of directories in this area is > 14, then the oldest entry is removed. Links to the last 7 regression results are appended at the end if the results page. @@ -416,21 +459,16 @@ return # Construct the paths - results_fname = 'results.html' - results_page = self.results_server_dir + '/' + results_fname - results_page_url = results_page.replace(self.results_server_prefix, - self.results_server_url_prefix) - - # Assume that a 'style.css' is available at root path - css_path = ( - (len(self.rel_path.split("/")) + 1) * "../") + "css/style.css" + results_page_url = self.results_server_page.replace( + self.results_server_prefix, self.results_server_url_prefix) # Timeformat for moving the dir tf = "%Y.%m.%d_%H.%M.%S" - # Extract the timestamp of the existing results_page - cmd = self.results_server_cmd + " ls -L " + results_page + " | " + \ - "grep \'Creation time:\'" + # Extract the timestamp of the existing self.results_server_page + cmd = self.results_server_cmd + " ls -L " + self.results_server_page + \ + " | grep \'Creation time:\'" + log.log(VERBOSE, cmd) cmd_output = subprocess.run(cmd, shell=True, @@ -510,7 +548,7 @@ for i in range(len(rdirs)): if i < 7: rdir_url = self.results_server_path + '/' + rdirs[ - i] + "/" + results_fname + i] + "/" + self.results_server_html rdir_url = rdir_url.replace(self.results_server_prefix, self.results_server_url_prefix) history_txt += "- [{}]({})\n".format(rdirs[i], rdir_url) @@ -525,15 +563,17 @@ # Publish the results page. # First, write the results html file temporarily to the scratch area. - results_html_file = self.results_file + ".html" + results_html_file = self.scratch_path + "/results_" + self.timestamp + ".html" f = open(results_html_file, 'w') - f.write(md_results_to_html(self.results_title, css_path, results_md)) + f.write( + md_results_to_html(self.results_title, + self.results_server_css_path, results_md)) f.close() - rm_cmd += "rm -rf " + results_html_file + "; " + rm_cmd += "/bin/rm -rf " + results_html_file + "; " log.info("Publishing results to %s", results_page_url) cmd = self.results_server_cmd + " cp " + results_html_file + " " + \ - results_page + "; " + rm_cmd + self.results_server_page + "; " + rm_cmd log.log(VERBOSE, cmd) try: cmd_output = subprocess.run(args=cmd, @@ -551,3 +591,33 @@ item._publish_results() if self.is_master_cfg: self.publish_results_summary() + + def publish_results_summary(self): + '''Public facing API for publishing md format results to the opentitan web server. + ''' + results_html_file = "summary_" + self.timestamp + ".html" + results_page_url = self.results_summary_server_page.replace( + self.results_server_prefix, self.results_server_url_prefix) + + # Publish the results page. + # First, write the results html file temporarily to the scratch area. + f = open(results_html_file, 'w') + f.write( + md_results_to_html(self.results_title, + self.results_server_css_path, + self.results_summary_md)) + f.close() + rm_cmd = "/bin/rm -rf " + results_html_file + "; " + + log.info("Publishing results summary to %s", results_page_url) + cmd = self.results_server_cmd + " cp " + results_html_file + " " + \ + self.results_summary_server_page + "; " + rm_cmd + log.log(VERBOSE, cmd) + try: + cmd_output = subprocess.run(args=cmd, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + log.log(VERBOSE, cmd_output.stdout.decode("utf-8")) + except Exception as e: + log.error("%s: Failed to publish results:\n\"%s\"", e, str(cmd))
diff --git a/util/dvsim/SimCfg.py b/util/dvsim/SimCfg.py index 9374c2d..d83b84b 100644 --- a/util/dvsim/SimCfg.py +++ b/util/dvsim/SimCfg.py
@@ -65,7 +65,6 @@ self.project = "" self.flow = "" self.flow_makefile = "" - self.scratch_path = "" self.build_dir = "" self.run_dir = "" self.sw_build_dir = "" @@ -104,9 +103,7 @@ # Parse the cfg_file file tree self.parse_flow_cfg(flow_cfg_file) - - # Stop here if this is a master cfg list - if self.is_master_cfg: return + self._post_parse_flow_cfg() # If build_unique is set, then add current timestamp to uniquify it if self.build_unique: @@ -123,14 +120,11 @@ ] self.__dict__ = find_and_substitute_wildcards(self.__dict__, self.__dict__, - ignored_wildcards) + ignored_wildcards, + self.is_master_cfg) - # TODO only support VCS coverage now - if self.tool != "vcs" and self.cov is True: - self.cov = False - log.warning( - "Coverage collection with tool \"%s\" is not supported yet", - self.tool) + # Set the title for simulation results. + self.results_title = self.name.upper() + " Simulation Results" # Print info log.info("Scratch path for %s: %s", self.name, self.scratch_path) @@ -150,8 +144,15 @@ self._process_exports() # Create objects from raw dicts - build_modes, sim_modes, run_modes, - # tests and regressions - self._create_objects() + # tests and regressions, only if not a master cfg obj + if not self.is_master_cfg: + # TODO: hack to prevent coverage collection if tool != vcs + if self.cov and self.tool != "vcs": + self.cov = False + log.warning( + "Coverage collection with tool \"%s\" is not supported yet", + self.tool) + self._create_objects() # Post init checks self.__post_init__() @@ -159,7 +160,6 @@ def __post_init__(self): # Run some post init checks super().__post_init__() - self.results_title = self.name.upper() + " Simulation Results" @staticmethod def create_instance(flow_cfg_file, proj_root, args): @@ -298,9 +298,10 @@ # Check if all items has been processed if items_list != []: - log.error("The items %s added for run were not found in \n%s!" + \ - "\nUse the --list switch to see a list of available tests / regressions.", \ - items_list, self.flow_cfg_file) + log.error( + "The items %s added for run were not found in \n%s!\n \ + Use the --list switch to see a list of available \ + tests / regressions.", items_list, self.flow_cfg_file) sys.exit(1) # Process reseed override and create the build_list @@ -400,6 +401,26 @@ if self.cov: Deploy.deploy(self.cov_deploys) + def _cov_analyze(self): + '''Use the last regression coverage data to open up the GUI tool to + analyze the coverage. + ''' + cov_analyze_deploy = CovAnalyze(self) + try: + proc = subprocess.Popen(args=cov_analyze_deploy.cmd, + shell=True, + close_fds=True) + except Exception as e: + log.fatal("Failed to run coverage analysis cmd:\n\"%s\"\n%s", + cov_analyze_deploy.cmd, e) + sys.exit(1) + + def cov_analyze(self): + '''Public facing API for analyzing coverage. + ''' + for item in self.cfgs: + item._cov_analyze() + def _gen_results(self): ''' The function is called after the regression has completed. It collates the @@ -482,18 +503,19 @@ results_str += self.cov_report_deploy.cov_results self.results_summary["Coverage"] = self.cov_report_deploy.cov_total else: - self.results_summary["Coverage"] = "N.A. %" + self.results_summary["Coverage"] = "-- %" # append link of detail result to block name - self.results_summary["Name"] = self.append_result_link(self.results_summary["Name"]) + self.results_summary["Name"] = self._get_results_page_link( + self.results_summary["Name"]) # Append failures for triage self.results_md = results_str + fail_msgs # Write results to the scratch area - self.results_file = self.scratch_path + "/results_" + self.timestamp + ".md" - log.info("Detailed results are available at %s", self.results_file) - f = open(self.results_file, 'w') + results_file = self.scratch_path + "/results_" + self.timestamp + ".md" + log.info("Detailed results are available at %s", results_file) + f = open(results_file, 'w') f.write(self.results_md) f.close() @@ -511,7 +533,7 @@ for title in item.results_summary: row.append(item.results_summary[title]) table.append(row) - self.results_summary_md = "## Simulation Summary Results\n" + self.results_summary_md = "## " + self.results_title + " (Summary)\n" self.results_summary_md += "### " + self.timestamp_long + "\n" self.results_summary_md += tabulate(table, headers="firstrow", @@ -520,32 +542,6 @@ print(self.results_summary_md) return self.results_summary_md - def append_result_link(self, link_name): - results_page = self.results_server_dir + '/' + 'results.html' - results_page_url = results_page.replace(self.results_server_prefix, - self.results_server_url_prefix) - return "[%s](%s)" % (link_name, results_page_url) - - def _cov_analyze(self): - '''Use the last regression coverage data to open up the GUI tool to - analyze the coverage. - ''' - cov_analyze_deploy = CovAnalyze(self) - try: - proc = subprocess.Popen(args=cov_analyze_deploy.cmd, - shell=True, - close_fds=True) - except Exception as e: - log.fatal("Failed to run coverage analysis cmd:\n\"%s\"\n%s", - cov_analyze_deploy.cmd, e) - sys.exit(1) - - def cov_analyze(self): - '''Public facing API for analyzing coverage. - ''' - for item in self.cfgs: - item._cov_analyze() - def _publish_results(self): '''Publish coverage results to the opentitan web server.''' super()._publish_results()
diff --git a/util/dvsim/utils.py b/util/dvsim/utils.py index 0e95a41..377bd67 100644 --- a/util/dvsim/utils.py +++ b/util/dvsim/utils.py
@@ -86,7 +86,7 @@ return hjson_cfg_dict -def subst_wildcards(var, mdict, ignored_wildcards=[]): +def subst_wildcards(var, mdict, ignored_wildcards=[], ignore_error=False): ''' If var has wildcards specified within {..}, find and substitute them. ''' @@ -96,7 +96,8 @@ if "{eval_cmd}" in var: idx = var.find("{eval_cmd}") + 11 - subst_var = subst_wildcards(var[idx:], mdict, ignored_wildcards) + subst_var = subst_wildcards(var[idx:], mdict, ignored_wildcards, + ignore_error) # If var has wildcards that were ignored, then skip running the command # for now, assume that it will be handled later. match = re.findall(r"{([A-Za-z0-9\_]+)}", subst_var) @@ -115,7 +116,8 @@ subst_found = [] for element in found: element = subst_wildcards( - element, mdict, ignored_wildcards) + element, mdict, ignored_wildcards, + ignore_error) subst_found.append(element) # Expand list into a str since list within list is # not supported. @@ -123,7 +125,8 @@ elif type(found) is str: found = subst_wildcards(found, mdict, - ignored_wildcards) + ignored_wildcards, + ignore_error) elif type(found) is bool: found = int(found) @@ -132,7 +135,7 @@ # Check if the wildcard exists as an environment variable env_var = os.environ.get(item) if env_var is not None: subst_list[item] = env_var - else: + elif not ignore_error: log.error( "Substitution for the wildcard \"%s\" not found", item) @@ -142,7 +145,10 @@ return var -def find_and_substitute_wildcards(sub_dict, full_dict, ignored_wildcards=[]): +def find_and_substitute_wildcards(sub_dict, + full_dict, + ignored_wildcards=[], + ignore_error=False): ''' Recursively find key values containing wildcards in sub_dict in full_dict and return resolved sub_dict. @@ -151,7 +157,7 @@ if type(sub_dict[key]) in [dict, OrderedDict]: # Recursively call this funciton in sub-dicts sub_dict[key] = find_and_substitute_wildcards( - sub_dict[key], full_dict, ignored_wildcards) + sub_dict[key], full_dict, ignored_wildcards, ignore_error) elif type(sub_dict[key]) is list: sub_dict_key_values = list(sub_dict[key]) @@ -162,18 +168,19 @@ # Recursively call this funciton in sub-dicts sub_dict_key_values[i] = \ find_and_substitute_wildcards(sub_dict_key_values[i], - full_dict, ignored_wildcards) + full_dict, ignored_wildcards, ignore_error) elif type(sub_dict_key_values[i]) is str: sub_dict_key_values[i] = subst_wildcards( - sub_dict_key_values[i], full_dict, ignored_wildcards) + sub_dict_key_values[i], full_dict, ignored_wildcards, + ignore_error) # Set the substituted key values back sub_dict[key] = sub_dict_key_values elif type(sub_dict[key]) is str: sub_dict[key] = subst_wildcards(sub_dict[key], full_dict, - ignored_wildcards) + ignored_wildcards, ignore_error) return sub_dict @@ -217,7 +224,7 @@ return op # List of 'not applicable' identifiers. - na_list = ['--', 'NA', 'N.A.', 'N.A', 'na', 'n.a.', 'n.a'] + na_list = ['--', 'NA', 'N.A.', 'N.A', 'N/A', 'na', 'n.a.', 'n.a', 'n/a'] na_list_patterns = '|'.join(na_list) # List of floating point patterns: '0', '0.0' & '.0'