| # Copyright lowRISC contributors. | |
| # Licensed under the Apache License, Version 2.0, see LICENSE for details. | |
| # SPDX-License-Identifier: Apache-2.0 | |
| import logging as log | |
| import os | |
| import pprint | |
| import random | |
| import re | |
| import shlex | |
| import subprocess | |
| import sys | |
| import time | |
| from collections import OrderedDict | |
| from sim_utils import get_cov_summary_table | |
| from tabulate import tabulate | |
| from utils import VERBOSE, find_and_substitute_wildcards, run_cmd | |
| class Deploy(): | |
| """ | |
| Abstraction for deploying builds and runs. | |
| """ | |
| # Timer in hours, minutes and seconds. | |
| hh = 0 | |
| mm = 0 | |
| ss = 0 | |
| # Maintain a list of dispatched items. | |
| dispatch_counter = 0 | |
| # Misc common deploy settings. | |
| print_legend = True | |
| print_interval = 5 | |
| max_parallel = 16 | |
| max_odirs = 5 | |
| # Max jobs dispatched in one go. | |
| slot_limit = 20 | |
| def __self_str__(self): | |
| if log.getLogger().isEnabledFor(VERBOSE): | |
| return pprint.pformat(self.__dict__) | |
| else: | |
| ret = self.cmd | |
| if self.sub != []: | |
| ret += "\nSub:\n" + str(self.sub) | |
| return ret | |
| def __str__(self): | |
| return self.__self_str__() | |
| def __repr__(self): | |
| return self.__self_str__() | |
| def __init__(self, sim_cfg): | |
| '''Initialize common class members.''' | |
| # Cross ref the whole cfg object for ease. | |
| self.sim_cfg = sim_cfg | |
| # Common vars | |
| self.identifier = "" | |
| self.cmd = "" | |
| self.odir = "" | |
| self.log = "" | |
| self.fail_msg = "" | |
| # Flag to indicate whether to 'overwrite' if odir already exists, | |
| # or to backup the existing one and create a new one. | |
| # For builds, we want to overwrite existing to leverage the tools' | |
| # incremental / partition compile features. For runs, we may want to | |
| # create a new one. | |
| self.renew_odir = False | |
| # List of vars required to be exported to sub-shell | |
| self.exports = {} | |
| # Deploy sub commands | |
| self.sub = [] | |
| # Process | |
| self.process = None | |
| self.log_fd = None | |
| self.status = None | |
| # These are mandatory class attributes that need to be extracted and | |
| # set from the sim_cfg object. These are explicitly used to construct | |
| # the command for deployment. | |
| self.mandatory_cmd_attrs = {} | |
| # These are mandatory class attributes that also need to be extracted | |
| # and set from the sim_cfg object. Some of these contribute to the | |
| # construction of the command. Others are used to determine pass / fail | |
| # conditions. | |
| self.mandatory_misc_attrs = { | |
| "name": False, | |
| "build_mode": False, | |
| "flow_makefile": False, | |
| "exports": False, | |
| "dry_run": False | |
| } | |
| # Function to parse a dict and extract the mandatory cmd and misc attrs. | |
| def parse_dict(self, ddict): | |
| if not hasattr(self, "target"): | |
| log.error( | |
| "Class %s does not have the mandatory attribute \"target\" defined", | |
| self.__class__.__name__) | |
| sys.exit(1) | |
| ddict_keys = ddict.keys() | |
| for key in self.mandatory_cmd_attrs.keys(): | |
| if self.mandatory_cmd_attrs[key] is False: | |
| if key in ddict_keys: | |
| setattr(self, key, ddict[key]) | |
| self.mandatory_cmd_attrs[key] = True | |
| for key in self.mandatory_misc_attrs.keys(): | |
| if self.mandatory_misc_attrs[key] is False: | |
| if key in ddict_keys: | |
| setattr(self, key, ddict[key]) | |
| self.mandatory_misc_attrs[key] = True | |
| def __post_init__(self): | |
| # Ensure all mandatory attrs are set | |
| for attr in self.mandatory_cmd_attrs.keys(): | |
| if self.mandatory_cmd_attrs[attr] is False: | |
| log.error("Attribute \"%s\" not found for \"%s\".", attr, | |
| self.name) | |
| sys.exit(1) | |
| for attr in self.mandatory_misc_attrs.keys(): | |
| if self.mandatory_misc_attrs[attr] is False: | |
| log.error("Attribute \"%s\" not found for \"%s\".", attr, | |
| self.name) | |
| sys.exit(1) | |
| # Recursively search and replace wildcards | |
| # First pass: search within self dict. We ignore errors since some | |
| # substitions may be available in the second pass. | |
| self.__dict__ = find_and_substitute_wildcards(self.__dict__, | |
| self.__dict__, [], True) | |
| # Second pass: search in sim_cfg dict, this time not ignoring errors. | |
| self.__dict__ = find_and_substitute_wildcards(self.__dict__, | |
| self.sim_cfg.__dict__, | |
| [], False) | |
| # Set identifier. | |
| self.identifier = self.sim_cfg.name + ":" + self.name | |
| # Set the command, output dir and log | |
| self.odir = getattr(self, self.target + "_dir") | |
| # Set the output dir link name to the basename of odir (by default) | |
| self.odir_ln = os.path.basename(os.path.normpath(self.odir)) | |
| self.log = self.odir + "/" + self.target + ".log" | |
| # If using LSF, redirect stdout and err to the log file | |
| self.cmd = self.construct_cmd() | |
| def construct_cmd(self): | |
| cmd = "make -f " + self.flow_makefile + " " + self.target | |
| if self.dry_run is True: | |
| cmd += " -n" | |
| for attr in sorted(self.mandatory_cmd_attrs.keys()): | |
| value = getattr(self, attr) | |
| if type(value) is list: | |
| pretty_value = [] | |
| for item in value: | |
| pretty_value.append(item.strip()) | |
| value = " ".join(pretty_value) | |
| if type(value) is bool: | |
| value = int(value) | |
| if type(value) is str: | |
| value = value.strip() | |
| cmd += " " + attr + "=\"" + str(value) + "\"" | |
| # TODO: If not running locally, redirect stdout and err to the log file | |
| # self.cmd += " > " + self.log + " 2>&1 &" | |
| return cmd | |
| def is_equivalent_job(self, item): | |
| '''Checks if job that would be dispatched with `item` is equivalent to | |
| `self`. | |
| Determines if `item` and `self` would behave exactly the same way when | |
| deployed. If so, then there is no point in keeping both. The caller can | |
| choose to discard `item` and pick `self` instead. To do so, we check | |
| the final resolved `cmd` & the exports. The `name` field will be unique | |
| to `item` and `self`, so we take that out of the comparison. | |
| ''' | |
| if type(self) != type(item): | |
| return False | |
| # Check if the cmd field is identical. | |
| item_cmd = item.cmd.replace(item.name, self.name) | |
| if self.cmd != item_cmd: | |
| return False | |
| # Check if exports have identical set of keys. | |
| if self.exports.keys() != item.exports.keys(): | |
| return False | |
| # Check if exports have identical values. | |
| for key, val in self.exports.items(): | |
| item_val = item.exports[key] | |
| if type(item_val) is str: | |
| item_val = item_val.replace(item.name, self.name) | |
| if val != item_val: | |
| return False | |
| log.log(VERBOSE, "Deploy job \"%s\" is equivalent to \"%s\"", | |
| item.name, self.name) | |
| return True | |
| def dispatch_cmd(self): | |
| self.exports.update(os.environ) | |
| args = shlex.split(self.cmd) | |
| try: | |
| # If renew_odir flag is True - then move it. | |
| if self.renew_odir: | |
| self.odir_limiter(odir=self.odir) | |
| os.system("mkdir -p " + self.odir) | |
| # Dump all env variables for ease of debug. | |
| with open(self.odir + "/env_vars", | |
| "w", | |
| encoding="UTF-8", | |
| errors="surrogateescape") as f: | |
| for var in sorted(self.exports.keys()): | |
| f.write("{}={}\n".format(var, self.exports[var])) | |
| f.close() | |
| os.system("ln -s " + self.odir + " " + self.sim_cfg.links['D'] + | |
| '/' + self.odir_ln) | |
| f = open(self.log, "w", encoding="UTF-8", errors="surrogateescape") | |
| f.write("[Executing]:\n{}\n\n".format(self.cmd)) | |
| f.flush() | |
| self.process = subprocess.Popen(args, | |
| bufsize=4096, | |
| universal_newlines=True, | |
| stdout=f, | |
| stderr=f, | |
| env=self.exports) | |
| self.log_fd = f | |
| self.status = "D" | |
| Deploy.dispatch_counter += 1 | |
| except IOError: | |
| log.error('IO Error: See %s', self.log) | |
| if self.log_fd: | |
| self.log_fd.close() | |
| self.status = "K" | |
| def odir_limiter(self, odir, max_odirs=-1): | |
| '''Function to backup previously run output directory to maintain a | |
| history of a limited number of output directories. It deletes the output | |
| directory with the oldest timestamps, if the limit is reached. It returns | |
| a list of directories that remain after deletion. | |
| Arguments: | |
| odir: The output directory to backup | |
| max_odirs: Maximum output directories to maintain as history. | |
| Returns: | |
| dirs: Space-separated list of directories that remain after deletion. | |
| ''' | |
| try: | |
| # If output directory exists, back it up. | |
| if os.path.exists(odir): | |
| ts = run_cmd("date '+" + self.sim_cfg.ts_format + "' -d \"" + | |
| "$(stat -c '%y' " + odir + ")\"") | |
| os.system('mv ' + odir + " " + odir + "_" + ts) | |
| except IOError: | |
| log.error('Failed to back up existing output directory %s', odir) | |
| dirs = "" | |
| # Delete older directories. | |
| try: | |
| pdir = os.path.realpath(odir + "/..") | |
| # Fatal out if pdir got set to root. | |
| if pdir == "/": | |
| log.fatal( | |
| "Something went wrong while processing \"%s\": odir = \"%s\"", | |
| self.name, odir) | |
| sys.exit(1) | |
| if os.path.exists(pdir): | |
| find_cmd = "find " + pdir + " -mindepth 1 -maxdepth 1 -type d " | |
| dirs = run_cmd(find_cmd) | |
| dirs = dirs.replace('\n', ' ') | |
| list_dirs = dirs.split() | |
| num_dirs = len(list_dirs) | |
| if max_odirs == -1: | |
| max_odirs = self.max_odirs | |
| num_rm_dirs = num_dirs - max_odirs | |
| if num_rm_dirs > -1: | |
| rm_dirs = run_cmd(find_cmd + | |
| "-printf '%T+ %p\n' | sort | head -n " + | |
| str(num_rm_dirs + 1) + | |
| " | awk '{print $2}'") | |
| rm_dirs = rm_dirs.replace('\n', ' ') | |
| dirs = dirs.replace(rm_dirs, "") | |
| os.system("/bin/rm -rf " + rm_dirs) | |
| except IOError: | |
| log.error("Failed to delete old run directories!") | |
| return dirs | |
| def set_status(self): | |
| self.status = 'P' | |
| if self.dry_run is False: | |
| seen_fail_pattern = False | |
| for fail_pattern in self.fail_patterns: | |
| # Return error message with the following 4 lines. | |
| grep_cmd = "grep -m 1 -A 4 -E \'" + fail_pattern + "\' " + self.log | |
| (status, rslt) = subprocess.getstatusoutput(grep_cmd) | |
| if rslt: | |
| msg = "```\n{}\n```\n".format(rslt) | |
| self.fail_msg += msg | |
| log.log(VERBOSE, msg) | |
| self.status = 'F' | |
| seen_fail_pattern = True | |
| break | |
| # If fail patterns were not encountered, but the job returned with non-zero exit code | |
| # for whatever reason, then show the last 10 lines of the log as the failure message, | |
| # which might help with the debug. | |
| if self.process.returncode != 0 and not seen_fail_pattern: | |
| msg = "Last 10 lines of the log:<br>\n" | |
| self.fail_msg += msg | |
| log.log(VERBOSE, msg) | |
| get_fail_msg_cmd = "tail -n 10 " + self.log | |
| msg = run_cmd(get_fail_msg_cmd) | |
| msg = "```\n{}\n```\n".format(msg) | |
| self.fail_msg += msg | |
| log.log(VERBOSE, msg) | |
| self.status = "F" | |
| # Return if status is fail - no need to look for pass patterns. | |
| if self.status == 'F': | |
| return | |
| # If fail patterns were not found, ensure pass patterns indeed were. | |
| for pass_pattern in self.pass_patterns: | |
| grep_cmd = "grep -c -m 1 -E \'" + pass_pattern + "\' " + self.log | |
| (status, rslt) = subprocess.getstatusoutput(grep_cmd) | |
| if rslt == "0": | |
| msg = "Pass pattern \"{}\" not found.<br>\n".format( | |
| pass_pattern) | |
| self.fail_msg += msg | |
| log.log(VERBOSE, msg) | |
| self.status = 'F' | |
| break | |
| # Recursively set sub-item's status if parent item fails | |
| def set_sub_status(self, status): | |
| for sub_item in self.sub: | |
| sub_item.status = status | |
| sub_item.set_sub_status(status) | |
| def link_odir(self): | |
| if self.status == '.': | |
| log.error("Method unexpectedly called!") | |
| else: | |
| old_link = self.sim_cfg.links['D'] + "/" + self.odir_ln | |
| new_link = self.sim_cfg.links[self.status] + "/" + self.odir_ln | |
| cmd = "ln -s " + self.odir + " " + new_link + "; " | |
| cmd += "rm " + old_link | |
| if os.system(cmd): | |
| log.error("Cmd \"%s\" could not be run", cmd) | |
| def get_status(self): | |
| if self.status != "D": | |
| return | |
| if self.process.poll() is not None: | |
| self.log_fd.close() | |
| self.set_status() | |
| log.debug("Item %s has completed execution: %s", self.name, | |
| self.status) | |
| Deploy.dispatch_counter -= 1 | |
| self.link_odir() | |
| del self.process | |
| def kill(self): | |
| '''Kill running processes. | |
| ''' | |
| if self.status == "D" and self.process.poll() is None: | |
| self.kill_remote_job() | |
| self.process.kill() | |
| if self.log_fd: | |
| self.log_fd.close() | |
| self.status = "K" | |
| # recurisvely kill sub target | |
| elif len(self.sub): | |
| for item in self.sub: | |
| item.kill() | |
| def kill_remote_job(self): | |
| ''' | |
| If jobs are run in remote server, need to use another command to kill them. | |
| ''' | |
| # TODO: Currently only support lsf, may need to add support for GCP later. | |
| # If use lsf, kill it by job ID. | |
| if re.match("^bsub", self.sim_cfg.job_prefix): | |
| # get job id from below string | |
| # Job <xxxxxx> is submitted to default queue | |
| grep_cmd = "grep -m 1 -E \'" + "^Job <" + "\' " + self.log | |
| (status, rslt) = subprocess.getstatusoutput(grep_cmd) | |
| if rslt != "": | |
| job_id = rslt.split('Job <')[1].split('>')[0] | |
| try: | |
| subprocess.run(["bkill", job_id], check=True) | |
| except Exception as e: | |
| log.error("%s: Failed to run bkill\n", e) | |
| @staticmethod | |
| def increment_timer(): | |
| # sub function that increments with overflow = 60 | |
| def _incr_ovf_60(val): | |
| if val >= 59: | |
| val = 0 | |
| return val, True | |
| else: | |
| val += 1 | |
| return val, False | |
| incr_hh = False | |
| Deploy.ss, incr_mm = _incr_ovf_60(Deploy.ss) | |
| if incr_mm: | |
| Deploy.mm, incr_hh = _incr_ovf_60(Deploy.mm) | |
| if incr_hh: | |
| Deploy.hh += 1 | |
| @staticmethod | |
| def deploy(items): | |
| dispatched_items = [] | |
| queued_items = [] | |
| # Print timer val in hh:mm:ss. | |
| def get_timer_val(): | |
| return "%02i:%02i:%02i" % (Deploy.hh, Deploy.mm, Deploy.ss) | |
| # Check if elapsed time has reached the next print interval. | |
| def has_print_interval_reached(): | |
| # Deploy.print_interval is expected to be < 1 hour. | |
| return (((Deploy.mm * 60 + Deploy.ss) % | |
| Deploy.print_interval) == 0) | |
| def dispatch_items(items): | |
| item_names = OrderedDict() | |
| for item in items: | |
| if item.target not in item_names.keys(): | |
| item_names[item.target] = "" | |
| if item.status is None: | |
| item_names[item.target] += item.identifier + ", " | |
| item.dispatch_cmd() | |
| for target in item_names.keys(): | |
| if item_names[target] != "": | |
| item_names[target] = " [" + item_names[target][:-2] + "]" | |
| log.log(VERBOSE, "[%s]: [%s]: [dispatch]:\n%s", | |
| get_timer_val(), target, item_names[target]) | |
| # Initialize status for a target, add '_stats_' for the said target | |
| # and initialize counters for queued, dispatched, passed, failed, | |
| # killed and total to 0. Also adds a boolean key to indicate if all | |
| # items in a given target are done. | |
| def init_status_target_stats(status, target): | |
| status[target] = OrderedDict() | |
| status[target]['_stats_'] = OrderedDict() | |
| status[target]['_stats_']['Q'] = 0 | |
| status[target]['_stats_']['D'] = 0 | |
| status[target]['_stats_']['P'] = 0 | |
| status[target]['_stats_']['F'] = 0 | |
| status[target]['_stats_']['K'] = 0 | |
| status[target]['_stats_']['T'] = 0 | |
| status[target]['_done_'] = False | |
| # Update status counter for a newly queued item. | |
| def add_status_target_queued(status, item): | |
| if item.target not in status.keys(): | |
| init_status_target_stats(status, item.target) | |
| status[item.target][item] = "Q" | |
| status[item.target]['_stats_']['Q'] += 1 | |
| status[item.target]['_stats_']['T'] += 1 | |
| # Update status counters for a target. | |
| def update_status_target_stats(status, item): | |
| old_status = status[item.target][item] | |
| status[item.target]['_stats_'][old_status] -= 1 | |
| status[item.target]['_stats_'][item.status] += 1 | |
| status[item.target][item] = item.status | |
| def check_if_done_and_print_status(status, print_status_flag): | |
| all_done = True | |
| for target in status.keys(): | |
| target_done_prev = status[target]['_done_'] | |
| target_done_curr = ((status[target]['_stats_']["Q"] == 0) and | |
| (status[target]['_stats_']["D"] == 0)) | |
| status[target]['_done_'] = target_done_curr | |
| all_done &= target_done_curr | |
| # Print if flag is set and target_done is not True for two | |
| # consecutive times. | |
| if not (target_done_prev and | |
| target_done_curr) and print_status_flag: | |
| stats = status[target]['_stats_'] | |
| width = "0{}d".format(len(str(stats["T"]))) | |
| msg = "[" | |
| for s in stats.keys(): | |
| msg += s + ": {:{}}, ".format(stats[s], width) | |
| msg = msg[:-2] + "]" | |
| log.info("[%s]: [%s]: %s", get_timer_val(), target, msg) | |
| return all_done | |
| # Print legend once at the start of the run. | |
| if Deploy.print_legend: | |
| log.info("[legend]: [Q: queued, D: dispatched, " | |
| "P: passed, F: failed, K: killed, T: total]") | |
| Deploy.print_legend = False | |
| status = OrderedDict() | |
| print_status_flag = True | |
| # Queue all items | |
| queued_items = items | |
| for item in queued_items: | |
| add_status_target_queued(status, item) | |
| all_done = False | |
| while not all_done: | |
| # Get status of dispatched items. | |
| for item in dispatched_items: | |
| if item.status == "D": | |
| item.get_status() | |
| if item.status != status[item.target][item]: | |
| print_status_flag = True | |
| if item.status != "D": | |
| if item.status != "P": | |
| # Kill its sub items if item did not pass. | |
| item.set_sub_status("K") | |
| log.error("[%s]: [%s]: [status] [%s: %s]", | |
| get_timer_val(), item.target, | |
| item.identifier, item.status) | |
| else: | |
| log.log(VERBOSE, "[%s]: [%s]: [status] [%s: %s]", | |
| get_timer_val(), item.target, | |
| item.identifier, item.status) | |
| # Queue items' sub-items if it is done. | |
| queued_items.extend(item.sub) | |
| for sub_item in item.sub: | |
| add_status_target_queued(status, sub_item) | |
| update_status_target_stats(status, item) | |
| # Dispatch items from the queue as slots free up. | |
| all_done = (len(queued_items) == 0) | |
| if not all_done: | |
| num_slots = Deploy.max_parallel - Deploy.dispatch_counter | |
| if num_slots > Deploy.slot_limit: | |
| num_slots = Deploy.slot_limit | |
| if num_slots > 0: | |
| if len(queued_items) > num_slots: | |
| dispatch_items(queued_items[0:num_slots]) | |
| dispatched_items.extend(queued_items[0:num_slots]) | |
| queued_items = queued_items[num_slots:] | |
| else: | |
| dispatch_items(queued_items) | |
| dispatched_items.extend(queued_items) | |
| queued_items = [] | |
| # Check if we are done and print the status periodically. | |
| all_done &= check_if_done_and_print_status(status, | |
| print_status_flag) | |
| # Advance time by 1s if there is more work to do. | |
| if not all_done: | |
| time.sleep(1) | |
| Deploy.increment_timer() | |
| print_status_flag = has_print_interval_reached() | |
| class CompileSim(Deploy): | |
| """ | |
| Abstraction for building the simulation executable. | |
| """ | |
| # Register all builds with the class | |
| items = [] | |
| def __init__(self, build_mode, sim_cfg): | |
| # Initialize common vars. | |
| super().__init__(sim_cfg) | |
| self.target = "build" | |
| self.pass_patterns = [] | |
| self.fail_patterns = [] | |
| self.mandatory_cmd_attrs.update({ | |
| # tool srcs | |
| "tool_srcs": False, | |
| "tool_srcs_dir": False, | |
| # Flist gen | |
| "sv_flist_gen_cmd": False, | |
| "sv_flist_gen_dir": False, | |
| "sv_flist_gen_opts": False, | |
| # Build | |
| "build_dir": False, | |
| "build_cmd": False, | |
| "build_opts": False | |
| }) | |
| self.mandatory_misc_attrs.update({ | |
| "cov_db_dir": False, | |
| "build_pass_patterns": False, | |
| "build_fail_patterns": False | |
| }) | |
| super().parse_dict(build_mode.__dict__) | |
| # Call this method again with the sim_cfg dict passed as the object, | |
| # since it may contain additional mandatory attrs. | |
| super().parse_dict(sim_cfg.__dict__) | |
| self.build_mode = self.name | |
| self.pass_patterns = self.build_pass_patterns | |
| self.fail_patterns = self.build_fail_patterns | |
| self.__post_init__() | |
| # Start fail message construction | |
| self.fail_msg = "\n**BUILD:** {}<br>\n".format(self.name) | |
| log_sub_path = self.log.replace(self.sim_cfg.scratch_path + '/', '') | |
| self.fail_msg += "**LOG:** $scratch_path/{}<br>\n".format(log_sub_path) | |
| CompileSim.items.append(self) | |
| def dispatch_cmd(self): | |
| # Delete previous cov_db_dir if it exists before dispatching new build. | |
| if os.path.exists(self.cov_db_dir): | |
| os.system("rm -rf " + self.cov_db_dir) | |
| super().dispatch_cmd() | |
| class CompileOneShot(Deploy): | |
| """ | |
| Abstraction for building the simulation executable. | |
| """ | |
| # Register all builds with the class | |
| items = [] | |
| def __init__(self, build_mode, sim_cfg): | |
| # Initialize common vars. | |
| super().__init__(sim_cfg) | |
| self.target = "build" | |
| self.pass_patterns = [] | |
| self.fail_patterns = [] | |
| self.mandatory_cmd_attrs.update({ | |
| # tool srcs | |
| "tool_srcs": False, | |
| "tool_srcs_dir": False, | |
| # Flist gen | |
| "sv_flist_gen_cmd": False, | |
| "sv_flist_gen_dir": False, | |
| "sv_flist_gen_opts": False, | |
| # Build | |
| "build_dir": False, | |
| "build_cmd": False, | |
| "build_opts": False, | |
| "build_log": False, | |
| # Report processing | |
| "report_cmd": False, | |
| "report_opts": False | |
| }) | |
| super().parse_dict(build_mode.__dict__) | |
| # Call this method again with the sim_cfg dict passed as the object, | |
| # since it may contain additional mandatory attrs. | |
| super().parse_dict(sim_cfg.__dict__) | |
| self.build_mode = self.name | |
| self.__post_init__() | |
| # Start fail message construction | |
| self.fail_msg = "\n**BUILD:** {}<br>\n".format(self.name) | |
| log_sub_path = self.log.replace(self.sim_cfg.scratch_path + '/', '') | |
| self.fail_msg += "**LOG:** $scratch_path/{}<br>\n".format(log_sub_path) | |
| CompileOneShot.items.append(self) | |
| class RunTest(Deploy): | |
| """ | |
| Abstraction for running tests. This is one per seed for each test. | |
| """ | |
| # Initial seed values when running tests (if available). | |
| seeds = [] | |
| fixed_seed = None | |
| # Register all runs with the class | |
| items = [] | |
| def __init__(self, index, test, sim_cfg): | |
| # Initialize common vars. | |
| super().__init__(sim_cfg) | |
| self.target = "run" | |
| self.pass_patterns = [] | |
| self.fail_patterns = [] | |
| self.mandatory_cmd_attrs.update({ | |
| "proj_root": False, | |
| "uvm_test": False, | |
| "uvm_test_seq": False, | |
| "run_opts": False, | |
| "sw_test": False, | |
| "sw_test_is_prebuilt": False, | |
| "sw_build_device": False, | |
| "sw_build_dir": False, | |
| "run_dir": False, | |
| "run_cmd": False, | |
| "run_opts": False | |
| }) | |
| self.mandatory_misc_attrs.update({ | |
| "run_dir_name": False, | |
| "cov_db_test_dir": False, | |
| "run_pass_patterns": False, | |
| "run_fail_patterns": False | |
| }) | |
| self.index = index | |
| self.seed = RunTest.get_seed() | |
| super().parse_dict(test.__dict__) | |
| # Call this method again with the sim_cfg dict passed as the object, | |
| # since it may contain additional mandatory attrs. | |
| super().parse_dict(sim_cfg.__dict__) | |
| self.test = self.name | |
| self.renew_odir = True | |
| self.build_mode = test.build_mode.name | |
| self.pass_patterns = self.run_pass_patterns | |
| self.fail_patterns = self.run_fail_patterns | |
| self.__post_init__() | |
| # For output dir link, use run_dir_name instead. | |
| self.odir_ln = self.run_dir_name | |
| # Start fail message construction | |
| self.fail_msg = "\n**TEST:** {}, ".format(self.name) | |
| self.fail_msg += "**SEED:** {}<br>\n".format(self.seed) | |
| log_sub_path = self.log.replace(self.sim_cfg.scratch_path + '/', '') | |
| self.fail_msg += "**LOG:** $scratch_path/{}<br>\n".format(log_sub_path) | |
| RunTest.items.append(self) | |
| def __post_init__(self): | |
| super().__post_init__() | |
| # Set identifier. | |
| self.identifier = self.sim_cfg.name + ":" + self.run_dir_name | |
| def get_status(self): | |
| '''Override base class get_status implementation for additional post-status | |
| actions.''' | |
| super().get_status() | |
| if self.status not in ["D", "P"]: | |
| # Delete the coverage data if available. | |
| if os.path.exists(self.cov_db_test_dir): | |
| log.log(VERBOSE, "Deleting coverage data of failing test:\n%s", | |
| self.cov_db_test_dir) | |
| os.system("/bin/rm -rf " + self.cov_db_test_dir) | |
| @staticmethod | |
| def get_seed(): | |
| # If --seeds option is passed, then those custom seeds are consumed | |
| # first. If --fixed-seed <val> is also passed, the subsequent tests | |
| # (once the custom seeds are consumed) will be run with the fixed seed. | |
| if not RunTest.seeds: | |
| if RunTest.fixed_seed: | |
| return RunTest.fixed_seed | |
| for i in range(1000): | |
| seed = random.getrandbits(32) | |
| RunTest.seeds.append(seed) | |
| return RunTest.seeds.pop(0) | |
| class CovMerge(Deploy): | |
| """ | |
| Abstraction for merging coverage databases. An item of this class is created AFTER | |
| the regression is completed. | |
| """ | |
| # Register all builds with the class | |
| items = [] | |
| def __init__(self, sim_cfg): | |
| # Initialize common vars. | |
| super().__init__(sim_cfg) | |
| self.target = "cov_merge" | |
| self.pass_patterns = [] | |
| self.fail_patterns = [] | |
| # Construct local 'special' variable from cov directories that need to | |
| # be merged. | |
| self.cov_db_dirs = "" | |
| self.mandatory_cmd_attrs.update({ | |
| "cov_merge_cmd": False, | |
| "cov_merge_opts": False | |
| }) | |
| self.mandatory_misc_attrs.update({ | |
| "cov_merge_dir": False, | |
| "cov_merge_db_dir": False | |
| }) | |
| super().parse_dict(sim_cfg.__dict__) | |
| self.__post_init__() | |
| # Override standard output and log patterns. | |
| self.odir = self.cov_merge_db_dir | |
| self.odir_ln = os.path.basename(os.path.normpath(self.odir)) | |
| # Start fail message construction | |
| self.fail_msg = "\n**COV_MERGE:** {}<br>\n".format(self.name) | |
| log_sub_path = self.log.replace(self.sim_cfg.scratch_path + '/', '') | |
| self.fail_msg += "**LOG:** $scratch_path/{}<br>\n".format(log_sub_path) | |
| CovMerge.items.append(self) | |
| def __post_init__(self): | |
| # Add cov db dirs from all the builds that were kicked off. | |
| for bld in self.sim_cfg.builds: | |
| self.cov_db_dirs += bld.cov_db_dir + " " | |
| # Recursively search and replace wildcards, ignoring cov_db_dirs. | |
| # We need to resolve it later based on cov_db_dirs value set below. | |
| # First pass: search within self dict. We ignore errors since some | |
| # substitions may be available in the second pass. | |
| self.__dict__ = find_and_substitute_wildcards( | |
| self.__dict__, | |
| self.__dict__, | |
| ignored_wildcards=["cov_db_dirs"], | |
| ignore_error=True) | |
| # Second pass: search in sim_cfg dict, this time not ignoring errors. | |
| self.__dict__ = find_and_substitute_wildcards( | |
| self.__dict__, | |
| self.sim_cfg.__dict__, | |
| ignored_wildcards=["cov_db_dirs"], | |
| ignore_error=False) | |
| # Prune previous merged cov directories. | |
| prev_cov_db_dirs = self.odir_limiter(odir=self.cov_merge_db_dir) | |
| # If a merged cov data base exists from a previous run, then consider | |
| # that as well for merging, if the --cov-merge-previous command line | |
| # switch is passed. | |
| if self.sim_cfg.cov_merge_previous: | |
| self.cov_db_dirs += prev_cov_db_dirs | |
| # Append cov_db_dirs to the list of exports. | |
| self.exports["cov_db_dirs"] = "\"{}\"".format(self.cov_db_dirs) | |
| # Call base class __post_init__ to do checks and substitutions | |
| super().__post_init__() | |
| class CovReport(Deploy): | |
| """ | |
| Abstraction for coverage report generation. An item of this class is created AFTER | |
| the regression is completed. | |
| """ | |
| # Register all builds with the class | |
| items = [] | |
| def __init__(self, sim_cfg): | |
| # Initialize common vars. | |
| super().__init__(sim_cfg) | |
| self.target = "cov_report" | |
| self.pass_patterns = [] | |
| self.fail_patterns = [] | |
| self.cov_total = "" | |
| self.cov_results = "" | |
| self.mandatory_cmd_attrs.update({ | |
| "cov_report_cmd": False, | |
| "cov_report_opts": False | |
| }) | |
| self.mandatory_misc_attrs.update({ | |
| "cov_report_dir": False, | |
| "cov_merge_db_dir": False, | |
| "cov_report_txt": False | |
| }) | |
| super().parse_dict(sim_cfg.__dict__) | |
| self.__post_init__() | |
| # Start fail message construction | |
| self.fail_msg = "\n**COV_REPORT:** {}<br>\n".format(self.name) | |
| log_sub_path = self.log.replace(self.sim_cfg.scratch_path + '/', '') | |
| self.fail_msg += "**LOG:** $scratch_path/{}<br>\n".format(log_sub_path) | |
| CovReport.items.append(self) | |
| def get_status(self): | |
| super().get_status() | |
| # Once passed, extract the cov results summary from the dashboard. | |
| if self.status == "P": | |
| results, self.cov_total, ex_msg = get_cov_summary_table( | |
| self.cov_report_txt, self.sim_cfg.tool) | |
| if not ex_msg: | |
| # Succeeded in obtaining the coverage data. | |
| colalign = (("center", ) * len(results[0])) | |
| self.cov_results = tabulate(results, | |
| headers="firstrow", | |
| tablefmt="pipe", | |
| colalign=colalign) | |
| else: | |
| self.fail_msg += ex_msg | |
| log.error(ex_msg) | |
| self.status = "F" | |
| if self.status == "P": | |
| # Delete the cov report - not needed. | |
| os.system("rm -rf " + self.log) | |
| class CovAnalyze(Deploy): | |
| """ | |
| Abstraction for coverage analysis tool. | |
| """ | |
| # Register all builds with the class | |
| items = [] | |
| def __init__(self, sim_cfg): | |
| # Initialize common vars. | |
| super().__init__(sim_cfg) | |
| self.target = "cov_analyze" | |
| self.pass_patterns = [] | |
| self.fail_patterns = [] | |
| self.mandatory_cmd_attrs.update({ | |
| # tool srcs | |
| "tool_srcs": False, | |
| "tool_srcs_dir": False, | |
| "cov_analyze_cmd": False, | |
| "cov_analyze_opts": False | |
| }) | |
| self.mandatory_misc_attrs.update({ | |
| "cov_analyze_dir": False, | |
| "cov_merge_db_dir": False | |
| }) | |
| super().parse_dict(sim_cfg.__dict__) | |
| self.__post_init__() | |
| # Start fail message construction | |
| self.fail_msg = "\n**COV_ANALYZE:** {}<br>\n".format(self.name) | |
| log_sub_path = self.log.replace(self.sim_cfg.scratch_path + '/', '') | |
| self.fail_msg += "**LOG:** $scratch_path/{}<br>\n".format(log_sub_path) | |
| CovAnalyze.items.append(self) |