| # Copyright lowRISC contributors. | |
| # Licensed under the Apache License, Version 2.0, see LICENSE for details. | |
| # SPDX-License-Identifier: Apache-2.0 | |
| r""" | |
| Utility functions common across dvsim. | |
| """ | |
| import logging as log | |
| import os | |
| import re | |
| import shlex | |
| import shutil | |
| import subprocess | |
| import sys | |
| import time | |
| from collections import OrderedDict | |
| from datetime import datetime | |
| from pathlib import Path | |
| import hjson | |
| import mistletoe | |
| from premailer import transform | |
| # For verbose logging | |
| VERBOSE = 15 | |
| # Timestamp format when creating directory backups. | |
| TS_FORMAT = "%y.%m.%d_%H.%M.%S" | |
| # Timestamp format when generating reports. | |
| TS_FORMAT_LONG = "%A %B %d %Y %H:%M:%S UTC" | |
| # Run a command and get the result. Exit with error if the command did not | |
| # succeed. This is a simpler version of the run_cmd function below. | |
| def run_cmd(cmd): | |
| (status, output) = subprocess.getstatusoutput(cmd) | |
| if status: | |
| print(f'cmd {cmd} returned with status {status}', file=sys.stderr) | |
| sys.exit(status) | |
| return output | |
| # Run a command with a specified timeout. If the command does not finish before | |
| # the timeout, then it returns -1. Else it returns the command output. If the | |
| # command fails, it throws an exception and returns the stderr. | |
| def run_cmd_with_timeout(cmd, timeout=-1, exit_on_failure=1): | |
| args = shlex.split(cmd) | |
| p = subprocess.Popen(args, | |
| stdout=subprocess.PIPE, | |
| stderr=subprocess.STDOUT) | |
| # If timeout is set, poll for the process to finish until timeout | |
| result = "" | |
| status = -1 | |
| if timeout == -1: | |
| p.wait() | |
| else: | |
| start = time.time() | |
| while time.time() - start < timeout: | |
| if p.poll() is not None: | |
| break | |
| time.sleep(.01) | |
| # Capture output and status if cmd exited, else kill it | |
| if p.poll() is not None: | |
| result = p.communicate()[0] | |
| status = p.returncode | |
| else: | |
| log.error("cmd \"%s\" timed out!", cmd) | |
| p.kill() | |
| if status != 0: | |
| log.error("cmd \"%s\" exited with status %d", cmd, status) | |
| if exit_on_failure == 1: | |
| sys.exit(status) | |
| return (result, status) | |
| # Parse hjson and return a dict | |
| def parse_hjson(hjson_file): | |
| hjson_cfg_dict = None | |
| try: | |
| log.debug("Parsing %s", hjson_file) | |
| f = open(hjson_file, 'rU') | |
| text = f.read() | |
| hjson_cfg_dict = hjson.loads(text, use_decimal=True) | |
| f.close() | |
| except Exception as e: | |
| log.fatal( | |
| "Failed to parse \"%s\" possibly due to bad path or syntax error.\n%s", | |
| hjson_file, e) | |
| sys.exit(1) | |
| return hjson_cfg_dict | |
| def _stringify_wildcard_value(value): | |
| '''Make sense of a wildcard value as a string (see subst_wildcards) | |
| Strings are passed through unchanged. Integer or boolean values are printed | |
| as numerical strings. Lists or other sequences have their items printed | |
| separated by spaces. | |
| ''' | |
| if type(value) is str: | |
| return value | |
| if type(value) in [bool, int]: | |
| return str(int(value)) | |
| try: | |
| return ' '.join(_stringify_wildcard_value(x) for x in value) | |
| except TypeError: | |
| raise ValueError('Wildcard had value {!r} which is not of a supported ' | |
| 'type.'.format(value)) | |
| def _subst_wildcards(var, mdict, ignored, ignore_error, seen): | |
| '''Worker function for subst_wildcards | |
| seen is a list of wildcards that have been expanded on the way to this call | |
| (used for spotting circular recursion). | |
| Returns (expanded, seen_err) where expanded is the new value of the string | |
| and seen_err is true if we stopped early because of an ignored error. | |
| ''' | |
| wildcard_re = re.compile(r"{([A-Za-z0-9\_]+)}") | |
| # Work from left to right, expanding each wildcard we find. idx is where we | |
| # should start searching (so that we don't keep finding a wildcard that | |
| # we've decided to ignore). | |
| idx = 0 | |
| any_err = False | |
| while True: | |
| right_str = var[idx:] | |
| match = wildcard_re.search(right_str) | |
| # If no match, we're done. | |
| if match is None: | |
| return (var, any_err) | |
| name = match.group(1) | |
| # If the name should be ignored, skip over it. | |
| if name in ignored: | |
| idx += match.end() | |
| continue | |
| # If the name has been seen already, we've spotted circular recursion. | |
| # That's not allowed! | |
| if name in seen: | |
| raise ValueError('String contains circular expansion of ' | |
| 'wildcard {!r}.'.format(match.group(0))) | |
| # Treat eval_cmd specially | |
| if name == 'eval_cmd': | |
| cmd = _subst_wildcards(right_str[match.end():], mdict, ignored, | |
| ignore_error, seen)[0] | |
| # Are there any wildcards left in cmd? If not, we can run the | |
| # command and we're done. | |
| cmd_matches = list(wildcard_re.finditer(cmd)) | |
| if not cmd_matches: | |
| var = var[:match.start()] + run_cmd(cmd) | |
| continue | |
| # Otherwise, check that each of them is ignored, or that | |
| # ignore_error is True. | |
| bad_names = False | |
| if not ignore_error: | |
| for cmd_match in cmd_matches: | |
| if cmd_match.group(1) not in ignored: | |
| bad_names = True | |
| if bad_names: | |
| raise ValueError('Cannot run eval_cmd because the command ' | |
| 'expands to {!r}, which still contains a ' | |
| 'wildcard.'.format(cmd)) | |
| # We can't run the command (because it still has wildcards), but we | |
| # don't want to report an error either because ignore_error is true | |
| # or because each wildcard that's left is ignored. Return the | |
| # partially evaluated version. | |
| return (var[:idx] + right_str[:match.end()] + cmd, True) | |
| # Otherwise, look up name in mdict. | |
| value = mdict.get(name) | |
| # If the value isn't set, check the environment | |
| if value is None: | |
| value = os.environ.get(name) | |
| if value is None: | |
| # Ignore missing values if ignore_error is True. | |
| if ignore_error: | |
| idx += match.end() | |
| continue | |
| raise ValueError('String to be expanded contains ' | |
| 'unknown wildcard, {!r}.'.format(match.group(0))) | |
| value = _stringify_wildcard_value(value) | |
| # Do any recursive expansion of value, adding name to seen (to avoid | |
| # circular recursion). | |
| value, saw_err = _subst_wildcards(value, mdict, ignored, ignore_error, | |
| seen + [name]) | |
| # Replace the original match with the result and go around again. If | |
| # saw_err, increment idx past what we just inserted. | |
| var = (var[:idx] + right_str[:match.start()] + value + | |
| right_str[match.end():]) | |
| if saw_err: | |
| any_err = True | |
| idx += match.start() + len(value) | |
| def subst_wildcards(var, mdict, ignored_wildcards=[], ignore_error=False): | |
| '''Substitute any "wildcard" variables in the string var. | |
| var is the string to be substituted. mdict is a dictionary mapping | |
| variables to strings. ignored_wildcards is a list of wildcards that | |
| shouldn't be substituted. ignore_error means to partially evaluate rather | |
| than exit on an error. | |
| A wildcard is written as a name (alphanumeric, allowing backslash and | |
| underscores) surrounded by braces. For example, | |
| subst_wildcards('foo {x} baz', {'x': 'bar'}) | |
| returns "foo bar baz". Dictionary values can be strings, booleans, integers | |
| or lists. For example: | |
| subst_wildcards('{a}, {b}, {c}, {d}', | |
| {'a': 'a', 'b': True, 'c': 42, 'd': ['a', 10]}) | |
| returns 'a, 1, 42, a 10'. | |
| If a wildcard is in ignored_wildcards, it is ignored. For example, | |
| subst_wildcards('{a} {b}', {'b': 'bee'}, ignored_wildcards=['a']) | |
| returns "{a} bee". | |
| If a wildcard appears in var but is not in mdict, the environment is | |
| checked for the variable. If the name still isn't found, the default | |
| behaviour is to log an error and exit. If ignore_error is True, the | |
| wildcard is ignored (as if it appeared in ignore_wildcards). | |
| If {eval_cmd} appears in the string and 'eval_cmd' is not in | |
| ignored_wildcards then the following text is recursively expanded. The | |
| result of this expansion is treated as a command to run and the text is | |
| replaced by the output of the command. | |
| If a wildcard has been ignored (either because of ignored_wildcards or | |
| ignore_error), the command to run in eval_cmd might contain a match for | |
| wildcard_re. If ignore_error is True, the command is not run. So | |
| subst_wildcards('{eval_cmd}{foo}', {}, ignore_error=True) | |
| will return '{eval_cmd}{foo}' unchanged. If ignore_error is False, the | |
| function logs an error and exits. | |
| Recursion is possible in subst_wildcards. For example, | |
| subst_wildcards('{a}', {'a': '{b}', 'b': 'c'}) | |
| returns 'c'. Circular recursion is detected, however. So | |
| subst_wildcards('{a}', {'a': '{b}', 'b': '{a}'}) | |
| will log an error and exit. This error is raised whether or not | |
| ignore_error is set. | |
| Since subst_wildcards works from left to right, it's possible to compute | |
| wildcard names with code like this: | |
| subst_wildcards('{a}b}', {'a': 'a {', 'b': 'bee'}) | |
| which returns 'a bee'. This is pretty hard to read though, so is probably | |
| not a good idea to use. | |
| ''' | |
| try: | |
| return _subst_wildcards(var, mdict, ignored_wildcards, ignore_error, | |
| [])[0] | |
| except ValueError as err: | |
| log.error(str(err)) | |
| sys.exit(1) | |
| def find_and_substitute_wildcards(sub_dict, | |
| full_dict, | |
| ignored_wildcards=[], | |
| ignore_error=False): | |
| ''' | |
| Recursively find key values containing wildcards in sub_dict in full_dict | |
| and return resolved sub_dict. | |
| ''' | |
| for key in sub_dict.keys(): | |
| if type(sub_dict[key]) in [dict, OrderedDict]: | |
| # Recursively call this funciton in sub-dicts | |
| sub_dict[key] = find_and_substitute_wildcards( | |
| sub_dict[key], full_dict, ignored_wildcards, ignore_error) | |
| elif type(sub_dict[key]) is list: | |
| sub_dict_key_values = list(sub_dict[key]) | |
| # Loop through the list of key's values and substitute each var | |
| # in case it contains a wildcard | |
| for i in range(len(sub_dict_key_values)): | |
| if type(sub_dict_key_values[i]) in [dict, OrderedDict]: | |
| # Recursively call this funciton in sub-dicts | |
| sub_dict_key_values[i] = \ | |
| find_and_substitute_wildcards(sub_dict_key_values[i], | |
| full_dict, ignored_wildcards, ignore_error) | |
| elif type(sub_dict_key_values[i]) is str: | |
| sub_dict_key_values[i] = subst_wildcards( | |
| sub_dict_key_values[i], full_dict, ignored_wildcards, | |
| ignore_error) | |
| # Set the substituted key values back | |
| sub_dict[key] = sub_dict_key_values | |
| elif type(sub_dict[key]) is str: | |
| sub_dict[key] = subst_wildcards(sub_dict[key], full_dict, | |
| ignored_wildcards, ignore_error) | |
| return sub_dict | |
| def md_results_to_html(title, css_file, md_text): | |
| '''Convert results in md format to html. Add a little bit of styling. | |
| ''' | |
| html_text = "<!DOCTYPE html>\n" | |
| html_text += "<html lang=\"en\">\n" | |
| html_text += "<head>\n" | |
| if title != "": | |
| html_text += " <title>{}</title>\n".format(title) | |
| html_text += "</head>\n" | |
| html_text += "<body>\n" | |
| html_text += "<div class=\"results\">\n" | |
| html_text += mistletoe.markdown(md_text) | |
| html_text += "</div>\n" | |
| html_text += "</body>\n" | |
| html_text += "</html>\n" | |
| html_text = htmc_color_pc_cells(html_text) | |
| # this function converts css style to inline html style | |
| html_text = transform(html_text, | |
| external_styles=css_file, | |
| cssutils_logging_level=log.ERROR) | |
| return html_text | |
| def htmc_color_pc_cells(text): | |
| '''This function finds cells in a html table that contain numerical values | |
| (and a few known strings) followed by a single space and an identifier. | |
| Depending on the identifier, it shades the cell in a specific way. A set of | |
| 12 color palettes for setting those shades are encoded in ./style.css. | |
| These are 'cna' (grey), 'c0' (red), 'c1' ... 'c10' (green). The shade 'cna' | |
| is used for items that are maked as 'not applicable'. The shades 'c1' to | |
| 'c9' form a gradient from red to lime-green to indicate 'levels of | |
| completeness'. 'cna' is used for greying out a box for 'not applicable' | |
| items, 'c0' is for items that are considered risky (or not yet started) and | |
| 'c10' for items that have completed successfully, or that are | |
| 'in good standing'. | |
| These are the supported identifiers: %, %u, G, B, E, W, EN, WN. | |
| The shading behavior for these is described below. | |
| %: Coloured percentage, where the number in front of the '%' sign is mapped | |
| to a color for the cell ranging from red ('c0') to green ('c10'). | |
| %u: Uncoloured percentage, where no markup is applied and '%u' is replaced | |
| with '%' in the output. | |
| G: This stands for 'Good' and results in a green cell. | |
| B: This stands for 'Bad' and results in a red cell. | |
| E: This stands for 'Errors' and the cell is colored with red if the number | |
| in front of the indicator is larger than 0. Otherwise the cell is | |
| colored with green. | |
| W: This stands for 'Warnings' and the cell is colored with yellow ('c6') | |
| if the number in front of the indicator is larger than 0. Otherwise | |
| the cell is colored with green. | |
| EN: This stands for 'Errors Negative', which behaves the same as 'E' except | |
| that the cell is colored red if the number in front of the indicator is | |
| negative. | |
| WN: This stands for 'Warnings Negative', which behaves the same as 'W' | |
| except that the cell is colored yellow if the number in front of the | |
| indicator is negative. | |
| N/A items can have any of the following indicators and need not be | |
| preceeded with a numerical value: | |
| '--', 'NA', 'N.A.', 'N.A', 'N/A', 'na', 'n.a.', 'n.a', 'n/a' | |
| ''' | |
| # Replace <td> with <td class="color-class"> based on the fp | |
| # value. "color-classes" are listed in ./style.css as follows: "cna" | |
| # for NA value, "c0" to "c10" for fp value falling between 0.00-9.99, | |
| # 10.00-19.99 ... 90.00-99.99, 100.0 respetively. | |
| def color_cell(cell, cclass, indicator="%"): | |
| op = cell.replace("<td", "<td class=\"" + cclass + "\"") | |
| # Remove the indicator. | |
| op = re.sub(r"\s*" + indicator + r"\s*", "", op) | |
| return op | |
| # List of 'not applicable' identifiers. | |
| na_list = ['--', 'NA', 'N.A.', 'N.A', 'N/A', 'na', 'n.a.', 'n.a', 'n/a'] | |
| na_list_patterns = '|'.join(na_list) | |
| # List of floating point patterns: '0', '0.0' & '.0' | |
| fp_patterns = r"[\+\-]?\d+\.?\d*" | |
| patterns = fp_patterns + '|' + na_list_patterns | |
| indicators = "%|%u|G|B|E|W|I|EN|WN" | |
| match = re.findall( | |
| r"(<td.*>\s*(" + patterns + r")\s+(" + indicators + r")\s*</td>)", | |
| text) | |
| if len(match) > 0: | |
| subst_list = {} | |
| fp_nums = [] | |
| for item in match: | |
| # item is a tuple - first is the full string indicating the table | |
| # cell which we want to replace, second is the floating point value. | |
| cell = item[0] | |
| fp_num = item[1] | |
| indicator = item[2] | |
| # Skip if fp_num is already processed. | |
| if (fp_num, indicator) in fp_nums: | |
| continue | |
| fp_nums.append((fp_num, indicator)) | |
| if fp_num in na_list: | |
| subst = color_cell(cell, "cna", indicator) | |
| else: | |
| # Item is a fp num. | |
| try: | |
| fp = float(fp_num) | |
| except ValueError: | |
| log.error( | |
| "Percentage item \"%s\" in cell \"%s\" is not an " | |
| "integer or a floating point number", fp_num, cell) | |
| continue | |
| # Percentage, colored. | |
| if indicator == "%": | |
| if fp >= 0.0 and fp < 10.0: | |
| subst = color_cell(cell, "c0") | |
| elif fp >= 10.0 and fp < 20.0: | |
| subst = color_cell(cell, "c1") | |
| elif fp >= 20.0 and fp < 30.0: | |
| subst = color_cell(cell, "c2") | |
| elif fp >= 30.0 and fp < 40.0: | |
| subst = color_cell(cell, "c3") | |
| elif fp >= 40.0 and fp < 50.0: | |
| subst = color_cell(cell, "c4") | |
| elif fp >= 50.0 and fp < 60.0: | |
| subst = color_cell(cell, "c5") | |
| elif fp >= 60.0 and fp < 70.0: | |
| subst = color_cell(cell, "c6") | |
| elif fp >= 70.0 and fp < 80.0: | |
| subst = color_cell(cell, "c7") | |
| elif fp >= 80.0 and fp < 90.0: | |
| subst = color_cell(cell, "c8") | |
| elif fp >= 90.0 and fp < 100.0: | |
| subst = color_cell(cell, "c9") | |
| elif fp >= 100.0: | |
| subst = color_cell(cell, "c10") | |
| # Percentage, uncolored. | |
| elif indicator == "%u": | |
| subst = cell.replace("%u", "%") | |
| # Good: green | |
| elif indicator == "G": | |
| subst = color_cell(cell, "c10", indicator) | |
| # Bad: red | |
| elif indicator == "B": | |
| subst = color_cell(cell, "c0", indicator) | |
| # Info, uncolored. | |
| elif indicator == "I": | |
| subst = cell.replace("I", "") | |
| # Bad if positive: red for errors, yellow for warnings, | |
| # otherwise green. | |
| elif indicator in ["E", "W"]: | |
| if fp <= 0: | |
| subst = color_cell(cell, "c10", indicator) | |
| elif indicator == "W": | |
| subst = color_cell(cell, "c6", indicator) | |
| elif indicator == "E": | |
| subst = color_cell(cell, "c0", indicator) | |
| # Bad if negative: red for errors, yellow for warnings, | |
| # otherwise green. | |
| elif indicator in ["EN", "WN"]: | |
| if fp >= 0: | |
| subst = color_cell(cell, "c10", indicator) | |
| elif indicator == "WN": | |
| subst = color_cell(cell, "c6", indicator) | |
| elif indicator == "EN": | |
| subst = color_cell(cell, "c0", indicator) | |
| subst_list[cell] = subst | |
| for item in subst_list: | |
| text = text.replace(item, subst_list[item]) | |
| return text | |
| def print_msg_list(msg_list_title, msg_list, max_msg_count=-1): | |
| '''This function prints a list of messages to Markdown. | |
| The argument msg_list_title contains a string for the list title, whereas | |
| the msg_list argument contains the actual list of message strings. | |
| max_msg_count limits the number of messages to be printed (set to negative | |
| number to print all messages). | |
| Example: | |
| print_msg_list("### Tool Warnings", ["Message A", "Message B"], 10) | |
| ''' | |
| md_results = "" | |
| if msg_list: | |
| md_results += msg_list_title + "\n" | |
| md_results += "```\n" | |
| for k, msg in enumerate(msg_list): | |
| if k <= max_msg_count or max_msg_count < 0: | |
| md_results += msg + "\n\n" | |
| else: | |
| md_results += "Note: %d more messages have been suppressed " % ( | |
| len(msg_list) - max_msg_count) | |
| md_results += "(max_msg_count = %d) \n\n" % (max_msg_count) | |
| break | |
| md_results += "```\n" | |
| return md_results | |
| def rm_path(path, ignore_error=False): | |
| '''Removes the specified path if it exists. | |
| 'path' is a Path-like object. If it does not exist, the function simply | |
| returns. If 'ignore_error' is set, then exception caught by the remove | |
| operation is raised, else it is ignored. | |
| ''' | |
| exc = None | |
| try: | |
| os.remove(path) | |
| except FileNotFoundError: | |
| pass | |
| except IsADirectoryError: | |
| try: | |
| shutil.rmtree(path) | |
| except OSError as e: | |
| exc = e | |
| except OSError as e: | |
| exc = e | |
| if exc: | |
| log.error("Failed to remove {}:\n{}.".format(path, exc)) | |
| if not ignore_error: | |
| raise exc | |
| def mk_path(path): | |
| '''Create the specified path if it does not exist. | |
| 'path' is a Path-like object. If it does exist, the function simply | |
| returns. If it does not exist, the function creates the path and its | |
| parent dictories if necessary. | |
| ''' | |
| try: | |
| Path(path).mkdir(parents=True, exist_ok=True) | |
| except PermissionError as e: | |
| log.fatal("Failed to create directory {}:\n{}.".format(path, e)) | |
| sys.exit(1) | |
| def mk_symlink(path, link): | |
| '''Create a symlink from the given path. | |
| 'link' is a Path-like object. If it does exist, remove the existing link and | |
| create a new symlink with this given path. | |
| If it does not exist, the function creates the symlink with the given path. | |
| ''' | |
| while True: | |
| try: | |
| os.symlink(path, link) | |
| break | |
| except FileExistsError: | |
| rm_path(link) | |
| def clean_odirs(odir, max_odirs, ts_format=TS_FORMAT): | |
| """Clean previous output directories. | |
| When running jobs, we may want to maintain a limited history of | |
| previous invocations. This method finds and deletes the output | |
| directories at the base of input arg 'odir' with the oldest timestamps, | |
| if that limit is reached. It returns a list of directories that | |
| remain after deletion. | |
| """ | |
| odir = Path(odir) | |
| if os.path.exists(odir): | |
| # If output directory exists, back it up. | |
| ts = datetime.fromtimestamp(os.stat(odir).st_ctime).strftime(ts_format) | |
| # Prior to Python 3.9, shutil may run into an error when passing in | |
| # Path objects (see https://bugs.python.org/issue32689). While this | |
| # has been fixed in Python 3.9, string casts are added so that this | |
| # also works with older versions. | |
| shutil.move(str(odir), str(odir.with_name(ts))) | |
| # Get list of past output directories sorted by creation time. | |
| pdir = odir.resolve().parent | |
| if not pdir.exists(): | |
| return [] | |
| dirs = sorted([old for old in pdir.iterdir() if (old.is_dir() and | |
| old != 'summary')], | |
| key=os.path.getctime, | |
| reverse=True) | |
| for old in dirs[max(0, max_odirs - 1):]: | |
| shutil.rmtree(old, ignore_errors=True) | |
| return [] if max_odirs == 0 else dirs[:max_odirs - 1] | |
| def check_bool(x): | |
| """check_bool checks if input 'x' either a bool or | |
| one of the following strings: ["true", "false"] | |
| It returns value as Bool type. | |
| """ | |
| if isinstance(x, bool): | |
| return x | |
| if not x.lower() in ["true", "false"]: | |
| raise RuntimeError("{} is not a boolean value.".format(x)) | |
| else: | |
| return (x.lower() == "true") | |
| def check_int(x): | |
| """check_int checks if input 'x' is decimal integer. | |
| It returns value as an int type. | |
| """ | |
| if isinstance(x, int): | |
| return x | |
| if not x.isdecimal(): | |
| raise RuntimeError("{} is not a decimal number".format(x)) | |
| return int(x) |