[util] Allow optional tools in check_tool_requirements tool_requirements.py lists minimum versions for tools, but the distinction between "tools that absolutely need to be installed" and "the minimum version of a tool if you want to run it" was baked in to check_tool_requirements.py This patch moves that distinction to the (more user-editable) tool_requirements.py. Tools that are marked as_needed aren't checked unless they are passed on the command line to check_tool_requirements.py. Signed-off-by: Rupert Swarbrick <rswarbrick@lowrisc.org>
diff --git a/doc/ug/install_instructions/index.md b/doc/ug/install_instructions/index.md index 5ca8710..479e7c0 100644 --- a/doc/ug/install_instructions/index.md +++ b/doc/ug/install_instructions/index.md
@@ -41,7 +41,7 @@ GCC 5 or Clang 3.5 should meet this requirement. * clang-format. The use of clang-format 6.0 is recommended to match the formatting enforced when submitting a pull request. -* [ninja](https://ninja-build.org/) {{< tool_version "ninja-build" >}} +* [ninja](https://ninja-build.org/) {{< tool_version "ninja" >}} * Bash * curl * xz tools
diff --git a/tool_requirements.py b/tool_requirements.py index 9f9a799..aeb277a 100644 --- a/tool_requirements.py +++ b/tool_requirements.py
@@ -4,10 +4,32 @@ # Version requirements for various tools. Checked by tooling (e.g. fusesoc), # and inserted into the documentation. +# +# Entries are keyed by tool name. The value is either a string giving the +# minimum version number or is a dictionary. If a dictionary, the following +# keys are recognised: +# +# min_version: Required string. Minimum version number. +# +# as_needed: Optional bool. Defaults to False. If set, this tool is not +# automatically required. If it is asked for, the rest of the +# entry gives the required version. +# __TOOL_REQUIREMENTS__ = { 'edalize': '0.2.0', - 'hugo_extended': '0.71.0', - 'ninja-build': '1.8.2', - 'verible': 'v0.0-808-g1e17daa', + 'ninja': '1.8.2', 'verilator': '4.104', + + 'hugo_extended': { + 'min_version': '0.71.0', + 'as_needed': True + }, + 'verible': { + 'min_version': 'v0.0-808-g1e17daa', + 'as_needed': True + }, + 'vcs': { + 'min_version': '2020.03-SP2', + 'as_needed': True + } }
diff --git a/util/build_docs.py b/util/build_docs.py index e1ec934..77cfbde 100755 --- a/util/build_docs.py +++ b/util/build_docs.py
@@ -38,8 +38,8 @@ # Version of hugo extended to be used to build the docs try: - tool_requirements = check_tool_requirements.read_tool_requirements() - HUGO_EXTENDED_VERSION = tool_requirements['hugo_extended'] + TOOL_REQUIREMENTS = check_tool_requirements.read_tool_requirements() + HUGO_EXTENDED_VERSION = TOOL_REQUIREMENTS['hugo_extended'].min_version except Exception as e: print("Unable to get required hugo version: %s" % str(e), file=sys.stderr) sys.exit(1) @@ -241,16 +241,12 @@ The version number per tool will be saved in outdir-generated/version_$TOOL_NAME.txt """ - # Populate __TOOL_REQUIREMENTS__ - requirements_file = str(SRCTREE_TOP.joinpath("tool_requirements.py")) - exec(open(requirements_file).read(), globals()) - # And then write a version file for every tool. - for tool in __TOOL_REQUIREMENTS__: # noqa: F821 + for tool, req in TOOL_REQUIREMENTS.items(): version_path = config["outdir-generated"].joinpath('version_' + tool + '.txt') version_path.parent.mkdir(parents=True, exist_ok=True) with open(str(version_path), mode='w') as fout: - fout.write(__TOOL_REQUIREMENTS__[tool]) # noqa: F821 + fout.write(req.min_version) def generate_dif_docs():
diff --git a/util/check_tool_requirements.py b/util/check_tool_requirements.py index 2e46ed1..77c469c 100755 --- a/util/check_tool_requirements.py +++ b/util/check_tool_requirements.py
@@ -3,10 +3,12 @@ # Licensed under the Apache License, Version 2.0, see LICENSE for details. # SPDX-License-Identifier: Apache-2.0 +import argparse from distutils.version import StrictVersion import logging as log import os import re +import shlex import subprocess import sys @@ -23,6 +25,298 @@ return os.path.join(top_src_dir, 'tool_requirements.py') +class ReqErr(Exception): + def __init__(self, path, msg): + self.path = path + self.msg = msg + + def __str__(self): + return ('Error parsing tool requirements from {!r}: {}' + .format(self.path, self.msg)) + + +class ToolReq: + # A subclass can set this to configure the command that's run to get the + # version of a tool. If tool_cmd is None, get_version will call "self.tool + # --version". + tool_cmd = None + + # Used by get_version. If not None, this is a dictionary that's added to + # the environment when running the command. + tool_env = None + + # A subclass can set this to configure _parse_version_output. If set, it + # should be a Regex object with a single capturing group that captures the + # version. + version_regex = None + + def __init__(self, tool, min_version): + self.tool = tool + self.min_version = min_version + self.optional = False + + def _get_tool_cmd(self): + '''Return the command to run to get the installed version''' + return self.tool_cmd or [self.tool, '--version'] + + def _get_version(self): + '''Run the tool to get the installed version. + + Raises a RuntimeError on failure. The default version uses the class + variable tool_cmd to figure out what to run. + + ''' + + def _parse_version_output(self, stdout): + '''Parse the nonempty stdout to get a version number + + Raises a ValueError on failure. The default implementation returns the + last word of the first line if version_regex is None or the first match + for version_regex if it is not None. + + ''' + if self.version_regex is None: + line0 = stdout.split('\n', 1)[0] + words = line0.rsplit(None, 1) + if not words: + raise ValueError('Empty first line.') + + return words[-1] + + for line in stdout.split('\n'): + match = self.version_regex.match(line.rstrip()) + if match is not None: + return match.group(1) + raise ValueError('No line matched version regex.') + + def get_version(self): + '''Run the tool to get a version. + + Returns a version string on success. Raises a RuntimeError on failure. + The default version uses the class variable tool_cmd to figure out what + to run. + + ''' + cmd = self._get_tool_cmd() + cmd_txt = ' '.join(shlex.quote(w) for w in cmd) + + env = None + if self.tool_env is not None: + env = os.environ.copy() + env.update(self.tool_env) + + try: + proc = subprocess.run(cmd, + check=True, + stdout=subprocess.PIPE, + universal_newlines=True, + env=env) + except (subprocess.CalledProcessError, FileNotFoundError) as err: + env_msg = ('' if not self.tool_env else + ' (with environment overrides: {})' + .format(', '.join('{}={}'.format(k, v) + for k, v in self.tool_env.items()))) + raise RuntimeError('Failed to run {!r}{} to check version: {}' + .format(cmd_txt, env_msg, err)) + + if not proc.stdout: + raise RuntimeError('No output from running {!r} to check version.' + .format(cmd_txt)) + + try: + return self._parse_version_output(proc.stdout) + except ValueError as err: + raise RuntimeError('Bad output from running {!r} ' + 'to check version: {}' + .format(cmd_txt, err)) + + def to_semver(self, version, from_req): + '''Convert a tool version to semantic versioning format + + If from_req is true, this version comes from the requirements file + (rather than being reported from an installed application). That might + mean stricter checking. If version is not a known format, raises a + ValueError. + + ''' + return version + + def check(self): + '''Get the installed version and check it matches the requirements + + Returns (is_good, msg). is_good is true if we matched the requirements + and false otherwise. msg describes what happened (an error message on + failure, or extra information on success). + + ''' + try: + min_semver = self.to_semver(self.min_version, True) + except ValueError as err: + return (False, + 'Failed to convert requirement to semantic version: {}' + .format(err)) + try: + min_sv = StrictVersion(min_semver) + except ValueError as err: + return (False, + 'Bad semver inferred from required version ({}): {}' + .format(min_semver, err)) + + try: + actual_version = self.get_version() + except RuntimeError as err: + return (False, str(err)) + + try: + actual_semver = self.to_semver(actual_version, False) + except ValueError as err: + return (False, + 'Failed to convert installed to semantic version: {}' + .format(err)) + try: + actual_sv = StrictVersion(actual_semver) + except ValueError as err: + return (False, + 'Bad semver inferred from installed version ({}): {}' + .format(actual_semver, err)) + + if actual_sv < min_sv: + return (False, + 'Installed version is too old: ' + 'found version {}, but need at least {}' + .format(actual_version, self.min_version)) + + return (True, + 'Sufficiently recent version (found {}; needed {})' + .format(actual_version, self.min_version)) + + +class VerilatorToolReq(ToolReq): + def get_version(self): + try: + # Note: "verilator" needs to be called through a shell and with all + # arguments in a string, as it doesn't have a shebang, but instead + # relies on perl magic to parse command line arguments. + version_str = subprocess.run('verilator --version', shell=True, + check=True, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True) + except subprocess.CalledProcessError as err: + raise RuntimeError('Unable to call Verilator to check version: {}' + .format(err)) from None + + return version_str.stdout.split(' ')[1].strip() + + +class VeribleToolReq(ToolReq): + tool_cmd = ['verible-verilog-lint', '--version'] + + def to_semver(self, version, from_req): + # Drop the hash suffix and convert into version string that + # is compatible with StrictVersion in check_version below. + # Example: v0.0-808-g1e17daa -> 0.0.808 + m = re.fullmatch(r'v([0-9]+)\.([0-9]+)-([0-9]+)-g[0-9a-f]+$', version) + if m is None: + raise ValueError("{} has invalid version string format." + .format(version)) + + return '.'.join(m.group(1, 2, 3)) + + +class VcsToolReq(ToolReq): + tool_cmd = ['vcs', '-full64', '-ID'] + tool_env = {'VCS_ARCH_OVERRIDE': 'linux'} + version_regex = re.compile(r'Compiler version = VCS [A-Z]-(.*)') + + def to_semver(self, version, from_req): + # VCS has a rather strange numbering scheme, where the most general + # versions look something like this: + # + # Q-2020.03-SP1-2 + # + # Our version_regex strips out the "Q" part (a "platform prefix") + # already. A version always has the "2020.03" (year and month) part, + # and may also have an -SP<n> and/or -<patch> suffix. + # + # Since StrictVersion expects a 3 digit versioning scheme, we multiply + # any SP number by 100, which should work as long as the patch version + # isn't greater than 99. + # + # Some VCS builds also report other cruft (like _Full64) after this + # version number. If from_req is False, allow (and ignore) that too. + regex = r'([0-9]+).([0-9]+)(?:-SP([0-9]+))?(?:-([0-9]+))?' + if from_req: + regex += '$' + + match = re.match(regex, version) + if match is None: + raise ValueError("{!r} is not a recognised VCS version string." + .format(version)) + major = match.group(1) + minor = match.group(2) + sp = int(match.group(3) or 0) + patch = int(match.group(4) or 0) + comb = str(sp * 100 + patch) + return '{}.{}{}'.format(major, minor, comb) + + +class PyModuleToolReq(ToolReq): + '''A tool in a Python module (its version can be found by running pip)''' + version_regex = re.compile(r'Version: (.*)') + + def _get_tool_cmd(self): + return ['pip3', 'show', self.tool] + + +def dict_to_tool_req(path, tool, raw): + '''Parse a dict (as read from Python) as a ToolReq + + Required keys: version. Optional keys: as_needed. + + ''' + where = 'Dict for {} in __TOOL_REQUIREMENTS__'.format(tool) + # We operate in place on the dictionary. Take a copy to avoid an + # obnoxious API. + raw = raw.copy() + + if 'min_version' not in raw: + raise ReqErr(path, + '{} is missing required key: "min_version".' + .format(where)) + min_version = raw['min_version'] + if not isinstance(min_version, str): + raise ReqErr(path, + '{} has min_version that is not a string.' + .format(where)) + del raw['min_version'] + + as_needed = False + if 'as_needed' in raw: + as_needed = raw['as_needed'] + if not isinstance(as_needed, bool): + raise ReqErr(path, + '{} has as_needed that is not a bool.' + .format(where)) + del raw['as_needed'] + + if raw: + raise ReqErr(path, + '{} has unexpected keys: {}.' + .format(where, ', '.join(raw.keys()))) + + classes = { + 'edalize': PyModuleToolReq, + 'vcs': VcsToolReq, + 'verible': VeribleToolReq, + 'verilator': VerilatorToolReq + } + cls = classes.get(tool, ToolReq) + + ret = cls(tool, min_version) + ret.as_needed = as_needed + return ret + + def read_tool_requirements(path=None): '''Read tool requirements from a Python file''' if path is None: @@ -34,148 +328,80 @@ # We expect the exec call to have populated globs with a # __TOOL_REQUIREMENTS__ dictionary. - reqs = globs.get('__TOOL_REQUIREMENTS__') - if reqs is None: - log.error('The Python file at {} did not define ' - '__TOOL_REQUIREMENTS__.' - .format(path)) - return None + raw = globs.get('__TOOL_REQUIREMENTS__') + if raw is None: + raise ReqErr(path, + 'The Python file at did not define ' + '__TOOL_REQUIREMENTS__.') - # reqs should be a dictionary (mapping tool name to minimum version) - if not isinstance(reqs, dict): - log.error('The Python file at {} defined ' - '__TOOL_REQUIREMENTS__, but it is not a dict.' - .format(path)) - return None + # raw should be a dictionary (keyed by tool name) + if not isinstance(raw, dict): + raise ReqErr(path, '__TOOL_REQUIREMENTS__ is not a dict.') + + reqs = {} + for tool, raw_val in raw.items(): + if not isinstance(tool, str): + raise ReqErr(path, + 'Invalid key in __TOOL_REQUIREMENTS__: {!r}' + .format(tool)) + + if isinstance(raw_val, str): + # Shorthand notation: value is just a string, which we + # interpret as a minimum version + raw_val = {'min_version': raw_val} + + if not isinstance(raw_val, dict): + raise ReqErr(path, + 'Value for {} in __TOOL_REQUIREMENTS__ ' + 'is not a string or dict.'.format(tool)) + + reqs[tool] = dict_to_tool_req(path, tool, raw_val) return reqs -def get_verilator_version(): - try: - # Note: "verilator" needs to be called through a shell and with all - # arguments in a string, as it doesn't have a shebang, but instead - # relies on perl magic to parse command line arguments. - version_str = subprocess.run('verilator --version', shell=True, - check=True, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - universal_newlines=True) - return version_str.stdout.split(' ')[1].strip() - - except subprocess.CalledProcessError as e: - log.error("Unable to call Verilator to check version: " + str(e)) - log.error(e.stdout) - return None - - -def convert_verible_version(version_string): - '''Convert Verible version string to semantic versioning format.''' - # Drop the hash suffix and convert into version string that - # is compatible with StrictVersion in check_version below. - # Example: v0.0-808-g1e17daa -> 0.0.808 - m = re.fullmatch(r'v([0-9]+)\.([0-9]+)-([0-9]+)-g[0-9a-f]+$', version_string) - - if m is None: - log.error("{} has invalid version string format.".format(version_string)) - return None - - return '.'.join(m.group(1, 2, 3)) - - -def get_verible_version(): - '''Run Verible to check its version''' - try: - version_str = subprocess.run('verible-verilog-lint --version', shell=True, - check=True, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - universal_newlines=True) - return version_str.stdout.split('\n')[0].strip() - - except subprocess.CalledProcessError as e: - log.error("Unable to call Verible to check version: " + str(e)) - log.error(e.stdout) - return None - - -def pip3_get_version(tool): - '''Run pip3 to find the version of an installed module''' - cmd = ['pip3', 'show', tool] - try: - proc = subprocess.run(cmd, - check=True, - stderr=subprocess.STDOUT, - stdout=subprocess.PIPE, - universal_newlines=True) - except subprocess.CalledProcessError as err: - log.error('pip3 command failed: {}'.format(err)) - log.error("Failed to get version of {} with pip3: is it installed?" - .format(tool)) - log.error(err.stdout) - return None - - version_re = 'Version: (.*)' - for line in proc.stdout.splitlines(): - match = re.match(version_re, line) - if match: - return match.group(1) - - # If we get here, we never saw a version line. - log.error('No output line from running {} started with "Version: ".' - .format(cmd)) - return None - - -def check_version(requirements, tool_name, getter, version_converter=None): - required_version = requirements.get(tool_name) - if required_version is None: - log.error('Requirements file does not specify version for {}.' - .format(tool_name)) - return False - - actual_version = getter() - if actual_version is None: - return False - - # If a version string converter is defined, call it. This is required - # for some version strings that are not compatible with StrictVersion. - if version_converter is not None: - required_version = version_converter(required_version) - actual_version = version_converter(actual_version) - - if StrictVersion(actual_version) < StrictVersion(required_version): - log.error("%s is too old: found version %s, need at least %s", - tool_name, actual_version, required_version) - return False - else: - log.info("Found sufficiently recent version of %s (found %s, need %s)", - tool_name, actual_version, required_version) - return True - - def main(): + parser = argparse.ArgumentParser() + parser.add_argument('tool', nargs='*') + args = parser.parse_args() + # Get tool requirements - tool_requirements = read_tool_requirements() - if tool_requirements is None: + try: + tool_requirements = read_tool_requirements() + except ReqErr as err: + log.error(str(err)) return 1 + pending_tools = set(args.tool) + missing_tools = [] + for tool, req in tool_requirements.items(): + if req.as_needed and tool not in pending_tools: + continue + pending_tools.discard(tool) + + good, msg = req.check() + if not good: + log.error('Failed tool requirement for {}: {}' + .format(tool, msg)) + missing_tools.append(tool) + else: + log.info('Tool {} present: {}' + .format(tool, msg)) + all_good = True - all_good &= check_version(tool_requirements, - 'verilator', - get_verilator_version) - all_good &= check_version(tool_requirements, - 'verible', - get_verible_version, - convert_verible_version) - all_good &= check_version(tool_requirements, - 'edalize', - lambda: pip3_get_version('edalize')) - - if not all_good: + if missing_tools: log.error("Tool requirements not fulfilled. " - "Please update the tools and retry.") - return 1 + "Please update tools ({}) and retry." + .format(', '.join(missing_tools))) + all_good = False - return 0 + if pending_tools: + log.error("Some tools specified on command line don't appear in " + "tool requirements file: {}" + .format(', '.join(sorted(pending_tools)))) + all_good = False + + return 0 if all_good else 1 if __name__ == "__main__":