[util] Allow optional tools in check_tool_requirements
tool_requirements.py lists minimum versions for tools, but the
distinction between "tools that absolutely need to be installed" and
"the minimum version of a tool if you want to run it" was baked in to
check_tool_requirements.py
This patch moves that distinction to the (more user-editable)
tool_requirements.py. Tools that are marked as_needed aren't checked
unless they are passed on the command line to
check_tool_requirements.py.
Signed-off-by: Rupert Swarbrick <rswarbrick@lowrisc.org>
diff --git a/doc/ug/install_instructions/index.md b/doc/ug/install_instructions/index.md
index 5ca8710..479e7c0 100644
--- a/doc/ug/install_instructions/index.md
+++ b/doc/ug/install_instructions/index.md
@@ -41,7 +41,7 @@
GCC 5 or Clang 3.5 should meet this requirement.
* clang-format.
The use of clang-format 6.0 is recommended to match the formatting enforced when submitting a pull request.
-* [ninja](https://ninja-build.org/) {{< tool_version "ninja-build" >}}
+* [ninja](https://ninja-build.org/) {{< tool_version "ninja" >}}
* Bash
* curl
* xz tools
diff --git a/tool_requirements.py b/tool_requirements.py
index 9f9a799..aeb277a 100644
--- a/tool_requirements.py
+++ b/tool_requirements.py
@@ -4,10 +4,32 @@
# Version requirements for various tools. Checked by tooling (e.g. fusesoc),
# and inserted into the documentation.
+#
+# Entries are keyed by tool name. The value is either a string giving the
+# minimum version number or is a dictionary. If a dictionary, the following
+# keys are recognised:
+#
+# min_version: Required string. Minimum version number.
+#
+# as_needed: Optional bool. Defaults to False. If set, this tool is not
+# automatically required. If it is asked for, the rest of the
+# entry gives the required version.
+#
__TOOL_REQUIREMENTS__ = {
'edalize': '0.2.0',
- 'hugo_extended': '0.71.0',
- 'ninja-build': '1.8.2',
- 'verible': 'v0.0-808-g1e17daa',
+ 'ninja': '1.8.2',
'verilator': '4.104',
+
+ 'hugo_extended': {
+ 'min_version': '0.71.0',
+ 'as_needed': True
+ },
+ 'verible': {
+ 'min_version': 'v0.0-808-g1e17daa',
+ 'as_needed': True
+ },
+ 'vcs': {
+ 'min_version': '2020.03-SP2',
+ 'as_needed': True
+ }
}
diff --git a/util/build_docs.py b/util/build_docs.py
index e1ec934..77cfbde 100755
--- a/util/build_docs.py
+++ b/util/build_docs.py
@@ -38,8 +38,8 @@
# Version of hugo extended to be used to build the docs
try:
- tool_requirements = check_tool_requirements.read_tool_requirements()
- HUGO_EXTENDED_VERSION = tool_requirements['hugo_extended']
+ TOOL_REQUIREMENTS = check_tool_requirements.read_tool_requirements()
+ HUGO_EXTENDED_VERSION = TOOL_REQUIREMENTS['hugo_extended'].min_version
except Exception as e:
print("Unable to get required hugo version: %s" % str(e), file=sys.stderr)
sys.exit(1)
@@ -241,16 +241,12 @@
The version number per tool will be saved in outdir-generated/version_$TOOL_NAME.txt
"""
- # Populate __TOOL_REQUIREMENTS__
- requirements_file = str(SRCTREE_TOP.joinpath("tool_requirements.py"))
- exec(open(requirements_file).read(), globals())
-
# And then write a version file for every tool.
- for tool in __TOOL_REQUIREMENTS__: # noqa: F821
+ for tool, req in TOOL_REQUIREMENTS.items():
version_path = config["outdir-generated"].joinpath('version_' + tool + '.txt')
version_path.parent.mkdir(parents=True, exist_ok=True)
with open(str(version_path), mode='w') as fout:
- fout.write(__TOOL_REQUIREMENTS__[tool]) # noqa: F821
+ fout.write(req.min_version)
def generate_dif_docs():
diff --git a/util/check_tool_requirements.py b/util/check_tool_requirements.py
index 2e46ed1..77c469c 100755
--- a/util/check_tool_requirements.py
+++ b/util/check_tool_requirements.py
@@ -3,10 +3,12 @@
# Licensed under the Apache License, Version 2.0, see LICENSE for details.
# SPDX-License-Identifier: Apache-2.0
+import argparse
from distutils.version import StrictVersion
import logging as log
import os
import re
+import shlex
import subprocess
import sys
@@ -23,6 +25,298 @@
return os.path.join(top_src_dir, 'tool_requirements.py')
+class ReqErr(Exception):
+ def __init__(self, path, msg):
+ self.path = path
+ self.msg = msg
+
+ def __str__(self):
+ return ('Error parsing tool requirements from {!r}: {}'
+ .format(self.path, self.msg))
+
+
+class ToolReq:
+ # A subclass can set this to configure the command that's run to get the
+ # version of a tool. If tool_cmd is None, get_version will call "self.tool
+ # --version".
+ tool_cmd = None
+
+ # Used by get_version. If not None, this is a dictionary that's added to
+ # the environment when running the command.
+ tool_env = None
+
+ # A subclass can set this to configure _parse_version_output. If set, it
+ # should be a Regex object with a single capturing group that captures the
+ # version.
+ version_regex = None
+
+ def __init__(self, tool, min_version):
+ self.tool = tool
+ self.min_version = min_version
+ self.optional = False
+
+ def _get_tool_cmd(self):
+ '''Return the command to run to get the installed version'''
+ return self.tool_cmd or [self.tool, '--version']
+
+ def _get_version(self):
+ '''Run the tool to get the installed version.
+
+ Raises a RuntimeError on failure. The default version uses the class
+ variable tool_cmd to figure out what to run.
+
+ '''
+
+ def _parse_version_output(self, stdout):
+ '''Parse the nonempty stdout to get a version number
+
+ Raises a ValueError on failure. The default implementation returns the
+ last word of the first line if version_regex is None or the first match
+ for version_regex if it is not None.
+
+ '''
+ if self.version_regex is None:
+ line0 = stdout.split('\n', 1)[0]
+ words = line0.rsplit(None, 1)
+ if not words:
+ raise ValueError('Empty first line.')
+
+ return words[-1]
+
+ for line in stdout.split('\n'):
+ match = self.version_regex.match(line.rstrip())
+ if match is not None:
+ return match.group(1)
+ raise ValueError('No line matched version regex.')
+
+ def get_version(self):
+ '''Run the tool to get a version.
+
+ Returns a version string on success. Raises a RuntimeError on failure.
+ The default version uses the class variable tool_cmd to figure out what
+ to run.
+
+ '''
+ cmd = self._get_tool_cmd()
+ cmd_txt = ' '.join(shlex.quote(w) for w in cmd)
+
+ env = None
+ if self.tool_env is not None:
+ env = os.environ.copy()
+ env.update(self.tool_env)
+
+ try:
+ proc = subprocess.run(cmd,
+ check=True,
+ stdout=subprocess.PIPE,
+ universal_newlines=True,
+ env=env)
+ except (subprocess.CalledProcessError, FileNotFoundError) as err:
+ env_msg = ('' if not self.tool_env else
+ ' (with environment overrides: {})'
+ .format(', '.join('{}={}'.format(k, v)
+ for k, v in self.tool_env.items())))
+ raise RuntimeError('Failed to run {!r}{} to check version: {}'
+ .format(cmd_txt, env_msg, err))
+
+ if not proc.stdout:
+ raise RuntimeError('No output from running {!r} to check version.'
+ .format(cmd_txt))
+
+ try:
+ return self._parse_version_output(proc.stdout)
+ except ValueError as err:
+ raise RuntimeError('Bad output from running {!r} '
+ 'to check version: {}'
+ .format(cmd_txt, err))
+
+ def to_semver(self, version, from_req):
+ '''Convert a tool version to semantic versioning format
+
+ If from_req is true, this version comes from the requirements file
+ (rather than being reported from an installed application). That might
+ mean stricter checking. If version is not a known format, raises a
+ ValueError.
+
+ '''
+ return version
+
+ def check(self):
+ '''Get the installed version and check it matches the requirements
+
+ Returns (is_good, msg). is_good is true if we matched the requirements
+ and false otherwise. msg describes what happened (an error message on
+ failure, or extra information on success).
+
+ '''
+ try:
+ min_semver = self.to_semver(self.min_version, True)
+ except ValueError as err:
+ return (False,
+ 'Failed to convert requirement to semantic version: {}'
+ .format(err))
+ try:
+ min_sv = StrictVersion(min_semver)
+ except ValueError as err:
+ return (False,
+ 'Bad semver inferred from required version ({}): {}'
+ .format(min_semver, err))
+
+ try:
+ actual_version = self.get_version()
+ except RuntimeError as err:
+ return (False, str(err))
+
+ try:
+ actual_semver = self.to_semver(actual_version, False)
+ except ValueError as err:
+ return (False,
+ 'Failed to convert installed to semantic version: {}'
+ .format(err))
+ try:
+ actual_sv = StrictVersion(actual_semver)
+ except ValueError as err:
+ return (False,
+ 'Bad semver inferred from installed version ({}): {}'
+ .format(actual_semver, err))
+
+ if actual_sv < min_sv:
+ return (False,
+ 'Installed version is too old: '
+ 'found version {}, but need at least {}'
+ .format(actual_version, self.min_version))
+
+ return (True,
+ 'Sufficiently recent version (found {}; needed {})'
+ .format(actual_version, self.min_version))
+
+
+class VerilatorToolReq(ToolReq):
+ def get_version(self):
+ try:
+ # Note: "verilator" needs to be called through a shell and with all
+ # arguments in a string, as it doesn't have a shebang, but instead
+ # relies on perl magic to parse command line arguments.
+ version_str = subprocess.run('verilator --version', shell=True,
+ check=True, stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ universal_newlines=True)
+ except subprocess.CalledProcessError as err:
+ raise RuntimeError('Unable to call Verilator to check version: {}'
+ .format(err)) from None
+
+ return version_str.stdout.split(' ')[1].strip()
+
+
+class VeribleToolReq(ToolReq):
+ tool_cmd = ['verible-verilog-lint', '--version']
+
+ def to_semver(self, version, from_req):
+ # Drop the hash suffix and convert into version string that
+ # is compatible with StrictVersion in check_version below.
+ # Example: v0.0-808-g1e17daa -> 0.0.808
+ m = re.fullmatch(r'v([0-9]+)\.([0-9]+)-([0-9]+)-g[0-9a-f]+$', version)
+ if m is None:
+ raise ValueError("{} has invalid version string format."
+ .format(version))
+
+ return '.'.join(m.group(1, 2, 3))
+
+
+class VcsToolReq(ToolReq):
+ tool_cmd = ['vcs', '-full64', '-ID']
+ tool_env = {'VCS_ARCH_OVERRIDE': 'linux'}
+ version_regex = re.compile(r'Compiler version = VCS [A-Z]-(.*)')
+
+ def to_semver(self, version, from_req):
+ # VCS has a rather strange numbering scheme, where the most general
+ # versions look something like this:
+ #
+ # Q-2020.03-SP1-2
+ #
+ # Our version_regex strips out the "Q" part (a "platform prefix")
+ # already. A version always has the "2020.03" (year and month) part,
+ # and may also have an -SP<n> and/or -<patch> suffix.
+ #
+ # Since StrictVersion expects a 3 digit versioning scheme, we multiply
+ # any SP number by 100, which should work as long as the patch version
+ # isn't greater than 99.
+ #
+ # Some VCS builds also report other cruft (like _Full64) after this
+ # version number. If from_req is False, allow (and ignore) that too.
+ regex = r'([0-9]+).([0-9]+)(?:-SP([0-9]+))?(?:-([0-9]+))?'
+ if from_req:
+ regex += '$'
+
+ match = re.match(regex, version)
+ if match is None:
+ raise ValueError("{!r} is not a recognised VCS version string."
+ .format(version))
+ major = match.group(1)
+ minor = match.group(2)
+ sp = int(match.group(3) or 0)
+ patch = int(match.group(4) or 0)
+ comb = str(sp * 100 + patch)
+ return '{}.{}{}'.format(major, minor, comb)
+
+
+class PyModuleToolReq(ToolReq):
+ '''A tool in a Python module (its version can be found by running pip)'''
+ version_regex = re.compile(r'Version: (.*)')
+
+ def _get_tool_cmd(self):
+ return ['pip3', 'show', self.tool]
+
+
+def dict_to_tool_req(path, tool, raw):
+ '''Parse a dict (as read from Python) as a ToolReq
+
+ Required keys: version. Optional keys: as_needed.
+
+ '''
+ where = 'Dict for {} in __TOOL_REQUIREMENTS__'.format(tool)
+ # We operate in place on the dictionary. Take a copy to avoid an
+ # obnoxious API.
+ raw = raw.copy()
+
+ if 'min_version' not in raw:
+ raise ReqErr(path,
+ '{} is missing required key: "min_version".'
+ .format(where))
+ min_version = raw['min_version']
+ if not isinstance(min_version, str):
+ raise ReqErr(path,
+ '{} has min_version that is not a string.'
+ .format(where))
+ del raw['min_version']
+
+ as_needed = False
+ if 'as_needed' in raw:
+ as_needed = raw['as_needed']
+ if not isinstance(as_needed, bool):
+ raise ReqErr(path,
+ '{} has as_needed that is not a bool.'
+ .format(where))
+ del raw['as_needed']
+
+ if raw:
+ raise ReqErr(path,
+ '{} has unexpected keys: {}.'
+ .format(where, ', '.join(raw.keys())))
+
+ classes = {
+ 'edalize': PyModuleToolReq,
+ 'vcs': VcsToolReq,
+ 'verible': VeribleToolReq,
+ 'verilator': VerilatorToolReq
+ }
+ cls = classes.get(tool, ToolReq)
+
+ ret = cls(tool, min_version)
+ ret.as_needed = as_needed
+ return ret
+
+
def read_tool_requirements(path=None):
'''Read tool requirements from a Python file'''
if path is None:
@@ -34,148 +328,80 @@
# We expect the exec call to have populated globs with a
# __TOOL_REQUIREMENTS__ dictionary.
- reqs = globs.get('__TOOL_REQUIREMENTS__')
- if reqs is None:
- log.error('The Python file at {} did not define '
- '__TOOL_REQUIREMENTS__.'
- .format(path))
- return None
+ raw = globs.get('__TOOL_REQUIREMENTS__')
+ if raw is None:
+ raise ReqErr(path,
+ 'The Python file at did not define '
+ '__TOOL_REQUIREMENTS__.')
- # reqs should be a dictionary (mapping tool name to minimum version)
- if not isinstance(reqs, dict):
- log.error('The Python file at {} defined '
- '__TOOL_REQUIREMENTS__, but it is not a dict.'
- .format(path))
- return None
+ # raw should be a dictionary (keyed by tool name)
+ if not isinstance(raw, dict):
+ raise ReqErr(path, '__TOOL_REQUIREMENTS__ is not a dict.')
+
+ reqs = {}
+ for tool, raw_val in raw.items():
+ if not isinstance(tool, str):
+ raise ReqErr(path,
+ 'Invalid key in __TOOL_REQUIREMENTS__: {!r}'
+ .format(tool))
+
+ if isinstance(raw_val, str):
+ # Shorthand notation: value is just a string, which we
+ # interpret as a minimum version
+ raw_val = {'min_version': raw_val}
+
+ if not isinstance(raw_val, dict):
+ raise ReqErr(path,
+ 'Value for {} in __TOOL_REQUIREMENTS__ '
+ 'is not a string or dict.'.format(tool))
+
+ reqs[tool] = dict_to_tool_req(path, tool, raw_val)
return reqs
-def get_verilator_version():
- try:
- # Note: "verilator" needs to be called through a shell and with all
- # arguments in a string, as it doesn't have a shebang, but instead
- # relies on perl magic to parse command line arguments.
- version_str = subprocess.run('verilator --version', shell=True,
- check=True, stdout=subprocess.PIPE,
- stderr=subprocess.STDOUT,
- universal_newlines=True)
- return version_str.stdout.split(' ')[1].strip()
-
- except subprocess.CalledProcessError as e:
- log.error("Unable to call Verilator to check version: " + str(e))
- log.error(e.stdout)
- return None
-
-
-def convert_verible_version(version_string):
- '''Convert Verible version string to semantic versioning format.'''
- # Drop the hash suffix and convert into version string that
- # is compatible with StrictVersion in check_version below.
- # Example: v0.0-808-g1e17daa -> 0.0.808
- m = re.fullmatch(r'v([0-9]+)\.([0-9]+)-([0-9]+)-g[0-9a-f]+$', version_string)
-
- if m is None:
- log.error("{} has invalid version string format.".format(version_string))
- return None
-
- return '.'.join(m.group(1, 2, 3))
-
-
-def get_verible_version():
- '''Run Verible to check its version'''
- try:
- version_str = subprocess.run('verible-verilog-lint --version', shell=True,
- check=True, stdout=subprocess.PIPE,
- stderr=subprocess.STDOUT,
- universal_newlines=True)
- return version_str.stdout.split('\n')[0].strip()
-
- except subprocess.CalledProcessError as e:
- log.error("Unable to call Verible to check version: " + str(e))
- log.error(e.stdout)
- return None
-
-
-def pip3_get_version(tool):
- '''Run pip3 to find the version of an installed module'''
- cmd = ['pip3', 'show', tool]
- try:
- proc = subprocess.run(cmd,
- check=True,
- stderr=subprocess.STDOUT,
- stdout=subprocess.PIPE,
- universal_newlines=True)
- except subprocess.CalledProcessError as err:
- log.error('pip3 command failed: {}'.format(err))
- log.error("Failed to get version of {} with pip3: is it installed?"
- .format(tool))
- log.error(err.stdout)
- return None
-
- version_re = 'Version: (.*)'
- for line in proc.stdout.splitlines():
- match = re.match(version_re, line)
- if match:
- return match.group(1)
-
- # If we get here, we never saw a version line.
- log.error('No output line from running {} started with "Version: ".'
- .format(cmd))
- return None
-
-
-def check_version(requirements, tool_name, getter, version_converter=None):
- required_version = requirements.get(tool_name)
- if required_version is None:
- log.error('Requirements file does not specify version for {}.'
- .format(tool_name))
- return False
-
- actual_version = getter()
- if actual_version is None:
- return False
-
- # If a version string converter is defined, call it. This is required
- # for some version strings that are not compatible with StrictVersion.
- if version_converter is not None:
- required_version = version_converter(required_version)
- actual_version = version_converter(actual_version)
-
- if StrictVersion(actual_version) < StrictVersion(required_version):
- log.error("%s is too old: found version %s, need at least %s",
- tool_name, actual_version, required_version)
- return False
- else:
- log.info("Found sufficiently recent version of %s (found %s, need %s)",
- tool_name, actual_version, required_version)
- return True
-
-
def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument('tool', nargs='*')
+ args = parser.parse_args()
+
# Get tool requirements
- tool_requirements = read_tool_requirements()
- if tool_requirements is None:
+ try:
+ tool_requirements = read_tool_requirements()
+ except ReqErr as err:
+ log.error(str(err))
return 1
+ pending_tools = set(args.tool)
+ missing_tools = []
+ for tool, req in tool_requirements.items():
+ if req.as_needed and tool not in pending_tools:
+ continue
+ pending_tools.discard(tool)
+
+ good, msg = req.check()
+ if not good:
+ log.error('Failed tool requirement for {}: {}'
+ .format(tool, msg))
+ missing_tools.append(tool)
+ else:
+ log.info('Tool {} present: {}'
+ .format(tool, msg))
+
all_good = True
- all_good &= check_version(tool_requirements,
- 'verilator',
- get_verilator_version)
- all_good &= check_version(tool_requirements,
- 'verible',
- get_verible_version,
- convert_verible_version)
- all_good &= check_version(tool_requirements,
- 'edalize',
- lambda: pip3_get_version('edalize'))
-
- if not all_good:
+ if missing_tools:
log.error("Tool requirements not fulfilled. "
- "Please update the tools and retry.")
- return 1
+ "Please update tools ({}) and retry."
+ .format(', '.join(missing_tools)))
+ all_good = False
- return 0
+ if pending_tools:
+ log.error("Some tools specified on command line don't appear in "
+ "tool requirements file: {}"
+ .format(', '.join(sorted(pending_tools))))
+ all_good = False
+
+ return 0 if all_good else 1
if __name__ == "__main__":
