mirror of
https://github.com/openhwgroup/cve2.git
synced 2025-04-20 03:57:25 -04:00
[util] Manually "vendor" latest check_tool_requirements.py
This comes from OpenTitan and can't currently be vendored in properly (because it doesn't live in its own directory). We'll sort that out eventually but, for now, copy in some recent changes by hand.
This commit is contained in:
Rupert Swarbrick
Rupert Swarbrick
parent
c7f44557d2
commit
5d7b7c1e6f
1 changed files with 357 additions and 92 deletions
|
|
@ -1,12 +1,14 @@
|
|||
#!/usr/bin/python3
|
||||
#!/usr/bin/env python3
|
||||
# Copyright lowRISC contributors.
|
||||
# Licensed under the Apache License, Version 2.0, see LICENSE for details.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import argparse
|
||||
from distutils.version import StrictVersion
|
||||
import logging as log
|
||||
import os
|
||||
import re
|
||||
import shlex
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
|
|
@ -23,6 +25,298 @@ def get_tool_requirements_path():
|
|||
return os.path.join(top_src_dir, 'tool_requirements.py')
|
||||
|
||||
|
||||
class ReqErr(Exception):
|
||||
def __init__(self, path, msg):
|
||||
self.path = path
|
||||
self.msg = msg
|
||||
|
||||
def __str__(self):
|
||||
return ('Error parsing tool requirements from {!r}: {}'
|
||||
.format(self.path, self.msg))
|
||||
|
||||
|
||||
class ToolReq:
|
||||
# A subclass can set this to configure the command that's run to get the
|
||||
# version of a tool. If tool_cmd is None, get_version will call "self.tool
|
||||
# --version".
|
||||
tool_cmd = None
|
||||
|
||||
# Used by get_version. If not None, this is a dictionary that's added to
|
||||
# the environment when running the command.
|
||||
tool_env = None
|
||||
|
||||
# A subclass can set this to configure _parse_version_output. If set, it
|
||||
# should be a Regex object with a single capturing group that captures the
|
||||
# version.
|
||||
version_regex = None
|
||||
|
||||
def __init__(self, tool, min_version):
|
||||
self.tool = tool
|
||||
self.min_version = min_version
|
||||
self.optional = False
|
||||
|
||||
def _get_tool_cmd(self):
|
||||
'''Return the command to run to get the installed version'''
|
||||
return self.tool_cmd or [self.tool, '--version']
|
||||
|
||||
def _get_version(self):
|
||||
'''Run the tool to get the installed version.
|
||||
|
||||
Raises a RuntimeError on failure. The default version uses the class
|
||||
variable tool_cmd to figure out what to run.
|
||||
|
||||
'''
|
||||
|
||||
def _parse_version_output(self, stdout):
|
||||
'''Parse the nonempty stdout to get a version number
|
||||
|
||||
Raises a ValueError on failure. The default implementation returns the
|
||||
last word of the first line if version_regex is None or the first match
|
||||
for version_regex if it is not None.
|
||||
|
||||
'''
|
||||
if self.version_regex is None:
|
||||
line0 = stdout.split('\n', 1)[0]
|
||||
words = line0.rsplit(None, 1)
|
||||
if not words:
|
||||
raise ValueError('Empty first line.')
|
||||
|
||||
return words[-1]
|
||||
|
||||
for line in stdout.split('\n'):
|
||||
match = self.version_regex.match(line.rstrip())
|
||||
if match is not None:
|
||||
return match.group(1)
|
||||
raise ValueError('No line matched version regex.')
|
||||
|
||||
def get_version(self):
|
||||
'''Run the tool to get a version.
|
||||
|
||||
Returns a version string on success. Raises a RuntimeError on failure.
|
||||
The default version uses the class variable tool_cmd to figure out what
|
||||
to run.
|
||||
|
||||
'''
|
||||
cmd = self._get_tool_cmd()
|
||||
cmd_txt = ' '.join(shlex.quote(w) for w in cmd)
|
||||
|
||||
env = None
|
||||
if self.tool_env is not None:
|
||||
env = os.environ.copy()
|
||||
env.update(self.tool_env)
|
||||
|
||||
try:
|
||||
proc = subprocess.run(cmd,
|
||||
check=True,
|
||||
stdout=subprocess.PIPE,
|
||||
universal_newlines=True,
|
||||
env=env)
|
||||
except (subprocess.CalledProcessError, FileNotFoundError) as err:
|
||||
env_msg = ('' if not self.tool_env else
|
||||
' (with environment overrides: {})'
|
||||
.format(', '.join('{}={}'.format(k, v)
|
||||
for k, v in self.tool_env.items())))
|
||||
raise RuntimeError('Failed to run {!r}{} to check version: {}'
|
||||
.format(cmd_txt, env_msg, err))
|
||||
|
||||
if not proc.stdout:
|
||||
raise RuntimeError('No output from running {!r} to check version.'
|
||||
.format(cmd_txt))
|
||||
|
||||
try:
|
||||
return self._parse_version_output(proc.stdout)
|
||||
except ValueError as err:
|
||||
raise RuntimeError('Bad output from running {!r} '
|
||||
'to check version: {}'
|
||||
.format(cmd_txt, err))
|
||||
|
||||
def to_semver(self, version, from_req):
|
||||
'''Convert a tool version to semantic versioning format
|
||||
|
||||
If from_req is true, this version comes from the requirements file
|
||||
(rather than being reported from an installed application). That might
|
||||
mean stricter checking. If version is not a known format, raises a
|
||||
ValueError.
|
||||
|
||||
'''
|
||||
return version
|
||||
|
||||
def check(self):
|
||||
'''Get the installed version and check it matches the requirements
|
||||
|
||||
Returns (is_good, msg). is_good is true if we matched the requirements
|
||||
and false otherwise. msg describes what happened (an error message on
|
||||
failure, or extra information on success).
|
||||
|
||||
'''
|
||||
try:
|
||||
min_semver = self.to_semver(self.min_version, True)
|
||||
except ValueError as err:
|
||||
return (False,
|
||||
'Failed to convert requirement to semantic version: {}'
|
||||
.format(err))
|
||||
try:
|
||||
min_sv = StrictVersion(min_semver)
|
||||
except ValueError as err:
|
||||
return (False,
|
||||
'Bad semver inferred from required version ({}): {}'
|
||||
.format(min_semver, err))
|
||||
|
||||
try:
|
||||
actual_version = self.get_version()
|
||||
except RuntimeError as err:
|
||||
return (False, str(err))
|
||||
|
||||
try:
|
||||
actual_semver = self.to_semver(actual_version, False)
|
||||
except ValueError as err:
|
||||
return (False,
|
||||
'Failed to convert installed to semantic version: {}'
|
||||
.format(err))
|
||||
try:
|
||||
actual_sv = StrictVersion(actual_semver)
|
||||
except ValueError as err:
|
||||
return (False,
|
||||
'Bad semver inferred from installed version ({}): {}'
|
||||
.format(actual_semver, err))
|
||||
|
||||
if actual_sv < min_sv:
|
||||
return (False,
|
||||
'Installed version is too old: '
|
||||
'found version {}, but need at least {}'
|
||||
.format(actual_version, self.min_version))
|
||||
|
||||
return (True,
|
||||
'Sufficiently recent version (found {}; needed {})'
|
||||
.format(actual_version, self.min_version))
|
||||
|
||||
|
||||
class VerilatorToolReq(ToolReq):
|
||||
def get_version(self):
|
||||
try:
|
||||
# Note: "verilator" needs to be called through a shell and with all
|
||||
# arguments in a string, as it doesn't have a shebang, but instead
|
||||
# relies on perl magic to parse command line arguments.
|
||||
version_str = subprocess.run('verilator --version', shell=True,
|
||||
check=True, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
universal_newlines=True)
|
||||
except subprocess.CalledProcessError as err:
|
||||
raise RuntimeError('Unable to call Verilator to check version: {}'
|
||||
.format(err)) from None
|
||||
|
||||
return version_str.stdout.split(' ')[1].strip()
|
||||
|
||||
|
||||
class VeribleToolReq(ToolReq):
|
||||
tool_cmd = ['verible-verilog-lint', '--version']
|
||||
|
||||
def to_semver(self, version, from_req):
|
||||
# Drop the hash suffix and convert into version string that
|
||||
# is compatible with StrictVersion in check_version below.
|
||||
# Example: v0.0-808-g1e17daa -> 0.0.808
|
||||
m = re.fullmatch(r'v([0-9]+)\.([0-9]+)-([0-9]+)-g[0-9a-f]+$', version)
|
||||
if m is None:
|
||||
raise ValueError("{} has invalid version string format."
|
||||
.format(version))
|
||||
|
||||
return '.'.join(m.group(1, 2, 3))
|
||||
|
||||
|
||||
class VcsToolReq(ToolReq):
|
||||
tool_cmd = ['vcs', '-full64', '-ID']
|
||||
tool_env = {'VCS_ARCH_OVERRIDE': 'linux'}
|
||||
version_regex = re.compile(r'Compiler version = VCS [A-Z]-(.*)')
|
||||
|
||||
def to_semver(self, version, from_req):
|
||||
# VCS has a rather strange numbering scheme, where the most general
|
||||
# versions look something like this:
|
||||
#
|
||||
# Q-2020.03-SP1-2
|
||||
#
|
||||
# Our version_regex strips out the "Q" part (a "platform prefix")
|
||||
# already. A version always has the "2020.03" (year and month) part,
|
||||
# and may also have an -SP<n> and/or -<patch> suffix.
|
||||
#
|
||||
# Since StrictVersion expects a 3 digit versioning scheme, we multiply
|
||||
# any SP number by 100, which should work as long as the patch version
|
||||
# isn't greater than 99.
|
||||
#
|
||||
# Some VCS builds also report other cruft (like _Full64) after this
|
||||
# version number. If from_req is False, allow (and ignore) that too.
|
||||
regex = r'([0-9]+).([0-9]+)(?:-SP([0-9]+))?(?:-([0-9]+))?'
|
||||
if from_req:
|
||||
regex += '$'
|
||||
|
||||
match = re.match(regex, version)
|
||||
if match is None:
|
||||
raise ValueError("{!r} is not a recognised VCS version string."
|
||||
.format(version))
|
||||
major = match.group(1)
|
||||
minor = match.group(2)
|
||||
sp = int(match.group(3) or 0)
|
||||
patch = int(match.group(4) or 0)
|
||||
comb = str(sp * 100 + patch)
|
||||
return '{}.{}{}'.format(major, minor, comb)
|
||||
|
||||
|
||||
class PyModuleToolReq(ToolReq):
|
||||
'''A tool in a Python module (its version can be found by running pip)'''
|
||||
version_regex = re.compile(r'Version: (.*)')
|
||||
|
||||
def _get_tool_cmd(self):
|
||||
return ['pip3', 'show', self.tool]
|
||||
|
||||
|
||||
def dict_to_tool_req(path, tool, raw):
|
||||
'''Parse a dict (as read from Python) as a ToolReq
|
||||
|
||||
Required keys: version. Optional keys: as_needed.
|
||||
|
||||
'''
|
||||
where = 'Dict for {} in __TOOL_REQUIREMENTS__'.format(tool)
|
||||
# We operate in place on the dictionary. Take a copy to avoid an
|
||||
# obnoxious API.
|
||||
raw = raw.copy()
|
||||
|
||||
if 'min_version' not in raw:
|
||||
raise ReqErr(path,
|
||||
'{} is missing required key: "min_version".'
|
||||
.format(where))
|
||||
min_version = raw['min_version']
|
||||
if not isinstance(min_version, str):
|
||||
raise ReqErr(path,
|
||||
'{} has min_version that is not a string.'
|
||||
.format(where))
|
||||
del raw['min_version']
|
||||
|
||||
as_needed = False
|
||||
if 'as_needed' in raw:
|
||||
as_needed = raw['as_needed']
|
||||
if not isinstance(as_needed, bool):
|
||||
raise ReqErr(path,
|
||||
'{} has as_needed that is not a bool.'
|
||||
.format(where))
|
||||
del raw['as_needed']
|
||||
|
||||
if raw:
|
||||
raise ReqErr(path,
|
||||
'{} has unexpected keys: {}.'
|
||||
.format(where, ', '.join(raw.keys())))
|
||||
|
||||
classes = {
|
||||
'edalize': PyModuleToolReq,
|
||||
'vcs': VcsToolReq,
|
||||
'verible': VeribleToolReq,
|
||||
'verilator': VerilatorToolReq
|
||||
}
|
||||
cls = classes.get(tool, ToolReq)
|
||||
|
||||
ret = cls(tool, min_version)
|
||||
ret.as_needed = as_needed
|
||||
return ret
|
||||
|
||||
|
||||
def read_tool_requirements(path=None):
|
||||
'''Read tool requirements from a Python file'''
|
||||
if path is None:
|
||||
|
|
@ -34,109 +328,80 @@ def read_tool_requirements(path=None):
|
|||
|
||||
# We expect the exec call to have populated globs with a
|
||||
# __TOOL_REQUIREMENTS__ dictionary.
|
||||
reqs = globs.get('__TOOL_REQUIREMENTS__')
|
||||
if reqs is None:
|
||||
log.error('The Python file at {} did not define '
|
||||
'__TOOL_REQUIREMENTS__.'
|
||||
.format(path))
|
||||
return None
|
||||
raw = globs.get('__TOOL_REQUIREMENTS__')
|
||||
if raw is None:
|
||||
raise ReqErr(path,
|
||||
'The Python file at did not define '
|
||||
'__TOOL_REQUIREMENTS__.')
|
||||
|
||||
# reqs should be a dictionary (mapping tool name to minimum version)
|
||||
if not isinstance(reqs, dict):
|
||||
log.error('The Python file at {} defined '
|
||||
'__TOOL_REQUIREMENTS__, but it is not a dict.'
|
||||
.format(path))
|
||||
return None
|
||||
# raw should be a dictionary (keyed by tool name)
|
||||
if not isinstance(raw, dict):
|
||||
raise ReqErr(path, '__TOOL_REQUIREMENTS__ is not a dict.')
|
||||
|
||||
reqs = {}
|
||||
for tool, raw_val in raw.items():
|
||||
if not isinstance(tool, str):
|
||||
raise ReqErr(path,
|
||||
'Invalid key in __TOOL_REQUIREMENTS__: {!r}'
|
||||
.format(tool))
|
||||
|
||||
if isinstance(raw_val, str):
|
||||
# Shorthand notation: value is just a string, which we
|
||||
# interpret as a minimum version
|
||||
raw_val = {'min_version': raw_val}
|
||||
|
||||
if not isinstance(raw_val, dict):
|
||||
raise ReqErr(path,
|
||||
'Value for {} in __TOOL_REQUIREMENTS__ '
|
||||
'is not a string or dict.'.format(tool))
|
||||
|
||||
reqs[tool] = dict_to_tool_req(path, tool, raw_val)
|
||||
|
||||
return reqs
|
||||
|
||||
|
||||
def get_verilator_version():
|
||||
try:
|
||||
# Note: "verilator" needs to be called through a shell and with all
|
||||
# arguments in a string, as it doesn't have a shebang, but instead
|
||||
# relies on perl magic to parse command line arguments.
|
||||
version_str = subprocess.run('verilator --version', shell=True,
|
||||
check=True, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
universal_newlines=True)
|
||||
return version_str.stdout.split(' ')[1].strip()
|
||||
|
||||
except subprocess.CalledProcessError as e:
|
||||
log.error("Unable to call Verilator to check version: " + str(e))
|
||||
log.error(e.stdout)
|
||||
return None
|
||||
|
||||
|
||||
def pip3_get_version(tool):
|
||||
'''Run pip3 to find the version of an installed module'''
|
||||
cmd = ['pip3', 'show', tool]
|
||||
try:
|
||||
proc = subprocess.run(cmd,
|
||||
check=True,
|
||||
stderr=subprocess.STDOUT,
|
||||
stdout=subprocess.PIPE,
|
||||
universal_newlines=True)
|
||||
except subprocess.CalledProcessError as err:
|
||||
log.error('pip3 command failed: {}'.format(err))
|
||||
log.error("Failed to get version of {} with pip3: is it installed?"
|
||||
.format(tool))
|
||||
log.error(err.stdout)
|
||||
return None
|
||||
|
||||
version_re = 'Version: (.*)'
|
||||
for line in proc.stdout.splitlines():
|
||||
match = re.match(version_re, line)
|
||||
if match:
|
||||
return match.group(1)
|
||||
|
||||
# If we get here, we never saw a version line.
|
||||
log.error('No output line from running {} started with "Version: ".'
|
||||
.format(cmd))
|
||||
return None
|
||||
|
||||
|
||||
def check_version(requirements, tool_name, getter):
|
||||
required_version = requirements.get(tool_name)
|
||||
if required_version is None:
|
||||
log.error('Requirements file does not specify version for {}.'
|
||||
.format(tool_name))
|
||||
return False
|
||||
|
||||
actual_version = getter()
|
||||
if actual_version is None:
|
||||
return False
|
||||
|
||||
if StrictVersion(actual_version) < StrictVersion(required_version):
|
||||
log.error("%s is too old: found version %s, need at least %s",
|
||||
tool_name, actual_version, required_version)
|
||||
return False
|
||||
else:
|
||||
log.info("Found sufficiently recent version of %s (found %s, need %s)",
|
||||
tool_name, actual_version, required_version)
|
||||
return True
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('tool', nargs='*')
|
||||
args = parser.parse_args()
|
||||
|
||||
# Get tool requirements
|
||||
tool_requirements = read_tool_requirements()
|
||||
if tool_requirements is None:
|
||||
try:
|
||||
tool_requirements = read_tool_requirements()
|
||||
except ReqErr as err:
|
||||
log.error(str(err))
|
||||
return 1
|
||||
|
||||
pending_tools = set(args.tool)
|
||||
missing_tools = []
|
||||
for tool, req in tool_requirements.items():
|
||||
if req.as_needed and tool not in pending_tools:
|
||||
continue
|
||||
pending_tools.discard(tool)
|
||||
|
||||
good, msg = req.check()
|
||||
if not good:
|
||||
log.error('Failed tool requirement for {}: {}'
|
||||
.format(tool, msg))
|
||||
missing_tools.append(tool)
|
||||
else:
|
||||
log.info('Tool {} present: {}'
|
||||
.format(tool, msg))
|
||||
|
||||
all_good = True
|
||||
all_good &= check_version(tool_requirements,
|
||||
'verilator',
|
||||
get_verilator_version)
|
||||
all_good &= check_version(tool_requirements,
|
||||
'edalize',
|
||||
lambda: pip3_get_version('edalize'))
|
||||
|
||||
if not all_good:
|
||||
if missing_tools:
|
||||
log.error("Tool requirements not fulfilled. "
|
||||
"Please update the tools and retry.")
|
||||
return 1
|
||||
"Please update tools ({}) and retry."
|
||||
.format(', '.join(missing_tools)))
|
||||
all_good = False
|
||||
|
||||
return 0
|
||||
if pending_tools:
|
||||
log.error("Some tools specified on command line don't appear in "
|
||||
"tool requirements file: {}"
|
||||
.format(', '.join(sorted(pending_tools))))
|
||||
all_good = False
|
||||
|
||||
return 0 if all_good else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue