github-mirrors/ibex
1
0
Fork 0
mirror of https://github.com/lowRISC/ibex.git synced 2025-04-23 13:27:10 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

[uvm,core_ibex] Run comparisons in parallel

Browse source 
This patch teaches Make which tests we're actually running (via the
list_tests.py script), which means that we can compare the ISS and RTL
results in parallel rather than serially.

There's a bit of duplicated code (both list_tests.py and sim.py
currently contain the code to get a list of tests and then filter by
Ibex configuration), but this should go away with a later patch that
runs the RTL simulations in parallel in a similar way.

Note: This might seem a little silly: trace comparison takes way less
time than the RTL simulation! The point is that it's probably easier
to work "from the bottom" than to start by parallelising the
simulations themselves.
This commit is contained in:
Rupert Swarbrick 2021-06-02 10:51:25 +01:00 • committed by Rupert Swarbrick
parent b40f5b8f55
commit 700f29b7b3
5 changed files with 466 additions and 150 deletions
Download patch file Download diff file Expand all files Collapse all files

53
dv/uvm/core_ibex/Makefile
View file

@ -210,6 +210,18 @@ FORCE:
# Call it as $(call vars-prereq,X,TGT,VS)
vars-prereq = $(if $(call vars-differ,$(1),$(2),$(3)),FORCE,)
###############################################################################
# Get a list of tests and seeds
#
# Run list_tests.py to list the things we need to run in the format
# TESTNAME.SEED and store it in a variable.
tests-and-seeds := \
$(shell ./list_tests.py \
--start_seed $(SEED) \
--test "$(TEST)" \
--iterations $(ITERATIONS) \
--ibex-config $(IBEX_CONFIG))
###############################################################################
# Generate random instructions
#
@ -402,17 +414,36 @@ rtl_sim: $(metadata)/rtl_sim.run.stamp
###############################################################################
# Compare ISS and RTL sim results
$(OUT-SEED)/regr.log: \
$(metadata)/instr_gen.iss.stamp \
$(metadata)/rtl_sim.run.stamp $(TESTLIST)
$(verb)rm -f $@
$(verb)./sim.py \
--o=$(OUT-SEED) \
--steps=compare \
--ibex_config=$(IBEX_CONFIG) \
${TEST_OPTS} \
--simulator="${SIMULATOR}" \
--iss="${ISS}"
#
# For a given TEST/SEED pair, the ISS and RTL logs appear at:
#
# $(OUT-SEED)/instr_gen/$(ISS)_sim/$(TEST).$(SEED).log
# $(OUT-SEED)/rtl_sim/$(TEST).$(SEED)/trace_core_00000000.log
#
# The comparison script compares these and writes to a result file at
#
# $(OUT-SEED)/rtl_sim/$(TEST).$(SEED)/comparison-result.txt
#
# with PASSED or FAILED, depending.
rtl-sim-dirs := $(addprefix $(OUT-SEED)/rtl_sim/,$(tests-and-seeds))
comp-results := $(addsuffix /comparison-result.txt,$(rtl-sim-dirs))
$(comp-results): \
%/comparison-result.txt: \
compare.py $(metadata)/instr_gen.iss.stamp $(metadata)/rtl_sim.run.stamp
@echo Comparing traces for $*
$(verb)./compare.py \
--iss $(ISS) \
--iss-log-dir $(OUT-SEED)/instr_gen/$(ISS)_sim \
--start-seed $(SEED) \
--test-dot-seed "$(notdir $*)" \
--output $@ \
--rtl-log-dir $(OUT-SEED)/rtl_sim/$(notdir $*)
$(OUT-SEED)/regr.log: collect_results.py $(comp-results)
@echo "Collecting up results (report at $@)"
$(verb)./collect_results.py -o $@ $(comp-results)
.PHONY: post_compare
post_compare: $(OUT-SEED)/regr.log

57
dv/uvm/core_ibex/collect_results.py Executable file
View file

@ -0,0 +1,57 @@
#!/usr/bin/env python3
# Copyright lowRISC contributors.
# Licensed under the Apache License, Version 2.0, see LICENSE for details.
# SPDX-License-Identifier: Apache-2.0
import argparse
import sys
def parse_log(path: str) -> bool:
first_line = '(empty file)'
with open(path, 'r', encoding='UTF-8') as log:
for line in log:
first_line = line.rstrip()
break
if first_line == 'PASS':
return True
if first_line.startswith('FAIL'):
return False
raise RuntimeError('Strange first line ({!r})'.format(first_line))
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument('--output', '-o', required=True)
parser.add_argument('log', nargs='*')
args = parser.parse_args()
fail_count = 0
pass_count = 0
for log_path in args.log:
try:
passed = parse_log(log_path)
except RuntimeError as e:
print(f'Failed to parse run results at {log_path:!r}: {e}',
file=sys.stderr)
passed = False
if passed:
pass_count += 1
else:
fail_count += 1
msg = '{} PASSED, {} FAILED'.format(pass_count, fail_count)
with open(args.output, 'w', encoding='UTF-8') as outfile:
print(msg, file=outfile)
print(msg)
# Succeed if no tests failed
return 1 if fail_count else 0
if __name__ == '__main__':
sys.exit(main())

225
dv/uvm/core_ibex/compare.py Executable file
View file

@ -0,0 +1,225 @@
#!/usr/bin/env python3
# Copyright lowRISC contributors.
# Licensed under the Apache License, Version 2.0, see LICENSE for details.
# SPDX-License-Identifier: Apache-2.0
'''
A script to compare an ISS and RTL run to make sure nothing has diverged.
'''
import argparse
import os
import re
import sys
from typing import Dict, List, Optional, TextIO, Tuple
_CORE_IBEX = os.path.normpath(os.path.join(os.path.dirname(__file__)))
_IBEX_ROOT = os.path.normpath(os.path.join(_CORE_IBEX, '../../..'))
_RISCV_DV_ROOT = os.path.join(_IBEX_ROOT, 'vendor/google_riscv-dv')
_OLD_SYS_PATH = sys.path
# Import riscv_trace_csv and lib from _DV_SCRIPTS before putting sys.path back
# as it started.
try:
sys.path = ([os.path.join(_CORE_IBEX, 'riscv_dv_extension'),
os.path.join(_RISCV_DV_ROOT, 'scripts')] +
sys.path)
from lib import process_regression_list # type: ignore
from spike_log_to_trace_csv import process_spike_sim_log # type: ignore
from ovpsim_log_to_trace_csv import process_ovpsim_sim_log # type: ignore
from instr_trace_compare import compare_trace_csv # type: ignore
from ibex_log_to_trace_csv import (process_ibex_sim_log, # type: ignore
check_ibex_uvm_log)
finally:
sys.path = _OLD_SYS_PATH
_TestEntry = Dict[str, object]
_TestEntries = List[_TestEntry]
_TestAndSeed = Tuple[str, int]
_CompareResult = Tuple[bool, Optional[str], Dict[str, str]]
def read_test_dot_seed(arg: str) -> _TestAndSeed:
'''Read a value for --test-dot-seed'''
match = re.match(r'([^.]+)\.([0-9]+)$', arg)
if match is None:
raise argparse.ArgumentTypeError('Bad --test-dot-seed ({}): '
'should be of the form TEST.SEED.'
.format(arg))
return (match.group(1), int(match.group(2), 10))
def get_test_entry(testname: str) -> _TestEntry:
matched_list = [] # type: _TestEntries
testlist = os.path.join(_CORE_IBEX, 'riscv_dv_extension', 'testlist.yaml')
process_regression_list(testlist, 'all', 0, matched_list, _RISCV_DV_ROOT)
for entry in matched_list:
if entry['test'] == testname:
return entry
raise RuntimeError('No matching test entry for {!r}'.format(testname))
def compare_test_run(test: _TestEntry,
idx: int,
seed: int,
rtl_log_dir: str,
iss: str,
iss_log_dir: str) -> _CompareResult:
'''Compare results for a single run of a single test
Here, test is a dictionary describing the test (read from the testlist YAML
file). idx is the iteration index and seed is the corresponding seed. iss
is the chosen instruction set simulator (currently supported: spike and
ovpsim).
rtl_log_dir is the directory containing log output from the RTL simulation.
iss_log_dir is the directory that contains logs for ISS runs.
Returns a _CompareResult with a pass/fail flag, together with some
information about the run (to be written to the log file).
'''
test_name = test['test']
assert isinstance(test_name, str)
uvm_log = os.path.join(rtl_log_dir, 'sim.log')
kv_data = {
'test name': test_name,
'iteration': str(idx),
'seed': str(seed),
'UVM log': uvm_log
}
# Have a look at the UVM log. Report a failure if an issue is seen in the
# log.
uvm_pass, uvm_log_lines = check_ibex_uvm_log(uvm_log)
if not uvm_pass:
return (False, 'simulation error', kv_data)
rtl_log = os.path.join(rtl_log_dir, 'trace_core_00000000.log')
rtl_csv = os.path.join(rtl_log_dir, 'trace_core_00000000.csv')
kv_data['rtl log'] = rtl_log
kv_data['rtl csv'] = rtl_csv
try:
# Convert the RTL log file to a trace CSV.
process_ibex_sim_log(rtl_log, rtl_csv, 1)
except (OSError, RuntimeError) as e:
return (False, f'RTL log processing failed ({e})', kv_data)
no_post_compare = test.get('no_post_compare', False)
assert isinstance(no_post_compare, bool)
# no_post_compare skips the final ISS v RTL log check, so if we've reached
# here we're done when no_post_compare is set.
if no_post_compare:
return (True, None, kv_data)
# There were no UVM errors. Process the log file from the ISS.
iss_log = os.path.join(iss_log_dir, '{}.{}.log'.format(test_name, idx))
iss_csv = os.path.join(iss_log_dir, '{}.{}.csv'.format(test_name, idx))
kv_data['ISS log'] = iss_log
kv_data['ISS csv'] = iss_csv
try:
if iss == "spike":
process_spike_sim_log(iss_log, iss_csv)
else:
assert iss == 'ovpsim' # (should be checked by argparse)
process_ovpsim_sim_log(iss_log, iss_csv)
except (OSError, RuntimeError) as e:
return (False, f'ISS log processing failed ({e})', kv_data)
compare_log = os.path.join(rtl_log_dir, 'compare.log')
kv_data['comparison log'] = compare_log
# Delete any existing file at compare_log (the compare_trace_csv function
# would append to it, which is rather confusing).
try:
os.remove(compare_log)
except FileNotFoundError:
pass
compare_result = \
compare_trace_csv(rtl_csv, iss_csv, "ibex", iss, compare_log,
**test.get('compare_opts', {}))
# Rather oddly, compare_result is a string. The comparison passed if it
# starts with '[PASSED]' and failed otherwise.
compare_passed = compare_result.startswith('[PASSED]: ')
if not compare_passed:
assert compare_result.startswith('[FAILED]: ')
# compare_result[10:] will look like "123 matched, 321 mismatch",
# meaning that 123 instructions matched and 321 instructions didn't.
kv_data['compared instructions'] = compare_result[10:]
return (False, 'mismatch between ISS and RTL', kv_data)
# compare_result[10:] will look like "123 matched", meaning that 123
# instructions matched.
kv_data['compared instructions'] = compare_result[10:]
return (True, None, kv_data)
def on_result(result: _CompareResult, output: TextIO) -> None:
passed, err_msg, kv_data = result
if passed:
assert err_msg is None
output.write('PASS\n\n')
else:
assert err_msg is not None
output.write('FAIL\n\n')
output.write(f'Test failed: {err_msg}\n')
output.write('---\n\n')
klen = 1
for k in kv_data:
klen = max(klen, len(k))
for k, v in kv_data.items():
kpad = ' ' * (klen - len(k))
output.write(f'{k}:{kpad} | {v}\n')
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument('--iss', required=True, choices=['spike', 'ovpsim'])
parser.add_argument('--iss-log-dir', required=True)
parser.add_argument('--start-seed', type=int, required=True)
parser.add_argument('--test-dot-seed',
type=read_test_dot_seed,
required=True)
parser.add_argument('--rtl-log-dir', required=True)
parser.add_argument('--output', required=True)
args = parser.parse_args()
if args.start_seed < 0:
raise RuntimeError('Invalid start seed: {}'.format(args.start_seed))
testname, seed = args.test_dot_seed
if seed < args.start_seed:
raise RuntimeError('Start seed is greater than test seed '
f'({args.start_seed} > {seed}).')
iteration = seed - args.start_seed
entry = get_test_entry(testname)
result = compare_test_run(entry, iteration, seed,
args.rtl_log_dir, args.iss, args.iss_log_dir)
with open(args.output, 'w', encoding='UTF-8') as outfile:
on_result(result, outfile)
# Always return 0 (success), even if the test failed. We've successfully
# generated a comparison log either way and we don't want to stop Make from
# gathering them all up for us.
return 0
if __name__ == '__main__':
sys.exit(main())

135
dv/uvm/core_ibex/list_tests.py Executable file
View file

@ -0,0 +1,135 @@
#!/usr/bin/env python3
# Copyright lowRISC contributors.
# Licensed under the Apache License, Version 2.0, see LICENSE for details.
# SPDX-License-Identifier: Apache-2.0
import argparse
import os
import sys
from typing import Dict, List
_CORE_IBEX = os.path.normpath(os.path.join(os.path.dirname(__file__)))
_IBEX_ROOT = os.path.normpath(os.path.join(_CORE_IBEX, '../../..'))
_RISCV_DV_ROOT = os.path.join(_IBEX_ROOT, 'vendor/google_riscv-dv')
_OLD_SYS_PATH = sys.path
# Import riscv_trace_csv and lib from _DV_SCRIPTS before putting sys.path back
# as it started.
try:
sys.path = ([os.path.join(_CORE_IBEX, 'riscv_dv_extension'),
os.path.join(_IBEX_ROOT, 'util'),
os.path.join(_RISCV_DV_ROOT, 'scripts')] +
sys.path)
from lib import process_regression_list # type: ignore
from ibex_config import parse_config # type: ignore
finally:
sys.path = _OLD_SYS_PATH
_TestEntry = Dict[str, object]
_TestEntries = List[_TestEntry]
def filter_tests_by_config(cfg: str, test_list: _TestEntries) -> _TestEntries:
'''Filter out any unsupported tests from being executed.
This function will parse the set of RTL parameters required by a given
test (if any) and ensure that those parameters are supported by the
selected core config.
Doing this allows the run flow to be smarter about running regressions
with different configs (useful for CI flows).
Arguments:
cfg: string name of the ibex config being tested, should match a
config name from ibex_configs.yaml.
test_list: list of test entry objects parsed from the YAML testlist
Returns:
filtered_test_list: a list of test entry objects, filtered such that
all tests incompatible with the specified ibex
config have been removed.
e.g. if the "small" config has been specified, this
function will filter out all tests that require
B-extension and PMP parameters
'''
filtered_test_list = []
config = parse_config(cfg, os.path.join(_IBEX_ROOT, "ibex_configs.yaml"))
for test in test_list:
good = True
if "rtl_params" in test:
param_dict = test['rtl_params']
assert isinstance(param_dict, dict)
for p, p_val in param_dict.items():
config_val = config.get(p, None)
# Throw an error if required RTL parameters in the testlist
# have been formatted incorrectly (typos, wrong parameters,
# etc)
if config_val is None:
raise ValueError('Parameter {} not found in config {}'
.format(p, cfg))
# Ibex has some enum parameters, so as a result some tests are
# able to run with several of these parameter values (like
# bitmanipulation tests). If this is the case, the testlist
# will specify all legal enum values, check if any of them
# match the config.
if isinstance(p_val, list):
good = (config_val in p_val)
else:
good = (p_val == config_val)
# If there is any parameter mismatch, we can terminate
# immediately and exclude the test from being executed
if not good:
break
if good:
filtered_test_list.append(test)
return filtered_test_list
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument('--start_seed', type=int, default=1)
parser.add_argument('--test', required=True)
parser.add_argument('--iterations', type=int, default=0)
parser.add_argument('--ibex-config', required=True)
args = parser.parse_args()
if args.iterations < 0:
raise RuntimeError('Bad --iterations argument: must be non-negative')
if args.start_seed < 0:
raise RuntimeError('Bad --start_seed argument: must be non-negative')
# Get all the tests that match --test, scaling as necessary with the
# --iterations argument.
matched_list = [] # type: _TestEntries
testlist = os.path.join(_CORE_IBEX, 'riscv_dv_extension', 'testlist.yaml')
process_regression_list(testlist, args.test, args.iterations,
matched_list, _RISCV_DV_ROOT)
if not matched_list:
raise RuntimeError("Cannot find {} in {}".format(args.test, testlist))
# Filter tests by the chosen configuration
matched_list = filter_tests_by_config(args.ibex_config, matched_list)
# Print the tests crossed by seeds, one to a line, in the format TEST.SEED.
for test in matched_list:
name = test['test']
iterations = test['iterations']
assert isinstance(name, str) and isinstance(iterations, int)
assert iterations > 0
for iteration in range(iterations):
print('{}.{}'.format(name, args.start_seed + iteration))
return 0
if __name__ == '__main__':
sys.exit(main())

146
dv/uvm/core_ibex/sim.py
View file

@ -40,12 +40,6 @@ try:
setup_logging, RET_SUCCESS, RET_FAIL)
import logging
from spike_log_to_trace_csv import process_spike_sim_log
from ovpsim_log_to_trace_csv import process_ovpsim_sim_log
from instr_trace_compare import compare_trace_csv
from ibex_log_to_trace_csv import process_ibex_sim_log, check_ibex_uvm_log
from ibex_config import parse_config
finally:
@ -408,125 +402,6 @@ def rtl_sim(sim_cmd, test_list,
run_sim_commands(cmd_list, lsf_cmd is not None)
def compare_test_run(test, idx, seed, iss, output_dir, report):
'''Compare results for a single run of a single test
Here, test is a dictionary describing the test (read from the testlist YAML
file). idx is the iteration index and seed is the corresponding seed. iss
is the chosen instruction set simulator (currently supported: spike and
ovpsim). output_dir is the base output directory (which should contain logs
from both the ISS and the test run itself). report is the path to the
regression report file we're writing.
Returns True if the test run passed and False otherwise.
'''
with open(report, 'a') as report_fd:
test_name = test['test']
elf = os.path.join(output_dir,
'instr_gen/asm_test/{}_{}.o'.format(test_name, idx))
logging.info("Comparing %s/DUT sim result : %s" % (iss, elf))
test_name_idx = '{}.{}'.format(test_name, idx)
test_underline = '-' * len(test_name_idx)
report_fd.write('\n{}\n{}\n'.format(test_name_idx, test_underline))
report_fd.write('Test binary: {}\n'.format(elf))
rtl_dir = os.path.join(output_dir, 'rtl_sim',
'{}.{}'.format(test_name, seed))
uvm_log = os.path.join(rtl_dir, 'sim.log')
# Have a look at the UVM log. Report a failure if an issue is seen in the
# log.
uvm_pass, uvm_log_lines = check_ibex_uvm_log(uvm_log)
report_fd.write('sim log: {}\n'.format(uvm_log))
if not uvm_pass:
for line in uvm_log_lines:
report_fd.write(line)
report_fd.write('[FAILED]: sim error seen\n')
return False
rtl_log = os.path.join(rtl_dir, 'trace_core_00000000.log')
rtl_csv = os.path.join(rtl_dir, 'trace_core_00000000.csv')
try:
# Convert the RTL log file to a trace CSV.
process_ibex_sim_log(rtl_log, rtl_csv, 1)
except (OSError, RuntimeError) as e:
report_fd.write('[FAILED]: Log processing failed: {}\n'.format(e))
return False
no_post_compare = test.get('no_post_compare', False)
assert isinstance(no_post_compare, bool)
# no_post_compare skips the final ISS v RTL log check, so if we've reached
# here we're done when no_post_compare is set.
if no_post_compare:
report_fd.write('[PASSED]\n')
return True
# There were no UVM errors. Process the log file from the ISS.
iss_dir = os.path.join(output_dir, 'instr_gen', '{}_sim'.format(iss))
iss_log = os.path.join(iss_dir, '{}.{}.log'.format(test_name, idx))
iss_csv = os.path.join(iss_dir, '{}.{}.csv'.format(test_name, idx))
try:
if iss == "spike":
process_spike_sim_log(iss_log, iss_csv)
else:
assert iss == 'ovpsim' # (should be checked by argparse)
process_ovpsim_sim_log(iss_log, iss_csv)
except (OSError, RuntimeError) as e:
report_fd.write('[FAILED]: Log processing failed: {}\n'.format(e))
return False
compare_result = \
compare_trace_csv(rtl_csv, iss_csv, "ibex", iss, report,
**test.get('compare_opts', {}))
# Rather oddly, compare_result is a string. The comparison passed if it
# starts with '[PASSED]'.
return compare_result.startswith('[PASSED]')
def compare(test_list, iss, start_seed, output_dir):
"""Compare RTL & ISS simulation reult
Here, test_list is a list of tests read from the testlist YAML file. iss is
the instruction set simulator that was used (must be 'spike' or 'ovpsim').
start_seed is the seed for index zero. output_dir is the output directory
which contains the results and where we'll write the regression log.
"""
report = os.path.join(output_dir, 'regr.log')
passes = 0
fails = 0
for test in test_list:
for idx in range(test['iterations']):
if compare_test_run(test, idx,
start_seed + idx, iss, output_dir, report):
passes += 1
else:
fails += 1
summary = "\n{} PASSED, {} FAILED".format(passes, fails)
with open(report, 'a') as report_fd:
report_fd.write(summary + '\n')
logging.info(summary)
logging.info("RTL & ISS regression report at {}".format(report))
return fails == 0
#TODO(udinator) - support DSim, and Riviera
def gen_cov(base_dir, simulator, lsf_cmd):
"""Generate a merged coverage directory.
@ -616,7 +491,7 @@ def main():
parser.add_argument("--en_wave", action='store_true',
help="Enable waveform dump")
parser.add_argument("--steps", type=str, default="all",
help="Run steps: compile,sim,compare,cov")
help="Run steps: compile,sim,cov")
parser.add_argument("--lsf_cmd", type=str,
help=("LSF command. Run locally if lsf "
"command is not specified"))
@ -658,7 +533,6 @@ def main():
steps = {
'compile': args.steps == "all" or 'compile' in args.steps,
'sim': args.steps == "all" or 'sim' in args.steps,
'compare': args.steps == "all" or 'compare' in args.steps,
'cov': args.steps == "all" or 'cov' in args.steps
}
@ -674,7 +548,12 @@ def main():
compile_cmds, sim_cmd = get_simulator_cmd(args.simulator,
args.simulator_yaml, enables)
if steps['sim'] or steps['compare']:
# Compile TB
if steps['compile']:
rtl_compile(compile_cmds, output_dir, args.lsf_cmd, args.cmp_opts)
# Run RTL simulation
if steps['sim']:
process_regression_list(args.testlist, args.test, args.iterations or 0,
matched_list, _RISCV_DV_ROOT)
if not matched_list:
@ -683,20 +562,9 @@ def main():
matched_list = filter_tests_by_config(args.ibex_config, matched_list)
# Compile TB
if steps['compile']:
rtl_compile(compile_cmds, output_dir, args.lsf_cmd, args.cmp_opts)
# Run RTL simulation
if steps['sim']:
rtl_sim(sim_cmd, matched_list, args.start_seed,
args.sim_opts, output_dir, bin_dir, args.lsf_cmd)
# Compare RTL & ISS simulation result.
if steps['compare']:
if not compare(matched_list, args.iss, args.start_seed, args.o):
return RET_FAIL
# Generate merged coverage directory and load it into appropriate GUI
if steps['cov']:
gen_cov(args.o, args.simulator, args.lsf_cmd)