github-mirrors/ibex
1
0
Fork 0
mirror of https://github.com/lowRISC/ibex.git synced 2025-04-22 04:47:25 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Split calls to run-instr-gen.py

Browse source
This commit is contained in:
Rupert Swarbrick 2022-04-20 07:51:18 +01:00 • committed by Rupert Swarbrick
parent a58fb02b68
commit a628007826
4 changed files with 140 additions and 150 deletions
Download patch file Download diff file Expand all files Collapse all files

36
dv/uvm/core_ibex/Makefile
View file

@ -294,29 +294,26 @@ instr_gen_build: $(OUT-DIR)instr_gen/.compile.stamp
###############################################################################
# Run the random instruction generator
#
RISCV_DV_ROOT := $(GEN_DIR)
INSTR_GEN_RUN_COMMANDS := $(shell mktemp)
$(metadata)/.instr_gen.run.stamp: \
test-asms := $(foreach ts,$(tests-and-seeds),$(OUT-SEED)/instr_gen/$(ts).S)
$(test-asms): \
$(OUT-SEED)/instr_gen/%.S: \
$(OUT-DIR)instr_gen/.compile.stamp \
$(TESTLIST) \
scripts/construct_makefile.py scripts/run-instr-gen.py | $(metadata)
$(verb)rm -rf $(OUT-SEED)/instr_gen/asm_test
+$(verb)scripts/run-instr-gen.py \
scripts/run-instr-gen.py | $(metadata)
$(verb)scripts/run-instr-gen.py \
$(verb-arg) \
--simulator $(SIMULATOR) \
--end-signature-addr $(SIGNATURE_ADDR) \
--output-dir $(@D) \
--gen-build-dir $(OUT-DIR)instr_gen \
--output $(OUT-SEED)/instr_gen \
--isa $(ISA) \
--test $(TEST) \
--start-seed $(SEED) \
--iterations $(ITERATIONS) \
--test-dot-seed $* \
--pmp-num-regions $(PMP_REGIONS) \
--pmp-granularity $(PMP_GRANULARITY)
@touch $@
.PHONY: instr_gen_run
instr_gen_run: $(metadata)/.instr_gen.run.stamp
instr_gen_run: $(test-asms)
###############################################################################
# Compile the generated assembly programs
@ -333,15 +330,14 @@ test-bins := $(foreach ts,$(tests-and-seeds),$(OUT-SEED)/instr_gen/$(ts).bin)
$(test-bins): \
$(OUT-SEED)/instr_gen/%.bin: \
$(metadata)/.instr_gen.run.stamp \
$(OUT-SEED)/instr_gen/%.S \
scripts/compile-generated-test.py
$(verb)scripts/compile-generated-test.py \
$(verb-arg) \
--output $@ \
--isa $(ISA) \
--input-dir $(OUT-SEED)/instr_gen/asm_test \
--test-dot-seed $* \
--start-seed $(SEED)
$(verb)scripts/compile-generated-test.py \
$(verb-arg) \
--output $@ \
--isa $(ISA) \
--input-dir $(OUT-SEED)/instr_gen \
--test-dot-seed $*
.PHONY: instr_gen_compile
instr_gen_compile: $(test-bins)

10
dv/uvm/core_ibex/scripts/compile-generated-test.py
View file

@ -22,18 +22,10 @@ def main() -> int:
parser.add_argument('--input-dir', required=True)
parser.add_argument('--test-dot-seed',
type=read_test_dot_seed, required=True)
parser.add_argument('--start-seed', type=int, required=True)
args = parser.parse_args()
if args.start_seed < 0:
raise RuntimeError('Invalid start seed: {}'.format(args.start_seed))
testname, seed = args.test_dot_seed
if seed < args.start_seed:
raise RuntimeError('Start seed is greater than test seed '
f'({args.start_seed} > {seed}).')
iteration = seed - args.start_seed
if not args.output.endswith('.bin'):
raise RuntimeError("Output argument must end with .bin: "
@ -42,7 +34,7 @@ def main() -> int:
out_riscv_dv_path = out_base + '.riscv-dv.log'
out_obj_path = out_base + '.o'
src_file = os.path.join(args.input_dir, f'{testname}_{iteration}.S')
src_file = os.path.join(args.input_dir, f'{testname}.{seed}.S')
if not os.path.exists(src_file):
raise RuntimeError(f'No such input file: {src_file!r}.')

57
dv/uvm/core_ibex/scripts/construct_makefile.py
View file

@ -1,57 +0,0 @@
#!/usr/bin/env python3
# Copyright lowRISC contributors.
# Licensed under the Apache License, Version 2.0, see LICENSE for details.
# SPDX-License-Identifier: Apache-2.0
import argparse
import sys
def transform(discard_stdstreams: bool,
cmdlist_path: str,
makefile_path: str) -> None:
'''Transform a list of commands to a Makefile'''
# Many commands come with a logfile argument, however some capture the
# stdout/stderr to a file. Handle both cases to ensure the logs are tidy.
tail = '\n'
if discard_stdstreams:
tail = ' >/dev/null 2>&1' + tail
with open(cmdlist_path) as f, \
open(makefile_path, 'w', encoding='UTF-8') as outfile:
ctr = 0
for line in f:
line = line.strip()
if not line:
continue
outfile.write(f'{ctr}:\n')
outfile.write('\t' + line.strip() + tail)
ctr += 1
outfile.write(f'CMDS := $(shell seq 0 {ctr - 1})\n')
outfile.write('all: $(CMDS)')
def main() -> int:
"""
Construct a trivial makefile from the --debug output of riscv-dv commands.
Creates a flat makefile, so if the commands have any ordering requirements
this will fall over.
"""
parser = argparse.ArgumentParser()
parser.add_argument('--test_cmds', required=True,
help='File containing --debug output')
parser.add_argument('--output', required=True,
help='Makefile to be constructed')
parser.add_argument("--discard_stdstreams", action='store_true',
help='Redirect stdstreams to /dev/null')
args = parser.parse_args()
transform(args.discard_stdstreams, args.test_cmds, args.output)
return 0
if __name__ == '__main__':
sys.exit(main())

187
dv/uvm/core_ibex/scripts/run-instr-gen.py
View file

@ -6,12 +6,14 @@
import argparse
import os
import re
import shlex
import shutil
import sys
import tempfile
from typing import List
import construct_makefile
from scripts_lib import start_riscv_dv_run_cmd, run_one
from scripts_lib import read_test_dot_seed, start_riscv_dv_run_cmd, run_one
def main() -> int:
@ -19,19 +21,20 @@ def main() -> int:
parser.add_argument('--verbose', action='store_true')
parser.add_argument('--simulator', required=True)
parser.add_argument('--end-signature-addr', required=True)
parser.add_argument('--output-dir', required=True)
parser.add_argument('--gen-build-dir', required=True)
parser.add_argument('--output', required=True)
parser.add_argument('--isa', required=True)
parser.add_argument('--test', required=True)
parser.add_argument('--start-seed', type=int, required=True)
parser.add_argument('--iterations', type=int, required=True)
parser.add_argument('--test-dot-seed',
type=read_test_dot_seed, required=True)
parser.add_argument('--pmp-num-regions', type=int, required=True)
parser.add_argument('--pmp-granularity', type=int, required=True)
args = parser.parse_args()
testname, seed = args.test_dot_seed
inst_overrides = [
'riscv_asm_program_gen',
'ibex_asm_program_gen',
@ -47,106 +50,162 @@ def main() -> int:
sim_opts_str = ' '.join('+{}={}'.format(k, v)
for k, v in sim_opts_dict.items())
output_makefile = os.path.join(args.output, 'run.mk')
output_pfx = os.path.join(args.output_dir, f'{testname}.{seed}')
riscv_dv_log = output_pfx + '.riscv-dv.log'
gen_log = output_pfx + '.gen.log'
gen_asm = output_pfx + '.S'
# Ensure that the output directory actually exists
os.makedirs(args.output_dir, exist_ok=True)
with tempfile.TemporaryDirectory() as td:
orig_list = os.path.join(td, 'orig-cmds.list')
reloc_list = os.path.join(td, 'reloc-cmds.list')
orig_list = os.path.join(td, 'cmds.list')
placeholder = os.path.join(td, '@@PLACEHOLDER@@')
cmd = (start_riscv_dv_run_cmd(args.verbose) +
['--so', '--steps=gen',
'--output', args.output,
'--output', placeholder,
'--simulator', args.simulator,
'--isa', args.isa,
'--test', args.test,
'--start_seed', str(args.start_seed),
'--iterations', str(args.iterations),
'--test', testname,
'--start_seed', str(seed),
'--iterations', '1',
'--sim_opts', sim_opts_str,
'--debug', orig_list])
# Run riscv-dv to generate a bunch of commands
gen_retcode = run_one(args.verbose, cmd)
# Run riscv-dv to generate commands. This is rather chatty, so redirect
# its output to a log file.
gen_retcode = run_one(args.verbose, cmd,
redirect_stdstreams=riscv_dv_log)
if gen_retcode:
return gen_retcode
# Those commands assume the riscv-dv directory layout, where the build
# and run directories are the same. Transform each of the commands as
# necessary to point at the built generator
reloc_commands(args.simulator, args.output, args.gen_build_dir,
orig_list, reloc_list)
cmds = reloc_commands(placeholder,
args.gen_build_dir,
td,
args.simulator,
testname,
orig_list)
# Convert the final command list to a Makefile
construct_makefile.transform(True, reloc_list, output_makefile)
# Run the commands in sequence to create "test_0.S" and "gen.log" in
# the temporary directory.
ret = 0
for cmd in cmds:
ret = run_one(args.verbose, cmd, redirect_stdstreams='/dev/null')
if ret != 0:
break
# Finally, run Make to run those commands
cmd = ['make', '-f', output_makefile, 'all']
if not args.verbose:
cmd.append('-s')
td_gen_log = os.path.join(td, 'gen.log')
td_gen_asm = os.path.join(td, 'test_0.S')
return run_one(args.verbose, cmd)
# At this point, we might have a "gen.log" in the temporary directory.
# Copy it back if so. If not, check that ret is nonzero.
if os.path.exists(td_gen_log):
shutil.copy(td_gen_log, gen_log)
elif ret == 0:
raise RuntimeError('Generation commands exited with zero status '
'but left no gen.log in scratch directory.')
# If we failed, exit now (rather than copying the .S file across)
if ret != 0:
return ret
# We succeeded. Check there is indeed a test_0.S in the temporary
# directory and copy it back.
if not os.path.exists(td_gen_asm):
raise RuntimeError('Generation commands exited with zero status '
'but left no test_0.S in scratch directory.')
shutil.copy(td_gen_asm, gen_asm)
return 0
def reloc_commands(simulator: str, run_dir: str, build_dir: str,
src: str, dst: str) -> None:
'''Reads all the lines in src and "relocate" them to dst.
def reloc_commands(placeholder_dir: str,
build_dir: str,
scratch_dir: str,
simulator: str,
testname: str,
src: str) -> List[List[str]]:
'''Reads the (one) line in src and apply relocations to it
More precisely, try to find paths in these lines that refer to things that
were actually built as part of the build step (compiling the instruction
generator) and switch those paths over to point at the correct directory.
The result should be a series of commands that build a single test into
scratch_dir/test_0.S, dumping a log into scratch_dir/gen.log.
'''
with open(src) as src_file, open(dst, 'w') as dst_file:
ret = []
with open(src) as src_file:
for line in src_file:
line = line.strip()
if not line:
continue
parts = shlex.split(line)
reloc_parts = [reloc_word(simulator, run_dir, build_dir, w)
for w in parts]
reloc_line = ' '.join([shlex.quote(w) for w in reloc_parts])
print(reloc_line, file=dst_file)
ret.append([reloc_word(simulator,
placeholder_dir, build_dir,
scratch_dir, testname, w)
for w in shlex.split(line)])
return ret
def reloc_word(simulator: str, run_dir: str, build_dir: str, word: str) -> str:
'''Helper function for reloc_commands that relocates just one word
This is a bit of a hack, made necessary because riscv-dv doesn't really
support building the instruction generator in a different directory from
where we run it.
'''
reloc_basenames = {
def reloc_word(simulator: str,
placeholder_dir: str, build_dir: str, scratch_dir: str,
testname: str, word: str) -> str:
'''Helper function for reloc_commands that relocates just one word'''
sim_relocs = {
'vcs': [
# The VCS binary
'vcs_simv'
# The VCS-generated binary
(os.path.join(placeholder_dir, 'vcs_simv'),
os.path.join(build_dir, 'vcs_simv'))
],
'xlm': [
# For Xcelium, the build directory gets passed as the
# "-xmlibdirpath" argument. The basename there will be "instr_gen".
'instr_gen'
# "-xmlibdirpath" argument.
(placeholder_dir, build_dir)
]
}
always_relocs = [
# The generated test. Since riscv-dv expects to make more than one of
# them, this gets supplied as a plusarg with just the test name (with
# no seed suffix).
(os.path.join(placeholder_dir, 'asm_test', testname),
os.path.join(scratch_dir, 'test')),
basename = os.path.basename(word)
if basename not in reloc_basenames[simulator]:
# This doesn't look like a file we care about tracking
return word
# The log file for generation itself
(os.path.join(placeholder_dir, f'sim_{testname}_0.log'),
os.path.join(scratch_dir, 'gen.log'))
]
abs_word = os.path.abspath(word)
abs_rundir = os.path.abspath(run_dir)
# Special handling for plusargs with filenames, which end up looking
# something like +my_argument=foo/bar/baz.
match = re.match(r'(\+[A-Za-z0-9_]+=)(.*)', word)
if match is not None:
pre = match.group(1)
post = match.group(2)
else:
pre = ''
post = word
common_path = os.path.commonpath([abs_word, abs_rundir])
if common_path != abs_rundir:
# The file wasn't in the run_dir tree.
return word
abs_post = os.path.abspath(post)
# Avoid a path like "a/b/c/." if abs_word happens to equal abs_rundir
if abs_word == abs_rundir:
return build_dir
for orig, reloc in sim_relocs[simulator] + always_relocs:
abs_orig = os.path.abspath(orig)
if abs_orig == abs_post:
post = reloc
break
rel_word = os.path.relpath(abs_word, abs_rundir)
return os.path.join(build_dir, rel_word)
reloc = pre + post
# Check there's no remaining occurrence of the placeholder
if placeholder_dir in reloc:
raise RuntimeError('Failed to replace an occurrence of the '
f'placeholder in {word} (got {reloc})')
return reloc
if __name__ == '__main__':