mirror of
https://github.com/openhwgroup/cvw.git
synced 2025-04-20 03:47:20 -04:00
Merge pull request #1156 from jordancarlin/update_ccov
Use cvw-arch-verif tests for code coverage
This commit is contained in:
David Harris
GitHub
commit
9b24938590
3 changed files with 58 additions and 43 deletions
|
|
@ -34,7 +34,7 @@ tests = [
|
|||
"arch32zba", "arch32zbb", "arch32zbc", "arch32zbs", "arch32zfh", "arch32zfh_fma",
|
||||
"arch32zfh_divsqrt", "arch32zfaf", "arch32zfad", "wally32a_lrsc", "wally32priv", "wally32periph", "arch32zcb",
|
||||
"arch32zbkb", "arch32zbkc", "arch32zbkx", "arch32zknd", "arch32zkne", "arch32zknh"]],
|
||||
["rv64i", ["arch64i"]]
|
||||
["rv64i", ["arch64i"]]
|
||||
]
|
||||
|
||||
# Separate test for short buildroot run through OpenSBI UART output
|
||||
|
|
@ -265,8 +265,8 @@ class bcolors:
|
|||
def addTests(tests, sim):
|
||||
sim_logdir = WALLY+ "/sim/" + sim + "/logs/"
|
||||
for test in tests:
|
||||
config = test[0];
|
||||
suites = test[1];
|
||||
config = test[0]
|
||||
suites = test[1]
|
||||
if (len(test) >= 3):
|
||||
args = " --args " + " ".join(test[2])
|
||||
else:
|
||||
|
|
@ -291,18 +291,35 @@ def addTests(tests, sim):
|
|||
configs.append(tc)
|
||||
|
||||
|
||||
def addLockstepTestsByDir(dir, config, sim, fcovMode):
|
||||
sim_logdir = WALLY+ "/sim/" + sim + "/logs/"
|
||||
if (coverStr != ""): # use --fcov in place of --lockstep
|
||||
cmdPrefix="wsim --sim " + sim + " " + coverStr + " " + config
|
||||
else:
|
||||
cmdPrefix="wsim --lockstep --sim " + sim + " " + config
|
||||
if (os.path.isdir(dir)):
|
||||
def addTestsByDir(dir, config, sim, lockstepMode=0):
|
||||
if os.path.isdir(dir):
|
||||
sim_logdir = WALLY+ "/sim/" + sim + "/logs/"
|
||||
if coverStr == "--fcov": # use --fcov in place of --lockstep
|
||||
cmdPrefix="wsim --sim " + sim + " " + coverStr + " " + config
|
||||
gs="Mismatches : 0"
|
||||
if ("cvw-arch-verif/tests" in dir and not "priv" in dir):
|
||||
fileEnd = "ALL.elf"
|
||||
else:
|
||||
fileEnd = ".elf"
|
||||
elif coverStr == "--ccov":
|
||||
cmdPrefix="wsim --sim " + sim + " " + coverStr + " " + config
|
||||
gs="Single Elf file tests are not signatured verified."
|
||||
if ("cvw-arch-verif/tests" in dir and not "priv" in dir):
|
||||
fileEnd = "ALL.elf"
|
||||
else:
|
||||
fileEnd = ".elf"
|
||||
elif lockstepMode:
|
||||
cmdPrefix="wsim --lockstep --sim " + sim + " " + config
|
||||
gs="Mismatches : 0"
|
||||
fileEnd = ".elf"
|
||||
else:
|
||||
cmdPrefix="wsim --sim " + sim + " " + config
|
||||
gs="Single Elf file tests are not signatured verified."
|
||||
fileEnd = ".elf"
|
||||
for dirpath, dirnames, filenames in os.walk(os.path.abspath(dir)):
|
||||
for file in filenames:
|
||||
# fcov lockstep only runs on WALLY-COV-ALL.elf files; other lockstep runs on all files
|
||||
if ((file.endswith(".elf") and (fcovMode == 0 or "tests/priv" in dir)) or
|
||||
(file.endswith("ALL.elf") and fcovMode == 1)):
|
||||
if file.endswith(fileEnd):
|
||||
fullfile = os.path.join(dirpath, file)
|
||||
fields = fullfile.rsplit('/', 3)
|
||||
if (fields[2] == "ref"):
|
||||
|
|
@ -318,7 +335,7 @@ def addLockstepTestsByDir(dir, config, sim, fcovMode):
|
|||
name=file,
|
||||
variant=config,
|
||||
cmd=cmdPrefix + " " + fullfile + " > " + sim_log,
|
||||
grepstr="Mismatches : 0",
|
||||
grepstr=gs,
|
||||
grepfile = sim_log)
|
||||
configs.append(tc)
|
||||
else:
|
||||
|
|
@ -379,14 +396,13 @@ parser.add_argument("--fcov", help="Functional Coverage", action="store_true")
|
|||
parser.add_argument("--nightly", help="Run large nightly regression", action="store_true")
|
||||
parser.add_argument("--buildroot", help="Include Buildroot Linux boot test (takes many hours, done along with --nightly)", action="store_true")
|
||||
parser.add_argument("--testfloat", help="Include Testfloat floating-point unit tests", action="store_true")
|
||||
parser.add_argument("--fp", help="Include floating-point tests in coverage (slower runtime)", action="store_true")
|
||||
parser.add_argument("--fp", help="Include floating-point tests in coverage (slower runtime)", action="store_true") # Currently not used
|
||||
parser.add_argument("--dryrun", help="Print commands invoked to console without running regression", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
if (args.nightly):
|
||||
nightMode = "--nightly";
|
||||
nightMode = "--nightly"
|
||||
sims = ["questa", "verilator", "vcs"] # exercise all simulators; can omit a sim if no license is available
|
||||
# sims = ["questa", "verilator"] # exercise all simulators; can omit a sim if no license is available
|
||||
else:
|
||||
nightMode = ""
|
||||
sims = [defaultsim]
|
||||
|
|
@ -396,7 +412,7 @@ if (args.ccov): # only run RV64GC tests in coverage mode
|
|||
elif (args.fcov): # only run RV64GC tests in lockstep in coverage mode
|
||||
coverStr = '--fcov'
|
||||
else:
|
||||
coverStr = ''
|
||||
coverStr = ''
|
||||
|
||||
|
||||
# Run Lint
|
||||
|
|
@ -417,15 +433,14 @@ if (args.buildroot):
|
|||
addTests(tests_buildrootbootlockstep, lockstepsim) # lockstep with Questa and ImperasDV runs overnight
|
||||
|
||||
if (args.ccov): # only run RV64GC tests on Questa in code coverage mode
|
||||
addTests(tests64gc_nofp, coveragesim)
|
||||
if (args.fp):
|
||||
addTests(tests64gc_fp, coveragesim)
|
||||
addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/rv64/", "rv64gc", coveragesim)
|
||||
addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/priv/rv64/", "rv64gc", coveragesim)
|
||||
addTestsByDir(WALLY+"/tests/coverage/", "rv64gc", coveragesim)
|
||||
elif (args.fcov): # run tests in lockstep in functional coverage mode
|
||||
addLockstepTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/rv32/", "rv32gc", coveragesim, 1)
|
||||
addLockstepTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/rv64/", "rv64gc", coveragesim, 1)
|
||||
addLockstepTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/priv/rv32/", "rv32gc", coveragesim, 1)
|
||||
addLockstepTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/priv/rv64/", "rv64gc", coveragesim, 1)
|
||||
#addLockstepTestsByDir(WALLY+"/tests/riscof/work/wally-riscv-arch-test/rv64i_m/privilege/src/", "rv64gc", coveragesim, 0)
|
||||
addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/rv32/", "rv32gc", coveragesim)
|
||||
addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/rv64/", "rv64gc", coveragesim)
|
||||
addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/priv/rv32/", "rv32gc", coveragesim)
|
||||
addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/priv/rv64/", "rv64gc", coveragesim)
|
||||
|
||||
else:
|
||||
for sim in sims:
|
||||
|
|
@ -437,9 +452,9 @@ else:
|
|||
|
||||
# run derivative configurations and lockstep tests in nightly regression
|
||||
if (args.nightly):
|
||||
addLockstepTestsByDir(WALLY+"/tests/coverage", "rv64gc", lockstepsim, 0)
|
||||
addLockstepTestsByDir(WALLY+"/tests/riscof/work/wally-riscv-arch-test/rv64i_m", "rv64gc", lockstepsim, 0)
|
||||
addLockstepTestsByDir(WALLY+"/tests/riscof/work/wally-riscv-arch-test/rv32i_m", "rv32gc", lockstepsim, 0)
|
||||
addTestsByDir(WALLY+"/tests/coverage", "rv64gc", lockstepsim, 1)
|
||||
addTestsByDir(WALLY+"/tests/riscof/work/wally-riscv-arch-test/rv64i_m", "rv64gc", lockstepsim, 1)
|
||||
addTestsByDir(WALLY+"/tests/riscof/work/wally-riscv-arch-test/rv32i_m", "rv32gc", lockstepsim, 1)
|
||||
addTests(derivconfigtests, defaultsim)
|
||||
# addTests(bpredtests, defaultsim) # This is currently broken in regression due to something related to the new wsim script.
|
||||
|
||||
|
|
@ -540,17 +555,17 @@ def main():
|
|||
else:
|
||||
ImperasDVLicenseCount = 10000 # effectively no license limit for non-lockstep tests
|
||||
with Pool(processes=min(len(configs),multiprocessing.cpu_count(), ImperasDVLicenseCount)) as pool:
|
||||
num_fail = 0
|
||||
results = {}
|
||||
for config in configs:
|
||||
results[config] = pool.apply_async(run_test_case,(config, args.dryrun))
|
||||
for (config,result) in results.items():
|
||||
try:
|
||||
num_fail+=result.get(timeout=TIMEOUT_DUR)
|
||||
except TimeoutError:
|
||||
pool.terminate()
|
||||
num_fail+=1
|
||||
print(f"{bcolors.FAIL}%s: Timeout - runtime exceeded %d seconds{bcolors.ENDC}" % (config.cmd, TIMEOUT_DUR))
|
||||
num_fail = 0
|
||||
results = {}
|
||||
for config in configs:
|
||||
results[config] = pool.apply_async(run_test_case,(config, args.dryrun))
|
||||
for (config,result) in results.items():
|
||||
try:
|
||||
num_fail+=result.get(timeout=TIMEOUT_DUR)
|
||||
except TimeoutError:
|
||||
pool.terminate()
|
||||
num_fail+=1
|
||||
print(f"{bcolors.FAIL}%s: Timeout - runtime exceeded %d seconds{bcolors.ENDC}" % (config.cmd, TIMEOUT_DUR))
|
||||
|
||||
# Coverage report
|
||||
if args.ccov:
|
||||
|
|
|
|||
4
bin/wsim
4
bin/wsim
|
|
@ -101,10 +101,10 @@ if (args.lockstep or args.lockstepverbose):
|
|||
if(args.locksteplog != 0): ImperasPlusArgs = " +IDV_TRACE2LOG=" + str(EnableLog) + " +IDV_TRACE2LOG_AFTER=" + str(args.locksteplog)
|
||||
else: ImperasPlusArgs = ""
|
||||
if(args.fcov):
|
||||
CovEnableStr = "1" if int(args.covlog) > 0 else "0";
|
||||
CovEnableStr = "1" if int(args.covlog) > 0 else "0"
|
||||
if(args.covlog >= 1): EnableLog = 1
|
||||
else: EnableLog = 0
|
||||
ImperasPlusArgs = " +IDV_TRACE2COV=" + str(EnableLog) + " +TRACE2LOG_AFTER=" + str(args.covlog) + " +TRACE2COV_ENABLE=" + CovEnableStr;
|
||||
ImperasPlusArgs = " +IDV_TRACE2COV=" + str(EnableLog) + " +TRACE2LOG_AFTER=" + str(args.covlog) + " +TRACE2COV_ENABLE=" + CovEnableStr
|
||||
suffix = ""
|
||||
else:
|
||||
CovEnableStr = ""
|
||||
|
|
|
|||
|
|
@ -29,8 +29,8 @@ deriv:
|
|||
|
||||
.PHONY: QuestaCodeCoverage collect_functcov combine_functcov remove_functcov_artifacts riscvdv riscvdv_functcov
|
||||
|
||||
QuestaCodeCoverage: questa/ucdb/rv64gc_arch64i.ucdb
|
||||
vcover merge -out questa/ucdb/cov.ucdb questa/ucdb/rv64gc_arch64i.ucdb questa/ucdb/rv64gc*.ucdb -logfile questa/cov/log
|
||||
QuestaCodeCoverage:
|
||||
vcover merge -out questa/ucdb/cov.ucdb questa/ucdb/rv64gc*.ucdb -logfile questa/cov/log
|
||||
# vcover merge -out questa/ucdb/cov.ucdb questa/ucdb/rv64gc_arch64i.ucdb questa/ucdb/rv64gc*.ucdb questa/ucdb/buildroot_buildroot.ucdb riscv.ucdb -logfile questa/cov/log
|
||||
vcover report -details questa/ucdb/cov.ucdb > questa/cov/rv64gc_coverage_details.rpt
|
||||
vcover report questa/ucdb/cov.ucdb -details -instance=/core/ebu. > questa/cov/rv64gc_coverage_ebu.rpt
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue