Add wall-clock timeout within rtl simulation to gracefully end

Use a DPI call to unix 'date' to implement a wall-clock timeout entirely within
a simulation. This allows the UVM environment to gracefully end when the
threshold is reached, and for things like logs and coverage databases to be
generated correctly.
Previously, a process-level timeout was used, which gave the running simulation
no time to commit any logs/databases to disk before ending. Hence we would not
gather any coverage from timed-out tests.

A plusarg 'test_timeout_s' can be specified to each test to set the timeout. The
default timeout is 1800s.

dv/uvm/core_ibex/common/date.c

@@ -0,0 +1,6 @@
+// Copyright lowRISC contributors.
+// Licensed under the Apache License, Version 2.0, see LICENSE for details.
+// SPDX-License-Identifier: Apache-2.0
+
+#include <time.h>
+long int get_unix_timestamp() { return time(NULL); }

dv/uvm/core_ibex/common/date_dpi.svh

@@ -0,0 +1,10 @@
+// Copyright lowRISC contributors.
+// Licensed under the Apache License, Version 2.0, see LICENSE for details.
+// SPDX-License-Identifier: Apache-2.0
+
+`ifndef DATE_DPI_SVH
+`define DATE_DPI_SVH
+
+import "DPI-C" function longint get_unix_timestamp();
+
+`endif

dv/uvm/core_ibex/ibex_dv.f

@@ -100,6 +100,8 @@ ${PRJ_DIR}/rtl/ibex_top.sv
 ${PRJ_DIR}/rtl/ibex_top_tracing.sv
 
 // Core DV files
++incdir+${PRJ_DIR}/dv/uvm/core_ibex/common
+${PRJ_DIR}/dv/uvm/core_ibex/common/date.c
 ${PRJ_DIR}/vendor/google_riscv-dv/src/riscv_signature_pkg.sv
 +incdir+${LOWRISC_IP_DIR}/dv/sv/dv_utils
 +incdir+${LOWRISC_IP_DIR}/dv/sv/dv_base_reg

dv/uvm/core_ibex/riscv_dv_extension/ibex_log_to_trace_csv.py

@@ -29,6 +29,7 @@ try:
 finally:
     sys.path = _OLD_SYS_PATH
 
+from test_run_result import Failure_Modes
 
 INSTR_RE = \
     re.compile(r"^\s*(?P<time>\d+)\s+(?P<cycle>\d+)\s+(?P<pc>[0-9a-f]+)\s+"
@@ -195,7 +196,9 @@ def check_ibex_uvm_log(uvm_log):
     failed = False
 
     error_linenum = None
+    error_line = None
     log_out = []
+    failure_mode = Failure_Modes.NONE
 
     with open(uvm_log, "r") as log:
         # Simulation log has report summary at the end, which references
@@ -212,6 +215,7 @@ def check_ibex_uvm_log(uvm_log):
                 'Error' in line) \
                     and not test_result_seen:
                 error_linenum = linenum
+                error_line = line
                 failed = True
 
             if 'RISC-V UVM TEST PASSED' in line:
@@ -238,7 +242,13 @@ def check_ibex_uvm_log(uvm_log):
                            for linenum, line in enumerate(log, 1)
                            if linenum in range(error_linenum-5, error_linenum+5)]
 
-    return (passed, log_out)
+            if ('Test failed due to wall-clock timeout.' in error_line):
+
+                failure_mode = Failure_Modes.TIMEOUT
+            else:
+                failure_mode = Failure_Modes.LOG_ERROR
+
+    return (passed, log_out, failure_mode)
 
 
 def main():

dv/uvm/core_ibex/scripts/check_logs.py

@@ -39,15 +39,19 @@ def compare_test_run(trr: TestRunResult) -> TestRunResult:
     # Have a look at the UVM log. Report a failure if an issue is seen.
     try:
         logger.debug(f"About to do Log processing: {trr.rtl_log}")
-        uvm_pass, uvm_log_lines = check_ibex_uvm_log(trr.rtl_log)
+        uvm_pass, uvm_log_lines, uvm_failure_mode = check_ibex_uvm_log(trr.rtl_log)
     except IOError as e:
         trr.passed = False
         trr.failure_mode = Failure_Modes.FILE_ERROR
         trr.failure_message = f"[FAILED] Could not open simulation log: {e}\n"
         return trr
     if not uvm_pass:
-        trr.failure_mode = Failure_Modes.LOG_ERROR
-        trr.failure_message = f"\n[FAILURE]: sim error seen in '{trr.rtl_log.name}'\n"
+        trr.failure_mode = uvm_failure_mode
+        if uvm_failure_mode == Failure_Modes.TIMEOUT:
+            trr.failure_message = "[FAILURE] Simulation ended gracefully due to timeout " \
+                                 f"[{trr.timeout_s}s].\n"
+        else:
+            trr.failure_message = f"\n[FAILED]: error seen in '{trr.rtl_log.name}'\n"
         if uvm_log_lines:
             trr.failure_message += \
                 "---------------*LOG-EXTRACT*----------------\n" + \

dv/uvm/core_ibex/scripts/collect_results.py

@@ -165,8 +165,6 @@ def main() -> int:
                 if trr.passed:
                     passing_tests.append(trr)
                 else:
-                    if (trr.failure_mode == Failure_Modes.TIMEOUT):
-                        trr.failure_message = f"[FAILURE] Simulation timed-out [{trr.timeout_s}s].\n"
                     failing_tests.append(trr)
             except RuntimeError as e:
                 failing_tests.append(

dv/uvm/core_ibex/scripts/run_rtl.py

@@ -46,6 +46,8 @@ def _main() -> int:
     if sim_opts_raw:
         sim_opts += sim_opts_raw.replace('\n', '')
 
+    trr.timeout_s = (testopts.get('timeout_s') if (testopts.get('timeout_s') is not None) else
+                     md.run_rtl_timeout_s)
     trr.rtl_log         = trr.dir_test / 'rtl_sim.log'
     trr.rtl_trace       = trr.dir_test / 'trace_core_00000000.log'
     trr.iss_cosim_trace = trr.dir_test / f'{md.iss}_cosim_trace_core_00000000.log'
@@ -62,6 +64,7 @@ def _main() -> int:
         'rtl_trace': trr.rtl_trace.parent/'trace_core',
         'iss_cosim_trace': trr.iss_cosim_trace,
         'sim_opts': (f"+signature_addr={md.signature_addr}\n" +
+                     f"+test_timeout_s={trr.timeout_s}\n" +
                      f"{get_sim_opts(md.ibex_config, md.simulator)}\n" +
                      sim_opts)
     }
@@ -78,8 +81,6 @@ def _main() -> int:
         user_subst_options=subst_vars_dict)
     logger.info(sim_cmds)
 
-    trr.timeout_s = (testopts.get('timeout_s') if (testopts.get('timeout_s') is not None) else
-                     md.run_rtl_timeout_s)
     trr.dir_test.mkdir(exist_ok=True, parents=True)
     trr.rtl_cmds   = [format_to_cmd(cmd) for cmd in sim_cmds]
     trr.rtl_stdout = trr.dir_test / 'rtl_sim_stdstreams.log'
@@ -97,10 +98,12 @@ def _main() -> int:
                 sim_fd.write(f"Running run-rtl command :\n{' '.join(cmd)}\n".encode())
                 run_one(md.verbose, cmd,
                         redirect_stdstreams=sim_fd,
-                        timeout_s=trr.timeout_s,
+                        timeout_s=md.run_rtl_timeout_s+60,  # Ideally we time-out inside the simulation
                         reraise=True)  # Allow us to catch timeout exceptions at this level
         except subprocess.TimeoutExpired:
             trr.failure_mode = Failure_Modes.TIMEOUT
+            trr.failure_message = "[FAILURE] Simulation process killed due to timeout " \
+                                 f"[{md.run_rtl_timeout_s+60}s].\n"
 
     trr.export(write_yaml=True)
     # Always return 0 (success), even if the test failed. We've successfully

dv/uvm/core_ibex/tests/core_ibex_base_test.sv

@@ -2,6 +2,8 @@
 // Licensed under the Apache License, Version 2.0, see LICENSE for details.
 // SPDX-License-Identifier: Apache-2.0
 
+`include "date_dpi.svh"
+
 class core_ibex_base_test extends uvm_test;
 
   core_ibex_env                                   env;
@@ -14,6 +16,7 @@ class core_ibex_base_test extends uvm_test;
   mem_model_pkg::mem_model                        mem;
   core_ibex_vseq                                  vseq;
   bit                                             enable_irq_seq;
+  longint                                         timeout_seconds = 1800; // wall-clock seconds
   int unsigned                                    timeout_in_cycles = 100000000;
   int unsigned                                    max_quit_count  = 1;
   // If no signature_addr handshake functionality is desired between the testbench and the generated
@@ -217,6 +220,7 @@ class core_ibex_base_test extends uvm_test;
 
   // Watch for all of the different critera for test pass/failure here
   virtual task wait_for_test_done();
+    longint timeout_timestamp, ts;
     bit result;
 
     fork
@@ -254,6 +258,23 @@ class core_ibex_base_test extends uvm_test;
         clk_vif.wait_clks(timeout_in_cycles);
         `uvm_fatal(`gfn, "TEST TIMEOUT!!")
       end
+      // - End the test gracefully by wall-clock timeout (gather coverage etc.)
+      //   The plusarg 'test_timeout_s' can be used to set this value.
+      begin
+        void'($value$plusargs("test_timeout_s=%0d", timeout_seconds));
+        `uvm_info(`gfn,
+                  $sformatf("Test wall-clock timeout is set to : %0ds", timeout_seconds),
+                  UVM_LOW)
+        timeout_timestamp = get_unix_timestamp() + timeout_seconds;
+        forever begin
+          // Check the wall-clock every 1000us of simulation time.
+          #1000us;
+          ts = get_unix_timestamp();
+          if (ts >= timeout_timestamp) break;
+        end
+        `uvm_fatal(`gfn,
+                   $sformatf("Test failed due to wall-clock timeout. [%0ds]", timeout_seconds))
+      end
     join_any
 
     test_done = 1'b1;