[dv/ibex] filter out tests on a per-config basis

This PR adds functionality to filter out tests during regressions for a
particular config.

e.g. if a full regression is kicked off using the `small` config, we
don't want to attempt to run any PMP and bitmanip tests as the RTL
parameter-set will not support it.

To do this, a new YAML field called `rtl_params` is added to relevant
test entries, to indicate what parameters (if any) are required to be
able to run the particular test, along with the required value of said
parameters.

`sim.py` will then parse this field (if it exists), and using
information from `ibex_configs.yaml` pertaining to the current config,
will remove tests from being run on-the-fly.

This also gives us the convenient side effect of not having to re-run
instruction generation if there is a parameter/config mismatch, we can
just rerun the RTL compilation and simulation stages safely.

Signed-off-by: Udi Jonnalagadda <udij@google.com>

dv/uvm/core_ibex/Makefile

@@ -353,6 +353,7 @@ $(OUT-DIR)rtl_sim/.compile.stamp: \
 	$(verb)./sim.py \
 	 --o=$(OUT-DIR) \
 	 --steps=compile \
+	 --ibex_config=$(IBEX_CONFIG) \
 	 ${COMMON_OPTS} \
 	 --simulator="${SIMULATOR}" --simulator_yaml=yaml/rtl_simulation.yaml \
 	 $(cov-arg) $(wave-arg) $(lsf-arg) \
@@ -389,6 +390,7 @@ $(metadata)/rtl_sim.run.stamp: \
 	$(verb)./sim.py \
 	 --o=$(OUT-SEED) \
 	 --steps=sim \
+	 --ibex_config=$(IBEX_CONFIG) \
 	 ${TEST_OPTS} \
 	 --simulator="${SIMULATOR}" --simulator_yaml=yaml/rtl_simulation.yaml \
 	 $(cov-arg) $(wave-arg) $(lsf-arg) \
@@ -407,6 +409,7 @@ $(OUT-SEED)/regr.log: \
 	$(verb)./sim.py \
      --o=$(OUT-SEED) \
      --steps=compare \
+     --ibex_config=$(IBEX_CONFIG) \
      ${TEST_OPTS} \
      --simulator="${SIMULATOR}" \
      --iss="${ISS}"

dv/uvm/core_ibex/riscv_dv_extension/testlist.yaml

@@ -642,6 +642,8 @@
     +pmp_max_offset=00024000
     +enable_write_pmp_csr=1
   rtl_test: core_ibex_base_test
+  rtl_params:
+    PMPEnable: 1
 
 - test: riscv_pmp_disable_all_regions_test
   desc: >
@@ -671,6 +673,8 @@
     +pmp_region_15=X:0,W:0,R:0
     +enable_write_pmp_csr=1
   rtl_test: core_ibex_base_test
+  rtl_params:
+    PMPEnable: 1
 
 - test: riscv_pmp_out_of_bounds_test
   desc: >
@@ -685,6 +689,8 @@
     +enable_write_pmp_csr=1
     +directed_instr_0=riscv_load_store_rand_addr_instr_stream,50
   rtl_test: core_ibex_base_test
+  rtl_params:
+    PMPEnable: 1
 
 - test: riscv_pmp_full_random_test
   desc: >
@@ -701,6 +707,8 @@
     +pmp_allow_addr_overlap=1
     +enable_write_pmp_csr=1
   rtl_test: core_ibex_base_test
+  rtl_params:
+    PMPEnable: 1
 
 - test: riscv_bitmanip_full_test
   desc: >
@@ -711,6 +719,8 @@
     +enable_b_extension=1
     +enable_bitmanip_groups=zbb,zb_tmp,zbt,zbs,zbp,zbf,zbe,zbc,zbr
   rtl_test: core_ibex_base_test
+  rtl_params:
+    RV32B: "ibex_pkg::RV32BFull"
 
 - test: riscv_bitmanip_balanced_test
   desc: >
@@ -721,3 +731,5 @@
     +enable_b_extension=1
     +enable_bitmanip_groups=zbb,zb_tmp,zbt,zbs,zbf
   rtl_test: core_ibex_base_test
+  rtl_params:
+    RV32B: ["ibex_pkg::RV32BFull", "ibex_pkg::RV32BBalanced"]

dv/uvm/core_ibex/sim.py

@@ -31,6 +31,7 @@ _OLD_SYS_PATH = sys.path
 # as it started.
 try:
     sys.path = ([os.path.join(_CORE_IBEX, 'riscv_dv_extension'),
+                 os.path.join(_IBEX_ROOT, 'util'),
                  os.path.join(_RISCV_DV_ROOT, 'scripts')] +
                 sys.path)
 
@@ -45,6 +46,8 @@ try:
 
     from ibex_log_to_trace_csv import process_ibex_sim_log, check_ibex_uvm_log
 
+    from ibex_config import parse_config
+
 finally:
     sys.path = _OLD_SYS_PATH
 
@@ -190,6 +193,75 @@ def get_simulator_cmd(simulator, yaml_path, enables):
             subst_cmd(entry['sim']['cmd'], enables, entry['sim'], env_vars))
 
 
+def filter_tests_by_config(cfg, test_list):
+    '''Filter out any unsupported tests from being executed.
+
+    This function will parse the set of RTL parameters required by a given
+    test (if any) and ensure that those parameters are supported by the
+    selected core config.
+
+    Doing this allows the run flow to be smarter about running regressions
+    with different configs (useful for CI flows).
+
+    Arguments:
+
+        cfg: string name of the ibex config being tested, should match a
+             config name from ibex_configs.yaml.
+
+        test_list: list of test entry objects parsed from the YAML testlist
+
+    Returns:
+
+        filtered_test_list: a list of test entry objects, filtered such that
+                            all tests incompatible with the specified ibex
+                            config have been removed.
+
+                            e.g. if the "small" config has been specified, this
+                            function will filter out all tests that require
+                            B-extension and PMP parameters
+    '''
+    filtered_test_list = []
+    config = parse_config(cfg, os.path.join(_IBEX_ROOT, "ibex_configs.yaml"))
+
+    for test in test_list:
+        if "rtl_params" in test:
+            param_dict = test['rtl_params']
+            matching_params = False
+            for p, p_val in param_dict.items():
+                config_val = config.get(p, None)
+                # Throw an error if required RTL parameters in the testlist
+                # have been formatted incorrectly (typos, wrong parameters, etc)
+                if config_val is None:
+                    logging.error('parameter % not found in config %' % (p, cfg))
+                    sys.exit(RET_FAIL)
+                else:
+                    # Ibex has some enum parameters, so as a result some tests
+                    # are able to run with several of these parameter values
+                    # (like bitmanipulation tests).
+                    # If this is the case, the testlist will specify all legal
+                    # enum values, check if any of them match the config.
+                    if isinstance(p_val, list):
+                        matching_params = (config_val in p_val)
+                    else:
+                        matching_params = (p_val == config_val)
+
+                    # If there is any parameter mismatch, we can terminate
+                    # immediately and exclude the test from being executed
+                    if not matching_params:
+                        break
+
+            if matching_params:
+                filtered_test_list.append(test)
+            else:
+                logging.info("""Cannot run test %s with config %s due to
+                                mismatching RTL parameter requirements %s""" %
+                             (test['test'], cfg, param_dict))
+        else:
+            filtered_test_list.append(test)
+
+    return filtered_test_list
+
+
 def rtl_compile(compile_cmds, output_dir, lsf_cmd, opts):
     """Compile the testbench RTL
 
@@ -559,6 +631,10 @@ def main():
     parser.add_argument("--lsf_cmd", type=str,
                         help=("LSF command. Run locally if lsf "
                               "command is not specified"))
+    parser.add_argument("--ibex_config", type=str,
+                        help=("Name of the Ibex config being tested. "
+                              "Should be a valid config name from "
+                              "ibex_configs.yaml"))
 
     sg = (parser.
           add_argument_group('Seeds and iterations',
@@ -630,6 +706,8 @@ def main():
             raise RuntimeError("Cannot find %s in %s" %
                                (args.test, args.testlist))
 
+        matched_list = filter_tests_by_config(args.ibex_config, matched_list);
+
     # Compile TB
     if steps['compile']:
         rtl_compile(compile_cmds, output_dir, args.lsf_cmd, args.cmp_opts)

util/ibex_config.py

@@ -214,6 +214,40 @@ def get_config_file_location():
     return os.environ.get('IBEX_CONFIG_FILE', _DEFAULT_CONFIG_FILE)
 
 
+def parse_config(config_name, config_filename):
+    """Parses the selected config file and returns selected config information.
+
+    Arguments:
+
+        config_name: Name of the chosen Ibex core config
+
+        config_filename: Name of the configuration filename to be parsed
+
+    Returns: the chosen Ibex config as a YAML object.
+
+    Raises a ConfigException if there are any error while parsing the YAML.
+
+    Raises a FileNotFoundError if there are errors opening the chosen config file.
+    """
+    try:
+        config_file = open(config_filename)
+        config_dicts = get_config_dicts(config_file)
+
+        if config_name not in config_dicts:
+            print('ERROR: configuration {!r} not found in {!r}.'.format(
+                  config_name, config_filename), file=sys.stderr)
+            sys.exit(1)
+        return config_dicts[config_name]
+    except ConfigException as ce:
+        print('ERROR: failure to process configuration from {!r} {!r}.'.format(
+              config_filename, ce), file=sys.stderr)
+        sys.exit(1)
+    except FileNotFoundError:
+        print('ERROR: could not find configuration file {!r}.'.format(
+              config_filename), file=sys.stderr)
+        sys.exit(1)
+
+
 def main():
     outputters = [
         FusesocOpts(),
@@ -270,32 +304,8 @@ def main():
         print('ERROR: No output format specified.')
         sys.exit(1)
 
-    try:
-        config_file = open(args.config_filename)
-        config_dicts = get_config_dicts(config_file)
-
-        if args.config_name not in config_dicts:
-            print('ERROR: configuration',
-                  args.config_name,
-                  'not found in',
-                  args.config_filename,
-                  file=sys.stderr)
-
-            sys.exit(1)
-
-        print(args.output_fn(config_dicts[args.config_name], args))
-    except ConfigException as ce:
-        print('ERROR: failure to process configuration from',
-              args.config_filename,
-              ce,
-              file=sys.stderr)
-        sys.exit(1)
-    except FileNotFoundError:
-        print('ERROR: could not find configuration file',
-              args.config_filename,
-              file=sys.stderr)
-        sys.exit(1)
-
+    parsed_ibex_config = parse_config(args.config_name, args.config_filename)
+    print(args.output_fn(parsed_ibex_config, args))
 
 if __name__ == "__main__":
     main()