[cosim] Add Simple System with cosim

dv/verilator/simple_system_cosim/README.md

@@ -0,0 +1,85 @@
+# Ibex Simple System with Co-simulation checking
+
+This augments the Ibex Simple System (`examples/simple_system`) to include the
+co-simulation system to check Ibex's execution. This runs Spike in lockstep with
+Ibex and checks each instruction Ibex retires matches what Spike has executed.
+In addition all data memory accesses are checked against memory acceses Spike
+has performed. More details on how the co-simulation works and how to build and
+run simple system with it included can be in found in the Ibex documentation
+under 'Co-simulation System' in the 'Ibex Reference Guide' section.
+
+## Quick Build and Run Instructions
+
+```
+# Get the Ibex co-simulation spike branch
+git clone -b ibex_cosim https://github.com/lowRISC/riscv-isa-sim.git riscv-isa-sim-cosim
+
+# Setup build directory
+cd riscv-isa-sim-cosim
+mkdir build
+cd build
+
+# Configure and build spike
+../configure --enable-commitlog --enable-misaligned --prefix=/opt/spike-cosim 
+# Installs in /opt/spike-cosim
+sudo make -j8 install
+
+# Setup IBEX_COSIM_ISS_ROOT for fusesoc build below
+export IBEX_COSIM_ISS_ROOT=/opt/spike-cosim
+
+# Spike's libsoftfloat.so needs to be accessible so add it to LD_LIBRARY_PATH
+export LD_LIBRARY_PATH=/opt/spike-cosim/lib:$LD_LIBRARY_PATH
+
+# Switch to a checkout of the Ibex repository
+cd <ibex_repo>
+
+# Build simulator
+fusesoc --cores-root=. run --target=sim --setup --build lowrisc:ibex:ibex_simple_system_cosim --RV32E=0 --RV32M=ibex_pkg::RV32MFast
+
+# Build coremark test binary, with performance counter dump disabled. The 
+# co-simulator system doesn't produce matching performance counters in spike so
+# any read of those CSRs results in a mismatch and a failure.
+make -C ./examples/sw/benchmarks/coremark SUPPRESS_PCOUNT_DUMP=1
+
+# Run coremark binary with co-simulation checking
+build/lowrisc_ibex_ibex_simple_system_cosim_0/sim-verilator/Vibex_simple_system --meminit=ram,examples/sw/benchmarks/coremark/coremark.elf
+```
+
+Sample output:
+
+```
+Simulation of Ibex
+==================
+
+Tracing can be toggled by sending SIGUSR1 to this process:
+$ kill -USR1 29121
+
+Simulation running, end by pressing CTRL-c.
+TOP.ibex_simple_system.u_top.u_ibex_tracer.unnamedblk1: Writing execution trace to trace_core_00000000.log
+Terminating simulation by software request.
+- ../src/lowrisc_ibex_sim_shared_0/./rtl/sim/simulator_ctrl.sv:93: Verilog $finish
+Received $finish() from Verilog, shutting down simulation.
+
+Simulation statistics
+=====================
+Executed cycles:  4116797
+Wallclock time:   17.053 s
+Simulation speed: 241412 cycles/s (241.412 kHz)
+Co-simulation matched 2789425 instructions
+
+Performance Counters
+====================
+Cycles:                     4055056
+NONE:                       0
+Instructions Retired:       2750348
+LSU Busy:                   684533
+Fetch Wait:                 187543
+Loads:                      541082
+Stores:                     143451
+Jumps:                      57169
+Conditional Branches:       523452
+Taken Conditional Branches: 187543
+Compressed Instructions:    0
+Multiply Wait:              187920
+Divide Wait:                0
+```

dv/verilator/simple_system_cosim/ibex_cosim_setup_check.core

@@ -0,0 +1,25 @@
+CAPI=2:
+# Copyright lowRISC contributors.
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+name: "lowrisc:tool:ibex_cosim_setup_check:0.1"
+description: "Check $IBEX_COSIM_ISS_ROOT is set"
+
+filesets:
+  files_ibex_cosim_setup_check:
+    files:
+      - ./util/ibex_cosim_setup_check.sh : { copyto: util/ibex_cosim_setup_check.sh }
+
+scripts:
+  ibex_cosim_setup_check:
+    cmd:
+      - sh
+      - util/ibex_cosim_setup_check.sh
+
+targets:
+  default:
+    filesets:
+      - files_ibex_cosim_setup_check
+    hooks:
+      pre_build:
+        - ibex_cosim_setup_check

dv/verilator/simple_system_cosim/ibex_simple_system_cosim.core

@@ -0,0 +1,160 @@
+CAPI=2:
+# Copyright lowRISC contributors.
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+name: "lowrisc:ibex:ibex_simple_system_cosim"
+description: "Generic simple system for running binaries on ibex using verilator"
+filesets:
+  files_cosim:
+    depend:
+      - lowrisc:dv:cosim_dpi
+      - lowrisc:ibex:ibex_simple_system_core
+      - lowrisc:tool:ibex_cosim_setup_check
+    files:
+      - simple_system_cosim.cc: { file_type: cppSource }
+      - ibex_simple_system_cosim_checker.sv
+      - ibex_simple_system_cosim_checker_bind.sv
+    file_type: systemVerilogSource
+
+parameters:
+  RV32E:
+    datatype: int
+    paramtype: vlogparam
+    default: 0
+    description: "Enable the E ISA extension (reduced register set) [0/1]"
+
+  RV32M:
+    datatype: str
+    default: ibex_pkg::RV32MFast
+    paramtype: vlogdefine
+    description: "RV32M implementation parameter enum. See the ibex_pkg::rv32m_e enum in ibex_pkg.sv for permitted values."
+
+  RV32B:
+    datatype: str
+    default: ibex_pkg::RV32BNone
+    paramtype: vlogdefine
+    description: "Bitmanip implementation parameter enum. See the ibex_pkg::rv32b_e enum in ibex_pkg.sv for permitted values."
+
+  RegFile:
+    datatype: str
+    default: ibex_pkg::RegFileFF
+    paramtype: vlogdefine
+    description: "Register file implementation parameter enum. See the ibex_pkg::regfile_e enum in ibex_pkg.sv for permitted values."
+
+  ICache:
+    datatype: int
+    default: 0
+    paramtype: vlogparam
+    description: "Enable instruction cache"
+
+  ICacheECC:
+    datatype: int
+    default: 0
+    paramtype: vlogparam
+    description: "Enable ECC protection in instruction cache"
+
+  SRAMInitFile:
+    datatype: str
+    paramtype: vlogparam
+    description: "Path to a vmem file to initialize the RAM with"
+
+  BranchTargetALU:
+    datatype: int
+    paramtype: vlogparam
+    default: 0
+    description: "Enables separate branch target ALU (increasing branch performance EXPERIMENTAL)"
+
+  WritebackStage:
+    datatype: int
+    paramtype: vlogparam
+    default: 0
+    description: "Enables third pipeline stage (EXPERIMENTAL)"
+
+  SecureIbex:
+    datatype: int
+    default: 0
+    paramtype: vlogparam
+    description: "Enables security hardening features (EXPERIMENTAL) [0/1]"
+
+  BranchPredictor:
+    datatype: int
+    paramtype: vlogparam
+    default: 0
+    description: "Enables static branch prediction (EXPERIMENTAL)"
+
+  PMPEnable:
+    datatype: int
+    default: 0
+    paramtype: vlogparam
+    description: "Enable PMP"
+
+  PMPGranularity:
+    datatype: int
+    default: 0
+    paramtype: vlogparam
+    description: "Granularity of NAPOT range, 0 = 4 byte, 1 = byte, 2 = 16 byte, 3 = 32 byte etc"
+
+  PMPNumRegions:
+    datatype: int
+    default: 4
+    paramtype: vlogparam
+    description: "Number of PMP regions"
+
+targets:
+  default: &default_target
+    filesets:
+      - files_cosim
+    toplevel: ibex_simple_system
+    parameters:
+      - RV32E
+      - RV32M
+      - RV32B
+      - RegFile
+      - ICache
+      - ICacheECC
+      - BranchTargetALU
+      - WritebackStage
+      - SecureIbex
+      - BranchPredictor
+      - PMPEnable
+      - PMPGranularity
+      - PMPNumRegions
+      - SRAMInitFile
+
+  lint:
+    <<: *default_target
+    default_tool: verilator
+    tools:
+      verilator:
+        mode: lint-only
+        verilator_options:
+          - "-Wall"
+          # RAM primitives wider than 64bit (required for ECC) fail to build in
+          # Verilator without increasing the unroll count (see Verilator#1266)
+          - "--unroll-count 72"
+
+  sim:
+    <<: *default_target
+    default_tool: verilator
+    tools:
+      vcs:
+        vcs_options:
+          - '-xlrm uniq_prior_final'
+          - '-debug_access+r'
+      verilator:
+        mode: cc
+        verilator_options:
+          # Disabling tracing reduces compile times but doesn't have a
+          # huge influence on runtime performance.
+          - '--trace'
+          - '--trace-fst' # this requires -DVM_TRACE_FMT_FST in CFLAGS below!
+          - '--trace-structs'
+          - '--trace-params'
+          - '--trace-max-array 1024'
+          - '-CFLAGS "-std=c++11 -Wall -DVL_USER_STOP -DVM_TRACE_FMT_FST -DTOPLEVEL_NAME=ibex_simple_system -g -I${IBEX_COSIM_ISS_ROOT}/include -I${IBEX_COSIM_ISS_ROOT}/include/softfloat"'
+          - '-LDFLAGS "-pthread -lutil -lelf -L${IBEX_COSIM_ISS_ROOT}/lib/ -g -lriscv -lsoftfloat -lfdt -ldisasm -ldl"'
+          - "-Wall"
+          - "-Wwarn-IMPERFECTSCH"
+          # RAM primitives wider than 64bit (required for ECC) fail to build in
+          # Verilator without increasing the unroll count (see Verilator#1266)
+          - "--unroll-count 72"

dv/verilator/simple_system_cosim/ibex_simple_system_cosim_checker.sv

@@ -0,0 +1,81 @@
+// Copyright lowRISC contributors.
+// Licensed under the Apache License, Version 2.0, see LICENSE for details.
+// SPDX-License-Identifier: Apache-2.0
+
+module ibex_simple_system_cosim_checker (
+  input clk_i,
+  input rst_ni,
+
+  input logic        host_dmem_req,
+  input logic        host_dmem_gnt,
+  input logic        host_dmem_we,
+  input logic [31:0] host_dmem_addr,
+  input logic [3:0]  host_dmem_be,
+  input logic [31:0] host_dmem_wdata,
+
+  input logic        host_dmem_rvalid,
+  input logic [31:0] host_dmem_rdata,
+  input logic        host_dmem_err
+);
+  import "DPI-C" function chandle get_spike_cosim;
+
+  chandle cosim_handle;
+
+  initial begin
+    cosim_handle = get_spike_cosim();
+  end
+
+  always @(posedge clk_i) begin
+    if (u_top.rvfi_valid & !u_top.rvfi_trap) begin
+      riscv_cosim_set_nmi(cosim_handle, u_top.rvfi_ext_nmi);
+      riscv_cosim_set_mip(cosim_handle, u_top.rvfi_ext_mip);
+      riscv_cosim_set_debug_req(cosim_handle, u_top.rvfi_ext_debug_req);
+      riscv_cosim_set_mcycle(cosim_handle, u_top.rvfi_ext_mcycle);
+
+      if (riscv_cosim_step(cosim_handle, u_top.rvfi_rd_addr, u_top.rvfi_rd_wdata,
+                           u_top.rvfi_pc_rdata, u_top.rvfi_trap) == 0)
+      begin
+        $display("FAILURE: Co-simulation mismatch at time %t", $time());
+        for (int i = 0;i < riscv_cosim_get_num_errors(cosim_handle); ++i) begin
+          $display(riscv_cosim_get_error(cosim_handle, i));
+        end
+        riscv_cosim_clear_errors(cosim_handle);
+
+        $fatal(1, "Co-simulation mismatch seen");
+      end
+    end
+  end
+
+  logic outstanding_store;
+  logic [31:0] outstanding_addr;
+  logic [3:0] outstanding_be;
+  logic [31:0] outstanding_store_data;
+  logic outstanding_misaligned_first;
+  logic outstanding_misaligned_second;
+
+  always @(posedge clk_i or negedge rst_ni) begin
+    if (!rst_ni) begin
+      outstanding_store <= 1'b0;
+    end else begin
+      if (host_dmem_req && host_dmem_gnt) begin
+        outstanding_store      <= host_dmem_we;
+        outstanding_addr       <= host_dmem_addr;
+        outstanding_be         <= host_dmem_be;
+        outstanding_store_data <= host_dmem_wdata;
+        outstanding_misaligned_first <=
+          u_top.u_ibex_top.u_ibex_core.load_store_unit_i.handle_misaligned_d |
+          ((u_top.u_ibex_top.u_ibex_core.load_store_unit_i.lsu_type_i == 2'b01) &
+           (u_top.u_ibex_top.u_ibex_core.load_store_unit_i.data_offset == 2'b01));
+
+        outstanding_misaligned_second <=
+          u_top.u_ibex_top.u_ibex_core.load_store_unit_i.addr_incr_req_o;
+      end
+
+      if (host_dmem_rvalid) begin
+        riscv_cosim_notify_dside_access(cosim_handle, outstanding_store, outstanding_addr,
+          outstanding_store ? outstanding_store_data : host_dmem_rdata, outstanding_be,
+          host_dmem_err, outstanding_misaligned_first, outstanding_misaligned_second);
+      end
+    end
+  end
+endmodule

dv/verilator/simple_system_cosim/ibex_simple_system_cosim_checker_bind.sv

@@ -0,0 +1,22 @@
+// Copyright lowRISC contributors.
+// Licensed under the Apache License, Version 2.0, see LICENSE for details.
+// SPDX-License-Identifier: Apache-2.0
+
+module ibex_simple_system_cosim_checker_bind;
+  bind ibex_simple_system ibex_simple_system_cosim_checker
+    u_ibex_simple_system_cosim_checker_bind (
+      .clk_i            (IO_CLK),
+      .rst_ni           (IO_RST_N),
+
+      .host_dmem_req    (host_req[CoreD]),
+      .host_dmem_gnt    (host_gnt[CoreD]),
+      .host_dmem_we     (host_we[CoreD]),
+      .host_dmem_addr   (host_addr[CoreD]),
+      .host_dmem_be     (host_be[CoreD]),
+      .host_dmem_wdata  (host_wdata[CoreD]),
+
+      .host_dmem_rvalid (host_rvalid[CoreD]),
+      .host_dmem_rdata  (host_rdata[CoreD]),
+      .host_dmem_err    (host_err[CoreD])
+    );
+endmodule

dv/verilator/simple_system_cosim/simple_system_cosim.cc

@@ -0,0 +1,73 @@
+// Copyright lowRISC contributors.
+// Licensed under the Apache License, Version 2.0, see LICENSE for details.
+// SPDX-License-Identifier: Apache-2.0
+
+#include <cassert>
+#include <memory>
+#include "cosim.h"
+#include "ibex_simple_system.h"
+#include "spike_cosim.h"
+#include "verilator_memutil.h"
+
+class SimpleSystemCosim : public SimpleSystem {
+ public:
+  std::unique_ptr<SpikeCosim> _cosim;
+
+  SimpleSystemCosim(const char *ram_hier_path, int ram_size_words)
+      : SimpleSystem(ram_hier_path, ram_size_words), _cosim(nullptr) {}
+
+  ~SimpleSystemCosim() {}
+
+ protected:
+  void CopyMemAreaToCosim(MemArea *area, uint32_t base_addr) {
+    auto mem_data = area->Read(0, area->GetSizeWords());
+    _cosim->backdoor_write_mem(base_addr, area->GetSizeBytes(), &mem_data[0]);
+  }
+
+  virtual int Setup(int argc, char **argv, bool &exit_app) override {
+    int ret_code = SimpleSystem::Setup(argc, argv, exit_app);
+    if (exit_app) {
+      return ret_code;
+    }
+
+    _cosim = std::make_unique<SpikeCosim>(
+        0x100080, 0x100001, "simple_system_cosim.log", false, false);
+
+    _cosim->add_memory(0x100000, 1024 * 1024);
+    _cosim->add_memory(0x20000, 4096);
+
+    CopyMemAreaToCosim(&_ram, 0x100000);
+
+    return 0;
+  }
+
+  virtual bool Finish() {
+    std::cout << "Co-simulation matched " << _cosim->get_insn_cnt()
+              << " instructions\n";
+
+    return SimpleSystem::Finish();
+  }
+};
+
+// Use raw pointer as destruction outside main can cause segment fault (due to
+// undefined instruction order vs VerilatorSimCtrl singleton).
+SimpleSystemCosim *simple_system_cosim;
+
+extern "C" {
+void *get_spike_cosim() {
+  assert(simple_system_cosim);
+  return static_cast<Cosim *>(simple_system_cosim->_cosim.get());
+}
+}
+
+int main(int argc, char **argv) {
+  simple_system_cosim = new SimpleSystemCosim(
+      "TOP.ibex_simple_system.u_ram.u_ram.gen_generic.u_impl_generic",
+      (1024 * 1024) / 4);
+
+  int ret_code = simple_system_cosim->Main(argc, argv);
+
+  delete simple_system_cosim;
+
+  return ret_code;
+}

dv/verilator/simple_system_cosim/util/ibex_cosim_setup_check.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+# Copyright lowRISC contributors.
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+
+if [[ ! -v IBEX_COSIM_ISS_ROOT ]]; then
+  echo "IBEX_COSIM_ISS_ROOT must be set to the root directory of a suitable" \
+       "modified spike implementation, see" \
+       "dv/verilator/simple_system_cosim/README.md for more details"
+  exit 1;
+fi
+