diff --git a/dv/verilator/simple_system_cosim/README.md b/dv/verilator/simple_system_cosim/README.md new file mode 100644 index 00000000..3d198683 --- /dev/null +++ b/dv/verilator/simple_system_cosim/README.md @@ -0,0 +1,85 @@ +# Ibex Simple System with Co-simulation checking + +This augments the Ibex Simple System (`examples/simple_system`) to include the +co-simulation system to check Ibex's execution. This runs Spike in lockstep with +Ibex and checks each instruction Ibex retires matches what Spike has executed. +In addition all data memory accesses are checked against memory acceses Spike +has performed. More details on how the co-simulation works and how to build and +run simple system with it included can be in found in the Ibex documentation +under 'Co-simulation System' in the 'Ibex Reference Guide' section. + +## Quick Build and Run Instructions + +``` +# Get the Ibex co-simulation spike branch +git clone -b ibex_cosim https://github.com/lowRISC/riscv-isa-sim.git riscv-isa-sim-cosim + +# Setup build directory +cd riscv-isa-sim-cosim +mkdir build +cd build + +# Configure and build spike +../configure --enable-commitlog --enable-misaligned --prefix=/opt/spike-cosim +# Installs in /opt/spike-cosim +sudo make -j8 install + +# Setup IBEX_COSIM_ISS_ROOT for fusesoc build below +export IBEX_COSIM_ISS_ROOT=/opt/spike-cosim + +# Spike's libsoftfloat.so needs to be accessible so add it to LD_LIBRARY_PATH +export LD_LIBRARY_PATH=/opt/spike-cosim/lib:$LD_LIBRARY_PATH + +# Switch to a checkout of the Ibex repository +cd + +# Build simulator +fusesoc --cores-root=. run --target=sim --setup --build lowrisc:ibex:ibex_simple_system_cosim --RV32E=0 --RV32M=ibex_pkg::RV32MFast + +# Build coremark test binary, with performance counter dump disabled. The +# co-simulator system doesn't produce matching performance counters in spike so +# any read of those CSRs results in a mismatch and a failure. +make -C ./examples/sw/benchmarks/coremark SUPPRESS_PCOUNT_DUMP=1 + +# Run coremark binary with co-simulation checking +build/lowrisc_ibex_ibex_simple_system_cosim_0/sim-verilator/Vibex_simple_system --meminit=ram,examples/sw/benchmarks/coremark/coremark.elf +``` + +Sample output: + +``` +Simulation of Ibex +================== + +Tracing can be toggled by sending SIGUSR1 to this process: +$ kill -USR1 29121 + +Simulation running, end by pressing CTRL-c. +TOP.ibex_simple_system.u_top.u_ibex_tracer.unnamedblk1: Writing execution trace to trace_core_00000000.log +Terminating simulation by software request. +- ../src/lowrisc_ibex_sim_shared_0/./rtl/sim/simulator_ctrl.sv:93: Verilog $finish +Received $finish() from Verilog, shutting down simulation. + +Simulation statistics +===================== +Executed cycles: 4116797 +Wallclock time: 17.053 s +Simulation speed: 241412 cycles/s (241.412 kHz) +Co-simulation matched 2789425 instructions + +Performance Counters +==================== +Cycles: 4055056 +NONE: 0 +Instructions Retired: 2750348 +LSU Busy: 684533 +Fetch Wait: 187543 +Loads: 541082 +Stores: 143451 +Jumps: 57169 +Conditional Branches: 523452 +Taken Conditional Branches: 187543 +Compressed Instructions: 0 +Multiply Wait: 187920 +Divide Wait: 0 +``` diff --git a/dv/verilator/simple_system_cosim/ibex_cosim_setup_check.core b/dv/verilator/simple_system_cosim/ibex_cosim_setup_check.core new file mode 100644 index 00000000..17bc59dc --- /dev/null +++ b/dv/verilator/simple_system_cosim/ibex_cosim_setup_check.core @@ -0,0 +1,25 @@ +CAPI=2: +# Copyright lowRISC contributors. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 +name: "lowrisc:tool:ibex_cosim_setup_check:0.1" +description: "Check $IBEX_COSIM_ISS_ROOT is set" + +filesets: + files_ibex_cosim_setup_check: + files: + - ./util/ibex_cosim_setup_check.sh : { copyto: util/ibex_cosim_setup_check.sh } + +scripts: + ibex_cosim_setup_check: + cmd: + - sh + - util/ibex_cosim_setup_check.sh + +targets: + default: + filesets: + - files_ibex_cosim_setup_check + hooks: + pre_build: + - ibex_cosim_setup_check diff --git a/dv/verilator/simple_system_cosim/ibex_simple_system_cosim.core b/dv/verilator/simple_system_cosim/ibex_simple_system_cosim.core new file mode 100644 index 00000000..bc9acfca --- /dev/null +++ b/dv/verilator/simple_system_cosim/ibex_simple_system_cosim.core @@ -0,0 +1,160 @@ +CAPI=2: +# Copyright lowRISC contributors. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 +name: "lowrisc:ibex:ibex_simple_system_cosim" +description: "Generic simple system for running binaries on ibex using verilator" +filesets: + files_cosim: + depend: + - lowrisc:dv:cosim_dpi + - lowrisc:ibex:ibex_simple_system_core + - lowrisc:tool:ibex_cosim_setup_check + files: + - simple_system_cosim.cc: { file_type: cppSource } + - ibex_simple_system_cosim_checker.sv + - ibex_simple_system_cosim_checker_bind.sv + file_type: systemVerilogSource + +parameters: + RV32E: + datatype: int + paramtype: vlogparam + default: 0 + description: "Enable the E ISA extension (reduced register set) [0/1]" + + RV32M: + datatype: str + default: ibex_pkg::RV32MFast + paramtype: vlogdefine + description: "RV32M implementation parameter enum. See the ibex_pkg::rv32m_e enum in ibex_pkg.sv for permitted values." + + RV32B: + datatype: str + default: ibex_pkg::RV32BNone + paramtype: vlogdefine + description: "Bitmanip implementation parameter enum. See the ibex_pkg::rv32b_e enum in ibex_pkg.sv for permitted values." + + RegFile: + datatype: str + default: ibex_pkg::RegFileFF + paramtype: vlogdefine + description: "Register file implementation parameter enum. See the ibex_pkg::regfile_e enum in ibex_pkg.sv for permitted values." + + ICache: + datatype: int + default: 0 + paramtype: vlogparam + description: "Enable instruction cache" + + ICacheECC: + datatype: int + default: 0 + paramtype: vlogparam + description: "Enable ECC protection in instruction cache" + + SRAMInitFile: + datatype: str + paramtype: vlogparam + description: "Path to a vmem file to initialize the RAM with" + + BranchTargetALU: + datatype: int + paramtype: vlogparam + default: 0 + description: "Enables separate branch target ALU (increasing branch performance EXPERIMENTAL)" + + WritebackStage: + datatype: int + paramtype: vlogparam + default: 0 + description: "Enables third pipeline stage (EXPERIMENTAL)" + + SecureIbex: + datatype: int + default: 0 + paramtype: vlogparam + description: "Enables security hardening features (EXPERIMENTAL) [0/1]" + + BranchPredictor: + datatype: int + paramtype: vlogparam + default: 0 + description: "Enables static branch prediction (EXPERIMENTAL)" + + PMPEnable: + datatype: int + default: 0 + paramtype: vlogparam + description: "Enable PMP" + + PMPGranularity: + datatype: int + default: 0 + paramtype: vlogparam + description: "Granularity of NAPOT range, 0 = 4 byte, 1 = byte, 2 = 16 byte, 3 = 32 byte etc" + + PMPNumRegions: + datatype: int + default: 4 + paramtype: vlogparam + description: "Number of PMP regions" + +targets: + default: &default_target + filesets: + - files_cosim + toplevel: ibex_simple_system + parameters: + - RV32E + - RV32M + - RV32B + - RegFile + - ICache + - ICacheECC + - BranchTargetALU + - WritebackStage + - SecureIbex + - BranchPredictor + - PMPEnable + - PMPGranularity + - PMPNumRegions + - SRAMInitFile + + lint: + <<: *default_target + default_tool: verilator + tools: + verilator: + mode: lint-only + verilator_options: + - "-Wall" + # RAM primitives wider than 64bit (required for ECC) fail to build in + # Verilator without increasing the unroll count (see Verilator#1266) + - "--unroll-count 72" + + sim: + <<: *default_target + default_tool: verilator + tools: + vcs: + vcs_options: + - '-xlrm uniq_prior_final' + - '-debug_access+r' + verilator: + mode: cc + verilator_options: + # Disabling tracing reduces compile times but doesn't have a + # huge influence on runtime performance. + - '--trace' + - '--trace-fst' # this requires -DVM_TRACE_FMT_FST in CFLAGS below! + - '--trace-structs' + - '--trace-params' + - '--trace-max-array 1024' + - '-CFLAGS "-std=c++11 -Wall -DVL_USER_STOP -DVM_TRACE_FMT_FST -DTOPLEVEL_NAME=ibex_simple_system -g -I${IBEX_COSIM_ISS_ROOT}/include -I${IBEX_COSIM_ISS_ROOT}/include/softfloat"' + - '-LDFLAGS "-pthread -lutil -lelf -L${IBEX_COSIM_ISS_ROOT}/lib/ -g -lriscv -lsoftfloat -lfdt -ldisasm -ldl"' + - "-Wall" + - "-Wwarn-IMPERFECTSCH" + # RAM primitives wider than 64bit (required for ECC) fail to build in + # Verilator without increasing the unroll count (see Verilator#1266) + - "--unroll-count 72" diff --git a/dv/verilator/simple_system_cosim/ibex_simple_system_cosim_checker.sv b/dv/verilator/simple_system_cosim/ibex_simple_system_cosim_checker.sv new file mode 100644 index 00000000..608e9edd --- /dev/null +++ b/dv/verilator/simple_system_cosim/ibex_simple_system_cosim_checker.sv @@ -0,0 +1,81 @@ +// Copyright lowRISC contributors. +// Licensed under the Apache License, Version 2.0, see LICENSE for details. +// SPDX-License-Identifier: Apache-2.0 + +module ibex_simple_system_cosim_checker ( + input clk_i, + input rst_ni, + + input logic host_dmem_req, + input logic host_dmem_gnt, + input logic host_dmem_we, + input logic [31:0] host_dmem_addr, + input logic [3:0] host_dmem_be, + input logic [31:0] host_dmem_wdata, + + input logic host_dmem_rvalid, + input logic [31:0] host_dmem_rdata, + input logic host_dmem_err +); + import "DPI-C" function chandle get_spike_cosim; + + chandle cosim_handle; + + initial begin + cosim_handle = get_spike_cosim(); + end + + always @(posedge clk_i) begin + if (u_top.rvfi_valid & !u_top.rvfi_trap) begin + riscv_cosim_set_nmi(cosim_handle, u_top.rvfi_ext_nmi); + riscv_cosim_set_mip(cosim_handle, u_top.rvfi_ext_mip); + riscv_cosim_set_debug_req(cosim_handle, u_top.rvfi_ext_debug_req); + riscv_cosim_set_mcycle(cosim_handle, u_top.rvfi_ext_mcycle); + + if (riscv_cosim_step(cosim_handle, u_top.rvfi_rd_addr, u_top.rvfi_rd_wdata, + u_top.rvfi_pc_rdata, u_top.rvfi_trap) == 0) + begin + $display("FAILURE: Co-simulation mismatch at time %t", $time()); + for (int i = 0;i < riscv_cosim_get_num_errors(cosim_handle); ++i) begin + $display(riscv_cosim_get_error(cosim_handle, i)); + end + riscv_cosim_clear_errors(cosim_handle); + + $fatal(1, "Co-simulation mismatch seen"); + end + end + end + + logic outstanding_store; + logic [31:0] outstanding_addr; + logic [3:0] outstanding_be; + logic [31:0] outstanding_store_data; + logic outstanding_misaligned_first; + logic outstanding_misaligned_second; + + always @(posedge clk_i or negedge rst_ni) begin + if (!rst_ni) begin + outstanding_store <= 1'b0; + end else begin + if (host_dmem_req && host_dmem_gnt) begin + outstanding_store <= host_dmem_we; + outstanding_addr <= host_dmem_addr; + outstanding_be <= host_dmem_be; + outstanding_store_data <= host_dmem_wdata; + outstanding_misaligned_first <= + u_top.u_ibex_top.u_ibex_core.load_store_unit_i.handle_misaligned_d | + ((u_top.u_ibex_top.u_ibex_core.load_store_unit_i.lsu_type_i == 2'b01) & + (u_top.u_ibex_top.u_ibex_core.load_store_unit_i.data_offset == 2'b01)); + + outstanding_misaligned_second <= + u_top.u_ibex_top.u_ibex_core.load_store_unit_i.addr_incr_req_o; + end + + if (host_dmem_rvalid) begin + riscv_cosim_notify_dside_access(cosim_handle, outstanding_store, outstanding_addr, + outstanding_store ? outstanding_store_data : host_dmem_rdata, outstanding_be, + host_dmem_err, outstanding_misaligned_first, outstanding_misaligned_second); + end + end + end +endmodule diff --git a/dv/verilator/simple_system_cosim/ibex_simple_system_cosim_checker_bind.sv b/dv/verilator/simple_system_cosim/ibex_simple_system_cosim_checker_bind.sv new file mode 100644 index 00000000..a650e68b --- /dev/null +++ b/dv/verilator/simple_system_cosim/ibex_simple_system_cosim_checker_bind.sv @@ -0,0 +1,22 @@ +// Copyright lowRISC contributors. +// Licensed under the Apache License, Version 2.0, see LICENSE for details. +// SPDX-License-Identifier: Apache-2.0 + +module ibex_simple_system_cosim_checker_bind; + bind ibex_simple_system ibex_simple_system_cosim_checker + u_ibex_simple_system_cosim_checker_bind ( + .clk_i (IO_CLK), + .rst_ni (IO_RST_N), + + .host_dmem_req (host_req[CoreD]), + .host_dmem_gnt (host_gnt[CoreD]), + .host_dmem_we (host_we[CoreD]), + .host_dmem_addr (host_addr[CoreD]), + .host_dmem_be (host_be[CoreD]), + .host_dmem_wdata (host_wdata[CoreD]), + + .host_dmem_rvalid (host_rvalid[CoreD]), + .host_dmem_rdata (host_rdata[CoreD]), + .host_dmem_err (host_err[CoreD]) + ); +endmodule diff --git a/dv/verilator/simple_system_cosim/simple_system_cosim.cc b/dv/verilator/simple_system_cosim/simple_system_cosim.cc new file mode 100644 index 00000000..c54e7f20 --- /dev/null +++ b/dv/verilator/simple_system_cosim/simple_system_cosim.cc @@ -0,0 +1,73 @@ +// Copyright lowRISC contributors. +// Licensed under the Apache License, Version 2.0, see LICENSE for details. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include "cosim.h" +#include "ibex_simple_system.h" +#include "spike_cosim.h" +#include "verilator_memutil.h" + +class SimpleSystemCosim : public SimpleSystem { + public: + std::unique_ptr _cosim; + + SimpleSystemCosim(const char *ram_hier_path, int ram_size_words) + : SimpleSystem(ram_hier_path, ram_size_words), _cosim(nullptr) {} + + ~SimpleSystemCosim() {} + + protected: + void CopyMemAreaToCosim(MemArea *area, uint32_t base_addr) { + auto mem_data = area->Read(0, area->GetSizeWords()); + _cosim->backdoor_write_mem(base_addr, area->GetSizeBytes(), &mem_data[0]); + } + + virtual int Setup(int argc, char **argv, bool &exit_app) override { + int ret_code = SimpleSystem::Setup(argc, argv, exit_app); + if (exit_app) { + return ret_code; + } + + _cosim = std::make_unique( + 0x100080, 0x100001, "simple_system_cosim.log", false, false); + + _cosim->add_memory(0x100000, 1024 * 1024); + _cosim->add_memory(0x20000, 4096); + + CopyMemAreaToCosim(&_ram, 0x100000); + + return 0; + } + + virtual bool Finish() { + std::cout << "Co-simulation matched " << _cosim->get_insn_cnt() + << " instructions\n"; + + return SimpleSystem::Finish(); + } +}; + +// Use raw pointer as destruction outside main can cause segment fault (due to +// undefined instruction order vs VerilatorSimCtrl singleton). +SimpleSystemCosim *simple_system_cosim; + +extern "C" { +void *get_spike_cosim() { + assert(simple_system_cosim); + return static_cast(simple_system_cosim->_cosim.get()); +} +} + +int main(int argc, char **argv) { + simple_system_cosim = new SimpleSystemCosim( + "TOP.ibex_simple_system.u_ram.u_ram.gen_generic.u_impl_generic", + (1024 * 1024) / 4); + + int ret_code = simple_system_cosim->Main(argc, argv); + + delete simple_system_cosim; + + return ret_code; +} diff --git a/dv/verilator/simple_system_cosim/util/ibex_cosim_setup_check.sh b/dv/verilator/simple_system_cosim/util/ibex_cosim_setup_check.sh new file mode 100755 index 00000000..281c5b4c --- /dev/null +++ b/dv/verilator/simple_system_cosim/util/ibex_cosim_setup_check.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +# Copyright lowRISC contributors. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 + +if [[ ! -v IBEX_COSIM_ISS_ROOT ]]; then + echo "IBEX_COSIM_ISS_ROOT must be set to the root directory of a suitable" \ + "modified spike implementation, see" \ + "dv/verilator/simple_system_cosim/README.md for more details" + exit 1; +fi +