diff --git a/Bender.yml b/Bender.yml index fdd7f1273..56f3b8135 100644 --- a/Bender.yml +++ b/Bender.yml @@ -110,6 +110,7 @@ sources: - core/ariane_regfile_ff.sv - core/ariane_regfile_fpga.sv - core/scoreboard.sv + - core/round_interval.sv - core/store_buffer.sv - core/amo_buffer.sv - core/store_unit.sv diff --git a/Flist.ariane b/Flist.ariane index 12b462940..816353278 100644 --- a/Flist.ariane +++ b/Flist.ariane @@ -94,6 +94,7 @@ core/mmu_sv39x4/ptw_sv39x4.sv core/ariane_regfile_ff.sv core/re_name.sv core/scoreboard.sv +core/round_interval.sv core/store_buffer.sv core/amo_buffer.sv core/store_unit.sv diff --git a/core/Flist.cva6 b/core/Flist.cva6 index de27ed54d..aa885261a 100644 --- a/core/Flist.cva6 +++ b/core/Flist.cva6 @@ -126,6 +126,7 @@ ${CVA6_REPO_DIR}/core/ariane_regfile_ff.sv ${CVA6_REPO_DIR}/core/ariane_regfile_fpga.sv // NOTE: scoreboard.sv modified for DSIM (unchanged for other simulators) ${CVA6_REPO_DIR}/core/scoreboard.sv +${CVA6_REPO_DIR}/core/round_interval.sv ${CVA6_REPO_DIR}/core/store_buffer.sv ${CVA6_REPO_DIR}/core/amo_buffer.sv ${CVA6_REPO_DIR}/core/store_unit.sv diff --git a/core/commit_stage.sv b/core/commit_stage.sv index 8fad0b870..129422e62 100644 --- a/core/commit_stage.sv +++ b/core/commit_stage.sv @@ -36,6 +36,8 @@ module commit_stage input logic single_step_i, // The instruction we want to commit - ISSUE_STAGE input scoreboard_entry_t [CVA6Cfg.NrCommitPorts-1:0] commit_instr_i, + // The instruction is cancelled - ISSUE_STAGE + input logic [CVA6Cfg.NrCommitPorts-1:0] commit_drop_i, // Acknowledge that we are indeed committing - ISSUE_STAGE output logic [CVA6Cfg.NrCommitPorts-1:0] commit_ack_o, // Acknowledge that we are indeed committing - CSR_REGFILE @@ -149,128 +151,151 @@ module commit_stage csr_write_fflags_o = 1'b0; flush_commit_o = 1'b0; - // we will not commit the instruction if we took an exception - // and we do not commit the instruction if we requested a halt - if (commit_instr_i[0].valid && !commit_instr_i[0].ex.valid && !halt_i) begin - if (CVA6Cfg.RVZCMP && commit_instr_i[0].is_macro_instr && commit_instr_i[0].is_last_macro_instr) - commit_macro_ack[0] = 1'b1; - else commit_macro_ack[0] = 1'b0; - // we can definitely write the register file - // if the instruction is not committing anything the destination - commit_ack_o[0] = 1'b1; - if (CVA6Cfg.FpPresent && ariane_pkg::is_rd_fpr(commit_instr_i[0].op)) begin - we_fpr_o[0] = 1'b1; + // we do not commit the instruction yet if we requested a halt + if (commit_instr_i[0].valid && !halt_i) begin + // we will not commit the instruction if we took an exception + if (commit_instr_i[0].ex.valid) begin + // However we can drop it (with its exception) + if (commit_drop_i[0]) begin + commit_ack_o[0] = 1'b1; + end end else begin - we_gpr_o[0] = 1'b1; - end - // check whether the instruction we retire was a store - if ((!CVA6Cfg.RVA && commit_instr_i[0].fu == STORE) || (CVA6Cfg.RVA && commit_instr_i[0].fu == STORE && !instr_0_is_amo)) begin - // check if the LSU is ready to accept another commit entry (e.g.: a non-speculative store) - if (commit_lsu_ready_i) begin - commit_ack_o[0] = 1'b1; - commit_lsu_o = 1'b1; - // stall in case the store buffer is not able to accept anymore instructions - end else begin - commit_ack_o[0] = 1'b0; + commit_ack_o[0] = 1'b1; + + if (CVA6Cfg.RVZCMP && commit_instr_i[0].is_macro_instr && commit_instr_i[0].is_last_macro_instr) + commit_macro_ack[0] = 1'b1; + else commit_macro_ack[0] = 1'b0; + + if (!commit_drop_i[0]) begin + // we can definitely write the register file + // if the instruction is not committing anything the destination + if (CVA6Cfg.FpPresent && ariane_pkg::is_rd_fpr(commit_instr_i[0].op)) begin + we_fpr_o[0] = 1'b1; + end else begin + we_gpr_o[0] = 1'b1; + end end - end - // --------- - // FPU Flags - // --------- - if (CVA6Cfg.FpPresent) begin - if (commit_instr_i[0].fu inside {FPU, FPU_VEC}) begin - // write the CSR with potential exception flags from retiring floating point instruction - csr_wdata_o = {{CVA6Cfg.XLEN - 5{1'b0}}, commit_instr_i[0].ex.cause[4:0]}; - csr_write_fflags_o = 1'b1; - commit_ack_o[0] = 1'b1; + + // check whether the instruction we retire was a store + if (commit_instr_i[0].fu == STORE && !(CVA6Cfg.RVA && instr_0_is_amo)) begin + // check if the LSU is ready to accept another commit entry (e.g.: a non-speculative store) + if (commit_lsu_ready_i) begin + commit_lsu_o = 1'b1; + // stall in case the store buffer is not able to accept anymore instructions + end else begin + commit_ack_o[0] = 1'b0; + end end - end - // --------- - // CSR Logic - // --------- - // check whether the instruction we retire was a CSR instruction and it did not - // throw an exception - if (commit_instr_i[0].fu == CSR) begin - // write the CSR file - csr_op_o = commit_instr_i[0].op; - csr_wdata_o = commit_instr_i[0].result; - if (!csr_exception_i.valid) begin - commit_csr_o = 1'b1; - wdata_o[0] = csr_rdata_i; - commit_ack_o[0] = 1'b1; - end else begin - commit_ack_o[0] = 1'b0; - we_gpr_o[0] = 1'b0; + // --------- + // FPU Flags + // --------- + if (CVA6Cfg.FpPresent) begin + if (commit_instr_i[0].fu inside {FPU, FPU_VEC}) begin + if (!commit_drop_i[0]) begin + // write the CSR with potential exception flags from retiring floating point instruction + csr_wdata_o = {{CVA6Cfg.XLEN - 5{1'b0}}, commit_instr_i[0].ex.cause[4:0]}; + csr_write_fflags_o = 1'b1; + end + end + end + // --------- + // CSR Logic + // --------- + // check whether the instruction we retire was a CSR instruction and it did not + // throw an exception + if (commit_instr_i[0].fu == CSR) begin + // write the CSR file + csr_op_o = commit_instr_i[0].op; + csr_wdata_o = commit_instr_i[0].result; + if (!commit_drop_i[0]) begin + if (!csr_exception_i.valid) begin + commit_csr_o = 1'b1; + wdata_o[0] = csr_rdata_i; + end else begin + commit_ack_o[0] = 1'b0; + we_gpr_o[0] = 1'b0; + end + end + end + // ------------------ + // SFENCE.VMA Logic + // ------------------ + // sfence.vma is idempotent so we can safely re-execute it after returning + // from interrupt service routine + // check if this instruction was a SFENCE_VMA + if (CVA6Cfg.RVS && commit_instr_i[0].op == SFENCE_VMA) begin + if (!commit_drop_i[0]) begin + // no store pending so we can flush the TLBs and pipeline + sfence_vma_o = no_st_pending_i; + // wait for the store buffer to drain until flushing the pipeline + commit_ack_o[0] = no_st_pending_i; + end + end + // ------------------ + // HFENCE.VVMA Logic + // ------------------ + // hfence.vvma is idempotent so we can safely re-execute it after returning + // from interrupt service routine + // check if this instruction was a HFENCE_VVMA + if (CVA6Cfg.RVH && commit_instr_i[0].op == HFENCE_VVMA) begin + if (!commit_drop_i[0]) begin + // no store pending so we can flush the TLBs and pipeline + hfence_vvma_o = no_st_pending_i; + // wait for the store buffer to drain until flushing the pipeline + commit_ack_o[0] = no_st_pending_i; + end + end + // ------------------ + // HFENCE.GVMA Logic + // ------------------ + // hfence.gvma is idempotent so we can safely re-execute it after returning + // from interrupt service routine + // check if this instruction was a HFENCE_GVMA + if (CVA6Cfg.RVH && commit_instr_i[0].op == HFENCE_GVMA) begin + if (!commit_drop_i[0]) begin + // no store pending so we can flush the TLBs and pipeline + hfence_gvma_o = no_st_pending_i; + // wait for the store buffer to drain until flushing the pipeline + commit_ack_o[0] = no_st_pending_i; + end + end + // ------------------ + // FENCE.I Logic + // ------------------ + // fence.i is idempotent so we can safely re-execute it after returning + // from interrupt service routine + // Fence synchronizes data and instruction streams. That means that we need to flush the private icache + // and the private dcache. This is the most expensive instruction. + if (commit_instr_i[0].op == FENCE_I || (flush_dcache_i && CVA6Cfg.DCacheType == config_pkg::WB && commit_instr_i[0].fu != STORE)) begin + if (!commit_drop_i[0]) begin + commit_ack_o[0] = no_st_pending_i; + // tell the controller to flush the I$ + fence_i_o = no_st_pending_i; + end + end + // ------------------ + // FENCE Logic + // ------------------ + // fence is idempotent so we can safely re-execute it after returning + // from interrupt service routine + if (commit_instr_i[0].op == FENCE) begin + if (!commit_drop_i[0]) begin + commit_ack_o[0] = no_st_pending_i; + // tell the controller to flush the D$ + fence_o = no_st_pending_i; + end + end + // ------------------ + // AMO + // ------------------ + if (CVA6Cfg.RVA && instr_0_is_amo) begin + // AMO finished + commit_ack_o[0] = amo_resp_i.ack; + // flush the pipeline + flush_commit_o = amo_resp_i.ack; + amo_valid_commit_o = 1'b1; + we_gpr_o[0] = amo_resp_i.ack; end - end - // ------------------ - // SFENCE.VMA Logic - // ------------------ - // sfence.vma is idempotent so we can safely re-execute it after returning - // from interrupt service routine - // check if this instruction was a SFENCE_VMA - if (CVA6Cfg.RVS && commit_instr_i[0].op == SFENCE_VMA) begin - // no store pending so we can flush the TLBs and pipeline - sfence_vma_o = no_st_pending_i; - // wait for the store buffer to drain until flushing the pipeline - commit_ack_o[0] = no_st_pending_i; - end - // ------------------ - // HFENCE.VVMA Logic - // ------------------ - // hfence.vvma is idempotent so we can safely re-execute it after returning - // from interrupt service routine - // check if this instruction was a HFENCE_VVMA - if (CVA6Cfg.RVH && commit_instr_i[0].op == HFENCE_VVMA) begin - // no store pending so we can flush the TLBs and pipeline - hfence_vvma_o = no_st_pending_i; - // wait for the store buffer to drain until flushing the pipeline - commit_ack_o[0] = no_st_pending_i; - end - // ------------------ - // HFENCE.GVMA Logic - // ------------------ - // hfence.gvma is idempotent so we can safely re-execute it after returning - // from interrupt service routine - // check if this instruction was a HFENCE_GVMA - if (CVA6Cfg.RVH && commit_instr_i[0].op == HFENCE_GVMA) begin - // no store pending so we can flush the TLBs and pipeline - hfence_gvma_o = no_st_pending_i; - // wait for the store buffer to drain until flushing the pipeline - commit_ack_o[0] = no_st_pending_i; - end - // ------------------ - // FENCE.I Logic - // ------------------ - // fence.i is idempotent so we can safely re-execute it after returning - // from interrupt service routine - // Fence synchronizes data and instruction streams. That means that we need to flush the private icache - // and the private dcache. This is the most expensive instruction. - if (commit_instr_i[0].op == FENCE_I || (flush_dcache_i && CVA6Cfg.DCacheType == config_pkg::WB && commit_instr_i[0].fu != STORE)) begin - commit_ack_o[0] = no_st_pending_i; - // tell the controller to flush the I$ - fence_i_o = no_st_pending_i; - end - // ------------------ - // FENCE Logic - // ------------------ - // fence is idempotent so we can safely re-execute it after returning - // from interrupt service routine - if (commit_instr_i[0].op == FENCE) begin - commit_ack_o[0] = no_st_pending_i; - // tell the controller to flush the D$ - fence_o = no_st_pending_i; - end - // ------------------ - // AMO - // ------------------ - if (CVA6Cfg.RVA && instr_0_is_amo) begin - // AMO finished - commit_ack_o[0] = amo_resp_i.ack; - // flush the pipeline - flush_commit_o = amo_resp_i.ack; - amo_valid_commit_o = 1'b1; - we_gpr_o[0] = amo_resp_i.ack; end end @@ -296,26 +321,29 @@ module commit_stage if (!exception_o.valid && !commit_instr_i[1].ex.valid && (commit_instr_i[1].fu inside {ALU, LOAD, CTRL_FLOW, MULT, FPU, FPU_VEC})) begin - if (CVA6Cfg.FpPresent && ariane_pkg::is_rd_fpr(commit_instr_i[1].op)) we_fpr_o[1] = 1'b1; - else we_gpr_o[1] = 1'b1; - if (commit_instr_i[1].is_macro_instr && commit_instr_i[1].is_last_macro_instr) commit_macro_ack[1] = 1'b1; else commit_macro_ack[1] = 1'b0; commit_ack_o[1] = 1'b1; - // additionally check if we are retiring an FPU instruction because we need to make sure that we write all - // exception flags - if (CVA6Cfg.FpPresent && commit_instr_i[1].fu inside {FPU, FPU_VEC}) begin - if (csr_write_fflags_o) - csr_wdata_o = { - {CVA6Cfg.XLEN - 5{1'b0}}, - (commit_instr_i[0].ex.cause[4:0] | commit_instr_i[1].ex.cause[4:0]) - }; - else csr_wdata_o = {{CVA6Cfg.XLEN - 5{1'b0}}, commit_instr_i[1].ex.cause[4:0]}; + if (!commit_drop_i[1]) begin + if (CVA6Cfg.FpPresent && ariane_pkg::is_rd_fpr(commit_instr_i[1].op)) + we_fpr_o[1] = 1'b1; + else we_gpr_o[1] = 1'b1; - csr_write_fflags_o = 1'b1; + // additionally check if we are retiring an FPU instruction because we need to make sure that we write all + // exception flags + if (CVA6Cfg.FpPresent && commit_instr_i[1].fu inside {FPU, FPU_VEC}) begin + if (csr_write_fflags_o) + csr_wdata_o = { + {CVA6Cfg.XLEN - 5{1'b0}}, + (commit_instr_i[0].ex.cause[4:0] | commit_instr_i[1].ex.cause[4:0]) + }; + else csr_wdata_o = {{CVA6Cfg.XLEN - 5{1'b0}}, commit_instr_i[1].ex.cause[4:0]}; + + csr_write_fflags_o = 1'b1; + end end end end @@ -344,7 +372,7 @@ module commit_stage exception_o.gva = 1'b0; // we need a valid instruction in the commit stage - if (commit_instr_i[0].valid) begin + if (commit_instr_i[0].valid && !commit_drop_i[0]) begin // ------------------------ // check for CSR exception // ------------------------ diff --git a/core/cva6.sv b/core/cva6.sv index e63601d3c..09a2946cb 100644 --- a/core/cva6.sv +++ b/core/cva6.sv @@ -459,6 +459,9 @@ module cva6 // ID <-> COMMIT // -------------- scoreboard_entry_t [CVA6Cfg.NrCommitPorts-1:0] commit_instr_id_commit; + logic [CVA6Cfg.NrCommitPorts-1:0] commit_drop_id_commit; + logic [CVA6Cfg.NrCommitPorts-1:0] commit_ack_commit_id; + // -------------- // RVFI // -------------- @@ -668,59 +671,36 @@ module cva6 exception_t [CVA6Cfg.NrWbPorts-1:0] ex_ex_ex_id; // exception from execute, ex_stage to id_stage logic [CVA6Cfg.NrWbPorts-1:0] wt_valid_ex_id; + assign trans_id_ex_id[FLU_WB] = flu_trans_id_ex_id; + assign wbdata_ex_id[FLU_WB] = flu_result_ex_id; + assign ex_ex_ex_id[FLU_WB] = flu_exception_ex_id; + assign wt_valid_ex_id[FLU_WB] = flu_valid_ex_id; + + assign trans_id_ex_id[STORE_WB] = store_trans_id_ex_id; + assign wbdata_ex_id[STORE_WB] = store_result_ex_id; + assign ex_ex_ex_id[STORE_WB] = store_exception_ex_id; + assign wt_valid_ex_id[STORE_WB] = store_valid_ex_id; + + assign trans_id_ex_id[LOAD_WB] = load_trans_id_ex_id; + assign wbdata_ex_id[LOAD_WB] = load_result_ex_id; + assign ex_ex_ex_id[LOAD_WB] = load_exception_ex_id; + assign wt_valid_ex_id[LOAD_WB] = load_valid_ex_id; + + assign trans_id_ex_id[FPU_WB] = fpu_trans_id_ex_id; + assign wbdata_ex_id[FPU_WB] = fpu_result_ex_id; + assign ex_ex_ex_id[FPU_WB] = fpu_exception_ex_id; + assign wt_valid_ex_id[FPU_WB] = fpu_valid_ex_id; + if (CVA6Cfg.CvxifEn) begin - assign trans_id_ex_id = { - x_trans_id_ex_id, - flu_trans_id_ex_id, - load_trans_id_ex_id, - store_trans_id_ex_id, - fpu_trans_id_ex_id - }; - assign wbdata_ex_id = { - x_result_ex_id, flu_result_ex_id, load_result_ex_id, store_result_ex_id, fpu_result_ex_id - }; - assign ex_ex_ex_id = { - x_exception_ex_id, - flu_exception_ex_id, - load_exception_ex_id, - store_exception_ex_id, - fpu_exception_ex_id - }; - assign wt_valid_ex_id = { - x_valid_ex_id, flu_valid_ex_id, load_valid_ex_id, store_valid_ex_id, fpu_valid_ex_id - }; + assign trans_id_ex_id[X_WB] = x_trans_id_ex_id; + assign wbdata_ex_id[X_WB] = x_result_ex_id; + assign ex_ex_ex_id[X_WB] = x_exception_ex_id; + assign wt_valid_ex_id[X_WB] = x_valid_ex_id; end else if (CVA6Cfg.EnableAccelerator) begin - assign trans_id_ex_id = { - flu_trans_id_ex_id, - load_trans_id_ex_id, - store_trans_id_ex_id, - fpu_trans_id_ex_id, - acc_trans_id_ex_id - }; - assign wbdata_ex_id = { - flu_result_ex_id, load_result_ex_id, store_result_ex_id, fpu_result_ex_id, acc_result_ex_id - }; - assign ex_ex_ex_id = { - flu_exception_ex_id, - load_exception_ex_id, - store_exception_ex_id, - fpu_exception_ex_id, - acc_exception_ex_id - }; - assign wt_valid_ex_id = { - flu_valid_ex_id, load_valid_ex_id, store_valid_ex_id, fpu_valid_ex_id, acc_valid_ex_id - }; - end else begin - assign trans_id_ex_id = { - flu_trans_id_ex_id, load_trans_id_ex_id, store_trans_id_ex_id, fpu_trans_id_ex_id - }; - assign wbdata_ex_id = { - flu_result_ex_id, load_result_ex_id, store_result_ex_id, fpu_result_ex_id - }; - assign ex_ex_ex_id = { - flu_exception_ex_id, load_exception_ex_id, store_exception_ex_id, fpu_exception_ex_id - }; - assign wt_valid_ex_id = {flu_valid_ex_id, load_valid_ex_id, store_valid_ex_id, fpu_valid_ex_id}; + assign trans_id_ex_id[ACC_WB] = acc_trans_id_ex_id; + assign wbdata_ex_id[ACC_WB] = acc_result_ex_id; + assign ex_ex_ex_id[ACC_WB] = acc_exception_ex_id; + assign wt_valid_ex_id[ACC_WB] = acc_valid_ex_id; end if (CVA6Cfg.CvxifEn && CVA6Cfg.EnableAccelerator) begin : gen_err_xif_and_acc @@ -797,7 +777,8 @@ module cva6 .we_gpr_i (we_gpr_commit_id), .we_fpr_i (we_fpr_commit_id), .commit_instr_o (commit_instr_id_commit), - .commit_ack_i (commit_ack), + .commit_drop_o (commit_drop_id_commit), + .commit_ack_i (commit_ack_commit_id), // Performance Counters .stall_issue_o (stall_issue), //RVFI @@ -960,7 +941,8 @@ module cva6 .dirty_fp_state_o (dirty_fp_state), .single_step_i (single_step_csr_commit || single_step_acc_commit), .commit_instr_i (commit_instr_id_commit), - .commit_ack_o (commit_ack), + .commit_drop_i (commit_drop_id_commit), + .commit_ack_o (commit_ack_commit_id), .commit_macro_ack_o(commit_macro_ack), .no_st_pending_i (no_st_pending_commit), .waddr_o (waddr_commit_id), @@ -988,6 +970,8 @@ module cva6 .* ); + assign commit_ack = commit_ack_commit_id & ~commit_drop_id_commit; + // --------- // CSR // --------- @@ -1633,6 +1617,7 @@ module cva6 .rs2_forwarding_i(rs2_forwarding_id_ex), .commit_instr_i(commit_instr_id_commit), + .commit_drop_i (commit_drop_id_commit), .ex_commit_i (ex_commit), .priv_lvl_i (priv_lvl), diff --git a/core/cva6_rvfi.sv b/core/cva6_rvfi.sv index 7fd2342c4..afe353452 100644 --- a/core/cva6_rvfi.sv +++ b/core/cva6_rvfi.sv @@ -74,6 +74,7 @@ module cva6_rvfi logic [CVA6Cfg.NrCommitPorts-1:0][REG_ADDR_SIZE-1:0] commit_instr_rd; logic [CVA6Cfg.NrCommitPorts-1:0][CVA6Cfg.XLEN-1:0] commit_instr_result; logic [CVA6Cfg.NrCommitPorts-1:0] commit_instr_valid; + logic [CVA6Cfg.NrCommitPorts-1:0] commit_drop; logic [CVA6Cfg.XLEN-1:0] ex_commit_cause; logic ex_commit_valid; @@ -139,6 +140,7 @@ module cva6_rvfi assign commit_instr_rd = instr.commit_instr_rd; assign commit_instr_result = instr.commit_instr_result; assign commit_instr_valid = instr.commit_instr_valid; + assign commit_drop = instr.commit_drop; assign ex_commit_cause = instr.ex_commit_cause; assign ex_commit_valid = instr.ex_commit_valid; @@ -253,7 +255,7 @@ module cva6_rvfi end else if (lsu_wmask != 0) begin mem_n[lsu_addr_trans_id].lsu_addr = lsu_addr; mem_n[lsu_addr_trans_id].lsu_wmask = lsu_wmask; - mem_n[lsu_addr_trans_id].lsu_wdata = wbdata[1]; + mem_n[lsu_addr_trans_id].lsu_wdata = wbdata[STORE_WB]; end end @@ -272,8 +274,8 @@ module cva6_rvfi always_ff @(posedge clk_i) begin for (int i = 0; i < CVA6Cfg.NrCommitPorts; i++) begin logic exception; - exception = commit_instr_valid[i] && ex_commit_valid; - rvfi_instr_o[i].valid <= (commit_ack[i] && !ex_commit_valid) || + exception = commit_instr_valid[i] && ex_commit_valid && !commit_drop[i]; + rvfi_instr_o[i].valid <= (commit_ack[i] && !ex_commit_valid && !commit_drop[i]) || (exception && (ex_commit_cause == riscv::ENV_CALL_MMODE || ex_commit_cause == riscv::ENV_CALL_SMODE || ex_commit_cause == riscv::ENV_CALL_UMODE)); diff --git a/core/cva6_rvfi_probes.sv b/core/cva6_rvfi_probes.sv index a150c320f..f49997731 100644 --- a/core/cva6_rvfi_probes.sv +++ b/core/cva6_rvfi_probes.sv @@ -39,6 +39,7 @@ module cva6_rvfi_probes input logic [SUPERSCALAR:0][CVA6Cfg.VLEN-1:0] rs2_forwarding_i, input scoreboard_entry_t [CVA6Cfg.NrCommitPorts-1:0] commit_instr_i, + input logic [CVA6Cfg.NrCommitPorts-1:0] commit_drop_i, input exception_t ex_commit_i, input riscv::priv_lvl_t priv_lvl_i, @@ -103,6 +104,7 @@ module cva6_rvfi_probes instr.commit_instr_valid[i] = commit_instr_i[i].valid; end + instr.commit_drop = commit_drop_i; instr.commit_ack = commit_ack_i; instr.wdata = wdata_i; diff --git a/core/include/ariane_pkg.sv b/core/include/ariane_pkg.sv index fe5659c89..5585db4b5 100644 --- a/core/include/ariane_pkg.sv +++ b/core/include/ariane_pkg.sv @@ -169,6 +169,7 @@ package ariane_pkg; localparam int unsigned FETCH_FIFO_DEPTH = 4; localparam int unsigned SUPERSCALAR = cva6_config_pkg::CVA6ConfigSuperscalarEn; + localparam int unsigned SPECULATIVE_SB = SUPERSCALAR; typedef enum logic [2:0] { NoCF, // No control flow prediction @@ -202,6 +203,14 @@ package ariane_pkg; ACCEL // 10 } fu_t; + // Index of writeback ports + localparam FLU_WB = 0; + localparam STORE_WB = 1; + localparam LOAD_WB = 2; + localparam FPU_WB = 3; + localparam ACC_WB = 4; + localparam X_WB = 4; + localparam EXC_OFF_RST = 8'h80; localparam SupervisorIrq = 1; diff --git a/core/include/rvfi_types.svh b/core/include/rvfi_types.svh index 3d33d212e..5fd98dce1 100644 --- a/core/include/rvfi_types.svh +++ b/core/include/rvfi_types.svh @@ -112,6 +112,7 @@ logic [Cfg.NrCommitPorts-1:0][ariane_pkg::REG_ADDR_SIZE-1:0] commit_instr_rd; \ logic [Cfg.NrCommitPorts-1:0][Cfg.XLEN-1:0] commit_instr_result; \ logic [Cfg.NrCommitPorts-1:0] commit_instr_valid; \ + logic [Cfg.NrCommitPorts-1:0] commit_drop; \ logic [Cfg.XLEN-1:0] ex_commit_cause; \ logic ex_commit_valid; \ riscv::priv_lvl_t priv_lvl; \ diff --git a/core/issue_read_operands.sv b/core/issue_read_operands.sv index eef1b818f..998943b90 100644 --- a/core/issue_read_operands.sv +++ b/core/issue_read_operands.sv @@ -217,14 +217,25 @@ module issue_read_operands unique case (issue_instr_i[0].fu) NONE: fus_busy[1].none = 1'b1; CTRL_FLOW: begin - // There are no branch misses on a JAL - if (issue_instr_i[0].op == ariane_pkg::ADD) begin + if (ariane_pkg::SPECULATIVE_SB) begin + // Issue speculative instruction, will be removed on BMISS fus_busy[1].alu = 1'b1; fus_busy[1].ctrl_flow = 1'b1; fus_busy[1].csr = 1'b1; + // Speculative non-idempotent loads are not supported yet + fus_busy[1].load = 1'b1; + // The store buffer cannot be partially flushed yet + fus_busy[1].store = 1'b1; end else begin - // Control hazard - fus_busy[1] = '1; + // There are no branch misses on a JAL + if (issue_instr_i[0].op == ariane_pkg::ADD) begin + fus_busy[1].alu = 1'b1; + fus_busy[1].ctrl_flow = 1'b1; + fus_busy[1].csr = 1'b1; + end else begin + // Control hazard + fus_busy[1] = '1; + end end end ALU, CSR: begin diff --git a/core/issue_stage.sv b/core/issue_stage.sv index 6d8da085c..3c318722e 100644 --- a/core/issue_stage.sv +++ b/core/issue_stage.sv @@ -116,6 +116,8 @@ module issue_stage input logic [CVA6Cfg.NrCommitPorts-1:0] we_fpr_i, // Instructions to commit - COMMIT_STAGE output scoreboard_entry_t [CVA6Cfg.NrCommitPorts-1:0] commit_instr_o, + // Instruction is cancelled - COMMIT_STAGE + output logic [CVA6Cfg.NrCommitPorts-1:0] commit_drop_o, // Commit acknowledge - COMMIT_STAGE input logic [CVA6Cfg.NrCommitPorts-1:0] commit_ack_i, // Issue stall - PERF_COUNTERS diff --git a/core/round_interval.sv b/core/round_interval.sv new file mode 100644 index 000000000..9026347bc --- /dev/null +++ b/core/round_interval.sv @@ -0,0 +1,53 @@ +// Copyright 2024 Thales Silicon Security +// +// Licensed under the Solderpad Hardware Licence, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// SPDX-License-Identifier: Apache-2.0 WITH SHL-2.0 +// You may obtain a copy of the License at https://solderpad.org/licenses/ +// +// Original Author: Côme ALLART - Thales + +module round_interval #( + parameter int unsigned S = 1, + parameter int unsigned L = 1 << S +) ( + // Start index + // Included in the interval + input logic [S-1:0] start_i, + // Stop index + // Not included in the interval + input logic [S-1:0] stop_i, + + // The interval from start to stop, rounding + // Considered full when start_i == stop_i + output logic [L-1:0] active_o +); + + // Bit high at index start/stop + logic [L-1:0] a; + logic [L-1:0] b; + + for (genvar i = 0; i < L; i++) begin + assign a[i] = start_i == i; + assign b[i] = stop_i == i; + end + + // Propagation to the higher indexes: >= + logic [L-1:0] ge_a; + logic [L-1:0] ge_b; + + assign ge_b[0] = b[0]; + assign ge_a[0] = a[0]; + for (genvar i = 1; i < L; i++) begin + assign ge_b[i] = ge_b[i-1] || b[i]; + assign ge_a[i] = ge_a[i-1] || a[i]; + end + + // < is the negation of >= + logic [L-1:0] lt_b; + assign lt_b = ~ge_b; + + // Build the interval + assign active_o = (start_i <= stop_i) ? lt_b & ge_a : lt_b | ge_a; + +endmodule diff --git a/core/scoreboard.sv b/core/scoreboard.sv index 8ad3b0a39..d456ac0e3 100644 --- a/core/scoreboard.sv +++ b/core/scoreboard.sv @@ -59,6 +59,8 @@ module scoreboard #( // TO_BE_COMPLETED - TO_BE_COMPLETED output scoreboard_entry_t [CVA6Cfg.NrCommitPorts-1:0] commit_instr_o, // TO_BE_COMPLETED - TO_BE_COMPLETED + output logic [CVA6Cfg.NrCommitPorts-1:0] commit_drop_o, + // TO_BE_COMPLETED - TO_BE_COMPLETED input logic [CVA6Cfg.NrCommitPorts-1:0] commit_ack_i, // instruction to put on top of scoreboard e.g.: top pointer @@ -104,35 +106,51 @@ module scoreboard #( // this is the FIFO struct of the issue queue typedef struct packed { logic issued; // this bit indicates whether we issued this instruction e.g.: if it is valid + logic cancelled; // this instruction was cancelled (speculative scoreboard) logic is_rd_fpr_flag; // redundant meta info, added for speed scoreboard_entry_t sbe; // this is the score board entry we will send to ex } sb_mem_t; sb_mem_t [CVA6Cfg.NR_SB_ENTRIES-1:0] mem_q, mem_n; + logic [CVA6Cfg.NR_SB_ENTRIES-1:0] still_issued; + + logic [ariane_pkg::SUPERSCALAR:0] issue_full; + logic [1:0][CVA6Cfg.NR_SB_ENTRIES/2-1:0] issued_instrs_even_odd; + + logic bmiss; + logic [CVA6Cfg.TRANS_ID_BITS-1:0] after_flu_wb; + logic [CVA6Cfg.NR_SB_ENTRIES-1:0] speculative_instrs; - logic issue_full; logic [ariane_pkg::SUPERSCALAR:0] num_issue; - logic [CVA6Cfg.TRANS_ID_BITS:0] issue_cnt_n, issue_cnt_q; logic [CVA6Cfg.TRANS_ID_BITS-1:0] issue_pointer_n, issue_pointer_q; logic [ariane_pkg::SUPERSCALAR+1:0][CVA6Cfg.TRANS_ID_BITS-1:0] issue_pointer; logic [CVA6Cfg.NrCommitPorts-1:0][CVA6Cfg.TRANS_ID_BITS-1:0] commit_pointer_n, commit_pointer_q; logic [$clog2(CVA6Cfg.NrCommitPorts):0] num_commit; - // the issue queue is full don't issue any new instructions - // works since aligned to power of 2 - if (ariane_pkg::SUPERSCALAR) begin - assign issue_full = (issue_cnt_q[CVA6Cfg.TRANS_ID_BITS] == 1'b1) || &issue_cnt_q[CVA6Cfg.TRANS_ID_BITS-1:0]; - end else begin - assign issue_full = (issue_cnt_q[CVA6Cfg.TRANS_ID_BITS] == 1'b1); + for (genvar i = 0; i < CVA6Cfg.NR_SB_ENTRIES; i++) begin + assign still_issued[i] = mem_q[i].issued & ~mem_q[i].cancelled; end - assign sb_full_o = issue_full; + for (genvar i = 0; i < CVA6Cfg.NR_SB_ENTRIES; i++) begin + assign issued_instrs_even_odd[i%2][i/2] = mem_q[i].issued; + end + + // the issue queue is full don't issue any new instructions + assign issue_full[0] = &issued_instrs_even_odd[0] && &issued_instrs_even_odd[1]; + if (ariane_pkg::SUPERSCALAR) begin : assign_issue_full + // Need two slots available to issue two instructions. + // They are next to each other so one must be even and one odd + assign issue_full[1] = &issued_instrs_even_odd[0] || &issued_instrs_even_odd[1]; + end + + assign sb_full_o = issue_full[0]; // output commit instruction directly always_comb begin : commit_ports for (int unsigned i = 0; i < CVA6Cfg.NrCommitPorts; i++) begin commit_instr_o[i] = mem_q[commit_pointer_q[i]].sbe; commit_instr_o[i].trans_id = commit_pointer_q[i]; + commit_drop_o[i] = mem_q[commit_pointer_q[i]].cancelled; end end @@ -150,8 +168,8 @@ module scoreboard #( // make sure we assign the correct trans ID issue_instr_o[i].trans_id = issue_pointer[i]; - issue_instr_valid_o[i] = decoded_instr_valid_i[i] & ~issue_full; - decoded_instr_ack_o[i] = issue_ack_i[i] & ~issue_full; + issue_instr_valid_o[i] = decoded_instr_valid_i[i] & ~issue_full[i]; + decoded_instr_ack_o[i] = issue_ack_i[i] & ~issue_full[i]; end end @@ -168,12 +186,11 @@ module scoreboard #( // the decoded instruction we put in there is valid (1st bit) // increase the issue counter and advance issue pointer num_issue += 'd1; - mem_n[issue_pointer[i]] = { - 1'b1, // valid bit - (CVA6Cfg.FpPresent && ariane_pkg::is_rd_fpr( - decoded_instr_i[i].op - )), // whether rd goes to the fpr - decoded_instr_i[i] // decoded instruction record + mem_n[issue_pointer[i]] = '{ + issued: 1'b1, + cancelled: 1'b0, + is_rd_fpr_flag: CVA6Cfg.FpPresent && ariane_pkg::is_rd_fpr(decoded_instr_i[i].op), + sbe: decoded_instr_i[i] }; end end @@ -220,6 +237,19 @@ module scoreboard #( end end + // ------------ + // Cancel + // ------------ + if (ariane_pkg::SPECULATIVE_SB) begin + if (bmiss) begin + for (int unsigned i = 0; i < CVA6Cfg.NR_SB_ENTRIES; i++) begin + if (speculative_instrs[i]) begin + mem_n[i].cancelled = 1'b1; + end + end + end + end + // ------------ // Commit Port // ------------ @@ -228,6 +258,7 @@ module scoreboard #( if (commit_ack_i[i]) begin // this instruction is no longer in issue e.g.: it is considered finished mem_n[commit_pointer_q[i]].issued = 1'b0; + mem_n[commit_pointer_q[i]].cancelled = 1'b0; mem_n[commit_pointer_q[i]].sbe.valid = 1'b0; end end @@ -239,12 +270,26 @@ module scoreboard #( for (int unsigned i = 0; i < CVA6Cfg.NR_SB_ENTRIES; i++) begin // set all valid flags for all entries to zero mem_n[i].issued = 1'b0; + mem_n[i].cancelled = 1'b0; mem_n[i].sbe.valid = 1'b0; mem_n[i].sbe.ex.valid = 1'b0; end end end + assign bmiss = resolved_branch_i.valid && resolved_branch_i.is_mispredict; + assign after_flu_wb = trans_id_i[ariane_pkg::FLU_WB] + 'd1; + + if (ariane_pkg::SPECULATIVE_SB) begin : find_speculative_instrs + round_interval #( + .S(CVA6Cfg.TRANS_ID_BITS) + ) i_speculative_instrs ( + .start_i (after_flu_wb), + .stop_i (issue_pointer_q), + .active_o(speculative_instrs) + ); + end + // FIFO counter updates if (CVA6Cfg.NrCommitPorts == 2) begin : gen_commit_ports assign num_commit = commit_ack_i[1] + commit_ack_i[0]; @@ -252,11 +297,12 @@ module scoreboard #( assign num_commit = commit_ack_i[0]; end - assign issue_cnt_n = (flush_i) ? '0 : issue_cnt_q - {{CVA6Cfg.TRANS_ID_BITS - $clog2( - CVA6Cfg.NrCommitPorts - ) {1'b0}}, num_commit} + num_issue; assign commit_pointer_n[0] = (flush_i) ? '0 : commit_pointer_q[0] + num_commit; - assign issue_pointer_n = (flush_i) ? '0 : issue_pointer[num_issue]; + + always_comb begin : assign_issue_pointer_n + issue_pointer_n = issue_pointer[num_issue]; + if (flush_i) issue_pointer_n = '0; + end // precompute offsets for commit slots for (genvar k = 1; k < CVA6Cfg.NrCommitPorts; k++) begin : gen_cnt_incr @@ -284,8 +330,8 @@ module scoreboard #( // check for all valid entries and set the clobber accordingly for (int unsigned i = 0; i < CVA6Cfg.NR_SB_ENTRIES; i++) begin - gpr_clobber_vld[mem_q[i].sbe.rd][i] = mem_q[i].issued & ~mem_q[i].is_rd_fpr_flag; - fpr_clobber_vld[mem_q[i].sbe.rd][i] = mem_q[i].issued & mem_q[i].is_rd_fpr_flag; + gpr_clobber_vld[mem_q[i].sbe.rd][i] = still_issued[i] & ~mem_q[i].is_rd_fpr_flag; + fpr_clobber_vld[mem_q[i].sbe.rd][i] = still_issued[i] & mem_q[i].is_rd_fpr_flag; clobber_fu[i] = mem_q[i].sbe.fu; end @@ -347,25 +393,25 @@ module scoreboard #( // WB ports have higher prio than entries for (genvar i = 0; i <= ariane_pkg::SUPERSCALAR; i++) begin for (genvar k = 0; unsigned'(k) < CVA6Cfg.NrWbPorts; k++) begin : gen_rs_wb - assign rs1_fwd_req[i][k] = (mem_q[trans_id_i[k]].sbe.rd == rs1_i[i]) & wt_valid_i[k] & (~ex_i[k].valid) & (mem_q[trans_id_i[k]].is_rd_fpr_flag == (CVA6Cfg.FpPresent && ariane_pkg::is_rs1_fpr( + assign rs1_fwd_req[i][k] = (mem_q[trans_id_i[k]].sbe.rd == rs1_i[i]) & (~mem_q[trans_id_i[k]].cancelled) & wt_valid_i[k] & (~ex_i[k].valid) & (mem_q[trans_id_i[k]].is_rd_fpr_flag == (CVA6Cfg.FpPresent && ariane_pkg::is_rs1_fpr( issue_instr_o[i].op ))); - assign rs2_fwd_req[i][k] = (mem_q[trans_id_i[k]].sbe.rd == rs2_i[i]) & wt_valid_i[k] & (~ex_i[k].valid) & (mem_q[trans_id_i[k]].is_rd_fpr_flag == (CVA6Cfg.FpPresent && ariane_pkg::is_rs2_fpr( + assign rs2_fwd_req[i][k] = (mem_q[trans_id_i[k]].sbe.rd == rs2_i[i]) & (~mem_q[trans_id_i[k]].cancelled) & wt_valid_i[k] & (~ex_i[k].valid) & (mem_q[trans_id_i[k]].is_rd_fpr_flag == (CVA6Cfg.FpPresent && ariane_pkg::is_rs2_fpr( issue_instr_o[i].op ))); - assign rs3_fwd_req[i][k] = (mem_q[trans_id_i[k]].sbe.rd == rs3_i[i]) & wt_valid_i[k] & (~ex_i[k].valid) & (mem_q[trans_id_i[k]].is_rd_fpr_flag == (CVA6Cfg.FpPresent && ariane_pkg::is_imm_fpr( + assign rs3_fwd_req[i][k] = (mem_q[trans_id_i[k]].sbe.rd == rs3_i[i]) & (~mem_q[trans_id_i[k]].cancelled) & wt_valid_i[k] & (~ex_i[k].valid) & (mem_q[trans_id_i[k]].is_rd_fpr_flag == (CVA6Cfg.FpPresent && ariane_pkg::is_imm_fpr( issue_instr_o[i].op ))); assign rs_data[i][k] = wbdata_i[k]; end for (genvar k = 0; unsigned'(k) < CVA6Cfg.NR_SB_ENTRIES; k++) begin : gen_rs_entries - assign rs1_fwd_req[i][k+CVA6Cfg.NrWbPorts] = (mem_q[k].sbe.rd == rs1_i[i]) & mem_q[k].issued & mem_q[k].sbe.valid & (mem_q[k].is_rd_fpr_flag == (CVA6Cfg.FpPresent && ariane_pkg::is_rs1_fpr( + assign rs1_fwd_req[i][k+CVA6Cfg.NrWbPorts] = (mem_q[k].sbe.rd == rs1_i[i]) & still_issued[k] & mem_q[k].sbe.valid & (mem_q[k].is_rd_fpr_flag == (CVA6Cfg.FpPresent && ariane_pkg::is_rs1_fpr( issue_instr_o[i].op ))); - assign rs2_fwd_req[i][k+CVA6Cfg.NrWbPorts] = (mem_q[k].sbe.rd == rs2_i[i]) & mem_q[k].issued & mem_q[k].sbe.valid & (mem_q[k].is_rd_fpr_flag == (CVA6Cfg.FpPresent && ariane_pkg::is_rs2_fpr( + assign rs2_fwd_req[i][k+CVA6Cfg.NrWbPorts] = (mem_q[k].sbe.rd == rs2_i[i]) & still_issued[k] & mem_q[k].sbe.valid & (mem_q[k].is_rd_fpr_flag == (CVA6Cfg.FpPresent && ariane_pkg::is_rs2_fpr( issue_instr_o[i].op ))); - assign rs3_fwd_req[i][k+CVA6Cfg.NrWbPorts] = (mem_q[k].sbe.rd == rs3_i[i]) & mem_q[k].issued & mem_q[k].sbe.valid & (mem_q[k].is_rd_fpr_flag == (CVA6Cfg.FpPresent && ariane_pkg::is_imm_fpr( + assign rs3_fwd_req[i][k+CVA6Cfg.NrWbPorts] = (mem_q[k].sbe.rd == rs3_i[i]) & still_issued[k] & mem_q[k].sbe.valid & (mem_q[k].is_rd_fpr_flag == (CVA6Cfg.FpPresent && ariane_pkg::is_imm_fpr( issue_instr_o[i].op ))); assign rs_data[i][k+CVA6Cfg.NrWbPorts] = mem_q[k].sbe.result; @@ -455,11 +501,9 @@ module scoreboard #( always_ff @(posedge clk_i or negedge rst_ni) begin : regs if (!rst_ni) begin mem_q <= '{default: sb_mem_t'(0)}; - issue_cnt_q <= '0; commit_pointer_q <= '0; issue_pointer_q <= '0; end else begin - issue_cnt_q <= issue_cnt_n; issue_pointer_q <= issue_pointer_n; mem_q <= mem_n; commit_pointer_q <= commit_pointer_n;