diff --git a/Bender.yml b/Bender.yml index 9bea4ef86..139ca8c32 100644 --- a/Bender.yml +++ b/Bender.yml @@ -25,10 +25,9 @@ sources: - core/include/cv64a6_imafdcv_sv39_config_pkg.sv - core/include/riscv_pkg.sv - core/include/ariane_pkg.sv - - core/cva6_mmu/cva6_tlb.sv - - core/cva6_mmu/cva6_shared_tlb.sv - - core/cva6_mmu/cva6_mmu.sv - - core/cva6_mmu/cva6_ptw.sv + - core/mmu_sv39/tlb.sv + - core/mmu_sv39/mmu.sv + - core/mmu_sv39/ptw.sv - corev_apu/tb/common/mock_uart.sv - target: cv64a6_imafdc_sv39 @@ -36,10 +35,9 @@ sources: - core/include/cv64a6_imafdc_sv39_config_pkg.sv - core/include/riscv_pkg.sv - core/include/ariane_pkg.sv - - core/cva6_mmu/cva6_tlb.sv - - core/cva6_mmu/cva6_shared_tlb.sv - - core/cva6_mmu/cva6_mmu.sv - - core/cva6_mmu/cva6_ptw.sv + - core/mmu_sv39/tlb.sv + - core/mmu_sv39/mmu.sv + - core/mmu_sv39/ptw.sv - core/cva6_accel_first_pass_decoder_stub.sv - target: cv64a6_imafdc_sv39_wb @@ -47,10 +45,9 @@ sources: - core/include/cv64a6_imafdc_sv39_wb_config_pkg.sv - core/include/riscv_pkg.sv - core/include/ariane_pkg.sv - - core/cva6_mmu/cva6_tlb.sv - - core/cva6_mmu/cva6_shared_tlb.sv - - core/cva6_mmu/cva6_mmu.sv - - core/cva6_mmu/cva6_ptw.sv + - core/mmu_sv39/tlb.sv + - core/mmu_sv39/mmu.sv + - core/mmu_sv39/ptw.sv - core/cva6_accel_first_pass_decoder_stub.sv - target: cv32a6_imac_sv0 @@ -58,10 +55,9 @@ sources: - core/include/cv32a6_imac_sv0_config_pkg.sv - core/include/riscv_pkg.sv - core/include/ariane_pkg.sv - - core/cva6_mmu/cva6_tlb.sv - - core/cva6_mmu/cva6_shared_tlb.sv - - core/cva6_mmu/cva6_mmu.sv - - core/cva6_mmu/cva6_ptw.sv + - core/mmu_sv32/cva6_tlb_sv32.sv + - core/mmu_sv32/cva6_mmu_sv32.sv + - core/mmu_sv32/cva6_ptw_sv32.sv - core/cva6_accel_first_pass_decoder_stub.sv - target: cv32a6_imac_sv32 @@ -69,10 +65,9 @@ sources: - core/include/cv32a6_imac_sv32_config_pkg.sv - core/include/riscv_pkg.sv - core/include/ariane_pkg.sv - - core/cva6_mmu/cva6_tlb.sv - - core/cva6_mmu/cva6_shared_tlb.sv - - core/cva6_mmu/cva6_mmu.sv - - core/cva6_mmu/cva6_ptw.sv + - core/mmu_sv32/cva6_tlb_sv32.sv + - core/mmu_sv32/cva6_mmu_sv32.sv + - core/mmu_sv32/cva6_ptw_sv32.sv - core/cva6_accel_first_pass_decoder_stub.sv - target: cv32a6_imafc_sv32 @@ -80,10 +75,9 @@ sources: - core/include/cv32a6_imafc_sv32_config_pkg.sv - core/include/riscv_pkg.sv - core/include/ariane_pkg.sv - - core/cva6_mmu/cva6_tlb.sv - - core/cva6_mmu/cva6_shared_tlb.sv - - core/cva6_mmu/cva6_mmu.sv - - core/cva6_mmu/cva6_ptw.sv + - core/mmu_sv32/cva6_tlb_sv32.sv + - core/mmu_sv32/cva6_mmu_sv32.sv + - core/mmu_sv32/cva6_ptw_sv32.sv - core/cva6_accel_first_pass_decoder_stub.sv # included via target core/include/${TARGET_CFG}_config_pkg.sv diff --git a/Flist.ariane b/Flist.ariane index ae1d1d06c..a1f497638 100644 --- a/Flist.ariane +++ b/Flist.ariane @@ -82,20 +82,19 @@ core/issue_stage.sv core/load_unit.sv core/load_store_unit.sv core/lsu_bypass.sv -core/cva6_mmu/cva6_mmu.sv +core/mmu_sv39/mmu.sv core/mult.sv core/multiplier.sv core/serdiv.sv core/perf_counters.sv -core/cva6_mmu/cva6_ptw.sv +core/mmu_sv39/ptw.sv core/ariane_regfile_ff.sv core/re_name.sv core/scoreboard.sv core/store_buffer.sv core/amo_buffer.sv core/store_unit.sv -core/cva6_mmu/cva6_tlb.sv -core/cva6_mmu/cva6_shared_tlb.sv +core/mmu_sv39/tlb.sv core/commit_stage.sv core/cache_subsystem/wt_dcache_ctrl.sv core/cache_subsystem/wt_dcache_mem.sv diff --git a/ariane.core b/ariane.core index a1d4b27e9..501f296c1 100644 --- a/ariane.core +++ b/ariane.core @@ -33,18 +33,20 @@ filesets: - src/lsu_arbiter.sv - src/lsu.sv - src/miss_handler.sv - - src/cva6_mmu/cva6_mmu.sv + - src/mmu_sv39/mmu.sv + - src/mmu_sv32/cva6_mmu_sv32.sv - src/mult.sv - src/nbdcache.sv - src/pcgen_stage.sv - src/perf_counters.sv - - src/cva6_mmu/cva6_ptw.sv + - src/mmu_sv39/ptw.sv + - src/mmu_sv32/cva6_ptw_sv32.sv - src/regfile_ff.sv - src/scoreboard.sv - src/store_buffer.sv - src/store_unit.sv - - src/cva6_mmu/cva6_tlb.sv - - src/cva6_mmu/cva6_shared_tlb.sv + - src/mmu_sv39/tlb.sv + - src/mmu_sv32/cva6_tlb_sv32.sv file_type : systemVerilogSource depend : - pulp-platform.org::axi_mem_if diff --git a/core/Flist.cva6 b/core/Flist.cva6 index 7bd972bf7..daf4b28a6 100644 --- a/core/Flist.cva6 +++ b/core/Flist.cva6 @@ -182,10 +182,15 @@ ${CVA6_REPO_DIR}/common/local/util/tc_sram_wrapper.sv ${CVA6_REPO_DIR}/vendor/pulp-platform/tech_cells_generic/src/rtl/tc_sram.sv ${CVA6_REPO_DIR}/common/local/util/sram.sv -// MMU -${CVA6_REPO_DIR}/core/cva6_mmu/cva6_mmu.sv -${CVA6_REPO_DIR}/core/cva6_mmu/cva6_ptw.sv -${CVA6_REPO_DIR}/core/cva6_mmu/cva6_tlb.sv -${CVA6_REPO_DIR}/core/cva6_mmu/cva6_shared_tlb.sv +// MMU Sv39 +${CVA6_REPO_DIR}/core/mmu_sv39/mmu.sv +${CVA6_REPO_DIR}/core/mmu_sv39/ptw.sv +${CVA6_REPO_DIR}/core/mmu_sv39/tlb.sv + +// MMU Sv32 +${CVA6_REPO_DIR}/core/mmu_sv32/cva6_mmu_sv32.sv +${CVA6_REPO_DIR}/core/mmu_sv32/cva6_ptw_sv32.sv +${CVA6_REPO_DIR}/core/mmu_sv32/cva6_tlb_sv32.sv +${CVA6_REPO_DIR}/core/mmu_sv32/cva6_shared_tlb_sv32.sv // end of manifest diff --git a/core/cva6_mmu/cva6_mmu.sv b/core/cva6_mmu/cva6_mmu.sv deleted file mode 100644 index 10e6dd1c4..000000000 --- a/core/cva6_mmu/cva6_mmu.sv +++ /dev/null @@ -1,884 +0,0 @@ -// Copyright (c) 2018 ETH Zurich and University of Bologna. -// Copyright (c) 2021 Thales. -// Copyright (c) 2022 Bruno Sá and Zero-Day Labs. -// Copyright (c) 2024 PlanV Technology -// SPDX-License-Identifier: Apache-2.0 WITH SHL-2.1 -// Copyright and related rights are licensed under the Solderpad Hardware -// License, Version 0.51 (the "License"); you may not use this file except in -// compliance with the License. You may obtain a copy of the License at -// http://solderpad.org/licenses/SHL-0.51. Unless required by applicable law -// or agreed to in writing, software, hardware and materials distributed under -// this License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. -// -// Author: Angela Gonzalez, PlanV Technology -// Date: 26/02/2024 -// -// Description: Memory Management Unit for CVA6, contains TLB and -// address translation unit. SV32 SV39 and SV39x4 as defined in RISC-V -// privilege specification 1.11-WIP. -// This module is an merge of the MMU Sv39 developed -// by Florian Zaruba, the MMU Sv32 developed by Sebastien Jacq and the MMU Sv39x4 developed by Bruno Sá. - - -module cva6_mmu - import ariane_pkg::*; -#( - parameter config_pkg::cva6_cfg_t CVA6Cfg = config_pkg::cva6_cfg_empty, - // parameter ariane_pkg::ariane_cfg_t ArianeCfg = ariane_pkg::ArianeDefaultConfig, //This is the required config param in the hypervisor version for now - parameter int unsigned INSTR_TLB_ENTRIES = 4, - parameter int unsigned DATA_TLB_ENTRIES = 4, - parameter logic HYP_EXT = 0, - parameter int ASID_WIDTH [HYP_EXT:0], - parameter int unsigned VPN_LEN = 1, - parameter int unsigned PT_LEVELS = 1 - -) ( - input logic clk_i, - input logic rst_ni, - input logic flush_i, - input logic [HYP_EXT*2:0] enable_translation_i, //[v_i,enable_g_translation,enable_translation] - input logic [HYP_EXT*2:0] en_ld_st_translation_i, // enable virtual memory translation for ld/st - // IF interface - input icache_arsp_t icache_areq_i, - output icache_areq_t icache_areq_o, - // input icache_areq_o_t icache_areq_i, // this is the data type in the hypervisor version for now - // output icache_areq_i_t icache_areq_o, - - // LSU interface - // this is a more minimalistic interface because the actual addressing logic is handled - // in the LSU as we distinguish load and stores, what we do here is simple address translation - input exception_t misaligned_ex_i, - input logic lsu_req_i, // request address translation - input logic [riscv::VLEN-1:0] lsu_vaddr_i, // virtual address in - input riscv::xlen_t lsu_tinst_i, // transformed instruction in - input logic lsu_is_store_i, // the translation is requested by a store - output logic csr_hs_ld_st_inst_o, // hyp load store instruction - // if we need to walk the page table we can't grant in the same cycle - // Cycle 0 - output logic lsu_dtlb_hit_o, // sent in same cycle as the request if translation hits in DTLB - output logic [riscv::PPNW-1:0] lsu_dtlb_ppn_o, // ppn (send same cycle as hit) - // Cycle 1 - output logic lsu_valid_o, // translation is valid - output logic [riscv::PLEN-1:0] lsu_paddr_o, // translated address - output exception_t lsu_exception_o, // address translation threw an exception - // General control signals - input riscv::priv_lvl_t priv_lvl_i, - input riscv::priv_lvl_t ld_st_priv_lvl_i, - input logic [HYP_EXT:0] sum_i, - input logic [HYP_EXT:0] mxr_i, - input logic hlvx_inst_i, - input logic hs_ld_st_inst_i, - // input logic flag_mprv_i, - input logic [riscv::PPNW-1:0] satp_ppn_i[HYP_EXT*2:0], //[hgatp,vsatp,satp] - - input logic [ASID_WIDTH[0]-1:0] asid_i [HYP_EXT*2:0], //[vmid,vs_asid,asid] - input logic [ASID_WIDTH[0]-1:0] asid_to_be_flushed_i [ HYP_EXT:0], - input logic [ riscv::VLEN-1:0] vaddr_to_be_flushed_i[ HYP_EXT:0], - - input logic [HYP_EXT*2:0] flush_tlb_i, - - // Performance counters - output logic itlb_miss_o, - output logic dtlb_miss_o, - // PTW memory interface - input dcache_req_o_t req_port_i, - output dcache_req_i_t req_port_o, - // PMP - input riscv::pmpcfg_t [15:0] pmpcfg_i, - input logic [15:0][riscv::PLEN-3:0] pmpaddr_i -); - logic [ASID_WIDTH[0]-1:0] dtlb_mmu_asid_i[HYP_EXT:0]; - logic [ASID_WIDTH[0]-1:0] itlb_mmu_asid_i[HYP_EXT:0]; - - genvar b; - generate - for (b = 0; b < HYP_EXT + 1; b++) begin : gen_tlbs_asid - assign dtlb_mmu_asid_i[b] = b==0 ? - ((en_ld_st_translation_i[2*HYP_EXT] || flush_tlb_i[HYP_EXT]) ? asid_i[HYP_EXT] : asid_i[0]): - asid_i[HYP_EXT*2]; - assign itlb_mmu_asid_i[b] = b==0 ? - (enable_translation_i[2*HYP_EXT] ? asid_i[HYP_EXT] : asid_i[0]): - asid_i[HYP_EXT*2]; - end - endgenerate - - // memory management, pte for cva6 - localparam type pte_cva6_t = struct packed { - logic [riscv::PPNW-1:0] ppn; // PPN length for - logic [1:0] rsw; - logic d; - logic a; - logic g; - logic u; - logic x; - logic w; - logic r; - logic v; - }; - - localparam type tlb_update_cva6_t = struct packed { - logic valid; - logic [PT_LEVELS-2:0][HYP_EXT:0] is_page; - logic [VPN_LEN-1:0] vpn; - logic [HYP_EXT:0][ASID_WIDTH[0]-1:0] asid; - logic [HYP_EXT*2:0] v_st_enbl; // v_i,g-stage enabled, s-stage enabled - pte_cva6_t [HYP_EXT:0] content; - }; - - logic [HYP_EXT:0] iaccess_err; // insufficient privilege to access this instruction page - logic [HYP_EXT:0] daccess_err; // insufficient privilege to access this data page - logic ptw_active; // PTW is currently walking a page table - logic walking_instr; // PTW is walking because of an ITLB miss - logic [HYP_EXT*2:0] ptw_error; // PTW threw an exception - logic ptw_access_exception; // PTW threw an access exception (PMPs) - logic [HYP_EXT:0][riscv::PLEN-1:0] ptw_bad_paddr; // PTW guest page fault bad guest physical addr - - logic [riscv::VLEN-1:0] update_vaddr, shared_tlb_vaddr; - - tlb_update_cva6_t update_itlb, update_dtlb, update_shared_tlb; - - logic itlb_lu_access; - pte_cva6_t [ HYP_EXT:0] itlb_content; - logic [ PT_LEVELS-2:0] itlb_is_page; - logic itlb_lu_hit; - logic [ riscv::GPLEN-1:0] itlb_gpaddr; - logic [ASID_WIDTH[0]-1:0] itlb_lu_asid; - - logic dtlb_lu_access; - pte_cva6_t [ HYP_EXT:0] dtlb_content; - logic [ PT_LEVELS-2:0] dtlb_is_page; - logic [ASID_WIDTH[0]-1:0] dtlb_lu_asid; - logic dtlb_lu_hit; - logic [ riscv::GPLEN-1:0] dtlb_gpaddr; - - logic shared_tlb_access; - logic shared_tlb_hit, itlb_req; - - // Assignments - - assign itlb_lu_access = icache_areq_i.fetch_req; - assign dtlb_lu_access = lsu_req_i; - - - cva6_tlb #( - .pte_cva6_t (pte_cva6_t), - .tlb_update_cva6_t(tlb_update_cva6_t), - .TLB_ENTRIES (INSTR_TLB_ENTRIES), - .HYP_EXT (HYP_EXT), - .ASID_WIDTH (ASID_WIDTH), - .VPN_LEN (VPN_LEN), - .PT_LEVELS (PT_LEVELS) - ) i_itlb ( - .clk_i (clk_i), - .rst_ni (rst_ni), - .flush_i (flush_tlb_i), - .v_st_enbl_i (enable_translation_i), - .update_i (update_itlb), - .lu_access_i (itlb_lu_access), - .lu_asid_i (itlb_mmu_asid_i), - .asid_to_be_flushed_i (asid_to_be_flushed_i), - .vaddr_to_be_flushed_i(vaddr_to_be_flushed_i), - .lu_vaddr_i (icache_areq_i.fetch_vaddr), - .lu_content_o (itlb_content), - .lu_gpaddr_o (itlb_gpaddr), - .lu_is_page_o (itlb_is_page), - .lu_hit_o (itlb_lu_hit) - ); - - cva6_tlb #( - .pte_cva6_t (pte_cva6_t), - .tlb_update_cva6_t(tlb_update_cva6_t), - .TLB_ENTRIES (DATA_TLB_ENTRIES), - .HYP_EXT (HYP_EXT), - .ASID_WIDTH (ASID_WIDTH), - .VPN_LEN (VPN_LEN), - .PT_LEVELS (PT_LEVELS) - ) i_dtlb ( - .clk_i (clk_i), - .rst_ni (rst_ni), - .flush_i (flush_tlb_i), - .v_st_enbl_i (en_ld_st_translation_i), - .update_i (update_dtlb), - .lu_access_i (dtlb_lu_access), - .lu_asid_i (dtlb_mmu_asid_i), - .asid_to_be_flushed_i (asid_to_be_flushed_i), - .vaddr_to_be_flushed_i(vaddr_to_be_flushed_i), - .lu_vaddr_i (lsu_vaddr_i), - .lu_content_o (dtlb_content), - .lu_gpaddr_o (dtlb_gpaddr), - .lu_is_page_o (dtlb_is_page), - .lu_hit_o (dtlb_lu_hit) - ); - - - cva6_shared_tlb #( - .SHARED_TLB_DEPTH (64), - .SHARED_TLB_WAYS (2), - .HYP_EXT (HYP_EXT), - .ASID_WIDTH (ASID_WIDTH), - .VPN_LEN (VPN_LEN), - .PT_LEVELS (PT_LEVELS), - .pte_cva6_t (pte_cva6_t), - .tlb_update_cva6_t(tlb_update_cva6_t) - ) i_shared_tlb ( - .clk_i(clk_i), - .rst_ni(rst_ni), - .flush_i(flush_tlb_i), - .v_st_enbl_i({enable_translation_i, en_ld_st_translation_i}), - - .dtlb_asid_i (dtlb_mmu_asid_i), - .itlb_asid_i (itlb_mmu_asid_i), - // from TLBs - // did we miss? - .itlb_access_i(itlb_lu_access), - .itlb_hit_i (itlb_lu_hit), - .itlb_vaddr_i (icache_areq_i.fetch_vaddr), - - .dtlb_access_i(dtlb_lu_access), - .dtlb_hit_i (dtlb_lu_hit), - .dtlb_vaddr_i (lsu_vaddr_i), - - // to TLBs, update logic - .itlb_update_o(update_itlb), - .dtlb_update_o(update_dtlb), - - // Performance counters - .itlb_miss_o(itlb_miss_o), - .dtlb_miss_o(dtlb_miss_o), - - .shared_tlb_access_o(shared_tlb_access), - .shared_tlb_hit_o (shared_tlb_hit), - .shared_tlb_vaddr_o (shared_tlb_vaddr), - - .itlb_req_o (itlb_req), - // to update shared tlb - .shared_tlb_update_i(update_shared_tlb) - ); - - cva6_ptw #( - .CVA6Cfg (CVA6Cfg), - // .ArianeCfg ( ArianeCfg ), // this is the configuration needed in the hypervisor extension for now - .pte_cva6_t (pte_cva6_t), - .tlb_update_cva6_t(tlb_update_cva6_t), - .HYP_EXT (HYP_EXT), - .ASID_WIDTH (ASID_WIDTH), - .VPN_LEN (VPN_LEN), - .PT_LEVELS (PT_LEVELS) - ) i_ptw ( - .clk_i (clk_i), - .rst_ni (rst_ni), - .flush_i(flush_i), - - .ptw_active_o (ptw_active), - .walking_instr_o (walking_instr), - .ptw_error_o (ptw_error), - .ptw_access_exception_o(ptw_access_exception), - - .enable_translation_i (enable_translation_i), - .en_ld_st_translation_i(en_ld_st_translation_i), - - .lsu_is_store_i(lsu_is_store_i), - // PTW memory interface - .req_port_i (req_port_i), - .req_port_o (req_port_o), - - .asid_i(asid_i), - - .update_vaddr_o(update_vaddr), - - // to Shared TLB, update logic - .shared_tlb_update_o(update_shared_tlb), - - - // from shared TLB - // did we miss? - .shared_tlb_access_i(shared_tlb_access), - .shared_tlb_hit_i (shared_tlb_hit), - .shared_tlb_vaddr_i (shared_tlb_vaddr), - - .itlb_req_i(itlb_req), - - .hlvx_inst_i(hlvx_inst_i), - // from CSR file - .satp_ppn_i (satp_ppn_i), - .mxr_i (mxr_i), - - // Performance counters - .shared_tlb_miss_o(), //open for now - - // PMP - .pmpcfg_i (pmpcfg_i), - .pmpaddr_i (pmpaddr_i), - .bad_paddr_o(ptw_bad_paddr) - - ); - - //----------------------- - // Instruction Interface - //----------------------- - logic match_any_execute_region; - logic pmp_instr_allow; - localparam int PPNWMin = (riscv::PPNW - 1 > 29) ? 29 : riscv::PPNW - 1; - - // The instruction interface is a simple request response interface - always_comb begin : instr_interface - // MMU disabled: just pass through - icache_areq_o.fetch_valid = icache_areq_i.fetch_req; - icache_areq_o.fetch_paddr = icache_areq_i.fetch_vaddr[((riscv::PLEN > riscv::VLEN) ? riscv::VLEN -1: riscv::PLEN -1 ):0]; - // two potential exception sources: - // 1. HPTW threw an exception -> signal with a page fault exception - // 2. We got an access error because of insufficient permissions -> throw an access exception - icache_areq_o.fetch_exception = '0; - // Check whether we are allowed to access this memory region from a fetch perspective - iaccess_err[0] = icache_areq_i.fetch_req && (enable_translation_i[0] || HYP_EXT == 0) && // - (((priv_lvl_i == riscv::PRIV_LVL_U) && ~itlb_content[0].u) // - || ((priv_lvl_i == riscv::PRIV_LVL_S) && itlb_content[0].u)); - - if (HYP_EXT == 1) - iaccess_err[HYP_EXT] = icache_areq_i.fetch_req && enable_translation_i[HYP_EXT] && !itlb_content[HYP_EXT].u; - // MMU enabled: address from TLB, request delayed until hit. Error when TLB - // hit and no access right or TLB hit and translated address not valid (e.g. - // AXI decode error), or when PTW performs walk due to ITLB miss and raises - // an error. - if ((|enable_translation_i[HYP_EXT:0])) begin - // we work with SV39 or SV32, so if VM is enabled, check that all bits [riscv::VLEN-1:riscv::SV-1] are equal - if (icache_areq_i.fetch_req && !((&icache_areq_i.fetch_vaddr[riscv::VLEN-1:riscv::SV-1]) == 1'b1 || (|icache_areq_i.fetch_vaddr[riscv::VLEN-1:riscv::SV-1]) == 1'b0)) - if (HYP_EXT == 1) - icache_areq_o.fetch_exception = { - riscv::INSTR_ACCESS_FAULT, - {riscv::XLEN'(icache_areq_i.fetch_vaddr)}, - {riscv::GPLEN{1'b0}}, - {riscv::XLEN{1'b0}}, - enable_translation_i[HYP_EXT*2], - 1'b1 - }; - else - icache_areq_o.fetch_exception = { - riscv::INSTR_ACCESS_FAULT, {riscv::XLEN'(icache_areq_i.fetch_vaddr)}, 1'b1 - }; - - icache_areq_o.fetch_valid = 1'b0; - - icache_areq_o.fetch_paddr = { - (enable_translation_i[HYP_EXT] && HYP_EXT == 1)? itlb_content[HYP_EXT].ppn : itlb_content[0].ppn, - icache_areq_i.fetch_vaddr[11:0] - }; - - if (itlb_is_page[0]) begin - - icache_areq_o.fetch_paddr[PPNWMin:12] = icache_areq_i.fetch_vaddr[PPNWMin:12]; - - end else if (PT_LEVELS == 3 && itlb_is_page[PT_LEVELS-2]) begin - - icache_areq_o.fetch_paddr[PPNWMin-(VPN_LEN/PT_LEVELS):12] = icache_areq_i.fetch_vaddr[PPNWMin-(VPN_LEN/PT_LEVELS):12]; - - end - // ---------// - // ITLB Hit - // --------// - // if we hit the ITLB output the request signal immediately - if (itlb_lu_hit) begin - icache_areq_o.fetch_valid = icache_areq_i.fetch_req; - if (HYP_EXT == 1 && iaccess_err[HYP_EXT]) - icache_areq_o.fetch_exception = { - riscv::INSTR_GUEST_PAGE_FAULT, - {riscv::XLEN'(icache_areq_i.fetch_vaddr)}, - itlb_gpaddr[riscv::GPLEN-1:0], - {riscv::XLEN{1'b0}}, - enable_translation_i[HYP_EXT*2], - 1'b1 - }; - // we got an access error - else if (iaccess_err[0]) - // throw a page fault - if (HYP_EXT == 1) - icache_areq_o.fetch_exception = { - riscv::INSTR_PAGE_FAULT, - {riscv::XLEN'(icache_areq_i.fetch_vaddr)}, - {riscv::GPLEN{1'b0}}, - {riscv::XLEN{1'b0}}, - enable_translation_i[HYP_EXT*2], - 1'b1 - }; - else - icache_areq_o.fetch_exception = { - riscv::INSTR_PAGE_FAULT, - {{riscv::XLEN - riscv::VLEN{1'b0}}, icache_areq_i.fetch_vaddr}, - 1'b1 - }; - else if (!pmp_instr_allow) - if (HYP_EXT == 1) - icache_areq_o.fetch_exception = { - riscv::INSTR_ACCESS_FAULT, - {riscv::XLEN'(icache_areq_i.fetch_vaddr)}, - {riscv::GPLEN{1'b0}}, - {riscv::XLEN{1'b0}}, - enable_translation_i[HYP_EXT*2], - 1'b1 - }; - else - icache_areq_o.fetch_exception = { - riscv::INSTR_ACCESS_FAULT, riscv::XLEN'(icache_areq_i.fetch_vaddr), 1'b1 - }; - end else if (ptw_active && walking_instr) begin - // ---------// - // ITLB Miss - // ---------// - // watch out for exceptions happening during walking the page table - icache_areq_o.fetch_valid = ptw_error[0] | ptw_access_exception; - if (ptw_error[0]) - if (HYP_EXT == 1 && ptw_error[HYP_EXT]) - icache_areq_o.fetch_exception = { - riscv::INSTR_GUEST_PAGE_FAULT, - {riscv::XLEN'(update_vaddr)}, - ptw_bad_paddr[HYP_EXT][riscv::GPLEN-1:0], - (ptw_error[HYP_EXT*2] ? (riscv::IS_XLEN64 ? riscv::READ_64_PSEUDOINSTRUCTION : riscv::READ_32_PSEUDOINSTRUCTION) : {riscv::XLEN{1'b0}}), - enable_translation_i[2*HYP_EXT], - 1'b1 - }; - else if (HYP_EXT == 1) - icache_areq_o.fetch_exception = { - riscv::INSTR_PAGE_FAULT, - {riscv::XLEN'(update_vaddr)}, - {riscv::GPLEN{1'b0}}, - {riscv::XLEN{1'b0}}, - enable_translation_i[2*HYP_EXT], - 1'b1 - }; - else - icache_areq_o.fetch_exception = { - riscv::INSTR_PAGE_FAULT, {riscv::XLEN'(update_vaddr)}, 1'b1 - }; - else if (HYP_EXT == 1) - icache_areq_o.fetch_exception = { - riscv::INSTR_ACCESS_FAULT, - {riscv::XLEN'(update_vaddr)}, - {riscv::GPLEN{1'b0}}, - {riscv::XLEN{1'b0}}, - enable_translation_i[2*HYP_EXT], - 1'b1 - }; - else - icache_areq_o.fetch_exception = { - riscv::INSTR_ACCESS_FAULT, - ptw_bad_paddr[0][riscv::PLEN-1:(riscv::PLEN>riscv::VLEN)?(riscv::PLEN-riscv::VLEN) : 0], - 1'b1 - }; - end - end - - // if it didn't match any execute region throw an `Instruction Access Fault` - // or: if we are not translating, check PMPs immediately on the paddr - if ((!match_any_execute_region && (!ptw_error[0]|| HYP_EXT==0) ) || (!(|enable_translation_i[HYP_EXT:0]) && !pmp_instr_allow)) - if (HYP_EXT == 1) - icache_areq_o.fetch_exception = { - riscv::INSTR_ACCESS_FAULT, - {riscv::XLEN'(icache_areq_o.fetch_paddr)}, - {riscv::GPLEN{1'b0}}, - {riscv::XLEN{1'b0}}, - enable_translation_i[2*HYP_EXT], - 1'b1 - }; - else - icache_areq_o.fetch_exception = { - riscv::INSTR_ACCESS_FAULT, - riscv::VLEN'(icache_areq_o.fetch_paddr[riscv::PLEN-1:(riscv::PLEN > riscv::VLEN) ? (riscv::PLEN - riscv::VLEN) : 0]), - 1'b1 - }; - end - - // check for execute flag on memory - assign match_any_execute_region = config_pkg::is_inside_execute_regions( - CVA6Cfg, {{64 - riscv::PLEN{1'b0}}, icache_areq_o.fetch_paddr} - ); - // assign match_any_execute_region = ariane_pkg::is_inside_execute_regions(ArianeCfg, {{64-riscv::PLEN{1'b0}}, icache_areq_o.fetch_paddr}); // this is the package used in the hypervisor extension for now - - // Instruction fetch - pmp #( - .CVA6Cfg (CVA6Cfg), //comment for hypervisor extension - .PLEN (riscv::PLEN), - .PMP_LEN (riscv::PLEN - 2), - .NR_ENTRIES(CVA6Cfg.NrPMPEntries) - // .NR_ENTRIES ( ArianeCfg.NrPMPEntries ) // configuration used in hypervisor extension - ) i_pmp_if ( - .addr_i (icache_areq_o.fetch_paddr), - .priv_lvl_i, - // we will always execute on the instruction fetch port - .access_type_i(riscv::ACCESS_EXEC), - // Configuration - .conf_addr_i (pmpaddr_i), - .conf_i (pmpcfg_i), - .allow_o (pmp_instr_allow) - ); - - //----------------------- - // Data Interface - //----------------------- - logic [HYP_EXT:0][riscv::VLEN-1:0] lsu_vaddr_n, lsu_vaddr_q; - logic [riscv::XLEN-1:0] lsu_tinst_n, lsu_tinst_q; - logic hs_ld_st_inst_n, hs_ld_st_inst_q; - pte_cva6_t [HYP_EXT:0] dtlb_pte_n, dtlb_pte_q; - exception_t misaligned_ex_n, misaligned_ex_q; - logic lsu_req_n, lsu_req_q; - logic lsu_is_store_n, lsu_is_store_q; - logic dtlb_hit_n, dtlb_hit_q; - logic [PT_LEVELS-2:0] dtlb_is_page_n, dtlb_is_page_q; - - // check if we need to do translation or if we are always ready (e.g.: we are not translating anything) - assign lsu_dtlb_hit_o = (en_ld_st_translation_i[HYP_EXT:0]) ? dtlb_lu_hit : 1'b1; - - // Wires to PMP checks - riscv::pmp_access_t pmp_access_type; - logic pmp_data_allow; - - - // The data interface is simpler and only consists of a request/response interface - always_comb begin : data_interface - // save request and DTLB response - lsu_vaddr_n[0] = lsu_vaddr_i; - lsu_req_n = lsu_req_i; - misaligned_ex_n = misaligned_ex_i; - dtlb_pte_n = dtlb_content; - dtlb_hit_n = dtlb_lu_hit; - lsu_is_store_n = lsu_is_store_i; - dtlb_is_page_n = dtlb_is_page; - - lsu_valid_o = lsu_req_q; - lsu_exception_o = misaligned_ex_q; - pmp_access_type = lsu_is_store_q ? riscv::ACCESS_WRITE : riscv::ACCESS_READ; - - // mute misaligned exceptions if there is no request otherwise they will throw accidental exceptions - misaligned_ex_n.valid = misaligned_ex_i.valid & lsu_req_i; - - // Check if the User flag is set, then we may only access it in supervisor mode - // if SUM is enabled - daccess_err[0] = (en_ld_st_translation_i[0] || HYP_EXT==0)&& - ((ld_st_priv_lvl_i == riscv::PRIV_LVL_S && (en_ld_st_translation_i[HYP_EXT*2] ? !sum_i[HYP_EXT] : !sum_i[0] ) && dtlb_pte_q[0].u) || // SUM is not set and we are trying to access a user page in supervisor mode - (ld_st_priv_lvl_i == riscv::PRIV_LVL_U && !dtlb_pte_q[0].u)); - - if (HYP_EXT == 1) begin - lsu_tinst_n = lsu_tinst_i; - hs_ld_st_inst_n = hs_ld_st_inst_i; - lsu_vaddr_n[HYP_EXT] = dtlb_gpaddr; - csr_hs_ld_st_inst_o = hs_ld_st_inst_i || hs_ld_st_inst_q; - daccess_err[HYP_EXT] = en_ld_st_translation_i[HYP_EXT] && !dtlb_pte_q[HYP_EXT].u; - end - - lsu_paddr_o = (riscv::PLEN)'(lsu_vaddr_q[0]); - lsu_dtlb_ppn_o = (riscv::PPNW)'(lsu_vaddr_n[0][((riscv::PLEN > riscv::VLEN) ? riscv::VLEN -1: riscv::PLEN -1 ):12]); - - // translation is enabled and no misaligned exception occurred - if ((|en_ld_st_translation_i[HYP_EXT:0]) && !misaligned_ex_q.valid) begin - lsu_valid_o = 1'b0; - - lsu_dtlb_ppn_o = (en_ld_st_translation_i[HYP_EXT] && HYP_EXT == 1)? dtlb_content[HYP_EXT].ppn :dtlb_content[0].ppn; - lsu_paddr_o = { - (en_ld_st_translation_i[HYP_EXT] && HYP_EXT == 1)? dtlb_pte_q[HYP_EXT].ppn : dtlb_pte_q[0].ppn, - lsu_vaddr_q[0][11:0] - }; - // Mega page - if (dtlb_is_page_q[0]) begin - - lsu_dtlb_ppn_o[PPNWMin:12] = lsu_vaddr_n[0][PPNWMin:12]; - lsu_paddr_o[PPNWMin:12] = lsu_vaddr_q[0][PPNWMin:12]; - - end else if (PT_LEVELS == 3 && dtlb_is_page_q[PT_LEVELS-2]) begin - - lsu_paddr_o[PPNWMin-(VPN_LEN/PT_LEVELS):12] = lsu_vaddr_q[0][PPNWMin-(VPN_LEN/PT_LEVELS):12]; - lsu_dtlb_ppn_o[PPNWMin-(VPN_LEN/PT_LEVELS):12] = lsu_vaddr_n[0][PPNWMin-(VPN_LEN/PT_LEVELS):12]; - - end - - // --------- - // DTLB Hit - // -------- - if (dtlb_hit_q && lsu_req_q) begin - lsu_valid_o = 1'b1; - // exception priority: - // PAGE_FAULTS have higher priority than ACCESS_FAULTS - // virtual memory based exceptions are PAGE_FAULTS - // physical memory based exceptions are ACCESS_FAULTS (PMA/PMP) - - // this is a store - if (lsu_is_store_q) begin - // check if the page is write-able and we are not violating privileges - // also check if the dirty flag is set - if(HYP_EXT==1 && en_ld_st_translation_i[HYP_EXT] && (!dtlb_pte_q[HYP_EXT].w || daccess_err[HYP_EXT] || !dtlb_pte_q[HYP_EXT].d)) begin - lsu_exception_o = { - riscv::STORE_GUEST_PAGE_FAULT, - {{riscv::XLEN - riscv::VLEN{lsu_vaddr_q[0][riscv::VLEN-1]}}, lsu_vaddr_q[0]}, - lsu_vaddr_q[HYP_EXT][riscv::GPLEN-1:0], - {riscv::XLEN{1'b0}}, - en_ld_st_translation_i[HYP_EXT*2], - 1'b1 - }; - end else if ((en_ld_st_translation_i[0] || HYP_EXT==0) && (!dtlb_pte_q[0].w || daccess_err[0] || !dtlb_pte_q[0].d)) begin - if (HYP_EXT == 1) begin - lsu_exception_o = { - riscv::STORE_PAGE_FAULT, - {{riscv::XLEN - riscv::VLEN{lsu_vaddr_q[0][riscv::VLEN-1]}}, lsu_vaddr_q[0]}, - {riscv::GPLEN{1'b0}}, - lsu_tinst_q, - en_ld_st_translation_i[HYP_EXT*2], - 1'b1 - }; - end else begin - lsu_exception_o = { - riscv::STORE_PAGE_FAULT, - {{riscv::XLEN - riscv::VLEN{lsu_vaddr_q[0][riscv::VLEN-1]}}, lsu_vaddr_q[0]}, - 1'b1 - }; - end - // Check if any PMPs are violated - end else if (!pmp_data_allow) begin - if (HYP_EXT == 1) begin - lsu_exception_o = { - riscv::ST_ACCESS_FAULT, - {riscv::XLEN'(lsu_paddr_o)}, - {riscv::GPLEN{1'b0}}, - lsu_tinst_q, - en_ld_st_translation_i[HYP_EXT*2], - 1'b1 - }; - end else begin - lsu_exception_o = { - riscv::ST_ACCESS_FAULT, - riscv::XLEN'(lsu_paddr_o[riscv::PLEN-1:(riscv::PLEN > riscv::VLEN) ? (riscv::PLEN - riscv::VLEN) : 0]), - 1'b1 - }; - end - end - - // this is a load - end else begin - if (HYP_EXT == 1 && daccess_err[HYP_EXT]) begin - lsu_exception_o = { - riscv::LOAD_GUEST_PAGE_FAULT, - {{riscv::XLEN - riscv::VLEN{lsu_vaddr_q[0][riscv::VLEN-1]}}, lsu_vaddr_q[0]}, - lsu_vaddr_q[HYP_EXT][riscv::GPLEN-1:0], - {riscv::XLEN{1'b0}}, - en_ld_st_translation_i[HYP_EXT*2], - 1'b1 - }; - // check for sufficient access privileges - throw a page fault if necessary - end else if (daccess_err[0]) begin - if (HYP_EXT == 1) begin - lsu_exception_o = { - riscv::LOAD_PAGE_FAULT, - {{riscv::XLEN - riscv::VLEN{lsu_vaddr_q[0][riscv::VLEN-1]}}, lsu_vaddr_q[0]}, - {riscv::GPLEN{1'b0}}, - lsu_tinst_q, - en_ld_st_translation_i[HYP_EXT*2], - 1'b1 - }; - end else begin - lsu_exception_o = { - riscv::LOAD_PAGE_FAULT, - {{riscv::XLEN - riscv::VLEN{lsu_vaddr_q[0][riscv::VLEN-1]}}, lsu_vaddr_q[0]}, - 1'b1 - }; - end - // Check if any PMPs are violated - end else if (!pmp_data_allow) begin - if (HYP_EXT == 1) begin - lsu_exception_o = { - riscv::LD_ACCESS_FAULT, - {{riscv::XLEN - riscv::VLEN{lsu_vaddr_q[0][riscv::VLEN-1]}}, lsu_vaddr_q[0]}, - {riscv::GPLEN{1'b0}}, - lsu_tinst_q, - en_ld_st_translation_i[HYP_EXT*2], - 1'b1 - }; - end else begin - lsu_exception_o = { - riscv::LD_ACCESS_FAULT, - lsu_paddr_o[riscv::PLEN-1:(riscv::PLEN>riscv::VLEN)?(riscv::PLEN-riscv::VLEN) : 0], - 1'b1 - }; - end - end - end - end else - - // --------- - // DTLB Miss - // --------- - // watch out for exceptions - if (ptw_active && !walking_instr) begin - // page table walker threw an exception - if (ptw_error[0]) begin - // an error makes the translation valid - lsu_valid_o = 1'b1; - // the page table walker can only throw page faults - if (lsu_is_store_q) begin - if (HYP_EXT == 1 && ptw_error[HYP_EXT]) begin - lsu_exception_o = { - riscv::STORE_GUEST_PAGE_FAULT, - {{riscv::XLEN - riscv::VLEN{lsu_vaddr_q[0][riscv::VLEN-1]}}, update_vaddr}, - ptw_bad_paddr[HYP_EXT][riscv::GPLEN-1:0], - (ptw_error[HYP_EXT*2] ? (riscv::IS_XLEN64 ? riscv::READ_64_PSEUDOINSTRUCTION : riscv::READ_32_PSEUDOINSTRUCTION) : {riscv::XLEN{1'b0}}), - en_ld_st_translation_i[HYP_EXT*2], - 1'b1 - }; - end else begin - if (HYP_EXT == 1) begin - lsu_exception_o = { - riscv::STORE_PAGE_FAULT, - {{riscv::XLEN - riscv::VLEN{lsu_vaddr_q[0][riscv::VLEN-1]}}, update_vaddr}, - {riscv::GPLEN{1'b0}}, - lsu_tinst_q, - en_ld_st_translation_i[HYP_EXT*2], - 1'b1 - }; - end else begin - lsu_exception_o = { - riscv::STORE_PAGE_FAULT, - {{riscv::XLEN - riscv::VLEN{lsu_vaddr_q[0][riscv::VLEN-1]}}, update_vaddr}, - 1'b1 - }; - end - end - end else begin - if (HYP_EXT == 1 && ptw_error[HYP_EXT]) begin - lsu_exception_o = { - riscv::LOAD_GUEST_PAGE_FAULT, - {{riscv::XLEN - riscv::VLEN{lsu_vaddr_q[0][riscv::VLEN-1]}}, update_vaddr}, - ptw_bad_paddr[HYP_EXT][riscv::GPLEN-1:0], - (ptw_error[HYP_EXT*2] ? (riscv::IS_XLEN64 ? riscv::READ_64_PSEUDOINSTRUCTION : riscv::READ_32_PSEUDOINSTRUCTION) : {riscv::XLEN{1'b0}}), - en_ld_st_translation_i[HYP_EXT*2], - 1'b1 - }; - end else begin - if (HYP_EXT == 1) begin - lsu_exception_o = { - riscv::LOAD_PAGE_FAULT, - {{riscv::XLEN - riscv::VLEN{lsu_vaddr_q[0][riscv::VLEN-1]}}, update_vaddr}, - {riscv::GPLEN{1'b0}}, - lsu_tinst_q, - en_ld_st_translation_i[HYP_EXT*2], - 1'b1 - }; - end else begin - lsu_exception_o = { - riscv::LOAD_PAGE_FAULT, - {{riscv::XLEN - riscv::VLEN{lsu_vaddr_q[0][riscv::VLEN-1]}}, update_vaddr}, - 1'b1 - }; - end - end - end - end - - if (ptw_access_exception) begin - // an error makes the translation valid - lsu_valid_o = 1'b1; - // the page table walker can only throw page faults - if (HYP_EXT == 1) begin - lsu_exception_o = { - riscv::LD_ACCESS_FAULT, - {{riscv::XLEN - riscv::VLEN{lsu_vaddr_q[0][riscv::VLEN-1]}}, update_vaddr}, - {riscv::GPLEN{1'b0}}, - lsu_tinst_q, - en_ld_st_translation_i[HYP_EXT*2], - 1'b1 - }; - end else begin - lsu_exception_o = { - riscv::LD_ACCESS_FAULT, - ptw_bad_paddr[0][riscv::PLEN-1:(riscv::PLEN > riscv::VLEN) ? (riscv::PLEN - riscv::VLEN) : 0], - 1'b1 - }; - end - end - end - end // If translation is not enabled, check the paddr immediately against PMPs -else if (lsu_req_q && !misaligned_ex_q.valid && !pmp_data_allow) begin - if (lsu_is_store_q) begin - if (HYP_EXT == 1) begin - lsu_exception_o = { - riscv::ST_ACCESS_FAULT, - {{riscv::XLEN - riscv::VLEN{lsu_vaddr_q[0][riscv::VLEN-1]}}, update_vaddr}, - {riscv::GPLEN{1'b0}}, - lsu_tinst_q, - en_ld_st_translation_i[HYP_EXT*2], - 1'b1 - }; - end else - lsu_exception_o = { - riscv::ST_ACCESS_FAULT, - lsu_paddr_o[riscv::PLEN-1:(riscv::PLEN>riscv::VLEN)?(riscv::PLEN-riscv::VLEN) : 0], - 1'b1 - }; - end else begin - if (HYP_EXT == 1) begin - lsu_exception_o = { - riscv::LD_ACCESS_FAULT, - {{riscv::XLEN - riscv::VLEN{lsu_vaddr_q[0][riscv::VLEN-1]}}, update_vaddr}, - {riscv::GPLEN{1'b0}}, - lsu_tinst_q, - en_ld_st_translation_i[HYP_EXT*2], - 1'b1 - }; - end else begin - lsu_exception_o = { - riscv::LD_ACCESS_FAULT, - lsu_paddr_o[riscv::PLEN-1:(riscv::PLEN>riscv::VLEN)?(riscv::PLEN-riscv::VLEN) : 0], - 1'b1 - }; - end - end - end - end - - // Load/store PMP check - pmp #( - .CVA6Cfg (CVA6Cfg), // COMMENT IN HYPERVISOR EXTENSION - .PLEN (riscv::PLEN), - .PMP_LEN (riscv::PLEN - 2), - .NR_ENTRIES(CVA6Cfg.NrPMPEntries) - // .NR_ENTRIES ( ArianeCfg.NrPMPEntries ) // CONFIGURATION USED IN HYPERVISOR EXTENSION - ) i_pmp_data ( - .addr_i (lsu_paddr_o), - .priv_lvl_i (ld_st_priv_lvl_i), - .access_type_i(pmp_access_type), - // Configuration - .conf_addr_i (pmpaddr_i), - .conf_i (pmpcfg_i), - .allow_o (pmp_data_allow) - ); - - // ---------- - // Registers - // ---------- - always_ff @(posedge clk_i or negedge rst_ni) begin - if (~rst_ni) begin - lsu_vaddr_q <= '0; - lsu_req_q <= '0; - misaligned_ex_q <= '0; - dtlb_pte_q <= '0; - dtlb_hit_q <= '0; - lsu_is_store_q <= '0; - dtlb_is_page_q <= '0; - - if (HYP_EXT == 1) begin - lsu_tinst_q <= '0; - hs_ld_st_inst_q <= '0; - end - end else begin - lsu_vaddr_q <= lsu_vaddr_n; - lsu_req_q <= lsu_req_n; - misaligned_ex_q <= misaligned_ex_n; - dtlb_pte_q <= dtlb_pte_n; - dtlb_hit_q <= dtlb_hit_n; - lsu_is_store_q <= lsu_is_store_n; - dtlb_is_page_q <= dtlb_is_page_n; - - if (HYP_EXT == 1) begin - lsu_tinst_q <= lsu_tinst_n; - hs_ld_st_inst_q <= hs_ld_st_inst_n; - end - end - end -endmodule diff --git a/core/cva6_mmu/cva6_ptw.sv b/core/cva6_mmu/cva6_ptw.sv deleted file mode 100644 index 5923c3623..000000000 --- a/core/cva6_mmu/cva6_ptw.sv +++ /dev/null @@ -1,626 +0,0 @@ -// Copyright (c) 2018 ETH Zurich and University of Bologna. -// Copyright (c) 2021 Thales. -// Copyright (c) 2022 Bruno Sá and Zero-Day Labs. -// Copyright (c) 2024 PlanV Technology -// SPDX-License-Identifier: Apache-2.0 WITH SHL-2.1 -// Copyright and related rights are licensed under the Solderpad Hardware -// License, Version 0.51 (the "License"); you may not use this file except in -// compliance with the License. You may obtain a copy of the License at -// http://solderpad.org/licenses/SHL-0.51. Unless required by applicable law -// or agreed to in writing, software, hardware and materials distributed under -// this License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. -// -// Author: Angela Gonzalez, PlanV Technology -// Date: 26/02/2024 -// Description: Hardware-PTW (Page-Table-Walker) for CVA6 supporting sv32, sv39 and sv39x4. -// This module is an merge of the PTW Sv39 developed by Florian Zaruba, -// the PTW Sv32 developed by Sebastien Jacq and the PTW Sv39x4 by Bruno Sá. - -/* verilator lint_off WIDTH */ - -module cva6_ptw - import ariane_pkg::*; -#( - parameter type pte_cva6_t = logic, - parameter type tlb_update_cva6_t = logic, - parameter int unsigned HYP_EXT = 0, - parameter int ASID_WIDTH[HYP_EXT:0], - parameter int unsigned VPN_LEN = 1, - parameter config_pkg::cva6_cfg_t CVA6Cfg = config_pkg::cva6_cfg_empty, - parameter int unsigned PT_LEVELS = 1 -) ( - input logic clk_i, // Clock - input logic rst_ni, // Asynchronous reset active low - input logic flush_i, // flush everything, we need to do this because - // actually everything we do is speculative at this stage - // e.g.: there could be a CSR instruction that changes everything - output logic ptw_active_o, - output logic walking_instr_o, // set when walking for TLB - output logic [HYP_EXT*2:0] ptw_error_o, // set when an error occurred - output logic ptw_access_exception_o, // set when an PMP access exception occured - input logic [HYP_EXT*2:0] enable_translation_i, //[v_i,enable_g_translation,enable_translation] - input logic [HYP_EXT*2:0] en_ld_st_translation_i, // enable virtual memory translation for load/stores - input logic hlvx_inst_i, // is a HLVX load/store instruction - - input logic lsu_is_store_i, // this translation was triggered by a store - // PTW memory interface - input dcache_req_o_t req_port_i, - output dcache_req_i_t req_port_o, - - - // to TLBs, update logic - output tlb_update_cva6_t shared_tlb_update_o, - - output logic [riscv::VLEN-1:0] update_vaddr_o, - - input logic [ASID_WIDTH[0]-1:0] asid_i[HYP_EXT*2:0], //[vmid,vs_asid,asid] - - // from TLBs - // did we miss? - input logic shared_tlb_access_i, - input logic shared_tlb_hit_i, - input logic [riscv::VLEN-1:0] shared_tlb_vaddr_i, - - input logic itlb_req_i, - - // from CSR file - input logic [riscv::PPNW-1:0] satp_ppn_i[HYP_EXT*2:0], //[hgatp,vsatp,satp] - input logic [ HYP_EXT:0] mxr_i, - - // Performance counters - output logic shared_tlb_miss_o, - - // PMP - - input riscv::pmpcfg_t [15:0] pmpcfg_i, - input logic [15:0][riscv::PLEN-3:0] pmpaddr_i, - output logic [HYP_EXT:0][riscv::PLEN-1:0] bad_paddr_o - -); - - // input registers - logic data_rvalid_q; - riscv::xlen_t data_rdata_q; - - pte_cva6_t [HYP_EXT*2:0] pte; //[gpte_d,gpte_q,pte] - // register to perform context switch between stages - // pte_cva6_t gpte_q, gpte_d; - assign pte[0] = pte_cva6_t'(data_rdata_q[riscv::PPNW+9:0]); - - enum logic [2:0] { - IDLE, - WAIT_GRANT, - PTE_LOOKUP, - WAIT_RVALID, - PROPAGATE_ERROR, - PROPAGATE_ACCESS_ERROR, - LATENCY - } - state_q, state_d; - - logic [PT_LEVELS-2:0] misaligned_page; - logic [HYP_EXT:0][PT_LEVELS-2:0] ptw_lvl_n, ptw_lvl_q; - - // define 3 PTW stages to be used in sv39x4. sv32 and sv39 are always in S_STAGE - // S_STAGE -> S/VS-stage normal translation controlled by the satp/vsatp CSRs - // G_INTERMED_STAGE -> Converts the S/VS-stage non-leaf GPA pointers to HPA (controlled by hgatp) - // G_FINAL_STAGE -> Converts the S/VS-stage final GPA to HPA (controlled by hgatp) - enum logic [1:0] { - S_STAGE, - G_INTERMED_STAGE, - G_FINAL_STAGE - } - ptw_stage_q, ptw_stage_d; - - // is this an instruction page table walk? - logic is_instr_ptw_q, is_instr_ptw_n; - logic global_mapping_q, global_mapping_n; - // latched tag signal - logic tag_valid_n, tag_valid_q; - // register the ASIDs - logic [HYP_EXT:0][ASID_WIDTH[0]-1:0] tlb_update_asid_q, tlb_update_asid_n; - // register the VPN we need to walk, SV39 defines a 39 bit virtual address - logic [riscv::VLEN-1:0] vaddr_q, vaddr_n; - logic [HYP_EXT*2:0][PT_LEVELS-2:0][(VPN_LEN/PT_LEVELS)-1:0] vaddr_lvl; - // register the VPN we need to walk, SV39x4 defines a 41 bit virtual address for the G-Stage - logic [riscv::GPLEN-1:0] gpaddr_q, gpaddr_n, gpaddr_base; - logic [PT_LEVELS-2:0][riscv::GPLEN-1:0] gpaddr; - // 4 byte aligned physical pointer - logic [riscv::PLEN-1:0] ptw_pptr_q, ptw_pptr_n; - logic [riscv::PLEN-1:0] gptw_pptr_q, gptw_pptr_n; - - // Assignments - assign update_vaddr_o = vaddr_q; - - assign ptw_active_o = (state_q != IDLE); - assign walking_instr_o = is_instr_ptw_q; - // directly output the correct physical address - assign req_port_o.address_index = ptw_pptr_q[DCACHE_INDEX_WIDTH-1:0]; - assign req_port_o.address_tag = ptw_pptr_q[DCACHE_INDEX_WIDTH+DCACHE_TAG_WIDTH-1:DCACHE_INDEX_WIDTH]; - // we are never going to kill this request - assign req_port_o.kill_req = '0; - // we are never going to write with the HPTW - assign req_port_o.data_wdata = '0; - // we only issue one single request at a time - assign req_port_o.data_id = '0; - - // ----------- - // TLB Update - // ----------- - - assign gpaddr_base = {pte[0].ppn[riscv::GPPNW-1:0], vaddr_q[11:0]}; - - genvar z, w; - generate - for (z = 0; z < PT_LEVELS - 1; z++) begin - - // check if the ppn is correctly aligned: - // 6. If i > 0 and pa.ppn[i − 1 : 0] != 0, this is a misaligned superpage; stop and raise a page-fault - // exception. - assign misaligned_page[z] = (ptw_lvl_q[0] == (z)) && (pte[0].ppn[(VPN_LEN/PT_LEVELS)*(PT_LEVELS-1-z)-1:0] != '0); - - //record the vaddr corresponding to each level - for (w = 0; w < HYP_EXT * 2 + 1; w++) begin - assign vaddr_lvl[w][z] = w==0 ? vaddr_q[12+((VPN_LEN/PT_LEVELS)*(PT_LEVELS-z-1))-1:12+((VPN_LEN/PT_LEVELS)*(PT_LEVELS-z-2))] : - w==1 ? gptw_pptr_q[12+((VPN_LEN/PT_LEVELS)*(PT_LEVELS-z-1))-1:12+((VPN_LEN/PT_LEVELS)*(PT_LEVELS-z-2))]: - gpaddr_q[12+((VPN_LEN/PT_LEVELS)*(PT_LEVELS-z-1))-1:12+((VPN_LEN/PT_LEVELS)*(PT_LEVELS-z-2))]; - end - - if (HYP_EXT == 1) begin - assign gpaddr[z][VPN_LEN-(VPN_LEN/PT_LEVELS):0]= (ptw_lvl_q[0] == z) ? vaddr_q[VPN_LEN-(VPN_LEN/PT_LEVELS):0] : gpaddr_base[VPN_LEN-(VPN_LEN/PT_LEVELS):0]; - assign gpaddr[z][VPN_LEN:VPN_LEN-(VPN_LEN/PT_LEVELS)+1]= (ptw_lvl_q[0] == 0) ? vaddr_q[VPN_LEN:VPN_LEN-(VPN_LEN/PT_LEVELS)+1] : gpaddr_base[VPN_LEN:VPN_LEN-(VPN_LEN/PT_LEVELS)+1]; - assign gpaddr[z][riscv::GPLEN-1:VPN_LEN+1] = gpaddr_base[riscv::GPLEN-1:VPN_LEN+1]; - end - - - end - endgenerate - - always_comb begin : tlb_update - // update the correct page table level - for (int unsigned y = 0; y < HYP_EXT + 1; y++) begin - for (int unsigned x = 0; x < PT_LEVELS - 1; x++) begin - if((&enable_translation_i[HYP_EXT:0] || &en_ld_st_translation_i[HYP_EXT:0])&& HYP_EXT==1) begin - shared_tlb_update_o.is_page[x][y] = (ptw_lvl_q[y==HYP_EXT?0 : 1] == x); - end else if (enable_translation_i[0] || en_ld_st_translation_i[0] || HYP_EXT == 0) begin - shared_tlb_update_o.is_page[x][y] = y == 0 ? (ptw_lvl_q[0] == x) : 1'b0; - end else begin - shared_tlb_update_o.is_page[x][y] = y != 0 ? (ptw_lvl_q[0] == x) : 1'b0; - end - end - - // set the global mapping bit - if ((enable_translation_i[HYP_EXT] || en_ld_st_translation_i[HYP_EXT]) && HYP_EXT == 1) begin - shared_tlb_update_o.content[y] = y == 0 ? pte[HYP_EXT] | (global_mapping_q << 5) : pte[0]; - end else begin - shared_tlb_update_o.content[y] = y == 0 ? (pte[0] | (global_mapping_q << 5)) : '0; - end - end - // output the correct ASIDs - shared_tlb_update_o.asid = tlb_update_asid_q; - - bad_paddr_o[0] = ptw_access_exception_o ? ptw_pptr_q : 'b0; - if (HYP_EXT == 1) - bad_paddr_o[HYP_EXT][riscv::GPLEN:0] = ptw_error_o[HYP_EXT] ? ((ptw_stage_q == G_INTERMED_STAGE) ? gptw_pptr_q[riscv::GPLEN:0] : gpaddr_q) : 'b0; - end - - assign req_port_o.tag_valid = tag_valid_q; - - logic allow_access; - - - - pmp #( - .PLEN (riscv::PLEN), - .PMP_LEN (riscv::PLEN - 2), - .NR_ENTRIES(CVA6Cfg.NrPMPEntries) - ) i_pmp_ptw ( - .addr_i (ptw_pptr_q), - // PTW access are always checked as if in S-Mode... - .priv_lvl_i (riscv::PRIV_LVL_S), - // ...and they are always loads - .access_type_i(riscv::ACCESS_READ), - // Configuration - .conf_addr_i (pmpaddr_i), - .conf_i (pmpcfg_i), - .allow_o (allow_access) - ); - - - assign req_port_o.data_be = riscv::XLEN == 32 ? be_gen_32( - req_port_o.address_index[1:0], req_port_o.data_size - ) : be_gen( - req_port_o.address_index[2:0], req_port_o.data_size - ); - - assign shared_tlb_update_o.vpn = VPN_LEN'(vaddr_q[riscv::SV+HYP_EXT*2-1:12]); - - //------------------- - // Page table walker - //------------------- - // A virtual address va is translated into a physical address pa as follows: - // 1. Let a be sptbr.ppn × PAGESIZE, and let i = LEVELS-1. (For Sv39, - // PAGESIZE=2^12 and LEVELS=3.) - // 2. Let pte be the value of the PTE at address a+va.vpn[i]×PTESIZE. (For - // Sv32, PTESIZE=4.) - // 3. If pte.v = 0, or if pte.r = 0 and pte.w = 1, or if any bits or encodings - // that are reserved for future standard use are set within pte, stop and raise - // a page-fault exception corresponding to the original access type. - // 4. Otherwise, the PTE is valid. If pte.r = 1 or pte.x = 1, go to step 5. - // Otherwise, this PTE is a pointer to the next level of the page table. - // Let i=i-1. If i < 0, stop and raise an access exception. Otherwise, let - // a = pte.ppn × PAGESIZE and go to step 2. - // 5. A leaf PTE has been found. Determine if the requested memory access - // is allowed by the pte.r, pte.w, and pte.x bits. If not, stop and - // raise an access exception. Otherwise, the translation is successful. - // Set pte.a to 1, and, if the memory access is a store, set pte.d to 1. - // The translated physical address is given as follows: - // - pa.pgoff = va.pgoff. - // - If i > 0, then this is a superpage translation and - // pa.ppn[i-1:0] = va.vpn[i-1:0]. - // - pa.ppn[LEVELS-1:i] = pte.ppn[LEVELS-1:i]. - always_comb begin : ptw - automatic logic [riscv::PLEN-1:0] pptr; - // automatic logic [riscv::GPLEN-1:0] gpaddr; - // default assignments - // PTW memory interface - tag_valid_n = 1'b0; - req_port_o.data_req = 1'b0; - req_port_o.data_size = 2'(PT_LEVELS); - req_port_o.data_we = 1'b0; - ptw_error_o = '0; - ptw_access_exception_o = 1'b0; - shared_tlb_update_o.valid = 1'b0; - is_instr_ptw_n = is_instr_ptw_q; - ptw_lvl_n = ptw_lvl_q; - ptw_pptr_n = ptw_pptr_q; - state_d = state_q; - ptw_stage_d = ptw_stage_q; - global_mapping_n = global_mapping_q; - // input registers - tlb_update_asid_n = tlb_update_asid_q; - vaddr_n = vaddr_q; - pptr = ptw_pptr_q; - - if (HYP_EXT == 1) begin - gpaddr_n = gpaddr_q; - gptw_pptr_n = gptw_pptr_q; - end - - shared_tlb_miss_o = 1'b0; - - if (HYP_EXT == 1) pte[HYP_EXT*2] = pte[HYP_EXT]; - - case (state_q) - - IDLE: begin - // by default we start with the top-most page table - ptw_lvl_n = '0; - global_mapping_n = 1'b0; - is_instr_ptw_n = 1'b0; - - - if (HYP_EXT == 1) begin - pte[HYP_EXT*2] = '0; - gpaddr_n = '0; - end - - - // if we got an ITLB miss - if ((|enable_translation_i[HYP_EXT:0] || |en_ld_st_translation_i[HYP_EXT:0] || HYP_EXT == 0) && shared_tlb_access_i && ~shared_tlb_hit_i) begin - if ((&enable_translation_i[HYP_EXT:0] || &en_ld_st_translation_i[HYP_EXT:0]) && HYP_EXT==1) begin - ptw_stage_d = G_INTERMED_STAGE; - pptr = { - satp_ppn_i[HYP_EXT], - shared_tlb_vaddr_i[riscv::SV-1:riscv::SV-(VPN_LEN/PT_LEVELS)], - (PT_LEVELS)'(0) - }; - gptw_pptr_n = pptr; - ptw_pptr_n = { - satp_ppn_i[HYP_EXT*2][riscv::PPNW-1:2], - pptr[riscv::SV+HYP_EXT*2-1:riscv::SV-(VPN_LEN/PT_LEVELS)], - (PT_LEVELS)'(0) - }; - end else if (((|enable_translation_i[HYP_EXT:0] && !enable_translation_i[0]) || (|en_ld_st_translation_i[HYP_EXT:0] && !en_ld_st_translation_i[0])) && HYP_EXT==1) begin - ptw_stage_d = G_FINAL_STAGE; - gpaddr_n = shared_tlb_vaddr_i[riscv::SV+HYP_EXT*2-1:0]; - ptw_pptr_n = { - satp_ppn_i[HYP_EXT*2][riscv::PPNW-1:2], - shared_tlb_vaddr_i[riscv::SV+HYP_EXT*2-1:riscv::SV-(VPN_LEN/PT_LEVELS)], - (PT_LEVELS)'(0) - }; - end else begin - ptw_stage_d = S_STAGE; - if((enable_translation_i[HYP_EXT*2] || en_ld_st_translation_i[HYP_EXT*2]) && HYP_EXT==1) - ptw_pptr_n = { - satp_ppn_i[HYP_EXT], - shared_tlb_vaddr_i[riscv::SV-1:riscv::SV-(VPN_LEN/PT_LEVELS)], - (PT_LEVELS)'(0) - }; - else - ptw_pptr_n = { - satp_ppn_i[0], - shared_tlb_vaddr_i[riscv::SV-1:riscv::SV-(VPN_LEN/PT_LEVELS)], - (PT_LEVELS)'(0) - }; - end - - is_instr_ptw_n = itlb_req_i; - vaddr_n = shared_tlb_vaddr_i; - state_d = WAIT_GRANT; - shared_tlb_miss_o = 1'b1; - - for (int unsigned b = 0; b < HYP_EXT + 1; b++) begin - tlb_update_asid_n[b] = b==0 ? ((enable_translation_i[2*HYP_EXT] || en_ld_st_translation_i[2*HYP_EXT]) ? asid_i[HYP_EXT] : asid_i[0]) : asid_i[HYP_EXT*2]; - end - end - end - - WAIT_GRANT: begin - // send a request out - req_port_o.data_req = 1'b1; - // wait for the WAIT_GRANT - if (req_port_i.data_gnt) begin - // send the tag valid signal one cycle later - tag_valid_n = 1'b1; - state_d = PTE_LOOKUP; - end - end - - PTE_LOOKUP: begin - // we wait for the valid signal - if (data_rvalid_q) begin - - // check if the global mapping bit is set - if (pte[0].g && (ptw_stage_q == S_STAGE || HYP_EXT == 0)) global_mapping_n = 1'b1; - - // ------------- - // Invalid PTE - // ------------- - // If pte.v = 0, or if pte.r = 0 and pte.w = 1, stop and raise a page-fault exception. - if (!pte[0].v || (!pte[0].r && pte[0].w)) // || (|pte.reserved)) - state_d = PROPAGATE_ERROR; - // ----------- - // Valid PTE - // ----------- - else begin - state_d = LATENCY; - // it is a valid PTE - // if pte.r = 1 or pte.x = 1 it is a valid PTE - if (pte[0].r || pte[0].x) begin - if (HYP_EXT == 1) begin - case (ptw_stage_q) - S_STAGE: begin - if ((is_instr_ptw_q && enable_translation_i[HYP_EXT]) || (!is_instr_ptw_q && en_ld_st_translation_i[HYP_EXT])) begin - state_d = WAIT_GRANT; - ptw_stage_d = G_FINAL_STAGE; - if (HYP_EXT == 1) pte[HYP_EXT*2] = pte[0]; - ptw_lvl_n[HYP_EXT] = ptw_lvl_q[0]; - gpaddr_n = gpaddr[ptw_lvl_q[0]]; - ptw_pptr_n = { - satp_ppn_i[HYP_EXT*2][riscv::PPNW-1:2], - gpaddr[ptw_lvl_q[0]][riscv::SV+HYP_EXT*2-1:riscv::SV-(VPN_LEN/PT_LEVELS)], - (PT_LEVELS)'(0) - }; - ptw_lvl_n[0] = '0; - end - end - G_INTERMED_STAGE: begin - state_d = WAIT_GRANT; - ptw_stage_d = S_STAGE; - ptw_lvl_n[0] = ptw_lvl_q[HYP_EXT]; - pptr = {pte[0].ppn[riscv::GPPNW-1:0], gptw_pptr_q[11:0]}; - if (ptw_lvl_q[0] == 1) pptr[20:0] = gptw_pptr_q[20:0]; - if (ptw_lvl_q[0] == 0) pptr[29:0] = gptw_pptr_q[29:0]; - ptw_pptr_n = pptr; - end - default: ; - endcase - end - // Valid translation found (either 1G, 2M or 4K entry) - if (is_instr_ptw_q) begin - // ------------ - // Update ITLB - // ------------ - // If page is not executable, we can directly raise an error. This - // doesn't put a useless entry into the TLB. The same idea applies - // to the access flag since we let the access flag be managed by SW. - if (!pte[0].x || !pte[0].a) begin - state_d = PROPAGATE_ERROR; - if (HYP_EXT == 1) ptw_stage_d = ptw_stage_q; - end else if((ptw_stage_q == G_FINAL_STAGE) || !enable_translation_i[HYP_EXT] || HYP_EXT==0) - shared_tlb_update_o.valid = 1'b1; - - end else begin - // ------------ - // Update DTLB - // ------------ - // Check if the access flag has been set, otherwise throw a page-fault - // and let the software handle those bits. - // If page is not readable (there are no write-only pages) - // we can directly raise an error. This doesn't put a useless - // entry into the TLB. - if (pte[0].a && ((pte[0].r && !hlvx_inst_i) || (pte[0].x && (mxr_i[0] || hlvx_inst_i || (ptw_stage_q == S_STAGE && mxr_i[HYP_EXT] && en_ld_st_translation_i[HYP_EXT*2] && HYP_EXT==1))))) begin - if((ptw_stage_q == G_FINAL_STAGE) || !en_ld_st_translation_i[HYP_EXT] || HYP_EXT==0) - shared_tlb_update_o.valid = 1'b1; - end else begin - state_d = PROPAGATE_ERROR; - if (HYP_EXT == 1) ptw_stage_d = ptw_stage_q; - end - // Request is a store: perform some additional checks - // If the request was a store and the page is not write-able, raise an error - // the same applies if the dirty flag is not set - if (lsu_is_store_i && (!pte[0].w || !pte[0].d)) begin - shared_tlb_update_o.valid = 1'b0; - state_d = PROPAGATE_ERROR; - if (HYP_EXT == 1) ptw_stage_d = ptw_stage_q; - end - end - - //if there is a misaligned page, propagate error - if (|misaligned_page) begin - state_d = PROPAGATE_ERROR; - if (HYP_EXT == 1) ptw_stage_d = ptw_stage_q; - shared_tlb_update_o.valid = 1'b0; - end - - // check if 63:41 are all zeros - if (HYP_EXT==1 && ((enable_translation_i[HYP_EXT*2] && is_instr_ptw_q) || (en_ld_st_translation_i[HYP_EXT*2] && !is_instr_ptw_q)) && ptw_stage_q == S_STAGE && !((|pte[0].ppn[riscv::PPNW-HYP_EXT:riscv::GPPNW]) == 1'b0)) begin - state_d = PROPAGATE_ERROR; - ptw_stage_d = G_FINAL_STAGE; - end - // this is a pointer to the next TLB level - end else begin - // pointer to next level of page table - - if (ptw_lvl_q[0] == PT_LEVELS - 1) begin - // Should already be the last level page table => Error - ptw_lvl_n[0] = ptw_lvl_q[0]; - state_d = PROPAGATE_ERROR; - if (HYP_EXT == 1) ptw_stage_d = ptw_stage_q; - - - end else begin - ptw_lvl_n[0] = ptw_lvl_q[0] + 1'b1; - state_d = WAIT_GRANT; - - if (HYP_EXT == 1) begin - case (ptw_stage_q) - S_STAGE: begin - if (HYP_EXT==1 && ((is_instr_ptw_q && enable_translation_i[HYP_EXT]) || (!is_instr_ptw_q && en_ld_st_translation_i[HYP_EXT]))) begin - ptw_stage_d = G_INTERMED_STAGE; - if (HYP_EXT == 1) pte[HYP_EXT*2] = pte[0]; - ptw_lvl_n[HYP_EXT] = ptw_lvl_q[0] + 1; - pptr = {pte[0].ppn, vaddr_lvl[0][ptw_lvl_q[0]], (PT_LEVELS)'(0)}; - gptw_pptr_n = pptr; - ptw_pptr_n = { - satp_ppn_i[HYP_EXT*2][riscv::PPNW-1:2], - pptr[riscv::SV+HYP_EXT*2-1:riscv::SV-(VPN_LEN/PT_LEVELS)], - (PT_LEVELS)'(0) - }; - ptw_lvl_n[0] = '0; - end else begin - ptw_pptr_n = {pte[0].ppn, vaddr_lvl[0][ptw_lvl_q[0]], (PT_LEVELS)'(0)}; - end - end - G_INTERMED_STAGE: begin - ptw_pptr_n = {pte[0].ppn, vaddr_lvl[HYP_EXT][ptw_lvl_q[0]], (PT_LEVELS)'(0)}; - end - G_FINAL_STAGE: begin - ptw_pptr_n = { - pte[0].ppn, vaddr_lvl[HYP_EXT*2][ptw_lvl_q[0]], (PT_LEVELS)'(0) - }; - end - endcase - end else ptw_pptr_n = {pte[0].ppn, vaddr_lvl[0][ptw_lvl_q[0]], (PT_LEVELS)'(0)}; - - if (HYP_EXT == 1 && (pte[0].a || pte[0].d || pte[0].u)) begin - state_d = PROPAGATE_ERROR; - ptw_stage_d = ptw_stage_q; - end - - end - - // check if 63:41 are all zeros - if (HYP_EXT==1 && (((enable_translation_i[HYP_EXT*2] && is_instr_ptw_q) || (en_ld_st_translation_i[HYP_EXT*2] && !is_instr_ptw_q)) && ptw_stage_q == S_STAGE && !((|pte[0].ppn[riscv::PPNW-1:riscv::GPPNW]) == 1'b0))) begin - state_d = PROPAGATE_ERROR; - ptw_stage_d = ptw_stage_q; - end - end - end - - // Check if this access was actually allowed from a PMP perspective - if (!allow_access) begin - shared_tlb_update_o.valid = 1'b0; - // we have to return the failed address in bad_addr - ptw_pptr_n = ptw_pptr_q; - if (HYP_EXT == 1) ptw_stage_d = ptw_stage_q; - state_d = PROPAGATE_ACCESS_ERROR; - end - end - // we've got a data WAIT_GRANT so tell the cache that the tag is valid - end - // Propagate error to MMU/LSU - PROPAGATE_ERROR: begin - state_d = LATENCY; - ptw_error_o[0] = 1'b1; - if (HYP_EXT == 1) begin - ptw_error_o[HYP_EXT] = (ptw_stage_q != S_STAGE) ? 1'b1 : 1'b0; - ptw_error_o[HYP_EXT*2] = (ptw_stage_q == G_INTERMED_STAGE) ? 1'b1 : 1'b0; - end - end - PROPAGATE_ACCESS_ERROR: begin - state_d = LATENCY; - ptw_access_exception_o = 1'b1; - end - // wait for the rvalid before going back to IDLE - WAIT_RVALID: begin - if (data_rvalid_q) state_d = IDLE; - end - LATENCY: begin - state_d = IDLE; - end - default: begin - state_d = IDLE; - end - endcase - - // ------- - // Flush - // ------- - // should we have flushed before we got an rvalid, wait for it until going back to IDLE - if (flush_i) begin - // on a flush check whether we are - // 1. in the PTE Lookup check whether we still need to wait for an rvalid - // 2. waiting for a grant, if so: wait for it - // if not, go back to idle - if (((state_q inside {PTE_LOOKUP, WAIT_RVALID}) && !data_rvalid_q) || ((state_q == WAIT_GRANT) && req_port_i.data_gnt)) - state_d = WAIT_RVALID; - else state_d = LATENCY; - end - end - - // sequential process - always_ff @(posedge clk_i or negedge rst_ni) begin - if (~rst_ni) begin - state_q <= IDLE; - is_instr_ptw_q <= 1'b0; - ptw_lvl_q <= '0; - tag_valid_q <= 1'b0; - tlb_update_asid_q <= '0; - vaddr_q <= '0; - ptw_pptr_q <= '0; - global_mapping_q <= 1'b0; - data_rdata_q <= '0; - data_rvalid_q <= 1'b0; - if (HYP_EXT == 1) begin - gpaddr_q <= '0; - gptw_pptr_q <= '0; - ptw_stage_q <= S_STAGE; - pte[HYP_EXT] <= '0; - end - end else begin - state_q <= state_d; - ptw_pptr_q <= ptw_pptr_n; - is_instr_ptw_q <= is_instr_ptw_n; - ptw_lvl_q <= ptw_lvl_n; - tag_valid_q <= tag_valid_n; - tlb_update_asid_q <= tlb_update_asid_n; - vaddr_q <= vaddr_n; - global_mapping_q <= global_mapping_n; - data_rdata_q <= req_port_i.data_rdata; - data_rvalid_q <= req_port_i.data_rvalid; - - if (HYP_EXT == 1) begin - gpaddr_q <= gpaddr_n; - gptw_pptr_q <= gptw_pptr_n; - ptw_stage_q <= ptw_stage_d; - pte[HYP_EXT] <= pte[HYP_EXT*2]; - end - end - end - -endmodule -/* verilator lint_on WIDTH */ diff --git a/core/cva6_mmu/cva6_tlb.sv b/core/cva6_mmu/cva6_tlb.sv deleted file mode 100644 index f5c736993..000000000 --- a/core/cva6_mmu/cva6_tlb.sv +++ /dev/null @@ -1,459 +0,0 @@ -// Copyright (c) 2018 ETH Zurich and University of Bologna. -// Copyright (c) 2021 Thales. -// Copyright (c) 2022 Bruno Sá and Zero-Day Labs. -// Copyright (c) 2024 PlanV Technology -// SPDX-License-Identifier: Apache-2.0 WITH SHL-2.1 -// Copyright and related rights are licensed under the Solderpad Hardware -// License, Version 0.51 (the "License"); you may not use this file except in -// compliance with the License. You may obtain a copy of the License at -// http://solderpad.org/licenses/SHL-0.51. Unless required by applicable law -// or agreed to in writing, software, hardware and materials distributed under -// this License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. -// -// Author: Angela Gonzalez PlanV Technology -// Date: 26/02/2024 -// -// Description: Translation Lookaside Buffer, parameterizable to Sv32 or Sv39 , -// or sv39x4 fully set-associative -// This module is an merge of the Sv32 TLB developed by Sebastien -// Jacq (Thales Research & Technology), the Sv39 TLB developed -// by Florian Zaruba and David Schaffenrath and the Sv39x4 by Bruno Sá. - -module cva6_tlb - import ariane_pkg::*; -#( - parameter type pte_cva6_t = logic, - parameter type tlb_update_cva6_t = logic, - parameter int unsigned TLB_ENTRIES = 4, - parameter int unsigned HYP_EXT = 0, - parameter int ASID_WIDTH[HYP_EXT:0], //[vmid_width,asid_width] - parameter int unsigned VPN_LEN = 1, - parameter int unsigned PT_LEVELS = 1 -) ( - input logic clk_i, // Clock - input logic rst_ni, // Asynchronous reset active low - input logic [HYP_EXT*2:0] flush_i, // Flush signal [g_stage,vs stage, normal translation signal] - input logic [HYP_EXT*2:0] v_st_enbl_i, // v_i,g-stage enabled, s-stage enabled - // Update TLB - input tlb_update_cva6_t update_i, - // Lookup signals - input logic lu_access_i, - input logic [ASID_WIDTH[0]-1:0] lu_asid_i[HYP_EXT:0], //[lu_vmid,lu_asid] - input logic [riscv::VLEN-1:0] lu_vaddr_i, - output logic [riscv::GPLEN-1:0] lu_gpaddr_o, - output pte_cva6_t [HYP_EXT:0] lu_content_o, - input logic [ASID_WIDTH[0]-1:0] asid_to_be_flushed_i[HYP_EXT:0], //[vmid,asid] - input logic [riscv::VLEN-1:0] vaddr_to_be_flushed_i[HYP_EXT:0], // [gpaddr,vaddr] - output logic [PT_LEVELS-2:0] lu_is_page_o, - output logic lu_hit_o -); - - // computes the paddr based on the page size, ppn and offset - function automatic logic [(riscv::GPLEN-1):0] make_gpaddr( - input logic s_st_enbl, input logic is_1G, input logic is_2M, - input logic [(riscv::VLEN-1):0] vaddr, input riscv::pte_t pte); - logic [(riscv::GPLEN-1):0] gpaddr; - if (s_st_enbl) begin - gpaddr = {pte.ppn[(riscv::GPPNW-1):0], vaddr[11:0]}; - // Giga page - if (is_1G) gpaddr[29:12] = vaddr[29:12]; - // Mega page - if (is_2M) gpaddr[20:12] = vaddr[20:12]; - end else begin - gpaddr = vaddr[(riscv::GPLEN-1):0]; - end - return gpaddr; - endfunction : make_gpaddr - - // computes the final gppn based on the guest physical address - function automatic logic [(riscv::GPPNW-1):0] make_gppn(input logic s_st_enbl, input logic is_1G, - input logic is_2M, input logic [28:0] vpn, - input riscv::pte_t pte); - logic [(riscv::GPPNW-1):0] gppn; - if (s_st_enbl) begin - gppn = pte.ppn[(riscv::GPPNW-1):0]; - if (is_2M) gppn[8:0] = vpn[8:0]; - if (is_1G) gppn[17:0] = vpn[17:0]; - end else begin - gppn = vpn; - end - return gppn; - endfunction : make_gppn - - // SV39 defines three levels of page tables - struct packed { - logic [HYP_EXT:0][ASID_WIDTH[0]-1:0] asid; - logic [PT_LEVELS+HYP_EXT-1:0][(VPN_LEN/PT_LEVELS)-1:0] vpn; - logic [PT_LEVELS-2:0][HYP_EXT:0] is_page; - logic [HYP_EXT*2:0] v_st_enbl; // v_i,g-stage enabled, s-stage enabled - logic valid; - } [TLB_ENTRIES-1:0] - tags_q, tags_n; - - pte_cva6_t [TLB_ENTRIES-1:0][HYP_EXT:0] content_q, content_n; - - logic [TLB_ENTRIES-1:0][PT_LEVELS-1:0] vpn_match; - logic [TLB_ENTRIES-1:0][PT_LEVELS-1:0] level_match; - logic [TLB_ENTRIES-1:0][HYP_EXT:0][PT_LEVELS-1:0] vaddr_vpn_match; - logic [TLB_ENTRIES-1:0][HYP_EXT:0][PT_LEVELS-1:0] vaddr_level_match; - logic [TLB_ENTRIES-1:0] lu_hit; // to replacement logic - logic [TLB_ENTRIES-1:0] replace_en; // replace the following entry, set by replacement strategy - logic [TLB_ENTRIES-1:0][HYP_EXT:0] match_asid; - logic [TLB_ENTRIES-1:0][PT_LEVELS-1:0] page_match; - logic [TLB_ENTRIES-1:0][HYP_EXT:0][PT_LEVELS-1:0] vpage_match; - logic [TLB_ENTRIES-1:0][PT_LEVELS-2:0] is_page_o; - logic [TLB_ENTRIES-1:0] match_stage; - pte_cva6_t g_content; - logic [TLB_ENTRIES-1:0][(riscv::GPPNW-1):0] gppn; - logic [HYP_EXT*2:0] v_st_enbl; - - assign v_st_enbl = (HYP_EXT == 1) ? v_st_enbl_i : '1; - //------------- - // Translation - //------------- - - genvar i, x, z, w; - generate - for (i = 0; i < TLB_ENTRIES; i++) begin - for (x = 0; x < PT_LEVELS; x++) begin - //identify page_match for all TLB Entries - assign page_match[i][x] = x==0 ? 1 :((HYP_EXT==0 || x==(PT_LEVELS-1)) ? // PAGE_MATCH CONTAINS THE MATCH INFORMATION FOR EACH TAG OF is_1G and is_2M in sv39x4. HIGHER LEVEL (Giga page), THEN THERE IS THE Mega page AND AT THE LOWER LEVEL IS ALWAYS 1 - &(tags_q[i].is_page[PT_LEVELS-1-x] | (~v_st_enbl[HYP_EXT:0])): - ((&v_st_enbl[HYP_EXT:0]) ? - ((tags_q[i].is_page[PT_LEVELS-1-x][0] && (tags_q[i].is_page[PT_LEVELS-2-x][HYP_EXT] || tags_q[i].is_page[PT_LEVELS-1-x][HYP_EXT])) - || (tags_q[i].is_page[PT_LEVELS-1-x][HYP_EXT] && (tags_q[i].is_page[PT_LEVELS-2-x][0] || tags_q[i].is_page[PT_LEVELS-1-x][0]))): - tags_q[i].is_page[PT_LEVELS-1-x][0] && v_st_enbl[0] || tags_q[i].is_page[PT_LEVELS-1-x][HYP_EXT] && v_st_enbl[HYP_EXT])); - - //identify if vpn matches at all PT levels for all TLB entries - assign vpn_match[i][x] = (HYP_EXT == 1 && x == (PT_LEVELS - 1) && ~v_st_enbl[0]) ? // - lu_vaddr_i[12+((VPN_LEN/PT_LEVELS)*(x+1))-1:12+((VPN_LEN/PT_LEVELS)*x)] == tags_q[i].vpn[x] && lu_vaddr_i[12+HYP_EXT*(VPN_LEN-1): 12+HYP_EXT*(VPN_LEN-(VPN_LEN%PT_LEVELS))] == tags_q[i].vpn[x+HYP_EXT][(VPN_LEN%PT_LEVELS)-HYP_EXT:0]: // - lu_vaddr_i[12+((VPN_LEN/PT_LEVELS)*(x+1))-1:12+((VPN_LEN/PT_LEVELS)*x)] == tags_q[i].vpn[x]; - - //identify if there is a hit at each PT level for all TLB entries - assign level_match[i][x] = &vpn_match[i][PT_LEVELS-1:x] && page_match[i][x]; - - //identify vpage_match for all TLB Entries and vaddr_level match (if there is a hit at each PT level for all TLB entries on the vaddr) - for (z = 0; z < HYP_EXT + 1; z++) begin - assign vpage_match[i][z][x] = x == 0 ? 1 : tags_q[i].is_page[PT_LEVELS-1-x][z]; - assign vaddr_level_match[i][z][x]= &vaddr_vpn_match[i][z][PT_LEVELS-1:x] && vpage_match[i][z][x]; - - end - //identify if virtual address vpn matches at all PT levels for all TLB entries - assign vaddr_vpn_match[i][0][x] = vaddr_to_be_flushed_i[0][12+((VPN_LEN/PT_LEVELS)*(x+1))-1:12+((VPN_LEN/PT_LEVELS)*x)] == tags_q[i].vpn[x]; - - end - - - - if (HYP_EXT == 1) begin - //identify if GPADDR matches the GPPN - assign vaddr_vpn_match[i][HYP_EXT][0] = (vaddr_to_be_flushed_i[HYP_EXT][20:12] == gppn[i][8:0]); - assign vaddr_vpn_match[i][HYP_EXT][HYP_EXT] = (vaddr_to_be_flushed_i[HYP_EXT][29:21] == gppn[i][17:9]); - assign vaddr_vpn_match[i][HYP_EXT][HYP_EXT*2] = (vaddr_to_be_flushed_i[HYP_EXT][30+riscv::GPPN2:30] == gppn[i][18+riscv::GPPN2:18]); - - end - - for (w = 0; w < PT_LEVELS - 1; w++) begin - assign is_page_o[i][w] = page_match[i][PT_LEVELS - 1 - w]; //THIS REORGANIZES THE PAGES TO MATCH THE OUTPUT STRUCTURE (2M,1G) - end - end - endgenerate - - always_comb begin : translation - - // default assignment - lu_hit = '{default: 0}; - lu_hit_o = 1'b0; - lu_content_o = '{default: 0}; - lu_is_page_o = '{default: 0}; - match_asid = '{default: 0}; - match_stage = '{default: 0}; - g_content = '{default: 0}; - lu_gpaddr_o = '{default: 0}; - - - for (int unsigned i = 0; i < TLB_ENTRIES; i++) begin - // first level match, this may be a giga page, check the ASID flags as well - // if the entry is associated to a global address, don't match the ASID (ASID is don't care) - match_asid[i][0] = (((lu_asid_i[0][ASID_WIDTH[0]-1:0] == tags_q[i].asid[0][ASID_WIDTH[0]-1:0]) || content_q[i][0].g) && v_st_enbl[0]) || !v_st_enbl[0]; - - if (HYP_EXT == 1) begin - match_asid[i][HYP_EXT] = (lu_asid_i[HYP_EXT][ASID_WIDTH[HYP_EXT]-1:0] == tags_q[i].asid[HYP_EXT][ASID_WIDTH[HYP_EXT]-1:0] && v_st_enbl[HYP_EXT]) || !v_st_enbl[HYP_EXT]; - end - - // check if translation is a: S-Stage and G-Stage, S-Stage only or G-Stage only translation and virtualization mode is on/off - match_stage[i] = tags_q[i].v_st_enbl == v_st_enbl; - - if (tags_q[i].valid && &match_asid[i] && match_stage[i]) begin - - if (HYP_EXT == 1 && vpn_match[i][HYP_EXT*2]) - lu_gpaddr_o = make_gpaddr( - v_st_enbl[0], - tags_q[i].is_page[0][0], - tags_q[i].is_page[1][0], - lu_vaddr_i, - content_q[i][0] - ); - - if (|level_match[i]) begin - lu_is_page_o = is_page_o[i]; - lu_content_o = content_q[i]; - lu_hit_o = 1'b1; - lu_hit[i] = 1'b1; - - if (HYP_EXT == 1) begin - // Compute G-Stage PPN based on the gpaddr - g_content = content_q[i][HYP_EXT]; - if (tags_q[i].is_page[HYP_EXT][HYP_EXT]) g_content.ppn[8:0] = lu_gpaddr_o[20:12]; - if (tags_q[i].is_page[0][HYP_EXT]) g_content.ppn[17:0] = lu_gpaddr_o[29:12]; - // Output G-stage and S-stage content - lu_content_o[HYP_EXT] = level_match[i][PT_LEVELS-1] ? content_q[i][HYP_EXT] : g_content; - end - end - end - end - end - - - - logic [HYP_EXT:0]asid_to_be_flushed_is0; // indicates that the ASID provided by SFENCE.VMA (rs2)is 0, active high - logic [HYP_EXT:0] vaddr_to_be_flushed_is0; // indicates that the VADDR provided by SFENCE.VMA (rs1)is 0, active high - - localparam int VADDR_WIDTH[1:0] = {riscv::GPLEN, riscv::VLEN}; - - genvar a; - generate - for (a = 0; a < HYP_EXT + 1; a++) begin - assign asid_to_be_flushed_is0[a] = ~(|asid_to_be_flushed_i[a][ASID_WIDTH[a]-1:0]); - assign vaddr_to_be_flushed_is0[a] = ~(|vaddr_to_be_flushed_i[a][VADDR_WIDTH[a]-1:0]); - end - endgenerate - - // ------------------ - // Update and Flush - // ------------------ - always_comb begin : update_flush - tags_n = tags_q; - content_n = content_q; - - for (int unsigned i = 0; i < TLB_ENTRIES; i++) begin - - - if (HYP_EXT == 1) begin - gppn[i] = make_gppn( - tags_q[i].v_st_enbl[0], - tags_q[i].is_page[0][0], - tags_q[i].is_page[HYP_EXT][0], - { - tags_q[i].vpn[3*HYP_EXT][(VPN_LEN%PT_LEVELS)-1:0], - tags_q[i].vpn[2*HYP_EXT], - tags_q[i].vpn[HYP_EXT], - tags_q[i].vpn[0] - }, - content_q[i][0] - ); - end - - - if (flush_i[0]) begin - if (!tags_q[i].v_st_enbl[HYP_EXT*2] || HYP_EXT == 0) begin - // invalidate logic - // flush everything if ASID is 0 and vaddr is 0 ("SFENCE.VMA x0 x0" case) - if (asid_to_be_flushed_is0[0] && vaddr_to_be_flushed_is0[0]) tags_n[i].valid = 1'b0; - // flush vaddr in all addressing space ("SFENCE.VMA vaddr x0" case), it should happen only for leaf pages - else if (asid_to_be_flushed_is0[0] && (|vaddr_level_match[i][0] ) && (~vaddr_to_be_flushed_is0[0])) - tags_n[i].valid = 1'b0; - // the entry is flushed if it's not global and asid and vaddr both matches with the entry to be flushed ("SFENCE.VMA vaddr asid" case) - else if ((!content_q[i][0].g) && (|vaddr_level_match[i][0]) && (asid_to_be_flushed_i[0][ASID_WIDTH[0]-1:0] == tags_q[i].asid[0][ASID_WIDTH[0]-1:0] ) && (!vaddr_to_be_flushed_is0[0]) && (!asid_to_be_flushed_is0[0])) - tags_n[i].valid = 1'b0; - // the entry is flushed if it's not global, and the asid matches and vaddr is 0. ("SFENCE.VMA 0 asid" case) - else if ((!content_q[i][0].g) && (vaddr_to_be_flushed_is0[0]) && (asid_to_be_flushed_i[0][ASID_WIDTH[0]-1:0] == tags_q[i].asid[0][ASID_WIDTH[0]-1:0] ) && (!asid_to_be_flushed_is0[0])) - tags_n[i].valid = 1'b0; - end - end else if (flush_i[HYP_EXT] && HYP_EXT == 1) begin - if (tags_q[i].v_st_enbl[HYP_EXT*2] && tags_q[i].v_st_enbl[0]) begin - // invalidate logic - // flush everything if current VMID matches and ASID is 0 and vaddr is 0 ("SFENCE.VMA/HFENCE.VVMA x0 x0" case) - if (asid_to_be_flushed_is0[0] && vaddr_to_be_flushed_is0[0] && ((tags_q[i].v_st_enbl[HYP_EXT] && lu_asid_i[HYP_EXT][ASID_WIDTH[HYP_EXT]-1:0] == tags_q[i].asid[HYP_EXT][ASID_WIDTH[HYP_EXT]-1:0]) || !tags_q[i].v_st_enbl[HYP_EXT])) - tags_n[i].valid = 1'b0; - // flush vaddr in all addressing space if current VMID matches ("SFENCE.VMA/HFENCE.VVMA vaddr x0" case), it should happen only for leaf pages - else if (asid_to_be_flushed_is0[0] && (|vaddr_level_match[i][0]) && (~vaddr_to_be_flushed_is0[0]) && ((tags_q[i].v_st_enbl[HYP_EXT] && lu_asid_i[HYP_EXT][ASID_WIDTH[HYP_EXT]-1:0] == tags_q[i].asid[HYP_EXT][ASID_WIDTH[HYP_EXT]-1:0]) || !tags_q[i].v_st_enbl[HYP_EXT])) - tags_n[i].valid = 1'b0; - // the entry is flushed if it's not global and asid and vaddr and current VMID matches with the entry to be flushed ("SFENCE.VMA/HFENCE.VVMA vaddr asid" case) - else if ((!content_q[i][0].g) && (|vaddr_level_match[i][0]) && (asid_to_be_flushed_i[0][ASID_WIDTH[0]-1:0] == tags_q[i].asid[0][ASID_WIDTH[0]-1:0] && ((tags_q[i].v_st_enbl[HYP_EXT] && lu_asid_i[HYP_EXT][ASID_WIDTH[HYP_EXT]-1:0] == tags_q[i].asid[HYP_EXT][ASID_WIDTH[HYP_EXT]-1:0]) || !tags_q[i].v_st_enbl[HYP_EXT])) && (!vaddr_to_be_flushed_is0[0]) && (!asid_to_be_flushed_is0[0])) - tags_n[i].valid = 1'b0; - // the entry is flushed if it's not global, and the asid and the current VMID matches and vaddr is 0. ("SFENCE.VMA/HFENCE.VVMA 0 asid" case) - else if ((!content_q[i][0].g) && (vaddr_to_be_flushed_is0[0]) && (asid_to_be_flushed_i[0][ASID_WIDTH[0]-1:0] == tags_q[i].asid[0][ASID_WIDTH[0]-1:0] && ((tags_q[i].v_st_enbl[HYP_EXT] && lu_asid_i[HYP_EXT][ASID_WIDTH[HYP_EXT]-1:0] == tags_q[i].asid[HYP_EXT][ASID_WIDTH[HYP_EXT]-1:0]) || !tags_q[i].v_st_enbl[HYP_EXT])) && (!asid_to_be_flushed_is0[0])) - tags_n[i].valid = 1'b0; - end - end else if (flush_i[HYP_EXT*2] && HYP_EXT == 1) begin - if (tags_q[i].v_st_enbl[HYP_EXT]) begin - // invalidate logic - // flush everything if vmid is 0 and addr is 0 ("HFENCE.GVMA x0 x0" case) - if (asid_to_be_flushed_is0[HYP_EXT] && vaddr_to_be_flushed_is0[HYP_EXT]) - tags_n[i].valid = 1'b0; - // flush gpaddr in all addressing space ("HFENCE.GVMA gpaddr x0" case), it should happen only for leaf pages - else if (asid_to_be_flushed_is0[HYP_EXT] && (|vaddr_level_match[i][HYP_EXT] ) && (~vaddr_to_be_flushed_is0[HYP_EXT])) - tags_n[i].valid = 1'b0; - // the entry vmid and gpaddr both matches with the entry to be flushed ("HFENCE.GVMA gpaddr vmid" case) - else if ((|vaddr_level_match[i][HYP_EXT]) && (asid_to_be_flushed_i[HYP_EXT][ASID_WIDTH[HYP_EXT]-1:0] == tags_q[i].asid[HYP_EXT][ASID_WIDTH[HYP_EXT]-1:0]) && (~vaddr_to_be_flushed_is0[HYP_EXT]) && (~asid_to_be_flushed_is0[HYP_EXT])) - tags_n[i].valid = 1'b0; - // the entry is flushed if the vmid matches and gpaddr is 0. ("HFENCE.GVMA 0 vmid" case) - else if ((vaddr_to_be_flushed_is0[HYP_EXT]) && (asid_to_be_flushed_i[HYP_EXT][ASID_WIDTH[HYP_EXT]-1:0] == tags_q[i].asid[HYP_EXT][ASID_WIDTH[HYP_EXT]-1:0]) && (!asid_to_be_flushed_is0[HYP_EXT])) - tags_n[i].valid = 1'b0; - end - // normal replacement - end else if (update_i.valid & replace_en[i] && !lu_hit_o) begin - //update tag - tags_n[i] = { - update_i.asid, - ((PT_LEVELS + HYP_EXT) * (VPN_LEN / PT_LEVELS))'(update_i.vpn), - update_i.is_page, - update_i.v_st_enbl, - 1'b1 - }; - // update content as well - content_n[i] = update_i.content; - end - end - end - - // ----------------------------------------------- - // PLRU - Pseudo Least Recently Used Replacement - // ----------------------------------------------- - logic [2*(TLB_ENTRIES-1)-1:0] plru_tree_q, plru_tree_n; - always_comb begin : plru_replacement - plru_tree_n = plru_tree_q; - // The PLRU-tree indexing: - // lvl0 0 - // / \ -// / \ -// lvl1 1 2 -// / \ / \ -// lvl2 3 4 5 6 -// / \ /\/\ /\ -// ... ... ... ... -// Just predefine which nodes will be set/cleared -// E.g. for a TLB with 8 entries, the for-loop is semantically -// equivalent to the following pseudo-code: -// unique case (1'b1) -// lu_hit[7]: plru_tree_n[0, 2, 6] = {1, 1, 1}; -// lu_hit[6]: plru_tree_n[0, 2, 6] = {1, 1, 0}; -// lu_hit[5]: plru_tree_n[0, 2, 5] = {1, 0, 1}; -// lu_hit[4]: plru_tree_n[0, 2, 5] = {1, 0, 0}; -// lu_hit[3]: plru_tree_n[0, 1, 4] = {0, 1, 1}; -// lu_hit[2]: plru_tree_n[0, 1, 4] = {0, 1, 0}; -// lu_hit[1]: plru_tree_n[0, 1, 3] = {0, 0, 1}; -// lu_hit[0]: plru_tree_n[0, 1, 3] = {0, 0, 0}; -// default: begin /* No hit */ end -// endcase -for ( - int unsigned i = 0; i < TLB_ENTRIES; i++ - ) begin - automatic int unsigned idx_base, shift, new_index; - // we got a hit so update the pointer as it was least recently used - if (lu_hit[i] & lu_access_i) begin - // Set the nodes to the values we would expect - for (int unsigned lvl = 0; lvl < $clog2(TLB_ENTRIES); lvl++) begin - idx_base = $unsigned((2 ** lvl) - 1); - // lvl0 <=> MSB, lvl1 <=> MSB-1, ... - shift = $clog2(TLB_ENTRIES) - lvl; - // to circumvent the 32 bit integer arithmetic assignment - new_index = ~((i >> (shift - 1)) & 32'b1); - plru_tree_n[idx_base+(i>>shift)] = new_index[0]; - end - end - end - // Decode tree to write enable signals - // Next for-loop basically creates the following logic for e.g. an 8 entry - // TLB (note: pseudo-code obviously): - // replace_en[7] = &plru_tree_q[ 6, 2, 0]; //plru_tree_q[0,2,6]=={1,1,1} - // replace_en[6] = &plru_tree_q[~6, 2, 0]; //plru_tree_q[0,2,6]=={1,1,0} - // replace_en[5] = &plru_tree_q[ 5,~2, 0]; //plru_tree_q[0,2,5]=={1,0,1} - // replace_en[4] = &plru_tree_q[~5,~2, 0]; //plru_tree_q[0,2,5]=={1,0,0} - // replace_en[3] = &plru_tree_q[ 4, 1,~0]; //plru_tree_q[0,1,4]=={0,1,1} - // replace_en[2] = &plru_tree_q[~4, 1,~0]; //plru_tree_q[0,1,4]=={0,1,0} - // replace_en[1] = &plru_tree_q[ 3,~1,~0]; //plru_tree_q[0,1,3]=={0,0,1} - // replace_en[0] = &plru_tree_q[~3,~1,~0]; //plru_tree_q[0,1,3]=={0,0,0} - // For each entry traverse the tree. If every tree-node matches, - // the corresponding bit of the entry's index, this is - // the next entry to replace. - for (int unsigned i = 0; i < TLB_ENTRIES; i += 1) begin - automatic logic en; - automatic int unsigned idx_base, shift, new_index; - en = 1'b1; - for (int unsigned lvl = 0; lvl < $clog2(TLB_ENTRIES); lvl++) begin - idx_base = $unsigned((2 ** lvl) - 1); - // lvl0 <=> MSB, lvl1 <=> MSB-1, ... - shift = $clog2(TLB_ENTRIES) - lvl; - - // en &= plru_tree_q[idx_base + (i>>shift)] == ((i >> (shift-1)) & 1'b1); - new_index = (i >> (shift - 1)) & 32'b1; - if (new_index[0]) begin - en &= plru_tree_q[idx_base+(i>>shift)]; - end else begin - en &= ~plru_tree_q[idx_base+(i>>shift)]; - end - end - replace_en[i] = en; - end - end - - // sequential process - always_ff @(posedge clk_i or negedge rst_ni) begin - if (~rst_ni) begin - tags_q <= '{default: 0}; - content_q <= '{default: 0}; - plru_tree_q <= '{default: 0}; - end else begin - tags_q <= tags_n; - content_q <= content_n; - plru_tree_q <= plru_tree_n; - end - end - //-------------- - // Sanity checks - //-------------- - - //pragma translate_off -`ifndef VERILATOR - - initial begin : p_assertions - assert ((TLB_ENTRIES % 2 == 0) && (TLB_ENTRIES > 1)) - else begin - $error("TLB size must be a multiple of 2 and greater than 1"); - $stop(); - end - assert (ASID_WIDTH[0] >= 1) - else begin - $error("ASID width must be at least 1"); - $stop(); - end - end - - // Just for checking - function int countSetBits(logic [TLB_ENTRIES-1:0] vector); - automatic int count = 0; - foreach (vector[idx]) begin - count += vector[idx]; - end - return count; - endfunction - - assert property (@(posedge clk_i) (countSetBits(lu_hit) <= 1)) - else begin - $error("More then one hit in TLB!"); - $stop(); - end - assert property (@(posedge clk_i) (countSetBits(replace_en) <= 1)) - else begin - $error("More then one TLB entry selected for next replace!"); - $stop(); - end - -`endif - //pragma translate_on - -endmodule diff --git a/core/include/cv64a6_imafdc_sv39_config_pkg.sv b/core/include/cv64a6_imafdc_sv39_config_pkg.sv index 2f648619b..29c549911 100644 --- a/core/include/cv64a6_imafdc_sv39_config_pkg.sv +++ b/core/include/cv64a6_imafdc_sv39_config_pkg.sv @@ -55,8 +55,8 @@ package cva6_config_pkg; localparam CVA6ConfigNrStorePipeRegs = 0; localparam CVA6ConfigNrLoadBufEntries = 2; - localparam CVA6ConfigInstrTlbEntries = 4; - localparam CVA6ConfigDataTlbEntries = 4; + localparam CVA6ConfigInstrTlbEntries = 16; + localparam CVA6ConfigDataTlbEntries = 16; localparam CVA6ConfigRASDepth = 2; localparam CVA6ConfigBTBEntries = 32; diff --git a/core/include/riscv_pkg.sv b/core/include/riscv_pkg.sv index fb4fcf25b..18ae2cfc2 100644 --- a/core/include/riscv_pkg.sv +++ b/core/include/riscv_pkg.sv @@ -40,10 +40,6 @@ package riscv; localparam VLEN = (XLEN == 32) ? 32 : 64; // virtual address length localparam PLEN = (XLEN == 32) ? 34 : 56; // physical address length - localparam GPLEN = (XLEN == 32) ? 34 : 41; - localparam GPPNW = (XLEN == 32) ? 22 : 29; - localparam GPPN2 = (XLEN == 32) ? riscv::VLEN - 33 : 10; - localparam IS_XLEN32 = (XLEN == 32) ? 1'b1 : 1'b0; localparam IS_XLEN64 = (XLEN == 32) ? 1'b0 : 1'b1; localparam ModeW = (XLEN == 32) ? 1 : 4; @@ -338,15 +334,10 @@ package riscv; localparam logic [XLEN-1:0] ST_ACCESS_FAULT = 7; // Illegal access as governed by PMPs and PMAs localparam logic [XLEN-1:0] ENV_CALL_UMODE = 8; // environment call from user mode localparam logic [XLEN-1:0] ENV_CALL_SMODE = 9; // environment call from supervisor mode - localparam logic [XLEN-1:0] ENV_CALL_VSMODE = 10; // environment call from virtual supervisor mode localparam logic [XLEN-1:0] ENV_CALL_MMODE = 11; // environment call from machine mode localparam logic [XLEN-1:0] INSTR_PAGE_FAULT = 12; // Instruction page fault localparam logic [XLEN-1:0] LOAD_PAGE_FAULT = 13; // Load page fault localparam logic [XLEN-1:0] STORE_PAGE_FAULT = 15; // Store page fault - localparam logic [XLEN-1:0] INSTR_GUEST_PAGE_FAULT = 20; // Instruction guest-page fault - localparam logic [XLEN-1:0] LOAD_GUEST_PAGE_FAULT = 21; // Load guest-page fault - localparam logic [XLEN-1:0] VIRTUAL_INSTRUCTION = 22; // virtual instruction - localparam logic [XLEN-1:0] STORE_GUEST_PAGE_FAULT = 23; // Store guest-page fault localparam logic [XLEN-1:0] DEBUG_REQUEST = 24; // Debug request localparam int unsigned IRQ_S_SOFT = 1; @@ -370,14 +361,6 @@ package riscv; localparam logic [XLEN-1:0] S_EXT_INTERRUPT = (1 << (XLEN - 1)) | XLEN'(IRQ_S_EXT); localparam logic [XLEN-1:0] M_EXT_INTERRUPT = (1 << (XLEN - 1)) | XLEN'(IRQ_M_EXT); - // ---------------------- - // PseudoInstructions Codes - // ---------------------- - localparam logic [XLEN-1:0] READ_32_PSEUDOINSTRUCTION = 32'h00002000; - localparam logic [XLEN-1:0] WRITE_32_PSEUDOINSTRUCTION = 32'h00002020; - localparam logic [XLEN-1:0] READ_64_PSEUDOINSTRUCTION = 64'h00003000; - localparam logic [XLEN-1:0] WRITE_64_PSEUDOINSTRUCTION = 64'h00003020; - // ----- // CSRs // ----- diff --git a/core/load_store_unit.sv b/core/load_store_unit.sv index 66a74c743..cb899c6c0 100644 --- a/core/load_store_unit.sv +++ b/core/load_store_unit.sv @@ -175,65 +175,63 @@ module load_store_unit // ------------------- // MMU e.g.: TLBs/PTW // ------------------- - if (MMU_PRESENT) begin : gen_mmu - - localparam HYP_EXT = 0; //CVA6Cfg.CVA6ConfigHExtEn - localparam VPN_LEN = (riscv::XLEN == 64) ? (HYP_EXT ? 29 : 27) : 20; - localparam PT_LEVELS = (riscv::XLEN == 64) ? 3 : 2; - localparam int mmu_ASID_WIDTH[HYP_EXT:0] = {ASID_WIDTH}; - - - cva6_mmu #( + if (MMU_PRESENT && (riscv::XLEN == 64)) begin : gen_mmu_sv39 + mmu #( .CVA6Cfg (CVA6Cfg), .INSTR_TLB_ENTRIES(ariane_pkg::INSTR_TLB_ENTRIES), .DATA_TLB_ENTRIES (ariane_pkg::DATA_TLB_ENTRIES), - .HYP_EXT (HYP_EXT), - .ASID_WIDTH (mmu_ASID_WIDTH), - .VPN_LEN (VPN_LEN), - .PT_LEVELS (PT_LEVELS) + .ASID_WIDTH (ASID_WIDTH) ) i_cva6_mmu ( - .clk_i (clk_i), - .rst_ni (rst_ni), - .flush_i (flush_i), - .enable_translation_i ({enable_translation_i}), - .en_ld_st_translation_i({en_ld_st_translation_i}), - .icache_areq_i (icache_areq_i), - .icache_areq_o (icache_areq_o), // misaligned bypass - .misaligned_ex_i (misaligned_exception), - .lsu_req_i (translation_req), - .lsu_vaddr_i (mmu_vaddr), - .lsu_tinst_i (0), - .lsu_is_store_i (st_translation_req), - .csr_hs_ld_st_inst_o (), - .lsu_dtlb_hit_o (dtlb_hit), // send in the same cycle as the request - .lsu_dtlb_ppn_o (dtlb_ppn), // send in the same cycle as the request - + .misaligned_ex_i(misaligned_exception), + .lsu_is_store_i (st_translation_req), + .lsu_req_i (translation_req), + .lsu_vaddr_i (mmu_vaddr), .lsu_valid_o (translation_valid), .lsu_paddr_o (mmu_paddr), .lsu_exception_o(mmu_exception), - - .priv_lvl_i (priv_lvl_i), - .ld_st_priv_lvl_i(ld_st_priv_lvl_i), - - .sum_i ({sum_i}), - .mxr_i ({mxr_i}), - .hlvx_inst_i (0), - .hs_ld_st_inst_i(0), - - .satp_ppn_i ({satp_ppn_i}), - .asid_i ({asid_i}), - .asid_to_be_flushed_i ({asid_to_be_flushed_i}), - .vaddr_to_be_flushed_i({vaddr_to_be_flushed_i}), - .flush_tlb_i ({flush_tlb_i}), - - .itlb_miss_o(itlb_miss_o), - .dtlb_miss_o(dtlb_miss_o), - - .req_port_i(dcache_req_ports_i[0]), - .req_port_o(dcache_req_ports_o[0]), + .lsu_dtlb_hit_o (dtlb_hit), // send in the same cycle as the request + .lsu_dtlb_ppn_o (dtlb_ppn), // send in the same cycle as the request + // connecting PTW to D$ IF + .req_port_i (dcache_req_ports_i[0]), + .req_port_o (dcache_req_ports_o[0]), + // icache address translation requests + .icache_areq_i (icache_areq_i), + .asid_to_be_flushed_i, + .vaddr_to_be_flushed_i, + .icache_areq_o (icache_areq_o), .pmpcfg_i, - .pmpaddr_i + .pmpaddr_i, + .* + ); + end else if (MMU_PRESENT && (riscv::XLEN == 32)) begin : gen_mmu_sv32 + cva6_mmu_sv32 #( + .CVA6Cfg (CVA6Cfg), + .INSTR_TLB_ENTRIES(ariane_pkg::INSTR_TLB_ENTRIES), + .DATA_TLB_ENTRIES (ariane_pkg::DATA_TLB_ENTRIES), + .ASID_WIDTH (ASID_WIDTH) + ) i_cva6_mmu ( + // misaligned bypass + .misaligned_ex_i(misaligned_exception), + .lsu_is_store_i (st_translation_req), + .lsu_req_i (translation_req), + .lsu_vaddr_i (mmu_vaddr), + .lsu_valid_o (translation_valid), + .lsu_paddr_o (mmu_paddr), + .lsu_exception_o(mmu_exception), + .lsu_dtlb_hit_o (dtlb_hit), // send in the same cycle as the request + .lsu_dtlb_ppn_o (dtlb_ppn), // send in the same cycle as the request + // connecting PTW to D$ IF + .req_port_i (dcache_req_ports_i[0]), + .req_port_o (dcache_req_ports_o[0]), + // icache address translation requests + .icache_areq_i (icache_areq_i), + .asid_to_be_flushed_i, + .vaddr_to_be_flushed_i, + .icache_areq_o (icache_areq_o), + .pmpcfg_i, + .pmpaddr_i, + .* ); end else begin : gen_no_mmu diff --git a/core/mmu_sv32/cva6_mmu_sv32.sv b/core/mmu_sv32/cva6_mmu_sv32.sv new file mode 100644 index 000000000..9c98793da --- /dev/null +++ b/core/mmu_sv32/cva6_mmu_sv32.sv @@ -0,0 +1,586 @@ +// Copyright (c) 2021 Thales. +// Copyright and related rights are licensed under the Solderpad Hardware +// License, Version 0.51 (the "License"); you may not use this file except in +// compliance with the License. You may obtain a copy of the License at +// http://solderpad.org/licenses/SHL-0.51. Unless required by applicable law +// or agreed to in writing, software, hardware and materials distributed under +// this License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. +// +// Author: Sebastien Jacq Thales Research & Technology +// Date: 17/07/2021 +// +// Additional contributions by: +// Sebastien Jacq - sjthales on github.com +// +// Description: Memory Management Unit for CV32A6, contains TLB and +// address translation unit. Sv32 as defined in RISC-V +// privilege specification 1.11-WIP. +// This module is an adaptation of the MMU Sv39 developed +// by Florian Zaruba to the Sv32 standard. +// +// =========================================================================== // +// Revisions : +// Date Version Author Description +// 2020-02-17 0.1 S.Jacq MMU Sv32 for CV32A6 +// =========================================================================== // + +module cva6_mmu_sv32 + import ariane_pkg::*; +#( + parameter config_pkg::cva6_cfg_t CVA6Cfg = config_pkg::cva6_cfg_empty, + parameter int unsigned INSTR_TLB_ENTRIES = 2, + parameter int unsigned DATA_TLB_ENTRIES = 2, + parameter int unsigned ASID_WIDTH = 1 +) ( + input logic clk_i, + input logic rst_ni, + input logic flush_i, + input logic enable_translation_i, + input logic en_ld_st_translation_i, // enable virtual memory translation for load/stores + // IF interface + input icache_arsp_t icache_areq_i, + output icache_areq_t icache_areq_o, + // LSU interface + // this is a more minimalistic interface because the actual addressing logic is handled + // in the LSU as we distinguish load and stores, what we do here is simple address translation + input exception_t misaligned_ex_i, + input logic lsu_req_i, // request address translation + input logic [riscv::VLEN-1:0] lsu_vaddr_i, // virtual address in + input logic lsu_is_store_i, // the translation is requested by a store + // if we need to walk the page table we can't grant in the same cycle + // Cycle 0 + output logic lsu_dtlb_hit_o, // sent in the same cycle as the request if translation hits in the DTLB + output logic [riscv::PPNW-1:0] lsu_dtlb_ppn_o, // ppn (send same cycle as hit) + // Cycle 1 + output logic lsu_valid_o, // translation is valid + output logic [riscv::PLEN-1:0] lsu_paddr_o, // translated address + output exception_t lsu_exception_o, // address translation threw an exception + // General control signals + input riscv::priv_lvl_t priv_lvl_i, + input riscv::priv_lvl_t ld_st_priv_lvl_i, + input logic sum_i, + input logic mxr_i, + // input logic flag_mprv_i, + input logic [riscv::PPNW-1:0] satp_ppn_i, + input logic [ASID_WIDTH-1:0] asid_i, + input logic [ASID_WIDTH-1:0] asid_to_be_flushed_i, + input logic [riscv::VLEN-1:0] vaddr_to_be_flushed_i, + input logic flush_tlb_i, + // Performance counters + output logic itlb_miss_o, + output logic dtlb_miss_o, + // PTW memory interface + input dcache_req_o_t req_port_i, + output dcache_req_i_t req_port_o, + // PMP + input riscv::pmpcfg_t [15:0] pmpcfg_i, + input logic [15:0][riscv::PLEN-3:0] pmpaddr_i +); + + logic iaccess_err; // insufficient privilege to access this instruction page + logic daccess_err; // insufficient privilege to access this data page + logic ptw_active; // PTW is currently walking a page table + logic walking_instr; // PTW is walking because of an ITLB miss + logic ptw_error; // PTW threw an exception + logic ptw_access_exception; // PTW threw an access exception (PMPs) + logic [riscv::PLEN-1:0] ptw_bad_paddr; // PTW PMP exception bad physical addr + + logic [riscv::VLEN-1:0] update_vaddr; + tlb_update_sv32_t update_itlb, update_dtlb, update_shared_tlb; + + logic itlb_lu_access; + riscv::pte_sv32_t itlb_content; + logic itlb_is_4M; + logic itlb_lu_hit; + + logic dtlb_lu_access; + riscv::pte_sv32_t dtlb_content; + logic dtlb_is_4M; + logic dtlb_lu_hit; + + logic shared_tlb_access; + logic [riscv::VLEN-1:0] shared_tlb_vaddr; + logic shared_tlb_hit; + + logic itlb_req; + + + // Assignments + assign itlb_lu_access = icache_areq_i.fetch_req; + assign dtlb_lu_access = lsu_req_i; + + + cva6_tlb_sv32 #( + .CVA6Cfg (CVA6Cfg), + .TLB_ENTRIES(INSTR_TLB_ENTRIES), + .ASID_WIDTH (ASID_WIDTH) + ) i_itlb ( + .clk_i (clk_i), + .rst_ni (rst_ni), + .flush_i(flush_tlb_i), + + .update_i(update_itlb), + + .lu_access_i (itlb_lu_access), + .lu_asid_i (asid_i), + .asid_to_be_flushed_i (asid_to_be_flushed_i), + .vaddr_to_be_flushed_i(vaddr_to_be_flushed_i), + .lu_vaddr_i (icache_areq_i.fetch_vaddr), + .lu_content_o (itlb_content), + + .lu_is_4M_o(itlb_is_4M), + .lu_hit_o (itlb_lu_hit) + ); + + cva6_tlb_sv32 #( + .CVA6Cfg (CVA6Cfg), + .TLB_ENTRIES(DATA_TLB_ENTRIES), + .ASID_WIDTH (ASID_WIDTH) + ) i_dtlb ( + .clk_i (clk_i), + .rst_ni (rst_ni), + .flush_i(flush_tlb_i), + + .update_i(update_dtlb), + + .lu_access_i (dtlb_lu_access), + .lu_asid_i (asid_i), + .asid_to_be_flushed_i (asid_to_be_flushed_i), + .vaddr_to_be_flushed_i(vaddr_to_be_flushed_i), + .lu_vaddr_i (lsu_vaddr_i), + .lu_content_o (dtlb_content), + + .lu_is_4M_o(dtlb_is_4M), + .lu_hit_o (dtlb_lu_hit) + ); + + cva6_shared_tlb_sv32 #( + .CVA6Cfg (CVA6Cfg), + .SHARED_TLB_DEPTH(64), + .SHARED_TLB_WAYS (2), + .ASID_WIDTH (ASID_WIDTH) + ) i_shared_tlb ( + .clk_i (clk_i), + .rst_ni (rst_ni), + .flush_i(flush_tlb_i), + + .enable_translation_i (enable_translation_i), + .en_ld_st_translation_i(en_ld_st_translation_i), + + .asid_i (asid_i), + // from TLBs + // did we miss? + .itlb_access_i(itlb_lu_access), + .itlb_hit_i (itlb_lu_hit), + .itlb_vaddr_i (icache_areq_i.fetch_vaddr), + + .dtlb_access_i(dtlb_lu_access), + .dtlb_hit_i (dtlb_lu_hit), + .dtlb_vaddr_i (lsu_vaddr_i), + + // to TLBs, update logic + .itlb_update_o(update_itlb), + .dtlb_update_o(update_dtlb), + + // Performance counters + .itlb_miss_o(itlb_miss_o), + .dtlb_miss_o(dtlb_miss_o), + + .shared_tlb_access_o(shared_tlb_access), + .shared_tlb_hit_o (shared_tlb_hit), + .shared_tlb_vaddr_o (shared_tlb_vaddr), + + .itlb_req_o (itlb_req), + // to update shared tlb + .shared_tlb_update_i(update_shared_tlb) + ); + + cva6_ptw_sv32 #( + .CVA6Cfg (CVA6Cfg), + .ASID_WIDTH(ASID_WIDTH) + ) i_ptw ( + .clk_i (clk_i), + .rst_ni (rst_ni), + .flush_i(flush_i), + + .ptw_active_o (ptw_active), + .walking_instr_o (walking_instr), + .ptw_error_o (ptw_error), + .ptw_access_exception_o(ptw_access_exception), + + .lsu_is_store_i(lsu_is_store_i), + // PTW memory interface + .req_port_i (req_port_i), + .req_port_o (req_port_o), + + // to Shared TLB, update logic + .shared_tlb_update_o(update_shared_tlb), + + .update_vaddr_o(update_vaddr), + + .asid_i(asid_i), + + // from shared TLB + // did we miss? + .shared_tlb_access_i(shared_tlb_access), + .shared_tlb_hit_i (shared_tlb_hit), + .shared_tlb_vaddr_i (shared_tlb_vaddr), + + .itlb_req_i(itlb_req), + + // from CSR file + .satp_ppn_i(satp_ppn_i), // ppn from satp + .mxr_i (mxr_i), + + // Performance counters + .shared_tlb_miss_o(), //open for now + + // PMP + .pmpcfg_i (pmpcfg_i), + .pmpaddr_i (pmpaddr_i), + .bad_paddr_o(ptw_bad_paddr) + + ); + + // ila_1 i_ila_1 ( + // .clk(clk_i), // input wire clk + // .probe0({req_port_o.address_tag, req_port_o.address_index}), + // .probe1(req_port_o.data_req), // input wire [63:0] probe1 + // .probe2(req_port_i.data_gnt), // input wire [0:0] probe2 + // .probe3(req_port_i.data_rdata), // input wire [0:0] probe3 + // .probe4(req_port_i.data_rvalid), // input wire [0:0] probe4 + // .probe5(ptw_error), // input wire [1:0] probe5 + // .probe6(update_vaddr), // input wire [0:0] probe6 + // .probe7(update_itlb.valid), // input wire [0:0] probe7 + // .probe8(update_dtlb.valid), // input wire [0:0] probe8 + // .probe9(dtlb_lu_access), // input wire [0:0] probe9 + // .probe10(lsu_vaddr_i), // input wire [0:0] probe10 + // .probe11(dtlb_lu_hit), // input wire [0:0] probe11 + // .probe12(itlb_lu_access), // input wire [0:0] probe12 + // .probe13(icache_areq_i.fetch_vaddr), // input wire [0:0] probe13 + // .probe14(itlb_lu_hit) // input wire [0:0] probe13 + // ); + + //----------------------- + // Instruction Interface + //----------------------- + logic match_any_execute_region; + logic pmp_instr_allow; + + // The instruction interface is a simple request response interface + always_comb begin : instr_interface + // MMU disabled: just pass through + icache_areq_o.fetch_valid = icache_areq_i.fetch_req; + if (riscv::PLEN > riscv::VLEN) + icache_areq_o.fetch_paddr = { + {riscv::PLEN - riscv::VLEN{1'b0}}, icache_areq_i.fetch_vaddr + }; // play through in case we disabled address translation + else + icache_areq_o.fetch_paddr = { + 2'b00, icache_areq_i.fetch_vaddr[riscv::VLEN-1:0] + }; // play through in case we disabled address translation + // two potential exception sources: + // 1. HPTW threw an exception -> signal with a page fault exception + // 2. We got an access error because of insufficient permissions -> throw an access exception + icache_areq_o.fetch_exception = '0; + // Check whether we are allowed to access this memory region from a fetch perspective + iaccess_err = icache_areq_i.fetch_req && (((priv_lvl_i == riscv::PRIV_LVL_U) && ~itlb_content.u) + || ((priv_lvl_i == riscv::PRIV_LVL_S) && itlb_content.u)); + + // MMU enabled: address from TLB, request delayed until hit. Error when TLB + // hit and no access right or TLB hit and translated address not valid (e.g. + // AXI decode error), or when PTW performs walk due to ITLB miss and raises + // an error. + if (enable_translation_i) begin + // we work with SV32, so if VM is enabled, check that all bits [riscv::VLEN-1:riscv::SV-1] are equal + if (icache_areq_i.fetch_req && !((&icache_areq_i.fetch_vaddr[riscv::VLEN-1:riscv::SV-1]) == 1'b1 || (|icache_areq_i.fetch_vaddr[riscv::VLEN-1:riscv::SV-1]) == 1'b0)) begin + icache_areq_o.fetch_exception.cause = riscv::INSTR_ACCESS_FAULT; + icache_areq_o.fetch_exception.valid = 1'b1; + if (CVA6Cfg.TvalEn) + icache_areq_o.fetch_exception.tval = { + {riscv::XLEN - riscv::VLEN{1'b0}}, icache_areq_i.fetch_vaddr + }; + end + + icache_areq_o.fetch_valid = 1'b0; + + // 4K page + icache_areq_o.fetch_paddr = {itlb_content.ppn, icache_areq_i.fetch_vaddr[11:0]}; + // Mega page + if (itlb_is_4M) begin + icache_areq_o.fetch_paddr[21:12] = icache_areq_i.fetch_vaddr[21:12]; + end + + + // --------- + // ITLB Hit + // -------- + // if we hit the ITLB output the request signal immediately + if (itlb_lu_hit) begin + icache_areq_o.fetch_valid = icache_areq_i.fetch_req; + // we got an access error + if (iaccess_err) begin + // throw a page fault + icache_areq_o.fetch_exception.cause = riscv::INSTR_PAGE_FAULT; + icache_areq_o.fetch_exception.valid = 1'b1; + if (CVA6Cfg.TvalEn) + icache_areq_o.fetch_exception.tval = { + {riscv::XLEN - riscv::VLEN{1'b0}}, icache_areq_i.fetch_vaddr + }; + //to check on wave --> not connected + end else if (!pmp_instr_allow) begin + icache_areq_o.fetch_exception.cause = riscv::INSTR_ACCESS_FAULT; + icache_areq_o.fetch_exception.valid = 1'b1; + if (CVA6Cfg.TvalEn) icache_areq_o.fetch_exception.tval = icache_areq_i.fetch_vaddr; + //to check on wave --> not connected + end + end else + // --------- + // ITLB Miss + // --------- + // watch out for exceptions happening during walking the page table + if (ptw_active && walking_instr) begin + icache_areq_o.fetch_valid = ptw_error | ptw_access_exception; + if (ptw_error) begin + icache_areq_o.fetch_exception.cause = riscv::INSTR_PAGE_FAULT; + icache_areq_o.fetch_exception.valid = 1'b1; + if (CVA6Cfg.TvalEn) + icache_areq_o.fetch_exception.tval = {{riscv::XLEN - riscv::VLEN{1'b0}}, update_vaddr}; + end //to check on wave + // TODO(moschn,zarubaf): What should the value of tval be in this case? + else begin + icache_areq_o.fetch_exception.cause = riscv::INSTR_ACCESS_FAULT; + icache_areq_o.fetch_exception.valid = 1'b1; + if (CVA6Cfg.TvalEn) icache_areq_o.fetch_exception.tval = ptw_bad_paddr[riscv::PLEN-1:2]; + end + end + end + // if it didn't match any execute region throw an `Instruction Access Fault` + // or: if we are not translating, check PMPs immediately on the paddr + if (!match_any_execute_region || (!enable_translation_i && !pmp_instr_allow)) begin + icache_areq_o.fetch_exception.cause = riscv::INSTR_ACCESS_FAULT; + icache_areq_o.fetch_exception.valid = 1'b1; + if (CVA6Cfg.TvalEn) + icache_areq_o.fetch_exception.tval = icache_areq_o.fetch_paddr[riscv::PLEN-1:2]; + //to check on wave --> not connected + end + end + + // check for execute flag on memory + assign match_any_execute_region = config_pkg::is_inside_execute_regions( + CVA6Cfg, {{64 - riscv::PLEN{1'b0}}, icache_areq_o.fetch_paddr} + ); + + // Instruction fetch + pmp #( + .PLEN (riscv::PLEN), + .PMP_LEN (riscv::PLEN - 2), + .NR_ENTRIES(CVA6Cfg.NrPMPEntries) + ) i_pmp_if ( + .addr_i (icache_areq_o.fetch_paddr), + .priv_lvl_i, + // we will always execute on the instruction fetch port + .access_type_i(riscv::ACCESS_EXEC), + // Configuration + .conf_addr_i (pmpaddr_i), + .conf_i (pmpcfg_i), + .allow_o (pmp_instr_allow) + ); + + //----------------------- + // Data Interface + //----------------------- + logic [riscv::VLEN-1:0] lsu_vaddr_n, lsu_vaddr_q; + riscv::pte_sv32_t dtlb_pte_n, dtlb_pte_q; + exception_t misaligned_ex_n, misaligned_ex_q; + logic lsu_req_n, lsu_req_q; + logic lsu_is_store_n, lsu_is_store_q; + logic dtlb_hit_n, dtlb_hit_q; + logic dtlb_is_4M_n, dtlb_is_4M_q; + + // check if we need to do translation or if we are always ready (e.g.: we are not translating anything) + assign lsu_dtlb_hit_o = (en_ld_st_translation_i) ? dtlb_lu_hit : 1'b1; + + // Wires to PMP checks + riscv::pmp_access_t pmp_access_type; + logic pmp_data_allow; + localparam PPNWMin = (riscv::PPNW - 1 > 29) ? 29 : riscv::PPNW - 1; + // The data interface is simpler and only consists of a request/response interface + always_comb begin : data_interface + // save request and DTLB response + lsu_vaddr_n = lsu_vaddr_i; + lsu_req_n = lsu_req_i; + misaligned_ex_n = misaligned_ex_i; + dtlb_pte_n = dtlb_content; + dtlb_hit_n = dtlb_lu_hit; + lsu_is_store_n = lsu_is_store_i; + dtlb_is_4M_n = dtlb_is_4M; + + if (riscv::PLEN > riscv::VLEN) begin + lsu_paddr_o = {{riscv::PLEN - riscv::VLEN{1'b0}}, lsu_vaddr_q}; + lsu_dtlb_ppn_o = {{riscv::PLEN - riscv::VLEN{1'b0}}, lsu_vaddr_n[riscv::VLEN-1:12]}; + end else begin + lsu_paddr_o = {2'b00, lsu_vaddr_q[riscv::VLEN-1:0]}; + lsu_dtlb_ppn_o = lsu_vaddr_n[riscv::PPNW-1:0]; + end + lsu_valid_o = lsu_req_q; + lsu_exception_o = misaligned_ex_q; + pmp_access_type = lsu_is_store_q ? riscv::ACCESS_WRITE : riscv::ACCESS_READ; + + // mute misaligned exceptions if there is no request otherwise they will throw accidental exceptions + misaligned_ex_n.valid = misaligned_ex_i.valid & lsu_req_i; + + // Check if the User flag is set, then we may only access it in supervisor mode + // if SUM is enabled + daccess_err = (ld_st_priv_lvl_i == riscv::PRIV_LVL_S && !sum_i && dtlb_pte_q.u) || // SUM is not set and we are trying to access a user page in supervisor mode + (ld_st_priv_lvl_i == riscv::PRIV_LVL_U && !dtlb_pte_q.u); // this is not a user page but we are in user mode and trying to access it + // translation is enabled and no misaligned exception occurred + if (en_ld_st_translation_i && !misaligned_ex_q.valid) begin + lsu_valid_o = 1'b0; + // 4K page + lsu_paddr_o = {dtlb_pte_q.ppn, lsu_vaddr_q[11:0]}; + lsu_dtlb_ppn_o = dtlb_content.ppn; + // Mega page + if (dtlb_is_4M_q) begin + lsu_paddr_o[21:12] = lsu_vaddr_q[21:12]; + lsu_dtlb_ppn_o[21:12] = lsu_vaddr_n[21:12]; + end + // --------- + // DTLB Hit + // -------- + if (dtlb_hit_q && lsu_req_q) begin + lsu_valid_o = 1'b1; + // exception priority: + // PAGE_FAULTS have higher priority than ACCESS_FAULTS + // virtual memory based exceptions are PAGE_FAULTS + // physical memory based exceptions are ACCESS_FAULTS (PMA/PMP) + + // this is a store + if (lsu_is_store_q) begin + // check if the page is write-able and we are not violating privileges + // also check if the dirty flag is set + if (!dtlb_pte_q.w || daccess_err || !dtlb_pte_q.d) begin + lsu_exception_o.cause = riscv::STORE_PAGE_FAULT; + lsu_exception_o.valid = 1'b1; + if (CVA6Cfg.TvalEn) + lsu_exception_o.tval = { + {riscv::XLEN - riscv::VLEN{lsu_vaddr_q[riscv::VLEN-1]}}, lsu_vaddr_q + }; + // to check on wave + // Check if any PMPs are violated + end else if (!pmp_data_allow) begin + lsu_exception_o.cause = riscv::ST_ACCESS_FAULT; + lsu_exception_o.valid = 1'b1; + if (CVA6Cfg.TvalEn) lsu_exception_o.tval = lsu_paddr_o[riscv::PLEN-1:2]; + //only 32 bits on 34b of lsu_paddr_o are returned. + end + + // this is a load + end else begin + // check for sufficient access privileges - throw a page fault if necessary + if (daccess_err) begin + lsu_exception_o.cause = riscv::LOAD_PAGE_FAULT; + lsu_exception_o.valid = 1'b1; + if (CVA6Cfg.TvalEn) + lsu_exception_o.tval = { + {riscv::XLEN - riscv::VLEN{lsu_vaddr_q[riscv::VLEN-1]}}, lsu_vaddr_q + }; + // Check if any PMPs are violated + end else if (!pmp_data_allow) begin + lsu_exception_o.cause = riscv::LD_ACCESS_FAULT; + lsu_exception_o.valid = 1'b1; + if (CVA6Cfg.TvalEn) lsu_exception_o.tval = lsu_paddr_o[riscv::PLEN-1:2]; + end + end + end else + + // --------- + // DTLB Miss + // --------- + // watch out for exceptions + if (ptw_active && !walking_instr) begin + // page table walker threw an exception + if (ptw_error) begin + // an error makes the translation valid + lsu_valid_o = 1'b1; + // the page table walker can only throw page faults + if (lsu_is_store_q) begin + lsu_exception_o.cause = riscv::STORE_PAGE_FAULT; + lsu_exception_o.valid = 1'b1; + if (CVA6Cfg.TvalEn) + lsu_exception_o.tval = { + {riscv::XLEN - riscv::VLEN{lsu_vaddr_q[riscv::VLEN-1]}}, update_vaddr + }; + end else begin + lsu_exception_o.cause = riscv::LOAD_PAGE_FAULT; + lsu_exception_o.valid = 1'b1; + if (CVA6Cfg.TvalEn) + lsu_exception_o.tval = { + {riscv::XLEN - riscv::VLEN{lsu_vaddr_q[riscv::VLEN-1]}}, update_vaddr + }; + end + end + + if (ptw_access_exception) begin + // an error makes the translation valid + lsu_valid_o = 1'b1; + // the page table walker can only throw page faults + lsu_exception_o.cause = riscv::LD_ACCESS_FAULT; + lsu_exception_o.valid = 1'b1; + if (CVA6Cfg.TvalEn) lsu_exception_o.tval = ptw_bad_paddr[riscv::PLEN-1:2]; + end + end + end // If translation is not enabled, check the paddr immediately against PMPs + else if (lsu_req_q && !misaligned_ex_q.valid && !pmp_data_allow) begin + if (lsu_is_store_q) begin + lsu_exception_o.cause = riscv::ST_ACCESS_FAULT; + lsu_exception_o.valid = 1'b1; + if (CVA6Cfg.TvalEn) lsu_exception_o.tval = lsu_paddr_o[riscv::PLEN-1:2]; + end else begin + lsu_exception_o.cause = riscv::LD_ACCESS_FAULT; + lsu_exception_o.valid = 1'b1; + if (CVA6Cfg.TvalEn) lsu_exception_o.tval = lsu_paddr_o[riscv::PLEN-1:2]; + end + end + end + + // Load/store PMP check + pmp #( + .PLEN (riscv::PLEN), + .PMP_LEN (riscv::PLEN - 2), + .NR_ENTRIES(CVA6Cfg.NrPMPEntries) + ) i_pmp_data ( + .addr_i (lsu_paddr_o), + .priv_lvl_i (ld_st_priv_lvl_i), + .access_type_i(pmp_access_type), + // Configuration + .conf_addr_i (pmpaddr_i), + .conf_i (pmpcfg_i), + .allow_o (pmp_data_allow) + ); + + // ---------- + // Registers + // ---------- + always_ff @(posedge clk_i or negedge rst_ni) begin + if (~rst_ni) begin + lsu_vaddr_q <= '0; + lsu_req_q <= '0; + misaligned_ex_q <= '0; + dtlb_pte_q <= '0; + dtlb_hit_q <= '0; + lsu_is_store_q <= '0; + dtlb_is_4M_q <= '0; + end else begin + lsu_vaddr_q <= lsu_vaddr_n; + lsu_req_q <= lsu_req_n; + misaligned_ex_q <= misaligned_ex_n; + dtlb_pte_q <= dtlb_pte_n; + dtlb_hit_q <= dtlb_hit_n; + lsu_is_store_q <= lsu_is_store_n; + dtlb_is_4M_q <= dtlb_is_4M_n; + end + end +endmodule diff --git a/core/mmu_sv32/cva6_ptw_sv32.sv b/core/mmu_sv32/cva6_ptw_sv32.sv new file mode 100644 index 000000000..4bd736bd3 --- /dev/null +++ b/core/mmu_sv32/cva6_ptw_sv32.sv @@ -0,0 +1,400 @@ +// Copyright (c) 2021 Thales. +// Copyright and related rights are licensed under the Solderpad Hardware +// License, Version 0.51 (the "License"); you may not use this file except in +// compliance with the License. You may obtain a copy of the License at +// http://solderpad.org/licenses/SHL-0.51. Unless required by applicable law +// or agreed to in writing, software, hardware and materials distributed under +// this License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. +// +// Author: Sebastien Jacq Thales Research & Technology +// Date: 17/07/2021 +// +// Additional contributions by: +// Sebastien Jacq - sjthales on github.com +// +// Description: Hardware-PTW (Page-Table-Walker) for MMU Sv32. +// This module is an adaptation of the Sv39 PTW developed +// by Florian Zaruba and David Schaffenrath to the Sv32 standard. +// +// =========================================================================== // +// Revisions : +// Date Version Author Description +// 2020-02-17 0.1 S.Jacq PTW Sv32 for CV32A6 +// =========================================================================== // + +/* verilator lint_off WIDTH */ + +module cva6_ptw_sv32 + import ariane_pkg::*; +#( + parameter config_pkg::cva6_cfg_t CVA6Cfg = config_pkg::cva6_cfg_empty, + parameter int ASID_WIDTH = 1 +) ( + input logic clk_i, // Clock + input logic rst_ni, // Asynchronous reset active low + input logic flush_i, // flush everything, we need to do this because + // actually everything we do is speculative at this stage + // e.g.: there could be a CSR instruction that changes everything + output logic ptw_active_o, + output logic walking_instr_o, // set when walking for TLB + output logic ptw_error_o, // set when an error occurred + output logic ptw_access_exception_o, // set when an PMP access exception occured + + input logic lsu_is_store_i, // this translation was triggered by a store + // PTW memory interface + input dcache_req_o_t req_port_i, + output dcache_req_i_t req_port_o, + + // to Shared TLB, update logic + output tlb_update_sv32_t shared_tlb_update_o, + + output logic [riscv::VLEN-1:0] update_vaddr_o, + + input logic [ASID_WIDTH-1:0] asid_i, + + // from shared TLB + input logic shared_tlb_access_i, + input logic shared_tlb_hit_i, + input logic [riscv::VLEN-1:0] shared_tlb_vaddr_i, + + input logic itlb_req_i, + + // from CSR file + input logic [riscv::PPNW-1:0] satp_ppn_i, // ppn from satp + input logic mxr_i, + + // Performance counters + output logic shared_tlb_miss_o, + + // PMP + input riscv::pmpcfg_t [15:0] pmpcfg_i, + input logic [15:0][riscv::PLEN-3:0] pmpaddr_i, + output logic [riscv::PLEN-1:0] bad_paddr_o + +); + + // input registers + logic data_rvalid_q; + riscv::xlen_t data_rdata_q; + + riscv::pte_sv32_t pte; + assign pte = riscv::pte_sv32_t'(data_rdata_q); + + + enum logic [2:0] { + IDLE, + WAIT_GRANT, + PTE_LOOKUP, + WAIT_RVALID, + PROPAGATE_ERROR, + PROPAGATE_ACCESS_ERROR, + LATENCY + } + state_q, state_d; + + // SV32 defines two levels of page tables + enum logic { + LVL1, + LVL2 + } + ptw_lvl_q, ptw_lvl_n; + + // is this an instruction page table walk? + logic is_instr_ptw_q, is_instr_ptw_n; + logic global_mapping_q, global_mapping_n; + // latched tag signal + logic tag_valid_n, tag_valid_q; + // register the ASID + logic [ASID_WIDTH-1:0] tlb_update_asid_q, tlb_update_asid_n; + // register the VPN we need to walk, SV32 defines a 32 bit virtual address + logic [riscv::VLEN-1:0] vaddr_q, vaddr_n; + // 4 byte aligned physical pointer + logic [riscv::PLEN-1:0] ptw_pptr_q, ptw_pptr_n; + + // Assignments + assign update_vaddr_o = vaddr_q; + + assign ptw_active_o = (state_q != IDLE); + //assign walking_instr_o = is_instr_ptw_q; + assign walking_instr_o = is_instr_ptw_q; + // directly output the correct physical address + assign req_port_o.address_index = ptw_pptr_q[DCACHE_INDEX_WIDTH-1:0]; + assign req_port_o.address_tag = ptw_pptr_q[DCACHE_INDEX_WIDTH+DCACHE_TAG_WIDTH-1:DCACHE_INDEX_WIDTH]; + // we are never going to kill this request + assign req_port_o.kill_req = '0; + // we are never going to write with the HPTW + assign req_port_o.data_wdata = '0; + // we only issue one single request at a time + assign req_port_o.data_id = '0; + + // ----------- + // Shared TLB Update + // ----------- + assign shared_tlb_update_o.vpn = vaddr_q[riscv::SV-1:12]; + // update the correct page table level + assign shared_tlb_update_o.is_4M = (ptw_lvl_q == LVL1); + // output the correct ASID + assign shared_tlb_update_o.asid = tlb_update_asid_q; + // set the global mapping bit + assign shared_tlb_update_o.content = pte | (global_mapping_q << 5); + + + assign req_port_o.tag_valid = tag_valid_q; + + logic allow_access; + + assign bad_paddr_o = ptw_access_exception_o ? ptw_pptr_q : 'b0; + + pmp #( + .CVA6Cfg (CVA6Cfg), + .PLEN (riscv::PLEN), + .PMP_LEN (riscv::PLEN - 2), + .NR_ENTRIES(CVA6Cfg.NrPMPEntries) + ) i_pmp_ptw ( + .addr_i (ptw_pptr_q), + // PTW access are always checked as if in S-Mode... + .priv_lvl_i (riscv::PRIV_LVL_S), + // ...and they are always loads + .access_type_i(riscv::ACCESS_READ), + // Configuration + .conf_addr_i (pmpaddr_i), + .conf_i (pmpcfg_i), + .allow_o (allow_access) + ); + + + assign req_port_o.data_be = be_gen_32(req_port_o.address_index[1:0], req_port_o.data_size); + + //------------------- + // Page table walker + //------------------- + // A virtual address va is translated into a physical address pa as follows: + // 1. Let a be sptbr.ppn × PAGESIZE, and let i = LEVELS-1. (For Sv39, + // PAGESIZE=2^12 and LEVELS=3.) + // 2. Let pte be the value of the PTE at address a+va.vpn[i]×PTESIZE. (For + // Sv32, PTESIZE=4.) + // 3. If pte.v = 0, or if pte.r = 0 and pte.w = 1, stop and raise an access + // exception. + // 4. Otherwise, the PTE is valid. If pte.r = 1 or pte.x = 1, go to step 5. + // Otherwise, this PTE is a pointer to the next level of the page table. + // Let i=i-1. If i < 0, stop and raise an access exception. Otherwise, let + // a = pte.ppn × PAGESIZE and go to step 2. + // 5. A leaf PTE has been found. Determine if the requested memory access + // is allowed by the pte.r, pte.w, and pte.x bits. If not, stop and + // raise an access exception. Otherwise, the translation is successful. + // Set pte.a to 1, and, if the memory access is a store, set pte.d to 1. + // The translated physical address is given as follows: + // - pa.pgoff = va.pgoff. + // - If i > 0, then this is a superpage translation and + // pa.ppn[i-1:0] = va.vpn[i-1:0]. + // - pa.ppn[LEVELS-1:i] = pte.ppn[LEVELS-1:i]. + always_comb begin : ptw + // default assignments + // PTW memory interface + tag_valid_n = 1'b0; + req_port_o.data_req = 1'b0; + req_port_o.data_size = 2'b10; + req_port_o.data_we = 1'b0; + ptw_error_o = 1'b0; + ptw_access_exception_o = 1'b0; + shared_tlb_update_o.valid = 1'b0; + is_instr_ptw_n = is_instr_ptw_q; + ptw_lvl_n = ptw_lvl_q; + ptw_pptr_n = ptw_pptr_q; + state_d = state_q; + global_mapping_n = global_mapping_q; + // input registers + tlb_update_asid_n = tlb_update_asid_q; + vaddr_n = vaddr_q; + + shared_tlb_miss_o = 1'b0; + + case (state_q) + + IDLE: begin + // by default we start with the top-most page table + ptw_lvl_n = LVL1; + global_mapping_n = 1'b0; + is_instr_ptw_n = 1'b0; + // if we got a Shared TLB miss + if (shared_tlb_access_i & ~shared_tlb_hit_i) begin + ptw_pptr_n = { + satp_ppn_i, shared_tlb_vaddr_i[riscv::SV-1:22], 2'b0 + }; // SATP.PPN * PAGESIZE + VPN*PTESIZE = SATP.PPN * 2^(12) + VPN*4 + is_instr_ptw_n = itlb_req_i; + tlb_update_asid_n = asid_i; + vaddr_n = shared_tlb_vaddr_i; + state_d = WAIT_GRANT; + shared_tlb_miss_o = 1'b1; + end + end + + WAIT_GRANT: begin + // send a request out + req_port_o.data_req = 1'b1; + // wait for the WAIT_GRANT + if (req_port_i.data_gnt) begin + // send the tag valid signal one cycle later + tag_valid_n = 1'b1; + state_d = PTE_LOOKUP; + end + end + + PTE_LOOKUP: begin + // we wait for the valid signal + if (data_rvalid_q) begin + + // check if the global mapping bit is set + if (pte.g) global_mapping_n = 1'b1; + + // ------------- + // Invalid PTE + // ------------- + // If pte.v = 0, or if pte.r = 0 and pte.w = 1, stop and raise a page-fault exception. + if (!pte.v || (!pte.r && pte.w)) state_d = PROPAGATE_ERROR; + // ----------- + // Valid PTE + // ----------- + else begin + //state_d = IDLE; + state_d = LATENCY; + // it is a valid PTE + // if pte.r = 1 or pte.x = 1 it is a valid PTE + if (pte.r || pte.x) begin + // Valid translation found (either 4M or 4K entry) + if (is_instr_ptw_q) begin + // ------------ + // Update ITLB + // ------------ + // If page is not executable, we can directly raise an error. This + // doesn't put a useless entry into the TLB. The same idea applies + // to the access flag since we let the access flag be managed by SW. + if (!pte.x || !pte.a) state_d = PROPAGATE_ERROR; + else shared_tlb_update_o.valid = 1'b1; + + end else begin + // ------------ + // Update DTLB + // ------------ + // Check if the access flag has been set, otherwise throw a page-fault + // and let the software handle those bits. + // If page is not readable (there are no write-only pages) + // we can directly raise an error. This doesn't put a useless + // entry into the TLB. + if (pte.a && (pte.r || (pte.x && mxr_i))) begin + shared_tlb_update_o.valid = 1'b1; + end else begin + state_d = PROPAGATE_ERROR; + end + // Request is a store: perform some additional checks + // If the request was a store and the page is not write-able, raise an error + // the same applies if the dirty flag is not set + if (lsu_is_store_i && (!pte.w || !pte.d)) begin + shared_tlb_update_o.valid = 1'b0; + state_d = PROPAGATE_ERROR; + end + end + // check if the ppn is correctly aligned: + // 6. If i > 0 and pa.ppn[i − 1 : 0] != 0, this is a misaligned superpage; stop and raise a page-fault + // exception. + if (ptw_lvl_q == LVL1 && pte.ppn[9:0] != '0) begin + state_d = PROPAGATE_ERROR; + shared_tlb_update_o.valid = 1'b0; + end + // this is a pointer to the next TLB level + end else begin + // pointer to next level of page table + if (ptw_lvl_q == LVL1) begin + // we are in the second level now + ptw_lvl_n = LVL2; + ptw_pptr_n = {pte.ppn, vaddr_q[21:12], 2'b0}; + end + + state_d = WAIT_GRANT; + + if (ptw_lvl_q == LVL2) begin + // Should already be the last level page table => Error + ptw_lvl_n = LVL2; + state_d = PROPAGATE_ERROR; + end + end + end + + // Check if this access was actually allowed from a PMP perspective + if (!allow_access) begin + shared_tlb_update_o.valid = 1'b0; + // we have to return the failed address in bad_addr + ptw_pptr_n = ptw_pptr_q; + state_d = PROPAGATE_ACCESS_ERROR; + end + end + // we've got a data WAIT_GRANT so tell the cache that the tag is valid + end + // Propagate error to MMU/LSU + PROPAGATE_ERROR: begin + state_d = LATENCY; + ptw_error_o = 1'b1; + end + PROPAGATE_ACCESS_ERROR: begin + state_d = LATENCY; + ptw_access_exception_o = 1'b1; + end + // wait for the rvalid before going back to IDLE + WAIT_RVALID: begin + if (data_rvalid_q) state_d = IDLE; + end + LATENCY: begin + state_d = IDLE; + end + default: begin + state_d = IDLE; + end + endcase + + // ------- + // Flush + // ------- + // should we have flushed before we got an rvalid, wait for it until going back to IDLE + if (flush_i) begin + // on a flush check whether we are + // 1. in the PTE Lookup check whether we still need to wait for an rvalid + // 2. waiting for a grant, if so: wait for it + // if not, go back to idle + if (((state_q inside {PTE_LOOKUP, WAIT_RVALID}) && !data_rvalid_q) || + ((state_q == WAIT_GRANT) && req_port_i.data_gnt)) + state_d = WAIT_RVALID; + else state_d = LATENCY; + end + end + + // sequential process + always_ff @(posedge clk_i or negedge rst_ni) begin + if (~rst_ni) begin + state_q <= IDLE; + is_instr_ptw_q <= 1'b0; + ptw_lvl_q <= LVL1; + tag_valid_q <= 1'b0; + tlb_update_asid_q <= '0; + vaddr_q <= '0; + ptw_pptr_q <= '0; + global_mapping_q <= 1'b0; + data_rdata_q <= '0; + data_rvalid_q <= 1'b0; + end else begin + state_q <= state_d; + ptw_pptr_q <= ptw_pptr_n; + is_instr_ptw_q <= is_instr_ptw_n; + ptw_lvl_q <= ptw_lvl_n; + tag_valid_q <= tag_valid_n; + tlb_update_asid_q <= tlb_update_asid_n; + vaddr_q <= vaddr_n; + global_mapping_q <= global_mapping_n; + data_rdata_q <= req_port_i.data_rdata; + data_rvalid_q <= req_port_i.data_rvalid; + end + end + +endmodule +/* verilator lint_on WIDTH */ diff --git a/core/cva6_mmu/cva6_shared_tlb.sv b/core/mmu_sv32/cva6_shared_tlb_sv32.sv similarity index 52% rename from core/cva6_mmu/cva6_shared_tlb.sv rename to core/mmu_sv32/cva6_shared_tlb_sv32.sv index 43157de29..98e2a044a 100644 --- a/core/cva6_mmu/cva6_shared_tlb.sv +++ b/core/mmu_sv32/cva6_shared_tlb_sv32.sv @@ -1,6 +1,4 @@ // Copyright (c) 2023 Thales. -// Copyright (c) 2024, PlanV Technology -// SPDX-License-Identifier: Apache-2.0 WITH SHL-2.1 // Copyright and related rights are licensed under the Solderpad Hardware // License, Version 0.51 (the "License"); you may not use this file except in // compliance with the License. You may obtain a copy of the License at @@ -10,33 +8,31 @@ // CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. // -// Author: Angela Gonzalez PlanV Technology -// Date: 26/02/2024 +// Author: Sebastien Jacq - Thales Research & Technology +// Date: 08/03/2023 // // Description: N-way associative shared TLB, it allows to reduce the number -// of ITLB and DTLB entries. This module is an update of the -// shared TLB sv32 developed by Sebastien Jacq (Thales Research & Technology) -// to be used with sv32, sv39 and sv39x4. +// of ITLB and DTLB entries. +// /* verilator lint_off WIDTH */ -module cva6_shared_tlb #( - parameter type pte_cva6_t = logic, - parameter type tlb_update_cva6_t = logic, +module cva6_shared_tlb_sv32 + import ariane_pkg::*; +#( + parameter config_pkg::cva6_cfg_t CVA6Cfg = config_pkg::cva6_cfg_empty, parameter int SHARED_TLB_DEPTH = 64, parameter int SHARED_TLB_WAYS = 2, - parameter int unsigned HYP_EXT = 0, - parameter int ASID_WIDTH[HYP_EXT:0], //[vmid_width,asid_width] - parameter int unsigned VPN_LEN = 1, - parameter int unsigned PT_LEVELS = 1 + parameter int ASID_WIDTH = 1 ) ( - input logic clk_i, // Clock + input logic clk_i, // Clock input logic rst_ni, // Asynchronous reset active low - input logic [HYP_EXT*2:0] flush_i, // Flush signal [g_stage,vs stage, normal translation signal] - input logic [1:0][HYP_EXT*2:0] v_st_enbl_i, // v_i,g-stage enabled, s-stage enabled + input logic flush_i, - input logic [ASID_WIDTH[0]-1:0] dtlb_asid_i[HYP_EXT:0], //[vmid,vs_asid,asid] - input logic [ASID_WIDTH[0]-1:0] itlb_asid_i[HYP_EXT:0], //[vmid,vs_asid,asid] + input logic enable_translation_i, // CSRs indicate to enable SV32 + input logic en_ld_st_translation_i, // enable virtual memory translation for load/stores + + input logic [ASID_WIDTH-1:0] asid_i, // from TLBs // did we miss? @@ -49,8 +45,8 @@ module cva6_shared_tlb #( input logic [riscv::VLEN-1:0] dtlb_vaddr_i, // to TLBs, update logic - output tlb_update_cva6_t itlb_update_o, - output tlb_update_cva6_t dtlb_update_o, + output tlb_update_sv32_t itlb_update_o, + output tlb_update_sv32_t dtlb_update_o, // Performance counters output logic itlb_miss_o, @@ -63,7 +59,7 @@ module cva6_shared_tlb #( output logic itlb_req_o, // Update shared TLB in case of miss - input tlb_update_cva6_t shared_tlb_update_i + input tlb_update_sv32_t shared_tlb_update_i ); @@ -76,10 +72,10 @@ module cva6_shared_tlb #( endfunction typedef struct packed { - logic [HYP_EXT:0][ASID_WIDTH[0]-1:0] asid; - logic [PT_LEVELS+HYP_EXT-1:0][(VPN_LEN/PT_LEVELS)-1:0] vpn; - logic [PT_LEVELS-2:0][HYP_EXT:0] is_page; - logic [HYP_EXT*2:0] v_st_enbl; // v_i,g-stage enabled, s-stage enabled + logic [8:0] asid; //9 bits wide + logic [9:0] vpn1; //10 bits wide + logic [9:0] vpn0; //10 bits wide + logic is_4M; } shared_tag_t; shared_tag_t shared_tag_wr; @@ -103,30 +99,24 @@ module cva6_shared_tlb #( logic [ SHARED_TLB_WAYS-1:0] pte_wr_en; logic [$clog2(SHARED_TLB_DEPTH)-1:0] pte_wr_addr; - logic [ $bits(pte_cva6_t)-1:0] pte_wr_data [ HYP_EXT:0]; + logic [$bits(riscv::pte_sv32_t)-1:0] pte_wr_data; logic [ SHARED_TLB_WAYS-1:0] pte_rd_en; logic [$clog2(SHARED_TLB_DEPTH)-1:0] pte_rd_addr; - logic [ $bits(pte_cva6_t)-1:0] pte_rd_data [SHARED_TLB_WAYS-1:0] [HYP_EXT:0]; + logic [$bits(riscv::pte_sv32_t)-1:0] pte_rd_data [SHARED_TLB_WAYS-1:0]; logic [ SHARED_TLB_WAYS-1:0] pte_req; logic [ SHARED_TLB_WAYS-1:0] pte_we; logic [$clog2(SHARED_TLB_DEPTH)-1:0] pte_addr; - logic [PT_LEVELS+HYP_EXT-1:0][(VPN_LEN/PT_LEVELS)-1:0] vpn_d, vpn_q; - logic [SHARED_TLB_WAYS-1:0][PT_LEVELS-1:0] vpn_match; - logic [SHARED_TLB_WAYS-1:0][PT_LEVELS-1:0] page_match; - logic [SHARED_TLB_WAYS-1:0][PT_LEVELS-1:0] level_match; + logic [9:0] vpn0_d, vpn1_d, vpn0_q, vpn1_q; - logic [SHARED_TLB_WAYS-1:0][HYP_EXT:0] match_asid; - logic [SHARED_TLB_WAYS-1:0] match_stage; - - pte_cva6_t [SHARED_TLB_WAYS-1:0][HYP_EXT:0] pte; + riscv::pte_sv32_t [SHARED_TLB_WAYS-1:0] pte; logic [riscv::VLEN-1-12:0] itlb_vpn_q; logic [riscv::VLEN-1-12:0] dtlb_vpn_q; - logic [ASID_WIDTH[0]-1:0] tlb_update_asid_q[HYP_EXT:0], tlb_update_asid_d[HYP_EXT:0]; + logic [ASID_WIDTH-1:0] tlb_update_asid_q, tlb_update_asid_d; logic shared_tlb_access_q, shared_tlb_access_d; logic shared_tlb_hit_d; @@ -135,8 +125,6 @@ module cva6_shared_tlb #( logic itlb_req_d, itlb_req_q; logic dtlb_req_d, dtlb_req_q; - int i_req_d, i_req_q; - // replacement strategy logic [SHARED_TLB_WAYS-1:0] way_valid; logic update_lfsr; // shift the LFSR @@ -152,53 +140,14 @@ module cva6_shared_tlb #( assign itlb_req_o = itlb_req_q; - genvar i, x; - generate - for (i = 0; i < SHARED_TLB_WAYS; i++) begin : gen_match_tlb_ways - //identify page_match for all TLB Entries - - for (x = 0; x < PT_LEVELS; x++) begin : gen_match - assign page_match[i][x] = x==0 ? 1 :((HYP_EXT==0 || x==(PT_LEVELS-1)) ? // PAGE_MATCH CONTAINS THE MATCH INFORMATION FOR EACH TAG OF is_1G and is_2M in sv39x4. HIGHER LEVEL (Giga page), THEN THERE IS THE Mega page AND AT THE LOWER LEVEL IS ALWAYS 1 - &(shared_tag_rd[i].is_page[PT_LEVELS-1-x] | (~v_st_enbl_i[i_req_q][HYP_EXT:0])): - ((&v_st_enbl_i[i_req_q][HYP_EXT:0]) ? - ((shared_tag_rd[i].is_page[PT_LEVELS-1-x][0] && (shared_tag_rd[i].is_page[PT_LEVELS-2-x][HYP_EXT] || shared_tag_rd[i].is_page[PT_LEVELS-1-x][HYP_EXT])) - || (shared_tag_rd[i].is_page[PT_LEVELS-1-x][HYP_EXT] && (shared_tag_rd[i].is_page[PT_LEVELS-2-x][0] || shared_tag_rd[i].is_page[PT_LEVELS-1-x][0]))): - shared_tag_rd[i].is_page[PT_LEVELS-1-x][0] && v_st_enbl_i[i_req_q][0] || shared_tag_rd[i].is_page[PT_LEVELS-1-x][HYP_EXT] && v_st_enbl_i[i_req_q][HYP_EXT])); - - //identify if vpn matches at all PT levels for all TLB entries - assign vpn_match[i][x] = (HYP_EXT==1 && x==(PT_LEVELS-1) && ~v_st_enbl_i[i_req_q][0]) ? // - vpn_q[x] == shared_tag_rd[i].vpn[x] && vpn_q[x+1][(VPN_LEN%PT_LEVELS)-1:0] == shared_tag_rd[i].vpn[x+1][(VPN_LEN%PT_LEVELS)-1:0]: // - vpn_q[x] == shared_tag_rd[i].vpn[x]; - - //identify if there is a hit at each PT level for all TLB entries - assign level_match[i][x] = &vpn_match[i][PT_LEVELS-1:x] && page_match[i][x]; - - end - end - endgenerate - - genvar w; - generate - for (w = 0; w < PT_LEVELS; w++) begin - assign vpn_d[w] = ((|v_st_enbl_i[1][HYP_EXT:0]) && itlb_access_i && ~itlb_hit_i && ~dtlb_access_i) ? // - itlb_vaddr_i[12+((VPN_LEN/PT_LEVELS)*(w+1))-1:12+((VPN_LEN/PT_LEVELS)*w)] : // - (((|v_st_enbl_i[0][HYP_EXT:0]) && dtlb_access_i && ~dtlb_hit_i) ? // - dtlb_vaddr_i[12+((VPN_LEN/PT_LEVELS)*(w+1))-1:12+((VPN_LEN/PT_LEVELS)*w)] : vpn_q[w]); - end - endgenerate - - if (HYP_EXT == 1) //THIS UPDATES THE EXTRA BITS OF VPN IN SV39x4 - assign vpn_d[PT_LEVELS][(VPN_LEN%PT_LEVELS)-1:0] = ((|v_st_enbl_i[1][HYP_EXT:0]) && itlb_access_i && ~itlb_hit_i && ~dtlb_access_i) ? // - itlb_vaddr_i[VPN_LEN-1:VPN_LEN-(VPN_LEN%PT_LEVELS)] : // - (((|v_st_enbl_i[0][HYP_EXT:0]) && dtlb_access_i && ~dtlb_hit_i) ? // - dtlb_vaddr_i[VPN_LEN-1: VPN_LEN-(VPN_LEN%PT_LEVELS)] : vpn_q[PT_LEVELS][(VPN_LEN%PT_LEVELS)-1:0]); - /////////////////////////////////////////////////////// // tag comparison, hit generation /////////////////////////////////////////////////////// always_comb begin : itlb_dtlb_miss itlb_miss_o = 1'b0; dtlb_miss_o = 1'b0; + vpn0_d = vpn0_q; + vpn1_d = vpn1_q; tag_rd_en = '0; pte_rd_en = '0; @@ -213,37 +162,42 @@ module cva6_shared_tlb #( tag_rd_addr = '0; pte_rd_addr = '0; - i_req_d = i_req_q; // if we got an ITLB miss - if ((|v_st_enbl_i[1][HYP_EXT:0]) & itlb_access_i & ~itlb_hit_i & ~dtlb_access_i) begin + if (enable_translation_i & itlb_access_i & ~itlb_hit_i & ~dtlb_access_i) begin tag_rd_en = '1; tag_rd_addr = itlb_vaddr_i[12+:$clog2(SHARED_TLB_DEPTH)]; pte_rd_en = '1; pte_rd_addr = itlb_vaddr_i[12+:$clog2(SHARED_TLB_DEPTH)]; + vpn0_d = itlb_vaddr_i[21:12]; + vpn1_d = itlb_vaddr_i[31:22]; + itlb_miss_o = 1'b1; itlb_req_d = 1'b1; - tlb_update_asid_d = itlb_asid_i; - shared_tlb_access_d = '1; + tlb_update_asid_d = asid_i; + + shared_tlb_access_d = 1'b1; shared_tlb_vaddr_d = itlb_vaddr_i; - i_req_d = 1; // we got an DTLB miss - end else if ((|v_st_enbl_i[0][HYP_EXT:0]) & dtlb_access_i & ~dtlb_hit_i) begin + end else if (en_ld_st_translation_i & dtlb_access_i & ~dtlb_hit_i) begin tag_rd_en = '1; tag_rd_addr = dtlb_vaddr_i[12+:$clog2(SHARED_TLB_DEPTH)]; pte_rd_en = '1; pte_rd_addr = dtlb_vaddr_i[12+:$clog2(SHARED_TLB_DEPTH)]; + vpn0_d = dtlb_vaddr_i[21:12]; + vpn1_d = dtlb_vaddr_i[31:22]; + dtlb_miss_o = 1'b1; dtlb_req_d = 1'b1; - tlb_update_asid_d = dtlb_asid_i; - shared_tlb_access_d = '1; + tlb_update_asid_d = asid_i; + + shared_tlb_access_d = 1'b1; shared_tlb_vaddr_d = dtlb_vaddr_i; - i_req_d = 0; end end //itlb_dtlb_miss @@ -251,41 +205,23 @@ module cva6_shared_tlb #( shared_tlb_hit_d = 1'b0; dtlb_update_o = '0; itlb_update_o = '0; - //number of ways for (int unsigned i = 0; i < SHARED_TLB_WAYS; i++) begin - // first level match, this may be a giga page, check the ASID flags as well - // if the entry is associated to a global address, don't match the ASID (ASID is don't care) - match_asid[i][0] = (((tlb_update_asid_q[0][ASID_WIDTH[0]-1:0] == shared_tag_rd[i].asid[0][ASID_WIDTH[0]-1:0]) || pte[i][0].g) && v_st_enbl_i[i_req_q][0]) || !v_st_enbl_i[i_req_q][0]; - - if (HYP_EXT == 1) begin - match_asid[i][HYP_EXT] = (tlb_update_asid_q[HYP_EXT][ASID_WIDTH[HYP_EXT]-1:0] == shared_tag_rd[i].asid[HYP_EXT][ASID_WIDTH[HYP_EXT]-1:0] && v_st_enbl_i[i_req_q][HYP_EXT]) || !v_st_enbl_i[i_req_q][HYP_EXT]; - end - - // check if translation is a: S-Stage and G-Stage, S-Stage only or G-Stage only translation and virtualization mode is on/off - match_stage[i] = shared_tag_rd[i].v_st_enbl == v_st_enbl_i[i_req_q]; - - if (shared_tag_valid[i] && &match_asid[i] && match_stage[i]) begin - if (|level_match[i]) begin + if (shared_tag_valid[i] && ((tlb_update_asid_q == shared_tag_rd[i].asid) || pte[i].g) && vpn1_q == shared_tag_rd[i].vpn1) begin + if (shared_tag_rd[i].is_4M || vpn0_q == shared_tag_rd[i].vpn0) begin shared_tlb_hit_d = 1'b1; if (itlb_req_q) begin itlb_update_o.valid = 1'b1; itlb_update_o.vpn = itlb_vpn_q; - itlb_update_o.is_page = shared_tag_rd[i].is_page; + itlb_update_o.is_4M = shared_tag_rd[i].is_4M; + itlb_update_o.asid = tlb_update_asid_q; itlb_update_o.content = pte[i]; - itlb_update_o.v_st_enbl = shared_tag_rd[i].v_st_enbl; - for (int unsigned a = 0; a < HYP_EXT + 1; a++) begin - itlb_update_o.asid[a] = tlb_update_asid_q[a]; - end end else if (dtlb_req_q) begin dtlb_update_o.valid = 1'b1; dtlb_update_o.vpn = dtlb_vpn_q; - dtlb_update_o.is_page = shared_tag_rd[i].is_page; + dtlb_update_o.is_4M = shared_tag_rd[i].is_4M; + dtlb_update_o.asid = tlb_update_asid_q; dtlb_update_o.content = pte[i]; - dtlb_update_o.v_st_enbl = shared_tag_rd[i].v_st_enbl; - for (int unsigned a = 0; a < HYP_EXT + 1; a++) begin - dtlb_update_o.asid[a] = tlb_update_asid_q[a]; - end end end end @@ -297,14 +233,14 @@ module cva6_shared_tlb #( if (~rst_ni) begin itlb_vpn_q <= '0; dtlb_vpn_q <= '0; - tlb_update_asid_q <= '{default: 0}; + tlb_update_asid_q <= '0; shared_tlb_access_q <= '0; shared_tlb_vaddr_q <= '0; shared_tag_valid_q <= '0; - vpn_q <= 0; + vpn0_q <= '0; + vpn1_q <= '0; itlb_req_q <= '0; dtlb_req_q <= '0; - i_req_q <= 0; shared_tag_valid <= '0; end else begin itlb_vpn_q <= itlb_vaddr_i[riscv::SV-1:12]; @@ -313,10 +249,10 @@ module cva6_shared_tlb #( shared_tlb_access_q <= shared_tlb_access_d; shared_tlb_vaddr_q <= shared_tlb_vaddr_d; shared_tag_valid_q <= shared_tag_valid_d; - vpn_q <= vpn_d; + vpn0_q <= vpn0_d; + vpn1_q <= vpn1_d; itlb_req_q <= itlb_req_d; dtlb_req_q <= dtlb_req_d; - i_req_q <= i_req_d; shared_tag_valid <= shared_tag_valid_q[tag_rd_addr]; end end @@ -329,7 +265,7 @@ module cva6_shared_tlb #( tag_wr_en = '0; pte_wr_en = '0; - if (|flush_i) begin + if (flush_i) begin shared_tag_valid_d = '0; end else if (shared_tlb_update_i.valid) begin for (int unsigned i = 0; i < SHARED_TLB_WAYS; i++) begin @@ -343,32 +279,15 @@ module cva6_shared_tlb #( end //update_flush assign shared_tag_wr.asid = shared_tlb_update_i.asid; - assign shared_tag_wr.is_page = shared_tlb_update_i.is_page; - assign shared_tag_wr.v_st_enbl = v_st_enbl_i[i_req_q]; - - genvar z; - generate - for (z = 0; z < PT_LEVELS; z++) begin : gen_shared_tag - assign shared_tag_wr.vpn[z] = shared_tlb_update_i.vpn[((VPN_LEN/PT_LEVELS)*(z+1))-1:((VPN_LEN/PT_LEVELS)*z)]; - end - if (HYP_EXT == 1) begin : gen_shared_tag_hyp - //THIS UPDATES THE EXTRA BITS OF VPN IN SV39x4 - assign shared_tag_wr.vpn[PT_LEVELS][(VPN_LEN%PT_LEVELS)-1:0] = shared_tlb_update_i.vpn[VPN_LEN-1: VPN_LEN-(VPN_LEN%PT_LEVELS)]; - end - endgenerate - + assign shared_tag_wr.vpn1 = shared_tlb_update_i.vpn[19:10]; + assign shared_tag_wr.vpn0 = shared_tlb_update_i.vpn[9:0]; + assign shared_tag_wr.is_4M = shared_tlb_update_i.is_4M; assign tag_wr_addr = shared_tlb_update_i.vpn[$clog2(SHARED_TLB_DEPTH)-1:0]; assign tag_wr_data = shared_tag_wr; assign pte_wr_addr = shared_tlb_update_i.vpn[$clog2(SHARED_TLB_DEPTH)-1:0]; - - genvar h; - generate - for (h = 0; h < HYP_EXT + 1; h++) begin : gen_pte_wr_data - assign pte_wr_data[h] = shared_tlb_update_i.content[h]; - end - endgenerate + assign pte_wr_data = shared_tlb_update_i.content; assign way_valid = shared_tag_valid_q[shared_tlb_update_i.vpn[$clog2(SHARED_TLB_DEPTH)-1:0]]; assign repl_way = (all_ways_valid) ? rnd_way : inv_way; @@ -425,25 +344,23 @@ module cva6_shared_tlb #( assign shared_tag_rd[i] = shared_tag_t'(tag_rd_data[i]); - for (genvar a = 0; a < HYP_EXT + 1; a++) begin : g_content_sram - // PTE RAM - sram #( - .DATA_WIDTH($bits(pte_cva6_t)), - .NUM_WORDS (SHARED_TLB_DEPTH) - ) pte_sram ( - .clk_i (clk_i), - .rst_ni (rst_ni), - .req_i (pte_req[i]), - .we_i (pte_we[i]), - .addr_i (pte_addr), - .wuser_i('0), - .wdata_i(pte_wr_data[a]), - .be_i ('1), - .ruser_o(), - .rdata_o(pte_rd_data[i][a]) - ); - assign pte[i][a] = pte_cva6_t'(pte_rd_data[i][a]); - end + // PTE RAM + sram #( + .DATA_WIDTH($bits(riscv::pte_sv32_t)), + .NUM_WORDS (SHARED_TLB_DEPTH) + ) pte_sram ( + .clk_i (clk_i), + .rst_ni (rst_ni), + .req_i (pte_req[i]), + .we_i (pte_we[i]), + .addr_i (pte_addr), + .wuser_i('0), + .wdata_i(pte_wr_data), + .be_i ('1), + .ruser_o(), + .rdata_o(pte_rd_data[i]) + ); + assign pte[i] = riscv::pte_sv32_t'(pte_rd_data[i]); end endmodule diff --git a/core/mmu_sv32/cva6_tlb_sv32.sv b/core/mmu_sv32/cva6_tlb_sv32.sv new file mode 100644 index 000000000..79a7c98dc --- /dev/null +++ b/core/mmu_sv32/cva6_tlb_sv32.sv @@ -0,0 +1,281 @@ +// Copyright (c) 2021 Thales. +// Copyright and related rights are licensed under the Solderpad Hardware +// License, Version 0.51 (the "License"); you may not use this file except in +// compliance with the License. You may obtain a copy of the License at +// http://solderpad.org/licenses/SHL-0.51. Unless required by applicable law +// or agreed to in writing, software, hardware and materials distributed under +// this License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. +// +// Author: Sebastien Jacq Thales Research & Technology +// Date: 17/07/2021 +// +// Additional contributions by: +// Sebastien Jacq - sjthales on github.com +// +// Description: Translation Lookaside Buffer, Sv32 , fully set-associative +// This module is an adaptation of the Sv39 TLB developed +// by Florian Zaruba and David Schaffenrath to the Sv32 standard. +// +// =========================================================================== // +// Revisions : +// Date Version Author Description +// 2020-02-17 0.1 S.Jacq TLB Sv32 for CV32A6 +// =========================================================================== // + +module cva6_tlb_sv32 + import ariane_pkg::*; +#( + parameter config_pkg::cva6_cfg_t CVA6Cfg = config_pkg::cva6_cfg_empty, + parameter int unsigned TLB_ENTRIES = 4, + parameter int unsigned ASID_WIDTH = 1 +) ( + input logic clk_i, // Clock + input logic rst_ni, // Asynchronous reset active low + input logic flush_i, // Flush signal + // Update TLB + input tlb_update_sv32_t update_i, + // Lookup signals + input logic lu_access_i, + input logic [ASID_WIDTH-1:0] lu_asid_i, + input logic [riscv::VLEN-1:0] lu_vaddr_i, + output riscv::pte_sv32_t lu_content_o, + input logic [ASID_WIDTH-1:0] asid_to_be_flushed_i, + input logic [riscv::VLEN-1:0] vaddr_to_be_flushed_i, + output logic lu_is_4M_o, + output logic lu_hit_o +); + + // Sv32 defines two levels of page tables + struct packed { + logic [8:0] asid; //9 bits wide + logic [9:0] vpn1; //10 bits wide + logic [9:0] vpn0; //10 bits wide + logic is_4M; + logic valid; + } [TLB_ENTRIES-1:0] + tags_q, tags_n; + + riscv::pte_sv32_t [TLB_ENTRIES-1:0] content_q, content_n; + logic [9:0] vpn0, vpn1; + logic [TLB_ENTRIES-1:0] lu_hit; // to replacement logic + logic [TLB_ENTRIES-1:0] replace_en; // replace the following entry, set by replacement strategy + //------------- + // Translation + //------------- + always_comb begin : translation + vpn0 = lu_vaddr_i[21:12]; + vpn1 = lu_vaddr_i[31:22]; + + + // default assignment + lu_hit = '{default: 0}; + lu_hit_o = 1'b0; + lu_content_o = '{default: 0}; + lu_is_4M_o = 1'b0; + + for (int unsigned i = 0; i < TLB_ENTRIES; i++) begin + // first level match, this may be a mega page, check the ASID flags as well + // if the entry is associated to a global address, don't match the ASID (ASID is don't care) + if (tags_q[i].valid && ((lu_asid_i == tags_q[i].asid[ASID_WIDTH-1:0]) || content_q[i].g) && vpn1 == tags_q[i].vpn1) begin + if (tags_q[i].is_4M || vpn0 == tags_q[i].vpn0) begin + lu_is_4M_o = tags_q[i].is_4M; + lu_content_o = content_q[i]; + lu_hit_o = 1'b1; + lu_hit[i] = 1'b1; + end + end + end + end + + logic asid_to_be_flushed_is0; // indicates that the ASID provided by SFENCE.VMA (rs2)is 0, active high + logic vaddr_to_be_flushed_is0; // indicates that the VADDR provided by SFENCE.VMA (rs1)is 0, active high + logic [TLB_ENTRIES-1:0] vaddr_vpn0_match; + logic [TLB_ENTRIES-1:0] vaddr_vpn1_match; + + + assign asid_to_be_flushed_is0 = ~(|asid_to_be_flushed_i); + assign vaddr_to_be_flushed_is0 = ~(|vaddr_to_be_flushed_i); + + // ------------------ + // Update and Flush + // ------------------ + always_comb begin : update_flush + tags_n = tags_q; + content_n = content_q; + + for (int unsigned i = 0; i < TLB_ENTRIES; i++) begin + + vaddr_vpn0_match[i] = (vaddr_to_be_flushed_i[21:12] == tags_q[i].vpn0); + vaddr_vpn1_match[i] = (vaddr_to_be_flushed_i[31:22] == tags_q[i].vpn1); + + if (flush_i) begin + // invalidate logic + // flush everything if ASID is 0 and vaddr is 0 ("SFENCE.VMA x0 x0" case) + if (asid_to_be_flushed_is0 && vaddr_to_be_flushed_is0) tags_n[i].valid = 1'b0; + // flush vaddr in all addressing space ("SFENCE.VMA vaddr x0" case), it should happen only for leaf pages + else if (asid_to_be_flushed_is0 && ( (vaddr_vpn0_match[i] && vaddr_vpn1_match[i]) || (vaddr_vpn1_match[i] && tags_q[i].is_4M) ) && (~vaddr_to_be_flushed_is0)) + tags_n[i].valid = 1'b0; + // the entry is flushed if it's not global and asid and vaddr both matches with the entry to be flushed ("SFENCE.VMA vaddr asid" case) + else if ((!content_q[i].g) && ((vaddr_vpn0_match[i] && vaddr_vpn1_match[i]) || (vaddr_vpn1_match[i] && tags_q[i].is_4M)) && (asid_to_be_flushed_i == tags_q[i].asid[ASID_WIDTH-1:0]) && (!vaddr_to_be_flushed_is0) && (!asid_to_be_flushed_is0)) + tags_n[i].valid = 1'b0; + // the entry is flushed if it's not global, and the asid matches and vaddr is 0. ("SFENCE.VMA 0 asid" case) + else if ((!content_q[i].g) && (vaddr_to_be_flushed_is0) && (asid_to_be_flushed_i == tags_q[i].asid[ASID_WIDTH-1:0]) && (!asid_to_be_flushed_is0)) + tags_n[i].valid = 1'b0; + // normal replacement + end else if (update_i.valid & replace_en[i]) begin + // update tag array + tags_n[i] = '{ + asid: update_i.asid, + vpn1: update_i.vpn[19:10], + vpn0: update_i.vpn[9:0], + is_4M: update_i.is_4M, + valid: 1'b1 + }; + // and content as well + content_n[i] = update_i.content; + end + end + end + + // ----------------------------------------------- + // PLRU - Pseudo Least Recently Used Replacement + // ----------------------------------------------- + logic [2*(TLB_ENTRIES-1)-1:0] plru_tree_q, plru_tree_n; + logic en; + int unsigned idx_base, shift, new_index; + always_comb begin : plru_replacement + plru_tree_n = plru_tree_q; + en = '0; + idx_base = '0; + shift = '0; + new_index = '0; + // The PLRU-tree indexing: + // lvl0 0 + // / \ + // / \ + // lvl1 1 2 + // / \ / \ + // lvl2 3 4 5 6 + // / \ /\/\ /\ + // ... ... ... ... + // Just predefine which nodes will be set/cleared + // E.g. for a TLB with 8 entries, the for-loop is semantically + // equivalent to the following pseudo-code: + // unique case (1'b1) + // lu_hit[7]: plru_tree_n[0, 2, 6] = {1, 1, 1}; + // lu_hit[6]: plru_tree_n[0, 2, 6] = {1, 1, 0}; + // lu_hit[5]: plru_tree_n[0, 2, 5] = {1, 0, 1}; + // lu_hit[4]: plru_tree_n[0, 2, 5] = {1, 0, 0}; + // lu_hit[3]: plru_tree_n[0, 1, 4] = {0, 1, 1}; + // lu_hit[2]: plru_tree_n[0, 1, 4] = {0, 1, 0}; + // lu_hit[1]: plru_tree_n[0, 1, 3] = {0, 0, 1}; + // lu_hit[0]: plru_tree_n[0, 1, 3] = {0, 0, 0}; + // default: begin /* No hit */ end + // endcase + for ( + int unsigned i = 0; i < TLB_ENTRIES; i++ + ) begin + // we got a hit so update the pointer as it was least recently used + if (lu_hit[i] & lu_access_i) begin + // Set the nodes to the values we would expect + for (int unsigned lvl = 0; lvl < $clog2(TLB_ENTRIES); lvl++) begin + idx_base = $unsigned((2 ** lvl) - 1); + // lvl0 <=> MSB, lvl1 <=> MSB-1, ... + shift = $clog2(TLB_ENTRIES) - lvl; + // to circumvent the 32 bit integer arithmetic assignment + new_index = ~((i >> (shift - 1)) & 32'b1); + plru_tree_n[idx_base+(i>>shift)] = new_index[0]; + end + end + end + // Decode tree to write enable signals + // Next for-loop basically creates the following logic for e.g. an 8 entry + // TLB (note: pseudo-code obviously): + // replace_en[7] = &plru_tree_q[ 6, 2, 0]; //plru_tree_q[0,2,6]=={1,1,1} + // replace_en[6] = &plru_tree_q[~6, 2, 0]; //plru_tree_q[0,2,6]=={1,1,0} + // replace_en[5] = &plru_tree_q[ 5,~2, 0]; //plru_tree_q[0,2,5]=={1,0,1} + // replace_en[4] = &plru_tree_q[~5,~2, 0]; //plru_tree_q[0,2,5]=={1,0,0} + // replace_en[3] = &plru_tree_q[ 4, 1,~0]; //plru_tree_q[0,1,4]=={0,1,1} + // replace_en[2] = &plru_tree_q[~4, 1,~0]; //plru_tree_q[0,1,4]=={0,1,0} + // replace_en[1] = &plru_tree_q[ 3,~1,~0]; //plru_tree_q[0,1,3]=={0,0,1} + // replace_en[0] = &plru_tree_q[~3,~1,~0]; //plru_tree_q[0,1,3]=={0,0,0} + // For each entry traverse the tree. If every tree-node matches, + // the corresponding bit of the entry's index, this is + // the next entry to replace. + for (int unsigned i = 0; i < TLB_ENTRIES; i += 1) begin + en = 1'b1; + for (int unsigned lvl = 0; lvl < $clog2(TLB_ENTRIES); lvl++) begin + idx_base = $unsigned((2 ** lvl) - 1); + // lvl0 <=> MSB, lvl1 <=> MSB-1, ... + shift = $clog2(TLB_ENTRIES) - lvl; + + // en &= plru_tree_q[idx_base + (i>>shift)] == ((i >> (shift-1)) & 1'b1); + new_index = (i >> (shift - 1)) & 32'b1; + if (new_index[0]) begin + en &= plru_tree_q[idx_base+(i>>shift)]; + end else begin + en &= ~plru_tree_q[idx_base+(i>>shift)]; + end + end + replace_en[i] = en; + end + end + + // sequential process + always_ff @(posedge clk_i or negedge rst_ni) begin + if (~rst_ni) begin + tags_q <= '{default: 0}; + content_q <= '{default: 0}; + plru_tree_q <= '{default: 0}; + end else begin + tags_q <= tags_n; + content_q <= content_n; + plru_tree_q <= plru_tree_n; + end + end + //-------------- + // Sanity checks + //-------------- + + //pragma translate_off +`ifndef VERILATOR + + initial begin : p_assertions + assert ((TLB_ENTRIES % 2 == 0) && (TLB_ENTRIES > 1)) + else begin + $error("TLB size must be a multiple of 2 and greater than 1"); + $stop(); + end + assert (ASID_WIDTH >= 1) + else begin + $error("ASID width must be at least 1"); + $stop(); + end + end + + // Just for checking + function int countSetBits(logic [TLB_ENTRIES-1:0] vector); + automatic int count = 0; + foreach (vector[idx]) begin + count += vector[idx]; + end + return count; + endfunction + + assert property (@(posedge clk_i) (countSetBits(lu_hit) <= 1)) + else begin + $error("More then one hit in TLB!"); + $stop(); + end + assert property (@(posedge clk_i) (countSetBits(replace_en) <= 1)) + else begin + $error("More then one TLB entry selected for next replace!"); + $stop(); + end + +`endif + //pragma translate_on + +endmodule diff --git a/core/mmu_sv39/mmu.sv b/core/mmu_sv39/mmu.sv new file mode 100644 index 000000000..56994f260 --- /dev/null +++ b/core/mmu_sv39/mmu.sv @@ -0,0 +1,534 @@ +// Copyright 2018 ETH Zurich and University of Bologna. +// Copyright and related rights are licensed under the Solderpad Hardware +// License, Version 0.51 (the "License"); you may not use this file except in +// compliance with the License. You may obtain a copy of the License at +// http://solderpad.org/licenses/SHL-0.51. Unless required by applicable law +// or agreed to in writing, software, hardware and materials distributed under +// this License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. +// +// Author: Florian Zaruba, ETH Zurich +// Date: 19/04/2017 +// Description: Memory Management Unit for Ariane, contains TLB and +// address translation unit. SV39 as defined in RISC-V +// privilege specification 1.11-WIP + + +module mmu + import ariane_pkg::*; +#( + parameter config_pkg::cva6_cfg_t CVA6Cfg = config_pkg::cva6_cfg_empty, + parameter int unsigned INSTR_TLB_ENTRIES = 4, + parameter int unsigned DATA_TLB_ENTRIES = 4, + parameter int unsigned ASID_WIDTH = 1 +) ( + input logic clk_i, + input logic rst_ni, + input logic flush_i, + input logic enable_translation_i, + input logic en_ld_st_translation_i, // enable virtual memory translation for load/stores + // IF interface + input icache_arsp_t icache_areq_i, + output icache_areq_t icache_areq_o, + // LSU interface + // this is a more minimalistic interface because the actual addressing logic is handled + // in the LSU as we distinguish load and stores, what we do here is simple address translation + input exception_t misaligned_ex_i, + input logic lsu_req_i, // request address translation + input logic [riscv::VLEN-1:0] lsu_vaddr_i, // virtual address in + input logic lsu_is_store_i, // the translation is requested by a store + // if we need to walk the page table we can't grant in the same cycle + // Cycle 0 + output logic lsu_dtlb_hit_o, // sent in the same cycle as the request if translation hits in the DTLB + output logic [riscv::PPNW-1:0] lsu_dtlb_ppn_o, // ppn (send same cycle as hit) + // Cycle 1 + output logic lsu_valid_o, // translation is valid + output logic [riscv::PLEN-1:0] lsu_paddr_o, // translated address + output exception_t lsu_exception_o, // address translation threw an exception + // General control signals + input riscv::priv_lvl_t priv_lvl_i, + input riscv::priv_lvl_t ld_st_priv_lvl_i, + input logic sum_i, + input logic mxr_i, + // input logic flag_mprv_i, + input logic [riscv::PPNW-1:0] satp_ppn_i, + input logic [ASID_WIDTH-1:0] asid_i, + input logic [ASID_WIDTH-1:0] asid_to_be_flushed_i, + input logic [riscv::VLEN-1:0] vaddr_to_be_flushed_i, + input logic flush_tlb_i, + // Performance counters + output logic itlb_miss_o, + output logic dtlb_miss_o, + // PTW memory interface + input dcache_req_o_t req_port_i, + output dcache_req_i_t req_port_o, + // PMP + input riscv::pmpcfg_t [15:0] pmpcfg_i, + input logic [15:0][riscv::PLEN-3:0] pmpaddr_i +); + + logic iaccess_err; // insufficient privilege to access this instruction page + logic daccess_err; // insufficient privilege to access this data page + logic ptw_active; // PTW is currently walking a page table + logic walking_instr; // PTW is walking because of an ITLB miss + logic ptw_error; // PTW threw an exception + logic ptw_access_exception; // PTW threw an access exception (PMPs) + logic [riscv::PLEN-1:0] ptw_bad_paddr; // PTW PMP exception bad physical addr + + logic [riscv::VLEN-1:0] update_vaddr; + tlb_update_t update_ptw_itlb, update_ptw_dtlb; + + logic itlb_lu_access; + riscv::pte_t itlb_content; + logic itlb_is_2M; + logic itlb_is_1G; + logic itlb_lu_hit; + + logic dtlb_lu_access; + riscv::pte_t dtlb_content; + logic dtlb_is_2M; + logic dtlb_is_1G; + logic dtlb_lu_hit; + + + // Assignments + assign itlb_lu_access = icache_areq_i.fetch_req; + assign dtlb_lu_access = lsu_req_i; + + + tlb #( + .CVA6Cfg (CVA6Cfg), + .TLB_ENTRIES(INSTR_TLB_ENTRIES), + .ASID_WIDTH (ASID_WIDTH) + ) i_itlb ( + .clk_i (clk_i), + .rst_ni (rst_ni), + .flush_i(flush_tlb_i), + + .update_i(update_ptw_itlb), + + .lu_access_i (itlb_lu_access), + .lu_asid_i (asid_i), + .asid_to_be_flushed_i (asid_to_be_flushed_i), + .vaddr_to_be_flushed_i(vaddr_to_be_flushed_i), + .lu_vaddr_i (icache_areq_i.fetch_vaddr), + .lu_content_o (itlb_content), + + .lu_is_2M_o(itlb_is_2M), + .lu_is_1G_o(itlb_is_1G), + .lu_hit_o (itlb_lu_hit) + ); + + tlb #( + .CVA6Cfg (CVA6Cfg), + .TLB_ENTRIES(DATA_TLB_ENTRIES), + .ASID_WIDTH (ASID_WIDTH) + ) i_dtlb ( + .clk_i (clk_i), + .rst_ni (rst_ni), + .flush_i(flush_tlb_i), + + .update_i(update_ptw_dtlb), + + .lu_access_i (dtlb_lu_access), + .lu_asid_i (asid_i), + .asid_to_be_flushed_i (asid_to_be_flushed_i), + .vaddr_to_be_flushed_i(vaddr_to_be_flushed_i), + .lu_vaddr_i (lsu_vaddr_i), + .lu_content_o (dtlb_content), + + .lu_is_2M_o(dtlb_is_2M), + .lu_is_1G_o(dtlb_is_1G), + .lu_hit_o (dtlb_lu_hit) + ); + + + ptw #( + .CVA6Cfg (CVA6Cfg), + .ASID_WIDTH(ASID_WIDTH) + ) i_ptw ( + .clk_i (clk_i), + .rst_ni (rst_ni), + .ptw_active_o (ptw_active), + .walking_instr_o (walking_instr), + .ptw_error_o (ptw_error), + .ptw_access_exception_o(ptw_access_exception), + .enable_translation_i (enable_translation_i), + + .update_vaddr_o(update_vaddr), + .itlb_update_o (update_ptw_itlb), + .dtlb_update_o (update_ptw_dtlb), + + .itlb_access_i(itlb_lu_access), + .itlb_hit_i (itlb_lu_hit), + .itlb_vaddr_i (icache_areq_i.fetch_vaddr), + + .dtlb_access_i(dtlb_lu_access), + .dtlb_hit_i (dtlb_lu_hit), + .dtlb_vaddr_i (lsu_vaddr_i), + + .req_port_i (req_port_i), + .req_port_o (req_port_o), + .pmpcfg_i, + .pmpaddr_i, + .bad_paddr_o(ptw_bad_paddr), + .* + ); + + // ila_1 i_ila_1 ( + // .clk(clk_i), // input wire clk + // .probe0({req_port_o.address_tag, req_port_o.address_index}), + // .probe1(req_port_o.data_req), // input wire [63:0] probe1 + // .probe2(req_port_i.data_gnt), // input wire [0:0] probe2 + // .probe3(req_port_i.data_rdata), // input wire [0:0] probe3 + // .probe4(req_port_i.data_rvalid), // input wire [0:0] probe4 + // .probe5(ptw_error), // input wire [1:0] probe5 + // .probe6(update_vaddr), // input wire [0:0] probe6 + // .probe7(update_ptw_itlb.valid), // input wire [0:0] probe7 + // .probe8(update_ptw_dtlb.valid), // input wire [0:0] probe8 + // .probe9(dtlb_lu_access), // input wire [0:0] probe9 + // .probe10(lsu_vaddr_i), // input wire [0:0] probe10 + // .probe11(dtlb_lu_hit), // input wire [0:0] probe11 + // .probe12(itlb_lu_access), // input wire [0:0] probe12 + // .probe13(icache_areq_i.fetch_vaddr), // input wire [0:0] probe13 + // .probe14(itlb_lu_hit) // input wire [0:0] probe13 + // ); + + //----------------------- + // Instruction Interface + //----------------------- + logic match_any_execute_region; + logic pmp_instr_allow; + + // The instruction interface is a simple request response interface + always_comb begin : instr_interface + // MMU disabled: just pass through + icache_areq_o.fetch_valid = icache_areq_i.fetch_req; + icache_areq_o.fetch_paddr = icache_areq_i.fetch_vaddr[riscv::PLEN-1:0]; // play through in case we disabled address translation + // two potential exception sources: + // 1. HPTW threw an exception -> signal with a page fault exception + // 2. We got an access error because of insufficient permissions -> throw an access exception + icache_areq_o.fetch_exception = '0; + // Check whether we are allowed to access this memory region from a fetch perspective + iaccess_err = icache_areq_i.fetch_req && enable_translation_i + && (((priv_lvl_i == riscv::PRIV_LVL_U) && ~itlb_content.u) + || ((priv_lvl_i == riscv::PRIV_LVL_S) && itlb_content.u)); + + // MMU enabled: address from TLB, request delayed until hit. Error when TLB + // hit and no access right or TLB hit and translated address not valid (e.g. + // AXI decode error), or when PTW performs walk due to ITLB miss and raises + // an error. + if (enable_translation_i) begin + // we work with SV39 or SV32, so if VM is enabled, check that all bits [riscv::VLEN-1:riscv::SV-1] are equal + if (icache_areq_i.fetch_req && !((&icache_areq_i.fetch_vaddr[riscv::VLEN-1:riscv::SV-1]) == 1'b1 || (|icache_areq_i.fetch_vaddr[riscv::VLEN-1:riscv::SV-1]) == 1'b0)) begin + icache_areq_o.fetch_exception.cause = riscv::INSTR_ACCESS_FAULT; + icache_areq_o.fetch_exception.valid = 1'b1; + if (CVA6Cfg.TvalEn) + icache_areq_o.fetch_exception.tval = { + {riscv::XLEN - riscv::VLEN{1'b0}}, icache_areq_i.fetch_vaddr + }; + end + + icache_areq_o.fetch_valid = 1'b0; + + // 4K page + icache_areq_o.fetch_paddr = {itlb_content.ppn, icache_areq_i.fetch_vaddr[11:0]}; + // Mega page + if (itlb_is_2M) begin + icache_areq_o.fetch_paddr[20:12] = icache_areq_i.fetch_vaddr[20:12]; + end + // Giga page + if (itlb_is_1G) begin + icache_areq_o.fetch_paddr[29:12] = icache_areq_i.fetch_vaddr[29:12]; + end + + // --------- + // ITLB Hit + // -------- + // if we hit the ITLB output the request signal immediately + if (itlb_lu_hit) begin + icache_areq_o.fetch_valid = icache_areq_i.fetch_req; + // we got an access error + if (iaccess_err) begin + // throw a page fault + icache_areq_o.fetch_exception.cause = riscv::INSTR_PAGE_FAULT; + icache_areq_o.fetch_exception.valid = 1'b1; + if (CVA6Cfg.TvalEn) + icache_areq_o.fetch_exception.tval = { + {riscv::XLEN - riscv::VLEN{1'b0}}, icache_areq_i.fetch_vaddr + }; + end else if (!pmp_instr_allow) begin + icache_areq_o.fetch_exception.cause = riscv::INSTR_ACCESS_FAULT; + icache_areq_o.fetch_exception.valid = 1'b1; + if (CVA6Cfg.TvalEn) + icache_areq_o.fetch_exception.tval = { + {riscv::XLEN - riscv::VLEN{1'b0}}, icache_areq_i.fetch_vaddr + }; + end + end else + // --------- + // ITLB Miss + // --------- + // watch out for exceptions happening during walking the page table + if (ptw_active && walking_instr) begin + icache_areq_o.fetch_valid = ptw_error | ptw_access_exception; + if (ptw_error) begin + icache_areq_o.fetch_exception.cause = riscv::INSTR_PAGE_FAULT; + icache_areq_o.fetch_exception.valid = 1'b1; + if (CVA6Cfg.TvalEn) + icache_areq_o.fetch_exception.tval = {{riscv::XLEN - riscv::VLEN{1'b0}}, update_vaddr}; + end else begin + icache_areq_o.fetch_exception.cause = riscv::INSTR_ACCESS_FAULT; + icache_areq_o.fetch_exception.valid = 1'b1; + if (CVA6Cfg.TvalEn) + icache_areq_o.fetch_exception.tval = {{riscv::XLEN - riscv::VLEN{1'b0}}, update_vaddr}; + end + end + end + // if it didn't match any execute region throw an `Instruction Access Fault` + // or: if we are not translating, check PMPs immediately on the paddr + if ((!match_any_execute_region && !ptw_error) || (!enable_translation_i && !pmp_instr_allow)) begin + icache_areq_o.fetch_exception.cause = riscv::INSTR_ACCESS_FAULT; + icache_areq_o.fetch_exception.valid = 1'b1; + if (CVA6Cfg.TvalEn) + icache_areq_o.fetch_exception.tval = { + {riscv::XLEN - riscv::PLEN{1'b0}}, icache_areq_o.fetch_paddr + }; + end + end + + // check for execute flag on memory + assign match_any_execute_region = config_pkg::is_inside_execute_regions( + CVA6Cfg, {{64 - riscv::PLEN{1'b0}}, icache_areq_o.fetch_paddr} + ); + + // Instruction fetch + pmp #( + .CVA6Cfg (CVA6Cfg), + .PLEN (riscv::PLEN), + .PMP_LEN (riscv::PLEN - 2), + .NR_ENTRIES(CVA6Cfg.NrPMPEntries) + ) i_pmp_if ( + .addr_i (icache_areq_o.fetch_paddr), + .priv_lvl_i, + // we will always execute on the instruction fetch port + .access_type_i(riscv::ACCESS_EXEC), + // Configuration + .conf_addr_i (pmpaddr_i), + .conf_i (pmpcfg_i), + .allow_o (pmp_instr_allow) + ); + + //----------------------- + // Data Interface + //----------------------- + logic [riscv::VLEN-1:0] lsu_vaddr_n, lsu_vaddr_q; + riscv::pte_t dtlb_pte_n, dtlb_pte_q; + exception_t misaligned_ex_n, misaligned_ex_q; + logic lsu_req_n, lsu_req_q; + logic lsu_is_store_n, lsu_is_store_q; + logic dtlb_hit_n, dtlb_hit_q; + logic dtlb_is_2M_n, dtlb_is_2M_q; + logic dtlb_is_1G_n, dtlb_is_1G_q; + + // check if we need to do translation or if we are always ready (e.g.: we are not translating anything) + assign lsu_dtlb_hit_o = (en_ld_st_translation_i) ? dtlb_lu_hit : 1'b1; + + // Wires to PMP checks + riscv::pmp_access_t pmp_access_type; + logic pmp_data_allow; + localparam PPNWMin = (riscv::PPNW - 1 > 29) ? 29 : riscv::PPNW - 1; + // The data interface is simpler and only consists of a request/response interface + always_comb begin : data_interface + // save request and DTLB response + lsu_vaddr_n = lsu_vaddr_i; + lsu_req_n = lsu_req_i; + misaligned_ex_n = misaligned_ex_i; + dtlb_pte_n = dtlb_content; + dtlb_hit_n = dtlb_lu_hit; + lsu_is_store_n = lsu_is_store_i; + dtlb_is_2M_n = dtlb_is_2M; + dtlb_is_1G_n = dtlb_is_1G; + + lsu_paddr_o = lsu_vaddr_q[riscv::PLEN-1:0]; + lsu_dtlb_ppn_o = lsu_vaddr_n[riscv::PLEN-1:12]; + lsu_valid_o = lsu_req_q; + lsu_exception_o = misaligned_ex_q; + pmp_access_type = lsu_is_store_q ? riscv::ACCESS_WRITE : riscv::ACCESS_READ; + + // mute misaligned exceptions if there is no request otherwise they will throw accidental exceptions + misaligned_ex_n.valid = misaligned_ex_i.valid & lsu_req_i; + + // Check if the User flag is set, then we may only access it in supervisor mode + // if SUM is enabled + daccess_err = en_ld_st_translation_i && ((ld_st_priv_lvl_i == riscv::PRIV_LVL_S && !sum_i && dtlb_pte_q.u) || // SUM is not set and we are trying to access a user page in supervisor mode + (ld_st_priv_lvl_i == riscv::PRIV_LVL_U && !dtlb_pte_q.u)); // this is not a user page but we are in user mode and trying to access it + // translation is enabled and no misaligned exception occurred + if (en_ld_st_translation_i && !misaligned_ex_q.valid) begin + lsu_valid_o = 1'b0; + // 4K page + lsu_paddr_o = {dtlb_pte_q.ppn, lsu_vaddr_q[11:0]}; + lsu_dtlb_ppn_o = dtlb_content.ppn; + // Mega page + if (dtlb_is_2M_q) begin + lsu_paddr_o[20:12] = lsu_vaddr_q[20:12]; + lsu_dtlb_ppn_o[20:12] = lsu_vaddr_n[20:12]; + end + // Giga page + if (dtlb_is_1G_q) begin + lsu_paddr_o[PPNWMin:12] = lsu_vaddr_q[PPNWMin:12]; + lsu_dtlb_ppn_o[PPNWMin:12] = lsu_vaddr_n[PPNWMin:12]; + end + // --------- + // DTLB Hit + // -------- + if (dtlb_hit_q && lsu_req_q) begin + lsu_valid_o = 1'b1; + // exception priority: + // PAGE_FAULTS have higher priority than ACCESS_FAULTS + // virtual memory based exceptions are PAGE_FAULTS + // physical memory based exceptions are ACCESS_FAULTS (PMA/PMP) + + // this is a store + if (lsu_is_store_q) begin + // check if the page is write-able and we are not violating privileges + // also check if the dirty flag is set + if (!dtlb_pte_q.w || daccess_err || !dtlb_pte_q.d) begin + lsu_exception_o.cause = riscv::STORE_PAGE_FAULT; + lsu_exception_o.valid = 1'b1; + if (CVA6Cfg.TvalEn) + lsu_exception_o.tval = { + {riscv::XLEN - riscv::VLEN{lsu_vaddr_q[riscv::VLEN-1]}}, lsu_vaddr_q + }; + // Check if any PMPs are violated + end else if (!pmp_data_allow) begin + lsu_exception_o.cause = riscv::ST_ACCESS_FAULT; + lsu_exception_o.valid = 1'b1; + if (CVA6Cfg.TvalEn) + lsu_exception_o.tval = { + {riscv::XLEN - riscv::VLEN{lsu_vaddr_q[riscv::VLEN-1]}}, lsu_vaddr_q + }; + end + + // this is a load + end else begin + // check for sufficient access privileges - throw a page fault if necessary + if (daccess_err) begin + lsu_exception_o.cause = riscv::LOAD_PAGE_FAULT; + lsu_exception_o.valid = 1'b1; + if (CVA6Cfg.TvalEn) + lsu_exception_o.tval = { + {riscv::XLEN - riscv::VLEN{lsu_vaddr_q[riscv::VLEN-1]}}, lsu_vaddr_q + }; + // Check if any PMPs are violated + end else if (!pmp_data_allow) begin + lsu_exception_o.cause = riscv::LD_ACCESS_FAULT; + lsu_exception_o.valid = 1'b1; + if (CVA6Cfg.TvalEn) + lsu_exception_o.tval = { + {riscv::XLEN - riscv::VLEN{lsu_vaddr_q[riscv::VLEN-1]}}, lsu_vaddr_q + }; + end + end + end else + + // --------- + // DTLB Miss + // --------- + // watch out for exceptions + if (ptw_active && !walking_instr) begin + // page table walker threw an exception + if (ptw_error) begin + // an error makes the translation valid + lsu_valid_o = 1'b1; + // the page table walker can only throw page faults + if (lsu_is_store_q) begin + lsu_exception_o.cause = riscv::STORE_PAGE_FAULT; + lsu_exception_o.valid = 1'b1; + if (CVA6Cfg.TvalEn) + lsu_exception_o.tval = { + {riscv::XLEN - riscv::VLEN{lsu_vaddr_q[riscv::VLEN-1]}}, update_vaddr + }; + end else begin + lsu_exception_o.cause = riscv::LOAD_PAGE_FAULT; + lsu_exception_o.valid = 1'b1; + if (CVA6Cfg.TvalEn) + lsu_exception_o.tval = { + {riscv::XLEN - riscv::VLEN{lsu_vaddr_q[riscv::VLEN-1]}}, update_vaddr + }; + end + end + + if (ptw_access_exception) begin + // an error makes the translation valid + lsu_valid_o = 1'b1; + // Any fault of the page table walk should be based of the original access type + if (lsu_is_store_q) begin + lsu_exception_o.cause = riscv::ST_ACCESS_FAULT; + lsu_exception_o.valid = 1'b1; + if (CVA6Cfg.TvalEn) + lsu_exception_o.tval = {{riscv::XLEN - riscv::VLEN{1'b0}}, lsu_vaddr_n}; + end else begin + lsu_exception_o.cause = riscv::LD_ACCESS_FAULT; + lsu_exception_o.valid = 1'b1; + if (CVA6Cfg.TvalEn) + lsu_exception_o.tval = {{riscv::XLEN - riscv::VLEN{1'b0}}, lsu_vaddr_n}; + end + end + end + end // If translation is not enabled, check the paddr immediately against PMPs + else if (lsu_req_q && !misaligned_ex_q.valid && !pmp_data_allow) begin + if (lsu_is_store_q) begin + lsu_exception_o.cause = riscv::ST_ACCESS_FAULT; + lsu_exception_o.valid = 1'b1; + if (CVA6Cfg.TvalEn) lsu_exception_o.tval = {{riscv::XLEN - riscv::PLEN{1'b0}}, lsu_paddr_o}; + end else begin + lsu_exception_o.cause = riscv::LD_ACCESS_FAULT; + lsu_exception_o.valid = 1'b1; + if (CVA6Cfg.TvalEn) lsu_exception_o.tval = {{riscv::XLEN - riscv::PLEN{1'b0}}, lsu_paddr_o}; + end + end + end + + // Load/store PMP check + pmp #( + .CVA6Cfg (CVA6Cfg), + .PLEN (riscv::PLEN), + .PMP_LEN (riscv::PLEN - 2), + .NR_ENTRIES(CVA6Cfg.NrPMPEntries) + ) i_pmp_data ( + .addr_i (lsu_paddr_o), + .priv_lvl_i (ld_st_priv_lvl_i), + .access_type_i(pmp_access_type), + // Configuration + .conf_addr_i (pmpaddr_i), + .conf_i (pmpcfg_i), + .allow_o (pmp_data_allow) + ); + + // ---------- + // Registers + // ---------- + always_ff @(posedge clk_i or negedge rst_ni) begin + if (~rst_ni) begin + lsu_vaddr_q <= '0; + lsu_req_q <= '0; + misaligned_ex_q <= '0; + dtlb_pte_q <= '0; + dtlb_hit_q <= '0; + lsu_is_store_q <= '0; + dtlb_is_2M_q <= '0; + dtlb_is_1G_q <= '0; + end else begin + lsu_vaddr_q <= lsu_vaddr_n; + lsu_req_q <= lsu_req_n; + misaligned_ex_q <= misaligned_ex_n; + dtlb_pte_q <= dtlb_pte_n; + dtlb_hit_q <= dtlb_hit_n; + lsu_is_store_q <= lsu_is_store_n; + dtlb_is_2M_q <= dtlb_is_2M_n; + dtlb_is_1G_q <= dtlb_is_1G_n; + end + end +endmodule diff --git a/core/mmu_sv39/ptw.sv b/core/mmu_sv39/ptw.sv new file mode 100644 index 000000000..2d0e3780a --- /dev/null +++ b/core/mmu_sv39/ptw.sv @@ -0,0 +1,409 @@ +// Copyright 2018 ETH Zurich and University of Bologna. +// Copyright and related rights are licensed under the Solderpad Hardware +// License, Version 0.51 (the "License"); you may not use this file except in +// compliance with the License. You may obtain a copy of the License at +// http://solderpad.org/licenses/SHL-0.51. Unless required by applicable law +// or agreed to in writing, software, hardware and materials distributed under +// this License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. +// +// Author: David Schaffenrath, TU Graz +// Author: Florian Zaruba, ETH Zurich +// Date: 24.4.2017 +// Description: Hardware-PTW + +/* verilator lint_off WIDTH */ + +module ptw + import ariane_pkg::*; +#( + parameter config_pkg::cva6_cfg_t CVA6Cfg = config_pkg::cva6_cfg_empty, + parameter int ASID_WIDTH = 1 +) ( + input logic clk_i, // Clock + input logic rst_ni, // Asynchronous reset active low + input logic flush_i, // flush everything, we need to do this because + // actually everything we do is speculative at this stage + // e.g.: there could be a CSR instruction that changes everything + output logic ptw_active_o, + output logic walking_instr_o, // set when walking for TLB + output logic ptw_error_o, // set when an error occurred + output logic ptw_access_exception_o, // set when an PMP access exception occured + input logic enable_translation_i, // CSRs indicate to enable SV39 + input logic en_ld_st_translation_i, // enable virtual memory translation for load/stores + + input logic lsu_is_store_i, // this translation was triggered by a store + // PTW memory interface + input dcache_req_o_t req_port_i, + output dcache_req_i_t req_port_o, + + + // to TLBs, update logic + output tlb_update_t itlb_update_o, + output tlb_update_t dtlb_update_o, + + output logic [riscv::VLEN-1:0] update_vaddr_o, + + input logic [ ASID_WIDTH-1:0] asid_i, + // from TLBs + // did we miss? + input logic itlb_access_i, + input logic itlb_hit_i, + input logic [riscv::VLEN-1:0] itlb_vaddr_i, + + input logic dtlb_access_i, + input logic dtlb_hit_i, + input logic [riscv::VLEN-1:0] dtlb_vaddr_i, + // from CSR file + input logic [riscv::PPNW-1:0] satp_ppn_i, // ppn from satp + input logic mxr_i, + // Performance counters + output logic itlb_miss_o, + output logic dtlb_miss_o, + // PMP + + input riscv::pmpcfg_t [15:0] pmpcfg_i, + input logic [15:0][riscv::PLEN-3:0] pmpaddr_i, + output logic [riscv::PLEN-1:0] bad_paddr_o + +); + + // input registers + logic data_rvalid_q; + logic [63:0] data_rdata_q; + + riscv::pte_t pte; + assign pte = riscv::pte_t'(data_rdata_q); + + enum logic [2:0] { + IDLE, + WAIT_GRANT, + PTE_LOOKUP, + WAIT_RVALID, + PROPAGATE_ERROR, + PROPAGATE_ACCESS_ERROR + } + state_q, state_d; + + // SV39 defines three levels of page tables + enum logic [1:0] { + LVL1, + LVL2, + LVL3 + } + ptw_lvl_q, ptw_lvl_n; + + // is this an instruction page table walk? + logic is_instr_ptw_q, is_instr_ptw_n; + logic global_mapping_q, global_mapping_n; + // latched tag signal + logic tag_valid_n, tag_valid_q; + // register the ASID + logic [ASID_WIDTH-1:0] tlb_update_asid_q, tlb_update_asid_n; + // register the VPN we need to walk, SV39 defines a 39 bit virtual address + logic [riscv::VLEN-1:0] vaddr_q, vaddr_n; + // 4 byte aligned physical pointer + logic [riscv::PLEN-1:0] ptw_pptr_q, ptw_pptr_n; + + // Assignments + assign update_vaddr_o = vaddr_q; + + assign ptw_active_o = (state_q != IDLE); + assign walking_instr_o = is_instr_ptw_q; + // directly output the correct physical address + assign req_port_o.address_index = ptw_pptr_q[DCACHE_INDEX_WIDTH-1:0]; + assign req_port_o.address_tag = ptw_pptr_q[DCACHE_INDEX_WIDTH+DCACHE_TAG_WIDTH-1:DCACHE_INDEX_WIDTH]; + // we are never going to kill this request + assign req_port_o.kill_req = '0; + // we are never going to write with the HPTW + assign req_port_o.data_wdata = 64'b0; + // we only issue one single request at a time + assign req_port_o.data_id = '0; + // ----------- + // TLB Update + // ----------- + assign itlb_update_o.vpn = {{39 - riscv::SV{1'b0}}, vaddr_q[riscv::SV-1:12]}; + assign dtlb_update_o.vpn = {{39 - riscv::SV{1'b0}}, vaddr_q[riscv::SV-1:12]}; + // update the correct page table level + assign itlb_update_o.is_2M = (ptw_lvl_q == LVL2); + assign itlb_update_o.is_1G = (ptw_lvl_q == LVL1); + assign dtlb_update_o.is_2M = (ptw_lvl_q == LVL2); + assign dtlb_update_o.is_1G = (ptw_lvl_q == LVL1); + // output the correct ASID + assign itlb_update_o.asid = tlb_update_asid_q; + assign dtlb_update_o.asid = tlb_update_asid_q; + // set the global mapping bit + assign itlb_update_o.content = pte | (global_mapping_q << 5); + assign dtlb_update_o.content = pte | (global_mapping_q << 5); + + assign req_port_o.tag_valid = tag_valid_q; + + logic allow_access; + + assign bad_paddr_o = ptw_access_exception_o ? ptw_pptr_q : 'b0; + + pmp #( + .CVA6Cfg (CVA6Cfg), + .PLEN (riscv::PLEN), + .PMP_LEN (riscv::PLEN - 2), + .NR_ENTRIES(CVA6Cfg.NrPMPEntries) + ) i_pmp_ptw ( + .addr_i (ptw_pptr_q), + // PTW access are always checked as if in S-Mode... + .priv_lvl_i (riscv::PRIV_LVL_S), + // ...and they are always loads + .access_type_i(riscv::ACCESS_READ), + // Configuration + .conf_addr_i (pmpaddr_i), + .conf_i (pmpcfg_i), + .allow_o (allow_access) + ); + + //------------------- + // Page table walker + //------------------- + // A virtual address va is translated into a physical address pa as follows: + // 1. Let a be sptbr.ppn × PAGESIZE, and let i = LEVELS-1. (For Sv39, + // PAGESIZE=2^12 and LEVELS=3.) + // 2. Let pte be the value of the PTE at address a+va.vpn[i]×PTESIZE. (For + // Sv32, PTESIZE=4.) + // 3. If pte.v = 0, or if pte.r = 0 and pte.w = 1, stop and raise an access + // exception. + // 4. Otherwise, the PTE is valid. If pte.r = 1 or pte.x = 1, go to step 5. + // Otherwise, this PTE is a pointer to the next level of the page table. + // Let i=i-1. If i < 0, stop and raise an access exception. Otherwise, let + // a = pte.ppn × PAGESIZE and go to step 2. + // 5. A leaf PTE has been found. Determine if the requested memory access + // is allowed by the pte.r, pte.w, and pte.x bits. If not, stop and + // raise an access exception. Otherwise, the translation is successful. + // Set pte.a to 1, and, if the memory access is a store, set pte.d to 1. + // The translated physical address is given as follows: + // - pa.pgoff = va.pgoff. + // - If i > 0, then this is a superpage translation and + // pa.ppn[i-1:0] = va.vpn[i-1:0]. + // - pa.ppn[LEVELS-1:i] = pte.ppn[LEVELS-1:i]. + always_comb begin : ptw + // default assignments + // PTW memory interface + tag_valid_n = 1'b0; + req_port_o.data_req = 1'b0; + req_port_o.data_be = 8'hFF; + req_port_o.data_size = 2'b11; + req_port_o.data_we = 1'b0; + ptw_error_o = 1'b0; + ptw_access_exception_o = 1'b0; + itlb_update_o.valid = 1'b0; + dtlb_update_o.valid = 1'b0; + is_instr_ptw_n = is_instr_ptw_q; + ptw_lvl_n = ptw_lvl_q; + ptw_pptr_n = ptw_pptr_q; + state_d = state_q; + global_mapping_n = global_mapping_q; + // input registers + tlb_update_asid_n = tlb_update_asid_q; + vaddr_n = vaddr_q; + + itlb_miss_o = 1'b0; + dtlb_miss_o = 1'b0; + + case (state_q) + + IDLE: begin + // by default we start with the top-most page table + ptw_lvl_n = LVL1; + global_mapping_n = 1'b0; + is_instr_ptw_n = 1'b0; + // if we got an ITLB miss + if (enable_translation_i & itlb_access_i & ~itlb_hit_i & ~dtlb_access_i) begin + ptw_pptr_n = {satp_ppn_i, itlb_vaddr_i[riscv::SV-1:30], 3'b0}; + is_instr_ptw_n = 1'b1; + tlb_update_asid_n = asid_i; + vaddr_n = itlb_vaddr_i; + state_d = WAIT_GRANT; + itlb_miss_o = 1'b1; + // we got an DTLB miss + end else if (en_ld_st_translation_i & dtlb_access_i & ~dtlb_hit_i) begin + ptw_pptr_n = {satp_ppn_i, dtlb_vaddr_i[riscv::SV-1:30], 3'b0}; + tlb_update_asid_n = asid_i; + vaddr_n = dtlb_vaddr_i; + state_d = WAIT_GRANT; + dtlb_miss_o = 1'b1; + end + end + + WAIT_GRANT: begin + // send a request out + req_port_o.data_req = 1'b1; + // wait for the WAIT_GRANT + if (req_port_i.data_gnt) begin + // send the tag valid signal one cycle later + tag_valid_n = 1'b1; + state_d = PTE_LOOKUP; + end + end + + PTE_LOOKUP: begin + // we wait for the valid signal + if (data_rvalid_q) begin + + // check if the global mapping bit is set + if (pte.g) global_mapping_n = 1'b1; + + // ------------- + // Invalid PTE + // ------------- + // If pte.v = 0, or if pte.r = 0 and pte.w = 1, stop and raise a page-fault exception. + if (!pte.v || (!pte.r && pte.w)) state_d = PROPAGATE_ERROR; + // ----------- + // Valid PTE + // ----------- + else begin + state_d = IDLE; + // it is a valid PTE + // if pte.r = 1 or pte.x = 1 it is a valid PTE + if (pte.r || pte.x) begin + // Valid translation found (either 1G, 2M or 4K entry) + if (is_instr_ptw_q) begin + // ------------ + // Update ITLB + // ------------ + // If page is not executable, we can directly raise an error. This + // doesn't put a useless entry into the TLB. The same idea applies + // to the access flag since we let the access flag be managed by SW. + if (!pte.x || !pte.a) state_d = PROPAGATE_ERROR; + else itlb_update_o.valid = 1'b1; + + end else begin + // ------------ + // Update DTLB + // ------------ + // Check if the access flag has been set, otherwise throw a page-fault + // and let the software handle those bits. + // If page is not readable (there are no write-only pages) + // we can directly raise an error. This doesn't put a useless + // entry into the TLB. + if (pte.a && (pte.r || (pte.x && mxr_i))) begin + dtlb_update_o.valid = 1'b1; + end else begin + state_d = PROPAGATE_ERROR; + end + // Request is a store: perform some additional checks + // If the request was a store and the page is not write-able, raise an error + // the same applies if the dirty flag is not set + if (lsu_is_store_i && (!pte.w || !pte.d)) begin + dtlb_update_o.valid = 1'b0; + state_d = PROPAGATE_ERROR; + end + end + // check if the ppn is correctly aligned: + // 6. If i > 0 and pa.ppn[i − 1 : 0] != 0, this is a misaligned superpage; stop and raise a page-fault + // exception. + if (ptw_lvl_q == LVL1 && pte.ppn[17:0] != '0) begin + state_d = PROPAGATE_ERROR; + dtlb_update_o.valid = 1'b0; + itlb_update_o.valid = 1'b0; + end else if (ptw_lvl_q == LVL2 && pte.ppn[8:0] != '0) begin + state_d = PROPAGATE_ERROR; + dtlb_update_o.valid = 1'b0; + itlb_update_o.valid = 1'b0; + end + // this is a pointer to the next TLB level + end else begin + // pointer to next level of page table + if (ptw_lvl_q == LVL1) begin + // we are in the second level now + ptw_lvl_n = LVL2; + ptw_pptr_n = {pte.ppn, vaddr_q[29:21], 3'b0}; + end + + if (ptw_lvl_q == LVL2) begin + // here we received a pointer to the third level + ptw_lvl_n = LVL3; + ptw_pptr_n = {pte.ppn, vaddr_q[20:12], 3'b0}; + end + + state_d = WAIT_GRANT; + + if (ptw_lvl_q == LVL3) begin + // Should already be the last level page table => Error + ptw_lvl_n = LVL3; + state_d = PROPAGATE_ERROR; + end + end + end + + // Check if this access was actually allowed from a PMP perspective + if (!allow_access) begin + itlb_update_o.valid = 1'b0; + dtlb_update_o.valid = 1'b0; + // we have to return the failed address in bad_addr + ptw_pptr_n = ptw_pptr_q; + state_d = PROPAGATE_ACCESS_ERROR; + end + end + // we've got a data WAIT_GRANT so tell the cache that the tag is valid + end + // Propagate error to MMU/LSU + PROPAGATE_ERROR: begin + state_d = IDLE; + ptw_error_o = 1'b1; + end + PROPAGATE_ACCESS_ERROR: begin + state_d = IDLE; + ptw_access_exception_o = 1'b1; + end + // wait for the rvalid before going back to IDLE + WAIT_RVALID: begin + if (data_rvalid_q) state_d = IDLE; + end + default: begin + state_d = IDLE; + end + endcase + + // ------- + // Flush + // ------- + // should we have flushed before we got an rvalid, wait for it until going back to IDLE + if (flush_i) begin + // on a flush check whether we are + // 1. in the PTE Lookup check whether we still need to wait for an rvalid + // 2. waiting for a grant, if so: wait for it + // if not, go back to idle + if (((state_q inside {PTE_LOOKUP, WAIT_RVALID}) && !data_rvalid_q) || + ((state_q == WAIT_GRANT) && req_port_i.data_gnt)) + state_d = WAIT_RVALID; + else state_d = IDLE; + end + end + + // sequential process + always_ff @(posedge clk_i or negedge rst_ni) begin + if (~rst_ni) begin + state_q <= IDLE; + is_instr_ptw_q <= 1'b0; + ptw_lvl_q <= LVL1; + tag_valid_q <= 1'b0; + tlb_update_asid_q <= '0; + vaddr_q <= '0; + ptw_pptr_q <= '0; + global_mapping_q <= 1'b0; + data_rdata_q <= '0; + data_rvalid_q <= 1'b0; + end else begin + state_q <= state_d; + ptw_pptr_q <= ptw_pptr_n; + is_instr_ptw_q <= is_instr_ptw_n; + ptw_lvl_q <= ptw_lvl_n; + tag_valid_q <= tag_valid_n; + tlb_update_asid_q <= tlb_update_asid_n; + vaddr_q <= vaddr_n; + global_mapping_q <= global_mapping_n; + data_rdata_q <= req_port_i.data_rdata; + data_rvalid_q <= req_port_i.data_rvalid; + end + end + +endmodule +/* verilator lint_on WIDTH */ diff --git a/core/mmu_sv39/tlb.sv b/core/mmu_sv39/tlb.sv new file mode 100644 index 000000000..3df2cb017 --- /dev/null +++ b/core/mmu_sv39/tlb.sv @@ -0,0 +1,290 @@ +// Copyright 2018 ETH Zurich and University of Bologna. +// Copyright and related rights are licensed under the Solderpad Hardware +// License, Version 0.51 (the "License"); you may not use this file except in +// compliance with the License. You may obtain a copy of the License at +// http://solderpad.org/licenses/SHL-0.51. Unless required by applicable law +// or agreed to in writing, software, hardware and materials distributed under +// this License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. +// +// Author: David Schaffenrath, TU Graz +// Author: Florian Zaruba, ETH Zurich +// Date: 21.4.2017 +// Description: Translation Lookaside Buffer, SV39 +// fully set-associative + + +module tlb + import ariane_pkg::*; +#( + parameter config_pkg::cva6_cfg_t CVA6Cfg = config_pkg::cva6_cfg_empty, + parameter int unsigned TLB_ENTRIES = 4, + parameter int unsigned ASID_WIDTH = 1 +) ( + input logic clk_i, // Clock + input logic rst_ni, // Asynchronous reset active low + input logic flush_i, // Flush signal + // Update TLB + input tlb_update_t update_i, + // Lookup signals + input logic lu_access_i, + input logic [ ASID_WIDTH-1:0] lu_asid_i, + input logic [riscv::VLEN-1:0] lu_vaddr_i, + output riscv::pte_t lu_content_o, + input logic [ ASID_WIDTH-1:0] asid_to_be_flushed_i, + input logic [riscv::VLEN-1:0] vaddr_to_be_flushed_i, + output logic lu_is_2M_o, + output logic lu_is_1G_o, + output logic lu_hit_o +); + + // SV39 defines three levels of page tables + struct packed { + logic [ASID_WIDTH-1:0] asid; + logic [riscv::VPN2:0] vpn2; + logic [8:0] vpn1; + logic [8:0] vpn0; + logic is_2M; + logic is_1G; + logic valid; + } [TLB_ENTRIES-1:0] + tags_q, tags_n; + + riscv::pte_t [TLB_ENTRIES-1:0] content_q, content_n; + logic [8:0] vpn0, vpn1; + logic [ riscv::VPN2:0] vpn2; + logic [TLB_ENTRIES-1:0] lu_hit; // to replacement logic + logic [TLB_ENTRIES-1:0] replace_en; // replace the following entry, set by replacement strategy + //------------- + // Translation + //------------- + always_comb begin : translation + vpn0 = lu_vaddr_i[20:12]; + vpn1 = lu_vaddr_i[29:21]; + vpn2 = lu_vaddr_i[30+riscv::VPN2:30]; + + // default assignment + lu_hit = '{default: 0}; + lu_hit_o = 1'b0; + lu_content_o = '{default: 0}; + lu_is_1G_o = 1'b0; + lu_is_2M_o = 1'b0; + + for (int unsigned i = 0; i < TLB_ENTRIES; i++) begin + // first level match, this may be a giga page, check the ASID flags as well + // if the entry is associated to a global address, don't match the ASID (ASID is don't care) + if (tags_q[i].valid && ((lu_asid_i == tags_q[i].asid) || content_q[i].g) && vpn2 == tags_q[i].vpn2) begin + // second level + if (tags_q[i].is_1G) begin + lu_is_1G_o = 1'b1; + lu_content_o = content_q[i]; + lu_hit_o = 1'b1; + lu_hit[i] = 1'b1; + // not a giga page hit so check further + end else if (vpn1 == tags_q[i].vpn1) begin + // this could be a 2 mega page hit or a 4 kB hit + // output accordingly + if (tags_q[i].is_2M || vpn0 == tags_q[i].vpn0) begin + lu_is_2M_o = tags_q[i].is_2M; + lu_content_o = content_q[i]; + lu_hit_o = 1'b1; + lu_hit[i] = 1'b1; + end + end + end + end + end + + + + logic asid_to_be_flushed_is0; // indicates that the ASID provided by SFENCE.VMA (rs2)is 0, active high + logic vaddr_to_be_flushed_is0; // indicates that the VADDR provided by SFENCE.VMA (rs1)is 0, active high + logic [TLB_ENTRIES-1:0] vaddr_vpn0_match; + logic [TLB_ENTRIES-1:0] vaddr_vpn1_match; + logic [TLB_ENTRIES-1:0] vaddr_vpn2_match; + + assign asid_to_be_flushed_is0 = ~(|asid_to_be_flushed_i); + assign vaddr_to_be_flushed_is0 = ~(|vaddr_to_be_flushed_i); + + // ------------------ + // Update and Flush + // ------------------ + always_comb begin : update_flush + tags_n = tags_q; + content_n = content_q; + + for (int unsigned i = 0; i < TLB_ENTRIES; i++) begin + + vaddr_vpn0_match[i] = (vaddr_to_be_flushed_i[20:12] == tags_q[i].vpn0); + vaddr_vpn1_match[i] = (vaddr_to_be_flushed_i[29:21] == tags_q[i].vpn1); + vaddr_vpn2_match[i] = (vaddr_to_be_flushed_i[30+riscv::VPN2:30] == tags_q[i].vpn2); + + if (flush_i) begin + // invalidate logic + // flush everything if ASID is 0 and vaddr is 0 ("SFENCE.VMA x0 x0" case) + if (asid_to_be_flushed_is0 && vaddr_to_be_flushed_is0) tags_n[i].valid = 1'b0; + // flush vaddr in all addressing space ("SFENCE.VMA vaddr x0" case), it should happen only for leaf pages + else if (asid_to_be_flushed_is0 && ((vaddr_vpn0_match[i] && vaddr_vpn1_match[i] && vaddr_vpn2_match[i]) || (vaddr_vpn2_match[i] && tags_q[i].is_1G) || (vaddr_vpn1_match[i] && vaddr_vpn2_match[i] && tags_q[i].is_2M) ) && (~vaddr_to_be_flushed_is0)) + tags_n[i].valid = 1'b0; + // the entry is flushed if it's not global and asid and vaddr both matches with the entry to be flushed ("SFENCE.VMA vaddr asid" case) + else if ((!content_q[i].g) && ((vaddr_vpn0_match[i] && vaddr_vpn1_match[i] && vaddr_vpn2_match[i]) || (vaddr_vpn2_match[i] && tags_q[i].is_1G) || (vaddr_vpn1_match[i] && vaddr_vpn2_match[i] && tags_q[i].is_2M)) && (asid_to_be_flushed_i == tags_q[i].asid) && (!vaddr_to_be_flushed_is0) && (!asid_to_be_flushed_is0)) + tags_n[i].valid = 1'b0; + // the entry is flushed if it's not global, and the asid matches and vaddr is 0. ("SFENCE.VMA 0 asid" case) + else if ((!content_q[i].g) && (vaddr_to_be_flushed_is0) && (asid_to_be_flushed_i == tags_q[i].asid) && (!asid_to_be_flushed_is0)) + tags_n[i].valid = 1'b0; + // normal replacement + end else if (update_i.valid & replace_en[i]) begin + // update tag array + tags_n[i] = '{ + asid: update_i.asid, + vpn2: update_i.vpn[18+riscv::VPN2:18], + vpn1: update_i.vpn[17:9], + vpn0: update_i.vpn[8:0], + is_1G: update_i.is_1G, + is_2M: update_i.is_2M, + valid: 1'b1 + }; + // and content as well + content_n[i] = update_i.content; + end + end + end + + // ----------------------------------------------- + // PLRU - Pseudo Least Recently Used Replacement + // ----------------------------------------------- + logic [2*(TLB_ENTRIES-1)-1:0] plru_tree_q, plru_tree_n; + always_comb begin : plru_replacement + plru_tree_n = plru_tree_q; + // The PLRU-tree indexing: + // lvl0 0 + // / \ + // / \ + // lvl1 1 2 + // / \ / \ + // lvl2 3 4 5 6 + // / \ /\/\ /\ + // ... ... ... ... + // Just predefine which nodes will be set/cleared + // E.g. for a TLB with 8 entries, the for-loop is semantically + // equivalent to the following pseudo-code: + // unique case (1'b1) + // lu_hit[7]: plru_tree_n[0, 2, 6] = {1, 1, 1}; + // lu_hit[6]: plru_tree_n[0, 2, 6] = {1, 1, 0}; + // lu_hit[5]: plru_tree_n[0, 2, 5] = {1, 0, 1}; + // lu_hit[4]: plru_tree_n[0, 2, 5] = {1, 0, 0}; + // lu_hit[3]: plru_tree_n[0, 1, 4] = {0, 1, 1}; + // lu_hit[2]: plru_tree_n[0, 1, 4] = {0, 1, 0}; + // lu_hit[1]: plru_tree_n[0, 1, 3] = {0, 0, 1}; + // lu_hit[0]: plru_tree_n[0, 1, 3] = {0, 0, 0}; + // default: begin /* No hit */ end + // endcase + for ( + int unsigned i = 0; i < TLB_ENTRIES; i++ + ) begin + automatic int unsigned idx_base, shift, new_index; + // we got a hit so update the pointer as it was least recently used + if (lu_hit[i] & lu_access_i) begin + // Set the nodes to the values we would expect + for (int unsigned lvl = 0; lvl < $clog2(TLB_ENTRIES); lvl++) begin + idx_base = $unsigned((2 ** lvl) - 1); + // lvl0 <=> MSB, lvl1 <=> MSB-1, ... + shift = $clog2(TLB_ENTRIES) - lvl; + // to circumvent the 32 bit integer arithmetic assignment + new_index = ~((i >> (shift - 1)) & 32'b1); + plru_tree_n[idx_base+(i>>shift)] = new_index[0]; + end + end + end + // Decode tree to write enable signals + // Next for-loop basically creates the following logic for e.g. an 8 entry + // TLB (note: pseudo-code obviously): + // replace_en[7] = &plru_tree_q[ 6, 2, 0]; //plru_tree_q[0,2,6]=={1,1,1} + // replace_en[6] = &plru_tree_q[~6, 2, 0]; //plru_tree_q[0,2,6]=={1,1,0} + // replace_en[5] = &plru_tree_q[ 5,~2, 0]; //plru_tree_q[0,2,5]=={1,0,1} + // replace_en[4] = &plru_tree_q[~5,~2, 0]; //plru_tree_q[0,2,5]=={1,0,0} + // replace_en[3] = &plru_tree_q[ 4, 1,~0]; //plru_tree_q[0,1,4]=={0,1,1} + // replace_en[2] = &plru_tree_q[~4, 1,~0]; //plru_tree_q[0,1,4]=={0,1,0} + // replace_en[1] = &plru_tree_q[ 3,~1,~0]; //plru_tree_q[0,1,3]=={0,0,1} + // replace_en[0] = &plru_tree_q[~3,~1,~0]; //plru_tree_q[0,1,3]=={0,0,0} + // For each entry traverse the tree. If every tree-node matches, + // the corresponding bit of the entry's index, this is + // the next entry to replace. + for (int unsigned i = 0; i < TLB_ENTRIES; i += 1) begin + automatic logic en; + automatic int unsigned idx_base, shift, new_index; + en = 1'b1; + for (int unsigned lvl = 0; lvl < $clog2(TLB_ENTRIES); lvl++) begin + idx_base = $unsigned((2 ** lvl) - 1); + // lvl0 <=> MSB, lvl1 <=> MSB-1, ... + shift = $clog2(TLB_ENTRIES) - lvl; + + // en &= plru_tree_q[idx_base + (i>>shift)] == ((i >> (shift-1)) & 1'b1); + new_index = (i >> (shift - 1)) & 32'b1; + if (new_index[0]) begin + en &= plru_tree_q[idx_base+(i>>shift)]; + end else begin + en &= ~plru_tree_q[idx_base+(i>>shift)]; + end + end + replace_en[i] = en; + end + end + + // sequential process + always_ff @(posedge clk_i or negedge rst_ni) begin + if (~rst_ni) begin + tags_q <= '{default: 0}; + content_q <= '{default: 0}; + plru_tree_q <= '{default: 0}; + end else begin + tags_q <= tags_n; + content_q <= content_n; + plru_tree_q <= plru_tree_n; + end + end + //-------------- + // Sanity checks + //-------------- + + //pragma translate_off +`ifndef VERILATOR + + initial begin : p_assertions + assert ((TLB_ENTRIES % 2 == 0) && (TLB_ENTRIES > 1)) + else begin + $error("TLB size must be a multiple of 2 and greater than 1"); + $stop(); + end + assert (ASID_WIDTH >= 1) + else begin + $error("ASID width must be at least 1"); + $stop(); + end + end + + // Just for checking + function int countSetBits(logic [TLB_ENTRIES-1:0] vector); + automatic int count = 0; + foreach (vector[idx]) begin + count += vector[idx]; + end + return count; + endfunction + + assert property (@(posedge clk_i) (countSetBits(lu_hit) <= 1)) + else begin + $error("More then one hit in TLB!"); + $stop(); + end + assert property (@(posedge clk_i) (countSetBits(replace_en) <= 1)) + else begin + $error("More then one TLB entry selected for next replace!"); + $stop(); + end + +`endif + //pragma translate_on + +endmodule diff --git a/docs/01_cva6_user/Programmer_View.rst b/docs/01_cva6_user/Programmer_View.rst index a2d0eab30..461e5b0c1 100644 --- a/docs/01_cva6_user/Programmer_View.rst +++ b/docs/01_cva6_user/Programmer_View.rst @@ -166,34 +166,19 @@ CV32A6 supports the RISC-V **Sv32** virtual memory when the ``MMUEn`` parameter CV64A6 supports the RISC-V **Sv39** virtual memory when the ``MMUEn`` parameter is set to 1 (and ``Xlen`` is set to 64). -Within CV64A6, the hypervisor extension is enabled and supports **Sv39x4** virtual memory when the ``CVA6ConfigHExtEn`` parameter is set to 1 (and ``Xlen`` is set to 64). - - -By default, CV32A6 and CV64A6 are in RISC-V **Bare** mode. **Sv32** or **Sv39** are enabled by writing the required configuration to satp register mode bits. - -In CV32A6 the mode bit of satp register is bit 31. **Sv32** is enabled by writing 1 to ``satp[31]``. - -In CV64A6 the mode bits of satp register are bits [63:60]. **Sv39** is enabled by writing ``8`` to ``satp[63:60]``. - -When the ``MMUEn`` parameter is set to 0, CV32A6 and CV64A6 are always in RISC-V **Bare** mode; satp mode bit(s) remain at 0 and writes to this register are ignored. - - -By default, the hypervisor extension is disabled. It can be enabled by setting bit 7 in the ``misa CSR``, which corresponds to the letter H. - -When ``CVA6ConfigHExtEn`` parameter is set to 0, the hypervisor extension is always disabled; bit 7 in the ``misa CSR`` remains at 0 and writes to this register are ignored. - -Even if Hypervisor extension is enabled, by default, address translation for Supervisor, Hypervisor and Virtual Supervisor are disabled. They can be enabled by writing the required configuration to satp, hgatp and vsatp registers respectively. - -**Sv39** is enabled for Supervisor or Virtual Supervisor by writing ``8`` to ``satp[63:60]`` or ``vsatp[63:60]`` respectively. - -**Sv39x4** is enabled for Hypervisor by writing ``8`` to ``hgatp[63:60]``. +By default, CV32A6 and CV64A6 are in RISC-V **Bare** mode. **Sv32** or **Sv39** are enabled by writing 1 to ``satp[0]`` register bit. +When the ``MMUEn`` parameter is set to 0, CV32A6 and CV64A6 are always in RISC-V **Bare** mode; ``satp[0]`` remains at 0 and writes to this register are ignored. Notes for the integrator: * The virtual memory is implemented by a memory management unit (MMU) that accelerates the translation from virtual memory addresses (as handled by the core) to physical memory addresses. The MMU integrates translation lookaside buffers (TLB) and a hardware page table walker (PTW). The number of instruction and data TLB entries are configured with ``InstrTlbEntries`` and ``DataTlbEntries``. -* The MMU is implemented with a microarchitectural optimization featuring two levels of TLB: level 1 TLB (sized by ``InstrTlbEntries`` and ``DataTlbEntries``) and a shared level 2 TLB. The optimization has no consequences on the programmer's view. +* The MMU will integrate a microarchitectural optimization featuring two levels of TLB: level 1 TBL (sized by ``InstrTlbEntries`` and ``DataTlbEntries``) and a shared level 2 TLB. The optimization has no consequences on the programmer's view. + +* The addition of the hypervisor support will come with **Sv39x4** virtual memory that is not yet documented here. + +*These are the addressing modes supported by the various CVA6 configurations:* CV32A60AX virtual memory ~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/src_files.yml b/src_files.yml index 1a8a206d0..84173c67c 100644 --- a/src_files.yml +++ b/src_files.yml @@ -34,7 +34,8 @@ ariane: src/load_unit.sv, src/load_store_unit.sv, src/miss_handler.sv, - src/cva6_mmu/cva6_mmu.sv, + src/mmu_sv39/mmu.sv, + src/mmu_sv32/cva6_mmu_sv32.sv, src/mult.sv, src/nbdcache.sv, src/vdregs.sv, @@ -42,13 +43,14 @@ ariane: src/sram_wrapper.sv, src/pcgen_stage.sv, src/perf_counters.sv, - src/cva6_mmu/cva6_ptw.sv, + src/mmu_sv39/ptw.sv, + src/mmu_sv32/cva6_ptw_sv32.sv, src/re_name.sv, src/scoreboard.sv, src/store_buffer.sv, src/store_unit.sv, - src/cva6_mmu/cva6_tlb.sv, - src/cva6_mmu/cva6_shared_tlb.sv, + src/mmu_sv39/tlb.sv, + src/mmu_sv32/cva6_tlb_sv32.sv, src/acc_dispatcher.sv, src/debug/dm_csrs.sv, src/debug/dm_mem.sv,