diff --git a/core/cva6_mmu/cva6_mmu.sv b/core/cva6_mmu/cva6_mmu.sv index cad2c6eaf..a7a17ff40 100644 --- a/core/cva6_mmu/cva6_mmu.sv +++ b/core/cva6_mmu/cva6_mmu.sv @@ -544,15 +544,15 @@ module cva6_mmu if ((en_ld_st_translation_i || en_ld_st_g_translation_i) && !misaligned_ex_q.valid) begin lsu_valid_o = 1'b0; - lsu_dtlb_ppn_o = (en_ld_st_g_translation_i && CVA6Cfg.RVH)? dtlb_g_content.ppn :dtlb_content.ppn; + lsu_dtlb_ppn_o = (en_ld_st_g_translation_i && CVA6Cfg.RVH)?dtlb_g_content.ppn:dtlb_content.ppn; lsu_paddr_o = { (en_ld_st_g_translation_i && CVA6Cfg.RVH) ? dtlb_gpte_q.ppn : dtlb_pte_q.ppn, lsu_vaddr_q[11:0] }; if (CVA6Cfg.PtLevels == 3 && dtlb_is_page_q[CVA6Cfg.PtLevels-2]) begin - lsu_paddr_o[PPNWMin-(CVA6Cfg.VpnLen/CVA6Cfg.PtLevels):9+CVA6Cfg.PtLevels] = lsu_vaddr_q[PPNWMin-(CVA6Cfg.VpnLen/CVA6Cfg.PtLevels):9+CVA6Cfg.PtLevels]; - lsu_dtlb_ppn_o[PPNWMin-(CVA6Cfg.VpnLen/CVA6Cfg.PtLevels):9+CVA6Cfg.PtLevels] = lsu_vaddr_n[PPNWMin-(CVA6Cfg.VpnLen/CVA6Cfg.PtLevels):9+CVA6Cfg.PtLevels]; + lsu_paddr_o[PPNWMin-(CVA6Cfg.VpnLen/CVA6Cfg.PtLevels):12] = lsu_vaddr_q[PPNWMin-(CVA6Cfg.VpnLen/CVA6Cfg.PtLevels):12]; + lsu_dtlb_ppn_o[PPNWMin-(CVA6Cfg.VpnLen/CVA6Cfg.PtLevels):12] = lsu_vaddr_n[PPNWMin-(CVA6Cfg.VpnLen/CVA6Cfg.PtLevels):12]; end if (dtlb_is_page_q[0]) begin @@ -584,7 +584,7 @@ module cva6_mmu {CVA6Cfg.XLEN - CVA6Cfg.VLEN{lsu_vaddr_q[CVA6Cfg.VLEN-1]}}, lsu_vaddr_q }; if (CVA6Cfg.RVH) begin - lsu_exception_o.tval2= CVA6Cfg.GPLEN'(lsu_gpaddr_q[(CVA6Cfg.XLEN==32?CVA6Cfg.VLEN : CVA6Cfg.GPLEN)-1:0]); + lsu_exception_o.tval2 = CVA6Cfg.GPLEN'(lsu_gpaddr_q[(CVA6Cfg.XLEN==32?CVA6Cfg.VLEN : CVA6Cfg.GPLEN)-1:0]); lsu_exception_o.tinst = '0; lsu_exception_o.gva = ld_st_v_i; end @@ -611,7 +611,7 @@ module cva6_mmu {CVA6Cfg.XLEN - CVA6Cfg.VLEN{lsu_vaddr_q[CVA6Cfg.VLEN-1]}}, lsu_vaddr_q }; if (CVA6Cfg.RVH) begin - lsu_exception_o.tval2= CVA6Cfg.GPLEN'(lsu_gpaddr_q[(CVA6Cfg.XLEN==32?CVA6Cfg.VLEN : CVA6Cfg.GPLEN)-1:0]); + lsu_exception_o.tval2 = CVA6Cfg.GPLEN'(lsu_gpaddr_q[(CVA6Cfg.XLEN==32?CVA6Cfg.VLEN : CVA6Cfg.GPLEN)-1:0]); lsu_exception_o.tinst = '0; lsu_exception_o.gva = ld_st_v_i; end diff --git a/core/cva6_mmu/cva6_ptw.sv b/core/cva6_mmu/cva6_ptw.sv index 08b2d3131..a7938415d 100644 --- a/core/cva6_mmu/cva6_ptw.sv +++ b/core/cva6_mmu/cva6_ptw.sv @@ -48,13 +48,11 @@ module cva6_ptw input logic v_i, // current virtualization mode bit input logic ld_st_v_i, // load/store virtualization mode bit input logic hlvx_inst_i, // is a HLVX load/store instruction - - input logic lsu_is_store_i, // this translation was triggered by a store + input logic lsu_is_store_i, // this translation was triggered by a store // PTW memory interface input dcache_req_o_t req_port_i, output dcache_req_i_t req_port_o, - // to TLBs, update logic output tlb_update_cva6_t shared_tlb_update_o, @@ -74,7 +72,7 @@ module cva6_ptw // from CSR file input logic [CVA6Cfg.PPNW-1:0] satp_ppn_i, // ppn from satp - input logic [CVA6Cfg.PPNW-1:0] vsatp_ppn_i, // ppn from satp + input logic [CVA6Cfg.PPNW-1:0] vsatp_ppn_i, // ppn from vsatp input logic [CVA6Cfg.PPNW-1:0] hgatp_ppn_i, // ppn from hgatp input logic mxr_i, input logic vmxr_i, @@ -110,6 +108,7 @@ module cva6_ptw state_q, state_d; logic [CVA6Cfg.PtLevels-2:0] misaligned_page; + logic tlb_update_valid; logic [HYP_EXT:0][CVA6Cfg.PtLevels-2:0] ptw_lvl_n, ptw_lvl_q; // define 3 PTW stages to be used in sv39x4. sv32 and sv39 are always in S_STAGE @@ -163,27 +162,26 @@ module cva6_ptw assign gpaddr_base = {pte.ppn[CVA6Cfg.GPPNW-1:0], vaddr_q[11:0]}; assign gpaddr[CVA6Cfg.PtLevels-1] = gpaddr_base; - assign shared_tlb_update_o.vpn = CVA6Cfg.VpnLen'(vaddr_q[CVA6Cfg.SV+HYP_EXT*2-1:12]); genvar z, w; generate for (z = 0; z < CVA6Cfg.PtLevels - 1; z++) begin - // check if the ppn is correctly aligned: + // Check if the ppn is correctly aligned: // 6. If i > 0 and pa.ppn[i − 1 : 0] != 0, this is a misaligned superpage; stop and raise a page-fault // exception. assign misaligned_page[z] = (ptw_lvl_q[0] == (z)) && (pte.ppn[(CVA6Cfg.VpnLen/CVA6Cfg.PtLevels)*(CVA6Cfg.PtLevels-1-z)-1:0] != '0); - //record the vaddr corresponding to each level + // Record the vaddr corresponding to each level for (w = 0; w < HYP_EXT * 2 + 1; w++) begin - assign vaddr_lvl[w][z] = w==0 ? vaddr_q[12+((CVA6Cfg.VpnLen/CVA6Cfg.PtLevels)*(CVA6Cfg.PtLevels-z-1))-1:12+((CVA6Cfg.VpnLen/CVA6Cfg.PtLevels)*(CVA6Cfg.PtLevels-z-2))] : - w==1 ? gptw_pptr_q[12+((CVA6Cfg.VpnLen/CVA6Cfg.PtLevels)*(CVA6Cfg.PtLevels-z-1))-1:12+((CVA6Cfg.VpnLen/CVA6Cfg.PtLevels)*(CVA6Cfg.PtLevels-z-2))]: - gpaddr_q[12+((CVA6Cfg.VpnLen/CVA6Cfg.PtLevels)*(CVA6Cfg.PtLevels-z-1))-1:12+((CVA6Cfg.VpnLen/CVA6Cfg.PtLevels)*(CVA6Cfg.PtLevels-z-2))]; + assign vaddr_lvl[w][z] = w==0 ? vaddr_q[12+((CVA6Cfg.VpnLen/CVA6Cfg.PtLevels)*(CVA6Cfg.PtLevels-z-1))-1:12+((CVA6Cfg.VpnLen/CVA6Cfg.PtLevels)*(CVA6Cfg.PtLevels-z-2))]: + w==1 ? gptw_pptr_q[12+((CVA6Cfg.VpnLen/CVA6Cfg.PtLevels)*(CVA6Cfg.PtLevels-z-1))-1:12+((CVA6Cfg.VpnLen/CVA6Cfg.PtLevels)*(CVA6Cfg.PtLevels-z-2))]: + gpaddr_q[12+((CVA6Cfg.VpnLen/CVA6Cfg.PtLevels)*(CVA6Cfg.PtLevels-z-1))-1:12+((CVA6Cfg.VpnLen/CVA6Cfg.PtLevels)*(CVA6Cfg.PtLevels-z-2))]; end if (CVA6Cfg.RVH) begin - assign gpaddr[z][CVA6Cfg.VpnLen-(CVA6Cfg.VpnLen/CVA6Cfg.PtLevels):0]= (ptw_lvl_q[0] == z) ? vaddr_q[CVA6Cfg.VpnLen-(CVA6Cfg.VpnLen/CVA6Cfg.PtLevels):0] : gpaddr_base[CVA6Cfg.VpnLen-(CVA6Cfg.VpnLen/CVA6Cfg.PtLevels):0]; - assign gpaddr[z][CVA6Cfg.VpnLen:CVA6Cfg.VpnLen-(CVA6Cfg.VpnLen/CVA6Cfg.PtLevels)+1]= (ptw_lvl_q[0] == 0) ? vaddr_q[CVA6Cfg.VpnLen:CVA6Cfg.VpnLen-(CVA6Cfg.VpnLen/CVA6Cfg.PtLevels)+1] : gpaddr_base[CVA6Cfg.VpnLen:CVA6Cfg.VpnLen-(CVA6Cfg.VpnLen/CVA6Cfg.PtLevels)+1]; + assign gpaddr[z][CVA6Cfg.VpnLen-(CVA6Cfg.VpnLen/CVA6Cfg.PtLevels):0] = (ptw_lvl_q[0] == z) ? vaddr_q[CVA6Cfg.VpnLen-(CVA6Cfg.VpnLen/CVA6Cfg.PtLevels):0] : gpaddr_base[CVA6Cfg.VpnLen-(CVA6Cfg.VpnLen/CVA6Cfg.PtLevels):0]; + assign gpaddr[z][CVA6Cfg.VpnLen:CVA6Cfg.VpnLen-(CVA6Cfg.VpnLen/CVA6Cfg.PtLevels)+1] = (ptw_lvl_q[0] == 0) ? vaddr_q[CVA6Cfg.VpnLen:CVA6Cfg.VpnLen-(CVA6Cfg.VpnLen/CVA6Cfg.PtLevels)+1] : gpaddr_base[CVA6Cfg.VpnLen:CVA6Cfg.VpnLen-(CVA6Cfg.VpnLen/CVA6Cfg.PtLevels)+1]; assign gpaddr[z][CVA6Cfg.GPLEN-1:CVA6Cfg.VpnLen+1] = gpaddr_base[CVA6Cfg.GPLEN-1:CVA6Cfg.VpnLen+1]; end @@ -192,32 +190,41 @@ module cva6_ptw endgenerate always_comb begin : tlb_update - // update the correct page table level - for (int unsigned y = 0; y < HYP_EXT + 1; y++) begin - for (int unsigned x = 0; x < CVA6Cfg.PtLevels - 1; x++) begin - if((enable_g_translation_i && enable_translation_i) || (en_ld_st_g_translation_i && en_ld_st_translation_i) && CVA6Cfg.RVH) begin - shared_tlb_update_o.is_page[x][y] = (ptw_lvl_q[y==HYP_EXT?0 : 1] == x); - end else if (enable_translation_i || en_ld_st_translation_i || !CVA6Cfg.RVH) begin - shared_tlb_update_o.is_page[x][y] = y == 0 ? (ptw_lvl_q[0] == x) : 1'b0; - end else begin - shared_tlb_update_o.is_page[x][y] = y != 0 ? (ptw_lvl_q[0] == x) : 1'b0; + if (tlb_update_valid) begin + // update the correct page table level + for (int unsigned y = 0; y < HYP_EXT + 1; y++) begin + for (int unsigned x = 0; x < CVA6Cfg.PtLevels - 1; x++) begin + // VS + G-Translation + if(((enable_g_translation_i && enable_translation_i) || (en_ld_st_g_translation_i && en_ld_st_translation_i)) && CVA6Cfg.RVH) begin + shared_tlb_update_o.is_page[x][y] = (ptw_lvl_q[y==1?0:1] == x); + // Non-V, S-Translation + end else if (enable_translation_i || en_ld_st_translation_i || !CVA6Cfg.RVH) begin + shared_tlb_update_o.is_page[x][y] = y == 0 ? (ptw_lvl_q[0] == x) : 1'b0; + // G-Translation + end else begin + shared_tlb_update_o.is_page[x][y] = y != 0 ? (ptw_lvl_q[0] == x) : 1'b0; + end end end - end - // set the global mapping bit - if ((enable_g_translation_i || en_ld_st_g_translation_i) && CVA6Cfg.RVH) begin - shared_tlb_update_o.content = gpte_q | (global_mapping_q << 5); - shared_tlb_update_o.g_content = pte; + // set the global mapping bit + if ((enable_g_translation_i || en_ld_st_g_translation_i) && CVA6Cfg.RVH) begin + shared_tlb_update_o.content = (gpte_q | (global_mapping_q << 5)); + shared_tlb_update_o.g_content = pte; + end else begin + shared_tlb_update_o.content = (pte | (global_mapping_q << 5)); + shared_tlb_update_o.g_content = '0; + end + + // output the correct ASIDs + shared_tlb_update_o.asid = tlb_update_asid_q; + shared_tlb_update_o.vmid = CVA6Cfg.RVH ? tlb_update_vmid_q : '0; + shared_tlb_update_o.vpn = vaddr_q[12+CVA6Cfg.VpnLen-1:12]; + shared_tlb_update_o.valid = 1'b1; end else begin - shared_tlb_update_o.content = (pte | (global_mapping_q << 5)); - shared_tlb_update_o.g_content = '0; + shared_tlb_update_o.valid = 1'b0; end - // output the correct ASIDs - shared_tlb_update_o.asid = tlb_update_asid_q; - shared_tlb_update_o.vmid = CVA6Cfg.RVH ? tlb_update_vmid_q : '0; - bad_paddr_o = ptw_access_exception_o ? ptw_pptr_q : 'b0; if (CVA6Cfg.RVH) bad_gpaddr_o[CVA6Cfg.GPLEN-1:0] = ptw_error_at_g_st_o ? ((ptw_stage_q == G_INTERMED_STAGE) ? gptw_pptr_q[CVA6Cfg.GPLEN-1:0] : gpaddr_q) : 'b0; @@ -276,7 +283,6 @@ module cva6_ptw // - pa.ppn[LEVELS-1:i] = pte.ppn[LEVELS-1:i]. always_comb begin : ptw automatic logic [CVA6Cfg.PLEN-1:0] pptr; - // automatic logic [CVA6Cfg.GPLEN-1:0] gpaddr; // default assignments // PTW memory interface tag_valid_n = 1'b0; @@ -287,7 +293,7 @@ module cva6_ptw ptw_error_at_g_st_o = 1'b0; ptw_err_at_g_int_st_o = 1'b0; ptw_access_exception_o = 1'b0; - shared_tlb_update_o.valid = 1'b0; + tlb_update_valid = 1'b0; is_instr_ptw_n = is_instr_ptw_q; ptw_lvl_n = ptw_lvl_q; ptw_pptr_n = ptw_pptr_q; @@ -407,8 +413,7 @@ module cva6_ptw // ----------- else begin state_d = LATENCY; - // it is a valid PTE - // if pte.r = 1 or pte.x = 1 it is a valid PTE + // It is a valid PTE if pte.r = 1 or pte.x = 1 if (pte.r || pte.x) begin if (CVA6Cfg.RVH) begin case (ptw_stage_q) @@ -421,7 +426,7 @@ module cva6_ptw gpaddr_n = gpaddr[ptw_lvl_q[0]]; ptw_pptr_n = { hgatp_ppn_i[CVA6Cfg.PPNW-1:2], - gpaddr[ptw_lvl_q[0]][CVA6Cfg.SV+HYP_EXT*2-1:CVA6Cfg.SV-(CVA6Cfg.VpnLen/CVA6Cfg.PtLevels)], + gpaddr[ptw_lvl_q[0]][CVA6Cfg.GPLEN-1:CVA6Cfg.SV-(CVA6Cfg.VpnLen/CVA6Cfg.PtLevels)], (CVA6Cfg.PtLevels)'(0) }; ptw_lvl_n[0] = '0; @@ -450,8 +455,8 @@ module cva6_ptw if (!pte.x || !pte.a) begin state_d = PROPAGATE_ERROR; if (CVA6Cfg.RVH) ptw_stage_d = ptw_stage_q; - end else if (CVA6Cfg.RVH && ((ptw_stage_q == G_FINAL_STAGE) || !enable_g_translation_i) || !CVA6Cfg.RVH) - shared_tlb_update_o.valid = 1'b1; + end else if ((CVA6Cfg.RVH && ((ptw_stage_q == G_FINAL_STAGE) || !enable_g_translation_i)) || !CVA6Cfg.RVH) + tlb_update_valid = 1'b1; end else begin // ------------ @@ -462,48 +467,45 @@ module cva6_ptw // If page is not readable (there are no write-only pages) // we can directly raise an error. This doesn't put a useless // entry into the TLB. - if (pte.a && ((pte.r && !hlvx_inst_i) || (pte.x && (mxr_i || hlvx_inst_i || (ptw_stage_q == S_STAGE && vmxr_i && ld_st_v_i && CVA6Cfg.RVH))))) begin - if (CVA6Cfg.RVH && ((ptw_stage_q == G_FINAL_STAGE) || !en_ld_st_g_translation_i) || !CVA6Cfg.RVH) - shared_tlb_update_o.valid = 1'b1; + if ( + (pte.a && ((pte.r && !hlvx_inst_i) || (pte.x && (mxr_i || hlvx_inst_i || (ptw_stage_q == S_STAGE && vmxr_i && ld_st_v_i && CVA6Cfg.RVH))))) + // Request is a store: perform some additional checks + // If the request was a store and the page is not write-able, raise an error + // the same applies if the dirty flag is not set + // g-intermediate nodes however never need write-permission + && (!lsu_is_store_i || (pte.w && pte.d) || (ptw_stage_q == G_INTERMED_STAGE && CVA6Cfg.RVH)) + ) begin + if ((CVA6Cfg.RVH && ((ptw_stage_q == G_FINAL_STAGE) || !en_ld_st_g_translation_i)) || !CVA6Cfg.RVH) + tlb_update_valid = 1'b1; end else begin state_d = PROPAGATE_ERROR; if (CVA6Cfg.RVH) ptw_stage_d = ptw_stage_q; end - // Request is a store: perform some additional checks - // If the request was a store and the page is not write-able, raise an error - // the same applies if the dirty flag is not set - if (lsu_is_store_i && (!pte.w || !pte.d) && (ptw_stage_q != G_INTERMED_STAGE)) begin //FIXME - shared_tlb_update_o.valid = 1'b0; - state_d = PROPAGATE_ERROR; - if (CVA6Cfg.RVH) ptw_stage_d = ptw_stage_q; - end end - //if there is a misaligned page, propagate error + // If there is a misaligned page, propagate error if (|misaligned_page) begin state_d = PROPAGATE_ERROR; if (CVA6Cfg.RVH) ptw_stage_d = ptw_stage_q; - shared_tlb_update_o.valid = 1'b0; + tlb_update_valid = 1'b0; end - // check if 63:41 are all zeros + // Check if 63:41 are all zeros if (CVA6Cfg.RVH) begin if (((v_i && is_instr_ptw_q) || (ld_st_v_i && !is_instr_ptw_q)) && ptw_stage_q == S_STAGE && !((|pte.ppn[CVA6Cfg.PPNW-1:CVA6Cfg.GPPNW-1+HYP_EXT]) == 1'b0)) begin state_d = PROPAGATE_ERROR; ptw_stage_d = G_FINAL_STAGE; end end - // this is a pointer to the next TLB level + // This is a pointer to the next TLB level end else begin - // pointer to next level of page table + // Pointer to next level of page table if (ptw_lvl_q[0] == CVA6Cfg.PtLevels - 1) begin // Should already be the last level page table => Error ptw_lvl_n[0] = ptw_lvl_q[0]; state_d = PROPAGATE_ERROR; if (CVA6Cfg.RVH) ptw_stage_d = ptw_stage_q; - - end else begin ptw_lvl_n[0] = ptw_lvl_q[0] + 1'b1; state_d = WAIT_GRANT; @@ -513,7 +515,6 @@ module cva6_ptw S_STAGE: begin if (CVA6Cfg.RVH && ((is_instr_ptw_q && enable_g_translation_i) || (!is_instr_ptw_q && en_ld_st_g_translation_i))) begin ptw_stage_d = G_INTERMED_STAGE; - if (CVA6Cfg.RVH) gpte_d = pte; ptw_lvl_n[HYP_EXT] = ptw_lvl_q[0] + 1; pptr = {pte.ppn, vaddr_lvl[0][ptw_lvl_q[0]], (CVA6Cfg.PtLevels)'(0)}; gptw_pptr_n = pptr; @@ -547,7 +548,7 @@ module cva6_ptw end - // check if 63:41 are all zeros + // Check if 63:41 are all zeros if (CVA6Cfg.RVH) begin if (((v_i && is_instr_ptw_q) || (ld_st_v_i && !is_instr_ptw_q)) && ptw_stage_q == S_STAGE && !((|pte.ppn[CVA6Cfg.PPNW-1:CVA6Cfg.GPPNW-1+HYP_EXT]) == 1'b0)) begin state_d = PROPAGATE_ERROR; @@ -559,14 +560,14 @@ module cva6_ptw // Check if this access was actually allowed from a PMP perspective if (!allow_access) begin - shared_tlb_update_o.valid = 1'b0; + tlb_update_valid = 1'b0; // we have to return the failed address in bad_addr ptw_pptr_n = ptw_pptr_q; if (CVA6Cfg.RVH) ptw_stage_d = ptw_stage_q; state_d = PROPAGATE_ACCESS_ERROR; end end - // we've got a data WAIT_GRANT so tell the cache that the tag is valid + // We've got a data WAIT_GRANT so tell the cache that the tag is valid end // Propagate error to MMU/LSU PROPAGATE_ERROR: begin @@ -581,7 +582,7 @@ module cva6_ptw state_d = LATENCY; ptw_access_exception_o = 1'b1; end - // wait for the rvalid before going back to IDLE + // Wait for the rvalid before going back to IDLE WAIT_RVALID: begin if (data_rvalid_q) state_d = IDLE; end diff --git a/core/cva6_mmu/cva6_tlb.sv b/core/cva6_mmu/cva6_tlb.sv index 68083f3e2..306f3e7d7 100644 --- a/core/cva6_mmu/cva6_tlb.sv +++ b/core/cva6_mmu/cva6_tlb.sv @@ -15,7 +15,7 @@ // Author: Angela Gonzalez PlanV Technology // Date: 26/02/2024 // -// Description: Translation Lookaside Buffer, parameterizable to Sv32 or Sv39 , +// Description: Translation Lookaside Buffer, parameterizable to Sv32 or Sv39, // or sv39x4 fully set-associative // This module is an merge of the Sv32 TLB developed by Sebastien // Jacq (Thales Research & Technology), the Sv39 TLB developed @@ -45,7 +45,7 @@ module cva6_tlb input logic [CVA6Cfg.ASID_WIDTH-1:0] lu_asid_i, input logic [CVA6Cfg.VMID_WIDTH-1:0] lu_vmid_i, input logic [CVA6Cfg.VLEN-1:0] lu_vaddr_i, - output logic [CVA6Cfg.GPLEN-1:0] lu_gpaddr_o, + output logic [CVA6Cfg.GPLEN-1:0] lu_gpaddr_o, // FIXME output pte_cva6_t lu_content_o, output pte_cva6_t lu_g_content_o, input logic [CVA6Cfg.ASID_WIDTH-1:0] asid_to_be_flushed_i, @@ -60,16 +60,21 @@ module cva6_tlb struct packed { logic [CVA6Cfg.ASID_WIDTH-1:0] asid; logic [CVA6Cfg.VMID_WIDTH-1:0] vmid; + // VPN is: + // [0] -> VPN0 + // [1] -> VPN1 + // [2] -> VPN2 + // [3] -> 2-bit supplementary PPN2 in case of GPA logic [CVA6Cfg.PtLevels+HYP_EXT-1:0][(CVA6Cfg.VpnLen/CVA6Cfg.PtLevels)-1:0] vpn; logic [CVA6Cfg.PtLevels-2:0][HYP_EXT:0] is_page; - logic [HYP_EXT*2:0] v_st_enbl; // v_i,g-stage enabled, s-stage enabled + logic [HYP_EXT*2:0] v_st_enbl; // v_i, g-stage enabled, s-stage enabled logic valid; } [TLB_ENTRIES-1:0] tags_q, tags_n; struct packed { - pte_cva6_t pte; - pte_cva6_t gpte; + pte_cva6_t pte; // Result of S-translation of the input + pte_cva6_t gpte; // Output of G-translation of the (possibly S-translated) input } [TLB_ENTRIES-1:0] content_q, content_n; @@ -87,62 +92,87 @@ module cva6_tlb logic [TLB_ENTRIES-1:0] match_stage; pte_cva6_t g_content; logic [TLB_ENTRIES-1:0][(CVA6Cfg.GPPNW-1):0] gppn; - logic [2:0] v_st_enbl; + logic [HYP_EXT*2:0] v_st_enbl; assign v_st_enbl = (CVA6Cfg.RVH) ? {v_i, g_st_enbl_i, s_st_enbl_i} : '1; + //------------- // Translation //------------- - genvar i, x, z, w; generate for (i = 0; i < TLB_ENTRIES; i++) begin for (x = 0; x < CVA6Cfg.PtLevels; x++) begin - //identify page_match for all TLB Entries - assign page_match[i][x] = x==0 ? 1 :((HYP_EXT==0 || x==(CVA6Cfg.PtLevels-1)) ? // PAGE_MATCH CONTAINS THE MATCH INFORMATION FOR EACH TAG OF is_1G and is_2M in sv39x4. HIGHER LEVEL (Giga page), THEN THERE IS THE Mega page AND AT THE LOWER LEVEL IS ALWAYS 1 - &(tags_q[i].is_page[CVA6Cfg.PtLevels-1-x] | (~v_st_enbl[HYP_EXT:0])): - ((&v_st_enbl[HYP_EXT:0]) ? - ((tags_q[i].is_page[CVA6Cfg.PtLevels-1-x][0] && (tags_q[i].is_page[CVA6Cfg.PtLevels-2-x][HYP_EXT] || tags_q[i].is_page[CVA6Cfg.PtLevels-1-x][HYP_EXT])) - || (tags_q[i].is_page[CVA6Cfg.PtLevels-1-x][HYP_EXT] && (tags_q[i].is_page[CVA6Cfg.PtLevels-2-x][0] || tags_q[i].is_page[CVA6Cfg.PtLevels-1-x][0]))): - tags_q[i].is_page[CVA6Cfg.PtLevels-1-x][0] && s_st_enbl_i || tags_q[i].is_page[CVA6Cfg.PtLevels-1-x][HYP_EXT] && g_st_enbl_i)); + // Identify if virtual address at level `x` matches the vaddr / gpaddr to be flushed + assign vaddr_vpn_match[i][0][x] = vaddr_to_be_flushed_i[12+((CVA6Cfg.VpnLen/CVA6Cfg.PtLevels)*(x+1))-1:12+((CVA6Cfg.VpnLen/CVA6Cfg.PtLevels)*x)] == tags_q[i].vpn[x]; + if (CVA6Cfg.RVH) begin + assign vaddr_vpn_match[i][HYP_EXT][0] = gpaddr_to_be_flushed_i[12+((CVA6Cfg.VpnLen/CVA6Cfg.PtLevels)*(x+1))-1:12+((CVA6Cfg.VpnLen/CVA6Cfg.PtLevels)*x)] == + gppn[i][ (CVA6Cfg.VpnLen/CVA6Cfg.PtLevels)*(x+1) : (CVA6Cfg.VpnLen/CVA6Cfg.PtLevels)*x]; + end + end - //identify if vpn matches at all PT levels for all TLB entries - assign vpn_match[i][x] = (CVA6Cfg.RVH && x == (CVA6Cfg.PtLevels - 1) && ~s_st_enbl_i) ? // - lu_vaddr_i[12+((CVA6Cfg.VpnLen/CVA6Cfg.PtLevels)*(x+1))-1:12+((CVA6Cfg.VpnLen/CVA6Cfg.PtLevels)*x)] == tags_q[i].vpn[x] && lu_vaddr_i[12+HYP_EXT*(CVA6Cfg.VpnLen-1): 12+HYP_EXT*(CVA6Cfg.VpnLen-(CVA6Cfg.VpnLen%CVA6Cfg.PtLevels))] == tags_q[i].vpn[x+HYP_EXT][(CVA6Cfg.VpnLen%CVA6Cfg.PtLevels)-HYP_EXT:0]: // - lu_vaddr_i[12+((CVA6Cfg.VpnLen/CVA6Cfg.PtLevels)*(x+1))-1:12+((CVA6Cfg.VpnLen/CVA6Cfg.PtLevels)*x)] == tags_q[i].vpn[x]; + for (x = 0; x < CVA6Cfg.PtLevels; x++) begin + // WARNING: `x` goes in the order {0 = 4K, 1 = 2M, 2 = 1G}. - //identify if there is a hit at each PT level for all TLB entries + // Identify page_match for all TLB Entries: + // `page_match[i][x] == 1` if the entry `i` represent a page of (non-stricly) bigger length than + // requested. + // 4K is always a match + // In case of H-mode, the length of a page in the TLB is the smallest of S-translation and + // G-translation + if (x==0) begin + assign page_match[i][x] = 1; + end else begin + if (HYP_EXT==0 || x==(CVA6Cfg.PtLevels-1)) begin + // No H-mode or Giga page. Then both condition must be true: + // - G-stage translation is *not* enabled or G-entry is a matching page (bit 1) + // - S-translation is *not* enabled or S-entry i is a matching page (bit 0) + assign page_match[i][x] = &(tags_q[i].is_page[CVA6Cfg.PtLevels-1-x][HYP_EXT:0] | (~v_st_enbl[HYP_EXT:0])); + end else begin + // Other cases: H-mode and mega page + assign page_match[i][x] = (&v_st_enbl[HYP_EXT:0])? + // If S-translation and G-translation are active, then either: + // - S-translation matchs and G-translation is Mega or Giga + // - G-translation matchs and S-translation is Mega or Giga + ((tags_q[i].is_page[CVA6Cfg.PtLevels-1-x][0] && (tags_q[i].is_page[CVA6Cfg.PtLevels-2-x][HYP_EXT] || tags_q[i].is_page[CVA6Cfg.PtLevels-1-x][HYP_EXT])) + || (tags_q[i].is_page[CVA6Cfg.PtLevels-1-x][HYP_EXT] && (tags_q[i].is_page[CVA6Cfg.PtLevels-2-x][0] || tags_q[i].is_page[CVA6Cfg.PtLevels-1-x][0]))) + : // Else, either S or G-level must match depending which is active + ((tags_q[i].is_page[CVA6Cfg.PtLevels-1-x][0] && s_st_enbl_i) || (tags_q[i].is_page[CVA6Cfg.PtLevels-1-x][HYP_EXT] && g_st_enbl_i)); + end + end + + // Identify if VPN matches at levels `x` for TLB entry `i` + if (CVA6Cfg.RVH && x == (CVA6Cfg.PtLevels - 1)) begin + // H-mode: extend the check on last level (Giga page) to include the supplementary bits in + // GPA (cfg.GPPNW = 29 bits in Sv39x4, compared to 27 bits in Sv39) + // /!\ Only in cases G-translation is active and not S-translation + assign vpn_match[i][x] = lu_vaddr_i[12+((CVA6Cfg.VpnLen/CVA6Cfg.PtLevels)*(x+1))-1:12+((CVA6Cfg.VpnLen/CVA6Cfg.PtLevels)*x)] == tags_q[i].vpn[x] + && (s_st_enbl_i || lu_vaddr_i[12+CVA6Cfg.GPPNW-1:12+((CVA6Cfg.VpnLen/CVA6Cfg.PtLevels)*(x+1))] == tags_q[i].vpn[x+HYP_EXT][(CVA6Cfg.GPPNW%(CVA6Cfg.VpnLen/CVA6Cfg.PtLevels))-1:0]); + + end else begin + // Standard match + assign vpn_match[i][x] = lu_vaddr_i[12+((CVA6Cfg.VpnLen/CVA6Cfg.PtLevels)*(x+1))-1:12+((CVA6Cfg.VpnLen/CVA6Cfg.PtLevels)*x)] == tags_q[i].vpn[x]; + end + + // Identify if there is a hit at level `x`: VPN and page length must match assign level_match[i][x] = &vpn_match[i][CVA6Cfg.PtLevels-1:x] && page_match[i][x]; - //identify vpage_match for all TLB Entries and vaddr_level match (if there is a hit at each PT level for all TLB entries on the vaddr) + // Identify `vpage_match` (matching page type) and deduce `vaddr_level` match (hit at all level on the virtual addr to be flushed + // and matching page type). for (z = 0; z < HYP_EXT + 1; z++) begin assign vpage_match[i][z][x] = x == 0 ? 1 : tags_q[i].is_page[CVA6Cfg.PtLevels-1-x][z]; - assign vaddr_level_match[i][z][x]= &vaddr_vpn_match[i][z][CVA6Cfg.PtLevels-1:x] && vpage_match[i][z][x]; - + assign vaddr_level_match[i][z][x] = &vaddr_vpn_match[i][z][CVA6Cfg.PtLevels-1:x] && vpage_match[i][z][x]; end - //identify if virtual address vpn matches at all PT levels for all TLB entries - assign vaddr_vpn_match[i][0][x] = vaddr_to_be_flushed_i[12+((CVA6Cfg.VpnLen/CVA6Cfg.PtLevels)*(x+1))-1:12+((CVA6Cfg.VpnLen/CVA6Cfg.PtLevels)*x)] == tags_q[i].vpn[x]; - - end - - - - if (CVA6Cfg.RVH) begin - //identify if GPADDR matches the GPPN - assign vaddr_vpn_match[i][HYP_EXT][0] = (gpaddr_to_be_flushed_i[20:12] == gppn[i][8:0]); - assign vaddr_vpn_match[i][HYP_EXT][HYP_EXT] = (gpaddr_to_be_flushed_i[29:21] == gppn[i][17:9]); - assign vaddr_vpn_match[i][HYP_EXT][HYP_EXT*2] = (gpaddr_to_be_flushed_i[30+GPPN2:30] == gppn[i][18+GPPN2:18]); - end + // Reorganise the output structure to match `is_page` tag order: [1G, 2M] for (w = 0; w < CVA6Cfg.PtLevels - 1; w++) begin - assign is_page_o[i][w] = page_match[i][CVA6Cfg.PtLevels - 1 - w]; //THIS REORGANIZES THE PAGES TO MATCH THE OUTPUT STRUCTURE (2M,1G) + assign is_page_o[i][w] = page_match[i][CVA6Cfg.PtLevels - 1 - w]; end end endgenerate - always_comb begin : translation - + always_comb begin: translation // default assignment lu_hit = '{default: 0}; lu_hit_o = 1'b0; @@ -153,10 +183,9 @@ module cva6_tlb match_vmid = CVA6Cfg.RVH ? '{default: 0} : '{default: 1}; match_stage = '{default: 0}; g_content = '{default: 0}; - lu_gpaddr_o = '{default: 0}; for (int unsigned i = 0; i < TLB_ENTRIES; i++) begin - // first level match, this may be a giga page, check the ASID flags as well + // First level match, this may be a giga page, check the ASID flags as well // if the entry is associated to a global address, don't match the ASID (ASID is don't care) match_asid[i] = ((lu_asid_i == tags_q[i].asid || content_q[i].pte.g) && s_st_enbl_i) || !s_st_enbl_i; @@ -164,45 +193,57 @@ module cva6_tlb match_vmid[i] = (lu_vmid_i == tags_q[i].vmid && g_st_enbl_i) || !g_st_enbl_i; end - // check if translation is a: S-Stage and G-Stage, S-Stage only or G-Stage only translation and virtualization mode is on/off + // Check if that S-stage and G-stage modes corresponds, and that + // and virtualization mode is on/off match_stage[i] = tags_q[i].v_st_enbl[HYP_EXT*2:0] == v_st_enbl[HYP_EXT*2:0]; - if (tags_q[i].valid && match_asid[i] && match_vmid[i] && match_stage[i]) begin - - if (CVA6Cfg.RVH && vpn_match[i][HYP_EXT*2]) begin + // There was a match, i.e.: + // - tag is valid + // - asid matches + // - vmid matches + // - virtualisations / S-stage / G-stage matches + // - there exists a PT level for which and entry exists and corresponding VPN(s) match + if (tags_q[i].valid && match_asid[i] && match_vmid[i] && match_stage[i] && |level_match[i]) begin + lu_is_page_o = is_page_o[i]; + lu_content_o = content_q[i].pte; + lu_hit_o = 1'b1; + lu_hit[i] = 1'b1; + // Translate S-stage to GPA: use `content_q[i].pte` to get PPN and use offset + // from input `lu_vaddr_i`. In case of mega/giga pages, PTE PPN must be aligned, + // so we should not overwite any useful bits. + if (CVA6Cfg.RVH) begin if (s_st_enbl_i) begin + // S-stage Normal page lu_gpaddr_o = {content_q[i].pte.ppn[(CVA6Cfg.GPPNW-1):0], lu_vaddr_i[11:0]}; - // Giga page - if (tags_q[i].is_page[0][0]) - lu_gpaddr_o[12+2*CVA6Cfg.VpnLen/CVA6Cfg.PtLevels-1:12] = lu_vaddr_i[12+2*CVA6Cfg.VpnLen/CVA6Cfg.PtLevels-1:12]; - // Mega page - if (tags_q[i].is_page[HYP_EXT][0]) + // S-stage Mega page + if (tags_q[i].is_page[1][0]) lu_gpaddr_o[12+CVA6Cfg.VpnLen/CVA6Cfg.PtLevels-1:12] = lu_vaddr_i[12+CVA6Cfg.VpnLen/CVA6Cfg.PtLevels-1:12]; + // S-stage Giga page + if (tags_q[i].is_page[0][0]) + lu_gpaddr_o[12+2*CVA6Cfg.VpnLen/CVA6Cfg.PtLevels-1:12] = lu_vaddr_i[12+2*(CVA6Cfg.VpnLen/CVA6Cfg.PtLevels)-1:12]; end else begin - lu_gpaddr_o =CVA6Cfg.GPLEN'(lu_vaddr_i[(CVA6Cfg.XLEN == 32 ? CVA6Cfg.VLEN: CVA6Cfg.GPLEN)-1:0]); + lu_gpaddr_o = CVA6Cfg.GPLEN'(lu_vaddr_i[(CVA6Cfg.XLEN == 32?CVA6Cfg.VLEN:CVA6Cfg.GPLEN)-1:0]); end - end - if (|level_match[i]) begin - lu_is_page_o = is_page_o[i]; - lu_content_o = content_q[i].pte; - lu_hit_o = 1'b1; - lu_hit[i] = 1'b1; - - if (CVA6Cfg.RVH) begin - // Compute G-Stage PPN based on the gpaddr - g_content = content_q[i].gpte; - if (tags_q[i].is_page[HYP_EXT][HYP_EXT]) g_content.ppn[8:0] = lu_gpaddr_o[20:12]; - if (tags_q[i].is_page[0][HYP_EXT]) g_content.ppn[17:0] = lu_gpaddr_o[29:12]; - // Output G-stage and S-stage content - lu_g_content_o = level_match[i][CVA6Cfg.PtLevels-1] ? content_q[i].gpte : g_content; + // G-translation (if requested), depending on `content[i].gpte` page type + if (g_st_enbl_i) begin + // Compute G-Stage PPN based on the GPA + // in case of mega/giga pages, GPTE PPN must be aligned so + // here again, we should not overwrite useful bits. + lu_g_content_o = content_q[i].gpte; + // Mega page + if (tags_q[i].is_page[1][HYP_EXT]) + lu_g_content_o.ppn[(CVA6Cfg.VpnLen/CVA6Cfg.PtLevels)-1:0] = lu_gpaddr_o[12+(CVA6Cfg.VpnLen/CVA6Cfg.PtLevels)-1:12]; + // Giga page + if (tags_q[i].is_page[0][HYP_EXT]) + lu_g_content_o.ppn[2*(CVA6Cfg.VpnLen/CVA6Cfg.PtLevels)-1:0] = lu_gpaddr_o[12+2*(CVA6Cfg.VpnLen/CVA6Cfg.PtLevels)-1:12]; end end end end end - logic [HYP_EXT:0]asid_to_be_flushed_is0; // indicates that the ASID provided by SFENCE.VMA (rs2)is 0, active high + logic [HYP_EXT:0] asid_to_be_flushed_is0; // indicates that the ASID provided by SFENCE.VMA (rs2)is 0, active high logic [HYP_EXT:0] vaddr_to_be_flushed_is0; // indicates that the VADDR provided by SFENCE.VMA (rs1)is 0, active high logic vmid_to_be_flushed_is0; // indicates that the VMID provided is 0, active high logic gpaddr_to_be_flushed_is0; // indicates that the GPADDR provided is 0, active high @@ -220,17 +261,16 @@ module cva6_tlb content_n = content_q; for (int unsigned i = 0; i < TLB_ENTRIES; i++) begin - - if (CVA6Cfg.RVH) begin - if (tags_q[i].v_st_enbl[0]) begin gppn[i] = content_q[i].pte.ppn[(CVA6Cfg.GPPNW-1):0]; - if (tags_q[i].is_page[HYP_EXT][0]) + // Mega Page + if (tags_q[i].is_page[1][0]) gppn[i][CVA6Cfg.VpnLen/CVA6Cfg.PtLevels-1:0] = tags_q[i].vpn[0]; + // Giga Page if (tags_q[i].is_page[0][0]) gppn[i][2*(CVA6Cfg.VpnLen/CVA6Cfg.PtLevels)-1:0] = { - tags_q[i].vpn[HYP_EXT], tags_q[i].vpn[0] + tags_q[i].vpn[1], tags_q[i].vpn[0] }; end else begin gppn[i][CVA6Cfg.VpnLen-1:0] = CVA6Cfg.VpnLen'(tags_q[i].vpn); @@ -286,10 +326,11 @@ module cva6_tlb end // normal replacement end else if (update_i.valid & replace_en[i] & !lu_hit_o) begin - //update tag + // update tag tags_n[i] = { update_i.asid, update_i.vmid, + // Zero-extended VPN to fit the tag width ((CVA6Cfg.PtLevels + HYP_EXT) * (CVA6Cfg.VpnLen / CVA6Cfg.PtLevels))'(update_i.vpn), update_i.is_page, update_i.v_st_enbl, @@ -311,27 +352,27 @@ module cva6_tlb // The PLRU-tree indexing: // lvl0 0 // / \ -// / \ -// lvl1 1 2 -// / \ / \ -// lvl2 3 4 5 6 -// / \ /\/\ /\ -// ... ... ... ... -// Just predefine which nodes will be set/cleared -// E.g. for a TLB with 8 entries, the for-loop is semantically -// equivalent to the following pseudo-code: -// unique case (1'b1) -// lu_hit[7]: plru_tree_n[0, 2, 6] = {1, 1, 1}; -// lu_hit[6]: plru_tree_n[0, 2, 6] = {1, 1, 0}; -// lu_hit[5]: plru_tree_n[0, 2, 5] = {1, 0, 1}; -// lu_hit[4]: plru_tree_n[0, 2, 5] = {1, 0, 0}; -// lu_hit[3]: plru_tree_n[0, 1, 4] = {0, 1, 1}; -// lu_hit[2]: plru_tree_n[0, 1, 4] = {0, 1, 0}; -// lu_hit[1]: plru_tree_n[0, 1, 3] = {0, 0, 1}; -// lu_hit[0]: plru_tree_n[0, 1, 3] = {0, 0, 0}; -// default: begin /* No hit */ end -// endcase -for ( + // / \ + // lvl1 1 2 + // / \ / \ + // lvl2 3 4 5 6 + // / \ /\/\ /\ + // ... ... ... ... + // Just predefine which nodes will be set/cleared + // E.g. for a TLB with 8 entries, the for-loop is semantically + // equivalent to the following pseudo-code: + // unique case (1'b1) + // lu_hit[7]: plru_tree_n[0, 2, 6] = {1, 1, 1}; + // lu_hit[6]: plru_tree_n[0, 2, 6] = {1, 1, 0}; + // lu_hit[5]: plru_tree_n[0, 2, 5] = {1, 0, 1}; + // lu_hit[4]: plru_tree_n[0, 2, 5] = {1, 0, 0}; + // lu_hit[3]: plru_tree_n[0, 1, 4] = {0, 1, 1}; + // lu_hit[2]: plru_tree_n[0, 1, 4] = {0, 1, 0}; + // lu_hit[1]: plru_tree_n[0, 1, 3] = {0, 0, 1}; + // lu_hit[0]: plru_tree_n[0, 1, 3] = {0, 0, 0}; + // default: begin /* No hit */ end + // endcase + for ( int unsigned i = 0; i < TLB_ENTRIES; i++ ) begin automatic int unsigned idx_base, shift, new_index; @@ -395,12 +436,11 @@ for ( plru_tree_q <= plru_tree_n; end end + //-------------- // Sanity checks //-------------- - //pragma translate_off - initial begin : p_assertions assert ((TLB_ENTRIES % 2 == 0) && (TLB_ENTRIES > 1)) else begin