From 366a96a0fc05f6a0031e1fa61734368eb0e33e6f Mon Sep 17 00:00:00 2001 From: Ross Thompson Date: Tue, 28 Mar 2023 14:47:08 -0500 Subject: [PATCH 1/6] Possible fix for issue 148. I found the problem. We use a Committed(F/M) signal to indicate the IFU or LSU has an ongoing cache or bus transaction and should not be interrupted. At the time of the mret, the IFU is fetching uncacheable invalid instructions asserting CommittedF. As the IFU finishes the request it unstalls the pipeline but continues to assert CommittedF. (This is not necessary for the IFU). In the same cycle the LSU d cache misses. Because CommittedF is blocking the interrupt the d cache submits a cache line fetch to the EBU. I am thinking out loud here. At it's core the Committed(F/M) ensure memory operations are atomic and caches don't get into inconsistent states. Once the memory operation is completed the LSU/IFU removes the stall but continues to hold Committed(F/M) because the memory operation has completed and it would be wrong to allow an interrupt to occur with a completed load/store. However this is not true of the IFU. If we lower CommittedF once the operation is complete then this problem is solved. The interrupt won't be masked and the LSU will flush the d cache miss. This requires a minor change in the cachebusfsm and cachefsm. I will report back after I've confirmed this works. --- src/cache/cachefsm.sv | 2 +- src/ebu/ahbcacheinterface.sv | 5 +++-- src/ebu/buscachefsm.sv | 5 +++-- src/ifu/ifu.sv | 2 +- src/lsu/lsu.sv | 2 +- 5 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/cache/cachefsm.sv b/src/cache/cachefsm.sv index c51257be7..cd1d43c55 100644 --- a/src/cache/cachefsm.sv +++ b/src/cache/cachefsm.sv @@ -135,7 +135,7 @@ module cachefsm #(parameter READ_ONLY_CACHE = 0) ( end // com back to CPU - assign CacheCommitted = CurrState != STATE_READY; + assign CacheCommitted = (CurrState != STATE_READY) & ~(READ_ONLY_CACHE & CurrState == STATE_READ_HOLD); assign CacheStall = (CurrState == STATE_READY & (FlushCache | AnyMiss)) | (CurrState == STATE_FETCH) | (CurrState == STATE_WRITEBACK) | diff --git a/src/ebu/ahbcacheinterface.sv b/src/ebu/ahbcacheinterface.sv index b30a15096..e2e7d3696 100644 --- a/src/ebu/ahbcacheinterface.sv +++ b/src/ebu/ahbcacheinterface.sv @@ -33,7 +33,8 @@ module ahbcacheinterface #( parameter BEATSPERLINE, // Number of AHBW words (beats) in cacheline parameter AHBWLOGBWPL, // Log2 of ^ parameter LINELEN, // Number of bits in cacheline - parameter LLENPOVERAHBW // Number of AHB beats in a LLEN word. AHBW cannot be larger than LLEN. (implementation limitation) + parameter LLENPOVERAHBW, // Number of AHB beats in a LLEN word. AHBW cannot be larger than LLEN. (implementation limitation) + parameter READ_ONLY_CACHE )( input logic HCLK, HRESETn, // bus interface controls @@ -115,7 +116,7 @@ module ahbcacheinterface #( flopen #(`AHBW/8) HWSTRBReg(HCLK, HREADY, BusByteMaskM[`AHBW/8-1:0], HWSTRB); - buscachefsm #(BeatCountThreshold, AHBWLOGBWPL) AHBBuscachefsm( + buscachefsm #(BeatCountThreshold, AHBWLOGBWPL, READ_ONLY_CACHE) AHBBuscachefsm( .HCLK, .HRESETn, .Flush, .BusRW, .Stall, .BusCommitted, .BusStall, .CaptureEn, .SelBusBeat, .CacheBusRW, .CacheBusAck, .BeatCount, .BeatCountDelayed, .HREADY, .HTRANS, .HWRITE, .HBURST); diff --git a/src/ebu/buscachefsm.sv b/src/ebu/buscachefsm.sv index c619c9135..e0efcf3a3 100644 --- a/src/ebu/buscachefsm.sv +++ b/src/ebu/buscachefsm.sv @@ -33,7 +33,8 @@ // HCLK and clk must be the same clock! module buscachefsm #( parameter BeatCountThreshold, // Largest beat index - parameter AHBWLOGBWPL // Log2 of BEATSPERLINE + parameter AHBWLOGBWPL, // Log2 of BEATSPERLINE + parameter READ_ONLY_CACHE )( input logic HCLK, input logic HRESETn, @@ -121,7 +122,7 @@ module buscachefsm #( (CurrState == DATA_PHASE) | (CurrState == CACHE_FETCH & ~HREADY) | (CurrState == CACHE_WRITEBACK & ~HREADY); - assign BusCommitted = CurrState != ADR_PHASE; + assign BusCommitted = (CurrState != ADR_PHASE) & ~(READ_ONLY_CACHE & CurrState == MEM3); // AHB bus interface assign HTRANS = (CurrState == ADR_PHASE & HREADY & ((|BusRW) | (|CacheBusRW)) & ~Flush) | diff --git a/src/ifu/ifu.sv b/src/ifu/ifu.sv index 2a411a737..c1556daeb 100644 --- a/src/ifu/ifu.sv +++ b/src/ifu/ifu.sv @@ -251,7 +251,7 @@ module ifu ( .NextSet(PCSpillNextF[11:0]), .PAdr(PCPF), .CacheCommitted(CacheCommittedF), .InvalidateCache(InvalidateICacheM)); - ahbcacheinterface #(WORDSPERLINE, LOGBWPL, LINELEN, LLENPOVERAHBW) + ahbcacheinterface #(WORDSPERLINE, LOGBWPL, LINELEN, LLENPOVERAHBW, 1) ahbcacheinterface(.HCLK(clk), .HRESETn(~reset), .HRDATA, .Flush(FlushD), .CacheBusRW, .HSIZE(IFUHSIZE), .HBURST(IFUHBURST), .HTRANS(IFUHTRANS), .HWSTRB(), diff --git a/src/lsu/lsu.sv b/src/lsu/lsu.sv index e3adc00f5..0b0bc81e3 100644 --- a/src/lsu/lsu.sv +++ b/src/lsu/lsu.sv @@ -275,7 +275,7 @@ module lsu ( .FetchBuffer, .CacheBusRW, .CacheBusAck(DCacheBusAck), .InvalidateCache(1'b0)); - ahbcacheinterface #(.BEATSPERLINE(BEATSPERLINE), .AHBWLOGBWPL(AHBWLOGBWPL), .LINELEN(LINELEN), .LLENPOVERAHBW(LLENPOVERAHBW)) ahbcacheinterface( + ahbcacheinterface #(.BEATSPERLINE(BEATSPERLINE), .AHBWLOGBWPL(AHBWLOGBWPL), .LINELEN(LINELEN), .LLENPOVERAHBW(LLENPOVERAHBW), .READ_ONLY_CACHE(0)) ahbcacheinterface( .HCLK(clk), .HRESETn(~reset), .Flush(FlushW), .HRDATA, .HWDATA(LSUHWDATA), .HWSTRB(LSUHWSTRB), .HSIZE(LSUHSIZE), .HBURST(LSUHBURST), .HTRANS(LSUHTRANS), .HWRITE(LSUHWRITE), .HREADY(LSUHREADY), From a5601ea264ee7a1a1d491f50db8dfc6cefbe035e Mon Sep 17 00:00:00 2001 From: Ross Thompson Date: Tue, 28 Mar 2023 16:09:54 -0500 Subject: [PATCH 2/6] Now have logging of i/d cache addresses, but the performance counter reports are x's. --- testbench/testbench.sv | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/testbench/testbench.sv b/testbench/testbench.sv index f07cfef12..0afb4310b 100644 --- a/testbench/testbench.sv +++ b/testbench/testbench.sv @@ -474,7 +474,7 @@ logic [3:0] dummy; // default start condiction is reset // default end condiction is end of test (DCacheFlushDone) assign StartSampleFirst = InReset; - flopr #(1) StartSampleReg(clk, reset, StartSampleFirst, StartSampleDelayed); + flop #(1) StartSampleReg(clk, StartSampleFirst, StartSampleDelayed); assign StartSample = StartSampleFirst & ~ StartSampleDelayed; assign EndSample = DCacheFlushStart & ~DCacheFlushDone; @@ -555,16 +555,19 @@ end int file; string LogFile; logic resetD, resetEdge; + logic Enable; + assign Enable = ~dut.core.StallD & ~dut.core.FlushD & dut.core.ifu.bus.icache.CacheRWF[1] & ~reset; flop #(1) ResetDReg(clk, reset, resetD); assign resetEdge = ~reset & resetD; initial begin LogFile = $psprintf("ICache.log"); file = $fopen(LogFile, "w"); + $fwrite(file, "BEGIN %s\n", memfilename); end always @(posedge clk) begin if(resetEdge) $fwrite(file, "TRAIN\n"); if(StartSample) $fwrite(file, "BEGIN %s\n", memfilename); - if(~dut.core.StallD & ~dut.core.FlushD) begin + if(Enable) begin // only log i cache reads $fwrite(file, "%h R\n", dut.core.ifu.PCPF); end if(EndSample) $fwrite(file, "END %s\n", memfilename); @@ -580,6 +583,7 @@ end initial begin LogFile = $psprintf("DCache.log"); file = $fopen(LogFile, "w"); + $fwrite(file, "BEGIN %s\n", memfilename); end always @(posedge clk) begin if(resetEdge) $fwrite(file, "TRAIN\n"); From ef2660068945fe00b27e6040d02da6bae310fd30 Mon Sep 17 00:00:00 2001 From: Ross Thompson Date: Tue, 28 Mar 2023 16:15:05 -0500 Subject: [PATCH 3/6] Restored performance counter reports. --- testbench/testbench.sv | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/testbench/testbench.sv b/testbench/testbench.sv index 0afb4310b..1af6f2b13 100644 --- a/testbench/testbench.sv +++ b/testbench/testbench.sv @@ -169,7 +169,8 @@ logic [3:0] dummy; logic InitializingMemories; integer ResetCount, ResetThreshold; logic InReset; - + logic Begin; + // instantiate device to be tested assign GPIOIN = 0; assign UARTSin = 1; @@ -417,7 +418,7 @@ logic [3:0] dummy; if(`PrintHPMCounters & `ZICOUNTERS_SUPPORTED) begin : HPMCSample integer HPMCindex; logic StartSampleFirst; - logic StartSampleDelayed; + logic StartSampleDelayed, BeginDelayed; logic EndSampleFirst, EndSampleDelayed; logic [`XLEN-1:0] InitialHPMCOUNTERH[`COUNTERS-1:0]; @@ -474,10 +475,13 @@ logic [3:0] dummy; // default start condiction is reset // default end condiction is end of test (DCacheFlushDone) assign StartSampleFirst = InReset; - flop #(1) StartSampleReg(clk, StartSampleFirst, StartSampleDelayed); + flopr #(1) StartSampleReg(clk, reset, StartSampleFirst, StartSampleDelayed); assign StartSample = StartSampleFirst & ~ StartSampleDelayed; - assign EndSample = DCacheFlushStart & ~DCacheFlushDone; + + flop #(1) BeginReg(clk, StartSampleFirst, BeginDelayed); + assign Begin = StartSampleFirst & ~ BeginDelayed; + end always @(negedge clk) begin @@ -566,7 +570,7 @@ end end always @(posedge clk) begin if(resetEdge) $fwrite(file, "TRAIN\n"); - if(StartSample) $fwrite(file, "BEGIN %s\n", memfilename); + if(Begin) $fwrite(file, "BEGIN %s\n", memfilename); if(Enable) begin // only log i cache reads $fwrite(file, "%h R\n", dut.core.ifu.PCPF); end @@ -587,7 +591,7 @@ end end always @(posedge clk) begin if(resetEdge) $fwrite(file, "TRAIN\n"); - if(StartSample) $fwrite(file, "BEGIN %s\n", memfilename); + if(Begin) $fwrite(file, "BEGIN %s\n", memfilename); if(~dut.core.StallW & ~dut.core.FlushW & dut.core.InstrValidM) begin if(dut.core.lsu.bus.dcache.CacheRWM == 2'b10) begin $fwrite(file, "%h R\n", dut.core.lsu.PAdrM); From 650a1a8d7ea6b29ee9fbc54137fe3077bc8a05c9 Mon Sep 17 00:00:00 2001 From: Ross Thompson Date: Tue, 28 Mar 2023 16:20:14 -0500 Subject: [PATCH 4/6] Now reports if there is a hit or miss. --- testbench/testbench.sv | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/testbench/testbench.sv b/testbench/testbench.sv index 1af6f2b13..aa11d377c 100644 --- a/testbench/testbench.sv +++ b/testbench/testbench.sv @@ -568,11 +568,13 @@ end file = $fopen(LogFile, "w"); $fwrite(file, "BEGIN %s\n", memfilename); end + string HitMissString; + assign HitMissString = dut.core.ifu.bus.icache.icache.CacheHit ? "H" : "M"; always @(posedge clk) begin if(resetEdge) $fwrite(file, "TRAIN\n"); if(Begin) $fwrite(file, "BEGIN %s\n", memfilename); if(Enable) begin // only log i cache reads - $fwrite(file, "%h R\n", dut.core.ifu.PCPF); + $fwrite(file, "%h R %s\n", dut.core.ifu.PCPF, HitMissString); end if(EndSample) $fwrite(file, "END %s\n", memfilename); end @@ -582,8 +584,10 @@ end int file; string LogFile; logic resetD, resetEdge; + string HitMissString; flop #(1) ResetDReg(clk, reset, resetD); assign resetEdge = ~reset & resetD; + assign HitMissString = dut.core.lsu.bus.dcache.dcache.CacheHit ? "H" : "M"; initial begin LogFile = $psprintf("DCache.log"); file = $fopen(LogFile, "w"); @@ -594,13 +598,13 @@ end if(Begin) $fwrite(file, "BEGIN %s\n", memfilename); if(~dut.core.StallW & ~dut.core.FlushW & dut.core.InstrValidM) begin if(dut.core.lsu.bus.dcache.CacheRWM == 2'b10) begin - $fwrite(file, "%h R\n", dut.core.lsu.PAdrM); + $fwrite(file, "%h R %s\n", dut.core.lsu.PAdrM, HitMissString); end else if (dut.core.lsu.bus.dcache.CacheRWM == 2'b01) begin - $fwrite(file, "%h W\n", dut.core.lsu.PAdrM); + $fwrite(file, "%h W %s\n", dut.core.lsu.PAdrM, HitMissString); end else if (dut.core.lsu.bus.dcache.CacheAtomicM[1] == 1'b1) begin // *** This may change - $fwrite(file, "%h A\n", dut.core.lsu.PAdrM); + $fwrite(file, "%h A %s\n", dut.core.lsu.PAdrM, HitMissString); end else if (dut.core.lsu.bus.dcache.FlushDCache) begin - $fwrite(file, "%h F\n", dut.core.lsu.PAdrM); + $fwrite(file, "%h F %s\n", dut.core.lsu.PAdrM, HitMissString); end end if(EndSample) $fwrite(file, "END %s\n", memfilename); From c65c9e52d49c46ddd6cf02624f7b89800dcede4f Mon Sep 17 00:00:00 2001 From: Ross Thompson Date: Tue, 28 Mar 2023 16:20:45 -0500 Subject: [PATCH 5/6] Disable loggers by default. --- testbench/testbench.sv | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testbench/testbench.sv b/testbench/testbench.sv index aa11d377c..cc0f0a241 100644 --- a/testbench/testbench.sv +++ b/testbench/testbench.sv @@ -29,9 +29,9 @@ `include "tests.vh" `define PrintHPMCounters 1 -`define BPRED_LOGGER 1 -`define I_CACHE_ADDR_LOGGER 1 -`define D_CACHE_ADDR_LOGGER 1 +`define BPRED_LOGGER 0 +`define I_CACHE_ADDR_LOGGER 0 +`define D_CACHE_ADDR_LOGGER 0 module testbench; parameter DEBUG=0; From 84860a062dc688c0783c53d6b8ec9b3521f10b9c Mon Sep 17 00:00:00 2001 From: Ross Thompson Date: Tue, 28 Mar 2023 16:27:54 -0500 Subject: [PATCH 6/6] Modified the testbench to not use the loggers for unsupported configurations. --- testbench/testbench.sv | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/testbench/testbench.sv b/testbench/testbench.sv index cc0f0a241..4372fd4e5 100644 --- a/testbench/testbench.sv +++ b/testbench/testbench.sv @@ -532,7 +532,7 @@ logic [3:0] dummy; // initialize the branch predictor - if (`BPRED_SUPPORTED == 1) begin + if (`BPRED_SUPPORTED) begin integer adrindex; always @(*) begin @@ -555,7 +555,7 @@ logic [3:0] dummy; end - if (`I_CACHE_ADDR_LOGGER == 1) begin + if (`ICACHE_SUPPORTED && `I_CACHE_ADDR_LOGGER) begin int file; string LogFile; logic resetD, resetEdge; @@ -580,7 +580,7 @@ end end end - if (`D_CACHE_ADDR_LOGGER == 1) begin + if (`DCACHE_SUPPORTED && `D_CACHE_ADDR_LOGGER) begin int file; string LogFile; logic resetD, resetEdge; @@ -611,7 +611,7 @@ end end end - if (`BPRED_SUPPORTED == 1) begin + if (`BPRED_SUPPORTED) begin if (`BPRED_LOGGER) begin string direction; int file;