github-mirrors/cva6
1
0
Fork 0
mirror of https://github.com/openhwgroup/cva6.git synced 2025-04-20 04:07:36 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

wb_dcache: Forward atomic transactions to AXI (#777)

Browse source 
* wb_dcache: Forward "atomic transactions" to AXI

* Correct bugs

* Forward LR/SC atomics

* Fix CI

* miss_handler: Route AMO port through arbiter

* axi_adapter: Correct LOAD AMOs handling

Accept read data only after (or together) handshake on B channel

* Restore old ID

* Correct atop encodings

* Correct AMOs AXI ID

* Correct wb_dcache testbench

Previously not comparing AMOs at all! Due to amo_exp_resp being 'x

* Realign and sign extend 32b request rdata

* Use axi_pkg definitions for ATOPs encoding

* Remove whitespace

* wb_dcache: Style corrections

Co-authored-by: Florian Zaruba <florianzaruba@gmail.com>

Co-authored-by: Florian Zaruba <florianzaruba@gmail.com>
This commit is contained in:
Luca Colagrande 2022-03-09 16:33:37 +01:00 • committed by GitHub
parent 062ec9170a
commit 75250868eb
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 365 additions and 259 deletions
Download patch file Download diff file Expand all files Collapse all files

2
Bender.yml
View file

@ -19,7 +19,6 @@ sources:
- core/include/riscv_pkg.sv
- corev_apu/riscv-dbg/src/dm_pkg.sv
- core/include/ariane_pkg.sv
- core/include/std_cache_pkg.sv
- core/include/wt_cache_pkg.sv
- corev_apu/axi/src/axi_pkg.sv
- corev_apu/register_interface/src/reg_intf.sv
@ -28,6 +27,7 @@ sources:
- corev_apu/tb/ariane_soc_pkg.sv
- corev_apu/tb/ariane_axi_soc_pkg.sv
- core/include/ariane_axi_pkg.sv
- core/include/std_cache_pkg.sv
- core/fpu/src/fpnew_pkg.sv
- core/fpu/src/fpu_div_sqrt_mvp/hdl/defs_div_sqrt_mvp.sv
# Stand-alone source files

2
Makefile
View file

@ -90,7 +90,6 @@ ariane_pkg += core/include/riscv_pkg.sv \
corev_apu/riscv-dbg/src/dm_pkg.sv \
core/include/ariane_pkg.sv \
core/include/ariane_rvfi_pkg.sv \
core/include/std_cache_pkg.sv \
core/include/wt_cache_pkg.sv \
core/include/cvxif_pkg.sv \
corev_apu/axi/src/axi_pkg.sv \
@ -100,6 +99,7 @@ ariane_pkg += core/include/riscv_pkg.sv \
corev_apu/tb/ariane_soc_pkg.sv \
corev_apu/tb/ariane_axi_soc_pkg.sv \
core/include/ariane_axi_pkg.sv \
core/include/std_cache_pkg.sv \
core/fpu/src/fpnew_pkg.sv \
common/submodules/common_cells/src/cf_math_pkg.sv \
core/cvxif_example/include/cvxif_instr_pkg.sv \

101
core/axi_adapter.sv
View file

@ -27,8 +27,8 @@ module axi_adapter #(
input logic req_i,
input ariane_axi::ad_req_t type_i,
input ariane_pkg::amo_t amo_i,
output logic gnt_o,
output logic [AXI_ID_WIDTH-1:0] gnt_id_o,
input logic [riscv::XLEN-1:0] addr_i,
input logic we_i,
input logic [(DATA_WIDTH/riscv::XLEN)-1:0][riscv::XLEN-1:0] wdata_i,
@ -51,7 +51,7 @@ module axi_adapter #(
enum logic [3:0] {
IDLE, WAIT_B_VALID, WAIT_AW_READY, WAIT_LAST_W_READY, WAIT_LAST_W_READY_AW_READY, WAIT_AW_READY_BURST,
WAIT_R_VALID, WAIT_R_VALID_MULTIPLE, COMPLETE_READ
WAIT_R_VALID, WAIT_R_VALID_MULTIPLE, COMPLETE_READ, WAIT_AMO_R_VALID
} state_q, state_d;
// counter for AXI transfers
@ -61,6 +61,8 @@ module axi_adapter #(
logic [(DATA_WIDTH/riscv::XLEN)-1:0] addr_offset_d, addr_offset_q;
logic [AXI_ID_WIDTH-1:0] id_d, id_q;
logic [ADDR_INDEX-1:0] index;
// save the atomic operation
ariane_pkg::amo_t amo_d, amo_q;
always_comb begin : axi_fsm
// Default assignments
@ -75,7 +77,7 @@ module axi_adapter #(
axi_req_o.aw.cache = 4'b0;
axi_req_o.aw.qos = 4'b0;
axi_req_o.aw.id = id_i;
axi_req_o.aw.atop = '0; // currently not used
axi_req_o.aw.atop = atop_from_amo(amo_i);
axi_req_o.aw.user = '0;
axi_req_o.ar_valid = 1'b0;
@ -103,7 +105,6 @@ module axi_adapter #(
axi_req_o.r_ready = 1'b0;
gnt_o = 1'b0;
gnt_id_o = id_i;
valid_o = 1'b0;
id_o = axi_resp_i.r.id;
@ -116,6 +117,7 @@ module axi_adapter #(
cache_line_d = cache_line_q;
addr_offset_d = addr_offset_q;
id_d = id_q;
amo_d = amo_q;
index = '0;
case (state_q)
@ -130,6 +132,8 @@ module axi_adapter #(
// the data is valid
axi_req_o.aw_valid = 1'b1;
axi_req_o.w_valid = 1'b1;
// store-conditional requires exclusive access
axi_req_o.aw.lock = amo_i == ariane_pkg::AMO_SC;
// its a single write
if (type_i == ariane_axi::SINGLE_REQ) begin
// only a single write so the data is already the last one
@ -143,8 +147,14 @@ module axi_adapter #(
default: state_d = IDLE;
endcase
if (axi_resp_i.aw_ready) amo_d = amo_i;
// its a request for the whole cache line
end else begin
// TODO colluca: bursts of AMOs unsupported
assert (amo_i == ariane_pkg::AMO_NONE)
else $fatal("Bursts of atomic operations are not supported");
axi_req_o.aw.len = BURST_SIZE; // number of bursts to do
axi_req_o.w.data = wdata_i[0];
axi_req_o.w.strb = be_i[0];
@ -165,8 +175,14 @@ module axi_adapter #(
end else begin
axi_req_o.ar_valid = 1'b1;
// load-reserved requires exclusive access
axi_req_o.ar.lock = amo_i == ariane_pkg::AMO_LR;
gnt_o = axi_resp_i.ar_ready;
if (type_i != ariane_axi::SINGLE_REQ) begin
assert (amo_i == ariane_pkg::AMO_NONE)
else $fatal("Bursts of atomic operations are not supported");
axi_req_o.ar.len = BURST_SIZE;
cnt_d = BURST_SIZE;
end
@ -186,6 +202,7 @@ module axi_adapter #(
if (axi_resp_i.aw_ready) begin
gnt_o = 1'b1;
state_d = WAIT_B_VALID;
amo_d = amo_i;
end
end
@ -264,13 +281,52 @@ module axi_adapter #(
// ~> finish write transaction
WAIT_B_VALID: begin
axi_req_o.b_ready = 1'b1;
id_o = axi_resp_i.b.id;
// Write is valid
if (axi_resp_i.b_valid) begin
state_d = IDLE;
valid_o = 1'b1;
axi_req_o.b_ready = 1'b1;
// some atomics must wait for read data
// we only accept it after accepting bvalid
if (amo_returns_data(amo_q)) begin
if (axi_resp_i.r_valid) begin
// return read data if valid
valid_o = 1'b1;
axi_req_o.r_ready = 1'b1;
state_d = IDLE;
rdata_o = axi_resp_i.r.data;
end else begin
// wait otherwise
state_d = WAIT_AMO_R_VALID;
end
end else begin
valid_o = 1'b1;
state_d = IDLE;
// store-conditional response
if (amo_q == ariane_pkg::AMO_SC) begin
if (axi_resp_i.b.resp == axi_pkg::RESP_EXOKAY) begin
// success -> return 0
rdata_o = 1'b0;
// TODO colluca: replace with else if (... == RESP_OKAY)?
end else begin
// failure -> return 1
rdata_o = 1'b1;
end
end
end
end
end
// ~> some atomics wait for read data
WAIT_AMO_R_VALID: begin
// acknowledge data and terminate atomic
if (axi_resp_i.r_valid) begin
axi_req_o.r_ready = 1'b1;
state_d = IDLE;
valid_o = 1'b1;
rdata_o = axi_resp_i.r.data;
end
end
@ -336,13 +392,44 @@ module axi_adapter #(
cache_line_q <= '0;
addr_offset_q <= '0;
id_q <= '0;
amo_q <= ariane_pkg::AMO_NONE;
end else begin
state_q <= state_d;
cnt_q <= cnt_d;
cache_line_q <= cache_line_d;
addr_offset_q <= addr_offset_d;
id_q <= id_d;
amo_q <= amo_d;
end
end
function automatic axi_pkg::atop_t atop_from_amo(ariane_pkg::amo_t amo);
axi_pkg::atop_t result = 6'b000000;
unique case(amo)
ariane_pkg::AMO_NONE: result = {axi_pkg::ATOP_NONE, 4'b0000};
ariane_pkg::AMO_SWAP: result = {axi_pkg::ATOP_ATOMICSWAP};
ariane_pkg::AMO_ADD : result = {axi_pkg::ATOP_ATOMICLOAD, axi_pkg::ATOP_LITTLE_END, axi_pkg::ATOP_ADD};
ariane_pkg::AMO_AND : result = {axi_pkg::ATOP_ATOMICLOAD, axi_pkg::ATOP_LITTLE_END, axi_pkg::ATOP_CLR};
ariane_pkg::AMO_OR : result = {axi_pkg::ATOP_ATOMICLOAD, axi_pkg::ATOP_LITTLE_END, axi_pkg::ATOP_SET};
ariane_pkg::AMO_XOR : result = {axi_pkg::ATOP_ATOMICLOAD, axi_pkg::ATOP_LITTLE_END, axi_pkg::ATOP_EOR};
ariane_pkg::AMO_MAX : result = {axi_pkg::ATOP_ATOMICLOAD, axi_pkg::ATOP_LITTLE_END, axi_pkg::ATOP_SMAX};
ariane_pkg::AMO_MAXU: result = {axi_pkg::ATOP_ATOMICLOAD, axi_pkg::ATOP_LITTLE_END, axi_pkg::ATOP_UMAX};
ariane_pkg::AMO_MIN : result = {axi_pkg::ATOP_ATOMICLOAD, axi_pkg::ATOP_LITTLE_END, axi_pkg::ATOP_SMIN};
ariane_pkg::AMO_MINU: result = {axi_pkg::ATOP_ATOMICLOAD, axi_pkg::ATOP_LITTLE_END, axi_pkg::ATOP_UMIN};
ariane_pkg::AMO_CAS1: result = {axi_pkg::ATOP_NONE, 4'b0000}; // Unsupported
ariane_pkg::AMO_CAS2: result = {axi_pkg::ATOP_NONE, 4'b0000}; // Unsupported
default: result = 6'b000000;
endcase
return result;
endfunction
function automatic logic amo_returns_data(ariane_pkg::amo_t amo);
axi_pkg::atop_t atop = atop_from_amo(amo);
logic is_load = atop[5:4] == axi_pkg::ATOP_ATOMICLOAD;
logic is_swap_or_cmp = atop[5:4] == axi_pkg::ATOP_ATOMICSWAP[5:4];
return is_load || is_swap_or_cmp;
endfunction
endmodule

424
core/cache_subsystem/miss_handler.sv
View file

@ -60,6 +60,9 @@ module miss_handler import ariane_pkg::*; import std_cache_pkg::*; #(
output logic we_o
);
// Three MSHR ports + AMO port
parameter NR_BYPASS_PORTS = NR_PORTS + 1;
// FSM states
enum logic [3:0] {
IDLE, // 0
@ -74,9 +77,8 @@ module miss_handler import ariane_pkg::*; import std_cache_pkg::*; #(
MISS_REPL, // 9
SAVE_CACHELINE, // A
INIT, // B
AMO_LOAD, // C
AMO_SAVE_LOAD, // D
AMO_STORE // E
AMO_REQ, // C
AMO_WAIT_RESP // D
} state_d, state_q;
// Registers
@ -96,6 +98,18 @@ module miss_handler import ariane_pkg::*; import std_cache_pkg::*; #(
logic [NR_PORTS-1:0][7:0] miss_req_be;
logic [NR_PORTS-1:0][1:0] miss_req_size;
// Bypass AMO port
bypass_req_t amo_bypass_req;
bypass_rsp_t amo_bypass_rsp;
// Bypass ports <-> Arbiter
bypass_req_t [NR_BYPASS_PORTS-1:0] bypass_ports_req;
bypass_rsp_t [NR_BYPASS_PORTS-1:0] bypass_ports_rsp;
// Arbiter <-> Bypass AXI adapter
bypass_req_t bypass_adapter_req;
bypass_rsp_t bypass_adapter_rsp;
// Cache Line Refill <-> AXI
logic req_fsm_miss_valid;
logic [63:0] req_fsm_miss_addr;
@ -115,12 +129,7 @@ module miss_handler import ariane_pkg::*; import std_cache_pkg::*; #(
logic [$clog2(DCACHE_SET_ASSOC-1)-1:0] lfsr_bin;
// AMOs
ariane_pkg::amo_t amo_op;
logic [63:0] amo_operand_a, amo_operand_b, amo_result_o;
struct packed {
logic [63:3] address;
logic valid;
} reservation_d, reservation_q;
logic [63:0] amo_operand_b;
// ------------------------------
// Cache Management
@ -154,6 +163,16 @@ module miss_handler import ariane_pkg::*; import std_cache_pkg::*; #(
req_fsm_miss_be = '0;
req_fsm_miss_req = ariane_axi::CACHE_LINE_REQ;
req_fsm_miss_size = 2'b11;
// to AXI bypass
amo_bypass_req.req = 1'b0;
amo_bypass_req.reqtype = ariane_axi::SINGLE_REQ;
amo_bypass_req.amo = ariane_pkg::AMO_NONE;
amo_bypass_req.addr = '0;
amo_bypass_req.we = 1'b0;
amo_bypass_req.wdata = '0;
amo_bypass_req.be = '0;
amo_bypass_req.size = 2'b11;
amo_bypass_req.id = 4'b1011;
// core
flush_ack_o = 1'b0;
miss_o = 1'b0; // to performance counter
@ -171,12 +190,8 @@ module miss_handler import ariane_pkg::*; import std_cache_pkg::*; #(
// AMOs
amo_resp_o.ack = 1'b0;
amo_resp_o.result = '0;
// silence the unit when not used
amo_op = amo_req_i.amo_op;
amo_operand_a = '0;
amo_operand_b = '0;
reservation_d = reservation_q;
case (state_q)
IDLE: begin
@ -189,7 +204,7 @@ module miss_handler import ariane_pkg::*; import std_cache_pkg::*; #(
cnt_d = '0;
// 2. Do the AMO
end else begin
state_d = AMO_LOAD;
state_d = AMO_REQ;
serve_amo_d = 1'b0;
end
end
@ -377,89 +392,71 @@ module miss_handler import ariane_pkg::*; import std_cache_pkg::*; #(
// ----------------------
// AMOs
// ----------------------
// TODO(zarubaf) Move this closer to memory
// ~> we are here because we need to do the AMO, the cache is clean at this point
// start by executing the load
AMO_LOAD: begin
req_fsm_miss_valid = 1'b1;
AMO_REQ: begin
amo_bypass_req.req = 1'b1;
amo_bypass_req.reqtype = ariane_axi::SINGLE_REQ;
amo_bypass_req.amo = amo_req_i.amo_op;
// address is in operand a
req_fsm_miss_addr = amo_req_i.operand_a;
req_fsm_miss_req = ariane_axi::SINGLE_REQ;
req_fsm_miss_size = amo_req_i.size;
// the request has been granted
if (gnt_miss_fsm) begin
state_d = AMO_SAVE_LOAD;
amo_bypass_req.addr = amo_req_i.operand_a;
if (amo_req_i.amo_op != AMO_LR) begin
amo_bypass_req.we = 1'b1;
end
end
// save the load value
AMO_SAVE_LOAD: begin
if (valid_miss_fsm) begin
// we are only concerned about the lower 64-bit
mshr_d.wdata = data_miss_fsm[0];
state_d = AMO_STORE;
end
end
// and do the store
AMO_STORE: begin
automatic logic [63:0] load_data;
// re-align load data
load_data = data_align(amo_req_i.operand_a[2:0], mshr_q.wdata);
// Sign-extend for word operation
if (amo_req_i.size == 2'b10) begin
amo_operand_a = sext32(load_data[31:0]);
amo_operand_b = sext32(amo_req_i.operand_b[31:0]);
amo_bypass_req.size = amo_req_i.size;
// AXI implements CLR op instead of AND, negate operand
if (amo_req_i.amo_op == AMO_AND) begin
amo_operand_b = ~amo_req_i.operand_b;
end else begin
amo_operand_a = load_data;
amo_operand_b = amo_req_i.operand_b;
end
// we do not need a store request for load reserved or a failing store conditional
// we can bail-out without making any further requests
if (amo_req_i.amo_op == AMO_LR ||
(amo_req_i.amo_op == AMO_SC &&
((reservation_q.valid && reservation_q.address != amo_req_i.operand_a[63:3]) || !reservation_q.valid))) begin
req_fsm_miss_valid = 1'b0;
state_d = IDLE;
amo_resp_o.ack = 1'b1;
// write-back the result
amo_resp_o.result = amo_operand_a;
// we know that the SC failed
if (amo_req_i.amo_op == AMO_SC) begin
amo_resp_o.result = 1'b1;
// also clear the reservation
reservation_d.valid = 1'b0;
end
// align data and byte-enable to correct byte lanes
amo_bypass_req.wdata = amo_operand_b;
if (amo_req_i.size == 2'b11) begin
// 64b transfer
amo_bypass_req.be = 8'b11111111;
end else begin
req_fsm_miss_valid = 1'b1;
// 32b transfer
if (amo_req_i.operand_a[2:0] == '0) begin
// 64b aligned -> activate lower 4 byte lanes
amo_bypass_req.be = 8'b00001111;
end else begin
// 64b unaligned -> activate upper 4 byte lanes
amo_bypass_req.be = 8'b11110000;
amo_bypass_req.wdata = amo_operand_b[31:0] << 32;
end
end
req_fsm_miss_we = 1'b1;
req_fsm_miss_req = ariane_axi::SINGLE_REQ;
req_fsm_miss_size = amo_req_i.size;
req_fsm_miss_addr = amo_req_i.operand_a;
req_fsm_miss_wdata = data_align(amo_req_i.operand_a[2:0], amo_result_o);
req_fsm_miss_be = be_gen(amo_req_i.operand_a[2:0], amo_req_i.size);
// place a reservation on the memory
if (amo_req_i.amo_op == AMO_LR) begin
reservation_d.address = amo_req_i.operand_a[63:3];
reservation_d.valid = 1'b1;
// when request is accepted, wait for response
if (amo_bypass_rsp.gnt) begin
if (amo_bypass_rsp.valid) begin
state_d = IDLE;
amo_resp_o.ack = 1'b1;
amo_resp_o.result = amo_bypass_rsp.rdata;
end else begin
state_d = AMO_WAIT_RESP;
end
end
// the request is valid or we didn't need to go for another store
if (valid_miss_fsm) begin
end
AMO_WAIT_RESP: begin
if (amo_bypass_rsp.valid) begin
state_d = IDLE;
amo_resp_o.ack = 1'b1;
// write-back the result
amo_resp_o.result = amo_operand_a;
if (amo_req_i.amo_op == AMO_SC) begin
amo_resp_o.result = 1'b0;
// An SC must fail if there is a nother SC (to any address) between the LR and the SC in program
// order (even to the same address).
// in any case destory the reservation
reservation_d.valid = 1'b0;
// Request is assumed to be still valid (ack not granted yet)
if (amo_req_i.size == 2'b10) begin
// 32b request
logic [31:0] halfword;
if (amo_req_i.operand_a[2:0] == '0) begin
// 64b aligned -> activate lower 4 byte lanes
halfword = amo_bypass_rsp.rdata[31:0];
end else begin
// 64b unaligned -> activate upper 4 byte lanes
halfword = amo_bypass_rsp.rdata[63:32];
end
// Sign-extend 32b requests as per RISC-V spec
amo_resp_o.result = {{32{halfword[31]}}, halfword};
end else begin
// 64b request
amo_resp_o.result = amo_bypass_rsp.rdata;
end
end
end
@ -495,7 +492,6 @@ module miss_handler import ariane_pkg::*; import std_cache_pkg::*; #(
evict_way_q <= '0;
evict_cl_q <= '0;
serve_amo_q <= 1'b0;
reservation_q <= '0;
end else begin
mshr_q <= mshr_d;
state_q <= state_d;
@ -503,7 +499,6 @@ module miss_handler import ariane_pkg::*; import std_cache_pkg::*; #(
evict_way_q <= evict_way_d;
evict_cl_q <= evict_cl_d;
serve_amo_q <= serve_amo_d;
reservation_q <= reservation_d;
end
end
@ -514,77 +509,80 @@ module miss_handler import ariane_pkg::*; import std_cache_pkg::*; #(
@(posedge clk_i) $onehot0(evict_way_q)) else $warning("Evict-way should be one-hot encoded");
`endif
//pragma translate_on
// ----------------------
// Bypass Arbiter
// ----------------------
// Connection Arbiter <-> AXI
logic req_fsm_bypass_valid;
logic [63:0] req_fsm_bypass_addr;
logic [63:0] req_fsm_bypass_wdata;
logic req_fsm_bypass_we;
logic [7:0] req_fsm_bypass_be;
logic [1:0] req_fsm_bypass_size;
logic gnt_bypass_fsm;
logic valid_bypass_fsm;
logic [63:0] data_bypass_fsm;
logic [$clog2(NR_PORTS)-1:0] id_fsm_bypass;
logic [3:0] id_bypass_fsm;
logic [3:0] gnt_id_bypass_fsm;
arbiter #(
.NR_PORTS ( NR_PORTS ),
.DATA_WIDTH ( 64 )
// ----------------------
// Pack bypass ports
// ----------------------
always_comb begin
logic [$clog2(NR_BYPASS_PORTS)-1:0] id;
// Pack MHSR ports first
for (id = 0; id < NR_PORTS; id++) begin
bypass_ports_req[id].req = miss_req_valid[id] & miss_req_bypass[id];
bypass_ports_req[id].reqtype = ariane_axi::SINGLE_REQ;
bypass_ports_req[id].amo = AMO_NONE;
bypass_ports_req[id].id = {2'b10, id};
bypass_ports_req[id].addr = miss_req_addr[id];
bypass_ports_req[id].wdata = miss_req_wdata[id];
bypass_ports_req[id].we = miss_req_we[id];
bypass_ports_req[id].be = miss_req_be[id];
bypass_ports_req[id].size = miss_req_size[id];
bypass_gnt_o[id] = bypass_ports_rsp[id].gnt;
bypass_valid_o[id] = bypass_ports_rsp[id].valid;
bypass_data_o[id] = bypass_ports_rsp[id].rdata;
end
// AMO port has lowest priority
bypass_ports_req[id] = amo_bypass_req;
amo_bypass_rsp = bypass_ports_rsp[id];
end
// ----------------------
// Arbitrate bypass ports
// ----------------------
axi_adapter_arbiter #(
.NR_PORTS(NR_BYPASS_PORTS),
.req_t (bypass_req_t),
.rsp_t (bypass_rsp_t)
) i_bypass_arbiter (
.clk_i (clk_i),
.rst_ni(rst_ni),
// Master Side
.data_req_i ( miss_req_valid & miss_req_bypass ),
.address_i ( miss_req_addr ),
.data_wdata_i ( miss_req_wdata ),
.data_we_i ( miss_req_we ),
.data_be_i ( miss_req_be ),
.data_size_i ( miss_req_size ),
.data_gnt_o ( bypass_gnt_o ),
.data_rvalid_o ( bypass_valid_o ),
.data_rdata_o ( bypass_data_o ),
// Slave Sid
.id_i ( id_bypass_fsm[$clog2(NR_PORTS)-1:0] ),
.id_o ( id_fsm_bypass ),
.gnt_id_i ( gnt_id_bypass_fsm[$clog2(NR_PORTS)-1:0] ),
.address_o ( req_fsm_bypass_addr ),
.data_wdata_o ( req_fsm_bypass_wdata ),
.data_req_o ( req_fsm_bypass_valid ),
.data_we_o ( req_fsm_bypass_we ),
.data_be_o ( req_fsm_bypass_be ),
.data_size_o ( req_fsm_bypass_size ),
.data_gnt_i ( gnt_bypass_fsm ),
.data_rvalid_i ( valid_bypass_fsm ),
.data_rdata_i ( data_bypass_fsm ),
.*
.req_i (bypass_ports_req),
.rsp_o (bypass_ports_rsp),
// Slave Side
.req_o (bypass_adapter_req),
.rsp_i (bypass_adapter_rsp)
);
// ----------------------
// Bypass AXI Interface
// ----------------------
axi_adapter #(
.DATA_WIDTH ( 64 ),
.AXI_ID_WIDTH ( 4 ),
.CACHELINE_BYTE_OFFSET ( DCACHE_BYTE_OFFSET )
.DATA_WIDTH (64),
.AXI_ID_WIDTH (4),
.CACHELINE_BYTE_OFFSET(DCACHE_BYTE_OFFSET)
) i_bypass_axi_adapter (
.clk_i,
.rst_ni,
.req_i ( req_fsm_bypass_valid ),
.type_i ( ariane_axi::SINGLE_REQ ),
.gnt_o ( gnt_bypass_fsm ),
.addr_i ( req_fsm_bypass_addr ),
.we_i ( req_fsm_bypass_we ),
.wdata_i ( req_fsm_bypass_wdata ),
.be_i ( req_fsm_bypass_be ),
.size_i ( req_fsm_bypass_size ),
.id_i ( {2'b10, id_fsm_bypass} ),
.valid_o ( valid_bypass_fsm ),
.rdata_o ( data_bypass_fsm ),
.gnt_id_o ( gnt_id_bypass_fsm ),
.id_o ( id_bypass_fsm ),
.critical_word_o ( ), // not used for single requests
.critical_word_valid_o ( ), // not used for single requests
.axi_req_o ( axi_bypass_o ),
.axi_resp_i ( axi_bypass_i )
.clk_i (clk_i),
.rst_ni (rst_ni),
.req_i (bypass_adapter_req.req),
.type_i (bypass_adapter_req.reqtype),
.amo_i (bypass_adapter_req.amo),
.id_i (bypass_adapter_req.id),
.addr_i (bypass_adapter_req.addr),
.wdata_i (bypass_adapter_req.wdata),
.we_i (bypass_adapter_req.we),
.be_i (bypass_adapter_req.be),
.size_i (bypass_adapter_req.size),
.gnt_o (bypass_adapter_rsp.gnt),
.valid_o (bypass_adapter_rsp.valid),
.rdata_o (bypass_adapter_rsp.rdata),
.id_o (), // not used, single outstanding request in arbiter
.critical_word_o (), // not used for single requests
.critical_word_valid_o(), // not used for single requests
.axi_req_o (axi_bypass_o),
.axi_resp_i (axi_bypass_i)
);
// ----------------------
@ -599,6 +597,7 @@ module miss_handler import ariane_pkg::*; import std_cache_pkg::*; #(
.rst_ni,
.req_i ( req_fsm_miss_valid ),
.type_i ( req_fsm_miss_req ),
.amo_i ( AMO_NONE ),
.gnt_o ( gnt_miss_fsm ),
.addr_i ( req_fsm_miss_addr ),
.we_i ( req_fsm_miss_we ),
@ -606,7 +605,6 @@ module miss_handler import ariane_pkg::*; import std_cache_pkg::*; #(
.be_i ( req_fsm_miss_be ),
.size_i ( req_fsm_miss_size ),
.id_i ( 4'b1100 ),
.gnt_id_o ( ), // open
.valid_o ( valid_miss_fsm ),
.rdata_o ( data_miss_fsm ),
.id_o ( ),
@ -626,16 +624,6 @@ module miss_handler import ariane_pkg::*; import std_cache_pkg::*; #(
.*
);
// -----------------
// AMO ALU
// -----------------
amo_alu i_amo_alu (
.amo_op_i ( amo_op ),
.amo_operand_a_i ( amo_operand_a ),
.amo_operand_b_i ( amo_operand_b ),
.amo_result_o ( amo_result_o )
);
// -----------------
// Struct Split
// -----------------
@ -658,105 +646,61 @@ endmodule
// --------------
// AXI Arbiter
// --------------s
// --------------
//
// Description: Arbitrates access to AXI refill/bypass
//
module arbiter #(
parameter int unsigned NR_PORTS = 3,
parameter int unsigned DATA_WIDTH = 64
module axi_adapter_arbiter #(
parameter NR_PORTS = 4,
parameter type req_t = std_cache_pkg::bypass_req_t,
parameter type rsp_t = std_cache_pkg::bypass_rsp_t
)(
input logic clk_i, // Clock
input logic rst_ni, // Asynchronous reset active low
// master ports
input logic [NR_PORTS-1:0] data_req_i,
input logic [NR_PORTS-1:0][63:0] address_i,
input logic [NR_PORTS-1:0][DATA_WIDTH-1:0] data_wdata_i,
input logic [NR_PORTS-1:0] data_we_i,
input logic [NR_PORTS-1:0][DATA_WIDTH/8-1:0] data_be_i,
input logic [NR_PORTS-1:0][1:0] data_size_i,
output logic [NR_PORTS-1:0] data_gnt_o,
output logic [NR_PORTS-1:0] data_rvalid_o,
output logic [NR_PORTS-1:0][DATA_WIDTH-1:0] data_rdata_o,
// slave port
input logic [$clog2(NR_PORTS)-1:0] id_i,
output logic [$clog2(NR_PORTS)-1:0] id_o,
input logic [$clog2(NR_PORTS)-1:0] gnt_id_i,
output logic data_req_o,
output logic [63:0] address_o,
output logic [DATA_WIDTH-1:0] data_wdata_o,
output logic data_we_o,
output logic [DATA_WIDTH/8-1:0] data_be_o,
output logic [1:0] data_size_o,
input logic data_gnt_i,
input logic data_rvalid_i,
input logic [DATA_WIDTH-1:0] data_rdata_i
input logic clk_i, // Clock
input logic rst_ni, // Asynchronous reset active low
// Master ports
input req_t [NR_PORTS-1:0] req_i,
output rsp_t [NR_PORTS-1:0] rsp_o,
// Slave port
output req_t req_o,
input rsp_t rsp_i
);
enum logic [1:0] { IDLE, REQ, SERVING } state_d, state_q;
enum logic { IDLE, SERVING } state_d, state_q;
struct packed {
logic [$clog2(NR_PORTS)-1:0] id;
logic [63:0] address;
logic [63:0] data;
logic [1:0] size;
logic [DATA_WIDTH/8-1:0] be;
logic we;
} req_d, req_q;
req_t req_d, req_q;
logic [NR_PORTS-1:0] sel_d, sel_q;
always_comb begin
automatic logic [$clog2(NR_PORTS)-1:0] request_index;
request_index = 0;
sel_d = sel_q;
state_d = state_q;
req_d = req_q;
// request port
data_req_o = 1'b0;
address_o = req_q.address;
data_wdata_o = req_q.data;
data_be_o = req_q.be;
data_size_o = req_q.size;
data_we_o = req_q.we;
id_o = req_q.id;
data_gnt_o = '0;
// read port
data_rvalid_o = '0;
data_rdata_o = '0;
data_rdata_o[req_q.id] = data_rdata_i;
req_o = req_q;
rsp_o = '0;
rsp_o[sel_q].rdata = rsp_i.rdata;
case (state_q)
IDLE: begin
// wait for incoming requests
for (int unsigned i = 0; i < NR_PORTS; i++) begin
if (data_req_i[i] == 1'b1) begin
data_req_o = data_req_i[i];
data_gnt_o[i] = data_req_i[i];
request_index = i[$bits(request_index)-1:0];
// save the request
req_d.address = address_i[i];
req_d.id = i[$bits(req_q.id)-1:0];
req_d.data = data_wdata_i[i];
req_d.size = data_size_i[i];
req_d.be = data_be_i[i];
req_d.we = data_we_i[i];
if (req_i[i].req == 1'b1) begin
sel_d = i[$bits(sel_d)-1:0];
state_d = SERVING;
break; // break here as this is a priority select
break;
end
end
address_o = address_i[request_index];
data_wdata_o = data_wdata_i[request_index];
data_be_o = data_be_i[request_index];
data_size_o = data_size_i[request_index];
data_we_o = data_we_i[request_index];
id_o = request_index;
req_d = req_i[sel_d];
req_o = req_i[sel_d];
rsp_o[sel_d].gnt = req_i[sel_d].req;
end
SERVING: begin
data_req_o = 1'b1;
if (data_rvalid_i) begin
data_rvalid_o[req_q.id] = 1'b1;
if (rsp_i.valid) begin
rsp_o[sel_q].valid = 1'b1;
state_d = IDLE;
end
end
@ -768,9 +712,11 @@ module arbiter #(
always_ff @(posedge clk_i or negedge rst_ni) begin
if (~rst_ni) begin
state_q <= IDLE;
sel_q <= '0;
req_q <= '0;
end else begin
state_q <= state_d;
sel_q <= sel_d;
req_q <= req_d;
end
end
@ -781,13 +727,13 @@ module arbiter #(
//pragma translate_off
`ifndef VERILATOR
// make sure that we eventually get an rvalid after we received a grant
assert property (@(posedge clk_i) data_gnt_i |-> ##[1:$] data_rvalid_i )
assert property (@(posedge clk_i) rsp_i.gnt |-> ##[1:$] rsp_i.valid )
else begin $error("There was a grant without a rvalid"); $stop(); end
// assert that there is no grant without a request
assert property (@(negedge clk_i) data_gnt_i |-> data_req_o)
assert property (@(negedge clk_i) rsp_i.gnt |-> req_o.req)
else begin $error("There was a grant without a request."); $stop(); end
// assert that the address does not contain X when request is sent
assert property ( @(posedge clk_i) (data_req_o) |-> (!$isunknown(address_o)) )
assert property ( @(posedge clk_i) (req_o.req) |-> (!$isunknown(req_o.addr)) )
else begin $error("address contains X when request is set"); $stop(); end
`endif

18
core/include/std_cache_pkg.sv
View file

@ -42,6 +42,24 @@ package std_cache_pkg;
logic bypass;
} miss_req_t;
typedef struct packed {
logic req;
ariane_axi::ad_req_t reqtype;
ariane_pkg::amo_t amo;
logic [3:0] id;
logic [63:0] addr;
logic [63:0] wdata;
logic we;
logic [7:0] be;
logic [1:0] size;
} bypass_req_t;
typedef struct packed {
logic gnt;
logic valid;
logic [63:0] rdata;
} bypass_rsp_t;
typedef struct packed {
logic [ariane_pkg::DCACHE_TAG_WIDTH-1:0] tag; // tag array
logic [ariane_pkg::DCACHE_LINE_WIDTH-1:0] data; // data array

2
corev_apu/fpga/src/ariane_xilinx.sv
View file

@ -565,8 +565,8 @@ axi_adapter #(
.rst_ni ( rst_n ),
.req_i ( dm_master_req ),
.type_i ( ariane_axi::SINGLE_REQ ),
.amo_i ( ariane_pkg::AMO_NONE ),
.gnt_o ( dm_master_gnt ),
.gnt_id_o ( ),
.addr_i ( dm_master_add ),
.we_i ( dm_master_we ),
.wdata_i ( dm_master_wdata ),

2
corev_apu/tb/ariane_testharness.sv
View file

@ -287,8 +287,8 @@ module ariane_testharness #(
.rst_ni ( rst_ni ),
.req_i ( dm_master_req ),
.type_i ( ariane_axi::SINGLE_REQ ),
.amo_i ( ariane_pkg::AMO_NONE ),
.gnt_o ( dm_master_gnt ),
.gnt_id_o ( ),
.addr_i ( dm_master_add ),
.we_i ( dm_master_we ),
.wdata_i ( dm_master_wdata ),

57
corev_apu/tb/tb_wb_dcache/hdl/tb.sv
View file

@ -186,6 +186,32 @@ module tb import ariane_pkg::*; import std_cache_pkg::*; import tb_pkg::*; #()()
tb_mem_port_t data_mem_port;
tb_mem_port_t bypass_mem_port;
AXI_BUS #(
.AXI_ADDR_WIDTH ( TbAxiAddrWidthFull ),
.AXI_DATA_WIDTH ( TbAxiDataWidthFull ),
.AXI_ID_WIDTH ( TbAxiIdWidthFull + 32'd1 ),
.AXI_USER_WIDTH ( TbAxiUserWidthFull )
) axi_bypass ();
AXI_BUS #(
.AXI_ADDR_WIDTH ( TbAxiAddrWidthFull ),
.AXI_DATA_WIDTH ( TbAxiDataWidthFull ),
.AXI_ID_WIDTH ( TbAxiIdWidthFull + 32'd1 ),
.AXI_USER_WIDTH ( TbAxiUserWidthFull )
) axi_bypass_amo_adapter ();
AXI_BUS_DV #(
.AXI_ADDR_WIDTH ( TbAxiAddrWidthFull ),
.AXI_DATA_WIDTH ( TbAxiDataWidthFull ),
.AXI_ID_WIDTH ( TbAxiIdWidthFull + 32'd1 ),
.AXI_USER_WIDTH ( TbAxiUserWidthFull )
) axi_bypass_amo_adapter_dv (
.clk_i ( clk_i )
);
`AXI_ASSIGN(axi_bypass, axi_bypass_dv)
`AXI_ASSIGN(axi_bypass_amo_adapter_dv, axi_bypass_amo_adapter)
///////////////////////////////////////////////////////////////////////////////
// Helper tasks
///////////////////////////////////////////////////////////////////////////////
@ -258,8 +284,8 @@ module tb import ariane_pkg::*; import std_cache_pkg::*; import tb_pkg::*; #()()
`APPL_WAIT_CYC(clk_i, 1)
forever begin
amo_exp_result = 'x;
`ACQ_WAIT_CYC(clk_i, 1)
amo_exp_result = 'x;
// Regular stores. These are directly written to shadow memory.
if(write_en) begin
@ -285,7 +311,8 @@ module tb import ariane_pkg::*; import std_cache_pkg::*; import tb_pkg::*; #()()
// The result that is expected to be returned by AMO and evantually to be stored in rd.
// For most AMOs, this is the previous memory content.
amo_exp_result[31:0] = amo_shadow[31:0];
// RISC-V spec requires: "For RV64, 32-bit AMOs always sign-extend the value placed in rd."
amo_exp_result = amo_op_a;
// 64-bit AMO
end else begin
@ -361,8 +388,8 @@ module tb import ariane_pkg::*; import std_cache_pkg::*; import tb_pkg::*; #()()
// Instantiate memory and AXI ports
initial begin : p_sim_mem
// Create AXI ports
data_mem_port = new(axi_data_dv , CACHED);
bypass_mem_port = new(axi_bypass_dv, BYPASS);
data_mem_port = new(axi_data_dv , CACHED);
bypass_mem_port = new(axi_bypass_amo_adapter_dv, BYPASS);
// Initialize AXI ports and memory
data_mem_port.reset();
@ -401,6 +428,24 @@ module tb import ariane_pkg::*; import std_cache_pkg::*; import tb_pkg::*; #()()
.axi_bypass_i ( axi_bypass_i )
);
///////////////////////////////////////////////////////////////////////////////
// AXI Atomics Adapter
///////////////////////////////////////////////////////////////////////////////
axi_riscv_atomics_wrap #(
.AXI_ADDR_WIDTH ( TbAxiAddrWidthFull ),
.AXI_DATA_WIDTH ( TbAxiDataWidthFull ),
.AXI_ID_WIDTH ( TbAxiIdWidthFull + 32'd1 ),
.AXI_USER_WIDTH ( TbAxiUserWidthFull ),
.AXI_MAX_WRITE_TXNS ( 1 ),
.RISCV_WORD_WIDTH ( 64 )
) i_amo_adapter (
.clk_i ( clk_i ),
.rst_ni ( rst_ni ),
.mst ( axi_bypass_amo_adapter.Master ),
.slv ( axi_bypass.Slave )
);
///////////////////////////////////////////////////////////////////////////////
// port emulation programs
///////////////////////////////////////////////////////////////////////////////
@ -924,8 +969,8 @@ module tb import ariane_pkg::*; import std_cache_pkg::*; import tb_pkg::*; #()()
req_rate = '{50,0,2};
// AMOs should use cache port
bypass_mem_port.set_region(0, 0);
data_mem_port.set_region(0, MemBytes-1);
bypass_mem_port.set_region(0, MemBytes-1);
data_mem_port.set_region(0, 0);
runAMOs(nAMOs,1); // Last sequence flag, terminates agents
flushCache();

10
corev_apu/tb/tb_wb_dcache/hdl/tb_amoport.sv
View file

@ -85,11 +85,13 @@ program tb_amoport import ariane_pkg::*; import std_cache_pkg::*; import tb_pkg:
void'(randomize(size) with {size >= 2; size <= 3;});
// For LRs/SCs, choose from only 4 adresses,
// so that valid LR/SC combinations become more likely
// so that valid LR/SC combinations become more likely.
// Note: AMOs to address 0 are not supported in axi_riscv_atomics module
// so we start at 8 (for alignment purposes)
if (amo_op inside {AMO_LR, AMO_SC})
void'(randomize(paddr) with {paddr >= 0; paddr < 4;});
void'(randomize(paddr) with {paddr >= 8; paddr < 12;});
else
void'(randomize(paddr) with {paddr >= 0; paddr < (MemWords<<3);});
void'(randomize(paddr) with {paddr >= 8; paddr < (MemWords<<3);});
// Align adress
if (size == 2)
@ -181,6 +183,8 @@ program tb_amoport import ariane_pkg::*; import std_cache_pkg::*; import tb_pkg:
for (int k=0;k<seq_num_amo_i;k++) begin
`ACQ_WAIT_SIG(clk_i, dut_amo_resp_port_i.ack)
// Assert expected data is not 'x, protects against ineffective ==? comparisons
assert(exp_result_i !== 'x) else $error("Expected result is unknown");
// note: wildcard as defined in right operand!
ok=(dut_amo_resp_port_i.result ==? exp_result_i) && (act_mem_i == exp_mem_i);

6
corev_apu/tb/tb_wb_dcache/tb.list
View file

@ -35,6 +35,12 @@
../../../common/submodules/common_cells/src/stream_demux.sv
../../../core/axi_adapter.sv
../../../common/local/util/sram.sv
../../src/axi_riscv_atomics/src/axi_res_tbl.sv
../../src/axi_riscv_atomics/src/axi_riscv_amos.sv
../../src/axi_riscv_atomics/src/axi_riscv_amos_alu.sv
../../src/axi_riscv_atomics/src/axi_riscv_lrsc.sv
../../src/axi_riscv_atomics/src/axi_riscv_atomics.sv
../../src/axi_riscv_atomics/src/axi_riscv_atomics_wrap.sv
../common/tb_dcache_pkg.sv
../common/tb_readport.sv