Protected a display with a verbosity check

// Copyright (c) 2016-2020 Bluespec, Inc. All Rights Reserved.

package MMU_Cache_AXI4_Adapter;

// ================================================================
// Adapter converting generic 64b-wide read/write requests into an
// AXI4 bus master.  Two 'clients' upstream:
// - a cache: requests/responses are for full cache lines
// - an MMIO: requests/responses are for 64b word or sub-word,
//            and where lane-alignment is already done.

// The AXI4 bus master can be used with 32b or 64b buses, and manages
// byte-lane alignment, number of beats in a burst, write-strobes,
// etc. accordingly.

// TODO: Wrapping bursts (when cache can handle that)

// ================================================================

export  MMU_Cache_AXI4_Adapter_IFC (..);
export  mkMMU_Cache_AXI4_Adapter;

// ================================================================
// BSV lib imports

import Vector       :: *;
import BRAMCore     :: *;
import ConfigReg    :: *;
import FIFOF        :: *;
import GetPut       :: *;
import ClientServer :: *;
import Assert       :: *;

// ----------------
// BSV additional libs

import Cur_Cycle     :: *;
import GetPut_Aux    :: *;
import Semi_FIFOF    :: *;

// ================================================================
// Project imports

`ifdef RV32
`ifdef SV32
import Cache_Decls_RV32_Sv32_8KB_2way :: *;
`else
import Cache_Decls_RV32_8KB_2way :: *;
`endif
`endif

`ifdef RV64
`ifdef SV39
import Cache_Decls_RV64_Sv39_8KB_2way :: *;
`else
import Cache_Decls_RV64_8KB_2way :: *;
`endif
`endif

import MMU_Cache_Common :: *;

import AXI4_Types  :: *;
import Fabric_Defs :: *;

// ================================================================
// This module collects and discards write-responses.
// There is no write-data response to either client.
// Errors on writes are reported on a separate method that can be used
// to trigger an interrupt.

// This module avoids interleaving read and write requests,
// i.e., it launches a read request only when no write-responses from
// the AXI4 fabric are pending.

// ================================================================
// MODULE INTERFACE

interface MMU_Cache_AXI4_Adapter_IFC;
   // ----------------
   // interface for cache-line read/write client

   interface Put #(Line_Req)  p_mem_line_req;
   interface Put #(Bit #(64)) p_mem_line_write_data;
   interface Get #(Read_Data) g_mem_line_read_data;

   // ----------------
   // interface for word/sub-word read/write client

   interface Put #(Single_Req) p_mem_single_req;
   interface Put #(Bit #(64))  p_mem_single_write_data;
   interface Get #(Read_Data)  g_mem_single_read_data;

   // ----------------
   // Write-error from memory
   (* always_ready *)
   method Bool mv_write_error;

   // ----------------
   // Fabric master interface
   interface AXI4_Master_IFC #(Wd_Id, Wd_Addr, Wd_Data, Wd_User) mem_master;
endinterface

// ================================================================
// Misc. help functions

// ----------------------------------------------------------------
// Byte-width of the fabric

Integer bytes_per_fabric_data = ((valueOf (Wd_Data) == 32) ? 4 : 8);

// ----------------------------------------------------------------
// Convert size code into AXI4_Size code (number of bytes in a beat).
// It just so happens that our coding coincides with AXI4's coding.

function AXI4_Size  fv_size_code_to_AXI4_Size (Bit #(2) size_code);
   return { 1'b0, size_code };
endfunction

// ----------------------------------------------------------------
// Check if addr is cache-line-aligned

Bit #(64) byte_in_line_mask = fromInteger (bytes_per_cline - 1);

function Bool fv_is_line_aligned (Bit #(64) addr);
   return ((addr & byte_in_line_mask) == 0);
endfunction

// ----------------------------------------------------------------
// Align an address to beginning of cache line containing the addr

function Bit #(64) fv_align_to_line (Bit #(64) addr);
   return (addr & (~ byte_in_line_mask));
endfunction

// ----------------------------------------------------------------
// Convert a 64-bit Address to an AXI4 Fabric Address
// For FABRIC64 this does nothing.
// For FABRIC32 it discards the upper 32 bits.

function Fabric_Addr fv_Addr_to_Fabric_Addr (Bit #(64) addr);
   return truncate (addr);
endfunction

// ================================================================
// MODULE IMPLEMENTATION

(* synthesize *)
module mkMMU_Cache_AXI4_Adapter #(parameter Bit #(3) verbosity)
                                (MMU_Cache_AXI4_Adapter_IFC);

   // Verbosity: 0=quiet, 1 = rule firings
   // Integer verbosity = 1;

   // FIFOs for line-client
   FIFOF #(Line_Req)   f_line_reqs       <- mkFIFOF;
   FIFOF #(Bit #(64))  f_line_write_data <- mkFIFOF;
   FIFOF #(Read_Data)  f_line_read_data  <- mkFIFOF;

   // FIFOs for single-word client
   FIFOF #(Single_Req) f_single_reqs       <- mkFIFOF;
   FIFOF #(Bit #(64))  f_single_write_data <- mkFIFOF;
   FIFOF #(Read_Data)  f_single_read_data  <- mkFIFOF;

   // Identifiers for requestor client
   Bit #(1) client_id_line   = 0;
   Bit #(1) client_id_single = 1;

   // Limit the number of reads/writes outstanding to 15
   // TODO: change these to concurrent up/down counters?
   Reg #(Bit #(4)) rg_rd_rsps_pending <- mkReg (0);
   Reg #(Bit #(4)) rg_wr_rsps_pending <- mkReg (0);

   // Record errors on write-responses from mem
   Reg #(Bool) rg_write_error <- mkReg (False);

   // AXI4 fabric request/response
   AXI4_Master_Xactor_IFC #(Wd_Id, Wd_Addr, Wd_Data, Wd_User) master_xactor <- mkAXI4_Master_Xactor;

   // ****************************************************************
   // BEHAVIOR: READ RESPONSES (for line and single clients)
   // The following registers identify the client, the beat, etc.

   Reg #(Bit #(1)) rg_rd_client_id <- mkRegU;

   // TODO: change mkFIFOF to mkSizedFIFOF (allowing multiple outstanding reads)
   // may reads can be outstanding
   FIFOF #(Tuple3 #(Bit #(2),     // size_code
		    Bit #(1),     // addr bit [2]
		    Bit #(8)))    // Num beats read-data
         f_rd_rsp_control <- mkFIFOF;
   match { .rd_req_size_code, .rd_req_addr_bit_2, .rd_req_beats } = f_rd_rsp_control.first;

   // Count beats in a single transaction.
   // Note: beat-count is for fabric, not for client-side data.
   // Former is 2 x latter for Fabric32 for 64b line data and 64b single data.

   Reg #(Bit #(8)) rg_rd_beat <- mkRegU;

   // The following are needed for FABRIC32 during each burst response
   // to assemble lower and upper 32b into a 64b response to client
   Reg #(Bool)       rg_rd_data_lower32_ok <- mkRegU;
   Reg #(Bit #(32))  rg_rd_data_lower32    <- mkRegU;

   rule rl_read_data (rg_rd_beat < rd_req_beats);
      rg_rd_beat <= rg_rd_beat + 1;
      Bool last_beat = (rg_rd_beat == (rd_req_beats - 1));
      if (last_beat) begin
	 f_rd_rsp_control.deq;
	 rg_rd_rsps_pending <= rg_rd_rsps_pending - 1;
      end

      let rd_data <- pop_o (master_xactor.o_rd_data);

      Bool      ok   = (rd_data.rresp == axi4_resp_okay);
      Bit #(64) data = zeroExtend (rd_data.rdata);

      Bool do_enq    = True;
      Bool even_beat = (rg_rd_beat [0] == 1'b0);

      // FABRIC32 adjustments
      if (valueOf (Wd_Data) == 32) begin
	 if (rd_req_size_code != 2'b11) begin
	    // B, H, W: only 1 beat
	    Bool in_upper32 = (rd_req_addr_bit_2 == 1'b1);
	    if (in_upper32)
	       data = { data [31:0], 32'b0 };
	    do_enq = True;
	 end
	 else if (even_beat) begin // D, even beat
	    // Just save lower 32b; enq response only after next beat
	    rg_rd_data_lower32_ok <= ok;
	    rg_rd_data_lower32    <= data [31:0];
	    do_enq = False;
	 end
	 else begin // D, odd beat
	    ok   = (rg_rd_data_lower32_ok && ok);
	    data = { data [31:0], rg_rd_data_lower32 };
	    do_enq = True;
	 end
      end

      if (do_enq) begin
	 let rsp = Read_Data {ok: ok, data: data};
	 if (rg_rd_client_id == client_id_line)
	    f_line_read_data.enq (rsp);
	 else
	    f_single_read_data.enq (rsp);
      end

      if (verbosity >= 1) begin
	 $display ("%0d: %m.rl_read_data: ", cur_cycle);
	 if (rg_rd_client_id == client_id_line) $write ("    line (beat %0d)", rg_rd_beat);
	 else                                   $write ("     single");
	 $write (" ok %0d data %0h", pack (ok), data);
	 if ((valueOf (Wd_Data) == 32) && even_beat) $write (" (lower 32b of 64b)");
	 $display ("");
      end
   endrule: rl_read_data

   // ****************************************************************
   // BEHAVIOR: LINE READ REQUESTS

   rule rl_line_read_req (f_line_reqs.first.is_read
			  && (rg_rd_rsps_pending < '1)
			  && (rg_wr_rsps_pending == 0));
      let req <- pop (f_line_reqs);

      dynamicAssert (fv_is_line_aligned (req.addr), "rl_line_read_req addr misaligned");

      let addr = fv_align_to_line (req.addr);

      Fabric_Addr fabric_addr = fv_Addr_to_Fabric_Addr (addr);
      AXI4_Size   fabric_size = ((valueOf (Wd_Data) == 32) ? axsize_4 : axsize_8);

      // Note: AXI4 codes a burst length of 'n' as 'n-1'
      Integer   num_beats  = bytes_per_cline / bytes_per_fabric_data;
      AXI4_Len  fabric_len = fromInteger (num_beats - 1);

      let mem_req_rd_addr = AXI4_Rd_Addr {arid:     fabric_default_id,
					  araddr:   fabric_addr,
					  arlen:    fabric_len,
					  arsize:   fabric_size,
					  arburst:  axburst_incr,
					  arlock:   fabric_default_lock,
					  arcache:  fabric_default_arcache,
					  arprot:   fabric_default_prot,
					  arqos:    fabric_default_qos,
					  arregion: fabric_default_region,
					  aruser:   fabric_default_user};
      master_xactor.i_rd_addr.enq (mem_req_rd_addr);

      f_rd_rsp_control.enq (tuple3 (2'b11, 1'b0, fromInteger (num_beats)));
      rg_rd_client_id    <= client_id_line;
      rg_rd_beat         <= 0;
      rg_rd_rsps_pending <= rg_rd_rsps_pending + 1;

      // Debugging
      if (verbosity >= 1) begin
	 $display ("%0d: %m.rl_line_read_req: ", cur_cycle);
	 $display ("    AXI4_Rd_Addr{araddr %0h arlen %0d (burst len %0d) ",
		   fabric_addr,
		   fabric_len, num_beats,
		   fshow_AXI4_Size (fabric_size),
		   " incr}");
      end
   endrule

   // ****************************************************************
   // BEHAVIOR: Single read requests (not a burst)

   rule rl_single_read_req (f_single_reqs.first.is_read
			    && (rg_rd_rsps_pending < '1)
			    && (rg_wr_rsps_pending == 0));
      let         req        <- pop (f_single_reqs);
      Fabric_Addr fabric_addr = fv_Addr_to_Fabric_Addr (req.addr);
      AXI4_Size   fabric_size = fv_size_code_to_AXI4_Size (req.size_code);

      if (verbosity >= 1)
	 $display ("%0d: %m.rl_single_read_req:\n    AXI4_Rd_Addr{araddr %0h arlen 0 (burst length 1) ",
		   cur_cycle, fabric_addr,  fshow_AXI4_Size (fabric_size), "}");

      Bit #(8)    num_beats   = 1;

      // FABRIC32 adjustments
      if ((valueOf (Wd_Data) == 32) && (req.size_code == 2'b11)) begin
	 fabric_size = axsize_4;
	 num_beats   = 2;
      end
      // Note: AXI4 codes a burst length of 'n' as 'n-1'
      AXI4_Len fabric_len = num_beats - 1;

      let mem_req_rd_addr = AXI4_Rd_Addr {arid:     fabric_default_id,
					  araddr:   fabric_addr,
					  arlen:    0,           // burst len = arlen+1
					  arsize:   fabric_size,
					  arburst:  fabric_default_burst,
					  arlock:   fabric_default_lock,
					  arcache:  fabric_default_arcache,
					  arprot:   fabric_default_prot,
					  arqos:    fabric_default_qos,
					  arregion: fabric_default_region,
					  aruser:   fabric_default_user};
      master_xactor.i_rd_addr.enq (mem_req_rd_addr);

      f_rd_rsp_control.enq (tuple3 (req.size_code, req.addr [2], num_beats));
      rg_rd_client_id    <= client_id_single;
      rg_rd_beat         <= 0;
      rg_rd_rsps_pending <= rg_rd_rsps_pending + 1;
   endrule

   // ****************************************************************
   // BEHAVIOR: WRITE RESPONSES (for line and single clients)
   // The following register identifies the client

   rule rl_write_rsp;
      let wr_resp <- pop_o (master_xactor.o_wr_resp);

      Bool err = False;
      if (rg_wr_rsps_pending == 0) begin
	 rg_write_error <= True;

	 $display ("%0d: %m.rl_write_rsp: ERROR not expecting any write-response:", cur_cycle);
	 $display ("    ", fshow (wr_resp));
      end
      else begin
	 rg_wr_rsps_pending <= rg_wr_rsps_pending - 1;
	 if (wr_resp.bresp != axi4_resp_okay) begin
	    rg_write_error <= True;
	    if (verbosity >= 1) begin
	       $display ("%0d: %m.rl_write_rsp: FABRIC RESPONSE ERROR", cur_cycle);
	       $display ("    ", fshow (wr_resp));
	    end
	 end
	 else if (verbosity >= 1) begin
	    $display ("%0d: %m.rl_write_rsp: pending=%0d, ",
		      cur_cycle, rg_wr_rsps_pending, fshow (wr_resp));
	 end
      end
   endrule

   // ****************************************************************
   // BEHAVIOR: Write data (for both line and single)
   // Assume that data is already lane-aligned for 64b data width.

   FIFOF #(Tuple4 #(Bit #(1),     // client_id
		    Bit #(2),     // size_code
		    Bit #(3),     // addr lsbs
		    Bit #(8)))    // Num beats in write-data
         f_wr_data_control <- mkFIFOF;
   match {.wr_client_id,
	  .wr_req_size_code,
	  .wr_req_addr_lsbs,
	  .wr_req_beats } = f_wr_data_control.first;

   // Count beats in a single transaction
   // Note: beat-count is for fabric, not for client-side data.
   // Former is 2 x latter for Fabric32 for 64b line data and 64b single data.
   Reg #(Bit #(8)) rg_wr_beat <- mkReg (0);

   rule rl_write_data (rg_wr_beat < wr_req_beats);
      Bool last = (rg_wr_beat == (wr_req_beats - 1));
      if (last) begin
	 f_wr_data_control.deq;
	 rg_wr_beat <= 0;
      end
      else
	 rg_wr_beat <= rg_wr_beat + 1;

      Bit #(64) data = ((wr_client_id == client_id_line)
			? f_line_write_data.first
			: f_single_write_data.first);

      // Compute strobe from size and address
      Bit #(8)  strb = case (wr_req_size_code)
			  2'b00: 8'h_01;
			  2'b01: 8'h_03;
			  2'b10: 8'h_0F;
			  2'b11: 8'h_FF;
		       endcase;
      strb = (strb << wr_req_addr_lsbs);

      Bool do_deq    = True;
      Bool even_beat = (rg_wr_beat [0] == 1'b0);

      // FABRIC32 adjustments
      if (valueOf (Wd_Data) == 32) begin
	 if (wr_req_size_code != 2'b11) begin
	    // B, H, W: only 1 beat
	    Bool in_upper32 = (wr_req_addr_lsbs [2] == 1'b1);
	    if (in_upper32) begin
	       data = { 32'b0, data [63:32] };
	       strb = {  4'b0, strb [7:4] };
	    end
	    do_deq = True;
	 end
	 else if (even_beat) begin  // D, even beat
	    do_deq = False;
	 end
	 else begin // D, odd beat
	    data   = { 32'b0, data [63:32] };
	    strb   = '1;
	    do_deq = True;
	 end
      end

      let mem_req_wr_data = AXI4_Wr_Data {wdata:  truncate (data),
					  wstrb:  truncate (strb),
					  wlast:  last,
					  wuser:  fabric_default_user};
      master_xactor.i_wr_data.enq (mem_req_wr_data);

      if (do_deq) begin
	 if (wr_client_id == client_id_line)
	    f_line_write_data.deq;
	 else
	    f_single_write_data.deq;
      end

      if (verbosity >= 1) begin
	 $display ("%0d: %m.rl_write_data: beat %0d/%0d", cur_cycle, rg_wr_beat, wr_req_beats);
	 $display ("    AXI4_Wr_Data{%0h strb %0h last %0d}", data, strb, pack (last));
      end
   endrule

   // ****************************************************************
   // BEHAVIOR: Line-write requests

   rule rl_line_write_req ((! f_line_reqs.first.is_read)
			   && (rg_rd_rsps_pending == 0)
			   && (rg_wr_rsps_pending < '1));
      let req <- pop (f_line_reqs);

      dynamicAssert (fv_is_line_aligned (req.addr), "rl_line_write_req addr misaligned");

      let addr = fv_align_to_line (req.addr);

      Fabric_Addr fabric_addr = fv_Addr_to_Fabric_Addr (addr);
      AXI4_Size   fabric_size = ((valueOf (Wd_Data) == 32) ? axsize_4 : axsize_8);

      // Note: AXI4 codes a burst length of 'n' as 'n-1'
      Integer   num_beats  = bytes_per_cline / bytes_per_fabric_data;
      AXI4_Len  fabric_len = fromInteger (num_beats - 1);

      let mem_req_wr_addr = AXI4_Wr_Addr {awid:     fabric_default_id,
					  awaddr:   fabric_addr,
					  awlen:    fabric_len,
					  awsize:   fabric_size,
					  awburst:  axburst_incr,
					  awlock:   fabric_default_lock,
					  awcache:  fabric_default_awcache,
					  awprot:   fabric_default_prot,
					  awqos:    fabric_default_qos,
					  awregion: fabric_default_region,
					  awuser:   fabric_default_user};
      master_xactor.i_wr_addr.enq (mem_req_wr_addr);

      f_wr_data_control.enq (tuple4 (client_id_line,
				     2'b11,        // size_code D
				     0,            // req.addr lsbs
				     fromInteger (num_beats)));
      rg_wr_rsps_pending <= rg_wr_rsps_pending + 1;

      // Debugging
      if (verbosity >= 1)
	 $display ("%0d: %m.rl_line_write_req: AXI4_Wr_Addr{awaddr %0h awlen %0d burst-length %0d ",
		   cur_cycle,
		   fabric_addr,
		   fabric_len, num_beats,
		   fshow_AXI4_Size (fabric_size),
		   " incr}");
   endrule

   // ****************************************************************
   // Scheduling

   (* descending_urgency = "rl_read_data,       rl_write_rsp" *)
   (* descending_urgency = "rl_write_rsp,       rl_line_read_req" *)
   (* descending_urgency = "rl_line_read_req,   rl_line_write_req" *)
   (* descending_urgency = "rl_line_write_req,  rl_write_data" *)
   (* descending_urgency = "rl_write_data,      rl_single_read_req" *)
   (* descending_urgency = "rl_single_read_req, rl_single_write_req" *)

   // The following should be implied by transitivity, but bsc still cites them
   (* descending_urgency = "rl_read_data,       rl_line_read_req" *)
   (* descending_urgency = "rl_line_read_req,   rl_single_read_req" *)
   (* descending_urgency = "rl_line_write_req,  rl_single_read_req" *)
   (* descending_urgency = "rl_write_rsp,       rl_line_write_req" *)
   (* descending_urgency = "rl_line_write_req,  rl_single_write_req" *)
   (* descending_urgency = "rl_line_read_req,   rl_single_write_req" *)

   // ****************************************************************
   // BEHAVIOR: Single write requests (not a burst)

   rule rl_single_write_req ((! f_single_reqs.first.is_read)
			     && (rg_rd_rsps_pending == 0)
			     && (rg_wr_rsps_pending < '1));
      let req <- pop (f_single_reqs);

      Fabric_Addr fabric_addr = fv_Addr_to_Fabric_Addr (req.addr);
      AXI4_Size   fabric_size = fv_size_code_to_AXI4_Size (req.size_code);
      Bit #(8)    num_beats   = 1;

      // FABRIC32 adjustments
      if ((valueOf (Wd_Data) == 32) && (req.size_code == 2'b11)) begin
	 fabric_size = axsize_4;
	 num_beats   = 2;
      end
      // Note: AXI4 codes a burst length of 'n' as 'n-1'
      AXI4_Len fabric_len = num_beats - 1;

      // AXI4 Write-Address channel
      let mem_req_wr_addr = AXI4_Wr_Addr {awid:     fabric_default_id,
					  awaddr:   fabric_addr,
					  awlen:    fabric_len,
					  awsize:   fabric_size,
					  awburst:  fabric_default_burst,
					  awlock:   fabric_default_lock,
					  awcache:  fabric_default_awcache,
					  awprot:   fabric_default_prot,
					  awqos:    fabric_default_qos,
					  awregion: fabric_default_region,
					  awuser:   fabric_default_user};
      master_xactor.i_wr_addr.enq (mem_req_wr_addr);

      f_wr_data_control.enq (tuple4 (client_id_single,
				     req.size_code,
				     req.addr [2:0],
				     num_beats));
      rg_wr_rsps_pending <= rg_wr_rsps_pending + 1;

      // Debugging
      if (verbosity >= 1)
	 $display ("%0d: %m.rl_single_write_req: AXI4_Wr_Addr{awaddr %0h awlen %0d burst-length %0d ",
		   cur_cycle,
		   fabric_addr,
		   fabric_len, num_beats,
		   fshow_AXI4_Size (fabric_size),
		   " incr}");
   endrule

   // ================================================================
   // INTERFACE

   // ----------------
   // interface for cache-line read/write client

   interface Put p_mem_line_req        = toPut (f_line_reqs);
   interface Put p_mem_line_write_data = toPut (f_line_write_data);
   interface Get g_mem_line_read_data  = toGet (f_line_read_data);

   // ----------------
   // interface for word/sub-word read/write client

   interface Put p_mem_single_req        = toPut (f_single_reqs);
   interface Put p_mem_single_write_data = toPut (f_single_write_data);
   interface Get g_mem_single_read_data  = toGet (f_single_read_data);

   // ----------------
   // Write-error from memory

   method Bool mv_write_error = rg_write_error;

   // ----------------
   // Fabric master interface
   interface mem_master = master_xactor.axi_side;
endmodule

// ================================================================

endpackage

// Copyright (c) 2016-2020 Bluespec, Inc. All Rights Reserved.

package MMU_Cache_AXI4_Adapter;

// ================================================================
// Adapter converting generic 64b-wide read/write requests into an
// AXI4 bus master.  Two 'clients' upstream:
// - a cache: requests/responses are for full cache lines
// - an MMIO: requests/responses are for 64b word or sub-word,
//            and where lane-alignment is already done.

// The AXI4 bus master can be used with 32b or 64b buses, and manages
// byte-lane alignment, number of beats in a burst, write-strobes,
// etc. accordingly.

// TODO: Wrapping bursts (when cache can handle that)

// ================================================================

export  MMU_Cache_AXI4_Adapter_IFC (..);
export  mkMMU_Cache_AXI4_Adapter;

// ================================================================
// BSV lib imports

import Vector       :: *;
import BRAMCore     :: *;
import ConfigReg    :: *;
import FIFOF        :: *;
import GetPut       :: *;
import ClientServer :: *;
import Assert       :: *;

// ----------------
// BSV additional libs

import Cur_Cycle     :: *;
import GetPut_Aux    :: *;
import Semi_FIFOF    :: *;

// ================================================================
// Project imports

`ifdef RV32
`ifdef SV32
import Cache_Decls_RV32_Sv32_8KB_2way :: *;
`else
import Cache_Decls_RV32_8KB_2way :: *;
`endif
`endif

`ifdef RV64
`ifdef SV39
import Cache_Decls_RV64_Sv39_8KB_2way :: *;
`else
import Cache_Decls_RV64_8KB_2way :: *;
`endif
`endif

import MMU_Cache_Common :: *;

import AXI4_Types  :: *;
import Fabric_Defs :: *;

// ================================================================
// This module collects and discards write-responses.
// There is no write-data response to either client.
// Errors on writes are reported on a separate method that can be used
// to trigger an interrupt.

// This module avoids interleaving read and write requests,
// i.e., it launches a read request only when no write-responses from
// the AXI4 fabric are pending.

// ================================================================
// MODULE INTERFACE

interface MMU_Cache_AXI4_Adapter_IFC;
   // ----------------
   // interface for cache-line read/write client

   interface Put #(Line_Req)  p_mem_line_req;
   interface Put #(Bit #(64)) p_mem_line_write_data;
   interface Get #(Read_Data) g_mem_line_read_data;

   // ----------------
   // interface for word/sub-word read/write client

   interface Put #(Single_Req) p_mem_single_req;
   interface Put #(Bit #(64))  p_mem_single_write_data;
   interface Get #(Read_Data)  g_mem_single_read_data;

   // ----------------
   // Write-error from memory
   (* always_ready *)
   method Bool mv_write_error;

   // ----------------
   // Fabric master interface
   interface AXI4_Master_IFC #(Wd_Id, Wd_Addr, Wd_Data, Wd_User) mem_master;
endinterface

// ================================================================
// Misc. help functions

// ----------------------------------------------------------------
// Byte-width of the fabric

Integer bytes_per_fabric_data = ((valueOf (Wd_Data) == 32) ? 4 : 8);

// ----------------------------------------------------------------
// Convert size code into AXI4_Size code (number of bytes in a beat).
// It just so happens that our coding coincides with AXI4's coding.

function AXI4_Size  fv_size_code_to_AXI4_Size (Bit #(2) size_code);
   return { 1'b0, size_code };
endfunction

// ----------------------------------------------------------------
// Check if addr is cache-line-aligned

Bit #(64) byte_in_line_mask = fromInteger (bytes_per_cline - 1);

function Bool fv_is_line_aligned (Bit #(64) addr);
   return ((addr & byte_in_line_mask) == 0);
endfunction

// ----------------------------------------------------------------
// Align an address to beginning of cache line containing the addr

function Bit #(64) fv_align_to_line (Bit #(64) addr);
   return (addr & (~ byte_in_line_mask));
endfunction

// ----------------------------------------------------------------
// Convert a 64-bit Address to an AXI4 Fabric Address
// For FABRIC64 this does nothing.
// For FABRIC32 it discards the upper 32 bits.

function Fabric_Addr fv_Addr_to_Fabric_Addr (Bit #(64) addr);
   return truncate (addr);
endfunction

// ================================================================
// MODULE IMPLEMENTATION

(* synthesize *)
module mkMMU_Cache_AXI4_Adapter #(parameter Bit #(3) verbosity)
                                (MMU_Cache_AXI4_Adapter_IFC);

   // Verbosity: 0=quiet, 1 = rule firings
   // Integer verbosity = 1;

   // FIFOs for line-client
   FIFOF #(Line_Req)   f_line_reqs       <- mkFIFOF;
   FIFOF #(Bit #(64))  f_line_write_data <- mkFIFOF;
   FIFOF #(Read_Data)  f_line_read_data  <- mkFIFOF;

   // FIFOs for single-word client
   FIFOF #(Single_Req) f_single_reqs       <- mkFIFOF;
   FIFOF #(Bit #(64))  f_single_write_data <- mkFIFOF;
   FIFOF #(Read_Data)  f_single_read_data  <- mkFIFOF;

   // Identifiers for requestor client
   Bit #(1) client_id_line   = 0;
   Bit #(1) client_id_single = 1;

   // Limit the number of reads/writes outstanding to 15
   // TODO: change these to concurrent up/down counters?
   Reg #(Bit #(4)) rg_rd_rsps_pending <- mkReg (0);
   Reg #(Bit #(4)) rg_wr_rsps_pending <- mkReg (0);

   // Record errors on write-responses from mem
   Reg #(Bool) rg_write_error <- mkReg (False);

   // AXI4 fabric request/response
   AXI4_Master_Xactor_IFC #(Wd_Id, Wd_Addr, Wd_Data, Wd_User) master_xactor <- mkAXI4_Master_Xactor;

   // ****************************************************************
   // BEHAVIOR: READ RESPONSES (for line and single clients)
   // The following registers identify the client, the beat, etc.

   Reg #(Bit #(1)) rg_rd_client_id <- mkRegU;

   // TODO: change mkFIFOF to mkSizedFIFOF (allowing multiple outstanding reads)
   // may reads can be outstanding
   FIFOF #(Tuple3 #(Bit #(2),     // size_code
		    Bit #(1),     // addr bit [2]
		    Bit #(8)))    // Num beats read-data
         f_rd_rsp_control <- mkFIFOF;
   match { .rd_req_size_code, .rd_req_addr_bit_2, .rd_req_beats } = f_rd_rsp_control.first;

   // Count beats in a single transaction.
   // Note: beat-count is for fabric, not for client-side data.
   // Former is 2 x latter for Fabric32 for 64b line data and 64b single data.

   Reg #(Bit #(8)) rg_rd_beat <- mkRegU;

   // The following are needed for FABRIC32 during each burst response
   // to assemble lower and upper 32b into a 64b response to client
   Reg #(Bool)       rg_rd_data_lower32_ok <- mkRegU;
   Reg #(Bit #(32))  rg_rd_data_lower32    <- mkRegU;

   rule rl_read_data (rg_rd_beat < rd_req_beats);
      rg_rd_beat <= rg_rd_beat + 1;
      Bool last_beat = (rg_rd_beat == (rd_req_beats - 1));
      if (last_beat) begin
	 f_rd_rsp_control.deq;
	 rg_rd_rsps_pending <= rg_rd_rsps_pending - 1;
      end

      let rd_data <- pop_o (master_xactor.o_rd_data);

      Bool      ok   = (rd_data.rresp == axi4_resp_okay);
      Bit #(64) data = zeroExtend (rd_data.rdata);

      Bool do_enq    = True;
      Bool even_beat = (rg_rd_beat [0] == 1'b0);

      // FABRIC32 adjustments
      if (valueOf (Wd_Data) == 32) begin
	 if (rd_req_size_code != 2'b11) begin
	    // B, H, W: only 1 beat
	    Bool in_upper32 = (rd_req_addr_bit_2 == 1'b1);
	    if (in_upper32)
	       data = { data [31:0], 32'b0 };
	    do_enq = True;
	 end
	 else if (even_beat) begin // D, even beat
	    // Just save lower 32b; enq response only after next beat
	    rg_rd_data_lower32_ok <= ok;
	    rg_rd_data_lower32    <= data [31:0];
	    do_enq = False;
	 end
	 else begin // D, odd beat
	    ok   = (rg_rd_data_lower32_ok && ok);
	    data = { data [31:0], rg_rd_data_lower32 };
	    do_enq = True;
	 end
      end

      if (do_enq) begin
	 let rsp = Read_Data {ok: ok, data: data};
	 if (rg_rd_client_id == client_id_line)
	    f_line_read_data.enq (rsp);
	 else
	    f_single_read_data.enq (rsp);
      end

      if (verbosity >= 1) begin
	 $display ("%0d: %m.rl_read_data: ", cur_cycle);
	 if (rg_rd_client_id == client_id_line) $write ("    line (beat %0d)", rg_rd_beat);
	 else                                   $write ("     single");
	 $write (" ok %0d data %0h", pack (ok), data);
	 if ((valueOf (Wd_Data) == 32) && even_beat) $write (" (lower 32b of 64b)");
	 $display ("");
      end
   endrule: rl_read_data

   // ****************************************************************
   // BEHAVIOR: LINE READ REQUESTS

   rule rl_line_read_req (f_line_reqs.first.is_read
			  && (rg_rd_rsps_pending < '1)
			  && (rg_wr_rsps_pending == 0));
      let req <- pop (f_line_reqs);

      dynamicAssert (fv_is_line_aligned (req.addr), "rl_line_read_req addr misaligned");

      let addr = fv_align_to_line (req.addr);

      Fabric_Addr fabric_addr = fv_Addr_to_Fabric_Addr (addr);
      AXI4_Size   fabric_size = ((valueOf (Wd_Data) == 32) ? axsize_4 : axsize_8);

      // Note: AXI4 codes a burst length of 'n' as 'n-1'
      Integer   num_beats  = bytes_per_cline / bytes_per_fabric_data;
      AXI4_Len  fabric_len = fromInteger (num_beats - 1);

      let mem_req_rd_addr = AXI4_Rd_Addr {arid:     fabric_default_id,
					  araddr:   fabric_addr,
					  arlen:    fabric_len,
					  arsize:   fabric_size,
					  arburst:  axburst_incr,
					  arlock:   fabric_default_lock,
					  arcache:  fabric_default_arcache,
					  arprot:   fabric_default_prot,
					  arqos:    fabric_default_qos,
					  arregion: fabric_default_region,
					  aruser:   fabric_default_user};
      master_xactor.i_rd_addr.enq (mem_req_rd_addr);

      f_rd_rsp_control.enq (tuple3 (2'b11, 1'b0, fromInteger (num_beats)));
      rg_rd_client_id    <= client_id_line;
      rg_rd_beat         <= 0;
      rg_rd_rsps_pending <= rg_rd_rsps_pending + 1;

      // Debugging
      if (verbosity >= 1) begin
	 $display ("%0d: %m.rl_line_read_req: ", cur_cycle);
	 $display ("    AXI4_Rd_Addr{araddr %0h arlen %0d (burst len %0d) ",
		   fabric_addr,
		   fabric_len, num_beats,
		   fshow_AXI4_Size (fabric_size),
		   " incr}");
      end
   endrule

   // ****************************************************************
   // BEHAVIOR: Single read requests (not a burst)

   rule rl_single_read_req (f_single_reqs.first.is_read
			    && (rg_rd_rsps_pending < '1)
			    && (rg_wr_rsps_pending == 0));
      let         req        <- pop (f_single_reqs);
      Fabric_Addr fabric_addr = fv_Addr_to_Fabric_Addr (req.addr);
      AXI4_Size   fabric_size = fv_size_code_to_AXI4_Size (req.size_code);

      if (verbosity >= 1)
	 $display ("%0d: %m.rl_single_read_req:\n    AXI4_Rd_Addr{araddr %0h arlen 0 (burst length 1) ",
		   cur_cycle, fabric_addr,  fshow_AXI4_Size (fabric_size), "}");

      Bit #(8)    num_beats   = 1;

      // FABRIC32 adjustments
      if ((valueOf (Wd_Data) == 32) && (req.size_code == 2'b11)) begin
	 fabric_size = axsize_4;
	 num_beats   = 2;
      end
      // Note: AXI4 codes a burst length of 'n' as 'n-1'
      AXI4_Len fabric_len = num_beats - 1;

      let mem_req_rd_addr = AXI4_Rd_Addr {arid:     fabric_default_id,
					  araddr:   fabric_addr,
					  arlen:    0,           // burst len = arlen+1
					  arsize:   fabric_size,
					  arburst:  fabric_default_burst,
					  arlock:   fabric_default_lock,
					  arcache:  fabric_default_arcache,
					  arprot:   fabric_default_prot,
					  arqos:    fabric_default_qos,
					  arregion: fabric_default_region,
					  aruser:   fabric_default_user};
      master_xactor.i_rd_addr.enq (mem_req_rd_addr);

      f_rd_rsp_control.enq (tuple3 (req.size_code, req.addr [2], num_beats));
      rg_rd_client_id    <= client_id_single;
      rg_rd_beat         <= 0;
      rg_rd_rsps_pending <= rg_rd_rsps_pending + 1;
   endrule

   // ****************************************************************
   // BEHAVIOR: WRITE RESPONSES (for line and single clients)
   // The following register identifies the client

   rule rl_write_rsp;
      let wr_resp <- pop_o (master_xactor.o_wr_resp);

      Bool err = False;
      if (rg_wr_rsps_pending == 0) begin
	 rg_write_error <= True;

	 $display ("%0d: %m.rl_write_rsp: ERROR not expecting any write-response:", cur_cycle);
	 $display ("    ", fshow (wr_resp));
      end
      else begin
	 rg_wr_rsps_pending <= rg_wr_rsps_pending - 1;
	 if (wr_resp.bresp != axi4_resp_okay) begin
	    rg_write_error <= True;
	    if (verbosity >= 1) begin
	       $display ("%0d: %m.rl_write_rsp: FABRIC RESPONSE ERROR", cur_cycle);
	       $display ("    ", fshow (wr_resp));
	    end
	 end
	 else if (verbosity >= 1) begin
	    $display ("%0d: %m.rl_write_rsp: pending=%0d, ",
		      cur_cycle, rg_wr_rsps_pending, fshow (wr_resp));
	 end
      end
   endrule

   // ****************************************************************
   // BEHAVIOR: Write data (for both line and single)
   // Assume that data is already lane-aligned for 64b data width.

   FIFOF #(Tuple4 #(Bit #(1),     // client_id
		    Bit #(2),     // size_code
		    Bit #(3),     // addr lsbs
		    Bit #(8)))    // Num beats in write-data
         f_wr_data_control <- mkFIFOF;
   match {.wr_client_id,
	  .wr_req_size_code,
	  .wr_req_addr_lsbs,
	  .wr_req_beats } = f_wr_data_control.first;

   // Count beats in a single transaction
   // Note: beat-count is for fabric, not for client-side data.
   // Former is 2 x latter for Fabric32 for 64b line data and 64b single data.
   Reg #(Bit #(8)) rg_wr_beat <- mkReg (0);

   rule rl_write_data (rg_wr_beat < wr_req_beats);
      Bool last = (rg_wr_beat == (wr_req_beats - 1));
      if (last) begin
	 f_wr_data_control.deq;
	 rg_wr_beat <= 0;
      end
      else
	 rg_wr_beat <= rg_wr_beat + 1;

      Bit #(64) data = ((wr_client_id == client_id_line)
			? f_line_write_data.first
			: f_single_write_data.first);

      // Compute strobe from size and address
      Bit #(8)  strb = case (wr_req_size_code)
			  2'b00: 8'h_01;
			  2'b01: 8'h_03;
			  2'b10: 8'h_0F;
			  2'b11: 8'h_FF;
		       endcase;
      strb = (strb << wr_req_addr_lsbs);

      Bool do_deq    = True;
      Bool even_beat = (rg_wr_beat [0] == 1'b0);

      // FABRIC32 adjustments
      if (valueOf (Wd_Data) == 32) begin
	 if (wr_req_size_code != 2'b11) begin
	    // B, H, W: only 1 beat
	    Bool in_upper32 = (wr_req_addr_lsbs [2] == 1'b1);
	    if (in_upper32) begin
	       data = { 32'b0, data [63:32] };
	       strb = {  4'b0, strb [7:4] };
	    end
	    do_deq = True;
	 end
	 else if (even_beat) begin  // D, even beat
	    do_deq = False;
	 end
	 else begin // D, odd beat
	    data   = { 32'b0, data [63:32] };
	    strb   = '1;
	    do_deq = True;
	 end
      end

      let mem_req_wr_data = AXI4_Wr_Data {wdata:  truncate (data),
					  wstrb:  truncate (strb),
					  wlast:  last,
					  wuser:  fabric_default_user};
      master_xactor.i_wr_data.enq (mem_req_wr_data);

      if (do_deq) begin
	 if (wr_client_id == client_id_line)
	    f_line_write_data.deq;
	 else
	    f_single_write_data.deq;
      end

      if (verbosity >= 1) begin
	 $display ("%0d: %m.rl_write_data: beat %0d/%0d", cur_cycle, rg_wr_beat, wr_req_beats);
	 $display ("    AXI4_Wr_Data{%0h strb %0h last %0d}", data, strb, pack (last));
      end
   endrule

   // ****************************************************************
   // BEHAVIOR: Line-write requests

   rule rl_line_write_req ((! f_line_reqs.first.is_read)
			   && (rg_rd_rsps_pending == 0)
			   && (rg_wr_rsps_pending < '1));
      let req <- pop (f_line_reqs);

      dynamicAssert (fv_is_line_aligned (req.addr), "rl_line_write_req addr misaligned");

      let addr = fv_align_to_line (req.addr);

      Fabric_Addr fabric_addr = fv_Addr_to_Fabric_Addr (addr);
      AXI4_Size   fabric_size = ((valueOf (Wd_Data) == 32) ? axsize_4 : axsize_8);

      // Note: AXI4 codes a burst length of 'n' as 'n-1'
      Integer   num_beats  = bytes_per_cline / bytes_per_fabric_data;
      AXI4_Len  fabric_len = fromInteger (num_beats - 1);

      let mem_req_wr_addr = AXI4_Wr_Addr {awid:     fabric_default_id,
					  awaddr:   fabric_addr,
					  awlen:    fabric_len,
					  awsize:   fabric_size,
					  awburst:  axburst_incr,
					  awlock:   fabric_default_lock,
					  awcache:  fabric_default_awcache,
					  awprot:   fabric_default_prot,
					  awqos:    fabric_default_qos,
					  awregion: fabric_default_region,
					  awuser:   fabric_default_user};
      master_xactor.i_wr_addr.enq (mem_req_wr_addr);

      f_wr_data_control.enq (tuple4 (client_id_line,
				     2'b11,        // size_code D
				     0,            // req.addr lsbs
				     fromInteger (num_beats)));
      rg_wr_rsps_pending <= rg_wr_rsps_pending + 1;

      // Debugging
      if (verbosity >= 1)
	 $display ("%0d: %m.rl_line_write_req: AXI4_Wr_Addr{awaddr %0h awlen %0d burst-length %0d ",
		   cur_cycle,
		   fabric_addr,
		   fabric_len, num_beats,
		   fshow_AXI4_Size (fabric_size),
		   " incr}");
   endrule

   // ****************************************************************
   // Scheduling

   (* descending_urgency = "rl_read_data,       rl_write_rsp" *)
   (* descending_urgency = "rl_write_rsp,       rl_line_read_req" *)
   (* descending_urgency = "rl_line_read_req,   rl_line_write_req" *)
   (* descending_urgency = "rl_line_write_req,  rl_write_data" *)
   (* descending_urgency = "rl_write_data,      rl_single_read_req" *)
   (* descending_urgency = "rl_single_read_req, rl_single_write_req" *)

   // The following should be implied by transitivity, but bsc still cites them
   (* descending_urgency = "rl_read_data,       rl_line_read_req" *)
   (* descending_urgency = "rl_line_read_req,   rl_single_read_req" *)
   (* descending_urgency = "rl_line_write_req,  rl_single_read_req" *)
   (* descending_urgency = "rl_write_rsp,       rl_line_write_req" *)
   (* descending_urgency = "rl_line_write_req,  rl_single_write_req" *)
   (* descending_urgency = "rl_line_read_req,   rl_single_write_req" *)

   // ****************************************************************
   // BEHAVIOR: Single write requests (not a burst)

   rule rl_single_write_req ((! f_single_reqs.first.is_read)
			     && (rg_rd_rsps_pending == 0)
			     && (rg_wr_rsps_pending < '1));
      let req <- pop (f_single_reqs);

      Fabric_Addr fabric_addr = fv_Addr_to_Fabric_Addr (req.addr);
      AXI4_Size   fabric_size = fv_size_code_to_AXI4_Size (req.size_code);
      Bit #(8)    num_beats   = 1;

      // FABRIC32 adjustments
      if ((valueOf (Wd_Data) == 32) && (req.size_code == 2'b11)) begin
	 fabric_size = axsize_4;
	 num_beats   = 2;
      end
      // Note: AXI4 codes a burst length of 'n' as 'n-1'
      AXI4_Len fabric_len = num_beats - 1;

      // AXI4 Write-Address channel
      let mem_req_wr_addr = AXI4_Wr_Addr {awid:     fabric_default_id,
					  awaddr:   fabric_addr,
					  awlen:    fabric_len,
					  awsize:   fabric_size,
					  awburst:  fabric_default_burst,
					  awlock:   fabric_default_lock,
					  awcache:  fabric_default_awcache,
					  awprot:   fabric_default_prot,
					  awqos:    fabric_default_qos,
					  awregion: fabric_default_region,
					  awuser:   fabric_default_user};
      master_xactor.i_wr_addr.enq (mem_req_wr_addr);

      f_wr_data_control.enq (tuple4 (client_id_single,
				     req.size_code,
				     req.addr [2:0],
				     num_beats));
      rg_wr_rsps_pending <= rg_wr_rsps_pending + 1;

      // Debugging
	 $display ("%0d: %m.rl_single_write_req: AXI4_Wr_Addr{awaddr %0h awlen %0d burst-length %0d ",
		   cur_cycle,
		   fabric_addr,
		   fabric_len, num_beats,
		   fshow_AXI4_Size (fabric_size),
		   " incr}");
   endrule

   // ================================================================
   // INTERFACE

   // ----------------
   // interface for cache-line read/write client

   interface Put p_mem_line_req        = toPut (f_line_reqs);
   interface Put p_mem_line_write_data = toPut (f_line_write_data);
   interface Get g_mem_line_read_data  = toGet (f_line_read_data);

   // ----------------
   // interface for word/sub-word read/write client

   interface Put p_mem_single_req        = toPut (f_single_reqs);
   interface Put p_mem_single_write_data = toPut (f_single_write_data);
   interface Get g_mem_single_read_data  = toGet (f_single_read_data);

   // ----------------
   // Write-error from memory

   method Bool mv_write_error = rg_write_error;

   // ----------------
   // Fabric master interface
   interface mem_master = master_xactor.axi_side;
endmodule

// ================================================================

endpackage


fixed iteration termination condition

fixed iteration termination condition

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
import FIFO::*;
import FIFOF::*;
import Vector::*;
import ClientServer::*;
import GetPut::*;
import MemTypes::*;
import MemWriteEngine::*;
import Pipe::*;
import Arith::*;
import ConnectalConfig::*;

`ifdef NumEngineServers
typedef `NumEngineServers NumEngineServers;
`else
typedef 1 NumEngineServers;
`endif

typedef TDiv#(DataBusWidth,32) DataBusWords;

interface MemwriteRequest;
   method Action startWrite(Bit#(32) pointer, Bit#(32) offset, Bit#(32) numWords, Bit#(32) burstLen, Bit#(32) iterCnt);
   method Action getStateDbg();
endinterface

interface Memwrite;
   interface MemwriteRequest request;
   interface Vector#(1,MemWriteClient#(DataBusWidth)) dmaClient;
endinterface

interface MemwriteIndication;
   method Action started(Bit#(32) numWords);
   method Action reportStateDbg(Bit#(32) wrCnt, Bit#(32) srcGen);
   method Action writeDone(Bit#(32) v);
endinterface

module  mkMemwrite#(MemwriteIndication indication) (Memwrite);

   Reg#(SGLId)   pointer <- mkReg(0);
   Reg#(Bit#(32))       numWords <- mkReg(0);
   Reg#(Bit#(32))       burstLen <- mkReg(0);
   FIFOF#(void)               cf <- mkSizedFIFOF(1);

   Vector#(NumEngineServers, Reg#(Bit#(32)))         srcGens <- replicateM(mkReg(0));
   Reg#(Bit#(32))                                writeOffset <- mkReg(0);
   Reg#(Bit#(32))                                    iterCnt <- mkReg(0);
   Vector#(NumEngineServers, Reg#(Bit#(32)))        iterCnts <- replicateM(mkReg(0));
   Vector#(NumEngineServers, FIFOF#(void))               cfs <- replicateM(mkSizedFIFOF(1));
   Vector#(NumEngineServers, FIFOF#(Bool))       finishFifos <- replicateM(mkFIFOF);
   MemWriteEngine#(DataBusWidth,DataBusWidth,2,NumEngineServers)                we <- mkMemWriteEngine;
   Bit#(32) chunk = extend(numWords)*4;

   for(Integer i = 0; i < valueOf(NumEngineServers); i=i+1) begin
      rule start (iterCnts[i] > 0);
	 we.writeServers[i].request.put(MemengineCmd{tag:0, sglId:pointer, base:extend(writeOffset)+(fromInteger(i)*chunk), len:truncate(chunk), burstLen:truncate(burstLen*4)});
	 Bit#(32) srcGen = (fromInteger(i) << 28);
	 srcGens[i] <= srcGen;
	 $display("start %d/%d, %h 0x%x %h chunk=%h", i, valueOf(NumEngineServers), srcGen, iterCnts[i], writeOffset, chunk);
	 cfs[i].enq(?);
	 iterCnts[i] <= iterCnts[i]-1;
      endrule
`ifdef MEMENGINE_REQUEST_CYCLES
      rule requestCycles;
	 let reqcycles <- toGet(we.writeServers[i].requestCycles).get();
	 $display("request %d took %d cycles", reqcycles.tag, reqcycles.cycles);
      endrule
`endif
      rule finish;
	 $display("finish %d 0x%x", i, iterCnts[i]);
	 let rv <- we.writeServers[i].done.get;
	 finishFifos[i].enq(rv);
      endrule
      rule src if (cfs[i].notEmpty);
	 Vector#(DataBusWords, Bit#(32)) v;
	 for (Integer j = 0; j < valueOf(DataBusWords); j = j + 1)
	    v[j] = srcGens[i]+fromInteger(j);
	 we.writeServers[i].data.enq(pack(v));
	 let new_srcGen = srcGens[i]+fromInteger(valueOf(DataBusWords));
	 srcGens[i] <= new_srcGen;
	 if (new_srcGen[27:0] >= truncate(numWords))
	    cfs[i].deq;
      endrule
   end

   PipeOut#(Vector#(NumEngineServers, Bool)) finishPipe <- mkJoinVector(id, map(toPipeOut, finishFifos));
   PipeOut#(Bool) finishReducePipe <- mkReducePipe(uncurry(booland), finishPipe);

   rule indicate_finish;
      let rv <- toGet(finishReducePipe).get();
      $display("indicate_finish rv=%d iterCnt=%d", rv, iterCnt);
      if (iterCnt == 1) begin
	 cf.deq;
	 indication.writeDone(0);
      end
      iterCnt <= iterCnt - 1;
   endrule

   interface MemWriteClient dmaClient = cons(we.dmaClient, nil);
   interface MemwriteRequest request;
       method Action startWrite(Bit#(32) wp, Bit#(32) off, Bit#(32) nw, Bit#(32) bl, Bit#(32) ic);
	  $display("startWrite pointer=%d offset=%d numWords=%h burstLen=%d iterCnt=%d", pointer, off, nw, bl, ic);
	  indication.started(nw);
	  pointer <= wp;
	  cf.enq(?);
	  numWords  <= nw;
	  burstLen  <= bl;
	  iterCnt <= ic;
	  writeOffset <= off*4;
	  for(Integer i = 0; i < valueOf(NumEngineServers); i=i+1)
	     iterCnts[i] <= ic;
       endmethod
   endinterface
endmodule

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
import FIFO::*;
import FIFOF::*;
import Vector::*;
import ClientServer::*;
import GetPut::*;
import MemTypes::*;
import MemWriteEngine::*;
import Pipe::*;
import Arith::*;
import ConnectalConfig::*;

`ifdef NumEngineServers
typedef `NumEngineServers NumEngineServers;
`else
typedef 1 NumEngineServers;
`endif

typedef TDiv#(DataBusWidth,32) DataBusWords;

interface MemwriteRequest;
   method Action startWrite(Bit#(32) pointer, Bit#(32) offset, Bit#(32) numWords, Bit#(32) burstLen, Bit#(32) iterCnt);
   method Action getStateDbg();
endinterface

interface Memwrite;
   interface MemwriteRequest request;
   interface Vector#(1,MemWriteClient#(DataBusWidth)) dmaClient;
endinterface

interface MemwriteIndication;
   method Action started(Bit#(32) numWords);
   method Action reportStateDbg(Bit#(32) wrCnt, Bit#(32) srcGen);
   method Action writeDone(Bit#(32) v);
endinterface

module  mkMemwrite#(MemwriteIndication indication) (Memwrite);

   Reg#(SGLId)   pointer <- mkReg(0);
   Reg#(Bit#(32))       numWords <- mkReg(0);
   Reg#(Bit#(32))       burstLen <- mkReg(0);
   FIFOF#(void)               cf <- mkSizedFIFOF(1);

   Vector#(NumEngineServers, Reg#(Bit#(32)))         srcGens <- replicateM(mkReg(0));
   Reg#(Bit#(32))                                writeOffset <- mkReg(0);
   Reg#(Bit#(32))                                    iterCnt <- mkReg(0);
   Vector#(NumEngineServers, Reg#(Bit#(32)))        iterCnts <- replicateM(mkReg(0));
   Vector#(NumEngineServers, FIFOF#(void))               cfs <- replicateM(mkSizedFIFOF(1));
   Vector#(NumEngineServers, FIFOF#(Bool))       finishFifos <- replicateM(mkFIFOF);
   MemWriteEngine#(DataBusWidth,DataBusWidth,2,NumEngineServers)                we <- mkMemWriteEngine;
   Bit#(32) chunk = extend(numWords)*4;

   for(Integer i = 0; i < valueOf(NumEngineServers); i=i+1) begin
      rule start (iterCnts[i] > 0);
	 we.writeServers[i].request.put(MemengineCmd{tag:0, sglId:pointer, base:extend(writeOffset)+(fromInteger(i)*chunk), len:truncate(chunk), burstLen:truncate(burstLen*4)});
	 Bit#(32) srcGen = (fromInteger(i) << 28);
	 srcGens[i] <= srcGen;
	 $display("start %d/%d, %h 0x%x %h chunk=%h", i, valueOf(NumEngineServers), srcGen, iterCnts[i], writeOffset, chunk);
	 cfs[i].enq(?);
	 iterCnts[i] <= iterCnts[i]-1;
      endrule
`ifdef MEMENGINE_REQUEST_CYCLES
      rule requestCycles;
	 let reqcycles <- toGet(we.writeServers[i].requestCycles).get();
	 $display("request %d took %d cycles", reqcycles.tag, reqcycles.cycles);
      endrule
`endif
      rule finish;
	 $display("finish %d 0x%x", i, iterCnts[i]);
	 let rv <- we.writeServers[i].done.get;
	 finishFifos[i].enq(rv);
      endrule
      rule src if (cfs[i].notEmpty);
	 Vector#(DataBusWords, Bit#(32)) v;
	 for (Integer j = 0; j < valueOf(DataBusWords); j = j + 1)
	    v[j] = srcGens[i]+fromInteger(j);
	 we.writeServers[i].data.enq(pack(v));
	 let new_srcGen = srcGens[i]+fromInteger(valueOf(DataBusWords));
	 srcGens[i] <= new_srcGen;
	 if (new_srcGen[27:0] == truncate(writeOffset/4))
	    cfs[i].deq;
      endrule
   end

   PipeOut#(Vector#(NumEngineServers, Bool)) finishPipe <- mkJoinVector(id, map(toPipeOut, finishFifos));
   PipeOut#(Bool) finishReducePipe <- mkReducePipe(uncurry(booland), finishPipe);

   rule indicate_finish;
      let rv <- toGet(finishReducePipe).get();
      $display("indicate_finish rv=%d iterCnt=%d", rv, iterCnt);
      if (iterCnt == 1) begin
	 cf.deq;
	 indication.writeDone(0);
      end
      iterCnt <= iterCnt - 1;
   endrule

   interface MemWriteClient dmaClient = cons(we.dmaClient, nil);
   interface MemwriteRequest request;
       method Action startWrite(Bit#(32) wp, Bit#(32) off, Bit#(32) nw, Bit#(32) bl, Bit#(32) ic);
	  $display("startWrite pointer=%d offset=%d numWords=%h burstLen=%d iterCnt=%d", pointer, off, nw, bl, ic);
	  indication.started(nw);
	  pointer <= wp;
	  cf.enq(?);
	  numWords  <= nw;
	  burstLen  <= bl;
	  iterCnt <= ic;
	  writeOffset <= off*4;
	  for(Integer i = 0; i < valueOf(NumEngineServers); i=i+1)
	     iterCnts[i] <= ic;
       endmethod
   endinterface
endmodule


Simulate DataBusWidth=256 in Bsim

Simulate DataBusWidth=256 in Bsim

// Copyright (c) 2014 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import Clocks            :: *;
import Vector            :: *;
import FIFOF             :: *;
import FIFO              :: *;
import SpecialFIFOs      :: *;
import GetPut            :: *;
import Connectable       :: *;
import StmtFSM           :: *;
import Portal            :: *;
import Leds              :: *;
import Top               :: *;
import MemTypes          :: *;
import HostInterface     :: *;
import CtrlMux           :: *;
import ClientServer      :: *;
import MemSlaveEngine    :: *;
import MemMasterEngine   :: *;
import PCIE              :: *;

`ifndef PinType
`define PinType Empty
`endif

typedef `PinType PinType;
typedef `NumberOfMasters NumberOfMasters;

// implemented in BsimCtrl.cxx
import "BDPI" function Action                 initPortal();
import "BDPI" function Bool                   checkForRequest(Bit#(32) v);
import "BDPI" function ActionValue#(Bit#(64)) getRequest32(Bit#(32) v);
import "BDPI" function Action                 readResponse32(Bit#(32) d, Bit#(32) tag);
import "BDPI" function Action                 interruptLevel(Bit#(1) d);

// implemented in BsimDma.cxx
import "BDPI" function Action write_pareff32(Bit#(32) handle, Bit#(32) addr, Bit#(32) v);
import "BDPI" function Action write_pareff64(Bit#(32) handle, Bit#(32) addr, Bit#(64) v);
import "BDPI" function ActionValue#(Bit#(32)) read_pareff32(Bit#(32) handle, Bit#(32) addr);
import "BDPI" function ActionValue#(Bit#(64)) read_pareff64(Bit#(32) handle, Bit#(32) addr);

module mkBsimCtrlReadWrite(PhysMemMaster#(clientAddrWidth, clientBusWidth))
   provisos(Add#(a__, 32, clientAddrWidth), Add#(b__, 32, clientBusWidth));
   FIFO#(Bit#(clientBusWidth)) wf <- mkPipelineFIFO;
   Reg#(Bit#(32)) cycles      <- mkReg(0);
   rule count;
      cycles <= cycles + 1;
   endrule

   let verbose = False;
   let init_fsm <- mkOnce(initPortal());

   rule init_rule;
      init_fsm.start;
   endrule
   interface PhysMemReadClient read_client;
     interface Get readReq;
	 method ActionValue#(PhysMemRequest#(clientAddrWidth)) get() if (checkForRequest(0));
	 //$write("req_ar: ");
	 let ra <- getRequest32(0);
	 //$display("ra=%h", ra);
	 let burstLen = fromInteger(valueOf(clientBusWidth) / 8);
	 if (verbose) $display("\n%d BsimHost.readReq addr=%h burstLen=%d", cycles, ra, burstLen);
	 return PhysMemRequest { addr: extend(ra[31:0]), burstLen: burstLen, tag: truncate(ra[63:32])};
	 endmethod
     endinterface
     interface Put readData;
	 method Action put(MemData#(clientBusWidth) rd);
	 //$display("resp_read: rd=%h", rd);
	 if (verbose) $display("%d BsimHost.readData %h", cycles, rd.data);
	 readResponse32(truncate(rd.data), extend(rd.tag));
	 endmethod
     endinterface
   endinterface
   interface PhysMemWriteClient write_client;
     interface Get writeReq;
	 method ActionValue#(PhysMemRequest#(clientAddrWidth)) get() if (checkForRequest(1));
	 let wd <- getRequest32(1);
	 wf.enq(extend(wd[63:32]));
	 let burstLen = fromInteger(valueOf(clientBusWidth) / 8);
	 if (verbose) $display("\n%d BsimHost.writeReq addr/data=%h burstLen=%d", cycles, wd, burstLen);
	 return PhysMemRequest { addr: extend(wd[31:0]), burstLen: burstLen, tag: 0 };
	 endmethod
     endinterface
     interface Get writeData;
	 method ActionValue#(MemData#(clientBusWidth)) get;
	 wf.deq;
	 if (verbose) $display("%d BsimHost.writeData %h", cycles, wf.first);
	 return MemData { data: wf.first, tag: 0, last: True };
	 endmethod
     endinterface
     interface Put writeDone;
	 method Action put(Bit#(MemTagSize) resp);
	 if (verbose) $display("%d BsimHost.writeDone %d", cycles, resp);
	 endmethod
     endinterface
   endinterface
endmodule

interface BsimRdmaReadWrite#(numeric type dsz);
   method Action write_pareff(Bit#(32) handle, Bit#(32) addr, Bit#(dsz) v);
   method ActionValue#(Bit#(dsz)) read_pareff(Bit#(32) handle, Bit#(32) addr);
endinterface

typeclass SelectBsimRdmaReadWrite#(numeric type dsz);
   module selectBsimRdmaReadWrite(BsimRdmaReadWrite#(dsz) ifc);
endtypeclass

instance SelectBsimRdmaReadWrite#(32);
   module selectBsimRdmaReadWrite(BsimRdmaReadWrite#(32) ifc);
       method Action write_pareff(Bit#(32) handle, Bit#(32) addr, Bit#(32) v);
	  write_pareff32(handle, addr, v);
       endmethod
       method ActionValue#(Bit#(32)) read_pareff(Bit#(32) handle, Bit#(32) addr);
	  let v <- read_pareff32(handle, addr);
	  return v;
       endmethod
   endmodule
endinstance
instance SelectBsimRdmaReadWrite#(64);
   module selectBsimRdmaReadWrite(BsimRdmaReadWrite#(64) ifc);
       method Action write_pareff(Bit#(32) handle, Bit#(32) addr, Bit#(64) v);
	  write_pareff64(handle, addr, v);
       endmethod
       method ActionValue#(Bit#(64)) read_pareff(Bit#(32) handle, Bit#(32) addr);
	  let v <- read_pareff64(handle, addr);
	  return v;
       endmethod
   endmodule
endinstance
instance SelectBsimRdmaReadWrite#(128);
   module selectBsimRdmaReadWrite(BsimRdmaReadWrite#(128) ifc);
       method Action write_pareff(Bit#(32) handle, Bit#(32) addr, Bit#(128) v);
	  write_pareff64(handle, addr, v[63:0]);
	  write_pareff64(handle, addr+8, v[127:64]);
       endmethod
       method ActionValue#(Bit#(128)) read_pareff(Bit#(32) handle, Bit#(32) addr);
	  let v0 <- read_pareff64(handle, addr);
	  let v1 <- read_pareff64(handle, addr+8);
	  return {v1,v0};
       endmethod
   endmodule
endinstance
instance SelectBsimRdmaReadWrite#(256);
   module selectBsimRdmaReadWrite(BsimRdmaReadWrite#(256) ifc);
       method Action write_pareff(Bit#(32) handle, Bit#(32) addr, Bit#(256) v);
	  write_pareff64(handle, addr, v[63:0]);
	  write_pareff64(handle, addr+8, v[127:64]);
	  write_pareff64(handle, addr+16, v[191:128]);
	  write_pareff64(handle, addr+24, v[255:192]);
       endmethod
       method ActionValue#(Bit#(256)) read_pareff(Bit#(32) handle, Bit#(32) addr);
	  let v0 <- read_pareff64(handle, addr);
	  let v1 <- read_pareff64(handle, addr+8);
	  let v2 <- read_pareff64(handle, addr+16);
	  let v3 <- read_pareff64(handle, addr+24);
	  return {v3,v2,v1,v0};
       endmethod
   endmodule
endinstance

module mkBsimDmaMaster(PhysMemSlave#(serverAddrWidth,serverBusWidth))
   provisos(Div#(serverBusWidth,8,dataWidthBytes),
	    Mul#(dataWidthBytes,8,serverBusWidth),
	    Log#(dataWidthBytes,beatShift),
	    SelectBsimRdmaReadWrite#(serverBusWidth));

   let verbose = False;
   BsimRdmaReadWrite#(serverBusWidth) rw <- selectBsimRdmaReadWrite();

   Reg#(Bit#(serverAddrWidth)) readAddrr <- mkReg(0);
   Reg#(Bit#(BurstLenSize))  readLen <- mkReg(0);
   Reg#(Bit#(MemTagSize)) readId <- mkReg(0);
   Reg#(Bit#(serverAddrWidth)) writeAddrr <- mkReg(0);
   Reg#(Bit#(BurstLenSize))  writeLen <- mkReg(0);
   Reg#(Bit#(MemTagSize)) writeId <- mkReg(0);

   let readLatency_I = 150;
   let writeLatency_I = 150;

   Bit#(64) readLatency = fromInteger(readLatency_I);
   Bit#(64) writeLatency = fromInteger(writeLatency_I);

   Reg#(Bit#(64)) req_ar_b_ts <- mkReg(0);
   Reg#(Bit#(64)) req_aw_b_ts <- mkReg(0);
   Reg#(Bit#(64)) cycles <- mkReg(0);
   Reg#(Bit#(64)) last_reqAr <- mkReg(0);
   Reg#(Bit#(64)) last_read_eob <- mkReg(0);
   Reg#(Bit#(64)) last_write_eob <- mkReg(0);

   FIFOF#(Tuple2#(Bit#(64), PhysMemRequest#(serverAddrWidth)))  readDelayFifo <- mkSizedFIFOF(readLatency_I);
   FIFOF#(Tuple2#(Bit#(64),PhysMemRequest#(serverAddrWidth))) writeDelayFifo <- mkSizedFIFOF(writeLatency_I);

   FIFOF#(Tuple2#(Bit#(64), Bit#(MemTagSize))) bFifo <- mkSizedFIFOF(writeLatency_I);

   rule increment_cycle;
      cycles <= cycles+1;
   endrule

   let read_jitter = True; //cycles[4:0] == 0;
   let write_jitter = True; //cycles[4:0] == 5;

   Reg#(Bit#(8))  burstReg <- mkReg(0);
   FIFO#(Bit#(8)) reqs <- mkSizedFIFO(32);

   let beat_shift = fromInteger(valueOf(beatShift));

   interface PhysMemReadServer read_server;
      interface Put readReq;
	 method Action put(PhysMemRequest#(serverAddrWidth) req);
            if (verbose) $display("%d axiSlave.read.readAddr %h bc %d", cycles, req.addr, req.burstLen);
	    //readAddrGenerator.request.put(req);
	    readDelayFifo.enq(tuple2(cycles,req));
	 endmethod
      endinterface
      interface Get readData;
	 method ActionValue#(MemData#(serverBusWidth)) get() if (((readLen > 0) || (readLen == 0 && (cycles-tpl_1(readDelayFifo.first)) > readLatency)) && read_jitter);
	 Bit#(BurstLenSize) read_len = ?;
	 Bit#(serverAddrWidth) read_addr = ?;
	 Bit#(MemTagSize) read_id = ?;
	 Bit#(8) handle = ?;
	 if (readLen == 0 && (cycles-tpl_1(readDelayFifo.first)) > readLatency) begin
	    req_ar_b_ts <= cycles;
	    let req = tpl_2(readDelayFifo.first);
	    readDelayFifo.deq;
	    read_len = req.burstLen>>beat_shift;
	    read_addr = req.addr;
	    read_id = req.tag;
	    handle = req.addr[39:32];
	    //if(id==1) $display("mkBsimHost::resp_read_a: %d %d %d", req.tag,  cycles-last_read_eob, (req.burstLen>>beat_shift)-1);
	    //last_read_eob <= cycles;
	 end
	 else begin
	    //$display("mkBsimHost::resp_read_b: %d %d", readId,  cycles-last_read_eob);
	    //last_read_eob <= cycles;
	    handle = readAddrr[39:32];
	    read_addr = readAddrr;
	    read_id = readId;
	    read_len = readLen;
	 end
	 Bit#(serverBusWidth) v <- rw.read_pareff(extend(handle), read_addr[31:0]);
	 readLen <= read_len - 1;
	 readId <= read_id;
	 readAddrr <= read_addr + fromInteger(valueOf(serverBusWidth)/8);
	 //$display("mkBsimHost::resp_read id=%d %d", read_id, read_len);
	 //return Axi3ReadResponse { data: v, resp: 0, last: pack(readLen == 1), id: read_id};
            //if (verbose) $display("%d read_server.readData (b) %h", cycles, data);
            return MemData { data: v, tag: read_id, last: readLen == 1};
	 endmethod
      endinterface
   endinterface
   interface PhysMemWriteServer write_server;
      interface Put writeReq;
	 method Action put(PhysMemRequest#(serverAddrWidth) req);
	 //$display("mkBsimHost::req_aw id=%d", req.tag);
	 writeDelayFifo.enq(tuple2(cycles,req));
	 endmethod
      endinterface
      interface Put writeData;
	 method Action put(MemData#(serverBusWidth) resp) if (((writeLen > 0) || (writeLen == 0 && (cycles-tpl_1(writeDelayFifo.first)) > writeLatency)) && write_jitter);
	    //let addrBeat <- writeAddrGenerator.addrBeat.get();
	    //let addr = addrBeat.addr;
	    //Bit#(bramAddrWidth) regFileAddr = truncate(addr/fromInteger(valueOf(TDiv#(serverBusWidth,8))));
            //br.request.put(BRAMRequest{write:True, responseOnWrite:False, address:regFileAddr, datain:resp.data});
	 Bit#(BurstLenSize) write_len = ?;
	 Bit#(serverAddrWidth) write_addr = ?;
	 Bit#(MemTagSize) write_id = ?;
	 Bit#(8) handle = ?;
	 if (writeLen == 0 && (cycles-tpl_1(writeDelayFifo.first)) > writeLatency) begin
	    req_aw_b_ts <= cycles;
	    let req = tpl_2(writeDelayFifo.first);
	    writeDelayFifo.deq;
	    write_addr = req.addr;
	    write_len = req.burstLen>>beat_shift;
	    write_id = req.tag;
	    handle = req.addr[39:32];
	    //$display("mkBsimHost::resp_write_a: %d %d", req.tag,  cycles-last_write_eob);
	    //last_write_eob <= cycles;
	 end
	 else begin
	    //$display("mkBsimHost::resp_write_b: %d %d", writeId,  cycles-last_write_eob);
	    //last_write_eob <= cycles;
	    handle = writeAddrr[39:32];
	    write_len = writeLen;
	    write_addr = writeAddrr;
	    write_id = writeId;
	 end
	 rw.write_pareff(extend(handle), write_addr[31:0], resp.data);
	 //$display("write_resp(%d): handle=%d addr=%h v=%h", cycles, handle, write_addr, resp.data);
	 writeId <= write_id;
	 writeLen <= write_len - 1;
	 writeAddrr <= write_addr + fromInteger(valueOf(serverBusWidth)/8);
	 if (write_len == 1) begin
	    bFifo.enq(tuple2(cycles,write_id));
	 end
            //if (verbose) $display("%d write_server.writeAddr %h bc %d", cycles, req.addr, req.burstLen);
            //if (verbose) $display("%d write_server.writeData %h %h %d", cycles, addr, resp.data, addrBeat.bc);
            //if (addrBeat.last)
               //writeTagFifo.enq(addrBeat.tag);
	 endmethod
      endinterface
      interface Get writeDone;
	 method ActionValue#(Bit#(MemTagSize)) get() if ((cycles-tpl_1(bFifo.first)) > writeLatency);
	 bFifo.deq();
	 return tpl_2(bFifo.first());
	 endmethod
      endinterface
   endinterface
endmodule

module  mkBsimHost#(Clock derived_clock, Reset derived_reset)(BsimHost#(clientAddrWidth, clientBusWidth, clientIdWidth,
			      serverAddrWidth, serverBusWidth, serverIdWidth,
			      nSlaves))
   provisos (SelectBsimRdmaReadWrite#(serverBusWidth),
             Add#(a__, 32, clientAddrWidth), Add#(b__, 32, clientBusWidth),
             Mul#(TDiv#(serverBusWidth, 8), 8, serverBusWidth));

   Vector#(nSlaves,PhysMemSlave#(serverAddrWidth,  serverBusWidth)) servers <- replicateM(mkBsimDmaMaster);
   PhysMemMaster#(clientAddrWidth, clientBusWidth) crw <- mkBsimCtrlReadWrite();

   interface mem_servers = servers;
   interface PhysMemMaster mem_client = crw;
   interface derivedClock = derived_clock;
   interface derivedReset = derived_reset;
endmodule

module  mkBsimTop(Empty)
   provisos (SelectBsimRdmaReadWrite#(DataBusWidth));
   let divider <- mkClockDivider(2);
   Clock derivedClock = divider.fastClock;
   Clock singleClock = divider.slowClock;
   Reset derivedReset <- exposeCurrentReset;
   let single_reset <- mkReset(2, True, singleClock);
   Reset singleReset = single_reset.new_rst;
   BsimHost#(32,32,12,PhysAddrWidth,DataBusWidth,6,NumberOfMasters) host <- mkBsimHost(clocked_by singleClock, reset_by singleReset, derivedClock, derivedReset);
   ConnectalTop#(PhysAddrWidth,DataBusWidth,PinType,NumberOfMasters) top <- mkConnectalTop(
`ifdef IMPORT_HOSTIF
       host,
`endif
       clocked_by singleClock, reset_by singleReset);
   mapM(uncurry(mkConnection),zip(top.masters, host.mem_servers), clocked_by singleClock, reset_by singleReset);
`ifndef BSIM_EXERCISE_MEM_MASTER_SLAVE
   mkConnection(host.mem_client, top.slave, clocked_by singleClock, reset_by singleReset);
`else
   PciId masterPciId = unpack(22);
   PciId slavePciId = unpack(23);
   MemMasterEngine masterEngine <- mkMemMasterEngine(masterPciId, clocked_by singleClock, reset_by singleReset);
   MemSlaveEngine#(32) slaveEngine <- mkMemSlaveEngine(slavePciId, clocked_by singleClock, reset_by singleReset);
   mkConnection(host.mem_client, slaveEngine.slave, clocked_by singleClock, reset_by singleReset);
   mkConnection(slaveEngine.tlp.request, masterEngine.tlp.response, clocked_by singleClock, reset_by singleReset);
   mkConnection(slaveEngine.tlp.response, masterEngine.tlp.request, clocked_by singleClock, reset_by singleReset);
   mkConnection(masterEngine.master, top.slave, clocked_by singleClock, reset_by singleReset);
`endif

   let intr_mux <- mkInterruptMux(top.interrupt);
   rule int_rule;
      interruptLevel(truncate(pack(intr_mux)));
   endrule

`ifdef BSIMRESPONDER
   `BSIMRESPONDER (top.pins);
`endif
endmodule

// Copyright (c) 2014 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import Clocks            :: *;
import Vector            :: *;
import FIFOF             :: *;
import FIFO              :: *;
import SpecialFIFOs      :: *;
import GetPut            :: *;
import Connectable       :: *;
import StmtFSM           :: *;
import Portal            :: *;
import Leds              :: *;
import Top               :: *;
import MemTypes          :: *;
import HostInterface     :: *;
import CtrlMux           :: *;
import ClientServer      :: *;
import MemSlaveEngine    :: *;
import MemMasterEngine   :: *;
import PCIE              :: *;

`ifndef PinType
`define PinType Empty
`endif

typedef `PinType PinType;
typedef `NumberOfMasters NumberOfMasters;

// implemented in BsimCtrl.cxx
import "BDPI" function Action                 initPortal();
import "BDPI" function Bool                   checkForRequest(Bit#(32) v);
import "BDPI" function ActionValue#(Bit#(64)) getRequest32(Bit#(32) v);
import "BDPI" function Action                 readResponse32(Bit#(32) d, Bit#(32) tag);
import "BDPI" function Action                 interruptLevel(Bit#(1) d);

// implemented in BsimDma.cxx
import "BDPI" function Action write_pareff32(Bit#(32) handle, Bit#(32) addr, Bit#(32) v);
import "BDPI" function Action write_pareff64(Bit#(32) handle, Bit#(32) addr, Bit#(64) v);
import "BDPI" function ActionValue#(Bit#(32)) read_pareff32(Bit#(32) handle, Bit#(32) addr);
import "BDPI" function ActionValue#(Bit#(64)) read_pareff64(Bit#(32) handle, Bit#(32) addr);

module mkBsimCtrlReadWrite(PhysMemMaster#(clientAddrWidth, clientBusWidth))
   provisos(Add#(a__, 32, clientAddrWidth), Add#(b__, 32, clientBusWidth));
   FIFO#(Bit#(clientBusWidth)) wf <- mkPipelineFIFO;
   Reg#(Bit#(32)) cycles      <- mkReg(0);
   rule count;
      cycles <= cycles + 1;
   endrule

   let verbose = False;
   let init_fsm <- mkOnce(initPortal());

   rule init_rule;
      init_fsm.start;
   endrule
   interface PhysMemReadClient read_client;
     interface Get readReq;
	 method ActionValue#(PhysMemRequest#(clientAddrWidth)) get() if (checkForRequest(0));
	 //$write("req_ar: ");
	 let ra <- getRequest32(0);
	 //$display("ra=%h", ra);
	 let burstLen = fromInteger(valueOf(clientBusWidth) / 8);
	 if (verbose) $display("\n%d BsimHost.readReq addr=%h burstLen=%d", cycles, ra, burstLen);
	 return PhysMemRequest { addr: extend(ra[31:0]), burstLen: burstLen, tag: truncate(ra[63:32])};
	 endmethod
     endinterface
     interface Put readData;
	 method Action put(MemData#(clientBusWidth) rd);
	 //$display("resp_read: rd=%h", rd);
	 if (verbose) $display("%d BsimHost.readData %h", cycles, rd.data);
	 readResponse32(truncate(rd.data), extend(rd.tag));
	 endmethod
     endinterface
   endinterface
   interface PhysMemWriteClient write_client;
     interface Get writeReq;
	 method ActionValue#(PhysMemRequest#(clientAddrWidth)) get() if (checkForRequest(1));
	 let wd <- getRequest32(1);
	 wf.enq(extend(wd[63:32]));
	 let burstLen = fromInteger(valueOf(clientBusWidth) / 8);
	 if (verbose) $display("\n%d BsimHost.writeReq addr/data=%h burstLen=%d", cycles, wd, burstLen);
	 return PhysMemRequest { addr: extend(wd[31:0]), burstLen: burstLen, tag: 0 };
	 endmethod
     endinterface
     interface Get writeData;
	 method ActionValue#(MemData#(clientBusWidth)) get;
	 wf.deq;
	 if (verbose) $display("%d BsimHost.writeData %h", cycles, wf.first);
	 return MemData { data: wf.first, tag: 0, last: True };
	 endmethod
     endinterface
     interface Put writeDone;
	 method Action put(Bit#(MemTagSize) resp);
	 if (verbose) $display("%d BsimHost.writeDone %d", cycles, resp);
	 endmethod
     endinterface
   endinterface
endmodule

interface BsimRdmaReadWrite#(numeric type dsz);
   method Action write_pareff(Bit#(32) handle, Bit#(32) addr, Bit#(dsz) v);
   method ActionValue#(Bit#(dsz)) read_pareff(Bit#(32) handle, Bit#(32) addr);
endinterface

typeclass SelectBsimRdmaReadWrite#(numeric type dsz);
   module selectBsimRdmaReadWrite(BsimRdmaReadWrite#(dsz) ifc);
endtypeclass

instance SelectBsimRdmaReadWrite#(32);
   module selectBsimRdmaReadWrite(BsimRdmaReadWrite#(32) ifc);
       method Action write_pareff(Bit#(32) handle, Bit#(32) addr, Bit#(32) v);
	  write_pareff32(handle, addr, v);
       endmethod
       method ActionValue#(Bit#(32)) read_pareff(Bit#(32) handle, Bit#(32) addr);
	  let v <- read_pareff32(handle, addr);
	  return v;
       endmethod
   endmodule
endinstance
instance SelectBsimRdmaReadWrite#(64);
   module selectBsimRdmaReadWrite(BsimRdmaReadWrite#(64) ifc);
       method Action write_pareff(Bit#(32) handle, Bit#(32) addr, Bit#(64) v);
	  write_pareff64(handle, addr, v);
       endmethod
       method ActionValue#(Bit#(64)) read_pareff(Bit#(32) handle, Bit#(32) addr);
	  let v <- read_pareff64(handle, addr);
	  return v;
       endmethod
   endmodule
endinstance
instance SelectBsimRdmaReadWrite#(128);
   module selectBsimRdmaReadWrite(BsimRdmaReadWrite#(128) ifc);
       method Action write_pareff(Bit#(32) handle, Bit#(32) addr, Bit#(128) v);
	  write_pareff64(handle, addr, v[63:0]);
	  write_pareff64(handle, addr+8, v[127:64]);
       endmethod
       method ActionValue#(Bit#(128)) read_pareff(Bit#(32) handle, Bit#(32) addr);
	  let v0 <- read_pareff64(handle, addr);
	  let v1 <- read_pareff64(handle, addr+8);
	  return {v1,v0};
       endmethod
   endmodule
endinstance

module mkBsimDmaMaster(PhysMemSlave#(serverAddrWidth,serverBusWidth))
   provisos(Div#(serverBusWidth,8,dataWidthBytes),
	    Mul#(dataWidthBytes,8,serverBusWidth),
	    Log#(dataWidthBytes,beatShift),
	    SelectBsimRdmaReadWrite#(serverBusWidth));

   let verbose = False;
   BsimRdmaReadWrite#(serverBusWidth) rw <- selectBsimRdmaReadWrite();

   Reg#(Bit#(serverAddrWidth)) readAddrr <- mkReg(0);
   Reg#(Bit#(BurstLenSize))  readLen <- mkReg(0);
   Reg#(Bit#(MemTagSize)) readId <- mkReg(0);
   Reg#(Bit#(serverAddrWidth)) writeAddrr <- mkReg(0);
   Reg#(Bit#(BurstLenSize))  writeLen <- mkReg(0);
   Reg#(Bit#(MemTagSize)) writeId <- mkReg(0);

   let readLatency_I = 150;
   let writeLatency_I = 150;

   Bit#(64) readLatency = fromInteger(readLatency_I);
   Bit#(64) writeLatency = fromInteger(writeLatency_I);

   Reg#(Bit#(64)) req_ar_b_ts <- mkReg(0);
   Reg#(Bit#(64)) req_aw_b_ts <- mkReg(0);
   Reg#(Bit#(64)) cycles <- mkReg(0);
   Reg#(Bit#(64)) last_reqAr <- mkReg(0);
   Reg#(Bit#(64)) last_read_eob <- mkReg(0);
   Reg#(Bit#(64)) last_write_eob <- mkReg(0);

   FIFOF#(Tuple2#(Bit#(64), PhysMemRequest#(serverAddrWidth)))  readDelayFifo <- mkSizedFIFOF(readLatency_I);
   FIFOF#(Tuple2#(Bit#(64),PhysMemRequest#(serverAddrWidth))) writeDelayFifo <- mkSizedFIFOF(writeLatency_I);

   FIFOF#(Tuple2#(Bit#(64), Bit#(MemTagSize))) bFifo <- mkSizedFIFOF(writeLatency_I);

   rule increment_cycle;
      cycles <= cycles+1;
   endrule

   let read_jitter = True; //cycles[4:0] == 0;
   let write_jitter = True; //cycles[4:0] == 5;

   Reg#(Bit#(8))  burstReg <- mkReg(0);
   FIFO#(Bit#(8)) reqs <- mkSizedFIFO(32);

   let beat_shift = fromInteger(valueOf(beatShift));

   interface PhysMemReadServer read_server;
      interface Put readReq;
	 method Action put(PhysMemRequest#(serverAddrWidth) req);
            if (verbose) $display("%d axiSlave.read.readAddr %h bc %d", cycles, req.addr, req.burstLen);
	    //readAddrGenerator.request.put(req);
	    readDelayFifo.enq(tuple2(cycles,req));
	 endmethod
      endinterface
      interface Get readData;
	 method ActionValue#(MemData#(serverBusWidth)) get() if (((readLen > 0) || (readLen == 0 && (cycles-tpl_1(readDelayFifo.first)) > readLatency)) && read_jitter);
	 Bit#(BurstLenSize) read_len = ?;
	 Bit#(serverAddrWidth) read_addr = ?;
	 Bit#(MemTagSize) read_id = ?;
	 Bit#(8) handle = ?;
	 if (readLen == 0 && (cycles-tpl_1(readDelayFifo.first)) > readLatency) begin
	    req_ar_b_ts <= cycles;
	    let req = tpl_2(readDelayFifo.first);
	    readDelayFifo.deq;
	    read_len = req.burstLen>>beat_shift;
	    read_addr = req.addr;
	    read_id = req.tag;
	    handle = req.addr[39:32];
	    //if(id==1) $display("mkBsimHost::resp_read_a: %d %d %d", req.tag,  cycles-last_read_eob, (req.burstLen>>beat_shift)-1);
	    //last_read_eob <= cycles;
	 end
	 else begin
	    //$display("mkBsimHost::resp_read_b: %d %d", readId,  cycles-last_read_eob);
	    //last_read_eob <= cycles;
	    handle = readAddrr[39:32];
	    read_addr = readAddrr;
	    read_id = readId;
	    read_len = readLen;
	 end
	 Bit#(serverBusWidth) v <- rw.read_pareff(extend(handle), read_addr[31:0]);
	 readLen <= read_len - 1;
	 readId <= read_id;
	 readAddrr <= read_addr + fromInteger(valueOf(serverBusWidth)/8);
	 //$display("mkBsimHost::resp_read id=%d %d", read_id, read_len);
	 //return Axi3ReadResponse { data: v, resp: 0, last: pack(readLen == 1), id: read_id};
            //if (verbose) $display("%d read_server.readData (b) %h", cycles, data);
            return MemData { data: v, tag: read_id, last: readLen == 1};
	 endmethod
      endinterface
   endinterface
   interface PhysMemWriteServer write_server;
      interface Put writeReq;
	 method Action put(PhysMemRequest#(serverAddrWidth) req);
	 //$display("mkBsimHost::req_aw id=%d", req.tag);
	 writeDelayFifo.enq(tuple2(cycles,req));
	 endmethod
      endinterface
      interface Put writeData;
	 method Action put(MemData#(serverBusWidth) resp) if (((writeLen > 0) || (writeLen == 0 && (cycles-tpl_1(writeDelayFifo.first)) > writeLatency)) && write_jitter);
	    //let addrBeat <- writeAddrGenerator.addrBeat.get();
	    //let addr = addrBeat.addr;
	    //Bit#(bramAddrWidth) regFileAddr = truncate(addr/fromInteger(valueOf(TDiv#(serverBusWidth,8))));
            //br.request.put(BRAMRequest{write:True, responseOnWrite:False, address:regFileAddr, datain:resp.data});
	 Bit#(BurstLenSize) write_len = ?;
	 Bit#(serverAddrWidth) write_addr = ?;
	 Bit#(MemTagSize) write_id = ?;
	 Bit#(8) handle = ?;
	 if (writeLen == 0 && (cycles-tpl_1(writeDelayFifo.first)) > writeLatency) begin
	    req_aw_b_ts <= cycles;
	    let req = tpl_2(writeDelayFifo.first);
	    writeDelayFifo.deq;
	    write_addr = req.addr;
	    write_len = req.burstLen>>beat_shift;
	    write_id = req.tag;
	    handle = req.addr[39:32];
	    //$display("mkBsimHost::resp_write_a: %d %d", req.tag,  cycles-last_write_eob);
	    //last_write_eob <= cycles;
	 end
	 else begin
	    //$display("mkBsimHost::resp_write_b: %d %d", writeId,  cycles-last_write_eob);
	    //last_write_eob <= cycles;
	    handle = writeAddrr[39:32];
	    write_len = writeLen;
	    write_addr = writeAddrr;
	    write_id = writeId;
	 end
	 rw.write_pareff(extend(handle), write_addr[31:0], resp.data);
	 //$display("write_resp(%d): handle=%d addr=%h v=%h", cycles, handle, write_addr, resp.data);
	 writeId <= write_id;
	 writeLen <= write_len - 1;
	 writeAddrr <= write_addr + fromInteger(valueOf(serverBusWidth)/8);
	 if (write_len == 1) begin
	    bFifo.enq(tuple2(cycles,write_id));
	 end
            //if (verbose) $display("%d write_server.writeAddr %h bc %d", cycles, req.addr, req.burstLen);
            //if (verbose) $display("%d write_server.writeData %h %h %d", cycles, addr, resp.data, addrBeat.bc);
            //if (addrBeat.last)
               //writeTagFifo.enq(addrBeat.tag);
	 endmethod
      endinterface
      interface Get writeDone;
	 method ActionValue#(Bit#(MemTagSize)) get() if ((cycles-tpl_1(bFifo.first)) > writeLatency);
	 bFifo.deq();
	 return tpl_2(bFifo.first());
	 endmethod
      endinterface
   endinterface
endmodule

module  mkBsimHost#(Clock derived_clock, Reset derived_reset)(BsimHost#(clientAddrWidth, clientBusWidth, clientIdWidth,
			      serverAddrWidth, serverBusWidth, serverIdWidth,
			      nSlaves))
   provisos (SelectBsimRdmaReadWrite#(serverBusWidth),
             Add#(a__, 32, clientAddrWidth), Add#(b__, 32, clientBusWidth),
             Mul#(TDiv#(serverBusWidth, 8), 8, serverBusWidth));

   Vector#(nSlaves,PhysMemSlave#(serverAddrWidth,  serverBusWidth)) servers <- replicateM(mkBsimDmaMaster);
   PhysMemMaster#(clientAddrWidth, clientBusWidth) crw <- mkBsimCtrlReadWrite();

   interface mem_servers = servers;
   interface PhysMemMaster mem_client = crw;
   interface derivedClock = derived_clock;
   interface derivedReset = derived_reset;
endmodule

module  mkBsimTop(Empty)
   provisos (SelectBsimRdmaReadWrite#(DataBusWidth));
   let divider <- mkClockDivider(2);
   Clock derivedClock = divider.fastClock;
   Clock singleClock = divider.slowClock;
   Reset derivedReset <- exposeCurrentReset;
   let single_reset <- mkReset(2, True, singleClock);
   Reset singleReset = single_reset.new_rst;
   BsimHost#(32,32,12,PhysAddrWidth,DataBusWidth,6,NumberOfMasters) host <- mkBsimHost(clocked_by singleClock, reset_by singleReset, derivedClock, derivedReset);
   ConnectalTop#(PhysAddrWidth,DataBusWidth,PinType,NumberOfMasters) top <- mkConnectalTop(
`ifdef IMPORT_HOSTIF
       host,
`endif
       clocked_by singleClock, reset_by singleReset);
   mapM(uncurry(mkConnection),zip(top.masters, host.mem_servers), clocked_by singleClock, reset_by singleReset);
`ifndef BSIM_EXERCISE_MEM_MASTER_SLAVE
   mkConnection(host.mem_client, top.slave, clocked_by singleClock, reset_by singleReset);
`else
   PciId masterPciId = unpack(22);
   PciId slavePciId = unpack(23);
   MemMasterEngine masterEngine <- mkMemMasterEngine(masterPciId, clocked_by singleClock, reset_by singleReset);
   MemSlaveEngine#(32) slaveEngine <- mkMemSlaveEngine(slavePciId, clocked_by singleClock, reset_by singleReset);
   mkConnection(host.mem_client, slaveEngine.slave, clocked_by singleClock, reset_by singleReset);
   mkConnection(slaveEngine.tlp.request, masterEngine.tlp.response, clocked_by singleClock, reset_by singleReset);
   mkConnection(slaveEngine.tlp.response, masterEngine.tlp.request, clocked_by singleClock, reset_by singleReset);
   mkConnection(masterEngine.master, top.slave, clocked_by singleClock, reset_by singleReset);
`endif

   let intr_mux <- mkInterruptMux(top.interrupt);
   rule int_rule;
      interruptLevel(truncate(pack(intr_mux)));
   endrule

`ifdef BSIMRESPONDER
   `BSIMRESPONDER (top.pins);
`endif
endmodule