pass correct tile numbers to zipWIth3M renameReads and renameWrites

// Copyright (c) 2015 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
import ConnectalConfig::*;
import Vector::*;
import BuildVector::*;
import Portal::*;
import HostInterface::*;
import MMU::*;
import MemServer::*;
import MemTypes::*;
import CtrlMux::*;
import FIFO::*;
import GetPut::*;
import SpecialFIFOs::*;
import Pipe::*;
import ConnectalMemory::*;
import MMURequest::*;
import MMUIndication::*;
import MemServerIndication::*;
import MemServerRequest::*;
import IfcNames::*;
`include "ConnectalProjectConfig.bsv"
import `PinTypeInclude::*;

interface Platform;
   interface PhysMemSlave#(32,32) slave;
   interface Vector#(NumberOfMasters,PhysMemMaster#(PhysAddrWidth, DataBusWidth)) masters;
   interface Vector#(MaxNumberOfPortals,ReadOnly#(Bool)) interrupt;
   interface `PinType pins;
endinterface

typedef TMax#(TLog#(TSub#(NumberOfTiles,1)),1) TileTagBits;
function Bit#(TSub#(MemTagSize,TileTagBits)) tagLsb(Bit#(MemTagSize) tag); return truncate(tag); endfunction
function Bit#(TileTagBits) tagMsb(Bit#(MemTagSize) tag); return truncate(tag >> valueOf(TSub#(MemTagSize,TileTagBits))); endfunction

module renameReads#(Integer tile, MemReadClient#(DataBusWidth) reader, MemServerIndication err)(MemReadClient#(DataBusWidth));
   interface Get readReq;
      method ActionValue#(MemRequest) get;
	 let req <- reader.readReq.get;
	 Bit#(TSub#(MemTagSize,TileTagBits)) lsb = tagLsb(req.tag);
	 Bit#(TileTagBits) msb = tagMsb(req.tag);
	 if(req.tag != extend(lsb) && valueOf(NumberOfTiles) > 2) begin // one mgmt tile and one user tile
	    $display("renameReads tile tag out of range: 'h%h", req.tag);
	    err.error(extend(pack(DmaErrorTileTagOutOfRange)), req.sglId, extend(req.tag), fromInteger(tile));
	 end
	 req.tag = {fromInteger(tile),lsb};
	 return req;
      endmethod
   endinterface
   interface Put readData;
      method Action put(MemData#(DataBusWidth) v);
	 reader.readData.put(MemData{data:v.data, tag:{0,tagLsb(v.tag)}, last:v.last});
      endmethod
   endinterface
endmodule

module renameWrites#(Integer tile, MemWriteClient#(DataBusWidth) writer, MemServerIndication err)(MemWriteClient#(DataBusWidth));
   interface Get writeReq;
      method ActionValue#(MemRequest) get;
	 let req <- writer.writeReq.get;
	 Bit#(TSub#(MemTagSize,TileTagBits)) lsb = tagLsb(req.tag);
	 Bit#(TileTagBits) msb = tagMsb(req.tag);
	 if(req.tag != extend(lsb) && valueOf(NumberOfTiles) > 2) begin // one mgmt tile and one user tile
	    $display("renameWrites tile tag out of range: 'h%h", req.tag);
	    err.error(extend(pack(DmaErrorTileTagOutOfRange)), req.sglId, extend(req.tag), fromInteger(tile));
	 end
	 req.tag = {fromInteger(tile),lsb};
	 return req;
      endmethod
   endinterface
   interface Get writeData;
      method ActionValue#(MemData#(DataBusWidth)) get;
	 let rv <- writer.writeData.get;
   	 return MemData{data:rv.data, tag:{0,tagLsb(rv.tag)}, last:rv.last};
      endmethod
   endinterface
   interface Put writeDone;
      method Action put(Bit#(MemTagSize) v);
	 writer.writeDone.put({0,tagLsb(v)});
      endmethod
   endinterface
endmodule

module mkPlatform#(Vector#(NumberOfUserTiles, ConnectalTop) tiles)(Platform);
   /////////////////////////////////////////////////////////////
   // connecting up the tiles

   Vector#(NumberOfUserTiles, PhysMemSlave#(18,32)) tile_slaves;
   Vector#(NumberOfUserTiles, ReadOnly#(Bool)) tile_interrupts;
   Vector#(NumberOfUserTiles, Vector#(NumReadClients, MemReadClient#(DataBusWidth))) tile_read_clients;
   Vector#(NumberOfUserTiles, Vector#(NumWriteClients, MemWriteClient#(DataBusWidth))) tile_write_clients;
   Vector#(NumberOfUserTiles, Vector#(NumReadClients, Integer)) read_client_tile_numbers;
   Vector#(NumberOfUserTiles, Vector#(NumWriteClients, Integer)) write_client_tile_numbers;
   for(Integer i = 0; i < valueOf(NumberOfUserTiles); i=i+1) begin
      tile_slaves[i] = tiles[i].slave;
      let imux <- mkInterruptMux(tiles[i].interrupt);
      //ReadOnly#(Bool) imux = tiles[i].interrupt;
      tile_interrupts[i] = imux;
      tile_read_clients[i] = tiles[i].readers;
      tile_write_clients[i] = tiles[i].writers;
      read_client_tile_numbers[i] = replicate(i);
      write_client_tile_numbers[i] = replicate(i);
   end

   /////////////////////////////////////////////////////////////
   // framework internal portals

   MMUIndicationProxy lMMUIndicationProxy <- mkMMUIndicationProxy(PlatformIfcNames_MMUIndicationH2S);
   MemServerIndicationProxy lMemServerIndicationProxy <- mkMemServerIndicationProxy(PlatformIfcNames_MemServerIndicationH2S);

   MMU#(PhysAddrWidth) lMMU <- mkMMU(0,True, lMMUIndicationProxy.ifc);
   Vector#(TMul#(NumberOfUserTiles,NumReadClients), MemReadClient#(DataBusWidth)) tile_read_clients_renamed <- zipWith3M(renameReads, concat(read_client_tile_numbers), concat(tile_read_clients), replicate(lMemServerIndicationProxy.ifc));
   Vector#(TMul#(NumberOfUserTiles,NumWriteClients), MemWriteClient#(DataBusWidth)) tile_write_clients_renamed <- zipWith3M(renameWrites, concat(write_client_tile_numbers), concat(tile_write_clients), replicate(lMemServerIndicationProxy.ifc));
   MemServer#(PhysAddrWidth,DataBusWidth,NumberOfMasters) lMemServer <- mkMemServer(tile_read_clients_renamed, tile_write_clients_renamed, vec(lMMU), lMemServerIndicationProxy.ifc);

   MMURequestWrapper lMMURequestWrapper <- mkMMURequestWrapper(PlatformIfcNames_MMURequestS2H, lMMU.request);
   MemServerRequestWrapper lMemServerRequestWrapper <- mkMemServerRequestWrapper(PlatformIfcNames_MemServerRequestS2H, lMemServer.request);

   Vector#(4,StdPortal) framework_portals;
   framework_portals[0] = lMMUIndicationProxy.portalIfc;
   framework_portals[1] = lMemServerIndicationProxy.portalIfc;
   framework_portals[2] = lMMURequestWrapper.portalIfc;
   framework_portals[3] = lMemServerRequestWrapper.portalIfc;
   PhysMemSlave#(18,32) framework_ctrl_mux <- mkSlaveMux(framework_portals);
   let framework_intr <- mkInterruptMux(getInterruptVector(framework_portals));

   /////////////////////////////////////////////////////////////
   // expose interface to top

   PhysMemSlave#(32,32) ctrl_mux <- mkPhysMemSlaveMux(cons(framework_ctrl_mux,tile_slaves));
   Vector#(MaxNumberOfPortals, ReadOnly#(Bool)) interrupts = replicate(interface ReadOnly; method Bool _read(); return False; endmethod endinterface);
   interrupts[0] = framework_intr;
   for (Integer i = 1; i < valueOf(NumberOfTiles); i = i + 1)
      interrupts[i] = tile_interrupts[i-1];
   interface interrupt = interrupts;
   interface slave = ctrl_mux;
   interface masters = lMemServer.masters;
   interface pins = tiles[0].pins;
endmodule

// Copyright (c) 2015 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
import ConnectalConfig::*;
import Vector::*;
import BuildVector::*;
import Portal::*;
import HostInterface::*;
import MMU::*;
import MemServer::*;
import MemTypes::*;
import CtrlMux::*;
import FIFO::*;
import GetPut::*;
import SpecialFIFOs::*;
import Pipe::*;
import ConnectalMemory::*;
import MMURequest::*;
import MMUIndication::*;
import MemServerIndication::*;
import MemServerRequest::*;
import IfcNames::*;
`include "ConnectalProjectConfig.bsv"
import `PinTypeInclude::*;

interface Platform;
   interface PhysMemSlave#(32,32) slave;
   interface Vector#(NumberOfMasters,PhysMemMaster#(PhysAddrWidth, DataBusWidth)) masters;
   interface Vector#(MaxNumberOfPortals,ReadOnly#(Bool)) interrupt;
   interface `PinType pins;
endinterface

typedef TMax#(TLog#(TSub#(NumberOfTiles,1)),1) TileTagBits;
function Bit#(TSub#(MemTagSize,TileTagBits)) tagLsb(Bit#(MemTagSize) tag); return truncate(tag); endfunction
function Bit#(TileTagBits) tagMsb(Bit#(MemTagSize) tag); return truncate(tag >> valueOf(TSub#(MemTagSize,TileTagBits))); endfunction

module renameReads#(Integer tile, MemReadClient#(DataBusWidth) reader, MemServerIndication err)(MemReadClient#(DataBusWidth));
   interface Get readReq;
      method ActionValue#(MemRequest) get;
	 let req <- reader.readReq.get;
	 Bit#(TSub#(MemTagSize,TileTagBits)) lsb = tagLsb(req.tag);
	 Bit#(TileTagBits) msb = tagMsb(req.tag);
	 if(req.tag != extend(lsb) && valueOf(NumberOfTiles) > 2) begin // one mgmt tile and one user tile
	    $display("renameReads tile tag out of range: 'h%h", req.tag);
	    err.error(extend(pack(DmaErrorTileTagOutOfRange)), req.sglId, extend(req.tag), fromInteger(tile));
	 end
	 req.tag = {fromInteger(tile),lsb};
	 return req;
      endmethod
   endinterface
   interface Put readData;
      method Action put(MemData#(DataBusWidth) v);
	 reader.readData.put(MemData{data:v.data, tag:{0,tagLsb(v.tag)}, last:v.last});
      endmethod
   endinterface
endmodule

module renameWrites#(Integer tile, MemWriteClient#(DataBusWidth) writer, MemServerIndication err)(MemWriteClient#(DataBusWidth));
   interface Get writeReq;
      method ActionValue#(MemRequest) get;
	 let req <- writer.writeReq.get;
	 Bit#(TSub#(MemTagSize,TileTagBits)) lsb = tagLsb(req.tag);
	 Bit#(TileTagBits) msb = tagMsb(req.tag);
	 if(req.tag != extend(lsb) && valueOf(NumberOfTiles) > 2) begin // one mgmt tile and one user tile
	    $display("renameWrites tile tag out of range: 'h%h", req.tag);
	    err.error(extend(pack(DmaErrorTileTagOutOfRange)), req.sglId, extend(req.tag), fromInteger(tile));
	 end
	 req.tag = {fromInteger(tile),lsb};
	 return req;
      endmethod
   endinterface
   interface Get writeData;
      method ActionValue#(MemData#(DataBusWidth)) get;
	 let rv <- writer.writeData.get;
   	 return MemData{data:rv.data, tag:{0,tagLsb(rv.tag)}, last:rv.last};
      endmethod
   endinterface
   interface Put writeDone;
      method Action put(Bit#(MemTagSize) v);
	 writer.writeDone.put({0,tagLsb(v)});
      endmethod
   endinterface
endmodule

module mkPlatform#(Vector#(NumberOfUserTiles, ConnectalTop) tiles)(Platform);
   /////////////////////////////////////////////////////////////
   // connecting up the tiles

   Vector#(NumberOfUserTiles, PhysMemSlave#(18,32)) tile_slaves;
   Vector#(NumberOfUserTiles, ReadOnly#(Bool)) tile_interrupts;
   Vector#(NumberOfUserTiles, Vector#(NumReadClients, MemReadClient#(DataBusWidth))) tile_read_clients;
   Vector#(NumberOfUserTiles, Vector#(NumWriteClients, MemWriteClient#(DataBusWidth))) tile_write_clients;
   for(Integer i = 0; i < valueOf(NumberOfUserTiles); i=i+1) begin
      tile_slaves[i] = tiles[i].slave;
      let imux <- mkInterruptMux(tiles[i].interrupt);
      //ReadOnly#(Bool) imux = tiles[i].interrupt;
      tile_interrupts[i] = imux;
      tile_read_clients[i] = tiles[i].readers;
      tile_write_clients[i] = tiles[i].writers;
   end

   /////////////////////////////////////////////////////////////
   // framework internal portals

   MMUIndicationProxy lMMUIndicationProxy <- mkMMUIndicationProxy(PlatformIfcNames_MMUIndicationH2S);
   MemServerIndicationProxy lMemServerIndicationProxy <- mkMemServerIndicationProxy(PlatformIfcNames_MemServerIndicationH2S);

   MMU#(PhysAddrWidth) lMMU <- mkMMU(0,True, lMMUIndicationProxy.ifc);
   Vector#(TMul#(NumberOfUserTiles,NumReadClients), MemReadClient#(DataBusWidth)) tile_read_clients_renamed <- zipWith3M(renameReads, genVector, concat(tile_read_clients), replicate(lMemServerIndicationProxy.ifc));
   Vector#(TMul#(NumberOfUserTiles,NumWriteClients), MemWriteClient#(DataBusWidth)) tile_write_clients_renamed <- zipWith3M(renameWrites, genVector, concat(tile_write_clients), replicate(lMemServerIndicationProxy.ifc));
   MemServer#(PhysAddrWidth,DataBusWidth,NumberOfMasters) lMemServer <- mkMemServer(tile_read_clients_renamed, tile_write_clients_renamed, vec(lMMU), lMemServerIndicationProxy.ifc);

   MMURequestWrapper lMMURequestWrapper <- mkMMURequestWrapper(PlatformIfcNames_MMURequestS2H, lMMU.request);
   MemServerRequestWrapper lMemServerRequestWrapper <- mkMemServerRequestWrapper(PlatformIfcNames_MemServerRequestS2H, lMemServer.request);

   Vector#(4,StdPortal) framework_portals;
   framework_portals[0] = lMMUIndicationProxy.portalIfc;
   framework_portals[1] = lMemServerIndicationProxy.portalIfc;
   framework_portals[2] = lMMURequestWrapper.portalIfc;
   framework_portals[3] = lMemServerRequestWrapper.portalIfc;
   PhysMemSlave#(18,32) framework_ctrl_mux <- mkSlaveMux(framework_portals);
   let framework_intr <- mkInterruptMux(getInterruptVector(framework_portals));

   /////////////////////////////////////////////////////////////
   // expose interface to top

   PhysMemSlave#(32,32) ctrl_mux <- mkPhysMemSlaveMux(cons(framework_ctrl_mux,tile_slaves));
   Vector#(MaxNumberOfPortals, ReadOnly#(Bool)) interrupts = replicate(interface ReadOnly; method Bool _read(); return False; endmethod endinterface);
   interrupts[0] = framework_intr;
   for (Integer i = 1; i < valueOf(NumberOfTiles); i = i + 1)
      interrupts[i] = tile_interrupts[i-1];
   interface interrupt = interrupts;
   interface slave = ctrl_mux;
   interface masters = lMemServer.masters;
   interface pins = tiles[0].pins;
endmodule


adjusted fifo types and added debug probes

// Copyright (c) 2016 Connectal Project

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import BRAM::*;
import BuildVector::*;
import Clocks::*;
import Connectable::*;
import FIFOF::*;
import GetPut::*;
import Probe::*;
import Vector::*;

import AddressGenerator::*;
import AxiBits::*;
import ConnectalClocks::*;
import ConnectalConfig::*;
import DefaultValue::*;
import HostInterface::*;
import MemTypes::*;
import PhysMemToBram::*;
import Pipe::*;
import TraceMemClient::*;
import XilinxCells::*;

import AxiPcieRootPort::*;
import NvmeIfc::*;
import NvmePins::*;

`ifndef TOP_SOURCES_PORTAL_CLOCK
import ConnectalBramFifo::*;
`else
import BRAMFIFO::*;
module mkDualClockBramFIFOF#(Clock clock1, Reset reset1, Clock clock2, Reset reset2)(FIFOF#(a))
   provisos (Bits#(a, asz), Add#(1, a__, asz));
   FIFOF#(a) fifo <- mkSizedBRAMFIFOF(512, clocked_by clock1, reset_by reset1);
   return fifo;
endmodule
`endif

interface Nvme;
   interface NvmeRequest request;
   interface NvmeTrace trace;
   interface NvmePins pins;
   interface Vector#(1, MemReadClient#(DataBusWidth)) dmaReadClient;
   interface Vector#(1, MemWriteClient#(DataBusWidth)) dmaWriteClient;
`ifdef TOP_SOURCES_PORTAL_CLOCK
   interface Clock portalClockSource;
`endif
endinterface

module mkNvme#(NvmeIndication ind, NvmeTrace trace)(Nvme);
   let clock <- exposeCurrentClock;
   let reset <- exposeCurrentReset;
   let refclk_p <- mkB2C1();
   let refclk_n <- mkB2C1();
   let pcie_clk_100mhz_buf <- mkClockIBUFDS_GTE2(
`ifdef ClockDefaultParam
       defaultValue,
`endif
      True, refclk_p.c, refclk_n.c);
`ifndef TOP_SOURCES_PORTAL_CLOCK
   let axiClockB2C    <- mkB2C1();
   let axiCtlClockB2C <- mkB2C1();
   let axiClock = axiClockB2C.c;
   let axiCtlClock = axiCtlClockB2C.c;
   let axiReset <- mkSyncReset(10, reset, axiClock);
   let axiCtlReset <- mkSyncReset(10, reset, axiCtlClock);
`else
   let axiClock = clock;
   let axiCtlClock = clock;
   let axiReset = reset;
   let axiCtlReset = reset;
`endif

   let axiRootPort <- mkAPRP(pcie_clk_100mhz_buf, reset, axiClock, axiReset, axiCtlClock, axiCtlReset);
`ifndef TOP_SOURCES_PORTAL_CLOCK
   let axiClockC2B <- mkC2B(axiRootPort.axi.aclk_out);
   let axiCtlClockC2B <- mkC2B(axiRootPort.axi.ctl_aclk_out);
   rule rl_connect_clocks;
      axiClockB2C.inputclock(axiClockC2B.o);
      axiCtlClockB2C.inputclock(axiClockC2B.o);
   endrule
`endif

   FIFOF#(Bit#(32)) dfifoCtl <- mkFIFOF();
   Axi4SlaveBits#(32,DataBusWidth,4,Empty) axiRootPortSlave    = toAxi4SlaveBits(axiRootPort.s_axi);
   Axi4SlaveLiteBits#(32,32)     axiRootPortSlaveCtl = toAxi4SlaveBits(axiRootPort.s_axi_ctl);
   PhysMemSlave#(32,DataBusWidth)          axiRootPortMemSlave    <- mkPhysMemSlave(axiRootPortSlave, clocked_by axiClock, reset_by axiReset);
   PhysMemSlave#(32,32)          axiRootPortMemSlaveCtl <- mkPhysMemSlave(axiRootPortSlaveCtl, clocked_by axiCtlClock, reset_by axiCtlReset);

   FIFOF#(PhysMemRequest#(32,DataBusWidth)) araddrFifo <- mkFIFOF();
   FIFOF#(PhysMemRequest#(32,DataBusWidth)) awaddrFifo <- mkFIFOF();
   FIFOF#(MemData#(DataBusWidth))           rdataFifo <- mkFIFOF();
   FIFOF#(MemData#(DataBusWidth))           wdataFifo <- mkFIFOF();
   FIFOF#(Bit#(6))                doneFifo <- mkFIFOF();

   let araddrCnx <- mkConnection(toGet(araddrFifo), axiRootPortMemSlave.read_server.readReq);
   let awaddrCnx <- mkConnection(toGet(awaddrFifo), axiRootPortMemSlave.write_server.writeReq);
   let rdataCnx  <- mkConnection(axiRootPortMemSlave.read_server.readData, toPut(rdataFifo));
   let wdataCnx  <- mkConnection(toGet(wdataFifo), axiRootPortMemSlave.write_server.writeData);
   let doneCnx   <- mkConnection(axiRootPortMemSlave.write_server.writeDone, toPut(doneFifo));

   rule rl_rdata;
      let rdata <- toGet(rdataFifo).get();
      ind.readDone(rdata.data);
   endrule

   rule rl_writeDone;
      let tag <- toGet(doneFifo).get();
      ind.writeDone();
   endrule

   FIFOF#(PhysMemRequest#(32,32)) araddrFifoCtl <- mkFIFOF();
   FIFOF#(PhysMemRequest#(32,32)) awaddrFifoCtl <- mkFIFOF();
   FIFOF#(MemData#(32))           rdataFifoCtl <- mkFIFOF();
   FIFOF#(MemData#(32))           wdataFifoCtl <- mkFIFOF();
   FIFOF#(Bit#(6))                doneFifoCtl <- mkFIFOF();

   let araddrCtlCnx <- mkConnection(toGet(araddrFifoCtl), axiRootPortMemSlaveCtl.read_server.readReq);
   let awaddrCtlCnx <- mkConnection(toGet(awaddrFifoCtl), axiRootPortMemSlaveCtl.write_server.writeReq);
   let rdataCtlCnx  <- mkConnection(axiRootPortMemSlaveCtl.read_server.readData, toPut(rdataFifoCtl));
   let wdataCtlCnx  <- mkConnection(toGet(wdataFifoCtl), axiRootPortMemSlaveCtl.write_server.writeData);
   let doneCtlCnx   <- mkConnection(axiRootPortMemSlaveCtl.write_server.writeDone, toPut(doneFifoCtl));

   rule rl_rdata_ctl;
      let rdata <- toGet(rdataFifoCtl).get();
      ind.readDone(extend(rdata.data));
   endrule

   rule rl_writeDone_ctl;
      let tag <- toGet(doneFifoCtl).get();
      ind.writeDone();
   endrule

   Axi4MasterBits#(32,DataBusWidth,MemTagSize,Empty) m_axi_mm = toAxi4MasterBits(axiRootPort.m_axi);
   let getObjId = (interface GetObjId;
		   method SGLId objId(Bit#(32) addr); return extend(addr[31:24]); endmethod
		   method Bit#(MemOffsetSize) addr(Bit#(32) axiAddr); return extend(axiAddr[23:0]); endmethod
		   endinterface);
   let memReadClient  <- mkMemReadClient(getObjId, m_axi_mm);
   let memWriteClient <- mkMemWriteClient(getObjId, m_axi_mm);

   FIFOF#(Tuple4#(DmaChannel,Bool,MemRequest,Bit#(32))) traceFifo <- mkSizedBRAMFIFOF(128);
   PipeIn#(Tuple4#(DmaChannel,Bool,MemRequest,Bit#(32))) tracePipe = toPipeIn(traceFifo);
   FIFOF#(Tuple4#(DmaChannel,Bool,MemData#(DataBusWidth),Bit#(32))) traceDataFifo <- mkSizedBRAMFIFOF(1024);
   PipeIn#(Tuple4#(DmaChannel,Bool,MemData#(DataBusWidth),Bit#(32))) traceDataPipe = toPipeIn(traceDataFifo);
   rule rl_trace1;
      match { .chan, .write, .req, .timestamp } <- toGet(traceFifo).get();
      trace.traceDmaRequest(chan, write, truncate(req.sglId), extend(req.offset), extend(req.burstLen), extend(req.tag), timestamp);
   endrule
   rule rl_trace_data;
      match { .chan, .write, .md, .timestamp } <- toGet(traceDataFifo).get();
      trace.traceDmaData(chan, write, md.data, md.last, extend(md.tag), timestamp);
   endrule

   let traceReadClient <- mkTraceReadClient(tracePipe,traceDataPipe,DMA_TX,memReadClient);
   let traceWriteClient <- mkTraceWriteClient(tracePipe,traceDataPipe,DMA_RX,memWriteClient);
   let traceClient = (interface MemClient#(DataBusWidth);
			 interface readClient = traceReadClient;
			 interface writeClient = traceWriteClient;
		      endinterface);

   let splitter <- mkSplitMemServer();
   BRAM_Configure bramConfig = defaultValue;
   bramConfig.latency = 2;
   bramConfig.memorySize = 1024;
   BRAM1Port#(Bit#(32),Bit#(DataBusWidth)) bram        <- mkBRAM1Server(bramConfig);
   MemServer#(DataBusWidth) bramMem     <- mkMemToBram(bram.portA);
   let traceMemCnx <- mkConnection(traceClient, splitter.server);
   let bramMemCnx  <- mkConnection(splitter.bramClient, bramMem);
   rule rl_data;
      let md <- toGet(splitter.data).get();
      trace.traceData(md.data, md.last, extend(md.tag));
   endrule

   interface NvmeRequest request;

      method Action status();
        ind.status(axiRootPort.mmcm.lock());
      endmethod

      method Action read32(Bit#(32) addr);
	 araddrFifo.enq(PhysMemRequest { addr: addr, burstLen: 4, tag: 0 });
      endmethod
      method Action write32(Bit#(32) addr, Bit#(32) value);
	 awaddrFifo.enq(PhysMemRequest { addr: addr, burstLen: 4, tag: 0 });
	 wdataFifo.enq(MemData {data: extend(value), tag: 0, last: True});
      endmethod
      method Action read(Bit#(32) addr);
	 araddrFifo.enq(PhysMemRequest { addr: addr, burstLen: fromInteger(valueOf(TDiv#(DataBusWidth,8))), tag: 0 });
      endmethod
      method Action write(Bit#(32) addr, Bit#(DataBusWidth) value);
	 awaddrFifo.enq(PhysMemRequest { addr: addr, burstLen: fromInteger(valueOf(TDiv#(DataBusWidth,8))), tag: 0 });
	 wdataFifo.enq(MemData {data: value, tag: 0, last: True});
      endmethod

      method Action readCtl(Bit#(32) addr);
	 araddrFifoCtl.enq(PhysMemRequest { addr: addr, burstLen: 4, tag: 0 });
      endmethod
      method Action writeCtl(Bit#(32) addr, Bit#(DataBusWidth) value);
	 awaddrFifoCtl.enq(PhysMemRequest { addr: addr, burstLen: 4, tag: 0 });
	 wdataFifoCtl.enq(MemData {data: truncate(value), tag: 0, last: True});
      endmethod
   endinterface
   interface Clock portalClockSource = axiRootPort.axi.aclk_out;
   interface NvmePins pins;
      interface deleteme_unused_clock = clock;
      interface pcie_sys_reset_n = reset;
      interface pcie = axiRootPort.pci;
      method Action pcie_refclk(Bit#(1) p, Bit#(1) n);
         refclk_p.inputclock(p);
         refclk_n.inputclock(n);
      endmethod
   endinterface
   interface Vector dmaReadClient = vec(splitter.busClient.readClient);
   interface Vector dmaWriteClient = vec(splitter.busClient.writeClient);
endmodule

instance ToAxi4SlaveBits#(Axi4SlaveBits#(32,DataBusWidth,4,Empty), AprpS_axi);
   function Axi4SlaveBits#(32,DataBusWidth,4,Empty) toAxi4SlaveBits(AprpS_axi s);
      return (interface Axi4SlaveBits#(32,DataBusWidth,4,Empty);
	 method araddr = compose(s.araddr, extend);
	 method arburst = s.arburst;
	 //method arcache = s.arcache;
	 method arid = s.arid;
	 method arlen = s.arlen;
	 //method arlock = s.arlock;
	 //method arprot = s.arprot;
	 //method arqos = s.arqos;
	 method arready = s.arready;
	 method arsize = s.arsize;
	 method arvalid = s.arvalid;

	 method awaddr = compose(s.awaddr, extend);
	 method awburst = s.awburst;
	 //method awcache = s.awcache;
	 method awid = s.awid;
	 method awlen = s.awlen;
	 //method awlock = s.awlock;
	 //method awprot = s.awprot;
	 //method awqos = s.awqos;
	 method awready = s.awready;
	 method awsize = s.awsize;
	 method awvalid = s.awvalid;

	 method bid = s.bid;
	 method bready = s.bready;
	 method bresp = s.bresp;
	 method bvalid = s.bvalid;
	 method rdata = s.rdata;
	 method rid = s.rid;
	 method rlast = s.rlast;
	 method rready = s.rready;
	 method rresp = s.rresp;
	 method rvalid = s.rvalid;
	 method wdata = s.wdata;
	 method wlast = s.wlast;
	 method wready = s.wready;
	 method wvalid = s.wvalid;
	 method wstrb = s.wstrb;
	 endinterface);
   endfunction
endinstance

instance ToAxi4SlaveBits#(Axi4SlaveLiteBits#(32,32), AprpS_axi_ctl);
   function Axi4SlaveLiteBits#(32,32) toAxi4SlaveBits(AprpS_axi_ctl s);
      return (interface Axi4SlaveLiteBits#(32,32);
	 method araddr = compose(s.araddr, extend);
	 method arready = s.arready;
	 method arvalid = s.arvalid;

	 method awaddr = compose(s.awaddr, extend);
	 method awready = s.awready;
	 method awvalid = s.awvalid;

	 method bready = s.bready;
	 method bresp = s.bresp;
	 method bvalid = s.bvalid;
	 method rdata = s.rdata;
	 method rready = s.rready;
	 method rresp = s.rresp;
	 method rvalid = s.rvalid;
	 method wdata = s.wdata;
	 method wready = s.wready;
	 method Action      wvalid(Bit#(1) v);
	    s.wvalid(v);
	    s.wstrb(pack(replicate(v)));
	 endmethod
	 endinterface);
   endfunction
endinstance

instance ToAxi4MasterBits#(Axi4MasterBits#(32,DataBusWidth,tagWidth,Empty), AprpM_axi);
function Axi4MasterBits#(32,DataBusWidth,tagWidth,Empty) toAxi4MasterBits(AprpM_axi m);
   return (interface Axi4MasterBits#(32,DataBusWidth,tagWidth,Empty);
	   method araddr = m.araddr;
	   method arburst = m.arburst;
	   method arcache = m.arcache;
	   method arlen = m.arlen;
	   method arlock = extend(m.arlock);
	   method arready = m.arready;
	   method arsize = m.arsize;
	   method arvalid = m.arvalid;
	   method Bit#(1) aresetn(); return 1; endmethod
	   method Bit#(tagWidth)     arid(); return 0; endmethod
	   method arprot = m.arprot;
	   method arqos = 0;
	   method awaddr = m.awaddr;
	   method awburst = m.awburst;
	   method awcache = m.awcache;
	   method Bit#(tagWidth)     awid(); return 0; endmethod
	   method awlen = m.awlen;
	   method awlock = extend(m.awlock);
	   method awprot = m.awprot;
	   method awready = m.awready;
	   method Bit#(4)     awqos(); return 0; endmethod
	   method awsize = m.awsize;
	   method awvalid = m.awvalid;
	   method Action      bid(Bit#(tagWidth) v); endmethod
	   method bready = m.bready;
	   method bresp = m.bresp;
	   method bvalid = m.bvalid;
	   method rdata = m.rdata;
	   method Action      rid(Bit#(tagWidth) v); endmethod
	   method rlast = m.rlast;
	   method rready = m.rready;
	   method rresp = m.rresp;
	   method rvalid = m.rvalid;
	   method wdata = m.wdata;
	   method Bit#(tagWidth)     wid(); return 0; endmethod
	   method wlast = m.wlast;
	   method wready = m.wready;
	   method wstrb = m.wstrb;
	   method wvalid = m.wvalid;
	 interface extra = ?;
	 endinterface);
   endfunction
endinstance

interface SplitMemServer;
   interface MemServer#(DataBusWidth) server;
   interface MemClient#(DataBusWidth) busClient;
   interface MemClient#(DataBusWidth) bramClient;
   interface PipeOut#(MemData#(DataBusWidth)) data;
endinterface

module mkSplitMemServer(SplitMemServer);
   let readReqFifo   <- mkFIFOF();
   let readDataFifo  <- mkFIFOF();
   let writeReqFifo  <- mkFIFOF();
   let writeDataFifo <- mkFIFOF();
   let writeDoneFifo <- mkFIFOF();

   let busReadReqFifo   <- mkFIFOF();
   let busReadDataFifo  <- mkSizedBRAMFIFOF(16);
   let busWriteReqFifo  <- mkFIFOF();
   let busWriteDataFifo <- mkSizedBRAMFIFOF(16);
   let busWriteDoneFifo <- mkFIFOF();

   let bramReadReqFifo   <- mkFIFOF();
   let bramReadDataFifo  <- mkSizedBRAMFIFOF(16);
   let bramWriteReqFifo  <- mkFIFOF();
   let bramWriteDataFifo <- mkSizedBRAMFIFOF(16);
   let bramWriteDoneFifo <- mkFIFOF();

   let doneFifo <- mkFIFOF();

   let dataFifo <- mkSizedBRAMFIFOF(4096);

   AddressGenerator#(24,DataBusWidth) readAddrGenerator <- mkAddressGenerator();
   AddressGenerator#(24,DataBusWidth) writeAddrGenerator <- mkAddressGenerator();

   let readReqObj <- mkProbe();
   let readReqDest <- mkProbe();
   let readReqOffset <- mkProbe();
   let readDataDest <- mkProbe();
   let readDataData <- mkProbe();
   let readDataLast <- mkProbe();
   let readBeatAddr <- mkProbe();
   let readBeatByteCount <- mkProbe();
   let readBeatLast <- mkProbe();
   let busReadDataData <- mkProbe();
   let busReadDataLast <- mkProbe();

   rule rl_rd_req;
      MemRequest req <- toGet(readReqFifo).get();
      let dest = req.sglId[5:4];
      if (dest == 2)
	 bramReadReqFifo.enq(req);
      else
	 busReadReqFifo.enq(req);
      readAddrGenerator.request.put(PhysMemRequest { addr: truncate(req.offset), burstLen: req.burstLen, tag: extend(dest) });

      readReqObj <= req.sglId;
      readReqDest <= dest;
      readReqOffset <= req.offset[31:0];
   endrule

   rule rl_rd_data;
      let addrBeat <- readAddrGenerator.addrBeat.get();
      let dest = addrBeat.tag[1:0];
      MemData#(DataBusWidth) md;
      if (dest == 2)
	 md <- toGet(bramReadDataFifo).get();
      else
	 md <- toGet(busReadDataFifo).get();
      readDataFifo.enq(md);

      readDataDest <= dest;
      readDataData <= md.data;
      readDataLast <= md.last;
      readBeatAddr <= addrBeat.addr[15:0];
      readBeatByteCount <= addrBeat.bc;
      readBeatLast <= addrBeat.last;
   endrule

   rule rl_wr_req;
      MemRequest req <- toGet(writeReqFifo).get();
      Bit#(2) dest = req.sglId[5:4];
      if (dest == 2)
	 bramWriteReqFifo.enq(req);
      else if (dest == 3) begin
	 // no need to send the request
      end else
	 busWriteReqFifo.enq(req);
      writeAddrGenerator.request.put(PhysMemRequest { addr: truncate(req.offset), burstLen: req.burstLen, tag: extend(dest) });
   endrule

   rule rl_wr_data;
      let addrBeat <- writeAddrGenerator.addrBeat.get();
      let dest = addrBeat.tag[1:0];
      MemData#(DataBusWidth) md <- toGet(writeDataFifo).get();
      if (dest == 2)
	 bramWriteDataFifo.enq(md);
      else if (dest == 3)
	 dataFifo.enq(md);
      else
	 busWriteDataFifo.enq(md);
      if (addrBeat.last)
	 doneFifo.enq(md.tag);
   endrule

   rule rl_wr_done;
      let tag <- toGet(doneFifo).get();
      let dest = tag[1:0];
      Bit#(MemTagSize) doneTag;
      if (dest == 2)
	 doneTag <- toGet(bramWriteDoneFifo).get();
      else if (dest == 3)
	 doneTag = 0;
      else
	 doneTag <- toGet(busWriteDoneFifo).get();
      writeDoneFifo.enq(doneTag);
   endrule

   interface MemServer server;
      interface MemReadServer readServer;
	 interface readReq  = toPut(readReqFifo);
	 interface readData = toGet(readDataFifo);
      endinterface
      interface MemWriteServer writeServer;
	 interface writeReq  = toPut(writeReqFifo);
	 interface writeData = toPut(writeDataFifo);
	 interface writeDone = toGet(writeDoneFifo);
      endinterface
   endinterface
   interface MemClient busClient;
      interface MemReadClient readClient;
	 interface readReq  = toGet(busReadReqFifo);
	 interface Put readData;
	    method Action put(MemData#(DataBusWidth) md);
	       busReadDataFifo.enq(md);
	       busReadDataData <= md.data;
	       busReadDataLast <= md.last;
	    endmethod
	 endinterface
      endinterface
      interface MemWriteClient writeClient;
	 interface writeReq  = toGet(busWriteReqFifo);
	 interface writeData = toGet(busWriteDataFifo);
	 interface writeDone = toPut(busWriteDoneFifo);
      endinterface
   endinterface
   interface MemClient bramClient;
      interface MemReadClient readClient;
	 interface readReq  = toGet(bramReadReqFifo);
	 interface readData = toPut(bramReadDataFifo);
      endinterface
      interface MemWriteClient writeClient;
	 interface writeReq  = toGet(bramWriteReqFifo);
	 interface writeData = toGet(bramWriteDataFifo);
	 interface writeDone = toPut(bramWriteDoneFifo);
      endinterface
   endinterface
   interface PipeOut data = toPipeOut(dataFifo);
endmodule

interface MemServerPortal;
   interface MemServerPortalRequest request;
endinterface

module mkPhysMemSlavePortal#(PhysMemSlave#(32,DataBusWidth) ms, MemServerPortalIndication ind)(MemServerPortal);

   FIFOF#(PhysMemRequest#(32,DataBusWidth)) araddrFifo <- mkFIFOF();
   FIFOF#(PhysMemRequest#(32,DataBusWidth)) awaddrFifo <- mkFIFOF();
   FIFOF#(MemData#(DataBusWidth))           rdataFifo <- mkFIFOF();
   FIFOF#(MemData#(DataBusWidth))           wdataFifo <- mkFIFOF();
   FIFOF#(Bit#(6))                doneFifo <- mkFIFOF();

   let araddrCnx <- mkConnection(toGet(araddrFifo), ms.read_server.readReq);
   let awaddrCnx <- mkConnection(toGet(awaddrFifo), ms.write_server.writeReq);
   let rdataCnx  <- mkConnection(ms.read_server.readData, toPut(rdataFifo));
   let wdataCnx  <- mkConnection(toGet(wdataFifo), ms.write_server.writeData);
   let doneCnx   <- mkConnection(ms.write_server.writeDone, toPut(doneFifo));

   rule rl_rdata;
      let rdata <- toGet(rdataFifo).get();
      ind.readDone(rdata.data);
   endrule

   rule rl_writeDone;
      let tag <- toGet(doneFifo).get();
      ind.writeDone();
   endrule

   interface MemServerPortalRequest request;
      method Action read(Bit#(32) addr);
	 araddrFifo.enq(PhysMemRequest { addr: addr, burstLen: fromInteger(valueOf(TDiv#(DataBusWidth,8))), tag: 0 });
      endmethod
      method Action write(Bit#(32) addr, Bit#(DataBusWidth) value);
	 awaddrFifo.enq(PhysMemRequest { addr: addr, burstLen: fromInteger(valueOf(TDiv#(DataBusWidth,8))), tag: 0 });
	 wdataFifo.enq(MemData {data: value, tag: 0, last: True});
      endmethod
   endinterface
endmodule

// Copyright (c) 2016 Connectal Project

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import BRAM::*;
import BuildVector::*;
import Clocks::*;
import Connectable::*;
import FIFOF::*;
import GetPut::*;
import Vector::*;

import AddressGenerator::*;
import AxiBits::*;
import ConnectalClocks::*;
import ConnectalConfig::*;
import DefaultValue::*;
import HostInterface::*;
import MemTypes::*;
import PhysMemToBram::*;
import Pipe::*;
import TraceMemClient::*;
import XilinxCells::*;

import AxiPcieRootPort::*;
import NvmeIfc::*;
import NvmePins::*;

`ifndef TOP_SOURCES_PORTAL_CLOCK
import ConnectalBramFifo::*;
`else
import BRAMFIFO::*;
module mkDualClockBramFIFOF#(Clock clock1, Reset reset1, Clock clock2, Reset reset2)(FIFOF#(a))
   provisos (Bits#(a, asz), Add#(1, a__, asz));
   FIFOF#(a) fifo <- mkSizedBRAMFIFOF(512, clocked_by clock1, reset_by reset1);
   return fifo;
endmodule
`endif

interface Nvme;
   interface NvmeRequest request;
   interface NvmeTrace trace;
   interface NvmePins pins;
   interface Vector#(1, MemReadClient#(DataBusWidth)) dmaReadClient;
   interface Vector#(1, MemWriteClient#(DataBusWidth)) dmaWriteClient;
`ifdef TOP_SOURCES_PORTAL_CLOCK
   interface Clock portalClockSource;
`endif
endinterface

module mkNvme#(NvmeIndication ind, NvmeTrace trace)(Nvme);
   let clock <- exposeCurrentClock;
   let reset <- exposeCurrentReset;
   let refclk_p <- mkB2C1();
   let refclk_n <- mkB2C1();
   let pcie_clk_100mhz_buf <- mkClockIBUFDS_GTE2(
`ifdef ClockDefaultParam
       defaultValue,
`endif
      True, refclk_p.c, refclk_n.c);
`ifndef TOP_SOURCES_PORTAL_CLOCK
   let axiClockB2C    <- mkB2C1();
   let axiCtlClockB2C <- mkB2C1();
   let axiClock = axiClockB2C.c;
   let axiCtlClock = axiCtlClockB2C.c;
   let axiReset <- mkSyncReset(10, reset, axiClock);
   let axiCtlReset <- mkSyncReset(10, reset, axiCtlClock);
`else
   let axiClock = clock;
   let axiCtlClock = clock;
   let axiReset = reset;
   let axiCtlReset = reset;
`endif

   let axiRootPort <- mkAPRP(pcie_clk_100mhz_buf, reset, axiClock, axiReset, axiCtlClock, axiCtlReset);
`ifndef TOP_SOURCES_PORTAL_CLOCK
   let axiClockC2B <- mkC2B(axiRootPort.axi.aclk_out);
   let axiCtlClockC2B <- mkC2B(axiRootPort.axi.ctl_aclk_out);
   rule rl_connect_clocks;
      axiClockB2C.inputclock(axiClockC2B.o);
      axiCtlClockB2C.inputclock(axiClockC2B.o);
   endrule
`endif

   FIFOF#(Bit#(32)) dfifoCtl <- mkFIFOF();
   Axi4SlaveBits#(32,DataBusWidth,4,Empty) axiRootPortSlave    = toAxi4SlaveBits(axiRootPort.s_axi);
   Axi4SlaveLiteBits#(32,32)     axiRootPortSlaveCtl = toAxi4SlaveBits(axiRootPort.s_axi_ctl);
   PhysMemSlave#(32,DataBusWidth)          axiRootPortMemSlave    <- mkPhysMemSlave(axiRootPortSlave, clocked_by axiClock, reset_by axiReset);
   PhysMemSlave#(32,32)          axiRootPortMemSlaveCtl <- mkPhysMemSlave(axiRootPortSlaveCtl, clocked_by axiCtlClock, reset_by axiCtlReset);

   FIFOF#(PhysMemRequest#(32,DataBusWidth)) araddrFifo <- mkDualClockBramFIFOF(clock, reset, axiClock, axiReset);
   FIFOF#(PhysMemRequest#(32,DataBusWidth)) awaddrFifo <- mkDualClockBramFIFOF(clock, reset, axiClock, axiReset);
   FIFOF#(MemData#(DataBusWidth))           rdataFifo <- mkDualClockBramFIFOF(axiClock, axiReset, clock, reset);
   FIFOF#(MemData#(DataBusWidth))           wdataFifo <- mkDualClockBramFIFOF(clock, reset, axiClock, axiReset);
   FIFOF#(Bit#(6))                doneFifo <- mkDualClockBramFIFOF(axiClock, axiReset, clock, reset);

   let araddrCnx <- mkConnection(toGet(araddrFifo), axiRootPortMemSlave.read_server.readReq);
   let awaddrCnx <- mkConnection(toGet(awaddrFifo), axiRootPortMemSlave.write_server.writeReq);
   let rdataCnx  <- mkConnection(axiRootPortMemSlave.read_server.readData, toPut(rdataFifo));
   let wdataCnx  <- mkConnection(toGet(wdataFifo), axiRootPortMemSlave.write_server.writeData);
   let doneCnx   <- mkConnection(axiRootPortMemSlave.write_server.writeDone, toPut(doneFifo));

   rule rl_rdata;
      let rdata <- toGet(rdataFifo).get();
      ind.readDone(rdata.data);
   endrule

   rule rl_writeDone;
      let tag <- toGet(doneFifo).get();
      ind.writeDone();
   endrule

   FIFOF#(PhysMemRequest#(32,32)) araddrFifoCtl <- mkDualClockBramFIFOF(clock, reset, axiCtlClock, axiCtlReset);
   FIFOF#(PhysMemRequest#(32,32)) awaddrFifoCtl <- mkDualClockBramFIFOF(clock, reset, axiCtlClock, axiCtlReset);
   FIFOF#(MemData#(32))           rdataFifoCtl <- mkDualClockBramFIFOF(axiCtlClock, axiCtlReset, clock, reset);
   FIFOF#(MemData#(32))           wdataFifoCtl <- mkDualClockBramFIFOF(clock, reset, axiCtlClock, axiCtlReset);
   FIFOF#(Bit#(6))                doneFifoCtl <- mkDualClockBramFIFOF(axiCtlClock, axiCtlReset, clock, reset);

   let araddrCtlCnx <- mkConnection(toGet(araddrFifoCtl), axiRootPortMemSlaveCtl.read_server.readReq);
   let awaddrCtlCnx <- mkConnection(toGet(awaddrFifoCtl), axiRootPortMemSlaveCtl.write_server.writeReq);
   let rdataCtlCnx  <- mkConnection(axiRootPortMemSlaveCtl.read_server.readData, toPut(rdataFifoCtl));
   let wdataCtlCnx  <- mkConnection(toGet(wdataFifoCtl), axiRootPortMemSlaveCtl.write_server.writeData);
   let doneCtlCnx   <- mkConnection(axiRootPortMemSlaveCtl.write_server.writeDone, toPut(doneFifoCtl));

   rule rl_rdata_ctl;
      let rdata <- toGet(rdataFifoCtl).get();
      ind.readDone(extend(rdata.data));
   endrule

   rule rl_writeDone_ctl;
      let tag <- toGet(doneFifoCtl).get();
      ind.writeDone();
   endrule

   Axi4MasterBits#(32,DataBusWidth,MemTagSize,Empty) m_axi_mm = toAxi4MasterBits(axiRootPort.m_axi);
   let getObjId = (interface GetObjId;
		   method SGLId objId(Bit#(32) addr); return extend(addr[31:24]); endmethod
		   method Bit#(MemOffsetSize) addr(Bit#(32) axiAddr); return extend(axiAddr[23:0]); endmethod
		   endinterface);
   let memReadClient  <- mkMemReadClient(getObjId, m_axi_mm);
   let memWriteClient <- mkMemWriteClient(getObjId, m_axi_mm);

   FIFOF#(Tuple4#(DmaChannel,Bool,MemRequest,Bit#(32))) traceFifo <- mkSizedBRAMFIFOF(128);
   PipeIn#(Tuple4#(DmaChannel,Bool,MemRequest,Bit#(32))) tracePipe = toPipeIn(traceFifo);
   FIFOF#(Tuple4#(DmaChannel,Bool,MemData#(DataBusWidth),Bit#(32))) traceDataFifo <- mkSizedBRAMFIFOF(512);
   PipeIn#(Tuple4#(DmaChannel,Bool,MemData#(DataBusWidth),Bit#(32))) traceDataPipe = toPipeIn(traceDataFifo);
   rule rl_trace1;
      match { .chan, .write, .req, .timestamp } <- toGet(traceFifo).get();
      trace.traceDmaRequest(chan, write, truncate(req.sglId), extend(req.offset), extend(req.burstLen), extend(req.tag), timestamp);
   endrule
   rule rl_trace_data;
      match { .chan, .write, .md, .timestamp } <- toGet(traceDataFifo).get();
      trace.traceDmaData(chan, write, md.data, md.last, extend(md.tag), timestamp);
   endrule

   let traceReadClient <- mkTraceReadClient(tracePipe,traceDataPipe,DMA_TX,memReadClient);
   let traceWriteClient <- mkTraceWriteClient(tracePipe,traceDataPipe,DMA_RX,memWriteClient);
   let traceClient = (interface MemClient#(DataBusWidth);
			 interface readClient = traceReadClient;
			 interface writeClient = traceWriteClient;
		      endinterface);

   let splitter <- mkSplitMemServer();
   let traceMemCnx <- mkConnection(traceClient, splitter.server);
   rule rl_data;
      let md <- toGet(splitter.data).get();
      trace.traceData(md.data, md.last, extend(md.tag));
   endrule

   interface NvmeRequest request;

      method Action status();
        ind.status(axiRootPort.mmcm.lock());
      endmethod

      method Action read32(Bit#(32) addr);
	 araddrFifo.enq(PhysMemRequest { addr: addr, burstLen: 4, tag: 0 });
      endmethod
      method Action write32(Bit#(32) addr, Bit#(32) value);
	 awaddrFifo.enq(PhysMemRequest { addr: addr, burstLen: 4, tag: 0 });
	 wdataFifo.enq(MemData {data: extend(value), tag: 0, last: True});
      endmethod
      method Action read(Bit#(32) addr);
	 araddrFifo.enq(PhysMemRequest { addr: addr, burstLen: fromInteger(valueOf(TDiv#(DataBusWidth,8))), tag: 0 });
      endmethod
      method Action write(Bit#(32) addr, Bit#(DataBusWidth) value);
	 awaddrFifo.enq(PhysMemRequest { addr: addr, burstLen: fromInteger(valueOf(TDiv#(DataBusWidth,8))), tag: 0 });
	 wdataFifo.enq(MemData {data: value, tag: 0, last: True});
      endmethod

      method Action readCtl(Bit#(32) addr);
	 araddrFifoCtl.enq(PhysMemRequest { addr: addr, burstLen: 4, tag: 0 });
      endmethod
      method Action writeCtl(Bit#(32) addr, Bit#(DataBusWidth) value);
	 awaddrFifoCtl.enq(PhysMemRequest { addr: addr, burstLen: 4, tag: 0 });
	 wdataFifoCtl.enq(MemData {data: truncate(value), tag: 0, last: True});
      endmethod
   endinterface
   interface Clock portalClockSource = axiRootPort.axi.aclk_out;
   interface NvmePins pins;
      interface deleteme_unused_clock = clock;
      interface pcie_sys_reset_n = reset;
      interface pcie = axiRootPort.pci;
      method Action pcie_refclk(Bit#(1) p, Bit#(1) n);
         refclk_p.inputclock(p);
         refclk_n.inputclock(n);
      endmethod
   endinterface
   interface Vector dmaReadClient = vec(splitter.busClient.readClient);
   interface Vector dmaWriteClient = vec(splitter.busClient.writeClient);
endmodule

instance ToAxi4SlaveBits#(Axi4SlaveBits#(32,DataBusWidth,4,Empty), AprpS_axi);
   function Axi4SlaveBits#(32,DataBusWidth,4,Empty) toAxi4SlaveBits(AprpS_axi s);
      return (interface Axi4SlaveBits#(32,DataBusWidth,4,Empty);
	 method araddr = compose(s.araddr, extend);
	 method arburst = s.arburst;
	 //method arcache = s.arcache;
	 method arid = s.arid;
	 method arlen = s.arlen;
	 //method arlock = s.arlock;
	 //method arprot = s.arprot;
	 //method arqos = s.arqos;
	 method arready = s.arready;
	 method arsize = s.arsize;
	 method arvalid = s.arvalid;

	 method awaddr = compose(s.awaddr, extend);
	 method awburst = s.awburst;
	 //method awcache = s.awcache;
	 method awid = s.awid;
	 method awlen = s.awlen;
	 //method awlock = s.awlock;
	 //method awprot = s.awprot;
	 //method awqos = s.awqos;
	 method awready = s.awready;
	 method awsize = s.awsize;
	 method awvalid = s.awvalid;

	 method bid = s.bid;
	 method bready = s.bready;
	 method bresp = s.bresp;
	 method bvalid = s.bvalid;
	 method rdata = s.rdata;
	 method rid = s.rid;
	 method rlast = s.rlast;
	 method rready = s.rready;
	 method rresp = s.rresp;
	 method rvalid = s.rvalid;
	 method wdata = s.wdata;
	 method wlast = s.wlast;
	 method wready = s.wready;
	 method wvalid = s.wvalid;
	 method wstrb = s.wstrb;
	 endinterface);
   endfunction
endinstance

instance ToAxi4SlaveBits#(Axi4SlaveLiteBits#(32,32), AprpS_axi_ctl);
   function Axi4SlaveLiteBits#(32,32) toAxi4SlaveBits(AprpS_axi_ctl s);
      return (interface Axi4SlaveLiteBits#(32,32);
	 method araddr = compose(s.araddr, extend);
	 method arready = s.arready;
	 method arvalid = s.arvalid;

	 method awaddr = compose(s.awaddr, extend);
	 method awready = s.awready;
	 method awvalid = s.awvalid;

	 method bready = s.bready;
	 method bresp = s.bresp;
	 method bvalid = s.bvalid;
	 method rdata = s.rdata;
	 method rready = s.rready;
	 method rresp = s.rresp;
	 method rvalid = s.rvalid;
	 method wdata = s.wdata;
	 method wready = s.wready;
	 method Action      wvalid(Bit#(1) v);
	    s.wvalid(v);
	    s.wstrb(pack(replicate(v)));
	 endmethod
	 endinterface);
   endfunction
endinstance

instance ToAxi4MasterBits#(Axi4MasterBits#(32,DataBusWidth,tagWidth,Empty), AprpM_axi);
function Axi4MasterBits#(32,DataBusWidth,tagWidth,Empty) toAxi4MasterBits(AprpM_axi m);
   return (interface Axi4MasterBits#(32,DataBusWidth,tagWidth,Empty);
	   method araddr = m.araddr;
	   method arburst = m.arburst;
	   method arcache = m.arcache;
	   method arlen = m.arlen;
	   method arlock = extend(m.arlock);
	   method arready = m.arready;
	   method arsize = m.arsize;
	   method arvalid = m.arvalid;
	   method Bit#(1) aresetn(); return 1; endmethod
	   method Bit#(tagWidth)     arid(); return 0; endmethod
	   method arprot = m.arprot;
	   method arqos = 0;
	   method awaddr = m.awaddr;
	   method awburst = m.awburst;
	   method awcache = m.awcache;
	   method Bit#(tagWidth)     awid(); return 0; endmethod
	   method awlen = m.awlen;
	   method awlock = extend(m.awlock);
	   method awprot = m.awprot;
	   method awready = m.awready;
	   method Bit#(4)     awqos(); return 0; endmethod
	   method awsize = m.awsize;
	   method awvalid = m.awvalid;
	   method Action      bid(Bit#(tagWidth) v); endmethod
	   method bready = m.bready;
	   method bresp = m.bresp;
	   method bvalid = m.bvalid;
	   method rdata = m.rdata;
	   method Action      rid(Bit#(tagWidth) v); endmethod
	   method rlast = m.rlast;
	   method rready = m.rready;
	   method rresp = m.rresp;
	   method rvalid = m.rvalid;
	   method wdata = m.wdata;
	   method Bit#(tagWidth)     wid(); return 0; endmethod
	   method wlast = m.wlast;
	   method wready = m.wready;
	   method wstrb = m.wstrb;
	   method wvalid = m.wvalid;
	 interface extra = ?;
	 endinterface);
   endfunction
endinstance

interface SplitMemServer;
   interface MemServer#(DataBusWidth) server;
   interface MemClient#(DataBusWidth) busClient;
   interface MemClient#(DataBusWidth) bramClient;
   interface PipeOut#(MemData#(DataBusWidth)) data;
endinterface

module mkSplitMemServer(SplitMemServer);
   let readReqFifo   <- mkFIFOF();
   let readDataFifo  <- mkFIFOF();
   let writeReqFifo  <- mkFIFOF();
   let writeDataFifo <- mkFIFOF();
   let writeDoneFifo <- mkFIFOF();

   let busReadReqFifo   <- mkFIFOF();
   let busReadDataFifo  <- mkFIFOF();
   let busWriteReqFifo  <- mkFIFOF();
   let busWriteDataFifo <- mkFIFOF();
   let busWriteDoneFifo <- mkFIFOF();

   let bramReadReqFifo   <- mkFIFOF();
   let bramReadDataFifo  <- mkFIFOF();
   let bramWriteReqFifo  <- mkFIFOF();
   let bramWriteDataFifo <- mkFIFOF();
   let bramWriteDoneFifo <- mkFIFOF();

   let doneFifo <- mkFIFOF();

   let dataFifo <- mkSizedBRAMFIFOF(128);

   AddressGenerator#(24,DataBusWidth) readAddrGenerator <- mkAddressGenerator();
   AddressGenerator#(24,DataBusWidth) writeAddrGenerator <- mkAddressGenerator();

   rule rl_rd_req;
      MemRequest req <- toGet(readReqFifo).get();
      let dest = req.sglId[5:4];
      if (dest == 2)
	 bramReadReqFifo.enq(req);
      else
	 busReadReqFifo.enq(req);
      readAddrGenerator.request.put(PhysMemRequest { addr: truncate(req.offset), burstLen: req.burstLen, tag: extend(dest) });
   endrule

   rule rl_rd_data;
      let addrBeat <- readAddrGenerator.addrBeat.get();
      let dest = addrBeat.tag[1:0];
      MemData#(DataBusWidth) md;
      if (dest == 2)
	 md <- toGet(bramReadDataFifo).get();
      else
	 md <- toGet(busReadDataFifo).get();
      readDataFifo.enq(md);
   endrule

   rule rl_wr_req;
      MemRequest req <- toGet(writeReqFifo).get();
      Bit#(2) dest = req.sglId[5:4];
      if (dest == 2)
	 bramWriteReqFifo.enq(req);
      else if (dest == 3) begin
	 // no need to send the request
      end else
	 busWriteReqFifo.enq(req);
      writeAddrGenerator.request.put(PhysMemRequest { addr: truncate(req.offset), burstLen: req.burstLen, tag: extend(dest) });
   endrule

   rule rl_wr_data;
      let addrBeat <- writeAddrGenerator.addrBeat.get();
      let dest = addrBeat.tag[1:0];
      MemData#(DataBusWidth) md <- toGet(writeDataFifo).get();
      if (dest == 2)
	 bramWriteDataFifo.enq(md);
      else if (dest == 3)
	 dataFifo.enq(md);
      else
	 busWriteDataFifo.enq(md);
      if (addrBeat.last)
	 doneFifo.enq(md.tag);
   endrule

   rule rl_wr_done;
      let tag <- toGet(doneFifo).get();
      let dest = tag[1:0];
      Bit#(MemTagSize) doneTag;
      if (dest == 2)
	 doneTag <- toGet(bramWriteDoneFifo).get();
      else if (dest == 3)
	 doneTag = 0;
      else
	 doneTag <- toGet(busWriteDoneFifo).get();
      writeDoneFifo.enq(doneTag);
   endrule

   interface MemServer server;
      interface MemReadServer readServer;
	 interface readReq  = toPut(readReqFifo);
	 interface readData = toGet(readDataFifo);
      endinterface
      interface MemWriteServer writeServer;
	 interface writeReq  = toPut(writeReqFifo);
	 interface writeData = toPut(writeDataFifo);
	 interface writeDone = toGet(writeDoneFifo);
      endinterface
   endinterface
   interface MemClient busClient;
      interface MemReadClient readClient;
	 interface readReq  = toGet(busReadReqFifo);
	 interface readData = toPut(busReadDataFifo);
      endinterface
      interface MemWriteClient writeClient;
	 interface writeReq  = toGet(busWriteReqFifo);
	 interface writeData = toGet(busWriteDataFifo);
	 interface writeDone = toPut(busWriteDoneFifo);
      endinterface
   endinterface
   interface MemClient bramClient;
      interface MemReadClient readClient;
	 interface readReq  = toGet(bramReadReqFifo);
	 interface readData = toPut(bramReadDataFifo);
      endinterface
      interface MemWriteClient writeClient;
	 interface writeReq  = toGet(bramWriteReqFifo);
	 interface writeData = toGet(bramWriteDataFifo);
	 interface writeDone = toPut(bramWriteDoneFifo);
      endinterface
   endinterface
   interface PipeOut data = toPipeOut(dataFifo);
endmodule

interface MemServerPortal;
   interface MemServerPortalRequest request;
endinterface

module mkPhysMemSlavePortal#(PhysMemSlave#(32,DataBusWidth) ms, MemServerPortalIndication ind)(MemServerPortal);

   FIFOF#(PhysMemRequest#(32,DataBusWidth)) araddrFifo <- mkFIFOF();
   FIFOF#(PhysMemRequest#(32,DataBusWidth)) awaddrFifo <- mkFIFOF();
   FIFOF#(MemData#(DataBusWidth))           rdataFifo <- mkFIFOF();
   FIFOF#(MemData#(DataBusWidth))           wdataFifo <- mkFIFOF();
   FIFOF#(Bit#(6))                doneFifo <- mkFIFOF();

   let araddrCnx <- mkConnection(toGet(araddrFifo), ms.read_server.readReq);
   let awaddrCnx <- mkConnection(toGet(awaddrFifo), ms.write_server.writeReq);
   let rdataCnx  <- mkConnection(ms.read_server.readData, toPut(rdataFifo));
   let wdataCnx  <- mkConnection(toGet(wdataFifo), ms.write_server.writeData);
   let doneCnx   <- mkConnection(ms.write_server.writeDone, toPut(doneFifo));

   rule rl_rdata;
      let rdata <- toGet(rdataFifo).get();
      ind.readDone(rdata.data);
   endrule

   rule rl_writeDone;
      let tag <- toGet(doneFifo).get();
      ind.writeDone();
   endrule

   interface MemServerPortalRequest request;
      method Action read(Bit#(32) addr);
	 araddrFifo.enq(PhysMemRequest { addr: addr, burstLen: fromInteger(valueOf(TDiv#(DataBusWidth,8))), tag: 0 });
      endmethod
      method Action write(Bit#(32) addr, Bit#(DataBusWidth) value);
	 awaddrFifo.enq(PhysMemRequest { addr: addr, burstLen: fromInteger(valueOf(TDiv#(DataBusWidth,8))), tag: 0 });
	 wdataFifo.enq(MemData {data: value, tag: 0, last: True});
      endmethod
   endinterface
endmodule


improve PhysMemToBramBE timing

improve PhysMemToBramBE timing

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import FIFOF::*;
import FIFO::*;
import GetPut::*;
import MemTypes::*;
import AddressGenerator::*;
import BRAM::*;
import Memory::*;

module mkPhysMemToBram#(BRAMServer#(Bit#(bramAddrWidth), Bit#(busDataWidth)) br) (PhysMemSlave#(busAddrWidth, busDataWidth))
   provisos(Add#(a__, bramAddrWidth, busAddrWidth));

   FIFOF#(Bit#(6))  readTagFifo <- mkFIFOF();
   FIFOF#(Bit#(6)) writeTagFifo <- mkFIFOF();
   FIFO#(Bool)     readLastFifo <- mkFIFO();
   AddressGenerator#(busAddrWidth,busDataWidth) readAddrGenerator <- mkAddressGenerator();
   AddressGenerator#(busAddrWidth,busDataWidth) writeAddrGenerator <- mkAddressGenerator();
   let verbose = False;

    Reg#(Bit#(32)) cycles      <- mkReg(0);
    rule count;
       cycles <= cycles + 1;
    endrule

   rule read_req;
      let addrBeat <- readAddrGenerator.addrBeat.get();
      let addr = addrBeat.addr;
      let tag = addrBeat.tag;
      let burstCount = addrBeat.bc;
      readTagFifo.enq(tag);
      readLastFifo.enq(addrBeat.last);
      Bit#(bramAddrWidth) regFileAddr = truncate(addr/fromInteger(valueOf(TDiv#(busDataWidth,8))));
      br.request.put(BRAMRequest{write:False, responseOnWrite:False, address:regFileAddr, datain:?});
      if (verbose) $display("%d read_server.readData (a) %h %d last=%d", cycles, addr, burstCount, addrBeat.last);
   endrule

   interface PhysMemReadServer read_server;
      interface Put readReq;
	 method Action put(PhysMemRequest#(busAddrWidth, busDataWidth) req);
            if (verbose) $display("%d axiSlave.read.readAddr %h bc %d", cycles, req.addr, req.burstLen);
	    readAddrGenerator.request.put(req);
	 endmethod
      endinterface
      interface Get readData;
	 method ActionValue#(MemData#(busDataWidth)) get();
   	    let tag = readTagFifo.first;
	    readTagFifo.deq;
	    readLastFifo.deq;
            let data <- br.response.get;
            if (verbose) $display("%d read_server.readData (b) %h", cycles, data);
            return MemData { data: data, tag: tag, last: readLastFifo.first };
	 endmethod
      endinterface
   endinterface
   interface PhysMemWriteServer write_server;
      interface Put writeReq;
	 method Action put(PhysMemRequest#(busAddrWidth, busDataWidth) req);
	    writeAddrGenerator.request.put(req);
            if (verbose) $display("%d write_server.writeAddr %h bc %d", cycles, req.addr, req.burstLen);
	 endmethod
      endinterface
      interface Put writeData;
	 method Action put(MemData#(busDataWidth) resp);
	    let addrBeat <- writeAddrGenerator.addrBeat.get();
	    let addr = addrBeat.addr;
	    Bit#(bramAddrWidth) regFileAddr = truncate(addr/fromInteger(valueOf(TDiv#(busDataWidth,8))));
            br.request.put(BRAMRequest{write:True, responseOnWrite:False, address:regFileAddr, datain:resp.data});
            if (verbose) $display("%d write_server.writeData %h %h %d", cycles, addr, resp.data, addrBeat.bc);
            if (addrBeat.last)
               writeTagFifo.enq(addrBeat.tag);
	 endmethod
      endinterface
      interface Get writeDone;
	 method ActionValue#(Bit#(6)) get();
	    writeTagFifo.deq;
            return writeTagFifo.first;
	 endmethod
      endinterface
   endinterface
endmodule

module mkPhysMemToBramBE#(BRAMServerBE#(Bit#(bramAddrWidth), Bit#(busDataWidth), dataWidthBytes) br) (PhysMemSlave#(busAddrWidth, busDataWidth))
   provisos(Add#(a__, bramAddrWidth, busAddrWidth)
           ,Mul#(dataWidthBytes, 8, busDataWidth)
           ,Div#(busDataWidth, 8, dataWidthBytes)
           );
   let verbose = False;

   FIFOF#(Bit#(6))  readTagFifo <- mkFIFOF();
   FIFOF#(Bit#(6)) writeTagFifo <- mkFIFOF();
   FIFO#(Bool)     readLastFifo <- mkFIFO();
   FIFOF#(Bit#(TDiv#(busDataWidth, 8))) readByteEnableFifo <- mkFIFOF;
   FIFOF#(Bit#(TDiv#(busDataWidth, 8))) writeByteEnableFifo <- mkFIFOF;

   AddressGenerator#(busAddrWidth,busDataWidth) readAddrGenerator <- mkAddressGenerator();
   AddressGenerator#(busAddrWidth,busDataWidth) writeAddrGenerator <- mkAddressGenerator();

   FIFO#(PhysMemRequest#(busAddrWidth, busDataWidth)) req_ars <- mkFIFO1;
   FIFO#(Bit#(bramAddrWidth)) req_addr <- mkFIFO1;
   FIFO#(BRAMRequestBE#(Bit#(bramAddrWidth), Bit#(busDataWidth), dataWidthBytes)) req_aws <- mkFIFO1;

   Reg#(Bit#(32)) cycles      <- mkReg(0);
   rule count if (verbose);
      cycles <= cycles + 1;
   endrule

   rule req_ar;
     let req <- toGet(req_ars).get;
     readAddrGenerator.request.put(req);
   endrule

   rule read_req;
      let addrBeat <- readAddrGenerator.addrBeat.get();
      let addr = addrBeat.addr;
      let tag = addrBeat.tag;
      let burstCount = addrBeat.bc;
      readTagFifo.enq(tag);
      readLastFifo.enq(addrBeat.last);
      Bit#(bramAddrWidth) regFileAddr = truncate(addr << fromInteger(valueOf(TLog#(dataWidthBytes))));
      req_addr.enq(regFileAddr);
      if (verbose) $display("%d read_server.readData (a) %h %d last=%d", cycles, addr, burstCount, addrBeat.last);
   endrule

   rule read_bram_req;
      let addr <- toGet(req_addr).get;
      br.request.put(BRAMRequestBE{writeen:0, responseOnWrite:False, address:addr, datain:?});
   endrule

   rule req_aw;
      let req <- toGet(req_aws).get;
      br.request.put(req);
   endrule

   interface PhysMemReadServer read_server;
      interface Put readReq;
	 method Action put(PhysMemRequest#(busAddrWidth, busDataWidth) req);
            if (verbose) $display("%d read_server.readAddr %h bc %d fbe %x lbe %x", cycles, req.addr, req.burstLen
`ifdef BYTE_ENABLES
               , req.firstbe, req.lastbe
`endif
               );
            req_ars.enq(req);
            readByteEnableFifo.enq(reqLastByteEnable(req));
	 endmethod
      endinterface
      interface Get readData;
	 method ActionValue#(MemData#(busDataWidth)) get();
   	    let tag = readTagFifo.first;
	    readTagFifo.deq;
	    readLastFifo.deq;
            let data <- br.response.get;
            let readBE = readByteEnableFifo.first;
            Bit#(dataWidthBytes) byteEnable = readLastFifo.first ? readBE : maxBound;
            let newdata = updateDataWithMask(0, data, byteEnable);
            if (verbose) $display("%d read_server.readData (b) %h, %h", cycles, data, newdata);
            if (readLastFifo.first) begin
               readByteEnableFifo.deq;
            end
            return MemData { data: newdata, tag: tag, last: readLastFifo.first };
	 endmethod
      endinterface
   endinterface
   interface PhysMemWriteServer write_server;
      interface Put writeReq;
	 method Action put(PhysMemRequest#(busAddrWidth, busDataWidth) req);
	    writeAddrGenerator.request.put(req);
            writeByteEnableFifo.enq(reqLastByteEnable(req));
            if (verbose) $display("%d write_server.writeAddr %h bc %d fbe %x lbe %x", cycles, req.addr, req.burstLen
`ifdef BYTE_ENABLES
               , req.firstbe, req.lastbe
`endif
               );
	 endmethod
      endinterface
      interface Put writeData;
	 method Action put(MemData#(busDataWidth) resp);
	    let addrBeat <- writeAddrGenerator.addrBeat.get();
	    let addr = addrBeat.addr;
	    Bit#(bramAddrWidth) regFileAddr = truncate(addr << fromInteger(valueOf(TLog#(TDiv#(busDataWidth,8)))));
            let writeBE = writeByteEnableFifo.first;
            Bit#(dataWidthBytes) byteEnable = addrBeat.last ? writeBE : maxBound;
            req_aws.enq(BRAMRequestBE{writeen:byteEnable, responseOnWrite:False, address:regFileAddr, datain:resp.data});
            if (verbose) $display("%d write_server.writeData %h %h %d", cycles, addr, resp.data, addrBeat.bc);
            if (addrBeat.last)
               writeByteEnableFifo.deq;
               writeTagFifo.enq(addrBeat.tag);
	 endmethod
      endinterface
      interface Get writeDone;
	 method ActionValue#(Bit#(6)) get();
	    writeTagFifo.deq;
            return writeTagFifo.first;
	 endmethod
      endinterface
   endinterface
endmodule

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import FIFOF::*;
import FIFO::*;
import GetPut::*;
import MemTypes::*;
import AddressGenerator::*;
import BRAM::*;
import Memory::*;

module mkPhysMemToBram#(BRAMServer#(Bit#(bramAddrWidth), Bit#(busDataWidth)) br) (PhysMemSlave#(busAddrWidth, busDataWidth))
   provisos(Add#(a__, bramAddrWidth, busAddrWidth));

   FIFOF#(Bit#(6))  readTagFifo <- mkFIFOF();
   FIFOF#(Bit#(6)) writeTagFifo <- mkFIFOF();
   FIFO#(Bool)     readLastFifo <- mkFIFO();
   AddressGenerator#(busAddrWidth,busDataWidth) readAddrGenerator <- mkAddressGenerator();
   AddressGenerator#(busAddrWidth,busDataWidth) writeAddrGenerator <- mkAddressGenerator();
   let verbose = False;

    Reg#(Bit#(32)) cycles      <- mkReg(0);
    rule count;
       cycles <= cycles + 1;
    endrule

   rule read_req;
      let addrBeat <- readAddrGenerator.addrBeat.get();
      let addr = addrBeat.addr;
      let tag = addrBeat.tag;
      let burstCount = addrBeat.bc;
      readTagFifo.enq(tag);
      readLastFifo.enq(addrBeat.last);
      Bit#(bramAddrWidth) regFileAddr = truncate(addr/fromInteger(valueOf(TDiv#(busDataWidth,8))));
      br.request.put(BRAMRequest{write:False, responseOnWrite:False, address:regFileAddr, datain:?});
      if (verbose) $display("%d read_server.readData (a) %h %d last=%d", cycles, addr, burstCount, addrBeat.last);
   endrule

   interface PhysMemReadServer read_server;
      interface Put readReq;
	 method Action put(PhysMemRequest#(busAddrWidth, busDataWidth) req);
            if (verbose) $display("%d axiSlave.read.readAddr %h bc %d", cycles, req.addr, req.burstLen);
	    readAddrGenerator.request.put(req);
	 endmethod
      endinterface
      interface Get readData;
	 method ActionValue#(MemData#(busDataWidth)) get();
   	    let tag = readTagFifo.first;
	    readTagFifo.deq;
	    readLastFifo.deq;
            let data <- br.response.get;
            if (verbose) $display("%d read_server.readData (b) %h", cycles, data);
            return MemData { data: data, tag: tag, last: readLastFifo.first };
	 endmethod
      endinterface
   endinterface
   interface PhysMemWriteServer write_server;
      interface Put writeReq;
	 method Action put(PhysMemRequest#(busAddrWidth, busDataWidth) req);
	    writeAddrGenerator.request.put(req);
            if (verbose) $display("%d write_server.writeAddr %h bc %d", cycles, req.addr, req.burstLen);
	 endmethod
      endinterface
      interface Put writeData;
	 method Action put(MemData#(busDataWidth) resp);
	    let addrBeat <- writeAddrGenerator.addrBeat.get();
	    let addr = addrBeat.addr;
	    Bit#(bramAddrWidth) regFileAddr = truncate(addr/fromInteger(valueOf(TDiv#(busDataWidth,8))));
            br.request.put(BRAMRequest{write:True, responseOnWrite:False, address:regFileAddr, datain:resp.data});
            if (verbose) $display("%d write_server.writeData %h %h %d", cycles, addr, resp.data, addrBeat.bc);
            if (addrBeat.last)
               writeTagFifo.enq(addrBeat.tag);
	 endmethod
      endinterface
      interface Get writeDone;
	 method ActionValue#(Bit#(6)) get();
	    writeTagFifo.deq;
            return writeTagFifo.first;
	 endmethod
      endinterface
   endinterface
endmodule

module mkPhysMemToBramBE#(BRAMServerBE#(Bit#(bramAddrWidth), Bit#(busDataWidth), dataWidthBytes) br) (PhysMemSlave#(busAddrWidth, busDataWidth))
   provisos(Add#(a__, bramAddrWidth, busAddrWidth)
           ,Mul#(dataWidthBytes, 8, busDataWidth)
           ,Div#(busDataWidth, 8, dataWidthBytes)
           );

   FIFOF#(Bit#(6))  readTagFifo <- mkFIFOF();
   FIFOF#(Bit#(6)) writeTagFifo <- mkFIFOF();
   FIFO#(Bool)     readLastFifo <- mkFIFO();
   FIFOF#(Bit#(TDiv#(busDataWidth, 8))) readByteEnableFifo <- mkFIFOF;
   FIFOF#(Bit#(TDiv#(busDataWidth, 8))) writeByteEnableFifo <- mkFIFOF;

   AddressGenerator#(busAddrWidth,busDataWidth) readAddrGenerator <- mkAddressGenerator();
   AddressGenerator#(busAddrWidth,busDataWidth) writeAddrGenerator <- mkAddressGenerator();
   let verbose = False;

    Reg#(Bit#(32)) cycles      <- mkReg(0);
    rule count;
       cycles <= cycles + 1;
    endrule

   rule read_req;
      let addrBeat <- readAddrGenerator.addrBeat.get();
      let addr = addrBeat.addr;
      let tag = addrBeat.tag;
      let burstCount = addrBeat.bc;
      readTagFifo.enq(tag);
      readLastFifo.enq(addrBeat.last);
      Bit#(bramAddrWidth) regFileAddr = truncate(addr << fromInteger(valueOf(TLog#(dataWidthBytes))));
      br.request.put(BRAMRequestBE{writeen:0, responseOnWrite:False, address:regFileAddr, datain:?});
      if (verbose) $display("%d read_server.readData (a) %h %d last=%d", cycles, addr, burstCount, addrBeat.last);
   endrule

   interface PhysMemReadServer read_server;
      interface Put readReq;
	 method Action put(PhysMemRequest#(busAddrWidth, busDataWidth) req);
            if (verbose) $display("%d read_server.readAddr %h bc %d fbe %x lbe %x", cycles, req.addr, req.burstLen
`ifdef BYTE_ENABLES
               , req.firstbe, req.lastbe
`endif
               );
	    readAddrGenerator.request.put(req);
            readByteEnableFifo.enq(reqLastByteEnable(req));
	 endmethod
      endinterface
      interface Get readData;
	 method ActionValue#(MemData#(busDataWidth)) get();
   	    let tag = readTagFifo.first;
	    readTagFifo.deq;
	    readLastFifo.deq;
            let data <- br.response.get;
            let readBE = readByteEnableFifo.first;
            Bit#(dataWidthBytes) byteEnable = readLastFifo.first ? readBE : maxBound;
            let newdata = updateDataWithMask(0, data, byteEnable);
            if (verbose) $display("%d read_server.readData (b) %h, %h", cycles, data, newdata);
            if (readLastFifo.first) begin
               readByteEnableFifo.deq;
            end
            return MemData { data: newdata, tag: tag, last: readLastFifo.first };
	 endmethod
      endinterface
   endinterface
   interface PhysMemWriteServer write_server;
      interface Put writeReq;
	 method Action put(PhysMemRequest#(busAddrWidth, busDataWidth) req);
	    writeAddrGenerator.request.put(req);
            writeByteEnableFifo.enq(reqLastByteEnable(req));
            if (verbose) $display("%d write_server.writeAddr %h bc %d fbe %x lbe %x", cycles, req.addr, req.burstLen
`ifdef BYTE_ENABLES
               , req.firstbe, req.lastbe
`endif
               );
	 endmethod
      endinterface
      interface Put writeData;
	 method Action put(MemData#(busDataWidth) resp);
	    let addrBeat <- writeAddrGenerator.addrBeat.get();
	    let addr = addrBeat.addr;
	    Bit#(bramAddrWidth) regFileAddr = truncate(addr << fromInteger(valueOf(TLog#(TDiv#(busDataWidth,8)))));
            let writeBE = writeByteEnableFifo.first;
            Bit#(dataWidthBytes) byteEnable = addrBeat.last ? writeBE : maxBound;
            br.request.put(BRAMRequestBE{writeen:byteEnable, responseOnWrite:False, address:regFileAddr, datain:resp.data});
            if (verbose) $display("%d write_server.writeData %h %h %d", cycles, addr, resp.data, addrBeat.bc);
            if (addrBeat.last)
               writeByteEnableFifo.deq;
               writeTagFifo.enq(addrBeat.tag);
	 endmethod
      endinterface
      interface Get writeDone;
	 method ActionValue#(Bit#(6)) get();
	    writeTagFifo.deq;
            return writeTagFifo.first;
	 endmethod
      endinterface
   endinterface
endmodule