update to reflect changes to mkMPStreamEngine

// Copyright (c) 2016 Connectal Project

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import Arbitrate::*;
import BRAM::*;
import BuildVector::*;
import Clocks::*;
import Connectable::*;
import FIFOF::*;
import Gearbox::*;
import GetPut::*;
import Probe::*;
import StmtFSM::*;
import Vector::*;

import AddressGenerator::*;
import AxiBits::*;
import AxiStream::*;
import ConnectalClocks::*;
import ConnectalConfig::*;
import DefaultValue::*;
import GearboxGetPut::*;
import HostInterface::*;
import MemReadEngine::*;
import MemTypes::*;
import PhysMemSlaveFromBram::*;
import Pipe::*;
import TraceMemClient::*;
import XilinxCells::*;
import MPEngine::*;

`ifndef PCIE3
import AxiPcieRootPort::*;
`else
import AxiPcie3RootPort::*;
`endif
import NvmeIfc::*;
import NvmePins::*;

typedef struct {
   Bit#(8)  opcode;
   Bit#(8)  flags;
   Bit#(16) requestId;
   Bit#(64) startBlock;
   Bit#(32) numBlocks;
   Bit#(32) dsm;
   } NvmeIoCommand deriving (Bits);

`ifndef TOP_SOURCES_PORTAL_CLOCK
import ConnectalBramFifo::*;
`else
import BRAMFIFO::*;
module mkDualClockBramFIFOF#(Clock clock1, Reset reset1, Clock clock2, Reset reset2)(FIFOF#(a))
   provisos (Bits#(a, asz), Add#(1, a__, asz));
   FIFOF#(a) fifo <- mkSizedBRAMFIFOF(512, clocked_by clock1, reset_by reset1);
   return fifo;
endmodule
`endif

function PipeIn#(t) sinkPipe();
   return (interface PipeIn#(a);
	      method Action enq(t v); endmethod
	      method Bool notFull(); return False; endmethod
	   endinterface);
endfunction

instance ArbRequestTC#(BRAMRequest#(a,b));
   function Bool isReadRequest(BRAMRequest#(a,b) x); return !x.write; endfunction
   function Bool isWriteRequest(BRAMRequest#(a,b) x); return x.write; endfunction
endinstance

interface Nvme;
   interface NvmeRequest request;
   interface NvmeDriverRequest driverRequest;
   interface MemServerPortalRequest bramRequest;
   interface NvmeTrace trace;
   interface NvmePins pins;
   interface Vector#(2, MemReadClient#(DataBusWidth)) dmaReadClient;
   interface Vector#(1, MemWriteClient#(DataBusWidth)) dmaWriteClient;
`ifdef TOP_SOURCES_PORTAL_CLOCK
   interface Clock portalClockSource;
`endif
endinterface

module mkNvme#(NvmeIndication ind, NvmeDriverIndication driverInd, NvmeTrace trace, MemServerPortalIndication bramIndication)(Nvme);
   let clock <- exposeCurrentClock;
   let reset <- exposeCurrentReset;
   let refclk_p <- mkB2C1();
   let refclk_n <- mkB2C1();
   let pcie_clk_100mhz_buf <- mkClockIBUFDS_GTE2(
`ifdef ClockDefaultParam
       defaultValue,
`endif
      True, refclk_p.c, refclk_n.c);
`ifndef TOP_SOURCES_PORTAL_CLOCK
   let axiClockB2C    <- mkB2C1();
   let axiCtlClockB2C <- mkB2C1();
   let axiClock = axiClockB2C.c;
   let axiCtlClock = axiCtlClockB2C.c;
   let axiReset <- mkSyncReset(10, reset, axiClock);
   let axiCtlReset <- mkSyncReset(10, reset, axiCtlClock);
`else
   let axiClock = clock;
   let axiCtlClock = clock;
   let axiReset = reset;
   let axiCtlReset = reset;
`endif

`ifndef PCIE3
   let axiRootPort <- mkAPRP(pcie_clk_100mhz_buf, reset, axiClock, axiReset, axiCtlClock, axiCtlReset);
`else
   let axiRootPort <- mkAPRP(axiCtlClock, pcie_clk_100mhz_buf, reset);
`endif
`ifndef TOP_SOURCES_PORTAL_CLOCK
   let axiClockC2B <- mkC2B(axiRootPort.axi.aclk_out);
   let axiCtlClockC2B <- mkC2B(axiRootPort.axi.ctl_aclk_out);
   rule rl_connect_clocks;
      axiClockB2C.inputclock(axiClockC2B.o);
      axiCtlClockB2C.inputclock(axiClockC2B.o);
   endrule
`endif

   FIFOF#(Bit#(32)) dfifoCtl <- mkFIFOF();
   Axi4SlaveBits#(32,PcieDataBusWidth,4,Empty) axiRootPortSlave = toAxi4SlaveBits(axiRootPort.s_axi);
   Axi4SlaveLiteBits#(32,32)                axiRootPortSlaveCtl = toAxi4SlaveBits(axiRootPort.s_axi_ctl);
   PhysMemSlave#(32,PcieDataBusWidth)       axiRootPortMemSlave <- mkPhysMemSlave(axiRootPortSlave, clocked_by axiClock, reset_by axiReset);
   PhysMemSlave#(32,32)                  axiRootPortMemSlaveCtl <- mkPhysMemSlave(axiRootPortSlaveCtl, clocked_by axiCtlClock, reset_by axiCtlReset);

   FIFOF#(PhysMemRequest#(32,PcieDataBusWidth)) araddrFifo <- mkFIFOF();
   FIFOF#(PhysMemRequest#(32,PcieDataBusWidth)) awaddrFifo <- mkFIFOF();
   FIFOF#(MemData#(PcieDataBusWidth))           rdataFifo <- mkFIFOF();
   FIFOF#(MemData#(PcieDataBusWidth))           wdataFifo <- mkFIFOF();
   FIFOF#(Bit#(6))                           doneFifo <- mkFIFOF();

   let araddrCnx <- mkConnection(toGet(araddrFifo), axiRootPortMemSlave.read_server.readReq);
   let awaddrCnx <- mkConnection(toGet(awaddrFifo), axiRootPortMemSlave.write_server.writeReq);
   let rdataCnx  <- mkConnection(axiRootPortMemSlave.read_server.readData, toPut(rdataFifo));
   let wdataCnx  <- mkConnection(toGet(wdataFifo), axiRootPortMemSlave.write_server.writeData);
   let doneCnx   <- mkConnection(axiRootPortMemSlave.write_server.writeDone, toPut(doneFifo));

   rule rl_rdata;
      let rdata <- toGet(rdataFifo).get();
      if (rdata.tag == 0)
	 driverInd.readDone(truncate(rdata.data));
   endrule

   rule rl_writeDone;
      let tag <- toGet(doneFifo).get();
      if (tag == 0)
	 driverInd.writeDone();
   endrule

   FIFOF#(PhysMemRequest#(32,32)) araddrFifoCtl <- mkFIFOF();
   FIFOF#(PhysMemRequest#(32,32)) awaddrFifoCtl <- mkFIFOF();
   FIFOF#(MemData#(32))           rdataFifoCtl <- mkFIFOF();
   FIFOF#(MemData#(32))           wdataFifoCtl <- mkFIFOF();
   FIFOF#(Bit#(6))                doneFifoCtl <- mkFIFOF();

   let araddrCtlCnx <- mkConnection(toGet(araddrFifoCtl), axiRootPortMemSlaveCtl.read_server.readReq);
   let awaddrCtlCnx <- mkConnection(toGet(awaddrFifoCtl), axiRootPortMemSlaveCtl.write_server.writeReq);
   let rdataCtlCnx  <- mkConnection(axiRootPortMemSlaveCtl.read_server.readData, toPut(rdataFifoCtl));
   let wdataCtlCnx  <- mkConnection(toGet(wdataFifoCtl), axiRootPortMemSlaveCtl.write_server.writeData);
   let doneCtlCnx   <- mkConnection(axiRootPortMemSlaveCtl.write_server.writeDone, toPut(doneFifoCtl));

   rule rl_rdata_ctl;
      let rdata <- toGet(rdataFifoCtl).get();
      driverInd.readDone(extend(rdata.data));
   endrule

   rule rl_writeDone_ctl;
      let tag <- toGet(doneFifoCtl).get();
      driverInd.writeDone();
   endrule

   Axi4MasterBits#(32,PcieDataBusWidth,MemTagSize,Empty) m_axi_mm = toAxi4MasterBits(axiRootPort.m_axi);
   let getObjId = (interface GetObjId;
		   method SGLId objId(Bit#(32) addr); return extend(addr[31:24]); endmethod
		   method Bit#(MemOffsetSize) addr(Bit#(32) axiAddr); return extend(axiAddr[23:0]); endmethod
		   endinterface);
   let memReadClient  <- mkMemReadClient(getObjId, m_axi_mm);
   let memWriteClient <- mkMemWriteClient(getObjId, m_axi_mm);

   Reg#(Bool) traceEnabled <- mkReg(False);

   FIFOF#(Tuple4#(DmaChannel,Bool,MemRequest,Bit#(32))) traceFifo <- mkSizedBRAMFIFOF(128);
   PipeIn#(Tuple4#(DmaChannel,Bool,MemRequest,Bit#(32))) tracePipe = traceEnabled ? toPipeIn(traceFifo) : sinkPipe();
   FIFOF#(Tuple4#(DmaChannel,Bool,MemData#(PcieDataBusWidth),Bit#(32))) traceDataFifo <- mkSizedBRAMFIFOF(2048);
   PipeIn#(Tuple4#(DmaChannel,Bool,MemData#(PcieDataBusWidth),Bit#(32))) traceDataPipe = traceEnabled ? toPipeIn(traceDataFifo) : sinkPipe();
   FIFOF#(Tuple2#(DmaChannel,Bit#(32)))                              traceDoneFifo <- mkSizedBRAMFIFOF(128);
   PipeIn#(Tuple2#(DmaChannel,Bit#(32)))                             traceDonePipe = traceEnabled ? toPipeIn(traceDoneFifo) : sinkPipe();

   rule rl_trace1;
      match { .chan, .write, .req, .timestamp } <- toGet(traceFifo).get();
      trace.traceDmaRequest(chan, write, truncate(req.sglId), extend(req.offset), extend(req.burstLen), extend(req.tag), timestamp);
   endrule
   rule rl_trace_data;
      match { .chan, .write, .md, .timestamp } <- toGet(traceDataFifo).get();
      trace.traceDmaData(chan, write, unpack(md.data), md.last, extend(md.tag), timestamp);
   endrule
   rule rl_trace_done;
      match { .chan, .timestamp } <- toGet(traceDoneFifo).get();
      trace.traceDmaDone(chan, 0, timestamp);
   endrule

   let traceReadClient <- mkTraceReadClient(tracePipe,traceDataPipe,DMA_TX,memReadClient);
   let traceWriteClient <- mkTraceWriteClient(tracePipe,traceDataPipe,traceDonePipe,DMA_RX,memWriteClient);
   let traceClient = (interface MemClient#(PcieDataBusWidth);
			 interface readClient = traceReadClient;
			 interface writeClient = traceWriteClient;
		      endinterface);

   SplitMemServer#(PcieDataBusWidth) splitter <- mkSplitMemServer();
   MemServerGearbox#(PcieDataBusWidth,DataBusWidth) axiGearbox <- mk1toNMemServerGearbox();
   let gearboxCnx <- mkConnection(splitter.busClient, axiGearbox.server);

   let requestSize = 64;
   let responseSize = 16;
   let bytesPerEntry = valueOf(TDiv#(PcieDataBusWidth,8));
   let wordsPerEntry = valueOf(TDiv#(PcieDataBusWidth,32));

   BRAM_Configure bramConfig = defaultValue;
   bramConfig.latency = 2;
   bramConfig.memorySize = 4096*4/bytesPerEntry; // 4 pages
   BRAM2Port#(Bit#(32),Bit#(PcieDataBusWidth)) bram <- mkBRAM2Server(bramConfig);

   let arbIfc <- mkFixedPriority();
   Arbiter#(3, BRAMRequest#(Bit#(32),Bit#(PcieDataBusWidth)),Bit#(PcieDataBusWidth)) arbiter <- mkArbiter(arbIfc,2);
   let arbCnx <- mkConnection(arbiter.master, bram.portB);

   MemServer#(PcieDataBusWidth)            bramMemA <- mkMemServerFromBram(bram.portA);
   PhysMemSlave#(32,PcieDataBusWidth)      bramMemB <- mkPhysMemSlaveFromBram(arbiter.users[0]);
   MemServerPortal          bramMemServerPortal <- mkPhysMemSlavePortal(bramMemB,bramIndication);

   let portB1 = arbiter.users[1];
   let portB2 = arbiter.users[2];
   Reg#(Bit#(16)) requestId <- mkReg(0);
   Reg#(Bit#(16)) responseId <- mkReg(0);
   Reg#(Bit#(16)) requestQueueEntryCount <- mkReg(8);
   Reg#(Bool) requestSlotAvailable <- mkReg(True);
   Reg#(Bool) requestInProgress <- mkReg(True);
   Reg#(Vector#(16,Bit#(32))) ioCommand <- mkReg(unpack(0));
   Reg#(Bit#(32)) requestStartTimestamp <- mkReg(0);

   FIFOF#(NvmeIoCommand) ioCommandFifo <- mkFIFOF();

   Reg#(Bit#(32)) cycles <- mkReg(0);
   rule rl_cycles;
      cycles <= cycles + 1;
   endrule

   Reg#(Bit#(32)) i <- mkReg(0);
   Reg#(Bit#(1)) phase <- mkReg(1);

   let bramQueryAddress <- mkProbe();
   let bramQueryResponse <- mkProbe();
   let requestCompleted <- mkProbe();

   let requestFsm <- mkAutoFSM(seq
      while (True) seq
	 action
	    let req <- toGet(ioCommandFifo).get();
	    Vector#(16,Bit#(32)) command = unpack(0);
	    command[0] = { req.requestId, req.flags, req.opcode };
	    command[1] = 1; // nsid
	    // prp1: send data to fifo
	    command[6] = 32'h30000000;
	    // p2p2: read PRP list from BRAM at offset 0x2000
	    command[8] = 32'h20002000;
	    command[10] = req.startBlock[31:0];
	    command[11] = req.startBlock[63:32];
	    command[12] = req.numBlocks-1;
	    command[13] = req.dsm;
	    requestId <= req.requestId;
	    ioCommand <= command;

	    requestSlotAvailable <= False;
	 endaction

	 while (!requestSlotAvailable) seq // wait for open request slot
	    portB1.request.put(BRAMRequest {address: (extend(requestId[2:0]) * fromInteger(responseSize/bytesPerEntry)) + fromInteger(12/bytesPerEntry),
					    write: False, responseOnWrite: False, datain: 0});
	    action
	       let response <- portB1.response.get();
	       if (response[16+(3*32 % valueOf(PcieDataBusWidth))] == ~phase) begin
		  requestSlotAvailable <= True;
	       end
	    endaction
	 endseq // wait for open request slot

	 // write a IO read request to BRAM
         for (i <= 0; i < fromInteger(requestSize/bytesPerEntry); i <= i + 1) seq
	    portB1.request.put(BRAMRequest {address: 'h1000/fromInteger(bytesPerEntry) + (extend(requestId[2:0]) * fromInteger(requestSize/bytesPerEntry)) + i,
					    write: True, responseOnWrite: False,
					    datain: truncate(pack(ioCommand) >> (i*fromInteger(valueOf(PcieDataBusWidth)))) });
         endseq

	 // tell the NVME the new submission queue tail pointer
	 action
	    awaddrFifo.enq(PhysMemRequest { addr: 'h1000 + (2*2+0)*(4<<0), burstLen: 4, tag: 1 });
	    wdataFifo.enq(MemData{data: zeroExtend((requestId+1)[2:0]), tag: 1, last: True});
	    requestInProgress <= True;
	    requestStartTimestamp <= cycles;
	    trace.traceData(unpack(0), False, truncate(requestId), cycles);
	 endaction
      endseq // while True
      endseq);

   let responseFsm <- mkAutoFSM(seq
	 // wait for response queue to be updated
      while (True) seq
	 requestInProgress <= True;
	 while (requestInProgress) seq
	    portB2.request.put(BRAMRequest {address: (extend(responseId[2:0]) * fromInteger(responseSize/bytesPerEntry)) + fromInteger(12/bytesPerEntry),
					    write: False, responseOnWrite: False, datain: 0});
	    bramQueryAddress <= (responseId * fromInteger(responseSize/bytesPerEntry)) + fromInteger(12/bytesPerEntry);
	    action
	       let response <- portB2.response.get();
	       bramQueryResponse <= response;
	       if (response[16+(3*32 % valueOf(PcieDataBusWidth))] == phase) begin
		  requestCompleted <= response;
		  // if status field written by NVME
		  ind.transferCompleted(responseId, truncate(response), cycles - requestStartTimestamp);
		  requestInProgress <= False;
	       end
	    endaction
	 endseq // while requestInProgress

	 // tell the NVME the new response queue head pointer
	 action
	    let nextRequestId = (responseId + 1);
	    if (nextRequestId[2:0] == 0) begin
	       phase <= ~phase;
	    end
	    responseId <= nextRequestId;
	    awaddrFifo.enq(PhysMemRequest { addr: 'h1000 + (2*2+1)*(4<<0), burstLen: 4, tag: 1 });
	    wdataFifo.enq(MemData{data: zeroExtend((nextRequestId)[2:0]), tag: 1, last: True});
	 endaction
      endseq // while True
      endseq);

   Reg#(Bit#(32))                       dataCounter <- mkReg(0);
   FIFOF#(Bit#(32))                  dataLengthFifo <- mkFIFOF();
   let                                     fifoToMp <- mkFIFOF();
   let                                 needleLenReg <- mkReg(0);
   MemReadEngine#(DataBusWidth,DataBusWidth,2,3) re <- mkMemReadEngine();
   MPStreamEngine#(PcieDataBusWidth,DataBusWidth)    mpEngine <- mkMPStreamEngine();
   mkConnection(re.readServers[0].data, mpEngine.needle);
   mkConnection(re.readServers[1].data, mpEngine.mpNext);
   mkConnection(toPipeOut(fifoToMp), mpEngine.haystack);

   rule rl_count_data_to_mp;
      let data <- toGet(splitter.data).get();
      if (dataLengthFifo.notEmpty()) begin
	 data.last = (dataCounter+fromInteger(valueOf(PcieDataBusWidth)/8)) >= dataLengthFifo.first;
	 let md = MemDataF {data: data.data, last: data.last};
	 fifoToMp.enq(md);
      end
      dataCounter <= dataCounter + 1;
   endrule

   let traceMemCnx <- mkConnection(traceClient, splitter.server);
   let bramMemCnx  <- mkConnection(splitter.bramClient, bramMemA);
   rule rl_locdone;
      let loc <- toGet(mpEngine.locdone).get();
      driverInd.strstrLoc(pack(loc));
   endrule

`ifdef NVME_ACCELERATOR_INTERFACE
   FIFOF#(Bit#(32)) msgInFifo <- mkFIFOF();
   FIFOF#(Bit#(32)) msgOutFifo <- mkFIFOF();
   FIFOF#(Bit#(PcieDataBusWidth)) dataOutFifo <- mkFIFOF();
   AxiStreamSlave#(32)                msgInStream <- mkAxiStream(msgInFifo);
   AxiStreamMaster#(32)               msgOutStream <- mkAxiStream(msgOutFifo);
   AxiStreamMaster#(PcieDataBusWidth) dataOutStream <- mkAxiStream(dataOutFifo);
`endif

   interface MemServerPortalRequest bramRequest = bramMemServerPortal.request;
   interface NvmeDriverRequest driverRequest;
      method Action status();
`ifndef PCIE3
	 let mmcmLock = axiRootPort.mmcm.lock();
`else
	 let mmcmLock = True;
`endif
         driverInd.status(mmcmLock, dataCounter);
      endmethod
      method Action trace(Bool enabled);
	 traceEnabled <= enabled;
      endmethod
      method Action read32(Bit#(32) addr);
	 araddrFifo.enq(PhysMemRequest { addr: addr, burstLen: 4, tag: 0 });
      endmethod
      method Action write32(Bit#(32) addr, Bit#(32) value);
	 awaddrFifo.enq(PhysMemRequest { addr: addr, burstLen: 4, tag: 0 });
	 wdataFifo.enq(MemData {data: extend(value), tag: 0, last: True});
      endmethod
      method Action read(Bit#(32) addr);
	 araddrFifo.enq(PhysMemRequest { addr: addr, burstLen: fromInteger(valueOf(TDiv#(DataBusWidth,8))), tag: 0 });
      endmethod
      method Action write(Bit#(32) addr, Bit#(DataBusWidth) value);
	 awaddrFifo.enq(PhysMemRequest { addr: addr, burstLen: fromInteger(valueOf(TDiv#(DataBusWidth,8))), tag: 0 });
	 wdataFifo.enq(MemData {data: extend(value), tag: 0, last: True});
      endmethod

      method Action readCtl(Bit#(32) addr);
	 araddrFifoCtl.enq(PhysMemRequest { addr: addr, burstLen: 4, tag: 0 });
      endmethod
      method Action writeCtl(Bit#(32) addr, Bit#(DataBusWidth) value);
	 awaddrFifoCtl.enq(PhysMemRequest { addr: addr, burstLen: 4, tag: 0 });
	 wdataFifoCtl.enq(MemData {data: truncate(value), tag: 0, last: True});
      endmethod
      method Action setSearchString(Bit#(32) needleSglId, Bit#(32) mpNextSglId, Bit#(32) needleLen);
	 mpEngine.clear();
	 needleLenReg <= needleLen;

	 let burstLen = fromInteger(valueOf(DataBusWidth)/8);
	 let mask = burstLen - 1;
	 needleLen = (needleLen + mask) & ~mask;
	 re.readServers[0].request.put(MemengineCmd {sglId: needleSglId, base: 0, burstLen: burstLen, len: needleLen, tag: 0});
	 re.readServers[1].request.put(MemengineCmd {sglId: mpNextSglId, base: 0, burstLen: burstLen, len: needleLen*4, tag: 0});
      endmethod
      method Action startSearch(Bit#(32) haystackLen);
	 mpEngine.start(needleLenReg);
	 dataLengthFifo.enq(haystackLen);
      endmethod
   endinterface
   interface NvmeRequest request;
      method Action startTransfer(Bit#(8) opcode, Bit#(8) flags, Bit#(16) requestId, Bit#(64) startBlock, Bit#(32) numBlocks, Bit#(32) dsm);
	 ioCommandFifo.enq(NvmeIoCommand{opcode: opcode, flags: flags, requestId: requestId, startBlock: startBlock, numBlocks: numBlocks, dsm: dsm });
      endmethod
   endinterface
`ifndef PCIE3
   interface Clock portalClockSource = axiRootPort.axi.aclk_out;
`else
   interface Clock portalClockSource = axiRootPort.axi.aclk;
`endif
   interface NvmePins pins;
      interface deleteme_unused_clock = clock;
      interface pcie_sys_reset_n = reset;
      interface pcie = axiRootPort.pci;
      method Action pcie_refclk(Bit#(1) p, Bit#(1) n);
         refclk_p.inputclock(p);
         refclk_n.inputclock(n);
      endmethod
`ifdef NVME_ACCELERATOR_INTERFACE
      interface NvmeAccelerator accel;
	 interface msgIn = msgInStream;
	 interface msgOut = msgOutStream;
	 interface dataOut = dataOutStream;
      endinterface
`endif
   endinterface
   interface Vector dmaReadClient = vec(axiGearbox.client.readClient, re.dmaClient);
   interface Vector dmaWriteClient = vec(axiGearbox.client.writeClient);
endmodule

instance ToAxi4SlaveBits#(Axi4SlaveBits#(32,PcieDataBusWidth,4,Empty), AprpS_axi);
   function Axi4SlaveBits#(32,PcieDataBusWidth,4,Empty) toAxi4SlaveBits(AprpS_axi s);
      return (interface Axi4SlaveBits#(32,PcieDataBusWidth,4,Empty);
	 method araddr = compose(s.araddr, extend);
	 method arburst = s.arburst;
	 //method arcache = s.arcache;
	 method arid = s.arid;
	 method arlen = s.arlen;
	 //method arlock = s.arlock;
	 //method arprot = s.arprot;
	 //method arqos = s.arqos;
	 method arready = s.arready;
	 method arsize = s.arsize;
	 method arvalid = s.arvalid;

	 method awaddr = compose(s.awaddr, extend);
	 method awburst = s.awburst;
	 //method awcache = s.awcache;
	 method awid = s.awid;
	 method awlen = s.awlen;
	 //method awlock = s.awlock;
	 //method awprot = s.awprot;
	 //method awqos = s.awqos;
	 method awready = s.awready;
	 method awsize = s.awsize;
	 method awvalid = s.awvalid;

	 method bid = s.bid;
	 method bready = s.bready;
	 method bresp = s.bresp;
	 method bvalid = s.bvalid;
	 method rdata = s.rdata;
	 method rid = s.rid;
	 method rlast = s.rlast;
	 method rready = s.rready;
	 method rresp = s.rresp;
	 method rvalid = s.rvalid;
	 method wdata = s.wdata;
	 method wlast = s.wlast;
	 method wready = s.wready;
	 method wvalid = s.wvalid;
	 method wstrb = s.wstrb;
	 endinterface);
   endfunction
endinstance

instance ToAxi4SlaveBits#(Axi4SlaveLiteBits#(32,32), AprpS_axi_ctl);
   function Axi4SlaveLiteBits#(32,32) toAxi4SlaveBits(AprpS_axi_ctl s);
      return (interface Axi4SlaveLiteBits#(32,32);
	 method araddr = compose(s.araddr, truncate);
	 method arready = s.arready;
	 method arvalid = s.arvalid;

	 method awaddr = compose(s.awaddr, truncate);
	 method awready = s.awready;
	 method awvalid = s.awvalid;

	 method bready = s.bready;
	 method bresp = s.bresp;
	 method bvalid = s.bvalid;
	 method rdata = s.rdata;
	 method rready = s.rready;
	 method rresp = s.rresp;
	 method rvalid = s.rvalid;
	 method wdata = s.wdata;
	 method wready = s.wready;
	 method Action      wvalid(Bit#(1) v);
	    s.wvalid(v);
	    s.wstrb(pack(replicate(v)));
	 endmethod
	 endinterface);
   endfunction
endinstance

instance ToAxi4MasterBits#(Axi4MasterBits#(32,PcieDataBusWidth,tagWidth,Empty), AprpM_axi);
function Axi4MasterBits#(32,PcieDataBusWidth,tagWidth,Empty) toAxi4MasterBits(AprpM_axi m);
   return (interface Axi4MasterBits#(32,PcieDataBusWidth,tagWidth,Empty);
	   method araddr = m.araddr;
	   method arburst = m.arburst;
	   method arcache = m.arcache;
	   method arlen = m.arlen;
	   method arlock = extend(m.arlock);
	   method arready = m.arready;
	   method arsize = m.arsize;
	   method arvalid = m.arvalid;
	   method Bit#(1) aresetn(); return 1; endmethod
	   method Bit#(tagWidth)     arid(); return 0; endmethod
	   method arprot = m.arprot;
	   method arqos = 0;
	   method awaddr = m.awaddr;
	   method awburst = m.awburst;
	   method awcache = m.awcache;
	   method Bit#(tagWidth)     awid(); return 0; endmethod
	   method awlen = m.awlen;
	   method awlock = extend(m.awlock);
	   method awprot = m.awprot;
	   method awready = m.awready;
	   method Bit#(4)     awqos(); return 0; endmethod
	   method awsize = m.awsize;
	   method awvalid = m.awvalid;
	   method Action      bid(Bit#(tagWidth) v); endmethod
	   method bready = m.bready;
	   method bresp = m.bresp;
	   method bvalid = m.bvalid;
	   method rdata = m.rdata;
	   method Action      rid(Bit#(tagWidth) v); endmethod
	   method rlast = m.rlast;
	   method rready = m.rready;
	   method rresp = m.rresp;
	   method rvalid = m.rvalid;
	   method wdata = m.wdata;
	   method Bit#(tagWidth)     wid(); return 0; endmethod
	   method wlast = m.wlast;
	   method wready = m.wready;
	   method wstrb = m.wstrb;
	   method wvalid = m.wvalid;
	 interface extra = ?;
	 endinterface);
   endfunction
endinstance

interface SplitMemServer#(numeric type dataBusWidth);
   interface MemServer#(dataBusWidth) server;
   interface MemClient#(dataBusWidth) busClient;
   interface MemClient#(dataBusWidth) bramClient;
   interface PipeOut#(MemData#(dataBusWidth)) data;
endinterface

module mkSplitMemServer(SplitMemServer#(dataBusWidth));
   let readReqFifo   <- mkFIFOF();
   let readDataFifo  <- mkFIFOF();
   let writeReqFifo  <- mkFIFOF();
   let writeDataFifo <- mkFIFOF();
   let writeDoneFifo <- mkFIFOF();

   let busReadReqFifo   <- mkFIFOF();
   let busReadDataFifo  <- mkSizedBRAMFIFOF(16);
   let busWriteReqFifo  <- mkFIFOF();
   let busWriteDataFifo <- mkSizedBRAMFIFOF(16);
   let busWriteDoneFifo <- mkFIFOF();

   let bramReadReqFifo   <- mkFIFOF();
   let bramReadDataFifo  <- mkSizedBRAMFIFOF(16);
   let bramWriteReqFifo  <- mkFIFOF();
   let bramWriteDataFifo <- mkSizedBRAMFIFOF(16);
   let bramWriteDoneFifo <- mkFIFOF();

   let doneFifo <- mkFIFOF();

   let dataFifo <- mkSizedBRAMFIFOF(4096);
   let readDestFifo <- mkFIFOF();
   let writeDestFifo <- mkFIFOF();

   rule rl_rd_req;
      MemRequest req <- toGet(readReqFifo).get();
      let dest = req.sglId[5:4];
      if (dest == 2)
	 bramReadReqFifo.enq(req);
      else
	 busReadReqFifo.enq(req);
      readDestFifo.enq(dest);
   endrule

   rule rl_rd_data;
      let dest = readDestFifo.first();
      MemData#(dataBusWidth) md;
      if (dest == 2)
	 md <- toGet(bramReadDataFifo).get();
      else
	 md <- toGet(busReadDataFifo).get();
      if (md.last)
	 readDestFifo.deq();
      readDataFifo.enq(md);
   endrule

   rule rl_wr_req;
      MemRequest req <- toGet(writeReqFifo).get();
      Bit#(2) dest = req.sglId[5:4];
      if (dest == 2)
	 bramWriteReqFifo.enq(req);
      else if (dest == 3) begin
	 // no need to send the request
      end else
	 busWriteReqFifo.enq(req);
      writeDestFifo.enq(dest);
   endrule

   rule rl_wr_data;
      let dest = writeDestFifo.first();
      MemData#(dataBusWidth) md <- toGet(writeDataFifo).get();
      if (dest == 2)
	 bramWriteDataFifo.enq(md);
      else if (dest == 3)
	 dataFifo.enq(md);
      else
	 busWriteDataFifo.enq(md);
      if (md.last) begin
	 writeDestFifo.deq();
	 doneFifo.enq(tuple2(dest, md.tag));
      end
   endrule

   rule rl_wr_done;
      match { .dest, .tag } <- toGet(doneFifo).get();
      Bit#(MemTagSize) doneTag;
      if (dest == 2)
	 doneTag <- toGet(bramWriteDoneFifo).get();
      else if (dest == 3)
	 doneTag = 0;
      else
	 doneTag <- toGet(busWriteDoneFifo).get();
      writeDoneFifo.enq(doneTag);
   endrule

   interface MemServer server;
      interface MemReadServer readServer;
	 interface readReq  = toPut(readReqFifo);
	 interface readData = toGet(readDataFifo);
      endinterface
      interface MemWriteServer writeServer;
	 interface writeReq  = toPut(writeReqFifo);
	 interface writeData = toPut(writeDataFifo);
	 interface writeDone = toGet(writeDoneFifo);
      endinterface
   endinterface
   interface MemClient busClient;
      interface MemReadClient readClient;
	 interface readReq  = toGet(busReadReqFifo);
	 interface Put readData;
	    method Action put(MemData#(dataBusWidth) md);
	       busReadDataFifo.enq(md);
	    endmethod
	 endinterface
      endinterface
      interface MemWriteClient writeClient;
	 interface writeReq  = toGet(busWriteReqFifo);
	 interface writeData = toGet(busWriteDataFifo);
	 interface writeDone = toPut(busWriteDoneFifo);
      endinterface
   endinterface
   interface MemClient bramClient;
      interface MemReadClient readClient;
	 interface readReq  = toGet(bramReadReqFifo);
	 interface readData = toPut(bramReadDataFifo);
      endinterface
      interface MemWriteClient writeClient;
	 interface writeReq  = toGet(bramWriteReqFifo);
	 interface writeData = toGet(bramWriteDataFifo);
	 interface writeDone = toPut(bramWriteDoneFifo);
      endinterface
   endinterface
   interface PipeOut data = toPipeOut(dataFifo);
endmodule

interface MemServerGearbox#(numeric type serverDataBusWidth, numeric type clientDataBusWidth);
   interface MemServer#(serverDataBusWidth) server;
   interface MemClient#(clientDataBusWidth) client;
endinterface

module mk1toNMemServerGearbox(MemServerGearbox#(serverDataBusWidth, clientDataBusWidth))
   provisos (Div#(serverDataBusWidth,clientDataBusWidth,k)
	     ,Mul#(clientDataBusWidth,k,serverDataBusWidth)
	     ,Add#(1, a__, TMul#(2, k))
	     ,Add#(k, b__, TMul#(2, k))
	     ,Add#(1, c__, k)
	     );
   let clock <- exposeCurrentClock();
   let reset <- exposeCurrentReset();

   let readReqFifo   <- mkFIFOF();
   let writeReqFifo  <- mkFIFOF();
   let writeDoneFifo <- mkFIFOF();

   Gearbox#(1,k,MemData#(clientDataBusWidth)) readDataGearbox <- mk1toNGearbox(clock, reset, clock, reset);
   Gearbox#(k,1,MemData#(clientDataBusWidth)) writeDataGearbox <- mkNto1Gearbox(clock, reset, clock, reset);

   FIFOF#(MemData#(clientDataBusWidth)) busReadDataFifo  <- mkSizedBRAMFIFOF(16);
   FIFOF#(MemData#(clientDataBusWidth)) busWriteDataFifo <- mkSizedBRAMFIFOF(16);

   rule readDataCnx;
      let md <- toGet(busReadDataFifo).get();
      readDataGearbox.enq(vec(md));
   endrule
   rule writeDataCnx;
      let md = writeDataGearbox.first[0];
      writeDataGearbox.deq();
      busWriteDataFifo.enq(md);
   endrule

   interface MemServer server;
      interface MemReadServer readServer;
	 interface readReq  = toPut(readReqFifo);
	 interface Get readData;
	    method ActionValue#(MemData#(serverDataBusWidth)) get();
	       let mds = readDataGearbox.first();
	       readDataGearbox.deq();
	       let md = MemData { data: pack(map(memDataData, mds)), tag: mds[0].tag, last: mds[valueOf(k)-1].last };
	       return md;
	    endmethod
	 endinterface
      endinterface
      interface MemWriteServer writeServer;
	 interface writeReq  = toPut(writeReqFifo);
	 interface Put writeData;
	    method Action put(MemData#(serverDataBusWidth) md);
	       Vector#(k, Bit#(clientDataBusWidth)) datavec = unpack(md.data);
	       Vector#(k, MemData#(clientDataBusWidth)) mds = newVector;
	       for (Integer i = 0; i < valueOf(k); i = i + 1)
		  mds[i] = MemData { data: datavec[i], tag: md.tag, last: (i == valueOf(k)-1) && md.last };
	       writeDataGearbox.enq(mds);
	    endmethod
	 endinterface
	 interface writeDone = toGet(writeDoneFifo);
      endinterface
   endinterface
   interface MemClient client;
      interface MemReadClient readClient;
	 interface readReq  = toGet(readReqFifo);
	 interface Put readData;
	    method Action put(MemData#(clientDataBusWidth) md);
	       busReadDataFifo.enq(md);
	       //busReadDataData <= md.data;
	       //busReadDataLast <= md.last;
	    endmethod
	 endinterface
      endinterface
      interface MemWriteClient writeClient;
	 interface writeReq  = toGet(writeReqFifo);
	 interface writeData = toGet(busWriteDataFifo);
	 interface writeDone = toPut(writeDoneFifo);
      endinterface
   endinterface

endmodule

interface MemServerPortal;
   interface MemServerPortalRequest request;
endinterface

module mkPhysMemSlavePortal#(PhysMemSlave#(32,dataBusWidth) ms, MemServerPortalIndication ind)(MemServerPortal)
   provisos (Add#(dataBusWidth,7,a__)
	     ,Add#(b__,64,dataBusWidth)
	     ,Bits#(MemTypes::MemData#(dataBusWidth), a__));

   FIFOF#(PhysMemRequest#(32,dataBusWidth)) araddrFifo <- mkFIFOF();
   FIFOF#(PhysMemRequest#(32,dataBusWidth)) awaddrFifo <- mkFIFOF();
   FIFOF#(MemData#(dataBusWidth))           rdataFifo <- mkFIFOF();
   FIFOF#(MemData#(dataBusWidth))           wdataFifo <- mkFIFOF();
   FIFOF#(Bit#(6))                doneFifo <- mkFIFOF();

   let araddrCnx <- mkConnection(toGet(araddrFifo), ms.read_server.readReq);
   let awaddrCnx <- mkConnection(toGet(awaddrFifo), ms.write_server.writeReq);
   let rdataCnx  <- mkConnection(ms.read_server.readData, toPut(rdataFifo));
   let wdataCnx  <- mkConnection(toGet(wdataFifo), ms.write_server.writeData);
   let doneCnx   <- mkConnection(ms.write_server.writeDone, toPut(doneFifo));

   rule rl_rdata;
      let rdata <- toGet(rdataFifo).get();
      ind.readDone(truncate(rdata.data));
   endrule

   rule rl_writeDone;
      let tag <- toGet(doneFifo).get();
      ind.writeDone();
   endrule

   interface MemServerPortalRequest request;
      method Action read(Bit#(32) addr);
	 araddrFifo.enq(PhysMemRequest { addr: addr, burstLen: fromInteger(valueOf(TDiv#(dataBusWidth,8))), tag: 0 });
      endmethod
      method Action write(Bit#(32) addr, Bit#(DataBusWidth) value);
	 awaddrFifo.enq(PhysMemRequest { addr: addr, burstLen: fromInteger(valueOf(TDiv#(dataBusWidth,8))), tag: 0 });
	 wdataFifo.enq(MemData {data: extend(value), tag: 0, last: True});
      endmethod
   endinterface
endmodule

// Copyright (c) 2016 Connectal Project

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import Arbitrate::*;
import BRAM::*;
import BuildVector::*;
import Clocks::*;
import Connectable::*;
import FIFOF::*;
import Gearbox::*;
import GetPut::*;
import Probe::*;
import StmtFSM::*;
import Vector::*;

import AddressGenerator::*;
import AxiBits::*;
import AxiStream::*;
import ConnectalClocks::*;
import ConnectalConfig::*;
import DefaultValue::*;
import GearboxGetPut::*;
import HostInterface::*;
import MemReadEngine::*;
import MemTypes::*;
import PhysMemSlaveFromBram::*;
import Pipe::*;
import TraceMemClient::*;
import XilinxCells::*;
import MPEngine::*;

`ifndef PCIE3
import AxiPcieRootPort::*;
`else
import AxiPcie3RootPort::*;
`endif
import NvmeIfc::*;
import NvmePins::*;

typedef struct {
   Bit#(8)  opcode;
   Bit#(8)  flags;
   Bit#(16) requestId;
   Bit#(64) startBlock;
   Bit#(32) numBlocks;
   Bit#(32) dsm;
   } NvmeIoCommand deriving (Bits);

`ifndef TOP_SOURCES_PORTAL_CLOCK
import ConnectalBramFifo::*;
`else
import BRAMFIFO::*;
module mkDualClockBramFIFOF#(Clock clock1, Reset reset1, Clock clock2, Reset reset2)(FIFOF#(a))
   provisos (Bits#(a, asz), Add#(1, a__, asz));
   FIFOF#(a) fifo <- mkSizedBRAMFIFOF(512, clocked_by clock1, reset_by reset1);
   return fifo;
endmodule
`endif

function PipeIn#(t) sinkPipe();
   return (interface PipeIn#(a);
	      method Action enq(t v); endmethod
	      method Bool notFull(); return False; endmethod
	   endinterface);
endfunction

instance ArbRequestTC#(BRAMRequest#(a,b));
   function Bool isReadRequest(BRAMRequest#(a,b) x); return !x.write; endfunction
   function Bool isWriteRequest(BRAMRequest#(a,b) x); return x.write; endfunction
endinstance

interface Nvme;
   interface NvmeRequest request;
   interface NvmeDriverRequest driverRequest;
   interface MemServerPortalRequest bramRequest;
   interface NvmeTrace trace;
   interface NvmePins pins;
   interface Vector#(2, MemReadClient#(DataBusWidth)) dmaReadClient;
   interface Vector#(1, MemWriteClient#(DataBusWidth)) dmaWriteClient;
`ifdef TOP_SOURCES_PORTAL_CLOCK
   interface Clock portalClockSource;
`endif
endinterface

module mkNvme#(NvmeIndication ind, NvmeDriverIndication driverInd, NvmeTrace trace, MemServerPortalIndication bramIndication)(Nvme);
   let clock <- exposeCurrentClock;
   let reset <- exposeCurrentReset;
   let refclk_p <- mkB2C1();
   let refclk_n <- mkB2C1();
   let pcie_clk_100mhz_buf <- mkClockIBUFDS_GTE2(
`ifdef ClockDefaultParam
       defaultValue,
`endif
      True, refclk_p.c, refclk_n.c);
`ifndef TOP_SOURCES_PORTAL_CLOCK
   let axiClockB2C    <- mkB2C1();
   let axiCtlClockB2C <- mkB2C1();
   let axiClock = axiClockB2C.c;
   let axiCtlClock = axiCtlClockB2C.c;
   let axiReset <- mkSyncReset(10, reset, axiClock);
   let axiCtlReset <- mkSyncReset(10, reset, axiCtlClock);
`else
   let axiClock = clock;
   let axiCtlClock = clock;
   let axiReset = reset;
   let axiCtlReset = reset;
`endif

`ifndef PCIE3
   let axiRootPort <- mkAPRP(pcie_clk_100mhz_buf, reset, axiClock, axiReset, axiCtlClock, axiCtlReset);
`else
   let axiRootPort <- mkAPRP(axiCtlClock, pcie_clk_100mhz_buf, reset);
`endif
`ifndef TOP_SOURCES_PORTAL_CLOCK
   let axiClockC2B <- mkC2B(axiRootPort.axi.aclk_out);
   let axiCtlClockC2B <- mkC2B(axiRootPort.axi.ctl_aclk_out);
   rule rl_connect_clocks;
      axiClockB2C.inputclock(axiClockC2B.o);
      axiCtlClockB2C.inputclock(axiClockC2B.o);
   endrule
`endif

   FIFOF#(Bit#(32)) dfifoCtl <- mkFIFOF();
   Axi4SlaveBits#(32,PcieDataBusWidth,4,Empty) axiRootPortSlave = toAxi4SlaveBits(axiRootPort.s_axi);
   Axi4SlaveLiteBits#(32,32)                axiRootPortSlaveCtl = toAxi4SlaveBits(axiRootPort.s_axi_ctl);
   PhysMemSlave#(32,PcieDataBusWidth)       axiRootPortMemSlave <- mkPhysMemSlave(axiRootPortSlave, clocked_by axiClock, reset_by axiReset);
   PhysMemSlave#(32,32)                  axiRootPortMemSlaveCtl <- mkPhysMemSlave(axiRootPortSlaveCtl, clocked_by axiCtlClock, reset_by axiCtlReset);

   FIFOF#(PhysMemRequest#(32,PcieDataBusWidth)) araddrFifo <- mkFIFOF();
   FIFOF#(PhysMemRequest#(32,PcieDataBusWidth)) awaddrFifo <- mkFIFOF();
   FIFOF#(MemData#(PcieDataBusWidth))           rdataFifo <- mkFIFOF();
   FIFOF#(MemData#(PcieDataBusWidth))           wdataFifo <- mkFIFOF();
   FIFOF#(Bit#(6))                           doneFifo <- mkFIFOF();

   let araddrCnx <- mkConnection(toGet(araddrFifo), axiRootPortMemSlave.read_server.readReq);
   let awaddrCnx <- mkConnection(toGet(awaddrFifo), axiRootPortMemSlave.write_server.writeReq);
   let rdataCnx  <- mkConnection(axiRootPortMemSlave.read_server.readData, toPut(rdataFifo));
   let wdataCnx  <- mkConnection(toGet(wdataFifo), axiRootPortMemSlave.write_server.writeData);
   let doneCnx   <- mkConnection(axiRootPortMemSlave.write_server.writeDone, toPut(doneFifo));

   rule rl_rdata;
      let rdata <- toGet(rdataFifo).get();
      if (rdata.tag == 0)
	 driverInd.readDone(truncate(rdata.data));
   endrule

   rule rl_writeDone;
      let tag <- toGet(doneFifo).get();
      if (tag == 0)
	 driverInd.writeDone();
   endrule

   FIFOF#(PhysMemRequest#(32,32)) araddrFifoCtl <- mkFIFOF();
   FIFOF#(PhysMemRequest#(32,32)) awaddrFifoCtl <- mkFIFOF();
   FIFOF#(MemData#(32))           rdataFifoCtl <- mkFIFOF();
   FIFOF#(MemData#(32))           wdataFifoCtl <- mkFIFOF();
   FIFOF#(Bit#(6))                doneFifoCtl <- mkFIFOF();

   let araddrCtlCnx <- mkConnection(toGet(araddrFifoCtl), axiRootPortMemSlaveCtl.read_server.readReq);
   let awaddrCtlCnx <- mkConnection(toGet(awaddrFifoCtl), axiRootPortMemSlaveCtl.write_server.writeReq);
   let rdataCtlCnx  <- mkConnection(axiRootPortMemSlaveCtl.read_server.readData, toPut(rdataFifoCtl));
   let wdataCtlCnx  <- mkConnection(toGet(wdataFifoCtl), axiRootPortMemSlaveCtl.write_server.writeData);
   let doneCtlCnx   <- mkConnection(axiRootPortMemSlaveCtl.write_server.writeDone, toPut(doneFifoCtl));

   rule rl_rdata_ctl;
      let rdata <- toGet(rdataFifoCtl).get();
      driverInd.readDone(extend(rdata.data));
   endrule

   rule rl_writeDone_ctl;
      let tag <- toGet(doneFifoCtl).get();
      driverInd.writeDone();
   endrule

   Axi4MasterBits#(32,PcieDataBusWidth,MemTagSize,Empty) m_axi_mm = toAxi4MasterBits(axiRootPort.m_axi);
   let getObjId = (interface GetObjId;
		   method SGLId objId(Bit#(32) addr); return extend(addr[31:24]); endmethod
		   method Bit#(MemOffsetSize) addr(Bit#(32) axiAddr); return extend(axiAddr[23:0]); endmethod
		   endinterface);
   let memReadClient  <- mkMemReadClient(getObjId, m_axi_mm);
   let memWriteClient <- mkMemWriteClient(getObjId, m_axi_mm);

   Reg#(Bool) traceEnabled <- mkReg(False);

   FIFOF#(Tuple4#(DmaChannel,Bool,MemRequest,Bit#(32))) traceFifo <- mkSizedBRAMFIFOF(128);
   PipeIn#(Tuple4#(DmaChannel,Bool,MemRequest,Bit#(32))) tracePipe = traceEnabled ? toPipeIn(traceFifo) : sinkPipe();
   FIFOF#(Tuple4#(DmaChannel,Bool,MemData#(PcieDataBusWidth),Bit#(32))) traceDataFifo <- mkSizedBRAMFIFOF(2048);
   PipeIn#(Tuple4#(DmaChannel,Bool,MemData#(PcieDataBusWidth),Bit#(32))) traceDataPipe = traceEnabled ? toPipeIn(traceDataFifo) : sinkPipe();
   FIFOF#(Tuple2#(DmaChannel,Bit#(32)))                              traceDoneFifo <- mkSizedBRAMFIFOF(128);
   PipeIn#(Tuple2#(DmaChannel,Bit#(32)))                             traceDonePipe = traceEnabled ? toPipeIn(traceDoneFifo) : sinkPipe();

   rule rl_trace1;
      match { .chan, .write, .req, .timestamp } <- toGet(traceFifo).get();
      trace.traceDmaRequest(chan, write, truncate(req.sglId), extend(req.offset), extend(req.burstLen), extend(req.tag), timestamp);
   endrule
   rule rl_trace_data;
      match { .chan, .write, .md, .timestamp } <- toGet(traceDataFifo).get();
      trace.traceDmaData(chan, write, unpack(md.data), md.last, extend(md.tag), timestamp);
   endrule
   rule rl_trace_done;
      match { .chan, .timestamp } <- toGet(traceDoneFifo).get();
      trace.traceDmaDone(chan, 0, timestamp);
   endrule

   let traceReadClient <- mkTraceReadClient(tracePipe,traceDataPipe,DMA_TX,memReadClient);
   let traceWriteClient <- mkTraceWriteClient(tracePipe,traceDataPipe,traceDonePipe,DMA_RX,memWriteClient);
   let traceClient = (interface MemClient#(PcieDataBusWidth);
			 interface readClient = traceReadClient;
			 interface writeClient = traceWriteClient;
		      endinterface);

   SplitMemServer#(PcieDataBusWidth) splitter <- mkSplitMemServer();
   MemServerGearbox#(PcieDataBusWidth,DataBusWidth) axiGearbox <- mk1toNMemServerGearbox();
   let gearboxCnx <- mkConnection(splitter.busClient, axiGearbox.server);

   let requestSize = 64;
   let responseSize = 16;
   let bytesPerEntry = valueOf(TDiv#(PcieDataBusWidth,8));
   let wordsPerEntry = valueOf(TDiv#(PcieDataBusWidth,32));

   BRAM_Configure bramConfig = defaultValue;
   bramConfig.latency = 2;
   bramConfig.memorySize = 4096*4/bytesPerEntry; // 4 pages
   BRAM2Port#(Bit#(32),Bit#(PcieDataBusWidth)) bram <- mkBRAM2Server(bramConfig);

   let arbIfc <- mkFixedPriority();
   Arbiter#(3, BRAMRequest#(Bit#(32),Bit#(PcieDataBusWidth)),Bit#(PcieDataBusWidth)) arbiter <- mkArbiter(arbIfc,2);
   let arbCnx <- mkConnection(arbiter.master, bram.portB);

   MemServer#(PcieDataBusWidth)            bramMemA <- mkMemServerFromBram(bram.portA);
   PhysMemSlave#(32,PcieDataBusWidth)      bramMemB <- mkPhysMemSlaveFromBram(arbiter.users[0]);
   MemServerPortal          bramMemServerPortal <- mkPhysMemSlavePortal(bramMemB,bramIndication);

   let portB1 = arbiter.users[1];
   let portB2 = arbiter.users[2];
   Reg#(Bit#(16)) requestId <- mkReg(0);
   Reg#(Bit#(16)) responseId <- mkReg(0);
   Reg#(Bit#(16)) requestQueueEntryCount <- mkReg(8);
   Reg#(Bool) requestSlotAvailable <- mkReg(True);
   Reg#(Bool) requestInProgress <- mkReg(True);
   Reg#(Vector#(16,Bit#(32))) ioCommand <- mkReg(unpack(0));
   Reg#(Bit#(32)) requestStartTimestamp <- mkReg(0);

   FIFOF#(NvmeIoCommand) ioCommandFifo <- mkFIFOF();

   Reg#(Bit#(32)) cycles <- mkReg(0);
   rule rl_cycles;
      cycles <= cycles + 1;
   endrule

   Reg#(Bit#(32)) i <- mkReg(0);
   Reg#(Bit#(1)) phase <- mkReg(1);

   let bramQueryAddress <- mkProbe();
   let bramQueryResponse <- mkProbe();
   let requestCompleted <- mkProbe();

   let requestFsm <- mkAutoFSM(seq
      while (True) seq
	 action
	    let req <- toGet(ioCommandFifo).get();
	    Vector#(16,Bit#(32)) command = unpack(0);
	    command[0] = { req.requestId, req.flags, req.opcode };
	    command[1] = 1; // nsid
	    // prp1: send data to fifo
	    command[6] = 32'h30000000;
	    // p2p2: read PRP list from BRAM at offset 0x2000
	    command[8] = 32'h20002000;
	    command[10] = req.startBlock[31:0];
	    command[11] = req.startBlock[63:32];
	    command[12] = req.numBlocks-1;
	    command[13] = req.dsm;
	    requestId <= req.requestId;
	    ioCommand <= command;

	    requestSlotAvailable <= False;
	 endaction

	 while (!requestSlotAvailable) seq // wait for open request slot
	    portB1.request.put(BRAMRequest {address: (extend(requestId[2:0]) * fromInteger(responseSize/bytesPerEntry)) + fromInteger(12/bytesPerEntry),
					    write: False, responseOnWrite: False, datain: 0});
	    action
	       let response <- portB1.response.get();
	       if (response[16+(3*32 % valueOf(PcieDataBusWidth))] == ~phase) begin
		  requestSlotAvailable <= True;
	       end
	    endaction
	 endseq // wait for open request slot

	 // write a IO read request to BRAM
         for (i <= 0; i < fromInteger(requestSize/bytesPerEntry); i <= i + 1) seq
	    portB1.request.put(BRAMRequest {address: 'h1000/fromInteger(bytesPerEntry) + (extend(requestId[2:0]) * fromInteger(requestSize/bytesPerEntry)) + i,
					    write: True, responseOnWrite: False,
					    datain: truncate(pack(ioCommand) >> (i*fromInteger(valueOf(PcieDataBusWidth)))) });
         endseq

	 // tell the NVME the new submission queue tail pointer
	 action
	    awaddrFifo.enq(PhysMemRequest { addr: 'h1000 + (2*2+0)*(4<<0), burstLen: 4, tag: 1 });
	    wdataFifo.enq(MemData{data: zeroExtend((requestId+1)[2:0]), tag: 1, last: True});
	    requestInProgress <= True;
	    requestStartTimestamp <= cycles;
	    trace.traceData(unpack(0), False, truncate(requestId), cycles);
	 endaction
      endseq // while True
      endseq);

   let responseFsm <- mkAutoFSM(seq
	 // wait for response queue to be updated
      while (True) seq
	 requestInProgress <= True;
	 while (requestInProgress) seq
	    portB2.request.put(BRAMRequest {address: (extend(responseId[2:0]) * fromInteger(responseSize/bytesPerEntry)) + fromInteger(12/bytesPerEntry),
					    write: False, responseOnWrite: False, datain: 0});
	    bramQueryAddress <= (responseId * fromInteger(responseSize/bytesPerEntry)) + fromInteger(12/bytesPerEntry);
	    action
	       let response <- portB2.response.get();
	       bramQueryResponse <= response;
	       if (response[16+(3*32 % valueOf(PcieDataBusWidth))] == phase) begin
		  requestCompleted <= response;
		  // if status field written by NVME
		  ind.transferCompleted(responseId, truncate(response), cycles - requestStartTimestamp);
		  requestInProgress <= False;
	       end
	    endaction
	 endseq // while requestInProgress

	 // tell the NVME the new response queue head pointer
	 action
	    let nextRequestId = (responseId + 1);
	    if (nextRequestId[2:0] == 0) begin
	       phase <= ~phase;
	    end
	    responseId <= nextRequestId;
	    awaddrFifo.enq(PhysMemRequest { addr: 'h1000 + (2*2+1)*(4<<0), burstLen: 4, tag: 1 });
	    wdataFifo.enq(MemData{data: zeroExtend((nextRequestId)[2:0]), tag: 1, last: True});
	 endaction
      endseq // while True
      endseq);

   Reg#(Bit#(32))                       dataCounter <- mkReg(0);
   FIFOF#(Bit#(32))                  dataLengthFifo <- mkFIFOF();
   let                                     fifoToMp <- mkFIFOF();
   MemReadEngine#(DataBusWidth,DataBusWidth,2,1) re <- mkMemReadEngine();
   MPEngine#(PcieDataBusWidth,DataBusWidth)    mpEngine <- mkMPStreamEngine(toPipeOut(fifoToMp), re.readServers[0]);

   rule rl_count_data_to_mp;
      let data <- toGet(splitter.data).get();
      if (dataLengthFifo.notEmpty()) begin
	 data.last = (dataCounter+fromInteger(valueOf(PcieDataBusWidth)/8)) >= dataLengthFifo.first;
	 fifoToMp.enq(data);
      end
      dataCounter <= dataCounter + 1;
   endrule

   let traceMemCnx <- mkConnection(traceClient, splitter.server);
   let bramMemCnx  <- mkConnection(splitter.bramClient, bramMemA);
   rule rl_locdone;
      let loc <- toGet(mpEngine.locdone).get();
      driverInd.strstrLoc(pack(loc));
   endrule

`ifdef NVME_ACCELERATOR_INTERFACE
   FIFOF#(Bit#(32)) msgInFifo <- mkFIFOF();
   FIFOF#(Bit#(32)) msgOutFifo <- mkFIFOF();
   FIFOF#(Bit#(PcieDataBusWidth)) dataOutFifo <- mkFIFOF();
   AxiStreamSlave#(32)                msgInStream <- mkAxiStream(msgInFifo);
   AxiStreamMaster#(32)               msgOutStream <- mkAxiStream(msgOutFifo);
   AxiStreamMaster#(PcieDataBusWidth) dataOutStream <- mkAxiStream(dataOutFifo);
`endif

   interface MemServerPortalRequest bramRequest = bramMemServerPortal.request;
   interface NvmeDriverRequest driverRequest;
      method Action status();
`ifndef PCIE3
	 let mmcmLock = axiRootPort.mmcm.lock();
`else
	 let mmcmLock = True;
`endif
         driverInd.status(mmcmLock, dataCounter);
      endmethod
      method Action trace(Bool enabled);
	 traceEnabled <= enabled;
      endmethod
      method Action read32(Bit#(32) addr);
	 araddrFifo.enq(PhysMemRequest { addr: addr, burstLen: 4, tag: 0 });
      endmethod
      method Action write32(Bit#(32) addr, Bit#(32) value);
	 awaddrFifo.enq(PhysMemRequest { addr: addr, burstLen: 4, tag: 0 });
	 wdataFifo.enq(MemData {data: extend(value), tag: 0, last: True});
      endmethod
      method Action read(Bit#(32) addr);
	 araddrFifo.enq(PhysMemRequest { addr: addr, burstLen: fromInteger(valueOf(TDiv#(DataBusWidth,8))), tag: 0 });
      endmethod
      method Action write(Bit#(32) addr, Bit#(DataBusWidth) value);
	 awaddrFifo.enq(PhysMemRequest { addr: addr, burstLen: fromInteger(valueOf(TDiv#(DataBusWidth,8))), tag: 0 });
	 wdataFifo.enq(MemData {data: extend(value), tag: 0, last: True});
      endmethod

      method Action readCtl(Bit#(32) addr);
	 araddrFifoCtl.enq(PhysMemRequest { addr: addr, burstLen: 4, tag: 0 });
      endmethod
      method Action writeCtl(Bit#(32) addr, Bit#(DataBusWidth) value);
	 awaddrFifoCtl.enq(PhysMemRequest { addr: addr, burstLen: 4, tag: 0 });
	 wdataFifoCtl.enq(MemData {data: truncate(value), tag: 0, last: True});
      endmethod
      method Action setSearchString(Bit#(32) needleSglId, Bit#(32) mpNextSglId, Bit#(32) needleLen);
	 mpEngine.setsearch.enq(tuple3(needleSglId, mpNextSglId, needleLen));
      endmethod
      method Action startSearch(Bit#(32) haystackLen);
	 dataLengthFifo.enq(haystackLen);
	 mpEngine.setsearch.enq(tuple3(/* unused */maxBound, haystackLen, /* unused */0));
      endmethod
   endinterface
   interface NvmeRequest request;
      method Action startTransfer(Bit#(8) opcode, Bit#(8) flags, Bit#(16) requestId, Bit#(64) startBlock, Bit#(32) numBlocks, Bit#(32) dsm);
	 ioCommandFifo.enq(NvmeIoCommand{opcode: opcode, flags: flags, requestId: requestId, startBlock: startBlock, numBlocks: numBlocks, dsm: dsm });
      endmethod
   endinterface
`ifndef PCIE3
   interface Clock portalClockSource = axiRootPort.axi.aclk_out;
`else
   interface Clock portalClockSource = axiRootPort.axi.aclk;
`endif
   interface NvmePins pins;
      interface deleteme_unused_clock = clock;
      interface pcie_sys_reset_n = reset;
      interface pcie = axiRootPort.pci;
      method Action pcie_refclk(Bit#(1) p, Bit#(1) n);
         refclk_p.inputclock(p);
         refclk_n.inputclock(n);
      endmethod
`ifdef NVME_ACCELERATOR_INTERFACE
      interface NvmeAccelerator accel;
	 interface msgIn = msgInStream;
	 interface msgOut = msgOutStream;
	 interface dataOut = dataOutStream;
      endinterface
`endif
   endinterface
   interface Vector dmaReadClient = vec(axiGearbox.client.readClient, re.dmaClient);
   interface Vector dmaWriteClient = vec(axiGearbox.client.writeClient);
endmodule

instance ToAxi4SlaveBits#(Axi4SlaveBits#(32,PcieDataBusWidth,4,Empty), AprpS_axi);
   function Axi4SlaveBits#(32,PcieDataBusWidth,4,Empty) toAxi4SlaveBits(AprpS_axi s);
      return (interface Axi4SlaveBits#(32,PcieDataBusWidth,4,Empty);
	 method araddr = compose(s.araddr, extend);
	 method arburst = s.arburst;
	 //method arcache = s.arcache;
	 method arid = s.arid;
	 method arlen = s.arlen;
	 //method arlock = s.arlock;
	 //method arprot = s.arprot;
	 //method arqos = s.arqos;
	 method arready = s.arready;
	 method arsize = s.arsize;
	 method arvalid = s.arvalid;

	 method awaddr = compose(s.awaddr, extend);
	 method awburst = s.awburst;
	 //method awcache = s.awcache;
	 method awid = s.awid;
	 method awlen = s.awlen;
	 //method awlock = s.awlock;
	 //method awprot = s.awprot;
	 //method awqos = s.awqos;
	 method awready = s.awready;
	 method awsize = s.awsize;
	 method awvalid = s.awvalid;

	 method bid = s.bid;
	 method bready = s.bready;
	 method bresp = s.bresp;
	 method bvalid = s.bvalid;
	 method rdata = s.rdata;
	 method rid = s.rid;
	 method rlast = s.rlast;
	 method rready = s.rready;
	 method rresp = s.rresp;
	 method rvalid = s.rvalid;
	 method wdata = s.wdata;
	 method wlast = s.wlast;
	 method wready = s.wready;
	 method wvalid = s.wvalid;
	 method wstrb = s.wstrb;
	 endinterface);
   endfunction
endinstance

instance ToAxi4SlaveBits#(Axi4SlaveLiteBits#(32,32), AprpS_axi_ctl);
   function Axi4SlaveLiteBits#(32,32) toAxi4SlaveBits(AprpS_axi_ctl s);
      return (interface Axi4SlaveLiteBits#(32,32);
	 method araddr = compose(s.araddr, truncate);
	 method arready = s.arready;
	 method arvalid = s.arvalid;

	 method awaddr = compose(s.awaddr, truncate);
	 method awready = s.awready;
	 method awvalid = s.awvalid;

	 method bready = s.bready;
	 method bresp = s.bresp;
	 method bvalid = s.bvalid;
	 method rdata = s.rdata;
	 method rready = s.rready;
	 method rresp = s.rresp;
	 method rvalid = s.rvalid;
	 method wdata = s.wdata;
	 method wready = s.wready;
	 method Action      wvalid(Bit#(1) v);
	    s.wvalid(v);
	    s.wstrb(pack(replicate(v)));
	 endmethod
	 endinterface);
   endfunction
endinstance

instance ToAxi4MasterBits#(Axi4MasterBits#(32,PcieDataBusWidth,tagWidth,Empty), AprpM_axi);
function Axi4MasterBits#(32,PcieDataBusWidth,tagWidth,Empty) toAxi4MasterBits(AprpM_axi m);
   return (interface Axi4MasterBits#(32,PcieDataBusWidth,tagWidth,Empty);
	   method araddr = m.araddr;
	   method arburst = m.arburst;
	   method arcache = m.arcache;
	   method arlen = m.arlen;
	   method arlock = extend(m.arlock);
	   method arready = m.arready;
	   method arsize = m.arsize;
	   method arvalid = m.arvalid;
	   method Bit#(1) aresetn(); return 1; endmethod
	   method Bit#(tagWidth)     arid(); return 0; endmethod
	   method arprot = m.arprot;
	   method arqos = 0;
	   method awaddr = m.awaddr;
	   method awburst = m.awburst;
	   method awcache = m.awcache;
	   method Bit#(tagWidth)     awid(); return 0; endmethod
	   method awlen = m.awlen;
	   method awlock = extend(m.awlock);
	   method awprot = m.awprot;
	   method awready = m.awready;
	   method Bit#(4)     awqos(); return 0; endmethod
	   method awsize = m.awsize;
	   method awvalid = m.awvalid;
	   method Action      bid(Bit#(tagWidth) v); endmethod
	   method bready = m.bready;
	   method bresp = m.bresp;
	   method bvalid = m.bvalid;
	   method rdata = m.rdata;
	   method Action      rid(Bit#(tagWidth) v); endmethod
	   method rlast = m.rlast;
	   method rready = m.rready;
	   method rresp = m.rresp;
	   method rvalid = m.rvalid;
	   method wdata = m.wdata;
	   method Bit#(tagWidth)     wid(); return 0; endmethod
	   method wlast = m.wlast;
	   method wready = m.wready;
	   method wstrb = m.wstrb;
	   method wvalid = m.wvalid;
	 interface extra = ?;
	 endinterface);
   endfunction
endinstance

interface SplitMemServer#(numeric type dataBusWidth);
   interface MemServer#(dataBusWidth) server;
   interface MemClient#(dataBusWidth) busClient;
   interface MemClient#(dataBusWidth) bramClient;
   interface PipeOut#(MemData#(dataBusWidth)) data;
endinterface

module mkSplitMemServer(SplitMemServer#(dataBusWidth));
   let readReqFifo   <- mkFIFOF();
   let readDataFifo  <- mkFIFOF();
   let writeReqFifo  <- mkFIFOF();
   let writeDataFifo <- mkFIFOF();
   let writeDoneFifo <- mkFIFOF();

   let busReadReqFifo   <- mkFIFOF();
   let busReadDataFifo  <- mkSizedBRAMFIFOF(16);
   let busWriteReqFifo  <- mkFIFOF();
   let busWriteDataFifo <- mkSizedBRAMFIFOF(16);
   let busWriteDoneFifo <- mkFIFOF();

   let bramReadReqFifo   <- mkFIFOF();
   let bramReadDataFifo  <- mkSizedBRAMFIFOF(16);
   let bramWriteReqFifo  <- mkFIFOF();
   let bramWriteDataFifo <- mkSizedBRAMFIFOF(16);
   let bramWriteDoneFifo <- mkFIFOF();

   let doneFifo <- mkFIFOF();

   let dataFifo <- mkSizedBRAMFIFOF(4096);
   let readDestFifo <- mkFIFOF();
   let writeDestFifo <- mkFIFOF();

   rule rl_rd_req;
      MemRequest req <- toGet(readReqFifo).get();
      let dest = req.sglId[5:4];
      if (dest == 2)
	 bramReadReqFifo.enq(req);
      else
	 busReadReqFifo.enq(req);
      readDestFifo.enq(dest);
   endrule

   rule rl_rd_data;
      let dest = readDestFifo.first();
      MemData#(dataBusWidth) md;
      if (dest == 2)
	 md <- toGet(bramReadDataFifo).get();
      else
	 md <- toGet(busReadDataFifo).get();
      if (md.last)
	 readDestFifo.deq();
      readDataFifo.enq(md);
   endrule

   rule rl_wr_req;
      MemRequest req <- toGet(writeReqFifo).get();
      Bit#(2) dest = req.sglId[5:4];
      if (dest == 2)
	 bramWriteReqFifo.enq(req);
      else if (dest == 3) begin
	 // no need to send the request
      end else
	 busWriteReqFifo.enq(req);
      writeDestFifo.enq(dest);
   endrule

   rule rl_wr_data;
      let dest = writeDestFifo.first();
      MemData#(dataBusWidth) md <- toGet(writeDataFifo).get();
      if (dest == 2)
	 bramWriteDataFifo.enq(md);
      else if (dest == 3)
	 dataFifo.enq(md);
      else
	 busWriteDataFifo.enq(md);
      if (md.last) begin
	 writeDestFifo.deq();
	 doneFifo.enq(tuple2(dest, md.tag));
      end
   endrule

   rule rl_wr_done;
      match { .dest, .tag } <- toGet(doneFifo).get();
      Bit#(MemTagSize) doneTag;
      if (dest == 2)
	 doneTag <- toGet(bramWriteDoneFifo).get();
      else if (dest == 3)
	 doneTag = 0;
      else
	 doneTag <- toGet(busWriteDoneFifo).get();
      writeDoneFifo.enq(doneTag);
   endrule

   interface MemServer server;
      interface MemReadServer readServer;
	 interface readReq  = toPut(readReqFifo);
	 interface readData = toGet(readDataFifo);
      endinterface
      interface MemWriteServer writeServer;
	 interface writeReq  = toPut(writeReqFifo);
	 interface writeData = toPut(writeDataFifo);
	 interface writeDone = toGet(writeDoneFifo);
      endinterface
   endinterface
   interface MemClient busClient;
      interface MemReadClient readClient;
	 interface readReq  = toGet(busReadReqFifo);
	 interface Put readData;
	    method Action put(MemData#(dataBusWidth) md);
	       busReadDataFifo.enq(md);
	    endmethod
	 endinterface
      endinterface
      interface MemWriteClient writeClient;
	 interface writeReq  = toGet(busWriteReqFifo);
	 interface writeData = toGet(busWriteDataFifo);
	 interface writeDone = toPut(busWriteDoneFifo);
      endinterface
   endinterface
   interface MemClient bramClient;
      interface MemReadClient readClient;
	 interface readReq  = toGet(bramReadReqFifo);
	 interface readData = toPut(bramReadDataFifo);
      endinterface
      interface MemWriteClient writeClient;
	 interface writeReq  = toGet(bramWriteReqFifo);
	 interface writeData = toGet(bramWriteDataFifo);
	 interface writeDone = toPut(bramWriteDoneFifo);
      endinterface
   endinterface
   interface PipeOut data = toPipeOut(dataFifo);
endmodule

interface MemServerGearbox#(numeric type serverDataBusWidth, numeric type clientDataBusWidth);
   interface MemServer#(serverDataBusWidth) server;
   interface MemClient#(clientDataBusWidth) client;
endinterface

module mk1toNMemServerGearbox(MemServerGearbox#(serverDataBusWidth, clientDataBusWidth))
   provisos (Div#(serverDataBusWidth,clientDataBusWidth,k)
	     ,Mul#(clientDataBusWidth,k,serverDataBusWidth)
	     ,Add#(1, a__, TMul#(2, k))
	     ,Add#(k, b__, TMul#(2, k))
	     ,Add#(1, c__, k)
	     );
   let clock <- exposeCurrentClock();
   let reset <- exposeCurrentReset();

   let readReqFifo   <- mkFIFOF();
   let writeReqFifo  <- mkFIFOF();
   let writeDoneFifo <- mkFIFOF();

   Gearbox#(1,k,MemData#(clientDataBusWidth)) readDataGearbox <- mk1toNGearbox(clock, reset, clock, reset);
   Gearbox#(k,1,MemData#(clientDataBusWidth)) writeDataGearbox <- mkNto1Gearbox(clock, reset, clock, reset);

   FIFOF#(MemData#(clientDataBusWidth)) busReadDataFifo  <- mkSizedBRAMFIFOF(16);
   FIFOF#(MemData#(clientDataBusWidth)) busWriteDataFifo <- mkSizedBRAMFIFOF(16);

   rule readDataCnx;
      let md <- toGet(busReadDataFifo).get();
      readDataGearbox.enq(vec(md));
   endrule
   rule writeDataCnx;
      let md = writeDataGearbox.first[0];
      writeDataGearbox.deq();
      busWriteDataFifo.enq(md);
   endrule

   interface MemServer server;
      interface MemReadServer readServer;
	 interface readReq  = toPut(readReqFifo);
	 interface Get readData;
	    method ActionValue#(MemData#(serverDataBusWidth)) get();
	       let mds = readDataGearbox.first();
	       readDataGearbox.deq();
	       let md = MemData { data: pack(map(memDataData, mds)), tag: mds[0].tag, last: mds[valueOf(k)-1].last };
	       return md;
	    endmethod
	 endinterface
      endinterface
      interface MemWriteServer writeServer;
	 interface writeReq  = toPut(writeReqFifo);
	 interface Put writeData;
	    method Action put(MemData#(serverDataBusWidth) md);
	       Vector#(k, Bit#(clientDataBusWidth)) datavec = unpack(md.data);
	       Vector#(k, MemData#(clientDataBusWidth)) mds = newVector;
	       for (Integer i = 0; i < valueOf(k); i = i + 1)
		  mds[i] = MemData { data: datavec[i], tag: md.tag, last: (i == valueOf(k)-1) && md.last };
	       writeDataGearbox.enq(mds);
	    endmethod
	 endinterface
	 interface writeDone = toGet(writeDoneFifo);
      endinterface
   endinterface
   interface MemClient client;
      interface MemReadClient readClient;
	 interface readReq  = toGet(readReqFifo);
	 interface Put readData;
	    method Action put(MemData#(clientDataBusWidth) md);
	       busReadDataFifo.enq(md);
	       //busReadDataData <= md.data;
	       //busReadDataLast <= md.last;
	    endmethod
	 endinterface
      endinterface
      interface MemWriteClient writeClient;
	 interface writeReq  = toGet(writeReqFifo);
	 interface writeData = toGet(busWriteDataFifo);
	 interface writeDone = toPut(writeDoneFifo);
      endinterface
   endinterface

endmodule

interface MemServerPortal;
   interface MemServerPortalRequest request;
endinterface

module mkPhysMemSlavePortal#(PhysMemSlave#(32,dataBusWidth) ms, MemServerPortalIndication ind)(MemServerPortal)
   provisos (Add#(dataBusWidth,7,a__)
	     ,Add#(b__,64,dataBusWidth)
	     ,Bits#(MemTypes::MemData#(dataBusWidth), a__));

   FIFOF#(PhysMemRequest#(32,dataBusWidth)) araddrFifo <- mkFIFOF();
   FIFOF#(PhysMemRequest#(32,dataBusWidth)) awaddrFifo <- mkFIFOF();
   FIFOF#(MemData#(dataBusWidth))           rdataFifo <- mkFIFOF();
   FIFOF#(MemData#(dataBusWidth))           wdataFifo <- mkFIFOF();
   FIFOF#(Bit#(6))                doneFifo <- mkFIFOF();

   let araddrCnx <- mkConnection(toGet(araddrFifo), ms.read_server.readReq);
   let awaddrCnx <- mkConnection(toGet(awaddrFifo), ms.write_server.writeReq);
   let rdataCnx  <- mkConnection(ms.read_server.readData, toPut(rdataFifo));
   let wdataCnx  <- mkConnection(toGet(wdataFifo), ms.write_server.writeData);
   let doneCnx   <- mkConnection(ms.write_server.writeDone, toPut(doneFifo));

   rule rl_rdata;
      let rdata <- toGet(rdataFifo).get();
      ind.readDone(truncate(rdata.data));
   endrule

   rule rl_writeDone;
      let tag <- toGet(doneFifo).get();
      ind.writeDone();
   endrule

   interface MemServerPortalRequest request;
      method Action read(Bit#(32) addr);
	 araddrFifo.enq(PhysMemRequest { addr: addr, burstLen: fromInteger(valueOf(TDiv#(dataBusWidth,8))), tag: 0 });
      endmethod
      method Action write(Bit#(32) addr, Bit#(DataBusWidth) value);
	 awaddrFifo.enq(PhysMemRequest { addr: addr, burstLen: fromInteger(valueOf(TDiv#(dataBusWidth,8))), tag: 0 });
	 wdataFifo.enq(MemData {data: extend(value), tag: 0, last: True});
      endmethod
   endinterface
endmodule


comments

comments

// Copyright (c) 2014 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import StmtFSM::*;
import BRAM::*;
import MCSAlgorithm::*;
import StackReg::*;

/* frame arguments */
typedef struct {
   Bit#(7) aStart;
   Bit#(7) bStart;
   Bit#(7) aLen;
   Bit#(7) bLen;
   } CArgs deriving(Bits);

typedef struct {
   Bit#(7) midi;
   Bit#(7) maxj;
   } CVars deriving(Bits);

typedef enum {HCSIdle, HCS1, HCS2, HCS3, HCS4, HCS5, HCS6,
   HCSComplete} HCState deriving (Bits, Eq);

module mkHirschC#(BRAMServer#(Bit#(strIndexWidth), Bit#(8)) strA, BRAMServer#(Bit#(strIndexWidth), Bit#(8)) strB, BRAMServer#(Bit#(lIndexWidth), Bit#(16)) matL, MCSAlgorithm chirschB0,  MCSAlgorithm chirschB1, BRAMServer#(Bit#(lIndexWidth), Bit#(16)) l0, BRAMServer#(Bit#(lIndexWidth), Bit#(16)) l1)(MCSAlgorithm)
         provisos(Add#(0, 7, strIndexWidth),
	       Add#(0, 14, lIndexWidth));

      /* pc, args, vars */
   StackReg#(128, HCState, CArgs, CVars) fr <- mkStackReg(128, HCSIdle);

   Reg#(Bit#(7)) aStartReg <- mkReg(0);
   Reg#(Bit#(7)) bStartReg <- mkReg(0);
   Reg#(Bit#(14)) rStartReg <- mkReg(0);
   Reg#(Bit#(7)) aLenReg <- mkReg(0);
   Reg#(Bit#(7)) bLenReg <- mkReg(0);
   Reg#(Bit#(7)) ii <- mkReg(0);
   Reg#(Bit#(7)) jj <- mkReg(0);
   Reg#(Bit#(8)) aData <- mkReg(0);
   Reg#(Bit#(8)) bData <- mkReg(0);
   Reg#(Bit#(14)) outcounter <- mkReg(0);
   Reg#(Bit#(16)) maxjvalue <- mkReg(0);

/*
 * HirschC(Astart, Alen, Bstart, Blen, output fifo)
 * implicit A storage, B storage
 *
 */

  // This FSM searches string B looking for the first char of string A
   Stmt hirschC2Stmt =
   seq
      $display("hirschC2Stmt running ");
      // read A[0]
      strA.request.put(BRAMRequest{write: False, responseOnWrite: False, address: fr.args.aStart, datain: ?});
      action
	 let tmp <- strA.response.get();
	 aData <= tmp;
      endaction
      // scan for it in B
      for (jj <= 0; jj < fr.args.bLen; jj <= jj + 1)
	 seq
	    strB.request.put(BRAMRequest{write: False, responseOnWrite: False, address: fr.args.bStart + jj, datain: ?});
	    action
	       let tmp <- strB.response.get();
	       bData <= tmp;
	    endaction
	    if (aData == bData)
	       seq
		  matL.request.put(BRAMRequest{write: True, responseOnWrite: False, address: outcounter, datain: zeroExtend(aData)});
		  outcounter <= outcounter + 1;
		  break;
	       endseq
	 endseq
      fr.doreturn();
   endseq;

   FSM hC2fsm <- mkFSM(hirschC2Stmt);

   // AlgC step 1, 2, and 3
   rule hc1 (fr.pc == HCS1);
      $display("HCS1 aStart %d aLen %d bStart %d bLen %d", fr.args.aStart, fr.args.aLen, fr.args.bStart, fr.args.bLen);
      if (fr.args.bLen == 0)
	fr.doreturn();
     else if (fr.args.aLen == 1)
	begin
	   hC2fsm.start();
	   fr.nextpc(HCS2);
	end
     else
	action
	   let midi = fr.args.aLen >> 1;
	   fr.vars.midi <= midi;
	   chirschB0.setupA(fr.args.aStart, midi);
	   chirschB0.setupB(fr.args.bStart, fr.args.bLen);
	   chirschB1.setupA(fr.args.aStart+midi, fr.args.aLen - midi);
	   chirschB1.setupB(fr.args.bStart, fr.args.bLen);
	   chirschB0.fsm.start();
	   chirschB1.fsm.start();
	   fr.nextpc(HCS3);
	endaction
   endrule

   // This FSM searches the results of the two calls to HirschB
   Stmt hirschC4Stmt =
   seq
      $display ("hirschC4A stmt running");
      maxjvalue <= 0;
      for (jj <= 0; jj <= fr.args.bLen; jj <= jj + 1)
	 seq
	    l0.request.put(BRAMRequest{write: False, responseOnWrite: False, address: zeroExtend(jj), datain: ?});
	    l1.request.put(BRAMRequest{write: False, responseOnWrite: False, address: zeroExtend(fr.args.bLen - jj), datain: ?});
	    action
	       let t1 <- l0.response.get();
	       let t2 <- l1.response.get();
	       if ((t1 + t2) > maxjvalue)
		  action
		     maxjvalue <= t1 + t2;
		     fr.vars.maxj <= jj;
		  endaction
	    endaction
	 endseq

      fr.docall(HCS1, HCS5, CArgs {aStart: fr.args.aStart, aLen: fr.vars.midi, bStart: fr.args.bStart, bLen: fr.vars.maxj}, fr.vars);
   endseq;

   FSM hc4fsm <- mkFSM(hirschC4Stmt);

   rule hc3 (fr.pc == HCS3 && chirschB0.fsm.done() && chirschB1.fsm.done());
      $display("HSC3");
      hc4fsm.start();
      fr.nextpc(HCS4);
   endrule

   rule hc5 (fr.pc == HCS5);
      $display("HSC35");
      fr.docall(HCS1, HCS6, CArgs{aStart: fr.args.aStart + fr.vars.midi, aLen: fr.args.aLen - fr.vars.midi, bStart: fr.args.bStart + fr.vars.maxj, bLen: fr.args.bLen - fr.vars.maxj}, fr.vars);
   endrule

   rule hc6 (fr.pc == HCS6);
      $display("HSCC");
      fr.doreturn();
   endrule

   method Action setupA(Bit#(7) start, Bit#(7) length);
      $display("HirschC setupA %d %d", start, length);
      aStartReg <= start;
      aLenReg <= length;
   endmethod

   method Action setupB(Bit#(7) start, Bit#(7) length);
      $display("HirschC setupB %d %d", start, length);
      bStartReg <= start;
      bLenReg <= length;
   endmethod

   method Action setupL(Bit#(14) start);
      rStartReg <= start;
   endmethod

   interface FSM fsm;
      method Action start();
         $display("HirschC running");
	 fr.docall(HCS1, HCSComplete, CArgs{aStart: 0, aLen: aLenReg,
	    bStart: 0, bLen: bLenReg}, CVars {midi: 0, maxj: 0});
      endmethod
      method Bool done();
	 return(fr.pc == HCSIdle);
      endmethod
      method Action waitTillDone();
      endmethod
      method Action abort();
      endmethod
   endinterface: fsm

endmodule

// Copyright (c) 2014 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import StmtFSM::*;
import BRAM::*;
import MCSAlgorithm::*;

/* frame arguments */
typedef struct {
   Bit#(7) aStart;
   Bit#(7) bStart;
   Bit#(7) aLen;
   Bit#(7) bLen;
   } CArgs deriving(Bits);

typedef struct {
   Bit#(7) midi;
   Bit#(7) maxj;
   } CVars deriving(Bits);

typedef enum {HCSIdle, HCS1, HCS2, HCSComplete} HCState deriving (Bits, Eq);

module mkHirschC#(BRAMServer#(Bit#(strIndexWidth), Bit#(8)) strA, BRAMServer#(Bit#(strIndexWidth), Bit#(8)) strB, BRAMServer#(Bit#(lIndexWidth), Bit#(16)) matL, MCSAlgorithm chirschB0,  MCSAlgorithm chirschB1, BRAMServer#(Bit#(lIndexWidth)) l0, BRAMServer#(Bit#(lIndexWidth)) l1)(MCSAlgorithm)
         provisos(Add#(0, 7, strIndexWidth),
	       Add#(0, 14, lIndexWidth));

      /* pc, args, vars */
   StackReg#(128, FSState, CArgs, CVars) fr <- mkStackReg(128, HCSIDLE);

   Reg#(Bit#(7)) aStartReg <- mkReg(0);
   Reg#(Bit#(7)) bStartReg <- mkReg(0);
   Reg#(Bit#(14)) rStartReg <- mkReg(0);
   Reg#(Bit#(7)) aLenReg <- mkReg(0);
   Reg#(Bit#(7)) bLenReg <- mkReg(0);
   Reg#(Bit#(7)) ii <- mkReg(0);
   Reg#(Bit#(7)) jj <- mkReg(0);
   Reg#(Bit#(8)) aData <- mkReg(0);
   Reg#(Bit#(8)) bData <- mkReg(0);
   Reg#(Bit#(14)) outcounter <- mkReg(0);

/*
 * HirschC(Astart, Alen, Bstart, Blen, output fifo)
 * implicit A storage, B storage
 *
 */

  // This FSM searches string B looking for the first char of string A
   Stmt hirschC2stmt =
   seq
      $display("hirschC running ");
      // read A[0]
      strA.request.put(BRAMRequest{write: False, responseOnWrite: False, address: fr.args.aStart, datain: ?});
      action
	 let tmp <- strA.response.get();
	 aData <= tmp;
      endaction
      // scan for it in B
      for (jj <= 0; jj < fr.args.bLen; jj <= jj + 1)
	 seq
	    strB.request.put(BRAMRequest{write: False, responseOnWrite: False, address: bstart + jj, datain: ?});
	    action
	       let tmp <- strB.response.get();
	       bData <= tmp;
	    endaction
	    if (aData == bData)
	       seq
		  matL.request.put(BRAMRequest{write: True, responseOnWrite: False, address: outcounter, datain: aData});
		  outcounter <= outcounter + 1;
		  break;
	       endseq
	 endseq
      fr.nextpc(HCSComplete);
   endseq;

   FSM hC2fsm <- mkFSM(hirschC2Stmt);

   // AlgC step 1, 2, and 3
   rule hc1 (fr.pc == HCS1);
      $display("HCS1 aStart %d aLen %d bStart %d bLen %d", fr.args.aStart, fr.args.aLen, fr.args.bStart, fr.args.bLen);
      if (fr.args.bLen == 0)
	doreturn();
     else if (fr.args.aLen == 1)
	begin
	   hc2fsm.start();
	   fr.nextpc(HCS2);
	end
     else
	action
	   let midi = fr.aLen >> 1;
	   fr.vars.midi <= midi;
	   chirschB0.setupA(fr.args.aStart, midi);
	   chirschB0.setupB(fr.args.bStart, fr.args.bLen);
	   chirschB1.setupA(fr.args.aStart+midi, fr.args.aLen - midi);
	   chirschB1.setupB(fr.args.bStart, fr.args.bLen);
	   chirschB0.start();
	   chirschB1.start();
	   fr.nextPC(HCS3);
	endaction
   endrule

   // This FSM searches the results of the two calls to HirschB
   Stmt hirschC4Stmt =
   seq
      maxjvalue <= 0;
      for (jj <= 0; jj <= fr.args.bLen; jj <= jj + 1)
	 seq
	    l0.request.put(BRAMRequest{write: False, responseOnWrite: False, address: jj, datain: ?});
	    l1.request.put(BRAMRequest{write: False, responseOnWrite: False, address: fr.args.bLen - jj, datain: ?});
	    action
	       let t1 <- lo.response.get();
	       let t2 <- l1.response.get();
	       if ((t1 + t2) > maxjvalue)
		  action
		     maxjvalue <= t1 + t2;
		     fr.vars.maxj <= jj;
		  endaction
	    endaction
	 endseq

      fr.docall(HCS1, HCS5, CArgs {aStart: fr.args.aStart, aLen: fr.vars.midi, bStart: fr.args.bStart, bLen: fr.vars.maxj}, fr.vars);
   endseq

   FSM hc4fsm <- mkFSM(hirschC4Stmt);

   rule hc3 (fr.pc == HCS3 && chirschB0.done() && chirschB1.done);
      hc4fsm.start();
      fr.nextPC(HCS4);
   endrule

   rule hc5 (fr.pc == HCS5);
      fr.docall(HCS1, HCS6, CArgs{aStart: fr.args.aStart + fr.vars.midi, aLen: fr.args.aLen - fr.vars.midi, bStart: fr.args.bStart + fr.vars.maxj, bLen: fr.args.bLen - fr.vars.maxj}, fr.vars);
   endrule

   rule hc6 (fr.pc == HCS6);
      fr.doreturn();
   endrule

   method Action setupA(Bit#(7) start, Bit#(7) length);
      aStartReg <= start;
      aLenReg <= length;
   endmethod

   method Action setupB(Bit#(7) start, Bit#(7) length);
      bStartReg <= start;
      bLenReg <= length;
   endmethod

   method Action setupL(Bit#(14) start);
      rStartReg <= start;
   endmethod

   method Action start();
      docall(HCS1, HCSComplere, {aStart: 0, aLen: aLenReg,
	 bStart: 0, bLen: bLenReg}, {0, 0})
   endmethod

   interface FSM fsm = hC;

endmodule


remove redundant FIFOs, use MKFIFO1 instead of mkSizedFIFO(1)

Change-Id: I9d154d9f20bdd22cb760fa7737a1d3cdbb2c8701

// Copyright (c) 2012 Nokia, Inc.
// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import Vector::*;
import GetPut::*;
import FIFOF::*;
import SpecialFIFOs::*;
import FIFO::*;
import Connectable::*;

import Portal::*;
import Directory::*;
import MemTypes::*;
import Pipe::*;

module mkInterruptMux#(Vector#(numPortals,ReadOnly#(Bool)) inputs) (ReadOnly#(Bool))
   provisos(Add#(nz, TLog#(numPortals), 4),
	    Add#(1, a__, numPortals));

   function Bool my_read(ReadOnly#(Bool) x);
      return x._read;
   endfunction

   function Bool my_or(Bool a, Bool b);
      return a || b;
   endfunction

   method Bool _read;
      return fold(my_or,map(my_read,inputs));
   endmethod

endmodule

module mkSlaveMux#(Directory#(aw,aw,dataWidth) dir,
		   Vector#(numPortals,MemPortal#(aw,dataWidth)) portals) (MemSlave#(addrWidth,dataWidth))
   provisos(Add#(1,numPortals,numInputs),
	    Add#(a__,TLog#(numInputs),4),
	    Min#(2,TLog#(numInputs),bpc),
	    FunnelPipesPipelined#(1, numInputs, ObjectData#(dataWidth), bpc)
	    );

   Vector#(numInputs, MemSlave#(aw,dataWidth)) portalIfcs = cons(dir.portalIfc.slave,map(getSlave, portals));
   let port_sel_low = valueOf(aw);
   let port_sel_high = valueOf(TAdd#(3,aw));
   function Bit#(4) psel(Bit#(addrWidth) a);
      return a[port_sel_high:port_sel_low];
   endfunction
   function Bit#(aw) asel(Bit#(addrWidth) a);
      return a[(port_sel_low-1):0];
   endfunction
   function Get#(ObjectData#(dataWidth)) getMemPortalReadData(MemSlave#(aw,dataWidth) x) = x.read_server.readData;
   function Put#(ObjectData#(dataWidth)) getMemPortalWriteData(MemSlave#(aw,dataWidth) x) = x.write_server.writeData;

   FIFO#(MemRequest#(aw)) req_ars <- mkSizedFIFO(1);
   FIFO#(Bit#(TLog#(numInputs))) rs <- mkFIFO1();
   Vector#(numInputs, PipeOut#(ObjectData#(dataWidth))) readDataPipes <- mapM(mkPipeOut, map(getMemPortalReadData,portalIfcs));
   FunnelPipe#(1, numInputs, ObjectData#(dataWidth), bpc) read_data_funnel <- mkFunnelPipesPipelined(readDataPipes);

   FIFO#(MemRequest#(aw)) req_aws <- mkFIFO1();
   FIFO#(Bit#(TLog#(numInputs))) ws <- mkFIFO1();
   FIFOF#(Tuple2#(Bit#(TLog#(numInputs)), ObjectData#(dataWidth))) write_data <- mkFIFOF;
   UnFunnelPipe#(1, numInputs, ObjectData#(dataWidth), bpc) write_data_unfunnel <- mkUnFunnelPipesPipelined(cons(toPipeOut(write_data),nil));
   Vector#(numInputs, PipeIn#(ObjectData#(dataWidth))) writeDataPipes <- mapM(mkPipeIn, map(getMemPortalWriteData,portalIfcs));
   zipWithM(mkConnection, write_data_unfunnel, writeDataPipes);

   rule req_aw;
      let req <- toGet(req_aws).get;
      portalIfcs[ws.first].write_server.writeReq.put(req);
   endrule

   rule req_ar;
      let req <- toGet(req_ars).get;
      portalIfcs[rs.first].read_server.readReq.put(req);
   endrule

   interface MemWriteServer write_server;
      interface Put writeReq;
	 method Action put(MemRequest#(addrWidth) req);
	    req_aws.enq(MemRequest{addr:asel(req.addr), burstLen:req.burstLen, tag:req.tag});
	    if (req.burstLen > 1) $display("**** \n\n mkSlaveMux.writeReq len=%d \n\n ****", req.burstLen);
	    ws.enq(truncate(psel(req.addr)));
	    //$display("mkSlaveMux.writeReq addr=%h aw=%d psel=%h", req.addr, valueOf(aw), psel(req.addr));
	 endmethod
      endinterface
      interface Put writeData;
	 method Action put(ObjectData#(dataWidth) wdata);
	    write_data.enq(tuple2(ws.first,wdata));
	 endmethod
      endinterface
      interface Get writeDone;
	 method ActionValue#(Bit#(6)) get();
	    let rv <- portalIfcs[ws.first].write_server.writeDone.get();
	    ws.deq();
	    return rv;
	 endmethod
      endinterface
   endinterface
   interface MemReadServer read_server;
      interface Put readReq;
	 method Action put(MemRequest#(addrWidth) req);
	    req_ars.enq(MemRequest{addr:asel(req.addr), burstLen:req.burstLen, tag:req.tag});
	    rs.enq(truncate(psel(req.addr)));
	    if (req.burstLen > 1) $display("**** \n\n mkSlaveMux.readReq len=%d \n\n ****", req.burstLen);
	    //$display("mkSlaveMux.readReq addr=%h aw=%d psel=%h", req.addr, valueOf(aw), psel(req.addr));
	 endmethod
      endinterface
      interface Get readData;
	 method ActionValue#(ObjectData#(dataWidth)) get();
	    let rv <- toGet(read_data_funnel[0]).get;
	    rs.deq();
	    //$display("mkSlaveMux.readData rs=%d data=%h", rs.first, rv.data);
	    return rv;
	 endmethod
      endinterface
   endinterface

endmodule

module mkMemSlaveMux#(Vector#(numSlaves,MemSlave#(aw,dataWidth)) slaves) (MemSlave#(addrWidth,dataWidth))
   provisos(Add#(selWidth,aw,addrWidth),
	    Add#(a__, TLog#(numSlaves), selWidth)
      );

   Vector#(numSlaves, MemSlave#(aw,dataWidth)) portalIfcs = take(slaves);
   let port_sel_low = valueOf(aw);
   let port_sel_high = valueOf(TSub#(addrWidth,1));
   function Bit#(selWidth) psel(Bit#(addrWidth) a);
      return a[port_sel_high:port_sel_low];
   endfunction
   function Bit#(aw) asel(Bit#(addrWidth) a);
      return a[(port_sel_low-1):0];
   endfunction

   FIFO#(MemRequest#(aw))   req_ars <- mkFIFO1();
   FIFO#(Bit#(TLog#(numSlaves))) rs <- mkFIFO1();

   FIFO#(MemRequest#(aw))   req_aws <- mkFIFO1();
   FIFO#(Bit#(TLog#(numSlaves))) ws <- mkFIFO1();

   rule req_aw;
      let req <- toGet(req_aws).get;
      portalIfcs[ws.first].write_server.writeReq.put(req);
   endrule

   rule req_ar;
      let req <- toGet(req_ars).get;
      portalIfcs[rs.first].read_server.readReq.put(req);
   endrule

   interface MemWriteServer write_server;
      interface Put writeReq;
	 method Action put(MemRequest#(addrWidth) req);
	    req_aws.enq(MemRequest{addr:asel(req.addr), burstLen:req.burstLen, tag:req.tag});
	    if (req.burstLen > 1) $display("**** \n\n mkMemSlaveMux.writeReq len=%d \n\n ****", req.burstLen);
	    //$display("mkMemSlaveMux.writeReq addr=%h selWidth=%d aw=%d psel=%h", req.addr, valueOf(selWidth), valueOf(aw), psel(req.addr));
	    ws.enq(truncate(psel(req.addr)));
	 endmethod
      endinterface
      interface Put writeData;
	 method Action put(ObjectData#(dataWidth) wdata);
	    //$display("mkMemSlaveMux.writeData aw=%d ws=%d data=%h", valueOf(aw), ws.first, wdata.data);
	    portalIfcs[ws.first].write_server.writeData.put(wdata);
	 endmethod
      endinterface
      interface Get writeDone;
	 method ActionValue#(Bit#(6)) get();
	    let rv <- portalIfcs[ws.first].write_server.writeDone.get();
	    ws.deq();
	    return rv;
	 endmethod
      endinterface
   endinterface
   interface MemReadServer read_server;
      interface Put readReq;
	 method Action put(MemRequest#(addrWidth) req);
	    req_ars.enq(MemRequest{addr:asel(req.addr), burstLen:req.burstLen, tag:req.tag});
	    //$display("mkMemSlaveMux.readReq addr=%h aw=%d psel=%h", req.addr, valueOf(aw), psel(req.addr));
	    if (req.burstLen > 1) $display("**** \n\n mkMemSlaveMux.readReq len=%d \n\n ****", req.burstLen);
	    rs.enq(truncate(psel(req.addr)));
	 endmethod
      endinterface
      interface Get readData;
	 method ActionValue#(ObjectData#(dataWidth)) get();
	    let rv <- portalIfcs[rs.first].read_server.readData.get();
	    //$display("mkMemSlaveMux.readData aw=%d rs=%d data=%h", valueOf(aw), rs.first, rv.data);
	    //if (rv.last) begin
	       rs.deq();
	    //end
	    return rv;
	 endmethod
      endinterface
   endinterface

endmodule

// Copyright (c) 2012 Nokia, Inc.
// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import Vector::*;
import GetPut::*;
import FIFOF::*;
import SpecialFIFOs::*;
import FIFO::*;
import Connectable::*;

import Portal::*;
import Directory::*;
import MemTypes::*;
import Pipe::*;

module mkInterruptMux#(Vector#(numPortals,ReadOnly#(Bool)) inputs) (ReadOnly#(Bool))
   provisos(Add#(nz, TLog#(numPortals), 4),
	    Add#(1, a__, numPortals));

   function Bool my_read(ReadOnly#(Bool) x);
      return x._read;
   endfunction

   function Bool my_or(Bool a, Bool b);
      return a || b;
   endfunction

   method Bool _read;
      return fold(my_or,map(my_read,inputs));
   endmethod

endmodule

module mkSlaveMux#(Directory#(aw,aw,dataWidth) dir,
		   Vector#(numPortals,MemPortal#(aw,dataWidth)) portals) (MemSlave#(addrWidth,dataWidth))
   provisos(Add#(1,numPortals,numInputs),
	    Add#(a__,TLog#(numInputs),4),
	    Min#(2,TLog#(numInputs),bpc),
	    FunnelPipesPipelined#(1, numInputs, ObjectData#(dataWidth), bpc)
	    );

   Vector#(numInputs, MemSlave#(aw,dataWidth)) portalIfcs = cons(dir.portalIfc.slave,map(getSlave, portals));
   let port_sel_low = valueOf(aw);
   let port_sel_high = valueOf(TAdd#(3,aw));
   function Bit#(4) psel(Bit#(addrWidth) a);
      return a[port_sel_high:port_sel_low];
   endfunction
   function Bit#(aw) asel(Bit#(addrWidth) a);
      return a[(port_sel_low-1):0];
   endfunction
   function Get#(ObjectData#(dataWidth)) getMemPortalReadData(MemSlave#(aw,dataWidth) x) = x.read_server.readData;
   function Put#(ObjectData#(dataWidth)) getMemPortalWriteData(MemSlave#(aw,dataWidth) x) = x.write_server.writeData;

   FIFO#(MemRequest#(aw)) req_ars <- mkSizedFIFO(1);
   FIFO#(void) req_ar_fifo <- mkSizedFIFO(1);
   FIFO#(Bit#(TLog#(numInputs))) rs <- mkFIFO();
   Vector#(numInputs, PipeOut#(ObjectData#(dataWidth))) readDataPipes <- mapM(mkPipeOut, map(getMemPortalReadData,portalIfcs));
   FunnelPipe#(1, numInputs, ObjectData#(dataWidth), bpc) read_data_funnel <- mkFunnelPipesPipelined(readDataPipes);

   FIFO#(MemRequest#(aw)) req_aws <- mkSizedFIFO(1);
   FIFO#(void) req_aw_fifo <- mkSizedFIFO(1);
   FIFO#(Bit#(TLog#(numInputs))) ws <- mkFIFO();
   FIFOF#(Tuple2#(Bit#(TLog#(numInputs)), ObjectData#(dataWidth))) write_data <- mkFIFOF;
   UnFunnelPipe#(1, numInputs, ObjectData#(dataWidth), bpc) write_data_unfunnel <- mkUnFunnelPipesPipelined(cons(toPipeOut(write_data),nil));
   Vector#(numInputs, PipeIn#(ObjectData#(dataWidth))) writeDataPipes <- mapM(mkPipeIn, map(getMemPortalWriteData,portalIfcs));
   zipWithM(mkConnection, write_data_unfunnel, writeDataPipes);

   rule req_aw;
      let req <- toGet(req_aws).get;
      portalIfcs[ws.first].write_server.writeReq.put(req);
   endrule

   rule req_ar;
      let req <- toGet(req_ars).get;
      portalIfcs[rs.first].read_server.readReq.put(req);
   endrule

   interface MemWriteServer write_server;
      interface Put writeReq;
	 method Action put(MemRequest#(addrWidth) req);
	    req_aws.enq(MemRequest{addr:asel(req.addr), burstLen:req.burstLen, tag:req.tag});
	    if (req.burstLen > 1) $display("**** \n\n mkSlaveMux.writeReq len=%d \n\n ****", req.burstLen);
	    req_aw_fifo.enq(?);
	    ws.enq(truncate(psel(req.addr)));
	    //$display("mkSlaveMux.writeReq addr=%h aw=%d psel=%h", req.addr, valueOf(aw), psel(req.addr));
	 endmethod
      endinterface
      interface Put writeData;
	 method Action put(ObjectData#(dataWidth) wdata);
	    write_data.enq(tuple2(ws.first,wdata));
	 endmethod
      endinterface
      interface Get writeDone;
	 method ActionValue#(Bit#(6)) get();
	    let rv <- portalIfcs[ws.first].write_server.writeDone.get();
	    req_aw_fifo.deq;
	    ws.deq();
	    return rv;
	 endmethod
      endinterface
   endinterface
   interface MemReadServer read_server;
      interface Put readReq;
	 method Action put(MemRequest#(addrWidth) req);
	    req_ars.enq(MemRequest{addr:asel(req.addr), burstLen:req.burstLen, tag:req.tag});
	    req_ar_fifo.enq(?);
	    rs.enq(truncate(psel(req.addr)));
	    if (req.burstLen > 1) $display("**** \n\n mkSlaveMux.readReq len=%d \n\n ****", req.burstLen);
	    //$display("mkSlaveMux.readReq addr=%h aw=%d psel=%h", req.addr, valueOf(aw), psel(req.addr));
	 endmethod
      endinterface
      interface Get readData;
	 method ActionValue#(ObjectData#(dataWidth)) get();
	    let rv <- toGet(read_data_funnel[0]).get;
	    req_ar_fifo.deq;
	    rs.deq();
	    //$display("mkSlaveMux.readData rs=%d data=%h", rs.first, rv.data);
	    return rv;
	 endmethod
      endinterface
   endinterface

endmodule

module mkMemSlaveMux#(Vector#(numSlaves,MemSlave#(aw,dataWidth)) slaves) (MemSlave#(addrWidth,dataWidth))
   provisos(Add#(selWidth,aw,addrWidth),
	    Add#(a__, TLog#(numSlaves), selWidth)
      );

   Vector#(numSlaves, MemSlave#(aw,dataWidth)) portalIfcs = take(slaves);
   let port_sel_low = valueOf(aw);
   let port_sel_high = valueOf(TSub#(addrWidth,1));
   function Bit#(selWidth) psel(Bit#(addrWidth) a);
      return a[port_sel_high:port_sel_low];
   endfunction
   function Bit#(aw) asel(Bit#(addrWidth) a);
      return a[(port_sel_low-1):0];
   endfunction

   FIFO#(MemRequest#(aw)) req_ars <- mkSizedFIFO(1);
   FIFO#(void) req_ar_fifo <- mkSizedFIFO(1);
   FIFO#(Bit#(TLog#(numSlaves))) rs <- mkFIFO();

   FIFO#(MemRequest#(aw)) req_aws <- mkSizedFIFO(1);
   FIFO#(void) req_aw_fifo <- mkSizedFIFO(1);
   FIFO#(Bit#(TLog#(numSlaves))) ws <- mkFIFO();

   rule req_aw;
      let req <- toGet(req_aws).get;
      portalIfcs[ws.first].write_server.writeReq.put(req);
   endrule

   rule req_ar;
      let req <- toGet(req_ars).get;
      portalIfcs[rs.first].read_server.readReq.put(req);
   endrule

   interface MemWriteServer write_server;
      interface Put writeReq;
	 method Action put(MemRequest#(addrWidth) req);
	    req_aws.enq(MemRequest{addr:asel(req.addr), burstLen:req.burstLen, tag:req.tag});
	    req_aw_fifo.enq(?);
	    if (req.burstLen > 1) $display("**** \n\n mkMemSlaveMux.writeReq len=%d \n\n ****", req.burstLen);
	    //$display("mkMemSlaveMux.writeReq addr=%h selWidth=%d aw=%d psel=%h", req.addr, valueOf(selWidth), valueOf(aw), psel(req.addr));
	    ws.enq(truncate(psel(req.addr)));
	 endmethod
      endinterface
      interface Put writeData;
	 method Action put(ObjectData#(dataWidth) wdata);
	    //$display("mkMemSlaveMux.writeData aw=%d ws=%d data=%h", valueOf(aw), ws.first, wdata.data);
	    portalIfcs[ws.first].write_server.writeData.put(wdata);
	 endmethod
      endinterface
      interface Get writeDone;
	 method ActionValue#(Bit#(6)) get();
	    let rv <- portalIfcs[ws.first].write_server.writeDone.get();
	    req_aw_fifo.deq;
	    ws.deq();
	    return rv;
	 endmethod
      endinterface
   endinterface
   interface MemReadServer read_server;
      interface Put readReq;
	 method Action put(MemRequest#(addrWidth) req);
	    req_ars.enq(MemRequest{addr:asel(req.addr), burstLen:req.burstLen, tag:req.tag});
	    req_ar_fifo.enq(?);
	    //$display("mkMemSlaveMux.readReq addr=%h aw=%d psel=%h", req.addr, valueOf(aw), psel(req.addr));
	    if (req.burstLen > 1) $display("**** \n\n mkMemSlaveMux.readReq len=%d \n\n ****", req.burstLen);
	    rs.enq(truncate(psel(req.addr)));
	 endmethod
      endinterface
      interface Get readData;
	 method ActionValue#(ObjectData#(dataWidth)) get();
	    let rv <- portalIfcs[rs.first].read_server.readData.get();
	    //$display("mkMemSlaveMux.readData aw=%d rs=%d data=%h", valueOf(aw), rs.first, rv.data);
	    //if (rv.last) begin
	       req_ar_fifo.deq;
	       rs.deq();
	    //end
	    return rv;
	 endmethod
      endinterface
   endinterface

endmodule