updated dma controller

updated dma controller

// Copyright (c) 2015 Connectal Project

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import Clocks::*;
import Vector::*;
import BuildVector::*;
import FIFO::*;
import FIFOF::*;
import GetPut::*;
import Connectable::*;
import Probe::*;

import ConnectalConfig::*;
import Pipe::*;
import MemTypes::*;
import MemReadEngine::*;
import MemWriteEngine::*;
import HostInterface::*;
`include "ConnectalProjectConfig.bsv"

interface DmaRequest;
   //
   // Configures burstLen used by DMA transfers. Only needed for performance tuning if default value does not perform well.
   //
   method Action writeRequestSize(Bit#(16) burstLenBytes);
   //
   // Sets the DMA read request size. May be larger than writeRequestSize, depending on the host system chipset and configuration.
   //
   method Action readRequestSize(Bit#(16) readRequestBytes);
   //
   // Requests a transferToFpga of system memory, streaming the data to the toFpga PipeOut
   // @param objId the reference to the memory object allocated by portalAlloc
   // @param base  offset, in bytes, from which to start reading
   // @param bytes number of bytes to read, must be a multiple of the buswidth in bytes
   // @param tag   identifier for the request
   method Action transferToFpga(Bit#(32) objId, Bit#(32) base, Bit#(32) bytes, Bit#(8) tag);
   //
   // Requests a transferFromFpga of system memory, streaming the data from the fromFpga PipeIn
   // @param objId the reference to the memory object allocated by portalAlloc
   // @param base  offset, in bytes, to which to start writing
   // @param bytes number of bytes to write, must be a multiple of the buswidth in bytes
   // @param tag   identifier for the request
   method Action transferFromFpga(Bit#(32) objId, Bit#(32) base, Bit#(32) bytes, Bit#(8) tag);
endinterface

interface DmaIndication;
   // Indicates completion of transferToFpga request, identified by tag, from offset base of objId
   method Action transferToFpgaDone(Bit#(32) objId, Bit#(32) base, Bit#(8) tag, Bit#(32) cycles);
   // Indicates completion of transferFromFpga request, identified by tag, to offset base of objId
   method Action transferFromFpgaDone(Bit#(32) objId, Bit#(32) base, Bit#(8) tag, Bit#(32) cycles);
endinterface

//
// DmaController controls multiple channels of DMA to/from system memory
// @param numChannels: the maximum number of simultaneous transferToFpga and transferFromFpga streams
interface DmaController#(numeric type numChannels);
   // request from software
   interface Vector#(numChannels,DmaRequest) request;
   // data out to application logic
   interface Vector#(numChannels,PipeOut#(MemDataF#(DataBusWidth))) toFpga;
   // data in from application logic
   interface Vector#(numChannels,PipeIn#(MemDataF#(DataBusWidth)))  fromFpga;
   // memory interfaces connected to MemServer
   interface Vector#(1,MemReadClient#(DataBusWidth))      readClient;
   interface Vector#(1,MemWriteClient#(DataBusWidth))     writeClient;
endinterface

typedef 31 NumOutstandingRequests;
typedef TMul#(NumOutstandingRequests,1024) BufferSizeBytes;

function Bit#(dsz) memdatafToData(MemDataF#(dsz) mdf); return mdf.data; endfunction

module mkDmaController#(Vector#(numChannels,DmaIndication) indication)(DmaController#(numChannels))
   provisos (Add#(1, a__, numChannels),
	     Add#(b__, TLog#(numChannels), TAdd#(1, TLog#(TMul#(NumOutstandingRequests, numChannels)))),
	     Add#(c__, TLog#(numChannels), MemTagSize), // from MemReadEngine
	     Add#(d__, TLog#(numChannels), TLog#(TMul#(NumOutstandingRequests, numChannels))),
	     FunnelPipesPipelined#(1, numChannels, MemTypes::MemData#(DataBusWidth), 2),
	     FunnelPipesPipelined#(1, numChannels, MemTypes::MemRequest, 2),
	     FunnelPipesPipelined#(1, numChannels, Bit#(6), 2)
	     );
   MemReadEngine#(DataBusWidth,DataBusWidth,NumOutstandingRequests,numChannels)  re <- mkMemReadEngineBuff(valueOf(BufferSizeBytes));
   MemWriteEngine#(DataBusWidth,DataBusWidth,NumOutstandingRequests,numChannels) we <- mkMemWriteEngineBuff(valueOf(BufferSizeBytes));

   Vector#(numChannels, FIFO#(MemengineCmd)) readCmds <- replicateM(mkSizedFIFO(valueOf(NumOutstandingRequests)));
   Vector#(numChannels, FIFO#(MemengineCmd)) writeCmds <- replicateM(mkSizedFIFO(valueOf(NumOutstandingRequests)));

   Vector#(numChannels, FIFO#(Tuple3#(Bit#(32),Bit#(32),Bit#(32)))) readReqs <- replicateM(mkSizedFIFO(valueOf(NumOutstandingRequests)));
   Vector#(numChannels, FIFO#(Tuple3#(Bit#(32),Bit#(32),Bit#(32)))) writeReqs <- replicateM(mkSizedFIFO(valueOf(NumOutstandingRequests)));
   Vector#(numChannels, FIFOF#(MemDataF#(DataBusWidth))) transferToFpgaFifo <- replicateM(mkFIFOF());
   Vector#(numChannels, FIFO#(Bit#(8))) writeTags <- replicateM(mkSizedFIFO(valueOf(NumOutstandingRequests)));
   Reg#(Bit#(BurstLenSize)) writeRequestSizeReg <- mkReg(64);
   Reg#(Bit#(BurstLenSize)) readRequestSizeReg <- mkReg(256);
   Reg#(Bit#(32)) cyclesReg <- mkReg(0);
   rule countCycles;
      cyclesReg <= cyclesReg + 1;
   endrule

   Vector#(numChannels, Probe#(Bit#(MemTagSize))) probe_readReq <- replicateM(mkProbe);
   Vector#(numChannels, Probe#(Bit#(8))) probe_readDone <- replicateM(mkProbe);

   for (Integer channel = 0; channel < valueOf(numChannels); channel = channel + 1) begin
      FIFO#(Bit#(8)) readTags <- mkSizedFIFO(valueOf(NumOutstandingRequests));
      Reg#(Bit#(32)) readCount <- mkReg(0);

      rule transferToFpgaReqRule;
	 let cmd <- toGet(readCmds[channel]).get();
	 readReqs[channel].enq(tuple3(cmd.sglId, cmd.base, cyclesReg));
	 probe_readReq[channel] <= cmd.tag;
	 re.readServers[channel].request.put(cmd);
      endrule
       rule readDataRule;
	  let mdf <- toGet(re.readServers[channel].data).get();
	  Bit#(32) count = readCount + 1;
	  if (mdf.last) begin
	     readTags.enq(extend(mdf.tag));
	     count = 0;
	  end
	  readCount <= count;
	  transferToFpgaFifo[channel].enq(mdf);
       endrule
       rule transferToFpgaDoneRule;
	  match { .objId, .base, .cycles } <- toGet(readReqs[channel]).get();
	  cycles = cycles - cyclesReg;
`ifdef MEMENGINE_REQUEST_CYCLES
	  let tagcycles <- toGet(re.readServers[channel].requestCycles).get();
	  cycles = tagcycles.cycles;
`endif
	  let tag <- toGet(readTags).get();
	  probe_readDone[channel] <= tag;
	  indication[channel].transferToFpgaDone(objId, base, tag, cycles);
       endrule
      rule transferFromFpgaReqRule;
	 let cmd <- toGet(writeCmds[channel]).get();
	 writeReqs[channel].enq(tuple3(cmd.sglId, cmd.base, cyclesReg));
	 we.writeServers[channel].request.put(cmd);
	 writeTags[channel].enq(extend(cmd.tag));
      endrule
       rule transferFromFpgaDoneRule;
	  match { .objId, .base, .cycles } <- toGet(writeReqs[channel]).get();
	  cycles = cycles - cyclesReg;
`ifdef MEMENGINE_REQUEST_CYCLES
	  let tagcycles <- toGet(we.writeServers[channel].requestCycles).get();
	  cycles = tagcycles.cycles;
`endif
	  let done <- we.writeServers[channel].done.get();
	  let tag <- toGet(writeTags[channel]).get();
	  indication[channel].transferFromFpgaDone(objId, base, tag, cycles);
       endrule
   end

   function DmaRequest dmaRequestInterface(Integer channel);
      return (interface DmaRequest;
	 method Action writeRequestSize(Bit#(16) burstLenBytes);
	      writeRequestSizeReg <= truncate(burstLenBytes);
	 endmethod
	 method Action readRequestSize(Bit#(16) burstLenBytes);
	      readRequestSizeReg <= truncate(burstLenBytes);
	 endmethod
	 method Action transferToFpga(Bit#(32) objId, Bit#(32) base, Bit#(32) bytes, Bit#(8) tag);
	      readCmds[channel].enq(MemengineCmd {sglId: truncate(objId),
						  base: extend(base),
						  burstLen: extend(readRequestSizeReg),
						  len: bytes,
						  tag: truncate(tag)
						  });
	 endmethod
	 method Action transferFromFpga(Bit#(32) objId, Bit#(32) base, Bit#(32) bytes, Bit#(8) tag);
	    writeCmds[channel].enq(MemengineCmd {sglId: truncate(objId),
						 base: extend(base),
						 burstLen: extend(writeRequestSizeReg),
						 len: bytes,
						 tag: truncate(tag)
						 });
	 endmethod
	 endinterface);
   endfunction
   function PipeIn#(Bit#(dsz)) writeServerData(MemWriteEngineServer#(dsz) s); return s.data; endfunction

   interface Vector request = genWith(dmaRequestInterface);
   interface readClient = vec(re.dmaClient);
   interface writeClient = vec(we.dmaClient);
   interface toFpga = map(toPipeOut,transferToFpgaFifo);
   interface fromFpga = map(mapPipeIn(memdatafToData), map(writeServerData, we.writeServers));
endmodule

// Copyright (c) 2015 Connectal Project

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import Clocks::*;
import Vector::*;
import BuildVector::*;
import FIFO::*;
import FIFOF::*;
import GetPut::*;
import Connectable::*;
import Probe::*;

import ConnectalConfig::*;
import Pipe::*;
import MemTypes::*;
import MemReadEngine::*;
import MemWriteEngine::*;
import HostInterface::*;

`include "ConnectalProjectConfig.bsv"

interface DmaRequest;
   //
   // Configures burstLen used by DMA transfers. Only needed for performance tuning if default value does not perform well.
   //
   method Action writeRequestSize(Bit#(16) burstLenBytes);
   //
   // Sets the DMA read request size. May be larger than writeRequestSize, depending on the host system chipset and configuration.
   //
   method Action readRequestSize(Bit#(16) readRequestBytes);
   //
   // Requests a transferToFpga of system memory, streaming the data to the toFpga PipeOut
   // @param objId the reference to the memory object allocated by portalAlloc
   // @param base  offset, in bytes, from which to start reading
   // @param bytes number of bytes to read, must be a multiple of the buswidth in bytes
   // @param tag   identifier for the request
   method Action transferToFpga(Bit#(32) objId, Bit#(32) base, Bit#(32) bytes, Bit#(8) tag);
   //
   // Requests a transferFromFpga of system memory, streaming the data from the fromFpga PipeIn
   // @param objId the reference to the memory object allocated by portalAlloc
   // @param base  offset, in bytes, to which to start writing
   // @param bytes number of bytes to write, must be a multiple of the buswidth in bytes
   // @param tag   identifier for the request
   method Action transferFromFpga(Bit#(32) objId, Bit#(32) base, Bit#(32) bytes, Bit#(8) tag);
endinterface

interface DmaIndication;
   // Indicates completion of transferToFpga request, identified by tag, from offset base of objId
   method Action transferToFpgaDone(Bit#(32) objId, Bit#(32) base, Bit#(8) tag, Bit#(32) cycles);
   // Indicates completion of transferFromFpga request, identified by tag, to offset base of objId
   method Action transferFromFpgaDone(Bit#(32) objId, Bit#(32) base, Bit#(8) tag, Bit#(32) cycles);
endinterface

//
// DmaController controls multiple channels of DMA to/from system memory
// @param numChannels: the maximum number of simultaneous transferToFpga and transferFromFpga streams
interface DmaController#(numeric type numChannels);
   // request from software
   interface Vector#(numChannels,DmaRequest) request;
   // data out to application logic
   interface Vector#(numChannels,PipeOut#(MemDataF#(DataBusWidth))) toFpga;
   // data in from application logic
   interface Vector#(numChannels,PipeIn#(MemDataF#(DataBusWidth)))  fromFpga;
   // memory interfaces connected to MemServer
   interface Vector#(1,MemReadClient#(DataBusWidth))      readClient;
   interface Vector#(1,MemWriteClient#(DataBusWidth))     writeClient;
endinterface

typedef 15 NumOutstandingRequests;
typedef TMul#(NumOutstandingRequests,TMul#(32,4)) BufferSizeBytes;

function Bit#(dsz) memdatafToData(MemDataF#(dsz) mdf); return mdf.data; endfunction

module mkDmaController#(Vector#(numChannels,DmaIndication) indication)(DmaController#(numChannels))
   provisos (Add#(1, a__, numChannels),
	     Add#(b__, TLog#(numChannels), TAdd#(1, TLog#(TMul#(NumOutstandingRequests, numChannels)))),
	     Add#(c__, TLog#(numChannels), MemTagSize), // from MemReadEngine
	     Add#(d__, TLog#(numChannels), TLog#(TMul#(NumOutstandingRequests, numChannels))),
	     FunnelPipesPipelined#(1, numChannels, MemTypes::MemData#(DataBusWidth), 2),
	     FunnelPipesPipelined#(1, numChannels, MemTypes::MemRequest, 2),
	     FunnelPipesPipelined#(1, numChannels, Bit#(6), 2)
	     );
   MemReadEngine#(DataBusWidth,DataBusWidth,NumOutstandingRequests,numChannels)  re <- mkMemReadEngineBuff(valueOf(BufferSizeBytes));
   MemWriteEngine#(DataBusWidth,DataBusWidth,NumOutstandingRequests,numChannels) we <- mkMemWriteEngineBuff(valueOf(BufferSizeBytes));

   Vector#(numChannels, FIFO#(MemengineCmd)) readCmds <- replicateM(mkSizedFIFO(valueOf(NumOutstandingRequests)));
   Vector#(numChannels, FIFO#(MemengineCmd)) writeCmds <- replicateM(mkSizedFIFO(valueOf(NumOutstandingRequests)));

   Vector#(numChannels, FIFO#(Tuple3#(Bit#(32),Bit#(32),Bit#(32)))) readReqs <- replicateM(mkSizedFIFO(valueOf(NumOutstandingRequests)));
   Vector#(numChannels, FIFO#(Tuple3#(Bit#(32),Bit#(32),Bit#(32)))) writeReqs <- replicateM(mkSizedFIFO(valueOf(NumOutstandingRequests)));
   Vector#(numChannels, FIFOF#(MemDataF#(DataBusWidth))) transferToFpgaFifo <- replicateM(mkFIFOF());
   Vector#(numChannels, FIFO#(Bit#(8))) writeTags <- replicateM(mkSizedFIFO(valueOf(NumOutstandingRequests)));
   Vector#(numChannels, FIFO#(Bit#(8))) readTags <- replicateM(mkSizedFIFO(valueOf(NumOutstandingRequests)));
   Reg#(Bit#(BurstLenSize)) writeRequestSizeReg <- mkReg(64);
   Reg#(Bit#(BurstLenSize)) readRequestSizeReg <- mkReg(256);
   Reg#(Bit#(32)) cyclesReg <- mkReg(0);
   rule countCycles;
      cyclesReg <= cyclesReg + 1;
   endrule

   Vector#(numChannels, Probe#(Bit#(MemTagSize))) probe_readReq <- replicateM(mkProbe);
   Vector#(numChannels, Probe#(Bool)) probe_readLast <- replicateM(mkProbe);
   Vector#(numChannels, Probe#(Bit#(8))) probe_readDone <- replicateM(mkProbe);
   Vector#(numChannels, Probe#(Bit#(32))) probe_readCount <- replicateM(mkProbe);

   for (Integer channel = 0; channel < valueOf(numChannels); channel = channel + 1) begin
      Reg#(Bit#(32)) readCount <- mkReg(0);

      rule transferToFpgaReqRule;
         let cmd <- toGet(readCmds[channel]).get();
         $display ("transferToFpgaReqRule [%d / %d]", channel, valueOf(numChannels));
         readReqs[channel].enq(tuple3(cmd.sglId, cmd.base, cyclesReg));
         probe_readReq[channel] <= cmd.tag;
         re.readServers[channel].request.put(cmd);
      endrule
      rule readDataRule;
         let mdf <- toGet(re.readServers[channel].data).get();
         probe_readLast[channel] <= mdf.last;
         Bit#(32) count = readCount + 1;
         if (mdf.last) begin
            $display ("readDataRule [%d] mdf.last", channel);
            readTags[channel].enq(extend(mdf.tag));
            count = 0;
         end
         probe_readCount[channel] <= count;
         readCount <= count;
         transferToFpgaFifo[channel].enq(mdf);
      endrule
      rule transferToFpgaDoneRule;
         match { .objId, .base, .cycles } <- toGet(readReqs[channel]).get();
         cycles = cycles - cyclesReg;
`ifdef MEMENGINE_REQUEST_CYCLES
         let tagcycles <- toGet(re.readServers[channel].requestCycles).get();
         cycles = tagcycles.cycles;
`endif
         //let done <- re.readServers[channel].done.get();
         let tag <- toGet(readTags[channel]).get();
         probe_readDone[channel] <= tag;
         indication[channel].transferToFpgaDone(objId, base, tag, cycles);
      endrule
      rule transferFromFpgaReqRule;
         let cmd <- toGet(writeCmds[channel]).get();
         $display ("transferfromFpgaReqRule [%d]", channel);
         writeReqs[channel].enq(tuple3(cmd.sglId, cmd.base, cyclesReg));
         we.writeServers[channel].request.put(cmd);
         writeTags[channel].enq(extend(cmd.tag));
      endrule
      rule transferFromFpgaDoneRule;
         match { .objId, .base, .cycles } <- toGet(writeReqs[channel]).get();
         cycles = cycles - cyclesReg;
`ifdef MEMENGINE_REQUEST_CYCLES
         let tagcycles <- toGet(we.writeServers[channel].requestCycles).get();
         cycles = tagcycles.cycles;
`endif
         let done <- we.writeServers[channel].done.get();
         let tag <- toGet(writeTags[channel]).get();
         indication[channel].transferFromFpgaDone(objId, base, tag, cycles);
      endrule
   end

   function DmaRequest dmaRequestInterface(Integer channel);
      return (interface DmaRequest;
	 method Action writeRequestSize(Bit#(16) burstLenBytes);
	      writeRequestSizeReg <= truncate(burstLenBytes);
	 endmethod
	 method Action readRequestSize(Bit#(16) burstLenBytes);
	      readRequestSizeReg <= truncate(burstLenBytes);
	 endmethod
	 method Action transferToFpga(Bit#(32) objId, Bit#(32) base, Bit#(32) bytes, Bit#(8) tag);
	      readCmds[channel].enq(MemengineCmd {sglId: truncate(objId),
						  base: extend(base),
						  burstLen: extend(readRequestSizeReg),
						  len: bytes,
						  tag: truncate(tag)
						  });
	 endmethod
	 method Action transferFromFpga(Bit#(32) objId, Bit#(32) base, Bit#(32) bytes, Bit#(8) tag);
	    writeCmds[channel].enq(MemengineCmd {sglId: truncate(objId),
						 base: extend(base),
						 burstLen: extend(writeRequestSizeReg),
						 len: bytes,
						 tag: truncate(tag)
						 });
	 endmethod
	 endinterface);
   endfunction
   function PipeIn#(Bit#(dsz)) writeServerData(MemWriteEngineServer#(dsz) s); return s.data; endfunction

   interface Vector request = genWith(dmaRequestInterface);
   interface readClient = vec(re.dmaClient);
   interface writeClient = vec(we.dmaClient);
   interface toFpga = map(toPipeOut,transferToFpgaFifo);
   interface fromFpga = map(mapPipeIn(memdatafToData), map(writeServerData, we.writeServers));
endmodule


take min of dwCount before truncating

take min of dwCount before truncating

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import FIFO         :: *;
import FIFOF        :: *;
import GetPut       :: *;
import Connectable  :: *;
import PCIE         :: *;
import DefaultValue :: *;
import MIMO         :: *;
import MIFO         :: *;
import Vector       :: *;
import ClientServer :: *;
import MemTypes     :: *;

import AxiMasterSlave :: *;

interface MemSlaveEngine#(numeric type buswidth);
    interface Client#(TLPData#(16), TLPData#(16)) tlp;
    interface MemSlave#(40,buswidth) slave;
    method Bool tlpOutFifoNotEmpty();
    interface Reg#(Bool) use4dw;
endinterface: MemSlaveEngine

module mkMemSlaveEngine#(PciId my_id)(MemSlaveEngine#(buswidth))
   provisos (Div#(buswidth, 8, busWidthBytes),
	     Div#(buswidth, 32, busWidthWords),
	     Bits#(Vector#(busWidthWords, Bit#(32)), buswidth),
            Log#(busWidthBytes,beatShift),
	     Add#(aaa, 32, buswidth),
	     Add#(bbb, buswidth, 256),
	     Add#(ccc, TMul#(8, busWidthWords), 64),
	     Add#(ddd, TMul#(32, busWidthWords), 256),
	     Add#(eee, busWidthWords, 8),
	     Add#(1, a__, busWidthWords),
	     Add#(busWidthWords, b__, 4)
      );

    let beat_shift = fromInteger(valueOf(beatShift));

    FIFOF#(TLPData#(16)) tlpOutFifo <- mkFIFOF;
    FIFOF#(TLPData#(16)) tlpInFifo <- mkFIFOF;
    FIFO#(TLPData#(16)) tlpWriteHeaderFifo <- mkFIFO;
    FIFO#(TLPLength)      writeDwCountFifo <- mkFIFO;
    FIFO#(Bool)             writeIs3dwFifo <- mkFIFO;

    Reg#(Bit#(7)) hitReg <- mkReg(0);
    Reg#(Bool) use4dwReg <- mkReg(True);

    // default configuration for MIMO is for guarded enq() and deq().
    // However, the implicit guard only checks for space for 1 element for enq(), and availability of 1 element for deq().
    MIMOConfiguration mimoCfg = defaultValue;
   MIFO#(4,busWidthWords,8,Bit#(32)) completionMimo <- mkMIFO();
   MIFO#(4,busWidthWords,8,TLPTag) completionTagMimo <- mkMIFO();

    MIMO#(busWidthWords,4,8,Bit#(32)) writeDataMimo <- mkMIMO(mimoCfg);
    Reg#(Bit#(9)) writeBurstCount <- mkReg(0);
    Reg#(TLPLength)  writeDwCount <- mkReg(0); // how many 4 byte (double) words to send
    Reg#(LUInt#(4))    tlpDwCount <- mkReg(0); // how many to send in the next tlp (at most 4)
    Reg#(Bool)            lastTlp <- mkReg(False); // if the next tlp sent is the last one
    Reg#(Bool)    writeInProgress <- mkReg(False);
    FIFOF#(TLPTag) writeTag <- mkSizedFIFOF(16);
    FIFOF#(TLPTag) doneTag <- mkSizedFIFOF(16);

    function LUInt#(4) tlpWordCount(TLPData#(16) tlp);
       if (tlp.be == 16'h0000)
	  return 0;
       else if (tlp.be == 16'h000f || tlp.be == 16'hf000)
	  return 1;
       else if (tlp.be == 16'h00ff || tlp.be == 16'hff00)
	  return 2;
       else if (tlp.be == 16'h0fff || tlp.be == 16'hfff0)
	  return 3;
       else if (tlp.be == 16'hffff)
	  return 4;
       else
	  return 0;
    endfunction

   rule writeHeaderTlp if (!writeInProgress && writeDataMimo.deqReadyN(1));
      let tlp <- toGet(tlpWriteHeaderFifo).get();
      let dwCount <- toGet(writeDwCountFifo).get();
      let is3dw <- toGet(writeIs3dwFifo).get();

      TLPMemory4DWHeader hdr_4dw = unpack(tlp.data);

      TLPMemoryIO3DWHeader hdr_3dw = unpack(tlp.data);
      if (is3dw) begin
	 if (hdr_3dw.format != MEM_WRITE_3DW_DATA)
	    $display("MemSlaveEngine: expecting MEM_WRITE_3DW_DATA, got %d", hdr_3dw.format);
	 Vector#(4, Bit#(32)) v = writeDataMimo.first();
	 writeDataMimo.deq(1);
	 hdr_3dw.data = byteSwap(v[0]);
	 tlp.be = 16'hffff;
	 if (dwCount == 1)
	    tlp.eof = True;
	 dwCount = dwCount - 1;
	 tlp.data = pack(hdr_3dw);
      end
      else begin
	 tlp.be = 16'hffff;
      end

      tlpOutFifo.enq(tlp);
      $display("writeHeaderTlp dwCount=%d", dwCount);
      writeDwCount <= dwCount;
      tlpDwCount <= truncate(min(4,unpack(dwCount)));
      lastTlp <= (dwCount <= 4);
      writeInProgress <= (dwCount != 0);
      if (dwCount == 0) begin
	 doneTag.enq(writeTag.first());
	 writeTag.deq();
      end

   endrule

   rule writeTlps if (writeInProgress && writeDataMimo.deqReadyN(tlpDwCount));
      TLPData#(16) tlp = defaultValue;
      tlp.sof = False;
      Vector#(4, Bit#(32)) v = unpack(0);

      // The MIMO implicit guard only checks for availability of 1 element
      // so we explicitly check for the number of elements required
      writeDataMimo.deq(tlpDwCount);
      v = writeDataMimo.first();
      let dwCount = writeDwCount - extend(pack(tlpDwCount));
      writeDwCount <= dwCount;
      tlpDwCount <= truncate(min(4,unpack(dwCount)));
      lastTlp <= (dwCount <= 4);
      if (tlpDwCount == 4)
	 tlp.be = 16'hffff;
      else if (tlpDwCount == 3)
	 tlp.be = 16'hfff0;
      else if (tlpDwCount == 2)
	 tlp.be = 16'hff00;
      else if (tlpDwCount == 1)
	 tlp.be = 16'hf000;
      tlp.eof = lastTlp;
      if (lastTlp) begin
	 writeInProgress <= False;
	 doneTag.enq(writeTag.first());
	 writeTag.deq();
	 $display("writeDwCount=%d will be zero", writeDwCount);
      end

      for (Integer i = 0; i < 4; i = i + 1)
	 tlp.data[(i+1)*32-1:i*32] = byteSwap(v[3-i]);
      tlpOutFifo.enq(tlp);
   endrule

   Reg#(TLPTag) lastTag <- mkReg(0);
   FIFOF#(TLPData#(16)) tlpDecodeFifo <- mkFIFOF();
   FIFOF#(LUInt#(4))    tlpWordCountFifo <- mkFIFOF();
   rule tlpInRule;
      let tlp <- toGet(tlpInFifo).get();
      tlpWordCountFifo.enq(tlpWordCount(tlp));
      tlpDecodeFifo.enq(tlp);
   endrule

   rule handleTlpRule;
      let tlp = tlpDecodeFifo.first;
      let count = tlpWordCountFifo.first;
      Bool handled = False;
      TLPMemoryIO3DWHeader h = unpack(tlp.data);
      hitReg <= tlp.hit;
      TLPMemoryIO3DWHeader hdr_3dw = unpack(tlp.data);
      TLPCompletionHeader hdr_completion = unpack(tlp.data);
      Vector#(4, Bit#(32)) vec = unpack(0);
      Vector#(4, Bit#(32)) tlpvec = unpack(tlp.data);

      if (!tlp.sof) begin
	 vec = reverse(tlpvec);
	 // The MIMO implicit guard only checks for space to enqueue 1 element
	 // so we explicitly check for the number of elements required
	 // otherwise elements in the queue will be overwritten.
	 if (completionMimo.enqReady()
	    && completionTagMimo.enqReady())
	    begin
	       completionMimo.enq(count, vec);
	       Vector#(4, TLPTag) tagvec = replicate(lastTag);
	       completionTagMimo.enq(count, tagvec);
	       handled = True;
	    end
      end
      else if (hdr_3dw.format == MEM_WRITE_3DW_DATA
	       && hdr_3dw.pkttype == COMPLETION
	       && completionMimo.enqReady()
	       && completionTagMimo.enqReady()) begin
	    vec[0] = hdr_3dw.data;
	    completionMimo.enq(1, vec);
            TLPTag tag = hdr_completion.tag;
	    lastTag <= tag;
	    completionTagMimo.enq(1, replicate(tag));
	    handled = True;
      end
      //$display("tlpIn handled=%d tlp=%h\n", handled, tlp);
      if (handled) begin
	 tlpDecodeFifo.deq();
	 tlpWordCountFifo.deq();
      end
   endrule

    interface Client        tlp;
        interface request = toGet(tlpOutFifo);
        interface response = toPut(tlpInFifo);
    endinterface
    interface MemSlave slave;
   interface MemWriteServer write_server;
      interface Put writeReq;
         method Action put(MemRequest#(40) req)
	      if (writeBurstCount == 0);

	      let burstLen = req.burstLen >> beat_shift;
	      let addr = req.addr;
	      let awid = req.tag;
	      let writeIs3dw = False;

	      TLPLength tlplen = fromInteger(valueOf(busWidthWords))*(extend(burstLen));
	      TLPData#(16) tlp = defaultValue;
	      tlp.sof = True;
	      tlp.eof = False;
	      tlp.hit = 7'h00;
	      tlp.be = 16'hffff;

	      $display("slave.writeAddr tlplen=%d burstLen=%d", tlplen, burstLen);
	      if ((addr >> 32) != 0) begin
		 TLPMemory4DWHeader hdr_4dw = defaultValue;
		 hdr_4dw.format = MEM_WRITE_4DW_DATA;
		 hdr_4dw.tag = extend(awid);
		 hdr_4dw.reqid = my_id;
		 hdr_4dw.nosnoop = SNOOPING_REQD;
		 hdr_4dw.addr = addr[40-1:2];
		 hdr_4dw.length = tlplen;
		 hdr_4dw.firstbe = 4'hf;
		 hdr_4dw.lastbe = (tlplen > 1) ? 4'hf : 0;
		 tlp.data = pack(hdr_4dw);
	      end
	      else begin
		 writeIs3dw = True;
		 TLPMemoryIO3DWHeader hdr_3dw = defaultValue;
		 hdr_3dw.format = MEM_WRITE_3DW_DATA;
		 hdr_3dw.tag = extend(awid);
		 hdr_3dw.reqid = my_id;
		 hdr_3dw.nosnoop = SNOOPING_REQD;
		 hdr_3dw.addr = addr[32-1:2];
		 hdr_3dw.length = tlplen;
		 hdr_3dw.firstbe = 4'hf;
		 hdr_3dw.lastbe = (tlplen > 1) ? 4'hf : 0;

		 tlp.be = 16'hfff0; // no data word in this TLP

		 tlp.data = pack(hdr_3dw);
	      end
	      tlpWriteHeaderFifo.enq(tlp);
	      writeDwCountFifo.enq(tlplen);
	      writeIs3dwFifo.enq(writeIs3dw);
	      writeBurstCount <= zeroExtend(burstLen);
	      writeTag.enq(extend(awid));
           endmethod
	endinterface
      interface Put writeData;
         method Action put(MemData#(buswidth) wdata)
	      provisos (Bits#(Vector#(busWidthWords, Bit#(32)), busWidth)) if (writeBurstCount > 0 && writeDataMimo.enqReadyN(fromInteger(valueOf(busWidthWords))));

	      writeBurstCount <= writeBurstCount - 1;
	      Vector#(busWidthWords, Bit#(32)) v = unpack(wdata.data);
	      writeDataMimo.enq(fromInteger(valueOf(busWidthWords)), v);
           endmethod
       endinterface
      interface Get writeDone;
         method ActionValue#(Bit#(ObjectTagSize)) get();
	      let tag = doneTag.first();
	      doneTag.deq();
	      return truncate(tag);
           endmethod
	endinterface
   endinterface
   interface MemReadServer read_server;
      interface Put readReq;
         method Action put(MemRequest#(40) req) if (!writeInProgress && !writeDataMimo.deqReady());
	      let burstLen = req.burstLen >> beat_shift;
	      let addr = req.addr;
	      let arid = req.tag;

	       TLPData#(16) tlp = defaultValue;
	       tlp.sof = True;
	       tlp.eof = True;
	       tlp.hit = 7'h00;
	       TLPLength tlplen = fromInteger(valueOf(busWidthWords))*(extend(burstLen));
	       if (addr[39:32] != 0) begin
		   TLPMemory4DWHeader hdr_4dw = defaultValue;
		   hdr_4dw.format = MEM_READ_4DW_NO_DATA;
		   hdr_4dw.tag = extend(arid);
		   hdr_4dw.reqid = my_id;
		   hdr_4dw.nosnoop = SNOOPING_REQD;
		   hdr_4dw.addr = addr[40-1:2];
		   hdr_4dw.length = tlplen;
		   hdr_4dw.firstbe = 4'hf;
		   hdr_4dw.lastbe = (tlplen > 1) ? 4'hf : 0;
		   tlp.data = pack(hdr_4dw);
		   tlp.be = 16'hffff;
	       end
	       else begin
		   TLPMemoryIO3DWHeader hdr_3dw = defaultValue;
		   hdr_3dw.format = MEM_READ_3DW_NO_DATA;
		   hdr_3dw.tag = extend(arid);
		   hdr_3dw.reqid = my_id;
		   hdr_3dw.nosnoop = SNOOPING_REQD;
		   hdr_3dw.addr = addr[32-1:2];
		   hdr_3dw.length = tlplen;
		   hdr_3dw.firstbe = 4'hf;
		   hdr_3dw.lastbe = (tlplen > 1) ? 4'hf : 0;
		   tlp.data = pack(hdr_3dw);
		   tlp.be = 16'hfff0;
	       end
	       tlpOutFifo.enq(tlp);
           endmethod
       endinterface
      interface Get     readData;
         method ActionValue#(MemData#(buswidth)) get() if (completionMimo.deqReady()
							   && completionTagMimo.deqReady());
	      let data_v = completionMimo.first;
	      let tag_v = completionTagMimo.first;
	      completionMimo.deq();
	      completionTagMimo.deq();
              Bit#(buswidth) v = 0;
	      for (Integer i = 0; i < valueOf(busWidthWords); i = i+1)
		 v[(i+1)*32-1:i*32] = byteSwap(data_v[i]);
	      return MemData { data: v, tag: truncate(tag_v[0])};
           endmethod
	endinterface
   endinterface
    endinterface: slave
   method Bool tlpOutFifoNotEmpty() = tlpOutFifo.notEmpty;
   interface Reg use4dw = use4dwReg;
endmodule: mkMemSlaveEngine

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import FIFO         :: *;
import FIFOF        :: *;
import GetPut       :: *;
import Connectable  :: *;
import PCIE         :: *;
import DefaultValue :: *;
import MIMO         :: *;
import MIFO         :: *;
import Vector       :: *;
import ClientServer :: *;
import MemTypes     :: *;

import AxiMasterSlave :: *;

interface MemSlaveEngine#(numeric type buswidth);
    interface Client#(TLPData#(16), TLPData#(16)) tlp;
    interface MemSlave#(40,buswidth) slave;
    method Bool tlpOutFifoNotEmpty();
    interface Reg#(Bool) use4dw;
endinterface: MemSlaveEngine

module mkMemSlaveEngine#(PciId my_id)(MemSlaveEngine#(buswidth))
   provisos (Div#(buswidth, 8, busWidthBytes),
	     Div#(buswidth, 32, busWidthWords),
	     Bits#(Vector#(busWidthWords, Bit#(32)), buswidth),
            Log#(busWidthBytes,beatShift),
	     Add#(aaa, 32, buswidth),
	     Add#(bbb, buswidth, 256),
	     Add#(ccc, TMul#(8, busWidthWords), 64),
	     Add#(ddd, TMul#(32, busWidthWords), 256),
	     Add#(eee, busWidthWords, 8),
	     Add#(1, a__, busWidthWords),
	     Add#(busWidthWords, b__, 4)
      );

    let beat_shift = fromInteger(valueOf(beatShift));

    FIFOF#(TLPData#(16)) tlpOutFifo <- mkFIFOF;
    FIFOF#(TLPData#(16)) tlpInFifo <- mkFIFOF;
    FIFO#(TLPData#(16)) tlpWriteHeaderFifo <- mkFIFO;
    FIFO#(TLPLength)      writeDwCountFifo <- mkFIFO;
    FIFO#(Bool)             writeIs3dwFifo <- mkFIFO;

    Reg#(Bit#(7)) hitReg <- mkReg(0);
    Reg#(Bool) use4dwReg <- mkReg(True);

    // default configuration for MIMO is for guarded enq() and deq().
    // However, the implicit guard only checks for space for 1 element for enq(), and availability of 1 element for deq().
    MIMOConfiguration mimoCfg = defaultValue;
   MIFO#(4,busWidthWords,8,Bit#(32)) completionMimo <- mkMIFO();
   MIFO#(4,busWidthWords,8,TLPTag) completionTagMimo <- mkMIFO();

    MIMO#(busWidthWords,4,8,Bit#(32)) writeDataMimo <- mkMIMO(mimoCfg);
    Reg#(Bit#(9)) writeBurstCount <- mkReg(0);
    Reg#(TLPLength)  writeDwCount <- mkReg(0); // how many 4 byte (double) words to send
    Reg#(LUInt#(4))    tlpDwCount <- mkReg(0); // how many to send in the next tlp (at most 4)
    Reg#(Bool)            lastTlp <- mkReg(False); // if the next tlp sent is the last one
    Reg#(Bool)    writeInProgress <- mkReg(False);
    FIFOF#(TLPTag) writeTag <- mkSizedFIFOF(16);
    FIFOF#(TLPTag) doneTag <- mkSizedFIFOF(16);

    function LUInt#(4) tlpWordCount(TLPData#(16) tlp);
       if (tlp.be == 16'h0000)
	  return 0;
       else if (tlp.be == 16'h000f || tlp.be == 16'hf000)
	  return 1;
       else if (tlp.be == 16'h00ff || tlp.be == 16'hff00)
	  return 2;
       else if (tlp.be == 16'h0fff || tlp.be == 16'hfff0)
	  return 3;
       else if (tlp.be == 16'hffff)
	  return 4;
       else
	  return 0;
    endfunction

   rule writeHeaderTlp if (!writeInProgress && writeDataMimo.deqReadyN(1));
      let tlp <- toGet(tlpWriteHeaderFifo).get();
      let dwCount <- toGet(writeDwCountFifo).get();
      let is3dw <- toGet(writeIs3dwFifo).get();

      TLPMemory4DWHeader hdr_4dw = unpack(tlp.data);

      TLPMemoryIO3DWHeader hdr_3dw = unpack(tlp.data);
      if (is3dw) begin
	 if (hdr_3dw.format != MEM_WRITE_3DW_DATA)
	    $display("MemSlaveEngine: expecting MEM_WRITE_3DW_DATA, got %d", hdr_3dw.format);
	 Vector#(4, Bit#(32)) v = writeDataMimo.first();
	 writeDataMimo.deq(1);
	 hdr_3dw.data = byteSwap(v[0]);
	 tlp.be = 16'hffff;
	 if (dwCount == 1)
	    tlp.eof = True;
	 dwCount = dwCount - 1;
	 tlp.data = pack(hdr_3dw);
      end
      else begin
	 tlp.be = 16'hffff;
      end

      tlpOutFifo.enq(tlp);
      $display("writeHeaderTlp dwCount=%d", dwCount);
      writeDwCount <= dwCount;
      tlpDwCount <= min(4,truncate(unpack(dwCount)));
      lastTlp <= (dwCount <= 4);
      writeInProgress <= (dwCount != 0);
      if (dwCount == 0) begin
	 doneTag.enq(writeTag.first());
	 writeTag.deq();
      end

   endrule

   rule writeTlps if (writeInProgress && writeDataMimo.deqReadyN(tlpDwCount));
      TLPData#(16) tlp = defaultValue;
      tlp.sof = False;
      Vector#(4, Bit#(32)) v = unpack(0);

      // The MIMO implicit guard only checks for availability of 1 element
      // so we explicitly check for the number of elements required
      writeDataMimo.deq(tlpDwCount);
      v = writeDataMimo.first();
      let dwCount = writeDwCount - extend(pack(tlpDwCount));
      writeDwCount <= dwCount;
      tlpDwCount <= min(4,truncate(unpack(dwCount)));
      lastTlp <= (dwCount <= 4);
      if (tlpDwCount == 4)
	 tlp.be = 16'hffff;
      else if (tlpDwCount == 3)
	 tlp.be = 16'hfff0;
      else if (tlpDwCount == 2)
	 tlp.be = 16'hff00;
      else if (tlpDwCount == 1)
	 tlp.be = 16'hf000;
      tlp.eof = lastTlp;
      if (lastTlp) begin
	 writeInProgress <= False;
	 doneTag.enq(writeTag.first());
	 writeTag.deq();
	 $display("writeDwCount=%d will be zero", writeDwCount);
      end

      for (Integer i = 0; i < 4; i = i + 1)
	 tlp.data[(i+1)*32-1:i*32] = byteSwap(v[3-i]);
      tlpOutFifo.enq(tlp);
   endrule

   Reg#(TLPTag) lastTag <- mkReg(0);
   FIFOF#(TLPData#(16)) tlpDecodeFifo <- mkFIFOF();
   FIFOF#(LUInt#(4))    tlpWordCountFifo <- mkFIFOF();
   rule tlpInRule;
      let tlp <- toGet(tlpInFifo).get();
      tlpWordCountFifo.enq(tlpWordCount(tlp));
      tlpDecodeFifo.enq(tlp);
   endrule

   rule handleTlpRule;
      let tlp = tlpDecodeFifo.first;
      let count = tlpWordCountFifo.first;
      Bool handled = False;
      TLPMemoryIO3DWHeader h = unpack(tlp.data);
      hitReg <= tlp.hit;
      TLPMemoryIO3DWHeader hdr_3dw = unpack(tlp.data);
      TLPCompletionHeader hdr_completion = unpack(tlp.data);
      Vector#(4, Bit#(32)) vec = unpack(0);
      Vector#(4, Bit#(32)) tlpvec = unpack(tlp.data);

      if (!tlp.sof) begin
	 vec = reverse(tlpvec);
	 // The MIMO implicit guard only checks for space to enqueue 1 element
	 // so we explicitly check for the number of elements required
	 // otherwise elements in the queue will be overwritten.
	 if (completionMimo.enqReady()
	    && completionTagMimo.enqReady())
	    begin
	       completionMimo.enq(count, vec);
	       Vector#(4, TLPTag) tagvec = replicate(lastTag);
	       completionTagMimo.enq(count, tagvec);
	       handled = True;
	    end
      end
      else if (hdr_3dw.format == MEM_WRITE_3DW_DATA
	       && hdr_3dw.pkttype == COMPLETION
	       && completionMimo.enqReady()
	       && completionTagMimo.enqReady()) begin
	    vec[0] = hdr_3dw.data;
	    completionMimo.enq(1, vec);
            TLPTag tag = hdr_completion.tag;
	    lastTag <= tag;
	    completionTagMimo.enq(1, replicate(tag));
	    handled = True;
      end
      //$display("tlpIn handled=%d tlp=%h\n", handled, tlp);
      if (handled) begin
	 tlpDecodeFifo.deq();
	 tlpWordCountFifo.deq();
      end
   endrule

    interface Client        tlp;
        interface request = toGet(tlpOutFifo);
        interface response = toPut(tlpInFifo);
    endinterface
    interface MemSlave slave;
   interface MemWriteServer write_server;
      interface Put writeReq;
         method Action put(MemRequest#(40) req)
	      if (writeBurstCount == 0);

	      let burstLen = req.burstLen >> beat_shift;
	      let addr = req.addr;
	      let awid = req.tag;
	      let writeIs3dw = False;

	      TLPLength tlplen = fromInteger(valueOf(busWidthWords))*(extend(burstLen));
	      TLPData#(16) tlp = defaultValue;
	      tlp.sof = True;
	      tlp.eof = False;
	      tlp.hit = 7'h00;
	      tlp.be = 16'hffff;

	      $display("slave.writeAddr tlplen=%d burstLen=%d", tlplen, burstLen);
	      if ((addr >> 32) != 0) begin
		 TLPMemory4DWHeader hdr_4dw = defaultValue;
		 hdr_4dw.format = MEM_WRITE_4DW_DATA;
		 hdr_4dw.tag = extend(awid);
		 hdr_4dw.reqid = my_id;
		 hdr_4dw.nosnoop = SNOOPING_REQD;
		 hdr_4dw.addr = addr[40-1:2];
		 hdr_4dw.length = tlplen;
		 hdr_4dw.firstbe = 4'hf;
		 hdr_4dw.lastbe = (tlplen > 1) ? 4'hf : 0;
		 tlp.data = pack(hdr_4dw);
	      end
	      else begin
		 writeIs3dw = True;
		 TLPMemoryIO3DWHeader hdr_3dw = defaultValue;
		 hdr_3dw.format = MEM_WRITE_3DW_DATA;
		 hdr_3dw.tag = extend(awid);
		 hdr_3dw.reqid = my_id;
		 hdr_3dw.nosnoop = SNOOPING_REQD;
		 hdr_3dw.addr = addr[32-1:2];
		 hdr_3dw.length = tlplen;
		 hdr_3dw.firstbe = 4'hf;
		 hdr_3dw.lastbe = (tlplen > 1) ? 4'hf : 0;

		 tlp.be = 16'hfff0; // no data word in this TLP

		 tlp.data = pack(hdr_3dw);
	      end
	      tlpWriteHeaderFifo.enq(tlp);
	      writeDwCountFifo.enq(tlplen);
	      writeIs3dwFifo.enq(writeIs3dw);
	      writeBurstCount <= zeroExtend(burstLen);
	      writeTag.enq(extend(awid));
           endmethod
	endinterface
      interface Put writeData;
         method Action put(MemData#(buswidth) wdata)
	      provisos (Bits#(Vector#(busWidthWords, Bit#(32)), busWidth)) if (writeBurstCount > 0 && writeDataMimo.enqReadyN(fromInteger(valueOf(busWidthWords))));

	      writeBurstCount <= writeBurstCount - 1;
	      Vector#(busWidthWords, Bit#(32)) v = unpack(wdata.data);
	      writeDataMimo.enq(fromInteger(valueOf(busWidthWords)), v);
           endmethod
       endinterface
      interface Get writeDone;
         method ActionValue#(Bit#(ObjectTagSize)) get();
	      let tag = doneTag.first();
	      doneTag.deq();
	      return truncate(tag);
           endmethod
	endinterface
   endinterface
   interface MemReadServer read_server;
      interface Put readReq;
         method Action put(MemRequest#(40) req) if (!writeInProgress && !writeDataMimo.deqReady());
	      let burstLen = req.burstLen >> beat_shift;
	      let addr = req.addr;
	      let arid = req.tag;

	       TLPData#(16) tlp = defaultValue;
	       tlp.sof = True;
	       tlp.eof = True;
	       tlp.hit = 7'h00;
	       TLPLength tlplen = fromInteger(valueOf(busWidthWords))*(extend(burstLen));
	       if (addr[39:32] != 0) begin
		   TLPMemory4DWHeader hdr_4dw = defaultValue;
		   hdr_4dw.format = MEM_READ_4DW_NO_DATA;
		   hdr_4dw.tag = extend(arid);
		   hdr_4dw.reqid = my_id;
		   hdr_4dw.nosnoop = SNOOPING_REQD;
		   hdr_4dw.addr = addr[40-1:2];
		   hdr_4dw.length = tlplen;
		   hdr_4dw.firstbe = 4'hf;
		   hdr_4dw.lastbe = (tlplen > 1) ? 4'hf : 0;
		   tlp.data = pack(hdr_4dw);
		   tlp.be = 16'hffff;
	       end
	       else begin
		   TLPMemoryIO3DWHeader hdr_3dw = defaultValue;
		   hdr_3dw.format = MEM_READ_3DW_NO_DATA;
		   hdr_3dw.tag = extend(arid);
		   hdr_3dw.reqid = my_id;
		   hdr_3dw.nosnoop = SNOOPING_REQD;
		   hdr_3dw.addr = addr[32-1:2];
		   hdr_3dw.length = tlplen;
		   hdr_3dw.firstbe = 4'hf;
		   hdr_3dw.lastbe = (tlplen > 1) ? 4'hf : 0;
		   tlp.data = pack(hdr_3dw);
		   tlp.be = 16'hfff0;
	       end
	       tlpOutFifo.enq(tlp);
           endmethod
       endinterface
      interface Get     readData;
         method ActionValue#(MemData#(buswidth)) get() if (completionMimo.deqReady()
							   && completionTagMimo.deqReady());
	      let data_v = completionMimo.first;
	      let tag_v = completionTagMimo.first;
	      completionMimo.deq();
	      completionTagMimo.deq();
              Bit#(buswidth) v = 0;
	      for (Integer i = 0; i < valueOf(busWidthWords); i = i+1)
		 v[(i+1)*32-1:i*32] = byteSwap(data_v[i]);
	      return MemData { data: v, tag: truncate(tag_v[0])};
           endmethod
	endinterface
   endinterface
    endinterface: slave
   method Bool tlpOutFifoNotEmpty() = tlpOutFifo.notEmpty;
   interface Reg use4dw = use4dwReg;
endmodule: mkMemSlaveEngine


use a Completion Buffer to handle out of order command ring buffer reads

// Copyright (c) 2013 Quanta Research Cambridge, Inc.
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import FIFO::*;
import FIFOF::*;
import BRAMFIFO::*;
import GetPut::*;
import StmtFSM::*;
import ClientServer::*;

import PortalMemory::*;
import Dma::*;
import GetPutF::*;
import CompletionBuffer::*;

import RingTypes::*;
import RingBuffer::*;
import CopyEngine::*;
import EchoEngine::*;
import NopEngine::*;

interface RingRequest;
   method Action set(Bit#(1) cmd, Bit#(3) regist, Bit#(64) addr);
   method Action setCmdLast(Bit#(64) addr);
   method Action setCmdFirst(Bit#(64) addr);
   method Action setStatusLast(Bit#(64) addr);
   method Action setStatusFirst(Bit#(64) addr);
   method Action get(Bit#(1) cmd, Bit#(3) regist);
   method Action hwenable(Bit#(1) en);
   method Action doCommandIndirect(Bit#(64) pointer, Bit#(64) addr);
   method Action doCommandImmediate(Bit#(64) data);
endinterface

interface RingIndication;
   method Action setResult(Bit#(1) cmd, Bit#(3) regist, Bit#(64) addr);
   method Action getResult(Bit#(1) cmd, Bit#(3) regist, Bit#(64) addr);
endinterface

module mkRingRequest#(RingIndication indication,
		      DmaReadServer#(64) dma_read_chan,
		      DmaWriteServer#(64) dma_write_chan,
		      DmaReadServer#(64) cmd_read_chan,
		      DmaWriteServer#(64) status_write_chan )(RingRequest);

   ServerF#(Bit#(64), Bit#(64)) copyEngine <- mkCopyEngine(dma_read_chan, dma_write_chan);
   ServerF#(Bit#(64), Bit#(64)) nopEngine <- mkNopEngine();
   ServerF#(Bit#(64), Bit#(64)) echoEngine <- mkEchoEngine();
   CompletionBuffer#(4,Void) fetchComplete <mkCompletionBuffer;

   RingBuffer cmdRing <- mkRingBuffer;
   RingBuffer statusRing <- mkRingBuffer;
   Reg#(Bool) hwenabled <- mkReg(False);
   Reg#(Bool) cmdBusy <- mkReg(False);
   Reg#(DmaData#(64)) cmd <- mkReg(DmaData{data:0, tag:0});
   Reg#(Bit#(4)) ii <- mkReg(0);
   Reg#(Bit#(4)) respCtr <- mkReg(0);
   Reg#(Bit#(4)) dispCtr <- mkReg(0);
   Reg#(Bit#(6)) statusTag <- mkReg(0);
   Reg#(Bool) cmdFetchEn <- mkReg(False);

   let engineselect = cmd.data[63:56];
   function ServerF#(Bit#(64), Bit#(64)) cmdifc();
      if (engineselect == zeroExtend(pack(CmdNOP)))
	 return nopEngine;
      else if (engineselect == zeroExtend(pack(CmdCOPY)))
	 return copyEngine;
      else if (engineselect == zeroExtend(pack(CmdECHO)))
	 return echoEngine;
      else
	 return nopEngine;
   endfunction

   Stmt cmdFetch =
   seq
      $display("cmdFetch FSM TOP");
      while (True)
	 seq
	    if (hwenabled)
	       seq
	       if (cmdRing.notEmpty())
		  seq
		     $display ("cmdFetch handle=%h address=%h burst=%h tag=%h",
		      cmdRing.mempointer, cmdRing.bufferlast, 8, cmdFetchTag);
		     cmd_read_chan.readReq.put(
			DmaRequest{pointer: cmdRing.mempointer,
			   offset: cmdRing.bufferlast, burstLen: 8, tag: cmdFetchTagfetchCompletion.reserve()});
		     cmdRing.popfetch();
		  endseq
	       endseq
	 endseq
   endseq;

   Stmt cmdDispatch =
   seq
      while (True) seq
	 $display("cmdDispatch FSM TOP");
	 seq
	    action
	       let rv <- cmd_read_chan.readData.get();
	       cmd <= rv;
	       completionBuffer.complete(Tuple2(rv.tag,void));
	    endaction
	    // wait a cycle so cmd is valid!
//	    $display("cmdDispatch 0 tag=%h %h", cmd.tag, cmd.data);
	    cmdifc.request.put(cmd.data);
	 endseq
	 for (dispCtr <= 1; dispCtr < 8; dispCtr <= dispCtr + 1)
	    action
	       let rv <- cmd_read_chan.readData.get();
//	       $display("  cmdDispatch %h tag=%h %h", dispCtr, rv.tag, rv.data);
	       cmdifc.request.put(rv.data);
	    endaction
      endseq
   endseq;

   rule finishCmdReads;
      let v <- completionBuffer.drain.get();
      cmdRing.popAck();
      $display("pop ack");
   endrule

   Stmt responseArbiter =
   seq
      while(True) seq
	 if (statusRing.notFull() && copyEngine.response.notEmpty())
	    seq
//	       $display("responseArbiter copyEngine completion");
//	       $display("status write handle=%d address=%h burst=%h tag=%h",
//		  statusRing.mempointer, statusRing.bufferfirst, 8, statusTag);
	       status_write_chan.writeReq.put(
		  DmaRequest{pointer: statusRing.mempointer,
		     offset: statusRing.bufferfirst, burstLen: 8, tag: statusTag});
	       for (respCtr <= 0; respCtr < 8; respCtr <= respCtr + 1)
		  action
		     let rv <- copyEngine.response.get();
		     status_write_chan.writeData.put(DmaData{data: rv, tag: statusTag});
		  endaction
	       statusRing.push(64);
	       statusTag <= statusTag + 1;
	    endseq

	 if (statusRing.notFull() && echoEngine.response.notEmpty())
	    seq
//	       $display("responseArbiter echoEngine completion");
//	       $display("status write handle=%d address=%h burst=%h tag=%h",
//		  statusRing.mempointer, statusRing.bufferfirst, 8, statusTag);
	       status_write_chan.writeReq.put(
		  DmaRequest{pointer: statusRing.mempointer,
		     offset: statusRing.bufferfirst, burstLen: 8, tag: statusTag});
	       for (respCtr <= 0; respCtr < 8; respCtr <= respCtr + 1)
		  action
		     let rv <- echoEngine.response.get();
		     status_write_chan.writeData.put(DmaData{data: rv, tag: statusTag});
		  endaction
	       statusRing.push(64);
	       statusTag <= statusTag + 1;
	    endseq

      endseq
   endseq;

   rule writeAck;
      let tag <- status_write_chan.writeDone.get();
      //$display("status write done tag=%h", tag);
   endrule

   mkAutoFSM (cmdFetch);
   mkAutoFSM (cmdDispatch);
   mkAutoFSM (responseArbiter);

      // to start a command, doCommand fires off a memory read to the
      // specified address. when it comes back, the doCommandRule will
      // handle it
      method Action doCommandIndirect(Bit#(64) pointer, Bit#(64) addr);
	 cmd_read_chan.readReq.put(
				   DmaRequest{pointer: truncate(pointer),
	 offset: truncate(addr), burstLen: 8, tag: cmdFetchTag});
   	 cmdFetchTag <= cmdFetchTag + 1;
      endmethod

      method Action doCommandImmediate(Bit#(64) data);
      	 $display("doCommandImmediate %h", data);
      endmethod

      method Action set(Bit#(1) _cmd, Bit#(3) regist, Bit#(64) addr);
	 if (_cmd == 0)
	    cmdRing.configifc.set(regist, truncate(addr));
	 else
	    statusRing.configifc.set(regist, truncate(addr));
	 indication.setResult(_cmd, regist, addr);
      endmethod

      method Action setCmdFirst(Bit#(64) addr);
	    cmdRing.configifc.setFirst(truncate(addr));
      endmethod

      method Action setStatusFirst(Bit#(64) addr);
	    statusRing.configifc.setFirst(truncate(addr));
      endmethod

      method Action setCmdLast(Bit#(64) addr);
	    cmdRing.configifc.setLast(truncate(addr));
      endmethod

      method Action setStatusLast(Bit#(64) addr);
	    statusRing.configifc.setLast(truncate(addr));
      endmethod

      method Action get(Bit#(1) _cmd, Bit#(3) regist);
	 if (_cmd == 0)
	    indication.getResult(0, regist,
	       zeroExtend(cmdRing.configifc.get(regist)));
	 else
	    indication.getResult(1, regist,
	       zeroExtend(statusRing.configifc.get(regist)));
      endmethod

      method Action hwenable(Bit#(1) en);
	 $display ("hwenable set to %h", en);
	 hwenabled <= en == 1;
      endmethod

endmodule

// Copyright (c) 2013 Quanta Research Cambridge, Inc.
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import FIFO::*;
import FIFOF::*;
import BRAMFIFO::*;
import GetPut::*;
import StmtFSM::*;
import ClientServer::*;

import PortalMemory::*;
import Dma::*;
import GetPutF::*;

import RingTypes::*;
import RingBuffer::*;
import CopyEngine::*;
import EchoEngine::*;
import NopEngine::*;

interface RingRequest;
   method Action set(Bit#(1) cmd, Bit#(3) regist, Bit#(64) addr);
   method Action setCmdLast(Bit#(64) addr);
   method Action setCmdFirst(Bit#(64) addr);
   method Action setStatusLast(Bit#(64) addr);
   method Action setStatusFirst(Bit#(64) addr);
   method Action get(Bit#(1) cmd, Bit#(3) regist);
   method Action hwenable(Bit#(1) en);
   method Action doCommandIndirect(Bit#(64) pointer, Bit#(64) addr);
   method Action doCommandImmediate(Bit#(64) data);
endinterface

interface RingIndication;
   method Action setResult(Bit#(1) cmd, Bit#(3) regist, Bit#(64) addr);
   method Action getResult(Bit#(1) cmd, Bit#(3) regist, Bit#(64) addr);
endinterface

module mkRingRequest#(RingIndication indication,
		      DmaReadServer#(64) dma_read_chan,
		      DmaWriteServer#(64) dma_write_chan,
		      DmaReadServer#(64) cmd_read_chan,
		      DmaWriteServer#(64) status_write_chan )(RingRequest);

   ServerF#(Bit#(64), Bit#(64)) copyEngine <- mkCopyEngine(dma_read_chan, dma_write_chan);
   ServerF#(Bit#(64), Bit#(64)) nopEngine <- mkNopEngine();
   ServerF#(Bit#(64), Bit#(64)) echoEngine <- mkEchoEngine();

   RingBuffer cmdRing <- mkRingBuffer;
   RingBuffer statusRing <- mkRingBuffer;
   Reg#(Bool) hwenabled <- mkReg(False);
   Reg#(Bool) cmdBusy <- mkReg(False);
   Reg#(DmaData#(64)) cmd <- mkReg(DmaData{data:0, tag:0});
   Reg#(Bit#(4)) ii <- mkReg(0);
   Reg#(Bit#(4)) respCtr <- mkReg(0);
   Reg#(Bit#(4)) dispCtr <- mkReg(0);
   Reg#(Bit#(6)) cmdFetchTag <- mkReg(0);
   Reg#(Bit#(6)) statusTag <- mkReg(0);
   Reg#(Bool) cmdFetchEn <- mkReg(False);

   let engineselect = cmd.data[63:56];
   function ServerF#(Bit#(64), Bit#(64)) cmdifc();
      if (engineselect == zeroExtend(pack(CmdNOP)))
	 return nopEngine;
      else if (engineselect == zeroExtend(pack(CmdCOPY)))
	 return copyEngine;
      else if (engineselect == zeroExtend(pack(CmdECHO)))
	 return echoEngine;
      else
	 return nopEngine;
   endfunction

   Stmt cmdFetch =
   seq
//      $display("cmdFetch FSM TOP");
      while (True)
	 seq
	    if (hwenabled)
	       seq
	       if (cmdRing.bufferfirst != cmdRing.bufferlast)
		  seq
//		     $display ("cmdFetch handle=%h address=%h burst=%h tag=%h",
//		      cmdRing.mempointer, cmdRing.bufferlast, 8, cmdFetchTag);
		     cmd_read_chan.readReq.put(
			DmaRequest{pointer: cmdRing.mempointer,
			   offset: cmdRing.bufferlast, burstLen: 8, tag: cmdFetchTag});
		     cmdRing.pop(64);
		     cmdFetchTag <= cmdFetchTag + 1;
		  endseq
	       endseq
	 endseq
   endseq;

   Stmt cmdDispatch =
   seq
      while (True) seq
//	 $display("cmdDispatch FSM TOP");
	 seq
	    action
	       let rv <- cmd_read_chan.readData.get();
	       cmd <= rv;
	    endaction
	    // wait a cycle so cmd is valid!
//	    $display("cmdDispatch 0 tag=%h %h", cmd.tag, cmd.data);
	    cmdifc.request.put(cmd.data);
	 endseq
	 for (dispCtr <= 1; dispCtr < 8; dispCtr <= dispCtr + 1)
	    action
	       let rv <- cmd_read_chan.readData.get();
//	       $display("  cmdDispatch %h tag=%h %h", dispCtr, rv.tag, rv.data);
	       cmdifc.request.put(rv.data);
	    endaction
      endseq
   endseq;

   Stmt responseArbiter =
   seq
      while(True) seq
	 if (statusRing.notFull() && copyEngine.response.notEmpty())
	    seq
//	       $display("responseArbiter copyEngine completion");
//	       $display("status write handle=%d address=%h burst=%h tag=%h",
//		  statusRing.mempointer, statusRing.bufferfirst, 8, statusTag);
	       status_write_chan.writeReq.put(
		  DmaRequest{pointer: statusRing.mempointer,
		     offset: statusRing.bufferfirst, burstLen: 8, tag: statusTag});
	       for (respCtr <= 0; respCtr < 8; respCtr <= respCtr + 1)
		  action
		     let rv <- copyEngine.response.get();
		     status_write_chan.writeData.put(DmaData{data: rv, tag: statusTag});
		  endaction
	       statusRing.push(64);
	       statusTag <= statusTag + 1;
	    endseq

	 if (statusRing.notFull() && echoEngine.response.notEmpty())
	    seq
//	       $display("responseArbiter echoEngine completion");
//	       $display("status write handle=%d address=%h burst=%h tag=%h",
//		  statusRing.mempointer, statusRing.bufferfirst, 8, statusTag);
	       status_write_chan.writeReq.put(
		  DmaRequest{pointer: statusRing.mempointer,
		     offset: statusRing.bufferfirst, burstLen: 8, tag: statusTag});
	       for (respCtr <= 0; respCtr < 8; respCtr <= respCtr + 1)
		  action
		     let rv <- echoEngine.response.get();
		     status_write_chan.writeData.put(DmaData{data: rv, tag: statusTag});
		  endaction
	       statusRing.push(64);
	       statusTag <= statusTag + 1;
	    endseq

      endseq
   endseq;

   rule writeAck;
      let tag <- status_write_chan.writeDone.get();
      //$display("status write done tag=%h", tag);
   endrule

   mkAutoFSM (cmdFetch);
   mkAutoFSM (cmdDispatch);
   mkAutoFSM (responseArbiter);

      // to start a command, doCommand fires off a memory read to the
      // specified address. when it comes back, the doCommandRule will
      // handle it
      method Action doCommandIndirect(Bit#(64) pointer, Bit#(64) addr);
	 cmd_read_chan.readReq.put(
				   DmaRequest{pointer: truncate(pointer),
	 offset: truncate(addr), burstLen: 8, tag: cmdFetchTag});
   	 cmdFetchTag <= cmdFetchTag + 1;
      endmethod

      method Action doCommandImmediate(Bit#(64) data);
      	 $display("doCommandImmediate %h", data);
      endmethod

      method Action set(Bit#(1) _cmd, Bit#(3) regist, Bit#(64) addr);
	 if (_cmd == 0)
	    cmdRing.configifc.set(regist, truncate(addr));
	 else
	    statusRing.configifc.set(regist, truncate(addr));
	 indication.setResult(_cmd, regist, addr);
      endmethod

      method Action setCmdFirst(Bit#(64) addr);
	    cmdRing.configifc.setFirst(truncate(addr));
      endmethod

      method Action setStatusFirst(Bit#(64) addr);
	    statusRing.configifc.setFirst(truncate(addr));
      endmethod

      method Action setCmdLast(Bit#(64) addr);
	    cmdRing.configifc.setLast(truncate(addr));
      endmethod

      method Action setStatusLast(Bit#(64) addr);
	    statusRing.configifc.setLast(truncate(addr));
      endmethod

      method Action get(Bit#(1) _cmd, Bit#(3) regist);
	 if (_cmd == 0)
	    indication.getResult(0, regist,
	       zeroExtend(cmdRing.configifc.get(regist)));
	 else
	    indication.getResult(1, regist,
	       zeroExtend(statusRing.configifc.get(regist)));
      endmethod

      method Action hwenable(Bit#(1) en);
	 $display ("hwenable set to %h", en);
	 hwenabled <= en == 1;
      endmethod

endmodule


connect AxiEth DMA

connect AxiEth DMA

import BuildVector::*;
import Connectable::*;
import GetPut::*;
import FIFOF::*;
import BRAM::*;
import Probe::*;
import StmtFSM::*;
import TriState::*;
import Vector::*;

import ConnectalXilinxCells::*;
import ConnectalConfig::*;
import CtrlMux::*;
import HostInterface::*;
import MemTypes::*;
import AxiBits::*;

import AxiIntcBvi::*;
import AxiEthBvi::*;
import AxiDmaBvi::*;
import EthPins::*;

interface AxiEthTestRequest;
   method Action reset();
   method Action setupDma(Bit#(32) memref);
   method Action read(Bit#(32) addr);
   method Action write(Bit#(32) addr, Bit#(32) value);
endinterface

interface AxiEthTestIndication;
   method Action irqChanged(Bit#(1) newIrq, Bit#(4) intrSources);
   method Action readDone(Bit#(32) value);
   method Action writeDone();
   method Action resetDone();
endinterface

interface AxiEth;
   interface AxiEthTestRequest request;
   interface Vector#(2, MemReadClient#(DataBusWidth)) dmaReadClient;
   interface Vector#(2, MemWriteClient#(DataBusWidth)) dmaWriteClient;
   interface AxiEthPins pins;
endinterface

module mkAxiEth#(HostInterface host, AxiEthTestIndication ind)(AxiEth);

   let clock <- exposeCurrentClock();
   let reset <- exposeCurrentReset();

   let axiIntcBvi <- mkAxiIntcBvi(clock, reset);
   let axiDmaBvi <- mkAxiDmaBvi(clock,clock,clock,clock,reset);
   let axiEthBvi <- mkAxiEthBvi(host.tsys_clk_200mhz_buf,
				clock, reset, clock,
				reset, reset, reset, reset);

   Reg#(Bit#(32)) objId <- mkReg(0);

   let irqLevel <- mkReg(0);
   let intrLevel <- mkReg(0);
   Bit#(4) intr = {
      axiDmaBvi.s2mm.introut(),
      axiDmaBvi.mm2s.introut(),
      axiEthBvi.mac.irq(),
      axiEthBvi.interrupt()
      };

   rule rl_intr;
      axiIntcBvi.intr(intr);
   endrule

   rule rl_intr_indication;
      let irq = axiIntcBvi.irq;
      irqLevel <= irq;
      intrLevel <= intr;

      if (irq != irqLevel || intr != intrLevel) begin
	 ind.irqChanged(irq, intr);
	 $display("irq changed irq=%h intr sources %h", irq, intr);
      end
   endrule

   FIFOF#(BRAMRequest#(Bit#(32),Bit#(32))) reqFifo <- mkFIFOF();
   FIFOF#(Bit#(32))                       dataFifo <- mkFIFOF();
   // packet data and status from the ethernet
   mkConnection(axiEthBvi.m_axis_rxd, axiDmaBvi.s_axis_s2mm);
   mkConnection(axiEthBvi.m_axis_rxs, axiDmaBvi.s_axis_s2mm_sts);

   // packet data and control to the ethernet
   mkConnection(axiDmaBvi.m_axis_mm2s,       axiEthBvi.s_axis_txd);
   mkConnection(axiDmaBvi.m_axis_mm2s_cntrl, axiEthBvi.s_axis_txc);

   Axi4MasterBits#(32,32,MemTagSize,Empty) m_axi_mm2s = toAxi4MasterBits(axiDmaBvi.m_axi_mm2s);
   Axi4MasterBits#(32,32,MemTagSize,Empty) m_axi_s2mm = toAxi4MasterBits(axiDmaBvi.m_axi_s2mm);
   Axi4MasterBits#(32,32,MemTagSize,Empty) m_axi_sg = toAxi4MasterBits(axiDmaBvi.m_axi_sg);

   Axi4SlaveLiteBits#(9,32) axiIntcSlaveLite = toAxi4SlaveBits(axiIntcBvi.s_axi);
   PhysMemSlave#(18,32) axiIntcMemSlave <- mkPhysMemSlave(axiIntcSlaveLite);
   PhysMemSlave#(18,32) axiDmaMemSlave <- mkPhysMemSlave(axiDmaBvi.s_axi_lite);
   Axi4SlaveLiteBits#(18,32) axiEthSlaveLite = toAxi4SlaveBits(axiEthBvi.s_axi);
   PhysMemSlave#(18,32) axiEthMemSlave <- mkPhysMemSlave(axiEthSlaveLite);
   PhysMemSlave#(20,32) memSlaveMux    <- mkPhysMemSlaveMux(vec(axiIntcMemSlave, axiDmaMemSlave, axiEthMemSlave));

   FIFOF#(Bit#(32)) dfifo <- mkFIFOF();

   rule rl_axieth;
      axiEthBvi.signal.detect(1); // drive to 1 if not using optical transceiver, else use signal from transceiver
   endrule

   rule rl_rdata;
      let rdata <- memSlaveMux.read_server.readData.get();
      ind.readDone(rdata.data);
   endrule

   rule rl_wdata;
      let wdata <- toGet(dfifo).get();
       memSlaveMux.write_server.writeData.put(MemData {data: wdata, tag: 0});
   endrule

   rule rl_writeDone;
      let tag <- memSlaveMux.write_server.writeDone.get();
      ind.writeDone();
   endrule

   interface AxiEthTestRequest request;
      method Action reset();
      endmethod
      method Action setupDma(Bit#(32) memref);
	 objId <= memref;
      endmethod
      method Action read(Bit#(32) addr);
	 memSlaveMux.read_server.readReq.put(PhysMemRequest { addr: truncate(addr), burstLen: 4, tag: 0 });
      endmethod
      method Action write(Bit#(32) addr, Bit#(32) value);
	 memSlaveMux.write_server.writeReq.put(PhysMemRequest { addr: truncate(addr), burstLen: 4, tag: 0 });
	 dfifo.enq(value);
      endmethod
   endinterface
   interface AxiEthPins pins;
      interface EthPins eth;
	 interface AxiethbviMgt mgt   = axiEthBvi.mgt;
	 interface AxiethbviMdio sfp = axiEthBvi.sfp;
	 interface Clock deleteme_unused_clock = clock;
	 interface Reset deleteme_unused_reset = reset;
      endinterface
   endinterface
   interface Vector dmaReadClient = map(toMemReadClient(objId), vec(m_axi_mm2s, m_axi_sg));
   interface Vector dmaWriteClient = map(toMemWriteClient(objId), vec(m_axi_s2mm, m_axi_sg));
endmodule

import BuildVector::*;
import Connectable::*;
import GetPut::*;
import FIFOF::*;
import BRAM::*;
import Probe::*;
import StmtFSM::*;
import TriState::*;
import Vector::*;

import ConnectalXilinxCells::*;
import ConnectalConfig::*;
import CtrlMux::*;
import HostInterface::*;
import MemTypes::*;
import AxiBits::*;

import AxiIntcBvi::*;
import AxiEthBvi::*;
import AxiDmaBvi::*;
import EthPins::*;

interface AxiEthTestRequest;
   method Action reset();
   method Action read(Bit#(32) addr);
   method Action write(Bit#(32) addr, Bit#(32) value);
endinterface

interface AxiEthTestIndication;
   method Action irqChanged(Bit#(1) newIrq, Bit#(4) intrSources);
   method Action readDone(Bit#(32) value);
   method Action writeDone();
   method Action resetDone();
endinterface

interface AxiEth;
   interface AxiEthTestRequest request;
   interface Vector#(2, MemReadClient#(DataBusWidth)) dmaReadClient;
   interface Vector#(2, MemWriteClient#(DataBusWidth)) dmaWriteClient;
   interface AxiEthPins pins;
endinterface

module mkAxiEth#(HostInterface host, AxiEthTestIndication ind)(AxiEth);

   let clock <- exposeCurrentClock();
   let reset <- exposeCurrentReset();

   let axiIntcBvi <- mkAxiIntcBvi(clock, reset);
   let axiDmaBvi <- mkAxiDmaBvi(clock,clock,clock,clock,reset);
   let axiEthBvi <- mkAxiEthBvi(host.tsys_clk_200mhz_buf,
				clock, reset, clock,
				reset, reset, reset, reset);

   let irqLevel <- mkReg(0);
   let intrLevel <- mkReg(0);
   Bit#(4) intr = {
      axiDmaBvi.s2mm.introut(),
      axiDmaBvi.mm2s.introut(),
      axiEthBvi.mac.irq(),
      axiEthBvi.interrupt()
      };

   rule rl_intr;
      axiIntcBvi.intr(intr);
   endrule

   rule rl_intr_indication;
      let irq = axiIntcBvi.irq;
      irqLevel <= irq;
      intrLevel <= intr;

      if (irq != irqLevel || intr != intrLevel) begin
	 ind.irqChanged(irq, intr);
	 $display("irq changed irq=%h intr sources %h", irq, intr);
      end
   endrule

   FIFOF#(BRAMRequest#(Bit#(32),Bit#(32))) reqFifo <- mkFIFOF();
   FIFOF#(Bit#(32))                       dataFifo <- mkFIFOF();
   // packet data and status from the ethernet
   mkConnection(axiEthBvi.m_axis_rxd, axiDmaBvi.s_axis_s2mm);
   mkConnection(axiEthBvi.m_axis_rxs, axiDmaBvi.s_axis_s2mm_sts);

   // packet data and control to the ethernet
   mkConnection(axiDmaBvi.m_axis_mm2s,       axiEthBvi.s_axis_txd);
   mkConnection(axiDmaBvi.m_axis_mm2s_cntrl, axiEthBvi.s_axis_txc);

   Axi4MasterBits#(32,32,MemTagSize,Empty) m_axi_mm2s = toAxi4MasterBits(axiDmaBvi.m_axi_mm2s);
   Axi4MasterBits#(32,32,MemTagSize,Empty) m_axi_s2mm = toAxi4MasterBits(axiDmaBvi.m_axi_s2mm);
   Axi4MasterBits#(32,32,MemTagSize,Empty) m_axi_sg = toAxi4MasterBits(axiDmaBvi.m_axi_sg);

   Axi4SlaveLiteBits#(9,32) axiIntcSlaveLite = toAxi4SlaveBits(axiIntcBvi.s_axi);
   PhysMemSlave#(18,32) axiIntcMemSlave <- mkPhysMemSlave(axiIntcSlaveLite);
   PhysMemSlave#(18,32) axiDmaMemSlave <- mkPhysMemSlave(axiDmaBvi.s_axi_lite);
   Axi4SlaveLiteBits#(18,32) axiEthSlaveLite = toAxi4SlaveBits(axiEthBvi.s_axi);
   PhysMemSlave#(18,32) axiEthMemSlave <- mkPhysMemSlave(axiEthSlaveLite);
   PhysMemSlave#(20,32) memSlaveMux    <- mkPhysMemSlaveMux(vec(axiIntcMemSlave, axiDmaMemSlave, axiEthMemSlave));

   FIFOF#(Bit#(32)) dfifo <- mkFIFOF();

   rule rl_axieth;
      axiEthBvi.signal.detect(1); // drive to 1 if not using optical transceiver, else use signal from transceiver
   endrule

   rule rl_rdata;
      let rdata <- memSlaveMux.read_server.readData.get();
      ind.readDone(rdata.data);
   endrule

   rule rl_wdata;
      let wdata <- toGet(dfifo).get();
       memSlaveMux.write_server.writeData.put(MemData {data: wdata, tag: 0});
   endrule

   rule rl_writeDone;
      let tag <- memSlaveMux.write_server.writeDone.get();
      ind.writeDone();
   endrule

   interface AxiEthTestRequest request;
      method Action reset();
      endmethod
      method Action read(Bit#(32) addr);
	 memSlaveMux.read_server.readReq.put(PhysMemRequest { addr: truncate(addr), burstLen: 4, tag: 0 });
      endmethod
      method Action write(Bit#(32) addr, Bit#(32) value);
	 memSlaveMux.write_server.writeReq.put(PhysMemRequest { addr: truncate(addr), burstLen: 4, tag: 0 });
	 dfifo.enq(value);
      endmethod
   endinterface
   interface AxiEthPins pins;
      interface EthPins eth;
	 interface AxiethbviMgt mgt   = axiEthBvi.mgt;
	 interface AxiethbviMdio sfp = axiEthBvi.sfp;
	 interface Clock deleteme_unused_clock = clock;
	 interface Reset deleteme_unused_reset = reset;
      endinterface
   endinterface
   interface Vector dmaReadClient = map(toMemReadClient, vec(m_axi_mm2s, m_axi_sg));
   //interface Vector dmaWriteClient = map(toMemWriteClient, vec(m_axi_s2mm, m_axi_sg));
endmodule


fixed asymmetricbram size parameter

fixed asymmetricbram size parameter

/*-
 * Copyright (c) 2013, 2014 Alexandre Joannou
 * Copyright (c) 2015 Han Wang
 * All rights reserved.
 *
 * This software was developed by SRI International and the University of
 * Cambridge Computer Laboratory under DARPA/AFRL contract FA8750-11-C-0249
 * ("MRC2"), as part of the DARPA MRC research programme.
 *
 * @BERI_LICENSE_HEADER_START@
 *
 * Licensed to BERI Open Systems C.I.C. (BERI) under one or more contributor
 * license agreements.  See the NOTICE file distributed with this work for
 * additional information regarding copyright ownership.  BERI licenses this
 * file to you under the BERI Hardware-Software License, Version 1.0 (the
 * "License"); you may not use this file except in compliance with the
 * License.  You may obtain a copy of the License at:
 *
 *   http://www.beri-open-systems.org/legal/license-1-0.txt
 *
 * Unless required by applicable law or agreed to in writing, Work distributed
 * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
 * CONDITIONS OF ANY KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations under the License.
 *
 * @BERI_LICENSE_HEADER_END@
 */

package AsymmetricBRAM;

import Connectable::*;
import ClientServer::*;
import FIFO :: *;
import FIFOF :: *;
import GetPut::*;
import SpecialFIFOs :: *;
import ConfigReg :: * ;
import Vector :: * ;
import Pipe::*;

// Interface to the asymmetric ram
interface AsymmetricBRAM#(  type rAddrT, type rDataT,
                            type wAddrT, type wDataT);
   interface Put#(Tuple2#(wAddrT, wDataT)) writeServer;
   interface Server#(rAddrT, rDataT) readServer;
endinterface

// Interface to the verilog module (using an altsyncram component)
interface VAsymBRAMIfc#(    type rAddrT,type rDataT,
                            type wAddrT, type wDataT);
    method Action read(rAddrT read_addr);
    method rDataT getRead();
    method Action write(wAddrT write_addr, wDataT write_data);
endinterface

// Assume narrower write port
// Wrapper for the verilog
import "BVI" AsymmetricBRAM =
module vAsymBRAM#(Bool hasOutputRegister)
                 (VAsymBRAMIfc#(raddr_t, rdata_t, waddr_t, wdata_t))
    provisos(
        Bits#(raddr_t, raddr_sz),
        Bits#(rdata_t, rdata_sz),
        Bits#(waddr_t, waddr_sz),
        Bits#(wdata_t, wdata_sz)
    );

    default_clock (CLK);
    default_reset no_reset;

    parameter   PIPELINED   = Bit#(1)'(pack(hasOutputRegister));
    parameter   WADDR_WIDTH = valueOf(waddr_sz);
    parameter   WDATA_WIDTH = valueOf(wdata_sz);
    parameter   RADDR_WIDTH = valueOf(raddr_sz);
    parameter   RDATA_WIDTH = valueOf(rdata_sz);
    parameter   MEMSIZE     = valueOf(TExp#(waddr_sz));

    method read(RADDR) enable(REN);
    method RDATA getRead();
    method write(WADDR,WDATA) enable(WEN);

    schedule (getRead) SBR (read, write);
    schedule (read)  C (read);
    schedule (write) C (write);
    schedule (getRead) CF (getRead);
    schedule (write) CF (read);

endmodule

module mkAsymmetricBRAM#(Bool hasOutputRegister, Bool hasForwarding, String name)
                               (AsymmetricBRAM#(raddr_t, rdata_t, waddr_t, wdata_t))
    provisos(
        Bits#(raddr_t, raddr_sz),
        Bits#(rdata_t, rdata_sz),
        Bits#(waddr_t, waddr_sz),
        Bits#(wdata_t, wdata_sz),
        Div#(rdata_sz,wdata_sz,ratio),
        Log#(ratio,offset_sz),
        Add#(raddr_sz,offset_sz,waddr_sz),
        Bits#(Vector#(ratio, wdata_t), rdata_sz)
    );
    AsymmetricBRAM#(raddr_t, rdata_t, waddr_t, wdata_t) ret_ifc;
    `ifndef SIMULATION
        ret_ifc <- mkAsymmetricBRAMVerilog(hasOutputRegister, hasForwarding);
    `else
        ret_ifc <- mkAsymmetricBRAMBluesim(hasOutputRegister, hasForwarding, name);
    `endif
    return ret_ifc;
endmodule

module mkAsymmetricBRAMVerilog#(Bool hasOutputRegister, Bool hasForwarding)
                               (AsymmetricBRAM#(raddr_t, rdata_t, waddr_t, wdata_t))
    provisos(
        Bits#(raddr_t, raddr_sz),
        Bits#(rdata_t, rdata_sz),
        Bits#(waddr_t, waddr_sz),
        Bits#(wdata_t, wdata_sz),
        Div#(rdata_sz,wdata_sz,ratio),
        Log#(ratio,offset_sz),
        Add#(raddr_sz,offset_sz,waddr_sz),
        Bits#(Vector#(ratio, wdata_t), rdata_sz)
    );
    FIFO#(Tuple2#(waddr_t, wdata_t)) writeReqFifo <- mkFIFO;
    FIFO#(raddr_t) readReqFifo <- mkFIFO;
    FIFO#(rdata_t) readDataFifo <- mkFIFO;
    FIFO#(void) readCtrlFifo <- mkFIFO;

    VAsymBRAMIfc#(raddr_t, rdata_t, waddr_t, wdata_t) bram <- vAsymBRAM(hasOutputRegister);

    rule do_read;
       let v <- toGet(readReqFifo).get;
       bram.read(v);
       readCtrlFifo.enq(?);
    endrule

    rule do_reponse;
       let v <- toGet(readCtrlFifo).get;
       let addr = bram.getRead;
       readDataFifo.enq(addr);
    endrule

    rule do_write;
       let v <- toGet(writeReqFifo).get;
       let addr = tpl_1(v);
       let data = tpl_2(v);
       bram.write(addr, data);
    endrule

    interface Put writeServer = toPut(writeReqFifo);
    interface readServer = (interface Server;
      interface Put request;
         method Action put(raddr_t addr);
            readReqFifo.enq(addr);
         endmethod
      endinterface
      interface Get response = toGet(readDataFifo);
   endinterface);

endmodule

import "BDPI" mem_create    = function ActionValue#(Bit#(64)) mem_create(msize_t size, rsize_t rsize, wsize_t wsize)
                              provisos (Bits#(msize_t, msize_sz),
                                        Bits#(rsize_t, rsize_sz),
                                        Bits#(wsize_t, wsize_sz));
import "BDPI" mem_clean     = function Action mem_clean(Bit#(64) mem_ptr);
import "BDPI" mem_read      = function ActionValue#(rdata_t) mem_read(Bit#(64) mem_ptr, raddr_t raddr)
                              provisos (Bits#(raddr_t, raddr_sz),
                                        Bits#(rdata_t, rdata_sz));
import "BDPI" mem_write     = function Action mem_write(Bit#(64) mem_ptr, waddr_t waddr, wdata_t wdata)
                              provisos (Bits#(waddr_t, waddr_sz),
                                        Bits#(wdata_t, wdata_sz));

module mkAsymmetricBRAMBluesim#(Bool hasOutputRegister, Bool hasForwarding, String name)
                               (AsymmetricBRAM#(raddr_t, rdata_t, waddr_t, wdata_t))
    provisos(
        Bits#(raddr_t, raddr_sz),
        Bits#(rdata_t, rdata_sz),
        Bits#(waddr_t, waddr_sz),
        Bits#(wdata_t, wdata_sz),
        Div#(rdata_sz,wdata_sz,ratio),
        Log#(ratio,offset_sz),
        Add#(raddr_sz,offset_sz,waddr_sz),
        Bits#(Vector#(ratio, wdata_t), rdata_sz)
    );
    let verbose = False;
    FIFO#(Tuple2#(waddr_t, wdata_t)) writeReqFifo <- mkFIFO;
    FIFO#(raddr_t) readReqFifo <- mkFIFO;
    FIFO#(rdata_t) readDataFifo <- mkFIFO;

    Reg#(Bit#(64))  mem_ptr         <- mkRegU();
    Reg#(Bool)      isInitialized   <- mkReg(False);

    Reg#(Bit#(32)) cntr <- mkReg(0);
    rule every1 if (verbose);
      cntr <= cntr + 1;
    endrule

    (* execution_order = "do_read, do_write" *)
    rule do_read (isInitialized);
       let v <- toGet(readReqFifo).get;
       rdata_t rdata  <- mem_read(mem_ptr, v);
       readDataFifo.enq(rdata);
       if (verbose) $display("%s %d: read data from %x with index %x = %x", name, cntr, mem_ptr, v, rdata);
    endrule

    rule do_write (isInitialized);
       let v <- toGet(writeReqFifo).get;
       let writeAddr = tpl_1(v);
       let writeData = tpl_2(v);
       mem_write(mem_ptr, writeAddr, writeData);
       if (verbose) $display("%s %d: write data to %x with index %x = %x", name, cntr, mem_ptr, writeAddr, writeData);
    endrule

    rule do_init (!isInitialized);
       let tmp <- mem_create(fromInteger(valueOf(TExp#(waddr_sz))),
                             fromInteger(valueOf(rdata_sz)),
                             fromInteger(valueOf(wdata_sz)));
       mem_ptr <= tmp;
       isInitialized <= True;

       if (valueOf(raddr_sz) > 64) begin
          $display("raddr_sz larger than 64 is not supported");
       end
    endrule

   interface Put writeServer = toPut(writeReqFifo);
   interface readServer = (interface Server;
      interface Put request;
         method Action put(raddr_t addr);
            readReqFifo.enq(addr);
         endmethod
      endinterface
      interface Get response = toGet(readDataFifo);
   endinterface);
endmodule

endpackage

/*-
 * Copyright (c) 2013, 2014 Alexandre Joannou
 * Copyright (c) 2015 Han Wang
 * All rights reserved.
 *
 * This software was developed by SRI International and the University of
 * Cambridge Computer Laboratory under DARPA/AFRL contract FA8750-11-C-0249
 * ("MRC2"), as part of the DARPA MRC research programme.
 *
 * @BERI_LICENSE_HEADER_START@
 *
 * Licensed to BERI Open Systems C.I.C. (BERI) under one or more contributor
 * license agreements.  See the NOTICE file distributed with this work for
 * additional information regarding copyright ownership.  BERI licenses this
 * file to you under the BERI Hardware-Software License, Version 1.0 (the
 * "License"); you may not use this file except in compliance with the
 * License.  You may obtain a copy of the License at:
 *
 *   http://www.beri-open-systems.org/legal/license-1-0.txt
 *
 * Unless required by applicable law or agreed to in writing, Work distributed
 * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
 * CONDITIONS OF ANY KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations under the License.
 *
 * @BERI_LICENSE_HEADER_END@
 */

package AsymmetricBRAM;

import Connectable::*;
import ClientServer::*;
import FIFO :: *;
import FIFOF :: *;
import GetPut::*;
import SpecialFIFOs :: *;
import ConfigReg :: * ;
import Vector :: * ;
import Pipe::*;

// Interface to the asymmetric ram
interface AsymmetricBRAM#(  type rAddrT, type rDataT,
                            type wAddrT, type wDataT);
   interface Put#(Tuple2#(wAddrT, wDataT)) writeServer;
   interface Server#(rAddrT, rDataT) readServer;
endinterface

// Interface to the verilog module (using an altsyncram component)
interface VAsymBRAMIfc#(    type rAddrT,type rDataT,
                            type wAddrT, type wDataT);
    method Action read(rAddrT read_addr);
    method rDataT getRead();
    method Action write(wAddrT write_addr, wDataT write_data);
endinterface

// Wrapper for the verilog
import "BVI" AsymmetricBRAM =
module vAsymBRAM#(Bool hasOutputRegister)
                 (VAsymBRAMIfc#(raddr_t, rdata_t, waddr_t, wdata_t))
    provisos(
        Bits#(raddr_t, raddr_sz),
        Bits#(rdata_t, rdata_sz),
        Bits#(waddr_t, waddr_sz),
        Bits#(wdata_t, wdata_sz)
    );

    default_clock (CLK);
    default_reset no_reset;

    parameter   PIPELINED   = Bit#(1)'(pack(hasOutputRegister));
    parameter   WADDR_WIDTH = valueOf(waddr_sz);
    parameter   WDATA_WIDTH = valueOf(wdata_sz);
    parameter   RADDR_WIDTH = valueOf(raddr_sz);
    parameter   RDATA_WIDTH = valueOf(rdata_sz);
    parameter   MEMSIZE     = valueOf(TExp#(raddr_sz));

    method read(RADDR) enable(REN);
    method RDATA getRead();
    method write(WADDR,WDATA) enable(WEN);

    schedule (getRead) SBR (read, write);
    schedule (read)  C (read);
    schedule (write) C (write);
    schedule (getRead) CF (getRead);
    schedule (write) CF (read);

endmodule

module mkAsymmetricBRAM#(Bool hasOutputRegister, Bool hasForwarding, String name)
                               (AsymmetricBRAM#(raddr_t, rdata_t, waddr_t, wdata_t))
    provisos(
        Bits#(raddr_t, raddr_sz),
        Bits#(rdata_t, rdata_sz),
        Bits#(waddr_t, waddr_sz),
        Bits#(wdata_t, wdata_sz),
        Div#(rdata_sz,wdata_sz,ratio),
        Log#(ratio,offset_sz),
        Add#(raddr_sz,offset_sz,waddr_sz),
        Bits#(Vector#(ratio, wdata_t), rdata_sz)
    );
    AsymmetricBRAM#(raddr_t, rdata_t, waddr_t, wdata_t) ret_ifc;
    `ifndef SIMULATION
        ret_ifc <- mkAsymmetricBRAMVerilog(hasOutputRegister, hasForwarding);
    `else
        ret_ifc <- mkAsymmetricBRAMBluesim(hasOutputRegister, hasForwarding, name);
    `endif
    return ret_ifc;
endmodule

module mkAsymmetricBRAMVerilog#(Bool hasOutputRegister, Bool hasForwarding)
                               (AsymmetricBRAM#(raddr_t, rdata_t, waddr_t, wdata_t))
    provisos(
        Bits#(raddr_t, raddr_sz),
        Bits#(rdata_t, rdata_sz),
        Bits#(waddr_t, waddr_sz),
        Bits#(wdata_t, wdata_sz),
        Div#(rdata_sz,wdata_sz,ratio),
        Log#(ratio,offset_sz),
        Add#(raddr_sz,offset_sz,waddr_sz),
        Bits#(Vector#(ratio, wdata_t), rdata_sz)
    );
    FIFO#(Tuple2#(waddr_t, wdata_t)) writeReqFifo <- mkFIFO;
    FIFO#(raddr_t) readReqFifo <- mkFIFO;
    FIFO#(rdata_t) readDataFifo <- mkFIFO;
    FIFO#(void) readCtrlFifo <- mkFIFO;

    VAsymBRAMIfc#(raddr_t, rdata_t, waddr_t, wdata_t) bram <- vAsymBRAM(hasOutputRegister);

    rule do_read;
       let v <- toGet(readReqFifo).get;
       bram.read(v);
       readCtrlFifo.enq(?);
    endrule

    rule do_reponse;
       let v <- toGet(readCtrlFifo).get;
       let addr = bram.getRead;
       readDataFifo.enq(addr);
    endrule

    rule do_write;
       let v <- toGet(writeReqFifo).get;
       let addr = tpl_1(v);
       let data = tpl_2(v);
       bram.write(addr, data);
    endrule

    interface Put writeServer = toPut(writeReqFifo);
    interface readServer = (interface Server;
      interface Put request;
         method Action put(raddr_t addr);
            readReqFifo.enq(addr);
         endmethod
      endinterface
      interface Get response = toGet(readDataFifo);
   endinterface);

endmodule

import "BDPI" mem_create    = function ActionValue#(Bit#(64)) mem_create(msize_t size, rsize_t rsize, wsize_t wsize)
                              provisos (Bits#(msize_t, msize_sz),
                                        Bits#(rsize_t, rsize_sz),
                                        Bits#(wsize_t, wsize_sz));
import "BDPI" mem_clean     = function Action mem_clean(Bit#(64) mem_ptr);
import "BDPI" mem_read      = function ActionValue#(rdata_t) mem_read(Bit#(64) mem_ptr, raddr_t raddr)
                              provisos (Bits#(raddr_t, raddr_sz),
                                        Bits#(rdata_t, rdata_sz));
import "BDPI" mem_write     = function Action mem_write(Bit#(64) mem_ptr, waddr_t waddr, wdata_t wdata)
                              provisos (Bits#(waddr_t, waddr_sz),
                                        Bits#(wdata_t, wdata_sz));

module mkAsymmetricBRAMBluesim#(Bool hasOutputRegister, Bool hasForwarding, String name)
                               (AsymmetricBRAM#(raddr_t, rdata_t, waddr_t, wdata_t))
    provisos(
        Bits#(raddr_t, raddr_sz),
        Bits#(rdata_t, rdata_sz),
        Bits#(waddr_t, waddr_sz),
        Bits#(wdata_t, wdata_sz),
        Div#(rdata_sz,wdata_sz,ratio),
        Log#(ratio,offset_sz),
        Add#(raddr_sz,offset_sz,waddr_sz),
        Bits#(Vector#(ratio, wdata_t), rdata_sz)
    );
    let verbose = False;
    FIFO#(Tuple2#(waddr_t, wdata_t)) writeReqFifo <- mkFIFO;
    FIFO#(raddr_t) readReqFifo <- mkFIFO;
    FIFO#(rdata_t) readDataFifo <- mkFIFO;

    Reg#(Bit#(64))  mem_ptr         <- mkRegU();
    Reg#(Bool)      isInitialized   <- mkReg(False);

    Reg#(Bit#(32)) cntr <- mkReg(0);
    rule every1 if (verbose);
      cntr <= cntr + 1;
    endrule

    (* execution_order = "do_read, do_write" *)
    rule do_read (isInitialized);
       let v <- toGet(readReqFifo).get;
       rdata_t rdata  <- mem_read(mem_ptr, v);
       readDataFifo.enq(rdata);
       if (verbose) $display("%s %d: read data from %x with index %x = %x", name, cntr, mem_ptr, v, rdata);
    endrule

    rule do_write (isInitialized);
       let v <- toGet(writeReqFifo).get;
       let writeAddr = tpl_1(v);
       let writeData = tpl_2(v);
       mem_write(mem_ptr, writeAddr, writeData);
       if (verbose) $display("%s %d: write data to %x with index %x = %x", name, cntr, mem_ptr, writeAddr, writeData);
    endrule

    rule do_init (!isInitialized);
       let tmp <- mem_create(fromInteger(valueOf(TExp#(waddr_sz))),
                             fromInteger(valueOf(rdata_sz)),
                             fromInteger(valueOf(wdata_sz)));
       mem_ptr <= tmp;
       isInitialized <= True;

       if (valueOf(raddr_sz) > 64) begin
          $display("raddr_sz larger than 64 is not supported");
       end
    endrule

   interface Put writeServer = toPut(writeReqFifo);
   interface readServer = (interface Server;
      interface Put request;
         method Action put(raddr_t addr);
            readReqFifo.enq(addr);
         endmethod
      endinterface
      interface Get response = toGet(readDataFifo);
   endinterface);
endmodule

endpackage