add BYTE_ENABLES to MemServerInternal

add BYTE_ENABLES to MemServerInternal

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

// BSV Libraries
import BRAMFIFO::*;
import FIFO::*;
import FIFOF::*;
import Vector::*;
import GetPut::*;
import ClientServer::*;
import Assert::*;
import BRAM::*;
import DefaultValue::*;

// CONNECTAL Libraries
import MemTypes::*;
import ConnectalMemory::*;
import ConnectalClocks::*;
import SharedBuffMMU::*;
import ConnectalCompletionBuffer::*;
import ConnectalConfig::*;
import Ethernet::*;
`include "ConnectalProjectConfig.bsv"

typedef 32 BeatCountSize;

typedef 9 MMU_PIPELINE_DEPTH;

interface DmaDbg;
   method ActionValue#(Bit#(64)) getMemoryTraffic();
   method ActionValue#(DmaDbgRec) dbg();
endinterface

interface MemWriteInternal#(numeric type addrWidth, numeric type busWidth, numeric type numTags, numeric type numServers);
   interface DmaDbg dbg;
   interface Put#(TileControl) tileControl;
   interface PhysMemWriteClient#(addrWidth,busWidth) client;
   interface Vector#(numServers, MemWriteServer#(busWidth)) servers;
endinterface

interface MemReadInternal#(numeric type addrWidth, numeric type busWidth, numeric type numTags, numeric type numServers);
   interface DmaDbg dbg;
   interface Put#(TileControl) tileControl;
   interface PhysMemReadClient#(addrWidth,busWidth) client;
   interface Vector#(numServers, MemReadServer#(busWidth)) servers;
endinterface

function Bool sglid_outofrange(SGLId p);
   return ((p[15:0]) >= fromInteger(valueOf(MaxNumSGLists)));
endfunction

import RegFile::*;

typedef struct {MemRequest req;
		Bit#(TLog#(TMax#(1,numClients))) client; } LRec#(numeric type numClients, numeric type addrWidth) deriving(Bits);

typedef struct {MemRequest req;
		Bit#(addrWidth) pa;
		Bit#(MemTagSize) rename_tag;
		Bit#(TLog#(TMax#(1,numClients))) client; } RRec#(numeric type numClients, numeric type addrWidth) deriving(Bits);

typedef struct {Bit#(MemTagSize) req_tag;
		Bit#(BurstLenSize) req_burstLen;
		Bit#(MemTagSize) rename_tag;
		Bit#(TLog#(TMax#(1,numClients))) client;
		Bool last;
   } DRec#(numeric type numClients, numeric type addrWidth) deriving(Bits);

typedef struct {Bit#(MemTagSize) orig_tag;
		Bit#(TLog#(TMax#(1,numClients))) client; } RResp#(numeric type numClients, numeric type addrWidth) deriving(Bits);

typedef struct {DmaErrorType errorType;
		Bit#(32) pref; } DmaError deriving (Bits);

module mkMemReadInternal#(MemServerIndication ind,
			  Vector#(numMMUs,Server#(AddrTransRequest,AddrTransResponse#(addrWidth))) mmus)
   (MemReadInternal#(addrWidth, busWidth, numTags, numServers))
   provisos(Log#(busWidthBytes,beatShift)
	    ,Div#(busWidth,8,busWidthBytes)
	    ,Add#(beatShift, a__, 8)
	    ,Add#(b__, TLog#(numTags), MemTagSize)
	    ,Add#(beatShift, c__, BurstLenSize)
`ifdef BYTE_ENABLES
	    ,Add#(d__, TDiv#(busWidth, 8), ByteEnableSize)
`endif
	    );

   // stopping/killing infra
   Vector#(4,Reg#(Bool)) killv <- replicateM(mkReg(False));
   Vector#(4,Reg#(Bool)) stopv <- replicateM(mkReg(False));

   // stage 0: address translation (latency = MMU_PIPELINE_DEPTH)
   FIFO#(LRec#(numServers,addrWidth)) clientRequest <- mkSizedFIFO(valueOf(MMU_PIPELINE_DEPTH));
   // stage 1: address validation (latency = 1)
   FIFO#(RRec#(numServers,addrWidth))  serverRequest <- mkFIFO;
   // stage 2: read commands
   BRAM_Configure bramConfig = defaultValue;
   if (mainClockPeriod < 8)
      bramConfig.latency = 2;
   BRAM2Port#(Bit#(TLog#(numTags)), DRec#(numServers,addrWidth)) serverProcessing <- mkBRAM2Server(bramConfig);
   //BRAM2Port#(Bit#(TAdd#(TLog#(numTags),TSub#(BurstLenSize,beatShift))), MemData#(busWidth)) clientData <- mkBRAM2Server(bramConfig);
   FIFOF#(MemData#(busWidth)) clientData <- mkSizedFIFOF(4);
   // stage 3: read data
   FIFO#(MemData#(busWidth)) serverData <- mkFIFO;

   let verbose = True;

   RegFile#(Bit#(TLog#(numTags)),Tuple2#(Bool,Bit#(BurstLenSize))) clientBurstLen <- mkRegFileFull();
   Reg#(Bit#(BurstLenSize)) burstReg <- mkReg(0);
   Reg#(Bool)               firstReg <- mkReg(True);

   Reg#(Bit#(32))  beatCount <- mkReg(0);
   let beat_shift = fromInteger(valueOf(beatShift));
   TagGen#(numTags) tag_gen <- mkTagGen;

   FIFO#(Bit#(TAdd#(1,TLog#(TMax#(1,numServers))))) clientSelect <- mkFIFO;

   // performance analytics
   Reg#(Bit#(64)) cycle_cnt <- mkReg(0);
   Reg#(Bit#(64)) last_loadClient <- mkReg(0);
   Reg#(Bit#(64)) last_mmuResp <- mkReg(0);
   Reg#(Bit#(64)) last_comp <- mkReg(0);
   Reg#(Bit#(64)) last_readReq <- mkReg(0);
   Reg#(Bit#(64)) last_readData <- mkReg(0);
   (* fire_when_enabled *)
   rule cycle;
      cycle_cnt <= cycle_cnt+1;
   endrule

   FIFO#(DmaError) dmaErrorFifo <- mkFIFO();
   rule dmaError;
      let error <- toGet(dmaErrorFifo).get();
      ind.error(extend(pack(error.errorType)), error.pref, 0, 0);
   endrule

   rule checkMmuResp;
      let request <- toGet(clientRequest).get();
      let addrTransResponse <- mmus[request.req.sglId[31:16]].response.get;
      let physAddr = addrTransResponse.physAddr;
      let rename_tag <- tag_gen.getTag;
      let burstLenBeats = request.req.burstLen >> beat_shift;
      clientBurstLen.upd(truncate(rename_tag), tuple2(burstLenBeats == 1, burstLenBeats));

      serverRequest.enq(RRec{req:request.req, pa:physAddr, client:request.client, rename_tag:extend(rename_tag)});
      if (verbose) $display("mkMemReadInternal::checkMmuResp: client=%d, tag=%d rename_tag=%d burstLen=%d", request.client, request.req.tag, rename_tag, burstLenBeats);
      if (verbose) $display("mkMemReadInternal::mmuResp %d %d", request.client, cycle_cnt-last_mmuResp);
      last_mmuResp <= cycle_cnt;
   endrule

   rule read_data;
      let response <- toGet(serverData).get();
      let drq <- serverProcessing.portA.response.get;
      let tag = drq.req_tag;
      match { .last, .burstLen } = clientBurstLen.sub(truncate(response.tag));
      let first   = firstReg;
      if (first) begin
	 dynamicAssert(last == (burstLen==1), "Last incorrect");
      end
      if (last && burstLen != 1)
	 $display("rename_tag=%d tag=%d burstLen=%d last=%d", response.tag, tag, burstLen, last);
      Bit#(TLog#(numTags)) tt = truncate(response.tag);
      let client = drq.client;
      clientSelect.enq(extend(client));
      clientData.enq(response);
      $display("response: ", fshow(response));
      if (last) begin
	 tag_gen.returnTag(truncate(response.tag));
     $display("response: last! ", fshow(response));
      end
      last_readData <= cycle_cnt;
      if (verbose) $display("mkMemReadInternal::read_data cyclediff %d", cycle_cnt-last_readData);
      clientBurstLen.upd(truncate(response.tag), tuple2((burstLen-1 == 1),burstLen-1));
      firstReg <= response.last;
   endrule

   rule tag_completed;
      let tag <- tag_gen.complete;
      //serverProcessing.portB.request.put(BRAMRequest{write:False, address:tag, datain: ?, responseOnWrite: ?});
      //serverTag.enq(tag);
      if(verbose) $display("mkMemReadInternal::complete_burst0 %h", tag);
   endrule

   Vector#(numServers, MemReadServer#(busWidth)) sv = newVector;
   for(Integer i = 0; i < valueOf(numServers); i=i+1)
      sv[i] = (interface MemReadServer;
		  interface Put readReq;
		     method Action put(MemRequest req);
			last_loadClient <= cycle_cnt;
			let mmusel = req.sglId[31:16];
      			if (verbose) $display("mkMemReadInternal::loadClient server %d mmusel %d burstLen %d tag %d cycle %d",
			   i, mmusel, req.burstLen >> beat_shift, req.tag, cycle_cnt-last_loadClient);
			if (mmusel >= fromInteger(valueOf(numMMUs)))
			   dmaErrorFifo.enq(DmaError { errorType: DmaErrorMMUOutOfRange_r, pref: req.sglId });
   			else if (sglid_outofrange(req.sglId))
			   dmaErrorFifo.enq(DmaError { errorType: DmaErrorSGLIdOutOfRange_r, pref: req.sglId });
   			else if (stopv[req.tag[5:4]] == False) begin
   			   clientRequest.enq(LRec{req:req, client:fromInteger(i)});
   			   mmus[mmusel].request.put(AddrTransRequest{id:truncate(req.sglId),off:req.offset});
   			end
		     endmethod
		  endinterface
		  interface Get readData;
		     method ActionValue#(MemData#(busWidth)) get if (clientSelect.first == fromInteger(i));
			clientSelect.deq;
            let data = clientData.first;
            clientData.deq;
			if (verbose) $display("mkMemReadInternal::comp server %d data %x cycle %d", i, data.data, cycle_cnt-last_comp);
			last_comp <= cycle_cnt;
			return data;
		     endmethod
		  endinterface
	       endinterface);

   interface servers = sv;
   interface PhysMemReadClient client;
      interface Get readReq;
	 method ActionValue#(PhysMemRequest#(addrWidth,busWidth)) get();
	    let request <- toGet(serverRequest).get;
	    let req = request.req;
	    if (False && request.pa[31:24] != 0)
	       $display("mkMemReadInternal::req_ar: funny physAddr req.sglId=%d req.offset=%h physAddr=%h", req.sglId, req.offset, request.pa);
	    serverProcessing.portB.request.put(BRAMRequest{write:True, responseOnWrite:False, address:truncate(request.rename_tag),
						   datain:DRec{req_tag:req.tag, req_burstLen: req.burstLen, client:request.client, rename_tag:request.rename_tag, last:(req.burstLen == fromInteger(valueOf(busWidthBytes)))}});
	    //$display("mkMemReadInternal::readReq: client=%d, rename_tag=%d, physAddr=%h req.burstLen=%d beat_shift=%d last=%d", request.client,request.rename_tag,request.pa, req.burstLen, beat_shift, req.burstLen == beat_shift);
	    if (verbose) $display("mkMemReadInternal::read_client.readReq %d", cycle_cnt-last_readReq);
	    last_readReq <= cycle_cnt;
	    return PhysMemRequest{addr:request.pa, burstLen:req.burstLen, tag:request.rename_tag
`ifdef BYTE_ENABLES
				  , firstbe: truncate(request.req.firstbe), lastbe: truncate(request.req.lastbe)
`endif
	       };
	 endmethod
      endinterface
      interface Put readData;
	 method Action put(MemData#(busWidth) response);
	    serverData.enq(response);
	    serverProcessing.portA.request.put(BRAMRequest{write:False, address:truncate(response.tag), datain: ?, responseOnWrite: ?});
	    beatCount <= beatCount+1;
	 endmethod
      endinterface
   endinterface
   interface Put tileControl;
      method Action put(TileControl tc);
	 let tile = tc.tile;
	 let kv = True;
	 let sv = True;
	 if (tc.state == Running || tc.state == Stopped)
	    kv = False;
	 if (tc.state == Running)
	    sv = False;
	 killv[tile] <= kv;
	 stopv[tile] <= sv;
      endmethod
   endinterface
   interface DmaDbg dbg;
      method ActionValue#(DmaDbgRec) dbg();
	 return DmaDbgRec{x:0, y:0, z:0, w:0};
      endmethod
      method ActionValue#(Bit#(64)) getMemoryTraffic();
	 return extend(beatCount);
      endmethod
   endinterface
endmodule

module mkMemWriteInternal#(MemServerIndication ind,
			   Vector#(numMMUs,Server#(AddrTransRequest,AddrTransResponse#(addrWidth))) mmus)
   (MemWriteInternal#(addrWidth, busWidth, numTags, numServers))
   provisos(Log#(busWidthBytes,beatShift)
	    ,Div#(busWidth,8,busWidthBytes)
	    ,Add#(beatShift, a__, 8)
	    ,Add#(b__, TLog#(numTags), MemTagSize)
	    ,Add#(beatShift, c__, BurstLenSize)
`ifdef BYTE_ENABLES
        ,Add#(d__, TDiv#(busWidth, 8), ByteEnableSize)
`endif
	    );

   let verbose = False;

   // stopping/killing infra
   Vector#(4,Reg#(Bool)) killv <- replicateM(mkReg(False));
   Vector#(4,Reg#(Bool)) stopv <- replicateM(mkReg(False));

   // stage 0: address translation (latency = MMU_PIPELINE_DEPTH)
   FIFO#(LRec#(numServers,addrWidth)) clientRequest <- mkSizedFIFO(valueOf(MMU_PIPELINE_DEPTH));
   // stage 1: address validation (latency = 1)
   FIFO#(RRec#(numServers,addrWidth))  serverRequest <- mkFIFO;
   // stage 2: write commands
   FIFO#(DRec#(numServers, addrWidth)) serverProcessing <- mkSizedFIFO(valueOf(numTags));
   // stage 3: write data
   BRAM2Port#(Bit#(TLog#(numTags)), RResp#(numServers,addrWidth)) respFifos <- mkBRAM2Server(defaultValue);
   TagGen#(numTags) tag_gen <- mkTagGen;
   FIFO#(RResp#(numServers,addrWidth)) clientResponse <- mkFIFO;

   Reg#(Bit#(BurstLenSize)) burstReg <- mkReg(0);
   Reg#(Bool)               firstReg <- mkReg(True);
   Reg#(Bool)               lastReg <- mkReg(False);
   Reg#(Bit#(BeatCountSize)) beatCount <- mkReg(0);
   let beat_shift = fromInteger(valueOf(beatShift));

   Reg#(Bit#(64)) cycle_cnt <- mkReg(0);
   Reg#(Bit#(64)) last_loadClient <- mkReg(0);
   Reg#(Bit#(64)) last_mmuResp <- mkReg(0);

   (* fire_when_enabled *)
   rule cycle;
      cycle_cnt <= cycle_cnt+1;
   endrule

   FIFO#(DmaError) dmaErrorFifo <- mkFIFO();
   rule dmaError;
      let error <- toGet(dmaErrorFifo).get();
      ind.error(extend(pack(error.errorType)), error.pref, 0, 0);
   endrule

   rule checkMmuResp;
      let request <- toGet(clientRequest).get;
      let req = request.req;
      let client = request.client;
      let addrTransResponse <- mmus[req.sglId[31:16]].response.get;
      let physAddr = addrTransResponse.physAddr;
      let rename_tag <- tag_gen.getTag;
      serverRequest.enq(RRec{req:req, pa:physAddr, client:client, rename_tag:extend(rename_tag)});
      //if (verbose) $display("mkMemWriteInternal::checkMmuResp: client=%d, rename_tag=%d", client,rename_tag);
      if (verbose) $display("mkMemWriteInternal::mmuResp %d %d", client, cycle_cnt-last_mmuResp);
      last_mmuResp <= cycle_cnt;
   endrule

   rule writeDoneComp0;
      let tag <- tag_gen.complete;
      respFifos.portB.request.put(BRAMRequest{write:False, address:tag, datain: ?, responseOnWrite: ?});
   endrule

   FIFO#(MemData#(busWidth)) memDataFifo <- mkFIFO();
   Vector#(numServers, FIFO#(MemData#(busWidth))) clientWriteData <- replicateM(mkFIFO);

   if(valueOf(numServers) > 0)
      rule memdata;
	 let drq = serverProcessing.first;
	 let req_tag = drq.req_tag;
	 let req_burstLen = drq.req_burstLen;
	 let rename_tag = drq.rename_tag;
	 let client = drq.client;
	 MemData#(busWidth) tagdata = unpack(0);
	 if (killv[req_tag[5:4]] == False) begin
	    tagdata = clientWriteData[client].first;
	    clientWriteData[client].deq;
	 end
	 let burstLen = burstReg;
	 let first    = firstReg;
	 let last     = lastReg;
	 if (first) begin
	    burstLen = req_burstLen >> beat_shift;
	    last     = serverProcessing.first.last;
	    respFifos.portA.request.put(BRAMRequest{write:True,responseOnWrite:False, address:truncate(rename_tag), datain:RResp{orig_tag:req_tag, client:client}});
	 end
	 burstReg <= burstLen-1;
	 firstReg <= (burstLen-1 == 0);
	 lastReg  <= (burstLen-1 == 1);
	 beatCount <= beatCount+1;
	 if (last)
	    serverProcessing.deq();
	 //$display("mkMemWriteInternal::writeData: client=%d, rename_tag=%d", client, rename_tag);
	 memDataFifo.enq(MemData { data: tagdata.data,  tag:extend(rename_tag), last: last });
      endrule

   rule fill_clientResponse;
      let rv <- respFifos.portB.response.get;
      clientResponse.enq(rv);
   endrule

   Vector#(numServers, MemWriteServer#(busWidth)) sv = newVector;
   for(Integer i = 0; i < valueOf(numServers); i=i+1)
      sv[i] = (interface MemWriteServer;
		  interface Put writeReq;
		     method Action put(MemRequest req);
      			if (verbose) $display("mkMemWriteInternal::loadClient %d %d", i, cycle_cnt-last_loadClient);
			last_loadClient <= cycle_cnt;
			let mmusel = req.sglId[31:16];
			if (mmusel >= fromInteger(valueOf(numMMUs)))
			   dmaErrorFifo.enq(DmaError { errorType: DmaErrorMMUOutOfRange_w, pref: req.sglId });
   			else if (sglid_outofrange(req.sglId))
			   dmaErrorFifo.enq(DmaError { errorType: DmaErrorSGLIdOutOfRange_w, pref: req.sglId });
   			else if (stopv[req.tag[5:4]] == False) begin
   			   clientRequest.enq(LRec{req:req, client:fromInteger(i)});
   			   mmus[mmusel].request.put(AddrTransRequest{id:truncate(req.sglId),off:req.offset});
   			end
		     endmethod
		  endinterface
		  interface Put writeData = toPut(clientWriteData[i]);
		  interface Get writeDone;
		     method ActionValue#(Bit#(MemTagSize)) get if (clientResponse.first.client == fromInteger(i));
			clientResponse.deq;
			return clientResponse.first.orig_tag;
		     endmethod
		  endinterface
	       endinterface);

   interface servers = sv;
   interface PhysMemWriteClient client;
      interface Get writeReq;
	 method ActionValue#(PhysMemRequest#(addrWidth,busWidth)) get();
	    let request <- toGet(serverRequest).get();
	    let req = request.req;
	    let physAddr = request.pa;
	    let client = request.client;
	    let rename_tag = request.rename_tag;
	    serverProcessing.enq(DRec{req_tag:req.tag, req_burstLen: req.burstLen, client:client, rename_tag:rename_tag, last: (req.burstLen == fromInteger(valueOf(busWidthBytes))) });
	    //$display("mkMemWriteInternal::writeReq: client=%d, rename_tag=%d", client,rename_tag);
	    return PhysMemRequest{addr:physAddr, burstLen:req.burstLen, tag:extend(rename_tag)
`ifdef BYTE_ENABLES
				  , firstbe: truncate(req.firstbe), lastbe: truncate(req.lastbe)
`endif
};
	 endmethod
      endinterface
      interface Get writeData = toGet(memDataFifo);
      interface Put writeDone;
	 method Action put(Bit#(MemTagSize) resp);
	    tag_gen.returnTag(truncate(resp));
	    if (verbose) $display("mkMemWriteInternal::writeDone: resp=%d", resp);
	 endmethod
      endinterface
   endinterface
   interface Put tileControl;
      method Action put(TileControl tc);
	 let tile = tc.tile;
	 let kv = True;
	 let sv = True;
	 if (tc.state == Running || tc.state == Stopped)
	    kv = False;
	 if (tc.state == Running)
	    sv = False;
	 killv[tile] <= kv;
	 stopv[tile] <= sv;
      endmethod
   endinterface
   interface DmaDbg dbg;
      method ActionValue#(DmaDbgRec) dbg();
	 return DmaDbgRec{x:fromInteger(valueOf(numServers)), y:?, z:?, w:?};
      endmethod
      method ActionValue#(Bit#(64)) getMemoryTraffic();
	 return extend(beatCount);
      endmethod
   endinterface
endmodule

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

// BSV Libraries
import BRAMFIFO::*;
import FIFO::*;
import FIFOF::*;
import Vector::*;
import GetPut::*;
import ClientServer::*;
import Assert::*;
import BRAM::*;
import DefaultValue::*;

// CONNECTAL Libraries
import MemTypes::*;
import ConnectalMemory::*;
import ConnectalClocks::*;
import SharedBuffMMU::*;
import ConnectalCompletionBuffer::*;
import ConnectalConfig::*;
import Ethernet::*;
`include "ConnectalProjectConfig.bsv"

typedef 32 BeatCountSize;

typedef 9 MMU_PIPELINE_DEPTH;

interface DmaDbg;
   method ActionValue#(Bit#(64)) getMemoryTraffic();
   method ActionValue#(DmaDbgRec) dbg();
endinterface

interface MemWriteInternal#(numeric type addrWidth, numeric type busWidth, numeric type numTags, numeric type numServers);
   interface DmaDbg dbg;
   interface Put#(TileControl) tileControl;
   interface PhysMemWriteClient#(addrWidth,busWidth) client;
   interface Vector#(numServers, MemWriteServer#(busWidth)) servers;
endinterface

interface MemReadInternal#(numeric type addrWidth, numeric type busWidth, numeric type numTags, numeric type numServers);
   interface DmaDbg dbg;
   interface Put#(TileControl) tileControl;
   interface PhysMemReadClient#(addrWidth,busWidth) client;
   interface Vector#(numServers, MemReadServer#(busWidth)) servers;
endinterface

function Bool sglid_outofrange(SGLId p);
   return ((p[15:0]) >= fromInteger(valueOf(MaxNumSGLists)));
endfunction

import RegFile::*;

typedef struct {MemRequest req;
		Bit#(TLog#(TMax#(1,numClients))) client; } LRec#(numeric type numClients, numeric type addrWidth) deriving(Bits);

typedef struct {MemRequest req;
		Bit#(addrWidth) pa;
		Bit#(MemTagSize) rename_tag;
		Bit#(TLog#(TMax#(1,numClients))) client; } RRec#(numeric type numClients, numeric type addrWidth) deriving(Bits);

typedef struct {Bit#(MemTagSize) req_tag;
		Bit#(BurstLenSize) req_burstLen;
		Bit#(MemTagSize) rename_tag;
		Bit#(TLog#(TMax#(1,numClients))) client;
		Bool last;
   } DRec#(numeric type numClients, numeric type addrWidth) deriving(Bits);

typedef struct {Bit#(MemTagSize) orig_tag;
		Bit#(TLog#(TMax#(1,numClients))) client; } RResp#(numeric type numClients, numeric type addrWidth) deriving(Bits);

typedef struct {DmaErrorType errorType;
		Bit#(32) pref; } DmaError deriving (Bits);

module mkMemReadInternal#(MemServerIndication ind,
			  Vector#(numMMUs,Server#(AddrTransRequest,AddrTransResponse#(addrWidth))) mmus)
   (MemReadInternal#(addrWidth, busWidth, numTags, numServers))
   provisos(Log#(busWidthBytes,beatShift)
	    ,Div#(busWidth,8,busWidthBytes)
	    ,Add#(beatShift, a__, 8)
	    ,Add#(b__, TLog#(numTags), MemTagSize)
	    ,Add#(beatShift, c__, BurstLenSize)
	    ,Add#(d__, TDiv#(busWidth, 8), ByteEnableSize)
	    );

   // stopping/killing infra
   Vector#(4,Reg#(Bool)) killv <- replicateM(mkReg(False));
   Vector#(4,Reg#(Bool)) stopv <- replicateM(mkReg(False));

   // stage 0: address translation (latency = MMU_PIPELINE_DEPTH)
   FIFO#(LRec#(numServers,addrWidth)) clientRequest <- mkSizedFIFO(valueOf(MMU_PIPELINE_DEPTH));
   // stage 1: address validation (latency = 1)
   FIFO#(RRec#(numServers,addrWidth))  serverRequest <- mkFIFO;
   // stage 2: read commands
   BRAM_Configure bramConfig = defaultValue;
   if (mainClockPeriod < 8)
      bramConfig.latency = 2;
   BRAM2Port#(Bit#(TLog#(numTags)), DRec#(numServers,addrWidth)) serverProcessing <- mkBRAM2Server(bramConfig);
   //BRAM2Port#(Bit#(TAdd#(TLog#(numTags),TSub#(BurstLenSize,beatShift))), MemData#(busWidth)) clientData <- mkBRAM2Server(bramConfig);
   FIFOF#(MemData#(busWidth)) clientData <- mkSizedFIFOF(4);
   // stage 3: read data
   FIFO#(MemData#(busWidth)) serverData <- mkFIFO;

   let verbose = True;

   RegFile#(Bit#(TLog#(numTags)),Tuple2#(Bool,Bit#(BurstLenSize))) clientBurstLen <- mkRegFileFull();
   Reg#(Bit#(BurstLenSize)) burstReg <- mkReg(0);
   Reg#(Bool)               firstReg <- mkReg(True);

   Reg#(Bit#(32))  beatCount <- mkReg(0);
   let beat_shift = fromInteger(valueOf(beatShift));
   TagGen#(numTags) tag_gen <- mkTagGen;

   FIFO#(Bit#(TAdd#(1,TLog#(TMax#(1,numServers))))) clientSelect <- mkFIFO;

   // performance analytics
   Reg#(Bit#(64)) cycle_cnt <- mkReg(0);
   Reg#(Bit#(64)) last_loadClient <- mkReg(0);
   Reg#(Bit#(64)) last_mmuResp <- mkReg(0);
   Reg#(Bit#(64)) last_comp <- mkReg(0);
   Reg#(Bit#(64)) last_readReq <- mkReg(0);
   Reg#(Bit#(64)) last_readData <- mkReg(0);
   (* fire_when_enabled *)
   rule cycle;
      cycle_cnt <= cycle_cnt+1;
   endrule

   FIFO#(DmaError) dmaErrorFifo <- mkFIFO();
   rule dmaError;
      let error <- toGet(dmaErrorFifo).get();
      ind.error(extend(pack(error.errorType)), error.pref, 0, 0);
   endrule

   rule checkMmuResp;
      let request <- toGet(clientRequest).get();
      let addrTransResponse <- mmus[request.req.sglId[31:16]].response.get;
      let physAddr = addrTransResponse.physAddr;
      let rename_tag <- tag_gen.getTag;
      let burstLenBeats = request.req.burstLen >> beat_shift;
      clientBurstLen.upd(truncate(rename_tag), tuple2(burstLenBeats == 1, burstLenBeats));

      serverRequest.enq(RRec{req:request.req, pa:physAddr, client:request.client, rename_tag:extend(rename_tag)});
      if (verbose) $display("mkMemReadInternal::checkMmuResp: client=%d, tag=%d rename_tag=%d burstLen=%d", request.client, request.req.tag, rename_tag, burstLenBeats);
      if (verbose) $display("mkMemReadInternal::mmuResp %d %d", request.client, cycle_cnt-last_mmuResp);
      last_mmuResp <= cycle_cnt;
   endrule

   rule read_data;
      let response <- toGet(serverData).get();
      let drq <- serverProcessing.portA.response.get;
      let tag = drq.req_tag;
      match { .last, .burstLen } = clientBurstLen.sub(truncate(response.tag));
      let first   = firstReg;
      if (first) begin
	 dynamicAssert(last == (burstLen==1), "Last incorrect");
      end
      if (last && burstLen != 1)
	 $display("rename_tag=%d tag=%d burstLen=%d last=%d", response.tag, tag, burstLen, last);
      Bit#(TLog#(numTags)) tt = truncate(response.tag);
      let client = drq.client;
      clientSelect.enq(extend(client));
      clientData.enq(response);
      $display("response: ", fshow(response));
      if (last) begin
	 tag_gen.returnTag(truncate(response.tag));
     $display("response: last! ", fshow(response));
      end
      last_readData <= cycle_cnt;
      if (verbose) $display("mkMemReadInternal::read_data cyclediff %d", cycle_cnt-last_readData);
      clientBurstLen.upd(truncate(response.tag), tuple2((burstLen-1 == 1),burstLen-1));
      firstReg <= response.last;
   endrule

   rule tag_completed;
      let tag <- tag_gen.complete;
      //serverProcessing.portB.request.put(BRAMRequest{write:False, address:tag, datain: ?, responseOnWrite: ?});
      //serverTag.enq(tag);
      if(verbose) $display("mkMemReadInternal::complete_burst0 %h", tag);
   endrule

   Vector#(numServers, MemReadServer#(busWidth)) sv = newVector;
   for(Integer i = 0; i < valueOf(numServers); i=i+1)
      sv[i] = (interface MemReadServer;
		  interface Put readReq;
		     method Action put(MemRequest req);
			last_loadClient <= cycle_cnt;
			let mmusel = req.sglId[31:16];
      			if (verbose) $display("mkMemReadInternal::loadClient server %d mmusel %d burstLen %d tag %d cycle %d",
			   i, mmusel, req.burstLen >> beat_shift, req.tag, cycle_cnt-last_loadClient);
			if (mmusel >= fromInteger(valueOf(numMMUs)))
			   dmaErrorFifo.enq(DmaError { errorType: DmaErrorMMUOutOfRange_r, pref: req.sglId });
   			else if (sglid_outofrange(req.sglId))
			   dmaErrorFifo.enq(DmaError { errorType: DmaErrorSGLIdOutOfRange_r, pref: req.sglId });
   			else if (stopv[req.tag[5:4]] == False) begin
   			   clientRequest.enq(LRec{req:req, client:fromInteger(i)});
   			   mmus[mmusel].request.put(AddrTransRequest{id:truncate(req.sglId),off:req.offset});
   			end
		     endmethod
		  endinterface
		  interface Get readData;
		     method ActionValue#(MemData#(busWidth)) get if (clientSelect.first == fromInteger(i));
			clientSelect.deq;
            let data = clientData.first;
            clientData.deq;
			if (verbose) $display("mkMemReadInternal::comp server %d data %x cycle %d", i, data.data, cycle_cnt-last_comp);
			last_comp <= cycle_cnt;
			return data;
		     endmethod
		  endinterface
	       endinterface);

   interface servers = sv;
   interface PhysMemReadClient client;
      interface Get readReq;
	 method ActionValue#(PhysMemRequest#(addrWidth,busWidth)) get();
	    let request <- toGet(serverRequest).get;
	    let req = request.req;
	    if (False && request.pa[31:24] != 0)
	       $display("mkMemReadInternal::req_ar: funny physAddr req.sglId=%d req.offset=%h physAddr=%h", req.sglId, req.offset, request.pa);
	    serverProcessing.portB.request.put(BRAMRequest{write:True, responseOnWrite:False, address:truncate(request.rename_tag),
						   datain:DRec{req_tag:req.tag, req_burstLen: req.burstLen, client:request.client, rename_tag:request.rename_tag, last:(req.burstLen == fromInteger(valueOf(busWidthBytes)))}});
	    //$display("mkMemReadInternal::readReq: client=%d, rename_tag=%d, physAddr=%h req.burstLen=%d beat_shift=%d last=%d", request.client,request.rename_tag,request.pa, req.burstLen, beat_shift, req.burstLen == beat_shift);
	    if (verbose) $display("mkMemReadInternal::read_client.readReq %d", cycle_cnt-last_readReq);
	    last_readReq <= cycle_cnt;
	    return PhysMemRequest{addr:request.pa, burstLen:req.burstLen, tag:request.rename_tag
`ifdef BYTE_ENABLES
				  , firstbe: truncate(request.req.firstbe), lastbe: truncate(request.req.lastbe)
`endif
	       };
	 endmethod
      endinterface
      interface Put readData;
	 method Action put(MemData#(busWidth) response);
	    serverData.enq(response);
	    serverProcessing.portA.request.put(BRAMRequest{write:False, address:truncate(response.tag), datain: ?, responseOnWrite: ?});
	    beatCount <= beatCount+1;
	 endmethod
      endinterface
   endinterface
   interface Put tileControl;
      method Action put(TileControl tc);
	 let tile = tc.tile;
	 let kv = True;
	 let sv = True;
	 if (tc.state == Running || tc.state == Stopped)
	    kv = False;
	 if (tc.state == Running)
	    sv = False;
	 killv[tile] <= kv;
	 stopv[tile] <= sv;
      endmethod
   endinterface
   interface DmaDbg dbg;
      method ActionValue#(DmaDbgRec) dbg();
	 return DmaDbgRec{x:0, y:0, z:0, w:0};
      endmethod
      method ActionValue#(Bit#(64)) getMemoryTraffic();
	 return extend(beatCount);
      endmethod
   endinterface
endmodule

module mkMemWriteInternal#(MemServerIndication ind,
			   Vector#(numMMUs,Server#(AddrTransRequest,AddrTransResponse#(addrWidth))) mmus)
   (MemWriteInternal#(addrWidth, busWidth, numTags, numServers))
   provisos(Log#(busWidthBytes,beatShift)
	    ,Div#(busWidth,8,busWidthBytes)
	    ,Add#(beatShift, a__, 8)
	    ,Add#(b__, TLog#(numTags), MemTagSize)
	    ,Add#(beatShift, c__, BurstLenSize)
            ,Add#(d__, TDiv#(busWidth, 8), ByteEnableSize)
	    );

   let verbose = False;

   // stopping/killing infra
   Vector#(4,Reg#(Bool)) killv <- replicateM(mkReg(False));
   Vector#(4,Reg#(Bool)) stopv <- replicateM(mkReg(False));

   // stage 0: address translation (latency = MMU_PIPELINE_DEPTH)
   FIFO#(LRec#(numServers,addrWidth)) clientRequest <- mkSizedFIFO(valueOf(MMU_PIPELINE_DEPTH));
   // stage 1: address validation (latency = 1)
   FIFO#(RRec#(numServers,addrWidth))  serverRequest <- mkFIFO;
   // stage 2: write commands
   FIFO#(DRec#(numServers, addrWidth)) serverProcessing <- mkSizedFIFO(valueOf(numTags));
   // stage 3: write data
   BRAM2Port#(Bit#(TLog#(numTags)), RResp#(numServers,addrWidth)) respFifos <- mkBRAM2Server(defaultValue);
   TagGen#(numTags) tag_gen <- mkTagGen;
   FIFO#(RResp#(numServers,addrWidth)) clientResponse <- mkFIFO;

   Reg#(Bit#(BurstLenSize)) burstReg <- mkReg(0);
   Reg#(Bool)               firstReg <- mkReg(True);
   Reg#(Bool)               lastReg <- mkReg(False);
   Reg#(Bit#(BeatCountSize)) beatCount <- mkReg(0);
   let beat_shift = fromInteger(valueOf(beatShift));

   Reg#(Bit#(64)) cycle_cnt <- mkReg(0);
   Reg#(Bit#(64)) last_loadClient <- mkReg(0);
   Reg#(Bit#(64)) last_mmuResp <- mkReg(0);

   (* fire_when_enabled *)
   rule cycle;
      cycle_cnt <= cycle_cnt+1;
   endrule

   FIFO#(DmaError) dmaErrorFifo <- mkFIFO();
   rule dmaError;
      let error <- toGet(dmaErrorFifo).get();
      ind.error(extend(pack(error.errorType)), error.pref, 0, 0);
   endrule

   rule checkMmuResp;
      let request <- toGet(clientRequest).get;
      let req = request.req;
      let client = request.client;
      let addrTransResponse <- mmus[req.sglId[31:16]].response.get;
      let physAddr = addrTransResponse.physAddr;
      let rename_tag <- tag_gen.getTag;
      serverRequest.enq(RRec{req:req, pa:physAddr, client:client, rename_tag:extend(rename_tag)});
      //if (verbose) $display("mkMemWriteInternal::checkMmuResp: client=%d, rename_tag=%d", client,rename_tag);
      if (verbose) $display("mkMemWriteInternal::mmuResp %d %d", client, cycle_cnt-last_mmuResp);
      last_mmuResp <= cycle_cnt;
   endrule

   rule writeDoneComp0;
      let tag <- tag_gen.complete;
      respFifos.portB.request.put(BRAMRequest{write:False, address:tag, datain: ?, responseOnWrite: ?});
   endrule

   FIFO#(MemData#(busWidth)) memDataFifo <- mkFIFO();
   Vector#(numServers, FIFO#(MemData#(busWidth))) clientWriteData <- replicateM(mkFIFO);

   if(valueOf(numServers) > 0)
      rule memdata;
	 let drq = serverProcessing.first;
	 let req_tag = drq.req_tag;
	 let req_burstLen = drq.req_burstLen;
	 let rename_tag = drq.rename_tag;
	 let client = drq.client;
	 MemData#(busWidth) tagdata = unpack(0);
	 if (killv[req_tag[5:4]] == False) begin
	    tagdata = clientWriteData[client].first;
	    clientWriteData[client].deq;
	 end
	 let burstLen = burstReg;
	 let first    = firstReg;
	 let last     = lastReg;
	 if (first) begin
	    burstLen = req_burstLen >> beat_shift;
	    last     = serverProcessing.first.last;
	    respFifos.portA.request.put(BRAMRequest{write:True,responseOnWrite:False, address:truncate(rename_tag), datain:RResp{orig_tag:req_tag, client:client}});
	 end
	 burstReg <= burstLen-1;
	 firstReg <= (burstLen-1 == 0);
	 lastReg  <= (burstLen-1 == 1);
	 beatCount <= beatCount+1;
	 if (last)
	    serverProcessing.deq();
	 //$display("mkMemWriteInternal::writeData: client=%d, rename_tag=%d", client, rename_tag);
	 memDataFifo.enq(MemData { data: tagdata.data,  tag:extend(rename_tag), last: last });
      endrule

   rule fill_clientResponse;
      let rv <- respFifos.portB.response.get;
      clientResponse.enq(rv);
   endrule

   Vector#(numServers, MemWriteServer#(busWidth)) sv = newVector;
   for(Integer i = 0; i < valueOf(numServers); i=i+1)
      sv[i] = (interface MemWriteServer;
		  interface Put writeReq;
		     method Action put(MemRequest req);
      			if (verbose) $display("mkMemWriteInternal::loadClient %d %d", i, cycle_cnt-last_loadClient);
			last_loadClient <= cycle_cnt;
			let mmusel = req.sglId[31:16];
			if (mmusel >= fromInteger(valueOf(numMMUs)))
			   dmaErrorFifo.enq(DmaError { errorType: DmaErrorMMUOutOfRange_w, pref: req.sglId });
   			else if (sglid_outofrange(req.sglId))
			   dmaErrorFifo.enq(DmaError { errorType: DmaErrorSGLIdOutOfRange_w, pref: req.sglId });
   			else if (stopv[req.tag[5:4]] == False) begin
   			   clientRequest.enq(LRec{req:req, client:fromInteger(i)});
   			   mmus[mmusel].request.put(AddrTransRequest{id:truncate(req.sglId),off:req.offset});
   			end
		     endmethod
		  endinterface
		  interface Put writeData = toPut(clientWriteData[i]);
		  interface Get writeDone;
		     method ActionValue#(Bit#(MemTagSize)) get if (clientResponse.first.client == fromInteger(i));
			clientResponse.deq;
			return clientResponse.first.orig_tag;
		     endmethod
		  endinterface
	       endinterface);

   interface servers = sv;
   interface PhysMemWriteClient client;
      interface Get writeReq;
	 method ActionValue#(PhysMemRequest#(addrWidth,busWidth)) get();
	    let request <- toGet(serverRequest).get();
	    let req = request.req;
	    let physAddr = request.pa;
	    let client = request.client;
	    let rename_tag = request.rename_tag;
	    serverProcessing.enq(DRec{req_tag:req.tag, req_burstLen: req.burstLen, client:client, rename_tag:rename_tag, last: (req.burstLen == fromInteger(valueOf(busWidthBytes))) });
	    //$display("mkMemWriteInternal::writeReq: client=%d, rename_tag=%d", client,rename_tag);
	    return PhysMemRequest{addr:physAddr, burstLen:req.burstLen, tag:extend(rename_tag)
`ifdef BYTE_ENABLES
				  , firstbe: truncate(req.firstbe), lastbe: truncate(req.lastbe)
`endif
};
	 endmethod
      endinterface
      interface Get writeData = toGet(memDataFifo);
      interface Put writeDone;
	 method Action put(Bit#(MemTagSize) resp);
	    tag_gen.returnTag(truncate(resp));
	    if (verbose) $display("mkMemWriteInternal::writeDone: resp=%d", resp);
	 endmethod
      endinterface
   endinterface
   interface Put tileControl;
      method Action put(TileControl tc);
	 let tile = tc.tile;
	 let kv = True;
	 let sv = True;
	 if (tc.state == Running || tc.state == Stopped)
	    kv = False;
	 if (tc.state == Running)
	    sv = False;
	 killv[tile] <= kv;
	 stopv[tile] <= sv;
      endmethod
   endinterface
   interface DmaDbg dbg;
      method ActionValue#(DmaDbgRec) dbg();
	 return DmaDbgRec{x:fromInteger(valueOf(numServers)), y:?, z:?, w:?};
      endmethod
      method ActionValue#(Bit#(64)) getMemoryTraffic();
	 return extend(beatCount);
      endmethod
   endinterface
endmodule


Revert "remove unneeded version of mkPareff"

This reverts commit 46228b0609296e3f6fb3e2b83a1f67882c7360ba.

// Copyright (c) 2015 Connectal Project

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import Vector::*;
import FIFO::*;
import FIFOF::*;
import GetPut::*;

import MemTypes::*;

interface Pareff#(numeric type dataWidth);
   method Action init(Bit#(32) id, Bit#(32) handle, Bit#(32) size);
   method Action initfd(Bit#(32) id, Bit#(32) fd);
   method Action write(Bit#(32) handle, Bit#(32) addr, Bit#(dataWidth) v);
   method Action readrequest(Bit#(32) handle, Bit#(32) addr);
   method ActionValue#(Bit#(dataWidth)) readresponse();
endinterface

`ifdef BSIM
import "BDPI" function ActionValue#(Bit#(32)) pareff_init(Bit#(32) id, Bit#(32) handle, Bit#(32) size);
import "BDPI" function ActionValue#(Bit#(32)) pareff_initfd(Bit#(32) id, Bit#(32) fd);

// implemented in BsimDma.cpp
import "BDPI" function Action write_pareff32(Bit#(32) handle, Bit#(32) addr, Bit#(32) v);
import "BDPI" function Action write_pareff64(Bit#(32) handle, Bit#(32) addr, Bit#(64) v);
import "BDPI" function ActionValue#(Bit#(32)) read_pareff32(Bit#(32) handle, Bit#(32) addr);
import "BDPI" function ActionValue#(Bit#(64)) read_pareff64(Bit#(32) handle, Bit#(32) addr);

module mkPareff(Pareff#(dataWidth) ifc)
   provisos (Mul#(TDiv#(dataWidth, 32), 32, dataWidth));
   FIFO#(Bit#(dataWidth)) dataFifo <- mkFIFO();
      method Action init(Bit#(32) id, Bit#(32) handle, Bit#(32) size);
	 let v <- pareff_init(id, handle, size);
	 //return v;
      endmethod
      method Action initfd(Bit#(32) id, Bit#(32) fd);
	 let v <- pareff_initfd(id, fd);
	 //return v;
      endmethod
      method Action write(Bit#(32) handle, Bit#(32) addr, Bit#(dataWidth) v);
	  Vector#(TDiv#(dataWidth, 32), Bit#(32)) vs = unpack(v);
	  function Action write32(Integer i, Bit#(32) vv);
	     action
		write_pareff32(handle, addr+4*fromInteger(i), vv);
	     endaction
	  endfunction
	  mapM_(uncurry(write32), zip(genVector(), vs));
      endmethod
      method Action  readrequest(Bit#(32) handle, Bit#(32) addr);
	  function ActionValue#(Bit#(32)) read32(Integer i);
	     actionvalue
		let v <- read_pareff32(handle, addr+4*fromInteger(i));
		return v;
	     endactionvalue
	  endfunction
   	  Vector#(TDiv#(dataWidth,32),Bit#(32)) vs <- mapM(read32, genVector());
	  dataFifo.enq(pack(vs));
      endmethod
      method ActionValue#(Bit#(dataWidth)) readresponse();
          let v <- toGet(dataFifo).get();
	  return v;
      endmethod
endmodule
`endif

`ifdef XSIM
interface XsimMemReadWrite;
   method Action init(Bit#(32) id, Bit#(32) handle, Bit#(32) size);
   method Action initfd(Bit#(32) id, Bit#(32) fd);
   method Action write32(Bit#(32) handle, Bit#(32) addr, Bit#(32) v);
   method Action readrequest(Bit#(32) handle, Bit#(32) addr);
   method ActionValue#(Bit#(32)) readresponse();
endinterface

import "BVI" XsimMemReadWrite =
module mkXsimReadWrite(XsimMemReadWrite);
   method init(init_id, init_handle, init_size) enable (en_init);
   method initfd(initfd_id, initfd_fd) enable (en_initfd);
   method write32(write32_handle, write32_addr, write32_data) enable (en_write32);
   method readrequest(readrequest_handle, readrequest_addr) enable (en_readrequest) ready (rdy_readrequest);
   method readresponse_data readresponse() enable (en_readresponse) ready (rdy_readresponse);
   schedule (init, initfd, write32, readrequest, readresponse) CF (init, initfd, write32, readrequest, readresponse);
endmodule

module mkPareff(Pareff#(dataWidth) ifc)
   provisos (Mul#(TDiv#(dataWidth, 32), 32, dataWidth));
   Vector#(TDiv#(dataWidth,32),XsimMemReadWrite) rws <- replicateM(mkXsimReadWrite());
   method Action init(Bit#(32) id, Bit#(32) handle, Bit#(32) size);
      rws[0].init(id, handle, size);
   endmethod
   method Action initfd(Bit#(32) id, Bit#(32) fd);
      rws[0].initfd(id, fd);
   endmethod
   method Action write(Bit#(32) handle, Bit#(32) addr, Bit#(dataWidth) v);
      Vector#(TDiv#(dataWidth, 32), Bit#(32)) vs = unpack(v);
      function Action write32(Integer i, Bit#(32) vv);
	 action
	    rws[i].write32(handle, addr+4*fromInteger(i), vv);
	 endaction
      endfunction
      mapM_(uncurry(write32), zip(genVector(), vs));
   endmethod
   method Action readrequest(Bit#(32) handle, Bit#(32) addr);
      function Action doreadrequest(Integer i);
	 action
	    rws[i].readrequest(handle, addr+4*fromInteger(i));
	 endaction
      endfunction
      Vector#(TDiv#(dataWidth,32),Integer) indexes = genVector();
      mapM_(doreadrequest, indexes);
   endmethod
   method ActionValue#(Bit#(dataWidth)) readresponse();
      function ActionValue#(Bit#(32)) readresponse32(Integer i);
	 actionvalue
	    let v <- rws[i].readresponse();
	    return v;
	 endactionvalue
      endfunction
      Vector#(TDiv#(dataWidth,32),Bit#(32)) vs <- mapM(readresponse32, genVector());
      return pack(vs);
   endmethod
endmodule
`endif

`ifndef BSIM
`ifndef XSIM
module mkPareff(Pareff#(dataWidth) ifc);
   method Action init(Bit#(32) id, Bit#(32) handle, Bit#(32) size);
   endmethod
   method Action initfd(Bit#(32) id, Bit#(32) fd);
   endmethod
   method Action write(Bit#(32) handle, Bit#(32) addr, Bit#(dataWidth) v);
   endmethod
   method ActionValue#(Bit#(dataWidth)) read(Bit#(32) handle, Bit#(32) addr);
      return 0;
   endmethod
endmodule
`endif
`endif

module mkPareffDmaMaster(PhysMemSlave#(serverAddrWidth,serverBusWidth))
   provisos(Div#(serverBusWidth,8,dataWidthBytes),
	    Mul#(dataWidthBytes,8,serverBusWidth),
	    Log#(dataWidthBytes,beatShift),
	    Mul#(TDiv#(serverBusWidth, 32), 32, serverBusWidth),
	    Bits#(Tuple2#(Bit#(64), PhysMemRequest#(serverAddrWidth)), a__)
	    );

   let verbose = False;
   Pareff#(serverBusWidth) rw <- mkPareff();

   Reg#(Bit#(BurstLenSize))  readLenReg <- mkReg(0);
   Reg#(Bit#(32))         readOffsetReg <- mkReg(0);

   Reg#(Bit#(BurstLenSize))  writeLenReg <- mkReg(0);
   Reg#(Bit#(32))         writeOffsetReg <- mkReg(0);

   let readLatency_I = 150;
   let writeLatency_I = 150;

   Bit#(64) readLatency = fromInteger(readLatency_I);
   Bit#(64) writeLatency = fromInteger(writeLatency_I);

   Reg#(Bit#(64)) req_ar_b_ts <- mkReg(0);
   Reg#(Bit#(64)) req_aw_b_ts <- mkReg(0);
   Reg#(Bit#(64)) cycles <- mkReg(0);
   Reg#(Bit#(64)) last_reqAr <- mkReg(0);
   Reg#(Bit#(64)) last_read_eob <- mkReg(0);
   Reg#(Bit#(64)) last_write_eob <- mkReg(0);

   FIFOF#(Tuple2#(Bit#(64), PhysMemRequest#(serverAddrWidth)))  readDelayFifo <- mkSizedFIFOF(readLatency_I);
   FIFOF#(Tuple2#(Bit#(64),PhysMemRequest#(serverAddrWidth))) writeDelayFifo <- mkSizedFIFOF(writeLatency_I);

   FIFOF#(Tuple2#(Bit#(64), Bit#(MemTagSize))) bFifo <- mkSizedFIFOF(writeLatency_I);
   FIFOF#(Tuple2#(Bit#(MemTagSize),Bool)) taglastfifo <- mkFIFOF();
   rule increment_cycle;
      cycles <= cycles+1;
   endrule

   let read_jitter = True; //cycles[4:0] == 0;
   let write_jitter = True; //cycles[4:0] == 5;

   Reg#(Bit#(8))  burstReg <- mkReg(0);
   FIFO#(Bit#(8)) reqs <- mkSizedFIFO(32);

   let beat_shift = fromInteger(valueOf(beatShift));

   rule read_rule if (readDelayFifo.notEmpty() && (cycles-tpl_1(readDelayFifo.first) > readLatency));
	 match { .reqTime, .req } = readDelayFifo.first;
	 Bit#(BurstLenSize) readLen = readLenReg;
	 Bit#(32) readOffset = readOffsetReg;
	 Bit#(MemTagSize) tag = req.tag;
	 Bit#(8) handle = req.addr[39:32];

	 if (readLen == 0) begin
	    req_ar_b_ts <= cycles;
	    readLen     = req.burstLen>>beat_shift;
	    readOffset  = 0;
	 end
	 rw.readrequest(extend(handle), req.addr[31:0]+readOffset);
	 let last = (readLen == 1);
	 if (last)
	    readDelayFifo.deq();
	 taglastfifo.enq(tuple2(tag, last));
	 readLenReg <= readLen - 1;
	 readOffsetReg <= readOffset + fromInteger(valueOf(serverBusWidth)/8);
   endrule

   interface PhysMemReadServer read_server;
      interface Put readReq;
	 method Action put(PhysMemRequest#(serverAddrWidth) req);
            if (verbose) $display("%d axiSlave.read.readAddr %h bc %d", cycles, req.addr, req.burstLen);
	    //readAddrGenerator.request.put(req);
	    readDelayFifo.enq(tuple2(cycles,req));
	 endmethod
      endinterface
      interface Get readData;
	 method ActionValue#(MemData#(serverBusWidth)) get();
	     match { .tag, .last } <- toGet(taglastfifo).get();
 	     let v <- rw.readresponse();
	     return MemData { data: v, tag: tag, last: last };
	 endmethod
      endinterface
   endinterface
   interface PhysMemWriteServer write_server;
      interface Put writeReq;
	 method Action put(PhysMemRequest#(serverAddrWidth) req);
	 //$display("mkBsimHost::req_aw id=%d", req.tag);
	 writeDelayFifo.enq(tuple2(cycles,req));
	 endmethod
      endinterface
      interface Put writeData;
	 method Action put(MemData#(serverBusWidth) resp) if (writeDelayFifo.notEmpty && (cycles-tpl_1(writeDelayFifo.first)) > writeLatency);
	    match { .reqTime, .req } = writeDelayFifo.first;
	    Bit#(BurstLenSize) writeLen = writeLenReg;
	    Bit#(32) writeOffset = writeOffsetReg;
	    Bit#(MemTagSize) tag = req.tag;
	    Bit#(8) handle = req.addr[39:32];
	    if (writeLenReg == 0) begin
	       req_aw_b_ts <= cycles;
	       writeLen = req.burstLen>>beat_shift;
	       writeOffset = 0;
	    end
	    rw.write(extend(handle), req.addr[31:0] + writeOffset, resp.data);
	    writeLenReg <= writeLen - 1;
	    writeOffsetReg <= writeOffset + fromInteger(valueOf(serverBusWidth)/8);
	    if (writeLen == 1) begin
	       bFifo.enq(tuple2(cycles,tag));
	       writeDelayFifo.deq;
	    end
	 endmethod
      endinterface
      interface Get writeDone;
	 method ActionValue#(Bit#(MemTagSize)) get() if ((cycles-tpl_1(bFifo.first)) > writeLatency);
	 bFifo.deq();
	 return tpl_2(bFifo.first());
	 endmethod
      endinterface
   endinterface
endmodule

// Copyright (c) 2015 Connectal Project

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import Vector::*;
import FIFO::*;
import FIFOF::*;
import GetPut::*;

import MemTypes::*;

interface Pareff#(numeric type dataWidth);
   method Action init(Bit#(32) id, Bit#(32) handle, Bit#(32) size);
   method Action initfd(Bit#(32) id, Bit#(32) fd);
   method Action write(Bit#(32) handle, Bit#(32) addr, Bit#(dataWidth) v);
   method Action readrequest(Bit#(32) handle, Bit#(32) addr);
   method ActionValue#(Bit#(dataWidth)) readresponse();
endinterface

`ifdef BSIM
import "BDPI" function ActionValue#(Bit#(32)) pareff_init(Bit#(32) id, Bit#(32) handle, Bit#(32) size);
import "BDPI" function ActionValue#(Bit#(32)) pareff_initfd(Bit#(32) id, Bit#(32) fd);

// implemented in BsimDma.cpp
import "BDPI" function Action write_pareff32(Bit#(32) handle, Bit#(32) addr, Bit#(32) v);
import "BDPI" function Action write_pareff64(Bit#(32) handle, Bit#(32) addr, Bit#(64) v);
import "BDPI" function ActionValue#(Bit#(32)) read_pareff32(Bit#(32) handle, Bit#(32) addr);
import "BDPI" function ActionValue#(Bit#(64)) read_pareff64(Bit#(32) handle, Bit#(32) addr);

module mkPareff(Pareff#(dataWidth) ifc)
   provisos (Mul#(TDiv#(dataWidth, 32), 32, dataWidth));
   FIFO#(Bit#(dataWidth)) dataFifo <- mkFIFO();
      method Action init(Bit#(32) id, Bit#(32) handle, Bit#(32) size);
	 let v <- pareff_init(id, handle, size);
	 //return v;
      endmethod
      method Action initfd(Bit#(32) id, Bit#(32) fd);
	 let v <- pareff_initfd(id, fd);
	 //return v;
      endmethod
      method Action write(Bit#(32) handle, Bit#(32) addr, Bit#(dataWidth) v);
	  Vector#(TDiv#(dataWidth, 32), Bit#(32)) vs = unpack(v);
	  function Action write32(Integer i, Bit#(32) vv);
	     action
		write_pareff32(handle, addr+4*fromInteger(i), vv);
	     endaction
	  endfunction
	  mapM_(uncurry(write32), zip(genVector(), vs));
      endmethod
      method Action  readrequest(Bit#(32) handle, Bit#(32) addr);
	  function ActionValue#(Bit#(32)) read32(Integer i);
	     actionvalue
		let v <- read_pareff32(handle, addr+4*fromInteger(i));
		return v;
	     endactionvalue
	  endfunction
   	  Vector#(TDiv#(dataWidth,32),Bit#(32)) vs <- mapM(read32, genVector());
	  dataFifo.enq(pack(vs));
      endmethod
      method ActionValue#(Bit#(dataWidth)) readresponse();
          let v <- toGet(dataFifo).get();
	  return v;
      endmethod
endmodule
`endif

`ifdef XSIM
interface XsimMemReadWrite;
   method Action init(Bit#(32) id, Bit#(32) handle, Bit#(32) size);
   method Action initfd(Bit#(32) id, Bit#(32) fd);
   method Action write32(Bit#(32) handle, Bit#(32) addr, Bit#(32) v);
   method Action readrequest(Bit#(32) handle, Bit#(32) addr);
   method ActionValue#(Bit#(32)) readresponse();
endinterface

import "BVI" XsimMemReadWrite =
module mkXsimReadWrite(XsimMemReadWrite);
   method init(init_id, init_handle, init_size) enable (en_init);
   method initfd(initfd_id, initfd_fd) enable (en_initfd);
   method write32(write32_handle, write32_addr, write32_data) enable (en_write32);
   method readrequest(readrequest_handle, readrequest_addr) enable (en_readrequest) ready (rdy_readrequest);
   method readresponse_data readresponse() enable (en_readresponse) ready (rdy_readresponse);
   schedule (init, initfd, write32, readrequest, readresponse) CF (init, initfd, write32, readrequest, readresponse);
endmodule

module mkPareff(Pareff#(dataWidth) ifc)
   provisos (Mul#(TDiv#(dataWidth, 32), 32, dataWidth));
   Vector#(TDiv#(dataWidth,32),XsimMemReadWrite) rws <- replicateM(mkXsimReadWrite());
   method Action init(Bit#(32) id, Bit#(32) handle, Bit#(32) size);
      rws[0].init(id, handle, size);
   endmethod
   method Action initfd(Bit#(32) id, Bit#(32) fd);
      rws[0].initfd(id, fd);
   endmethod
   method Action write(Bit#(32) handle, Bit#(32) addr, Bit#(dataWidth) v);
      Vector#(TDiv#(dataWidth, 32), Bit#(32)) vs = unpack(v);
      function Action write32(Integer i, Bit#(32) vv);
	 action
	    rws[i].write32(handle, addr+4*fromInteger(i), vv);
	 endaction
      endfunction
      mapM_(uncurry(write32), zip(genVector(), vs));
   endmethod
   method Action readrequest(Bit#(32) handle, Bit#(32) addr);
      function Action doreadrequest(Integer i);
	 action
	    rws[i].readrequest(handle, addr+4*fromInteger(i));
	 endaction
      endfunction
      Vector#(TDiv#(dataWidth,32),Integer) indexes = genVector();
      mapM_(doreadrequest, indexes);
   endmethod
   method ActionValue#(Bit#(dataWidth)) readresponse();
      function ActionValue#(Bit#(32)) readresponse32(Integer i);
	 actionvalue
	    let v <- rws[i].readresponse();
	    return v;
	 endactionvalue
      endfunction
      Vector#(TDiv#(dataWidth,32),Bit#(32)) vs <- mapM(readresponse32, genVector());
      return pack(vs);
   endmethod
endmodule
`endif

module mkPareffDmaMaster(PhysMemSlave#(serverAddrWidth,serverBusWidth))
   provisos(Div#(serverBusWidth,8,dataWidthBytes),
	    Mul#(dataWidthBytes,8,serverBusWidth),
	    Log#(dataWidthBytes,beatShift),
	    Mul#(TDiv#(serverBusWidth, 32), 32, serverBusWidth),
	    Bits#(Tuple2#(Bit#(64), PhysMemRequest#(serverAddrWidth)), a__)
	    );

   let verbose = False;
   Pareff#(serverBusWidth) rw <- mkPareff();

   Reg#(Bit#(BurstLenSize))  readLenReg <- mkReg(0);
   Reg#(Bit#(32))         readOffsetReg <- mkReg(0);

   Reg#(Bit#(BurstLenSize))  writeLenReg <- mkReg(0);
   Reg#(Bit#(32))         writeOffsetReg <- mkReg(0);

   let readLatency_I = 150;
   let writeLatency_I = 150;

   Bit#(64) readLatency = fromInteger(readLatency_I);
   Bit#(64) writeLatency = fromInteger(writeLatency_I);

   Reg#(Bit#(64)) req_ar_b_ts <- mkReg(0);
   Reg#(Bit#(64)) req_aw_b_ts <- mkReg(0);
   Reg#(Bit#(64)) cycles <- mkReg(0);
   Reg#(Bit#(64)) last_reqAr <- mkReg(0);
   Reg#(Bit#(64)) last_read_eob <- mkReg(0);
   Reg#(Bit#(64)) last_write_eob <- mkReg(0);

   FIFOF#(Tuple2#(Bit#(64), PhysMemRequest#(serverAddrWidth)))  readDelayFifo <- mkSizedFIFOF(readLatency_I);
   FIFOF#(Tuple2#(Bit#(64),PhysMemRequest#(serverAddrWidth))) writeDelayFifo <- mkSizedFIFOF(writeLatency_I);

   FIFOF#(Tuple2#(Bit#(64), Bit#(MemTagSize))) bFifo <- mkSizedFIFOF(writeLatency_I);
   FIFOF#(Tuple2#(Bit#(MemTagSize),Bool)) taglastfifo <- mkFIFOF();
   rule increment_cycle;
      cycles <= cycles+1;
   endrule

   let read_jitter = True; //cycles[4:0] == 0;
   let write_jitter = True; //cycles[4:0] == 5;

   Reg#(Bit#(8))  burstReg <- mkReg(0);
   FIFO#(Bit#(8)) reqs <- mkSizedFIFO(32);

   let beat_shift = fromInteger(valueOf(beatShift));

   rule read_rule if (readDelayFifo.notEmpty() && (cycles-tpl_1(readDelayFifo.first) > readLatency));
	 match { .reqTime, .req } = readDelayFifo.first;
	 Bit#(BurstLenSize) readLen = readLenReg;
	 Bit#(32) readOffset = readOffsetReg;
	 Bit#(MemTagSize) tag = req.tag;
	 Bit#(8) handle = req.addr[39:32];

	 if (readLen == 0) begin
	    req_ar_b_ts <= cycles;
	    readLen     = req.burstLen>>beat_shift;
	    readOffset  = 0;
	 end
	 rw.readrequest(extend(handle), req.addr[31:0]+readOffset);
	 let last = (readLen == 1);
	 if (last)
	    readDelayFifo.deq();
	 taglastfifo.enq(tuple2(tag, last));
	 readLenReg <= readLen - 1;
	 readOffsetReg <= readOffset + fromInteger(valueOf(serverBusWidth)/8);
   endrule

   interface PhysMemReadServer read_server;
      interface Put readReq;
	 method Action put(PhysMemRequest#(serverAddrWidth) req);
            if (verbose) $display("%d axiSlave.read.readAddr %h bc %d", cycles, req.addr, req.burstLen);
	    //readAddrGenerator.request.put(req);
	    readDelayFifo.enq(tuple2(cycles,req));
	 endmethod
      endinterface
      interface Get readData;
	 method ActionValue#(MemData#(serverBusWidth)) get();
	     match { .tag, .last } <- toGet(taglastfifo).get();
 	     let v <- rw.readresponse();
	     return MemData { data: v, tag: tag, last: last };
	 endmethod
      endinterface
   endinterface
   interface PhysMemWriteServer write_server;
      interface Put writeReq;
	 method Action put(PhysMemRequest#(serverAddrWidth) req);
	 //$display("mkBsimHost::req_aw id=%d", req.tag);
	 writeDelayFifo.enq(tuple2(cycles,req));
	 endmethod
      endinterface
      interface Put writeData;
	 method Action put(MemData#(serverBusWidth) resp) if (writeDelayFifo.notEmpty && (cycles-tpl_1(writeDelayFifo.first)) > writeLatency);
	    match { .reqTime, .req } = writeDelayFifo.first;
	    Bit#(BurstLenSize) writeLen = writeLenReg;
	    Bit#(32) writeOffset = writeOffsetReg;
	    Bit#(MemTagSize) tag = req.tag;
	    Bit#(8) handle = req.addr[39:32];
	    if (writeLenReg == 0) begin
	       req_aw_b_ts <= cycles;
	       writeLen = req.burstLen>>beat_shift;
	       writeOffset = 0;
	    end
	    rw.write(extend(handle), req.addr[31:0] + writeOffset, resp.data);
	    writeLenReg <= writeLen - 1;
	    writeOffsetReg <= writeOffset + fromInteger(valueOf(serverBusWidth)/8);
	    if (writeLen == 1) begin
	       bFifo.enq(tuple2(cycles,tag));
	       writeDelayFifo.deq;
	    end
	 endmethod
      endinterface
      interface Get writeDone;
	 method ActionValue#(Bit#(MemTagSize)) get() if ((cycles-tpl_1(bFifo.first)) > writeLatency);
	 bFifo.deq();
	 return tpl_2(bFifo.first());
	 endmethod
      endinterface
   endinterface
endmodule