make sure to shift readData.tag right by serverIdxSz to recover original tag

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
import Vector::*;
import Cntrs::*;
import FIFOF::*;
import FIFO::*;
import GetPut::*;
import Assert::*;
import ClientServer::*;
import BRAM::*;
import BRAMFIFO::*;
import ConfigCounter::*;
import Connectable::*;
import ConnectalMemory::*;
import ConnectalBramFifo::*;
import MemTypes::*;
import Pipe::*;
import MemUtils::*;

`include "ConnectalProjectConfig.bsv"

module mkMemReadEngine(MemReadEngine#(busWidth, userWidth, cmdQDepth, numServers))
   provisos( Mul#(TDiv#(busWidth, 8), 8, busWidth)
	    ,Add#(1, a__, numServers)
	    ,Add#(busWidth, 0, userWidth)
	    ,Add#(b__, TLog#(numServers), TAdd#(1, TLog#(TMul#(cmdQDepth,numServers))))
	    ,Add#(c__, TLog#(numServers), TLog#(TMul#(cmdQDepth, numServers)))
	    ,Add#(d__, TLog#(numServers), 6)
	    );
   let rv <- mkMemReadEngineBuff(valueOf(cmdQDepth) * valueOf(TExp#(BurstLenSize)));
   return rv;
endmodule

typedef struct {
    Bit#(BurstLenSize) len;
    Bool               last;
} NextReq deriving (Bits, Eq);

function NextReq getNext(Bit#(32) len, Bit#(BurstLenSize) burst);
   NextReq v;
   v.last = (len <= extend(burst));
   v.len = v.last ? truncate(len) : burst;
   return v;
endfunction

typedef struct {
   Bit#(indexSize)    idx;
   SGLId              sglId;
   Bit#(32)           base;
   Bit#(MemTagSize)   tag;
   Bool               last;
   Bit#(BurstLenSize) burstLen;
} ServerRequest#(numeric type indexSize) deriving (Bits, Eq);

module mkMemReadEngineBuff#(Integer bufferSizeBytes) (MemReadEngine#(busWidth, userWidth, cmdQDepth, numServers))
   provisos (Div#(busWidth,8,busWidthBytes),
	     Mul#(busWidthBytes,8,busWidth),
	     Log#(busWidthBytes,beatShift),
	     Log#(cmdQDepth,logCmdQDepth),
	     Mul#(cmdQDepth,numServers,cmdBuffSz),
	     Log#(cmdBuffSz, cmdBuffAddrSz),
	     Log#(numServers, serverIdxSz),
	     Add#(busWidth, 0, userWidth),
	     Add#(1,logCmdQDepth, outCntSz),
	     Add#(1, c__, numServers),
	     Add#(b__, TLog#(numServers), cmdBuffAddrSz),
	     Add#(e__, TLog#(numServers), TAdd#(1, cmdBuffAddrSz)),
	     Add#(a__, serverIdxSz, cmdBuffAddrSz),
	     Min#(2,TLog#(numServers),bpc),
	     Add#(d__, TLog#(numServers), TAdd#(1, serverIdxSz)),
	     Add#(f__, serverIdxSz, 6));
   let verbose = False;
   let beatShift = fromInteger(valueOf(beatShift));

   let clock <- exposeCurrentClock();
   let reset <- exposeCurrentReset();

   Integer bufferSizeBeats = bufferSizeBytes/valueOf(busWidthBytes);
   Vector#(numServers, Reg#(Bool))          clientInFlight <- replicateM(mkReg(False));
   Vector#(numServers, ConfigCounter#(16))  clientAvail <- replicateM(mkConfigCounter(fromInteger(bufferSizeBeats)));
   Vector#(numServers, Reg#(MemengineCmd))  clientCommand <- replicateM(mkReg(unpack(0)));
`ifdef USE_DUAL_CLOCK_FIFOF
   Vector#(numServers, FIFOF#(MemDataF#(userWidth))) clientDataFifo <- replicateM(mkDualClockBramFIFOF(clock, reset, clock, reset));
`else
   Vector#(numServers, FIFOF#(MemDataF#(userWidth))) clientDataFifo <- replicateM(mkSizedBRAMFIFOF(bufferSizeBeats));
`endif
   Vector#(numServers, FIFO#(MemengineCmd)) clientRequest <- replicateM(mkFIFO());
   Vector#(numServers, Reg#(Bit#(32)))      clientLen <- replicateM(mkReg(unpack(0)));
   Vector#(numServers, Reg#(Bit#(32)))      clientBase <- replicateM(mkReg(unpack(0)));
   Vector#(numServers, Reg#(NextReq))       clientNext <- replicateM(mkReg(unpack(0)));
   Vector#(numServers, Count#(Bit#(32)))    clientCycles <- replicateM(mkCount(0));
   Vector#(numServers, FIFOF#(MemRequestCycles)) clientCyclesFifo <- replicateM(mkFIFOF());
   Vector#(numServers, FIFOF#(ServerRequest#(serverIdxSz))) clientSReq <- replicateM(mkSizedFIFOF(1));

   Vector#(numServers, FIFO#(Tuple3#(Bool,Bool,Bit#(BurstLenSize)))) serverCheckAvail <- replicateM(mkSizedFIFO(1));
   FIFO#(ServerRequest#(serverIdxSz))       serverRequest <- mkSizedFIFO(valueOf(cmdQDepth));
   FIFO#(Tuple3#(Bit#(8),Bit#(serverIdxSz),Bool)) serverProcessing <- mkSizedFIFO(valueOf(cmdQDepth));
   FIFOF#(MemData#(busWidth))                       serverDataFifo <- mkFIFOF;

   Reg#(Bit#(8))                    respCnt <- mkReg(0);
   Reg#(Bit#(32)) counter <- mkReg(0);

   mkFunnelNodeRR(map(toPipeOut, clientSReq), valueOf(numServers), toPut(serverRequest));

   rule incCounter;
      counter <= counter + 1;
   endrule

   for (Integer idx = 0; idx < valueOf(numServers); idx = idx + 1) begin
      rule rule_startNew if (!clientInFlight[idx]);
	 let cmd <- toGet(clientRequest[idx]).get();
	 clientInFlight[idx] <= True;
	 clientCommand[idx] <= cmd;
         clientLen[idx] <= cmd.len - extend(cmd.burstLen);
         clientBase[idx] <= cmd.base;
         clientNext[idx] <= getNext(cmd.len, cmd.burstLen);
	 clientCycles[idx] <= 0;
	 if (verbose) $display("mkMemReadEngineBuff::%d rule_startNew %d %d", counter, idx, clientAvail[idx].read);
      endrule
      rule rule_cycles;
	 clientCycles[idx].incr(1);
      endrule
      rule rule_checkAvail if (clientInFlight[idx]);
         let cmd_len = clientNext[idx].len;
         let last_burst = clientNext[idx].last;
         let cond0 <- clientAvail[idx].maybeDecrement(unpack(extend(cmd_len>>beatShift)));
         serverCheckAvail[idx].enq(tuple3(cond0,last_burst,cmd_len));
         if (verbose) $display("mkMemReadEngineBuff::%d rule_checkAvail avail[%d] %d burstLen %d cond0 %d last_burst %d", counter, idx, clientAvail[idx].read(), cmd_len>>beatShift, cond0, last_burst);
      endrule

      rule rule_requestServer if (clientInFlight[idx]);
         match {.cond0,.last_burst,.cmd_len} <- toGet(serverCheckAvail[idx]).get;
         if  (cond0) begin
	    if (verbose) $display("mkMemReadEngineBuff::%d rule_requestServer clientLen %d idx %d cond0 %d last_burst %d", counter, clientLen[idx], idx, cond0, last_burst);
	    clientSReq[idx].enq(ServerRequest{
                 idx:fromInteger(idx),sglId:clientCommand[idx].sglId,base:clientBase[idx],
                 tag:clientCommand[idx].tag,last:last_burst,burstLen:cmd_len});
            clientBase[idx] <= clientBase[idx] + extend(cmd_len);
            clientLen[idx] <= clientLen[idx] - extend(cmd_len);
            clientNext[idx] <= getNext(clientLen[idx], clientCommand[idx].burstLen);
	    if (last_burst) begin
	       if (verbose) $display("mkMemReadEngineBuff::%d rule_requestServer last_burst %d", counter, last_burst);
	       clientInFlight[idx] <= False;
`ifdef MEMENGINE_REQUEST_CYCLES
	       $display("clientCycles[%d] = %d", idx, clientCycles[idx]);
	       clientCyclesFifo[idx].enq(MemRequestCycles { tag:clientCommand[idx].tag, cycles: clientCycles[idx] });
`endif
	    end
         end
      endrule
   end

   rule read_data_rule;
      let d <- toGet(serverDataFifo).get();
      match {.rc, .idx, .last_burst} = serverProcessing.first;
      let new_respCnt = respCnt+1;
      let l = False;
      if (verbose) $display("mkMemReadEngineBuff::%d data %h new_respCnt %d rc %d last_burst %d idx %d clientInFlight %d eob %d", counter, d.data, new_respCnt, rc, last_burst, idx, clientInFlight[idx], d.last);
      if (new_respCnt == rc) begin
	 respCnt <= 0;
	 serverProcessing.deq;
	 //$display("eob %d", idx);
	 l = last_burst;
      end
      else begin
	 respCnt <= new_respCnt;
      end
      clientDataFifo[idx].enq(MemDataF { data: d.data, tag: d.tag >> serverIdxSz, first: (respCnt == 0), last: l});
   endrule

   Vector#(numServers, MemReadEngineServer#(userWidth)) rs;
   for(Integer i = 0; i < valueOf(numServers); i=i+1)
      rs[i] = (interface MemReadEngineServer#(userWidth);
		  interface Put request;
		     method Action put(MemengineCmd cmd);
`ifdef SIMULATION
			Bit#(32) bsb = fromInteger(bufferSizeBytes);
			Bit#(32) dw = fromInteger(valueOf(busWidthBytes));
			let mdw = ((cmd.len)/dw)*dw != cmd.len;
			let bbl = extend(cmd.burstLen) > bsb;
			if(bbl || mdw) begin
			   if (bbl)
			      $display("XXXXXXXXXX mkMemReadEngineBuff::unsupported burstLen %d %d", bsb, cmd.burstLen);
			   if (mdw)
			      $display("XXXXXXXXXX mkMemReadEngineBuff::unsupported len %d", cmd.len);
			end
			else
`endif
                           begin
			   clientRequest[i].enq(cmd);
			   end
 		     endmethod
		  endinterface
                  interface data = interface PipeOut;
	             method MemDataF#(userWidth) first;
	                return clientDataFifo[i].first;
	             endmethod
	             method Action deq;
	                if (verbose) $display("mkMemReadEngineBuff::check_out: idx %d data %h clientAvail %d eob %d", i, clientDataFifo[i].first.data, clientAvail[i].read(), clientDataFifo[i].first.last);
	                clientDataFifo[i].deq;
	                clientAvail[i].increment(1);
	             endmethod
	             method Bool notEmpty = clientDataFifo[i].notEmpty;
                  endinterface;
	       interface requestCycles = toPipeOut(clientCyclesFifo[i]);
               endinterface);
   interface readServers = rs;
   interface MemReadClient dmaClient;
      interface Get readReq;
	 method ActionValue#(MemRequest) get();
            let v <- toGet(serverRequest).get;
	    serverProcessing.enq(tuple3(truncate(v.burstLen>>beatShift), v.idx, v.last));
	    if (verbose) $display("MemReadEngine::%d readReq idx %d offset %h burstLenBytes %h last %d", counter, v.idx, v.base, v.burstLen, v.last);
	    return MemRequest { sglId: v.sglId, offset: extend(v.base),
               burstLen:v.burstLen, tag: (v.tag << valueOf(serverIdxSz)) | extend(v.idx)
`ifdef BYTE_ENABLES
			       , firstbe: maxBound, lastbe: maxBound
`endif
               };
	 endmethod
      endinterface
      interface Put readData = toPut(serverDataFifo);
   endinterface
endmodule

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
import Vector::*;
import Cntrs::*;
import FIFOF::*;
import FIFO::*;
import GetPut::*;
import Assert::*;
import ClientServer::*;
import BRAM::*;
import BRAMFIFO::*;
import ConfigCounter::*;
import Connectable::*;
import ConnectalMemory::*;
import ConnectalBramFifo::*;
import MemTypes::*;
import Pipe::*;
import MemUtils::*;

`include "ConnectalProjectConfig.bsv"

module mkMemReadEngine(MemReadEngine#(busWidth, userWidth, cmdQDepth, numServers))
   provisos( Mul#(TDiv#(busWidth, 8), 8, busWidth)
	    ,Add#(1, a__, numServers)
	    ,Add#(busWidth, 0, userWidth)
	    ,Add#(b__, TLog#(numServers), TAdd#(1, TLog#(TMul#(cmdQDepth,numServers))))
	    ,Add#(c__, TLog#(numServers), TLog#(TMul#(cmdQDepth, numServers)))
	    ,Add#(d__, TLog#(numServers), 6)
	    );
   let rv <- mkMemReadEngineBuff(valueOf(cmdQDepth) * valueOf(TExp#(BurstLenSize)));
   return rv;
endmodule

typedef struct {
    Bit#(BurstLenSize) len;
    Bool               last;
} NextReq deriving (Bits, Eq);

function NextReq getNext(Bit#(32) len, Bit#(BurstLenSize) burst);
   NextReq v;
   v.last = (len <= extend(burst));
   v.len = v.last ? truncate(len) : burst;
   return v;
endfunction

typedef struct {
   Bit#(indexSize)    idx;
   SGLId              sglId;
   Bit#(32)           base;
   Bit#(MemTagSize)   tag;
   Bool               last;
   Bit#(BurstLenSize) burstLen;
} ServerRequest#(numeric type indexSize) deriving (Bits, Eq);

module mkMemReadEngineBuff#(Integer bufferSizeBytes) (MemReadEngine#(busWidth, userWidth, cmdQDepth, numServers))
   provisos (Div#(busWidth,8,busWidthBytes),
	     Mul#(busWidthBytes,8,busWidth),
	     Log#(busWidthBytes,beatShift),
	     Log#(cmdQDepth,logCmdQDepth),
	     Mul#(cmdQDepth,numServers,cmdBuffSz),
	     Log#(cmdBuffSz, cmdBuffAddrSz),
	     Log#(numServers, serverIdxSz),
	     Add#(busWidth, 0, userWidth),
	     Add#(1,logCmdQDepth, outCntSz),
	     Add#(1, c__, numServers),
	     Add#(b__, TLog#(numServers), cmdBuffAddrSz),
	     Add#(e__, TLog#(numServers), TAdd#(1, cmdBuffAddrSz)),
	     Add#(a__, serverIdxSz, cmdBuffAddrSz),
	     Min#(2,TLog#(numServers),bpc),
	     Add#(d__, TLog#(numServers), TAdd#(1, serverIdxSz)),
	     Add#(f__, serverIdxSz, 6));
   let verbose = False;
   let beatShift = fromInteger(valueOf(beatShift));

   let clock <- exposeCurrentClock();
   let reset <- exposeCurrentReset();

   Integer bufferSizeBeats = bufferSizeBytes/valueOf(busWidthBytes);
   Vector#(numServers, Reg#(Bool))          clientInFlight <- replicateM(mkReg(False));
   Vector#(numServers, ConfigCounter#(16))  clientAvail <- replicateM(mkConfigCounter(fromInteger(bufferSizeBeats)));
   Vector#(numServers, Reg#(MemengineCmd))  clientCommand <- replicateM(mkReg(unpack(0)));
`ifdef USE_DUAL_CLOCK_FIFOF
   Vector#(numServers, FIFOF#(MemDataF#(userWidth))) clientDataFifo <- replicateM(mkDualClockBramFIFOF(clock, reset, clock, reset));
`else
   Vector#(numServers, FIFOF#(MemDataF#(userWidth))) clientDataFifo <- replicateM(mkSizedBRAMFIFOF(bufferSizeBeats));
`endif
   Vector#(numServers, FIFO#(MemengineCmd)) clientRequest <- replicateM(mkFIFO());
   Vector#(numServers, Reg#(Bit#(32)))      clientLen <- replicateM(mkReg(unpack(0)));
   Vector#(numServers, Reg#(Bit#(32)))      clientBase <- replicateM(mkReg(unpack(0)));
   Vector#(numServers, Reg#(NextReq))       clientNext <- replicateM(mkReg(unpack(0)));
   Vector#(numServers, Count#(Bit#(32)))    clientCycles <- replicateM(mkCount(0));
   Vector#(numServers, FIFOF#(MemRequestCycles)) clientCyclesFifo <- replicateM(mkFIFOF());
   Vector#(numServers, FIFOF#(ServerRequest#(serverIdxSz))) clientSReq <- replicateM(mkSizedFIFOF(1));

   Vector#(numServers, FIFO#(Tuple3#(Bool,Bool,Bit#(BurstLenSize)))) serverCheckAvail <- replicateM(mkSizedFIFO(1));
   FIFO#(ServerRequest#(serverIdxSz))       serverRequest <- mkSizedFIFO(valueOf(cmdQDepth));
   FIFO#(Tuple3#(Bit#(8),Bit#(serverIdxSz),Bool)) serverProcessing <- mkSizedFIFO(valueOf(cmdQDepth));
   FIFOF#(MemData#(busWidth))                       serverDataFifo <- mkFIFOF;

   Reg#(Bit#(8))                    respCnt <- mkReg(0);
   Reg#(Bit#(32)) counter <- mkReg(0);

   mkFunnelNodeRR(map(toPipeOut, clientSReq), valueOf(numServers), toPut(serverRequest));

   rule incCounter;
      counter <= counter + 1;
   endrule

   for (Integer idx = 0; idx < valueOf(numServers); idx = idx + 1) begin
      rule rule_startNew if (!clientInFlight[idx]);
	 let cmd <- toGet(clientRequest[idx]).get();
	 clientInFlight[idx] <= True;
	 clientCommand[idx] <= cmd;
         clientLen[idx] <= cmd.len - extend(cmd.burstLen);
         clientBase[idx] <= cmd.base;
         clientNext[idx] <= getNext(cmd.len, cmd.burstLen);
	 clientCycles[idx] <= 0;
	 if (verbose) $display("mkMemReadEngineBuff::%d rule_startNew %d %d", counter, idx, clientAvail[idx].read);
      endrule
      rule rule_cycles;
	 clientCycles[idx].incr(1);
      endrule
      rule rule_checkAvail if (clientInFlight[idx]);
         let cmd_len = clientNext[idx].len;
         let last_burst = clientNext[idx].last;
         let cond0 <- clientAvail[idx].maybeDecrement(unpack(extend(cmd_len>>beatShift)));
         serverCheckAvail[idx].enq(tuple3(cond0,last_burst,cmd_len));
         if (verbose) $display("mkMemReadEngineBuff::%d rule_checkAvail avail[%d] %d burstLen %d cond0 %d last_burst %d", counter, idx, clientAvail[idx].read(), cmd_len>>beatShift, cond0, last_burst);
      endrule

      rule rule_requestServer if (clientInFlight[idx]);
         match {.cond0,.last_burst,.cmd_len} <- toGet(serverCheckAvail[idx]).get;
         if  (cond0) begin
	    if (verbose) $display("mkMemReadEngineBuff::%d rule_requestServer clientLen %d idx %d cond0 %d last_burst %d", counter, clientLen[idx], idx, cond0, last_burst);
	    clientSReq[idx].enq(ServerRequest{
                 idx:fromInteger(idx),sglId:clientCommand[idx].sglId,base:clientBase[idx],
                 tag:clientCommand[idx].tag,last:last_burst,burstLen:cmd_len});
            clientBase[idx] <= clientBase[idx] + extend(cmd_len);
            clientLen[idx] <= clientLen[idx] - extend(cmd_len);
            clientNext[idx] <= getNext(clientLen[idx], clientCommand[idx].burstLen);
	    if (last_burst) begin
	       if (verbose) $display("mkMemReadEngineBuff::%d rule_requestServer last_burst %d", counter, last_burst);
	       clientInFlight[idx] <= False;
`ifdef MEMENGINE_REQUEST_CYCLES
	       $display("clientCycles[%d] = %d", idx, clientCycles[idx]);
	       clientCyclesFifo[idx].enq(MemRequestCycles { tag:clientCommand[idx].tag, cycles: clientCycles[idx] });
`endif
	    end
         end
      endrule
   end

   rule read_data_rule;
      let d <- toGet(serverDataFifo).get();
      match {.rc, .idx, .last_burst} = serverProcessing.first;
      let new_respCnt = respCnt+1;
      let l = False;
      if (verbose) $display("mkMemReadEngineBuff::%d data %h new_respCnt %d rc %d last_burst %d idx %d clientInFlight %d eob %d", counter, d.data, new_respCnt, rc, last_burst, idx, clientInFlight[idx], d.last);
      if (new_respCnt == rc) begin
	 respCnt <= 0;
	 serverProcessing.deq;
	 //$display("eob %d", idx);
	 l = last_burst;
      end
      else begin
	 respCnt <= new_respCnt;
      end
      clientDataFifo[idx].enq(MemDataF { data: d.data, tag: d.tag, first: (respCnt == 0), last: l});
   endrule

   Vector#(numServers, MemReadEngineServer#(userWidth)) rs;
   for(Integer i = 0; i < valueOf(numServers); i=i+1)
      rs[i] = (interface MemReadEngineServer#(userWidth);
		  interface Put request;
		     method Action put(MemengineCmd cmd);
`ifdef SIMULATION
			Bit#(32) bsb = fromInteger(bufferSizeBytes);
			Bit#(32) dw = fromInteger(valueOf(busWidthBytes));
			let mdw = ((cmd.len)/dw)*dw != cmd.len;
			let bbl = extend(cmd.burstLen) > bsb;
			if(bbl || mdw) begin
			   if (bbl)
			      $display("XXXXXXXXXX mkMemReadEngineBuff::unsupported burstLen %d %d", bsb, cmd.burstLen);
			   if (mdw)
			      $display("XXXXXXXXXX mkMemReadEngineBuff::unsupported len %d", cmd.len);
			end
			else
`endif
                           begin
			   clientRequest[i].enq(cmd);
			   end
 		     endmethod
		  endinterface
                  interface data = interface PipeOut;
	             method MemDataF#(userWidth) first;
	                return clientDataFifo[i].first;
	             endmethod
	             method Action deq;
	                if (verbose) $display("mkMemReadEngineBuff::check_out: idx %d data %h clientAvail %d eob %d", i, clientDataFifo[i].first.data, clientAvail[i].read(), clientDataFifo[i].first.last);
	                clientDataFifo[i].deq;
	                clientAvail[i].increment(1);
	             endmethod
	             method Bool notEmpty = clientDataFifo[i].notEmpty;
                  endinterface;
	       interface requestCycles = toPipeOut(clientCyclesFifo[i]);
               endinterface);
   interface readServers = rs;
   interface MemReadClient dmaClient;
      interface Get readReq;
	 method ActionValue#(MemRequest) get();
            let v <- toGet(serverRequest).get;
	    serverProcessing.enq(tuple3(truncate(v.burstLen>>beatShift), v.idx, v.last));
	    if (verbose) $display("MemReadEngine::%d readReq idx %d offset %h burstLenBytes %h last %d", counter, v.idx, v.base, v.burstLen, v.last);
	    return MemRequest { sglId: v.sglId, offset: extend(v.base),
               burstLen:v.burstLen, tag: (v.tag << valueOf(serverIdxSz)) | extend(v.idx)
`ifdef BYTE_ENABLES
			       , firstbe: maxBound, lastbe: maxBound
`endif
               };
	 endmethod
      endinterface
      interface Put readData = toPut(serverDataFifo);
   endinterface
endmodule


comment

comment

// Copyright (c) 2014 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

// bsv libraries
import Vector::*;

// Connectal libraries
import CtrlMux::*;
import Portal::*;
import Leds::*;
import MemTypes::*;
import MemPortal::*;
import HostInterface::*;

// generated by tool
import LedControllerRequest::*;

// defined by user
import LedController::*;

typedef enum {LedControllerRequestPortal} IfcNames deriving (Eq,Bits);

module mkConnectalTop(StdConnectalTop#(PhysAddrWidth));

   // instantiate user portals
   LedController ledController <- mkLedControllerRequest();
   LedControllerRequestWrapper ledControllerRequestWrapper <- mkLedControllerRequestWrapper(LedControllerRequestPortal,
											    ledController.request);

   Vector#(1,StdPortal) portals;
   portals[0] = ledControllerRequestWrapper.portalIfc;
   let ctrl_mux <- mkSlaveMux(portals);

   interface interrupt = getInterruptVector(portals);
   interface slave = ctrl_mux;
   interface masters = nil;
   interface leds = ledController.leds;

endmodule : mkConnectalTop

// Copyright (c) 2014 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

// bsv libraries
import Vector::*;

// portz libraries
import CtrlMux::*;
import Portal::*;
import Leds::*;
import MemTypes::*;
import MemPortal::*;
import HostInterface::*;

// generated by tool
import LedControllerRequest::*;

// defined by user
import LedController::*;

typedef enum {LedControllerRequestPortal} IfcNames deriving (Eq,Bits);

module mkConnectalTop(StdConnectalTop#(PhysAddrWidth));

   // instantiate user portals
   LedController ledController <- mkLedControllerRequest();
   LedControllerRequestWrapper ledControllerRequestWrapper <- mkLedControllerRequestWrapper(LedControllerRequestPortal,
											    ledController.request);

   Vector#(1,StdPortal) portals;
   portals[0] = ledControllerRequestWrapper.portalIfc;
   let ctrl_mux <- mkSlaveMux(portals);

   interface interrupt = getInterruptVector(portals);
   interface slave = ctrl_mux;
   interface masters = nil;
   interface leds = ledController.leds;

endmodule : mkConnectalTop


remove dependence on aggressive conditions from mkMemReadInternal.read_client.readData.put

Change-Id: I50d38ddf6d0061ad31807ecae0f926e38c3346c5

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

// BSV Libraries
import BRAMFIFO::*;
import FIFO::*;
import FIFOF::*;
import Vector::*;
import GetPut::*;
import ClientServer::*;
import Assert::*;

// XBSV Libraries
import MemTypes::*;
import PortalMemory::*;
import SGList::*;

typedef 9 SGL_PIPELINE_DEPTH;
typedef 32 TAG_DEPTH;

interface MemWriteInternal#(numeric type addrWidth, numeric type dataWidth);
   interface DmaDbg dbg;
   interface MemWriteClient#(addrWidth,dataWidth) write_client;
endinterface

interface MemReadInternal#(numeric type addrWidth, numeric type dataWidth);
   interface DmaDbg dbg;
   interface MemReadClient#(addrWidth,dataWidth) read_client;
endinterface

function Bool bad_pointer(ObjectPointer p);
   return (p > fromInteger(valueOf(MaxNumSGLists)) || p == 0);
endfunction

interface TagGen#(numeric type numClients, numeric type numTags);
   method Action tag_request(Bit#(TLog#(numClients)) client, Bit#(6) orig_tag);
   method ActionValue#(Bit#(TLog#(numTags)))  tag_response;
   method Action return_tag(Bit#(TLog#(numTags)) tag);
endinterface

module mkTagGenOO(TagGen#(numClients,numTags))
   provisos(Log#(numTags,tagWidth),
	    Log#(numClients,clientWidth));

   let request_fifo0 <- mkSizedFIFOF(valueOf(SGL_PIPELINE_DEPTH));
   let request_fifo1 <- mkSizedFIFOF(valueOf(SGL_PIPELINE_DEPTH));
   let return_fifo <- mkFIFO;
   Vector#(numTags, Reg#(Bit#(TLog#(TAG_DEPTH)))) tag_regs <- replicateM(mkReg(0));
   Vector#(numTags, Reg#(Maybe#(Tuple2#(Bit#(clientWidth),Bit#(6))))) client_map <- replicateM(mkReg(tagged Invalid));
   Maybe#(UInt#(tagWidth)) next_free = findElem(0, readVReg(tag_regs));

   rule return_tag_rule;
      return_fifo.deq;
      let tag = return_fifo.first;
      tag_regs[tag] <= tag_regs[tag]-1;
      if (tag_regs[tag]==1)
	 client_map[tag] <= tagged Invalid;
   endrule

   rule tag_request_rule;
      request_fifo0.deq;
      let client = tpl_2(request_fifo0.first);
      let orig_tag = tpl_3(request_fifo0.first);
      let rv = case  (tpl_1(request_fifo0.first)) matches
		  tagged Valid .tag: return (_when_(tag_regs[tag] < maxBound) (tag));
		  tagged Invalid: return (_when_(isValid(next_free)) (fromMaybe(?, next_free)));
	       endcase;
      request_fifo1.enq(tuple3(rv,client,orig_tag));
   endrule

   method Action tag_request(Bit#(clientWidth) client, Bit#(6) orig_tag) if (request_fifo0.notFull);
      let rv = findElem(tagged Valid tuple2(client,orig_tag), readVReg(client_map));
      request_fifo0.enq(tuple3(rv,client,orig_tag));
   endmethod

   method ActionValue#(Bit#(tagWidth)) tag_response;
      request_fifo1.deq;
      let rv = tpl_1(request_fifo1.first);
      let client = tpl_2(request_fifo1.first);
      let orig_tag = tpl_3(request_fifo1.first);
      client_map[rv] <= tagged Valid tuple2(client,orig_tag);
      tag_regs[rv] <= tag_regs[rv]+1;
      return extend(pack(rv));
   endmethod

   method Action return_tag(Bit#(tagWidth) tag);
      return_fifo.enq(tag);
   endmethod

endmodule

module mkTagGenIO(TagGen#(numClients,numTags))
   provisos(Log#(numTags,tagWidth),
	    Log#(numClients,clientWidth),
	    Bits#(Bit#(clientWidth), tagWidth));

   let request_fifo <- mkSizedFIFO(valueOf(SGL_PIPELINE_DEPTH));

   method Action tag_request(Bit#(clientWidth) client, Bit#(6) client_tag);
      request_fifo.enq(client);
   endmethod

   method ActionValue#(Bit#(tagWidth)) tag_response;
      request_fifo.deq;
      let client = request_fifo.first;
      return pack(client);
   endmethod

   method Action return_tag(Bit#(tagWidth) tag);
      noAction;
   endmethod

endmodule

typedef struct {ObjectRequest req;
		Bit#(TLog#(numClients)) client; } LRec#(numeric type numClients, numeric type numTags, numeric type addrWidth) deriving(Bits);

typedef struct {ObjectRequest req;
		Bit#(addrWidth) pa;
		Bit#(TLog#(numTags)) rename_tag;
		Bit#(TLog#(numClients)) client; } RRec#(numeric type numClients, numeric type numTags, numeric type addrWidth) deriving(Bits);

typedef struct {ObjectRequest req;
		Bit#(TLog#(numTags)) rename_tag;
		Bit#(TLog#(numClients)) client; } DRec#(numeric type numClients, numeric type numTags, numeric type addrWidth) deriving(Bits);

typedef struct {Bit#(6) orig_tag;
		Bit#(TLog#(numClients)) client; } RResp#(numeric type numClients, numeric type numTags, numeric type addrWidth) deriving(Bits);

module mkMemReadInternal#(Integer id,
			  Vector#(numClients, ObjectReadClient#(dataWidth)) readClients,
			  DmaIndication dmaIndication,
			  Server#(Tuple2#(SGListId,Bit#(ObjectOffsetSize)),Bit#(addrWidth)) sgl,
			  TagGen#(numClients, numTags) tag_gen)
   (MemReadInternal#(addrWidth, dataWidth))

   provisos(Add#(b__, addrWidth, 64),
	    Add#(c__, 12, addrWidth),
	    Add#(1, c__, d__),
	    Div#(dataWidth,8,dataWidthBytes),
	    Mul#(dataWidthBytes,8,dataWidth),
	    Log#(dataWidthBytes,beatShift),
	    Add#(a__, TLog#(numTags), 6));

   // stage 0: address translation (latency = SGL_PIPELINE_DEPTH)
   FIFO#(LRec#(numClients,numTags,addrWidth)) lreqFifo <- mkSizedFIFO(valueOf(SGL_PIPELINE_DEPTH));
   // stage 1: address validation (latency = 1)
   FIFO#(RRec#(numClients,numTags,addrWidth))  reqFifo <- mkFIFO;
   // stage 2: read commands (maximum buffering to handle high latency read response times)
   Vector#(numTags, FIFO#(DRec#(numClients,numTags,addrWidth))) dreqFifos <- replicateM(mkSizedFIFO(valueOf(TAG_DEPTH)));
   // stage 3: read data (minimal buffering required)
   FIFO#(Tuple2#(DRec#(numClients,numTags,addrWidth),MemData#(dataWidth))) readDataPipelineFifo <- mkFIFO;
   FIFO#(MemData#(dataWidth)) responseFifo <- mkFIFO;
   Vector#(numTags, Reg#(Bit#(8)))           burstRegs <- replicateM(mkReg(0));
   Reg#(Bit#(64))  beatCount <- mkReg(0);
   let beat_shift = fromInteger(valueOf(beatShift));

   // performance analytics
   Reg#(Bit#(64)) cycle_cnt <- mkReg(0);
   Reg#(Bit#(64)) last_loadClient <- mkReg(0);
   Reg#(Bit#(64)) last_sglResp <- mkReg(0);
   Reg#(Bit#(64)) last_eob <- mkReg(0);
   (* fire_when_enabled *)
   rule cycle;
      cycle_cnt <= cycle_cnt+1;
   endrule

   for (Integer selectReg = 0; selectReg < valueOf(numClients); selectReg = selectReg + 1)
      rule loadClient;
      	 //$display("mkMemReadInternal::loadClient %d %d", selectReg, cycle_cnt-last_loadClient);
	 //last_loadClient <= cycle_cnt;
   	 ObjectRequest req <- readClients[selectReg].readReq.get();
   	 if (bad_pointer(req.pointer))
   	    dmaIndication.badPointer(req.pointer);
   	 else begin
	    tag_gen.tag_request(fromInteger(selectReg), req.tag);
   	    lreqFifo.enq(LRec{req:req, client:fromInteger(selectReg)});
   	    sgl.request.put(tuple2(truncate(req.pointer),req.offset));
   	 end
      endrule

   rule checkSglResp;
      let physAddr <- sgl.response.get;
      let req = lreqFifo.first.req;
      let client = lreqFifo.first.client;
      let rename_tag <- tag_gen.tag_response;
      lreqFifo.deq();
      if (physAddr <= (1 << valueOf(SGListPageShift0))) begin
	 // squash request
	 $display("dmaRead: badAddr pointer=%d offset=%h physAddr=%h", req.pointer, req.offset, physAddr);
	 dmaIndication.badAddr(req.pointer, extend(req.offset), extend(physAddr));
	 tag_gen.return_tag(rename_tag);
      end
      else begin
	 if (False && physAddr[31:24] != 0)
	    $display("checkSglResp: funny physAddr req.pointer=%d req.offset=%h physAddr=%h", req.pointer, req.offset, physAddr);
	 reqFifo.enq(RRec{req:req, pa:physAddr, client:client, rename_tag:extend(rename_tag)});
	 //$display("checkSglResp: client=%d, rename_tag=%d", client,rename_tag);
      end
      //$display("mkMemReadInternal::sglResp %d %d", client, cycle_cnt-last_sglResp);
      //last_sglResp <= cycle_cnt;
   endrule

   rule readDataComp;
      readDataPipelineFifo.deq;
      let drq = tpl_1(readDataPipelineFifo.first);
      let response = tpl_2(readDataPipelineFifo.first);
      let client = drq.client;
      let req = drq.req;
      readClients[client].readData.put(ObjectData { data: response.data, tag: req.tag, last: False});
      beatCount <= beatCount+1;
   endrule

   rule read_client_response;
      let response <- toGet(responseFifo).get();
      Bit#(6) response_tag = response.tag;
      let dreqFifo = dreqFifos[response_tag];
      dynamicAssert(truncate(response_tag) == dreqFifo.first.rename_tag, "mkMemReadInternal");
      readDataPipelineFifo.enq(tuple2(dreqFifo.first, response));
      let burstLen = burstRegs[response_tag];
      if (burstLen == 0)
	 burstLen = dreqFifo.first.req.burstLen >> beat_shift;
      if (burstLen == 1) begin
	 //$display("mkMemReadInternal::eob %d", cycle_cnt-last_eob);
	 last_eob <= cycle_cnt;
	 dreqFifo.deq();
	 tag_gen.return_tag(truncate(response_tag));
      end
      burstRegs[response_tag] <= burstLen-1;
   endrule

   interface MemReadClient read_client;
      interface Get readReq;
	 method ActionValue#(MemRequest#(addrWidth)) get();
	    reqFifo.deq;
	    let req = reqFifo.first.req;
	    let physAddr = reqFifo.first.pa;
	    let client = reqFifo.first.client;
	    let rename_tag = reqFifo.first.rename_tag;
	    if (False && physAddr[31:24] != 0)
	       $display("req_ar: funny physAddr req.pointer=%d req.offset=%h physAddr=%h", req.pointer, req.offset, physAddr);
	    dreqFifos[rename_tag].enq(DRec{req:req, client:client, rename_tag:rename_tag});
	    //$display("readReq: client=%d, rename_tag=%d, physAddr=%h", client,rename_tag,physAddr);
	    return MemRequest{addr:physAddr, burstLen:req.burstLen, tag:extend(rename_tag)};
	 endmethod
      endinterface
      interface Put readData = toPut(responseFifo);
   endinterface
   interface DmaDbg dbg;
      method ActionValue#(DmaDbgRec) dbg();
	 return DmaDbgRec{x:fromInteger(valueOf(numClients)), y:0, z:0, w:0};
      endmethod
      method ActionValue#(Bit#(64)) getMemoryTraffic();
	 return beatCount;
      endmethod
   endinterface
endmodule

module mkMemWriteInternal#(Integer iid,
			   Vector#(numClients, ObjectWriteClient#(dataWidth)) writeClients,
			   DmaIndication dmaIndication,
			   Server#(Tuple2#(SGListId,Bit#(ObjectOffsetSize)),Bit#(addrWidth)) sgl,
			   TagGen#(numClients, numTags) tag_gen)
   (MemWriteInternal#(addrWidth, dataWidth))

   provisos(Add#(b__, addrWidth, 64),
	    Add#(c__, 12, addrWidth),
	    Add#(1, c__, d__),
	    Div#(dataWidth,8,dataWidthBytes),
	    Mul#(dataWidthBytes,8,dataWidth),
	    Log#(dataWidthBytes,beatShift),
	    Add#(a__, TLog#(numTags), 6));

   // stage 0: address translation (latency = SGL_PIPELINE_DEPTH)
   FIFO#(LRec#(numClients,numTags,addrWidth)) lreqFifo <- mkSizedFIFO(valueOf(SGL_PIPELINE_DEPTH));
   // stage 1: address validation (latency = 1)
   FIFO#(RRec#(numClients,numTags,addrWidth))  reqFifo <- mkFIFO;
   // stage 2: write commands (maximum buffering to handle high latency writes)
   FIFO#(DRec#(numClients,numTags,addrWidth)) dreqFifo <- mkSizedBRAMFIFO(valueOf(TMul#(TAG_DEPTH,numTags)));
   // stage 3: write data (maximum buffering, though I have no idea if any hosts will begin the next data transfer before sending the write ack)
   Vector#(numTags, FIFO#(RResp#(numClients,numTags,addrWidth))) respFifos <- replicateM(mkSizedFIFO(valueOf(TAG_DEPTH)));
   // stage 4: write done (minimal buffering required)
   FIFO#(Tuple2#(RResp#(numClients,numTags,addrWidth),Bit#(6))) writeDonePipelineFifo <- mkFIFO;

   Reg#(Bit#(8)) burstReg <- mkReg(0);
   Reg#(Bit#(64)) beatCount <- mkReg(0);
   let beat_shift = fromInteger(valueOf(beatShift));

   Reg#(Bit#(64)) cycle_cnt <- mkReg(0);
   Reg#(Bit#(64)) last_loadClient <- mkReg(0);
   Reg#(Bit#(64)) last_sglResp <- mkReg(0);
   Reg#(Bit#(64)) last_eob <- mkReg(0);
   (* fire_when_enabled *)
   rule cycle;
      cycle_cnt <= cycle_cnt+1;
   endrule

   for (Integer selectReg = 0; selectReg < valueOf(numClients); selectReg = selectReg + 1)
       rule loadClient;
      	  //$display("mkMemWriteInternal::loadClient %d %d", selectReg, cycle_cnt-last_loadClient);
	  //last_loadClient <= cycle_cnt;
   	  ObjectRequest req <- writeClients[selectReg].writeReq.get();
   	  if (bad_pointer(req.pointer))
   	     dmaIndication.badPointer(req.pointer);
   	  else begin
	     tag_gen.tag_request(fromInteger(selectReg), req.tag);
   	     lreqFifo.enq(LRec{req:req, client:fromInteger(selectReg)});
   	     sgl.request.put(tuple2(truncate(req.pointer),req.offset));
   	  end
       endrule

   rule checkSglResp;
      let physAddr <- sgl.response.get;
      let req = lreqFifo.first.req;
      let client = lreqFifo.first.client;
      let rename_tag <- tag_gen.tag_response;
      lreqFifo.deq();
      if (physAddr <= (1 << valueOf(SGListPageShift0))) begin
	 // squash request
	 $display("dmaWrite: badAddr handle=%d addr=%h physAddr=%h", req.pointer, req.offset, physAddr);
	 dmaIndication.badAddr(req.pointer, extend(req.offset), extend(physAddr));
	 tag_gen.return_tag(rename_tag);
      end
      else begin
	 reqFifo.enq(RRec{req:req, pa:physAddr, client:client, rename_tag:extend(rename_tag)});
	 //$display("checkSglResp: client=%d, rename_tag=%d", client,rename_tag);
      end
      //$display("mkMemWriteInternal::sglResp %d %d", client, cycle_cnt-last_sglResp);
      //last_sglResp <= cycle_cnt;
   endrule

   rule writeDoneComp;
      writeDonePipelineFifo.deq;
      let client = tpl_1(writeDonePipelineFifo.first).client;
      let orig_tag = tpl_1(writeDonePipelineFifo.first).orig_tag;
      let response_tag = tpl_2(writeDonePipelineFifo.first);
      writeClients[client].writeDone.put(orig_tag);
      tag_gen.return_tag(truncate(response_tag));
   endrule

   interface MemWriteClient write_client;
      interface Get writeReq;
	 method ActionValue#(MemRequest#(addrWidth)) get();
	    let req = reqFifo.first.req;
	    let physAddr = reqFifo.first.pa;
	    let client = reqFifo.first.client;
	    let rename_tag = reqFifo.first.rename_tag;
	    reqFifo.deq;
	    dreqFifo.enq(DRec{req:req, client:client, rename_tag:rename_tag});
	    //$display("writeReq: client=%d, rename_tag=%d", client,rename_tag);
	    return MemRequest{addr:physAddr, burstLen:req.burstLen, tag:extend(rename_tag)};
	 endmethod
      endinterface
      interface Get writeData;
	 method ActionValue#(MemData#(dataWidth)) get();
	    let client = dreqFifo.first.client;
	    let req = dreqFifo.first.req;
	    let rename_tag =dreqFifo.first.rename_tag;
	    ObjectData#(dataWidth) tagdata <- writeClients[client].writeData.get();
	    let burstLen = burstReg;
	    if (burstLen == 0) begin
	       burstLen = req.burstLen >> beat_shift;
	       respFifos[rename_tag].enq(RResp{orig_tag:req.tag, client:client});
	    end
	    burstReg <= burstLen-1;
	    beatCount <= beatCount+1;
	    if (burstLen == 1)
	       dreqFifo.deq();
	    //$display("writeData: client=%d, rename_tag=%d", client, rename_tag);
	    return MemData { data: tagdata.data,  tag:extend(rename_tag), last: False };
	 endmethod
      endinterface
      interface Put writeDone;
	 method Action put(Bit#(6) resp);
	    let response_tag = resp;
	    let respFifo = respFifos[response_tag];
	    writeDonePipelineFifo.enq(tuple2(respFifo.first,resp));
	    respFifo.deq;
	 endmethod
      endinterface
   endinterface
   interface DmaDbg dbg;
      method ActionValue#(DmaDbgRec) dbg();
	 return DmaDbgRec{x:fromInteger(valueOf(numClients)), y:?, z:?, w:?};
      endmethod
      method ActionValue#(Bit#(64)) getMemoryTraffic();
	 return beatCount;
      endmethod
   endinterface
endmodule

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

// BSV Libraries
import BRAMFIFO::*;
import FIFO::*;
import FIFOF::*;
import Vector::*;
import GetPut::*;
import ClientServer::*;
import Assert::*;

// XBSV Libraries
import MemTypes::*;
import PortalMemory::*;
import SGList::*;

typedef 9 SGL_PIPELINE_DEPTH;
typedef 32 TAG_DEPTH;

interface MemWriteInternal#(numeric type addrWidth, numeric type dataWidth);
   interface DmaDbg dbg;
   interface MemWriteClient#(addrWidth,dataWidth) write_client;
endinterface

interface MemReadInternal#(numeric type addrWidth, numeric type dataWidth);
   interface DmaDbg dbg;
   interface MemReadClient#(addrWidth,dataWidth) read_client;
endinterface

function Bool bad_pointer(ObjectPointer p);
   return (p > fromInteger(valueOf(MaxNumSGLists)) || p == 0);
endfunction

interface TagGen#(numeric type numClients, numeric type numTags);
   method Action tag_request(Bit#(TLog#(numClients)) client, Bit#(6) orig_tag);
   method ActionValue#(Bit#(TLog#(numTags)))  tag_response;
   method Action return_tag(Bit#(TLog#(numTags)) tag);
endinterface

module mkTagGenOO(TagGen#(numClients,numTags))
   provisos(Log#(numTags,tagWidth),
	    Log#(numClients,clientWidth));

   let request_fifo0 <- mkSizedFIFOF(valueOf(SGL_PIPELINE_DEPTH));
   let request_fifo1 <- mkSizedFIFOF(valueOf(SGL_PIPELINE_DEPTH));
   let return_fifo <- mkFIFO;
   Vector#(numTags, Reg#(Bit#(TLog#(TAG_DEPTH)))) tag_regs <- replicateM(mkReg(0));
   Vector#(numTags, Reg#(Maybe#(Tuple2#(Bit#(clientWidth),Bit#(6))))) client_map <- replicateM(mkReg(tagged Invalid));
   Maybe#(UInt#(tagWidth)) next_free = findElem(0, readVReg(tag_regs));

   rule return_tag_rule;
      return_fifo.deq;
      let tag = return_fifo.first;
      tag_regs[tag] <= tag_regs[tag]-1;
      if (tag_regs[tag]==1)
	 client_map[tag] <= tagged Invalid;
   endrule

   rule tag_request_rule;
      request_fifo0.deq;
      let client = tpl_2(request_fifo0.first);
      let orig_tag = tpl_3(request_fifo0.first);
      let rv = case  (tpl_1(request_fifo0.first)) matches
		  tagged Valid .tag: return (_when_(tag_regs[tag] < maxBound) (tag));
		  tagged Invalid: return (_when_(isValid(next_free)) (fromMaybe(?, next_free)));
	       endcase;
      request_fifo1.enq(tuple3(rv,client,orig_tag));
   endrule

   method Action tag_request(Bit#(clientWidth) client, Bit#(6) orig_tag) if (request_fifo0.notFull);
      let rv = findElem(tagged Valid tuple2(client,orig_tag), readVReg(client_map));
      request_fifo0.enq(tuple3(rv,client,orig_tag));
   endmethod

   method ActionValue#(Bit#(tagWidth)) tag_response;
      request_fifo1.deq;
      let rv = tpl_1(request_fifo1.first);
      let client = tpl_2(request_fifo1.first);
      let orig_tag = tpl_3(request_fifo1.first);
      client_map[rv] <= tagged Valid tuple2(client,orig_tag);
      tag_regs[rv] <= tag_regs[rv]+1;
      return extend(pack(rv));
   endmethod

   method Action return_tag(Bit#(tagWidth) tag);
      return_fifo.enq(tag);
   endmethod

endmodule

module mkTagGenIO(TagGen#(numClients,numTags))
   provisos(Log#(numTags,tagWidth),
	    Log#(numClients,clientWidth),
	    Bits#(Bit#(clientWidth), tagWidth));

   let request_fifo <- mkSizedFIFO(valueOf(SGL_PIPELINE_DEPTH));

   method Action tag_request(Bit#(clientWidth) client, Bit#(6) client_tag);
      request_fifo.enq(client);
   endmethod

   method ActionValue#(Bit#(tagWidth)) tag_response;
      request_fifo.deq;
      let client = request_fifo.first;
      return pack(client);
   endmethod

   method Action return_tag(Bit#(tagWidth) tag);
      noAction;
   endmethod

endmodule

typedef struct {ObjectRequest req;
		Bit#(TLog#(numClients)) client; } LRec#(numeric type numClients, numeric type numTags, numeric type addrWidth) deriving(Bits);

typedef struct {ObjectRequest req;
		Bit#(addrWidth) pa;
		Bit#(TLog#(numTags)) rename_tag;
		Bit#(TLog#(numClients)) client; } RRec#(numeric type numClients, numeric type numTags, numeric type addrWidth) deriving(Bits);

typedef struct {ObjectRequest req;
		Bit#(TLog#(numTags)) rename_tag;
		Bit#(TLog#(numClients)) client; } DRec#(numeric type numClients, numeric type numTags, numeric type addrWidth) deriving(Bits);

typedef struct {Bit#(6) orig_tag;
		Bit#(TLog#(numClients)) client; } RResp#(numeric type numClients, numeric type numTags, numeric type addrWidth) deriving(Bits);

module mkMemReadInternal#(Integer id,
			  Vector#(numClients, ObjectReadClient#(dataWidth)) readClients,
			  DmaIndication dmaIndication,
			  Server#(Tuple2#(SGListId,Bit#(ObjectOffsetSize)),Bit#(addrWidth)) sgl,
			  TagGen#(numClients, numTags) tag_gen)
   (MemReadInternal#(addrWidth, dataWidth))

   provisos(Add#(b__, addrWidth, 64),
	    Add#(c__, 12, addrWidth),
	    Add#(1, c__, d__),
	    Div#(dataWidth,8,dataWidthBytes),
	    Mul#(dataWidthBytes,8,dataWidth),
	    Log#(dataWidthBytes,beatShift),
	    Add#(a__, TLog#(numTags), 6));

   // stage 0: address translation (latency = SGL_PIPELINE_DEPTH)
   FIFO#(LRec#(numClients,numTags,addrWidth)) lreqFifo <- mkSizedFIFO(valueOf(SGL_PIPELINE_DEPTH));
   // stage 1: address validation (latency = 1)
   FIFO#(RRec#(numClients,numTags,addrWidth))  reqFifo <- mkFIFO;
   // stage 2: read commands (maximum buffering to handle high latency read response times)
   Vector#(numTags, FIFO#(DRec#(numClients,numTags,addrWidth))) dreqFifos <- replicateM(mkSizedFIFO(valueOf(TAG_DEPTH)));
   // stage 3: read data (minimal buffering required)
   FIFO#(Tuple2#(DRec#(numClients,numTags,addrWidth),MemData#(dataWidth))) readDataPipelineFifo <- mkFIFO;
   Vector#(numTags, Reg#(Bit#(8)))           burstRegs <- replicateM(mkReg(0));
   Reg#(Bit#(64))  beatCount <- mkReg(0);
   let beat_shift = fromInteger(valueOf(beatShift));

   // performance analytics
   Reg#(Bit#(64)) cycle_cnt <- mkReg(0);
   Reg#(Bit#(64)) last_loadClient <- mkReg(0);
   Reg#(Bit#(64)) last_sglResp <- mkReg(0);
   Reg#(Bit#(64)) last_eob <- mkReg(0);
   (* fire_when_enabled *)
   rule cycle;
      cycle_cnt <= cycle_cnt+1;
   endrule

   for (Integer selectReg = 0; selectReg < valueOf(numClients); selectReg = selectReg + 1)
      rule loadClient;
      	 //$display("mkMemReadInternal::loadClient %d %d", selectReg, cycle_cnt-last_loadClient);
	 //last_loadClient <= cycle_cnt;
   	 ObjectRequest req <- readClients[selectReg].readReq.get();
   	 if (bad_pointer(req.pointer))
   	    dmaIndication.badPointer(req.pointer);
   	 else begin
	    tag_gen.tag_request(fromInteger(selectReg), req.tag);
   	    lreqFifo.enq(LRec{req:req, client:fromInteger(selectReg)});
   	    sgl.request.put(tuple2(truncate(req.pointer),req.offset));
   	 end
      endrule

   rule checkSglResp;
      let physAddr <- sgl.response.get;
      let req = lreqFifo.first.req;
      let client = lreqFifo.first.client;
      let rename_tag <- tag_gen.tag_response;
      lreqFifo.deq();
      if (physAddr <= (1 << valueOf(SGListPageShift0))) begin
	 // squash request
	 $display("dmaRead: badAddr pointer=%d offset=%h physAddr=%h", req.pointer, req.offset, physAddr);
	 dmaIndication.badAddr(req.pointer, extend(req.offset), extend(physAddr));
	 tag_gen.return_tag(rename_tag);
      end
      else begin
	 if (False && physAddr[31:24] != 0)
	    $display("checkSglResp: funny physAddr req.pointer=%d req.offset=%h physAddr=%h", req.pointer, req.offset, physAddr);
	 reqFifo.enq(RRec{req:req, pa:physAddr, client:client, rename_tag:extend(rename_tag)});
	 //$display("checkSglResp: client=%d, rename_tag=%d", client,rename_tag);
      end
      //$display("mkMemReadInternal::sglResp %d %d", client, cycle_cnt-last_sglResp);
      //last_sglResp <= cycle_cnt;
   endrule

   rule readDataComp;
      readDataPipelineFifo.deq;
      let drq = tpl_1(readDataPipelineFifo.first);
      let response = tpl_2(readDataPipelineFifo.first);
      let client = drq.client;
      let req = drq.req;
      readClients[client].readData.put(ObjectData { data: response.data, tag: req.tag, last: False});
      beatCount <= beatCount+1;
   endrule

   interface MemReadClient read_client;
      interface Get readReq;
	 method ActionValue#(MemRequest#(addrWidth)) get();
	    reqFifo.deq;
	    let req = reqFifo.first.req;
	    let physAddr = reqFifo.first.pa;
	    let client = reqFifo.first.client;
	    let rename_tag = reqFifo.first.rename_tag;
	    if (False && physAddr[31:24] != 0)
	       $display("req_ar: funny physAddr req.pointer=%d req.offset=%h physAddr=%h", req.pointer, req.offset, physAddr);
	    dreqFifos[rename_tag].enq(DRec{req:req, client:client, rename_tag:rename_tag});
	    //$display("readReq: client=%d, rename_tag=%d, physAddr=%h", client,rename_tag,physAddr);
	    return MemRequest{addr:physAddr, burstLen:req.burstLen, tag:extend(rename_tag)};
	 endmethod
      endinterface
      interface Put readData;
	 method Action put(MemData#(dataWidth) response);
	    Bit#(6) response_tag = response.tag;
	    let dreqFifo = dreqFifos[response_tag];
	    dynamicAssert(truncate(response_tag) == dreqFifo.first.rename_tag, "mkMemReadInternal");
	    readDataPipelineFifo.enq(tuple2(dreqFifo.first, response));
	    let burstLen = burstRegs[response_tag];
	    if (burstLen == 0)
	       burstLen = dreqFifo.first.req.burstLen >> beat_shift;
	    if (burstLen == 1) begin
	       //$display("mkMemReadInternal::eob %d", cycle_cnt-last_eob);
	       last_eob <= cycle_cnt;
	       dreqFifo.deq();
	       tag_gen.return_tag(truncate(response_tag));
	    end
	    burstRegs[response_tag] <= burstLen-1;
	 endmethod
      endinterface
   endinterface
   interface DmaDbg dbg;
      method ActionValue#(DmaDbgRec) dbg();
	 return DmaDbgRec{x:fromInteger(valueOf(numClients)), y:0, z:0, w:0};
      endmethod
      method ActionValue#(Bit#(64)) getMemoryTraffic();
	 return beatCount;
      endmethod
   endinterface
endmodule

module mkMemWriteInternal#(Integer iid,
			   Vector#(numClients, ObjectWriteClient#(dataWidth)) writeClients,
			   DmaIndication dmaIndication,
			   Server#(Tuple2#(SGListId,Bit#(ObjectOffsetSize)),Bit#(addrWidth)) sgl,
			   TagGen#(numClients, numTags) tag_gen)
   (MemWriteInternal#(addrWidth, dataWidth))

   provisos(Add#(b__, addrWidth, 64),
	    Add#(c__, 12, addrWidth),
	    Add#(1, c__, d__),
	    Div#(dataWidth,8,dataWidthBytes),
	    Mul#(dataWidthBytes,8,dataWidth),
	    Log#(dataWidthBytes,beatShift),
	    Add#(a__, TLog#(numTags), 6));

   // stage 0: address translation (latency = SGL_PIPELINE_DEPTH)
   FIFO#(LRec#(numClients,numTags,addrWidth)) lreqFifo <- mkSizedFIFO(valueOf(SGL_PIPELINE_DEPTH));
   // stage 1: address validation (latency = 1)
   FIFO#(RRec#(numClients,numTags,addrWidth))  reqFifo <- mkFIFO;
   // stage 2: write commands (maximum buffering to handle high latency writes)
   FIFO#(DRec#(numClients,numTags,addrWidth)) dreqFifo <- mkSizedBRAMFIFO(valueOf(TMul#(TAG_DEPTH,numTags)));
   // stage 3: write data (maximum buffering, though I have no idea if any hosts will begin the next data transfer before sending the write ack)
   Vector#(numTags, FIFO#(RResp#(numClients,numTags,addrWidth))) respFifos <- replicateM(mkSizedFIFO(valueOf(TAG_DEPTH)));
   // stage 4: write done (minimal buffering required)
   FIFO#(Tuple2#(RResp#(numClients,numTags,addrWidth),Bit#(6))) writeDonePipelineFifo <- mkFIFO;

   Reg#(Bit#(8)) burstReg <- mkReg(0);
   Reg#(Bit#(64)) beatCount <- mkReg(0);
   let beat_shift = fromInteger(valueOf(beatShift));

   Reg#(Bit#(64)) cycle_cnt <- mkReg(0);
   Reg#(Bit#(64)) last_loadClient <- mkReg(0);
   Reg#(Bit#(64)) last_sglResp <- mkReg(0);
   Reg#(Bit#(64)) last_eob <- mkReg(0);
   (* fire_when_enabled *)
   rule cycle;
      cycle_cnt <= cycle_cnt+1;
   endrule

   for (Integer selectReg = 0; selectReg < valueOf(numClients); selectReg = selectReg + 1)
       rule loadClient;
      	  //$display("mkMemWriteInternal::loadClient %d %d", selectReg, cycle_cnt-last_loadClient);
	  //last_loadClient <= cycle_cnt;
   	  ObjectRequest req <- writeClients[selectReg].writeReq.get();
   	  if (bad_pointer(req.pointer))
   	     dmaIndication.badPointer(req.pointer);
   	  else begin
	     tag_gen.tag_request(fromInteger(selectReg), req.tag);
   	     lreqFifo.enq(LRec{req:req, client:fromInteger(selectReg)});
   	     sgl.request.put(tuple2(truncate(req.pointer),req.offset));
   	  end
       endrule

   rule checkSglResp;
      let physAddr <- sgl.response.get;
      let req = lreqFifo.first.req;
      let client = lreqFifo.first.client;
      let rename_tag <- tag_gen.tag_response;
      lreqFifo.deq();
      if (physAddr <= (1 << valueOf(SGListPageShift0))) begin
	 // squash request
	 $display("dmaWrite: badAddr handle=%d addr=%h physAddr=%h", req.pointer, req.offset, physAddr);
	 dmaIndication.badAddr(req.pointer, extend(req.offset), extend(physAddr));
	 tag_gen.return_tag(rename_tag);
      end
      else begin
	 reqFifo.enq(RRec{req:req, pa:physAddr, client:client, rename_tag:extend(rename_tag)});
	 //$display("checkSglResp: client=%d, rename_tag=%d", client,rename_tag);
      end
      //$display("mkMemWriteInternal::sglResp %d %d", client, cycle_cnt-last_sglResp);
      //last_sglResp <= cycle_cnt;
   endrule

   rule writeDoneComp;
      writeDonePipelineFifo.deq;
      let client = tpl_1(writeDonePipelineFifo.first).client;
      let orig_tag = tpl_1(writeDonePipelineFifo.first).orig_tag;
      let response_tag = tpl_2(writeDonePipelineFifo.first);
      writeClients[client].writeDone.put(orig_tag);
      tag_gen.return_tag(truncate(response_tag));
   endrule

   interface MemWriteClient write_client;
      interface Get writeReq;
	 method ActionValue#(MemRequest#(addrWidth)) get();
	    let req = reqFifo.first.req;
	    let physAddr = reqFifo.first.pa;
	    let client = reqFifo.first.client;
	    let rename_tag = reqFifo.first.rename_tag;
	    reqFifo.deq;
	    dreqFifo.enq(DRec{req:req, client:client, rename_tag:rename_tag});
	    //$display("writeReq: client=%d, rename_tag=%d", client,rename_tag);
	    return MemRequest{addr:physAddr, burstLen:req.burstLen, tag:extend(rename_tag)};
	 endmethod
      endinterface
      interface Get writeData;
	 method ActionValue#(MemData#(dataWidth)) get();
	    let client = dreqFifo.first.client;
	    let req = dreqFifo.first.req;
	    let rename_tag =dreqFifo.first.rename_tag;
	    ObjectData#(dataWidth) tagdata <- writeClients[client].writeData.get();
	    let burstLen = burstReg;
	    if (burstLen == 0) begin
	       burstLen = req.burstLen >> beat_shift;
	       respFifos[rename_tag].enq(RResp{orig_tag:req.tag, client:client});
	    end
	    burstReg <= burstLen-1;
	    beatCount <= beatCount+1;
	    if (burstLen == 1)
	       dreqFifo.deq();
	    //$display("writeData: client=%d, rename_tag=%d", client, rename_tag);
	    return MemData { data: tagdata.data,  tag:extend(rename_tag), last: False };
	 endmethod
      endinterface
      interface Put writeDone;
	 method Action put(Bit#(6) resp);
	    let response_tag = resp;
	    let respFifo = respFifos[response_tag];
	    writeDonePipelineFifo.enq(tuple2(respFifo.first,resp));
	    respFifo.deq;
	 endmethod
      endinterface
   endinterface
   interface DmaDbg dbg;
      method ActionValue#(DmaDbgRec) dbg();
	 return DmaDbgRec{x:fromInteger(valueOf(numClients)), y:?, z:?, w:?};
      endmethod
      method ActionValue#(Bit#(64)) getMemoryTraffic();
	 return beatCount;
      endmethod
   endinterface
endmodule


increasing the size of the respFifo in mkAxiDmaWriteInternal increases write bw utilization on zynq platform from .5 to 1.0

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

// BSV Libraries
import FIFO::*;
import Vector::*;
import GetPut::*;
import ClientServer::*;
import Assert::*;

// XBSV Libraries
import AxiMasterSlave::*;
import Dma::*;
import PortalMemory::*;
import Adapter::*;
import SGList::*;

`ifdef BSIM
import "BDPI" function ActionValue#(Bit#(32)) pareff(Bit#(32) handle, Bit#(32) size);
`endif

//
// @brief AxiDma provides the configuration and AXI bus interface for Dma
//
// @param dsz Number of bits in the data bus
//

interface AxiDmaServer#(numeric type addrWidth, numeric type dsz);
   interface DmaConfig request;
   interface Axi3Master#(addrWidth,dsz,6) m_axi;
endinterface

interface AxiDmaWriteInternal#(numeric type addrWidth, numeric type dsz);
   interface DmaDbg dbg;
   interface Axi3Master#(addrWidth,dsz,6) m_axi;
   interface Get#(Tuple2#(Bit#(6),Bit#(6))) tagMismatch;
endinterface

interface AxiDmaReadInternal#(numeric type addrWidth, numeric type dsz);
   interface DmaDbg dbg;
   interface Axi3Master#(addrWidth,dsz,6) m_axi;
   interface Get#(Tuple2#(Bit#(6),Bit#(6))) tagMismatch;
endinterface

function Bool bad_pointer(DmaPointer p);
   return (p > fromInteger(valueOf(MaxNumSGLists)) || p == 0);
endfunction

typedef enum {Idle, Translate, Address, Data, Done} InternalState deriving(Eq,Bits);

typedef struct {DmaRequest req;
		Bit#(6)    id;
		Bit#(addrWidth) pa;
		DmaChannelId chan; } IRec#(type addrWidth) deriving(Bits);

module mkAxiDmaReadInternal#(Vector#(numReadClients, DmaReadClient#(dsz)) readClients,
			     DmaIndication dmaIndication,
			     Server#(Tuple2#(SGListId,Bit#(DmaOffsetSize)),Bit#(addrWidth)) sgl)
   (AxiDmaReadInternal#(addrWidth, dsz))

   provisos(Add#(1,a__,dsz),
	    Add#(b__, addrWidth, 64),
	    Add#(c__, 12, addrWidth),
	    Add#(1, c__, d__));

   FIFO#(IRec#(addrWidth)) lreqFifo <- mkSizedFIFO(1);
   FIFO#(IRec#(addrWidth))  reqFifo <- mkSizedFIFO(1);
   FIFO#(IRec#(addrWidth)) dreqFifo <- mkSizedFIFO(32);

   Reg#(Bit#(8))           burstReg <- mkReg(0);
   Vector#(numReadClients, Reg#(Bit#(64))) beatCounts <- replicateM(mkReg(0));

   // the choice of 5 is based on PCIE limitations.
   // uniqueness is enforced by the depth of dreqFIFO
   Reg#(Bit#(5))  id_gen   <- mkReg(0);
   // report a tag mismatch for oo completions (in which
   // case we will need to introduce completion buffers)
   FIFO#(Tuple2#(Bit#(6),Bit#(6))) tag_mismatch <- mkSizedFIFO(32);

   for (Integer selectReg = 0; selectReg < valueOf(numReadClients); selectReg = selectReg + 1)
       rule loadChannel;
	  DmaRequest req <- readClients[selectReg].readReq.get();
	  //$display("dmaread.loadChannel activeChan=%d handle=%h addr=%h burst=%h", selectReg, req.pointer, req.offset, req.burstLen);
	  if (bad_pointer(req.pointer))
	     dmaIndication.badPointer(req.pointer);
	  else begin
	     lreqFifo.enq(IRec{req:req, id:?, pa:?, chan:fromInteger(selectReg)});
	     sgl.request.put(tuple2(truncate(req.pointer),req.offset));
	  end
       endrule

   rule checkSglResp;
      let physAddr <- sgl.response.get;
      let req = lreqFifo.first.req;
      let chan = lreqFifo.first.chan;
      lreqFifo.deq();
      if (physAddr <= (1 << valueOf(SGListPageShift0))) begin
	 // squash request
	 $display("dmaRead: badAddr pointer=%d offset=%h physAddr=%h", req.pointer, req.offset, physAddr);
	 dmaIndication.badAddr(req.pointer, extend(req.offset), extend(physAddr));
      end
      else begin
	 if (False && physAddr[31:24] != 0)
	    $display("checkSglResp: funny physAddr req.pointer=%d req.offset=%h physAddr=%h", req.pointer, req.offset, physAddr);
	 //$display("mkAxiDmaReadInternal::checkSglResp tag=%d, id=%d, activeChan=%d", req.tag, id_gen, chan);
	 reqFifo.enq(IRec{req:req, id:{0,id_gen}, pa:physAddr, chan:chan});
	 id_gen <= id_gen+1;
      end
   endrule

   interface DmaDbg dbg;
      method ActionValue#(DmaDbgRec) dbg();
	 return DmaDbgRec{x:fromInteger(valueOf(numReadClients)), y:?, z:?, w:?};
      endmethod
      method ActionValue#(Bit#(64)) getMemoryTraffic(Bit#(32) client);
	 return (valueOf(numReadClients) > 0 && client < fromInteger(valueOf(numReadClients))) ? beatCounts[client] : 0;
      endmethod
   endinterface

   interface Axi3Master m_axi;
      interface Get req_ar;
	 method ActionValue#(Axi3ReadRequest#(addrWidth,6)) get();
	    let req = reqFifo.first.req;
	    let physAddr = reqFifo.first.pa;
	    let id = reqFifo.first.id;
	    reqFifo.deq;
	    //$display("mkAxiDmaReadInternal::req_ar tag=%d id=%d len=%d activeChan=%d", req.tag, id, req.burstLen, reqFifo.first.chan);
	    if (False && physAddr[31:24] != 0)
	       $display("req_ar: funny physAddr req.pointer=%d req.offset=%h physAddr=%h", req.pointer, req.offset, physAddr);

	    dreqFifo.enq(reqFifo.first);
	    return Axi3ReadRequest{address:physAddr, len:truncate(req.burstLen-1), id:id,
				   size: axiBusSize(valueOf(dsz)), burst: 1, prot: 0, cache: 3, lock:0, qos:0};
	 endmethod
      endinterface
      interface Put resp_read;
	 method Action put(Axi3ReadResponse#(dsz,6) response);
	    let activeChan = dreqFifo.first.chan;
	    let req = dreqFifo.first.req;
	    let id = dreqFifo.first.id;
	    if (valueOf(numReadClients) > 0)
	      readClients[activeChan].readData.put(DmaData { data: response.data, tag: req.tag});

	    let burstLen = burstReg;
	    if (burstLen == 0)
	       burstLen = req.burstLen;

	    if (burstLen == 1  && valueOf(numReadClients) > 0) begin
	       dreqFifo.deq();
	    end

	    if (response.id != id) begin
	       tag_mismatch.enq(tuple2(response.id,id));
	       $display("mkAxiDmaReadInternal::tag_mismatch %d %d", response.id, id);
	    end
	    //$display("mkAxiDmaReadInternal::resp_read id=%d burstLen=%d activeChan=%d", id, burstLen, activeChan);
	    burstReg <= burstLen-1;
	    if(valueOf(numReadClients) > 0)
	       beatCounts[activeChan] <= beatCounts[activeChan]+1;
	 endmethod
      endinterface
      interface Get req_aw = ?;
      interface Get resp_write = ?;
      interface Put resp_b = ?;
   endinterface
   interface Get tagMismatch = fifoToGet(tag_mismatch);
endmodule

module mkAxiDmaWriteInternal#(Vector#(numWriteClients, DmaWriteClient#(dsz)) writeClients,
			      DmaIndication dmaIndication,
			      Server#(Tuple2#(SGListId,Bit#(DmaOffsetSize)),Bit#(addrWidth)) sgl)

   (AxiDmaWriteInternal#(addrWidth, dsz))

   provisos(Add#(1,a__,dsz),
	    Add#(b__, addrWidth, 64),
	    Add#(c__, 12, addrWidth),
	    Add#(1, c__, d__));

   FIFO#(IRec#(addrWidth)) lreqFifo <- mkSizedFIFO(1);
   FIFO#(IRec#(addrWidth))  reqFifo <- mkSizedFIFO(1);
   FIFO#(IRec#(addrWidth)) dreqFifo <- mkSizedFIFO(32);
   FIFO#(IRec#(addrWidth)) respFifo <- mkSizedFIFO(32);

   Reg#(Bit#(8))         burstReg <- mkReg(0);
   Vector#(numWriteClients, Reg#(Bit#(64))) beatCounts <- replicateM(mkReg(0));

   // the choice of 5 is based on PCIE limitations.
   // uniqueness is enforced by the depth of dreqFIFO
   Reg#(Bit#(5))  id_gen   <- mkReg(0);
   // report a tag mismatch for oo completions (in which
   // case we will need to introduce completion buffers)
   FIFO#(Tuple2#(Bit#(6),Bit#(6))) tag_mismatch <- mkSizedFIFO(32);

   for (Integer selectReg = 0; selectReg < valueOf(numWriteClients); selectReg = selectReg + 1)
       rule loadChannel;
	  DmaRequest req <- writeClients[selectReg].writeReq.get();
	  //$display("dmawrite.loadChannel activeChan=%d handle=%h addr=%h burst=%h debugReq=%d", selectReg, req.pointer, req.offset, req.burstLen, debugReg);
	  if (bad_pointer(req.pointer))
	     dmaIndication.badPointer(req.pointer);
	  else begin
	     lreqFifo.enq(IRec{req:req, id:?, pa:?, chan:fromInteger(selectReg)});
	     sgl.request.put(tuple2(truncate(req.pointer),req.offset));
	  end
       endrule

   rule checkSglResp;
      let physAddr <- sgl.response.get;
      let req = lreqFifo.first.req;
      let chan = lreqFifo.first.chan;
      lreqFifo.deq();
      if (physAddr <= (1 << valueOf(SGListPageShift0))) begin
	 // squash request
	 $display("dmaWrite: badAddr handle=%d addr=%h physAddr=%h", req.pointer, req.offset, physAddr);
	 dmaIndication.badAddr(req.pointer, extend(req.offset), extend(physAddr));
      end
      else begin
	 reqFifo.enq(IRec{req:req, id:{0,id_gen}, pa:physAddr, chan:chan});
	 id_gen <= id_gen+1;
      end
   endrule

   interface DmaDbg dbg;
      method ActionValue#(DmaDbgRec) dbg();
	 return DmaDbgRec{x:fromInteger(valueOf(numWriteClients)), y:?, z:?, w:?};
      endmethod
      method ActionValue#(Bit#(64)) getMemoryTraffic(Bit#(32) client);
	 return (valueOf(numWriteClients) > 0 && client < fromInteger(valueOf(numWriteClients))) ? beatCounts[client] : 0;
      endmethod
   endinterface

   interface Axi3Master m_axi;
   interface Get req_ar = ?;
   interface Put resp_read = ?;
   interface Get req_aw;
      method ActionValue#(Axi3WriteRequest#(addrWidth,6)) get();
	 let req = reqFifo.first.req;
	 let physAddr = reqFifo.first.pa;
	 let id = reqFifo.first.id;
	 reqFifo.deq;
	 //$display("dmaWrite addr physAddr=%h burstReg=%d", physAddr, req.burstLen);

	 dreqFifo.enq(reqFifo.first);
	 return Axi3WriteRequest{address:physAddr, len:truncate(req.burstLen-1), id:id,
				 size: axiBusSize(valueOf(dsz)), burst: 1, prot: 0, cache: 3, lock:0, qos:0};
      endmethod
   endinterface
   interface Get resp_write;
	 method ActionValue#(Axi3WriteData#(dsz,6)) get();
	    let activeChan = dreqFifo.first.chan;
	    let req = dreqFifo.first.req;
	    let id = dreqFifo.first.id;
	    DmaData#(dsz) tagdata = unpack(0);
	    if (valueOf(numWriteClients) > 0)
	       tagdata <- writeClients[activeChan].writeData.get();
	    let burstLen = burstReg;
	    if (burstLen == 0)
	       burstLen = req.burstLen;

	    if (burstLen == 1) begin
	       dreqFifo.deq();
	       respFifo.enq(dreqFifo.first);
	    end

	    //$display("dmaWrite data data=%h burstLen=%d", tagdata.data, burstLen);
	    burstReg <= burstLen-1;
	    if(valueOf(numWriteClients) > 0)
	       beatCounts[activeChan] <= beatCounts[activeChan]+1;

	    Bit#(1) last = burstLen == 1 ? 1'b1 : 1'b0;
	    return Axi3WriteData { data: tagdata.data, byteEnable: maxBound, last: last, id: id };
	 endmethod
      endinterface
      interface Put resp_b;
	 method Action put(Axi3WriteResponse#(6) resp);
	    let activeChan = respFifo.first.chan;
	    let id = respFifo.first.id;
	    let tag = respFifo.first.req.tag;
	    if (resp.id != id)
	       tag_mismatch.enq(tuple2(resp.id,id));
	    respFifo.deq();
	    if (valueOf(numWriteClients) > 0) begin
	       writeClients[activeChan].writeDone.put(tag);
	    end
	 endmethod
      endinterface
   endinterface
   interface Get tagMismatch = fifoToGet(tag_mismatch);
endmodule

//
// @brief Creates a Dma controller for Dma read and write clients
//
// @param dmaIndication Interface for notifying software
// @param readClients The read clients.
// @param writeClients The writeclients.
//
module mkAxiDmaServer#(DmaIndication dmaIndication,
		       Vector#(numReadClients, DmaReadClient#(dsz)) readClients,
		       Vector#(numWriteClients, DmaWriteClient#(dsz)) writeClients)

   (AxiDmaServer#(addrWidth, dsz))

   provisos (Add#(1,a__,dsz),
        Add#(b__, TSub#(addrWidth, 12), 32),
        Add#(c__, 12, addrWidth),
        Add#(d__, addrWidth, 64),
        Add#(e__, TSub#(addrWidth, 12), DmaOffsetSize),
        Add#(f__, c__, DmaOffsetSize),
	Add#(g__, addrWidth, 40));

   SGListMMU#(addrWidth) sgl <- mkSGListMMU(dmaIndication);
   FIFO#(void)   addrReqFifo <- mkFIFO;

   AxiDmaReadInternal#(addrWidth, dsz) reader <- mkAxiDmaReadInternal(readClients, dmaIndication, sgl.addr[0]);
   AxiDmaWriteInternal#(addrWidth, dsz) writer <- mkAxiDmaWriteInternal(writeClients, dmaIndication, sgl.addr[1]);

   rule tag_mismatch_read;
      let rv <- reader.tagMismatch.get;
      dmaIndication.tagMismatch(Read, extend(tpl_1(rv)), extend(tpl_2(rv)));
   endrule

   rule tag_mismatch_write;
      let rv <- writer.tagMismatch.get;
      dmaIndication.tagMismatch(Write, extend(tpl_1(rv)), extend(tpl_2(rv)));
   endrule

   rule sglistEntry;
      addrReqFifo.deq;
      let physAddr <- sgl.addr[0].response.get;
      dmaIndication.addrResponse(zeroExtend(physAddr));
   endrule

   interface DmaConfig request;
      method Action getStateDbg(ChannelType rc);
	 let rv = ?;
	 if (rc == Read)
	    rv <- reader.dbg.dbg;
	 else
	    rv <- writer.dbg.dbg;
	 dmaIndication.reportStateDbg(rv);
      endmethod
      method Action getMemoryTraffic(ChannelType rc, Bit#(32) client);
	 if (rc == Read) begin
	    let rv <- reader.dbg.getMemoryTraffic(client);
	    dmaIndication.reportMemoryTraffic(rv);
	 end
	 else begin
	    let rv <- writer.dbg.getMemoryTraffic(client);
	    dmaIndication.reportMemoryTraffic(rv);
	 end
      endmethod
      method Action sglist(Bit#(32) pref, Bit#(DmaOffsetSize) addr, Bit#(32) len);
	 if (bad_pointer(pref))
	    dmaIndication.badPointer(pref);
`ifdef BSIM
	 let va <- pareff(pref, len);
         addr[39:32] = truncate(pref);
`endif
	 sgl.sglist(pref, addr, len);
      endmethod
      method Action region(Bit#(32) pointer, Bit#(40) barr8, Bit#(8) off8, Bit#(40) barr4, Bit#(8) off4, Bit#(40) barr0, Bit#(8) off0);
	 sgl.region(pointer,barr8,off8,barr4,off4,barr0,off0);
      endmethod
      method Action addrRequest(Bit#(32) pointer, Bit#(32) offset);
	 addrReqFifo.enq(?);
	 sgl.addr[0].request.put(tuple2(truncate(pointer), extend(offset)));
      endmethod
   endinterface
   interface Axi3Master m_axi;
      interface Get req_ar = reader.m_axi.req_ar;
      interface Put resp_read = reader.m_axi.resp_read;

      interface Get req_aw = writer.m_axi.req_aw;
      interface Get resp_write = writer.m_axi.resp_write;
      interface Put resp_b = writer.m_axi.resp_b;
   endinterface
endmodule

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

// BSV Libraries
import FIFO::*;
import Vector::*;
import GetPut::*;
import ClientServer::*;
import Assert::*;

// XBSV Libraries
import AxiMasterSlave::*;
import Dma::*;
import PortalMemory::*;
import Adapter::*;
import SGList::*;

`ifdef BSIM
import "BDPI" function ActionValue#(Bit#(32)) pareff(Bit#(32) handle, Bit#(32) size);
`endif

//
// @brief AxiDma provides the configuration and AXI bus interface for Dma
//
// @param dsz Number of bits in the data bus
//

interface AxiDmaServer#(numeric type addrWidth, numeric type dsz);
   interface DmaConfig request;
   interface Axi3Master#(addrWidth,dsz,6) m_axi;
endinterface

interface AxiDmaWriteInternal#(numeric type addrWidth, numeric type dsz);
   interface DmaDbg dbg;
   interface Axi3Master#(addrWidth,dsz,6) m_axi;
   interface Get#(Tuple2#(Bit#(6),Bit#(6))) tagMismatch;
endinterface

interface AxiDmaReadInternal#(numeric type addrWidth, numeric type dsz);
   interface DmaDbg dbg;
   interface Axi3Master#(addrWidth,dsz,6) m_axi;
   interface Get#(Tuple2#(Bit#(6),Bit#(6))) tagMismatch;
endinterface

function Bool bad_pointer(DmaPointer p);
   return (p > fromInteger(valueOf(MaxNumSGLists)) || p == 0);
endfunction

typedef enum {Idle, Translate, Address, Data, Done} InternalState deriving(Eq,Bits);

typedef struct {DmaRequest req;
		Bit#(6)    id;
		Bit#(addrWidth) pa;
		DmaChannelId chan; } IRec#(type addrWidth) deriving(Bits);

module mkAxiDmaReadInternal#(Vector#(numReadClients, DmaReadClient#(dsz)) readClients,
			     DmaIndication dmaIndication,
			     Server#(Tuple2#(SGListId,Bit#(DmaOffsetSize)),Bit#(addrWidth)) sgl)
   (AxiDmaReadInternal#(addrWidth, dsz))

   provisos(Add#(1,a__,dsz),
	    Add#(b__, addrWidth, 64),
	    Add#(c__, 12, addrWidth),
	    Add#(1, c__, d__));

   FIFO#(IRec#(addrWidth)) lreqFifo <- mkSizedFIFO(1);
   FIFO#(IRec#(addrWidth))  reqFifo <- mkSizedFIFO(1);
   FIFO#(IRec#(addrWidth)) dreqFifo <- mkSizedFIFO(32);

   Reg#(Bit#(8))           burstReg <- mkReg(0);
   Vector#(numReadClients, Reg#(Bit#(64))) beatCounts <- replicateM(mkReg(0));

   // the choice of 5 is based on PCIE limitations.
   // uniqueness is enforced by the depth of dreqFIFO
   Reg#(Bit#(5))  id_gen   <- mkReg(0);
   // report a tag mismatch for oo completions (in which
   // case we will need to introduce completion buffers)
   FIFO#(Tuple2#(Bit#(6),Bit#(6))) tag_mismatch <- mkSizedFIFO(32);

   for (Integer selectReg = 0; selectReg < valueOf(numReadClients); selectReg = selectReg + 1)
       rule loadChannel;
	  DmaRequest req <- readClients[selectReg].readReq.get();
	  //$display("dmaread.loadChannel activeChan=%d handle=%h addr=%h burst=%h", selectReg, req.pointer, req.offset, req.burstLen);
	  if (bad_pointer(req.pointer))
	     dmaIndication.badPointer(req.pointer);
	  else begin
	     lreqFifo.enq(IRec{req:req, id:?, pa:?, chan:fromInteger(selectReg)});
	     sgl.request.put(tuple2(truncate(req.pointer),req.offset));
	  end
       endrule

   rule checkSglResp;
      let physAddr <- sgl.response.get;
      let req = lreqFifo.first.req;
      let chan = lreqFifo.first.chan;
      lreqFifo.deq();
      if (physAddr <= (1 << valueOf(SGListPageShift0))) begin
	 // squash request
	 $display("dmaRead: badAddr pointer=%d offset=%h physAddr=%h", req.pointer, req.offset, physAddr);
	 dmaIndication.badAddr(req.pointer, extend(req.offset), extend(physAddr));
      end
      else begin
	 if (False && physAddr[31:24] != 0)
	    $display("checkSglResp: funny physAddr req.pointer=%d req.offset=%h physAddr=%h", req.pointer, req.offset, physAddr);
	 //$display("mkAxiDmaReadInternal::checkSglResp tag=%d, id=%d, activeChan=%d", req.tag, id_gen, chan);
	 reqFifo.enq(IRec{req:req, id:{0,id_gen}, pa:physAddr, chan:chan});
	 id_gen <= id_gen+1;
      end
   endrule

   interface DmaDbg dbg;
      method ActionValue#(DmaDbgRec) dbg();
	 return DmaDbgRec{x:fromInteger(valueOf(numReadClients)), y:?, z:?, w:?};
      endmethod
      method ActionValue#(Bit#(64)) getMemoryTraffic(Bit#(32) client);
	 return (valueOf(numReadClients) > 0 && client < fromInteger(valueOf(numReadClients))) ? beatCounts[client] : 0;
      endmethod
   endinterface

   interface Axi3Master m_axi;
      interface Get req_ar;
	 method ActionValue#(Axi3ReadRequest#(addrWidth,6)) get();
	    let req = reqFifo.first.req;
	    let physAddr = reqFifo.first.pa;
	    let id = reqFifo.first.id;
	    reqFifo.deq;
	    //$display("mkAxiDmaReadInternal::req_ar tag=%d id=%d len=%d activeChan=%d", req.tag, id, req.burstLen, reqFifo.first.chan);
	    if (False && physAddr[31:24] != 0)
	       $display("req_ar: funny physAddr req.pointer=%d req.offset=%h physAddr=%h", req.pointer, req.offset, physAddr);

	    dreqFifo.enq(reqFifo.first);
	    return Axi3ReadRequest{address:physAddr, len:truncate(req.burstLen-1), id:id,
				   size: axiBusSize(valueOf(dsz)), burst: 1, prot: 0, cache: 3, lock:0, qos:0};
	 endmethod
      endinterface
      interface Put resp_read;
	 method Action put(Axi3ReadResponse#(dsz,6) response);
	    let activeChan = dreqFifo.first.chan;
	    let req = dreqFifo.first.req;
	    let id = dreqFifo.first.id;
	    if (valueOf(numReadClients) > 0)
	      readClients[activeChan].readData.put(DmaData { data: response.data, tag: req.tag});

	    let burstLen = burstReg;
	    if (burstLen == 0)
	       burstLen = req.burstLen;

	    if (burstLen == 1  && valueOf(numReadClients) > 0) begin
	       dreqFifo.deq();
	    end

	    if (response.id != id) begin
	       tag_mismatch.enq(tuple2(response.id,id));
	       $display("mkAxiDmaReadInternal::tag_mismatch %d %d", response.id, id);
	    end
	    //$display("mkAxiDmaReadInternal::resp_read id=%d burstLen=%d activeChan=%d", id, burstLen, activeChan);
	    burstReg <= burstLen-1;
	    if(valueOf(numReadClients) > 0)
	       beatCounts[activeChan] <= beatCounts[activeChan]+1;
	 endmethod
      endinterface
      interface Get req_aw = ?;
      interface Get resp_write = ?;
      interface Put resp_b = ?;
   endinterface
   interface Get tagMismatch = fifoToGet(tag_mismatch);
endmodule

module mkAxiDmaWriteInternal#(Vector#(numWriteClients, DmaWriteClient#(dsz)) writeClients,
			      DmaIndication dmaIndication,
			      Server#(Tuple2#(SGListId,Bit#(DmaOffsetSize)),Bit#(addrWidth)) sgl)

   (AxiDmaWriteInternal#(addrWidth, dsz))

   provisos(Add#(1,a__,dsz),
	    Add#(b__, addrWidth, 64),
	    Add#(c__, 12, addrWidth),
	    Add#(1, c__, d__));

   FIFO#(IRec#(addrWidth)) lreqFifo <- mkSizedFIFO(1);
   FIFO#(IRec#(addrWidth))  reqFifo <- mkSizedFIFO(1);
   FIFO#(IRec#(addrWidth)) dreqFifo <- mkSizedFIFO(32);
   FIFO#(IRec#(addrWidth)) respFifo <- mkSizedFIFO(1);

   Reg#(Bit#(8))         burstReg <- mkReg(0);
   Vector#(numWriteClients, Reg#(Bit#(64))) beatCounts <- replicateM(mkReg(0));

   // the choice of 5 is based on PCIE limitations.
   // uniqueness is enforced by the depth of dreqFIFO
   Reg#(Bit#(5))  id_gen   <- mkReg(0);
   // report a tag mismatch for oo completions (in which
   // case we will need to introduce completion buffers)
   FIFO#(Tuple2#(Bit#(6),Bit#(6))) tag_mismatch <- mkSizedFIFO(32);

   for (Integer selectReg = 0; selectReg < valueOf(numWriteClients); selectReg = selectReg + 1)
       rule loadChannel;
	  DmaRequest req <- writeClients[selectReg].writeReq.get();
	  //$display("dmawrite.loadChannel activeChan=%d handle=%h addr=%h burst=%h debugReq=%d", selectReg, req.pointer, req.offset, req.burstLen, debugReg);
	  if (bad_pointer(req.pointer))
	     dmaIndication.badPointer(req.pointer);
	  else begin
	     lreqFifo.enq(IRec{req:req, id:?, pa:?, chan:fromInteger(selectReg)});
	     sgl.request.put(tuple2(truncate(req.pointer),req.offset));
	  end
       endrule

   rule checkSglResp;
      let physAddr <- sgl.response.get;
      let req = lreqFifo.first.req;
      let chan = lreqFifo.first.chan;
      lreqFifo.deq();
      if (physAddr <= (1 << valueOf(SGListPageShift0))) begin
	 // squash request
	 $display("dmaWrite: badAddr handle=%d addr=%h physAddr=%h", req.pointer, req.offset, physAddr);
	 dmaIndication.badAddr(req.pointer, extend(req.offset), extend(physAddr));
      end
      else begin
	 reqFifo.enq(IRec{req:req, id:{0,id_gen}, pa:physAddr, chan:chan});
	 id_gen <= id_gen+1;
      end
   endrule

   interface DmaDbg dbg;
      method ActionValue#(DmaDbgRec) dbg();
	 return DmaDbgRec{x:fromInteger(valueOf(numWriteClients)), y:?, z:?, w:?};
      endmethod
      method ActionValue#(Bit#(64)) getMemoryTraffic(Bit#(32) client);
	 return (valueOf(numWriteClients) > 0 && client < fromInteger(valueOf(numWriteClients))) ? beatCounts[client] : 0;
      endmethod
   endinterface

   interface Axi3Master m_axi;
   interface Get req_ar = ?;
   interface Put resp_read = ?;
   interface Get req_aw;
      method ActionValue#(Axi3WriteRequest#(addrWidth,6)) get();
	 let req = reqFifo.first.req;
	 let physAddr = reqFifo.first.pa;
	 let id = reqFifo.first.id;
	 reqFifo.deq;
	 //$display("dmaWrite addr physAddr=%h burstReg=%d", physAddr, req.burstLen);

	 dreqFifo.enq(reqFifo.first);
	 return Axi3WriteRequest{address:physAddr, len:truncate(req.burstLen-1), id:id,
				 size: axiBusSize(valueOf(dsz)), burst: 1, prot: 0, cache: 3, lock:0, qos:0};
      endmethod
   endinterface
   interface Get resp_write;
	 method ActionValue#(Axi3WriteData#(dsz,6)) get();
	    let activeChan = dreqFifo.first.chan;
	    let req = dreqFifo.first.req;
	    let id = dreqFifo.first.id;
	    DmaData#(dsz) tagdata = unpack(0);
	    if (valueOf(numWriteClients) > 0)
	       tagdata <- writeClients[activeChan].writeData.get();
	    let burstLen = burstReg;
	    if (burstLen == 0)
	       burstLen = req.burstLen;

	    if (burstLen == 1) begin
	       dreqFifo.deq();
	       respFifo.enq(dreqFifo.first);
	    end

	    //$display("dmaWrite data data=%h burstLen=%d", tagdata.data, burstLen);
	    burstReg <= burstLen-1;
	    if(valueOf(numWriteClients) > 0)
	       beatCounts[activeChan] <= beatCounts[activeChan]+1;

	    Bit#(1) last = burstLen == 1 ? 1'b1 : 1'b0;
	    return Axi3WriteData { data: tagdata.data, byteEnable: maxBound, last: last, id: id };
	 endmethod
      endinterface
      interface Put resp_b;
	 method Action put(Axi3WriteResponse#(6) resp);
	    let activeChan = respFifo.first.chan;
	    let id = respFifo.first.id;
	    let tag = respFifo.first.req.tag;
	    if (resp.id != id)
	       tag_mismatch.enq(tuple2(resp.id,id));
	    respFifo.deq();
	    if (valueOf(numWriteClients) > 0) begin
	       writeClients[activeChan].writeDone.put(tag);
	    end
	 endmethod
      endinterface
   endinterface
   interface Get tagMismatch = fifoToGet(tag_mismatch);
endmodule

//
// @brief Creates a Dma controller for Dma read and write clients
//
// @param dmaIndication Interface for notifying software
// @param readClients The read clients.
// @param writeClients The writeclients.
//
module mkAxiDmaServer#(DmaIndication dmaIndication,
		       Vector#(numReadClients, DmaReadClient#(dsz)) readClients,
		       Vector#(numWriteClients, DmaWriteClient#(dsz)) writeClients)

   (AxiDmaServer#(addrWidth, dsz))

   provisos (Add#(1,a__,dsz),
        Add#(b__, TSub#(addrWidth, 12), 32),
        Add#(c__, 12, addrWidth),
        Add#(d__, addrWidth, 64),
        Add#(e__, TSub#(addrWidth, 12), DmaOffsetSize),
        Add#(f__, c__, DmaOffsetSize),
	Add#(g__, addrWidth, 40));

   SGListMMU#(addrWidth) sgl <- mkSGListMMU(dmaIndication);
   FIFO#(void)   addrReqFifo <- mkFIFO;

   AxiDmaReadInternal#(addrWidth, dsz) reader <- mkAxiDmaReadInternal(readClients, dmaIndication, sgl.addr[0]);
   AxiDmaWriteInternal#(addrWidth, dsz) writer <- mkAxiDmaWriteInternal(writeClients, dmaIndication, sgl.addr[1]);

   rule tag_mismatch_read;
      let rv <- reader.tagMismatch.get;
      dmaIndication.tagMismatch(Read, extend(tpl_1(rv)), extend(tpl_2(rv)));
   endrule

   rule tag_mismatch_write;
      let rv <- writer.tagMismatch.get;
      dmaIndication.tagMismatch(Write, extend(tpl_1(rv)), extend(tpl_2(rv)));
   endrule

   rule sglistEntry;
      addrReqFifo.deq;
      let physAddr <- sgl.addr[0].response.get;
      dmaIndication.addrResponse(zeroExtend(physAddr));
   endrule

   interface DmaConfig request;
      method Action getStateDbg(ChannelType rc);
	 let rv = ?;
	 if (rc == Read)
	    rv <- reader.dbg.dbg;
	 else
	    rv <- writer.dbg.dbg;
	 dmaIndication.reportStateDbg(rv);
      endmethod
      method Action getMemoryTraffic(ChannelType rc, Bit#(32) client);
	 if (rc == Read) begin
	    let rv <- reader.dbg.getMemoryTraffic(client);
	    dmaIndication.reportMemoryTraffic(rv);
	 end
	 else begin
	    let rv <- writer.dbg.getMemoryTraffic(client);
	    dmaIndication.reportMemoryTraffic(rv);
	 end
      endmethod
      method Action sglist(Bit#(32) pref, Bit#(DmaOffsetSize) addr, Bit#(32) len);
	 if (bad_pointer(pref))
	    dmaIndication.badPointer(pref);
`ifdef BSIM
	 let va <- pareff(pref, len);
         addr[39:32] = truncate(pref);
`endif
	 sgl.sglist(pref, addr, len);
      endmethod
      method Action region(Bit#(32) pointer, Bit#(40) barr8, Bit#(8) off8, Bit#(40) barr4, Bit#(8) off4, Bit#(40) barr0, Bit#(8) off0);
	 sgl.region(pointer,barr8,off8,barr4,off4,barr0,off0);
      endmethod
      method Action addrRequest(Bit#(32) pointer, Bit#(32) offset);
	 addrReqFifo.enq(?);
	 sgl.addr[0].request.put(tuple2(truncate(pointer), extend(offset)));
      endmethod
   endinterface
   interface Axi3Master m_axi;
      interface Get req_ar = reader.m_axi.req_ar;
      interface Put resp_read = reader.m_axi.resp_read;

      interface Get req_aw = writer.m_axi.req_aw;
      interface Get resp_write = writer.m_axi.resp_write;
      interface Put resp_b = writer.m_axi.resp_b;
   endinterface
endmodule