simplified writeHeaderTlp and writeTlps

simplified writeHeaderTlp and writeTlps

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import FIFOF        :: *;
import GetPut       :: *;
import Connectable  :: *;
import PCIE         :: *;
import DefaultValue :: *;
import MIMO         :: *;
import MIFO         :: *;
import Vector       :: *;
import ClientServer :: *;
import MemTypes     :: *;

import AxiMasterSlave :: *;

interface MemSlaveEngine#(numeric type buswidth);
    interface Client#(TLPData#(16), TLPData#(16)) tlp;
    interface MemSlave#(40,buswidth) slave;
    method Bool tlpOutFifoNotEmpty();
    interface Reg#(Bool) use4dw;
endinterface: MemSlaveEngine

module mkMemSlaveEngine#(PciId my_id)(MemSlaveEngine#(buswidth))
   provisos (Div#(buswidth, 8, busWidthBytes),
	     Div#(buswidth, 32, busWidthWords),
	     Bits#(Vector#(busWidthWords, Bit#(32)), buswidth),
            Log#(busWidthBytes,beatShift),
	     Add#(aaa, 32, buswidth),
	     Add#(bbb, buswidth, 256),
	     Add#(ccc, TMul#(8, busWidthWords), 64),
	     Add#(ddd, TMul#(32, busWidthWords), 256),
	     Add#(eee, busWidthWords, 8),
	     Add#(1, a__, busWidthWords),
	     Add#(busWidthWords, b__, 4)
      );

    let beat_shift = fromInteger(valueOf(beatShift));

    FIFOF#(TLPData#(16)) tlpOutFifo <- mkFIFOF;
    FIFOF#(TLPData#(16)) tlpInFifo <- mkFIFOF;
    FIFOF#(TLPData#(16)) tlpWriteHeaderFifo <- mkFIFOF;

    Reg#(Bit#(7)) hitReg <- mkReg(0);
    Reg#(Bool) use4dwReg <- mkReg(True);

    // default configuration for MIMO is for guarded enq() and deq().
    // However, the implicit guard only checks for space for 1 element for enq(), and availability of 1 element for deq().
    MIMOConfiguration mimoCfg = defaultValue;
   MIFO#(4,busWidthWords,8,Bit#(32)) completionMimo <- mkMIFO();
   MIFO#(4,busWidthWords,8,TLPTag) completionTagMimo <- mkMIFO();

    MIMO#(busWidthWords,4,8,Bit#(32)) writeDataMimo <- mkMIMO(mimoCfg);
    Reg#(Bit#(9)) writeBurstCount <- mkReg(0);
    Reg#(TLPLength)  writeDwCount <- mkReg(0); // how many 4 byte (double) words to send
    Reg#(LUInt#(4))    tlpDwCount <- mkReg(0); // how many to send in the next tlp (at most 4)
    Reg#(Bool)            lastTlp <- mkReg(False); // if the next tlp sent is the last one
    Reg#(Bool)    writeInProgress <- mkReg(False);
    FIFOF#(TLPTag) writeTag <- mkSizedFIFOF(16);
    FIFOF#(TLPTag) doneTag <- mkSizedFIFOF(16);

    function LUInt#(4) tlpWordCount(TLPData#(16) tlp);
       if (tlp.be == 16'h0000)
	  return 0;
       else if (tlp.be == 16'h000f || tlp.be == 16'hf000)
	  return 1;
       else if (tlp.be == 16'h00ff || tlp.be == 16'hff00)
	  return 2;
       else if (tlp.be == 16'h0fff || tlp.be == 16'hfff0)
	  return 3;
       else if (tlp.be == 16'hffff)
	  return 4;
       else
	  return 0;
    endfunction

   rule writeHeaderTlp if (!writeInProgress && writeDataMimo.deqReadyN(1));
      let tlp = tlpWriteHeaderFifo.first;

      TLPMemory4DWHeader hdr_4dw = unpack(tlp.data);
      TLPLength dwCount = hdr_4dw.length;

      TLPMemoryIO3DWHeader hdr_3dw = unpack(tlp.data);
      if (hdr_3dw.format == MEM_WRITE_3DW_DATA) begin
	 dwCount = hdr_3dw.length;
	 Vector#(4, Bit#(32)) v = writeDataMimo.first();
	 writeDataMimo.deq(1);
	 hdr_3dw.data = byteSwap(v[0]);
	 tlp.be = 16'hffff;
	 if (dwCount == 1)
	    tlp.eof = True;
	 dwCount = dwCount - 1;
	 tlp.data = pack(hdr_3dw);
      end
      else if (hdr_4dw.format == MEM_WRITE_4DW_DATA && writeDataMimo.deqReadyN(1)) begin
	 tlp.be = 16'hffff;
      end

      tlpWriteHeaderFifo.deq();
      tlpOutFifo.enq(tlp);
      $display("writeHeaderTlp dwCount=%d", dwCount);
      writeDwCount <= dwCount;
      tlpDwCount <= min(4,truncate(unpack(dwCount)));
      lastTlp <= (dwCount <= 4);
      writeInProgress <= (dwCount != 0);
      if (dwCount == 0) begin
	 doneTag.enq(writeTag.first());
	 writeTag.deq();
      end

   endrule

   rule writeTlps if (writeInProgress && writeDataMimo.deqReadyN(tlpDwCount));
      TLPData#(16) tlp = defaultValue;
      tlp.sof = False;
      Vector#(4, Bit#(32)) v = unpack(0);

      // The MIMO implicit guard only checks for availability of 1 element
      // so we explicitly check for the number of elements required
      writeDataMimo.deq(tlpDwCount);
      v = writeDataMimo.first();
      let dwCount = writeDwCount - extend(pack(tlpDwCount));
      writeDwCount <= dwCount;
      tlpDwCount <= min(4,truncate(unpack(dwCount)));
      lastTlp <= (dwCount <= 4);
      if (tlpDwCount == 4)
	 tlp.be = 16'hffff;
      else if (tlpDwCount == 3)
	 tlp.be = 16'hfff0;
      else if (tlpDwCount == 2)
	 tlp.be = 16'hff00;
      else if (tlpDwCount == 1)
	 tlp.be = 16'hf000;
      tlp.eof = lastTlp;
      if (lastTlp) begin
	 writeInProgress <= False;
	 doneTag.enq(writeTag.first());
	 writeTag.deq();
	 $display("writeDwCount=%d will be zero", writeDwCount);
      end

      for (Integer i = 0; i < 4; i = i + 1)
	 tlp.data[(i+1)*32-1:i*32] = byteSwap(v[3-i]);
      tlpOutFifo.enq(tlp);
   endrule

   Reg#(TLPTag) lastTag <- mkReg(0);
   FIFOF#(TLPData#(16)) tlpDecodeFifo <- mkFIFOF();
   FIFOF#(LUInt#(4))    tlpWordCountFifo <- mkFIFOF();
   rule tlpInRule;
      let tlp <- toGet(tlpInFifo).get();
      tlpWordCountFifo.enq(tlpWordCount(tlp));
      tlpDecodeFifo.enq(tlp);
   endrule

   rule handleTlpRule;
      let tlp = tlpDecodeFifo.first;
      let count = tlpWordCountFifo.first;
      Bool handled = False;
      TLPMemoryIO3DWHeader h = unpack(tlp.data);
      hitReg <= tlp.hit;
      TLPMemoryIO3DWHeader hdr_3dw = unpack(tlp.data);
      TLPCompletionHeader hdr_completion = unpack(tlp.data);
      Vector#(4, Bit#(32)) vec = unpack(0);
      Vector#(4, Bit#(32)) tlpvec = unpack(tlp.data);

      if (!tlp.sof) begin
	 vec = reverse(tlpvec);
	 // The MIMO implicit guard only checks for space to enqueue 1 element
	 // so we explicitly check for the number of elements required
	 // otherwise elements in the queue will be overwritten.
	 if (completionMimo.enqReady()
	    && completionTagMimo.enqReady())
	    begin
	       completionMimo.enq(count, vec);
	       Vector#(4, TLPTag) tagvec = replicate(lastTag);
	       completionTagMimo.enq(count, tagvec);
	       handled = True;
	    end
      end
      else if (hdr_3dw.format == MEM_WRITE_3DW_DATA
	       && hdr_3dw.pkttype == COMPLETION
	       && completionMimo.enqReady()
	       && completionTagMimo.enqReady()) begin
	    vec[0] = hdr_3dw.data;
	    completionMimo.enq(1, vec);
            TLPTag tag = hdr_completion.tag;
	    lastTag <= tag;
	    completionTagMimo.enq(1, replicate(tag));
	    handled = True;
      end
      //$display("tlpIn handled=%d tlp=%h\n", handled, tlp);
      if (handled) begin
	 tlpDecodeFifo.deq();
	 tlpWordCountFifo.deq();
      end
   endrule

    interface Client        tlp;
        interface request = toGet(tlpOutFifo);
        interface response = toPut(tlpInFifo);
    endinterface
    interface MemSlave slave;
   interface MemWriteServer write_server;
      interface Put writeReq;
         method Action put(MemRequest#(40) req)
	      if (writeBurstCount == 0);

	      let burstLen = req.burstLen >> beat_shift;
	      let addr = req.addr;
	      let awid = req.tag;

	      TLPLength tlplen = fromInteger(valueOf(busWidthWords))*(extend(burstLen));
	      TLPData#(16) tlp = defaultValue;
	      tlp.sof = True;
	      tlp.eof = False;
	      tlp.hit = 7'h00;
	      tlp.be = 16'hffff;

	      $display("slave.writeAddr tlplen=%d burstLen=%d", tlplen, burstLen);
	      if ((addr >> 32) != 0) begin
		 TLPMemory4DWHeader hdr_4dw = defaultValue;
		 hdr_4dw.format = MEM_WRITE_4DW_DATA;
		 hdr_4dw.tag = extend(awid);
		 hdr_4dw.reqid = my_id;
		 hdr_4dw.nosnoop = SNOOPING_REQD;
		 hdr_4dw.addr = addr[40-1:2];
		 hdr_4dw.length = tlplen;
		 hdr_4dw.firstbe = 4'hf;
		 hdr_4dw.lastbe = (tlplen > 1) ? 4'hf : 0;
		 tlp.data = pack(hdr_4dw);
	      end
	      else begin
		 TLPMemoryIO3DWHeader hdr_3dw = defaultValue;
		 hdr_3dw.format = MEM_WRITE_3DW_DATA;
		 hdr_3dw.tag = extend(awid);
		 hdr_3dw.reqid = my_id;
		 hdr_3dw.nosnoop = SNOOPING_REQD;
		 hdr_3dw.addr = addr[32-1:2];
		 hdr_3dw.length = tlplen;
		 hdr_3dw.firstbe = 4'hf;
		 hdr_3dw.lastbe = (tlplen > 1) ? 4'hf : 0;

		 tlp.be = 16'hfff0; // no data word in this TLP

		 tlp.data = pack(hdr_3dw);
	      end
	      tlpWriteHeaderFifo.enq(tlp);
	      writeBurstCount <= zeroExtend(burstLen);
	      writeTag.enq(extend(awid));
           endmethod
	endinterface
      interface Put writeData;
         method Action put(MemData#(buswidth) wdata)
	      provisos (Bits#(Vector#(busWidthWords, Bit#(32)), busWidth)) if (writeBurstCount > 0 && writeDataMimo.enqReadyN(fromInteger(valueOf(busWidthWords))));

	      writeBurstCount <= writeBurstCount - 1;
	      Vector#(busWidthWords, Bit#(32)) v = unpack(wdata.data);
	      writeDataMimo.enq(fromInteger(valueOf(busWidthWords)), v);
           endmethod
       endinterface
      interface Get writeDone;
         method ActionValue#(Bit#(ObjectTagSize)) get();
	      let tag = doneTag.first();
	      doneTag.deq();
	      return truncate(tag);
           endmethod
	endinterface
   endinterface
   interface MemReadServer read_server;
      interface Put readReq;
         method Action put(MemRequest#(40) req) if (!writeInProgress && !writeDataMimo.deqReady());
	      let burstLen = req.burstLen >> beat_shift;
	      let addr = req.addr;
	      let arid = req.tag;

	       TLPData#(16) tlp = defaultValue;
	       tlp.sof = True;
	       tlp.eof = True;
	       tlp.hit = 7'h00;
	       TLPLength tlplen = fromInteger(valueOf(busWidthWords))*(extend(burstLen));
	       if (addr[39:32] != 0) begin
		   TLPMemory4DWHeader hdr_4dw = defaultValue;
		   hdr_4dw.format = MEM_READ_4DW_NO_DATA;
		   hdr_4dw.tag = extend(arid);
		   hdr_4dw.reqid = my_id;
		   hdr_4dw.nosnoop = SNOOPING_REQD;
		   hdr_4dw.addr = addr[40-1:2];
		   hdr_4dw.length = tlplen;
		   hdr_4dw.firstbe = 4'hf;
		   hdr_4dw.lastbe = (tlplen > 1) ? 4'hf : 0;
		   tlp.data = pack(hdr_4dw);
		   tlp.be = 16'hffff;
	       end
	       else begin
		   TLPMemoryIO3DWHeader hdr_3dw = defaultValue;
		   hdr_3dw.format = MEM_READ_3DW_NO_DATA;
		   hdr_3dw.tag = extend(arid);
		   hdr_3dw.reqid = my_id;
		   hdr_3dw.nosnoop = SNOOPING_REQD;
		   hdr_3dw.addr = addr[32-1:2];
		   hdr_3dw.length = tlplen;
		   hdr_3dw.firstbe = 4'hf;
		   hdr_3dw.lastbe = (tlplen > 1) ? 4'hf : 0;
		   tlp.data = pack(hdr_3dw);
		   tlp.be = 16'hfff0;
	       end
	       tlpOutFifo.enq(tlp);
           endmethod
       endinterface
      interface Get     readData;
         method ActionValue#(MemData#(buswidth)) get() if (completionMimo.deqReady()
							   && completionTagMimo.deqReady());
	      let data_v = completionMimo.first;
	      let tag_v = completionTagMimo.first;
	      completionMimo.deq();
	      completionTagMimo.deq();
              Bit#(buswidth) v = 0;
	      for (Integer i = 0; i < valueOf(busWidthWords); i = i+1)
		 v[(i+1)*32-1:i*32] = byteSwap(data_v[i]);
	      return MemData { data: v, tag: truncate(tag_v[0])};
           endmethod
	endinterface
   endinterface
    endinterface: slave
   method Bool tlpOutFifoNotEmpty() = tlpOutFifo.notEmpty;
   interface Reg use4dw = use4dwReg;
endmodule: mkMemSlaveEngine

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import FIFOF        :: *;
import GetPut       :: *;
import Connectable  :: *;
import PCIE         :: *;
import DefaultValue :: *;
import MIMO         :: *;
import MIFO         :: *;
import Vector       :: *;
import ClientServer :: *;
import MemTypes     :: *;

import AxiMasterSlave :: *;

interface MemSlaveEngine#(numeric type buswidth);
    interface Client#(TLPData#(16), TLPData#(16)) tlp;
    interface MemSlave#(40,buswidth) slave;
    method Bool tlpOutFifoNotEmpty();
    interface Reg#(Bool) use4dw;
endinterface: MemSlaveEngine

module mkMemSlaveEngine#(PciId my_id)(MemSlaveEngine#(buswidth))
   provisos (Div#(buswidth, 8, busWidthBytes),
	     Div#(buswidth, 32, busWidthWords),
	     Bits#(Vector#(busWidthWords, Bit#(32)), buswidth),
            Log#(busWidthBytes,beatShift),
	     Add#(aaa, 32, buswidth),
	     Add#(bbb, buswidth, 256),
	     Add#(ccc, TMul#(8, busWidthWords), 64),
	     Add#(ddd, TMul#(32, busWidthWords), 256),
	     Add#(eee, busWidthWords, 8),
	     Add#(1, a__, busWidthWords),
	     Add#(busWidthWords, b__, 4)
      );

    let beat_shift = fromInteger(valueOf(beatShift));

    FIFOF#(TLPData#(16)) tlpOutFifo <- mkFIFOF;
    FIFOF#(TLPData#(16)) tlpInFifo <- mkFIFOF;
    FIFOF#(TLPData#(16)) tlpWriteHeaderFifo <- mkFIFOF;

    Reg#(Bit#(7)) hitReg <- mkReg(0);
    Reg#(Bool) use4dwReg <- mkReg(True);

    // default configuration for MIMO is for guarded enq() and deq().
    // However, the implicit guard only checks for space for 1 element for enq(), and availability of 1 element for deq().
    MIMOConfiguration mimoCfg = defaultValue;
   MIFO#(4,busWidthWords,8,Bit#(32)) completionMimo <- mkMIFO();
   MIFO#(4,busWidthWords,8,TLPTag) completionTagMimo <- mkMIFO();

    Vector#(4,FIFOF#(Bit#(32))) completionFifos <- replicateM(mkFIFOF);
    Vector#(4,FIFOF#(TLPTag)) completionTagFifos <- replicateM(mkFIFOF);

    MIMO#(busWidthWords,4,8,Bit#(32)) writeDataMimo <- mkMIMO(mimoCfg);
    Reg#(Bit#(9)) writeBurstCount <- mkReg(0);
    Reg#(TLPLength)  writeDwCount <- mkReg(0);
    Reg#(Bool)    writeInProgress <- mkReg(False);
    FIFOF#(TLPTag) writeTag <- mkSizedFIFOF(16);
    FIFOF#(TLPTag) doneTag <- mkSizedFIFOF(16);

    function LUInt#(4) tlpWordCount(TLPData#(16) tlp);
       if (tlp.be == 16'h0000)
	  return 0;
       else if (tlp.be == 16'h000f || tlp.be == 16'hf000)
	  return 1;
       else if (tlp.be == 16'h00ff || tlp.be == 16'hff00)
	  return 2;
       else if (tlp.be == 16'h0fff || tlp.be == 16'hfff0)
	  return 3;
       else if (tlp.be == 16'hffff)
	  return 4;
       else
	  return 0;
    endfunction

   rule writeHeaderTlp if (!writeInProgress);
      let tlp = tlpWriteHeaderFifo.first;

      TLPMemory4DWHeader hdr_4dw = unpack(tlp.data);
      TLPLength dwCount = hdr_4dw.length;

      TLPMemoryIO3DWHeader hdr_3dw = unpack(tlp.data);
      Bool sendit = False;
      if (hdr_3dw.format == MEM_WRITE_3DW_DATA && writeDataMimo.deqReadyN(1)) begin
	 dwCount = hdr_3dw.length;
	 Vector#(4, Bit#(32)) v = writeDataMimo.first();
	 writeDataMimo.deq(1);
	 hdr_3dw.data = byteSwap(v[0]);
	 tlp.be = 16'hffff;
	 if (dwCount == 1)
	    tlp.eof = True;
	 dwCount = dwCount - 1;
	 tlp.data = pack(hdr_3dw);
	 sendit = True;
      end
      else if (hdr_3dw.format == MEM_WRITE_3DW_DATA) begin
	 // retry until the data is available in writeDataMimo
      end
      else if (hdr_4dw.format == MEM_WRITE_4DW_DATA && writeDataMimo.deqReadyN(1)) begin
	 tlp.be = 16'hffff;
	 sendit = True;
      end
      else if (hdr_4dw.format == MEM_WRITE_4DW_DATA) begin
	 sendit = False;
      end
      else begin
	 sendit = True;
      end
      if (sendit) begin
	 tlpWriteHeaderFifo.deq();
	 tlpOutFifo.enq(tlp);
	 $display("writeHeaderTlp dwCount=%d", dwCount);
	 writeDwCount <= dwCount;
	 writeInProgress <= (dwCount != 0);
	 if (dwCount == 0) begin
	    doneTag.enq(writeTag.first());
	    writeTag.deq();
	 end
      end
   endrule

   rule writeTlps if (writeInProgress);
      TLPData#(16) tlp = defaultValue;
      tlp.sof = False;
      Vector#(4, Bit#(32)) v = unpack(0);
      Bool sendit = False;
      // The MIMO implicit guard only checks for availability of 1 element
      // so we explicitly check for the number of elements required
      if (writeDwCount > 4 && writeDataMimo.deqReadyN(4)) begin
	 v = writeDataMimo.first();

	 writeDataMimo.deq(4);
	 writeDwCount <= writeDwCount - 4;
	 tlp.eof = False;
	 tlp.be = 16'hffff;
	 sendit = True;
      end
      else if (writeDwCount <= 4 && writeDataMimo.deqReadyN(unpack(truncate(writeDwCount)))) begin
	 v = writeDataMimo.first();
	 writeDataMimo.deq(unpack(truncate(writeDwCount)));
	 writeDwCount <= 0;
	 writeInProgress <= False;
	 doneTag.enq(writeTag.first());
	 writeTag.deq();
	 $display("writeDwCount=%d will be zero", writeDwCount);
	 tlp.eof = True;
	 if (writeDwCount == 4)
	    tlp.be = 16'hffff;
	 else if (writeDwCount == 3)
	    tlp.be = 16'hfff0;
	 else if (writeDwCount == 2)
	    tlp.be = 16'hff00;
	 else if (writeDwCount == 1)
	    tlp.be = 16'hf000;
	 sendit = True;
      end
      else begin
	 // wait for more data in writeDataMimo
	 $display("waiting for more data dwCount=%d count=%d writeBurstCount=%d enqReady=%d",
	    writeDwCount, writeDataMimo.count(), writeBurstCount, writeDataMimo.enqReadyN(fromInteger(valueOf(busWidthWords))));
      end
      if (sendit) begin
	 for (Integer i = 0; i < 4; i = i + 1)
	    tlp.data[(i+1)*32-1:i*32] = byteSwap(v[3-i]);
	 tlpOutFifo.enq(tlp);
      end
   endrule

   Reg#(TLPTag) lastTag <- mkReg(0);
   FIFOF#(TLPData#(16)) tlpDecodeFifo <- mkFIFOF();
   FIFOF#(LUInt#(4))    tlpWordCountFifo <- mkFIFOF();
   rule tlpInRule;
      let tlp <- toGet(tlpInFifo).get();
      tlpWordCountFifo.enq(tlpWordCount(tlp));
      tlpDecodeFifo.enq(tlp);
   endrule

   rule handleTlpRule;
      let tlp = tlpDecodeFifo.first;
      let count = tlpWordCountFifo.first;
      Bool handled = False;
      TLPMemoryIO3DWHeader h = unpack(tlp.data);
      hitReg <= tlp.hit;
      TLPMemoryIO3DWHeader hdr_3dw = unpack(tlp.data);
      TLPCompletionHeader hdr_completion = unpack(tlp.data);
      Vector#(4, Bit#(32)) vec = unpack(0);
      Vector#(4, Bit#(32)) tlpvec = unpack(tlp.data);

      if (!tlp.sof) begin
	 vec = reverse(tlpvec);
	 // The MIMO implicit guard only checks for space to enqueue 1 element
	 // so we explicitly check for the number of elements required
	 // otherwise elements in the queue will be overwritten.
	 if (completionMimo.enqReady()
	    && completionTagMimo.enqReady())
	    begin
	       completionMimo.enq(count, vec);
	       Vector#(4, TLPTag) tagvec = replicate(lastTag);
	       completionTagMimo.enq(count, tagvec);
	       handled = True;
	    end
      end
      else if (hdr_3dw.format == MEM_WRITE_3DW_DATA
	       && hdr_3dw.pkttype == COMPLETION
	       && completionMimo.enqReady()
	       && completionTagMimo.enqReady()) begin
	    vec[0] = hdr_3dw.data;
	    completionMimo.enq(1, vec);
            TLPTag tag = hdr_completion.tag;
	    lastTag <= tag;
	    completionTagMimo.enq(1, replicate(tag));
	    handled = True;
      end
      //$display("tlpIn handled=%d tlp=%h\n", handled, tlp);
      if (handled) begin
	 tlpDecodeFifo.deq();
	 tlpWordCountFifo.deq();
      end
   endrule

    interface Client        tlp;
        interface request = toGet(tlpOutFifo);
        interface response = toPut(tlpInFifo);
    endinterface
    interface MemSlave slave;
   interface MemWriteServer write_server;
      interface Put writeReq;
         method Action put(MemRequest#(40) req)
	      if (writeBurstCount == 0);

	      let burstLen = req.burstLen >> beat_shift;
	      let addr = req.addr;
	      let awid = req.tag;

	      TLPLength tlplen = fromInteger(valueOf(busWidthWords))*(extend(burstLen));
	      TLPData#(16) tlp = defaultValue;
	      tlp.sof = True;
	      tlp.eof = False;
	      tlp.hit = 7'h00;
	      tlp.be = 16'hffff;

	      $display("slave.writeAddr tlplen=%d burstLen=%d", tlplen, burstLen);
	      if ((addr >> 32) != 0) begin
		 TLPMemory4DWHeader hdr_4dw = defaultValue;
		 hdr_4dw.format = MEM_WRITE_4DW_DATA;
		 hdr_4dw.tag = extend(awid);
		 hdr_4dw.reqid = my_id;
		 hdr_4dw.nosnoop = SNOOPING_REQD;
		 hdr_4dw.addr = addr[40-1:2];
		 hdr_4dw.length = tlplen;
		 hdr_4dw.firstbe = 4'hf;
		 hdr_4dw.lastbe = (tlplen > 1) ? 4'hf : 0;
		 tlp.data = pack(hdr_4dw);
	      end
	      else begin
		 TLPMemoryIO3DWHeader hdr_3dw = defaultValue;
		 hdr_3dw.format = MEM_WRITE_3DW_DATA;
		 hdr_3dw.tag = extend(awid);
		 hdr_3dw.reqid = my_id;
		 hdr_3dw.nosnoop = SNOOPING_REQD;
		 hdr_3dw.addr = addr[32-1:2];
		 hdr_3dw.length = tlplen;
		 hdr_3dw.firstbe = 4'hf;
		 hdr_3dw.lastbe = (tlplen > 1) ? 4'hf : 0;

		 tlp.be = 16'hfff0; // no data word in this TLP

		 tlp.data = pack(hdr_3dw);
	      end
	      tlpWriteHeaderFifo.enq(tlp);
	      writeBurstCount <= zeroExtend(burstLen);
	      writeTag.enq(extend(awid));
           endmethod
	endinterface
      interface Put writeData;
         method Action put(MemData#(buswidth) wdata)
	      provisos (Bits#(Vector#(busWidthWords, Bit#(32)), busWidth)) if (writeBurstCount > 0 && writeDataMimo.enqReadyN(fromInteger(valueOf(busWidthWords))));

	      writeBurstCount <= writeBurstCount - 1;
	      Vector#(busWidthWords, Bit#(32)) v = unpack(wdata.data);
	      writeDataMimo.enq(fromInteger(valueOf(busWidthWords)), v);
           endmethod
       endinterface
      interface Get writeDone;
         method ActionValue#(Bit#(ObjectTagSize)) get();
	      let tag = doneTag.first();
	      doneTag.deq();
	      return truncate(tag);
           endmethod
	endinterface
   endinterface
   interface MemReadServer read_server;
      interface Put readReq;
         method Action put(MemRequest#(40) req) if (!writeInProgress && !writeDataMimo.deqReady());
	      let burstLen = req.burstLen >> beat_shift;
	      let addr = req.addr;
	      let arid = req.tag;

	       TLPData#(16) tlp = defaultValue;
	       tlp.sof = True;
	       tlp.eof = True;
	       tlp.hit = 7'h00;
	       TLPLength tlplen = fromInteger(valueOf(busWidthWords))*(extend(burstLen));
	       if (addr[39:32] != 0) begin
		   TLPMemory4DWHeader hdr_4dw = defaultValue;
		   hdr_4dw.format = MEM_READ_4DW_NO_DATA;
		   hdr_4dw.tag = extend(arid);
		   hdr_4dw.reqid = my_id;
		   hdr_4dw.nosnoop = SNOOPING_REQD;
		   hdr_4dw.addr = addr[40-1:2];
		   hdr_4dw.length = tlplen;
		   hdr_4dw.firstbe = 4'hf;
		   hdr_4dw.lastbe = (tlplen > 1) ? 4'hf : 0;
		   tlp.data = pack(hdr_4dw);
		   tlp.be = 16'hffff;
	       end
	       else begin
		   TLPMemoryIO3DWHeader hdr_3dw = defaultValue;
		   hdr_3dw.format = MEM_READ_3DW_NO_DATA;
		   hdr_3dw.tag = extend(arid);
		   hdr_3dw.reqid = my_id;
		   hdr_3dw.nosnoop = SNOOPING_REQD;
		   hdr_3dw.addr = addr[32-1:2];
		   hdr_3dw.length = tlplen;
		   hdr_3dw.firstbe = 4'hf;
		   hdr_3dw.lastbe = (tlplen > 1) ? 4'hf : 0;
		   tlp.data = pack(hdr_3dw);
		   tlp.be = 16'hfff0;
	       end
	       tlpOutFifo.enq(tlp);
           endmethod
       endinterface
      interface Get     readData;
         method ActionValue#(MemData#(buswidth)) get() if (completionMimo.deqReady()
							   && completionTagMimo.deqReady());
	      let data_v = completionMimo.first;
	      let tag_v = completionTagMimo.first;
	      completionMimo.deq();
	      completionTagMimo.deq();
              Bit#(buswidth) v = 0;
	      for (Integer i = 0; i < valueOf(busWidthWords); i = i+1)
		 v[(i+1)*32-1:i*32] = byteSwap(data_v[i]);
	      return MemData { data: v, tag: truncate(tag_v[0])};
           endmethod
	endinterface
   endinterface
    endinterface: slave
   method Bool tlpOutFifoNotEmpty() = tlpOutFifo.notEmpty;
   interface Reg use4dw = use4dwReg;
endmodule: mkMemSlaveEngine


added mkMemServerFromBram

added mkMemServerFromBram

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import FIFOF::*;
import FIFO::*;
import GetPut::*;
import MemTypes::*;
import AddressGenerator::*;
import BRAM::*;
import Memory::*;

module mkPhysMemToBram#(BRAMServer#(Bit#(bramAddrWidth), Bit#(busDataWidth)) br) (PhysMemSlave#(busAddrWidth, busDataWidth))
   provisos(Add#(a__, bramAddrWidth, busAddrWidth));

   FIFOF#(Bit#(6))  readTagFifo <- mkFIFOF();
   FIFOF#(Bit#(6)) writeTagFifo <- mkFIFOF();
   FIFO#(Bool)     readLastFifo <- mkFIFO();
   AddressGenerator#(busAddrWidth,busDataWidth) readAddrGenerator <- mkAddressGenerator();
   AddressGenerator#(busAddrWidth,busDataWidth) writeAddrGenerator <- mkAddressGenerator();
   let verbose = False;

    Reg#(Bit#(32)) cycles      <- mkReg(0);
    rule count;
       cycles <= cycles + 1;
    endrule

   rule read_req;
      let addrBeat <- readAddrGenerator.addrBeat.get();
      let addr = addrBeat.addr;
      let tag = addrBeat.tag;
      let burstCount = addrBeat.bc;
      readTagFifo.enq(tag);
      readLastFifo.enq(addrBeat.last);
      Bit#(bramAddrWidth) regFileAddr = truncate(addr/fromInteger(valueOf(TDiv#(busDataWidth,8))));
      br.request.put(BRAMRequest{write:False, responseOnWrite:False, address:regFileAddr, datain:?});
      if (verbose) $display("%d read_server.readData (a) %h %d last=%d", cycles, addr, burstCount, addrBeat.last);
   endrule

   interface PhysMemReadServer read_server;
      interface Put readReq;
	 method Action put(PhysMemRequest#(busAddrWidth, busDataWidth) req);
            if (verbose) $display("%d axiSlave.read.readAddr %h bc %d", cycles, req.addr, req.burstLen);
	    readAddrGenerator.request.put(req);
	 endmethod
      endinterface
      interface Get readData;
	 method ActionValue#(MemData#(busDataWidth)) get();
   	    let tag = readTagFifo.first;
	    readTagFifo.deq;
	    readLastFifo.deq;
            let data <- br.response.get;
            if (verbose) $display("%d read_server.readData (b) %h", cycles, data);
            return MemData { data: data, tag: tag, last: readLastFifo.first };
	 endmethod
      endinterface
   endinterface
   interface PhysMemWriteServer write_server;
      interface Put writeReq;
	 method Action put(PhysMemRequest#(busAddrWidth, busDataWidth) req);
	    writeAddrGenerator.request.put(req);
            if (verbose) $display("%d write_server.writeAddr %h bc %d", cycles, req.addr, req.burstLen);
	 endmethod
      endinterface
      interface Put writeData;
	 method Action put(MemData#(busDataWidth) resp);
	    let addrBeat <- writeAddrGenerator.addrBeat.get();
	    let addr = addrBeat.addr;
	    Bit#(bramAddrWidth) regFileAddr = truncate(addr/fromInteger(valueOf(TDiv#(busDataWidth,8))));
            br.request.put(BRAMRequest{write:True, responseOnWrite:False, address:regFileAddr, datain:resp.data});
            if (verbose) $display("%d write_server.writeData %h %h %d", cycles, addr, resp.data, addrBeat.bc);
            if (addrBeat.last)
               writeTagFifo.enq(addrBeat.tag);
	 endmethod
      endinterface
      interface Get writeDone;
	 method ActionValue#(Bit#(6)) get();
	    writeTagFifo.deq;
            return writeTagFifo.first;
	 endmethod
      endinterface
   endinterface
endmodule

module mkPhysMemToBramBE#(BRAMServerBE#(Bit#(bramAddrWidth), Bit#(busDataWidth), dataWidthBytes) br) (PhysMemSlave#(busAddrWidth, busDataWidth))
   provisos(Add#(a__, bramAddrWidth, busAddrWidth)
           ,Mul#(dataWidthBytes, 8, busDataWidth)
           ,Div#(busDataWidth, 8, dataWidthBytes)
           );
   let verbose = False;

   FIFOF#(Bit#(6))  readTagFifo <- mkFIFOF();
   FIFOF#(Bit#(6)) writeTagFifo <- mkFIFOF();
   FIFO#(Bool)     readLastFifo <- mkFIFO();
   FIFOF#(Bit#(TDiv#(busDataWidth, 8))) readByteEnableFifo <- mkFIFOF;
   FIFOF#(Bit#(TDiv#(busDataWidth, 8))) writeByteEnableFifo <- mkFIFOF;

   AddressGenerator#(busAddrWidth,busDataWidth) readAddrGenerator <- mkAddressGenerator();
   AddressGenerator#(busAddrWidth,busDataWidth) writeAddrGenerator <- mkAddressGenerator();

   FIFO#(PhysMemRequest#(busAddrWidth, busDataWidth)) req_ars <- mkFIFO1;
   FIFO#(Bit#(bramAddrWidth)) req_addr <- mkFIFO1;
   FIFO#(BRAMRequestBE#(Bit#(bramAddrWidth), Bit#(busDataWidth), dataWidthBytes)) req_aws <- mkFIFO1;

   Reg#(Bit#(32)) cycles      <- mkReg(0);
   rule count if (verbose);
      cycles <= cycles + 1;
   endrule

   rule req_ar;
     let req <- toGet(req_ars).get;
     readAddrGenerator.request.put(req);
   endrule

   rule read_req;
      let addrBeat <- readAddrGenerator.addrBeat.get();
      let addr = addrBeat.addr;
      let tag = addrBeat.tag;
      let burstCount = addrBeat.bc;
      readTagFifo.enq(tag);
      readLastFifo.enq(addrBeat.last);
      Bit#(bramAddrWidth) regFileAddr = truncate(addr << fromInteger(valueOf(TLog#(dataWidthBytes))));
      req_addr.enq(regFileAddr);
      if (verbose) $display("%d read_server.readData (a) %h %d last=%d", cycles, addr, burstCount, addrBeat.last);
   endrule

   rule read_bram_req;
      let addr <- toGet(req_addr).get;
      br.request.put(BRAMRequestBE{writeen:0, responseOnWrite:False, address:addr, datain:?});
   endrule

   rule req_aw;
      let req <- toGet(req_aws).get;
      br.request.put(req);
   endrule

   interface PhysMemReadServer read_server;
      interface Put readReq;
	 method Action put(PhysMemRequest#(busAddrWidth, busDataWidth) req);
            if (verbose) $display("%d read_server.readAddr %h bc %d fbe %x lbe %x", cycles, req.addr, req.burstLen
`ifdef BYTE_ENABLES
               , req.firstbe, req.lastbe
`endif
               );
            req_ars.enq(req);
            readByteEnableFifo.enq(reqLastByteEnable(req));
	 endmethod
      endinterface
      interface Get readData;
	 method ActionValue#(MemData#(busDataWidth)) get();
   	    let tag = readTagFifo.first;
	    readTagFifo.deq;
	    readLastFifo.deq;
            let data <- br.response.get;
            let readBE = readByteEnableFifo.first;
            Bit#(dataWidthBytes) byteEnable = readLastFifo.first ? readBE : maxBound;
            let newdata = updateDataWithMask(0, data, byteEnable);
            if (verbose) $display("%d read_server.readData (b) %h, %h", cycles, data, newdata);
            if (readLastFifo.first) begin
               readByteEnableFifo.deq;
            end
            return MemData { data: newdata, tag: tag, last: readLastFifo.first };
	 endmethod
      endinterface
   endinterface
   interface PhysMemWriteServer write_server;
      interface Put writeReq;
	 method Action put(PhysMemRequest#(busAddrWidth, busDataWidth) req);
	    writeAddrGenerator.request.put(req);
            writeByteEnableFifo.enq(reqLastByteEnable(req));
            if (verbose) $display("%d write_server.writeAddr %h bc %d fbe %x lbe %x", cycles, req.addr, req.burstLen
`ifdef BYTE_ENABLES
               , req.firstbe, req.lastbe
`endif
               );
	 endmethod
      endinterface
      interface Put writeData;
	 method Action put(MemData#(busDataWidth) resp);
	    let addrBeat <- writeAddrGenerator.addrBeat.get();
	    let addr = addrBeat.addr;
	    Bit#(bramAddrWidth) regFileAddr = truncate(addr << fromInteger(valueOf(TLog#(TDiv#(busDataWidth,8)))));
            let writeBE = writeByteEnableFifo.first;
            Bit#(dataWidthBytes) byteEnable = addrBeat.last ? writeBE : maxBound;
            req_aws.enq(BRAMRequestBE{writeen:byteEnable, responseOnWrite:False, address:regFileAddr, datain:resp.data});
            if (verbose) $display("%d write_server.writeData %h %h %d", cycles, addr, resp.data, addrBeat.bc);
            if (addrBeat.last)
               writeByteEnableFifo.deq;
               writeTagFifo.enq(addrBeat.tag);
	 endmethod
      endinterface
      interface Get writeDone;
	 method ActionValue#(Bit#(6)) get();
	    writeTagFifo.deq;
            return writeTagFifo.first;
	 endmethod
      endinterface
   endinterface
endmodule

module mkMemServerFromBram#(BRAMServer#(Bit#(bramAddrWidth), Bit#(busDataWidth)) br) (MemServer#(busDataWidth))
   provisos(Add#(a__, bramAddrWidth, MemOffsetSize));

   FIFOF#(Bit#(6))  readTagFifo <- mkFIFOF();
   FIFOF#(Bit#(6)) writeTagFifo <- mkFIFOF();
   FIFO#(Bool)     readLastFifo <- mkFIFO();
   AddressGenerator#(bramAddrWidth,busDataWidth) readAddrGenerator <- mkAddressGenerator();
   AddressGenerator#(bramAddrWidth,busDataWidth) writeAddrGenerator <- mkAddressGenerator();
   let verbose = False;

    Reg#(Bit#(32)) cycles      <- mkReg(0);
    rule count;
       cycles <= cycles + 1;
    endrule

   rule read_req;
      let addrBeat <- readAddrGenerator.addrBeat.get();
      let addr = addrBeat.addr;
      let tag = addrBeat.tag;
      let burstCount = addrBeat.bc;
      readTagFifo.enq(tag);
      readLastFifo.enq(addrBeat.last);
      Bit#(bramAddrWidth) regFileAddr = truncate(addr/fromInteger(valueOf(TDiv#(busDataWidth,8))));
      br.request.put(BRAMRequest{write:False, responseOnWrite:False, address:regFileAddr, datain:?});
      if (verbose) $display("%d read_server.readData (a) %h %d last=%d", cycles, addr, burstCount, addrBeat.last);
   endrule

   interface MemReadServer readServer;
      interface Put readReq;
	 method Action put(MemRequest req);
            if (verbose) $display("%d axiSlave.read.readAddr %h bc %d", cycles, req.offset, req.burstLen);
	    readAddrGenerator.request.put(PhysMemRequest { addr: truncate(req.offset), burstLen: req.burstLen, tag: req.tag });
	 endmethod
      endinterface
      interface Get readData;
	 method ActionValue#(MemData#(busDataWidth)) get();
	    let tag = readTagFifo.first;
	    readTagFifo.deq;
	    readLastFifo.deq;
            let data <- br.response.get;
            if (verbose) $display("%d read_server.readData (b) %h", cycles, data);
            return MemData { data: data, tag: tag, last: readLastFifo.first };
	 endmethod
      endinterface
   endinterface
   interface MemWriteServer writeServer;
      interface Put writeReq;
	 method Action put(MemRequest req);
	    writeAddrGenerator.request.put(PhysMemRequest { addr: truncate(req.offset), burstLen: req.burstLen, tag: req.tag});
            if (verbose) $display("%d write_server.writeAddr %h bc %d", cycles, req.offset, req.burstLen);
	 endmethod
      endinterface
      interface Put writeData;
	 method Action put(MemData#(busDataWidth) resp);
	    let addrBeat <- writeAddrGenerator.addrBeat.get();
	    let addr = addrBeat.addr;
	    Bit#(bramAddrWidth) regFileAddr = truncate(addr/fromInteger(valueOf(TDiv#(busDataWidth,8))));
            br.request.put(BRAMRequest{write:True, responseOnWrite:False, address:regFileAddr, datain:resp.data});
            if (verbose) $display("%d write_server.writeData %h %h %d", cycles, addr, resp.data, addrBeat.bc);
            if (addrBeat.last)
               writeTagFifo.enq(addrBeat.tag);
	 endmethod
      endinterface
      interface Get writeDone;
	 method ActionValue#(Bit#(6)) get();
	    writeTagFifo.deq;
            return writeTagFifo.first;
	 endmethod
      endinterface
   endinterface
endmodule


remove unused import

remove unused import

// Copyright (c) 2014 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import Vector            :: *;
import Clocks            :: *;
import GetPut            :: *;
import FIFO              :: *;
import Connectable       :: *;
import ClientServer      :: *;
import DefaultValue      :: *;
import PcieSplitter      :: *;
import Xilinx            :: *;
import PCIEWRAPPER       :: *;
import Portal            :: *;
import Leds              :: *;
import Top               :: *;
import MemSlaveEngine    :: *;
import MemMasterEngine   :: *;
import PcieCsr           :: *;
import MemTypes          :: *;
import Bscan             :: *;
`ifndef BSIM
import PcieEndpointX7    :: *;
`endif
import PcieHost         :: *;
import HostInterface    :: *;

`ifndef DataBusWidth
`define DataBusWidth 64
`endif
`ifndef PinType
`define PinType Empty
`endif

typedef `PinType PinType;

`ifndef BSIM
(* no_default_clock, no_default_reset *)
`endif
module mkPcieTop #(Clock pci_sys_clk_p, Clock pci_sys_clk_n, Clock sys_clk_p, Clock sys_clk_n, Reset pci_sys_reset_n)
   (PcieTop#(PinType));

   PcieHostTop host <- mkPcieHostTop(pci_sys_clk_p, pci_sys_clk_n, sys_clk_p, sys_clk_n, pci_sys_reset_n);

`ifdef IMPORT_HOSTIF
   ConnectalTop#(PhysAddrWidth, DataBusWidth, PinType, NumberOfMasters) portalTop <- mkConnectalTop(host, clocked_by host.portalClock, reset_by host.portalReset);
`else
   ConnectalTop#(PhysAddrWidth, DataBusWidth, PinType, NumberOfMasters) portalTop <- mkConnectalTop(clocked_by host.portalClock, reset_by host.portalReset);
`endif
   mkConnection(host.tpciehost.master, portalTop.slave, clocked_by host.portalClock, reset_by host.portalReset);
   if (valueOf(NumberOfMasters) > 0) begin
      mapM(uncurry(mkConnection),zip(portalTop.masters, host.tpciehost.slave));
   end

   // going from level to edge-triggered interrupt
   Vector#(16, Reg#(Bool)) interruptRequested <- replicateM(mkReg(False, clocked_by host.portalClock, reset_by host.portalReset));
   rule interrupt_rule;
     Maybe#(Bit#(4)) intr = tagged Invalid;
     for (Integer i = 0; i < 16; i = i + 1) begin
	 if (portalTop.interrupt[i] && !interruptRequested[i])
             intr = tagged Valid fromInteger(i);
	 interruptRequested[i] <= portalTop.interrupt[i];
     end
     if (intr matches tagged Valid .intr_num) begin
        ReadOnly_MSIX_Entry msixEntry = host.tpciehost.msixEntry[intr_num];
        host.tpciehost.interruptRequest.put(tuple2({msixEntry.addr_hi, msixEntry.addr_lo}, msixEntry.msg_data));
     end
   endrule

`ifndef BSIM
   interface pcie = host.tep7.pcie;
   method Bit#(NumLeds) leds();
      return extend({host.tep7.user.lnk_up(),3'd2});
   endmethod
   interface pins = portalTop.pins;
`endif
endmodule

// Copyright (c) 2014 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import Vector            :: *;
import Clocks            :: *;
import GetPut            :: *;
import FIFO              :: *;
import Connectable       :: *;
import ClientServer      :: *;
import DefaultValue      :: *;
import PcieSplitter      :: *;
import PcieTracer        :: *;
import Xilinx            :: *;
import PCIEWRAPPER       :: *;
import Portal            :: *;
import Leds              :: *;
import Top               :: *;
import MemSlaveEngine    :: *;
import MemMasterEngine   :: *;
import PcieCsr           :: *;
import MemTypes          :: *;
import Bscan             :: *;
`ifndef BSIM
import PcieEndpointX7    :: *;
`endif
import PcieHost         :: *;
import HostInterface    :: *;

`ifndef DataBusWidth
`define DataBusWidth 64
`endif
`ifndef PinType
`define PinType Empty
`endif

typedef `PinType PinType;

`ifndef BSIM
(* no_default_clock, no_default_reset *)
`endif
module mkPcieTop #(Clock pci_sys_clk_p, Clock pci_sys_clk_n, Clock sys_clk_p, Clock sys_clk_n, Reset pci_sys_reset_n)
   (PcieTop#(PinType));

   PcieHostTop host <- mkPcieHostTop(pci_sys_clk_p, pci_sys_clk_n, sys_clk_p, sys_clk_n, pci_sys_reset_n);

`ifdef IMPORT_HOSTIF
   ConnectalTop#(PhysAddrWidth, DataBusWidth, PinType, NumberOfMasters) portalTop <- mkConnectalTop(host, clocked_by host.portalClock, reset_by host.portalReset);
`else
   ConnectalTop#(PhysAddrWidth, DataBusWidth, PinType, NumberOfMasters) portalTop <- mkConnectalTop(clocked_by host.portalClock, reset_by host.portalReset);
`endif
   mkConnection(host.tpciehost.master, portalTop.slave, clocked_by host.portalClock, reset_by host.portalReset);
   if (valueOf(NumberOfMasters) > 0) begin
      mapM(uncurry(mkConnection),zip(portalTop.masters, host.tpciehost.slave));
   end

   // going from level to edge-triggered interrupt
   Vector#(16, Reg#(Bool)) interruptRequested <- replicateM(mkReg(False, clocked_by host.portalClock, reset_by host.portalReset));
   rule interrupt_rule;
     Maybe#(Bit#(4)) intr = tagged Invalid;
     for (Integer i = 0; i < 16; i = i + 1) begin
	 if (portalTop.interrupt[i] && !interruptRequested[i])
             intr = tagged Valid fromInteger(i);
	 interruptRequested[i] <= portalTop.interrupt[i];
     end
     if (intr matches tagged Valid .intr_num) begin
        ReadOnly_MSIX_Entry msixEntry = host.tpciehost.msixEntry[intr_num];
        host.tpciehost.interruptRequest.put(tuple2({msixEntry.addr_hi, msixEntry.addr_lo}, msixEntry.msg_data));
     end
   endrule

`ifndef BSIM
   interface pcie = host.tep7.pcie;
   method Bit#(NumLeds) leds();
      return extend({host.tep7.user.lnk_up(),3'd2});
   endmethod
   interface pins = portalTop.pins;
`endif
endmodule


split the accumOutFifo

split the accumOutFifo

// Copyright (c) 2014 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import FIFO::*;
import FIFOF::*;
import MIMO::*;
import DefaultValue::*;
import SpecialFIFOs::*;
import Vector::*;
import BRAM::*;
import DmaVector::*;
import PortalMemory::*;
import Dma::*;
import FloatingPoint::*;
import Pipe::*;
import FloatOps::*;
import Timer::*;
import RbmTypes::*;
import Assert::*;
import Connectable::*;

interface DotProdServer#(numeric type n);
   interface Reg#(UInt#(20)) numElts;
   interface Put#(Tuple2#(Vector#(n,Float),Vector#(n,Float))) request;
   interface PipeOut#(Float) pipe;
endinterface

(* synthesize *)
module [Module] mkDotProdServer#(UInt#(TLog#(TMul#(J,K))) label)(DotProdServer#(N));

   let n = valueOf(N);
   let add_depth = valueOf(FP_ADD_DEPTH);
   let mul_depth = valueOf(FP_MUL_DEPTH);
   Bool verbose = False; //label==0;

   Reg#(Bit#(TAdd#(TLog#(FP_ADD_DEPTH),1))) initCnt <- mkReg(0);
   FIFO#(void) initCtrl <- mkSizedFIFO(1);

   Reg#(UInt#(20)) numEltsReg <- mkReg(0);
   Reg#(UInt#(20)) countInReg <- mkReg(0);

   Vector#(N, FloatAlu#(FP_MUL_DEPTH)) muls <- replicateM(mkFloatMultiplier(defaultValue));
   Vector#(N, FloatAlu#(FP_ADD_DEPTH)) adders <- replicateM(mkFloatAdder(defaultValue));

   Vector#(N,FIFOF#(Tuple2#(Float,Float))) abfifos <- replicateM(mkFIFOF());
   Vector#(N,FIFOF#(Bool)) lastFifos <- replicateM(mkSizedFIFOF(mul_depth));

   Vector#(N, Reg#(Bit#(TAdd#(TLog#(FP_ADD_DEPTH),1)))) drainCnts <- replicateM(mkReg(0));
   Vector#(N, Reg#(Bool)) drained <- replicateM(mkReg(False));

   Reg#(Maybe#(Float)) accum <- mkReg(Nothing);
   FIFOF#(Float) dotfifo <- mkFIFOF;

   function Bit#(TLog#(N)) i_v(Integer i) = fromInteger(i);

   rule init if (initCnt > 0);
      for(Integer i = 0; i < n; i = i + 1)
	 adders[i].request.put(tuple2(0,0));
      initCnt <= initCnt - 1;
      if (verbose) $display($format(fshow("label=")+fshow(label)+fshow(" initCnt=")+fshow(initCnt)));
   endrule

   for (Integer i = 0; i < n; i = i + 1)
      rule mul;
	 // this rule could be folded into the 'put' method to reduce latency
	 match {.x,.y} <- toGet(abfifos[i]).get;
	 muls[i].request.put(tuple2(x, y));
	 if (verbose) $display($format(fshow("label=")+fshow(label)+fshow(" mul=")+fshow(i_v(i))));
      endrule

   for (Integer i = 0; i < n; i = i + 1)
      rule acc if (drainCnts[i] == 0 && initCnt == 0);
	 match {.resp,.*} <- muls[i].response.get();
	 match {.acc,.*} <- adders[i].response.get;
	 adders[i].request.put(tuple2(resp,acc));
	 let last <- toGet(lastFifos[i]).get;
	 if (last) begin
	    if (i>0)
	       drainCnts[i] <= fromInteger(add_depth);
	    else
	       drainCnts[i] <= fromInteger((add_depth*2)-1); // is this correct??
	 end
	 if (verbose) $display($format(fshow("label=")+fshow(label)+fshow(" acc=")+fshow(i_v(i))+fshow(" last=")+fshow(last)));
      endrule

   for (Integer i = 1; i < n; i = i + 1)
      rule gather if (drainCnts[i] > 0);
	 let new_cnt = drainCnts[i]-1;
	 drained[i] <= (new_cnt==0);
	 drainCnts[i] <= new_cnt;
	 match {.a,.*} <- adders[0].response.get;
	 match {.b,.*} <- adders[i].response.get;
	 adders[0].request.put(tuple2(a,b));
	 if (verbose) $display($format(fshow("label=")+fshow(label)+fshow(" gather=")+fshow(i_v(i))+fshow(" drainCnt=")+fshow(drainCnts[i])));
      endrule

   // the reference-guide says this should work, but it doesn't compile:
   // let gathered = and(tail(drained));
   function Bool is_true(Bool b) = b;
   let gathered = all(is_true, readVReg(tail(drained)));
   // this will only work correctly when add_depth is odd
   rule drain if (gathered && drainCnts[0] > 0);
      let new_cnt = drainCnts[0]-1;
      match {.a,.*} <- adders[0].response.get;
      drainCnts[0] <= new_cnt;
      let enq = False;
      if(accum matches tagged Valid .v) begin
	 adders[0].request.put(tuple2(a,v));
	 dynamicAssert(new_cnt > 0, "mkDotProdServer::drain");
	 enq = True;
	 accum <= tagged Invalid;
      end
      else begin
	 if (new_cnt == 0) begin
	    dotfifo.enq(a);
	    initCtrl.deq;
	 end
	 else
	    accum <= tagged Valid a;
      end
      if (verbose) $display($format(fshow("label=")+fshow(label)+fshow(" drain0=")+fshow(enq)+fshow(" drainCnt=")+fshow(drainCnts[0])));
   endrule

   PipeOut#(Float) dotpipe = toPipeOut(dotfifo);
   interface Put request;
      method Action put(Tuple2#(Vector#(N,Float),Vector#(N,Float)) tpl);
	 if (countInReg == 0) begin
	    initCtrl.enq(?);
	    initCnt <= fromInteger(add_depth);
	 end
	 let c = countInReg+fromInteger(n);
	 Bool isLast = (c >= numEltsReg);
	 if (isLast) begin
	    c = 0;
	 end
	 countInReg <= c;
	 match { .avec, .bvec } = tpl;
	 function Action enqvalues(Integer i);
	    action
	       abfifos[i].enq(tuple2(avec[i], bvec[i]));
	       lastFifos[i].enq(isLast);
	       //if (verbose) $display($format(fshow("label=")+fshow(label)+fshow(" countInReg=(")+fshow(countInReg)+fshow("/")+fshow(numEltsReg)+fshow(") dotprod x=") + fshow(avec[i]) + fshow(" y=") + fshow(bvec[i])));
	    endaction
	 endfunction
	 Vector#(N, Integer) indices = genVector();
	 mapM_(enqvalues, indices);
      endmethod
   endinterface : request
   interface PipeOut pipe = dotpipe;
   interface Reg numElts = numEltsReg;
endmodule : mkDotProdServer

function Vector#(TMul#(j,k), etype) flattenMatrix(Vector#(j, Vector#(k, etype)) mat);
   function etype flatten(Integer i); return mat[i/valueOf(k)][i%valueOf(k)]; endfunction
   return genWith(flatten);
endfunction

typedef struct {
   a xbase;
   a xlimit;
   a xstep;
   a ybase;
   a ylimit;
   a ystep;
} XYRangeConfig#(type a) deriving (Bits, FShow);

interface XYRangePipeIfc#(type a);
   interface PipeOut#(Tuple2#(a,a)) pipe;
   method Action start(XYRangeConfig#(a) cfg);
   method Action display();
endinterface

module mkXYRangePipeOut(XYRangePipeIfc#(a)) provisos (Arith#(a), Bits#(a,awidth), Eq#(a), Ord#(a));
   Reg#(a) x <- mkReg(0);
   Reg#(a) y <- mkReg(0);
   Reg#(a) xbase <- mkReg(0);
   Reg#(a) ybase <- mkReg(0);
   Reg#(a) xstep <- mkReg(0);
   Reg#(a) ystep <- mkReg(0);
   Reg#(a) xlimit <- mkReg(0);
   Reg#(a) ylimit <- mkReg(0);

   interface PipeOut pipe;
      method Tuple2#(a,a) first() if (x < xlimit && y < ylimit);
	 return tuple2(x,y);
      endmethod
      method Action deq if (x < xlimit && y < ylimit);
	 let newx = x;
	 let newy = y+ystep;
	 if (newy >= ylimit && x < xlimit) begin
	    newy = ybase;
	    newx = newx + xstep;
	 end
	 x <= newx;
	 y <= newy;
      endmethod
      method Bool notEmpty();
	 return (x < xlimit && y < ylimit);
      endmethod
   endinterface
   method Action start(XYRangeConfig#(a) cfg) if (x >= xlimit);
      //$display("XYRangePipe x=%d xlimit=%d xstep=%d y=%d ylimit=%d ystep=%d", cfg.xbase, cfg.xlimit, cfg.xstep, cfg.ybase, cfg.ylimit, cfg.ystep);
      x <= cfg.xbase;
      y <= cfg.ybase;
      xbase <= cfg.xbase;
      ybase <= cfg.ybase;
      xstep <= cfg.xstep;
      ystep <= cfg.ystep;
      xlimit <= cfg.xlimit;
      ylimit <= cfg.ylimit;
   endmethod
   method Action display();
      $display("XYRangePipe x=%d xlimit=%d y=%d ylimit=%d xstep=%d ystep=%d", x, xlimit, xstep, y, ylimit, ystep);
   endmethod
endmodule: mkXYRangePipeOut

typedef struct {
   ObjectPointer pointer;
   addrtype base;
   addrtype numRows;
   addrtype numColumns;
} MatrixDescriptor#(type addrtype) deriving (Bits);

// row major layout
interface DmaMatrixMultiplyIfc#(numeric type addrwidth, numeric type dsz);
   interface Vector#(J, ObjectWriteClient#(dsz)) writeClients;
   method Action start(ObjectPointer pointerA, UInt#(addrwidth) numRowsA, UInt#(addrwidth) numColumnsA,
		       ObjectPointer pointerB, UInt#(addrwidth) numRowsB, UInt#(addrwidth) numColumnsB,
		       ObjectPointer pointerC);
   method ActionValue#(Bool) finish();
endinterface

typedef enum {
   Idle, Ready, Running, Done
   } MMState deriving (Bits, Eq);

/*!
 * Multiplies two matrices A and B and writes the result to memory.
 * Fetches J rows at a time from A and K rows at a time from B.
 * Each cycle, it can fetch N elements of a row or column.
 *
 * Just considering memory bandwidth, every J+K cycles it is ready to perform J*K*N multiply accumulates.
 *
 */
module [Module] mkDmaMatrixMultiply#(Vector#(J, VectorSource#(dsz, Vector#(N, Float))) sourceA,
				     Vector#(K, VectorSource#(dsz, Vector#(N, Float))) sourceB,
				     function Module#(DmaVectorSink#(dsz, Vector#(N, Float))) mkSink(PipeOut#(Vector#(N, Float)) pipe_in)
				     )(DmaMatrixMultiplyIfc#(addrwidth, dsz))
   provisos (  Add#(N,n__,K)
	     , Mul#(N,m__,K)
	     , Add#(1,o__,J)
	     , Log#(N,nshift)
	     , FShow#(Float)
	     , Arith#(Float)
	     , Bits#(Vector#(N, Float), dsz)
	     , Bits#(MatrixDescriptor#(UInt#(addrwidth)), mdsz)
	     , Bits#(Tuple2#(UInt#(addrwidth), UInt#(addrwidth)), tplsz)
	     , Add#(b__, 20, addrwidth)
	     , Add#(a__, addrwidth, 40)
      );

   let n = valueOf(N);
   let jj = valueOf(J);
   let kk = valueOf(K);
   let nshift = valueOf(nshift);
   Bool verbose = False;
   Bool verbose1 = False;
   Bool timing = True;

   Reg#(UInt#(32)) cycles <- mkReg(0);
   Reg#(Bool) doneReg <- mkReg(False);
   Reg#(MatrixDescriptor#(UInt#(addrwidth))) descriptorA <- mkReg(unpack(0));
   Reg#(MatrixDescriptor#(UInt#(addrwidth))) descriptorB <- mkReg(unpack(0));
   Reg#(MatrixDescriptor#(UInt#(addrwidth))) descriptorC <- mkReg(unpack(0));
   Reg#(UInt#(addrwidth)) dotprodCount <- mkReg(0);

   Vector#(J, Vector#(K, PipeOut#(Vector#(N,Float)))) aPipes  <- mapM(mkSizedForkVector(valueOf(SourceBufferSize)), map(vectorSourcePipe, sourceA));
   Vector#(K, Vector#(J, PipeOut#(Vector#(N,Float)))) bPipesT <- mapM(mkForkVector, map(vectorSourcePipe, sourceB));
   Vector#(J, Vector#(K, PipeOut#(Vector#(N,Float)))) bPipes  =  transpose(bPipesT);

   rule countCycles;
      cycles <= cycles+1;
   endrule

//`define OLDWAY
`ifdef OLDWAY
   function Module#(DotProdServer#(N)) mkFxDotProd(Integer i);
      return mkDotProdServer(fromInteger(i));
   endfunction
   function Put#(Tuple2#(Vector#(n,Float),Vector#(n,Float))) getDotProdServerRequestInterface(DotProdServer#(n) s); return s.request; endfunction
   function PipeOut#(Float) getDotProdServerPipe(DotProdServer#(n) s); return s.pipe; endfunction
   Vector#(TMul#(J,K), DotProdServer#(N)) fxdotprods <- genWithM(mkFxDotProd);
   Vector#(TMul#(J,K), PipeOut#(Float)) fxpipes = map(getDotProdServerPipe, fxdotprods);

   for (Integer k = 0; k < kk; k = k+1) begin
      for (Integer j = 0; j < jj; j = j + 1) begin
	 rule connectDotProd;
	    let index = j*kk+k;
	    let a <- toGet(aPipes[j][k]).get();
	    let b <- toGet(bPipes[j][k]).get();
	    int jint = fromInteger(j);
	    int kint = fromInteger(k);
	    //$display($format(fshow("connectDotProd pos=")+fshow(tuple2(jint,kint))+fshow(" a=")+fshow(pack(a))+fshow(" b=")+fshow(pack(b))));
	    fxdotprods[index].request.put(tuple2(a, b));
	 endrule
      end
   end
`else
   FirstLastPipe#(UInt#(addrwidth)) firstLastPipe <- mkFirstLastPipe();
   Vector#(TAdd#(TMul#(J,K),1), PipeOut#(Tuple2#(Bool,Bool))) firstLastPipes      <- mkForkVector(firstLastPipe.pipe);

   Vector#(TMul#(J,K), FIFOF#(Float))   accumFifos  <- replicateM(mkFIFOF);
   Vector#(TMul#(J,K), PipeOut#(Float)) accumPipes  =  map(toPipeOut, accumFifos);
   Vector#(M, PipeOut#(Float))          accumFunnels <- mkFunnelPipes(accumPipes);

   rule primeTheAccum;
      match { .first, .last } <- toGet(firstLastPipes[valueOf(TMul#(J,K))]).get();
      if (first)
	 for (Integer i = 0; i < valueOf(TMul#(J,K)); i = i + 1)
	    accumFifos[i].enq(unpack(0));
   endrule

   Vector#(TMul#(J,K), PipeOut#(Vector#(1,Float))) a1Pipes <- mapM(mkFunnel, flattenMatrix(aPipes));
   Vector#(TMul#(J,K), PipeOut#(Vector#(1,Float))) b1Pipes <- mapM(mkFunnel, flattenMatrix(bPipes));
   Vector#(TMul#(J,K), PipeOut#(Tuple2#(Vector#(1,Float),Vector#(1,Float)))) ab1Pipes = map(uncurry(zipPipeOut), zip(a1Pipes, b1Pipes));

   Vector#(M, PipeOut#(Tuple2#(Vector#(1,Float),Vector#(1,Float)))) abFunnels <- mkFunnelPipes(ab1Pipes);
   Vector#(M, FloatAlu#(FP_MUL_DEPTH)) muls <- replicateM(mkFloatMultiplier(defaultValue));
   Vector#(M, FloatAlu#(FP_ADD_DEPTH)) adders <- replicateM(mkFloatAdder(defaultValue));
   rule mulreq;
      for (Integer i = 0; i < valueOf(M); i = i + 1) begin
	 match { .avec, .bvec } <- toGet(abFunnels[i]).get();
	 muls[i].request.put(tuple2(avec[0], bvec[0]));
      end
   endrule
   rule accumreq;
      for (Integer i = 0; i < valueOf(M); i = i + 1) begin
	 let accum <- toGet(accumFunnels[i]).get();
	 match {.resp, .*} <- muls[i].response.get();
	 adders[i].request.put(tuple2(resp, accum));
      end
   endrule
   Vector#(M, FIFOF#(Float))   accumOutFifos      <- replicateM(mkFIFOF());
   Vector#(M, PipeOut#(Float)) accumOutFunnelPipes =  map(toPipeOut, accumOutFifos);
   for (Integer i = 0; i < valueOf(M); i = i + 1) begin
      rule accumresp;
	 match {.resp, .*} <- adders[i].response.get();
	 accumOutFifos[i].enq(resp);
      endrule
   end
   Vector#(TMul#(J,K), PipeOut#(Float)) accumOutPipes <- mkUnfunnelPipes(accumOutFunnelPipes);
   Vector#(TMul#(J,K), FIFOF#(Float))     resultFifos <- replicateM(mkFIFOF());
   for (Integer i = 0; i < valueOf(TMul#(J,K)); i = i + 1)
      rule feedback;
	 let accumOut <- toGet(accumOutPipes[i]).get();
	 match { .first, .last } <- toGet(firstLastPipes[i]).get();
	 if (last)
	    resultFifos[i].enq(accumOut);
	 else
	    accumFifos[i].enq(accumOut);
      endrule
   Vector#(TMul#(J,K), PipeOut#(Float)) fxpipes = map(toPipeOut, resultFifos);

`endif
   MIMOConfiguration mimoCfg = defaultValue;
   MIMO#(K,N,TAdd#(K,N),Float) dfifo <- mkMIMO(mimoCfg);
   Vector#(J, MIMO#(K,N,TAdd#(K,N),Float)) dfifos <- replicateM(mkMIMO(mimoCfg));
   let sinks <- mapM(mkSink, map(toPipeOut, dfifos));

   XYRangePipeIfc#(UInt#(addrwidth)) indexpipeifc <- mkXYRangePipeOut();
   XYRangePipeIfc#(UInt#(addrwidth)) offsetpipeA <- mkXYRangePipeOut();
   XYRangePipeIfc#(UInt#(addrwidth)) offsetpipeB <- mkXYRangePipeOut();
   XYRangePipeIfc#(UInt#(addrwidth)) offsetpipeC <- mkXYRangePipeOut();

   Vector#(TAdd#(TAdd#(J,J),K), PipeOut#(Tuple2#(UInt#(addrwidth),UInt#(addrwidth)))) indexpipes <- mkSizedForkVector(valueOf(SourceBufferSize), indexpipeifc.pipe);
   Vector#(J, PipeOut#(Tuple2#(UInt#(addrwidth),UInt#(addrwidth))))        offsetpipesA <- mkSizedForkVector(valueOf(SourceBufferSize), offsetpipeA.pipe);
   Vector#(K, PipeOut#(Tuple2#(UInt#(addrwidth),UInt#(addrwidth))))        offsetpipesB <- mkSizedForkVector(valueOf(SourceBufferSize), offsetpipeB.pipe);
   Vector#(J, PipeOut#(Tuple2#(UInt#(addrwidth),UInt#(addrwidth))))        offsetpipesC <- mkSizedForkVector(valueOf(SourceBufferSize), offsetpipeC.pipe);

   Reg#(Bool) running <- mkReg(False);
   FIFOF#(Bool) doneFifo <- mkFIFOF();

   Vector#(J, Reg#(UInt#(addrwidth))) startAOffset <- replicateM(mkReg(0));
   Vector#(K, Reg#(UInt#(addrwidth))) startBOffset <- replicateM(mkReg(0));
   Vector#(J, Reg#(UInt#(addrwidth))) startCOffset <- replicateM(mkReg(0));
   for (Integer k = 0; k < kk; k = k + 1) begin
      rule startDotProds;
	 Tuple2#(UInt#(addrwidth),UInt#(addrwidth)) index <- toGet(indexpipes[k]).get();
	 match { .unusedB, .startBBase } <- toGet(offsetpipesB[k]).get();

	 int kint = fromInteger(k);

	 let row = tpl_1(index);
	 let col = tpl_2(index)+fromInteger(k);

	 let startB = startBBase + startBOffset[k];

	 if (timing || verbose) $display($format(fshow(cycles)+fshow("    startB index=")+fshow(tuple2(row,col))
	    +fshow(" startB=")+fshow(startB)
	    +fshow(" k=")+fshow(kint)));

	 if (verbose || verbose1) $display($format(fshow(cycles)+fshow("    sourceB[")+fshow(kint)+fshow("].start")+fshow(startB)));

	 sourceB[k].start(descriptorB.pointer, pack(extend(startB>>nshift)), pack(extend(descriptorB.numColumns>>nshift)));
      endrule
      rule finishSourceB;
	 UInt#(TLog#(K)) in = fromInteger(k);
	 int kint = fromInteger(k);
	 if (verbose || verbose1) $display($format(fshow(cycles)+fshow("    sourceB[")+fshow(kint)+fshow("].finish")));
	 let b <- sourceB[k].finish();
      endrule
   end
   for (Integer j = 0; j < jj; j = j + 1) begin

      rule startSourceAndSink;
	 Tuple2#(UInt#(addrwidth),UInt#(addrwidth)) index <- toGet(indexpipes[j+kk]).get();

	 let row = tpl_1(index)+fromInteger(j);
	 let col = tpl_2(index);

	 match { .startABase, .unusedA } <- toGet(offsetpipesA[j]).get();
	 match { .startCBase, .offsetC } <- toGet(offsetpipesC[j]).get();
	 let startA = startABase + startAOffset[j];
	 let startC = startCBase + startCOffset[j] + offsetC;

	 int jint = fromInteger(j);
	 if (timing || verbose) $display($format(fshow(cycles)+fshow("    start A/C index=")+fshow(tuple2(row,col))
	    +fshow(" startA=")+fshow(startA)
	    +fshow(" startC=")+fshow(startC)
	    +fshow(" startC>>nshift=")+fshow(startC>>nshift)
	    +fshow(" j=")+fshow(jint)));

	 sourceA[j].start(descriptorA.pointer, pack(extend(startA>>nshift)), pack(extend(descriptorA.numColumns>>nshift)));
	 if (verbose || verbose1) $display($format(fshow(cycles)+fshow("    sourceA[")+fshow(jint)+fshow("].start")+fshow(startA)));
	 sinks[j].vector.start(descriptorC.pointer, pack(extend(startC>>nshift)), fromInteger(kk/n));
	 if (verbose || verbose1) $display($format(fshow(cycles)+fshow("      sinks[")+fshow(jint)+fshow("].start")+fshow(startC)));

	 // this one counts by one rather than 1/n because the source values are unfunneled before mul/add
	 if (j == 0)
	    firstLastPipe.start(truncate(descriptorA.numColumns));
      endrule

      rule finishSourceA;
	 if (verbose || verbose1) $display($format(fshow(cycles)+fshow("    sourceA[0].finish ")));
	 let b <- sourceA[j].finish();
      endrule

      rule dotProdValue;
	 Tuple2#(UInt#(addrwidth),UInt#(addrwidth)) index <- toGet(indexpipes[jj+kk+j]).get();
	 Vector#(K,Float) vs;
	 for (Integer k = 0; k < kk; k = k + 1) begin
	    let dpnum = j*kk+k;
	    let v <- toGet(fxpipes[dpnum]).get();
	    //let indexi = tuple2(tpl_1(index), tpl_2(index)+fromInteger(k));
	    //if (verbose) $display($format(fshow(cycles)+fshow("    dotprodvalue index=")+fshow(indexi)+fshow(" dotprod=")+fshow(v)));
	    int jint = fromInteger(j);
	    int kint = fromInteger(k);
	    if (verbose1) $display($format(fshow(cycles)+fshow("    dotprodvalue pos=")+fshow(index)+fshow(tuple2(jint,kint))+fshow(" dotprod=")+fshow(pack(v))));
	    vs[k] = v;
	 end
	 dfifos[j].enq(fromInteger(kk), vs);
      endrule

      rule sinkDone;
	 // each time we write a burst of k values via sinks
	 //let index <- toGet(indexpipes[jj+kk+1]).get();
	 let b <- sinks[j].vector.finish();
	 let c = dotprodCount-fromInteger(kk);
	 int jint = fromInteger(j);
	 if (verbose1) $display($format(fshow(cycles)+fshow("    sinkDone c")+fshow(c)+fshow(" j=")+fshow(jint)));
	 dotprodCount <= c;
	 if (c == 0) begin
	    running <= False;
	    doneFifo.enq(?);
	 end
      endrule
   end

   method Action start(ObjectPointer pointerA, UInt#(addrwidth) numRowsA, UInt#(addrwidth) numColumnsA,
		       ObjectPointer pointerB, UInt#(addrwidth) numRowsB, UInt#(addrwidth) numColumnsB,
		       ObjectPointer pointerC) if (!running);
      XYRangeConfig#(UInt#(addrwidth)) indexcfg  = XYRangeConfig {xbase: 0, xlimit: numRowsA, xstep: fromInteger(jj),
								  ybase: 0, ylimit: numRowsB, ystep: fromInteger(kk) };
      XYRangeConfig#(UInt#(addrwidth)) offsetcfgA = XYRangeConfig {xbase: 0, xlimit: numRowsA*numColumnsA, xstep: numColumnsA*fromInteger(jj),
								  ybase: 0, ylimit: numRowsB, ystep: fromInteger(kk) };
      XYRangeConfig#(UInt#(addrwidth)) offsetcfgB = XYRangeConfig {xbase: 0, xlimit: numRowsA, xstep: fromInteger(jj),
								  ybase: 0, ylimit: numRowsB*numColumnsB, ystep: fromInteger(kk)*numColumnsB };
      XYRangeConfig#(UInt#(addrwidth)) offsetcfgC = XYRangeConfig {xbase: 0, xlimit: numRowsA*numRowsB, xstep: numRowsB*fromInteger(jj),
								  ybase: 0, ylimit: numRowsB, ystep: fromInteger(kk) };
      descriptorA <= MatrixDescriptor { pointer: pointerA, base: 0, numRows: numRowsA, numColumns: numColumnsA};
      descriptorB <= MatrixDescriptor { pointer: pointerB, base: 0, numRows: numRowsB, numColumns: numColumnsB};
      descriptorC <= MatrixDescriptor { pointer: pointerC, base: 0, numRows: numRowsA, numColumns: numRowsB};
      dotprodCount <= numRowsA*numRowsB;
      running <= True;

      if (verbose) $display("mm pointerA=%d pointerB=%d pointerC=%d\n", pointerA, pointerB, pointerC);
      if (verbose) $display("mm.start ra=%d ca=%d rb=%d cb=%d dotprodCount=%d", numRowsA, numColumnsA, numRowsB, numColumnsB, dotprodCount);
      if (verbose) $display($format(fshow("mm.start ")+fshow(indexcfg)));
      indexpipeifc.start(indexcfg);
      offsetpipeA.start(offsetcfgA);
      offsetpipeB.start(offsetcfgB);
      offsetpipeC.start(offsetcfgC);
      for (Integer j = 0; j < jj; j = j + 1) begin
	 startAOffset[j] <= fromInteger(j)*numColumnsA;
	 startCOffset[j] <= fromInteger(j)*numRowsB;
      end
      for (Integer k = 0; k < kk; k = k + 1) begin
	 for (Integer j = 0; j < jj; j = j + 1) begin
	    let index = j*kk + k;
`ifdef OLDWAY
	    fxdotprods[index].numElts <= truncate(numColumnsA);
`endif
	 end
	 startBOffset[k] <= fromInteger(k)*numColumnsB;
      end
   endmethod
   method ActionValue#(Bool) finish();
      if (verbose) $display("mm.finish()");
      doneFifo.deq();
      return True;
   endmethod

   interface Vector writeClients = map(getSinkWriteClient, sinks);
endmodule

typedef 20 MMSize;

interface DramMatrixMultiply#(numeric type n, numeric type dmasz);
   interface Vector#(TAdd#(K,J), ObjectReadClient#(dmasz)) readClients;
   interface Vector#(J, ObjectWriteClient#(dmasz)) writeClients;
   method Action start(ObjectPointer pointerA, UInt#(MMSize) numRowsA, UInt#(MMSize) numColumnsA,
		       ObjectPointer pointerB, UInt#(MMSize) numRowsB, UInt#(MMSize) numColumnsB,
		       ObjectPointer pointerC);
   method ActionValue#(Bool) finish();
   method Bit#(32) dbg();
endinterface

(* synthesize *)
module [Module] mkDramMatrixMultiply(DramMatrixMultiply#(N,TMul#(N,32)));
   Vector#(TAdd#(K,J), DmaVectorSource#(DmaSz, Vector#(N,Float))) vfsources <- replicateM(mkDmaVectorSource());
   Vector#(J, VectorSource#(DmaSz, Vector#(N,Float))) xvfsources = takeAt(0,          map(dmaVectorSourceVector, vfsources));
   Vector#(K, VectorSource#(DmaSz, Vector#(N,Float))) yvfsources = takeAt(valueOf(J), map(dmaVectorSourceVector, vfsources));
   DmaMatrixMultiplyIfc#(MMSize,DmaSz) dmaMMF <- mkDmaMatrixMultiply(xvfsources, yvfsources, mkDmaVectorSink);
   interface Vector readClients = map(getSourceReadClient, vfsources);
   interface Vector writeClients = dmaMMF.writeClients;
   method start = dmaMMF.start;
   method finish = dmaMMF.finish;
   method Bit#(32) dbg();
      Bit#(32) d = 0;
      d[0] = pack(vfsources[0].vector.pipe.notEmpty());
      d[1] = pack(vfsources[1].vector.pipe.notEmpty());
      return d;
   endmethod
endmodule

interface Mm#(numeric type n);
   interface MmRequest mmRequest;
   interface TimerRequest timerRequest;
   interface Vector#(TAdd#(K,J), ObjectReadClient#(TMul#(32,N))) readClients;
   interface Vector#(J, ObjectWriteClient#(TMul#(32,n))) writeClients;
endinterface

module [Module] mkMm#(MmIndication ind, TimerIndication timerInd)(Mm#(N))
   provisos (Add#(1,a__,N),
	     Add#(N,0,n),
	     Mul#(N,32,DmaSz)
      );

   let n = valueOf(n);

   DramMatrixMultiply#(N, TMul#(N,32)) dmaMMF <- mkDramMatrixMultiply();

   Reg#(Bit#(64)) mmfCycles <- mkReg(0);
   rule countMmfCycles;
      mmfCycles <= mmfCycles + 1;
   endrule

   FIFOF#(Bool) busyFifo <- mkFIFOF();
   rule mmfDone;
      let d <- dmaMMF.finish();
      busyFifo.deq();
      ind.mmfDone(mmfCycles);
   endrule

   FIFOF#(Bool) timerRunning <- mkFIFOF();
   Reg#(Bit#(64)) cycleCount <- mkReg(0);
   Reg#(Bit#(64)) idleCount <- mkReg(0);
   rule countCycles if (timerRunning.notEmpty());
      cycleCount <= cycleCount + 1;
      if (!busyFifo.notEmpty())
	 idleCount <= idleCount + 1;
   endrule

   interface TimerRequest timerRequest;
      method Action startTimer() if (!timerRunning.notEmpty());
	 cycleCount <= 0;
	 idleCount <= 0;
	 timerRunning.enq(True);
      endmethod
      method Action stopTimer();
	 timerRunning.deq();
	 timerInd.elapsedCycles(cycleCount, idleCount);
      endmethod
   endinterface

   interface MmRequest mmRequest;
      method Action mmf(Bit#(32) h1, Bit#(32) r1, Bit#(32) c1,
			Bit#(32) h2, Bit#(32) r2, Bit#(32) c2,
			Bit#(32) h3);
	 dmaMMF.start(h1, unpack(truncate(r1)), unpack(truncate(c1)),
		      h2, unpack(truncate(r2)), unpack(truncate(c2)),
		      h3);
	 mmfCycles <= 0;
	 busyFifo.enq(True);
      endmethod
   endinterface

   interface Vector readClients = dmaMMF.readClients;
   interface Vector writeClients =  dmaMMF.writeClients;

endmodule

// Copyright (c) 2014 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import FIFO::*;
import FIFOF::*;
import MIMO::*;
import DefaultValue::*;
import SpecialFIFOs::*;
import Vector::*;
import BRAM::*;
import DmaVector::*;
import PortalMemory::*;
import Dma::*;
import FloatingPoint::*;
import Pipe::*;
import FloatOps::*;
import Timer::*;
import RbmTypes::*;
import Assert::*;
import Connectable::*;

interface DotProdServer#(numeric type n);
   interface Reg#(UInt#(20)) numElts;
   interface Put#(Tuple2#(Vector#(n,Float),Vector#(n,Float))) request;
   interface PipeOut#(Float) pipe;
endinterface

(* synthesize *)
module [Module] mkDotProdServer#(UInt#(TLog#(TMul#(J,K))) label)(DotProdServer#(N));

   let n = valueOf(N);
   let add_depth = valueOf(FP_ADD_DEPTH);
   let mul_depth = valueOf(FP_MUL_DEPTH);
   Bool verbose = False; //label==0;

   Reg#(Bit#(TAdd#(TLog#(FP_ADD_DEPTH),1))) initCnt <- mkReg(0);
   FIFO#(void) initCtrl <- mkSizedFIFO(1);

   Reg#(UInt#(20)) numEltsReg <- mkReg(0);
   Reg#(UInt#(20)) countInReg <- mkReg(0);

   Vector#(N, FloatAlu#(FP_MUL_DEPTH)) muls <- replicateM(mkFloatMultiplier(defaultValue));
   Vector#(N, FloatAlu#(FP_ADD_DEPTH)) adders <- replicateM(mkFloatAdder(defaultValue));

   Vector#(N,FIFOF#(Tuple2#(Float,Float))) abfifos <- replicateM(mkFIFOF());
   Vector#(N,FIFOF#(Bool)) lastFifos <- replicateM(mkSizedFIFOF(mul_depth));

   Vector#(N, Reg#(Bit#(TAdd#(TLog#(FP_ADD_DEPTH),1)))) drainCnts <- replicateM(mkReg(0));
   Vector#(N, Reg#(Bool)) drained <- replicateM(mkReg(False));

   Reg#(Maybe#(Float)) accum <- mkReg(Nothing);
   FIFOF#(Float) dotfifo <- mkFIFOF;

   function Bit#(TLog#(N)) i_v(Integer i) = fromInteger(i);

   rule init if (initCnt > 0);
      for(Integer i = 0; i < n; i = i + 1)
	 adders[i].request.put(tuple2(0,0));
      initCnt <= initCnt - 1;
      if (verbose) $display($format(fshow("label=")+fshow(label)+fshow(" initCnt=")+fshow(initCnt)));
   endrule

   for (Integer i = 0; i < n; i = i + 1)
      rule mul;
	 // this rule could be folded into the 'put' method to reduce latency
	 match {.x,.y} <- toGet(abfifos[i]).get;
	 muls[i].request.put(tuple2(x, y));
	 if (verbose) $display($format(fshow("label=")+fshow(label)+fshow(" mul=")+fshow(i_v(i))));
      endrule

   for (Integer i = 0; i < n; i = i + 1)
      rule acc if (drainCnts[i] == 0 && initCnt == 0);
	 match {.resp,.*} <- muls[i].response.get();
	 match {.acc,.*} <- adders[i].response.get;
	 adders[i].request.put(tuple2(resp,acc));
	 let last <- toGet(lastFifos[i]).get;
	 if (last) begin
	    if (i>0)
	       drainCnts[i] <= fromInteger(add_depth);
	    else
	       drainCnts[i] <= fromInteger((add_depth*2)-1); // is this correct??
	 end
	 if (verbose) $display($format(fshow("label=")+fshow(label)+fshow(" acc=")+fshow(i_v(i))+fshow(" last=")+fshow(last)));
      endrule

   for (Integer i = 1; i < n; i = i + 1)
      rule gather if (drainCnts[i] > 0);
	 let new_cnt = drainCnts[i]-1;
	 drained[i] <= (new_cnt==0);
	 drainCnts[i] <= new_cnt;
	 match {.a,.*} <- adders[0].response.get;
	 match {.b,.*} <- adders[i].response.get;
	 adders[0].request.put(tuple2(a,b));
	 if (verbose) $display($format(fshow("label=")+fshow(label)+fshow(" gather=")+fshow(i_v(i))+fshow(" drainCnt=")+fshow(drainCnts[i])));
      endrule

   // the reference-guide says this should work, but it doesn't compile:
   // let gathered = and(tail(drained));
   function Bool is_true(Bool b) = b;
   let gathered = all(is_true, readVReg(tail(drained)));
   // this will only work correctly when add_depth is odd
   rule drain if (gathered && drainCnts[0] > 0);
      let new_cnt = drainCnts[0]-1;
      match {.a,.*} <- adders[0].response.get;
      drainCnts[0] <= new_cnt;
      let enq = False;
      if(accum matches tagged Valid .v) begin
	 adders[0].request.put(tuple2(a,v));
	 dynamicAssert(new_cnt > 0, "mkDotProdServer::drain");
	 enq = True;
	 accum <= tagged Invalid;
      end
      else begin
	 if (new_cnt == 0) begin
	    dotfifo.enq(a);
	    initCtrl.deq;
	 end
	 else
	    accum <= tagged Valid a;
      end
      if (verbose) $display($format(fshow("label=")+fshow(label)+fshow(" drain0=")+fshow(enq)+fshow(" drainCnt=")+fshow(drainCnts[0])));
   endrule

   PipeOut#(Float) dotpipe = toPipeOut(dotfifo);
   interface Put request;
      method Action put(Tuple2#(Vector#(N,Float),Vector#(N,Float)) tpl);
	 if (countInReg == 0) begin
	    initCtrl.enq(?);
	    initCnt <= fromInteger(add_depth);
	 end
	 let c = countInReg+fromInteger(n);
	 Bool isLast = (c >= numEltsReg);
	 if (isLast) begin
	    c = 0;
	 end
	 countInReg <= c;
	 match { .avec, .bvec } = tpl;
	 function Action enqvalues(Integer i);
	    action
	       abfifos[i].enq(tuple2(avec[i], bvec[i]));
	       lastFifos[i].enq(isLast);
	       //if (verbose) $display($format(fshow("label=")+fshow(label)+fshow(" countInReg=(")+fshow(countInReg)+fshow("/")+fshow(numEltsReg)+fshow(") dotprod x=") + fshow(avec[i]) + fshow(" y=") + fshow(bvec[i])));
	    endaction
	 endfunction
	 Vector#(N, Integer) indices = genVector();
	 mapM_(enqvalues, indices);
      endmethod
   endinterface : request
   interface PipeOut pipe = dotpipe;
   interface Reg numElts = numEltsReg;
endmodule : mkDotProdServer

function Vector#(TMul#(j,k), etype) flattenMatrix(Vector#(j, Vector#(k, etype)) mat);
   function etype flatten(Integer i); return mat[i/valueOf(k)][i%valueOf(k)]; endfunction
   return genWith(flatten);
endfunction

typedef struct {
   a xbase;
   a xlimit;
   a xstep;
   a ybase;
   a ylimit;
   a ystep;
} XYRangeConfig#(type a) deriving (Bits, FShow);

interface XYRangePipeIfc#(type a);
   interface PipeOut#(Tuple2#(a,a)) pipe;
   method Action start(XYRangeConfig#(a) cfg);
   method Action display();
endinterface

module mkXYRangePipeOut(XYRangePipeIfc#(a)) provisos (Arith#(a), Bits#(a,awidth), Eq#(a), Ord#(a));
   Reg#(a) x <- mkReg(0);
   Reg#(a) y <- mkReg(0);
   Reg#(a) xbase <- mkReg(0);
   Reg#(a) ybase <- mkReg(0);
   Reg#(a) xstep <- mkReg(0);
   Reg#(a) ystep <- mkReg(0);
   Reg#(a) xlimit <- mkReg(0);
   Reg#(a) ylimit <- mkReg(0);

   interface PipeOut pipe;
      method Tuple2#(a,a) first() if (x < xlimit && y < ylimit);
	 return tuple2(x,y);
      endmethod
      method Action deq if (x < xlimit && y < ylimit);
	 let newx = x;
	 let newy = y+ystep;
	 if (newy >= ylimit && x < xlimit) begin
	    newy = ybase;
	    newx = newx + xstep;
	 end
	 x <= newx;
	 y <= newy;
      endmethod
      method Bool notEmpty();
	 return (x < xlimit && y < ylimit);
      endmethod
   endinterface
   method Action start(XYRangeConfig#(a) cfg) if (x >= xlimit);
      //$display("XYRangePipe x=%d xlimit=%d xstep=%d y=%d ylimit=%d ystep=%d", cfg.xbase, cfg.xlimit, cfg.xstep, cfg.ybase, cfg.ylimit, cfg.ystep);
      x <= cfg.xbase;
      y <= cfg.ybase;
      xbase <= cfg.xbase;
      ybase <= cfg.ybase;
      xstep <= cfg.xstep;
      ystep <= cfg.ystep;
      xlimit <= cfg.xlimit;
      ylimit <= cfg.ylimit;
   endmethod
   method Action display();
      $display("XYRangePipe x=%d xlimit=%d y=%d ylimit=%d xstep=%d ystep=%d", x, xlimit, xstep, y, ylimit, ystep);
   endmethod
endmodule: mkXYRangePipeOut

typedef struct {
   ObjectPointer pointer;
   addrtype base;
   addrtype numRows;
   addrtype numColumns;
} MatrixDescriptor#(type addrtype) deriving (Bits);

// row major layout
interface DmaMatrixMultiplyIfc#(numeric type addrwidth, numeric type dsz);
   interface Vector#(J, ObjectWriteClient#(dsz)) writeClients;
   method Action start(ObjectPointer pointerA, UInt#(addrwidth) numRowsA, UInt#(addrwidth) numColumnsA,
		       ObjectPointer pointerB, UInt#(addrwidth) numRowsB, UInt#(addrwidth) numColumnsB,
		       ObjectPointer pointerC);
   method ActionValue#(Bool) finish();
endinterface

typedef enum {
   Idle, Ready, Running, Done
   } MMState deriving (Bits, Eq);

/*!
 * Multiplies two matrices A and B and writes the result to memory.
 * Fetches J rows at a time from A and K rows at a time from B.
 * Each cycle, it can fetch N elements of a row or column.
 *
 * Just considering memory bandwidth, every J+K cycles it is ready to perform J*K*N multiply accumulates.
 *
 */
module [Module] mkDmaMatrixMultiply#(Vector#(J, VectorSource#(dsz, Vector#(N, Float))) sourceA,
				     Vector#(K, VectorSource#(dsz, Vector#(N, Float))) sourceB,
				     function Module#(DmaVectorSink#(dsz, Vector#(N, Float))) mkSink(PipeOut#(Vector#(N, Float)) pipe_in)
				     )(DmaMatrixMultiplyIfc#(addrwidth, dsz))
   provisos (  Add#(N,n__,K)
	     , Mul#(N,m__,K)
	     , Add#(1,o__,J)
	     , Log#(N,nshift)
	     , FShow#(Float)
	     , Arith#(Float)
	     , Bits#(Vector#(N, Float), dsz)
	     , Bits#(MatrixDescriptor#(UInt#(addrwidth)), mdsz)
	     , Bits#(Tuple2#(UInt#(addrwidth), UInt#(addrwidth)), tplsz)
	     , Add#(b__, 20, addrwidth)
	     , Add#(a__, addrwidth, 40)
      );

   let n = valueOf(N);
   let jj = valueOf(J);
   let kk = valueOf(K);
   let nshift = valueOf(nshift);
   Bool verbose = False;
   Bool verbose1 = False;
   Bool timing = True;

   Reg#(UInt#(32)) cycles <- mkReg(0);
   Reg#(Bool) doneReg <- mkReg(False);
   Reg#(MatrixDescriptor#(UInt#(addrwidth))) descriptorA <- mkReg(unpack(0));
   Reg#(MatrixDescriptor#(UInt#(addrwidth))) descriptorB <- mkReg(unpack(0));
   Reg#(MatrixDescriptor#(UInt#(addrwidth))) descriptorC <- mkReg(unpack(0));
   Reg#(UInt#(addrwidth)) dotprodCount <- mkReg(0);

   Vector#(J, Vector#(K, PipeOut#(Vector#(N,Float)))) aPipes  <- mapM(mkSizedForkVector(valueOf(SourceBufferSize)), map(vectorSourcePipe, sourceA));
   Vector#(K, Vector#(J, PipeOut#(Vector#(N,Float)))) bPipesT <- mapM(mkForkVector, map(vectorSourcePipe, sourceB));
   Vector#(J, Vector#(K, PipeOut#(Vector#(N,Float)))) bPipes  =  transpose(bPipesT);

   rule countCycles;
      cycles <= cycles+1;
   endrule

//`define OLDWAY
`ifdef OLDWAY
   function Module#(DotProdServer#(N)) mkFxDotProd(Integer i);
      return mkDotProdServer(fromInteger(i));
   endfunction
   function Put#(Tuple2#(Vector#(n,Float),Vector#(n,Float))) getDotProdServerRequestInterface(DotProdServer#(n) s); return s.request; endfunction
   function PipeOut#(Float) getDotProdServerPipe(DotProdServer#(n) s); return s.pipe; endfunction
   Vector#(TMul#(J,K), DotProdServer#(N)) fxdotprods <- genWithM(mkFxDotProd);
   Vector#(TMul#(J,K), PipeOut#(Float)) fxpipes = map(getDotProdServerPipe, fxdotprods);

   for (Integer k = 0; k < kk; k = k+1) begin
      for (Integer j = 0; j < jj; j = j + 1) begin
	 rule connectDotProd;
	    let index = j*kk+k;
	    let a <- toGet(aPipes[j][k]).get();
	    let b <- toGet(bPipes[j][k]).get();
	    int jint = fromInteger(j);
	    int kint = fromInteger(k);
	    //$display($format(fshow("connectDotProd pos=")+fshow(tuple2(jint,kint))+fshow(" a=")+fshow(pack(a))+fshow(" b=")+fshow(pack(b))));
	    fxdotprods[index].request.put(tuple2(a, b));
	 endrule
      end
   end
`else
   FirstLastPipe#(UInt#(addrwidth)) firstLastPipe <- mkFirstLastPipe();
   Vector#(2, PipeOut#(Tuple2#(Bool,Bool))) firstLastPipes      <- mkForkVector(firstLastPipe.pipe);

   Vector#(TMul#(J,K), FIFOF#(Float))   accumFifos  <- replicateM(mkFIFOF);
   Vector#(TMul#(J,K), PipeOut#(Float)) accumPipes  =  map(toPipeOut, accumFifos);
   Vector#(M, PipeOut#(Float))          accumFunnels <- mkFunnelPipes(accumPipes);

   rule primeTheAccum;
      match { .first, .last } <- toGet(firstLastPipes[0]).get();
      if (first)
	 for (Integer i = 0; i < valueOf(TMul#(J,K)); i = i + 1)
	    accumFifos[i].enq(unpack(0));
   endrule

   Vector#(TMul#(J,K), PipeOut#(Vector#(1,Float))) a1Pipes <- mapM(mkFunnel, flattenMatrix(aPipes));
   Vector#(TMul#(J,K), PipeOut#(Vector#(1,Float))) b1Pipes <- mapM(mkFunnel, flattenMatrix(bPipes));
   Vector#(TMul#(J,K), PipeOut#(Tuple2#(Vector#(1,Float),Vector#(1,Float)))) ab1Pipes = map(uncurry(zipPipeOut), zip(a1Pipes, b1Pipes));

   Vector#(M, PipeOut#(Tuple2#(Vector#(1,Float),Vector#(1,Float)))) abFunnels <- mkFunnelPipes(ab1Pipes);
   Vector#(M, FloatAlu#(FP_MUL_DEPTH)) muls <- replicateM(mkFloatMultiplier(defaultValue));
   Vector#(M, FloatAlu#(FP_ADD_DEPTH)) adders <- replicateM(mkFloatAdder(defaultValue));
   rule mulreq;
      for (Integer i = 0; i < valueOf(M); i = i + 1) begin
	 match { .avec, .bvec } <- toGet(abFunnels[i]).get();
	 muls[i].request.put(tuple2(avec[0], bvec[0]));
      end
   endrule
   rule accumreq;
      for (Integer i = 0; i < valueOf(M); i = i + 1) begin
	 let accum <- toGet(accumFunnels[i]).get();
	 match {.resp, .*} <- muls[i].response.get();
	 adders[i].request.put(tuple2(resp, accum));
      end
   endrule
   FIFOF#(Vector#(M, Float))   accumOutFunnel     <- mkFIFOF();
   PipeOut#(Vector#(M, Float)) accumOutFunnelPipe =  toPipeOut(accumOutFunnel);
   rule accumresp;
      function ActionValue#(Float) getAccumResponse(FloatAlu#(depth) adder); actionvalue match {.resp, .*} <- adder.response.get(); return resp; endactionvalue endfunction
      Vector#(M,Float) accum <- mapM(getAccumResponse, adders);
      accumOutFunnel.enq(accum);
   endrule
   PipeOut#(Vector#(TMul#(J,K), (Float))) accumOutPipe <- mkUnfunnel(accumOutFunnelPipe);
   Vector#(TMul#(J,K), FIFOF#(Float)) resultFifos <- replicateM(mkFIFOF());
   rule feedback;
      let accumOut <- toGet(accumOutPipe).get();
      match { .first, .last } <- toGet(firstLastPipes[1]).get();
      for (Integer i = 0; i < valueOf(TMul#(J,K)); i = i + 1)
	 if (last)
	    resultFifos[i].enq(accumOut[i]);
	 else
	    accumFifos[i].enq(accumOut[i]);
   endrule
   Vector#(TMul#(J,K), PipeOut#(Float)) fxpipes = map(toPipeOut, resultFifos);

`endif
   MIMOConfiguration mimoCfg = defaultValue;
   MIMO#(K,N,TAdd#(K,N),Float) dfifo <- mkMIMO(mimoCfg);
   Vector#(J, MIMO#(K,N,TAdd#(K,N),Float)) dfifos <- replicateM(mkMIMO(mimoCfg));
   let sinks <- mapM(mkSink, map(toPipeOut, dfifos));

   XYRangePipeIfc#(UInt#(addrwidth)) indexpipeifc <- mkXYRangePipeOut();
   XYRangePipeIfc#(UInt#(addrwidth)) offsetpipeA <- mkXYRangePipeOut();
   XYRangePipeIfc#(UInt#(addrwidth)) offsetpipeB <- mkXYRangePipeOut();
   XYRangePipeIfc#(UInt#(addrwidth)) offsetpipeC <- mkXYRangePipeOut();

   Vector#(TAdd#(TAdd#(J,J),K), PipeOut#(Tuple2#(UInt#(addrwidth),UInt#(addrwidth)))) indexpipes <- mkSizedForkVector(valueOf(SourceBufferSize), indexpipeifc.pipe);
   Vector#(J, PipeOut#(Tuple2#(UInt#(addrwidth),UInt#(addrwidth))))        offsetpipesA <- mkSizedForkVector(valueOf(SourceBufferSize), offsetpipeA.pipe);
   Vector#(K, PipeOut#(Tuple2#(UInt#(addrwidth),UInt#(addrwidth))))        offsetpipesB <- mkSizedForkVector(valueOf(SourceBufferSize), offsetpipeB.pipe);
   Vector#(J, PipeOut#(Tuple2#(UInt#(addrwidth),UInt#(addrwidth))))        offsetpipesC <- mkSizedForkVector(valueOf(SourceBufferSize), offsetpipeC.pipe);

   Reg#(Bool) running <- mkReg(False);
   FIFOF#(Bool) doneFifo <- mkFIFOF();

   Vector#(J, Reg#(UInt#(addrwidth))) startAOffset <- replicateM(mkReg(0));
   Vector#(K, Reg#(UInt#(addrwidth))) startBOffset <- replicateM(mkReg(0));
   Vector#(J, Reg#(UInt#(addrwidth))) startCOffset <- replicateM(mkReg(0));
   for (Integer k = 0; k < kk; k = k + 1) begin
      rule startDotProds;
	 Tuple2#(UInt#(addrwidth),UInt#(addrwidth)) index <- toGet(indexpipes[k]).get();
	 match { .unusedB, .startBBase } <- toGet(offsetpipesB[k]).get();

	 int kint = fromInteger(k);

	 let row = tpl_1(index);
	 let col = tpl_2(index)+fromInteger(k);

	 let startB = startBBase + startBOffset[k];

	 if (timing || verbose) $display($format(fshow(cycles)+fshow("    startB index=")+fshow(tuple2(row,col))
	    +fshow(" startB=")+fshow(startB)
	    +fshow(" k=")+fshow(kint)));

	 if (verbose || verbose1) $display($format(fshow(cycles)+fshow("    sourceB[")+fshow(kint)+fshow("].start")+fshow(startB)));

	 sourceB[k].start(descriptorB.pointer, pack(extend(startB>>nshift)), pack(extend(descriptorB.numColumns>>nshift)));
      endrule
      rule finishSourceB;
	 UInt#(TLog#(K)) in = fromInteger(k);
	 int kint = fromInteger(k);
	 if (verbose || verbose1) $display($format(fshow(cycles)+fshow("    sourceB[")+fshow(kint)+fshow("].finish")));
	 let b <- sourceB[k].finish();
      endrule
   end
   for (Integer j = 0; j < jj; j = j + 1) begin

      rule startSourceAndSink;
	 Tuple2#(UInt#(addrwidth),UInt#(addrwidth)) index <- toGet(indexpipes[j+kk]).get();

	 let row = tpl_1(index)+fromInteger(j);
	 let col = tpl_2(index);

	 match { .startABase, .unusedA } <- toGet(offsetpipesA[j]).get();
	 match { .startCBase, .offsetC } <- toGet(offsetpipesC[j]).get();
	 let startA = startABase + startAOffset[j];
	 let startC = startCBase + startCOffset[j] + offsetC;

	 int jint = fromInteger(j);
	 if (timing || verbose) $display($format(fshow(cycles)+fshow("    start A/C index=")+fshow(tuple2(row,col))
	    +fshow(" startA=")+fshow(startA)
	    +fshow(" startC=")+fshow(startC)
	    +fshow(" startC>>nshift=")+fshow(startC>>nshift)
	    +fshow(" j=")+fshow(jint)));

	 sourceA[j].start(descriptorA.pointer, pack(extend(startA>>nshift)), pack(extend(descriptorA.numColumns>>nshift)));
	 if (verbose || verbose1) $display($format(fshow(cycles)+fshow("    sourceA[")+fshow(jint)+fshow("].start")+fshow(startA)));
	 sinks[j].vector.start(descriptorC.pointer, pack(extend(startC>>nshift)), fromInteger(kk/n));
	 if (verbose || verbose1) $display($format(fshow(cycles)+fshow("      sinks[")+fshow(jint)+fshow("].start")+fshow(startC)));

	 // this one counts by one rather than 1/n because the source values are unfunneled before mul/add
	 if (j == 0)
	    firstLastPipe.start(truncate(descriptorA.numColumns));
      endrule

      rule finishSourceA;
	 if (verbose || verbose1) $display($format(fshow(cycles)+fshow("    sourceA[0].finish ")));
	 let b <- sourceA[j].finish();
      endrule

      rule dotProdValue;
	 Tuple2#(UInt#(addrwidth),UInt#(addrwidth)) index <- toGet(indexpipes[jj+kk+j]).get();
	 Vector#(K,Float) vs;
	 for (Integer k = 0; k < kk; k = k + 1) begin
	    let dpnum = j*kk+k;
	    let v <- toGet(fxpipes[dpnum]).get();
	    //let indexi = tuple2(tpl_1(index), tpl_2(index)+fromInteger(k));
	    //if (verbose) $display($format(fshow(cycles)+fshow("    dotprodvalue index=")+fshow(indexi)+fshow(" dotprod=")+fshow(v)));
	    int jint = fromInteger(j);
	    int kint = fromInteger(k);
	    if (verbose1) $display($format(fshow(cycles)+fshow("    dotprodvalue pos=")+fshow(index)+fshow(tuple2(jint,kint))+fshow(" dotprod=")+fshow(pack(v))));
	    vs[k] = v;
	 end
	 dfifos[j].enq(fromInteger(kk), vs);
      endrule

      rule sinkDone;
	 // each time we write a burst of k values via sinks
	 //let index <- toGet(indexpipes[jj+kk+1]).get();
	 let b <- sinks[j].vector.finish();
	 let c = dotprodCount-fromInteger(kk);
	 int jint = fromInteger(j);
	 if (verbose1) $display($format(fshow(cycles)+fshow("    sinkDone c")+fshow(c)+fshow(" j=")+fshow(jint)));
	 dotprodCount <= c;
	 if (c == 0) begin
	    running <= False;
	    doneFifo.enq(?);
	 end
      endrule
   end

   method Action start(ObjectPointer pointerA, UInt#(addrwidth) numRowsA, UInt#(addrwidth) numColumnsA,
		       ObjectPointer pointerB, UInt#(addrwidth) numRowsB, UInt#(addrwidth) numColumnsB,
		       ObjectPointer pointerC) if (!running);
      XYRangeConfig#(UInt#(addrwidth)) indexcfg  = XYRangeConfig {xbase: 0, xlimit: numRowsA, xstep: fromInteger(jj),
								  ybase: 0, ylimit: numRowsB, ystep: fromInteger(kk) };
      XYRangeConfig#(UInt#(addrwidth)) offsetcfgA = XYRangeConfig {xbase: 0, xlimit: numRowsA*numColumnsA, xstep: numColumnsA*fromInteger(jj),
								  ybase: 0, ylimit: numRowsB, ystep: fromInteger(kk) };
      XYRangeConfig#(UInt#(addrwidth)) offsetcfgB = XYRangeConfig {xbase: 0, xlimit: numRowsA, xstep: fromInteger(jj),
								  ybase: 0, ylimit: numRowsB*numColumnsB, ystep: fromInteger(kk)*numColumnsB };
      XYRangeConfig#(UInt#(addrwidth)) offsetcfgC = XYRangeConfig {xbase: 0, xlimit: numRowsA*numRowsB, xstep: numRowsB*fromInteger(jj),
								  ybase: 0, ylimit: numRowsB, ystep: fromInteger(kk) };
      descriptorA <= MatrixDescriptor { pointer: pointerA, base: 0, numRows: numRowsA, numColumns: numColumnsA};
      descriptorB <= MatrixDescriptor { pointer: pointerB, base: 0, numRows: numRowsB, numColumns: numColumnsB};
      descriptorC <= MatrixDescriptor { pointer: pointerC, base: 0, numRows: numRowsA, numColumns: numRowsB};
      dotprodCount <= numRowsA*numRowsB;
      running <= True;

      if (verbose) $display("mm pointerA=%d pointerB=%d pointerC=%d\n", pointerA, pointerB, pointerC);
      if (verbose) $display("mm.start ra=%d ca=%d rb=%d cb=%d dotprodCount=%d", numRowsA, numColumnsA, numRowsB, numColumnsB, dotprodCount);
      if (verbose) $display($format(fshow("mm.start ")+fshow(indexcfg)));
      indexpipeifc.start(indexcfg);
      offsetpipeA.start(offsetcfgA);
      offsetpipeB.start(offsetcfgB);
      offsetpipeC.start(offsetcfgC);
      for (Integer j = 0; j < jj; j = j + 1) begin
	 startAOffset[j] <= fromInteger(j)*numColumnsA;
	 startCOffset[j] <= fromInteger(j)*numRowsB;
      end
      for (Integer k = 0; k < kk; k = k + 1) begin
	 for (Integer j = 0; j < jj; j = j + 1) begin
	    let index = j*kk + k;
`ifdef OLDWAY
	    fxdotprods[index].numElts <= truncate(numColumnsA);
`endif
	 end
	 startBOffset[k] <= fromInteger(k)*numColumnsB;
      end
   endmethod
   method ActionValue#(Bool) finish();
      if (verbose) $display("mm.finish()");
      doneFifo.deq();
      return True;
   endmethod

   interface Vector writeClients = map(getSinkWriteClient, sinks);
endmodule

typedef 20 MMSize;

interface DramMatrixMultiply#(numeric type n, numeric type dmasz);
   interface Vector#(TAdd#(K,J), ObjectReadClient#(dmasz)) readClients;
   interface Vector#(J, ObjectWriteClient#(dmasz)) writeClients;
   method Action start(ObjectPointer pointerA, UInt#(MMSize) numRowsA, UInt#(MMSize) numColumnsA,
		       ObjectPointer pointerB, UInt#(MMSize) numRowsB, UInt#(MMSize) numColumnsB,
		       ObjectPointer pointerC);
   method ActionValue#(Bool) finish();
   method Bit#(32) dbg();
endinterface

(* synthesize *)
module [Module] mkDramMatrixMultiply(DramMatrixMultiply#(N,TMul#(N,32)));
   Vector#(TAdd#(K,J), DmaVectorSource#(DmaSz, Vector#(N,Float))) vfsources <- replicateM(mkDmaVectorSource());
   Vector#(J, VectorSource#(DmaSz, Vector#(N,Float))) xvfsources = takeAt(0,          map(dmaVectorSourceVector, vfsources));
   Vector#(K, VectorSource#(DmaSz, Vector#(N,Float))) yvfsources = takeAt(valueOf(J), map(dmaVectorSourceVector, vfsources));
   DmaMatrixMultiplyIfc#(MMSize,DmaSz) dmaMMF <- mkDmaMatrixMultiply(xvfsources, yvfsources, mkDmaVectorSink);
   interface Vector readClients = map(getSourceReadClient, vfsources);
   interface Vector writeClients = dmaMMF.writeClients;
   method start = dmaMMF.start;
   method finish = dmaMMF.finish;
   method Bit#(32) dbg();
      Bit#(32) d = 0;
      d[0] = pack(vfsources[0].vector.pipe.notEmpty());
      d[1] = pack(vfsources[1].vector.pipe.notEmpty());
      return d;
   endmethod
endmodule

interface Mm#(numeric type n);
   interface MmRequest mmRequest;
   interface TimerRequest timerRequest;
   interface Vector#(TAdd#(K,J), ObjectReadClient#(TMul#(32,N))) readClients;
   interface Vector#(J, ObjectWriteClient#(TMul#(32,n))) writeClients;
endinterface

module [Module] mkMm#(MmIndication ind, TimerIndication timerInd)(Mm#(N))
   provisos (Add#(1,a__,N),
	     Add#(N,0,n),
	     Mul#(N,32,DmaSz)
      );

   let n = valueOf(n);

   DramMatrixMultiply#(N, TMul#(N,32)) dmaMMF <- mkDramMatrixMultiply();

   Reg#(Bit#(64)) mmfCycles <- mkReg(0);
   rule countMmfCycles;
      mmfCycles <= mmfCycles + 1;
   endrule

   FIFOF#(Bool) busyFifo <- mkFIFOF();
   rule mmfDone;
      let d <- dmaMMF.finish();
      busyFifo.deq();
      ind.mmfDone(mmfCycles);
   endrule

   FIFOF#(Bool) timerRunning <- mkFIFOF();
   Reg#(Bit#(64)) cycleCount <- mkReg(0);
   Reg#(Bit#(64)) idleCount <- mkReg(0);
   rule countCycles if (timerRunning.notEmpty());
      cycleCount <= cycleCount + 1;
      if (!busyFifo.notEmpty())
	 idleCount <= idleCount + 1;
   endrule

   interface TimerRequest timerRequest;
      method Action startTimer() if (!timerRunning.notEmpty());
	 cycleCount <= 0;
	 idleCount <= 0;
	 timerRunning.enq(True);
      endmethod
      method Action stopTimer();
	 timerRunning.deq();
	 timerInd.elapsedCycles(cycleCount, idleCount);
      endmethod
   endinterface

   interface MmRequest mmRequest;
      method Action mmf(Bit#(32) h1, Bit#(32) r1, Bit#(32) c1,
			Bit#(32) h2, Bit#(32) r2, Bit#(32) c2,
			Bit#(32) h3);
	 dmaMMF.start(h1, unpack(truncate(r1)), unpack(truncate(c1)),
		      h2, unpack(truncate(r2)), unpack(truncate(c2)),
		      h3);
	 mmfCycles <= 0;
	 busyFifo.enq(True);
      endmethod
   endinterface

   interface Vector readClients = dmaMMF.readClients;
   interface Vector writeClients =  dmaMMF.writeClients;

endmodule