foo -> selectMMUPort

foo -> selectMMUPort

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
import FIFO::*;
import Vector::*;
import List::*;
import GetPut::*;
import ClientServer::*;
import Assert::*;
import StmtFSM::*;
import SpecialFIFOs::*;
import Connectable::*;
import HostInterface::*;
import MemTypes::*;
import ConnectalMemory::*;
import MMU::*;
import MemServerInternal::*;

interface MemServer#(numeric type addrWidth, numeric type busWidth, numeric type nMasters);
   interface MemServerRequest request;
   interface Vector#(nMasters,PhysMemMaster#(addrWidth, busWidth)) masters;
endinterface

interface MemServerWithMMU#(numeric type addrWidth, numeric type busWidth, numeric type nMasters);
   interface MemServerRequest memServerRequest;
   interface Vector#(nMasters,PhysMemMaster#(addrWidth, busWidth)) masters;
   interface MMURequest mmuRequest;
endinterface

interface MemServerRead#(numeric type addrWidth, numeric type busWidth, numeric type numClients, numeric type numServers);
   interface MemServerRequest request;
   interface Vector#(numClients, PhysMemReadClient#(addrWidth,busWidth)) clients;
   interface Vector#(numServers, MemReadServer#(busWidth)) servers;
endinterface

interface MemServerWrite#(numeric type addrWidth, numeric type busWidth, numeric type numClients, numeric type numServers);
   interface MemServerRequest request;
   interface Vector#(numClients, PhysMemWriteClient#(addrWidth,busWidth)) clients;
   interface Vector#(numServers, MemWriteServer#(busWidth)) servers;
endinterface

typedef struct {
   DmaErrorType errorType;
   Bit#(32) pref;
   } DmaError deriving (Bits);

module mkMemServer#(Vector#(numReadClients, MemReadClient#(busWidth)) readClients,
		    Vector#(numWriteClients, MemWriteClient#(busWidth)) writeClients,
		    Vector#(numMMUs,MMU#(addrWidth)) mmus,
		    MemServerIndication indication)
   (MemServer#(addrWidth, busWidth, nMasters))
   provisos(Mul#(TDiv#(numWriteClients, nMasters),nMasters,nws)
	    ,Mul#(TDiv#(numReadClients, nMasters),nMasters,nrs)
	    ,Add#(TLog#(TDiv#(busWidth, 8)), d__, 8)
	    ,Add#(TLog#(TDiv#(busWidth, 8)), e__, BurstLenSize)
	    ,Add#(c__, addrWidth, 64)
	    ,Add#(numWriteClients, a__, nws)
	    ,Add#(numReadClients, b__, nrs)
	    );

   MemServerRead#(addrWidth,busWidth,nMasters,nrs)  reader <- mkMemServerRead(indication, mmus);
   MemServerWrite#(addrWidth,busWidth,nMasters,nws) writer <- mkMemServerWrite(indication, mmus);

   zipWithM(mkConnection,readClients,take(reader.servers));
   zipWithM(mkConnection,writeClients,take(writer.servers));

   function PhysMemMaster#(addrWidth,busWidth) mkm(Integer i) = (interface PhysMemMaster#(addrWidth,busWidth);
		     interface PhysMemReadClient read_client = reader.clients[i];
		     interface PhysMemWriteClient write_client = writer.clients[i];
		  endinterface);

   interface MemServerRequest request;
      method Action setTileState(TileControl tc);
	 reader.request.setTileState(tc);
	 writer.request.setTileState(tc);
      endmethod
      method Action stateDbg(ChannelType rc);
	 if (rc == ChannelType_Read)
	    reader.request.stateDbg(rc);
	 else
	    writer.request.stateDbg(rc);
      endmethod
      method Action memoryTraffic(ChannelType rc);
	 if (rc == ChannelType_Read)
	    reader.request.memoryTraffic(rc);
	 else
	    writer.request.memoryTraffic(rc);
      endmethod
      method Action addrTrans(Bit#(32) pointer, Bit#(32) offset);
	 writer.request.addrTrans(pointer,offset);
      endmethod
   endinterface
   interface masters = map(mkm,genVector);
endmodule

module mkMemServerRead#(MemServerIndication indication,
			Vector#(numMMUs,MMU#(addrWidth)) mmus)
   (MemServerRead#(addrWidth, busWidth, numClients, numServers))
   provisos(Mul#(nrc, numClients, numServers)
	    ,Add#(a__, addrWidth, 64)
	    ,Add#(TLog#(TDiv#(busWidth, 8)), b__, 8)
	    ,Add#(TLog#(TDiv#(busWidth, 8)), c__, BurstLenSize)
	    );

   FIFO#(Bit#(32))   addrReqFifo <- mkFIFO;
   Reg#(Bit#(8)) dbgPtr <- mkReg(0);
   Reg#(Bit#(8)) trafficPtr <- mkReg(0);
   Reg#(Bit#(64)) trafficAccum <- mkReg(0);

   function Server#(MMURequest,Bit#(addrWidth)) selectMMUPort(Integer i);
      return mmus[i].addr[0];
   endfunction
   Vector#(numMMUs,ArbitratedMMU#(addrWidth,numClients)) mmu_servers <- mapM(mkArbitratedMMU,map(genVector));
   Vector#(numClients,MemReadInternal#(addrWidth,busWidth,MemServerTags,nrc)) readers;
   Vector#(numClients, PhysMemReadClient#(addrWidth,busWidth)) read_clients;
   Vector#(numServers, MemReadServer#(busWidth)) read_servers;

   for(Integer i = 0; i < valueOf(numClients); i = i+1) begin
      Vector#(numMMUs,Server#(ReqTup,Bit#(addrWidth))) ss;
      for(Integer j = 0; j < valueOf(numMMUs); j=j+1)
	 ss[j] = mmu_servers[j].servers[i];
      readers[i] <- mkMemReadInternal(indication,ss);
      read_clients[i] = readers[i].client;
      for(Integer j = 0; j < valueOf(nrc); j=j+1)
	 read_servers[i*valueOf(nrc)+j] = readers[i].servers[j];
   end

   rule mmuEntry;
      addrReqFifo.deq;
      let physAddr <- mmus[addrReqFifo.first[31:16]].addr[0].response.get;
      indication.addrResponse(zeroExtend(physAddr));
   endrule

   Stmt dbgStmt =
   seq
      for(dbgPtr <= 0; dbgPtr < fromInteger(valueOf(numClients)); dbgPtr <= dbgPtr+1)
	 (action
	     let rv <- readers[dbgPtr].dbg.dbg;
	     indication.reportStateDbg(rv);
	  endaction);
   endseq;
   FSM dbgFSM <- mkFSM(dbgStmt);

   Stmt trafficStmt =
   seq
      trafficAccum <= 0;
      for(trafficPtr <= 0; trafficPtr < fromInteger(valueOf(numClients)); trafficPtr <= trafficPtr+1)
	 (action
	     let rv <- readers[trafficPtr].dbg.getMemoryTraffic();
	     trafficAccum <= trafficAccum + rv;
	  endaction);
      indication.reportMemoryTraffic(trafficAccum);
   endseq;
   FSM trafficFSM <- mkFSM(trafficStmt);

   interface servers = read_servers;
   interface clients = read_clients;
   interface MemServerRequest request;
      method Action setTileState(TileControl tc);
	 for(Integer i = 0; i < valueOf(numClients); i=i+1)
	    readers[i].tileControl.put(tc);
      endmethod
      method Action stateDbg(ChannelType rc);
	 if (rc == ChannelType_Read)
	    dbgFSM.start;
      endmethod
      method Action memoryTraffic(ChannelType rc);
	 if (rc == ChannelType_Read)
	    trafficFSM.start;
      endmethod
      method Action addrTrans(Bit#(32) pointer, Bit#(32) offset);
	 addrReqFifo.enq(pointer);
	 mmus[pointer[31:16]].addr[0].request.put(ReqTup{id:truncate(pointer), off:extend(offset)});
      endmethod
   endinterface
endmodule

module mkMemServerWrite#(MemServerIndication indication,
		     Vector#(numMMUs,MMU#(addrWidth)) mmus)
   (MemServerWrite#(addrWidth, busWidth, numClients, numServers))
   provisos(Mul#(nwc, numClients, numServers)
	    ,Add#(a__, addrWidth, 64)
	    ,Add#(TLog#(TDiv#(busWidth, 8)), b__, 8)
	    ,Add#(TLog#(TDiv#(busWidth, 8)), c__, BurstLenSize)
	    );

   FIFO#(Bit#(32))   addrReqFifo <- mkFIFO;
   Reg#(Bit#(8)) dbgPtr <- mkReg(0);
   Reg#(Bit#(8)) trafficPtr <- mkReg(0);
   Reg#(Bit#(64)) trafficAccum <- mkReg(0);

   function Server#(MMURequest,Bit#(addrWidth)) selectMMUPort(Integer i);
      return mmus[i].addr[1];
   endfunction
   Vector#(numMMUs,ArbitratedMMU#(addrWidth,numClients)) mmu_servers <- mapM(mkArbitratedMMU,map(genVector));
   Vector#(numClients,MemWriteInternal#(addrWidth,busWidth,MemServerTags,nwc)) writers;
   Vector#(numClients, PhysMemWriteClient#(addrWidth,busWidth)) write_clients;
   Vector#(numServers, MemWriteServer#(busWidth)) write_servers;

   for(Integer i = 0; i < valueOf(numClients); i = i+1) begin
      Vector#(numMMUs,Server#(ReqTup,Bit#(addrWidth))) ss;
      for(Integer j = 0; j < valueOf(numMMUs); j=j+1)
	 ss[j] = mmu_servers[j].servers[i];
      writers[i] <- mkMemWriteInternal(indication, ss);
      write_clients[i] = writers[i].client;
      for(Integer j = 0; j < valueOf(nwc); j=j+1)
	 write_servers[i*valueOf(nwc)+j] = writers[i].servers[j];
   end

   rule mmuEntry;
      addrReqFifo.deq;
      let physAddr <- mmus[addrReqFifo.first[31:16]].addr[1].response.get;
      indication.addrResponse(zeroExtend(physAddr));
   endrule

   Stmt dbgStmt =
   seq
      for(dbgPtr <= 0; dbgPtr < fromInteger(valueOf(numClients)); dbgPtr <= dbgPtr+1)
	 (action
	     let rv <- writers[dbgPtr].dbg.dbg;
	     indication.reportStateDbg(rv);
	  endaction);
   endseq;
   FSM dbgFSM <- mkFSM(dbgStmt);

   Stmt trafficStmt =
   seq
      trafficAccum <= 0;
      for(trafficPtr <= 0; trafficPtr < fromInteger(valueOf(numClients)); trafficPtr <= trafficPtr+1)
	 (action
	     let rv <- writers[trafficPtr].dbg.getMemoryTraffic();
	     trafficAccum <= trafficAccum + rv;
	  endaction);
      indication.reportMemoryTraffic(trafficAccum);
   endseq;
   FSM trafficFSM <- mkFSM(trafficStmt);

   interface servers = write_servers;
   interface clients = write_clients;
   interface MemServerRequest request;
      method Action setTileState(TileControl tc);
	 for(Integer i = 0; i < valueOf(numClients); i=i+1)
	    writers[i].tileControl.put(tc);
      endmethod
      method Action stateDbg(ChannelType rc);
	 if (rc == ChannelType_Write)
	    dbgFSM.start;
      endmethod
      method Action memoryTraffic(ChannelType rc);
	 if (rc == ChannelType_Write)
	    trafficFSM.start;
      endmethod
      method Action addrTrans(Bit#(32) pointer, Bit#(32) offset);
	 addrReqFifo.enq(pointer);
	 mmus[pointer[31:16]].addr[1].request.put(ReqTup{id:truncate(pointer), off:extend(offset)});
      endmethod
   endinterface
endmodule

module mkMemServerWithMMU#(Vector#(numReadClients, MemReadClient#(busWidth)) readClients,
			  Vector#(numWriteClients, MemWriteClient#(busWidth)) writeClients,
			  MemServerIndication indication,
			  MMUIndication mmuIndication)(MemServerWithMMU#(addrWidth, busWidth,nMasters))

   provisos(Add#(TLog#(TDiv#(busWidth, 8)), e__, 8)
	    ,Add#(TLog#(TDiv#(busWidth, 8)), f__, BurstLenSize)
	    ,Add#(c__, addrWidth, 64)
	    ,Add#(d__, addrWidth, MemOffsetSize)
	    ,Add#(numWriteClients, a__, TMul#(TDiv#(numWriteClients, nMasters),nMasters))
	    ,Add#(numReadClients, b__, TMul#(TDiv#(numReadClients, nMasters),nMasters))
	    );

   MMU#(addrWidth) hostMMU <- mkMMU(0, True, mmuIndication);
   MemServer#(addrWidth,busWidth,nMasters) dma <- mkMemServer(readClients, writeClients, cons(hostMMU,nil), indication);

   interface MemServerRequest memServerRequest = dma.request;
   interface MMURequest mmuRequest = hostMMU.request;
   interface Vector masters = dma.masters;
endmodule

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
import FIFO::*;
import Vector::*;
import List::*;
import GetPut::*;
import ClientServer::*;
import Assert::*;
import StmtFSM::*;
import SpecialFIFOs::*;
import Connectable::*;
import HostInterface::*;
import MemTypes::*;
import ConnectalMemory::*;
import MMU::*;
import MemServerInternal::*;

interface MemServer#(numeric type addrWidth, numeric type busWidth, numeric type nMasters);
   interface MemServerRequest request;
   interface Vector#(nMasters,PhysMemMaster#(addrWidth, busWidth)) masters;
endinterface

interface MemServerWithMMU#(numeric type addrWidth, numeric type busWidth, numeric type nMasters);
   interface MemServerRequest memServerRequest;
   interface Vector#(nMasters,PhysMemMaster#(addrWidth, busWidth)) masters;
   interface MMURequest mmuRequest;
endinterface

interface MemServerRead#(numeric type addrWidth, numeric type busWidth, numeric type numClients, numeric type numServers);
   interface MemServerRequest request;
   interface Vector#(numClients, PhysMemReadClient#(addrWidth,busWidth)) clients;
   interface Vector#(numServers, MemReadServer#(busWidth)) servers;
endinterface

interface MemServerWrite#(numeric type addrWidth, numeric type busWidth, numeric type numClients, numeric type numServers);
   interface MemServerRequest request;
   interface Vector#(numClients, PhysMemWriteClient#(addrWidth,busWidth)) clients;
   interface Vector#(numServers, MemWriteServer#(busWidth)) servers;
endinterface

typedef struct {
   DmaErrorType errorType;
   Bit#(32) pref;
   } DmaError deriving (Bits);

module mkMemServer#(Vector#(numReadClients, MemReadClient#(busWidth)) readClients,
		    Vector#(numWriteClients, MemWriteClient#(busWidth)) writeClients,
		    Vector#(numMMUs,MMU#(addrWidth)) mmus,
		    MemServerIndication indication)
   (MemServer#(addrWidth, busWidth, nMasters))
   provisos(Mul#(TDiv#(numWriteClients, nMasters),nMasters,nws)
	    ,Mul#(TDiv#(numReadClients, nMasters),nMasters,nrs)
	    ,Add#(TLog#(TDiv#(busWidth, 8)), d__, 8)
	    ,Add#(TLog#(TDiv#(busWidth, 8)), e__, BurstLenSize)
	    ,Add#(c__, addrWidth, 64)
	    ,Add#(numWriteClients, a__, nws)
	    ,Add#(numReadClients, b__, nrs)
	    );

   MemServerRead#(addrWidth,busWidth,nMasters,nrs)  reader <- mkMemServerRead(indication, mmus);
   MemServerWrite#(addrWidth,busWidth,nMasters,nws) writer <- mkMemServerWrite(indication, mmus);

   zipWithM(mkConnection,readClients,take(reader.servers));
   zipWithM(mkConnection,writeClients,take(writer.servers));

   function PhysMemMaster#(addrWidth,busWidth) mkm(Integer i) = (interface PhysMemMaster#(addrWidth,busWidth);
		     interface PhysMemReadClient read_client = reader.clients[i];
		     interface PhysMemWriteClient write_client = writer.clients[i];
		  endinterface);

   interface MemServerRequest request;
      method Action setTileState(TileControl tc);
	 reader.request.setTileState(tc);
	 writer.request.setTileState(tc);
      endmethod
      method Action stateDbg(ChannelType rc);
	 if (rc == ChannelType_Read)
	    reader.request.stateDbg(rc);
	 else
	    writer.request.stateDbg(rc);
      endmethod
      method Action memoryTraffic(ChannelType rc);
	 if (rc == ChannelType_Read)
	    reader.request.memoryTraffic(rc);
	 else
	    writer.request.memoryTraffic(rc);
      endmethod
      method Action addrTrans(Bit#(32) pointer, Bit#(32) offset);
	 writer.request.addrTrans(pointer,offset);
      endmethod
   endinterface
   interface masters = map(mkm,genVector);
endmodule

module mkMemServerRead#(MemServerIndication indication,
			Vector#(numMMUs,MMU#(addrWidth)) mmus)
   (MemServerRead#(addrWidth, busWidth, numClients, numServers))
   provisos(Mul#(nrc, numClients, numServers)
	    ,Add#(a__, addrWidth, 64)
	    ,Add#(TLog#(TDiv#(busWidth, 8)), b__, 8)
	    ,Add#(TLog#(TDiv#(busWidth, 8)), c__, BurstLenSize)
	    );

   FIFO#(Bit#(32))   addrReqFifo <- mkFIFO;
   Reg#(Bit#(8)) dbgPtr <- mkReg(0);
   Reg#(Bit#(8)) trafficPtr <- mkReg(0);
   Reg#(Bit#(64)) trafficAccum <- mkReg(0);

   module foo#(Integer i) (ArbitratedMMU#(addrWidth,numClients));
      let rv <- mkArbitratedMMU(mmus[i].addr[0]);
      return rv;
   endmodule
   Vector#(numMMUs,ArbitratedMMU#(addrWidth,numClients)) mmu_servers <- mapM(foo,genVector);
   Vector#(numClients,MemReadInternal#(addrWidth,busWidth,MemServerTags,nrc)) readers;
   Vector#(numClients, PhysMemReadClient#(addrWidth,busWidth)) read_clients;
   Vector#(numServers, MemReadServer#(busWidth)) read_servers;

   for(Integer i = 0; i < valueOf(numClients); i = i+1) begin
      Vector#(numMMUs,Server#(ReqTup,Bit#(addrWidth))) ss;
      for(Integer j = 0; j < valueOf(numMMUs); j=j+1)
	 ss[j] = mmu_servers[j].servers[i];
      readers[i] <- mkMemReadInternal(indication,ss);
      read_clients[i] = readers[i].client;
      for(Integer j = 0; j < valueOf(nrc); j=j+1)
	 read_servers[i*valueOf(nrc)+j] = readers[i].servers[j];
   end

   rule mmuEntry;
      addrReqFifo.deq;
      let physAddr <- mmus[addrReqFifo.first[31:16]].addr[0].response.get;
      indication.addrResponse(zeroExtend(physAddr));
   endrule

   Stmt dbgStmt =
   seq
      for(dbgPtr <= 0; dbgPtr < fromInteger(valueOf(numClients)); dbgPtr <= dbgPtr+1)
	 (action
	     let rv <- readers[dbgPtr].dbg.dbg;
	     indication.reportStateDbg(rv);
	  endaction);
   endseq;
   FSM dbgFSM <- mkFSM(dbgStmt);

   Stmt trafficStmt =
   seq
      trafficAccum <= 0;
      for(trafficPtr <= 0; trafficPtr < fromInteger(valueOf(numClients)); trafficPtr <= trafficPtr+1)
	 (action
	     let rv <- readers[trafficPtr].dbg.getMemoryTraffic();
	     trafficAccum <= trafficAccum + rv;
	  endaction);
      indication.reportMemoryTraffic(trafficAccum);
   endseq;
   FSM trafficFSM <- mkFSM(trafficStmt);

   interface servers = read_servers;
   interface clients = read_clients;
   interface MemServerRequest request;
      method Action setTileState(TileControl tc);
	 for(Integer i = 0; i < valueOf(numClients); i=i+1)
	    readers[i].tileControl.put(tc);
      endmethod
      method Action stateDbg(ChannelType rc);
	 if (rc == ChannelType_Read)
	    dbgFSM.start;
      endmethod
      method Action memoryTraffic(ChannelType rc);
	 if (rc == ChannelType_Read)
	    trafficFSM.start;
      endmethod
      method Action addrTrans(Bit#(32) pointer, Bit#(32) offset);
	 addrReqFifo.enq(pointer);
	 mmus[pointer[31:16]].addr[0].request.put(ReqTup{id:truncate(pointer), off:extend(offset)});
      endmethod
   endinterface
endmodule

module mkMemServerWrite#(MemServerIndication indication,
		     Vector#(numMMUs,MMU#(addrWidth)) mmus)
   (MemServerWrite#(addrWidth, busWidth, numClients, numServers))
   provisos(Mul#(nwc, numClients, numServers)
	    ,Add#(a__, addrWidth, 64)
	    ,Add#(TLog#(TDiv#(busWidth, 8)), b__, 8)
	    ,Add#(TLog#(TDiv#(busWidth, 8)), c__, BurstLenSize)
	    );

   FIFO#(Bit#(32))   addrReqFifo <- mkFIFO;
   Reg#(Bit#(8)) dbgPtr <- mkReg(0);
   Reg#(Bit#(8)) trafficPtr <- mkReg(0);
   Reg#(Bit#(64)) trafficAccum <- mkReg(0);

   module foo#(Integer i) (ArbitratedMMU#(addrWidth,numClients));
      let rv <- mkArbitratedMMU(mmus[i].addr[1]);
      return rv;
   endmodule
   Vector#(numMMUs,ArbitratedMMU#(addrWidth,numClients)) mmu_servers <- mapM(foo,genVector);
   Vector#(numClients,MemWriteInternal#(addrWidth,busWidth,MemServerTags,nwc)) writers;
   Vector#(numClients, PhysMemWriteClient#(addrWidth,busWidth)) write_clients;
   Vector#(numServers, MemWriteServer#(busWidth)) write_servers;

   for(Integer i = 0; i < valueOf(numClients); i = i+1) begin
      Vector#(numMMUs,Server#(ReqTup,Bit#(addrWidth))) ss;
      for(Integer j = 0; j < valueOf(numMMUs); j=j+1)
	 ss[j] = mmu_servers[j].servers[i];
      writers[i] <- mkMemWriteInternal(indication, ss);
      write_clients[i] = writers[i].client;
      for(Integer j = 0; j < valueOf(nwc); j=j+1)
	 write_servers[i*valueOf(nwc)+j] = writers[i].servers[j];
   end

   rule mmuEntry;
      addrReqFifo.deq;
      let physAddr <- mmus[addrReqFifo.first[31:16]].addr[1].response.get;
      indication.addrResponse(zeroExtend(physAddr));
   endrule

   Stmt dbgStmt =
   seq
      for(dbgPtr <= 0; dbgPtr < fromInteger(valueOf(numClients)); dbgPtr <= dbgPtr+1)
	 (action
	     let rv <- writers[dbgPtr].dbg.dbg;
	     indication.reportStateDbg(rv);
	  endaction);
   endseq;
   FSM dbgFSM <- mkFSM(dbgStmt);

   Stmt trafficStmt =
   seq
      trafficAccum <= 0;
      for(trafficPtr <= 0; trafficPtr < fromInteger(valueOf(numClients)); trafficPtr <= trafficPtr+1)
	 (action
	     let rv <- writers[trafficPtr].dbg.getMemoryTraffic();
	     trafficAccum <= trafficAccum + rv;
	  endaction);
      indication.reportMemoryTraffic(trafficAccum);
   endseq;
   FSM trafficFSM <- mkFSM(trafficStmt);

   interface servers = write_servers;
   interface clients = write_clients;
   interface MemServerRequest request;
      method Action setTileState(TileControl tc);
	 for(Integer i = 0; i < valueOf(numClients); i=i+1)
	    writers[i].tileControl.put(tc);
      endmethod
      method Action stateDbg(ChannelType rc);
	 if (rc == ChannelType_Write)
	    dbgFSM.start;
      endmethod
      method Action memoryTraffic(ChannelType rc);
	 if (rc == ChannelType_Write)
	    trafficFSM.start;
      endmethod
      method Action addrTrans(Bit#(32) pointer, Bit#(32) offset);
	 addrReqFifo.enq(pointer);
	 mmus[pointer[31:16]].addr[1].request.put(ReqTup{id:truncate(pointer), off:extend(offset)});
      endmethod
   endinterface
endmodule

module mkMemServerWithMMU#(Vector#(numReadClients, MemReadClient#(busWidth)) readClients,
			  Vector#(numWriteClients, MemWriteClient#(busWidth)) writeClients,
			  MemServerIndication indication,
			  MMUIndication mmuIndication)(MemServerWithMMU#(addrWidth, busWidth,nMasters))

   provisos(Add#(TLog#(TDiv#(busWidth, 8)), e__, 8)
	    ,Add#(TLog#(TDiv#(busWidth, 8)), f__, BurstLenSize)
	    ,Add#(c__, addrWidth, 64)
	    ,Add#(d__, addrWidth, MemOffsetSize)
	    ,Add#(numWriteClients, a__, TMul#(TDiv#(numWriteClients, nMasters),nMasters))
	    ,Add#(numReadClients, b__, TMul#(TDiv#(numReadClients, nMasters),nMasters))
	    );

   MMU#(addrWidth) hostMMU <- mkMMU(0, True, mmuIndication);
   MemServer#(addrWidth,busWidth,nMasters) dma <- mkMemServer(readClients, writeClients, cons(hostMMU,nil), indication);

   interface MemServerRequest memServerRequest = dma.request;
   interface MMURequest mmuRequest = hostMMU.request;
   interface Vector masters = dma.masters;
endmodule


use ConnectalMimo in MemToPcie

use ConnectalMimo in MemToPcie

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import FIFO         :: *;
import FIFOF        :: *;
import GetPut       :: *;
import Connectable  :: *;
import PCIE         :: *;
import DefaultValue :: *;
import Vector       :: *;
import ClientServer :: *;
import MIMO         :: *;
import Probe        :: *;

import ConnectalMimo :: *;
import MIFO         :: *;
import MemTypes     :: *;
import ConfigCounter ::*;

typedef struct {
   TLPData#(16) tlp;
   TLPLength    dwCount;
   Bool         is3dw;
   Bool         isHeaderOnly;
   } TlpWriteHeaderInfo deriving (Bits);

interface MemToPcie#(numeric type buswidth);
    interface Client#(TLPData#(16), TLPData#(16)) tlp;
    interface PhysMemSlave#(40,buswidth) slave;
    method Bool tlpOutFifoNotEmpty();
    interface Reg#(Bool) use4dw;
endinterface: MemToPcie

`ifdef XILINX
   `define AXI
`elsif BSIM
   `define AXI
`elsif ALTERA
   `define AVALON
`endif

typedef 64 WriteDataMimoSize;
typedef TAdd#(1,TLog#(WriteDataMimoSize)) WriteDataBurstLenSize;
module mkMemToPcie#(PciId my_id)(MemToPcie#(buswidth))
   provisos (Div#(buswidth, 8, busWidthBytes),
	     Div#(buswidth, 32, busWidthWords),
	     Bits#(Vector#(busWidthWords, Bit#(32)), buswidth),
            Log#(busWidthBytes,beatShift),
	     Add#(aaa, 32, buswidth),
	     Add#(bbb, buswidth, 256),
	     Add#(ccc, TMul#(8, busWidthWords), 64),
	     Add#(ddd, TMul#(32, busWidthWords), 256),
	     Add#(eee, busWidthWords, 8),
	     Add#(1, a__, busWidthWords),
	     Add#(busWidthWords, b__, 4),
	     Add#(c__, busWidthWords, WriteDataMimoSize)
      );

    let beat_shift = fromInteger(valueOf(beatShift));

    FIFOF#(TLPData#(16)) tlpOutFifo <- mkFIFOF;
    FIFOF#(TLPData#(16)) tlpInFifo <- mkFIFOF;
    FIFOF#(TlpWriteHeaderInfo) tlpWriteHeaderFifo <- mkFIFOF;

    Reg#(Bit#(7)) hitReg <- mkReg(0);
    Reg#(Bool) use4dwReg <- mkReg(True);

    // default configuration for MIMO is for guarded enq() and deq().
    // However, the implicit guard only checks for space for 1 element for enq(), and availability of 1 element for deq().
    MIMOConfiguration mimoCfg = defaultValue;
   MIFO#(4,busWidthWords,16,Bit#(32)) completionMimo <- mkMIFO();
   MIFO#(4,busWidthWords,16,TLPTag) completionTagMimo <- mkMIFO();

   mimoCfg.bram_based = True;
   mimoCfg.unguarded = True;
    MIMO#(busWidthWords,4,WriteDataMimoSize,Bit#(32)) writeDataMimo <- ConnectalMimo::mkMIMOBram(mimoCfg);
    ConfigCounter#(8) writeDataCnt <- mkConfigCounter(0);
    Reg#(Bit#(WriteDataBurstLenSize)) writeBurstCount <- mkReg(0);
    FIFO#(Bit#(WriteDataBurstLenSize)) writeBurstCountFifo <- mkFIFO();
    Reg#(TLPLength)  writeDwCount <- mkReg(0); // how many 4 byte (double) words to send
    Reg#(LUInt#(4))    tlpDwCount <- mkReg(0); // how many to send in the next tlp (at most 4)
    Reg#(Bool)            lastTlp <- mkReg(False); // if the next tlp sent is the last one
    Reg#(Bool)    writeInProgress <- mkReg(False);
    FIFOF#(TLPTag) writeTag <- mkSizedFIFOF(16);
    FIFOF#(TLPTag) doneTag <- mkSizedFIFOF(16);

    Reg#(Bool) quadAlignedTlpHandled <- mkReg(True);

   Wire#(Bool) writeHeaderTlpWire <- mkDWire(False);
   Probe#(Bool) writeHeaderAvailableProbe <- mkProbe;
   Wire#(Bool) writeDataMimoEnqWire <- mkDWire(False);
   Probe#(Bool) writeDataMimoEnqProbe <- mkProbe;
   let    writeDataMimoCountProbe <- mkProbe;
   let writeDataMimoEnqReadyProbe <- mkProbe;
   Probe#(Bool) writeInProgressProbe <- mkProbe;
   Probe#(Bit#(10)) writeBurstCountProbe <- mkProbe;
   rule updateProbe0;
      writeDataMimoCountProbe    <= writeDataMimo.count();
      writeDataMimoEnqReadyProbe <= writeDataMimo.enqReadyN(fromInteger(valueOf(busWidthWords)));
   endrule
   rule updateProbe2;
      writeHeaderAvailableProbe <= writeHeaderTlpWire;
      writeDataMimoEnqProbe <= writeDataMimoEnqWire;
   endrule

   Reg#(Bool) writeDataCntAboveThreshold <- mkReg(False);
   Reg#(Bool) writeDataMimoHasRoom <- mkReg(False);
   rule updateWriteDataMimoHasRoom;
      writeDataMimoHasRoom <= (writeDataCnt.read <= fromInteger(valueOf(WriteDataMimoSize) - valueOf(busWidthWords)));
   endrule
   rule noheader if (!tlpWriteHeaderFifo.notEmpty());
      writeDataCntAboveThreshold <= False;
   endrule
   rule updateWriteDataCntAboveThreshold if (writeInProgress || !writeDataCntAboveThreshold);
      writeDataCntAboveThreshold <= (writeDataCnt.read >= truncate(unpack(tlpWriteHeaderFifo.first.dwCount)));
   endrule
   (* descending_urgency = "writeHeaderTlp,updateWriteDataCntAboveThreshold" *)
   rule writeHeaderTlp if (!writeInProgress && writeDataCntAboveThreshold);
      // update for next cycle
      writeDataCntAboveThreshold <= (writeDataCnt.read >= truncate(unpack(tlpWriteHeaderFifo.first.dwCount)));

      writeHeaderTlpWire <= True;
      let info <- toGet(tlpWriteHeaderFifo).get();
      let tlp     = info.tlp;
      let dwCount = info.dwCount;
      let is3dw   = info.is3dw;
      let isHeaderOnly = info.isHeaderOnly;

      TLPMemory4DWHeader hdr_4dw = unpack(tlp.data);

      TLPMemoryIO3DWHeader hdr_3dw = unpack(tlp.data);
      if (is3dw) begin
	 if (hdr_3dw.format != MEM_WRITE_3DW_DATA)
	    $display("MemToPcie: expecting MEM_WRITE_3DW_DATA, got %d", hdr_3dw.format);
	 Vector#(4, Bit#(32)) v = unpack(0);
         tlp.sof = True;
`ifdef AXI
         v = writeDataMimo.first();
	 writeDataMimo.deq(1);
	 hdr_3dw.data = byteSwap(v[0]);
         tlp.eof = (dwCount == 1) ? True : False;
         dwCount = dwCount - 1;
         writeDataCnt.decrement(1);
`elsif AVALON
         let quadWordAligned = isQuadWordAligned(getLowerAddr(hdr_3dw.addr, hdr_3dw.firstbe));
         // if quad-word aligned, insert bubble.
         if (!quadWordAligned) begin
            v = writeDataMimo.first();
            writeDataMimo.deq(1);
            hdr_3dw.data = v[0];
         end
         else begin
            hdr_3dw.data = unpack(0);
         end
         tlp.eof = (dwCount == 1 && !quadWordAligned) ? True : False;
         if (!quadWordAligned) begin
            dwCount = dwCount - 1;
            writeDataCnt.decrement(1);
         end
`endif
	 tlp.be = 16'hffff;
	 tlp.data = pack(hdr_3dw);
      end
      else begin
         quadAlignedTlpHandled <= isQuadWordAligned(getLowerAddr(truncate(unpack(hdr_4dw.addr)), hdr_4dw.firstbe));
	 tlp.be = 16'hffff;
      end

      tlpOutFifo.enq(tlp);
      $display("writeHeaderTlp dwCount=%d", dwCount);
      writeDwCount <= dwCount;
      tlpDwCount <= truncate(min(4,unpack(dwCount)));
      lastTlp <= (dwCount <= 4);
      writeInProgress <= (dwCount != 0);
      writeInProgressProbe <= (dwCount != 0);
      if (isHeaderOnly) begin
	 doneTag.enq(writeTag.first());
	 writeTag.deq();
      end

   endrule

   rule writeTlps if (writeInProgress); // already verified  writeDataMimo.deqReadyN(tlpDwCount)) for this transaction
      TLPData#(16) tlp = defaultValue;
      tlp.sof = False;
      Vector#(4, Bit#(32)) v = unpack(0);
      Integer currDwCount = 4;

`ifdef AVALON
      if (!quadAlignedTlpHandled) begin
         currDwCount = 3; // 3 Data Dwords in this cycle.
         quadAlignedTlpHandled <= True;
      end
`endif

      // The MIMO implicit guard only checks for availability of 1 element
      // so we explicitly check for the number of elements required
      writeDataMimo.deq(tlpDwCount);
      v = writeDataMimo.first();
      let dwCount = writeDwCount - extend(pack(tlpDwCount));
      writeDwCount <= dwCount;
      tlpDwCount <= truncate(min(fromInteger(currDwCount),unpack(dwCount)));
      writeDataCnt.decrement(unpack(extend(pack(tlpDwCount))));
      lastTlp <= (dwCount <= 4);
      if (tlpDwCount == 4)
	 tlp.be = 16'hffff;
      else if (tlpDwCount == 3)
	 tlp.be = 16'hfff0;
      else if (tlpDwCount == 2)
	 tlp.be = 16'hff00;
      else if (tlpDwCount == 1)
	 tlp.be = 16'hf000;
      tlp.eof = lastTlp;
      if (lastTlp) begin
	 writeInProgress <= False;
	 writeInProgressProbe <= False;
	 doneTag.enq(writeTag.first());
	 writeTag.deq();
	 $display("writeDwCount=%d will be zero", writeDwCount);
      end

      for (Integer i = 0; i < currDwCount; i = i + 1) begin
`ifdef AXI
	 tlp.data[(i+1)*32-1:i*32] = byteSwap(v[(currDwCount-1)-i]);
`elsif AVALON
	 tlp.data[(i+1)*32-1:i*32] = v[(currDwCount-1)-i];
`endif
      end

      tlpOutFifo.enq(tlp);
   endrule

   Reg#(TLPTag) lastTag <- mkReg(0);
   FIFOF#(TLPData#(16)) tlpDecodeFifo <- mkFIFOF();
   Reg#(TLPLength) wordCountReg <- mkReg(0);
   rule tlpInRule;
      let tlp <- toGet(tlpInFifo).get();
      tlpDecodeFifo.enq(tlp);
   endrule

   rule handleTlpRule;
      let tlp = tlpDecodeFifo.first;
      Bool handled = False;
      TLPMemoryIO3DWHeader h = unpack(tlp.data);
      hitReg <= tlp.hit;
      TLPMemoryIO3DWHeader hdr_3dw = unpack(tlp.data);
      TLPCompletionHeader hdr_completion = unpack(tlp.data);
      Vector#(4, Bit#(32)) vec = unpack(0);
      Vector#(4, Bit#(32)) tlpvec = unpack(tlp.data);
      let wordCount = wordCountReg;
`ifdef AXI
      let dataInSecondTlp = False;
`elsif AVALON
      let quadWordAligned = isQuadWordAligned(getLowerAddr(hdr_3dw.addr, hdr_3dw.firstbe));
      let dataInSecondTlp = quadWordAligned;
`endif
      if (!tlp.sof) begin
	 vec = reverse(tlpvec);
	 // The MIMO implicit guard only checks for space to enqueue 1 element
	 // so we explicitly check for the number of elements required
	 // otherwise elements in the queue will be overwritten.
	 if (completionMimo.enqReady()
	    && completionTagMimo.enqReady())
	    begin
	       UInt#(3) count = 4;
	       if (tlp.eof) begin
		  count = truncate(unpack(wordCountReg));
	       end
	       wordCount = wordCountReg - extend(pack(count));
	       completionMimo.enq(count, vec);
	       Vector#(4, TLPTag) tagvec = replicate(lastTag);
	       completionTagMimo.enq(count, tagvec);
	       handled = True;
	    end
      end
      else if (hdr_3dw.format == MEM_WRITE_3DW_DATA
	       && hdr_3dw.pkttype == COMPLETION
	       && completionMimo.enqReady()
	       && completionTagMimo.enqReady()) begin
            TLPTag tag = hdr_completion.tag;
            lastTag <= tag;
            if (!dataInSecondTlp) begin
               vec[0] = hdr_3dw.data;
               wordCount = hdr_3dw.length - 1;
               completionMimo.enq(1, vec);
               completionTagMimo.enq(1, replicate(tag));
            end
	    handled = True;
      end
      wordCountReg <= wordCount;
      $display("tlpIn handled=%d tlp=%h\n", handled, tlp);
      if (handled) begin
	 tlpDecodeFifo.deq();
      end
   endrule

   FIFO#(PhysMemRequest#(40,buswidth)) readReqFifo <- mkFIFO();
   rule readReqRule if (!writeInProgress); // && !writeDataMimo.deqReady());
      let req <- toGet(readReqFifo).get();
      let burstLen = req.burstLen >> beat_shift;
      let addr = req.addr;
      let arid = req.tag;

      TLPData#(16) tlp = defaultValue;
      tlp.sof = True;
      tlp.eof = True;
      tlp.hit = 7'h00;
      TLPLength tlplen = fromInteger(valueOf(busWidthWords))*extend(burstLen);
      if (addr[39:32] != 0) begin
	 TLPMemory4DWHeader hdr_4dw = defaultValue;
	 hdr_4dw.format = MEM_READ_4DW_NO_DATA;
	 hdr_4dw.tag = extend(arid);
	 hdr_4dw.reqid = my_id;
	 hdr_4dw.nosnoop = SNOOPING_REQD;
	 hdr_4dw.addr = addr[40-1:2];
	 hdr_4dw.length = tlplen;
	 hdr_4dw.firstbe = reqFirstByteEnable(req)[3:0];
	 hdr_4dw.lastbe = (tlplen > 1) ? reqLastByteEnable(req)[valueOf(busWidthBytes)-1:valueOf(busWidthBytes)-4] : 0;
	 tlp.data = pack(hdr_4dw);
	 tlp.be = 16'hffff;
      end
      else begin
	 TLPMemoryIO3DWHeader hdr_3dw = defaultValue;
	 hdr_3dw.format = MEM_READ_3DW_NO_DATA;
	 hdr_3dw.tag = extend(arid);
	 hdr_3dw.reqid = my_id;
	 hdr_3dw.nosnoop = SNOOPING_REQD;
	 hdr_3dw.addr = addr[32-1:2];
	 hdr_3dw.length = tlplen;
	 hdr_3dw.firstbe = reqFirstByteEnable(req)[3:0];
	 hdr_3dw.lastbe = (tlplen > 1) ? reqLastByteEnable(req)[valueOf(busWidthBytes)-1:valueOf(busWidthBytes)-4] : 0;
	 tlp.data = pack(hdr_3dw);
	 tlp.be = 16'hfff0;
      end
      tlpOutFifo.enq(tlp);
   endrule

    interface Client        tlp;
        interface request = toGet(tlpOutFifo);
        interface response = toPut(tlpInFifo);
    endinterface
    interface PhysMemSlave slave;
   interface PhysMemWriteServer write_server;
      interface Put writeReq;
         method Action put(PhysMemRequest#(40,buswidth) req); // if (writeBurstCount == 0);
	    Bit#(WriteDataBurstLenSize) burstLen = truncate(req.burstLen >> beat_shift);
	    let addr = req.addr;
	    let awid = req.tag;
	    let writeIs3dw = False;

	    TLPLength tlplen = fromInteger(valueOf(busWidthWords))*extend(burstLen);
	    TLPData#(16) tlp = defaultValue;
	    tlp.sof = True;
	    tlp.eof = False;
	    tlp.hit = 7'h00;
	    tlp.be = 16'hffff;

	    $display("slave.writeAddr tlplen=%d burstLen=%d", tlplen, burstLen);
	    if ((addr >> 32) != 0) begin
	       TLPMemory4DWHeader hdr_4dw = defaultValue;
	       hdr_4dw.format = MEM_WRITE_4DW_DATA;
	       hdr_4dw.tag = extend(awid);
	       hdr_4dw.reqid = my_id;
	       hdr_4dw.nosnoop = SNOOPING_REQD;
	       hdr_4dw.addr = addr[40-1:2];
	       hdr_4dw.length = tlplen;
	       hdr_4dw.firstbe = reqFirstByteEnable(req)[3:0];
	       hdr_4dw.lastbe = (tlplen > 1) ? reqLastByteEnable(req)[valueOf(busWidthBytes)-1:valueOf(busWidthBytes)-4] : 0;
	       tlp.data = pack(hdr_4dw);
	    end
	    else begin
	       writeIs3dw = True;
	       TLPMemoryIO3DWHeader hdr_3dw = defaultValue;
	       hdr_3dw.format = MEM_WRITE_3DW_DATA;
	       hdr_3dw.tag = extend(awid);
	       hdr_3dw.reqid = my_id;
	       hdr_3dw.nosnoop = SNOOPING_REQD;
	       hdr_3dw.addr = addr[32-1:2];
	       hdr_3dw.length = tlplen;
	       hdr_3dw.firstbe = reqFirstByteEnable(req)[3:0];
	       hdr_3dw.lastbe = (tlplen > 1) ? reqLastByteEnable(req)[valueOf(busWidthBytes)-1:valueOf(busWidthBytes)-4] : 0;
	       tlp.be = 16'hfff0; // no data word in this TLP

	       tlp.data = pack(hdr_3dw);
	    end
	    tlpWriteHeaderFifo.enq(TlpWriteHeaderInfo {tlp: tlp, dwCount: tlplen, is3dw: writeIs3dw, isHeaderOnly: (writeIs3dw && tlplen == 1) });
	    writeBurstCountFifo.enq(burstLen);
	    writeTag.enq(extend(awid));
         endmethod
	endinterface
      interface Put writeData;
         method Action put(MemData#(buswidth) wdata)
	    provisos (Bits#(Vector#(busWidthWords, Bit#(32)), busWidth)) if (writeDataMimoHasRoom); //.enqReadyN(fromInteger(valueOf(busWidthWords))));

	    let burstLen = writeBurstCount;
	    if (burstLen == 0) begin
	       burstLen <- toGet(writeBurstCountFifo).get();
	    end
	    writeBurstCount <= extend(burstLen-1);
	    writeBurstCountProbe <= extend(burstLen-1);

	    Vector#(busWidthWords, Bit#(32)) v = unpack(wdata.data);
	    writeDataMimo.enq(fromInteger(valueOf(busWidthWords)), v);
            writeDataCnt.increment(fromInteger(valueOf(busWidthWords)));
	    writeDataMimoEnqWire <= True;
         endmethod
       endinterface
      interface Get writeDone;
         method ActionValue#(Bit#(MemTagSize)) get();
	      let tag = doneTag.first();
	      doneTag.deq();
	      return truncate(tag);
           endmethod
	endinterface
   endinterface
   interface PhysMemReadServer read_server;
      interface Put readReq;
         method Action put(PhysMemRequest#(40,buswidth) req);
	    readReqFifo.enq(req);
         endmethod
       endinterface
      interface Get     readData;
         method ActionValue#(MemData#(buswidth)) get() if (completionMimo.deqReady()
							   && completionTagMimo.deqReady());
	      let data_v = completionMimo.first;
	      let tag_v = completionTagMimo.first;
	      completionMimo.deq();
	      completionTagMimo.deq();
              Bit#(buswidth) v = 0;
              for (Integer i = 0; i < valueOf(busWidthWords); i = i+1) begin
`ifdef AXI
		 v[(i+1)*32-1:i*32] = byteSwap(data_v[i]);
`elsif AVALON
		 v[(i+1)*32-1:i*32] = data_v[i];
`endif
              end
	      return MemData { data: v, tag: truncate(tag_v[0]), last: True};
           endmethod
	endinterface
   endinterface
    endinterface: slave
   method Bool tlpOutFifoNotEmpty() = tlpOutFifo.notEmpty;
   interface Reg use4dw = use4dwReg;
endmodule: mkMemToPcie

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import FIFO         :: *;
import FIFOF        :: *;
import GetPut       :: *;
import Connectable  :: *;
import PCIE         :: *;
import DefaultValue :: *;
import MIMO         :: *;
import MIFO         :: *;
import Vector       :: *;
import ClientServer :: *;
import MemTypes     :: *;
import Probe        :: *;
import ConfigCounter ::*;

typedef struct {
   TLPData#(16) tlp;
   TLPLength    dwCount;
   Bool         is3dw;
   Bool         isHeaderOnly;
   } TlpWriteHeaderInfo deriving (Bits);

interface MemToPcie#(numeric type buswidth);
    interface Client#(TLPData#(16), TLPData#(16)) tlp;
    interface PhysMemSlave#(40,buswidth) slave;
    method Bool tlpOutFifoNotEmpty();
    interface Reg#(Bool) use4dw;
endinterface: MemToPcie

`ifdef XILINX
   `define AXI
`elsif BSIM
   `define AXI
`elsif ALTERA
   `define AVALON
`endif

typedef 64 WriteDataMimoSize;
typedef TAdd#(1,TLog#(WriteDataMimoSize)) WriteDataBurstLenSize;
module mkMemToPcie#(PciId my_id)(MemToPcie#(buswidth))
   provisos (Div#(buswidth, 8, busWidthBytes),
	     Div#(buswidth, 32, busWidthWords),
	     Bits#(Vector#(busWidthWords, Bit#(32)), buswidth),
            Log#(busWidthBytes,beatShift),
	     Add#(aaa, 32, buswidth),
	     Add#(bbb, buswidth, 256),
	     Add#(ccc, TMul#(8, busWidthWords), 64),
	     Add#(ddd, TMul#(32, busWidthWords), 256),
	     Add#(eee, busWidthWords, 8),
	     Add#(1, a__, busWidthWords),
	     Add#(busWidthWords, b__, 4),
	     Add#(c__, busWidthWords, WriteDataMimoSize)
      );

    let beat_shift = fromInteger(valueOf(beatShift));

    FIFOF#(TLPData#(16)) tlpOutFifo <- mkFIFOF;
    FIFOF#(TLPData#(16)) tlpInFifo <- mkFIFOF;
    FIFOF#(TlpWriteHeaderInfo) tlpWriteHeaderFifo <- mkFIFOF;

    Reg#(Bit#(7)) hitReg <- mkReg(0);
    Reg#(Bool) use4dwReg <- mkReg(True);

    // default configuration for MIMO is for guarded enq() and deq().
    // However, the implicit guard only checks for space for 1 element for enq(), and availability of 1 element for deq().
    MIMOConfiguration mimoCfg = defaultValue;
   MIFO#(4,busWidthWords,16,Bit#(32)) completionMimo <- mkMIFO();
   MIFO#(4,busWidthWords,16,TLPTag) completionTagMimo <- mkMIFO();

   mimoCfg.bram_based = True;
   mimoCfg.unguarded = True;
    MIMO#(busWidthWords,4,WriteDataMimoSize,Bit#(32)) writeDataMimo <- mkMIMO(mimoCfg);
    ConfigCounter#(8) writeDataCnt <- mkConfigCounter(0);
    Reg#(Bit#(WriteDataBurstLenSize)) writeBurstCount <- mkReg(0);
    FIFO#(Bit#(WriteDataBurstLenSize)) writeBurstCountFifo <- mkFIFO();
    Reg#(TLPLength)  writeDwCount <- mkReg(0); // how many 4 byte (double) words to send
    Reg#(LUInt#(4))    tlpDwCount <- mkReg(0); // how many to send in the next tlp (at most 4)
    Reg#(Bool)            lastTlp <- mkReg(False); // if the next tlp sent is the last one
    Reg#(Bool)    writeInProgress <- mkReg(False);
    FIFOF#(TLPTag) writeTag <- mkSizedFIFOF(16);
    FIFOF#(TLPTag) doneTag <- mkSizedFIFOF(16);

    Reg#(Bool) quadAlignedTlpHandled <- mkReg(True);

   Wire#(Bool) writeHeaderTlpWire <- mkDWire(False);
   Probe#(Bool) writeHeaderAvailableProbe <- mkProbe;
   Wire#(Bool) writeDataMimoEnqWire <- mkDWire(False);
   Probe#(Bool) writeDataMimoEnqProbe <- mkProbe;
   let    writeDataMimoCountProbe <- mkProbe;
   let writeDataMimoEnqReadyProbe <- mkProbe;
   Probe#(Bool) writeInProgressProbe <- mkProbe;
   Probe#(Bit#(10)) writeBurstCountProbe <- mkProbe;
   rule updateProbe0;
      writeDataMimoCountProbe    <= writeDataMimo.count();
      writeDataMimoEnqReadyProbe <= writeDataMimo.enqReadyN(fromInteger(valueOf(busWidthWords)));
   endrule
   rule updateProbe2;
      writeHeaderAvailableProbe <= writeHeaderTlpWire;
      writeDataMimoEnqProbe <= writeDataMimoEnqWire;
   endrule

   Reg#(Bool) writeDataCntAboveThreshold <- mkReg(False);
   Reg#(Bool) writeDataMimoHasRoom <- mkReg(False);
   rule updateWriteDataMimoHasRoom;
      writeDataMimoHasRoom <= (writeDataCnt.read <= fromInteger(valueOf(WriteDataMimoSize) - valueOf(busWidthWords)));
   endrule
   rule noheader if (!tlpWriteHeaderFifo.notEmpty());
      writeDataCntAboveThreshold <= False;
   endrule
   rule updateWriteDataCntAboveThreshold if (writeInProgress || !writeDataCntAboveThreshold);
      writeDataCntAboveThreshold <= (writeDataCnt.read >= truncate(unpack(tlpWriteHeaderFifo.first.dwCount)));
   endrule
   (* descending_urgency = "writeHeaderTlp,updateWriteDataCntAboveThreshold" *)
   rule writeHeaderTlp if (!writeInProgress && writeDataCntAboveThreshold);
      // update for next cycle
      writeDataCntAboveThreshold <= (writeDataCnt.read >= truncate(unpack(tlpWriteHeaderFifo.first.dwCount)));

      writeHeaderTlpWire <= True;
      let info <- toGet(tlpWriteHeaderFifo).get();
      let tlp     = info.tlp;
      let dwCount = info.dwCount;
      let is3dw   = info.is3dw;
      let isHeaderOnly = info.isHeaderOnly;

      TLPMemory4DWHeader hdr_4dw = unpack(tlp.data);

      TLPMemoryIO3DWHeader hdr_3dw = unpack(tlp.data);
      if (is3dw) begin
	 if (hdr_3dw.format != MEM_WRITE_3DW_DATA)
	    $display("MemToPcie: expecting MEM_WRITE_3DW_DATA, got %d", hdr_3dw.format);
	 Vector#(4, Bit#(32)) v = unpack(0);
         tlp.sof = True;
`ifdef AXI
         v = writeDataMimo.first();
	 writeDataMimo.deq(1);
	 hdr_3dw.data = byteSwap(v[0]);
         tlp.eof = (dwCount == 1) ? True : False;
         dwCount = dwCount - 1;
         writeDataCnt.decrement(1);
`elsif AVALON
         let quadWordAligned = isQuadWordAligned(getLowerAddr(hdr_3dw.addr, hdr_3dw.firstbe));
         // if quad-word aligned, insert bubble.
         if (!quadWordAligned) begin
            v = writeDataMimo.first();
            writeDataMimo.deq(1);
            hdr_3dw.data = v[0];
         end
         else begin
            hdr_3dw.data = unpack(0);
         end
         tlp.eof = (dwCount == 1 && !quadWordAligned) ? True : False;
         if (!quadWordAligned) begin
            dwCount = dwCount - 1;
            writeDataCnt.decrement(1);
         end
`endif
	 tlp.be = 16'hffff;
	 tlp.data = pack(hdr_3dw);
      end
      else begin
         quadAlignedTlpHandled <= isQuadWordAligned(getLowerAddr(truncate(unpack(hdr_4dw.addr)), hdr_4dw.firstbe));
	 tlp.be = 16'hffff;
      end

      tlpOutFifo.enq(tlp);
      $display("writeHeaderTlp dwCount=%d", dwCount);
      writeDwCount <= dwCount;
      tlpDwCount <= truncate(min(4,unpack(dwCount)));
      lastTlp <= (dwCount <= 4);
      writeInProgress <= (dwCount != 0);
      writeInProgressProbe <= (dwCount != 0);
      if (isHeaderOnly) begin
	 doneTag.enq(writeTag.first());
	 writeTag.deq();
      end

   endrule

   rule writeTlps if (writeInProgress); // already verified  writeDataMimo.deqReadyN(tlpDwCount)) for this transaction
      TLPData#(16) tlp = defaultValue;
      tlp.sof = False;
      Vector#(4, Bit#(32)) v = unpack(0);
      Integer currDwCount = 4;

`ifdef AVALON
      if (!quadAlignedTlpHandled) begin
         currDwCount = 3; // 3 Data Dwords in this cycle.
         quadAlignedTlpHandled <= True;
      end
`endif

      // The MIMO implicit guard only checks for availability of 1 element
      // so we explicitly check for the number of elements required
      writeDataMimo.deq(tlpDwCount);
      v = writeDataMimo.first();
      let dwCount = writeDwCount - extend(pack(tlpDwCount));
      writeDwCount <= dwCount;
      tlpDwCount <= truncate(min(fromInteger(currDwCount),unpack(dwCount)));
      writeDataCnt.decrement(unpack(extend(pack(tlpDwCount))));
      lastTlp <= (dwCount <= 4);
      if (tlpDwCount == 4)
	 tlp.be = 16'hffff;
      else if (tlpDwCount == 3)
	 tlp.be = 16'hfff0;
      else if (tlpDwCount == 2)
	 tlp.be = 16'hff00;
      else if (tlpDwCount == 1)
	 tlp.be = 16'hf000;
      tlp.eof = lastTlp;
      if (lastTlp) begin
	 writeInProgress <= False;
	 writeInProgressProbe <= False;
	 doneTag.enq(writeTag.first());
	 writeTag.deq();
	 $display("writeDwCount=%d will be zero", writeDwCount);
      end

      for (Integer i = 0; i < currDwCount; i = i + 1) begin
`ifdef AXI
	 tlp.data[(i+1)*32-1:i*32] = byteSwap(v[(currDwCount-1)-i]);
`elsif AVALON
	 tlp.data[(i+1)*32-1:i*32] = v[(currDwCount-1)-i];
`endif
      end

      tlpOutFifo.enq(tlp);
   endrule

   Reg#(TLPTag) lastTag <- mkReg(0);
   FIFOF#(TLPData#(16)) tlpDecodeFifo <- mkFIFOF();
   Reg#(TLPLength) wordCountReg <- mkReg(0);
   rule tlpInRule;
      let tlp <- toGet(tlpInFifo).get();
      tlpDecodeFifo.enq(tlp);
   endrule

   rule handleTlpRule;
      let tlp = tlpDecodeFifo.first;
      Bool handled = False;
      TLPMemoryIO3DWHeader h = unpack(tlp.data);
      hitReg <= tlp.hit;
      TLPMemoryIO3DWHeader hdr_3dw = unpack(tlp.data);
      TLPCompletionHeader hdr_completion = unpack(tlp.data);
      Vector#(4, Bit#(32)) vec = unpack(0);
      Vector#(4, Bit#(32)) tlpvec = unpack(tlp.data);
      let wordCount = wordCountReg;
`ifdef AXI
      let dataInSecondTlp = False;
`elsif AVALON
      let quadWordAligned = isQuadWordAligned(getLowerAddr(hdr_3dw.addr, hdr_3dw.firstbe));
      let dataInSecondTlp = quadWordAligned;
`endif
      if (!tlp.sof) begin
	 vec = reverse(tlpvec);
	 // The MIMO implicit guard only checks for space to enqueue 1 element
	 // so we explicitly check for the number of elements required
	 // otherwise elements in the queue will be overwritten.
	 if (completionMimo.enqReady()
	    && completionTagMimo.enqReady())
	    begin
	       UInt#(3) count = 4;
	       if (tlp.eof) begin
		  count = truncate(unpack(wordCountReg));
	       end
	       wordCount = wordCountReg - extend(pack(count));
	       completionMimo.enq(count, vec);
	       Vector#(4, TLPTag) tagvec = replicate(lastTag);
	       completionTagMimo.enq(count, tagvec);
	       handled = True;
	    end
      end
      else if (hdr_3dw.format == MEM_WRITE_3DW_DATA
	       && hdr_3dw.pkttype == COMPLETION
	       && completionMimo.enqReady()
	       && completionTagMimo.enqReady()) begin
            TLPTag tag = hdr_completion.tag;
            lastTag <= tag;
            if (!dataInSecondTlp) begin
               vec[0] = hdr_3dw.data;
               wordCount = hdr_3dw.length - 1;
               completionMimo.enq(1, vec);
               completionTagMimo.enq(1, replicate(tag));
            end
	    handled = True;
      end
      wordCountReg <= wordCount;
      $display("tlpIn handled=%d tlp=%h\n", handled, tlp);
      if (handled) begin
	 tlpDecodeFifo.deq();
      end
   endrule

   FIFO#(PhysMemRequest#(40,buswidth)) readReqFifo <- mkFIFO();
   rule readReqRule if (!writeInProgress); // && !writeDataMimo.deqReady());
      let req <- toGet(readReqFifo).get();
      let burstLen = req.burstLen >> beat_shift;
      let addr = req.addr;
      let arid = req.tag;

      TLPData#(16) tlp = defaultValue;
      tlp.sof = True;
      tlp.eof = True;
      tlp.hit = 7'h00;
      TLPLength tlplen = fromInteger(valueOf(busWidthWords))*extend(burstLen);
      if (addr[39:32] != 0) begin
	 TLPMemory4DWHeader hdr_4dw = defaultValue;
	 hdr_4dw.format = MEM_READ_4DW_NO_DATA;
	 hdr_4dw.tag = extend(arid);
	 hdr_4dw.reqid = my_id;
	 hdr_4dw.nosnoop = SNOOPING_REQD;
	 hdr_4dw.addr = addr[40-1:2];
	 hdr_4dw.length = tlplen;
	 hdr_4dw.firstbe = reqFirstByteEnable(req)[3:0];
	 hdr_4dw.lastbe = (tlplen > 1) ? reqLastByteEnable(req)[valueOf(busWidthBytes)-1:valueOf(busWidthBytes)-4] : 0;
	 tlp.data = pack(hdr_4dw);
	 tlp.be = 16'hffff;
      end
      else begin
	 TLPMemoryIO3DWHeader hdr_3dw = defaultValue;
	 hdr_3dw.format = MEM_READ_3DW_NO_DATA;
	 hdr_3dw.tag = extend(arid);
	 hdr_3dw.reqid = my_id;
	 hdr_3dw.nosnoop = SNOOPING_REQD;
	 hdr_3dw.addr = addr[32-1:2];
	 hdr_3dw.length = tlplen;
	 hdr_3dw.firstbe = reqFirstByteEnable(req)[3:0];
	 hdr_3dw.lastbe = (tlplen > 1) ? reqLastByteEnable(req)[valueOf(busWidthBytes)-1:valueOf(busWidthBytes)-4] : 0;
	 tlp.data = pack(hdr_3dw);
	 tlp.be = 16'hfff0;
      end
      tlpOutFifo.enq(tlp);
   endrule

    interface Client        tlp;
        interface request = toGet(tlpOutFifo);
        interface response = toPut(tlpInFifo);
    endinterface
    interface PhysMemSlave slave;
   interface PhysMemWriteServer write_server;
      interface Put writeReq;
         method Action put(PhysMemRequest#(40,buswidth) req); // if (writeBurstCount == 0);
	    Bit#(WriteDataBurstLenSize) burstLen = truncate(req.burstLen >> beat_shift);
	    let addr = req.addr;
	    let awid = req.tag;
	    let writeIs3dw = False;

	    TLPLength tlplen = fromInteger(valueOf(busWidthWords))*extend(burstLen);
	    TLPData#(16) tlp = defaultValue;
	    tlp.sof = True;
	    tlp.eof = False;
	    tlp.hit = 7'h00;
	    tlp.be = 16'hffff;

	    $display("slave.writeAddr tlplen=%d burstLen=%d", tlplen, burstLen);
	    if ((addr >> 32) != 0) begin
	       TLPMemory4DWHeader hdr_4dw = defaultValue;
	       hdr_4dw.format = MEM_WRITE_4DW_DATA;
	       hdr_4dw.tag = extend(awid);
	       hdr_4dw.reqid = my_id;
	       hdr_4dw.nosnoop = SNOOPING_REQD;
	       hdr_4dw.addr = addr[40-1:2];
	       hdr_4dw.length = tlplen;
	       hdr_4dw.firstbe = reqFirstByteEnable(req)[3:0];
	       hdr_4dw.lastbe = (tlplen > 1) ? reqLastByteEnable(req)[valueOf(busWidthBytes)-1:valueOf(busWidthBytes)-4] : 0;
	       tlp.data = pack(hdr_4dw);
	    end
	    else begin
	       writeIs3dw = True;
	       TLPMemoryIO3DWHeader hdr_3dw = defaultValue;
	       hdr_3dw.format = MEM_WRITE_3DW_DATA;
	       hdr_3dw.tag = extend(awid);
	       hdr_3dw.reqid = my_id;
	       hdr_3dw.nosnoop = SNOOPING_REQD;
	       hdr_3dw.addr = addr[32-1:2];
	       hdr_3dw.length = tlplen;
	       hdr_3dw.firstbe = reqFirstByteEnable(req)[3:0];
	       hdr_3dw.lastbe = (tlplen > 1) ? reqLastByteEnable(req)[valueOf(busWidthBytes)-1:valueOf(busWidthBytes)-4] : 0;
	       tlp.be = 16'hfff0; // no data word in this TLP

	       tlp.data = pack(hdr_3dw);
	    end
	    tlpWriteHeaderFifo.enq(TlpWriteHeaderInfo {tlp: tlp, dwCount: tlplen, is3dw: writeIs3dw, isHeaderOnly: (writeIs3dw && tlplen == 1) });
	    writeBurstCountFifo.enq(burstLen);
	    writeTag.enq(extend(awid));
         endmethod
	endinterface
      interface Put writeData;
         method Action put(MemData#(buswidth) wdata)
	    provisos (Bits#(Vector#(busWidthWords, Bit#(32)), busWidth)) if (writeDataMimoHasRoom); //.enqReadyN(fromInteger(valueOf(busWidthWords))));

	    let burstLen = writeBurstCount;
	    if (burstLen == 0) begin
	       burstLen <- toGet(writeBurstCountFifo).get();
	    end
	    writeBurstCount <= extend(burstLen-1);
	    writeBurstCountProbe <= extend(burstLen-1);

	    Vector#(busWidthWords, Bit#(32)) v = unpack(wdata.data);
	    writeDataMimo.enq(fromInteger(valueOf(busWidthWords)), v);
            writeDataCnt.increment(fromInteger(valueOf(busWidthWords)));
	    writeDataMimoEnqWire <= True;
         endmethod
       endinterface
      interface Get writeDone;
         method ActionValue#(Bit#(MemTagSize)) get();
	      let tag = doneTag.first();
	      doneTag.deq();
	      return truncate(tag);
           endmethod
	endinterface
   endinterface
   interface PhysMemReadServer read_server;
      interface Put readReq;
         method Action put(PhysMemRequest#(40,buswidth) req);
	    readReqFifo.enq(req);
         endmethod
       endinterface
      interface Get     readData;
         method ActionValue#(MemData#(buswidth)) get() if (completionMimo.deqReady()
							   && completionTagMimo.deqReady());
	      let data_v = completionMimo.first;
	      let tag_v = completionTagMimo.first;
	      completionMimo.deq();
	      completionTagMimo.deq();
              Bit#(buswidth) v = 0;
              for (Integer i = 0; i < valueOf(busWidthWords); i = i+1) begin
`ifdef AXI
		 v[(i+1)*32-1:i*32] = byteSwap(data_v[i]);
`elsif AVALON
		 v[(i+1)*32-1:i*32] = data_v[i];
`endif
              end
	      return MemData { data: v, tag: truncate(tag_v[0]), last: True};
           endmethod
	endinterface
   endinterface
    endinterface: slave
   method Bool tlpOutFifoNotEmpty() = tlpOutFifo.notEmpty;
   interface Reg use4dw = use4dwReg;
endmodule: mkMemToPcie


Generates interrupts via PCIe

Generates interrupts via PCIe

import Clocks            :: *;
import Connectable       :: *;
import Xilinx            :: *;
import XilinxPCIE        :: *;
import Kintex7PcieBridge :: *;
import EchoWrapper       :: *;

(* synthesize, no_default_clock, no_default_reset *)
module mkEchoTop #(Clock pci_sys_clk_p, Clock pci_sys_clk_n,
		  Clock sys_clk_p,     Clock sys_clk_n,
		  Clock user_clk_p, Clock user_clk_n,
		  Reset pci_sys_reset_n)
                 (KC705_FPGA);

   Clock user_clk <- mkClockIBUFDS(user_clk_p, user_clk_n);

   K7PcieBridgeIfc#(8) k7pcie <- mkK7PcieBridge( pci_sys_clk_p, pci_sys_clk_n, sys_clk_p, sys_clk_n, pci_sys_reset_n,
                                                 64'h05ce_0006_ecc0_2608 );

   Reg#(Bool) interruptRequested <- mkReg(False, clocked_by k7pcie.clock125, reset_by k7pcie.reset125);
   EchoWrapper echoWrapper <- mkEchoWrapper(clocked_by k7pcie.clock125, reset_by k7pcie.reset125);
   mkConnection(k7pcie.portal0, echoWrapper.ctrl, clocked_by k7pcie.clock125, reset_by k7pcie.reset125);
   mkConnection(echoWrapper.trace, k7pcie.trace);

   rule requestInterrupt;
      Bool interrupt = (echoWrapper.interrupts[0] == 1);
      if (interrupt && !interruptRequested)
	 k7pcie.interrupt();
      interruptRequested <= interrupt;
   endrule

   interface pcie = k7pcie.pcie;
   //interface ddr3 = k7pcie.ddr3;
   method leds = zeroExtend({ pack(k7pcie.isCalibrated)
			     ,pack(False)
			     ,pack(False)
			     ,pack(k7pcie.isLinkUp)
			     });
endmodule: mkEchoTop

import Clocks            :: *;
import Connectable       :: *;
import Xilinx            :: *;
import XilinxPCIE        :: *;
import Kintex7PcieBridge :: *;
import EchoWrapper       :: *;

(* synthesize, no_default_clock, no_default_reset *)
module mkEchoTop #(Clock pci_sys_clk_p, Clock pci_sys_clk_n,
		  Clock sys_clk_p,     Clock sys_clk_n,
		  Clock user_clk_p, Clock user_clk_n,
		  Reset pci_sys_reset_n)
                 (KC705_FPGA);

   Clock user_clk <- mkClockIBUFDS(user_clk_p, user_clk_n);

   K7PcieBridgeIfc#(8) k7pcie <- mkK7PcieBridge( pci_sys_clk_p, pci_sys_clk_n, sys_clk_p, sys_clk_n, pci_sys_reset_n,
                                                 64'h05ce_0006_ecc0_2604 );

   EchoWrapper echoWrapper <- mkEchoWrapper(clocked_by k7pcie.clock125, reset_by k7pcie.reset125);
   mkConnection(k7pcie.portal0, echoWrapper.ctrl, clocked_by k7pcie.clock125, reset_by k7pcie.reset125);
   mkConnection(echoWrapper.trace, k7pcie.trace);

   interface pcie = k7pcie.pcie;
   //interface ddr3 = k7pcie.ddr3;
   method leds = zeroExtend({ pack(k7pcie.isCalibrated)
			     ,pack(False)
			     ,pack(False)
			     ,pack(k7pcie.isLinkUp)
			     });
endmodule: mkEchoTop


Precompute first and last to reduce combinational path.

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

// BSV Libraries
import BRAMFIFO::*;
import FIFO::*;
import FIFOF::*;
import Vector::*;
import GetPut::*;
import ClientServer::*;
import Assert::*;

// XBSV Libraries
import MemTypes::*;
import PortalMemory::*;
import SGList::*;

typedef 9 SGL_PIPELINE_DEPTH;
typedef 32 TAG_DEPTH;

interface MemWriteInternal#(numeric type addrWidth, numeric type dataWidth);
   interface DmaDbg dbg;
   interface MemWriteClient#(addrWidth,dataWidth) write_client;
endinterface

interface MemReadInternal#(numeric type addrWidth, numeric type dataWidth);
   interface DmaDbg dbg;
   interface MemReadClient#(addrWidth,dataWidth) read_client;
endinterface

function Bool bad_pointer(ObjectPointer p);
   return ((p >>8) > fromInteger(valueOf(MaxNumSGLists)));
endfunction

interface TagGen#(numeric type numClients, numeric type numTags);
   method Action tag_request(Bit#(TLog#(numClients)) client, Bit#(6) orig_tag);
   method ActionValue#(Bit#(TLog#(numTags)))  tag_response;
   method Action return_tag(Bit#(TLog#(numTags)) tag);
endinterface

module mkTagGenOO(TagGen#(numClients,numTags))
   provisos(Log#(numTags,tagWidth),
	    Log#(numClients,clientWidth));

   let request_fifo0 <- mkSizedFIFOF(valueOf(SGL_PIPELINE_DEPTH));
   let request_fifo1 <- mkSizedFIFOF(valueOf(SGL_PIPELINE_DEPTH));
   let return_fifo <- mkFIFO;
   Vector#(numTags, Reg#(Bit#(TLog#(TAG_DEPTH)))) tag_regs <- replicateM(mkReg(0));
   Vector#(numTags, Reg#(Maybe#(Tuple2#(Bit#(clientWidth),Bit#(6))))) client_map <- replicateM(mkReg(tagged Invalid));
   Maybe#(UInt#(tagWidth)) next_free = findElem(0, readVReg(tag_regs));

   rule return_tag_rule;
      return_fifo.deq;
      let tag = return_fifo.first;
      tag_regs[tag] <= tag_regs[tag]-1;
      if (tag_regs[tag]==1)
	 client_map[tag] <= tagged Invalid;
   endrule

   rule tag_request_rule;
      request_fifo0.deq;
      let client = tpl_2(request_fifo0.first);
      let orig_tag = tpl_3(request_fifo0.first);
      let rv = case  (tpl_1(request_fifo0.first)) matches
		  tagged Valid .tag: return (_when_(tag_regs[tag] < maxBound) (tag));
		  tagged Invalid: return (_when_(isValid(next_free)) (fromMaybe(?, next_free)));
	       endcase;
      request_fifo1.enq(tuple3(rv,client,orig_tag));
   endrule

   method Action tag_request(Bit#(clientWidth) client, Bit#(6) orig_tag) if (request_fifo0.notFull);
      let rv = findElem(tagged Valid tuple2(client,orig_tag), readVReg(client_map));
      request_fifo0.enq(tuple3(rv,client,orig_tag));
   endmethod

   method ActionValue#(Bit#(tagWidth)) tag_response;
      request_fifo1.deq;
      let rv = tpl_1(request_fifo1.first);
      let client = tpl_2(request_fifo1.first);
      let orig_tag = tpl_3(request_fifo1.first);
      client_map[rv] <= tagged Valid tuple2(client,orig_tag);
      tag_regs[rv] <= tag_regs[rv]+1;
      return extend(pack(rv));
   endmethod

   method Action return_tag(Bit#(tagWidth) tag);
      return_fifo.enq(tag);
   endmethod

endmodule

module mkTagGenIO(TagGen#(numClients,numTags))
   provisos(Log#(numTags,tagWidth),
	    Log#(numClients,clientWidth),
	    Bits#(Bit#(clientWidth), tagWidth));

   let request_fifo <- mkSizedFIFO(valueOf(SGL_PIPELINE_DEPTH));

   method Action tag_request(Bit#(clientWidth) client, Bit#(6) client_tag);
      request_fifo.enq(client);
   endmethod

   method ActionValue#(Bit#(tagWidth)) tag_response;
      request_fifo.deq;
      let client = request_fifo.first;
      return pack(client);
   endmethod

   method Action return_tag(Bit#(tagWidth) tag);
      noAction;
   endmethod

endmodule

typedef struct {ObjectRequest req;
		Bit#(TLog#(numClients)) client; } LRec#(numeric type numClients, numeric type numTags, numeric type addrWidth) deriving(Bits);

typedef struct {ObjectRequest req;
		Bit#(addrWidth) pa;
		Bit#(TLog#(numTags)) rename_tag;
		Bit#(TLog#(numClients)) client; } RRec#(numeric type numClients, numeric type numTags, numeric type addrWidth) deriving(Bits);

typedef struct {ObjectRequest req;
		Bit#(TLog#(numTags)) rename_tag;
		Bit#(TLog#(numClients)) client;
		Bool last; } DRec#(numeric type numClients, numeric type numTags, numeric type addrWidth) deriving(Bits);

typedef struct {Bit#(6) orig_tag;
		Bit#(TLog#(numClients)) client; } RResp#(numeric type numClients, numeric type numTags, numeric type addrWidth) deriving(Bits);

module mkMemReadInternal#(Integer id,
			  Vector#(numClients, ObjectReadClient#(dataWidth)) readClients,
			  DmaIndication dmaIndication,
			  Server#(Tuple2#(SGListId,Bit#(ObjectOffsetSize)),Bit#(addrWidth)) sgl,
			  TagGen#(numClients, numTags) tag_gen)
   (MemReadInternal#(addrWidth, dataWidth))

   provisos(Add#(b__, addrWidth, 64),
	    Add#(c__, 12, addrWidth),
	    Add#(1, c__, d__),
	    Div#(dataWidth,8,dataWidthBytes),
	    Mul#(dataWidthBytes,8,dataWidth),
	    Log#(dataWidthBytes,beatShift),
	    Add#(a__, TLog#(numTags), 6));

   // stage 0: address translation (latency = SGL_PIPELINE_DEPTH)
   FIFO#(LRec#(numClients,numTags,addrWidth)) lreqFifo <- mkSizedFIFO(valueOf(SGL_PIPELINE_DEPTH));
   // stage 1: address validation (latency = 1)
   FIFO#(RRec#(numClients,numTags,addrWidth))  reqFifo <- mkFIFO;
   // stage 2: read commands (maximum buffering to handle high latency read response times)
   Vector#(numTags, FIFOF#(DRec#(numClients,numTags,addrWidth))) dreqFifos <- replicateM(mkSizedBRAMFIFOF(valueOf(TAG_DEPTH)));
   // stage 3: read data (minimal buffering required)
   FIFO#(Tuple2#(DRec#(numClients,numTags,addrWidth),MemData#(dataWidth))) readDataPipelineFifo <- mkFIFO;
   FIFO#(MemData#(dataWidth)) responseFifo <- mkFIFO;
   Vector#(numTags, Reg#(Bit#(8)))           burstRegs <- replicateM(mkReg(0));
   Vector#(numTags, Reg#(Bool))              firstRegs <- replicateM(mkReg(True));
   Vector#(numTags, Reg#(Bool))               lastRegs <- replicateM(mkReg(False));
   Reg#(Bit#(64))  beatCount <- mkReg(0);
   let beat_shift = fromInteger(valueOf(beatShift));

   // performance analytics
   Reg#(Bit#(64)) cycle_cnt <- mkReg(0);
   Reg#(Bit#(64)) last_loadClient <- mkReg(0);
   Reg#(Bit#(64)) last_sglResp <- mkReg(0);
   Reg#(Bit#(64)) last_eob <- mkReg(0);
   (* fire_when_enabled *)
   rule cycle;
      cycle_cnt <= cycle_cnt+1;
   endrule

   for (Integer selectReg = 0; selectReg < valueOf(numClients); selectReg = selectReg + 1)
      rule loadClient;
      	 //$display("mkMemReadInternal::loadClient %d %d", selectReg, cycle_cnt-last_loadClient);
	 //last_loadClient <= cycle_cnt;
   	 ObjectRequest req <- readClients[selectReg].readReq.get();
   	 if (bad_pointer(req.pointer))
   	    dmaIndication.dmaError(extend(pack(DmaErrorBadPointer4)), req.pointer, 0, 0);
   	 else begin
	    tag_gen.tag_request(fromInteger(selectReg), req.tag);
   	    lreqFifo.enq(LRec{req:req, client:fromInteger(selectReg)});
   	    sgl.request.put(tuple2(truncate(req.pointer),req.offset));
   	 end
      endrule

   rule checkSglResp;
      let physAddr <- sgl.response.get;
      let req = lreqFifo.first.req;
      let client = lreqFifo.first.client;
      let rename_tag <- tag_gen.tag_response;
      lreqFifo.deq();
      reqFifo.enq(RRec{req:req, pa:physAddr, client:client, rename_tag:extend(rename_tag)});
      //$display("checkSglResp: client=%d, rename_tag=%d", client,rename_tag);
      //$display("mkMemReadInternal::sglResp %d %d", client, cycle_cnt-last_sglResp);
      //last_sglResp <= cycle_cnt;
   endrule

   rule readDataComp;
      readDataPipelineFifo.deq;
      let drq = tpl_1(readDataPipelineFifo.first);
      let response = tpl_2(readDataPipelineFifo.first);
      let client = drq.client;
      let req = drq.req;
      readClients[client].readData.put(ObjectData { data: response.data, tag: req.tag, last: False});
      //$display("readDataComp: %d %h", client, response.data);
      beatCount <= beatCount+1;
   endrule

   rule read_client_response;
      let response <- toGet(responseFifo).get();
      Bit#(6) response_tag = response.tag;
      let dreqFifo = dreqFifos[response_tag];
      dynamicAssert(truncate(response_tag) == dreqFifo.first.rename_tag, "mkMemReadInternal");
      readDataPipelineFifo.enq(tuple2(dreqFifo.first, response));
      let burstLen = burstRegs[response_tag];
      let first =    firstRegs[response_tag];
      let last  =    lastRegs[response_tag];
      if (first) begin
	 burstLen = dreqFifo.first.req.burstLen >> beat_shift;
	 last = dreqFifo.first.last;
      end
      if (last) begin
	 //$display("mkMemReadInternal::eob %d", cycle_cnt-last_eob);
	 last_eob <= cycle_cnt;
	 dreqFifo.deq();
	 tag_gen.return_tag(truncate(response_tag));
      end
      burstRegs[response_tag] <= burstLen-1;
      firstRegs[response_tag] <= (burstLen-1 == 0);
      lastRegs[response_tag] <= (burstLen-1 == 1);
   endrule

   interface MemReadClient read_client;
      interface Get readReq;
	 method ActionValue#(MemRequest#(addrWidth)) get();
	    reqFifo.deq;
	    let req = reqFifo.first.req;
	    let physAddr = reqFifo.first.pa;
	    let client = reqFifo.first.client;
	    let rename_tag = reqFifo.first.rename_tag;
	    if (False && physAddr[31:24] != 0)
	       $display("req_ar: funny physAddr req.pointer=%d req.offset=%h physAddr=%h", req.pointer, req.offset, physAddr);
	    dreqFifos[rename_tag].enq(DRec{req:req, client:client, rename_tag:rename_tag, last:(req.burstLen == beat_shift)});
	    //$display("readReq: client=%d, rename_tag=%d, physAddr=%h", client,rename_tag,physAddr);
	    return MemRequest{addr:physAddr, burstLen:req.burstLen, tag:extend(rename_tag)};
	 endmethod
      endinterface
      interface Put readData = toPut(responseFifo);
   endinterface
   interface DmaDbg dbg;
      method ActionValue#(DmaDbgRec) dbg();
	 // Bit#(1) ne0 = pack(dreqFifos[0].notEmpty);
	 // Bit#(1) ne1 = pack(dreqFifos[1].notEmpty);
	 // return DmaDbgRec{x:extend(burstRegs[0]), y:extend(burstRegs[1]), z:extend(ne0), w:extend(ne1)};
	 return DmaDbgRec{x:0, y:0, z:0, w:0};
      endmethod
      method ActionValue#(Bit#(64)) getMemoryTraffic();
	 return beatCount;
      endmethod
   endinterface
endmodule

module mkMemWriteInternal#(Integer iid,
			   Vector#(numClients, ObjectWriteClient#(dataWidth)) writeClients,
			   DmaIndication dmaIndication,
			   Server#(Tuple2#(SGListId,Bit#(ObjectOffsetSize)),Bit#(addrWidth)) sgl,
			   TagGen#(numClients, numTags) tag_gen)
   (MemWriteInternal#(addrWidth, dataWidth))

   provisos(Add#(b__, addrWidth, 64),
	    Add#(c__, 12, addrWidth),
	    Add#(1, c__, d__),
	    Div#(dataWidth,8,dataWidthBytes),
	    Mul#(dataWidthBytes,8,dataWidth),
	    Log#(dataWidthBytes,beatShift),
	    Add#(a__, TLog#(numTags), 6));

   // stage 0: address translation (latency = SGL_PIPELINE_DEPTH)
   FIFO#(LRec#(numClients,numTags,addrWidth)) lreqFifo <- mkSizedFIFO(valueOf(SGL_PIPELINE_DEPTH));
   // stage 1: address validation (latency = 1)
   FIFO#(RRec#(numClients,numTags,addrWidth))  reqFifo <- mkFIFO;
   // stage 2: write commands (maximum buffering to handle high latency writes)
   FIFO#(DRec#(numClients,numTags,addrWidth)) dreqFifo <- mkSizedBRAMFIFO(valueOf(TMul#(TAG_DEPTH,numTags)));
   // stage 3: write data (maximum buffering, though I have no idea if any hosts will begin the next data transfer before sending the write ack)
   Vector#(numTags, FIFO#(RResp#(numClients,numTags,addrWidth))) respFifos <- replicateM(mkSizedBRAMFIFO(valueOf(TAG_DEPTH)));
   // stage 4: write done (minimal buffering required)
   FIFO#(Tuple2#(RResp#(numClients,numTags,addrWidth),Bit#(6))) writeDonePipelineFifo <- mkFIFO;

   Reg#(Bit#(8)) burstReg <- mkReg(0);
   Reg#(Bit#(64)) beatCount <- mkReg(0);
   let beat_shift = fromInteger(valueOf(beatShift));

   Reg#(Bit#(64)) cycle_cnt <- mkReg(0);
   Reg#(Bit#(64)) last_loadClient <- mkReg(0);
   Reg#(Bit#(64)) last_sglResp <- mkReg(0);
   Reg#(Bit#(64)) last_eob <- mkReg(0);
   (* fire_when_enabled *)
   rule cycle;
      cycle_cnt <= cycle_cnt+1;
   endrule

   for (Integer selectReg = 0; selectReg < valueOf(numClients); selectReg = selectReg + 1)
       rule loadClient;
      	  //$display("mkMemWriteInternal::loadClient %d %d", selectReg, cycle_cnt-last_loadClient);
	  //last_loadClient <= cycle_cnt;
   	  ObjectRequest req <- writeClients[selectReg].writeReq.get();
   	  if (bad_pointer(req.pointer))
   	     dmaIndication.dmaError(extend(pack(DmaErrorBadPointer5)), req.pointer, 0, 0);
   	  else begin
	     tag_gen.tag_request(fromInteger(selectReg), req.tag);
   	     lreqFifo.enq(LRec{req:req, client:fromInteger(selectReg)});
   	     sgl.request.put(tuple2(truncate(req.pointer),req.offset));
   	  end
       endrule

   rule checkSglResp;
      let physAddr <- sgl.response.get;
      let req = lreqFifo.first.req;
      let client = lreqFifo.first.client;
      let rename_tag <- tag_gen.tag_response;
      lreqFifo.deq();
      reqFifo.enq(RRec{req:req, pa:physAddr, client:client, rename_tag:extend(rename_tag)});
      //$display("checkSglResp: client=%d, rename_tag=%d", client,rename_tag);
      //$display("mkMemWriteInternal::sglResp %d %d", client, cycle_cnt-last_sglResp);
      //last_sglResp <= cycle_cnt;
   endrule

   rule writeDoneComp;
      writeDonePipelineFifo.deq;
      let client = tpl_1(writeDonePipelineFifo.first).client;
      let orig_tag = tpl_1(writeDonePipelineFifo.first).orig_tag;
      let response_tag = tpl_2(writeDonePipelineFifo.first);
      writeClients[client].writeDone.put(orig_tag);
      tag_gen.return_tag(truncate(response_tag));
   endrule

   interface MemWriteClient write_client;
      interface Get writeReq;
	 method ActionValue#(MemRequest#(addrWidth)) get();
	    let req = reqFifo.first.req;
	    let physAddr = reqFifo.first.pa;
	    let client = reqFifo.first.client;
	    let rename_tag = reqFifo.first.rename_tag;
	    reqFifo.deq;
	    dreqFifo.enq(DRec{req:req, client:client, rename_tag:rename_tag});
	    //$display("writeReq: client=%d, rename_tag=%d", client,rename_tag);
	    return MemRequest{addr:physAddr, burstLen:req.burstLen, tag:extend(rename_tag)};
	 endmethod
      endinterface
      interface Get writeData;
	 method ActionValue#(MemData#(dataWidth)) get();
	    let client = dreqFifo.first.client;
	    let req = dreqFifo.first.req;
	    let rename_tag =dreqFifo.first.rename_tag;
	    ObjectData#(dataWidth) tagdata <- writeClients[client].writeData.get();
	    let burstLen = burstReg;
	    if (burstLen == 0) begin
	       burstLen = req.burstLen >> beat_shift;
	       respFifos[rename_tag].enq(RResp{orig_tag:req.tag, client:client});
	    end
	    burstReg <= burstLen-1;
	    beatCount <= beatCount+1;
	    if (burstLen == 1)
	       dreqFifo.deq();
	    //$display("writeData: client=%d, rename_tag=%d", client, rename_tag);
	    return MemData { data: tagdata.data,  tag:extend(rename_tag), last: False };
	 endmethod
      endinterface
      interface Put writeDone;
	 method Action put(Bit#(6) resp);
	    let response_tag = resp;
	    let respFifo = respFifos[response_tag];
	    writeDonePipelineFifo.enq(tuple2(respFifo.first,resp));
	    respFifo.deq;
	 endmethod
      endinterface
   endinterface
   interface DmaDbg dbg;
      method ActionValue#(DmaDbgRec) dbg();
	 return DmaDbgRec{x:fromInteger(valueOf(numClients)), y:?, z:?, w:?};
      endmethod
      method ActionValue#(Bit#(64)) getMemoryTraffic();
	 return beatCount;
      endmethod
   endinterface
endmodule

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

// BSV Libraries
import BRAMFIFO::*;
import FIFO::*;
import FIFOF::*;
import Vector::*;
import GetPut::*;
import ClientServer::*;
import Assert::*;

// XBSV Libraries
import MemTypes::*;
import PortalMemory::*;
import SGList::*;

typedef 9 SGL_PIPELINE_DEPTH;
typedef 32 TAG_DEPTH;

interface MemWriteInternal#(numeric type addrWidth, numeric type dataWidth);
   interface DmaDbg dbg;
   interface MemWriteClient#(addrWidth,dataWidth) write_client;
endinterface

interface MemReadInternal#(numeric type addrWidth, numeric type dataWidth);
   interface DmaDbg dbg;
   interface MemReadClient#(addrWidth,dataWidth) read_client;
endinterface

function Bool bad_pointer(ObjectPointer p);
   return ((p >>8) > fromInteger(valueOf(MaxNumSGLists)));
endfunction

interface TagGen#(numeric type numClients, numeric type numTags);
   method Action tag_request(Bit#(TLog#(numClients)) client, Bit#(6) orig_tag);
   method ActionValue#(Bit#(TLog#(numTags)))  tag_response;
   method Action return_tag(Bit#(TLog#(numTags)) tag);
endinterface

module mkTagGenOO(TagGen#(numClients,numTags))
   provisos(Log#(numTags,tagWidth),
	    Log#(numClients,clientWidth));

   let request_fifo0 <- mkSizedFIFOF(valueOf(SGL_PIPELINE_DEPTH));
   let request_fifo1 <- mkSizedFIFOF(valueOf(SGL_PIPELINE_DEPTH));
   let return_fifo <- mkFIFO;
   Vector#(numTags, Reg#(Bit#(TLog#(TAG_DEPTH)))) tag_regs <- replicateM(mkReg(0));
   Vector#(numTags, Reg#(Maybe#(Tuple2#(Bit#(clientWidth),Bit#(6))))) client_map <- replicateM(mkReg(tagged Invalid));
   Maybe#(UInt#(tagWidth)) next_free = findElem(0, readVReg(tag_regs));

   rule return_tag_rule;
      return_fifo.deq;
      let tag = return_fifo.first;
      tag_regs[tag] <= tag_regs[tag]-1;
      if (tag_regs[tag]==1)
	 client_map[tag] <= tagged Invalid;
   endrule

   rule tag_request_rule;
      request_fifo0.deq;
      let client = tpl_2(request_fifo0.first);
      let orig_tag = tpl_3(request_fifo0.first);
      let rv = case  (tpl_1(request_fifo0.first)) matches
		  tagged Valid .tag: return (_when_(tag_regs[tag] < maxBound) (tag));
		  tagged Invalid: return (_when_(isValid(next_free)) (fromMaybe(?, next_free)));
	       endcase;
      request_fifo1.enq(tuple3(rv,client,orig_tag));
   endrule

   method Action tag_request(Bit#(clientWidth) client, Bit#(6) orig_tag) if (request_fifo0.notFull);
      let rv = findElem(tagged Valid tuple2(client,orig_tag), readVReg(client_map));
      request_fifo0.enq(tuple3(rv,client,orig_tag));
   endmethod

   method ActionValue#(Bit#(tagWidth)) tag_response;
      request_fifo1.deq;
      let rv = tpl_1(request_fifo1.first);
      let client = tpl_2(request_fifo1.first);
      let orig_tag = tpl_3(request_fifo1.first);
      client_map[rv] <= tagged Valid tuple2(client,orig_tag);
      tag_regs[rv] <= tag_regs[rv]+1;
      return extend(pack(rv));
   endmethod

   method Action return_tag(Bit#(tagWidth) tag);
      return_fifo.enq(tag);
   endmethod

endmodule

module mkTagGenIO(TagGen#(numClients,numTags))
   provisos(Log#(numTags,tagWidth),
	    Log#(numClients,clientWidth),
	    Bits#(Bit#(clientWidth), tagWidth));

   let request_fifo <- mkSizedFIFO(valueOf(SGL_PIPELINE_DEPTH));

   method Action tag_request(Bit#(clientWidth) client, Bit#(6) client_tag);
      request_fifo.enq(client);
   endmethod

   method ActionValue#(Bit#(tagWidth)) tag_response;
      request_fifo.deq;
      let client = request_fifo.first;
      return pack(client);
   endmethod

   method Action return_tag(Bit#(tagWidth) tag);
      noAction;
   endmethod

endmodule

typedef struct {ObjectRequest req;
		Bit#(TLog#(numClients)) client; } LRec#(numeric type numClients, numeric type numTags, numeric type addrWidth) deriving(Bits);

typedef struct {ObjectRequest req;
		Bit#(addrWidth) pa;
		Bit#(TLog#(numTags)) rename_tag;
		Bit#(TLog#(numClients)) client; } RRec#(numeric type numClients, numeric type numTags, numeric type addrWidth) deriving(Bits);

typedef struct {ObjectRequest req;
		Bit#(TLog#(numTags)) rename_tag;
		Bit#(TLog#(numClients)) client; } DRec#(numeric type numClients, numeric type numTags, numeric type addrWidth) deriving(Bits);

typedef struct {Bit#(6) orig_tag;
		Bit#(TLog#(numClients)) client; } RResp#(numeric type numClients, numeric type numTags, numeric type addrWidth) deriving(Bits);

module mkMemReadInternal#(Integer id,
			  Vector#(numClients, ObjectReadClient#(dataWidth)) readClients,
			  DmaIndication dmaIndication,
			  Server#(Tuple2#(SGListId,Bit#(ObjectOffsetSize)),Bit#(addrWidth)) sgl,
			  TagGen#(numClients, numTags) tag_gen)
   (MemReadInternal#(addrWidth, dataWidth))

   provisos(Add#(b__, addrWidth, 64),
	    Add#(c__, 12, addrWidth),
	    Add#(1, c__, d__),
	    Div#(dataWidth,8,dataWidthBytes),
	    Mul#(dataWidthBytes,8,dataWidth),
	    Log#(dataWidthBytes,beatShift),
	    Add#(a__, TLog#(numTags), 6));

   // stage 0: address translation (latency = SGL_PIPELINE_DEPTH)
   FIFO#(LRec#(numClients,numTags,addrWidth)) lreqFifo <- mkSizedFIFO(valueOf(SGL_PIPELINE_DEPTH));
   // stage 1: address validation (latency = 1)
   FIFO#(RRec#(numClients,numTags,addrWidth))  reqFifo <- mkFIFO;
   // stage 2: read commands (maximum buffering to handle high latency read response times)
   Vector#(numTags, FIFOF#(DRec#(numClients,numTags,addrWidth))) dreqFifos <- replicateM(mkSizedBRAMFIFOF(valueOf(TAG_DEPTH)));
   // stage 3: read data (minimal buffering required)
   FIFO#(Tuple2#(DRec#(numClients,numTags,addrWidth),MemData#(dataWidth))) readDataPipelineFifo <- mkFIFO;
   FIFO#(MemData#(dataWidth)) responseFifo <- mkFIFO;
   Vector#(numTags, Reg#(Bit#(8)))           burstRegs <- replicateM(mkReg(0));
   Reg#(Bit#(64))  beatCount <- mkReg(0);
   let beat_shift = fromInteger(valueOf(beatShift));

   // performance analytics
   Reg#(Bit#(64)) cycle_cnt <- mkReg(0);
   Reg#(Bit#(64)) last_loadClient <- mkReg(0);
   Reg#(Bit#(64)) last_sglResp <- mkReg(0);
   Reg#(Bit#(64)) last_eob <- mkReg(0);
   (* fire_when_enabled *)
   rule cycle;
      cycle_cnt <= cycle_cnt+1;
   endrule

   for (Integer selectReg = 0; selectReg < valueOf(numClients); selectReg = selectReg + 1)
      rule loadClient;
      	 //$display("mkMemReadInternal::loadClient %d %d", selectReg, cycle_cnt-last_loadClient);
	 //last_loadClient <= cycle_cnt;
   	 ObjectRequest req <- readClients[selectReg].readReq.get();
   	 if (bad_pointer(req.pointer))
   	    dmaIndication.dmaError(extend(pack(DmaErrorBadPointer4)), req.pointer, 0, 0);
   	 else begin
	    tag_gen.tag_request(fromInteger(selectReg), req.tag);
   	    lreqFifo.enq(LRec{req:req, client:fromInteger(selectReg)});
   	    sgl.request.put(tuple2(truncate(req.pointer),req.offset));
   	 end
      endrule

   rule checkSglResp;
      let physAddr <- sgl.response.get;
      let req = lreqFifo.first.req;
      let client = lreqFifo.first.client;
      let rename_tag <- tag_gen.tag_response;
      lreqFifo.deq();
      reqFifo.enq(RRec{req:req, pa:physAddr, client:client, rename_tag:extend(rename_tag)});
      //$display("checkSglResp: client=%d, rename_tag=%d", client,rename_tag);
      //$display("mkMemReadInternal::sglResp %d %d", client, cycle_cnt-last_sglResp);
      //last_sglResp <= cycle_cnt;
   endrule

   rule readDataComp;
      readDataPipelineFifo.deq;
      let drq = tpl_1(readDataPipelineFifo.first);
      let response = tpl_2(readDataPipelineFifo.first);
      let client = drq.client;
      let req = drq.req;
      readClients[client].readData.put(ObjectData { data: response.data, tag: req.tag, last: False});
      //$display("readDataComp: %d %h", client, response.data);
      beatCount <= beatCount+1;
   endrule

   rule read_client_response;
      let response <- toGet(responseFifo).get();
      Bit#(6) response_tag = response.tag;
      let dreqFifo = dreqFifos[response_tag];
      dynamicAssert(truncate(response_tag) == dreqFifo.first.rename_tag, "mkMemReadInternal");
      readDataPipelineFifo.enq(tuple2(dreqFifo.first, response));
      let burstLen = burstRegs[response_tag];
      if (burstLen == 0)
	 burstLen = dreqFifo.first.req.burstLen >> beat_shift;
      if (burstLen == 1) begin
	 //$display("mkMemReadInternal::eob %d", cycle_cnt-last_eob);
	 last_eob <= cycle_cnt;
	 dreqFifo.deq();
	 tag_gen.return_tag(truncate(response_tag));
      end
      burstRegs[response_tag] <= burstLen-1;
   endrule

   interface MemReadClient read_client;
      interface Get readReq;
	 method ActionValue#(MemRequest#(addrWidth)) get();
	    reqFifo.deq;
	    let req = reqFifo.first.req;
	    let physAddr = reqFifo.first.pa;
	    let client = reqFifo.first.client;
	    let rename_tag = reqFifo.first.rename_tag;
	    if (False && physAddr[31:24] != 0)
	       $display("req_ar: funny physAddr req.pointer=%d req.offset=%h physAddr=%h", req.pointer, req.offset, physAddr);
	    dreqFifos[rename_tag].enq(DRec{req:req, client:client, rename_tag:rename_tag});
	    //$display("readReq: client=%d, rename_tag=%d, physAddr=%h", client,rename_tag,physAddr);
	    return MemRequest{addr:physAddr, burstLen:req.burstLen, tag:extend(rename_tag)};
	 endmethod
      endinterface
      interface Put readData = toPut(responseFifo);
   endinterface
   interface DmaDbg dbg;
      method ActionValue#(DmaDbgRec) dbg();
	 // Bit#(1) ne0 = pack(dreqFifos[0].notEmpty);
	 // Bit#(1) ne1 = pack(dreqFifos[1].notEmpty);
	 // return DmaDbgRec{x:extend(burstRegs[0]), y:extend(burstRegs[1]), z:extend(ne0), w:extend(ne1)};
	 return DmaDbgRec{x:0, y:0, z:0, w:0};
      endmethod
      method ActionValue#(Bit#(64)) getMemoryTraffic();
	 return beatCount;
      endmethod
   endinterface
endmodule

module mkMemWriteInternal#(Integer iid,
			   Vector#(numClients, ObjectWriteClient#(dataWidth)) writeClients,
			   DmaIndication dmaIndication,
			   Server#(Tuple2#(SGListId,Bit#(ObjectOffsetSize)),Bit#(addrWidth)) sgl,
			   TagGen#(numClients, numTags) tag_gen)
   (MemWriteInternal#(addrWidth, dataWidth))

   provisos(Add#(b__, addrWidth, 64),
	    Add#(c__, 12, addrWidth),
	    Add#(1, c__, d__),
	    Div#(dataWidth,8,dataWidthBytes),
	    Mul#(dataWidthBytes,8,dataWidth),
	    Log#(dataWidthBytes,beatShift),
	    Add#(a__, TLog#(numTags), 6));

   // stage 0: address translation (latency = SGL_PIPELINE_DEPTH)
   FIFO#(LRec#(numClients,numTags,addrWidth)) lreqFifo <- mkSizedFIFO(valueOf(SGL_PIPELINE_DEPTH));
   // stage 1: address validation (latency = 1)
   FIFO#(RRec#(numClients,numTags,addrWidth))  reqFifo <- mkFIFO;
   // stage 2: write commands (maximum buffering to handle high latency writes)
   FIFO#(DRec#(numClients,numTags,addrWidth)) dreqFifo <- mkSizedBRAMFIFO(valueOf(TMul#(TAG_DEPTH,numTags)));
   // stage 3: write data (maximum buffering, though I have no idea if any hosts will begin the next data transfer before sending the write ack)
   Vector#(numTags, FIFO#(RResp#(numClients,numTags,addrWidth))) respFifos <- replicateM(mkSizedBRAMFIFO(valueOf(TAG_DEPTH)));
   // stage 4: write done (minimal buffering required)
   FIFO#(Tuple2#(RResp#(numClients,numTags,addrWidth),Bit#(6))) writeDonePipelineFifo <- mkFIFO;

   Reg#(Bit#(8)) burstReg <- mkReg(0);
   Reg#(Bit#(64)) beatCount <- mkReg(0);
   let beat_shift = fromInteger(valueOf(beatShift));

   Reg#(Bit#(64)) cycle_cnt <- mkReg(0);
   Reg#(Bit#(64)) last_loadClient <- mkReg(0);
   Reg#(Bit#(64)) last_sglResp <- mkReg(0);
   Reg#(Bit#(64)) last_eob <- mkReg(0);
   (* fire_when_enabled *)
   rule cycle;
      cycle_cnt <= cycle_cnt+1;
   endrule

   for (Integer selectReg = 0; selectReg < valueOf(numClients); selectReg = selectReg + 1)
       rule loadClient;
      	  //$display("mkMemWriteInternal::loadClient %d %d", selectReg, cycle_cnt-last_loadClient);
	  //last_loadClient <= cycle_cnt;
   	  ObjectRequest req <- writeClients[selectReg].writeReq.get();
   	  if (bad_pointer(req.pointer))
   	     dmaIndication.dmaError(extend(pack(DmaErrorBadPointer5)), req.pointer, 0, 0);
   	  else begin
	     tag_gen.tag_request(fromInteger(selectReg), req.tag);
   	     lreqFifo.enq(LRec{req:req, client:fromInteger(selectReg)});
   	     sgl.request.put(tuple2(truncate(req.pointer),req.offset));
   	  end
       endrule

   rule checkSglResp;
      let physAddr <- sgl.response.get;
      let req = lreqFifo.first.req;
      let client = lreqFifo.first.client;
      let rename_tag <- tag_gen.tag_response;
      lreqFifo.deq();
      reqFifo.enq(RRec{req:req, pa:physAddr, client:client, rename_tag:extend(rename_tag)});
      //$display("checkSglResp: client=%d, rename_tag=%d", client,rename_tag);
      //$display("mkMemWriteInternal::sglResp %d %d", client, cycle_cnt-last_sglResp);
      //last_sglResp <= cycle_cnt;
   endrule

   rule writeDoneComp;
      writeDonePipelineFifo.deq;
      let client = tpl_1(writeDonePipelineFifo.first).client;
      let orig_tag = tpl_1(writeDonePipelineFifo.first).orig_tag;
      let response_tag = tpl_2(writeDonePipelineFifo.first);
      writeClients[client].writeDone.put(orig_tag);
      tag_gen.return_tag(truncate(response_tag));
   endrule

   interface MemWriteClient write_client;
      interface Get writeReq;
	 method ActionValue#(MemRequest#(addrWidth)) get();
	    let req = reqFifo.first.req;
	    let physAddr = reqFifo.first.pa;
	    let client = reqFifo.first.client;
	    let rename_tag = reqFifo.first.rename_tag;
	    reqFifo.deq;
	    dreqFifo.enq(DRec{req:req, client:client, rename_tag:rename_tag});
	    //$display("writeReq: client=%d, rename_tag=%d", client,rename_tag);
	    return MemRequest{addr:physAddr, burstLen:req.burstLen, tag:extend(rename_tag)};
	 endmethod
      endinterface
      interface Get writeData;
	 method ActionValue#(MemData#(dataWidth)) get();
	    let client = dreqFifo.first.client;
	    let req = dreqFifo.first.req;
	    let rename_tag =dreqFifo.first.rename_tag;
	    ObjectData#(dataWidth) tagdata <- writeClients[client].writeData.get();
	    let burstLen = burstReg;
	    if (burstLen == 0) begin
	       burstLen = req.burstLen >> beat_shift;
	       respFifos[rename_tag].enq(RResp{orig_tag:req.tag, client:client});
	    end
	    burstReg <= burstLen-1;
	    beatCount <= beatCount+1;
	    if (burstLen == 1)
	       dreqFifo.deq();
	    //$display("writeData: client=%d, rename_tag=%d", client, rename_tag);
	    return MemData { data: tagdata.data,  tag:extend(rename_tag), last: False };
	 endmethod
      endinterface
      interface Put writeDone;
	 method Action put(Bit#(6) resp);
	    let response_tag = resp;
	    let respFifo = respFifos[response_tag];
	    writeDonePipelineFifo.enq(tuple2(respFifo.first,resp));
	    respFifo.deq;
	 endmethod
      endinterface
   endinterface
   interface DmaDbg dbg;
      method ActionValue#(DmaDbgRec) dbg();
	 return DmaDbgRec{x:fromInteger(valueOf(numClients)), y:?, z:?, w:?};
      endmethod
      method ActionValue#(Bit#(64)) getMemoryTraffic();
	 return beatCount;
      endmethod
   endinterface
endmodule


flow control

flow control

// Copyright (c) 2014 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import Complex::*;
import FixedPoint::*;
//import StmtFSM::*;
import BRAM::*;
import Pipe::*;
import Vector::*;

typedef Complex#(FixedPoint#(2,23)) DDSOutType;
typedef FixedPoint#(2,23) PhaseType;

interface DDS;
   method Action setPhaseAdvance(FixedPoint#(10,32) v);
   interface PipeOut#(DDSOutType) osc;
endinterface

module mkDDS(DDS);
   BRAM_Configure cfg = defaultValue;
   cfg.memorySize = 1024;
   cfg.loadFormat = tagged Binary "sine.bin";
   BRAM1Port#(Bit#(10), DDSOutType) ram <-mkBRAM1Server(cfg);
   FIFO#(DDSOutType) ddsout <- mkFIFO();
   Reg#(PhaseType) phase <- mkReg(0);
   Reg#(PhaseType) phaseAdvance <- mkReg(0);
      /*
   Reg#(UInt#(12)) idx <- mkReg(0);
   Stmt dumpRam =
   seq
      for (idx <= 0; idx < 1024; idx <= idx + 1)
	 seq
	    ram.portA.request.put(BRAMRequest{write: False, responseOnWrite: False, address: truncate(pack(idx)), datain: ?});
	   action
	      let v <- ram.portA.response.get();
	      $display("adr %d", idx);
	      $write( "re " ) ; fxptWrite( 10, v.rel ) ; $display("" ) ;
	      $write( "im " ) ; fxptWrite( 10, v.img ) ; $display("" ) ;
	      endaction
	    endseq
   endseq;

   mkAutoFSM (dumpRam);
   */

   rule filter_phase;
      phase <= phase + phaseAdvance;
      ram.portA.request.put(BRAMRequest{write: False, responseOnWrite: False, address: truncate(rotateBitsBy(phase.f, 10)), datain: ?});
   endrule

   rule ddsoutrule;
      let v <- ram.portA.response.get();
      ddsout.enq(v);
   endrule

   method Action setPhaseAdvance(FixedPoint#(10,32) v);
      phaseAdvance <= PhaseType{i: truncate(v.i), f: truncate(rotateBitsBy(v.f, 10))};
      endmethod
   interface PipeOut osc = toPipeOut(ddsout);

endmodule

// Copyright (c) 2014 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import Complex::*;
import FixedPoint::*;
//import StmtFSM::*;
import BRAM::*;
import Pipe::*;
import Vector::*;

typedef Complex#(FixedPoint#(2,23)) DDSOutType;
typedef FixedPoint#(2,23) PhaseType;

interface DDS;
   method Action setPhaseAdvance(FixedPoint#(10,32) v);
   interface PipeOut#(DDSOutType) osc;
endinterface

module mkDDS(DDS);
   BRAM_Configure cfg = defaultValue;
   cfg.memorySize = 1024;
   cfg.loadFormat = tagged Binary "sine.bin";
   BRAM1Port#(Bit#(10), DDSOutType) ram <-mkBRAM1Server(cfg);
   Reg#(DDSOutType) ddsout <- mkReg(?);
   Reg#(PhaseType) phase <- mkReg(0);
   Reg#(PhaseType) phaseAdvance <- mkReg(0);
      /*
   Reg#(UInt#(12)) idx <- mkReg(0);
   Stmt dumpRam =
   seq
      for (idx <= 0; idx < 1024; idx <= idx + 1)
	 seq
	    ram.portA.request.put(BRAMRequest{write: False, responseOnWrite: False, address: truncate(pack(idx)), datain: ?});
	   action
	      let v <- ram.portA.response.get();
	      $display("adr %d", idx);
	      $write( "re " ) ; fxptWrite( 10, v.rel ) ; $display("" ) ;
	      $write( "im " ) ; fxptWrite( 10, v.img ) ; $display("" ) ;
	      endaction
	    endseq
   endseq;

   mkAutoFSM (dumpRam);
   */

   rule filter_phase;
      phase <= phase + phaseAdvance;
      ram.portA.request.put(BRAMRequest{write: False, responseOnWrite: False, address: truncate(rotateBitsBy(phase.f, 10)), datain: ?});
   endrule

   rule ddsoutrule;
      let v <- ram.portA.response.get();
      ddsout <= v;
   endrule

   method Action setPhaseAdvance(FixedPoint#(10,32) v);
      phaseAdvance <= PhaseType{i: truncate(v.i), f: truncate(rotateBitsBy(v.f, 10))};
      endmethod
   interface PipeOut osc = toPipeOut(ddsout);

endmodule