fill in other bus width encodings in axiBusSize

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import FIFOF::*;
import FIFO::*;
import GetPut::*;
import GetPutWithClocks::*;
import Connectable::*;
import RegFile::*;

typeclass RegToWriteOnly#(type a);
   function WriteOnly#(a) regToWriteOnly(Reg#(a) x);
endtypeclass

instance RegToWriteOnly#(a);
   function WriteOnly#(a) regToWriteOnly(Reg#(a) x);
      return (interface WriteOnly;
		 method Action _write(a v);
		    x._write(v);
		 endmethod
	      endinterface);
   endfunction
endinstance

typedef struct {
   Bit#(addrWidth) address;
   Bit#(4) len;
   Bit#(3) size; // assume matches bus width of Axi3Master
   Bit#(2) burst;  // drive with 2'b01
   Bit#(3) prot; // drive with 3'b000
   Bit#(4) cache; // drive with 4'b0011
   Bit#(idWidth) id;
   Bit#(2) lock;
   Bit#(4) qos;
} Axi3ReadRequest#(numeric type addrWidth, numeric type idWidth) deriving (Bits);

function Bit#(3) axiBusSize(Integer busWidth);
   if (busWidth == 32)
      return 3'b010; // 3'b010: 32bit, 3'b011: 64bit, 3'b100: 128bit
   else if (busWidth == 64)
      return 3'b011;
   else if (busWidth == 128)
      return 3'b100;
   else if (busWidth == 256)
      return 3'b101;
   else if (busWidth == 512)
      return 3'b110;
   else if (busWidth == 1024)
      return 3'b111;
   else
      return 3'b000;
endfunction

typedef struct {
   Bit#(busWidth) data;
   Bit#(2) resp;
   Bit#(1) last;
   Bit#(idWidth) id;
} Axi3ReadResponse#(numeric type busWidth, numeric type idWidth) deriving (Bits);

typedef struct {
   Bit#(addrWidth) address;
   Bit#(4) len;
   Bit#(3) size; // assume matches bus width of Axi3Master
   Bit#(2) burst;  // drive with 2'b01
   Bit#(3) prot; // drive with 3'b000
   Bit#(4) cache; // drive with 4'b0011
   Bit#(idWidth) id;
   Bit#(2) lock;
   Bit#(4) qos;
} Axi3WriteRequest#(numeric type addrWidth, numeric type idWidth) deriving (Bits);

typedef struct {
    Bit#(busWidth) data;
    Bit#(TDiv#(busWidth,8)) byteEnable;
    Bit#(1)        last;
    Bit#(idWidth) id;
} Axi3WriteData#(numeric type busWidth, numeric type idWidth) deriving (Bits);

typedef struct {
    Bit#(2) resp;
    Bit#(idWidth) id;
} Axi3WriteResponse#(numeric type idWidth) deriving (Bits);

function Get#(Axi3ReadResponse#(_b,_c)) get_resp_read(Axi3Slave#(_a,_b,_c) x);
   return x.resp_read;
endfunction

function Get#(Axi3WriteResponse#(_c)) get_resp_b(Axi3Slave#(_a,_b,_c) x);
   return x.resp_b;
endfunction

interface Axi3Master#(numeric type addrWidth, numeric type busWidth, numeric type idWidth);
   interface Get#(Axi3ReadRequest#(addrWidth, idWidth)) req_ar;
   interface Put#(Axi3ReadResponse#(busWidth, idWidth)) resp_read;
   interface Get#(Axi3WriteRequest#(addrWidth, idWidth)) req_aw;
   interface Get#(Axi3WriteData#(busWidth, idWidth)) resp_write;
   interface Put#(Axi3WriteResponse#(idWidth)) resp_b;
endinterface

interface Axi3Slave#(numeric type addrWidth, numeric type busWidth, numeric type idWidth);
   interface Put#(Axi3ReadRequest#(addrWidth, idWidth)) req_ar;
   interface Get#(Axi3ReadResponse#(busWidth, idWidth)) resp_read;
   interface Put#(Axi3WriteRequest#(addrWidth, idWidth)) req_aw;
   interface Put#(Axi3WriteData#(busWidth, idWidth)) resp_write;
   interface Get#(Axi3WriteResponse#(idWidth)) resp_b;
endinterface

function Put#(t) null_put();
   return (interface Put;
	      method Action put(t x) if (False);
		 noAction;
	      endmethod
	   endinterface);
endfunction

function Get#(t) null_get();
   return (interface Get;
	      method ActionValue#(t) get() if (False);
		 return ?;
	      endmethod
	   endinterface);
endfunction

function  Axi3Master#(addrWidth, busWidth, idWidth) null_axi_master();
   return (interface Axi3Master;
	      interface Get req_ar = null_get;
	      interface Put resp_read = null_put;
	      interface Get req_aw = null_get;
	      interface Get resp_write = null_get;
	      interface Put resp_b = null_put;
	   endinterface);
endfunction

instance Connectable#(Axi3Master#(addrWidth, busWidth,idWidth), Axi3Slave#(addrWidth, busWidth,idWidth));
   module mkConnection#(Axi3Master#(addrWidth, busWidth,idWidth) m, Axi3Slave#(addrWidth, busWidth,idWidth) s)(Empty);

      mkConnection(m.req_ar, s.req_ar);
      mkConnection(s.resp_read, m.resp_read);

      mkConnection(m.req_aw, s.req_aw);
      mkConnection(m.resp_write, s.resp_write);
      mkConnection(s.resp_b, m.resp_b);

   endmodule
endinstance

instance ConnectableWithClocks#(Axi3Master#(addrWidth, busWidth,idWidth), Axi3Slave#(addrWidth, busWidth,idWidth))
   provisos (ConnectableWithClocks#(Get#(Axi3ReadRequest#(addrWidth, idWidth)), Put#(Axi3ReadRequest#(addrWidth, idWidth)))
	     ,ConnectableWithClocks#(Get#(Axi3ReadResponse#(busWidth, idWidth)), Put#(Axi3ReadResponse#(busWidth, idWidth)))
	     ,ConnectableWithClocks#(Get#(Axi3WriteRequest#(addrWidth, idWidth)), Put#(Axi3WriteRequest#(addrWidth, idWidth)))
	     ,ConnectableWithClocks#(Get#(Axi3WriteData#(busWidth, idWidth)), Put#(Axi3WriteData#(busWidth, idWidth)))
	     ,ConnectableWithClocks#(Get#(Axi3WriteResponse#(idWidth)), Put#(Axi3WriteResponse#(idWidth)))
	     );

   module mkConnectionWithClocks2#(Axi3Master#(addrWidth, busWidth,idWidth) m, Axi3Slave#(addrWidth, busWidth,idWidth) s)(Empty);
      mkConnectionWithClocks(clockOf(m), resetOf(m), clockOf(s), resetOf(s), m, s);
   endmodule

   module mkConnectionWithClocks#(Clock mClock, Reset mReset, Clock sClock, Reset sReset,
				  Axi3Master#(addrWidth, busWidth,idWidth) m, Axi3Slave#(addrWidth, busWidth,idWidth) s)(Empty);

      mkConnectionWithClocks(mClock, mReset, sClock, sReset, m.req_ar, s.req_ar);
      mkConnectionWithClocks(sClock, sReset, mClock, mReset, s.resp_read, m.resp_read);

      mkConnectionWithClocks(mClock, mReset, sClock, sReset, m.req_aw, s.req_aw);
      mkConnectionWithClocks(mClock, mReset, sClock, sReset, m.resp_write, s.resp_write);
      mkConnectionWithClocks(sClock, sReset, mClock, mReset, s.resp_b, m.resp_b);

   endmodule
endinstance

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import FIFOF::*;
import FIFO::*;
import GetPut::*;
import GetPutWithClocks::*;
import Connectable::*;
import RegFile::*;

typeclass RegToWriteOnly#(type a);
   function WriteOnly#(a) regToWriteOnly(Reg#(a) x);
endtypeclass

instance RegToWriteOnly#(a);
   function WriteOnly#(a) regToWriteOnly(Reg#(a) x);
      return (interface WriteOnly;
		 method Action _write(a v);
		    x._write(v);
		 endmethod
	      endinterface);
   endfunction
endinstance

typedef struct {
   Bit#(addrWidth) address;
   Bit#(4) len;
   Bit#(3) size; // assume matches bus width of Axi3Master
   Bit#(2) burst;  // drive with 2'b01
   Bit#(3) prot; // drive with 3'b000
   Bit#(4) cache; // drive with 4'b0011
   Bit#(idWidth) id;
   Bit#(2) lock;
   Bit#(4) qos;
} Axi3ReadRequest#(numeric type addrWidth, numeric type idWidth) deriving (Bits);

function Bit#(3) axiBusSize(Integer busWidth);
   if (busWidth == 32)
      return 3'b010; // 3'b010: 32bit, 3'b011: 64bit, 3'b100: 128bit
   else if (busWidth == 64)
      return 3'b011;
   else if (busWidth == 128)
      return 3'b100;
   else
      return 3'b011;
endfunction

typedef struct {
   Bit#(busWidth) data;
   Bit#(2) resp;
   Bit#(1) last;
   Bit#(idWidth) id;
} Axi3ReadResponse#(numeric type busWidth, numeric type idWidth) deriving (Bits);

typedef struct {
   Bit#(addrWidth) address;
   Bit#(4) len;
   Bit#(3) size; // assume matches bus width of Axi3Master
   Bit#(2) burst;  // drive with 2'b01
   Bit#(3) prot; // drive with 3'b000
   Bit#(4) cache; // drive with 4'b0011
   Bit#(idWidth) id;
   Bit#(2) lock;
   Bit#(4) qos;
} Axi3WriteRequest#(numeric type addrWidth, numeric type idWidth) deriving (Bits);

typedef struct {
    Bit#(busWidth) data;
    Bit#(TDiv#(busWidth,8)) byteEnable;
    Bit#(1)        last;
    Bit#(idWidth) id;
} Axi3WriteData#(numeric type busWidth, numeric type idWidth) deriving (Bits);

typedef struct {
    Bit#(2) resp;
    Bit#(idWidth) id;
} Axi3WriteResponse#(numeric type idWidth) deriving (Bits);

function Get#(Axi3ReadResponse#(_b,_c)) get_resp_read(Axi3Slave#(_a,_b,_c) x);
   return x.resp_read;
endfunction

function Get#(Axi3WriteResponse#(_c)) get_resp_b(Axi3Slave#(_a,_b,_c) x);
   return x.resp_b;
endfunction

interface Axi3Master#(numeric type addrWidth, numeric type busWidth, numeric type idWidth);
   interface Get#(Axi3ReadRequest#(addrWidth, idWidth)) req_ar;
   interface Put#(Axi3ReadResponse#(busWidth, idWidth)) resp_read;
   interface Get#(Axi3WriteRequest#(addrWidth, idWidth)) req_aw;
   interface Get#(Axi3WriteData#(busWidth, idWidth)) resp_write;
   interface Put#(Axi3WriteResponse#(idWidth)) resp_b;
endinterface

interface Axi3Slave#(numeric type addrWidth, numeric type busWidth, numeric type idWidth);
   interface Put#(Axi3ReadRequest#(addrWidth, idWidth)) req_ar;
   interface Get#(Axi3ReadResponse#(busWidth, idWidth)) resp_read;
   interface Put#(Axi3WriteRequest#(addrWidth, idWidth)) req_aw;
   interface Put#(Axi3WriteData#(busWidth, idWidth)) resp_write;
   interface Get#(Axi3WriteResponse#(idWidth)) resp_b;
endinterface

function Put#(t) null_put();
   return (interface Put;
	      method Action put(t x) if (False);
		 noAction;
	      endmethod
	   endinterface);
endfunction

function Get#(t) null_get();
   return (interface Get;
	      method ActionValue#(t) get() if (False);
		 return ?;
	      endmethod
	   endinterface);
endfunction

function  Axi3Master#(addrWidth, busWidth, idWidth) null_axi_master();
   return (interface Axi3Master;
	      interface Get req_ar = null_get;
	      interface Put resp_read = null_put;
	      interface Get req_aw = null_get;
	      interface Get resp_write = null_get;
	      interface Put resp_b = null_put;
	   endinterface);
endfunction

instance Connectable#(Axi3Master#(addrWidth, busWidth,idWidth), Axi3Slave#(addrWidth, busWidth,idWidth));
   module mkConnection#(Axi3Master#(addrWidth, busWidth,idWidth) m, Axi3Slave#(addrWidth, busWidth,idWidth) s)(Empty);

      mkConnection(m.req_ar, s.req_ar);
      mkConnection(s.resp_read, m.resp_read);

      mkConnection(m.req_aw, s.req_aw);
      mkConnection(m.resp_write, s.resp_write);
      mkConnection(s.resp_b, m.resp_b);

   endmodule
endinstance

instance ConnectableWithClocks#(Axi3Master#(addrWidth, busWidth,idWidth), Axi3Slave#(addrWidth, busWidth,idWidth))
   provisos (ConnectableWithClocks#(Get#(Axi3ReadRequest#(addrWidth, idWidth)), Put#(Axi3ReadRequest#(addrWidth, idWidth)))
	     ,ConnectableWithClocks#(Get#(Axi3ReadResponse#(busWidth, idWidth)), Put#(Axi3ReadResponse#(busWidth, idWidth)))
	     ,ConnectableWithClocks#(Get#(Axi3WriteRequest#(addrWidth, idWidth)), Put#(Axi3WriteRequest#(addrWidth, idWidth)))
	     ,ConnectableWithClocks#(Get#(Axi3WriteData#(busWidth, idWidth)), Put#(Axi3WriteData#(busWidth, idWidth)))
	     ,ConnectableWithClocks#(Get#(Axi3WriteResponse#(idWidth)), Put#(Axi3WriteResponse#(idWidth)))
	     );

   module mkConnectionWithClocks2#(Axi3Master#(addrWidth, busWidth,idWidth) m, Axi3Slave#(addrWidth, busWidth,idWidth) s)(Empty);
      mkConnectionWithClocks(clockOf(m), resetOf(m), clockOf(s), resetOf(s), m, s);
   endmodule

   module mkConnectionWithClocks#(Clock mClock, Reset mReset, Clock sClock, Reset sReset,
				  Axi3Master#(addrWidth, busWidth,idWidth) m, Axi3Slave#(addrWidth, busWidth,idWidth) s)(Empty);

      mkConnectionWithClocks(mClock, mReset, sClock, sReset, m.req_ar, s.req_ar);
      mkConnectionWithClocks(sClock, sReset, mClock, mReset, s.resp_read, m.resp_read);

      mkConnectionWithClocks(mClock, mReset, sClock, sReset, m.req_aw, s.req_aw);
      mkConnectionWithClocks(mClock, mReset, sClock, sReset, m.resp_write, s.resp_write);
      mkConnectionWithClocks(sClock, sReset, mClock, mReset, s.resp_b, m.resp_b);

   endmodule
endinstance


initialize gather_phase to False

initialize gather_phase to False

Change-Id: Ic95aee62f7179a72c716ce7074a2edd4413fd6ff

// Copyright (c) 2014 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import FIFO::*;
import FIFOF::*;
import MIMO::*;
import DefaultValue::*;
import SpecialFIFOs::*;
import Vector::*;
import BRAM::*;
import DmaVector::*;
import PortalMemory::*;
import MemTypes::*;
import MemreadEngine::*;
import MemwriteEngine::*;
import MemUtils::*;
import FloatingPoint::*;
import Pipe::*;
import Arith::*;
import FloatOps::*;
import Timer::*;
import RbmTypes::*;
import Assert::*;
import Connectable::*;
import Clocks::*;
import Gearbox::*;
import XilinxCells::*;
import HostInterface::*;

interface SharedDotProdDebug#(numeric type k);
   interface PipeOut#(Bit#(32)) macCount;
endinterface

interface SharedDotProdServer#(numeric type k);
   interface Put#(MmToken)                 aInput;
   interface Put#(MmToken)                 bInput;
   interface Vector#(k, PipeOut#(MmToken)) pipes;
   interface SharedDotProdDebug#(k) debug;
endinterface

typedef struct {
`ifdef TAGGED_TOKENS
   UInt#(32) row;
   UInt#(32) col;
`endif
   Float v;
   Bool first;
   Bool last;
} MmToken deriving (Eq,Bits);

typedef 8 UB_MulLat; // upper bound on MUL latency?
typedef 8 UB_AddLat; // upper bound on ADD latency?

(* synthesize *)
module  mkSharedInterleavedDotProdServer#(UInt#(TLog#(TMul#(J,K))) label)(SharedDotProdServer#(K))
   provisos(Div#(UB_AddLat,K,gatherSz));

   let ub_MulLat = valueOf(UB_MulLat);
   let ub_AddLat = valueOf(UB_AddLat);
   let kk = valueOf(K);

   Bool verbose = False; //label == 0;

   Reg#(UInt#(20)) countReg     <- mkReg(0);

   FloatAlu mul   <- mkFloatMultiplier(defaultValue);
   FloatAlu adder <- mkFloatAdder(defaultValue);
   FIFOF#(Float) adder_buffer <- mkSizedFIFOF(valueOf(TMul#(K,gatherSz)));

`ifdef TAGGED_TOKENS
   FIFO#(Tuple2#(UInt#(32),UInt#(32))) tag_fifo <- mkSizedFIFO(ub_MulLat);
   Vector#(K,Reg#(Tuple2#(UInt#(32),UInt#(32)))) tag_regs <- replicateM(mkRegU);
`endif

   Reg#(Bit#(32)) cycles <- mkReg(0);
   rule countCycles;
      cycles <= cycles + 1;
   endrule

   FIFOF#(MmToken)                          afifo   <- mkFIFOF();
   PipeOut#(MmToken)                        aFunnel = toPipeOut(afifo);

   FIFOF#(MmToken)                          bfifo <- mkFIFOF();
   PipeOut#(MmToken)                        bFunnel = toPipeOut(bfifo);

   Reg#(Bit#(16)) firstCnt <- mkReg(0);
   Reg#(Bool)         gReg <- mkReg(False);
   FIFOF#(Float)     gFifo <- mkSizedFIFOF(kk);
   Reg#(Bit#(16))  lastCnt <- mkReg(0);
   Reg#(Bit#(16))gatherCnt <- mkReg(0);
   Reg#(Bool) gather_phase <- mkReg(False);
   //invariant: gather_phase = lastCnt == fromInteger(kk);

   FIFOF#(Tuple2#(Bool,Bool))    flFifo <- mkSizedFIFOF(ub_MulLat);
   Vector#(K,FIFOF#(MmToken))    dotfifos <- replicateM(mkFIFOF1);
   Reg#(Bit#(TAdd#(1,TLog#(K)))) rowReg <- mkReg(0);

   Reg#(Bit#(32)) lastMul <- mkReg(0);
   Reg#(Bit#(32)) lastAcc <- mkReg(0);
   Reg#(Bit#(32)) lastGather <- mkReg(0);
   Reg#(Bit#(32)) macs <- mkReg(0);

   (* fire_when_enabled *)
   rule connect_adder_buffer;
      match {.acc,.*} <- adder.response.get();
      adder_buffer.enq(acc);
      macs <= macs+1;
   endrule

   (* fire_when_enabled *)
   rule multiply;
      lastMul <= cycles;
      let a <- toGet(aFunnel).get();
      let b <- toGet(bFunnel).get();
      flFifo.enq(tuple2(a.first,a.last));
      if (a.first != b.first)
	 $display("****\n    Warning: a.first=%d != b.first=%d\n****", a.first, b.first);
      if (a.last != b.last)
	 $display("****\n    Warning: a.last=%d != b.last=%d\n****", a.last, b.last);
      if (verbose)
	 $display("%08d multiply: label=%d mulin first=%d last=%d", cycles-lastMul, label, a.first, a.last);
      mul.request.put(tuple2(a.v, b.v));
`ifdef TAGGED_TOKENS
      if(label==0) $display("%d (%d,%d)(%d,%d)", label, a.row, a.col, b.row, b.col);
      tag_fifo.enq(tuple2(a.row,b.col));
`endif
   endrule

   function Action incrementRowReg =
      (action
	  // I have to do this check (instead of relying on wrap-around) because
	  // rowReg has an extra bit to compenseate for bsc's silly Bit#(0) handling
	  if (rowReg+1 == fromInteger(kk)) rowReg <= 0;
	  else rowReg <= rowReg+1;
       endaction);

   (* fire_when_enabled *)
   rule accumulate if (!gather_phase);
      incrementRowReg;
      lastAcc <= cycles;
      match {.first, .last} <- toGet(flFifo).get();
      if (verbose) $display("%08d accumulate: label=%d mulout first=%d last=%d firstCnt=%d lastCnt=%d",
			    cycles-lastAcc, label, first, last, firstCnt, lastCnt);
      match {.resp,.*} <- mul.response.get;
      let acc = unpack(0);
      if (firstCnt == fromInteger(valueOf(TMul#(K,gatherSz))))
	 acc <- toGet(adder_buffer).get;
      else begin
	 firstCnt <= firstCnt+1;
      end
      adder.request.put(tuple2(resp,acc));
      if(last) begin
	 lastCnt <= lastCnt+1;
	 gather_phase <= (lastCnt == fromInteger(kk)-1);
      end
`ifdef TAGGED_TOKENS
      let row = rowReg;
      let t <- toGet(tag_fifo).get;
      tag_regs[row] <= t;
`endif
   endrule

   (* fire_when_enabled *)
   rule gather if (gather_phase);
      incrementRowReg;
      lastGather <= cycles;
      let row = rowReg;
      let last_row = row == fromInteger(kk-1);
      let last_pass = gatherCnt+1 == fromInteger(valueOf(gatherSz));
      if (verbose)
	 $display("%08d gather: gather=%d row=%d last_pass=%d last_row=%d, gReg=%d",
		  cycles-lastGather, gatherCnt, row, last_pass, last_row, gReg);
      let x <- toGet(adder_buffer).get;
      if (!last_pass) begin
	 if (last_row) begin
	    if (gReg) gatherCnt <= gatherCnt+1;
	    gReg <= !gReg;
	 end
	 if(gReg) begin
	    let y <- toGet(gFifo).get;
	    adder.request.put(tuple2(x,y));
	 end
	 else begin
	    gFifo.enq(x);
	 end
      end
      else begin
`ifdef TAGGED_TOKENS
      	 let row = tpl_1(tag_regs[row]);
      	 let col = tpl_2(tag_regs[row]);
	 dotfifos[row].enq(MmToken{row:row, col:col, v:x});
`else
	 dotfifos[row].enq(MmToken{v:x});
`endif
	 if (last_row) begin
	    gatherCnt <= 0;
	    lastCnt <= 0;
	    firstCnt <= 0;
	    gather_phase <= False;
	 end
      end
   endrule

   Vector#(K,PipeOut#(MmToken)) dotpipes = map(toPipeOut, dotfifos);

   interface Put aInput;
      method Action put(MmToken a);
   	 afifo.enq(a);
	 countReg <= countReg+1;
      endmethod
   endinterface
   interface Put bInput   = toPut(bfifo);
   interface Vector pipes = dotpipes;
   interface SharedDotProdDebug debug;
      interface PipeOut  macCount = toPipeOut(macs._read);
   endinterface
endmodule : mkSharedInterleavedDotProdServer

interface MmTileDebug;
   interface PipeOut#(Bit#(32)) macCount;
endinterface

interface MmTile;
   interface Vector#(RowsPerTile, Put#(MmToken)) aInputs;
   interface Vector#(RowsPerTile, Put#(MmToken)) bInputs;
   interface Vector#(RowsPerTile, PipeOut#(Vector#(N, MmToken))) fxPipes;
   interface MmTileDebug debug;
endinterface

function Put#(a) toCountedPut(Reg#(Bit#(n)) r, Put#(a) p);
   return (interface Put#(a);
      method Action put(a v);
	 r <= r+1;
	 p.put(v);
      endmethod
      endinterface);
endfunction

(* synthesize *)
module  mkMmTile#(Clock slowClock, Reset slowReset, UInt#(TLog#(T)) tile)(MmTile);

   let rowsPerTile = valueOf(RowsPerTile);
   let kk = valueOf(K);

   Vector#(RowsPerTile, Reg#(Bit#(32))) aMmTokensPutRegs <- replicateM(mkReg(0));
   Vector#(RowsPerTile, Reg#(Bit#(32))) bMmTokensPutRegs <- replicateM(mkReg(0));
   Vector#(RowsPerTile, Reg#(Bit#(32))) aMmTokensReadRegs <- replicateM(mkReg(0));
   Vector#(RowsPerTile, Reg#(Bit#(32))) bMmTokensReadRegs <- replicateM(mkReg(0));

   Vector#(RowsPerTile, FIFOF#(MmToken))   aFifos <- replicateM(mkFIFOF);
   Vector#(RowsPerTile, PipeOut#(MmToken)) aPipes = zipWith(toCountedPipeOut, aMmTokensReadRegs, map(toPipeOut, aFifos));
   Vector#(RowsPerTile,  FIFOF#(MmToken))   bFifos <- replicateM(mkFIFOF);
   Vector#(RowsPerTile,  PipeOut#(MmToken)) bPipes = zipWith(toCountedPipeOut, bMmTokensReadRegs, map(toPipeOut, bFifos));

   function Vector#(k,PipeOut#(MmToken)) getDotProdServerPipes(SharedDotProdServer#(k) s); return s.pipes; endfunction
   Vector#(RowsPerTile, SharedDotProdServer#(K)) fxdotprods <- mapM(mkSharedInterleavedDotProdServer, map(fromInteger,genVector));
   Vector#(RowsPerTile, Vector#(K, PipeOut#(MmToken))) fxpipes = map(getDotProdServerPipes, fxdotprods);
//`define USE_MIMO_DFIFOS // this version is faster
   let fastClock <- exposeCurrentClock();
   let fastReset <- exposeCurrentReset();
`ifndef USE_MIMO_DFIFOS
   Vector#(RowsPerTile, PipeOut#(Vector#(K, MmToken))) fxPipesK <- mapM(mkJoinVector(id), fxpipes);
   Vector#(RowsPerTile, PipeOut#(MmToken)) fxPipes1MmToken <- mapM(mkFunnel1, fxPipesK);
   Vector#(RowsPerTile, PipeOut#(Vector#(1, MmToken))) fxPipes1 = map(mapPipe(replicate), fxPipes1MmToken);
`else
   MIMOConfiguration mimoCfg = defaultValue;
   Vector#(RowsPerTile, MIMO#(K,1,TAdd#(K,1),MmToken)) dfifos <- replicateM(mkMIMO(mimoCfg));
   Vector#(RowsPerTile, PipeOut#(Vector#(1, MmToken))) fxPipes1 = map(toPipeOut, dfifos);
`endif
   Vector#(RowsPerTile, Gearbox#(1, N, MmToken)) gearboxes <- replicateM(mk1toNGearbox(fastClock, fastReset, slowClock, slowReset));
   Vector#(RowsPerTile, PipeIn#(Vector#(1,MmToken))) toGearboxes = map(toPipeIn, gearboxes);
   Vector#(RowsPerTile, PipeOut#(Vector#(N, MmToken))) fromGearboxes = map(toPipeOut, gearboxes);
   mapM(uncurry(mkConnection), zip(fxPipes1, toGearboxes));
   // introduce a buffer to help vivado meet timing on vc707
   Vector#(RowsPerTile, FIFOF#(Vector#(N,MmToken)))    tokenfifos <- replicateM(mkFIFOF(clocked_by slowClock, reset_by slowReset));
   Vector#(RowsPerTile, PipeIn#(Vector#(N,MmToken))) toMmTokenFifos = map(toPipeIn, tokenfifos);
   mapM(uncurry(mkConnection), zip(fromGearboxes, toMmTokenFifos), clocked_by slowClock, reset_by slowReset);
   Vector#(RowsPerTile, PipeOut#(Vector#(N, MmToken))) fxPipesN = map(toPipeOut, tokenfifos);

   FirstLastPipe#(UInt#(MMSize)) firstLastPipe          <- mkFirstLastPipe();
   Vector#(2, PipeOut#(Tuple2#(Bool,Bool))) firstLastPipes <- mkForkVector(firstLastPipe.pipe);

   for (Integer j = 0; j < rowsPerTile; j = j + 1) begin
      //mkConnection(toGet(aPipes[j]), fxdotprods[j].aInput);
      //mkConnection(toGet(bPipes[j]), fxdotprods[j].bInput);
      (* fire_when_enabled *)
      rule connectA;
	 let x <- toGet(aPipes[j]).get;
	 fxdotprods[j].aInput.put(x);
      endrule
      (* fire_when_enabled *)
      rule connectB;
	 let x <- toGet(bPipes[j]).get;
	 fxdotprods[j].bInput.put(x);
      endrule
   end

`ifdef USE_MIMO_DFIFOS
   for (Integer j = 0; j < rowsPerTile; j = j + 1) begin
      rule dotProdValue;
	 Vector#(K,MmToken) vs;
	 for (Integer k = 0; k < kk; k = k + 1) begin
	    let v <- toGet(fxpipes[j][k]).get();
	    vs[k] = v;
	 end
	 dfifos[j].enq(fromInteger(kk), vs);
      endrule
   end
`endif

   function Bool fifofNotEmpty(FIFOF#(a) fifof); return fifof.notEmpty(); endfunction
   function PipeOut#(Bit#(32)) dotProdMacCount(SharedDotProdServer#(K) dotprodserver); return dotprodserver.debug.macCount; endfunction
   PipeOut#(Bit#(32)) macCountPipe <- mkReducePipes(uncurry(add), map(dotProdMacCount, fxdotprods));

   interface Vector aInputs = zipWith(toCountedPut, aMmTokensPutRegs, map(toPut, aFifos));
   interface Vector bInputs = zipWith(toCountedPut, bMmTokensPutRegs, map(toPut, bFifos));
   interface Vector fxPipes = fxPipesN;
   interface MmTileDebug debug;
      interface PipeOut macCount = macCountPipe;
   endinterface
endmodule : mkMmTile

typedef struct {
   ObjectPointer pointer;
   addrtype base;
   addrtype numRows;
   addrtype numColumns;
} MatrixDescriptor#(type addrtype) deriving (Bits);

interface DmaMatrixMultiplyDebug;
   method Bit#(32) macCount();
endinterface

// Copyright (c) 2014 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import FIFO::*;
import FIFOF::*;
import MIMO::*;
import DefaultValue::*;
import SpecialFIFOs::*;
import Vector::*;
import BRAM::*;
import DmaVector::*;
import PortalMemory::*;
import MemTypes::*;
import MemreadEngine::*;
import MemwriteEngine::*;
import MemUtils::*;
import FloatingPoint::*;
import Pipe::*;
import Arith::*;
import FloatOps::*;
import Timer::*;
import RbmTypes::*;
import Assert::*;
import Connectable::*;
import Clocks::*;
import Gearbox::*;
import XilinxCells::*;
import HostInterface::*;

interface SharedDotProdDebug#(numeric type k);
   interface PipeOut#(Bit#(32)) macCount;
endinterface

interface SharedDotProdServer#(numeric type k);
   interface Put#(MmToken)                 aInput;
   interface Put#(MmToken)                 bInput;
   interface Vector#(k, PipeOut#(MmToken)) pipes;
   interface SharedDotProdDebug#(k) debug;
endinterface

typedef struct {
`ifdef TAGGED_TOKENS
   UInt#(32) row;
   UInt#(32) col;
`endif
   Float v;
   Bool first;
   Bool last;
} MmToken deriving (Eq,Bits);

typedef 8 UB_MulLat; // upper bound on MUL latency?
typedef 8 UB_AddLat; // upper bound on ADD latency?

(* synthesize *)
module  mkSharedInterleavedDotProdServer#(UInt#(TLog#(TMul#(J,K))) label)(SharedDotProdServer#(K))
   provisos(Div#(UB_AddLat,K,gatherSz));

   let ub_MulLat = valueOf(UB_MulLat);
   let ub_AddLat = valueOf(UB_AddLat);
   let kk = valueOf(K);

   Bool verbose = False; //label == 0;

   Reg#(UInt#(20)) countReg     <- mkReg(0);

   FloatAlu mul   <- mkFloatMultiplier(defaultValue);
   FloatAlu adder <- mkFloatAdder(defaultValue);
   FIFOF#(Float) adder_buffer <- mkSizedFIFOF(valueOf(TMul#(K,gatherSz)));

`ifdef TAGGED_TOKENS
   FIFO#(Tuple2#(UInt#(32),UInt#(32))) tag_fifo <- mkSizedFIFO(ub_MulLat);
   Vector#(K,Reg#(Tuple2#(UInt#(32),UInt#(32)))) tag_regs <- replicateM(mkRegU);
`endif

   Reg#(Bit#(32)) cycles <- mkReg(0);
   rule countCycles;
      cycles <= cycles + 1;
   endrule

   FIFOF#(MmToken)                          afifo   <- mkFIFOF();
   PipeOut#(MmToken)                        aFunnel = toPipeOut(afifo);

   FIFOF#(MmToken)                          bfifo <- mkFIFOF();
   PipeOut#(MmToken)                        bFunnel = toPipeOut(bfifo);

   Reg#(Bit#(16)) firstCnt <- mkReg(0);
   Reg#(Bool)         gReg <- mkReg(False);
   FIFOF#(Float)     gFifo <- mkSizedFIFOF(kk);
   Reg#(Bit#(16))  lastCnt <- mkReg(0);
   Reg#(Bit#(16))gatherCnt <- mkReg(0);
   Reg#(Bool) gather_phase <- mkReg(0);
   //invariant: gather_phase = lastCnt == fromInteger(kk);

   FIFOF#(Tuple2#(Bool,Bool))    flFifo <- mkSizedFIFOF(ub_MulLat);
   Vector#(K,FIFOF#(MmToken))    dotfifos <- replicateM(mkFIFOF1);
   Reg#(Bit#(TAdd#(1,TLog#(K)))) rowReg <- mkReg(0);

   Reg#(Bit#(32)) lastMul <- mkReg(0);
   Reg#(Bit#(32)) lastAcc <- mkReg(0);
   Reg#(Bit#(32)) lastGather <- mkReg(0);
   Reg#(Bit#(32)) macs <- mkReg(0);

   (* fire_when_enabled *)
   rule connect_adder_buffer;
      match {.acc,.*} <- adder.response.get();
      adder_buffer.enq(acc);
      macs <= macs+1;
   endrule

   (* fire_when_enabled *)
   rule multiply;
      lastMul <= cycles;
      let a <- toGet(aFunnel).get();
      let b <- toGet(bFunnel).get();
      flFifo.enq(tuple2(a.first,a.last));
      if (a.first != b.first)
	 $display("****\n    Warning: a.first=%d != b.first=%d\n****", a.first, b.first);
      if (a.last != b.last)
	 $display("****\n    Warning: a.last=%d != b.last=%d\n****", a.last, b.last);
      if (verbose)
	 $display("%08d multiply: label=%d mulin first=%d last=%d", cycles-lastMul, label, a.first, a.last);
      mul.request.put(tuple2(a.v, b.v));
`ifdef TAGGED_TOKENS
      if(label==0) $display("%d (%d,%d)(%d,%d)", label, a.row, a.col, b.row, b.col);
      tag_fifo.enq(tuple2(a.row,b.col));
`endif
   endrule

   function Action incrementRowReg =
      (action
	  // I have to do this check (instead of relying on wrap-around) because
	  // rowReg has an extra bit to compenseate for bsc's silly Bit#(0) handling
	  if (rowReg+1 == fromInteger(kk)) rowReg <= 0;
	  else rowReg <= rowReg+1;
       endaction);

   (* fire_when_enabled *)
   rule accumulate if (!gather_phase);
      incrementRowReg;
      lastAcc <= cycles;
      match {.first, .last} <- toGet(flFifo).get();
      if (verbose) $display("%08d accumulate: label=%d mulout first=%d last=%d firstCnt=%d lastCnt=%d",
			    cycles-lastAcc, label, first, last, firstCnt, lastCnt);
      match {.resp,.*} <- mul.response.get;
      let acc = unpack(0);
      if (firstCnt == fromInteger(valueOf(TMul#(K,gatherSz))))
	 acc <- toGet(adder_buffer).get;
      else begin
	 firstCnt <= firstCnt+1;
      end
      adder.request.put(tuple2(resp,acc));
      if(last) begin
	 lastCnt <= lastCnt+1;
	 gather_phase <= (lastCnt == fromInteger(kk)-1);
      end
`ifdef TAGGED_TOKENS
      let row = rowReg;
      let t <- toGet(tag_fifo).get;
      tag_regs[row] <= t;
`endif
   endrule

   (* fire_when_enabled *)
   rule gather if (gather_phase);
      incrementRowReg;
      lastGather <= cycles;
      let row = rowReg;
      let last_row = row == fromInteger(kk-1);
      let last_pass = gatherCnt+1 == fromInteger(valueOf(gatherSz));
      if (verbose)
	 $display("%08d gather: gather=%d row=%d last_pass=%d last_row=%d, gReg=%d",
		  cycles-lastGather, gatherCnt, row, last_pass, last_row, gReg);
      let x <- toGet(adder_buffer).get;
      if (!last_pass) begin
	 if (last_row) begin
	    if (gReg) gatherCnt <= gatherCnt+1;
	    gReg <= !gReg;
	 end
	 if(gReg) begin
	    let y <- toGet(gFifo).get;
	    adder.request.put(tuple2(x,y));
	 end
	 else begin
	    gFifo.enq(x);
	 end
      end
      else begin
`ifdef TAGGED_TOKENS
      	 let row = tpl_1(tag_regs[row]);
      	 let col = tpl_2(tag_regs[row]);
	 dotfifos[row].enq(MmToken{row:row, col:col, v:x});
`else
	 dotfifos[row].enq(MmToken{v:x});
`endif
	 if (last_row) begin
	    gatherCnt <= 0;
	    lastCnt <= 0;
	    firstCnt <= 0;
	    gather_phase <= False;
	 end
      end
   endrule

   Vector#(K,PipeOut#(MmToken)) dotpipes = map(toPipeOut, dotfifos);

   interface Put aInput;
      method Action put(MmToken a);
   	 afifo.enq(a);
	 countReg <= countReg+1;
      endmethod
   endinterface
   interface Put bInput   = toPut(bfifo);
   interface Vector pipes = dotpipes;
   interface SharedDotProdDebug debug;
      interface PipeOut  macCount = toPipeOut(macs._read);
   endinterface
endmodule : mkSharedInterleavedDotProdServer

interface MmTileDebug;
   interface PipeOut#(Bit#(32)) macCount;
endinterface

interface MmTile;
   interface Vector#(RowsPerTile, Put#(MmToken)) aInputs;
   interface Vector#(RowsPerTile, Put#(MmToken)) bInputs;
   interface Vector#(RowsPerTile, PipeOut#(Vector#(N, MmToken))) fxPipes;
   interface MmTileDebug debug;
endinterface

function Put#(a) toCountedPut(Reg#(Bit#(n)) r, Put#(a) p);
   return (interface Put#(a);
      method Action put(a v);
	 r <= r+1;
	 p.put(v);
      endmethod
      endinterface);
endfunction

(* synthesize *)
module  mkMmTile#(Clock slowClock, Reset slowReset, UInt#(TLog#(T)) tile)(MmTile);

   let rowsPerTile = valueOf(RowsPerTile);
   let kk = valueOf(K);

   Vector#(RowsPerTile, Reg#(Bit#(32))) aMmTokensPutRegs <- replicateM(mkReg(0));
   Vector#(RowsPerTile, Reg#(Bit#(32))) bMmTokensPutRegs <- replicateM(mkReg(0));
   Vector#(RowsPerTile, Reg#(Bit#(32))) aMmTokensReadRegs <- replicateM(mkReg(0));
   Vector#(RowsPerTile, Reg#(Bit#(32))) bMmTokensReadRegs <- replicateM(mkReg(0));

   Vector#(RowsPerTile, FIFOF#(MmToken))   aFifos <- replicateM(mkFIFOF);
   Vector#(RowsPerTile, PipeOut#(MmToken)) aPipes = zipWith(toCountedPipeOut, aMmTokensReadRegs, map(toPipeOut, aFifos));
   Vector#(RowsPerTile,  FIFOF#(MmToken))   bFifos <- replicateM(mkFIFOF);
   Vector#(RowsPerTile,  PipeOut#(MmToken)) bPipes = zipWith(toCountedPipeOut, bMmTokensReadRegs, map(toPipeOut, bFifos));

   function Vector#(k,PipeOut#(MmToken)) getDotProdServerPipes(SharedDotProdServer#(k) s); return s.pipes; endfunction
   Vector#(RowsPerTile, SharedDotProdServer#(K)) fxdotprods <- mapM(mkSharedInterleavedDotProdServer, map(fromInteger,genVector));
   Vector#(RowsPerTile, Vector#(K, PipeOut#(MmToken))) fxpipes = map(getDotProdServerPipes, fxdotprods);
//`define USE_MIMO_DFIFOS // this version is faster
   let fastClock <- exposeCurrentClock();
   let fastReset <- exposeCurrentReset();
`ifndef USE_MIMO_DFIFOS
   Vector#(RowsPerTile, PipeOut#(Vector#(K, MmToken))) fxPipesK <- mapM(mkJoinVector(id), fxpipes);
   Vector#(RowsPerTile, PipeOut#(MmToken)) fxPipes1MmToken <- mapM(mkFunnel1, fxPipesK);
   Vector#(RowsPerTile, PipeOut#(Vector#(1, MmToken))) fxPipes1 = map(mapPipe(replicate), fxPipes1MmToken);
`else
   MIMOConfiguration mimoCfg = defaultValue;
   Vector#(RowsPerTile, MIMO#(K,1,TAdd#(K,1),MmToken)) dfifos <- replicateM(mkMIMO(mimoCfg));
   Vector#(RowsPerTile, PipeOut#(Vector#(1, MmToken))) fxPipes1 = map(toPipeOut, dfifos);
`endif
   Vector#(RowsPerTile, Gearbox#(1, N, MmToken)) gearboxes <- replicateM(mk1toNGearbox(fastClock, fastReset, slowClock, slowReset));
   Vector#(RowsPerTile, PipeIn#(Vector#(1,MmToken))) toGearboxes = map(toPipeIn, gearboxes);
   Vector#(RowsPerTile, PipeOut#(Vector#(N, MmToken))) fromGearboxes = map(toPipeOut, gearboxes);
   mapM(uncurry(mkConnection), zip(fxPipes1, toGearboxes));
   // introduce a buffer to help vivado meet timing on vc707
   Vector#(RowsPerTile, FIFOF#(Vector#(N,MmToken)))    tokenfifos <- replicateM(mkFIFOF(clocked_by slowClock, reset_by slowReset));
   Vector#(RowsPerTile, PipeIn#(Vector#(N,MmToken))) toMmTokenFifos = map(toPipeIn, tokenfifos);
   mapM(uncurry(mkConnection), zip(fromGearboxes, toMmTokenFifos), clocked_by slowClock, reset_by slowReset);
   Vector#(RowsPerTile, PipeOut#(Vector#(N, MmToken))) fxPipesN = map(toPipeOut, tokenfifos);

   FirstLastPipe#(UInt#(MMSize)) firstLastPipe          <- mkFirstLastPipe();
   Vector#(2, PipeOut#(Tuple2#(Bool,Bool))) firstLastPipes <- mkForkVector(firstLastPipe.pipe);

   for (Integer j = 0; j < rowsPerTile; j = j + 1) begin
      //mkConnection(toGet(aPipes[j]), fxdotprods[j].aInput);
      //mkConnection(toGet(bPipes[j]), fxdotprods[j].bInput);
      (* fire_when_enabled *)
      rule connectA;
	 let x <- toGet(aPipes[j]).get;
	 fxdotprods[j].aInput.put(x);
      endrule
      (* fire_when_enabled *)
      rule connectB;
	 let x <- toGet(bPipes[j]).get;
	 fxdotprods[j].bInput.put(x);
      endrule
   end

`ifdef USE_MIMO_DFIFOS
   for (Integer j = 0; j < rowsPerTile; j = j + 1) begin
      rule dotProdValue;
	 Vector#(K,MmToken) vs;
	 for (Integer k = 0; k < kk; k = k + 1) begin
	    let v <- toGet(fxpipes[j][k]).get();
	    vs[k] = v;
	 end
	 dfifos[j].enq(fromInteger(kk), vs);
      endrule
   end
`endif

   function Bool fifofNotEmpty(FIFOF#(a) fifof); return fifof.notEmpty(); endfunction
   function PipeOut#(Bit#(32)) dotProdMacCount(SharedDotProdServer#(K) dotprodserver); return dotprodserver.debug.macCount; endfunction
   PipeOut#(Bit#(32)) macCountPipe <- mkReducePipes(uncurry(add), map(dotProdMacCount, fxdotprods));

   interface Vector aInputs = zipWith(toCountedPut, aMmTokensPutRegs, map(toPut, aFifos));
   interface Vector bInputs = zipWith(toCountedPut, bMmTokensPutRegs, map(toPut, bFifos));
   interface Vector fxPipes = fxPipesN;
   interface MmTileDebug debug;
      interface PipeOut macCount = macCountPipe;
   endinterface
endmodule : mkMmTile

typedef struct {
   ObjectPointer pointer;
   addrtype base;
   addrtype numRows;
   addrtype numColumns;
} MatrixDescriptor#(type addrtype) deriving (Bits);

interface DmaMatrixMultiplyDebug;
   method Bit#(32) macCount();
endinterface


shared memory indication portals nearly working

Change-Id: I248d833c91f447435a5983581e066c40aed7ed7b

// Copyright (c) 2014 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import Vector::*;
import GetPut::*;
import FIFO::*;
import FIFOF::*;
import GetPut::*;
import ClientServer::*;
import CtrlMux::*;
import Gearbox::*;
import Clocks::*;

import MIFO::*;
import Pipe::*;
import Portal::*;
import MemTypes::*;
import AddressGenerator::*;
import MemTypes::*;
import MemreadEngine::*;
import MemwriteEngine::*;
import ConnectalMemory::*;

typedef enum {
   Idle,
   HeadRequested,
   TailRequested,
   RequestMessage,
   MessageHeaderRequested,
   MessageRequested,
   Drain,
   UpdateTail,
   UpdateHead,
   UpdateHead2,
   Waiting,
   SendHeader,
   SendMessage,
   Stop
   } SharedMemoryPortalState deriving (Bits,Eq);

module mkSharedMemoryRequestPortal#(PipePortal#(numRequests, numIndications, 32) portal,
				    MemreadServer#(64) readEngine,
				    MemwriteServer#(64) writeEngine,
				    Reg#(Bit#(32)) sglIdReg,
				    Reg#(Bool)     readyReg)(SharedMemoryPortal#(64));
      // read the head and tail pointers, if they are different, then read a request
      Reg#(Bit#(32)) reqLimitReg <- mkReg(0);
      Reg#(Bit#(32)) reqHeadReg <- mkReg(0);
      Reg#(Bit#(32)) reqTailReg <- mkReg(0);
      Reg#(Bit#(16)) wordCountReg <- mkReg(0);
      Reg#(Bit#(16)) messageWordsReg <- mkReg(0);
      Reg#(Bit#(16)) methodIdReg <- mkReg(0);
      Reg#(SharedMemoryPortalState) reqState <- mkReg(Idle);

   let verbose = False;

      rule updateReqHeadTail if (reqState == Idle && readyReg);
	 readEngine.cmdServer.request.put(MemengineCmd { sglId: sglIdReg,
							base: 0,
							burstLen: 16,
							len: 16
							});
	 reqState <= HeadRequested;
      endrule

      rule receiveReqHeadTail if (reqState == HeadRequested || reqState == TailRequested);
	 let data <- toGet(readEngine.dataPipe).get();
	 let w0 = data[31:0];
	 let w1 = data[63:32];
	 let head = reqHeadReg;
	 let tail = reqTailReg;
	 if (reqState == HeadRequested) begin
	    reqLimitReg <= w0;
	    reqHeadReg <= w1;
	    head = w1;
	    reqState <= TailRequested;
	 end
	 else begin
	    if (reqTailReg == 0) begin
	       tail = w0;
	       reqTailReg <= tail;
	    end
	    if (tail != reqHeadReg)
	       reqState <= RequestMessage;
	    else
	       reqState <= Idle;
	 end
	 if (verbose)
	    $display("receiveReqHeadTail state=%d w0=%x w1=%x head=%d tail=%d limit=%d", reqState, w0, w1, head, tail, reqLimitReg);
      endrule

      rule requestMessage if (reqState == RequestMessage);
	 Bit#(32) wordCount = reqHeadReg - reqTailReg;
	 if ((reqTailReg & 1) == 1) begin
	    $display("WARNING requestMessage: reqTail=%d is odd.", reqTailReg);
	 end
	 let tail = reqTailReg + wordCount;
	 if (reqHeadReg < reqTailReg) begin
	    $display("requestMessage wrapped: head=%d tail=%d", reqHeadReg, reqTailReg);
	    wordCount = reqLimitReg - reqTailReg;
	    tail = 4;
	 end
	 if (verbose) $display("requestMessage id=%d tail=%h head=%h wordCount=%d", sglIdReg, reqTailReg, reqHeadReg, wordCount);

	 reqTailReg <= tail;
	 wordCountReg <= truncate(wordCount);
	 readEngine.cmdServer.request.put(MemengineCmd {sglId: sglIdReg,
							base: extend(reqTailReg << 2),
							burstLen: 16,
							len: wordCount << 2
							});
	 reqState <= MessageHeaderRequested;
      endrule

      MIFO#(4,1,4,Bit#(32)) readMifo <- mkMIFO();
      rule demuxwords if (readMifo.enqReady());
	 let data <- toGet(readEngine.dataPipe).get();
	 Vector#(2,Bit#(32)) dvec = unpack(data);
	 let enqCount = 2;
	 Vector#(4,Bit#(32)) dvec4;
	 dvec4[0] = dvec[0];
	 dvec4[1] = dvec[1];
	 dvec4[2] = 0;
	 dvec4[3] = 0;
	 readMifo.enq(enqCount, dvec4);
      endrule

      rule receiveMessageHeader if (reqState == MessageHeaderRequested);
	 let vec <- toGet(readMifo).get();
	 let hdr = vec[0];
	 let methodId = hdr[31:16];
	 let messageWords = hdr[15:0];
	 methodIdReg <= methodId;
	 if (verbose)
	    $display("receiveMessageHeader hdr=%x methodId=%x messageWords=%d wordCount=%d", hdr, methodId, messageWords, wordCountReg);
	 wordCountReg <= wordCountReg - 1;
	 messageWordsReg <= messageWords - 1;
	 if (hdr == 0) begin
	    if (wordCountReg == 1)
	       reqState <= UpdateTail;
	    else
	       reqState <= Drain;
	 end
	 else if (wordCountReg <= messageWords)
	    reqState <= UpdateTail;
	 else if (messageWords == 1)
	    reqState <= MessageHeaderRequested;
	 else
	    reqState <= MessageRequested;
      endrule

      rule drain if (reqState == Drain);
	 let vec <- toGet(readMifo).get();
	 if (wordCountReg == 1)
	    reqState <= UpdateTail;
	 wordCountReg <= wordCountReg - 1;
      endrule

      rule receiveMessage if (reqState == MessageRequested);
	 let vec <- toGet(readMifo).get();
	 let data = vec[0];
	 if (verbose)
	    $display("receiveMessage data=%x messageWords=%d wordCount=%d", data, messageWordsReg, wordCountReg);
	 if (methodIdReg != 16'hFFFF)
	    portal.requests[methodIdReg].enq(data);

	 messageWordsReg <= messageWordsReg - 1;
	 wordCountReg <= wordCountReg - 1;
	 if (messageWordsReg == 1)
	    reqState <= MessageHeaderRequested;
	 else if (wordCountReg <= 1)
	    reqState <= UpdateTail;
      endrule

      rule updateTail if (reqState == UpdateTail);
	 if (verbose)
	    $display("updateTail: tail=%d", reqTailReg);
	 // update the tail pointer
	 writeEngine.cmdServer.request.put(MemengineCmd {sglId: sglIdReg,
							 base: 8,
							 len: 8,
							 burstLen: 8
							 });
	 writeEngine.dataPipe.enq(extend(reqTailReg));
	 reqState <= Waiting;
      endrule
      rule waiting if (reqState == Waiting);
	 let done <- writeEngine.cmdServer.response.get();
	 reqState <= Idle;
      endrule

      rule consumeResponse;
	 let response <- readEngine.cmdServer.response.get();
      endrule
endmodule

module mkSharedMemoryIndicationPortal#(PipePortal#(numRequests, numIndications, 32) portal,
				       MemreadServer#(64) readEngine,
				       MemwriteServer#(64) writeEngine,
				       Reg#(Bit#(32)) sglIdReg,
				       Reg#(Bool)     readyReg)(SharedMemoryPortal#(64));
      // read the head and tail pointers, if they are different, then read a request
      Reg#(Bit#(16)) indLimitReg <- mkReg(0);
      Reg#(Bit#(16)) indHeadReg <- mkReg(0);
      Reg#(Bit#(16)) indTailReg <- mkReg(0);
      Reg#(Bit#(16)) wordCountReg <- mkReg(0);
      Reg#(Bit#(16)) messageWordsReg <- mkReg(0);
      Reg#(Bit#(16)) methodIdReg <- mkReg(0);
      Reg#(SharedMemoryPortalState) indState <- mkReg(Idle);

   let verbose = True;

   function Bool pipeOutNotEmpty(PipeOut#(a) po); return po.notEmpty(); endfunction
   Vector#(numIndications, Bool) readyBits = map(pipeOutNotEmpty, portal.indications);
   Bool      interruptStatus = False;
   Bit#(16)  readyChannel = -1;
   for (Integer i = 0; i < valueOf(numIndications); i = i + 1) begin
      if (readyBits[i]) begin
         interruptStatus = True;
         readyChannel = fromInteger(i);
      end
   end

      rule updateIndHeadTail if (indState == Idle && readyReg);
	 readEngine.cmdServer.request.put(MemengineCmd { sglId: sglIdReg,
							base: 0,
							burstLen: 16,
							len: 16
							});
	 indState <= HeadRequested;
      endrule

      rule receiveIndHeadTail if (indState == HeadRequested || indState == TailRequested);
	 let data <- toGet(readEngine.dataPipe).get();
	 let w0 = data[31:0];
	 let w1 = data[63:32];
	 let head = indHeadReg;
	 let tail = indTailReg;
	 if (indState == HeadRequested) begin
	    indLimitReg <= truncate(w0);
	    indHeadReg <= truncate(w1);
	    head = truncate(w1);
	    indState <= TailRequested;
	 end
	 else begin
	    if (indTailReg == 0) begin
	       tail = truncate(w0);
	       indTailReg <= tail;
	    end
	    //if (tail != indHeadReg)
	       indState <= SendHeader;
	    //else
	       //indState <= Idle;
	 end
	 if (verbose)
	    $display("receiveIndHeadTail state=%d w0=%x w1=%x head=%d tail=%d limit=%d", indState, w0, w1, head, tail, indLimitReg);
      endrule

   let defaultClock <- exposeCurrentClock;
   let defaultReset <- exposeCurrentReset;
   Gearbox#(1,2,Bit#(32)) gb <- mk1toNGearbox(defaultClock, defaultReset, defaultClock, defaultReset);
   rule send64bits;
      let v = gb.first;
      gb.deq();
      writeEngine.dataPipe.enq(pack(v));
   endrule
   rule sendHeader if (indState == SendHeader && interruptStatus);
      Bit#(16) numWords = portal.messageSize(readyChannel) >> 5;
      Bit#(16) totalWords = numWords + 1;
      if (numWords[0] == 0)
	 totalWords = numWords + 2;
      Bit#(32) hdr = extend(readyChannel) << 16 | extend(totalWords);
      $display("sendHeader hdr=%h numWords=%d", hdr, numWords);

      indHeadReg <= indHeadReg + totalWords;
      messageWordsReg <= numWords;
      methodIdReg <= readyChannel;
      gb.enq(replicate(hdr));
      writeEngine.cmdServer.request.put(MemengineCmd { sglId: sglIdReg,
						      base: extend(indHeadReg) << 2,
						      burstLen: 8,
						      len: extend(totalWords) << 2 });
      indState <= SendMessage;
   endrule
   rule sendMessage if (indState == SendMessage);
      messageWordsReg <= messageWordsReg - 1;
      let v = portal.indications[methodIdReg].first;
      portal.indications[methodIdReg].deq();
      gb.enq(replicate(v));
      $display("sendMessage v=%h messageWords=%d", v, messageWordsReg);

      if (messageWordsReg == 1)
	 indState <= UpdateHead;
   endrule
   rule updateHead if (indState == UpdateHead);
      $display("updateIndHead limit=%d head=%d", indLimitReg, indHeadReg);
      gb.enq(replicate(extend(indLimitReg)));
      writeEngine.cmdServer.request.put(MemengineCmd { sglId: sglIdReg,
						      base: 0 << 2,
						      burstLen: 8,
						      len: 2 << 2 });
      indState <= UpdateHead2;
   endrule
   rule updateHead2 if (indState == UpdateHead2);
      $display("updateIndHead2");
      gb.enq(replicate(extend(indHeadReg)));
      indState <= SendHeader;
   endrule
   rule done;
      let done <- writeEngine.cmdServer.response.get();
   endrule

endmodule

module mkSharedMemoryPortal#(PipePortal#(numRequests, numIndications, 32) portal)(SharedMemoryPortal#(64));

   MemreadEngineV#(64,2,1) readEngine <- mkMemreadEngine();
   MemwriteEngineV#(64,2,1) writeEngine <- mkMemwriteEngine();

   Bool verbose = False;

   Reg#(Bit#(32)) sglIdReg <- mkReg(0);
   Reg#(Bool)     readyReg   <- mkReg(False);

   if (valueOf(numRequests) > 0) begin
      let readPortal <- mkSharedMemoryRequestPortal(portal, readEngine.read_servers[0], writeEngine.write_servers[0], sglIdReg, readyReg);
   end
   else if (valueOf(numIndications) > 0) begin
      let writePortal <- mkSharedMemoryIndicationPortal(portal, readEngine.read_servers[0], writeEngine.write_servers[0], sglIdReg, readyReg);
   end
   interface SharedMemoryPortalConfig cfg;
      method Action setSglId(Bit#(32) id);
	 sglIdReg <= id;
	 readyReg <= True;
      endmethod
   endinterface
   interface MemReadClient  readClient = readEngine.dmaClient;
   interface MemWriteClient writeClient = writeEngine.dmaClient;
   interface ReadOnly interrupt;
   endinterface
endmodule

// Copyright (c) 2014 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import Vector::*;
import GetPut::*;
import FIFO::*;
import FIFOF::*;
import GetPut::*;
import ClientServer::*;
import CtrlMux::*;

import MIFO::*;
import Pipe::*;
import Portal::*;
import MemTypes::*;
import AddressGenerator::*;
import MemTypes::*;
import MemreadEngine::*;
import MemwriteEngine::*;
import ConnectalMemory::*;

typedef enum {
   Idle,
   HeadRequested,
   TailRequested,
   RequestMessage,
   MessageHeaderRequested,
   MessageRequested,
   Drain,
   UpdateTail,
   Waiting,
   Stop
   } SharedMemoryPortalState deriving (Bits,Eq);

module mkSharedMemoryRequestPortal#(PipePortal#(numRequests, numIndications, 32) portal,
				    MemreadServer#(64) readEngine,
				    MemwriteServer#(64) writeEngine,
				    Reg#(Bit#(32)) sglIdReg,
				    Reg#(Bool)     readyReg)(SharedMemoryPortal#(64));
      // read the head and tail pointers, if they are different, then read a request
      Reg#(Bit#(32)) reqLimitReg <- mkReg(0);
      Reg#(Bit#(32)) reqHeadReg <- mkReg(0);
      Reg#(Bit#(32)) reqTailReg <- mkReg(0);
      Reg#(Bit#(16)) wordCountReg <- mkReg(0);
      Reg#(Bit#(16)) messageWordsReg <- mkReg(0);
      Reg#(Bit#(16)) methodIdReg <- mkReg(0);
      Reg#(SharedMemoryPortalState) reqState <- mkReg(Idle);

   let verbose = False;

      rule updateReqHeadTail if (reqState == Idle && readyReg);
	 readEngine.cmdServer.request.put(MemengineCmd { sglId: sglIdReg,
							base: 0,
							burstLen: 16,
							len: 16
							});
	 reqState <= HeadRequested;
      endrule

      rule receiveReqHeadTail if (reqState == HeadRequested || reqState == TailRequested);
	 let data <- toGet(readEngine.dataPipe).get();
	 let w0 = data[31:0];
	 let w1 = data[63:32];
	 let head = reqHeadReg;
	 let tail = reqTailReg;
	 if (reqState == HeadRequested) begin
	    reqLimitReg <= w0;
	    reqHeadReg <= w1;
	    head = w1;
	    reqState <= TailRequested;
	 end
	 else begin
	    if (reqTailReg == 0) begin
	       tail = w0;
	       reqTailReg <= tail;
	    end
	    if (tail != reqHeadReg)
	       reqState <= RequestMessage;
	    else
	       reqState <= Idle;
	 end
	 if (verbose)
	    $display("receiveReqHeadTail state=%d w0=%x w1=%x head=%d tail=%d limit=%d", reqState, w0, w1, head, tail, reqLimitReg);
      endrule

      rule requestMessage if (reqState == RequestMessage);
	 Bit#(32) wordCount = reqHeadReg - reqTailReg;
	 if ((reqTailReg & 1) == 1) begin
	    $display("WARNING requestMessage: reqTail=%d is odd.", reqTailReg);
	 end
	 let tail = reqTailReg + wordCount;
	 if (reqHeadReg < reqTailReg) begin
	    $display("requestMessage wrapped: head=%d tail=%d", reqHeadReg, reqTailReg);
	    wordCount = reqLimitReg - reqTailReg;
	    tail = 4;
	 end
	 if (verbose) $display("requestMessage id=%d tail=%h head=%h wordCount=%d", sglIdReg, reqTailReg, reqHeadReg, wordCount);

	 reqTailReg <= tail;
	 wordCountReg <= truncate(wordCount);
	 readEngine.cmdServer.request.put(MemengineCmd {sglId: sglIdReg,
							base: extend(reqTailReg << 2),
							burstLen: 16,
							len: wordCount << 2
							});
	 reqState <= MessageHeaderRequested;
      endrule

      MIFO#(4,1,4,Bit#(32)) readMifo <- mkMIFO();
      rule demuxwords if (readMifo.enqReady());
	 let data <- toGet(readEngine.dataPipe).get();
	 Vector#(2,Bit#(32)) dvec = unpack(data);
	 let enqCount = 2;
	 Vector#(4,Bit#(32)) dvec4;
	 dvec4[0] = dvec[0];
	 dvec4[1] = dvec[1];
	 dvec4[2] = 0;
	 dvec4[3] = 0;
	 readMifo.enq(enqCount, dvec4);
      endrule

      rule receiveMessageHeader if (reqState == MessageHeaderRequested);
	 let vec <- toGet(readMifo).get();
	 let hdr = vec[0];
	 let methodId = hdr[31:16];
	 let messageWords = hdr[15:0];
	 methodIdReg <= methodId;
	 if (verbose)
	    $display("receiveMessageHeader hdr=%x methodId=%x messageWords=%d wordCount=%d", hdr, methodId, messageWords, wordCountReg);
	 wordCountReg <= wordCountReg - 1;
	 messageWordsReg <= messageWords - 1;
	 if (hdr == 0) begin
	    if (wordCountReg == 1)
	       reqState <= UpdateTail;
	    else
	       reqState <= Drain;
	 end
	 else if (wordCountReg <= messageWords)
	    reqState <= UpdateTail;
	 else if (messageWords == 1)
	    reqState <= MessageHeaderRequested;
	 else
	    reqState <= MessageRequested;
      endrule

      rule drain if (reqState == Drain);
	 let vec <- toGet(readMifo).get();
	 if (wordCountReg == 1)
	    reqState <= UpdateTail;
	 wordCountReg <= wordCountReg - 1;
      endrule

      rule receiveMessage if (reqState == MessageRequested);
	 let vec <- toGet(readMifo).get();
	 let data = vec[0];
	 if (verbose)
	    $display("receiveMessage data=%x messageWords=%d wordCount=%d", data, messageWordsReg, wordCountReg);
	 if (methodIdReg != 16'hFFFF)
	    portal.requests[methodIdReg].enq(data);

	 messageWordsReg <= messageWordsReg - 1;
	 wordCountReg <= wordCountReg - 1;
	 if (messageWordsReg == 1)
	    reqState <= MessageHeaderRequested;
	 else if (wordCountReg <= 1)
	    reqState <= UpdateTail;
      endrule

      rule updateTail if (reqState == UpdateTail);
	 if (verbose)
	    $display("updateTail: tail=%d", reqTailReg);
	 // update the tail pointer
	 writeEngine.cmdServer.request.put(MemengineCmd {sglId: sglIdReg,
							 base: 8,
							 len: 8,
							 burstLen: 8
							 });
	 writeEngine.dataPipe.enq(extend(reqTailReg));
	 reqState <= Waiting;
      endrule
      rule waiting if (reqState == Waiting);
	 let done <- writeEngine.cmdServer.response.get();
	 reqState <= Idle;
      endrule

      rule consumeResponse;
	 let response <- readEngine.cmdServer.response.get();
      endrule
endmodule

module mkSharedMemoryIndicationPortal#(PipePortal#(numRequests, numIndications, 32) portal,
				    MemreadServer#(64) readEngine,
				    MemwriteServer#(64) writeEngine,
				    Reg#(Bit#(32)) sglIdReg,
				    Reg#(Bool)     readyReg)(SharedMemoryPortal#(64));
      // read the head and tail pointers, if they are different, then read a request
      Reg#(Bit#(32)) indLimitReg <- mkReg(0);
      Reg#(Bit#(32)) indHeadReg <- mkReg(0);
      Reg#(Bit#(32)) indTailReg <- mkReg(0);
      Reg#(Bit#(16)) wordCountReg <- mkReg(0);
      Reg#(Bit#(16)) messageWordsReg <- mkReg(0);
      Reg#(Bit#(16)) methodIdReg <- mkReg(0);
      Reg#(SharedMemoryPortalState) indState <- mkReg(Idle);

   let verbose = True;

      rule updateIndHeadTail if (indState == Idle && readyReg);
	 readEngine.cmdServer.request.put(MemengineCmd { sglId: sglIdReg,
							base: 0,
							burstLen: 16,
							len: 16
							});
	 indState <= HeadRequested;
      endrule

      rule receiveIndHeadTail if (indState == HeadRequested || indState == TailRequested);
	 let data <- toGet(readEngine.dataPipe).get();
	 let w0 = data[31:0];
	 let w1 = data[63:32];
	 let head = indHeadReg;
	 let tail = indTailReg;
	 if (indState == HeadRequested) begin
	    indLimitReg <= w0;
	    indHeadReg <= w1;
	    head = w1;
	    indState <= TailRequested;
	 end
	 else begin
	    if (indTailReg == 0) begin
	       tail = w0;
	       indTailReg <= tail;
	    end
	    if (tail != indHeadReg)
	       indState <= RequestMessage;
	    else
	       indState <= Idle;
	 end
	 if (verbose)
	    $display("receiveIndHeadTail state=%d w0=%x w1=%x head=%d tail=%d limit=%d", indState, w0, w1, head, tail, indLimitReg);
      endrule
endmodule

module mkSharedMemoryPortal#(PipePortal#(numRequests, numIndications, 32) portal)(SharedMemoryPortal#(64));

   MemreadEngineV#(64,2,1) readEngine <- mkMemreadEngine();
   MemwriteEngineV#(64,2,1) writeEngine <- mkMemwriteEngine();

   Bool verbose = False;

   Reg#(Bit#(32)) sglIdReg <- mkReg(0);
   Reg#(Bool)     readyReg   <- mkReg(False);

   if (valueOf(numRequests) > 0) begin
      let readPortal <- mkSharedMemoryRequestPortal(portal, readEngine.read_servers[0], writeEngine.write_servers[0], sglIdReg, readyReg);
   end
   else if (valueOf(numIndications) > 0) begin
      let writePortal <- mkSharedMemoryIndicationPortal(portal, readEngine.read_servers[0], writeEngine.write_servers[0], sglIdReg, readyReg);
   end
   interface SharedMemoryPortalConfig cfg;
      method Action setSglId(Bit#(32) id);
	 sglIdReg <= id;
	 readyReg <= True;
      endmethod
   endinterface
   interface MemReadClient  readClient = readEngine.dmaClient;
   interface MemWriteClient writeClient = writeEngine.dmaClient;
   interface ReadOnly interrupt;
   endinterface
endmodule


Wait for reset to actually finish before sending resetDone indication.

// Copyright (c) 2016 Massachusetts Institute of Technology

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

`include "ProcConfig.bsv"

// ProcConnectal.bsv
// This is a wrapper to translate the generic Proc interface to an interface
// that is accepted by connectal.

import ConnectalConfig::*;

// This assumes a Proc.bsv file that contains the mkProc definition
import Abstraction::*;
import BuildVector::*;
import ClientServer::*;
import Clocks::*;
import Connectable::*;
import GetPut::*;
import Proc::*;
import Vector::*;
import VerificationPacket::*;
import MemTypes::*;
import RVTypes::*;
import SharedMemoryBridge::*;
import UncachedBridge::*;
import PerfMonitorConnectal::*;

// ProcControl
interface ProcControlRequest;
    method Action reset;
    method Action start(Bit#(64) startPc, Bit#(64) verificationPacketsToIgnore, Bool sendSynchronizationPackets);
    method Action stop;
endinterface
interface ProcControlIndication;
    method Action resetDone;
endinterface
// Platform
interface PlatformRequest;
    method Action configure(Bit#(32) ramSharedMemRefPointer, Bit#(64) ramSize, Bit#(32) romSharedMemRefPointer, Bit#(64) romSize);
endinterface
// Verification
interface VerificationIndication;
    method Action getVerificationPacket(VerificationPacket packet);
endinterface
// PerfMonitor interfaces defined in PerfMonitor package
// ExternalMMIO
interface ExternalMMIOResponse;
    method Action response(Bool write, Bit#(64) data);
    method Action triggerExternalInterrupt;
endinterface
interface ExternalMMIORequest;
    method Action request(Bool write, Bit#(4) length, Bit#(64) addr, Bit#(64) data);
endinterface

// This is the interface of all the requests, the indications are passed in as
// parameters to the mkProcConnectal module.
interface ProcConnectal;
    interface ProcControlRequest procControlRequest;
    interface PlatformRequest platformRequest;
    interface PerfMonitorRequest perfMonitorRequest;
    interface ExternalMMIOResponse externalMMIOResponse;
    interface Vector#(1, MemReadClient#(DataBusWidth)) dmaReadClient;
    interface Vector#(1, MemWriteClient#(DataBusWidth)) dmaWriteClient;
    interface Vector#(1, MemReadClient#(DataBusWidth)) romReadClient;
endinterface

module [Module] mkProcConnectal#(ProcControlIndication procControlIndication,
                                 VerificationIndication verificationIndication,
                                 PerfMonitorIndication perfMonitorIndication,
                                 ExternalMMIORequest externalMMIORequest)
                                (ProcConnectal);
    let clock <- exposeCurrentClock;
    let reset <- exposeCurrentReset;
    let procReset <- mkReset(10, True, clock);
    Reg#(Bool) resetSent <- mkReg(False);

    Proc#(MainMemoryWidth) proc <- mkProc(reset_by procReset.new_rst);

    // Address space: 0 - romSz
    SharedMemoryBridge#(MainMemoryWidth) romSharedMemoryBridge <- mkSharedMemoryBridge;
    let romToSharedMem <- mkConnection(proc.rom, romSharedMemoryBridge.to_proc);

    // Address space: 0 - ramSz
    SharedMemoryBridge#(MainMemoryWidth) ramSharedMemoryBridge <- mkSharedMemoryBridge;
    let ramToSharedMem <- mkConnection(proc.ram, ramSharedMemoryBridge.to_proc);

    // Address space: ?
    UncachedBridge uncachedBridge <- mkUncachedBridge;
    let memToUncachedMem <- mkConnection(proc.mmio, uncachedBridge.toProc);

    // rules for connecting indications
    rule finishReset(resetSent && (!procReset.isAsserted)
                               && (ramSharedMemoryBridge.numberFlyingOperations == 0)
                               && (romSharedMemoryBridge.numberFlyingOperations == 0));
        // wait for procReset to finish
        resetSent <= False;
        procControlIndication.resetDone;
    endrule
    rule connectVerificationIndication;
        let msg <- proc.getVerificationPacket;
        verificationIndication.getVerificationPacket(msg);
    endrule
    rule connectExternalMMIORequest;
        let msg <- uncachedBridge.externalMMIO.request.get;
        Bit#(4) length = (case (msg.size)
                B: 1;
                H: 2;
                W: 4;
                D: 8;
            endcase);
        // zero extend data in case XLEN is not 64
        externalMMIORequest.request(msg.write, length, msg.addr, zeroExtend(msg.data));
    endrule

    // request interfaces
    interface ProcControlRequest procControlRequest;
        method Action reset() if (!resetSent);
            // resets the processor
            procReset.assertReset();
            // flushes the pending memory requests
            ramSharedMemoryBridge.flushRespReqMem;
            romSharedMemoryBridge.flushRespReqMem;
            resetSent <= True;
        endmethod
        method Action start(Bit#(64) startPc, Bit#(64) verificationPacketsToIgnore, Bool sendSynchronizationPackets);
            proc.start(startPc, verificationPacketsToIgnore, sendSynchronizationPackets);
        endmethod
        method Action stop();
            proc.stop;
        endmethod
    endinterface
    interface PlatformRequest platformRequest;
        method Action configure(Bit#(32) ramSharedMemRefPointer, Bit#(64) ramSize, Bit#(32) romSharedMemRefPointer, Bit#(64) romSize);
            // configure shared memory
            ramSharedMemoryBridge.initSharedMem(ramSharedMemRefPointer, ramSize);
            // configure ROM
            romSharedMemoryBridge.initSharedMem(romSharedMemRefPointer, romSize);
        endmethod
    endinterface
    interface PerfMonitorRequest perfMonitorRequest;
        method Action reset;
            noAction;
        endmethod
        method Action setEnable(Bool en);
            noAction;
        endmethod
        method Action req(Bit#(32) index);
            perfMonitorIndication.resp(0);
        endmethod
    endinterface
    interface ExternalMMIOResponse externalMMIOResponse;
        method Action response(Bool write, Bit#(64) data);
            // truncate data in case XLEN is not 64
            uncachedBridge.externalMMIO.response.put( UncachedMemResp{write: write, data: truncate(data)} );
        endmethod
        method Action triggerExternalInterrupt;
            proc.triggerExternalInterrupt;
        endmethod
    endinterface

    // dma interfaces
    interface MemReadClient dmaReadClient = vec(ramSharedMemoryBridge.to_host_read);
    interface MemWriteClient dmaWriteClient = vec(ramSharedMemoryBridge.to_host_write);
    interface MemReadClient romReadClient = vec(romSharedMemoryBridge.to_host_read);
endmodule

// Copyright (c) 2016 Massachusetts Institute of Technology

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

`include "ProcConfig.bsv"

// ProcConnectal.bsv
// This is a wrapper to translate the generic Proc interface to an interface
// that is accepted by connectal.

import ConnectalConfig::*;

// This assumes a Proc.bsv file that contains the mkProc definition
import Abstraction::*;
import BuildVector::*;
import ClientServer::*;
import Clocks::*;
import Connectable::*;
import GetPut::*;
import Proc::*;
import Vector::*;
import VerificationPacket::*;
import MemTypes::*;
import RVTypes::*;
import SharedMemoryBridge::*;
import UncachedBridge::*;
import PerfMonitorConnectal::*;

// ProcControl
interface ProcControlRequest;
    method Action reset;
    method Action start(Bit#(64) startPc, Bit#(64) verificationPacketsToIgnore, Bool sendSynchronizationPackets);
    method Action stop;
endinterface
interface ProcControlIndication;
    method Action resetDone;
endinterface
// Platform
interface PlatformRequest;
    method Action configure(Bit#(32) ramSharedMemRefPointer, Bit#(64) ramSize, Bit#(32) romSharedMemRefPointer, Bit#(64) romSize);
endinterface
// Verification
interface VerificationIndication;
    method Action getVerificationPacket(VerificationPacket packet);
endinterface
// PerfMonitor interfaces defined in PerfMonitor package
// ExternalMMIO
interface ExternalMMIOResponse;
    method Action response(Bool write, Bit#(64) data);
    method Action triggerExternalInterrupt;
endinterface
interface ExternalMMIORequest;
    method Action request(Bool write, Bit#(4) length, Bit#(64) addr, Bit#(64) data);
endinterface

// This is the interface of all the requests, the indications are passed in as
// parameters to the mkProcConnectal module.
interface ProcConnectal;
    interface ProcControlRequest procControlRequest;
    interface PlatformRequest platformRequest;
    interface PerfMonitorRequest perfMonitorRequest;
    interface ExternalMMIOResponse externalMMIOResponse;
    interface Vector#(1, MemReadClient#(DataBusWidth)) dmaReadClient;
    interface Vector#(1, MemWriteClient#(DataBusWidth)) dmaWriteClient;
    interface Vector#(1, MemReadClient#(DataBusWidth)) romReadClient;
endinterface

module [Module] mkProcConnectal#(ProcControlIndication procControlIndication,
                                 VerificationIndication verificationIndication,
                                 PerfMonitorIndication perfMonitorIndication,
                                 ExternalMMIORequest externalMMIORequest)
                                (ProcConnectal);
    let clock <- exposeCurrentClock;
    let reset <- exposeCurrentReset;
    let procReset <- mkReset(10, True, clock);
    Reg#(Bool) resetSent <- mkReg(False);

    Proc#(MainMemoryWidth) proc <- mkProc(reset_by procReset.new_rst);

    // Address space: 0 - romSz
    SharedMemoryBridge#(MainMemoryWidth) romSharedMemoryBridge <- mkSharedMemoryBridge;
    let romToSharedMem <- mkConnection(proc.rom, romSharedMemoryBridge.to_proc);

    // Address space: 0 - ramSz
    SharedMemoryBridge#(MainMemoryWidth) ramSharedMemoryBridge <- mkSharedMemoryBridge;
    let ramToSharedMem <- mkConnection(proc.ram, ramSharedMemoryBridge.to_proc);

    // Address space: ?
    UncachedBridge uncachedBridge <- mkUncachedBridge;
    let memToUncachedMem <- mkConnection(proc.mmio, uncachedBridge.toProc);

    // rules for connecting indications
    rule finishReset(resetSent && (ramSharedMemoryBridge.numberFlyingOperations == 0)
                               && (romSharedMemoryBridge.numberFlyingOperations == 0));
        resetSent <= False;
        procControlIndication.resetDone;
    endrule
    rule connectVerificationIndication;
        let msg <- proc.getVerificationPacket;
        verificationIndication.getVerificationPacket(msg);
    endrule
    rule connectExternalMMIORequest;
        let msg <- uncachedBridge.externalMMIO.request.get;
        Bit#(4) length = (case (msg.size)
                B: 1;
                H: 2;
                W: 4;
                D: 8;
            endcase);
        // zero extend data in case XLEN is not 64
        externalMMIORequest.request(msg.write, length, msg.addr, zeroExtend(msg.data));
    endrule

    // request interfaces
    interface ProcControlRequest procControlRequest;
        method Action reset() if (!resetSent);
            // resets the processor
            procReset.assertReset();
            // flushes the pending memory requests
            ramSharedMemoryBridge.flushRespReqMem;
            romSharedMemoryBridge.flushRespReqMem;
            resetSent <= True;
        endmethod
        method Action start(Bit#(64) startPc, Bit#(64) verificationPacketsToIgnore, Bool sendSynchronizationPackets);
            proc.start(startPc, verificationPacketsToIgnore, sendSynchronizationPackets);
        endmethod
        method Action stop();
            proc.stop;
        endmethod
    endinterface
    interface PlatformRequest platformRequest;
        method Action configure(Bit#(32) ramSharedMemRefPointer, Bit#(64) ramSize, Bit#(32) romSharedMemRefPointer, Bit#(64) romSize);
            // configure shared memory
            ramSharedMemoryBridge.initSharedMem(ramSharedMemRefPointer, ramSize);
            // configure ROM
            romSharedMemoryBridge.initSharedMem(romSharedMemRefPointer, romSize);
        endmethod
    endinterface
    interface PerfMonitorRequest perfMonitorRequest;
        method Action reset;
            noAction;
        endmethod
        method Action setEnable(Bool en);
            noAction;
        endmethod
        method Action req(Bit#(32) index);
            perfMonitorIndication.resp(0);
        endmethod
    endinterface
    interface ExternalMMIOResponse externalMMIOResponse;
        method Action response(Bool write, Bit#(64) data);
            // truncate data in case XLEN is not 64
            uncachedBridge.externalMMIO.response.put( UncachedMemResp{write: write, data: truncate(data)} );
        endmethod
        method Action triggerExternalInterrupt;
            proc.triggerExternalInterrupt;
        endmethod
    endinterface

    // dma interfaces
    interface MemReadClient dmaReadClient = vec(ramSharedMemoryBridge.to_host_read);
    interface MemWriteClient dmaWriteClient = vec(ramSharedMemoryBridge.to_host_write);
    interface MemReadClient romReadClient = vec(romSharedMemoryBridge.to_host_read);
endmodule


simplify

simplify

// Copyright (c) 2014 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import Vector            :: *;
import Clocks            :: *;
import GetPut            :: *;
import FIFO              :: *;
import Connectable       :: *;
import ClientServer      :: *;
import Xilinx            :: *;
import DefaultValue      :: *;
import PcieSplitter      :: *;
import PcieTracer        :: *;
import PcieGearbox       :: *;
import XbsvXilinx7Pcie   :: *;
import PCIEWRAPPER       :: *;
import Portal            :: *;
import Leds              :: *;
import Top               :: *;
import AxiSlaveEngine    :: *;
import MemMasterEngine   :: *;
import AxiMasterSlave    :: *;
import AxiDma            :: *;
import PcieCsr           :: *;
import MemSlave          :: *;
import Dma               :: *;

`ifdef Artix7
typedef 4 PcieLanes;
typedef 4 NumLeds;
`else
typedef 8 PcieLanes;
typedef 8 NumLeds;
`endif
`ifndef DataBusWidth
`define DataBusWidth 64
`endif
`ifndef NumberOfMasters
`define NumberOfMasters 1
`endif
`ifndef PinType
`define PinType Empty
`endif

typedef (function Module#(PortalTop#(40, dsz, ipins, `NumberOfMasters)) mkPortalTop()) MkPortalTop#(numeric type dsz, type ipins);

interface PcieTop#(type ipins);
   (* prefix="PCIE" *)
   interface PciewrapPci_exp#(PcieLanes) pcie;
   (* always_ready *)
   method Bit#(NumLeds) leds();
   interface ipins       pins;
endinterface

interface PcieHost#(numeric type dsz);
   interface Vector#(16,MSIX_Entry) msixEntry;
   interface MemMaster#(32,32) master;
   interface Axi3Slave#(40,dsz,6)  slave;
   interface Put#(Tuple2#(Bit#(64),Bit#(32))) interruptRequest;
endinterface

module [Module] mkPcieHost #(Clock epClock250, Reset epReset250, PciId my_pciId,
Server#(TLPData#(8), TLPData#(8)) ep_tlp)(PcieHost#(dsz))
provisos(
   Mul#(TDiv#(dsz, 8), 8, dsz),
    Add#(a__, TDiv#(dsz, 32), 8),
    Add#(b__, TMul#(32, TDiv#(dsz, 32)), 256),
    Add#(c__, TMul#(8, TDiv#(dsz, 32)), 64),
    Add#(d__, dsz, 256),
    Add#(e__, 32, dsz),
    Mul#(TDiv#(dsz, 32), 32, dsz));
   Clock epClock125 <- exposeCurrentClock();
   Reset epReset125 <- exposeCurrentReset();
   let dispatcher <- mkTLPDispatcher;
   let arbiter    <- mkTLPArbiter;
   Axi3Slave#(40,dsz,6) slav = ?;
   Put#(Tuple2#(Bit#(64),Bit#(32))) intreq = ?;

   Vector#(PortMax, MemMaster#(32,32)) mvec;
   for (Integer i = 0; i < valueOf(PortMax); i=i+1) begin
       let serv = (interface Server;
                     interface response = dispatcher.out[i];
                     interface request = arbiter.in[i];
                  endinterface);
       if (i == portAxi) begin
           AxiSlaveEngine#(dsz) sEngine <- mkAxiSlaveEngine(my_pciId);
           mkConnection(serv, sEngine.tlp);
           slav = sEngine.slave;
       end
       else begin
           MemMasterEngine mEngine <- mkMemMasterEngine(my_pciId);
           mkConnection(serv, mEngine.tlp);
           mvec[i] = mEngine.master;
           if (i == portPortal)
               intreq = mEngine.interruptRequest;
       end
   end

   // The PCIE endpoint is processing TLPData#(8)s at 250MHz.  The
   // AXI bridge is accepting TLPData#(16)s at 125 MHz. The
   // connection between the endpoint and the AXI contains GearBox
   // instances for the TLPData#(8)@250 <--> TLPData#(16)@125
   // conversion.
   PcieGearbox gb <- mkPcieGearbox(epClock250, epReset250, epClock125, epReset125);
   mkConnection(ep_tlp, gb.tlp, clocked_by epClock250, reset_by epReset250);

   PcieTracer  traceif <- mkPcieTracer();
   mkConnection(gb.pci, traceif.pci);
   mkConnection(traceif.bus, (interface Client;
                                 interface request = arbiter.outToBus;
                                 interface response = dispatcher.inFromBus;
                              endinterface));

   PcieControlAndStatusRegs csr <- mkPcieControlAndStatusRegs(traceif.tlpdata);
   MemSlave#(32,32) my_slave <- mkMemSlave(csr.client);
   mkConnection(mvec[portConfig], my_slave);

   interface msixEntry = csr.msixEntry;
   interface master = mvec[portPortal];
   interface slave = slav;
   interface interruptRequest = intreq;
endmodule: mkPcieHost

(* no_default_clock, no_default_reset *)
module [Module] mkPcieTopFromPortal #(Clock pci_sys_clk_p, Clock pci_sys_clk_n, Clock sys_clk_p, Clock sys_clk_n, Reset pci_sys_reset_n,
				      MkPortalTop#(dsz, ipins) mkPortalTop)
   (PcieTop#(ipins))
   provisos (Mul#(TDiv#(dsz, 32), 32, dsz),
	     Add#(b__, 32, dsz),
	     Add#(c__, dsz, 256),
	     Add#(d__, TMul#(8, TDiv#(dsz, 32)), 64),
	     Add#(e__, TMul#(32, TDiv#(dsz, 32)), 256),
	     Add#(f__, TDiv#(dsz, 32), 8),
	     Mul#(TDiv#(dsz, 8), 8, dsz)
      );

   Clock sys_clk_200mhz <- mkClockIBUFDS(sys_clk_p, sys_clk_n);
   Clock sys_clk_200mhz_buf <- mkClockBUFG(clocked_by sys_clk_200mhz);
   Clock pci_clk_100mhz_buf <- mkClockIBUFDS_GTE2(True, pci_sys_clk_p, pci_sys_clk_n);

   // Instantiate the PCIE endpoint
   PCIExpressX7#(PcieLanes) _ep <- mkPCIExpressEndpointX7( defaultValue
						  , clocked_by pci_clk_100mhz_buf
						  , reset_by pci_sys_reset_n
						  );
   // The PCIe endpoint exports full (250MHz) and half-speed (125MHz) clocks
   Clock epClock250 = _ep.user.clk_out;
   Reset user_reset_n <- mkResetInverter(_ep.user.reset_out);
   Reset epReset250 <- mkAsyncReset(4, user_reset_n, epClock250);

   ClockGenerator7Params     params = defaultValue;
   params.clkin1_period    = 4.000;
   params.clkin_buffer     = False;
   params.clkfbout_mult_f  = 4.000;
   params.clkout0_divide_f = 8.000;
   ClockGenerator7           clkgen <- mkClockGenerator7(params, clocked_by _ep.user.clk_out, reset_by user_reset_n);
   Clock epClock125 = clkgen.clkout0; /* half speed user_clk */
   Reset epReset125 <- mkAsyncReset(4, user_reset_n, epClock125);

   let portalTop <- mkPortalTop(clocked_by epClock125, reset_by epReset125);
   PcieHost#(dsz) pciehost <- mkPcieHost(epClock250, epReset250,
         PciId{ bus:  _ep.cfg.bus_number(), dev: _ep.cfg.device_number(), func: _ep.cfg.function_number()},
         _ep.tlp, clocked_by epClock125, reset_by epReset125);

   mkConnection(pciehost.master, portalTop.slave, clocked_by epClock125, reset_by epReset125);
   if (`NumberOfMasters > 0) begin
      Vector#(`NumberOfMasters,Axi3Master#(40,dsz,6)) m_axis;
      m_axis[0] <- mkAxiDmaMaster(portalTop.masters[0], clocked_by epClock125, reset_by epReset125);
      mkConnection(m_axis[0], pciehost.slave, clocked_by epClock125, reset_by epReset125);
   end

   // going from level to edge-triggered interrupt
   Vector#(15, Reg#(Bool)) interruptRequested <- replicateM(mkReg(False, clocked_by epClock125, reset_by epReset125));
   rule interrupt_rule;
     Integer intr_num = 0;
     for (Integer i = 0; i < 15; i = i + 1) begin
	 if (intr_num == 0 && portalTop.interrupt[i] && !interruptRequested[i])
             intr_num = i+1;
	 interruptRequested[i] <= portalTop.interrupt[i];
     end
     if (intr_num != 0) begin // i= 0 for the directory
        MSIX_Entry msixEntry = pciehost.msixEntry[intr_num];
        pciehost.interruptRequest.put(tuple2({msixEntry.addr_hi, msixEntry.addr_lo}, msixEntry.msg_data));
     end
   endrule

   interface pcie = _ep.pcie;
   method Bit#(NumLeds) leds();
      return extend({_ep.user.lnk_up(),3'd2});
   endmethod
   interface pins = portalTop.pins;
endmodule: mkPcieTopFromPortal

(* synthesize *)
module mkSynthesizeablePortalTop(PortalTop#(40, `DataBusWidth, Empty, `NumberOfMasters));
   let top <- mkPortalTop();
   interface masters = top.masters;
   interface slave = top.slave;
   interface interrupt = top.interrupt;
   interface leds = top.leds;
   interface pins = top.pins;
endmodule

module mkPcieTop #(Clock pci_sys_clk_p, Clock pci_sys_clk_n, Clock sys_clk_p, Clock sys_clk_n, Reset pci_sys_reset_n)
   (PcieTop#(`PinType));
   let top <- mkPcieTopFromPortal(pci_sys_clk_p, pci_sys_clk_n, sys_clk_p, sys_clk_n, pci_sys_reset_n,mkSynthesizeablePortalTop);
   return top;
endmodule: mkPcieTop

// Copyright (c) 2014 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import Vector            :: *;
import Clocks            :: *;
import GetPut            :: *;
import FIFO              :: *;
import Connectable       :: *;
import ClientServer      :: *;
import Xilinx            :: *;
import DefaultValue      :: *;
import PcieSplitter      :: *;
import PcieTracer        :: *;
import PcieGearbox       :: *;
import XbsvXilinx7Pcie   :: *;
import PCIEWRAPPER       :: *;
import Portal            :: *;
import Leds              :: *;
import Top               :: *;
import AxiSlaveEngine    :: *;
import MemMasterEngine   :: *;
import AxiMasterSlave    :: *;
import AxiDma            :: *;
import PcieCsr           :: *;
import MemSlave          :: *;
import Dma               :: *;

`ifdef Artix7
typedef 4 PcieLanes;
typedef 4 NumLeds;
`else
typedef 8 PcieLanes;
typedef 8 NumLeds;
`endif
`ifndef DataBusWidth
`define DataBusWidth 64
`endif
`ifndef NumberOfMasters
`define NumberOfMasters 1
`endif
`ifndef PinType
`define PinType Empty
`endif

typedef (function Module#(PortalTop#(40, dsz, ipins, `NumberOfMasters)) mkPortalTop()) MkPortalTop#(numeric type dsz, type ipins);

interface PcieTop#(type ipins);
   (* prefix="PCIE" *)
   interface PciewrapPci_exp#(PcieLanes) pcie;
   (* always_ready *)
   method Bit#(NumLeds) leds();
   interface ipins       pins;
endinterface

interface PcieHost#(numeric type dsz);
   interface Vector#(16,MSIX_Entry) msixEntry;
   interface MemMaster#(32,32) master;
   interface Axi3Slave#(40,dsz,6)  slave;
   interface Put#(Tuple2#(Bit#(64),Bit#(32))) interruptRequest;
endinterface

module [Module] mkPcieHost #(Clock epClock250, Reset epReset250, PciId my_pciId,
Server#(TLPData#(8), TLPData#(8)) ep_tlp)(PcieHost#(dsz))
provisos(
   Mul#(TDiv#(dsz, 8), 8, dsz),
    Add#(a__, TDiv#(dsz, 32), 8),
    Add#(b__, TMul#(32, TDiv#(dsz, 32)), 256),
    Add#(c__, TMul#(8, TDiv#(dsz, 32)), 64),
    Add#(d__, dsz, 256),
    Add#(e__, 32, dsz),
    Mul#(TDiv#(dsz, 32), 32, dsz));
   Clock epClock125 <- exposeCurrentClock();
   Reset epReset125 <- exposeCurrentReset();

   let dispatcher <- mkTLPDispatcher;
   let arbiter    <- mkTLPArbiter;

   //Client#(TLPData#(16),TLPData#(16)) busClient;
   Vector#(PortMax, Server#(TLPData#(16),TLPData#(16))) servers;
   for (Integer i = 0; i < valueOf(PortMax); i=i+1) begin
       servers[i] = (interface Server;
                     interface response = dispatcher.out[i];
                     interface request = arbiter.in[i];
                  endinterface);
   end

   MemMasterEngine splitEngine <- mkMemMasterEngine(my_pciId);
   mkConnection(servers[portConfig], splitEngine.tlp);
   MemMasterEngine portalEngine <- mkMemMasterEngine(my_pciId);
   mkConnection(servers[portPortal], portalEngine.tlp);
   AxiSlaveEngine#(dsz) dmaEngine <- mkAxiSlaveEngine(my_pciId);
   mkConnection(servers[portAxi], dmaEngine.tlp);

   // The PCIE endpoint is processing TLPData#(8)s at 250MHz.  The
   // AXI bridge is accepting TLPData#(16)s at 125 MHz. The
   // connection between the endpoint and the AXI contains GearBox
   // instances for the TLPData#(8)@250 <--> TLPData#(16)@125
   // conversion.
   PcieGearbox gb <- mkPcieGearbox(epClock250, epReset250, epClock125, epReset125);
   mkConnection(ep_tlp, gb.tlp, clocked_by epClock250, reset_by epReset250);

   PcieTracer  traceif <- mkPcieTracer();
   mkConnection(gb.pci, traceif.pci);
   mkConnection(traceif.bus, (
       interface Client;
          interface request = arbiter.outToBus;
          interface response = dispatcher.inFromBus;
       endinterface));

   PcieControlAndStatusRegs csr <- mkPcieControlAndStatusRegs(traceif.tlpdata);
   MemSlave#(32,32) my_slave <- mkMemSlave(csr.client);
   mkConnection(splitEngine.master, my_slave);

   interface msixEntry = csr.msixEntry;
   interface master = portalEngine.master;
   interface slave = dmaEngine.slave;
   interface interruptRequest = portalEngine.interruptRequest;
endmodule: mkPcieHost

(* no_default_clock, no_default_reset *)
module [Module] mkPcieTopFromPortal #(Clock pci_sys_clk_p, Clock pci_sys_clk_n, Clock sys_clk_p, Clock sys_clk_n, Reset pci_sys_reset_n,
				      MkPortalTop#(dsz, ipins) mkPortalTop)
   (PcieTop#(ipins))
   provisos (Mul#(TDiv#(dsz, 32), 32, dsz),
	     Add#(b__, 32, dsz),
	     Add#(c__, dsz, 256),
	     Add#(d__, TMul#(8, TDiv#(dsz, 32)), 64),
	     Add#(e__, TMul#(32, TDiv#(dsz, 32)), 256),
	     Add#(f__, TDiv#(dsz, 32), 8),
	     Mul#(TDiv#(dsz, 8), 8, dsz)
      );

   Clock sys_clk_200mhz <- mkClockIBUFDS(sys_clk_p, sys_clk_n);
   Clock sys_clk_200mhz_buf <- mkClockBUFG(clocked_by sys_clk_200mhz);
   Clock pci_clk_100mhz_buf <- mkClockIBUFDS_GTE2(True, pci_sys_clk_p, pci_sys_clk_n);

   // Instantiate the PCIE endpoint
   PCIExpressX7#(PcieLanes) _ep <- mkPCIExpressEndpointX7( defaultValue
						  , clocked_by pci_clk_100mhz_buf
						  , reset_by pci_sys_reset_n
						  );
   // The PCIe endpoint exports full (250MHz) and half-speed (125MHz) clocks
   Clock epClock250 = _ep.user.clk_out;
   Reset user_reset_n <- mkResetInverter(_ep.user.reset_out);
   Reset epReset250 <- mkAsyncReset(4, user_reset_n, epClock250);

   ClockGenerator7Params     params = defaultValue;
   params.clkin1_period    = 4.000;
   params.clkin_buffer     = False;
   params.clkfbout_mult_f  = 4.000;
   params.clkout0_divide_f = 8.000;
   ClockGenerator7           clkgen <- mkClockGenerator7(params, clocked_by _ep.user.clk_out, reset_by user_reset_n);
   Clock epClock125 = clkgen.clkout0; /* half speed user_clk */
   Reset epReset125 <- mkAsyncReset(4, user_reset_n, epClock125);

   let portalTop <- mkPortalTop(clocked_by epClock125, reset_by epReset125);
   PcieHost#(dsz) pciehost <- mkPcieHost(epClock250, epReset250,
         PciId{ bus:  _ep.cfg.bus_number(), dev: _ep.cfg.device_number(), func: _ep.cfg.function_number()},
         _ep.tlp, clocked_by epClock125, reset_by epReset125);

   mkConnection(pciehost.master, portalTop.slave, clocked_by epClock125, reset_by epReset125);
   if (`NumberOfMasters > 0) begin
      Vector#(`NumberOfMasters,Axi3Master#(40,dsz,6)) m_axis;
      m_axis[0] <- mkAxiDmaMaster(portalTop.masters[0], clocked_by epClock125, reset_by epReset125);
      mkConnection(m_axis[0], pciehost.slave, clocked_by epClock125, reset_by epReset125);
   end

   // going from level to edge-triggered interrupt
   Vector#(15, Reg#(Bool)) interruptRequested <- replicateM(mkReg(False, clocked_by epClock125, reset_by epReset125));
   rule interrupt_rule;
     Integer intr_num = 0;
     for (Integer i = 0; i < 15; i = i + 1) begin
	 if (intr_num == 0 && portalTop.interrupt[i] && !interruptRequested[i])
             intr_num = i+1;
	 interruptRequested[i] <= portalTop.interrupt[i];
     end
     if (intr_num != 0) begin // i= 0 for the directory
        MSIX_Entry msixEntry = pciehost.msixEntry[intr_num];
        pciehost.interruptRequest.put(tuple2({msixEntry.addr_hi, msixEntry.addr_lo}, msixEntry.msg_data));
     end
   endrule

   interface pcie = _ep.pcie;
   method Bit#(NumLeds) leds();
      return extend({_ep.user.lnk_up(),3'd2});
   endmethod
   interface pins = portalTop.pins;
endmodule: mkPcieTopFromPortal

(* synthesize *)
module mkSynthesizeablePortalTop(PortalTop#(40, `DataBusWidth, Empty, `NumberOfMasters));
   let top <- mkPortalTop();
   interface masters = top.masters;
   interface slave = top.slave;
   interface interrupt = top.interrupt;
   interface leds = top.leds;
   interface pins = top.pins;
endmodule

module mkPcieTop #(Clock pci_sys_clk_p, Clock pci_sys_clk_n, Clock sys_clk_p, Clock sys_clk_n, Reset pci_sys_reset_n)
   (PcieTop#(`PinType));
   let top <- mkPcieTopFromPortal(pci_sys_clk_p, pci_sys_clk_n, sys_clk_p, sys_clk_n, pci_sys_reset_n,mkSynthesizeablePortalTop);
   return top;
endmodule: mkPcieTop