pre-calculate last_row

pre-calculate last_row

Change-Id: I2d51b457f2cb46aaf2edde9f711c7aed7401faab

// Copyright (c) 2014 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import FIFO::*;
import FIFOF::*;
import MIMO::*;
import DefaultValue::*;
import SpecialFIFOs::*;
import Vector::*;
import BRAM::*;
import DmaVector::*;
import PortalMemory::*;
import MemTypes::*;
import MemreadEngine::*;
import MemwriteEngine::*;
import MemUtils::*;
import FloatingPoint::*;
import Pipe::*;
import Arith::*;
import FloatOps::*;
import Timer::*;
import RbmTypes::*;
import Assert::*;
import Connectable::*;
import Clocks::*;
import Gearbox::*;
import XilinxCells::*;
import HostInterface::*;

interface SharedDotProdDebug#(numeric type k);
   interface PipeOut#(Bit#(32)) macCount;
endinterface

interface SharedDotProdServer#(numeric type k);
   interface Put#(MmToken)                 aInput;
   interface Put#(MmToken)                 bInput;
   interface Vector#(k, PipeOut#(MmToken)) pipes;
   interface SharedDotProdDebug#(k) debug;
endinterface

typedef struct {
`ifdef TAGGED_TOKENS
   UInt#(32) row;
   UInt#(32) col;
`endif
   Float v;
   Bool first;
   Bool last;
} MmToken deriving (Eq,Bits);

typedef 8 UB_MulLat; // upper bound on MUL latency?
typedef 8 UB_AddLat; // upper bound on ADD latency?

(* synthesize *)
module  mkSharedInterleavedDotProdServer#(UInt#(TLog#(TMul#(J,K))) label)(SharedDotProdServer#(K))
   provisos(Div#(UB_AddLat,K,gatherSz));

   let ub_MulLat = valueOf(UB_MulLat);
   let ub_AddLat = valueOf(UB_AddLat);
   let kk = valueOf(K);

   Bool verbose = False; //label == 0;

   Reg#(UInt#(20)) countReg     <- mkReg(0);

   FloatAlu mul   <- mkFloatMultiplier(defaultValue);
   FloatAlu adder <- mkFloatAdder(defaultValue);
   FIFOF#(Float) adder_buffer <- mkSizedFIFOF(valueOf(TMul#(K,gatherSz)));

`ifdef TAGGED_TOKENS
   FIFO#(Tuple2#(UInt#(32),UInt#(32))) tag_fifo <- mkSizedFIFO(ub_MulLat);
   Vector#(K,Reg#(Tuple2#(UInt#(32),UInt#(32)))) tag_regs <- replicateM(mkRegU);
`endif

   Reg#(Bit#(32)) cycles <- mkReg(0);
   rule countCycles;
      cycles <= cycles + 1;
   endrule

   FIFOF#(MmToken)                          afifo   <- mkFIFOF();
   PipeOut#(MmToken)                        aFunnel = toPipeOut(afifo);

   FIFOF#(MmToken)                          bfifo <- mkFIFOF();
   PipeOut#(MmToken)                        bFunnel = toPipeOut(bfifo);

   Reg#(Bit#(16)) firstCnt <- mkReg(0);
   Reg#(Bool)         gReg <- mkReg(False);
   FIFOF#(Float)     gFifo <- mkSizedFIFOF(kk);
   Reg#(Bit#(16))  lastCnt <- mkReg(0);
   Reg#(Bit#(16))gatherCnt <- mkReg(0);
   Reg#(Bool) gather_phase <- mkReg(False);
   //invariant: gather_phase = lastCnt == fromInteger(kk);

   FIFOF#(Tuple2#(Bool,Bool))    flFifo <- mkSizedFIFOF(ub_MulLat);
   Vector#(K,FIFOF#(MmToken))    dotfifos <- replicateM(mkFIFOF1);
   Reg#(Bit#(TAdd#(1,TLog#(K)))) rowReg <- mkReg(0);
   Reg#(Bool)                lastRowReg <- mkReg(False);

   Reg#(Bit#(32)) lastMul <- mkReg(0);
   Reg#(Bit#(32)) lastAcc <- mkReg(0);
   Reg#(Bit#(32)) lastGather <- mkReg(0);
   Reg#(Bit#(32)) macs <- mkReg(0);

   (* fire_when_enabled *)
   rule connect_adder_buffer;
      match {.acc,.*} <- adder.response.get();
      adder_buffer.enq(acc);
      macs <= macs+1;
   endrule

   (* fire_when_enabled *)
   rule multiply;
      lastMul <= cycles;
      let a <- toGet(aFunnel).get();
      let b <- toGet(bFunnel).get();
      flFifo.enq(tuple2(a.first,a.last));
      if (a.first != b.first)
	 $display("****\n    Warning: a.first=%d != b.first=%d\n****", a.first, b.first);
      if (a.last != b.last)
	 $display("****\n    Warning: a.last=%d != b.last=%d\n****", a.last, b.last);
      if (verbose)
	 $display("%08d multiply: label=%d mulin first=%d last=%d", cycles-lastMul, label, a.first, a.last);
      mul.request.put(tuple2(a.v, b.v));
`ifdef TAGGED_TOKENS
      if(label==0) $display("%d (%d,%d)(%d,%d)", label, a.row, a.col, b.row, b.col);
      tag_fifo.enq(tuple2(a.row,b.col));
`endif
   endrule

   function Action incrementRowReg =
      (action
	  // I have to do this check (instead of relying on wrap-around) because
	  // rowReg has an extra bit to compenseate for bsc's silly Bit#(0) handling
	  if (rowReg+1 == fromInteger(kk)) begin
	     rowReg <= 0;
	     lastRowReg <= (0 == fromInteger(kk-1));
	  end
	  else begin
	     rowReg <= rowReg+1;
	     lastRowReg <= (rowReg == fromInteger(kk-2));
	  end
       endaction);

   (* fire_when_enabled *)
   rule accumulate if (!gather_phase);
      incrementRowReg;
      lastAcc <= cycles;
      match {.first, .last} <- toGet(flFifo).get();
      if (verbose) $display("%08d accumulate: label=%d mulout first=%d last=%d firstCnt=%d lastCnt=%d",
			    cycles-lastAcc, label, first, last, firstCnt, lastCnt);
      match {.resp,.*} <- mul.response.get;
      let acc = unpack(0);
      if (firstCnt == fromInteger(valueOf(TMul#(K,gatherSz))))
	 acc <- toGet(adder_buffer).get;
      else begin
	 firstCnt <= firstCnt+1;
      end
      adder.request.put(tuple2(resp,acc));
      if(last) begin
	 lastCnt <= lastCnt+1;
	 gather_phase <= (lastCnt == fromInteger(kk)-1);
      end
`ifdef TAGGED_TOKENS
      let row = rowReg;
      let t <- toGet(tag_fifo).get;
      tag_regs[row] <= t;
`endif
   endrule

   (* fire_when_enabled *)
   rule gather if (gather_phase);
      incrementRowReg;
      lastGather <= cycles;
      let row = rowReg;
      let last_row = lastRowReg;
      let last_pass = gatherCnt+1 == fromInteger(valueOf(gatherSz));
      if (verbose)
	 $display("%08d gather: gather=%d row=%d last_pass=%d last_row=%d, gReg=%d",
		  cycles-lastGather, gatherCnt, row, last_pass, last_row, gReg);
      let x <- toGet(adder_buffer).get;
      if (!last_pass) begin
	 if (last_row) begin
	    if (gReg) gatherCnt <= gatherCnt+1;
	    gReg <= !gReg;
	 end
	 if(gReg) begin
	    let y <- toGet(gFifo).get;
	    adder.request.put(tuple2(x,y));
	 end
	 else begin
	    gFifo.enq(x);
	 end
      end
      else begin
`ifdef TAGGED_TOKENS
      	 let row = tpl_1(tag_regs[row]);
      	 let col = tpl_2(tag_regs[row]);
	 dotfifos[row].enq(MmToken{row:row, col:col, v:x});
`else
	 dotfifos[row].enq(MmToken{v:x});
`endif
	 if (last_row) begin
	    gatherCnt <= 0;
	    lastCnt <= 0;
	    firstCnt <= 0;
	    gather_phase <= False;
	 end
      end
   endrule

   Vector#(K,PipeOut#(MmToken)) dotpipes = map(toPipeOut, dotfifos);

   interface Put aInput;
      method Action put(MmToken a);
   	 afifo.enq(a);
	 countReg <= countReg+1;
      endmethod
   endinterface
   interface Put bInput   = toPut(bfifo);
   interface Vector pipes = dotpipes;
   interface SharedDotProdDebug debug;
      interface PipeOut  macCount = toPipeOut(macs._read);
   endinterface
endmodule : mkSharedInterleavedDotProdServer

interface MmTileDebug;
   interface PipeOut#(Bit#(32)) macCount;
endinterface

interface MmTile;
   interface Vector#(RowsPerTile, Put#(MmToken)) aInputs;
   interface Vector#(RowsPerTile, Put#(MmToken)) bInputs;
   interface Vector#(RowsPerTile, PipeOut#(Vector#(N, MmToken))) fxPipes;
   interface MmTileDebug debug;
endinterface

function Put#(a) toCountedPut(Reg#(Bit#(n)) r, Put#(a) p);
   return (interface Put#(a);
      method Action put(a v);
	 r <= r+1;
	 p.put(v);
      endmethod
      endinterface);
endfunction

(* synthesize *)
module  mkMmTile#(Clock slowClock, Reset slowReset, UInt#(TLog#(T)) tile)(MmTile);

   let rowsPerTile = valueOf(RowsPerTile);
   let kk = valueOf(K);

   Vector#(RowsPerTile, Reg#(Bit#(32))) aMmTokensPutRegs <- replicateM(mkReg(0));
   Vector#(RowsPerTile, Reg#(Bit#(32))) bMmTokensPutRegs <- replicateM(mkReg(0));
   Vector#(RowsPerTile, Reg#(Bit#(32))) aMmTokensReadRegs <- replicateM(mkReg(0));
   Vector#(RowsPerTile, Reg#(Bit#(32))) bMmTokensReadRegs <- replicateM(mkReg(0));

   Vector#(RowsPerTile, FIFOF#(MmToken))   aFifos <- replicateM(mkFIFOF);
   Vector#(RowsPerTile, PipeOut#(MmToken)) aPipes = zipWith(toCountedPipeOut, aMmTokensReadRegs, map(toPipeOut, aFifos));
   Vector#(RowsPerTile,  FIFOF#(MmToken))   bFifos <- replicateM(mkFIFOF);
   Vector#(RowsPerTile,  PipeOut#(MmToken)) bPipes = zipWith(toCountedPipeOut, bMmTokensReadRegs, map(toPipeOut, bFifos));

   function Vector#(k,PipeOut#(MmToken)) getDotProdServerPipes(SharedDotProdServer#(k) s); return s.pipes; endfunction
   Vector#(RowsPerTile, SharedDotProdServer#(K)) fxdotprods <- mapM(mkSharedInterleavedDotProdServer, map(fromInteger,genVector));
   Vector#(RowsPerTile, Vector#(K, PipeOut#(MmToken))) fxpipes = map(getDotProdServerPipes, fxdotprods);
//`define USE_MIMO_DFIFOS // this version is faster
   let fastClock <- exposeCurrentClock();
   let fastReset <- exposeCurrentReset();
`ifndef USE_MIMO_DFIFOS
   Vector#(RowsPerTile, PipeOut#(Vector#(K, MmToken))) fxPipesK <- mapM(mkJoinVector(id), fxpipes);
   Vector#(RowsPerTile, PipeOut#(MmToken)) fxPipes1MmToken <- mapM(mkFunnel1, fxPipesK);
   Vector#(RowsPerTile, PipeOut#(Vector#(1, MmToken))) fxPipes1 = map(mapPipe(replicate), fxPipes1MmToken);
`else
   MIMOConfiguration mimoCfg = defaultValue;
   Vector#(RowsPerTile, MIMO#(K,1,TAdd#(K,1),MmToken)) dfifos <- replicateM(mkMIMO(mimoCfg));
   Vector#(RowsPerTile, PipeOut#(Vector#(1, MmToken))) fxPipes1 = map(toPipeOut, dfifos);
`endif
   Vector#(RowsPerTile, Gearbox#(1, N, MmToken)) gearboxes <- replicateM(mk1toNGearbox(fastClock, fastReset, slowClock, slowReset));
   Vector#(RowsPerTile, PipeIn#(Vector#(1,MmToken))) toGearboxes = map(toPipeIn, gearboxes);
   Vector#(RowsPerTile, PipeOut#(Vector#(N, MmToken))) fromGearboxes = map(toPipeOut, gearboxes);
   mapM(uncurry(mkConnection), zip(fxPipes1, toGearboxes));
   // introduce a buffer to help vivado meet timing on vc707
   Vector#(RowsPerTile, FIFOF#(Vector#(N,MmToken)))    tokenfifos <- replicateM(mkFIFOF(clocked_by slowClock, reset_by slowReset));
   Vector#(RowsPerTile, PipeIn#(Vector#(N,MmToken))) toMmTokenFifos = map(toPipeIn, tokenfifos);
   mapM(uncurry(mkConnection), zip(fromGearboxes, toMmTokenFifos), clocked_by slowClock, reset_by slowReset);
   Vector#(RowsPerTile, PipeOut#(Vector#(N, MmToken))) fxPipesN = map(toPipeOut, tokenfifos);

   FirstLastPipe#(UInt#(MMSize)) firstLastPipe          <- mkFirstLastPipe();
   Vector#(2, PipeOut#(Tuple2#(Bool,Bool))) firstLastPipes <- mkForkVector(firstLastPipe.pipe);

   for (Integer j = 0; j < rowsPerTile; j = j + 1) begin
      //mkConnection(toGet(aPipes[j]), fxdotprods[j].aInput);
      //mkConnection(toGet(bPipes[j]), fxdotprods[j].bInput);
      (* fire_when_enabled *)
      rule connectA;
	 let x <- toGet(aPipes[j]).get;
	 fxdotprods[j].aInput.put(x);
      endrule
      (* fire_when_enabled *)
      rule connectB;
	 let x <- toGet(bPipes[j]).get;
	 fxdotprods[j].bInput.put(x);
      endrule
   end

`ifdef USE_MIMO_DFIFOS
   for (Integer j = 0; j < rowsPerTile; j = j + 1) begin
      rule dotProdValue;
	 Vector#(K,MmToken) vs;
	 for (Integer k = 0; k < kk; k = k + 1) begin
	    let v <- toGet(fxpipes[j][k]).get();
	    vs[k] = v;
	 end
	 dfifos[j].enq(fromInteger(kk), vs);
      endrule
   end
`endif

   function Bool fifofNotEmpty(FIFOF#(a) fifof); return fifof.notEmpty(); endfunction
   function PipeOut#(Bit#(32)) dotProdMacCount(SharedDotProdServer#(K) dotprodserver); return dotprodserver.debug.macCount; endfunction
   PipeOut#(Bit#(32)) macCountPipe <- mkReducePipes(uncurry(add), map(dotProdMacCount, fxdotprods));

   interface Vector aInputs = zipWith(toCountedPut, aMmTokensPutRegs, map(toPut, aFifos));
   interface Vector bInputs = zipWith(toCountedPut, bMmTokensPutRegs, map(toPut, bFifos));
   interface Vector fxPipes = fxPipesN;
   interface MmTileDebug debug;
      interface PipeOut macCount = macCountPipe;
   endinterface
endmodule : mkMmTile

typedef struct {
   ObjectPointer pointer;
   addrtype base;
   addrtype numRows;
   addrtype numColumns;
} MatrixDescriptor#(type addrtype) deriving (Bits);

interface DmaMatrixMultiplyDebug;
   method Bit#(32) macCount();
endinterface

// Copyright (c) 2014 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import FIFO::*;
import FIFOF::*;
import MIMO::*;
import DefaultValue::*;
import SpecialFIFOs::*;
import Vector::*;
import BRAM::*;
import DmaVector::*;
import PortalMemory::*;
import MemTypes::*;
import MemreadEngine::*;
import MemwriteEngine::*;
import MemUtils::*;
import FloatingPoint::*;
import Pipe::*;
import Arith::*;
import FloatOps::*;
import Timer::*;
import RbmTypes::*;
import Assert::*;
import Connectable::*;
import Clocks::*;
import Gearbox::*;
import XilinxCells::*;
import HostInterface::*;

interface SharedDotProdDebug#(numeric type k);
   interface PipeOut#(Bit#(32)) macCount;
endinterface

interface SharedDotProdServer#(numeric type k);
   interface Put#(MmToken)                 aInput;
   interface Put#(MmToken)                 bInput;
   interface Vector#(k, PipeOut#(MmToken)) pipes;
   interface SharedDotProdDebug#(k) debug;
endinterface

typedef struct {
`ifdef TAGGED_TOKENS
   UInt#(32) row;
   UInt#(32) col;
`endif
   Float v;
   Bool first;
   Bool last;
} MmToken deriving (Eq,Bits);

typedef 8 UB_MulLat; // upper bound on MUL latency?
typedef 8 UB_AddLat; // upper bound on ADD latency?

(* synthesize *)
module  mkSharedInterleavedDotProdServer#(UInt#(TLog#(TMul#(J,K))) label)(SharedDotProdServer#(K))
   provisos(Div#(UB_AddLat,K,gatherSz));

   let ub_MulLat = valueOf(UB_MulLat);
   let ub_AddLat = valueOf(UB_AddLat);
   let kk = valueOf(K);

   Bool verbose = False; //label == 0;

   Reg#(UInt#(20)) countReg     <- mkReg(0);

   FloatAlu mul   <- mkFloatMultiplier(defaultValue);
   FloatAlu adder <- mkFloatAdder(defaultValue);
   FIFOF#(Float) adder_buffer <- mkSizedFIFOF(valueOf(TMul#(K,gatherSz)));

`ifdef TAGGED_TOKENS
   FIFO#(Tuple2#(UInt#(32),UInt#(32))) tag_fifo <- mkSizedFIFO(ub_MulLat);
   Vector#(K,Reg#(Tuple2#(UInt#(32),UInt#(32)))) tag_regs <- replicateM(mkRegU);
`endif

   Reg#(Bit#(32)) cycles <- mkReg(0);
   rule countCycles;
      cycles <= cycles + 1;
   endrule

   FIFOF#(MmToken)                          afifo   <- mkFIFOF();
   PipeOut#(MmToken)                        aFunnel = toPipeOut(afifo);

   FIFOF#(MmToken)                          bfifo <- mkFIFOF();
   PipeOut#(MmToken)                        bFunnel = toPipeOut(bfifo);

   Reg#(Bit#(16)) firstCnt <- mkReg(0);
   Reg#(Bool)         gReg <- mkReg(False);
   FIFOF#(Float)     gFifo <- mkSizedFIFOF(kk);
   Reg#(Bit#(16))  lastCnt <- mkReg(0);
   Reg#(Bit#(16))gatherCnt <- mkReg(0);
   Reg#(Bool) gather_phase <- mkReg(False);
   //invariant: gather_phase = lastCnt == fromInteger(kk);

   FIFOF#(Tuple2#(Bool,Bool))    flFifo <- mkSizedFIFOF(ub_MulLat);
   Vector#(K,FIFOF#(MmToken))    dotfifos <- replicateM(mkFIFOF1);
   Reg#(Bit#(TAdd#(1,TLog#(K)))) rowReg <- mkReg(0);

   Reg#(Bit#(32)) lastMul <- mkReg(0);
   Reg#(Bit#(32)) lastAcc <- mkReg(0);
   Reg#(Bit#(32)) lastGather <- mkReg(0);
   Reg#(Bit#(32)) macs <- mkReg(0);

   (* fire_when_enabled *)
   rule connect_adder_buffer;
      match {.acc,.*} <- adder.response.get();
      adder_buffer.enq(acc);
      macs <= macs+1;
   endrule

   (* fire_when_enabled *)
   rule multiply;
      lastMul <= cycles;
      let a <- toGet(aFunnel).get();
      let b <- toGet(bFunnel).get();
      flFifo.enq(tuple2(a.first,a.last));
      if (a.first != b.first)
	 $display("****\n    Warning: a.first=%d != b.first=%d\n****", a.first, b.first);
      if (a.last != b.last)
	 $display("****\n    Warning: a.last=%d != b.last=%d\n****", a.last, b.last);
      if (verbose)
	 $display("%08d multiply: label=%d mulin first=%d last=%d", cycles-lastMul, label, a.first, a.last);
      mul.request.put(tuple2(a.v, b.v));
`ifdef TAGGED_TOKENS
      if(label==0) $display("%d (%d,%d)(%d,%d)", label, a.row, a.col, b.row, b.col);
      tag_fifo.enq(tuple2(a.row,b.col));
`endif
   endrule

   function Action incrementRowReg =
      (action
	  // I have to do this check (instead of relying on wrap-around) because
	  // rowReg has an extra bit to compenseate for bsc's silly Bit#(0) handling
	  if (rowReg+1 == fromInteger(kk)) rowReg <= 0;
	  else rowReg <= rowReg+1;
       endaction);

   (* fire_when_enabled *)
   rule accumulate if (!gather_phase);
      incrementRowReg;
      lastAcc <= cycles;
      match {.first, .last} <- toGet(flFifo).get();
      if (verbose) $display("%08d accumulate: label=%d mulout first=%d last=%d firstCnt=%d lastCnt=%d",
			    cycles-lastAcc, label, first, last, firstCnt, lastCnt);
      match {.resp,.*} <- mul.response.get;
      let acc = unpack(0);
      if (firstCnt == fromInteger(valueOf(TMul#(K,gatherSz))))
	 acc <- toGet(adder_buffer).get;
      else begin
	 firstCnt <= firstCnt+1;
      end
      adder.request.put(tuple2(resp,acc));
      if(last) begin
	 lastCnt <= lastCnt+1;
	 gather_phase <= (lastCnt == fromInteger(kk)-1);
      end
`ifdef TAGGED_TOKENS
      let row = rowReg;
      let t <- toGet(tag_fifo).get;
      tag_regs[row] <= t;
`endif
   endrule

   (* fire_when_enabled *)
   rule gather if (gather_phase);
      incrementRowReg;
      lastGather <= cycles;
      let row = rowReg;
      let last_row = row == fromInteger(kk-1);
      let last_pass = gatherCnt+1 == fromInteger(valueOf(gatherSz));
      if (verbose)
	 $display("%08d gather: gather=%d row=%d last_pass=%d last_row=%d, gReg=%d",
		  cycles-lastGather, gatherCnt, row, last_pass, last_row, gReg);
      let x <- toGet(adder_buffer).get;
      if (!last_pass) begin
	 if (last_row) begin
	    if (gReg) gatherCnt <= gatherCnt+1;
	    gReg <= !gReg;
	 end
	 if(gReg) begin
	    let y <- toGet(gFifo).get;
	    adder.request.put(tuple2(x,y));
	 end
	 else begin
	    gFifo.enq(x);
	 end
      end
      else begin
`ifdef TAGGED_TOKENS
      	 let row = tpl_1(tag_regs[row]);
      	 let col = tpl_2(tag_regs[row]);
	 dotfifos[row].enq(MmToken{row:row, col:col, v:x});
`else
	 dotfifos[row].enq(MmToken{v:x});
`endif
	 if (last_row) begin
	    gatherCnt <= 0;
	    lastCnt <= 0;
	    firstCnt <= 0;
	    gather_phase <= False;
	 end
      end
   endrule

   Vector#(K,PipeOut#(MmToken)) dotpipes = map(toPipeOut, dotfifos);

   interface Put aInput;
      method Action put(MmToken a);
   	 afifo.enq(a);
	 countReg <= countReg+1;
      endmethod
   endinterface
   interface Put bInput   = toPut(bfifo);
   interface Vector pipes = dotpipes;
   interface SharedDotProdDebug debug;
      interface PipeOut  macCount = toPipeOut(macs._read);
   endinterface
endmodule : mkSharedInterleavedDotProdServer

interface MmTileDebug;
   interface PipeOut#(Bit#(32)) macCount;
endinterface

interface MmTile;
   interface Vector#(RowsPerTile, Put#(MmToken)) aInputs;
   interface Vector#(RowsPerTile, Put#(MmToken)) bInputs;
   interface Vector#(RowsPerTile, PipeOut#(Vector#(N, MmToken))) fxPipes;
   interface MmTileDebug debug;
endinterface

function Put#(a) toCountedPut(Reg#(Bit#(n)) r, Put#(a) p);
   return (interface Put#(a);
      method Action put(a v);
	 r <= r+1;
	 p.put(v);
      endmethod
      endinterface);
endfunction

(* synthesize *)
module  mkMmTile#(Clock slowClock, Reset slowReset, UInt#(TLog#(T)) tile)(MmTile);

   let rowsPerTile = valueOf(RowsPerTile);
   let kk = valueOf(K);

   Vector#(RowsPerTile, Reg#(Bit#(32))) aMmTokensPutRegs <- replicateM(mkReg(0));
   Vector#(RowsPerTile, Reg#(Bit#(32))) bMmTokensPutRegs <- replicateM(mkReg(0));
   Vector#(RowsPerTile, Reg#(Bit#(32))) aMmTokensReadRegs <- replicateM(mkReg(0));
   Vector#(RowsPerTile, Reg#(Bit#(32))) bMmTokensReadRegs <- replicateM(mkReg(0));

   Vector#(RowsPerTile, FIFOF#(MmToken))   aFifos <- replicateM(mkFIFOF);
   Vector#(RowsPerTile, PipeOut#(MmToken)) aPipes = zipWith(toCountedPipeOut, aMmTokensReadRegs, map(toPipeOut, aFifos));
   Vector#(RowsPerTile,  FIFOF#(MmToken))   bFifos <- replicateM(mkFIFOF);
   Vector#(RowsPerTile,  PipeOut#(MmToken)) bPipes = zipWith(toCountedPipeOut, bMmTokensReadRegs, map(toPipeOut, bFifos));

   function Vector#(k,PipeOut#(MmToken)) getDotProdServerPipes(SharedDotProdServer#(k) s); return s.pipes; endfunction
   Vector#(RowsPerTile, SharedDotProdServer#(K)) fxdotprods <- mapM(mkSharedInterleavedDotProdServer, map(fromInteger,genVector));
   Vector#(RowsPerTile, Vector#(K, PipeOut#(MmToken))) fxpipes = map(getDotProdServerPipes, fxdotprods);
//`define USE_MIMO_DFIFOS // this version is faster
   let fastClock <- exposeCurrentClock();
   let fastReset <- exposeCurrentReset();
`ifndef USE_MIMO_DFIFOS
   Vector#(RowsPerTile, PipeOut#(Vector#(K, MmToken))) fxPipesK <- mapM(mkJoinVector(id), fxpipes);
   Vector#(RowsPerTile, PipeOut#(MmToken)) fxPipes1MmToken <- mapM(mkFunnel1, fxPipesK);
   Vector#(RowsPerTile, PipeOut#(Vector#(1, MmToken))) fxPipes1 = map(mapPipe(replicate), fxPipes1MmToken);
`else
   MIMOConfiguration mimoCfg = defaultValue;
   Vector#(RowsPerTile, MIMO#(K,1,TAdd#(K,1),MmToken)) dfifos <- replicateM(mkMIMO(mimoCfg));
   Vector#(RowsPerTile, PipeOut#(Vector#(1, MmToken))) fxPipes1 = map(toPipeOut, dfifos);
`endif
   Vector#(RowsPerTile, Gearbox#(1, N, MmToken)) gearboxes <- replicateM(mk1toNGearbox(fastClock, fastReset, slowClock, slowReset));
   Vector#(RowsPerTile, PipeIn#(Vector#(1,MmToken))) toGearboxes = map(toPipeIn, gearboxes);
   Vector#(RowsPerTile, PipeOut#(Vector#(N, MmToken))) fromGearboxes = map(toPipeOut, gearboxes);
   mapM(uncurry(mkConnection), zip(fxPipes1, toGearboxes));
   // introduce a buffer to help vivado meet timing on vc707
   Vector#(RowsPerTile, FIFOF#(Vector#(N,MmToken)))    tokenfifos <- replicateM(mkFIFOF(clocked_by slowClock, reset_by slowReset));
   Vector#(RowsPerTile, PipeIn#(Vector#(N,MmToken))) toMmTokenFifos = map(toPipeIn, tokenfifos);
   mapM(uncurry(mkConnection), zip(fromGearboxes, toMmTokenFifos), clocked_by slowClock, reset_by slowReset);
   Vector#(RowsPerTile, PipeOut#(Vector#(N, MmToken))) fxPipesN = map(toPipeOut, tokenfifos);

   FirstLastPipe#(UInt#(MMSize)) firstLastPipe          <- mkFirstLastPipe();
   Vector#(2, PipeOut#(Tuple2#(Bool,Bool))) firstLastPipes <- mkForkVector(firstLastPipe.pipe);

   for (Integer j = 0; j < rowsPerTile; j = j + 1) begin
      //mkConnection(toGet(aPipes[j]), fxdotprods[j].aInput);
      //mkConnection(toGet(bPipes[j]), fxdotprods[j].bInput);
      (* fire_when_enabled *)
      rule connectA;
	 let x <- toGet(aPipes[j]).get;
	 fxdotprods[j].aInput.put(x);
      endrule
      (* fire_when_enabled *)
      rule connectB;
	 let x <- toGet(bPipes[j]).get;
	 fxdotprods[j].bInput.put(x);
      endrule
   end

`ifdef USE_MIMO_DFIFOS
   for (Integer j = 0; j < rowsPerTile; j = j + 1) begin
      rule dotProdValue;
	 Vector#(K,MmToken) vs;
	 for (Integer k = 0; k < kk; k = k + 1) begin
	    let v <- toGet(fxpipes[j][k]).get();
	    vs[k] = v;
	 end
	 dfifos[j].enq(fromInteger(kk), vs);
      endrule
   end
`endif

   function Bool fifofNotEmpty(FIFOF#(a) fifof); return fifof.notEmpty(); endfunction
   function PipeOut#(Bit#(32)) dotProdMacCount(SharedDotProdServer#(K) dotprodserver); return dotprodserver.debug.macCount; endfunction
   PipeOut#(Bit#(32)) macCountPipe <- mkReducePipes(uncurry(add), map(dotProdMacCount, fxdotprods));

   interface Vector aInputs = zipWith(toCountedPut, aMmTokensPutRegs, map(toPut, aFifos));
   interface Vector bInputs = zipWith(toCountedPut, bMmTokensPutRegs, map(toPut, bFifos));
   interface Vector fxPipes = fxPipesN;
   interface MmTileDebug debug;
      interface PipeOut macCount = macCountPipe;
   endinterface
endmodule : mkMmTile

typedef struct {
   ObjectPointer pointer;
   addrtype base;
   addrtype numRows;
   addrtype numColumns;
} MatrixDescriptor#(type addrtype) deriving (Bits);

interface DmaMatrixMultiplyDebug;
   method Bit#(32) macCount();
endinterface


Decreased the memory space allocation for BAR2 to 16 * 64kb

portal memory is only 64kb/portal (from mkDirectoryDbg in bsv/Directory.bsv)


simplify pixel unpack

simplify pixel unpack

// Copyright (c) 2012 Nokia, Inc.
// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import FIFO::*;
import FIFOF::*;
import BRAMFIFO::*;
import Vector::*;
import Clocks::*;
import GetPut::*;
import ClientServer::*;
import PCIE::*;
import GetPutWithClocks::*;
import Connectable::*;
import PortalMemory::*;
import MemTypes::*;
import DmaUtils::*;
import ClientServer::*;
import AxiMasterSlave::*;
import MemreadEngine::*;
import HDMI::*;
import XADC::*;
import YUV::*;
import Gearbox::*;
import MIMO::*;
import BlueScope::*;

interface HdmiDisplayRequest;
   method Action startFrameBuffer(Int#(32) base, UInt#(32) rows, UInt#(32) cols, UInt#(32) pixels);
   method Action stopFrameBuffer();
   method Action getTransferStats();
   method Action setTraceTransfers(Bit#(1) trace);
endinterface
interface HdmiDisplayIndication;
   method Action transferStarted(Bit#(32) count);
   method Action transferFinished(Bit#(32) count);
   method Action transferStats(Bit#(32) count, Bit#(32) transferCycles, Bit#(64) sumOfCycles);
endinterface

interface HdmiDisplay;
`ifdef HDMI_BLUESCOPE
   interface BlueScopeRequest  bluescopeRequest;
   interface ObjectWriteClient#(64) bluescopeWriteClient;
`endif
    interface HdmiDisplayRequest displayRequest;
    interface HdmiInternalRequest internalRequest;
    interface ObjectReadClient#(64) dmaClient;
    interface HDMI#(Bit#(HdmiBits)) hdmi;
    interface XADC xadc;
endinterface

module mkHdmiDisplay#(Clock hdmi_clock,
		      HdmiDisplayIndication hdmiDisplayIndication,
		      HdmiInternalIndication hdmiInternalIndication
`ifdef HDMI_BLUESCOPE
		      , BlueScopeIndication bluescopeIndication
`endif
)(HdmiDisplay);
    Clock defaultClock <- exposeCurrentClock;
    Reset defaultReset <- exposeCurrentReset;
    Reset hdmi_reset <- mkAsyncReset(2, defaultReset, hdmi_clock);

   Reg#(UInt#(16)) rowsReg <- mkReg(1080);
   Reg#(UInt#(16)) colsReg <- mkReg(1920);
   Reg#(UInt#(24)) pixelCountReg <- mkReg(1080*1920);

    Reg#(Bool) sendVsyncIndication <- mkReg(False);
    SyncPulseIfc vsyncPulse <- mkSyncHandshake(hdmi_clock, hdmi_reset, defaultClock);
    Reg#(Bit#(1)) bozobit <- mkReg(0, clocked_by hdmi_clock, reset_by hdmi_reset);

    Reg#(Maybe#(Bit#(32))) referenceReg <- mkReg(tagged Invalid);
    FIFO#(Bit#(64)) mrFifo <- mkSizedBRAMFIFO(1024);
    MemreadEngine#(64,16) memreadEngine <- mkMemreadEngine;

    HdmiGenerator#(Rgb888) hdmiGen <- mkHdmiGenerator(defaultClock, defaultReset,
							vsyncPulse, hdmiInternalIndication, clocked_by hdmi_clock, reset_by hdmi_reset);
`ifndef ZC706
   Rgb888ToYyuv converter <- mkRgb888ToYyuv(clocked_by hdmi_clock, reset_by hdmi_reset);
   mkConnection(hdmiGen.rgb888, converter.rgb888);
   HDMI#(Bit#(HdmiBits)) hdmisignals <- mkHDMI(converter.yyuv, clocked_by hdmi_clock, reset_by hdmi_reset);
`else
   HDMI#(Bit#(HdmiBits)) hdmisignals <- mkHDMI(hdmiGen.rgb888, clocked_by hdmi_clock, reset_by hdmi_reset);
`endif
`ifdef HDMI_BLUESCOPE
   let bluescope <- mkSyncBlueScope(65536, bluescopeIndication, hdmi_clock, hdmi_reset, defaultClock, defaultReset);
   MIMO#(1, 16, 64, Bit#(4)) mimo <- mkMIMO(MIMOConfiguration { unguarded: False, bram_based: False }, clocked_by hdmi_clock, reset_by hdmi_reset);
   Reg#(Bool) triggered <- mkReg(False, clocked_by hdmi_clock, reset_by hdmi_reset);
   rule toGearbox if ((hdmisignals.hdmi_vsync == 1) || triggered);
      Bit#(4) v = 0;
      v[0] = hdmisignals.hdmi_vsync;
      v[1] = hdmisignals.hdmi_de;
      v[2] = hdmisignals.hdmi_hsync;
      v[3] = hdmisignals.hdmi_vsync;
      triggered <= True;
      if (mimo.enqReadyN(1))
	 mimo.enq(1, cons(v,nil));
      else
	 $display("mimo.stalled mimo.count=%d", mimo.count);
   endrule
   rule gearboxToBlueScope if (mimo.deqReadyN(16));
      Bit#(64) v = pack(mimo.first());
      mimo.deq(16);
      bluescope.dataIn(v, v);
   endrule
`endif

   SyncFIFOIfc#(Bit#(64)) synchronizer <- mkSyncFIFO(32, defaultClock, defaultReset, hdmi_clock);
   rule fromMemread;
      let v <- toGet(memreadEngine.dataPipes[0]).get;
      mrFifo.enq(v);
   endrule
   rule toSynch;
      let v <- toGet(mrFifo).get();
      synchronizer.enq(v);
   endrule
   Reg#(Bool) evenOdd <- mkReg(True, clocked_by hdmi_clock, reset_by hdmi_reset);
   Reg#(Bit#(32)) doublePixelReg <- mkReg(0, clocked_by hdmi_clock, reset_by hdmi_reset);
   rule doPut;
      let pixel = doublePixelReg;
      if (evenOdd) begin
	 Vector#(2,Bit#(32)) doublePixel = unpack(synchronizer.first);
	 synchronizer.deq;
         doublePixelReg <= doublePixel[1];
	 pixel = doublePixel[0];
      end
      evenOdd <= !evenOdd;
      hdmiGen.request.put(pixel);
   endrule

   Reg#(Bit#(32)) transferCount <- mkReg(0);
   Reg#(Bit#(32)) transferCyclesSnapshot <- mkReg(0);
   Reg#(Bit#(32)) transferCycles <- mkReg(0);
   Reg#(Bit#(48)) transferSumOfCycles<- mkReg(0);
   Reg#(Bit#(32)) vsyncCount <- mkReg(0);

   FIFOF#(Bool) vsyncFifo <- mkFIFOF();
   rule vsyncrule if (vsyncPulse.pulse());
      vsyncCount <= vsyncCount + 1;
      if (vsyncFifo.notFull())
	 vsyncFifo.enq(True);
   endrule

   Reg#(Bool) traceTransfers <- mkReg(False);
   rule notransfer if (referenceReg matches tagged Invalid);
      vsyncFifo.deq();
   endrule
   rule startTransfer if (referenceReg matches tagged Valid .reference);
      memreadEngine.readServers[0].request.put(MemengineCmd{pointer:reference, base:0, len:pack(extend(pixelCountReg))*4, burstLen:64});
      if (traceTransfers)
	 hdmiDisplayIndication.transferStarted(transferCount);
      transferCyclesSnapshot <= transferCycles;
      vsyncFifo.deq();
   endrule
   rule countCycles;
      transferCycles <= transferCycles + 1;
   endrule
   rule finishTransferRule;
      let b <- memreadEngine.readServers[0].response.get;
      transferCount <= transferCount + 1;
      let tc = transferCycles - transferCyclesSnapshot;
      transferSumOfCycles <= transferSumOfCycles + extend(tc);
      if (traceTransfers)
	 hdmiDisplayIndication.transferFinished(transferCount);
   endrule

    rule bozobit_rule;
        bozobit <= ~bozobit;
    endrule

    interface HdmiDisplayRequest displayRequest;
	method Action startFrameBuffer(Int#(32) base, UInt#(32) rows, UInt#(32) cols, UInt#(32) pixels);
	   rowsReg <= truncate(rows);
	   colsReg <= truncate(cols);
	   pixelCountReg <= truncate(pixels);
	   $display("startFrameBuffer %h", base);
           referenceReg <= tagged Valid truncate(pack(base));
	   hdmiGen.control.setTestPattern(0);
	endmethod
       method Action stopFrameBuffer();
	  referenceReg <= tagged Invalid;
	  hdmiGen.control.setTestPattern(1);
       endmethod
       method Action getTransferStats();
          hdmiDisplayIndication.transferStats(transferCount, transferCycles-transferCyclesSnapshot, extend(transferSumOfCycles));
       endmethod
       method Action setTraceTransfers(Bit#(1) trace);
	  traceTransfers <= unpack(trace);
       endmethod
    endinterface: displayRequest

    interface ObjectReadClient dmaClient = memreadEngine.dmaClient;
    interface HDMI hdmi = hdmisignals;
    interface HdmiInternalRequest internalRequest = hdmiGen.control;
`ifdef HDMI_BLUESCOPE
    interface BlueScopeRequest bluescopeRequest = bluescope.requestIfc;
    interface ObjectWriteClient bluescopeWriteClient = bluescope.writeClient;
`endif
    interface XADC xadc;
        method Bit#(4) gpio;
            return { bozobit, hdmisignals.hdmi_vsync,
                hdmisignals.hdmi_data[8], hdmisignals.hdmi_data[0]};
                //hdmiGen.hdmi.hsync, hdmi_de};
        endmethod
    endinterface: xadc
endmodule

// Copyright (c) 2012 Nokia, Inc.
// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import FIFO::*;
import FIFOF::*;
import BRAMFIFO::*;
import Vector::*;
import Clocks::*;
import GetPut::*;
import ClientServer::*;
import PCIE::*;
import GetPutWithClocks::*;
import Connectable::*;
import PortalMemory::*;
import MemTypes::*;
import DmaUtils::*;
import ClientServer::*;
import AxiMasterSlave::*;
import MemreadEngine::*;
import HDMI::*;
import XADC::*;
import YUV::*;
import Gearbox::*;
import MIMO::*;
import BlueScope::*;

interface HdmiDisplayRequest;
   method Action startFrameBuffer(Int#(32) base, UInt#(32) rows, UInt#(32) cols, UInt#(32) pixels);
   method Action stopFrameBuffer();
   method Action getTransferStats();
   method Action setTraceTransfers(Bit#(1) trace);
endinterface
interface HdmiDisplayIndication;
   method Action transferStarted(Bit#(32) count);
   method Action transferFinished(Bit#(32) count);
   method Action transferStats(Bit#(32) count, Bit#(32) transferCycles, Bit#(64) sumOfCycles);
endinterface

interface HdmiDisplay;
`ifdef HDMI_BLUESCOPE
   interface BlueScopeRequest  bluescopeRequest;
   interface ObjectWriteClient#(64) bluescopeWriteClient;
`endif
    interface HdmiDisplayRequest displayRequest;
    interface HdmiInternalRequest internalRequest;
    interface ObjectReadClient#(64) dmaClient;
    interface HDMI#(Bit#(HdmiBits)) hdmi;
    interface XADC xadc;
endinterface

module mkHdmiDisplay#(Clock hdmi_clock,
		      HdmiDisplayIndication hdmiDisplayIndication,
		      HdmiInternalIndication hdmiInternalIndication
`ifdef HDMI_BLUESCOPE
		      , BlueScopeIndication bluescopeIndication
`endif
)(HdmiDisplay);
    Clock defaultClock <- exposeCurrentClock;
    Reset defaultReset <- exposeCurrentReset;
    Reset hdmi_reset <- mkAsyncReset(2, defaultReset, hdmi_clock);

   Reg#(UInt#(16)) rowsReg <- mkReg(1080);
   Reg#(UInt#(16)) colsReg <- mkReg(1920);
   Reg#(UInt#(24)) pixelCountReg <- mkReg(1080*1920);

    Reg#(Bool) sendVsyncIndication <- mkReg(False);
    SyncPulseIfc vsyncPulse <- mkSyncHandshake(hdmi_clock, hdmi_reset, defaultClock);
    Reg#(Bit#(1)) bozobit <- mkReg(0, clocked_by hdmi_clock, reset_by hdmi_reset);

    Reg#(Maybe#(Bit#(32))) referenceReg <- mkReg(tagged Invalid);
    FIFO#(Bit#(64)) mrFifo <- mkSizedBRAMFIFO(1024);
    MemreadEngine#(64,16) memreadEngine <- mkMemreadEngine;

    HdmiGenerator#(Rgb888) hdmiGen <- mkHdmiGenerator(defaultClock, defaultReset,
							vsyncPulse, hdmiInternalIndication, clocked_by hdmi_clock, reset_by hdmi_reset);
`ifndef ZC706
   Rgb888ToYyuv converter <- mkRgb888ToYyuv(clocked_by hdmi_clock, reset_by hdmi_reset);
   mkConnection(hdmiGen.rgb888, converter.rgb888);
   HDMI#(Bit#(HdmiBits)) hdmisignals <- mkHDMI(converter.yyuv, clocked_by hdmi_clock, reset_by hdmi_reset);
`else
   HDMI#(Bit#(HdmiBits)) hdmisignals <- mkHDMI(hdmiGen.rgb888, clocked_by hdmi_clock, reset_by hdmi_reset);
`endif
`ifdef HDMI_BLUESCOPE
   let bluescope <- mkSyncBlueScope(65536, bluescopeIndication, hdmi_clock, hdmi_reset, defaultClock, defaultReset);
   MIMO#(1, 16, 64, Bit#(4)) mimo <- mkMIMO(MIMOConfiguration { unguarded: False, bram_based: False }, clocked_by hdmi_clock, reset_by hdmi_reset);
   Reg#(Bool) triggered <- mkReg(False, clocked_by hdmi_clock, reset_by hdmi_reset);
   rule toGearbox if ((hdmisignals.hdmi_vsync == 1) || triggered);
      Bit#(4) v = 0;
      v[0] = hdmisignals.hdmi_vsync;
      v[1] = hdmisignals.hdmi_de;
      v[2] = hdmisignals.hdmi_hsync;
      v[3] = hdmisignals.hdmi_vsync;
      triggered <= True;
      if (mimo.enqReadyN(1))
	 mimo.enq(1, cons(v,nil));
      else
	 $display("mimo.stalled mimo.count=%d", mimo.count);
   endrule
   rule gearboxToBlueScope if (mimo.deqReadyN(16));
      Bit#(64) v = pack(mimo.first());
      mimo.deq(16);
      bluescope.dataIn(v, v);
   endrule
`endif

   SyncFIFOIfc#(Bit#(64)) synchronizer <- mkSyncFIFO(32, defaultClock, defaultReset, hdmi_clock);
   rule fromMemread;
      let v <- toGet(memreadEngine.dataPipes[0]).get;
      mrFifo.enq(v);
   endrule
   rule toSynch;
      let v <- toGet(mrFifo).get();
      synchronizer.enq(v);
   endrule
   Reg#(Bit#(1)) evenOdd <- mkReg(0, clocked_by hdmi_clock, reset_by hdmi_reset);
   Reg#(Vector#(2,Bit#(32))) doublePixelReg <- mkReg(unpack(0), clocked_by hdmi_clock, reset_by hdmi_reset);
   rule doPut;
      Vector#(2,Bit#(32)) doublePixel = doublePixelReg;
      let pixel = doublePixel[evenOdd];
      if (evenOdd == 0) begin
	 doublePixel = unpack(synchronizer.first);
	 synchronizer.deq;
	 pixel = doublePixel[0];
      end
      doublePixelReg <= doublePixel;
      evenOdd <= evenOdd + 1;
      hdmiGen.request.put(pixel);
   endrule

   Reg#(Bit#(32)) transferCount <- mkReg(0);
   Reg#(Bit#(32)) transferCyclesSnapshot <- mkReg(0);
   Reg#(Bit#(32)) transferCycles <- mkReg(0);
   Reg#(Bit#(48)) transferSumOfCycles<- mkReg(0);
   Reg#(Bit#(32)) vsyncCount <- mkReg(0);

   FIFOF#(Bool) vsyncFifo <- mkFIFOF();
   rule vsyncrule if (vsyncPulse.pulse());
      vsyncCount <= vsyncCount + 1;
      if (vsyncFifo.notFull())
	 vsyncFifo.enq(True);
   endrule

   Reg#(Bool) traceTransfers <- mkReg(False);
   rule notransfer if (referenceReg matches tagged Invalid);
      vsyncFifo.deq();
   endrule
   rule startTransfer if (referenceReg matches tagged Valid .reference);
      memreadEngine.readServers[0].request.put(MemengineCmd{pointer:reference, base:0, len:pack(extend(pixelCountReg))*4, burstLen:64});
      if (traceTransfers)
	 hdmiDisplayIndication.transferStarted(transferCount);
      transferCyclesSnapshot <= transferCycles;
      vsyncFifo.deq();
   endrule
   rule countCycles;
      transferCycles <= transferCycles + 1;
   endrule
   rule finishTransferRule;
      let b <- memreadEngine.readServers[0].response.get;
      transferCount <= transferCount + 1;
      let tc = transferCycles - transferCyclesSnapshot;
      transferSumOfCycles <= transferSumOfCycles + extend(tc);
      if (traceTransfers)
	 hdmiDisplayIndication.transferFinished(transferCount);
   endrule

    rule bozobit_rule;
        bozobit <= ~bozobit;
    endrule

    interface HdmiDisplayRequest displayRequest;
	method Action startFrameBuffer(Int#(32) base, UInt#(32) rows, UInt#(32) cols, UInt#(32) pixels);
	   rowsReg <= truncate(rows);
	   colsReg <= truncate(cols);
	   pixelCountReg <= truncate(pixels);
	   $display("startFrameBuffer %h", base);
           referenceReg <= tagged Valid truncate(pack(base));
	   hdmiGen.control.setTestPattern(0);
	endmethod
       method Action stopFrameBuffer();
	  referenceReg <= tagged Invalid;
	  hdmiGen.control.setTestPattern(1);
       endmethod
       method Action getTransferStats();
          hdmiDisplayIndication.transferStats(transferCount, transferCycles-transferCyclesSnapshot, extend(transferSumOfCycles));
       endmethod
       method Action setTraceTransfers(Bit#(1) trace);
	  traceTransfers <= unpack(trace);
       endmethod
    endinterface: displayRequest

    interface ObjectReadClient dmaClient = memreadEngine.dmaClient;
    interface HDMI hdmi = hdmisignals;
    interface HdmiInternalRequest internalRequest = hdmiGen.control;
`ifdef HDMI_BLUESCOPE
    interface BlueScopeRequest bluescopeRequest = bluescope.requestIfc;
    interface ObjectWriteClient bluescopeWriteClient = bluescope.writeClient;
`endif
    interface XADC xadc;
        method Bit#(4) gpio;
            return { bozobit, hdmisignals.hdmi_vsync,
                hdmisignals.hdmi_data[8], hdmisignals.hdmi_data[0]};
                //hdmiGen.hdmi.hsync, hdmi_de};
        endmethod
    endinterface: xadc
endmodule


updated dma i/f configReadChan(...) -> configChan(0,...)

// Copyright (c) 2012 Nokia, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import FIFOF::*;
import Clocks::*;
import GetPut::*;
import PCIE::*;
import GetPutWithClocks::*;
import Connectable::*;
import PortalMemory::*;
import PortalSMemory::*;
import AxiSDMA::*;
import BsimSDMA::*;
import AxiMasterSlave::*;
import AxiClientServer::*;
import HDMI::*;
import XADC::*;
import YUV::*;

interface HdmiControlRequest;
    method Action startFrameBuffer0(Int#(32) base);
endinterface
interface HdmiControlIndication;
endinterface

interface HdmiDisplayRequest;
    interface HdmiControlRequest coreRequest;
    interface HdmiInternalRequest coRequest;
    interface DMARequest dmaRequest;
    interface Axi3Client#(40,64,8,12) m_axi;
    interface HDMI hdmi;
    interface XADC xadc;
endinterface
interface HdmiDisplayIndication;
    interface HdmiControlIndication coreIndication;
    interface HdmiInternalIndication coIndication;
    interface DMAIndication dmaIndication;
endinterface

module mkHdmiDisplayRequest#(Clock processing_system7_1_fclk_clk1, HdmiDisplayIndication indication)(HdmiDisplayRequest);
    Clock defaultClock <- exposeCurrentClock;
    Reset defaultReset <- exposeCurrentReset;
    Clock hdmi_clock = processing_system7_1_fclk_clk1;
    Reset hdmi_reset <- mkAsyncReset(2, defaultReset, hdmi_clock);
    Reg#(Bool) sendVsyncIndication <- mkReg(False);
    SyncPulseIfc vsyncPulse <- mkSyncHandshake(hdmi_clock, hdmi_reset, defaultClock);
    Reg#(Bit#(1)) bozobit <- mkReg(0, clocked_by hdmi_clock, reset_by hdmi_reset);
    Reg#(Bit#(8)) segmentIndexReg <- mkReg(0);
    Reg#(Bit#(24)) segmentOffsetReg <- mkReg(0);
`ifdef BSIM
    BsimDMA    dma <- mkBsimDMA(indication.dmaIndication);
`else
    AxiDMA     dma <- mkAxiDMA(indication.dmaIndication);
`endif
    ReadChan dma_stream_read_chan = dma.read.readChannels[0];
    Reg#(Int#(32)) referenceReg <- mkReg(-1);
    Reg#(Bit#(32)) streamRdCnt <- mkReg(0);

    HdmiGenerator hdmiGen <- mkHdmiGenerator(defaultClock, defaultReset,
        vsyncPulse, indication.coIndication, clocked_by hdmi_clock, reset_by hdmi_reset);

    rule readReq if(referenceReg >= 0);
        streamRdCnt <= streamRdCnt - 16;
        dma_stream_read_chan.readReq.put(?);
    endrule
    mkConnectionWithClocks(dma_stream_read_chan.readData, hdmiGen.request, defaultClock, defaultReset, hdmi_clock, hdmi_reset);

    rule vsyncrule if (vsyncPulse.pulse() && referenceReg >= 0);
        dma.request.configChan(0, 0, pack(referenceReg), 16);
    endrule

    rule bozobit_rule;
        bozobit <= ~bozobit;
    endrule

    interface HdmiControlRequest coreRequest;
	method Action startFrameBuffer0(Int#(32) base);
	    $display("startFrameBuffer %h", base);
            referenceReg <= base;
	    hdmiGen.control.setTestPattern(0);
	endmethod
    endinterface: coreRequest

`ifndef BSIM
    interface Axi3Client m_axi = dma.m_axi;
`endif
    interface DMARequest dmaRequest = dma.request;
    interface HDMI hdmi = hdmiGen.hdmi;
    interface HdmiInternalRequest coRequest = hdmiGen.control;
    interface XADC xadc;
        method Bit#(4) gpio;
            return { bozobit, hdmiGen.hdmi.vsync,
                hdmiGen.hdmi.data[8], hdmiGen.hdmi.data[0]};
                //hdmiGen.hdmi.hsync, hdmi_de};
        endmethod
    endinterface: xadc
endmodule

// Copyright (c) 2012 Nokia, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import FIFOF::*;
import Clocks::*;
import GetPut::*;
import PCIE::*;
import GetPutWithClocks::*;
import Connectable::*;
import PortalMemory::*;
import PortalSMemory::*;
import AxiSDMA::*;
import BsimSDMA::*;
import AxiMasterSlave::*;
import AxiClientServer::*;
import HDMI::*;
import XADC::*;
import YUV::*;

interface HdmiControlRequest;
    method Action startFrameBuffer0(Int#(32) base);
endinterface
interface HdmiControlIndication;
endinterface

interface HdmiDisplayRequest;
    interface HdmiControlRequest coreRequest;
    interface HdmiInternalRequest coRequest;
    interface DMARequest dmaRequest;
    interface Axi3Client#(40,64,8,12) m_axi;
    interface HDMI hdmi;
    interface XADC xadc;
endinterface
interface HdmiDisplayIndication;
    interface HdmiControlIndication coreIndication;
    interface HdmiInternalIndication coIndication;
    interface DMAIndication dmaIndication;
endinterface

module mkHdmiDisplayRequest#(Clock processing_system7_1_fclk_clk1, HdmiDisplayIndication indication)(HdmiDisplayRequest);
    Clock defaultClock <- exposeCurrentClock;
    Reset defaultReset <- exposeCurrentReset;
    Clock hdmi_clock = processing_system7_1_fclk_clk1;
    Reset hdmi_reset <- mkAsyncReset(2, defaultReset, hdmi_clock);
    Reg#(Bool) sendVsyncIndication <- mkReg(False);
    SyncPulseIfc vsyncPulse <- mkSyncHandshake(hdmi_clock, hdmi_reset, defaultClock);
    Reg#(Bit#(1)) bozobit <- mkReg(0, clocked_by hdmi_clock, reset_by hdmi_reset);
    Reg#(Bit#(8)) segmentIndexReg <- mkReg(0);
    Reg#(Bit#(24)) segmentOffsetReg <- mkReg(0);
`ifdef BSIM
    BsimDMA    dma <- mkBsimDMA(indication.dmaIndication);
`else
    AxiDMA     dma <- mkAxiDMA(indication.dmaIndication);
`endif
    ReadChan dma_stream_read_chan = dma.read.readChannels[0];
    Reg#(Int#(32)) referenceReg <- mkReg(-1);
    Reg#(Bit#(32)) streamRdCnt <- mkReg(0);

    HdmiGenerator hdmiGen <- mkHdmiGenerator(defaultClock, defaultReset,
        vsyncPulse, indication.coIndication, clocked_by hdmi_clock, reset_by hdmi_reset);

    rule readReq if(referenceReg >= 0);
        streamRdCnt <= streamRdCnt - 16;
        dma_stream_read_chan.readReq.put(?);
    endrule
    mkConnectionWithClocks(dma_stream_read_chan.readData, hdmiGen.request, defaultClock, defaultReset, hdmi_clock, hdmi_reset);

    rule vsyncrule if (vsyncPulse.pulse() && referenceReg >= 0);
        dma.request.configReadChan(0, pack(referenceReg), 16);
    endrule

    rule bozobit_rule;
        bozobit <= ~bozobit;
    endrule

    interface HdmiControlRequest coreRequest;
	method Action startFrameBuffer0(Int#(32) base);
	    $display("startFrameBuffer %h", base);
            referenceReg <= base;
	    hdmiGen.control.setTestPattern(0);
	endmethod
    endinterface: coreRequest

`ifndef BSIM
    interface Axi3Client m_axi = dma.m_axi;
`endif
    interface DMARequest dmaRequest = dma.request;
    interface HDMI hdmi = hdmiGen.hdmi;
    interface HdmiInternalRequest coRequest = hdmiGen.control;
    interface XADC xadc;
        method Bit#(4) gpio;
            return { bozobit, hdmiGen.hdmi.vsync,
                hdmiGen.hdmi.data[8], hdmiGen.hdmi.data[0]};
                //hdmiGen.hdmi.hsync, hdmi_de};
        endmethod
    endinterface: xadc
endmodule