added support for BSV_POSITIVE_RESET to ConnectalBramFifo

// Copyright (c) 2015 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import Clocks::*;
import Vector::*;
import FIFO::*;
import FIFOF::*;
import BRAMFIFO::*;
import CBus::*; // extendNP and truncateNP

import Arith::*;

(* always_ready, always_enabled *)
interface X7FifoSyncMacro#(numeric type data_width);
   method Bit#(1) empty();
   method Bit#(1) full();
   method Action din(Bit#(data_width) v);
   method Action wren(Bit#(1) wren);
   method Bit#(data_width) dout();
   method Action rden(Bit#(1) rden);
   method Bit#(9) wrcount();
   method Bit#(9) rdcount();
endinterface

import "BVI" FIFO_DUALCLOCK_MACRO =
module  vmkBramFifo#(String fifo_size, Clock wrClock, Reset wrReset, Clock rdClock, Reset rdReset)(X7FifoSyncMacro#(data_width));
`ifndef BSV_POSITIVE_RESET
   let fifoReset <- mkResetInverter(wrReset, clocked_by wrClock);
`else
   let fifoReset = wrReset;
`endif
   parameter DEVICE = "7SERIES";
   parameter DATA_WIDTH = valueOf(data_width);
   parameter FIFO_SIZE = fifo_size;
   parameter FIRST_WORD_FALL_THROUGH = 1;
   default_clock wrClock(WRCLK) = wrClock;
   no_reset;
   input_reset wrReset(RST) = fifoReset;
   input_clock rdClock(RDCLK) = rdClock;
   method EMPTY empty() clocked_by (rdClock) reset_by (wrReset);
   method FULL full() clocked_by (wrClock) reset_by (wrReset);
   method din(DI) enable ((*inhigh*)EN_di) clocked_by (wrClock) reset_by (wrReset);
   method wren(WREN) enable ((*inhigh*)EN_wren) clocked_by (wrClock) reset_by (wrReset);
   method DO dout() clocked_by (rdClock) reset_by (wrReset);
   method rden(RDEN) enable ((*inhigh*)EN_rden) clocked_by (rdClock) reset_by (wrReset);
   // wrcount and rdcount ports are needed for xsim
   method WRCOUNT wrcount() clocked_by (wrClock) reset_by (wrReset);
   method RDCOUNT rdcount() clocked_by (rdClock) reset_by (wrReset);
   schedule (empty, full, dout, din, wren, rden, wrcount, rdcount) CF (empty, full, dout, din, wren, rden, wrcount, rdcount);
endmodule

module mkDualClockBramFIFOF#(Clock srcClock, Reset srcReset, Clock dstClock, Reset dstReset)(FIFOF#(t))
   provisos (Bits#(t,sizet),
	     Add#(1,a__,sizet));
   String fifo_size = "18Kb";
   Vector#(TDiv#(sizet,36),X7FifoSyncMacro#(36)) fifos <- replicateM(vmkBramFifo(fifo_size, srcClock, srcReset, dstClock, dstReset));
   Wire#(Bit#(1)) rdenWire <- mkDWire(0, clocked_by dstClock, reset_by dstReset);
   Wire#(Bit#(1)) wrenWire <- mkDWire(0, clocked_by srcClock, reset_by srcReset);
   Vector#(TDiv#(sizet,36),Wire#(Bit#(36))) dinWires <- replicateM(mkDWire(0, clocked_by srcClock, reset_by srcReset));

   for (Integer i = 0; i < valueOf(TDiv#(sizet,36)); i = i+1) begin
      Reg#(Bit#(9)) rdcount <- mkReg(0, clocked_by dstClock, reset_by dstReset);
      Reg#(Bit#(9)) wrcount <- mkReg(0, clocked_by srcClock, reset_by srcReset);
      rule rdenRule;
	 fifos[i].rden(rdenWire);
      endrule
      rule wrenRule;
	 fifos[i].wren(wrenWire);
      endrule
      rule inputs;
	 fifos[i].din(dinWires[i]);
      endrule
      rule countrds;
	 rdcount <= fifos[i].rdcount();
      endrule
      rule countwrs;
	 wrcount <= fifos[i].wrcount();
      endrule
   end

   function Bool fifoNotEmpty(Integer i); return fifos[i].empty == 0; endfunction
   function Bool fifoNotFull(Integer i); return fifos[i].full == 0; endfunction
   Vector#(TDiv#(sizet,36), Bool) vNotEmpty = genWith(fifoNotEmpty);
   Vector#(TDiv#(sizet,36), Bool) vNotFull = genWith(fifoNotFull);

   method t first() if (fifos[0].empty == 0);
      function Bit#(36) fifoFirst(Integer i); return fifos[i].dout(); endfunction
      Vector#(TDiv#(sizet,36), Bit#(36)) v = genWith(fifoFirst);
      return unpack(truncateNP(pack(v)));
   endmethod
   method Action deq() if (fifos[0].empty == 0);
      rdenWire <= 1;
   endmethod
   method notEmpty = (fifos[0].empty == 0);
   method Action enq(t v) if (fifos[0].full == 0);
      Vector#(TDiv#(sizet,36), Bit#(36)) vs = unpack(extendNP(pack(v)));
      Vector#(TDiv#(sizet,36), Integer) indices = genVector();
      function Action fifoEnq(Integer i); action dinWires[i] <= vs[i]; endaction endfunction
      mapM_(fifoEnq, indices);
      wrenWire <= 1;
   endmethod
   method notFull = (fifos[0].full == 0);
endmodule

module mkSizedBRAMFIFOF#(Integer m)(FIFOF#(t))
   provisos (Bits#(t,sizet),
	     Add#(1,a__,sizet));
   let clock <- exposeCurrentClock();
   let reset <- exposeCurrentReset();
   let fifo <- mkDualClockBramFIFOF(clock, reset, clock, reset);
   return fifo;
endmodule
module mkSizedBRAMFIFO#(Integer m)(FIFO#(t))
   provisos (Bits#(t,sizet),
	     Add#(1,a__,sizet));
   let clock <- exposeCurrentClock();
   let reset <- exposeCurrentReset();
   let fifo <- mkDualClockBramFIFOF(clock, reset, clock, reset);
   method first = fifo.first;
   method deq = fifo.deq;
   method enq = fifo.enq;
endmodule

// Copyright (c) 2015 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import Clocks::*;
import Vector::*;
import FIFO::*;
import FIFOF::*;
import BRAMFIFO::*;
import CBus::*; // extendNP and truncateNP

import Arith::*;

(* always_ready, always_enabled *)
interface X7FifoSyncMacro#(numeric type data_width);
   method Bit#(1) empty();
   method Bit#(1) full();
   method Action din(Bit#(data_width) v);
   method Action wren(Bit#(1) wren);
   method Bit#(data_width) dout();
   method Action rden(Bit#(1) rden);
   method Bit#(9) wrcount();
   method Bit#(9) rdcount();
endinterface

import "BVI" FIFO_DUALCLOCK_MACRO =
module  vmkBramFifo#(String fifo_size, Clock wrClock, Reset wrReset, Clock rdClock, Reset rdReset)(X7FifoSyncMacro#(data_width));
   let invertReset <- mkResetInverter(wrReset, clocked_by wrClock);
   //let invertReset <- mkAsyncReset(1, invertReset0, wrClock, clocked_by wrClock);
   parameter DEVICE = "7SERIES";
   parameter DATA_WIDTH = valueOf(data_width);
   parameter FIFO_SIZE = fifo_size;
   parameter FIRST_WORD_FALL_THROUGH = 1;
   default_clock wrClock(WRCLK) = wrClock;
   no_reset;
   input_reset wrReset(RST) = invertReset;
   input_clock rdClock(RDCLK) = rdClock;
   method EMPTY empty() clocked_by (rdClock) reset_by (wrReset);
   method FULL full() clocked_by (wrClock) reset_by (wrReset);
   method din(DI) enable ((*inhigh*)EN_di) clocked_by (wrClock) reset_by (wrReset);
   method wren(WREN) enable ((*inhigh*)EN_wren) clocked_by (wrClock) reset_by (wrReset);
   method DO dout() clocked_by (rdClock) reset_by (wrReset);
   method rden(RDEN) enable ((*inhigh*)EN_rden) clocked_by (rdClock) reset_by (wrReset);
   // wrcount and rdcount ports are needed for xsim
   method WRCOUNT wrcount() clocked_by (wrClock) reset_by (wrReset);
   method RDCOUNT rdcount() clocked_by (rdClock) reset_by (wrReset);
   schedule (empty, full, dout, din, wren, rden, wrcount, rdcount) CF (empty, full, dout, din, wren, rden, wrcount, rdcount);
endmodule

module mkDualClockBramFIFOF#(Clock srcClock, Reset srcReset, Clock dstClock, Reset dstReset)(FIFOF#(t))
   provisos (Bits#(t,sizet),
	     Add#(1,a__,sizet));
   String fifo_size = "18Kb";
   Vector#(TDiv#(sizet,36),X7FifoSyncMacro#(36)) fifos <- replicateM(vmkBramFifo(fifo_size, srcClock, srcReset, dstClock, dstReset));
   Wire#(Bit#(1)) rdenWire <- mkDWire(0, clocked_by dstClock, reset_by dstReset);
   Wire#(Bit#(1)) wrenWire <- mkDWire(0, clocked_by srcClock, reset_by srcReset);
   Vector#(TDiv#(sizet,36),Wire#(Bit#(36))) dinWires <- replicateM(mkDWire(0, clocked_by srcClock, reset_by srcReset));

   for (Integer i = 0; i < valueOf(TDiv#(sizet,36)); i = i+1) begin
      Reg#(Bit#(9)) rdcount <- mkReg(0, clocked_by dstClock, reset_by dstReset);
      Reg#(Bit#(9)) wrcount <- mkReg(0, clocked_by srcClock, reset_by srcReset);
      rule rdenRule;
	 fifos[i].rden(rdenWire);
      endrule
      rule wrenRule;
	 fifos[i].wren(wrenWire);
      endrule
      rule inputs;
	 fifos[i].din(dinWires[i]);
      endrule
      rule countrds;
	 rdcount <= fifos[i].rdcount();
      endrule
      rule countwrs;
	 wrcount <= fifos[i].wrcount();
      endrule
   end

   function Bool fifoNotEmpty(Integer i); return fifos[i].empty == 0; endfunction
   function Bool fifoNotFull(Integer i); return fifos[i].full == 0; endfunction
   Vector#(TDiv#(sizet,36), Bool) vNotEmpty = genWith(fifoNotEmpty);
   Vector#(TDiv#(sizet,36), Bool) vNotFull = genWith(fifoNotFull);

   method t first() if (fifos[0].empty == 0);
      function Bit#(36) fifoFirst(Integer i); return fifos[i].dout(); endfunction
      Vector#(TDiv#(sizet,36), Bit#(36)) v = genWith(fifoFirst);
      return unpack(truncateNP(pack(v)));
   endmethod
   method Action deq() if (fifos[0].empty == 0);
      rdenWire <= 1;
   endmethod
   method notEmpty = (fifos[0].empty == 0);
   method Action enq(t v) if (fifos[0].full == 0);
      Vector#(TDiv#(sizet,36), Bit#(36)) vs = unpack(extendNP(pack(v)));
      Vector#(TDiv#(sizet,36), Integer) indices = genVector();
      function Action fifoEnq(Integer i); action dinWires[i] <= vs[i]; endaction endfunction
      mapM_(fifoEnq, indices);
      wrenWire <= 1;
   endmethod
   method notFull = (fifos[0].full == 0);
endmodule

module mkSizedBRAMFIFOF#(Integer m)(FIFOF#(t))
   provisos (Bits#(t,sizet),
	     Add#(1,a__,sizet));
   let clock <- exposeCurrentClock();
   let reset <- exposeCurrentReset();
   let fifo <- mkDualClockBramFIFOF(clock, reset, clock, reset);
   return fifo;
endmodule
module mkSizedBRAMFIFO#(Integer m)(FIFO#(t))
   provisos (Bits#(t,sizet),
	     Add#(1,a__,sizet));
   let clock <- exposeCurrentClock();
   let reset <- exposeCurrentReset();
   let fifo <- mkDualClockBramFIFOF(clock, reset, clock, reset);
   method first = fifo.first;
   method deq = fifo.deq;
   method enq = fifo.enq;
endmodule


temporary fix to write throughput problems

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import Vector::*;
import FIFOF::*;
import FIFO::*;
import GetPut::*;
import GetPut::*;
import ClientServer::*;
import BRAM::*;
import BRAMFIFO::*;
import Connectable::*;

import Ratchet::*;
import PortalMemory::*;
import MemTypes::*;
import Pipe::*;
import MemUtils::*;

module mkMemwriteEngine(MemwriteEngineV#(dataWidth, cmdQDepth, numServers))
   provisos( Mul#(TDiv#(dataWidth, 8), 8, dataWidth)
	    ,Add#(1, a__, numServers)
	    ,Add#(b__, TLog#(numServers), TAdd#(1, TLog#(TMul#(cmdQDepth,numServers))))
	    ,Pipe::FunnelPipesPipelined#(1, numServers,Tuple2#(Bit#(TLog#(numServers)), MemTypes::MemengineCmd), TMin#(2,TLog#(numServers)))
	    ,Pipe::FunnelPipesPipelined#(1, numServers,Tuple2#(Bit#(dataWidth),Bool),TMin#(2, TLog#(numServers)))
	    ,Add#(c__, TLog#(numServers), TLog#(TMul#(cmdQDepth, numServers)))
	    ,Add#(1, d__, dataWidth)
	    ,FunnelPipesPipelined#(1, numServers, Tuple3#(Bit#(2),Bit#(dataWidth),Bool), TMin#(2, TLog#(numServers)))
	    ,FunnelPipesPipelined#(1, numServers,Tuple3#(Bit#(TLog#(numServers)), Bit#(dataWidth), Bool), TMin#(2,TLog#(numServers)))
	    );
   let rv <- mkMemwriteEngineBuff(256);
   return rv;
endmodule

interface BurstFunnel#(numeric type k, numeric type w);
   method Action loadIdx(Bit#(TLog#(k)) i);
   interface Vector#(k, PipeIn#(Bit#(w))) dataIn;
   interface Vector#(k, Reg#(Bit#(8))) burstLen;
   interface PipeOut#(Tuple2#(Bit#(TLog#(k)),Bit#(w))) dataOut;
endinterface

module mkBurstFunnelOld#(Integer __x)(BurstFunnel#(k,w))
   provisos( Log#(k,logk)
	    ,Min#(2,logk,bpc)
	    ,FunnelPipesPipelined#(1, k, Tuple3#(Bit#(logk),Bit#(w),Bool), bpc)
	    );
   // 'mutex' is an ugly hack which should be replaced by something which will actually make timing and achieve full throughput (mdk)
   FIFO#(void) mutex <- mkSizedFIFO(1);
   Vector#(k, FIFOF#(Tuple3#(Bit#(logk), Bit#(w),Bool))) data_in <- replicateM(mkFIFOF);
   Vector#(k,Reg#(Bit#(8))) burst_len <- replicateM(mkReg(0));
   Vector#(k,Reg#(Bit#(8))) inj_ctrl <- replicateM(mkReg(0));
   //TAdd#(1,logk) is because bsc is wierd about comparing literal '0' to a value of Bit#(0)
   FIFO#(Bit#(TAdd#(1,logk))) loadIdxs <- mkSizedFIFO(32);
   function PipeIn#(Bit#(w)) enter_data(FIFOF#(Tuple3#(Bit#(logk), Bit#(w), Bool)) f, Integer i) =
      (interface PipeIn;
	  method Bool notFull = f.notFull;
	  method Action enq(Bit#(w) v) if (loadIdxs.first == fromInteger(i));
	     let first = inj_ctrl[i] == 0;
	     let cnt = first ? burst_len[i] : inj_ctrl[i];
	     let new_cnt = cnt-1;
	     inj_ctrl[i] <= new_cnt;
	     if (first)
		mutex.enq(?);
	     if (new_cnt == 0)
		loadIdxs.deq;
	     //$display("enq (%d) %h", i, v);
	     f.enq(tuple3(fromInteger(i), v, first));
	  endmethod
       endinterface);
      Vector#(k, PipeIn#(Bit#(w))) data_in_pipes = zipWith(enter_data, data_in, genVector);
      // this should use bpc, not logk, but I need to think of a clever way to stop bursts from overtaking oneanother (mdk)
      FunnelPipe#(1, k, Tuple3#(Bit#(logk), Bit#(w),Bool),bpc) data_in_funnel <- mkFunnelPipesPipelined(map(toPipeOut,data_in));
      method Action loadIdx(Bit#(logk) idx);
	 loadIdxs.enq(extend(idx));
	 //$display("loadIdxs %d", idx);
      endmethod
      interface burstLen = burst_len;
	 interface dataIn = data_in_pipes;
	    interface PipeOut dataOut;
	       method Tuple2#(Bit#(logk), Bit#(w)) first;
		  match{.a,.b,.c} = data_in_funnel[0].first;
		  return tuple2(a,b);
	       endmethod
	       method Action deq;
		  if(tpl_3(data_in_funnel[0].first))
		     mutex.deq;
		  data_in_funnel[0].deq;
	       endmethod
	       method Bool notEmpty = data_in_funnel[0].notEmpty;
	    endinterface
endmodule

// this works correctly, but has throughput problems
module mkBurstFunnel#(Integer maxBurstLen)(BurstFunnel#(k,w))
   provisos( Log#(k,logk)
	    ,Min#(2,logk,bpc)
	    ,FunnelPipesPipelined#(1, k, Tuple3#(Bit#(2),Bit#(w),Bool), bpc)
	    );

   Reg#(Bit#(2)) nameGen <- mkReg(0);
   UGBramFifos#(4,16,Tuple2#(Bit#(w),Bool)) complBuff <- mkUGBramFifos;
   Vector#(4, Ratchet#(16)) compCnts <- replicateM(mkRatchet(0));
   Vector#(k,FIFOF#(Tuple3#(Bit#(2), Bit#(w), Bool))) data_in <- replicateM(mkFIFOF);
   Vector#(k,Reg#(Bit#(8))) burst_len <- replicateM(mkReg(0));
   Vector#(k,Reg#(Bit#(8))) drain_cnt <- replicateM(mkReg(0));
   Vector#(k,Reg#(Bit#(8))) inj_ctrl <- replicateM(mkReg(0));
   FIFO#(Tuple2#(Bit#(TAdd#(1,logk)),Bit#(2))) loadIdxs <- mkSizedFIFO(1);
   FIFO#(Tuple2#(Bit#(TAdd#(1,logk)),Bit#(2))) inFlight <- mkSizedFIFO(3);
   FunnelPipe#(1, k, Tuple3#(Bit#(2),Bit#(w),Bool),bpc) data_in_funnel <- mkFunnelPipesPipelined(map(toPipeOut,data_in));
   Reg#(Bit#(8)) drainCnt <- mkReg(0);
   FIFOF#(Tuple2#(Bit#(TLog#(k)),Bit#(w))) exit_data <- mkFIFOF;
   Reg#(Bit#(2)) newName <- mkReg(0);
   FIFO#(Bit#(logk)) drainRename <- mkFIFO;

   function PipeIn#(Bit#(w)) enter_data(FIFOF#(Tuple3#(Bit#(2), Bit#(w), Bool)) f, Integer i) =
      (interface PipeIn;
   	  method Bool notFull = f.notFull;
   	  method Action enq(Bit#(w) v) if (tpl_1(loadIdxs.first) == fromInteger(i));
	     match {.old_name, .new_name} = loadIdxs.first;
	     let first = inj_ctrl[i] == 0;
	     let cnt = first ? burst_len[i] : inj_ctrl[i];
	     let new_cnt = cnt-1;
	     let last = new_cnt == 0;
	     inj_ctrl[i] <= new_cnt;
	     if (first)
		inFlight.enq(loadIdxs.first);
	     if (last)
		loadIdxs.deq;
	     f.enq(tuple3(new_name, v, last));
	  endmethod
       endinterface);
   Vector#(k, PipeIn#(Bit#(w))) data_in_pipes = zipWith(enter_data, data_in, genVector);

   function Reg#(Bit#(8)) check(Reg#(Bit#(8)) r) =
      (interface Reg;
	  method Action _write(Bit#(8) v);
	     if(v > 16) begin
		$display("ERROR mkBurstFunnel: burstLen too large");
		$finish;
	     end
	     r <= v;
	  endmethod
	  method Bit#(8) _read = r._read;
       endinterface);

   (* fire_when_enabled *)
   rule drain_funnel;
      match{.new_name,.data,.last} = data_in_funnel[0].first;
      data_in_funnel[0].deq;
      complBuff.enq(new_name,tuple2(data,last));
      compCnts[new_name].increment(1);
   endrule

   rule start_drain if (drainCnt == 0);
      match {.old_name, .new_name} = inFlight.first;
      newName <= new_name;
      drainCnt <= burst_len[old_name];
      inFlight.deq;
      drainRename.enq(truncate(old_name));
   endrule

   (* fire_when_enabled *)
   rule drain_req (drainCnt > 0 && compCnts[newName].read > 0);
      complBuff.first_req(newName);
      drainCnt <= drainCnt-1;
      compCnts[newName].decrement(1);
      complBuff.deq(newName);
   endrule

   (* fire_when_enabled *)
   rule drain_resp;
      match {.data,.last} <- complBuff.first_resp;
      if (last)
	 drainRename.deq;
      exit_data.enq(tuple2(drainRename.first,data));
   endrule

   method Action loadIdx(Bit#(logk) idx);
      loadIdxs.enq(tuple2(extend(idx),nameGen));
      nameGen <= nameGen+1;
   endmethod
   interface burstLen = map(check,burst_len);
   interface dataIn = data_in_pipes;
   interface PipeOut dataOut = toPipeOut(exit_data);
endmodule

module mkMemwriteEngineBuff#(Integer bufferSizeBytes)(MemwriteEngineV#(dataWidth, cmdQDepth, numServers))
   provisos ( Div#(dataWidth,8,dataWidthBytes)
	     ,Mul#(dataWidthBytes,8,dataWidth)
	     ,Log#(dataWidthBytes,beatShift)
	     ,Log#(cmdQDepth,logCmdQDepth)
	     ,Mul#(cmdQDepth,numServers,cmdBuffSz)
	     ,Log#(cmdBuffSz, cmdBuffAddrSz)
	     ,Log#(numServers, serverIdxSz)
	     ,Add#(1,logCmdQDepth, outCntSz)
	     ,Add#(1, c__, numServers)
	     ,Add#(b__, TLog#(numServers), cmdBuffAddrSz)
	     ,Add#(e__, TLog#(numServers), TAdd#(1, cmdBuffAddrSz))
	     ,Add#(a__, serverIdxSz, cmdBuffAddrSz)
	     ,Min#(2,TLog#(numServers),bpc)
	     ,FunnelPipesPipelined#(1,numServers,Tuple2#(Bit#(serverIdxSz),MemengineCmd),bpc)
	     ,FunnelPipesPipelined#(1,numServers,Tuple2#(Bit#(dataWidth),Bool),bpc)
	     ,FunnelPipesPipelined#(1, numServers, Tuple3#(Bit#(2),Bit#(dataWidth),Bool), TMin#(2, serverIdxSz))
	     ,Add#(1, d__, dataWidth)
	     ,FunnelPipesPipelined#(1, numServers, Tuple3#(Bit#(serverIdxSz),Bit#(dataWidth), Bool), TMin#(2, serverIdxSz))
	     );

   Integer bufferSizeBeats = bufferSizeBytes/valueOf(dataWidthBytes);
   Vector#(numServers, Reg#(Bit#(outCntSz)))     outs1 <- replicateM(mkReg(0));
   Vector#(numServers, Reg#(Bit#(outCntSz)))     outs0 <- replicateM(mkReg(0));
   Vector#(numServers, Ratchet#(16))           buffCap <- replicateM(mkRatchet(0));
   UGBramFifos#(numServers,cmdQDepth,MemengineCmd) cmdBuf <- mkUGBramFifos;

   FIFO#(Bit#(serverIdxSz))                       loadf_a <- mkSizedFIFO(1);
   FIFO#(Tuple2#(Bit#(serverIdxSz),MemengineCmd)) loadf_b <- mkSizedFIFO(1);
   FIFO#(Tuple3#(Bit#(8),Bit#(serverIdxSz),Bool))   workf <- mkSizedFIFO(32); // is this the right size?
   FIFO#(Tuple2#(Bit#(serverIdxSz),Bool))           donef <- mkSizedFIFO(32); // is this the right size?

   Vector#(numServers, FIFO#(void))              outfs <- replicateM(mkSizedFIFO(1));
   Vector#(numServers, FIFOF#(Tuple2#(Bit#(serverIdxSz), MemengineCmd))) cmds_in <- replicateM(mkSizedFIFOF(1));
   FunnelPipe#(1, numServers, Tuple2#(Bit#(serverIdxSz), MemengineCmd),bpc) cmds_in_funnel <- mkFunnelPipesPipelined(map(toPipeOut,cmds_in));
   Vector#(numServers, FIFOF#(Bit#(dataWidth)))  write_data_buffs <- replicateM(mkSizedBRAMFIFOF(bufferSizeBeats));
   Vector#(numServers, PipeOut#(Bit#(dataWidth))) foo = map(toPipeOut, write_data_buffs);
   BurstFunnel#(numServers,dataWidth) write_data_funnel <- mkBurstFunnelOld(bufferSizeBeats);
   zipWithM(mkConnection, foo, write_data_funnel.dataIn);

   Reg#(Bit#(8))                               respCnt <- mkReg(0);
   Reg#(Bit#(serverIdxSz))                     loadIdx <- mkReg(0);
   let beat_shift = fromInteger(valueOf(beatShift));
   let cmd_q_depth = fromInteger(valueOf(cmdQDepth));

   rule store_cmd;
      match {.idx, .cmd} <- toGet(cmds_in_funnel[0]).get;
      outs1[idx] <= outs1[idx]+1;
      cmdBuf.enq(idx,cmd);
      //$display("store_cmd %d", idx);
   endrule

   rule load_ctxt_a;
      loadIdx <= loadIdx+1;
      if (outs1[loadIdx] > 0) begin
	 cmdBuf.first_req(loadIdx);
	 loadf_a.enq(loadIdx);
	 //$display("load_ctxt_a %d", loadIdx);
      end
   endrule

   rule load_ctxt_b;
      let idx <- toGet(loadf_a).get;
      let cmd <- cmdBuf.first_resp;
      //$display("%d %d", buffCap[idx].read(), cmd.burstLen>>beat_shift);
      if (outs1[idx] > 0 && buffCap[idx].read() >= unpack(extend(cmd.burstLen>>beat_shift))) begin
	 //$display("load_ctxt_b %h %d", cmd.base, idx);
	 buffCap[idx].decrement(unpack(extend(cmd.burstLen>>beat_shift)));
	 loadf_b.enq(tuple2(idx,cmd));
	 write_data_funnel.loadIdx(idx);
	 if (cmd.len <= extend(cmd.burstLen)) begin
	    outs1[idx] <= outs1[idx]-1;
	    cmdBuf.deq(idx);
	 end
	 else begin
	    let new_cmd = MemengineCmd{pointer:cmd.pointer, base:cmd.base+extend(cmd.burstLen), burstLen:cmd.burstLen, len:cmd.len-extend(cmd.burstLen)};
	    cmdBuf.upd_head(idx,new_cmd);
	 end
      end
   endrule

   function PipeIn#(Bit#(w)) check_in(FIFOF#(Bit#(w)) f, Integer i) =
      (interface PipeIn;
   	  method Bool notFull = f.notFull;
   	  method Action enq(Bit#(w) v);
	     f.enq(v);
	     buffCap[i].increment(1);
	     //$display("check_in %d", i);
	     // if(i==2)
	     // 	for(Integer j = 0; j < valueOf(w); j=j+32) begin
	     // 	   Bit#(32) xx = v[j+31:j];
	     // 	   $display("%h", xx);
	     // 	end
	  endmethod
       endinterface);

   Vector#(numServers, Server#(MemengineCmd,Bool)) rs;
   for(Integer i = 0; i < valueOf(numServers); i=i+1)
      rs[i] = (interface Server#(MemengineCmd,Bool);
		  interface Put request;
		     method Action put(MemengineCmd c) if (outs0[i] < cmd_q_depth);
			Bit#(32) bsb = fromInteger(bufferSizeBytes);
			if(extend(c.burstLen) > bsb)
			   $display("mkMemwriteEngineV::unsupportedBurstLen");
			outs0[i] <= outs0[i]+1;
			cmds_in[i].enq(tuple2(fromInteger(i),c));
			write_data_funnel.burstLen[i] <= c.burstLen >> beat_shift;
			//$display("(%d) %h %h %h", i, c.base, c.len, c.burstLen);
 		     endmethod
		  endinterface
		  interface Get response;
		     method ActionValue#(Bool) get;
			outfs[i].deq;
	 		outs0[i] <= outs0[i]-1;
			return True;
		     endmethod
		  endinterface
	       endinterface);
   interface writeServers = rs;
   interface ObjectWriteClient dmaClient;
      interface Get writeReq;
	 method ActionValue#(ObjectRequest) get();
	    match {.idx, .cmd} <- toGet(loadf_b).get;
	    Bit#(8) bl = cmd.burstLen;
	    Bool last = False;
	    if (cmd.len <= extend(bl)) begin
	       last = True;
	       bl = truncate(cmd.len);
	    end
	    workf.enq(tuple3(truncate(bl>>beat_shift), idx, last));
	    //$display("writeReq %d, %h %h %h", idx, cmd.base, bl, last);
	    return ObjectRequest { pointer: cmd.pointer, offset: cmd.base, burstLen:bl, tag: 0 };
	 endmethod
      endinterface
      interface Get writeData;
	 method ActionValue#(ObjectData#(dataWidth)) get;
	    match {.rc, .idx, .last} = workf.first;
	    let new_respCnt = respCnt+1;
	    if (new_respCnt == rc) begin
	       respCnt <= 0;
	       workf.deq;
	       donef.enq(tuple2(idx,last));
	    end
	    else begin
	       respCnt <= new_respCnt;
	    end
	    match {._idx, .wd} <- toGet(write_data_funnel.dataOut).get;
	    if(_idx != idx) begin
	       $display("ERROR mkMemwriteEngineBuf: bursts oo %d %d", idx, _idx);
	       $finish;
	    end
	    return ObjectData{data:wd, tag:0, last: False};
	 endmethod
      endinterface
      interface Put writeDone;
	 method Action put(Bit#(6) tag);
	    match {.idx, .last} <- toGet(donef).get;
	    if (last)
	       outfs[idx].enq(?);
	    //$display("writeDone %d %d", idx, last);
	 endmethod
      endinterface
   endinterface
   interface dataPipes = zipWith(check_in, write_data_buffs, genVector);
endmodule

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import Vector::*;
import FIFOF::*;
import FIFO::*;
import GetPut::*;
import GetPut::*;
import ClientServer::*;
import BRAM::*;
import BRAMFIFO::*;
import Connectable::*;

import Ratchet::*;
import PortalMemory::*;
import MemTypes::*;
import Pipe::*;
import MemUtils::*;

module mkMemwriteEngine(MemwriteEngineV#(dataWidth, cmdQDepth, numServers))
   provisos( Mul#(TDiv#(dataWidth, 8), 8, dataWidth)
	    ,Add#(1, a__, numServers)
	    ,Add#(b__, TLog#(numServers), TAdd#(1, TLog#(TMul#(cmdQDepth,numServers))))
	    ,Pipe::FunnelPipesPipelined#(1, numServers,Tuple2#(Bit#(TLog#(numServers)), MemTypes::MemengineCmd), TMin#(2,TLog#(numServers)))
	    ,Pipe::FunnelPipesPipelined#(1, numServers,Tuple2#(Bit#(dataWidth),Bool),TMin#(2, TLog#(numServers)))
	    ,Add#(c__, TLog#(numServers), TLog#(TMul#(cmdQDepth, numServers)))
	    ,Add#(1, d__, dataWidth)
	    ,FunnelPipesPipelined#(1, numServers, Tuple3#(Bit#(2),Bit#(dataWidth),Bool), TMin#(2, TLog#(numServers)))
	    );
   let rv <- mkMemwriteEngineBuff(256);
   return rv;
endmodule

interface BurstFunnel#(numeric type k, numeric type w);
   method Action loadIdx(Bit#(TLog#(k)) i);
   interface Vector#(k, PipeIn#(Bit#(w))) dataIn;
   interface Vector#(k, Reg#(Bit#(8))) burstLen;
   interface PipeOut#(Tuple2#(Bit#(TLog#(k)),Bit#(w))) dataOut;
endinterface

module mkBurstFunnel#(Integer maxBurstLen)(BurstFunnel#(k,w))
   provisos( Log#(k,logk)
	    ,Min#(2,logk,bpc)
	    ,FunnelPipesPipelined#(1, k, Tuple3#(Bit#(2),Bit#(w),Bool), bpc)
	    );

   Reg#(Bit#(2)) nameGen <- mkReg(0);
   UGBramFifos#(4,16,Tuple2#(Bit#(w),Bool)) complBuff <- mkUGBramFifos;
   Vector#(4, Ratchet#(16)) compCnts <- replicateM(mkRatchet(0));
   Vector#(k,FIFOF#(Tuple3#(Bit#(2), Bit#(w), Bool))) data_in <- replicateM(mkFIFOF);
   Vector#(k,Reg#(Bit#(8))) burst_len <- replicateM(mkReg(0));
   Vector#(k,Reg#(Bit#(8))) drain_cnt <- replicateM(mkReg(0));
   Vector#(k,Reg#(Bit#(8))) inj_ctrl <- replicateM(mkReg(0));
   FIFO#(Tuple2#(Bit#(TAdd#(1,logk)),Bit#(2))) loadIdxs <- mkSizedFIFO(1);
   FIFO#(Tuple2#(Bit#(TAdd#(1,logk)),Bit#(2))) inFlight <- mkSizedFIFO(3);
   FunnelPipe#(1, k, Tuple3#(Bit#(2),Bit#(w),Bool),bpc) data_in_funnel <- mkFunnelPipesPipelined(map(toPipeOut,data_in));
   Reg#(Bit#(8)) drainCnt <- mkReg(0);
   FIFOF#(Tuple2#(Bit#(TLog#(k)),Bit#(w))) exit_data <- mkFIFOF;
   Reg#(Bit#(2)) newName <- mkReg(0);
   FIFO#(Bit#(logk)) drainRename <- mkFIFO;

   function PipeIn#(Bit#(w)) enter_data(FIFOF#(Tuple3#(Bit#(2), Bit#(w), Bool)) f, Integer i) =
      (interface PipeIn;
   	  method Bool notFull = f.notFull;
   	  method Action enq(Bit#(w) v) if (tpl_1(loadIdxs.first) == fromInteger(i));
	     match {.old_name, .new_name} = loadIdxs.first;
	     let first = inj_ctrl[i] == 0;
	     let cnt = first ? burst_len[i] : inj_ctrl[i];
	     let new_cnt = cnt-1;
	     let last = new_cnt == 0;
	     inj_ctrl[i] <= new_cnt;
	     if (first)
		inFlight.enq(loadIdxs.first);
	     if (last)
		loadIdxs.deq;
	     f.enq(tuple3(new_name, v, last));
	  endmethod
       endinterface);
   Vector#(k, PipeIn#(Bit#(w))) data_in_pipes = zipWith(enter_data, data_in, genVector);

   function Reg#(Bit#(8)) check(Reg#(Bit#(8)) r) =
      (interface Reg;
	  method Action _write(Bit#(8) v);
	     if(v > 16) begin
		$display("ERROR mkBurstFunnel: burstLen too large");
		$finish;
	     end
	     r <= v;
	  endmethod
	  method Bit#(8) _read = r._read;
       endinterface);

   rule drain_funnel;
      match{.new_name,.data,.last} = data_in_funnel[0].first;
      data_in_funnel[0].deq;
      complBuff.enq(new_name,tuple2(data,last));
      compCnts[new_name].increment(1);
   endrule

   rule start_drain if (drainCnt == 0);
      match {.old_name, .new_name} = inFlight.first;
      newName <= new_name;
      drainCnt <= burst_len[old_name];
      inFlight.deq;
      drainRename.enq(truncate(old_name));
   endrule

   rule drain_req (drainCnt > 0 && compCnts[newName].read > 0);
      complBuff.first_req(newName);
      drainCnt <= drainCnt-1;
      compCnts[newName].decrement(1);
      complBuff.deq(newName);
   endrule

   rule drain_resp;
      match {.data,.last} <- complBuff.first_resp;
      if (last)
	 drainRename.deq;
      exit_data.enq(tuple2(drainRename.first,data));
   endrule

   method Action loadIdx(Bit#(logk) idx);
      loadIdxs.enq(tuple2(extend(idx),nameGen));
      nameGen <= nameGen+1;
   endmethod
   interface burstLen = map(check,burst_len);
   interface dataIn = data_in_pipes;
   interface PipeOut dataOut = toPipeOut(exit_data);
endmodule

module mkMemwriteEngineBuff#(Integer bufferSizeBytes)(MemwriteEngineV#(dataWidth, cmdQDepth, numServers))
   provisos ( Div#(dataWidth,8,dataWidthBytes)
	     ,Mul#(dataWidthBytes,8,dataWidth)
	     ,Log#(dataWidthBytes,beatShift)
	     ,Log#(cmdQDepth,logCmdQDepth)
	     ,Mul#(cmdQDepth,numServers,cmdBuffSz)
	     ,Log#(cmdBuffSz, cmdBuffAddrSz)
	     ,Log#(numServers, serverIdxSz)
	     ,Add#(1,logCmdQDepth, outCntSz)
	     ,Add#(1, c__, numServers)
	     ,Add#(b__, TLog#(numServers), cmdBuffAddrSz)
	     ,Add#(e__, TLog#(numServers), TAdd#(1, cmdBuffAddrSz))
	     ,Add#(a__, serverIdxSz, cmdBuffAddrSz)
	     ,Min#(2,TLog#(numServers),bpc)
	     ,FunnelPipesPipelined#(1,numServers,Tuple2#(Bit#(serverIdxSz),MemengineCmd),bpc)
	     ,FunnelPipesPipelined#(1,numServers,Tuple2#(Bit#(dataWidth),Bool),bpc)
	     ,FunnelPipesPipelined#(1, numServers, Tuple3#(Bit#(2),Bit#(dataWidth),Bool), TMin#(2, serverIdxSz))
	     ,Add#(1, d__, dataWidth)
	     );

   Integer bufferSizeBeats = bufferSizeBytes/valueOf(dataWidthBytes);
   Vector#(numServers, Reg#(Bit#(outCntSz)))     outs1 <- replicateM(mkReg(0));
   Vector#(numServers, Reg#(Bit#(outCntSz)))     outs0 <- replicateM(mkReg(0));
   Vector#(numServers, Ratchet#(16))           buffCap <- replicateM(mkRatchet(0));
   UGBramFifos#(numServers,cmdQDepth,MemengineCmd) cmdBuf <- mkUGBramFifos;

   FIFO#(Bit#(serverIdxSz))                       loadf_a <- mkSizedFIFO(1);
   FIFO#(Tuple2#(Bit#(serverIdxSz),MemengineCmd)) loadf_b <- mkSizedFIFO(1);
   FIFO#(Tuple3#(Bit#(8),Bit#(serverIdxSz),Bool))   workf <- mkSizedFIFO(32); // is this the right size?
   FIFO#(Tuple2#(Bit#(serverIdxSz),Bool))           donef <- mkSizedFIFO(32); // is this the right size?

   Vector#(numServers, FIFO#(void))              outfs <- replicateM(mkSizedFIFO(1));
   Vector#(numServers, FIFOF#(Tuple2#(Bit#(serverIdxSz), MemengineCmd))) cmds_in <- replicateM(mkSizedFIFOF(1));
   FunnelPipe#(1, numServers, Tuple2#(Bit#(serverIdxSz), MemengineCmd),bpc) cmds_in_funnel <- mkFunnelPipesPipelined(map(toPipeOut,cmds_in));
   Vector#(numServers, FIFOF#(Bit#(dataWidth)))  write_data_buffs <- replicateM(mkSizedBRAMFIFOF(bufferSizeBeats));
   Vector#(numServers, PipeOut#(Bit#(dataWidth))) foo = map(toPipeOut, write_data_buffs);
   BurstFunnel#(numServers,dataWidth) write_data_funnel <- mkBurstFunnel(bufferSizeBeats);
   zipWithM(mkConnection, foo, write_data_funnel.dataIn);

   Reg#(Bit#(8))                               respCnt <- mkReg(0);
   Reg#(Bit#(serverIdxSz))                     loadIdx <- mkReg(0);
   let beat_shift = fromInteger(valueOf(beatShift));
   let cmd_q_depth = fromInteger(valueOf(cmdQDepth));

   rule store_cmd;
      match {.idx, .cmd} <- toGet(cmds_in_funnel[0]).get;
      outs1[idx] <= outs1[idx]+1;
      cmdBuf.enq(idx,cmd);
      //$display("store_cmd %d", idx);
   endrule

   rule load_ctxt_a;
      loadIdx <= loadIdx+1;
      if (outs1[loadIdx] > 0) begin
	 cmdBuf.first_req(loadIdx);
	 loadf_a.enq(loadIdx);
	 //$display("load_ctxt_a %d", loadIdx);
      end
   endrule

   rule load_ctxt_b;
      let idx <- toGet(loadf_a).get;
      let cmd <- cmdBuf.first_resp;
      //$display("%d %d", buffCap[idx].read(), cmd.burstLen>>beat_shift);
      if (outs1[idx] > 0 && buffCap[idx].read() >= unpack(extend(cmd.burstLen>>beat_shift))) begin
	 //$display("load_ctxt_b %h %d", cmd.base, idx);
	 buffCap[idx].decrement(unpack(extend(cmd.burstLen>>beat_shift)));
	 loadf_b.enq(tuple2(idx,cmd));
	 write_data_funnel.loadIdx(idx);
	 if (cmd.len <= extend(cmd.burstLen)) begin
	    outs1[idx] <= outs1[idx]-1;
	    cmdBuf.deq(idx);
	 end
	 else begin
	    let new_cmd = MemengineCmd{pointer:cmd.pointer, base:cmd.base+extend(cmd.burstLen), burstLen:cmd.burstLen, len:cmd.len-extend(cmd.burstLen)};
	    cmdBuf.upd_head(idx,new_cmd);
	 end
      end
   endrule

   function PipeIn#(Bit#(w)) check_in(FIFOF#(Bit#(w)) f, Integer i) =
      (interface PipeIn;
   	  method Bool notFull = f.notFull;
   	  method Action enq(Bit#(w) v);
	     f.enq(v);
	     buffCap[i].increment(1);
	     //$display("check_in %d", i);
	     // if(i==2)
	     // 	for(Integer j = 0; j < valueOf(w); j=j+32) begin
	     // 	   Bit#(32) xx = v[j+31:j];
	     // 	   $display("%h", xx);
	     // 	end
	  endmethod
       endinterface);

   Vector#(numServers, Server#(MemengineCmd,Bool)) rs;
   for(Integer i = 0; i < valueOf(numServers); i=i+1)
      rs[i] = (interface Server#(MemengineCmd,Bool);
		  interface Put request;
		     method Action put(MemengineCmd c) if (outs0[i] < cmd_q_depth);
			Bit#(32) bsb = fromInteger(bufferSizeBytes);
			if(extend(c.burstLen) > bsb)
			   $display("mkMemwriteEngineV::unsupportedBurstLen");
			outs0[i] <= outs0[i]+1;
			cmds_in[i].enq(tuple2(fromInteger(i),c));
			write_data_funnel.burstLen[i] <= c.burstLen >> beat_shift;
			//$display("(%d) %h %h %h", i, c.base, c.len, c.burstLen);
 		     endmethod
		  endinterface
		  interface Get response;
		     method ActionValue#(Bool) get;
			outfs[i].deq;
	 		outs0[i] <= outs0[i]-1;
			return True;
		     endmethod
		  endinterface
	       endinterface);
   interface writeServers = rs;
   interface ObjectWriteClient dmaClient;
      interface Get writeReq;
	 method ActionValue#(ObjectRequest) get();
	    match {.idx, .cmd} <- toGet(loadf_b).get;
	    Bit#(8) bl = cmd.burstLen;
	    Bool last = False;
	    if (cmd.len <= extend(bl)) begin
	       last = True;
	       bl = truncate(cmd.len);
	    end
	    workf.enq(tuple3(truncate(bl>>beat_shift), idx, last));
	    //$display("writeReq %d, %h %h %h", idx, cmd.base, bl, last);
	    return ObjectRequest { pointer: cmd.pointer, offset: cmd.base, burstLen:bl, tag: 0 };
	 endmethod
      endinterface
      interface Get writeData;
	 method ActionValue#(ObjectData#(dataWidth)) get;
	    match {.rc, .idx, .last} = workf.first;
	    let new_respCnt = respCnt+1;
	    if (new_respCnt == rc) begin
	       respCnt <= 0;
	       workf.deq;
	       donef.enq(tuple2(idx,last));
	    end
	    else begin
	       respCnt <= new_respCnt;
	    end
	    match {._idx, .wd} <- toGet(write_data_funnel.dataOut).get;
	    if(_idx != idx) begin
	       $display("ERROR mkMemwriteEngineBuf: bursts oo %d %d", idx, _idx);
	       $finish;
	    end
	    return ObjectData{data:wd, tag:0, last: False};
	 endmethod
      endinterface
      interface Put writeDone;
	 method Action put(Bit#(6) tag);
	    match {.idx, .last} <- toGet(donef).get;
	    if (last)
	       outfs[idx].enq(?);
	    //$display("writeDone %d %d", idx, last);
	 endmethod
      endinterface
   endinterface
   interface dataPipes = zipWith(check_in, write_data_buffs, genVector);
endmodule


use smaller fifos, linux still boots on TINY

// Copyright (c) 2017 Massachusetts Institute of Technology
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

`include "ProcConfig.bsv"

import BrPred::*;
import DirPredictor::*;
import Btb::*;
import ClientServer::*;
import Connectable::*;
import Decode::*;
import Ehr::*;
import Fifo::*;
import GetPut::*;
import MemoryTypes::*;
import Types::*;
import ProcTypes::*;
import CCTypes::*;
import Ras::*;
import EpochManager::*;
import Performance::*;
import Vector::*;
import Assert::*;
import Cntrs::*;
import ConfigReg::*;
import TlbTypes::*;
import ITlb::*;
import CCTypes::*;
import L1CoCache::*;
import MMIOInst::*;

interface FetchStage;
    // pipeline
    interface Vector#(SupSize, SupFifoDeq#(FromFetchStage)) pipelines;

    // tlb and mem connections
    interface ITlb iTlbIfc;
    interface ICoCache iMemIfc;
    interface MMIOInstToCore mmioIfc;

    // starting and stopping
    method Action start(Addr pc);
    method Action stop();

    // redirection methods
    method Action setWaitRedirect;
    method Action redirect(Addr pc);
    method Action done_flushing();
    method Action train_predictors(
        Addr pc, Addr next_pc, IType iType, Bool taken,
        DirPredTrainInfo dpTrain, Bool mispred
    );

    // debug
    method FetchDebugState getFetchState;

    // performance
    interface Perf#(DecStagePerfType) perf;
endinterface

typedef struct {
    Addr pc;
    Epoch mainEp;
    Bool waitForRedirect;
    Bool waitForFlush;
} FetchDebugState deriving(Bits, Eq, FShow);

typedef struct {
    Addr pc;
    Addr pred_next_pc;
    Bool decode_epoch;
    Epoch main_epoch;
} Fetch1ToFetch2 deriving(Bits, Eq, FShow);

typedef struct {
    Addr pc;
    Addr phys_pc;
    Addr pred_next_pc;
    Maybe#(Exception) cause;
    Bool access_mmio; // inst fetch from MMIO
    Bool decode_epoch;
    Epoch main_epoch;
} Fetch2ToFetch3 deriving(Bits, Eq, FShow);

typedef struct {
  Addr pc;
  Addr ppc;
  Bool decode_epoch;
  Epoch main_epoch;
  Instruction inst;
  Maybe#(Exception) cause;
} Fetch3ToDecode deriving(Bits, Eq, FShow);

typedef struct {
  Addr pc;
  Addr ppc;
  Epoch main_epoch;
  DirPredTrainInfo dpTrain;
  Instruction inst;
  DecodedInst dInst;
  ArchRegs regs;
  Maybe#(Exception) cause;
} FromFetchStage deriving (Bits, Eq, FShow);

// train next addr pred (BTB)
typedef struct {
    Addr pc;
    Addr nextPc;
} TrainNAP deriving(Bits, Eq, FShow);

(* synthesize *)
module mkFetchStage(FetchStage);
    // rule ordering: Fetch1 (BTB+TLB) < Fetch3 (decode & dir pred) < redirect method
    // Fetch1 < Fetch3 to avoid bypassing path on PC and epochs

    let verbose = True;

    // Basic State Elements
    Reg#(Bool) started <- mkReg(False);

    // Stall fetch when trap happens or system inst is renamed
    // All inst younger than the trap/system inst will be killed
    // Since CSR may be modified, sending wrong path request to TLB may cause problem
    // So we stall until the next redirection happens
    // The next redirect is either by the trap/system inst or an older one
    Reg#(Bool) waitForRedirect <- mkReg(False);
    // We don't want setWaitForRedirect method and redirect method to happen together
    // make them conflict
    RWire#(void) setWaitRedirect_redirect_conflict <- mkRWire;

    // Stall fetch during the flush triggered by the procesing trap/system inst in commit stage
    // We stall until the flush is done
    Reg#(Bool) waitForFlush <- mkReg(False);

    Ehr#(3, Addr) pc_reg <- mkEhr(0);
    Integer pc_fetch1_port = 0;
    Integer pc_decode_port = 1;
    Integer pc_redirect_port = 2;

    // Epochs
    Reg#(Bool) decode_epoch <- mkReg(False);
    Reg#(Epoch) f_main_epoch <- mkReg(0); // fetch estimate of main epoch

    // Pipeline Stage FIFOs
    Fifo#(2, Tuple2#(Bit#(TLog#(SupSize)),Fetch1ToFetch2)) f12f2 <- mkCFFifo;
    Fifo#(4, Tuple2#(Bit#(TLog#(SupSize)),Fetch2ToFetch3)) f22f3 <- mkCFFifo; // FIFO should match I$ latency
    Fifo#(2, Tuple2#(Bit#(TLog#(SupSize)),Fetch2ToFetch3)) f32d <- mkCFFifo;
    Fifo#(2, Vector#(SupSize,Maybe#(Instruction))) instdata <- mkPipelineFifo();
    SupFifo#(SupSize, 2, FromFetchStage) out_fifo <- mkSupFifo;
       // Can the fifo size be smaller?

    // Branch Predictors
    NextAddrPred    nextAddrPred <- mkBtb;
    let             dirPred      <- mkDirPredictor;
    ReturnAddrStack ras          <- mkRas;
    // Wire to train next addr pred (NAP)
    RWire#(TrainNAP) napTrainByDec <- mkRWire;
    RWire#(TrainNAP) napTrainByExe <- mkRWire;

    // TLB and Cache connections
    ITlb iTlb <- mkITlb;
    ICoCache iMem <- mkICoCache;
    MMIOInst mmio <- mkMMIOInst;
    Server#(Addr, TlbResp) tlb_server = iTlb.to_proc;
    Server#(Addr, Vector#(SupSize, Maybe#(Instruction))) mem_server = iMem.to_proc;

    // performance counters
    Fifo#(1, DecStagePerfType) perfReqQ <- mkCFFifo; // perf req FIFO
`ifdef PERF_COUNT
    Reg#(Bool) doStats <- mkConfigReg(False);
    // decode stage redirect
    Count#(Data) decRedirectBrCnt <- mkCount(0);
    Count#(Data) decRedirectJmpCnt <- mkCount(0);
    Count#(Data) decRedirectJrCnt <- mkCount(0);
    Count#(Data) decRedirectOtherCnt <- mkCount(0);
    // perf resp FIFO
    Fifo#(1, PerfResp#(DecStagePerfType)) perfRespQ <- mkCFFifo;

    rule doPerfReq;
        let t <- toGet(perfReqQ).get;
        Data d = (case(t)
            DecRedirectBr: decRedirectBrCnt;
            DecRedirectJmp: decRedirectJmpCnt;
            DecRedirectJr: decRedirectJrCnt;
            DecRedirectOther: decRedirectOtherCnt;
            default: 0;
        endcase);
        perfRespQ.enq(PerfResp {
            pType: t,
            data: d
        });
    endrule
`endif

    // We don't send req to TLB when waiting for redirect or TLB flush. Since
    // there is no FIFO between doFetch1 and TLB, when OOO commit stage wait
    // TLB idle to change VM CSR / signal flush TLB, there is no wrong path
    // request afterwards to race with the system code that manage paget table.
    rule doFetch1(started && !waitForRedirect && !waitForFlush);
        let pc = pc_reg[pc_fetch1_port];

        // Chain of prediction for the next instructions
        // We need a BTB with a register file with enough ports!
        // Instead of cascading predictions, we can always feed pc+4*i into
        // predictor, because we will break superscaler fetch if nextpc != pc+4
        Vector#(SupSize, Addr) pred_future_pc;
        for(Integer i = 0; i < valueof(SupSize); i = i+1) begin
            pred_future_pc[i] = nextAddrPred.predPc(pc + fromInteger(4 * i));
        end

        // Next pc is the first nextPc that breaks the chain of pc+4 or
        // that is at the end of a cacheline.
        Vector#(SupSize,Integer) indexes = genVector;
        function Bool findNextPc(Addr pc, Integer i);
            Bool notLastInst = getLineInstOffset(pc + fromInteger(4*i)) != maxBound;
            Bool noJump = pred_future_pc[i] == pc + fromInteger(4*(i+1));
            return (!(notLastInst && noJump));
        endfunction
        Integer posLastSup = fromMaybe(valueof(SupSize) - 1, find(findNextPc(pc), indexes));
        Addr pred_next_pc = pred_future_pc[posLastSup];
        pc_reg[pc_fetch1_port] <= pred_next_pc;

        // Send TLB request
        tlb_server.request.put(pc);

        let out = Fetch1ToFetch2 {
            pc: pc,
            pred_next_pc: pred_next_pc,
            decode_epoch: decode_epoch,
            main_epoch: f_main_epoch};
        f12f2.enq(tuple2(fromInteger(posLastSup),out));
        if (verbose) $display("Fetch1: ", fshow(out));
    endrule

    rule doFetch2;
        let {nbSup,in} = f12f2.first;
        f12f2.deq;

        // Get TLB response
        match {.phys_pc, .cause} <- tlb_server.response.get;

        // Access main mem or boot rom
        Bool access_mmio = False;
        if (!isValid(cause)) begin
            case(mmio.getFetchTarget(phys_pc))
                MainMem: begin
                    // Send ICache request
                    mem_server.request.put(phys_pc);
                end
                BootRom: begin
                    // Send MMIO req. Luckily boot rom is also aligned with
                    // cache line size, so all nbSup+1 insts can be fetched
                    // from boot rom. It won't happen that insts fetched from
                    // boot rom is less than requested.
                    mmio.bootRomReq(phys_pc, nbSup);
                    access_mmio = True;
                end
                default: begin
                    // Access fault
                    cause = Valid (InstAccessFault);
                end
            endcase
        end

        let out = Fetch2ToFetch3 {
            pc: in.pc,
            phys_pc: phys_pc,
            pred_next_pc: in.pred_next_pc,
            cause: cause,
            access_mmio: access_mmio,
            decode_epoch: in.decode_epoch,
            main_epoch: in.main_epoch };
        f22f3.enq(tuple2(nbSup,out));
        if (verbose) $display("Fetch2: ", fshow(out));
    endrule

// Break out of i$
    rule doFetch3;
        let {nbSup, fetch3In} = f22f3.first;
        f22f3.deq();
        if (verbose) $display("Fetch3 %d",fetch3In.pc);

        // Get ICache/MMIO response if no exception
        // In case of exception, we still need to process at least inst_data[0]
        // (it will be turned to an exception later), so inst_data[0] must be
        // valid.
        Vector#(SupSize,Maybe#(Instruction)) inst_d = replicate(tagged Valid (0));
        if(!isValid(fetch3In.cause)) begin
            if(fetch3In.access_mmio) begin
                if(verbose) $display("get answer from MMIO %d", fetch3In.pc);
                inst_d <- mmio.bootRomResp;
            end
            else begin
                if(verbose) $display("get answer from memory %d", fetch3In.pc);
                inst_d <- mem_server.response.get;
            end
        end
        if(verbose) $display("epoch instr: %d, epoch main : %d", fetch3In.main_epoch, f_main_epoch);
        instdata.enq(inst_d);
        f32d.enq(f22f3.first);
    endrule

    rule doDecode;
        let {nbSup, fetch3In} = f32d.first;
        f32d.deq();
        let inst_data = instdata.first();
        instdata.deq();
        // The main_epoch check is required to make sure this stage doesn't
        // redirect the PC if a later stage already redirected the PC.
        if (fetch3In.main_epoch == f_main_epoch) begin
            Bool decode_epoch_local = decode_epoch; // next value for decode epoch
            Maybe#(Addr) redirectPc = Invalid; // next pc redirect by branch predictor
            Maybe#(TrainNAP) trainNAP = Invalid; // training data sent to next addr pred
`ifdef PERF_COUNT
            // performance counter: inst being redirect by decode stage
            // Note that only 1 redirection may happen in a cycle
            Maybe#(IType) redirectInst = Invalid;
`endif

            for (Integer i = 0; i < valueof(SupSize); i=i+1) begin
                if (inst_data[i] != tagged Invalid && fromInteger(i) <= nbSup) begin
                    // get the input to decode
                    let in = Fetch3ToDecode {
                        pc: fetch3In.pc+fromInteger(4*i),
                        // last inst, next pc may not be pc+4
                        ppc: fromInteger(i) == nbSup ? fetch3In.pred_next_pc :
                                                       fetch3In.pc+fromInteger(4*(i+1)),
                        decode_epoch: fetch3In.decode_epoch,
                        main_epoch: fetch3In.main_epoch,
                        inst: fromMaybe(?,inst_data[i]),
                        cause: fetch3In.cause
                    };
                    let cause = in.cause;
                    if (verbose) $display("Decode %d\n",i);

                    // do decode and branch prediction
                    // Drop here if does not match the decode_epoch.
                    if (in.decode_epoch == decode_epoch_local) begin
                        doAssert(in.main_epoch == f_main_epoch, "main epoch must match");

                        let decode_result = decode(in.inst);

                        // update cause if there was not an early detected exception
                        if (!isValid(cause)) begin
                            cause = decode_result.illegalInst ? tagged Valid IllegalInst : tagged Invalid;
                        end

                        let dInst = decode_result.dInst;
                        let regs = decode_result.regs;
                        DirPredTrainInfo dp_train = ?; // dir pred training bookkeeping

                        // update predicted next pc
                        if (!isValid(cause)) begin
                            // direction predict
                            Bool pred_taken = False;
                            if(dInst.iType == Br) begin
                                let pred_res <- dirPred.pred[i].pred(in.pc);
                                pred_taken = pred_res.taken;
                                dp_train = pred_res.train;
                            end
                            Maybe#(Addr) nextPc = decodeBrPred(in.pc, dInst, pred_taken);

                            // return address stack
                            if (dInst.iType == J && isValid(regs.dst)) begin
                                // function call -- push return address to stack
                                ras.pushAddress[i](in.pc + 4);
                            end
                            else if (dInst.iType == Jr) begin
                                let new_ppc = ras.firstAddress[i];
                                ras.popAddress[i];
                                nextPc = tagged Valid new_ppc;
                            end

                            if(verbose) begin
                                $display("Branch prediction: ", fshow(dInst.iType), " ; ", fshow(in.pc), " ; ",
                                         fshow(in.ppc), " ; ", fshow(pred_taken), " ; ", fshow(nextPc));
                            end

                            // check previous mispred
                            if (nextPc matches tagged Valid .decode_pred_next_pc &&& decode_pred_next_pc != in.ppc) begin
                                if (verbose) $display("ppc and decodeppc :  %h %h", in.ppc, decode_pred_next_pc);
                                decode_epoch_local = !decode_epoch_local;
                                redirectPc = Valid (decode_pred_next_pc); // record redirect next pc
                                in.ppc = decode_pred_next_pc;
                                // train next addr pred when mispredict
                                trainNAP = Valid (TrainNAP {pc: in.pc, nextPc: decode_pred_next_pc});
`ifdef PERF_COUNT
                                // performance stats: record decode redirect
                                doAssert(redirectInst == Invalid, "at most 1 decode redirect per cycle");
                                redirectInst = Valid (dInst.iType);
`endif
                            end
                        end
                        let out = FromFetchStage{pc: in.pc,
                                                 ppc: in.ppc,
                                                 main_epoch: in.main_epoch,
                                                 dpTrain: dp_train,
                                                 inst: in.inst,
                                                 dInst: dInst,
                                                 regs: decode_result.regs,
                                                 cause: cause };
                        out_fifo.enqS[i].enq(out);
                        if (verbose) $display("Decode: ", fshow(out));
                    end
                    else begin
                        if (verbose) $display("Drop decoded within a superscalar");
                        // just drop wrong path instructions
                    end
                end
                else if (inst_data[i] == tagged Invalid && fromInteger(i) <= nbSup) begin
                    // inst num is less than expected; this should not happen
                    // because both I$ and boot rom are aligned to cache line
                    // size.
                    doAssert(False, "Fetched insts not enough");
                end
            end

            // update PC and epoch
            if(redirectPc matches tagged Valid .nextPc) begin
                pc_reg[pc_decode_port] <= nextPc;
            end
            decode_epoch <= decode_epoch_local;
            // send training data for next addr pred
            if(trainNAP matches tagged Valid .x) begin
                napTrainByDec.wset(x);
            end
`ifdef PERF_COUNT
            // performance counter: check whether redirect happens
            if(redirectInst matches tagged Valid .iType &&& doStats) begin
                case(iType)
                    Br: decRedirectBrCnt.incr(1);
                    J : decRedirectJmpCnt.incr(1);
                    Jr: decRedirectJrCnt.incr(1);
                    default: decRedirectOtherCnt.incr(1);
                endcase
            end
`endif
        end
        else begin
            if (verbose) $display("drop in fetch3decode");
        end
    endrule

    // train next addr pred
    (* fire_when_enabled, no_implicit_conditions *)
    rule doTrainNAP(isValid(napTrainByDec.wget) || isValid(napTrainByExe.wget));
        // give priority to train data from exe
        TrainNAP train = fromMaybe(validValue(napTrainByDec.wget), napTrainByExe.wget);
        nextAddrPred.update(train.pc, train.nextPc, train.nextPc != train.pc + 4);
    endrule

    interface Vector pipelines = out_fifo.deqS;
    interface iTlbIfc = iTlb;
    interface iMemIfc = iMem;
    interface mmioIfc = mmio.toCore;

    method Action start(Addr start_pc);
        pc_reg[0] <= start_pc;
        started <= True;
        waitForRedirect <= False;
        waitForFlush <= False;
    endmethod
    method Action stop();
        started <= False;
    endmethod

    method Action setWaitRedirect;
        waitForRedirect <= True;
        setWaitRedirect_redirect_conflict.wset(?); // conflict with redirect
    endmethod
    method Action redirect(Addr new_pc);
        if (verbose) $display("Redirect: newpc %h, old f_main_epoch %d, new f_main_epoch %d",new_pc,f_main_epoch,f_main_epoch+1);
        pc_reg[pc_redirect_port] <= new_pc;
        f_main_epoch <= (f_main_epoch == fromInteger(valueOf(NumEpochs)-1)) ? 0 : f_main_epoch + 1;
        // redirect comes, stop stalling for redirect
        waitForRedirect <= False;
        setWaitRedirect_redirect_conflict.wset(?); // conflict with setWaitForRedirect
        // this redirect may be caused by a trap/system inst in commit stage
        // we conservatively set wait for flush TODO make this an input parameter
        waitForFlush <= True;
    endmethod
    method Action done_flushing() if (waitForFlush);
        // signal that the pipeline can resume fetching
        waitForFlush <= False;
        // XXX The guard prevents the readyToFetch rule in Core.bsv from firing every cycle
        // The guard also makes this method sequence before (restricted) redirect method
        // So the effect of setting waitForFlush in redirect method will not be overwritten
        // Then we don't need to make two methods conflict
        // It's fine for the effect of this method to be overwritten, because it fires very often
    endmethod

    method Action train_predictors(
        Addr pc, Addr next_pc, IType iType, Bool taken,
        DirPredTrainInfo dpTrain, Bool mispred
    );
        //if (iType == J || (iType == Br && next_pc < pc)) begin
        //    // Only train the next address predictor for jumps and backward branches
        //    // next_pc != pc + 4 is a substitute for taken
        //    nextAddrPred.update(pc, next_pc, taken);
        //end
        if (iType == Br) begin
            // Train the direction predictor for all branches
            dirPred.update(pc, taken, dpTrain, mispred);
        end
        // train next addr pred when mispred
        if(mispred) begin
            napTrainByExe.wset(TrainNAP {pc: pc, nextPc: next_pc});
        end
    endmethod

    method FetchDebugState getFetchState;
        return FetchDebugState {
            pc: pc_reg[0],
            waitForRedirect: waitForRedirect,
            waitForFlush: waitForFlush,
            mainEp: f_main_epoch
        };
    endmethod

    interface Perf perf;
        method Action setStatus(Bool stats);
`ifdef PERF_COUNT
            doStats <= stats;
`else
            noAction;
`endif
        endmethod

        method Action req(DecStagePerfType r);
            perfReqQ.enq(r);
        endmethod

        method ActionValue#(PerfResp#(DecStagePerfType)) resp;
`ifdef PERF_COUNT
            perfRespQ.deq;
            return perfRespQ.first;
`else
            perfReqQ.deq;
            return PerfResp {
                pType: perfReqQ.first,
                data: 0
            };
`endif
        endmethod

`ifdef PERF_COUNT
        method Bool respValid = perfRespQ.notEmpty;
`else
        method Bool respValid = perfReqQ.notEmpty;
`endif
    endinterface
endmodule

// Copyright (c) 2017 Massachusetts Institute of Technology
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

`include "ProcConfig.bsv"

import BrPred::*;
import DirPredictor::*;
import Btb::*;
import ClientServer::*;
import Connectable::*;
import Decode::*;
import Ehr::*;
import Fifo::*;
import GetPut::*;
import MemoryTypes::*;
import Types::*;
import ProcTypes::*;
import CCTypes::*;
import Ras::*;
import EpochManager::*;
import Performance::*;
import Vector::*;
import Assert::*;
import Cntrs::*;
import ConfigReg::*;
import TlbTypes::*;
import ITlb::*;
import CCTypes::*;
import L1CoCache::*;
import MMIOInst::*;

interface FetchStage;
    // pipeline
    interface Vector#(SupSize, SupFifoDeq#(FromFetchStage)) pipelines;

    // tlb and mem connections
    interface ITlb iTlbIfc;
    interface ICoCache iMemIfc;
    interface MMIOInstToCore mmioIfc;

    // starting and stopping
    method Action start(Addr pc);
    method Action stop();

    // redirection methods
    method Action setWaitRedirect;
    method Action redirect(Addr pc);
    method Action done_flushing();
    method Action train_predictors(
        Addr pc, Addr next_pc, IType iType, Bool taken,
        DirPredTrainInfo dpTrain, Bool mispred
    );

    // debug
    method FetchDebugState getFetchState;

    // performance
    interface Perf#(DecStagePerfType) perf;
endinterface

typedef struct {
    Addr pc;
    Epoch mainEp;
    Bool waitForRedirect;
    Bool waitForFlush;
} FetchDebugState deriving(Bits, Eq, FShow);

typedef struct {
    Addr pc;
    Addr pred_next_pc;
    Bool decode_epoch;
    Epoch main_epoch;
} Fetch1ToFetch2 deriving(Bits, Eq, FShow);

typedef struct {
    Addr pc;
    Addr phys_pc;
    Addr pred_next_pc;
    Maybe#(Exception) cause;
    Bool access_mmio; // inst fetch from MMIO
    Bool decode_epoch;
    Epoch main_epoch;
} Fetch2ToFetch3 deriving(Bits, Eq, FShow);

typedef struct {
  Addr pc;
  Addr ppc;
  Bool decode_epoch;
  Epoch main_epoch;
  Instruction inst;
  Maybe#(Exception) cause;
} Fetch3ToDecode deriving(Bits, Eq, FShow);

typedef struct {
  Addr pc;
  Addr ppc;
  Epoch main_epoch;
  DirPredTrainInfo dpTrain;
  Instruction inst;
  DecodedInst dInst;
  ArchRegs regs;
  Maybe#(Exception) cause;
} FromFetchStage deriving (Bits, Eq, FShow);

// train next addr pred (BTB)
typedef struct {
    Addr pc;
    Addr nextPc;
} TrainNAP deriving(Bits, Eq, FShow);

(* synthesize *)
module mkFetchStage(FetchStage);
    // rule ordering: Fetch1 (BTB+TLB) < Fetch3 (decode & dir pred) < redirect method
    // Fetch1 < Fetch3 to avoid bypassing path on PC and epochs

    let verbose = True;

    // Basic State Elements
    Reg#(Bool) started <- mkReg(False);

    // Stall fetch when trap happens or system inst is renamed
    // All inst younger than the trap/system inst will be killed
    // Since CSR may be modified, sending wrong path request to TLB may cause problem
    // So we stall until the next redirection happens
    // The next redirect is either by the trap/system inst or an older one
    Reg#(Bool) waitForRedirect <- mkReg(False);
    // We don't want setWaitForRedirect method and redirect method to happen together
    // make them conflict
    RWire#(void) setWaitRedirect_redirect_conflict <- mkRWire;

    // Stall fetch during the flush triggered by the procesing trap/system inst in commit stage
    // We stall until the flush is done
    Reg#(Bool) waitForFlush <- mkReg(False);

    Ehr#(3, Addr) pc_reg <- mkEhr(0);
    Integer pc_fetch1_port = 0;
    Integer pc_decode_port = 1;
    Integer pc_redirect_port = 2;

    // Epochs
    Reg#(Bool) decode_epoch <- mkReg(False);
    Reg#(Epoch) f_main_epoch <- mkReg(0); // fetch estimate of main epoch

    // Pipeline Stage FIFOs
    Fifo#(2, Tuple2#(Bit#(TLog#(SupSize)),Fetch1ToFetch2)) f12f2 <- mkCFFifo;
    Fifo#(4, Tuple2#(Bit#(TLog#(SupSize)),Fetch2ToFetch3)) f22f3 <- mkCFFifo; // FIFO should match I$ latency
    Fifo#(4, Tuple2#(Bit#(TLog#(SupSize)),Fetch2ToFetch3)) f32d <- mkCFFifo; // FIFO should match I$ latency
    SupFifo#(SupSize, 2, Fetch3ToDecode) f32decode <- mkSupFifo;
    SupFifo#(SupSize, 2, FromFetchStage) out_fifo <- mkSupFifo;
       // Can the fifo size be smaller?
    Fifo#(4,Vector#(SupSize,Maybe#(Instruction))) instdata <- mkPipelineFifo();

    // Branch Predictors
    NextAddrPred    nextAddrPred <- mkBtb;
    let             dirPred      <- mkDirPredictor;
    ReturnAddrStack ras          <- mkRas;
    // Wire to train next addr pred (NAP)
    RWire#(TrainNAP) napTrainByDec <- mkRWire;
    RWire#(TrainNAP) napTrainByExe <- mkRWire;

    // TLB and Cache connections
    ITlb iTlb <- mkITlb;
    ICoCache iMem <- mkICoCache;
    MMIOInst mmio <- mkMMIOInst;
    Server#(Addr, TlbResp) tlb_server = iTlb.to_proc;
    Server#(Addr, Vector#(SupSize, Maybe#(Instruction))) mem_server = iMem.to_proc;

    // performance counters
    Fifo#(1, DecStagePerfType) perfReqQ <- mkCFFifo; // perf req FIFO
`ifdef PERF_COUNT
    Reg#(Bool) doStats <- mkConfigReg(False);
    // decode stage redirect
    Count#(Data) decRedirectBrCnt <- mkCount(0);
    Count#(Data) decRedirectJmpCnt <- mkCount(0);
    Count#(Data) decRedirectJrCnt <- mkCount(0);
    Count#(Data) decRedirectOtherCnt <- mkCount(0);
    // perf resp FIFO
    Fifo#(1, PerfResp#(DecStagePerfType)) perfRespQ <- mkCFFifo;

    rule doPerfReq;
        let t <- toGet(perfReqQ).get;
        Data d = (case(t)
            DecRedirectBr: decRedirectBrCnt;
            DecRedirectJmp: decRedirectJmpCnt;
            DecRedirectJr: decRedirectJrCnt;
            DecRedirectOther: decRedirectOtherCnt;
            default: 0;
        endcase);
        perfRespQ.enq(PerfResp {
            pType: t,
            data: d
        });
    endrule
`endif

    // We don't send req to TLB when waiting for redirect or TLB flush. Since
    // there is no FIFO between doFetch1 and TLB, when OOO commit stage wait
    // TLB idle to change VM CSR / signal flush TLB, there is no wrong path
    // request afterwards to race with the system code that manage paget table.
    rule doFetch1(started && !waitForRedirect && !waitForFlush);
        let pc = pc_reg[pc_fetch1_port];

        // Chain of prediction for the next instructions
        // We need a BTB with a register file with enough ports!
        // Instead of cascading predictions, we can always feed pc+4*i into
        // predictor, because we will break superscaler fetch if nextpc != pc+4
        Vector#(SupSize, Addr) pred_future_pc;
        for(Integer i = 0; i < valueof(SupSize); i = i+1) begin
            pred_future_pc[i] = nextAddrPred.predPc(pc + fromInteger(4 * i));
        end

        // Next pc is the first nextPc that breaks the chain of pc+4 or
        // that is at the end of a cacheline.
        Vector#(SupSize,Integer) indexes = genVector;
        function Bool findNextPc(Addr pc, Integer i);
            Bool notLastInst = getLineInstOffset(pc + fromInteger(4*i)) != maxBound;
            Bool noJump = pred_future_pc[i] == pc + fromInteger(4*(i+1));
            return (!(notLastInst && noJump));
        endfunction
        Integer posLastSup = fromMaybe(valueof(SupSize) - 1, find(findNextPc(pc), indexes));
        Addr pred_next_pc = pred_future_pc[posLastSup];
        pc_reg[pc_fetch1_port] <= pred_next_pc;

        // Send TLB request
        tlb_server.request.put(pc);

        let out = Fetch1ToFetch2 {
            pc: pc,
            pred_next_pc: pred_next_pc,
            decode_epoch: decode_epoch,
            main_epoch: f_main_epoch};
        f12f2.enq(tuple2(fromInteger(posLastSup),out));
        if (verbose) $display("Fetch1: ", fshow(out));
    endrule

    rule doFetch2;
        let {nbSup,in} = f12f2.first;
        f12f2.deq;

        // Get TLB response
        match {.phys_pc, .cause} <- tlb_server.response.get;

        // Access main mem or boot rom
        Bool access_mmio = False;
        if (!isValid(cause)) begin
            case(mmio.getFetchTarget(phys_pc))
                MainMem: begin
                    // Send ICache request
                    mem_server.request.put(phys_pc);
                end
                BootRom: begin
                    // Send MMIO req. Luckily boot rom is also aligned with
                    // cache line size, so all nbSup+1 insts can be fetched
                    // from boot rom. It won't happen that insts fetched from
                    // boot rom is less than requested.
                    mmio.bootRomReq(phys_pc, nbSup);
                    access_mmio = True;
                end
                default: begin
                    // Access fault
                    cause = Valid (InstAccessFault);
                end
            endcase
        end

        let out = Fetch2ToFetch3 {
            pc: in.pc,
            phys_pc: phys_pc,
            pred_next_pc: in.pred_next_pc,
            cause: cause,
            access_mmio: access_mmio,
            decode_epoch: in.decode_epoch,
            main_epoch: in.main_epoch };
        f22f3.enq(tuple2(nbSup,out));
        if (verbose) $display("Fetch2: ", fshow(out));
    endrule

// Break out of i$
    rule doFetch3;
        let {nbSup, fetch3In} = f22f3.first;
        f22f3.deq();
        if (verbose) $display("Fetch3 %d",fetch3In.pc);

        // Get ICache/MMIO response if no exception
        // In case of exception, we still need to process at least inst_data[0]
        // (it will be turned to an exception later), so inst_data[0] must be
        // valid.
        Vector#(SupSize,Maybe#(Instruction)) inst_d = replicate(tagged Valid (0));
        if(!isValid(fetch3In.cause)) begin
            if(fetch3In.access_mmio) begin
                if(verbose) $display("get answer from MMIO %d", fetch3In.pc);
                inst_d <- mmio.bootRomResp;
            end
            else begin
                if(verbose) $display("get answer from memory %d", fetch3In.pc);
                inst_d <- mem_server.response.get;
            end
        end
        if(verbose) $display("epoch instr: %d, epoch main : %d", fetch3In.main_epoch, f_main_epoch);
        instdata.enq(inst_d);
        f32d.enq(f22f3.first);
    endrule

    rule doDecode;
        let {nbSup, fetch3In} = f32d.first;
        f32d.deq();
	let inst_data = instdata.first();
	instdata.deq();
        // The main_epoch check is required to make sure this stage doesn't
        // redirect the PC if a later stage already redirected the PC.
        if (fetch3In.main_epoch == f_main_epoch) begin
            Bool decode_epoch_local = decode_epoch; // next value for decode epoch
            Maybe#(Addr) redirectPc = Invalid; // next pc redirect by branch predictor
            Maybe#(TrainNAP) trainNAP = Invalid; // training data sent to next addr pred
`ifdef PERF_COUNT
            // performance counter: inst being redirect by decode stage
            // Note that only 1 redirection may happen in a cycle
            Maybe#(IType) redirectInst = Invalid;
`endif

            for (Integer i = 0; i < valueof(SupSize); i=i+1) begin
                if (inst_data[i] != tagged Invalid && fromInteger(i) <= nbSup) begin
                    // get the input to decode
                    let in = Fetch3ToDecode {
                        pc: fetch3In.pc+fromInteger(4*i),
                        // last inst, next pc may not be pc+4
                        ppc: fromInteger(i) == nbSup ? fetch3In.pred_next_pc :
                                                       fetch3In.pc+fromInteger(4*(i+1)),
                        decode_epoch: fetch3In.decode_epoch,
                        main_epoch: fetch3In.main_epoch,
                        inst: fromMaybe(?,inst_data[i]),
                        cause: fetch3In.cause
                    };
                    let cause = in.cause;
                    if (verbose) $display("Decode %d\n",i);

                    // do decode and branch prediction
                    // Drop here if does not match the decode_epoch.
                    if (in.decode_epoch == decode_epoch_local) begin
                        doAssert(in.main_epoch == f_main_epoch, "main epoch must match");

                        let decode_result = decode(in.inst);

                        // update cause if there was not an early detected exception
                        if (!isValid(cause)) begin
                            cause = decode_result.illegalInst ? tagged Valid IllegalInst : tagged Invalid;
                        end

                        let dInst = decode_result.dInst;
                        let regs = decode_result.regs;
                        DirPredTrainInfo dp_train = ?; // dir pred training bookkeeping

                        // update predicted next pc
                        if (!isValid(cause)) begin
                            // direction predict
                            Bool pred_taken = False;
                            if(dInst.iType == Br) begin
                                let pred_res <- dirPred.pred[i].pred(in.pc);
                                pred_taken = pred_res.taken;
                                dp_train = pred_res.train;
                            end
                            Maybe#(Addr) nextPc = decodeBrPred(in.pc, dInst, pred_taken);

                            // return address stack
                            if (dInst.iType == J && isValid(regs.dst)) begin
                                // function call -- push return address to stack
                                ras.pushAddress[i](in.pc + 4);
                            end
                            else if (dInst.iType == Jr) begin
                                let new_ppc = ras.firstAddress[i];
                                ras.popAddress[i];
                                nextPc = tagged Valid new_ppc;
                            end

                            if(verbose) begin
                                $display("Branch prediction: ", fshow(dInst.iType), " ; ", fshow(in.pc), " ; ",
                                         fshow(in.ppc), " ; ", fshow(pred_taken), " ; ", fshow(nextPc));
                            end

                            // check previous mispred
                            if (nextPc matches tagged Valid .decode_pred_next_pc &&& decode_pred_next_pc != in.ppc) begin
                                if (verbose) $display("ppc and decodeppc :  %h %h", in.ppc, decode_pred_next_pc);
                                decode_epoch_local = !decode_epoch_local;
                                redirectPc = Valid (decode_pred_next_pc); // record redirect next pc
                                in.ppc = decode_pred_next_pc;
                                // train next addr pred when mispredict
                                trainNAP = Valid (TrainNAP {pc: in.pc, nextPc: decode_pred_next_pc});
`ifdef PERF_COUNT
                                // performance stats: record decode redirect
                                doAssert(redirectInst == Invalid, "at most 1 decode redirect per cycle");
                                redirectInst = Valid (dInst.iType);
`endif
                            end
                        end
                        let out = FromFetchStage{pc: in.pc,
                                                 ppc: in.ppc,
                                                 main_epoch: in.main_epoch,
                                                 dpTrain: dp_train,
                                                 inst: in.inst,
                                                 dInst: dInst,
                                                 regs: decode_result.regs,
                                                 cause: cause };
                        out_fifo.enqS[i].enq(out);
                        if (verbose) $display("Decode: ", fshow(out));
                    end
                    else begin
                        if (verbose) $display("Drop decoded within a superscalar");
                        // just drop wrong path instructions
                    end
                end
                else if (inst_data[i] == tagged Invalid && fromInteger(i) <= nbSup) begin
                    // inst num is less than expected; this should not happen
                    // because both I$ and boot rom are aligned to cache line
                    // size.
                    doAssert(False, "Fetched insts not enough");
                end
            end

            // update PC and epoch
            if(redirectPc matches tagged Valid .nextPc) begin
                pc_reg[pc_decode_port] <= nextPc;
            end
            decode_epoch <= decode_epoch_local;
            // send training data for next addr pred
            if(trainNAP matches tagged Valid .x) begin
                napTrainByDec.wset(x);
            end
`ifdef PERF_COUNT
            // performance counter: check whether redirect happens
            if(redirectInst matches tagged Valid .iType &&& doStats) begin
                case(iType)
                    Br: decRedirectBrCnt.incr(1);
                    J : decRedirectJmpCnt.incr(1);
                    Jr: decRedirectJrCnt.incr(1);
                    default: decRedirectOtherCnt.incr(1);
                endcase
            end
`endif
        end
        else begin
            if (verbose) $display("drop in fetch3decode");
        end
    endrule

    // train next addr pred
    (* fire_when_enabled, no_implicit_conditions *)
    rule doTrainNAP(isValid(napTrainByDec.wget) || isValid(napTrainByExe.wget));
        // give priority to train data from exe
        TrainNAP train = fromMaybe(validValue(napTrainByDec.wget), napTrainByExe.wget);
        nextAddrPred.update(train.pc, train.nextPc, train.nextPc != train.pc + 4);
    endrule

    interface Vector pipelines = out_fifo.deqS;
    interface iTlbIfc = iTlb;
    interface iMemIfc = iMem;
    interface mmioIfc = mmio.toCore;

    method Action start(Addr start_pc);
        pc_reg[0] <= start_pc;
        started <= True;
        waitForRedirect <= False;
        waitForFlush <= False;
    endmethod
    method Action stop();
        started <= False;
    endmethod

    method Action setWaitRedirect;
        waitForRedirect <= True;
        setWaitRedirect_redirect_conflict.wset(?); // conflict with redirect
    endmethod
    method Action redirect(Addr new_pc);
        if (verbose) $display("Redirect: newpc %h, old f_main_epoch %d, new f_main_epoch %d",new_pc,f_main_epoch,f_main_epoch+1);
        pc_reg[pc_redirect_port] <= new_pc;
        f_main_epoch <= (f_main_epoch == fromInteger(valueOf(NumEpochs)-1)) ? 0 : f_main_epoch + 1;
        // redirect comes, stop stalling for redirect
        waitForRedirect <= False;
        setWaitRedirect_redirect_conflict.wset(?); // conflict with setWaitForRedirect
        // this redirect may be caused by a trap/system inst in commit stage
        // we conservatively set wait for flush TODO make this an input parameter
        waitForFlush <= True;
    endmethod
    method Action done_flushing() if (waitForFlush);
        // signal that the pipeline can resume fetching
        waitForFlush <= False;
        // XXX The guard prevents the readyToFetch rule in Core.bsv from firing every cycle
        // The guard also makes this method sequence before (restricted) redirect method
        // So the effect of setting waitForFlush in redirect method will not be overwritten
        // Then we don't need to make two methods conflict
        // It's fine for the effect of this method to be overwritten, because it fires very often
    endmethod

    method Action train_predictors(
        Addr pc, Addr next_pc, IType iType, Bool taken,
        DirPredTrainInfo dpTrain, Bool mispred
    );
        //if (iType == J || (iType == Br && next_pc < pc)) begin
        //    // Only train the next address predictor for jumps and backward branches
        //    // next_pc != pc + 4 is a substitute for taken
        //    nextAddrPred.update(pc, next_pc, taken);
        //end
        if (iType == Br) begin
            // Train the direction predictor for all branches
            dirPred.update(pc, taken, dpTrain, mispred);
        end
        // train next addr pred when mispred
        if(mispred) begin
            napTrainByExe.wset(TrainNAP {pc: pc, nextPc: next_pc});
        end
    endmethod

    method FetchDebugState getFetchState;
        return FetchDebugState {
            pc: pc_reg[0],
            waitForRedirect: waitForRedirect,
            waitForFlush: waitForFlush,
            mainEp: f_main_epoch
        };
    endmethod

    interface Perf perf;
        method Action setStatus(Bool stats);
`ifdef PERF_COUNT
            doStats <= stats;
`else
            noAction;
`endif
        endmethod

        method Action req(DecStagePerfType r);
            perfReqQ.enq(r);
        endmethod

        method ActionValue#(PerfResp#(DecStagePerfType)) resp;
`ifdef PERF_COUNT
            perfRespQ.deq;
            return perfRespQ.first;
`else
            perfReqQ.deq;
            return PerfResp {
                pType: perfReqQ.first,
                data: 0
            };
`endif
        endmethod

`ifdef PERF_COUNT
        method Bool respValid = perfRespQ.notEmpty;
`else
        method Bool respValid = perfReqQ.notEmpty;
`endif
    endinterface
endmodule