use atomic maybeDecrement

use atomic maybeDecrement

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import Vector::*;
import FIFOF::*;
import FIFO::*;
import GetPut::*;
import ClientServer::*;
import BRAM::*;
import BRAMFIFO::*;
import Connectable::*;

import ConfigCounter::*;
import ConnectalMemory::*;
import MemTypes::*;
import Pipe::*;
import MemUtils::*;

module mkMemwriteEngine(MemwriteEngine#(dataWidth, cmdQDepth, numServers))
   provisos( Mul#(TDiv#(dataWidth, 8), 8, dataWidth)
	    ,Add#(1, a__, numServers)
	    ,Add#(b__, TLog#(numServers), TAdd#(1, TLog#(TMul#(cmdQDepth,numServers))))
	    ,Pipe::FunnelPipesPipelined#(1, numServers,Tuple2#(Bit#(TLog#(numServers)), MemTypes::MemengineCmd), TMin#(2,TLog#(numServers)))
	    ,Pipe::FunnelPipesPipelined#(1, numServers,Tuple2#(Bit#(dataWidth),Bool),TMin#(2, TLog#(numServers)))
	    ,Add#(c__, TLog#(numServers), TLog#(TMul#(cmdQDepth, numServers)))
	    ,Add#(1, d__, dataWidth)
	    ,FunnelPipesPipelined#(1, numServers, Tuple3#(Bit#(2),Bit#(dataWidth),Bool), TMin#(2, TLog#(numServers)))
	    ,FunnelPipesPipelined#(1, numServers,Tuple3#(Bit#(TLog#(numServers)), Bit#(dataWidth), Bool), TMin#(2,TLog#(numServers)))
	    ,Add#(e__, TLog#(numServers), 6)
	    );
   let rv <- mkMemwriteEngineBuff(valueOf(TExp#(BurstLenSize)));
   return rv;
endmodule

interface BurstFunnel#(numeric type k, numeric type w);
   method Action loadIdx(Bit#(TLog#(k)) i);
   interface Vector#(k, PipeIn#(Bit#(w))) dataIn;
   interface Vector#(k, Reg#(Bit#(BurstLenSize))) burstLen;
   interface PipeOut#(Tuple2#(Bit#(TLog#(k)),Bit#(w))) dataOut;
endinterface

module mkBurstFunnel#(Integer maxBurstLen)(BurstFunnel#(k,w))
   provisos( Log#(k,logk)
	    ,Min#(2,logk,bpc)
	    ,FunnelPipesPipelined#(1, k, Tuple3#(Bit#(2),Bit#(w),Bool), bpc)
	    );

   Reg#(Bit#(2)) nameGen <- mkReg(0);
   UGBramFifos#(4,16,Tuple2#(Bit#(w),Bool)) complBuff <- mkUGBramFifos;
   Vector#(4, ConfigCounter#(16)) compCnts <- replicateM(mkConfigCounter(0));
   Vector#(k,FIFOF#(Tuple3#(Bit#(2), Bit#(w), Bool))) data_in <- replicateM(mkFIFOF);
   Vector#(k,Reg#(Bit#(BurstLenSize))) burst_len <- replicateM(mkReg(0));
   Vector#(k,Reg#(Bit#(BurstLenSize))) drain_cnt <- replicateM(mkReg(0));
   Reg#(Bit#(BurstLenSize)) inj_ctrl <- mkReg(0);
   FIFO#(Tuple2#(Bit#(TAdd#(1,logk)),Bit#(2))) loadIdxs <- mkSizedBRAMFIFO(32);
   FIFO#(Tuple2#(Bit#(TAdd#(1,logk)),Bit#(2))) inFlight <- mkSizedBRAMFIFO(4);
   FunnelPipe#(1, k, Tuple3#(Bit#(2),Bit#(w),Bool),bpc) data_in_funnel <- mkFunnelPipesPipelined(map(toPipeOut,data_in));
   Reg#(Bit#(BurstLenSize)) drainCnt <- mkReg(0);
   FIFOF#(Tuple2#(Bit#(TLog#(k)),Bit#(w))) exit_data <- mkFIFOF;
   FIFO#(Bit#(logk)) drainRename <- mkFIFO;

   Reg#(Bit#(32)) cycle <- mkReg(0);
   Reg#(Bit#(32)) last_entry <- mkReg(0);

   rule cyc;
      cycle <= cycle+1;
   endrule

   function PipeIn#(Bit#(w)) enter_data(FIFOF#(Tuple3#(Bit#(2), Bit#(w), Bool)) f, Integer i) =
      (interface PipeIn;
   	  method Bool notFull = f.notFull;
   	  method Action enq(Bit#(w) v) if (tpl_1(loadIdxs.first) == fromInteger(i));
	     last_entry <= cycle;
	     match {.old_name, .new_name} = loadIdxs.first;
	     let first = inj_ctrl == 0;
	     let cnt = first ? burst_len[i] : inj_ctrl;
	     let new_cnt = cnt-1;
	     let last = new_cnt == 0;
	     inj_ctrl <= new_cnt;
	     if (first)
		inFlight.enq(loadIdxs.first);
	     if (last)
		loadIdxs.deq;
	     f.enq(tuple3(new_name, v, last));
	     //$display("%d enq %d", cycle-last_entry, i);
	  endmethod
       endinterface);
   Vector#(k, PipeIn#(Bit#(w))) data_in_pipes = zipWith(enter_data, data_in, genVector);

   function Reg#(Bit#(BurstLenSize)) check(Reg#(Bit#(BurstLenSize)) r) =
      (interface Reg;
	  method Action _write(Bit#(BurstLenSize) v);
	     if(v > 16) begin
		$display("ERROR mkBurstFunnel: burstLen too large");
		$finish;
	     end
	     r <= v;
	  endmethod
	  method Bit#(BurstLenSize) _read = r._read;
       endinterface);

   rule drain_funnel;
      match{.new_name,.data,.last} = data_in_funnel[0].first;
      data_in_funnel[0].deq;
      complBuff.enq(new_name,tuple2(data,last));
      compCnts[new_name].increment(1);
   endrule

   rule drain_req (compCnts[tpl_2(inFlight.first)].read > 0);
      match {.old_name, .new_name} = inFlight.first;
      let new_drainCnt = drainCnt-1;
      if (drainCnt == 0) begin
	 new_drainCnt = burst_len[old_name]-1;
	 drainRename.enq(truncate(old_name));
      end
      if (new_drainCnt == 0) begin
	 inFlight.deq;
      end
      complBuff.first_req(new_name);
      drainCnt <= new_drainCnt;
      compCnts[new_name].decrement(1);
      complBuff.deq(new_name);
   endrule

   rule drain_resp;
      match {.data,.last} <- complBuff.first_resp;
      if (last)
	 drainRename.deq;
      exit_data.enq(tuple2(drainRename.first,data));
   endrule

   method Action loadIdx(Bit#(logk) idx);
      loadIdxs.enq(tuple2(extend(idx),nameGen));
      nameGen <= nameGen+1;
   endmethod
   interface burstLen = map(check,burst_len);
   interface dataIn = data_in_pipes;
   interface PipeOut dataOut = toPipeOut(exit_data);
endmodule

module mkMemwriteEngineBuff#(Integer bufferSizeBytes)(MemwriteEngine#(dataWidth, cmdQDepth, numServers))
   provisos ( Div#(dataWidth,8,dataWidthBytes)
	     ,Mul#(dataWidthBytes,8,dataWidth)
	     ,Log#(dataWidthBytes,beatShift)
	     ,Log#(cmdQDepth,logCmdQDepth)
	     ,Mul#(cmdQDepth,numServers,cmdBuffSz)
	     ,Log#(cmdBuffSz, cmdBuffAddrSz)
	     ,Log#(numServers, serverIdxSz)
	     ,Add#(1,logCmdQDepth, outCntSz)
	     ,Add#(1, c__, numServers)
	     ,Add#(b__, TLog#(numServers), cmdBuffAddrSz)
	     ,Add#(e__, TLog#(numServers), TAdd#(1, cmdBuffAddrSz))
	     ,Add#(a__, serverIdxSz, cmdBuffAddrSz)
	     ,Min#(2,TLog#(numServers),bpc)
	     ,FunnelPipesPipelined#(1,numServers,Tuple2#(Bit#(serverIdxSz),MemengineCmd),bpc)
	     ,FunnelPipesPipelined#(1,numServers,Tuple2#(Bit#(dataWidth),Bool),bpc)
	     ,FunnelPipesPipelined#(1, numServers, Tuple3#(Bit#(2),Bit#(dataWidth),Bool), TMin#(2, serverIdxSz))
	     ,Add#(1, d__, dataWidth)
	     ,FunnelPipesPipelined#(1, numServers, Tuple3#(Bit#(serverIdxSz),Bit#(dataWidth), Bool), TMin#(2, serverIdxSz))
	     ,Add#(f__, TLog#(numServers), TAdd#(1, serverIdxSz))
	     ,Add#(g__, serverIdxSz, 6)
	     );

   Integer bufferSizeBeats = bufferSizeBytes/valueOf(dataWidthBytes);
   Vector#(numServers, Reg#(Bit#(outCntSz)))     outs1 <- replicateM(mkReg(0));
   Vector#(numServers, Reg#(Bit#(outCntSz)))     outs0 <- replicateM(mkReg(0));
   Vector#(numServers, ConfigCounter#(16))           buffCap <- replicateM(mkConfigCounter(0));
   UGBramFifos#(numServers,cmdQDepth,MemengineCmd) cmdBuf <- mkUGBramFifos;

   Reg#(Bool) load_in_progress <- mkReg(False);
   FIFO#(Tuple3#(MemengineCmd,Bool,Bool))         loadf_b <- mkSizedFIFO(1);
   FIFO#(Tuple2#(Bit#(serverIdxSz),MemengineCmd)) loadf_c <- mkSizedFIFO(1);
   FIFO#(Tuple3#(Bit#(8),Bit#(MemTagSize),Bool))    workf <- mkSizedBRAMFIFO(32); // is this the right size?
   FIFO#(Tuple2#(Bit#(serverIdxSz),Bool))           donef <- mkSizedBRAMFIFO(32); // is this the right size?

   Vector#(numServers, FIFO#(Bool))              outfs <- replicateM(mkSizedFIFO(1));
   Vector#(numServers, FIFOF#(Tuple2#(Bit#(serverIdxSz), MemengineCmd))) cmds_in <- replicateM(mkSizedFIFOF(1));
   FunnelPipe#(1, numServers, Tuple2#(Bit#(serverIdxSz), MemengineCmd),bpc) cmds_in_funnel <- mkFunnelPipesPipelined(map(toPipeOut,cmds_in));
   Vector#(numServers, FIFOF#(Bit#(dataWidth)))  write_data_buffs <- replicateM(mkSizedBRAMFIFOF(bufferSizeBeats));
   Vector#(numServers, PipeOut#(Bit#(dataWidth))) foo = map(toPipeOut, write_data_buffs);
   BurstFunnel#(numServers,dataWidth) write_data_funnel <- mkBurstFunnel(bufferSizeBeats);
   zipWithM(mkConnection, foo, write_data_funnel.dataIn);

   Reg#(Bit#(8))                    respCnt <- mkReg(0);
   Reg#(Bit#(TAdd#(1,serverIdxSz))) loadIdx <- mkReg(0);
   let beat_shift = fromInteger(valueOf(beatShift));
   let cmd_q_depth = fromInteger(valueOf(cmdQDepth));

   function Action incr_loadIdx =
      (action
       if(loadIdx+1 >= fromInteger(valueOf(numServers)))
	  loadIdx <= 0;
       else
	  loadIdx <= loadIdx+1;
       endaction);

   rule store_cmd;
      match {.idx, .cmd} <- toGet(cmds_in_funnel[0]).get;
      outs1[idx] <= outs1[idx]+1;
      cmdBuf.enq(idx,cmd);
      //$display("store_cmd %d", idx);
   endrule

   rule load_ctxt_a (!load_in_progress);
      if (outs1[loadIdx] > 0) begin
	 load_in_progress <= True;
	 cmdBuf.first_req(truncate(loadIdx));
      end
      else begin
	 incr_loadIdx;
      end
   endrule

   rule load_ctxt_b;
      let cmd <- cmdBuf.first_resp;
      let cond0 <- buffCap[loadIdx].maybeDecrement(unpack(extend(cmd.burstLen>>beat_shift)));
      let cond1 = cmd.len <= extend(cmd.burstLen);
      loadf_b.enq(tuple3(cmd,cond0,cond1));
   endrule

   rule load_ctxt_c;
      load_in_progress <= False;
      incr_loadIdx;
      match {.cmd,.cond0,.cond1} <- toGet(loadf_b).get;
      if  (cond0) begin
	 //$display("load_ctxt_b %h %d", cmd.base, idx);
	 let x = cmd.burstLen;
	 if (cmd.len < extend(cmd.burstLen))
	    x = truncate(cmd.len);
	 loadf_c.enq(tuple2(truncate(loadIdx),cmd));
	 write_data_funnel.loadIdx(truncate(loadIdx));
	 if (cond1) begin
	    outs1[loadIdx] <= outs1[loadIdx]-1;
	    cmdBuf.deq(truncate(loadIdx));
	 end
	 else begin
	    let new_cmd = MemengineCmd{sglId:cmd.sglId, base:cmd.base+extend(cmd.burstLen), burstLen:cmd.burstLen, len:cmd.len-extend(cmd.burstLen), tag:cmd.tag};
	    cmdBuf.upd_head(truncate(loadIdx),new_cmd);
	 end
      end
   endrule

   function PipeIn#(Bit#(w)) check_in(FIFOF#(Bit#(w)) f, Integer i) =
      (interface PipeIn;
   	  method Bool notFull = f.notFull;
   	  method Action enq(Bit#(w) v);
	     f.enq(v);
	     buffCap[i].increment(1);
	     //$display("check_in %d", i);
	     // if(i==2)
	     // 	for(Integer j = 0; j < valueOf(w); j=j+32) begin
	     // 	   Bit#(32) xx = v[j+31:j];
	     // 	   $display("%h", xx);
	     // 	end
	  endmethod
       endinterface);

   function MemwriteServer#(dataWidth) toMemwriteServer(Server#(MemengineCmd,Bool) cs, PipeIn#(Bit#(dataWidth)) p) =
      (interface MemwriteServer;
	  interface cmdServer = cs;
	  interface dataPipe  = p;
       endinterface);

   Vector#(numServers, Server#(MemengineCmd,Bool)) rs;
   for(Integer i = 0; i < valueOf(numServers); i=i+1)
      rs[i] = (interface Server#(MemengineCmd,Bool);
		  interface Put request;
		     method Action put(MemengineCmd c) if (outs0[i] < cmd_q_depth);
			Bit#(32) bsb = fromInteger(bufferSizeBytes);
`ifdef BSIM
			Bit#(32) dw = fromInteger(valueOf(dataWidthBytes));
			Bit#(32) bl = extend(c.burstLen);
			// this is because bsc lifts the divide operation (below)
			// and on startup the simulator gets a floating-point exception
	  		if (bl ==0)
			   bl = 1;
			let mdw0 = ((c.len)/bl)*bl != c.len;
			let mdw1 = ((c.len)/dw)*dw != c.len;
			let bbl = extend(c.burstLen) > bsb;
			if(bbl || mdw0 || mdw1 || c.len == 0) begin
			   if (bbl)
			      $display("XXXXXXXXXX mkMemwriteEngineBuff::unsupported burstLen %d %d", bsb, c.burstLen);
			   if (mdw0 || mdw1 || c.len == 0)
			      $display("XXXXXXXXXX mkMemwriteEngineBuff::unsupported len %h mdw0=%d mdw1=%d", c.len, mdw0, mdw1);
			end
			else begin
`endif
			   outs0[i] <= outs0[i]+1;
			   cmds_in[i].enq(tuple2(fromInteger(i),c));
			   write_data_funnel.burstLen[i] <= c.burstLen >> beat_shift;
			   //$display("(%d) %h %h %h", i, c.base, c.len, c.burstLen);
`ifdef BSIM
			end
`endif
 		     endmethod
		  endinterface
		  interface Get response;
		     method ActionValue#(Bool) get;
			let rv <- toGet(outfs[i]).get;
	 		outs0[i] <= outs0[i]-1;
			return rv;
		     endmethod
		  endinterface
	       endinterface);
   interface writeServers = rs;
   interface MemWriteClient dmaClient;
      interface Get writeReq;
	 method ActionValue#(MemRequest) get();
	    match {.idx, .cmd} <- toGet(loadf_c).get;
	    Bit#(BurstLenSize) bl = cmd.burstLen;
	    Bool last = False;
	    if (cmd.len <= extend(bl)) begin
	       last = True;
	       bl = truncate(cmd.len);
	    end
	    let new_tag = (cmd.tag << valueOf(serverIdxSz)) | extend(idx);
	    workf.enq(tuple3(truncate(bl>>beat_shift), new_tag, last));
	    //$display("writeReq %d, %h %h %h", idx, cmd.base, bl, last);
	    return MemRequest { sglId: cmd.sglId, offset: cmd.base, burstLen:bl, tag: new_tag};
	 endmethod
      endinterface
      interface Get writeData;
	 method ActionValue#(MemData#(dataWidth)) get;
	    match {.rc, .new_tag, .last} = workf.first;
	    Bit#(serverIdxSz) idx = truncate(new_tag);
	    let new_respCnt = respCnt+1;
	    if (new_respCnt == rc) begin
	       respCnt <= 0;
	       workf.deq;
	       donef.enq(tuple2(idx,last));
	    end
	    else begin
	       respCnt <= new_respCnt;
	    end
	    match {._idx, .wd} <- toGet(write_data_funnel.dataOut).get;
	    if(_idx != idx) begin
	       $display("ERROR mkMemwriteEngineBuf: bursts oo %d %d", idx, _idx);
	       $finish;
	    end
	    // TODO: this last field should be set correctly to mark the end of bursts
	    return MemData{data:wd, tag:new_tag, last:False};
	 endmethod
      endinterface
      interface Put writeDone;
	 method Action put(Bit#(MemTagSize) tag);
	    match {.idx, .last} <- toGet(donef).get;
	    if (last)
	       outfs[idx].enq(True);
	    //$display("writeDone %d %d", idx, last);
	 endmethod
      endinterface
   endinterface
   interface dataPipes = zipWith(check_in, write_data_buffs, genVector);
   interface write_servers = zipWith(toMemwriteServer, rs, zipWith(check_in, write_data_buffs, genVector));
endmodule

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import Vector::*;
import FIFOF::*;
import FIFO::*;
import GetPut::*;
import ClientServer::*;
import BRAM::*;
import BRAMFIFO::*;
import Connectable::*;

import ConfigCounter::*;
import ConnectalMemory::*;
import MemTypes::*;
import Pipe::*;
import MemUtils::*;

module mkMemwriteEngine(MemwriteEngine#(dataWidth, cmdQDepth, numServers))
   provisos( Mul#(TDiv#(dataWidth, 8), 8, dataWidth)
	    ,Add#(1, a__, numServers)
	    ,Add#(b__, TLog#(numServers), TAdd#(1, TLog#(TMul#(cmdQDepth,numServers))))
	    ,Pipe::FunnelPipesPipelined#(1, numServers,Tuple2#(Bit#(TLog#(numServers)), MemTypes::MemengineCmd), TMin#(2,TLog#(numServers)))
	    ,Pipe::FunnelPipesPipelined#(1, numServers,Tuple2#(Bit#(dataWidth),Bool),TMin#(2, TLog#(numServers)))
	    ,Add#(c__, TLog#(numServers), TLog#(TMul#(cmdQDepth, numServers)))
	    ,Add#(1, d__, dataWidth)
	    ,FunnelPipesPipelined#(1, numServers, Tuple3#(Bit#(2),Bit#(dataWidth),Bool), TMin#(2, TLog#(numServers)))
	    ,FunnelPipesPipelined#(1, numServers,Tuple3#(Bit#(TLog#(numServers)), Bit#(dataWidth), Bool), TMin#(2,TLog#(numServers)))
	    ,Add#(e__, TLog#(numServers), 6)
	    );
   let rv <- mkMemwriteEngineBuff(valueOf(TExp#(BurstLenSize)));
   return rv;
endmodule

interface BurstFunnel#(numeric type k, numeric type w);
   method Action loadIdx(Bit#(TLog#(k)) i);
   interface Vector#(k, PipeIn#(Bit#(w))) dataIn;
   interface Vector#(k, Reg#(Bit#(BurstLenSize))) burstLen;
   interface PipeOut#(Tuple2#(Bit#(TLog#(k)),Bit#(w))) dataOut;
endinterface

module mkBurstFunnel#(Integer maxBurstLen)(BurstFunnel#(k,w))
   provisos( Log#(k,logk)
	    ,Min#(2,logk,bpc)
	    ,FunnelPipesPipelined#(1, k, Tuple3#(Bit#(2),Bit#(w),Bool), bpc)
	    );

   Reg#(Bit#(2)) nameGen <- mkReg(0);
   UGBramFifos#(4,16,Tuple2#(Bit#(w),Bool)) complBuff <- mkUGBramFifos;
   Vector#(4, ConfigCounter#(16)) compCnts <- replicateM(mkConfigCounter(0));
   Vector#(k,FIFOF#(Tuple3#(Bit#(2), Bit#(w), Bool))) data_in <- replicateM(mkFIFOF);
   Vector#(k,Reg#(Bit#(BurstLenSize))) burst_len <- replicateM(mkReg(0));
   Vector#(k,Reg#(Bit#(BurstLenSize))) drain_cnt <- replicateM(mkReg(0));
   Reg#(Bit#(BurstLenSize)) inj_ctrl <- mkReg(0);
   FIFO#(Tuple2#(Bit#(TAdd#(1,logk)),Bit#(2))) loadIdxs <- mkSizedBRAMFIFO(32);
   FIFO#(Tuple2#(Bit#(TAdd#(1,logk)),Bit#(2))) inFlight <- mkSizedBRAMFIFO(4);
   FunnelPipe#(1, k, Tuple3#(Bit#(2),Bit#(w),Bool),bpc) data_in_funnel <- mkFunnelPipesPipelined(map(toPipeOut,data_in));
   Reg#(Bit#(BurstLenSize)) drainCnt <- mkReg(0);
   FIFOF#(Tuple2#(Bit#(TLog#(k)),Bit#(w))) exit_data <- mkFIFOF;
   FIFO#(Bit#(logk)) drainRename <- mkFIFO;

   Reg#(Bit#(32)) cycle <- mkReg(0);
   Reg#(Bit#(32)) last_entry <- mkReg(0);

   rule cyc;
      cycle <= cycle+1;
   endrule

   function PipeIn#(Bit#(w)) enter_data(FIFOF#(Tuple3#(Bit#(2), Bit#(w), Bool)) f, Integer i) =
      (interface PipeIn;
   	  method Bool notFull = f.notFull;
   	  method Action enq(Bit#(w) v) if (tpl_1(loadIdxs.first) == fromInteger(i));
	     last_entry <= cycle;
	     match {.old_name, .new_name} = loadIdxs.first;
	     let first = inj_ctrl == 0;
	     let cnt = first ? burst_len[i] : inj_ctrl;
	     let new_cnt = cnt-1;
	     let last = new_cnt == 0;
	     inj_ctrl <= new_cnt;
	     if (first)
		inFlight.enq(loadIdxs.first);
	     if (last)
		loadIdxs.deq;
	     f.enq(tuple3(new_name, v, last));
	     //$display("%d enq %d", cycle-last_entry, i);
	  endmethod
       endinterface);
   Vector#(k, PipeIn#(Bit#(w))) data_in_pipes = zipWith(enter_data, data_in, genVector);

   function Reg#(Bit#(BurstLenSize)) check(Reg#(Bit#(BurstLenSize)) r) =
      (interface Reg;
	  method Action _write(Bit#(BurstLenSize) v);
	     if(v > 16) begin
		$display("ERROR mkBurstFunnel: burstLen too large");
		$finish;
	     end
	     r <= v;
	  endmethod
	  method Bit#(BurstLenSize) _read = r._read;
       endinterface);

   rule drain_funnel;
      match{.new_name,.data,.last} = data_in_funnel[0].first;
      data_in_funnel[0].deq;
      complBuff.enq(new_name,tuple2(data,last));
      compCnts[new_name].increment(1);
   endrule

   rule drain_req (compCnts[tpl_2(inFlight.first)].read > 0);
      match {.old_name, .new_name} = inFlight.first;
      let new_drainCnt = drainCnt-1;
      if (drainCnt == 0) begin
	 new_drainCnt = burst_len[old_name]-1;
	 drainRename.enq(truncate(old_name));
      end
      if (new_drainCnt == 0) begin
	 inFlight.deq;
      end
      complBuff.first_req(new_name);
      drainCnt <= new_drainCnt;
      compCnts[new_name].decrement(1);
      complBuff.deq(new_name);
   endrule

   rule drain_resp;
      match {.data,.last} <- complBuff.first_resp;
      if (last)
	 drainRename.deq;
      exit_data.enq(tuple2(drainRename.first,data));
   endrule

   method Action loadIdx(Bit#(logk) idx);
      loadIdxs.enq(tuple2(extend(idx),nameGen));
      nameGen <= nameGen+1;
   endmethod
   interface burstLen = map(check,burst_len);
   interface dataIn = data_in_pipes;
   interface PipeOut dataOut = toPipeOut(exit_data);
endmodule

module mkMemwriteEngineBuff#(Integer bufferSizeBytes)(MemwriteEngine#(dataWidth, cmdQDepth, numServers))
   provisos ( Div#(dataWidth,8,dataWidthBytes)
	     ,Mul#(dataWidthBytes,8,dataWidth)
	     ,Log#(dataWidthBytes,beatShift)
	     ,Log#(cmdQDepth,logCmdQDepth)
	     ,Mul#(cmdQDepth,numServers,cmdBuffSz)
	     ,Log#(cmdBuffSz, cmdBuffAddrSz)
	     ,Log#(numServers, serverIdxSz)
	     ,Add#(1,logCmdQDepth, outCntSz)
	     ,Add#(1, c__, numServers)
	     ,Add#(b__, TLog#(numServers), cmdBuffAddrSz)
	     ,Add#(e__, TLog#(numServers), TAdd#(1, cmdBuffAddrSz))
	     ,Add#(a__, serverIdxSz, cmdBuffAddrSz)
	     ,Min#(2,TLog#(numServers),bpc)
	     ,FunnelPipesPipelined#(1,numServers,Tuple2#(Bit#(serverIdxSz),MemengineCmd),bpc)
	     ,FunnelPipesPipelined#(1,numServers,Tuple2#(Bit#(dataWidth),Bool),bpc)
	     ,FunnelPipesPipelined#(1, numServers, Tuple3#(Bit#(2),Bit#(dataWidth),Bool), TMin#(2, serverIdxSz))
	     ,Add#(1, d__, dataWidth)
	     ,FunnelPipesPipelined#(1, numServers, Tuple3#(Bit#(serverIdxSz),Bit#(dataWidth), Bool), TMin#(2, serverIdxSz))
	     ,Add#(f__, TLog#(numServers), TAdd#(1, serverIdxSz))
	     ,Add#(g__, serverIdxSz, 6)
	     );

   Integer bufferSizeBeats = bufferSizeBytes/valueOf(dataWidthBytes);
   Vector#(numServers, Reg#(Bit#(outCntSz)))     outs1 <- replicateM(mkReg(0));
   Vector#(numServers, Reg#(Bit#(outCntSz)))     outs0 <- replicateM(mkReg(0));
   Vector#(numServers, ConfigCounter#(16))           buffCap <- replicateM(mkConfigCounter(0));
   UGBramFifos#(numServers,cmdQDepth,MemengineCmd) cmdBuf <- mkUGBramFifos;

   Reg#(Bool) load_in_progress <- mkReg(False);
   FIFO#(Tuple3#(MemengineCmd,Bool,Bool))         loadf_b <- mkSizedFIFO(1);
   FIFO#(Tuple2#(Bit#(serverIdxSz),MemengineCmd)) loadf_c <- mkSizedFIFO(1);
   FIFO#(Tuple3#(Bit#(8),Bit#(MemTagSize),Bool))    workf <- mkSizedBRAMFIFO(32); // is this the right size?
   FIFO#(Tuple2#(Bit#(serverIdxSz),Bool))           donef <- mkSizedBRAMFIFO(32); // is this the right size?

   Vector#(numServers, FIFO#(Bool))              outfs <- replicateM(mkSizedFIFO(1));
   Vector#(numServers, FIFOF#(Tuple2#(Bit#(serverIdxSz), MemengineCmd))) cmds_in <- replicateM(mkSizedFIFOF(1));
   FunnelPipe#(1, numServers, Tuple2#(Bit#(serverIdxSz), MemengineCmd),bpc) cmds_in_funnel <- mkFunnelPipesPipelined(map(toPipeOut,cmds_in));
   Vector#(numServers, FIFOF#(Bit#(dataWidth)))  write_data_buffs <- replicateM(mkSizedBRAMFIFOF(bufferSizeBeats));
   Vector#(numServers, PipeOut#(Bit#(dataWidth))) foo = map(toPipeOut, write_data_buffs);
   BurstFunnel#(numServers,dataWidth) write_data_funnel <- mkBurstFunnel(bufferSizeBeats);
   zipWithM(mkConnection, foo, write_data_funnel.dataIn);

   Reg#(Bit#(8))                    respCnt <- mkReg(0);
   Reg#(Bit#(TAdd#(1,serverIdxSz))) loadIdx <- mkReg(0);
   let beat_shift = fromInteger(valueOf(beatShift));
   let cmd_q_depth = fromInteger(valueOf(cmdQDepth));

   function Action incr_loadIdx =
      (action
       if(loadIdx+1 >= fromInteger(valueOf(numServers)))
	  loadIdx <= 0;
       else
	  loadIdx <= loadIdx+1;
       endaction);

   rule store_cmd;
      match {.idx, .cmd} <- toGet(cmds_in_funnel[0]).get;
      outs1[idx] <= outs1[idx]+1;
      cmdBuf.enq(idx,cmd);
      //$display("store_cmd %d", idx);
   endrule

   rule load_ctxt_a (!load_in_progress);
      if (outs1[loadIdx] > 0) begin
	 load_in_progress <= True;
	 cmdBuf.first_req(truncate(loadIdx));
      end
      else begin
	 incr_loadIdx;
      end
   endrule

   rule load_ctxt_b;
      let cmd <- cmdBuf.first_resp;
      let cond0 = buffCap[loadIdx].read() >= unpack(extend(cmd.burstLen>>beat_shift));
      let cond1 = cmd.len <= extend(cmd.burstLen);
      loadf_b.enq(tuple3(cmd,cond0,cond1));
   endrule

   rule load_ctxt_c;
      load_in_progress <= False;
      incr_loadIdx;
      match {.cmd,.cond0,.cond1} <- toGet(loadf_b).get;
      if  (cond0) begin
	 //$display("load_ctxt_b %h %d", cmd.base, idx);
	 let x = cmd.burstLen;
	 if (cmd.len < extend(cmd.burstLen))
	    x = truncate(cmd.len);
	 buffCap[loadIdx].decrement(unpack(extend(x>>beat_shift)));
	 loadf_c.enq(tuple2(truncate(loadIdx),cmd));
	 write_data_funnel.loadIdx(truncate(loadIdx));
	 if (cond1) begin
	    outs1[loadIdx] <= outs1[loadIdx]-1;
	    cmdBuf.deq(truncate(loadIdx));
	 end
	 else begin
	    let new_cmd = MemengineCmd{sglId:cmd.sglId, base:cmd.base+extend(cmd.burstLen), burstLen:cmd.burstLen, len:cmd.len-extend(cmd.burstLen), tag:cmd.tag};
	    cmdBuf.upd_head(truncate(loadIdx),new_cmd);
	 end
      end
   endrule

   function PipeIn#(Bit#(w)) check_in(FIFOF#(Bit#(w)) f, Integer i) =
      (interface PipeIn;
   	  method Bool notFull = f.notFull;
   	  method Action enq(Bit#(w) v);
	     f.enq(v);
	     buffCap[i].increment(1);
	     //$display("check_in %d", i);
	     // if(i==2)
	     // 	for(Integer j = 0; j < valueOf(w); j=j+32) begin
	     // 	   Bit#(32) xx = v[j+31:j];
	     // 	   $display("%h", xx);
	     // 	end
	  endmethod
       endinterface);

   function MemwriteServer#(dataWidth) toMemwriteServer(Server#(MemengineCmd,Bool) cs, PipeIn#(Bit#(dataWidth)) p) =
      (interface MemwriteServer;
	  interface cmdServer = cs;
	  interface dataPipe  = p;
       endinterface);

   Vector#(numServers, Server#(MemengineCmd,Bool)) rs;
   for(Integer i = 0; i < valueOf(numServers); i=i+1)
      rs[i] = (interface Server#(MemengineCmd,Bool);
		  interface Put request;
		     method Action put(MemengineCmd c) if (outs0[i] < cmd_q_depth);
			Bit#(32) bsb = fromInteger(bufferSizeBytes);
`ifdef BSIM
			Bit#(32) dw = fromInteger(valueOf(dataWidthBytes));
			Bit#(32) bl = extend(c.burstLen);
			// this is because bsc lifts the divide operation (below)
			// and on startup the simulator gets a floating-point exception
	  		if (bl ==0)
			   bl = 1;
			let mdw0 = ((c.len)/bl)*bl != c.len;
			let mdw1 = ((c.len)/dw)*dw != c.len;
			let bbl = extend(c.burstLen) > bsb;
			if(bbl || mdw0 || mdw1 || c.len == 0) begin
			   if (bbl)
			      $display("XXXXXXXXXX mkMemwriteEngineBuff::unsupported burstLen %d %d", bsb, c.burstLen);
			   if (mdw0 || mdw1 || c.len == 0)
			      $display("XXXXXXXXXX mkMemwriteEngineBuff::unsupported len %h mdw0=%d mdw1=%d", c.len, mdw0, mdw1);
			end
			else begin
`endif
			   outs0[i] <= outs0[i]+1;
			   cmds_in[i].enq(tuple2(fromInteger(i),c));
			   write_data_funnel.burstLen[i] <= c.burstLen >> beat_shift;
			   //$display("(%d) %h %h %h", i, c.base, c.len, c.burstLen);
`ifdef BSIM
			end
`endif
 		     endmethod
		  endinterface
		  interface Get response;
		     method ActionValue#(Bool) get;
			let rv <- toGet(outfs[i]).get;
	 		outs0[i] <= outs0[i]-1;
			return rv;
		     endmethod
		  endinterface
	       endinterface);
   interface writeServers = rs;
   interface MemWriteClient dmaClient;
      interface Get writeReq;
	 method ActionValue#(MemRequest) get();
	    match {.idx, .cmd} <- toGet(loadf_c).get;
	    Bit#(BurstLenSize) bl = cmd.burstLen;
	    Bool last = False;
	    if (cmd.len <= extend(bl)) begin
	       last = True;
	       bl = truncate(cmd.len);
	    end
	    let new_tag = (cmd.tag << valueOf(serverIdxSz)) | extend(idx);
	    workf.enq(tuple3(truncate(bl>>beat_shift), new_tag, last));
	    //$display("writeReq %d, %h %h %h", idx, cmd.base, bl, last);
	    return MemRequest { sglId: cmd.sglId, offset: cmd.base, burstLen:bl, tag: new_tag};
	 endmethod
      endinterface
      interface Get writeData;
	 method ActionValue#(MemData#(dataWidth)) get;
	    match {.rc, .new_tag, .last} = workf.first;
	    Bit#(serverIdxSz) idx = truncate(new_tag);
	    let new_respCnt = respCnt+1;
	    if (new_respCnt == rc) begin
	       respCnt <= 0;
	       workf.deq;
	       donef.enq(tuple2(idx,last));
	    end
	    else begin
	       respCnt <= new_respCnt;
	    end
	    match {._idx, .wd} <- toGet(write_data_funnel.dataOut).get;
	    if(_idx != idx) begin
	       $display("ERROR mkMemwriteEngineBuf: bursts oo %d %d", idx, _idx);
	       $finish;
	    end
	    // TODO: this last field should be set correctly to mark the end of bursts
	    return MemData{data:wd, tag:new_tag, last:False};
	 endmethod
      endinterface
      interface Put writeDone;
	 method Action put(Bit#(MemTagSize) tag);
	    match {.idx, .last} <- toGet(donef).get;
	    if (last)
	       outfs[idx].enq(True);
	    //$display("writeDone %d %d", idx, last);
	 endmethod
      endinterface
   endinterface
   interface dataPipes = zipWith(check_in, write_data_buffs, genVector);
   interface write_servers = zipWith(toMemwriteServer, rs, zipWith(check_in, write_data_buffs, genVector));
endmodule


had frc and frq reversed in interface tlpr

// Copyright (c) 2014-2015 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

package Pcie3EndpointX7;

import Clocks            ::*;
import Vector            ::*;
import Connectable       ::*;
import GetPut            ::*;
import Reserved          ::*;
import TieOff            ::*;
import DefaultValue      ::*;
import DReg              ::*;
import Gearbox           ::*;
import FIFO              ::*;
import FIFOF             ::*;
import SpecialFIFOs      ::*;
import ClientServer      ::*;
import Real              ::*;
import XilinxVirtex7PCIE ::*;
import BUtils            ::*;

import ConnectalClocks   ::*;
import ConnectalXilinxCells   ::*;
import XilinxCells       ::*;
import PCIE              ::*;
import PCIEWRAPPER3      ::*;
import Bufgctrl           ::*;
import PcieGearbox       :: *;

interface PcieEndpointX7#(numeric type lanes);
   interface PciewrapPci_exp#(lanes)           pcie;
   interface PciewrapUser#(lanes)              user;
   interface PciewrapPipe#(lanes)              pipe;
   interface PciewrapCommon#(lanes)            common;
   interface Server#(TLPData#(16), TLPData#(16)) tlpr;
   interface Server#(TLPData#(16), TLPData#(16)) tlpc;
   interface Clock epClock125;
   interface Reset epReset125;
   interface Clock epClock250;
   interface Reset epReset250;
   interface Clock epPcieClock;
   interface Reset epPcieReset;
   interface Clock epPortalClock;
   interface Reset epPortalReset;
   interface Clock epDerivedClock;
   interface Reset epDerivedReset;
endinterface

typedef struct {
   Bit #(256)     data;
   Bool          sop;
   Bool          eop;
   Bit #(8)      keep;
   TLPFirstDWBE  first_be;
   TLPFirstDWBE  last_be;
} AxiStCq deriving (Bits, Eq);

typedef struct {
   Bit #(256)     data;
   Bit #(8)      keep;
   Bool          last;
} AxiStCc deriving (Bits, Eq);

typedef struct {
   Bit #(256)     data;
   Bool          last;
   Bit #(8)      keep;
   Bit #(4)      first_be;
   Bit #(4)      last_be;
} AxiStRq deriving (Bits, Eq);

typedef struct {
   Bit #(256)     data;
   Bool          sop;
   Bool          eop;
   Bit #(8)      keep;
   Bit #(8)      be;
} AxiStRc deriving (Bits, Eq);

`ifdef BOARD_vc709
typedef 8 PcieLanes;
typedef 8 NumLeds;
`endif
`ifdef BOARD_nfsume
typedef 8 PcieLanes;
typedef 2 NumLeds;
`endif

(* synthesize *)
module mkPcieEndpointX7(PcieEndpointX7#(PcieLanes));

   PCIEParams params = defaultValue;
   Clock defaultClock <- exposeCurrentClock();
   Reset defaultReset <- exposeCurrentReset();
   Reset defaultResetInverted <- mkResetInverter(defaultReset, clocked_by defaultClock);
   PcieWrap#(PcieLanes) pcie_ep <- mkPcieWrap(defaultClock, defaultResetInverted);

   // The PCIe endpoint exports full (250MHz) and half-speed (125MHz) clocks
   Clock clock250 = pcie_ep.user_clk;
   Reset user_reset_n <- mkResetInverter(pcie_ep.user_reset, clocked_by pcie_ep.user_clk);
   Reset reset250 <- mkAsyncReset(4, user_reset_n, clock250);

   ClockGenerator7Params     clkgenParams = defaultValue;
   clkgenParams.clkin1_period    = 4.000; //  250MHz
   clkgenParams.clkin1_period    = 4.000;
   clkgenParams.clkin_buffer     = False;
   clkgenParams.clkfbout_mult_f  = 4.000; // 1000MHz
   clkgenParams.clkout0_divide_f = 8.000; //  125MHz
   clkgenParams.clkout1_divide     = round(derivedClockPeriod);
   clkgenParams.clkout1_duty_cycle = 0.5;
   clkgenParams.clkout1_phase      = 0.0000;
   ClockGenerator7           clkgen <- mkClockGenerator7(clkgenParams, clocked_by clock250, reset_by reset250);
   Clock clock125 = clkgen.clkout0; /* half speed user_clk */
   Reset reset125 <- mkAsyncReset(4, reset250, clock125);
   Clock derivedClock = clkgen.clkout1;
   Reset derivedReset <- mkAsyncReset(4, reset250, derivedClock);
   Reset user_reset <- mkAsyncReset(2, pcie_ep.user_reset, pcie_ep.user_clk);

   FIFOF#(AxiStCq) fAxiCq <- mkBypassFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   FIFOF#(AxiStRc) fAxiRc <- mkBypassFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);

   FIFOF#(AxiStRq) fAxiRq <- mkBypassFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   FIFOF#(AxiStCc) fAxiCc <- mkBypassFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);

   FIFOF#(TLPData#(16)) fcq <- mkFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   FIFOF#(TLPData#(16)) frc <- mkFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   FIFOF#(TLPData#(16)) fcc <- mkFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   FIFOF#(TLPData#(16)) frq <- mkFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);

   // Drive s_axis_rq
   let rq_txready = (pcie_ep.s_axis_rq.tready != 0 && fAxiRq.notEmpty);

   //(* fire_when_enabled, no_implicit_conditions *)
   rule drive_axi_rq if (rq_txready);
      let info = fAxiRq.first; fAxiRq.deq;
      pcie_ep.s_axis_rq.tvalid(1);
      pcie_ep.s_axis_rq.tlast(pack(info.last));
      pcie_ep.s_axis_rq.tdata(info.data);
      pcie_ep.s_axis_rq.tkeep(info.keep);
      pcie_ep.s_axis_rq.tuser({0, info.last_be, info.first_be});
   endrule

   (* fire_when_enabled, no_implicit_conditions *)
   rule drive_axi_rq2 if (!rq_txready);
      pcie_ep.s_axis_rq.tvalid(0);
      pcie_ep.s_axis_rq.tlast(0);
      pcie_ep.s_axis_rq.tdata(0);
      pcie_ep.s_axis_rq.tkeep(0);
      pcie_ep.s_axis_rq.tuser(0);
   endrule

   // Drive s_axis_cc
   let cc_txready = (pcie_ep.s_axis_cc.tready != 0 && fAxiCc.notEmpty);

   //(* fire_when_enabled, no_implicit_conditions *)
   rule drive_axi_cc if (cc_txready);
      let info = fAxiCc.first; fAxiCc.deq;
      $display("drive axi_cc, data: %h, keep: %h, last: %h", info.data, info.keep, info.last);
      pcie_ep.s_axis_cc.tvalid(1);
      pcie_ep.s_axis_cc.tlast(pack(info.last));
      pcie_ep.s_axis_cc.tdata(info.data);
      pcie_ep.s_axis_cc.tkeep(info.keep);
      pcie_ep.s_axis_cc.tuser(0);
   endrule

   (* fire_when_enabled, no_implicit_conditions *)
   rule drive_axi_cc2 if (!cc_txready);
      pcie_ep.s_axis_cc.tvalid(0);
      pcie_ep.s_axis_cc.tlast(0);
      pcie_ep.s_axis_cc.tdata(0);
      pcie_ep.s_axis_cc.tkeep(0);
      pcie_ep.s_axis_cc.tuser(0);
   endrule

   (* fire_when_enabled, no_implicit_conditions *)
   rule drive_axi_rc_ready;
      pcie_ep.m_axis_rc.tready (duplicate (pack (fAxiRc.notFull)));
   endrule

   // Drive m_axis_rc
   (* fire_when_enabled *)
   rule sink_axi_rc if (pcie_ep.m_axis_rc.tvalid != 0 && fAxiRc.notFull);
      let rc = AxiStRc {data:pcie_ep.m_axis_rc.tdata,
			sop: unpack (pcie_ep.m_axis_rc.tuser [32]),         // tuser.is_sof_0
			eop: unpack (pcie_ep.m_axis_rc.tlast),
			keep:pcie_ep.m_axis_rc.tkeep,
			be:  truncate (pcie_ep.m_axis_rc.tuser [31:0])};    // tuser.byte_en
      fAxiRc.enq (rc);
   endrule

   (* fire_when_enabled, no_implicit_conditions *)
   rule drive_axi_cq_ready;
      pcie_ep.m_axis_cq.tready (duplicate (pack (fAxiCq.notFull)));
   endrule

   (* fire_when_enabled *)
   rule sink_axi_cq if (pcie_ep.m_axis_cq.tvalid != 0 && fAxiCq.notFull);
      let cq = AxiStCq {data:     pcie_ep.m_axis_cq.tdata,
			sop:      unpack (pcie_ep.m_axis_cq.tuser [40]),  // tuser.sop
			eop:      unpack (pcie_ep.m_axis_cq.tlast),
			keep:     pcie_ep.m_axis_cq.tkeep,
			first_be: pcie_ep.m_axis_cq.tuser [3:0],    // tuser.first_be,
			last_be:  pcie_ep.m_axis_cq.tuser [7:4]};   // tuser.last_be
      fAxiCq.enq (cq);
   endrule

   // CQ.
   CQDescriptor cq_desc = unpack(fAxiCq.first.data [127:0]);

   rule rl_cq_wr_header (fAxiCq.first.sop && ((cq_desc.reqtype == MEMORY_WRITE) || (cq_desc.reqtype == IO_WRITE)));
      Bit#(32) data = fAxiCq.first.data[159:128];
      // get data;
      TLPData#(16) tlp16 = convertCQDescriptorToTLP16(cq_desc, data, fAxiCq.first.first_be, fAxiCq.first.last_be);
      $display("cq_desc.reqtype=%h", cq_desc.reqtype);
      // enqueue?
      fcq.enq(tlp16);
      fAxiCq.deq;
   endrule

   // Write data payload, no data remaining
   rule rl_cq_wr_payload((!fAxiCq.first.sop));
      fAxiCq.deq;
   endrule

   // Write data payload, 1 to 3 DWs remaining
   // Write data payload, 4 or more DWs remaining

   rule rl_cq_rd_header (fAxiCq.first.sop && ((cq_desc.reqtype == MEMORY_READ) || (cq_desc.reqtype == IO_READ)));
      Bit#(32) data = 0;
      TLPData#(16) tlp16 = convertCQDescriptorToTLP16(cq_desc, data, fAxiCq.first.first_be, fAxiCq.first.last_be);
      $display("rl_cq_rd_header: cq_desc = %16x", cq_desc);
      fcq.enq(tlp16);
      fAxiCq.deq;
   endrule

   // RC.
   Reg#(DWCount) rg_dwcount <- mkRegU(clocked_by pcie_ep.user_clk, reset_by user_reset_n);

   rule rl_rc_header (fAxiRc.first.sop);
      RCDescriptor rc_desc = unpack(fAxiRc.first.data [95:0]);
      Bit#(32) data = fAxiRc.first.data[31:0];
      TLPData#(16) tlp16 = convertRCDescriptorToTLP16(rc_desc, data);
      rg_dwcount <= (rc_desc.dwcount == 0) ? 0 : rc_desc.dwcount - 1;
      frc.enq(tlp16);
      fAxiRc.deq;
   endrule

   rule rl_rc_data ((!(fAxiRc.first.sop)) && (rg_dwcount != 0));
      Bit#(16) be16;
      case (rg_dwcount)
         1: be16 = 16'hF000;
         2: be16 = 16'hFF00;
         3: be16 = 16'hFFF0;
         default: be16 = 16'hFFFF;
      endcase
      TLPData#(16) tlp16 = TLPData{sof: False,
                                   eof: (rg_dwcount <= 4),
                                   hit: 0,
                                   be: be16,
                                   data: pack(fAxiRc.first.data[127:0])};
      frc.enq(tlp16);
      fAxiRc.deq;
   endrule

   Reg#(DWCount) cc_dwcount <- mkReg(0, clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   FIFOF#(TLPData#(16)) fcc_tlps <- mkPipelineFIFOF (clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   // CC.
   rule get_cc_tlps;
      let tlp <- toGet(fcc).get;
      fcc_tlps.enq(tlp);
   endrule

   rule rl_cc_header(fcc_tlps.first.sof);
      match { .cc_desc, .dw} = convertTLP16ToCCDescriptor(fcc_tlps.first);
      cc_dwcount <= cc_desc.dwcount - 1;
      fAxiCc.enq(AxiStCc {data: zeroExtend({dw, pack(cc_desc)[95:0]}),
                       last: fcc_tlps.first.eof,
                       keep: 8'h0F});
      fcc_tlps.deq;
   endrule

   rule rl_cc_data((!fcc_tlps.first.sof) && (cc_dwcount != 0));
      Bit#(256) x = zeroExtend(fcc_tlps.first.data); //FIXME
      fAxiCc.enq(AxiStCc {data: {x},
                       last: cc_dwcount <= 4,
                       keep: (cc_dwcount == 3) ? 8'h0F : 8'hFF});
      fcc_tlps.deq;
   endrule

   // RQ.
   FIFOF#(TLPData#(16)) frq_tlps <- mkPipelineFIFOF (clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   Reg#(DWCount) rq_dwcount <- mkReg(0, clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   Reg#(Maybe#(Bit#(32))) rq_mdw <- mkRegU(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   Reg#(Bit#(4)) rq_first_be <- mkRegU(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   Reg#(Bit#(4)) rq_last_be <- mkRegU(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   rule get_rq_tlps;
      let tlp <- toGet(frq).get;
      frq_tlps.enq(tlp);
   endrule

   rule rl_rq_header(frq_tlps.first.sof);
      match { .rq_desc, .first_be, .last_be, .mdata} = convertTLP16ToRQDescriptor(frq_tlps.first);
      rq_dwcount <= ((rq_desc.reqtype == MEMORY_WRITE) ? rq_desc.dwcount : 0);
      rq_mdw <= mdata;
      rq_first_be <= first_be;
      rq_last_be <= last_be;
      fAxiRq.enq(AxiStRq {data: zeroExtend(pack(rq_desc)), //FIXME:
                       last: frq_tlps.first.eof,
                       keep: 8'hFF,
                       first_be: first_be,
                       last_be: last_be});
      frq_tlps.deq;
   endrule

   // rq_mdw contains last DW
   rule rl_rq_data_a (rq_mdw matches tagged Valid .dw &&& (rq_dwcount == 1));
      rq_dwcount <= 0;
      rq_mdw <= tagged Invalid;
   endrule

   // rq_mdw Valid, and there are more DWs.
   rule rl_rq_data_b(rq_mdw matches tagged Valid .dw &&& (rq_dwcount != 1));

   endrule

   // rq_mdw Invalid, and there are more DWs.
   rule rl_rq_data_c(rq_mdw matches tagged Invalid &&& (rq_dwcount != 1));

   endrule

`ifdef PCIE_250MHZ
   Clock portalClock = clock250;
   Reset portalReset = reset250;
`else
   Clock portalClock = clock125;
   Reset portalReset = reset125;
`endif
   // The PCIE endpoint is processing Gen3 descriptors at 250MHz. The
   // AXI bridge is accepting TLPData#(16)s at 250 MHz. The
   // conversion uses half of Gen3 descriptor.
   //mkConnection(tlp8, gb.tlp, clocked_by portalClock, reset_by portalReset);

   interface Server tlpr;
      interface request = toPut(frq);
      interface response = toGet(frc);
   endinterface
   interface Server tlpc;
      interface request = toPut(fcc);
      interface response = toGet(fcq);
   endinterface
   interface pcie    = pcie_ep.pci_exp;
   interface Pcie3wrapUser user = pcie_ep.user;
   interface PciewrapPipe pipe = pcie_ep.pipe;
   interface PciewrapCommon common= pcie_ep.common;
   interface Clock epClock125 = clock125;
   interface Reset epReset125 = reset125;
   interface Clock epClock250 = clock250;
   interface Reset epReset250 = reset250;
   interface Clock epPcieClock = clock250;
   interface Reset epPcieReset = reset250;
   interface Clock epPortalClock = clock250;
   interface Reset epPortalReset = reset250;
   interface Clock epDerivedClock = derivedClock;
   interface Reset epDerivedReset = derivedReset;
endmodule: mkPcieEndpointX7

endpackage: Pcie3EndpointX7

// Copyright (c) 2014-2015 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

package Pcie3EndpointX7;

import Clocks            ::*;
import Vector            ::*;
import Connectable       ::*;
import GetPut            ::*;
import Reserved          ::*;
import TieOff            ::*;
import DefaultValue      ::*;
import DReg              ::*;
import Gearbox           ::*;
import FIFO              ::*;
import FIFOF             ::*;
import SpecialFIFOs      ::*;
import ClientServer      ::*;
import Real              ::*;
import XilinxVirtex7PCIE ::*;
import BUtils            ::*;

import ConnectalClocks   ::*;
import ConnectalXilinxCells   ::*;
import XilinxCells       ::*;
import PCIE              ::*;
import PCIEWRAPPER3      ::*;
import Bufgctrl           ::*;
import PcieGearbox       :: *;

interface PcieEndpointX7#(numeric type lanes);
   interface PciewrapPci_exp#(lanes)           pcie;
   interface PciewrapUser#(lanes)              user;
   interface PciewrapPipe#(lanes)              pipe;
   interface PciewrapCommon#(lanes)            common;
   interface Server#(TLPData#(16), TLPData#(16)) tlpr;
   interface Server#(TLPData#(16), TLPData#(16)) tlpc;
   interface Clock epClock125;
   interface Reset epReset125;
   interface Clock epClock250;
   interface Reset epReset250;
   interface Clock epPcieClock;
   interface Reset epPcieReset;
   interface Clock epPortalClock;
   interface Reset epPortalReset;
   interface Clock epDerivedClock;
   interface Reset epDerivedReset;
endinterface

typedef struct {
   Bit #(256)     data;
   Bool          sop;
   Bool          eop;
   Bit #(8)      keep;
   TLPFirstDWBE  first_be;
   TLPFirstDWBE  last_be;
} AxiStCq deriving (Bits, Eq);

typedef struct {
   Bit #(256)     data;
   Bit #(8)      keep;
   Bool          last;
} AxiStCc deriving (Bits, Eq);

typedef struct {
   Bit #(256)     data;
   Bool          last;
   Bit #(8)      keep;
   Bit #(4)      first_be;
   Bit #(4)      last_be;
} AxiStRq deriving (Bits, Eq);

typedef struct {
   Bit #(256)     data;
   Bool          sop;
   Bool          eop;
   Bit #(8)      keep;
   Bit #(8)      be;
} AxiStRc deriving (Bits, Eq);

`ifdef BOARD_vc709
typedef 8 PcieLanes;
typedef 8 NumLeds;
`endif
`ifdef BOARD_nfsume
typedef 8 PcieLanes;
typedef 2 NumLeds;
`endif

(* synthesize *)
module mkPcieEndpointX7(PcieEndpointX7#(PcieLanes));

   PCIEParams params = defaultValue;
   Clock defaultClock <- exposeCurrentClock();
   Reset defaultReset <- exposeCurrentReset();
   Reset defaultResetInverted <- mkResetInverter(defaultReset, clocked_by defaultClock);
   PcieWrap#(PcieLanes) pcie_ep <- mkPcieWrap(defaultClock, defaultResetInverted);

   // The PCIe endpoint exports full (250MHz) and half-speed (125MHz) clocks
   Clock clock250 = pcie_ep.user_clk;
   Reset user_reset_n <- mkResetInverter(pcie_ep.user_reset, clocked_by pcie_ep.user_clk);
   Reset reset250 <- mkAsyncReset(4, user_reset_n, clock250);

   ClockGenerator7Params     clkgenParams = defaultValue;
   clkgenParams.clkin1_period    = 4.000; //  250MHz
   clkgenParams.clkin1_period    = 4.000;
   clkgenParams.clkin_buffer     = False;
   clkgenParams.clkfbout_mult_f  = 4.000; // 1000MHz
   clkgenParams.clkout0_divide_f = 8.000; //  125MHz
   clkgenParams.clkout1_divide     = round(derivedClockPeriod);
   clkgenParams.clkout1_duty_cycle = 0.5;
   clkgenParams.clkout1_phase      = 0.0000;
   ClockGenerator7           clkgen <- mkClockGenerator7(clkgenParams, clocked_by clock250, reset_by reset250);
   Clock clock125 = clkgen.clkout0; /* half speed user_clk */
   Reset reset125 <- mkAsyncReset(4, reset250, clock125);
   Clock derivedClock = clkgen.clkout1;
   Reset derivedReset <- mkAsyncReset(4, reset250, derivedClock);
   Reset user_reset <- mkAsyncReset(2, pcie_ep.user_reset, pcie_ep.user_clk);

   FIFOF#(AxiStCq) fAxiCq <- mkBypassFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   FIFOF#(AxiStRc) fAxiRc <- mkBypassFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);

   FIFOF#(AxiStRq) fAxiRq <- mkBypassFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   FIFOF#(AxiStCc) fAxiCc <- mkBypassFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);

   FIFOF#(TLPData#(16)) fcq <- mkFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   FIFOF#(TLPData#(16)) frc <- mkFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   FIFOF#(TLPData#(16)) fcc <- mkFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   FIFOF#(TLPData#(16)) frq <- mkFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);

   // Drive s_axis_rq
   let rq_txready = (pcie_ep.s_axis_rq.tready != 0 && fAxiRq.notEmpty);

   //(* fire_when_enabled, no_implicit_conditions *)
   rule drive_axi_rq if (rq_txready);
      let info = fAxiRq.first; fAxiRq.deq;
      pcie_ep.s_axis_rq.tvalid(1);
      pcie_ep.s_axis_rq.tlast(pack(info.last));
      pcie_ep.s_axis_rq.tdata(info.data);
      pcie_ep.s_axis_rq.tkeep(info.keep);
      pcie_ep.s_axis_rq.tuser({0, info.last_be, info.first_be});
   endrule

   (* fire_when_enabled, no_implicit_conditions *)
   rule drive_axi_rq2 if (!rq_txready);
      pcie_ep.s_axis_rq.tvalid(0);
      pcie_ep.s_axis_rq.tlast(0);
      pcie_ep.s_axis_rq.tdata(0);
      pcie_ep.s_axis_rq.tkeep(0);
      pcie_ep.s_axis_rq.tuser(0);
   endrule

   // Drive s_axis_cc
   let cc_txready = (pcie_ep.s_axis_cc.tready != 0 && fAxiCc.notEmpty);

   //(* fire_when_enabled, no_implicit_conditions *)
   rule drive_axi_cc if (cc_txready);
      let info = fAxiCc.first; fAxiCc.deq;
      $display("drive axi_cc, data: %h, keep: %h, last: %h", info.data, info.keep, info.last);
      pcie_ep.s_axis_cc.tvalid(1);
      pcie_ep.s_axis_cc.tlast(pack(info.last));
      pcie_ep.s_axis_cc.tdata(info.data);
      pcie_ep.s_axis_cc.tkeep(info.keep);
      pcie_ep.s_axis_cc.tuser(0);
   endrule

   (* fire_when_enabled, no_implicit_conditions *)
   rule drive_axi_cc2 if (!cc_txready);
      pcie_ep.s_axis_cc.tvalid(0);
      pcie_ep.s_axis_cc.tlast(0);
      pcie_ep.s_axis_cc.tdata(0);
      pcie_ep.s_axis_cc.tkeep(0);
      pcie_ep.s_axis_cc.tuser(0);
   endrule

   (* fire_when_enabled, no_implicit_conditions *)
   rule drive_axi_rc_ready;
      pcie_ep.m_axis_rc.tready (duplicate (pack (fAxiRc.notFull)));
   endrule

   // Drive m_axis_rc
   (* fire_when_enabled *)
   rule sink_axi_rc if (pcie_ep.m_axis_rc.tvalid != 0 && fAxiRc.notFull);
      let rc = AxiStRc {data:pcie_ep.m_axis_rc.tdata,
			sop: unpack (pcie_ep.m_axis_rc.tuser [32]),         // tuser.is_sof_0
			eop: unpack (pcie_ep.m_axis_rc.tlast),
			keep:pcie_ep.m_axis_rc.tkeep,
			be:  truncate (pcie_ep.m_axis_rc.tuser [31:0])};    // tuser.byte_en
      fAxiRc.enq (rc);
   endrule

   (* fire_when_enabled, no_implicit_conditions *)
   rule drive_axi_cq_ready;
      pcie_ep.m_axis_cq.tready (duplicate (pack (fAxiCq.notFull)));
   endrule

   (* fire_when_enabled *)
   rule sink_axi_cq if (pcie_ep.m_axis_cq.tvalid != 0 && fAxiCq.notFull);
      let cq = AxiStCq {data:     pcie_ep.m_axis_cq.tdata,
			sop:      unpack (pcie_ep.m_axis_cq.tuser [40]),  // tuser.sop
			eop:      unpack (pcie_ep.m_axis_cq.tlast),
			keep:     pcie_ep.m_axis_cq.tkeep,
			first_be: pcie_ep.m_axis_cq.tuser [3:0],    // tuser.first_be,
			last_be:  pcie_ep.m_axis_cq.tuser [7:4]};   // tuser.last_be
      fAxiCq.enq (cq);
   endrule

   // CQ.
   CQDescriptor cq_desc = unpack(fAxiCq.first.data [127:0]);

   rule rl_cq_wr_header (fAxiCq.first.sop && ((cq_desc.reqtype == MEMORY_WRITE) || (cq_desc.reqtype == IO_WRITE)));
      Bit#(32) data = fAxiCq.first.data[159:128];
      // get data;
      TLPData#(16) tlp16 = convertCQDescriptorToTLP16(cq_desc, data, fAxiCq.first.first_be, fAxiCq.first.last_be);
      $display("cq_desc.reqtype=%h", cq_desc.reqtype);
      // enqueue?
      fcq.enq(tlp16);
      fAxiCq.deq;
   endrule

   // Write data payload, no data remaining
   rule rl_cq_wr_payload((!fAxiCq.first.sop));
      fAxiCq.deq;
   endrule

   // Write data payload, 1 to 3 DWs remaining
   // Write data payload, 4 or more DWs remaining

   rule rl_cq_rd_header (fAxiCq.first.sop && ((cq_desc.reqtype == MEMORY_READ) || (cq_desc.reqtype == IO_READ)));
      Bit#(32) data = 0;
      TLPData#(16) tlp16 = convertCQDescriptorToTLP16(cq_desc, data, fAxiCq.first.first_be, fAxiCq.first.last_be);
      $display("rl_cq_rd_header: cq_desc = %16x", cq_desc);
      fcq.enq(tlp16);
      fAxiCq.deq;
   endrule

   // RC.
   Reg#(DWCount) rg_dwcount <- mkRegU(clocked_by pcie_ep.user_clk, reset_by user_reset_n);

   rule rl_rc_header (fAxiRc.first.sop);
      RCDescriptor rc_desc = unpack(fAxiRc.first.data [95:0]);
      Bit#(32) data = fAxiRc.first.data[31:0];
      TLPData#(16) tlp16 = convertRCDescriptorToTLP16(rc_desc, data);
      rg_dwcount <= (rc_desc.dwcount == 0) ? 0 : rc_desc.dwcount - 1;
      frc.enq(tlp16);
      fAxiRc.deq;
   endrule

   rule rl_rc_data ((!(fAxiRc.first.sop)) && (rg_dwcount != 0));
      Bit#(16) be16;
      case (rg_dwcount)
         1: be16 = 16'hF000;
         2: be16 = 16'hFF00;
         3: be16 = 16'hFFF0;
         default: be16 = 16'hFFFF;
      endcase
      TLPData#(16) tlp16 = TLPData{sof: False,
                                   eof: (rg_dwcount <= 4),
                                   hit: 0,
                                   be: be16,
                                   data: pack(fAxiRc.first.data[127:0])};
      frc.enq(tlp16);
      fAxiRc.deq;
   endrule

   Reg#(DWCount) cc_dwcount <- mkReg(0, clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   FIFOF#(TLPData#(16)) fcc_tlps <- mkPipelineFIFOF (clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   // CC.
   rule get_cc_tlps;
      let tlp <- toGet(fcc).get;
      fcc_tlps.enq(tlp);
   endrule

   rule rl_cc_header(fcc_tlps.first.sof);
      match { .cc_desc, .dw} = convertTLP16ToCCDescriptor(fcc_tlps.first);
      cc_dwcount <= cc_desc.dwcount - 1;
      fAxiCc.enq(AxiStCc {data: zeroExtend({dw, pack(cc_desc)[95:0]}),
                       last: fcc_tlps.first.eof,
                       keep: 8'h0F});
      fcc_tlps.deq;
   endrule

   rule rl_cc_data((!fcc_tlps.first.sof) && (cc_dwcount != 0));
      Bit#(256) x = zeroExtend(fcc_tlps.first.data); //FIXME
      fAxiCc.enq(AxiStCc {data: {x},
                       last: cc_dwcount <= 4,
                       keep: (cc_dwcount == 3) ? 8'h0F : 8'hFF});
      fcc_tlps.deq;
   endrule

   // RQ.
   FIFOF#(TLPData#(16)) frq_tlps <- mkPipelineFIFOF (clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   Reg#(DWCount) rq_dwcount <- mkReg(0, clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   Reg#(Maybe#(Bit#(32))) rq_mdw <- mkRegU(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   Reg#(Bit#(4)) rq_first_be <- mkRegU(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   Reg#(Bit#(4)) rq_last_be <- mkRegU(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   rule get_rq_tlps;
      let tlp <- toGet(frq).get;
      frq_tlps.enq(tlp);
   endrule

   rule rl_rq_header(frq_tlps.first.sof);
      match { .rq_desc, .first_be, .last_be, .mdata} = convertTLP16ToRQDescriptor(frq_tlps.first);
      rq_dwcount <= ((rq_desc.reqtype == MEMORY_WRITE) ? rq_desc.dwcount : 0);
      rq_mdw <= mdata;
      rq_first_be <= first_be;
      rq_last_be <= last_be;
      fAxiRq.enq(AxiStRq {data: zeroExtend(pack(rq_desc)), //FIXME:
                       last: frq_tlps.first.eof,
                       keep: 8'hFF,
                       first_be: first_be,
                       last_be: last_be});
      frq_tlps.deq;
   endrule

   // rq_mdw contains last DW
   rule rl_rq_data_a (rq_mdw matches tagged Valid .dw &&& (rq_dwcount == 1));
      rq_dwcount <= 0;
      rq_mdw <= tagged Invalid;
   endrule

   // rq_mdw Valid, and there are more DWs.
   rule rl_rq_data_b(rq_mdw matches tagged Valid .dw &&& (rq_dwcount != 1));

   endrule

   // rq_mdw Invalid, and there are more DWs.
   rule rl_rq_data_c(rq_mdw matches tagged Invalid &&& (rq_dwcount != 1));

   endrule

`ifdef PCIE_250MHZ
   Clock portalClock = clock250;
   Reset portalReset = reset250;
`else
   Clock portalClock = clock125;
   Reset portalReset = reset125;
`endif
   // The PCIE endpoint is processing Gen3 descriptors at 250MHz. The
   // AXI bridge is accepting TLPData#(16)s at 250 MHz. The
   // conversion uses half of Gen3 descriptor.
   //mkConnection(tlp8, gb.tlp, clocked_by portalClock, reset_by portalReset);

   interface Server tlpr;
      interface request = toPut(frc);
      interface response = toGet(frq);
   endinterface
   interface Server tlpc;
      interface request = toPut(fcc);
      interface response = toGet(fcq);
   endinterface
   interface pcie    = pcie_ep.pci_exp;
   interface Pcie3wrapUser user = pcie_ep.user;
   interface PciewrapPipe pipe = pcie_ep.pipe;
   interface PciewrapCommon common= pcie_ep.common;
   interface Clock epClock125 = clock125;
   interface Reset epReset125 = reset125;
   interface Clock epClock250 = clock250;
   interface Reset epReset250 = reset250;
   interface Clock epPcieClock = clock250;
   interface Reset epPcieReset = reset250;
   interface Clock epPortalClock = clock250;
   interface Reset epPortalReset = reset250;
   interface Clock epDerivedClock = derivedClock;
   interface Reset epDerivedReset = derivedReset;
endmodule: mkPcieEndpointX7

endpackage: Pcie3EndpointX7


reduce AXI/Avalon differences in MemMasterEngine

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import FIFOF        :: *;
import Vector       :: *;
import GetPut       :: *;
import Connectable  :: *;
import MIMO         :: *;
import PCIE         :: *;
import DefaultValue :: *;
import ClientServer :: *;
import MemTypes     :: *;

//
// Top interface: PCIe transaction level packets (TLPs)
// Bottom interface: MemMaster that sends read/write requests to an MemSlave
// Also sources interrupt MSIX requests
interface MemMasterEngine;
    interface Client#(TLPData#(16), TLPData#(16)) tlp;
    interface PhysMemMaster#(32,32) master;
endinterface

`ifdef XILINX
   `define AXI
`elsif BSIM
   `define AXI
`elsif ALTERA
   `define AVALON
`elsif VSIM
   `define AVALON
`endif

(* synthesize *)
module mkMemMasterEngine#(PciId my_id)(MemMasterEngine);
    Reg#(Bit#(7)) hitReg <- mkReg(0);
    FIFOF#(TLPMemoryIO3DWHeader) readHeaderFifo <- mkSizedFIFOF(8);
    FIFOF#(TLPMemoryIO3DWHeader) readDataFifo <- mkSizedFIFOF(8);
    FIFOF#(TLPMemoryIO3DWHeader) writeHeaderFifo <- mkSizedFIFOF(8);
    FIFOF#(TLPMemoryIO3DWHeader) writeDataFifo <- mkSizedFIFOF(8);
    FIFOF#(TLPData#(16)) tlpOutFifo <- mkSizedFIFOF(8);
    Reg#(TLPTag) tlpTag <- mkReg(0);

    MIMOConfiguration mimoCfg = defaultValue;
    MIMO#(1,4,4,Bit#(32)) completionMimo <- mkMIMO(mimoCfg);
   Reg#(TLPLength)             readBurstCount <- mkReg(0);
   Reg#(LUInt#(4))     completionMimoDeqCount <- mkReg(0);
   Reg#(Bool)      readBurstCountGreaterThan4 <- mkReg(False);
   Reg#(Bool)                  readInProgress <- mkReg(False);
   Reg#(Bit#(7))               address <- mkReg(0);

   function Bool isQuarWordAligned(Bit#(7) lower_addr);
      return (lower_addr[2:0]==3'b0);
   endfunction

   rule completionHeader if (!readInProgress && readDataFifo.notEmpty() && completionMimo.deqReadyN(1));
      let hdr = readDataFifo.first;
      TLPLength rbc = hdr.length;

      Vector#(4, Bit#(32)) dvec = unpack(0);
`ifdef AXI
      dvec = completionMimo.first();
      completionMimo.deq(1);
`elsif AVALON
      let quadWordAligned = isQuarWordAligned(getLowerAddr(hdr.addr, hdr.firstbe));
      // if quad-word aligned, insert bubble.
      if (!quadWordAligned) begin
         dvec = completionMimo.first();
         completionMimo.deq(1);
      end
`endif
      //$display("completionHeader length=%d rbc=%d addr=%x", hdr.length, rbc, hdr.addr);
      TLPCompletionHeader completion = defaultValue;
      completion.format = MEM_WRITE_3DW_DATA;
      completion.pkttype = COMPLETION;
      completion.relaxed = hdr.relaxed;
      completion.nosnoop = hdr.nosnoop;
      completion.length = hdr.length;
      completion.tclass = hdr.tclass;
      completion.cmplid = my_id;
      completion.tag = truncate(hdr.tag);
      completion.bytecount = 4;
      completion.reqid = hdr.reqid;
      completion.loweraddr = getLowerAddr(hdr.addr, hdr.firstbe);
`ifdef AXI
      completion.data = byteSwap(dvec[0]);
`elsif AVALON
      if (!quadWordAligned) begin
         completion.data = (dvec[0]);
      end
      else begin
         completion.data = unpack(0);
      end
`endif
      TLPData#(16) tlp = defaultValue;
      tlp.data = pack(completion);
      tlp.sof = True;

`ifdef AXI
      tlp.eof = (rbc == 1) ? True : False;
`elsif AVALON
      tlp.eof = (rbc == 1 && !quadWordAligned) ? True : False;
`endif
      tlp.be = 16'hFFFF;
      tlp.hit = hitReg;
      tlpOutFifo.enq(tlp);

`ifdef AXI
      rbc = rbc - 1;
`elsif AVALON
      if (!quadWordAligned) begin
         rbc = rbc - 1;
      end
`endif
      readBurstCount <= rbc;
      completionMimoDeqCount <= truncate(min(4,unpack(rbc)));
      readInProgress <= (rbc != 0);
      readBurstCountGreaterThan4 <= (rbc > 4);
      if (rbc == 0) begin
	 readDataFifo.deq;
      end
   endrule

    function Bit#(16) tlpBe(TLPLength len);
       if (len == 0)
	  return 0;
       else if (len == 1)
	  return 16'hf000;
       else if (len == 2)
	  return 16'hff00;
       else if (len == 3)
	  return 16'hfff0;
       else
	  return 16'hffff;
    endfunction

   rule continuation if (readInProgress && completionMimo.deqReadyN(completionMimoDeqCount));
      let rbc = readBurstCount;
      TLPData#(16) tlp = defaultValue;
      Vector#(4, Bit#(32)) dvec = unpack(0);
      tlp.sof = False;
      dvec = completionMimo.first();
      completionMimo.deq(completionMimoDeqCount);

      if (readBurstCountGreaterThan4) begin
	 rbc = rbc - 4;
	 tlp.be = tlpBe(4);
	 tlp.eof = False;
      end
      else begin
	 tlp.be = tlpBe(rbc);
	 //$display("tlp.data=%h tlp.be=%h", tlp.data, tlp.be);
	 tlp.eof = True;
	 rbc = 0;
      end

      readBurstCount <= rbc;
      completionMimoDeqCount <= truncate(min(4,unpack(rbc)));
      readBurstCountGreaterThan4 <= (rbc > 4);
      if (!readBurstCountGreaterThan4) begin
	 readDataFifo.deq();
	 readInProgress <= False;
      end
      for (Integer i = 0; i < 4; i = i + 1)
`ifdef AXI
	 tlp.data[(i+1)*32-1:i*32] = byteSwap(dvec[3-i]);
`elsif AVALON
	 tlp.data[(i+1)*32-1:i*32] = (dvec[3-i]);
         tlp.hit = hitReg;
`endif
      tlpOutFifo.enq(tlp);
   endrule

    interface Client        tlp;
    interface Put response;
        method Action put(TLPData#(16) tlp);
	    //$display("MemMasterEngine.put tlp=%h", tlp);
	    TLPMemoryIO3DWHeader h = unpack(tlp.data);
	    hitReg <= tlp.hit;
	    TLPMemoryIO3DWHeader hdr_3dw = unpack(tlp.data);
`ifdef AVALON
            // For Altera, the position of payload in a 3DW TLP depends on alignment.
            let quadWordAligned = isQuarWordAligned(getLowerAddr(hdr_3dw.addr, hdr_3dw.firstbe));
`endif
	    if (tlp.sof && hdr_3dw.format == MEM_READ_3DW_NO_DATA) begin
	       if (readHeaderFifo.notFull())
	          readHeaderFifo.enq(hdr_3dw);
	       else begin
		  // FIXME: should generate a response or host will lock up
	       end
	    end
	    else begin
               //FIXME: should rewrite to allow burst write.
               if (tlp.sof && writeHeaderFifo.notFull())
		  writeHeaderFifo.enq(hdr_3dw);
	    end
	endmethod
    endinterface
    interface Get request = toGet(tlpOutFifo);
    endinterface: tlp
    interface PhysMemMaster master;
    interface PhysMemWriteClient write_client;
        interface Get    writeReq;
	  method ActionValue#(PhysMemRequest#(32)) get();
	     let hdr = writeHeaderFifo.first;
	     writeHeaderFifo.deq;
	     writeDataFifo.enq(hdr);
	     let burstLen = truncate(hdr.length << 2);
	     return PhysMemRequest { addr: extend(writeHeaderFifo.first.addr) << 2, burstLen: burstLen, tag: truncate(writeHeaderFifo.first.tag)};
	  endmethod
       endinterface
        interface Get writeData;
	  method ActionValue#(MemData#(32)) get();
	     let hdr <- toGet(writeDataFifo).get();
`ifdef AXI
	     let data = byteSwap(hdr.data);
`elsif AVALON
	     let data = hdr.data;
`endif
	     return MemData { data: data, tag: truncate(hdr.tag), last: True};
	  endmethod
       endinterface
        interface Put       writeDone;
	  method Action put(Bit#(MemTagSize) resp);
	  endmethod
       endinterface
     endinterface
    interface PhysMemReadClient read_client;
        interface Get    readReq;
	  method ActionValue#(PhysMemRequest#(32)) get();
	     let hdr = readHeaderFifo.first;
	     readHeaderFifo.deq;
	     //$display("req_ar hdr.length=%d hdr.addr=%h", hdr.length, hdr.addr);
	     readDataFifo.enq(hdr);
	     let burstLen = truncate(hdr.length << 2);
	     return PhysMemRequest { addr: extend(readHeaderFifo.first.addr) << 2, burstLen: burstLen, tag: truncate(readHeaderFifo.first.tag)};
	    endmethod
       endinterface
        interface Put readData;
	  method Action put(MemData#(32) resp) if (completionMimo.enqReadyN(1));
	     Vector#(1, Bit#(32)) vec = cons(resp.data, nil);
	     completionMimo.enq(1, vec);
	  endmethod
	endinterface
    endinterface
    endinterface: master
endmodule: mkMemMasterEngine

interface MemInterrupt;
    interface Client#(TLPData#(16), TLPData#(16)) tlp;
    interface Put#(Tuple2#(Bit#(64),Bit#(32))) interruptRequest;
    interface Get#(Tuple2#(Bit#(64),Bit#(32))) interruptTrace;
endinterface

typedef struct {
   Bit#(64) addr;
   Bit#(32) data;
   Bool     mswIsZero;
   Bool     lswIsZero;
   } InterruptRequest deriving (Bits);

(* synthesize *)
module mkMemInterrupt#(PciId my_id)(MemInterrupt);
    FIFOF#(InterruptRequest) interruptRequestFifo <- mkFIFOF();
    FIFOF#(InterruptRequest) interruptTraceFifo <- mkSizedFIFOF(4);
    Reg#(Maybe#(Bit#(32))) interruptSecondHalf <- mkReg(tagged Invalid);
    Reg#(TLPTag) tlpTag <- mkReg(0);
    FIFOF#(TLPData#(16)) tlpOutFifo <- mkSizedFIFOF(8);

    function Bool isQuarWordAligned(Bit#(7) lower_addr);
       return (lower_addr[2:0]==3'b0);
    endfunction

    rule interruptTlpOut if (interruptRequestFifo.notEmpty &&& interruptSecondHalf matches tagged Invalid);
       TLPData#(16) tlp = defaultValue;
       tlp.sof = True;
       tlp.eof = False;
       tlp.hit = 7'h00;
       tlp.be = 16'hffff;

       let deqInterruptRequestFifo = False;
       let sendInterrupt = False;

       let interruptRequest = interruptRequestFifo.first;
       let interruptAddr = interruptRequest.addr;
       let interruptData = interruptRequest.data;
       let mswIsZero = interruptRequest.mswIsZero;
       let lswIsZero = interruptRequest.lswIsZero;

`ifdef AXI
       let dataInSecondTlp = False;
`elsif AVALON
       let dataInSecondTlp = isQuarWordAligned(truncate(interruptAddr));
`endif

       if (mswIsZero && lswIsZero) begin
	  // do not write to 0 -- it wedges the host
	  deqInterruptRequestFifo = True;
       end
       else if (mswIsZero) begin
          TLPMemoryIO3DWHeader hdr_3dw = defaultValue();
          hdr_3dw.format = MEM_WRITE_3DW_DATA;
	  //hdr_3dw.pkttype = MEM_READ_WRITE;
          hdr_3dw.tag = tlpTag;
          hdr_3dw.reqid = my_id;
          hdr_3dw.length = 1;
          hdr_3dw.firstbe = '1;
          hdr_3dw.lastbe = '0;
          hdr_3dw.addr = interruptAddr[31:2];
`ifdef AXI
	  hdr_3dw.data = byteSwap(interruptData);
`elsif AVALON
	  hdr_3dw.data = interruptData;
`endif

	  tlp.data = pack(hdr_3dw);
          if (dataInSecondTlp) begin
	     tlp.eof = False;
             interruptSecondHalf <= tagged Valid interruptData;
          end
	  else begin
	     tlp.eof = True;
	     deqInterruptRequestFifo = True;
	  end
	  sendInterrupt = True;
       end
       else begin
	  TLPMemory4DWHeader hdr_4dw = defaultValue;
	  hdr_4dw.format = MEM_WRITE_4DW_DATA;
	  //hdr_4dw.pkttype = MEM_READ_WRITE;
	  hdr_4dw.tag = tlpTag;
	  hdr_4dw.reqid = my_id;
	  hdr_4dw.nosnoop = SNOOPING_REQD;
	  hdr_4dw.addr = interruptAddr[40-1:2];
	  hdr_4dw.length = 1;
	  hdr_4dw.firstbe = 4'hf;
	  hdr_4dw.lastbe = 0;
	  tlp.data = pack(hdr_4dw);

	  sendInterrupt = True;
	  interruptSecondHalf <= tagged Valid interruptData;
       end

       if (deqInterruptRequestFifo) begin
	  interruptRequestFifo.deq();
       end
       if (sendInterrupt)
	  tlpOutFifo.enq(tlp);
    endrule

    rule interruptTlpDataOut if (interruptSecondHalf matches tagged Valid .interruptData);
       TLPData#(16) tlp = defaultValue;
       tlp.sof = False;
       tlp.eof = True;
       tlp.hit = 7'h00;
       tlp.be = 16'hf000;
`ifdef AXI
       tlp.data[7+8*15:8*12] = byteSwap(interruptData);
`elsif AVALON
       tlp.data[7+8*15:8*12] = interruptData;
`endif
       tlpOutFifo.enq(tlp);
       interruptSecondHalf <= tagged Invalid;
       interruptRequestFifo.deq();
    endrule

    interface Client        tlp;
    interface Put response;
        method Action put(TLPData#(16) tlp);
	endmethod
    endinterface
    interface Get request = toGet(tlpOutFifo);
    endinterface: tlp
    interface Put interruptRequest;
       method Action put(Tuple2#(Bit#(64),Bit#(32)) intr);
	  match { .addr, .data } = intr;
	  Bool mswIsZero = (addr[63:32] == 0);
	  Bool lswIsZero = (addr[31:0] == 0);
	  let interruptRequest = InterruptRequest { addr: addr, data: data, mswIsZero: mswIsZero, lswIsZero: lswIsZero };
          interruptRequestFifo.enq(interruptRequest);
	  if (interruptTraceFifo.notFull())
	      interruptTraceFifo.enq(interruptRequest);
       endmethod
    endinterface
    interface Get interruptTrace;
       method ActionValue#(Tuple2#(Bit#(64),Bit#(32))) get();
           let req = interruptTraceFifo.first();
           interruptTraceFifo.deq();
	   return tuple2(req.addr, req.data);
       endmethod
    endinterface
endmodule: mkMemInterrupt

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import FIFOF        :: *;
import Vector       :: *;
import GetPut       :: *;
import Connectable  :: *;
import MIMO         :: *;
import PCIE         :: *;
import DefaultValue :: *;
import ClientServer :: *;
import MemTypes     :: *;

//
// Top interface: PCIe transaction level packets (TLPs)
// Bottom interface: MemMaster that sends read/write requests to an MemSlave
// Also sources interrupt MSIX requests
interface MemMasterEngine;
    interface Client#(TLPData#(16), TLPData#(16)) tlp;
    interface PhysMemMaster#(32,32) master;
endinterface

`ifdef XILINX
   `define AXI
`elsif BSIM
   `define AXI
`elsif ALTERA
   `define AVALON
`elsif VSIM
   `define AVALON
`endif

(* synthesize *)
module mkMemMasterEngine#(PciId my_id)(MemMasterEngine);
    Reg#(Bit#(7)) hitReg <- mkReg(0);
    FIFOF#(TLPMemoryIO3DWHeader) readHeaderFifo <- mkSizedFIFOF(8);
    FIFOF#(TLPMemoryIO3DWHeader) readDataFifo <- mkSizedFIFOF(8);
    FIFOF#(TLPMemoryIO3DWHeader) writeHeaderFifo <- mkSizedFIFOF(8);
    FIFOF#(TLPMemoryIO3DWHeader) writeDataFifo <- mkSizedFIFOF(8);
    FIFOF#(TLPData#(16)) tlpOutFifo <- mkSizedFIFOF(8);
    Reg#(TLPTag) tlpTag <- mkReg(0);

    MIMOConfiguration mimoCfg = defaultValue;
    MIMO#(1,4,4,Bit#(32)) completionMimo <- mkMIMO(mimoCfg);
   Reg#(TLPLength)             readBurstCount <- mkReg(0);
   Reg#(LUInt#(4))     completionMimoDeqCount <- mkReg(0);
   Reg#(Bool)      readBurstCountGreaterThan4 <- mkReg(False);
   Reg#(Bool)                  readInProgress <- mkReg(False);
   Reg#(Bit#(7))               address <- mkReg(0);

   function Bool isQuarWordAligned(Bit#(7) lower_addr);
      return (lower_addr[2:0]==3'b0);
   endfunction

   rule completionHeader if (!readInProgress && readDataFifo.notEmpty() && completionMimo.deqReadyN(1));
      let hdr = readDataFifo.first;
      TLPLength rbc = hdr.length;

      Vector#(4, Bit#(32)) dvec = unpack(0);
`ifdef AXI
      dvec = completionMimo.first();
      completionMimo.deq(1);
`elsif AVALON
      let quadWordAligned = isQuarWordAligned(getLowerAddr(hdr.addr, hdr.firstbe));
      // if quad-word aligned, insert bubble.
      if (!quadWordAligned) begin
         dvec = completionMimo.first();
         completionMimo.deq(1);
      end
`endif
      //$display("completionHeader length=%d rbc=%d addr=%x", hdr.length, rbc, hdr.addr);
      TLPCompletionHeader completion = defaultValue;
      completion.format = MEM_WRITE_3DW_DATA;
      completion.pkttype = COMPLETION;
      completion.relaxed = hdr.relaxed;
      completion.nosnoop = hdr.nosnoop;
      completion.length = hdr.length;
      completion.tclass = hdr.tclass;
      completion.cmplid = my_id;
      completion.tag = truncate(hdr.tag);
      completion.bytecount = 4;
      completion.reqid = hdr.reqid;
      completion.loweraddr = getLowerAddr(hdr.addr, hdr.firstbe);
`ifdef AXI
      completion.data = byteSwap(dvec[0]);
`elsif AVALON
      if (!quadWordAligned) begin
         completion.data = (dvec[0]);
      end
      else begin
         completion.data = unpack(0);
      end
`endif
      TLPData#(16) tlp = defaultValue;
      tlp.data = pack(completion);
      tlp.sof = True;

`ifdef AXI
      tlp.eof = (rbc == 1) ? True : False;
`elsif AVALON
      tlp.eof = (rbc == 1 && !quadWordAligned) ? True : False;
`endif
      tlp.be = 16'hFFFF;
      tlp.hit = hitReg;
      tlpOutFifo.enq(tlp);

`ifdef AXI
      rbc = rbc - 1;
`elsif AVALON
      if (!quadWordAligned) begin
         rbc = rbc - 1;
      end
`endif
      readBurstCount <= rbc;
      completionMimoDeqCount <= truncate(min(4,unpack(rbc)));
      readInProgress <= (rbc != 0);
      readBurstCountGreaterThan4 <= (rbc > 4);
      if (rbc == 0) begin
	 readDataFifo.deq;
      end
   endrule

    function Bit#(16) tlpBe(TLPLength len);
       if (len == 0)
	  return 0;
       else if (len == 1)
	  return 16'hf000;
       else if (len == 2)
	  return 16'hff00;
       else if (len == 3)
	  return 16'hfff0;
       else
	  return 16'hffff;
    endfunction

   rule continuation if (readInProgress && completionMimo.deqReadyN(completionMimoDeqCount));
      let rbc = readBurstCount;
      TLPData#(16) tlp = defaultValue;
      Vector#(4, Bit#(32)) dvec = unpack(0);
      tlp.sof = False;
      dvec = completionMimo.first();
      completionMimo.deq(completionMimoDeqCount);

      if (readBurstCountGreaterThan4) begin
	 rbc = rbc - 4;
	 tlp.be = tlpBe(4);
	 tlp.eof = False;
      end
      else begin
	 tlp.be = tlpBe(rbc);
	 //$display("tlp.data=%h tlp.be=%h", tlp.data, tlp.be);
	 tlp.eof = True;
	 rbc = 0;
      end

      readBurstCount <= rbc;
      completionMimoDeqCount <= truncate(min(4,unpack(rbc)));
      readBurstCountGreaterThan4 <= (rbc > 4);
      if (!readBurstCountGreaterThan4) begin
	 readDataFifo.deq();
	 readInProgress <= False;
      end
      for (Integer i = 0; i < 4; i = i + 1)
`ifdef AXI
	 tlp.data[(i+1)*32-1:i*32] = byteSwap(dvec[3-i]);
`elsif AVALON
	 tlp.data[(i+1)*32-1:i*32] = (dvec[3-i]);
         tlp.hit = hitReg;
`endif
      tlpOutFifo.enq(tlp);
   endrule

    interface Client        tlp;
    interface Put response;
        method Action put(TLPData#(16) tlp);
	    //$display("MemMasterEngine.put tlp=%h", tlp);
	    TLPMemoryIO3DWHeader h = unpack(tlp.data);
	    hitReg <= tlp.hit;
	    TLPMemoryIO3DWHeader hdr_3dw = unpack(tlp.data);
`ifdef AVALON
            // For Altera, the position of payload in a 3DW TLP depends on alignment.
            let quadWordAligned = isQuarWordAligned(getLowerAddr(hdr_3dw.addr, hdr_3dw.firstbe));
`endif
	    if (tlp.sof && hdr_3dw.format == MEM_READ_3DW_NO_DATA) begin
	       if (readHeaderFifo.notFull())
	          readHeaderFifo.enq(hdr_3dw);
	       else begin
		  // FIXME: should generate a response or host will lock up
	       end
	    end
	    else begin
               //FIXME: should rewrite to allow burst write.
               if (tlp.sof && writeHeaderFifo.notFull())
		  writeHeaderFifo.enq(hdr_3dw);
	    end
	endmethod
    endinterface
    interface Get request = toGet(tlpOutFifo);
    endinterface: tlp
    interface PhysMemMaster master;
    interface PhysMemWriteClient write_client;
        interface Get    writeReq;
	  method ActionValue#(PhysMemRequest#(32)) get();
	     let hdr = writeHeaderFifo.first;
	     writeHeaderFifo.deq;
	     writeDataFifo.enq(hdr);
	     let burstLen = truncate(hdr.length << 2);
	     return PhysMemRequest { addr: extend(writeHeaderFifo.first.addr) << 2, burstLen: burstLen, tag: truncate(writeHeaderFifo.first.tag)};
	  endmethod
       endinterface
        interface Get writeData;
	  method ActionValue#(MemData#(32)) get();
	     let hdr <- toGet(writeDataFifo).get();
`ifdef AXI
	     let data = byteSwap(hdr.data);
`elsif AVALON
	     let data = hdr.data;
`endif
	     return MemData { data: data, tag: truncate(hdr.tag), last: True};
	  endmethod
       endinterface
        interface Put       writeDone;
	  method Action put(Bit#(MemTagSize) resp);
	  endmethod
       endinterface
     endinterface
    interface PhysMemReadClient read_client;
        interface Get    readReq;
	  method ActionValue#(PhysMemRequest#(32)) get();
	     let hdr = readHeaderFifo.first;
	     readHeaderFifo.deq;
	     //$display("req_ar hdr.length=%d hdr.addr=%h", hdr.length, hdr.addr);
	     readDataFifo.enq(hdr);
	     let burstLen = truncate(hdr.length << 2);
	     return PhysMemRequest { addr: extend(readHeaderFifo.first.addr) << 2, burstLen: burstLen, tag: truncate(readHeaderFifo.first.tag)};
	    endmethod
       endinterface
        interface Put readData;
	  method Action put(MemData#(32) resp) if (completionMimo.enqReadyN(1));
	     Vector#(1, Bit#(32)) vec = cons(resp.data, nil);
	     completionMimo.enq(1, vec);
	  endmethod
	endinterface
    endinterface
    endinterface: master
endmodule: mkMemMasterEngine

interface MemInterrupt;
    interface Client#(TLPData#(16), TLPData#(16)) tlp;
    interface Put#(Tuple2#(Bit#(64),Bit#(32))) interruptRequest;
    interface Get#(Tuple2#(Bit#(64),Bit#(32))) interruptTrace;
endinterface

typedef struct {
   Bit#(64) addr;
   Bit#(32) data;
   Bool     mswIsZero;
   Bool     lswIsZero;
   } InterruptRequest deriving (Bits);

(* synthesize *)
module mkMemInterrupt#(PciId my_id)(MemInterrupt);
    FIFOF#(InterruptRequest) interruptRequestFifo <- mkFIFOF();
    FIFOF#(InterruptRequest) interruptTraceFifo <- mkSizedFIFOF(4);
    Reg#(Maybe#(Bit#(32))) interruptSecondHalf <- mkReg(tagged Invalid);
    Reg#(TLPTag) tlpTag <- mkReg(0);
    FIFOF#(TLPData#(16)) tlpOutFifo <- mkSizedFIFOF(8);

    function Bool isQuarWordAligned(Bit#(7) lower_addr);
       return (lower_addr[2:0]==3'b0);
    endfunction

    rule interruptTlpOut if (interruptRequestFifo.notEmpty &&& interruptSecondHalf matches tagged Invalid);
       TLPData#(16) tlp = defaultValue;
       tlp.sof = True;
       tlp.eof = False;
       tlp.hit = 7'h00;
       tlp.be = 16'hffff;

       let interruptRequested = True;
       let sendInterrupt = False;

       let interruptRequest = interruptRequestFifo.first;
       let interruptAddr = interruptRequest.addr;
       let interruptData = interruptRequest.data;
       let mswIsZero = interruptRequest.mswIsZero;
       let lswIsZero = interruptRequest.lswIsZero;

       let quadWordAligned = isQuarWordAligned(truncate(interruptAddr));

       if (mswIsZero && lswIsZero) begin
	  // do not write to 0 -- it wedges the host
	  interruptRequested = False;
       end
       else if (mswIsZero) begin
          TLPMemoryIO3DWHeader hdr_3dw = defaultValue();
          hdr_3dw.format = MEM_WRITE_3DW_DATA;
	  //hdr_3dw.pkttype = MEM_READ_WRITE;
          hdr_3dw.tag = tlpTag;
          hdr_3dw.reqid = my_id;
          hdr_3dw.length = 1;
          hdr_3dw.firstbe = '1;
          hdr_3dw.lastbe = '0;
          hdr_3dw.addr = interruptAddr[31:2];
`ifdef AXI
	  hdr_3dw.data = byteSwap(interruptData);
`elsif AVALON
	  hdr_3dw.data = interruptData;
`endif

	  tlp.data = pack(hdr_3dw);
`ifdef AXI
	  tlp.eof = True;
`elsif AVALON
          tlp.eof = (!quadWordAligned) ? True : False;
          if (quadWordAligned) begin
             interruptSecondHalf <= tagged Valid interruptData;
          end
`endif
	  sendInterrupt = True;
	  interruptRequested = False;
       end
       else begin
	  TLPMemory4DWHeader hdr_4dw = defaultValue;
	  hdr_4dw.format = MEM_WRITE_4DW_DATA;
	  //hdr_4dw.pkttype = MEM_READ_WRITE;
	  hdr_4dw.tag = tlpTag;
	  hdr_4dw.reqid = my_id;
	  hdr_4dw.nosnoop = SNOOPING_REQD;
	  hdr_4dw.addr = interruptAddr[40-1:2];
	  hdr_4dw.length = 1;
	  hdr_4dw.firstbe = 4'hf;
	  hdr_4dw.lastbe = 0;
	  tlp.data = pack(hdr_4dw);

	  sendInterrupt = True;
	  interruptSecondHalf <= tagged Valid interruptData;
       end

       if (!interruptRequested) begin
	  interruptRequestFifo.deq();
       end
       if (sendInterrupt)
	  tlpOutFifo.enq(tlp);
    endrule

    rule interruptTlpDataOut if (interruptSecondHalf matches tagged Valid .interruptData);
       TLPData#(16) tlp = defaultValue;
       tlp.sof = False;
       tlp.eof = True;
       tlp.hit = 7'h00;
       tlp.be = 16'hf000;
`ifdef AXI
       tlp.data[7+8*15:8*12] = byteSwap(interruptData);
`elsif AVALON
       tlp.data[7+8*15:8*12] = interruptData;
`endif
       tlpOutFifo.enq(tlp);
       interruptSecondHalf <= tagged Invalid;
`ifdef AXI
       interruptRequestFifo.deq();
`endif
    endrule

    interface Client        tlp;
    interface Put response;
        method Action put(TLPData#(16) tlp);
	endmethod
    endinterface
    interface Get request = toGet(tlpOutFifo);
    endinterface: tlp
    interface Put interruptRequest;
       method Action put(Tuple2#(Bit#(64),Bit#(32)) intr);
	  match { .addr, .data } = intr;
	  Bool mswIsZero = (addr[63:32] == 0);
	  Bool lswIsZero = (addr[31:0] == 0);
	  let interruptRequest = InterruptRequest { addr: addr, data: data, mswIsZero: mswIsZero, lswIsZero: lswIsZero };
          interruptRequestFifo.enq(interruptRequest);
	  if (interruptTraceFifo.notFull())
	      interruptTraceFifo.enq(interruptRequest);
       endmethod
    endinterface
    interface Get interruptTrace;
       method ActionValue#(Tuple2#(Bit#(64),Bit#(32))) get();
           let req = interruptTraceFifo.first();
           interruptTraceFifo.deq();
	   return tuple2(req.addr, req.data);
       endmethod
    endinterface
endmodule: mkMemInterrupt


Fixed up TV_Taps for the new DM_Req/Rsp types between the debug module and the CPU

// Copyright (c) 2018-2019 Bluespec, Inc. All Rights Reserved.

package TV_Taps;

// ================================================================
// This package defines 'taps' on connections between
// - DM and CPU, on which DM accesses CPU GPRs, FPRs and CSRs
// - DM and memory bus, on which DM accesses memory
// Each tap snoops 'writes', and produces a corresponsing Trace_Data
// write-memory command for the Tandem Verifier, so that it keeps its
// GPRs, FPRs, CSRs and memories in sync.

// ================================================================
// BSV library imports

import Assert        :: *;
import BUtils        :: *;
import FIFOF         :: *;
import GetPut        :: *;
import ClientServer  :: *;
import Connectable   :: *;

// ----------------
// BSV additional libs

import Semi_FIFOF  :: *;
import GetPut_Aux  :: *;

// ================================================================
// Project imports

import ISA_Decls      :: *;
import DM_CPU_Req_Rsp :: *;
import TV_Info        :: *;

import AXI4_Types   :: *;
import Fabric_Defs  :: *;

// ================================================================
// DM-to-memory tap

interface DM_Mem_Tap_IFC;
   interface AXI4_Slave_IFC  #(Wd_Id, Wd_Addr, Wd_Data, Wd_User)  slave;
   interface AXI4_Master_IFC #(Wd_Id, Wd_Addr, Wd_Data, Wd_User)  master;
   interface Get #(Trace_Data)                                    trace_data_out;
endinterface

(* synthesize *)
module mkDM_Mem_Tap (DM_Mem_Tap_IFC);

   // Transactor facing DM
   AXI4_Slave_Xactor_IFC  #(Wd_Id, Wd_Addr, Wd_Data, Wd_User) slave_xactor  <- mkAXI4_Slave_Xactor;

   // Transactor facing memory bus
   AXI4_Master_Xactor_IFC #(Wd_Id, Wd_Addr, Wd_Data, Wd_User) master_xactor <- mkAXI4_Master_Xactor;

   // Tap output
   FIFOF #(Trace_Data)  f_trace_data <- mkFIFOF;

   // ----------------
   // AXI requests

   // Snoop write requests
   rule write_reqs;
      let wr_addr = slave_xactor.o_wr_addr.first;
      slave_xactor.o_wr_addr.deq;

      let wr_data = slave_xactor.o_wr_data.first;
      slave_xactor.o_wr_data.deq;

      // Pass-through
      master_xactor.i_wr_addr.enq (wr_addr);
      master_xactor.i_wr_data.enq (wr_data);

      // Tap
      Bit #(64) paddr = ?;
      Bit #(64) stval = ?;
`ifdef FABRIC64
      if (wr_data.wstrb == 'h0f) begin
	 paddr = zeroExtend (wr_addr.awaddr);
 	 stval = (wr_data.wdata & 'h_FFFF_FFFF);
      end
      else if (wr_data.wstrb == 'hf0) begin
	 paddr = zeroExtend (wr_addr.awaddr);
	 stval = ((wr_data.wdata >> 32) & 'h_FFFF_FFFF);
      end
      else
	 dynamicAssert(False, "mkDM_Mem_Tap: unsupported byte enables");
`else
      paddr = zeroExtend (wr_addr.awaddr);
      stval = zeroExtend (wr_data.wdata);
`endif
      Trace_Data td = mkTrace_MEM_WRITE (f3_SIZE_W, truncate (stval), paddr);
      f_trace_data.enq (td);
   endrule

   // Read requests, write responses and read responses are not snooped
   mkConnection (slave_xactor.o_rd_addr, master_xactor.i_rd_addr);
   mkConnection (slave_xactor.i_wr_resp, master_xactor.o_wr_resp);
   mkConnection (slave_xactor.i_rd_data, master_xactor.o_rd_data);

   // ================================================================
   // INTERFACE

   // Facing DM
   interface slave  = slave_xactor.axi_side;
   // Facing bus
   interface master = master_xactor.axi_side;
   // Tap towards verifier
   interface Get trace_data_out = toGet (f_trace_data);

endmodule: mkDM_Mem_Tap

// ================================================================
// DM-to-CPU GPR tap (for writes to GPRs)

interface DM_GPR_Tap_IFC;
   interface Client #(DM_CPU_Req #(5,  XLEN), DM_CPU_Rsp #(XLEN))  client;
   interface Server #(DM_CPU_Req #(5,  XLEN), DM_CPU_Rsp #(XLEN))  server;
   interface Get #(Trace_Data)        trace_data_out;
endinterface

(* synthesize *)
module mkDM_GPR_Tap (DM_GPR_Tap_IFC);
   // req from DM
   FIFOF #(DM_CPU_Req #(5,  XLEN)) f_req_in     <- mkFIFOF;
   // req to CPU
   FIFOF #(DM_CPU_Req #(5,  XLEN)) f_req_out    <- mkFIFOF;
   // resp CPU->DM
   FIFOF #(DM_CPU_Rsp #(XLEN))     f_rsp        <- mkFIFOF;
   // Tap to TV
   FIFOF #(Trace_Data)             f_trace_data <- mkFIFOF;

   rule request;
      let req <- pop (f_req_in);

      // Pass-through to CPU
      f_req_out.enq(req);

      // Snoop writes and send trace data to TV
      if (req.write) begin
	 Trace_Data td;
	 td = mkTrace_GPR_WRITE (req.address, req.data);
	 f_trace_data.enq (td);
      end
   endrule

   interface Client client = toGPClient (f_req_out, f_rsp);
   interface Server server = toGPServer (f_req_in,  f_rsp);

   interface Get trace_data_out = toGet (f_trace_data);
endmodule: mkDM_GPR_Tap

// ================================================================
// DM-to-CPU FPR tap (for writes to FPRs)

`ifdef ISA_F_OR_D

interface DM_FPR_Tap_IFC;
   interface Client #(DM_CPU_Req #(5,  XLEN), DM_CPU_Rsp #(XLEN)) client;
   interface Server #(DM_CPU_Req #(5,  XLEN), DM_CPU_Rsp #(XLEN)) server;
   interface Get #(Trace_Data) trace_data_out;
endinterface

(* synthesize *)
module mkDM_FPR_Tap (DM_FPR_Tap_IFC);
   // req from DM
   FIFOF #(DM_CPU_Req #(5,  XLEN)) f_req_in     <- mkFIFOF;
   // req to CPU
   FIFOF #(DM_CPU_Req #(5,  XLEN)) f_req_out    <- mkFIFOF;
   // resp CPU->DM
   FIFOF #(DM_CPU_Rsp #(XLEN))     f_rsp        <- mkFIFOF;
   // Tap to TV
   FIFOF #(Trace_Data)             f_trace_data <- mkFIFOF;

   rule request;
      let req <- pop (f_req_in);

      // Pass-through to CPU
      f_req_out.enq(req);

      // Snoop writes and send trace data to TV
      if (req.write) begin
	 Trace_Data td;
	 td = mkTrace_FPR_WRITE (req.address, req.data);
	 f_trace_data.enq (td);
      end
   endrule

   interface Client client = toGPClient (f_req_out, f_rsp);
   interface Server server = toGPServer (f_req_in,  f_rsp);

   interface Get trace_data_out = toGet (f_trace_data);
endmodule: mkDM_FPR_Tap

`endif

// ================================================================
// DM-to-CPU CSR tap (for writes to CSRs)

interface DM_CSR_Tap_IFC;
   interface Client #(DM_CPU_Req #(12,  XLEN), DM_CPU_Rsp #(XLEN)) client;
   interface Server #(DM_CPU_Req #(12,  XLEN), DM_CPU_Rsp #(XLEN)) server;
   interface Get #(Trace_Data)  trace_data_out;
endinterface

(* synthesize *)
module mkDM_CSR_Tap (DM_CSR_Tap_IFC);
   // req from DM
   FIFOF #(DM_CPU_Req #(12,  XLEN)) f_req_in     <- mkFIFOF;
   // req to CPU
   FIFOF #(DM_CPU_Req #(12,  XLEN)) f_req_out    <- mkFIFOF;
   // resp CPU->DM
   FIFOF #(DM_CPU_Rsp #(XLEN))      f_rsp        <- mkFIFOF;
   // Tap to TV
   FIFOF #(Trace_Data)              f_trace_data <- mkFIFOF;

   rule request;
      let req <- pop (f_req_in);

      // Pass-through to CPU
      f_req_out.enq(req);

      // Snoop writes and send trace data to TV
      if (req.write) begin
	 Trace_Data td = mkTrace_CSR_WRITE (req.address, req.data);
	 f_trace_data.enq (td);
      end
   endrule

   interface Client client = toGPClient (f_req_out, f_rsp);
   interface Server server = toGPServer (f_req_in,  f_rsp);

   interface Get trace_data_out = toGet (f_trace_data);
endmodule: mkDM_CSR_Tap

// ================================================================

endpackage

// Copyright (c) 2018-2019 Bluespec, Inc. All Rights Reserved.

package TV_Taps;

// ================================================================
// This package defines 'taps' on connections between
// - DM and CPU, on which DM accesses CPU GPRs, FPRs and CSRs
// - DM and memory bus, on which DM accesses memory
// Each tap snoops 'writes', and produces a corresponsing Trace_Data
// write-memory command for the Tandem Verifier, so that it keeps its
// GPRs, FPRs, CSRs and memories in sync.

// ================================================================
// BSV library imports

import Assert        :: *;
import BUtils        :: *;
import FIFOF         :: *;
import GetPut        :: *;
import ClientServer  :: *;
import Connectable   :: *;
import Memory        :: *;

// ----------------
// BSV additional libs

import Semi_FIFOF  :: *;
import GetPut_Aux  :: *;

// ================================================================
// Project imports

import ISA_Decls  :: *;
import TV_Info    :: *;

import AXI4_Types   :: *;
import Fabric_Defs  :: *;

// ================================================================
// DM-to-memory tap

interface DM_Mem_Tap_IFC;
   interface AXI4_Slave_IFC  #(Wd_Id, Wd_Addr, Wd_Data, Wd_User)  slave;
   interface AXI4_Master_IFC #(Wd_Id, Wd_Addr, Wd_Data, Wd_User)  master;
   interface Get #(Trace_Data)                                    trace_data_out;
endinterface

(* synthesize *)
module mkDM_Mem_Tap (DM_Mem_Tap_IFC);

   // Transactor facing DM
   AXI4_Slave_Xactor_IFC  #(Wd_Id, Wd_Addr, Wd_Data, Wd_User) slave_xactor  <- mkAXI4_Slave_Xactor;

   // Transactor facing memory bus
   AXI4_Master_Xactor_IFC #(Wd_Id, Wd_Addr, Wd_Data, Wd_User) master_xactor <- mkAXI4_Master_Xactor;

   // Tap output
   FIFOF #(Trace_Data)  f_trace_data <- mkFIFOF;

   // ----------------
   // AXI requests

   // Snoop write requests
   rule write_reqs;
      let wr_addr = slave_xactor.o_wr_addr.first;
      slave_xactor.o_wr_addr.deq;

      let wr_data = slave_xactor.o_wr_data.first;
      slave_xactor.o_wr_data.deq;

      // Pass-through
      master_xactor.i_wr_addr.enq (wr_addr);
      master_xactor.i_wr_data.enq (wr_data);

      // Tap
      Bit #(64) paddr = ?;
      Bit #(64) stval = ?;
`ifdef FABRIC64
      if (wr_data.wstrb == 'h0f) begin
	 paddr = zeroExtend (wr_addr.awaddr);
 	 stval = (wr_data.wdata & 'h_FFFF_FFFF);
      end
      else if (wr_data.wstrb == 'hf0) begin
	 paddr = zeroExtend (wr_addr.awaddr);
	 stval = ((wr_data.wdata >> 32) & 'h_FFFF_FFFF);
      end
      else
	 dynamicAssert(False, "mkDM_Mem_Tap: unsupported byte enables");
`else
      paddr = zeroExtend (wr_addr.awaddr);
      stval = zeroExtend (wr_data.wdata);
`endif
      Trace_Data td = mkTrace_MEM_WRITE (f3_SIZE_W, truncate (stval), paddr);
      f_trace_data.enq (td);
   endrule

   // Read requests, write responses and read responses are not snooped
   mkConnection (slave_xactor.o_rd_addr, master_xactor.i_rd_addr);
   mkConnection (slave_xactor.i_wr_resp, master_xactor.o_wr_resp);
   mkConnection (slave_xactor.i_rd_data, master_xactor.o_rd_data);

   // ================================================================
   // INTERFACE

   // Facing DM
   interface slave  = slave_xactor.axi_side;
   // Facing bus
   interface master = master_xactor.axi_side;
   // Tap towards verifier
   interface Get trace_data_out = toGet (f_trace_data);

endmodule: mkDM_Mem_Tap

// ================================================================
// DM-to-CPU GPR tap (for writes to GPRs)

interface DM_GPR_Tap_IFC;
   interface MemoryClient #(5, XLEN)  client;
   interface MemoryServer #(5, XLEN)  server;
   interface Get #(Trace_Data)        trace_data_out;
endinterface

(* synthesize *)
module mkDM_GPR_Tap (DM_GPR_Tap_IFC);
   // req from DM
   FIFOF #(MemoryRequest #(5, XLEN)) f_req_in     <- mkFIFOF;
   // req to CPU
   FIFOF #(MemoryRequest #(5, XLEN)) f_req_out    <- mkFIFOF;
   // resp CPU->DM
   FIFOF #(MemoryResponse #(XLEN))   f_rsp        <- mkFIFOF;
   // Tap to TV
   FIFOF #(Trace_Data)               f_trace_data <- mkFIFOF;

   rule request;
      let req <- pop (f_req_in);

      // Pass-through to CPU
      f_req_out.enq(req);

      // Snoop writes and send trace data to TV
      if (req.write) begin
	 Trace_Data td;
	 td = mkTrace_GPR_WRITE (req.address, req.data);
	 f_trace_data.enq (td);
      end
   endrule

   interface MemoryClient client = toGPClient (f_req_out, f_rsp);
   interface MemoryServer server = toGPServer (f_req_in,  f_rsp);

   interface Get trace_data_out = toGet (f_trace_data);
endmodule: mkDM_GPR_Tap

// ================================================================
// DM-to-CPU FPR tap (for writes to FPRs)

`ifdef ISA_F_OR_D

interface DM_FPR_Tap_IFC;
   interface MemoryClient #(5, FLEN)  client;
   interface MemoryServer #(5, FLEN)  server;
   interface Get #(Trace_Data)        trace_data_out;
endinterface

(* synthesize *)
module mkDM_FPR_Tap (DM_FPR_Tap_IFC);
   // req from DM
   FIFOF #(MemoryRequest #(5, FLEN)) f_req_in     <- mkFIFOF;
   // req to CPU
   FIFOF #(MemoryRequest #(5, FLEN)) f_req_out    <- mkFIFOF;
   // resp CPU->DM
   FIFOF #(MemoryResponse #(FLEN))   f_rsp        <- mkFIFOF;
   // Tap to TV
   FIFOF #(Trace_Data)               f_trace_data <- mkFIFOF;

   rule request;
      let req <- pop (f_req_in);

      // Pass-through to CPU
      f_req_out.enq(req);

      // Snoop writes and send trace data to TV
      if (req.write) begin
	 Trace_Data td;
	 td = mkTrace_FPR_WRITE (req.address, req.data);
	 f_trace_data.enq (td);
      end
   endrule

   interface MemoryClient client = toGPClient (f_req_out, f_rsp);
   interface MemoryServer server = toGPServer (f_req_in,  f_rsp);

   interface Get trace_data_out = toGet (f_trace_data);
endmodule: mkDM_FPR_Tap

`endif

// ================================================================
// DM-to-CPU CSR tap (for writes to CSRs)

interface DM_CSR_Tap_IFC;
   interface MemoryClient #(12, XLEN)  client;
   interface MemoryServer #(12, XLEN)  server;
   interface Get #(Trace_Data)         trace_data_out;
endinterface

(* synthesize *)
module mkDM_CSR_Tap (DM_CSR_Tap_IFC);
   // req from DM
   FIFOF #(MemoryRequest #(12, XLEN)) f_req_in     <- mkFIFOF;
   // req to CPU
   FIFOF #(MemoryRequest #(12, XLEN)) f_req_out    <- mkFIFOF;
   // resp CPU->DM
   FIFOF #(MemoryResponse #(XLEN))    f_rsp        <- mkFIFOF;
   // Tap to TV
   FIFOF #(Trace_Data)                f_trace_data <- mkFIFOF;

   rule request;
      let req <- pop (f_req_in);

      // Pass-through to CPU
      f_req_out.enq(req);

      // Snoop writes and send trace data to TV
      if (req.write) begin
	 Trace_Data td = mkTrace_CSR_WRITE (req.address, req.data);
	 f_trace_data.enq (td);
      end
   endrule

   interface MemoryClient client = toGPClient (f_req_out, f_rsp);
   interface MemoryServer server = toGPServer (f_req_in,  f_rsp);

   interface Get trace_data_out = toGet (f_trace_data);
endmodule: mkDM_CSR_Tap

// ================================================================

endpackage