switch MemSlaveEngine back to using MIMO

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import FIFO         :: *;
import FIFOF        :: *;
import GetPut       :: *;
import Connectable  :: *;
import PCIE         :: *;
import DefaultValue :: *;
import MIMO         :: *;
import MIFO         :: *;
import Vector       :: *;
import ClientServer :: *;
import MemTypes     :: *;
import Probe        :: *;
import PcieTracer   :: *;

interface MemSlaveEngine#(numeric type buswidth);
    interface Client#(TLPData#(16), TLPData#(16)) tlp;
    interface PhysMemSlave#(40,buswidth) slave;
    interface Get#(TimestampedTlpData) trace;
    method Bool tlpOutFifoNotEmpty();
    interface Reg#(Bool) use4dw;
endinterface: MemSlaveEngine

module mkMemSlaveEngine#(PciId my_id)(MemSlaveEngine#(buswidth))
   provisos (Div#(buswidth, 8, busWidthBytes),
	     Div#(buswidth, 32, busWidthWords),
	     Bits#(Vector#(busWidthWords, Bit#(32)), buswidth),
            Log#(busWidthBytes,beatShift),
	     Add#(aaa, 32, buswidth),
	     Add#(bbb, buswidth, 256),
	     Add#(bbbb, buswidth, 1024),
	     Add#(ccc, TMul#(8, busWidthWords), 256),
	     Add#(ddd, TMul#(32, busWidthWords), 256),
	     Add#(eee, busWidthWords, 8),
      	     Add#(fff, busWidthWords, 32),
	     Add#(1, a__, busWidthWords),
	     Add#(busWidthWords, b__, 4),
	     Add#(c__, busWidthWords, 16)
      );

    let beat_shift = fromInteger(valueOf(beatShift));

    FIFOF#(TLPData#(16)) tlpOutFifo <- mkFIFOF;
    FIFOF#(TLPData#(16)) tlpInFifo <- mkFIFOF;
    FIFO#(TLPData#(16)) tlpWriteHeaderFifo <- mkFIFO;
    FIFO#(TLPLength)      writeDwCountFifo <- mkFIFO;
    FIFO#(Bool)             writeIs3dwFifo <- mkFIFO;
    FIFO#(Bool)             writeIsHeaderOnlyFifo <- mkFIFO;

    Reg#(Bit#(7)) hitReg <- mkReg(0);
    Reg#(Bool) use4dwReg <- mkReg(True);

    // default configuration for MIMO is for guarded enq() and deq().
    // However, the implicit guard only checks for space for 1 element for enq(), and availability of 1 element for deq().
    MIMOConfiguration mimoCfg = defaultValue;
    MIMO#(4,busWidthWords,32,Bit#(32)) completionMimo <- mkMIMO(mimoCfg);
    MIMO#(4,busWidthWords,32,TLPTag) completionTagMimo <- mkMIMO(mimoCfg);

    MIMO#(busWidthWords,4,32,Bit#(32)) writeDataMimo <- mkMIMO(mimoCfg);
    Reg#(Bit#(10)) writeBurstCount <- mkReg(0);
   FIFO#(Bit#(10)) writeBurstCountFifo <- mkFIFO();
    Reg#(TLPLength)  writeDwCount <- mkReg(0); // how many 4 byte (double) words to send
    Reg#(LUInt#(4))    tlpDwCount <- mkReg(0); // how many to send in the next tlp (at most 4)
    Reg#(Bool)            lastTlp <- mkReg(False); // if the next tlp sent is the last one
    Reg#(Bool)    writeInProgress <- mkReg(False);
    FIFOF#(TLPTag) writeTag <- mkSizedFIFOF(16);
    FIFOF#(TLPTag) doneTag <- mkSizedFIFOF(16);

   Wire#(Bool) writeHeaderTlpWire <- mkDWire(False);
   Probe#(Bool) writeHeaderAvailableProbe <- mkProbe;
   Wire#(Bool) writeDataMimoEnqWire <- mkDWire(False);
   Probe#(Bool) writeDataMimoEnqProbe <- mkProbe;
   let    writeDataMimoCountProbe <- mkProbe;
   let writeDataMimoEnqReadyProbe <- mkProbe;
   Probe#(Bool) writeInProgressProbe <- mkProbe;
   Probe#(Bit#(10)) writeBurstCountProbe <- mkProbe;
   rule updateProbe0;
      writeDataMimoCountProbe    <= writeDataMimo.count();
      writeDataMimoEnqReadyProbe <= writeDataMimo.enqReadyN(fromInteger(valueOf(busWidthWords)));
   endrule
   rule updateProbe2;
      writeHeaderAvailableProbe <= writeHeaderTlpWire;
      writeDataMimoEnqProbe <= writeDataMimoEnqWire;
   endrule

   rule writeHeaderTlp if (!writeInProgress && (!writeIs3dwFifo.first || writeDataMimo.deqReadyN(1)));
      writeHeaderTlpWire <= True;
      let tlp <- toGet(tlpWriteHeaderFifo).get();
      let dwCount <- toGet(writeDwCountFifo).get();
      let is3dw <- toGet(writeIs3dwFifo).get();
      let isHeaderOnly <- toGet(writeIsHeaderOnlyFifo).get();

      TLPMemory4DWHeader hdr_4dw = unpack(tlp.data);

      TLPMemoryIO3DWHeader hdr_3dw = unpack(tlp.data);
      if (is3dw) begin
	 if (hdr_3dw.format != MEM_WRITE_3DW_DATA)
	    $display("MemSlaveEngine: expecting MEM_WRITE_3DW_DATA, got %d", hdr_3dw.format);
	 Vector#(4, Bit#(32)) v = writeDataMimo.first();
	 writeDataMimo.deq(1);
	 hdr_3dw.data = byteSwap(v[0]);
	 tlp.be = 16'hffff;
	 if (dwCount == 1)
	    tlp.eof = True;
	 dwCount = dwCount - 1;
	 tlp.data = pack(hdr_3dw);
      end
      else begin
	 tlp.be = 16'hffff;
      end

      tlpOutFifo.enq(tlp);
      //$display("writeHeaderTlp dwCount=%d", dwCount);
      writeDwCount <= dwCount;
      tlpDwCount <= truncate(min(4,unpack(dwCount)));
      lastTlp <= (dwCount <= 4);
      writeInProgress <= (dwCount != 0);
      writeInProgressProbe <= (dwCount != 0);
      if (isHeaderOnly) begin
	 doneTag.enq(writeTag.first());
	 writeTag.deq();
      end

   endrule

   rule writeTlps if (writeInProgress && writeDataMimo.deqReadyN(tlpDwCount));
      TLPData#(16) tlp = defaultValue;
      tlp.sof = False;
      Vector#(4, Bit#(32)) v = unpack(0);

      // The MIMO implicit guard only checks for availability of 1 element
      // so we explicitly check for the number of elements required
      writeDataMimo.deq(tlpDwCount);
      v = writeDataMimo.first();
      let dwCount = writeDwCount - extend(pack(tlpDwCount));
      writeDwCount <= dwCount;
      tlpDwCount <= truncate(min(4,unpack(dwCount)));
      lastTlp <= (dwCount <= 4);
      if (tlpDwCount == 4)
	 tlp.be = 16'hffff;
      else if (tlpDwCount == 3)
	 tlp.be = 16'hfff0;
      else if (tlpDwCount == 2)
	 tlp.be = 16'hff00;
      else if (tlpDwCount == 1)
	 tlp.be = 16'hf000;
      tlp.eof = lastTlp;
      if (lastTlp) begin
	 writeInProgress <= False;
	 writeInProgressProbe <= False;
	 doneTag.enq(writeTag.first());
	 writeTag.deq();
	 //$display("writeDwCount=%d will be zero", writeDwCount);
      end

      for (Integer i = 0; i < 4; i = i + 1)
	 tlp.data[(i+1)*32-1:i*32] = byteSwap(v[3-i]);
      tlpOutFifo.enq(tlp);
   endrule

   Reg#(TLPTag) lastTag <- mkReg(0);
   FIFOF#(TLPData#(16)) tlpDecodeFifo <- mkFIFOF();
   Reg#(TLPLength) wordCountReg <- mkReg(0);
   FIFO#(TimestampedTlpData) tlpTraceFifo <- mkFIFO();
   Reg#(Bit#(6)) tlpAgeReg <- mkReg(0);
   Reg#(Bit#(32)) cycles <- mkReg(0);
   rule count;
      cycles <= cycles + 1;
   endrule

   rule tlpInRule;
      let tlp <- toGet(tlpInFifo).get();
      tlpDecodeFifo.enq(tlp);
   endrule

   rule handleTlpRule;
      let tlp = tlpDecodeFifo.first;
      let tlpAge = tlpAgeReg + 1;
      Bool handled = False;
      TLPMemoryIO3DWHeader h = unpack(tlp.data);
      hitReg <= tlp.hit;
      TLPMemoryIO3DWHeader hdr_3dw = unpack(tlp.data);
      TLPCompletionHeader hdr_completion = unpack(tlp.data);
      Vector#(4, Bit#(32)) vec = unpack(0);
      Vector#(4, Bit#(32)) tlpvec = unpack(tlp.data);
      let wordCount = wordCountReg;
      if (!tlp.sof) begin
	 vec = reverse(tlpvec);
	 // The MIMO implicit guard only checks for space to enqueue 1 element
	 // so we explicitly check for the number of elements required
	 // otherwise elements in the queue will be overwritten.
	 UInt#(3) count = 4;
	 if (tlp.eof) begin
	    count = truncate(unpack(wordCountReg));
	 end
	 if (completionMimo.enqReadyN(count)
	    && completionTagMimo.enqReadyN(count))
	    begin
	       wordCount = wordCountReg - extend(pack(count));
	       completionMimo.enq(count, vec);
	       Vector#(4, TLPTag) tagvec = replicate(lastTag);
	       completionTagMimo.enq(count, tagvec);
	       handled = True;
	    end
      end
      else if (hdr_3dw.format == MEM_WRITE_3DW_DATA
	       && hdr_3dw.pkttype == COMPLETION
	       && completionMimo.enqReady()
	       && completionTagMimo.enqReady()) begin
	    vec[0] = hdr_3dw.data;
            wordCount = hdr_3dw.length - 1;
	    completionMimo.enq(1, vec);
            TLPTag tag = hdr_completion.tag;
	    lastTag <= tag;
	    completionTagMimo.enq(1, replicate(tag));
	    handled = True;
      end
      wordCountReg <= wordCount;
      //$display("tlpIn handled=%d tlp=%h\n", handled, tlp);
      if (tlpAge == 15 && !handled) begin
	 let ttd = TimestampedTlpData { timestamp: cycles, source: 48, tlp: tlp };
	 tlpTraceFifo.enq(ttd);
	 handled = True;
      end
      if (handled) begin
	 tlpDecodeFifo.deq();
	 tlpAge = 0;
      end
      tlpAgeReg <= tlpAge;
   endrule

   FIFO#(PhysMemRequest#(40)) readReqFifo <- mkFIFO();
   rule readReqRule if (!writeInProgress && !writeDataMimo.deqReady());
      let req <- toGet(readReqFifo).get();
      let burstLen = req.burstLen >> beat_shift;
      let addr = req.addr;
      let arid = req.tag;

      TLPData#(16) tlp = defaultValue;
      tlp.sof = True;
      tlp.eof = True;
      tlp.hit = 7'h00;
      TLPLength tlplen = fromInteger(valueOf(busWidthWords))*extend(burstLen);
      if (addr[39:32] != 0) begin
	 TLPMemory4DWHeader hdr_4dw = defaultValue;
	 hdr_4dw.format = MEM_READ_4DW_NO_DATA;
	 hdr_4dw.tag = extend(arid);
	 hdr_4dw.reqid = my_id;
	 hdr_4dw.nosnoop = SNOOPING_REQD;
	 hdr_4dw.addr = addr[40-1:2];
	 hdr_4dw.length = tlplen;
	 hdr_4dw.firstbe = 4'hf;
	 hdr_4dw.lastbe = (tlplen > 1) ? 4'hf : 0;
	 tlp.data = pack(hdr_4dw);
	 tlp.be = 16'hffff;
      end
      else begin
	 TLPMemoryIO3DWHeader hdr_3dw = defaultValue;
	 hdr_3dw.format = MEM_READ_3DW_NO_DATA;
	 hdr_3dw.tag = extend(arid);
	 hdr_3dw.reqid = my_id;
	 hdr_3dw.nosnoop = SNOOPING_REQD;
	 hdr_3dw.addr = addr[32-1:2];
	 hdr_3dw.length = tlplen;
	 hdr_3dw.firstbe = 4'hf;
	 hdr_3dw.lastbe = (tlplen > 1) ? 4'hf : 0;
	 tlp.data = pack(hdr_3dw);
	 tlp.be = 16'hfff0;
      end
      tlpOutFifo.enq(tlp);
   endrule

    interface Client        tlp;
        interface request = toGet(tlpOutFifo);
        interface response = toPut(tlpInFifo);
    endinterface
    interface PhysMemSlave slave;
   interface PhysMemWriteServer write_server;
      interface Put writeReq;
         method Action put(PhysMemRequest#(40) req); // if (writeBurstCount == 0);
	    let burstLen = req.burstLen >> beat_shift;
	    let addr = req.addr;
	    let awid = req.tag;
	    let writeIs3dw = False;

	    TLPLength tlplen = fromInteger(valueOf(busWidthWords))*extend(burstLen);
	    TLPData#(16) tlp = defaultValue;
	    tlp.sof = True;
	    tlp.eof = False;
	    tlp.hit = 7'h00;
	    tlp.be = 16'hffff;

	    $display("slave.writeAddr tlplen=%d burstLen=%d", tlplen, burstLen);
	    if ((addr >> 32) != 0) begin
	       TLPMemory4DWHeader hdr_4dw = defaultValue;
	       hdr_4dw.format = MEM_WRITE_4DW_DATA;
	       hdr_4dw.tag = extend(awid);
	       hdr_4dw.reqid = my_id;
	       hdr_4dw.nosnoop = SNOOPING_REQD;
	       hdr_4dw.addr = addr[40-1:2];
	       hdr_4dw.length = tlplen;
	       hdr_4dw.firstbe = 4'hf;
	       hdr_4dw.lastbe = (tlplen > 1) ? 4'hf : 0;
	       tlp.data = pack(hdr_4dw);
	    end
	    else begin
	       writeIs3dw = True;
	       TLPMemoryIO3DWHeader hdr_3dw = defaultValue;
	       hdr_3dw.format = MEM_WRITE_3DW_DATA;
	       hdr_3dw.tag = extend(awid);
	       hdr_3dw.reqid = my_id;
	       hdr_3dw.nosnoop = SNOOPING_REQD;
	       hdr_3dw.addr = addr[32-1:2];
	       hdr_3dw.length = tlplen;
	       hdr_3dw.firstbe = 4'hf;
	       hdr_3dw.lastbe = (tlplen > 1) ? 4'hf : 0;

	       tlp.be = 16'hfff0; // no data word in this TLP

	       tlp.data = pack(hdr_3dw);
	    end
	    tlpWriteHeaderFifo.enq(tlp);
	    writeDwCountFifo.enq(tlplen);
	    writeIs3dwFifo.enq(writeIs3dw);
	    writeIsHeaderOnlyFifo.enq(writeIs3dw && tlplen == 1);
	    writeBurstCountFifo.enq(extend(burstLen));
	    writeTag.enq(extend(awid));
         endmethod
	endinterface
      interface Put writeData;
         method Action put(MemData#(buswidth) wdata)
	      provisos (Bits#(Vector#(busWidthWords, Bit#(32)), busWidth)) if (writeDataMimo.enqReadyN(fromInteger(valueOf(busWidthWords))));

	    let burstLen = writeBurstCount;
	    if (burstLen == 0) begin
	       burstLen <- toGet(writeBurstCountFifo).get();
	    end
	    writeBurstCount <= extend(burstLen-1);
	    writeBurstCountProbe <= extend(burstLen-1);

	    Vector#(busWidthWords, Bit#(32)) v = unpack(wdata.data);
	    writeDataMimo.enq(fromInteger(valueOf(busWidthWords)), v);
	    writeDataMimoEnqWire <= True;
         endmethod
       endinterface
      interface Get writeDone;
         method ActionValue#(Bit#(MemTagSize)) get();
	      let tag = doneTag.first();
	      doneTag.deq();
	      return truncate(tag);
           endmethod
	endinterface
   endinterface
   interface PhysMemReadServer read_server;
      interface Put readReq;
         method Action put(PhysMemRequest#(40) req);
	    readReqFifo.enq(req);
         endmethod
       endinterface
      interface Get     readData;
         method ActionValue#(MemData#(buswidth)) get() if (completionMimo.deqReady()
							   && completionTagMimo.deqReady());
	      let data_v = completionMimo.first;
	      let tag_v = completionTagMimo.first;
	      completionMimo.deq(fromInteger(valueOf(busWidthWords)));
	      completionTagMimo.deq(fromInteger(valueOf(busWidthWords)));
              Bit#(buswidth) v = 0;
	      for (Integer i = 0; i < valueOf(busWidthWords); i = i+1)
		 v[(i+1)*32-1:i*32] = byteSwap(data_v[i]);
	      return MemData { data: v, tag: truncate(tag_v[0]), last: True};
           endmethod
	endinterface
   endinterface
    endinterface: slave
   method Bool tlpOutFifoNotEmpty() = tlpOutFifo.notEmpty;
   interface Reg use4dw = use4dwReg;
   interface Get trace = fifoToGet(tlpTraceFifo);
endmodule: mkMemSlaveEngine

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import FIFO         :: *;
import FIFOF        :: *;
import GetPut       :: *;
import Connectable  :: *;
import PCIE         :: *;
import DefaultValue :: *;
import MIMO         :: *;
import MIFO         :: *;
import Vector       :: *;
import ClientServer :: *;
import MemTypes     :: *;
import Probe        :: *;
import PcieTracer   :: *;

interface MemSlaveEngine#(numeric type buswidth);
    interface Client#(TLPData#(16), TLPData#(16)) tlp;
    interface PhysMemSlave#(40,buswidth) slave;
    interface Get#(TimestampedTlpData) trace;
    method Bool tlpOutFifoNotEmpty();
    interface Reg#(Bool) use4dw;
endinterface: MemSlaveEngine

module mkMemSlaveEngine#(PciId my_id)(MemSlaveEngine#(buswidth))
   provisos (Div#(buswidth, 8, busWidthBytes),
	     Div#(buswidth, 32, busWidthWords),
	     Bits#(Vector#(busWidthWords, Bit#(32)), buswidth),
            Log#(busWidthBytes,beatShift),
	     Add#(aaa, 32, buswidth),
	     Add#(bbb, buswidth, 256),
	     Add#(ccc, TMul#(8, busWidthWords), 64),
	     Add#(ddd, TMul#(32, busWidthWords), 256),
	     Add#(eee, busWidthWords, 8),
	     Add#(1, a__, busWidthWords),
	     Add#(busWidthWords, b__, 4),
	     Add#(c__, busWidthWords, 16)
      );

    let beat_shift = fromInteger(valueOf(beatShift));

    FIFOF#(TLPData#(16)) tlpOutFifo <- mkFIFOF;
    FIFOF#(TLPData#(16)) tlpInFifo <- mkFIFOF;
    FIFO#(TLPData#(16)) tlpWriteHeaderFifo <- mkFIFO;
    FIFO#(TLPLength)      writeDwCountFifo <- mkFIFO;
    FIFO#(Bool)             writeIs3dwFifo <- mkFIFO;
    FIFO#(Bool)             writeIsHeaderOnlyFifo <- mkFIFO;

    Reg#(Bit#(7)) hitReg <- mkReg(0);
    Reg#(Bool) use4dwReg <- mkReg(True);

    // default configuration for MIMO is for guarded enq() and deq().
    // However, the implicit guard only checks for space for 1 element for enq(), and availability of 1 element for deq().
    MIMOConfiguration mimoCfg = defaultValue;
   MIFO#(4,busWidthWords,16,Bit#(32)) completionMimo <- mkMIFO();
   MIFO#(4,busWidthWords,16,TLPTag) completionTagMimo <- mkMIFO();

    MIMO#(busWidthWords,4,16,Bit#(32)) writeDataMimo <- mkMIMO(mimoCfg);
    Reg#(Bit#(10)) writeBurstCount <- mkReg(0);
   FIFO#(Bit#(10)) writeBurstCountFifo <- mkFIFO();
    Reg#(TLPLength)  writeDwCount <- mkReg(0); // how many 4 byte (double) words to send
    Reg#(LUInt#(4))    tlpDwCount <- mkReg(0); // how many to send in the next tlp (at most 4)
    Reg#(Bool)            lastTlp <- mkReg(False); // if the next tlp sent is the last one
    Reg#(Bool)    writeInProgress <- mkReg(False);
    FIFOF#(TLPTag) writeTag <- mkSizedFIFOF(16);
    FIFOF#(TLPTag) doneTag <- mkSizedFIFOF(16);

   Wire#(Bool) writeHeaderTlpWire <- mkDWire(False);
   Probe#(Bool) writeHeaderAvailableProbe <- mkProbe;
   Wire#(Bool) writeDataMimoEnqWire <- mkDWire(False);
   Probe#(Bool) writeDataMimoEnqProbe <- mkProbe;
   let    writeDataMimoCountProbe <- mkProbe;
   let writeDataMimoEnqReadyProbe <- mkProbe;
   Probe#(Bool) writeInProgressProbe <- mkProbe;
   Probe#(Bit#(10)) writeBurstCountProbe <- mkProbe;
   rule updateProbe0;
      writeDataMimoCountProbe    <= writeDataMimo.count();
      writeDataMimoEnqReadyProbe <= writeDataMimo.enqReadyN(fromInteger(valueOf(busWidthWords)));
   endrule
   rule updateProbe2;
      writeHeaderAvailableProbe <= writeHeaderTlpWire;
      writeDataMimoEnqProbe <= writeDataMimoEnqWire;
   endrule

   rule writeHeaderTlp if (!writeInProgress && (!writeIs3dwFifo.first || writeDataMimo.deqReadyN(1)));
      writeHeaderTlpWire <= True;
      let tlp <- toGet(tlpWriteHeaderFifo).get();
      let dwCount <- toGet(writeDwCountFifo).get();
      let is3dw <- toGet(writeIs3dwFifo).get();
      let isHeaderOnly <- toGet(writeIsHeaderOnlyFifo).get();

      TLPMemory4DWHeader hdr_4dw = unpack(tlp.data);

      TLPMemoryIO3DWHeader hdr_3dw = unpack(tlp.data);
      if (is3dw) begin
	 if (hdr_3dw.format != MEM_WRITE_3DW_DATA)
	    $display("MemSlaveEngine: expecting MEM_WRITE_3DW_DATA, got %d", hdr_3dw.format);
	 Vector#(4, Bit#(32)) v = writeDataMimo.first();
	 writeDataMimo.deq(1);
	 hdr_3dw.data = byteSwap(v[0]);
	 tlp.be = 16'hffff;
	 if (dwCount == 1)
	    tlp.eof = True;
	 dwCount = dwCount - 1;
	 tlp.data = pack(hdr_3dw);
      end
      else begin
	 tlp.be = 16'hffff;
      end

      tlpOutFifo.enq(tlp);
      //$display("writeHeaderTlp dwCount=%d", dwCount);
      writeDwCount <= dwCount;
      tlpDwCount <= truncate(min(4,unpack(dwCount)));
      lastTlp <= (dwCount <= 4);
      writeInProgress <= (dwCount != 0);
      writeInProgressProbe <= (dwCount != 0);
      if (isHeaderOnly) begin
	 doneTag.enq(writeTag.first());
	 writeTag.deq();
      end

   endrule

   rule writeTlps if (writeInProgress && writeDataMimo.deqReadyN(tlpDwCount));
      TLPData#(16) tlp = defaultValue;
      tlp.sof = False;
      Vector#(4, Bit#(32)) v = unpack(0);

      // The MIMO implicit guard only checks for availability of 1 element
      // so we explicitly check for the number of elements required
      writeDataMimo.deq(tlpDwCount);
      v = writeDataMimo.first();
      let dwCount = writeDwCount - extend(pack(tlpDwCount));
      writeDwCount <= dwCount;
      tlpDwCount <= truncate(min(4,unpack(dwCount)));
      lastTlp <= (dwCount <= 4);
      if (tlpDwCount == 4)
	 tlp.be = 16'hffff;
      else if (tlpDwCount == 3)
	 tlp.be = 16'hfff0;
      else if (tlpDwCount == 2)
	 tlp.be = 16'hff00;
      else if (tlpDwCount == 1)
	 tlp.be = 16'hf000;
      tlp.eof = lastTlp;
      if (lastTlp) begin
	 writeInProgress <= False;
	 writeInProgressProbe <= False;
	 doneTag.enq(writeTag.first());
	 writeTag.deq();
	 //$display("writeDwCount=%d will be zero", writeDwCount);
      end

      for (Integer i = 0; i < 4; i = i + 1)
	 tlp.data[(i+1)*32-1:i*32] = byteSwap(v[3-i]);
      tlpOutFifo.enq(tlp);
   endrule

   Reg#(TLPTag) lastTag <- mkReg(0);
   FIFOF#(TLPData#(16)) tlpDecodeFifo <- mkFIFOF();
   Reg#(TLPLength) wordCountReg <- mkReg(0);
   FIFO#(TimestampedTlpData) tlpTraceFifo <- mkFIFO();
   Reg#(Bit#(6)) tlpAgeReg <- mkReg(0);
   Reg#(Bit#(32)) cycles <- mkReg(0);
   rule count;
      cycles <= cycles + 1;
   endrule

   rule tlpInRule;
      let tlp <- toGet(tlpInFifo).get();
      tlpDecodeFifo.enq(tlp);
   endrule

   rule handleTlpRule;
      let tlp = tlpDecodeFifo.first;
      let tlpAge = tlpAgeReg + 1;
      Bool handled = False;
      TLPMemoryIO3DWHeader h = unpack(tlp.data);
      hitReg <= tlp.hit;
      TLPMemoryIO3DWHeader hdr_3dw = unpack(tlp.data);
      TLPCompletionHeader hdr_completion = unpack(tlp.data);
      Vector#(4, Bit#(32)) vec = unpack(0);
      Vector#(4, Bit#(32)) tlpvec = unpack(tlp.data);
      let wordCount = wordCountReg;
      if (!tlp.sof) begin
	 vec = reverse(tlpvec);
	 // The MIMO implicit guard only checks for space to enqueue 1 element
	 // so we explicitly check for the number of elements required
	 // otherwise elements in the queue will be overwritten.
	 UInt#(3) count = 4;
	 if (tlp.eof) begin
	    count = truncate(unpack(wordCountReg));
	 end
	 if (completionMimo.enqReady()
	    && completionTagMimo.enqReady())
	    begin
	       wordCount = wordCountReg - extend(pack(count));
	       completionMimo.enq(count, vec);
	       Vector#(4, TLPTag) tagvec = replicate(lastTag);
	       completionTagMimo.enq(count, tagvec);
	       handled = True;
	    end
      end
      else if (hdr_3dw.format == MEM_WRITE_3DW_DATA
	       && hdr_3dw.pkttype == COMPLETION
	       && completionMimo.enqReady()
	       && completionTagMimo.enqReady()) begin
	    vec[0] = hdr_3dw.data;
            wordCount = hdr_3dw.length - 1;
	    completionMimo.enq(1, vec);
            TLPTag tag = hdr_completion.tag;
	    lastTag <= tag;
	    completionTagMimo.enq(1, replicate(tag));
	    handled = True;
      end
      wordCountReg <= wordCount;
      //$display("tlpIn handled=%d tlp=%h\n", handled, tlp);
      if (tlpAge == 15 && !handled) begin
	 let ttd = TimestampedTlpData { timestamp: cycles, source: 48, tlp: tlp };
	 tlpTraceFifo.enq(ttd);
	 handled = True;
      end
      if (handled) begin
	 tlpDecodeFifo.deq();
	 tlpAge = 0;
      end
      tlpAgeReg <= tlpAge;
   endrule

   FIFO#(PhysMemRequest#(40)) readReqFifo <- mkFIFO();
   rule readReqRule if (!writeInProgress && !writeDataMimo.deqReady());
      let req <- toGet(readReqFifo).get();
      let burstLen = req.burstLen >> beat_shift;
      let addr = req.addr;
      let arid = req.tag;

      TLPData#(16) tlp = defaultValue;
      tlp.sof = True;
      tlp.eof = True;
      tlp.hit = 7'h00;
      TLPLength tlplen = fromInteger(valueOf(busWidthWords))*extend(burstLen);
      if (addr[39:32] != 0) begin
	 TLPMemory4DWHeader hdr_4dw = defaultValue;
	 hdr_4dw.format = MEM_READ_4DW_NO_DATA;
	 hdr_4dw.tag = extend(arid);
	 hdr_4dw.reqid = my_id;
	 hdr_4dw.nosnoop = SNOOPING_REQD;
	 hdr_4dw.addr = addr[40-1:2];
	 hdr_4dw.length = tlplen;
	 hdr_4dw.firstbe = 4'hf;
	 hdr_4dw.lastbe = (tlplen > 1) ? 4'hf : 0;
	 tlp.data = pack(hdr_4dw);
	 tlp.be = 16'hffff;
      end
      else begin
	 TLPMemoryIO3DWHeader hdr_3dw = defaultValue;
	 hdr_3dw.format = MEM_READ_3DW_NO_DATA;
	 hdr_3dw.tag = extend(arid);
	 hdr_3dw.reqid = my_id;
	 hdr_3dw.nosnoop = SNOOPING_REQD;
	 hdr_3dw.addr = addr[32-1:2];
	 hdr_3dw.length = tlplen;
	 hdr_3dw.firstbe = 4'hf;
	 hdr_3dw.lastbe = (tlplen > 1) ? 4'hf : 0;
	 tlp.data = pack(hdr_3dw);
	 tlp.be = 16'hfff0;
      end
      tlpOutFifo.enq(tlp);
   endrule

    interface Client        tlp;
        interface request = toGet(tlpOutFifo);
        interface response = toPut(tlpInFifo);
    endinterface
    interface PhysMemSlave slave;
   interface PhysMemWriteServer write_server;
      interface Put writeReq;
         method Action put(PhysMemRequest#(40) req); // if (writeBurstCount == 0);
	    let burstLen = req.burstLen >> beat_shift;
	    let addr = req.addr;
	    let awid = req.tag;
	    let writeIs3dw = False;

	    TLPLength tlplen = fromInteger(valueOf(busWidthWords))*extend(burstLen);
	    TLPData#(16) tlp = defaultValue;
	    tlp.sof = True;
	    tlp.eof = False;
	    tlp.hit = 7'h00;
	    tlp.be = 16'hffff;

	    $display("slave.writeAddr tlplen=%d burstLen=%d", tlplen, burstLen);
	    if ((addr >> 32) != 0) begin
	       TLPMemory4DWHeader hdr_4dw = defaultValue;
	       hdr_4dw.format = MEM_WRITE_4DW_DATA;
	       hdr_4dw.tag = extend(awid);
	       hdr_4dw.reqid = my_id;
	       hdr_4dw.nosnoop = SNOOPING_REQD;
	       hdr_4dw.addr = addr[40-1:2];
	       hdr_4dw.length = tlplen;
	       hdr_4dw.firstbe = 4'hf;
	       hdr_4dw.lastbe = (tlplen > 1) ? 4'hf : 0;
	       tlp.data = pack(hdr_4dw);
	    end
	    else begin
	       writeIs3dw = True;
	       TLPMemoryIO3DWHeader hdr_3dw = defaultValue;
	       hdr_3dw.format = MEM_WRITE_3DW_DATA;
	       hdr_3dw.tag = extend(awid);
	       hdr_3dw.reqid = my_id;
	       hdr_3dw.nosnoop = SNOOPING_REQD;
	       hdr_3dw.addr = addr[32-1:2];
	       hdr_3dw.length = tlplen;
	       hdr_3dw.firstbe = 4'hf;
	       hdr_3dw.lastbe = (tlplen > 1) ? 4'hf : 0;

	       tlp.be = 16'hfff0; // no data word in this TLP

	       tlp.data = pack(hdr_3dw);
	    end
	    tlpWriteHeaderFifo.enq(tlp);
	    writeDwCountFifo.enq(tlplen);
	    writeIs3dwFifo.enq(writeIs3dw);
	    writeIsHeaderOnlyFifo.enq(writeIs3dw && tlplen == 1);
	    writeBurstCountFifo.enq(extend(burstLen));
	    writeTag.enq(extend(awid));
         endmethod
	endinterface
      interface Put writeData;
         method Action put(MemData#(buswidth) wdata)
	      provisos (Bits#(Vector#(busWidthWords, Bit#(32)), busWidth)) if (writeDataMimo.enqReadyN(fromInteger(valueOf(busWidthWords))));

	    let burstLen = writeBurstCount;
	    if (burstLen == 0) begin
	       burstLen <- toGet(writeBurstCountFifo).get();
	    end
	    writeBurstCount <= extend(burstLen-1);
	    writeBurstCountProbe <= extend(burstLen-1);

	    Vector#(busWidthWords, Bit#(32)) v = unpack(wdata.data);
	    writeDataMimo.enq(fromInteger(valueOf(busWidthWords)), v);
	    writeDataMimoEnqWire <= True;
         endmethod
       endinterface
      interface Get writeDone;
         method ActionValue#(Bit#(MemTagSize)) get();
	      let tag = doneTag.first();
	      doneTag.deq();
	      return truncate(tag);
           endmethod
	endinterface
   endinterface
   interface PhysMemReadServer read_server;
      interface Put readReq;
         method Action put(PhysMemRequest#(40) req);
	    readReqFifo.enq(req);
         endmethod
       endinterface
      interface Get     readData;
         method ActionValue#(MemData#(buswidth)) get() if (completionMimo.deqReady()
							   && completionTagMimo.deqReady());
	      let data_v = completionMimo.first;
	      let tag_v = completionTagMimo.first;
	      completionMimo.deq();
	      completionTagMimo.deq();
              Bit#(buswidth) v = 0;
	      for (Integer i = 0; i < valueOf(busWidthWords); i = i+1)
		 v[(i+1)*32-1:i*32] = byteSwap(data_v[i]);
	      return MemData { data: v, tag: truncate(tag_v[0]), last: True};
           endmethod
	endinterface
   endinterface
    endinterface: slave
   method Bool tlpOutFifoNotEmpty() = tlpOutFifo.notEmpty;
   interface Reg use4dw = use4dwReg;
   interface Get trace = fifoToGet(tlpTraceFifo);
endmodule: mkMemSlaveEngine


fixed apparent deadlock in write path

fixed apparent deadlock in write path

// Copyright (c) 2015 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import Clocks            :: *;
import Vector            :: *;
import FIFOF             :: *;
import FIFO              :: *;
import SpecialFIFOs      :: *;
import GetPut            :: *;
import Connectable       :: *;
import StmtFSM           :: *;
import Portal            :: *;
import Leds              :: *;
import Top               :: *;
import MemTypes          :: *;
import HostInterface     :: *;
import CtrlMux           :: *;
import ClientServer      :: *;
import PCIE              :: *;

`ifndef PinType
`define PinType Empty
`endif

typedef `PinType PinType;
typedef `NumberOfMasters NumberOfMasters;

module  mkXsimHost#(Clock derivedClock, Reset derivedReset)(XsimHost);
   interface derivedClock = derivedClock;
   interface derivedReset = derivedReset;
endmodule

interface XsimTop;
   method Bit#(1) rd();
   method ActionValue#(Bit#(32)) directoryEntry();
   method Action read(Bit#(32) addr);
   method ActionValue#(Bit#(32)) readData();
   method Action write(Bit#(32) addr, Bit#(32) data);

   interface Clock singleClock;
   interface Reset singleReset;
endinterface

module  mkXsimTop(XsimTop);
   //let divider <- mkClockDivider(2);
   Clock derivedClock <- exposeCurrentClock; //= divider.fastClock;
   Clock single_clock = derivedClock; //divider.slowClock;
   Reset derivedReset <- exposeCurrentReset;
   //let sr <- mkReset(2, True, single_clock);
   Reset single_reset = derivedReset; //sr.new_rst;

   XsimHost host <- mkXsimHost(clocked_by single_clock, reset_by single_reset, derivedClock, derivedReset);
   ConnectalTop#(PhysAddrWidth,DataBusWidth,PinType,NumberOfMasters) top <- mkConnectalTop(
`ifdef IMPORT_HOSTIF
       host,
`endif
       clocked_by single_clock, reset_by single_reset);
   //mapM(uncurry(mkConnection),zip(top.masters, host.mem_servers), clocked_by single_clock, reset_by single_reset);

   let intr_mux <- mkInterruptMux(top.interrupt);
   rule int_rule;
      //interruptLevel(truncate(pack(intr_mux)));
   endrule

   Reg#(Bool) readingDirectory <- mkReg(True);
   Reg#(Bool) token <- mkReg(True);
   Reg#(Bit#(8)) portalNumber <- mkReg(0);
   Reg#(Bit#(8)) offset <- mkReg(8'h10);
   Reg#(Bit#(32)) idReg <- mkReg(0);
   FIFO#(Bit#(32)) directoryFifo <- mkFIFO();
   rule readDirectory if (readingDirectory && portalNumber < 16 && token);
      let o = offset;
      let pn = portalNumber;
      top.slave.read_server.readReq.put(PhysMemRequest { addr: (extend(portalNumber) << 16) | extend(offset), burstLen: 4, tag: (o == 8'h10) ? 7 : 9});
      if (offset == 8'h14) begin
	 o = 8'h10;
	 pn = pn + 1;
      end
      else begin
	 o = o + 4;
      end
      offset <= o;
      portalNumber <= pn;
      token <= False;
   endrule
   rule directoryValue if (readingDirectory && !token);
      let memData <- top.slave.read_server.readData.get();

      let t = token;
      if (memData.tag == 7) begin
	 idReg <= memData.data;
	 t = True;
      end
      else begin
	 directoryFifo.enq((extend(memData.data[0]) << 31) | idReg);
	 // if this is not the last one, then read the next portal
	 t = (memData.data[0] == 0);
      end
      token <= t;
   endrule

   FIFO#(Bit#(32)) writeDataFifo <- mkFIFO();
   rule writedatarule;
      let data <- toGet(writeDataFifo).get();
      top.slave.write_server.writeData.put(MemData { data: data, tag: 0, last: True });
   endrule

   method Bit#(1) rd();
      return readingDirectory ? 1 : 0;
   endmethod
   method ActionValue#(Bit#(32)) directoryEntry() if (readingDirectory);
      let v <- toGet(directoryFifo).get();
      if (v[31] == 1)
	 readingDirectory <= False;
      return v;
   endmethod

   method Action read(Bit#(32) addr) if (!readingDirectory);
      $display("read addr=%h", addr);
      top.slave.read_server.readReq.put(PhysMemRequest { addr: addr, burstLen: 4, tag: 0 });
   endmethod

   method ActionValue#(Bit#(32)) readData() if (!readingDirectory);
      let memData <- top.slave.read_server.readData.get();
      $display("read data=%h", memData.data);
      return memData.data;
   endmethod

   method Action write(Bit#(32) addr, Bit#(32) data);
      top.slave.write_server.writeReq.put(PhysMemRequest { addr: addr, burstLen: 4, tag: 0});
      writeDataFifo.enq(data);
   endmethod

`ifdef BSIMRESPONDER
   `BSIMRESPONDER (top.pins);
`endif
   interface Clock singleClock = single_clock;
   interface Reset singleReset = single_reset;
endmodule

// Copyright (c) 2015 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import Clocks            :: *;
import Vector            :: *;
import FIFOF             :: *;
import FIFO              :: *;
import SpecialFIFOs      :: *;
import GetPut            :: *;
import Connectable       :: *;
import StmtFSM           :: *;
import Portal            :: *;
import Leds              :: *;
import Top               :: *;
import MemTypes          :: *;
import HostInterface     :: *;
import CtrlMux           :: *;
import ClientServer      :: *;
import PCIE              :: *;

`ifndef PinType
`define PinType Empty
`endif

typedef `PinType PinType;
typedef `NumberOfMasters NumberOfMasters;

module  mkXsimHost#(Clock derivedClock, Reset derivedReset)(XsimHost);
   interface derivedClock = derivedClock;
   interface derivedReset = derivedReset;
endmodule

interface XsimTop;
   method Bit#(1) rd();
   method ActionValue#(Bit#(32)) directoryEntry();
   method Action read(Bit#(32) addr);
   method ActionValue#(Bit#(32)) readData();
   method Action write(Bit#(32) addr, Bit#(32) data);

   interface Clock singleClock;
   interface Reset singleReset;
endinterface

module  mkXsimTop(XsimTop);
   //let divider <- mkClockDivider(2);
   Clock derivedClock <- exposeCurrentClock; //= divider.fastClock;
   Clock single_clock = derivedClock; //divider.slowClock;
   Reset derivedReset <- exposeCurrentReset;
   //let sr <- mkReset(2, True, single_clock);
   Reset single_reset = derivedReset; //sr.new_rst;

   XsimHost host <- mkXsimHost(clocked_by single_clock, reset_by single_reset, derivedClock, derivedReset);
   ConnectalTop#(PhysAddrWidth,DataBusWidth,PinType,NumberOfMasters) top <- mkConnectalTop(
`ifdef IMPORT_HOSTIF
       host,
`endif
       clocked_by single_clock, reset_by single_reset);
   //mapM(uncurry(mkConnection),zip(top.masters, host.mem_servers), clocked_by single_clock, reset_by single_reset);

   let intr_mux <- mkInterruptMux(top.interrupt);
   rule int_rule;
      //interruptLevel(truncate(pack(intr_mux)));
   endrule

   Reg#(Bool) readingDirectory <- mkReg(True);
   Reg#(Bool) token <- mkReg(True);
   Reg#(Bit#(8)) portalNumber <- mkReg(0);
   Reg#(Bit#(8)) offset <- mkReg(8'h10);
   Reg#(Bit#(32)) idReg <- mkReg(0);
   FIFO#(Bit#(32)) directoryFifo <- mkFIFO();
   rule readDirectory if (readingDirectory && portalNumber < 16 && token);
      let o = offset;
      let pn = portalNumber;
      top.slave.read_server.readReq.put(PhysMemRequest { addr: (extend(portalNumber) << 16) | extend(offset), burstLen: 4, tag: (o == 8'h10) ? 7 : 9});
      if (offset == 8'h14) begin
	 o = 8'h10;
	 pn = pn + 1;
      end
      else begin
	 o = o + 4;
      end
      offset <= o;
      portalNumber <= pn;
      token <= False;
   endrule
   rule directoryValue if (readingDirectory && !token);
      let memData <- top.slave.read_server.readData.get();

      let t = token;
      if (memData.tag == 7) begin
	 idReg <= memData.data;
	 t = True;
      end
      else begin
	 directoryFifo.enq((extend(memData.data[0]) << 31) | idReg);
	 // if this is not the last one, then read the next portal
	 t = (memData.data[0] == 0);
      end
      token <= t;
   endrule

   method Bit#(1) rd();
      return readingDirectory ? 1 : 0;
   endmethod
   method ActionValue#(Bit#(32)) directoryEntry() if (readingDirectory);
      let v <- toGet(directoryFifo).get();
      if (v[31] == 1)
	 readingDirectory <= False;
      return v;
   endmethod

   method Action read(Bit#(32) addr) if (!readingDirectory);
      $display("read addr=%h", addr);
      top.slave.read_server.readReq.put(PhysMemRequest { addr: addr, burstLen: 4, tag: 0 });
   endmethod

   method ActionValue#(Bit#(32)) readData() if (!readingDirectory);
      let memData <- top.slave.read_server.readData.get();
      $display("read data=%h", memData.data);
      return memData.data;
   endmethod

   method Action write(Bit#(32) addr, Bit#(32) data);
      top.slave.write_server.writeReq.put(PhysMemRequest { addr: addr, burstLen: 4, tag: 0});
      top.slave.write_server.writeData.put(MemData { data: data, tag: 0, last: True });
   endmethod

`ifdef BSIMRESPONDER
   `BSIMRESPONDER (top.pins);
`endif
   interface Clock singleClock = single_clock;
   interface Reset singleReset = single_reset;
endmodule


Fixed default verbosity for FBox_Top to 0

// Copyright (c) 2016-2019 Bluespec, Inc. All Rights Reserved

package CPU_Stage2;

// ================================================================
// This is Stage 2 of the CPU.
// It is the "DM" stage ("Data Memory"), which is the main function.

// However, this stage also contains all other (potentially) long-latency
// operations:
//    MBox ("M" extension ops, integer multiply/divide)
//    FDBox ("FD" extension ops, single and double precision floating point)

// This stage sends out Tandem Verifier information for pipelined instructions

// Note: $displays are indented by (stage num x 4) spaces.
// for traditional pipeline display
//     IF
//         DM
//             WB
// i.e., 8 spaces for this stage.

// ================================================================
// Exports

export
CPU_Stage2_IFC (..),
mkCPU_Stage2;

// ================================================================
// BSV library imports

import FIFOF        :: *;
import GetPut       :: *;
import ClientServer :: *;
import ConfigReg    :: *;

// ----------------
// BSV additional libs

import Cur_Cycle  :: *;

// ================================================================
// Project imports

import ISA_Decls     :: *;

import TV_Info       :: *;

import CPU_Globals   :: *;
import Near_Mem_IFC  :: *;
import CSR_RegFile   :: *;    // For SATP, SSTATUS, MSTATUS

`ifdef SHIFT_SERIAL
import Shifter_Box  :: *;
`endif

`ifdef ISA_M
import RISCV_MBox  :: *;
`endif

`ifdef ISA_F
import FBox_Top    :: *;
import FBox_Core   :: *;   // For fv_nanbox function
`endif

// ================================================================
// Interface

interface CPU_Stage2_IFC;
   // ---- Reset
   interface Server #(Token, Token) server_reset;

   // ---- Output
   (* always_ready *)
   method Output_Stage2  out;

   (* always_ready *)
   method Action deq;

   // ---- Input
   (* always_ready *)
   method Action enq (Data_Stage1_to_Stage2 x);

   (* always_ready *)
   method Action set_full (Bool full);
endinterface

// ================================================================
// Implementation module

module mkCPU_Stage2 #(Bit #(4)         verbosity,
		      CSR_RegFile_IFC  csr_regfile,    // for SATP and SSTATUS: TODO carry in Data_Stage1_to_Stage2
		      DMem_IFC         dcache)
                    (CPU_Stage2_IFC);

   FIFOF #(Token) f_reset_reqs <- mkFIFOF;
   FIFOF #(Token) f_reset_rsps <- mkFIFOF;

   Reg #(Bool)                  rg_resetting  <- mkReg (False);
   Reg #(Bool)                  rg_full       <- mkReg (False);
   Reg #(Data_Stage1_to_Stage2) rg_stage2     <- mkRegU;    // From Stage 1

   // ----------------
   // Serial shifter box

`ifdef SHIFT_SERIAL
   Shifter_Box_IFC shifter_box <- mkShifter_Box;
`endif

   // ----------------
   // Integer multiply/divide box

`ifdef ISA_M
   RISCV_MBox_IFC mbox <- mkRISCV_MBox;
`endif

   // ----------------
   // Floating point box

`ifdef ISA_F
   FBox_Top_IFC fbox <- mkFBox_Top (0);
`endif

   // ----------------

   let bypass_base = Bypass {bypass_state: BYPASS_RD_NONE,
			     rd:           rg_stage2.rd,
			     rd_val:       rg_stage2.val1
			     };

`ifdef ISA_F
   let fbypass_base = FBypass {bypass_state: BYPASS_RD_NONE,
			       rd:           rg_stage2.rd,
			       rd_val:       rg_stage2.fval1
			       };
`endif

   let data_to_stage3_base = Data_Stage2_to_Stage3 {priv:      rg_stage2.priv,
						    pc:        rg_stage2.pc,
						    instr:     rg_stage2.instr,
`ifdef ISA_F
                                                    rd_in_fpr: False,
                                                    upd_flags: False,
                                                    fpr_flags: 0,
                                                    frd_val  : rg_stage2.fval1,
`endif
						    rd_valid:  False,
						    rd:        rg_stage2.rd,
						    rd_val:    rg_stage2.val1};

   let  trap_info_dmem = Trap_Info {epc:      rg_stage2.pc,
				    exc_code: dcache.exc_code,
				    tval:     rg_stage2.addr };

`ifdef ISA_F
   // The FBox can only generate ILLEGAL Instruction exceptions
   let  trap_info_fbox = Trap_Info {epc:      rg_stage2.pc,
				    exc_code: exc_code_ILLEGAL_INSTRUCTION,
				    tval:     0 };
`endif

   // ----------------------------------------------------------------
   // BEHAVIOR

   rule rl_reset_begin;
      f_reset_reqs.deq;
      rg_full <= False;
      rg_resetting <= True;
`ifdef ISA_F
      fbox.server_reset.request.put (?);
`endif
   endrule

   rule rl_reset_end (rg_resetting);
      rg_resetting <= False;

`ifdef ISA_F
      let res <- fbox.server_reset.response.get;
`endif

      f_reset_rsps.enq (?);
   endrule

   // ----------------
   // Combinational output function

   function Output_Stage2 fv_out;
      Output_Stage2 output_stage2 = ?;

      // This stage is empty
      if (! rg_full) begin
	 output_stage2 = Output_Stage2 {
              ostatus         : OSTATUS_EMPTY
            , trap_info       : ?
            , data_to_stage3  : ?
            , bypass          : no_bypass
`ifdef ISA_F
            , fbypass         : no_fbypass
`endif
`ifdef INCLUDE_TANDEM_VERIF
	    , trace_data      : ?
`endif
	 };
      end

      // This stage is just relaying ALU results from previous stage to next stage
      else if (rg_stage2.op_stage2 == OP_Stage2_ALU) begin
	 let data_to_stage3 = data_to_stage3_base;
	 data_to_stage3.rd_valid = True;

	 let bypass = bypass_base;
	 bypass.bypass_state = BYPASS_RD_RDVAL;

`ifdef INCLUDE_TANDEM_VERIF
	 let trace_data = rg_stage2.trace_data;
`endif

	 output_stage2 = Output_Stage2 {
              ostatus         : OSTATUS_PIPE
            , trap_info       : ?
            , data_to_stage3  : data_to_stage3
            , bypass          : bypass
`ifdef ISA_F
            , fbypass         : no_fbypass
`endif
`ifdef INCLUDE_TANDEM_VERIF
            , trace_data      : trace_data
`endif
         };
      end

      // This stage is doing a LOAD or AMO
      else if (   (rg_stage2.op_stage2 == OP_Stage2_LD)
`ifdef ISA_A
	       || (rg_stage2.op_stage2 == OP_Stage2_AMO)
`endif
	       )
	 begin
	    let ostatus = (  (! dcache.valid)
			   ? OSTATUS_BUSY
			   : (  dcache.exc
			      ? OSTATUS_NONPIPE
			      : OSTATUS_PIPE));

	    WordXL result = truncate (dcache.word64);

            let funct3 = instr_funct3 (rg_stage2.instr);

	    let data_to_stage3 = data_to_stage3_base;
	    data_to_stage3.rd_valid = (ostatus == OSTATUS_PIPE);

`ifdef ISA_F
            data_to_stage3.rd_in_fpr = rg_stage2.rd_in_fpr;
            // A FPR load
            if (rg_stage2.rd_in_fpr) begin
`ifdef ISA_D
               // Both FLW and FLD are legal instructions
               // A FLW result
               if (funct3 == f3_FLW)
                  // needs nan-boxing when destined for a DP register file
                  data_to_stage3.frd_val = fv_nanbox (dcache.word64);

               // A FLD result
               else
                  data_to_stage3.frd_val = dcache.word64;
`else
               // Only FLW is a legal instruction
               data_to_stage3.frd_val = truncate (dcache.word64);
`endif
            end
`endif
            // GPR loads
	    data_to_stage3.rd_val   = result;

            // Update the bypass channel, if not trapping (NONPIPE)
	    let bypass = bypass_base;
`ifdef ISA_F
	    let fbypass = fbypass_base;
`endif

	    if (ostatus != OSTATUS_NONPIPE) begin
`ifdef ISA_F
               // Bypassing FPR value.
               if (rg_stage2.rd_in_fpr) begin
		  // Choose one of the following two options

		  // Option 1: longer critical path, since the data is bypassed back into previous stage.
		  // We use data_to_stage3.rd_val since nanboxing has been done.
		  // fbypass.bypass_state = ((ostatus == OSTATUS_PIPE) ? BYPASS_RD_RDVAL : BYPASS_RD);
		  // fbypass.rd_val       = data_to_stage3.frd_val;

		  // Option 2: shorter critical path, since the data is not bypassed into previous stage,
		  // (the bypassing is effectively delayed until the next stage).
		  fbypass.bypass_state = BYPASS_RD;
               end
`endif

               // Bypassing GPR values
               if (rg_stage2.rd != 0) begin    // TODO: is this test necessary?
		  // Choose one of the following two options

		  // Option 1: longer critical path, since the data is bypassed back into previous stage.
		  // We use data_to_stage3.rd_val since nanboxing has been done.
		  // bypass.bypass_state = ((ostatus == OSTATUS_PIPE) ? BYPASS_RD_RDVAL : BYPASS_RD);
		  // bypass.rd_val       = result;

		  // Option 2: shorter critical path, since the data is not bypassed into previous stage,
		  // (the bypassing is effectively delayed until the next stage).
		  bypass.bypass_state = BYPASS_RD;
	       end
	    end

`ifdef INCLUDE_TANDEM_VERIF
	    let trace_data   = rg_stage2.trace_data;
`ifdef ISA_F
            if (rg_stage2.rd_in_fpr) begin
               trace_data.word5 = data_to_stage3.frd_val;

               // Update MSTATUS.FS in trace packet
               trace_data = fv_trace_update_mstatus_fs (trace_data, fs_xs_dirty);
            end else
`endif
               trace_data.word1 = data_to_stage3.rd_val;
`endif
            output_stage2 = Output_Stage2 {
                 ostatus         : ostatus
               , trap_info       : trap_info_dmem
               , data_to_stage3  : data_to_stage3
               , bypass          : bypass
`ifdef ISA_F
               , fbypass         : fbypass
`endif
`ifdef INCLUDE_TANDEM_VERIF
               , trace_data      : trace_data
`endif
            };
	 end

      // This stage is doing a STORE
      else if (rg_stage2.op_stage2 == OP_Stage2_ST) begin
	 let ostatus = (  (! dcache.valid)
			     ? OSTATUS_BUSY
			     : (  dcache.exc
				? OSTATUS_NONPIPE
				: OSTATUS_PIPE));

	 let data_to_stage3 = data_to_stage3_base;
	 data_to_stage3.rd_valid = (ostatus == OSTATUS_PIPE);
	 data_to_stage3.rd       = 0;

`ifdef INCLUDE_TANDEM_VERIF
	 let trace_data   = rg_stage2.trace_data;
`endif

	 output_stage2 = Output_Stage2 {
              ostatus         : ostatus
            , trap_info       : trap_info_dmem
            , data_to_stage3  : data_to_stage3
            , bypass          : no_bypass
`ifdef ISA_F
            , fbypass         : no_fbypass
`endif
`ifdef INCLUDE_TANDEM_VERIF
            , trace_data      : trace_data
`endif
         };
      end

`ifdef SHIFT_SERIAL
      // This stage is doing a serial shift
      else if (rg_stage2.op_stage2 == OP_Stage2_SH) begin
	 let ostatus = ((! shifter_box.valid) ? OSTATUS_BUSY : OSTATUS_PIPE);

	 let result = shifter_box.word;

	 let data_to_stage3 = data_to_stage3_base;
	 data_to_stage3.rd_valid = (ostatus == OSTATUS_PIPE);
	 data_to_stage3.rd_val   = result;

	 let bypass = bypass_base;
	 bypass.bypass_state = ((ostatus == OSTATUS_PIPE) ? BYPASS_RD_RDVAL : BYPASS_RD);
	 bypass.rd_val       = result;

`ifdef INCLUDE_TANDEM_VERIF
	 let trace_data   = rg_stage2.trace_data;
	 trace_data.word1 = result;
`endif

	 output_stage2 = Output_Stage2 {
              ostatus         : ostatus
            , trap_info       : ?
            , data_to_stage3  : data_to_stage3
            , bypass          : bypass
`ifdef ISA_F
            , fbypass         : no_fbypass
`endif
`ifdef INCLUDE_TANDEM_VERIF
            , trace_data      : trace_data
`endif
         };
      end
`endif

`ifdef ISA_M
      // This stage is doing an integer multiply/divide
      else if (rg_stage2.op_stage2 == OP_Stage2_M) begin
	 let ostatus = ((! mbox.valid) ? OSTATUS_BUSY : OSTATUS_PIPE);

	 let result = mbox.word;

	 let data_to_stage3 = data_to_stage3_base;
	 data_to_stage3.rd_valid = (ostatus == OSTATUS_PIPE);
	 data_to_stage3.rd_val   = result;

	 let bypass = bypass_base;
	 bypass.bypass_state = ((ostatus == OSTATUS_PIPE) ? BYPASS_RD_RDVAL : BYPASS_RD);
	 bypass.rd_val       = result;

`ifdef INCLUDE_TANDEM_VERIF
	 let trace_data   = rg_stage2.trace_data;
	 trace_data.word1 = result;
`endif

	 output_stage2 = Output_Stage2 {
              ostatus         : ostatus
            , trap_info       : ?
            , data_to_stage3  : data_to_stage3
            , bypass          : bypass
`ifdef ISA_F
            , fbypass         : no_fbypass
`endif
`ifdef INCLUDE_TANDEM_VERIF
            , trace_data      : trace_data
`endif
         };
      end
`endif

`ifdef ISA_F
      // This stage is doing a floating point op
      else if (rg_stage2.op_stage2 == OP_Stage2_FD) begin
	 let ostatus = ((! fbox.valid) ? OSTATUS_BUSY : OSTATUS_PIPE);

         // Extract fields from FBOX result
	 match {.value, .fflags} = fbox.word;

	 let data_to_stage3      = data_to_stage3_base;
	 data_to_stage3.rd_valid = (ostatus == OSTATUS_PIPE);
`ifdef ISA_D
	 data_to_stage3.frd_val  = value;
`else
	 data_to_stage3.frd_val  = truncate (value);
`endif
         data_to_stage3.rd_in_fpr= rg_stage2.rd_in_fpr;
         data_to_stage3.upd_flags= True;
         data_to_stage3.fpr_flags= fflags;

         // result is meant for a FPR
	 let bypass              = bypass_base;
         let fbypass             = fbypass_base;
         if (rg_stage2.rd_in_fpr) begin
            fbypass.bypass_state    = ((ostatus==OSTATUS_PIPE) ? BYPASS_RD_RDVAL
                                                               : BYPASS_RD);
`ifdef ISA_D
            fbypass.rd_val          = value;
`else
            fbypass.rd_val          = truncate (value);
`endif
         end

         // result is meant for a GPR
         else begin
            bypass.bypass_state     = ((ostatus==OSTATUS_PIPE) ? BYPASS_RD_RDVAL
                                                               : BYPASS_RD);
`ifdef RV64
            bypass.rd_val           = (value);
            data_to_stage3.rd_val   = value;
`else
            bypass.rd_val           = truncate (value);
            data_to_stage3.rd_val   = truncate (value);
`endif
         end

         // -----
`ifdef INCLUDE_TANDEM_VERIF
	 let trace_data = rg_stage2.trace_data;

         if (rg_stage2.rd_in_fpr) begin
            trace_data.word5 = data_to_stage3.frd_val;
         end else begin
            trace_data.word1 = data_to_stage3.rd_val;
         end

         // Update MSTATUS.FS and FCSR.FFLAGS in trace packet
         trace_data = fv_trace_update_mstatus_fs (trace_data, fs_xs_dirty);
         trace_data = fv_trace_update_fcsr_fflags (trace_data, fflags);

`endif

	 output_stage2 = Output_Stage2 {
              ostatus         : ostatus
            , trap_info       : trap_info_fbox
            , data_to_stage3  : data_to_stage3
            , bypass          : bypass
`ifdef ISA_F
            , fbypass         : fbypass
`endif
`ifdef INCLUDE_TANDEM_VERIF
            , trace_data      : trace_data
`endif
         };
      end
`endif

      return output_stage2;
   endfunction

   // ----------------
   // Initiate DM, Shifter box, MBox or FBox op

   function Action fa_enq (Data_Stage1_to_Stage2 x);
      action
	 rg_stage2  <= x;

	 let funct3 = instr_funct3 (x.instr);

	 // If DMem access, initiate it
`ifdef ISA_A
	 Bool op_stage2_amo = (x.op_stage2 == OP_Stage2_AMO);
	 Bit #(7) amo_funct7 = x.val1 [6:0];
`else
	 Bool op_stage2_amo = False;
	 Bit #(7) amo_funct7 = 0;
`endif
	 if ((x.op_stage2 == OP_Stage2_LD) || (x.op_stage2 == OP_Stage2_ST) || op_stage2_amo) begin
	    WordXL   mstatus     = csr_regfile.read_mstatus;
`ifdef ISA_PRIV_S
	    Bit #(1) sstatus_SUM = (csr_regfile.read_sstatus) [18];
`else
	    Bit #(1) sstatus_SUM = 0;
`endif
	    Bit #(1) mstatus_MXR = mstatus [19];
	    Priv_Mode  mem_priv = x.priv;
	    if (mstatus [17] == 1'b1) begin
	       mem_priv = mstatus [12:11];
	       // $display ("    S2.fa_enq: mem_priv %0d => %0d (mstatus.MPP) due to mstatus.MPRV", x.priv, mem_priv);
	    end

	    CacheOp cache_op = ?;
	    if      (x.op_stage2 == OP_Stage2_LD)  cache_op = CACHE_LD;
	    else if (x.op_stage2 == OP_Stage2_ST)  cache_op = CACHE_ST;
`ifdef ISA_A
	    else if (x.op_stage2 == OP_Stage2_AMO) cache_op = CACHE_AMO;
`endif

            // Prepare the store value
`ifdef RV64
            Bit# (64) wdata_from_gpr = x.val2;
`else
            Bit# (64) wdata_from_gpr = zeroExtend (x.val2);
`endif

`ifdef ISA_F
`ifdef ISA_D
            Bit# (64) wdata_from_fpr = x.fval2;
`else
            Bit# (64) wdata_from_fpr = zeroExtend (x.fval2);
`endif
`endif
	    dcache.req (cache_op,
			instr_funct3 (x.instr),
`ifdef ISA_A
			amo_funct7,
`endif
			x.addr,
`ifdef ISA_F
			(x.rs_frm_fpr ? wdata_from_fpr : wdata_from_gpr),
`else
			wdata_from_gpr,
`endif
			mem_priv,
			sstatus_SUM,
			mstatus_MXR,
			csr_regfile.read_satp);
	 end

`ifdef SHIFT_SERIAL
	 // If Shifter box op, initiate it
	 else if (x.op_stage2 == OP_Stage2_SH)
	    shifter_box.req (unpack (funct3 [2]), x.val1, x.val2);
`endif

`ifdef ISA_M
	 // If MBox op, initiate it
	 else if (x.op_stage2 == OP_Stage2_M) begin
            // Instr fields required for decode for F/D opcodes
	    Bool is_OP_not_OP_32 = (x.instr [3] == 1'b0);
            mbox.req (is_OP_not_OP_32, funct3, x.val1, x.val2);
	 end
`endif

`ifdef ISA_F
	 // If FBox op, initiate it
	 else if (x.op_stage2 == OP_Stage2_FD) begin
	    // Instr fields required for decode for F/D opcodes
            let opcode = instr_opcode (x.instr);
	    let funct7 = instr_funct7 (x.instr);
            let rs2    = instr_rs2    (x.instr);
            Bit #(64) val1 = x.val1_frm_gpr ? extend (x.val1)
                                            : extend (x.fval1);

	    fbox.req (  opcode
		      , funct7
		      , x.rounding_mode   // rm
		      , rs2
		      , val1
		      , extend (x.fval2)
		      , extend (x.fval3)
		     );
         end
`endif
      endaction
   endfunction

   // ----------------------------------------------------------------
   // INTERFACE

   // ---- Reset
   interface server_reset = toGPServer (f_reset_reqs, f_reset_rsps);

   // ---- Output
   method Output_Stage2  out;
      return fv_out;
   endmethod

   method Action deq ();
      noAction;
   endmethod

   // ---- Input
   method Action enq (Data_Stage1_to_Stage2 x);
      fa_enq (x);

      if (verbosity > 1)
	 $display ("    CPU_Stage2.enq (Data_Stage1_to_Stage2) ", fshow (x));
   endmethod

   method Action set_full (Bool full);
      rg_full <= full;
   endmethod
endmodule

// ================================================================

endpackage

// Copyright (c) 2016-2019 Bluespec, Inc. All Rights Reserved

package CPU_Stage2;

// ================================================================
// This is Stage 2 of the CPU.
// It is the "DM" stage ("Data Memory"), which is the main function.

// However, this stage also contains all other (potentially) long-latency
// operations:
//    MBox ("M" extension ops, integer multiply/divide)
//    FDBox ("FD" extension ops, single and double precision floating point)

// This stage sends out Tandem Verifier information for pipelined instructions

// Note: $displays are indented by (stage num x 4) spaces.
// for traditional pipeline display
//     IF
//         DM
//             WB
// i.e., 8 spaces for this stage.

// ================================================================
// Exports

export
CPU_Stage2_IFC (..),
mkCPU_Stage2;

// ================================================================
// BSV library imports

import FIFOF        :: *;
import GetPut       :: *;
import ClientServer :: *;
import ConfigReg    :: *;

// ----------------
// BSV additional libs

import Cur_Cycle  :: *;

// ================================================================
// Project imports

import ISA_Decls     :: *;

import TV_Info       :: *;

import CPU_Globals   :: *;
import Near_Mem_IFC  :: *;
import CSR_RegFile   :: *;    // For SATP, SSTATUS, MSTATUS

`ifdef SHIFT_SERIAL
import Shifter_Box  :: *;
`endif

`ifdef ISA_M
import RISCV_MBox  :: *;
`endif

`ifdef ISA_F
import FBox_Top    :: *;
import FBox_Core   :: *;   // For fv_nanbox function
`endif

// ================================================================
// Interface

interface CPU_Stage2_IFC;
   // ---- Reset
   interface Server #(Token, Token) server_reset;

   // ---- Output
   (* always_ready *)
   method Output_Stage2  out;

   (* always_ready *)
   method Action deq;

   // ---- Input
   (* always_ready *)
   method Action enq (Data_Stage1_to_Stage2 x);

   (* always_ready *)
   method Action set_full (Bool full);
endinterface

// ================================================================
// Implementation module

module mkCPU_Stage2 #(Bit #(4)         verbosity,
		      CSR_RegFile_IFC  csr_regfile,    // for SATP and SSTATUS: TODO carry in Data_Stage1_to_Stage2
		      DMem_IFC         dcache)
                    (CPU_Stage2_IFC);

   FIFOF #(Token) f_reset_reqs <- mkFIFOF;
   FIFOF #(Token) f_reset_rsps <- mkFIFOF;

   Reg #(Bool)                  rg_resetting  <- mkReg (False);
   Reg #(Bool)                  rg_full       <- mkReg (False);
   Reg #(Data_Stage1_to_Stage2) rg_stage2     <- mkRegU;    // From Stage 1

   // ----------------
   // Serial shifter box

`ifdef SHIFT_SERIAL
   Shifter_Box_IFC shifter_box <- mkShifter_Box;
`endif

   // ----------------
   // Integer multiply/divide box

`ifdef ISA_M
   RISCV_MBox_IFC mbox <- mkRISCV_MBox;
`endif

   // ----------------
   // Floating point box

`ifdef ISA_F
   FBox_Top_IFC fbox <- mkFBox_Top (3);
`endif

   // ----------------

   let bypass_base = Bypass {bypass_state: BYPASS_RD_NONE,
			     rd:           rg_stage2.rd,
			     rd_val:       rg_stage2.val1
			     };

`ifdef ISA_F
   let fbypass_base = FBypass {bypass_state: BYPASS_RD_NONE,
			       rd:           rg_stage2.rd,
			       rd_val:       rg_stage2.fval1
			       };
`endif

   let data_to_stage3_base = Data_Stage2_to_Stage3 {priv:      rg_stage2.priv,
						    pc:        rg_stage2.pc,
						    instr:     rg_stage2.instr,
`ifdef ISA_F
                                                    rd_in_fpr: False,
                                                    upd_flags: False,
                                                    fpr_flags: 0,
                                                    frd_val  : rg_stage2.fval1,
`endif
						    rd_valid:  False,
						    rd:        rg_stage2.rd,
						    rd_val:    rg_stage2.val1};

   let  trap_info_dmem = Trap_Info {epc:      rg_stage2.pc,
				    exc_code: dcache.exc_code,
				    tval:     rg_stage2.addr };

`ifdef ISA_F
   // The FBox can only generate ILLEGAL Instruction exceptions
   let  trap_info_fbox = Trap_Info {epc:      rg_stage2.pc,
				    exc_code: exc_code_ILLEGAL_INSTRUCTION,
				    tval:     0 };
`endif

   // ----------------------------------------------------------------
   // BEHAVIOR

   rule rl_reset_begin;
      f_reset_reqs.deq;
      rg_full <= False;
      rg_resetting <= True;
`ifdef ISA_F
      fbox.server_reset.request.put (?);
`endif
   endrule

   rule rl_reset_end (rg_resetting);
      rg_resetting <= False;

`ifdef ISA_F
      let res <- fbox.server_reset.response.get;
`endif

      f_reset_rsps.enq (?);
   endrule

   // ----------------
   // Combinational output function

   function Output_Stage2 fv_out;
      Output_Stage2 output_stage2 = ?;

      // This stage is empty
      if (! rg_full) begin
	 output_stage2 = Output_Stage2 {
              ostatus         : OSTATUS_EMPTY
            , trap_info       : ?
            , data_to_stage3  : ?
            , bypass          : no_bypass
`ifdef ISA_F
            , fbypass         : no_fbypass
`endif
`ifdef INCLUDE_TANDEM_VERIF
	    , trace_data      : ?
`endif
	 };
      end

      // This stage is just relaying ALU results from previous stage to next stage
      else if (rg_stage2.op_stage2 == OP_Stage2_ALU) begin
	 let data_to_stage3 = data_to_stage3_base;
	 data_to_stage3.rd_valid = True;

	 let bypass = bypass_base;
	 bypass.bypass_state = BYPASS_RD_RDVAL;

`ifdef INCLUDE_TANDEM_VERIF
	 let trace_data = rg_stage2.trace_data;
`endif

	 output_stage2 = Output_Stage2 {
              ostatus         : OSTATUS_PIPE
            , trap_info       : ?
            , data_to_stage3  : data_to_stage3
            , bypass          : bypass
`ifdef ISA_F
            , fbypass         : no_fbypass
`endif
`ifdef INCLUDE_TANDEM_VERIF
            , trace_data      : trace_data
`endif
         };
      end

      // This stage is doing a LOAD or AMO
      else if (   (rg_stage2.op_stage2 == OP_Stage2_LD)
`ifdef ISA_A
	       || (rg_stage2.op_stage2 == OP_Stage2_AMO)
`endif
	       )
	 begin
	    let ostatus = (  (! dcache.valid)
			   ? OSTATUS_BUSY
			   : (  dcache.exc
			      ? OSTATUS_NONPIPE
			      : OSTATUS_PIPE));

	    WordXL result = truncate (dcache.word64);

            let funct3 = instr_funct3 (rg_stage2.instr);

	    let data_to_stage3 = data_to_stage3_base;
	    data_to_stage3.rd_valid = (ostatus == OSTATUS_PIPE);

`ifdef ISA_F
            data_to_stage3.rd_in_fpr = rg_stage2.rd_in_fpr;
            // A FPR load
            if (rg_stage2.rd_in_fpr) begin
`ifdef ISA_D
               // Both FLW and FLD are legal instructions
               // A FLW result
               if (funct3 == f3_FLW)
                  // needs nan-boxing when destined for a DP register file
                  data_to_stage3.frd_val = fv_nanbox (dcache.word64);

               // A FLD result
               else
                  data_to_stage3.frd_val = dcache.word64;
`else
               // Only FLW is a legal instruction
               data_to_stage3.frd_val = truncate (dcache.word64);
`endif
            end
`endif
            // GPR loads
	    data_to_stage3.rd_val   = result;

            // Update the bypass channel, if not trapping (NONPIPE)
	    let bypass = bypass_base;
`ifdef ISA_F
	    let fbypass = fbypass_base;
`endif

	    if (ostatus != OSTATUS_NONPIPE) begin
`ifdef ISA_F
               // Bypassing FPR value.
               if (rg_stage2.rd_in_fpr) begin
		  // Choose one of the following two options

		  // Option 1: longer critical path, since the data is bypassed back into previous stage.
		  // We use data_to_stage3.rd_val since nanboxing has been done.
		  // fbypass.bypass_state = ((ostatus == OSTATUS_PIPE) ? BYPASS_RD_RDVAL : BYPASS_RD);
		  // fbypass.rd_val       = data_to_stage3.frd_val;

		  // Option 2: shorter critical path, since the data is not bypassed into previous stage,
		  // (the bypassing is effectively delayed until the next stage).
		  fbypass.bypass_state = BYPASS_RD;
               end
`endif

               // Bypassing GPR values
               if (rg_stage2.rd != 0) begin    // TODO: is this test necessary?
		  // Choose one of the following two options

		  // Option 1: longer critical path, since the data is bypassed back into previous stage.
		  // We use data_to_stage3.rd_val since nanboxing has been done.
		  // bypass.bypass_state = ((ostatus == OSTATUS_PIPE) ? BYPASS_RD_RDVAL : BYPASS_RD);
		  // bypass.rd_val       = result;

		  // Option 2: shorter critical path, since the data is not bypassed into previous stage,
		  // (the bypassing is effectively delayed until the next stage).
		  bypass.bypass_state = BYPASS_RD;
	       end
	    end

`ifdef INCLUDE_TANDEM_VERIF
	    let trace_data   = rg_stage2.trace_data;
`ifdef ISA_F
            if (rg_stage2.rd_in_fpr) begin
               trace_data.word5 = data_to_stage3.frd_val;

               // Update MSTATUS.FS in trace packet
               trace_data = fv_trace_update_mstatus_fs (trace_data, fs_xs_dirty);
            end else
`endif
               trace_data.word1 = data_to_stage3.rd_val;
`endif
            output_stage2 = Output_Stage2 {
                 ostatus         : ostatus
               , trap_info       : trap_info_dmem
               , data_to_stage3  : data_to_stage3
               , bypass          : bypass
`ifdef ISA_F
               , fbypass         : fbypass
`endif
`ifdef INCLUDE_TANDEM_VERIF
               , trace_data      : trace_data
`endif
            };
	 end

      // This stage is doing a STORE
      else if (rg_stage2.op_stage2 == OP_Stage2_ST) begin
	 let ostatus = (  (! dcache.valid)
			     ? OSTATUS_BUSY
			     : (  dcache.exc
				? OSTATUS_NONPIPE
				: OSTATUS_PIPE));

	 let data_to_stage3 = data_to_stage3_base;
	 data_to_stage3.rd_valid = (ostatus == OSTATUS_PIPE);
	 data_to_stage3.rd       = 0;

`ifdef INCLUDE_TANDEM_VERIF
	 let trace_data   = rg_stage2.trace_data;
`endif

	 output_stage2 = Output_Stage2 {
              ostatus         : ostatus
            , trap_info       : trap_info_dmem
            , data_to_stage3  : data_to_stage3
            , bypass          : no_bypass
`ifdef ISA_F
            , fbypass         : no_fbypass
`endif
`ifdef INCLUDE_TANDEM_VERIF
            , trace_data      : trace_data
`endif
         };
      end

`ifdef SHIFT_SERIAL
      // This stage is doing a serial shift
      else if (rg_stage2.op_stage2 == OP_Stage2_SH) begin
	 let ostatus = ((! shifter_box.valid) ? OSTATUS_BUSY : OSTATUS_PIPE);

	 let result = shifter_box.word;

	 let data_to_stage3 = data_to_stage3_base;
	 data_to_stage3.rd_valid = (ostatus == OSTATUS_PIPE);
	 data_to_stage3.rd_val   = result;

	 let bypass = bypass_base;
	 bypass.bypass_state = ((ostatus == OSTATUS_PIPE) ? BYPASS_RD_RDVAL : BYPASS_RD);
	 bypass.rd_val       = result;

`ifdef INCLUDE_TANDEM_VERIF
	 let trace_data   = rg_stage2.trace_data;
	 trace_data.word1 = result;
`endif

	 output_stage2 = Output_Stage2 {
              ostatus         : ostatus
            , trap_info       : ?
            , data_to_stage3  : data_to_stage3
            , bypass          : bypass
`ifdef ISA_F
            , fbypass         : no_fbypass
`endif
`ifdef INCLUDE_TANDEM_VERIF
            , trace_data      : trace_data
`endif
         };
      end
`endif

`ifdef ISA_M
      // This stage is doing an integer multiply/divide
      else if (rg_stage2.op_stage2 == OP_Stage2_M) begin
	 let ostatus = ((! mbox.valid) ? OSTATUS_BUSY : OSTATUS_PIPE);

	 let result = mbox.word;

	 let data_to_stage3 = data_to_stage3_base;
	 data_to_stage3.rd_valid = (ostatus == OSTATUS_PIPE);
	 data_to_stage3.rd_val   = result;

	 let bypass = bypass_base;
	 bypass.bypass_state = ((ostatus == OSTATUS_PIPE) ? BYPASS_RD_RDVAL : BYPASS_RD);
	 bypass.rd_val       = result;

`ifdef INCLUDE_TANDEM_VERIF
	 let trace_data   = rg_stage2.trace_data;
	 trace_data.word1 = result;
`endif

	 output_stage2 = Output_Stage2 {
              ostatus         : ostatus
            , trap_info       : ?
            , data_to_stage3  : data_to_stage3
            , bypass          : bypass
`ifdef ISA_F
            , fbypass         : no_fbypass
`endif
`ifdef INCLUDE_TANDEM_VERIF
            , trace_data      : trace_data
`endif
         };
      end
`endif

`ifdef ISA_F
      // This stage is doing a floating point op
      else if (rg_stage2.op_stage2 == OP_Stage2_FD) begin
	 let ostatus = ((! fbox.valid) ? OSTATUS_BUSY : OSTATUS_PIPE);

         // Extract fields from FBOX result
	 match {.value, .fflags} = fbox.word;

	 let data_to_stage3      = data_to_stage3_base;
	 data_to_stage3.rd_valid = (ostatus == OSTATUS_PIPE);
`ifdef ISA_D
	 data_to_stage3.frd_val  = value;
`else
	 data_to_stage3.frd_val  = truncate (value);
`endif
         data_to_stage3.rd_in_fpr= rg_stage2.rd_in_fpr;
         data_to_stage3.upd_flags= True;
         data_to_stage3.fpr_flags= fflags;

         // result is meant for a FPR
	 let bypass              = bypass_base;
         let fbypass             = fbypass_base;
         if (rg_stage2.rd_in_fpr) begin
            fbypass.bypass_state    = ((ostatus==OSTATUS_PIPE) ? BYPASS_RD_RDVAL
                                                               : BYPASS_RD);
`ifdef ISA_D
            fbypass.rd_val          = value;
`else
            fbypass.rd_val          = truncate (value);
`endif
         end

         // result is meant for a GPR
         else begin
            bypass.bypass_state     = ((ostatus==OSTATUS_PIPE) ? BYPASS_RD_RDVAL
                                                               : BYPASS_RD);
`ifdef RV64
            bypass.rd_val           = (value);
            data_to_stage3.rd_val   = value;
`else
            bypass.rd_val           = truncate (value);
            data_to_stage3.rd_val   = truncate (value);
`endif
         end

         // -----
`ifdef INCLUDE_TANDEM_VERIF
	 let trace_data = rg_stage2.trace_data;

         if (rg_stage2.rd_in_fpr) begin
            trace_data.word5 = data_to_stage3.frd_val;
         end else begin
            trace_data.word1 = data_to_stage3.rd_val;
         end

         // Update MSTATUS.FS and FCSR.FFLAGS in trace packet
         trace_data = fv_trace_update_mstatus_fs (trace_data, fs_xs_dirty);
         trace_data = fv_trace_update_fcsr_fflags (trace_data, fflags);

`endif

	 output_stage2 = Output_Stage2 {
              ostatus         : ostatus
            , trap_info       : trap_info_fbox
            , data_to_stage3  : data_to_stage3
            , bypass          : bypass
`ifdef ISA_F
            , fbypass         : fbypass
`endif
`ifdef INCLUDE_TANDEM_VERIF
            , trace_data      : trace_data
`endif
         };
      end
`endif

      return output_stage2;
   endfunction

   // ----------------
   // Initiate DM, Shifter box, MBox or FBox op

   function Action fa_enq (Data_Stage1_to_Stage2 x);
      action
	 rg_stage2  <= x;

	 let funct3 = instr_funct3 (x.instr);

	 // If DMem access, initiate it
`ifdef ISA_A
	 Bool op_stage2_amo = (x.op_stage2 == OP_Stage2_AMO);
	 Bit #(7) amo_funct7 = x.val1 [6:0];
`else
	 Bool op_stage2_amo = False;
	 Bit #(7) amo_funct7 = 0;
`endif
	 if ((x.op_stage2 == OP_Stage2_LD) || (x.op_stage2 == OP_Stage2_ST) || op_stage2_amo) begin
	    WordXL   mstatus     = csr_regfile.read_mstatus;
`ifdef ISA_PRIV_S
	    Bit #(1) sstatus_SUM = (csr_regfile.read_sstatus) [18];
`else
	    Bit #(1) sstatus_SUM = 0;
`endif
	    Bit #(1) mstatus_MXR = mstatus [19];
	    Priv_Mode  mem_priv = x.priv;
	    if (mstatus [17] == 1'b1) begin
	       mem_priv = mstatus [12:11];
	       // $display ("    S2.fa_enq: mem_priv %0d => %0d (mstatus.MPP) due to mstatus.MPRV", x.priv, mem_priv);
	    end

	    CacheOp cache_op = ?;
	    if      (x.op_stage2 == OP_Stage2_LD)  cache_op = CACHE_LD;
	    else if (x.op_stage2 == OP_Stage2_ST)  cache_op = CACHE_ST;
`ifdef ISA_A
	    else if (x.op_stage2 == OP_Stage2_AMO) cache_op = CACHE_AMO;
`endif

            // Prepare the store value
`ifdef RV64
            Bit# (64) wdata_from_gpr = x.val2;
`else
            Bit# (64) wdata_from_gpr = zeroExtend (x.val2);
`endif

`ifdef ISA_F
`ifdef ISA_D
            Bit# (64) wdata_from_fpr = x.fval2;
`else
            Bit# (64) wdata_from_fpr = zeroExtend (x.fval2);
`endif
`endif
	    dcache.req (cache_op,
			instr_funct3 (x.instr),
`ifdef ISA_A
			amo_funct7,
`endif
			x.addr,
`ifdef ISA_F
			(x.rs_frm_fpr ? wdata_from_fpr : wdata_from_gpr),
`else
			wdata_from_gpr,
`endif
			mem_priv,
			sstatus_SUM,
			mstatus_MXR,
			csr_regfile.read_satp);
	 end

`ifdef SHIFT_SERIAL
	 // If Shifter box op, initiate it
	 else if (x.op_stage2 == OP_Stage2_SH)
	    shifter_box.req (unpack (funct3 [2]), x.val1, x.val2);
`endif

`ifdef ISA_M
	 // If MBox op, initiate it
	 else if (x.op_stage2 == OP_Stage2_M) begin
            // Instr fields required for decode for F/D opcodes
	    Bool is_OP_not_OP_32 = (x.instr [3] == 1'b0);
            mbox.req (is_OP_not_OP_32, funct3, x.val1, x.val2);
	 end
`endif

`ifdef ISA_F
	 // If FBox op, initiate it
	 else if (x.op_stage2 == OP_Stage2_FD) begin
	    // Instr fields required for decode for F/D opcodes
            let opcode = instr_opcode (x.instr);
	    let funct7 = instr_funct7 (x.instr);
            let rs2    = instr_rs2    (x.instr);
            Bit #(64) val1 = x.val1_frm_gpr ? extend (x.val1)
                                            : extend (x.fval1);

	    fbox.req (  opcode
		      , funct7
		      , x.rounding_mode   // rm
		      , rs2
		      , val1
		      , extend (x.fval2)
		      , extend (x.fval3)
		     );
         end
`endif
      endaction
   endfunction

   // ----------------------------------------------------------------
   // INTERFACE

   // ---- Reset
   interface server_reset = toGPServer (f_reset_reqs, f_reset_rsps);

   // ---- Output
   method Output_Stage2  out;
      return fv_out;
   endmethod

   method Action deq ();
      noAction;
   endmethod

   // ---- Input
   method Action enq (Data_Stage1_to_Stage2 x);
      fa_enq (x);

      if (verbosity > 1)
	 $display ("    CPU_Stage2.enq (Data_Stage1_to_Stage2) ", fshow (x));
   endmethod

   method Action set_full (Bool full);
      rg_full <= full;
   endmethod
endmodule

// ================================================================

endpackage