Byte ordering of Altera is different from Xilinx

// Copyright (c) 2016 Cornell University.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

// NOTE:
// Implement a store-and-forward mechanism between ring-buffer and
// main packet memory. Packets are buffered completely in ring buffer
// before it is sent to main memory and vice versa.

import BuildVector::*;
import Connectable::*;
import Clocks::*;
import Cntrs::*;
import ClientServer::*;
import DefaultValue::*;
import DbgTypes::*;
import Ethernet::*;
import EthMac::*;
import FIFO::*;
import FIFOF::*;
import GetPut::*;
import Gearbox::*;
import MemTypes::*;
import MemServer::*;
import MemServerInternal::*;
import MemMgmt::*;
import PacketBuffer::*;
import SharedBuff::*;
import SharedBuffMMU::*;
import SpecialFIFOs::*;
import Vector::*;
import Pipe::*;
import DbgTypes::*;

interface StoreAndFwdFromRingToMem;
   interface PktReadClient readClient;
   interface MemAllocClient malloc;
   interface MemWriteClient#(`DataBusWidth) writeClient;
   interface PipeOut#(PacketInstance) eventPktCommitted;
endinterface

module mkStoreAndFwdFromRingToMem(StoreAndFwdFromRingToMem)
   provisos (Div#(`DataBusWidth, 8, bytesPerBeat)
            ,Log#(bytesPerBeat, beatShift));

   let verbose = False;

   // RingBuffer Read Client
   FIFO#(EtherData) readDataFifo <- mkFIFO;
   FIFO#(Bit#(EtherLen)) readLenFifo <- mkFIFO;
   FIFO#(EtherReq) readReqFifo <- mkFIFO;

   // Memory Client
   FIFO#(MemRequest) writeReqFifo <- mkSizedFIFO(4);
   FIFO#(MemData#(`DataBusWidth)) writeDataFifo <- mkSizedFIFO(16);
   FIFO#(Bit#(MemTagSize)) writeDoneFifo <- mkSizedFIFO(4);
   MemWriteClient#(`DataBusWidth) dmaWriteClient = (interface MemWriteClient;
   interface Get writeReq = toGet(writeReqFifo);
   interface Get writeData = toGet(writeDataFifo);
   interface Put writeDone = toPut(writeDoneFifo);
   endinterface);

   FIFO#(Bit#(EtherLen)) mallocReqFifo <- mkFIFO;
   FIFO#(Bit#(EtherLen)) pktLenFifo <- mkFIFO;
   FIFO#(Maybe#(PktId)) mallocDoneFifo <- mkFIFO;
   Reg#(Bool) readStarted <- mkReg(False);
   Reg#(Bool) mallocd <- mkReg(False);

   FIFOF#(PacketInstance) eventPktReceivedFifo <- mkFIFOF;
   FIFOF#(PacketInstance) eventPktCommittedFifo <- mkFIFOF;

   Reg#(Bit#(32)) cycle <- mkReg(0);
   rule every1 if (verbose);
      cycle <= cycle + 1;
   endrule

   rule packetReadStart if (!readStarted);
      let pktLen <- toGet(readLenFifo).get;
      if (verbose) $display("StoreAndForward::packetReadStart %d: ReadLen %d", cycle, pktLen);
      mallocReqFifo.enq(pktLen);
      pktLenFifo.enq(pktLen);
      readStarted <= True;
   endrule

   rule allocMemory;
      let pktLen <- toGet(pktLenFifo).get;
      let allocId <- toGet(mallocDoneFifo).get;
      let bytesPerBeatMinusOne = fromInteger(valueOf(bytesPerBeat))-1;
      // roundup to 16 byte boundary
      let burstLen = ((pktLen + bytesPerBeatMinusOne) & ~(bytesPerBeatMinusOne));
      let mask = (1<< (pktLen % fromInteger(valueOf(bytesPerBeat))))-1;
      if (isValid(allocId)) begin
         mallocd <= True;
         readReqFifo.enq(EtherReq{len: truncate(pktLen)});
         //FIXME use correct sglId
         writeReqFifo.enq(MemRequest {sglId: extend(fromMaybe(?, allocId)), offset: 0,
                                      burstLen: truncate(burstLen), tag:0
`ifdef BYTE_ENABLES
                                      , firstbe: 'hffff, lastbe: mask
`endif
                                     });
         if (verbose) $display("StoreAndForward::allocMemory %d: alloc done", cycle);
         eventPktReceivedFifo.enq(PacketInstance {id: fromMaybe(?, allocId), size: pktLen});
      end
   endrule

   rule packetReadInProgress if (readStarted && mallocd);
      let v <- toGet(readDataFifo).get;
      if (v.eop) begin
         readStarted <= False;
         mallocd <= False;
         if (verbose) $display("StoreAndForward:: %d: packet finished", cycle);
      end
      if (verbose) $display("StoreAndForward::writeData: %d: data:%h, tag:%h, last:%h", cycle, v.data, 0, v.eop);
      writeDataFifo.enq(MemData {data: v.data, tag: 0, last: v.eop});
   endrule

   rule packetReadDone;
      let v <- toGet(writeDoneFifo).get;
      let recvd <- toGet(eventPktReceivedFifo).get;
      eventPktCommittedFifo.enq(recvd);
      if (verbose) $display("StoreAndForward::packetReadDone %d: packet written to memory %h", cycle, v);
   endrule

   interface PktReadClient readClient;
      interface readData = toPut(readDataFifo);
      interface readLen = toPut(readLenFifo);
      interface readReq = toGet(readReqFifo);
   endinterface

   interface malloc = (interface MemAllocClient;
      interface Get mallocReq = toGet(mallocReqFifo);
      interface Put mallocDone = toPut(mallocDoneFifo);
   endinterface);
   interface writeClient = dmaWriteClient;
   interface PipeOut eventPktCommitted = toPipeOut(eventPktCommittedFifo);
endmodule

interface StoreAndFwdFromMemToRing;
   interface PktWriteClient writeClient;
   interface MemReadClient#(`DataBusWidth) readClient;
   interface PipeIn#(PacketInstance) eventPktSend;
   interface MemFreeClient free;
endinterface

module mkStoreAndFwdFromMemToRing(StoreAndFwdFromMemToRing)
   provisos (Div#(`DataBusWidth, 8, bytesPerBeat)
            ,Log#(bytesPerBeat, beatShift));

   let verbose = False;

   // Ring Buffer Write Client
   FIFO#(EtherData) writeDataFifo <- mkFIFO;

   // read client interface
   FIFO#(MemRequest) readReqFifo <-mkSizedFIFO(4);
   FIFO#(MemData#(`DataBusWidth)) readDataFifo <- mkSizedFIFO(32);
   MemReadClient#(`DataBusWidth) dmaReadClient = (interface MemReadClient;
   interface Get readReq = toGet(readReqFifo);
   interface Put readData = toPut(readDataFifo);
   endinterface);

   FIFOF#(PacketInstance) eventPktSendFifo <- mkSizedFIFOF(4);
   FIFO#(PktId) freeReqFifo <- mkSizedFIFO(4);

   Reg#(Bool)                 outPacket <- mkReg(False);
   Reg#(PktId)            currPacketId <- mkReg(0);
   Reg#(Bit#(EtherLen))   readBurstCount <- mkReg(0);
   Reg#(Bit#(EtherLen))   readBurstLen <- mkReg(0);

   Reg#(Bit#(32)) cycle <- mkReg(0);
   rule every1 if (verbose);
      cycle <= cycle + 1;
   endrule

   rule packetReadStart (!outPacket);
      let pkt <- toGet(eventPktSendFifo).get;
      let bytesPerBeatMinusOne = fromInteger(valueOf(bytesPerBeat))-1;
      // roundup to 16 byte boundary
      let burstLen = ((pkt.size + bytesPerBeatMinusOne) & ~(bytesPerBeatMinusOne));
      if (verbose) $display("StoreAndForward:: packetReadStart: %h, burstLen = %h", pkt.size, burstLen);
      let mask = (1<< (pkt.size % fromInteger(valueOf(bytesPerBeat))))-1;
      readReqFifo.enq(MemRequest{sglId: extend(pkt.id), offset: 0,
                                 burstLen: truncate(burstLen), tag: 0
`ifdef BYTE_ENABLES
                                 , firstbe: 'hffff, lastbe: mask
`endif
                                });
      if (verbose) $display("StoreAndForward::packetReadStart %d: send a new packet with size %h %h", cycle, burstLen, pack(mask));
      outPacket <= True;
      currPacketId <= pkt.id;
      readBurstLen <= pkt.size;
      readBurstCount <= pkt.size;
   endrule

   rule packetReadInProgress (outPacket);
      let d <- toGet(readDataFifo).get;
      let _bytesPerBeat= fromInteger(valueOf(bytesPerBeat));

      let sop = (readBurstLen == readBurstCount) ? True : False;
      let eop = (readBurstCount <= _bytesPerBeat) ? True : False;

      Bit#(bytesPerBeat) mask;
      if (readBurstCount <= _bytesPerBeat)
         mask = (1<<readBurstCount)-1;
      else
         mask = (1<<_bytesPerBeat)-1;

      writeDataFifo.enq(EtherData{data: d.data, mask: mask, sop: sop, eop: eop});

      if (verbose) $display("StoreAndForward::readdata %d: %h %h %h %h %h", cycle, readBurstCount, d.data, pack(mask), sop, eop);

      if (readBurstCount > _bytesPerBeat)
         readBurstCount <= readBurstCount - _bytesPerBeat;

      if (eop) begin
         outPacket <= False;
         freeReqFifo.enq(currPacketId);
      end
   endrule

   interface PktWriteClient writeClient;
      interface writeData = toGet(writeDataFifo);
   endinterface
   interface readClient = dmaReadClient;
   interface PipeIn eventPktSend = toPipeIn(eventPktSendFifo);
   interface free = (interface MemFreeClient;
      interface Get freeReq = toGet(freeReqFifo);
   endinterface);
endmodule

interface StoreAndFwdFromRingToMac;
   interface PktReadClient readClient;
   interface Get#(PacketDataT#(64)) macTx;
   method TxThruDbgRec dbg;
   method ThruDbgRec sdbg;
endinterface

// store data from ring buffer to network
// big-endianess -> little-endianess
module mkStoreAndFwdFromRingToMac#(Clock txClock, Reset txReset)(StoreAndFwdFromRingToMac);
   let verbose = False;
   Clock defaultClock <- exposeCurrentClock();
   Reset defaultReset <- exposeCurrentReset();

   Reg#(Bit#(64)) cycle_cnt <- mkReg(0);
   Reg#(Bit#(64)) last_startofpacket <- mkReg(0);
   Reg#(Bit#(64)) last_endofpacket <- mkReg(0);
   Reg#(Bit#(64)) goodputCount <- mkReg(0);
   Reg#(Bit#(64)) idleCount <- mkReg(0);

   // stats
   Reg#(Bit#(64)) total_cycles <- mkReg(0, clocked_by txClock, reset_by txReset);
   Reg#(Bit#(64)) idle_cycles <- mkReg(0, clocked_by txClock, reset_by txReset);
   Reg#(Bit#(64)) sopCount <- mkReg(0, clocked_by txClock, reset_by txReset);
   Reg#(Bit#(64)) eopCount <- mkReg(0, clocked_by txClock, reset_by txReset);
   Reg#(Bit#(64)) data_bytes <- mkReg(0, clocked_by txClock, reset_by txReset);

   // RingBuffer Read Client
   FIFO#(EtherData) readDataFifo <- mkFIFO;
   FIFO#(Bit#(EtherLen)) readLenFifo <- mkFIFO;
   FIFO#(EtherReq) readReqFifo <- mkFIFO;

   // Mac Facing Fifo
   FIFOF#(PacketDataT#(64)) writeMacFifo <- mkFIFOF(clocked_by txClock, reset_by txReset);
   Gearbox#(2, 1, PacketDataT#(64)) fifoTxData <- mkNto1Gearbox(txClock, txReset, txClock, txReset);
   SyncFIFOIfc#(EtherData) tx_fifo <- mkSyncFIFO(5, defaultClock, defaultReset, txClock);

   rule cycle;
      cycle_cnt <= cycle_cnt + 1;
   endrule

   rule total_cycle;
      total_cycles <= total_cycles + 1;
   endrule

   rule readDataStart;
      let pktLen <- toGet(readLenFifo).get;
      if (verbose) $display(fshow(" read packt ") + fshow(pktLen));
      readReqFifo.enq(EtherReq{len: pktLen});
   endrule

   function Vector#(2, PacketDataT#(64)) split(EtherData in);
      Vector#(2, PacketDataT#(64)) v = defaultValue;
      Vector#(8, Bit#(8)) v0_data = unpack(in.data[63:0]);
      Vector#(8, Bit#(8)) v1_data = unpack(in.data[127:64]);
      v[0].sop = pack(in.sop);
`ifdef ALTERA
      v[0].data = pack(reverse(v0_data));
`elsif XILINX
      v[0].data = pack(v0_data);
`endif
      v[0].eop = (in.mask[15:8] == 0) ? pack(in.eop) : 0;
      v[0].mask = in.mask[7:0];
      v[1].sop = 0;
`ifdef ALTERA
      v[1].data = pack(reverse(v1_data));
`elsif XILINX
      v[1].data = pack(v1_data);
`endif
      v[1].eop = pack(in.eop);
      v[1].mask = in.mask[15:8];
      return v;
   endfunction

   rule cross_clocking;
      let v <- toGet(readDataFifo).get;
      tx_fifo.enq(v);

      // performance analysis
      if (v.sop) begin
         last_startofpacket <= cycle_cnt;
         idleCount <= (last_endofpacket != 0) ? (idleCount + (cycle_cnt - last_endofpacket)) : 0;
      end
      if (v.eop) begin
         last_endofpacket <= cycle_cnt;
         goodputCount <= (last_startofpacket != 0) ? (goodputCount + (cycle_cnt - last_startofpacket)) : 0;
      end
   endrule

   rule process_incoming_packet;
      let v <- toGet(tx_fifo).get;
      fifoTxData.enq(split(v));
   endrule

   rule process_outgoing_packet;
      let data = fifoTxData.first; fifoTxData.deq;
      let temp = head(data);
      let bytes = zeroExtend(pack(countOnes(temp.mask)));
      if (unpack(temp.sop)) sopCount <= sopCount + 1;
      data_bytes <= data_bytes + bytes;
      if (temp.mask != 0) begin
         if (verbose) $display("ringToMac:: tx data %h", temp.data);
         if (unpack(temp.eop)) eopCount <= eopCount + 1;
         writeMacFifo.enq(temp);
      end
      else begin
         idle_cycles <= idle_cycles + 1;
      end
   endrule

   rule count_idle_cycles (!fifoTxData.notEmpty);
      idle_cycles <= idle_cycles + 1;
   endrule

   interface PktReadClient readClient;
      interface readData = toPut(readDataFifo);
      interface readLen = toPut(readLenFifo);
      interface readReq = toGet(readReqFifo);
   endinterface
   interface Get macTx = toGet(writeMacFifo);
   method TxThruDbgRec dbg;
      return TxThruDbgRec {goodputCount: goodputCount, idleCount: idleCount};
   endmethod
   method ThruDbgRec sdbg;
      return ThruDbgRec {data_bytes: data_bytes, sops: sopCount, eops: eopCount, idle_cycles: idle_cycles, total_cycles: total_cycles};
   endmethod
endmodule

interface StoreAndFwdFromMacToRing;
   interface PktWriteClient writeClient;
   interface Put#(PacketDataT#(64)) macRx;
   method ThruDbgRec sdbg;
endinterface

// store data from network to ring buffer
// little-endianess -> big-endianess
module mkStoreAndFwdFromMacToRing#(Clock rxClock, Reset rxReset)(StoreAndFwdFromMacToRing);
   let verbose = False;
   Clock defaultClock <- exposeCurrentClock();
   Reset defaultReset <- exposeCurrentReset();

   // Ring Buffer WriteClient
   FIFO#(EtherData) writeDataFifo <- mkFIFO;

   // stats
   Reg#(Bit#(64)) total_cycles <- mkReg(0, clocked_by rxClock, reset_by rxReset);
   Reg#(Bit#(64)) idle_cycles <- mkReg(0, clocked_by rxClock, reset_by rxReset);
   Reg#(Bit#(64)) sopCount <- mkReg(0, clocked_by rxClock, reset_by rxReset);
   Reg#(Bit#(64)) eopCount <- mkReg(0, clocked_by rxClock, reset_by rxReset);
   Reg#(Bit#(64)) data_bytes <- mkReg(0, clocked_by rxClock, reset_by rxReset);

   // Mac facing fifos
   Reg#(Bool) inProgress <- mkReg(False, clocked_by rxClock, reset_by rxReset);
   Reg#(Bool) oddBeat    <- mkReg(True, clocked_by rxClock, reset_by rxReset);
   Reg#(PacketDataT#(64)) v_prev <- mkReg(defaultValue, clocked_by rxClock, reset_by rxReset);

   FIFO#(PacketDataT#(64)) readMacFifo <- mkFIFO(clocked_by rxClock, reset_by rxReset);
   SyncFIFOIfc#(EtherData) rx_fifo <- mkSyncFIFO(5, rxClock, rxReset, defaultClock);

   rule total_cycle;
      total_cycles <= total_cycles + 1;
   endrule

   function EtherData combine(Vector#(2, PacketDataT#(64)) v);
      EtherData data = defaultValue;
      Vector#(8, Bit#(8)) v0_data = unpack(v[0].data);
      Vector#(8, Bit#(8)) v1_data = unpack(v[1].data);
`ifdef ALTERA
      data.data = {pack(reverse(v1_data)), pack(reverse(v0_data))};
`elsif XILINX
      data.data = {pack(v1_data), pack(v0_data)};
`endif
      data.mask = {v[1].mask, v[0].mask};
      data.sop = unpack(v[0].sop);
      data.eop = unpack(v[0].eop) || unpack(v[1].eop);
      return data;
   endfunction

   rule startOfPacket if (!inProgress);
      let v = readMacFifo.first;
      inProgress <= unpack(v.sop);
      if (v.sop == 0)
         readMacFifo.deq;
      else
         sopCount <= sopCount + 1;
      if (verbose) $display("macToRing:: start");
   endrule

   rule readPacketOdd if (inProgress && oddBeat);
      let v <- toGet(readMacFifo).get;
      let bytes = zeroExtend(pack(countOnes(v.mask)));
      data_bytes <= data_bytes + bytes;
      if (verbose) $display("macToRing:: read odd beat %h", v.data);
      if (unpack(v.eop)) begin
         PacketDataT#(64) vo = defaultValue;
         rx_fifo.enq(combine(vec(v, vo)));
         if (verbose) $display("macToRing:: odd eop %h %h", v.data, v.mask);
         inProgress <= False;
         eopCount <= eopCount +1;
      end
      else begin
         oddBeat <= !oddBeat;
      end
      v_prev <= v;
   endrule

   rule readPacketEven if (inProgress && !oddBeat);
      let v <- toGet(readMacFifo).get;
      let bytes = zeroExtend(pack(countOnes(v.mask)));
      data_bytes <= data_bytes + bytes;
      rx_fifo.enq(combine(vec(v_prev, v)));
      if (verbose) $display("macToRing:: read even beat %h", v.data);
      if (unpack(v.eop)) begin
         inProgress <= False;
         if (verbose) $display("macToRing:: even eop %h %h %h %h", v.data, v.mask, v_prev.data, v_prev.mask);
         eopCount <= eopCount +1;
      end
      oddBeat <= !oddBeat;
   endrule

   rule write_data;
      let v <- toGet(rx_fifo).get;
      writeDataFifo.enq(v);
      if (verbose) $display("macToRing:: writeToFifo");
   endrule

   rule count_idle_cycles (!inProgress);
      idle_cycles <= idle_cycles + 1;
   endrule

   interface PktWriteClient writeClient;
      interface writeData = toGet(writeDataFifo);
   endinterface
   interface Put macRx = toPut(readMacFifo);
   method ThruDbgRec sdbg;
      return ThruDbgRec {data_bytes: data_bytes, sops: sopCount, eops: eopCount, idle_cycles: idle_cycles, total_cycles: total_cycles};
   endmethod
endmodule

// Copyright (c) 2016 Cornell University.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

// NOTE:
// Implement a store-and-forward mechanism between ring-buffer and
// main packet memory. Packets are buffered completely in ring buffer
// before it is sent to main memory and vice versa.

import BuildVector::*;
import Connectable::*;
import Clocks::*;
import Cntrs::*;
import ClientServer::*;
import DefaultValue::*;
import DbgTypes::*;
import Ethernet::*;
import EthMac::*;
import FIFO::*;
import FIFOF::*;
import GetPut::*;
import Gearbox::*;
import MemTypes::*;
import MemServer::*;
import MemServerInternal::*;
import MemMgmt::*;
import PacketBuffer::*;
import SharedBuff::*;
import SharedBuffMMU::*;
import SpecialFIFOs::*;
import Vector::*;
import Pipe::*;
import DbgTypes::*;

interface StoreAndFwdFromRingToMem;
   interface PktReadClient readClient;
   interface MemAllocClient malloc;
   interface MemWriteClient#(`DataBusWidth) writeClient;
   interface PipeOut#(PacketInstance) eventPktCommitted;
endinterface

module mkStoreAndFwdFromRingToMem(StoreAndFwdFromRingToMem)
   provisos (Div#(`DataBusWidth, 8, bytesPerBeat)
            ,Log#(bytesPerBeat, beatShift));

   let verbose = False;

   // RingBuffer Read Client
   FIFO#(EtherData) readDataFifo <- mkFIFO;
   FIFO#(Bit#(EtherLen)) readLenFifo <- mkFIFO;
   FIFO#(EtherReq) readReqFifo <- mkFIFO;

   // Memory Client
   FIFO#(MemRequest) writeReqFifo <- mkSizedFIFO(4);
   FIFO#(MemData#(`DataBusWidth)) writeDataFifo <- mkSizedFIFO(16);
   FIFO#(Bit#(MemTagSize)) writeDoneFifo <- mkSizedFIFO(4);
   MemWriteClient#(`DataBusWidth) dmaWriteClient = (interface MemWriteClient;
   interface Get writeReq = toGet(writeReqFifo);
   interface Get writeData = toGet(writeDataFifo);
   interface Put writeDone = toPut(writeDoneFifo);
   endinterface);

   FIFO#(Bit#(EtherLen)) mallocReqFifo <- mkFIFO;
   FIFO#(Bit#(EtherLen)) pktLenFifo <- mkFIFO;
   FIFO#(Maybe#(PktId)) mallocDoneFifo <- mkFIFO;
   Reg#(Bool) readStarted <- mkReg(False);
   Reg#(Bool) mallocd <- mkReg(False);

   FIFOF#(PacketInstance) eventPktReceivedFifo <- mkFIFOF;
   FIFOF#(PacketInstance) eventPktCommittedFifo <- mkFIFOF;

   Reg#(Bit#(32)) cycle <- mkReg(0);
   rule every1 if (verbose);
      cycle <= cycle + 1;
   endrule

   rule packetReadStart if (!readStarted);
      let pktLen <- toGet(readLenFifo).get;
      if (verbose) $display("StoreAndForward::packetReadStart %d: ReadLen %d", cycle, pktLen);
      mallocReqFifo.enq(pktLen);
      pktLenFifo.enq(pktLen);
      readStarted <= True;
   endrule

   rule allocMemory;
      let pktLen <- toGet(pktLenFifo).get;
      let allocId <- toGet(mallocDoneFifo).get;
      let bytesPerBeatMinusOne = fromInteger(valueOf(bytesPerBeat))-1;
      // roundup to 16 byte boundary
      let burstLen = ((pktLen + bytesPerBeatMinusOne) & ~(bytesPerBeatMinusOne));
      let mask = (1<< (pktLen % fromInteger(valueOf(bytesPerBeat))))-1;
      if (isValid(allocId)) begin
         mallocd <= True;
         readReqFifo.enq(EtherReq{len: truncate(pktLen)});
         //FIXME use correct sglId
         writeReqFifo.enq(MemRequest {sglId: extend(fromMaybe(?, allocId)), offset: 0,
                                      burstLen: truncate(burstLen), tag:0
`ifdef BYTE_ENABLES
                                      , firstbe: 'hffff, lastbe: mask
`endif
                                     });
         if (verbose) $display("StoreAndForward::allocMemory %d: alloc done", cycle);
         eventPktReceivedFifo.enq(PacketInstance {id: fromMaybe(?, allocId), size: pktLen});
      end
   endrule

   rule packetReadInProgress if (readStarted && mallocd);
      let v <- toGet(readDataFifo).get;
      if (v.eop) begin
         readStarted <= False;
         mallocd <= False;
         if (verbose) $display("StoreAndForward:: %d: packet finished", cycle);
      end
      if (verbose) $display("StoreAndForward::writeData: %d: data:%h, tag:%h, last:%h", cycle, v.data, 0, v.eop);
      writeDataFifo.enq(MemData {data: v.data, tag: 0, last: v.eop});
   endrule

   rule packetReadDone;
      let v <- toGet(writeDoneFifo).get;
      let recvd <- toGet(eventPktReceivedFifo).get;
      eventPktCommittedFifo.enq(recvd);
      if (verbose) $display("StoreAndForward::packetReadDone %d: packet written to memory %h", cycle, v);
   endrule

   interface PktReadClient readClient;
      interface readData = toPut(readDataFifo);
      interface readLen = toPut(readLenFifo);
      interface readReq = toGet(readReqFifo);
   endinterface

   interface malloc = (interface MemAllocClient;
      interface Get mallocReq = toGet(mallocReqFifo);
      interface Put mallocDone = toPut(mallocDoneFifo);
   endinterface);
   interface writeClient = dmaWriteClient;
   interface PipeOut eventPktCommitted = toPipeOut(eventPktCommittedFifo);
endmodule

interface StoreAndFwdFromMemToRing;
   interface PktWriteClient writeClient;
   interface MemReadClient#(`DataBusWidth) readClient;
   interface PipeIn#(PacketInstance) eventPktSend;
   interface MemFreeClient free;
endinterface

module mkStoreAndFwdFromMemToRing(StoreAndFwdFromMemToRing)
   provisos (Div#(`DataBusWidth, 8, bytesPerBeat)
            ,Log#(bytesPerBeat, beatShift));

   let verbose = False;

   // Ring Buffer Write Client
   FIFO#(EtherData) writeDataFifo <- mkFIFO;

   // read client interface
   FIFO#(MemRequest) readReqFifo <-mkSizedFIFO(4);
   FIFO#(MemData#(`DataBusWidth)) readDataFifo <- mkSizedFIFO(32);
   MemReadClient#(`DataBusWidth) dmaReadClient = (interface MemReadClient;
   interface Get readReq = toGet(readReqFifo);
   interface Put readData = toPut(readDataFifo);
   endinterface);

   FIFOF#(PacketInstance) eventPktSendFifo <- mkSizedFIFOF(4);
   FIFO#(PktId) freeReqFifo <- mkSizedFIFO(4);

   Reg#(Bool)                 outPacket <- mkReg(False);
   Reg#(PktId)            currPacketId <- mkReg(0);
   Reg#(Bit#(EtherLen))   readBurstCount <- mkReg(0);
   Reg#(Bit#(EtherLen))   readBurstLen <- mkReg(0);

   Reg#(Bit#(32)) cycle <- mkReg(0);
   rule every1 if (verbose);
      cycle <= cycle + 1;
   endrule

   rule packetReadStart (!outPacket);
      let pkt <- toGet(eventPktSendFifo).get;
      let bytesPerBeatMinusOne = fromInteger(valueOf(bytesPerBeat))-1;
      // roundup to 16 byte boundary
      let burstLen = ((pkt.size + bytesPerBeatMinusOne) & ~(bytesPerBeatMinusOne));
      if (verbose) $display("StoreAndForward:: packetReadStart: %h, burstLen = %h", pkt.size, burstLen);
      let mask = (1<< (pkt.size % fromInteger(valueOf(bytesPerBeat))))-1;
      readReqFifo.enq(MemRequest{sglId: extend(pkt.id), offset: 0,
                                 burstLen: truncate(burstLen), tag: 0
`ifdef BYTE_ENABLES
                                 , firstbe: 'hffff, lastbe: mask
`endif
                                });
      if (verbose) $display("StoreAndForward::packetReadStart %d: send a new packet with size %h %h", cycle, burstLen, pack(mask));
      outPacket <= True;
      currPacketId <= pkt.id;
      readBurstLen <= pkt.size;
      readBurstCount <= pkt.size;
   endrule

   rule packetReadInProgress (outPacket);
      let d <- toGet(readDataFifo).get;
      let _bytesPerBeat= fromInteger(valueOf(bytesPerBeat));

      let sop = (readBurstLen == readBurstCount) ? True : False;
      let eop = (readBurstCount <= _bytesPerBeat) ? True : False;

      Bit#(bytesPerBeat) mask;
      if (readBurstCount <= _bytesPerBeat)
         mask = (1<<readBurstCount)-1;
      else
         mask = (1<<_bytesPerBeat)-1;

      writeDataFifo.enq(EtherData{data: d.data, mask: mask, sop: sop, eop: eop});

      if (verbose) $display("StoreAndForward::readdata %d: %h %h %h %h %h", cycle, readBurstCount, d.data, pack(mask), sop, eop);

      if (readBurstCount > _bytesPerBeat)
         readBurstCount <= readBurstCount - _bytesPerBeat;

      if (eop) begin
         outPacket <= False;
         freeReqFifo.enq(currPacketId);
      end
   endrule

   interface PktWriteClient writeClient;
      interface writeData = toGet(writeDataFifo);
   endinterface
   interface readClient = dmaReadClient;
   interface PipeIn eventPktSend = toPipeIn(eventPktSendFifo);
   interface free = (interface MemFreeClient;
      interface Get freeReq = toGet(freeReqFifo);
   endinterface);
endmodule

interface StoreAndFwdFromRingToMac;
   interface PktReadClient readClient;
   interface Get#(PacketDataT#(64)) macTx;
   method TxThruDbgRec dbg;
   method ThruDbgRec sdbg;
endinterface

// store data from ring buffer to network
// big-endianess -> little-endianess
module mkStoreAndFwdFromRingToMac#(Clock txClock, Reset txReset)(StoreAndFwdFromRingToMac);
   let verbose = False;
   Clock defaultClock <- exposeCurrentClock();
   Reset defaultReset <- exposeCurrentReset();

   Reg#(Bit#(64)) cycle_cnt <- mkReg(0);
   Reg#(Bit#(64)) last_startofpacket <- mkReg(0);
   Reg#(Bit#(64)) last_endofpacket <- mkReg(0);
   Reg#(Bit#(64)) goodputCount <- mkReg(0);
   Reg#(Bit#(64)) idleCount <- mkReg(0);

   // stats
   Reg#(Bit#(64)) total_cycles <- mkReg(0, clocked_by txClock, reset_by txReset);
   Reg#(Bit#(64)) idle_cycles <- mkReg(0, clocked_by txClock, reset_by txReset);
   Reg#(Bit#(64)) sopCount <- mkReg(0, clocked_by txClock, reset_by txReset);
   Reg#(Bit#(64)) eopCount <- mkReg(0, clocked_by txClock, reset_by txReset);
   Reg#(Bit#(64)) data_bytes <- mkReg(0, clocked_by txClock, reset_by txReset);

   // RingBuffer Read Client
   FIFO#(EtherData) readDataFifo <- mkFIFO;
   FIFO#(Bit#(EtherLen)) readLenFifo <- mkFIFO;
   FIFO#(EtherReq) readReqFifo <- mkFIFO;

   // Mac Facing Fifo
   FIFOF#(PacketDataT#(64)) writeMacFifo <- mkFIFOF(clocked_by txClock, reset_by txReset);
   Gearbox#(2, 1, PacketDataT#(64)) fifoTxData <- mkNto1Gearbox(txClock, txReset, txClock, txReset);
   SyncFIFOIfc#(EtherData) tx_fifo <- mkSyncFIFO(5, defaultClock, defaultReset, txClock);

   rule cycle;
      cycle_cnt <= cycle_cnt + 1;
   endrule

   rule total_cycle;
      total_cycles <= total_cycles + 1;
   endrule

   rule readDataStart;
      let pktLen <- toGet(readLenFifo).get;
      if (verbose) $display(fshow(" read packt ") + fshow(pktLen));
      readReqFifo.enq(EtherReq{len: pktLen});
   endrule

   function Vector#(2, PacketDataT#(64)) split(EtherData in);
      Vector#(2, PacketDataT#(64)) v = defaultValue;
      Vector#(8, Bit#(8)) v0_data = unpack(in.data[63:0]);
      Vector#(8, Bit#(8)) v1_data = unpack(in.data[127:64]);
      v[0].sop = pack(in.sop);
      v[0].data = pack(v0_data);
      v[0].eop = (in.mask[15:8] == 0) ? pack(in.eop) : 0;
      v[0].mask = in.mask[7:0];
      v[1].sop = 0;
      v[1].data = pack(v1_data);
      v[1].eop = pack(in.eop);
      v[1].mask = in.mask[15:8];
      return v;
   endfunction

   rule cross_clocking;
      let v <- toGet(readDataFifo).get;
      tx_fifo.enq(v);

      // performance analysis
      if (v.sop) begin
         last_startofpacket <= cycle_cnt;
         idleCount <= (last_endofpacket != 0) ? (idleCount + (cycle_cnt - last_endofpacket)) : 0;
      end
      if (v.eop) begin
         last_endofpacket <= cycle_cnt;
         goodputCount <= (last_startofpacket != 0) ? (goodputCount + (cycle_cnt - last_startofpacket)) : 0;
      end
   endrule

   rule process_incoming_packet;
      let v <- toGet(tx_fifo).get;
      fifoTxData.enq(split(v));
   endrule

   rule process_outgoing_packet;
      let data = fifoTxData.first; fifoTxData.deq;
      let temp = head(data);
      let bytes = zeroExtend(pack(countOnes(temp.mask)));
      if (unpack(temp.sop)) sopCount <= sopCount + 1;
      data_bytes <= data_bytes + bytes;
      if (temp.mask != 0) begin
         if (verbose) $display("ringToMac:: tx data %h", temp.data);
         if (unpack(temp.eop)) eopCount <= eopCount + 1;
         writeMacFifo.enq(temp);
      end
      else begin
         idle_cycles <= idle_cycles + 1;
      end
   endrule

   rule count_idle_cycles (!fifoTxData.notEmpty);
      idle_cycles <= idle_cycles + 1;
   endrule

   interface PktReadClient readClient;
      interface readData = toPut(readDataFifo);
      interface readLen = toPut(readLenFifo);
      interface readReq = toGet(readReqFifo);
   endinterface
   interface Get macTx = toGet(writeMacFifo);
   method TxThruDbgRec dbg;
      return TxThruDbgRec {goodputCount: goodputCount, idleCount: idleCount};
   endmethod
   method ThruDbgRec sdbg;
      return ThruDbgRec {data_bytes: data_bytes, sops: sopCount, eops: eopCount, idle_cycles: idle_cycles, total_cycles: total_cycles};
   endmethod
endmodule

interface StoreAndFwdFromMacToRing;
   interface PktWriteClient writeClient;
   interface Put#(PacketDataT#(64)) macRx;
   method ThruDbgRec sdbg;
endinterface

// store data from network to ring buffer
// little-endianess -> big-endianess
module mkStoreAndFwdFromMacToRing#(Clock rxClock, Reset rxReset)(StoreAndFwdFromMacToRing);
   let verbose = False;
   Clock defaultClock <- exposeCurrentClock();
   Reset defaultReset <- exposeCurrentReset();

   // Ring Buffer WriteClient
   FIFO#(EtherData) writeDataFifo <- mkFIFO;

   // stats
   Reg#(Bit#(64)) total_cycles <- mkReg(0, clocked_by rxClock, reset_by rxReset);
   Reg#(Bit#(64)) idle_cycles <- mkReg(0, clocked_by rxClock, reset_by rxReset);
   Reg#(Bit#(64)) sopCount <- mkReg(0, clocked_by rxClock, reset_by rxReset);
   Reg#(Bit#(64)) eopCount <- mkReg(0, clocked_by rxClock, reset_by rxReset);
   Reg#(Bit#(64)) data_bytes <- mkReg(0, clocked_by rxClock, reset_by rxReset);

   // Mac facing fifos
   Reg#(Bool) inProgress <- mkReg(False, clocked_by rxClock, reset_by rxReset);
   Reg#(Bool) oddBeat    <- mkReg(True, clocked_by rxClock, reset_by rxReset);
   Reg#(PacketDataT#(64)) v_prev <- mkReg(defaultValue, clocked_by rxClock, reset_by rxReset);

   FIFO#(PacketDataT#(64)) readMacFifo <- mkFIFO(clocked_by rxClock, reset_by rxReset);
   SyncFIFOIfc#(EtherData) rx_fifo <- mkSyncFIFO(5, rxClock, rxReset, defaultClock);

   rule total_cycle;
      total_cycles <= total_cycles + 1;
   endrule

   function EtherData combine(Vector#(2, PacketDataT#(64)) v);
      EtherData data = defaultValue;
      Vector#(8, Bit#(8)) v0_data = unpack(v[0].data);
      Vector#(8, Bit#(8)) v1_data = unpack(v[1].data);
      data.data = {pack(v1_data), pack(v0_data)};
      data.mask = {v[1].mask, v[0].mask};
      data.sop = unpack(v[0].sop);
      data.eop = unpack(v[0].eop) || unpack(v[1].eop);
      return data;
   endfunction

   rule startOfPacket if (!inProgress);
      let v = readMacFifo.first;
      inProgress <= unpack(v.sop);
      if (v.sop == 0)
         readMacFifo.deq;
      else
         sopCount <= sopCount + 1;
      if (verbose) $display("macToRing:: start");
   endrule

   rule readPacketOdd if (inProgress && oddBeat);
      let v <- toGet(readMacFifo).get;
      let bytes = zeroExtend(pack(countOnes(v.mask)));
      data_bytes <= data_bytes + bytes;
      if (verbose) $display("macToRing:: read odd beat %h", v.data);
      if (unpack(v.eop)) begin
         PacketDataT#(64) vo = defaultValue;
         rx_fifo.enq(combine(vec(v, vo)));
         if (verbose) $display("macToRing:: odd eop %h %h", v.data, v.mask);
         inProgress <= False;
         eopCount <= eopCount +1;
      end
      else begin
         oddBeat <= !oddBeat;
      end
      v_prev <= v;
   endrule

   rule readPacketEven if (inProgress && !oddBeat);
      let v <- toGet(readMacFifo).get;
      let bytes = zeroExtend(pack(countOnes(v.mask)));
      data_bytes <= data_bytes + bytes;
      rx_fifo.enq(combine(vec(v_prev, v)));
      if (verbose) $display("macToRing:: read even beat %h", v.data);
      if (unpack(v.eop)) begin
         inProgress <= False;
         if (verbose) $display("macToRing:: even eop %h %h %h %h", v.data, v.mask, v_prev.data, v_prev.mask);
         eopCount <= eopCount +1;
      end
      oddBeat <= !oddBeat;
   endrule

   rule write_data;
      let v <- toGet(rx_fifo).get;
      writeDataFifo.enq(v);
      if (verbose) $display("macToRing:: writeToFifo");
   endrule

   rule count_idle_cycles (!inProgress);
      idle_cycles <= idle_cycles + 1;
   endrule

   interface PktWriteClient writeClient;
      interface writeData = toGet(writeDataFifo);
   endinterface
   interface Put macRx = toPut(readMacFifo);
   method ThruDbgRec sdbg;
      return ThruDbgRec {data_bytes: data_bytes, sops: sopCount, eops: eopCount, idle_cycles: idle_cycles, total_cycles: total_cycles};
   endmethod
endmodule


Remove deq from nap exe

Remove deq from nap exe

// Copyright (c) 2017 Massachusetts Institute of Technology
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

`include "ProcConfig.bsv"

import BrPred::*;
import DirPredictor::*;
import Btb::*;
import ClientServer::*;
import Connectable::*;
import Decode::*;
import Ehr::*;
import Fifo::*;
import GetPut::*;
import MemoryTypes::*;
import Types::*;
import ProcTypes::*;
import CCTypes::*;
import Ras::*;
import EpochManager::*;
import Performance::*;
import Vector::*;
import Assert::*;
import Cntrs::*;
import ConfigReg::*;
import TlbTypes::*;
import ITlb::*;
import CCTypes::*;
import L1CoCache::*;
import MMIOInst::*;

interface FetchStage;
    // pipeline
    interface Vector#(SupSize, SupFifoDeq#(FromFetchStage)) pipelines;

    // tlb and mem connections
    interface ITlb iTlbIfc;
    interface ICoCache iMemIfc;
    interface MMIOInstToCore mmioIfc;

    // starting and stopping
    method Action start(Addr pc);
    method Action stop();

    // redirection methods
    method Action setWaitRedirect;
    method Action redirect(Addr pc);
    method Action done_flushing();
    method Action train_predictors(
        Addr pc, Addr next_pc, IType iType, Bool taken,
        DirPredTrainInfo dpTrain, Bool mispred
    );

    // debug
    method FetchDebugState getFetchState;

    // performance
    interface Perf#(DecStagePerfType) perf;
endinterface

typedef struct {
    Addr pc;
    Epoch mainEp;
    Bool waitForRedirect;
    Bool waitForFlush;
} FetchDebugState deriving(Bits, Eq, FShow);

typedef struct {
    Addr pc;
    Addr pred_next_pc;
    Bool decode_epoch;
    Epoch main_epoch;
} Fetch1ToFetch2 deriving(Bits, Eq, FShow);

typedef struct {
    Addr pc;
    Addr phys_pc;
    Addr pred_next_pc;
    Maybe#(Exception) cause;
    Bool access_mmio; // inst fetch from MMIO
    Bool decode_epoch;
    Epoch main_epoch;
} Fetch2ToFetch3 deriving(Bits, Eq, FShow);

typedef struct {
  Addr pc;
  Addr ppc;
  Bool decode_epoch;
  Epoch main_epoch;
  Instruction inst;
  Maybe#(Exception) cause;
} Fetch3ToDecode deriving(Bits, Eq, FShow);

typedef struct {
  Addr pc;
  Addr ppc;
  Epoch main_epoch;
  DirPredTrainInfo dpTrain;
  Instruction inst;
  DecodedInst dInst;
  ArchRegs regs;
  Maybe#(Exception) cause;
} FromFetchStage deriving (Bits, Eq, FShow);

// train next addr pred (BTB)
typedef struct {
    Addr pc;
    Addr nextPc;
} TrainNAP deriving(Bits, Eq, FShow);

(* synthesize *)
module mkFetchStage(FetchStage);
    // rule ordering: Fetch1 (BTB+TLB) < Fetch3 (decode & dir pred) < redirect method
    // Fetch1 < Fetch3 to avoid bypassing path on PC and epochs

    let verbose = True;

    // Basic State Elements
    Reg#(Bool) started <- mkReg(False);

    // Stall fetch when trap happens or system inst is renamed
    // All inst younger than the trap/system inst will be killed
    // Since CSR may be modified, sending wrong path request to TLB may cause problem
    // So we stall until the next redirection happens
    // The next redirect is either by the trap/system inst or an older one
    Reg#(Bool) waitForRedirect <- mkReg(False);
    // We don't want setWaitForRedirect method and redirect method to happen together
    // make them conflict
    RWire#(void) setWaitRedirect_redirect_conflict <- mkRWire;

    // Stall fetch during the flush triggered by the procesing trap/system inst in commit stage
    // We stall until the flush is done
    Reg#(Bool) waitForFlush <- mkReg(False);

    Ehr#(3, Addr) pc_reg <- mkEhr(0);
    Integer pc_fetch1_port = 0;
    Integer pc_decode_port = 1;
    Integer pc_redirect_port = 2;

    // Epochs
    Reg#(Bool) decode_epoch <- mkReg(False);
    Reg#(Epoch) f_main_epoch <- mkReg(0); // fetch estimate of main epoch

    // Pipeline Stage FIFOs
    Fifo#(2, Tuple2#(Bit#(TLog#(SupSize)),Fetch1ToFetch2)) f12f2 <- mkCFFifo;
    Fifo#(4, Tuple2#(Bit#(TLog#(SupSize)),Fetch2ToFetch3)) f22f3 <- mkCFFifo; // FIFO should match I$ latency
    Fifo#(2, Tuple2#(Bit#(TLog#(SupSize)),Fetch2ToFetch3)) f32d <- mkCFFifo;
    Fifo#(2, Vector#(SupSize,Maybe#(Instruction))) instdata <- mkPipelineFifo();
    SupFifo#(SupSize, 2, FromFetchStage) out_fifo <- mkSupFifo;
       // Can the fifo size be smaller?

    // Branch Predictors
    NextAddrPred    nextAddrPred <- mkBtb;
    let             dirPred      <- mkDirPredictor;
    ReturnAddrStack ras          <- mkRas;
    // Wire to train next addr pred (NAP)
    RWire#(TrainNAP) napTrainByExe <- mkRWire;
    Fifo#(2,TrainNAP) napTrainByDec <- mkCFFifo;

    // TLB and Cache connections
    ITlb iTlb <- mkITlb;
    ICoCache iMem <- mkICoCache;
    MMIOInst mmio <- mkMMIOInst;
    Server#(Addr, TlbResp) tlb_server = iTlb.to_proc;
    Server#(Addr, Vector#(SupSize, Maybe#(Instruction))) mem_server = iMem.to_proc;

    // performance counters
    Fifo#(1, DecStagePerfType) perfReqQ <- mkCFFifo; // perf req FIFO
`ifdef PERF_COUNT
    Reg#(Bool) doStats <- mkConfigReg(False);
    // decode stage redirect
    Count#(Data) decRedirectBrCnt <- mkCount(0);
    Count#(Data) decRedirectJmpCnt <- mkCount(0);
    Count#(Data) decRedirectJrCnt <- mkCount(0);
    Count#(Data) decRedirectOtherCnt <- mkCount(0);
    // perf resp FIFO
    Fifo#(1, PerfResp#(DecStagePerfType)) perfRespQ <- mkCFFifo;

    rule doPerfReq;
        let t <- toGet(perfReqQ).get;
        Data d = (case(t)
            DecRedirectBr: decRedirectBrCnt;
            DecRedirectJmp: decRedirectJmpCnt;
            DecRedirectJr: decRedirectJrCnt;
            DecRedirectOther: decRedirectOtherCnt;
            default: 0;
        endcase);
        perfRespQ.enq(PerfResp {
            pType: t,
            data: d
        });
    endrule
`endif

    // We don't send req to TLB when waiting for redirect or TLB flush. Since
    // there is no FIFO between doFetch1 and TLB, when OOO commit stage wait
    // TLB idle to change VM CSR / signal flush TLB, there is no wrong path
    // request afterwards to race with the system code that manage paget table.
    rule doFetch1(started && !waitForRedirect && !waitForFlush);
        let pc = pc_reg[pc_fetch1_port];

        // Chain of prediction for the next instructions
        // We need a BTB with a register file with enough ports!
        // Instead of cascading predictions, we can always feed pc+4*i into
        // predictor, because we will break superscaler fetch if nextpc != pc+4
        Vector#(SupSize, Addr) pred_future_pc;
        for(Integer i = 0; i < valueof(SupSize); i = i+1) begin
            pred_future_pc[i] = nextAddrPred.predPc(pc + fromInteger(4 * i));
        end

        // Next pc is the first nextPc that breaks the chain of pc+4 or
        // that is at the end of a cacheline.
        Vector#(SupSize,Integer) indexes = genVector;
        function Bool findNextPc(Addr pc, Integer i);
            Bool notLastInst = getLineInstOffset(pc + fromInteger(4*i)) != maxBound;
            Bool noJump = pred_future_pc[i] == pc + fromInteger(4*(i+1));
            return (!(notLastInst && noJump));
        endfunction
        Integer posLastSup = fromMaybe(valueof(SupSize) - 1, find(findNextPc(pc), indexes));
        Addr pred_next_pc = pred_future_pc[posLastSup];
        pc_reg[pc_fetch1_port] <= pred_next_pc;

        // Send TLB request
        tlb_server.request.put(pc);

        let out = Fetch1ToFetch2 {
            pc: pc,
            pred_next_pc: pred_next_pc,
            decode_epoch: decode_epoch,
            main_epoch: f_main_epoch};
        f12f2.enq(tuple2(fromInteger(posLastSup),out));
        if (verbose) $display("Fetch1: ", fshow(out));
    endrule

    rule doFetch2;
        let {nbSup,in} = f12f2.first;
        f12f2.deq;

        // Get TLB response
        match {.phys_pc, .cause} <- tlb_server.response.get;

        // Access main mem or boot rom
        Bool access_mmio = False;
        if (!isValid(cause)) begin
            case(mmio.getFetchTarget(phys_pc))
                MainMem: begin
                    // Send ICache request
                    mem_server.request.put(phys_pc);
                end
                BootRom: begin
                    // Send MMIO req. Luckily boot rom is also aligned with
                    // cache line size, so all nbSup+1 insts can be fetched
                    // from boot rom. It won't happen that insts fetched from
                    // boot rom is less than requested.
                    mmio.bootRomReq(phys_pc, nbSup);
                    access_mmio = True;
                end
                default: begin
                    // Access fault
                    cause = Valid (InstAccessFault);
                end
            endcase
        end

        let out = Fetch2ToFetch3 {
            pc: in.pc,
            phys_pc: phys_pc,
            pred_next_pc: in.pred_next_pc,
            cause: cause,
            access_mmio: access_mmio,
            decode_epoch: in.decode_epoch,
            main_epoch: in.main_epoch };
        f22f3.enq(tuple2(nbSup,out));
        if (verbose) $display("Fetch2: ", fshow(out));
    endrule

// Break out of i$
    rule doFetch3;
        let {nbSup, fetch3In} = f22f3.first;
        f22f3.deq();
        if (verbose) $display("Fetch3 %d",fetch3In.pc);

        // Get ICache/MMIO response if no exception
        // In case of exception, we still need to process at least inst_data[0]
        // (it will be turned to an exception later), so inst_data[0] must be
        // valid.
        Vector#(SupSize,Maybe#(Instruction)) inst_d = replicate(tagged Valid (0));
        if(!isValid(fetch3In.cause)) begin
            if(fetch3In.access_mmio) begin
                if(verbose) $display("get answer from MMIO %d", fetch3In.pc);
                inst_d <- mmio.bootRomResp;
            end
            else begin
                if(verbose) $display("get answer from memory %d", fetch3In.pc);
                inst_d <- mem_server.response.get;
            end
        end
        if(verbose) $display("epoch instr: %d, epoch main : %d", fetch3In.main_epoch, f_main_epoch);
        instdata.enq(inst_d);
        f32d.enq(f22f3.first);
    endrule

    rule doDecode;
        let {nbSup, fetch3In} = f32d.first;
        f32d.deq();
        let inst_data = instdata.first();
        instdata.deq();
        // The main_epoch check is required to make sure this stage doesn't
        // redirect the PC if a later stage already redirected the PC.
        if (fetch3In.main_epoch == f_main_epoch) begin
            Bool decode_epoch_local = decode_epoch; // next value for decode epoch
            Maybe#(Addr) redirectPc = Invalid; // next pc redirect by branch predictor
            Maybe#(TrainNAP) trainNAP = Invalid; // training data sent to next addr pred
`ifdef PERF_COUNT
            // performance counter: inst being redirect by decode stage
            // Note that only 1 redirection may happen in a cycle
            Maybe#(IType) redirectInst = Invalid;
`endif

            for (Integer i = 0; i < valueof(SupSize); i=i+1) begin
                if (inst_data[i] != tagged Invalid && fromInteger(i) <= nbSup) begin
                    // get the input to decode
                    let in = Fetch3ToDecode {
                        pc: fetch3In.pc+fromInteger(4*i),
                        // last inst, next pc may not be pc+4
                        ppc: fromInteger(i) == nbSup ? fetch3In.pred_next_pc :
                                                       fetch3In.pc+fromInteger(4*(i+1)),
                        decode_epoch: fetch3In.decode_epoch,
                        main_epoch: fetch3In.main_epoch,
                        inst: fromMaybe(?,inst_data[i]),
                        cause: fetch3In.cause
                    };
                    let cause = in.cause;
                    if (verbose) $display("Decode %d\n",i);

                    // do decode and branch prediction
                    // Drop here if does not match the decode_epoch.
                    if (in.decode_epoch == decode_epoch_local) begin
                        doAssert(in.main_epoch == f_main_epoch, "main epoch must match");

                        let decode_result = decode(in.inst);

                        // update cause if there was not an early detected exception
                        if (!isValid(cause)) begin
                            cause = decode_result.illegalInst ? tagged Valid IllegalInst : tagged Invalid;
                        end

                        let dInst = decode_result.dInst;
                        let regs = decode_result.regs;
                        DirPredTrainInfo dp_train = ?; // dir pred training bookkeeping

                        // update predicted next pc
                        if (!isValid(cause)) begin
                            // direction predict
                            Bool pred_taken = False;
                            if(dInst.iType == Br) begin
                                let pred_res <- dirPred.pred[i].pred(in.pc);
                                pred_taken = pred_res.taken;
                                dp_train = pred_res.train;
                            end
                            Maybe#(Addr) nextPc = decodeBrPred(in.pc, dInst, pred_taken);
                            function Bool linkedR(Maybe#(ArchRIndx) register);
                                    let res=False;
                                    if (register matches tagged Valid .r &&& (r == tagged Gpr 1 || r == tagged Gpr 5)) res=True;
				    return res;
                            endfunction

                            // return address stack only push  when rd = x1 or x5
                            if (dInst.iType == J && linkedR(regs.dst)) begin
                                // function call -- push return address to stack
                                ras.pushAddress[i](in.pc + 4);
                            end
                            else if (dInst.iType == Jr ) begin // jalr
                                // we push if rs1 != rd and both are x1 or x5
                                if( regs.src1 != regs.dst && linkedR(regs.src1) && linkedR(regs.dst)) begin
                                    ras.pushAddress[i](in.pc+4);
                                    let new_ppc = ras.firstAddress[i];
                                    ras.popAddress[i];
                                    nextPc = tagged Valid new_ppc;
                                end
                                // we push if rd = x1 or x5 and rs1 is not x1 && x5
                                if (!linkedR(regs.src1) && linkedR(regs.dst)) begin
                                    ras.pushAddress[i](in.pc+4);
                                end
                                // we push if rd = rs1 = x1 or x5
                                if (regs.src1 == regs.dst && linkedR(regs.src1)) begin
                                    ras.pushAddress[i](in.pc+4);
                                end
                                // we pop if rd != x1 or x5, and rs1 = x1 or x5
                                if (!linkedR(regs.dst) && linkedR(regs.src1)) begin
                                    let new_ppc = ras.firstAddress[i];
                                    ras.popAddress[i];
                                    nextPc = tagged Valid new_ppc;
                                end
                            end

                            if(verbose) begin
                                $display("Branch prediction: ", fshow(dInst.iType), " ; ", fshow(in.pc), " ; ",
                                         fshow(in.ppc), " ; ", fshow(pred_taken), " ; ", fshow(nextPc));
                            end

                            // check previous mispred
                            if (nextPc matches tagged Valid .decode_pred_next_pc &&& decode_pred_next_pc != in.ppc) begin
                                if (verbose) $display("ppc and decodeppc :  %h %h", in.ppc, decode_pred_next_pc);
                                decode_epoch_local = !decode_epoch_local;
                                redirectPc = Valid (decode_pred_next_pc); // record redirect next pc
                                in.ppc = decode_pred_next_pc;
                                // train next addr pred when mispredict
                                trainNAP = Valid (TrainNAP {pc: in.pc, nextPc: decode_pred_next_pc});
`ifdef PERF_COUNT
                                // performance stats: record decode redirect
                                doAssert(redirectInst == Invalid, "at most 1 decode redirect per cycle");
                                redirectInst = Valid (dInst.iType);
`endif
                            end
                        end
                        let out = FromFetchStage{pc: in.pc,
                                                 ppc: in.ppc,
                                                 main_epoch: in.main_epoch,
                                                 dpTrain: dp_train,
                                                 inst: in.inst,
                                                 dInst: dInst,
                                                 regs: decode_result.regs,
                                                 cause: cause };
                        out_fifo.enqS[i].enq(out);
                        if (verbose) $display("Decode: ", fshow(out));
                    end
                    else begin
                        if (verbose) $display("Drop decoded within a superscalar");
                        // just drop wrong path instructions
                    end
                end
                else if (inst_data[i] == tagged Invalid && fromInteger(i) <= nbSup) begin
                    // inst num is less than expected; this should not happen
                    // because both I$ and boot rom are aligned to cache line
                    // size.
                    doAssert(False, "Fetched insts not enough");
                end
            end

            // update PC and epoch
            if(redirectPc matches tagged Valid .nextPc) begin
                pc_reg[pc_decode_port] <= nextPc;
            end
            decode_epoch <= decode_epoch_local;
            // send training data for next addr pred
            if (trainNAP matches tagged Valid .x) begin
                napTrainByDec.enq(x);
            end
`ifdef PERF_COUNT
            // performance counter: check whether redirect happens
            if(redirectInst matches tagged Valid .iType &&& doStats) begin
                case(iType)
                    Br: decRedirectBrCnt.incr(1);
                    J : decRedirectJmpCnt.incr(1);
                    Jr: decRedirectJrCnt.incr(1);
                    default: decRedirectOtherCnt.incr(1);
                endcase
            end
`endif
        end
        else begin
            if (verbose) $display("drop in fetch3decode");
        end
    endrule

        // train next addr pred
    (* fire_when_enabled*)
    rule doTrainNAP( isValid(napTrainByExe.wget));
        // give priority to train data from exe
        TrainNAP train = fromMaybe(?, napTrainByExe.wget);
        nextAddrPred.update(train.pc, train.nextPc, train.nextPc != train.pc + 4);
    endrule
    rule doTrainNAPDec;
        TrainNAP train = napTrainByDec.first();
        napTrainByDec.deq();
        nextAddrPred.update(train.pc, train.nextPc, train.nextPc != train.pc + 4);
    endrule

    interface Vector pipelines = out_fifo.deqS;
    interface iTlbIfc = iTlb;
    interface iMemIfc = iMem;
    interface mmioIfc = mmio.toCore;

    method Action start(Addr start_pc);
        pc_reg[0] <= start_pc;
        started <= True;
        waitForRedirect <= False;
        waitForFlush <= False;
    endmethod
    method Action stop();
        started <= False;
    endmethod

    method Action setWaitRedirect;
        waitForRedirect <= True;
        setWaitRedirect_redirect_conflict.wset(?); // conflict with redirect
    endmethod
    method Action redirect(Addr new_pc);
        if (verbose) $display("Redirect: newpc %h, old f_main_epoch %d, new f_main_epoch %d",new_pc,f_main_epoch,f_main_epoch+1);
        pc_reg[pc_redirect_port] <= new_pc;
        f_main_epoch <= (f_main_epoch == fromInteger(valueOf(NumEpochs)-1)) ? 0 : f_main_epoch + 1;
        // redirect comes, stop stalling for redirect
        waitForRedirect <= False;
        setWaitRedirect_redirect_conflict.wset(?); // conflict with setWaitForRedirect
        // this redirect may be caused by a trap/system inst in commit stage
        // we conservatively set wait for flush TODO make this an input parameter
        waitForFlush <= True;
    endmethod
    method Action done_flushing() if (waitForFlush);
        // signal that the pipeline can resume fetching
        waitForFlush <= False;
        // XXX The guard prevents the readyToFetch rule in Core.bsv from firing every cycle
        // The guard also makes this method sequence before (restricted) redirect method
        // So the effect of setting waitForFlush in redirect method will not be overwritten
        // Then we don't need to make two methods conflict
        // It's fine for the effect of this method to be overwritten, because it fires very often
    endmethod

    method Action train_predictors(
        Addr pc, Addr next_pc, IType iType, Bool taken,
        DirPredTrainInfo dpTrain, Bool mispred
    );
        //if (iType == J || (iType == Br && next_pc < pc)) begin
        //    // Only train the next address predictor for jumps and backward branches
        //    // next_pc != pc + 4 is a substitute for taken
        //    nextAddrPred.update(pc, next_pc, taken);
        //end
        if (iType == Br) begin
            // Train the direction predictor for all branches
            dirPred.update(pc, taken, dpTrain, mispred);
        end
        // train next addr pred when mispred
        if(mispred) begin
            napTrainByExe.wset(TrainNAP {pc: pc, nextPc: next_pc});
        end
    endmethod

    method FetchDebugState getFetchState;
        return FetchDebugState {
            pc: pc_reg[0],
            waitForRedirect: waitForRedirect,
            waitForFlush: waitForFlush,
            mainEp: f_main_epoch
        };
    endmethod

    interface Perf perf;
        method Action setStatus(Bool stats);
`ifdef PERF_COUNT
            doStats <= stats;
`else
            noAction;
`endif
        endmethod

        method Action req(DecStagePerfType r);
            perfReqQ.enq(r);
        endmethod

        method ActionValue#(PerfResp#(DecStagePerfType)) resp;
`ifdef PERF_COUNT
            perfRespQ.deq;
            return perfRespQ.first;
`else
            perfReqQ.deq;
            return PerfResp {
                pType: perfReqQ.first,
                data: 0
            };
`endif
        endmethod

`ifdef PERF_COUNT
        method Bool respValid = perfRespQ.notEmpty;
`else
        method Bool respValid = perfReqQ.notEmpty;
`endif
    endinterface
endmodule

// Copyright (c) 2017 Massachusetts Institute of Technology
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

`include "ProcConfig.bsv"

import BrPred::*;
import DirPredictor::*;
import Btb::*;
import ClientServer::*;
import Connectable::*;
import Decode::*;
import Ehr::*;
import Fifo::*;
import GetPut::*;
import MemoryTypes::*;
import Types::*;
import ProcTypes::*;
import CCTypes::*;
import Ras::*;
import EpochManager::*;
import Performance::*;
import Vector::*;
import Assert::*;
import Cntrs::*;
import ConfigReg::*;
import TlbTypes::*;
import ITlb::*;
import CCTypes::*;
import L1CoCache::*;
import MMIOInst::*;

interface FetchStage;
    // pipeline
    interface Vector#(SupSize, SupFifoDeq#(FromFetchStage)) pipelines;

    // tlb and mem connections
    interface ITlb iTlbIfc;
    interface ICoCache iMemIfc;
    interface MMIOInstToCore mmioIfc;

    // starting and stopping
    method Action start(Addr pc);
    method Action stop();

    // redirection methods
    method Action setWaitRedirect;
    method Action redirect(Addr pc);
    method Action done_flushing();
    method Action train_predictors(
        Addr pc, Addr next_pc, IType iType, Bool taken,
        DirPredTrainInfo dpTrain, Bool mispred
    );

    // debug
    method FetchDebugState getFetchState;

    // performance
    interface Perf#(DecStagePerfType) perf;
endinterface

typedef struct {
    Addr pc;
    Epoch mainEp;
    Bool waitForRedirect;
    Bool waitForFlush;
} FetchDebugState deriving(Bits, Eq, FShow);

typedef struct {
    Addr pc;
    Addr pred_next_pc;
    Bool decode_epoch;
    Epoch main_epoch;
} Fetch1ToFetch2 deriving(Bits, Eq, FShow);

typedef struct {
    Addr pc;
    Addr phys_pc;
    Addr pred_next_pc;
    Maybe#(Exception) cause;
    Bool access_mmio; // inst fetch from MMIO
    Bool decode_epoch;
    Epoch main_epoch;
} Fetch2ToFetch3 deriving(Bits, Eq, FShow);

typedef struct {
  Addr pc;
  Addr ppc;
  Bool decode_epoch;
  Epoch main_epoch;
  Instruction inst;
  Maybe#(Exception) cause;
} Fetch3ToDecode deriving(Bits, Eq, FShow);

typedef struct {
  Addr pc;
  Addr ppc;
  Epoch main_epoch;
  DirPredTrainInfo dpTrain;
  Instruction inst;
  DecodedInst dInst;
  ArchRegs regs;
  Maybe#(Exception) cause;
} FromFetchStage deriving (Bits, Eq, FShow);

// train next addr pred (BTB)
typedef struct {
    Addr pc;
    Addr nextPc;
} TrainNAP deriving(Bits, Eq, FShow);

(* synthesize *)
module mkFetchStage(FetchStage);
    // rule ordering: Fetch1 (BTB+TLB) < Fetch3 (decode & dir pred) < redirect method
    // Fetch1 < Fetch3 to avoid bypassing path on PC and epochs

    let verbose = True;

    // Basic State Elements
    Reg#(Bool) started <- mkReg(False);

    // Stall fetch when trap happens or system inst is renamed
    // All inst younger than the trap/system inst will be killed
    // Since CSR may be modified, sending wrong path request to TLB may cause problem
    // So we stall until the next redirection happens
    // The next redirect is either by the trap/system inst or an older one
    Reg#(Bool) waitForRedirect <- mkReg(False);
    // We don't want setWaitForRedirect method and redirect method to happen together
    // make them conflict
    RWire#(void) setWaitRedirect_redirect_conflict <- mkRWire;

    // Stall fetch during the flush triggered by the procesing trap/system inst in commit stage
    // We stall until the flush is done
    Reg#(Bool) waitForFlush <- mkReg(False);

    Ehr#(3, Addr) pc_reg <- mkEhr(0);
    Integer pc_fetch1_port = 0;
    Integer pc_decode_port = 1;
    Integer pc_redirect_port = 2;

    // Epochs
    Reg#(Bool) decode_epoch <- mkReg(False);
    Reg#(Epoch) f_main_epoch <- mkReg(0); // fetch estimate of main epoch

    // Pipeline Stage FIFOs
    Fifo#(2, Tuple2#(Bit#(TLog#(SupSize)),Fetch1ToFetch2)) f12f2 <- mkCFFifo;
    Fifo#(4, Tuple2#(Bit#(TLog#(SupSize)),Fetch2ToFetch3)) f22f3 <- mkCFFifo; // FIFO should match I$ latency
    Fifo#(2, Tuple2#(Bit#(TLog#(SupSize)),Fetch2ToFetch3)) f32d <- mkCFFifo;
    Fifo#(2, Vector#(SupSize,Maybe#(Instruction))) instdata <- mkPipelineFifo();
    SupFifo#(SupSize, 2, FromFetchStage) out_fifo <- mkSupFifo;
       // Can the fifo size be smaller?

    // Branch Predictors
    NextAddrPred    nextAddrPred <- mkBtb;
    let             dirPred      <- mkDirPredictor;
    ReturnAddrStack ras          <- mkRas;
    // Wire to train next addr pred (NAP)
    RWire#(TrainNAP) napTrainByExe <- mkRWire;
    Fifo#(2,TrainNAP) napTrainByDec <- mkCFFifo;

    // TLB and Cache connections
    ITlb iTlb <- mkITlb;
    ICoCache iMem <- mkICoCache;
    MMIOInst mmio <- mkMMIOInst;
    Server#(Addr, TlbResp) tlb_server = iTlb.to_proc;
    Server#(Addr, Vector#(SupSize, Maybe#(Instruction))) mem_server = iMem.to_proc;

    // performance counters
    Fifo#(1, DecStagePerfType) perfReqQ <- mkCFFifo; // perf req FIFO
`ifdef PERF_COUNT
    Reg#(Bool) doStats <- mkConfigReg(False);
    // decode stage redirect
    Count#(Data) decRedirectBrCnt <- mkCount(0);
    Count#(Data) decRedirectJmpCnt <- mkCount(0);
    Count#(Data) decRedirectJrCnt <- mkCount(0);
    Count#(Data) decRedirectOtherCnt <- mkCount(0);
    // perf resp FIFO
    Fifo#(1, PerfResp#(DecStagePerfType)) perfRespQ <- mkCFFifo;

    rule doPerfReq;
        let t <- toGet(perfReqQ).get;
        Data d = (case(t)
            DecRedirectBr: decRedirectBrCnt;
            DecRedirectJmp: decRedirectJmpCnt;
            DecRedirectJr: decRedirectJrCnt;
            DecRedirectOther: decRedirectOtherCnt;
            default: 0;
        endcase);
        perfRespQ.enq(PerfResp {
            pType: t,
            data: d
        });
    endrule
`endif

    // We don't send req to TLB when waiting for redirect or TLB flush. Since
    // there is no FIFO between doFetch1 and TLB, when OOO commit stage wait
    // TLB idle to change VM CSR / signal flush TLB, there is no wrong path
    // request afterwards to race with the system code that manage paget table.
    rule doFetch1(started && !waitForRedirect && !waitForFlush);
        let pc = pc_reg[pc_fetch1_port];

        // Chain of prediction for the next instructions
        // We need a BTB with a register file with enough ports!
        // Instead of cascading predictions, we can always feed pc+4*i into
        // predictor, because we will break superscaler fetch if nextpc != pc+4
        Vector#(SupSize, Addr) pred_future_pc;
        for(Integer i = 0; i < valueof(SupSize); i = i+1) begin
            pred_future_pc[i] = nextAddrPred.predPc(pc + fromInteger(4 * i));
        end

        // Next pc is the first nextPc that breaks the chain of pc+4 or
        // that is at the end of a cacheline.
        Vector#(SupSize,Integer) indexes = genVector;
        function Bool findNextPc(Addr pc, Integer i);
            Bool notLastInst = getLineInstOffset(pc + fromInteger(4*i)) != maxBound;
            Bool noJump = pred_future_pc[i] == pc + fromInteger(4*(i+1));
            return (!(notLastInst && noJump));
        endfunction
        Integer posLastSup = fromMaybe(valueof(SupSize) - 1, find(findNextPc(pc), indexes));
        Addr pred_next_pc = pred_future_pc[posLastSup];
        pc_reg[pc_fetch1_port] <= pred_next_pc;

        // Send TLB request
        tlb_server.request.put(pc);

        let out = Fetch1ToFetch2 {
            pc: pc,
            pred_next_pc: pred_next_pc,
            decode_epoch: decode_epoch,
            main_epoch: f_main_epoch};
        f12f2.enq(tuple2(fromInteger(posLastSup),out));
        if (verbose) $display("Fetch1: ", fshow(out));
    endrule

    rule doFetch2;
        let {nbSup,in} = f12f2.first;
        f12f2.deq;

        // Get TLB response
        match {.phys_pc, .cause} <- tlb_server.response.get;

        // Access main mem or boot rom
        Bool access_mmio = False;
        if (!isValid(cause)) begin
            case(mmio.getFetchTarget(phys_pc))
                MainMem: begin
                    // Send ICache request
                    mem_server.request.put(phys_pc);
                end
                BootRom: begin
                    // Send MMIO req. Luckily boot rom is also aligned with
                    // cache line size, so all nbSup+1 insts can be fetched
                    // from boot rom. It won't happen that insts fetched from
                    // boot rom is less than requested.
                    mmio.bootRomReq(phys_pc, nbSup);
                    access_mmio = True;
                end
                default: begin
                    // Access fault
                    cause = Valid (InstAccessFault);
                end
            endcase
        end

        let out = Fetch2ToFetch3 {
            pc: in.pc,
            phys_pc: phys_pc,
            pred_next_pc: in.pred_next_pc,
            cause: cause,
            access_mmio: access_mmio,
            decode_epoch: in.decode_epoch,
            main_epoch: in.main_epoch };
        f22f3.enq(tuple2(nbSup,out));
        if (verbose) $display("Fetch2: ", fshow(out));
    endrule

// Break out of i$
    rule doFetch3;
        let {nbSup, fetch3In} = f22f3.first;
        f22f3.deq();
        if (verbose) $display("Fetch3 %d",fetch3In.pc);

        // Get ICache/MMIO response if no exception
        // In case of exception, we still need to process at least inst_data[0]
        // (it will be turned to an exception later), so inst_data[0] must be
        // valid.
        Vector#(SupSize,Maybe#(Instruction)) inst_d = replicate(tagged Valid (0));
        if(!isValid(fetch3In.cause)) begin
            if(fetch3In.access_mmio) begin
                if(verbose) $display("get answer from MMIO %d", fetch3In.pc);
                inst_d <- mmio.bootRomResp;
            end
            else begin
                if(verbose) $display("get answer from memory %d", fetch3In.pc);
                inst_d <- mem_server.response.get;
            end
        end
        if(verbose) $display("epoch instr: %d, epoch main : %d", fetch3In.main_epoch, f_main_epoch);
        instdata.enq(inst_d);
        f32d.enq(f22f3.first);
    endrule

    rule doDecode;
        let {nbSup, fetch3In} = f32d.first;
        f32d.deq();
        let inst_data = instdata.first();
        instdata.deq();
        // The main_epoch check is required to make sure this stage doesn't
        // redirect the PC if a later stage already redirected the PC.
        if (fetch3In.main_epoch == f_main_epoch) begin
            Bool decode_epoch_local = decode_epoch; // next value for decode epoch
            Maybe#(Addr) redirectPc = Invalid; // next pc redirect by branch predictor
            Maybe#(TrainNAP) trainNAP = Invalid; // training data sent to next addr pred
`ifdef PERF_COUNT
            // performance counter: inst being redirect by decode stage
            // Note that only 1 redirection may happen in a cycle
            Maybe#(IType) redirectInst = Invalid;
`endif

            for (Integer i = 0; i < valueof(SupSize); i=i+1) begin
                if (inst_data[i] != tagged Invalid && fromInteger(i) <= nbSup) begin
                    // get the input to decode
                    let in = Fetch3ToDecode {
                        pc: fetch3In.pc+fromInteger(4*i),
                        // last inst, next pc may not be pc+4
                        ppc: fromInteger(i) == nbSup ? fetch3In.pred_next_pc :
                                                       fetch3In.pc+fromInteger(4*(i+1)),
                        decode_epoch: fetch3In.decode_epoch,
                        main_epoch: fetch3In.main_epoch,
                        inst: fromMaybe(?,inst_data[i]),
                        cause: fetch3In.cause
                    };
                    let cause = in.cause;
                    if (verbose) $display("Decode %d\n",i);

                    // do decode and branch prediction
                    // Drop here if does not match the decode_epoch.
                    if (in.decode_epoch == decode_epoch_local) begin
                        doAssert(in.main_epoch == f_main_epoch, "main epoch must match");

                        let decode_result = decode(in.inst);

                        // update cause if there was not an early detected exception
                        if (!isValid(cause)) begin
                            cause = decode_result.illegalInst ? tagged Valid IllegalInst : tagged Invalid;
                        end

                        let dInst = decode_result.dInst;
                        let regs = decode_result.regs;
                        DirPredTrainInfo dp_train = ?; // dir pred training bookkeeping

                        // update predicted next pc
                        if (!isValid(cause)) begin
                            // direction predict
                            Bool pred_taken = False;
                            if(dInst.iType == Br) begin
                                let pred_res <- dirPred.pred[i].pred(in.pc);
                                pred_taken = pred_res.taken;
                                dp_train = pred_res.train;
                            end
                            Maybe#(Addr) nextPc = decodeBrPred(in.pc, dInst, pred_taken);
                            function Bool linkedR(Maybe#(ArchRIndx) register);
                                    let res=False;
                                    if (register matches tagged Valid .r &&& (r == tagged Gpr 1 || r == tagged Gpr 5)) res=True;
				    return res;
                            endfunction

                            // return address stack only push  when rd = x1 or x5
                            if (dInst.iType == J && linkedR(regs.dst)) begin
                                // function call -- push return address to stack
                                ras.pushAddress[i](in.pc + 4);
                            end
                            else if (dInst.iType == Jr ) begin // jalr
                                // we push if rs1 != rd and both are x1 or x5
                                if( regs.src1 != regs.dst && linkedR(regs.src1) && linkedR(regs.dst)) begin
                                    ras.pushAddress[i](in.pc+4);
                                    let new_ppc = ras.firstAddress[i];
                                    ras.popAddress[i];
                                    nextPc = tagged Valid new_ppc;
                                end
                                // we push if rd = x1 or x5 and rs1 is not x1 && x5
                                if (!linkedR(regs.src1) && linkedR(regs.dst)) begin
                                    ras.pushAddress[i](in.pc+4);
                                end
                                // we push if rd = rs1 = x1 or x5
                                if (regs.src1 == regs.dst && linkedR(regs.src1)) begin
                                    ras.pushAddress[i](in.pc+4);
                                end
                                // we pop if rd != x1 or x5, and rs1 = x1 or x5
                                if (!linkedR(regs.dst) && linkedR(regs.src1)) begin
                                    let new_ppc = ras.firstAddress[i];
                                    ras.popAddress[i];
                                    nextPc = tagged Valid new_ppc;
                                end
                            end

                            if(verbose) begin
                                $display("Branch prediction: ", fshow(dInst.iType), " ; ", fshow(in.pc), " ; ",
                                         fshow(in.ppc), " ; ", fshow(pred_taken), " ; ", fshow(nextPc));
                            end

                            // check previous mispred
                            if (nextPc matches tagged Valid .decode_pred_next_pc &&& decode_pred_next_pc != in.ppc) begin
                                if (verbose) $display("ppc and decodeppc :  %h %h", in.ppc, decode_pred_next_pc);
                                decode_epoch_local = !decode_epoch_local;
                                redirectPc = Valid (decode_pred_next_pc); // record redirect next pc
                                in.ppc = decode_pred_next_pc;
                                // train next addr pred when mispredict
                                trainNAP = Valid (TrainNAP {pc: in.pc, nextPc: decode_pred_next_pc});
`ifdef PERF_COUNT
                                // performance stats: record decode redirect
                                doAssert(redirectInst == Invalid, "at most 1 decode redirect per cycle");
                                redirectInst = Valid (dInst.iType);
`endif
                            end
                        end
                        let out = FromFetchStage{pc: in.pc,
                                                 ppc: in.ppc,
                                                 main_epoch: in.main_epoch,
                                                 dpTrain: dp_train,
                                                 inst: in.inst,
                                                 dInst: dInst,
                                                 regs: decode_result.regs,
                                                 cause: cause };
                        out_fifo.enqS[i].enq(out);
                        if (verbose) $display("Decode: ", fshow(out));
                    end
                    else begin
                        if (verbose) $display("Drop decoded within a superscalar");
                        // just drop wrong path instructions
                    end
                end
                else if (inst_data[i] == tagged Invalid && fromInteger(i) <= nbSup) begin
                    // inst num is less than expected; this should not happen
                    // because both I$ and boot rom are aligned to cache line
                    // size.
                    doAssert(False, "Fetched insts not enough");
                end
            end

            // update PC and epoch
            if(redirectPc matches tagged Valid .nextPc) begin
                pc_reg[pc_decode_port] <= nextPc;
            end
            decode_epoch <= decode_epoch_local;
            // send training data for next addr pred
            if (trainNAP matches tagged Valid .x) begin
                napTrainByDec.enq(x);
            end
`ifdef PERF_COUNT
            // performance counter: check whether redirect happens
            if(redirectInst matches tagged Valid .iType &&& doStats) begin
                case(iType)
                    Br: decRedirectBrCnt.incr(1);
                    J : decRedirectJmpCnt.incr(1);
                    Jr: decRedirectJrCnt.incr(1);
                    default: decRedirectOtherCnt.incr(1);
                endcase
            end
`endif
        end
        else begin
            if (verbose) $display("drop in fetch3decode");
        end
    endrule

        // train next addr pred
    (* fire_when_enabled*)
    rule doTrainNAP( isValid(napTrainByExe.wget));
        // give priority to train data from exe
        TrainNAP train = fromMaybe(?, napTrainByExe.wget);
        napTrainByDec.deq();
        nextAddrPred.update(train.pc, train.nextPc, train.nextPc != train.pc + 4);
    endrule
    rule doTrainNAPDec;
        TrainNAP train = napTrainByDec.first();
        napTrainByDec.deq();
        nextAddrPred.update(train.pc, train.nextPc, train.nextPc != train.pc + 4);
    endrule

    interface Vector pipelines = out_fifo.deqS;
    interface iTlbIfc = iTlb;
    interface iMemIfc = iMem;
    interface mmioIfc = mmio.toCore;

    method Action start(Addr start_pc);
        pc_reg[0] <= start_pc;
        started <= True;
        waitForRedirect <= False;
        waitForFlush <= False;
    endmethod
    method Action stop();
        started <= False;
    endmethod

    method Action setWaitRedirect;
        waitForRedirect <= True;
        setWaitRedirect_redirect_conflict.wset(?); // conflict with redirect
    endmethod
    method Action redirect(Addr new_pc);
        if (verbose) $display("Redirect: newpc %h, old f_main_epoch %d, new f_main_epoch %d",new_pc,f_main_epoch,f_main_epoch+1);
        pc_reg[pc_redirect_port] <= new_pc;
        f_main_epoch <= (f_main_epoch == fromInteger(valueOf(NumEpochs)-1)) ? 0 : f_main_epoch + 1;
        // redirect comes, stop stalling for redirect
        waitForRedirect <= False;
        setWaitRedirect_redirect_conflict.wset(?); // conflict with setWaitForRedirect
        // this redirect may be caused by a trap/system inst in commit stage
        // we conservatively set wait for flush TODO make this an input parameter
        waitForFlush <= True;
    endmethod
    method Action done_flushing() if (waitForFlush);
        // signal that the pipeline can resume fetching
        waitForFlush <= False;
        // XXX The guard prevents the readyToFetch rule in Core.bsv from firing every cycle
        // The guard also makes this method sequence before (restricted) redirect method
        // So the effect of setting waitForFlush in redirect method will not be overwritten
        // Then we don't need to make two methods conflict
        // It's fine for the effect of this method to be overwritten, because it fires very often
    endmethod

    method Action train_predictors(
        Addr pc, Addr next_pc, IType iType, Bool taken,
        DirPredTrainInfo dpTrain, Bool mispred
    );
        //if (iType == J || (iType == Br && next_pc < pc)) begin
        //    // Only train the next address predictor for jumps and backward branches
        //    // next_pc != pc + 4 is a substitute for taken
        //    nextAddrPred.update(pc, next_pc, taken);
        //end
        if (iType == Br) begin
            // Train the direction predictor for all branches
            dirPred.update(pc, taken, dpTrain, mispred);
        end
        // train next addr pred when mispred
        if(mispred) begin
            napTrainByExe.wset(TrainNAP {pc: pc, nextPc: next_pc});
        end
    endmethod

    method FetchDebugState getFetchState;
        return FetchDebugState {
            pc: pc_reg[0],
            waitForRedirect: waitForRedirect,
            waitForFlush: waitForFlush,
            mainEp: f_main_epoch
        };
    endmethod

    interface Perf perf;
        method Action setStatus(Bool stats);
`ifdef PERF_COUNT
            doStats <= stats;
`else
            noAction;
`endif
        endmethod

        method Action req(DecStagePerfType r);
            perfReqQ.enq(r);
        endmethod

        method ActionValue#(PerfResp#(DecStagePerfType)) resp;
`ifdef PERF_COUNT
            perfRespQ.deq;
            return perfRespQ.first;
`else
            perfReqQ.deq;
            return PerfResp {
                pType: perfReqQ.first,
                data: 0
            };
`endif
        endmethod

`ifdef PERF_COUNT
        method Bool respValid = perfRespQ.notEmpty;
`else
        method Bool respValid = perfReqQ.notEmpty;
`endif
    endinterface
endmodule


added a bunch of debug probes

added a bunch of debug probes

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
import Clocks::*;
import Vector::*;
import BuildVector::*;
import GetPut::*;
import Connectable::*;
import ClientServer::*;
import ConnectalMemory::*;
import FIFOF::*;
import Gearbox::*;
import GearboxGetPut::*;
import MemTypes::*;
import MemReadEngine::*;
import MemWriteEngine::*;
import Pipe::*;
import AxiDdr3Controller::*;
import GetPutWithClocks::*;
import AxiMasterSlave::*;
import Axi4MasterSlave::*;
import AxiDdr3Wrapper  ::*;
import AxiDma::*;
import ConnectalConfig::*;
import HostInterface::*;
import Probe::*;

interface Ddr3TestRequest;
   method Action startWriteDram(Bit#(32) sglId);
   method Action startReadDram(Bit#(32) sglId);
endinterface

interface Ddr3TestIndication;
   method Action writeDone(Bit#(32) v);
   method Action readDone(Bit#(32) v);
endinterface

interface Ddr3Test;
   interface Ddr3TestRequest request;
   interface Vector#(1, MemReadClient#(DataBusWidth)) readClient;
   interface Vector#(1, MemWriteClient#(DataBusWidth)) writeClient;
   interface Ddr3Pins ddr3;
endinterface

typedef TDiv#(Ddr3DataWidth,DataBusWidth) BusRatio;
typedef TDiv#(Ddr3DataWidth,8) Ddr3DataBytes;

module mkDdr3Test#(HostInterface host, Ddr3TestIndication indication)(Ddr3Test);

   let clock <- exposeCurrentClock();
   let reset <- exposeCurrentReset();

   let transferLen = 256;

   Clock clk200 = host.tsys_clk_200mhz_buf;

   let ddr3Controller <- mkDdr3(clk200);
   MemReadEngine#(DataBusWidth,DataBusWidth,1,1)  re <- mkMemReadEngine();
   MemWriteEngine#(DataBusWidth,DataBusWidth,1,1)  we <- mkMemWriteEngine();

   FIFOF#(Bit#(32))   writeReqFifo <- mkFIFOF();
   FIFOF#(Bit#(32))   readReqFifo <- mkFIFOF();

   Probe#(Bit#(Ddr3AddrWidth)) aw_req_probe <- mkProbe();
   Reg#(Bit#(Ddr3AddrWidth)) dramWriteLimitProbe <- mkReg(0);
   Reg#(Bit#(Ddr3AddrWidth)) dramReadLimitProbe <- mkReg(0);

   Probe#(Bit#(Ddr3AddrWidth)) awAddrProbe <- mkProbe(clocked_by ddr3Controller.uiClock, reset_by ddr3Controller.uiReset);
   Probe#(Bit#(8)) awLenProbe <- mkProbe(clocked_by ddr3Controller.uiClock, reset_by ddr3Controller.uiReset);
   Probe#(Bit#(3)) awSizeProbe <- mkProbe(clocked_by ddr3Controller.uiClock, reset_by ddr3Controller.uiReset);
   Probe#(Vector#(BusRatio,Bit#(DataBusWidth))) wdataProbe <- mkProbe(clocked_by ddr3Controller.uiClock, reset_by ddr3Controller.uiReset);
   Probe#(Axi4WriteResponse#(6)) bProbe <- mkProbe(clocked_by ddr3Controller.uiClock, reset_by ddr3Controller.uiReset);
   Probe#(Bit#(Ddr3AddrWidth)) arAddrProbe <- mkProbe(clocked_by ddr3Controller.uiClock, reset_by ddr3Controller.uiReset);
   Probe#(Bit#(8)) arLenProbe <- mkProbe(clocked_by ddr3Controller.uiClock, reset_by ddr3Controller.uiReset);
   Probe#(Bit#(3)) arSizeProbe <- mkProbe(clocked_by ddr3Controller.uiClock, reset_by ddr3Controller.uiReset);
   Probe#(Vector#(BusRatio,Bit#(DataBusWidth))) rdataProbe <- mkProbe(clocked_by ddr3Controller.uiClock, reset_by ddr3Controller.uiReset);
   let sglWriteProbe <- mkProbe();
   let sglReadProbe <- mkProbe();

   Gearbox#(1,BusRatio,Bit#(DataBusWidth)) dramWriteGearbox <- mk1toNGearbox(clock, reset, ddr3Controller.uiClock, ddr3Controller.uiReset);
   SyncFIFOIfc#(Axi4WriteRequest#(Ddr3AddrWidth,6)) awfifo <- mkSyncFIFO(8, clock, reset, ddr3Controller.uiClock);
   SyncFIFOIfc#(Axi4WriteResponse#(6)) bfifo <- mkSyncFIFO(8, ddr3Controller.uiClock, ddr3Controller.uiReset, clock);
   //mkConnection(toGet(awfifo), ddr3Controller.slave.req_aw);
   //mkConnection(ddr3Controller.slave.resp_b, toPut(bfifo));
   rule rl_awfifo;
      let req <- toGet(awfifo).get();
      awAddrProbe <= req.address;
      awLenProbe <= req.len;
      awSizeProbe <= req.size;
      ddr3Controller.slave.req_aw.put(req);
   endrule
   rule rl_bfifo;
      let b <- ddr3Controller.slave.resp_b.get();
      bProbe <= b;
      bfifo.enq(b);
   endrule

   Reg#(Bit#(Ddr3AddrWidth)) dramWriteOffset <- mkReg(0);
   Reg#(Bit#(Ddr3AddrWidth)) dramWriteLimit <- mkReg(0);
   rule rl_req_aw if (dramWriteOffset < dramWriteLimit);
      Axi4WriteRequest#(Ddr3AddrWidth,6) req = Axi4WriteRequest {
	 address: truncate(dramWriteOffset),
	 len: 0, // indicates 1 beat of data
	 size: axiBusSize(valueOf(Ddr3DataWidth)),
	 id: 0,
	 burst: 2'b01,
	 prot: 3'b000,   //ignored
	 cache: 4'b0011, //ignored
	 lock: 2'b00,    //ignored
	 qos: 4'b0000    //ignored
	 };
      awfifo.enq(req);
      aw_req_probe <= req.address;
      dramWriteOffset <= dramWriteOffset + 1;
   endrule

   rule rl_wdata_gb;
      let rdata <- toGet(re.readServers[0].data).get();
      dramWriteGearbox.enq(vec(rdata.data));
   endrule
   rule rl_wdata;
      let mds <- toGet(dramWriteGearbox).get();
      wdataProbe <= mds;
      ddr3Controller.slave.resp_write.put(Axi4WriteData {
	 data: pack(mds),
	 byteEnable: maxBound,
	 last: 1,
	 id: 0
	 });
   endrule

   rule rl_b;
      // consume the writeDone
      let b <- toGet(bfifo).get();
      // let's see an indication, but we should be counting how many words were sent
      indication.writeDone(extend(b.id));
   endrule

   rule rl_write_start;
      let sglId <- toGet(writeReqFifo).get();
      dramWriteOffset <= 0;
      dramWriteLimit <= transferLen/fromInteger(valueOf(Ddr3DataBytes));
      dramWriteLimitProbe <= transferLen/fromInteger(valueOf(Ddr3DataBytes));
      re.readServers[0].request.put(MemengineCmd { sglId: sglId,
						  base: 0,
						  burstLen: 128,
						  len: transferLen,
						  tag: 0
						  });
   endrule

   Gearbox#(BusRatio,1,Bit#(DataBusWidth)) dramReadGearbox <- mkNto1Gearbox(ddr3Controller.uiClock, ddr3Controller.uiReset, clock, reset);
   SyncFIFOIfc#(Axi4ReadRequest#(Ddr3AddrWidth,6)) arfifo <- mkSyncFIFO(8, clock, reset, ddr3Controller.uiClock);
   //mkConnection(toGet(arfifo), ddr3Controller.slave.req_ar);
   rule rl_arfifo;
      let req <- toGet(arfifo).get();
      arAddrProbe <= req.address;
      arLenProbe <= req.len;
      arSizeProbe <= req.size;
      ddr3Controller.slave.req_ar.put(req);
   endrule

   Reg#(Bit#(Ddr3AddrWidth)) dramReadOffset <- mkReg(0);
   Reg#(Bit#(Ddr3AddrWidth)) dramReadLimit <- mkReg(0);
   rule rl_req_ar if (dramReadOffset < dramReadLimit);
      Axi4ReadRequest#(Ddr3AddrWidth,6) req = Axi4ReadRequest {
	 address: truncate(dramReadOffset),
	 len: 0, // indicates one beat of data
	 size: axiBusSize(valueOf(Ddr3DataWidth)),
	 id: 0,
	 burst: 2'b01,
	 prot: 3'b000,   //ignored
	 cache: 4'b0011, //ignored
	 lock: 2'b00,    //ignored
	 qos: 4'b0000    //ignored
	 };
      arfifo.enq(req);
      dramReadOffset <= dramReadOffset + 1;
   endrule

   rule rl_rdata;
      let resp <- ddr3Controller.slave.resp_read.get();
      Vector#(BusRatio, Bit#(DataBusWidth)) data = unpack(resp.data);
      rdataProbe <= data;
      dramReadGearbox.enq(data);
   endrule
   FIFOF#(Bit#(DataBusWidth)) slackFifo <- mkFIFOF();
   rule rl_rdata_gb;
      Bit#(DataBusWidth) rdata <- toGet(dramReadGearbox).get();
      slackFifo.enq(rdata);
   endrule
   rule rl_rdata_slack;
      let rdata <- toGet(slackFifo).get();
      //fixme last field
      we.writeServers[0].data.enq(rdata);
   endrule

   rule rl_read_start;
      let sglId <- toGet(readReqFifo).get();
      dramReadOffset <= 0;
      dramReadLimit <= transferLen/fromInteger(valueOf(Ddr3DataBytes));
      dramReadLimitProbe <= transferLen/fromInteger(valueOf(Ddr3DataBytes));
      we.writeServers[0].request.put(MemengineCmd { sglId: sglId,
						   base: 0,
						   burstLen: 128,
						   len: transferLen,
						   tag: 0
						   });
   endrule

   rule rl_read_done;
      let done <- we.writeServers[0].done.get();
      indication.readDone(0);
   endrule

   interface Ddr3TestRequest request;
      method Action startWriteDram(Bit#(32) sglId);
	 sglWriteProbe <= sglId;
	 writeReqFifo.enq(sglId);
      endmethod
      method Action startReadDram(Bit#(32) sglId);
	 sglReadProbe <= sglId;
	 readReqFifo.enq(sglId);
      endmethod
   endinterface
   interface MemReadClient readClient = vec(re.dmaClient);
   interface MemWriteClient writeClient = vec(we.dmaClient);
   interface AxiDdr3 ddr3 = ddr3Controller.ddr3;
endmodule

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
import Clocks::*;
import Vector::*;
import BuildVector::*;
import GetPut::*;
import Connectable::*;
import ClientServer::*;
import ConnectalMemory::*;
import FIFOF::*;
import Gearbox::*;
import GearboxGetPut::*;
import MemTypes::*;
import MemReadEngine::*;
import MemWriteEngine::*;
import Pipe::*;
import AxiDdr3Controller::*;
import GetPutWithClocks::*;
import AxiMasterSlave::*;
import Axi4MasterSlave::*;
import AxiDdr3Wrapper  ::*;
import AxiDma::*;
import ConnectalConfig::*;
import HostInterface::*;

interface Ddr3TestRequest;
   method Action startWriteDram(Bit#(32) sglId);
   method Action startReadDram(Bit#(32) sglId);
endinterface

interface Ddr3TestIndication;
   method Action writeDone(Bit#(32) v);
   method Action readDone(Bit#(32) v);
endinterface

interface Ddr3Test;
   interface Ddr3TestRequest request;
   interface Vector#(1, MemReadClient#(DataBusWidth)) readClient;
   interface Vector#(1, MemWriteClient#(DataBusWidth)) writeClient;
   interface Ddr3Pins ddr3;
endinterface

typedef TDiv#(Ddr3DataWidth,DataBusWidth) BusRatio;
typedef TDiv#(Ddr3DataWidth,8) Ddr3DataBytes;

module mkDdr3Test#(HostInterface host, Ddr3TestIndication indication)(Ddr3Test);

   let clock <- exposeCurrentClock();
   let reset <- exposeCurrentReset();

   let transferLen = 256;

   Clock clk200 = host.tsys_clk_200mhz_buf;

   let ddr3Controller <- mkDdr3(clk200);
   MemReadEngine#(DataBusWidth,DataBusWidth,1,1)  re <- mkMemReadEngine();
   MemWriteEngine#(DataBusWidth,DataBusWidth,1,1)  we <- mkMemWriteEngine();

   FIFOF#(Bit#(32))   writeReqFifo <- mkFIFOF();
   FIFOF#(Bit#(32))   readReqFifo <- mkFIFOF();

   Gearbox#(1,BusRatio,Bit#(DataBusWidth)) dramWriteGearbox <- mk1toNGearbox(clock, reset, ddr3Controller.uiClock, ddr3Controller.uiReset);
   SyncFIFOIfc#(Axi4WriteRequest#(Ddr3AddrWidth,6)) awfifo <- mkSyncFIFO(8, clock, reset, ddr3Controller.uiClock);
   SyncFIFOIfc#(Axi4WriteResponse#(6)) bfifo <- mkSyncFIFO(8, ddr3Controller.uiClock, ddr3Controller.uiReset, clock);
   mkConnection(toGet(awfifo), ddr3Controller.slave.req_aw);
   mkConnection(ddr3Controller.slave.resp_b, toPut(bfifo));

   Reg#(Bit#(Ddr3AddrWidth)) dramWriteOffset <- mkReg(0);
   Reg#(Bit#(Ddr3AddrWidth)) dramWriteLimit <- mkReg(0);
   rule rl_req_aw if (dramWriteOffset < dramWriteLimit);
      awfifo.enq(Axi4WriteRequest {
	 address: truncate(dramWriteOffset),
	 len: 1,
	 size: axiBusSize(valueOf(Ddr3DataWidth)),
	 id: 0,
	 burst: 2'b01,
	 prot: 3'b000,   //ignored
	 cache: 4'b0011, //ignored
	 lock: 2'b00,    //ignored
	 qos: 4'b0000    //ignored
	 });
      dramWriteOffset <= dramWriteOffset + 1;
   endrule

   rule rl_wdata_gb;
      let rdata <- toGet(re.readServers[0].data).get();
      dramWriteGearbox.enq(vec(rdata.data));
   endrule
   rule rl_wdata;
      let mds <- toGet(dramWriteGearbox).get();
      ddr3Controller.slave.resp_write.put(Axi4WriteData {
	 data: pack(mds),
	 byteEnable: maxBound,
	 last: 1,
	 id: 0
	 });
   endrule

   rule rl_b;
      // consume the writeDone
      let b <- toGet(bfifo).get();
      // let's see an indication, but we should be counting how many words were sent
      indication.writeDone(extend(b.id));
   endrule

   rule rl_write_start;
      let sglId <- toGet(writeReqFifo).get();
      dramWriteOffset <= 0;
      dramWriteLimit <= transferLen/fromInteger(valueOf(Ddr3DataBytes));
      dramWriteLimitProbe <= transferLen/fromInteger(valueOf(Ddr3DataBytes));
      re.readServers[0].request.put(MemengineCmd { sglId: sglId,
						  base: 0,
						  burstLen: 128,
						  len: transferLen,
						  tag: 0
						  });
   endrule

   Gearbox#(BusRatio,1,Bit#(DataBusWidth)) dramReadGearbox <- mkNto1Gearbox(ddr3Controller.uiClock, ddr3Controller.uiReset, clock, reset);
   SyncFIFOIfc#(Axi4ReadRequest#(Ddr3AddrWidth,6)) arfifo <- mkSyncFIFO(8, clock, reset, ddr3Controller.uiClock);
   mkConnection(toGet(arfifo), ddr3Controller.slave.req_ar);

   Reg#(Bit#(Ddr3AddrWidth)) dramReadOffset <- mkReg(0);
   Reg#(Bit#(Ddr3AddrWidth)) dramReadLimit <- mkReg(0);
   rule rl_req_ar if (dramReadOffset < dramReadLimit);
      arfifo.enq(Axi4ReadRequest {
	 address: truncate(dramReadOffset),
	 len: 0, // indicates one beat of data
	 size: axiBusSize(valueOf(Ddr3DataWidth)),
	 id: 0,
	 burst: 2'b01,
	 prot: 3'b000,   //ignored
	 cache: 4'b0011, //ignored
	 lock: 2'b00,    //ignored
	 qos: 4'b0000    //ignored
	 });
      dramReadOffset <= dramReadOffset + 1;
   endrule

   rule rl_rdata;
      let resp <- ddr3Controller.slave.resp_read.get();
      Vector#(BusRatio, Bit#(DataBusWidth)) data = unpack(resp.data);
      dramReadGearbox.enq(data);
   endrule
   FIFOF#(Bit#(DataBusWidth)) slackFifo <- mkFIFOF();
   rule rl_rdata_gb;
      Bit#(DataBusWidth) rdata <- toGet(dramReadGearbox).get();
      slackFifo.enq(rdata);
   endrule
   rule rl_rdata_slack;
      let rdata <- toGet(slackFifo).get();
      //fixme last field
      we.writeServers[0].data.enq(rdata);
   endrule

   rule rl_read_start;
      let sglId <- toGet(readReqFifo).get();
      dramReadOffset <= 0;
      dramReadLimit <= transferLen/fromInteger(valueOf(Ddr3DataBytes));
      dramReadLimitProbe <= transferLen/fromInteger(valueOf(Ddr3DataBytes));
      we.writeServers[0].request.put(MemengineCmd { sglId: sglId,
						   base: 0,
						   burstLen: 128,
						   len: transferLen,
						   tag: 0
						   });
   endrule

   rule rl_read_done;
      let done <- we.writeServers[0].done.get();
      indication.readDone(0);
   endrule

   interface Ddr3TestRequest request;
      method Action startWriteDram(Bit#(32) sglId);
	 writeReqFifo.enq(sglId);
      endmethod
      method Action startReadDram(Bit#(32) sglId);
	 readReqFifo.enq(sglId);
      endmethod
   endinterface
   interface MemReadClient readClient = vec(re.dmaClient);
   interface MemWriteClient writeClient = vec(we.dmaClient);
   interface AxiDdr3 ddr3 = ddr3Controller.ddr3;
endmodule