MIFO -> MIMO

MIFO -> MIMO

// Copyright (c) 2014 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
import Vector::*;
import GetPut::*;
import ClientServer::*;
import Gearbox::*;
import MIFO::*;
import DefaultValue::*;
import MIMO::*;
import Pipe::*;
import Portal::*;
import MemTypes::*;
import ConnectalMemory::*;

typedef enum {
   Idle,
   HeadRequested,
   TailRequested,
   RequestMessage,
   MessageHeaderRequested,
   MessageRequested,
   Drain,
   UpdateTail,
   UpdateHead,
   UpdateHead2,
   Waiting,
   SendHeader,
   SendMessage,
   SendPadding,
   Stop
   } SharedMemoryPortalState deriving (Bits,Eq);

module mkSharedMemoryRequestPortal#(PipePortal#(numRequests, numIndications, 32) portal,
    MemreadServer#(64) readEngine, MemwriteServer#(64) writeEngine)(SharedMemoryPortal#(64));
   // read the head and tail pointers, if they are different, then read a request
   Reg#(Bit#(32)) limitReg <- mkReg(0);
   Reg#(Bit#(32)) headReg <- mkReg(0);
   Reg#(Bit#(32)) tailReg <- mkReg(0);
   Reg#(Bit#(16)) countReg <- mkReg(0);
   Reg#(Bit#(16)) messageWordsReg <- mkReg(0);
   Reg#(Bit#(16)) methodIdReg <- mkReg(0);
   Reg#(SharedMemoryPortalState) state <- mkReg(Idle);
   Reg#(Bit#(32)) sglIdReg <- mkReg(0);
   Reg#(Bool)     readyReg   <- mkReg(False);
   MIMOConfiguration mimoConfig = defaultValue;
   MIMO#(4,1,4,Bit#(32)) readMimo <- mkMIMO(mimoConfig);

   let verbose = False;

   rule updateReqHeadTail if (state == Idle && readyReg);
      readEngine.request.put(
          MemengineCmd {sglId: sglIdReg, base: 0, burstLen: 16, len: 16, tag: 0});
      state <= HeadRequested;
   endrule

   rule receiveReqHeadTail if (state == HeadRequested || state == TailRequested);
      let v <- toGet(readEngine.data).get();
      let w0 = v.data[31:0];
      let w1 = v.data[63:32];
      let head = headReg;
      let tail = tailReg;
      if (state == HeadRequested) begin
         limitReg <= w0;
         headReg <= w1;
         head = w1;
         state <= TailRequested;
      end
      else begin
         if (tailReg == 0) begin
            tail = w0;
            tailReg <= tail;
         end
         if (tail != headReg)
            state <= RequestMessage;
         else
            state <= Idle;
      end
      if (verbose)
         $display("receiveReqHeadTail state=%d w0=%x w1=%x head=%d tail=%d limit=%d", state, w0, w1, head, tail, limitReg);
   endrule

   rule requestMessage if (state == RequestMessage);
      Bit#(32) wordCount = headReg - tailReg;
      if ((tailReg & 1) == 1) begin
         $display("WARNING requestMessage: reqTail=%d is odd.", tailReg);
      end
      let tail = tailReg + wordCount;
      if (headReg < tailReg) begin
         $display("requestMessage wrapped: head=%d tail=%d", headReg, tailReg);
         wordCount = limitReg - tailReg;
         tail = 4;
      end
      if (verbose) $display("requestMessage id=%d tail=%h head=%h wordCount=%d", sglIdReg, tailReg, headReg, wordCount);
      tailReg <= tail;
      countReg <= truncate(wordCount);
      readEngine.request.put( MemengineCmd
          {sglId: sglIdReg, base: extend(tailReg << 2), burstLen: 16, len: wordCount << 2, tag: 0});
      state <= MessageHeaderRequested;
   endrule

   let enqCount = 2;
   rule demuxwords if (state != HeadRequested && state != TailRequested && readMimo.enqReadyN(enqCount));
      let v <- toGet(readEngine.data).get();
      Vector#(2,Bit#(32)) dvec = unpack(v.data);
      Vector#(4,Bit#(32)) dvec4;
      dvec4[0] = dvec[0];
      dvec4[1] = dvec[1];
      dvec4[2] = 0;
      dvec4[3] = 0;
      readMimo.enq(enqCount, dvec4);
   endrule

   rule receiveMessageHeader if (state == MessageHeaderRequested);
      let vec <- toGet(toPipeOut(readMimo)).get();
      let hdr = vec[0];
      let methodId = hdr[31:16];
      let messageWords = hdr[15:0];
      methodIdReg <= methodId;
      if (verbose)
         $display("receiveMessageHeader hdr=%x methodId=%x messageWords=%d wordCount=%d", hdr, methodId, messageWords, countReg);
      countReg <= countReg - 1;
      messageWordsReg <= messageWords - 1;
      if (hdr == 0) begin
         if (countReg == 1)
            state <= UpdateTail;
         else
            state <= Drain;
      end
      else if (countReg == 1)
         state <= UpdateTail;
      else if (messageWords == 1)
         state <= MessageHeaderRequested;
      else
         state <= MessageRequested;
   endrule

   rule drain if (state == Drain);
      let vec <- toGet(readMimo).get();
      if (countReg == 1)
         state <= UpdateTail;
      countReg <= countReg - 1;
   endrule

   rule receiveMessage if (state == MessageRequested);
      let vec <- toGet(toPipeOut(readMimo)).get();
      let data = vec[0];
      if (verbose)
         $display("receiveMessage data=%x messageWords=%d wordCount=%d", data, messageWordsReg, countReg);
      if (methodIdReg != 16'hFFFF)
         portal.requests[methodIdReg].enq(data);
      messageWordsReg <= messageWordsReg - 1;
      countReg <= countReg - 1;
      if (countReg <= 1)
         state <= UpdateTail;
      else if (messageWordsReg == 1)
         state <= MessageHeaderRequested;
   endrule

   rule updateTail if (state == UpdateTail);
      if (verbose)
         $display("updateTail: tail=%d", tailReg);
      // update the tail pointer
      writeEngine.request.put(
          MemengineCmd {sglId: sglIdReg, base: 8, len: 8, burstLen: 8, tag: 0});
      writeEngine.data.enq(extend(tailReg));
      state <= Waiting;
   endrule

   rule waiting if (state == Waiting);
      let done <- writeEngine.done.get();
      state <= Idle;
   endrule

   interface SharedMemoryPortalConfig cfg;
      method Action setSglId(Bit#(32) id);
         sglIdReg <= id;
         readyReg <= True;
      endmethod
   endinterface
endmodule

module mkSharedMemoryIndicationPortal#(PipePortal#(numRequests, numIndications, 32) portal,
    MemreadServer#(64) readEngine, MemwriteServer#(64) writeEngine)(SharedMemoryPortal#(64));
   let defaultClock <- exposeCurrentClock;
   let defaultReset <- exposeCurrentReset;
   // read the head and tail pointers, if they are different, then read a request
   Reg#(Bit#(16)) limitReg <- mkReg(0);
   Reg#(Bit#(16)) headReg <- mkReg(0);
   Reg#(Bit#(16)) tailReg <- mkReg(0);
   Reg#(Bit#(16)) messageWordsReg <- mkReg(0);
   Reg#(Bit#(16)) methodIdReg <- mkReg(0);
   Reg#(Bool) paddingReg <- mkReg(False);
   Reg#(SharedMemoryPortalState) state <- mkReg(Idle);
   Reg#(Bit#(32)) sglIdReg <- mkReg(0);
   Reg#(Bool)     readyReg   <- mkReg(False);
   Vector#(numIndications, Bool) readyBits = map(pipeOutNotEmpty, portal.indications);
   Bool      interruptStatus = False;
   Bit#(16)  readyChannel = -1;
   function Bool pipeOutNotEmpty(PipeOut#(a) po); return po.notEmpty(); endfunction
   Gearbox#(1,2,Bit#(32)) gb <- mk1toNGearbox(defaultClock, defaultReset, defaultClock, defaultReset);

   let verbose = True;

   for (Integer i = valueOf(numIndications) - 1; i >= 0; i = i - 1) begin
      if (readyBits[i]) begin
         interruptStatus = True;
         readyChannel = fromInteger(i);
      end
   end

   rule updateIndHeadTail if (state == Idle && readyReg);
      readEngine.request.put(
          MemengineCmd {sglId: sglIdReg, base: 0, burstLen: 16, len: 16, tag: 0});
      state <= HeadRequested;
   endrule

   rule receiveIndHeadTail if (state == HeadRequested || state == TailRequested);
      let md <- toGet(readEngine.data).get();
      let data = md.data;
      let w0 = data[31:0];
      let w1 = data[63:32];
      let head = headReg;
      let tail = tailReg;
      if (state == HeadRequested) begin
         limitReg <= truncate(w0);
         headReg <= truncate(w1);
         head = truncate(w1);
         state <= TailRequested;
      end
      else begin
         if (tailReg == 0) begin
            tail = truncate(w0);
            tailReg <= tail;
         end
         //if (tail != headReg)
            state <= SendHeader;
         //else
            //state <= Idle;
      end
      if (verbose)
         $display("receiveIndHeadTail state=%d w0=%x w1=%x head=%d tail=%d limit=%d", state, w0, w1, head, tail, limitReg);
   endrule

   rule send64bits;
      let v = gb.first;
      gb.deq();
      writeEngine.data.enq(pack(v));
   endrule

   rule sendHeader if (state == SendHeader && interruptStatus);
      Bit#(16) messageBits = portal.messageSize.size(readyChannel);
      Bit#(16) roundup = messageBits[4:0] == 0 ? 0 : 1;
      Bit#(16) numWords = (messageBits >> 5) + roundup;
      Bit#(16) totalWords = numWords + 1;
      Bit#(32) hdr = extend(readyChannel) << 16 | extend(numWords + 1);
      if (numWords[0] == 0)
         totalWords = numWords + 2;
      paddingReg <= numWords[0] == 0;
      $display("sendHeader hdr=%h messageBits=%d numWords=%d totalWords=%d paddingReg=%d headReg=%h", hdr, messageBits, numWords, totalWords, paddingReg, headReg);
      headReg <= headReg + totalWords;
      messageWordsReg <= numWords;
      methodIdReg <= readyChannel;
      gb.enq(replicate(hdr));
      writeEngine.request.put( MemengineCmd
          {sglId: sglIdReg, base: extend(headReg) << 2, burstLen: 8, len: extend(totalWords) << 2, tag: 0});
      state <= SendMessage;
   endrule

   rule sendMessage if (state == SendMessage);
      messageWordsReg <= messageWordsReg - 1;
      let v = portal.indications[methodIdReg].first;
      portal.indications[methodIdReg].deq();
      gb.enq(replicate(v));
      $display("sendMessage v=%h messageWords=%d", v, messageWordsReg);
      if (messageWordsReg == 1) begin
         if (paddingReg)
            state <= SendPadding;
         else
            state <= UpdateHead;
      end
   endrule

   rule sendPadding if (state == SendPadding);
      $display("sendPadding");
      gb.enq(replicate(32'hffff0001));
      state <= UpdateHead;
   endrule

   rule updateHead if (state == UpdateHead);
      $display("updateIndHead limit=%d head=%d", limitReg, headReg);
      gb.enq(replicate(extend(limitReg)));
      writeEngine.request.put(
             MemengineCmd {sglId: sglIdReg, base: 0 << 2, burstLen: 8, len: 2 << 2, tag: 0});
      state <= UpdateHead2;
   endrule

   rule updateHead2 if (state == UpdateHead2);
      $display("updateIndHead2");
      gb.enq(replicate(extend(headReg)));
      state <= SendHeader;
   endrule

   rule done;
      let done <- writeEngine.done.get();
   endrule

   interface SharedMemoryPortalConfig cfg;
      method Action setSglId(Bit#(32) id);
         sglIdReg <= id;
         readyReg <= True;
      endmethod
   endinterface
endmodule

// Copyright (c) 2014 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
import Vector::*;
import GetPut::*;
import ClientServer::*;
import Gearbox::*;
import MIFO::*;
import Pipe::*;
import Portal::*;
import MemTypes::*;
import ConnectalMemory::*;

typedef enum {
   Idle,
   HeadRequested,
   TailRequested,
   RequestMessage,
   MessageHeaderRequested,
   MessageRequested,
   Drain,
   UpdateTail,
   UpdateHead,
   UpdateHead2,
   Waiting,
   SendHeader,
   SendMessage,
   SendPadding,
   Stop
   } SharedMemoryPortalState deriving (Bits,Eq);

module mkSharedMemoryRequestPortal#(PipePortal#(numRequests, numIndications, 32) portal,
    MemreadServer#(64) readEngine, MemwriteServer#(64) writeEngine)(SharedMemoryPortal#(64));
   // read the head and tail pointers, if they are different, then read a request
   Reg#(Bit#(32)) limitReg <- mkReg(0);
   Reg#(Bit#(32)) headReg <- mkReg(0);
   Reg#(Bit#(32)) tailReg <- mkReg(0);
   Reg#(Bit#(16)) countReg <- mkReg(0);
   Reg#(Bit#(16)) messageWordsReg <- mkReg(0);
   Reg#(Bit#(16)) methodIdReg <- mkReg(0);
   Reg#(SharedMemoryPortalState) state <- mkReg(Idle);
   Reg#(Bit#(32)) sglIdReg <- mkReg(0);
   Reg#(Bool)     readyReg   <- mkReg(False);
   MIFO#(4,1,4,Bit#(32)) readMifo <- mkMIFO();

   let verbose = False;

   rule updateReqHeadTail if (state == Idle && readyReg);
      readEngine.request.put(
          MemengineCmd {sglId: sglIdReg, base: 0, burstLen: 16, len: 16, tag: 0});
      state <= HeadRequested;
   endrule

   rule receiveReqHeadTail if (state == HeadRequested || state == TailRequested);
      let v <- toGet(readEngine.data).get();
      let w0 = v.data[31:0];
      let w1 = v.data[63:32];
      let head = headReg;
      let tail = tailReg;
      if (state == HeadRequested) begin
         limitReg <= w0;
         headReg <= w1;
         head = w1;
         state <= TailRequested;
      end
      else begin
         if (tailReg == 0) begin
            tail = w0;
            tailReg <= tail;
         end
         if (tail != headReg)
            state <= RequestMessage;
         else
            state <= Idle;
      end
      if (verbose)
         $display("receiveReqHeadTail state=%d w0=%x w1=%x head=%d tail=%d limit=%d", state, w0, w1, head, tail, limitReg);
   endrule

   rule requestMessage if (state == RequestMessage);
      Bit#(32) wordCount = headReg - tailReg;
      if ((tailReg & 1) == 1) begin
         $display("WARNING requestMessage: reqTail=%d is odd.", tailReg);
      end
      let tail = tailReg + wordCount;
      if (headReg < tailReg) begin
         $display("requestMessage wrapped: head=%d tail=%d", headReg, tailReg);
         wordCount = limitReg - tailReg;
         tail = 4;
      end
      if (verbose) $display("requestMessage id=%d tail=%h head=%h wordCount=%d", sglIdReg, tailReg, headReg, wordCount);
      tailReg <= tail;
      countReg <= truncate(wordCount);
      readEngine.request.put( MemengineCmd
          {sglId: sglIdReg, base: extend(tailReg << 2), burstLen: 16, len: wordCount << 2, tag: 0});
      state <= MessageHeaderRequested;
   endrule

   rule demuxwords if (state != HeadRequested && state != TailRequested && readMifo.enqReady());
      let v <- toGet(readEngine.data).get();
      Vector#(2,Bit#(32)) dvec = unpack(v.data);
      let enqCount = 2;
      Vector#(4,Bit#(32)) dvec4;
      dvec4[0] = dvec[0];
      dvec4[1] = dvec[1];
      dvec4[2] = 0;
      dvec4[3] = 0;
      readMifo.enq(enqCount, dvec4);
   endrule

   rule receiveMessageHeader if (state == MessageHeaderRequested);
      let vec <- toGet(readMifo).get();
      let hdr = vec[0];
      let methodId = hdr[31:16];
      let messageWords = hdr[15:0];
      methodIdReg <= methodId;
      if (verbose)
         $display("receiveMessageHeader hdr=%x methodId=%x messageWords=%d wordCount=%d", hdr, methodId, messageWords, countReg);
      countReg <= countReg - 1;
      messageWordsReg <= messageWords - 1;
      if (hdr == 0) begin
         if (countReg == 1)
            state <= UpdateTail;
         else
            state <= Drain;
      end
      else if (countReg == 1)
         state <= UpdateTail;
      else if (messageWords == 1)
         state <= MessageHeaderRequested;
      else
         state <= MessageRequested;
   endrule

   rule drain if (state == Drain);
      let vec <- toGet(readMifo).get();
      if (countReg == 1)
         state <= UpdateTail;
      countReg <= countReg - 1;
   endrule

   rule receiveMessage if (state == MessageRequested);
      let vec <- toGet(readMifo).get();
      let data = vec[0];
      if (verbose)
         $display("receiveMessage data=%x messageWords=%d wordCount=%d", data, messageWordsReg, countReg);
      if (methodIdReg != 16'hFFFF)
         portal.requests[methodIdReg].enq(data);
      messageWordsReg <= messageWordsReg - 1;
      countReg <= countReg - 1;
      if (countReg <= 1)
         state <= UpdateTail;
      else if (messageWordsReg == 1)
         state <= MessageHeaderRequested;
   endrule

   rule updateTail if (state == UpdateTail);
      if (verbose)
         $display("updateTail: tail=%d", tailReg);
      // update the tail pointer
      writeEngine.request.put(
          MemengineCmd {sglId: sglIdReg, base: 8, len: 8, burstLen: 8, tag: 0});
      writeEngine.data.enq(extend(tailReg));
      state <= Waiting;
   endrule

   rule waiting if (state == Waiting);
      let done <- writeEngine.done.get();
      state <= Idle;
   endrule

   interface SharedMemoryPortalConfig cfg;
      method Action setSglId(Bit#(32) id);
         sglIdReg <= id;
         readyReg <= True;
      endmethod
   endinterface
endmodule

module mkSharedMemoryIndicationPortal#(PipePortal#(numRequests, numIndications, 32) portal,
    MemreadServer#(64) readEngine, MemwriteServer#(64) writeEngine)(SharedMemoryPortal#(64));
   let defaultClock <- exposeCurrentClock;
   let defaultReset <- exposeCurrentReset;
   // read the head and tail pointers, if they are different, then read a request
   Reg#(Bit#(16)) limitReg <- mkReg(0);
   Reg#(Bit#(16)) headReg <- mkReg(0);
   Reg#(Bit#(16)) tailReg <- mkReg(0);
   Reg#(Bit#(16)) messageWordsReg <- mkReg(0);
   Reg#(Bit#(16)) methodIdReg <- mkReg(0);
   Reg#(Bool) paddingReg <- mkReg(False);
   Reg#(SharedMemoryPortalState) state <- mkReg(Idle);
   Reg#(Bit#(32)) sglIdReg <- mkReg(0);
   Reg#(Bool)     readyReg   <- mkReg(False);
   Vector#(numIndications, Bool) readyBits = map(pipeOutNotEmpty, portal.indications);
   Bool      interruptStatus = False;
   Bit#(16)  readyChannel = -1;
   function Bool pipeOutNotEmpty(PipeOut#(a) po); return po.notEmpty(); endfunction
   Gearbox#(1,2,Bit#(32)) gb <- mk1toNGearbox(defaultClock, defaultReset, defaultClock, defaultReset);

   let verbose = True;

   for (Integer i = valueOf(numIndications) - 1; i >= 0; i = i - 1) begin
      if (readyBits[i]) begin
         interruptStatus = True;
         readyChannel = fromInteger(i);
      end
   end

   rule updateIndHeadTail if (state == Idle && readyReg);
      readEngine.request.put(
          MemengineCmd {sglId: sglIdReg, base: 0, burstLen: 16, len: 16, tag: 0});
      state <= HeadRequested;
   endrule

   rule receiveIndHeadTail if (state == HeadRequested || state == TailRequested);
      let md <- toGet(readEngine.data).get();
      let data = md.data;
      let w0 = data[31:0];
      let w1 = data[63:32];
      let head = headReg;
      let tail = tailReg;
      if (state == HeadRequested) begin
         limitReg <= truncate(w0);
         headReg <= truncate(w1);
         head = truncate(w1);
         state <= TailRequested;
      end
      else begin
         if (tailReg == 0) begin
            tail = truncate(w0);
            tailReg <= tail;
         end
         //if (tail != headReg)
            state <= SendHeader;
         //else
            //state <= Idle;
      end
      if (verbose)
         $display("receiveIndHeadTail state=%d w0=%x w1=%x head=%d tail=%d limit=%d", state, w0, w1, head, tail, limitReg);
   endrule

   rule send64bits;
      let v = gb.first;
      gb.deq();
      writeEngine.data.enq(pack(v));
   endrule

   rule sendHeader if (state == SendHeader && interruptStatus);
      Bit#(16) messageBits = portal.messageSize.size(readyChannel);
      Bit#(16) roundup = messageBits[4:0] == 0 ? 0 : 1;
      Bit#(16) numWords = (messageBits >> 5) + roundup;
      Bit#(16) totalWords = numWords + 1;
      Bit#(32) hdr = extend(readyChannel) << 16 | extend(numWords + 1);
      if (numWords[0] == 0)
         totalWords = numWords + 2;
      paddingReg <= numWords[0] == 0;
      $display("sendHeader hdr=%h messageBits=%d numWords=%d totalWords=%d paddingReg=%d headReg=%h", hdr, messageBits, numWords, totalWords, paddingReg, headReg);
      headReg <= headReg + totalWords;
      messageWordsReg <= numWords;
      methodIdReg <= readyChannel;
      gb.enq(replicate(hdr));
      writeEngine.request.put( MemengineCmd
          {sglId: sglIdReg, base: extend(headReg) << 2, burstLen: 8, len: extend(totalWords) << 2, tag: 0});
      state <= SendMessage;
   endrule

   rule sendMessage if (state == SendMessage);
      messageWordsReg <= messageWordsReg - 1;
      let v = portal.indications[methodIdReg].first;
      portal.indications[methodIdReg].deq();
      gb.enq(replicate(v));
      $display("sendMessage v=%h messageWords=%d", v, messageWordsReg);
      if (messageWordsReg == 1) begin
         if (paddingReg)
            state <= SendPadding;
         else
            state <= UpdateHead;
      end
   endrule

   rule sendPadding if (state == SendPadding);
      $display("sendPadding");
      gb.enq(replicate(32'hffff0001));
      state <= UpdateHead;
   endrule

   rule updateHead if (state == UpdateHead);
      $display("updateIndHead limit=%d head=%d", limitReg, headReg);
      gb.enq(replicate(extend(limitReg)));
      writeEngine.request.put(
             MemengineCmd {sglId: sglIdReg, base: 0 << 2, burstLen: 8, len: 2 << 2, tag: 0});
      state <= UpdateHead2;
   endrule

   rule updateHead2 if (state == UpdateHead2);
      $display("updateIndHead2");
      gb.enq(replicate(extend(headReg)));
      state <= SendHeader;
   endrule

   rule done;
      let done <- writeEngine.done.get();
   endrule

   interface SharedMemoryPortalConfig cfg;
      method Action setSglId(Bit#(32) id);
         sglIdReg <= id;
         readyReg <= True;
      endmethod
   endinterface
endmodule


fix spelling of zeroExtend

fix spelling of zeroExtend

// Copyright (c) 2013 Quanta Research Cambridge, Inc.
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import FIFO::*;
import FIFOF::*;
import BRAMFIFO::*;
import GetPut::*;
import StmtFSM::*;
import ClientServer::*;

import PortalMemory::*;
import Dma::*;
import GetPutF::*;

import RingTypes::*;
import RingBuffer::*;
import CopyEngine::*;
import EchoEngine::*;
import NopEngine::*;

interface RingRequest;
   method Action set(Bit#(1) cmd, Bit#(3) regist, Bit#(64) addr);
   method Action get(Bit#(1) cmd, Bit#(3) regist);
   method Action hwenable(Bit#(1) en);
   method Action doCommandIndirect(Bit#(64) pointer, Bit#(64) addr);
   method Action doCommandImmediate(Bit#(64) data);
endinterface

interface RingIndication;
   method Action setResult(Bit#(1) cmd, Bit#(3) regist, Bit#(64) addr);
   method Action getResult(Bit#(1) cmd, Bit#(3) regist, Bit#(64) addr);
endinterface

module mkRingRequest#(RingIndication indication,
		      DmaReadServer#(64) dma_read_chan,
		      DmaWriteServer#(64) dma_write_chan,
		      DmaReadServer#(64) cmd_read_chan,
		      DmaWriteServer#(64) status_write_chan )(RingRequest);

   ServerF#(Bit#(64), Bit#(64)) copyEngine <- mkCopyEngine(dma_read_chan, dma_write_chan);
   ServerF#(Bit#(64), Bit#(64)) nopEngine <- mkNopEngine();
   ServerF#(Bit#(64), Bit#(64)) echoEngine <- mkEchoEngine();

   RingBuffer cmdRing <- mkRingBuffer;
   RingBuffer statusRing <- mkRingBuffer;
   Reg#(Bool) hwenabled <- mkReg(False);
   Reg#(Bool) cmdBusy <- mkReg(False);
   Reg#(DmaData#(64)) cmd <- mkReg(DmaData{data:0, tag:0});
   Reg#(Bit#(4)) ii <- mkReg(0);
   Reg#(Bit#(4)) respCtr <- mkReg(0);
   Reg#(Bit#(4)) dispCtr <- mkReg(0);
   Reg#(Bit#(6)) cmdFetchTag <- mkReg(0);
   Reg#(Bit#(6)) statusTag <- mkReg(0);
   Reg#(Bool) cmdFetchEn <- mkReg(False);

   let engineselect = cmd.data[63:56];
   function ServerF#(Bit#(64), Bit#(64)) cmdifc();
      if (engineselect == zeroExtend(pack(CmdNOP)))
	 return nopEngine;
      else if (engineselect == zeroExtend(pack(CmdCOPY)))
	 return copyEngine;
      else if (engineselect == zeroExtend(pack(CmdECHO)))
	 return echoEngine;
      else
	 return nopEngine;
   endfunction

   Stmt cmdFetch =
   seq
//      $display("cmdFetch FSM TOP");
      while (True)
	 seq
	    if (hwenabled)
	       seq
	       if (cmdRing.bufferfirst != cmdRing.bufferlast)
		  seq
		     //$display ("cmdFetch handle=%h address=%h burst=%h tag=%h", cmdRing.mempointer, cmdRing.bufferlast, 8, cmdFetchTag);
		     cmd_read_chan.readReq.put(
			DmaRequest{pointer: cmdRing.mempointer,
			   offset: cmdRing.bufferlast, burstLen: 8, tag: cmdFetchTag});
		     cmdRing.pop(64);
		     cmdFetchTag <= cmdFetchTag + 1;
		  endseq
	       endseq
	 endseq
   endseq;

   Stmt cmdDispatch =
   seq
      while (True) seq
//	 $display("cmdDispatch FSM TOP");
	 seq
	    action
	       let rv <- cmd_read_chan.readData.get();
	       cmd <= rv;
	    endaction
	    // wait a cycle so cmd is valid!
	    //$display("cmdDispatch 0 tag=%h %h", cmd.tag, cmd.data);
	    cmdifc.request.put(cmd.data);
	 endseq
	 for (dispCtr <= 1; dispCtr < 8; dispCtr <= dispCtr + 1)
	    action
	       let rv <- cmd_read_chan.readData.get();
	       //$display("  cmdDispatch %h tag=%h %h", dispCtr, rv.tag, rv.data);
	       cmdifc.request.put(rv.data);
	    endaction
      endseq
   endseq;

   Stmt responseArbiter =
   seq
      while(True) seq
	 if (statusRing.notFull() && copyEngine.response.notEmpty())
	    seq
	       //$display("responseArbiter copyEngine completion");
	       //$display("status write handle=%d address=%h burst=%h tag=%h",
	//	  statusRing.mempointer, statusRing.bufferfirst, 8, statusTag);
	       status_write_chan.writeReq.put(
		  DmaRequest{pointer: statusRing.mempointer,
		     offset: statusRing.bufferfirst, burstLen: 8, tag: statusTag});
	       for (respCtr <= 0; respCtr < 8; respCtr <= respCtr + 1)
		  action
		     let rv <- copyEngine.response.get();
		     status_write_chan.writeData.put(DmaData{data: rv, tag: statusTag});
		  endaction
	       statusRing.push(64);
	       statusTag <= statusTag + 1;
	    endseq

	 if (statusRing.notFull() && echoEngine.response.notEmpty())
	    seq
	       //$display("responseArbiter echoEngine completion");
	       //$display("status write handle=%d address=%h burst=%h tag=%h",
	//	  statusRing.mempointer, statusRing.bufferfirst, 8, statusTag);
	       status_write_chan.writeReq.put(
		  DmaRequest{pointer: statusRing.mempointer,
		     offset: statusRing.bufferfirst, burstLen: 8, tag: statusTag});
	       for (respCtr <= 0; respCtr < 8; respCtr <= respCtr + 1)
		  action
		     let rv <- echoEngine.response.get();
		     status_write_chan.writeData.put(DmaData{data: rv, tag: statusTag});
		  endaction
	       statusRing.push(64);
	       statusTag <= statusTag + 1;
	    endseq

      endseq
   endseq;

   rule writeAck;
      let tag <- status_write_chan.writeDone.get();
      //$display("status write done tag=%h", tag);
   endrule

   mkAutoFSM (cmdFetch);
   mkAutoFSM (cmdDispatch);
   mkAutoFSM (responseArbiter);

      // to start a command, doCommand fires off a memory read to the
      // specified address. when it comes back, the doCommandRule will
      // handle it
      method Action doCommandIndirect(Bit#(64) pointer, Bit#(64) addr);
	 cmd_read_chan.readReq.put(
				   DmaRequest{pointer: truncate(pointer),
	 offset: truncate(addr), burstLen: 8, tag: cmdFetchTag});
   	 cmdFetchTag <= cmdFetchTag + 1;
      endmethod

      method Action doCommandImmediate(Bit#(64) data);
      	 $display("doCommandImmediate %h", data);
      endmethod

      method Action set(Bit#(1) _cmd, Bit#(3) regist, Bit#(64) addr);
	 if (_cmd == 0)
	    cmdRing.configifc.set(regist, truncate(addr));
	 else
	    statusRing.configifc.set(regist, truncate(addr));
	 indication.setResult(_cmd, regist, addr);
      endmethod

      method Action get(Bit#(1) _cmd, Bit#(3) regist);
	 if (_cmd == 0)
	    indication.getResult(0, regist,
	       zeroExtend(cmdRing.configifc.get(regist)));
	 else
	    indication.getResult(0, regist,
	       zeroExtend(statusRing.configifc.get(regist)));
      endmethod

      method Action hwenable(Bit#(1) en);
	 $display ("hwenable set to %h", en);
	 hwenabled <= en == 1;
      endmethod

endmodule

// Copyright (c) 2013 Quanta Research Cambridge, Inc.
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import FIFO::*;
import FIFOF::*;
import BRAMFIFO::*;
import GetPut::*;
import StmtFSM::*;
import ClientServer::*;

import PortalMemory::*;
import Dma::*;
import GetPutF::*;

import RingTypes::*;
import RingBuffer::*;
import CopyEngine::*;
import EchoEngine::*;
import NopEngine::*;

interface RingRequest;
   method Action set(Bit#(1) cmd, Bit#(3) regist, Bit#(64) addr);
   method Action get(Bit#(1) cmd, Bit#(3) regist);
   method Action hwenable(Bit#(1) en);
   method Action doCommandIndirect(Bit#(64) addr);
   method Action doCommandImmediate(Bit#(64) data);
endinterface

interface RingIndication;
   method Action setResult(Bit#(1) cmd, Bit#(3) regist, Bit#(64) addr);
   method Action getResult(Bit#(1) cmd, Bit#(3) regist, Bit#(64) addr);
endinterface

module mkRingRequest#(RingIndication indication,
		      DmaReadServer#(64) dma_read_chan,
		      DmaWriteServer#(64) dma_write_chan,
		      DmaReadServer#(64) cmd_read_chan,
		      DmaWriteServer#(64) status_write_chan )(RingRequest);

   ServerF#(Bit#(64), Bit#(64)) copyEngine <- mkCopyEngine(dma_read_chan, dma_write_chan);
   ServerF#(Bit#(64), Bit#(64)) nopEngine <- mkNopEngine();
   ServerF#(Bit#(64), Bit#(64)) echoEngine <- mkEchoEngine();

   RingBuffer cmdRing <- mkRingBuffer;
   RingBuffer statusRing <- mkRingBuffer;
   Reg#(Bool) hwenabled <- mkReg(False);
   Reg#(Bool) cmdBusy <- mkReg(False);
   Reg#(DmaData#(64)) cmd <- mkReg(DmaData{data:0, tag:0});
   Reg#(Bit#(4)) ii <- mkReg(0);
   Reg#(Bit#(4)) respCtr <- mkReg(0);
   Reg#(Bit#(4)) dispCtr <- mkReg(0);
   Reg#(Bit#(6)) cmdFetchTag <- mkReg(0);
   Reg#(Bit#(6)) statusTag <- mkReg(0);
   Reg#(Bool) cmdFetchEn <- mkReg(False);

   let engineselect = cmd.data[63:56];
   function ServerF#(Bit#(64), Bit#(64)) cmdifc();
      if (engineselect == zeroExtend(pack(CmdNOP)))
	 return nopEngine;
      else if (engineselect == zeroExtend(pack(CmdCOPY)))
	 return copyEngine;
      else if (engineselect == zeroExtend(pack(CmdECHO)))
	 return echoEngine;
      else
	 return nopEngine;
   endfunction

   Stmt cmdFetch =
   seq
//      $display("cmdFetch FSM TOP");
      while (True)
	 seq
	    if (hwenabled)
	       seq
	       if (cmdRing.bufferfirst != cmdRing.bufferlast)
		  seq
		     //$display ("cmdFetch handle=%h address=%h burst=%h tag=%h", cmdRing.memhandle, cmdRing.bufferlast, 8, cmdFetchTag);
		     cmd_read_chan.readReq.put(
			DmaRequest{handle: cmdRing.memhandle,
			   address: cmdRing.bufferlast, burstLen: 8, tag: cmdFetchTag});
		     cmdRing.pop(64);
		     cmdFetchTag <= cmdFetchTag + 1;
		  endseq
	       endseq
	 endseq
   endseq;

   Stmt cmdDispatch =
   seq
      while (True) seq
//	 $display("cmdDispatch FSM TOP");
	 seq
	    action
	       let rv <- cmd_read_chan.readData.get();
	       cmd <= rv;
	    endaction
	    // wait a cycle so cmd is valid!
	    //$display("cmdDispatch 0 tag=%h %h", cmd.tag, cmd.data);
	    cmdifc.request.put(cmd.data);
	 endseq
	 for (dispCtr <= 1; dispCtr < 8; dispCtr <= dispCtr + 1)
	    action
	       let rv <- cmd_read_chan.readData.get();
	       //$display("  cmdDispatch %h tag=%h %h", dispCtr, rv.tag, rv.data);
	       cmdifc.request.put(rv.data);
	    endaction
      endseq
   endseq;

   Stmt responseArbiter =
   seq
      while(True) seq
	 if (statusRing.notFull() && copyEngine.response.notEmpty())
	    seq
	       //$display("responseArbiter copyEngine completion");
	       //$display("status write handle=%d address=%h burst=%h tag=%h",
	//	  statusRing.memhandle, statusRing.bufferfirst, 8, statusTag);
	       status_write_chan.writeReq.put(
		  DmaRequest{handle: statusRing.memhandle,
		     address: statusRing.bufferfirst, burstLen: 8, tag: statusTag});
	       for (respCtr <= 0; respCtr < 8; respCtr <= respCtr + 1)
		  action
		     let rv <- copyEngine.response.get();
		     status_write_chan.writeData.put(DmaData{data: rv, tag: statusTag});
		  endaction
	       statusRing.push(64);
	       statusTag <= statusTag + 1;
	    endseq

	 if (statusRing.notFull() && echoEngine.response.notEmpty())
	    seq
	       //$display("responseArbiter echoEngine completion");
	       //$display("status write handle=%d address=%h burst=%h tag=%h",
	//	  statusRing.memhandle, statusRing.bufferfirst, 8, statusTag);
	       status_write_chan.writeReq.put(
		  DmaRequest{handle: statusRing.memhandle,
		     address: statusRing.bufferfirst, burstLen: 8, tag: statusTag});
	       for (respCtr <= 0; respCtr < 8; respCtr <= respCtr + 1)
		  action
		     let rv <- echoEngine.response.get();
		     status_write_chan.writeData.put(DmaData{data: rv, tag: statusTag});
		  endaction
	       statusRing.push(64);
	       statusTag <= statusTag + 1;
	    endseq

      endseq
   endseq;

   rule writeAck;
      let tag <- status_write_chan.writeDone.get();
      //$display("status write done tag=%h", tag);
   endrule

   mkAutoFSM (cmdFetch);
   mkAutoFSM (cmdDispatch);
   mkAutoFSM (responseArbiter);

      // to start a command, doCommand fires off a memory read to the
      // specified address. when it comes back, the doCommandRule will
      // handle it
      method Action doCommandIndirect(Bit#(64) addr);
	 cmd_read_chan.readReq.put(
				   DmaRequest{handle: addr[63:32],
	 address: truncate(addr[31:0]), burstLen: 8, tag: cmdFetchTag});
   	 cmdFetchTag <= cmdFetchTag + 1;
      endmethod

      method Action doCommandImmediate(Bit#(64) data);
      	 $display("doCommandImmediate %h", data);
      endmethod

      method Action set(Bit#(1) _cmd, Bit#(3) regist, Bit#(64) addr);
	 if (_cmd == 0)
	    cmdRing.configifc.set(regist, truncate(addr));
	 else
	    statusRing.configifc.set(regist, truncate(addr));
	 indication.setResult(_cmd, regist, addr);
      endmethod

      method Action get(Bit#(1) _cmd, Bit#(3) regist);
	 if (_cmd == 0)
	    indication.getResult(0, regist,
	       zeroextend(cmdRing.configifc.get(regist)));
	 else
	    indication.getResult(0, regist,
	       zeroextend(statusRing.configifc.get(regist)));
      endmethod

      method Action hwenable(Bit#(1) en);
	 $display ("hwenable set to %h", en);
	 hwenabled <= en == 1;
      endmethod

endmodule


decoupled loadClient rule from dmaError indication

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

// BSV Libraries
import BRAMFIFO::*;
import FIFO::*;
import FIFOF::*;
import Vector::*;
import GetPut::*;
import ClientServer::*;
import Assert::*;

// XBSV Libraries
import MemTypes::*;
import PortalMemory::*;
import SGList::*;

typedef 9 SGL_PIPELINE_DEPTH;
typedef 32 TAG_DEPTH;

interface MemWriteInternal#(numeric type addrWidth, numeric type dataWidth);
   interface DmaDbg dbg;
   interface MemWriteClient#(addrWidth,dataWidth) write_client;
endinterface

interface MemReadInternal#(numeric type addrWidth, numeric type dataWidth);
   interface DmaDbg dbg;
   interface MemReadClient#(addrWidth,dataWidth) read_client;
endinterface

function Bool bad_pointer(ObjectPointer p);
   return ((p >>8) > fromInteger(valueOf(MaxNumSGLists)));
endfunction

interface TagGen#(numeric type numClients, numeric type numTags);
   method Action tag_request(Bit#(TLog#(numClients)) client, Bit#(6) orig_tag);
   method ActionValue#(Bit#(TLog#(numTags)))  tag_response;
   method Action return_tag(Bit#(TLog#(numTags)) tag);
endinterface

module mkTagGenOO(TagGen#(numClients,numTags))
   provisos(Log#(numTags,tagWidth),
	    Log#(numClients,clientWidth));

   let request_fifo0 <- mkSizedFIFOF(valueOf(SGL_PIPELINE_DEPTH));
   let request_fifo1 <- mkSizedFIFOF(valueOf(SGL_PIPELINE_DEPTH));
   let return_fifo <- mkFIFO;
   Vector#(numTags, Reg#(Bit#(TLog#(TAG_DEPTH)))) tag_regs <- replicateM(mkReg(0));
   Vector#(numTags, Reg#(Maybe#(Tuple2#(Bit#(clientWidth),Bit#(6))))) client_map <- replicateM(mkReg(tagged Invalid));
   Maybe#(UInt#(tagWidth)) next_free = findElem(0, readVReg(tag_regs));

   rule return_tag_rule;
      return_fifo.deq;
      let tag = return_fifo.first;
      tag_regs[tag] <= tag_regs[tag]-1;
      if (tag_regs[tag]==1)
	 client_map[tag] <= tagged Invalid;
   endrule

   rule tag_request_rule;
      request_fifo0.deq;
      let client = tpl_2(request_fifo0.first);
      let orig_tag = tpl_3(request_fifo0.first);
      let rv = case  (tpl_1(request_fifo0.first)) matches
		  tagged Valid .tag: return (_when_(tag_regs[tag] < maxBound) (tag));
		  tagged Invalid: return (_when_(isValid(next_free)) (fromMaybe(?, next_free)));
	       endcase;
      request_fifo1.enq(tuple3(rv,client,orig_tag));
   endrule

   method Action tag_request(Bit#(clientWidth) client, Bit#(6) orig_tag) if (request_fifo0.notFull);
      let rv = findElem(tagged Valid tuple2(client,orig_tag), readVReg(client_map));
      request_fifo0.enq(tuple3(rv,client,orig_tag));
   endmethod

   method ActionValue#(Bit#(tagWidth)) tag_response;
      request_fifo1.deq;
      let rv = tpl_1(request_fifo1.first);
      let client = tpl_2(request_fifo1.first);
      let orig_tag = tpl_3(request_fifo1.first);
      client_map[rv] <= tagged Valid tuple2(client,orig_tag);
      tag_regs[rv] <= tag_regs[rv]+1;
      return extend(pack(rv));
   endmethod

   method Action return_tag(Bit#(tagWidth) tag);
      return_fifo.enq(tag);
   endmethod

endmodule

module mkTagGenIO(TagGen#(numClients,numTags))
   provisos(Log#(numTags,tagWidth),
	    Log#(numClients,clientWidth),
	    Bits#(Bit#(clientWidth), tagWidth));

   let request_fifo <- mkSizedFIFO(valueOf(SGL_PIPELINE_DEPTH));

   method Action tag_request(Bit#(clientWidth) client, Bit#(6) client_tag);
      request_fifo.enq(client);
   endmethod

   method ActionValue#(Bit#(tagWidth)) tag_response;
      request_fifo.deq;
      let client = request_fifo.first;
      return pack(client);
   endmethod

   method Action return_tag(Bit#(tagWidth) tag);
      noAction;
   endmethod

endmodule

typedef struct {ObjectRequest req;
		Bit#(TLog#(numClients)) client; } LRec#(numeric type numClients, numeric type numTags, numeric type addrWidth) deriving(Bits);

typedef struct {ObjectRequest req;
		Bit#(addrWidth) pa;
		Bit#(TLog#(numTags)) rename_tag;
		Bit#(TLog#(numClients)) client; } RRec#(numeric type numClients, numeric type numTags, numeric type addrWidth) deriving(Bits);

typedef struct {ObjectRequest req;
		Bit#(TLog#(numTags)) rename_tag;
		Bit#(TLog#(numClients)) client;
		Bool last; } DRec#(numeric type numClients, numeric type numTags, numeric type addrWidth) deriving(Bits);

typedef struct {Bit#(6) orig_tag;
		Bit#(TLog#(numClients)) client; } RResp#(numeric type numClients, numeric type numTags, numeric type addrWidth) deriving(Bits);

typedef struct {DmaErrorType errorType;
		Bit#(32) pref; } DmaError deriving (Bits);

module mkMemReadInternal#(Integer id,
			  Vector#(numClients, ObjectReadClient#(dataWidth)) readClients,
			  DmaIndication dmaIndication,
			  Server#(Tuple2#(SGListId,Bit#(ObjectOffsetSize)),Bit#(addrWidth)) sgl,
			  TagGen#(numClients, numTags) tag_gen)
   (MemReadInternal#(addrWidth, dataWidth))

   provisos(Add#(b__, addrWidth, 64),
	    Add#(c__, 12, addrWidth),
	    Add#(1, c__, d__),
	    Div#(dataWidth,8,dataWidthBytes),
	    Mul#(dataWidthBytes,8,dataWidth),
	    Log#(dataWidthBytes,beatShift),
	    Add#(a__, TLog#(numTags), 6));

   // stage 0: address translation (latency = SGL_PIPELINE_DEPTH)
   FIFO#(LRec#(numClients,numTags,addrWidth)) lreqFifo <- mkSizedFIFO(valueOf(SGL_PIPELINE_DEPTH));
   // stage 1: address validation (latency = 1)
   FIFO#(RRec#(numClients,numTags,addrWidth))  reqFifo <- mkFIFO;
   // stage 2: read commands (maximum buffering to handle high latency read response times)
   Vector#(numTags, FIFOF#(DRec#(numClients,numTags,addrWidth))) dreqFifos <- replicateM(mkSizedBRAMFIFOF(valueOf(TAG_DEPTH)));
   // stage 3: read data (minimal buffering required)
   FIFO#(Tuple2#(DRec#(numClients,numTags,addrWidth),MemData#(dataWidth))) readDataPipelineFifo <- mkFIFO;
   FIFO#(MemData#(dataWidth)) responseFifo <- mkFIFO;
   Vector#(numTags, Reg#(Bit#(8)))           burstRegs <- replicateM(mkReg(0));
   Vector#(numTags, Reg#(Bool))              firstRegs <- replicateM(mkReg(True));
   Vector#(numTags, Reg#(Bool))               lastRegs <- replicateM(mkReg(False));
   Reg#(Bit#(64))  beatCount <- mkReg(0);
   let beat_shift = fromInteger(valueOf(beatShift));

   // performance analytics
   Reg#(Bit#(64)) cycle_cnt <- mkReg(0);
   Reg#(Bit#(64)) last_loadClient <- mkReg(0);
   Reg#(Bit#(64)) last_sglResp <- mkReg(0);
   Reg#(Bit#(64)) last_eob <- mkReg(0);
   (* fire_when_enabled *)
   rule cycle;
      cycle_cnt <= cycle_cnt+1;
   endrule

   FIFO#(DmaError) dmaErrorFifo <- mkFIFO();
   rule dmaError;
      let error <- toGet(dmaErrorFifo).get();
      dmaIndication.dmaError(extend(pack(error.errorType)), error.pref, 0, 0);
   endrule

   for (Integer selectReg = 0; selectReg < valueOf(numClients); selectReg = selectReg + 1)
      rule loadClient;
      	 //$display("mkMemReadInternal::loadClient %d %d", selectReg, cycle_cnt-last_loadClient);
	 //last_loadClient <= cycle_cnt;
   	 ObjectRequest req <- readClients[selectReg].readReq.get();
   	 if (bad_pointer(req.pointer))
	    dmaErrorFifo.enq(DmaError { errorType: DmaErrorBadPointer4, pref: req.pointer });
   	 else begin
	    tag_gen.tag_request(fromInteger(selectReg), req.tag);
   	    lreqFifo.enq(LRec{req:req, client:fromInteger(selectReg)});
   	    sgl.request.put(tuple2(truncate(req.pointer),req.offset));
   	 end
      endrule

   rule checkSglResp;
      let physAddr <- sgl.response.get;
      let req = lreqFifo.first.req;
      let client = lreqFifo.first.client;
      let rename_tag <- tag_gen.tag_response;
      lreqFifo.deq();
      reqFifo.enq(RRec{req:req, pa:physAddr, client:client, rename_tag:extend(rename_tag)});
      //$display("checkSglResp: client=%d, rename_tag=%d", client,rename_tag);
      //$display("mkMemReadInternal::sglResp %d %d", client, cycle_cnt-last_sglResp);
      //last_sglResp <= cycle_cnt;
   endrule

   rule readDataComp;
      readDataPipelineFifo.deq;
      let drq = tpl_1(readDataPipelineFifo.first);
      let response = tpl_2(readDataPipelineFifo.first);
      let client = drq.client;
      let req = drq.req;
      readClients[client].readData.put(ObjectData { data: response.data, tag: req.tag, last: False});
      //$display("readDataComp: %d %h", client, response.data);
      beatCount <= beatCount+1;
   endrule

   rule read_client_response;
      let response <- toGet(responseFifo).get();
      Bit#(6) response_tag = response.tag;
      let dreqFifo = dreqFifos[response_tag];
      dynamicAssert(truncate(response_tag) == dreqFifo.first.rename_tag, "mkMemReadInternal");
      readDataPipelineFifo.enq(tuple2(dreqFifo.first, response));
      let burstLen = burstRegs[response_tag];
      let first =    firstRegs[response_tag];
      let last  =    lastRegs[response_tag];
      if (first) begin
	 burstLen = dreqFifo.first.req.burstLen >> beat_shift;
	 last = dreqFifo.first.last;
      end
      if (last) begin
	 //$display("mkMemReadInternal::eob %d", cycle_cnt-last_eob);
	 last_eob <= cycle_cnt;
	 dreqFifo.deq();
	 tag_gen.return_tag(truncate(response_tag));
      end
      burstRegs[response_tag] <= burstLen-1;
      firstRegs[response_tag] <= (burstLen-1 == 0);
      lastRegs[response_tag] <= (burstLen-1 == 1);
   endrule

   interface MemReadClient read_client;
      interface Get readReq;
	 method ActionValue#(MemRequest#(addrWidth)) get();
	    reqFifo.deq;
	    let req = reqFifo.first.req;
	    let physAddr = reqFifo.first.pa;
	    let client = reqFifo.first.client;
	    let rename_tag = reqFifo.first.rename_tag;
	    if (False && physAddr[31:24] != 0)
	       $display("req_ar: funny physAddr req.pointer=%d req.offset=%h physAddr=%h", req.pointer, req.offset, physAddr);
	    dreqFifos[rename_tag].enq(DRec{req:req, client:client, rename_tag:rename_tag, last:(req.burstLen == beat_shift)});
	    //$display("readReq: client=%d, rename_tag=%d, physAddr=%h", client,rename_tag,physAddr);
	    return MemRequest{addr:physAddr, burstLen:req.burstLen, tag:extend(rename_tag)};
	 endmethod
      endinterface
      interface Put readData = toPut(responseFifo);
   endinterface
   interface DmaDbg dbg;
      method ActionValue#(DmaDbgRec) dbg();
	 // Bit#(1) ne0 = pack(dreqFifos[0].notEmpty);
	 // Bit#(1) ne1 = pack(dreqFifos[1].notEmpty);
	 // return DmaDbgRec{x:extend(burstRegs[0]), y:extend(burstRegs[1]), z:extend(ne0), w:extend(ne1)};
	 return DmaDbgRec{x:0, y:0, z:0, w:0};
      endmethod
      method ActionValue#(Bit#(64)) getMemoryTraffic();
	 return beatCount;
      endmethod
   endinterface
endmodule

module mkMemWriteInternal#(Integer iid,
			   Vector#(numClients, ObjectWriteClient#(dataWidth)) writeClients,
			   DmaIndication dmaIndication,
			   Server#(Tuple2#(SGListId,Bit#(ObjectOffsetSize)),Bit#(addrWidth)) sgl,
			   TagGen#(numClients, numTags) tag_gen)
   (MemWriteInternal#(addrWidth, dataWidth))

   provisos(Add#(b__, addrWidth, 64),
	    Add#(c__, 12, addrWidth),
	    Add#(1, c__, d__),
	    Div#(dataWidth,8,dataWidthBytes),
	    Mul#(dataWidthBytes,8,dataWidth),
	    Log#(dataWidthBytes,beatShift),
	    Add#(a__, TLog#(numTags), 6));

   // stage 0: address translation (latency = SGL_PIPELINE_DEPTH)
   FIFO#(LRec#(numClients,numTags,addrWidth)) lreqFifo <- mkSizedFIFO(valueOf(SGL_PIPELINE_DEPTH));
   // stage 1: address validation (latency = 1)
   FIFO#(RRec#(numClients,numTags,addrWidth))  reqFifo <- mkFIFO;
   // stage 2: write commands (maximum buffering to handle high latency writes)
   FIFO#(DRec#(numClients,numTags,addrWidth)) dreqFifo <- mkSizedBRAMFIFO(valueOf(TMul#(TAG_DEPTH,numTags)));
   // stage 3: write data (maximum buffering, though I have no idea if any hosts will begin the next data transfer before sending the write ack)
   Vector#(numTags, FIFO#(RResp#(numClients,numTags,addrWidth))) respFifos <- replicateM(mkSizedBRAMFIFO(valueOf(TAG_DEPTH)));
   // stage 4: write done (minimal buffering required)
   FIFO#(Tuple2#(RResp#(numClients,numTags,addrWidth),Bit#(6))) writeDonePipelineFifo <- mkFIFO;

   Reg#(Bit#(8)) burstReg <- mkReg(0);
   Reg#(Bit#(64)) beatCount <- mkReg(0);
   let beat_shift = fromInteger(valueOf(beatShift));

   Reg#(Bit#(64)) cycle_cnt <- mkReg(0);
   Reg#(Bit#(64)) last_loadClient <- mkReg(0);
   Reg#(Bit#(64)) last_sglResp <- mkReg(0);
   Reg#(Bit#(64)) last_eob <- mkReg(0);
   (* fire_when_enabled *)
   rule cycle;
      cycle_cnt <= cycle_cnt+1;
   endrule

   FIFO#(DmaError) dmaErrorFifo <- mkFIFO();
   rule dmaError;
      let error <- toGet(dmaErrorFifo).get();
      dmaIndication.dmaError(extend(pack(error.errorType)), error.pref, 0, 0);
   endrule

   for (Integer selectReg = 0; selectReg < valueOf(numClients); selectReg = selectReg + 1)
       rule loadClient;
      	  //$display("mkMemWriteInternal::loadClient %d %d", selectReg, cycle_cnt-last_loadClient);
	  //last_loadClient <= cycle_cnt;
   	  ObjectRequest req <- writeClients[selectReg].writeReq.get();
   	  if (bad_pointer(req.pointer))
	     dmaErrorFifo.enq(DmaError { errorType: DmaErrorBadPointer5, pref: req.pointer });
   	  else begin
	     tag_gen.tag_request(fromInteger(selectReg), req.tag);
   	     lreqFifo.enq(LRec{req:req, client:fromInteger(selectReg)});
   	     sgl.request.put(tuple2(truncate(req.pointer),req.offset));
   	  end
       endrule

   rule checkSglResp;
      let physAddr <- sgl.response.get;
      let req = lreqFifo.first.req;
      let client = lreqFifo.first.client;
      let rename_tag <- tag_gen.tag_response;
      lreqFifo.deq();
      reqFifo.enq(RRec{req:req, pa:physAddr, client:client, rename_tag:extend(rename_tag)});
      //$display("checkSglResp: client=%d, rename_tag=%d", client,rename_tag);
      //$display("mkMemWriteInternal::sglResp %d %d", client, cycle_cnt-last_sglResp);
      //last_sglResp <= cycle_cnt;
   endrule

   rule writeDoneComp;
      writeDonePipelineFifo.deq;
      let client = tpl_1(writeDonePipelineFifo.first).client;
      let orig_tag = tpl_1(writeDonePipelineFifo.first).orig_tag;
      let response_tag = tpl_2(writeDonePipelineFifo.first);
      writeClients[client].writeDone.put(orig_tag);
      tag_gen.return_tag(truncate(response_tag));
   endrule

   interface MemWriteClient write_client;
      interface Get writeReq;
	 method ActionValue#(MemRequest#(addrWidth)) get();
	    let req = reqFifo.first.req;
	    let physAddr = reqFifo.first.pa;
	    let client = reqFifo.first.client;
	    let rename_tag = reqFifo.first.rename_tag;
	    reqFifo.deq;
	    dreqFifo.enq(DRec{req:req, client:client, rename_tag:rename_tag});
	    //$display("writeReq: client=%d, rename_tag=%d", client,rename_tag);
	    return MemRequest{addr:physAddr, burstLen:req.burstLen, tag:extend(rename_tag)};
	 endmethod
      endinterface
      interface Get writeData;
	 method ActionValue#(MemData#(dataWidth)) get();
	    let client = dreqFifo.first.client;
	    let req = dreqFifo.first.req;
	    let rename_tag =dreqFifo.first.rename_tag;
	    ObjectData#(dataWidth) tagdata <- writeClients[client].writeData.get();
	    let burstLen = burstReg;
	    if (burstLen == 0) begin
	       burstLen = req.burstLen >> beat_shift;
	       respFifos[rename_tag].enq(RResp{orig_tag:req.tag, client:client});
	    end
	    burstReg <= burstLen-1;
	    beatCount <= beatCount+1;
	    if (burstLen == 1)
	       dreqFifo.deq();
	    //$display("writeData: client=%d, rename_tag=%d", client, rename_tag);
	    return MemData { data: tagdata.data,  tag:extend(rename_tag), last: False };
	 endmethod
      endinterface
      interface Put writeDone;
	 method Action put(Bit#(6) resp);
	    let response_tag = resp;
	    let respFifo = respFifos[response_tag];
	    writeDonePipelineFifo.enq(tuple2(respFifo.first,resp));
	    respFifo.deq;
	 endmethod
      endinterface
   endinterface
   interface DmaDbg dbg;
      method ActionValue#(DmaDbgRec) dbg();
	 return DmaDbgRec{x:fromInteger(valueOf(numClients)), y:?, z:?, w:?};
      endmethod
      method ActionValue#(Bit#(64)) getMemoryTraffic();
	 return beatCount;
      endmethod
   endinterface
endmodule

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

// BSV Libraries
import BRAMFIFO::*;
import FIFO::*;
import FIFOF::*;
import Vector::*;
import GetPut::*;
import ClientServer::*;
import Assert::*;

// XBSV Libraries
import MemTypes::*;
import PortalMemory::*;
import SGList::*;

typedef 9 SGL_PIPELINE_DEPTH;
typedef 32 TAG_DEPTH;

interface MemWriteInternal#(numeric type addrWidth, numeric type dataWidth);
   interface DmaDbg dbg;
   interface MemWriteClient#(addrWidth,dataWidth) write_client;
endinterface

interface MemReadInternal#(numeric type addrWidth, numeric type dataWidth);
   interface DmaDbg dbg;
   interface MemReadClient#(addrWidth,dataWidth) read_client;
endinterface

function Bool bad_pointer(ObjectPointer p);
   return ((p >>8) > fromInteger(valueOf(MaxNumSGLists)));
endfunction

interface TagGen#(numeric type numClients, numeric type numTags);
   method Action tag_request(Bit#(TLog#(numClients)) client, Bit#(6) orig_tag);
   method ActionValue#(Bit#(TLog#(numTags)))  tag_response;
   method Action return_tag(Bit#(TLog#(numTags)) tag);
endinterface

module mkTagGenOO(TagGen#(numClients,numTags))
   provisos(Log#(numTags,tagWidth),
	    Log#(numClients,clientWidth));

   let request_fifo0 <- mkSizedFIFOF(valueOf(SGL_PIPELINE_DEPTH));
   let request_fifo1 <- mkSizedFIFOF(valueOf(SGL_PIPELINE_DEPTH));
   let return_fifo <- mkFIFO;
   Vector#(numTags, Reg#(Bit#(TLog#(TAG_DEPTH)))) tag_regs <- replicateM(mkReg(0));
   Vector#(numTags, Reg#(Maybe#(Tuple2#(Bit#(clientWidth),Bit#(6))))) client_map <- replicateM(mkReg(tagged Invalid));
   Maybe#(UInt#(tagWidth)) next_free = findElem(0, readVReg(tag_regs));

   rule return_tag_rule;
      return_fifo.deq;
      let tag = return_fifo.first;
      tag_regs[tag] <= tag_regs[tag]-1;
      if (tag_regs[tag]==1)
	 client_map[tag] <= tagged Invalid;
   endrule

   rule tag_request_rule;
      request_fifo0.deq;
      let client = tpl_2(request_fifo0.first);
      let orig_tag = tpl_3(request_fifo0.first);
      let rv = case  (tpl_1(request_fifo0.first)) matches
		  tagged Valid .tag: return (_when_(tag_regs[tag] < maxBound) (tag));
		  tagged Invalid: return (_when_(isValid(next_free)) (fromMaybe(?, next_free)));
	       endcase;
      request_fifo1.enq(tuple3(rv,client,orig_tag));
   endrule

   method Action tag_request(Bit#(clientWidth) client, Bit#(6) orig_tag) if (request_fifo0.notFull);
      let rv = findElem(tagged Valid tuple2(client,orig_tag), readVReg(client_map));
      request_fifo0.enq(tuple3(rv,client,orig_tag));
   endmethod

   method ActionValue#(Bit#(tagWidth)) tag_response;
      request_fifo1.deq;
      let rv = tpl_1(request_fifo1.first);
      let client = tpl_2(request_fifo1.first);
      let orig_tag = tpl_3(request_fifo1.first);
      client_map[rv] <= tagged Valid tuple2(client,orig_tag);
      tag_regs[rv] <= tag_regs[rv]+1;
      return extend(pack(rv));
   endmethod

   method Action return_tag(Bit#(tagWidth) tag);
      return_fifo.enq(tag);
   endmethod

endmodule

module mkTagGenIO(TagGen#(numClients,numTags))
   provisos(Log#(numTags,tagWidth),
	    Log#(numClients,clientWidth),
	    Bits#(Bit#(clientWidth), tagWidth));

   let request_fifo <- mkSizedFIFO(valueOf(SGL_PIPELINE_DEPTH));

   method Action tag_request(Bit#(clientWidth) client, Bit#(6) client_tag);
      request_fifo.enq(client);
   endmethod

   method ActionValue#(Bit#(tagWidth)) tag_response;
      request_fifo.deq;
      let client = request_fifo.first;
      return pack(client);
   endmethod

   method Action return_tag(Bit#(tagWidth) tag);
      noAction;
   endmethod

endmodule

typedef struct {ObjectRequest req;
		Bit#(TLog#(numClients)) client; } LRec#(numeric type numClients, numeric type numTags, numeric type addrWidth) deriving(Bits);

typedef struct {ObjectRequest req;
		Bit#(addrWidth) pa;
		Bit#(TLog#(numTags)) rename_tag;
		Bit#(TLog#(numClients)) client; } RRec#(numeric type numClients, numeric type numTags, numeric type addrWidth) deriving(Bits);

typedef struct {ObjectRequest req;
		Bit#(TLog#(numTags)) rename_tag;
		Bit#(TLog#(numClients)) client;
		Bool last; } DRec#(numeric type numClients, numeric type numTags, numeric type addrWidth) deriving(Bits);

typedef struct {Bit#(6) orig_tag;
		Bit#(TLog#(numClients)) client; } RResp#(numeric type numClients, numeric type numTags, numeric type addrWidth) deriving(Bits);

module mkMemReadInternal#(Integer id,
			  Vector#(numClients, ObjectReadClient#(dataWidth)) readClients,
			  DmaIndication dmaIndication,
			  Server#(Tuple2#(SGListId,Bit#(ObjectOffsetSize)),Bit#(addrWidth)) sgl,
			  TagGen#(numClients, numTags) tag_gen)
   (MemReadInternal#(addrWidth, dataWidth))

   provisos(Add#(b__, addrWidth, 64),
	    Add#(c__, 12, addrWidth),
	    Add#(1, c__, d__),
	    Div#(dataWidth,8,dataWidthBytes),
	    Mul#(dataWidthBytes,8,dataWidth),
	    Log#(dataWidthBytes,beatShift),
	    Add#(a__, TLog#(numTags), 6));

   // stage 0: address translation (latency = SGL_PIPELINE_DEPTH)
   FIFO#(LRec#(numClients,numTags,addrWidth)) lreqFifo <- mkSizedFIFO(valueOf(SGL_PIPELINE_DEPTH));
   // stage 1: address validation (latency = 1)
   FIFO#(RRec#(numClients,numTags,addrWidth))  reqFifo <- mkFIFO;
   // stage 2: read commands (maximum buffering to handle high latency read response times)
   Vector#(numTags, FIFOF#(DRec#(numClients,numTags,addrWidth))) dreqFifos <- replicateM(mkSizedBRAMFIFOF(valueOf(TAG_DEPTH)));
   // stage 3: read data (minimal buffering required)
   FIFO#(Tuple2#(DRec#(numClients,numTags,addrWidth),MemData#(dataWidth))) readDataPipelineFifo <- mkFIFO;
   FIFO#(MemData#(dataWidth)) responseFifo <- mkFIFO;
   Vector#(numTags, Reg#(Bit#(8)))           burstRegs <- replicateM(mkReg(0));
   Vector#(numTags, Reg#(Bool))              firstRegs <- replicateM(mkReg(True));
   Vector#(numTags, Reg#(Bool))               lastRegs <- replicateM(mkReg(False));
   Reg#(Bit#(64))  beatCount <- mkReg(0);
   let beat_shift = fromInteger(valueOf(beatShift));

   // performance analytics
   Reg#(Bit#(64)) cycle_cnt <- mkReg(0);
   Reg#(Bit#(64)) last_loadClient <- mkReg(0);
   Reg#(Bit#(64)) last_sglResp <- mkReg(0);
   Reg#(Bit#(64)) last_eob <- mkReg(0);
   (* fire_when_enabled *)
   rule cycle;
      cycle_cnt <= cycle_cnt+1;
   endrule

   for (Integer selectReg = 0; selectReg < valueOf(numClients); selectReg = selectReg + 1)
      rule loadClient;
      	 //$display("mkMemReadInternal::loadClient %d %d", selectReg, cycle_cnt-last_loadClient);
	 //last_loadClient <= cycle_cnt;
   	 ObjectRequest req <- readClients[selectReg].readReq.get();
   	 if (bad_pointer(req.pointer))
   	    dmaIndication.dmaError(extend(pack(DmaErrorBadPointer4)), req.pointer, 0, 0);
   	 else begin
	    tag_gen.tag_request(fromInteger(selectReg), req.tag);
   	    lreqFifo.enq(LRec{req:req, client:fromInteger(selectReg)});
   	    sgl.request.put(tuple2(truncate(req.pointer),req.offset));
   	 end
      endrule

   rule checkSglResp;
      let physAddr <- sgl.response.get;
      let req = lreqFifo.first.req;
      let client = lreqFifo.first.client;
      let rename_tag <- tag_gen.tag_response;
      lreqFifo.deq();
      reqFifo.enq(RRec{req:req, pa:physAddr, client:client, rename_tag:extend(rename_tag)});
      //$display("checkSglResp: client=%d, rename_tag=%d", client,rename_tag);
      //$display("mkMemReadInternal::sglResp %d %d", client, cycle_cnt-last_sglResp);
      //last_sglResp <= cycle_cnt;
   endrule

   rule readDataComp;
      readDataPipelineFifo.deq;
      let drq = tpl_1(readDataPipelineFifo.first);
      let response = tpl_2(readDataPipelineFifo.first);
      let client = drq.client;
      let req = drq.req;
      readClients[client].readData.put(ObjectData { data: response.data, tag: req.tag, last: False});
      //$display("readDataComp: %d %h", client, response.data);
      beatCount <= beatCount+1;
   endrule

   rule read_client_response;
      let response <- toGet(responseFifo).get();
      Bit#(6) response_tag = response.tag;
      let dreqFifo = dreqFifos[response_tag];
      dynamicAssert(truncate(response_tag) == dreqFifo.first.rename_tag, "mkMemReadInternal");
      readDataPipelineFifo.enq(tuple2(dreqFifo.first, response));
      let burstLen = burstRegs[response_tag];
      let first =    firstRegs[response_tag];
      let last  =    lastRegs[response_tag];
      if (first) begin
	 burstLen = dreqFifo.first.req.burstLen >> beat_shift;
	 last = dreqFifo.first.last;
      end
      if (last) begin
	 //$display("mkMemReadInternal::eob %d", cycle_cnt-last_eob);
	 last_eob <= cycle_cnt;
	 dreqFifo.deq();
	 tag_gen.return_tag(truncate(response_tag));
      end
      burstRegs[response_tag] <= burstLen-1;
      firstRegs[response_tag] <= (burstLen-1 == 0);
      lastRegs[response_tag] <= (burstLen-1 == 1);
   endrule

   interface MemReadClient read_client;
      interface Get readReq;
	 method ActionValue#(MemRequest#(addrWidth)) get();
	    reqFifo.deq;
	    let req = reqFifo.first.req;
	    let physAddr = reqFifo.first.pa;
	    let client = reqFifo.first.client;
	    let rename_tag = reqFifo.first.rename_tag;
	    if (False && physAddr[31:24] != 0)
	       $display("req_ar: funny physAddr req.pointer=%d req.offset=%h physAddr=%h", req.pointer, req.offset, physAddr);
	    dreqFifos[rename_tag].enq(DRec{req:req, client:client, rename_tag:rename_tag, last:(req.burstLen == beat_shift)});
	    //$display("readReq: client=%d, rename_tag=%d, physAddr=%h", client,rename_tag,physAddr);
	    return MemRequest{addr:physAddr, burstLen:req.burstLen, tag:extend(rename_tag)};
	 endmethod
      endinterface
      interface Put readData = toPut(responseFifo);
   endinterface
   interface DmaDbg dbg;
      method ActionValue#(DmaDbgRec) dbg();
	 // Bit#(1) ne0 = pack(dreqFifos[0].notEmpty);
	 // Bit#(1) ne1 = pack(dreqFifos[1].notEmpty);
	 // return DmaDbgRec{x:extend(burstRegs[0]), y:extend(burstRegs[1]), z:extend(ne0), w:extend(ne1)};
	 return DmaDbgRec{x:0, y:0, z:0, w:0};
      endmethod
      method ActionValue#(Bit#(64)) getMemoryTraffic();
	 return beatCount;
      endmethod
   endinterface
endmodule

module mkMemWriteInternal#(Integer iid,
			   Vector#(numClients, ObjectWriteClient#(dataWidth)) writeClients,
			   DmaIndication dmaIndication,
			   Server#(Tuple2#(SGListId,Bit#(ObjectOffsetSize)),Bit#(addrWidth)) sgl,
			   TagGen#(numClients, numTags) tag_gen)
   (MemWriteInternal#(addrWidth, dataWidth))

   provisos(Add#(b__, addrWidth, 64),
	    Add#(c__, 12, addrWidth),
	    Add#(1, c__, d__),
	    Div#(dataWidth,8,dataWidthBytes),
	    Mul#(dataWidthBytes,8,dataWidth),
	    Log#(dataWidthBytes,beatShift),
	    Add#(a__, TLog#(numTags), 6));

   // stage 0: address translation (latency = SGL_PIPELINE_DEPTH)
   FIFO#(LRec#(numClients,numTags,addrWidth)) lreqFifo <- mkSizedFIFO(valueOf(SGL_PIPELINE_DEPTH));
   // stage 1: address validation (latency = 1)
   FIFO#(RRec#(numClients,numTags,addrWidth))  reqFifo <- mkFIFO;
   // stage 2: write commands (maximum buffering to handle high latency writes)
   FIFO#(DRec#(numClients,numTags,addrWidth)) dreqFifo <- mkSizedBRAMFIFO(valueOf(TMul#(TAG_DEPTH,numTags)));
   // stage 3: write data (maximum buffering, though I have no idea if any hosts will begin the next data transfer before sending the write ack)
   Vector#(numTags, FIFO#(RResp#(numClients,numTags,addrWidth))) respFifos <- replicateM(mkSizedBRAMFIFO(valueOf(TAG_DEPTH)));
   // stage 4: write done (minimal buffering required)
   FIFO#(Tuple2#(RResp#(numClients,numTags,addrWidth),Bit#(6))) writeDonePipelineFifo <- mkFIFO;

   Reg#(Bit#(8)) burstReg <- mkReg(0);
   Reg#(Bit#(64)) beatCount <- mkReg(0);
   let beat_shift = fromInteger(valueOf(beatShift));

   Reg#(Bit#(64)) cycle_cnt <- mkReg(0);
   Reg#(Bit#(64)) last_loadClient <- mkReg(0);
   Reg#(Bit#(64)) last_sglResp <- mkReg(0);
   Reg#(Bit#(64)) last_eob <- mkReg(0);
   (* fire_when_enabled *)
   rule cycle;
      cycle_cnt <= cycle_cnt+1;
   endrule

   for (Integer selectReg = 0; selectReg < valueOf(numClients); selectReg = selectReg + 1)
       rule loadClient;
      	  //$display("mkMemWriteInternal::loadClient %d %d", selectReg, cycle_cnt-last_loadClient);
	  //last_loadClient <= cycle_cnt;
   	  ObjectRequest req <- writeClients[selectReg].writeReq.get();
   	  if (bad_pointer(req.pointer))
   	     dmaIndication.dmaError(extend(pack(DmaErrorBadPointer5)), req.pointer, 0, 0);
   	  else begin
	     tag_gen.tag_request(fromInteger(selectReg), req.tag);
   	     lreqFifo.enq(LRec{req:req, client:fromInteger(selectReg)});
   	     sgl.request.put(tuple2(truncate(req.pointer),req.offset));
   	  end
       endrule

   rule checkSglResp;
      let physAddr <- sgl.response.get;
      let req = lreqFifo.first.req;
      let client = lreqFifo.first.client;
      let rename_tag <- tag_gen.tag_response;
      lreqFifo.deq();
      reqFifo.enq(RRec{req:req, pa:physAddr, client:client, rename_tag:extend(rename_tag)});
      //$display("checkSglResp: client=%d, rename_tag=%d", client,rename_tag);
      //$display("mkMemWriteInternal::sglResp %d %d", client, cycle_cnt-last_sglResp);
      //last_sglResp <= cycle_cnt;
   endrule

   rule writeDoneComp;
      writeDonePipelineFifo.deq;
      let client = tpl_1(writeDonePipelineFifo.first).client;
      let orig_tag = tpl_1(writeDonePipelineFifo.first).orig_tag;
      let response_tag = tpl_2(writeDonePipelineFifo.first);
      writeClients[client].writeDone.put(orig_tag);
      tag_gen.return_tag(truncate(response_tag));
   endrule

   interface MemWriteClient write_client;
      interface Get writeReq;
	 method ActionValue#(MemRequest#(addrWidth)) get();
	    let req = reqFifo.first.req;
	    let physAddr = reqFifo.first.pa;
	    let client = reqFifo.first.client;
	    let rename_tag = reqFifo.first.rename_tag;
	    reqFifo.deq;
	    dreqFifo.enq(DRec{req:req, client:client, rename_tag:rename_tag});
	    //$display("writeReq: client=%d, rename_tag=%d", client,rename_tag);
	    return MemRequest{addr:physAddr, burstLen:req.burstLen, tag:extend(rename_tag)};
	 endmethod
      endinterface
      interface Get writeData;
	 method ActionValue#(MemData#(dataWidth)) get();
	    let client = dreqFifo.first.client;
	    let req = dreqFifo.first.req;
	    let rename_tag =dreqFifo.first.rename_tag;
	    ObjectData#(dataWidth) tagdata <- writeClients[client].writeData.get();
	    let burstLen = burstReg;
	    if (burstLen == 0) begin
	       burstLen = req.burstLen >> beat_shift;
	       respFifos[rename_tag].enq(RResp{orig_tag:req.tag, client:client});
	    end
	    burstReg <= burstLen-1;
	    beatCount <= beatCount+1;
	    if (burstLen == 1)
	       dreqFifo.deq();
	    //$display("writeData: client=%d, rename_tag=%d", client, rename_tag);
	    return MemData { data: tagdata.data,  tag:extend(rename_tag), last: False };
	 endmethod
      endinterface
      interface Put writeDone;
	 method Action put(Bit#(6) resp);
	    let response_tag = resp;
	    let respFifo = respFifos[response_tag];
	    writeDonePipelineFifo.enq(tuple2(respFifo.first,resp));
	    respFifo.deq;
	 endmethod
      endinterface
   endinterface
   interface DmaDbg dbg;
      method ActionValue#(DmaDbgRec) dbg();
	 return DmaDbgRec{x:fromInteger(valueOf(numClients)), y:?, z:?, w:?};
      endmethod
      method ActionValue#(Bit#(64)) getMemoryTraffic();
	 return beatCount;
      endmethod
   endinterface
endmodule