introduced mkAxiTop and fixed AXI master connection in memread

// bsv libraries
import SpecialFIFOs::*;
import Vector::*;
import StmtFSM::*;
import FIFO::*;

// portz libraries
import AxiMasterSlave::*;
import Directory::*;
import CtrlMux::*;
import Portal::*;
import Leds::*;
import PortalMemory::*;
import AxiRDMA::*;
import BsimRDMA::*;
import PortalMemory::*;
import PortalRMemory::*;

// for PCIE
import Connectable::*;
import Xilinx            :: *;
import XilinxPCIE        :: *;
import Xilinx7PcieBridge :: *;
import PcieToAxiBridge   :: *;

// generated by tool
import MemreadRequestWrapper::*;
import DMARequestWrapper::*;
import MemreadIndicationProxy::*;
import DMAIndicationProxy::*;

// defined by user
import Memread::*;

interface AxiTop;
   interface StdAxi3Slave     ctrl;
   interface StdAxi3Master    m_axi;
   interface ReadOnly#(Bool)  interrupt;
   interface LEDS             leds;
endinterface

module mkAxiTop(AxiTop);

   DMAIndicationProxy dmaIndicationProxy <- mkDMAIndicationProxy(9);

   MemreadIndicationProxy memreadIndicationProxy <- mkMemreadIndicationProxy(7);
   Memread memread <- mkMemread(memreadIndicationProxy.ifc);
   MemreadRequestWrapper memreadRequestWrapper <- mkMemreadRequestWrapper(1008,memread.request);

   Vector#(1, DMAReadClient#(64)) clients = cons(memread.dmaClient, nil);
`ifdef BSIM
   BsimDMAServer#(64)  dma <- mkBsimDMAServer(dmaIndicationProxy.ifc, clients, nil);
`else
   Integer             numRequests = 2;
   AxiDMAServer#(64,8) dma <- mkAxiDMAServer(dmaIndicationProxy.ifc, numRequests, clients, nil);
`endif

   DMARequestWrapper dmaRequestWrapper <- mkDMARequestWrapper(1005,dma.request);

   Vector#(4,StdPortal) portals;
   portals[0] = memreadRequestWrapper.portalIfc;
   portals[1] = memreadIndicationProxy.portalIfc;
   portals[2] = dmaRequestWrapper.portalIfc;
   portals[3] = dmaIndicationProxy.portalIfc;

   Directory dir <- mkDirectory(portals);
   Vector#(1,StdPortal) directories;
   directories[0] = dir.portalIfc;

   // when constructing ctrl and interrupt muxes, directories must be the first argument
   let ctrl_mux <- mkAxiSlaveMux(directories,portals);
   let interrupt_mux <- mkInterruptMux(directories,portals);
`ifndef BSIM
   let axi_master <- mkAxi3Master(dma.m_axi);
`endif

   interface ReadOnly interrupt = interrupt_mux;
   interface StdAxi3Slave ctrl = ctrl_mux;
`ifndef BSIM
   interface StdAxi3Master m_axi = axi_master;
`endif
endmodule : mkAxiTop

module mkZynqTop(AxiTop);
   let axiTop <- mkAxiTop();
   return axiTop;
endmodule : mkZynqTop

import "BDPI" function Action      initPortal(Bit#(32) d);
import "BDPI" function Bool                    writeReq();
import "BDPI" function ActionValue#(Bit#(32)) writeAddr();
import "BDPI" function ActionValue#(Bit#(32)) writeData();
import "BDPI" function Bool                     readReq();
import "BDPI" function ActionValue#(Bit#(32))  readAddr();
import "BDPI" function Action        readData(Bit#(32) d);

module mkBsimTop();
   AxiTop top <- mkAxiTop;
   let wf <- mkPipelineFIFO;
   let init_seq = (action
		      initPortal(0);
		      initPortal(1);
		      initPortal(2);
		      initPortal(3);
		      initPortal(4);
                   endaction);
   let init_fsm <- mkOnce(init_seq);
   rule init_rule;
      init_fsm.start;
   endrule
   rule wrReq (writeReq());
      let wa <- writeAddr;
      let wd <- writeData;
      top.ctrl.write.writeAddr(wa,0,0,0,0,0,0);
      wf.enq(wd);
   endrule
   rule wrData;
      wf.deq;
      top.ctrl.write.writeData(wf.first,0,0,0);
   endrule
   rule rdReq (readReq());
      let ra <- readAddr;
      top.ctrl.read.readAddr(ra,0,0,0,0,0,0);
   endrule
   rule rdResp;
      let rd <- top.ctrl.read.readData;
      readData(rd);
   endrule
endmodule

(* no_default_clock, no_default_reset *)
module mkPcieTop #(Clock pci_sys_clk_p, Clock pci_sys_clk_n,
                          Clock sys_clk_p,     Clock sys_clk_n,
                          Reset pci_sys_reset_n)
                         (VC707_FPGA);

   let contentId = 64'h4563686f;

   X7PcieBridgeIfc#(8) x7pcie <- mkX7PcieBridge( pci_sys_clk_p, pci_sys_clk_n, sys_clk_p, sys_clk_n, pci_sys_reset_n,
                                                 contentId );

   Reg#(Bool) interruptRequested <- mkReg(False, clocked_by x7pcie.clock125, reset_by x7pcie.reset125);

   // instantiate user portals
   let axiTop <- mkAxiTop(clocked_by x7pcie.clock125, reset_by x7pcie.reset125);

   // connect them to PCIE
   mkConnection(x7pcie.portal0, axiTop.ctrl, clocked_by x7pcie.clock125, reset_by x7pcie.reset125);

   AxiSlaveEngine#(64,8) axiSlaveEngine <- mkAxiSlaveEngine(x7pcie.pciId(), clocked_by x7pcie.clock125, reset_by x7pcie.reset125);
   mkConnection(tpl_1(x7pcie.slave), tpl_2(axiSlaveEngine.tlps), clocked_by x7pcie.clock125, reset_by x7pcie.reset125);
   mkConnection(tpl_1(axiSlaveEngine.tlps), tpl_2(x7pcie.slave), clocked_by x7pcie.clock125, reset_by x7pcie.reset125);
   mkConnection(axiTop.m_axi, axiSlaveEngine.slave3, clocked_by x7pcie.clock125, reset_by x7pcie.reset125);

   rule requestInterrupt;
      if (axiTop.interrupt && !interruptRequested)
	 x7pcie.interrupt();
      interruptRequested <= axiTop.interrupt;
   endrule

   interface pcie = x7pcie.pcie;
   //interface ddr3 = x7pcie.ddr3;
   method leds = zeroExtend({  pack(x7pcie.isCalibrated)
			     , pack(True)
			     , pack(False)
			     , pack(x7pcie.isLinkUp)
			     });

endmodule: mkPcieTop

// bsv libraries
import SpecialFIFOs::*;
import Vector::*;
import StmtFSM::*;
import FIFO::*;

// portz libraries
import AxiMasterSlave::*;
import Directory::*;
import CtrlMux::*;
import Portal::*;
import Leds::*;
import PortalMemory::*;
import AxiRDMA::*;
import BsimRDMA::*;
import PortalMemory::*;
import PortalRMemory::*;

// for PCIE
import Connectable::*;
import Xilinx            :: *;
import XilinxPCIE        :: *;
import Xilinx7PcieBridge :: *;
import PcieToAxiBridge   :: *;

// generated by tool
import MemreadRequestWrapper::*;
import DMARequestWrapper::*;
import MemreadIndicationProxy::*;
import DMAIndicationProxy::*;

// defined by user
import Memread::*;

interface Top;
   interface StdAxi3Slave     ctrl;
   interface StdAxi3Master    m_axi;
   interface ReadOnly#(Bool)  interrupt;
   interface LEDS             leds;
endinterface

module mkZynqTop(Top);

   DMAIndicationProxy dmaIndicationProxy <- mkDMAIndicationProxy(9);

   MemreadIndicationProxy memreadIndicationProxy <- mkMemreadIndicationProxy(7);
   Memread memread <- mkMemread(memreadIndicationProxy.ifc);
   MemreadRequestWrapper memreadRequestWrapper <- mkMemreadRequestWrapper(1008,memread.request);

   Vector#(1, DMAReadClient#(64)) clients = cons(memread.dmaClient, nil);
`ifdef BSIM
   BsimDMAServer#(64)  dma <- mkBsimDMAServer(dmaIndicationProxy.ifc, clients, nil);
`else
   Integer             numRequests = 2;
   AxiDMAServer#(64,8) dma <- mkAxiDMAServer(dmaIndicationProxy.ifc, numRequests, clients, nil);
`endif

   DMARequestWrapper dmaRequestWrapper <- mkDMARequestWrapper(1005,dma.request);

   Vector#(4,StdPortal) portals;
   portals[0] = memreadRequestWrapper.portalIfc;
   portals[1] = memreadIndicationProxy.portalIfc;
   portals[2] = dmaRequestWrapper.portalIfc;
   portals[3] = dmaIndicationProxy.portalIfc;

   Directory dir <- mkDirectory(portals);
   Vector#(1,StdPortal) directories;
   directories[0] = dir.portalIfc;

   // when constructing ctrl and interrupt muxes, directories must be the first argument
   let ctrl_mux <- mkAxiSlaveMux(directories,portals);
   let interrupt_mux <- mkInterruptMux(directories,portals);
`ifndef BSIM
   let axi_master <- mkAxi3Master(dma.m_axi);
`endif

   interface ReadOnly interrupt = interrupt_mux;
   interface StdAxi3Slave ctrl = ctrl_mux;
`ifndef BSIM
   interface StdAxi3Master m_axi = axi_master;
`endif
endmodule

import "BDPI" function Action      initPortal(Bit#(32) d);
import "BDPI" function Bool                    writeReq();
import "BDPI" function ActionValue#(Bit#(32)) writeAddr();
import "BDPI" function ActionValue#(Bit#(32)) writeData();
import "BDPI" function Bool                     readReq();
import "BDPI" function ActionValue#(Bit#(32))  readAddr();
import "BDPI" function Action        readData(Bit#(32) d);

module mkBsimTop();
   Top top <- mkZynqTop;
   let wf <- mkPipelineFIFO;
   let init_seq = (action
		      initPortal(0);
		      initPortal(1);
		      initPortal(2);
		      initPortal(3);
		      initPortal(4);
                   endaction);
   let init_fsm <- mkOnce(init_seq);
   rule init_rule;
      init_fsm.start;
   endrule
   rule wrReq (writeReq());
      let wa <- writeAddr;
      let wd <- writeData;
      top.ctrl.write.writeAddr(wa,0,0,0,0,0,0);
      wf.enq(wd);
   endrule
   rule wrData;
      wf.deq;
      top.ctrl.write.writeData(wf.first,0,0,0);
   endrule
   rule rdReq (readReq());
      let ra <- readAddr;
      top.ctrl.read.readAddr(ra,0,0,0,0,0,0);
   endrule
   rule rdResp;
      let rd <- top.ctrl.read.readData;
      readData(rd);
   endrule
endmodule

(* no_default_clock, no_default_reset *)
module mkPcieTop #(Clock pci_sys_clk_p, Clock pci_sys_clk_n,
                          Clock sys_clk_p,     Clock sys_clk_n,
                          Reset pci_sys_reset_n)
                         (VC707_FPGA);

   let contentId = 64'h4563686f;

   X7PcieBridgeIfc#(8) x7pcie <- mkX7PcieBridge( pci_sys_clk_p, pci_sys_clk_n, sys_clk_p, sys_clk_n, pci_sys_reset_n,
                                                 contentId );

   Reg#(Bool) interruptRequested <- mkReg(False, clocked_by x7pcie.clock125, reset_by x7pcie.reset125);

   // instantiate user portals
   DMAIndicationProxy dmaIndicationProxy <- mkDMAIndicationProxy(9, clocked_by x7pcie.clock125, reset_by x7pcie.reset125);

   MemreadIndicationProxy memreadIndicationProxy <- mkMemreadIndicationProxy(7, clocked_by x7pcie.clock125, reset_by x7pcie.reset125);
   Memread memread <- mkMemread(memreadIndicationProxy.ifc, clocked_by x7pcie.clock125, reset_by x7pcie.reset125);
   MemreadRequestWrapper memreadRequestWrapper <- mkMemreadRequestWrapper(1008,memread.request, clocked_by x7pcie.clock125, reset_by x7pcie.reset125);

   Vector#(1, DMAReadClient#(64)) clients = cons(memread.dmaClient, nil);
   Integer             numRequests = 2;
   AxiDMAServer#(64,8) dma <- mkAxiDMAServer(dmaIndicationProxy.ifc, numRequests, clients, nil,
					     clocked_by x7pcie.clock125, reset_by x7pcie.reset125);

   DMARequestWrapper dmaRequestWrapper <- mkDMARequestWrapper(1005,dma.request, clocked_by x7pcie.clock125, reset_by x7pcie.reset125);

   Vector#(4,StdPortal) portals;
   portals[0] = memreadRequestWrapper.portalIfc;
   portals[1] = memreadIndicationProxy.portalIfc;
   portals[2] = dmaRequestWrapper.portalIfc;
   portals[3] = dmaIndicationProxy.portalIfc;

   Directory dir <- mkDirectory(portals, clocked_by x7pcie.clock125, reset_by x7pcie.reset125);
   Vector#(1,StdPortal) directories;
   directories[0] = dir.portalIfc;

   // when constructing ctrl and interrupt muxes, directories must be the first argument
   let ctrl_mux <- mkAxiSlaveMux(directories,portals, clocked_by x7pcie.clock125, reset_by x7pcie.reset125);
   let interrupt_mux <- mkInterruptMux(directories,portals, clocked_by x7pcie.clock125, reset_by x7pcie.reset125);
   let axi_master <- mkAxi3Master(dma.m_axi, clocked_by x7pcie.clock125, reset_by x7pcie.reset125);

   // connect them to PCIE
   mkConnection(x7pcie.portal0, ctrl_mux, clocked_by x7pcie.clock125, reset_by x7pcie.reset125);

   rule requestInterrupt;
      if (interrupt_mux && !interruptRequested)
	 x7pcie.interrupt();
      interruptRequested <= interrupt_mux;
   endrule

   interface pcie = x7pcie.pcie;
   //interface ddr3 = x7pcie.ddr3;
   method leds = zeroExtend({  pack(x7pcie.isCalibrated)
			     , pack(True)
			     , pack(False)
			     , pack(x7pcie.isLinkUp)
			     });

endmodule: mkPcieTop


debug printout

debug printout

// Copyright (c) 2014 Quanta Research Cambridge, Inc.
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import SerialFIFO::*;
import NocNode::*;
import LinkHost::*;
import Connectable::*;
import StmtFSM::*;
import Vector::*;
import FIFOF::*;
import Pipe::*;

interface NocIndication;
   method Action ack(Bit#(4) recvnode, Bit#(4) to, Bit#(32) message);
endinterface

interface NocRequest;
   method Action send(Bit#(4) sendnode, Bit#(4) to, Bit#(32) message);
endinterface

module mkNocRequest#(NocIndication indication)(NocRequest);

   SerialFIFO#(DataMessage) xxx <- mkSerialFIFO();
   Vector#(5, SerialFIFO#(DataMessage)) we <- replicateM( mkSerialFIFO );
   Vector#(5, SerialFIFO#(DataMessage)) ew <- replicateM( mkSerialFIFO );

   // discard traffic from loose ends
   rule discardeast;
      $display("we[4] discard %x", we[4].out.first);
      we[4].out.deq();
      endrule

   rule discardwest;
      $display("ew[0] discard %x", ew[0].out.first);
      ew[0].out.deq();
      endrule

   Vector#(4, SerialFIFO#(DataMessage)) node;

    for (Bit#(4) i = 0; i < 4; i = i + 1)
    begin
        node[i] <- mkNocNode(unpack(i),
	    SerialFIFO {in: ew[i+0].in, out: we[i+0].out},
	    SerialFIFO {in: we[i+1].in, out: ew[i+1].out});
    end

  // fsm to read from host ports and generate indications

  Reg#(Bit#(4)) id <- mkReg(0);

  Stmt readindications =
    seq
    while(True) seq
      for(id <= 0; id < 4; id <= id + 1)
          if (node[id].out.notEmpty())
	      seq
		 $display("recv at %d to %d m %x",
		    id,
		    node[id].out.first.address,
	            node[id].out.first.payload);
		 indication.ack(id,
		    node[id].out.first.address,
	            node[id].out.first.payload);
		 node[id].out.deq();
              endseq
      endseq
    endseq;

    mkAutoFSM(readindications);

   method Action send(Bit#(4) sendnode, Bit#(4) to, Bit#(32) message);
      $display("send f %d t %d m %x", sendnode, to, message);
      node[sendnode].in.enq(DataMessage{address: to, payload: message});
   endmethod

endmodule

// Copyright (c) 2014 Quanta Research Cambridge, Inc.
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import SerialFIFO::*;
import NocNode::*;
import LinkHost::*;
import Connectable::*;
import StmtFSM::*;
import Vector::*;
import FIFOF::*;
import Pipe::*;

interface NocIndication;
   method Action ack(Bit#(4) recvnode, Bit#(4) to, Bit#(32) message);
endinterface

interface NocRequest;
   method Action send(Bit#(4) sendnode, Bit#(4) to, Bit#(32) message);
endinterface

module mkNocRequest#(NocIndication indication)(NocRequest);

   SerialFIFO#(DataMessage) xxx <- mkSerialFIFO();
   Vector#(5, SerialFIFO#(DataMessage)) we <- replicateM( mkSerialFIFO );
   Vector#(5, SerialFIFO#(DataMessage)) ew <- replicateM( mkSerialFIFO );

   // discard traffic from loose ends
   rule discardeast;
      $display("we[4] discard %x", we[4].out.first);
      we[4].out.deq();
      endrule

   rule discardwest;
      $display("ew[0] discard %x", ew[0].out.first);
      ew[0].out.deq();
      endrule

   Vector#(4, SerialFIFO#(DataMessage)) node;

    for (Bit#(4) i = 0; i < 4; i = i + 1)
    begin
        node[i] <- mkNocNode(unpack(i),
	    SerialFIFO {in: ew[i+0].in, out: we[i+0].out},
	    SerialFIFO {in: we[i+1].in, out: ew[i+1].out});
    end

  // fsm to read from host ports and generate indications

  Reg#(Bit#(4)) id <- mkReg(0);

  Stmt readindications =
    seq
    while(True) seq
      for(id <= 0; id < 4; id <= id + 1)
          if (node[id].out.notEmpty())
	      seq
		 indication.ack(id, node[id].out.first.address,
	            node[id].out.first.payload);
		 node[id].out.deq();
              endseq
      endseq
    endseq;

    mkAutoFSM(readindications);

   method Action send(Bit#(4) sendnode, Bit#(4) to, Bit#(32) message);
      node[sendnode].in.enq(DataMessage{address: to, payload: message});
   endmethod

endmodule


Update Paxos messagetype enum

Update Paxos messagetype enum

import ClientServer::*;

typedef 16 RoundSize;
typedef 16 MsgTypeSize;
typedef 16 DatapathSize;
typedef 32 InstanceSize;
typedef 256 ValueSize;
typedef 1024 InstanceCount;

typedef enum {
   PAXOS_1A = 0,
   PAXOS_1B = 1,
   PAXOS_2A = 2,
   PAXOS_2B = 3
} MsgType deriving (Bits, Eq);

typedef struct {
   Bit#(addrSz) addr;
   Bit#(dataSz) data;
   Bool write;
} RegRequest#(numeric type addrSz, numeric type dataSz) deriving (Bits, Eq);

typedef struct {
   Bit#(dataSz) data;
} RegResponse#(numeric type dataSz) deriving (Bits, Eq);

typedef RegRequest#(TLog#(InstanceCount), RoundSize) RoundRegRequest;
typedef RegResponse#(RoundSize) RoundRegResponse;
typedef RegRequest#(1, DatapathSize) DatapathIdRegRequest;
typedef RegResponse#(DatapathSize) DatapathIdRegResponse;
typedef RegRequest#(1, InstanceSize) InstanceRegRequest;
typedef RegResponse#(InstanceSize) InstanceRegResponse;
typedef RegRequest#(TLog#(InstanceCount), RoundSize) VRoundRegRequest;
typedef RegResponse#(RoundSize) VRoundRegResponse;
typedef RegRequest#(TLog#(InstanceCount), ValueSize) ValueRegRequest;
typedef RegResponse#(ValueSize) ValueRegResponse;

typedef enum {
   ACCEPTOR,
   COORDINATOR,
   FORWARDER
} Role deriving (Bits, Eq);
instance FShow#(Role);
   function Fmt fshow(Role role);
      case(role)
         ACCEPTOR: return fshow("ACCEPTOR");
         COORDINATOR: return fshow("COORDINATOR");
         FORWARDER: return fshow("FORWARDER");
      endcase
   endfunction
endinstance

typedef RegRequest#(1, SizeOf#(Role)) RoleRegRequest;
typedef RegResponse#(SizeOf#(Role)) RoleRegResponse;

typedef struct {
   Role role;
} RoleT deriving (Bits, Eq);

//typedef struct {
//   Bit#(1) addr;
//   Role data;
//   Bool write;
//} RoleRegRequest deriving (Bits);
//
//typedef struct {
//   Role data;
//} RoleRegResponse deriving (Bits);
//
typedef Client#(RoundRegRequest, RoundRegResponse) RoundRegClient;
typedef Server#(RoundRegRequest, RoundRegResponse) RoundRegServer;
typedef Client#(RoleRegRequest, RoleRegResponse) RoleRegClient;
typedef Server#(RoleRegRequest, RoleRegResponse) RoleRegServer;
typedef Client#(DatapathIdRegRequest, DatapathIdRegResponse) DatapathIdRegClient;
typedef Server#(DatapathIdRegRequest, DatapathIdRegResponse) DatapathIdRegServer;
typedef Client#(InstanceRegRequest, InstanceRegResponse) InstanceRegClient;
typedef Server#(InstanceRegRequest, InstanceRegResponse) InstanceRegServer;
typedef Client#(VRoundRegRequest, VRoundRegResponse) VRoundRegClient;
typedef Server#(VRoundRegRequest, VRoundRegResponse) VRoundRegServer;
typedef Client#(ValueRegRequest, ValueRegResponse) ValueRegClient;
typedef Server#(ValueRegRequest, ValueRegResponse) ValueRegServer;

/* Tables */
typedef enum {
    FORWARD,
    BROADCAST
} DmacTblActionT deriving (Bits, Eq, FShow);

typedef enum {
    Nop,
    IncreaseInstance
} SequenceTblActionT deriving (Bits, Eq, FShow);

typedef enum {
    Handle1A,
    Handle2A,
    Drop,
    Unused
} AcceptorTblActionT deriving (Bits, Eq, FShow);

//typedef struct {
//   AcceptorTblAction act;
//} AcceptorTblActionT deriving (Bits, Eq, FShow);

import ClientServer::*;

typedef 16 RoundSize;
typedef 16 MsgTypeSize;
typedef 16 DatapathSize;
typedef 32 InstanceSize;
typedef 256 ValueSize;
typedef 1024 InstanceCount;

typedef enum {
   PAXOS_1A = 1,
   PAXOS_1B = 2,
   PAXOS_2A = 3,
   PAXOS_2B = 4
} MsgType deriving (Bits, Eq);

typedef struct {
   Bit#(addrSz) addr;
   Bit#(dataSz) data;
   Bool write;
} RegRequest#(numeric type addrSz, numeric type dataSz) deriving (Bits, Eq);

typedef struct {
   Bit#(dataSz) data;
} RegResponse#(numeric type dataSz) deriving (Bits, Eq);

typedef RegRequest#(TLog#(InstanceCount), RoundSize) RoundRegRequest;
typedef RegResponse#(RoundSize) RoundRegResponse;
typedef RegRequest#(1, DatapathSize) DatapathIdRegRequest;
typedef RegResponse#(DatapathSize) DatapathIdRegResponse;
typedef RegRequest#(1, InstanceSize) InstanceRegRequest;
typedef RegResponse#(InstanceSize) InstanceRegResponse;
typedef RegRequest#(TLog#(InstanceCount), RoundSize) VRoundRegRequest;
typedef RegResponse#(RoundSize) VRoundRegResponse;
typedef RegRequest#(TLog#(InstanceCount), ValueSize) ValueRegRequest;
typedef RegResponse#(ValueSize) ValueRegResponse;

typedef enum {
   ACCEPTOR,
   COORDINATOR,
   FORWARDER
} Role deriving (Bits, Eq);
instance FShow#(Role);
   function Fmt fshow(Role role);
      case(role)
         ACCEPTOR: return fshow("ACCEPTOR");
         COORDINATOR: return fshow("COORDINATOR");
         FORWARDER: return fshow("FORWARDER");
      endcase
   endfunction
endinstance

typedef RegRequest#(1, SizeOf#(Role)) RoleRegRequest;
typedef RegResponse#(SizeOf#(Role)) RoleRegResponse;

typedef struct {
   Role role;
} RoleT deriving (Bits, Eq);

//typedef struct {
//   Bit#(1) addr;
//   Role data;
//   Bool write;
//} RoleRegRequest deriving (Bits);
//
//typedef struct {
//   Role data;
//} RoleRegResponse deriving (Bits);
//
typedef Client#(RoundRegRequest, RoundRegResponse) RoundRegClient;
typedef Server#(RoundRegRequest, RoundRegResponse) RoundRegServer;
typedef Client#(RoleRegRequest, RoleRegResponse) RoleRegClient;
typedef Server#(RoleRegRequest, RoleRegResponse) RoleRegServer;
typedef Client#(DatapathIdRegRequest, DatapathIdRegResponse) DatapathIdRegClient;
typedef Server#(DatapathIdRegRequest, DatapathIdRegResponse) DatapathIdRegServer;
typedef Client#(InstanceRegRequest, InstanceRegResponse) InstanceRegClient;
typedef Server#(InstanceRegRequest, InstanceRegResponse) InstanceRegServer;
typedef Client#(VRoundRegRequest, VRoundRegResponse) VRoundRegClient;
typedef Server#(VRoundRegRequest, VRoundRegResponse) VRoundRegServer;
typedef Client#(ValueRegRequest, ValueRegResponse) ValueRegClient;
typedef Server#(ValueRegRequest, ValueRegResponse) ValueRegServer;

/* Tables */
typedef enum {
    FORWARD,
    BROADCAST
} DmacTblActionT deriving (Bits, Eq, FShow);

typedef enum {
    Nop,
    IncreaseInstance
} SequenceTblActionT deriving (Bits, Eq, FShow);

typedef enum {
    Handle1A,
    Handle2A,
    Drop,
    Unused
} AcceptorTblActionT deriving (Bits, Eq, FShow);

//typedef struct {
//   AcceptorTblAction act;
//} AcceptorTblActionT deriving (Bits, Eq, FShow);


change decimation to 4 for testing

change decimation to 4 for testing

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import DDS::*;
import Gearbox::*;
import ChannelSelect::*;

import Complex::*;
import SDRTypes::*;
import FPCMult::*;
import FixedPoint::*;
import Pipe::*;
import Vector::*;
import ChannelSelectTestInterfaces::*;

module mkChannelSelectTestRequest#(ChannelSelectTestIndication indication) (ChannelSelectTestRequest)
   provisos(Bits#(CoeffData, a__),
	    Bits#(ProductData, b__),
	    Bits#(MulData, c__));
   Clock clk <- exposeCurrentClock;
   Reset rst <- exposeCurrentReset;
   Gearbox#(1, 2, Complex#(Signal)) gb <- mk1toNGearbox(clk, rst, clk, rst);
   DDS dds <- mkDDS();
   ChannelSelect cs <- mkChannelSelect(4, dds);

   rule processRF;
      let data = gb.first();
      gb.deq();
      cs.rfreq.enq(data);
   endrule

   rule processIF;
      let data = cs.ifreq.first();
      cs.ifreq.deq();
      indication.ifreqData(zeroExtend(pack(data.rel)), zeroExtend(pack(data.img)));
   endrule

   method Action rfreqDataWrite(Bit#(32) dataRe, Bit#(32) dataIm);
      Signal re = unpack(truncate(dataRe));
      Signal im = unpack(truncate(dataIm));
      Vector#(1, Complex#(Signal)) x = newVector();
      x[0] = Complex{rel: re, img: im};
      gb.enq(x);
      indication.setDataResp();
   endmethod

   method Action setCoeff(Bit#(11) addr, Bit#(32) valueRe, Bit#(32) valueIm);
    FixedPoint#(2, 23) re = unpack(pack(truncate(valueRe)));
    FixedPoint#(2, 23) im = unpack(pack(truncate(valueIm)));
      cs.setCoeff(addr, Complex{rel: re, img:im});
      indication.setCoeffResp();
   endmethod
endmodule

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import DDS::*;
import Gearbox::*;
import ChannelSelect::*;

import Complex::*;
import SDRTypes::*;
import FPCMult::*;
import FixedPoint::*;
import Pipe::*;
import Vector::*;
import ChannelSelectTestInterfaces::*;

module mkChannelSelectTestRequest#(ChannelSelectTestIndication indication) (ChannelSelectTestRequest)
   provisos(Bits#(CoeffData, a__),
	    Bits#(ProductData, b__),
	    Bits#(MulData, c__));
   Clock clk <- exposeCurrentClock;
   Reset rst <- exposeCurrentReset;
   Gearbox#(1, 2, Complex#(Signal)) gb <- mk1toNGearbox(clk, rst, clk, rst);
   DDS dds <- mkDDS();
   ChannelSelect cs <- mkChannelSelect(64, dds);

   rule processRF;
      let data = gb.first();
      gb.deq();
      cs.rfreq.enq(data);
   endrule

   rule processIF;
      let data = cs.ifreq.first();
      cs.ifreq.deq();
      indication.ifreqData(zeroExtend(pack(data.rel)), zeroExtend(pack(data.img)));
   endrule

   method Action rfreqDataWrite(Bit#(32) dataRe, Bit#(32) dataIm);
      Signal re = unpack(truncate(dataRe));
      Signal im = unpack(truncate(dataIm));
      Vector#(1, Complex#(Signal)) x = newVector();
      x[0] = Complex{rel: re, img: im};
      gb.enq(x);
      indication.setDataResp();
   endmethod

   method Action setCoeff(Bit#(11) addr, Bit#(32) valueRe, Bit#(32) valueIm);
    FixedPoint#(2, 23) re = unpack(pack(truncate(valueRe)));
    FixedPoint#(2, 23) im = unpack(pack(truncate(valueIm)));
      cs.setCoeff(addr, Complex{rel: re, img:im});
      indication.setCoeffResp();
   endmethod
endmodule


checking for blocked tlps seemed to break things

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import FIFO         :: *;
import FIFOF        :: *;
import GetPut       :: *;
import Connectable  :: *;
import PCIE         :: *;
import DefaultValue :: *;
import MIMO         :: *;
import MIFO         :: *;
import Vector       :: *;
import ClientServer :: *;
import MemTypes     :: *;
import Probe        :: *;
import PcieTracer   :: *;

interface MemSlaveEngine#(numeric type buswidth);
    interface Client#(TLPData#(16), TLPData#(16)) tlp;
    interface PhysMemSlave#(40,buswidth) slave;
    interface Get#(TimestampedTlpData) trace;
    method Bool tlpOutFifoNotEmpty();
    interface Reg#(Bool) use4dw;
endinterface: MemSlaveEngine

module mkMemSlaveEngine#(PciId my_id)(MemSlaveEngine#(buswidth))
   provisos (Div#(buswidth, 8, busWidthBytes),
	     Div#(buswidth, 32, busWidthWords),
	     Bits#(Vector#(busWidthWords, Bit#(32)), buswidth),
            Log#(busWidthBytes,beatShift),
	     Add#(aaa, 32, buswidth),
	     Add#(bbb, buswidth, 256),
	     Add#(bbbb, buswidth, 1024),
	     Add#(ccc, TMul#(8, busWidthWords), 256),
	     Add#(ddd, TMul#(32, busWidthWords), 256),
	     Add#(eee, busWidthWords, 8),
      	     Add#(fff, busWidthWords, 32),
	     Add#(1, a__, busWidthWords),
	     Add#(busWidthWords, b__, 4),
	     Add#(c__, busWidthWords, 16)
      );

    let beat_shift = fromInteger(valueOf(beatShift));

    FIFOF#(TLPData#(16)) tlpOutFifo <- mkFIFOF;
    FIFOF#(TLPData#(16)) tlpInFifo <- mkFIFOF;
    FIFO#(TLPData#(16)) tlpWriteHeaderFifo <- mkFIFO;
    FIFO#(TLPLength)      writeDwCountFifo <- mkFIFO;
    FIFO#(Bool)             writeIs3dwFifo <- mkFIFO;
    FIFO#(Bool)             writeIsHeaderOnlyFifo <- mkFIFO;

    Reg#(Bit#(7)) hitReg <- mkReg(0);
    Reg#(Bool) use4dwReg <- mkReg(True);

    // default configuration for MIMO is for guarded enq() and deq().
    // However, the implicit guard only checks for space for 1 element for enq(), and availability of 1 element for deq().
    MIMOConfiguration mimoCfg = defaultValue;
    MIMO#(4,busWidthWords,32,Bit#(32)) completionMimo <- mkMIMO(mimoCfg);
    MIMO#(4,busWidthWords,32,TLPTag) completionTagMimo <- mkMIMO(mimoCfg);

    MIMO#(busWidthWords,4,32,Bit#(32)) writeDataMimo <- mkMIMO(mimoCfg);
    Reg#(Bit#(10)) writeBurstCount <- mkReg(0);
   FIFO#(Bit#(10)) writeBurstCountFifo <- mkFIFO();
    Reg#(TLPLength)  writeDwCount <- mkReg(0); // how many 4 byte (double) words to send
    Reg#(LUInt#(4))    tlpDwCount <- mkReg(0); // how many to send in the next tlp (at most 4)
    Reg#(Bool)            lastTlp <- mkReg(False); // if the next tlp sent is the last one
    Reg#(Bool)    writeInProgress <- mkReg(False);
    FIFOF#(TLPTag) writeTag <- mkSizedFIFOF(16);
    FIFOF#(TLPTag) doneTag <- mkSizedFIFOF(16);

   Wire#(Bool) writeHeaderTlpWire <- mkDWire(False);
   Probe#(Bool) writeHeaderAvailableProbe <- mkProbe;
   Wire#(Bool) writeDataMimoEnqWire <- mkDWire(False);
   Probe#(Bool) writeDataMimoEnqProbe <- mkProbe;
   let    writeDataMimoCountProbe <- mkProbe;
   let writeDataMimoEnqReadyProbe <- mkProbe;
   Probe#(Bool) writeInProgressProbe <- mkProbe;
   Probe#(Bit#(10)) writeBurstCountProbe <- mkProbe;
   rule updateProbe0;
      writeDataMimoCountProbe    <= writeDataMimo.count();
      writeDataMimoEnqReadyProbe <= writeDataMimo.enqReadyN(fromInteger(valueOf(busWidthWords)));
   endrule
   rule updateProbe2;
      writeHeaderAvailableProbe <= writeHeaderTlpWire;
      writeDataMimoEnqProbe <= writeDataMimoEnqWire;
   endrule

   rule writeHeaderTlp if (!writeInProgress && (!writeIs3dwFifo.first || writeDataMimo.deqReadyN(1)));
      writeHeaderTlpWire <= True;
      let tlp <- toGet(tlpWriteHeaderFifo).get();
      let dwCount <- toGet(writeDwCountFifo).get();
      let is3dw <- toGet(writeIs3dwFifo).get();
      let isHeaderOnly <- toGet(writeIsHeaderOnlyFifo).get();

      TLPMemory4DWHeader hdr_4dw = unpack(tlp.data);

      TLPMemoryIO3DWHeader hdr_3dw = unpack(tlp.data);
      if (is3dw) begin
	 if (hdr_3dw.format != MEM_WRITE_3DW_DATA)
	    $display("MemSlaveEngine: expecting MEM_WRITE_3DW_DATA, got %d", hdr_3dw.format);
	 Vector#(4, Bit#(32)) v = writeDataMimo.first();
	 writeDataMimo.deq(1);
	 hdr_3dw.data = byteSwap(v[0]);
	 tlp.be = 16'hffff;
	 if (dwCount == 1)
	    tlp.eof = True;
	 dwCount = dwCount - 1;
	 tlp.data = pack(hdr_3dw);
      end
      else begin
	 tlp.be = 16'hffff;
      end

      tlpOutFifo.enq(tlp);
      //$display("writeHeaderTlp dwCount=%d", dwCount);
      writeDwCount <= dwCount;
      tlpDwCount <= truncate(min(4,unpack(dwCount)));
      lastTlp <= (dwCount <= 4);
      writeInProgress <= (dwCount != 0);
      writeInProgressProbe <= (dwCount != 0);
      if (isHeaderOnly) begin
	 doneTag.enq(writeTag.first());
	 writeTag.deq();
      end

   endrule

   rule writeTlps if (writeInProgress && writeDataMimo.deqReadyN(tlpDwCount));
      TLPData#(16) tlp = defaultValue;
      tlp.sof = False;
      Vector#(4, Bit#(32)) v = unpack(0);

      // The MIMO implicit guard only checks for availability of 1 element
      // so we explicitly check for the number of elements required
      writeDataMimo.deq(tlpDwCount);
      v = writeDataMimo.first();
      let dwCount = writeDwCount - extend(pack(tlpDwCount));
      writeDwCount <= dwCount;
      tlpDwCount <= truncate(min(4,unpack(dwCount)));
      lastTlp <= (dwCount <= 4);
      if (tlpDwCount == 4)
	 tlp.be = 16'hffff;
      else if (tlpDwCount == 3)
	 tlp.be = 16'hfff0;
      else if (tlpDwCount == 2)
	 tlp.be = 16'hff00;
      else if (tlpDwCount == 1)
	 tlp.be = 16'hf000;
      tlp.eof = lastTlp;
      if (lastTlp) begin
	 writeInProgress <= False;
	 writeInProgressProbe <= False;
	 doneTag.enq(writeTag.first());
	 writeTag.deq();
	 //$display("writeDwCount=%d will be zero", writeDwCount);
      end

      for (Integer i = 0; i < 4; i = i + 1)
	 tlp.data[(i+1)*32-1:i*32] = byteSwap(v[3-i]);
      tlpOutFifo.enq(tlp);
   endrule

   Reg#(TLPTag) lastTag <- mkReg(0);
   FIFOF#(TLPData#(16)) tlpDecodeFifo <- mkFIFOF();
   Reg#(TLPLength) wordCountReg <- mkReg(0);
   FIFOF#(TimestampedTlpData) tlpTraceFifo <- mkSizedFIFOF(8);
   Reg#(Bit#(6)) tlpAgeReg <- mkReg(0);
   Reg#(Bool) tlpAgeStartReg <- mkReg(False);
   Reg#(Bit#(32)) cycles <- mkReg(0);
   rule count;
      cycles <= cycles + 1;
   endrule

   rule tlpInRule;
      let tlp <- toGet(tlpInFifo).get();
      tlpDecodeFifo.enq(tlp);
   endrule

   rule handleTlpRule;
      let tlp = tlpDecodeFifo.first;
      let tlpAge = tlpAgeReg + 1;
      if (tlpAgeStartReg) // first cycle for this TLP
	 tlpAge = 1;
      Bool tlpAgeStart = False;
      Bool handled = False;
      TLPMemoryIO3DWHeader h = unpack(tlp.data);
      hitReg <= tlp.hit;
      TLPMemoryIO3DWHeader hdr_3dw = unpack(tlp.data);
      TLPCompletionHeader hdr_completion = unpack(tlp.data);
      Vector#(4, Bit#(32)) vec = unpack(0);
      Vector#(4, Bit#(32)) tlpvec = unpack(tlp.data);
      let wordCount = wordCountReg;
      if (!tlp.sof) begin
	 vec = reverse(tlpvec);
	 // The MIMO implicit guard only checks for space to enqueue 1 element
	 // so we explicitly check for the number of elements required
	 // otherwise elements in the queue will be overwritten.
	 UInt#(3) count = 4;
	 if (tlp.eof) begin
	    count = truncate(unpack(wordCountReg));
	 end
	 if (completionMimo.enqReadyN(count)
	    && completionTagMimo.enqReadyN(count))
	    begin
	       wordCount = wordCountReg - extend(pack(count));
	       completionMimo.enq(count, vec);
	       Vector#(4, TLPTag) tagvec = replicate(lastTag);
	       completionTagMimo.enq(count, tagvec);
	       handled = True;
	    end
      end
      else if (hdr_3dw.format == MEM_WRITE_3DW_DATA
	       && hdr_3dw.pkttype == COMPLETION
	       && completionMimo.enqReady()
	       && completionTagMimo.enqReady()) begin
	    vec[0] = hdr_3dw.data;
            wordCount = hdr_3dw.length - 1;
	    completionMimo.enq(1, vec);
            TLPTag tag = hdr_completion.tag;
	    lastTag <= tag;
	    completionTagMimo.enq(1, replicate(tag));
	    handled = True;
      end
      wordCountReg <= wordCount;
      //$display("tlpIn handled=%d tlp=%h\n", handled, tlp);
      if (False && tlpAge == 15 && !handled) begin
	 let ttd = TimestampedTlpData { timestamp: cycles, source: 38, tlp: tlp };
	 if (tlpTraceFifo.notFull())
	    tlpTraceFifo.enq(ttd);
	 tlpAge = 0;
	 //handled = True;
      end
      if (handled) begin
	 tlpDecodeFifo.deq();
	 tlpAge = 0;
	 tlpAgeStart = True;
      end
      tlpAgeReg <= tlpAge;
      tlpAgeStartReg <= tlpAgeStart;
   endrule

   FIFO#(PhysMemRequest#(40)) readReqFifo <- mkFIFO();
   rule readReqRule if (!writeInProgress && !writeDataMimo.deqReady());
      let req <- toGet(readReqFifo).get();
      let burstLen = req.burstLen >> beat_shift;
      let addr = req.addr;
      let arid = req.tag;

      TLPData#(16) tlp = defaultValue;
      tlp.sof = True;
      tlp.eof = True;
      tlp.hit = 7'h00;
      TLPLength tlplen = fromInteger(valueOf(busWidthWords))*extend(burstLen);
      if (addr[39:32] != 0) begin
	 TLPMemory4DWHeader hdr_4dw = defaultValue;
	 hdr_4dw.format = MEM_READ_4DW_NO_DATA;
	 hdr_4dw.tag = extend(arid);
	 hdr_4dw.reqid = my_id;
	 hdr_4dw.nosnoop = SNOOPING_REQD;
	 hdr_4dw.addr = addr[40-1:2];
	 hdr_4dw.length = tlplen;
	 hdr_4dw.firstbe = 4'hf;
	 hdr_4dw.lastbe = (tlplen > 1) ? 4'hf : 0;
	 tlp.data = pack(hdr_4dw);
	 tlp.be = 16'hffff;
      end
      else begin
	 TLPMemoryIO3DWHeader hdr_3dw = defaultValue;
	 hdr_3dw.format = MEM_READ_3DW_NO_DATA;
	 hdr_3dw.tag = extend(arid);
	 hdr_3dw.reqid = my_id;
	 hdr_3dw.nosnoop = SNOOPING_REQD;
	 hdr_3dw.addr = addr[32-1:2];
	 hdr_3dw.length = tlplen;
	 hdr_3dw.firstbe = 4'hf;
	 hdr_3dw.lastbe = (tlplen > 1) ? 4'hf : 0;
	 tlp.data = pack(hdr_3dw);
	 tlp.be = 16'hfff0;
      end
      tlpOutFifo.enq(tlp);
   endrule

    interface Client        tlp;
        interface request = toGet(tlpOutFifo);
        interface response = toPut(tlpInFifo);
    endinterface
    interface PhysMemSlave slave;
   interface PhysMemWriteServer write_server;
      interface Put writeReq;
         method Action put(PhysMemRequest#(40) req); // if (writeBurstCount == 0);
	    let burstLen = req.burstLen >> beat_shift;
	    let addr = req.addr;
	    let awid = req.tag;
	    let writeIs3dw = False;

	    TLPLength tlplen = fromInteger(valueOf(busWidthWords))*extend(burstLen);
	    TLPData#(16) tlp = defaultValue;
	    tlp.sof = True;
	    tlp.eof = False;
	    tlp.hit = 7'h00;
	    tlp.be = 16'hffff;

	    $display("slave.writeAddr tlplen=%d burstLen=%d", tlplen, burstLen);
	    if ((addr >> 32) != 0) begin
	       TLPMemory4DWHeader hdr_4dw = defaultValue;
	       hdr_4dw.format = MEM_WRITE_4DW_DATA;
	       hdr_4dw.tag = extend(awid);
	       hdr_4dw.reqid = my_id;
	       hdr_4dw.nosnoop = SNOOPING_REQD;
	       hdr_4dw.addr = addr[40-1:2];
	       hdr_4dw.length = tlplen;
	       hdr_4dw.firstbe = 4'hf;
	       hdr_4dw.lastbe = (tlplen > 1) ? 4'hf : 0;
	       tlp.data = pack(hdr_4dw);
	    end
	    else begin
	       writeIs3dw = True;
	       TLPMemoryIO3DWHeader hdr_3dw = defaultValue;
	       hdr_3dw.format = MEM_WRITE_3DW_DATA;
	       hdr_3dw.tag = extend(awid);
	       hdr_3dw.reqid = my_id;
	       hdr_3dw.nosnoop = SNOOPING_REQD;
	       hdr_3dw.addr = addr[32-1:2];
	       hdr_3dw.length = tlplen;
	       hdr_3dw.firstbe = 4'hf;
	       hdr_3dw.lastbe = (tlplen > 1) ? 4'hf : 0;

	       tlp.be = 16'hfff0; // no data word in this TLP

	       tlp.data = pack(hdr_3dw);
	    end
	    tlpWriteHeaderFifo.enq(tlp);
	    writeDwCountFifo.enq(tlplen);
	    writeIs3dwFifo.enq(writeIs3dw);
	    writeIsHeaderOnlyFifo.enq(writeIs3dw && tlplen == 1);
	    writeBurstCountFifo.enq(extend(burstLen));
	    writeTag.enq(extend(awid));
         endmethod
	endinterface
      interface Put writeData;
         method Action put(MemData#(buswidth) wdata)
	      provisos (Bits#(Vector#(busWidthWords, Bit#(32)), busWidth)) if (writeDataMimo.enqReadyN(fromInteger(valueOf(busWidthWords))));

	    let burstLen = writeBurstCount;
	    if (burstLen == 0) begin
	       burstLen <- toGet(writeBurstCountFifo).get();
	    end
	    writeBurstCount <= extend(burstLen-1);
	    writeBurstCountProbe <= extend(burstLen-1);

	    Vector#(busWidthWords, Bit#(32)) v = unpack(wdata.data);
	    writeDataMimo.enq(fromInteger(valueOf(busWidthWords)), v);
	    writeDataMimoEnqWire <= True;
         endmethod
       endinterface
      interface Get writeDone;
         method ActionValue#(Bit#(MemTagSize)) get();
	      let tag = doneTag.first();
	      doneTag.deq();
	      return truncate(tag);
           endmethod
	endinterface
   endinterface
   interface PhysMemReadServer read_server;
      interface Put readReq;
         method Action put(PhysMemRequest#(40) req);
	    readReqFifo.enq(req);
         endmethod
       endinterface
      interface Get     readData;
         method ActionValue#(MemData#(buswidth)) get() if (completionMimo.deqReady()
							   && completionTagMimo.deqReady());
	      let data_v = completionMimo.first;
	      let tag_v = completionTagMimo.first;
	      completionMimo.deq(fromInteger(valueOf(busWidthWords)));
	      completionTagMimo.deq(fromInteger(valueOf(busWidthWords)));
              Bit#(buswidth) v = 0;
	      for (Integer i = 0; i < valueOf(busWidthWords); i = i+1)
		 v[(i+1)*32-1:i*32] = byteSwap(data_v[i]);
	      return MemData { data: v, tag: truncate(tag_v[0]), last: True};
           endmethod
	endinterface
   endinterface
    endinterface: slave
   method Bool tlpOutFifoNotEmpty() = tlpOutFifo.notEmpty;
   interface Reg use4dw = use4dwReg;
   interface Get trace = toGet(tlpTraceFifo);
endmodule: mkMemSlaveEngine

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import FIFO         :: *;
import FIFOF        :: *;
import GetPut       :: *;
import Connectable  :: *;
import PCIE         :: *;
import DefaultValue :: *;
import MIMO         :: *;
import MIFO         :: *;
import Vector       :: *;
import ClientServer :: *;
import MemTypes     :: *;
import Probe        :: *;
import PcieTracer   :: *;

interface MemSlaveEngine#(numeric type buswidth);
    interface Client#(TLPData#(16), TLPData#(16)) tlp;
    interface PhysMemSlave#(40,buswidth) slave;
    interface Get#(TimestampedTlpData) trace;
    method Bool tlpOutFifoNotEmpty();
    interface Reg#(Bool) use4dw;
endinterface: MemSlaveEngine

module mkMemSlaveEngine#(PciId my_id)(MemSlaveEngine#(buswidth))
   provisos (Div#(buswidth, 8, busWidthBytes),
	     Div#(buswidth, 32, busWidthWords),
	     Bits#(Vector#(busWidthWords, Bit#(32)), buswidth),
            Log#(busWidthBytes,beatShift),
	     Add#(aaa, 32, buswidth),
	     Add#(bbb, buswidth, 256),
	     Add#(bbbb, buswidth, 1024),
	     Add#(ccc, TMul#(8, busWidthWords), 256),
	     Add#(ddd, TMul#(32, busWidthWords), 256),
	     Add#(eee, busWidthWords, 8),
      	     Add#(fff, busWidthWords, 32),
	     Add#(1, a__, busWidthWords),
	     Add#(busWidthWords, b__, 4),
	     Add#(c__, busWidthWords, 16)
      );

    let beat_shift = fromInteger(valueOf(beatShift));

    FIFOF#(TLPData#(16)) tlpOutFifo <- mkFIFOF;
    FIFOF#(TLPData#(16)) tlpInFifo <- mkFIFOF;
    FIFO#(TLPData#(16)) tlpWriteHeaderFifo <- mkFIFO;
    FIFO#(TLPLength)      writeDwCountFifo <- mkFIFO;
    FIFO#(Bool)             writeIs3dwFifo <- mkFIFO;
    FIFO#(Bool)             writeIsHeaderOnlyFifo <- mkFIFO;

    Reg#(Bit#(7)) hitReg <- mkReg(0);
    Reg#(Bool) use4dwReg <- mkReg(True);

    // default configuration for MIMO is for guarded enq() and deq().
    // However, the implicit guard only checks for space for 1 element for enq(), and availability of 1 element for deq().
    MIMOConfiguration mimoCfg = defaultValue;
    MIMO#(4,busWidthWords,32,Bit#(32)) completionMimo <- mkMIMO(mimoCfg);
    MIMO#(4,busWidthWords,32,TLPTag) completionTagMimo <- mkMIMO(mimoCfg);

    MIMO#(busWidthWords,4,32,Bit#(32)) writeDataMimo <- mkMIMO(mimoCfg);
    Reg#(Bit#(10)) writeBurstCount <- mkReg(0);
   FIFO#(Bit#(10)) writeBurstCountFifo <- mkFIFO();
    Reg#(TLPLength)  writeDwCount <- mkReg(0); // how many 4 byte (double) words to send
    Reg#(LUInt#(4))    tlpDwCount <- mkReg(0); // how many to send in the next tlp (at most 4)
    Reg#(Bool)            lastTlp <- mkReg(False); // if the next tlp sent is the last one
    Reg#(Bool)    writeInProgress <- mkReg(False);
    FIFOF#(TLPTag) writeTag <- mkSizedFIFOF(16);
    FIFOF#(TLPTag) doneTag <- mkSizedFIFOF(16);

   Wire#(Bool) writeHeaderTlpWire <- mkDWire(False);
   Probe#(Bool) writeHeaderAvailableProbe <- mkProbe;
   Wire#(Bool) writeDataMimoEnqWire <- mkDWire(False);
   Probe#(Bool) writeDataMimoEnqProbe <- mkProbe;
   let    writeDataMimoCountProbe <- mkProbe;
   let writeDataMimoEnqReadyProbe <- mkProbe;
   Probe#(Bool) writeInProgressProbe <- mkProbe;
   Probe#(Bit#(10)) writeBurstCountProbe <- mkProbe;
   rule updateProbe0;
      writeDataMimoCountProbe    <= writeDataMimo.count();
      writeDataMimoEnqReadyProbe <= writeDataMimo.enqReadyN(fromInteger(valueOf(busWidthWords)));
   endrule
   rule updateProbe2;
      writeHeaderAvailableProbe <= writeHeaderTlpWire;
      writeDataMimoEnqProbe <= writeDataMimoEnqWire;
   endrule

   rule writeHeaderTlp if (!writeInProgress && (!writeIs3dwFifo.first || writeDataMimo.deqReadyN(1)));
      writeHeaderTlpWire <= True;
      let tlp <- toGet(tlpWriteHeaderFifo).get();
      let dwCount <- toGet(writeDwCountFifo).get();
      let is3dw <- toGet(writeIs3dwFifo).get();
      let isHeaderOnly <- toGet(writeIsHeaderOnlyFifo).get();

      TLPMemory4DWHeader hdr_4dw = unpack(tlp.data);

      TLPMemoryIO3DWHeader hdr_3dw = unpack(tlp.data);
      if (is3dw) begin
	 if (hdr_3dw.format != MEM_WRITE_3DW_DATA)
	    $display("MemSlaveEngine: expecting MEM_WRITE_3DW_DATA, got %d", hdr_3dw.format);
	 Vector#(4, Bit#(32)) v = writeDataMimo.first();
	 writeDataMimo.deq(1);
	 hdr_3dw.data = byteSwap(v[0]);
	 tlp.be = 16'hffff;
	 if (dwCount == 1)
	    tlp.eof = True;
	 dwCount = dwCount - 1;
	 tlp.data = pack(hdr_3dw);
      end
      else begin
	 tlp.be = 16'hffff;
      end

      tlpOutFifo.enq(tlp);
      //$display("writeHeaderTlp dwCount=%d", dwCount);
      writeDwCount <= dwCount;
      tlpDwCount <= truncate(min(4,unpack(dwCount)));
      lastTlp <= (dwCount <= 4);
      writeInProgress <= (dwCount != 0);
      writeInProgressProbe <= (dwCount != 0);
      if (isHeaderOnly) begin
	 doneTag.enq(writeTag.first());
	 writeTag.deq();
      end

   endrule

   rule writeTlps if (writeInProgress && writeDataMimo.deqReadyN(tlpDwCount));
      TLPData#(16) tlp = defaultValue;
      tlp.sof = False;
      Vector#(4, Bit#(32)) v = unpack(0);

      // The MIMO implicit guard only checks for availability of 1 element
      // so we explicitly check for the number of elements required
      writeDataMimo.deq(tlpDwCount);
      v = writeDataMimo.first();
      let dwCount = writeDwCount - extend(pack(tlpDwCount));
      writeDwCount <= dwCount;
      tlpDwCount <= truncate(min(4,unpack(dwCount)));
      lastTlp <= (dwCount <= 4);
      if (tlpDwCount == 4)
	 tlp.be = 16'hffff;
      else if (tlpDwCount == 3)
	 tlp.be = 16'hfff0;
      else if (tlpDwCount == 2)
	 tlp.be = 16'hff00;
      else if (tlpDwCount == 1)
	 tlp.be = 16'hf000;
      tlp.eof = lastTlp;
      if (lastTlp) begin
	 writeInProgress <= False;
	 writeInProgressProbe <= False;
	 doneTag.enq(writeTag.first());
	 writeTag.deq();
	 //$display("writeDwCount=%d will be zero", writeDwCount);
      end

      for (Integer i = 0; i < 4; i = i + 1)
	 tlp.data[(i+1)*32-1:i*32] = byteSwap(v[3-i]);
      tlpOutFifo.enq(tlp);
   endrule

   Reg#(TLPTag) lastTag <- mkReg(0);
   FIFOF#(TLPData#(16)) tlpDecodeFifo <- mkFIFOF();
   Reg#(TLPLength) wordCountReg <- mkReg(0);
   FIFOF#(TimestampedTlpData) tlpTraceFifo <- mkSizedFIFOF(8);
   Reg#(Bit#(6)) tlpAgeReg <- mkReg(0);
   Reg#(Bool) tlpAgeStartReg <- mkReg(False);
   Reg#(Bit#(32)) cycles <- mkReg(0);
   rule count;
      cycles <= cycles + 1;
   endrule

   rule tlpInRule;
      let tlp <- toGet(tlpInFifo).get();
      tlpDecodeFifo.enq(tlp);
   endrule

   rule handleTlpRule;
      let tlp = tlpDecodeFifo.first;
      let tlpAge = tlpAgeReg + 1;
      if (tlpAgeStartReg) // first cycle for this TLP
	 tlpAge = 1;
      Bool tlpAgeStart = False;
      Bool handled = False;
      TLPMemoryIO3DWHeader h = unpack(tlp.data);
      hitReg <= tlp.hit;
      TLPMemoryIO3DWHeader hdr_3dw = unpack(tlp.data);
      TLPCompletionHeader hdr_completion = unpack(tlp.data);
      Vector#(4, Bit#(32)) vec = unpack(0);
      Vector#(4, Bit#(32)) tlpvec = unpack(tlp.data);
      let wordCount = wordCountReg;
      if (!tlp.sof) begin
	 vec = reverse(tlpvec);
	 // The MIMO implicit guard only checks for space to enqueue 1 element
	 // so we explicitly check for the number of elements required
	 // otherwise elements in the queue will be overwritten.
	 UInt#(3) count = 4;
	 if (tlp.eof) begin
	    count = truncate(unpack(wordCountReg));
	 end
	 if (completionMimo.enqReadyN(count)
	    && completionTagMimo.enqReadyN(count))
	    begin
	       wordCount = wordCountReg - extend(pack(count));
	       completionMimo.enq(count, vec);
	       Vector#(4, TLPTag) tagvec = replicate(lastTag);
	       completionTagMimo.enq(count, tagvec);
	       handled = True;
	    end
      end
      else if (hdr_3dw.format == MEM_WRITE_3DW_DATA
	       && hdr_3dw.pkttype == COMPLETION
	       && completionMimo.enqReady()
	       && completionTagMimo.enqReady()) begin
	    vec[0] = hdr_3dw.data;
            wordCount = hdr_3dw.length - 1;
	    completionMimo.enq(1, vec);
            TLPTag tag = hdr_completion.tag;
	    lastTag <= tag;
	    completionTagMimo.enq(1, replicate(tag));
	    handled = True;
      end
      wordCountReg <= wordCount;
      //$display("tlpIn handled=%d tlp=%h\n", handled, tlp);
      if (tlpAge == 15 && !handled) begin
	 let ttd = TimestampedTlpData { timestamp: cycles, source: 38, tlp: tlp };
	 if (tlpTraceFifo.notFull())
	    tlpTraceFifo.enq(ttd);
	 tlpAge = 0;
	 //handled = True;
      end
      if (handled) begin
	 tlpDecodeFifo.deq();
	 tlpAge = 0;
	 tlpAgeStart = True;
      end
      tlpAgeReg <= tlpAge;
      tlpAgeStartReg <= tlpAgeStart;
   endrule

   FIFO#(PhysMemRequest#(40)) readReqFifo <- mkFIFO();
   rule readReqRule if (!writeInProgress && !writeDataMimo.deqReady());
      let req <- toGet(readReqFifo).get();
      let burstLen = req.burstLen >> beat_shift;
      let addr = req.addr;
      let arid = req.tag;

      TLPData#(16) tlp = defaultValue;
      tlp.sof = True;
      tlp.eof = True;
      tlp.hit = 7'h00;
      TLPLength tlplen = fromInteger(valueOf(busWidthWords))*extend(burstLen);
      if (addr[39:32] != 0) begin
	 TLPMemory4DWHeader hdr_4dw = defaultValue;
	 hdr_4dw.format = MEM_READ_4DW_NO_DATA;
	 hdr_4dw.tag = extend(arid);
	 hdr_4dw.reqid = my_id;
	 hdr_4dw.nosnoop = SNOOPING_REQD;
	 hdr_4dw.addr = addr[40-1:2];
	 hdr_4dw.length = tlplen;
	 hdr_4dw.firstbe = 4'hf;
	 hdr_4dw.lastbe = (tlplen > 1) ? 4'hf : 0;
	 tlp.data = pack(hdr_4dw);
	 tlp.be = 16'hffff;
      end
      else begin
	 TLPMemoryIO3DWHeader hdr_3dw = defaultValue;
	 hdr_3dw.format = MEM_READ_3DW_NO_DATA;
	 hdr_3dw.tag = extend(arid);
	 hdr_3dw.reqid = my_id;
	 hdr_3dw.nosnoop = SNOOPING_REQD;
	 hdr_3dw.addr = addr[32-1:2];
	 hdr_3dw.length = tlplen;
	 hdr_3dw.firstbe = 4'hf;
	 hdr_3dw.lastbe = (tlplen > 1) ? 4'hf : 0;
	 tlp.data = pack(hdr_3dw);
	 tlp.be = 16'hfff0;
      end
      tlpOutFifo.enq(tlp);
   endrule

    interface Client        tlp;
        interface request = toGet(tlpOutFifo);
        interface response = toPut(tlpInFifo);
    endinterface
    interface PhysMemSlave slave;
   interface PhysMemWriteServer write_server;
      interface Put writeReq;
         method Action put(PhysMemRequest#(40) req); // if (writeBurstCount == 0);
	    let burstLen = req.burstLen >> beat_shift;
	    let addr = req.addr;
	    let awid = req.tag;
	    let writeIs3dw = False;

	    TLPLength tlplen = fromInteger(valueOf(busWidthWords))*extend(burstLen);
	    TLPData#(16) tlp = defaultValue;
	    tlp.sof = True;
	    tlp.eof = False;
	    tlp.hit = 7'h00;
	    tlp.be = 16'hffff;

	    $display("slave.writeAddr tlplen=%d burstLen=%d", tlplen, burstLen);
	    if ((addr >> 32) != 0) begin
	       TLPMemory4DWHeader hdr_4dw = defaultValue;
	       hdr_4dw.format = MEM_WRITE_4DW_DATA;
	       hdr_4dw.tag = extend(awid);
	       hdr_4dw.reqid = my_id;
	       hdr_4dw.nosnoop = SNOOPING_REQD;
	       hdr_4dw.addr = addr[40-1:2];
	       hdr_4dw.length = tlplen;
	       hdr_4dw.firstbe = 4'hf;
	       hdr_4dw.lastbe = (tlplen > 1) ? 4'hf : 0;
	       tlp.data = pack(hdr_4dw);
	    end
	    else begin
	       writeIs3dw = True;
	       TLPMemoryIO3DWHeader hdr_3dw = defaultValue;
	       hdr_3dw.format = MEM_WRITE_3DW_DATA;
	       hdr_3dw.tag = extend(awid);
	       hdr_3dw.reqid = my_id;
	       hdr_3dw.nosnoop = SNOOPING_REQD;
	       hdr_3dw.addr = addr[32-1:2];
	       hdr_3dw.length = tlplen;
	       hdr_3dw.firstbe = 4'hf;
	       hdr_3dw.lastbe = (tlplen > 1) ? 4'hf : 0;

	       tlp.be = 16'hfff0; // no data word in this TLP

	       tlp.data = pack(hdr_3dw);
	    end
	    tlpWriteHeaderFifo.enq(tlp);
	    writeDwCountFifo.enq(tlplen);
	    writeIs3dwFifo.enq(writeIs3dw);
	    writeIsHeaderOnlyFifo.enq(writeIs3dw && tlplen == 1);
	    writeBurstCountFifo.enq(extend(burstLen));
	    writeTag.enq(extend(awid));
         endmethod
	endinterface
      interface Put writeData;
         method Action put(MemData#(buswidth) wdata)
	      provisos (Bits#(Vector#(busWidthWords, Bit#(32)), busWidth)) if (writeDataMimo.enqReadyN(fromInteger(valueOf(busWidthWords))));

	    let burstLen = writeBurstCount;
	    if (burstLen == 0) begin
	       burstLen <- toGet(writeBurstCountFifo).get();
	    end
	    writeBurstCount <= extend(burstLen-1);
	    writeBurstCountProbe <= extend(burstLen-1);

	    Vector#(busWidthWords, Bit#(32)) v = unpack(wdata.data);
	    writeDataMimo.enq(fromInteger(valueOf(busWidthWords)), v);
	    writeDataMimoEnqWire <= True;
         endmethod
       endinterface
      interface Get writeDone;
         method ActionValue#(Bit#(MemTagSize)) get();
	      let tag = doneTag.first();
	      doneTag.deq();
	      return truncate(tag);
           endmethod
	endinterface
   endinterface
   interface PhysMemReadServer read_server;
      interface Put readReq;
         method Action put(PhysMemRequest#(40) req);
	    readReqFifo.enq(req);
         endmethod
       endinterface
      interface Get     readData;
         method ActionValue#(MemData#(buswidth)) get() if (completionMimo.deqReady()
							   && completionTagMimo.deqReady());
	      let data_v = completionMimo.first;
	      let tag_v = completionTagMimo.first;
	      completionMimo.deq(fromInteger(valueOf(busWidthWords)));
	      completionTagMimo.deq(fromInteger(valueOf(busWidthWords)));
              Bit#(buswidth) v = 0;
	      for (Integer i = 0; i < valueOf(busWidthWords); i = i+1)
		 v[(i+1)*32-1:i*32] = byteSwap(data_v[i]);
	      return MemData { data: v, tag: truncate(tag_v[0]), last: True};
           endmethod
	endinterface
   endinterface
    endinterface: slave
   method Bool tlpOutFifoNotEmpty() = tlpOutFifo.notEmpty;
   interface Reg use4dw = use4dwReg;
   interface Get trace = toGet(tlpTraceFifo);
endmodule: mkMemSlaveEngine


improve throughput in mkMemWriteEngine

improve throughput in mkMemWriteEngine

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
import Vector::*;
import BuildVector::*;
import Cntrs::*;
import FIFOF::*;
import FIFO::*;
import GetPut::*;
import Connectable::*;
import BRAMFIFO::*;
import ConfigCounter::*;
import MemTypes::*;
import Pipe::*;
import ConnectalConfig::*;

`include "ConnectalProjectConfig.bsv"

module mkMemWriteEngine(MemWriteEngine#(busWidth, userWidth, cmdQDepth, numServers))
   provisos( Add#(1, d__, busWidth)
	    ,Add#(1, d__, userWidth)
	    ,Add#(e__, TLog#(numServers), MemTagSize)
	    ,FunnelPipesPipelined#(1, numServers, MemData#(userWidth), 2)
	    ,FunnelPipesPipelined#(1, numServers, MemRequest, 2)
	    ,FunnelPipesPipelined#(1, numServers, Bit#(MemTagSize), 2)
	    );
   let rv <- mkMemWriteEngineBuff(valueOf(TExp#(BurstLenSize)));
   return rv;
endmodule

interface MemWriteChannel#(numeric type busWidth, numeric type userWidth, numeric type cmdQDepth);
   interface PipeIn#(Bit#(MemTagSize))        writeGnt; // grants request
   interface PipeOut#(MemRequest)             writeReq;
   interface PipeOut#(MemData#(userWidth))    writeData;
   interface PipeIn#(Bit#(MemTagSize))        writeDone;
   interface MemWriteEngineServer#(userWidth) writeServer;
endinterface

module mkMemWriteChannel#(Integer bufferSizeBytes, Integer channelNumber,
			  PipeOut#(Bit#(MemTagSize)) writeGntPipe, PipeOut#(Bit#(MemTagSize)) writeDonePipe)
   (MemWriteChannel#(busWidth, userWidth, cmdQDepth))
   provisos ( Div#(busWidth,8,busWidthBytes)
	     ,Log#(busWidthBytes,beatShift)
	     ,Add#(1, d__, userWidth)
	     ,Add#(userWidth, 0, busWidth)
	     );

   Integer bufferSizeBeats = bufferSizeBytes/valueOf(busWidthBytes);
   Reg#(Bool) load_in_progress <- mkReg(False);
   FIFO#(Tuple3#(MemengineCmd,Bool,Bool))       serverCond <- mkFIFO1();
   FIFO#(Tuple2#(Bit#(MemTagSize),MemengineCmd)) serverReq <- mkSizedFIFO(valueOf(cmdQDepth));
   FIFO#(Tuple3#(Bit#(8),Bit#(MemTagSize),Bool))inProgress <- mkSizedFIFO(valueOf(cmdQDepth));
   FIFO#(Tuple3#(Bit#(MemTagSize),Bit#(MemTagSize),Bool)) serverDone <- mkSizedFIFO(valueOf(cmdQDepth));
   FIFOF#(MemRequest)           writeReqFifo <- mkFIFOF();
   FIFOF#(MemData#(userWidth)) writeDataFifo <- mkFIFOF();

   Reg#(Bool)              clientInFlight <- mkReg(False);
   Reg#(Bool)              clientBursts <- mkReg(False);
   ConfigCounter#(16)      clientAvail <- mkConfigCounter(0);
   Reg#(MemengineCmd)      clientStart <- mkReg(unpack(0));
   FIFO#(Bool)             clientFinished <- mkSizedFIFO(1);
   FIFOF#(MemengineCmd)    clientCommand <- mkSizedFIFOF(1);
   Count#(Bit#(32))        clientCycles     <- mkCount(0);
   FIFOF#(MemRequestCycles) clientCyclesFifo <- mkFIFOF();
   FIFOF#(Bit#(userWidth)) dataBuffer <- mkSizedBRAMFIFOF(bufferSizeBeats);
   Reg#(Bit#(32)) cycles <- mkReg(0);
   rule rl_cycles;
      cycles <= cycles + 1;
   endrule

   Reg#(Bit#(8))                    respCnt <- mkReg(0);
   let beat_shift = fromInteger(valueOf(beatShift));

   rule store_cmd if (!clientInFlight);
      let cmd <- toGet(clientCommand).get();
      clientInFlight <= True;
      clientBursts <= True;
      clientStart <= cmd;
      $display("cycles %d starting request %d bytes %d", cycles, cmd.tag, cmd.len);
      clientCycles <= 0;
   endrule
   rule rule_request_cycles;
      clientCycles.incr(1);
   endrule

   rule load_ctxt_a if (!load_in_progress);
      if (clientBursts) begin
	 load_in_progress <= True;
	 let cmd = clientStart;
	 let cond0 <- clientAvail.maybeDecrement(unpack(extend(cmd.burstLen>>beat_shift)));
	 let cond1 = cmd.len <= extend(cmd.burstLen);
	 serverCond.enq(tuple3(cmd,cond0,cond1));
      end
   endrule

   rule load_ctxt_b if (load_in_progress);
      load_in_progress <= False;
      match {.cmd,.cond0,.cond1} <- toGet(serverCond).get;
      if  (cond0) begin
	 //$display("load_ctxt_b cycles %d %h", cycles, cmd.base);
	 serverReq.enq(tuple2(0,cmd));
	 if (cond1) begin
	    clientBursts <= False;
	 end
	 else begin
	    clientStart <= MemengineCmd{sglId:cmd.sglId, base:cmd.base+extend(cmd.burstLen),
               burstLen:cmd.burstLen, len:cmd.len-extend(cmd.burstLen), tag:cmd.tag};
	 end
      end
   endrule

   rule rlWriteReq;
      match {.idx, .cmd} <- toGet(serverReq).get;
      Bit#(BurstLenSize) bl = cmd.burstLen;
      Bool last = False;
      if (cmd.len <= extend(bl)) begin
	 last = True;
	 bl = truncate(cmd.len);
      end
      inProgress.enq(tuple3(truncate(bl>>beat_shift), cmd.tag, last));
      //$display("writeReq %d, %h %h %h", channelNumber, cmd.base, bl, last);
      writeReqFifo.enq(MemRequest { sglId: cmd.sglId, offset: extend(cmd.base), burstLen:bl, tag: fromInteger(channelNumber)});
   endrule

   rule rlWriteData;
      match {.rc, .client_tag, .last} = inProgress.first;
      let gnt = writeGntPipe.first;
      let new_respCnt = respCnt+1;
      let lastBeat = False;
      if (new_respCnt == rc) begin
	 respCnt <= 0;
	 inProgress.deq();
	 writeGntPipe.deq();
	 serverDone.enq(tuple3(0,client_tag,last));
	 lastBeat = True;
      end
      else begin
	 respCnt <= new_respCnt;
      end
      let wd <- toGet(dataBuffer).get();
      writeDataFifo.enq(MemData{data:wd, tag:fromInteger(channelNumber), last:lastBeat});
   endrule

   rule rlWriteDone;
      let tag <- toGet(writeDonePipe).get();
      match {.idx, .req_tag, .last} <- toGet(serverDone).get;
      if (last) begin
	 clientInFlight <= False;
	 clientFinished.enq(True);
`ifdef MEMENGINE_REQUEST_CYCLES
	 $display("cycles %d req_tag %d clientCycles = %d", cycles, req_tag, clientCycles);
	 clientCyclesFifo.enq(MemRequestCycles { tag: req_tag, cycles: clientCycles });
`endif
      end
      //$display("writeDone %d %d", channelNumber, last);
   endrule

   MemWriteEngineServer#(userWidth) ws = (interface MemWriteEngineServer#(userWidth);
      interface Put request;
	 method Action put(MemengineCmd cmd);
	    Bit#(32) bsb = fromInteger(bufferSizeBytes);
      `ifdef SIMULATION
	    Bit#(32) dw = fromInteger(valueOf(busWidthBytes));
	    Bit#(32) bl = extend(cmd.burstLen);
	    // this is because bsc lifts the divide operation (below)
	    // and on startup the simulator gets a floating-point exception
	    if (bl ==0)
	       bl = 1;
	    let mdw0 = ((cmd.len)/bl)*bl != cmd.len;
	    let mdw1 = ((cmd.len)/dw)*dw != cmd.len;
	    let bbl = extend(cmd.burstLen) > bsb;
	    if(bbl || mdw0 || mdw1 || cmd.len == 0) begin
	       if (bbl)
					  $display("XXXXXXXXXX mkMemWriteEngineBuff::unsupported burstLen %d %d", bsb, cmd.burstLen);
	       if (mdw0 || mdw1 || cmd.len == 0)
					  $display("XXXXXXXXXX mkMemWriteEngineBuff::unsupported len %h mdw0=%d mdw1=%d", cmd.len, mdw0, mdw1);
	    end
	    else
      `endif
	       begin
		  clientCommand.enq(cmd);
		  $display("(%d) %h %h %h", channelNumber, cmd.base, cmd.len, cmd.burstLen);
	       end
	 endmethod
      endinterface
      interface Get done;
	 method ActionValue#(Bool) get = toGet(clientFinished).get;
      endinterface
      interface PipeIn data = interface PipeIn;
				 method Bool notFull = dataBuffer.notFull;
   				 method Action enq(Bit#(userWidth) v);
				    dataBuffer.enq(v);
				    clientAvail.increment(1);
				 endmethod
			      endinterface;
	 interface PipeOut requestCycles = toPipeOut(clientCyclesFifo);
	 endinterface);
   interface writeServer = ws;
   interface writeReq = toPipeOut(writeReqFifo);
   interface writeData = toPipeOut(writeDataFifo);
endmodule

module mkMemWriteEngineBuff#(Integer bufferSizeBytes)(MemWriteEngine#(busWidth, userWidth, cmdQDepth, numServers))
   provisos ( Div#(busWidth,8,busWidthBytes)
	     ,Log#(busWidthBytes,beatShift)
	     ,Add#(1, a__, userWidth)
	     ,Add#(userWidth, 0, busWidth)
	     ,Add#(b__, TLog#(numServers), MemTagSize)
             ,FunnelPipesPipelined#(1, numServers, MemData#(userWidth), 2)
             ,FunnelPipesPipelined#(1, numServers, MemRequest, 2)
             ,FunnelPipesPipelined#(1, numServers, Bit#(MemTagSize), 2)
	     );

   FIFOF#(Bit#(MemTagSize)) writeDoneFifo <- mkFIFOF();
   FIFOF#(Bit#(MemTagSize))       arbFifo <- mkFIFOF1();
   function Tuple2#(Bit#(TLog#(numServers)),Bit#(MemTagSize)) tagDone(Bit#(MemTagSize) tag);
      return tuple2(truncate(tag), tag);
   endfunction
   UnFunnelPipe#(1,numServers,Bit#(MemTagSize),2) arbPipes <- mkUnFunnelPipesPipelined(vec(mapPipe(tagDone, toPipeOut(arbFifo))));
   UnFunnelPipe#(1,numServers,Bit#(MemTagSize),2) donePipes <- mkUnFunnelPipesPipelined(vec(mapPipe(tagDone, toPipeOut(writeDoneFifo))));
   Vector#(numServers, MemWriteChannel#(busWidth,userWidth,cmdQDepth)) writeChannels <- zipWith3M(mkMemWriteChannel(bufferSizeBytes),
												  genVector(),
												  arbPipes,
												  donePipes);
   function PipeOut#(MemRequest) writeChannelDmaWriteReq(Integer i);
      return writeChannels[i].writeReq;
   endfunction
   function PipeOut#(MemData#(userWidth)) writeChannelDmaWriteData(Integer i);
      return writeChannels[i].writeData;
   endfunction
   function MemWriteEngineServer#(userWidth) writeChannelServer(Integer i);
      return writeChannels[i].writeServer;
   endfunction

   Reg#(Bool)         reqInFlight <- mkReg(False);
   FIFOF#(MemRequest)          writeReqFifo <- mkFIFOF();
   FIFOF#(MemData#(userWidth)) writeDataFifo <- mkSizedFIFOF(16);
   FunnelPipe#(1,numServers,MemRequest,2) reqFunnel <- mkFunnelPipesPipelined(genWith(writeChannelDmaWriteReq));
   FunnelPipe#(1,numServers,MemData#(userWidth),2) dataFunnel <- mkFunnelPipesPipelined(genWith(writeChannelDmaWriteData));

   rule rl_arbitration if (!reqInFlight);
      let req <- toGet(reqFunnel[0]).get();
      // tag is channel number
      arbFifo.enq(req.tag);
      writeReqFifo.enq(req);
      reqInFlight <= True;
   endrule
   rule rl_writeData if (reqInFlight);
      MemData#(userWidth) md <- toGet(dataFunnel[0]).get();
      if (md.last)
	 reqInFlight <= False;
      writeDataFifo.enq(md);
   endrule

   interface writeServers = genWith(writeChannelServer);
   interface MemWriteClient dmaClient;
      interface writeReq = toGet(writeReqFifo);
      interface writeData = toGet(writeDataFifo);
      interface writeDone = toPut(writeDoneFifo);
   endinterface: dmaClient

endmodule

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
import Vector::*;
import BuildVector::*;
import Cntrs::*;
import FIFOF::*;
import FIFO::*;
import GetPut::*;
import Connectable::*;
import BRAMFIFO::*;
import ConfigCounter::*;
import MemTypes::*;
import Pipe::*;
import ConnectalConfig::*;

`include "ConnectalProjectConfig.bsv"

module mkMemWriteEngine(MemWriteEngine#(busWidth, userWidth, cmdQDepth, numServers))
   provisos( Add#(1, d__, busWidth)
	    ,Add#(1, d__, userWidth)
	    ,Add#(e__, TLog#(numServers), MemTagSize)
	    ,FunnelPipesPipelined#(1, numServers, MemData#(userWidth), 2)
	    ,FunnelPipesPipelined#(1, numServers, MemRequest, 2)
	    ,FunnelPipesPipelined#(1, numServers, Bit#(MemTagSize), 2)
	    );
   let rv <- mkMemWriteEngineBuff(valueOf(TExp#(BurstLenSize)));
   return rv;
endmodule

interface MemWriteChannel#(numeric type busWidth, numeric type userWidth, numeric type cmdQDepth);
   interface PipeIn#(Bit#(MemTagSize))        writeGnt; // grants request
   interface PipeOut#(MemRequest)             writeReq;
   interface PipeOut#(MemData#(userWidth))    writeData;
   interface PipeIn#(Bit#(MemTagSize))        writeDone;
   interface MemWriteEngineServer#(userWidth) writeServer;
endinterface

module mkMemWriteChannel#(Integer bufferSizeBytes, Integer channelNumber,
			  PipeOut#(Bit#(MemTagSize)) writeGntPipe, PipeOut#(Bit#(MemTagSize)) writeDonePipe)
   (MemWriteChannel#(busWidth, userWidth, cmdQDepth))
   provisos ( Div#(busWidth,8,busWidthBytes)
	     ,Log#(busWidthBytes,beatShift)
	     ,Add#(1, d__, userWidth)
	     ,Add#(userWidth, 0, busWidth)
	     );

   Integer bufferSizeBeats = bufferSizeBytes/valueOf(busWidthBytes);
   Reg#(Bool) load_in_progress <- mkReg(False);
   FIFO#(Tuple3#(MemengineCmd,Bool,Bool))       serverCond <- mkFIFO1();
   FIFO#(Tuple2#(Bit#(MemTagSize),MemengineCmd)) serverReq <- mkSizedFIFO(valueOf(cmdQDepth));
   FIFO#(Tuple3#(Bit#(8),Bit#(MemTagSize),Bool))inProgress <- mkSizedFIFO(valueOf(cmdQDepth));
   FIFO#(Tuple3#(Bit#(MemTagSize),Bit#(MemTagSize),Bool)) serverDone <- mkSizedFIFO(valueOf(cmdQDepth));
   FIFOF#(MemRequest)           writeReqFifo <- mkFIFOF();
   FIFOF#(MemData#(userWidth)) writeDataFifo <- mkFIFOF();

   Reg#(Bool)              clientInFlight <- mkReg(False);
   Reg#(Bool)              clientBursts <- mkReg(False);
   ConfigCounter#(16)      clientAvail <- mkConfigCounter(0);
   Reg#(MemengineCmd)      clientStart <- mkReg(unpack(0));
   FIFO#(Bool)             clientFinished <- mkSizedFIFO(1);
   FIFOF#(MemengineCmd)    clientCommand <- mkSizedFIFOF(1);
   Count#(Bit#(32))        clientCycles     <- mkCount(0);
   FIFOF#(MemRequestCycles) clientCyclesFifo <- mkFIFOF();
   FIFOF#(Bit#(userWidth)) dataBuffer <- mkSizedBRAMFIFOF(bufferSizeBeats);
   Reg#(Bit#(32)) cycles <- mkReg(0);
   rule rl_cycles;
      cycles <= cycles + 1;
   endrule

   Reg#(Bit#(8))                    respCnt <- mkReg(0);
   let beat_shift = fromInteger(valueOf(beatShift));

   rule store_cmd if (!clientInFlight);
      let cmd <- toGet(clientCommand).get();
      clientInFlight <= True;
      clientBursts <= True;
      clientStart <= cmd;
      $display("cycles %d starting request %d bytes %d", cycles, cmd.tag, cmd.len);
      clientCycles <= 0;
   endrule
   rule rule_request_cycles;
      clientCycles.incr(1);
   endrule

   rule load_ctxt_a if (!load_in_progress);
      if (clientBursts) begin
	 load_in_progress <= True;
	 let cmd = clientStart;
	 let cond0 <- clientAvail.maybeDecrement(unpack(extend(cmd.burstLen>>beat_shift)));
	 let cond1 = cmd.len <= extend(cmd.burstLen);
	 serverCond.enq(tuple3(cmd,cond0,cond1));
      end
   endrule

   rule load_ctxt_b if (load_in_progress);
      load_in_progress <= False;
      match {.cmd,.cond0,.cond1} <- toGet(serverCond).get;
      if  (cond0) begin
	 //$display("load_ctxt_b cycles %d %h", cycles, cmd.base);
	 serverReq.enq(tuple2(0,cmd));
	 if (cond1) begin
	    clientBursts <= False;
	 end
	 else begin
	    clientStart <= MemengineCmd{sglId:cmd.sglId, base:cmd.base+extend(cmd.burstLen),
               burstLen:cmd.burstLen, len:cmd.len-extend(cmd.burstLen), tag:cmd.tag};
	 end
      end
   endrule

   rule rlWriteReq;
      match {.idx, .cmd} <- toGet(serverReq).get;
      Bit#(BurstLenSize) bl = cmd.burstLen;
      Bool last = False;
      if (cmd.len <= extend(bl)) begin
	 last = True;
	 bl = truncate(cmd.len);
      end
      inProgress.enq(tuple3(truncate(bl>>beat_shift), cmd.tag, last));
      //$display("writeReq %d, %h %h %h", channelNumber, cmd.base, bl, last);
      writeReqFifo.enq(MemRequest { sglId: cmd.sglId, offset: extend(cmd.base), burstLen:bl, tag: fromInteger(channelNumber)});
   endrule

   rule rlWriteData;
      match {.rc, .client_tag, .last} = inProgress.first;
      let gnt = writeGntPipe.first;
      let new_respCnt = respCnt+1;
      let lastBeat = False;
      if (new_respCnt == rc) begin
	 respCnt <= 0;
	 inProgress.deq();
	 writeGntPipe.deq();
	 serverDone.enq(tuple3(0,client_tag,last));
	 lastBeat = True;
      end
      else begin
	 respCnt <= new_respCnt;
      end
      let wd <- toGet(dataBuffer).get();
      writeDataFifo.enq(MemData{data:wd, tag:fromInteger(channelNumber), last:lastBeat});
   endrule

   rule rlWriteDone;
      let tag <- toGet(writeDonePipe).get();
      match {.idx, .req_tag, .last} <- toGet(serverDone).get;
      if (last) begin
	 clientInFlight <= False;
	 clientFinished.enq(True);
`ifdef MEMENGINE_REQUEST_CYCLES
	 $display("cycles %d req_tag %d clientCycles = %d", cycles, req_tag, clientCycles);
	 clientCyclesFifo.enq(MemRequestCycles { tag: req_tag, cycles: clientCycles });
`endif
      end
      //$display("writeDone %d %d", channelNumber, last);
   endrule

   MemWriteEngineServer#(userWidth) ws = (interface MemWriteEngineServer#(userWidth);
      interface Put request;
	 method Action put(MemengineCmd cmd);
	    Bit#(32) bsb = fromInteger(bufferSizeBytes);
      `ifdef SIMULATION
	    Bit#(32) dw = fromInteger(valueOf(busWidthBytes));
	    Bit#(32) bl = extend(cmd.burstLen);
	    // this is because bsc lifts the divide operation (below)
	    // and on startup the simulator gets a floating-point exception
	    if (bl ==0)
	       bl = 1;
	    let mdw0 = ((cmd.len)/bl)*bl != cmd.len;
	    let mdw1 = ((cmd.len)/dw)*dw != cmd.len;
	    let bbl = extend(cmd.burstLen) > bsb;
	    if(bbl || mdw0 || mdw1 || cmd.len == 0) begin
	       if (bbl)
					  $display("XXXXXXXXXX mkMemWriteEngineBuff::unsupported burstLen %d %d", bsb, cmd.burstLen);
	       if (mdw0 || mdw1 || cmd.len == 0)
					  $display("XXXXXXXXXX mkMemWriteEngineBuff::unsupported len %h mdw0=%d mdw1=%d", cmd.len, mdw0, mdw1);
	    end
	    else
      `endif
	       begin
		  clientCommand.enq(cmd);
		  $display("(%d) %h %h %h", channelNumber, cmd.base, cmd.len, cmd.burstLen);
	       end
	 endmethod
      endinterface
      interface Get done;
	 method ActionValue#(Bool) get = toGet(clientFinished).get;
      endinterface
      interface PipeIn data = interface PipeIn;
				 method Bool notFull = dataBuffer.notFull;
   				 method Action enq(Bit#(userWidth) v);
				    dataBuffer.enq(v);
				    clientAvail.increment(1);
				 endmethod
			      endinterface;
	 interface PipeOut requestCycles = toPipeOut(clientCyclesFifo);
	 endinterface);
   interface writeServer = ws;
   interface writeReq = toPipeOut(writeReqFifo);
   interface writeData = toPipeOut(writeDataFifo);
endmodule

module mkMemWriteEngineBuff#(Integer bufferSizeBytes)(MemWriteEngine#(busWidth, userWidth, cmdQDepth, numServers))
   provisos ( Div#(busWidth,8,busWidthBytes)
	     ,Log#(busWidthBytes,beatShift)
	     ,Add#(1, a__, userWidth)
	     ,Add#(userWidth, 0, busWidth)
	     ,Add#(b__, TLog#(numServers), MemTagSize)
             ,FunnelPipesPipelined#(1, numServers, MemData#(userWidth), 2)
             ,FunnelPipesPipelined#(1, numServers, MemRequest, 2)
             ,FunnelPipesPipelined#(1, numServers, Bit#(MemTagSize), 2)
	     );

   FIFOF#(Bit#(MemTagSize)) writeDoneFifo <- mkFIFOF();
   FIFOF#(Bit#(MemTagSize))       arbFifo <- mkFIFOF1();
   function Tuple2#(Bit#(TLog#(numServers)),Bit#(MemTagSize)) tagDone(Bit#(MemTagSize) tag);
      return tuple2(truncate(tag), tag);
   endfunction
   UnFunnelPipe#(1,numServers,Bit#(MemTagSize),2) arbPipes <- mkUnFunnelPipesPipelined(vec(mapPipe(tagDone, toPipeOut(arbFifo))));
   UnFunnelPipe#(1,numServers,Bit#(MemTagSize),2) donePipes <- mkUnFunnelPipesPipelined(vec(mapPipe(tagDone, toPipeOut(writeDoneFifo))));
   Vector#(numServers, MemWriteChannel#(busWidth,userWidth,cmdQDepth)) writeChannels <- zipWith3M(mkMemWriteChannel(bufferSizeBytes),
												  genVector(),
												  arbPipes,
												  donePipes);
   function PipeOut#(MemRequest) writeChannelDmaWriteReq(Integer i);
      return writeChannels[i].writeReq;
   endfunction
   function PipeOut#(MemData#(userWidth)) writeChannelDmaWriteData(Integer i);
      return writeChannels[i].writeData;
   endfunction
   function MemWriteEngineServer#(userWidth) writeChannelServer(Integer i);
      return writeChannels[i].writeServer;
   endfunction

   Reg#(Bool)         reqInFlight <- mkReg(False);
   FIFOF#(MemRequest)          writeReqFifo <- mkFIFOF();
   FIFOF#(MemData#(userWidth)) writeDataFifo <- mkFIFOF();
   FunnelPipe#(1,numServers,MemRequest,2) reqFunnel <- mkFunnelPipesPipelined(genWith(writeChannelDmaWriteReq));
   FunnelPipe#(1,numServers,MemData#(userWidth),2) dataFunnel <- mkFunnelPipesPipelined(genWith(writeChannelDmaWriteData));

   rule rl_arbitration if (!reqInFlight);
      let req <- toGet(reqFunnel[0]).get();
      // tag is channel number
      arbFifo.enq(req.tag);
      writeReqFifo.enq(req);
      reqInFlight <= True;
   endrule
   rule rl_writeData if (reqInFlight);
      MemData#(userWidth) md <- toGet(dataFunnel[0]).get();
      if (md.last)
	 reqInFlight <= False;
      writeDataFifo.enq(md);
   endrule

   interface writeServers = genWith(writeChannelServer);
   interface MemWriteClient dmaClient;
      interface writeReq = toGet(writeReqFifo);
      interface writeData = toGet(writeDataFifo);
      interface writeDone = toPut(writeDoneFifo);
   endinterface: dmaClient

endmodule