add support for any register type up to 32 bits

// Copyright (c) 2014 Quanta Research Cambridge, Inc.
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import RegC::*;
import Connectable::*;

/* This is a simple serial bus
 * The frame bit indicates the valid period of a message
 *
 */

interface SpiTapIn;
   method Action frame(bit f);
   method Action data(bit d);
endinterface

interface SpiTapOut;
   method bit frame();
   method bit data();
endinterface

interface SpiTap;
   interface SpiTapIn in;
   interface SpiTapOut out;
endinterface

interface SpiReg#(type a);
   interface SpiTap tap;
   interface Reg#(a) r;
endinterface

instance Connectable#(SpiTapOut, SpiTapIn);
   module mkConnection#(SpiTapOut out, SpiTapIn in)(Empty);
      rule move_data;
	 in.frame(out.frame());
	 in.data(out.data());
	 endrule
   endmodule
endinstance

module mkSpiReg#(Bit#(32) id)(SpiReg#(a))
   provisos(Bits#(a,asize),
      Add#(a__, asize, 32));
   Reg#(bit) frameinbit <- mkReg(0);
   Reg#(bit) datainbit <- mkReg(0);
   Wire#(bit) dataoutwire <- mkDWire(0);

   Reg#(Bit#(6)) count <- mkReg(0);
   Reg#(Bit#(32)) shifter <- mkReg(0);
   Reg#(Bool) addressmatch <- mkReg(False);
   Reg#(Bool) iswrite <- mkReg(False);
   Reg#(Bit#(asize)) data <- mkReg(0);

   rule handleFrame;
      if (frameinbit == 0)
	 begin
            count <= 0;
	    addressmatch <= False;
	 end
      else
	 count <= count + 1;
   endrule

   rule handleShift (frameinbit == 1);
      Bit#(32) tmp = shifter;
      tmp = tmp >> 1;
      tmp[31] = datainbit;
      shifter <= tmp;
      if (count == 31)
	 begin
	    iswrite <= tmp[0] == 1;
            addressmatch <= (id[31:1] == tmp[31:1]);
         end
      if ((count == 63) && addressmatch && iswrite)
	 data <= truncate(tmp);
      if ((count[5] == 1) && addressmatch && (!iswrite))
          begin
	     if (valueof(asize) == 32)
		dataoutwire <= data[count & 31];
	     else
		begin
		   if ((count & 31) < fromInteger(valueof(asize)))
                      dataoutwire <= data[count & 31];
		end
	  end
      else
	 dataoutwire <= datainbit;
   endrule

   interface SpiTap tap;

      interface SpiTapIn in;

	 method Action frame(bit i );
	    frameinbit <= i ;
	 endmethod

	 method Action data( bit i );
	    datainbit <= i;
	 endmethod

      endinterface

      interface SpiTapOut out;

	 method bit frame();
	    return frameinbit;
	 endmethod

	 method bit data();
	    return dataoutwire;
	 endmethod

      endinterface

   endinterface

   interface Reg r;

      method Action _write(a v);
	 data <= pack(v);
      endmethod

      method a _read();
	 return(unpack(data));
      endmethod

   endinterface

endmodule

// Copyright (c) 2014 Quanta Research Cambridge, Inc.
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import RegC::*;
import Connectable::*;

/* This is a simple serial bus
 * The frame bit indicates the valid period of a message
 *
 */

interface SpiTapIn;
   method Action frame(bit f);
   method Action data(bit d);
endinterface

interface SpiTapOut;
   method bit frame();
   method bit data();
endinterface

interface SpiTap;
   interface SpiTapIn in;
   interface SpiTapOut out;
endinterface

instance Connectable#(SpiTapOut, SpiTapIn);
   module mkConnection#(SpiTapOut out, SpiTapIn in)(Empty);
      rule move_data;
	 in.frame(out.frame());
	 in.data(out.data());
	 endrule
   endmodule
endinstance

(* synthesize *)
module mkSpiTap#(Bit#(32) id)(SpiTap);
   Reg#(bit) frameinbit <- mkReg(0);
   Reg#(bit) datainbit <- mkReg(0);
   Wire#(bit) dataoutwire <- mkDWire(0);

   Reg#(Bit#(6)) count <- mkReg(0);
   Reg#(Bit#(32)) shifter <- mkReg(0);
   Reg#(Bit#(32)) address <- mkReg(0);
   Reg#(Bit#(32)) data <- mkReg(0);

   rule handleFrame;
      if (frameinbit == 0)
	 count <= 0;
      else
	 count <= count + 1;
   endrule

   rule handleShift (frameinbit == 1);
      Bit#(32) tmp = shifter;
      tmp = tmp >> 1;
      tmp[31] = datainbit;
      shifter <= tmp;
      if (count == 31)
	 address <= tmp;
      if ((count == 63) && (address[31:1] == id[31:1]) && (address[0] == 1))
	 data <= tmp;
      if ((count[5] == 1) && (address[31:1] == id[31:1]) && (address[0] == 0))
	 dataoutwire <= data[count & 31];
      else
	 dataoutwire <= datainbit;
   endrule

   interface SpiTapIn in;

      method Action frame(bit i );
	 frameinbit <= i ;
      endmethod

      method Action data( bit i );
	 datainbit <= i;
      endmethod

   endinterface

   interface SpiTapOut out;

      method bit frame();
	 return frameinbit;
      endmethod

      method bit data();
	 return dataoutwire;
      endmethod

   endinterface

endmodule


Simplify AxiGather a bit

Simplify AxiGather a bit

// Copyright (c) 2015 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
import GetPut::*;
import Vector::*;
import PPS7LIB::*;
import AxiMasterSlave::*;
import AxiBits::*;

interface AxiMasterCommon#(numeric type addrWidth, numeric type dataWidth, numeric type tagWidth);
    method Bit#(1)            aresetn();
    interface Axi3Master#(addrWidth,dataWidth,tagWidth) client;
endinterface

interface AxiSlaveCommon#(numeric type addrWidth, numeric type dataWidth, numeric type tagWidth, type extraType);
    method Bit#(1)            aresetn();
    interface Axi3Slave#(addrWidth,dataWidth,tagWidth) server;
    interface extraType   extra;
endinterface

module mkAxi3MasterGather#(AxiMasterBits#(addrWidth, dataWidth, tagWidth, Empty) axiWires)(AxiMasterCommon#(addrWidth, dataWidth, tagWidth));
   Wire#(Bit#(1)) arready <- mkDWire(0);
   Wire#(Bit#(1)) awready <- mkDWire(0);
   Wire#(Bit#(1)) rvalid <- mkDWire(0);
   Wire#(Bit#(1)) wready <- mkDWire(0);
   Wire#(Bit#(tagWidth)) rid <- mkDWire(0);
   Wire#(Bit#(2)) rresp <- mkDWire(0);
   Wire#(Bit#(dataWidth)) rdata <- mkDWire(0);
   Wire#(Bit#(1)) rlast <- mkDWire(0);
   Wire#(Bit#(1)) bvalid <- mkDWire(0);
   Wire#(Bit#(tagWidth)) bid <- mkDWire(0);
   Wire#(Bit#(2)) bresp <- mkDWire(0);

   rule handshake1;
        axiWires.arready(arready);
   endrule
   rule handshake2;
        axiWires.awready(awready);
   endrule
   rule handshake3;
        axiWires.rid(rid);
        axiWires.rresp(rresp);
        axiWires.rdata(rdata);
        axiWires.rlast(rlast);
        axiWires.rvalid(rvalid);
   endrule
   rule handshake4;
        axiWires.wready(wready);
   endrule
   rule handshake5;
        axiWires.bvalid(bvalid);
        axiWires.bid(bid);
        axiWires.bresp(bresp);
   endrule

   interface Axi3Master client;
        interface Get req_ar;
             method ActionValue#(Axi3ReadRequest#(addrWidth,tagWidth)) get() if (axiWires.arvalid() != 0);
                 Axi3ReadRequest#(addrWidth,tagWidth) v;
                 v.address = axiWires.araddr();
                 v.burst = axiWires.arburst();
                 v.cache = axiWires.arcache();
                 v.id = axiWires.arid();
                 v.len = axiWires.arlen();
                 v.lock = axiWires.arlock();
                 v.prot = axiWires.arprot();
                 v.qos = axiWires.arqos();
                 v.size = {0, axiWires.arsize()};

                arready <= 1;
                return v;
            endmethod
        endinterface
        interface Get req_aw;
            method ActionValue#(Axi3WriteRequest#(addrWidth,tagWidth)) get() if (axiWires.awvalid() != 0);
                Axi3WriteRequest#(addrWidth,tagWidth) v;
                v.address = axiWires.awaddr();
                v.burst = axiWires.awburst();
                v.cache = axiWires.awcache();
                v.id = axiWires.awid();
                v.len = axiWires.awlen();
                v.lock = axiWires.awlock();
                v.prot = axiWires.awprot();
                v.qos = axiWires.awqos();
                v.size = {0, axiWires.awsize()};

                awready <= 1;
                return v;
           endmethod
        endinterface
        interface Put resp_read;
            method Action put(Axi3ReadResponse#(dataWidth, tagWidth) v) if (axiWires.rready() != 0);
                rid <= v.id;
                rresp <= v.resp;
                rdata <= v.data;
                rlast <= v.last;
                rvalid <= 1;
            endmethod
        endinterface
        interface Get resp_write;
            method ActionValue#(Axi3WriteData#(dataWidth,tagWidth)) get() if (axiWires.wvalid() != 0);
                Axi3WriteData#(dataWidth,tagWidth) v;
                v.id = axiWires.wid();
                v.byteEnable = axiWires.wstrb();
                v.data = axiWires.wdata();
                v.last = axiWires.wlast();

                wready <= 1;
                return v;
            endmethod
        endinterface
        interface Put resp_b;
            method Action put(Axi3WriteResponse#(tagWidth) v) if (axiWires.bready() != 0);
                bvalid <= 1;
                bid    <= v.id;
                bresp  <= v.resp;
            endmethod
        endinterface
    endinterface
    method aresetn = axiWires.aresetn;
endmodule

module mkAxi3SlaveGather#(AxiSlaveBits#(addrWidth, dataWidth, tagWidth,
    extraType) axiWires)(AxiSlaveCommon#(addrWidth, dataWidth,tagWidth,extraType));
    Wire#(Bit#(addrWidth)) araddr <- mkDWire(0);
    Wire#(Bit#(2)) arburst <- mkDWire(0);
    Wire#(Bit#(4)) arcache <- mkDWire(0);
    Wire#(Bit#(tagWidth)) arid <- mkDWire(0);
    Wire#(Bit#(4)) arlen <- mkDWire(0);
    Wire#(Bit#(2)) arlock <- mkDWire(0);
    Wire#(Bit#(3)) arprot <- mkDWire(0);
    Wire#(Bit#(4)) arqos <- mkDWire(0);
    Wire#(Bit#(2)) arsize <- mkDWire(0);
    Wire#(Bit#(1)) arvalid <- mkDWire(0);

    Wire#(Bit#(addrWidth)) awaddr <- mkDWire(0);
    Wire#(Bit#(2)) awburst <- mkDWire(0);
    Wire#(Bit#(4)) awcache <- mkDWire(0);
    Wire#(Bit#(tagWidth)) awid <- mkDWire(0);
    Wire#(Bit#(4)) awlen <- mkDWire(0);
    Wire#(Bit#(2)) awlock <- mkDWire(0);
    Wire#(Bit#(3)) awprot <- mkDWire(0);
    Wire#(Bit#(4)) awqos <- mkDWire(0);
    Wire#(Bit#(2)) awsize <- mkDWire(0);
    Wire#(Bit#(1)) awvalid <- mkDWire(0);

    Wire#(Bit#(1)) rready <- mkDWire(0);
    Wire#(Bit#(tagWidth)) wid <- mkDWire(0);
    Wire#(Bit#(TDiv#(dataWidth,8))) wstrb <- mkDWire(0);
    Wire#(Bit#(dataWidth)) wdata <- mkDWire(0);
    Wire#(Bit#(1)) wlast <- mkDWire(0);
    Wire#(Bit#(1)) wvalid <- mkDWire(0);
    Wire#(Bit#(1)) bready <- mkDWire(0);

    rule handshake1;
          axiWires.araddr(araddr);
          axiWires.arburst(arburst);
          axiWires.arcache(arcache);
          axiWires.arid(arid);
          axiWires.arlen(arlen);
          axiWires.arlock(arlock);
          axiWires.arprot(arprot);
          axiWires.arqos(arqos);
          axiWires.arsize(arsize);
          axiWires.arvalid(arvalid);
    endrule
    rule handshake2;
          axiWires.awaddr(awaddr);
          axiWires.awburst(awburst);
          axiWires.awcache(awcache);
          axiWires.awid(awid);
          axiWires.awlen(awlen);
          axiWires.awlock(awlock);
          axiWires.awprot(awprot);
          axiWires.awqos(awqos);
          axiWires.awsize(awsize);
          axiWires.awvalid(awvalid);
    endrule
    rule handshake3;
         axiWires.rready(rready);
    endrule
    rule handshake4;
          axiWires.wid(wid);
          axiWires.wstrb(wstrb);
          axiWires.wdata(wdata);
          axiWires.wlast(wlast);
          axiWires.wvalid(wvalid);
    endrule
    rule handshake5;
         axiWires.bready(bready);
    endrule
    interface Axi3Slave server;
    interface Put req_ar;
        method Action put(Axi3ReadRequest#(addrWidth,tagWidth) v) if (axiWires.arready() != 0);
           araddr <= v.address;
           arburst <= v.burst;
           arcache <= v.cache;
           arid <= v.id;
           arlen <= v.len;
           arlock <= v.lock;
           arprot <= v.prot;
           arqos <= v.qos;
           arsize <= v.size[1:0];

           arvalid <= 1;
        endmethod
    endinterface
    interface Put req_aw;
        method Action put(Axi3WriteRequest#(addrWidth,tagWidth) v) if (axiWires.awready() != 0);
           awaddr <= v.address;
           awburst <= v.burst;
           awcache <= v.cache;
           awid <= v.id;
           awlen <= v.len;
           awlock <= v.lock;
           awprot <= v.prot;
           awqos <= v.qos;
           awsize <= v.size[1:0];

           awvalid <= 1;
        endmethod
    endinterface
    interface Put resp_write;
        method Action put(Axi3WriteData#(dataWidth,tagWidth) v) if (axiWires.wready() != 0);
           wid <= v.id;
           wstrb <= v.byteEnable;
           wdata <= v.data;
           wlast <= v.last;

           wvalid <= 1;
        endmethod
    endinterface
    interface Get resp_read;
        method ActionValue#(Axi3ReadResponse#(dataWidth, tagWidth)) get() if (axiWires.rvalid() != 0);
            Axi3ReadResponse#(dataWidth, tagWidth) v;
            v.id = axiWires.rid();
            v.resp = axiWires.rresp();
            v.data = axiWires.rdata();
            v.last = axiWires.rlast();

            rready <= 1;
            return v;
        endmethod
         endinterface
    interface Get resp_b;
        method ActionValue#(Axi3WriteResponse#(tagWidth)) get() if (axiWires.bvalid() != 0);
            Axi3WriteResponse#(tagWidth) v;
            v.id = axiWires.bid();
            v.resp = axiWires.bresp();

            bready <= 1;
            return v;
        endmethod
     endinterface
     endinterface
     interface extra = axiWires.extra;
     method aresetn = axiWires.aresetn;
endmodule

// Copyright (c) 2015 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
import GetPut::*;
import Vector::*;
import PPS7LIB::*;
import AxiMasterSlave::*;
import AxiBits::*;

interface AxiMasterCommon#(numeric type addrWidth, numeric type dataWidth, numeric type tagWidth);
    method Bit#(1)            aresetn();
    interface Axi3Master#(addrWidth,dataWidth,tagWidth) client;
endinterface

interface AxiSlaveCommon#(numeric type addrWidth, numeric type dataWidth, numeric type tagWidth, type extraType);
    method Bit#(1)            aresetn();
    interface Axi3Slave#(addrWidth,dataWidth,tagWidth) server;
    interface extraType   extra;
endinterface

interface AxiMasterWires#(numeric type addrWidth, numeric type dataWidth, numeric type tagWidth);
   interface Wire#(Bit#(1)) arready;
   interface Wire#(Bit#(1)) awready;
   interface Wire#(Bit#(1)) rvalid;
   interface Wire#(Bit#(1)) wready;
   interface Wire#(Bit#(1)) bvalid;

   interface Wire#(Bit#(tagWidth)) rid;
   interface Wire#(Bit#(2))  rresp;
   interface Wire#(Bit#(dataWidth)) rdata;
   interface Wire#(Bit#(1))  rlast;
   interface Wire#(Bit#(tagWidth)) bid;
   interface Wire#(Bit#(2)) bresp;
endinterface

interface AxiSlaveWires#(numeric type addrWidth, numeric type dataWidth, numeric type tagWidth);
   interface Wire#(Bit#(addrWidth)) araddr;
   interface Wire#(Bit#(2)) arburst;
   interface Wire#(Bit#(4)) arcache;
   interface Wire#(Bit#(tagWidth)) arid;
   interface Wire#(Bit#(4)) arlen;
   interface Wire#(Bit#(2)) arlock;
   interface Wire#(Bit#(3)) arprot;
   interface Wire#(Bit#(4)) arqos;
   interface Wire#(Bit#(2)) arsize;
   interface Wire#(Bit#(1)) arvalid;
   interface Wire#(Bit#(addrWidth)) awaddr;
   interface Wire#(Bit#(2)) awburst;
   interface Wire#(Bit#(4)) awcache;
   interface Wire#(Bit#(tagWidth)) awid;
   interface Wire#(Bit#(4)) awlen;
   interface Wire#(Bit#(2)) awlock;
   interface Wire#(Bit#(3)) awprot;
   interface Wire#(Bit#(4)) awqos;
   interface Wire#(Bit#(2)) awsize;
   interface Wire#(Bit#(1)) awvalid;
   interface Wire#(Bit#(1)) rready;
   interface Wire#(Bit#(tagWidth)) wid;
   interface Wire#(Bit#(TDiv#(dataWidth,8))) wstrb;
   interface Wire#(Bit#(dataWidth)) wdata;
   interface Wire#(Bit#(1)) wlast;
   interface Wire#(Bit#(1)) wvalid;
   interface Wire#(Bit#(1)) bready;
endinterface

module mkAxiMasterWires(AxiMasterWires#(addrWidth, dataWidth, tagWidth));
   Vector#(6, Wire#(Bit#(1))) wires <- replicateM(mkDWire(0));
   Vector#(1, Wire#(Bit#(dataWidth))) datawires <- replicateM(mkDWire(0));
   Vector#(2, Wire#(Bit#(tagWidth))) idwires <- replicateM(mkDWire(0));
   Vector#(2, Wire#(Bit#(2))) respwires <- replicateM(mkDWire(0));
   interface Wire arready = wires[0];
   interface Wire awready = wires[1];
   interface Wire rvalid = wires[2];
   interface Wire wready = wires[3];
   interface Wire rid  = idwires[0];
   interface Wire rresp  = respwires[0];
   interface Wire rdata  = datawires[0];
   interface Wire rlast  = wires[4];
   interface Wire bvalid = wires[5];
   interface Wire bid    = idwires[1];
   interface Wire bresp  = respwires[1];
endmodule

module mkAxiSlaveWires(AxiSlaveWires#(addrWidth, dataWidth, tagWidth));
   Vector#(5, Wire#(Bit#(1))) wires <- replicateM(mkDWire(0));
   Vector#(2, Wire#(Bit#(addrWidth))) addrwires <- replicateM(mkDWire(0));
   Vector#(1, Wire#(Bit#(dataWidth))) datawires <- replicateM(mkDWire(0));
   Vector#(2, Wire#(Bit#(2))) burstwires <- replicateM(mkDWire(0));
   Vector#(2, Wire#(Bit#(4))) cachewires <- replicateM(mkDWire(0));
   Vector#(3, Wire#(Bit#(tagWidth))) idwires <- replicateM(mkDWire(0));
   Vector#(1, Wire#(Bit#(TDiv#(dataWidth,8)))) strbwires <- replicateM(mkDWire(0));
   Vector#(2, Wire#(Bit#(4))) lenwires <- replicateM(mkDWire(0));
   Vector#(2, Wire#(Bit#(2))) lockwires <- replicateM(mkDWire(0));
   Vector#(2, Wire#(Bit#(3))) protwires <- replicateM(mkDWire(0));
   Vector#(2, Wire#(Bit#(4))) qoswires <- replicateM(mkDWire(0));
   Vector#(3, Wire#(Bit#(1))) validwires <- replicateM(mkDWire(0));
   Vector#(2, Wire#(Bit#(2))) sizewires <- replicateM(mkDWire(0));
   Vector#(3, Wire#(Bit#(1))) readywires <- replicateM(mkDWire(0));
   Vector#(2, Wire#(Bit#(5))) userwires <- replicateM(mkDWire(0));
   Vector#(1, Wire#(Bit#(1))) lastwires <- replicateM(mkDWire(0));
   interface Wire araddr = addrwires[0];
   interface Wire arburst = burstwires[0];
   interface Wire arcache = cachewires[0];
   interface Wire arid = idwires[0];
   interface Wire arlen = lenwires[0];
   interface Wire arlock = lockwires[0];
   interface Wire arprot = protwires[0];
   interface Wire arqos = qoswires[0];
   interface Wire arsize = sizewires[0];
   interface Wire arvalid = validwires[0];

   interface Wire awaddr = addrwires[1];
   interface Wire awburst = burstwires[1];
   interface Wire awcache = cachewires[1];
   interface Wire awid = idwires[1];
   interface Wire awlen = lenwires[1];
   interface Wire awlock = lockwires[1];
   interface Wire awprot = protwires[1];
   interface Wire awqos = qoswires[1];
   interface Wire awsize = sizewires[1];
   interface Wire awvalid = validwires[1];

   interface Wire rready = readywires[0];
   interface Wire wid = idwires[2];
   interface Wire wstrb = strbwires[0];
   interface Wire wdata = datawires[0];
   interface Wire wlast = lastwires[0];
   interface Wire wvalid = validwires[2];
   interface Wire bready = readywires[1];
endmodule

module mkAxi3MasterGather#(AxiMasterBits#(addrWidth, dataWidth, tagWidth, Empty) axiWires)(AxiMasterCommon#(addrWidth, dataWidth, tagWidth));
//provisos(Div#(dataWidth, 8, 4));
   AxiMasterWires#(addrWidth, dataWidth, tagWidth) vtopmw <- mkAxiMasterWires;
   rule handshake1;
        axiWires.arready(vtopmw.arready);
   endrule
   rule handshake2;
        axiWires.awready(vtopmw.awready);
   endrule
   rule handshake3;
        axiWires.rid(vtopmw.rid);
        axiWires.rresp(vtopmw.rresp);
        axiWires.rdata(vtopmw.rdata);
        axiWires.rlast(vtopmw.rlast);
        axiWires.rvalid(vtopmw.rvalid);
   endrule
   rule handshake4;
        axiWires.wready(vtopmw.wready);
   endrule
   rule handshake5;
        axiWires.bvalid(vtopmw.bvalid);
        axiWires.bid(vtopmw.bid);
        axiWires.bresp(vtopmw.bresp);
   endrule

   interface Axi3Master client;
        interface Get req_ar;
             method ActionValue#(Axi3ReadRequest#(addrWidth,tagWidth)) get() if (axiWires.arvalid() != 0);
                 Axi3ReadRequest#(addrWidth,tagWidth) v;
                 v.address = axiWires.araddr();
                 v.burst = axiWires.arburst();
                 v.cache = axiWires.arcache();
                 v.id = axiWires.arid();
                 v.len = axiWires.arlen();
                 v.lock = axiWires.arlock();
                 v.prot = axiWires.arprot();
                 v.qos = axiWires.arqos();
                 v.size = {0, axiWires.arsize()};

                vtopmw.arready <= 1;
                return v;
            endmethod
        endinterface
        interface Get req_aw;
            method ActionValue#(Axi3WriteRequest#(addrWidth,tagWidth)) get() if (axiWires.awvalid() != 0);
                Axi3WriteRequest#(addrWidth,tagWidth) v;
                v.address = axiWires.awaddr();
                v.burst = axiWires.awburst();
                v.cache = axiWires.awcache();
                v.id = axiWires.awid();
                v.len = axiWires.awlen();
                v.lock = axiWires.awlock();
                v.prot = axiWires.awprot();
                v.qos = axiWires.awqos();
                v.size = {0, axiWires.awsize()};

                vtopmw.awready <= 1;
                return v;
           endmethod
        endinterface
        interface Put resp_read;
            method Action put(Axi3ReadResponse#(dataWidth, tagWidth) v) if (axiWires.rready() != 0);
                vtopmw.rid <= v.id;
                vtopmw.rresp <= v.resp;
                vtopmw.rdata <= v.data;
                vtopmw.rlast <= v.last;
                vtopmw.rvalid <= 1;
            endmethod
        endinterface
        interface Get resp_write;
            method ActionValue#(Axi3WriteData#(dataWidth,tagWidth)) get() if (axiWires.wvalid() != 0);
                Axi3WriteData#(dataWidth,tagWidth) v;
                v.id = axiWires.wid();
                v.byteEnable = axiWires.wstrb();
                v.data = axiWires.wdata();
                v.last = axiWires.wlast();

                vtopmw.wready <= 1;
                return v;
            endmethod
        endinterface
        interface Put resp_b;
            method Action put(Axi3WriteResponse#(tagWidth) v) if (axiWires.bready() != 0);
                vtopmw.bvalid <= 1;
                vtopmw.bid    <= v.id;
                vtopmw.bresp  <= v.resp;
            endmethod
        endinterface
    endinterface
    method aresetn = axiWires.aresetn;
endmodule

module mkAxi3SlaveGather#(AxiSlaveBits#(addrWidth, dataWidth, tagWidth,
    extraType) axiWires)(AxiSlaveCommon#(addrWidth, dataWidth,tagWidth,extraType));
    AxiSlaveWires#(addrWidth,dataWidth,tagWidth) vtopsw <- mkAxiSlaveWires;
    rule handshake1;
          axiWires.araddr(vtopsw.araddr);
          axiWires.arburst(vtopsw.arburst);
          axiWires.arcache(vtopsw.arcache);
          axiWires.arid(vtopsw.arid);
          axiWires.arlen(vtopsw.arlen);
          axiWires.arlock(vtopsw.arlock);
          axiWires.arprot(vtopsw.arprot);
          axiWires.arqos(vtopsw.arqos);
          axiWires.arsize(vtopsw.arsize);
          axiWires.arvalid(vtopsw.arvalid);
    endrule
    rule handshake2;
          axiWires.awaddr(vtopsw.awaddr);
          axiWires.awburst(vtopsw.awburst);
          axiWires.awcache(vtopsw.awcache);
          axiWires.awid(vtopsw.awid);
          axiWires.awlen(vtopsw.awlen);
          axiWires.awlock(vtopsw.awlock);
          axiWires.awprot(vtopsw.awprot);
          axiWires.awqos(vtopsw.awqos);
          axiWires.awsize(vtopsw.awsize);
          axiWires.awvalid(vtopsw.awvalid);
    endrule
    rule handshake3;
         axiWires.rready(vtopsw.rready);
    endrule
    rule handshake4;
          axiWires.wid(vtopsw.wid);
          axiWires.wstrb(vtopsw.wstrb);
          axiWires.wdata(vtopsw.wdata);
          axiWires.wlast(vtopsw.wlast);
          axiWires.wvalid(vtopsw.wvalid);
    endrule
    rule handshake5;
         axiWires.bready(vtopsw.bready);
    endrule
    interface Axi3Slave server;
    interface Put req_ar;
        method Action put(Axi3ReadRequest#(addrWidth,tagWidth) v) if (axiWires.arready() != 0);
           vtopsw.araddr <= v.address;
           vtopsw.arburst <= v.burst;
           vtopsw.arcache <= v.cache;
           vtopsw.arid <= v.id;
           vtopsw.arlen <= v.len;
           vtopsw.arlock <= v.lock;
           vtopsw.arprot <= v.prot;
           vtopsw.arqos <= v.qos;
           vtopsw.arsize <= v.size[1:0];

           vtopsw.arvalid <= 1;
        endmethod
    endinterface
    interface Put req_aw;
        method Action put(Axi3WriteRequest#(addrWidth,tagWidth) v) if (axiWires.awready() != 0);
           vtopsw.awaddr <= v.address;
           vtopsw.awburst <= v.burst;
           vtopsw.awcache <= v.cache;
           vtopsw.awid <= v.id;
           vtopsw.awlen <= v.len;
           vtopsw.awlock <= v.lock;
           vtopsw.awprot <= v.prot;
           vtopsw.awqos <= v.qos;
           vtopsw.awsize <= v.size[1:0];

           vtopsw.awvalid <= 1;
        endmethod
    endinterface
    interface Put resp_write;
        method Action put(Axi3WriteData#(dataWidth,tagWidth) v) if (axiWires.wready() != 0);
           vtopsw.wid <= v.id;
           vtopsw.wstrb <= v.byteEnable;
           vtopsw.wdata <= v.data;
           vtopsw.wlast <= v.last;

           vtopsw.wvalid <= 1;
        endmethod
    endinterface
    interface Get resp_read;
        method ActionValue#(Axi3ReadResponse#(dataWidth, tagWidth)) get() if (axiWires.rvalid() != 0);
            Axi3ReadResponse#(dataWidth, tagWidth) v;
            v.id = axiWires.rid();
            v.resp = axiWires.rresp();
            v.data = axiWires.rdata();
            v.last = axiWires.rlast();

            vtopsw.rready <= 1;
            return v;
        endmethod
         endinterface
    interface Get resp_b;
        method ActionValue#(Axi3WriteResponse#(tagWidth)) get() if (axiWires.bvalid() != 0);
            Axi3WriteResponse#(tagWidth) v;
            v.id = axiWires.bid();
            v.resp = axiWires.bresp();

            vtopsw.bready <= 1;
            return v;
        endmethod
     endinterface
     endinterface
     interface extra = axiWires.extra;
     method aresetn = axiWires.aresetn;
endmodule


fixing DMA-to-fpga not working for sudden dta length

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import FIFO         :: *;
import FIFOF        :: *;
import GetPut       :: *;
import Connectable  :: *;
import PCIE         :: *;
import DefaultValue :: *;
import Vector       :: *;
import ClientServer :: *;
import MIMO         :: *;
import Probe        :: *;

import ConnectalMimo :: *;
import MIFO         :: *;
import MemTypes     :: *;
import ConfigCounter ::*;
import ConnectalConfig::*;

`include "ConnectalProjectConfig.bsv"

typedef struct {
   TLPData#(TlpDataBytes) tlp;
   TLPLength    dwCount;
   Bool         is3dw;
   Bool         isHeaderOnly;
   } TlpWriteHeaderInfo deriving (Bits);

interface MemToPcie#(numeric type buswidth);
    interface Client#(TLPData#(TlpDataBytes), TLPData#(TlpDataBytes)) tlp;
    interface PhysMemSlave#(40,buswidth) slave;
    method Bool tlpOutFifoNotEmpty();
    interface Reg#(Bool) use4dw;
endinterface: MemToPcie

`ifdef XILINX
   `define AXI
`elsif SIMULATION
   `define AXI
`elsif ALTERA
   `define AVALON
`endif

`ifdef PCIE3
typedef 1024 WriteDataBurstLen; // max payload size is 1024 bytes
`else
typedef 256 WriteDataBurstLen; // max payload size is 256 bytes for Xilinx gen2 core
`endif
typedef TDiv#(WriteDataBurstLen,4) WriteDataMimoSize; // number of words to hold in the MIMO
typedef BurstLenSize WriteDataBurstLenSize;
module mkMemToPcie#(PciId my_id)(MemToPcie#(buswidth))
   provisos (Div#(buswidth, 8, busWidthBytes),
	     Div#(buswidth, 32, busWidthWords),
	     Bits#(Vector#(busWidthWords, Bit#(32)), buswidth),
	     Log#(busWidthBytes,beatShift),
	     Add#(aaa, 32, buswidth),
	     Add#(bbb, buswidth, 256),
	     Add#(ccc, TMul#(8, busWidthWords), 64),
	     Add#(ddd, TMul#(32, busWidthWords), 256),
	     Add#(eee, busWidthWords, 8),
	     Add#(1, a__, busWidthWords),
	     Add#(b__, busWidthWords, TlpDataWords),
	     Add#(c__, busWidthWords, WriteDataMimoSize),
	     Add#(0,16,TlpDataBytes)
	     );

    let beat_shift = fromInteger(valueOf(beatShift));

    FIFOF#(TLPData#(TlpDataBytes)) tlpOutFifo <- mkFIFOF;
    FIFOF#(TLPData#(TlpDataBytes)) tlpInFifo <- mkFIFOF;
    FIFOF#(TlpWriteHeaderInfo) tlpWriteHeaderFifo <- mkFIFOF;

    Reg#(Bit#(7)) hitReg <- mkReg(0);
    Reg#(Bool) use4dwReg <- mkReg(True);

    // default configuration for MIMO is for guarded enq() and deq().
    // However, the implicit guard only checks for space for 1 element for enq(), and availability of 1 element for deq().
    MIMOConfiguration mimoCfg = defaultValue;
   MIFO#(TlpDataWords,busWidthWords,16,Bit#(32)) completionMimo <- mkMIFO();
   MIFO#(TlpDataWords,busWidthWords,16,Tuple2#(TLPTag,Bool)) completionTagMimo <- mkMIFO(); // tag, last beat of burst

   mimoCfg.bram_based = True;
   mimoCfg.unguarded = True;
    MIMO#(busWidthWords,TlpDataWords,WriteDataMimoSize,Bit#(32)) writeDataMimo <- ConnectalMimo::mkMIMOBram(mimoCfg);
    ConfigCounter#(8) writeDataCnt <- mkConfigCounter(0);
    Reg#(Bit#(WriteDataBurstLenSize)) writeBurstCount <- mkReg(0);
    FIFO#(Bit#(WriteDataBurstLenSize)) writeBurstCountFifo <- mkFIFO();
    Reg#(TLPLength)  writeDwCount <- mkReg(0); // how many 4 byte (double) words to send
    Reg#(LUInt#(TlpDataWords)) tlpDwCount <- mkReg(0); // how many to send in the next tlp (at most 4)
    Reg#(Bool)            lastTlp <- mkReg(False); // if the next tlp sent is the last one
    Reg#(Bool)    writeInProgress <- mkReg(False);
    FIFOF#(TLPTag) writeTag <- mkSizedFIFOF(16);
    FIFOF#(TLPTag) doneTag <- mkSizedFIFOF(16);

    Reg#(Bool) quadAlignedTlpHandled <- mkReg(True);

   Wire#(Bool) writeHeaderTlpWire <- mkDWire(False);
   Wire#(Bool) writeDataMimoEnqWire <- mkDWire(False);

   Reg#(Bool) writeDataCntAboveThreshold <- mkReg(False);
   Reg#(Bool) writeDataMimoHasRoom <- mkReg(False);
   rule updateWriteDataMimoHasRoom;
      writeDataMimoHasRoom <= (writeDataCnt.read <= fromInteger(valueOf(WriteDataMimoSize) - valueOf(busWidthWords)));
   endrule
   rule noheader if (!tlpWriteHeaderFifo.notEmpty());
      writeDataCntAboveThreshold <= False;
   endrule
   rule updateWriteDataCntAboveThreshold if (writeInProgress || !writeDataCntAboveThreshold);
      writeDataCntAboveThreshold <= (writeDataCnt.read >= truncate(unpack(tlpWriteHeaderFifo.first.dwCount)));
   endrule
   (* descending_urgency = "writeHeaderTlp,updateWriteDataCntAboveThreshold" *)
   rule writeHeaderTlp if (!writeInProgress && writeDataCntAboveThreshold);
      // update for next cycle
      writeDataCntAboveThreshold <= (writeDataCnt.read >= truncate(unpack(tlpWriteHeaderFifo.first.dwCount)));

      writeHeaderTlpWire <= True;
      let info <- toGet(tlpWriteHeaderFifo).get();
      let tlp     = info.tlp;
      let dwCount = info.dwCount;
      let is3dw   = info.is3dw;
      let isHeaderOnly = info.isHeaderOnly;

      TLPMemory4DWHeader hdr_4dw = unpack(truncate(tlp.data));

      TLPMemoryIO3DWHeader hdr_3dw = unpack(truncate(tlp.data));
      if (is3dw) begin
	 if (hdr_3dw.format != MEM_WRITE_3DW_DATA)
	    $display("MemToPcie: expecting MEM_WRITE_3DW_DATA, got %d", hdr_3dw.format);
	 Vector#(TlpDataWords, Bit#(32)) v = unpack(0);
         tlp.sof = True;
`ifdef AXI
         v = writeDataMimo.first();
	 writeDataMimo.deq(1);
	 hdr_3dw.data = byteSwap(v[0]);
         tlp.eof = (dwCount == 1) ? True : False;
         dwCount = dwCount - 1;
         writeDataCnt.decrement(1);
`elsif AVALON
         let quadWordAligned = isQuadWordAligned(getLowerAddr(hdr_3dw.addr, hdr_3dw.firstbe));
         // if quad-word aligned, insert bubble.
         if (!quadWordAligned) begin
            v = writeDataMimo.first();
            writeDataMimo.deq(1);
            hdr_3dw.data = v[0];
         end
         else begin
            hdr_3dw.data = unpack(0);
         end
         tlp.eof = (dwCount == 1 && !quadWordAligned) ? True : False;
         if (!quadWordAligned) begin
            dwCount = dwCount - 1;
            writeDataCnt.decrement(1);
         end
`endif
	 tlp.be = 'hffff;
	 tlp.data = extend(pack(hdr_3dw));
      end
      else begin
         quadAlignedTlpHandled <= isQuadWordAligned(getLowerAddr(truncate(unpack(hdr_4dw.addr)), hdr_4dw.firstbe));
	 tlp.be = 'hffff;
      end

      tlpOutFifo.enq(tlp);
      $display("writeHeaderTlp dwCount=%d", dwCount);
      writeDwCount <= dwCount;
      tlpDwCount <= truncate(min(fromInteger(valueOf(TlpDataWords)),unpack(dwCount)));
      lastTlp <= (dwCount <= fromInteger(valueOf(TlpDataWords)));
      writeInProgress <= (dwCount != 0);
      if (isHeaderOnly) begin
	 doneTag.enq(writeTag.first());
	 writeTag.deq();
      end

   endrule

   rule writeTlps if (writeInProgress); // already verified  writeDataMimo.deqReadyN(tlpDwCount)) for this transaction
      TLPData#(TlpDataBytes) tlp = defaultValue;
      tlp.sof = False;
      Vector#(TlpDataWords, Bit#(32)) v = unpack(0);
      Integer currDwCount = valueOf(TlpDataWords);

`ifdef AVALON
      if (!quadAlignedTlpHandled) begin
         currDwCount = 3; // 3 Data Dwords in this cycle.
         quadAlignedTlpHandled <= True;
      end
`endif

      // The MIMO implicit guard only checks for availability of 1 element
      // so we explicitly check for the number of elements required
      writeDataMimo.deq(tlpDwCount);
      v = writeDataMimo.first();
      let dwCount = writeDwCount - extend(pack(tlpDwCount));
      writeDwCount <= dwCount;
      tlpDwCount <= truncate(min(fromInteger(currDwCount),unpack(dwCount)));
      writeDataCnt.decrement(unpack(extend(pack(tlpDwCount))));
      lastTlp <= (dwCount <= fromInteger(valueOf(TlpDataWords)));
      tlp.be = maxBound << (4*(fromInteger(valueOf(TlpDataWords))-tlpDwCount));
      tlp.eof = lastTlp;
      if (lastTlp) begin
	 writeInProgress <= False;
	 doneTag.enq(writeTag.first());
	 writeTag.deq();
	 $display("writeDwCount=%d will be zero", writeDwCount);
      end

      for (Integer i = 0; i < currDwCount; i = i + 1) begin
`ifdef AXI
`ifdef PCIE3
	 tlp.data[(i+1)*32-1:i*32] = v[i];
`else
	 tlp.data[(i+1)*32-1:i*32] = byteSwap(v[(currDwCount-1)-i]);
`endif
`elsif AVALON
	 tlp.data[(i+1)*32-1:i*32] = v[(currDwCount-1)-i];
`endif
      end

      tlpOutFifo.enq(tlp);
   endrule: writeTlps

   Reg#(TLPTag) lastTag <- mkReg(0);
   FIFOF#(TLPData#(TlpDataBytes)) tlpDecodeFifo <- mkFIFOF();
   Reg#(TLPLength) wordCountReg <- mkReg(0);
   rule tlpInRule;
      let tlp <- toGet(tlpInFifo).get();
      tlpDecodeFifo.enq(tlp);
   endrule

   rule handleTlpRule;
      let tlp = tlpDecodeFifo.first;
      Bool handled = False;
      TLPMemoryIO3DWHeader h = unpack(truncate(tlp.data));
      hitReg <= tlp.hit;
      TLPMemoryIO3DWHeader hdr_3dw = unpack(truncate(tlp.data));
      TLPCompletionHeader hdr_completion = unpack(truncate(tlp.data));
      Vector#(TlpDataWords, Bit#(32)) vec = unpack(0);
      Vector#(TlpDataWords, Bit#(32)) tlpvec = unpack(tlp.data);
      let wordCount = wordCountReg;
`ifdef AXI
      let dataInSecondTlp = False;
`elsif AVALON
      let quadWordAligned = isQuadWordAligned(getLowerAddr(hdr_3dw.addr, hdr_3dw.firstbe));
      let dataInSecondTlp = quadWordAligned;
`endif
      if (!tlp.sof) begin
`ifdef PCIE3
	 vec = tlpvec;
`else
	 vec = reverse(tlpvec);
`endif
	 // The MIMO implicit guard only checks for space to enqueue 1 element
	 // so we explicitly check for the number of elements required
	 // otherwise elements in the queue will be overwritten.
	 if (completionMimo.enqReady()
	    && completionTagMimo.enqReady())
	    begin
	       LUInt#(TlpDataWords) count = fromInteger(valueOf(TlpDataWords));
	       if (tlp.eof) begin
		  count = truncate(unpack(wordCountReg));
	       end
	       wordCount = wordCountReg - extend(pack(count));
	       completionMimo.enq(count, vec);
	       function Tuple2#(TLPTag,Bool) taglast(Integer i); return tuple2(lastTag, (fromInteger(i) == (count-1)) ? tlp.eof : False); endfunction
	       Vector#(TlpDataWords, Tuple2#(TLPTag,Bool)) tagvec = genWith(taglast);
	       completionTagMimo.enq(count, tagvec);
	       handled = True;
	    end
      end
      else if (hdr_3dw.format == MEM_WRITE_3DW_DATA
	       && hdr_3dw.pkttype == COMPLETION
	       && completionMimo.enqReady()
	       && completionTagMimo.enqReady()) begin
            TLPTag tag = hdr_completion.tag;
            lastTag <= tag;
            if (!dataInSecondTlp) begin
               vec[0] = hdr_3dw.data;
               wordCount = hdr_3dw.length - 1;
               completionMimo.enq(1, vec);
               completionTagMimo.enq(1, replicate(tuple2(tag,tlp.eof)));
            end
            else begin
               wordCount = hdr_3dw.length;
            end
	    handled = True;
      end
      wordCountReg <= wordCount;
      $display("tlpIn handled=%d tlp=%h\n", handled, tlp);
      if (handled) begin
	 tlpDecodeFifo.deq();
      end
   endrule

   FIFO#(PhysMemRequest#(40,buswidth)) readReqFifo <- mkFIFO();
   rule readReqRule if (!writeInProgress); // && !writeDataMimo.deqReady());
      let req <- toGet(readReqFifo).get();
      let burstLen = req.burstLen >> beat_shift;
      let addr = req.addr;
      let arid = req.tag;

      TLPData#(TlpDataBytes) tlp = defaultValue;
      tlp.sof = True;
      tlp.eof = True;
      tlp.hit = 7'h00;
      TLPLength tlplen = fromInteger(valueOf(busWidthWords))*truncate(burstLen);
      if (addr[39:32] != 0) begin
	 TLPMemory4DWHeader hdr_4dw = defaultValue;
	 hdr_4dw.format = MEM_READ_4DW_NO_DATA;
	 hdr_4dw.tag = extend(arid);
	 hdr_4dw.reqid = my_id;
	 hdr_4dw.nosnoop = SNOOPING_REQD;
	 hdr_4dw.addr = addr[40-1:2];
	 hdr_4dw.length = tlplen;
	 Bit#(TDiv#(buswidth,8)) firstbe = reqFirstByteEnable(req);
	 Bit#(TDiv#(buswidth,8)) lastbe = reqLastByteEnable(req);
	 hdr_4dw.firstbe = firstbe[3:0];
	 hdr_4dw.lastbe = (tlplen > 1) ? lastbe[valueOf(busWidthBytes)-1:valueOf(busWidthBytes)-4] : 0;
	 tlp.data = extend(pack(hdr_4dw));
	 tlp.be = 'hffff;
      end
      else begin
	 TLPMemoryIO3DWHeader hdr_3dw = defaultValue;
	 hdr_3dw.format = MEM_READ_3DW_NO_DATA;
	 hdr_3dw.tag = extend(arid);
	 hdr_3dw.reqid = my_id;
	 hdr_3dw.nosnoop = SNOOPING_REQD;
	 hdr_3dw.addr = addr[32-1:2];
	 hdr_3dw.length = tlplen;
	 Bit#(TDiv#(buswidth,8)) firstbe = reqFirstByteEnable(req);
	 Bit#(TDiv#(buswidth,8)) lastbe = reqLastByteEnable(req);
	 hdr_3dw.firstbe = firstbe[3:0];
	 hdr_3dw.lastbe = (tlplen > 1) ? lastbe[valueOf(busWidthBytes)-1:valueOf(busWidthBytes)-4] : 0;
	 tlp.data = extend(pack(hdr_3dw));
	 tlp.be = 'hfff0;
      end
      tlpOutFifo.enq(tlp);
   endrule

    interface Client        tlp;
        interface request = toGet(tlpOutFifo);
        interface response = toPut(tlpInFifo);
    endinterface
    interface PhysMemSlave slave;
   interface PhysMemWriteServer write_server;
      interface Put writeReq;
         method Action put(PhysMemRequest#(40,buswidth) req); // if (writeBurstCount == 0);
	    Bit#(WriteDataBurstLenSize) burstLen = truncate(req.burstLen >> beat_shift);
	    let addr = req.addr;
	    let awid = req.tag;
	    let writeIs3dw = False;
	    let use3dw = True;
`ifdef PCIE3
	    awid = awid | (1 << (valueOf(MemTagSize)-1));
	    use3dw = False;
`endif
	    TLPLength tlplen = fromInteger(valueOf(busWidthWords))*truncate(burstLen);
	    TLPData#(TlpDataBytes) tlp = defaultValue;
	    tlp.sof = True;
	    tlp.eof = False;
	    tlp.hit = 7'h00;
	    tlp.be = 'hffff;

	    $display("slave.writeAddr tlplen=%d burstLen=%d", tlplen, burstLen);
	    if ((addr >> 32) != 0 || !use3dw) begin
	       TLPMemory4DWHeader hdr_4dw = defaultValue;
	       hdr_4dw.format = MEM_WRITE_4DW_DATA;
	       hdr_4dw.tag = extend(awid);
	       hdr_4dw.reqid = my_id;
	       hdr_4dw.nosnoop = SNOOPING_REQD;
	       hdr_4dw.addr = addr[40-1:2];
	       hdr_4dw.length = tlplen;
	       Bit#(TDiv#(buswidth,8)) firstbe = reqFirstByteEnable(req);
	       Bit#(TDiv#(buswidth,8)) lastbe = reqLastByteEnable(req);
	       hdr_4dw.firstbe = firstbe[3:0];
	       hdr_4dw.lastbe = (tlplen > 1) ? lastbe[valueOf(busWidthBytes)-1:valueOf(busWidthBytes)-4] : 0;
	       tlp.data = extend(pack(hdr_4dw));
	    end
	    else begin
	       writeIs3dw = True;
	       TLPMemoryIO3DWHeader hdr_3dw = defaultValue;
	       hdr_3dw.format = MEM_WRITE_3DW_DATA;
	       hdr_3dw.tag = extend(awid);
	       hdr_3dw.reqid = my_id;
	       hdr_3dw.nosnoop = SNOOPING_REQD;
	       hdr_3dw.addr = addr[32-1:2];
	       hdr_3dw.length = tlplen;
	       Bit#(TDiv#(buswidth,8)) firstbe = reqFirstByteEnable(req);
	       Bit#(TDiv#(buswidth,8)) lastbe = reqLastByteEnable(req);
	       hdr_3dw.firstbe = firstbe[3:0];
	       hdr_3dw.lastbe = (tlplen > 1) ? lastbe[valueOf(busWidthBytes)-1:valueOf(busWidthBytes)-4] : 0;
	       tlp.be = 'hfff0; // no data word in this TLP

	       tlp.data = extend(pack(hdr_3dw));
	    end
	    tlpWriteHeaderFifo.enq(TlpWriteHeaderInfo {tlp: tlp, dwCount: tlplen, is3dw: writeIs3dw, isHeaderOnly: (writeIs3dw && tlplen == 1) });
	    writeBurstCountFifo.enq(burstLen);
	    writeTag.enq(extend(awid));
         endmethod
	endinterface
      interface Put writeData;
         method Action put(MemData#(buswidth) wdata)
	    provisos (Bits#(Vector#(busWidthWords, Bit#(32)), busWidth)) if (writeDataMimoHasRoom); //.enqReadyN(fromInteger(valueOf(busWidthWords))));

	    let burstLen = writeBurstCount;
	    if (burstLen == 0) begin
	       burstLen <- toGet(writeBurstCountFifo).get();
	    end
	    writeBurstCount <= extend(burstLen-1);

	    Vector#(busWidthWords, Bit#(32)) v = unpack(wdata.data);
	    writeDataMimo.enq(fromInteger(valueOf(busWidthWords)), v);
            writeDataCnt.increment(fromInteger(valueOf(busWidthWords)));
	    writeDataMimoEnqWire <= True;
         endmethod
       endinterface
      interface Get writeDone;
         method ActionValue#(Bit#(MemTagSize)) get();
	      let tag = doneTag.first();
	      doneTag.deq();
	      return truncate(tag);
           endmethod
	endinterface
   endinterface
   interface PhysMemReadServer read_server;
      interface Put readReq;
         method Action put(PhysMemRequest#(40,buswidth) req);
	    readReqFifo.enq(req);
         endmethod
       endinterface
      interface Get     readData;
         method ActionValue#(MemData#(buswidth)) get() if (completionMimo.deqReady()
							   && completionTagMimo.deqReady());
	      let data_v = completionMimo.first;
	      let tag_last_v = completionTagMimo.first;
	      match { .tag, .last } = tag_last_v[fromInteger(valueOf(busWidthWords))-1];
	      completionMimo.deq();
	      completionTagMimo.deq();
              Bit#(buswidth) v = 0;
              for (Integer i = 0; i < valueOf(busWidthWords); i = i+1) begin
`ifdef AXI
`ifdef PCIE3
		 v[(i+1)*32-1:i*32] = data_v[i];
`else
		 v[(i+1)*32-1:i*32] = byteSwap(data_v[i]);
`endif
`elsif AVALON
		 v[(i+1)*32-1:i*32] = data_v[i];
`endif
              end
	      return MemData { data: v, tag: truncate(tag), last: last}; // last beat of this response burst
           endmethod
	endinterface
   endinterface
    endinterface: slave
   method Bool tlpOutFifoNotEmpty() = tlpOutFifo.notEmpty;
   interface Reg use4dw = use4dwReg;
endmodule: mkMemToPcie

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import FIFO         :: *;
import FIFOF        :: *;
import GetPut       :: *;
import Connectable  :: *;
import PCIE         :: *;
import DefaultValue :: *;
import Vector       :: *;
import ClientServer :: *;
import MIMO         :: *;
import Probe        :: *;

import ConnectalMimo :: *;
import MIFO         :: *;
import MemTypes     :: *;
import ConfigCounter ::*;
import ConnectalConfig::*;

`include "ConnectalProjectConfig.bsv"

typedef struct {
   TLPData#(TlpDataBytes) tlp;
   TLPLength    dwCount;
   Bool         is3dw;
   Bool         isHeaderOnly;
   } TlpWriteHeaderInfo deriving (Bits);

interface MemToPcie#(numeric type buswidth);
    interface Client#(TLPData#(TlpDataBytes), TLPData#(TlpDataBytes)) tlp;
    interface PhysMemSlave#(40,buswidth) slave;
    method Bool tlpOutFifoNotEmpty();
    interface Reg#(Bool) use4dw;
endinterface: MemToPcie

`ifdef XILINX
   `define AXI
`elsif SIMULATION
   `define AXI
`elsif ALTERA
   `define AVALON
`endif

`ifdef PCIE3
typedef 1024 WriteDataBurstLen; // max payload size is 1024 bytes
`else
typedef 256 WriteDataBurstLen; // max payload size is 256 bytes for Xilinx gen2 core
`endif
typedef TDiv#(WriteDataBurstLen,4) WriteDataMimoSize; // number of words to hold in the MIMO
typedef BurstLenSize WriteDataBurstLenSize;
module mkMemToPcie#(PciId my_id)(MemToPcie#(buswidth))
   provisos (Div#(buswidth, 8, busWidthBytes),
	     Div#(buswidth, 32, busWidthWords),
	     Bits#(Vector#(busWidthWords, Bit#(32)), buswidth),
	     Log#(busWidthBytes,beatShift),
	     Add#(aaa, 32, buswidth),
	     Add#(bbb, buswidth, 256),
	     Add#(ccc, TMul#(8, busWidthWords), 64),
	     Add#(ddd, TMul#(32, busWidthWords), 256),
	     Add#(eee, busWidthWords, 8),
	     Add#(1, a__, busWidthWords),
	     Add#(b__, busWidthWords, TlpDataWords),
	     Add#(c__, busWidthWords, WriteDataMimoSize),
	     Add#(0,16,TlpDataBytes)
	     );

    let beat_shift = fromInteger(valueOf(beatShift));

    FIFOF#(TLPData#(TlpDataBytes)) tlpOutFifo <- mkFIFOF;
    FIFOF#(TLPData#(TlpDataBytes)) tlpInFifo <- mkFIFOF;
    FIFOF#(TlpWriteHeaderInfo) tlpWriteHeaderFifo <- mkFIFOF;

    Reg#(Bit#(7)) hitReg <- mkReg(0);
    Reg#(Bool) use4dwReg <- mkReg(True);

    // default configuration for MIMO is for guarded enq() and deq().
    // However, the implicit guard only checks for space for 1 element for enq(), and availability of 1 element for deq().
    MIMOConfiguration mimoCfg = defaultValue;
   MIFO#(TlpDataWords,busWidthWords,16,Bit#(32)) completionMimo <- mkMIFO();
   MIFO#(TlpDataWords,busWidthWords,16,Tuple2#(TLPTag,Bool)) completionTagMimo <- mkMIFO(); // tag, last beat of burst

   mimoCfg.bram_based = True;
   mimoCfg.unguarded = True;
    MIMO#(busWidthWords,TlpDataWords,WriteDataMimoSize,Bit#(32)) writeDataMimo <- ConnectalMimo::mkMIMOBram(mimoCfg);
    ConfigCounter#(8) writeDataCnt <- mkConfigCounter(0);
    Reg#(Bit#(WriteDataBurstLenSize)) writeBurstCount <- mkReg(0);
    FIFO#(Bit#(WriteDataBurstLenSize)) writeBurstCountFifo <- mkFIFO();
    Reg#(TLPLength)  writeDwCount <- mkReg(0); // how many 4 byte (double) words to send
    Reg#(LUInt#(TlpDataWords)) tlpDwCount <- mkReg(0); // how many to send in the next tlp (at most 4)
    Reg#(Bool)            lastTlp <- mkReg(False); // if the next tlp sent is the last one
    Reg#(Bool)    writeInProgress <- mkReg(False);
    FIFOF#(TLPTag) writeTag <- mkSizedFIFOF(16);
    FIFOF#(TLPTag) doneTag <- mkSizedFIFOF(16);

    Reg#(Bool) quadAlignedTlpHandled <- mkReg(True);

   Wire#(Bool) writeHeaderTlpWire <- mkDWire(False);
   Wire#(Bool) writeDataMimoEnqWire <- mkDWire(False);

   Reg#(Bool) writeDataCntAboveThreshold <- mkReg(False);
   Reg#(Bool) writeDataMimoHasRoom <- mkReg(False);
   rule updateWriteDataMimoHasRoom;
      writeDataMimoHasRoom <= (writeDataCnt.read <= fromInteger(valueOf(WriteDataMimoSize) - valueOf(busWidthWords)));
   endrule
   rule noheader if (!tlpWriteHeaderFifo.notEmpty());
      writeDataCntAboveThreshold <= False;
   endrule
   rule updateWriteDataCntAboveThreshold if (writeInProgress || !writeDataCntAboveThreshold);
      writeDataCntAboveThreshold <= (writeDataCnt.read >= truncate(unpack(tlpWriteHeaderFifo.first.dwCount)));
   endrule
   (* descending_urgency = "writeHeaderTlp,updateWriteDataCntAboveThreshold" *)
   rule writeHeaderTlp if (!writeInProgress && writeDataCntAboveThreshold);
      // update for next cycle
      writeDataCntAboveThreshold <= (writeDataCnt.read >= truncate(unpack(tlpWriteHeaderFifo.first.dwCount)));

      writeHeaderTlpWire <= True;
      let info <- toGet(tlpWriteHeaderFifo).get();
      let tlp     = info.tlp;
      let dwCount = info.dwCount;
      let is3dw   = info.is3dw;
      let isHeaderOnly = info.isHeaderOnly;

      TLPMemory4DWHeader hdr_4dw = unpack(truncate(tlp.data));

      TLPMemoryIO3DWHeader hdr_3dw = unpack(truncate(tlp.data));
      if (is3dw) begin
	 if (hdr_3dw.format != MEM_WRITE_3DW_DATA)
	    $display("MemToPcie: expecting MEM_WRITE_3DW_DATA, got %d", hdr_3dw.format);
	 Vector#(TlpDataWords, Bit#(32)) v = unpack(0);
         tlp.sof = True;
`ifdef AXI
         v = writeDataMimo.first();
	 writeDataMimo.deq(1);
	 hdr_3dw.data = byteSwap(v[0]);
         tlp.eof = (dwCount == 1) ? True : False;
         dwCount = dwCount - 1;
         writeDataCnt.decrement(1);
`elsif AVALON
         let quadWordAligned = isQuadWordAligned(getLowerAddr(hdr_3dw.addr, hdr_3dw.firstbe));
         // if quad-word aligned, insert bubble.
         if (!quadWordAligned) begin
            v = writeDataMimo.first();
            writeDataMimo.deq(1);
            hdr_3dw.data = v[0];
         end
         else begin
            hdr_3dw.data = unpack(0);
         end
         tlp.eof = (dwCount == 1 && !quadWordAligned) ? True : False;
         if (!quadWordAligned) begin
            dwCount = dwCount - 1;
            writeDataCnt.decrement(1);
         end
`endif
	 tlp.be = 'hffff;
	 tlp.data = extend(pack(hdr_3dw));
      end
      else begin
         quadAlignedTlpHandled <= isQuadWordAligned(getLowerAddr(truncate(unpack(hdr_4dw.addr)), hdr_4dw.firstbe));
	 tlp.be = 'hffff;
      end

      tlpOutFifo.enq(tlp);
      $display("writeHeaderTlp dwCount=%d", dwCount);
      writeDwCount <= dwCount;
      tlpDwCount <= truncate(min(fromInteger(valueOf(TlpDataWords)),unpack(dwCount)));
      lastTlp <= (dwCount <= fromInteger(valueOf(TlpDataWords)));
      writeInProgress <= (dwCount != 0);
      if (isHeaderOnly) begin
	 doneTag.enq(writeTag.first());
	 writeTag.deq();
      end

   endrule

   rule writeTlps if (writeInProgress); // already verified  writeDataMimo.deqReadyN(tlpDwCount)) for this transaction
      TLPData#(TlpDataBytes) tlp = defaultValue;
      tlp.sof = False;
      Vector#(TlpDataWords, Bit#(32)) v = unpack(0);
      Integer currDwCount = valueOf(TlpDataWords);

`ifdef AVALON
      if (!quadAlignedTlpHandled) begin
         currDwCount = 3; // 3 Data Dwords in this cycle.
         quadAlignedTlpHandled <= True;
      end
`endif

      // The MIMO implicit guard only checks for availability of 1 element
      // so we explicitly check for the number of elements required
      writeDataMimo.deq(tlpDwCount);
      v = writeDataMimo.first();
      let dwCount = writeDwCount - extend(pack(tlpDwCount));
      writeDwCount <= dwCount;
      tlpDwCount <= truncate(min(fromInteger(currDwCount),unpack(dwCount)));
      writeDataCnt.decrement(unpack(extend(pack(tlpDwCount))));
      lastTlp <= (dwCount <= fromInteger(valueOf(TlpDataWords)));
      tlp.be = maxBound << (4*(fromInteger(valueOf(TlpDataWords))-tlpDwCount));
      tlp.eof = lastTlp;
      if (lastTlp) begin
	 writeInProgress <= False;
	 doneTag.enq(writeTag.first());
	 writeTag.deq();
	 $display("writeDwCount=%d will be zero", writeDwCount);
      end

      for (Integer i = 0; i < currDwCount; i = i + 1) begin
`ifdef AXI
`ifdef PCIE3
	 tlp.data[(i+1)*32-1:i*32] = v[i];
`else
	 tlp.data[(i+1)*32-1:i*32] = byteSwap(v[(currDwCount-1)-i]);
`endif
`elsif AVALON
	 tlp.data[(i+1)*32-1:i*32] = v[(currDwCount-1)-i];
`endif
      end

      tlpOutFifo.enq(tlp);
   endrule: writeTlps

   Reg#(TLPTag) lastTag <- mkReg(0);
   FIFOF#(TLPData#(TlpDataBytes)) tlpDecodeFifo <- mkFIFOF();
   Reg#(TLPLength) wordCountReg <- mkReg(0);
   rule tlpInRule;
      let tlp <- toGet(tlpInFifo).get();
      tlpDecodeFifo.enq(tlp);
   endrule

   rule handleTlpRule;
      let tlp = tlpDecodeFifo.first;
      Bool handled = False;
      TLPMemoryIO3DWHeader h = unpack(truncate(tlp.data));
      hitReg <= tlp.hit;
      TLPMemoryIO3DWHeader hdr_3dw = unpack(truncate(tlp.data));
      TLPCompletionHeader hdr_completion = unpack(truncate(tlp.data));
      Vector#(TlpDataWords, Bit#(32)) vec = unpack(0);
      Vector#(TlpDataWords, Bit#(32)) tlpvec = unpack(tlp.data);
      let wordCount = wordCountReg;
`ifdef AXI
      let dataInSecondTlp = False;
`elsif AVALON
      let quadWordAligned = isQuadWordAligned(getLowerAddr(hdr_3dw.addr, hdr_3dw.firstbe));
      let dataInSecondTlp = quadWordAligned;
`endif
      if (!tlp.sof) begin
`ifdef PCIE3
	 vec = tlpvec;
`else
	 vec = reverse(tlpvec);
`endif
	 // The MIMO implicit guard only checks for space to enqueue 1 element
	 // so we explicitly check for the number of elements required
	 // otherwise elements in the queue will be overwritten.
	 if (completionMimo.enqReady()
	    && completionTagMimo.enqReady())
	    begin
	       LUInt#(TlpDataWords) count = fromInteger(valueOf(TlpDataWords));
	       if (tlp.eof) begin
		  count = truncate(unpack(wordCountReg));
	       end
	       wordCount = wordCountReg - extend(pack(count));
	       completionMimo.enq(count, vec);
	       function Tuple2#(TLPTag,Bool) taglast(Integer i); return tuple2(lastTag, (fromInteger(i) == (count-1)) ? tlp.eof : False); endfunction
	       Vector#(TlpDataWords, Tuple2#(TLPTag,Bool)) tagvec = genWith(taglast);
	       completionTagMimo.enq(count, tagvec);
	       handled = True;
	    end
      end
      else if (hdr_3dw.format == MEM_WRITE_3DW_DATA
	       && hdr_3dw.pkttype == COMPLETION
	       && completionMimo.enqReady()
	       && completionTagMimo.enqReady()) begin
            TLPTag tag = hdr_completion.tag;
            lastTag <= tag;
            if (!dataInSecondTlp) begin
               vec[0] = hdr_3dw.data;
               wordCount = hdr_3dw.length - 1;
               completionMimo.enq(1, vec);
               completionTagMimo.enq(1, replicate(tuple2(tag,tlp.eof)));
            end
	    handled = True;
      end
      wordCountReg <= wordCount;
      $display("tlpIn handled=%d tlp=%h\n", handled, tlp);
      if (handled) begin
	 tlpDecodeFifo.deq();
      end
   endrule

   FIFO#(PhysMemRequest#(40,buswidth)) readReqFifo <- mkFIFO();
   rule readReqRule if (!writeInProgress); // && !writeDataMimo.deqReady());
      let req <- toGet(readReqFifo).get();
      let burstLen = req.burstLen >> beat_shift;
      let addr = req.addr;
      let arid = req.tag;

      TLPData#(TlpDataBytes) tlp = defaultValue;
      tlp.sof = True;
      tlp.eof = True;
      tlp.hit = 7'h00;
      TLPLength tlplen = fromInteger(valueOf(busWidthWords))*truncate(burstLen);
      if (addr[39:32] != 0) begin
	 TLPMemory4DWHeader hdr_4dw = defaultValue;
	 hdr_4dw.format = MEM_READ_4DW_NO_DATA;
	 hdr_4dw.tag = extend(arid);
	 hdr_4dw.reqid = my_id;
	 hdr_4dw.nosnoop = SNOOPING_REQD;
	 hdr_4dw.addr = addr[40-1:2];
	 hdr_4dw.length = tlplen;
	 Bit#(TDiv#(buswidth,8)) firstbe = reqFirstByteEnable(req);
	 Bit#(TDiv#(buswidth,8)) lastbe = reqLastByteEnable(req);
	 hdr_4dw.firstbe = firstbe[3:0];
	 hdr_4dw.lastbe = (tlplen > 1) ? lastbe[valueOf(busWidthBytes)-1:valueOf(busWidthBytes)-4] : 0;
	 tlp.data = extend(pack(hdr_4dw));
	 tlp.be = 'hffff;
      end
      else begin
	 TLPMemoryIO3DWHeader hdr_3dw = defaultValue;
	 hdr_3dw.format = MEM_READ_3DW_NO_DATA;
	 hdr_3dw.tag = extend(arid);
	 hdr_3dw.reqid = my_id;
	 hdr_3dw.nosnoop = SNOOPING_REQD;
	 hdr_3dw.addr = addr[32-1:2];
	 hdr_3dw.length = tlplen;
	 Bit#(TDiv#(buswidth,8)) firstbe = reqFirstByteEnable(req);
	 Bit#(TDiv#(buswidth,8)) lastbe = reqLastByteEnable(req);
	 hdr_3dw.firstbe = firstbe[3:0];
	 hdr_3dw.lastbe = (tlplen > 1) ? lastbe[valueOf(busWidthBytes)-1:valueOf(busWidthBytes)-4] : 0;
	 tlp.data = extend(pack(hdr_3dw));
	 tlp.be = 'hfff0;
      end
      tlpOutFifo.enq(tlp);
   endrule

    interface Client        tlp;
        interface request = toGet(tlpOutFifo);
        interface response = toPut(tlpInFifo);
    endinterface
    interface PhysMemSlave slave;
   interface PhysMemWriteServer write_server;
      interface Put writeReq;
         method Action put(PhysMemRequest#(40,buswidth) req); // if (writeBurstCount == 0);
	    Bit#(WriteDataBurstLenSize) burstLen = truncate(req.burstLen >> beat_shift);
	    let addr = req.addr;
	    let awid = req.tag;
	    let writeIs3dw = False;
	    let use3dw = True;
`ifdef PCIE3
	    awid = awid | (1 << (valueOf(MemTagSize)-1));
	    use3dw = False;
`endif
	    TLPLength tlplen = fromInteger(valueOf(busWidthWords))*truncate(burstLen);
	    TLPData#(TlpDataBytes) tlp = defaultValue;
	    tlp.sof = True;
	    tlp.eof = False;
	    tlp.hit = 7'h00;
	    tlp.be = 'hffff;

	    $display("slave.writeAddr tlplen=%d burstLen=%d", tlplen, burstLen);
	    if ((addr >> 32) != 0 || !use3dw) begin
	       TLPMemory4DWHeader hdr_4dw = defaultValue;
	       hdr_4dw.format = MEM_WRITE_4DW_DATA;
	       hdr_4dw.tag = extend(awid);
	       hdr_4dw.reqid = my_id;
	       hdr_4dw.nosnoop = SNOOPING_REQD;
	       hdr_4dw.addr = addr[40-1:2];
	       hdr_4dw.length = tlplen;
	       Bit#(TDiv#(buswidth,8)) firstbe = reqFirstByteEnable(req);
	       Bit#(TDiv#(buswidth,8)) lastbe = reqLastByteEnable(req);
	       hdr_4dw.firstbe = firstbe[3:0];
	       hdr_4dw.lastbe = (tlplen > 1) ? lastbe[valueOf(busWidthBytes)-1:valueOf(busWidthBytes)-4] : 0;
	       tlp.data = extend(pack(hdr_4dw));
	    end
	    else begin
	       writeIs3dw = True;
	       TLPMemoryIO3DWHeader hdr_3dw = defaultValue;
	       hdr_3dw.format = MEM_WRITE_3DW_DATA;
	       hdr_3dw.tag = extend(awid);
	       hdr_3dw.reqid = my_id;
	       hdr_3dw.nosnoop = SNOOPING_REQD;
	       hdr_3dw.addr = addr[32-1:2];
	       hdr_3dw.length = tlplen;
	       Bit#(TDiv#(buswidth,8)) firstbe = reqFirstByteEnable(req);
	       Bit#(TDiv#(buswidth,8)) lastbe = reqLastByteEnable(req);
	       hdr_3dw.firstbe = firstbe[3:0];
	       hdr_3dw.lastbe = (tlplen > 1) ? lastbe[valueOf(busWidthBytes)-1:valueOf(busWidthBytes)-4] : 0;
	       tlp.be = 'hfff0; // no data word in this TLP

	       tlp.data = extend(pack(hdr_3dw));
	    end
	    tlpWriteHeaderFifo.enq(TlpWriteHeaderInfo {tlp: tlp, dwCount: tlplen, is3dw: writeIs3dw, isHeaderOnly: (writeIs3dw && tlplen == 1) });
	    writeBurstCountFifo.enq(burstLen);
	    writeTag.enq(extend(awid));
         endmethod
	endinterface
      interface Put writeData;
         method Action put(MemData#(buswidth) wdata)
	    provisos (Bits#(Vector#(busWidthWords, Bit#(32)), busWidth)) if (writeDataMimoHasRoom); //.enqReadyN(fromInteger(valueOf(busWidthWords))));

	    let burstLen = writeBurstCount;
	    if (burstLen == 0) begin
	       burstLen <- toGet(writeBurstCountFifo).get();
	    end
	    writeBurstCount <= extend(burstLen-1);

	    Vector#(busWidthWords, Bit#(32)) v = unpack(wdata.data);
	    writeDataMimo.enq(fromInteger(valueOf(busWidthWords)), v);
            writeDataCnt.increment(fromInteger(valueOf(busWidthWords)));
	    writeDataMimoEnqWire <= True;
         endmethod
       endinterface
      interface Get writeDone;
         method ActionValue#(Bit#(MemTagSize)) get();
	      let tag = doneTag.first();
	      doneTag.deq();
	      return truncate(tag);
           endmethod
	endinterface
   endinterface
   interface PhysMemReadServer read_server;
      interface Put readReq;
         method Action put(PhysMemRequest#(40,buswidth) req);
	    readReqFifo.enq(req);
         endmethod
       endinterface
      interface Get     readData;
         method ActionValue#(MemData#(buswidth)) get() if (completionMimo.deqReady()
							   && completionTagMimo.deqReady());
	      let data_v = completionMimo.first;
	      let tag_last_v = completionTagMimo.first;
	      match { .tag, .last } = tag_last_v[fromInteger(valueOf(busWidthWords))-1];
	      completionMimo.deq();
	      completionTagMimo.deq();
              Bit#(buswidth) v = 0;
              for (Integer i = 0; i < valueOf(busWidthWords); i = i+1) begin
`ifdef AXI
`ifdef PCIE3
		 v[(i+1)*32-1:i*32] = data_v[i];
`else
		 v[(i+1)*32-1:i*32] = byteSwap(data_v[i]);
`endif
`elsif AVALON
		 v[(i+1)*32-1:i*32] = data_v[i];
`endif
              end
	      return MemData { data: v, tag: truncate(tag), last: last}; // last beat of this response burst
           endmethod
	endinterface
   endinterface
    endinterface: slave
   method Bool tlpOutFifoNotEmpty() = tlpOutFifo.notEmpty;
   interface Reg use4dw = use4dwReg;
endmodule: mkMemToPcie


reduce conditionalization on use of addr generators

// Copyright (c) 2012 Nokia, Inc.
// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import Vector::*;
import GetPut::*;
import FIFOF::*;
import SpecialFIFOs::*;
import FIFO::*;
import Connectable::*;
import AddressGenerator::*;

import Portal::*;
import MemTypes::*;
import Arith::*;
import Pipe::*;

module mkInterruptMux#(Vector#(numPortals,ReadOnly#(Bool)) inputs) (ReadOnly#(Bool))
   provisos(Add#(nz, TLog#(numPortals), 4),
	    Add#(1, a__, numPortals));
   function Bool my_read(ReadOnly#(Bool) x);
      return x._read;
   endfunction
   method Bool _read;
      return fold(boolor,map(my_read,inputs));
   endmethod
endmodule

module mkPortalInterrupt#(Vector#(numIndications, PipeOut#(Bit#(dataWidth))) indicationPipes)
   (PortalInterrupt#(dataWidth));
   Bool      interruptStatus = False;
   function Bool pipeOutNotEmpty(PipeOut#(a) po); return po.notEmpty(); endfunction
   Vector#(numIndications, Bool) readyBits = map(pipeOutNotEmpty, indicationPipes);

   Bit#(dataWidth)  readyChannel = -1;
   for (Integer i = valueOf(numIndications) - 1; i >= 0; i = i - 1) begin
      if (readyBits[i]) begin
         interruptStatus = True;
         readyChannel = fromInteger(i);
      end
   end
   method Bool status();
      return interruptStatus;
   endmethod
   method Bit#(dataWidth) channel();
      return readyChannel;
   endmethod
endmodule

module mkSlaveMux#(Vector#(numPortals,MemPortal#(aw,dataWidth)) portals) (PhysMemSlave#(addrWidth,dataWidth))
   provisos(Add#(selWidth,aw,addrWidth)
	    ,Add#(a__, TLog#(numPortals), selWidth)
	    ,FunnelPipesPipelined#(1, numPortals, MemData#(dataWidth),TMin#(2, TLog#(numPortals)))
	    );
   Vector#(numPortals, PhysMemSlave#(aw,dataWidth)) slaves = map(getSlave, portals);
   for(Integer i = 0; i < valueOf(numPortals); i=i+1)
      rule writeTop;
	 portals[i].num_portals <= fromInteger(valueOf(numPortals));
      endrule
   let rv <- mkMemPortalMux(slaves);
   return rv;
endmodule

module mkMemMethodMuxIn#(PhysMemSlave#(aw,dataWidth) ctrl, Vector#(numRequests, PipeIn#(Bit#(dataWidth))) requests
)(PhysMemSlave#(addrWidth,dataWidth))
   provisos(Add#(selWidth,aw,addrWidth)
	    , Add#(a__, TLog#(numRequests), selWidth)
            , Add#(1, b__, dataWidth)
      );
   AddressGenerator#(aw,dataWidth) fifoReadAddrGenerator  <- mkAddressGenerator();
   AddressGenerator#(aw,dataWidth) fifoWriteAddrGenerator <- mkAddressGenerator();
   FIFO#(Bit#(MemTagSize))                fifoWriteDoneFifo <- mkFIFO();
   let port_sel_low = valueOf(aw);
   let port_sel_high = valueOf(TSub#(addrWidth,1));
   function Bit#(selWidth) psel(Bit#(addrWidth) a);
      Bit#(selWidth) v = a[port_sel_high:port_sel_low];
      return v - 1;
   endfunction
   function Bool pselCtrl(Bit#(addrWidth) a);
      Bit#(selWidth) v = a[port_sel_high:port_sel_low];
      return v == 0;
   endfunction
   function Bit#(aw) asel(Bit#(addrWidth) a);
      return a[(port_sel_low-1):0];
   endfunction

   FIFO#(Bit#(MemTagSize)) doneFifo          <- mkFIFO1();
   FIFO#(PhysMemRequest#(aw))   req_ars <- mkFIFO1();
   FIFO#(Bit#(TLog#(numRequests))) rs <- mkFIFO1();
   FIFO#(Bool)                   rsCtrl <- mkFIFO1();
   FIFO#(PhysMemRequest#(aw))   req_aws <- mkFIFO1();
   FIFO#(Bit#(TLog#(numRequests))) ws <- mkFIFO1();
   FIFO#(Bool)                   wsCtrl <- mkFIFO1();

   rule write_done;
      let rv;
      if (wsCtrl.first)
         rv <- ctrl.write_server.writeDone.get();
      else
         rv <- toGet(fifoWriteDoneFifo).get();
      ws.deq();
      wsCtrl.deq();
      doneFifo.enq(rv);
   endrule

   rule req_aw;
      let req <- toGet(req_aws).get;
      fifoWriteAddrGenerator.request.put(req);
      if (wsCtrl.first)
         ctrl.write_server.writeReq.put(req);
   endrule

   rule req_ar;
      let req <- toGet(req_ars).get;
      fifoReadAddrGenerator.request.put(req);
      if (rsCtrl.first)
         ctrl.read_server.readReq.put(req);
   endrule

   FIFO#(MemData#(dataWidth)) rvFifo <- mkFIFO;
   rule rvrule;
      let rv;
      let b <- fifoReadAddrGenerator.addrBeat.get();
      if (rsCtrl.first)
	 rv <- ctrl.read_server.readData.get();
      else begin
	 let v = 0;
	 if (b.addr == 4)
	    v = extend(pack(requests[rs.first].notFull()));
	 rv = MemData { data: v, tag: b.tag, last: b.last };
      end
      rvFifo.enq(rv);
      //$display("mkMemMethodMux.readData aw=%d rs=%d data=%h", valueOf(aw), rs.first, rv.data);
      if (b.last) begin
	 rs.deq();
	 rsCtrl.deq();
      end
   endrule

   interface PhysMemWriteServer write_server;
      interface Put writeReq;
	 method Action put(PhysMemRequest#(addrWidth) req);
	    req_aws.enq(PhysMemRequest{addr:asel(req.addr), burstLen:req.burstLen, tag:req.tag});
	    if (req.burstLen > 4) $display("**** \n\n mkMemMethodMux.writeReq len=%d \n\n ****", req.burstLen);
	    //$display("mkMemMethodMux.writeReq addr=%h selWidth=%d aw=%d psel=%h pselCtrl=%x", req.addr, valueOf(selWidth), valueOf(aw), psel(req.addr), pselCtrl(req.addr));
	    ws.enq(truncate(psel(req.addr)));
            wsCtrl.enq(pselCtrl(req.addr));
	 endmethod
      endinterface
      interface Put writeData;
	 method Action put(MemData#(dataWidth) wdata);
	    //$display("mkMemMethodMux.writeData aw=%d ws=%d data=%h", valueOf(aw), ws.first, wdata.data);
	    let b <- fifoWriteAddrGenerator.addrBeat.get();
            if (wsCtrl.first)
	       ctrl.write_server.writeData.put(wdata);
            else begin
	       //$display("mkPipeInMemSlave.writeData.put addr=%h data=%h", b.addr, wdata.data);
	       if (b.last)
	          fifoWriteDoneFifo.enq(b.tag);
	       requests[ws.first].enq(wdata.data);
	       // this used to be where we triggered putFailed
            end
	 endmethod
      endinterface
      interface Get writeDone;
	 method ActionValue#(Bit#(MemTagSize)) get();
	    let rv <- toGet(doneFifo).get();
	    return rv;
	 endmethod
      endinterface
   endinterface
   interface PhysMemReadServer read_server;
      interface Put readReq;
	 method Action put(PhysMemRequest#(addrWidth) req);
	    req_ars.enq(PhysMemRequest{addr:asel(req.addr), burstLen:req.burstLen, tag:req.tag});
	    //$display("mkMemMethodMux.readReq addr=%h aw=%d psel=%h pselCtrl=%x", req.addr, valueOf(aw), psel(req.addr), pselCtrl(req.addr));
	    if (req.burstLen > 4) $display("**** \n\n mkMemMethodMux.readReq len=%d \n\n ****", req.burstLen);
	    rs.enq(truncate(psel(req.addr)));
            rsCtrl.enq(pselCtrl(req.addr));
	 endmethod
      endinterface
      interface Get readData;
	 method ActionValue#(MemData#(dataWidth)) get();
	    let rv <- toGet(rvFifo).get();
	    return rv;
	 endmethod
      endinterface
   endinterface
endmodule

module mkMemMethodMuxOut#(PhysMemSlave#(aw,dataWidth) ctrl, Vector#(numIndications, PipeOut#(Bit#(dataWidth))) indications)(PhysMemSlave#(addrWidth,dataWidth))
   provisos(Add#(selWidth,aw,addrWidth)
	    , Add#(a__, TLog#(numIndications), selWidth)
            , Add#(1, b__, dataWidth)
      );
   AddressGenerator#(aw,dataWidth) fifoReadAddrGenerator <- mkAddressGenerator();
   AddressGenerator#(aw,dataWidth) fifoWriteAddrGenerator <- mkAddressGenerator();
   FIFO#(Bit#(MemTagSize))                fifoWriteDoneFifo <- mkFIFO();
   FIFO#(MemData#(dataWidth))             fifoReadDataFifo <- mkFIFO();
   let port_sel_low = valueOf(aw);
   let port_sel_high = valueOf(TSub#(addrWidth,1));
   function Bit#(selWidth) psel(Bit#(addrWidth) a);
      Bit#(selWidth) v = a[port_sel_high:port_sel_low];
      return v - 1;
   endfunction
   function Bool pselCtrl(Bit#(addrWidth) a);
      Bit#(selWidth) v = a[port_sel_high:port_sel_low];
      return v == 0;
   endfunction
   function Bit#(aw) asel(Bit#(addrWidth) a);
      return a[(port_sel_low-1):0];
   endfunction

   FIFO#(Bit#(MemTagSize)) doneFifo          <- mkFIFO1();
   FIFO#(PhysMemRequest#(aw))   req_ars <- mkFIFO1();
   FIFO#(Bit#(TLog#(numIndications))) rs <- mkFIFO1();
   FIFO#(Bool)                   rsCtrl <- mkFIFO1();
   FIFO#(PhysMemRequest#(aw))   req_aws <- mkFIFO1();
   FIFO#(Bit#(TLog#(numIndications))) ws <- mkFIFO1();
   FIFO#(Bool)                   wsCtrl <- mkFIFO1();

   rule readDataRule;
      let b <- fifoReadAddrGenerator.addrBeat.get();
      let v = 0;
      if (!rsCtrl.first) begin
	  if (b.addr == 0)
	     v <- toGet(indications[rs.first]).get();
	  else if (b.addr == 4)
	     v = extend(pack(indications[rs.first].notEmpty()));
	  //$display("mkPipeOutMemSlave.readData.get addr=%h data=%h", b.addr, data);
	  fifoReadDataFifo.enq(MemData { data: v, tag: b.tag, last: b.last });
      end
   endrule

   rule write_done;
      let rv;
      if (wsCtrl.first)
         rv <- ctrl.write_server.writeDone.get();
      else
         rv <- toGet(fifoWriteDoneFifo).get();
      ws.deq();
      wsCtrl.deq();
      doneFifo.enq(rv);
   endrule

   rule req_aw;
      let req <- toGet(req_aws).get;
      fifoWriteAddrGenerator.request.put(req);
      if (wsCtrl.first)
         ctrl.write_server.writeReq.put(req);
   endrule

   rule req_ar;
      let req <- toGet(req_ars).get;
      fifoReadAddrGenerator.request.put(req);
      if (rsCtrl.first)
         ctrl.read_server.readReq.put(req);
   endrule

   FIFO#(MemData#(dataWidth)) rvFifo <- mkFIFO;
   rule rvrule;
      let rv;
      if (rsCtrl.first)
	 rv <- ctrl.read_server.readData.get();
      else
	 rv <- toGet(fifoReadDataFifo).get();
      rvFifo.enq(rv);
      //$display("mkMemMethodMux.readData aw=%d rs=%d data=%h", valueOf(aw), rs.first, rv.data);
      rs.deq();
      rsCtrl.deq();
   endrule

   interface PhysMemWriteServer write_server;
      interface Put writeReq;
	 method Action put(PhysMemRequest#(addrWidth) req);
	    req_aws.enq(PhysMemRequest{addr:asel(req.addr), burstLen:req.burstLen, tag:req.tag});
	    if (req.burstLen > 4) $display("**** \n\n mkMemMethodMux.writeReq len=%d \n\n ****", req.burstLen);
	    //$display("mkMemMethodMux.writeReq addr=%h selWidth=%d aw=%d psel=%h pselCtrl=%x", req.addr, valueOf(selWidth), valueOf(aw), psel(req.addr), pselCtrl(req.addr));
	    ws.enq(truncate(psel(req.addr)));
            wsCtrl.enq(pselCtrl(req.addr));
	 endmethod
      endinterface
      interface Put writeData;
	 method Action put(MemData#(dataWidth) wdata);
	    //$display("mkMemMethodMux.writeData aw=%d ws=%d data=%h", valueOf(aw), ws.first, wdata.data);
	    let b <- fifoWriteAddrGenerator.addrBeat.get();
            if (wsCtrl.first)
	       ctrl.write_server.writeData.put(wdata);
            else begin
	       //$display("mkPipeOutMemSlave.writeData.put addr=%h data=%h", b.addr, d.data);
	       if (b.last)
	          fifoWriteDoneFifo.enq(b.tag);
            end
	 endmethod
      endinterface
      interface Get writeDone;
	 method ActionValue#(Bit#(MemTagSize)) get();
	    let rv <- toGet(doneFifo).get();
	    return rv;
	 endmethod
      endinterface
   endinterface
   interface PhysMemReadServer read_server;
      interface Put readReq;
	 method Action put(PhysMemRequest#(addrWidth) req);
	    req_ars.enq(PhysMemRequest{addr:asel(req.addr), burstLen:req.burstLen, tag:req.tag});
	    //$display("mkMemMethodMux.readReq addr=%h aw=%d psel=%h pselCtrl=%x", req.addr, valueOf(aw), psel(req.addr), pselCtrl(req.addr));
	    if (req.burstLen > 4) $display("**** \n\n mkMemMethodMux.readReq len=%d \n\n ****", req.burstLen);
	    rs.enq(truncate(psel(req.addr)));
            rsCtrl.enq(pselCtrl(req.addr));
	 endmethod
      endinterface
      interface Get readData;
	 method ActionValue#(MemData#(dataWidth)) get();
	    let rv <- toGet(rvFifo).get();
	    return rv;
	 endmethod
      endinterface
   endinterface
endmodule

module mkMemPortalMux#(Vector#(numSlaves,PhysMemSlave#(aw,dataWidth)) slaves) (PhysMemSlave#(addrWidth,dataWidth))
   provisos(Add#(selWidth,aw,addrWidth)
	    ,Add#(a__, TLog#(numSlaves), selWidth)
	    ,Min#(2,TLog#(numSlaves),bpc)
	    ,Pipe::FunnelPipesPipelined#(1, numSlaves, MemTypes::MemData#(dataWidth),bpc)
	    );
   let port_sel_low = valueOf(aw);
   let port_sel_high = valueOf(TSub#(addrWidth,1));
   function Bit#(selWidth) psel(Bit#(addrWidth) a);
      return a[port_sel_high:port_sel_low];
   endfunction
   function Bit#(aw) asel(Bit#(addrWidth) a);
      return a[(port_sel_low-1):0];
   endfunction
   function Get#(MemData#(dataWidth)) getMemPortalReadData(PhysMemSlave#(aw,dataWidth) x) = x.read_server.readData;
   function Put#(MemData#(dataWidth)) getMemPortalWriteData(PhysMemSlave#(aw,dataWidth) x) = x.write_server.writeData;

   FIFO#(Bit#(MemTagSize))        doneFifo <- mkFIFO1();
   FIFO#(PhysMemRequest#(aw)) req_ars <- mkSizedFIFO(1);
   FIFO#(Bit#(TLog#(numSlaves))) rs <- mkFIFO1();
   Vector#(numSlaves, PipeOut#(MemData#(dataWidth))) readDataPipes <- mapM(mkPipeOut, map(getMemPortalReadData,slaves));
   FunnelPipe#(1, numSlaves, MemData#(dataWidth), bpc) read_data_funnel <- mkFunnelPipesPipelined(readDataPipes);

   FIFO#(PhysMemRequest#(aw)) req_aws <- mkFIFO1();
   FIFO#(Bit#(TLog#(numSlaves))) ws <- mkFIFO1();
   FIFOF#(Tuple2#(Bit#(TLog#(numSlaves)), MemData#(dataWidth))) write_data <- mkFIFOF;
   UnFunnelPipe#(1, numSlaves, MemData#(dataWidth), bpc) write_data_unfunnel <- mkUnFunnelPipesPipelined(cons(toPipeOut(write_data),nil));
   Vector#(numSlaves, PipeIn#(MemData#(dataWidth))) writeDataPipes <- mapM(mkPipeIn, map(getMemPortalWriteData,slaves));
   zipWithM(mkConnection, write_data_unfunnel, writeDataPipes);

   rule req_aw;
      let req <- toGet(req_aws).get;
      slaves[ws.first].write_server.writeReq.put(req);
   endrule

   rule req_ar;
      let req <- toGet(req_ars).get;
      slaves[rs.first].read_server.readReq.put(req);
   endrule

   rule write_done_rule;
      let rv <- slaves[ws.first].write_server.writeDone.get();
      ws.deq();
      doneFifo.enq(rv);
   endrule

   let verbose = False;

   interface PhysMemWriteServer write_server;
      interface Put writeReq;
	 method Action put(PhysMemRequest#(addrWidth) req);
	    req_aws.enq(PhysMemRequest{addr:asel(req.addr), burstLen:req.burstLen, tag:req.tag});
	    if (req.burstLen > 4) $display("**** \n\n mkSlaveMux.writeReq len=%d \n\n ****", req.burstLen);
	    ws.enq(truncate(psel(req.addr)));
	    if(verbose) $display("mkSlaveMux.writeReq addr=%h aw=%d psel=%h", req.addr, valueOf(aw), psel(req.addr));
	 endmethod
      endinterface
      interface Put writeData;
	 method Action put(MemData#(dataWidth) wdata);
	    write_data.enq(tuple2(ws.first,wdata));
	    if(verbose) $display("mkSlaveMux.writeData dst=%h wdata=%h", ws.first,wdata);
	 endmethod
      endinterface
      interface Get writeDone;
	 method ActionValue#(Bit#(MemTagSize)) get();
	    let rv <- toGet(doneFifo).get();
	    return rv;
	 endmethod
      endinterface
   endinterface
   interface PhysMemReadServer read_server;
      interface Put readReq;
	 method Action put(PhysMemRequest#(addrWidth) req);
	    req_ars.enq(PhysMemRequest{addr:asel(req.addr), burstLen:req.burstLen, tag:req.tag});
	    rs.enq(truncate(psel(req.addr)));
	    if (req.burstLen > 4) $display("**** \n\n mkSlaveMux.readReq len=%d \n\n ****", req.burstLen);
	    if(verbose) $display("mkSlaveMux.readReq addr=%h aw=%d psel=%h", req.addr, valueOf(aw), psel(req.addr));
	 endmethod
      endinterface
      interface Get readData;
	 method ActionValue#(MemData#(dataWidth)) get();
	    let rv <- toGet(read_data_funnel[0]).get;
	    rs.deq();
	    if(verbose) $display("mkSlaveMux.readData rs=%d data=%h", rs.first, rv.data);
	    return rv;
	 endmethod
      endinterface
   endinterface
endmodule

// Copyright (c) 2012 Nokia, Inc.
// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import Vector::*;
import GetPut::*;
import FIFOF::*;
import SpecialFIFOs::*;
import FIFO::*;
import Connectable::*;
import AddressGenerator::*;

import Portal::*;
import MemTypes::*;
import Arith::*;
import Pipe::*;

module mkInterruptMux#(Vector#(numPortals,ReadOnly#(Bool)) inputs) (ReadOnly#(Bool))
   provisos(Add#(nz, TLog#(numPortals), 4),
	    Add#(1, a__, numPortals));
   function Bool my_read(ReadOnly#(Bool) x);
      return x._read;
   endfunction
   method Bool _read;
      return fold(boolor,map(my_read,inputs));
   endmethod
endmodule

module mkPortalInterrupt#(Vector#(numIndications, PipeOut#(Bit#(dataWidth))) indicationPipes)
   (PortalInterrupt#(dataWidth));
   Bool      interruptStatus = False;
   function Bool pipeOutNotEmpty(PipeOut#(a) po); return po.notEmpty(); endfunction
   Vector#(numIndications, Bool) readyBits = map(pipeOutNotEmpty, indicationPipes);

   Bit#(dataWidth)  readyChannel = -1;
   for (Integer i = valueOf(numIndications) - 1; i >= 0; i = i - 1) begin
      if (readyBits[i]) begin
         interruptStatus = True;
         readyChannel = fromInteger(i);
      end
   end
   method Bool status();
      return interruptStatus;
   endmethod
   method Bit#(dataWidth) channel();
      return readyChannel;
   endmethod
endmodule

module mkSlaveMux#(Vector#(numPortals,MemPortal#(aw,dataWidth)) portals) (PhysMemSlave#(addrWidth,dataWidth))
   provisos(Add#(selWidth,aw,addrWidth)
	    ,Add#(a__, TLog#(numPortals), selWidth)
	    ,FunnelPipesPipelined#(1, numPortals, MemData#(dataWidth),TMin#(2, TLog#(numPortals)))
	    );
   Vector#(numPortals, PhysMemSlave#(aw,dataWidth)) slaves = map(getSlave, portals);
   for(Integer i = 0; i < valueOf(numPortals); i=i+1)
      rule writeTop;
	 portals[i].num_portals <= fromInteger(valueOf(numPortals));
      endrule
   let rv <- mkMemPortalMux(slaves);
   return rv;
endmodule

module mkMemMethodMuxIn#(PhysMemSlave#(aw,dataWidth) ctrl, Vector#(numRequests, PipeIn#(Bit#(dataWidth))) requests
)(PhysMemSlave#(addrWidth,dataWidth))
   provisos(Add#(selWidth,aw,addrWidth)
	    , Add#(a__, TLog#(numRequests), selWidth)
            , Add#(1, b__, dataWidth)
      );
   AddressGenerator#(aw,dataWidth) fifoReadAddrGenerator  <- mkAddressGenerator();
   AddressGenerator#(aw,dataWidth) fifoWriteAddrGenerator <- mkAddressGenerator();
   FIFO#(Bit#(MemTagSize))                fifoWriteDoneFifo <- mkFIFO();
   let port_sel_low = valueOf(aw);
   let port_sel_high = valueOf(TSub#(addrWidth,1));
   function Bit#(selWidth) psel(Bit#(addrWidth) a);
      Bit#(selWidth) v = a[port_sel_high:port_sel_low];
      return v - 1;
   endfunction
   function Bool pselCtrl(Bit#(addrWidth) a);
      Bit#(selWidth) v = a[port_sel_high:port_sel_low];
      return v == 0;
   endfunction
   function Bit#(aw) asel(Bit#(addrWidth) a);
      return a[(port_sel_low-1):0];
   endfunction

   FIFO#(Bit#(MemTagSize)) doneFifo          <- mkFIFO1();
   FIFO#(PhysMemRequest#(aw))   req_ars <- mkFIFO1();
   FIFO#(Bit#(TLog#(numRequests))) rs <- mkFIFO1();
   FIFO#(Bool)                   rsCtrl <- mkFIFO1();
   FIFO#(PhysMemRequest#(aw))   req_aws <- mkFIFO1();
   FIFO#(Bit#(TLog#(numRequests))) ws <- mkFIFO1();
   FIFO#(Bool)                   wsCtrl <- mkFIFO1();

   rule write_done;
      let rv;
      if (wsCtrl.first)
         rv <- ctrl.write_server.writeDone.get();
      else
         rv <- toGet(fifoWriteDoneFifo).get();
      ws.deq();
      wsCtrl.deq();
      doneFifo.enq(rv);
   endrule

   rule req_aw;
      let req <- toGet(req_aws).get;
      if (wsCtrl.first)
         ctrl.write_server.writeReq.put(req);
      else
         fifoWriteAddrGenerator.request.put(req);
   endrule

   rule req_ar;
      let req <- toGet(req_ars).get;
      if (rsCtrl.first)
         ctrl.read_server.readReq.put(req);
      else
         fifoReadAddrGenerator.request.put(req);
   endrule

   FIFO#(MemData#(dataWidth)) rvFifo <- mkFIFO;
   rule rvrule;
      let rv;
      if (rsCtrl.first)
	 rv <- ctrl.read_server.readData.get();
      else begin
	 let b <- fifoReadAddrGenerator.addrBeat.get();
	 let v = 0;
	 if (b.addr == 4)
	    v = extend(pack(requests[rs.first].notFull()));
	 rv = MemData { data: v, tag: b.tag, last: b.last };
      end
      rvFifo.enq(rv);
      //$display("mkMemMethodMux.readData aw=%d rs=%d data=%h", valueOf(aw), rs.first, rv.data);
      rs.deq();
      rsCtrl.deq();
   endrule

   interface PhysMemWriteServer write_server;
      interface Put writeReq;
	 method Action put(PhysMemRequest#(addrWidth) req);
	    req_aws.enq(PhysMemRequest{addr:asel(req.addr), burstLen:req.burstLen, tag:req.tag});
	    if (req.burstLen > 4) $display("**** \n\n mkMemMethodMux.writeReq len=%d \n\n ****", req.burstLen);
	    //$display("mkMemMethodMux.writeReq addr=%h selWidth=%d aw=%d psel=%h pselCtrl=%x", req.addr, valueOf(selWidth), valueOf(aw), psel(req.addr), pselCtrl(req.addr));
	    ws.enq(truncate(psel(req.addr)));
            wsCtrl.enq(pselCtrl(req.addr));
	 endmethod
      endinterface
      interface Put writeData;
	 method Action put(MemData#(dataWidth) wdata);
	    //$display("mkMemMethodMux.writeData aw=%d ws=%d data=%h", valueOf(aw), ws.first, wdata.data);
            if (wsCtrl.first)
	       ctrl.write_server.writeData.put(wdata);
            else begin
	       let b <- fifoWriteAddrGenerator.addrBeat.get();
	       //$display("mkPipeInMemSlave.writeData.put addr=%h data=%h", b.addr, wdata.data);
	       if (b.last)
	          fifoWriteDoneFifo.enq(b.tag);
	       requests[ws.first].enq(wdata.data);
	       // this used to be where we triggered putFailed
            end
	 endmethod
      endinterface
      interface Get writeDone;
	 method ActionValue#(Bit#(MemTagSize)) get();
	    let rv <- toGet(doneFifo).get();
	    return rv;
	 endmethod
      endinterface
   endinterface
   interface PhysMemReadServer read_server;
      interface Put readReq;
	 method Action put(PhysMemRequest#(addrWidth) req);
	    req_ars.enq(PhysMemRequest{addr:asel(req.addr), burstLen:req.burstLen, tag:req.tag});
	    //$display("mkMemMethodMux.readReq addr=%h aw=%d psel=%h pselCtrl=%x", req.addr, valueOf(aw), psel(req.addr), pselCtrl(req.addr));
	    if (req.burstLen > 4) $display("**** \n\n mkMemMethodMux.readReq len=%d \n\n ****", req.burstLen);
	    rs.enq(truncate(psel(req.addr)));
            rsCtrl.enq(pselCtrl(req.addr));
	 endmethod
      endinterface
      interface Get readData;
	 method ActionValue#(MemData#(dataWidth)) get();
	    let rv <- toGet(rvFifo).get();
	    return rv;
	 endmethod
      endinterface
   endinterface
endmodule

module mkMemMethodMuxOut#(PhysMemSlave#(aw,dataWidth) ctrl, Vector#(numIndications, PipeOut#(Bit#(dataWidth))) indications)(PhysMemSlave#(addrWidth,dataWidth))
   provisos(Add#(selWidth,aw,addrWidth)
	    , Add#(a__, TLog#(numIndications), selWidth)
            , Add#(1, b__, dataWidth)
      );
   AddressGenerator#(aw,dataWidth) fifoReadAddrGenerator <- mkAddressGenerator();
   AddressGenerator#(aw,dataWidth) fifoWriteAddrGenerator <- mkAddressGenerator();
   FIFO#(Bit#(MemTagSize))                fifoWriteDoneFifo <- mkFIFO();
   FIFO#(MemData#(dataWidth))             fifoReadDataFifo <- mkFIFO();
   let port_sel_low = valueOf(aw);
   let port_sel_high = valueOf(TSub#(addrWidth,1));
   function Bit#(selWidth) psel(Bit#(addrWidth) a);
      Bit#(selWidth) v = a[port_sel_high:port_sel_low];
      return v - 1;
   endfunction
   function Bool pselCtrl(Bit#(addrWidth) a);
      Bit#(selWidth) v = a[port_sel_high:port_sel_low];
      return v == 0;
   endfunction
   function Bit#(aw) asel(Bit#(addrWidth) a);
      return a[(port_sel_low-1):0];
   endfunction

   FIFO#(Bit#(MemTagSize)) doneFifo          <- mkFIFO1();
   FIFO#(PhysMemRequest#(aw))   req_ars <- mkFIFO1();
   FIFO#(Bit#(TLog#(numIndications))) rs <- mkFIFO1();
   FIFO#(Bool)                   rsCtrl <- mkFIFO1();
   FIFO#(PhysMemRequest#(aw))   req_aws <- mkFIFO1();
   FIFO#(Bit#(TLog#(numIndications))) ws <- mkFIFO1();
   FIFO#(Bool)                   wsCtrl <- mkFIFO1();

   rule readDataRule;
      let b <- fifoReadAddrGenerator.addrBeat.get();
      let v = 0;
      if (b.addr == 0)
	 v <- toGet(indications[rs.first]).get();
      else if (b.addr == 4)
	 v = extend(pack(indications[rs.first].notEmpty()));
      //$display("mkPipeOutMemSlave.readData.get addr=%h data=%h", b.addr, data);
      fifoReadDataFifo.enq(MemData { data: v, tag: b.tag, last: b.last });
   endrule

   rule write_done;
      let rv;
      if (wsCtrl.first)
         rv <- ctrl.write_server.writeDone.get();
      else
         rv <- toGet(fifoWriteDoneFifo).get();
      ws.deq();
      wsCtrl.deq();
      doneFifo.enq(rv);
   endrule

   rule req_aw;
      let req <- toGet(req_aws).get;
      if (wsCtrl.first)
         ctrl.write_server.writeReq.put(req);
      else
         fifoWriteAddrGenerator.request.put(req);
   endrule

   rule req_ar;
      let req <- toGet(req_ars).get;
      if (rsCtrl.first)
         ctrl.read_server.readReq.put(req);
      else
         fifoReadAddrGenerator.request.put(req);
   endrule

   FIFO#(MemData#(dataWidth)) rvFifo <- mkFIFO;
   rule rvrule;
      let rv;
      if (rsCtrl.first)
	 rv <- ctrl.read_server.readData.get();
      else
	 rv <- toGet(fifoReadDataFifo).get();
      rvFifo.enq(rv);
      //$display("mkMemMethodMux.readData aw=%d rs=%d data=%h", valueOf(aw), rs.first, rv.data);
      rs.deq();
      rsCtrl.deq();
   endrule

   interface PhysMemWriteServer write_server;
      interface Put writeReq;
	 method Action put(PhysMemRequest#(addrWidth) req);
	    req_aws.enq(PhysMemRequest{addr:asel(req.addr), burstLen:req.burstLen, tag:req.tag});
	    if (req.burstLen > 4) $display("**** \n\n mkMemMethodMux.writeReq len=%d \n\n ****", req.burstLen);
	    //$display("mkMemMethodMux.writeReq addr=%h selWidth=%d aw=%d psel=%h pselCtrl=%x", req.addr, valueOf(selWidth), valueOf(aw), psel(req.addr), pselCtrl(req.addr));
	    ws.enq(truncate(psel(req.addr)));
            wsCtrl.enq(pselCtrl(req.addr));
	 endmethod
      endinterface
      interface Put writeData;
	 method Action put(MemData#(dataWidth) wdata);
	    //$display("mkMemMethodMux.writeData aw=%d ws=%d data=%h", valueOf(aw), ws.first, wdata.data);
            if (wsCtrl.first)
	       ctrl.write_server.writeData.put(wdata);
            else begin
	       let b <- fifoWriteAddrGenerator.addrBeat.get();
	       //$display("mkPipeOutMemSlave.writeData.put addr=%h data=%h", b.addr, d.data);
	       if (b.last)
	          fifoWriteDoneFifo.enq(b.tag);
            end
	 endmethod
      endinterface
      interface Get writeDone;
	 method ActionValue#(Bit#(MemTagSize)) get();
	    let rv <- toGet(doneFifo).get();
	    return rv;
	 endmethod
      endinterface
   endinterface
   interface PhysMemReadServer read_server;
      interface Put readReq;
	 method Action put(PhysMemRequest#(addrWidth) req);
	    req_ars.enq(PhysMemRequest{addr:asel(req.addr), burstLen:req.burstLen, tag:req.tag});
	    //$display("mkMemMethodMux.readReq addr=%h aw=%d psel=%h pselCtrl=%x", req.addr, valueOf(aw), psel(req.addr), pselCtrl(req.addr));
	    if (req.burstLen > 4) $display("**** \n\n mkMemMethodMux.readReq len=%d \n\n ****", req.burstLen);
	    rs.enq(truncate(psel(req.addr)));
            rsCtrl.enq(pselCtrl(req.addr));
	 endmethod
      endinterface
      interface Get readData;
	 method ActionValue#(MemData#(dataWidth)) get();
	    let rv <- toGet(rvFifo).get();
	    return rv;
	 endmethod
      endinterface
   endinterface
endmodule

module mkMemPortalMux#(Vector#(numSlaves,PhysMemSlave#(aw,dataWidth)) slaves) (PhysMemSlave#(addrWidth,dataWidth))
   provisos(Add#(selWidth,aw,addrWidth)
	    ,Add#(a__, TLog#(numSlaves), selWidth)
	    ,Min#(2,TLog#(numSlaves),bpc)
	    ,Pipe::FunnelPipesPipelined#(1, numSlaves, MemTypes::MemData#(dataWidth),bpc)
	    );
   let port_sel_low = valueOf(aw);
   let port_sel_high = valueOf(TSub#(addrWidth,1));
   function Bit#(selWidth) psel(Bit#(addrWidth) a);
      return a[port_sel_high:port_sel_low];
   endfunction
   function Bit#(aw) asel(Bit#(addrWidth) a);
      return a[(port_sel_low-1):0];
   endfunction
   function Get#(MemData#(dataWidth)) getMemPortalReadData(PhysMemSlave#(aw,dataWidth) x) = x.read_server.readData;
   function Put#(MemData#(dataWidth)) getMemPortalWriteData(PhysMemSlave#(aw,dataWidth) x) = x.write_server.writeData;

   FIFO#(Bit#(MemTagSize))        doneFifo <- mkFIFO1();
   FIFO#(PhysMemRequest#(aw)) req_ars <- mkSizedFIFO(1);
   FIFO#(Bit#(TLog#(numSlaves))) rs <- mkFIFO1();
   Vector#(numSlaves, PipeOut#(MemData#(dataWidth))) readDataPipes <- mapM(mkPipeOut, map(getMemPortalReadData,slaves));
   FunnelPipe#(1, numSlaves, MemData#(dataWidth), bpc) read_data_funnel <- mkFunnelPipesPipelined(readDataPipes);

   FIFO#(PhysMemRequest#(aw)) req_aws <- mkFIFO1();
   FIFO#(Bit#(TLog#(numSlaves))) ws <- mkFIFO1();
   FIFOF#(Tuple2#(Bit#(TLog#(numSlaves)), MemData#(dataWidth))) write_data <- mkFIFOF;
   UnFunnelPipe#(1, numSlaves, MemData#(dataWidth), bpc) write_data_unfunnel <- mkUnFunnelPipesPipelined(cons(toPipeOut(write_data),nil));
   Vector#(numSlaves, PipeIn#(MemData#(dataWidth))) writeDataPipes <- mapM(mkPipeIn, map(getMemPortalWriteData,slaves));
   zipWithM(mkConnection, write_data_unfunnel, writeDataPipes);

   rule req_aw;
      let req <- toGet(req_aws).get;
      slaves[ws.first].write_server.writeReq.put(req);
   endrule

   rule req_ar;
      let req <- toGet(req_ars).get;
      slaves[rs.first].read_server.readReq.put(req);
   endrule

   rule write_done_rule;
      let rv <- slaves[ws.first].write_server.writeDone.get();
      ws.deq();
      doneFifo.enq(rv);
   endrule

   let verbose = False;

   interface PhysMemWriteServer write_server;
      interface Put writeReq;
	 method Action put(PhysMemRequest#(addrWidth) req);
	    req_aws.enq(PhysMemRequest{addr:asel(req.addr), burstLen:req.burstLen, tag:req.tag});
	    if (req.burstLen > 4) $display("**** \n\n mkSlaveMux.writeReq len=%d \n\n ****", req.burstLen);
	    ws.enq(truncate(psel(req.addr)));
	    if(verbose) $display("mkSlaveMux.writeReq addr=%h aw=%d psel=%h", req.addr, valueOf(aw), psel(req.addr));
	 endmethod
      endinterface
      interface Put writeData;
	 method Action put(MemData#(dataWidth) wdata);
	    write_data.enq(tuple2(ws.first,wdata));
	    if(verbose) $display("mkSlaveMux.writeData dst=%h wdata=%h", ws.first,wdata);
	 endmethod
      endinterface
      interface Get writeDone;
	 method ActionValue#(Bit#(MemTagSize)) get();
	    let rv <- toGet(doneFifo).get();
	    return rv;
	 endmethod
      endinterface
   endinterface
   interface PhysMemReadServer read_server;
      interface Put readReq;
	 method Action put(PhysMemRequest#(addrWidth) req);
	    req_ars.enq(PhysMemRequest{addr:asel(req.addr), burstLen:req.burstLen, tag:req.tag});
	    rs.enq(truncate(psel(req.addr)));
	    if (req.burstLen > 4) $display("**** \n\n mkSlaveMux.readReq len=%d \n\n ****", req.burstLen);
	    if(verbose) $display("mkSlaveMux.readReq addr=%h aw=%d psel=%h", req.addr, valueOf(aw), psel(req.addr));
	 endmethod
      endinterface
      interface Get readData;
	 method ActionValue#(MemData#(dataWidth)) get();
	    let rv <- toGet(read_data_funnel[0]).get;
	    rs.deq();
	    if(verbose) $display("mkSlaveMux.readData rs=%d data=%h", rs.first, rv.data);
	    return rv;
	 endmethod
      endinterface
   endinterface
endmodule