use mkDualClockBramFIFOF

use mkDualClockBramFIFOF

import GetPut::*;
import Clocks::*;
import FIFO::*;
import FIFOF::*;
import GetPut::*;
import Connectable::*;

import HostInterface::*;
import Dsp48E1::*;
import InnerProdInterface::*;
import ConnectalBramFifo::*;

interface InnerProd;
   interface InnerProdRequest request;
endinterface

interface InnerProdTile;
   interface Put#(Tuple7#(Int#(16),Int#(16),Bool,Bool,Bit#(4),Bit#(5),Bit#(7))) request;
   interface Get#(Int#(48)) response;
endinterface

(* synthesize *)
module mkInnerProdTile(InnerProdTile);

   let dsp <- mkDsp48E1();

   interface Put request;
      method Action put(Tuple7#(Int#(16),Int#(16),Bool,Bool,Bit#(4),Bit#(5),Bit#(7)) req);
	 match { .a, .b, .first, .last, .alumode, .inmode, .opmode } = req;
	 dsp.a(extend(pack(a)));
	 dsp.b(extend(pack(b)));
	 dsp.c('h22);
	 dsp.d(0);
	 //let opmode = 7'h45; // p = M + P
	 //if (first)
	 //opmode = 7'h05; // P = M + 0
	 dsp.opmode(opmode);
	 dsp.inmode(inmode);
	 dsp.alumode(alumode);
	 dsp.last(pack(last));
      endmethod
   endinterface
   interface Get response;
      method ActionValue#(Int#(48)) get();
	 $display("InnerProdTile response.get %h", dsp.p());
	 return unpack(dsp.p());
      endmethod
   endinterface
endmodule

module mkInnerProd#(
`ifdef IMPORT_HOSTIF
		    HostInterface host,
`endif
		    InnerProdIndication ind)(InnerProd);

   let defaultClock <- exposeCurrentClock;
   let defaultReset <- exposeCurrentReset;
`ifdef IMPORT_HOSTIF
   let derivedClock = host.derivedClock;
`else
   let derivedClock = defaultClock;
`endif
   let derivedReset <- mkAsyncReset(2, defaultReset, derivedClock);
   let optionalReset = derivedReset; // noReset
   let syncIn <- mkSyncFIFO(16, defaultClock, defaultReset, derivedClock);
   FIFOF#(Int#(48)) bramFifo <- mkDualClockBramFIFOF(derivedClock, derivedReset, defaultClock, defaultReset);

   Reg#(Bit#(32)) cycles <- mkReg(0, clocked_by derivedClock, reset_by derivedReset);
   rule cyclesRule;
      cycles <= cycles+1;
   endrule

   let tile <- mkInnerProdTile(clocked_by derivedClock, reset_by optionalReset);
   rule syncRequestRule;
      let req <- toGet(syncIn).get();
      $display("syncRequestRule a=%h b=%h", tpl_1(req), tpl_2(req));
      tile.request.put(req);
   endrule
   mkConnection(tile.response, toPut(bramFifo), clocked_by derivedClock, reset_by derivedReset);
   rule indRule;
      let r <- toGet(bramFifo).get();
      $display("%d: indRule v=%x %d", cycles, r, r);
      ind.innerProd(pack(r));
   endrule

   interface InnerProdRequest request;
      method Action innerProd(Bit#(16) a, Bit#(16) b, Bool first, Bool last, Bit#(4) alumode, Bit#(5) inmode, Bit#(7) opmode);
	 $display("request.innerProd a=%h b=%h", a, b);
	 syncIn.enq(tuple7(unpack(a),unpack(b),first,last, alumode, inmode, opmode));
      endmethod
      method Action finish();
	 $dumpflush();
	 $finish();
      endmethod
   endinterface
endmodule

import GetPut::*;
import Clocks::*;
import FIFO::*;
import FIFOF::*;
import GetPut::*;
import Connectable::*;

import HostInterface::*;
import Dsp48E1::*;
import InnerProdInterface::*;
import ConnectalBramFifo::*;

interface InnerProd;
   interface InnerProdRequest request;
endinterface

interface InnerProdTile;
   interface Put#(Tuple7#(Int#(16),Int#(16),Bool,Bool,Bit#(4),Bit#(5),Bit#(7))) request;
   interface Get#(Int#(48)) response;
endinterface

(* synthesize *)
module mkInnerProdTile(InnerProdTile);

   let dsp <- mkDsp48E1();

   interface Put request;
      method Action put(Tuple7#(Int#(16),Int#(16),Bool,Bool,Bit#(4),Bit#(5),Bit#(7)) req);
	 match { .a, .b, .first, .last, .alumode, .inmode, .opmode } = req;
	 dsp.a(extend(pack(a)));
	 dsp.b(extend(pack(b)));
	 dsp.c(0);
	 dsp.d(0);
	 //let opmode = 7'h45; // p = M + P
	 //if (first)
	 //opmode = 7'h05; // P = M + 0
	 dsp.opmode(opmode);
	 dsp.inmode(inmode);
	 dsp.alumode(alumode);
	 dsp.last(pack(last));
      endmethod
   endinterface
   interface Get response;
      method ActionValue#(Int#(48)) get();
	 $display("InnerProdTile response.get %h", dsp.p());
	 return unpack(dsp.p());
      endmethod
   endinterface
endmodule

module mkInnerProd#(
`ifdef IMPORT_HOSTIF
		    HostInterface host,
`endif
		    InnerProdIndication ind)(InnerProd);

   let defaultClock <- exposeCurrentClock;
   let defaultReset <- exposeCurrentReset;
`ifdef IMPORT_HOSTIF
   let derivedClock = host.derivedClock;
`else
   let derivedClock = defaultClock;
`endif
   let derivedReset <- mkAsyncReset(2, defaultReset, derivedClock);
   let optionalReset = derivedReset; // noReset
   let syncIn <- mkSyncFIFO(16, defaultClock, defaultReset, derivedClock);
   SyncFIFOIfc#(Int#(48)) syncOut <- mkSyncFIFO(16, derivedClock, derivedReset, defaultClock);
   FIFOF#(Int#(48)) bramFifo <- mkSizedBRAMFIFOF(512, clocked_by derivedClock, reset_by derivedReset);

   Reg#(Bit#(32)) cycles <- mkReg(0, clocked_by derivedClock, reset_by derivedReset);
   rule cyclesRule;
      cycles <= cycles+1;
   endrule

   let tile <- mkInnerProdTile(clocked_by derivedClock, reset_by optionalReset);
   rule syncRequestRule;
      let req <- toGet(syncIn).get();
      $display("syncRequestRule a=%h b=%h", tpl_1(req), tpl_2(req));
      tile.request.put(req);
   endrule
   mkConnection(tile.response, toPut(bramFifo), clocked_by derivedClock, reset_by derivedReset);
   mkConnection(toGet(bramFifo), toPut(syncOut), clocked_by derivedClock, reset_by derivedReset);
   rule indRule;
      let r <- toGet(syncOut).get();
      $display("%d: indRule v=%x %d", cycles, r, r);
      ind.innerProd(pack(r));
   endrule

   interface InnerProdRequest request;
      method Action innerProd(Bit#(16) a, Bit#(16) b, Bool first, Bool last, Bit#(4) alumode, Bit#(5) inmode, Bit#(7) opmode);
	 $display("request.innerProd a=%h b=%h", a, b);
	 syncIn.enq(tuple7(unpack(a),unpack(b),first,last, alumode, inmode, opmode));
      endmethod
      method Action finish();
	 $dumpflush();
	 $finish();
      endmethod
   endinterface
endmodule


hide dds again

hide dds again

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import DDS::*;
import Gearbox::*;
import ChannelSelect::*;
import Complex::*;
import FPCMult::*;
import FixedPoint::*;
import Pipe::*;
import Vector::*;

/* rfreqDataWrite data is FixedPoint(2,14) */
/* setCoeff data is FixedPoint(2, 23) */
interface ChannelSelectTestRequest;
   method Action rfreqDataWrite(Bit#(32) dataRe, Bit#(32) dataIm);
   method Action setCoeff(Bit#(10) addr, Bit#(32) valueRe, Bit#(32) valueIm);
endinterface

/* rfreqDataWrite data is FixedPoint(2,14) */
interface ChannelSelectTestIndication;
   method Action ifreqData(Bit#(32) dataRe, Bit#(32) dataIm);
endinterface

module mkChannelSelectTestRequest#(ChannelSelectTestIndication indication) (ChannelSelectTestRequest)
   provisos(Bits#(CoeffData, a__),
	    Bits#(ProductData, b__),
	    Bits#(MulData, c__));
   Clock clk <- exposeCurrentClock;
   Reset rst <- exposeCurrentReset;
   Gearbox#(1, 2, Complex#(Signal)) gb <- mk1toNGearbox(clk, rst, clk, rst);
   ChannelSelect cs <- mkChannelSelect(64);

   rule processRF;
      let data = gb.first();
      gb.deq();
      cs.rfreq.enq(data);
   endrule

   rule processIF;
      let data = cs.ifreq.first();
      cs.ifreq.deq();
      indication.ifreqData(zeroExtend(pack(data.rel)), zeroExtend(pack(data.img)));
   endrule

   method Action rfreqDataWrite(Bit#(32) dataRe, Bit#(32) dataIm);
      Signal re = unpack(truncate(dataRe));
      Signal im = unpack(truncate(dataIm));
      Vector#(1, Complex#(Signal)) x = newVector();
      x[0] = Complex{rel: re, img: im};
      gb.enq(x);
   endmethod

   method Action setCoeff(Bit#(10) addr, Bit#(32) valueRe, Bit#(32) valueIm);
    FixedPoint#(2, 23) re = unpack(pack(truncate(valueRe)));
    FixedPoint#(2, 23) im = unpack(pack(truncate(valueIm)));
      cs.setCoeff(addr, Complex{rel: re, img:im});
   endmethod
endmodule

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import DDS::*;
import Gearbox::*;
import ChannelSelect::*;
import Complex::*;
import FPCMult::*;
import FixedPoint::*;
import Pipe::*;
import Vector::*;

/* rfreqDataWrite data is FixedPoint(2,14) */
/* setCoeff data is FixedPoint(2, 23) */
interface ChannelSelectTestRequest;
   method Action rfreqDataWrite(Bit#(32) dataRe, Bit#(32) dataIm);
   method Action setCoeff(Bit#(10) addr, Bit#(32) valueRe, Bit#(32) valueIm);
endinterface

/* rfreqDataWrite data is FixedPoint(2,14) */
interface ChannelSelectTestIndication;
   method Action ifreqData(Bit#(32) dataRe, Bit#(32) dataIm);
endinterface

module mkChannelSelectTestRequest#(ChannelSelectTestIndication indication) (ChannelSelectTestRequest)
   provisos(Bits#(CoeffData, a__),
	    Bits#(ProductData, b__),
	    Bits#(MulData, c__));
   Clock clk <- exposeCurrentClock;
   Reset rst <- exposeCurrentReset;
   Gearbox#(1, 2, Complex#(Signal)) gb <- mk1toNGearbox(clk, rst, clk, rst);
   DDS dds <- mkDDS();
   ChannelSelect cs <- mkChannelSelect(64, dds);

   rule processRF;
      let data = gb.first();
      gb.deq();
      cs.rfreq.enq(data);
   endrule

   rule processIF;
      let data = cs.ifreq.first();
      cs.ifreq.deq();
      indication.ifreqData(zeroExtend(pack(data.rel)), zeroExtend(pack(data.img)));
   endrule

   method Action rfreqDataWrite(Bit#(32) dataRe, Bit#(32) dataIm);
      Signal re = unpack(truncate(dataRe));
      Signal im = unpack(truncate(dataIm));
      Vector#(1, Complex#(Signal)) x = newVector();
      x[0] = Complex{rel: re, img: im};
      gb.enq(x);
   endmethod

   method Action setCoeff(Bit#(10) addr, Bit#(32) valueRe, Bit#(32) valueIm);
    FixedPoint#(2, 23) re = unpack(pack(truncate(valueRe)));
    FixedPoint#(2, 23) im = unpack(pack(truncate(valueIm)));
      cs.setCoeff(addr, Complex{rel: re, img:im});
   endmethod
endmodule


1. reverting DmaUtils since the pipelining mods didn't work

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import BRAM::*;
import FIFO::*;
import Vector::*;
import Gearbox::*;
import FIFOF::*;
import SpecialFIFOs::*;

import BRAMFIFOFLevel::*;
import GetPut::*;
import Dma::*;

function ObjectReadClient#(dataWidth) orc(DmaReadBuffer#(dataWidth,bufferDepth) rb) = rb.dmaClient;
function ObjectWriteClient#(dataWidth) owc(DmaWriteBuffer#(dataWidth,bufferDepth) wb) = wb.dmaClient;
function ObjectReadServer#(dataWidth) ors(DmaReadBuffer#(dataWidth,bufferDepth) rb) = rb.dmaServer;
function ObjectWriteServer#(dataWidth) ows(DmaWriteBuffer#(dataWidth,bufferDepth) wb) = wb.dmaServer;

//
// @brief A buffer for reading from a bus of width dataWidth.
//
// @param dataWidth The number of bits in the bus.
// @param bufferDepth The depth of the internal buffer
//
interface DmaReadBuffer#(numeric type dataWidth, numeric type bufferDepth);
   interface ObjectReadServer #(dataWidth) dmaServer;
   interface ObjectReadClient#(dataWidth) dmaClient;
endinterface

//
// @brief A buffer for writing to a bus of width dataWidth.
//
// @param dataWidth The number of bits in the bus.
// @param bufferDepth The depth of the internal buffer
//
interface DmaWriteBuffer#(numeric type dataWidth, numeric type bufferDepth);
   interface ObjectWriteServer#(dataWidth) dmaServer;
   interface ObjectWriteClient#(dataWidth) dmaClient;
endinterface

//
// @brief Makes a Dma buffer for reading wordSize words from memory.
//
// @param dataWidth The width of the bus in bits.
// @param bufferDepth The depth of the internal buffer
//
module mkDmaReadBuffer(DmaReadBuffer#(dataWidth, bufferDepth))
   provisos(Add#(b__, TAdd#(1,TLog#(bufferDepth)), 8),
	    Div#(dataWidth,8,dataWidthBytes),
	    Mul#(dataWidthBytes,8,dataWidth),
	    Log#(dataWidthBytes,beatShift));

   FIFOFLevel#(ObjectData#(dataWidth),bufferDepth)  readBuffer <- mkBRAMFIFOFLevel;
   FIFOF#(ObjectRequest)        reqOutstanding <- mkFIFOF();
   Ratchet#(TAdd#(1,TLog#(bufferDepth))) unfulfilled <- mkRatchet(0);
   let beat_shift = fromInteger(valueOf(beatShift));

   // only issue the readRequest when sufficient buffering is available.  This includes the bufering we have already comitted.
   Bit#(TAdd#(1,TLog#(bufferDepth))) sreq = pack(satPlus(Sat_Bound, unpack(truncate(reqOutstanding.first.burstLen>>beat_shift)), unfulfilled.read()));

   interface ObjectReadServer dmaServer;
      interface Put readReq = toPut(reqOutstanding);
      interface Get readData = toGet(readBuffer);
   endinterface
   interface ObjectReadClient dmaClient;
      interface Get readReq;
	 method ActionValue#(ObjectRequest) get if (readBuffer.lowWater(sreq));
	    reqOutstanding.deq;
	    unfulfilled.increment(unpack(truncate(reqOutstanding.first.burstLen>>beat_shift)));
	    return reqOutstanding.first;
	 endmethod
      endinterface
      interface Put readData;
	 method Action put(ObjectData#(dataWidth) x);
	    readBuffer.fifo.enq(x);
	    unfulfilled.decrement(1);
	 endmethod
      endinterface
   endinterface
endmodule

//
// @brief Makes a Dma channel for writing wordSize words from memory.
//
// @param dataWidth The width of the bus in bits.
// @param bufferDepth The depth of the internal buffer
//
module mkDmaWriteBuffer(DmaWriteBuffer#(dataWidth, bufferDepth))
   provisos(Add#(b__, TAdd#(1, TLog#(bufferDepth)), 8),
	    Div#(dataWidth,8,dataWidthBytes),
	    Mul#(dataWidthBytes,8,dataWidth),
	    Log#(dataWidthBytes,beatShift));

   FIFOFLevel#(ObjectData#(dataWidth),bufferDepth) writeBuffer <- mkBRAMFIFOFLevel;
   FIFOF#(ObjectRequest)        reqOutstanding <- mkFIFOF();
   FIFOF#(Bit#(6))                        doneTags <- mkFIFOF();
   Ratchet#(TAdd#(1,TLog#(bufferDepth)))  unfulfilled <- mkRatchet(0);
   let beat_shift = fromInteger(valueOf(beatShift));

   // only issue the writeRequest when sufficient data is available.  This includes the data we have already comitted.
   Bit#(TAdd#(1,TLog#(bufferDepth))) sreq = pack(satPlus(Sat_Bound, unpack(truncate(reqOutstanding.first.burstLen>>beat_shift)), unfulfilled.read()));

   interface ObjectWriteServer dmaServer;
      interface Put writeReq = toPut(reqOutstanding);
      interface Put writeData = toPut(writeBuffer);
      interface Get writeDone = toGet(doneTags);
   endinterface
   interface ObjectWriteClient dmaClient;
      interface Get writeReq;
	 method ActionValue#(ObjectRequest) get if (writeBuffer.highWater(sreq));
	    reqOutstanding.deq;
	    unfulfilled.increment(unpack(truncate(reqOutstanding.first.burstLen>>beat_shift)));
	    return reqOutstanding.first;
	 endmethod
      endinterface
      interface Get writeData;
	 method ActionValue#(ObjectData#(dataWidth)) get();
	    unfulfilled.decrement(1);
	    writeBuffer.fifo.deq;
	    return writeBuffer.fifo.first;
	 endmethod
      endinterface
      interface Put writeDone = toPut(doneTags);
   endinterface
endmodule

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import BRAM::*;
import FIFO::*;
import Vector::*;
import Gearbox::*;
import FIFOF::*;
import SpecialFIFOs::*;

import BRAMFIFOFLevel::*;
import GetPut::*;
import Dma::*;

function ObjectReadClient#(dataWidth) orc(DmaReadBuffer#(dataWidth,bufferDepth) rb) = rb.dmaClient;
function ObjectWriteClient#(dataWidth) owc(DmaWriteBuffer#(dataWidth,bufferDepth) wb) = wb.dmaClient;
function ObjectReadServer#(dataWidth) ors(DmaReadBuffer#(dataWidth,bufferDepth) rb) = rb.dmaServer;
function ObjectWriteServer#(dataWidth) ows(DmaWriteBuffer#(dataWidth,bufferDepth) wb) = wb.dmaServer;

//
// @brief A buffer for reading from a bus of width dataWidth.
//
// @param dataWidth The number of bits in the bus.
// @param bufferDepth The depth of the internal buffer
//
interface DmaReadBuffer#(numeric type dataWidth, numeric type bufferDepth);
   interface ObjectReadServer #(dataWidth) dmaServer;
   interface ObjectReadClient#(dataWidth) dmaClient;
endinterface

//
// @brief A buffer for writing to a bus of width dataWidth.
//
// @param dataWidth The number of bits in the bus.
// @param bufferDepth The depth of the internal buffer
//
interface DmaWriteBuffer#(numeric type dataWidth, numeric type bufferDepth);
   interface ObjectWriteServer#(dataWidth) dmaServer;
   interface ObjectWriteClient#(dataWidth) dmaClient;
endinterface

//
// @brief Makes a Dma buffer for reading wordSize words from memory.
//
// @param dataWidth The width of the bus in bits.
// @param bufferDepth The depth of the internal buffer
//
module mkDmaReadBuffer(DmaReadBuffer#(dataWidth, bufferDepth))
   provisos(Add#(b__, TAdd#(1,TLog#(bufferDepth)), 8),
	    Div#(dataWidth,8,dataWidthBytes),
	    Mul#(dataWidthBytes,8,dataWidth),
	    Log#(dataWidthBytes,beatShift));

   FIFO#(ObjectData#(dataWidth))  readBuffer <- mkSizedFIFO(valueOf(bufferDepth));
   FIFOF#(ObjectRequest)        reqOutstanding <- mkFIFOF();
   FIFOF#(ObjectRequest)        reqReady       <- mkFIFOF();
   Ratchet#(TAdd#(1,TLog#(bufferDepth))) availableBuffers <- mkRatchet(fromInteger(valueOf(bufferDepth)));
   let beat_shift = fromInteger(valueOf(beatShift));

   Bit#(TAdd#(1,TLog#(bufferDepth))) requested = truncate(reqOutstanding.first.burstLen>>beat_shift);
   Bool ready = reqOutstanding.notEmpty && (unpack(requested) <= availableBuffers.read());
   // rule updateReady;
   //    Bit#(TAdd#(1,TLog#(bufferDepth))) requested = truncate(reqOutstanding.first.burstLen>>beat_shift);
   //    Bool ready = (unpack(requested) <= availableBuffers.read());
   //    if (ready) begin
   // 	 let req <- toGet(reqOutstanding).get();
   // 	 reqReady.enq(req);
   // 	 availableBuffers.decrement(unpack(requested));
   //    end
   // endrule

   // only issue the readRequest when sufficient buffering is available.  This includes the buffering we have already committed.
   interface ObjectReadServer dmaServer;
      interface Put readReq = toPut(reqOutstanding);
      interface Get readData;
	 method ActionValue#(ObjectData#(dataWidth)) get();
	    availableBuffers.increment(1);
	    let resp <- toGet(readBuffer).get();
	    return resp;
	 endmethod
      endinterface
   endinterface
   interface ObjectReadClient dmaClient;
      interface Get readReq;
	 method ActionValue#(ObjectRequest) get if (ready);
	    let req <- toGet(reqOutstanding).get();
	    availableBuffers.decrement(unpack(requested));
	    return req;
	 endmethod
      endinterface
      interface Put readData;
	 method Action put(ObjectData#(dataWidth) x);
	    readBuffer.enq(x);
	 endmethod
      endinterface
   endinterface
endmodule

//
// @brief Makes a Dma channel for writing wordSize words from memory.
//
// @param dataWidth The width of the bus in bits.
// @param bufferDepth The depth of the internal buffer
//
module mkDmaWriteBuffer(DmaWriteBuffer#(dataWidth, bufferDepth))
   provisos(Add#(b__, TAdd#(1, TLog#(bufferDepth)), 8),
	    Div#(dataWidth,8,dataWidthBytes),
	    Mul#(dataWidthBytes,8,dataWidth),
	    Log#(dataWidthBytes,beatShift));

   FIFO#(ObjectData#(dataWidth)) writeBuffer <- mkSizedFIFO(valueOf(bufferDepth));
   FIFOF#(ObjectRequest)        reqOutstanding <- mkFIFOF();
   FIFOF#(Bit#(ObjectTagSize))  doneTags <- mkFIFOF();
   Ratchet#(TAdd#(1,TLog#(bufferDepth))) availableWords <- mkRatchet(fromInteger(valueOf(bufferDepth)));
   let beat_shift = fromInteger(valueOf(beatShift));

   FIFOF#(Bool) hasEnoughCapacity <- mkFIFOF();
   rule updateReady;
      Bit#(TAdd#(1,TLog#(bufferDepth))) requested = truncate(reqOutstanding.first.burstLen>>beat_shift);
      if (unpack(truncate(reqOutstanding.first.burstLen>>beat_shift)) <= availableWords.read())
	 hasEnoughCapacity.enq(True);
   endrule

   // only issue the writeRequest when sufficient data is available.  This includes the data we have already committed.
   interface ObjectWriteServer dmaServer;
      interface Put writeReq = toPut(reqOutstanding);
      interface Put writeData;
	 method Action put(ObjectData#(dataWidth) d);
	    writeBuffer.enq(d);
	    availableWords.increment(1);
	 endmethod
      endinterface
      interface Get writeDone = toGet(doneTags);
   endinterface
   interface ObjectWriteClient dmaClient;
      interface Get writeReq;
	 method ActionValue#(ObjectRequest) get if (hasEnoughCapacity.notEmpty());
	    hasEnoughCapacity.deq();
	    reqOutstanding.deq;
	    availableWords.decrement(unpack(truncate(reqOutstanding.first.burstLen>>beat_shift)));
	    return reqOutstanding.first;
	 endmethod
      endinterface
      interface Get writeData;
	 method ActionValue#(ObjectData#(dataWidth)) get();
	    writeBuffer.deq;
	    return writeBuffer.first;
	 endmethod
      endinterface
      interface Put writeDone = toPut(doneTags);
   endinterface
endmodule

module mkDmaReadMux#(Vector#(numClients,ObjectReadClient#(dataWidth)) readClients)(ObjectReadClient#(dataWidth))
   provisos(Log#(numClients,tagsz),
	    Add#(tagsz,a__,ObjectTagSize));

   FIFO#(ObjectRequest)          readReqFifo  <- mkFIFO();
   FIFO#(ObjectData#(dataWidth)) readRespFifo <- mkFIFO();

   for (Integer i = 0; i < valueOf(numClients); i = i + 1) begin
      // assume fixed tag per client
      Reg#(Bit#(ObjectTagSize)) tagReg <- mkReg(0);
      rule getreq;
	 let req <- readClients[i].readReq.get();
	 tagReg <= req.tag;
	 req.tag = fromInteger(i);
	 readReqFifo.enq(req);
      endrule
      rule sendresp if (readRespFifo.first.tag == fromInteger(i));
	 let resp <- toGet(readRespFifo).get();
	 resp.tag = tagReg;
	 readClients[i].readData.put(resp);
      endrule
   end

   interface Get readReq = toGet(readReqFifo);
   interface Put readData = toPut(readRespFifo);
endmodule

module mkDmaWriteMux#(Vector#(numClients,ObjectWriteClient#(dataWidth)) writeClients)(ObjectWriteClient#(dataWidth))
   provisos(Log#(numClients,tagsz),
	    Add#(tagsz,a__,ObjectTagSize));

   FIFO#(ObjectRequest)          writeReqFifo  <- mkFIFO();
   FIFO#(ObjectData#(dataWidth)) writeDataFifo <- mkFIFO();
   FIFO#(Bit#(ObjectTagSize))    writeDoneFifo <- mkFIFO();
   FIFO#(Bit#(ObjectTagSize))    arbFifo       <- mkFIFO();

   for (Integer i = 0; i < valueOf(numClients); i = i + 1) begin
      // assume fixed tag per client
      Reg#(Bit#(ObjectTagSize)) tagReg <- mkReg(0);
      rule getreq;
	 let req <- writeClients[i].writeReq.get();
	 tagReg <= req.tag;
	 req.tag = fromInteger(i);
	 writeReqFifo.enq(req);
	 arbFifo.enq(req.tag);
      endrule
      rule senddata if (fromInteger(i) == arbFifo.first);
	 let data <- writeClients[i].writeData.get();
	 data.tag = tagReg;
	 writeDataFifo.enq(data);
      endrule
      rule senddone if (writeDoneFifo.first == fromInteger(i));
	 arbFifo.deq();
	 let done <- toGet(writeDoneFifo).get();
	 writeClients[i].writeDone.put(tagReg);
      endrule
   end

   interface Get writeReq = toGet(writeReqFifo);
   interface Get writeData = toGet(writeDataFifo);
   interface Put writeDone = toPut(writeDoneFifo);
endmodule


pipeline check of upper and lower half of interrupt address

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import FIFOF        :: *;
import Vector       :: *;
import GetPut       :: *;
import Connectable  :: *;
import MIMO         :: *;
import PCIE         :: *;
import DefaultValue :: *;
import ClientServer :: *;
import MemTypes     :: *;

//
// Top interface: PCIe transaction level packets (TLPs)
// Bottom interface: MemMaster that sends read/write requests to an MemSlave
// Also sources interrupt MSIX requests
interface MemMasterEngine;
    interface Client#(TLPData#(16), TLPData#(16)) tlp;
    interface MemMaster#(32,32) master;
endinterface

(* synthesize *)
module mkMemMasterEngine#(PciId my_id)(MemMasterEngine);
    Reg#(Bit#(7)) hitReg <- mkReg(0);
    FIFOF#(TLPMemoryIO3DWHeader) readHeaderFifo <- mkSizedFIFOF(8);
    FIFOF#(TLPMemoryIO3DWHeader) readDataFifo <- mkSizedFIFOF(8);
    FIFOF#(TLPMemoryIO3DWHeader) writeHeaderFifo <- mkSizedFIFOF(8);
    FIFOF#(TLPMemoryIO3DWHeader) writeDataFifo <- mkSizedFIFOF(8);
    FIFOF#(TLPData#(16)) tlpOutFifo <- mkSizedFIFOF(8);
    Reg#(TLPTag) tlpTag <- mkReg(0);

    MIMOConfiguration mimoCfg = defaultValue;
    MIMO#(1,4,4,Bit#(32)) completionMimo <- mkMIMO(mimoCfg);
   Reg#(TLPLength)             readBurstCount <- mkReg(0);
   Reg#(Bool)      readBurstCountGreaterThan4 <- mkReg(False);
   Reg#(Bool)                  readInProgress <- mkReg(False);
   rule completionHeader if (!readInProgress && readDataFifo.notEmpty() && completionMimo.deqReadyN(1));
      let hdr = readDataFifo.first;
      TLPLength rbc = hdr.length;

      Vector#(4, Bit#(32)) dvec = completionMimo.first();
      completionMimo.deq(1);

      //$display("completionHeader length=%d rbc=%d addr=%x", hdr.length, rbc, hdr.addr);
      TLPCompletionHeader completion = defaultValue;
      completion.format = MEM_WRITE_3DW_DATA;
      completion.pkttype = COMPLETION;
      completion.relaxed = hdr.relaxed;
      completion.nosnoop = hdr.nosnoop;
      completion.length = hdr.length;
      completion.tclass = hdr.tclass;
      completion.cmplid = my_id;
      completion.tag = truncate(hdr.tag);
      completion.bytecount = 4;
      completion.reqid = hdr.reqid;
      completion.loweraddr = getLowerAddr(hdr.addr, hdr.firstbe);
      completion.data = byteSwap(dvec[0]);
      TLPData#(16) tlp = defaultValue;
      tlp.data = pack(completion);
      tlp.sof = True;
      tlp.eof = (rbc == 1) ? True : False;
      tlp.be = 16'hFFFF;
      tlp.hit = hitReg;
      tlpOutFifo.enq(tlp);

      rbc = rbc - 1;
      readBurstCount <= rbc;
      readInProgress <= (rbc != 0);
      readBurstCountGreaterThan4 <= (rbc > 4);
      if (rbc == 0) begin
	 readDataFifo.deq;
      end
   endrule

    function Bit#(16) tlpBe(TLPLength len);
       if (len == 0)
	  return 0;
       else if (len == 1)
	  return 16'hf000;
       else if (len == 2)
	  return 16'hff00;
       else if (len == 3)
	  return 16'hfff0;
       else
	  return 16'hffff;
    endfunction

   rule continuation if (readInProgress);
      let rbc = readBurstCount;
      let sendit = False;
      TLPData#(16) tlp = defaultValue;
      Vector#(4, Bit#(32)) dvec = unpack(0);
      tlp.sof = False;
      //$display("continuation rbc=%d", rbc);
      if (readBurstCountGreaterThan4) begin
	 if (completionMimo.deqReadyN(4)) begin
	    rbc = rbc - 4;
	    dvec = completionMimo.first();
	    completionMimo.deq(4);
	    tlp.be = tlpBe(4);
	    tlp.eof = False;
	    sendit = True;
	 end
      end
      else begin
	 UInt#(3) deqCount = truncate(unpack(rbc));
	 if (completionMimo.deqReadyN(deqCount)) begin
	    dvec = completionMimo.first();
	    completionMimo.deq(deqCount);
	    tlp.be = tlpBe(rbc);
	    //$display("tlp.data=%h tlp.be=%h", tlp.data, tlp.be);
	    tlp.eof = True;
	    rbc = 0;
	    sendit = True;
	 end
      end

      readBurstCount <= rbc;
      readBurstCountGreaterThan4 <= (rbc > 4);
      if (rbc == 0) begin
	 readDataFifo.deq();
	 readInProgress <= False;
      end
      if (sendit) begin
	 for (Integer i = 0; i < 4; i = i + 1)
	    tlp.data[(i+1)*32-1:i*32] = byteSwap(dvec[3-i]);
	 tlpOutFifo.enq(tlp);
      end
   endrule

    interface Client        tlp;
    interface Put response;
        method Action put(TLPData#(16) tlp);
	    //$display("MemMasterEngine.put tlp=%h", tlp);
	    TLPMemoryIO3DWHeader h = unpack(tlp.data);
	    hitReg <= tlp.hit;
	    TLPMemoryIO3DWHeader hdr_3dw = unpack(tlp.data);
	    if (hdr_3dw.format == MEM_READ_3DW_NO_DATA) begin
	       if (readHeaderFifo.notFull())
	          readHeaderFifo.enq(hdr_3dw);
	       else begin
		  // FIXME: should generate a response or host will lock up
	       end
	    end
	    else begin
	       if (writeHeaderFifo.notFull())
		  writeHeaderFifo.enq(hdr_3dw);
	    end
	endmethod
    endinterface
    interface Get request = toGet(tlpOutFifo);
    endinterface: tlp
    interface MemMaster master;
    interface MemWriteClient write_client;
        interface Get    writeReq;
	  method ActionValue#(MemRequest#(32)) get();
	     let hdr = writeHeaderFifo.first;
	     writeHeaderFifo.deq;
	     writeDataFifo.enq(hdr);
	     let axilen = hdr.length;
	     return MemRequest { addr: extend(writeHeaderFifo.first.addr) << 2, burstLen: truncate(axilen), tag: truncate(writeHeaderFifo.first.tag)};
	  endmethod
       endinterface
        interface Get writeData;
	  method ActionValue#(MemData#(32)) get();
	     let hdr <- toGet(writeDataFifo).get();
	     let data = byteSwap(hdr.data);
	     return MemData { data: data, tag: truncate(hdr.tag), last: True};
	  endmethod
       endinterface
        interface Put       writeDone;
	  method Action put(Bit#(ObjectTagSize) resp);
	  endmethod
       endinterface
     endinterface
    interface MemReadClient read_client;
        interface Get    readReq;
	  method ActionValue#(MemRequest#(32)) get();
	     let hdr = readHeaderFifo.first;
	     readHeaderFifo.deq;
	     //$display("req_ar hdr.length=%d hdr.addr=%h", hdr.length, hdr.addr);
	     readDataFifo.enq(hdr);
	     let axilen = hdr.length;
	     return MemRequest { addr: extend(readHeaderFifo.first.addr) << 2, burstLen: truncate(axilen), tag: truncate(readHeaderFifo.first.tag)};
	    endmethod
       endinterface
        interface Put readData;
	  method Action put(MemData#(32) resp) if (completionMimo.enqReadyN(1));
	     Vector#(1, Bit#(32)) vec = cons(resp.data, nil);
	     completionMimo.enq(1, vec);
	  endmethod
	endinterface
    endinterface
    endinterface: master
endmodule: mkMemMasterEngine

interface MemInterrupt;
    interface Client#(TLPData#(16), TLPData#(16)) tlp;
    interface Put#(Tuple2#(Bit#(64),Bit#(32))) interruptRequest;
endinterface

//(* synthesize *)
module mkMemInterrupt#(PciId my_id)(MemInterrupt);
    FIFOF#(Tuple4#(Bit#(64),Bit#(32),Bool,Bool)) interruptRequestFifo <- mkFIFOF();
    Reg#(Maybe#(Bit#(32))) interruptSecondHalf <- mkReg(tagged Invalid);
    Reg#(TLPTag) tlpTag <- mkReg(0);
    FIFOF#(TLPData#(16)) tlpOutFifo <- mkSizedFIFOF(8);

    rule interruptTlpOut if (interruptRequestFifo.notEmpty &&& interruptSecondHalf matches tagged Invalid);
       TLPData#(16) tlp = defaultValue;
       tlp.sof = True;
       tlp.eof = False;
       tlp.hit = 7'h00;
       tlp.be = 16'hffff;

       let interruptRequested = True;
       let sendInterrupt = False;

       match { .interruptAddr, .interruptData, .mswIsZero, .lswIsZero } = interruptRequestFifo.first;
       if (mswIsZero && lswIsZero) begin
	  // do not write to 0 -- it wedges the host
	  interruptRequested = False;
       end
       else if (mswIsZero) begin
          TLPMemoryIO3DWHeader hdr_3dw = defaultValue();
          hdr_3dw.format = MEM_WRITE_3DW_DATA;
	  //hdr_3dw.pkttype = MEM_READ_WRITE;
          hdr_3dw.tag = tlpTag;
          hdr_3dw.reqid = my_id;
          hdr_3dw.length = 1;
          hdr_3dw.firstbe = '1;
          hdr_3dw.lastbe = '0;
          hdr_3dw.addr = interruptAddr[31:2];
	  hdr_3dw.data = byteSwap(interruptData);
	  tlp.data = pack(hdr_3dw);
	  tlp.eof = True;
	  sendInterrupt = True;
	  interruptRequested = False;
       end
       else begin
	  TLPMemory4DWHeader hdr_4dw = defaultValue;
	  hdr_4dw.format = MEM_WRITE_4DW_DATA;
	  //hdr_4dw.pkttype = MEM_READ_WRITE;
	  hdr_4dw.tag = tlpTag;
	  hdr_4dw.reqid = my_id;
	  hdr_4dw.nosnoop = SNOOPING_REQD;
	  hdr_4dw.addr = interruptAddr[40-1:2];
	  hdr_4dw.length = 1;
	  hdr_4dw.firstbe = 4'hf;
	  hdr_4dw.lastbe = 0;
	  tlp.data = pack(hdr_4dw);

	  sendInterrupt = True;
	  interruptSecondHalf <= tagged Valid interruptData;
       end

       if (!interruptRequested)
	  interruptRequestFifo.deq();
       if (sendInterrupt)
	  tlpOutFifo.enq(tlp);
    endrule

    rule interruptTlpDataOut if (interruptSecondHalf matches tagged Valid .interruptData);
       TLPData#(16) tlp = defaultValue;
       tlp.sof = False;
       tlp.eof = True;
       tlp.hit = 7'h00;
       tlp.be = 16'hf000;
       tlp.data[7+8*15:8*12] = byteSwap(interruptData);
       tlpOutFifo.enq(tlp);
       interruptSecondHalf <= tagged Invalid;
       interruptRequestFifo.deq();
    endrule

    interface Client        tlp;
    interface Put response;
        method Action put(TLPData#(16) tlp);
	endmethod
    endinterface
    interface Get request = toGet(tlpOutFifo);
    endinterface: tlp
    interface Put interruptRequest;
       method Action put(Tuple2#(Bit#(64),Bit#(32)) intr);
	  match { .addr, .data } = intr;
	  Bool mswIsZero = (addr[63:32] == 0);
	  Bool lswIsZero = (addr[31:0] == 0);
          interruptRequestFifo.enq(tuple4(addr, data, lswIsZero, mswIsZero));
       endmethod
    endinterface
endmodule: mkMemInterrupt

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import FIFOF        :: *;
import Vector       :: *;
import GetPut       :: *;
import Connectable  :: *;
import MIMO         :: *;
import PCIE         :: *;
import DefaultValue :: *;
import ClientServer :: *;
import MemTypes     :: *;

//
// Top interface: PCIe transaction level packets (TLPs)
// Bottom interface: MemMaster that sends read/write requests to an MemSlave
// Also sources interrupt MSIX requests
interface MemMasterEngine;
    interface Client#(TLPData#(16), TLPData#(16)) tlp;
    interface MemMaster#(32,32) master;
endinterface

(* synthesize *)
module mkMemMasterEngine#(PciId my_id)(MemMasterEngine);
    Reg#(Bit#(7)) hitReg <- mkReg(0);
    FIFOF#(TLPMemoryIO3DWHeader) readHeaderFifo <- mkSizedFIFOF(8);
    FIFOF#(TLPMemoryIO3DWHeader) readDataFifo <- mkSizedFIFOF(8);
    FIFOF#(TLPMemoryIO3DWHeader) writeHeaderFifo <- mkSizedFIFOF(8);
    FIFOF#(TLPMemoryIO3DWHeader) writeDataFifo <- mkSizedFIFOF(8);
    FIFOF#(TLPData#(16)) tlpOutFifo <- mkSizedFIFOF(8);
    Reg#(TLPTag) tlpTag <- mkReg(0);

    MIMOConfiguration mimoCfg = defaultValue;
    MIMO#(1,4,4,Bit#(32)) completionMimo <- mkMIMO(mimoCfg);
   Reg#(TLPLength)             readBurstCount <- mkReg(0);
   Reg#(Bool)      readBurstCountGreaterThan4 <- mkReg(False);
   Reg#(Bool)                  readInProgress <- mkReg(False);
   rule completionHeader if (!readInProgress && readDataFifo.notEmpty() && completionMimo.deqReadyN(1));
      let hdr = readDataFifo.first;
      TLPLength rbc = hdr.length;

      Vector#(4, Bit#(32)) dvec = completionMimo.first();
      completionMimo.deq(1);

      //$display("completionHeader length=%d rbc=%d addr=%x", hdr.length, rbc, hdr.addr);
      TLPCompletionHeader completion = defaultValue;
      completion.format = MEM_WRITE_3DW_DATA;
      completion.pkttype = COMPLETION;
      completion.relaxed = hdr.relaxed;
      completion.nosnoop = hdr.nosnoop;
      completion.length = hdr.length;
      completion.tclass = hdr.tclass;
      completion.cmplid = my_id;
      completion.tag = truncate(hdr.tag);
      completion.bytecount = 4;
      completion.reqid = hdr.reqid;
      completion.loweraddr = getLowerAddr(hdr.addr, hdr.firstbe);
      completion.data = byteSwap(dvec[0]);
      TLPData#(16) tlp = defaultValue;
      tlp.data = pack(completion);
      tlp.sof = True;
      tlp.eof = (rbc == 1) ? True : False;
      tlp.be = 16'hFFFF;
      tlp.hit = hitReg;
      tlpOutFifo.enq(tlp);

      rbc = rbc - 1;
      readBurstCount <= rbc;
      readInProgress <= (rbc != 0);
      readBurstCountGreaterThan4 <= (rbc > 4);
      if (rbc == 0) begin
	 readDataFifo.deq;
      end
   endrule

    function Bit#(16) tlpBe(TLPLength len);
       if (len == 0)
	  return 0;
       else if (len == 1)
	  return 16'hf000;
       else if (len == 2)
	  return 16'hff00;
       else if (len == 3)
	  return 16'hfff0;
       else
	  return 16'hffff;
    endfunction

   rule continuation if (readInProgress);
      let rbc = readBurstCount;
      let sendit = False;
      TLPData#(16) tlp = defaultValue;
      Vector#(4, Bit#(32)) dvec = unpack(0);
      tlp.sof = False;
      //$display("continuation rbc=%d", rbc);
      if (readBurstCountGreaterThan4) begin
	 if (completionMimo.deqReadyN(4)) begin
	    rbc = rbc - 4;
	    dvec = completionMimo.first();
	    completionMimo.deq(4);
	    tlp.be = tlpBe(4);
	    tlp.eof = False;
	    sendit = True;
	 end
      end
      else begin
	 UInt#(3) deqCount = truncate(unpack(rbc));
	 if (completionMimo.deqReadyN(deqCount)) begin
	    dvec = completionMimo.first();
	    completionMimo.deq(deqCount);
	    tlp.be = tlpBe(rbc);
	    //$display("tlp.data=%h tlp.be=%h", tlp.data, tlp.be);
	    tlp.eof = True;
	    rbc = 0;
	    sendit = True;
	 end
      end

      readBurstCount <= rbc;
      readBurstCountGreaterThan4 <= (rbc > 4);
      if (rbc == 0) begin
	 readDataFifo.deq();
	 readInProgress <= False;
      end
      if (sendit) begin
	 for (Integer i = 0; i < 4; i = i + 1)
	    tlp.data[(i+1)*32-1:i*32] = byteSwap(dvec[3-i]);
	 tlpOutFifo.enq(tlp);
      end
   endrule

    interface Client        tlp;
    interface Put response;
        method Action put(TLPData#(16) tlp);
	    //$display("MemMasterEngine.put tlp=%h", tlp);
	    TLPMemoryIO3DWHeader h = unpack(tlp.data);
	    hitReg <= tlp.hit;
	    TLPMemoryIO3DWHeader hdr_3dw = unpack(tlp.data);
	    if (hdr_3dw.format == MEM_READ_3DW_NO_DATA) begin
	       if (readHeaderFifo.notFull())
	          readHeaderFifo.enq(hdr_3dw);
	       else begin
		  // FIXME: should generate a response or host will lock up
	       end
	    end
	    else begin
	       if (writeHeaderFifo.notFull())
		  writeHeaderFifo.enq(hdr_3dw);
	    end
	endmethod
    endinterface
    interface Get request = toGet(tlpOutFifo);
    endinterface: tlp
    interface MemMaster master;
    interface MemWriteClient write_client;
        interface Get    writeReq;
	  method ActionValue#(MemRequest#(32)) get();
	     let hdr = writeHeaderFifo.first;
	     writeHeaderFifo.deq;
	     writeDataFifo.enq(hdr);
	     let axilen = hdr.length;
	     return MemRequest { addr: extend(writeHeaderFifo.first.addr) << 2, burstLen: truncate(axilen), tag: truncate(writeHeaderFifo.first.tag)};
	  endmethod
       endinterface
        interface Get writeData;
	  method ActionValue#(MemData#(32)) get();
	     let hdr <- toGet(writeDataFifo).get();
	     let data = byteSwap(hdr.data);
	     return MemData { data: data, tag: truncate(hdr.tag), last: True};
	  endmethod
       endinterface
        interface Put       writeDone;
	  method Action put(Bit#(ObjectTagSize) resp);
	  endmethod
       endinterface
     endinterface
    interface MemReadClient read_client;
        interface Get    readReq;
	  method ActionValue#(MemRequest#(32)) get();
	     let hdr = readHeaderFifo.first;
	     readHeaderFifo.deq;
	     //$display("req_ar hdr.length=%d hdr.addr=%h", hdr.length, hdr.addr);
	     readDataFifo.enq(hdr);
	     let axilen = hdr.length;
	     return MemRequest { addr: extend(readHeaderFifo.first.addr) << 2, burstLen: truncate(axilen), tag: truncate(readHeaderFifo.first.tag)};
	    endmethod
       endinterface
        interface Put readData;
	  method Action put(MemData#(32) resp) if (completionMimo.enqReadyN(1));
	     Vector#(1, Bit#(32)) vec = cons(resp.data, nil);
	     completionMimo.enq(1, vec);
	  endmethod
	endinterface
    endinterface
    endinterface: master
endmodule: mkMemMasterEngine

interface MemInterrupt;
    interface Client#(TLPData#(16), TLPData#(16)) tlp;
    interface Put#(Tuple2#(Bit#(64),Bit#(32))) interruptRequest;
endinterface

//(* synthesize *)
module mkMemInterrupt#(PciId my_id)(MemInterrupt);
    FIFOF#(Tuple2#(Bit#(64),Bit#(32))) interruptRequestFifo <- mkSizedFIFOF(16);
    Reg#(Maybe#(Bit#(32))) interruptSecondHalf <- mkReg(tagged Invalid);
    Reg#(TLPTag) tlpTag <- mkReg(0);
    FIFOF#(TLPData#(16)) tlpOutFifo <- mkSizedFIFOF(8);

    rule interruptTlpOut if (interruptRequestFifo.notEmpty &&& interruptSecondHalf matches tagged Invalid);
       TLPData#(16) tlp = defaultValue;
       tlp.sof = True;
       tlp.eof = False;
       tlp.hit = 7'h00;
       tlp.be = 16'hffff;

       let interruptRequested = True;
       let sendInterrupt = False;

       Bit#(64) interruptAddr = tpl_1(interruptRequestFifo.first);
       Bit#(32) interruptData = tpl_2(interruptRequestFifo.first);
       if (interruptAddr == '0) begin
	  // do not write to 0 -- it wedges the host
	  interruptRequested = False;
       end
       else if (interruptAddr[63:32] == '0) begin
          TLPMemoryIO3DWHeader hdr_3dw = defaultValue();
          hdr_3dw.format = MEM_WRITE_3DW_DATA;
	  //hdr_3dw.pkttype = MEM_READ_WRITE;
          hdr_3dw.tag = tlpTag;
          hdr_3dw.reqid = my_id;
          hdr_3dw.length = 1;
          hdr_3dw.firstbe = '1;
          hdr_3dw.lastbe = '0;
          hdr_3dw.addr = interruptAddr[31:2];
	  hdr_3dw.data = byteSwap(interruptData);
	  tlp.data = pack(hdr_3dw);
	  tlp.eof = True;
	  sendInterrupt = True;
	  interruptRequested = False;
       end
       else begin
	  TLPMemory4DWHeader hdr_4dw = defaultValue;
	  hdr_4dw.format = MEM_WRITE_4DW_DATA;
	  //hdr_4dw.pkttype = MEM_READ_WRITE;
	  hdr_4dw.tag = tlpTag;
	  hdr_4dw.reqid = my_id;
	  hdr_4dw.nosnoop = SNOOPING_REQD;
	  hdr_4dw.addr = interruptAddr[40-1:2];
	  hdr_4dw.length = 1;
	  hdr_4dw.firstbe = 4'hf;
	  hdr_4dw.lastbe = 0;
	  tlp.data = pack(hdr_4dw);

	  sendInterrupt = True;
	  interruptSecondHalf <= tagged Valid interruptData;
       end

       if (!interruptRequested)
	  interruptRequestFifo.deq();
       if (sendInterrupt)
	  tlpOutFifo.enq(tlp);
    endrule

    rule interruptTlpDataOut if (interruptSecondHalf matches tagged Valid .interruptData);
       TLPData#(16) tlp = defaultValue;
       tlp.sof = False;
       tlp.eof = True;
       tlp.hit = 7'h00;
       tlp.be = 16'hf000;
       tlp.data[7+8*15:8*12] = byteSwap(interruptData);
       tlpOutFifo.enq(tlp);
       interruptSecondHalf <= tagged Invalid;
       interruptRequestFifo.deq();
    endrule

    interface Client        tlp;
    interface Put response;
        method Action put(TLPData#(16) tlp);
	endmethod
    endinterface
    interface Get request = toGet(tlpOutFifo);
    endinterface: tlp
    interface Put interruptRequest;
       method Action put(Tuple2#(Bit#(64),Bit#(32)) intr);
          interruptRequestFifo.enq(intr);
       endmethod
    endinterface
endmodule: mkMemInterrupt