better debug printing for arbitrate

better debug printing for arbitrate

// Copyright (c) 2014 Quanta Research Cambridge, Inc.
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import Connectable::*;
import SerialFIFO::*;
import FIFOF::*;
import Vector::*;
import Pipe::*;
import Arbiter::*;

typedef struct {
   Vector#(2, Bit#(4)) address;
   Bit#(32) payload;
   } DataMessage deriving(Bits);

interface NocNode#(numeric type dim);
   interface PipeIn#(DataMessage) hosttonode;
   interface PipeOut#(DataMessage) nodetohost;
   interface Vector#(dim, PipeIn#(Bit#(1))) linkupin;
   interface Vector#(dim, PipeOut#(Bit#(1))) linkupout;
   interface Vector#(dim, PipeIn#(Bit#(1))) linkdownin;
   interface Vector#(dim, PipeOut#(Bit#(1))) linkdownout;
endinterface

function PipeOut#(Bit#(1)) selectoutput(SerialFIFOTX#(DataMessage) x);
   return x.out;
endfunction

function PipeIn#(Bit#(1)) selectinput(SerialFIFORX#(DataMessage) x);
   return x.in;
endfunction

function Action move(PipeOut#(DataMessage) from, PipeIn#(DataMessage) to);
   return action
	     $display("move %x", from.first);
	     to.enq(from.first);
	     from.deq();
	  endaction;
endfunction

module mkNocArbitrate#(Vector#(n, Bit#(4)) id, Vector#(r, PipeOut#(a)) in, PipeIn#(a) out)(Empty)
/*   provisos(
      Add#(0,n,2),
      Add#(0,r,5)
      ) */ ;
   Arbiter_IFC#(r) arb <- mkArbiter(False);
   for (Integer i = 0; i < valueOf(r); i = i + 1)
      rule send_request (out.notFull && in[i].notEmpty);
	 arb.clients[i].request();
      endrule

   rule move;
      if (out.notFull && in[arb.grant_id].notEmpty)
	 action
	    $display("arb id [%d,%d] link %d from %d", id[0], id[1], i, arb.grant_id);
	    out.enq(in[arb.grant_id].first());
	    in[arb.grant_id].deq();
	 endaction
   endrule
endmodule

module mkDistributor#(Vector#(n, Bit#(4)) id, PipeOut#(DataMessage) in, Vector#(r, PipeIn#(DataMessage)) out)(Empty)
/*    provisos(
      Add#(0,n,2),
      Add#(0,r,5)
      ) */
;
   rule move;
      $display("distrib [%d,%d] to [%d,%d] v %x",
	 id[0], id[1], in.first.address[0], in.first.address[1],
	 in.first.payload);
      if (in.first.address[0] < id[0])
	 begin
	    $display(" to link 1");
	    move(in, out[1]);
	 end
      else if (in.first.address[0] > id[0])
	 begin
	    $display(" to link 0");
	    move(in, out[0]);
	 end
      else /* in.first.address[0] == id[0] */
	 begin
	    if (in.first.address[1] < id[1])
	       begin
		  $display(" to link 3");
		  move (in, out[3]);
	       end
	    else if (in.first.address[1] > id[1])
	       begin
		  $display(" to link 2");
		  move (in, out[2]);
	       end
	    else
	       begin
		  $display(" to link 4");
		  move(in, out[4]);
	       end
	 end
   endrule
endmodule

// This makes a FIFO which throws away data and is never ready to read
module mkDiscard(FIFOF#(DataMessage));
   method Action enq(DataMessage d);
   endmethod
   method Bool notFull;
      return True;
   endmethod
   method DataMessage first() = ?;
   method Bool notEmpty();
      return False;
   endmethod
   method Action deq;
      if (False) noAction;
   endmethod
   method Action clear = ?;
endmodule

typedef 2 NumDims;
(* synthesize *)
module mkNocNode#(Vector#(NumDims, Bit#(4)) id)(NocNode#(NumDims));
   Integer radix = (valueOf(NumDims) * 2) + 1;

   // host Links
   FIFOF#(DataMessage) fifofromhost <- mkSizedFIFOF(4);
   FIFOF#(DataMessage) fifotohost <- mkSizedFIFOF(4);
   PipeIn#(DataMessage) tohost = toPipeIn(fifotohost);
   PipeOut#(DataMessage) fromhost = toPipeOut(fifofromhost);

   Vector#(NumDims, SerialFIFOTX#(DataMessage)) txup <- replicateM(mkSerialFIFOTX);
   Vector#(NumDims, SerialFIFORX#(DataMessage)) rxup <- replicateM(mkSerialFIFORX);
   Vector#(NumDims, SerialFIFOTX#(DataMessage)) txdown <- replicateM(mkSerialFIFOTX);
   Vector#(NumDims, SerialFIFORX#(DataMessage)) rxdown <- replicateM(mkSerialFIFORX);

   // sources
   Vector#(TAdd#(TMul#(NumDims, 2), 1), PipeOut#(DataMessage)) switchin = newVector;
   Vector#(TAdd#(TMul#(NumDims, 2), 1), PipeIn#(DataMessage)) switchout = newVector;
   // buffers for crossbar switch
   Vector#(TAdd#(TMul#(NumDims, 2), 1), Vector#(TAdd#(TMul#(NumDims, 2), 1), FIFOF#(DataMessage))) xp = replicate(newVector);
   // temps for building switch
   Vector#(TAdd#(TMul#(NumDims, 2), 1), PipeIn#(DataMessage)) tmpin = newVector;
   Vector#(TAdd#(TMul#(NumDims, 2), 1), PipeOut#(DataMessage)) tmpout = newVector;

   for (Integer i = 0; i < valueOf(NumDims); i = i + 1)
      begin
	 switchin[(2*i) + 0] = rxup[i].out;
	 switchin[(2*i) + 1] = rxdown[i].out;
	 switchout[(2*i) + 0] = txup[i].in;
	 switchout[(2*i) + 1] = txdown[i].in;
      end
   switchin[radix - 1] = fromhost;
   switchout[radix - 1] = tohost;

   for (Integer x = 0; x < radix; x = x + 1)
      for (Integer y = 0; y < radix; y = y + 1)
	 if (x != y) xp[x][y] <- mkSizedFIFOF(4);

   for (Integer x = 0; x < (radix - 1); x = x + 1)
      xp[x][x] <- mkDiscard();

   xp[radix - 1][radix - 1] <- mkSizedFIFOF(4);

   // create distributors

   for (Integer x = 0; x < radix; x = x + 1)
      begin
	 for (Integer y = 0; y < radix; y = y + 1)
	    tmpin[y] = toPipeIn(xp[x][y]);
	 mkDistributor(id, switchin[x], tmpin);
      end
   // create arbiters
   for (Integer y = 0; y < radix; y = y + 1)
      begin
	 for (Integer x = 0; x < radix; x = x + 1)
	    tmpout[x] = toPipeOut(xp[x][y]);
	 mkNocArbitrate(id, tmpout, switchout[y]);
      end

  // interface wiring

   interface PipeIn hosttonode = toPipeIn(fifofromhost);
   interface PipeOut nodetohost = toPipeOut(fifotohost);
   interface Vector linkupin = map(selectinput, rxup);
   interface Vector linkupout = map(selectoutput, txup);
   interface Vector linkdownin = map(selectinput, rxdown);
   interface Vector linkdownout = map(selectoutput, txdown);

endmodule

// Copyright (c) 2014 Quanta Research Cambridge, Inc.
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import Connectable::*;
import SerialFIFO::*;
import FIFOF::*;
import Vector::*;
import Pipe::*;
import Arbiter::*;

typedef struct {
   Vector#(2, Bit#(4)) address;
   Bit#(32) payload;
   } DataMessage deriving(Bits);

interface NocNode#(numeric type dim);
   interface PipeIn#(DataMessage) hosttonode;
   interface PipeOut#(DataMessage) nodetohost;
   interface Vector#(dim, PipeIn#(Bit#(1))) linkupin;
   interface Vector#(dim, PipeOut#(Bit#(1))) linkupout;
   interface Vector#(dim, PipeIn#(Bit#(1))) linkdownin;
   interface Vector#(dim, PipeOut#(Bit#(1))) linkdownout;
endinterface

function PipeOut#(Bit#(1)) selectoutput(SerialFIFOTX#(DataMessage) x);
   return x.out;
endfunction

function PipeIn#(Bit#(1)) selectinput(SerialFIFORX#(DataMessage) x);
   return x.in;
endfunction

function Action move(PipeOut#(DataMessage) from, PipeIn#(DataMessage) to);
   return action
	     $display("move %x", from.first);
	     to.enq(from.first);
	     from.deq();
	  endaction;
endfunction

module mkNocArbitrate#(Vector#(n, Bit#(4)) id, Vector#(r, PipeOut#(a)) in, PipeIn#(a) out)(Empty)
/*   provisos(
      Add#(0,n,2),
      Add#(0,r,5)
      ) */ ;
   Arbiter_IFC#(r) arb <- mkArbiter(False);
   for (Integer i = 0; i < valueOf(r); i = i + 1)
      rule send_request (out.notFull && in[i].notEmpty);
	 arb.clients[i].request();
      endrule

   rule move;
      if (out.notFull && in[arb.grant_id].notEmpty)
	 action
	    $display("arb id [%d,%d] from %d", id[0], id[1], arb.grant_id);
	    out.enq(in[arb.grant_id].first());
	    in[arb.grant_id].deq();
	 endaction
   endrule
endmodule

module mkDistributor#(Vector#(n, Bit#(4)) id, PipeOut#(DataMessage) in, Vector#(r, PipeIn#(DataMessage)) out)(Empty)
/*    provisos(
      Add#(0,n,2),
      Add#(0,r,5)
      ) */
;
   rule move;
      $display("distrib [%d,%d] to [%d,%d] v %x",
	 id[0], id[1], in.first.address[0], in.first.address[1],
	 in.first.payload);
      if (in.first.address[0] < id[0])
	 begin
	    $display(" to link 1");
	    move(in, out[1]);
	 end
      else if (in.first.address[0] > id[0])
	 begin
	    $display(" to link 0");
	    move(in, out[0]);
	 end
      else /* in.first.address[0] == id[0] */
	 begin
	    if (in.first.address[1] < id[1])
	       begin
		  $display(" to link 3");
		  move (in, out[3]);
	       end
	    else if (in.first.address[1] > id[1])
	       begin
		  $display(" to link 2");
		  move (in, out[2]);
	       end
	    else
	       begin
		  $display(" to link 4");
		  move(in, out[4]);
	       end
	 end
   endrule
endmodule

// This makes a FIFO which throws away data and is never ready to read
module mkDiscard(FIFOF#(DataMessage));
   method Action enq(DataMessage d);
   endmethod
   method Bool notFull;
      return True;
   endmethod
   method DataMessage first() = ?;
   method Bool notEmpty();
      return False;
   endmethod
   method Action deq;
      if (False) noAction;
   endmethod
   method Action clear = ?;
endmodule

typedef 2 NumDims;
(* synthesize *)
module mkNocNode#(Vector#(NumDims, Bit#(4)) id)(NocNode#(NumDims));
   Integer radix = (valueOf(NumDims) * 2) + 1;

   // host Links
   FIFOF#(DataMessage) fifofromhost <- mkSizedFIFOF(4);
   FIFOF#(DataMessage) fifotohost <- mkSizedFIFOF(4);
   PipeIn#(DataMessage) tohost = toPipeIn(fifotohost);
   PipeOut#(DataMessage) fromhost = toPipeOut(fifofromhost);

   Vector#(NumDims, SerialFIFOTX#(DataMessage)) txup <- replicateM(mkSerialFIFOTX);
   Vector#(NumDims, SerialFIFORX#(DataMessage)) rxup <- replicateM(mkSerialFIFORX);
   Vector#(NumDims, SerialFIFOTX#(DataMessage)) txdown <- replicateM(mkSerialFIFOTX);
   Vector#(NumDims, SerialFIFORX#(DataMessage)) rxdown <- replicateM(mkSerialFIFORX);

   // sources
   Vector#(TAdd#(TMul#(NumDims, 2), 1), PipeOut#(DataMessage)) switchin = newVector;
   Vector#(TAdd#(TMul#(NumDims, 2), 1), PipeIn#(DataMessage)) switchout = newVector;
   // buffers for crossbar switch
   Vector#(TAdd#(TMul#(NumDims, 2), 1), Vector#(TAdd#(TMul#(NumDims, 2), 1), FIFOF#(DataMessage))) xp = replicate(newVector);
   // temps for building switch
   Vector#(TAdd#(TMul#(NumDims, 2), 1), PipeIn#(DataMessage)) tmpin = newVector;
   Vector#(TAdd#(TMul#(NumDims, 2), 1), PipeOut#(DataMessage)) tmpout = newVector;

   for (Integer i = 0; i < valueOf(NumDims); i = i + 1)
      begin
	 switchin[(2*i) + 0] = rxup[i].out;
	 switchin[(2*i) + 1] = rxdown[i].out;
	 switchout[(2*i) + 0] = txup[i].in;
	 switchout[(2*i) + 1] = txdown[i].in;
      end
   switchin[radix - 1] = fromhost;
   switchout[radix - 1] = tohost;

   for (Integer x = 0; x < radix; x = x + 1)
      for (Integer y = 0; y < radix; y = y + 1)
	 if (x != y) xp[x][y] <- mkSizedFIFOF(4);

   for (Integer x = 0; x < (radix - 1); x = x + 1)
      xp[x][x] <- mkDiscard();

   xp[radix - 1][radix - 1] <- mkSizedFIFOF(4);

   // create distributors

   for (Integer x = 0; x < radix; x = x + 1)
      begin
	 for (Integer y = 0; y < radix; y = y + 1)
	    tmpin[y] = toPipeIn(xp[x][y]);
	 mkDistributor(id, switchin[x], tmpin);
      end
   // create arbiters
   for (Integer y = 0; y < radix; y = y + 1)
      begin
	 for (Integer x = 0; x < radix; x = x + 1)
	    tmpout[x] = toPipeOut(xp[x][y]);
	 mkNocArbitrate(id, tmpout, switchout[y]);
      end

  // interface wiring

   interface PipeIn hosttonode = toPipeIn(fifofromhost);
   interface PipeOut nodetohost = toPipeOut(fifotohost);
   interface Vector linkupin = map(selectinput, rxup);
   interface Vector linkupout = map(selectoutput, txup);
   interface Vector linkdownin = map(selectinput, rxdown);
   interface Vector linkdownout = map(selectoutput, txdown);

endmodule


connect the dma done trace

connect the dma done trace

// Copyright (c) 2016 Connectal Project

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import BRAM::*;
import BuildVector::*;
import Clocks::*;
import Connectable::*;
import FIFOF::*;
import GetPut::*;
import Probe::*;
import Vector::*;

import AddressGenerator::*;
import AxiBits::*;
import ConnectalClocks::*;
import ConnectalConfig::*;
import DefaultValue::*;
import HostInterface::*;
import MemReadEngine::*;
import MemTypes::*;
import PhysMemSlaveFromBram::*;
import Pipe::*;
import TraceMemClient::*;
import XilinxCells::*;
import MPEngine::*;

import AxiPcieRootPort::*;
import NvmeIfc::*;
import NvmePins::*;

`ifndef TOP_SOURCES_PORTAL_CLOCK
import ConnectalBramFifo::*;
`else
import BRAMFIFO::*;
module mkDualClockBramFIFOF#(Clock clock1, Reset reset1, Clock clock2, Reset reset2)(FIFOF#(a))
   provisos (Bits#(a, asz), Add#(1, a__, asz));
   FIFOF#(a) fifo <- mkSizedBRAMFIFOF(512, clocked_by clock1, reset_by reset1);
   return fifo;
endmodule
`endif

interface Nvme;
   interface NvmeRequest request;
   interface MemServerPortalRequest bramRequest;
   interface NvmeTrace trace;
   interface NvmePins pins;
   interface Vector#(2, MemReadClient#(DataBusWidth)) dmaReadClient;
   interface Vector#(1, MemWriteClient#(DataBusWidth)) dmaWriteClient;
`ifdef TOP_SOURCES_PORTAL_CLOCK
   interface Clock portalClockSource;
`endif
endinterface

module mkNvme#(NvmeIndication ind, NvmeTrace trace, MemServerPortalIndication bramIndication)(Nvme);
   let clock <- exposeCurrentClock;
   let reset <- exposeCurrentReset;
   let refclk_p <- mkB2C1();
   let refclk_n <- mkB2C1();
   let pcie_clk_100mhz_buf <- mkClockIBUFDS_GTE2(
`ifdef ClockDefaultParam
       defaultValue,
`endif
      True, refclk_p.c, refclk_n.c);
`ifndef TOP_SOURCES_PORTAL_CLOCK
   let axiClockB2C    <- mkB2C1();
   let axiCtlClockB2C <- mkB2C1();
   let axiClock = axiClockB2C.c;
   let axiCtlClock = axiCtlClockB2C.c;
   let axiReset <- mkSyncReset(10, reset, axiClock);
   let axiCtlReset <- mkSyncReset(10, reset, axiCtlClock);
`else
   let axiClock = clock;
   let axiCtlClock = clock;
   let axiReset = reset;
   let axiCtlReset = reset;
`endif

   let axiRootPort <- mkAPRP(pcie_clk_100mhz_buf, reset, axiClock, axiReset, axiCtlClock, axiCtlReset);
`ifndef TOP_SOURCES_PORTAL_CLOCK
   let axiClockC2B <- mkC2B(axiRootPort.axi.aclk_out);
   let axiCtlClockC2B <- mkC2B(axiRootPort.axi.ctl_aclk_out);
   rule rl_connect_clocks;
      axiClockB2C.inputclock(axiClockC2B.o);
      axiCtlClockB2C.inputclock(axiClockC2B.o);
   endrule
`endif

   FIFOF#(Bit#(32)) dfifoCtl <- mkFIFOF();
   Axi4SlaveBits#(32,DataBusWidth,4,Empty) axiRootPortSlave    = toAxi4SlaveBits(axiRootPort.s_axi);
   Axi4SlaveLiteBits#(32,32)     axiRootPortSlaveCtl = toAxi4SlaveBits(axiRootPort.s_axi_ctl);
   PhysMemSlave#(32,DataBusWidth)          axiRootPortMemSlave    <- mkPhysMemSlave(axiRootPortSlave, clocked_by axiClock, reset_by axiReset);
   PhysMemSlave#(32,32)          axiRootPortMemSlaveCtl <- mkPhysMemSlave(axiRootPortSlaveCtl, clocked_by axiCtlClock, reset_by axiCtlReset);

   FIFOF#(PhysMemRequest#(32,DataBusWidth)) araddrFifo <- mkFIFOF();
   FIFOF#(PhysMemRequest#(32,DataBusWidth)) awaddrFifo <- mkFIFOF();
   FIFOF#(MemData#(DataBusWidth))           rdataFifo <- mkFIFOF();
   FIFOF#(MemData#(DataBusWidth))           wdataFifo <- mkFIFOF();
   FIFOF#(Bit#(6))                           doneFifo <- mkFIFOF();

   let araddrCnx <- mkConnection(toGet(araddrFifo), axiRootPortMemSlave.read_server.readReq);
   let awaddrCnx <- mkConnection(toGet(awaddrFifo), axiRootPortMemSlave.write_server.writeReq);
   let rdataCnx  <- mkConnection(axiRootPortMemSlave.read_server.readData, toPut(rdataFifo));
   let wdataCnx  <- mkConnection(toGet(wdataFifo), axiRootPortMemSlave.write_server.writeData);
   let doneCnx   <- mkConnection(axiRootPortMemSlave.write_server.writeDone, toPut(doneFifo));

   rule rl_rdata;
      let rdata <- toGet(rdataFifo).get();
      ind.readDone(rdata.data);
   endrule

   rule rl_writeDone;
      let tag <- toGet(doneFifo).get();
      ind.writeDone();
   endrule

   FIFOF#(PhysMemRequest#(32,32)) araddrFifoCtl <- mkFIFOF();
   FIFOF#(PhysMemRequest#(32,32)) awaddrFifoCtl <- mkFIFOF();
   FIFOF#(MemData#(32))           rdataFifoCtl <- mkFIFOF();
   FIFOF#(MemData#(32))           wdataFifoCtl <- mkFIFOF();
   FIFOF#(Bit#(6))                doneFifoCtl <- mkFIFOF();

   let araddrCtlCnx <- mkConnection(toGet(araddrFifoCtl), axiRootPortMemSlaveCtl.read_server.readReq);
   let awaddrCtlCnx <- mkConnection(toGet(awaddrFifoCtl), axiRootPortMemSlaveCtl.write_server.writeReq);
   let rdataCtlCnx  <- mkConnection(axiRootPortMemSlaveCtl.read_server.readData, toPut(rdataFifoCtl));
   let wdataCtlCnx  <- mkConnection(toGet(wdataFifoCtl), axiRootPortMemSlaveCtl.write_server.writeData);
   let doneCtlCnx   <- mkConnection(axiRootPortMemSlaveCtl.write_server.writeDone, toPut(doneFifoCtl));

   rule rl_rdata_ctl;
      let rdata <- toGet(rdataFifoCtl).get();
      ind.readDone(extend(rdata.data));
   endrule

   rule rl_writeDone_ctl;
      let tag <- toGet(doneFifoCtl).get();
      ind.writeDone();
   endrule

   Axi4MasterBits#(32,DataBusWidth,MemTagSize,Empty) m_axi_mm = toAxi4MasterBits(axiRootPort.m_axi);
   let getObjId = (interface GetObjId;
		   method SGLId objId(Bit#(32) addr); return extend(addr[31:24]); endmethod
		   method Bit#(MemOffsetSize) addr(Bit#(32) axiAddr); return extend(axiAddr[23:0]); endmethod
		   endinterface);
   let memReadClient  <- mkMemReadClient(getObjId, m_axi_mm);
   let memWriteClient <- mkMemWriteClient(getObjId, m_axi_mm);

   FIFOF#(Tuple4#(DmaChannel,Bool,MemRequest,Bit#(32))) traceFifo <- mkSizedBRAMFIFOF(128);
   PipeIn#(Tuple4#(DmaChannel,Bool,MemRequest,Bit#(32))) tracePipe = toPipeIn(traceFifo);
   FIFOF#(Tuple4#(DmaChannel,Bool,MemData#(DataBusWidth),Bit#(32))) traceDataFifo <- mkSizedBRAMFIFOF(1024);
   PipeIn#(Tuple4#(DmaChannel,Bool,MemData#(DataBusWidth),Bit#(32))) traceDataPipe = toPipeIn(traceDataFifo);
   FIFOF#(Tuple2#(DmaChannel,Bit#(32)))                              traceDoneFifo <- mkSizedBRAMFIFOF(1024);
   PipeIn#(Tuple2#(DmaChannel,Bit#(32)))                             traceDonePipe = toPipeIn(traceDoneFifo);

   rule rl_trace1;
      match { .chan, .write, .req, .timestamp } <- toGet(traceFifo).get();
      trace.traceDmaRequest(chan, write, truncate(req.sglId), extend(req.offset), extend(req.burstLen), extend(req.tag), timestamp);
   endrule
   rule rl_trace_data;
      match { .chan, .write, .md, .timestamp } <- toGet(traceDataFifo).get();
      trace.traceDmaData(chan, write, md.data, md.last, extend(md.tag), timestamp);
   endrule
   rule rl_trace_done;
      match { .chan, .timestamp } <- toGet(traceDoneFifo).get();
      trace.traceDmaDone(chan, 0, timestamp);
   endrule

   let traceReadClient <- mkTraceReadClient(tracePipe,traceDataPipe,DMA_TX,memReadClient);
   let traceWriteClient <- mkTraceWriteClient(tracePipe,traceDataPipe,traceDonePipe,DMA_RX,memWriteClient);
   let traceClient = (interface MemClient#(DataBusWidth);
			 interface readClient = traceReadClient;
			 interface writeClient = traceWriteClient;
		      endinterface);

   let splitter <- mkSplitMemServer();
   BRAM_Configure bramConfig = defaultValue;
   bramConfig.latency = 2;
   bramConfig.memorySize = 1024;
   BRAM2Port#(Bit#(32),Bit#(DataBusWidth)) bram <- mkBRAM2Server(bramConfig);
   MemServer#(DataBusWidth)            bramMemA <- mkMemServerFromBram(bram.portA);
   PhysMemSlave#(32,DataBusWidth)      bramMemB <- mkPhysMemSlaveFromBram(bram.portB);
   MemServerPortal          bramMemServerPortal <- mkPhysMemSlavePortal(bramMemB,bramIndication);

   MemReadEngine#(DataBusWidth,DataBusWidth,2,1) re <- mkMemReadEngine();
   MPEngine#(DataBusWidth,DataBusWidth)    mpEngine <- mkMPStreamEngine(splitter.data, re.readServers[0]);

   let traceMemCnx <- mkConnection(traceClient, splitter.server);
   let bramMemCnx  <- mkConnection(splitter.bramClient, bramMemA);
   // rule rl_data;
   //    let md <- toGet(splitter.data).get();
   //    trace.traceData(md.data, md.last, extend(md.tag));
   // endrule
   rule rl_locdone;
      let loc <- toGet(mpEngine.locdone).get();
      ind.strstrLoc(pack(loc));
   endrule

   interface MemServerPortalRequest bramRequest = bramMemServerPortal.request;
   interface NvmeRequest request;

      method Action status();
        ind.status(axiRootPort.mmcm.lock());
      endmethod

      method Action read32(Bit#(32) addr);
	 araddrFifo.enq(PhysMemRequest { addr: addr, burstLen: 4, tag: 0 });
      endmethod
      method Action write32(Bit#(32) addr, Bit#(32) value);
	 awaddrFifo.enq(PhysMemRequest { addr: addr, burstLen: 4, tag: 0 });
	 wdataFifo.enq(MemData {data: extend(value), tag: 0, last: True});
      endmethod
      method Action read(Bit#(32) addr);
	 araddrFifo.enq(PhysMemRequest { addr: addr, burstLen: fromInteger(valueOf(TDiv#(DataBusWidth,8))), tag: 0 });
      endmethod
      method Action write(Bit#(32) addr, Bit#(DataBusWidth) value);
	 awaddrFifo.enq(PhysMemRequest { addr: addr, burstLen: fromInteger(valueOf(TDiv#(DataBusWidth,8))), tag: 0 });
	 wdataFifo.enq(MemData {data: value, tag: 0, last: True});
      endmethod

      method Action readCtl(Bit#(32) addr);
	 araddrFifoCtl.enq(PhysMemRequest { addr: addr, burstLen: 4, tag: 0 });
      endmethod
      method Action writeCtl(Bit#(32) addr, Bit#(DataBusWidth) value);
	 awaddrFifoCtl.enq(PhysMemRequest { addr: addr, burstLen: 4, tag: 0 });
	 wdataFifoCtl.enq(MemData {data: truncate(value), tag: 0, last: True});
      endmethod
      method Action setSearchString(Bit#(32) needleSglId, Bit#(32) mpNextSglId, Bit#(32) needleLen);
	 mpEngine.setsearch.enq(tuple3(needleSglId, mpNextSglId, needleLen));
      endmethod
      method Action startSearch(Bit#(32) haystackLen);
	 mpEngine.setsearch.enq(tuple3(/* unused */maxBound, haystackLen, /* unused */0));
      endmethod
   endinterface
   interface Clock portalClockSource = axiRootPort.axi.aclk_out;
   interface NvmePins pins;
      interface deleteme_unused_clock = clock;
      interface pcie_sys_reset_n = reset;
      interface pcie = axiRootPort.pci;
      method Action pcie_refclk(Bit#(1) p, Bit#(1) n);
         refclk_p.inputclock(p);
         refclk_n.inputclock(n);
      endmethod
   endinterface
   interface Vector dmaReadClient = vec(splitter.busClient.readClient, re.dmaClient);
   interface Vector dmaWriteClient = vec(splitter.busClient.writeClient);
endmodule

instance ToAxi4SlaveBits#(Axi4SlaveBits#(32,DataBusWidth,4,Empty), AprpS_axi);
   function Axi4SlaveBits#(32,DataBusWidth,4,Empty) toAxi4SlaveBits(AprpS_axi s);
      return (interface Axi4SlaveBits#(32,DataBusWidth,4,Empty);
	 method araddr = compose(s.araddr, extend);
	 method arburst = s.arburst;
	 //method arcache = s.arcache;
	 method arid = s.arid;
	 method arlen = s.arlen;
	 //method arlock = s.arlock;
	 //method arprot = s.arprot;
	 //method arqos = s.arqos;
	 method arready = s.arready;
	 method arsize = s.arsize;
	 method arvalid = s.arvalid;

	 method awaddr = compose(s.awaddr, extend);
	 method awburst = s.awburst;
	 //method awcache = s.awcache;
	 method awid = s.awid;
	 method awlen = s.awlen;
	 //method awlock = s.awlock;
	 //method awprot = s.awprot;
	 //method awqos = s.awqos;
	 method awready = s.awready;
	 method awsize = s.awsize;
	 method awvalid = s.awvalid;

	 method bid = s.bid;
	 method bready = s.bready;
	 method bresp = s.bresp;
	 method bvalid = s.bvalid;
	 method rdata = s.rdata;
	 method rid = s.rid;
	 method rlast = s.rlast;
	 method rready = s.rready;
	 method rresp = s.rresp;
	 method rvalid = s.rvalid;
	 method wdata = s.wdata;
	 method wlast = s.wlast;
	 method wready = s.wready;
	 method wvalid = s.wvalid;
	 method wstrb = s.wstrb;
	 endinterface);
   endfunction
endinstance

instance ToAxi4SlaveBits#(Axi4SlaveLiteBits#(32,32), AprpS_axi_ctl);
   function Axi4SlaveLiteBits#(32,32) toAxi4SlaveBits(AprpS_axi_ctl s);
      return (interface Axi4SlaveLiteBits#(32,32);
	 method araddr = compose(s.araddr, extend);
	 method arready = s.arready;
	 method arvalid = s.arvalid;

	 method awaddr = compose(s.awaddr, extend);
	 method awready = s.awready;
	 method awvalid = s.awvalid;

	 method bready = s.bready;
	 method bresp = s.bresp;
	 method bvalid = s.bvalid;
	 method rdata = s.rdata;
	 method rready = s.rready;
	 method rresp = s.rresp;
	 method rvalid = s.rvalid;
	 method wdata = s.wdata;
	 method wready = s.wready;
	 method Action      wvalid(Bit#(1) v);
	    s.wvalid(v);
	    s.wstrb(pack(replicate(v)));
	 endmethod
	 endinterface);
   endfunction
endinstance

instance ToAxi4MasterBits#(Axi4MasterBits#(32,DataBusWidth,tagWidth,Empty), AprpM_axi);
function Axi4MasterBits#(32,DataBusWidth,tagWidth,Empty) toAxi4MasterBits(AprpM_axi m);
   return (interface Axi4MasterBits#(32,DataBusWidth,tagWidth,Empty);
	   method araddr = m.araddr;
	   method arburst = m.arburst;
	   method arcache = m.arcache;
	   method arlen = m.arlen;
	   method arlock = extend(m.arlock);
	   method arready = m.arready;
	   method arsize = m.arsize;
	   method arvalid = m.arvalid;
	   method Bit#(1) aresetn(); return 1; endmethod
	   method Bit#(tagWidth)     arid(); return 0; endmethod
	   method arprot = m.arprot;
	   method arqos = 0;
	   method awaddr = m.awaddr;
	   method awburst = m.awburst;
	   method awcache = m.awcache;
	   method Bit#(tagWidth)     awid(); return 0; endmethod
	   method awlen = m.awlen;
	   method awlock = extend(m.awlock);
	   method awprot = m.awprot;
	   method awready = m.awready;
	   method Bit#(4)     awqos(); return 0; endmethod
	   method awsize = m.awsize;
	   method awvalid = m.awvalid;
	   method Action      bid(Bit#(tagWidth) v); endmethod
	   method bready = m.bready;
	   method bresp = m.bresp;
	   method bvalid = m.bvalid;
	   method rdata = m.rdata;
	   method Action      rid(Bit#(tagWidth) v); endmethod
	   method rlast = m.rlast;
	   method rready = m.rready;
	   method rresp = m.rresp;
	   method rvalid = m.rvalid;
	   method wdata = m.wdata;
	   method Bit#(tagWidth)     wid(); return 0; endmethod
	   method wlast = m.wlast;
	   method wready = m.wready;
	   method wstrb = m.wstrb;
	   method wvalid = m.wvalid;
	 interface extra = ?;
	 endinterface);
   endfunction
endinstance

interface SplitMemServer;
   interface MemServer#(DataBusWidth) server;
   interface MemClient#(DataBusWidth) busClient;
   interface MemClient#(DataBusWidth) bramClient;
   interface PipeOut#(MemData#(DataBusWidth)) data;
endinterface

module mkSplitMemServer(SplitMemServer);
   let readReqFifo   <- mkFIFOF();
   let readDataFifo  <- mkFIFOF();
   let writeReqFifo  <- mkFIFOF();
   let writeDataFifo <- mkFIFOF();
   let writeDoneFifo <- mkFIFOF();

   let busReadReqFifo   <- mkFIFOF();
   let busReadDataFifo  <- mkSizedBRAMFIFOF(16);
   let busWriteReqFifo  <- mkFIFOF();
   let busWriteDataFifo <- mkSizedBRAMFIFOF(16);
   let busWriteDoneFifo <- mkFIFOF();

   let bramReadReqFifo   <- mkFIFOF();
   let bramReadDataFifo  <- mkSizedBRAMFIFOF(16);
   let bramWriteReqFifo  <- mkFIFOF();
   let bramWriteDataFifo <- mkSizedBRAMFIFOF(16);
   let bramWriteDoneFifo <- mkFIFOF();

   let doneFifo <- mkFIFOF();

   let dataFifo <- mkSizedBRAMFIFOF(4096);

   AddressGenerator#(24,DataBusWidth) readAddrGenerator <- mkAddressGenerator();
   AddressGenerator#(24,DataBusWidth) writeAddrGenerator <- mkAddressGenerator();

   let readReqObj <- mkProbe();
   let readReqDest <- mkProbe();
   let readReqOffset <- mkProbe();
   let readDataDest <- mkProbe();
   let readDataData <- mkProbe();
   let readDataLast <- mkProbe();
   let readBeatAddr <- mkProbe();
   let readBeatByteCount <- mkProbe();
   let readBeatLast <- mkProbe();
   let busReadDataData <- mkProbe();
   let busReadDataLast <- mkProbe();
   let writeReqObj <- mkProbe();
   let writeReqDest <- mkProbe();
   let writeReqOffset <- mkProbe();
   let writeDataDest <- mkProbe();
   let writeDataData <- mkProbe();
   let writeDataLast <- mkProbe();
   let writeBeatAddr <- mkProbe();
   let writeBeatByteCount <- mkProbe();
   let writeBeatLast <- mkProbe();
   let writeDoneTag <- mkProbe();
   let writeDoneDest <- mkProbe();

   rule rl_rd_req;
      MemRequest req <- toGet(readReqFifo).get();
      let dest = req.sglId[5:4];
      if (dest == 2)
	 bramReadReqFifo.enq(req);
      else
	 busReadReqFifo.enq(req);
      readAddrGenerator.request.put(PhysMemRequest { addr: truncate(req.offset), burstLen: req.burstLen, tag: extend(dest) });

      readReqObj <= req.sglId;
      readReqDest <= dest;
      readReqOffset <= req.offset[31:0];
   endrule

   rule rl_rd_data;
      let addrBeat <- readAddrGenerator.addrBeat.get();
      let dest = addrBeat.tag[1:0];
      MemData#(DataBusWidth) md;
      if (dest == 2)
	 md <- toGet(bramReadDataFifo).get();
      else
	 md <- toGet(busReadDataFifo).get();
      readDataFifo.enq(md);

      readDataDest <= dest;
      readDataData <= md.data;
      readDataLast <= md.last;
      readBeatAddr <= addrBeat.addr[15:0];
      readBeatByteCount <= addrBeat.bc;
      readBeatLast <= addrBeat.last;
   endrule

   rule rl_wr_req;
      MemRequest req <- toGet(writeReqFifo).get();
      Bit#(2) dest = req.sglId[5:4];
      if (dest == 2)
	 bramWriteReqFifo.enq(req);
      else if (dest == 3) begin
	 // no need to send the request
      end else
	 busWriteReqFifo.enq(req);
      writeAddrGenerator.request.put(PhysMemRequest { addr: truncate(req.offset), burstLen: req.burstLen, tag: extend(dest) });

      writeReqObj <= req.sglId;
      writeReqDest <= dest;
      writeReqOffset <= req.offset[31:0];
   endrule

   rule rl_wr_data;
      let addrBeat <- writeAddrGenerator.addrBeat.get();
      let dest = addrBeat.tag[1:0];
      MemData#(DataBusWidth) md <- toGet(writeDataFifo).get();
      if (dest == 2)
	 bramWriteDataFifo.enq(md);
      else if (dest == 3)
	 dataFifo.enq(md);
      else
	 busWriteDataFifo.enq(md);
      if (addrBeat.last)
	 doneFifo.enq(tuple2(dest, md.tag));
      else if (md.last)
	 doneFifo.enq(tuple2(dest, maxBound));

      writeDataDest <= dest;
      writeDataData <= md.data;
      writeDataLast <= md.last;
      writeBeatAddr <= addrBeat.addr[15:0];
      writeBeatByteCount <= addrBeat.bc;
      writeBeatLast <= addrBeat.last;

   endrule

   rule rl_wr_done;
      match { .dest, .tag } <- toGet(doneFifo).get();
      Bit#(MemTagSize) doneTag;
      if (dest == 2)
	 doneTag <- toGet(bramWriteDoneFifo).get();
      else if (dest == 3)
	 doneTag = 0;
      else
	 doneTag <- toGet(busWriteDoneFifo).get();
      writeDoneFifo.enq(doneTag);

      writeDoneTag <= doneTag;
      writeDoneDest <= dest;
   endrule

   interface MemServer server;
      interface MemReadServer readServer;
	 interface readReq  = toPut(readReqFifo);
	 interface readData = toGet(readDataFifo);
      endinterface
      interface MemWriteServer writeServer;
	 interface writeReq  = toPut(writeReqFifo);
	 interface writeData = toPut(writeDataFifo);
	 interface writeDone = toGet(writeDoneFifo);
      endinterface
   endinterface
   interface MemClient busClient;
      interface MemReadClient readClient;
	 interface readReq  = toGet(busReadReqFifo);
	 interface Put readData;
	    method Action put(MemData#(DataBusWidth) md);
	       busReadDataFifo.enq(md);
	       busReadDataData <= md.data;
	       busReadDataLast <= md.last;
	    endmethod
	 endinterface
      endinterface
      interface MemWriteClient writeClient;
	 interface writeReq  = toGet(busWriteReqFifo);
	 interface writeData = toGet(busWriteDataFifo);
	 interface writeDone = toPut(busWriteDoneFifo);
      endinterface
   endinterface
   interface MemClient bramClient;
      interface MemReadClient readClient;
	 interface readReq  = toGet(bramReadReqFifo);
	 interface readData = toPut(bramReadDataFifo);
      endinterface
      interface MemWriteClient writeClient;
	 interface writeReq  = toGet(bramWriteReqFifo);
	 interface writeData = toGet(bramWriteDataFifo);
	 interface writeDone = toPut(bramWriteDoneFifo);
      endinterface
   endinterface
   interface PipeOut data = toPipeOut(dataFifo);
endmodule

interface MemServerPortal;
   interface MemServerPortalRequest request;
endinterface

module mkPhysMemSlavePortal#(PhysMemSlave#(32,DataBusWidth) ms, MemServerPortalIndication ind)(MemServerPortal);

   FIFOF#(PhysMemRequest#(32,DataBusWidth)) araddrFifo <- mkFIFOF();
   FIFOF#(PhysMemRequest#(32,DataBusWidth)) awaddrFifo <- mkFIFOF();
   FIFOF#(MemData#(DataBusWidth))           rdataFifo <- mkFIFOF();
   FIFOF#(MemData#(DataBusWidth))           wdataFifo <- mkFIFOF();
   FIFOF#(Bit#(6))                doneFifo <- mkFIFOF();

   let araddrCnx <- mkConnection(toGet(araddrFifo), ms.read_server.readReq);
   let awaddrCnx <- mkConnection(toGet(awaddrFifo), ms.write_server.writeReq);
   let rdataCnx  <- mkConnection(ms.read_server.readData, toPut(rdataFifo));
   let wdataCnx  <- mkConnection(toGet(wdataFifo), ms.write_server.writeData);
   let doneCnx   <- mkConnection(ms.write_server.writeDone, toPut(doneFifo));

   rule rl_rdata;
      let rdata <- toGet(rdataFifo).get();
      ind.readDone(rdata.data);
   endrule

   rule rl_writeDone;
      let tag <- toGet(doneFifo).get();
      ind.writeDone();
   endrule

   interface MemServerPortalRequest request;
      method Action read(Bit#(32) addr);
	 araddrFifo.enq(PhysMemRequest { addr: addr, burstLen: fromInteger(valueOf(TDiv#(DataBusWidth,8))), tag: 0 });
      endmethod
      method Action write(Bit#(32) addr, Bit#(DataBusWidth) value);
	 awaddrFifo.enq(PhysMemRequest { addr: addr, burstLen: fromInteger(valueOf(TDiv#(DataBusWidth,8))), tag: 0 });
	 wdataFifo.enq(MemData {data: value, tag: 0, last: True});
      endmethod
   endinterface
endmodule

// Copyright (c) 2016 Connectal Project

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import BRAM::*;
import BuildVector::*;
import Clocks::*;
import Connectable::*;
import FIFOF::*;
import GetPut::*;
import Probe::*;
import Vector::*;

import AddressGenerator::*;
import AxiBits::*;
import ConnectalClocks::*;
import ConnectalConfig::*;
import DefaultValue::*;
import HostInterface::*;
import MemReadEngine::*;
import MemTypes::*;
import PhysMemSlaveFromBram::*;
import Pipe::*;
import TraceMemClient::*;
import XilinxCells::*;
import MPEngine::*;

import AxiPcieRootPort::*;
import NvmeIfc::*;
import NvmePins::*;

`ifndef TOP_SOURCES_PORTAL_CLOCK
import ConnectalBramFifo::*;
`else
import BRAMFIFO::*;
module mkDualClockBramFIFOF#(Clock clock1, Reset reset1, Clock clock2, Reset reset2)(FIFOF#(a))
   provisos (Bits#(a, asz), Add#(1, a__, asz));
   FIFOF#(a) fifo <- mkSizedBRAMFIFOF(512, clocked_by clock1, reset_by reset1);
   return fifo;
endmodule
`endif

interface Nvme;
   interface NvmeRequest request;
   interface MemServerPortalRequest bramRequest;
   interface NvmeTrace trace;
   interface NvmePins pins;
   interface Vector#(2, MemReadClient#(DataBusWidth)) dmaReadClient;
   interface Vector#(1, MemWriteClient#(DataBusWidth)) dmaWriteClient;
`ifdef TOP_SOURCES_PORTAL_CLOCK
   interface Clock portalClockSource;
`endif
endinterface

module mkNvme#(NvmeIndication ind, NvmeTrace trace, MemServerPortalIndication bramIndication)(Nvme);
   let clock <- exposeCurrentClock;
   let reset <- exposeCurrentReset;
   let refclk_p <- mkB2C1();
   let refclk_n <- mkB2C1();
   let pcie_clk_100mhz_buf <- mkClockIBUFDS_GTE2(
`ifdef ClockDefaultParam
       defaultValue,
`endif
      True, refclk_p.c, refclk_n.c);
`ifndef TOP_SOURCES_PORTAL_CLOCK
   let axiClockB2C    <- mkB2C1();
   let axiCtlClockB2C <- mkB2C1();
   let axiClock = axiClockB2C.c;
   let axiCtlClock = axiCtlClockB2C.c;
   let axiReset <- mkSyncReset(10, reset, axiClock);
   let axiCtlReset <- mkSyncReset(10, reset, axiCtlClock);
`else
   let axiClock = clock;
   let axiCtlClock = clock;
   let axiReset = reset;
   let axiCtlReset = reset;
`endif

   let axiRootPort <- mkAPRP(pcie_clk_100mhz_buf, reset, axiClock, axiReset, axiCtlClock, axiCtlReset);
`ifndef TOP_SOURCES_PORTAL_CLOCK
   let axiClockC2B <- mkC2B(axiRootPort.axi.aclk_out);
   let axiCtlClockC2B <- mkC2B(axiRootPort.axi.ctl_aclk_out);
   rule rl_connect_clocks;
      axiClockB2C.inputclock(axiClockC2B.o);
      axiCtlClockB2C.inputclock(axiClockC2B.o);
   endrule
`endif

   FIFOF#(Bit#(32)) dfifoCtl <- mkFIFOF();
   Axi4SlaveBits#(32,DataBusWidth,4,Empty) axiRootPortSlave    = toAxi4SlaveBits(axiRootPort.s_axi);
   Axi4SlaveLiteBits#(32,32)     axiRootPortSlaveCtl = toAxi4SlaveBits(axiRootPort.s_axi_ctl);
   PhysMemSlave#(32,DataBusWidth)          axiRootPortMemSlave    <- mkPhysMemSlave(axiRootPortSlave, clocked_by axiClock, reset_by axiReset);
   PhysMemSlave#(32,32)          axiRootPortMemSlaveCtl <- mkPhysMemSlave(axiRootPortSlaveCtl, clocked_by axiCtlClock, reset_by axiCtlReset);

   FIFOF#(PhysMemRequest#(32,DataBusWidth)) araddrFifo <- mkFIFOF();
   FIFOF#(PhysMemRequest#(32,DataBusWidth)) awaddrFifo <- mkFIFOF();
   FIFOF#(MemData#(DataBusWidth))           rdataFifo <- mkFIFOF();
   FIFOF#(MemData#(DataBusWidth))           wdataFifo <- mkFIFOF();
   FIFOF#(Bit#(6))                           doneFifo <- mkFIFOF();

   let araddrCnx <- mkConnection(toGet(araddrFifo), axiRootPortMemSlave.read_server.readReq);
   let awaddrCnx <- mkConnection(toGet(awaddrFifo), axiRootPortMemSlave.write_server.writeReq);
   let rdataCnx  <- mkConnection(axiRootPortMemSlave.read_server.readData, toPut(rdataFifo));
   let wdataCnx  <- mkConnection(toGet(wdataFifo), axiRootPortMemSlave.write_server.writeData);
   let doneCnx   <- mkConnection(axiRootPortMemSlave.write_server.writeDone, toPut(doneFifo));

   rule rl_rdata;
      let rdata <- toGet(rdataFifo).get();
      ind.readDone(rdata.data);
   endrule

   rule rl_writeDone;
      let tag <- toGet(doneFifo).get();
      ind.writeDone();
   endrule

   FIFOF#(PhysMemRequest#(32,32)) araddrFifoCtl <- mkFIFOF();
   FIFOF#(PhysMemRequest#(32,32)) awaddrFifoCtl <- mkFIFOF();
   FIFOF#(MemData#(32))           rdataFifoCtl <- mkFIFOF();
   FIFOF#(MemData#(32))           wdataFifoCtl <- mkFIFOF();
   FIFOF#(Bit#(6))                doneFifoCtl <- mkFIFOF();

   let araddrCtlCnx <- mkConnection(toGet(araddrFifoCtl), axiRootPortMemSlaveCtl.read_server.readReq);
   let awaddrCtlCnx <- mkConnection(toGet(awaddrFifoCtl), axiRootPortMemSlaveCtl.write_server.writeReq);
   let rdataCtlCnx  <- mkConnection(axiRootPortMemSlaveCtl.read_server.readData, toPut(rdataFifoCtl));
   let wdataCtlCnx  <- mkConnection(toGet(wdataFifoCtl), axiRootPortMemSlaveCtl.write_server.writeData);
   let doneCtlCnx   <- mkConnection(axiRootPortMemSlaveCtl.write_server.writeDone, toPut(doneFifoCtl));

   rule rl_rdata_ctl;
      let rdata <- toGet(rdataFifoCtl).get();
      ind.readDone(extend(rdata.data));
   endrule

   rule rl_writeDone_ctl;
      let tag <- toGet(doneFifoCtl).get();
      ind.writeDone();
   endrule

   Axi4MasterBits#(32,DataBusWidth,MemTagSize,Empty) m_axi_mm = toAxi4MasterBits(axiRootPort.m_axi);
   let getObjId = (interface GetObjId;
		   method SGLId objId(Bit#(32) addr); return extend(addr[31:24]); endmethod
		   method Bit#(MemOffsetSize) addr(Bit#(32) axiAddr); return extend(axiAddr[23:0]); endmethod
		   endinterface);
   let memReadClient  <- mkMemReadClient(getObjId, m_axi_mm);
   let memWriteClient <- mkMemWriteClient(getObjId, m_axi_mm);

   FIFOF#(Tuple4#(DmaChannel,Bool,MemRequest,Bit#(32))) traceFifo <- mkSizedBRAMFIFOF(128);
   PipeIn#(Tuple4#(DmaChannel,Bool,MemRequest,Bit#(32))) tracePipe = toPipeIn(traceFifo);
   FIFOF#(Tuple4#(DmaChannel,Bool,MemData#(DataBusWidth),Bit#(32))) traceDataFifo <- mkSizedBRAMFIFOF(1024);
   PipeIn#(Tuple4#(DmaChannel,Bool,MemData#(DataBusWidth),Bit#(32))) traceDataPipe = toPipeIn(traceDataFifo);
   FIFOF#(Tuple2#(DmaChannel,Bit#(32)))                              traceDoneFifo <- mkSizedBRAMFIFOF(1024);
   PipeIn#(Tuple2#(DmaChannel,Bit#(32)))                             traceDonePipe = toPipeIn(traceDoneFifo);

   rule rl_trace1;
      match { .chan, .write, .req, .timestamp } <- toGet(traceFifo).get();
      trace.traceDmaRequest(chan, write, truncate(req.sglId), extend(req.offset), extend(req.burstLen), extend(req.tag), timestamp);
   endrule
   rule rl_trace_data;
      match { .chan, .write, .md, .timestamp } <- toGet(traceDataFifo).get();
      trace.traceDmaData(chan, write, md.data, md.last, extend(md.tag), timestamp);
   endrule

   let traceReadClient <- mkTraceReadClient(tracePipe,traceDataPipe,DMA_TX,memReadClient);
   let traceWriteClient <- mkTraceWriteClient(tracePipe,traceDataPipe,traceDonePipe,DMA_RX,memWriteClient);
   let traceClient = (interface MemClient#(DataBusWidth);
			 interface readClient = traceReadClient;
			 interface writeClient = traceWriteClient;
		      endinterface);

   let splitter <- mkSplitMemServer();
   BRAM_Configure bramConfig = defaultValue;
   bramConfig.latency = 2;
   bramConfig.memorySize = 1024;
   BRAM2Port#(Bit#(32),Bit#(DataBusWidth)) bram <- mkBRAM2Server(bramConfig);
   MemServer#(DataBusWidth)            bramMemA <- mkMemServerFromBram(bram.portA);
   PhysMemSlave#(32,DataBusWidth)      bramMemB <- mkPhysMemSlaveFromBram(bram.portB);
   MemServerPortal          bramMemServerPortal <- mkPhysMemSlavePortal(bramMemB,bramIndication);

   MemReadEngine#(DataBusWidth,DataBusWidth,2,1) re <- mkMemReadEngine();
   MPEngine#(DataBusWidth,DataBusWidth)    mpEngine <- mkMPStreamEngine(splitter.data, re.readServers[0]);

   let traceMemCnx <- mkConnection(traceClient, splitter.server);
   let bramMemCnx  <- mkConnection(splitter.bramClient, bramMemA);
   // rule rl_data;
   //    let md <- toGet(splitter.data).get();
   //    trace.traceData(md.data, md.last, extend(md.tag));
   // endrule
   rule rl_locdone;
      let loc <- toGet(mpEngine.locdone).get();
      ind.strstrLoc(pack(loc));
   endrule

   interface MemServerPortalRequest bramRequest = bramMemServerPortal.request;
   interface NvmeRequest request;

      method Action status();
        ind.status(axiRootPort.mmcm.lock());
      endmethod

      method Action read32(Bit#(32) addr);
	 araddrFifo.enq(PhysMemRequest { addr: addr, burstLen: 4, tag: 0 });
      endmethod
      method Action write32(Bit#(32) addr, Bit#(32) value);
	 awaddrFifo.enq(PhysMemRequest { addr: addr, burstLen: 4, tag: 0 });
	 wdataFifo.enq(MemData {data: extend(value), tag: 0, last: True});
      endmethod
      method Action read(Bit#(32) addr);
	 araddrFifo.enq(PhysMemRequest { addr: addr, burstLen: fromInteger(valueOf(TDiv#(DataBusWidth,8))), tag: 0 });
      endmethod
      method Action write(Bit#(32) addr, Bit#(DataBusWidth) value);
	 awaddrFifo.enq(PhysMemRequest { addr: addr, burstLen: fromInteger(valueOf(TDiv#(DataBusWidth,8))), tag: 0 });
	 wdataFifo.enq(MemData {data: value, tag: 0, last: True});
      endmethod

      method Action readCtl(Bit#(32) addr);
	 araddrFifoCtl.enq(PhysMemRequest { addr: addr, burstLen: 4, tag: 0 });
      endmethod
      method Action writeCtl(Bit#(32) addr, Bit#(DataBusWidth) value);
	 awaddrFifoCtl.enq(PhysMemRequest { addr: addr, burstLen: 4, tag: 0 });
	 wdataFifoCtl.enq(MemData {data: truncate(value), tag: 0, last: True});
      endmethod
      method Action setSearchString(Bit#(32) needleSglId, Bit#(32) mpNextSglId, Bit#(32) needleLen);
	 mpEngine.setsearch.enq(tuple3(needleSglId, mpNextSglId, needleLen));
      endmethod
      method Action startSearch(Bit#(32) haystackLen);
	 mpEngine.setsearch.enq(tuple3(/* unused */maxBound, haystackLen, /* unused */0));
      endmethod
   endinterface
   interface Clock portalClockSource = axiRootPort.axi.aclk_out;
   interface NvmePins pins;
      interface deleteme_unused_clock = clock;
      interface pcie_sys_reset_n = reset;
      interface pcie = axiRootPort.pci;
      method Action pcie_refclk(Bit#(1) p, Bit#(1) n);
         refclk_p.inputclock(p);
         refclk_n.inputclock(n);
      endmethod
   endinterface
   interface Vector dmaReadClient = vec(splitter.busClient.readClient, re.dmaClient);
   interface Vector dmaWriteClient = vec(splitter.busClient.writeClient);
endmodule

instance ToAxi4SlaveBits#(Axi4SlaveBits#(32,DataBusWidth,4,Empty), AprpS_axi);
   function Axi4SlaveBits#(32,DataBusWidth,4,Empty) toAxi4SlaveBits(AprpS_axi s);
      return (interface Axi4SlaveBits#(32,DataBusWidth,4,Empty);
	 method araddr = compose(s.araddr, extend);
	 method arburst = s.arburst;
	 //method arcache = s.arcache;
	 method arid = s.arid;
	 method arlen = s.arlen;
	 //method arlock = s.arlock;
	 //method arprot = s.arprot;
	 //method arqos = s.arqos;
	 method arready = s.arready;
	 method arsize = s.arsize;
	 method arvalid = s.arvalid;

	 method awaddr = compose(s.awaddr, extend);
	 method awburst = s.awburst;
	 //method awcache = s.awcache;
	 method awid = s.awid;
	 method awlen = s.awlen;
	 //method awlock = s.awlock;
	 //method awprot = s.awprot;
	 //method awqos = s.awqos;
	 method awready = s.awready;
	 method awsize = s.awsize;
	 method awvalid = s.awvalid;

	 method bid = s.bid;
	 method bready = s.bready;
	 method bresp = s.bresp;
	 method bvalid = s.bvalid;
	 method rdata = s.rdata;
	 method rid = s.rid;
	 method rlast = s.rlast;
	 method rready = s.rready;
	 method rresp = s.rresp;
	 method rvalid = s.rvalid;
	 method wdata = s.wdata;
	 method wlast = s.wlast;
	 method wready = s.wready;
	 method wvalid = s.wvalid;
	 method wstrb = s.wstrb;
	 endinterface);
   endfunction
endinstance

instance ToAxi4SlaveBits#(Axi4SlaveLiteBits#(32,32), AprpS_axi_ctl);
   function Axi4SlaveLiteBits#(32,32) toAxi4SlaveBits(AprpS_axi_ctl s);
      return (interface Axi4SlaveLiteBits#(32,32);
	 method araddr = compose(s.araddr, extend);
	 method arready = s.arready;
	 method arvalid = s.arvalid;

	 method awaddr = compose(s.awaddr, extend);
	 method awready = s.awready;
	 method awvalid = s.awvalid;

	 method bready = s.bready;
	 method bresp = s.bresp;
	 method bvalid = s.bvalid;
	 method rdata = s.rdata;
	 method rready = s.rready;
	 method rresp = s.rresp;
	 method rvalid = s.rvalid;
	 method wdata = s.wdata;
	 method wready = s.wready;
	 method Action      wvalid(Bit#(1) v);
	    s.wvalid(v);
	    s.wstrb(pack(replicate(v)));
	 endmethod
	 endinterface);
   endfunction
endinstance

instance ToAxi4MasterBits#(Axi4MasterBits#(32,DataBusWidth,tagWidth,Empty), AprpM_axi);
function Axi4MasterBits#(32,DataBusWidth,tagWidth,Empty) toAxi4MasterBits(AprpM_axi m);
   return (interface Axi4MasterBits#(32,DataBusWidth,tagWidth,Empty);
	   method araddr = m.araddr;
	   method arburst = m.arburst;
	   method arcache = m.arcache;
	   method arlen = m.arlen;
	   method arlock = extend(m.arlock);
	   method arready = m.arready;
	   method arsize = m.arsize;
	   method arvalid = m.arvalid;
	   method Bit#(1) aresetn(); return 1; endmethod
	   method Bit#(tagWidth)     arid(); return 0; endmethod
	   method arprot = m.arprot;
	   method arqos = 0;
	   method awaddr = m.awaddr;
	   method awburst = m.awburst;
	   method awcache = m.awcache;
	   method Bit#(tagWidth)     awid(); return 0; endmethod
	   method awlen = m.awlen;
	   method awlock = extend(m.awlock);
	   method awprot = m.awprot;
	   method awready = m.awready;
	   method Bit#(4)     awqos(); return 0; endmethod
	   method awsize = m.awsize;
	   method awvalid = m.awvalid;
	   method Action      bid(Bit#(tagWidth) v); endmethod
	   method bready = m.bready;
	   method bresp = m.bresp;
	   method bvalid = m.bvalid;
	   method rdata = m.rdata;
	   method Action      rid(Bit#(tagWidth) v); endmethod
	   method rlast = m.rlast;
	   method rready = m.rready;
	   method rresp = m.rresp;
	   method rvalid = m.rvalid;
	   method wdata = m.wdata;
	   method Bit#(tagWidth)     wid(); return 0; endmethod
	   method wlast = m.wlast;
	   method wready = m.wready;
	   method wstrb = m.wstrb;
	   method wvalid = m.wvalid;
	 interface extra = ?;
	 endinterface);
   endfunction
endinstance

interface SplitMemServer;
   interface MemServer#(DataBusWidth) server;
   interface MemClient#(DataBusWidth) busClient;
   interface MemClient#(DataBusWidth) bramClient;
   interface PipeOut#(MemData#(DataBusWidth)) data;
endinterface

module mkSplitMemServer(SplitMemServer);
   let readReqFifo   <- mkFIFOF();
   let readDataFifo  <- mkFIFOF();
   let writeReqFifo  <- mkFIFOF();
   let writeDataFifo <- mkFIFOF();
   let writeDoneFifo <- mkFIFOF();

   let busReadReqFifo   <- mkFIFOF();
   let busReadDataFifo  <- mkSizedBRAMFIFOF(16);
   let busWriteReqFifo  <- mkFIFOF();
   let busWriteDataFifo <- mkSizedBRAMFIFOF(16);
   let busWriteDoneFifo <- mkFIFOF();

   let bramReadReqFifo   <- mkFIFOF();
   let bramReadDataFifo  <- mkSizedBRAMFIFOF(16);
   let bramWriteReqFifo  <- mkFIFOF();
   let bramWriteDataFifo <- mkSizedBRAMFIFOF(16);
   let bramWriteDoneFifo <- mkFIFOF();

   let doneFifo <- mkFIFOF();

   let dataFifo <- mkSizedBRAMFIFOF(4096);

   AddressGenerator#(24,DataBusWidth) readAddrGenerator <- mkAddressGenerator();
   AddressGenerator#(24,DataBusWidth) writeAddrGenerator <- mkAddressGenerator();

   let readReqObj <- mkProbe();
   let readReqDest <- mkProbe();
   let readReqOffset <- mkProbe();
   let readDataDest <- mkProbe();
   let readDataData <- mkProbe();
   let readDataLast <- mkProbe();
   let readBeatAddr <- mkProbe();
   let readBeatByteCount <- mkProbe();
   let readBeatLast <- mkProbe();
   let busReadDataData <- mkProbe();
   let busReadDataLast <- mkProbe();
   let writeReqObj <- mkProbe();
   let writeReqDest <- mkProbe();
   let writeReqOffset <- mkProbe();
   let writeDataDest <- mkProbe();
   let writeDataData <- mkProbe();
   let writeDataLast <- mkProbe();
   let writeBeatAddr <- mkProbe();
   let writeBeatByteCount <- mkProbe();
   let writeBeatLast <- mkProbe();
   let writeDoneTag <- mkProbe();
   let writeDoneDest <- mkProbe();

   rule rl_rd_req;
      MemRequest req <- toGet(readReqFifo).get();
      let dest = req.sglId[5:4];
      if (dest == 2)
	 bramReadReqFifo.enq(req);
      else
	 busReadReqFifo.enq(req);
      readAddrGenerator.request.put(PhysMemRequest { addr: truncate(req.offset), burstLen: req.burstLen, tag: extend(dest) });

      readReqObj <= req.sglId;
      readReqDest <= dest;
      readReqOffset <= req.offset[31:0];
   endrule

   rule rl_rd_data;
      let addrBeat <- readAddrGenerator.addrBeat.get();
      let dest = addrBeat.tag[1:0];
      MemData#(DataBusWidth) md;
      if (dest == 2)
	 md <- toGet(bramReadDataFifo).get();
      else
	 md <- toGet(busReadDataFifo).get();
      readDataFifo.enq(md);

      readDataDest <= dest;
      readDataData <= md.data;
      readDataLast <= md.last;
      readBeatAddr <= addrBeat.addr[15:0];
      readBeatByteCount <= addrBeat.bc;
      readBeatLast <= addrBeat.last;
   endrule

   rule rl_wr_req;
      MemRequest req <- toGet(writeReqFifo).get();
      Bit#(2) dest = req.sglId[5:4];
      if (dest == 2)
	 bramWriteReqFifo.enq(req);
      else if (dest == 3) begin
	 // no need to send the request
      end else
	 busWriteReqFifo.enq(req);
      writeAddrGenerator.request.put(PhysMemRequest { addr: truncate(req.offset), burstLen: req.burstLen, tag: extend(dest) });

      writeReqObj <= req.sglId;
      writeReqDest <= dest;
      writeReqOffset <= req.offset[31:0];
   endrule

   rule rl_wr_data;
      let addrBeat <- writeAddrGenerator.addrBeat.get();
      let dest = addrBeat.tag[1:0];
      MemData#(DataBusWidth) md <- toGet(writeDataFifo).get();
      if (dest == 2)
	 bramWriteDataFifo.enq(md);
      else if (dest == 3)
	 dataFifo.enq(md);
      else
	 busWriteDataFifo.enq(md);
      if (addrBeat.last)
	 doneFifo.enq(tuple2(dest, md.tag));
      else if (md.last)
	 doneFifo.enq(tuple2(dest, maxBound));

      writeDataDest <= dest;
      writeDataData <= md.data;
      writeDataLast <= md.last;
      writeBeatAddr <= addrBeat.addr[15:0];
      writeBeatByteCount <= addrBeat.bc;
      writeBeatLast <= addrBeat.last;

   endrule

   rule rl_wr_done;
      match { .dest, .tag } <- toGet(doneFifo).get();
      Bit#(MemTagSize) doneTag;
      if (dest == 2)
	 doneTag <- toGet(bramWriteDoneFifo).get();
      else if (dest == 3)
	 doneTag = 0;
      else
	 doneTag <- toGet(busWriteDoneFifo).get();
      writeDoneFifo.enq(doneTag);

      writeDoneTag <= doneTag;
      writeDoneDest <= dest;
   endrule

   interface MemServer server;
      interface MemReadServer readServer;
	 interface readReq  = toPut(readReqFifo);
	 interface readData = toGet(readDataFifo);
      endinterface
      interface MemWriteServer writeServer;
	 interface writeReq  = toPut(writeReqFifo);
	 interface writeData = toPut(writeDataFifo);
	 interface writeDone = toGet(writeDoneFifo);
      endinterface
   endinterface
   interface MemClient busClient;
      interface MemReadClient readClient;
	 interface readReq  = toGet(busReadReqFifo);
	 interface Put readData;
	    method Action put(MemData#(DataBusWidth) md);
	       busReadDataFifo.enq(md);
	       busReadDataData <= md.data;
	       busReadDataLast <= md.last;
	    endmethod
	 endinterface
      endinterface
      interface MemWriteClient writeClient;
	 interface writeReq  = toGet(busWriteReqFifo);
	 interface writeData = toGet(busWriteDataFifo);
	 interface writeDone = toPut(busWriteDoneFifo);
      endinterface
   endinterface
   interface MemClient bramClient;
      interface MemReadClient readClient;
	 interface readReq  = toGet(bramReadReqFifo);
	 interface readData = toPut(bramReadDataFifo);
      endinterface
      interface MemWriteClient writeClient;
	 interface writeReq  = toGet(bramWriteReqFifo);
	 interface writeData = toGet(bramWriteDataFifo);
	 interface writeDone = toPut(bramWriteDoneFifo);
      endinterface
   endinterface
   interface PipeOut data = toPipeOut(dataFifo);
endmodule

interface MemServerPortal;
   interface MemServerPortalRequest request;
endinterface

module mkPhysMemSlavePortal#(PhysMemSlave#(32,DataBusWidth) ms, MemServerPortalIndication ind)(MemServerPortal);

   FIFOF#(PhysMemRequest#(32,DataBusWidth)) araddrFifo <- mkFIFOF();
   FIFOF#(PhysMemRequest#(32,DataBusWidth)) awaddrFifo <- mkFIFOF();
   FIFOF#(MemData#(DataBusWidth))           rdataFifo <- mkFIFOF();
   FIFOF#(MemData#(DataBusWidth))           wdataFifo <- mkFIFOF();
   FIFOF#(Bit#(6))                doneFifo <- mkFIFOF();

   let araddrCnx <- mkConnection(toGet(araddrFifo), ms.read_server.readReq);
   let awaddrCnx <- mkConnection(toGet(awaddrFifo), ms.write_server.writeReq);
   let rdataCnx  <- mkConnection(ms.read_server.readData, toPut(rdataFifo));
   let wdataCnx  <- mkConnection(toGet(wdataFifo), ms.write_server.writeData);
   let doneCnx   <- mkConnection(ms.write_server.writeDone, toPut(doneFifo));

   rule rl_rdata;
      let rdata <- toGet(rdataFifo).get();
      ind.readDone(rdata.data);
   endrule

   rule rl_writeDone;
      let tag <- toGet(doneFifo).get();
      ind.writeDone();
   endrule

   interface MemServerPortalRequest request;
      method Action read(Bit#(32) addr);
	 araddrFifo.enq(PhysMemRequest { addr: addr, burstLen: fromInteger(valueOf(TDiv#(DataBusWidth,8))), tag: 0 });
      endmethod
      method Action write(Bit#(32) addr, Bit#(DataBusWidth) value);
	 awaddrFifo.enq(PhysMemRequest { addr: addr, burstLen: fromInteger(valueOf(TDiv#(DataBusWidth,8))), tag: 0 });
	 wdataFifo.enq(MemData {data: value, tag: 0, last: True});
      endmethod
   endinterface
endmodule


fix TestBench bug in mkDct32

fix TestBench bug in mkDct32

// Copyright (c) 2016 Min Chen
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this
// list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//

import Vector::*;
import GetPut::*;
import Connectable::*;
import ConfigReg::*;
import FIFOF::*;
import SpecialFIFOs::*;
import BUtils::*;
import BRAMCore::*;
import SMul::*;
import ARAM::*;
import Utils::*;

// Internal ROM Table
Integer g_t32[32][16] =
{
    { 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 },
    { 90, 90, 88, 85, 82, 78, 73, 67, 61, 54, 46, 38, 31, 22, 13,  4 },
    { 90, 87, 80, 70, 57, 43, 25,  9, -9,-25,-43,-57,-70,-80,-87,-90 },
    { 90, 82, 67, 46, 22, -4,-31,-54,-73,-85,-90,-88,-78,-61,-38,-13 },
    { 89, 75, 50, 18,-18,-50,-75,-89,-89,-75,-50,-18, 18, 50, 75, 89 },
    { 88, 67, 31,-13,-54,-82,-90,-78,-46, -4, 38, 73, 90, 85, 61, 22 },
    { 87, 57,  9,-43,-80,-90,-70,-25, 25, 70, 90, 80, 43, -9,-57,-87 },
    { 85, 46,-13,-67,-90,-73,-22, 38, 82, 88, 54, -4,-61,-90,-78,-31 },
    { 83, 36,-36,-83,-83,-36, 36, 83, 83, 36,-36,-83,-83,-36, 36, 83 },
    { 82, 22,-54,-90,-61, 13, 78, 85, 31,-46,-90,-67,  4, 73, 88, 38 },
    { 80,  9,-70,-87,-25, 57, 90, 43,-43,-90,-57, 25, 87, 70, -9,-80 },
    { 78, -4,-82,-73, 13, 85, 67,-22,-88,-61, 31, 90, 54,-38,-90,-46 },
    { 75,-18,-89,-50, 50, 89, 18,-75,-75, 18, 89, 50,-50,-89,-18, 75 },
    { 73,-31,-90,-22, 78, 67,-38,-90,-13, 82, 61,-46,-88, -4, 85, 54 },
    { 70,-43,-87,  9, 90, 25,-80,-57, 57, 80,-25,-90, -9, 87, 43,-70 },
    { 67,-54,-78, 38, 85,-22,-90,  4, 90, 13,-88,-31, 82, 46,-73,-61 },
    { 64,-64,-64, 64, 64,-64,-64, 64, 64,-64,-64, 64, 64,-64,-64, 64 },
    { 61,-73,-46, 82, 31,-88,-13, 90, -4,-90, 22, 85,-38,-78, 54, 67 },
    { 57,-80,-25, 90, -9,-87, 43, 70,-70,-43, 87,  9,-90, 25, 80,-57 },
    { 54,-85, -4, 88,-46,-61, 82, 13,-90, 38, 67,-78,-22, 90,-31,-73 },
    { 50,-89, 18, 75,-75,-18, 89,-50,-50, 89,-18,-75, 75, 18,-89, 50 },
    { 46,-90, 38, 54,-90, 31, 61,-88, 22, 67,-85, 13, 73,-82,  4, 78 },
    { 43,-90, 57, 25,-87, 70,  9,-80, 80, -9,-70, 87,-25,-57, 90,-43 },
    { 38,-88, 73, -4,-67, 90,-46,-31, 85,-78, 13, 61,-90, 54, 22,-82 },
    { 36,-83, 83,-36,-36, 83,-83, 36, 36,-83, 83,-36,-36, 83,-83, 36 },
    { 31,-78, 90,-61,  4, 54,-88, 82,-38,-22, 73,-90, 67,-13,-46, 85 },
    { 25,-70, 90,-80, 43,  9,-57, 87,-87, 57, -9,-43, 80,-90, 70,-25 },
    { 22,-61, 85,-90, 73,-38, -4, 46,-78, 90,-82, 54,-13,-31, 67,-88 },
    { 18,-50, 75,-89, 89,-75, 50,-18,-18, 50,-75, 89,-89, 75,-50, 18 },
    { 13,-38, 61,-78, 88,-90, 85,-73, 54,-31,  4, 22,-46, 67,-82, 90 },
    {  9,-25, 43,-57, 70,-80, 87,-90, 90,-87, 80,-70, 57,-43, 25, -9 },
    {  4,-13, 22,-31, 38,-46, 54,-61, 67,-73, 78,-82, 85,-88, 90,-90 }
};

// External interface
interface IDct32;
   method Action setSize(Bit#(2) dsize);
   interface Put#(Vector#(2, Vector#(32, Bit#(9)))) io_in;
   interface Get#(Vector#(32, Bit#(16))) io_out;
endinterface

interface IDct32Core#(numeric type a);
   method Vector#(15, Bit#(16)) dct(Vector#(16, Bit#(a)) x, Bit#(5) idx);
endinterface

module mkDct32Core(IDct32Core#(a))
   provisos(
      Add#(0, 8, b),
      Add#(1, c, TAdd#(a, b)),
      Add#(c, 1, c1),
      Add#(c, 2, c2),
      Add#(a, 10/*(5-1+6)*/, bs),
      Add#(shift32, 16, bs),
      Add#(1, shift16, shift32),
      Add#(2, shift8, shift32),
      Add#(3, shift4, shift32),
      Add#(0, TExp#(TSub#(shift32, 1)), round),
      Add#(_x1, 9, a),
      Add#(_x2, 16, c),
      Add#(_x3, 16, c1),
      Add#(_x4, 16, c2)
   );

   Vector#(16, SMUL#(a,b)) smul <- replicateM(mkSMUL);

   method Vector#(15, Bit#(16)) dct(Vector#(16, Bit#(a)) x, Bit#(5) idx);
      Bit#(c) x0[16];
      for(Integer i = 0; i < 16; i = i + 1)
         x0[i] = smul[i].c(x[i], fromInteger(g_t32[idx][i]));

      // Export 4 of DCT4 result from here
      Bit#(c1) x1[8];
      for(Integer i = 0; i < 8; i = i + 1)
         x1[i] = sExtend(x0[2 * i + 0]) + sExtend(x0[2 * i + 1]);

      // Export DCT8 result from here
      Bit#(c2) x2[4];
      for(Integer i = 0; i < 4; i = i + 1)
         x2[i] = sExtend(x1[2 * i + 0]) + sExtend(x1[2 * i + 1]);

      // Export DCT16 result from here
      Bit#(bs) x3[2];
      for(Integer i = 0; i < 2; i = i + 1)
         x3[i] = sExtend(x2[2 * i + 0]) + sExtend(x2[2 * i + 1]);

      // Export DCT32 result from here
      Bit#(bs) x4 = sExtend(x3[0]) + sExtend(x3[1]);

      // Generate DCT{4,8,16,32} coeff
      Vector#(15, Bit#(16)) coeff;

      for(Integer i = 0; i < 8; i = i + 1)
         coeff[0 + i] = roundN(x0[i], valueOf(shift4));

      for(Integer i = 0; i < 4; i = i + 1)
         coeff[8 + i] = roundN(x1[i], valueOf(shift8));

      for(Integer i = 0; i < 2; i = i + 1)
         coeff[12 + i] = roundN(x2[i], valueOf(shift16));

      coeff[14] = roundN(x4, valueOf(shift32));

      return coeff;
   endmethod
endmodule

(* synthesize *)
module mkDct32(IDct32);
   FIFOF#(Vector#(2, Vector#(32, Bit#(10)))) fifo_in     <- mkPipelineFIFOF;         // [E[0..15] O[0..15]]
   FIFOF#(Vector#(32, Bit#(16)))             fifo_out    <- mkPipelineFIFOF;
   Reg#(Bit#(2))                             dct_size    <- mkReg(3);

   Reg#(Bit#(16)) cycles <- mkReg(0);
   rule do_cycles;
      cycles <= cycles + 1;
   endrule

   // Internal
   Vector#(16, BRAM_SDP_PORT#(5,64,6,32))  mem <- replicateM(mkRAM_Nx1);

   Reg#(Bit#(9)) rowIdx <- mkConfigReg(0);
   Reg#(Bit#(6)) colCnt <- mkConfigReg(0);
   Reg#(Bit#(3)) colIdx <- mkConfigReg(0);

   Vector#(4, IDct32Core#(10)) dct_1D <- replicateM(mkDct32Core);
   Vector#(4, IDct32Core#(17)) dct_2D <- replicateM(mkDct32Core);
   Wire#(Bit#(1)) w_syncCol <- mkDWire(0);
   Reg#(Bit#(2)) syncCol <- mkReg(0);
   FIFOF#(Vector#(32, Bit#(17))) fifo_col <- mkPipelineFIFOF;        // [E[0..15] O[0..15]]
   RWire#(Tuple3#(Bit#(2), Bit#(9), Vector#(32, Bit#(16)))) wr_1D <- mkRWire;

   rule write_bram_1D(wr_1D.wget matches tagged Valid .x);
      let sz  = tpl_1(x);
      let idx = tpl_2(x);
      let dat = tpl_3(x);

      case(sz)
         0: begin
           mem[{idx[0], 3'd0}].write(0, {dat[ 5], dat[ 4], dat[ 1], dat[ 0]});
           mem[{idx[0], 3'd1}].write(0, {dat[ 7], dat[ 6], dat[ 3], dat[ 2]});
           mem[{idx[0], 3'd2}].write(0, {dat[13], dat[12], dat[ 9], dat[ 8]});
           mem[{idx[0], 3'd3}].write(0, {dat[15], dat[14], dat[10], dat[11]});
           mem[{idx[0], 3'd4}].write(0, {dat[21], dat[20], dat[17], dat[16]});
           mem[{idx[0], 3'd5}].write(0, {dat[23], dat[22], dat[19], dat[18]});
           mem[{idx[0], 3'd6}].write(0, {dat[29], dat[28], dat[25], dat[24]});
           mem[{idx[0], 3'd7}].write(0, {dat[31], dat[30], dat[27], dat[26]});
         end

         1: begin
           mem[{idx[1:0], 2'd0}].write(0, {dat[ 3], dat[ 2], dat[ 1], dat[ 0]});
           mem[{idx[1:0], 2'd1}].write(0, {dat[ 7], dat[ 6], dat[ 5], dat[ 4]});
           mem[{idx[1:0], 2'd2}].write(0, {dat[11], dat[10], dat[ 9], dat[ 8]});
           mem[{idx[1:0], 2'd3}].write(0, {dat[15], dat[14], dat[13], dat[12]});
         end

         2: begin
           mem[{idx[2:0], 1'd0}].write({2'd0, idx[5:3]}, {dat[ 3], dat[ 2], dat[ 1], dat[ 0]});
           mem[{idx[2:0], 1'd1}].write({2'd0, idx[5:3]}, {dat[ 7], dat[ 6], dat[ 5], dat[ 4]});
         end

         default: begin
            mem[idx[7:4]].write({idx[8],idx[3:0]}, pack(take(dat)));
            $display("[%6d] (%2d,%2d) %04X, %04X, %04X, %04X", cycles, idx[7:4], idx[3:0], dat[0], dat[1], dat[2], dat[3]);
         end
      endcase
   endrule

   rule do_dct_1D(fifo_in.notEmpty);
      let x = fifo_in.first;

      /*
      if (rowIdx == 0) begin
         $write("E: ");
         for(Integer i = 0; i < 32 / 2; i = i + 1) begin
            Bit#(16) tmp = sExtend(x[0][i]);
            $write("%04X, ", tmp);
         end
         $write("\n");
      end
      */

      let y0 = dct_1D[0].dct(take    (x[0]), {rowIdx[3:0], 1'd0});
      let y1 = dct_1D[1].dct(take    (x[1]), {rowIdx[3:0], 1'd0});
      let y2 = dct_1D[2].dct(takeTail(x[0]), {rowIdx[3:0], 1'd1});
      let y3 = dct_1D[3].dct(takeTail(x[1]), {rowIdx[3:0], 1'd1});

      let idxTmpA = rowIdx[3:0];
      let idxTmpB = rowIdx[7:4];
      case(dct_size)
         0: begin
            Vector#(8, Bit#(16)) m0 = takeAt(0, y0);
            Vector#(8, Bit#(16)) m1 = takeAt(0, y1);
            Vector#(8, Bit#(16)) m2 = takeAt(0, y2);
            Vector#(8, Bit#(16)) m3 = takeAt(0, y3);
            wr_1D.wset(tuple3(dct_size, rowIdx, unpack({?, pack(m3), pack(m2), pack(m1), pack(m0)})));

            // Update Row & Col sync
            idxTmpA = idxTmpA | 4'b1110;
            idxTmpB = idxTmpB | 4'b1111;
         end

         1: begin
            Vector#(4, Bit#(16)) m0 = takeAt(8, y0);
            Vector#(4, Bit#(16)) m1 = takeAt(8, y1);
            Vector#(4, Bit#(16)) m2 = takeAt(8, y2);
            Vector#(4, Bit#(16)) m3 = takeAt(8, y3);
            wr_1D.wset(tuple3(dct_size, rowIdx, unpack({?, pack(m3), pack(m2), pack(m1), pack(m0)})));

            // Update Row & Col sync
            idxTmpA = idxTmpA | 4'b1100;
            idxTmpB = idxTmpB | 4'b1100;
         end

         2: begin
            Vector#(2, Bit#(16)) m0 = takeAt(12, y0);
            Vector#(2, Bit#(16)) m1 = takeAt(12, y1);
            Vector#(2, Bit#(16)) m2 = takeAt(12, y2);
            Vector#(2, Bit#(16)) m3 = takeAt(12, y3);
            wr_1D.wset(tuple3(dct_size, rowIdx, unpack({?, pack(m3), pack(m2), pack(m1), pack(m0)})));

            // Update Row & Col sync
            idxTmpA = idxTmpA | 4'b1000;
            idxTmpB = {2'b11, rowIdx[4:3]};
         end

         default: begin
            wr_1D.wset(tuple3(dct_size, rowIdx, unpack({?, y3[14], y2[14], y1[14], y0[14]})));
         end
      endcase

      // Update Row & Col sync
      if (idxTmpA == '1) begin
         fifo_in.deq;
      end
      if (idxTmpB == '1) begin
         w_syncCol <= 1;
      end
      rowIdx <= rowIdx + 1;
   endrule

   rule read_mem;
      for(Integer i = 0; i < 16; i = i + 1) begin
         mem[i].read(colCnt);
      end
      syncCol <= {syncCol[0], w_syncCol};
   endrule

   rule pre_2D({syncCol[1],colCnt[4:0]} != 0);
      Vector#(32, Bit#(16)) x;
      Vector#(32, Bit#(17)) y;

      for(Integer i = 0; i < 32 / 2; i = i + 1) begin
         Vector#(2,Bit#(16)) t = unpack(mem[i].read_resp());
         x[i*2+0] = t[0];
         x[i*2+1] = t[1];
      end

      if (False) begin
         $write("[%6d] INP-2D: ", cycles);
         for(Integer i = 0; i < 32; i = i + 1) begin
            $write("%04X, ", x[i]);
         end
         $write("\n");
         //$write("             -O: ");
         //for(Integer i = 0; i < 16; i = i + 1) begin
         //   $write("%04X, ", x[16+i]);
         //end
         $write("\n");
      end

      for(Integer i = 0; i < 32 / 2; i = i + 1) begin
         y[ 0 + i] = sExtend(x[i]) + sExtend(x[31 - i]);
         y[16 + i] = sExtend(x[i]) - sExtend(x[31 - i]);
      end

      fifo_col.enq(y);
      colCnt <= colCnt + 1;
   endrule

   rule do_dct_2D;
      let x = fifo_col.first;

      if (False) begin
         $write("[%6d] EO-2D: ", cycles);
         for(Integer i = 0; i < 32; i = i + 1) begin
            Bit#(16) t = truncate(x[i]);
            $write("%04X, ", t);
         end
         $write("\n");
      end

      let y0 = dct_2D[0].dct(take    (x), {colIdx, 2'd0});
      let y1 = dct_2D[1].dct(takeTail(x), {colIdx, 2'd1});
      let y2 = dct_2D[2].dct(take    (x), {colIdx, 2'd2});
      let y3 = dct_2D[3].dct(takeTail(x), {colIdx, 2'd3});

      case(dct_size)
         0: begin
            Vector#(8, Bit#(16)) m0 = takeAt(0, y0);
            Vector#(8, Bit#(16)) m1 = takeAt(0, y1);
            Vector#(8, Bit#(16)) m2 = takeAt(0, y2);
            Vector#(8, Bit#(16)) m3 = takeAt(0, y3);
            fifo_out.enq(unpack({?, pack(m3), pack(m2), pack(m1), pack(m0)}));
         end

         1: begin
            Vector#(4, Bit#(16)) m0 = takeAt(8, y0);
            Vector#(4, Bit#(16)) m1 = takeAt(8, y1);
            Vector#(4, Bit#(16)) m2 = takeAt(8, y2);
            Vector#(4, Bit#(16)) m3 = takeAt(8, y3);
            fifo_out.enq(unpack({?, pack(m3), pack(m2), pack(m1), pack(m0)}));
         end

         2: begin
            Vector#(2, Bit#(16)) m0 = takeAt(12, y0);
            Vector#(2, Bit#(16)) m1 = takeAt(12, y1);
            Vector#(2, Bit#(16)) m2 = takeAt(12, y2);
            Vector#(2, Bit#(16)) m3 = takeAt(12, y3);
            fifo_out.enq(unpack({?, pack(m3), pack(m2), pack(m1), pack(m0)}));
         end

         default: begin
            fifo_out.enq(unpack({?, y3[14], y2[14], y1[14], y0[14]}));
         end
      endcase
      //$display("[%6d] %3d: %04X, %04X, %04X, %04X", cycles, colIdx, y0, y1, y2, y3);

      colIdx <= colIdx + 1;
      if (colIdx == '1) begin
         fifo_col.deq;
      end
   endrule

   // External Interface
   method Action setSize(Bit#(2) dsize);
      dct_size <= dsize;
   endmethod

   interface Put io_in = interface Put;
                            method Action put(Vector#(2, Vector#(32, Bit#(9))) x);
                               Vector#(2, Vector#(32, Bit#(10))) y = ?;

                               for(Integer i = 0; i < 32 / 2; i = i + 1) begin
                                  y[0][ 0 + i] = sExtend(x[0][i]) + sExtend(x[0][31 - i]);
                                  y[0][16 + i] = sExtend(x[0][i]) - sExtend(x[0][31 - i]);
                                  y[1][ 0 + i] = sExtend(x[1][i]) + sExtend(x[1][31 - i]);
                                  y[1][16 + i] = sExtend(x[1][i]) - sExtend(x[1][31 - i]);
                               end
                               fifo_in.enq(y);
                            endmethod
                         endinterface;

   interface Get io_out = toGet(fifo_out);

endmodule : mkDct32

`ifdef TEST_BENCH_mkDct32
import "BDPI" function Action dct32_genNew();
import "BDPI" function ActionValue#(Vector#(2, Vector#(32, Bit#(16)))) dct32_getDiff();
import "BDPI" function ActionValue#(Vector#(4, Bit#(16))) dct32_getDct();

(* synthesize *)
module mkTb(Empty);
   FIFOF#(Vector#(2, Vector#(32, Bit#(9)))) fifo_in <- mkPipelineFIFOF;
   FIFOF#(Vector#(32, Bit#(16))) fifo_out <- mkPipelineFIFOF;
   Reg#(Bit#(16)) cycles <- mkReg(0);
   Reg#(Bit#(9))  states <- mkReg(0);
   Reg#(Bit#(8))  passed <- mkReg(0);

   IDct32 dct32 <- mkDct32;

   mkConnection(toGet(fifo_in), dct32.io_in);
   mkConnection(dct32.io_out, toPut(fifo_out));

   rule do_cycles;
      cycles <= cycles + 1;
   endrule

   rule do_init(states == 0);
        dct32_genNew();
        $display("[%6d] Init DCT test data", cycles);
        states <= 1;
   endrule

   rule do_input(states >= 1 && states <= 16);
         let x <- dct32_getDiff;
         Vector#(2, Vector#(32, Bit#(9))) y;
         y[0] = map(truncate, x[0]);
         y[1] = map(truncate, x[1]);

         fifo_in.enq( y );

         $write("[%6d] ", cycles);
         for(Integer i = 0; i < 32; i = i + 1) begin
            $write("%04X, ", x[0][i]);
         end
         $write("\n         ");
         for(Integer i = 0; i < 32; i = i + 1) begin
            $write("%04X, ", x[1][i]);
         end
         $write("\n");
         states <= states + 1;
   endrule

   rule do_check_32(states >= 17 && states < 17+32*8);
      let x <- dct32_getDct;
      Vector#(4, Bit#(16)) y = take(fifo_out.first);

      if (x != y) begin
         $display("Verify failed: %X -> %X", x, y);
         $finish;
      end
      else begin
         $display("[%6d] (%3d) %X pass", cycles, states-17, x);
      end

      fifo_out.deq;
      states <= states + 1;
   endrule

   rule do_end(states >= 17+32*8);
      $display("[%d] All test passed!", passed);
      states <= 0;
      passed <= passed + 1;
      if (passed >= 10)
         $finish;
   endrule

      //states <= (states == 17+32*8) ? 0 : states + 1;
endmodule
`endif

// Copyright (c) 2016 Min Chen
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this
// list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//

import Vector::*;
import GetPut::*;
import Connectable::*;
import ConfigReg::*;
import FIFOF::*;
import SpecialFIFOs::*;
import BUtils::*;
import BRAMCore::*;
import SMul::*;
import ARAM::*;
import Utils::*;

// Internal ROM Table
Integer g_t32[32][16] =
{
    { 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 },
    { 90, 90, 88, 85, 82, 78, 73, 67, 61, 54, 46, 38, 31, 22, 13,  4 },
    { 90, 87, 80, 70, 57, 43, 25,  9, -9,-25,-43,-57,-70,-80,-87,-90 },
    { 90, 82, 67, 46, 22, -4,-31,-54,-73,-85,-90,-88,-78,-61,-38,-13 },
    { 89, 75, 50, 18,-18,-50,-75,-89,-89,-75,-50,-18, 18, 50, 75, 89 },
    { 88, 67, 31,-13,-54,-82,-90,-78,-46, -4, 38, 73, 90, 85, 61, 22 },
    { 87, 57,  9,-43,-80,-90,-70,-25, 25, 70, 90, 80, 43, -9,-57,-87 },
    { 85, 46,-13,-67,-90,-73,-22, 38, 82, 88, 54, -4,-61,-90,-78,-31 },
    { 83, 36,-36,-83,-83,-36, 36, 83, 83, 36,-36,-83,-83,-36, 36, 83 },
    { 82, 22,-54,-90,-61, 13, 78, 85, 31,-46,-90,-67,  4, 73, 88, 38 },
    { 80,  9,-70,-87,-25, 57, 90, 43,-43,-90,-57, 25, 87, 70, -9,-80 },
    { 78, -4,-82,-73, 13, 85, 67,-22,-88,-61, 31, 90, 54,-38,-90,-46 },
    { 75,-18,-89,-50, 50, 89, 18,-75,-75, 18, 89, 50,-50,-89,-18, 75 },
    { 73,-31,-90,-22, 78, 67,-38,-90,-13, 82, 61,-46,-88, -4, 85, 54 },
    { 70,-43,-87,  9, 90, 25,-80,-57, 57, 80,-25,-90, -9, 87, 43,-70 },
    { 67,-54,-78, 38, 85,-22,-90,  4, 90, 13,-88,-31, 82, 46,-73,-61 },
    { 64,-64,-64, 64, 64,-64,-64, 64, 64,-64,-64, 64, 64,-64,-64, 64 },
    { 61,-73,-46, 82, 31,-88,-13, 90, -4,-90, 22, 85,-38,-78, 54, 67 },
    { 57,-80,-25, 90, -9,-87, 43, 70,-70,-43, 87,  9,-90, 25, 80,-57 },
    { 54,-85, -4, 88,-46,-61, 82, 13,-90, 38, 67,-78,-22, 90,-31,-73 },
    { 50,-89, 18, 75,-75,-18, 89,-50,-50, 89,-18,-75, 75, 18,-89, 50 },
    { 46,-90, 38, 54,-90, 31, 61,-88, 22, 67,-85, 13, 73,-82,  4, 78 },
    { 43,-90, 57, 25,-87, 70,  9,-80, 80, -9,-70, 87,-25,-57, 90,-43 },
    { 38,-88, 73, -4,-67, 90,-46,-31, 85,-78, 13, 61,-90, 54, 22,-82 },
    { 36,-83, 83,-36,-36, 83,-83, 36, 36,-83, 83,-36,-36, 83,-83, 36 },
    { 31,-78, 90,-61,  4, 54,-88, 82,-38,-22, 73,-90, 67,-13,-46, 85 },
    { 25,-70, 90,-80, 43,  9,-57, 87,-87, 57, -9,-43, 80,-90, 70,-25 },
    { 22,-61, 85,-90, 73,-38, -4, 46,-78, 90,-82, 54,-13,-31, 67,-88 },
    { 18,-50, 75,-89, 89,-75, 50,-18,-18, 50,-75, 89,-89, 75,-50, 18 },
    { 13,-38, 61,-78, 88,-90, 85,-73, 54,-31,  4, 22,-46, 67,-82, 90 },
    {  9,-25, 43,-57, 70,-80, 87,-90, 90,-87, 80,-70, 57,-43, 25, -9 },
    {  4,-13, 22,-31, 38,-46, 54,-61, 67,-73, 78,-82, 85,-88, 90,-90 }
};

// External interface
interface IDct32;
   method Action setSize(Bit#(2) dsize);
   interface Put#(Vector#(2, Vector#(32, Bit#(9)))) io_in;
   interface Get#(Vector#(32, Bit#(16))) io_out;
endinterface

interface IDct32Core#(numeric type a);
   method Vector#(15, Bit#(16)) dct(Vector#(16, Bit#(a)) x, Bit#(5) idx);
endinterface

module mkDct32Core(IDct32Core#(a))
   provisos(
      Add#(0, 8, b),
      Add#(1, c, TAdd#(a, b)),
      Add#(c, 1, c1),
      Add#(c, 2, c2),
      Add#(a, 10/*(5-1+6)*/, bs),
      Add#(shift32, 16, bs),
      Add#(1, shift16, shift32),
      Add#(2, shift8, shift32),
      Add#(3, shift4, shift32),
      Add#(0, TExp#(TSub#(shift32, 1)), round),
      Add#(_x1, 9, a),
      Add#(_x2, 16, c),
      Add#(_x3, 16, c1),
      Add#(_x4, 16, c2)
   );

   Vector#(16, SMUL#(a,b)) smul <- replicateM(mkSMUL);

   method Vector#(15, Bit#(16)) dct(Vector#(16, Bit#(a)) x, Bit#(5) idx);
      Bit#(c) x0[16];
      for(Integer i = 0; i < 16; i = i + 1)
         x0[i] = smul[i].c(x[i], fromInteger(g_t32[idx][i]));

      // Export 4 of DCT4 result from here
      Bit#(c1) x1[8];
      for(Integer i = 0; i < 8; i = i + 1)
         x1[i] = sExtend(x0[2 * i + 0]) + sExtend(x0[2 * i + 1]);

      // Export DCT8 result from here
      Bit#(c2) x2[4];
      for(Integer i = 0; i < 4; i = i + 1)
         x2[i] = sExtend(x1[2 * i + 0]) + sExtend(x1[2 * i + 1]);

      // Export DCT16 result from here
      Bit#(bs) x3[2];
      for(Integer i = 0; i < 2; i = i + 1)
         x3[i] = sExtend(x2[2 * i + 0]) + sExtend(x2[2 * i + 1]);

      // Export DCT32 result from here
      Bit#(bs) x4 = sExtend(x3[0]) + sExtend(x3[1]);

      // Generate DCT{4,8,16,32} coeff
      Vector#(15, Bit#(16)) coeff;

      for(Integer i = 0; i < 8; i = i + 1)
         coeff[0 + i] = roundN(x0[i], valueOf(shift4));

      for(Integer i = 0; i < 4; i = i + 1)
         coeff[8 + i] = roundN(x1[i], valueOf(shift8));

      for(Integer i = 0; i < 2; i = i + 1)
         coeff[12 + i] = roundN(x2[i], valueOf(shift16));

      coeff[14] = roundN(x4, valueOf(shift32));

      return coeff;
   endmethod
endmodule

(* synthesize *)
module mkDct32(IDct32);
   FIFOF#(Vector#(2, Vector#(32, Bit#(10)))) fifo_in     <- mkPipelineFIFOF;         // [E[0..15] O[0..15]]
   FIFOF#(Vector#(32, Bit#(16)))             fifo_out    <- mkPipelineFIFOF;
   Reg#(Bit#(2))                             dct_size    <- mkReg(3);

   Reg#(Bit#(16)) cycles <- mkReg(0);
   rule do_cycles;
      cycles <= cycles + 1;
   endrule

   // Internal
   Vector#(16, BRAM_SDP_PORT#(5,64,6,32))  mem <- replicateM(mkRAM_Nx1);

   Reg#(Bit#(9)) rowIdx <- mkConfigReg(0);
   Reg#(Bit#(6)) colCnt <- mkConfigReg(0);
   Reg#(Bit#(3)) colIdx <- mkConfigReg(0);

   Vector#(4, IDct32Core#(10)) dct_1D <- replicateM(mkDct32Core);
   Vector#(4, IDct32Core#(17)) dct_2D <- replicateM(mkDct32Core);
   Wire#(Bit#(1)) w_syncCol <- mkDWire(0);
   Reg#(Bit#(2)) syncCol <- mkReg(0);
   FIFOF#(Vector#(32, Bit#(17))) fifo_col <- mkPipelineFIFOF;        // [E[0..15] O[0..15]]
   RWire#(Tuple3#(Bit#(2), Bit#(9), Vector#(32, Bit#(16)))) wr_1D <- mkRWire;

   rule write_bram_1D(wr_1D.wget matches tagged Valid .x);
      let sz  = tpl_1(x);
      let idx = tpl_2(x);
      let dat = tpl_3(x);

      case(sz)
         0: begin
           mem[{idx[0], 3'd0}].write(0, {dat[ 5], dat[ 4], dat[ 1], dat[ 0]});
           mem[{idx[0], 3'd1}].write(0, {dat[ 7], dat[ 6], dat[ 3], dat[ 2]});
           mem[{idx[0], 3'd2}].write(0, {dat[13], dat[12], dat[ 9], dat[ 8]});
           mem[{idx[0], 3'd3}].write(0, {dat[15], dat[14], dat[10], dat[11]});
           mem[{idx[0], 3'd4}].write(0, {dat[21], dat[20], dat[17], dat[16]});
           mem[{idx[0], 3'd5}].write(0, {dat[23], dat[22], dat[19], dat[18]});
           mem[{idx[0], 3'd6}].write(0, {dat[29], dat[28], dat[25], dat[24]});
           mem[{idx[0], 3'd7}].write(0, {dat[31], dat[30], dat[27], dat[26]});
         end

         1: begin
           mem[{idx[1:0], 2'd0}].write(0, {dat[ 3], dat[ 2], dat[ 1], dat[ 0]});
           mem[{idx[1:0], 2'd1}].write(0, {dat[ 7], dat[ 6], dat[ 5], dat[ 4]});
           mem[{idx[1:0], 2'd2}].write(0, {dat[11], dat[10], dat[ 9], dat[ 8]});
           mem[{idx[1:0], 2'd3}].write(0, {dat[15], dat[14], dat[13], dat[12]});
         end

         2: begin
           mem[{idx[2:0], 1'd0}].write({2'd0, idx[5:3]}, {dat[ 3], dat[ 2], dat[ 1], dat[ 0]});
           mem[{idx[2:0], 1'd1}].write({2'd0, idx[5:3]}, {dat[ 7], dat[ 6], dat[ 5], dat[ 4]});
         end

         default: begin
            mem[idx[7:4]].write({idx[8],idx[3:0]}, pack(take(dat)));
            $display("[%6d] (%2d,%2d) %04X, %04X, %04X, %04X", cycles, idx[7:4], idx[3:0], dat[0], dat[1], dat[2], dat[3]);
         end
      endcase
   endrule

   rule do_dct_1D(fifo_in.notEmpty);
      let x = fifo_in.first;

      /*
      if (rowIdx == 0) begin
         $write("E: ");
         for(Integer i = 0; i < 32 / 2; i = i + 1) begin
            Bit#(16) tmp = sExtend(x[0][i]);
            $write("%04X, ", tmp);
         end
         $write("\n");
      end
      */

      let y0 = dct_1D[0].dct(take    (x[0]), {rowIdx[3:0], 1'd0});
      let y1 = dct_1D[1].dct(take    (x[1]), {rowIdx[3:0], 1'd0});
      let y2 = dct_1D[2].dct(takeTail(x[0]), {rowIdx[3:0], 1'd1});
      let y3 = dct_1D[3].dct(takeTail(x[1]), {rowIdx[3:0], 1'd1});

      let idxTmpA = rowIdx[3:0];
      let idxTmpB = rowIdx[7:4];
      case(dct_size)
         0: begin
            Vector#(8, Bit#(16)) m0 = takeAt(0, y0);
            Vector#(8, Bit#(16)) m1 = takeAt(0, y1);
            Vector#(8, Bit#(16)) m2 = takeAt(0, y2);
            Vector#(8, Bit#(16)) m3 = takeAt(0, y3);
            wr_1D.wset(tuple3(dct_size, rowIdx, unpack({?, pack(m3), pack(m2), pack(m1), pack(m0)})));

            // Update Row & Col sync
            idxTmpA = idxTmpA | 4'b1110;
            idxTmpB = idxTmpB | 4'b1111;
         end

         1: begin
            Vector#(4, Bit#(16)) m0 = takeAt(8, y0);
            Vector#(4, Bit#(16)) m1 = takeAt(8, y1);
            Vector#(4, Bit#(16)) m2 = takeAt(8, y2);
            Vector#(4, Bit#(16)) m3 = takeAt(8, y3);
            wr_1D.wset(tuple3(dct_size, rowIdx, unpack({?, pack(m3), pack(m2), pack(m1), pack(m0)})));

            // Update Row & Col sync
            idxTmpA = idxTmpA | 4'b1100;
            idxTmpB = idxTmpB | 4'b1100;
         end

         2: begin
            Vector#(2, Bit#(16)) m0 = takeAt(12, y0);
            Vector#(2, Bit#(16)) m1 = takeAt(12, y1);
            Vector#(2, Bit#(16)) m2 = takeAt(12, y2);
            Vector#(2, Bit#(16)) m3 = takeAt(12, y3);
            wr_1D.wset(tuple3(dct_size, rowIdx, unpack({?, pack(m3), pack(m2), pack(m1), pack(m0)})));

            // Update Row & Col sync
            idxTmpA = idxTmpA | 4'b1000;
            idxTmpB = {2'b11, rowIdx[4:3]};
         end

         default: begin
            wr_1D.wset(tuple3(dct_size, rowIdx, unpack({?, y3[14], y2[14], y1[14], y0[14]})));
         end
      endcase

      // Update Row & Col sync
      if (idxTmpA == '1) begin
         fifo_in.deq;
      end
      if (idxTmpB == '1) begin
         w_syncCol <= 1;
      end
      rowIdx <= rowIdx + 1;
   endrule

   rule read_mem;
      for(Integer i = 0; i < 16; i = i + 1) begin
         mem[i].read(colCnt);
      end
      syncCol <= {syncCol[0], w_syncCol};
   endrule

   rule pre_2D({syncCol[1],colCnt[4:0]} != 0);
      Vector#(32, Bit#(16)) x;
      Vector#(32, Bit#(17)) y;

      for(Integer i = 0; i < 32 / 2; i = i + 1) begin
         Vector#(2,Bit#(16)) t = unpack(mem[i].read_resp());
         x[i*2+0] = t[0];
         x[i*2+1] = t[1];
      end

      if (False) begin
         $write("[%6d] INP-2D: ", cycles);
         for(Integer i = 0; i < 32; i = i + 1) begin
            $write("%04X, ", x[i]);
         end
         $write("\n");
         //$write("             -O: ");
         //for(Integer i = 0; i < 16; i = i + 1) begin
         //   $write("%04X, ", x[16+i]);
         //end
         $write("\n");
      end

      for(Integer i = 0; i < 32 / 2; i = i + 1) begin
         y[ 0 + i] = sExtend(x[i]) + sExtend(x[31 - i]);
         y[16 + i] = sExtend(x[i]) - sExtend(x[31 - i]);
      end

      fifo_col.enq(y);
      colCnt <= colCnt + 1;
   endrule

   rule do_dct_2D;
      let x = fifo_col.first;

      if (False) begin
         $write("[%6d] EO-2D: ", cycles);
         for(Integer i = 0; i < 32; i = i + 1) begin
            Bit#(16) t = truncate(x[i]);
            $write("%04X, ", t);
         end
         $write("\n");
      end

      let y0 = dct_2D[0].dct(take    (x), {colIdx, 2'd0});
      let y1 = dct_2D[1].dct(takeTail(x), {colIdx, 2'd1});
      let y2 = dct_2D[2].dct(take    (x), {colIdx, 2'd2});
      let y3 = dct_2D[3].dct(takeTail(x), {colIdx, 2'd3});

      case(dct_size)
         0: begin
            Vector#(8, Bit#(16)) m0 = takeAt(0, y0);
            Vector#(8, Bit#(16)) m1 = takeAt(0, y1);
            Vector#(8, Bit#(16)) m2 = takeAt(0, y2);
            Vector#(8, Bit#(16)) m3 = takeAt(0, y3);
            fifo_out.enq(unpack({?, pack(m3), pack(m2), pack(m1), pack(m0)}));
         end

         1: begin
            Vector#(4, Bit#(16)) m0 = takeAt(8, y0);
            Vector#(4, Bit#(16)) m1 = takeAt(8, y1);
            Vector#(4, Bit#(16)) m2 = takeAt(8, y2);
            Vector#(4, Bit#(16)) m3 = takeAt(8, y3);
            fifo_out.enq(unpack({?, pack(m3), pack(m2), pack(m1), pack(m0)}));
         end

         2: begin
            Vector#(2, Bit#(16)) m0 = takeAt(12, y0);
            Vector#(2, Bit#(16)) m1 = takeAt(12, y1);
            Vector#(2, Bit#(16)) m2 = takeAt(12, y2);
            Vector#(2, Bit#(16)) m3 = takeAt(12, y3);
            fifo_out.enq(unpack({?, pack(m3), pack(m2), pack(m1), pack(m0)}));
         end

         default: begin
            fifo_out.enq(unpack({?, y3[14], y2[14], y1[14], y0[14]}));
         end
      endcase
      //$display("[%6d] %3d: %04X, %04X, %04X, %04X", cycles, colIdx, y0, y1, y2, y3);

      colIdx <= colIdx + 1;
      if (colIdx == '1) begin
         fifo_col.deq;
      end
   endrule

   // External Interface
   method Action setSize(Bit#(2) dsize);
      dct_size <= dsize;
   endmethod

   interface Put io_in = interface Put;
                            method Action put(Vector#(2, Vector#(32, Bit#(9))) x);
                               Vector#(2, Vector#(32, Bit#(10))) y = ?;

                               for(Integer i = 0; i < 32 / 2; i = i + 1) begin
                                  y[0][ 0 + i] = sExtend(x[0][i]) + sExtend(x[0][31 - i]);
                                  y[0][16 + i] = sExtend(x[0][i]) - sExtend(x[0][31 - i]);
                                  y[1][ 0 + i] = sExtend(x[1][i]) + sExtend(x[1][31 - i]);
                                  y[1][16 + i] = sExtend(x[1][i]) - sExtend(x[1][31 - i]);
                               end
                               fifo_in.enq(y);
                            endmethod
                         endinterface;

   interface Get io_out = toGet(fifo_out);

endmodule : mkDct32

`ifdef TEST_BENCH_mkDct32
import "BDPI" function Action dct32_genNew();
import "BDPI" function ActionValue#(Vector#(2, Vector#(32, Bit#(16)))) dct32_getDiff();
import "BDPI" function ActionValue#(Vector#(4, Bit#(16))) dct32_getDct();

(* synthesize *)
module mkTb(Empty);
   FIFOF#(Vector#(2, Vector#(32, Bit#(9)))) fifo_in <- mkPipelineFIFOF;
   FIFOF#(Vector#(4, Bit#(16))) fifo_out <- mkPipelineFIFOF;
   Reg#(Bit#(16)) cycles <- mkReg(0);
   Reg#(Bit#(9))  states <- mkReg(0);
   Reg#(Bit#(8))  passed <- mkReg(0);

   IDct32 dct32 <- mkDct32;

   mkConnection(toGet(fifo_in), dct32.io_in);
   mkConnection(dct32.io_out, toPut(fifo_out));

   rule do_cycles;
      cycles <= cycles + 1;
   endrule

   rule do_init(states == 0);
        dct32_genNew();
        $display("[%6d] Init DCT test data", cycles);
        states <= 1;
   endrule

   rule do_input(states >= 1 && states <= 16);
         let x <- dct32_getDiff;
         Vector#(2, Vector#(32, Bit#(9))) y;
         y[0] = map(truncate, x[0]);
         y[1] = map(truncate, x[1]);

         fifo_in.enq( y );

         $write("[%6d] ", cycles);
         for(Integer i = 0; i < 32; i = i + 1) begin
            $write("%04X, ", x[0][i]);
         end
         $write("\n         ");
         for(Integer i = 0; i < 32; i = i + 1) begin
            $write("%04X, ", x[1][i]);
         end
         $write("\n");
         states <= states + 1;
   endrule

   rule do_check(states >= 17 && states < 17+32*8);
      let x <- dct32_getDct;
      let y = fifo_out.first;

      if (x != y) begin
         $display("Verify failed: %X -> %X", x, y);
         $finish;
      end
      else begin
         $display("[%6d] (%3d) %X pass", cycles, states-17, x);
      end

      fifo_out.deq;
      states <= states + 1;
   endrule

   rule do_end(states >= 17+32*8);
      $display("[%d] All test passed!", passed);
      states <= 0;
      passed <= passed + 1;
      if (passed >= 10)
         $finish;
   endrule

      //states <= (states == 17+32*8) ? 0 : states + 1;
endmodule
`endif