added toPipeOut(MIMO)

added toPipeOut(MIMO)

// Copyright (c) 2014 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import FIFOF::*;
import GetPut::*;
import Connectable::*;
import Vector::*;
import MIMO::*;
import DefaultValue::*;

interface PipeOut#(type a);
   method a first();
   method Action deq();
   method Bool notEmpty();
endinterface

typeclass PipeInOut#(type a, type b);
   function PipeOut#(a) toPipeOut(b in);
endtypeclass

typeclass MkPipeInOut#(type a, type b);
   module mkPipeOut#(b in)(PipeOut#(a));
endtypeclass

instance PipeInOut#(a, FIFOF#(a));
   function PipeOut#(a) toPipeOut(FIFOF#(a) in);
      return (interface PipeOut#(a);
		 method first = in.first;
		 method deq = in.deq;
		 method notEmpty = in.notEmpty;
	      endinterface);
   endfunction
endinstance

instance PipeInOut#(Vector#(n,a), MIMO#(k,n,sz,a));
   function PipeOut#(Vector#(n,a)) toPipeOut(MIMO#(k,n,sz,a) in);
      return (interface PipeOut#(a);
		 method first = in.first;
		 method Action deq() if (in.deqReadyN(fromInteger(valueOf(n))));
		    in.deq(fromInteger(valueOf(n)));
		 endmethod
		 method Bool notEmpty();
		    return in.deqReadyN(fromInteger(valueOf(n)));
		 endmethod
	      endinterface);
   endfunction
endinstance

instance MkPipeInOut#(a, Get#(a))
   provisos (Bits#(a, asz));
   module mkPipeOut#(Get#(a) in)(PipeOut#(a));
      FIFOF#(a) fifo <- mkFIFOF();
      rule connect;
	 let v <- in.get();
	 fifo.enq(v);
      endrule
      return toPipeOut(fifo);
   endmodule
endinstance

instance Connectable#(PipeOut#(a),Put#(a));
   module mkConnection#(PipeOut#(a) in, Put#(a) out)(Empty);
      rule connect;
	 let v = in.first;
	 in.deq();
	 out.put(v);
      endrule
   endmodule
endinstance

module mkUnfunnel#(PipeOut#(Vector#(m,a)) in)(PipeOut#(Vector#(mk, a)))
   provisos (Mul#(m, k, mk),
	     Bits#(a, asz),
	     Add#(1, b__, asz),
	     Add#(2, c__, mk),
	     Add#(d__, m, mk),
	     Add#(asz, m, e__),
	     Add#(asz, mk, f__));
   let m = fromInteger(valueOf(m));
   let mk = fromInteger(valueOf(mk));

   MIMOConfiguration cfg = defaultValue();
   MIMO#(m, mk, mk, a) mimo <- mkMIMO(cfg);
   rule consumer if (mimo.enqReadyN(m));
      Vector#(m, a) v = in.first();
      in.deq();
      mimo.enq(m, v);
   endrule

   method Vector#(mk, a) first() if (mimo.deqReadyN(mk));
      return mimo.first();
   endmethod
   method Action deq() if (mimo.deqReadyN(mk));
      mimo.deq(mk);
   endmethod
   method notEmpty();
      return mimo.deqReadyN(mk);
   endmethod
endmodule

module mkForkVector#(PipeOut#(a) inpipe)(Vector#(n, PipeOut#(a)))
   provisos (Bits#(a, asz));
   Vector#(n, FIFOF#(a)) fifos <- replicateM(mkFIFOF());
   rule forkelts;
      let v = inpipe.first();
      inpipe.deq;
      for (Integer i = 0; i < valueOf(n); i = i + 1) begin
	 fifos[i].enq(v);
      end
   endrule
   return map(toPipeOut, fifos);
endmodule

module mkJoin#(function c f(a av, b bv), PipeOut#(a) apipe, PipeOut#(b) bpipe)(PipeOut#(c));
   method c first();
      let av = apipe.first();
      let bv = bpipe.first();
      return f(av, bv);
   endmethod
   method Action deq();
      apipe.deq();
      bpipe.deq();
   endmethod
   method Bool notEmpty();
      return apipe.notEmpty() && bpipe.notEmpty();
   endmethod
endmodule

module mkMap#(function b f(a av), PipeOut#(a) apipe)(PipeOut#(b));
   method b first();
      let av = apipe.first();
      return f(av);
   endmethod
   method Action deq();
      apipe.deq();
   endmethod
   method Bool notEmpty();
      return apipe.notEmpty();
   endmethod
endmodule

typedef (function Module #(PipeOut #(tb)) mkPipeOut(PipeOut#(ta) ifc)) MkPipeOut#(type ta, type tb);

typeclass ReducePipe#( numeric type n, type a);
   module [Module] mkReducePipe (MkPipeOut#(Tuple2#(a,a), a) combinepipe,
				 PipeOut#(Vector#(n,a)) inpipe,
				 PipeOut#(a) ifc);
endtypeclass
instance ReducePipe#(1, a);
   module [Module] mkReducePipe (MkPipeOut#(Tuple2#(a,a), a) combinepipe,
				 PipeOut#(Vector#(1,a)) inpipe,
				 PipeOut#(a) ifc);
      let pipe <- mkMap(head, inpipe);
      return pipe;
   endmodule
endinstance
instance ReducePipe#(2, a);
   module [Module] mkReducePipe (MkPipeOut#(Tuple2#(a,a), a) combinepipe,
				 PipeOut#(Vector#(2,a)) inpipe,
				 PipeOut#(a) ifc);
      function Tuple2#(a,a) foo(Vector#(2,a) invec); return tuple2(invec[0], invec[1]); endfunction
      PipeOut#(Tuple2#(a,a)) zippipe <- mkMap(foo, inpipe);
      let pipe <- combinepipe(zippipe);
      return pipe;
   endmodule
endinstance

instance ReducePipe#(n, a)
   provisos (Add#(TDiv#(n,2), a__, n),
	     Bits#(Vector#(TDiv#(n,2), a), b__),
	     ReducePipe#(TDiv#(n,2),a));
   module [Module] mkReducePipe (MkPipeOut#(Tuple2#(a,a), a) combinepipe,
				 PipeOut#(Vector#(n,a)) inpipe,
				 PipeOut#(a) ifc);
      FIFOF#(Vector#(TDiv#(n,2),a)) infifo0 <- mkFIFOF;
      FIFOF#(Vector#(TDiv#(n,2),a)) infifo1 <- mkFIFOF;
      rule splitinput;
	 let v = inpipe.first();
	 inpipe.deq();
	 infifo0.enq(takeAt(0, v));
	 infifo1.enq(takeAt(valueOf(TDiv#(n,2)), v));
      endrule
      PipeOut#(Vector#(TDiv#(n,2),a)) inpipe0 = toPipeOut(infifo0);
      PipeOut#(Vector#(TDiv#(n,2),a)) inpipe1 = toPipeOut(infifo1);

      PipeOut#(a) p0 <- mkReducePipe(combinepipe, inpipe0);
      PipeOut#(a) p1 <- mkReducePipe(combinepipe, inpipe1);

      PipeOut#(Tuple2#(a,a)) tplpipe = (interface PipeOut;
					method Tuple2#(a,a) first(); return tuple2(p0.first, p1.first); endmethod
					method Action deq(); p0.deq(); p1.deq(); endmethod
					method notEmpty(); return p0.notEmpty() && p1.notEmpty(); endmethod
					endinterface);
      PipeOut#(a) outpipe <- combinepipe(tplpipe);
      return outpipe;
   endmodule
endinstance

// Copyright (c) 2014 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import FIFOF::*;
import GetPut::*;
import Connectable::*;
import Vector::*;
import MIMO::*;
import DefaultValue::*;

interface PipeOut#(type a);
   method a first();
   method Action deq();
   method Bool notEmpty();
endinterface

typeclass PipeInOut#(type a, type b);
   function PipeOut#(a) toPipeOut(b in);
endtypeclass

typeclass MkPipeInOut#(type a, type b);
   module mkPipeOut#(b in)(PipeOut#(a));
endtypeclass

instance PipeInOut#(a, FIFOF#(a));
   function PipeOut#(a) toPipeOut(FIFOF#(a) in);
      return (interface PipeOut#(a);
		 method first = in.first;
		 method deq = in.deq;
		 method notEmpty = in.notEmpty;
	      endinterface);
   endfunction
endinstance

instance MkPipeInOut#(a, Get#(a))
   provisos (Bits#(a, asz));
   module mkPipeOut#(Get#(a) in)(PipeOut#(a));
      FIFOF#(a) fifo <- mkFIFOF();
      rule connect;
	 let v <- in.get();
	 fifo.enq(v);
      endrule
      return toPipeOut(fifo);
   endmodule
endinstance

instance Connectable#(PipeOut#(a),Put#(a));
   module mkConnection#(PipeOut#(a) in, Put#(a) out)(Empty);
      rule connect;
	 let v = in.first;
	 in.deq();
	 out.put(v);
      endrule
   endmodule
endinstance

module mkUnfunnel#(PipeOut#(Vector#(m,a)) in)(PipeOut#(Vector#(mk, a)))
   provisos (Mul#(m, k, mk),
	     Bits#(a, asz),
	     Add#(1, b__, asz),
	     Add#(2, c__, mk),
	     Add#(d__, m, mk),
	     Add#(asz, m, e__),
	     Add#(asz, mk, f__));
   let m = fromInteger(valueOf(m));
   let mk = fromInteger(valueOf(mk));

   MIMOConfiguration cfg = defaultValue();
   MIMO#(m, mk, mk, a) mimo <- mkMIMO(cfg);
   rule consumer if (mimo.enqReadyN(m));
      Vector#(m, a) v = in.first();
      in.deq();
      mimo.enq(m, v);
   endrule

   method Vector#(mk, a) first() if (mimo.deqReadyN(mk));
      return mimo.first();
   endmethod
   method Action deq() if (mimo.deqReadyN(mk));
      mimo.deq(mk);
   endmethod
   method notEmpty();
      return mimo.deqReadyN(mk);
   endmethod
endmodule

module mkForkVector#(PipeOut#(a) inpipe)(Vector#(n, PipeOut#(a)))
   provisos (Bits#(a, asz));
   Vector#(n, FIFOF#(a)) fifos <- replicateM(mkFIFOF());
   rule forkelts;
      let v = inpipe.first();
      inpipe.deq;
      for (Integer i = 0; i < valueOf(n); i = i + 1) begin
	 fifos[i].enq(v);
      end
   endrule
   return map(toPipeOut, fifos);
endmodule

module mkJoin#(function c f(a av, b bv), PipeOut#(a) apipe, PipeOut#(b) bpipe)(PipeOut#(c));
   method c first();
      let av = apipe.first();
      let bv = bpipe.first();
      return f(av, bv);
   endmethod
   method Action deq();
      apipe.deq();
      bpipe.deq();
   endmethod
   method Bool notEmpty();
      return apipe.notEmpty() && bpipe.notEmpty();
   endmethod
endmodule

module mkMap#(function b f(a av), PipeOut#(a) apipe)(PipeOut#(b));
   method b first();
      let av = apipe.first();
      return f(av);
   endmethod
   method Action deq();
      apipe.deq();
   endmethod
   method Bool notEmpty();
      return apipe.notEmpty();
   endmethod
endmodule

typedef (function Module #(PipeOut #(tb)) mkPipeOut(PipeOut#(ta) ifc)) MkPipeOut#(type ta, type tb);

typeclass ReducePipe#( numeric type n, type a);
   module [Module] mkReducePipe (MkPipeOut#(Tuple2#(a,a), a) combinepipe,
				 PipeOut#(Vector#(n,a)) inpipe,
				 PipeOut#(a) ifc);
endtypeclass
instance ReducePipe#(1, a);
   module [Module] mkReducePipe (MkPipeOut#(Tuple2#(a,a), a) combinepipe,
				 PipeOut#(Vector#(1,a)) inpipe,
				 PipeOut#(a) ifc);
      let pipe <- mkMap(head, inpipe);
      return pipe;
   endmodule
endinstance
instance ReducePipe#(2, a);
   module [Module] mkReducePipe (MkPipeOut#(Tuple2#(a,a), a) combinepipe,
				 PipeOut#(Vector#(2,a)) inpipe,
				 PipeOut#(a) ifc);
      function Tuple2#(a,a) foo(Vector#(2,a) invec); return tuple2(invec[0], invec[1]); endfunction
      PipeOut#(Tuple2#(a,a)) zippipe <- mkMap(foo, inpipe);
      let pipe <- combinepipe(zippipe);
      return pipe;
   endmodule
endinstance

instance ReducePipe#(n, a)
   provisos (Add#(TDiv#(n,2), a__, n),
	     Bits#(Vector#(TDiv#(n,2), a), b__),
	     ReducePipe#(TDiv#(n,2),a));
   module [Module] mkReducePipe (MkPipeOut#(Tuple2#(a,a), a) combinepipe,
				 PipeOut#(Vector#(n,a)) inpipe,
				 PipeOut#(a) ifc);
      FIFOF#(Vector#(TDiv#(n,2),a)) infifo0 <- mkFIFOF;
      FIFOF#(Vector#(TDiv#(n,2),a)) infifo1 <- mkFIFOF;
      rule splitinput;
	 let v = inpipe.first();
	 inpipe.deq();
	 infifo0.enq(takeAt(0, v));
	 infifo1.enq(takeAt(valueOf(TDiv#(n,2)), v));
      endrule
      PipeOut#(Vector#(TDiv#(n,2),a)) inpipe0 = toPipeOut(infifo0);
      PipeOut#(Vector#(TDiv#(n,2),a)) inpipe1 = toPipeOut(infifo1);

      PipeOut#(a) p0 <- mkReducePipe(combinepipe, inpipe0);
      PipeOut#(a) p1 <- mkReducePipe(combinepipe, inpipe1);

      PipeOut#(Tuple2#(a,a)) tplpipe = (interface PipeOut;
					method Tuple2#(a,a) first(); return tuple2(p0.first, p1.first); endmethod
					method Action deq(); p0.deq(); p1.deq(); endmethod
					method notEmpty(); return p0.notEmpty() && p1.notEmpty(); endmethod
					endinterface);
      PipeOut#(a) outpipe <- combinepipe(tplpipe);
      return outpipe;
   endmodule
endinstance


print arbiter sim size

print arbiter sim size

// Copyright (c) 2017 Massachusetts Institute of Technology
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

`include "ProcConfig.bsv"

import GetPut::*;
import Assert::*;
import CacheUtils::*;
import Fifo::*;
import Types::*;
import ProcTypes::*;
import CCTypes::*;
import LLPipe::*;
import LLCRqMshr::*;
import LLCRqMshrSecureModel::*;
import LLBank::*;
import L1CoCache::*;
import LLCDmaConnect::*;
import Performance::*;

// Last-Level

// whether we model the effect of MSHR partition for security purpose
`ifdef SECURITY
`ifndef DISABLE_SECURE_LLC_MSHR
`define USE_LLC_MSHR_SECURE_MODEL
`endif
`endif

// whether we model the effect of the circular/fair arbiter at the entry point
// of LLC pipeline for security purpose
`ifdef SECURITY
`ifndef DISABLE_SECURE_LLC_ARBITER
`define USE_LLC_ARBITER_SECURE_MODEL
`endif
`endif

typedef `LOG_LLC_LINES LgLLLineNum;
typedef `LOG_LLC_WAYS LgLLWayNum;
typedef TExp#(LgLLWayNum) LLWayNum;
typedef 0 LgLLBankNum;
typedef TSub#(LgLLLineNum, TAdd#(LgLLWayNum, LgLLBankNum)) LgLLSetNum;

typedef Bit#(LgLLBankNum) LLBankId;
typedef LgLLSetNum LLIndexSz;
typedef Bit#(LLIndexSz) LLIndex;
typedef GetTagSz#(LgLLBankNum, LgLLSetNum) LLTagSz;
typedef Bit#(LLTagSz) LLTag;
typedef Bit#(TLog#(LLWayNum)) LLWay;

`ifdef USE_LLC_MSHR_SECURE_MODEL
typedef TDiv#(DramMaxReqs, 2) LLCRqNum; // SECURITY: limit MSHR size <= DRAM bandwidth
`else
typedef LLWayNum LLCRqNum;
`endif
typedef Bit#(TLog#(LLCRqNum)) LLCRqMshrIdx;

// all L1$ are children
typedef L1Num LLChildNum;
typedef Bit#(TLog#(LLChildNum)) LLChild;
typedef L1Way LLCRqId;

`ifdef USE_LLC_MSHR_SECURE_MODEL
module mkLastLvCRqMshr(
    LLCRqMshr#(LLChildNum, LLCRqNum, LLWay, LLTag, cRqT)
) provisos(
    Alias#(cRqT, LLRq#(LLCRqId, LLCDmaReqId, LLChild))
);
    function Addr getAddr(cRqT r) = r.addr;
    LLCRqMshrSecureModel#(
        `SIM_LOG_LLC_MSHR_BANK_NUM, LLChildNum, LLCRqNum, LLWay, LLTag, cRqT
    ) m <- mkLLCRqMshrSecureModel(getAddr);
    return m.mshr;
endmodule

`else

(* synthesize *)
module mkLastLvCRqMshr(
    LLCRqMshr#(LLChildNum, LLCRqNum, LLWay, LLTag, cRqT)
) provisos(
    Alias#(cRqT, LLRq#(LLCRqId, LLCDmaReqId, LLChild))
);
    function Addr getAddr(cRqT r) = r.addr;
    let m <- mkLLCRqMshr(getAddr);
    return m;
endmodule
`endif

`ifdef USE_LLC_ARBITER_SECURE_MODEL
typedef `SIM_LLC_ARBITER_NUM SimLLCArbNum;
(* synthesize *)
module mkLLPipeline(
    LLPipe#(LgLLBankNum, LLChildNum, LLWayNum, LLIndex, LLTag, LLCRqMshrIdx)
);
    // pipeline
    LLPipe#(LgLLBankNum, LLChildNum, LLWayNum, LLIndex, LLTag, LLCRqMshrIdx) m <- mkLLPipe;

    // round-robin reg: only allow entry to pipeline when turn == 0. This
    // models the effect of a circular/fair arbiter.
    Reg#(Bit#(TLog#(SimLLCArbNum))) turn <- mkReg(0);

    (* fire_when_enabled, no_implicit_conditions *)
    rule incrTurn;
        turn <= turn == fromInteger(valueof(SimLLCArbNum) - 1) ? 0 : turn + 1;
    endrule

`ifdef BSIM
    // print the arbiter num
    Reg#(Bool) showArbiter <- mkReg(True);
    rule doShowArbiter(showArbiter);
        $display("[LLPipe] Arbiter size %d", valueof(SimLLCArbNum));
        showArbiter <= False;
    endrule
`endif

    method Action send(LLPipeIn#(LLChild, LLWay, LLCRqMshrIdx) r) if(turn == 0);
        m.send(r);
    endmethod

    method notEmpty = m.notEmpty;
    method first = m.first;
    method unguard_first = m.unguard_first;
    method deqWrite = m.deqWrite;
endmodule

`else

(* synthesize *)
module mkLLPipeline(
    LLPipe#(LgLLBankNum, LLChildNum, LLWayNum, LLIndex, LLTag, LLCRqMshrIdx)
);
    let m <- mkLLPipe;
    return m;
endmodule
`endif

typedef LLBank#(LgLLBankNum, LLChildNum, LLWayNum, LLIndexSz, LLTagSz, LLCRqNum, LLCRqId, LLCDmaReqId) LLBankWrapper;
//typedef MemFifoClient#(LdMemRqId#(LLCRqMshrIdx), void) LLCMemFifoClient;
typedef LLCRqStuck#(LLChildNum, LLCRqId, LLCDmaReqId) LLCStuck;

interface LLCache;
    interface ParentCacheToChild#(LLCRqId, LLChild) to_child;
    interface DmaServer#(LLCDmaReqId) dma;
    interface MemFifoClient#(LdMemRqId#(LLCRqMshrIdx), void) to_mem;
    // detect deadlock: only in use when macro CHECK_DEADLOCK is defined
    interface Get#(LLCStuck) cRqStuck;
    // performance
    interface Perf#(LLCPerfType) perf;
endinterface

`ifdef SECURITY
// We rotate the addr in/out LLC to achieve set partition
// FIXME This is a hack: we simulate the performance of partitioning a large
// LLC using a smaller LLC with less partitions. So LgLLCPartitionNum should
// NOT be viewed as the number of DRAM regions.
`ifdef SIM_LOG_LLC_PARTITION_NUM
typedef `SIM_LOG_LLC_PARTITION_NUM LgLLCPartitionNum;
`else
typedef `LOG_DRAM_REGION_NUM LgLLCPartitionNum;
`endif
typedef `LOG_DRAM_REGION_SIZE LgDramRegionSz;
typedef TAdd#(TAdd#(LLIndexSz, LgLLBankNum), LgLineSzBytes) LLIndexBankOffsetSz;

function Addr secureRotateAddr(Addr addr) provisos(
    // region/partition id cannot be wider than index + bank id
    Add#(LgLLCPartitionNum, a__, TAdd#(LLIndexSz, LgLLBankNum))
);
    // low bits: index + bank id + line offset without the higher bits which
    // will be replaced by region/partition id
    Bit#(TSub#(LLIndexBankOffsetSz, LgLLCPartitionNum)) low = truncate(addr);
    // swap bits: higher bits of index + bank id to be swapped with region/partition
    // id
    Bit#(LgLLCPartitionNum) swap = truncate(addr >> (valueof(LLIndexBankOffsetSz) - valueof(LgLLCPartitionNum)));
    // middle bits between swap and region
    Bit#(TSub#(LgDramRegionSz, LLIndexBankOffsetSz)) mid = truncate(addr >> valueof(LLIndexBankOffsetSz));
    // region/partition id
    Bit#(LgLLCPartitionNum) region = truncate(addr >> valueof(LgDramRegionSz));
    // high bits beyond phy mem boundary
    Bit#(TSub#(AddrSz, TAdd#(LgLLCPartitionNum, LgDramRegionSz))) high = truncateLSB(addr);
    // exchange swap bits with region bits
    return {high, swap, mid, region, low};
endfunction
`endif

(* synthesize *)
module mkLLCache(LLCache);
`ifdef DEBUG_DMA
    staticAssert(False, "DEBUG_DMA should not be defined");
`endif

    LLBankWrapper cache <- mkLLBank(mkLastLvCRqMshr, mkLLPipeline);

    // perf counters
    Fifo#(1, LLCPerfType) perfReqQ <- mkCFFifo;
`ifdef PERF_COUNT
    Fifo#(1, PerfResp#(LLCPerfType)) perfRespQ <- mkCFFifo;

    rule doPerf;
        let t <- toGet(perfReqQ).get;
        let d = cache.getPerfData(t);
        perfRespQ.enq(PerfResp {
            pType: t,
            data: d
        });
    endrule
`endif

`ifdef SECURITY
`ifndef DISABLE_SECURE_LLC

`ifdef BSIM
    // print the LLC partition in simulation
    Reg#(Bool) showLLCPartition <- mkReg(True);
    rule doShowLLCPartition(showLLCPartition);
        $fdisplay(stderr, "[LLCache] log LLC partition = %d", valueof(LgLLCPartitionNum));
        showLLCPartition <= False;
    endrule
`endif

    // rotate addr to achieve LLC set partition
    interface ParentCacheToChild to_child;
        interface FifoEnq rsFromC;
            method notFull = cache.to_child.rsFromC.notFull;
            method Action enq(CRsMsg#(LLChild) x);
                let y = x;
                y.addr = secureRotateAddr(x.addr);
                cache.to_child.rsFromC.enq(y);
            endmethod
        endinterface
        interface FifoEnq rqFromC;
            method notFull = cache.to_child.rqFromC.notFull;
            method Action enq(CRqMsg#(LLCRqId, LLChild) x);
                let y = x;
                y.addr = secureRotateAddr(x.addr);
                cache.to_child.rqFromC.enq(y);
            endmethod
        endinterface
        interface FifoDeq toC;
            method notEmpty = cache.to_child.toC.notEmpty;
            method deq = cache.to_child.toC.deq;
            method PRqRsMsg#(LLCRqId, LLChild) first;
                case(cache.to_child.toC.first) matches
                    tagged PRq .x: begin
                        let y = x;
                        y.addr = secureRotateAddr(x.addr);
                        return PRq (y);
                    end
                    tagged PRs .x: begin
                        let y = x;
                        y.addr = secureRotateAddr(x.addr);
                        return PRs (y);
                    end
                    default: return ?;
                endcase
            endmethod
        endinterface
    endinterface

    interface DmaServer dma;
        interface FifoEnq memReq;
            method notFull = cache.dma.memReq.notFull;
            method Action enq(DmaRq#(LLCDmaReqId) x);
                let y = x;
                y.addr = secureRotateAddr(x.addr);
                cache.dma.memReq.enq(y);
            endmethod
        endinterface
        interface respLd = cache.dma.respLd;
        interface respSt = cache.dma.respSt;
    endinterface

    interface MemFifoClient to_mem;
        interface FifoDeq toM;
            method notEmpty = cache.to_mem.toM.notEmpty;
            method deq = cache.to_mem.toM.deq;
            method ToMemMsg#(LdMemRqId#(LLCRqMshrIdx), void) first;
                case(cache.to_mem.toM.first) matches
                    tagged Ld .x: begin
                        let y = x;
                        y.addr = secureRotateAddr(x.addr);
                        return Ld (y);
                    end
                    tagged Wb .x: begin
                        let y = x;
                        y.addr = secureRotateAddr(x.addr);
                        return Wb (y);
                    end
                    default: return ?;
                endcase
            endmethod
        endinterface
        interface rsFromM = cache.to_mem.rsFromM;
    endinterface

    interface Get cRqStuck;
        method ActionValue#(LLCStuck) get;
            let x <- cache.cRqStuck.get;
            let y = x;
            y.addr = secureRotateAddr(x.addr);
            return y;
        endmethod
    endinterface

`else // DISABLE_SECURE_LLC

    interface to_child = cache.to_child;
    interface dma = cache.dma;
    interface to_mem = cache.to_mem;
    interface cRqStuck = cache.cRqStuck;

`endif // DISABLE_SECURE_LLC
`else // SECURITY

    interface to_child = cache.to_child;
    interface dma = cache.dma;
    interface to_mem = cache.to_mem;
    interface cRqStuck = cache.cRqStuck;

`endif // SECURITY

    interface Perf perf;
        method Action setStatus(Bool stats);
            cache.setPerfStatus(stats);
        endmethod
        method Action req(LLCPerfType r);
            perfReqQ.enq(r);
        endmethod
        method ActionValue#(PerfResp#(LLCPerfType)) resp;
`ifdef PERF_COUNT
            perfRespQ.deq;
            return perfRespQ.first;
`else
            perfReqQ.deq;
            return PerfResp {
                pType: perfReqQ.first,
                data: 0
            };
`endif
        endmethod
        method Bool respValid;
`ifdef PERF_COUNT
            return perfRespQ.notEmpty;
`else
            return perfReqQ.notEmpty;
`endif
        endmethod
    endinterface
endmodule

// Copyright (c) 2017 Massachusetts Institute of Technology
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

`include "ProcConfig.bsv"

import GetPut::*;
import Assert::*;
import CacheUtils::*;
import Fifo::*;
import Types::*;
import ProcTypes::*;
import CCTypes::*;
import LLPipe::*;
import LLCRqMshr::*;
import LLCRqMshrSecureModel::*;
import LLBank::*;
import L1CoCache::*;
import LLCDmaConnect::*;
import Performance::*;

// Last-Level

// whether we model the effect of MSHR partition for security purpose
`ifdef SECURITY
`ifndef DISABLE_SECURE_LLC_MSHR
`define USE_LLC_MSHR_SECURE_MODEL
`endif
`endif

// whether we model the effect of the circular/fair arbiter at the entry point
// of LLC pipeline for security purpose
`ifdef SECURITY
`ifndef DISABLE_SECURE_LLC_ARBITER
`define USE_LLC_ARBITER_SECURE_MODEL
`endif
`endif

typedef `LOG_LLC_LINES LgLLLineNum;
typedef `LOG_LLC_WAYS LgLLWayNum;
typedef TExp#(LgLLWayNum) LLWayNum;
typedef 0 LgLLBankNum;
typedef TSub#(LgLLLineNum, TAdd#(LgLLWayNum, LgLLBankNum)) LgLLSetNum;

typedef Bit#(LgLLBankNum) LLBankId;
typedef LgLLSetNum LLIndexSz;
typedef Bit#(LLIndexSz) LLIndex;
typedef GetTagSz#(LgLLBankNum, LgLLSetNum) LLTagSz;
typedef Bit#(LLTagSz) LLTag;
typedef Bit#(TLog#(LLWayNum)) LLWay;

`ifdef USE_LLC_MSHR_SECURE_MODEL
typedef TDiv#(DramMaxReqs, 2) LLCRqNum; // SECURITY: limit MSHR size <= DRAM bandwidth
`else
typedef LLWayNum LLCRqNum;
`endif
typedef Bit#(TLog#(LLCRqNum)) LLCRqMshrIdx;

// all L1$ are children
typedef L1Num LLChildNum;
typedef Bit#(TLog#(LLChildNum)) LLChild;
typedef L1Way LLCRqId;

`ifdef USE_LLC_MSHR_SECURE_MODEL
module mkLastLvCRqMshr(
    LLCRqMshr#(LLChildNum, LLCRqNum, LLWay, LLTag, cRqT)
) provisos(
    Alias#(cRqT, LLRq#(LLCRqId, LLCDmaReqId, LLChild))
);
    function Addr getAddr(cRqT r) = r.addr;
    LLCRqMshrSecureModel#(
        `SIM_LOG_LLC_MSHR_BANK_NUM, LLChildNum, LLCRqNum, LLWay, LLTag, cRqT
    ) m <- mkLLCRqMshrSecureModel(getAddr);
    return m.mshr;
endmodule

`else

(* synthesize *)
module mkLastLvCRqMshr(
    LLCRqMshr#(LLChildNum, LLCRqNum, LLWay, LLTag, cRqT)
) provisos(
    Alias#(cRqT, LLRq#(LLCRqId, LLCDmaReqId, LLChild))
);
    function Addr getAddr(cRqT r) = r.addr;
    let m <- mkLLCRqMshr(getAddr);
    return m;
endmodule
`endif

`ifdef USE_LLC_ARBITER_SECURE_MODEL
typedef `SIM_LLC_ARBITER_NUM SimLLCArbNum;
(* synthesize *)
module mkLLPipeline(
    LLPipe#(LgLLBankNum, LLChildNum, LLWayNum, LLIndex, LLTag, LLCRqMshrIdx)
);
    // pipeline
    LLPipe#(LgLLBankNum, LLChildNum, LLWayNum, LLIndex, LLTag, LLCRqMshrIdx) m <- mkLLPipe;

    // round-robin reg: only allow entry to pipeline when turn == 0. This
    // models the effect of a circular/fair arbiter.
    Reg#(Bit#(TLog#(SimLLCArbNum))) turn <- mkReg(0);

    (* fire_when_enabled, no_implicit_conditions *)
    rule incrTurn;
        turn <= turn == fromInteger(valueof(SimLLCArbNum) - 1) ? 0 : turn + 1;
    endrule

    method Action send(LLPipeIn#(LLChild, LLWay, LLCRqMshrIdx) r) if(turn == 0);
        m.send(r);
    endmethod

    method notEmpty = m.notEmpty;
    method first = m.first;
    method unguard_first = m.unguard_first;
    method deqWrite = m.deqWrite;
endmodule

`else

(* synthesize *)
module mkLLPipeline(
    LLPipe#(LgLLBankNum, LLChildNum, LLWayNum, LLIndex, LLTag, LLCRqMshrIdx)
);
    let m <- mkLLPipe;
    return m;
endmodule
`endif

typedef LLBank#(LgLLBankNum, LLChildNum, LLWayNum, LLIndexSz, LLTagSz, LLCRqNum, LLCRqId, LLCDmaReqId) LLBankWrapper;
//typedef MemFifoClient#(LdMemRqId#(LLCRqMshrIdx), void) LLCMemFifoClient;
typedef LLCRqStuck#(LLChildNum, LLCRqId, LLCDmaReqId) LLCStuck;

interface LLCache;
    interface ParentCacheToChild#(LLCRqId, LLChild) to_child;
    interface DmaServer#(LLCDmaReqId) dma;
    interface MemFifoClient#(LdMemRqId#(LLCRqMshrIdx), void) to_mem;
    // detect deadlock: only in use when macro CHECK_DEADLOCK is defined
    interface Get#(LLCStuck) cRqStuck;
    // performance
    interface Perf#(LLCPerfType) perf;
endinterface

`ifdef SECURITY
// We rotate the addr in/out LLC to achieve set partition
// FIXME This is a hack: we simulate the performance of partitioning a large
// LLC using a smaller LLC with less partitions. So LgLLCPartitionNum should
// NOT be viewed as the number of DRAM regions.
`ifdef SIM_LOG_LLC_PARTITION_NUM
typedef `SIM_LOG_LLC_PARTITION_NUM LgLLCPartitionNum;
`else
typedef `LOG_DRAM_REGION_NUM LgLLCPartitionNum;
`endif
typedef `LOG_DRAM_REGION_SIZE LgDramRegionSz;
typedef TAdd#(TAdd#(LLIndexSz, LgLLBankNum), LgLineSzBytes) LLIndexBankOffsetSz;

function Addr secureRotateAddr(Addr addr) provisos(
    // region/partition id cannot be wider than index + bank id
    Add#(LgLLCPartitionNum, a__, TAdd#(LLIndexSz, LgLLBankNum))
);
    // low bits: index + bank id + line offset without the higher bits which
    // will be replaced by region/partition id
    Bit#(TSub#(LLIndexBankOffsetSz, LgLLCPartitionNum)) low = truncate(addr);
    // swap bits: higher bits of index + bank id to be swapped with region/partition
    // id
    Bit#(LgLLCPartitionNum) swap = truncate(addr >> (valueof(LLIndexBankOffsetSz) - valueof(LgLLCPartitionNum)));
    // middle bits between swap and region
    Bit#(TSub#(LgDramRegionSz, LLIndexBankOffsetSz)) mid = truncate(addr >> valueof(LLIndexBankOffsetSz));
    // region/partition id
    Bit#(LgLLCPartitionNum) region = truncate(addr >> valueof(LgDramRegionSz));
    // high bits beyond phy mem boundary
    Bit#(TSub#(AddrSz, TAdd#(LgLLCPartitionNum, LgDramRegionSz))) high = truncateLSB(addr);
    // exchange swap bits with region bits
    return {high, swap, mid, region, low};
endfunction
`endif

(* synthesize *)
module mkLLCache(LLCache);
`ifdef DEBUG_DMA
    staticAssert(False, "DEBUG_DMA should not be defined");
`endif

    LLBankWrapper cache <- mkLLBank(mkLastLvCRqMshr, mkLLPipeline);

    // perf counters
    Fifo#(1, LLCPerfType) perfReqQ <- mkCFFifo;
`ifdef PERF_COUNT
    Fifo#(1, PerfResp#(LLCPerfType)) perfRespQ <- mkCFFifo;

    rule doPerf;
        let t <- toGet(perfReqQ).get;
        let d = cache.getPerfData(t);
        perfRespQ.enq(PerfResp {
            pType: t,
            data: d
        });
    endrule
`endif

`ifdef SECURITY
`ifndef DISABLE_SECURE_LLC

`ifdef BSIM
    // print the LLC partition in simulation
    Reg#(Bool) showLLCPartition <- mkReg(True);
    rule doShowLLCPartition(showLLCPartition);
        $fdisplay(stderr, "[LLCache] log LLC partition = %d", valueof(LgLLCPartitionNum));
        showLLCPartition <= False;
    endrule
`endif

    // rotate addr to achieve LLC set partition
    interface ParentCacheToChild to_child;
        interface FifoEnq rsFromC;
            method notFull = cache.to_child.rsFromC.notFull;
            method Action enq(CRsMsg#(LLChild) x);
                let y = x;
                y.addr = secureRotateAddr(x.addr);
                cache.to_child.rsFromC.enq(y);
            endmethod
        endinterface
        interface FifoEnq rqFromC;
            method notFull = cache.to_child.rqFromC.notFull;
            method Action enq(CRqMsg#(LLCRqId, LLChild) x);
                let y = x;
                y.addr = secureRotateAddr(x.addr);
                cache.to_child.rqFromC.enq(y);
            endmethod
        endinterface
        interface FifoDeq toC;
            method notEmpty = cache.to_child.toC.notEmpty;
            method deq = cache.to_child.toC.deq;
            method PRqRsMsg#(LLCRqId, LLChild) first;
                case(cache.to_child.toC.first) matches
                    tagged PRq .x: begin
                        let y = x;
                        y.addr = secureRotateAddr(x.addr);
                        return PRq (y);
                    end
                    tagged PRs .x: begin
                        let y = x;
                        y.addr = secureRotateAddr(x.addr);
                        return PRs (y);
                    end
                    default: return ?;
                endcase
            endmethod
        endinterface
    endinterface

    interface DmaServer dma;
        interface FifoEnq memReq;
            method notFull = cache.dma.memReq.notFull;
            method Action enq(DmaRq#(LLCDmaReqId) x);
                let y = x;
                y.addr = secureRotateAddr(x.addr);
                cache.dma.memReq.enq(y);
            endmethod
        endinterface
        interface respLd = cache.dma.respLd;
        interface respSt = cache.dma.respSt;
    endinterface

    interface MemFifoClient to_mem;
        interface FifoDeq toM;
            method notEmpty = cache.to_mem.toM.notEmpty;
            method deq = cache.to_mem.toM.deq;
            method ToMemMsg#(LdMemRqId#(LLCRqMshrIdx), void) first;
                case(cache.to_mem.toM.first) matches
                    tagged Ld .x: begin
                        let y = x;
                        y.addr = secureRotateAddr(x.addr);
                        return Ld (y);
                    end
                    tagged Wb .x: begin
                        let y = x;
                        y.addr = secureRotateAddr(x.addr);
                        return Wb (y);
                    end
                    default: return ?;
                endcase
            endmethod
        endinterface
        interface rsFromM = cache.to_mem.rsFromM;
    endinterface

    interface Get cRqStuck;
        method ActionValue#(LLCStuck) get;
            let x <- cache.cRqStuck.get;
            let y = x;
            y.addr = secureRotateAddr(x.addr);
            return y;
        endmethod
    endinterface

`else // DISABLE_SECURE_LLC

    interface to_child = cache.to_child;
    interface dma = cache.dma;
    interface to_mem = cache.to_mem;
    interface cRqStuck = cache.cRqStuck;

`endif // DISABLE_SECURE_LLC
`else // SECURITY

    interface to_child = cache.to_child;
    interface dma = cache.dma;
    interface to_mem = cache.to_mem;
    interface cRqStuck = cache.cRqStuck;

`endif // SECURITY

    interface Perf perf;
        method Action setStatus(Bool stats);
            cache.setPerfStatus(stats);
        endmethod
        method Action req(LLCPerfType r);
            perfReqQ.enq(r);
        endmethod
        method ActionValue#(PerfResp#(LLCPerfType)) resp;
`ifdef PERF_COUNT
            perfRespQ.deq;
            return perfRespQ.first;
`else
            perfReqQ.deq;
            return PerfResp {
                pType: perfReqQ.first,
                data: 0
            };
`endif
        endmethod
        method Bool respValid;
`ifdef PERF_COUNT
            return perfRespQ.notEmpty;
`else
            return perfReqQ.notEmpty;
`endif
        endmethod
    endinterface
endmodule


use zipWIthM instead of genWithM(uncurry(f), zip(l1, l2)) in mkMemReadEngineBuff

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
import Vector::*;
import BuildVector::*;
import Cntrs::*;
import FIFOF::*;
import FIFO::*;
import GetPut::*;
import Assert::*;
import ClientServer::*;
import BRAM::*;
import BRAMFIFO::*;
import ConfigCounter::*;
import Connectable::*;
import ConnectalMemory::*;
import ConnectalBramFifo::*;
import MemTypes::*;
import Pipe::*;
import MemUtils::*;
import ConnectalConfig::*;

`include "ConnectalProjectConfig.bsv"

typedef 2 MemReadFunnelBPC;

module mkMemReadEngine(MemReadEngine#(busWidth, userWidth, cmdQDepth, numServers))
   provisos( Mul#(TDiv#(busWidth, 8), 8, busWidth)
	    ,Add#(1, a__, numServers)
	    ,Add#(busWidth, 0, userWidth)
//	     ,Min#(MemReadFunnelBPC, TLog#(numServers), bpc)
	     ,Add#(0,MemReadFunnelBPC,bpc)
	    ,FunnelPipesPipelined#(1, numServers, MemTypes::MemData#(userWidth), bpc)
	    ,Pipe::FunnelPipesPipelined#(1, numServers, MemTypes::MemRequest, bpc)
	    ,Add#(b__, TLog#(numServers), MemTagSize)
      	    );
   let rv <- mkMemReadEngineBuff(valueOf(cmdQDepth) * valueOf(TExp#(BurstLenSize)));
   return rv;
endmodule

typedef struct {
    Bit#(BurstLenSize) len;
    Bool               last;
} NextReq deriving (Bits, Eq);

function NextReq getNext(Bit#(32) len, Bit#(BurstLenSize) burst);
   NextReq v;
   v.last = (len <= extend(burst));
   v.len = v.last ? truncate(len) : burst;
   return v;
endfunction

interface MemReadChannel#(numeric type busWidth, numeric type userWidth, numeric type cmdQDepth);
   interface PipeOut#(MemRequest)        readReq;
   interface PipeIn#(MemData#(busWidth)) readData;
   interface MemReadEngineServer#(userWidth) readServer;
endinterface

module mkMemReadChannel#(Integer bufferSizeBytes, Integer channelNumber, PipeOut#(MemData#(userWidth)) dataPipe)
   (MemReadChannel#(busWidth, userWidth, cmdQDepth))
   provisos (Div#(busWidth,8,busWidthBytes),
	     Mul#(busWidthBytes,8,busWidth),
	     Log#(busWidthBytes,beatShift),
	     Log#(cmdQDepth,logCmdQDepth),
	     Add#(busWidth, 0, userWidth),
	     Add#(1,logCmdQDepth, outCntSz));
   let verbose = False;
   let beatShift = fromInteger(valueOf(beatShift));

   let clock <- exposeCurrentClock();
   let reset <- exposeCurrentReset();

   Integer bufferSizeBeats = bufferSizeBytes/valueOf(busWidthBytes);
   Reg#(Bool)          clientInFlight <- mkReg(False);
   ConfigCounter#(16)  clientAvail <- mkConfigCounter(fromInteger(bufferSizeBeats));
   Reg#(MemengineCmd)  clientCommand <- mkReg(unpack(0));
`ifdef USE_DUAL_CLOCK_FIFOF
   FIFOF#(MemDataF#(userWidth)) clientDataFifo <- mkDualClockBramFIFOF(clock, reset, clock, reset);
`else
   FIFOF#(MemDataF#(userWidth)) clientDataFifo <- mkSizedBRAMFIFOF(bufferSizeBeats);
`endif
   FIFO#(MemengineCmd) clientRequest <- mkFIFO();
   Reg#(Bit#(32))      clientLen <- mkReg(unpack(0));
   Reg#(Bit#(32))      clientBase <- mkReg(unpack(0));
   Reg#(NextReq)       clientNext <- mkReg(unpack(0));
   Count#(Bit#(32))    clientCycles <- mkCount(0);
   FIFOF#(MemRequestCycles) clientCyclesFifo <- mkFIFOF();

   FIFO#(Tuple3#(Bool,Bool,Bit#(BurstLenSize))) serverCheckAvail <- mkSizedFIFO(1);
   FIFOF#(MemRequest)                          dmaRequest <- mkSizedFIFOF(valueOf(cmdQDepth));
   FIFO#(Tuple3#(Bit#(8),Bit#(MemTagSize),Bool)) serverProcessing <- mkSizedFIFO(valueOf(cmdQDepth));
   FIFOF#(MemData#(busWidth))                       serverDataFifo <- mkFIFOF;

   Reg#(Bit#(8))                    respCnt <- mkReg(0);
   Reg#(Bit#(32)) counter <- mkReg(0);

   rule incCounter;
      counter <= counter + 1;
   endrule

   rule rule_cycles;
      clientCycles.incr(1);
   endrule
   rule rule_startNew if (!clientInFlight);
      let cmd <- toGet(clientRequest).get();
      clientInFlight <= True;
      clientCommand <= cmd;
      clientLen <= cmd.len - extend(cmd.burstLen);
      clientBase <= cmd.base;
      clientNext <= getNext(cmd.len, cmd.burstLen);
      clientCycles <= 0;
      if (verbose) $display("mkMemReadEngineBuff::%d rule_startNew %d", counter, clientAvail.read);
   endrule
   rule rule_checkAvail if (clientInFlight);
      let cmd_len = clientNext.len;
      let last_burst = clientNext.last;
      let cond0 <- clientAvail.maybeDecrement(unpack(extend(cmd_len>>beatShift)));
      serverCheckAvail.enq(tuple3(cond0,last_burst,cmd_len));
      if (verbose) $display("mkMemReadEngineBuff::%d rule_checkAvail avail %d burstLen %d cond0 %d last_burst %d", counter, clientAvail.read(), cmd_len>>beatShift, cond0, last_burst);
   endrule

   rule rule_requestServer if (clientInFlight);
      match {.cond0,.last_burst,.cmd_len} <- toGet(serverCheckAvail).get;
      if  (cond0) begin
	 if (verbose) $display("mkMemReadEngineBuff::%d rule_requestServer clientLen %d cond0 %d last_burst %d", counter, clientLen, cond0, last_burst);
	 serverProcessing.enq(tuple3(truncate(cmd_len>>beatShift), clientCommand.tag, last_burst));
	 if (verbose) $display("MemReadEngine::%d readReq idx %d offset %h burstLenBytes %h last %d", counter, 0, clientBase, cmd_len, last_burst);

	 dmaRequest.enq(MemRequest { sglId: clientCommand.sglId, offset: extend(clientBase),
	    burstLen:cmd_len, tag: fromInteger(channelNumber)
	    `ifdef BYTE_ENABLES
				    , firstbe: maxBound, lastbe: maxBound
	    `endif
	    });
         clientBase <= clientBase + extend(cmd_len);
         clientLen <= clientLen - extend(cmd_len);
         clientNext <= getNext(clientLen, clientCommand.burstLen);
	 if (last_burst) begin
	    if (verbose) $display("mkMemReadEngineBuff::%d rule_requestServer last_burst %d", counter, last_burst);
	    clientInFlight <= False;
	    `ifdef MEMENGINE_REQUEST_CYCLES
	    $display("clientCycles = %d", clientCycles);
	    clientCyclesFifo.enq(MemRequestCycles { tag:clientCommand.tag, cycles: clientCycles });
	    `endif
	 end
      end
   endrule

   rule read_data_rule;
      let d <- toGet(dataPipe).get();
      match {.rc, .tag, .last_burst} = serverProcessing.first;
      let new_respCnt = respCnt+1;
      let l = False;
      if (verbose) $display("mkMemReadEngineBuff::%d data %h new_respCnt %d rc %d last_burst %d tag %d clientInFlight %d eob %d", counter, d.data, new_respCnt, rc, last_burst, tag, clientInFlight, d.last);
      if (new_respCnt == rc) begin
	 respCnt <= 0;
	 serverProcessing.deq;
	 l = last_burst;
      end
      else begin
	 respCnt <= new_respCnt;
      end
      clientDataFifo.enq(MemDataF { data: d.data, tag: d.tag, first: (respCnt == 0), last: l});
   endrule

   MemReadEngineServer#(userWidth) rs = (interface MemReadEngineServer#(userWidth);
		  interface Put request;
		     method Action put(MemengineCmd cmd);
`ifdef SIMULATION
			Bit#(32) bsb = fromInteger(bufferSizeBytes);
			Bit#(32) dw = fromInteger(valueOf(busWidthBytes));
			let mdw = ((cmd.len)/dw)*dw != cmd.len;
			let bbl = extend(cmd.burstLen) > bsb;
			if(bbl || mdw) begin
			   if (bbl)
			      $display("XXXXXXXXXX mkMemReadEngineBuff::unsupported burstLen %d %d", bsb, cmd.burstLen);
			   if (mdw)
			      $display("XXXXXXXXXX mkMemReadEngineBuff::unsupported len %d", cmd.len);
			end
			else
`endif
                           begin
			   clientRequest.enq(cmd);
			   end
 		     endmethod
		  endinterface
                  interface data = interface PipeOut;
	             method MemDataF#(userWidth) first;
	                return clientDataFifo.first;
	             endmethod
	             method Action deq;
	                if (verbose) $display("mkMemReadEngineBuff::check_out: data %h clientAvail %d eob %d", clientDataFifo.first.data, clientAvail.read(), clientDataFifo.first.last);
	                clientDataFifo.deq;
	                clientAvail.increment(1);
	             endmethod
	             method Bool notEmpty = clientDataFifo.notEmpty;
                  endinterface;
	       interface requestCycles = toPipeOut(clientCyclesFifo);
               endinterface);
   interface readServer = rs;
   interface PipeOut readReq = toPipeOut(dmaRequest);
endmodule

module mkMemReadEngineBuff#(Integer bufferSizeBytes) (MemReadEngine#(busWidth, userWidth, cmdQDepth, numServers))
   provisos (Div#(busWidth,8,busWidthBytes),
	     Mul#(busWidthBytes,8,busWidth),
	     Add#(busWidth, 0, userWidth),
//	     Min#(MemReadFunnelBPC, TLog#(numServers), bpc),
	     Add#(0,MemReadFunnelBPC,bpc),
	     FunnelPipesPipelined#(1, numServers, MemTypes::MemData#(userWidth), bpc),
	     FunnelPipesPipelined#(1, numServers, MemTypes::MemRequest, bpc),
	     Add#(a__, TLog#(numServers), MemTagSize)
      );
   let verbose = False;

   let clock <- exposeCurrentClock();
   let reset <- exposeCurrentReset();

   Integer bufferSizeBeats = bufferSizeBytes/valueOf(busWidthBytes);

   FIFOF#(MemData#(busWidth)) readDataFifo <- mkFIFOF();
   function Tuple2#(Bit#(TLog#(numServers)),MemData#(userWidth)) tagData(MemData#(userWidth) md);
      return tuple2(truncate(md.tag), md);
   endfunction
   UnFunnelPipe#(1,numServers,MemData#(userWidth),bpc) dataPipes <- mkUnFunnelPipesPipelined(vec(mapPipe(tagData, toPipeOut(readDataFifo))));

   Vector#(numServers, MemReadChannel#(busWidth,userWidth,cmdQDepth)) readChannels <- zipWithM(mkMemReadChannel(bufferSizeBytes),
											       genVector(), dataPipes);
   Vector#(numServers, FIFOF#(Bit#(MemTagSize))) readtagFifos <- replicateM(mkSizedFIFOF(valueOf(cmdQDepth)));
   function PipeOut#(MemRequest) readChannelDmaReadReq(Integer i);
      return readChannels[i].readReq;
   endfunction
   function MemReadEngineServer#(userWidth) readChannelServer(Integer i);
      return readChannels[i].readServer;
   endfunction

   FIFOF#(MemRequest) reqFifo <- mkFIFOF();
   FunnelPipe#(1,numServers,MemRequest,bpc) reqFunnel <- mkFunnelPipesPipelined(genWith(readChannelDmaReadReq));

   interface Vector  readServers = genWith(readChannelServer);
   interface MemReadClient dmaClient;
      interface Get    readReq = toGet(reqFunnel[0]);
      interface Put   readData = toPut(readDataFifo);
   endinterface

endmodule

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
import Vector::*;
import BuildVector::*;
import Cntrs::*;
import FIFOF::*;
import FIFO::*;
import GetPut::*;
import Assert::*;
import ClientServer::*;
import BRAM::*;
import BRAMFIFO::*;
import ConfigCounter::*;
import Connectable::*;
import ConnectalMemory::*;
import ConnectalBramFifo::*;
import MemTypes::*;
import Pipe::*;
import MemUtils::*;
import ConnectalConfig::*;

`include "ConnectalProjectConfig.bsv"

typedef 2 MemReadFunnelBPC;

module mkMemReadEngine(MemReadEngine#(busWidth, userWidth, cmdQDepth, numServers))
   provisos( Mul#(TDiv#(busWidth, 8), 8, busWidth)
	    ,Add#(1, a__, numServers)
	    ,Add#(busWidth, 0, userWidth)
//	     ,Min#(MemReadFunnelBPC, TLog#(numServers), bpc)
	     ,Add#(0,MemReadFunnelBPC,bpc)
	    ,FunnelPipesPipelined#(1, numServers, MemTypes::MemData#(userWidth), bpc)
	    ,Pipe::FunnelPipesPipelined#(1, numServers, MemTypes::MemRequest, bpc)
	    ,Add#(b__, TLog#(numServers), MemTagSize)
      	    );
   let rv <- mkMemReadEngineBuff(valueOf(cmdQDepth) * valueOf(TExp#(BurstLenSize)));
   return rv;
endmodule

typedef struct {
    Bit#(BurstLenSize) len;
    Bool               last;
} NextReq deriving (Bits, Eq);

function NextReq getNext(Bit#(32) len, Bit#(BurstLenSize) burst);
   NextReq v;
   v.last = (len <= extend(burst));
   v.len = v.last ? truncate(len) : burst;
   return v;
endfunction

interface MemReadChannel#(numeric type busWidth, numeric type userWidth, numeric type cmdQDepth);
   interface PipeOut#(MemRequest)        readReq;
   interface PipeIn#(MemData#(busWidth)) readData;
   interface MemReadEngineServer#(userWidth) readServer;
endinterface

module mkMemReadChannel#(Integer bufferSizeBytes, Integer channelNumber, PipeOut#(MemData#(userWidth)) dataPipe)
   (MemReadChannel#(busWidth, userWidth, cmdQDepth))
   provisos (Div#(busWidth,8,busWidthBytes),
	     Mul#(busWidthBytes,8,busWidth),
	     Log#(busWidthBytes,beatShift),
	     Log#(cmdQDepth,logCmdQDepth),
	     Add#(busWidth, 0, userWidth),
	     Add#(1,logCmdQDepth, outCntSz));
   let verbose = False;
   let beatShift = fromInteger(valueOf(beatShift));

   let clock <- exposeCurrentClock();
   let reset <- exposeCurrentReset();

   Integer bufferSizeBeats = bufferSizeBytes/valueOf(busWidthBytes);
   Reg#(Bool)          clientInFlight <- mkReg(False);
   ConfigCounter#(16)  clientAvail <- mkConfigCounter(fromInteger(bufferSizeBeats));
   Reg#(MemengineCmd)  clientCommand <- mkReg(unpack(0));
`ifdef USE_DUAL_CLOCK_FIFOF
   FIFOF#(MemDataF#(userWidth)) clientDataFifo <- mkDualClockBramFIFOF(clock, reset, clock, reset);
`else
   FIFOF#(MemDataF#(userWidth)) clientDataFifo <- mkSizedBRAMFIFOF(bufferSizeBeats);
`endif
   FIFO#(MemengineCmd) clientRequest <- mkFIFO();
   Reg#(Bit#(32))      clientLen <- mkReg(unpack(0));
   Reg#(Bit#(32))      clientBase <- mkReg(unpack(0));
   Reg#(NextReq)       clientNext <- mkReg(unpack(0));
   Count#(Bit#(32))    clientCycles <- mkCount(0);
   FIFOF#(MemRequestCycles) clientCyclesFifo <- mkFIFOF();

   FIFO#(Tuple3#(Bool,Bool,Bit#(BurstLenSize))) serverCheckAvail <- mkSizedFIFO(1);
   FIFOF#(MemRequest)                          dmaRequest <- mkSizedFIFOF(valueOf(cmdQDepth));
   FIFO#(Tuple3#(Bit#(8),Bit#(MemTagSize),Bool)) serverProcessing <- mkSizedFIFO(valueOf(cmdQDepth));
   FIFOF#(MemData#(busWidth))                       serverDataFifo <- mkFIFOF;

   Reg#(Bit#(8))                    respCnt <- mkReg(0);
   Reg#(Bit#(32)) counter <- mkReg(0);

   rule incCounter;
      counter <= counter + 1;
   endrule

   rule rule_cycles;
      clientCycles.incr(1);
   endrule
   rule rule_startNew if (!clientInFlight);
      let cmd <- toGet(clientRequest).get();
      clientInFlight <= True;
      clientCommand <= cmd;
      clientLen <= cmd.len - extend(cmd.burstLen);
      clientBase <= cmd.base;
      clientNext <= getNext(cmd.len, cmd.burstLen);
      clientCycles <= 0;
      if (verbose) $display("mkMemReadEngineBuff::%d rule_startNew %d", counter, clientAvail.read);
   endrule
   rule rule_checkAvail if (clientInFlight);
      let cmd_len = clientNext.len;
      let last_burst = clientNext.last;
      let cond0 <- clientAvail.maybeDecrement(unpack(extend(cmd_len>>beatShift)));
      serverCheckAvail.enq(tuple3(cond0,last_burst,cmd_len));
      if (verbose) $display("mkMemReadEngineBuff::%d rule_checkAvail avail %d burstLen %d cond0 %d last_burst %d", counter, clientAvail.read(), cmd_len>>beatShift, cond0, last_burst);
   endrule

   rule rule_requestServer if (clientInFlight);
      match {.cond0,.last_burst,.cmd_len} <- toGet(serverCheckAvail).get;
      if  (cond0) begin
	 if (verbose) $display("mkMemReadEngineBuff::%d rule_requestServer clientLen %d cond0 %d last_burst %d", counter, clientLen, cond0, last_burst);
	 serverProcessing.enq(tuple3(truncate(cmd_len>>beatShift), clientCommand.tag, last_burst));
	 if (verbose) $display("MemReadEngine::%d readReq idx %d offset %h burstLenBytes %h last %d", counter, 0, clientBase, cmd_len, last_burst);

	 dmaRequest.enq(MemRequest { sglId: clientCommand.sglId, offset: extend(clientBase),
	    burstLen:cmd_len, tag: fromInteger(channelNumber)
	    `ifdef BYTE_ENABLES
				    , firstbe: maxBound, lastbe: maxBound
	    `endif
	    });
         clientBase <= clientBase + extend(cmd_len);
         clientLen <= clientLen - extend(cmd_len);
         clientNext <= getNext(clientLen, clientCommand.burstLen);
	 if (last_burst) begin
	    if (verbose) $display("mkMemReadEngineBuff::%d rule_requestServer last_burst %d", counter, last_burst);
	    clientInFlight <= False;
	    `ifdef MEMENGINE_REQUEST_CYCLES
	    $display("clientCycles = %d", clientCycles);
	    clientCyclesFifo.enq(MemRequestCycles { tag:clientCommand.tag, cycles: clientCycles });
	    `endif
	 end
      end
   endrule

   rule read_data_rule;
      let d <- toGet(dataPipe).get();
      match {.rc, .tag, .last_burst} = serverProcessing.first;
      let new_respCnt = respCnt+1;
      let l = False;
      if (verbose) $display("mkMemReadEngineBuff::%d data %h new_respCnt %d rc %d last_burst %d tag %d clientInFlight %d eob %d", counter, d.data, new_respCnt, rc, last_burst, tag, clientInFlight, d.last);
      if (new_respCnt == rc) begin
	 respCnt <= 0;
	 serverProcessing.deq;
	 l = last_burst;
      end
      else begin
	 respCnt <= new_respCnt;
      end
      clientDataFifo.enq(MemDataF { data: d.data, tag: d.tag, first: (respCnt == 0), last: l});
   endrule

   MemReadEngineServer#(userWidth) rs = (interface MemReadEngineServer#(userWidth);
		  interface Put request;
		     method Action put(MemengineCmd cmd);
`ifdef SIMULATION
			Bit#(32) bsb = fromInteger(bufferSizeBytes);
			Bit#(32) dw = fromInteger(valueOf(busWidthBytes));
			let mdw = ((cmd.len)/dw)*dw != cmd.len;
			let bbl = extend(cmd.burstLen) > bsb;
			if(bbl || mdw) begin
			   if (bbl)
			      $display("XXXXXXXXXX mkMemReadEngineBuff::unsupported burstLen %d %d", bsb, cmd.burstLen);
			   if (mdw)
			      $display("XXXXXXXXXX mkMemReadEngineBuff::unsupported len %d", cmd.len);
			end
			else
`endif
                           begin
			   clientRequest.enq(cmd);
			   end
 		     endmethod
		  endinterface
                  interface data = interface PipeOut;
	             method MemDataF#(userWidth) first;
	                return clientDataFifo.first;
	             endmethod
	             method Action deq;
	                if (verbose) $display("mkMemReadEngineBuff::check_out: data %h clientAvail %d eob %d", clientDataFifo.first.data, clientAvail.read(), clientDataFifo.first.last);
	                clientDataFifo.deq;
	                clientAvail.increment(1);
	             endmethod
	             method Bool notEmpty = clientDataFifo.notEmpty;
                  endinterface;
	       interface requestCycles = toPipeOut(clientCyclesFifo);
               endinterface);
   interface readServer = rs;
   interface PipeOut readReq = toPipeOut(dmaRequest);
endmodule

module mkMemReadEngineBuff#(Integer bufferSizeBytes) (MemReadEngine#(busWidth, userWidth, cmdQDepth, numServers))
   provisos (Div#(busWidth,8,busWidthBytes),
	     Mul#(busWidthBytes,8,busWidth),
	     Add#(busWidth, 0, userWidth),
//	     Min#(MemReadFunnelBPC, TLog#(numServers), bpc),
	     Add#(0,MemReadFunnelBPC,bpc),
	     FunnelPipesPipelined#(1, numServers, MemTypes::MemData#(userWidth), bpc),
	     FunnelPipesPipelined#(1, numServers, MemTypes::MemRequest, bpc),
	     Add#(a__, TLog#(numServers), MemTagSize)
      );
   let verbose = False;

   let clock <- exposeCurrentClock();
   let reset <- exposeCurrentReset();

   Integer bufferSizeBeats = bufferSizeBytes/valueOf(busWidthBytes);

   FIFOF#(MemData#(busWidth)) readDataFifo <- mkFIFOF();
   function Tuple2#(Bit#(TLog#(numServers)),MemData#(userWidth)) tagData(MemData#(userWidth) md);
      return tuple2(truncate(md.tag), md);
   endfunction
   UnFunnelPipe#(1,numServers,MemData#(userWidth),bpc) dataPipes <- mkUnFunnelPipesPipelined(vec(mapPipe(tagData, toPipeOut(readDataFifo))));

   Vector#(numServers, MemReadChannel#(busWidth,userWidth,cmdQDepth)) readChannels <- mapM(uncurry(mkMemReadChannel(bufferSizeBytes)),
											   zip(genVector(), dataPipes));
   Vector#(numServers, FIFOF#(Bit#(MemTagSize))) readtagFifos <- replicateM(mkSizedFIFOF(valueOf(cmdQDepth)));
   function PipeOut#(MemRequest) readChannelDmaReadReq(Integer i);
      return readChannels[i].readReq;
   endfunction
   function MemReadEngineServer#(userWidth) readChannelServer(Integer i);
      return readChannels[i].readServer;
   endfunction

   FIFOF#(MemRequest) reqFifo <- mkFIFOF();
   FunnelPipe#(1,numServers,MemRequest,bpc) reqFunnel <- mkFunnelPipesPipelined(genWith(readChannelDmaReadReq));

   interface Vector  readServers = genWith(readChannelServer);
   interface MemReadClient dmaClient;
      interface Get    readReq = toGet(reqFunnel[0]);
      interface Put   readData = toPut(readDataFifo);
   endinterface

endmodule


derive Eq and FShow for DmaErrorType

derive Eq and FShow for DmaErrorType

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import GetPut::*;
import Vector::*;

//
// Dma channel type
//
typedef enum {
   ChannelType_Read, ChannelType_Write
   } ChannelType deriving (Bits,Eq,FShow);

//
// @brief Channel Identifier
//
//typedef Bit#(16) DmaChannelId;

typedef struct {
   Bit#(32) x;
   Bit#(32) y;
   Bit#(32) z;
   Bit#(32) w;
   } DmaDbgRec deriving(Bits);

typedef enum {
   DmaErrorNone,
   DmaErrorSGLIdOutOfRange_r,
   DmaErrorSGLIdOutOfRange_w,
   DmaErrorMMUOutOfRange_r,
   DmaErrorMMUOutOfRange_w,
   DmaErrorOffsetOutOfRange,
   DmaErrorSGLIdInvalid,
   DmaErrorTileTagOutOfRange
   } DmaErrorType deriving (Bits,Eq,FShow);

//
// @brief Events sent from a Dma engine
//
interface MemServerIndication;
   method Action addrResponse(Bit#(64) physAddr);
   method Action reportStateDbg(DmaDbgRec rec);
   method Action reportMemoryTraffic(Bit#(64) words);
   method Action error(Bit#(32) code, Bit#(32) sglId, Bit#(64) offset, Bit#(64) extra);
endinterface

//
// @brief Events sent from a MMU
//
interface MMUIndication;
   method Action idResponse(Bit#(32) sglId);
   method Action configResp(Bit#(32) sglId);
   method Action error(Bit#(32) code, Bit#(32) sglId, Bit#(64) offset, Bit#(64) extra);
endinterface

typedef Bit#(32) SpecialTypeForSendingFd;
//
// @brief Configuration interface to an MMU
//
interface MMURequest;
   //
   // @brief Adds an address translation entry to the scatter-gather list for an object
   //
   // @param sglId Specifies the object to be translated
   // @param addr Physical address of the segment
   // @param len Length of the segment
   //
   method Action sglist(Bit#(32) sglId, Bit#(32) sglIndex, Bit#(64) addr,  Bit#(32) len);
   method Action region(Bit#(32) sglId, Bit#(64) barr12, Bit#(32) index12, Bit#(64) barr8, Bit#(32) index8, Bit#(64) barr4, Bit#(32) index4, Bit#(64) barr0, Bit#(32) index0);
   method Action idRequest(SpecialTypeForSendingFd fd);
   method Action idReturn(Bit#(32) sglId);
   method Action setInterface(Bit#(32) interfaceId, Bit#(32) sglId);
endinterface

typedef enum {
   Idle, Stopped, Running
   } TileState deriving (Bits,Eq);

typedef struct {
   Bit#(2) tile;
   TileState state;
   } TileControl deriving (Bits);

//
// @brief Control interface to Dma engine
//
interface MemServerRequest;
   // @brief Requests an address translation
   //
   method Action addrTrans(Bit#(32) sglId, Bit#(32) offset);

   // @brief Changes tile status
   //
   method Action setTileState(TileControl tc);
   //
   // @brief Requests debug info for the specified channel type
   //
   method Action stateDbg(ChannelType rc);
   method Action memoryTraffic(ChannelType rc);
endinterface

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import GetPut::*;
import Vector::*;

//
// Dma channel type
//
typedef enum {
   ChannelType_Read, ChannelType_Write
   } ChannelType deriving (Bits,Eq,FShow);

//
// @brief Channel Identifier
//
//typedef Bit#(16) DmaChannelId;

typedef struct {
   Bit#(32) x;
   Bit#(32) y;
   Bit#(32) z;
   Bit#(32) w;
   } DmaDbgRec deriving(Bits);

typedef enum {
   DmaErrorNone,
   DmaErrorSGLIdOutOfRange_r,
   DmaErrorSGLIdOutOfRange_w,
   DmaErrorMMUOutOfRange_r,
   DmaErrorMMUOutOfRange_w,
   DmaErrorOffsetOutOfRange,
   DmaErrorSGLIdInvalid,
   DmaErrorTileTagOutOfRange
   } DmaErrorType deriving (Bits);

//
// @brief Events sent from a Dma engine
//
interface MemServerIndication;
   method Action addrResponse(Bit#(64) physAddr);
   method Action reportStateDbg(DmaDbgRec rec);
   method Action reportMemoryTraffic(Bit#(64) words);
   method Action error(Bit#(32) code, Bit#(32) sglId, Bit#(64) offset, Bit#(64) extra);
endinterface

//
// @brief Events sent from a MMU
//
interface MMUIndication;
   method Action idResponse(Bit#(32) sglId);
   method Action configResp(Bit#(32) sglId);
   method Action error(Bit#(32) code, Bit#(32) sglId, Bit#(64) offset, Bit#(64) extra);
endinterface

typedef Bit#(32) SpecialTypeForSendingFd;
//
// @brief Configuration interface to an MMU
//
interface MMURequest;
   //
   // @brief Adds an address translation entry to the scatter-gather list for an object
   //
   // @param sglId Specifies the object to be translated
   // @param addr Physical address of the segment
   // @param len Length of the segment
   //
   method Action sglist(Bit#(32) sglId, Bit#(32) sglIndex, Bit#(64) addr,  Bit#(32) len);
   method Action region(Bit#(32) sglId, Bit#(64) barr12, Bit#(32) index12, Bit#(64) barr8, Bit#(32) index8, Bit#(64) barr4, Bit#(32) index4, Bit#(64) barr0, Bit#(32) index0);
   method Action idRequest(SpecialTypeForSendingFd fd);
   method Action idReturn(Bit#(32) sglId);
   method Action setInterface(Bit#(32) interfaceId, Bit#(32) sglId);
endinterface

typedef enum {
   Idle, Stopped, Running
   } TileState deriving (Bits,Eq);

typedef struct {
   Bit#(2) tile;
   TileState state;
   } TileControl deriving (Bits);

//
// @brief Control interface to Dma engine
//
interface MemServerRequest;
   // @brief Requests an address translation
   //
   method Action addrTrans(Bit#(32) sglId, Bit#(32) offset);

   // @brief Changes tile status
   //
   method Action setTileState(TileControl tc);
   //
   // @brief Requests debug info for the specified channel type
   //
   method Action stateDbg(ChannelType rc);
   method Action memoryTraffic(ChannelType rc);
endinterface


even -> rg_even

even -> rg_even

// Copyright (c) 2014-2015 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

package Pcie3EndpointX7;

import Clocks            ::*;
import Vector            ::*;
import Connectable       ::*;
import GetPut            ::*;
import Reserved          ::*;
import TieOff            ::*;
import DefaultValue      ::*;
import DReg              ::*;
import Gearbox           ::*;
import FIFO              ::*;
import FIFOF             ::*;
import SpecialFIFOs      ::*;
import ClientServer      ::*;
import Real              ::*;
import XilinxVirtex7PCIE ::*;
import BUtils            ::*;

import ConnectalClocks   ::*;
import ConnectalXilinxCells   ::*;
import XilinxCells       ::*;
import PCIE              ::*;
import PCIEWRAPPER3      ::*;
import Bufgctrl           ::*;
import PcieGearbox       :: *;

interface PcieEndpointX7#(numeric type lanes);
   interface PciewrapPci_exp#(lanes)           pcie;
   interface PciewrapUser#(lanes)              user;
   interface PciewrapPipe#(lanes)              pipe;
   interface PciewrapCommon#(lanes)            common;
   interface Server#(TLPData#(16), TLPData#(16)) tlpr;
   interface Server#(TLPData#(16), TLPData#(16)) tlpc;
   interface Clock epPcieClock;
   interface Reset epPcieReset;
   interface Clock epPortalClock;
   interface Reset epPortalReset;
   interface Clock epDerivedClock;
   interface Reset epDerivedReset;
endinterface

typedef struct {
   Bit #(256)     data;
   Bool          sop;
   Bool          eop;
   Bit #(8)      keep;
   TLPFirstDWBE  first_be;
   TLPFirstDWBE  last_be;
} AxiStCq deriving (Bits, Eq);

typedef struct {
   Bit #(256)     data;
   Bit #(8)      keep;
   Bool          last;
} AxiStCc deriving (Bits, Eq);

typedef struct {
   Bit #(256)     data;
   Bool          last;
   Bit #(8)      keep;
   Bit #(4)      first_be;
   Bit #(4)      last_be;
} AxiStRq deriving (Bits, Eq);

typedef struct {
   Bit #(256)     data;
   Bool          sop;
   Bool          eop;
   Bit #(8)      keep;
   Bit #(8)      be;
} AxiStRc deriving (Bits, Eq);

function TLPData#(16) convertCQDescriptorToTLP16(CQDescriptor desc, Bit#(32) data, TLPFirstDWBE first, TLPLastDWBE last);
   TLPMemoryIO3DWHeader header = defaultValue;
   header.format     = tpl_1(convertCQReqTypeToTLPFmtType(desc.reqtype));
   header.pkttype    = tpl_2(convertCQReqTypeToTLPFmtType(desc.reqtype));
   header.tclass     = desc.tclass;
   header.relaxed    = desc.relaxed;
   header.nosnoop    = desc.nosnoop;
   header.length     = (desc.dwcount == 1024) ? 0 : truncate(desc.dwcount);
   header.reqid      = desc.reqid;
   header.tag        = desc.tag;
   header.lastbe     = last;
   header.firstbe    = first;
   header.addr       = truncate(desc.address);
   header.data       = convertDW(data);

   Bool is3DW = isReadReqType(desc.reqtype);
   Bool is3Or4DW = isReadReqType(desc.reqtype) || (desc.dwcount == 1);

   TLPData#(16) retval = defaultValue;
   retval.sof   = True;
   retval.eof   = is3Or4DW;
   retval.hit   = (1 << pack(desc.barid));
   retval.data  = pack(header);
   retval.be    = (is3DW ? 16'hFFF0 : 16'hFFFF);

   return retval;
endfunction

function TLPData#(16) convertRCDescriptorToTLP16(RCDescriptor desc, Bit#(32) data);
   TLPCompletionHeader header = defaultValue;
   header.tclass    = desc.tclass;
   header.relaxed   = desc.relaxed;
   header.nosnoop   = desc.nosnoop;
   header.cmplid    = desc.complid;
   header.tag       = desc.tag;
   header.reqid     = desc.reqid;
   header.poison    = desc.poisoned;
   header.cstatus   = desc.status;
   header.length    = (desc.dwcount == 1024) ? 0 : truncate(desc.dwcount);
   header.bytecount = (desc.bytecount == 4096) ? 0 : truncate(desc.bytecount);
   header.loweraddr = truncate(desc.loweraddr);
   header.data      = convertDW(data);

   Bool is3DW = (desc.dwcount == 0);
   Bool is3Or4DW = (desc.dwcount == 0) || (desc.dwcount == 1);
   TLPData#(16) retval = defaultValue;
   retval.sof   = True;
   retval.eof   = is3Or4DW;
   retval.hit   = 1; // XXX
   retval.data  = pack(header);
   retval.be    = (is3DW ? 16'hFFF0 : 16'hFFFF);

   return retval;
endfunction

`ifdef BOARD_vc709
typedef 8 PcieLanes;
typedef 8 NumLeds;
`endif
`ifdef BOARD_nfsume
typedef 8 PcieLanes;
typedef 2 NumLeds;
`endif

(* synthesize *)
module mkPcieEndpointX7(PcieEndpointX7#(PcieLanes));

   PCIEParams params = defaultValue;
   Clock defaultClock <- exposeCurrentClock();
   Reset defaultReset <- exposeCurrentReset();
   Reset defaultResetInverted <- mkResetInverter(defaultReset, clocked_by defaultClock);
   PcieWrap#(PcieLanes) pcie_ep <- mkPcieWrap(defaultClock, defaultResetInverted);

   // The PCIe endpoint exports full (250MHz) and half-speed (125MHz) clocks
   Clock pcieClock250 = pcie_ep.user_clk;
   Reset user_reset_n <- mkResetInverter(pcie_ep.user_reset, clocked_by pcie_ep.user_clk);
   Reset pcieReset250 <- mkAsyncReset(4, user_reset_n, pcieClock250);

   ClockGenerator7Params     clkgenParams = defaultValue;
   clkgenParams.clkin1_period    = 4.000; //  250MHz
   clkgenParams.clkin1_period    = 4.000;
   clkgenParams.clkin_buffer     = False;
   clkgenParams.clkfbout_mult_f  = 4.000; // 1000MHz
   clkgenParams.clkout0_divide_f = derivedClockPeriod;
   clkgenParams.clkout1_divide     = round(mainClockPeriod);
   clkgenParams.clkout1_duty_cycle = 0.5;
   clkgenParams.clkout1_phase      = 0.0000;
   ClockGenerator7           clkgen <- mkClockGenerator7(clkgenParams, clocked_by pcieClock250, reset_by pcieReset250);
   Clock mainClock = clkgen.clkout1;
   Reset mainReset <- mkAsyncReset(4, pcieReset250, mainClock);
   Clock derivedClock = clkgen.clkout0;
   Reset derivedReset <- mkAsyncReset(4, pcieReset250, derivedClock);
   Reset user_reset <- mkAsyncReset(2, pcie_ep.user_reset, pcie_ep.user_clk);

   FIFOF#(AxiStCq) fAxiCq <- mkFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   FIFOF#(AxiStRc) fAxiRc <- mkFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);

   FIFOF#(AxiStRq) fAxiRq <- mkFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   FIFOF#(AxiStCc) fAxiCc <- mkFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);

   FIFOF#(TLPData#(16)) fcq <- mkFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   FIFOF#(TLPData#(16)) frc <- mkFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   FIFOF#(TLPData#(16)) fcc <- mkFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   FIFOF#(TLPData#(16)) frq <- mkFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);

   // Drive s_axis_rq
   let rq_txready = (pcie_ep.s_axis_rq.tready != 0 && fAxiRq.notEmpty);

   //(* fire_when_enabled, no_implicit_conditions *)
   rule drive_axi_rq if (rq_txready);
      let info = fAxiRq.first; fAxiRq.deq;
      pcie_ep.s_axis_rq.tvalid(1);
      pcie_ep.s_axis_rq.tlast(pack(info.last));
      pcie_ep.s_axis_rq.tdata(info.data);
      pcie_ep.s_axis_rq.tkeep(info.keep);
      pcie_ep.s_axis_rq.tuser({0, info.last_be, info.first_be});
   endrule

   (* fire_when_enabled, no_implicit_conditions *)
   rule drive_axi_rq2 if (!rq_txready);
      pcie_ep.s_axis_rq.tvalid(0);
      pcie_ep.s_axis_rq.tlast(0);
      pcie_ep.s_axis_rq.tdata(0);
      pcie_ep.s_axis_rq.tkeep(0);
      pcie_ep.s_axis_rq.tuser(0);
   endrule

   // Drive s_axis_cc
   let cc_txready = (pcie_ep.s_axis_cc.tready != 0 && fAxiCc.notEmpty);

   //(* fire_when_enabled, no_implicit_conditions *)
   rule drive_axi_cc if (cc_txready);
      let info = fAxiCc.first; fAxiCc.deq;
      $display("drive axi_cc, data: %h, keep: %h, last: %h", info.data, info.keep, info.last);
      pcie_ep.s_axis_cc.tvalid(1);
      pcie_ep.s_axis_cc.tlast(pack(info.last));
      pcie_ep.s_axis_cc.tdata(info.data);
      pcie_ep.s_axis_cc.tkeep(info.keep);
      pcie_ep.s_axis_cc.tuser(0);
   endrule

   (* fire_when_enabled, no_implicit_conditions *)
   rule drive_axi_cc2 if (!cc_txready);
      pcie_ep.s_axis_cc.tvalid(0);
      pcie_ep.s_axis_cc.tlast(0);
      pcie_ep.s_axis_cc.tdata(0);
      pcie_ep.s_axis_cc.tkeep(0);
      pcie_ep.s_axis_cc.tuser(0);
   endrule

   (* fire_when_enabled, no_implicit_conditions *)
   rule drive_axi_rc_ready;
      pcie_ep.m_axis_rc.tready (duplicate (pack (fAxiRc.notFull)));
   endrule

   // Drive m_axis_rc
   (* fire_when_enabled *)
   rule sink_axi_rc if (pcie_ep.m_axis_rc.tvalid != 0 && fAxiRc.notFull);
      let rc = AxiStRc {data:pcie_ep.m_axis_rc.tdata,
			sop: unpack (pcie_ep.m_axis_rc.tuser [32]),         // tuser.is_sof_0
			eop: unpack (pcie_ep.m_axis_rc.tlast),
			keep:pcie_ep.m_axis_rc.tkeep,
			be:  truncate (pcie_ep.m_axis_rc.tuser [31:0])};    // tuser.byte_en
      fAxiRc.enq (rc);
   endrule

   (* fire_when_enabled, no_implicit_conditions *)
   rule drive_axi_cq_ready;
      pcie_ep.m_axis_cq.tready (duplicate (pack (fAxiCq.notFull)));
   endrule

   (* fire_when_enabled *)
   rule sink_axi_cq if (pcie_ep.m_axis_cq.tvalid != 0 && fAxiCq.notFull);
      let cq = AxiStCq {data:     pcie_ep.m_axis_cq.tdata,
			sop:      unpack (pcie_ep.m_axis_cq.tuser [40]),  // tuser.sop
			eop:      unpack (pcie_ep.m_axis_cq.tlast),
			keep:     pcie_ep.m_axis_cq.tkeep,
			first_be: pcie_ep.m_axis_cq.tuser [3:0],    // tuser.first_be,
			last_be:  pcie_ep.m_axis_cq.tuser [7:4]};   // tuser.last_be
      fAxiCq.enq (cq);
   endrule

   // CQ.
   CQDescriptor cq_desc = unpack(fAxiCq.first.data [127:0]);

   rule rl_cq_wr_header (fAxiCq.first.sop && ((cq_desc.reqtype == MEMORY_WRITE) || (cq_desc.reqtype == IO_WRITE)));
      Bit#(32) data = fAxiCq.first.data[159:128];
      // get data;
      TLPData#(16) tlp16 = convertCQDescriptorToTLP16(cq_desc, data, fAxiCq.first.first_be, fAxiCq.first.last_be);
      $display("cq_desc.reqtype=%h", cq_desc.reqtype);
      // enqueue?
      fcq.enq(tlp16);
      fAxiCq.deq;
   endrule

   // Write data payload, no data remaining
   rule rl_cq_wr_payload((!fAxiCq.first.sop));
      fAxiCq.deq;
   endrule

   // Write data payload, 1 to 3 DWs remaining
   // Write data payload, 4 or more DWs remaining

   rule rl_cq_rd_header (fAxiCq.first.sop && ((cq_desc.reqtype == MEMORY_READ) || (cq_desc.reqtype == IO_READ)));
      Bit#(32) data = 0;
      TLPData#(16) tlp16 = convertCQDescriptorToTLP16(cq_desc, data, fAxiCq.first.first_be, fAxiCq.first.last_be);
      $display("rl_cq_rd_header: cq_desc = %16x", cq_desc);
      fcq.enq(tlp16);
      fAxiCq.deq;
   endrule

   // RC.
   Reg#(DWCount) rg_dwcount <- mkRegU(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   Reg#(Bool) rg_even <- mkReg(True, clocked_by pcie_ep.user_clk, reset_by user_reset_n);

   rule rl_rc_header (fAxiRc.first.sop && rg_even);
      RCDescriptor rc_desc = unpack(fAxiRc.first.data [95:0]);
      // RC descriptor always 96 bytes with first data word in bits 127:96
      Bit#(32) data = fAxiRc.first.data[127:96] | 32'hbeef0000;
      TLPData#(16) tlp16 = convertRCDescriptorToTLP16(rc_desc, data);
      rg_dwcount <= (rc_desc.dwcount == 0) ? 0 : rc_desc.dwcount - 1;
      frc.enq(tlp16);
      let even = False;
      if (rc_desc.dwcount == 0 || rc_desc.dwcount == 1) begin
        fAxiRc.deq;
        even = True;
      end
      rg_even <= even;
   endrule

   rule rl_rc_data ((!rg_even || !(fAxiRc.first.sop)) && (rg_dwcount != 0));
      Bit#(16) be16;
      case (rg_dwcount)
         1: be16 = 16'hF000;
         2: be16 = 16'hFF00;
         3: be16 = 16'hFFF0;
         default: be16 = 16'hFFFF;
      endcase
      let last = (rg_dwcount <= 4);
      let data = (rg_even) ? fAxiRc.first.data[127:0]: fAxiRc.first.data[255:128];
      TLPData#(16) tlp16 = TLPData{sof: False,
                                   eof: last,
                                   hit: 0,
                                   be: be16,
                                   data: pack(data)};
      frc.enq(tlp16);
      if (last || !rg_even) begin
         fAxiRc.deq;
      end
      rg_even <= (last) ? True : !rg_even;
   endrule

   Reg#(DWCount) cc_dwcount <- mkReg(0, clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   FIFOF#(TLPData#(16)) fcc_tlps <- mkFIFOF (clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   // CC.
   rule get_cc_tlps;
      let tlp <- toGet(fcc).get;
      fcc_tlps.enq(tlp);
   endrule

   rule rl_cc_header(fcc_tlps.first.sof);
      match { .cc_desc, .dw} = convertTLP16ToCCDescriptor(fcc_tlps.first);
      cc_dwcount <= cc_desc.dwcount - 1;
      fAxiCc.enq(AxiStCc {data: zeroExtend({dw, pack(cc_desc)[95:0]}),
                       last: fcc_tlps.first.eof,
                       keep: 8'h0F});
      fcc_tlps.deq;
   endrule

   rule rl_cc_data((!fcc_tlps.first.sof) && (cc_dwcount != 0));
      Bit#(256) x = zeroExtend(fcc_tlps.first.data); //FIXME
      fAxiCc.enq(AxiStCc {data: {x},
                       last: cc_dwcount <= 4,
                       keep: (cc_dwcount == 3) ? 8'h0F : 8'hFF});
      fcc_tlps.deq;
   endrule

   // RQ.
   FIFOF#(TLPData#(16)) frq_tlps <- mkFIFOF (clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   Reg#(DWCount) rq_dwcount <- mkReg(0, clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   Reg#(Maybe#(Bit#(32))) rq_mdw <- mkRegU(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   Reg#(Bit#(4)) rq_first_be <- mkRegU(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   Reg#(Bit#(4)) rq_last_be <- mkRegU(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   rule get_rq_tlps;
      let tlp <- toGet(frq).get;
      frq_tlps.enq(tlp);
   endrule

   rule rl_rq_header(frq_tlps.first.sof);
      match { .rq_desc, .first_be, .last_be, .mdata} = convertTLP16ToRQDescriptor(frq_tlps.first);
      rq_dwcount <= ((rq_desc.reqtype == MEMORY_WRITE) ? rq_desc.dwcount : 0);
      rq_mdw <= mdata;
      rq_first_be <= first_be;
      rq_last_be <= last_be;
      fAxiRq.enq(AxiStRq {data: zeroExtend(pack(rq_desc)), //FIXME:
                       last: frq_tlps.first.eof,
                       keep: 8'h0F,
                       first_be: first_be,
                       last_be: last_be});
      frq_tlps.deq;
   endrule

   // rq_mdw contains last DW
   rule rl_rq_data_a (rq_mdw matches tagged Valid .dw &&& (rq_dwcount == 1));
      rq_dwcount <= 0;
      rq_mdw <= tagged Invalid;
   endrule

   // rq_mdw Valid, and there are more DWs.
   rule rl_rq_data_b(rq_mdw matches tagged Valid .dw &&& (rq_dwcount != 1));

   endrule

   // rq_mdw Invalid, and there are more DWs.
   rule rl_rq_data_c(rq_mdw matches tagged Invalid &&& (rq_dwcount != 1));

   endrule

   // The PCIE endpoint is processing Gen3 descriptors at 250MHz. The
   // AXI bridge is accepting TLPData#(16)s at 250 MHz. The
   // conversion uses half of Gen3 descriptor.
   //mkConnection(tlp8, gb.tlp, clocked_by portalClock, reset_by portalReset);

   let portalClock = (mainClockPeriod == pcieClockPeriod) ? pcieClock250 : mainClock;
   let portalReset = (mainClockPeriod == pcieClockPeriod) ? pcieReset250 : mainReset;

   interface Server tlpr;
      interface request = toPut(frq);
      interface response = toGet(frc);
   endinterface
   // Requests from other PCIe devices
   interface Server tlpc;
      interface request = toPut(fcc);
      interface response = toGet(fcq);
   endinterface
   interface pcie    = pcie_ep.pci_exp;
   interface Pcie3wrapUser user = pcie_ep.user;
   interface PciewrapPipe pipe = pcie_ep.pipe;
   interface PciewrapCommon common= pcie_ep.common;
   interface Clock epPcieClock = pcieClock250;
   interface Reset epPcieReset = pcieReset250;
   interface Clock epPortalClock = portalClock;
   interface Reset epPortalReset = portalReset;
   interface Clock epDerivedClock = derivedClock;
   interface Reset epDerivedReset = derivedReset;
endmodule: mkPcieEndpointX7

endpackage: Pcie3EndpointX7

// Copyright (c) 2014-2015 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

package Pcie3EndpointX7;

import Clocks            ::*;
import Vector            ::*;
import Connectable       ::*;
import GetPut            ::*;
import Reserved          ::*;
import TieOff            ::*;
import DefaultValue      ::*;
import DReg              ::*;
import Gearbox           ::*;
import FIFO              ::*;
import FIFOF             ::*;
import SpecialFIFOs      ::*;
import ClientServer      ::*;
import Real              ::*;
import XilinxVirtex7PCIE ::*;
import BUtils            ::*;

import ConnectalClocks   ::*;
import ConnectalXilinxCells   ::*;
import XilinxCells       ::*;
import PCIE              ::*;
import PCIEWRAPPER3      ::*;
import Bufgctrl           ::*;
import PcieGearbox       :: *;

interface PcieEndpointX7#(numeric type lanes);
   interface PciewrapPci_exp#(lanes)           pcie;
   interface PciewrapUser#(lanes)              user;
   interface PciewrapPipe#(lanes)              pipe;
   interface PciewrapCommon#(lanes)            common;
   interface Server#(TLPData#(16), TLPData#(16)) tlpr;
   interface Server#(TLPData#(16), TLPData#(16)) tlpc;
   interface Clock epPcieClock;
   interface Reset epPcieReset;
   interface Clock epPortalClock;
   interface Reset epPortalReset;
   interface Clock epDerivedClock;
   interface Reset epDerivedReset;
endinterface

typedef struct {
   Bit #(256)     data;
   Bool          sop;
   Bool          eop;
   Bit #(8)      keep;
   TLPFirstDWBE  first_be;
   TLPFirstDWBE  last_be;
} AxiStCq deriving (Bits, Eq);

typedef struct {
   Bit #(256)     data;
   Bit #(8)      keep;
   Bool          last;
} AxiStCc deriving (Bits, Eq);

typedef struct {
   Bit #(256)     data;
   Bool          last;
   Bit #(8)      keep;
   Bit #(4)      first_be;
   Bit #(4)      last_be;
} AxiStRq deriving (Bits, Eq);

typedef struct {
   Bit #(256)     data;
   Bool          sop;
   Bool          eop;
   Bit #(8)      keep;
   Bit #(8)      be;
} AxiStRc deriving (Bits, Eq);

function TLPData#(16) convertCQDescriptorToTLP16(CQDescriptor desc, Bit#(32) data, TLPFirstDWBE first, TLPLastDWBE last);
   TLPMemoryIO3DWHeader header = defaultValue;
   header.format     = tpl_1(convertCQReqTypeToTLPFmtType(desc.reqtype));
   header.pkttype    = tpl_2(convertCQReqTypeToTLPFmtType(desc.reqtype));
   header.tclass     = desc.tclass;
   header.relaxed    = desc.relaxed;
   header.nosnoop    = desc.nosnoop;
   header.length     = (desc.dwcount == 1024) ? 0 : truncate(desc.dwcount);
   header.reqid      = desc.reqid;
   header.tag        = desc.tag;
   header.lastbe     = last;
   header.firstbe    = first;
   header.addr       = truncate(desc.address);
   header.data       = convertDW(data);

   Bool is3DW = isReadReqType(desc.reqtype);
   Bool is3Or4DW = isReadReqType(desc.reqtype) || (desc.dwcount == 1);

   TLPData#(16) retval = defaultValue;
   retval.sof   = True;
   retval.eof   = is3Or4DW;
   retval.hit   = (1 << pack(desc.barid));
   retval.data  = pack(header);
   retval.be    = (is3DW ? 16'hFFF0 : 16'hFFFF);

   return retval;
endfunction

function TLPData#(16) convertRCDescriptorToTLP16(RCDescriptor desc, Bit#(32) data);
   TLPCompletionHeader header = defaultValue;
   header.tclass    = desc.tclass;
   header.relaxed   = desc.relaxed;
   header.nosnoop   = desc.nosnoop;
   header.cmplid    = desc.complid;
   header.tag       = desc.tag;
   header.reqid     = desc.reqid;
   header.poison    = desc.poisoned;
   header.cstatus   = desc.status;
   header.length    = (desc.dwcount == 1024) ? 0 : truncate(desc.dwcount);
   header.bytecount = (desc.bytecount == 4096) ? 0 : truncate(desc.bytecount);
   header.loweraddr = truncate(desc.loweraddr);
   header.data      = convertDW(data);

   Bool is3DW = (desc.dwcount == 0);
   Bool is3Or4DW = (desc.dwcount == 0) || (desc.dwcount == 1);
   TLPData#(16) retval = defaultValue;
   retval.sof   = True;
   retval.eof   = is3Or4DW;
   retval.hit   = 1; // XXX
   retval.data  = pack(header);
   retval.be    = (is3DW ? 16'hFFF0 : 16'hFFFF);

   return retval;
endfunction

`ifdef BOARD_vc709
typedef 8 PcieLanes;
typedef 8 NumLeds;
`endif
`ifdef BOARD_nfsume
typedef 8 PcieLanes;
typedef 2 NumLeds;
`endif

(* synthesize *)
module mkPcieEndpointX7(PcieEndpointX7#(PcieLanes));

   PCIEParams params = defaultValue;
   Clock defaultClock <- exposeCurrentClock();
   Reset defaultReset <- exposeCurrentReset();
   Reset defaultResetInverted <- mkResetInverter(defaultReset, clocked_by defaultClock);
   PcieWrap#(PcieLanes) pcie_ep <- mkPcieWrap(defaultClock, defaultResetInverted);

   // The PCIe endpoint exports full (250MHz) and half-speed (125MHz) clocks
   Clock pcieClock250 = pcie_ep.user_clk;
   Reset user_reset_n <- mkResetInverter(pcie_ep.user_reset, clocked_by pcie_ep.user_clk);
   Reset pcieReset250 <- mkAsyncReset(4, user_reset_n, pcieClock250);

   ClockGenerator7Params     clkgenParams = defaultValue;
   clkgenParams.clkin1_period    = 4.000; //  250MHz
   clkgenParams.clkin1_period    = 4.000;
   clkgenParams.clkin_buffer     = False;
   clkgenParams.clkfbout_mult_f  = 4.000; // 1000MHz
   clkgenParams.clkout0_divide_f = derivedClockPeriod;
   clkgenParams.clkout1_divide     = round(mainClockPeriod);
   clkgenParams.clkout1_duty_cycle = 0.5;
   clkgenParams.clkout1_phase      = 0.0000;
   ClockGenerator7           clkgen <- mkClockGenerator7(clkgenParams, clocked_by pcieClock250, reset_by pcieReset250);
   Clock mainClock = clkgen.clkout1;
   Reset mainReset <- mkAsyncReset(4, pcieReset250, mainClock);
   Clock derivedClock = clkgen.clkout0;
   Reset derivedReset <- mkAsyncReset(4, pcieReset250, derivedClock);
   Reset user_reset <- mkAsyncReset(2, pcie_ep.user_reset, pcie_ep.user_clk);

   FIFOF#(AxiStCq) fAxiCq <- mkFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   FIFOF#(AxiStRc) fAxiRc <- mkFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);

   FIFOF#(AxiStRq) fAxiRq <- mkFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   FIFOF#(AxiStCc) fAxiCc <- mkFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);

   FIFOF#(TLPData#(16)) fcq <- mkFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   FIFOF#(TLPData#(16)) frc <- mkFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   FIFOF#(TLPData#(16)) fcc <- mkFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   FIFOF#(TLPData#(16)) frq <- mkFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);

   // Drive s_axis_rq
   let rq_txready = (pcie_ep.s_axis_rq.tready != 0 && fAxiRq.notEmpty);

   //(* fire_when_enabled, no_implicit_conditions *)
   rule drive_axi_rq if (rq_txready);
      let info = fAxiRq.first; fAxiRq.deq;
      pcie_ep.s_axis_rq.tvalid(1);
      pcie_ep.s_axis_rq.tlast(pack(info.last));
      pcie_ep.s_axis_rq.tdata(info.data);
      pcie_ep.s_axis_rq.tkeep(info.keep);
      pcie_ep.s_axis_rq.tuser({0, info.last_be, info.first_be});
   endrule

   (* fire_when_enabled, no_implicit_conditions *)
   rule drive_axi_rq2 if (!rq_txready);
      pcie_ep.s_axis_rq.tvalid(0);
      pcie_ep.s_axis_rq.tlast(0);
      pcie_ep.s_axis_rq.tdata(0);
      pcie_ep.s_axis_rq.tkeep(0);
      pcie_ep.s_axis_rq.tuser(0);
   endrule

   // Drive s_axis_cc
   let cc_txready = (pcie_ep.s_axis_cc.tready != 0 && fAxiCc.notEmpty);

   //(* fire_when_enabled, no_implicit_conditions *)
   rule drive_axi_cc if (cc_txready);
      let info = fAxiCc.first; fAxiCc.deq;
      $display("drive axi_cc, data: %h, keep: %h, last: %h", info.data, info.keep, info.last);
      pcie_ep.s_axis_cc.tvalid(1);
      pcie_ep.s_axis_cc.tlast(pack(info.last));
      pcie_ep.s_axis_cc.tdata(info.data);
      pcie_ep.s_axis_cc.tkeep(info.keep);
      pcie_ep.s_axis_cc.tuser(0);
   endrule

   (* fire_when_enabled, no_implicit_conditions *)
   rule drive_axi_cc2 if (!cc_txready);
      pcie_ep.s_axis_cc.tvalid(0);
      pcie_ep.s_axis_cc.tlast(0);
      pcie_ep.s_axis_cc.tdata(0);
      pcie_ep.s_axis_cc.tkeep(0);
      pcie_ep.s_axis_cc.tuser(0);
   endrule

   (* fire_when_enabled, no_implicit_conditions *)
   rule drive_axi_rc_ready;
      pcie_ep.m_axis_rc.tready (duplicate (pack (fAxiRc.notFull)));
   endrule

   // Drive m_axis_rc
   (* fire_when_enabled *)
   rule sink_axi_rc if (pcie_ep.m_axis_rc.tvalid != 0 && fAxiRc.notFull);
      let rc = AxiStRc {data:pcie_ep.m_axis_rc.tdata,
			sop: unpack (pcie_ep.m_axis_rc.tuser [32]),         // tuser.is_sof_0
			eop: unpack (pcie_ep.m_axis_rc.tlast),
			keep:pcie_ep.m_axis_rc.tkeep,
			be:  truncate (pcie_ep.m_axis_rc.tuser [31:0])};    // tuser.byte_en
      fAxiRc.enq (rc);
   endrule

   (* fire_when_enabled, no_implicit_conditions *)
   rule drive_axi_cq_ready;
      pcie_ep.m_axis_cq.tready (duplicate (pack (fAxiCq.notFull)));
   endrule

   (* fire_when_enabled *)
   rule sink_axi_cq if (pcie_ep.m_axis_cq.tvalid != 0 && fAxiCq.notFull);
      let cq = AxiStCq {data:     pcie_ep.m_axis_cq.tdata,
			sop:      unpack (pcie_ep.m_axis_cq.tuser [40]),  // tuser.sop
			eop:      unpack (pcie_ep.m_axis_cq.tlast),
			keep:     pcie_ep.m_axis_cq.tkeep,
			first_be: pcie_ep.m_axis_cq.tuser [3:0],    // tuser.first_be,
			last_be:  pcie_ep.m_axis_cq.tuser [7:4]};   // tuser.last_be
      fAxiCq.enq (cq);
   endrule

   // CQ.
   CQDescriptor cq_desc = unpack(fAxiCq.first.data [127:0]);

   rule rl_cq_wr_header (fAxiCq.first.sop && ((cq_desc.reqtype == MEMORY_WRITE) || (cq_desc.reqtype == IO_WRITE)));
      Bit#(32) data = fAxiCq.first.data[159:128];
      // get data;
      TLPData#(16) tlp16 = convertCQDescriptorToTLP16(cq_desc, data, fAxiCq.first.first_be, fAxiCq.first.last_be);
      $display("cq_desc.reqtype=%h", cq_desc.reqtype);
      // enqueue?
      fcq.enq(tlp16);
      fAxiCq.deq;
   endrule

   // Write data payload, no data remaining
   rule rl_cq_wr_payload((!fAxiCq.first.sop));
      fAxiCq.deq;
   endrule

   // Write data payload, 1 to 3 DWs remaining
   // Write data payload, 4 or more DWs remaining

   rule rl_cq_rd_header (fAxiCq.first.sop && ((cq_desc.reqtype == MEMORY_READ) || (cq_desc.reqtype == IO_READ)));
      Bit#(32) data = 0;
      TLPData#(16) tlp16 = convertCQDescriptorToTLP16(cq_desc, data, fAxiCq.first.first_be, fAxiCq.first.last_be);
      $display("rl_cq_rd_header: cq_desc = %16x", cq_desc);
      fcq.enq(tlp16);
      fAxiCq.deq;
   endrule

   // RC.
   Reg#(DWCount) rg_dwcount <- mkRegU(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   Reg#(Bool) even <- mkReg(True, clocked_by pcie_ep.user_clk, reset_by user_reset_n);

   rule rl_rc_header (fAxiRc.first.sop && even);
      RCDescriptor rc_desc = unpack(fAxiRc.first.data [95:0]);
      // RC descriptor always 96 bytes with first data word in bits 127:96
      Bit#(32) data = fAxiRc.first.data[127:96] | 32'hbeef0000;
      TLPData#(16) tlp16 = convertRCDescriptorToTLP16(rc_desc, data);
      rg_dwcount <= (rc_desc.dwcount == 0) ? 0 : rc_desc.dwcount - 1;
      frc.enq(tlp16);
      let e = False;
      if (rc_desc.dwcount == 0 || rc_desc.dwcount == 1) begin
        fAxiRc.deq;
        e = True;
      end
      even <= e;
   endrule

   rule rl_rc_data ((!even || !(fAxiRc.first.sop)) && (rg_dwcount != 0));
      Bit#(16) be16;
      case (rg_dwcount)
         1: be16 = 16'hF000;
         2: be16 = 16'hFF00;
         3: be16 = 16'hFFF0;
         default: be16 = 16'hFFFF;
      endcase
      let last = (rg_dwcount <= 4);
      let data = (even) ? fAxiRc.first.data[127:0]: fAxiRc.first.data[255:128];
      TLPData#(16) tlp16 = TLPData{sof: False,
                                   eof: last,
                                   hit: 0,
                                   be: be16,
                                   data: pack(data)};
      frc.enq(tlp16);
      if (last || !even)
         fAxiRc.deq;
      even <= (last) ? True : !even;
   endrule

   Reg#(DWCount) cc_dwcount <- mkReg(0, clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   FIFOF#(TLPData#(16)) fcc_tlps <- mkFIFOF (clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   // CC.
   rule get_cc_tlps;
      let tlp <- toGet(fcc).get;
      fcc_tlps.enq(tlp);
   endrule

   rule rl_cc_header(fcc_tlps.first.sof);
      match { .cc_desc, .dw} = convertTLP16ToCCDescriptor(fcc_tlps.first);
      cc_dwcount <= cc_desc.dwcount - 1;
      fAxiCc.enq(AxiStCc {data: zeroExtend({dw, pack(cc_desc)[95:0]}),
                       last: fcc_tlps.first.eof,
                       keep: 8'h0F});
      fcc_tlps.deq;
   endrule

   rule rl_cc_data((!fcc_tlps.first.sof) && (cc_dwcount != 0));
      Bit#(256) x = zeroExtend(fcc_tlps.first.data); //FIXME
      fAxiCc.enq(AxiStCc {data: {x},
                       last: cc_dwcount <= 4,
                       keep: (cc_dwcount == 3) ? 8'h0F : 8'hFF});
      fcc_tlps.deq;
   endrule

   // RQ.
   FIFOF#(TLPData#(16)) frq_tlps <- mkFIFOF (clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   Reg#(DWCount) rq_dwcount <- mkReg(0, clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   Reg#(Maybe#(Bit#(32))) rq_mdw <- mkRegU(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   Reg#(Bit#(4)) rq_first_be <- mkRegU(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   Reg#(Bit#(4)) rq_last_be <- mkRegU(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   rule get_rq_tlps;
      let tlp <- toGet(frq).get;
      frq_tlps.enq(tlp);
   endrule

   rule rl_rq_header(frq_tlps.first.sof);
      match { .rq_desc, .first_be, .last_be, .mdata} = convertTLP16ToRQDescriptor(frq_tlps.first);
      rq_dwcount <= ((rq_desc.reqtype == MEMORY_WRITE) ? rq_desc.dwcount : 0);
      rq_mdw <= mdata;
      rq_first_be <= first_be;
      rq_last_be <= last_be;
      fAxiRq.enq(AxiStRq {data: zeroExtend(pack(rq_desc)), //FIXME:
                       last: frq_tlps.first.eof,
                       keep: 8'h0F,
                       first_be: first_be,
                       last_be: last_be});
      frq_tlps.deq;
   endrule

   // rq_mdw contains last DW
   rule rl_rq_data_a (rq_mdw matches tagged Valid .dw &&& (rq_dwcount == 1));
      rq_dwcount <= 0;
      rq_mdw <= tagged Invalid;
   endrule

   // rq_mdw Valid, and there are more DWs.
   rule rl_rq_data_b(rq_mdw matches tagged Valid .dw &&& (rq_dwcount != 1));

   endrule

   // rq_mdw Invalid, and there are more DWs.
   rule rl_rq_data_c(rq_mdw matches tagged Invalid &&& (rq_dwcount != 1));

   endrule

   // The PCIE endpoint is processing Gen3 descriptors at 250MHz. The
   // AXI bridge is accepting TLPData#(16)s at 250 MHz. The
   // conversion uses half of Gen3 descriptor.
   //mkConnection(tlp8, gb.tlp, clocked_by portalClock, reset_by portalReset);

   let portalClock = (mainClockPeriod == pcieClockPeriod) ? pcieClock250 : mainClock;
   let portalReset = (mainClockPeriod == pcieClockPeriod) ? pcieReset250 : mainReset;

   interface Server tlpr;
      interface request = toPut(frq);
      interface response = toGet(frc);
   endinterface
   // Requests from other PCIe devices
   interface Server tlpc;
      interface request = toPut(fcc);
      interface response = toGet(fcq);
   endinterface
   interface pcie    = pcie_ep.pci_exp;
   interface Pcie3wrapUser user = pcie_ep.user;
   interface PciewrapPipe pipe = pcie_ep.pipe;
   interface PciewrapCommon common= pcie_ep.common;
   interface Clock epPcieClock = pcieClock250;
   interface Reset epPcieReset = pcieReset250;
   interface Clock epPortalClock = portalClock;
   interface Reset epPortalReset = portalReset;
   interface Clock epDerivedClock = derivedClock;
   interface Reset epDerivedReset = derivedReset;
endmodule: mkPcieEndpointX7

endpackage: Pcie3EndpointX7