increase to 14 bits

increase to 14 bits

// Copyright (c) 2014 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import StmtFSM::*;

interface MCSAlgorithm;
   method Action setupA (Bit#(14) start, Bit#(14) length);
   method Action setupB (Bit#(14) start, Bit#(14) length);
   method Action setupL (Bit#(14) start);
   method Bit#(14) result();
   interface FSM fsm;
endinterface

// Copyright (c) 2014 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import StmtFSM::*;

interface MCSAlgorithm;
   method Action setupA (Bit#(7) start, Bit#(7) length);
   method Action setupB (Bit#(7) start, Bit#(7) length);
   method Action setupL (Bit#(14) start);
   method Bit#(14) result();
   interface FSM fsm;
endinterface


only call simDma.initfd if hostMapped

only call simDma.initfd if hostMapped

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

// BSV Libraries
import RegFile::*;
import FIFO::*;
import FIFOF::*;
import Vector::*;
import GetPut::*;
import Connectable::*;
import BRAMFIFO::*;
import BRAM::*;
import Probe::*;

import ConnectalBram::*;
import MemTypes::*;
import StmtFSM::*;
import ClientServer::*;
import ConnectalMemory::*;
import ConnectalCompletionBuffer::*;
import SimDma::*;

`include "ConnectalProjectConfig.bsv"

typedef 32 MaxNumSGLists;
typedef Bit#(TLog#(MaxNumSGLists)) SGListId;
typedef 12 SGListPageShift0;
typedef 16 SGListPageShift4;
typedef 20 SGListPageShift8;
typedef 24 SGListPageShift12;
typedef Bit#(TLog#(MaxNumSGLists)) RegionsIdx;

typedef 8 IndexWidth;

typedef struct {
   SGListId             id;
   Bit#(MemOffsetSize) off;
} AddrTransRequest deriving (Eq,Bits,FShow);

typedef struct {
   DmaErrorType    error;
   Bit#(addrWidth) physAddr;
} AddrTransResponse#(numeric type addrWidth) deriving (Eq,Bits,FShow);

interface MMU#(numeric type addrWidth);
   interface MMURequest request;
   interface Vector#(2,Server#(AddrTransRequest,AddrTransResponse#(addrWidth))) addr;
endinterface

module mkSimpleMMU#(Integer iid, Bool hostMapped, MMUIndication mmuIndication)(MMU#(addrWidth))
   provisos (Add#(b__,8,addrWidth)
	     ,Add#(c__,24,addrWidth)
	     ,Bits#(AddrTransResponse#(addrWidth),d__)
	     );
   Vector#(2,FIFOF#(AddrTransRequest)) reqFifos <- replicateM(mkFIFOF());
   Vector#(2,FIFOF#(AddrTransResponse#(addrWidth))) respFifos <- replicateM(mkFIFOF());

   let probePhysAddr <- mkProbe();

   for (Integer i = 0; i < 2; i = i + 1) begin
      rule rl_process_req;
	 let req <- toGet(reqFifos[i]).get();
	 Bit#(addrWidth) physAddr = extend(req.off[23:0]);
	 physAddr[31:24] = extend(req.id);
	 probePhysAddr <= physAddr;
	 respFifos[i].enq(AddrTransResponse { error: DmaErrorNone, physAddr: physAddr });
      endrule
   end

   function Server#(AddrTransRequest,AddrTransResponse#(addrWidth)) genServer(Integer i);
      return (interface Server#(AddrTransRequest,AddrTransResponse#(addrWidth));
		 interface request = toPut(reqFifos[i]);
	 interface response = toGet(respFifos[i]);
	 endinterface);
   endfunction

   interface MMURequest request;
      method Action idRequest(SpecialTypeForSendingFd fd);
	 mmuIndication.idResponse(0);
      endmethod
      method Action idReturn(Bit#(32) sglId);
	 mmuIndication.idResponse(sglId);
      endmethod
      method Action region(Bit#(32) pointer, Bit#(64) barr12, Bit#(32) index12, Bit#(64) barr8, Bit#(32) index8, Bit#(64) barr4, Bit#(32) index4, Bit#(64) barr0, Bit#(32) index0);
	 mmuIndication.configResp(extend(pointer));
      endmethod
      method Action sglist(Bit#(32) pointer, Bit#(32) pointerIndex, Bit#(64) addr,  Bit#(32) len);
	 // no response
      endmethod
      method Action setInterface(Bit#(32) interfaceId, Bit#(32) sglId);
	 /* this method is only implemented in s/w responders */
      endmethod
   endinterface
   interface Vector addr = genWith(genServer);
endmodule

typedef struct {
   DmaErrorType error;
   Bit#(3) pageSize;
   Bit#(SGListPageShift12) value;
} Offset deriving (Eq,Bits,FShow);

typedef Bit#(TSub#(MemOffsetSize,SGListPageShift0)) Page0;
typedef Bit#(TSub#(MemOffsetSize,SGListPageShift4)) Page4;
typedef Bit#(TSub#(MemOffsetSize,SGListPageShift8)) Page8;
typedef Bit#(TSub#(MemOffsetSize,SGListPageShift12)) Page12;

typedef struct {
   Bit#(TSub#(MemOffsetSize,SGListPageShift0)) barrier;
   Bit#(IndexWidth) idxOffset;
   } SingleRegion deriving (Eq,Bits,FShow);

typedef struct {
   SingleRegion reg12;
   SingleRegion reg8;
   SingleRegion reg4;
   SingleRegion reg0;
   } Region deriving (Eq,Bits,FShow);

typedef struct {DmaErrorType errorType;
		Bit#(32) pref;
		Bit#(MemOffsetSize) off;
   } DmaError deriving (Bits);

typedef struct {
   Bool cond12;
   Bool cond8;
   Bool cond4;
   Bool cond0;
   Bit#(IndexWidth) idxOffset12;
   Bit#(IndexWidth) idxOffset8;
   Bit#(IndexWidth) idxOffset4;
   Bit#(IndexWidth) idxOffset0;
   AddrTransRequest req;
   } Stage3Params deriving (Bits);

typedef struct {
   Offset off;
   Bit#(IndexWidth) pbase;
   Bit#(IndexWidth) idxOffset;
   SGListId ptr;
   } Stage4Params deriving (Bits);

// the address translation servers (addr[0], addr[1]) have a latency of 8 and are fully pipelined
module mkMMU#(Integer iid, Bool hostMapped, MMUIndication mmuIndication)(MMU#(addrWidth))
   provisos(Log#(MaxNumSGLists, listIdxSize),
	    Add#(listIdxSize,8, entryIdxSize),
	    Add#(a__,addrWidth,MemOffsetSize));

   let verbose = False;
   TagGen#(MaxNumSGLists) sglId_gen <- mkTagGen();
   rule complete_sglId_gen;
      let __x <- sglId_gen.complete;
   endrule

   // for simulators
   SimDma#(32) simDma <- mkSimDma();

   // stage 0 (latency == 1)
   Vector#(2, FIFO#(AddrTransRequest)) incomingReqs <- replicateM(mkFIFO);

   // stage 1 (latency == 2)
   BRAM_Configure bramConfig = defaultValue;
   bramConfig.latency        = 2;
   BRAM2Port#(RegionsIdx, Maybe#(Region)) regall <- ConnectalBram::mkBRAM2Server(bramConfig);
   Vector#(2,FIFOF#(AddrTransRequest))          reqs0 <- replicateM(mkSizedFIFOF(3));

   // stage 2 (latency == 1)
   Vector#(2, FIFOF#(Stage3Params)) stage3Params <- replicateM(mkFIFOF);

   // stage 3 (latency == 1)
   Vector#(2, FIFOF#(Stage4Params)) stage4Params <- replicateM(mkFIFOF);

   // stage 4 (latency == 2)
   BRAM2Port#(Bit#(entryIdxSize),Page0) translationTable <- ConnectalBram::mkBRAM2Server(bramConfig);
   Vector#(2,FIFOF#(Offset))           offs1 <- replicateM(mkSizedFIFOF(3));

   // stage 4 (latnecy == 1)
   Vector#(2,FIFOF#(AddrTransResponse#(addrWidth))) pageResponseFifos <- replicateM(mkFIFOF);

   FIFO#(DmaError) dmaErrorFifo <- mkFIFO();
   Vector#(2,FIFO#(DmaError)) dmaErrorFifos <- replicateM(mkFIFO());
   for (Integer i = 0; i < 2; i = i + 1)
      mkConnection(toGet(dmaErrorFifos[i]), toPut(dmaErrorFifo));
   rule dmaError;
      let error <- toGet(dmaErrorFifo).get();
      mmuIndication.error(extend(pack(error.errorType)), error.pref, extend(error.off), fromInteger(iid));
   endrule

   let page_shift0 = fromInteger(valueOf(SGListPageShift0));
   let page_shift4 = fromInteger(valueOf(SGListPageShift4));
   let page_shift8 = fromInteger(valueOf(SGListPageShift8));
   let page_shift12 = fromInteger(valueOf(SGListPageShift12));

   function BRAMServer#(a,b) portsel(BRAM2Port#(a,b) x, Integer i);
      if(i==0) return x.portA;
      else return x.portB;
   endfunction

   for (Integer i = 0; i < 2; i=i+1) begin
      rule stage1;  // first read in the address cutoff values between regions
	 AddrTransRequest req <- toGet(incomingReqs[i]).get();
	 portsel(regall, i).request.put(BRAMRequest{write:False, responseOnWrite:False,
            address:truncate(req.id), datain:?});
	 reqs0[i].enq(req);
      endrule

      // pipeline the address lookup
      rule stage2; // Now compare address cutoffs with requested offset
	 AddrTransRequest req <- toGet(reqs0[i]).get;
	 Maybe#(Region) m_regionall <- portsel(regall,i).response.get;

	 case (m_regionall) matches
	    tagged Valid .regionall: begin
               Page0 off0 = truncate(req.off >> valueOf(SGListPageShift0));
               Page4 off4 = truncate(req.off >> valueOf(SGListPageShift4));
               Page8 off8 = truncate(req.off >> valueOf(SGListPageShift8));
               Page12 off12 = truncate(req.off >> valueOf(SGListPageShift12));
	       let cond12 = off12 < truncate(regionall.reg12.barrier);
	       let cond8 = off8 < truncate(regionall.reg8.barrier);
	       let cond4 = off4 < truncate(regionall.reg4.barrier);
	       let cond0 = off0 < regionall.reg0.barrier;

	       if (verbose) $display("mkMMU::stage2: id=%d off=%h (%h %h %h) (%h %h %h)", req.id, req.off,
				     regionall.reg8.barrier, regionall.reg4.barrier, regionall.reg0.barrier,
				     off8, off4, off0);

	       stage3Params[i].enq(Stage3Params {cond12: cond12, cond8: cond8, cond4: cond4, cond0: cond0,
						 idxOffset12: regionall.reg12.idxOffset,idxOffset8: regionall.reg8.idxOffset,
						 idxOffset4: regionall.reg4.idxOffset, idxOffset0: regionall.reg0.idxOffset,
						 req: req });
	    end
	    tagged Invalid:
	       dmaErrorFifos[0].enq(DmaError { errorType: DmaErrorSGLIdInvalid, pref: extend(req.id), off:req.off });
	 endcase
      endrule
      rule stage3; // Based on results of comparision, select a region, putting it into 'o.pageSize'.  idxOffset holds offset in sglist table of relevant entry
	 let params <- toGet(stage3Params[i]).get();
	 AddrTransRequest req = params.req;
	 Offset o = Offset { error: DmaErrorNone, pageSize: 0, value: truncate(req.off)};
	 Bit#(IndexWidth) pbase = 0;
	 Bit#(IndexWidth) idxOffset = 0;

	 if (params.cond12) begin
	    if (verbose) $display("mkMMU::request: req.id=%h req.off=%h", req.id, req.off);
	    o.pageSize = 4;
	    pbase = truncate(req.off>>page_shift12);
	    idxOffset = params.idxOffset12;
	 end
	 else if (params.cond8) begin
	    if (verbose) $display("mkMMU::request: req.id=%h req.off=%h", req.id, req.off);
	    o.pageSize = 3;
	    pbase = truncate(req.off>>page_shift8);
	    idxOffset = params.idxOffset8;
	 end
	 else if (params.cond4) begin
	    if (verbose) $display("mkMMU::request: req.id=%h req.off=%h", req.id, req.off);
	    o.pageSize = 2;
	    pbase = truncate(req.off>>page_shift4);
	    idxOffset = params.idxOffset4;
	 end
	 else if (params.cond0) begin
	    if (verbose) $display("mkMMU::request: req.id=%h req.off=%h", req.id, req.off);
	    o.pageSize = 1;
	    pbase = truncate(req.off>>page_shift0);
	    idxOffset = params.idxOffset0;
	 end
	 stage4Params[i].enq(Stage4Params { off: o, pbase: pbase, idxOffset: idxOffset, ptr: req.id });
      endrule
      (* descending_urgency = "stage2, stage4" *)
      rule stage4; // Read relevant sglist entry
	 let params <- toGet(stage4Params[i]).get();
	 let off = params.off;
	 let pbase = params.pbase;
	 let idxOffset = params.idxOffset;
	 let ptr = params.ptr;
	 Bit#(IndexWidth) p = pbase + idxOffset;
	 if (off.pageSize == 0) begin
	    if (verbose) $display("mkMMU::addr[%d].request.put: ERROR   ptr=%h off=%h\n", i, ptr, off);
	    dmaErrorFifos[1].enq(DmaError { errorType: DmaErrorOffsetOutOfRange, pref: extend(ptr), off:extend(off.value) });
	    off.error = DmaErrorOffsetOutOfRange;
	    p         = 0; // FIXME
	 end

	 if (verbose) $display("mkMMU::translationTable[%d].read %h", i, {ptr,p});
	 portsel(translationTable, i).request.put(BRAMRequest{write:False, responseOnWrite:False,
							      address:{ptr,p}, datain:?});
	 offs1[i].enq(off);

      endrule
      rule stage5; // Concatenate page base address from sglist entry with LSB offset bits from request and return
	 Page0 page <- portsel(translationTable, i).response.get;
	 let offset <- toGet(offs1[i]).get();
	 if (verbose) $display("mkMMU::p ages[%d].response page=%h offset=%h", i, page, offset);
	 Bit#(MemOffsetSize) rv = ?;
	 Page4 b4 = truncate(page);
	 Page8 b8 = truncate(page);
	 Page12 b12 = truncate(page);
	 case (offset.pageSize)
	    1: rv = {page,truncate(offset.value)};
	    2: rv = {b4,truncate(offset.value)};
	    3: rv = {b8,truncate(offset.value)};
	    4: rv = {b12,truncate(offset.value)};
	 endcase
	 pageResponseFifos[i].enq(AddrTransResponse { error: offset.error, physAddr: truncate(rv) });
      endrule
   end

   FIFO#(SGListId) configRespFifo <- mkFIFO;
   rule sendConfigResp;
      let ptr <- toGet(configRespFifo).get();
      mmuIndication.configResp(extend(ptr));
   endrule

   // given that the BRAM is faster than the connection from software, I see no need for a SizedBRAMFIFOF here. -Jamey
   FIFOF#(Bit#(32)) idReturnFifo <- mkFIFOF();
   rule idReturnRule;
      let sglId <- toGet(idReturnFifo).get;
      sglId_gen.returnTag(truncate(sglId));
      portsel(regall, 1).request.put(BRAMRequest{write:True, responseOnWrite:False, address: truncate(sglId), datain: tagged Invalid });
      $display("idReturn %d", sglId);
   endrule

   function Server#(AddrTransRequest,AddrTransResponse#(addrWidth)) addrServer(Integer i);
   return
      (interface Server#(AddrTransRequest,Bit#(addrWidth));
	  interface Put request;
	     method Action put(AddrTransRequest req);
		incomingReqs[i].enq(req);
	     endmethod
	  endinterface
	  interface Get response;
	     method ActionValue#(AddrTransResponse#(addrWidth)) get();
		let rv <- toGet(pageResponseFifos[i]).get();
`ifdef SIMULATION
		rv.physAddr = rv.physAddr | (fromInteger(iid)<<valueOf(addrWidth)-3);
`endif
		return rv;
	     endmethod
	  endinterface
       endinterface);
   endfunction

   interface MMURequest request;
   method Action idRequest(SpecialTypeForSendingFd fd);
      let nextId <- sglId_gen.getTag;
      let resp = (fromInteger(iid) << 16) | extend(nextId);
      if (verbose) $display("mkMMU::idRequest %d", fd);
      if (hostMapped) begin
	 let va <- simDma.initfd(resp, fd);
      end
      mmuIndication.idResponse(resp);
   endmethod
   method Action idReturn(Bit#(32) sglId);
      idReturnFifo.enq(sglId);
      if (hostMapped)
	 simDma.idreturn(sglId);
   endmethod
   method Action region(Bit#(32) pointer, Bit#(64) barr12, Bit#(32) index12, Bit#(64) barr8, Bit#(32) index8, Bit#(64) barr4, Bit#(32) index4, Bit#(64) barr0, Bit#(32) index0);
      portsel(regall, 1).request.put(BRAMRequest{write:True, responseOnWrite:False,
          address: truncate(pointer), datain: tagged Valid Region{
             reg12: SingleRegion{barrier: truncate(barr12), idxOffset: truncate(index12)},
             reg8: SingleRegion{barrier: truncate(barr8), idxOffset: truncate(index8)},
             reg4: SingleRegion{barrier: truncate(barr4), idxOffset: truncate(index4)},
             reg0: SingleRegion{barrier: truncate(barr0), idxOffset: truncate(index0)}} });
      if (verbose) $display("mkMMU::region pointer=%d barr12=%h barr8=%h barr4=%h barr0=%h", pointer, barr12, barr8, barr4, barr0);
      configRespFifo.enq(truncate(pointer));
   endmethod

   method Action sglist(Bit#(32) pointer, Bit#(32) pointerIndex, Bit#(64) addr,  Bit#(32) len);
         if (fromInteger(iid) != pointer[31:16]) begin
	    $display("mkMMU::sglist ERROR");
	    $finish();
	 end
	 if(hostMapped)
	    let va <- simDma.init({0,pointer[31:16]}, {0,pointer[15:0]}, len);
         Bit#(IndexWidth) ind = truncate(pointerIndex);
	 portsel(translationTable, 0).request.put(BRAMRequest{write:True, responseOnWrite:False,
             address:{truncate(pointer),ind}, datain:truncate(addr)});
         if (verbose) $display("mkMMU::sglist pointer=%d pointerIndex=%d addr=%d len=%d", pointer, pointerIndex, addr, len);
   endmethod
   method Action setInterface(Bit#(32) interfaceId, Bit#(32) sglId);
       /* this method is only implemented in s/w responders */
   endmethod
   endinterface
   interface addr = genWith(addrServer);

endmodule

interface ArbitratedMMU#(numeric type addrWidth, numeric type numServers);
   interface Vector#(numServers,Server#(AddrTransRequest,AddrTransResponse#(addrWidth))) servers;
endinterface

module mkArbitratedMMU#(Server#(AddrTransRequest,AddrTransResponse#(addrWidth)) server) (ArbitratedMMU#(addrWidth,numServers));

   FIFOF#(Bit#(TAdd#(1,TLog#(numServers)))) tokFifo <- mkSizedFIFOF(9);
   Reg#(Bit#(TLog#(numServers))) arb <- mkReg(0);

   // this is a very crude arbiter.  something more sophisticated may be required (mdk)
   rule inc_arb;
      arb <= arb+1;
   endrule

   function Server#(AddrTransRequest,AddrTransResponse#(addrWidth)) arbitratedServer(Integer i);
   return
      (interface Server#(AddrTransRequest,AddrTransResponse#(addrWidth));
	  interface Put request;
	     method Action put(AddrTransRequest req) if (arb == fromInteger(i));
		tokFifo.enq(fromInteger(i));
		server.request.put(req);
	     endmethod
	  endinterface
	  interface Get response;
	     method ActionValue#(AddrTransResponse#(addrWidth)) get() if (tokFifo.first == fromInteger(i));
		tokFifo.deq;
		let rv <- server.response.get;
		return rv;
	     endmethod
	  endinterface
       endinterface);
   endfunction

   interface servers = genWith(arbitratedServer);

endmodule

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

// BSV Libraries
import RegFile::*;
import FIFO::*;
import FIFOF::*;
import Vector::*;
import GetPut::*;
import Connectable::*;
import BRAMFIFO::*;
import BRAM::*;
import Probe::*;

import ConnectalBram::*;
import MemTypes::*;
import StmtFSM::*;
import ClientServer::*;
import ConnectalMemory::*;
import ConnectalCompletionBuffer::*;
import SimDma::*;

`include "ConnectalProjectConfig.bsv"

typedef 32 MaxNumSGLists;
typedef Bit#(TLog#(MaxNumSGLists)) SGListId;
typedef 12 SGListPageShift0;
typedef 16 SGListPageShift4;
typedef 20 SGListPageShift8;
typedef 24 SGListPageShift12;
typedef Bit#(TLog#(MaxNumSGLists)) RegionsIdx;

typedef 8 IndexWidth;

typedef struct {
   SGListId             id;
   Bit#(MemOffsetSize) off;
} AddrTransRequest deriving (Eq,Bits,FShow);

typedef struct {
   DmaErrorType    error;
   Bit#(addrWidth) physAddr;
} AddrTransResponse#(numeric type addrWidth) deriving (Eq,Bits,FShow);

interface MMU#(numeric type addrWidth);
   interface MMURequest request;
   interface Vector#(2,Server#(AddrTransRequest,AddrTransResponse#(addrWidth))) addr;
endinterface

module mkSimpleMMU#(Integer iid, Bool hostMapped, MMUIndication mmuIndication)(MMU#(addrWidth))
   provisos (Add#(b__,8,addrWidth)
	     ,Add#(c__,24,addrWidth)
	     ,Bits#(AddrTransResponse#(addrWidth),d__)
	     );
   Vector#(2,FIFOF#(AddrTransRequest)) reqFifos <- replicateM(mkFIFOF());
   Vector#(2,FIFOF#(AddrTransResponse#(addrWidth))) respFifos <- replicateM(mkFIFOF());

   let probePhysAddr <- mkProbe();

   for (Integer i = 0; i < 2; i = i + 1) begin
      rule rl_process_req;
	 let req <- toGet(reqFifos[i]).get();
	 Bit#(addrWidth) physAddr = extend(req.off[23:0]);
	 physAddr[31:24] = extend(req.id);
	 probePhysAddr <= physAddr;
	 respFifos[i].enq(AddrTransResponse { error: DmaErrorNone, physAddr: physAddr });
      endrule
   end

   function Server#(AddrTransRequest,AddrTransResponse#(addrWidth)) genServer(Integer i);
      return (interface Server#(AddrTransRequest,AddrTransResponse#(addrWidth));
		 interface request = toPut(reqFifos[i]);
	 interface response = toGet(respFifos[i]);
	 endinterface);
   endfunction

   interface MMURequest request;
      method Action idRequest(SpecialTypeForSendingFd fd);
	 mmuIndication.idResponse(0);
      endmethod
      method Action idReturn(Bit#(32) sglId);
	 mmuIndication.idResponse(sglId);
      endmethod
      method Action region(Bit#(32) pointer, Bit#(64) barr12, Bit#(32) index12, Bit#(64) barr8, Bit#(32) index8, Bit#(64) barr4, Bit#(32) index4, Bit#(64) barr0, Bit#(32) index0);
	 mmuIndication.configResp(extend(pointer));
      endmethod
      method Action sglist(Bit#(32) pointer, Bit#(32) pointerIndex, Bit#(64) addr,  Bit#(32) len);
	 // no response
      endmethod
      method Action setInterface(Bit#(32) interfaceId, Bit#(32) sglId);
	 /* this method is only implemented in s/w responders */
      endmethod
   endinterface
   interface Vector addr = genWith(genServer);
endmodule

typedef struct {
   DmaErrorType error;
   Bit#(3) pageSize;
   Bit#(SGListPageShift12) value;
} Offset deriving (Eq,Bits,FShow);

typedef Bit#(TSub#(MemOffsetSize,SGListPageShift0)) Page0;
typedef Bit#(TSub#(MemOffsetSize,SGListPageShift4)) Page4;
typedef Bit#(TSub#(MemOffsetSize,SGListPageShift8)) Page8;
typedef Bit#(TSub#(MemOffsetSize,SGListPageShift12)) Page12;

typedef struct {
   Bit#(TSub#(MemOffsetSize,SGListPageShift0)) barrier;
   Bit#(IndexWidth) idxOffset;
   } SingleRegion deriving (Eq,Bits,FShow);

typedef struct {
   SingleRegion reg12;
   SingleRegion reg8;
   SingleRegion reg4;
   SingleRegion reg0;
   } Region deriving (Eq,Bits,FShow);

typedef struct {DmaErrorType errorType;
		Bit#(32) pref;
		Bit#(MemOffsetSize) off;
   } DmaError deriving (Bits);

typedef struct {
   Bool cond12;
   Bool cond8;
   Bool cond4;
   Bool cond0;
   Bit#(IndexWidth) idxOffset12;
   Bit#(IndexWidth) idxOffset8;
   Bit#(IndexWidth) idxOffset4;
   Bit#(IndexWidth) idxOffset0;
   AddrTransRequest req;
   } Stage3Params deriving (Bits);

typedef struct {
   Offset off;
   Bit#(IndexWidth) pbase;
   Bit#(IndexWidth) idxOffset;
   SGListId ptr;
   } Stage4Params deriving (Bits);

// the address translation servers (addr[0], addr[1]) have a latency of 8 and are fully pipelined
module mkMMU#(Integer iid, Bool hostMapped, MMUIndication mmuIndication)(MMU#(addrWidth))
   provisos(Log#(MaxNumSGLists, listIdxSize),
	    Add#(listIdxSize,8, entryIdxSize),
	    Add#(a__,addrWidth,MemOffsetSize));

   let verbose = False;
   TagGen#(MaxNumSGLists) sglId_gen <- mkTagGen();
   rule complete_sglId_gen;
      let __x <- sglId_gen.complete;
   endrule

   // for simulators
   SimDma#(32) simDma <- mkSimDma();

   // stage 0 (latency == 1)
   Vector#(2, FIFO#(AddrTransRequest)) incomingReqs <- replicateM(mkFIFO);

   // stage 1 (latency == 2)
   BRAM_Configure bramConfig = defaultValue;
   bramConfig.latency        = 2;
   BRAM2Port#(RegionsIdx, Maybe#(Region)) regall <- ConnectalBram::mkBRAM2Server(bramConfig);
   Vector#(2,FIFOF#(AddrTransRequest))          reqs0 <- replicateM(mkSizedFIFOF(3));

   // stage 2 (latency == 1)
   Vector#(2, FIFOF#(Stage3Params)) stage3Params <- replicateM(mkFIFOF);

   // stage 3 (latency == 1)
   Vector#(2, FIFOF#(Stage4Params)) stage4Params <- replicateM(mkFIFOF);

   // stage 4 (latency == 2)
   BRAM2Port#(Bit#(entryIdxSize),Page0) translationTable <- ConnectalBram::mkBRAM2Server(bramConfig);
   Vector#(2,FIFOF#(Offset))           offs1 <- replicateM(mkSizedFIFOF(3));

   // stage 4 (latnecy == 1)
   Vector#(2,FIFOF#(AddrTransResponse#(addrWidth))) pageResponseFifos <- replicateM(mkFIFOF);

   FIFO#(DmaError) dmaErrorFifo <- mkFIFO();
   Vector#(2,FIFO#(DmaError)) dmaErrorFifos <- replicateM(mkFIFO());
   for (Integer i = 0; i < 2; i = i + 1)
      mkConnection(toGet(dmaErrorFifos[i]), toPut(dmaErrorFifo));
   rule dmaError;
      let error <- toGet(dmaErrorFifo).get();
      mmuIndication.error(extend(pack(error.errorType)), error.pref, extend(error.off), fromInteger(iid));
   endrule

   let page_shift0 = fromInteger(valueOf(SGListPageShift0));
   let page_shift4 = fromInteger(valueOf(SGListPageShift4));
   let page_shift8 = fromInteger(valueOf(SGListPageShift8));
   let page_shift12 = fromInteger(valueOf(SGListPageShift12));

   function BRAMServer#(a,b) portsel(BRAM2Port#(a,b) x, Integer i);
      if(i==0) return x.portA;
      else return x.portB;
   endfunction

   for (Integer i = 0; i < 2; i=i+1) begin
      rule stage1;  // first read in the address cutoff values between regions
	 AddrTransRequest req <- toGet(incomingReqs[i]).get();
	 portsel(regall, i).request.put(BRAMRequest{write:False, responseOnWrite:False,
            address:truncate(req.id), datain:?});
	 reqs0[i].enq(req);
      endrule

      // pipeline the address lookup
      rule stage2; // Now compare address cutoffs with requested offset
	 AddrTransRequest req <- toGet(reqs0[i]).get;
	 Maybe#(Region) m_regionall <- portsel(regall,i).response.get;

	 case (m_regionall) matches
	    tagged Valid .regionall: begin
               Page0 off0 = truncate(req.off >> valueOf(SGListPageShift0));
               Page4 off4 = truncate(req.off >> valueOf(SGListPageShift4));
               Page8 off8 = truncate(req.off >> valueOf(SGListPageShift8));
               Page12 off12 = truncate(req.off >> valueOf(SGListPageShift12));
	       let cond12 = off12 < truncate(regionall.reg12.barrier);
	       let cond8 = off8 < truncate(regionall.reg8.barrier);
	       let cond4 = off4 < truncate(regionall.reg4.barrier);
	       let cond0 = off0 < regionall.reg0.barrier;

	       if (verbose) $display("mkMMU::stage2: id=%d off=%h (%h %h %h) (%h %h %h)", req.id, req.off,
				     regionall.reg8.barrier, regionall.reg4.barrier, regionall.reg0.barrier,
				     off8, off4, off0);

	       stage3Params[i].enq(Stage3Params {cond12: cond12, cond8: cond8, cond4: cond4, cond0: cond0,
						 idxOffset12: regionall.reg12.idxOffset,idxOffset8: regionall.reg8.idxOffset,
						 idxOffset4: regionall.reg4.idxOffset, idxOffset0: regionall.reg0.idxOffset,
						 req: req });
	    end
	    tagged Invalid:
	       dmaErrorFifos[0].enq(DmaError { errorType: DmaErrorSGLIdInvalid, pref: extend(req.id), off:req.off });
	 endcase
      endrule
      rule stage3; // Based on results of comparision, select a region, putting it into 'o.pageSize'.  idxOffset holds offset in sglist table of relevant entry
	 let params <- toGet(stage3Params[i]).get();
	 AddrTransRequest req = params.req;
	 Offset o = Offset { error: DmaErrorNone, pageSize: 0, value: truncate(req.off)};
	 Bit#(IndexWidth) pbase = 0;
	 Bit#(IndexWidth) idxOffset = 0;

	 if (params.cond12) begin
	    if (verbose) $display("mkMMU::request: req.id=%h req.off=%h", req.id, req.off);
	    o.pageSize = 4;
	    pbase = truncate(req.off>>page_shift12);
	    idxOffset = params.idxOffset12;
	 end
	 else if (params.cond8) begin
	    if (verbose) $display("mkMMU::request: req.id=%h req.off=%h", req.id, req.off);
	    o.pageSize = 3;
	    pbase = truncate(req.off>>page_shift8);
	    idxOffset = params.idxOffset8;
	 end
	 else if (params.cond4) begin
	    if (verbose) $display("mkMMU::request: req.id=%h req.off=%h", req.id, req.off);
	    o.pageSize = 2;
	    pbase = truncate(req.off>>page_shift4);
	    idxOffset = params.idxOffset4;
	 end
	 else if (params.cond0) begin
	    if (verbose) $display("mkMMU::request: req.id=%h req.off=%h", req.id, req.off);
	    o.pageSize = 1;
	    pbase = truncate(req.off>>page_shift0);
	    idxOffset = params.idxOffset0;
	 end
	 stage4Params[i].enq(Stage4Params { off: o, pbase: pbase, idxOffset: idxOffset, ptr: req.id });
      endrule
      (* descending_urgency = "stage2, stage4" *)
      rule stage4; // Read relevant sglist entry
	 let params <- toGet(stage4Params[i]).get();
	 let off = params.off;
	 let pbase = params.pbase;
	 let idxOffset = params.idxOffset;
	 let ptr = params.ptr;
	 Bit#(IndexWidth) p = pbase + idxOffset;
	 if (off.pageSize == 0) begin
	    if (verbose) $display("mkMMU::addr[%d].request.put: ERROR   ptr=%h off=%h\n", i, ptr, off);
	    dmaErrorFifos[1].enq(DmaError { errorType: DmaErrorOffsetOutOfRange, pref: extend(ptr), off:extend(off.value) });
	    off.error = DmaErrorOffsetOutOfRange;
	    p         = 0; // FIXME
	 end

	 if (verbose) $display("mkMMU::translationTable[%d].read %h", i, {ptr,p});
	 portsel(translationTable, i).request.put(BRAMRequest{write:False, responseOnWrite:False,
							      address:{ptr,p}, datain:?});
	 offs1[i].enq(off);

      endrule
      rule stage5; // Concatenate page base address from sglist entry with LSB offset bits from request and return
	 Page0 page <- portsel(translationTable, i).response.get;
	 let offset <- toGet(offs1[i]).get();
	 if (verbose) $display("mkMMU::p ages[%d].response page=%h offset=%h", i, page, offset);
	 Bit#(MemOffsetSize) rv = ?;
	 Page4 b4 = truncate(page);
	 Page8 b8 = truncate(page);
	 Page12 b12 = truncate(page);
	 case (offset.pageSize)
	    1: rv = {page,truncate(offset.value)};
	    2: rv = {b4,truncate(offset.value)};
	    3: rv = {b8,truncate(offset.value)};
	    4: rv = {b12,truncate(offset.value)};
	 endcase
	 pageResponseFifos[i].enq(AddrTransResponse { error: offset.error, physAddr: truncate(rv) });
      endrule
   end

   FIFO#(SGListId) configRespFifo <- mkFIFO;
   rule sendConfigResp;
      let ptr <- toGet(configRespFifo).get();
      mmuIndication.configResp(extend(ptr));
   endrule

   // given that the BRAM is faster than the connection from software, I see no need for a SizedBRAMFIFOF here. -Jamey
   FIFOF#(Bit#(32)) idReturnFifo <- mkFIFOF();
   rule idReturnRule;
      let sglId <- toGet(idReturnFifo).get;
      sglId_gen.returnTag(truncate(sglId));
      portsel(regall, 1).request.put(BRAMRequest{write:True, responseOnWrite:False, address: truncate(sglId), datain: tagged Invalid });
      $display("idReturn %d", sglId);
   endrule

   function Server#(AddrTransRequest,AddrTransResponse#(addrWidth)) addrServer(Integer i);
   return
      (interface Server#(AddrTransRequest,Bit#(addrWidth));
	  interface Put request;
	     method Action put(AddrTransRequest req);
		incomingReqs[i].enq(req);
	     endmethod
	  endinterface
	  interface Get response;
	     method ActionValue#(AddrTransResponse#(addrWidth)) get();
		let rv <- toGet(pageResponseFifos[i]).get();
`ifdef SIMULATION
		rv.physAddr = rv.physAddr | (fromInteger(iid)<<valueOf(addrWidth)-3);
`endif
		return rv;
	     endmethod
	  endinterface
       endinterface);
   endfunction

   interface MMURequest request;
   method Action idRequest(SpecialTypeForSendingFd fd);
      let nextId <- sglId_gen.getTag;
      let resp = (fromInteger(iid) << 16) | extend(nextId);
      if (verbose) $display("mkMMU::idRequest %d", fd);
      let va <- simDma.initfd(resp, fd);
      mmuIndication.idResponse(resp);
   endmethod
   method Action idReturn(Bit#(32) sglId);
      idReturnFifo.enq(sglId);
      if (hostMapped)
	 simDma.idreturn(sglId);
   endmethod
   method Action region(Bit#(32) pointer, Bit#(64) barr12, Bit#(32) index12, Bit#(64) barr8, Bit#(32) index8, Bit#(64) barr4, Bit#(32) index4, Bit#(64) barr0, Bit#(32) index0);
      portsel(regall, 1).request.put(BRAMRequest{write:True, responseOnWrite:False,
          address: truncate(pointer), datain: tagged Valid Region{
             reg12: SingleRegion{barrier: truncate(barr12), idxOffset: truncate(index12)},
             reg8: SingleRegion{barrier: truncate(barr8), idxOffset: truncate(index8)},
             reg4: SingleRegion{barrier: truncate(barr4), idxOffset: truncate(index4)},
             reg0: SingleRegion{barrier: truncate(barr0), idxOffset: truncate(index0)}} });
      if (verbose) $display("mkMMU::region pointer=%d barr12=%h barr8=%h barr4=%h barr0=%h", pointer, barr12, barr8, barr4, barr0);
      configRespFifo.enq(truncate(pointer));
   endmethod

   method Action sglist(Bit#(32) pointer, Bit#(32) pointerIndex, Bit#(64) addr,  Bit#(32) len);
         if (fromInteger(iid) != pointer[31:16]) begin
	    $display("mkMMU::sglist ERROR");
	    $finish();
	 end
	 if(hostMapped)
	    let va <- simDma.init({0,pointer[31:16]}, {0,pointer[15:0]}, len);
         Bit#(IndexWidth) ind = truncate(pointerIndex);
	 portsel(translationTable, 0).request.put(BRAMRequest{write:True, responseOnWrite:False,
             address:{truncate(pointer),ind}, datain:truncate(addr)});
         if (verbose) $display("mkMMU::sglist pointer=%d pointerIndex=%d addr=%d len=%d", pointer, pointerIndex, addr, len);
   endmethod
   method Action setInterface(Bit#(32) interfaceId, Bit#(32) sglId);
       /* this method is only implemented in s/w responders */
   endmethod
   endinterface
   interface addr = genWith(addrServer);

endmodule

interface ArbitratedMMU#(numeric type addrWidth, numeric type numServers);
   interface Vector#(numServers,Server#(AddrTransRequest,AddrTransResponse#(addrWidth))) servers;
endinterface

module mkArbitratedMMU#(Server#(AddrTransRequest,AddrTransResponse#(addrWidth)) server) (ArbitratedMMU#(addrWidth,numServers));

   FIFOF#(Bit#(TAdd#(1,TLog#(numServers)))) tokFifo <- mkSizedFIFOF(9);
   Reg#(Bit#(TLog#(numServers))) arb <- mkReg(0);

   // this is a very crude arbiter.  something more sophisticated may be required (mdk)
   rule inc_arb;
      arb <= arb+1;
   endrule

   function Server#(AddrTransRequest,AddrTransResponse#(addrWidth)) arbitratedServer(Integer i);
   return
      (interface Server#(AddrTransRequest,AddrTransResponse#(addrWidth));
	  interface Put request;
	     method Action put(AddrTransRequest req) if (arb == fromInteger(i));
		tokFifo.enq(fromInteger(i));
		server.request.put(req);
	     endmethod
	  endinterface
	  interface Get response;
	     method ActionValue#(AddrTransResponse#(addrWidth)) get() if (tokFifo.first == fromInteger(i));
		tokFifo.deq;
		let rv <- server.response.get;
		return rv;
	     endmethod
	  endinterface
       endinterface);
   endfunction

   interface servers = genWith(arbitratedServer);

endmodule


minor fixes

minor fixes

// Copyright (c) 2014-2015 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

package Pcie3EndpointX7;

import Clocks            ::*;
import Vector            ::*;
import Connectable       ::*;
import GetPut            ::*;
import Reserved          ::*;
import TieOff            ::*;
import DefaultValue      ::*;
import DReg              ::*;
import Gearbox           ::*;
import FIFO              ::*;
import FIFOF             ::*;
import SpecialFIFOs      ::*;
import ClientServer      ::*;
import Real              ::*;
import XilinxVirtex7PCIE ::*;
import BUtils            ::*;

import ConnectalClocks   ::*;
import ConnectalXilinxCells   ::*;
import XilinxCells       ::*;
import PCIE              ::*;
import PCIEWRAPPER3      ::*;
import Bufgctrl           ::*;
import PcieGearbox       :: *;

interface PcieEndpointX7#(numeric type lanes);
   interface PciewrapPci_exp#(lanes)           pcie;
   interface PciewrapUser#(lanes)              user;
   interface PciewrapPipe#(lanes)              pipe;
   interface PciewrapCommon#(lanes)            common;
   interface Server#(TLPData#(16), TLPData#(16)) tlpr;
   interface Server#(TLPData#(16), TLPData#(16)) tlpc;
   interface Clock epPcieClock;
   interface Reset epPcieReset;
   interface Clock epPortalClock;
   interface Reset epPortalReset;
   interface Clock epDerivedClock;
   interface Reset epDerivedReset;
endinterface

typedef struct {
   Bit #(256)     data;
   Bool          sop;
   Bool          eop;
   Bit #(8)      keep;
   TLPFirstDWBE  first_be;
   TLPFirstDWBE  last_be;
} AxiStCq deriving (Bits, Eq);

typedef struct {
   Bit #(256)     data;
   Bit #(8)      keep;
   Bool          last;
} AxiStCc deriving (Bits, Eq);

typedef struct {
   Bit #(256)     data;
   Bool          last;
   Bit #(8)      keep;
   Bit #(4)      first_be;
   Bit #(4)      last_be;
} AxiStRq deriving (Bits, Eq);

typedef struct {
   Bit #(256)     data;
   Bool          sop;
   Bool          eop;
   Bit #(8)      keep;
   Bit #(8)      be;
} AxiStRc deriving (Bits, Eq);

`ifdef BOARD_vc709
typedef 8 PcieLanes;
typedef 8 NumLeds;
`endif
`ifdef BOARD_nfsume
typedef 8 PcieLanes;
typedef 2 NumLeds;
`endif

(* synthesize *)
module mkPcieEndpointX7(PcieEndpointX7#(PcieLanes));

   PCIEParams params = defaultValue;
   Clock defaultClock <- exposeCurrentClock();
   Reset defaultReset <- exposeCurrentReset();
   Reset defaultResetInverted <- mkResetInverter(defaultReset, clocked_by defaultClock);
   PcieWrap#(PcieLanes) pcie_ep <- mkPcieWrap(defaultClock, defaultResetInverted);

   // The PCIe endpoint exports full (250MHz) and half-speed (125MHz) clocks
   Clock pcieClock250 = pcie_ep.user_clk;
   Reset user_reset_n <- mkResetInverter(pcie_ep.user_reset, clocked_by pcie_ep.user_clk);
   Reset pcieReset250 <- mkAsyncReset(4, user_reset_n, pcieClock250);

   ClockGenerator7Params     clkgenParams = defaultValue;
   clkgenParams.clkin1_period    = 4.000; //  250MHz
   clkgenParams.clkin1_period    = 4.000;
   clkgenParams.clkin_buffer     = False;
   clkgenParams.clkfbout_mult_f  = 4.000; // 1000MHz
   clkgenParams.clkout0_divide_f = derivedClockPeriod;
   clkgenParams.clkout1_divide     = round(mainClockPeriod);
   clkgenParams.clkout1_duty_cycle = 0.5;
   clkgenParams.clkout1_phase      = 0.0000;
   ClockGenerator7           clkgen <- mkClockGenerator7(clkgenParams, clocked_by pcieClock250, reset_by pcieReset250);
   Clock mainClock = clkgen.clkout1;
   Reset mainReset <- mkAsyncReset(4, pcieReset250, mainClock);
   Clock derivedClock = clkgen.clkout0;
   Reset derivedReset <- mkAsyncReset(4, pcieReset250, derivedClock);
   Reset user_reset <- mkAsyncReset(2, pcie_ep.user_reset, pcie_ep.user_clk);

   FIFOF#(AxiStCq) fAxiCq <- mkFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   FIFOF#(AxiStRc) fAxiRc <- mkFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);

   FIFOF#(AxiStRq) fAxiRq <- mkFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   FIFOF#(AxiStCc) fAxiCc <- mkFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);

   FIFOF#(TLPData#(16)) fcq <- mkFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   FIFOF#(TLPData#(16)) frc <- mkFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   FIFOF#(TLPData#(16)) fcc <- mkFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   FIFOF#(TLPData#(16)) frq <- mkFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);

   // Drive s_axis_rq
   let rq_txready = (pcie_ep.s_axis_rq.tready != 0 && fAxiRq.notEmpty);

   //(* fire_when_enabled, no_implicit_conditions *)
   rule drive_axi_rq if (rq_txready);
      let info = fAxiRq.first; fAxiRq.deq;
      pcie_ep.s_axis_rq.tvalid(1);
      pcie_ep.s_axis_rq.tlast(pack(info.last));
      pcie_ep.s_axis_rq.tdata(info.data);
      pcie_ep.s_axis_rq.tkeep(info.keep);
      pcie_ep.s_axis_rq.tuser({0, info.last_be, info.first_be});
   endrule

   (* fire_when_enabled, no_implicit_conditions *)
   rule drive_axi_rq2 if (!rq_txready);
      pcie_ep.s_axis_rq.tvalid(0);
      pcie_ep.s_axis_rq.tlast(0);
      pcie_ep.s_axis_rq.tdata(0);
      pcie_ep.s_axis_rq.tkeep(0);
      pcie_ep.s_axis_rq.tuser(0);
   endrule

   // Drive s_axis_cc
   let cc_txready = (pcie_ep.s_axis_cc.tready != 0 && fAxiCc.notEmpty);

   //(* fire_when_enabled, no_implicit_conditions *)
   rule drive_axi_cc if (cc_txready);
      let info = fAxiCc.first; fAxiCc.deq;
      $display("drive axi_cc, data: %h, keep: %h, last: %h", info.data, info.keep, info.last);
      pcie_ep.s_axis_cc.tvalid(1);
      pcie_ep.s_axis_cc.tlast(pack(info.last));
      pcie_ep.s_axis_cc.tdata(info.data);
      pcie_ep.s_axis_cc.tkeep(info.keep);
      pcie_ep.s_axis_cc.tuser(0);
   endrule

   (* fire_when_enabled, no_implicit_conditions *)
   rule drive_axi_cc2 if (!cc_txready);
      pcie_ep.s_axis_cc.tvalid(0);
      pcie_ep.s_axis_cc.tlast(0);
      pcie_ep.s_axis_cc.tdata(0);
      pcie_ep.s_axis_cc.tkeep(0);
      pcie_ep.s_axis_cc.tuser(0);
   endrule

   (* fire_when_enabled, no_implicit_conditions *)
   rule drive_axi_rc_ready;
      pcie_ep.m_axis_rc.tready (duplicate (pack (fAxiRc.notFull)));
   endrule

   // Drive m_axis_rc
   (* fire_when_enabled *)
   rule sink_axi_rc if (pcie_ep.m_axis_rc.tvalid != 0 && fAxiRc.notFull);
      let rc = AxiStRc {data:pcie_ep.m_axis_rc.tdata,
			sop: unpack (pcie_ep.m_axis_rc.tuser [32]),         // tuser.is_sof_0
			eop: unpack (pcie_ep.m_axis_rc.tlast),
			keep:pcie_ep.m_axis_rc.tkeep,
			be:  truncate (pcie_ep.m_axis_rc.tuser [31:0])};    // tuser.byte_en
      fAxiRc.enq (rc);
   endrule

   (* fire_when_enabled, no_implicit_conditions *)
   rule drive_axi_cq_ready;
      pcie_ep.m_axis_cq.tready (duplicate (pack (fAxiCq.notFull)));
   endrule

   (* fire_when_enabled *)
   rule sink_axi_cq if (pcie_ep.m_axis_cq.tvalid != 0 && fAxiCq.notFull);
      let cq = AxiStCq {data:     pcie_ep.m_axis_cq.tdata,
			sop:      unpack (pcie_ep.m_axis_cq.tuser [40]),  // tuser.sop
			eop:      unpack (pcie_ep.m_axis_cq.tlast),
			keep:     pcie_ep.m_axis_cq.tkeep,
			first_be: pcie_ep.m_axis_cq.tuser [3:0],    // tuser.first_be,
			last_be:  pcie_ep.m_axis_cq.tuser [7:4]};   // tuser.last_be
      fAxiCq.enq (cq);
   endrule

   // CQ.
   CQDescriptor cq_desc = unpack(fAxiCq.first.data [127:0]);

   rule rl_cq_wr_header (fAxiCq.first.sop && ((cq_desc.reqtype == MEMORY_WRITE) || (cq_desc.reqtype == IO_WRITE)));
      Bit#(32) data = fAxiCq.first.data[159:128];
      // get data;
      TLPData#(16) tlp16 = convertCQDescriptorToTLP16(cq_desc, data, fAxiCq.first.first_be, fAxiCq.first.last_be);
      $display("cq_desc.reqtype=%h", cq_desc.reqtype);
      // enqueue?
      fcq.enq(tlp16);
      fAxiCq.deq;
   endrule

   // Write data payload, no data remaining
   rule rl_cq_wr_payload((!fAxiCq.first.sop));
      fAxiCq.deq;
   endrule

   // Write data payload, 1 to 3 DWs remaining
   // Write data payload, 4 or more DWs remaining

   rule rl_cq_rd_header (fAxiCq.first.sop && ((cq_desc.reqtype == MEMORY_READ) || (cq_desc.reqtype == IO_READ)));
      Bit#(32) data = 0;
      TLPData#(16) tlp16 = convertCQDescriptorToTLP16(cq_desc, data, fAxiCq.first.first_be, fAxiCq.first.last_be);
      $display("rl_cq_rd_header: cq_desc = %16x", cq_desc);
      fcq.enq(tlp16);
      fAxiCq.deq;
   endrule

   // RC.
   Reg#(DWCount) rg_dwcount <- mkRegU(clocked_by pcie_ep.user_clk, reset_by user_reset_n);

   rule rl_rc_header (fAxiRc.first.sop);
      RCDescriptor rc_desc = unpack(fAxiRc.first.data [95:0]);
      Bit#(32) data = fAxiRc.first.data[128+32-1:128];
      TLPData#(16) tlp16 = convertRCDescriptorToTLP16(rc_desc, data);
      rg_dwcount <= (rc_desc.dwcount == 0) ? 0 : rc_desc.dwcount - 1;
      frc.enq(tlp16);
      fAxiRc.deq;
   endrule

   rule rl_rc_data ((!(fAxiRc.first.sop)) && (rg_dwcount != 0));
      Bit#(16) be16;
      case (rg_dwcount)
         1: be16 = 16'hF000;
         2: be16 = 16'hFF00;
         3: be16 = 16'hFFF0;
         default: be16 = 16'hFFFF;
      endcase
      TLPData#(16) tlp16 = TLPData{sof: False,
                                   eof: (rg_dwcount <= 4),
                                   hit: 0,
                                   be: be16,
                                   data: pack(fAxiRc.first.data[127:0])};
      frc.enq(tlp16);
      fAxiRc.deq;
   endrule

   Reg#(DWCount) cc_dwcount <- mkReg(0, clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   FIFOF#(TLPData#(16)) fcc_tlps <- mkPipelineFIFOF (clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   // CC.
   rule get_cc_tlps;
      let tlp <- toGet(fcc).get;
      fcc_tlps.enq(tlp);
   endrule

   rule rl_cc_header(fcc_tlps.first.sof);
      match { .cc_desc, .dw} = convertTLP16ToCCDescriptor(fcc_tlps.first);
      cc_dwcount <= cc_desc.dwcount - 1;
      fAxiCc.enq(AxiStCc {data: zeroExtend({dw, pack(cc_desc)[95:0]}),
                       last: fcc_tlps.first.eof,
                       keep: 8'h0F});
      fcc_tlps.deq;
   endrule

   rule rl_cc_data((!fcc_tlps.first.sof) && (cc_dwcount != 0));
      Bit#(256) x = zeroExtend(fcc_tlps.first.data); //FIXME
      fAxiCc.enq(AxiStCc {data: {x},
                       last: cc_dwcount <= 4,
                       keep: (cc_dwcount == 3) ? 8'h0F : 8'hFF});
      fcc_tlps.deq;
   endrule

   // RQ.
   FIFOF#(TLPData#(16)) frq_tlps <- mkPipelineFIFOF (clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   Reg#(DWCount) rq_dwcount <- mkReg(0, clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   Reg#(Maybe#(Bit#(32))) rq_mdw <- mkRegU(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   Reg#(Bit#(4)) rq_first_be <- mkRegU(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   Reg#(Bit#(4)) rq_last_be <- mkRegU(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   rule get_rq_tlps;
      let tlp <- toGet(frq).get;
      frq_tlps.enq(tlp);
   endrule

   rule rl_rq_header(frq_tlps.first.sof);
      match { .rq_desc, .first_be, .last_be, .mdata} = convertTLP16ToRQDescriptor(frq_tlps.first);
      rq_dwcount <= ((rq_desc.reqtype == MEMORY_WRITE) ? rq_desc.dwcount : 0);
      rq_mdw <= mdata;
      rq_first_be <= first_be;
      rq_last_be <= last_be;
      fAxiRq.enq(AxiStRq {data: zeroExtend(pack(rq_desc)), //FIXME:
                       last: frq_tlps.first.eof,
                       keep: 8'hFF,
                       first_be: first_be,
                       last_be: last_be});
      frq_tlps.deq;
   endrule

   // rq_mdw contains last DW
   rule rl_rq_data_a (rq_mdw matches tagged Valid .dw &&& (rq_dwcount == 1));
      rq_dwcount <= 0;
      rq_mdw <= tagged Invalid;
   endrule

   // rq_mdw Valid, and there are more DWs.
   rule rl_rq_data_b(rq_mdw matches tagged Valid .dw &&& (rq_dwcount != 1));

   endrule

   // rq_mdw Invalid, and there are more DWs.
   rule rl_rq_data_c(rq_mdw matches tagged Invalid &&& (rq_dwcount != 1));

   endrule

   // The PCIE endpoint is processing Gen3 descriptors at 250MHz. The
   // AXI bridge is accepting TLPData#(16)s at 250 MHz. The
   // conversion uses half of Gen3 descriptor.
   //mkConnection(tlp8, gb.tlp, clocked_by portalClock, reset_by portalReset);

   let portalClock = (mainClockPeriod == pcieClockPeriod) ? pcieClock250 : mainClock;
   let portalReset = (mainClockPeriod == pcieClockPeriod) ? pcieReset250 : mainReset;

   interface Server tlpr;
      interface request = toPut(frq);
      interface response = toGet(frc);
   endinterface
   // Requests from other PCIe devices
   interface Server tlpc;
      interface request = toPut(fcc);
      interface response = toGet(fcq);
   endinterface
   interface pcie    = pcie_ep.pci_exp;
   interface Pcie3wrapUser user = pcie_ep.user;
   interface PciewrapPipe pipe = pcie_ep.pipe;
   interface PciewrapCommon common= pcie_ep.common;
   interface Clock epPcieClock = pcieClock250;
   interface Reset epPcieReset = pcieReset250;
   interface Clock epPortalClock = portalClock;
   interface Reset epPortalReset = portalReset;
   interface Clock epDerivedClock = derivedClock;
   interface Reset epDerivedReset = derivedReset;
endmodule: mkPcieEndpointX7

endpackage: Pcie3EndpointX7

// Copyright (c) 2014-2015 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

package Pcie3EndpointX7;

import Clocks            ::*;
import Vector            ::*;
import Connectable       ::*;
import GetPut            ::*;
import Reserved          ::*;
import TieOff            ::*;
import DefaultValue      ::*;
import DReg              ::*;
import Gearbox           ::*;
import FIFO              ::*;
import FIFOF             ::*;
import SpecialFIFOs      ::*;
import ClientServer      ::*;
import Real              ::*;
import XilinxVirtex7PCIE ::*;
import BUtils            ::*;

import ConnectalClocks   ::*;
import ConnectalXilinxCells   ::*;
import XilinxCells       ::*;
import PCIE              ::*;
import PCIEWRAPPER3      ::*;
import Bufgctrl           ::*;
import PcieGearbox       :: *;

interface PcieEndpointX7#(numeric type lanes);
   interface PciewrapPci_exp#(lanes)           pcie;
   interface PciewrapUser#(lanes)              user;
   interface PciewrapPipe#(lanes)              pipe;
   interface PciewrapCommon#(lanes)            common;
   interface Server#(TLPData#(16), TLPData#(16)) tlpr;
   interface Server#(TLPData#(16), TLPData#(16)) tlpc;
   interface Clock epPcieClock;
   interface Reset epPcieReset;
   interface Clock epPortalClock;
   interface Reset epPortalReset;
   interface Clock epDerivedClock;
   interface Reset epDerivedReset;
endinterface

typedef struct {
   Bit #(256)     data;
   Bool          sop;
   Bool          eop;
   Bit #(8)      keep;
   TLPFirstDWBE  first_be;
   TLPFirstDWBE  last_be;
} AxiStCq deriving (Bits, Eq);

typedef struct {
   Bit #(256)     data;
   Bit #(8)      keep;
   Bool          last;
} AxiStCc deriving (Bits, Eq);

typedef struct {
   Bit #(256)     data;
   Bool          last;
   Bit #(8)      keep;
   Bit #(4)      first_be;
   Bit #(4)      last_be;
} AxiStRq deriving (Bits, Eq);

typedef struct {
   Bit #(256)     data;
   Bool          sop;
   Bool          eop;
   Bit #(8)      keep;
   Bit #(8)      be;
} AxiStRc deriving (Bits, Eq);

`ifdef BOARD_vc709
typedef 8 PcieLanes;
typedef 8 NumLeds;
`endif
`ifdef BOARD_nfsume
typedef 8 PcieLanes;
typedef 2 NumLeds;
`endif

(* synthesize *)
module mkPcieEndpointX7(PcieEndpointX7#(PcieLanes));

   PCIEParams params = defaultValue;
   Clock defaultClock <- exposeCurrentClock();
   Reset defaultReset <- exposeCurrentReset();
   Reset defaultResetInverted <- mkResetInverter(defaultReset, clocked_by defaultClock);
   PcieWrap#(PcieLanes) pcie_ep <- mkPcieWrap(defaultClock, defaultResetInverted);

   // The PCIe endpoint exports full (250MHz) and half-speed (125MHz) clocks
   Clock pcieClock250 = pcie_ep.user_clk;
   Reset user_reset_n <- mkResetInverter(pcie_ep.user_reset, clocked_by pcie_ep.user_clk);
   Reset pcieReset250 <- mkAsyncReset(4, user_reset_n, pcieClock250);

   ClockGenerator7Params     clkgenParams = defaultValue;
   clkgenParams.clkin1_period    = 4.000; //  250MHz
   clkgenParams.clkin1_period    = 4.000;
   clkgenParams.clkin_buffer     = False;
   clkgenParams.clkfbout_mult_f  = 4.000; // 1000MHz
   clkgenParams.clkout0_divide_f = derivedClockPeriod;
   clkgenParams.clkout1_divide     = round(mainClockPeriod);
   clkgenParams.clkout1_duty_cycle = 0.5;
   clkgenParams.clkout1_phase      = 0.0000;
   ClockGenerator7           clkgen <- mkClockGenerator7(clkgenParams, clocked_by pcieClock250, reset_by pcieReset250);
   Clock mainClock = clkgen.clkout1;
   Reset mainReset <- mkAsyncReset(4, pcieReset250, mainClock);
   Clock derivedClock = clkgen.clkout0;
   Reset derivedReset <- mkAsyncReset(4, pcieReset250, derivedClock);
   Reset user_reset <- mkAsyncReset(2, pcie_ep.user_reset, pcie_ep.user_clk);

   FIFOF#(AxiStCq) fAxiCq <- mkBypassFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   FIFOF#(AxiStRc) fAxiRc <- mkBypassFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);

   FIFOF#(AxiStRq) fAxiRq <- mkBypassFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   FIFOF#(AxiStCc) fAxiCc <- mkBypassFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);

   FIFOF#(TLPData#(16)) fcq <- mkFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   FIFOF#(TLPData#(16)) frc <- mkFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   FIFOF#(TLPData#(16)) fcc <- mkFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   FIFOF#(TLPData#(16)) frq <- mkFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);

   // Drive s_axis_rq
   let rq_txready = (pcie_ep.s_axis_rq.tready != 0 && fAxiRq.notEmpty);

   //(* fire_when_enabled, no_implicit_conditions *)
   rule drive_axi_rq if (rq_txready);
      let info = fAxiRq.first; fAxiRq.deq;
      pcie_ep.s_axis_rq.tvalid(1);
      pcie_ep.s_axis_rq.tlast(pack(info.last));
      pcie_ep.s_axis_rq.tdata(info.data);
      pcie_ep.s_axis_rq.tkeep(info.keep);
      pcie_ep.s_axis_rq.tuser({0, info.last_be, info.first_be});
   endrule

   (* fire_when_enabled, no_implicit_conditions *)
   rule drive_axi_rq2 if (!rq_txready);
      pcie_ep.s_axis_rq.tvalid(0);
      pcie_ep.s_axis_rq.tlast(0);
      pcie_ep.s_axis_rq.tdata(0);
      pcie_ep.s_axis_rq.tkeep(0);
      pcie_ep.s_axis_rq.tuser(0);
   endrule

   // Drive s_axis_cc
   let cc_txready = (pcie_ep.s_axis_cc.tready != 0 && fAxiCc.notEmpty);

   //(* fire_when_enabled, no_implicit_conditions *)
   rule drive_axi_cc if (cc_txready);
      let info = fAxiCc.first; fAxiCc.deq;
      $display("drive axi_cc, data: %h, keep: %h, last: %h", info.data, info.keep, info.last);
      pcie_ep.s_axis_cc.tvalid(1);
      pcie_ep.s_axis_cc.tlast(pack(info.last));
      pcie_ep.s_axis_cc.tdata(info.data);
      pcie_ep.s_axis_cc.tkeep(info.keep);
      pcie_ep.s_axis_cc.tuser(0);
   endrule

   (* fire_when_enabled, no_implicit_conditions *)
   rule drive_axi_cc2 if (!cc_txready);
      pcie_ep.s_axis_cc.tvalid(0);
      pcie_ep.s_axis_cc.tlast(0);
      pcie_ep.s_axis_cc.tdata(0);
      pcie_ep.s_axis_cc.tkeep(0);
      pcie_ep.s_axis_cc.tuser(0);
   endrule

   (* fire_when_enabled, no_implicit_conditions *)
   rule drive_axi_rc_ready;
      pcie_ep.m_axis_rc.tready (duplicate (pack (fAxiRc.notFull)));
   endrule

   // Drive m_axis_rc
   (* fire_when_enabled *)
   rule sink_axi_rc if (pcie_ep.m_axis_rc.tvalid != 0 && fAxiRc.notFull);
      let rc = AxiStRc {data:pcie_ep.m_axis_rc.tdata,
			sop: unpack (pcie_ep.m_axis_rc.tuser [32]),         // tuser.is_sof_0
			eop: unpack (pcie_ep.m_axis_rc.tlast),
			keep:pcie_ep.m_axis_rc.tkeep,
			be:  truncate (pcie_ep.m_axis_rc.tuser [31:0])};    // tuser.byte_en
      fAxiRc.enq (rc);
   endrule

   (* fire_when_enabled, no_implicit_conditions *)
   rule drive_axi_cq_ready;
      pcie_ep.m_axis_cq.tready (duplicate (pack (fAxiCq.notFull)));
   endrule

   (* fire_when_enabled *)
   rule sink_axi_cq if (pcie_ep.m_axis_cq.tvalid != 0 && fAxiCq.notFull);
      let cq = AxiStCq {data:     pcie_ep.m_axis_cq.tdata,
			sop:      unpack (pcie_ep.m_axis_cq.tuser [40]),  // tuser.sop
			eop:      unpack (pcie_ep.m_axis_cq.tlast),
			keep:     pcie_ep.m_axis_cq.tkeep,
			first_be: pcie_ep.m_axis_cq.tuser [3:0],    // tuser.first_be,
			last_be:  pcie_ep.m_axis_cq.tuser [7:4]};   // tuser.last_be
      fAxiCq.enq (cq);
   endrule

   // CQ.
   CQDescriptor cq_desc = unpack(fAxiCq.first.data [127:0]);

   rule rl_cq_wr_header (fAxiCq.first.sop && ((cq_desc.reqtype == MEMORY_WRITE) || (cq_desc.reqtype == IO_WRITE)));
      Bit#(32) data = fAxiCq.first.data[159:128];
      // get data;
      TLPData#(16) tlp16 = convertCQDescriptorToTLP16(cq_desc, data, fAxiCq.first.first_be, fAxiCq.first.last_be);
      $display("cq_desc.reqtype=%h", cq_desc.reqtype);
      // enqueue?
      fcq.enq(tlp16);
      fAxiCq.deq;
   endrule

   // Write data payload, no data remaining
   rule rl_cq_wr_payload((!fAxiCq.first.sop));
      fAxiCq.deq;
   endrule

   // Write data payload, 1 to 3 DWs remaining
   // Write data payload, 4 or more DWs remaining

   rule rl_cq_rd_header (fAxiCq.first.sop && ((cq_desc.reqtype == MEMORY_READ) || (cq_desc.reqtype == IO_READ)));
      Bit#(32) data = 0;
      TLPData#(16) tlp16 = convertCQDescriptorToTLP16(cq_desc, data, fAxiCq.first.first_be, fAxiCq.first.last_be);
      $display("rl_cq_rd_header: cq_desc = %16x", cq_desc);
      fcq.enq(tlp16);
      fAxiCq.deq;
   endrule

   // RC.
   Reg#(DWCount) rg_dwcount <- mkRegU(clocked_by pcie_ep.user_clk, reset_by user_reset_n);

   rule rl_rc_header (fAxiRc.first.sop);
      RCDescriptor rc_desc = unpack(fAxiRc.first.data [95:0]);
      Bit#(32) data = fAxiRc.first.data[31:0];
      TLPData#(16) tlp16 = convertRCDescriptorToTLP16(rc_desc, data);
      rg_dwcount <= (rc_desc.dwcount == 0) ? 0 : rc_desc.dwcount - 1;
      frc.enq(tlp16);
      fAxiRc.deq;
   endrule

   rule rl_rc_data ((!(fAxiRc.first.sop)) && (rg_dwcount != 0));
      Bit#(16) be16;
      case (rg_dwcount)
         1: be16 = 16'hF000;
         2: be16 = 16'hFF00;
         3: be16 = 16'hFFF0;
         default: be16 = 16'hFFFF;
      endcase
      TLPData#(16) tlp16 = TLPData{sof: False,
                                   eof: (rg_dwcount <= 4),
                                   hit: 0,
                                   be: be16,
                                   data: pack(fAxiRc.first.data[127:0])};
      frc.enq(tlp16);
      fAxiRc.deq;
   endrule

   Reg#(DWCount) cc_dwcount <- mkReg(0, clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   FIFOF#(TLPData#(16)) fcc_tlps <- mkPipelineFIFOF (clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   // CC.
   rule get_cc_tlps;
      let tlp <- toGet(fcc).get;
      fcc_tlps.enq(tlp);
   endrule

   rule rl_cc_header(fcc_tlps.first.sof);
      match { .cc_desc, .dw} = convertTLP16ToCCDescriptor(fcc_tlps.first);
      cc_dwcount <= cc_desc.dwcount - 1;
      fAxiCc.enq(AxiStCc {data: zeroExtend({dw, pack(cc_desc)[95:0]}),
                       last: fcc_tlps.first.eof,
                       keep: 8'h0F});
      fcc_tlps.deq;
   endrule

   rule rl_cc_data((!fcc_tlps.first.sof) && (cc_dwcount != 0));
      Bit#(256) x = zeroExtend(fcc_tlps.first.data); //FIXME
      fAxiCc.enq(AxiStCc {data: {x},
                       last: cc_dwcount <= 4,
                       keep: (cc_dwcount == 3) ? 8'h0F : 8'hFF});
      fcc_tlps.deq;
   endrule

   // RQ.
   FIFOF#(TLPData#(16)) frq_tlps <- mkPipelineFIFOF (clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   Reg#(DWCount) rq_dwcount <- mkReg(0, clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   Reg#(Maybe#(Bit#(32))) rq_mdw <- mkRegU(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   Reg#(Bit#(4)) rq_first_be <- mkRegU(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   Reg#(Bit#(4)) rq_last_be <- mkRegU(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   rule get_rq_tlps;
      let tlp <- toGet(frq).get;
      frq_tlps.enq(tlp);
   endrule

   rule rl_rq_header(frq_tlps.first.sof);
      match { .rq_desc, .first_be, .last_be, .mdata} = convertTLP16ToRQDescriptor(frq_tlps.first);
      rq_dwcount <= ((rq_desc.reqtype == MEMORY_WRITE) ? rq_desc.dwcount : 0);
      rq_mdw <= mdata;
      rq_first_be <= first_be;
      rq_last_be <= last_be;
      fAxiRq.enq(AxiStRq {data: zeroExtend(pack(rq_desc)), //FIXME:
                       last: frq_tlps.first.eof,
                       keep: 8'hFF,
                       first_be: first_be,
                       last_be: last_be});
      frq_tlps.deq;
   endrule

   // rq_mdw contains last DW
   rule rl_rq_data_a (rq_mdw matches tagged Valid .dw &&& (rq_dwcount == 1));
      rq_dwcount <= 0;
      rq_mdw <= tagged Invalid;
   endrule

   // rq_mdw Valid, and there are more DWs.
   rule rl_rq_data_b(rq_mdw matches tagged Valid .dw &&& (rq_dwcount != 1));

   endrule

   // rq_mdw Invalid, and there are more DWs.
   rule rl_rq_data_c(rq_mdw matches tagged Invalid &&& (rq_dwcount != 1));

   endrule

   // The PCIE endpoint is processing Gen3 descriptors at 250MHz. The
   // AXI bridge is accepting TLPData#(16)s at 250 MHz. The
   // conversion uses half of Gen3 descriptor.
   //mkConnection(tlp8, gb.tlp, clocked_by portalClock, reset_by portalReset);

   let portalClock = (mainClockPeriod == pcieClockPeriod) ? pcieClock250 : mainClock;
   let portalReset = (mainClockPeriod == pcieClockPeriod) ? pcieReset250 : mainReset;

   interface Server tlpr;
      interface request = toPut(frq);
      interface response = toGet(frc);
   endinterface
   interface Server tlpc;
      interface request = toPut(fcc);
      interface response = toGet(fcq);
   endinterface
   interface pcie    = pcie_ep.pci_exp;
   interface Pcie3wrapUser user = pcie_ep.user;
   interface PciewrapPipe pipe = pcie_ep.pipe;
   interface PciewrapCommon common= pcie_ep.common;
   interface Clock epPcieClock = pcieClock250;
   interface Reset epPcieReset = pcieReset250;
   interface Clock epPortalClock = portalClock;
   interface Reset epPortalReset = portalReset;
   interface Clock epDerivedClock = derivedClock;
   interface Reset epDerivedReset = derivedReset;
endmodule: mkPcieEndpointX7

endpackage: Pcie3EndpointX7


Integer iid -> Bit#(4) mmuid

Integer iid -> Bit#(4) mmuid

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

// BSV Libraries
import RegFile::*;
import FIFO::*;
import FIFOF::*;
import Vector::*;
import GetPut::*;
import Connectable::*;
import BRAMFIFO::*;
import BRAM::*;
import Probe::*;

import ConnectalBram::*;
import MemTypes::*;
import StmtFSM::*;
import ClientServer::*;
import ConnectalMemory::*;
import ConnectalCompletionBuffer::*;
import SimDma::*;

`include "ConnectalProjectConfig.bsv"

typedef 32 MaxNumSGLists;
typedef Bit#(TLog#(MaxNumSGLists)) SGListId;
typedef 12 SGListPageShift0;
typedef 16 SGListPageShift4;
typedef 20 SGListPageShift8;
typedef 24 SGListPageShift12;
typedef Bit#(TLog#(MaxNumSGLists)) RegionsIdx;

typedef 8 IndexWidth;

typedef struct {
   SGListId             id;
   Bit#(MemOffsetSize) off;
} AddrTransRequest deriving (Eq,Bits,FShow);

typedef struct {
   DmaErrorType    error;
   Bit#(addrWidth) physAddr;
} AddrTransResponse#(numeric type addrWidth) deriving (Eq,Bits,FShow);

interface MMU#(numeric type addrWidth);
   interface MMURequest request;
   interface Vector#(2,Server#(AddrTransRequest,AddrTransResponse#(addrWidth))) addr;
endinterface

module mkSimpleMMU#(Bit#(4) mmuid, Bool hostMapped, MMUIndication mmuIndication)(MMU#(addrWidth))
   provisos (Add#(b__,8,addrWidth)
	     ,Add#(c__,24,addrWidth)
	     ,Bits#(AddrTransResponse#(addrWidth),d__)
	     );
   Vector#(2,FIFOF#(AddrTransRequest)) reqFifos <- replicateM(mkFIFOF());
   Vector#(2,FIFOF#(AddrTransResponse#(addrWidth))) respFifos <- replicateM(mkFIFOF());

   let probePhysAddr <- mkProbe();

   for (Integer i = 0; i < 2; i = i + 1) begin
      rule rl_process_req;
	 let req <- toGet(reqFifos[i]).get();
	 Bit#(addrWidth) physAddr = extend(req.off[23:0]);
	 physAddr[31:24] = extend(req.id);
	 probePhysAddr <= physAddr;
	 respFifos[i].enq(AddrTransResponse { error: DmaErrorNone, physAddr: physAddr });
      endrule
   end

   function Server#(AddrTransRequest,AddrTransResponse#(addrWidth)) genServer(Integer i);
      return (interface Server#(AddrTransRequest,AddrTransResponse#(addrWidth));
		 interface request = toPut(reqFifos[i]);
	 interface response = toGet(respFifos[i]);
	 endinterface);
   endfunction

   interface MMURequest request;
      method Action idRequest(SpecialTypeForSendingFd fd);
	 mmuIndication.idResponse(0);
      endmethod
      method Action idReturn(Bit#(32) sglId);
	 mmuIndication.idResponse(sglId);
      endmethod
      method Action region(Bit#(32) pointer, Bit#(64) barr12, Bit#(32) index12, Bit#(64) barr8, Bit#(32) index8, Bit#(64) barr4, Bit#(32) index4, Bit#(64) barr0, Bit#(32) index0);
	 mmuIndication.configResp(extend(pointer));
      endmethod
      method Action sglist(Bit#(32) pointer, Bit#(32) pointerIndex, Bit#(64) addr,  Bit#(32) len);
	 // no response
      endmethod
      method Action setInterface(Bit#(32) interfaceId, Bit#(32) sglId);
	 /* this method is only implemented in s/w responders */
      endmethod
   endinterface
   interface Vector addr = genWith(genServer);
endmodule

typedef struct {
   DmaErrorType error;
   Bit#(3) pageSize;
   Bit#(SGListPageShift12) value;
} Offset deriving (Eq,Bits,FShow);

typedef Bit#(TSub#(MemOffsetSize,SGListPageShift0)) Page0;
typedef Bit#(TSub#(MemOffsetSize,SGListPageShift4)) Page4;
typedef Bit#(TSub#(MemOffsetSize,SGListPageShift8)) Page8;
typedef Bit#(TSub#(MemOffsetSize,SGListPageShift12)) Page12;

typedef struct {
   Bit#(TSub#(MemOffsetSize,SGListPageShift0)) barrier;
   Bit#(IndexWidth) idxOffset;
   } SingleRegion deriving (Eq,Bits,FShow);

typedef struct {
   SingleRegion reg12;
   SingleRegion reg8;
   SingleRegion reg4;
   SingleRegion reg0;
   } Region deriving (Eq,Bits,FShow);

typedef struct {DmaErrorType errorType;
		Bit#(32) pref;
		Bit#(MemOffsetSize) off;
   } DmaError deriving (Bits);

typedef struct {
   Bool cond12;
   Bool cond8;
   Bool cond4;
   Bool cond0;
   Bit#(IndexWidth) idxOffset12;
   Bit#(IndexWidth) idxOffset8;
   Bit#(IndexWidth) idxOffset4;
   Bit#(IndexWidth) idxOffset0;
   AddrTransRequest req;
   } Stage3Params deriving (Bits);

typedef struct {
   Offset off;
   Bit#(IndexWidth) pbase;
   Bit#(IndexWidth) idxOffset;
   SGListId ptr;
   } Stage4Params deriving (Bits);

// the address translation servers (addr[0], addr[1]) have a latency of 8 and are fully pipelined
module mkMMU#(Bit#(4) mmuid, Bool hostMapped, MMUIndication mmuIndication)(MMU#(addrWidth))
   provisos(Log#(MaxNumSGLists, listIdxSize),
	    Add#(listIdxSize,8, entryIdxSize),
	    Add#(a__,addrWidth,MemOffsetSize));

   let verbose = False;
   TagGen#(MaxNumSGLists) sglId_gen <- mkTagGen();
   rule complete_sglId_gen;
      let __x <- sglId_gen.complete;
   endrule

   // for simulators
   SimDma#(32) simDma <- mkSimDma();

   // stage 0 (latency == 1)
   Vector#(2, FIFO#(AddrTransRequest)) incomingReqs <- replicateM(mkFIFO);

   // stage 1 (latency == 2)
   BRAM_Configure bramConfig = defaultValue;
   bramConfig.latency        = 2;
   BRAM2Port#(RegionsIdx, Maybe#(Region)) regall <- ConnectalBram::mkBRAM2Server(bramConfig);
   Vector#(2,FIFOF#(AddrTransRequest))          reqs0 <- replicateM(mkSizedFIFOF(3));

   // stage 2 (latency == 1)
   Vector#(2, FIFOF#(Stage3Params)) stage3Params <- replicateM(mkFIFOF);

   // stage 3 (latency == 1)
   Vector#(2, FIFOF#(Stage4Params)) stage4Params <- replicateM(mkFIFOF);

   // stage 4 (latency == 2)
   BRAM2Port#(Bit#(entryIdxSize),Page0) translationTable <- ConnectalBram::mkBRAM2Server(bramConfig);
   Vector#(2,FIFOF#(Offset))           offs1 <- replicateM(mkSizedFIFOF(3));

   // stage 4 (latnecy == 1)
   Vector#(2,FIFOF#(AddrTransResponse#(addrWidth))) pageResponseFifos <- replicateM(mkFIFOF);

   FIFO#(DmaError) dmaErrorFifo <- mkFIFO();
   Vector#(2,FIFO#(DmaError)) dmaErrorFifos <- replicateM(mkFIFO());
   for (Integer i = 0; i < 2; i = i + 1)
      mkConnection(toGet(dmaErrorFifos[i]), toPut(dmaErrorFifo));
   rule dmaError;
      let error <- toGet(dmaErrorFifo).get();
      mmuIndication.error(extend(pack(error.errorType)), error.pref, extend(error.off), extend(mmuid));
   endrule

   let page_shift0 = fromInteger(valueOf(SGListPageShift0));
   let page_shift4 = fromInteger(valueOf(SGListPageShift4));
   let page_shift8 = fromInteger(valueOf(SGListPageShift8));
   let page_shift12 = fromInteger(valueOf(SGListPageShift12));

   function BRAMServer#(a,b) portsel(BRAM2Port#(a,b) x, Integer i);
      if(i==0) return x.portA;
      else return x.portB;
   endfunction

   for (Integer i = 0; i < 2; i=i+1) begin
      rule stage1;  // first read in the address cutoff values between regions
	 AddrTransRequest req <- toGet(incomingReqs[i]).get();
	 portsel(regall, i).request.put(BRAMRequest{write:False, responseOnWrite:False,
            address:truncate(req.id), datain:?});
	 reqs0[i].enq(req);
      endrule

      // pipeline the address lookup
      rule stage2; // Now compare address cutoffs with requested offset
	 AddrTransRequest req <- toGet(reqs0[i]).get;
	 Maybe#(Region) m_regionall <- portsel(regall,i).response.get;

	 case (m_regionall) matches
	    tagged Valid .regionall: begin
               Page0 off0 = truncate(req.off >> valueOf(SGListPageShift0));
               Page4 off4 = truncate(req.off >> valueOf(SGListPageShift4));
               Page8 off8 = truncate(req.off >> valueOf(SGListPageShift8));
               Page12 off12 = truncate(req.off >> valueOf(SGListPageShift12));
	       let cond12 = off12 < truncate(regionall.reg12.barrier);
	       let cond8 = off8 < truncate(regionall.reg8.barrier);
	       let cond4 = off4 < truncate(regionall.reg4.barrier);
	       let cond0 = off0 < regionall.reg0.barrier;

	       if (verbose) $display("mkMMU::stage2: id=%d off=%h (%h %h %h) (%h %h %h)", req.id, req.off,
				     regionall.reg8.barrier, regionall.reg4.barrier, regionall.reg0.barrier,
				     off8, off4, off0);

	       stage3Params[i].enq(Stage3Params {cond12: cond12, cond8: cond8, cond4: cond4, cond0: cond0,
						 idxOffset12: regionall.reg12.idxOffset,idxOffset8: regionall.reg8.idxOffset,
						 idxOffset4: regionall.reg4.idxOffset, idxOffset0: regionall.reg0.idxOffset,
						 req: req });
	    end
	    tagged Invalid:
	       dmaErrorFifos[0].enq(DmaError { errorType: DmaErrorSGLIdInvalid, pref: extend(req.id), off:req.off });
	 endcase
      endrule
      rule stage3; // Based on results of comparision, select a region, putting it into 'o.pageSize'.  idxOffset holds offset in sglist table of relevant entry
	 let params <- toGet(stage3Params[i]).get();
	 AddrTransRequest req = params.req;
	 Offset o = Offset { error: DmaErrorNone, pageSize: 0, value: truncate(req.off)};
	 Bit#(IndexWidth) pbase = 0;
	 Bit#(IndexWidth) idxOffset = 0;

	 if (params.cond12) begin
	    if (verbose) $display("mkMMU::request: req.id=%h req.off=%h", req.id, req.off);
	    o.pageSize = 4;
	    pbase = truncate(req.off>>page_shift12);
	    idxOffset = params.idxOffset12;
	 end
	 else if (params.cond8) begin
	    if (verbose) $display("mkMMU::request: req.id=%h req.off=%h", req.id, req.off);
	    o.pageSize = 3;
	    pbase = truncate(req.off>>page_shift8);
	    idxOffset = params.idxOffset8;
	 end
	 else if (params.cond4) begin
	    if (verbose) $display("mkMMU::request: req.id=%h req.off=%h", req.id, req.off);
	    o.pageSize = 2;
	    pbase = truncate(req.off>>page_shift4);
	    idxOffset = params.idxOffset4;
	 end
	 else if (params.cond0) begin
	    if (verbose) $display("mkMMU::request: req.id=%h req.off=%h", req.id, req.off);
	    o.pageSize = 1;
	    pbase = truncate(req.off>>page_shift0);
	    idxOffset = params.idxOffset0;
	 end
	 stage4Params[i].enq(Stage4Params { off: o, pbase: pbase, idxOffset: idxOffset, ptr: req.id });
      endrule
      (* descending_urgency = "stage2, stage4" *)
      rule stage4; // Read relevant sglist entry
	 let params <- toGet(stage4Params[i]).get();
	 let off = params.off;
	 let pbase = params.pbase;
	 let idxOffset = params.idxOffset;
	 let ptr = params.ptr;
	 Bit#(IndexWidth) p = pbase + idxOffset;
	 if (off.pageSize == 0) begin
	    if (verbose) $display("mkMMU::addr[%d].request.put: ERROR   ptr=%h off=%h\n", i, ptr, off);
	    dmaErrorFifos[1].enq(DmaError { errorType: DmaErrorOffsetOutOfRange, pref: extend(ptr), off:extend(off.value) });
	    off.error = DmaErrorOffsetOutOfRange;
	    p         = 0; // FIXME
	 end

	 if (verbose) $display("mkMMU::translationTable[%d].read %h", i, {ptr,p});
	 portsel(translationTable, i).request.put(BRAMRequest{write:False, responseOnWrite:False,
							      address:{ptr,p}, datain:?});
	 offs1[i].enq(off);

      endrule
      rule stage5; // Concatenate page base address from sglist entry with LSB offset bits from request and return
	 Page0 page <- portsel(translationTable, i).response.get;
	 let offset <- toGet(offs1[i]).get();
	 if (verbose) $display("mkMMU::p ages[%d].response page=%h offset=%h", i, page, offset);
	 Bit#(MemOffsetSize) rv = ?;
	 Page4 b4 = truncate(page);
	 Page8 b8 = truncate(page);
	 Page12 b12 = truncate(page);
	 case (offset.pageSize)
	    1: rv = {page,truncate(offset.value)};
	    2: rv = {b4,truncate(offset.value)};
	    3: rv = {b8,truncate(offset.value)};
	    4: rv = {b12,truncate(offset.value)};
	 endcase
	 pageResponseFifos[i].enq(AddrTransResponse { error: offset.error, physAddr: truncate(rv) });
      endrule
   end

   FIFO#(SGListId) configRespFifo <- mkFIFO;
   rule sendConfigResp;
      let ptr <- toGet(configRespFifo).get();
      mmuIndication.configResp(extend(ptr));
   endrule

   // given that the BRAM is faster than the connection from software, I see no need for a SizedBRAMFIFOF here. -Jamey
   FIFOF#(Bit#(32)) idReturnFifo <- mkFIFOF();
   rule idReturnRule;
      let sglId <- toGet(idReturnFifo).get;
      sglId_gen.returnTag(truncate(sglId));
      portsel(regall, 1).request.put(BRAMRequest{write:True, responseOnWrite:False, address: truncate(sglId), datain: tagged Invalid });
      $display("idReturn %d", sglId);
   endrule

   function Server#(AddrTransRequest,AddrTransResponse#(addrWidth)) addrServer(Integer i);
   return
      (interface Server#(AddrTransRequest,Bit#(addrWidth));
	  interface Put request;
	     method Action put(AddrTransRequest req);
		incomingReqs[i].enq(req);
	     endmethod
	  endinterface
	  interface Get response;
	     method ActionValue#(AddrTransResponse#(addrWidth)) get();
		let rv <- toGet(pageResponseFifos[i]).get();
`ifdef SIMULATION
		rv.physAddr = rv.physAddr | (extend(mmuid)<<valueOf(addrWidth)-3);
`endif
		return rv;
	     endmethod
	  endinterface
       endinterface);
   endfunction

   interface MMURequest request;
   method Action idRequest(SpecialTypeForSendingFd fd);
      let nextId <- sglId_gen.getTag;
      let resp = (extend(mmuid) << 16) | extend(nextId);
      if (verbose) $display("mkMMU::idRequest %d", fd);
      if (hostMapped) begin
	 let va <- simDma.initfd(resp, fd);
      end
      mmuIndication.idResponse(resp);
   endmethod
   method Action idReturn(Bit#(32) sglId);
      idReturnFifo.enq(sglId);
      if (hostMapped)
	 simDma.idreturn(sglId);
   endmethod
   method Action region(Bit#(32) pointer, Bit#(64) barr12, Bit#(32) index12, Bit#(64) barr8, Bit#(32) index8, Bit#(64) barr4, Bit#(32) index4, Bit#(64) barr0, Bit#(32) index0);
      portsel(regall, 1).request.put(BRAMRequest{write:True, responseOnWrite:False,
          address: truncate(pointer), datain: tagged Valid Region{
             reg12: SingleRegion{barrier: truncate(barr12), idxOffset: truncate(index12)},
             reg8: SingleRegion{barrier: truncate(barr8), idxOffset: truncate(index8)},
             reg4: SingleRegion{barrier: truncate(barr4), idxOffset: truncate(index4)},
             reg0: SingleRegion{barrier: truncate(barr0), idxOffset: truncate(index0)}} });
      if (verbose) $display("mkMMU::region pointer=%d barr12=%h barr8=%h barr4=%h barr0=%h", pointer, barr12, barr8, barr4, barr0);
      configRespFifo.enq(truncate(pointer));
   endmethod

   method Action sglist(Bit#(32) pointer, Bit#(32) pointerIndex, Bit#(64) addr,  Bit#(32) len);
         if (extend(mmuid) != pointer[31:16]) begin
	    $display("mkMMU::sglist ERROR");
	    $finish();
	 end
	 if(hostMapped)
	    let va <- simDma.init({0,pointer[31:16]}, {0,pointer[15:0]}, len);
         Bit#(IndexWidth) ind = truncate(pointerIndex);
	 portsel(translationTable, 0).request.put(BRAMRequest{write:True, responseOnWrite:False,
             address:{truncate(pointer),ind}, datain:truncate(addr)});
         if (verbose) $display("mkMMU::sglist pointer=%d pointerIndex=%d addr=%d len=%d", pointer, pointerIndex, addr, len);
   endmethod
   method Action setInterface(Bit#(32) interfaceId, Bit#(32) sglId);
       /* this method is only implemented in s/w responders */
   endmethod
   endinterface
   interface addr = genWith(addrServer);

endmodule

interface ArbitratedMMU#(numeric type addrWidth, numeric type numServers);
   interface Vector#(numServers,Server#(AddrTransRequest,AddrTransResponse#(addrWidth))) servers;
endinterface

module mkArbitratedMMU#(Server#(AddrTransRequest,AddrTransResponse#(addrWidth)) server) (ArbitratedMMU#(addrWidth,numServers));

   FIFOF#(Bit#(TAdd#(1,TLog#(numServers)))) tokFifo <- mkSizedFIFOF(9);
   Reg#(Bit#(TLog#(numServers))) arb <- mkReg(0);

   // this is a very crude arbiter.  something more sophisticated may be required (mdk)
   rule inc_arb;
      arb <= arb+1;
   endrule

   function Server#(AddrTransRequest,AddrTransResponse#(addrWidth)) arbitratedServer(Integer i);
   return
      (interface Server#(AddrTransRequest,AddrTransResponse#(addrWidth));
	  interface Put request;
	     method Action put(AddrTransRequest req) if (arb == fromInteger(i));
		tokFifo.enq(fromInteger(i));
		server.request.put(req);
	     endmethod
	  endinterface
	  interface Get response;
	     method ActionValue#(AddrTransResponse#(addrWidth)) get() if (tokFifo.first == fromInteger(i));
		tokFifo.deq;
		let rv <- server.response.get;
		return rv;
	     endmethod
	  endinterface
       endinterface);
   endfunction

   interface servers = genWith(arbitratedServer);

endmodule

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

// BSV Libraries
import RegFile::*;
import FIFO::*;
import FIFOF::*;
import Vector::*;
import GetPut::*;
import Connectable::*;
import BRAMFIFO::*;
import BRAM::*;
import Probe::*;

import ConnectalBram::*;
import MemTypes::*;
import StmtFSM::*;
import ClientServer::*;
import ConnectalMemory::*;
import ConnectalCompletionBuffer::*;
import SimDma::*;

`include "ConnectalProjectConfig.bsv"

typedef 32 MaxNumSGLists;
typedef Bit#(TLog#(MaxNumSGLists)) SGListId;
typedef 12 SGListPageShift0;
typedef 16 SGListPageShift4;
typedef 20 SGListPageShift8;
typedef 24 SGListPageShift12;
typedef Bit#(TLog#(MaxNumSGLists)) RegionsIdx;

typedef 8 IndexWidth;

typedef struct {
   SGListId             id;
   Bit#(MemOffsetSize) off;
} AddrTransRequest deriving (Eq,Bits,FShow);

typedef struct {
   DmaErrorType    error;
   Bit#(addrWidth) physAddr;
} AddrTransResponse#(numeric type addrWidth) deriving (Eq,Bits,FShow);

interface MMU#(numeric type addrWidth);
   interface MMURequest request;
   interface Vector#(2,Server#(AddrTransRequest,AddrTransResponse#(addrWidth))) addr;
endinterface

module mkSimpleMMU#(Integer iid, Bool hostMapped, MMUIndication mmuIndication)(MMU#(addrWidth))
   provisos (Add#(b__,8,addrWidth)
	     ,Add#(c__,24,addrWidth)
	     ,Bits#(AddrTransResponse#(addrWidth),d__)
	     );
   Vector#(2,FIFOF#(AddrTransRequest)) reqFifos <- replicateM(mkFIFOF());
   Vector#(2,FIFOF#(AddrTransResponse#(addrWidth))) respFifos <- replicateM(mkFIFOF());

   let probePhysAddr <- mkProbe();

   for (Integer i = 0; i < 2; i = i + 1) begin
      rule rl_process_req;
	 let req <- toGet(reqFifos[i]).get();
	 Bit#(addrWidth) physAddr = extend(req.off[23:0]);
	 physAddr[31:24] = extend(req.id);
	 probePhysAddr <= physAddr;
	 respFifos[i].enq(AddrTransResponse { error: DmaErrorNone, physAddr: physAddr });
      endrule
   end

   function Server#(AddrTransRequest,AddrTransResponse#(addrWidth)) genServer(Integer i);
      return (interface Server#(AddrTransRequest,AddrTransResponse#(addrWidth));
		 interface request = toPut(reqFifos[i]);
	 interface response = toGet(respFifos[i]);
	 endinterface);
   endfunction

   interface MMURequest request;
      method Action idRequest(SpecialTypeForSendingFd fd);
	 mmuIndication.idResponse(0);
      endmethod
      method Action idReturn(Bit#(32) sglId);
	 mmuIndication.idResponse(sglId);
      endmethod
      method Action region(Bit#(32) pointer, Bit#(64) barr12, Bit#(32) index12, Bit#(64) barr8, Bit#(32) index8, Bit#(64) barr4, Bit#(32) index4, Bit#(64) barr0, Bit#(32) index0);
	 mmuIndication.configResp(extend(pointer));
      endmethod
      method Action sglist(Bit#(32) pointer, Bit#(32) pointerIndex, Bit#(64) addr,  Bit#(32) len);
	 // no response
      endmethod
      method Action setInterface(Bit#(32) interfaceId, Bit#(32) sglId);
	 /* this method is only implemented in s/w responders */
      endmethod
   endinterface
   interface Vector addr = genWith(genServer);
endmodule

typedef struct {
   DmaErrorType error;
   Bit#(3) pageSize;
   Bit#(SGListPageShift12) value;
} Offset deriving (Eq,Bits,FShow);

typedef Bit#(TSub#(MemOffsetSize,SGListPageShift0)) Page0;
typedef Bit#(TSub#(MemOffsetSize,SGListPageShift4)) Page4;
typedef Bit#(TSub#(MemOffsetSize,SGListPageShift8)) Page8;
typedef Bit#(TSub#(MemOffsetSize,SGListPageShift12)) Page12;

typedef struct {
   Bit#(TSub#(MemOffsetSize,SGListPageShift0)) barrier;
   Bit#(IndexWidth) idxOffset;
   } SingleRegion deriving (Eq,Bits,FShow);

typedef struct {
   SingleRegion reg12;
   SingleRegion reg8;
   SingleRegion reg4;
   SingleRegion reg0;
   } Region deriving (Eq,Bits,FShow);

typedef struct {DmaErrorType errorType;
		Bit#(32) pref;
		Bit#(MemOffsetSize) off;
   } DmaError deriving (Bits);

typedef struct {
   Bool cond12;
   Bool cond8;
   Bool cond4;
   Bool cond0;
   Bit#(IndexWidth) idxOffset12;
   Bit#(IndexWidth) idxOffset8;
   Bit#(IndexWidth) idxOffset4;
   Bit#(IndexWidth) idxOffset0;
   AddrTransRequest req;
   } Stage3Params deriving (Bits);

typedef struct {
   Offset off;
   Bit#(IndexWidth) pbase;
   Bit#(IndexWidth) idxOffset;
   SGListId ptr;
   } Stage4Params deriving (Bits);

// the address translation servers (addr[0], addr[1]) have a latency of 8 and are fully pipelined
module mkMMU#(Integer iid, Bool hostMapped, MMUIndication mmuIndication)(MMU#(addrWidth))
   provisos(Log#(MaxNumSGLists, listIdxSize),
	    Add#(listIdxSize,8, entryIdxSize),
	    Add#(a__,addrWidth,MemOffsetSize));

   let verbose = False;
   TagGen#(MaxNumSGLists) sglId_gen <- mkTagGen();
   rule complete_sglId_gen;
      let __x <- sglId_gen.complete;
   endrule

   // for simulators
   SimDma#(32) simDma <- mkSimDma();

   // stage 0 (latency == 1)
   Vector#(2, FIFO#(AddrTransRequest)) incomingReqs <- replicateM(mkFIFO);

   // stage 1 (latency == 2)
   BRAM_Configure bramConfig = defaultValue;
   bramConfig.latency        = 2;
   BRAM2Port#(RegionsIdx, Maybe#(Region)) regall <- ConnectalBram::mkBRAM2Server(bramConfig);
   Vector#(2,FIFOF#(AddrTransRequest))          reqs0 <- replicateM(mkSizedFIFOF(3));

   // stage 2 (latency == 1)
   Vector#(2, FIFOF#(Stage3Params)) stage3Params <- replicateM(mkFIFOF);

   // stage 3 (latency == 1)
   Vector#(2, FIFOF#(Stage4Params)) stage4Params <- replicateM(mkFIFOF);

   // stage 4 (latency == 2)
   BRAM2Port#(Bit#(entryIdxSize),Page0) translationTable <- ConnectalBram::mkBRAM2Server(bramConfig);
   Vector#(2,FIFOF#(Offset))           offs1 <- replicateM(mkSizedFIFOF(3));

   // stage 4 (latnecy == 1)
   Vector#(2,FIFOF#(AddrTransResponse#(addrWidth))) pageResponseFifos <- replicateM(mkFIFOF);

   FIFO#(DmaError) dmaErrorFifo <- mkFIFO();
   Vector#(2,FIFO#(DmaError)) dmaErrorFifos <- replicateM(mkFIFO());
   for (Integer i = 0; i < 2; i = i + 1)
      mkConnection(toGet(dmaErrorFifos[i]), toPut(dmaErrorFifo));
   rule dmaError;
      let error <- toGet(dmaErrorFifo).get();
      mmuIndication.error(extend(pack(error.errorType)), error.pref, extend(error.off), fromInteger(iid));
   endrule

   let page_shift0 = fromInteger(valueOf(SGListPageShift0));
   let page_shift4 = fromInteger(valueOf(SGListPageShift4));
   let page_shift8 = fromInteger(valueOf(SGListPageShift8));
   let page_shift12 = fromInteger(valueOf(SGListPageShift12));

   function BRAMServer#(a,b) portsel(BRAM2Port#(a,b) x, Integer i);
      if(i==0) return x.portA;
      else return x.portB;
   endfunction

   for (Integer i = 0; i < 2; i=i+1) begin
      rule stage1;  // first read in the address cutoff values between regions
	 AddrTransRequest req <- toGet(incomingReqs[i]).get();
	 portsel(regall, i).request.put(BRAMRequest{write:False, responseOnWrite:False,
            address:truncate(req.id), datain:?});
	 reqs0[i].enq(req);
      endrule

      // pipeline the address lookup
      rule stage2; // Now compare address cutoffs with requested offset
	 AddrTransRequest req <- toGet(reqs0[i]).get;
	 Maybe#(Region) m_regionall <- portsel(regall,i).response.get;

	 case (m_regionall) matches
	    tagged Valid .regionall: begin
               Page0 off0 = truncate(req.off >> valueOf(SGListPageShift0));
               Page4 off4 = truncate(req.off >> valueOf(SGListPageShift4));
               Page8 off8 = truncate(req.off >> valueOf(SGListPageShift8));
               Page12 off12 = truncate(req.off >> valueOf(SGListPageShift12));
	       let cond12 = off12 < truncate(regionall.reg12.barrier);
	       let cond8 = off8 < truncate(regionall.reg8.barrier);
	       let cond4 = off4 < truncate(regionall.reg4.barrier);
	       let cond0 = off0 < regionall.reg0.barrier;

	       if (verbose) $display("mkMMU::stage2: id=%d off=%h (%h %h %h) (%h %h %h)", req.id, req.off,
				     regionall.reg8.barrier, regionall.reg4.barrier, regionall.reg0.barrier,
				     off8, off4, off0);

	       stage3Params[i].enq(Stage3Params {cond12: cond12, cond8: cond8, cond4: cond4, cond0: cond0,
						 idxOffset12: regionall.reg12.idxOffset,idxOffset8: regionall.reg8.idxOffset,
						 idxOffset4: regionall.reg4.idxOffset, idxOffset0: regionall.reg0.idxOffset,
						 req: req });
	    end
	    tagged Invalid:
	       dmaErrorFifos[0].enq(DmaError { errorType: DmaErrorSGLIdInvalid, pref: extend(req.id), off:req.off });
	 endcase
      endrule
      rule stage3; // Based on results of comparision, select a region, putting it into 'o.pageSize'.  idxOffset holds offset in sglist table of relevant entry
	 let params <- toGet(stage3Params[i]).get();
	 AddrTransRequest req = params.req;
	 Offset o = Offset { error: DmaErrorNone, pageSize: 0, value: truncate(req.off)};
	 Bit#(IndexWidth) pbase = 0;
	 Bit#(IndexWidth) idxOffset = 0;

	 if (params.cond12) begin
	    if (verbose) $display("mkMMU::request: req.id=%h req.off=%h", req.id, req.off);
	    o.pageSize = 4;
	    pbase = truncate(req.off>>page_shift12);
	    idxOffset = params.idxOffset12;
	 end
	 else if (params.cond8) begin
	    if (verbose) $display("mkMMU::request: req.id=%h req.off=%h", req.id, req.off);
	    o.pageSize = 3;
	    pbase = truncate(req.off>>page_shift8);
	    idxOffset = params.idxOffset8;
	 end
	 else if (params.cond4) begin
	    if (verbose) $display("mkMMU::request: req.id=%h req.off=%h", req.id, req.off);
	    o.pageSize = 2;
	    pbase = truncate(req.off>>page_shift4);
	    idxOffset = params.idxOffset4;
	 end
	 else if (params.cond0) begin
	    if (verbose) $display("mkMMU::request: req.id=%h req.off=%h", req.id, req.off);
	    o.pageSize = 1;
	    pbase = truncate(req.off>>page_shift0);
	    idxOffset = params.idxOffset0;
	 end
	 stage4Params[i].enq(Stage4Params { off: o, pbase: pbase, idxOffset: idxOffset, ptr: req.id });
      endrule
      (* descending_urgency = "stage2, stage4" *)
      rule stage4; // Read relevant sglist entry
	 let params <- toGet(stage4Params[i]).get();
	 let off = params.off;
	 let pbase = params.pbase;
	 let idxOffset = params.idxOffset;
	 let ptr = params.ptr;
	 Bit#(IndexWidth) p = pbase + idxOffset;
	 if (off.pageSize == 0) begin
	    if (verbose) $display("mkMMU::addr[%d].request.put: ERROR   ptr=%h off=%h\n", i, ptr, off);
	    dmaErrorFifos[1].enq(DmaError { errorType: DmaErrorOffsetOutOfRange, pref: extend(ptr), off:extend(off.value) });
	    off.error = DmaErrorOffsetOutOfRange;
	    p         = 0; // FIXME
	 end

	 if (verbose) $display("mkMMU::translationTable[%d].read %h", i, {ptr,p});
	 portsel(translationTable, i).request.put(BRAMRequest{write:False, responseOnWrite:False,
							      address:{ptr,p}, datain:?});
	 offs1[i].enq(off);

      endrule
      rule stage5; // Concatenate page base address from sglist entry with LSB offset bits from request and return
	 Page0 page <- portsel(translationTable, i).response.get;
	 let offset <- toGet(offs1[i]).get();
	 if (verbose) $display("mkMMU::p ages[%d].response page=%h offset=%h", i, page, offset);
	 Bit#(MemOffsetSize) rv = ?;
	 Page4 b4 = truncate(page);
	 Page8 b8 = truncate(page);
	 Page12 b12 = truncate(page);
	 case (offset.pageSize)
	    1: rv = {page,truncate(offset.value)};
	    2: rv = {b4,truncate(offset.value)};
	    3: rv = {b8,truncate(offset.value)};
	    4: rv = {b12,truncate(offset.value)};
	 endcase
	 pageResponseFifos[i].enq(AddrTransResponse { error: offset.error, physAddr: truncate(rv) });
      endrule
   end

   FIFO#(SGListId) configRespFifo <- mkFIFO;
   rule sendConfigResp;
      let ptr <- toGet(configRespFifo).get();
      mmuIndication.configResp(extend(ptr));
   endrule

   // given that the BRAM is faster than the connection from software, I see no need for a SizedBRAMFIFOF here. -Jamey
   FIFOF#(Bit#(32)) idReturnFifo <- mkFIFOF();
   rule idReturnRule;
      let sglId <- toGet(idReturnFifo).get;
      sglId_gen.returnTag(truncate(sglId));
      portsel(regall, 1).request.put(BRAMRequest{write:True, responseOnWrite:False, address: truncate(sglId), datain: tagged Invalid });
      $display("idReturn %d", sglId);
   endrule

   function Server#(AddrTransRequest,AddrTransResponse#(addrWidth)) addrServer(Integer i);
   return
      (interface Server#(AddrTransRequest,Bit#(addrWidth));
	  interface Put request;
	     method Action put(AddrTransRequest req);
		incomingReqs[i].enq(req);
	     endmethod
	  endinterface
	  interface Get response;
	     method ActionValue#(AddrTransResponse#(addrWidth)) get();
		let rv <- toGet(pageResponseFifos[i]).get();
`ifdef SIMULATION
		rv.physAddr = rv.physAddr | (fromInteger(iid)<<valueOf(addrWidth)-3);
`endif
		return rv;
	     endmethod
	  endinterface
       endinterface);
   endfunction

   interface MMURequest request;
   method Action idRequest(SpecialTypeForSendingFd fd);
      let nextId <- sglId_gen.getTag;
      let resp = (fromInteger(iid) << 16) | extend(nextId);
      if (verbose) $display("mkMMU::idRequest %d", fd);
      if (hostMapped) begin
	 let va <- simDma.initfd(resp, fd);
      end
      mmuIndication.idResponse(resp);
   endmethod
   method Action idReturn(Bit#(32) sglId);
      idReturnFifo.enq(sglId);
      if (hostMapped)
	 simDma.idreturn(sglId);
   endmethod
   method Action region(Bit#(32) pointer, Bit#(64) barr12, Bit#(32) index12, Bit#(64) barr8, Bit#(32) index8, Bit#(64) barr4, Bit#(32) index4, Bit#(64) barr0, Bit#(32) index0);
      portsel(regall, 1).request.put(BRAMRequest{write:True, responseOnWrite:False,
          address: truncate(pointer), datain: tagged Valid Region{
             reg12: SingleRegion{barrier: truncate(barr12), idxOffset: truncate(index12)},
             reg8: SingleRegion{barrier: truncate(barr8), idxOffset: truncate(index8)},
             reg4: SingleRegion{barrier: truncate(barr4), idxOffset: truncate(index4)},
             reg0: SingleRegion{barrier: truncate(barr0), idxOffset: truncate(index0)}} });
      if (verbose) $display("mkMMU::region pointer=%d barr12=%h barr8=%h barr4=%h barr0=%h", pointer, barr12, barr8, barr4, barr0);
      configRespFifo.enq(truncate(pointer));
   endmethod

   method Action sglist(Bit#(32) pointer, Bit#(32) pointerIndex, Bit#(64) addr,  Bit#(32) len);
         if (fromInteger(iid) != pointer[31:16]) begin
	    $display("mkMMU::sglist ERROR");
	    $finish();
	 end
	 if(hostMapped)
	    let va <- simDma.init({0,pointer[31:16]}, {0,pointer[15:0]}, len);
         Bit#(IndexWidth) ind = truncate(pointerIndex);
	 portsel(translationTable, 0).request.put(BRAMRequest{write:True, responseOnWrite:False,
             address:{truncate(pointer),ind}, datain:truncate(addr)});
         if (verbose) $display("mkMMU::sglist pointer=%d pointerIndex=%d addr=%d len=%d", pointer, pointerIndex, addr, len);
   endmethod
   method Action setInterface(Bit#(32) interfaceId, Bit#(32) sglId);
       /* this method is only implemented in s/w responders */
   endmethod
   endinterface
   interface addr = genWith(addrServer);

endmodule

interface ArbitratedMMU#(numeric type addrWidth, numeric type numServers);
   interface Vector#(numServers,Server#(AddrTransRequest,AddrTransResponse#(addrWidth))) servers;
endinterface

module mkArbitratedMMU#(Server#(AddrTransRequest,AddrTransResponse#(addrWidth)) server) (ArbitratedMMU#(addrWidth,numServers));

   FIFOF#(Bit#(TAdd#(1,TLog#(numServers)))) tokFifo <- mkSizedFIFOF(9);
   Reg#(Bit#(TLog#(numServers))) arb <- mkReg(0);

   // this is a very crude arbiter.  something more sophisticated may be required (mdk)
   rule inc_arb;
      arb <= arb+1;
   endrule

   function Server#(AddrTransRequest,AddrTransResponse#(addrWidth)) arbitratedServer(Integer i);
   return
      (interface Server#(AddrTransRequest,AddrTransResponse#(addrWidth));
	  interface Put request;
	     method Action put(AddrTransRequest req) if (arb == fromInteger(i));
		tokFifo.enq(fromInteger(i));
		server.request.put(req);
	     endmethod
	  endinterface
	  interface Get response;
	     method ActionValue#(AddrTransResponse#(addrWidth)) get() if (tokFifo.first == fromInteger(i));
		tokFifo.deq;
		let rv <- server.response.get;
		return rv;
	     endmethod
	  endinterface
       endinterface);
   endfunction

   interface servers = genWith(arbitratedServer);

endmodule