composition of mkFunnelNode

composition of mkFunnelNode

// Copyright (c) 2014 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import FIFOF::*;
import GetPut::*;
import Connectable::*;
import Vector::*;
import BuildVector::*;
import MIMO::*;
import DefaultValue::*;
import Gearbox::*;
import Clocks::*;

typedef Tuple3#(x,x,x) Tripple#(type x);
typedef Tuple2#(x,x)   Pair#(type x);

interface PipeIn#(type a);
   method Action enq(a v);
   method Bool notFull();
endinterface

interface PipeOut#(type a);
   method a first();
   method Action deq();
   method Bool notEmpty();
endinterface

function Bool pipeInNotFull(PipeIn#(a) pipein); return pipein.notFull(); endfunction
function Bool pipeOutNotEmpty(PipeOut#(a) pipein); return pipein.notEmpty(); endfunction

typeclass ToPipeIn#(type a, type b) dependencies (b determines a);
   function PipeIn#(a) toPipeIn(b in);
endtypeclass

typeclass ToPipeOut#(type a, type b) dependencies ( b determines a);
   function PipeOut#(a) toPipeOut(b in);
endtypeclass

typeclass MkPipeOut#(type a, type b) dependencies ( b determines a);
   module mkPipeOut#(b in)(PipeOut#(a));
endtypeclass

typeclass MkPipeIn#(type a, type b) dependencies ( b determines a);
   module mkPipeIn#(b in)(PipeIn#(a));
endtypeclass

instance ToPipeIn#(a, FIFOF#(a));
   function PipeIn#(a) toPipeIn(FIFOF#(a) in);
      return (interface PipeIn#(a);
		 method enq = in.enq;
		 method notFull = in.notFull;
	      endinterface);
   endfunction
endinstance

instance ToPipeOut#(a, function a pipefn());
   function PipeOut#(a) toPipeOut(function a pipefn());
      return (interface PipeOut#(a);
		 method first(); return pipefn(); endmethod
		 method Action deq(); endmethod
		 method Bool notEmpty(); return False; endmethod
	      endinterface);
   endfunction
endinstance

instance ToPipeOut#(a, Reg#(a));
   function PipeOut#(a) toPipeOut(Reg#(a) in);
      return (interface PipeOut#(a);
		 method first(); return in; endmethod
		 method Action deq(); endmethod
		 method Bool notEmpty(); return False; endmethod
	      endinterface);
   endfunction
endinstance

instance ToPipeIn#(Vector#(m, a), Gearbox#(m, n, a));
   function PipeIn#(Vector#(m, a)) toPipeIn(Gearbox#(m, n, a) in);
      return (interface PipeIn#(Vector#(m, a));
		 method enq = in.enq;
		 method notFull = in.notFull;
	      endinterface);
   endfunction
endinstance

instance ToPipeOut#(a, FIFOF#(a));
   function PipeOut#(a) toPipeOut(FIFOF#(a) in);
      return (interface PipeOut#(a);
		 method first = in.first;
		 method deq = in.deq;
		 method notEmpty = in.notEmpty;
	      endinterface);
   endfunction
endinstance

instance ToPipeOut#(Vector#(n,a), MIMO#(k,n,sz,a));
   function PipeOut#(Vector#(n,a)) toPipeOut(MIMO#(k,n,sz,a) in);
      return (interface PipeOut#(a);
		 method first = in.first;
		 method Action deq() if (in.deqReadyN(fromInteger(valueOf(n))));
		    in.deq(fromInteger(valueOf(n)));
		 endmethod
		 method Bool notEmpty();
		    return in.deqReadyN(fromInteger(valueOf(n)));
		 endmethod
	      endinterface);
   endfunction
endinstance

instance ToPipeOut#(Vector#(n, a), Gearbox#(m, n, a));
   function PipeOut#(Vector#(n, a)) toPipeOut(Gearbox#(m, n, a) in);
      return (interface PipeOut#(Vector#(n,a));
		 method first = in.first;
		 method deq = in.deq;
		 method notEmpty = in.notEmpty;
	      endinterface);
   endfunction
endinstance

instance ToPipeIn#(a, SyncFIFOIfc#(a));
   function PipeIn#(a) toPipeIn(SyncFIFOIfc#(a) in);
      return (interface PipeIn#(a);
		 method enq = in.enq;
		 method notFull = in.notFull;
	      endinterface);
   endfunction
endinstance

instance ToPipeOut#(a, SyncFIFOIfc#(a));
   function PipeOut#(a) toPipeOut(SyncFIFOIfc#(a) in);
      return (interface PipeOut#(a);
		 method first = in.first;
		 method deq = in.deq;
		 method notEmpty = in.notEmpty;
	      endinterface);
   endfunction
endinstance

instance MkPipeOut#(a, ActionValue#(a))
   provisos (Bits#(a, asz));
   module mkPipeOut#(ActionValue#(a) in)(PipeOut#(a));
      FIFOF#(a) fifo <- mkFIFOF();
      rule connect;
	 let v <- in;
	 fifo.enq(v);
      endrule
      return toPipeOut(fifo);
   endmodule
endinstance

instance MkPipeOut#(a, Get#(a))
   provisos (Bits#(a, asz));
   module mkPipeOut#(Get#(a) in)(PipeOut#(a));
      FIFOF#(a) fifo <- mkFIFOF();
      rule connect;
	 let v <- in.get();
	 fifo.enq(v);
      endrule
      return toPipeOut(fifo);
   endmodule
endinstance

instance MkPipeIn#(a, Put#(a))
   provisos (Bits#(a, asz));
   module mkPipeIn#(Put#(a) out)(PipeIn#(a));
      FIFOF#(a) fifo <- mkFIFOF();
      rule connect;
	 let v <- toGet(fifo).get;
	 out.put(v);
      endrule
      return toPipeIn(fifo);
   endmodule
endinstance

function PipeOut#(a) toCountedPipeOut(Reg#(Bit#(n)) r, PipeOut#(a) pipe);
   return (interface PipeOut#(Vector#(n,a));
	      method first = pipe.first;
	      method Action deq(); pipe.deq; r <= r + 1; endmethod
	      method notEmpty = pipe.notEmpty;
	   endinterface);
endfunction

instance ToGet #(PipeOut #(a), a);
   function Get #(a) toGet (PipeOut #(a) po);
      return (interface Get;
                 method ActionValue #(a) get ();
                    po.deq ();
                    return po.first ();
                 endmethod
              endinterface);
   endfunction
endinstance

instance ToPut #(PipeIn #(a), a);
   function Put #(a) toPut (PipeIn #(a) pi);
      return (interface Put;
		 method Action put(a v);
                    pi.enq (v);
                 endmethod
              endinterface);
   endfunction
endinstance

instance Connectable#(PipeOut#(a),Put#(a));
   module mkConnection#(PipeOut#(a) in, Put#(a) out)(Empty);
      rule connect;
	 let v = in.first;
	 in.deq();
	 out.put(v);
      endrule
   endmodule
endinstance

instance Connectable#(PipeOut#(a),PipeIn#(a));
   module mkConnection#(PipeOut#(a) in, PipeIn#(a) out)(Empty);
      rule connect;
	 let v = in.first;
	 in.deq();
	 out.enq(v);
      endrule
   endmodule
endinstance

function PipeOut#(a) unvectorPipeOut(PipeOut#(Vector#(1,a)) in);
   return (interface PipeOut#(a);
	      method first = in.first[0];
	      method deq = in.deq;
	      method notEmpty = in.notEmpty;
	   endinterface);
endfunction

function PipeOut#(Tuple2#(a,b)) zipPipeOut(PipeOut#(a) ina, PipeOut#(b) inb);
      return (interface PipeOut#(Tuple2#(a,b));
		 method Tuple2#(a,b) first(); return tuple2(ina.first, inb.first); endmethod
		 method Action deq(); ina.deq(); inb.deq(); endmethod
		 method Bool notEmpty(); return ina.notEmpty() && inb.notEmpty(); endmethod
	      endinterface);
   endfunction

module mkFunnel#(PipeOut#(Vector#(mk,a)) in)(PipeOut#(Vector#(m, a)))
   provisos (Mul#(m, k, mk),
	     Bits#(a, asz),
	     Add#(a__, TMul#(asz, m), TMul#(asz, mk)),
	     Add#(1, b__, asz),
	     Add#(2, c__, mk),
	     Add#(d__, m, mk),
	     Add#(asz, m, e__),
	     Add#(asz, mk, f__));
   let m = fromInteger(valueOf(m));
   let mk = fromInteger(valueOf(mk));

   MIMOConfiguration cfg = defaultValue();
   MIMO#(mk, m, mk, a) mimo <- mkMIMO(cfg);
   rule consumer if (mimo.enqReadyN(mk));
      Vector#(mk, a) v = in.first();
      in.deq();
      mimo.enq(mk, v);
   endrule

   method Vector#(m, a) first() if (mimo.deqReadyN(m));
      return mimo.first();
   endmethod
   method Action deq() if (mimo.deqReadyN(m));
      mimo.deq(m);
   endmethod
   method notEmpty();
      return mimo.deqReadyN(m);
   endmethod
endmodule

module mkFunnel1#(PipeOut#(Vector#(k,a)) in)(PipeOut#(a))
   provisos (Bits#(a, asz), Log#(k,ksz));

   Reg#(Bit#(ksz)) selector <- mkReg(0);

   method a first();
      return in.first[selector];
   endmethod
   method Action deq();
      if (selector == fromInteger(valueOf(k)-1)) begin
	 in.deq();
	 selector <= 0;
      end
      else
	 selector <= selector + 1;
   endmethod
   method notEmpty();
      return in.notEmpty();
   endmethod
endmodule

module mkFunnelGB1#(Clock slowClock, Reset slowReset, Clock fastClock, Reset fastReset, PipeOut#(Vector#(k,a)) in)(PipeOut#(a))
   provisos (Bits#(a, asz), Log#(k,ksz), Add#(1,a__,k));

   Gearbox#(k,1,a) gb <- mkNto1Gearbox(slowClock, slowReset, fastClock, fastReset);
   PipeIn#(Vector#(k,a)) toGb = toPipeIn(gb);
   mkConnection(in, toGb);
   PipeOut#(Vector#(1,a)) fromGb = toPipeOut(gb);
   return mapPipe(head, fromGb);
endmodule

// 'j' is the width of the narrow end, and 'k' is the width of the wide end
typedef Vector#(j,PipeOut#(a))   FunnelPipe#(numeric type j, numeric type k, type a, numeric type bitsPerCycle);
typedef Vector#(k,PipeOut#(a)) UnFunnelPipe#(numeric type j, numeric type k, type a, numeric type bitsPerCycle);

typeclass FunnelPipesPipelined#(numeric type j, numeric type k, type a, numeric type bpc);
   module mkFunnelPipesPipelined#(Vector#(k,PipeOut#(a)) in) (FunnelPipe#(j,k,a,bpc));
   module mkFunnelPipesPipelinedRR#(Vector#(k,PipeOut#(a)) in, Integer c) (FunnelPipe#(j,k,a,bpc));
   module mkUnFunnelPipesPipelined#(Vector#(j,PipeOut#(Tuple2#(Bit#(TLog#(k)),a))) in) (UnFunnelPipe#(j,k,a,bpc));
   module mkUnFunnelPipesPipelinedRR#(Vector#(j,PipeOut#(a)) in, Integer c) (UnFunnelPipe#(j,k,a,bpc));
endtypeclass

function PipeOut#(b) pipeSecond(PipeOut#(Tuple2#(a,b)) x) =
   (interface PipeOut;
       method b first;
	  return tpl_2(x.first);
       endmethod
       method Action deq = x.deq;
       method Bool notEmpty = x.notEmpty;
    endinterface);

instance FunnelPipesPipelined#(1,1,a,bpc)
   provisos (Bits#(a,a__));
   module mkFunnelPipesPipelined#(Vector#(1,PipeOut#(a)) in) (FunnelPipe#(1,1,a,bpc));
      return in;
   endmodule
   module mkFunnelPipesPipelinedRR#(Vector#(1,PipeOut#(a)) in, Integer c) (FunnelPipe#(1,1,a,bpc));
      return in;
   endmodule
   module mkUnFunnelPipesPipelined#(Vector#(1,PipeOut#(Tuple2#(Bit#(0),a))) in) (UnFunnelPipe#(1,1,a,bpc));
      return map(pipeSecond, in);
   endmodule
   module mkUnFunnelPipesPipelinedRR#(Vector#(1,PipeOut#(a)) in, Integer c) (UnFunnelPipe#(1,1,a,bpc));
      return in;
   endmodule
endinstance

module mkUnFunnelPipesPipelinedInternal#(Vector#(1, PipeOut#(Tuple2#(Bit#(TLog#(k)),a))) in) (UnFunnelPipe#(1,k,a,bpc))
   provisos (Log#(k, logk),
	     Bits#(a,a__),
	     Add#(1,b__,k),
	     Div#(logk,bpc,stages));
   Vector#(k, PipeOut#(Tuple2#(Bit#(logk),a))) ins  = append(in,replicate(?));
   Vector#(k, PipeOut#(Tuple2#(Bit#(logk),a))) outs = newVector;
   for(Integer j = 0; j < valueOf(stages); j=j+1) begin
      for(Integer i = 0; i < 2**(j*valueOf(bpc)); i=i+1) begin
	 Integer bits = (j == valueOf(stages)-1) ? valueOf(logk)-(j*valueOf(bpc)) : valueOf(bpc);
	 function Bit#(bpc) sh(Bit#(bpc) x) = x<<(valueOf(bpc)-bits);
	 for(Integer l = 0; l < 2**bits; l=l+1)  begin
	    let buff <- mkFIFOF;
	    // extra conditional in case 'k' is not a power of 2
	    let idx = (2**bits)*i+l;
	    if (idx < valueOf(k)) begin
	       outs[idx] = toPipeOut(buff);
	       rule xfer if(tpl_1(ins[i].first)[(valueOf(logk)-1):(valueOf(logk)-valueOf(bpc))] == sh(fromInteger(l)));
		  match{.idx, .v} <- toGet(ins[i]).get;
		  buff.enq(tuple2(idx<<valueOf(bpc), v));
	       endrule
	    end
	 end
      end
      ins = outs;
   end
   return map(pipeSecond,outs);
endmodule

module mkFunnelNode#(Vector#(n, PipeOut#(a)) inpipes, Integer numPipes, Put#(a) outpipe)(Empty);
   rule funnel;
      a v = ?;
      Bool send = False;
      for (Integer i = 0; i < valueOf(n) && i < numPipes; i = i+1)
	 if (!send && inpipes[i].notEmpty) begin
	    v <- toGet(inpipes[i]).get();
	    send = True;
	 end
      if (send)
	 outpipe.put(v);
   endrule
endmodule

instance FunnelPipesPipelined#(1,k,a,bpc)
   provisos (Log#(k, logk),
	     Bits#(a,a__),
	     Add#(1,b__,k),
	     Div#(logk,bpc,stages),
	     Add#(k, c__, TExp#(logk)),
	     Add#(TExp#(bpc), d__, TExp#(logk))
	     );
   module mkFunnelPipesPipelined#(Vector#(k,PipeOut#(a)) in) (FunnelPipe#(1,k,a,bpc));
      Vector#(stages, Vector#(k, FIFOF#(a))) buffs  <- replicateM(replicateM(mkFIFOF));
      Vector#(TAdd#(stages,1), Vector#(k, PipeOut#(a))) infss = append(map(map(toPipeOut),buffs), vec(in));
      for(Integer j = valueOf(stages); j > 0; j=j-1) begin
	 Integer width = 2**(j*valueOf(bpc));
	 Integer stride = 2**valueOf(bpc);
	 for(Integer i = 0; i < width && i < valueOf(k); i=i+stride) begin
	    Vector#(TExp#(logk),PipeOut#(a)) paddedPipes = append(infss[j], replicate(?));
	    Vector#(TExp#(bpc),PipeOut#(a)) inpipes = takeAt(i, paddedPipes);
	    Integer numPipes = stride;
	    if (i + stride > valueOf(k))
	       numPipes = valueOf(k) - i;
	    mkFunnelNode(inpipes, numPipes, toPut(buffs[j-1][i/stride]));
	 end
      end
      return vec(infss[0][0]);
   endmodule
   module mkFunnelPipesPipelinedRR#(Vector#(k,PipeOut#(a)) in, Integer c) (FunnelPipe#(1,k,a,bpc));
      Vector#(stages, Vector#(k, FIFOF#(a))) buffs  <- replicateM(replicateM(mkFIFOF));
      Vector#(TAdd#(stages,1), Vector#(k, PipeOut#(a))) infss = append(map(map(toPipeOut),buffs), cons(in,nil));
      for(Integer j = valueOf(stages); j > 0; j=j-1) begin
	 Vector#(k, FIFOF#(void)) ctrl  <- replicateM(mkFIFOF1());
   	 for(Integer i = 0; i < 2**(j*valueOf(bpc)) && i < valueOf(k); i=i+1) begin
   	    let first = i==0;
   	    Reg#(Bit#(32)) cnt <- mkReg(0);
   	    let maxp = (2**((valueOf(stages)-j)*valueOf(bpc)))*c;
   	    let last = (maxp*(i+1) >= valueOf(k)*c);
   	    if (maxp*(i+1) > valueOf(k)*c) begin
   	       maxp = valueOf(k)*c-maxp*i;
	    end
   	    let xfer_guard = True;
   	    if (!first)
   	       xfer_guard = xfer_guard && ctrl[(i-1)].notEmpty;
   	    if (!last)
   	       xfer_guard = xfer_guard && (!ctrl[i].notEmpty);
   	    rule xfer if (xfer_guard);
   	       let new_cnt = cnt+1;
   	       if (new_cnt==fromInteger(maxp)) begin
   		  cnt <= 0;
   		  if (!last)
   		     ctrl[i].enq(?);
   		  if (last)
   		     for(Integer ff = 0; ff < i; ff=ff+1)
   		     	ctrl[ff].deq;
   	       end
   	       else begin
   		  cnt <= new_cnt;
   	       end
   	       let v <- toGet(infss[j][i]).get;
   	       toPut(buffs[j-1][i/(2**valueOf(bpc))]).put(v);
   	    endrule
   	 end
      end
      return cons(infss[0][0],nil);
   endmodule
   module mkUnFunnelPipesPipelined#(Vector#(1, PipeOut#(Tuple2#(Bit#(logk),a))) in) (UnFunnelPipe#(1,k,a,bpc));
      let rv <- mkUnFunnelPipesPipelinedInternal(in);
      return rv;
   endmodule
   module mkUnFunnelPipesPipelinedRR#(Vector#(1, PipeOut#(a)) in, Integer c) (UnFunnelPipe#(1,k,a,bpc));
      Vector#(1, FIFOF#(Tuple2#(Bit#(logk),a))) tagged_in_buffers <- replicateM(mkFIFOF);
      Vector#(1, PipeOut#(Tuple2#(Bit#(logk),a))) tagged_in = map(toPipeOut, tagged_in_buffers);
      let rv <- mkUnFunnelPipesPipelinedInternal(tagged_in);
      Reg#(Bit#(TAdd#(logk,1))) dest <- mkReg(0);
      Reg#(Bit#(32)) cnt <- mkReg(0);
      rule fill;
	 let new_cnt = cnt+1;
	 let new_dest = dest;
	 if (new_cnt == fromInteger(c)) begin
	    new_cnt = 0;
	    new_dest = dest+1;
	    if(new_dest==fromInteger(valueOf(k)))
	       new_dest=0;
	 end
	 cnt <= new_cnt;
	 dest <= new_dest;
	 let v <- toGet(in[0]).get;
	 tagged_in_buffers[0].enq(tuple2(truncate(dest),v));
	 //$display("mkUnFunnelPipesPipelinedInternal::fill %d", dest);
      endrule
      return rv;
   endmodule
endinstance

module mkUnfunnel#(PipeOut#(Vector#(m,a)) in)(PipeOut#(Vector#(mk, a)))
   provisos (Mul#(m, k, mk),
	     Bits#(a, asz),
	     Add#(1, b__, asz),
	     Add#(2, c__, mk),
	     Add#(d__, m, mk),
	     Add#(asz, m, e__),
	     Add#(asz, mk, f__));
   let m = fromInteger(valueOf(m));
   let mk = fromInteger(valueOf(mk));

   MIMOConfiguration cfg = defaultValue();
   MIMO#(m, mk, mk, a) mimo <- mkMIMO(cfg);
   rule consumer if (mimo.enqReadyN(m));
      Vector#(m, a) v = in.first();
      in.deq();
      mimo.enq(m, v);
   endrule

   method Vector#(mk, a) first() if (mimo.deqReadyN(mk));
      return mimo.first();
   endmethod
   method Action deq() if (mimo.deqReadyN(mk));
      mimo.deq(mk);
   endmethod
   method notEmpty();
      return mimo.deqReadyN(mk);
   endmethod
endmodule

module mkUnfunnelGB#(Clock slowClock, Reset slowReset, Clock fastClock, Reset fastReset, PipeOut#(Vector#(1,a)) in)(PipeOut#(Vector#(k, a)))
   provisos (Bits#(a, asz),
	     Add#(1, a__, k),
	     Add#(1, b__, asz),
	     Add#(1, c__, TMul#(2,k)),
	     Add#(k, d__, TMul#(2,k))
      );
   let k = fromInteger(valueOf(k));

   Gearbox#(1,k,a) gb <- mk1toNGearbox(fastClock, fastReset, slowClock, slowReset);
   PipeIn#(Vector#(1,a)) toGb = toPipeIn(gb);
   PipeOut#(Vector#(k,a)) fromGb = toPipeOut(gb);
   mkConnection(in,toGb);
   return fromGb;
endmodule: mkUnfunnelGB

module mkFunnelPipes#(Vector#(mk, PipeOut#(a)) ins)(Vector#(m, PipeOut#(a)))
   provisos (Mul#(m, k, mk),
	     Bits#(a, asz),
	     Log#(k,ksz)
      );
   let k = fromInteger(valueOf(k));
   let m = fromInteger(valueOf(m));
   let mk = fromInteger(valueOf(mk));

   Vector#(m, FIFOF#(a)) fifos <- replicateM(mkFIFOF);
   for (Integer i = 0; i < m; i = i+1) begin
      Reg#(Bit#(asz)) which <- mkReg(0);
      rule consumer;
	 let index = (which << valueOf(ksz)) + fromInteger(i);
	 let v <- toGet(ins[index]).get();
	 fifos[i].enq(v);
	 which <= (which + 1) % k;
      endrule
   end

   return map(toPipeOut, fifos);
endmodule

module mkFunnelPipes1#(Vector#(k, PipeOut#(a)) ins)(PipeOut#(a))
   provisos (Bits#(a, asz),
	     Log#(k,ksz)
      );
   let k = fromInteger(valueOf(k));
   Reg#(Bit#(ksz)) selector <- mkReg(0);

   method a first();
      return ins[selector].first();
   endmethod
   method Action deq();
      ins[selector].deq();
      if (selector == fromInteger(valueOf(k)-1))
	 selector <= 0;
   else
      selector <= selector + 1;
   endmethod
   method Bool notEmpty();
      return ins[selector].notEmpty();
   endmethod
endmodule

module mkUnfunnelPipes#(Vector#(m, PipeOut#(a)) ins)(Vector#(mk, PipeOut#(a)))
   provisos (Mul#(m, k, mk),
	     Log#(k,ksz),
	     Bits#(a,asz));

   let m = fromInteger(valueOf(m));
   let k = fromInteger(valueOf(k));
   let mk = fromInteger(valueOf(mk));

   Vector#(mk, FIFOF#(a)) fifos <- replicateM(mkFIFOF);
   for (Integer i = 0; i < m; i = i + 1) begin
      Reg#(Bit#(TAdd#(1,ksz))) which <- mkReg(0);
      rule consumer;
	 let index = which + fromInteger(i)*k;
	 let v <- toGet(ins[i]).get();
	 fifos[index].enq(v);
	 which <= (which + 1) % k;
      endrule
   end
   return map(toPipeOut, fifos);
endmodule

module mkRepeat#(UInt#(n) repetitions, PipeOut#(a) inpipe)(PipeOut#(a));
   Reg#(UInt#(n)) count <- mkReg(0);
   method first = inpipe.first;
   method Action deq();
      let c = count + 1;
      if (count == (repetitions - 1)) begin
	 c = 0;
	 inpipe.deq();
      end
      count <= c;
   endmethod
   method notEmpty = inpipe.notEmpty;
endmodule

module mkPipelinedForkVector#(PipeOut#(a) inpipe, Integer id)(UnFunnelPipe#(1,k,a,bpc))
   provisos ( Bits#(a,a__)
	     ,Add#(1,b__,k)
	     ,Log#(k,logk)
	     ,Div#(logk,bpc,stages));
   Vector#(k, FIFOF#(a))  buffs = newVector;
   Vector#(k, PipeOut#(a)) infs = cons(inpipe,replicate(?));
   for(Integer j = 0; j < valueOf(stages); j=j+1)begin
      for(Integer i = 0; i < 2**((j+1)*valueOf(bpc)) && i < valueOf(k); i=i+1)
	 buffs[i] <- mkFIFOF;
      rule xfer;
      	 for(Integer i = 0; i < 2**(j*valueOf(bpc)) && i < valueOf(k); i=i+1) begin
      	    for(Integer l = 0; l < 2**valueOf(bpc) && l < valueOf(k); l=l+1) begin
	       Integer idx = (i*(2**valueOf(bpc)))+l;
      	       if (idx < valueOf(k))
		  buffs[idx].enq(infs[i].first);
      	    end
      	    infs[i].deq;
      	 end
      endrule
      infs = map(toPipeOut, buffs);
   end
   return infs;
endmodule

module mkForkVector#(PipeOut#(a) inpipe)(Vector#(n, PipeOut#(a)))
   provisos (Bits#(a, asz));
   Vector#(n, FIFOF#(a)) fifos <- replicateM(mkFIFOF());
   rule forkelts;
      let v = inpipe.first();
      inpipe.deq;
      for (Integer i = 0; i < valueOf(n); i = i + 1) begin
	 fifos[i].enq(v);
      end
   endrule
   return map(toPipeOut, fifos);
endmodule

module mkSizedForkVector#(Integer size, PipeOut#(a) inpipe)(Vector#(n, PipeOut#(a)))
   provisos (Bits#(a, asz));
   Vector#(n, FIFOF#(a)) fifos <- replicateM(mkSizedFIFOF(size));
   rule forkelts;
      let v = inpipe.first();
      inpipe.deq;
      for (Integer i = 0; i < valueOf(n); i = i + 1) begin
	 fifos[i].enq(v);
      end
   endrule
   return map(toPipeOut, fifos);
endmodule

module mkJoin#(function c f(a av, b bv), PipeOut#(a) apipe, PipeOut#(b) bpipe)(PipeOut#(c));
   method c first();
      let av = apipe.first();
      let bv = bpipe.first();
      return f(av, bv);
   endmethod
   method Action deq();
      apipe.deq();
      bpipe.deq();
   endmethod
   method Bool notEmpty();
      return apipe.notEmpty() && bpipe.notEmpty();
   endmethod
endmodule

module mkJoinBuffered#(function c f(a av, b bv), PipeOut#(a) apipe, PipeOut#(b) bpipe)(PipeOut#(c))
   provisos (Bits#(c, csz));
   FIFOF#(c) joinFifo <- mkFIFOF();
   rule joinrule;
      let av <- toGet(apipe).get();
      let bv <- toGet(bpipe).get();
      joinFifo.enq(f(av, bv));
   endrule
   return toPipeOut(joinFifo);
endmodule

module mkJoinVector#(function b f(Vector#(n, a) av), Vector#(n, PipeOut#(a)) apipes)(PipeOut#(b))
   provisos (Bits#(Vector#(n,a),vasz));
   method b first();
      function a getfirst(PipeOut#(a) pipein); return pipein.first(); endfunction
      Vector#(n,a) vec = map(getfirst, apipes);
      return f(vec);
   endmethod
   method Action deq();
      function a getfirst(PipeOut#(a) pipein); return pipein.first(); endfunction
      for (Integer i = 0; i < valueOf(n); i = i + 1)
	 apipes[i].deq();
   endmethod
   method Bool notEmpty();
      function Bool myand(Bool a, Bool b); return a && b; endfunction
      return foldl(myand, True, map(pipeOutNotEmpty, apipes));
   endmethod
endmodule

function PipeOut#(b) mapPipe(function b f(a av), PipeOut#(a) apipe);
   return (interface PipeOut#(b);
      method b first();
	 let av = apipe.first();
	 return f(av);
      endmethod
      method Action deq();
	 apipe.deq();
      endmethod
      method Bool notEmpty();
	 return apipe.notEmpty();
      endmethod
      endinterface);
endfunction

function PipeIn#(a) mapPipeIn(function b f(a av), PipeIn#(b) apipe);
   return (interface PipeIn#(b);
	      method Action enq(a v);
		 apipe.enq(f(v));
	      endmethod
	      method Bool notFull();
		 return apipe.notFull();
	      endmethod
	   endinterface);
endfunction

// buffered version of mapPipe
module mkMapPipe#(function b f(a av), PipeOut#(a) apipe)(PipeOut#(b))
   provisos (Bits#(b,bsz));
   FIFOF#(b) fifo <- mkFIFOF();
   rule compute;
      let v <- toGet(apipe).get();
      fifo.enq(f(v));
   endrule
   return toPipeOut(fifo);
endmodule

typedef (function tb f(ta x)) CombinePipe#(type ta, type tb);

typeclass ReducePipe#( numeric type n, type a);
   module  mkReducePipe (CombinePipe#(Tuple2#(a,a), a) combinepipe,
			 PipeOut#(Vector#(n,a)) inpipe,
			 PipeOut#(a) ifc);
   module  mkReducePipes (CombinePipe#(Tuple2#(a,a), a) combinepipe,
			  Vector#(n,PipeOut#(a)) inpipe,
			  PipeOut#(a) ifc);
endtypeclass
instance ReducePipe#(1, a);
   module  mkReducePipe (CombinePipe#(Tuple2#(a,a), a) combinepipe,
				 PipeOut#(Vector#(1,a)) inpipe,
				 PipeOut#(a) ifc);
      let pipe = mapPipe(head, inpipe);
      return pipe;
   endmodule
   module  mkReducePipes (CombinePipe#(Tuple2#(a,a), a) combinepipe,
				  Vector#(1,PipeOut#(a)) inpipes,
				  PipeOut#(a) ifc);
      return inpipes[0];
   endmodule
endinstance
instance ReducePipe#(2, a)
   provisos(Bits#(a,a__));
   module  mkReducePipe (CombinePipe#(Tuple2#(a,a), a) combinepipe,
			 PipeOut#(Vector#(2,a)) inpipe,
			 PipeOut#(a) ifc);
      function a foo(Vector#(2,a) invec);
	 return combinepipe(tuple2(invec[0], invec[1]));
      endfunction
      let pipe <- mkMapPipe(foo, inpipe);
      return pipe;
   endmodule
   module  mkReducePipes (CombinePipe#(Tuple2#(a,a), a) combinepipe,
			  Vector#(2,PipeOut#(a)) inpipes,
			  PipeOut#(a) ifc);
      function a foo(Tuple2#(a,a) invec);
	 return combinepipe(invec);
      endfunction
      let pipe <- mkMapPipe(foo, zipPipeOut(inpipes[0], inpipes[1]));
      return pipe;
   endmodule
endinstance

instance ReducePipe#(n, a)
   provisos (Add#(TDiv#(n,2), a__, n),
	     Bits#(Vector#(TDiv#(n,2), a), b__),
	     ReducePipe#(TDiv#(n,2),a),
	     ReducePipe#(TSub#(n, TDiv#(n, 2)), a)
      );
   module  mkReducePipe (CombinePipe#(Tuple2#(a,a), a) combinepipe,
			 PipeOut#(Vector#(n,a)) inpipe,
			 PipeOut#(a) ifc);
      FIFOF#(Vector#(TDiv#(n,2),a)) infifo0 <- mkFIFOF;
      FIFOF#(Vector#(TSub#(n,TDiv#(n,2)),a)) infifo1 <- mkFIFOF;
      rule splitinput;
	 let v = inpipe.first();
	 inpipe.deq();
	 infifo0.enq(takeAt(0, v));
	 infifo1.enq(takeAt(valueOf(TDiv#(n,2)), v));
      endrule
      PipeOut#(Vector#(TDiv#(n,2),a)) inpipe0 = toPipeOut(infifo0);
      PipeOut#(Vector#(TSub#(n,TDiv#(n,2)),a)) inpipe1 = toPipeOut(infifo1);

      PipeOut#(a) p0 <- mkReducePipe(combinepipe, inpipe0);
      PipeOut#(a) p1 <- mkReducePipe(combinepipe, inpipe1);

      function a foo(Tuple2#(a,a) invec);
	 return combinepipe(invec);
      endfunction
      let pipe <- mkMapPipe(foo,zipPipeOut(p0, p1));
      return pipe;
   endmodule

   module  mkReducePipes (CombinePipe#(Tuple2#(a,a), a) combinepipe,
			  Vector#(n, PipeOut#(a)) inpipes,
			  PipeOut#(a) ifc);
      Vector#(TDiv#(n,2),PipeOut#(a)) pipes0 = takeAt(0, inpipes);
      Vector#(TSub#(n,TDiv#(n,2)),PipeOut#(a)) pipes1 = takeAt(valueOf(TDiv#(n,2)), inpipes);

      PipeOut#(a) p0 <- mkReducePipes(combinepipe, pipes0);
      PipeOut#(a) p1 <- mkReducePipes(combinepipe, pipes1);

      function a foo(Tuple2#(a,a) invec);
	 return combinepipe(invec);
      endfunction
      let pipe <- mkMapPipe(foo,zipPipeOut(p0, p1));
      return pipe;
   endmodule
endinstance

interface FirstLastPipe#(type a);
   interface PipeOut#(Tuple2#(Bool,Bool)) pipe;
   method Action start(a count);
endinterface

module mkFirstLastPipe(FirstLastPipe#(a))
   provisos (Bits#(a,asz), Ord#(a), Arith#(a), Eq#(a));
   Reg#(a) countReg <- mkReg(0);
   Reg#(Bool) firstReg <- mkReg(False);
   Reg#(Bool) lastReg <- mkReg(False);
   interface PipeOut pipe;
      method Tuple2#(Bool, Bool) first();
	 return tuple2(firstReg, lastReg);
      endmethod
      method Action deq() if (countReg > 0);
	 firstReg <= False;
	 let c = countReg - 1;
	 if (c == 1)
	    lastReg <= True;
	 countReg <= c;
      endmethod
      method Bool notEmpty();
	 return countReg > 0;
      endmethod
   endinterface
   method Action start(a count) if (countReg == 0);
      firstReg <= True;
      lastReg <= False;
      countReg <= count;
   endmethod
endmodule

typedef struct {
   a xbase;
   a xlimit;
   a xstep;
} RangeConfig#(type a) deriving (Bits, FShow);

interface RangePipeIfc#(type a);
   interface PipeOut#(a) pipe;
   method Bool isFirst();
   method Bool isLast();
   method Action start(RangeConfig#(a) cfg);
endinterface

module mkRangePipeOut(RangePipeIfc#(a)) provisos (Arith#(a), Bits#(a,awidth), Eq#(a), Ord#(a));
   Reg#(a) x <- mkReg(1);
   Reg#(a) xbase <- mkReg(0);
   Reg#(a) xstep <- mkReg(1);
   // inclusive limit
   Reg#(a) xlimit <- mkReg(0);
   Reg#(Bool) first <- mkReg(False);
   Reg#(Bool) last <- mkReg(False);
   Bool verbose = False;
   interface PipeOut pipe;
      method a first();
	 return x;
      endmethod
      method Action deq if (x <= xlimit);
	 x <= x + xstep;
	 first <= False;
	 last <= (x+xstep >= xlimit);
      endmethod
      method Bool notEmpty();
	 return (x <= xlimit);
      endmethod
   endinterface
   method Action start(RangeConfig#(a) cfg) if (x > xlimit);
      x <= cfg.xbase;
      xbase <= cfg.xbase;
      xstep <= cfg.xstep;

      // inclusive limit, should update interface accordingly
      xlimit <= cfg.xlimit-cfg.xstep;

      first <= True;
      last <= (cfg.xbase+cfg.xstep >= cfg.xlimit);
      if (verbose) $display("mkRangePipeOut xbase=%d xstep=%d cfg.xlimit=%d xlimit=%d last=%d", cfg.xbase, cfg.xstep, cfg.xlimit, (cfg.xlimit-cfg.xstep), (cfg.xbase+cfg.xstep >= cfg.xlimit));
   endmethod
   method Bool isFirst() = first;
   method Bool isLast() = last;
endmodule: mkRangePipeOut

// Copyright (c) 2014 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import FIFOF::*;
import GetPut::*;
import Connectable::*;
import Vector::*;
import BuildVector::*;
import MIMO::*;
import DefaultValue::*;
import Gearbox::*;
import Clocks::*;

typedef Tuple3#(x,x,x) Tripple#(type x);
typedef Tuple2#(x,x)   Pair#(type x);

interface PipeIn#(type a);
   method Action enq(a v);
   method Bool notFull();
endinterface

interface PipeOut#(type a);
   method a first();
   method Action deq();
   method Bool notEmpty();
endinterface

function Bool pipeInNotFull(PipeIn#(a) pipein); return pipein.notFull(); endfunction
function Bool pipeOutNotEmpty(PipeOut#(a) pipein); return pipein.notEmpty(); endfunction

typeclass ToPipeIn#(type a, type b) dependencies (b determines a);
   function PipeIn#(a) toPipeIn(b in);
endtypeclass

typeclass ToPipeOut#(type a, type b) dependencies ( b determines a);
   function PipeOut#(a) toPipeOut(b in);
endtypeclass

typeclass MkPipeOut#(type a, type b) dependencies ( b determines a);
   module mkPipeOut#(b in)(PipeOut#(a));
endtypeclass

typeclass MkPipeIn#(type a, type b) dependencies ( b determines a);
   module mkPipeIn#(b in)(PipeIn#(a));
endtypeclass

instance ToPipeIn#(a, FIFOF#(a));
   function PipeIn#(a) toPipeIn(FIFOF#(a) in);
      return (interface PipeIn#(a);
		 method enq = in.enq;
		 method notFull = in.notFull;
	      endinterface);
   endfunction
endinstance

instance ToPipeOut#(a, function a pipefn());
   function PipeOut#(a) toPipeOut(function a pipefn());
      return (interface PipeOut#(a);
		 method first(); return pipefn(); endmethod
		 method Action deq(); endmethod
		 method Bool notEmpty(); return False; endmethod
	      endinterface);
   endfunction
endinstance

instance ToPipeOut#(a, Reg#(a));
   function PipeOut#(a) toPipeOut(Reg#(a) in);
      return (interface PipeOut#(a);
		 method first(); return in; endmethod
		 method Action deq(); endmethod
		 method Bool notEmpty(); return False; endmethod
	      endinterface);
   endfunction
endinstance

instance ToPipeIn#(Vector#(m, a), Gearbox#(m, n, a));
   function PipeIn#(Vector#(m, a)) toPipeIn(Gearbox#(m, n, a) in);
      return (interface PipeIn#(Vector#(m, a));
		 method enq = in.enq;
		 method notFull = in.notFull;
	      endinterface);
   endfunction
endinstance

instance ToPipeOut#(a, FIFOF#(a));
   function PipeOut#(a) toPipeOut(FIFOF#(a) in);
      return (interface PipeOut#(a);
		 method first = in.first;
		 method deq = in.deq;
		 method notEmpty = in.notEmpty;
	      endinterface);
   endfunction
endinstance

instance ToPipeOut#(Vector#(n,a), MIMO#(k,n,sz,a));
   function PipeOut#(Vector#(n,a)) toPipeOut(MIMO#(k,n,sz,a) in);
      return (interface PipeOut#(a);
		 method first = in.first;
		 method Action deq() if (in.deqReadyN(fromInteger(valueOf(n))));
		    in.deq(fromInteger(valueOf(n)));
		 endmethod
		 method Bool notEmpty();
		    return in.deqReadyN(fromInteger(valueOf(n)));
		 endmethod
	      endinterface);
   endfunction
endinstance

instance ToPipeOut#(Vector#(n, a), Gearbox#(m, n, a));
   function PipeOut#(Vector#(n, a)) toPipeOut(Gearbox#(m, n, a) in);
      return (interface PipeOut#(Vector#(n,a));
		 method first = in.first;
		 method deq = in.deq;
		 method notEmpty = in.notEmpty;
	      endinterface);
   endfunction
endinstance

instance ToPipeIn#(a, SyncFIFOIfc#(a));
   function PipeIn#(a) toPipeIn(SyncFIFOIfc#(a) in);
      return (interface PipeIn#(a);
		 method enq = in.enq;
		 method notFull = in.notFull;
	      endinterface);
   endfunction
endinstance

instance ToPipeOut#(a, SyncFIFOIfc#(a));
   function PipeOut#(a) toPipeOut(SyncFIFOIfc#(a) in);
      return (interface PipeOut#(a);
		 method first = in.first;
		 method deq = in.deq;
		 method notEmpty = in.notEmpty;
	      endinterface);
   endfunction
endinstance

instance MkPipeOut#(a, ActionValue#(a))
   provisos (Bits#(a, asz));
   module mkPipeOut#(ActionValue#(a) in)(PipeOut#(a));
      FIFOF#(a) fifo <- mkFIFOF();
      rule connect;
	 let v <- in;
	 fifo.enq(v);
      endrule
      return toPipeOut(fifo);
   endmodule
endinstance

instance MkPipeOut#(a, Get#(a))
   provisos (Bits#(a, asz));
   module mkPipeOut#(Get#(a) in)(PipeOut#(a));
      FIFOF#(a) fifo <- mkFIFOF();
      rule connect;
	 let v <- in.get();
	 fifo.enq(v);
      endrule
      return toPipeOut(fifo);
   endmodule
endinstance

instance MkPipeIn#(a, Put#(a))
   provisos (Bits#(a, asz));
   module mkPipeIn#(Put#(a) out)(PipeIn#(a));
      FIFOF#(a) fifo <- mkFIFOF();
      rule connect;
	 let v <- toGet(fifo).get;
	 out.put(v);
      endrule
      return toPipeIn(fifo);
   endmodule
endinstance

function PipeOut#(a) toCountedPipeOut(Reg#(Bit#(n)) r, PipeOut#(a) pipe);
   return (interface PipeOut#(Vector#(n,a));
	      method first = pipe.first;
	      method Action deq(); pipe.deq; r <= r + 1; endmethod
	      method notEmpty = pipe.notEmpty;
	   endinterface);
endfunction

instance ToGet #(PipeOut #(a), a);
   function Get #(a) toGet (PipeOut #(a) po);
      return (interface Get;
                 method ActionValue #(a) get ();
                    po.deq ();
                    return po.first ();
                 endmethod
              endinterface);
   endfunction
endinstance

instance ToPut #(PipeIn #(a), a);
   function Put #(a) toPut (PipeIn #(a) pi);
      return (interface Put;
		 method Action put(a v);
                    pi.enq (v);
                 endmethod
              endinterface);
   endfunction
endinstance

instance Connectable#(PipeOut#(a),Put#(a));
   module mkConnection#(PipeOut#(a) in, Put#(a) out)(Empty);
      rule connect;
	 let v = in.first;
	 in.deq();
	 out.put(v);
      endrule
   endmodule
endinstance

instance Connectable#(PipeOut#(a),PipeIn#(a));
   module mkConnection#(PipeOut#(a) in, PipeIn#(a) out)(Empty);
      rule connect;
	 let v = in.first;
	 in.deq();
	 out.enq(v);
      endrule
   endmodule
endinstance

function PipeOut#(a) unvectorPipeOut(PipeOut#(Vector#(1,a)) in);
   return (interface PipeOut#(a);
	      method first = in.first[0];
	      method deq = in.deq;
	      method notEmpty = in.notEmpty;
	   endinterface);
endfunction

function PipeOut#(Tuple2#(a,b)) zipPipeOut(PipeOut#(a) ina, PipeOut#(b) inb);
      return (interface PipeOut#(Tuple2#(a,b));
		 method Tuple2#(a,b) first(); return tuple2(ina.first, inb.first); endmethod
		 method Action deq(); ina.deq(); inb.deq(); endmethod
		 method Bool notEmpty(); return ina.notEmpty() && inb.notEmpty(); endmethod
	      endinterface);
   endfunction

module mkFunnel#(PipeOut#(Vector#(mk,a)) in)(PipeOut#(Vector#(m, a)))
   provisos (Mul#(m, k, mk),
	     Bits#(a, asz),
	     Add#(a__, TMul#(asz, m), TMul#(asz, mk)),
	     Add#(1, b__, asz),
	     Add#(2, c__, mk),
	     Add#(d__, m, mk),
	     Add#(asz, m, e__),
	     Add#(asz, mk, f__));
   let m = fromInteger(valueOf(m));
   let mk = fromInteger(valueOf(mk));

   MIMOConfiguration cfg = defaultValue();
   MIMO#(mk, m, mk, a) mimo <- mkMIMO(cfg);
   rule consumer if (mimo.enqReadyN(mk));
      Vector#(mk, a) v = in.first();
      in.deq();
      mimo.enq(mk, v);
   endrule

   method Vector#(m, a) first() if (mimo.deqReadyN(m));
      return mimo.first();
   endmethod
   method Action deq() if (mimo.deqReadyN(m));
      mimo.deq(m);
   endmethod
   method notEmpty();
      return mimo.deqReadyN(m);
   endmethod
endmodule

module mkFunnel1#(PipeOut#(Vector#(k,a)) in)(PipeOut#(a))
   provisos (Bits#(a, asz), Log#(k,ksz));

   Reg#(Bit#(ksz)) selector <- mkReg(0);

   method a first();
      return in.first[selector];
   endmethod
   method Action deq();
      if (selector == fromInteger(valueOf(k)-1)) begin
	 in.deq();
	 selector <= 0;
      end
      else
	 selector <= selector + 1;
   endmethod
   method notEmpty();
      return in.notEmpty();
   endmethod
endmodule

module mkFunnelGB1#(Clock slowClock, Reset slowReset, Clock fastClock, Reset fastReset, PipeOut#(Vector#(k,a)) in)(PipeOut#(a))
   provisos (Bits#(a, asz), Log#(k,ksz), Add#(1,a__,k));

   Gearbox#(k,1,a) gb <- mkNto1Gearbox(slowClock, slowReset, fastClock, fastReset);
   PipeIn#(Vector#(k,a)) toGb = toPipeIn(gb);
   mkConnection(in, toGb);
   PipeOut#(Vector#(1,a)) fromGb = toPipeOut(gb);
   return mapPipe(head, fromGb);
endmodule

// 'j' is the width of the narrow end, and 'k' is the width of the wide end
typedef Vector#(j,PipeOut#(a))   FunnelPipe#(numeric type j, numeric type k, type a, numeric type bitsPerCycle);
typedef Vector#(k,PipeOut#(a)) UnFunnelPipe#(numeric type j, numeric type k, type a, numeric type bitsPerCycle);

typeclass FunnelPipesPipelined#(numeric type j, numeric type k, type a, numeric type bpc);
   module mkFunnelPipesPipelined#(Vector#(k,PipeOut#(a)) in) (FunnelPipe#(j,k,a,bpc));
   module mkFunnelPipesPipelinedRR#(Vector#(k,PipeOut#(a)) in, Integer c) (FunnelPipe#(j,k,a,bpc));
   module mkUnFunnelPipesPipelined#(Vector#(j,PipeOut#(Tuple2#(Bit#(TLog#(k)),a))) in) (UnFunnelPipe#(j,k,a,bpc));
   module mkUnFunnelPipesPipelinedRR#(Vector#(j,PipeOut#(a)) in, Integer c) (UnFunnelPipe#(j,k,a,bpc));
endtypeclass

function PipeOut#(b) pipeSecond(PipeOut#(Tuple2#(a,b)) x) =
   (interface PipeOut;
       method b first;
	  return tpl_2(x.first);
       endmethod
       method Action deq = x.deq;
       method Bool notEmpty = x.notEmpty;
    endinterface);

instance FunnelPipesPipelined#(1,1,a,bpc)
   provisos (Bits#(a,a__));
   module mkFunnelPipesPipelined#(Vector#(1,PipeOut#(a)) in) (FunnelPipe#(1,1,a,bpc));
      return in;
   endmodule
   module mkFunnelPipesPipelinedRR#(Vector#(1,PipeOut#(a)) in, Integer c) (FunnelPipe#(1,1,a,bpc));
      return in;
   endmodule
   module mkUnFunnelPipesPipelined#(Vector#(1,PipeOut#(Tuple2#(Bit#(0),a))) in) (UnFunnelPipe#(1,1,a,bpc));
      return map(pipeSecond, in);
   endmodule
   module mkUnFunnelPipesPipelinedRR#(Vector#(1,PipeOut#(a)) in, Integer c) (UnFunnelPipe#(1,1,a,bpc));
      return in;
   endmodule
endinstance

module mkUnFunnelPipesPipelinedInternal#(Vector#(1, PipeOut#(Tuple2#(Bit#(TLog#(k)),a))) in) (UnFunnelPipe#(1,k,a,bpc))
   provisos (Log#(k, logk),
	     Bits#(a,a__),
	     Add#(1,b__,k),
	     Div#(logk,bpc,stages));
   Vector#(k, PipeOut#(Tuple2#(Bit#(logk),a))) ins  = append(in,replicate(?));
   Vector#(k, PipeOut#(Tuple2#(Bit#(logk),a))) outs = newVector;
   for(Integer j = 0; j < valueOf(stages); j=j+1) begin
      for(Integer i = 0; i < 2**(j*valueOf(bpc)); i=i+1) begin
	 Integer bits = (j == valueOf(stages)-1) ? valueOf(logk)-(j*valueOf(bpc)) : valueOf(bpc);
	 function Bit#(bpc) sh(Bit#(bpc) x) = x<<(valueOf(bpc)-bits);
	 for(Integer l = 0; l < 2**bits; l=l+1)  begin
	    let buff <- mkFIFOF;
	    // extra conditional in case 'k' is not a power of 2
	    let idx = (2**bits)*i+l;
	    if (idx < valueOf(k)) begin
	       outs[idx] = toPipeOut(buff);
	       rule xfer if(tpl_1(ins[i].first)[(valueOf(logk)-1):(valueOf(logk)-valueOf(bpc))] == sh(fromInteger(l)));
		  match{.idx, .v} <- toGet(ins[i]).get;
		  buff.enq(tuple2(idx<<valueOf(bpc), v));
	       endrule
	    end
	 end
      end
      ins = outs;
   end
   return map(pipeSecond,outs);
endmodule

module mkFunnelNode#(Vector#(n, PipeOut#(a)) inpipes, Integer numPipes, Put#(a) outpipe)(Empty);
   rule funnel;
      a v = ?;
      Bool send = False;
      for (Integer i = 0; i < valueOf(n) && i < numPipes; i = i+1)
	 if (!send && inpipes[i].notEmpty) begin
	    v <- toGet(inpipes[i]).get();
	    send = True;
	 end
      if (send)
	 outpipe.put(v);
   endrule
endmodule

instance FunnelPipesPipelined#(1,k,a,bpc)
   provisos (Log#(k, logk),
	     Bits#(a,a__),
	     Add#(1,b__,k),
	     Div#(logk,bpc,stages),
	     Add#(k, c__, TExp#(logk)),
	     Add#(TExp#(bpc), d__, TExp#(logk))
	     );
   module mkFunnelPipesPipelined#(Vector#(k,PipeOut#(a)) in) (FunnelPipe#(1,k,a,bpc));
      Vector#(stages, Vector#(k, FIFOF#(a))) buffs  <- replicateM(replicateM(mkFIFOF));
      Vector#(TAdd#(stages,1), Vector#(k, PipeOut#(a))) infss = append(map(map(toPipeOut),buffs), vec(in));
      for(Integer j = valueOf(stages); j > 0; j=j-1) begin
	 Integer width = 2**(j*valueOf(bpc));
	 Integer stride = 2**valueOf(bpc);
	 for(Integer i = 0; i < width && i < valueOf(k); i=i+stride) begin
	    Vector#(TExp#(logk),PipeOut#(a)) paddedPipes = append(infss[j], replicate(?));
	    Vector#(TExp#(bpc),PipeOut#(a)) inpipes = takeAt(i, paddedPipes);
	    Integer numPipes = stride;
	    if (i + stride > valueOf(k))
	       numPipes = valueOf(k) - i + 1;
	    mkFunnelNode(inpipes, numPipes, toPut(buffs[j-1][i/width]));
	 end
      end
      return vec(infss[0][0]);
   endmodule
   module mkFunnelPipesPipelinedRR#(Vector#(k,PipeOut#(a)) in, Integer c) (FunnelPipe#(1,k,a,bpc));
      Vector#(stages, Vector#(k, FIFOF#(a))) buffs  <- replicateM(replicateM(mkFIFOF));
      Vector#(TAdd#(stages,1), Vector#(k, PipeOut#(a))) infss = append(map(map(toPipeOut),buffs), cons(in,nil));
      for(Integer j = valueOf(stages); j > 0; j=j-1) begin
	 Vector#(k, FIFOF#(void)) ctrl  <- replicateM(mkFIFOF1());
   	 for(Integer i = 0; i < 2**(j*valueOf(bpc)) && i < valueOf(k); i=i+1) begin
   	    let first = i==0;
   	    Reg#(Bit#(32)) cnt <- mkReg(0);
   	    let maxp = (2**((valueOf(stages)-j)*valueOf(bpc)))*c;
   	    let last = (maxp*(i+1) >= valueOf(k)*c);
   	    if (maxp*(i+1) > valueOf(k)*c) begin
   	       maxp = valueOf(k)*c-maxp*i;
	    end
   	    let xfer_guard = True;
   	    if (!first)
   	       xfer_guard = xfer_guard && ctrl[(i-1)].notEmpty;
   	    if (!last)
   	       xfer_guard = xfer_guard && (!ctrl[i].notEmpty);
   	    rule xfer if (xfer_guard);
   	       let new_cnt = cnt+1;
   	       if (new_cnt==fromInteger(maxp)) begin
   		  cnt <= 0;
   		  if (!last)
   		     ctrl[i].enq(?);
   		  if (last)
   		     for(Integer ff = 0; ff < i; ff=ff+1)
   		     	ctrl[ff].deq;
   	       end
   	       else begin
   		  cnt <= new_cnt;
   	       end
   	       let v <- toGet(infss[j][i]).get;
   	       toPut(buffs[j-1][i/(2**valueOf(bpc))]).put(v);
   	    endrule
   	 end
      end
      return cons(infss[0][0],nil);
   endmodule
   module mkUnFunnelPipesPipelined#(Vector#(1, PipeOut#(Tuple2#(Bit#(logk),a))) in) (UnFunnelPipe#(1,k,a,bpc));
      let rv <- mkUnFunnelPipesPipelinedInternal(in);
      return rv;
   endmodule
   module mkUnFunnelPipesPipelinedRR#(Vector#(1, PipeOut#(a)) in, Integer c) (UnFunnelPipe#(1,k,a,bpc));
      Vector#(1, FIFOF#(Tuple2#(Bit#(logk),a))) tagged_in_buffers <- replicateM(mkFIFOF);
      Vector#(1, PipeOut#(Tuple2#(Bit#(logk),a))) tagged_in = map(toPipeOut, tagged_in_buffers);
      let rv <- mkUnFunnelPipesPipelinedInternal(tagged_in);
      Reg#(Bit#(TAdd#(logk,1))) dest <- mkReg(0);
      Reg#(Bit#(32)) cnt <- mkReg(0);
      rule fill;
	 let new_cnt = cnt+1;
	 let new_dest = dest;
	 if (new_cnt == fromInteger(c)) begin
	    new_cnt = 0;
	    new_dest = dest+1;
	    if(new_dest==fromInteger(valueOf(k)))
	       new_dest=0;
	 end
	 cnt <= new_cnt;
	 dest <= new_dest;
	 let v <- toGet(in[0]).get;
	 tagged_in_buffers[0].enq(tuple2(truncate(dest),v));
	 //$display("mkUnFunnelPipesPipelinedInternal::fill %d", dest);
      endrule
      return rv;
   endmodule
endinstance

module mkUnfunnel#(PipeOut#(Vector#(m,a)) in)(PipeOut#(Vector#(mk, a)))
   provisos (Mul#(m, k, mk),
	     Bits#(a, asz),
	     Add#(1, b__, asz),
	     Add#(2, c__, mk),
	     Add#(d__, m, mk),
	     Add#(asz, m, e__),
	     Add#(asz, mk, f__));
   let m = fromInteger(valueOf(m));
   let mk = fromInteger(valueOf(mk));

   MIMOConfiguration cfg = defaultValue();
   MIMO#(m, mk, mk, a) mimo <- mkMIMO(cfg);
   rule consumer if (mimo.enqReadyN(m));
      Vector#(m, a) v = in.first();
      in.deq();
      mimo.enq(m, v);
   endrule

   method Vector#(mk, a) first() if (mimo.deqReadyN(mk));
      return mimo.first();
   endmethod
   method Action deq() if (mimo.deqReadyN(mk));
      mimo.deq(mk);
   endmethod
   method notEmpty();
      return mimo.deqReadyN(mk);
   endmethod
endmodule

module mkUnfunnelGB#(Clock slowClock, Reset slowReset, Clock fastClock, Reset fastReset, PipeOut#(Vector#(1,a)) in)(PipeOut#(Vector#(k, a)))
   provisos (Bits#(a, asz),
	     Add#(1, a__, k),
	     Add#(1, b__, asz),
	     Add#(1, c__, TMul#(2,k)),
	     Add#(k, d__, TMul#(2,k))
      );
   let k = fromInteger(valueOf(k));

   Gearbox#(1,k,a) gb <- mk1toNGearbox(fastClock, fastReset, slowClock, slowReset);
   PipeIn#(Vector#(1,a)) toGb = toPipeIn(gb);
   PipeOut#(Vector#(k,a)) fromGb = toPipeOut(gb);
   mkConnection(in,toGb);
   return fromGb;
endmodule: mkUnfunnelGB

module mkFunnelPipes#(Vector#(mk, PipeOut#(a)) ins)(Vector#(m, PipeOut#(a)))
   provisos (Mul#(m, k, mk),
	     Bits#(a, asz),
	     Log#(k,ksz)
      );
   let k = fromInteger(valueOf(k));
   let m = fromInteger(valueOf(m));
   let mk = fromInteger(valueOf(mk));

   Vector#(m, FIFOF#(a)) fifos <- replicateM(mkFIFOF);
   for (Integer i = 0; i < m; i = i+1) begin
      Reg#(Bit#(asz)) which <- mkReg(0);
      rule consumer;
	 let index = (which << valueOf(ksz)) + fromInteger(i);
	 let v <- toGet(ins[index]).get();
	 fifos[i].enq(v);
	 which <= (which + 1) % k;
      endrule
   end

   return map(toPipeOut, fifos);
endmodule

module mkFunnelPipes1#(Vector#(k, PipeOut#(a)) ins)(PipeOut#(a))
   provisos (Bits#(a, asz),
	     Log#(k,ksz)
      );
   let k = fromInteger(valueOf(k));
   Reg#(Bit#(ksz)) selector <- mkReg(0);

   method a first();
      return ins[selector].first();
   endmethod
   method Action deq();
      ins[selector].deq();
      if (selector == fromInteger(valueOf(k)-1))
	 selector <= 0;
   else
      selector <= selector + 1;
   endmethod
   method Bool notEmpty();
      return ins[selector].notEmpty();
   endmethod
endmodule

module mkUnfunnelPipes#(Vector#(m, PipeOut#(a)) ins)(Vector#(mk, PipeOut#(a)))
   provisos (Mul#(m, k, mk),
	     Log#(k,ksz),
	     Bits#(a,asz));

   let m = fromInteger(valueOf(m));
   let k = fromInteger(valueOf(k));
   let mk = fromInteger(valueOf(mk));

   Vector#(mk, FIFOF#(a)) fifos <- replicateM(mkFIFOF);
   for (Integer i = 0; i < m; i = i + 1) begin
      Reg#(Bit#(TAdd#(1,ksz))) which <- mkReg(0);
      rule consumer;
	 let index = which + fromInteger(i)*k;
	 let v <- toGet(ins[i]).get();
	 fifos[index].enq(v);
	 which <= (which + 1) % k;
      endrule
   end
   return map(toPipeOut, fifos);
endmodule

module mkRepeat#(UInt#(n) repetitions, PipeOut#(a) inpipe)(PipeOut#(a));
   Reg#(UInt#(n)) count <- mkReg(0);
   method first = inpipe.first;
   method Action deq();
      let c = count + 1;
      if (count == (repetitions - 1)) begin
	 c = 0;
	 inpipe.deq();
      end
      count <= c;
   endmethod
   method notEmpty = inpipe.notEmpty;
endmodule

module mkPipelinedForkVector#(PipeOut#(a) inpipe, Integer id)(UnFunnelPipe#(1,k,a,bpc))
   provisos ( Bits#(a,a__)
	     ,Add#(1,b__,k)
	     ,Log#(k,logk)
	     ,Div#(logk,bpc,stages));
   Vector#(k, FIFOF#(a))  buffs = newVector;
   Vector#(k, PipeOut#(a)) infs = cons(inpipe,replicate(?));
   for(Integer j = 0; j < valueOf(stages); j=j+1)begin
      for(Integer i = 0; i < 2**((j+1)*valueOf(bpc)) && i < valueOf(k); i=i+1)
	 buffs[i] <- mkFIFOF;
      rule xfer;
      	 for(Integer i = 0; i < 2**(j*valueOf(bpc)) && i < valueOf(k); i=i+1) begin
      	    for(Integer l = 0; l < 2**valueOf(bpc) && l < valueOf(k); l=l+1) begin
	       Integer idx = (i*(2**valueOf(bpc)))+l;
      	       if (idx < valueOf(k))
		  buffs[idx].enq(infs[i].first);
      	    end
      	    infs[i].deq;
      	 end
      endrule
      infs = map(toPipeOut, buffs);
   end
   return infs;
endmodule

module mkForkVector#(PipeOut#(a) inpipe)(Vector#(n, PipeOut#(a)))
   provisos (Bits#(a, asz));
   Vector#(n, FIFOF#(a)) fifos <- replicateM(mkFIFOF());
   rule forkelts;
      let v = inpipe.first();
      inpipe.deq;
      for (Integer i = 0; i < valueOf(n); i = i + 1) begin
	 fifos[i].enq(v);
      end
   endrule
   return map(toPipeOut, fifos);
endmodule

module mkSizedForkVector#(Integer size, PipeOut#(a) inpipe)(Vector#(n, PipeOut#(a)))
   provisos (Bits#(a, asz));
   Vector#(n, FIFOF#(a)) fifos <- replicateM(mkSizedFIFOF(size));
   rule forkelts;
      let v = inpipe.first();
      inpipe.deq;
      for (Integer i = 0; i < valueOf(n); i = i + 1) begin
	 fifos[i].enq(v);
      end
   endrule
   return map(toPipeOut, fifos);
endmodule

module mkJoin#(function c f(a av, b bv), PipeOut#(a) apipe, PipeOut#(b) bpipe)(PipeOut#(c));
   method c first();
      let av = apipe.first();
      let bv = bpipe.first();
      return f(av, bv);
   endmethod
   method Action deq();
      apipe.deq();
      bpipe.deq();
   endmethod
   method Bool notEmpty();
      return apipe.notEmpty() && bpipe.notEmpty();
   endmethod
endmodule

module mkJoinBuffered#(function c f(a av, b bv), PipeOut#(a) apipe, PipeOut#(b) bpipe)(PipeOut#(c))
   provisos (Bits#(c, csz));
   FIFOF#(c) joinFifo <- mkFIFOF();
   rule joinrule;
      let av <- toGet(apipe).get();
      let bv <- toGet(bpipe).get();
      joinFifo.enq(f(av, bv));
   endrule
   return toPipeOut(joinFifo);
endmodule

module mkJoinVector#(function b f(Vector#(n, a) av), Vector#(n, PipeOut#(a)) apipes)(PipeOut#(b))
   provisos (Bits#(Vector#(n,a),vasz));
   method b first();
      function a getfirst(PipeOut#(a) pipein); return pipein.first(); endfunction
      Vector#(n,a) vec = map(getfirst, apipes);
      return f(vec);
   endmethod
   method Action deq();
      function a getfirst(PipeOut#(a) pipein); return pipein.first(); endfunction
      for (Integer i = 0; i < valueOf(n); i = i + 1)
	 apipes[i].deq();
   endmethod
   method Bool notEmpty();
      function Bool myand(Bool a, Bool b); return a && b; endfunction
      return foldl(myand, True, map(pipeOutNotEmpty, apipes));
   endmethod
endmodule

function PipeOut#(b) mapPipe(function b f(a av), PipeOut#(a) apipe);
   return (interface PipeOut#(b);
      method b first();
	 let av = apipe.first();
	 return f(av);
      endmethod
      method Action deq();
	 apipe.deq();
      endmethod
      method Bool notEmpty();
	 return apipe.notEmpty();
      endmethod
      endinterface);
endfunction

function PipeIn#(a) mapPipeIn(function b f(a av), PipeIn#(b) apipe);
   return (interface PipeIn#(b);
	      method Action enq(a v);
		 apipe.enq(f(v));
	      endmethod
	      method Bool notFull();
		 return apipe.notFull();
	      endmethod
	   endinterface);
endfunction

// buffered version of mapPipe
module mkMapPipe#(function b f(a av), PipeOut#(a) apipe)(PipeOut#(b))
   provisos (Bits#(b,bsz));
   FIFOF#(b) fifo <- mkFIFOF();
   rule compute;
      let v <- toGet(apipe).get();
      fifo.enq(f(v));
   endrule
   return toPipeOut(fifo);
endmodule

typedef (function tb f(ta x)) CombinePipe#(type ta, type tb);

typeclass ReducePipe#( numeric type n, type a);
   module  mkReducePipe (CombinePipe#(Tuple2#(a,a), a) combinepipe,
			 PipeOut#(Vector#(n,a)) inpipe,
			 PipeOut#(a) ifc);
   module  mkReducePipes (CombinePipe#(Tuple2#(a,a), a) combinepipe,
			  Vector#(n,PipeOut#(a)) inpipe,
			  PipeOut#(a) ifc);
endtypeclass
instance ReducePipe#(1, a);
   module  mkReducePipe (CombinePipe#(Tuple2#(a,a), a) combinepipe,
				 PipeOut#(Vector#(1,a)) inpipe,
				 PipeOut#(a) ifc);
      let pipe = mapPipe(head, inpipe);
      return pipe;
   endmodule
   module  mkReducePipes (CombinePipe#(Tuple2#(a,a), a) combinepipe,
				  Vector#(1,PipeOut#(a)) inpipes,
				  PipeOut#(a) ifc);
      return inpipes[0];
   endmodule
endinstance
instance ReducePipe#(2, a)
   provisos(Bits#(a,a__));
   module  mkReducePipe (CombinePipe#(Tuple2#(a,a), a) combinepipe,
			 PipeOut#(Vector#(2,a)) inpipe,
			 PipeOut#(a) ifc);
      function a foo(Vector#(2,a) invec);
	 return combinepipe(tuple2(invec[0], invec[1]));
      endfunction
      let pipe <- mkMapPipe(foo, inpipe);
      return pipe;
   endmodule
   module  mkReducePipes (CombinePipe#(Tuple2#(a,a), a) combinepipe,
			  Vector#(2,PipeOut#(a)) inpipes,
			  PipeOut#(a) ifc);
      function a foo(Tuple2#(a,a) invec);
	 return combinepipe(invec);
      endfunction
      let pipe <- mkMapPipe(foo, zipPipeOut(inpipes[0], inpipes[1]));
      return pipe;
   endmodule
endinstance

instance ReducePipe#(n, a)
   provisos (Add#(TDiv#(n,2), a__, n),
	     Bits#(Vector#(TDiv#(n,2), a), b__),
	     ReducePipe#(TDiv#(n,2),a),
	     ReducePipe#(TSub#(n, TDiv#(n, 2)), a)
      );
   module  mkReducePipe (CombinePipe#(Tuple2#(a,a), a) combinepipe,
			 PipeOut#(Vector#(n,a)) inpipe,
			 PipeOut#(a) ifc);
      FIFOF#(Vector#(TDiv#(n,2),a)) infifo0 <- mkFIFOF;
      FIFOF#(Vector#(TSub#(n,TDiv#(n,2)),a)) infifo1 <- mkFIFOF;
      rule splitinput;
	 let v = inpipe.first();
	 inpipe.deq();
	 infifo0.enq(takeAt(0, v));
	 infifo1.enq(takeAt(valueOf(TDiv#(n,2)), v));
      endrule
      PipeOut#(Vector#(TDiv#(n,2),a)) inpipe0 = toPipeOut(infifo0);
      PipeOut#(Vector#(TSub#(n,TDiv#(n,2)),a)) inpipe1 = toPipeOut(infifo1);

      PipeOut#(a) p0 <- mkReducePipe(combinepipe, inpipe0);
      PipeOut#(a) p1 <- mkReducePipe(combinepipe, inpipe1);

      function a foo(Tuple2#(a,a) invec);
	 return combinepipe(invec);
      endfunction
      let pipe <- mkMapPipe(foo,zipPipeOut(p0, p1));
      return pipe;
   endmodule

   module  mkReducePipes (CombinePipe#(Tuple2#(a,a), a) combinepipe,
			  Vector#(n, PipeOut#(a)) inpipes,
			  PipeOut#(a) ifc);
      Vector#(TDiv#(n,2),PipeOut#(a)) pipes0 = takeAt(0, inpipes);
      Vector#(TSub#(n,TDiv#(n,2)),PipeOut#(a)) pipes1 = takeAt(valueOf(TDiv#(n,2)), inpipes);

      PipeOut#(a) p0 <- mkReducePipes(combinepipe, pipes0);
      PipeOut#(a) p1 <- mkReducePipes(combinepipe, pipes1);

      function a foo(Tuple2#(a,a) invec);
	 return combinepipe(invec);
      endfunction
      let pipe <- mkMapPipe(foo,zipPipeOut(p0, p1));
      return pipe;
   endmodule
endinstance

interface FirstLastPipe#(type a);
   interface PipeOut#(Tuple2#(Bool,Bool)) pipe;
   method Action start(a count);
endinterface

module mkFirstLastPipe(FirstLastPipe#(a))
   provisos (Bits#(a,asz), Ord#(a), Arith#(a), Eq#(a));
   Reg#(a) countReg <- mkReg(0);
   Reg#(Bool) firstReg <- mkReg(False);
   Reg#(Bool) lastReg <- mkReg(False);
   interface PipeOut pipe;
      method Tuple2#(Bool, Bool) first();
	 return tuple2(firstReg, lastReg);
      endmethod
      method Action deq() if (countReg > 0);
	 firstReg <= False;
	 let c = countReg - 1;
	 if (c == 1)
	    lastReg <= True;
	 countReg <= c;
      endmethod
      method Bool notEmpty();
	 return countReg > 0;
      endmethod
   endinterface
   method Action start(a count) if (countReg == 0);
      firstReg <= True;
      lastReg <= False;
      countReg <= count;
   endmethod
endmodule

typedef struct {
   a xbase;
   a xlimit;
   a xstep;
} RangeConfig#(type a) deriving (Bits, FShow);

interface RangePipeIfc#(type a);
   interface PipeOut#(a) pipe;
   method Bool isFirst();
   method Bool isLast();
   method Action start(RangeConfig#(a) cfg);
endinterface

module mkRangePipeOut(RangePipeIfc#(a)) provisos (Arith#(a), Bits#(a,awidth), Eq#(a), Ord#(a));
   Reg#(a) x <- mkReg(1);
   Reg#(a) xbase <- mkReg(0);
   Reg#(a) xstep <- mkReg(1);
   // inclusive limit
   Reg#(a) xlimit <- mkReg(0);
   Reg#(Bool) first <- mkReg(False);
   Reg#(Bool) last <- mkReg(False);
   Bool verbose = False;
   interface PipeOut pipe;
      method a first();
	 return x;
      endmethod
      method Action deq if (x <= xlimit);
	 x <= x + xstep;
	 first <= False;
	 last <= (x+xstep >= xlimit);
      endmethod
      method Bool notEmpty();
	 return (x <= xlimit);
      endmethod
   endinterface
   method Action start(RangeConfig#(a) cfg) if (x > xlimit);
      x <= cfg.xbase;
      xbase <= cfg.xbase;
      xstep <= cfg.xstep;

      // inclusive limit, should update interface accordingly
      xlimit <= cfg.xlimit-cfg.xstep;

      first <= True;
      last <= (cfg.xbase+cfg.xstep >= cfg.xlimit);
      if (verbose) $display("mkRangePipeOut xbase=%d xstep=%d cfg.xlimit=%d xlimit=%d last=%d", cfg.xbase, cfg.xstep, cfg.xlimit, (cfg.xlimit-cfg.xstep), (cfg.xbase+cfg.xstep >= cfg.xlimit));
   endmethod
   method Bool isFirst() = first;
   method Bool isLast() = last;
endmodule: mkRangePipeOut


minor edit

minor edit

// Copyright (c) 2014-2015 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

package Pcie3EndpointX7;

import Clocks            ::*;
import Vector            ::*;
import Connectable       ::*;
import GetPut            ::*;
import Reserved          ::*;
import TieOff            ::*;
import DefaultValue      ::*;
import DReg              ::*;
import Gearbox           ::*;
import FIFO              ::*;
import FIFOF             ::*;
import SpecialFIFOs      ::*;
import ClientServer      ::*;
import Real              ::*;
import XilinxVirtex7PCIE ::*;
import BUtils            ::*;
import Probe             ::*;

import ConnectalClocks   ::*;
import ConnectalXilinxCells   ::*;
import XilinxCells       ::*;
import PCIE              ::*;
import PCIEWRAPPER3      ::*;
import Bufgctrl           ::*;
import PcieGearbox       :: *;

interface PcieEndpointX7#(numeric type lanes);
   interface PciewrapPci_exp#(lanes)           pcie;
   interface PciewrapUser#(lanes)              user;
   interface PciewrapPipe#(lanes)              pipe;
   interface PciewrapCommon#(lanes)            common;
   interface Server#(TLPData#(16), TLPData#(16)) tlpr;
   interface Server#(TLPData#(16), TLPData#(16)) tlpc;
   interface Put#(Tuple2#(Bit#(64),Bit#(32)))  interruptRequest;
   interface Clock epPcieClock;
   interface Reset epPcieReset;
   interface Clock epPortalClock;
   interface Reset epPortalReset;
   interface Clock epDerivedClock;
   interface Reset epDerivedReset;
endinterface

typedef struct {
   Bit #(256)     data;
   Bool          sop;
   Bool          eop;
   Bit #(8)      keep;
   TLPFirstDWBE  first_be;
   TLPFirstDWBE  last_be;
} AxiStCq deriving (Bits, Eq);

typedef struct {
   Bit #(256)     data;
   Bit #(8)      keep;
   Bool          last;
} AxiStCc deriving (Bits, Eq);

typedef struct {
   Bit #(256)     data;
   Bool          last;
   Bit #(8)      keep;
   Bit #(4)      first_be;
   Bit #(4)      last_be;
} AxiStRq deriving (Bits, Eq);

typedef struct {
   Bit #(256)     data;
   Bool          sop;
   Bool          eop;
   Bit #(8)      keep;
   Bit #(8)      be;
} AxiStRc deriving (Bits, Eq);

function TLPData#(16) convertCQDescriptorToTLP16(CQDescriptor desc, Bit#(32) data, TLPFirstDWBE first, TLPLastDWBE last);
   TLPMemoryIO3DWHeader header = defaultValue;
   header.format     = tpl_1(convertCQReqTypeToTLPFmtType(desc.reqtype));
   header.pkttype    = tpl_2(convertCQReqTypeToTLPFmtType(desc.reqtype));
   header.tclass     = desc.tclass;
   header.relaxed    = desc.relaxed;
   header.nosnoop    = desc.nosnoop;
   header.length     = (desc.dwcount == 1024) ? 0 : truncate(desc.dwcount);
   header.reqid      = desc.reqid;
   header.tag        = desc.tag;
   header.lastbe     = last;
   header.firstbe    = first;
   header.addr       = truncate(desc.address);
   header.data       = convertDW(data);

   Bool is3DW = isReadReqType(desc.reqtype);
   Bool is3Or4DW = isReadReqType(desc.reqtype) || (desc.dwcount == 1);

   TLPData#(16) retval = defaultValue;
   retval.sof   = True;
   retval.eof   = is3Or4DW;
   retval.hit   = (1 << pack(desc.barid));
   retval.data  = pack(header);
   retval.be    = (is3DW ? 16'hFFF0 : 16'hFFFF);

   return retval;
endfunction

function TLPData#(16) convertRCDescriptorToTLP16(RCDescriptor desc, Bit#(32) data);
   TLPCompletionHeader header = defaultValue;
   header.tclass    = desc.tclass;
   header.relaxed   = desc.relaxed;
   header.nosnoop   = desc.nosnoop;
   header.cmplid    = desc.complid;
   header.tag       = desc.tag;
   header.reqid     = desc.reqid;
   header.poison    = desc.poisoned;
   header.cstatus   = desc.status;
   header.length    = (desc.dwcount == 1024) ? 0 : truncate(desc.dwcount);
   header.bytecount = (desc.bytecount == 4096) ? 0 : truncate(desc.bytecount);
   header.loweraddr = truncate(desc.loweraddr);
   header.data      = convertDW(data);

   Bool is3DW = (desc.dwcount == 0);
   Bool is3Or4DW = (desc.dwcount == 0) || (desc.dwcount == 1);
   TLPData#(16) retval = defaultValue;
   retval.sof   = True;
   retval.eof   = is3Or4DW;
   retval.hit   = 1; // XXX
   retval.data  = pack(header);
   retval.be    = (is3DW ? 16'hFFF0 : 16'hFFFF);

   return retval;
endfunction

`ifdef BOARD_vc709
typedef 8 PcieLanes;
typedef 8 NumLeds;
`endif
`ifdef BOARD_nfsume
typedef 8 PcieLanes;
typedef 2 NumLeds;
`endif

(* synthesize *)
module mkPcieEndpointX7(PcieEndpointX7#(PcieLanes));

   PCIEParams params = defaultValue;
   Clock defaultClock <- exposeCurrentClock();
   Reset defaultReset <- exposeCurrentReset();
   Reset defaultResetInverted <- mkResetInverter(defaultReset, clocked_by defaultClock);
   PcieWrap#(PcieLanes) pcie_ep <- mkPcieWrap(defaultClock, defaultResetInverted);

   // The PCIe endpoint exports full (250MHz) and half-speed (125MHz) clocks
   Clock pcieClock250 = pcie_ep.user_clk;
   Reset user_reset_n <- mkResetInverter(pcie_ep.user_reset, clocked_by pcie_ep.user_clk);
   Reset pcieReset250 <- mkAsyncReset(4, user_reset_n, pcieClock250);

   ClockGenerator7Params     clkgenParams = defaultValue;
   clkgenParams.clkin1_period    = 4.000; //  250MHz
   clkgenParams.clkin1_period    = 4.000;
   clkgenParams.clkin_buffer     = False;
   clkgenParams.clkfbout_mult_f  = 4.000; // 1000MHz
   clkgenParams.clkout0_divide_f = derivedClockPeriod;
   clkgenParams.clkout1_divide     = round(mainClockPeriod);
   clkgenParams.clkout1_duty_cycle = 0.5;
   clkgenParams.clkout1_phase      = 0.0000;
   ClockGenerator7           clkgen <- mkClockGenerator7(clkgenParams, clocked_by pcieClock250, reset_by pcieReset250);
   Clock mainClock = clkgen.clkout1;
   Reset mainReset <- mkAsyncReset(4, pcieReset250, mainClock);
   Clock derivedClock = clkgen.clkout0;
   Reset derivedReset <- mkAsyncReset(4, pcieReset250, derivedClock);
   Reset user_reset <- mkAsyncReset(2, pcie_ep.user_reset, pcie_ep.user_clk);

   // FIFOS
   FIFOF#(AxiStCq) fAxiCq <- mkFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   FIFOF#(AxiStRc) fAxiRc <- mkFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);

   FIFOF#(AxiStRq) fAxiRq <- mkFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   FIFOF#(AxiStCc) fAxiCc <- mkFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);

   FIFOF#(TLPData#(16)) fcq <- mkFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   FIFOF#(TLPData#(16)) frc <- mkFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   FIFOF#(TLPData#(16)) fcc <- mkFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   FIFOF#(TLPData#(16)) frq <- mkFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);

   FIFOF#(Tuple2#(Bit#(64),Bit#(32))) intrFifo <- mkFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);

   // Drive s_axis_rq
   let rq_txready = (pcie_ep.s_axis_rq.tready != 0 && fAxiRq.notEmpty);

   //(* fire_when_enabled, no_implicit_conditions *)
   rule drive_axi_rq if (rq_txready);
      let info = fAxiRq.first; fAxiRq.deq;
      pcie_ep.s_axis_rq.tvalid(1);
      pcie_ep.s_axis_rq.tlast(pack(info.last));
      pcie_ep.s_axis_rq.tdata(info.data);
      pcie_ep.s_axis_rq.tkeep(info.keep);
      pcie_ep.s_axis_rq.tuser({0, info.last_be, info.first_be});
   endrule

   (* fire_when_enabled, no_implicit_conditions *)
   rule drive_axi_rq2 if (!rq_txready);
      pcie_ep.s_axis_rq.tvalid(0);
      pcie_ep.s_axis_rq.tlast(0);
      pcie_ep.s_axis_rq.tdata(0);
      pcie_ep.s_axis_rq.tkeep(0);
      pcie_ep.s_axis_rq.tuser(0);
   endrule

   // Drive s_axis_cc
   let cc_txready = (pcie_ep.s_axis_cc.tready != 0 && fAxiCc.notEmpty);

   //(* fire_when_enabled, no_implicit_conditions *)
   rule drive_axi_cc if (cc_txready);
      let info = fAxiCc.first; fAxiCc.deq;
      $display("drive axi_cc, data: %h, keep: %h, last: %h", info.data, info.keep, info.last);
      pcie_ep.s_axis_cc.tvalid(1);
      pcie_ep.s_axis_cc.tlast(pack(info.last));
      pcie_ep.s_axis_cc.tdata(info.data);
      pcie_ep.s_axis_cc.tkeep(info.keep);
      pcie_ep.s_axis_cc.tuser(0);
   endrule

   (* fire_when_enabled, no_implicit_conditions *)
   rule drive_axi_cc2 if (!cc_txready);
      pcie_ep.s_axis_cc.tvalid(0);
      pcie_ep.s_axis_cc.tlast(0);
      pcie_ep.s_axis_cc.tdata(0);
      pcie_ep.s_axis_cc.tkeep(0);
      pcie_ep.s_axis_cc.tuser(0);
   endrule

   (* fire_when_enabled, no_implicit_conditions *)
   rule drive_axi_rc_ready;
      pcie_ep.m_axis_rc.tready (duplicate (pack (fAxiRc.notFull)));
   endrule

   // Drive m_axis_rc
   (* fire_when_enabled *)
   rule sink_axi_rc if (pcie_ep.m_axis_rc.tvalid != 0 && fAxiRc.notFull);
      let rc = AxiStRc {data:pcie_ep.m_axis_rc.tdata,
			sop: unpack (pcie_ep.m_axis_rc.tuser [32]),         // tuser.is_sof_0
			eop: unpack (pcie_ep.m_axis_rc.tlast),
			keep:pcie_ep.m_axis_rc.tkeep,
			be:  truncate (pcie_ep.m_axis_rc.tuser [31:0])};    // tuser.byte_en
      fAxiRc.enq (rc);
   endrule

   (* fire_when_enabled, no_implicit_conditions *)
   rule drive_axi_cq_ready;
      pcie_ep.m_axis_cq.tready (duplicate (pack (fAxiCq.notFull)));
   endrule

   (* fire_when_enabled *)
   rule sink_axi_cq if (pcie_ep.m_axis_cq.tvalid != 0 && fAxiCq.notFull);
      let cq = AxiStCq {data:     pcie_ep.m_axis_cq.tdata,
			sop:      unpack (pcie_ep.m_axis_cq.tuser [40]),  // tuser.sop
			eop:      unpack (pcie_ep.m_axis_cq.tlast),
			keep:     pcie_ep.m_axis_cq.tkeep,
			first_be: pcie_ep.m_axis_cq.tuser [3:0],    // tuser.first_be,
			last_be:  pcie_ep.m_axis_cq.tuser [7:4]};   // tuser.last_be
      fAxiCq.enq (cq);
   endrule

   // CQ.
   CQDescriptor cq_desc = unpack(fAxiCq.first.data [127:0]);

   rule rl_cq_wr_header (fAxiCq.first.sop && ((cq_desc.reqtype == MEMORY_WRITE) || (cq_desc.reqtype == IO_WRITE)));
      Bit#(32) data = fAxiCq.first.data[159:128];
      // get data;
      TLPData#(16) tlp16 = convertCQDescriptorToTLP16(cq_desc, data, fAxiCq.first.first_be, fAxiCq.first.last_be);
      $display("cq_desc.reqtype=%h", cq_desc.reqtype);
      // enqueue?
      fcq.enq(tlp16);
      fAxiCq.deq;
   endrule

   // Write data payload, no data remaining
   rule rl_cq_wr_payload((!fAxiCq.first.sop));
      fAxiCq.deq;
   endrule

   // Write data payload, 1 to 3 DWs remaining
   // Write data payload, 4 or more DWs remaining

   rule rl_cq_rd_header (fAxiCq.first.sop && ((cq_desc.reqtype == MEMORY_READ) || (cq_desc.reqtype == IO_READ)));
      Bit#(32) data = 0;
      TLPData#(16) tlp16 = convertCQDescriptorToTLP16(cq_desc, data, fAxiCq.first.first_be, fAxiCq.first.last_be);
      $display("rl_cq_rd_header: cq_desc = %16x", cq_desc);
      fcq.enq(tlp16);
      fAxiCq.deq;
   endrule

   // RC.
   Reg#(DWCount) rc_dwcount <- mkRegU(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   Reg#(Bool) rc_even <- mkReg(True, clocked_by pcie_ep.user_clk, reset_by user_reset_n);

   rule rl_rc_header (fAxiRc.first.sop && rc_even);
      RCDescriptor rc_desc = unpack(fAxiRc.first.data [95:0]);
      // RC descriptor always 96 bytes with first data word in bits 127:96
      Bit#(32) data = byteSwap(fAxiRc.first.data[127:96]);
      TLPData#(16) tlp16 = convertRCDescriptorToTLP16(rc_desc, data);
      rc_dwcount <= (rc_desc.dwcount == 0) ? 0 : rc_desc.dwcount - 1;
      frc.enq(tlp16);
      let even = False;
      if (rc_desc.dwcount == 0 || rc_desc.dwcount == 1) begin
        fAxiRc.deq;
        even = True;
      end
      rc_even <= even;
   endrule

   rule rl_rc_data ((!rc_even || !(fAxiRc.first.sop)) && (rc_dwcount != 0));
      Bit#(16) be16;
      case (rc_dwcount)
         1: be16 = 16'hF000;
         2: be16 = 16'hFF00;
         3: be16 = 16'hFFF0;
         default: be16 = 16'hFFFF;
      endcase
      let last = (rc_dwcount <= 4);
      let dwcount = rc_dwcount - 4;
      if (last) dwcount = 0;
      let data = (rc_even) ? fAxiRc.first.data[127:0]: fAxiRc.first.data[255:128];
      TLPData#(16) tlp16 = TLPData{sof: False,
                                   eof: last,
                                   hit: 0,
                                   be: be16,
                                   data: pack(data)};
      frc.enq(tlp16);
      if (last || !rc_even) begin
         fAxiRc.deq;
      end
      rc_dwcount <= dwcount;
      rc_even <= (last) ? True : !rc_even;
   endrule

   Reg#(DWCount) cc_dwcount <- mkReg(0, clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   FIFOF#(TLPData#(16)) fcc_tlps <- mkFIFOF (clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   // CC.
   rule get_cc_tlps;
      let tlp <- toGet(fcc).get;
      fcc_tlps.enq(tlp);
   endrule

   rule rl_cc_header(fcc_tlps.first.sof);
      match { .cc_desc, .dw} = convertTLP16ToCCDescriptor(fcc_tlps.first);
      cc_dwcount <= cc_desc.dwcount - 1;
      fAxiCc.enq(AxiStCc {data: zeroExtend({dw, pack(cc_desc)[95:0]}),
                       last: fcc_tlps.first.eof,
                       keep: 8'h0F});
      fcc_tlps.deq;
   endrule

   rule rl_cc_data((!fcc_tlps.first.sof) && (cc_dwcount != 0));
      Bit#(256) x = zeroExtend(fcc_tlps.first.data); //FIXME
      fAxiCc.enq(AxiStCc {data: {x},
                       last: cc_dwcount <= 4,
                       keep: (cc_dwcount == 3) ? 8'h0F : 8'hFF});
      fcc_tlps.deq;
   endrule

   // RQ.
   FIFOF#(TLPData#(16)) frq_tlps <- mkFIFOF (clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   Reg#(Bit#(4)) rq_first_be <- mkReg(0, clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   Reg#(Bit#(4)) rq_last_be <- mkReg(0, clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   Reg#(Bool)    rq_even    <- mkRegU(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   Reg#(DWCount) rq_dwcount <- mkReg(0, clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   Reg#(AxiStRq) rq_rq <- mkRegU(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   rule rl_rq_tlps;
      let tlp <- toGet(frq).get;
      frq_tlps.enq(tlp);
   endrule

   rule rl_rq_header if (frq_tlps.first.sof);
      TLPData#(16) tlp <- toGet(frq_tlps).get();
      match { .rq_desc, .first_be, .last_be, .mdata} = convertTLP16ToRQDescriptor(tlp);

      let dwcount = ((rq_desc.reqtype == MEMORY_WRITE) ? rq_desc.dwcount : 0);
      rq_dwcount <= dwcount;
      rq_even <= False;
      rq_first_be <= first_be;
      rq_last_be <= last_be;
      let last = (rq_desc.reqtype == MEMORY_WRITE) ? (dwcount <= 4) : True;
      let rq = AxiStRq {data: zeroExtend(pack(rq_desc)), //FIXME:
			last: last,
			keep: 8'h0F,
			first_be: first_be,
			last_be: last_be};
      if (rq_desc.reqtype == MEMORY_WRITE)
	 rq_rq <= rq;
      else
	 fAxiRq.enq(rq);
   endrule

   // more data
   rule rl_rq_data if (rq_dwcount != 0);
      TLPData#(16) tlp <- toGet(frq_tlps).get();
      let rq = rq_rq;
      let last = (rq_dwcount <= 4);
      let dwcount = rq_dwcount - 4;
      Bit#(8) keep;
      if (last)
	dwcount = 0;
      if (rq_even) begin
	 rq.data[127:0] = tlp.data;
	case (rq_dwcount)
	   1: keep = 8'h01;
	   2: keep = 8'h03;
	   3: keep = 8'h07;
	   default: keep = 8'h0f;
	endcase
      end
      else begin
	rq.data[255:128] = tlp.data;
	case (rq_dwcount)
	   1: keep = 8'h1f;
	   2: keep = 8'h3f;
	   3: keep = 8'h7f;
	   default: keep = 8'hff;
	endcase
      end
      rq.last = last;
      rq.first_be = rq_first_be;
      rq.last_be = rq_last_be;
      rq.keep = keep;
      if (!rq_even || last)
	 fAxiRq.enq(rq);
      if (rq_even)
	rq_rq <= rq;
      rq_dwcount <= dwcount;
      rq_even <= (last) ? False : !rq_even;
   endrule

   FIFO#(Bool) intrMutex <- mkFIFO1(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   rule rl_intr;
      let addr = 0;
      let data = 0;
      let enable = 0;
      if (intrFifo.notEmpty() && pcie_ep.cfg.interrupt_msix_enable[0] == 1) begin
	 match { .a, .d } <- toGet(intrFifo).get();
	 addr = a;
	 data = d;
	 enable = 1;
	 intrMutex.enq(True);
      end
      pcie_ep.cfg.interrupt_msix_address(addr);
      pcie_ep.cfg.interrupt_msix_data(data);
      pcie_ep.cfg.interrupt_msix_int(enable);
   endrule: rl_intr
   rule rl_intr_sent if (pcie_ep.cfg.interrupt_msix_sent() == 1|| pcie_ep.cfg.interrupt_msix_fail() == 1);
      intrMutex.deq();
   endrule

   // The PCIE endpoint is processing Gen3 descriptors at 250MHz. The
   // AXI bridge is accepting TLPData#(16)s at 250 MHz. The
   // conversion uses half of Gen3 descriptor.
   //mkConnection(tlp8, gb.tlp, clocked_by portalClock, reset_by portalReset);

   let portalClock = (mainClockPeriod == pcieClockPeriod) ? pcieClock250 : mainClock;
   let portalReset = (mainClockPeriod == pcieClockPeriod) ? pcieReset250 : mainReset;

   interface Server tlpr;
      interface request = toPut(frq);
      interface response = toGet(frc);
   endinterface
   // Requests from other PCIe devices
   interface Server tlpc;
      interface request = toPut(fcc);
      interface response = toGet(fcq);
   endinterface
   interface interruptRequest = toPut(intrFifo);
   interface pcie    = pcie_ep.pci_exp;
   interface Pcie3wrapUser user = pcie_ep.user;
   interface PciewrapPipe pipe = pcie_ep.pipe;
   interface PciewrapCommon common= pcie_ep.common;
   interface Clock epPcieClock = pcieClock250;
   interface Reset epPcieReset = pcieReset250;
   interface Clock epPortalClock = portalClock;
   interface Reset epPortalReset = portalReset;
   interface Clock epDerivedClock = derivedClock;
   interface Reset epDerivedReset = derivedReset;
endmodule: mkPcieEndpointX7

endpackage: Pcie3EndpointX7

// Copyright (c) 2014-2015 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

package Pcie3EndpointX7;

import Clocks            ::*;
import Vector            ::*;
import Connectable       ::*;
import GetPut            ::*;
import Reserved          ::*;
import TieOff            ::*;
import DefaultValue      ::*;
import DReg              ::*;
import Gearbox           ::*;
import FIFO              ::*;
import FIFOF             ::*;
import SpecialFIFOs      ::*;
import ClientServer      ::*;
import Real              ::*;
import XilinxVirtex7PCIE ::*;
import BUtils            ::*;

import ConnectalClocks   ::*;
import ConnectalXilinxCells   ::*;
import XilinxCells       ::*;
import PCIE              ::*;
import PCIEWRAPPER3      ::*;
import Bufgctrl           ::*;
import PcieGearbox       :: *;

interface PcieEndpointX7#(numeric type lanes);
   interface PciewrapPci_exp#(lanes)           pcie;
   interface PciewrapUser#(lanes)              user;
   interface PciewrapPipe#(lanes)              pipe;
   interface PciewrapCommon#(lanes)            common;
   interface Server#(TLPData#(16), TLPData#(16)) tlpr;
   interface Server#(TLPData#(16), TLPData#(16)) tlpc;
   interface Put#(Tuple2#(Bit#(64),Bit#(32)))  interruptRequest;
   interface Clock epPcieClock;
   interface Reset epPcieReset;
   interface Clock epPortalClock;
   interface Reset epPortalReset;
   interface Clock epDerivedClock;
   interface Reset epDerivedReset;
endinterface

typedef struct {
   Bit #(256)     data;
   Bool          sop;
   Bool          eop;
   Bit #(8)      keep;
   TLPFirstDWBE  first_be;
   TLPFirstDWBE  last_be;
} AxiStCq deriving (Bits, Eq);

typedef struct {
   Bit #(256)     data;
   Bit #(8)      keep;
   Bool          last;
} AxiStCc deriving (Bits, Eq);

typedef struct {
   Bit #(256)     data;
   Bool          last;
   Bit #(8)      keep;
   Bit #(4)      first_be;
   Bit #(4)      last_be;
} AxiStRq deriving (Bits, Eq);

typedef struct {
   Bit #(256)     data;
   Bool          sop;
   Bool          eop;
   Bit #(8)      keep;
   Bit #(8)      be;
} AxiStRc deriving (Bits, Eq);

function TLPData#(16) convertCQDescriptorToTLP16(CQDescriptor desc, Bit#(32) data, TLPFirstDWBE first, TLPLastDWBE last);
   TLPMemoryIO3DWHeader header = defaultValue;
   header.format     = tpl_1(convertCQReqTypeToTLPFmtType(desc.reqtype));
   header.pkttype    = tpl_2(convertCQReqTypeToTLPFmtType(desc.reqtype));
   header.tclass     = desc.tclass;
   header.relaxed    = desc.relaxed;
   header.nosnoop    = desc.nosnoop;
   header.length     = (desc.dwcount == 1024) ? 0 : truncate(desc.dwcount);
   header.reqid      = desc.reqid;
   header.tag        = desc.tag;
   header.lastbe     = last;
   header.firstbe    = first;
   header.addr       = truncate(desc.address);
   header.data       = convertDW(data);

   Bool is3DW = isReadReqType(desc.reqtype);
   Bool is3Or4DW = isReadReqType(desc.reqtype) || (desc.dwcount == 1);

   TLPData#(16) retval = defaultValue;
   retval.sof   = True;
   retval.eof   = is3Or4DW;
   retval.hit   = (1 << pack(desc.barid));
   retval.data  = pack(header);
   retval.be    = (is3DW ? 16'hFFF0 : 16'hFFFF);

   return retval;
endfunction

function TLPData#(16) convertRCDescriptorToTLP16(RCDescriptor desc, Bit#(32) data);
   TLPCompletionHeader header = defaultValue;
   header.tclass    = desc.tclass;
   header.relaxed   = desc.relaxed;
   header.nosnoop   = desc.nosnoop;
   header.cmplid    = desc.complid;
   header.tag       = desc.tag;
   header.reqid     = desc.reqid;
   header.poison    = desc.poisoned;
   header.cstatus   = desc.status;
   header.length    = (desc.dwcount == 1024) ? 0 : truncate(desc.dwcount);
   header.bytecount = (desc.bytecount == 4096) ? 0 : truncate(desc.bytecount);
   header.loweraddr = truncate(desc.loweraddr);
   header.data      = convertDW(data);

   Bool is3DW = (desc.dwcount == 0);
   Bool is3Or4DW = (desc.dwcount == 0) || (desc.dwcount == 1);
   TLPData#(16) retval = defaultValue;
   retval.sof   = True;
   retval.eof   = is3Or4DW;
   retval.hit   = 1; // XXX
   retval.data  = pack(header);
   retval.be    = (is3DW ? 16'hFFF0 : 16'hFFFF);

   return retval;
endfunction

`ifdef BOARD_vc709
typedef 8 PcieLanes;
typedef 8 NumLeds;
`endif
`ifdef BOARD_nfsume
typedef 8 PcieLanes;
typedef 2 NumLeds;
`endif

(* synthesize *)
module mkPcieEndpointX7(PcieEndpointX7#(PcieLanes));

   PCIEParams params = defaultValue;
   Clock defaultClock <- exposeCurrentClock();
   Reset defaultReset <- exposeCurrentReset();
   Reset defaultResetInverted <- mkResetInverter(defaultReset, clocked_by defaultClock);
   PcieWrap#(PcieLanes) pcie_ep <- mkPcieWrap(defaultClock, defaultResetInverted);

   // The PCIe endpoint exports full (250MHz) and half-speed (125MHz) clocks
   Clock pcieClock250 = pcie_ep.user_clk;
   Reset user_reset_n <- mkResetInverter(pcie_ep.user_reset, clocked_by pcie_ep.user_clk);
   Reset pcieReset250 <- mkAsyncReset(4, user_reset_n, pcieClock250);

   ClockGenerator7Params     clkgenParams = defaultValue;
   clkgenParams.clkin1_period    = 4.000; //  250MHz
   clkgenParams.clkin1_period    = 4.000;
   clkgenParams.clkin_buffer     = False;
   clkgenParams.clkfbout_mult_f  = 4.000; // 1000MHz
   clkgenParams.clkout0_divide_f = derivedClockPeriod;
   clkgenParams.clkout1_divide     = round(mainClockPeriod);
   clkgenParams.clkout1_duty_cycle = 0.5;
   clkgenParams.clkout1_phase      = 0.0000;
   ClockGenerator7           clkgen <- mkClockGenerator7(clkgenParams, clocked_by pcieClock250, reset_by pcieReset250);
   Clock mainClock = clkgen.clkout1;
   Reset mainReset <- mkAsyncReset(4, pcieReset250, mainClock);
   Clock derivedClock = clkgen.clkout0;
   Reset derivedReset <- mkAsyncReset(4, pcieReset250, derivedClock);
   Reset user_reset <- mkAsyncReset(2, pcie_ep.user_reset, pcie_ep.user_clk);

   FIFOF#(AxiStCq) fAxiCq <- mkFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   FIFOF#(AxiStRc) fAxiRc <- mkFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);

   FIFOF#(AxiStRq) fAxiRq <- mkFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   FIFOF#(AxiStCc) fAxiCc <- mkFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);

   FIFOF#(TLPData#(16)) fcq <- mkFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   FIFOF#(TLPData#(16)) frc <- mkFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   FIFOF#(TLPData#(16)) fcc <- mkFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   FIFOF#(TLPData#(16)) frq <- mkFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);

   FIFOF#(Tuple2#(Bit#(64),Bit#(32))) intrFifo <- mkFIFOF(clocked_by pcie_ep.user_clk, reset_by user_reset_n);

   // Drive s_axis_rq
   let rq_txready = (pcie_ep.s_axis_rq.tready != 0 && fAxiRq.notEmpty);

   //(* fire_when_enabled, no_implicit_conditions *)
   rule drive_axi_rq if (rq_txready);
      let info = fAxiRq.first; fAxiRq.deq;
      pcie_ep.s_axis_rq.tvalid(1);
      pcie_ep.s_axis_rq.tlast(pack(info.last));
      pcie_ep.s_axis_rq.tdata(info.data);
      pcie_ep.s_axis_rq.tkeep(info.keep);
      pcie_ep.s_axis_rq.tuser({0, info.last_be, info.first_be});
   endrule

   (* fire_when_enabled, no_implicit_conditions *)
   rule drive_axi_rq2 if (!rq_txready);
      pcie_ep.s_axis_rq.tvalid(0);
      pcie_ep.s_axis_rq.tlast(0);
      pcie_ep.s_axis_rq.tdata(0);
      pcie_ep.s_axis_rq.tkeep(0);
      pcie_ep.s_axis_rq.tuser(0);
   endrule

   // Drive s_axis_cc
   let cc_txready = (pcie_ep.s_axis_cc.tready != 0 && fAxiCc.notEmpty);

   //(* fire_when_enabled, no_implicit_conditions *)
   rule drive_axi_cc if (cc_txready);
      let info = fAxiCc.first; fAxiCc.deq;
      $display("drive axi_cc, data: %h, keep: %h, last: %h", info.data, info.keep, info.last);
      pcie_ep.s_axis_cc.tvalid(1);
      pcie_ep.s_axis_cc.tlast(pack(info.last));
      pcie_ep.s_axis_cc.tdata(info.data);
      pcie_ep.s_axis_cc.tkeep(info.keep);
      pcie_ep.s_axis_cc.tuser(0);
   endrule

   (* fire_when_enabled, no_implicit_conditions *)
   rule drive_axi_cc2 if (!cc_txready);
      pcie_ep.s_axis_cc.tvalid(0);
      pcie_ep.s_axis_cc.tlast(0);
      pcie_ep.s_axis_cc.tdata(0);
      pcie_ep.s_axis_cc.tkeep(0);
      pcie_ep.s_axis_cc.tuser(0);
   endrule

   (* fire_when_enabled, no_implicit_conditions *)
   rule drive_axi_rc_ready;
      pcie_ep.m_axis_rc.tready (duplicate (pack (fAxiRc.notFull)));
   endrule

   // Drive m_axis_rc
   (* fire_when_enabled *)
   rule sink_axi_rc if (pcie_ep.m_axis_rc.tvalid != 0 && fAxiRc.notFull);
      let rc = AxiStRc {data:pcie_ep.m_axis_rc.tdata,
			sop: unpack (pcie_ep.m_axis_rc.tuser [32]),         // tuser.is_sof_0
			eop: unpack (pcie_ep.m_axis_rc.tlast),
			keep:pcie_ep.m_axis_rc.tkeep,
			be:  truncate (pcie_ep.m_axis_rc.tuser [31:0])};    // tuser.byte_en
      fAxiRc.enq (rc);
   endrule

   (* fire_when_enabled, no_implicit_conditions *)
   rule drive_axi_cq_ready;
      pcie_ep.m_axis_cq.tready (duplicate (pack (fAxiCq.notFull)));
   endrule

   (* fire_when_enabled *)
   rule sink_axi_cq if (pcie_ep.m_axis_cq.tvalid != 0 && fAxiCq.notFull);
      let cq = AxiStCq {data:     pcie_ep.m_axis_cq.tdata,
			sop:      unpack (pcie_ep.m_axis_cq.tuser [40]),  // tuser.sop
			eop:      unpack (pcie_ep.m_axis_cq.tlast),
			keep:     pcie_ep.m_axis_cq.tkeep,
			first_be: pcie_ep.m_axis_cq.tuser [3:0],    // tuser.first_be,
			last_be:  pcie_ep.m_axis_cq.tuser [7:4]};   // tuser.last_be
      fAxiCq.enq (cq);
   endrule

   // CQ.
   CQDescriptor cq_desc = unpack(fAxiCq.first.data [127:0]);

   rule rl_cq_wr_header (fAxiCq.first.sop && ((cq_desc.reqtype == MEMORY_WRITE) || (cq_desc.reqtype == IO_WRITE)));
      Bit#(32) data = fAxiCq.first.data[159:128];
      // get data;
      TLPData#(16) tlp16 = convertCQDescriptorToTLP16(cq_desc, data, fAxiCq.first.first_be, fAxiCq.first.last_be);
      $display("cq_desc.reqtype=%h", cq_desc.reqtype);
      // enqueue?
      fcq.enq(tlp16);
      fAxiCq.deq;
   endrule

   // Write data payload, no data remaining
   rule rl_cq_wr_payload((!fAxiCq.first.sop));
      fAxiCq.deq;
   endrule

   // Write data payload, 1 to 3 DWs remaining
   // Write data payload, 4 or more DWs remaining

   rule rl_cq_rd_header (fAxiCq.first.sop && ((cq_desc.reqtype == MEMORY_READ) || (cq_desc.reqtype == IO_READ)));
      Bit#(32) data = 0;
      TLPData#(16) tlp16 = convertCQDescriptorToTLP16(cq_desc, data, fAxiCq.first.first_be, fAxiCq.first.last_be);
      $display("rl_cq_rd_header: cq_desc = %16x", cq_desc);
      fcq.enq(tlp16);
      fAxiCq.deq;
   endrule

   // RC.
   Reg#(DWCount) rc_dwcount <- mkRegU(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   Reg#(Bool) rc_even <- mkReg(True, clocked_by pcie_ep.user_clk, reset_by user_reset_n);

   rule rl_rc_header (fAxiRc.first.sop && rc_even);
      RCDescriptor rc_desc = unpack(fAxiRc.first.data [95:0]);
      // RC descriptor always 96 bytes with first data word in bits 127:96
      Bit#(32) data = byteSwap(fAxiRc.first.data[127:96]);
      TLPData#(16) tlp16 = convertRCDescriptorToTLP16(rc_desc, data);
      rc_dwcount <= (rc_desc.dwcount == 0) ? 0 : rc_desc.dwcount - 1;
      frc.enq(tlp16);
      let even = False;
      if (rc_desc.dwcount == 0 || rc_desc.dwcount == 1) begin
        fAxiRc.deq;
        even = True;
      end
      rc_even <= even;
   endrule

   rule rl_rc_data ((!rc_even || !(fAxiRc.first.sop)) && (rc_dwcount != 0));
      Bit#(16) be16;
      case (rc_dwcount)
         1: be16 = 16'hF000;
         2: be16 = 16'hFF00;
         3: be16 = 16'hFFF0;
         default: be16 = 16'hFFFF;
      endcase
      let last = (rc_dwcount <= 4);
      let dwcount = rc_dwcount - 4;
      if (last) dwcount = 0;
      let data = (rc_even) ? fAxiRc.first.data[127:0]: fAxiRc.first.data[255:128];
      TLPData#(16) tlp16 = TLPData{sof: False,
                                   eof: last,
                                   hit: 0,
                                   be: be16,
                                   data: pack(data)};
      frc.enq(tlp16);
      if (last || !rc_even) begin
         fAxiRc.deq;
      end
      rc_dwcount <= dwcount;
      rc_even <= (last) ? True : !rc_even;
   endrule

   Reg#(DWCount) cc_dwcount <- mkReg(0, clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   FIFOF#(TLPData#(16)) fcc_tlps <- mkFIFOF (clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   // CC.
   rule get_cc_tlps;
      let tlp <- toGet(fcc).get;
      fcc_tlps.enq(tlp);
   endrule

   rule rl_cc_header(fcc_tlps.first.sof);
      match { .cc_desc, .dw} = convertTLP16ToCCDescriptor(fcc_tlps.first);
      cc_dwcount <= cc_desc.dwcount - 1;
      fAxiCc.enq(AxiStCc {data: zeroExtend({dw, pack(cc_desc)[95:0]}),
                       last: fcc_tlps.first.eof,
                       keep: 8'h0F});
      fcc_tlps.deq;
   endrule

   rule rl_cc_data((!fcc_tlps.first.sof) && (cc_dwcount != 0));
      Bit#(256) x = zeroExtend(fcc_tlps.first.data); //FIXME
      fAxiCc.enq(AxiStCc {data: {x},
                       last: cc_dwcount <= 4,
                       keep: (cc_dwcount == 3) ? 8'h0F : 8'hFF});
      fcc_tlps.deq;
   endrule

   // RQ.
   FIFOF#(TLPData#(16)) frq_tlps <- mkFIFOF (clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   Reg#(Bit#(4)) rq_first_be <- mkReg(0, clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   Reg#(Bit#(4)) rq_last_be <- mkReg(0, clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   Reg#(Bool)    rq_even    <- mkRegU(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   Reg#(DWCount) rq_dwcount <- mkReg(0, clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   Reg#(AxiStRq) rq_rq <- mkRegU(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   rule get_rq_tlps;
      let tlp <- toGet(frq).get;
      frq_tlps.enq(tlp);
   endrule

   rule rl_rq_header if (frq_tlps.first.sof);
      TLPData#(16) tlp <- toGet(frq_tlps).get();
      match { .rq_desc, .first_be, .last_be, .mdata} = convertTLP16ToRQDescriptor(tlp);
      let dwcount = ((rq_desc.reqtype == MEMORY_WRITE) ? rq_desc.dwcount : 0);
      rq_dwcount <= dwcount;
      rq_even <= False;
      rq_first_be <= first_be;
      rq_last_be <= last_be;
      let last = (rq_desc.reqtype == MEMORY_WRITE) ? (dwcount <= 4) : True;
      let rq = AxiStRq {data: zeroExtend(pack(rq_desc)), //FIXME:
			last: last,
			keep: 8'h0F,
			first_be: first_be,
			last_be: last_be};
      if (rq_desc.reqtype == MEMORY_WRITE)
	 rq_rq <= rq;
      else
	 fAxiRq.enq(rq);
   endrule

   // more data
   rule rl_rq_data if (rq_dwcount != 0);
      TLPData#(16) tlp <- toGet(frq_tlps).get();
      let rq = rq_rq;
      let last = (rq_dwcount <= 4);
      let dwcount = rq_dwcount - 4;
      Bit#(8) keep;
      if (last)
	dwcount = 0;
      if (rq_even) begin
	 rq.data[127:0] = tlp.data;
	case (rq_dwcount)
	   1: keep = 8'h01;
	   2: keep = 8'h03;
	   3: keep = 8'h07;
	   default: keep = 8'h0f;
	endcase
      end
      else begin
	rq.data[255:128] = tlp.data;
	case (rq_dwcount)
	   1: keep = 8'h1f;
	   2: keep = 8'h3f;
	   3: keep = 8'h7f;
	   default: keep = 8'hff;
	endcase
      end
      rq.last = last;
      rq.first_be = rq_first_be;
      rq.last_be = rq_last_be;
      rq.keep = keep;
      if (!rq_even || last)
	 fAxiRq.enq(rq);
      if (rq_even)
	rq_rq <= rq;
      rq_dwcount <= dwcount;
      rq_even <= (last) ? False : !rq_even;
   endrule

   FIFO#(Bool) intrMutex <- mkFIFO1(clocked_by pcie_ep.user_clk, reset_by user_reset_n);
   rule rl_intr;
      let addr = 0;
      let data = 0;
      let enable = 0;
      if (intrFifo.notEmpty() && pcie_ep.cfg.interrupt_msix_enable[0] == 1) begin
	 match { .a, .d } <- toGet(intrFifo).get();
	 addr = a;
	 data = d;
	 enable = 1;
	 intrMutex.enq(True);
      end
      pcie_ep.cfg.interrupt_msix_address(addr);
      pcie_ep.cfg.interrupt_msix_data(data);
      pcie_ep.cfg.interrupt_msix_int(enable);
   endrule: rl_intr
   rule rl_intr_sent if (pcie_ep.cfg.interrupt_msix_sent() == 1|| pcie_ep.cfg.interrupt_msix_fail() == 1);
      intrMutex.deq();
   endrule

   // The PCIE endpoint is processing Gen3 descriptors at 250MHz. The
   // AXI bridge is accepting TLPData#(16)s at 250 MHz. The
   // conversion uses half of Gen3 descriptor.
   //mkConnection(tlp8, gb.tlp, clocked_by portalClock, reset_by portalReset);

   let portalClock = (mainClockPeriod == pcieClockPeriod) ? pcieClock250 : mainClock;
   let portalReset = (mainClockPeriod == pcieClockPeriod) ? pcieReset250 : mainReset;

   interface Server tlpr;
      interface request = toPut(frq);
      interface response = toGet(frc);
   endinterface
   // Requests from other PCIe devices
   interface Server tlpc;
      interface request = toPut(fcc);
      interface response = toGet(fcq);
   endinterface
   interface interruptRequest = toPut(intrFifo);
   interface pcie    = pcie_ep.pci_exp;
   interface Pcie3wrapUser user = pcie_ep.user;
   interface PciewrapPipe pipe = pcie_ep.pipe;
   interface PciewrapCommon common= pcie_ep.common;
   interface Clock epPcieClock = pcieClock250;
   interface Reset epPcieReset = pcieReset250;
   interface Clock epPortalClock = portalClock;
   interface Reset epPortalReset = portalReset;
   interface Clock epDerivedClock = derivedClock;
   interface Reset epDerivedReset = derivedReset;
endmodule: mkPcieEndpointX7

endpackage: Pcie3EndpointX7