added mkFPMultipler

added mkFPMultipler

import FIFOF::*;
import GetPut::*;
import ClientServer::*;
import FloatingPoint::*;
import DefaultValue::*;
import Randomizable::*;
import Vector::*;
import StmtFSM::*;
import Pipe::*;
import FIFO::*;
import BUtils::*;
import PipeMul::*;

////////////////////////////////////////////////////////////////////////////////
/// Floating point fused multiple accumulate
////////////////////////////////////////////////////////////////////////////////
///
/// copied from FloatingPoint.bsv and modified.
/// this version is no longer IEEE compliant :)
///
////////////////////////////////////////////////////////////////////////////////

typedef struct {
   Maybe#(FloatingPoint#(e,m)) res;
   Exception exc;
   RoundMode rmode;
   } CommonState#(numeric type e, numeric type m) deriving (Bits, Eq);

function Bit#(e) unbias( FloatingPoint#(e,m) din );
   return (din.exp - fromInteger(bias(din)));
endfunction

function Bit#(m) zExtendLSB(Bit#(n) value)
   provisos( Add#(n,m,k) );
   Bit#(k) out = { value, 0 };
   return out[valueof(k)-1:valueof(n)];
endfunction

function Integer minexp( FloatingPoint#(e,m) din );
  return 1-bias(din);
endfunction

function Bit#(1) getHiddenBit( FloatingPoint#(e,m) din );
   return (isSubNormal(din)) ? 0 : 1;
endfunction

function Integer bias( FloatingPoint#(e,m) din );
   return (2 ** (valueof(e)-1)) - 1;
endfunction

function Integer minexp_subnormal( FloatingPoint#(e,m) din );
   return minexp(din)-valueof(m);
endfunction

function Integer maxexp( FloatingPoint#(e,m) din );
   return bias(din);
endfunction

function Tuple2#(FloatingPoint#(e,m),Exception) round( RoundMode rmode, FloatingPoint#(e,m) din, Bit#(2) guard )
   provisos(  Add#(m, 1, m1)
	    , Add#(m, 2, m2)
	    );

   FloatingPoint#(e,m) out = defaultValue;
   Exception exc = defaultValue;

   if (isNaNOrInfinity(din)) begin
      out = din;
   end
   else begin
      let din_inc = din;

      Bit#(TAdd#(m,2)) sfd = unpack({1'b0, getHiddenBit(din), din.sfd}) + 1;

      if (msb(sfd) == 1) begin
	 if (din.exp == fromInteger(maxexp(din) + bias(out))) begin
	    din_inc = infinity(din_inc.sign);
	 end
	 else begin
	    din_inc.exp = din_inc.exp + 1;
	    din_inc.sfd = truncate(sfd >> 1);
	 end
      end
      else if ((din.exp == 0) && (truncateLSB(sfd) == 2'b01)) begin
	 din_inc.exp = 1;
	 din_inc.sfd = truncate(sfd);
      end
      else begin
	 din_inc.sfd = truncate(sfd);
      end

      if (guard != 0) begin
	 exc.inexact = True;
      end

      case(rmode)
	 Rnd_Nearest_Even:
	 begin
	    case (guard)
	       'b00: out = din;
	       'b01: out = din;
	       'b10: out = (lsb(din.sfd) == 1) ? din_inc : din;
	       'b11: out = din_inc;
	    endcase
	 end

	 Rnd_Nearest_Away_Zero:
	 begin
	    case (guard)
	       'b00: out = din;
	       'b01: out = din_inc;
	       'b10: out = din_inc;
	       'b11: out = din_inc;
	    endcase
	 end

	 Rnd_Plus_Inf:
	 begin
	    if (guard == 0)
	       out = din;
	    else if (din.sign)
	       out = din;
	    else
	       out = din_inc;
	 end

	 Rnd_Minus_Inf:
	 begin
	    if (guard == 0)
	       out = din;
	    else if (din.sign)
	       out = din_inc;
	    else
	       out = din;
	 end

	 Rnd_Zero:
	 begin
	    out = din;
	 end
      endcase
   end

   if (isInfinity(out)) begin
      exc.overflow = True;
   end

   return tuple2(out,exc);
endfunction

function Tuple3#(FloatingPoint#(e,m),Bit#(2),Exception) normalize( FloatingPoint#(e,m) din, Bit#(x) sfdin )
   provisos(
      Add#(1, a__, x),
      Add#(m, b__, x),
      // per request of bsc
      Add#(c__, TLog#(TAdd#(1, x)), TAdd#(e, 1))
      );

   FloatingPoint#(e,m) out = din;
   Bit#(2) guard = 0;
   Exception exc = defaultValue;

   Int#(TAdd#(e,1)) exp = isSubNormal(out) ? fromInteger(minexp(out)) : signExtend(unpack(unbias(out)));
   let zeros = countZerosMSB(sfdin);

   if ((zeros == 0) && (exp == fromInteger(maxexp(out)))) begin
      out.exp = maxBound - 1;
      out.sfd = maxBound;
      guard = '1;
      exc.overflow = True;
      exc.inexact = True;
   end
   else begin
      if (zeros == 0) begin
	 // carry, no sfd adjust necessary

	 if (out.exp == 0)
	    out.exp = 2;
	 else
	    out.exp = out.exp + 1;

	 // carry bit
	 sfdin = sfdin << 1;
      end
      else if (zeros == 1) begin
	 // already normalized

	 if (out.exp == 0)
	    out.exp = 1;

	 // carry, hidden bits
	 sfdin = sfdin << 2;
      end
      else if (zeros == fromInteger(valueOf(x))) begin
	 // exactly zero
	 out.exp = 0;
      end
      else begin
	 // try to normalize
	 Int#(TAdd#(e,1)) shift = zeroExtend(unpack(pack(zeros - 1)));
	 Int#(TAdd#(e,1)) maxshift = exp - fromInteger(minexp(out));

	 if (shift > maxshift) begin
	    // result will be subnormal

	    sfdin = sfdin << maxshift;
	    out.exp = 0;
	 end
	 else begin
	    // result will be normal

	    sfdin = sfdin << shift;
	    out.exp = out.exp - truncate(pack(shift));
	 end

 	 // carry, hidden bits
	 sfdin = sfdin << 2;
      end

      out.sfd = unpack(truncateLSB(sfdin));
      sfdin = sfdin << fromInteger(valueOf(m));

      guard[1] = unpack(truncateLSB(sfdin));
      sfdin = sfdin << 1;

      guard[0] = |sfdin;
   end

   if ((out.exp == 0) && (guard != 0))
      exc.underflow = True;

   return tuple3(out,guard,exc);
endfunction

function Bool isNaNOrInfinity( FloatingPoint#(e,m) din );
   return (din.exp == '1);
endfunction

module mkFpMac#(RoundMode rmode)(Server#(Tuple3#(Maybe#(FloatingPoint#(e,m)), FloatingPoint#(e,m), FloatingPoint#(e,m)), Tuple2#(FloatingPoint#(e,m),Exception)))
   provisos(
      Add#(e,2,ebits),
      Add#(m,1,mbits),
      Add#(m,5,m5bits),
      Add#(mbits,mbits,mmbits),
      // per request of bsc
      Add#(1, a__, mmbits),
      Add#(m, b__, mmbits),
      Add#(c__, TLog#(TAdd#(1, mmbits)), TAdd#(e, 1)),
      Add#(d__, TLog#(TAdd#(1, m5bits)), TAdd#(e, 1)),
      Add#(1, TAdd#(1, TAdd#(m, 3)), m5bits),
      Add#(e__, mmbits, TMul#(2, mmbits)),
      // for PipeMul2
      Add#(16, f__, mbits),
      Add#(16, g__, TMul#(2, mbits)),
      Add#(h__, mbits, TMul#(2, mbits)),
      Add#(16, i__, mmbits),
      Add#(j__, TSub#(mbits, 16), mmbits),
      Add#(mmbits,0,48)
      );

   FIFOF#(Tuple3#(Maybe#(FloatingPoint#(e,m)),
		 FloatingPoint#(e,m),
		 FloatingPoint#(e,m))) fOperand_S0 <- mkLFIFOF();

   Reg#(Tuple7#(CommonState#(e,m),
		Bool,
		FloatingPoint#(e,m),
		Bool,
		Int#(ebits),
		Bit#(mbits),
		Bit#(mbits))) rState_S1 <- mkReg(unpack(0));
   Reg#(Bool) rValid_S1 <- mkReg(False);

   FIFOF#(Tuple2#(FloatingPoint#(e,m),Exception)) fResult_S9 <- mkLFIFOF();

   Reg#(Tuple5#(CommonState#(e,m),
		Bool,
		FloatingPoint#(e,m),
		Bool,
		Int#(ebits))) rState_S2 <- mkReg(unpack(0));
   Reg#(Bool) rValid_S2 <- mkReg(False);
   Reg#(UInt#(mmbits)) rProd_S2lsb <- mkReg(0);
   Reg#(UInt#(mmbits)) rProd_S2msb <- mkReg(0);

   Reg#(Tuple5#(CommonState#(e,m),
      Bool,
      FloatingPoint#(e,m),
      Bool,
      Int#(ebits))) rState_S3 <- mkReg(unpack(0));
   Reg#(Bool) rValid_S3 <- mkReg(False);
   Reg#(UInt#(mmbits)) rProd_S3 <- mkReg(0);
   Reg#(Tuple4#(Bool, Bool, Bool, Bool)) rConditions_S3 <- mkReg(unpack(0));

   Reg#(CommonState#(e,m)) rState_S4_s <- mkReg(unpack(0));
   Reg#(Bool) rState_S4_acc <- mkReg(False);
   Reg#(FloatingPoint#(e,m)) rState_S4_opA <- mkReg(unpack(0));
   Reg#(FloatingPoint#(e,m)) rState_S4_bc <- mkReg(unpack(0));
   Reg#(Bit#(2)) rState_S4_guardBC <- mkReg(unpack(0));
   Reg#(Bool) rValid_S4 <- mkReg(False);

   Reg#(Tuple8#(CommonState#(e,m),
		Bool,
		Bool,
		Bool,
		Int#(ebits),
		Int#(ebits),
		Bit#(m5bits),
		Bit#(m5bits))) rState_S5 <- mkReg(unpack(0));
   Reg#(Bool) rValid_S5 <- mkReg(False);

   Reg#(Tuple7#(CommonState#(e,m),
		Bool,
		Bool,
		Bool,
		Int#(ebits),
		Bit#(m5bits),
		Bit#(m5bits))) rState_S6 <- mkReg(unpack(0));
   Reg#(Bool) rValid_S6 <- mkReg(False);

   Reg#(Tuple7#(CommonState#(e,m),
		Bool,
		Bool,
		Bool,
		Int#(ebits),
		Bit#(m5bits),
		Bit#(m5bits))) rState_S7 <- mkReg(unpack(0));
   Reg#(Bool) rValid_S7 <- mkReg(False);

   Reg#(Tuple5#(CommonState#(e,m),
		 Bool,
		 FloatingPoint#(e,m),
		 Bit#(2),
		 Bool)) rState_S8 <- mkReg(unpack(0));
   Reg#(Bool) rValid_S8 <- mkReg(False);

   // check operands, compute exponent for multiply
   rule s1_stage;
      begin
	 Tuple3#(Maybe#(FloatingPoint#(e,m)),FloatingPoint#(e,m),FloatingPoint#(e,m)) req = unpack(0);
	 let valid = False;
	 if (fOperand_S0.notEmpty()) begin
	    req = fOperand_S0.first();
	    fOperand_S0.deq();
	    valid = True;
	 end
	 match { .mopA, .opB, .opC } = req;

	 CommonState#(e,m) s = CommonState {
	    res: tagged Invalid,
	    exc: defaultValue,
	    rmode: rmode
	    };

	 Bool acc = False;
	 FloatingPoint#(e,m) opA = 0;

	 if (mopA matches tagged Valid .opA_) begin
	    opA = opA_;
	    acc = True;
	 end

	 Int#(ebits) expB = isSubNormal(opB) ? fromInteger(minexp(opB)) : signExtend(unpack(unbias(opB)));
	 Int#(ebits) expC = isSubNormal(opC) ? fromInteger(minexp(opC)) : signExtend(unpack(unbias(opC)));

	 Bit#(mbits) sfdB = { getHiddenBit(opB), opB.sfd };
	 Bit#(mbits) sfdC = { getHiddenBit(opC), opC.sfd };

	 Bool sgnBC = opB.sign != opC.sign;
	 Bool infBC = isInfinity(opB) || isInfinity(opC);
	 Bool zeroBC = isZero(opB) || isZero(opC);
	 Int#(ebits) expBC = expB + expC;

	 if (isSNaN(opA)) begin
	    s.res = tagged Valid nanQuiet(opA);
	    s.exc.invalid_op = True;
	 end
	 else if (isSNaN(opB)) begin
	    s.res = tagged Valid nanQuiet(opB);
	    s.exc.invalid_op = True;
	 end
	 else if (isSNaN(opC)) begin
	    s.res = tagged Valid nanQuiet(opC);
	    s.exc.invalid_op = True;
	 end
	 else if (isQNaN(opA)) begin
	    s.res = tagged Valid opA;
	 end
	 else if (isQNaN(opB)) begin
	    s.res = tagged Valid opB;
	 end
	 else if (isQNaN(opC)) begin
	    s.res = tagged Valid opC;
	 end
	 else if ((isInfinity(opB) && isZero(opC)) || (isZero(opB) && isInfinity(opC)) || (isInfinity(opA) && infBC && (opA.sign != sgnBC))) begin
	    // product of zero and infinity or addition of opposite sign infinity
	    s.res = tagged Valid qnan();
	    s.exc.invalid_op = True;
	 end
	 else if (isInfinity(opA)) begin
	    s.res = tagged Valid opA;
	 end
	 else if (infBC) begin
	    s.res = tagged Valid infinity(sgnBC);
	 end
	 else if (isZero(opA) && zeroBC && (opA.sign == sgnBC)) begin
	    s.res = tagged Valid opA;
	 end

	 rState_S1 <= tuple7(s,
			     acc,
			     opA,
			     sgnBC,
			     expBC,
			     sfdB,
			     sfdC);
	 rValid_S1 <= valid;
      end
   //endrule

   // start multiply
      //rule s2_stage if (fResult_S9.notFull());
      begin
	 match { .s, .acc, .opA, .sgnBC, .expBC, .sfdB, .sfdC } = rState_S1;
	 let valid = rValid_S1;

	 UInt#(mbits) sfdClsb = extend(unpack(sfdC[15:0]));
	 UInt#(TSub#(mbits,16)) sfdCmsb = unpack(sfdC[valueOf(mbits)-1:16]);
	 rProd_S2lsb <= extend(unpack(sfdB))*extend(sfdClsb);
	 rProd_S2msb <= extend(unpack(sfdB))*extend(sfdCmsb);

	 rState_S2 <= tuple5(s,
			     acc,
			     opA,
			     sgnBC,
			     expBC);
	 rValid_S2 <= valid;
      end
   //endrule

   //   add partial multiplies
      //rule s3_stage if (fResult_S9.notFull());
      begin
	 match { .s, .acc, .opA, .sgnBC, .expBC } = rState_S2;
	 let valid = rValid_S2;

	 UInt#(mmbits) sfdBC = rProd_S2lsb + (rProd_S2msb << 16);
	 rProd_S3 <= sfdBC;

	 rState_S3 <= tuple5(s,
			     acc,
			     opA,
			     sgnBC,
			     expBC);
	 rValid_S3 <= valid;

	 Bool overflow = False;
	 Bool underflow = False;
	 Bool inbounds = False;
	 Bool denorm = False;
	 let shift = 0;
	 FloatingPoint#(e,m) bc = defaultValue;
	 if (s.res matches tagged Invalid) begin
	    if (expBC > fromInteger(maxexp(bc))) begin
	       overflow = True;
	    end
	    else if (expBC < (fromInteger(minexp_subnormal(bc))-2)) begin
	       underflow = True;
	    end
	    else begin
	       inbounds = True;
	       shift = fromInteger(minexp(bc)) - expBC;
	       if (shift > 0) begin
		  denorm = True;
	       end
	    end
	 end
	 rConditions_S3 <= tuple4(overflow, underflow, inbounds, denorm);
      end
   //endrule

   // normalize multiplication result
      //rule s4_stage if (fResult_S9.notFull());
      begin
	 match { .s, .acc, .opA, .sgnBC, .expBC } = rState_S3;
	 match { .overflow, .underflow, .inbounds, .denorm } = rConditions_S3;
	 let valid = rValid_S3;

	 let sfdBC = pack(rProd_S3);

	 FloatingPoint#(e,m) bc = defaultValue;
	 Bit#(2) guardBC = ?;

	 //if (s.res matches tagged Invalid) begin
	 //if (expBC > fromInteger(maxexp(bc))) begin
	 if (overflow) begin
	       bc.sign = sgnBC;
	       bc.exp = maxBound - 1;
	       bc.sfd = maxBound;
	       guardBC = '1;

	       s.exc.overflow = True;
	       s.exc.inexact = True;
	 end
	 //else if (expBC < (fromInteger(minexp_subnormal(bc))-2)) begin
	 else if (underflow) begin
	       bc.sign = sgnBC;
	       bc.exp = 0;
	       bc.sfd = 0;
	       guardBC = 0;

	       if (|sfdBC == 1) begin
		  guardBC = 1;
		  s.exc.underflow = True;
		  s.exc.inexact = True;
	       end
	    end
	 //else begin
	 else if (inbounds) begin
	    //if (shift > 0) begin
	    if (denorm) begin
	       // subnormal
	       //Bit#(1) sfdlsb = |(sfdBC << (fromInteger(valueOf(mmbits)) - shift));
	       //sfdBC = sfdBC >> shift;
	       //sfdBC[0] = sfdBC[0] | sfdlsb;
	       //bc.exp = 0;
	       bc.exp = 0;
	       sfdBC = 0;
	       s.exc.underflow = True;
	       s.exc.inexact = True;
	    end
	    else begin
	       bc.exp = cExtend(expBC + fromInteger(bias(bc)));
	    end

	    bc.sign = sgnBC;
	    let y = normalize(bc, sfdBC);
	    bc = tpl_1(y);
	    guardBC = tpl_2(y);
	    s.exc = s.exc | tpl_3(y);
	 end
	 //end

	 rState_S4_s <= s;
	 rState_S4_acc <= acc;
	 rState_S4_opA <= opA;
	 rState_S4_bc <= bc;
	 rState_S4_guardBC <= guardBC;
	 rValid_S4 <= valid;
      end
   //endrule

   // calculate shift and sign for add
      //rule s5_stage if (fResult_S9.notFull());
      begin
	 //match { .s, .acc, .opA, .opBC, .guardBC } = rState_S4;
	 let s = rState_S4_s;
	 let acc = rState_S4_acc;
	 let opA = rState_S4_opA;
	 let opBC = rState_S4_bc;
	 let guardBC = rState_S4_guardBC;
	 let valid = rValid_S4;

	 Int#(ebits) expA = isSubNormal(opA) ? fromInteger(minexp(opA)) : signExtend(unpack(unbias(opA)));
	 Int#(ebits) expBC = isSubNormal(opBC) ? fromInteger(minexp(opBC)) : signExtend(unpack(unbias(opBC)));

	 Bit#(m5bits) sfdA = {1'b0, getHiddenBit(opA), opA.sfd, 3'b0};
	 Bit#(m5bits) sfdBC = {1'b0, getHiddenBit(opBC), opBC.sfd, guardBC, 1'b0};

	 Bool sub = opA.sign != opBC.sign;

	 Int#(ebits) exp = ?;
	 Int#(ebits) shift = ?;
	 Bit#(m5bits) x = ?;
	 Bit#(m5bits) y = ?;
	 Bool sgn = ?;

	 if ((!acc) || (expBC > expA) || ((expBC == expA) && (sfdBC > sfdA))) begin
	    exp = expBC;
	    shift = expBC - expA;
	    x = sfdBC;
	    y = sfdA;
	    sgn = opBC.sign;
	 end
	 else begin
	    exp = expA;
	    shift = expA - expBC;
	    x = sfdA;
	    y = sfdBC;
	    sgn = opA.sign;
	 end

	 rState_S5 <= tuple8(s,
			     acc,
			     sub,
			     sgn,
			     exp,
			     shift,
			     x,
			     y);
	 rValid_S5 <= valid;
      end
   //endrule

   // shift second add operand
   //rule s6_stage if (fResult_S9.notFull());
      begin
	 match { .s, .acc, .sub, .sgn, .exp, .shift, .x, .y } = rState_S5;
	 let valid = rValid_S5;

	 if (s.res matches tagged Invalid) begin
	    if (shift < fromInteger(valueOf(m5bits))) begin
	       Bit#(m5bits) guard;

	       guard = y << (fromInteger(valueOf(m5bits)) - shift);
	       y = y >> shift;
	       y[0] = y[0] | (|guard);
	    end
	    else if (|y == 1) begin
	       y = 1;
	    end
	 end

	 rState_S6 <= tuple7(s,
			     acc,
			     sub,
			     sgn,
			     exp,
			     x,
			     y);
	 rValid_S6 <= valid;
      end
   //endrule

   // add/subtract sfd
   //rule s7_stage if (fResult_S9.notFull());
      begin
	 match { .s, .acc, .sub, .sgn, .exp, .x, .y } = rState_S6;
	 let valid = rValid_S6;

	 let sum = x + y;
	 let diff = x - y;

	 rState_S7 <= tuple7(s,
			     acc,
			     sub,
			     sgn,
			     exp,
			     sum,
			     diff);
	 rValid_S7 <= valid;
      end
   //endrule

   // normalize addition result
   //rule s8_stage if (fResult_S9.notFull());
      begin
	 match { .s, .acc, .sub, .sgn, .exp, .sum, .diff } = rState_S7;
	 let valid = rValid_S7;

	 FloatingPoint#(e,m) out = defaultValue;
	 Bit#(2) guard = 0;

	 if (s.res matches tagged Invalid) begin
	    Bit#(m5bits) sfd;

	    sfd = sub ? diff : sum;

	    out.sign = sgn;
	    out.exp = cExtend(exp + fromInteger(bias(out)));

	    let y = normalize(out, sfd);
	    out = tpl_1(y);
	    guard = tpl_2(y);
	    s.exc = s.exc | tpl_3(y);
	 end

	 rState_S8 <= tuple5(s,
			     acc,
			     out,
			     guard,
			     sub);
	 rValid_S8 <= valid;
      end
   //endrule

   // round result
   //rule s9_stage if (fResult_S9.notFull());
      begin
	 match { .s, .acc, .out, .guard, .sub } = rState_S8;
	 let valid = rValid_S8;

	 if (s.res matches tagged Valid .x) begin
	    out = x;
	 end
	 else begin
	    let y = round(rmode, out, guard);
	    out = tpl_1(y);
	    s.exc = s.exc | tpl_2(y);

	    // adjust sign for exact zero result
	    if (acc && isZero(out) && !s.exc.inexact && sub) begin
	       out.sign = (rmode == Rnd_Minus_Inf);
	    end
	 end

	 if (valid)
	    fResult_S9.enq(tuple2(out,s.exc));
      end
   endrule

   interface Put request;
   method Action put(Tuple3#(Maybe#(FloatingPoint#(e,m)),FloatingPoint#(e,m),FloatingPoint#(e,m)) req);
      fOperand_S0.enq(req);
   endmethod
   endinterface
   interface Get response;
      method ActionValue#(Tuple2#(FloatingPoint#(e,m),Exception)) get();
	 let resp <- toGet(fResult_S9).get();
	 return resp;
      endmethod
   endinterface
endmodule

////////////////////////////////////////////////////////////////////////////////
/// Pipelined Floating Point Multiplier
////////////////////////////////////////////////////////////////////////////////
module mkFPMultiplier#(RoundMode rmode)(Server#(Tuple2#(FloatingPoint#(e,m), FloatingPoint#(e,m)), Tuple2#(FloatingPoint#(e,m),Exception)))
   provisos(
      // per request of bsc
      Add#(a__, TLog#(TAdd#(1, TAdd#(TAdd#(m, 1), TAdd#(m, 1)))), TAdd#(e, 1))
      );

   ////////////////////////////////////////////////////////////////////////////////
   /// S0
   ////////////////////////////////////////////////////////////////////////////////
   FIFO#(Tuple2#(FloatingPoint#(e,m),
		 FloatingPoint#(e,m)))                 fOperands_S0        <- mkLFIFO;

   ////////////////////////////////////////////////////////////////////////////////
   /// S1 - calculate the new exponent/sign
   ////////////////////////////////////////////////////////////////////////////////
   FIFO#(Tuple5#(CommonState#(e,m),
		 Bit#(TAdd#(m,1)),
		 Bit#(TAdd#(m,1)),
		 Int#(TAdd#(e,2)),
		 Bool)) fState_S1 <- mkLFIFO;

   rule s1_stage;
      match { .opA, .opB } <- toGet(fOperands_S0).get;

      CommonState#(e,m) s = CommonState {
	 res: tagged Invalid,
	 exc: defaultValue,
	 rmode: rmode
	 };

      Int#(TAdd#(e,2)) expA = isSubNormal(opA) ? fromInteger(minexp(opA)) : signExtend(unpack(unbias(opA)));
      Int#(TAdd#(e,2)) expB = isSubNormal(opB) ? fromInteger(minexp(opB)) : signExtend(unpack(unbias(opB)));
      Int#(TAdd#(e,2)) newexp = expA + expB;

      Bool sign = (opA.sign != opB.sign);

      Bit#(TAdd#(m,1)) opAsfd = { getHiddenBit(opA), opA.sfd };
      Bit#(TAdd#(m,1)) opBsfd = { getHiddenBit(opB), opB.sfd };

      if (isSNaN(opA)) begin
	 s.res = tagged Valid nanQuiet(opA);
	 s.exc.invalid_op = True;
      end
      else if (isSNaN(opB)) begin
	 s.res = tagged Valid nanQuiet(opB);
	 s.exc.invalid_op = True;
      end
      else if (isQNaN(opA)) begin
	 s.res = tagged Valid opA;
      end
      else if (isQNaN(opB)) begin
	 s.res = tagged Valid opB;
      end
      else if ((isInfinity(opA) && isZero(opB)) || (isZero(opA) && isInfinity(opB))) begin
	 s.res = tagged Valid qnan();
	 s.exc.invalid_op = True;
      end
      else if (isInfinity(opA) || isInfinity(opB)) begin
	 s.res = tagged Valid infinity(opA.sign != opB.sign);
      end
      else if (isZero(opA) || isZero(opB)) begin
	 s.res = tagged Valid zero(opA.sign != opB.sign);
      end
      else if (newexp > fromInteger(maxexp(opA))) begin
	 FloatingPoint#(e,m) out;
	 out.sign = (opA.sign != opB.sign);
	 out.exp = maxBound - 1;
	 out.sfd = maxBound;

	 s.exc.overflow = True;
	 s.exc.inexact = True;

	 let y = round(rmode, out, '1);
	 s.res = tagged Valid tpl_1(y);
	 s.exc = s.exc | tpl_2(y);
      end
      else if (newexp < (fromInteger(minexp_subnormal(opA))-2)) begin
	 FloatingPoint#(e,m) out;
	 out.sign = (opA.sign != opB.sign);
	 out.exp = 0;
	 out.sfd = 0;

	 s.exc.underflow = True;
	 s.exc.inexact = True;

	 let y = round(rmode, out, 'b01);
	 s.res = tagged Valid tpl_1(y);
	 s.exc = s.exc | tpl_2(y);
      end

      fState_S1.enq(tuple5(s,
			   opAsfd,
			   opBsfd,
			   newexp,
			   sign));
   endrule

   ////////////////////////////////////////////////////////////////////////////////
   /// S2
   ////////////////////////////////////////////////////////////////////////////////
   FIFO#(Tuple4#(CommonState#(e,m),
		 Bit#(TAdd#(TAdd#(m,1),TAdd#(m,1))),
		 Int#(TAdd#(e,2)),
		 Bool)) fState_S2 <- mkLFIFO;

   rule s2_stage;
      match {.s, .opAsfd, .opBsfd, .exp, .sign} <- toGet(fState_S1).get;

      Bit#(TAdd#(TAdd#(m,1),TAdd#(m,1))) sfdres = primMul(opAsfd, opBsfd);

      fState_S2.enq(tuple4(s,
			   sfdres,
			   exp,
			   sign));
   endrule

   ////////////////////////////////////////////////////////////////////////////////
   /// S3
   ////////////////////////////////////////////////////////////////////////////////
   FIFO#(Tuple4#(CommonState#(e,m),
		 Bit#(TAdd#(TAdd#(m,1),TAdd#(m,1))),
		 Int#(TAdd#(e,2)),
		 Bool)) fState_S3 <- mkLFIFO;

   rule s3_stage;
      let x <- toGet(fState_S2).get;
      fState_S3.enq(x);
   endrule

   ////////////////////////////////////////////////////////////////////////////////
   /// S4
   ////////////////////////////////////////////////////////////////////////////////
   FIFO#(Tuple3#(CommonState#(e,m),
		 FloatingPoint#(e,m),
		 Bit#(2))) fState_S4 <- mkLFIFO;

   rule s4_stage;
      match {.s, .sfdres, .exp, .sign} <- toGet(fState_S3).get;

      FloatingPoint#(e,m) result = defaultValue;
      Bit#(2) guard = ?;

      if (s.res matches tagged Invalid) begin
	 //$display("sfdres = 'h%x", sfdres);

	 let shift = fromInteger(minexp(result)) - exp;
	 if (shift > 0) begin
	    // subnormal
	    Bit#(1) sfdlsb = |(sfdres << (fromInteger(valueOf(TAdd#(TAdd#(m,1),TAdd#(m,1)))) - shift));

	    //$display("sfdlsb = |'h%x = 'b%b", (sfdres << (fromInteger(valueOf(TAdd#(TAdd#(m,1),TAdd#(m,1)))) - shift)), sfdlsb);

            sfdres = sfdres >> shift;
            sfdres[0] = sfdres[0] | sfdlsb;

	    result.exp = 0;
	 end
	 else begin
	    result.exp = cExtend(exp + fromInteger(bias(result)));
	 end

	 // $display("shift = %d", shift);
	 // $display("sfdres = 'h%x", sfdres);
	 // $display("result = ", fshow(result));
	 // $display("exc = 'b%b", pack(exc));
	 // $display("zeros = %d", countZerosMSB(sfdres));

	 result.sign = sign;
	 let y = normalize(result, sfdres);
	 result = tpl_1(y);
	 guard = tpl_2(y);
	 s.exc = s.exc | tpl_3(y);

	 // $display("result = ", fshow(result));
	 // $display("exc = 'b%b", pack(exc));
      end

      fState_S4.enq(tuple3(s,
			   result,
			   guard));
   endrule

   ////////////////////////////////////////////////////////////////////////////////
   /// S5
   ////////////////////////////////////////////////////////////////////////////////
   FIFO#(Tuple2#(FloatingPoint#(e,m),Exception)) fResult_S5          <- mkLFIFO;

   rule s5_stage;
      match {.s, .rnd, .guard} <- toGet(fState_S4).get;

      FloatingPoint#(e,m) out = rnd;

      if (s.res matches tagged Valid .x)
	 out = x;
      else begin
	 let y = round(s.rmode, out, guard);
	 out = tpl_1(y);
	 s.exc = s.exc | tpl_2(y);
      end

      fResult_S5.enq(tuple2(out,s.exc));
   endrule

   ////////////////////////////////////////////////////////////////////////////////
   /// Interface Connections / Methods
   ////////////////////////////////////////////////////////////////////////////////
   interface request = toPut(fOperands_S0);
   interface response = toGet(fResult_S5);

endmodule: mkFPMultiplier
////////////////////////////////////////////////////////////////////////////////


fill in missing methods

fill in missing methods

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import BRAM::*;
import FIFO::*;
import Vector::*;
import Gearbox::*;
import FIFOF::*;
import SpecialFIFOs::*;

import BRAMFIFOFLevel::*;
import GetPutF::*;
import Dma::*;
import MemreadEngine::*;
import MemwriteEngine::*;

interface BRAMReadClient#(type bramIdx, numeric type busWidth);
   method Action start(DmaPointer h, Bit#(DmaOffsetSize) rbase, bramIdx num, bramIdx wbase);
   method ActionValue#(Bool) finish();
   interface DmaReadClient#(busWidth) dmaClient;
endinterface

interface BRAMWriteClient#(type bramIdx, numeric type busWidth);
   method Action start(DmaPointer h, Bit#(DmaOffsetSize) rbase, bramIdx num, bramIdx wbase);
   method ActionValue#(Bool) finish();
   interface DmaWriteClient#(busWidth) dmaClient;
endinterface

module mkBRAMReadClient#(BRAMServer#(bramIdx,d) br)(BRAMReadClient#(bramIdx,busWidth))
   provisos(Bits#(d,dsz),
	    Div#(busWidth,dsz,nd),
	    Mul#(nd,dsz,busWidth),
	    Add#(1,a__,nd),
	    Eq#(bramIdx),
	    Ord#(bramIdx),
	    Arith#(bramIdx),
	    Bits#(bramIdx,b__));

   Clock clk <- exposeCurrentClock;
   Reset rst <- exposeCurrentReset;

   FIFO#(void) f <- mkSizedFIFO(1);
   Gearbox#(nd,1,d) gb <- mkNto1Gearbox(clk,rst,clk,rst);
   Reg#(bramIdx) i <- mkReg(0);
   Reg#(bramIdx) j <- mkReg(0);
   Reg#(bramIdx) n <- mkReg(0);
   Reg#(bramIdx) wbase <- mkReg(0);
   Reg#(DmaPointer) ptr <- mkReg(0);
   Reg#(Bit#(DmaOffsetSize)) roff <- mkReg(0);
   Reg#(Bit#(DmaOffsetSize)) rbase <- mkReg(0);

   let readFifo <- mkFIFOF;
   MemreadEngine#(busWidth) re <- mkMemreadEngine(readFifo);
   let bus_width_in_bytes = fromInteger(valueOf(busWidth)/8);

   rule loadReq(i < n);
      //$display("mkBRAMReadClient::loadReq ptr=%d %d %d", ptr, roff+rbase, bus_width_in_bytes);
      re.start(ptr, roff + rbase, bus_width_in_bytes, bus_width_in_bytes);
      i <= i+fromInteger(valueOf(nd));
      roff <= roff+bus_width_in_bytes;
   endrule

   rule loadResp;
      let __x <- re.finish;
      readFifo.deq;
      let rv = readFifo.first;
      Vector#(nd,d) rvv = unpack(rv);
      gb.enq(rvv);
      //$display("mkBRAMReadClient::loadResp ptr=%d", ptr);
   endrule

   rule load(j < n);
      br.request.put(BRAMRequest{write:True, responseOnWrite:False, address:j+wbase, datain:gb.first[0]});
      gb.deq;
      j <= j+1;
      if (j+1 == n)
	 f.enq(?);
      //$display("mkBRAMReadClient::load ptr=%d", ptr);
   endrule

   rule discard(j >= n);
      gb.deq;
      //$display("mkBRAMReadClient::discard ptr=%d", ptr);
   endrule

   method Action start(DmaPointer h, Bit#(DmaOffsetSize) rb, bramIdx num, bramIdx wb);
      $display("mkBRAMReadClient::start(%h, %h, %h %h)", h, wb, num, rb);
      i <= 0;
      j <= 0;
      n <= num;
      ptr <= h;
      roff <= 0;
      rbase <= rb;
      wbase <= wb;
   endmethod

   method ActionValue#(Bool) finish();
      f.deq;
      return True;
   endmethod

   interface dmaClient = re.dmaClient;

endmodule

module mkBRAMWriteClient#(BRAMServer#(bramIdx,d) br)(BRAMWriteClient#(bramIdx,busWidth))

   provisos(Bits#(d,dsz),
	    Div#(busWidth,dsz,nd),
	    Mul#(nd,dsz,busWidth),
	    Add#(1,a__,nd),
	    Add#(1, b__, TMul#(2, nd)),
	    Add#(nd, c__, TMul#(2, nd)),
	    Eq#(bramIdx),
	    Bits#(bramIdx, d__),
	    Ord#(bramIdx),
	    Arith#(bramIdx));

   Clock clk <- exposeCurrentClock;
   Reset rst <- exposeCurrentReset;

   FIFO#(void) f <- mkSizedFIFO(1);
   Reg#(bramIdx) n <- mkReg(0);
   Reg#(bramIdx) i <- mkReg(0);
   Reg#(bramIdx) j <- mkReg(0);
   Reg#(bramIdx) rbase <- mkReg(0);
   Reg#(DmaPointer) ptr <- mkReg(0);
   Reg#(Bit#(DmaOffsetSize)) woff <- mkReg(0);
   Reg#(Bit#(DmaOffsetSize)) wbase <- mkReg(0);
   Gearbox#(1,nd,Bit#(dsz)) gb <- mk1toNGearbox(clk,rst,clk,rst);

   FIFOF#(Bit#(busWidth)) writeFifo = (interface FIFOF;
				       method Bit#(busWidth) first(); return pack(gb.first); endmethod
				       method Bool notEmpty(); return gb.notEmpty(); endmethod
				       method Action enq(Bit#(busWidth) d); endmethod
				       method Action deq; endmethod
				       method Action clear; endmethod
				       method Bool notFull(); return(False); endmethod

				       endinterface);
   MemwriteEngine#(busWidth) we <- mkMemwriteEngine(writeFifo);
   let bus_width_in_bytes = fromInteger(valueOf(busWidth)/8);

   rule bramReq(j < n);
      br.request.put(BRAMRequest{write:False, responseOnWrite:False, address:j+rbase, datain:?});
      j <= j+1;
   endrule

   rule bramResp;
      d rv <- br.response.get;
      gb.enq(cons(pack(rv), nil));
   endrule

   rule loadReq(i < n);
      we.start(ptr, woff + wbase, bus_width_in_bytes, bus_width_in_bytes);
      i <= i+1;
      woff <= woff+bus_width_in_bytes;
   endrule

   rule loadResp;
      let __x <- we.finish;
      if (i == n)
	 f.enq(?);
   endrule

   method Action start(DmaPointer h, Bit#(DmaOffsetSize) wb, bramIdx num, bramIdx rb);
      //$display("mkBRAMWriteClient::start(%h, %h, %h %h)", h, wb, num, rb);
      i <= 0;
      j <= 0;
      n <= num;
      ptr <= h;
      rbase <= rb;
      wbase <= wb;
      woff <= 0;
   endmethod

   method ActionValue#(Bool) finish();
      f.deq;
      return True;
   endmethod

   interface dmaClient = we.dmaClient;

endmodule

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import BRAM::*;
import FIFO::*;
import Vector::*;
import Gearbox::*;
import FIFOF::*;
import SpecialFIFOs::*;

import BRAMFIFOFLevel::*;
import GetPutF::*;
import Dma::*;
import MemreadEngine::*;
import MemwriteEngine::*;

interface BRAMReadClient#(type bramIdx, numeric type busWidth);
   method Action start(DmaPointer h, Bit#(DmaOffsetSize) rbase, bramIdx num, bramIdx wbase);
   method ActionValue#(Bool) finish();
   interface DmaReadClient#(busWidth) dmaClient;
endinterface

interface BRAMWriteClient#(type bramIdx, numeric type busWidth);
   method Action start(DmaPointer h, Bit#(DmaOffsetSize) rbase, bramIdx num, bramIdx wbase);
   method ActionValue#(Bool) finish();
   interface DmaWriteClient#(busWidth) dmaClient;
endinterface

module mkBRAMReadClient#(BRAMServer#(bramIdx,d) br)(BRAMReadClient#(bramIdx,busWidth))
   provisos(Bits#(d,dsz),
	    Div#(busWidth,dsz,nd),
	    Mul#(nd,dsz,busWidth),
	    Add#(1,a__,nd),
	    Eq#(bramIdx),
	    Ord#(bramIdx),
	    Arith#(bramIdx),
	    Bits#(bramIdx,b__));

   Clock clk <- exposeCurrentClock;
   Reset rst <- exposeCurrentReset;

   FIFO#(void) f <- mkSizedFIFO(1);
   Gearbox#(nd,1,d) gb <- mkNto1Gearbox(clk,rst,clk,rst);
   Reg#(bramIdx) i <- mkReg(0);
   Reg#(bramIdx) j <- mkReg(0);
   Reg#(bramIdx) n <- mkReg(0);
   Reg#(bramIdx) wbase <- mkReg(0);
   Reg#(DmaPointer) ptr <- mkReg(0);
   Reg#(Bit#(DmaOffsetSize)) roff <- mkReg(0);
   Reg#(Bit#(DmaOffsetSize)) rbase <- mkReg(0);

   let readFifo <- mkFIFOF;
   MemreadEngine#(busWidth) re <- mkMemreadEngine(readFifo);
   let bus_width_in_bytes = fromInteger(valueOf(busWidth)/8);

   rule loadReq(i < n);
      //$display("mkBRAMReadClient::loadReq ptr=%d %d %d", ptr, roff+rbase, bus_width_in_bytes);
      re.start(ptr, roff + rbase, bus_width_in_bytes, bus_width_in_bytes);
      i <= i+fromInteger(valueOf(nd));
      roff <= roff+bus_width_in_bytes;
   endrule

   rule loadResp;
      let __x <- re.finish;
      readFifo.deq;
      let rv = readFifo.first;
      Vector#(nd,d) rvv = unpack(rv);
      gb.enq(rvv);
      //$display("mkBRAMReadClient::loadResp ptr=%d", ptr);
   endrule

   rule load(j < n);
      br.request.put(BRAMRequest{write:True, responseOnWrite:False, address:j+wbase, datain:gb.first[0]});
      gb.deq;
      j <= j+1;
      if (j+1 == n)
	 f.enq(?);
      //$display("mkBRAMReadClient::load ptr=%d", ptr);
   endrule

   rule discard(j >= n);
      gb.deq;
      //$display("mkBRAMReadClient::discard ptr=%d", ptr);
   endrule

   method Action start(DmaPointer h, Bit#(DmaOffsetSize) rb, bramIdx num, bramIdx wb);
      $display("mkBRAMReadClient::start(%h, %h, %h %h)", h, wb, num, rb);
      i <= 0;
      j <= 0;
      n <= num;
      ptr <= h;
      roff <= 0;
      rbase <= rb;
      wbase <= wb;
   endmethod

   method ActionValue#(Bool) finish();
      f.deq;
      return True;
   endmethod

   interface dmaClient = re.dmaClient;

endmodule

module mkBRAMWriteClient#(BRAMServer#(bramIdx,d) br)(BRAMWriteClient#(bramIdx,busWidth))

   provisos(Bits#(d,dsz),
	    Div#(busWidth,dsz,nd),
	    Mul#(nd,dsz,busWidth),
	    Add#(1,a__,nd),
	    Add#(1, b__, TMul#(2, nd)),
	    Add#(nd, c__, TMul#(2, nd)),
	    Eq#(bramIdx),
	    Bits#(bramIdx, d__),
	    Ord#(bramIdx),
	    Arith#(bramIdx));

   Clock clk <- exposeCurrentClock;
   Reset rst <- exposeCurrentReset;

   FIFO#(void) f <- mkSizedFIFO(1);
   Reg#(bramIdx) n <- mkReg(0);
   Reg#(bramIdx) i <- mkReg(0);
   Reg#(bramIdx) j <- mkReg(0);
   Reg#(bramIdx) rbase <- mkReg(0);
   Reg#(DmaPointer) ptr <- mkReg(0);
   Reg#(Bit#(DmaOffsetSize)) woff <- mkReg(0);
   Reg#(Bit#(DmaOffsetSize)) wbase <- mkReg(0);
   Gearbox#(1,nd,Bit#(dsz)) gb <- mk1toNGearbox(clk,rst,clk,rst);

   FIFOF#(Bit#(busWidth)) writeFifo = (interface FIFOF;
				       method Bit#(busWidth) first(); return pack(gb.first); endmethod
				       method Bool notEmpty(); return gb.notEmpty(); endmethod
				       endinterface);
   MemwriteEngine#(busWidth) we <- mkMemwriteEngine(writeFifo);
   let bus_width_in_bytes = fromInteger(valueOf(busWidth)/8);

   rule bramReq(j < n);
      br.request.put(BRAMRequest{write:False, responseOnWrite:False, address:j+rbase, datain:?});
      j <= j+1;
   endrule

   rule bramResp;
      d rv <- br.response.get;
      gb.enq(cons(pack(rv), nil));
   endrule

   rule loadReq(i < n);
      we.start(ptr, woff + wbase, bus_width_in_bytes, bus_width_in_bytes);
      i <= i+1;
      woff <= woff+bus_width_in_bytes;
   endrule

   rule loadResp;
      let __x <- we.finish;
      if (i == n)
	 f.enq(?);
   endrule

   method Action start(DmaPointer h, Bit#(DmaOffsetSize) wb, bramIdx num, bramIdx rb);
      //$display("mkBRAMWriteClient::start(%h, %h, %h %h)", h, wb, num, rb);
      i <= 0;
      j <= 0;
      n <= num;
      ptr <= h;
      rbase <= rb;
      wbase <= wb;
      woff <= 0;
   endmethod

   method ActionValue#(Bool) finish();
      f.deq;
      return True;
   endmethod

   interface dmaClient = we.dmaClient;

endmodule


pipeline memory access in MemToRing to improve throughput

// Copyright (c) 2016 Cornell University.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

// NOTE:
// Implement a store-and-forward mechanism between ring-buffer and
// main packet memory. Packets are buffered completely in ring buffer
// before it is sent to main memory and vice versa.

import BuildVector::*;
import Connectable::*;
import Clocks::*;
import Cntrs::*;
import ClientServer::*;
import DefaultValue::*;
import DbgTypes::*;
import Ethernet::*;
import EthMac::*;
import FIFO::*;
import FIFOF::*;
import GetPut::*;
import Gearbox::*;
import MemTypes::*;
import MemServer::*;
import MemServerInternal::*;
import MemMgmt::*;
import PacketBuffer::*;
import SharedBuff::*;
import SharedBuffMMU::*;
import SpecialFIFOs::*;
import Vector::*;
import Pipe::*;
import DbgTypes::*;

interface StoreAndFwdFromRingToMem;
   interface PktReadClient readClient;
   interface MemAllocClient malloc;
   interface MemWriteClient#(`DataBusWidth) writeClient;
   interface PipeOut#(PacketInstance) eventPktCommitted;
endinterface

module mkStoreAndFwdFromRingToMem(StoreAndFwdFromRingToMem)
   provisos (Div#(`DataBusWidth, 8, bytesPerBeat)
            ,Log#(bytesPerBeat, beatShift));

   let verbose = False;

   // RingBuffer Read Client
   FIFO#(EtherData) readDataFifo <- mkFIFO;
   FIFO#(Bit#(EtherLen)) readLenFifo <- mkFIFO;
   FIFO#(EtherReq) readReqFifo <- mkFIFO;

   // Memory Client
   FIFO#(MemRequest) writeReqFifo <- mkSizedFIFO(4);
   FIFO#(MemData#(`DataBusWidth)) writeDataFifo <- mkSizedFIFO(16);
   FIFO#(Bit#(MemTagSize)) writeDoneFifo <- mkSizedFIFO(4);
   MemWriteClient#(`DataBusWidth) dmaWriteClient = (interface MemWriteClient;
   interface Get writeReq = toGet(writeReqFifo);
   interface Get writeData = toGet(writeDataFifo);
   interface Put writeDone = toPut(writeDoneFifo);
   endinterface);

   FIFO#(Bit#(EtherLen)) mallocReqFifo <- mkFIFO;
   FIFO#(Bit#(EtherLen)) pktLenFifo <- mkFIFO;
   FIFO#(Maybe#(PktId)) mallocDoneFifo <- mkFIFO;
   Reg#(Bool) readStarted <- mkReg(False);
   Reg#(Bool) mallocd <- mkReg(False);

   FIFOF#(PacketInstance) eventPktReceivedFifo <- mkFIFOF;
   FIFOF#(PacketInstance) eventPktCommittedFifo <- mkFIFOF;

   Reg#(Bit#(32)) cycle <- mkReg(0);
   rule every1 if (verbose);
      cycle <= cycle + 1;
   endrule

   rule packetReadStart if (!readStarted);
      let pktLen <- toGet(readLenFifo).get;
      if (verbose) $display("StoreAndForward::packetReadStart %d: ReadLen %d", cycle, pktLen);
      mallocReqFifo.enq(pktLen);
      pktLenFifo.enq(pktLen);
      readStarted <= True;
   endrule

   rule allocMemory;
      let pktLen <- toGet(pktLenFifo).get;
      let allocId <- toGet(mallocDoneFifo).get;
      let bytesPerBeatMinusOne = fromInteger(valueOf(bytesPerBeat))-1;
      // roundup to 16 byte boundary
      let burstLen = ((pktLen + bytesPerBeatMinusOne) & ~(bytesPerBeatMinusOne));
      let mask = (1<< (pktLen % fromInteger(valueOf(bytesPerBeat))))-1;
      if (isValid(allocId)) begin
         mallocd <= True;
         readReqFifo.enq(EtherReq{len: truncate(pktLen)});
         //FIXME use correct sglId
         writeReqFifo.enq(MemRequest {sglId: extend(fromMaybe(?, allocId)), offset: 0,
                                      burstLen: truncate(burstLen), tag:0
`ifdef BYTE_ENABLES
                                      , firstbe: 'hffff, lastbe: mask
`endif
                                     });
         if (verbose) $display("StoreAndForward::allocMemory %d: alloc done", cycle);
         eventPktReceivedFifo.enq(PacketInstance {id: fromMaybe(?, allocId), size: pktLen});
      end
   endrule

   rule packetReadInProgress if (readStarted && mallocd);
      let v <- toGet(readDataFifo).get;
      if (v.eop) begin
         readStarted <= False;
         mallocd <= False;
         if (verbose) $display("StoreAndForward:: %d: packet finished", cycle);
      end
      if (verbose) $display("StoreAndForward::writeData: %d: data:%h, tag:%h, last:%h", cycle, v.data, 0, v.eop);
      writeDataFifo.enq(MemData {data: v.data, tag: 0, last: v.eop});
   endrule

   rule packetReadDone;
      let v <- toGet(writeDoneFifo).get;
      let recvd <- toGet(eventPktReceivedFifo).get;
      eventPktCommittedFifo.enq(recvd);
      if (verbose) $display("StoreAndForward::packetReadDone %d: packet written to memory %h", cycle, v);
   endrule

   interface PktReadClient readClient;
      interface readData = toPut(readDataFifo);
      interface readLen = toPut(readLenFifo);
      interface readReq = toGet(readReqFifo);
   endinterface

   interface malloc = (interface MemAllocClient;
      interface Get mallocReq = toGet(mallocReqFifo);
      interface Put mallocDone = toPut(mallocDoneFifo);
   endinterface);
   interface writeClient = dmaWriteClient;
   interface PipeOut eventPktCommitted = toPipeOut(eventPktCommittedFifo);
endmodule

interface StoreAndFwdFromMemToRing;
   interface PktWriteClient writeClient;
   interface MemReadClient#(`DataBusWidth) readClient;
   interface PipeIn#(PacketInstance) eventPktSend;
   interface MemFreeClient free;
endinterface

module mkStoreAndFwdFromMemToRing(StoreAndFwdFromMemToRing)
   provisos (Div#(`DataBusWidth, 8, bytesPerBeat)
            ,Log#(bytesPerBeat, beatShift));

   let verbose = False;

   // Ring Buffer Write Client
   FIFO#(EtherData) writeDataFifo <- mkFIFO;

   // read client interface
   FIFO#(MemRequest) readReqFifo <-mkSizedFIFO(4);
   FIFO#(MemData#(`DataBusWidth)) readDataFifo <- mkSizedFIFO(32);
   MemReadClient#(`DataBusWidth) dmaReadClient = (interface MemReadClient;
   interface Get readReq = toGet(readReqFifo);
   interface Put readData = toPut(readDataFifo);
   endinterface);

   FIFOF#(PacketInstance) eventPktSendFifo <- mkSizedFIFOF(16);
   FIFOF#(Bit#(EtherLen)) readBurstLenFifo <- mkSizedFIFOF(16);
   FIFOF#(PktId) currPacketIdFifo <- mkSizedFIFOF(16);
   FIFO#(PktId) freeReqFifo <- mkSizedFIFO(4);

   Reg#(Bool)                 outPacket <- mkReg(False);
   Reg#(Bit#(EtherLen))   readBurstCount <- mkReg(maxBound);
   Reg#(Bool)                 started <- mkReg(False);

   Reg#(Bit#(32)) cycle <- mkReg(0);
   rule every1 if (verbose);
      cycle <= cycle + 1;
   endrule

   rule packetReadStart;
      let pkt <- toGet(eventPktSendFifo).get;
      let bytesPerBeatMinusOne = fromInteger(valueOf(bytesPerBeat))-1;
      // roundup to 16 byte boundary
      let burstLen = ((pkt.size + bytesPerBeatMinusOne) & ~(bytesPerBeatMinusOne));
      if (verbose) $display("StoreAndForward:: packetReadStart: %h, burstLen = %h", pkt.size, burstLen);
      let mask = (1<< (pkt.size % fromInteger(valueOf(bytesPerBeat))))-1;
      readReqFifo.enq(MemRequest{sglId: extend(pkt.id), offset: 0,
                                 burstLen: truncate(burstLen), tag: 0
`ifdef BYTE_ENABLES
                                 , firstbe: 'hffff, lastbe: mask
`endif
                                });
      if (verbose) $display("StoreAndForward::packetReadStart %d: send a new packet with size %h %h", cycle, burstLen, pack(mask));
      currPacketIdFifo.enq(pkt.id);
      readBurstLenFifo.enq(pkt.size);
   endrule

   rule packetReadInProgress if (readBurstLenFifo.notEmpty);
      let d <- toGet(readDataFifo).get;
      let _bytesPerBeat= fromInteger(valueOf(bytesPerBeat));

      let readBurstLen = readBurstLenFifo.first;
      let currPacketId = currPacketIdFifo.first;

      let sop = False;
      let eop = False;
      if (!started) begin
         readBurstCount <= readBurstLen - _bytesPerBeat;
         started <= True;
         sop = True;
      end
      else if (readBurstCount <= _bytesPerBeat && started) begin
         readBurstLenFifo.deq;
         currPacketIdFifo.deq;
         readBurstCount <= maxBound;
         started <= False;
         eop = True;
         freeReqFifo.enq(currPacketId);
      end
      else if (readBurstCount > _bytesPerBeat && started) begin
         readBurstCount <= readBurstCount - _bytesPerBeat;
      end

      Bit#(bytesPerBeat) mask;
      if (readBurstCount <= _bytesPerBeat)
         mask = (1<<readBurstCount)-1;
      else
         mask = (1<<_bytesPerBeat)-1;

      writeDataFifo.enq(EtherData{data: d.data, mask: mask, sop: sop, eop: eop});

      if (verbose) $display("StoreAndForward::readdata %d: %h %h %h %h %h", cycle, readBurstCount, d.data, pack(mask), sop, eop);
   endrule

   interface PktWriteClient writeClient;
      interface writeData = toGet(writeDataFifo);
   endinterface
   interface readClient = dmaReadClient;
   interface PipeIn eventPktSend = toPipeIn(eventPktSendFifo);
   interface free = (interface MemFreeClient;
      interface Get freeReq = toGet(freeReqFifo);
   endinterface);
endmodule

interface StoreAndFwdFromRingToMac;
   interface PktReadClient readClient;
   interface Get#(PacketDataT#(64)) macTx;
   method TxThruDbgRec dbg;
   method ThruDbgRec sdbg;
endinterface

// store data from ring buffer to network
// big-endianess -> little-endianess
module mkStoreAndFwdFromRingToMac#(Clock txClock, Reset txReset)(StoreAndFwdFromRingToMac);
   let verbose = False;
   Clock defaultClock <- exposeCurrentClock();
   Reset defaultReset <- exposeCurrentReset();

   Reg#(Bit#(64)) cycle_cnt <- mkReg(0);
   Reg#(Bit#(64)) last_startofpacket <- mkReg(0);
   Reg#(Bit#(64)) last_endofpacket <- mkReg(0);
   Reg#(Bit#(64)) goodputCount <- mkReg(0);
   Reg#(Bit#(64)) idleCount <- mkReg(0);

   // stats
   Reg#(Bit#(64)) total_cycles <- mkReg(0, clocked_by txClock, reset_by txReset);
   Reg#(Bit#(64)) idle_cycles <- mkReg(0, clocked_by txClock, reset_by txReset);
   Reg#(Bit#(64)) sopCount <- mkReg(0, clocked_by txClock, reset_by txReset);
   Reg#(Bit#(64)) eopCount <- mkReg(0, clocked_by txClock, reset_by txReset);
   Reg#(Bit#(64)) data_bytes <- mkReg(0, clocked_by txClock, reset_by txReset);

   // RingBuffer Read Client
   FIFO#(EtherData) readDataFifo <- mkFIFO;
   FIFO#(Bit#(EtherLen)) readLenFifo <- mkFIFO;
   FIFO#(EtherReq) readReqFifo <- mkFIFO;

   // Mac Facing Fifo
   FIFOF#(PacketDataT#(64)) writeMacFifo <- mkFIFOF(clocked_by txClock, reset_by txReset);
   Gearbox#(2, 1, PacketDataT#(64)) fifoTxData <- mkNto1Gearbox(txClock, txReset, txClock, txReset);
   SyncFIFOIfc#(EtherData) tx_fifo <- mkSyncFIFO(5, defaultClock, defaultReset, txClock);

   rule cycle;
      cycle_cnt <= cycle_cnt + 1;
   endrule

   rule total_cycle;
      total_cycles <= total_cycles + 1;
   endrule

   rule readDataStart;
      let pktLen <- toGet(readLenFifo).get;
      if (verbose) $display(fshow(" read packt ") + fshow(pktLen));
      readReqFifo.enq(EtherReq{len: pktLen});
   endrule

   function Vector#(2, PacketDataT#(64)) split(EtherData in);
      Vector#(2, PacketDataT#(64)) v = defaultValue;
      Vector#(8, Bit#(8)) v0_data = unpack(in.data[63:0]);
      Vector#(8, Bit#(8)) v1_data = unpack(in.data[127:64]);
      v[0].sop = pack(in.sop);
`ifdef ALTERA
      v[0].data = pack(reverse(v0_data));
`else
      v[0].data = pack(v0_data);
`endif
      v[0].eop = (in.mask[15:8] == 0) ? pack(in.eop) : 0;
      v[0].mask = in.mask[7:0];
      v[1].sop = 0;
`ifdef ALTERA
      v[1].data = pack(reverse(v1_data));
`else
      v[1].data = pack(v1_data);
`endif
      v[1].eop = pack(in.eop);
      v[1].mask = in.mask[15:8];
      return v;
   endfunction

   rule cross_clocking;
      let v <- toGet(readDataFifo).get;
      tx_fifo.enq(v);

      // performance analysis
      if (v.sop) begin
         last_startofpacket <= cycle_cnt;
         idleCount <= (last_endofpacket != 0) ? (idleCount + (cycle_cnt - last_endofpacket)) : 0;
      end
      if (v.eop) begin
         last_endofpacket <= cycle_cnt;
         goodputCount <= (last_startofpacket != 0) ? (goodputCount + (cycle_cnt - last_startofpacket)) : 0;
      end
   endrule

   rule process_incoming_packet;
      let v <- toGet(tx_fifo).get;
      fifoTxData.enq(split(v));
   endrule

   rule process_outgoing_packet;
      let data = fifoTxData.first; fifoTxData.deq;
      let temp = head(data);
      let bytes = zeroExtend(pack(countOnes(temp.mask)));
      if (unpack(temp.sop)) sopCount <= sopCount + 1;
      data_bytes <= data_bytes + bytes;
      if (temp.mask != 0) begin
         if (verbose) $display("ringToMac:: tx data %h", temp.data);
         if (unpack(temp.eop)) eopCount <= eopCount + 1;
         writeMacFifo.enq(temp);
      end
      else begin
         idle_cycles <= idle_cycles + 1;
      end
   endrule

   rule count_idle_cycles (!fifoTxData.notEmpty);
      idle_cycles <= idle_cycles + 1;
   endrule

   interface PktReadClient readClient;
      interface readData = toPut(readDataFifo);
      interface readLen = toPut(readLenFifo);
      interface readReq = toGet(readReqFifo);
   endinterface
   interface Get macTx = toGet(writeMacFifo);
   method TxThruDbgRec dbg;
      return TxThruDbgRec {goodputCount: goodputCount, idleCount: idleCount};
   endmethod
   method ThruDbgRec sdbg;
      return ThruDbgRec {data_bytes: data_bytes, sops: sopCount, eops: eopCount, idle_cycles: idle_cycles, total_cycles: total_cycles};
   endmethod
endmodule

interface StoreAndFwdFromMacToRing;
   interface PktWriteClient writeClient;
   interface Put#(PacketDataT#(64)) macRx;
   method ThruDbgRec sdbg;
endinterface

// store data from network to ring buffer
// little-endianess -> big-endianess
module mkStoreAndFwdFromMacToRing#(Clock rxClock, Reset rxReset)(StoreAndFwdFromMacToRing);
   let verbose = False;
   Clock defaultClock <- exposeCurrentClock();
   Reset defaultReset <- exposeCurrentReset();

   // Ring Buffer WriteClient
   FIFO#(EtherData) writeDataFifo <- mkFIFO;

   // stats
   Reg#(Bit#(64)) total_cycles <- mkReg(0, clocked_by rxClock, reset_by rxReset);
   Reg#(Bit#(64)) idle_cycles <- mkReg(0, clocked_by rxClock, reset_by rxReset);
   Reg#(Bit#(64)) sopCount <- mkReg(0, clocked_by rxClock, reset_by rxReset);
   Reg#(Bit#(64)) eopCount <- mkReg(0, clocked_by rxClock, reset_by rxReset);
   Reg#(Bit#(64)) data_bytes <- mkReg(0, clocked_by rxClock, reset_by rxReset);

   // Mac facing fifos
   Reg#(Bool) inProgress <- mkReg(False, clocked_by rxClock, reset_by rxReset);
   Reg#(Bool) oddBeat    <- mkReg(True, clocked_by rxClock, reset_by rxReset);
   Reg#(PacketDataT#(64)) v_prev <- mkReg(defaultValue, clocked_by rxClock, reset_by rxReset);

   FIFO#(PacketDataT#(64)) readMacFifo <- mkFIFO(clocked_by rxClock, reset_by rxReset);
   SyncFIFOIfc#(EtherData) rx_fifo <- mkSyncFIFO(5, rxClock, rxReset, defaultClock);

   rule total_cycle;
      total_cycles <= total_cycles + 1;
   endrule

   function EtherData combine(Vector#(2, PacketDataT#(64)) v);
      EtherData data = defaultValue;
      Vector#(8, Bit#(8)) v0_data = unpack(v[0].data);
      Vector#(8, Bit#(8)) v1_data = unpack(v[1].data);
`ifdef ALTERA
      data.data = {pack(reverse(v1_data)), pack(reverse(v0_data))};
`else
      data.data = {pack(v1_data), pack(v0_data)};
`endif
      data.mask = {v[1].mask, v[0].mask};
      data.sop = unpack(v[0].sop);
      data.eop = unpack(v[0].eop) || unpack(v[1].eop);
      return data;
   endfunction

   rule startOfPacket if (!inProgress);
      let v = readMacFifo.first;
      inProgress <= unpack(v.sop);
      if (v.sop == 0)
         readMacFifo.deq;
      else
         sopCount <= sopCount + 1;
      if (verbose) $display("macToRing:: start");
   endrule

   rule readPacketOdd if (inProgress && oddBeat);
      let v <- toGet(readMacFifo).get;
      let bytes = zeroExtend(pack(countOnes(v.mask)));
      data_bytes <= data_bytes + bytes;
      if (verbose) $display("macToRing:: read odd beat %h", v.data);
      if (unpack(v.eop)) begin
         PacketDataT#(64) vo = defaultValue;
         rx_fifo.enq(combine(vec(v, vo)));
         if (verbose) $display("macToRing:: odd eop %h %h", v.data, v.mask);
         inProgress <= False;
         eopCount <= eopCount +1;
      end
      else begin
         oddBeat <= !oddBeat;
      end
      v_prev <= v;
   endrule

   rule readPacketEven if (inProgress && !oddBeat);
      let v <- toGet(readMacFifo).get;
      let bytes = zeroExtend(pack(countOnes(v.mask)));
      data_bytes <= data_bytes + bytes;
      rx_fifo.enq(combine(vec(v_prev, v)));
      if (verbose) $display("macToRing:: read even beat %h", v.data);
      if (unpack(v.eop)) begin
         inProgress <= False;
         if (verbose) $display("macToRing:: even eop %h %h %h %h", v.data, v.mask, v_prev.data, v_prev.mask);
         eopCount <= eopCount +1;
      end
      oddBeat <= !oddBeat;
   endrule

   rule write_data;
      let v <- toGet(rx_fifo).get;
      writeDataFifo.enq(v);
      if (verbose) $display("macToRing:: writeToFifo");
   endrule

   rule count_idle_cycles (!inProgress);
      idle_cycles <= idle_cycles + 1;
   endrule

   interface PktWriteClient writeClient;
      interface writeData = toGet(writeDataFifo);
   endinterface
   interface Put macRx = toPut(readMacFifo);
   method ThruDbgRec sdbg;
      return ThruDbgRec {data_bytes: data_bytes, sops: sopCount, eops: eopCount, idle_cycles: idle_cycles, total_cycles: total_cycles};
   endmethod
endmodule

// Copyright (c) 2016 Cornell University.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

// NOTE:
// Implement a store-and-forward mechanism between ring-buffer and
// main packet memory. Packets are buffered completely in ring buffer
// before it is sent to main memory and vice versa.

import BuildVector::*;
import Connectable::*;
import Clocks::*;
import Cntrs::*;
import ClientServer::*;
import DefaultValue::*;
import DbgTypes::*;
import Ethernet::*;
import EthMac::*;
import FIFO::*;
import FIFOF::*;
import GetPut::*;
import Gearbox::*;
import MemTypes::*;
import MemServer::*;
import MemServerInternal::*;
import MemMgmt::*;
import PacketBuffer::*;
import SharedBuff::*;
import SharedBuffMMU::*;
import SpecialFIFOs::*;
import Vector::*;
import Pipe::*;
import DbgTypes::*;

interface StoreAndFwdFromRingToMem;
   interface PktReadClient readClient;
   interface MemAllocClient malloc;
   interface MemWriteClient#(`DataBusWidth) writeClient;
   interface PipeOut#(PacketInstance) eventPktCommitted;
endinterface

module mkStoreAndFwdFromRingToMem(StoreAndFwdFromRingToMem)
   provisos (Div#(`DataBusWidth, 8, bytesPerBeat)
            ,Log#(bytesPerBeat, beatShift));

   let verbose = False;

   // RingBuffer Read Client
   FIFO#(EtherData) readDataFifo <- mkFIFO;
   FIFO#(Bit#(EtherLen)) readLenFifo <- mkFIFO;
   FIFO#(EtherReq) readReqFifo <- mkFIFO;

   // Memory Client
   FIFO#(MemRequest) writeReqFifo <- mkSizedFIFO(4);
   FIFO#(MemData#(`DataBusWidth)) writeDataFifo <- mkSizedFIFO(16);
   FIFO#(Bit#(MemTagSize)) writeDoneFifo <- mkSizedFIFO(4);
   MemWriteClient#(`DataBusWidth) dmaWriteClient = (interface MemWriteClient;
   interface Get writeReq = toGet(writeReqFifo);
   interface Get writeData = toGet(writeDataFifo);
   interface Put writeDone = toPut(writeDoneFifo);
   endinterface);

   FIFO#(Bit#(EtherLen)) mallocReqFifo <- mkFIFO;
   FIFO#(Bit#(EtherLen)) pktLenFifo <- mkFIFO;
   FIFO#(Maybe#(PktId)) mallocDoneFifo <- mkFIFO;
   Reg#(Bool) readStarted <- mkReg(False);
   Reg#(Bool) mallocd <- mkReg(False);

   FIFOF#(PacketInstance) eventPktReceivedFifo <- mkFIFOF;
   FIFOF#(PacketInstance) eventPktCommittedFifo <- mkFIFOF;

   Reg#(Bit#(32)) cycle <- mkReg(0);
   rule every1 if (verbose);
      cycle <= cycle + 1;
   endrule

   rule packetReadStart if (!readStarted);
      let pktLen <- toGet(readLenFifo).get;
      if (verbose) $display("StoreAndForward::packetReadStart %d: ReadLen %d", cycle, pktLen);
      mallocReqFifo.enq(pktLen);
      pktLenFifo.enq(pktLen);
      readStarted <= True;
   endrule

   rule allocMemory;
      let pktLen <- toGet(pktLenFifo).get;
      let allocId <- toGet(mallocDoneFifo).get;
      let bytesPerBeatMinusOne = fromInteger(valueOf(bytesPerBeat))-1;
      // roundup to 16 byte boundary
      let burstLen = ((pktLen + bytesPerBeatMinusOne) & ~(bytesPerBeatMinusOne));
      let mask = (1<< (pktLen % fromInteger(valueOf(bytesPerBeat))))-1;
      if (isValid(allocId)) begin
         mallocd <= True;
         readReqFifo.enq(EtherReq{len: truncate(pktLen)});
         //FIXME use correct sglId
         writeReqFifo.enq(MemRequest {sglId: extend(fromMaybe(?, allocId)), offset: 0,
                                      burstLen: truncate(burstLen), tag:0
`ifdef BYTE_ENABLES
                                      , firstbe: 'hffff, lastbe: mask
`endif
                                     });
         if (verbose) $display("StoreAndForward::allocMemory %d: alloc done", cycle);
         eventPktReceivedFifo.enq(PacketInstance {id: fromMaybe(?, allocId), size: pktLen});
      end
   endrule

   rule packetReadInProgress if (readStarted && mallocd);
      let v <- toGet(readDataFifo).get;
      if (v.eop) begin
         readStarted <= False;
         mallocd <= False;
         if (verbose) $display("StoreAndForward:: %d: packet finished", cycle);
      end
      if (verbose) $display("StoreAndForward::writeData: %d: data:%h, tag:%h, last:%h", cycle, v.data, 0, v.eop);
      writeDataFifo.enq(MemData {data: v.data, tag: 0, last: v.eop});
   endrule

   rule packetReadDone;
      let v <- toGet(writeDoneFifo).get;
      let recvd <- toGet(eventPktReceivedFifo).get;
      eventPktCommittedFifo.enq(recvd);
      if (verbose) $display("StoreAndForward::packetReadDone %d: packet written to memory %h", cycle, v);
   endrule

   interface PktReadClient readClient;
      interface readData = toPut(readDataFifo);
      interface readLen = toPut(readLenFifo);
      interface readReq = toGet(readReqFifo);
   endinterface

   interface malloc = (interface MemAllocClient;
      interface Get mallocReq = toGet(mallocReqFifo);
      interface Put mallocDone = toPut(mallocDoneFifo);
   endinterface);
   interface writeClient = dmaWriteClient;
   interface PipeOut eventPktCommitted = toPipeOut(eventPktCommittedFifo);
endmodule

interface StoreAndFwdFromMemToRing;
   interface PktWriteClient writeClient;
   interface MemReadClient#(`DataBusWidth) readClient;
   interface PipeIn#(PacketInstance) eventPktSend;
   interface MemFreeClient free;
endinterface

module mkStoreAndFwdFromMemToRing(StoreAndFwdFromMemToRing)
   provisos (Div#(`DataBusWidth, 8, bytesPerBeat)
            ,Log#(bytesPerBeat, beatShift));

   let verbose = False;

   // Ring Buffer Write Client
   FIFO#(EtherData) writeDataFifo <- mkFIFO;

   // read client interface
   FIFO#(MemRequest) readReqFifo <-mkSizedFIFO(4);
   FIFO#(MemData#(`DataBusWidth)) readDataFifo <- mkSizedFIFO(32);
   MemReadClient#(`DataBusWidth) dmaReadClient = (interface MemReadClient;
   interface Get readReq = toGet(readReqFifo);
   interface Put readData = toPut(readDataFifo);
   endinterface);

   FIFOF#(PacketInstance) eventPktSendFifo <- mkSizedFIFOF(4);
   FIFO#(PktId) freeReqFifo <- mkSizedFIFO(4);

   Reg#(Bool)                 outPacket <- mkReg(False);
   Reg#(PktId)            currPacketId <- mkReg(0);
   Reg#(Bit#(EtherLen))   readBurstCount <- mkReg(0);
   Reg#(Bit#(EtherLen))   readBurstLen <- mkReg(0);

   Reg#(Bit#(32)) cycle <- mkReg(0);
   rule every1 if (verbose);
      cycle <= cycle + 1;
   endrule

   rule packetReadStart (!outPacket);
      let pkt <- toGet(eventPktSendFifo).get;
      let bytesPerBeatMinusOne = fromInteger(valueOf(bytesPerBeat))-1;
      // roundup to 16 byte boundary
      let burstLen = ((pkt.size + bytesPerBeatMinusOne) & ~(bytesPerBeatMinusOne));
      if (verbose) $display("StoreAndForward:: packetReadStart: %h, burstLen = %h", pkt.size, burstLen);
      let mask = (1<< (pkt.size % fromInteger(valueOf(bytesPerBeat))))-1;
      readReqFifo.enq(MemRequest{sglId: extend(pkt.id), offset: 0,
                                 burstLen: truncate(burstLen), tag: 0
`ifdef BYTE_ENABLES
                                 , firstbe: 'hffff, lastbe: mask
`endif
                                });
      if (verbose) $display("StoreAndForward::packetReadStart %d: send a new packet with size %h %h", cycle, burstLen, pack(mask));
      outPacket <= True;
      currPacketId <= pkt.id;
      readBurstLen <= pkt.size;
      readBurstCount <= pkt.size;
   endrule

   rule packetReadInProgress (outPacket);
      let d <- toGet(readDataFifo).get;
      let _bytesPerBeat= fromInteger(valueOf(bytesPerBeat));

      let sop = (readBurstLen == readBurstCount) ? True : False;
      let eop = (readBurstCount <= _bytesPerBeat) ? True : False;

      Bit#(bytesPerBeat) mask;
      if (readBurstCount <= _bytesPerBeat)
         mask = (1<<readBurstCount)-1;
      else
         mask = (1<<_bytesPerBeat)-1;

      writeDataFifo.enq(EtherData{data: d.data, mask: mask, sop: sop, eop: eop});

      if (verbose) $display("StoreAndForward::readdata %d: %h %h %h %h %h", cycle, readBurstCount, d.data, pack(mask), sop, eop);

      if (readBurstCount > _bytesPerBeat)
         readBurstCount <= readBurstCount - _bytesPerBeat;

      if (eop) begin
         outPacket <= False;
         freeReqFifo.enq(currPacketId);
      end
   endrule

   interface PktWriteClient writeClient;
      interface writeData = toGet(writeDataFifo);
   endinterface
   interface readClient = dmaReadClient;
   interface PipeIn eventPktSend = toPipeIn(eventPktSendFifo);
   interface free = (interface MemFreeClient;
      interface Get freeReq = toGet(freeReqFifo);
   endinterface);
endmodule

interface StoreAndFwdFromRingToMac;
   interface PktReadClient readClient;
   interface Get#(PacketDataT#(64)) macTx;
   method TxThruDbgRec dbg;
   method ThruDbgRec sdbg;
endinterface

// store data from ring buffer to network
// big-endianess -> little-endianess
module mkStoreAndFwdFromRingToMac#(Clock txClock, Reset txReset)(StoreAndFwdFromRingToMac);
   let verbose = False;
   Clock defaultClock <- exposeCurrentClock();
   Reset defaultReset <- exposeCurrentReset();

   Reg#(Bit#(64)) cycle_cnt <- mkReg(0);
   Reg#(Bit#(64)) last_startofpacket <- mkReg(0);
   Reg#(Bit#(64)) last_endofpacket <- mkReg(0);
   Reg#(Bit#(64)) goodputCount <- mkReg(0);
   Reg#(Bit#(64)) idleCount <- mkReg(0);

   // stats
   Reg#(Bit#(64)) total_cycles <- mkReg(0, clocked_by txClock, reset_by txReset);
   Reg#(Bit#(64)) idle_cycles <- mkReg(0, clocked_by txClock, reset_by txReset);
   Reg#(Bit#(64)) sopCount <- mkReg(0, clocked_by txClock, reset_by txReset);
   Reg#(Bit#(64)) eopCount <- mkReg(0, clocked_by txClock, reset_by txReset);
   Reg#(Bit#(64)) data_bytes <- mkReg(0, clocked_by txClock, reset_by txReset);

   // RingBuffer Read Client
   FIFO#(EtherData) readDataFifo <- mkFIFO;
   FIFO#(Bit#(EtherLen)) readLenFifo <- mkFIFO;
   FIFO#(EtherReq) readReqFifo <- mkFIFO;

   // Mac Facing Fifo
   FIFOF#(PacketDataT#(64)) writeMacFifo <- mkFIFOF(clocked_by txClock, reset_by txReset);
   Gearbox#(2, 1, PacketDataT#(64)) fifoTxData <- mkNto1Gearbox(txClock, txReset, txClock, txReset);
   SyncFIFOIfc#(EtherData) tx_fifo <- mkSyncFIFO(5, defaultClock, defaultReset, txClock);

   rule cycle;
      cycle_cnt <= cycle_cnt + 1;
   endrule

   rule total_cycle;
      total_cycles <= total_cycles + 1;
   endrule

   rule readDataStart;
      let pktLen <- toGet(readLenFifo).get;
      if (verbose) $display(fshow(" read packt ") + fshow(pktLen));
      readReqFifo.enq(EtherReq{len: pktLen});
   endrule

   function Vector#(2, PacketDataT#(64)) split(EtherData in);
      Vector#(2, PacketDataT#(64)) v = defaultValue;
      Vector#(8, Bit#(8)) v0_data = unpack(in.data[63:0]);
      Vector#(8, Bit#(8)) v1_data = unpack(in.data[127:64]);
      v[0].sop = pack(in.sop);
`ifdef ALTERA
      v[0].data = pack(reverse(v0_data));
`else
      v[0].data = pack(v0_data);
`endif
      v[0].eop = (in.mask[15:8] == 0) ? pack(in.eop) : 0;
      v[0].mask = in.mask[7:0];
      v[1].sop = 0;
`ifdef ALTERA
      v[1].data = pack(reverse(v1_data));
`else
      v[1].data = pack(v1_data);
`endif
      v[1].eop = pack(in.eop);
      v[1].mask = in.mask[15:8];
      return v;
   endfunction

   rule cross_clocking;
      let v <- toGet(readDataFifo).get;
      tx_fifo.enq(v);

      // performance analysis
      if (v.sop) begin
         last_startofpacket <= cycle_cnt;
         idleCount <= (last_endofpacket != 0) ? (idleCount + (cycle_cnt - last_endofpacket)) : 0;
      end
      if (v.eop) begin
         last_endofpacket <= cycle_cnt;
         goodputCount <= (last_startofpacket != 0) ? (goodputCount + (cycle_cnt - last_startofpacket)) : 0;
      end
   endrule

   rule process_incoming_packet;
      let v <- toGet(tx_fifo).get;
      fifoTxData.enq(split(v));
   endrule

   rule process_outgoing_packet;
      let data = fifoTxData.first; fifoTxData.deq;
      let temp = head(data);
      let bytes = zeroExtend(pack(countOnes(temp.mask)));
      if (unpack(temp.sop)) sopCount <= sopCount + 1;
      data_bytes <= data_bytes + bytes;
      if (temp.mask != 0) begin
         if (verbose) $display("ringToMac:: tx data %h", temp.data);
         if (unpack(temp.eop)) eopCount <= eopCount + 1;
         writeMacFifo.enq(temp);
      end
      else begin
         idle_cycles <= idle_cycles + 1;
      end
   endrule

   rule count_idle_cycles (!fifoTxData.notEmpty);
      idle_cycles <= idle_cycles + 1;
   endrule

   interface PktReadClient readClient;
      interface readData = toPut(readDataFifo);
      interface readLen = toPut(readLenFifo);
      interface readReq = toGet(readReqFifo);
   endinterface
   interface Get macTx = toGet(writeMacFifo);
   method TxThruDbgRec dbg;
      return TxThruDbgRec {goodputCount: goodputCount, idleCount: idleCount};
   endmethod
   method ThruDbgRec sdbg;
      return ThruDbgRec {data_bytes: data_bytes, sops: sopCount, eops: eopCount, idle_cycles: idle_cycles, total_cycles: total_cycles};
   endmethod
endmodule

interface StoreAndFwdFromMacToRing;
   interface PktWriteClient writeClient;
   interface Put#(PacketDataT#(64)) macRx;
   method ThruDbgRec sdbg;
endinterface

// store data from network to ring buffer
// little-endianess -> big-endianess
module mkStoreAndFwdFromMacToRing#(Clock rxClock, Reset rxReset)(StoreAndFwdFromMacToRing);
   let verbose = False;
   Clock defaultClock <- exposeCurrentClock();
   Reset defaultReset <- exposeCurrentReset();

   // Ring Buffer WriteClient
   FIFO#(EtherData) writeDataFifo <- mkFIFO;

   // stats
   Reg#(Bit#(64)) total_cycles <- mkReg(0, clocked_by rxClock, reset_by rxReset);
   Reg#(Bit#(64)) idle_cycles <- mkReg(0, clocked_by rxClock, reset_by rxReset);
   Reg#(Bit#(64)) sopCount <- mkReg(0, clocked_by rxClock, reset_by rxReset);
   Reg#(Bit#(64)) eopCount <- mkReg(0, clocked_by rxClock, reset_by rxReset);
   Reg#(Bit#(64)) data_bytes <- mkReg(0, clocked_by rxClock, reset_by rxReset);

   // Mac facing fifos
   Reg#(Bool) inProgress <- mkReg(False, clocked_by rxClock, reset_by rxReset);
   Reg#(Bool) oddBeat    <- mkReg(True, clocked_by rxClock, reset_by rxReset);
   Reg#(PacketDataT#(64)) v_prev <- mkReg(defaultValue, clocked_by rxClock, reset_by rxReset);

   FIFO#(PacketDataT#(64)) readMacFifo <- mkFIFO(clocked_by rxClock, reset_by rxReset);
   SyncFIFOIfc#(EtherData) rx_fifo <- mkSyncFIFO(5, rxClock, rxReset, defaultClock);

   rule total_cycle;
      total_cycles <= total_cycles + 1;
   endrule

   function EtherData combine(Vector#(2, PacketDataT#(64)) v);
      EtherData data = defaultValue;
      Vector#(8, Bit#(8)) v0_data = unpack(v[0].data);
      Vector#(8, Bit#(8)) v1_data = unpack(v[1].data);
`ifdef ALTERA
      data.data = {pack(reverse(v1_data)), pack(reverse(v0_data))};
`else
      data.data = {pack(v1_data), pack(v0_data)};
`endif
      data.mask = {v[1].mask, v[0].mask};
      data.sop = unpack(v[0].sop);
      data.eop = unpack(v[0].eop) || unpack(v[1].eop);
      return data;
   endfunction

   rule startOfPacket if (!inProgress);
      let v = readMacFifo.first;
      inProgress <= unpack(v.sop);
      if (v.sop == 0)
         readMacFifo.deq;
      else
         sopCount <= sopCount + 1;
      if (verbose) $display("macToRing:: start");
   endrule

   rule readPacketOdd if (inProgress && oddBeat);
      let v <- toGet(readMacFifo).get;
      let bytes = zeroExtend(pack(countOnes(v.mask)));
      data_bytes <= data_bytes + bytes;
      if (verbose) $display("macToRing:: read odd beat %h", v.data);
      if (unpack(v.eop)) begin
         PacketDataT#(64) vo = defaultValue;
         rx_fifo.enq(combine(vec(v, vo)));
         if (verbose) $display("macToRing:: odd eop %h %h", v.data, v.mask);
         inProgress <= False;
         eopCount <= eopCount +1;
      end
      else begin
         oddBeat <= !oddBeat;
      end
      v_prev <= v;
   endrule

   rule readPacketEven if (inProgress && !oddBeat);
      let v <- toGet(readMacFifo).get;
      let bytes = zeroExtend(pack(countOnes(v.mask)));
      data_bytes <= data_bytes + bytes;
      rx_fifo.enq(combine(vec(v_prev, v)));
      if (verbose) $display("macToRing:: read even beat %h", v.data);
      if (unpack(v.eop)) begin
         inProgress <= False;
         if (verbose) $display("macToRing:: even eop %h %h %h %h", v.data, v.mask, v_prev.data, v_prev.mask);
         eopCount <= eopCount +1;
      end
      oddBeat <= !oddBeat;
   endrule

   rule write_data;
      let v <- toGet(rx_fifo).get;
      writeDataFifo.enq(v);
      if (verbose) $display("macToRing:: writeToFifo");
   endrule

   rule count_idle_cycles (!inProgress);
      idle_cycles <= idle_cycles + 1;
   endrule

   interface PktWriteClient writeClient;
      interface writeData = toGet(writeDataFifo);
   endinterface
   interface Put macRx = toPut(readMacFifo);
   method ThruDbgRec sdbg;
      return ThruDbgRec {data_bytes: data_bytes, sops: sopCount, eops: eopCount, idle_cycles: idle_cycles, total_cycles: total_cycles};
   endmethod
endmodule


removed fake GetPut.bsv

removed fake GetPut.bsv

interface Get#(type a);
    method ActionValue#(a) get();
endinterface

interface Put#(type a);
    method Action put(a v);
endinterface


reduce verbosity

reduce verbosity

Change-Id: I45c766b3795724957e43214344623f55c3814296

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import Vector::*;
import FIFOF::*;
import FIFO::*;
import GetPut::*;
import Assert::*;
import ClientServer::*;
import BRAM::*;

import PortalMemory::*;
import Dma::*;
import Pipe::*;

typedef struct {ObjectPointer pointer;
		Bit#(ObjectOffsetSize) base;
		Bit#(8) burstLen;
		Bit#(32) readLen;
		} MemengineCmd deriving (Eq,Bits);

interface MemreadEngineV#(numeric type dataWidth, numeric type cmdQDepth, numeric type numServers);
   interface Vector#(numServers, Server#(MemengineCmd,Bool)) readServers;
   interface ObjectReadClient#(dataWidth) dmaClient;
   interface Vector#(numServers, PipeOut#(Bit#(dataWidth))) dataPipes;
endinterface

module mkMemreadEngineV(MemreadEngineV#(dataWidth, cmdQDepth, numServers))
   provisos (Div#(dataWidth,8,dataWidthBytes),
	     Mul#(dataWidthBytes,8,dataWidth),
	     Log#(dataWidthBytes,beatShift),
	     Mul#(cmdQDepth,numServers,cmdBuffSz),
	     Log#(cmdBuffSz, cmdBuffAddrSz),
	     Log#(numServers, serverIdxSz),
	     Add#(1,cmdQDepth, outCntSz),
	     Add#(1, c__, numServers),
	     Add#(b__, TLog#(numServers), cmdBuffAddrSz),
	     Add#(a__, serverIdxSz, cmdBuffAddrSz));

   function Bit#(cmdBuffAddrSz) hf(Integer i) = fromInteger(i*valueOf(cmdQDepth));
   Vector#(numServers, Reg#(Bit#(outCntSz)))     outs1 <- replicateM(mkReg(0));
   Vector#(numServers, Reg#(Bit#(outCntSz)))     outs0 <- replicateM(mkReg(0));
   Vector#(numServers, Reg#(Bit#(cmdBuffAddrSz))) head <- mapM(mkReg, genWith(hf));
   Vector#(numServers, Reg#(Bit#(cmdBuffAddrSz))) tail <- mapM(mkReg, genWith(hf));

   BRAM1Port#(Bit#(cmdBuffAddrSz),MemengineCmd) cmdBuf <- mkBRAM1Server(defaultValue);
   FIFO#(Bit#(serverIdxSz))                      loadf <- mkSizedFIFO(1);
   FIFO#(Tuple3#(Bit#(8),Bit#(serverIdxSz),Bool))workf <- mkSizedFIFO(32); // isthis the right size?

   Vector#(numServers, FIFO#(void))              outfs <- replicateM(mkSizedFIFO(1));
   Vector#(numServers, FIFOF#(Tuple2#(Bit#(serverIdxSz), MemengineCmd))) cmds_in <- replicateM(mkSizedFIFOF(1));
   PipeOut#(Tuple2#(Bit#(serverIdxSz), MemengineCmd)) in_funnel <- mkFunnel1PipesPipelined(map(toPipeOut,cmds_in));
   FIFOF#(Tuple2#(Bit#(TLog#(numServers)), Tuple2#(Bit#(dataWidth),Bool))) read_data <- mkFIFOF;
   Vector#(numServers, PipeOut#(Tuple2#(Bit#(dataWidth),Bool))) out_unfunnel <- mkUnFunnel1PipesPipelined(toPipeOut(read_data));
   function PipeOut#(Bit#(dataWidth)) check_out(PipeOut#(Tuple2#(Bit#(dataWidth),Bool)) x, Integer i) =
      (interface PipeOut;
	  method Bit#(dataWidth) first;
	     return tpl_1(x.first);
	  endmethod
	  method Action deq;
	     x.deq;
	     if (tpl_2(x.first))
		outfs[i].enq(?);
	  endmethod
	  method Bool notEmpty = x.notEmpty;
       endinterface);
   Vector#(numServers, PipeOut#(Bit#(dataWidth))) out_data_pipes = zipWith(check_out, out_unfunnel, genVector);

   Reg#(Bit#(8))                               respCnt <- mkReg(0);
   Reg#(Bit#(serverIdxSz))                     loadIdx <- mkReg(0);
   let beat_shift = fromInteger(valueOf(beatShift));
   let cmd_q_depth = fromInteger(valueOf(cmdQDepth));

   rule store_cmd;
      match {.idx, .cmd} <- toGet(in_funnel).get;
      let new_tail = tail[idx]+1;
      if (new_tail >= extend(idx+1)*cmd_q_depth)
	 new_tail = extend(idx)*cmd_q_depth;
      tail[idx] <= new_tail;
      outs1[idx] <= outs1[idx]+1;
      cmdBuf.portA.request.put(BRAMRequest{write:True, responseOnWrite:False, address:tail[idx], datain:cmd});
      //$display("MemreadEngineV.store_cmd: %d %h", idx, tail[idx]);
   endrule

   rule load_ctxt;
      loadIdx <= loadIdx+1;
      if (outs1[loadIdx] > 0) begin
	 cmdBuf.portA.request.put(BRAMRequest{write:False, responseOnWrite:False, address:head[loadIdx], datain:?});
	 loadf.enq(loadIdx);
	 //$display("MemreadEngineV.load_ctxt: %d, %h", loadIdx, head[loadIdx]);
      end
   endrule

   Vector#(numServers, Server#(MemengineCmd,Bool)) rs;
   for(Integer i = 0; i < valueOf(numServers); i=i+1)
      rs[i] =  (interface Server#(MemengineCmd,Bool);
		   interface Put request;
		      method Action put(MemengineCmd c) if (outs0[i] < cmd_q_depth);
			 outs0[i] <= outs0[i]+1;
			 cmds_in[i].enq(tuple2(fromInteger(i),c));
 		      endmethod
		   endinterface
		   interface Get response;
		      method ActionValue#(Bool) get;
			 outfs[i].deq;
	 		 outs0[i] <= outs0[i]-1;
			 return True;
		      endmethod
		   endinterface
		endinterface);
   interface readServers = rs;
   interface ObjectReadClient dmaClient;
      interface Get readReq;
	 method ActionValue#(ObjectRequest) get();
	    let cmd <- cmdBuf.portA.response.get;
	    let idx <- toGet(loadf).get;
	    Bit#(8) bl = cmd.burstLen;
	    let last = False;
	    if (cmd.readLen <= extend(bl)) begin
	       last = True;
	       bl = truncate(cmd.readLen);
	       outs1[idx] <= outs1[idx]-1;
	       let new_head = head[idx]+1;
	       if (new_head >= extend(idx+1)*cmd_q_depth)
		  new_head = extend(idx)*cmd_q_depth;
	       head[idx] <= new_head;
	       //$display("new_head %d %d", idx, new_head);
	    end
	    let new_cmd = MemengineCmd{pointer:cmd.pointer, base:cmd.base+extend(bl), burstLen:cmd.burstLen, readLen:cmd.readLen-extend(bl)};
	    cmdBuf.portA.request.put(BRAMRequest{write:True, responseOnWrite:False, address:head[idx], datain:new_cmd});
	    workf.enq(tuple3(truncate(bl>>beat_shift), idx, last));
	    //$display("readReq %d, %h %h %h", idx, cmd.base, bl, last);
	    return ObjectRequest { pointer: cmd.pointer, offset: cmd.base, burstLen:bl, tag: 0 };
	 endmethod
      endinterface
      interface Put readData;
	 method Action put(ObjectData#(dataWidth) d);
	    match {.rc, .idx, .last} = workf.first;
	    let new_respCnt = respCnt+1;
	    let l = False;
	    //$display("%h %d", d.data, idx);
	    if (new_respCnt == rc) begin
	       respCnt <= 0;
	       workf.deq;
	       //$display("eob %d", idx);
	       l = last;
	    end
	    else begin
	       respCnt <= new_respCnt;
	    end
	    read_data.enq(tuple2(idx,tuple2(d.data,l)));
	 endmethod
      endinterface
   endinterface
   interface dataPipes = out_data_pipes;
endmodule

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import Vector::*;
import FIFOF::*;
import FIFO::*;
import GetPut::*;
import Assert::*;
import ClientServer::*;
import BRAM::*;

import PortalMemory::*;
import Dma::*;
import Pipe::*;

typedef struct {ObjectPointer pointer;
		Bit#(ObjectOffsetSize) base;
		Bit#(8) burstLen;
		Bit#(32) readLen;
		} MemengineCmd deriving (Eq,Bits);

interface MemreadEngineV#(numeric type dataWidth, numeric type cmdQDepth, numeric type numServers);
   interface Vector#(numServers, Server#(MemengineCmd,Bool)) readServers;
   interface ObjectReadClient#(dataWidth) dmaClient;
   interface Vector#(numServers, PipeOut#(Bit#(dataWidth))) dataPipes;
endinterface

module mkMemreadEngineV(MemreadEngineV#(dataWidth, cmdQDepth, numServers))
   provisos (Div#(dataWidth,8,dataWidthBytes),
	     Mul#(dataWidthBytes,8,dataWidth),
	     Log#(dataWidthBytes,beatShift),
	     Mul#(cmdQDepth,numServers,cmdBuffSz),
	     Log#(cmdBuffSz, cmdBuffAddrSz),
	     Log#(numServers, serverIdxSz),
	     Add#(1,cmdQDepth, outCntSz),
	     Add#(1, c__, numServers),
	     Add#(b__, TLog#(numServers), cmdBuffAddrSz),
	     Add#(a__, serverIdxSz, cmdBuffAddrSz));

   function Bit#(cmdBuffAddrSz) hf(Integer i) = fromInteger(i*valueOf(cmdQDepth));
   Vector#(numServers, Reg#(Bit#(outCntSz)))     outs1 <- replicateM(mkReg(0));
   Vector#(numServers, Reg#(Bit#(outCntSz)))     outs0 <- replicateM(mkReg(0));
   Vector#(numServers, Reg#(Bit#(cmdBuffAddrSz))) head <- mapM(mkReg, genWith(hf));
   Vector#(numServers, Reg#(Bit#(cmdBuffAddrSz))) tail <- mapM(mkReg, genWith(hf));

   BRAM1Port#(Bit#(cmdBuffAddrSz),MemengineCmd) cmdBuf <- mkBRAM1Server(defaultValue);
   FIFO#(Bit#(serverIdxSz))                      loadf <- mkSizedFIFO(1);
   FIFO#(Tuple3#(Bit#(8),Bit#(serverIdxSz),Bool))workf <- mkSizedFIFO(32); // isthis the right size?

   Vector#(numServers, FIFO#(void))              outfs <- replicateM(mkSizedFIFO(1));
   Vector#(numServers, FIFOF#(Tuple2#(Bit#(serverIdxSz), MemengineCmd))) cmds_in <- replicateM(mkSizedFIFOF(1));
   PipeOut#(Tuple2#(Bit#(serverIdxSz), MemengineCmd)) in_funnel <- mkFunnel1PipesPipelined(map(toPipeOut,cmds_in));
   FIFOF#(Tuple2#(Bit#(TLog#(numServers)), Tuple2#(Bit#(dataWidth),Bool))) read_data <- mkFIFOF;
   Vector#(numServers, PipeOut#(Tuple2#(Bit#(dataWidth),Bool))) out_unfunnel <- mkUnFunnel1PipesPipelined(toPipeOut(read_data));
   function PipeOut#(Bit#(dataWidth)) check_out(PipeOut#(Tuple2#(Bit#(dataWidth),Bool)) x, Integer i) =
      (interface PipeOut;
	  method Bit#(dataWidth) first;
	     return tpl_1(x.first);
	  endmethod
	  method Action deq;
	     x.deq;
	     if (tpl_2(x.first))
		outfs[i].enq(?);
	  endmethod
	  method Bool notEmpty = x.notEmpty;
       endinterface);
   Vector#(numServers, PipeOut#(Bit#(dataWidth))) out_data_pipes = zipWith(check_out, out_unfunnel, genVector);

   Reg#(Bit#(8))                               respCnt <- mkReg(0);
   Reg#(Bit#(serverIdxSz))                     loadIdx <- mkReg(0);
   let beat_shift = fromInteger(valueOf(beatShift));
   let cmd_q_depth = fromInteger(valueOf(cmdQDepth));

   rule store_cmd;
      match {.idx, .cmd} <- toGet(in_funnel).get;
      let new_tail = tail[idx]+1;
      if (new_tail >= extend(idx+1)*cmd_q_depth)
	 new_tail = extend(idx)*cmd_q_depth;
      tail[idx] <= new_tail;
      outs1[idx] <= outs1[idx]+1;
      cmdBuf.portA.request.put(BRAMRequest{write:True, responseOnWrite:False, address:tail[idx], datain:cmd});
      $display("store_cmd: %d %h", idx, tail[idx]);
   endrule

   rule load_ctxt;
      loadIdx <= loadIdx+1;
      if (outs1[loadIdx] > 0) begin
	 cmdBuf.portA.request.put(BRAMRequest{write:False, responseOnWrite:False, address:head[loadIdx], datain:?});
	 loadf.enq(loadIdx);
	 //$display("load_ctxt %d, %h", loadIdx, head[loadIdx]);
      end
   endrule

   Vector#(numServers, Server#(MemengineCmd,Bool)) rs;
   for(Integer i = 0; i < valueOf(numServers); i=i+1)
      rs[i] =  (interface Server#(MemengineCmd,Bool);
		   interface Put request;
		      method Action put(MemengineCmd c) if (outs0[i] < cmd_q_depth);
			 outs0[i] <= outs0[i]+1;
			 cmds_in[i].enq(tuple2(fromInteger(i),c));
 		      endmethod
		   endinterface
		   interface Get response;
		      method ActionValue#(Bool) get;
			 outfs[i].deq;
	 		 outs0[i] <= outs0[i]-1;
			 return True;
		      endmethod
		   endinterface
		endinterface);
   interface readServers = rs;
   interface ObjectReadClient dmaClient;
      interface Get readReq;
	 method ActionValue#(ObjectRequest) get();
	    let cmd <- cmdBuf.portA.response.get;
	    let idx <- toGet(loadf).get;
	    Bit#(8) bl = cmd.burstLen;
	    let last = False;
	    if (cmd.readLen <= extend(bl)) begin
	       last = True;
	       bl = truncate(cmd.readLen);
	       outs1[idx] <= outs1[idx]-1;
	       let new_head = head[idx]+1;
	       if (new_head >= extend(idx+1)*cmd_q_depth)
		  new_head = extend(idx)*cmd_q_depth;
	       head[idx] <= new_head;
	       //$display("new_head %d %d", idx, new_head);
	    end
	    let new_cmd = MemengineCmd{pointer:cmd.pointer, base:cmd.base+extend(bl), burstLen:cmd.burstLen, readLen:cmd.readLen-extend(bl)};
	    cmdBuf.portA.request.put(BRAMRequest{write:True, responseOnWrite:False, address:head[idx], datain:new_cmd});
	    workf.enq(tuple3(truncate(bl>>beat_shift), idx, last));
	    //$display("readReq %d, %h %h %h", idx, cmd.base, bl, last);
	    return ObjectRequest { pointer: cmd.pointer, offset: cmd.base, burstLen:bl, tag: 0 };
	 endmethod
      endinterface
      interface Put readData;
	 method Action put(ObjectData#(dataWidth) d);
	    match {.rc, .idx, .last} = workf.first;
	    let new_respCnt = respCnt+1;
	    let l = False;
	    //$display("%h %d", d.data, idx);
	    if (new_respCnt == rc) begin
	       respCnt <= 0;
	       workf.deq;
	       //$display("eob %d", idx);
	       l = last;
	    end
	    else begin
	       respCnt <= new_respCnt;
	    end
	    read_data.enq(tuple2(idx,tuple2(d.data,l)));
	 endmethod
      endinterface
   endinterface
   interface dataPipes = out_data_pipes;
endmodule


fixed $display in MemWrite.startWrite

fixed $display in MemWrite.startWrite

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
`include "ConnectalProjectConfig.bsv"
import FIFO::*;
import FIFOF::*;
import Vector::*;
import ClientServer::*;
import GetPut::*;
import ConnectalMemTypes::*;
import MemWriteEngine::*;
import Pipe::*;
import Arith::*;
import ConnectalConfig::*;

`ifdef NumEngineServers
typedef `NumEngineServers NumEngineServers;
`else
typedef 1 NumEngineServers;
`endif

typedef TDiv#(DataBusWidth,32) DataBusWords;

interface MemwriteRequest;
   method Action startWrite(Bit#(32) pointer, Bit#(32) offset, Bit#(32) numWords, Bit#(32) burstLen, Bit#(32) iterCnt);
   method Action getStateDbg();
endinterface

interface Memwrite;
   interface MemwriteRequest request;
   interface Vector#(1,MemWriteClient#(DataBusWidth)) dmaClient;
endinterface

interface MemwriteIndication;
   method Action started(Bit#(32) numWords);
   method Action reportStateDbg(Bit#(32) wrCnt, Bit#(32) srcGen);
   method Action writeDone(Bit#(32) v);
endinterface

module  mkMemwrite#(MemwriteIndication indication) (Memwrite);

   Reg#(SGLId)   pointer <- mkReg(0);
   Reg#(Bit#(32))       numWords <- mkReg(0);
   Reg#(Bit#(32))       burstLen <- mkReg(0);
   FIFOF#(void)               cf <- mkSizedFIFOF(1);

   Vector#(NumEngineServers, Reg#(Bit#(32)))         srcGens <- replicateM(mkReg(0));
   Reg#(Bit#(32))                                writeOffset <- mkReg(0);
   Reg#(Bit#(32))                                    iterCnt <- mkReg(0);
   Vector#(NumEngineServers, Reg#(Bit#(32)))        iterCnts <- replicateM(mkReg(0));
   Vector#(NumEngineServers, FIFOF#(void))               cfs <- replicateM(mkSizedFIFOF(1));
   Vector#(NumEngineServers, FIFOF#(Bool))       finishFifos <- replicateM(mkFIFOF);
   MemWriteEngine#(DataBusWidth,DataBusWidth,2,NumEngineServers)                we <- mkMemWriteEngine;
   Bit#(32) chunk = extend(numWords)*4;

   for(Integer i = 0; i < valueOf(NumEngineServers); i=i+1) begin
      rule start (iterCnts[i] > 0);
	 we.writeServers[i].request.put(MemengineCmd{tag:0, sglId:pointer, base:extend(writeOffset)+(fromInteger(i)*chunk), len:truncate(chunk), burstLen:truncate(burstLen*4)});
	 Bit#(32) srcGen = (fromInteger(i) << 28);
	 srcGens[i] <= srcGen;
	 $display("start %d/%d, %h 0x%x %h chunk=%h", i, valueOf(NumEngineServers), srcGen, iterCnts[i], writeOffset, chunk);
	 cfs[i].enq(?);
	 iterCnts[i] <= iterCnts[i]-1;
      endrule
`ifdef MEMENGINE_REQUEST_CYCLES
      rule requestCycles;
	 let reqcycles <- toGet(we.writeServers[i].requestCycles).get();
	 $display("request %d took %d cycles", reqcycles.tag, reqcycles.cycles);
      endrule
`endif
      rule finish;
	 $display("finish %d 0x%x", i, iterCnts[i]);
	 let rv <- we.writeServers[i].done.get;
	 finishFifos[i].enq(rv);
      endrule
      rule src if (cfs[i].notEmpty);
	 Vector#(DataBusWords, Bit#(32)) v;
	 for (Integer j = 0; j < valueOf(DataBusWords); j = j + 1)
	    v[j] = srcGens[i]+fromInteger(j);
	 we.writeServers[i].data.enq(pack(v));
	 let new_srcGen = srcGens[i]+fromInteger(valueOf(DataBusWords));
	 srcGens[i] <= new_srcGen;
	 if (new_srcGen[27:0] >= truncate(numWords))
	    cfs[i].deq;
      endrule
   end

   PipeOut#(Vector#(NumEngineServers, Bool)) finishPipe <- mkJoinVector(id, map(toPipeOut, finishFifos));
   PipeOut#(Bool) finishReducePipe <- mkReducePipe(uncurry(booland), finishPipe);

   rule indicate_finish;
      let rv <- toGet(finishReducePipe).get();
      $display("indicate_finish rv=%d iterCnt=%d", rv, iterCnt);
      if (iterCnt == 1) begin
	 cf.deq;
	 indication.writeDone(0);
      end
      iterCnt <= iterCnt - 1;
   endrule

   interface MemWriteClient dmaClient = cons(we.dmaClient, nil);
   interface MemwriteRequest request;
       method Action startWrite(Bit#(32) wp, Bit#(32) off, Bit#(32) nw, Bit#(32) bl, Bit#(32) ic);
	  $display("startWrite pointer=%d offset=%d numWords=%h burstLen=%d iterCnt=%d", wp, off, nw, bl, ic);
	  indication.started(nw);
	  pointer <= wp;
	  cf.enq(?);
	  numWords  <= nw;
	  burstLen  <= bl;
	  iterCnt <= ic;
	  writeOffset <= off*4;
	  for(Integer i = 0; i < valueOf(NumEngineServers); i=i+1)
	     iterCnts[i] <= ic;
       endmethod
   endinterface
endmodule

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
`include "ConnectalProjectConfig.bsv"
import FIFO::*;
import FIFOF::*;
import Vector::*;
import ClientServer::*;
import GetPut::*;
import ConnectalMemTypes::*;
import MemWriteEngine::*;
import Pipe::*;
import Arith::*;
import ConnectalConfig::*;

`ifdef NumEngineServers
typedef `NumEngineServers NumEngineServers;
`else
typedef 1 NumEngineServers;
`endif

typedef TDiv#(DataBusWidth,32) DataBusWords;

interface MemwriteRequest;
   method Action startWrite(Bit#(32) pointer, Bit#(32) offset, Bit#(32) numWords, Bit#(32) burstLen, Bit#(32) iterCnt);
   method Action getStateDbg();
endinterface

interface Memwrite;
   interface MemwriteRequest request;
   interface Vector#(1,MemWriteClient#(DataBusWidth)) dmaClient;
endinterface

interface MemwriteIndication;
   method Action started(Bit#(32) numWords);
   method Action reportStateDbg(Bit#(32) wrCnt, Bit#(32) srcGen);
   method Action writeDone(Bit#(32) v);
endinterface

module  mkMemwrite#(MemwriteIndication indication) (Memwrite);

   Reg#(SGLId)   pointer <- mkReg(0);
   Reg#(Bit#(32))       numWords <- mkReg(0);
   Reg#(Bit#(32))       burstLen <- mkReg(0);
   FIFOF#(void)               cf <- mkSizedFIFOF(1);

   Vector#(NumEngineServers, Reg#(Bit#(32)))         srcGens <- replicateM(mkReg(0));
   Reg#(Bit#(32))                                writeOffset <- mkReg(0);
   Reg#(Bit#(32))                                    iterCnt <- mkReg(0);
   Vector#(NumEngineServers, Reg#(Bit#(32)))        iterCnts <- replicateM(mkReg(0));
   Vector#(NumEngineServers, FIFOF#(void))               cfs <- replicateM(mkSizedFIFOF(1));
   Vector#(NumEngineServers, FIFOF#(Bool))       finishFifos <- replicateM(mkFIFOF);
   MemWriteEngine#(DataBusWidth,DataBusWidth,2,NumEngineServers)                we <- mkMemWriteEngine;
   Bit#(32) chunk = extend(numWords)*4;

   for(Integer i = 0; i < valueOf(NumEngineServers); i=i+1) begin
      rule start (iterCnts[i] > 0);
	 we.writeServers[i].request.put(MemengineCmd{tag:0, sglId:pointer, base:extend(writeOffset)+(fromInteger(i)*chunk), len:truncate(chunk), burstLen:truncate(burstLen*4)});
	 Bit#(32) srcGen = (fromInteger(i) << 28);
	 srcGens[i] <= srcGen;
	 $display("start %d/%d, %h 0x%x %h chunk=%h", i, valueOf(NumEngineServers), srcGen, iterCnts[i], writeOffset, chunk);
	 cfs[i].enq(?);
	 iterCnts[i] <= iterCnts[i]-1;
      endrule
`ifdef MEMENGINE_REQUEST_CYCLES
      rule requestCycles;
	 let reqcycles <- toGet(we.writeServers[i].requestCycles).get();
	 $display("request %d took %d cycles", reqcycles.tag, reqcycles.cycles);
      endrule
`endif
      rule finish;
	 $display("finish %d 0x%x", i, iterCnts[i]);
	 let rv <- we.writeServers[i].done.get;
	 finishFifos[i].enq(rv);
      endrule
      rule src if (cfs[i].notEmpty);
	 Vector#(DataBusWords, Bit#(32)) v;
	 for (Integer j = 0; j < valueOf(DataBusWords); j = j + 1)
	    v[j] = srcGens[i]+fromInteger(j);
	 we.writeServers[i].data.enq(pack(v));
	 let new_srcGen = srcGens[i]+fromInteger(valueOf(DataBusWords));
	 srcGens[i] <= new_srcGen;
	 if (new_srcGen[27:0] >= truncate(numWords))
	    cfs[i].deq;
      endrule
   end

   PipeOut#(Vector#(NumEngineServers, Bool)) finishPipe <- mkJoinVector(id, map(toPipeOut, finishFifos));
   PipeOut#(Bool) finishReducePipe <- mkReducePipe(uncurry(booland), finishPipe);

   rule indicate_finish;
      let rv <- toGet(finishReducePipe).get();
      $display("indicate_finish rv=%d iterCnt=%d", rv, iterCnt);
      if (iterCnt == 1) begin
	 cf.deq;
	 indication.writeDone(0);
      end
      iterCnt <= iterCnt - 1;
   endrule

   interface MemWriteClient dmaClient = cons(we.dmaClient, nil);
   interface MemwriteRequest request;
       method Action startWrite(Bit#(32) wp, Bit#(32) off, Bit#(32) nw, Bit#(32) bl, Bit#(32) ic);
	  $display("startWrite pointer=%d offset=%d numWords=%h burstLen=%d iterCnt=%d", pointer, off, nw, bl, ic);
	  indication.started(nw);
	  pointer <= wp;
	  cf.enq(?);
	  numWords  <= nw;
	  burstLen  <= bl;
	  iterCnt <= ic;
	  writeOffset <= off*4;
	  for(Integer i = 0; i < valueOf(NumEngineServers); i=i+1)
	     iterCnts[i] <= ic;
       endmethod
   endinterface
endmodule