fixed declarations to get rid of compilation warnings

// Copyright (c) 2012 MIT
// Copyright (c) 2012 Nokia, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import GetPut         ::*;
import FIFOF          ::*;
import SpecialFIFOs   ::*;
import StmtFSM        ::*;
import Assert         ::*;
import Pipe           ::*;

function Bit#(a) rtruncate(Bit#(b) x) provisos(Add#(k,a,b));
   match {.v,.*} = split(x);
   return v;
endfunction

interface ToBit#(numeric type n, type a);
   method Action enq(a v);
   method Bit#(n) first;
   method Action deq();
   method Bool notEmpty();
   method Bool notFull();
endinterface

interface FromBit#(numeric type n, type a);
   method Action enq(Bit#(n) v);
   method a first();
   method Action deq();
   method Bool notEmpty();
   method Bool notFull();
endinterface

instance ToGet#(FromBit#(n,a),a);
   function Get#(a) toGet(FromBit#(n,a) x);
      return (interface Get
		 method ActionValue #(a) get();
		    actionvalue
		       x.deq;
		       return x.first;
		    endactionvalue
		 endmethod
	      endinterface);
   endfunction
endinstance

instance ToPut#(ToBit#(n,a),a);
   function Put#(a) toPut(ToBit#(n,a) x);
      return (interface Put
		 method Action put(a v);
		    action
		       x.enq(v);
		    endaction
		 endmethod
	      endinterface);
   endfunction
endinstance

instance ToPipeIn#(Bit#(n), FromBit#(n,a));
   function PipeIn#(Bit#(n)) toPipeIn(FromBit#(n,a) frombit);
      return (interface PipeIn#(Bit#(n));
		 method enq = frombit.enq;
		 method notFull = frombit.notFull;
	      endinterface);
   endfunction
endinstance
instance ToPipeOut#(a, FromBit#(n,a));
   function PipeOut#(a) toPipeOut(FromBit#(n,a) frombit);
      return (interface PipeOut#(a);
		 method first = frombit.first;
		 method deq = frombit.deq;
		 method notEmpty = frombit.notEmpty;
	      endinterface);
   endfunction
endinstance

instance ToPipeIn#(a, ToBit#(n,a));
   function PipeIn#(a) toPipeIn(ToBit#(n,a) tobit);
      return (interface PipeIn#(Bit#(n));
		 method enq = tobit.enq;
		 method notFull = tobit.notFull;
	      endinterface);
   endfunction
endinstance
instance ToPipeOut#(Bit#(n), ToBit#(n,a));
   function PipeOut#(Bit#(n)) toPipeOut(ToBit#(n,a) tobit);
      return (interface PipeOut#(Bit#(n));
		 method first = tobit.first;
		 method deq = tobit.deq;
		 method notEmpty = tobit.notEmpty;
	      endinterface);
   endfunction
endinstance

module mkToBit(ToBit#(n,a))
   provisos(Bits#(a,asz),
            Add#(1, z, asz),
            Div#(asz,n,nwords),
            Mul#(n,nwords,aszn),
            Add#(n,a__,aszn),
            Add#(asz,paddingsz,aszn));

   Bit#(32) max  = fromInteger(valueOf(nwords)) - 1;
   Bit#(paddingsz) padding = 0;

   FIFOF#(Bit#(aszn)) fifo <- mkFIFOF1();
   Reg#(Bit#(32))   count <- mkReg(0);

   let nv = valueOf(n);

   method Action enq(a val);
      fifo.enq({padding,pack(val)});
   endmethod

   method Bit#(n) first() if (fifo.notEmpty);
      return rtruncate(fifo.first << (count*fromInteger(nv)));
   endmethod

   method Action deq() if (fifo.notEmpty);
      if (count == max)
         begin
            count <= 0;
            fifo.deq;
         end
      else
         begin
            count <= count + 1;
         end
   endmethod
   method Bool notEmpty = fifo.notEmpty;
   method Bool notFull = fifo.notFull;
endmodule

module mkFromBit(FromBit#(n,a))
   provisos(Bits#(a,asz),
            Add#(1,z,asz),
            Div#(asz,n,nwords),
            Mul#(n,nwords,aszn),
            Add#(n,a__,aszn),
            Add#(asz,paddingsz,aszn));

   Bit#(32) max    = fromInteger(valueOf(nwords))-1;

   Reg#(Bit#(aszn)) fbnbuff <- mkReg(0);
   Reg#(Bit#(32))    count <- mkReg(0);
   FIFOF#(Bit#(asz)) fifo <- mkFIFOF1;

   method Action enq(Bit#(n) x) if (count < max || fifo.notFull);
      Bit#(aszn) newbuff = truncate({fbnbuff,x});
      fbnbuff <= newbuff;
      if (count == max)
         begin
            count <= 0;
            fifo.enq(truncate(newbuff));
         end
      else
         begin
            count <= count+1;
         end
   endmethod

   method a first;
       return unpack(fifo.first);
   endmethod

   method Action deq;
       fifo.deq;
   endmethod

   method Bool notEmpty() = fifo.notEmpty;
   method Bool notFull() = fifo.notFull;
endmodule

module mkAdapterTb(Empty);
   ToBit#(32,Bit#(72)) tb32_72 <- mkToBit();
   ToBit#(32,Bit#(17)) tb32_17 <- mkToBit();

   FromBit#(32,Bit#(72)) fb32_72 <- mkFromBit();
   FromBit#(32,Bit#(17)) fb32_17 <- mkFromBit();

   Reg#(Bit#(10)) timer <- mkReg(0);
   rule timeout;
       timer <= timer+1;
       dynamicAssert(timer < 128, "Timeout");
   endrule

   mkAutoFSM(
      seq

       // test to bit-32
       tb32_72.enq(72'h090807060504030201);
       dynamicAssert(tb32_72.notEmpty, "Adapter not empty");
       dynamicAssert(!tb32_72.notFull, "Adapter full");
       $display("tb32_72 notEmpty %d notFull %d", tb32_72.notEmpty, tb32_72.notFull);
       $display("tb32_72 word 0 %h", tb32_72.first());
       dynamicAssert(tb32_72.first == 32'h00000009, "expecting 00000009");
       tb32_72.deq;
       $display("tb32_72 word 1 %h", tb32_72.first());
       dynamicAssert(tb32_72.first == 32'h08070605, "expecting 08070605");
       tb32_72.deq;
       $display("tb32_72 word 2 %h", tb32_72.first());
       dynamicAssert(tb32_72.first == 32'h04030201, "expecting 04030201");
       tb32_72.deq;
       dynamicAssert(!tb32_72.notEmpty && tb32_72.notFull, "Adapter empty and not full");
       $display("tb32_72 notEmpty %d notFull %d", tb32_72.notEmpty, tb32_72.notFull);
       dynamicAssert(!tb32_17.notEmpty, "tb32_17 empty");
       dynamicAssert(tb32_17.notFull, "tb32_17 !full");
       tb32_17.enq(17'h10203);
       dynamicAssert(tb32_17.notEmpty, "tb32_17 not empty");
       dynamicAssert(!tb32_17.notFull, "tb32_17 full");
       $display("tb32_17.first %h", tb32_17.first);
       dynamicAssert(tb32_17.first == (32'h00010203), "Expected 00010203");
       tb32_17.deq;
       dynamicAssert(!tb32_17.notEmpty, "tb32_17 empty");
       dynamicAssert(tb32_17.notFull, "tb32_17 !full");

       //test from bit-32
       dynamicAssert(!fb32_72.notEmpty, "Adapter empty");
       dynamicAssert(fb32_72.notFull, "Adapter not full");
       $display("fb32_72 notEmpty %d notFull %d", fb32_72.notEmpty, fb32_72.notFull);
       fb32_72.enq(32'h00000009);
       fb32_72.enq(32'h08070605);
       fb32_72.enq(32'h04030201);
       $display("fb32_72.first %h", fb32_72.first);
       dynamicAssert(fb32_72.first == 72'h090807060504030201, "Expected 090807060504030201");
       fb32_72.deq;
       fb32_72.enq(32'h09080706);
       dynamicAssert(!fb32_72.notEmpty, "Adapter not empty");
       dynamicAssert(fb32_72.notFull, "Adapter not full");
       fb32_72.enq(32'h05040302);
       fb32_72.enq(32'h01000000);
       dynamicAssert(fb32_72.notEmpty, "Adapter not empty");
       dynamicAssert(!fb32_72.notFull, "Adapter full");
       $display("fb32_72.first %h", fb32_72.first);
       dynamicAssert(fb32_72.first == 72'h060504030201000000, "Expected 060504030201000000");
       fb32_72.deq;
       $display("fb32_72 notEmpty %d notFull %d", fb32_72.notEmpty, fb32_72.notFull);
       dynamicAssert(!fb32_72.notEmpty, "Adapter empty");
       dynamicAssert(fb32_72.notFull, "Adapter not full");
       fb32_17.enq(32'h10203);
       dynamicAssert(fb32_17.notEmpty, "Adapter not empty");
       dynamicAssert(!fb32_17.notFull, "Adapter full");
       $display("fb32_17.first %h", fb32_17.first);
       dynamicAssert(fb32_17.first == 17'h10203, "Expected 10203");
       fb32_17.deq;
       dynamicAssert(!fb32_17.notEmpty, "Adapter empty");
       dynamicAssert(fb32_17.notFull, "Adapter not full");

       // they should be duals
       tb32_72.enq(72'h090807060504030201);
       fb32_72.enq(tb32_72.first);
       tb32_72.deq;
       fb32_72.enq(tb32_72.first);
       tb32_72.deq;
       fb32_72.enq(tb32_72.first);
       tb32_72.deq;
       dynamicAssert(fb32_72.first == 72'h090807060504030201, "Expected 090807060504030201");
       fb32_72.deq;

       tb32_17.enq(17'h10203);
       fb32_17.enq(tb32_17.first);
       tb32_17.deq;
       dynamicAssert(fb32_17.first == 17'h10203, "Expected 10203");
       fb32_17.deq;

       $finish(0);
   endseq
   );

endmodule

// Copyright (c) 2012 MIT
// Copyright (c) 2012 Nokia, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import GetPut         ::*;
import FIFOF          ::*;
import SpecialFIFOs   ::*;
import StmtFSM        ::*;
import Assert         ::*;
import Pipe           ::*;

function Bit#(a) rtruncate(Bit#(b) x) provisos(Add#(k,a,b));
   match {.v,.*} = split(x);
   return v;
endfunction

interface ToBit#(numeric type n, type a);
   method Action enq(a v);
   method Bit#(n) first;
   method Action deq();
   method Bool notEmpty();
   method Bool notFull();
endinterface

interface FromBit#(numeric type n, type a);
   method Action enq(Bit#(n) v);
   method a first();
   method Action deq();
   method Bool notEmpty();
   method Bool notFull();
endinterface

instance ToGet#(FromBit#(n,a),a);
   function Get#(a) toGet(FromBit#(n,a) x);
      return (interface Get
		 method get();
		    actionvalue
		       x.deq;
		       return x.first;
		    endactionvalue
		 endmethod
	      endinterface);
   endfunction
endinstance

instance ToPut#(ToBit#(n,a),a);
   function Put#(a) toPut(ToBit#(n,a) x);
      return (interface Put
		 method put(a v);
		    action
		       x.enq(v);
		    endaction
		 endmethod
	      endinterface);
   endfunction
endinstance

instance ToPipeIn#(Bit#(n), FromBit#(n,a));
   function PipeIn#(Bit#(n)) toPipeIn(FromBit#(n,a) frombit);
      return (interface PipeIn#(Bit#(n));
		 method enq = frombit.enq;
		 method notFull = frombit.notFull;
	      endinterface);
   endfunction
endinstance
instance ToPipeOut#(a, FromBit#(n,a));
   function PipeOut#(a) toPipeOut(FromBit#(n,a) frombit);
      return (interface PipeOut#(a);
		 method first = frombit.first;
		 method deq = frombit.deq;
		 method notEmpty = frombit.notEmpty;
	      endinterface);
   endfunction
endinstance

instance ToPipeIn#(a, ToBit#(n,a));
   function PipeIn#(a) toPipeIn(ToBit#(n,a) tobit);
      return (interface PipeIn#(Bit#(n));
		 method enq = tobit.enq;
		 method notFull = tobit.notFull;
	      endinterface);
   endfunction
endinstance
instance ToPipeOut#(Bit#(n), ToBit#(n,a));
   function PipeOut#(Bit#(n)) toPipeOut(ToBit#(n,a) tobit);
      return (interface PipeOut#(Bit#(n));
		 method first = tobit.first;
		 method deq = tobit.deq;
		 method notEmpty = tobit.notEmpty;
	      endinterface);
   endfunction
endinstance

module mkToBit(ToBit#(n,a))
   provisos(Bits#(a,asz),
            Add#(1, z, asz),
            Div#(asz,n,nwords),
            Mul#(n,nwords,aszn),
            Add#(n,a__,aszn),
            Add#(asz,paddingsz,aszn));

   Bit#(32) max  = fromInteger(valueOf(nwords)) - 1;
   Bit#(paddingsz) padding = 0;

   FIFOF#(Bit#(aszn)) fifo <- mkFIFOF1();
   Reg#(Bit#(32))   count <- mkReg(0);

   let nv = valueOf(n);

   method Action enq(a val);
      fifo.enq({padding,pack(val)});
   endmethod

   method Bit#(n) first() if (fifo.notEmpty);
      return rtruncate(fifo.first << (count*fromInteger(nv)));
   endmethod

   method Action deq() if (fifo.notEmpty);
      if (count == max)
         begin
            count <= 0;
            fifo.deq;
         end
      else
         begin
            count <= count + 1;
         end
   endmethod
   method Bool notEmpty = fifo.notEmpty;
   method Bool notFull = fifo.notFull;
endmodule

module mkFromBit(FromBit#(n,a))
   provisos(Bits#(a,asz),
            Add#(1,z,asz),
            Div#(asz,n,nwords),
            Mul#(n,nwords,aszn),
            Add#(n,a__,aszn),
            Add#(asz,paddingsz,aszn));

   Bit#(32) max    = fromInteger(valueOf(nwords))-1;

   Reg#(Bit#(aszn)) fbnbuff <- mkReg(0);
   Reg#(Bit#(32))    count <- mkReg(0);
   FIFOF#(Bit#(asz)) fifo <- mkFIFOF1;

   method Action enq(Bit#(n) x) if (count < max || fifo.notFull);
      Bit#(aszn) newbuff = truncate({fbnbuff,x});
      fbnbuff <= newbuff;
      if (count == max)
         begin
            count <= 0;
            fifo.enq(truncate(newbuff));
         end
      else
         begin
            count <= count+1;
         end
   endmethod

   method a first;
       return unpack(fifo.first);
   endmethod

   method Action deq;
       fifo.deq;
   endmethod

   method Bool notEmpty() = fifo.notEmpty;
   method Bool notFull() = fifo.notFull;
endmodule

module mkAdapterTb(Empty);
   ToBit#(32,Bit#(72)) tb32_72 <- mkToBit();
   ToBit#(32,Bit#(17)) tb32_17 <- mkToBit();

   FromBit#(32,Bit#(72)) fb32_72 <- mkFromBit();
   FromBit#(32,Bit#(17)) fb32_17 <- mkFromBit();

   Reg#(Bit#(10)) timer <- mkReg(0);
   rule timeout;
       timer <= timer+1;
       dynamicAssert(timer < 128, "Timeout");
   endrule

   mkAutoFSM(
      seq

       // test to bit-32
       tb32_72.enq(72'h090807060504030201);
       dynamicAssert(tb32_72.notEmpty, "Adapter not empty");
       dynamicAssert(!tb32_72.notFull, "Adapter full");
       $display("tb32_72 notEmpty %d notFull %d", tb32_72.notEmpty, tb32_72.notFull);
       $display("tb32_72 word 0 %h", tb32_72.first());
       dynamicAssert(tb32_72.first == 32'h00000009, "expecting 00000009");
       tb32_72.deq;
       $display("tb32_72 word 1 %h", tb32_72.first());
       dynamicAssert(tb32_72.first == 32'h08070605, "expecting 08070605");
       tb32_72.deq;
       $display("tb32_72 word 2 %h", tb32_72.first());
       dynamicAssert(tb32_72.first == 32'h04030201, "expecting 04030201");
       tb32_72.deq;
       dynamicAssert(!tb32_72.notEmpty && tb32_72.notFull, "Adapter empty and not full");
       $display("tb32_72 notEmpty %d notFull %d", tb32_72.notEmpty, tb32_72.notFull);
       dynamicAssert(!tb32_17.notEmpty, "tb32_17 empty");
       dynamicAssert(tb32_17.notFull, "tb32_17 !full");
       tb32_17.enq(17'h10203);
       dynamicAssert(tb32_17.notEmpty, "tb32_17 not empty");
       dynamicAssert(!tb32_17.notFull, "tb32_17 full");
       $display("tb32_17.first %h", tb32_17.first);
       dynamicAssert(tb32_17.first == (32'h00010203), "Expected 00010203");
       tb32_17.deq;
       dynamicAssert(!tb32_17.notEmpty, "tb32_17 empty");
       dynamicAssert(tb32_17.notFull, "tb32_17 !full");

       //test from bit-32
       dynamicAssert(!fb32_72.notEmpty, "Adapter empty");
       dynamicAssert(fb32_72.notFull, "Adapter not full");
       $display("fb32_72 notEmpty %d notFull %d", fb32_72.notEmpty, fb32_72.notFull);
       fb32_72.enq(32'h00000009);
       fb32_72.enq(32'h08070605);
       fb32_72.enq(32'h04030201);
       $display("fb32_72.first %h", fb32_72.first);
       dynamicAssert(fb32_72.first == 72'h090807060504030201, "Expected 090807060504030201");
       fb32_72.deq;
       fb32_72.enq(32'h09080706);
       dynamicAssert(!fb32_72.notEmpty, "Adapter not empty");
       dynamicAssert(fb32_72.notFull, "Adapter not full");
       fb32_72.enq(32'h05040302);
       fb32_72.enq(32'h01000000);
       dynamicAssert(fb32_72.notEmpty, "Adapter not empty");
       dynamicAssert(!fb32_72.notFull, "Adapter full");
       $display("fb32_72.first %h", fb32_72.first);
       dynamicAssert(fb32_72.first == 72'h060504030201000000, "Expected 060504030201000000");
       fb32_72.deq;
       $display("fb32_72 notEmpty %d notFull %d", fb32_72.notEmpty, fb32_72.notFull);
       dynamicAssert(!fb32_72.notEmpty, "Adapter empty");
       dynamicAssert(fb32_72.notFull, "Adapter not full");
       fb32_17.enq(32'h10203);
       dynamicAssert(fb32_17.notEmpty, "Adapter not empty");
       dynamicAssert(!fb32_17.notFull, "Adapter full");
       $display("fb32_17.first %h", fb32_17.first);
       dynamicAssert(fb32_17.first == 17'h10203, "Expected 10203");
       fb32_17.deq;
       dynamicAssert(!fb32_17.notEmpty, "Adapter empty");
       dynamicAssert(fb32_17.notFull, "Adapter not full");

       // they should be duals
       tb32_72.enq(72'h090807060504030201);
       fb32_72.enq(tb32_72.first);
       tb32_72.deq;
       fb32_72.enq(tb32_72.first);
       tb32_72.deq;
       fb32_72.enq(tb32_72.first);
       tb32_72.deq;
       dynamicAssert(fb32_72.first == 72'h090807060504030201, "Expected 090807060504030201");
       fb32_72.deq;

       tb32_17.enq(17'h10203);
       fb32_17.enq(tb32_17.first);
       tb32_17.deq;
       dynamicAssert(fb32_17.first == 17'h10203, "Expected 10203");
       fb32_17.deq;

       $finish(0);
   endseq
   );

endmodule


Fixed performance issue in CPU_Fetch_C.bsv for RVC: removed pipeline bubbles on 32-bit instrs that are 16b- but not 32b-aligned

// Copyright (c) 2016-2019 Bluespec, Inc. All Rights Reserved

package CPU_Fetch_C;

// ================================================================
// mkCPU_Fetch_C is an IMem_IFC->Module#(IMem_IFC) wrapper.

// The input parameter interface is for 32-bit-aligned 32-bit instruction-fetches.

// This module completely encapsulates the logic needed to extend it to
// 16-bit "compressed" instructions ('C' extension)
// which then also includes 32-bit instructions which may be only 16-bit-aligned,
// since 32b and 16b instructions can be freely mixed.

// ================================================================
// Exports

export mkCPU_Fetch_C;

// ================================================================
// BSV lib imports

import GetPut       :: *;
import ClientServer :: *;

// ----------------
// BSV additional libs

import Cur_Cycle :: *;

// ================================================================
// Project imports

import ISA_Decls    :: *;
import Near_Mem_IFC :: *;

// ================================================================
// Functions concerning address-alignment of instructions.
// 'Even': aligned to an even 16b boundary (so, also aligned to 32b boundary)
// 'Odd':  aligned to an odd  16b boundary (so, not  aligned to 32b boundary)

function Bool is_addr_even16 (WordXL pc);
   return (pc [1:0] == 2'b00);
endfunction

function Bool is_addr_odd16 (WordXL pc);
   return (pc [1:0] != 2'b00);
endfunction

function WordXL fn_to_b32_addr (WordXL addr);
   return ((addr >> 2) << 2);
endfunction

// Equality of 32-bit aligned addresses containing a1 and a2 (i.e., ignoring [1:0])
function Bool eq_b32_addr (WordXL a1, WordXL a2) = ((a1 >> 2) == (a2 >> 2));

// ================================================================
// Functions concerning whether an instr is 32b (RV) or 16b (RVC)

function Bool is_32b_instr (Bit #(n) instr);
   return (instr [1:0] == 2'b11);
endfunction

function Bool is_16b_instr (Bit #(n) instr);
   return (instr [1:0] != 2'b11);
endfunction

// ================================================================
// Wrapper: wraps 32-bit aligned IMem to allow 16-bit instrs and 32-bit instrs,
// aligned to 16b or 32b boundaries.

module mkCPU_Fetch_C #(IMem_IFC  imem32) (IMem_IFC);

   // imem32 is the underlying 32-bit-aligned memory.
   // imem32.req is only given 32-bit aligned addrs,
   // and returns the 32b at that addr.
   // imem32.pc (output pc) is always the last requested address, always 32b-aligned.
   // ( Assert: imem32.is_32b_not_16b == True )

   Integer verbosity = 0;

   // This register holds the actual PC value (which can be +/-2 from
   // the address given to the ICache.
   Reg #(WordXL)  rg_pc  <- mkRegU;

   // These registers caches the last output of imem32 (imem32.pc, and imem32.instr [31:16])
   Reg #(WordXL)    rg_cache_addr <- mkReg ('1);
   Reg #(Bit #(16)) rg_cache_b16  <- mkRegU;

   // The following hold args of the 'req' method.
   Reg #(Bit #(3))  rg_f3          <- mkRegU;
   Reg #(Priv_Mode) rg_priv        <- mkRegU;
   Reg #(Bit #(1))  rg_sstatus_SUM <- mkRegU;
   Reg #(Bit #(1))  rg_mstatus_MXR <- mkRegU;
   Reg #(WordXL)    rg_satp        <- mkRegU;

   // Holds a faulting address
   Reg #(WordXL)    rg_tval        <- mkRegU;

   // ----------------------------------------------------------------
   // Conditions for selecting 16b and 32b instruction

   // Condition: 32b instr from {imem [15:0], rg_cache_b16}
   Bool cond_i32_odd = (   is_addr_odd16 (rg_pc)
			&& eq_b32_addr (rg_pc, rg_cache_addr)
			&& (imem32.pc == rg_pc + 2)
			&& is_32b_instr (rg_cache_b16));

   // Condition: 32b instr from imem [31:0]
   Bool cond_i32_even = (   is_addr_even16 (rg_pc)
			 && eq_b32_addr (rg_pc, imem32.pc)
			 && is_32b_instr (imem32.instr));

   // Condition: 16b instr from imem [31:16]
   Bool cond_i16_odd  = (   is_addr_odd16 (rg_pc)
			 && (   (   eq_b32_addr (rg_pc, imem32.pc)
				 && is_16b_instr (imem32.instr [31:16]))
			     || (   eq_b32_addr (rg_pc, rg_cache_addr)
				 && is_16b_instr (rg_cache_b16))));

   // Condition: 16b instr from imem [15:0]
   Bool cond_i16_even = (   is_addr_even16 (rg_pc)
			 && eq_b32_addr (rg_pc, imem32.pc)
			 && is_16b_instr (imem32.instr));

   // Condition: 32b instr from { ...next imem [15:0], imem [31:16] }
   Bool cond_i32_odd_fetch_next = (   is_addr_odd16 (rg_pc)
				   && imem32.valid
				   && eq_b32_addr (rg_pc, imem32.pc)
				   && is_32b_instr (imem32.instr [31:16]));

   // ----------------
   // Compose the 32b output 'instr' (either a 32b instr, or { 16'b0, 16b instr })

   function Bit #(32) fn_instr_out ();
      Bit #(32) instr_out = imem32.instr;
      if (cond_i32_odd)
	 instr_out = { imem32.instr [15:0], rg_cache_b16 };

      else if (cond_i16_even)
	 instr_out = { 16'b0, imem32.instr [15:0] };

      else if (cond_i16_odd) begin
	 if (eq_b32_addr (rg_pc, imem32.pc))
	    instr_out = { 16'b0, imem32.instr [31:16] };
	 else
	    instr_out = { 16'b0, rg_cache_b16 };
      end
      return instr_out;
   endfunction

   // ================================================================
   // Assert: The underlying imem32 only knows about 32-bit aligned words

   rule rl_assert_fail (! imem32.is_i32_not_i16);
      $display ("%0d: ERROR: CPU_Fetch_C: imem32.is_i32_not_i16 is False", cur_cycle);
      $finish (1);
   endrule

   rule rl_debug_conds (False && (verbosity != 0) && imem32.valid);
      $display ("%0d: imem32.pc 0x%0h  pc 0x%0h  rg_f3 %0d  imem32.instr 0x%08h  rg_cache_b16 0x%04h",
		cur_cycle, imem32.pc,  rg_pc,  rg_f3,  imem32.instr,  rg_cache_b16);
      $display ("    cond_i32_odd  = ", fshow (cond_i32_odd));
      $display ("    cond_i32_even = ", fshow (cond_i32_even));
      $display ("    cond_i16_odd  = ", fshow (cond_i16_odd));
      $display ("    cond_i16_even = ", fshow (cond_i16_even));
      $display ("    cond_i32_odd_fetch_next             = ", fshow (cond_i32_odd_fetch_next));
      $display ("    eq_b32_addr (rg_pc, imem32.pc)      = ", fshow (eq_b32_addr (rg_pc, imem32.pc)));
      $display ("    is_addr_odd16 (rg_pc)               = ", fshow (is_addr_odd16 (rg_pc)));
      $display ("    is_32b_instr (imem32.instr [31:16]) = ", fshow (is_32b_instr (imem32.instr [31:16])));
   endrule

   // ================================================================
   // When imem32.instr [31:15] has lower half of 32b instr, cache it and fetch next 32 bits

   (* no_implicit_conditions, fire_when_enabled *)
   rule rl_fetch_next_32b (imem32.valid && cond_i32_odd_fetch_next);
      Addr next_b32_addr = imem32.pc + 4;
      imem32.req (rg_f3, next_b32_addr, rg_priv, rg_sstatus_SUM, rg_mstatus_MXR, rg_satp);
      rg_cache_addr <= imem32.pc;
      rg_cache_b16  <= imem32.instr [31:16];
      rg_tval       <= next_b32_addr;

      if (verbosity != 0)
	 $display ("%0d: CPU_Fetch_C.rl_fetch_next_32b:  imem32.pc 0x%0h  next_b32_addr 0x%0h",
		   cur_cycle, imem32.pc, next_b32_addr);
   endrule

   // ================================================================
   // INTERFACE

   // CPU side: IMem request
   method Action  req (Bit #(3)   f3,
		       WordXL     addr,
		       // The following  args for VM
		       Priv_Mode  priv,
		       Bit #(1)   sstatus_SUM,
		       Bit #(1)   mstatus_MXR,
		       WordXL     satp               // { VM_Mode, ASID, PPN_for_page_table }
		       ) if (! cond_i32_odd_fetch_next);
      rg_f3          <= f3;
      rg_pc          <= addr;
      rg_priv        <= priv;
      rg_sstatus_SUM <= sstatus_SUM;
      rg_mstatus_MXR <= mstatus_MXR;
      rg_satp        <= satp;
      rg_tval        <= addr;

      // Cache the previous output, if valid
      if (imem32.valid) begin
	 rg_cache_addr     <= imem32.pc;
	 rg_cache_b16 <= imem32.instr [31:16];
      end

      WordXL addr_of_b32 = fn_to_b32_addr (addr);

      // Fetch next 32b word if request is odd-16b aligned, we've already got those 16b, and it's a 32b instr
      // (the 16b we've already got is saved in r16_cache_b16).
      // Note: since we know it's a 32b instr, this next fetch is not speculative, so ok if it page faults.
      if (   is_addr_odd16 (addr)
	  && imem32.valid
	  && (addr_of_b32 == imem32.pc)
	  && is_32b_instr (imem32.instr [31:16]))
	 begin
	    addr_of_b32 = addr_of_b32 + 4;
	 end

      imem32.req (f3, addr_of_b32, priv, sstatus_SUM, mstatus_MXR, satp);
      if (verbosity > 0) begin
	 $display ("CPU_Fetch_C.req: addr 0x%0h, addr_of_b32 0x%0h", addr, addr_of_b32);
      end
   endmethod

   // CPU side: IMem response
   method Bool     valid    = (imem32.valid && (   cond_i32_odd
						|| cond_i32_even
						|| cond_i16_odd
						|| cond_i16_even));

   method Bool     is_i32_not_i16 = (cond_i32_odd || cond_i32_even);

   method WordXL   pc       = rg_pc;
   method Instr    instr    = fn_instr_out ();
   method Bool     exc      = imem32.exc;
   method Exc_Code exc_code = imem32.exc_code;
   method WordXL   tval     = rg_tval;        // Can be different from rg_pc

endmodule

// ================================================================

endpackage

// Copyright (c) 2016-2019 Bluespec, Inc. All Rights Reserved

package CPU_Fetch_C;

// ================================================================
// mkCPU_Fetch_C is an IMem_IFC->Module#(IMem_IFC) wrapper.

// The input parameter interface is for 32-bit-aligned 32-bit instruction-fetches.

// This module completely encapsulates the logic needed to extend it to
// 16-bit "compressed" instructions ('C' extension)
// which then also includes 16-bit-aligned 32-bit instructions.

// ================================================================
// Exports

export mkCPU_Fetch_C;

// ================================================================
// BSV lib imports

import GetPut       :: *;
import ClientServer :: *;

// ----------------
// BSV additional libs

import Cur_Cycle :: *;

// ================================================================
// Project imports

import ISA_Decls    :: *;
import Near_Mem_IFC :: *;

// ================================================================

module mkCPU_Fetch_C #(IMem_IFC  imem32) (IMem_IFC);

   // Assert: imem32.is_32b_not_16b == True

   Integer verbosity = 0;

   // This register holds the actual PC value (which can be +/-2 from
   // the address given to the ICache.
   Reg #(WordXL)  rg_pc  <- mkRegU;

   // This register holds [15:0] of a 32b instr that was in the
   // upper-half of a word from the ICache.
   Reg #(Bit #(16))  rg_instr_15_0  <- mkRegU;

   // The following hold args for the mmu-cache that are needed when
   // doing the second 32-bit fetch of an odd-aligned 32b instr
   Reg #(Bit #(3))  rg_f3          <- mkRegU;
   Reg #(Priv_Mode) rg_priv        <- mkRegU;
   Reg #(Bit #(1))  rg_sstatus_SUM <- mkRegU;
   Reg #(Bit #(1))  rg_mstatus_MXR <- mkRegU;
   Reg #(WordXL)    rg_satp        <- mkRegU;

   // Holds the faulting address
   Reg #(WordXL) rg_tval <- mkRegU;

   // ----------------------------------------------------------------

   function Bool is_even_h_pc (WordXL pc);
      return (pc [1:0] == 2'b00);
   endfunction

   function Bool is_odd_h_pc (WordXL pc);
      return (pc [1:0] != 2'b00);
   endfunction

   function Bool is_32b_instr (Bit #(n) instr);
      return (instr [1:0] == 2'b11);
   endfunction

   function Bool is_16b_instr (Bit #(n) instr);
      return (instr [1:0] != 2'b11);
   endfunction

   Bool eqw_rg_pc_imem_pc = ((imem32.pc >> 2) == (rg_pc >> 2));

   // ----------------------------------------------------------------
   // Conditions for selecting 16b and 32b instruction

   // Condition: 32b instr from {imem [15:0], rg_instr_15_0}
   Bool cond_i32_odd = (imem32.pc == rg_pc + 2);    // Assert: is_odd_h_pc (rg_pc)

   // Condition: 32b instr from imem [31:0]
   Bool cond_i32_even = (   eqw_rg_pc_imem_pc
			 && is_even_h_pc (rg_pc)
			 && (is_32b_instr (imem32.instr)));

   // Condition: 16b instr from imem [31:16]
   Bool cond_i16_odd  = (   eqw_rg_pc_imem_pc
			 && is_odd_h_pc (rg_pc)
			 && (is_16b_instr (imem32.instr [31:16])));

   // Condition: 16b instr from imem [15:0]
   Bool cond_i16_even = (   eqw_rg_pc_imem_pc
			 && is_even_h_pc (rg_pc)
			 && (is_16b_instr (imem32.instr)));

   // Condition: 32b instr from { ...next imem [15:0], imem [31:16] }
   Bool cond_i32_odd_fetch_next = (   eqw_rg_pc_imem_pc
				   && is_odd_h_pc (rg_pc)
				   && (is_32b_instr (imem32.instr [31:16])));

   // ================================================================
   // Assert: The underlying imem32 only knows about 32-bit aligned words

   rule rl_assert_fail (! imem32.is_i32_not_i16);
      $display ("ERROR: CPU_Fetch_C: imem32.is_i32_not_i16 is False");
      $finish (1);
   endrule

   rule rl_debug_conds (False && (verbosity != 0) && imem32.valid);
      $display ("    imem32.pc 0x%0h  pc 0x%0h  rg_f3 %0d  imem32.instr 0x%08h  rg_instr_15_0 0x%04h",
		imem32.pc,  rg_pc,  rg_f3,  imem32.instr,  rg_instr_15_0);
      $display ("    cond_i32_odd  = ", fshow (cond_i32_odd));
      $display ("    cond_i32_even = ", fshow (cond_i32_even));
      $display ("    cond_i16_odd  = ", fshow (cond_i16_odd));
      $display ("    cond_i16_even = ", fshow (cond_i16_even));
      $display ("    cond_i32_odd_fetch_next             = ", fshow (cond_i32_odd_fetch_next));
      $display ("    eqw_rg_pc_imem_pc                   = ", fshow (eqw_rg_pc_imem_pc));
      $display ("    is_odd_h_pc (rg_pc)                 = ", fshow (is_odd_h_pc (rg_pc)));
      $display ("    is_32b_instr (imem32.instr [31:16]) = ", fshow (is_32b_instr (imem32.instr [31:16])));
   endrule

   // ================================================================
   // On 32b odd instr in imem32.instr [31:15], fetch next 32 bits

   (* no_implicit_conditions, fire_when_enabled *)
   rule rl_fetch_next_32b (imem32.valid && cond_i32_odd_fetch_next);
      Addr next_word_addr = rg_pc + 2;
      imem32.req (rg_f3, next_word_addr, rg_priv, rg_sstatus_SUM, rg_mstatus_MXR, rg_satp);
      rg_instr_15_0 <= imem32.instr [31:16];
      rg_tval <= next_word_addr;
      if (verbosity != 0)
	 $display ("CPU_Fetch_C.rl_fetch_next_32b:  imem32.pc 0x%0h  next_word_addr 0x%0h",
		   imem32.pc, next_word_addr);
   endrule

   // ----------------
   // Compose the 'instr' 32b output

   Instr instr_out = imem32.instr;
   Bool  is_32b_not_16b = True;
   if (cond_i32_odd)
      instr_out = { imem32.instr [15:0], rg_instr_15_0 };

   else if (cond_i16_even) begin
      instr_out = { 16'b0, imem32.instr [15:0] };
      is_32b_not_16b = False;
   end

   else if (cond_i16_odd) begin
      instr_out = { 16'b0, imem32.instr [31:16] };
      is_32b_not_16b = False;
   end

   // ================================================================
   // INTERFACE

   // CPU side: IMem request
   method Action  req (Bit #(3) f3,
		       WordXL addr,
		       // The following  args for VM
		       Priv_Mode  priv,
		       Bit #(1)   sstatus_SUM,
		       Bit #(1)   mstatus_MXR,
		       WordXL     satp               // { VM_Mode, ASID, PPN_for_page_table }
		       ) if (! cond_i32_odd_fetch_next);
      rg_f3          <= f3;
      rg_pc          <= addr;
      rg_priv        <= priv;
      rg_sstatus_SUM <= sstatus_SUM;
      rg_mstatus_MXR <= mstatus_MXR;
      rg_satp        <= satp;
      rg_tval        <= addr;
      WordXL even_addr = { addr [xlen-1:2], 2'b_00 };
      imem32.req (f3, even_addr, priv, sstatus_SUM, mstatus_MXR, satp);
      if (verbosity > 0) begin
	 $display ("CPU_Fetch_C.req: addr 0x%0h, even_addr 0x%0h", addr, even_addr);
      end
   endmethod

   // CPU side: IMem response
   method Bool     valid    = (imem32.valid && (   cond_i32_odd
						|| cond_i32_even
						|| cond_i16_odd
						|| cond_i16_even));

   method Bool     is_i32_not_i16 = (cond_i32_odd || cond_i32_even);

   method WordXL   pc       = rg_pc;
   method Instr    instr    = instr_out;
   method Bool     exc      = imem32.exc;
   method Exc_Code exc_code = imem32.exc_code;
   method WordXL   tval     = rg_tval;        // Can be different from rg_pc

endmodule

// ================================================================

endpackage


use mkSharedDotProductServer to reduce number of MACs

// Copyright (c) 2014 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import FIFO::*;
import FIFOF::*;
import MIMO::*;
import DefaultValue::*;
import SpecialFIFOs::*;
import Vector::*;
import BRAM::*;
import DmaVector::*;
import PortalMemory::*;
import Dma::*;
import FloatingPoint::*;
import Pipe::*;
import FloatOps::*;
import Timer::*;
import RbmTypes::*;
import Assert::*;
import Connectable::*;

interface DotProdServer#(numeric type n);
   interface Reg#(UInt#(20)) numElts;
   interface Put#(Tuple2#(Vector#(n,Float),Vector#(n,Float))) request;
   interface PipeOut#(Float) pipe;
endinterface

(* synthesize *)
module [Module] mkDotProdServer#(UInt#(TLog#(TMul#(J,K))) label)(DotProdServer#(N));

   let n = valueOf(N);
   let add_depth = valueOf(FP_ADD_DEPTH);
   let mul_depth = valueOf(FP_MUL_DEPTH);
   Bool verbose = False; //label==0;

   Reg#(Bit#(TAdd#(TLog#(FP_ADD_DEPTH),1))) initCnt <- mkReg(0);
   FIFO#(void) initCtrl <- mkSizedFIFO(1);

   Reg#(UInt#(20)) numEltsReg <- mkReg(0);
   Reg#(UInt#(20)) countInReg <- mkReg(0);

   Vector#(N, FloatAlu#(FP_MUL_DEPTH)) muls <- replicateM(mkFloatMultiplier(defaultValue));
   Vector#(N, FloatAlu#(FP_ADD_DEPTH)) adders <- replicateM(mkFloatAdder(defaultValue));

   Vector#(N,FIFOF#(Tuple2#(Float,Float))) abfifos <- replicateM(mkFIFOF());
   Vector#(N,FIFOF#(Bool)) lastFifos <- replicateM(mkSizedFIFOF(mul_depth));

   Vector#(N, Reg#(Bit#(TAdd#(TLog#(FP_ADD_DEPTH),1)))) drainCnts <- replicateM(mkReg(0));
   Vector#(N, Reg#(Bool)) drained <- replicateM(mkReg(False));

   Reg#(Maybe#(Float)) accum <- mkReg(Nothing);
   FIFOF#(Float) dotfifo <- mkFIFOF;

   function Bit#(TLog#(N)) i_v(Integer i) = fromInteger(i);

   rule init if (initCnt > 0);
      for(Integer i = 0; i < n; i = i + 1)
	 adders[i].request.put(tuple2(0,0));
      initCnt <= initCnt - 1;
      if (verbose) $display($format(fshow("label=")+fshow(label)+fshow(" initCnt=")+fshow(initCnt)));
   endrule

   for (Integer i = 0; i < n; i = i + 1)
      rule mul;
	 // this rule could be folded into the 'put' method to reduce latency
	 match {.x,.y} <- toGet(abfifos[i]).get;
	 muls[i].request.put(tuple2(x, y));
	 if (verbose) $display($format(fshow("label=")+fshow(label)+fshow(" mul=")+fshow(i_v(i))));
      endrule

   for (Integer i = 0; i < n; i = i + 1)
      rule acc if (drainCnts[i] == 0 && initCnt == 0);
	 match {.resp,.*} <- muls[i].response.get();
	 match {.acc,.*} <- adders[i].response.get;
	 adders[i].request.put(tuple2(resp,acc));
	 let last <- toGet(lastFifos[i]).get;
	 if (last) begin
	    if (i>0)
	       drainCnts[i] <= fromInteger(add_depth);
	    else
	       drainCnts[i] <= fromInteger((add_depth*2)-1); // is this correct??
	 end
	 if (verbose) $display($format(fshow("label=")+fshow(label)+fshow(" acc=")+fshow(i_v(i))+fshow(" last=")+fshow(last)));
      endrule

   for (Integer i = 1; i < n; i = i + 1)
      rule gather if (drainCnts[i] > 0);
	 let new_cnt = drainCnts[i]-1;
	 drained[i] <= (new_cnt==0);
	 drainCnts[i] <= new_cnt;
	 match {.a,.*} <- adders[0].response.get;
	 match {.b,.*} <- adders[i].response.get;
	 adders[0].request.put(tuple2(a,b));
	 if (verbose) $display($format(fshow("label=")+fshow(label)+fshow(" gather=")+fshow(i_v(i))+fshow(" drainCnt=")+fshow(drainCnts[i])));
      endrule

   // the reference-guide says this should work, but it doesn't compile:
   // let gathered = and(tail(drained));
   function Bool is_true(Bool b) = b;
   let gathered = all(is_true, readVReg(tail(drained)));
   // this will only work correctly when add_depth is odd
   rule drain if (gathered && drainCnts[0] > 0);
      let new_cnt = drainCnts[0]-1;
      match {.a,.*} <- adders[0].response.get;
      drainCnts[0] <= new_cnt;
      let enq = False;
      if(accum matches tagged Valid .v) begin
	 adders[0].request.put(tuple2(a,v));
	 dynamicAssert(new_cnt > 0, "mkDotProdServer::drain");
	 enq = True;
	 accum <= tagged Invalid;
      end
      else begin
	 if (new_cnt == 0) begin
	    dotfifo.enq(a);
	    initCtrl.deq;
	 end
	 else
	    accum <= tagged Valid a;
      end
      if (verbose) $display($format(fshow("label=")+fshow(label)+fshow(" drain0=")+fshow(enq)+fshow(" drainCnt=")+fshow(drainCnts[0])));
   endrule

   PipeOut#(Float) dotpipe = toPipeOut(dotfifo);
   interface Put request;
      method Action put(Tuple2#(Vector#(N,Float),Vector#(N,Float)) tpl);
	 if (countInReg == 0) begin
	    initCtrl.enq(?);
	    initCnt <= fromInteger(add_depth);
	 end
	 let c = countInReg+fromInteger(n);
	 Bool isLast = (c >= numEltsReg);
	 if (isLast) begin
	    c = 0;
	 end
	 countInReg <= c;
	 match { .avec, .bvec } = tpl;
	 function Action enqvalues(Integer i);
	    action
	       abfifos[i].enq(tuple2(avec[i], bvec[i]));
	       lastFifos[i].enq(isLast);
	       //if (verbose) $display($format(fshow("label=")+fshow(label)+fshow(" countInReg=(")+fshow(countInReg)+fshow("/")+fshow(numEltsReg)+fshow(") dotprod x=") + fshow(avec[i]) + fshow(" y=") + fshow(bvec[i])));
	    endaction
	 endfunction
	 Vector#(N, Integer) indices = genVector();
	 mapM_(enqvalues, indices);
      endmethod
   endinterface : request
   interface PipeOut pipe = dotpipe;
   interface Reg numElts = numEltsReg;
endmodule : mkDotProdServer

interface SharedDotProdServer#(numeric type k, numeric type n);
   interface Reg#(UInt#(20)) numElts;
   interface Put#(Vector#(n,Float)) aInput;
   interface Vector#(k, Put#(Vector#(n,Float))) bInput;
   interface Vector#(k, PipeOut#(Float)) pipes;
endinterface

(* synthesize *)
module [Module] mkSharedDotProdServer#(UInt#(TLog#(TMul#(J,K))) label)(SharedDotProdServer#(K,N));

   let n = valueOf(N);
   let add_depth = valueOf(FP_ADD_DEPTH);
   let mul_depth = valueOf(FP_MUL_DEPTH);
   Bool verbose = False; //label==0;

   Reg#(Bit#(TAdd#(TLog#(FP_ADD_DEPTH),1))) initCnt <- mkReg(0);
   FIFO#(void) initCtrl <- mkSizedFIFO(1);

   Reg#(UInt#(20)) numEltsReg <- mkReg(0);
   Reg#(UInt#(20)) countInReg <- mkReg(0);

   FloatAlu#(FP_MUL_DEPTH) mul   <- mkFloatMultiplier(defaultValue);
   FloatAlu#(FP_ADD_DEPTH) adder <- mkFloatAdder(defaultValue);
   Vector#(K,FloatAlu#(FP_MUL_DEPTH)) muls   <- replicateM(mkFloatMultiplier(defaultValue));
   Vector#(K,FloatAlu#(FP_ADD_DEPTH)) adders <- replicateM(mkFloatAdder(defaultValue));

   // This could be done in fewer steps
   FIFOF#(Vector#(N,Float)) afifo <- mkFIFOF();
   Vector#(K, PipeOut#(Vector#(N,Float))) aPipesN <- mkForkVector(toPipeOut(afifo));
   Vector#(K, PipeOut#(Vector#(1,Float))) aPipes  = aPipesN; //<- mapM(mkFunnel, aPipesN);
   PipeOut#(Vector#(K,Float))             aPipe   <- mkJoinVector(id, map(unvectorPipeOut, aPipes));
   PipeOut#(Vector#(1,Float))             aFunnelV <- mkFunnel(aPipe);
   PipeOut#(Float)                        aFunnel = unvectorPipeOut(aFunnelV);

   Vector#(K, FIFOF#(Vector#(N,Float)))   bfifos <- replicateM(mkFIFOF());
   Vector#(K, PipeOut#(Vector#(N,Float))) bPipesN = map(toPipeOut,bfifos);
   Vector#(K, PipeOut#(Vector#(1,Float))) bPipes = bPipesN; //<- mapM(mkFunnel, bPipesN);
   PipeOut#(Vector#(K,Float))             bPipe  <- mkJoinVector(id, map(unvectorPipeOut, bPipes));
   PipeOut#(Vector#(1,Float))             bFunnelV <- mkFunnel(bPipe);
   PipeOut#(Float)                        bFunnel = unvectorPipeOut(bFunnelV);

   Vector#(K,FIFOF#(Bool)) firstFifos <- replicateM(mkSizedFIFOF(valueOf(K)));
   Vector#(K,FIFOF#(Bool)) lastFifos <- replicateM(mkSizedFIFOF(valueOf(K)));

   Vector#(N, Reg#(Bit#(TAdd#(TLog#(FP_ADD_DEPTH),1)))) drainCnts <- replicateM(mkReg(0));
   Vector#(N, Reg#(Bool)) drained <- replicateM(mkReg(False));

   Vector#(K, FIFOF#(Float))  accumFifos <- replicateM(mkFIFOF);
   Vector#(K,FIFOF#(Float)) dotfifos <- replicateM(mkFIFOF);

   Reg#(Bit#(TLog#(K))) chanReg <- mkReg(0);
   Vector#(2,FIFO#(Bit#(TLog#(K)))) chanFifos <- replicateM(mkSizedFIFO(valueOf(K)));

   function Bit#(TLog#(N)) i_v(Integer i) = fromInteger(i);

   Reg#(Bit#(32)) cycles <- mkReg(0);
   rule countCycles;
      cycles <= cycles + 1;
   endrule

   Vector#(K, Reg#(Bit#(32))) lastMulin <- replicateM(mkReg(0));
   Vector#(12, Reg#(Bool))   latencyReported <- replicateM(mkReg(False));

   (* descending_urgency = "mulin,accout" *)
   rule mulin;
      let chan = chanReg;
      begin
	 //$display("%08d label=%d mulin chan=%d chanReg=%d", cycles, label, chan, chanReg);
	 let latency = cycles-lastMulin[chan];
	 if ((lastMulin[chan] - cycles) < 6)
	    $display("%08d label=%d mulin chan=%d latency", cycles, label, chan, latency);
	 if (latency > 11) latency = 11;
	 if (!latencyReported[latency]) begin
	    $display("%08d label=%d mulin chan=%d latency", cycles, label, chan, latency);
	    latencyReported[latency] <= True;
	 end
	 lastMulin[chan] <= cycles;
      end

      for (Integer i = 0; i < 2; i=i+1)
	 chanFifos[i].enq(chan);
      chanReg <= (chan + 1);
      let a <- toGet(aFunnel).get();
      let b <- toGet(bFunnel).get();
      let first <- toGet(firstFifos[chan]).get();
      if (first)
	 accumFifos[chan].enq(unpack(0));
      //mul.request.put(tuple2(a, b));
      muls[chan].request.put(tuple2(a, b));
   endrule

   rule mulout;
      let chan <- toGet(chanFifos[0]).get();
      //$display("label=%d mulout chan=%d chanReg=%d", label, chan, chanReg);
      match {.resp,.*} <- muls[chan].response.get();
      let acc <- toGet(accumFifos[chan]).get();
      //adder.request.put(tuple2(resp,acc));
      adders[chan].request.put(tuple2(resp,acc));
   endrule

   rule accout;
      let chan <- toGet(chanFifos[1]).get();
      //$display("label=%d accout chan=%d chanReg=%d", label, chan, chanReg);
      let last <- toGet(lastFifos[chan]).get;
      match {.acc,.*} <- adders[chan].response.get();
      if (last)
	 dotfifos[chan].enq(acc);
      else
	 accumFifos[chan].enq(acc);
   endrule

   Vector#(K,PipeOut#(Float)) dotpipes = map(toPipeOut, dotfifos);

   interface Put aInput;
      method Action put(Vector#(N,Float) avec);

   	 afifo.enq(avec);

	 let c = countInReg+fromInteger(n);
	 Bool isFirst = (countInReg == 0);
	 Bool isLast = (c >= numEltsReg);
	 if (isLast) begin
	    c = 0;
	 end
	 countInReg <= c;
	 for (Integer k = 0; k < valueOf(K); k = k + 1) begin
	    lastFifos[k].enq(isLast);
	    firstFifos[k].enq(isFirst);
	 end
      endmethod
   endinterface
   interface Vector bInput = map(toPut, bfifos);
   interface Vector pipes = dotpipes;
   interface Reg numElts = numEltsReg;
endmodule : mkSharedDotProdServer

function Vector#(TMul#(j,k), etype) flattenMatrix(Vector#(j, Vector#(k, etype)) mat);
   function etype flatten(Integer i); return mat[i/valueOf(k)][i%valueOf(k)]; endfunction
   return genWith(flatten);
endfunction

typedef struct {
   a xbase;
   a xlimit;
   a xstep;
   a ybase;
   a ylimit;
   a ystep;
} XYRangeConfig#(type a) deriving (Bits, FShow);

interface XYRangePipeIfc#(type a);
   interface PipeOut#(Tuple2#(a,a)) pipe;
   method Action start(XYRangeConfig#(a) cfg);
   method Action display();
endinterface

module mkXYRangePipeOut(XYRangePipeIfc#(a)) provisos (Arith#(a), Bits#(a,awidth), Eq#(a), Ord#(a));
   Reg#(a) x <- mkReg(0);
   Reg#(a) y <- mkReg(0);
   Reg#(a) xbase <- mkReg(0);
   Reg#(a) ybase <- mkReg(0);
   Reg#(a) xstep <- mkReg(0);
   Reg#(a) ystep <- mkReg(0);
   Reg#(a) xlimit <- mkReg(0);
   Reg#(a) ylimit <- mkReg(0);

   interface PipeOut pipe;
      method Tuple2#(a,a) first() if (x < xlimit && y < ylimit);
	 return tuple2(x,y);
      endmethod
      method Action deq if (x < xlimit && y < ylimit);
	 let newx = x;
	 let newy = y+ystep;
	 if (newy >= ylimit && x < xlimit) begin
	    newy = ybase;
	    newx = newx + xstep;
	 end
	 x <= newx;
	 y <= newy;
      endmethod
      method Bool notEmpty();
	 return (x < xlimit && y < ylimit);
      endmethod
   endinterface
   method Action start(XYRangeConfig#(a) cfg) if (x >= xlimit);
      //$display("XYRangePipe x=%d xlimit=%d xstep=%d y=%d ylimit=%d ystep=%d", cfg.xbase, cfg.xlimit, cfg.xstep, cfg.ybase, cfg.ylimit, cfg.ystep);
      x <= cfg.xbase;
      y <= cfg.ybase;
      xbase <= cfg.xbase;
      ybase <= cfg.ybase;
      xstep <= cfg.xstep;
      ystep <= cfg.ystep;
      xlimit <= cfg.xlimit;
      ylimit <= cfg.ylimit;
   endmethod
   method Action display();
      $display("XYRangePipe x=%d xlimit=%d y=%d ylimit=%d xstep=%d ystep=%d", x, xlimit, xstep, y, ylimit, ystep);
   endmethod
endmodule: mkXYRangePipeOut

typedef struct {
   ObjectPointer pointer;
   addrtype base;
   addrtype numRows;
   addrtype numColumns;
} MatrixDescriptor#(type addrtype) deriving (Bits);

// row major layout
interface DmaMatrixMultiplyIfc#(numeric type addrwidth, numeric type dsz);
   interface Vector#(J, ObjectWriteClient#(dsz)) writeClients;
   method Action start(ObjectPointer pointerA, UInt#(addrwidth) numRowsA, UInt#(addrwidth) numColumnsA,
		       ObjectPointer pointerB, UInt#(addrwidth) numRowsB, UInt#(addrwidth) numColumnsB,
		       ObjectPointer pointerC);
   method ActionValue#(Bool) finish();
endinterface

typedef enum {
   Idle, Ready, Running, Done
   } MMState deriving (Bits, Eq);

/*!
 * Multiplies two matrices A and B and writes the result to memory.
 * Fetches J rows at a time from A and K rows at a time from B.
 * Each cycle, it can fetch N elements of a row or column.
 *
 * Just considering memory bandwidth, every J+K cycles it is ready to perform J*K*N multiply accumulates.
 *
 */
module [Module] mkDmaMatrixMultiply#(Vector#(J, VectorSource#(dsz, Vector#(N, Float))) sourceA,
				     Vector#(K, VectorSource#(dsz, Vector#(N, Float))) sourceB,
				     function Module#(DmaVectorSink#(dsz, Vector#(N, Float))) mkSink(PipeOut#(Vector#(N, Float)) pipe_in)
				     )(DmaMatrixMultiplyIfc#(addrwidth, dsz))
   provisos (  Add#(N,n__,K)
	     , Mul#(N,m__,K)
	     , Add#(1,o__,J)
	     , Log#(N,nshift)
	     , FShow#(Float)
	     , Arith#(Float)
	     , Bits#(Vector#(N, Float), dsz)
	     , Bits#(MatrixDescriptor#(UInt#(addrwidth)), mdsz)
	     , Bits#(Tuple2#(UInt#(addrwidth), UInt#(addrwidth)), tplsz)
	     , Add#(b__, 20, addrwidth)
	     , Add#(a__, addrwidth, 40)
      );

   let n = valueOf(N);
   let jj = valueOf(J);
   let kk = valueOf(K);
   let nshift = valueOf(nshift);
   Bool verbose = False;
   Bool verbose1 = False;
   Bool timing = True;

   Reg#(UInt#(32)) cycles <- mkReg(0);
   Reg#(Bool) doneReg <- mkReg(False);
   Reg#(MatrixDescriptor#(UInt#(addrwidth))) descriptorA <- mkReg(unpack(0));
   Reg#(MatrixDescriptor#(UInt#(addrwidth))) descriptorB <- mkReg(unpack(0));
   Reg#(MatrixDescriptor#(UInt#(addrwidth))) descriptorC <- mkReg(unpack(0));
   Reg#(UInt#(addrwidth)) dotprodCount <- mkReg(0);

   Vector#(J, PipeOut#(Vector#(N,Float))) aPipes  = map(vectorSourcePipe, sourceA);
   Vector#(K, Vector#(J, PipeOut#(Vector#(N,Float)))) bPipes  <- mapM(mkForkVector, map(vectorSourcePipe, sourceB));

   rule countCycles;
      cycles <= cycles+1;
   endrule

   function Module#(SharedDotProdServer#(K,N)) mkFxDotProd(Integer i);
      return mkSharedDotProdServer(fromInteger(i));
   endfunction
   function Vector#(k,PipeOut#(Float)) getDotProdServerPipes(SharedDotProdServer#(k,n) s); return s.pipes; endfunction
   Vector#(J, SharedDotProdServer#(K,N)) fxdotprods <- genWithM(mkFxDotProd);
   Vector#(J, Vector#(K, PipeOut#(Float))) fxpipes = map(getDotProdServerPipes, fxdotprods);

   FirstLastPipe#(UInt#(addrwidth)) firstLastPipe <- mkFirstLastPipe();
   Vector#(2, PipeOut#(Tuple2#(Bool,Bool))) firstLastPipes      <- mkForkVector(firstLastPipe.pipe);

   for (Integer j = 0; j < jj; j = j + 1)
      mkConnection(toGet(aPipes[j]), fxdotprods[j].aInput);
   for (Integer k = 0; k < kk; k = k+1) begin
      for (Integer j = 0; j < jj; j = j + 1) begin
	 mkConnection(toGet(bPipes[k][j]), fxdotprods[j].bInput[k]);
      end
   end

   MIMOConfiguration mimoCfg = defaultValue;
   MIMO#(K,N,TAdd#(K,N),Float) dfifo <- mkMIMO(mimoCfg);
   Vector#(J, MIMO#(K,N,TAdd#(K,N),Float)) dfifos <- replicateM(mkMIMO(mimoCfg));
   let sinks <- mapM(mkSink, map(toPipeOut, dfifos));

   XYRangePipeIfc#(UInt#(addrwidth)) indexpipeifc <- mkXYRangePipeOut();
   XYRangePipeIfc#(UInt#(addrwidth)) offsetpipeA <- mkXYRangePipeOut();
   XYRangePipeIfc#(UInt#(addrwidth)) offsetpipeB <- mkXYRangePipeOut();
   XYRangePipeIfc#(UInt#(addrwidth)) offsetpipeC <- mkXYRangePipeOut();

   Vector#(TAdd#(TAdd#(J,J),K), PipeOut#(Tuple2#(UInt#(addrwidth),UInt#(addrwidth)))) indexpipes <- mkSizedForkVector(valueOf(SourceBufferSize), indexpipeifc.pipe);
   Vector#(J, PipeOut#(Tuple2#(UInt#(addrwidth),UInt#(addrwidth))))        offsetpipesA <- mkSizedForkVector(valueOf(SourceBufferSize), offsetpipeA.pipe);
   Vector#(K, PipeOut#(Tuple2#(UInt#(addrwidth),UInt#(addrwidth))))        offsetpipesB <- mkSizedForkVector(valueOf(SourceBufferSize), offsetpipeB.pipe);
   Vector#(J, PipeOut#(Tuple2#(UInt#(addrwidth),UInt#(addrwidth))))        offsetpipesC <- mkSizedForkVector(valueOf(SourceBufferSize), offsetpipeC.pipe);

   Reg#(Bool) running <- mkReg(False);
   FIFOF#(Bool) doneFifo <- mkFIFOF();

   Vector#(J, Reg#(UInt#(addrwidth))) startAOffset <- replicateM(mkReg(0));
   Vector#(K, Reg#(UInt#(addrwidth))) startBOffset <- replicateM(mkReg(0));
   Vector#(J, Reg#(UInt#(addrwidth))) startCOffset <- replicateM(mkReg(0));
   for (Integer k = 0; k < kk; k = k + 1) begin
      rule startDotProds;
	 Tuple2#(UInt#(addrwidth),UInt#(addrwidth)) index <- toGet(indexpipes[k]).get();
	 match { .unusedB, .startBBase } <- toGet(offsetpipesB[k]).get();

	 int kint = fromInteger(k);

	 let row = tpl_1(index);
	 let col = tpl_2(index)+fromInteger(k);

	 let startB = startBBase + startBOffset[k];

	 if (timing || verbose) $display($format(fshow(cycles)+fshow("    startB index=")+fshow(tuple2(row,col))
	    +fshow(" startB=")+fshow(startB)
	    +fshow(" k=")+fshow(kint)));

	 if (verbose || verbose1) $display($format(fshow(cycles)+fshow("    sourceB[")+fshow(kint)+fshow("].start")+fshow(startB)));

	 sourceB[k].start(descriptorB.pointer, pack(extend(startB>>nshift)), pack(extend(descriptorB.numColumns>>nshift)));
      endrule
      rule finishSourceB;
	 UInt#(TLog#(K)) in = fromInteger(k);
	 int kint = fromInteger(k);
	 if (verbose || verbose1) $display($format(fshow(cycles)+fshow("    sourceB[")+fshow(kint)+fshow("].finish")));
	 let b <- sourceB[k].finish();
      endrule
   end
   for (Integer j = 0; j < jj; j = j + 1) begin

      rule startSourceAndSink;
	 Tuple2#(UInt#(addrwidth),UInt#(addrwidth)) index <- toGet(indexpipes[j+kk]).get();

	 let row = tpl_1(index)+fromInteger(j);
	 let col = tpl_2(index);

	 match { .startABase, .unusedA } <- toGet(offsetpipesA[j]).get();
	 match { .startCBase, .offsetC } <- toGet(offsetpipesC[j]).get();
	 let startA = startABase + startAOffset[j];
	 let startC = startCBase + startCOffset[j] + offsetC;

	 int jint = fromInteger(j);
	 if (timing || verbose) $display($format(fshow(cycles)+fshow("    start A/C index=")+fshow(tuple2(row,col))
	    +fshow(" startA=")+fshow(startA)
	    +fshow(" startC=")+fshow(startC)
	    +fshow(" startC>>nshift=")+fshow(startC>>nshift)
	    +fshow(" j=")+fshow(jint)));

	 sourceA[j].start(descriptorA.pointer, pack(extend(startA>>nshift)), pack(extend(descriptorA.numColumns>>nshift)));
	 if (verbose || verbose1) $display($format(fshow(cycles)+fshow("    sourceA[")+fshow(jint)+fshow("].start")+fshow(startA)));
	 sinks[j].vector.start(descriptorC.pointer, pack(extend(startC>>nshift)), fromInteger(kk/n));
	 if (verbose || verbose1) $display($format(fshow(cycles)+fshow("      sinks[")+fshow(jint)+fshow("].start")+fshow(startC)));

      endrule

      rule finishSourceA;
	 if (verbose || verbose1) $display($format(fshow(cycles)+fshow("    sourceA[0].finish ")));
	 let b <- sourceA[j].finish();
      endrule

      rule dotProdValue;
	 Tuple2#(UInt#(addrwidth),UInt#(addrwidth)) index <- toGet(indexpipes[jj+kk+j]).get();
	 Vector#(K,Float) vs;
	 for (Integer k = 0; k < kk; k = k + 1) begin
	    let dpnum = j*kk+k;
	    let v <- toGet(fxpipes[j][k]).get();
	    //let indexi = tuple2(tpl_1(index), tpl_2(index)+fromInteger(k));
	    //if (verbose) $display($format(fshow(cycles)+fshow("    dotprodvalue index=")+fshow(indexi)+fshow(" dotprod=")+fshow(v)));
	    int jint = fromInteger(j);
	    int kint = fromInteger(k);
	    if (verbose1) $display($format(fshow(cycles)+fshow("    dotprodvalue pos=")+fshow(index)+fshow(tuple2(jint,kint))+fshow(" dotprod=")+fshow(pack(v))));
	    vs[k] = v;
	 end
	 dfifos[j].enq(fromInteger(kk), vs);
      endrule

      rule sinkDone;
	 // each time we write a burst of k values via sinks
	 //let index <- toGet(indexpipes[jj+kk+1]).get();
	 let b <- sinks[j].vector.finish();
	 let c = dotprodCount-fromInteger(kk);
	 int jint = fromInteger(j);
	 if (verbose1) $display($format(fshow(cycles)+fshow("    sinkDone c")+fshow(c)+fshow(" j=")+fshow(jint)));
	 dotprodCount <= c;
	 if (c == 0) begin
	    running <= False;
	    doneFifo.enq(?);
	 end
      endrule
   end

   method Action start(ObjectPointer pointerA, UInt#(addrwidth) numRowsA, UInt#(addrwidth) numColumnsA,
		       ObjectPointer pointerB, UInt#(addrwidth) numRowsB, UInt#(addrwidth) numColumnsB,
		       ObjectPointer pointerC) if (!running);
      XYRangeConfig#(UInt#(addrwidth)) indexcfg  = XYRangeConfig {xbase: 0, xlimit: numRowsA, xstep: fromInteger(jj),
								  ybase: 0, ylimit: numRowsB, ystep: fromInteger(kk) };
      XYRangeConfig#(UInt#(addrwidth)) offsetcfgA = XYRangeConfig {xbase: 0, xlimit: numRowsA*numColumnsA, xstep: numColumnsA*fromInteger(jj),
								  ybase: 0, ylimit: numRowsB, ystep: fromInteger(kk) };
      XYRangeConfig#(UInt#(addrwidth)) offsetcfgB = XYRangeConfig {xbase: 0, xlimit: numRowsA, xstep: fromInteger(jj),
								  ybase: 0, ylimit: numRowsB*numColumnsB, ystep: fromInteger(kk)*numColumnsB };
      XYRangeConfig#(UInt#(addrwidth)) offsetcfgC = XYRangeConfig {xbase: 0, xlimit: numRowsA*numRowsB, xstep: numRowsB*fromInteger(jj),
								  ybase: 0, ylimit: numRowsB, ystep: fromInteger(kk) };
      descriptorA <= MatrixDescriptor { pointer: pointerA, base: 0, numRows: numRowsA, numColumns: numColumnsA};
      descriptorB <= MatrixDescriptor { pointer: pointerB, base: 0, numRows: numRowsB, numColumns: numColumnsB};
      descriptorC <= MatrixDescriptor { pointer: pointerC, base: 0, numRows: numRowsA, numColumns: numRowsB};
      dotprodCount <= numRowsA*numRowsB;
      running <= True;

      if (verbose) $display("mm pointerA=%d pointerB=%d pointerC=%d\n", pointerA, pointerB, pointerC);
      if (verbose) $display("mm.start ra=%d ca=%d rb=%d cb=%d dotprodCount=%d", numRowsA, numColumnsA, numRowsB, numColumnsB, dotprodCount);
      if (verbose) $display($format(fshow("mm.start ")+fshow(indexcfg)));
      indexpipeifc.start(indexcfg);
      offsetpipeA.start(offsetcfgA);
      offsetpipeB.start(offsetcfgB);
      offsetpipeC.start(offsetcfgC);
      for (Integer j = 0; j < jj; j = j + 1) begin
	 startAOffset[j] <= fromInteger(j)*numColumnsA;
	 startCOffset[j] <= fromInteger(j)*numRowsB;
	 fxdotprods[j].numElts <= truncate(numColumnsA);
      end
      for (Integer k = 0; k < kk; k = k + 1) begin
	 startBOffset[k] <= fromInteger(k)*numColumnsB;
      end
   endmethod
   method ActionValue#(Bool) finish();
      if (verbose) $display("mm.finish()");
      doneFifo.deq();
      return True;
   endmethod

   interface Vector writeClients = map(getSinkWriteClient, sinks);
endmodule

typedef 20 MMSize;

interface DramMatrixMultiply#(numeric type n, numeric type dmasz);
   interface Vector#(TAdd#(K,J), ObjectReadClient#(dmasz)) readClients;
   interface Vector#(J, ObjectWriteClient#(dmasz)) writeClients;
   method Action start(ObjectPointer pointerA, UInt#(MMSize) numRowsA, UInt#(MMSize) numColumnsA,
		       ObjectPointer pointerB, UInt#(MMSize) numRowsB, UInt#(MMSize) numColumnsB,
		       ObjectPointer pointerC);
   method ActionValue#(Bool) finish();
   method Bit#(32) dbg();
endinterface

(* synthesize *)
module [Module] mkDramMatrixMultiply(DramMatrixMultiply#(N,TMul#(N,32)));
   Vector#(TAdd#(K,J), DmaVectorSource#(DmaSz, Vector#(N,Float))) vfsources <- replicateM(mkDmaVectorSource());
   Vector#(J, VectorSource#(DmaSz, Vector#(N,Float))) xvfsources = takeAt(0,          map(dmaVectorSourceVector, vfsources));
   Vector#(K, VectorSource#(DmaSz, Vector#(N,Float))) yvfsources = takeAt(valueOf(J), map(dmaVectorSourceVector, vfsources));
   DmaMatrixMultiplyIfc#(MMSize,DmaSz) dmaMMF <- mkDmaMatrixMultiply(xvfsources, yvfsources, mkDmaVectorSink);
   interface Vector readClients = map(getSourceReadClient, vfsources);
   interface Vector writeClients = dmaMMF.writeClients;
   method start = dmaMMF.start;
   method finish = dmaMMF.finish;
   method Bit#(32) dbg();
      Bit#(32) d = 0;
      d[0] = pack(vfsources[0].vector.pipe.notEmpty());
      d[1] = pack(vfsources[1].vector.pipe.notEmpty());
      return d;
   endmethod
endmodule

interface Mm#(numeric type n);
   interface MmRequest mmRequest;
   interface TimerRequest timerRequest;
   interface Vector#(TAdd#(K,J), ObjectReadClient#(TMul#(32,N))) readClients;
   interface Vector#(J, ObjectWriteClient#(TMul#(32,n))) writeClients;
endinterface

module [Module] mkMm#(MmIndication ind, TimerIndication timerInd)(Mm#(N))
   provisos (Add#(1,a__,N),
	     Add#(N,0,n),
	     Mul#(N,32,DmaSz)
      );

   let n = valueOf(n);

   DramMatrixMultiply#(N, TMul#(N,32)) dmaMMF <- mkDramMatrixMultiply();

   Reg#(Bit#(64)) mmfCycles <- mkReg(0);
   rule countMmfCycles;
      mmfCycles <= mmfCycles + 1;
   endrule

   FIFOF#(Bool) busyFifo <- mkFIFOF();
   rule mmfDone;
      let d <- dmaMMF.finish();
      busyFifo.deq();
      ind.mmfDone(mmfCycles);
   endrule

   FIFOF#(Bool) timerRunning <- mkFIFOF();
   Reg#(Bit#(64)) cycleCount <- mkReg(0);
   Reg#(Bit#(64)) idleCount <- mkReg(0);
   rule countCycles if (timerRunning.notEmpty());
      cycleCount <= cycleCount + 1;
      if (!busyFifo.notEmpty())
	 idleCount <= idleCount + 1;
   endrule

   interface TimerRequest timerRequest;
      method Action startTimer() if (!timerRunning.notEmpty());
	 cycleCount <= 0;
	 idleCount <= 0;
	 timerRunning.enq(True);
      endmethod
      method Action stopTimer();
	 timerRunning.deq();
	 timerInd.elapsedCycles(cycleCount, idleCount);
      endmethod
   endinterface

   interface MmRequest mmRequest;
      method Action mmf(Bit#(32) h1, Bit#(32) r1, Bit#(32) c1,
			Bit#(32) h2, Bit#(32) r2, Bit#(32) c2,
			Bit#(32) h3);
	 dmaMMF.start(h1, unpack(truncate(r1)), unpack(truncate(c1)),
		      h2, unpack(truncate(r2)), unpack(truncate(c2)),
		      h3);
	 mmfCycles <= 0;
	 busyFifo.enq(True);
      endmethod
   endinterface

   interface Vector readClients = dmaMMF.readClients;
   interface Vector writeClients =  dmaMMF.writeClients;

endmodule

// Copyright (c) 2014 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import FIFO::*;
import FIFOF::*;
import MIMO::*;
import DefaultValue::*;
import SpecialFIFOs::*;
import Vector::*;
import BRAM::*;
import DmaVector::*;
import PortalMemory::*;
import Dma::*;
import FloatingPoint::*;
import Pipe::*;
import FloatOps::*;
import Timer::*;
import RbmTypes::*;
import Assert::*;
import Connectable::*;

interface DotProdServer#(numeric type n);
   interface Reg#(UInt#(20)) numElts;
   interface Put#(Tuple2#(Vector#(n,Float),Vector#(n,Float))) request;
   interface PipeOut#(Float) pipe;
endinterface

(* synthesize *)
module [Module] mkDotProdServer#(UInt#(TLog#(TMul#(J,K))) label)(DotProdServer#(N));

   let n = valueOf(N);
   let add_depth = valueOf(FP_ADD_DEPTH);
   let mul_depth = valueOf(FP_MUL_DEPTH);
   Bool verbose = False; //label==0;

   Reg#(Bit#(TAdd#(TLog#(FP_ADD_DEPTH),1))) initCnt <- mkReg(0);
   FIFO#(void) initCtrl <- mkSizedFIFO(1);

   Reg#(UInt#(20)) numEltsReg <- mkReg(0);
   Reg#(UInt#(20)) countInReg <- mkReg(0);

   Vector#(N, FloatAlu#(FP_MUL_DEPTH)) muls <- replicateM(mkFloatMultiplier(defaultValue));
   Vector#(N, FloatAlu#(FP_ADD_DEPTH)) adders <- replicateM(mkFloatAdder(defaultValue));

   Vector#(N,FIFOF#(Tuple2#(Float,Float))) abfifos <- replicateM(mkFIFOF());
   Vector#(N,FIFOF#(Bool)) lastFifos <- replicateM(mkSizedFIFOF(mul_depth));

   Vector#(N, Reg#(Bit#(TAdd#(TLog#(FP_ADD_DEPTH),1)))) drainCnts <- replicateM(mkReg(0));
   Vector#(N, Reg#(Bool)) drained <- replicateM(mkReg(False));

   Reg#(Maybe#(Float)) accum <- mkReg(Nothing);
   FIFOF#(Float) dotfifo <- mkFIFOF;

   function Bit#(TLog#(N)) i_v(Integer i) = fromInteger(i);

   rule init if (initCnt > 0);
      for(Integer i = 0; i < n; i = i + 1)
	 adders[i].request.put(tuple2(0,0));
      initCnt <= initCnt - 1;
      if (verbose) $display($format(fshow("label=")+fshow(label)+fshow(" initCnt=")+fshow(initCnt)));
   endrule

   for (Integer i = 0; i < n; i = i + 1)
      rule mul;
	 // this rule could be folded into the 'put' method to reduce latency
	 match {.x,.y} <- toGet(abfifos[i]).get;
	 muls[i].request.put(tuple2(x, y));
	 if (verbose) $display($format(fshow("label=")+fshow(label)+fshow(" mul=")+fshow(i_v(i))));
      endrule

   for (Integer i = 0; i < n; i = i + 1)
      rule acc if (drainCnts[i] == 0 && initCnt == 0);
	 match {.resp,.*} <- muls[i].response.get();
	 match {.acc,.*} <- adders[i].response.get;
	 adders[i].request.put(tuple2(resp,acc));
	 let last <- toGet(lastFifos[i]).get;
	 if (last) begin
	    if (i>0)
	       drainCnts[i] <= fromInteger(add_depth);
	    else
	       drainCnts[i] <= fromInteger((add_depth*2)-1); // is this correct??
	 end
	 if (verbose) $display($format(fshow("label=")+fshow(label)+fshow(" acc=")+fshow(i_v(i))+fshow(" last=")+fshow(last)));
      endrule

   for (Integer i = 1; i < n; i = i + 1)
      rule gather if (drainCnts[i] > 0);
	 let new_cnt = drainCnts[i]-1;
	 drained[i] <= (new_cnt==0);
	 drainCnts[i] <= new_cnt;
	 match {.a,.*} <- adders[0].response.get;
	 match {.b,.*} <- adders[i].response.get;
	 adders[0].request.put(tuple2(a,b));
	 if (verbose) $display($format(fshow("label=")+fshow(label)+fshow(" gather=")+fshow(i_v(i))+fshow(" drainCnt=")+fshow(drainCnts[i])));
      endrule

   // the reference-guide says this should work, but it doesn't compile:
   // let gathered = and(tail(drained));
   function Bool is_true(Bool b) = b;
   let gathered = all(is_true, readVReg(tail(drained)));
   // this will only work correctly when add_depth is odd
   rule drain if (gathered && drainCnts[0] > 0);
      let new_cnt = drainCnts[0]-1;
      match {.a,.*} <- adders[0].response.get;
      drainCnts[0] <= new_cnt;
      let enq = False;
      if(accum matches tagged Valid .v) begin
	 adders[0].request.put(tuple2(a,v));
	 dynamicAssert(new_cnt > 0, "mkDotProdServer::drain");
	 enq = True;
	 accum <= tagged Invalid;
      end
      else begin
	 if (new_cnt == 0) begin
	    dotfifo.enq(a);
	    initCtrl.deq;
	 end
	 else
	    accum <= tagged Valid a;
      end
      if (verbose) $display($format(fshow("label=")+fshow(label)+fshow(" drain0=")+fshow(enq)+fshow(" drainCnt=")+fshow(drainCnts[0])));
   endrule

   PipeOut#(Float) dotpipe = toPipeOut(dotfifo);
   interface Put request;
      method Action put(Tuple2#(Vector#(N,Float),Vector#(N,Float)) tpl);
	 if (countInReg == 0) begin
	    initCtrl.enq(?);
	    initCnt <= fromInteger(add_depth);
	 end
	 let c = countInReg+fromInteger(n);
	 Bool isLast = (c >= numEltsReg);
	 if (isLast) begin
	    c = 0;
	 end
	 countInReg <= c;
	 match { .avec, .bvec } = tpl;
	 function Action enqvalues(Integer i);
	    action
	       abfifos[i].enq(tuple2(avec[i], bvec[i]));
	       lastFifos[i].enq(isLast);
	       //if (verbose) $display($format(fshow("label=")+fshow(label)+fshow(" countInReg=(")+fshow(countInReg)+fshow("/")+fshow(numEltsReg)+fshow(") dotprod x=") + fshow(avec[i]) + fshow(" y=") + fshow(bvec[i])));
	    endaction
	 endfunction
	 Vector#(N, Integer) indices = genVector();
	 mapM_(enqvalues, indices);
      endmethod
   endinterface : request
   interface PipeOut pipe = dotpipe;
   interface Reg numElts = numEltsReg;
endmodule : mkDotProdServer

function Vector#(TMul#(j,k), etype) flattenMatrix(Vector#(j, Vector#(k, etype)) mat);
   function etype flatten(Integer i); return mat[i/valueOf(k)][i%valueOf(k)]; endfunction
   return genWith(flatten);
endfunction

typedef struct {
   a xbase;
   a xlimit;
   a xstep;
   a ybase;
   a ylimit;
   a ystep;
} XYRangeConfig#(type a) deriving (Bits, FShow);

interface XYRangePipeIfc#(type a);
   interface PipeOut#(Tuple2#(a,a)) pipe;
   method Action start(XYRangeConfig#(a) cfg);
   method Action display();
endinterface

module mkXYRangePipeOut(XYRangePipeIfc#(a)) provisos (Arith#(a), Bits#(a,awidth), Eq#(a), Ord#(a));
   Reg#(a) x <- mkReg(0);
   Reg#(a) y <- mkReg(0);
   Reg#(a) xbase <- mkReg(0);
   Reg#(a) ybase <- mkReg(0);
   Reg#(a) xstep <- mkReg(0);
   Reg#(a) ystep <- mkReg(0);
   Reg#(a) xlimit <- mkReg(0);
   Reg#(a) ylimit <- mkReg(0);

   interface PipeOut pipe;
      method Tuple2#(a,a) first() if (x < xlimit && y < ylimit);
	 return tuple2(x,y);
      endmethod
      method Action deq if (x < xlimit && y < ylimit);
	 let newx = x;
	 let newy = y+ystep;
	 if (newy >= ylimit && x < xlimit) begin
	    newy = ybase;
	    newx = newx + xstep;
	 end
	 x <= newx;
	 y <= newy;
      endmethod
      method Bool notEmpty();
	 return (x < xlimit && y < ylimit);
      endmethod
   endinterface
   method Action start(XYRangeConfig#(a) cfg) if (x >= xlimit);
      //$display("XYRangePipe x=%d xlimit=%d xstep=%d y=%d ylimit=%d ystep=%d", cfg.xbase, cfg.xlimit, cfg.xstep, cfg.ybase, cfg.ylimit, cfg.ystep);
      x <= cfg.xbase;
      y <= cfg.ybase;
      xbase <= cfg.xbase;
      ybase <= cfg.ybase;
      xstep <= cfg.xstep;
      ystep <= cfg.ystep;
      xlimit <= cfg.xlimit;
      ylimit <= cfg.ylimit;
   endmethod
   method Action display();
      $display("XYRangePipe x=%d xlimit=%d y=%d ylimit=%d xstep=%d ystep=%d", x, xlimit, xstep, y, ylimit, ystep);
   endmethod
endmodule: mkXYRangePipeOut

typedef struct {
   ObjectPointer pointer;
   addrtype base;
   addrtype numRows;
   addrtype numColumns;
} MatrixDescriptor#(type addrtype) deriving (Bits);

// row major layout
interface DmaMatrixMultiplyIfc#(numeric type addrwidth, numeric type dsz);
   interface Vector#(J, ObjectWriteClient#(dsz)) writeClients;
   method Action start(ObjectPointer pointerA, UInt#(addrwidth) numRowsA, UInt#(addrwidth) numColumnsA,
		       ObjectPointer pointerB, UInt#(addrwidth) numRowsB, UInt#(addrwidth) numColumnsB,
		       ObjectPointer pointerC);
   method ActionValue#(Bool) finish();
endinterface

typedef enum {
   Idle, Ready, Running, Done
   } MMState deriving (Bits, Eq);

/*!
 * Multiplies two matrices A and B and writes the result to memory.
 * Fetches J rows at a time from A and K rows at a time from B.
 * Each cycle, it can fetch N elements of a row or column.
 *
 * Just considering memory bandwidth, every J+K cycles it is ready to perform J*K*N multiply accumulates.
 *
 */
module [Module] mkDmaMatrixMultiply#(Vector#(J, VectorSource#(dsz, Vector#(N, Float))) sourceA,
				     Vector#(K, VectorSource#(dsz, Vector#(N, Float))) sourceB,
				     function Module#(DmaVectorSink#(dsz, Vector#(N, Float))) mkSink(PipeOut#(Vector#(N, Float)) pipe_in)
				     )(DmaMatrixMultiplyIfc#(addrwidth, dsz))
   provisos (  Add#(N,n__,K)
	     , Mul#(N,m__,K)
	     , Add#(1,o__,J)
	     , Log#(N,nshift)
	     , FShow#(Float)
	     , Arith#(Float)
	     , Bits#(Vector#(N, Float), dsz)
	     , Bits#(MatrixDescriptor#(UInt#(addrwidth)), mdsz)
	     , Bits#(Tuple2#(UInt#(addrwidth), UInt#(addrwidth)), tplsz)
	     , Add#(b__, 20, addrwidth)
	     , Add#(a__, addrwidth, 40)
      );

   let n = valueOf(N);
   let jj = valueOf(J);
   let kk = valueOf(K);
   let nshift = valueOf(nshift);
   Bool verbose = False;
   Bool verbose1 = False;
   Bool timing = True;

   Reg#(UInt#(32)) cycles <- mkReg(0);
   Reg#(Bool) doneReg <- mkReg(False);
   Reg#(MatrixDescriptor#(UInt#(addrwidth))) descriptorA <- mkReg(unpack(0));
   Reg#(MatrixDescriptor#(UInt#(addrwidth))) descriptorB <- mkReg(unpack(0));
   Reg#(MatrixDescriptor#(UInt#(addrwidth))) descriptorC <- mkReg(unpack(0));
   Reg#(UInt#(addrwidth)) dotprodCount <- mkReg(0);

   Vector#(J, Vector#(K, PipeOut#(Vector#(N,Float)))) aPipes  <- mapM(mkSizedForkVector(valueOf(SourceBufferSize)), map(vectorSourcePipe, sourceA));
   Vector#(K, Vector#(J, PipeOut#(Vector#(N,Float)))) bPipesT <- mapM(mkForkVector, map(vectorSourcePipe, sourceB));
   Vector#(J, Vector#(K, PipeOut#(Vector#(N,Float)))) bPipes  =  transpose(bPipesT);

   rule countCycles;
      cycles <= cycles+1;
   endrule

//`define OLDWAY
`ifdef OLDWAY
   function Module#(DotProdServer#(N)) mkFxDotProd(Integer i);
      return mkDotProdServer(fromInteger(i));
   endfunction
   function Put#(Tuple2#(Vector#(n,Float),Vector#(n,Float))) getDotProdServerRequestInterface(DotProdServer#(n) s); return s.request; endfunction
   function PipeOut#(Float) getDotProdServerPipe(DotProdServer#(n) s); return s.pipe; endfunction
   Vector#(TMul#(J,K), DotProdServer#(N)) fxdotprods <- genWithM(mkFxDotProd);
   Vector#(TMul#(J,K), PipeOut#(Float)) fxpipes = map(getDotProdServerPipe, fxdotprods);

   for (Integer k = 0; k < kk; k = k+1) begin
      for (Integer j = 0; j < jj; j = j + 1) begin
	 rule connectDotProd;
	    let index = j*kk+k;
	    let a <- toGet(aPipes[j][k]).get();
	    let b <- toGet(bPipes[j][k]).get();
	    int jint = fromInteger(j);
	    int kint = fromInteger(k);
	    //$display($format(fshow("connectDotProd pos=")+fshow(tuple2(jint,kint))+fshow(" a=")+fshow(pack(a))+fshow(" b=")+fshow(pack(b))));
	    fxdotprods[index].request.put(tuple2(a, b));
	 endrule
      end
   end
`else
   FirstLastPipe#(UInt#(addrwidth)) firstLastPipe <- mkFirstLastPipe();
   Vector#(2, PipeOut#(Tuple2#(Bool,Bool))) firstLastPipes      <- mkForkVector(firstLastPipe.pipe);

   Vector#(TMul#(J,K), FIFOF#(Float))   accumFifos  <- replicateM(mkFIFOF);
   Vector#(TMul#(J,K), PipeOut#(Float)) accumPipes  =  map(toPipeOut, accumFifos);
   PipeOut#(Vector#(TMul#(J,K), Float)) accumPipe   <- mkJoinVector(id, accumPipes);
   PipeOut#(Vector#(M, Float))          accumFunnel <- mkFunnel(accumPipe);

   rule primeTheAccum;
      match { .first, .last } <- toGet(firstLastPipes[0]).get();
      if (first)
	 for (Integer i = 0; i < valueOf(TMul#(J,K)); i = i + 1)
	    accumFifos[i].enq(unpack(0));
   endrule

   Vector#(TMul#(J,K), PipeOut#(Vector#(1,Float))) aPipeVec <- mapM(mkFunnel, flattenMatrix(aPipes));
   Vector#(TMul#(J,K), PipeOut#(Vector#(1,Float))) bPipeVec <- mapM(mkFunnel, flattenMatrix(bPipes));
   Vector#(TMul#(J,K), PipeOut#(Tuple2#(Vector#(1,Float),Vector#(1,Float)))) abPipeVec = map(uncurry(zipPipeOut), zip(aPipeVec, bPipeVec));
   PipeOut#(Vector#(TMul#(J,K), Tuple2#(Vector#(1,Float),Vector#(1,Float)))) abPipe <- mkJoinVector(id, abPipeVec);

   PipeOut#(Vector#(M, Tuple2#(Vector#(1,Float),Vector#(1,Float)))) abFunnel <- mkFunnel(abPipe);
   Vector#(M, FloatAlu#(FP_MUL_DEPTH)) muls <- replicateM(mkFloatMultiplier(defaultValue));
   Vector#(M, FloatAlu#(FP_ADD_DEPTH)) adders <- replicateM(mkFloatAdder(defaultValue));
   rule mulreq;
      let vec <- toGet(abFunnel).get();
      $display("%d mulreq", cycles);
      for (Integer i = 0; i < valueOf(M); i = i + 1) begin
	 match { .avec, .bvec } = vec[i];
	 muls[i].request.put(tuple2(avec[0], bvec[0]));
      end
   endrule
   rule accumreq;
      $display("%d accumreq", cycles);
      let accum <- toGet(accumFunnel).get();
      for (Integer i = 0; i < valueOf(M); i = i + 1) begin
	 match {.resp, .*} <- muls[i].response.get();
	 adders[i].request.put(tuple2(resp, accum[i]));
      end
   endrule
   FIFOF#(Vector#(M, Float))   accumOutFunnel     <- mkFIFOF();
   PipeOut#(Vector#(M, Float)) accumOutFunnelPipe =  toPipeOut(accumOutFunnel);
   rule accumresp;
      function ActionValue#(Float) getAccumResponse(FloatAlu#(depth) adder); actionvalue match {.resp, .*} <- adder.response.get(); return resp; endactionvalue endfunction
      Vector#(M,Float) accum <- mapM(getAccumResponse, adders);
      accumOutFunnel.enq(accum);
   endrule
   PipeOut#(Vector#(TMul#(J,K), (Float))) accumOutPipe <- mkUnfunnel(accumOutFunnelPipe);
   Vector#(TMul#(J,K), FIFOF#(Float)) resultFifos <- replicateM(mkFIFOF());
   rule feedback;
      let accumOut <- toGet(accumOutPipe).get();
      match { .first, .last } <- toGet(firstLastPipes[1]).get();
      for (Integer i = 0; i < valueOf(TMul#(J,K)); i = i + 1)
	 if (last)
	    resultFifos[i].enq(accumOut[i]);
	 else
	    accumFifos[i].enq(accumOut[i]);
   endrule
   Vector#(TMul#(J,K), PipeOut#(Float)) fxpipes = map(toPipeOut, resultFifos);

`endif
   MIMOConfiguration mimoCfg = defaultValue;
   MIMO#(K,N,TAdd#(K,N),Float) dfifo <- mkMIMO(mimoCfg);
   Vector#(J, MIMO#(K,N,TAdd#(K,N),Float)) dfifos <- replicateM(mkMIMO(mimoCfg));
   let sinks <- mapM(mkSink, map(toPipeOut, dfifos));

   XYRangePipeIfc#(UInt#(addrwidth)) indexpipeifc <- mkXYRangePipeOut();
   XYRangePipeIfc#(UInt#(addrwidth)) offsetpipeA <- mkXYRangePipeOut();
   XYRangePipeIfc#(UInt#(addrwidth)) offsetpipeB <- mkXYRangePipeOut();
   XYRangePipeIfc#(UInt#(addrwidth)) offsetpipeC <- mkXYRangePipeOut();

   Vector#(TAdd#(TAdd#(J,J),K), PipeOut#(Tuple2#(UInt#(addrwidth),UInt#(addrwidth)))) indexpipes <- mkSizedForkVector(valueOf(SourceBufferSize), indexpipeifc.pipe);
   Vector#(J, PipeOut#(Tuple2#(UInt#(addrwidth),UInt#(addrwidth))))        offsetpipesA <- mkSizedForkVector(valueOf(SourceBufferSize), offsetpipeA.pipe);
   Vector#(K, PipeOut#(Tuple2#(UInt#(addrwidth),UInt#(addrwidth))))        offsetpipesB <- mkSizedForkVector(valueOf(SourceBufferSize), offsetpipeB.pipe);
   Vector#(J, PipeOut#(Tuple2#(UInt#(addrwidth),UInt#(addrwidth))))        offsetpipesC <- mkSizedForkVector(valueOf(SourceBufferSize), offsetpipeC.pipe);

   Reg#(Bool) running <- mkReg(False);
   FIFOF#(Bool) doneFifo <- mkFIFOF();

   Vector#(J, Reg#(UInt#(addrwidth))) startAOffset <- replicateM(mkReg(0));
   Vector#(K, Reg#(UInt#(addrwidth))) startBOffset <- replicateM(mkReg(0));
   Vector#(J, Reg#(UInt#(addrwidth))) startCOffset <- replicateM(mkReg(0));
   for (Integer k = 0; k < kk; k = k + 1) begin
      rule startDotProds;
	 Tuple2#(UInt#(addrwidth),UInt#(addrwidth)) index <- toGet(indexpipes[k]).get();
	 match { .unusedB, .startBBase } <- toGet(offsetpipesB[k]).get();

	 int kint = fromInteger(k);

	 let row = tpl_1(index);
	 let col = tpl_2(index)+fromInteger(k);

	 let startB = startBBase + startBOffset[k];

	 if (timing || verbose) $display($format(fshow(cycles)+fshow("    startB index=")+fshow(tuple2(row,col))
	    +fshow(" startB=")+fshow(startB)
	    +fshow(" k=")+fshow(kint)));

	 if (verbose || verbose1) $display($format(fshow(cycles)+fshow("    sourceB[")+fshow(kint)+fshow("].start")+fshow(startB)));

	 sourceB[k].start(descriptorB.pointer, pack(extend(startB>>nshift)), pack(extend(descriptorB.numColumns>>nshift)));
      endrule
      rule finishSourceB;
	 UInt#(TLog#(K)) in = fromInteger(k);
	 int kint = fromInteger(k);
	 if (verbose || verbose1) $display($format(fshow(cycles)+fshow("    sourceB[")+fshow(kint)+fshow("].finish")));
	 let b <- sourceB[k].finish();
      endrule
   end
   for (Integer j = 0; j < jj; j = j + 1) begin

      rule startSourceAndSink;
	 Tuple2#(UInt#(addrwidth),UInt#(addrwidth)) index <- toGet(indexpipes[j+kk]).get();

	 let row = tpl_1(index)+fromInteger(j);
	 let col = tpl_2(index);

	 match { .startABase, .unusedA } <- toGet(offsetpipesA[j]).get();
	 match { .startCBase, .offsetC } <- toGet(offsetpipesC[j]).get();
	 let startA = startABase + startAOffset[j];
	 let startC = startCBase + startCOffset[j] + offsetC;

	 int jint = fromInteger(j);
	 if (timing || verbose) $display($format(fshow(cycles)+fshow("    start A/C index=")+fshow(tuple2(row,col))
	    +fshow(" startA=")+fshow(startA)
	    +fshow(" startC=")+fshow(startC)
	    +fshow(" startC>>nshift=")+fshow(startC>>nshift)
	    +fshow(" j=")+fshow(jint)));

	 sourceA[j].start(descriptorA.pointer, pack(extend(startA>>nshift)), pack(extend(descriptorA.numColumns>>nshift)));
	 if (verbose || verbose1) $display($format(fshow(cycles)+fshow("    sourceA[")+fshow(jint)+fshow("].start")+fshow(startA)));
	 sinks[j].vector.start(descriptorC.pointer, pack(extend(startC>>nshift)), fromInteger(kk/n));
	 if (verbose || verbose1) $display($format(fshow(cycles)+fshow("      sinks[")+fshow(jint)+fshow("].start")+fshow(startC)));

	 // this one counts by one rather than 1/n because the source values are unfunneled before mul/add
	 if (j == 0)
	    firstLastPipe.start(truncate(descriptorA.numColumns));
      endrule

      rule finishSourceA;
	 if (verbose || verbose1) $display($format(fshow(cycles)+fshow("    sourceA[0].finish ")));
	 let b <- sourceA[j].finish();
      endrule

      rule dotProdValue;
	 Tuple2#(UInt#(addrwidth),UInt#(addrwidth)) index <- toGet(indexpipes[jj+kk+j]).get();
	 Vector#(K,Float) vs;
	 for (Integer k = 0; k < kk; k = k + 1) begin
	    let dpnum = j*kk+k;
	    let v <- toGet(fxpipes[dpnum]).get();
	    //let indexi = tuple2(tpl_1(index), tpl_2(index)+fromInteger(k));
	    //if (verbose) $display($format(fshow(cycles)+fshow("    dotprodvalue index=")+fshow(indexi)+fshow(" dotprod=")+fshow(v)));
	    int jint = fromInteger(j);
	    int kint = fromInteger(k);
	    if (verbose1) $display($format(fshow(cycles)+fshow("    dotprodvalue pos=")+fshow(index)+fshow(tuple2(jint,kint))+fshow(" dotprod=")+fshow(pack(v))));
	    vs[k] = v;
	 end
	 dfifos[j].enq(fromInteger(kk), vs);
      endrule

      rule sinkDone;
	 // each time we write a burst of k values via sinks
	 //let index <- toGet(indexpipes[jj+kk+1]).get();
	 let b <- sinks[j].vector.finish();
	 let c = dotprodCount-fromInteger(kk);
	 int jint = fromInteger(j);
	 if (verbose1) $display($format(fshow(cycles)+fshow("    sinkDone c")+fshow(c)+fshow(" j=")+fshow(jint)));
	 dotprodCount <= c;
	 if (c == 0) begin
	    running <= False;
	    doneFifo.enq(?);
	 end
      endrule
   end

   method Action start(ObjectPointer pointerA, UInt#(addrwidth) numRowsA, UInt#(addrwidth) numColumnsA,
		       ObjectPointer pointerB, UInt#(addrwidth) numRowsB, UInt#(addrwidth) numColumnsB,
		       ObjectPointer pointerC) if (!running);
      XYRangeConfig#(UInt#(addrwidth)) indexcfg  = XYRangeConfig {xbase: 0, xlimit: numRowsA, xstep: fromInteger(jj),
								  ybase: 0, ylimit: numRowsB, ystep: fromInteger(kk) };
      XYRangeConfig#(UInt#(addrwidth)) offsetcfgA = XYRangeConfig {xbase: 0, xlimit: numRowsA*numColumnsA, xstep: numColumnsA*fromInteger(jj),
								  ybase: 0, ylimit: numRowsB, ystep: fromInteger(kk) };
      XYRangeConfig#(UInt#(addrwidth)) offsetcfgB = XYRangeConfig {xbase: 0, xlimit: numRowsA, xstep: fromInteger(jj),
								  ybase: 0, ylimit: numRowsB*numColumnsB, ystep: fromInteger(kk)*numColumnsB };
      XYRangeConfig#(UInt#(addrwidth)) offsetcfgC = XYRangeConfig {xbase: 0, xlimit: numRowsA*numRowsB, xstep: numRowsB*fromInteger(jj),
								  ybase: 0, ylimit: numRowsB, ystep: fromInteger(kk) };
      descriptorA <= MatrixDescriptor { pointer: pointerA, base: 0, numRows: numRowsA, numColumns: numColumnsA};
      descriptorB <= MatrixDescriptor { pointer: pointerB, base: 0, numRows: numRowsB, numColumns: numColumnsB};
      descriptorC <= MatrixDescriptor { pointer: pointerC, base: 0, numRows: numRowsA, numColumns: numRowsB};
      dotprodCount <= numRowsA*numRowsB;
      running <= True;

      if (verbose) $display("mm pointerA=%d pointerB=%d pointerC=%d\n", pointerA, pointerB, pointerC);
      if (verbose) $display("mm.start ra=%d ca=%d rb=%d cb=%d dotprodCount=%d", numRowsA, numColumnsA, numRowsB, numColumnsB, dotprodCount);
      if (verbose) $display($format(fshow("mm.start ")+fshow(indexcfg)));
      indexpipeifc.start(indexcfg);
      offsetpipeA.start(offsetcfgA);
      offsetpipeB.start(offsetcfgB);
      offsetpipeC.start(offsetcfgC);
      for (Integer j = 0; j < jj; j = j + 1) begin
	 startAOffset[j] <= fromInteger(j)*numColumnsA;
	 startCOffset[j] <= fromInteger(j)*numRowsB;
      end
      for (Integer k = 0; k < kk; k = k + 1) begin
	 for (Integer j = 0; j < jj; j = j + 1) begin
	    let index = j*kk + k;
`ifdef OLDWAY
	    fxdotprods[index].numElts <= truncate(numColumnsA);
`endif
	 end
	 startBOffset[k] <= fromInteger(k)*numColumnsB;
      end
   endmethod
   method ActionValue#(Bool) finish();
      if (verbose) $display("mm.finish()");
      doneFifo.deq();
      return True;
   endmethod

   interface Vector writeClients = map(getSinkWriteClient, sinks);
endmodule

typedef 20 MMSize;

interface DramMatrixMultiply#(numeric type n, numeric type dmasz);
   interface Vector#(TAdd#(K,J), ObjectReadClient#(dmasz)) readClients;
   interface Vector#(J, ObjectWriteClient#(dmasz)) writeClients;
   method Action start(ObjectPointer pointerA, UInt#(MMSize) numRowsA, UInt#(MMSize) numColumnsA,
		       ObjectPointer pointerB, UInt#(MMSize) numRowsB, UInt#(MMSize) numColumnsB,
		       ObjectPointer pointerC);
   method ActionValue#(Bool) finish();
   method Bit#(32) dbg();
endinterface

(* synthesize *)
module [Module] mkDramMatrixMultiply(DramMatrixMultiply#(N,TMul#(N,32)));
   Vector#(TAdd#(K,J), DmaVectorSource#(DmaSz, Vector#(N,Float))) vfsources <- replicateM(mkDmaVectorSource());
   Vector#(J, VectorSource#(DmaSz, Vector#(N,Float))) xvfsources = takeAt(0,          map(dmaVectorSourceVector, vfsources));
   Vector#(K, VectorSource#(DmaSz, Vector#(N,Float))) yvfsources = takeAt(valueOf(J), map(dmaVectorSourceVector, vfsources));
   DmaMatrixMultiplyIfc#(MMSize,DmaSz) dmaMMF <- mkDmaMatrixMultiply(xvfsources, yvfsources, mkDmaVectorSink);
   interface Vector readClients = map(getSourceReadClient, vfsources);
   interface Vector writeClients = dmaMMF.writeClients;
   method start = dmaMMF.start;
   method finish = dmaMMF.finish;
   method Bit#(32) dbg();
      Bit#(32) d = 0;
      d[0] = pack(vfsources[0].vector.pipe.notEmpty());
      d[1] = pack(vfsources[1].vector.pipe.notEmpty());
      return d;
   endmethod
endmodule

interface Mm#(numeric type n);
   interface MmRequest mmRequest;
   interface TimerRequest timerRequest;
   interface Vector#(TAdd#(K,J), ObjectReadClient#(TMul#(32,N))) readClients;
   interface Vector#(J, ObjectWriteClient#(TMul#(32,n))) writeClients;
endinterface

module [Module] mkMm#(MmIndication ind, TimerIndication timerInd)(Mm#(N))
   provisos (Add#(1,a__,N),
	     Add#(N,0,n),
	     Mul#(N,32,DmaSz)
      );

   let n = valueOf(n);

   DramMatrixMultiply#(N, TMul#(N,32)) dmaMMF <- mkDramMatrixMultiply();

   Reg#(Bit#(64)) mmfCycles <- mkReg(0);
   rule countMmfCycles;
      mmfCycles <= mmfCycles + 1;
   endrule

   FIFOF#(Bool) busyFifo <- mkFIFOF();
   rule mmfDone;
      let d <- dmaMMF.finish();
      busyFifo.deq();
      ind.mmfDone(mmfCycles);
   endrule

   FIFOF#(Bool) timerRunning <- mkFIFOF();
   Reg#(Bit#(64)) cycleCount <- mkReg(0);
   Reg#(Bit#(64)) idleCount <- mkReg(0);
   rule countCycles if (timerRunning.notEmpty());
      cycleCount <= cycleCount + 1;
      if (!busyFifo.notEmpty())
	 idleCount <= idleCount + 1;
   endrule

   interface TimerRequest timerRequest;
      method Action startTimer() if (!timerRunning.notEmpty());
	 cycleCount <= 0;
	 idleCount <= 0;
	 timerRunning.enq(True);
      endmethod
      method Action stopTimer();
	 timerRunning.deq();
	 timerInd.elapsedCycles(cycleCount, idleCount);
      endmethod
   endinterface

   interface MmRequest mmRequest;
      method Action mmf(Bit#(32) h1, Bit#(32) r1, Bit#(32) c1,
			Bit#(32) h2, Bit#(32) r2, Bit#(32) c2,
			Bit#(32) h3);
	 dmaMMF.start(h1, unpack(truncate(r1)), unpack(truncate(c1)),
		      h2, unpack(truncate(r2)), unpack(truncate(c2)),
		      h3);
	 mmfCycles <= 0;
	 busyFifo.enq(True);
      endmethod
   endinterface

   interface Vector readClients = dmaMMF.readClients;
   interface Vector writeClients =  dmaMMF.writeClients;

endmodule