move sync processing under pixel conditional

// Copyright (c) 2013 Nokia, Inc.
// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import Vector::*;
import Clocks::*;
import FIFO::*;
import FIFOF::*;
import SpecialFIFOs::*;
import GetPut::*;
import SyncBits::*;
import YUV::*;
import Arith::*;

`ifdef ZC706
typedef 24 HdmiBits;
`else
typedef 16 HdmiBits;
`endif

interface HDMI#(type pixelType);
    method Bit#(1) hdmi_vsync;
    method Bit#(1) hdmi_hsync;
    method Bit#(1) hdmi_de;
    method pixelType hdmi_data;
    interface Clock hdmi_clock_if;
    interface Reset deleteme_unused_reset;
endinterface

interface HdmiInternalRequest;
    method Action setTestPattern(Bit#(1) v);
    method Action setPatternColor(Bit#(32) v);
    method Action setDePixel(Bit#(12) porch, Bit#(12) width, Bit#(12) visible, Bit#(12) last, Bit#(12) mid);
    method Action setDeLine(Bit#(11) porch, Bit#(11) width, Bit#(11) visible, Bit#(11) last, Bit#(11) mid);
    method Action waitForVsync(Bit#(32) unused);
endinterface
interface HdmiInternalIndication;
    method Action vsync(Bit#(64) v, Bit#(32) vs);
endinterface

interface HdmiGenerator#(type pixelType);
    interface HdmiInternalRequest control;
    interface Get#(VideoData#(pixelType)) rgb888;
    interface Put#(Bit#(32)) request;
endinterface

module mkHdmiGenerator#(Clock axi_clock, Reset axi_reset,
   SyncPulseIfc startDMA, HdmiInternalIndication indication)(HdmiGenerator#(Rgb888));
    Clock defaultClock <- exposeCurrentClock();
    Reset defaultReset <- exposeCurrentReset();
    // 1920 * 1080
    Reg#(Bit#(12)) dePixelEnd <- mkSyncReg(0, axi_clock, axi_reset, defaultClock);
    Reg#(Bit#(12)) dePixelWidth <- mkSyncReg(3, axi_clock, axi_reset, defaultClock);
    Reg#(Bit#(12)) dePixelPorch <- mkSyncReg(192, axi_clock, axi_reset, defaultClock);
    Reg#(Bit#(12)) dePixelVisible <- mkSyncReg(1920 + 192, axi_clock, axi_reset, defaultClock);
    Reg#(Bit#(12)) pixelMidpoint <- mkSyncReg((1920/2) + 192, axi_clock, axi_reset, defaultClock);
    Reg#(Bit#(11)) deLineEnd <- mkSyncReg(0, axi_clock, axi_reset, defaultClock);
    Reg#(Bit#(11)) deLineWidth <- mkSyncReg(3, axi_clock, axi_reset, defaultClock);
    Reg#(Bit#(11)) deLinePorch <- mkSyncReg(41, axi_clock, axi_reset, defaultClock);
    Reg#(Bit#(11)) deLineVisible <- mkSyncReg(1080+41, axi_clock, axi_reset, defaultClock);
    Reg#(Bit#(11)) lineMidpoint <- mkSyncReg((1080/2) + 41, axi_clock, axi_reset, defaultClock);
    Vector#(4, Reg#(Bit#(24))) patternRegs <- replicateM(mkSyncReg(24'h00FFFFFF, axi_clock, axi_reset, defaultClock));
    Reg#(Bit#(1)) shadowTestPatternEnabled <- mkSyncReg(1, axi_clock, axi_reset, defaultClock);
    Reg#(Bool) waitingForVsync <- mkSyncReg(False, axi_clock, axi_reset, defaultClock);
    SyncPulseIfc sendVsyncIndication <- mkSyncHandshake(defaultClock, defaultReset, axi_clock);

    Reg#(Bit#(11)) lineCount <- mkReg(0);
    Reg#(Bit#(12)) pixelCount <- mkReg(0);
    Reg#(Bit#(1)) patternIndex0 <- mkReg(0);
    Reg#(Bit#(1)) patternIndex1 <- mkReg(0);
    Reg#(Bit#(1)) testPatternEnabled <- mkReg(1);
    Reg#(Bool) dataEnable <- mkReg(False);
    Reg#(Bool) lineVisible <- mkReg(False);
    Reg#(Bit#(1)) vsync <- mkReg(0);
    Reg#(Bit#(1)) hsync <- mkReg(0);

    Reg#(VideoData#(Rgb888)) rgb888StageReg <- mkReg(unpack(0));
    Reg#(Bool) evenOddPixelReg <- mkReg(False);

    Reg#(Bit#(32)) underflowCount <- mkReg(0);
    Reg#(Bit#(32)) underflowCountAxi <- mkSyncReg(0, defaultClock, defaultReset, axi_clock);
    Reg#(Bit#(32)) counter <- mkReg(0, clocked_by axi_clock, reset_by axi_reset);
    Reg#(Bit#(32)) vsyncCounter <- mkReg(0, clocked_by axi_clock, reset_by axi_reset);
    Reg#(Bit#(32)) elapsed <- mkReg(0, clocked_by axi_clock, reset_by axi_reset);
    Reg#(Bit#(32)) elapsedVsync <- mkReg(0, clocked_by axi_clock, reset_by axi_reset);

    rule axicyclecount;
       counter <= counter + 1;
    endrule

    rule vsyncCount if (startDMA.pulse());
       vsyncCounter <= vsyncCounter+1;
    endrule
    rule vsyncReceived if (sendVsyncIndication.pulse());
       elapsed <= counter;
       elapsedVsync <= vsyncCounter;
       indication.vsync(extend(elapsed - counter), vsyncCounter - vsyncCounter);
       //waitingForVsync <= False;
    endrule

    rule init_pattern;
        patternRegs[1] <= 24'h00FF0000; // blue
        patternRegs[2] <= 24'h0000FF00; // green
        patternRegs[3] <= 24'h000000FF; // red
        //patternRegs[1] <= 32'h80ff80ff; // yuv422 white
        //patternRegs[2] <= 32'h2c961596; // yuv422 green
        //patternRegs[3] <= 32'hff1d6b1d; // yuv422 red
    endrule

    rule inc_counters;
        if (pixelCount == dePixelEnd) begin
           pixelCount <= 0;
           dataEnable <= False;
           patternIndex0 <= 0;
           if (lineCount == deLineEnd) begin
               lineCount <= 0;
               patternIndex1 <= 0;
               lineVisible <= False;
               testPatternEnabled <= shadowTestPatternEnabled;
               if (waitingForVsync)
	          sendVsyncIndication.send();
           end
           else begin
               if (lineCount == deLineVisible)
                   lineVisible <= True;
               lineCount <= lineCount+1;
               if (lineCount >= lineMidpoint)
                   patternIndex1 <= 1;
           end
        end
        else begin
           if (lineCount == deLinePorch) begin
              vsync <= 1;
              if (pixelCount == 0 && testPatternEnabled == 0)
                  startDMA.send();
           end
           else if (lineCount == deLineWidth)
              vsync <= 0;
           if (pixelCount == dePixelPorch)
              hsync <= 1;
           else if (pixelCount == dePixelWidth)
              hsync <= 0;
           if (pixelCount == dePixelVisible)
               dataEnable <= lineVisible;
           pixelCount <= pixelCount + 1;
           if (pixelCount == pixelMidpoint)
               patternIndex0 <= 1;
        end
    endrule

    rule output_data_rule if (!dataEnable);
        rgb888StageReg <= VideoData {de: 0, pixel: unpack(0), vsync: vsync, hsync: hsync };
    endrule

    rule testpattern_rule if (testPatternEnabled != 0 && dataEnable);
        rgb888StageReg <= VideoData {de: 1, vsync: 0, hsync: 0, pixel: unpack(patternRegs[{patternIndex1, patternIndex0}])};
    endrule

    interface Put request;
        method Action put(Bit#(32) v) if (testPatternEnabled == 0 && dataEnable);
           rgb888StageReg <= VideoData {de: 1, vsync: 0, hsync: 0, pixel: unpack(v[23:0])};
        endmethod
    endinterface: request

    interface HdmiInternalRequest control;
        method Action setPatternColor(Bit#(32) v);
            patternRegs[0] <= v[23:0];
        endmethod
        method Action setTestPattern(Bit#(1) v);
            shadowTestPatternEnabled <= v;
        endmethod
        method Action setDePixel(Bit#(12) porch, Bit#(12) width, Bit#(12) visible, Bit#(12) last, Bit#(12) mid);
            dePixelWidth <= width;
            dePixelPorch <= porch;
            dePixelVisible <= visible;
            dePixelEnd <= last;
            pixelMidpoint <= mid;
        endmethod
        method Action setDeLine(Bit#(11) porch, Bit#(11) width, Bit#(11) visible, Bit#(11) last, Bit#(11) mid);
            deLineWidth <= width;
            deLinePorch <= porch;
            deLineVisible <= visible;
            deLineEnd <= last;
            lineMidpoint <= mid;
        endmethod
        method Action waitForVsync(Bit#(32) unused);
            waitingForVsync <= True;
        endmethod
    endinterface
   interface Get rgb888;
      method ActionValue#(VideoData#(Rgb888)) get();
	 return rgb888StageReg;
      endmethod
   endinterface
endmodule

module mkHDMI#(Get#(VideoData#(pixelType)) videoInput)(HDMI#(Bit#(pixelsz)))
   provisos (Bits#(VideoData#(pixelType), a__), Bits#(pixelType,pixelsz));
   Clock defaultClock <- exposeCurrentClock();
   Reset defaultReset <- exposeCurrentReset();
   Wire#(VideoData#(pixelType)) video <- mkDWire(unpack(0));
   rule getvideo;
      let v <- videoInput.get();
      video <= v;
   endrule

   method Bit#(1) hdmi_vsync;
      return video.vsync;
   endmethod
   method Bit#(1) hdmi_hsync;
      return video.hsync;
   endmethod
   method Bit#(1) hdmi_de;
      return video.de;
   endmethod
   method Bit#(pixelsz) hdmi_data;
      return pack(video.pixel);
   endmethod
   interface hdmi_clock_if = defaultClock;
   interface deleteme_unused_reset = defaultReset;
endmodule

interface Rgb888ToYyuv;
   interface Put#(VideoData#(Rgb888)) rgb888;
   interface Get#(VideoData#(Yyuv)) yyuv;
endinterface

(* synthesize *)
module mkRgb888ToYyuv(Rgb888ToYyuv);
    Reg#(VideoData#(Rgb888))                        stage0Reg <- mkReg(unpack(0));
    Reg#(VideoData#(Yuv444Intermediates))           stage1Reg <- mkReg(unpack(0));
    Reg#(VideoData#(Vector#(2,Vector#(3,Bit#(16))))) stage2Reg <- mkReg(unpack(0));
    Reg#(VideoData#(Yuv444))                        stage3Reg <- mkReg(unpack(0));
    Reg#(VideoData#(Yyuv))                          stage4Reg <- mkReg(unpack(0));
    Reg#(Bool) evenOddPixelReg <- mkReg(False);

    rule stage1_rule;
        let previous = stage0Reg;
        let pixel = previous.pixel;
        stage1Reg <= VideoData {
            vsync: previous.vsync, hsync: previous.hsync, de: previous.de,
            pixel: (previous.de != 0) ? rgbToYuvIntermediates(pixel) : unpack(0)
        };
    endrule

    rule stage2_rule;
        let previous = stage1Reg;
       Vector#(4, Vector#(3, Bit#(16))) vprev = previous.pixel;
       Vector#(2, Vector#(3, Bit#(16))) vnext;
       vnext[0] = vadd(vprev[0], vprev[1]);
       vnext[1] = vadd(vprev[2], vprev[3]);

       stage2Reg <= VideoData {
            vsync: previous.vsync, hsync: previous.hsync, de: previous.de,
	    pixel: (previous.de != 0) ? vnext : unpack(0)
        };
    endrule

   rule stage3_rule;
      let previous = stage2Reg;
       Vector#(2, Vector#(3, Bit#(16))) vprev = previous.pixel;
      Yuv444 pixel = yuv444FromVector(vrshift(vadd(vprev[0], vprev[1]), 8));

      stage3Reg <= VideoData {
            vsync: previous.vsync, hsync: previous.hsync, de: previous.de,
	 pixel: (previous.de != 0) ? pixel : unpack(0) };
   endrule

    rule stage4_rule;
        let previous = stage3Reg;
        if (previous.de != 0)
            evenOddPixelReg <= !evenOddPixelReg;
        Yyuv data = Yyuv { uv: evenOddPixelReg ? previous.pixel.u : previous.pixel.v,
                           yy: previous.pixel.y };
        stage4Reg <= VideoData {
            vsync: previous.vsync, hsync: previous.hsync, de: previous.de,
            pixel: data
        };
    endrule

   interface Put rgb888;
      method Action put(VideoData#(Rgb888) v);
	 stage0Reg <= v;
      endmethod
   endinterface
   interface Get yyuv;
      method ActionValue#(VideoData#(Yyuv)) get();
	 return stage4Reg;
      endmethod
   endinterface
endmodule

// Copyright (c) 2013 Nokia, Inc.
// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import Vector::*;
import Clocks::*;
import FIFO::*;
import FIFOF::*;
import SpecialFIFOs::*;
import GetPut::*;
import SyncBits::*;
import YUV::*;
import Arith::*;

`ifdef ZC706
typedef 24 HdmiBits;
`else
typedef 16 HdmiBits;
`endif

interface HDMI#(type pixelType);
    method Bit#(1) hdmi_vsync;
    method Bit#(1) hdmi_hsync;
    method Bit#(1) hdmi_de;
    method pixelType hdmi_data;
    interface Clock hdmi_clock_if;
    interface Reset deleteme_unused_reset;
endinterface

interface HdmiInternalRequest;
    method Action setTestPattern(Bit#(1) v);
    method Action setPatternColor(Bit#(32) v);
    method Action setDePixel(Bit#(12) porch, Bit#(12) width, Bit#(12) visible, Bit#(12) last, Bit#(12) mid);
    method Action setDeLine(Bit#(11) porch, Bit#(11) width, Bit#(11) visible, Bit#(11) last, Bit#(11) mid);
    method Action waitForVsync(Bit#(32) unused);
endinterface
interface HdmiInternalIndication;
    method Action vsync(Bit#(64) v, Bit#(32) vs);
endinterface

interface HdmiGenerator#(type pixelType);
    interface HdmiInternalRequest control;
    interface Get#(VideoData#(pixelType)) rgb888;
    interface Put#(Bit#(32)) request;
endinterface

module mkHdmiGenerator#(Clock axi_clock, Reset axi_reset,
   SyncPulseIfc startDMA, HdmiInternalIndication indication)(HdmiGenerator#(Rgb888));
    Clock defaultClock <- exposeCurrentClock();
    Reset defaultReset <- exposeCurrentReset();
    // 1920 * 1080
    Reg#(Bit#(12)) dePixelEnd <- mkSyncReg(0, axi_clock, axi_reset, defaultClock);
    Reg#(Bit#(12)) dePixelWidth <- mkSyncReg(3, axi_clock, axi_reset, defaultClock);
    Reg#(Bit#(12)) dePixelPorch <- mkSyncReg(192, axi_clock, axi_reset, defaultClock);
    Reg#(Bit#(12)) dePixelVisible <- mkSyncReg(1920 + 192, axi_clock, axi_reset, defaultClock);
    Reg#(Bit#(12)) pixelMidpoint <- mkSyncReg((1920/2) + 192, axi_clock, axi_reset, defaultClock);
    Reg#(Bit#(11)) deLineEnd <- mkSyncReg(0, axi_clock, axi_reset, defaultClock);
    Reg#(Bit#(11)) deLineWidth <- mkSyncReg(3, axi_clock, axi_reset, defaultClock);
    Reg#(Bit#(11)) deLinePorch <- mkSyncReg(41, axi_clock, axi_reset, defaultClock);
    Reg#(Bit#(11)) deLineVisible <- mkSyncReg(1080+41, axi_clock, axi_reset, defaultClock);
    Reg#(Bit#(11)) lineMidpoint <- mkSyncReg((1080/2) + 41, axi_clock, axi_reset, defaultClock);
    Vector#(4, Reg#(Bit#(24))) patternRegs <- replicateM(mkSyncReg(24'h00FFFFFF, axi_clock, axi_reset, defaultClock));
    Reg#(Bit#(1)) shadowTestPatternEnabled <- mkSyncReg(1, axi_clock, axi_reset, defaultClock);
    Reg#(Bool) waitingForVsync <- mkSyncReg(False, axi_clock, axi_reset, defaultClock);
    SyncPulseIfc sendVsyncIndication <- mkSyncHandshake(defaultClock, defaultReset, axi_clock);

    Reg#(Bit#(11)) lineCount <- mkReg(0);
    Reg#(Bit#(12)) pixelCount <- mkReg(0);
    Reg#(Bit#(1)) patternIndex0 <- mkReg(0);
    Reg#(Bit#(1)) patternIndex1 <- mkReg(0);
    Reg#(Bit#(1)) testPatternEnabled <- mkReg(1);
    Reg#(Bool) dataEnable <- mkReg(False);
    Reg#(Bool) lineVisible <- mkReg(False);
    Reg#(Bit#(1)) vsync <- mkReg(0);
    Reg#(Bit#(1)) hsync <- mkReg(0);

    Reg#(VideoData#(Rgb888)) rgb888StageReg <- mkReg(unpack(0));
    Reg#(Bool) evenOddPixelReg <- mkReg(False);

    Reg#(Bit#(32)) underflowCount <- mkReg(0);
    Reg#(Bit#(32)) underflowCountAxi <- mkSyncReg(0, defaultClock, defaultReset, axi_clock);
    Reg#(Bit#(32)) counter <- mkReg(0, clocked_by axi_clock, reset_by axi_reset);
    Reg#(Bit#(32)) vsyncCounter <- mkReg(0, clocked_by axi_clock, reset_by axi_reset);
    Reg#(Bit#(32)) elapsed <- mkReg(0, clocked_by axi_clock, reset_by axi_reset);
    Reg#(Bit#(32)) elapsedVsync <- mkReg(0, clocked_by axi_clock, reset_by axi_reset);

    rule axicyclecount;
       counter <= counter + 1;
    endrule

    rule vsyncCount if (startDMA.pulse());
       vsyncCounter <= vsyncCounter+1;
    endrule
    rule vsyncReceived if (sendVsyncIndication.pulse());
       elapsed <= counter;
       elapsedVsync <= vsyncCounter;
       indication.vsync(extend(elapsed - counter), vsyncCounter - vsyncCounter);
       //waitingForVsync <= False;
    endrule

    rule init_pattern;
        patternRegs[1] <= 24'h00FF0000; // blue
        patternRegs[2] <= 24'h0000FF00; // green
        patternRegs[3] <= 24'h000000FF; // red
        //patternRegs[1] <= 32'h80ff80ff; // yuv422 white
        //patternRegs[2] <= 32'h2c961596; // yuv422 green
        //patternRegs[3] <= 32'hff1d6b1d; // yuv422 red
    endrule

    rule inc_counters;
        if (lineCount == deLinePorch) begin
           vsync <= 1;
           if (pixelCount == 0)
               startDMA.send();
        end
        else if (lineCount == deLineWidth)
           vsync <= 0;
        if (pixelCount == dePixelPorch)
           hsync <= 1;
        else if (pixelCount == dePixelWidth)
           hsync <= 0;
        if (pixelCount == dePixelEnd) begin
           pixelCount <= 0;
           dataEnable <= False;
           patternIndex0 <= 0;
           if (lineCount == deLineEnd) begin
               lineCount <= 0;
               patternIndex1 <= 0;
               lineVisible <= False;
               testPatternEnabled <= shadowTestPatternEnabled;
               if (waitingForVsync)
	          sendVsyncIndication.send();
           end
           else begin
               if (lineCount == deLineVisible)
                   lineVisible <= True;
               lineCount <= lineCount+1;
               if (lineCount >= lineMidpoint)
                   patternIndex1 <= 1;
           end
        end
        else begin
           if (pixelCount == dePixelVisible)
               dataEnable <= lineVisible;
           pixelCount <= pixelCount + 1;
           if (pixelCount == pixelMidpoint)
               patternIndex0 <= 1;
        end
    endrule

    rule output_data_rule if (!dataEnable);
        rgb888StageReg <= VideoData {de: 0, pixel: unpack(0), vsync: vsync, hsync: hsync };
    endrule

    rule testpattern_rule if (testPatternEnabled != 0 && dataEnable);
        Bit#(24) v = patternRegs[{patternIndex1, patternIndex0}];
        //if (lineCount < deLineVisible + 20) begin
            //if (pixelCount < dePixelVisible + 20)
                //v = 0;
            //if (pixelCount > dePixelEnd-30)
                //v = 24'hffffff;
        //end
        rgb888StageReg <= VideoData {de: 1, vsync: 0, hsync: 0, pixel: unpack(v)};
    endrule

    interface Put request;
        method Action put(Bit#(32) v) if (testPatternEnabled == 0 && dataEnable);
           rgb888StageReg <= VideoData {de: 1, vsync: 0, hsync: 0, pixel: unpack(v[23:0])};
        endmethod
    endinterface: request

    interface HdmiInternalRequest control;
        method Action setPatternColor(Bit#(32) v);
            patternRegs[0] <= v[23:0];
        endmethod
        method Action setTestPattern(Bit#(1) v);
            shadowTestPatternEnabled <= v;
        endmethod
        method Action setDePixel(Bit#(12) porch, Bit#(12) width, Bit#(12) visible, Bit#(12) last, Bit#(12) mid);
            dePixelWidth <= width;
            dePixelPorch <= porch;
            dePixelVisible <= visible;
            dePixelEnd <= last;
            pixelMidpoint <= mid;
        endmethod
        method Action setDeLine(Bit#(11) porch, Bit#(11) width, Bit#(11) visible, Bit#(11) last, Bit#(11) mid);
            deLineWidth <= width;
            deLinePorch <= porch;
            deLineVisible <= visible;
            deLineEnd <= last;
            lineMidpoint <= mid;
        endmethod
        method Action waitForVsync(Bit#(32) unused);
            waitingForVsync <= True;
        endmethod
    endinterface
   interface Get rgb888;
      method ActionValue#(VideoData#(Rgb888)) get();
	 return rgb888StageReg;
      endmethod
   endinterface
endmodule

module mkHDMI#(Get#(VideoData#(pixelType)) videoInput)(HDMI#(Bit#(pixelsz)))
   provisos (Bits#(VideoData#(pixelType), a__), Bits#(pixelType,pixelsz));
   Clock defaultClock <- exposeCurrentClock();
   Reset defaultReset <- exposeCurrentReset();
   Wire#(VideoData#(pixelType)) video <- mkDWire(unpack(0));
   rule getvideo;
      let v <- videoInput.get();
      video <= v;
   endrule

   method Bit#(1) hdmi_vsync;
      return video.vsync;
   endmethod
   method Bit#(1) hdmi_hsync;
      return video.hsync;
   endmethod
   method Bit#(1) hdmi_de;
      return video.de;
   endmethod
   method Bit#(pixelsz) hdmi_data;
      return pack(video.pixel);
   endmethod
   interface hdmi_clock_if = defaultClock;
   interface deleteme_unused_reset = defaultReset;
endmodule

interface Rgb888ToYyuv;
   interface Put#(VideoData#(Rgb888)) rgb888;
   interface Get#(VideoData#(Yyuv)) yyuv;
endinterface

(* synthesize *)
module mkRgb888ToYyuv(Rgb888ToYyuv);
    Reg#(VideoData#(Rgb888))                        stage0Reg <- mkReg(unpack(0));
    Reg#(VideoData#(Yuv444Intermediates))           stage1Reg <- mkReg(unpack(0));
    Reg#(VideoData#(Vector#(2,Vector#(3,Bit#(16))))) stage2Reg <- mkReg(unpack(0));
    Reg#(VideoData#(Yuv444))                        stage3Reg <- mkReg(unpack(0));
    Reg#(VideoData#(Yyuv))                          stage4Reg <- mkReg(unpack(0));
    Reg#(Bool) evenOddPixelReg <- mkReg(False);

    rule stage1_rule;
        let previous = stage0Reg;
        let pixel = previous.pixel;
        stage1Reg <= VideoData {
            vsync: previous.vsync, hsync: previous.hsync, de: previous.de,
            pixel: (previous.de != 0) ? rgbToYuvIntermediates(pixel) : unpack(0)
        };
    endrule

    rule stage2_rule;
        let previous = stage1Reg;
       Vector#(4, Vector#(3, Bit#(16))) vprev = previous.pixel;
       Vector#(2, Vector#(3, Bit#(16))) vnext;
       vnext[0] = vadd(vprev[0], vprev[1]);
       vnext[1] = vadd(vprev[2], vprev[3]);

       stage2Reg <= VideoData {
            vsync: previous.vsync, hsync: previous.hsync, de: previous.de,
	    pixel: (previous.de != 0) ? vnext : unpack(0)
        };
    endrule

   rule stage3_rule;
      let previous = stage2Reg;
       Vector#(2, Vector#(3, Bit#(16))) vprev = previous.pixel;
      Yuv444 pixel = yuv444FromVector(vrshift(vadd(vprev[0], vprev[1]), 8));

      stage3Reg <= VideoData {
            vsync: previous.vsync, hsync: previous.hsync, de: previous.de,
	 pixel: (previous.de != 0) ? pixel : unpack(0) };
   endrule

    rule stage4_rule;
        let previous = stage3Reg;
        if (previous.de != 0)
            evenOddPixelReg <= !evenOddPixelReg;
        Yyuv data = Yyuv { uv: evenOddPixelReg ? previous.pixel.u : previous.pixel.v,
                           yy: previous.pixel.y };
        stage4Reg <= VideoData {
            vsync: previous.vsync, hsync: previous.hsync, de: previous.de,
            pixel: data
        };
    endrule

   interface Put rgb888;
      method Action put(VideoData#(Rgb888) v);
	 stage0Reg <= v;
      endmethod
   endinterface
   interface Get yyuv;
      method ActionValue#(VideoData#(Yyuv)) get();
	 return stage4Reg;
      endmethod
   endinterface
endmodule


one more tweak to Rbm.bsv

one more tweak to Rbm.bsv

// Copyright (c) 2014 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import Vector::*;
import FIFO::*;
import FIFOF::*;
import DefaultValue::*;
import GetPut::*;
import ClientServer::*;
import Connectable::*;
import FloatingPoint::*;
import BRAM::*;

import AxiDma::*;
import PortalMemory::*;
import Dma::*;
import DmaVector::*;
import AxiMasterSlave::*;

import Matrix::*;
import Sigmoid::*;
import FloatOps::*;
import Pipe::*;
import Timer::*;
import RbmTypes::*;

function ObjectReadClient#(asz) getSourceReadClient(DmaVectorSource#(asz,a) s); return s.dmaClient; endfunction
function ObjectWriteClient#(asz) getSinkWriteClient(DmaVectorSink#(asz,a) s); return s.dmaClient; endfunction

interface DmaStatesPipe#(numeric type n, numeric type dmasz);
   interface Vector#(1, ObjectReadClient#(dmasz)) sources;
   interface Vector#(1, ObjectWriteClient#(dmasz)) sinks;
   method Action start(Bit#(32) readPointer, Bit#(32) readOffset,
		       Bit#(32) writePointer, Bit#(32) writeOffset, Bit#(32) numElts);
   method ActionValue#(Bool) finish();
endinterface

(* synthesize *)
module [Module] mkDmaStatesPipe(DmaStatesPipe#(N, DmaSz))
   provisos (Bits#(Vector#(N, Float), DmaSz)
	     );
   DmaVectorSource#(DmaSz, Vector#(N,Float)) statesource <- mkDmaVectorSource();
   PipeOut#(Vector#(N, Float)) dmaStatesPipe <- mkComputeStatesPipe(statesource.vector.pipe);
   DmaVectorSink#(DmaSz, Vector#(N, Float)) dmaStatesSink <- mkDmaVectorSink(dmaStatesPipe);

   interface Vector sources = cons(statesource.dmaClient, nil);
   interface Vector sinks = cons(dmaStatesSink.dmaClient, nil);
   method Action start(Bit#(32) readPointer, Bit#(32) readOffset,
				  Bit#(32) writePointer, Bit#(32) writeOffset, Bit#(32) numElts);
      statesource.vector.start(readPointer, extend(readOffset), extend(unpack(numElts)));
      dmaStatesSink.vector.start(writePointer, extend(writeOffset), extend(unpack(numElts)));
   endmethod
   method ActionValue#(Bool) finish();
      let b <- dmaStatesSink.vector.finish();
      return b;
   endmethod
endmodule

interface DmaStatesPipe2#(numeric type n, numeric type dmasz);
   interface Vector#(2, ObjectReadClient#(dmasz))  sources;
   interface Vector#(1, ObjectWriteClient#(dmasz)) sinks;
   method Action start(Bit#(32) readPointer, Bit#(32) readOffset,
		       Bit#(32) readPointer2, Bit#(32) readOffset2,
		       Bit#(32) writePointer, Bit#(32) writeOffset, Bit#(32) numElts);
   method ActionValue#(Bool) finish();
endinterface

(* synthesize *)
module [Module] mkDmaStatesPipe2(DmaStatesPipe2#(N, DmaSz))
   provisos (Bits#(Vector#(N, Float), DmaSz)
	     );
   Vector#(2, DmaVectorSource#(DmaSz, Vector#(N,Float))) statesources <- replicateM(mkDmaVectorSource());
   PipeOut#(Vector#(N, Float)) dmaStatesPipe <- mkComputeStatesPipe2(statesources[0].vector.pipe, statesources[1].vector.pipe);
   DmaVectorSink#(DmaSz, Vector#(N, Float)) dmaStatesSink <- mkDmaVectorSink(dmaStatesPipe);

   interface Vector sources = map(getSourceReadClient, statesources);
   interface Vector sinks = cons(dmaStatesSink.dmaClient, nil);
   method Action start(Bit#(32) readPointer, Bit#(32) readOffset,
		       Bit#(32) readPointer2, Bit#(32) readOffset2,
		       Bit#(32) writePointer, Bit#(32) writeOffset, Bit#(32) numElts);
      statesources[0].vector.start(readPointer, extend(readOffset), extend(unpack(numElts)));
      statesources[1].vector.start(readPointer2, extend(readOffset2), extend(unpack(numElts)));
      dmaStatesSink.vector.start(writePointer, extend(writeOffset), extend(unpack(numElts)));
   endmethod
   method ActionValue#(Bool) finish();
      let b <- dmaStatesSink.vector.finish();
      return b;
   endmethod
endmodule

interface DmaUpdateWeights#(numeric type n, numeric type dmasz);
   interface Vector#(3, ObjectReadClient#(dmasz)) readClients;
   interface Vector#(1, ObjectWriteClient#(dmasz)) writeClients;
   method Action start(Bit#(32) posAssociationsPointer, Bit#(32) negAssociationsPointer, Bit#(32) weightsPointer, Bit#(32) numElts, Float learningRateOverNumExamples);
   method ActionValue#(Bool) finish();
endinterface

(* synthesize *)
module [Module] mkDmaUpdateWeights(DmaUpdateWeights#(N, DmaSz))
   provisos (Bits#(Vector#(N, Float), DmaSz)
	     );
   Vector#(3, DmaVectorSource#(DmaSz, Vector#(N,Float))) sources <- replicateM(mkDmaVectorSource());

   let n = valueOf(N);

   Reg#(Float) learningRateOverNumExamples <- mkReg(defaultValue);

   FIFOF#(Vector#(N,Float)) wfifo <- mkFIFOF();
   Vector#(N, FloatServer2#(Float)) adders <- replicateM(mkFloatAdder());
   Vector#(N, FloatServer2#(Float)) adders2 <- replicateM(mkFloatAdder());
   Vector#(N, FloatServer2#(Float)) multipliers <- replicateM(mkFloatMultiplier());
   DmaVectorSink#(DmaSz, Vector#(N, Float)) sink <- mkDmaVectorSink(toPipeOut(wfifo));

// weights += learningRate * (pos_associations - neg_associations) / num_examples;
   rule sub;
      let pa = sources[0].vector.pipe.first();
      sources[0].vector.pipe.deq();
      let na = sources[1].vector.pipe.first();
      sources[1].vector.pipe.deq();
      for (Integer i = 0; i < n; i = i+1) begin
	 adders[i].request.put(tuple3(pa[i], -na[i], defaultValue));
      end
   endrule
   rule mul;
      for (Integer i = 0; i < n; i = i+1) begin
	 let sumexc <- adders[i].response.get();
	 multipliers[i].request.put(tuple3(learningRateOverNumExamples, tpl_1(sumexc), defaultValue));
      end
   endrule
   rule add;
      let weights = sources[2].vector.pipe.first();
      sources[2].vector.pipe.deq();
      for (Integer i = 0; i < n; i = i+1) begin
	 let resultexc <- multipliers[i].response.get();
	 adders2[i].request.put(tuple3(weights[i], tpl_1(resultexc), defaultValue));
      end
   endrule
   rule result;
      Vector#(N, Float) r;
      for (Integer i = 0; i < n; i = i+1) begin
	 let resultexc <- adders2[i].response.get();
	 r[i] = tpl_1(resultexc);
      end
      wfifo.enq(r);
   endrule

   for (Integer i = 0; i < 3; i = i + 1)
      rule finishSources;
	 let b <- sources[i].vector.finish();
      endrule

   //interface Vector sources = cons(statesource, nil);
   interface Vector readClients = map(getSourceReadClient, sources);
   interface Vector writeClients = cons(getSinkWriteClient(sink), nil);
   method Action start(Bit#(32) posAssociationsPointer, Bit#(32) negAssociationsPointer, Bit#(32) weightsPointer, Bit#(32) numElts, Float lrone);
      learningRateOverNumExamples <= lrone;
      sources[0].vector.start(posAssociationsPointer, 0, extend(numElts));
      sources[1].vector.start(negAssociationsPointer, 0, extend(numElts));
      sources[2].vector.start(weightsPointer, 0, extend(numElts));
      sink.vector.start(weightsPointer, 0, extend(numElts));
   endmethod
   method ActionValue#(Bool) finish();
      let b <- sink.vector.finish();
      return b;
   endmethod
endmodule
interface DmaSumOfErrorSquared#(numeric type n, numeric type dmasz);
   interface Vector#(2, ObjectReadClient#(dmasz)) readClients;
   interface PipeOut#(Float) pipe;
   method Action start(Bit#(32) dataPointer, Bit#(32) predPointer, Bit#(32) numElts);
endinterface

(* synthesize *)
module [Module] mkDmaSumOfErrorSquared(DmaSumOfErrorSquared#(N, DmaSz))
   provisos (Bits#(Vector#(N, Float), DmaSz)
	     );
   Vector#(2, DmaVectorSource#(DmaSz, Vector#(N,Float))) sources <- replicateM(mkDmaVectorSource());

   let n = valueOf(N);

   Reg#(Bit#(32)) resultCount <- mkReg(0);
   Vector#(N, FloatServer2#(Float)) adders <- replicateM(mkFloatAdder());
   Vector#(N, Server#(Tuple4#(Maybe#(Float), Float, Float, RoundMode), Tuple2#(Float,Exception))) macs <- replicateM(mkFloatMac());
   FIFOF#(Vector#(N, Float)) accfifo <- mkFIFOF();
   FIFOF#(Vector#(N, Float)) resultFifo <- mkFIFOF();

   // compute sum((data - pred)^2)

   rule sub;
      let data = sources[0].vector.pipe.first();
      sources[0].vector.pipe.deq();
      let pred = sources[1].vector.pipe.first();
      sources[1].vector.pipe.deq();
      for (Integer i = 0; i < n; i = i+1) begin
	 adders[i].request.put(tuple3(data[i], -pred[i], defaultValue));
      end
   endrule
   rule mac;
      let sum = accfifo.first;
      accfifo.deq;
      for (Integer i = 0; i < n; i = i+1) begin
	 let sumexc <- adders[i].response.get();
	 macs[i].request.put(tuple4(tagged Valid sum[i], tpl_1(sumexc), tpl_1(sumexc), defaultValue));
      end
   endrule
   rule acc if (resultCount > 0);
      Vector#(N, Float) sum = replicate(defaultValue);
      for (Integer i = 0; i < n; i = i+1) begin
	 let resultexc <- macs[i].response.get();
	 sum[i] = tpl_1(resultexc);
      end
      if (resultCount > 1)
	 accfifo.enq(sum);
      else
	 resultFifo.enq(sum);
      resultCount <= resultCount - 1;
   endrule

   PipeOut#(Vector#(N, Float)) sumPipe = toPipeOut(resultFifo);
   PipeOut#(Float) foldedPipe <- mkReducePipe(mkFloatAddPipe, sumPipe);

   for (Integer i = 0; i < 2; i = i + 1)
      rule finishSources;
	 let b <- sources[i].vector.finish();
      endrule

   //interface Vector sources = cons(statesource, nil);
   interface Vector readClients = map(getSourceReadClient, sources);
   interface PipeOut pipe = foldedPipe;
   method Action start(Bit#(32) dataPointer, Bit#(32) predPointer, Bit#(32) numElts) if (resultCount == 0);
      sources[0].vector.start(dataPointer, 0, extend(numElts));
      sources[1].vector.start(predPointer, 0, extend(numElts));
      Vector#(N, Float) sum = replicate(fromReal(0.0));
      accfifo.enq(sum);

      resultCount <= numElts;
   endmethod
endmodule

interface Rbm#(numeric type n);
   interface RbmRequest rbmRequest;
   interface MmRequest mmRequest;
   interface SigmoidRequest sigmoidRequest;
   interface TimerRequest timerRequest;
   interface Vector#(TAdd#(11,N), ObjectReadClient#(TMul#(32,n))) readClients;
   interface Vector#(5, ObjectWriteClient#(TMul#(32,n))) writeClients;
endinterface

module [Module] mkRbm#(RbmIndication rbmInd, MmIndication mmInd, SigmoidIndication sigmoidInd, TimerIndication timerInd)(Rbm#(N))
   provisos (Add#(1,a__,N),
	     Add#(N,0,n),
	     Mul#(N,32,DmaSz)
      );

   let n = valueOf(n);

   DramMatrixMultiply#(N, TMul#(N,32)) dmaMMF <- mkDramMatrixMultiply();
   //DramBramMatrixMultiply#(N, TMul#(N,32)) bramMMF <- mkDramBramMatrixMultiply();

   DmaSigmoidIfc#(TMul#(32,n)) dmaSigmoid <- mkDmaSigmoid();
   Vector#(2,ObjectReadClient#(TMul#(32,n))) sigmoidsources = dmaSigmoid.readClients;

   DmaStatesPipe#(N, DmaSz) dmaStates <- mkDmaStatesPipe();
   DmaStatesPipe2#(N, DmaSz) dmaStates2 <- mkDmaStatesPipe2();

   DmaUpdateWeights#(N, DmaSz) dmaUpdateWeights <- mkDmaUpdateWeights();
   DmaSumOfErrorSquared#(N, DmaSz) dmaSumOfErrorSquared <- mkDmaSumOfErrorSquared();

   FIFOF#(Bool) busyFifo <- mkFIFOF();
   rule mmfDone;
      $display("mmfDone");
      let d <- dmaMMF.finish();
      busyFifo.deq();
      mmInd.mmfDone();
   endrule
   rule sigmoidDone;
      $display("sigmoidDone");
      let d <- dmaSigmoid.finish();
      busyFifo.deq();
      sigmoidInd.sigmoidDone();
   endrule
   rule sigmoidTableUpdateDone;
      $display("sigmoidDone");
      let d <- dmaSigmoid.sigmoidTableUpdated();
      busyFifo.deq();
      sigmoidInd.sigmoidTableUpdated(0);
   endrule
   rule dmaStatesDone;
      $display("dmaStatesDone");
      let b <- dmaStates.finish();
      busyFifo.deq();
      rbmInd.statesDone();
   endrule
   rule dmaStatesDone2;
      $display("dmaStatesDone2");
      let b <- dmaStates2.finish();
      busyFifo.deq();
      rbmInd.statesDone2();
   endrule

   rule dmaUpdateWeightsDone;
      $display("dmaUpdateWeightsDone");
      let b <- dmaUpdateWeights.finish();
      busyFifo.deq();
      rbmInd.updateWeightsDone();
   endrule

   rule dmaSumOfErrorSquaredDone;
      $display("dmaSumOfErrorSquaredDone");
      dmaSumOfErrorSquared.pipe.deq();
      busyFifo.deq();
      rbmInd.sumOfErrorSquared(pack(dmaSumOfErrorSquared.pipe.first()));
   endrule

   FIFOF#(Bool) timerRunning <- mkFIFOF();
   Reg#(Bit#(64)) cycleCount <- mkReg(0);
   Reg#(Bit#(64)) idleCount <- mkReg(0);
   rule countCycles if (timerRunning.notEmpty());
      cycleCount <= cycleCount + 1;
      if (!busyFifo.notEmpty())
	 idleCount <= idleCount + 1;
   endrule

   interface TimerRequest timerRequest;
         method Action startTimer() if (!timerRunning.notEmpty());
	 cycleCount <= 0;
	 idleCount <= 0;
	 timerRunning.enq(True);
      endmethod
      method Action stopTimer();
	 timerRunning.deq();
	 timerInd.elapsedCycles(cycleCount, idleCount);
      endmethod
   endinterface

   interface MmRequest mmRequest;
      method Action mmf(Bit#(32) h1, Bit#(32) r1, Bit#(32) c1,
			Bit#(32) h2, Bit#(32) r2, Bit#(32) c2,
			Bit#(32) h3);
	 dmaMMF.start(h1, unpack(extend(r1)), unpack(extend(c1)),
		      h2, unpack(extend(r2)), unpack(extend(c2)),
		      h3);
	 busyFifo.enq(True);
      endmethod
   endinterface

   interface SigmoidRequest sigmoidRequest;
      method Action sigmoid(Bit#(32) readPointer, Bit#(32) readOffset,
			    Bit#(32) writePointer, Bit#(32) writeOffset, Bit#(32) numElts);
	 dmaSigmoid.start(readPointer, writePointer, unpack(extend(numElts)));
	 busyFifo.enq(True);
      endmethod
      method Action setSigmoidLimits(Bit#(32) rscale, Bit#(32) llimit, Bit#(32) ulimit);
	 $display("rbm.setSigmoidLimits");
	 dmaSigmoid.setSigmoidLimits(unpack(rscale), unpack(llimit), unpack(ulimit));
	 busyFifo.enq(True);
      endmethod
      method Action updateSigmoidTable(Bit#(32) readPointer, Bit#(32) readOffset, Bit#(32) numElts);
	 $display("rbm.updateSigmoidTable pointer=%x addr=%h numElts=%d", readPointer, readOffset, numElts);
	 dmaSigmoid.updateSigmoidTable(readPointer, extend(readOffset), extend(numElts));
	 busyFifo.enq(True);
      endmethod
      method Action sigmoidTableSize();
	 sigmoidInd.sigmoidTableSize(dmaSigmoid.tableSize());
      endmethod
   endinterface

   interface RbmRequest rbmRequest;
      method Action bramMmf(Bit#(32) h1, Bit#(32) r1, Bit#(32) c1,
			    Bit#(32) h2, Bit#(32) r2, Bit#(32) c2,
			    Bit#(32) h3);
	 // bramMMF.start(h1, unpack(extend(r1)), unpack(extend(c1)),
	 // 	       h2, unpack(extend(r2)), unpack(extend(c2)),
	 // 	       h3);
	 // busyFifo.enq(True);
      endmethod
      method Action toBram(Bit#(32) off, Bit#(32) pointer, Bit#(32) offset, Bit#(32) numElts);
	 // bramMMF.toBram(off, pointer, offset, numElts);
	 // busyFifo.enq(True);
      endmethod

      method Action fromBram(Bit#(32) off, Bit#(32) pointer, Bit#(32) offset, Bit#(32) numElts);
	 // bramMMF.fromBram(off, pointer, offset, numElts);
	 // busyFifo.enq(True);
      endmethod

      method Action dbg();
      endmethod

      method Action finish();
	 $finish(0);
      endmethod
      method Action computeStates(Bit#(32) readPointer, Bit#(32) readOffset,
				  Bit#(32) writePointer, Bit#(32) writeOffset, Bit#(32) numElts);
	 //$display("computeStates rh=%d wh=%d len=%d", readPointer, writePointer, numElts);
	 dmaStates.start(readPointer, readOffset,
			 writePointer, writeOffset, numElts);
	 busyFifo.enq(True);
      endmethod
      method Action computeStates2(Bit#(32) readPointer, Bit#(32) readOffset,
				   Bit#(32) readPointer2, Bit#(32) readOffset2,
				   Bit#(32) writePointer, Bit#(32) writeOffset, Bit#(32) numElts);
	 //$display("computeStates2 rh=%d wh=%d len=%d", readPointer, writePointer, numElts);
	 dmaStates2.start(readPointer, readOffset,
			  readPointer2, readOffset2,
			  writePointer,writeOffset, numElts);

	 busyFifo.enq(True);
      endmethod
      method Action updateWeights(Bit#(32) posAssociationsPointer,
	 Bit#(32) negAssociationsPointer,
				  Bit#(32) weightsPointer,
				  Bit#(32) numElts,
				  Bit#(32) learningRateOverNumExamples);
	 dmaUpdateWeights.start(posAssociationsPointer, negAssociationsPointer, weightsPointer,
				numElts, unpack(learningRateOverNumExamples));
	 busyFifo.enq(True);
      endmethod
      method Action sumOfErrorSquared(Bit#(32) dataPointer, Bit#(32) predPointer, Bit#(32) numElts);
	 dmaSumOfErrorSquared.start(dataPointer, predPointer, numElts);
	 busyFifo.enq(True);
      endmethod
   endinterface

   interface Vector readClients = append(
					 //append(
					    dmaMMF.readClients,
					    //bramMMF.readClients
					    //),
					 append(
						append(dmaUpdateWeights.readClients,
						       dmaSumOfErrorSquared.readClients),
						append(
						   sigmoidsources,
						   append(
						      dmaStates.sources,
						      dmaStates2.sources)
						   )));

   interface Vector writeClients = append(
      append(dmaUpdateWeights.writeClients,
	     cons(dmaSigmoid.dmaClient, cons(dmaStates.sinks[0], cons(dmaStates2.sinks[0], nil)))),
//      append(
	 dmaMMF.writeClients //,	     bramMMF.writeClients)
      );
endmodule

// Copyright (c) 2014 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import Vector::*;
import FIFO::*;
import FIFOF::*;
import DefaultValue::*;
import GetPut::*;
import ClientServer::*;
import Connectable::*;
import FloatingPoint::*;
import BRAM::*;

import AxiDma::*;
import PortalMemory::*;
import Dma::*;
import DmaVector::*;
import AxiMasterSlave::*;

import Matrix::*;
import Sigmoid::*;
import FloatOps::*;
import Pipe::*;
import Timer::*;
import RbmTypes::*;

function ObjectReadClient#(asz) getSourceReadClient(DmaVectorSource#(asz,a) s); return s.dmaClient; endfunction
function ObjectWriteClient#(asz) getSinkWriteClient(DmaVectorSink#(asz,a) s); return s.dmaClient; endfunction

interface DmaStatesPipe#(numeric type n, numeric type dmasz);
   interface Vector#(1, ObjectReadClient#(dmasz)) sources;
   interface Vector#(1, ObjectWriteClient#(dmasz)) sinks;
   method Action start(Bit#(32) readPointer, Bit#(32) readOffset,
		       Bit#(32) writePointer, Bit#(32) writeOffset, Bit#(32) numElts);
   method ActionValue#(Bool) finish();
endinterface

(* synthesize *)
module [Module] mkDmaStatesPipe(DmaStatesPipe#(N, DmaSz))
   provisos (Bits#(Vector#(N, Float), DmaSz)
	     );
   DmaVectorSource#(DmaSz, Vector#(N,Float)) statesource <- mkDmaVectorSource();
   PipeOut#(Vector#(N, Float)) dmaStatesPipe <- mkComputeStatesPipe(statesource.vector.pipe);
   DmaVectorSink#(DmaSz, Vector#(N, Float)) dmaStatesSink <- mkDmaVectorSink(dmaStatesPipe);

   interface Vector sources = cons(statesource.dmaClient, nil);
   interface Vector sinks = cons(dmaStatesSink.dmaClient, nil);
   method Action start(Bit#(32) readPointer, Bit#(32) readOffset,
				  Bit#(32) writePointer, Bit#(32) writeOffset, Bit#(32) numElts);
      statesource.vector.start(readPointer, extend(readOffset), extend(unpack(numElts)));
      dmaStatesSink.vector.start(writePointer, extend(writeOffset), extend(unpack(numElts)));
   endmethod
   method ActionValue#(Bool) finish();
      let b <- dmaStatesSink.vector.finish();
      return b;
   endmethod
endmodule

interface DmaStatesPipe2#(numeric type n, numeric type dmasz);
   interface Vector#(2, ObjectReadClient#(dmasz))  sources;
   interface Vector#(1, ObjectWriteClient#(dmasz)) sinks;
   method Action start(Bit#(32) readPointer, Bit#(32) readOffset,
		       Bit#(32) readPointer2, Bit#(32) readOffset2,
		       Bit#(32) writePointer, Bit#(32) writeOffset, Bit#(32) numElts);
   method ActionValue#(Bool) finish();
endinterface

(* synthesize *)
module [Module] mkDmaStatesPipe2(DmaStatesPipe2#(N, DmaSz))
   provisos (Bits#(Vector#(N, Float), DmaSz)
	     );
   Vector#(2, DmaVectorSource#(DmaSz, Vector#(N,Float))) statesources <- replicateM(mkDmaVectorSource());
   PipeOut#(Vector#(N, Float)) dmaStatesPipe <- mkComputeStatesPipe2(statesources[0].vector.pipe, statesources[1].vector.pipe);
   DmaVectorSink#(DmaSz, Vector#(N, Float)) dmaStatesSink <- mkDmaVectorSink(dmaStatesPipe);

   interface Vector sources = map(getSourceReadClient, statesources);
   interface Vector sinks = cons(dmaStatesSink.dmaClient, nil);
   method Action start(Bit#(32) readPointer, Bit#(32) readOffset,
		       Bit#(32) readPointer2, Bit#(32) readOffset2,
		       Bit#(32) writePointer, Bit#(32) writeOffset, Bit#(32) numElts);
      statesources[0].vector.start(readPointer, extend(readOffset), extend(unpack(numElts)));
      statesources[1].vector.start(readPointer2, extend(readOffset2), extend(unpack(numElts)));
      dmaStatesSink.vector.start(writePointer, extend(writeOffset), extend(unpack(numElts)));
   endmethod
   method ActionValue#(Bool) finish();
      let b <- dmaStatesSink.vector.finish();
      return b;
   endmethod
endmodule

interface DmaUpdateWeights#(numeric type n, numeric type dmasz);
   interface Vector#(3, ObjectReadClient#(dmasz)) readClients;
   interface Vector#(1, ObjectWriteClient#(dmasz)) writeClients;
   method Action start(Bit#(32) posAssociationsPointer, Bit#(32) negAssociationsPointer, Bit#(32) weightsPointer, Bit#(32) numElts, Float learningRateOverNumExamples);
   method ActionValue#(Bool) finish();
endinterface

(* synthesize *)
module [Module] mkDmaUpdateWeights(DmaUpdateWeights#(N, DmaSz))
   provisos (Bits#(Vector#(N, Float), DmaSz)
	     );
   Vector#(3, DmaVectorSource#(DmaSz, Vector#(N,Float))) sources <- replicateM(mkDmaVectorSource());

   let n = valueOf(N);

   Reg#(Float) learningRateOverNumExamples <- mkReg(defaultValue);

   FIFOF#(Vector#(N,Float)) wfifo <- mkFIFOF();
   Vector#(N, FloatServer2#(Float)) adders <- replicateM(mkFloatAdder());
   Vector#(N, FloatServer2#(Float)) adders2 <- replicateM(mkFloatAdder());
   Vector#(N, FloatServer2#(Float)) multipliers <- replicateM(mkFloatMultiplier());
   DmaVectorSink#(DmaSz, Vector#(N, Float)) sink <- mkDmaVectorSink(toPipeOut(wfifo));

// weights += learningRate * (pos_associations - neg_associations) / num_examples;
   rule sub;
      let pa = sources[0].vector.pipe.first();
      sources[0].vector.pipe.deq();
      let na = sources[1].vector.pipe.first();
      sources[1].vector.pipe.deq();
      for (Integer i = 0; i < n; i = i+1) begin
	 adders[i].request.put(tuple3(pa[i], -na[i], defaultValue));
      end
   endrule
   rule mul;
      for (Integer i = 0; i < n; i = i+1) begin
	 let sumexc <- adders[i].response.get();
	 multipliers[i].request.put(tuple3(learningRateOverNumExamples, tpl_1(sumexc), defaultValue));
      end
   endrule
   rule add;
      let weights = sources[2].vector.pipe.first();
      sources[2].vector.pipe.deq();
      for (Integer i = 0; i < n; i = i+1) begin
	 let resultexc <- multipliers[i].response.get();
	 adders2[i].request.put(tuple3(weights[i], tpl_1(resultexc), defaultValue));
      end
   endrule
   rule result;
      Vector#(N, Float) r;
      for (Integer i = 0; i < n; i = i+1) begin
	 let resultexc <- adders2[i].response.get();
	 r[i] = tpl_1(resultexc);
      end
      wfifo.enq(r);
   endrule

   for (Integer i = 0; i < 3; i = i + 1)
      rule finishSources;
	 let b <- sources[i].vector.finish();
      endrule

   //interface Vector sources = cons(statesource, nil);
   interface Vector readClients = map(getSourceReadClient, sources);
   interface Vector writeClients = cons(getSinkWriteClient(sink), nil);
   method Action start(Bit#(32) posAssociationsPointer, Bit#(32) negAssociationsPointer, Bit#(32) weightsPointer, Bit#(32) numElts, Float lrone);
      learningRateOverNumExamples <= lrone;
      sources[0].vector.start(posAssociationsPointer, 0, extend(numElts));
      sources[1].vector.start(negAssociationsPointer, 0, extend(numElts));
      sources[2].vector.start(weightsPointer, 0, extend(numElts));
      sink.vector.start(weightsPointer, 0, extend(numElts));
   endmethod
   method ActionValue#(Bool) finish();
      let b <- sink.vector.finish();
      return b;
   endmethod
endmodule
interface DmaSumOfErrorSquared#(numeric type n, numeric type dmasz);
   interface Vector#(2, ObjectReadClient#(dmasz)) readClients;
   interface PipeOut#(Float) pipe;
   method Action start(Bit#(32) dataPointer, Bit#(32) predPointer, Bit#(32) numElts);
endinterface

(* synthesize *)
module [Module] mkDmaSumOfErrorSquared(DmaSumOfErrorSquared#(N, DmaSz))
   provisos (Bits#(Vector#(N, Float), DmaSz)
	     );
   Vector#(2, DmaVectorSource#(DmaSz, Vector#(N,Float))) sources <- replicateM(mkDmaVectorSource());

   let n = valueOf(N);

   Reg#(Bit#(32)) resultCount <- mkReg(0);
   Vector#(N, FloatServer2#(Float)) adders <- replicateM(mkFloatAdder());
   Vector#(N, Server#(Tuple4#(Maybe#(Float), Float, Float, RoundMode), Tuple2#(Float,Exception))) macs <- replicateM(mkFloatMac());
   FIFOF#(Vector#(N, Float)) accfifo <- mkFIFOF();
   FIFOF#(Vector#(N, Float)) resultFifo <- mkFIFOF();

   // compute sum((data - pred)^2)

   rule sub;
      let data = sources[0].vector.pipe.first();
      sources[0].vector.pipe.deq();
      let pred = sources[1].vector.pipe.first();
      sources[1].vector.pipe.deq();
      for (Integer i = 0; i < n; i = i+1) begin
	 adders[i].request.put(tuple3(data[i], -pred[i], defaultValue));
      end
   endrule
   rule mac;
      let sum = accfifo.first;
      accfifo.deq;
      for (Integer i = 0; i < n; i = i+1) begin
	 let sumexc <- adders[i].response.get();
	 macs[i].request.put(tuple4(tagged Valid sum[i], tpl_1(sumexc), tpl_1(sumexc), defaultValue));
      end
   endrule
   rule acc if (resultCount > 0);
      Vector#(N, Float) sum = replicate(defaultValue);
      for (Integer i = 0; i < n; i = i+1) begin
	 let resultexc <- macs[i].response.get();
	 sum[i] = tpl_1(resultexc);
      end
      if (resultCount > 1)
	 accfifo.enq(sum);
      else
	 resultFifo.enq(sum);
      resultCount <= resultCount - 1;
   endrule

   PipeOut#(Vector#(N, Float)) sumPipe = toPipeOut(resultFifo);
   PipeOut#(Float) foldedPipe <- mkReducePipe(mkFloatAddPipe, sumPipe);

   for (Integer i = 0; i < 2; i = i + 1)
      rule finishSources;
	 let b <- sources[i].vector.finish();
      endrule

   //interface Vector sources = cons(statesource, nil);
   interface Vector readClients = map(getSourceReadClient, sources);
   interface PipeOut pipe = foldedPipe;
   method Action start(Bit#(32) dataPointer, Bit#(32) predPointer, Bit#(32) numElts) if (resultCount == 0);
      sources[0].vector.start(dataPointer, 0, extend(numElts));
      sources[1].vector.start(predPointer, 0, extend(numElts));
      Vector#(N, Float) sum = replicate(fromReal(0.0));
      accfifo.enq(sum);

      resultCount <= numElts;
   endmethod
endmodule

interface Rbm#(numeric type n);
   interface RbmRequest rbmRequest;
   interface MmRequest mmRequest;
   interface TimerRequest timerRequest;
   interface Vector#(TAdd#(11,N), ObjectReadClient#(TMul#(32,n))) readClients;
   interface Vector#(5, ObjectWriteClient#(TMul#(32,n))) writeClients;
endinterface

module [Module] mkRbm#(RbmIndication rbmInd, MmIndication mmInd, SigmoidIndication sigmoidInd, TimerIndication timerInd)(Rbm#(N))
   provisos (Add#(1,a__,N),
	     Add#(N,0,n),
	     Mul#(N,32,DmaSz)
      );

   let n = valueOf(n);

   DramMatrixMultiply#(N, TMul#(N,32)) dmaMMF <- mkDramMatrixMultiply();
   //DramBramMatrixMultiply#(N, TMul#(N,32)) bramMMF <- mkDramBramMatrixMultiply();

   DmaSigmoidIfc#(TMul#(32,n)) dmaSigmoid <- mkDmaSigmoid();
   Vector#(2,ObjectReadClient#(TMul#(32,n))) sigmoidsources = dmaSigmoid.readClients;

   DmaStatesPipe#(N, DmaSz) dmaStates <- mkDmaStatesPipe();
   DmaStatesPipe2#(N, DmaSz) dmaStates2 <- mkDmaStatesPipe2();

   DmaUpdateWeights#(N, DmaSz) dmaUpdateWeights <- mkDmaUpdateWeights();
   DmaSumOfErrorSquared#(N, DmaSz) dmaSumOfErrorSquared <- mkDmaSumOfErrorSquared();

   FIFOF#(Bool) busyFifo <- mkFIFOF();
   rule mmfDone;
      $display("mmfDone");
      let d <- dmaMMF.finish();
      busyFifo.deq();
      mmInd.mmfDone();
   endrule
   rule sigmoidDone;
      $display("sigmoidDone");
      let d <- dmaSigmoid.finish();
      busyFifo.deq();
      sigmoidInd.sigmoidDone();
   endrule
   rule sigmoidTableUpdateDone;
      $display("sigmoidDone");
      let d <- dmaSigmoid.sigmoidTableUpdated();
      busyFifo.deq();
      sigmoidInd.sigmoidTableUpdated(0);
   endrule
   rule dmaStatesDone;
      $display("dmaStatesDone");
      let b <- dmaStates.finish();
      busyFifo.deq();
      rbmInd.statesDone();
   endrule
   rule dmaStatesDone2;
      $display("dmaStatesDone2");
      let b <- dmaStates2.finish();
      busyFifo.deq();
      rbmInd.statesDone2();
   endrule

   rule dmaUpdateWeightsDone;
      $display("dmaUpdateWeightsDone");
      let b <- dmaUpdateWeights.finish();
      busyFifo.deq();
      rbmInd.updateWeightsDone();
   endrule

   rule dmaSumOfErrorSquaredDone;
      $display("dmaSumOfErrorSquaredDone");
      dmaSumOfErrorSquared.pipe.deq();
      busyFifo.deq();
      rbmInd.sumOfErrorSquared(pack(dmaSumOfErrorSquared.pipe.first()));
   endrule

   FIFOF#(Bool) timerRunning <- mkFIFOF();
   Reg#(Bit#(64)) cycleCount <- mkReg(0);
   Reg#(Bit#(64)) idleCount <- mkReg(0);
   rule countCycles if (timerRunning.notEmpty());
      cycleCount <= cycleCount + 1;
      if (!busyFifo.notEmpty())
	 idleCount <= idleCount + 1;
   endrule

   interface TimerRequest timerRequest;
         method Action startTimer() if (!timerRunning.notEmpty());
	 cycleCount <= 0;
	 idleCount <= 0;
	 timerRunning.enq(True);
      endmethod
      method Action stopTimer();
	 timerRunning.deq();
	 timerInd.elapsedCycles(cycleCount, idleCount);
      endmethod
   endinterface

   interface MmRequest mmRequest;
      method Action mmf(Bit#(32) h1, Bit#(32) r1, Bit#(32) c1,
			Bit#(32) h2, Bit#(32) r2, Bit#(32) c2,
			Bit#(32) h3);
	 dmaMMF.start(h1, unpack(extend(r1)), unpack(extend(c1)),
		      h2, unpack(extend(r2)), unpack(extend(c2)),
		      h3);
	 busyFifo.enq(True);
      endmethod
   endinterface

   interface SigmoidRequest sigmoidRequest;
      method Action sigmoid(Bit#(32) readPointer, Bit#(32) readOffset,
			    Bit#(32) writePointer, Bit#(32) writeOffset, Bit#(32) numElts);
	 dmaSigmoid.start(readPointer, writePointer, unpack(extend(numElts)));
	 busyFifo.enq(True);
      endmethod
      method Action setSigmoidLimits(Bit#(32) rscale, Bit#(32) llimit, Bit#(32) ulimit);
	 $display("rbm.setSigmoidLimits");
	 dmaSigmoid.setSigmoidLimits(unpack(rscale), unpack(llimit), unpack(ulimit));
	 busyFifo.enq(True);
      endmethod
      method Action updateSigmoidTable(Bit#(32) readPointer, Bit#(32) readOffset, Bit#(32) numElts);
	 $display("rbm.updateSigmoidTable pointer=%x addr=%h numElts=%d", readPointer, readOffset, numElts);
	 dmaSigmoid.updateSigmoidTable(readPointer, extend(readOffset), extend(numElts));
	 busyFifo.enq(True);
      endmethod
      method Action sigmoidTableSize();
	 sigmoidInd.sigmoidTableSize(dmaSigmoid.tableSize());
      endmethod
   endinterface

   interface RbmRequest rbmRequest;
      method Action bramMmf(Bit#(32) h1, Bit#(32) r1, Bit#(32) c1,
			    Bit#(32) h2, Bit#(32) r2, Bit#(32) c2,
			    Bit#(32) h3);
	 // bramMMF.start(h1, unpack(extend(r1)), unpack(extend(c1)),
	 // 	       h2, unpack(extend(r2)), unpack(extend(c2)),
	 // 	       h3);
	 // busyFifo.enq(True);
      endmethod
      method Action toBram(Bit#(32) off, Bit#(32) pointer, Bit#(32) offset, Bit#(32) numElts);
	 // bramMMF.toBram(off, pointer, offset, numElts);
	 // busyFifo.enq(True);
      endmethod

      method Action fromBram(Bit#(32) off, Bit#(32) pointer, Bit#(32) offset, Bit#(32) numElts);
	 // bramMMF.fromBram(off, pointer, offset, numElts);
	 // busyFifo.enq(True);
      endmethod

      method Action dbg();
      endmethod

      method Action finish();
	 $finish(0);
      endmethod
      method Action computeStates(Bit#(32) readPointer, Bit#(32) readOffset,
				  Bit#(32) writePointer, Bit#(32) writeOffset, Bit#(32) numElts);
	 //$display("computeStates rh=%d wh=%d len=%d", readPointer, writePointer, numElts);
	 dmaStates.start(readPointer, readOffset,
			 writePointer, writeOffset, numElts);
	 busyFifo.enq(True);
      endmethod
      method Action computeStates2(Bit#(32) readPointer, Bit#(32) readOffset,
				   Bit#(32) readPointer2, Bit#(32) readOffset2,
				   Bit#(32) writePointer, Bit#(32) writeOffset, Bit#(32) numElts);
	 //$display("computeStates2 rh=%d wh=%d len=%d", readPointer, writePointer, numElts);
	 dmaStates2.start(readPointer, readOffset,
			  readPointer2, readOffset2,
			  writePointer,writeOffset, numElts);

	 busyFifo.enq(True);
      endmethod
      method Action updateWeights(Bit#(32) posAssociationsPointer,
	 Bit#(32) negAssociationsPointer,
				  Bit#(32) weightsPointer,
				  Bit#(32) numElts,
				  Bit#(32) learningRateOverNumExamples);
	 dmaUpdateWeights.start(posAssociationsPointer, negAssociationsPointer, weightsPointer,
				numElts, unpack(learningRateOverNumExamples));
	 busyFifo.enq(True);
      endmethod
      method Action sumOfErrorSquared(Bit#(32) dataPointer, Bit#(32) predPointer, Bit#(32) numElts);
	 dmaSumOfErrorSquared.start(dataPointer, predPointer, numElts);
	 busyFifo.enq(True);
      endmethod
   endinterface

   interface Vector readClients = append(
					 //append(
					    dmaMMF.readClients,
					    //bramMMF.readClients
					    //),
					 append(
						append(dmaUpdateWeights.readClients,
						       dmaSumOfErrorSquared.readClients),
						append(
						   sigmoidsources,
						   append(
						      dmaStates.sources,
						      dmaStates2.sources)
						   )));

   interface Vector writeClients = append(
      append(dmaUpdateWeights.writeClients,
	     cons(dmaSigmoid.dmaClient, cons(dmaStates.sinks[0], cons(dmaStates2.sinks[0], nil)))),
//      append(
	 dmaMMF.writeClients //,	     bramMMF.writeClients)
      );
endmodule


minor readability text cleanups to MemServerInternal

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

// BSV Libraries
import BRAMFIFO::*;
import FIFO::*;
import FIFOF::*;
import Vector::*;
import GetPut::*;
import ClientServer::*;
import Assert::*;
import BRAM::*;
import DefaultValue::*;

// CONNECTAL Libraries
import MemTypes::*;
import ConnectalMemory::*;
import ConnectalClocks::*;
import MMU::*;
import ConnectalCompletionBuffer::*;

typedef 32 BeatCountSize;

typedef 9 MMU_PIPELINE_DEPTH;

interface DmaDbg;
   method ActionValue#(Bit#(64)) getMemoryTraffic();
   method ActionValue#(DmaDbgRec) dbg();
endinterface

interface MemWriteInternal#(numeric type addrWidth, numeric type dataWidth, numeric type numTags, numeric type numServers);
   interface DmaDbg dbg;
   interface Put#(TileControl) tileControl;
   interface PhysMemWriteClient#(addrWidth,dataWidth) client;
   interface Vector#(numServers, MemWriteServer#(dataWidth)) servers;
endinterface

interface MemReadInternal#(numeric type addrWidth, numeric type dataWidth, numeric type numTags, numeric type numServers);
   interface DmaDbg dbg;
   interface Put#(TileControl) tileControl;
   interface PhysMemReadClient#(addrWidth,dataWidth) client;
   interface Vector#(numServers, MemReadServer#(dataWidth)) servers;
endinterface

function Bool sglid_outofrange(SGLId p);
   return ((p[15:0]) >= fromInteger(valueOf(MaxNumSGLists)));
endfunction

typedef struct {MemRequest req;
		Bit#(TLog#(TMax#(1,numClients))) client; } LRec#(numeric type numClients, numeric type addrWidth) deriving(Bits);

typedef struct {MemRequest req;
		Bit#(addrWidth) pa;
		Bit#(MemTagSize) rename_tag;
		Bit#(TLog#(TMax#(1,numClients))) client; } RRec#(numeric type numClients, numeric type addrWidth) deriving(Bits);

typedef struct {Bit#(MemTagSize) req_tag;
		Bit#(BurstLenSize) req_burstLen;
		Bit#(MemTagSize) rename_tag;
		Bit#(TLog#(TMax#(1,numClients))) client;
		Bool last;
   } DRec#(numeric type numClients, numeric type addrWidth) deriving(Bits);

typedef struct {Bit#(MemTagSize) orig_tag;
		Bit#(TLog#(TMax#(1,numClients))) client; } RResp#(numeric type numClients, numeric type addrWidth) deriving(Bits);

typedef struct {DmaErrorType errorType;
		Bit#(32) pref; } DmaError deriving (Bits);

module mkMemReadInternal#(MemServerIndication ind,
			  Vector#(numMMUs,Server#(ReqTup,Bit#(addrWidth))) mmus)
   (MemReadInternal#(addrWidth, dataWidth, numTags, numServers))
   provisos(Log#(dataWidthBytes,beatShift)
	    ,Div#(dataWidth,8,dataWidthBytes)
	    ,Add#(beatShift, a__, 8)
	    ,Add#(b__, TLog#(numTags), 6)
	    );

   // stopping/killing infra
   Vector#(4,Reg#(Bool)) killv <- replicateM(mkReg(False));
   Vector#(4,Reg#(Bool)) stopv <- replicateM(mkReg(False));

   // stage 0: address translation (latency = MMU_PIPELINE_DEPTH)
   FIFO#(LRec#(numServers,addrWidth)) clientRequest <- mkSizedFIFO(valueOf(MMU_PIPELINE_DEPTH));
   // stage 1: address validation (latency = 1)
   FIFO#(RRec#(numServers,addrWidth))  serverRequest <- mkFIFO;
   // stage 2: read commands
   BRAM_Configure bramConfig = defaultValue;
   if (mainClockPeriod < 8)
      bramConfig.latency = 2;
   BRAM2Port#(Bit#(TLog#(numTags)), DRec#(numServers,addrWidth)) serverProcessing <- mkBRAM2Server(bramConfig);
   BRAM2Port#(Bit#(TAdd#(TLog#(numTags),TSub#(BurstLenSize,beatShift))), MemData#(dataWidth)) clientData <- mkBRAM2Server(bramConfig);
   // stage 3: read data
   FIFO#(MemData#(dataWidth)) serverData <- mkFIFO;

   let verbose = False;

   Reg#(Bit#(BurstLenSize)) burstReg <- mkReg(0);
   Reg#(Bool)               firstReg <- mkReg(True);
   Reg#(Bool)                lastReg <- mkReg(False);

   Reg#(Bit#(32))  beatCount <- mkReg(0);
   let beat_shift = fromInteger(valueOf(beatShift));
   TagGen#(numTags) tag_gen <- mkTagGen;

   Reg#(Bit#(BurstLenSize))      compCountReg <- mkReg(0);
   Reg#(Bit#(TLog#(numTags)))    compTagReg <- mkReg(0);
   Reg#(Bit#(TLog#(TMax#(1,numServers)))) compClientReg <- mkReg(0);
   Reg#(Bit#(2))                 compTileReg <- mkReg(0);
   FIFO#(Bit#(TAdd#(1,TLog#(TMax#(1,numServers))))) clientSelect <- mkFIFO;
   FIFO#(Bit#(TLog#(numTags)))   serverTag <- mkFIFO;

   // performance analytics
   Reg#(Bit#(64)) cycle_cnt <- mkReg(0);
   Reg#(Bit#(64)) last_loadClient <- mkReg(0);
   Reg#(Bit#(64)) last_mmuResp <- mkReg(0);
   Reg#(Bit#(64)) last_comp <- mkReg(0);
   Reg#(Bit#(64)) last_readReq <- mkReg(0);
   Reg#(Bit#(64)) last_readData <- mkReg(0);
   (* fire_when_enabled *)
   rule cycle;
      cycle_cnt <= cycle_cnt+1;
   endrule

   FIFO#(DmaError) dmaErrorFifo <- mkFIFO();
   rule dmaError;
      let error <- toGet(dmaErrorFifo).get();
      ind.error(extend(pack(error.errorType)), error.pref, 0, 0);
   endrule

   rule checkMmuResp;
      let request <- toGet(clientRequest).get();
      let physAddr <- mmus[request.req.sglId[31:16]].response.get;
      let rename_tag <- tag_gen.getTag;
      serverRequest.enq(RRec{req:request.req, pa:physAddr, client:request.client, rename_tag:extend(rename_tag)});
      if (verbose) $display("mkMemReadInternal::checkMmuResp: client=%d, rename_tag=%d", request.client,rename_tag);
      if (verbose) $display("mkMemReadInternal::mmuResp %d %d", request.client, cycle_cnt-last_mmuResp);
      last_mmuResp <= cycle_cnt;
   endrule

   rule read_data;
      let response <- toGet(serverData).get();
      Bit#(MemTagSize) response_tag = response.tag;
      let drq <- serverProcessing.portA.response.get;
      let otag = drq.req_tag;
      let burstLen = burstReg;
      let first =    firstReg;
      let last  =    lastReg;
      if (first) begin
	 burstLen = drq.req_burstLen >> beat_shift;
	 last = drq.last;
	 dynamicAssert(last == (burstLen==1), "Last incorrect");
      end
      Bit#(TLog#(numTags)) tt = truncate(response_tag);
      clientData.portA.request.put(BRAMRequest{write:True, responseOnWrite:False, datain:MemData{data: response.data, tag: otag, last: last}, address:{tt,truncate(burstLen)}});
      if (last) begin
	 tag_gen.returnTag(truncate(response_tag));
      end
      last_readData <= cycle_cnt;
      if (verbose) $display("mkMemReadInternal::read_data cyclediff %d", cycle_cnt-last_readData);
      burstReg <= burstLen-1;
      firstReg <= burstLen-1 == 0;
      lastReg  <= burstLen-1 == 1;
   endrule

   rule tag_completed;
      let tag <- tag_gen.complete;
      serverProcessing.portB.request.put(BRAMRequest{write:False, address:tag, datain: ?, responseOnWrite: ?});
      serverTag.enq(tag);
      if(verbose) $display("mkMemReadInternal::complete_burst0 %h", tag);
   endrule

   rule complete_burst1a if (compCountReg==0);
      let drq <- serverProcessing.portB.response.get;
      let req_burstLen = drq.req_burstLen;
      let client = drq.client;
      let cnt = req_burstLen >> beat_shift;
      let tag <- toGet(serverTag).get;
      if(killv[drq.req_tag[5:4]] == False) begin
	 clientSelect.enq(extend(client));
	 clientData.portB.request.put(BRAMRequest{write:False, address:{tag,truncate(cnt)}, datain: ?, responseOnWrite: ?});
      end
      compCountReg <= cnt-1;
      compTagReg <= tag;
      compClientReg <= client;
      compTileReg <= drq.req_tag[5:4];
      if(verbose) $display("mkMemReadInternal::complete_burst1a %h", client);
   endrule

   rule burst_remainder if (compCountReg > 0);
      let cnt = compCountReg;
      let tag = compTagReg;
      let client = compClientReg;
      if(killv[compTileReg] == False) begin
	 clientSelect.enq(extend(client));
	 clientData.portB.request.put(BRAMRequest{write:False, address:{tag,truncate(cnt)}, datain: ?, responseOnWrite: ?});
      end
      compCountReg <= cnt-1;
      if(verbose) $display("mkMemReadInternal::complete_burst1b count %h", compCountReg);
   endrule

   Vector#(numServers, MemReadServer#(dataWidth)) sv = newVector;
   for(Integer i = 0; i < valueOf(numServers); i=i+1)
      sv[i] = (interface MemReadServer;
		  interface Put readReq;
		     method Action put(MemRequest req);
			last_loadClient <= cycle_cnt;
			let mmusel = req.sglId[31:16];
      			if (verbose) $display("mkMemReadInternal::loadClient server %d mmusel %d cycle %d", i, mmusel, cycle_cnt-last_loadClient);
			if (mmusel >= fromInteger(valueOf(numMMUs)))
			   dmaErrorFifo.enq(DmaError { errorType: DmaErrorMMUOutOfRange_r, pref: req.sglId });
   			else if (sglid_outofrange(req.sglId))
			   dmaErrorFifo.enq(DmaError { errorType: DmaErrorSGLIdOutOfRange_r, pref: req.sglId });
   			else if (stopv[req.tag[5:4]] == False) begin
   			   clientRequest.enq(LRec{req:req, client:fromInteger(i)});
   			   mmus[mmusel].request.put(ReqTup{id:truncate(req.sglId),off:req.offset});
   			end
		     endmethod
		  endinterface
		  interface Get readData;
		     method ActionValue#(MemData#(dataWidth)) get if (clientSelect.first == fromInteger(i));
			clientSelect.deq;
			let data <- clientData.portB.response.get;
			if (verbose) $display("mkMemReadInternal::comp server %d data %x cycle %d", i, data.data, cycle_cnt-last_comp);
			last_comp <= cycle_cnt;
			return data;
		     endmethod
		  endinterface
	       endinterface);

   interface servers = sv;
   interface PhysMemReadClient client;
      interface Get readReq;
	 method ActionValue#(PhysMemRequest#(addrWidth)) get();
	    let request <- toGet(serverRequest).get;
	    let req = request.req;
	    if (False && request.pa[31:24] != 0)
	       $display("mkMemReadInternal::req_ar: funny physAddr req.sglId=%d req.offset=%h physAddr=%h", req.sglId, req.offset, request.pa);
	    serverProcessing.portB.request.put(BRAMRequest{write:True, responseOnWrite:False, address:truncate(request.rename_tag),
						   datain:DRec{req_tag:req.tag, req_burstLen: req.burstLen, client:request.client, rename_tag:request.rename_tag, last:(req.burstLen == fromInteger(valueOf(dataWidthBytes)))}});
	    //$display("mkMemReadInternal::readReq: client=%d, rename_tag=%d, physAddr=%h req.burstLen=%d beat_shift=%d last=%d", request.client,request.rename_tag,request.pa, req.burstLen, beat_shift, req.burstLen == beat_shift);
	    if (verbose) $display("mkMemReadInternal::read_client.readReq %d", cycle_cnt-last_readReq);
	    last_readReq <= cycle_cnt;
	    return PhysMemRequest{addr:request.pa, burstLen:req.burstLen, tag:request.rename_tag};
	 endmethod
      endinterface
      interface Put readData;
	 method Action put(MemData#(dataWidth) response);
	    serverData.enq(response);
	    serverProcessing.portA.request.put(BRAMRequest{write:False, address:truncate(response.tag), datain: ?, responseOnWrite: ?});
	    beatCount <= beatCount+1;
	 endmethod
      endinterface
   endinterface
   interface Put tileControl;
      method Action put(TileControl tc);
	 let tile = tc.tile;
	 let kv = True;
	 let sv = True;
	 if (tc.state == Running || tc.state == Stopped)
	    kv = False;
	 if (tc.state == Running)
	    sv = False;
	 killv[tile] <= kv;
	 stopv[tile] <= sv;
      endmethod
   endinterface
   interface DmaDbg dbg;
      method ActionValue#(DmaDbgRec) dbg();
	 return DmaDbgRec{x:0, y:0, z:0, w:0};
      endmethod
      method ActionValue#(Bit#(64)) getMemoryTraffic();
	 return extend(beatCount);
      endmethod
   endinterface
endmodule

module mkMemWriteInternal#(MemServerIndication ind,
			   Vector#(numMMUs,Server#(ReqTup,Bit#(addrWidth))) mmus)
   (MemWriteInternal#(addrWidth, dataWidth, numTags, numServers))
   provisos(Log#(dataWidthBytes,beatShift)
	    ,Div#(dataWidth,8,dataWidthBytes)
	    ,Add#(beatShift, a__, 8)
	    ,Add#(b__, TLog#(numTags), 6)
	    );

   let verbose = False;

   // stopping/killing infra
   Vector#(4,Reg#(Bool)) killv <- replicateM(mkReg(False));
   Vector#(4,Reg#(Bool)) stopv <- replicateM(mkReg(False));

   // stage 0: address translation (latency = MMU_PIPELINE_DEPTH)
   FIFO#(LRec#(numServers,addrWidth)) clientRequest <- mkSizedFIFO(valueOf(MMU_PIPELINE_DEPTH));
   // stage 1: address validation (latency = 1)
   FIFO#(RRec#(numServers,addrWidth))  serverRequest <- mkFIFO;
   // stage 2: write commands
   FIFO#(DRec#(numServers, addrWidth)) serverProcessing <- mkSizedFIFO(valueOf(numTags));
   // stage 3: write data
   BRAM2Port#(Bit#(TLog#(numTags)), RResp#(numServers,addrWidth)) respFifos <- mkBRAM2Server(defaultValue);
   TagGen#(numTags) tag_gen <- mkTagGen;
   FIFO#(RResp#(numServers,addrWidth)) clientResponse <- mkFIFO;

   Reg#(Bit#(BurstLenSize)) burstReg <- mkReg(0);
   Reg#(Bool)               firstReg <- mkReg(True);
   Reg#(Bool)               lastReg <- mkReg(False);
   Reg#(Bit#(BeatCountSize)) beatCount <- mkReg(0);
   let beat_shift = fromInteger(valueOf(beatShift));

   Reg#(Bit#(64)) cycle_cnt <- mkReg(0);
   Reg#(Bit#(64)) last_loadClient <- mkReg(0);
   Reg#(Bit#(64)) last_mmuResp <- mkReg(0);

   (* fire_when_enabled *)
   rule cycle;
      cycle_cnt <= cycle_cnt+1;
   endrule

   FIFO#(DmaError) dmaErrorFifo <- mkFIFO();
   rule dmaError;
      let error <- toGet(dmaErrorFifo).get();
      ind.error(extend(pack(error.errorType)), error.pref, 0, 0);
   endrule

   rule checkMmuResp;
      let request <- toGet(clientRequest).get;
      let req = request.req;
      let client = request.client;
      let physAddr <- mmus[req.sglId[31:16]].response.get;
      let rename_tag <- tag_gen.getTag;
      serverRequest.enq(RRec{req:req, pa:physAddr, client:client, rename_tag:extend(rename_tag)});
      //if (verbose) $display("mkMemWriteInternal::checkMmuResp: client=%d, rename_tag=%d", client,rename_tag);
      if (verbose) $display("mkMemWriteInternal::mmuResp %d %d", client, cycle_cnt-last_mmuResp);
      last_mmuResp <= cycle_cnt;
   endrule

   rule writeDoneComp0;
      let tag <- tag_gen.complete;
      respFifos.portB.request.put(BRAMRequest{write:False, address:tag, datain: ?, responseOnWrite: ?});
   endrule

   FIFO#(MemData#(dataWidth)) memDataFifo <- mkFIFO();
   Vector#(numServers, FIFO#(MemData#(dataWidth))) clientWriteData <- replicateM(mkFIFO);

   if(valueOf(numServers) > 0)
      rule memdata;
	 let drq = serverProcessing.first;
	 let req_tag = drq.req_tag;
	 let req_burstLen = drq.req_burstLen;
	 let rename_tag = drq.rename_tag;
	 let client = drq.client;
	 MemData#(dataWidth) tagdata = unpack(0);
	 if (killv[req_tag[5:4]] == False) begin
	    tagdata = clientWriteData[client].first;
	    clientWriteData[client].deq;
	 end
	 let burstLen = burstReg;
	 let first    = firstReg;
	 let last     = lastReg;
	 if (first) begin
	    burstLen = req_burstLen >> beat_shift;
	    last     = serverProcessing.first.last;
	    respFifos.portA.request.put(BRAMRequest{write:True,responseOnWrite:False, address:truncate(rename_tag), datain:RResp{orig_tag:req_tag, client:client}});
	 end
	 burstReg <= burstLen-1;
	 firstReg <= (burstLen-1 == 0);
	 lastReg  <= (burstLen-1 == 1);
	 beatCount <= beatCount+1;
	 if (last)
	    serverProcessing.deq();
	 //$display("mkMemWriteInternal::writeData: client=%d, rename_tag=%d", client, rename_tag);
	 memDataFifo.enq(MemData { data: tagdata.data,  tag:extend(rename_tag), last: last });
      endrule

   rule fill_clientResponse;
      let rv <- respFifos.portB.response.get;
      clientResponse.enq(rv);
   endrule

   Vector#(numServers, MemWriteServer#(dataWidth)) sv = newVector;
   for(Integer i = 0; i < valueOf(numServers); i=i+1)
      sv[i] = (interface MemWriteServer;
		  interface Put writeReq;
		     method Action put(MemRequest req);
      			if (verbose) $display("mkMemWriteInternal::loadClient %d %d", i, cycle_cnt-last_loadClient);
			last_loadClient <= cycle_cnt;
			let mmusel = req.sglId[31:16];
			if (mmusel >= fromInteger(valueOf(numMMUs)))
			   dmaErrorFifo.enq(DmaError { errorType: DmaErrorMMUOutOfRange_w, pref: req.sglId });
   			else if (sglid_outofrange(req.sglId))
			   dmaErrorFifo.enq(DmaError { errorType: DmaErrorSGLIdOutOfRange_w, pref: req.sglId });
   			else if (stopv[req.tag[5:4]] == False) begin
   			   clientRequest.enq(LRec{req:req, client:fromInteger(i)});
   			   mmus[mmusel].request.put(ReqTup{id:truncate(req.sglId),off:req.offset});
   			end
		     endmethod
		  endinterface
		  interface Put writeData = toPut(clientWriteData[i]);
		  interface Get writeDone;
		     method ActionValue#(Bit#(MemTagSize)) get if (clientResponse.first.client == fromInteger(i));
			clientResponse.deq;
			return clientResponse.first.orig_tag;
		     endmethod
		  endinterface
	       endinterface);

   interface servers = sv;
   interface PhysMemWriteClient client;
      interface Get writeReq;
	 method ActionValue#(PhysMemRequest#(addrWidth)) get();
	    let request <- toGet(serverRequest).get();
	    let req = request.req;
	    let physAddr = request.pa;
	    let client = request.client;
	    let rename_tag = request.rename_tag;
	    serverProcessing.enq(DRec{req_tag:req.tag, req_burstLen: req.burstLen, client:client, rename_tag:rename_tag, last: (req.burstLen == fromInteger(valueOf(dataWidthBytes))) });
	    //$display("mkMemWriteInternal::writeReq: client=%d, rename_tag=%d", client,rename_tag);
	    return PhysMemRequest{addr:physAddr, burstLen:req.burstLen, tag:extend(rename_tag)};
	 endmethod
      endinterface
      interface Get writeData = toGet(memDataFifo);
      interface Put writeDone;
	 method Action put(Bit#(MemTagSize) resp);
	    tag_gen.returnTag(truncate(resp));
	    if (verbose) $display("mkMemWriteInternal::writeDone: resp=%d", resp);
	 endmethod
      endinterface
   endinterface
   interface Put tileControl;
      method Action put(TileControl tc);
	 let tile = tc.tile;
	 let kv = True;
	 let sv = True;
	 if (tc.state == Running || tc.state == Stopped)
	    kv = False;
	 if (tc.state == Running)
	    sv = False;
	 killv[tile] <= kv;
	 stopv[tile] <= sv;
      endmethod
   endinterface
   interface DmaDbg dbg;
      method ActionValue#(DmaDbgRec) dbg();
	 return DmaDbgRec{x:fromInteger(valueOf(numServers)), y:?, z:?, w:?};
      endmethod
      method ActionValue#(Bit#(64)) getMemoryTraffic();
	 return extend(beatCount);
      endmethod
   endinterface
endmodule

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

// BSV Libraries
import BRAMFIFO::*;
import FIFO::*;
import FIFOF::*;
import Vector::*;
import GetPut::*;
import ClientServer::*;
import Assert::*;
import BRAM::*;
import DefaultValue::*;

// CONNECTAL Libraries
import MemTypes::*;
import ConnectalMemory::*;
import ConnectalClocks::*;
import MMU::*;
import ConnectalCompletionBuffer::*;

typedef 32 BeatCountSize;

typedef 9 MMU_PIPELINE_DEPTH;

interface DmaDbg;
   method ActionValue#(Bit#(64)) getMemoryTraffic();
   method ActionValue#(DmaDbgRec) dbg();
endinterface

interface MemWriteInternal#(numeric type addrWidth, numeric type dataWidth, numeric type numTags, numeric type numServers);
   interface DmaDbg dbg;
   interface Put#(TileControl) tileControl;
   interface PhysMemWriteClient#(addrWidth,dataWidth) client;
   interface Vector#(numServers, MemWriteServer#(dataWidth)) servers;
endinterface

interface MemReadInternal#(numeric type addrWidth, numeric type dataWidth, numeric type numTags, numeric type numServers);
   interface DmaDbg dbg;
   interface Put#(TileControl) tileControl;
   interface PhysMemReadClient#(addrWidth,dataWidth) client;
   interface Vector#(numServers, MemReadServer#(dataWidth)) servers;
endinterface

function Bool sglid_outofrange(SGLId p);
   return ((p[15:0]) >= fromInteger(valueOf(MaxNumSGLists)));
endfunction

typedef struct {MemRequest req;
		Bit#(TLog#(TMax#(1,numClients))) client; } LRec#(numeric type numClients, numeric type addrWidth) deriving(Bits);

typedef struct {MemRequest req;
		Bit#(addrWidth) pa;
		Bit#(MemTagSize) rename_tag;
		Bit#(TLog#(TMax#(1,numClients))) client; } RRec#(numeric type numClients, numeric type addrWidth) deriving(Bits);

typedef struct {Bit#(MemTagSize) req_tag;
		Bit#(BurstLenSize) req_burstLen;
		Bit#(MemTagSize) rename_tag;
		Bit#(TLog#(TMax#(1,numClients))) client;
		Bool last;
   } DRec#(numeric type numClients, numeric type addrWidth) deriving(Bits);

typedef struct {Bit#(MemTagSize) orig_tag;
		Bit#(TLog#(TMax#(1,numClients))) client; } RResp#(numeric type numClients, numeric type addrWidth) deriving(Bits);

typedef struct {DmaErrorType errorType;
		Bit#(32) pref; } DmaError deriving (Bits);

module mkMemReadInternal#(MemServerIndication ind,
			  Vector#(numMMUs,Server#(ReqTup,Bit#(addrWidth))) mmus)
   (MemReadInternal#(addrWidth, dataWidth, numTags, numServers))
   provisos(Log#(dataWidthBytes,beatShift)
	    ,Div#(dataWidth,8,dataWidthBytes)
	    ,Add#(beatShift, a__, 8)
	    ,Add#(b__, TLog#(numTags), 6)
	    );

   // stopping/killing infra
   Vector#(4,Reg#(Bool)) killv <- replicateM(mkReg(False));
   Vector#(4,Reg#(Bool)) stopv <- replicateM(mkReg(False));

   // stage 0: address translation (latency = MMU_PIPELINE_DEPTH)
   FIFO#(LRec#(numServers,addrWidth)) clientRequest <- mkSizedFIFO(valueOf(MMU_PIPELINE_DEPTH));
   // stage 1: address validation (latency = 1)
   FIFO#(RRec#(numServers,addrWidth))  serverRequest <- mkFIFO;
   // stage 2: read commands
   BRAM_Configure bramConfig = defaultValue;
   if (mainClockPeriod < 8)
      bramConfig.latency = 2;
   BRAM2Port#(Bit#(TLog#(numTags)), DRec#(numServers,addrWidth)) serverProcessing <- mkBRAM2Server(bramConfig);
   BRAM2Port#(Bit#(TAdd#(TLog#(numTags),TSub#(BurstLenSize,beatShift))), MemData#(dataWidth)) clientData <- mkBRAM2Server(bramConfig);
   // stage 3: read data
   FIFO#(MemData#(dataWidth)) serverData <- mkFIFO;

   let verbose = False;

   Reg#(Bit#(BurstLenSize)) burstReg <- mkReg(0);
   Reg#(Bool)               firstReg <- mkReg(True);
   Reg#(Bool)                lastReg <- mkReg(False);

   Reg#(Bit#(32))  beatCount <- mkReg(0);
   let beat_shift = fromInteger(valueOf(beatShift));
   TagGen#(numTags) tag_gen <- mkTagGen;

   Reg#(Bit#(BurstLenSize))      compCountReg <- mkReg(0);
   Reg#(Bit#(TLog#(numTags)))    compTagReg <- mkReg(0);
   Reg#(Bit#(TLog#(TMax#(1,numServers)))) compClientReg <- mkReg(0);
   Reg#(Bit#(2))                 compTileReg <- mkReg(0);
   FIFO#(Bit#(TAdd#(1,TLog#(TMax#(1,numServers))))) clientSelect <- mkFIFO;
   FIFO#(Bit#(TLog#(numTags)))   serverTag <- mkFIFO;

   // performance analytics
   Reg#(Bit#(64)) cycle_cnt <- mkReg(0);
   Reg#(Bit#(64)) last_loadClient <- mkReg(0);
   Reg#(Bit#(64)) last_mmuResp <- mkReg(0);
   Reg#(Bit#(64)) last_comp <- mkReg(0);
   Reg#(Bit#(64)) last_readReq <- mkReg(0);
   Reg#(Bit#(64)) last_readData <- mkReg(0);
   (* fire_when_enabled *)
   rule cycle;
      cycle_cnt <= cycle_cnt+1;
   endrule

   FIFO#(DmaError) dmaErrorFifo <- mkFIFO();
   rule dmaError;
      let error <- toGet(dmaErrorFifo).get();
      ind.error(extend(pack(error.errorType)), error.pref, 0, 0);
   endrule

   rule complete_burst0;
      let tag <- tag_gen.complete;
      serverProcessing.portB.request.put(BRAMRequest{write:False, address:tag, datain: ?, responseOnWrite: ?});
      serverTag.enq(tag);
      if(verbose) $display("mkMemReadInternal::complete_burst0 %h", tag);
   endrule

   rule complete_burst1a if (compCountReg==0);
      let drq <- serverProcessing.portB.response.get;
      let req_burstLen = drq.req_burstLen;
      let client = drq.client;
      let cnt = req_burstLen >> beat_shift;
      let tag <- toGet(serverTag).get;
      if(killv[drq.req_tag[5:4]] == False) begin
	 clientSelect.enq(extend(client));
	 clientData.portB.request.put(BRAMRequest{write:False, address:{tag,truncate(cnt)}, datain: ?, responseOnWrite: ?});
      end
      compCountReg <= cnt-1;
      compTagReg <= tag;
      compClientReg <= client;
      compTileReg <= drq.req_tag[5:4];
      if(verbose) $display("mkMemReadInternal::complete_burst1a %h", client);
   endrule

   rule complete_burst1b if (compCountReg > 0);
      let cnt = compCountReg;
      let tag = compTagReg;
      let client = compClientReg;
      if(killv[compTileReg] == False) begin
	 clientSelect.enq(extend(client));
	 clientData.portB.request.put(BRAMRequest{write:False, address:{tag,truncate(cnt)}, datain: ?, responseOnWrite: ?});
      end
      compCountReg <= cnt-1;
      if(verbose) $display("mkMemReadInternal::complete_burst1b count %h", compCountReg);
   endrule

   rule checkMmuResp;
      let req = clientRequest.first.req;
      let client = clientRequest.first.client;
      let physAddr <- mmus[req.sglId[31:16]].response.get;
      let rename_tag <- tag_gen.getTag;
      clientRequest.deq();
      serverRequest.enq(RRec{req:req, pa:physAddr, client:client, rename_tag:extend(rename_tag)});
      if (verbose) $display("mkMemReadInternal::checkMmuResp: client=%d, rename_tag=%d", client,rename_tag);
      if (verbose) $display("mkMemReadInternal::mmuResp %d %d", client, cycle_cnt-last_mmuResp);
      last_mmuResp <= cycle_cnt;
   endrule

   rule read_data;
      let response <- toGet(serverData).get();
      Bit#(MemTagSize) response_tag = response.tag;
      let drq <- serverProcessing.portA.response.get;
      let otag = drq.req_tag;
      let burstLen = burstReg;
      let first =    firstReg;
      let last  =    lastReg;
      if (first) begin
	 burstLen = drq.req_burstLen >> beat_shift;
	 last = drq.last;
	 dynamicAssert(last == (burstLen==1), "Last incorrect");
      end
      Bit#(TLog#(numTags)) tt = truncate(response_tag);
      clientData.portA.request.put(BRAMRequest{write:True, responseOnWrite:False, datain:MemData{data: response.data, tag: otag, last: last}, address:{tt,truncate(burstLen)}});
      if (last) begin
	 tag_gen.returnTag(truncate(response_tag));
      end
      last_readData <= cycle_cnt;
      if (verbose) $display("mkMemReadInternal::read_data cyclediff %d", cycle_cnt-last_readData);
      burstReg <= burstLen-1;
      firstReg <= burstLen-1 == 0;
      lastReg  <= burstLen-1 == 1;
   endrule

   Vector#(numServers, MemReadServer#(dataWidth)) sv = newVector;
   for(Integer i = 0; i < valueOf(numServers); i=i+1)
      sv[i] = (interface MemReadServer;
		  interface Put readReq;
		     method Action put(MemRequest req);
			last_loadClient <= cycle_cnt;
			let mmusel = req.sglId[31:16];
      			if (verbose) $display("mkMemReadInternal::loadClient server %d mmusel %d cycle %d", i, mmusel, cycle_cnt-last_loadClient);
			if (mmusel >= fromInteger(valueOf(numMMUs)))
			   dmaErrorFifo.enq(DmaError { errorType: DmaErrorMMUOutOfRange_r, pref: req.sglId });
   			else if (sglid_outofrange(req.sglId))
			   dmaErrorFifo.enq(DmaError { errorType: DmaErrorSGLIdOutOfRange_r, pref: req.sglId });
   			else if (stopv[req.tag[5:4]] == False) begin
   			   clientRequest.enq(LRec{req:req, client:fromInteger(i)});
   			   mmus[mmusel].request.put(ReqTup{id:truncate(req.sglId),off:req.offset});
   			end
		     endmethod
		  endinterface
		  interface Get readData;
		     method ActionValue#(MemData#(dataWidth)) get if (clientSelect.first == fromInteger(i));
			clientSelect.deq;
			let data <- clientData.portB.response.get;
			if (verbose) $display("mkMemReadInternal::comp server %d data %x cycle %d", i, data.data, cycle_cnt-last_comp);
			last_comp <= cycle_cnt;
			return data;
		     endmethod
		  endinterface
	       endinterface);

   interface servers = sv;
   interface PhysMemReadClient client;
      interface Get readReq;
	 method ActionValue#(PhysMemRequest#(addrWidth)) get();
	    serverRequest.deq;
	    let req = serverRequest.first.req;
	    let physAddr = serverRequest.first.pa;
	    let client = serverRequest.first.client;
	    let rename_tag = serverRequest.first.rename_tag;
	    if (False && physAddr[31:24] != 0)
	       $display("mkMemReadInternal::req_ar: funny physAddr req.sglId=%d req.offset=%h physAddr=%h", req.sglId, req.offset, physAddr);
	    serverProcessing.portB.request.put(BRAMRequest{write:True, responseOnWrite:False, address:truncate(rename_tag),
						   datain:DRec{req_tag:req.tag, req_burstLen: req.burstLen, client:client, rename_tag:rename_tag, last:(req.burstLen == fromInteger(valueOf(dataWidthBytes)))}});
	    //$display("mkMemReadInternal::readReq: client=%d, rename_tag=%d, physAddr=%h req.burstLen=%d beat_shift=%d last=%d", client,rename_tag,physAddr, req.burstLen, beat_shift, req.burstLen == beat_shift);
	    if (verbose) $display("mkMemReadInternal::read_client.readReq %d", cycle_cnt-last_readReq);
	    last_readReq <= cycle_cnt;
	    return PhysMemRequest{addr:physAddr, burstLen:req.burstLen, tag:rename_tag};
	 endmethod
      endinterface
      interface Put readData;
	 method Action put(MemData#(dataWidth) response);
	    serverData.enq(response);
	    serverProcessing.portA.request.put(BRAMRequest{write:False, address:truncate(response.tag), datain: ?, responseOnWrite: ?});
	    beatCount <= beatCount+1;
	 endmethod
      endinterface
   endinterface
   interface Put tileControl;
      method Action put(TileControl tc);
	 let tile = tc.tile;
	 let kv = True;
	 let sv = True;
	 if (tc.state == Running || tc.state == Stopped)
	    kv = False;
	 if (tc.state == Running)
	    sv = False;
	 killv[tile] <= kv;
	 stopv[tile] <= sv;
      endmethod
   endinterface
   interface DmaDbg dbg;
      method ActionValue#(DmaDbgRec) dbg();
	 return DmaDbgRec{x:0, y:0, z:0, w:0};
      endmethod
      method ActionValue#(Bit#(64)) getMemoryTraffic();
	 return extend(beatCount);
      endmethod
   endinterface
endmodule

module mkMemWriteInternal#(MemServerIndication ind,
			   Vector#(numMMUs,Server#(ReqTup,Bit#(addrWidth))) mmus)
   (MemWriteInternal#(addrWidth, dataWidth, numTags, numServers))
   provisos(Log#(dataWidthBytes,beatShift)
	    ,Div#(dataWidth,8,dataWidthBytes)
	    ,Add#(beatShift, a__, 8)
	    ,Add#(b__, TLog#(numTags), 6)
	    );

   let verbose = False;

   // stopping/killing infra
   Vector#(4,Reg#(Bool)) killv <- replicateM(mkReg(False));
   Vector#(4,Reg#(Bool)) stopv <- replicateM(mkReg(False));

   // stage 0: address translation (latency = MMU_PIPELINE_DEPTH)
   FIFO#(LRec#(numServers,addrWidth)) clientRequest <- mkSizedFIFO(valueOf(MMU_PIPELINE_DEPTH));
   // stage 1: address validation (latency = 1)
   FIFO#(RRec#(numServers,addrWidth))  serverRequest <- mkFIFO;
   // stage 2: write commands
   FIFO#(DRec#(numServers, addrWidth)) serverProcessing <- mkSizedFIFO(valueOf(numTags));
   // stage 3: write data
   BRAM2Port#(Bit#(TLog#(numTags)), RResp#(numServers,addrWidth)) respFifos <- mkBRAM2Server(defaultValue);
   TagGen#(numTags) tag_gen <- mkTagGen;
   FIFO#(RResp#(numServers,addrWidth)) clientResponse <- mkFIFO;

   Reg#(Bit#(BurstLenSize)) burstReg <- mkReg(0);
   Reg#(Bool)               firstReg <- mkReg(True);
   Reg#(Bool)               lastReg <- mkReg(False);
   Reg#(Bit#(BeatCountSize)) beatCount <- mkReg(0);
   let beat_shift = fromInteger(valueOf(beatShift));

   Reg#(Bit#(64)) cycle_cnt <- mkReg(0);
   Reg#(Bit#(64)) last_loadClient <- mkReg(0);
   Reg#(Bit#(64)) last_mmuResp <- mkReg(0);

   (* fire_when_enabled *)
   rule cycle;
      cycle_cnt <= cycle_cnt+1;
   endrule

   FIFO#(DmaError) dmaErrorFifo <- mkFIFO();
   rule dmaError;
      let error <- toGet(dmaErrorFifo).get();
      ind.error(extend(pack(error.errorType)), error.pref, 0, 0);
   endrule

   rule checkMmuResp;
      let req = clientRequest.first.req;
      let client = clientRequest.first.client;
      let physAddr <- mmus[req.sglId[31:16]].response.get;
      let rename_tag <- tag_gen.getTag;
      clientRequest.deq();
      serverRequest.enq(RRec{req:req, pa:physAddr, client:client, rename_tag:extend(rename_tag)});
      //if (verbose) $display("mkMemWriteInternal::checkMmuResp: client=%d, rename_tag=%d", client,rename_tag);
      if (verbose) $display("mkMemWriteInternal::mmuResp %d %d", client, cycle_cnt-last_mmuResp);
      last_mmuResp <= cycle_cnt;
   endrule

   rule writeDoneComp0;
      let tag <- tag_gen.complete;
      respFifos.portB.request.put(BRAMRequest{write:False, address:tag, datain: ?, responseOnWrite: ?});
   endrule

   FIFO#(MemData#(dataWidth)) memDataFifo <- mkFIFO();
   Vector#(numServers, FIFO#(MemData#(dataWidth))) clientWriteData <- replicateM(mkFIFO);

   if(valueOf(numServers) > 0)
      rule memdata;
	 let drq = serverProcessing.first;
	 let req_tag = drq.req_tag;
	 let req_burstLen = drq.req_burstLen;
	 let rename_tag = drq.rename_tag;
	 let client = drq.client;
	 MemData#(dataWidth) tagdata = unpack(0);
	 if (killv[req_tag[5:4]] == False) begin
	    tagdata = clientWriteData[client].first;
	    clientWriteData[client].deq;
	 end
	 let burstLen = burstReg;
	 let first    = firstReg;
	 let last     = lastReg;
	 if (first) begin
	    burstLen = req_burstLen >> beat_shift;
	    last     = serverProcessing.first.last;
	    respFifos.portA.request.put(BRAMRequest{write:True,responseOnWrite:False, address:truncate(rename_tag), datain:RResp{orig_tag:req_tag, client:client}});
	 end
	 burstReg <= burstLen-1;
	 firstReg <= (burstLen-1 == 0);
	 lastReg  <= (burstLen-1 == 1);
	 beatCount <= beatCount+1;
	 if (last)
	    serverProcessing.deq();
	 //$display("mkMemWriteInternal::writeData: client=%d, rename_tag=%d", client, rename_tag);
	 memDataFifo.enq(MemData { data: tagdata.data,  tag:extend(rename_tag), last: last });
      endrule

   rule fill_clientResponse;
      let rv <- respFifos.portB.response.get;
      clientResponse.enq(rv);
   endrule

   Vector#(numServers, MemWriteServer#(dataWidth)) sv = newVector;
   for(Integer i = 0; i < valueOf(numServers); i=i+1)
      sv[i] = (interface MemWriteServer;
		  interface Put writeReq;
		     method Action put(MemRequest req);
      			if (verbose) $display("mkMemWriteInternal::loadClient %d %d", i, cycle_cnt-last_loadClient);
			last_loadClient <= cycle_cnt;
			let mmusel = req.sglId[31:16];
			if (mmusel >= fromInteger(valueOf(numMMUs)))
			   dmaErrorFifo.enq(DmaError { errorType: DmaErrorMMUOutOfRange_w, pref: req.sglId });
   			else if (sglid_outofrange(req.sglId))
			   dmaErrorFifo.enq(DmaError { errorType: DmaErrorSGLIdOutOfRange_w, pref: req.sglId });
   			else if (stopv[req.tag[5:4]] == False) begin
   			   clientRequest.enq(LRec{req:req, client:fromInteger(i)});
   			   mmus[mmusel].request.put(ReqTup{id:truncate(req.sglId),off:req.offset});
   			end
		     endmethod
		  endinterface
		  interface Put writeData = toPut(clientWriteData[i]);
		  interface Get writeDone;
		     method ActionValue#(Bit#(MemTagSize)) get if (clientResponse.first.client == fromInteger(i));
			clientResponse.deq;
			return clientResponse.first.orig_tag;
		     endmethod
		  endinterface
	       endinterface);

   interface servers = sv;
   interface PhysMemWriteClient client;
      interface Get writeReq;
	 method ActionValue#(PhysMemRequest#(addrWidth)) get();
	    let req = serverRequest.first.req;
	    let physAddr = serverRequest.first.pa;
	    let client = serverRequest.first.client;
	    let rename_tag = serverRequest.first.rename_tag;
	    serverRequest.deq;
	    serverProcessing.enq(DRec{req_tag:req.tag, req_burstLen: req.burstLen, client:client, rename_tag:rename_tag, last: (req.burstLen == fromInteger(valueOf(dataWidthBytes))) });
	    //$display("mkMemWriteInternal::writeReq: client=%d, rename_tag=%d", client,rename_tag);
	    return PhysMemRequest{addr:physAddr, burstLen:req.burstLen, tag:extend(rename_tag)};
	 endmethod
      endinterface
      interface Get writeData = toGet(memDataFifo);
      interface Put writeDone;
	 method Action put(Bit#(MemTagSize) resp);
	    tag_gen.returnTag(truncate(resp));
	    if (verbose) $display("mkMemWriteInternal::writeDone: resp=%d", resp);
	 endmethod
      endinterface
   endinterface
   interface Put tileControl;
      method Action put(TileControl tc);
	 let tile = tc.tile;
	 let kv = True;
	 let sv = True;
	 if (tc.state == Running || tc.state == Stopped)
	    kv = False;
	 if (tc.state == Running)
	    sv = False;
	 killv[tile] <= kv;
	 stopv[tile] <= sv;
      endmethod
   endinterface
   interface DmaDbg dbg;
      method ActionValue#(DmaDbgRec) dbg();
	 return DmaDbgRec{x:fromInteger(valueOf(numServers)), y:?, z:?, w:?};
      endmethod
      method ActionValue#(Bit#(64)) getMemoryTraffic();
	 return extend(beatCount);
      endmethod
   endinterface
endmodule