added ifdef MTABLE ... endif around matchtable code

// Copyright (c) 2015 Cornell University.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import BRAMFIFO::*;
import BuildVector::*;
import Clocks::*;
import DefaultValue::*;
import FIFO::*;
import FIFOF::*;
import GetPut::*;
import ClientServer::*;
import Vector::*;
import Connectable::*;
import StmtFSM::*;
import Pipe::*;
import HostInterface::*;
import MemTypes::*;
import MemReadEngine::*;
import MemWriteEngine::*;
import MemServerIndication::*;
import MMUIndication::*;
`ifdef MTABLE
import MatchTable::*;
import MatchTableTypes::*;
`endif

import AlteraExtra::*;
import AlteraEthPhy::*;
import EthMac::*;
import ALTERA_SI570_WRAPPER          ::*;
import ALTERA_EDGE_DETECTOR_WRAPPER  ::*;
import ConnectalClocks::*;
import LedController::*;
import Leds::*;
import PushButtonController::*;

import Ethernet::*;
import IngressPipeline::*;
import PacketBuffer::*;
import Parser::*;
import Types::*;
import SharedBuff::*;
import `PinTypeInclude::*;

typedef TDiv#(DataBusWidth, 32) WordsPerBeat;

interface P4TopIndication;
   method Action sonic_read_version_resp(Bit#(32) version);
`ifdef MTABLE
   method Action matchTableResponse(Bit#(32) key, Bit#(32) value);
`endif
endinterface

interface P4TopRequest;
   method Action sonic_read_version();
   method Action writePacketData(Vector#(2, Bit#(64)) data, Bit#(1) sop, Bit#(1) eop);
   method Action readPacketBuffer(Bit#(16) addr);
   method Action writePacketBuffer(Bit#(16) addr, Bit#(64) data);
`ifdef MTABLE
   method Action matchTableRequest(Bit#(32) key, Bit#(32) value, Bit#(32) op);
`endif
   method Action port_mapping_add_entry(Bit#(32) table_name, MatchInput_port_mapping match_key);
   method Action port_mapping_set_default_action(Bit#(32) table_name);
   method Action port_mapping_delete_entry(Bit#(32) table_name, Bit#(32) id);
   method Action port_mapping_modify_entry(Bit#(32) table_name, Bit#(32) id, ActionInput_port_mapping actions);
   method Action bd_add_entry(Bit#(32) table_name, MatchInput_bd match_key);
   method Action bd_set_default_action(Bit#(32) table_name);
   method Action bd_delete_entry(Bit#(32) table_name, Bit#(32) id);
   method Action bd_modify_entry(Bit#(32) table_name, Bit#(32) id, ActionInput_bd actions);
   method Action ipv4_fib_add_entry(Bit#(32) table_name, MatchInput_ipv4_fib match_key);
   method Action ipv4_fib_set_default_action(Bit#(32) table_name);
   method Action ipv4_fib_delete_entry(Bit#(32) table_name, Bit#(32) id);
   method Action ipv4_fib_modify_entry(Bit#(32) table_name, Bit#(32) id, ActionInput_ipv4_fib actions);
   method Action ipv4_fib_lpm_add_entry(Bit#(32) table_name, MatchInput_ipv4_fib_lpm match_key);
   method Action ipv4_fib_lpm_set_default_action(Bit#(32) table_name);
   method Action ipv4_fib_lpm_delete_entry(Bit#(32) table_name, Bit#(32) id);
   method Action ipv4_fib_lpm_modify_entry(Bit#(32) table_name, Bit#(32) id, ActionInput_ipv4_fib_lpm actions);
   method Action nexthop_add_entry(Bit#(32) table_name, MatchInput_nexthop match_key);
   method Action nexthop_set_default_action(Bit#(32) table_name);
   method Action nexthop_delete_entry(Bit#(32) table_name, Bit#(32) id);
   method Action nexthop_modify_entry(Bit#(32) table_name, Bit#(32) id, ActionInput_nexthop actions);
   method Action rewrite_mac_add_entry(Bit#(32) table_name, MatchInput_rewrite_mac match_key);
   method Action rewrite_mac_set_default_action(Bit#(32) table_name);
   method Action rewrite_mac_delete_entry(Bit#(32) table_name, Bit#(32) id);
   method Action rewrite_mac_modify_entry(Bit#(32) table_name, Bit#(32) id, ActionInput_rewrite_mac actions);
endinterface

interface P4Top;
   interface P4TopRequest request;
   interface `PinType pins;
endinterface

//module mkP4Top#(Clock derivedClock, Reset derivedReset, P4TopIndication indication)(P4Top);
module mkP4Top#(P4TopIndication indication)(P4Top);
   Clock defaultClock <- exposeCurrentClock();
   Reset defaultReset <- exposeCurrentReset();

`ifndef BSIM
   B2C iclock_50 <- mkB2C();
   B2C1 iclock_644 <- mkB2C1();
//   B2C iclock_pcie <- mkB2C();

   Vector#(4, PushButtonController) buttons <- replicateM(mkPushButtonController(iclock_50.c, clocked_by iclock_50.c, reset_by iclock_50.r));

   Reset rst_644   <- mkResetInverter(iclock_50.r, clocked_by iclock_644.c);
   Reset rst_644_n <- mkAsyncReset(2, iclock_50.r, iclock_644.c);
   // PLL Input:   SFP REFCLK from SI570
   // PLL Output:  156.25MHz
   // PLL Reset:   Active High, must invert default Reset
   PLL156 pll156 <- mkPLL156(iclock_644.c, rst_644, clocked_by iclock_644.c, reset_by rst_644);
   Clock clk_156_25 = pll156.outclk_0;
   Reset rst_156   <- mkResetInverter(iclock_50.r, clocked_by clk_156_25);
   Reset rst_156_n <- mkAsyncReset(1, rst_156, clk_156_25);
   Si570Wrap si570 <- mkSi570Wrap(iclock_50.c, iclock_50.r, clocked_by iclock_50.c, reset_by iclock_50.r);
   EdgeDetectorWrap edgedetect <- mkEdgeDetectorWrap(iclock_50.c, iclock_50.r, clocked_by iclock_50.c, reset_by iclock_50.r);

   rule si570_connections;
      let ifreq_mode = 3'b110;  //644.53125 MHZ
      si570.ifreq.mode(ifreq_mode);
      si570.istart.go(edgedetect.odebounce.out);
   endrule

   rule button_to_iclock50_r;
      iclock_50.inputreset(pack(buttons[0]));
   endrule

   rule button_to_si570;
      edgedetect.itrigger.in(pack(buttons[1]));
   endrule

//   Reset xcvr_reset_n <- mkAsyncReset(2, iclock_50.r, eth.ifcs.clk_xcvr[0]);

//   LedController pci_led <- mkLedController(False, clocked_by iclock_pcie.c, reset_by iclock_pcie.r);
//   LedController eth_tx_led <- mkLedController(False, clocked_by clk_156_25, reset_by rst_156_n);
//   LedController eth_rx_led <- mkLedController(False, clocked_by eth.ifcs.clk_xcvr[0], reset_by xcvr_reset_n);
   LedController sfp_led <- mkLedController(False, clocked_by iclock_644.c, reset_by rst_644_n);

//   rule led_pcie;
//      pci_led.setPeriod(led_off, 500, led_on_max, 500);
//   endrule
//   rule led_eth_tx;
//      eth_tx_led.setPeriod(led_off, 500, led_on_max, 500);
//   endrule
//   rule led_eth_rx;
//      eth_rx_led.setPeriod(led_off, 500, led_on_max, 500);
//   endrule
   rule led_sfp;
      sfp_led.setPeriod(led_off, 500, led_on_max, 500);
   endrule

   Vector#(4, LedController) led_xcvr_ <- replicateM(mkLedController(False, clocked_by iclock_50.c, reset_by iclock_50.r));
   Led#(4) led_xcvr = combineLeds(led_xcvr_);

   rule led_xcvr_ready;
      for (Integer i = 0; i<4; i=i+1) begin
         led_xcvr_[i].setPeriod(led_off, 500, led_on_max, 500);
      end
   endrule

//   EthPhyIfc phy <- mkAlteraEthPhy(clk_50_b4a_buf.outclk, iclock_644.c, clk_156_25, iclock_50.r, clocked_by clk_156_25, reset_by rst_156_n);
//   Clock xgmii_rx_clk = phy.rx_clkout;
//   EthMacIfc mac <- mkEthMac(clk_50_b4a_buf.outclk, clk_156_25, replicate(xgmii_rx_clk), rst_156_n, clocked_by clk_156_25, reset_by rst_156_n);
//   mapM(uncurry(mkConnection), zip(mac.tx, phy.tx));
//   mapM(uncurry(mkConnection), zip(phy.rx, mac.rx));

`endif

   let verbose = True;
   Reg#(Cycle_t) cycle <- mkReg(defaultValue);
   rule every1 if (verbose);
      cycle.cnt <= cycle.cnt + 1;
   endrule

   Reg#(Standard_metadata_t) standard_metadata <- mkReg(defaultValue);
   Reg#(Ingress_metadata_t) ingress_metadata <- mkReg(defaultValue);

   PacketBuffer rxPktBuff <- mkPacketBuffer();
   Parser parser <- mkParser();
   Pipeline_port_mapping ingress_port_mapping <- mkIngressPipeline_port_mapping();

`ifdef MTABLE
   /* Match Table Functionalities */
   function RequestType makeRequest(Bit#(32) key, Bit#(32) value, Operation op);
       return RequestType {
           key : key,
           value : value,
           addrIdx : 0,
           op : op
       };
   endfunction

   Server#(RequestType, ResponseType) matchTable <- mkMatchTable();
`endif

   // read client interface
   FIFO#(MemRequest) reqFifo <-mkSizedFIFO(4);
   FIFO#(MemData#(DataBusWidth)) dataFifo <- mkSizedFIFO(32);
   MemReadClient#(DataBusWidth) dmaClient = (interface MemReadClient;
      interface Get readReq = toGet(reqFifo);
      interface Put readData = toPut(dataFifo);
   endinterface);

   // write client interface
   FIFO#(MemRequest) writeReqFifo <- mkSizedFIFO(4);
   FIFO#(MemData#(DataBusWidth)) writeDataFifo <- mkSizedFIFO(32);
   FIFO#(Bit#(MemTagSize)) writeDoneFifo <- mkSizedFIFO(4);
   MemWriteClient#(DataBusWidth) dmaWriteClient = (interface MemWriteClient;
      interface Get writeReq = toGet(writeReqFifo);
      interface Get writeData = toGet(writeDataFifo);
      interface Put writeDone = toPut(writeDoneFifo);
   endinterface);

   MemServerIndicationOutput memServerIndication <- mkMemServerIndicationOutput;
   MMUIndicationOutput mmuIndication <- mkMMUIndicationOutput;
//   SharedBuffer#(12, 128, 1) buff <- mkSharedBuffer(vec(dmaClient), vec(dmaWriteClient), memServerIndication.ifc, mmuIndication.ifc);

   Reg#(Bit#(EtherLen)) pktLen <- mkReg(0);
   Reg#(Bit#(9)) rAddr_wires <- mkReg(0);

   rule packetParseStart;
      let pktLen <- rxPktBuff.readServer.readLen.get;
      rxPktBuff.readServer.readReq.put(EtherReq{len: truncate(pktLen)});
      if (verbose) $display(fshow(cycle) + fshow(" read packt ") + fshow(pktLen));
   endrule

   rule packetParseInProgress;
      let v <- rxPktBuff.readServer.readData.get;
      if (verbose) $display(fshow(cycle) + fshow(" load packet ") + fshow(v));
      parser.frameIn.enq(v);
      if (v.eop) begin
         if (verbose) $display(fshow(cycle) + fshow(" eop."));
      end
   endrule

   rule matchTableStart;
      let v <- toGet(parser.parseDone).get;
      $display("Parse Done");
      parser.parserReset();
   endrule

   rule forwardPayload;
      let v <- toGet(parser.payloadOut).get;
   endrule

   rule readData;
      let v <- toGet(dataFifo).get;
      $display("%d: Received data %x", cycle, v.data);
   endrule

   rule writeDone;
      let v <- toGet(writeDoneFifo).get;
      $display("%d: Write done", cycle);
   endrule
   //mkConnection(parser.phvOut, ingress_port_mapping.phvIn);

`ifdef MTABLE
   rule matchTableRes;
       let res <- matchTable.response.get;
       indication.matchTableResponse(res.key, res.value);
   endrule
`endif

   interface P4TopRequest request;
      method Action sonic_read_version();
         let v= `NicVersion;
         indication.sonic_read_version_resp(v);
      endmethod
      method Action writePacketData(Vector#(2, Bit#(64)) data, Bit#(1) sop, Bit#(1) eop);
         EtherData beat = defaultValue;
         beat.data = pack(reverse(data));
         beat.sop = unpack(sop);
         beat.eop = unpack(eop);
         rxPktBuff.writeServer.writeData.put(beat);
      endmethod

      method Action readPacketBuffer(Bit#(16) addr);
         Bit#(ByteEnableSize) firstbe = 'hff;
         Bit#(ByteEnableSize) lastbe = 'hff;
         reqFifo.enq(MemRequest {sglId: 0, offset: 0, burstLen: 16, tag:0, firstbe: firstbe, lastbe: lastbe});
      endmethod

      method Action writePacketBuffer(Bit#(16) addr, Bit#(64) data);
         Bit#(ByteEnableSize) firstbe = 'hff;
         Bit#(ByteEnableSize) lastbe = 'hff;
         writeReqFifo.enq(MemRequest {sglId:0, offset:0, burstLen:16, tag:0, firstbe: firstbe, lastbe: lastbe});

         function Bit#(32) plusi(Integer i); return fromInteger(i); endfunction
         Vector#(WordsPerBeat, Bit#(32)) v = genWith(plusi);
         writeDataFifo.enq(MemData {data: pack(v), tag:0, last:True});
      endmethod

`ifdef MTABLE
      method Action matchTableRequest(Bit#(32) key, Bit#(32) value, Bit#(32) op);
        if (op == 0)
            matchTable.request.put(makeRequest(key, value, GET));
        else if (op == 1)
            matchTable.request.put(makeRequest(key, value, PUT));
        else if (op == 2)
            matchTable.request.put(makeRequest(key, value, UPDATE));
        else if (op == 3)
            matchTable.request.put(makeRequest(key, value, REMOVE));
      endmethod
`endif

      // Generate fixed standard set of API function
      // In software, we should hide the details of match table different behind api
      // expose thrift-server to software to take advantage of existing thrift infra.
      // expose raw connectal cpp generated interface to test application.

      method Action port_mapping_add_entry(Bit#(32) table_name, MatchInput_port_mapping match_key);
         // write entries to table.
         // queue read flow_id
      endmethod

      method Action port_mapping_set_default_action(Bit#(32) table_name);
         // write default action to action engine
      endmethod

      method Action port_mapping_delete_entry(Bit#(32) table_name, Bit#(32) flow_id);
         // invalidate entries in table
      endmethod

      method Action port_mapping_modify_entry(Bit#(32) table_name, Bit#(32) flow_id, ActionInput_port_mapping actions);
         // enqueue match key
      endmethod

      method Action bd_add_entry(Bit#(32) table_name, MatchInput_bd match_key);
         // write entries to table.
         // queue read flow_id
      endmethod

      method Action bd_set_default_action(Bit#(32) table_name);
         // write default action to action engine
      endmethod

      method Action bd_delete_entry(Bit#(32) table_name, Bit#(32) flow_id);
         // invalidate entries in table
      endmethod

      method Action bd_modify_entry(Bit#(32) table_name, Bit#(32) flow_id, ActionInput_bd actions);
         // enqueue match key
      endmethod

      method Action ipv4_fib_add_entry(Bit#(32) table_name, MatchInput_ipv4_fib match_key);
         // write entries to table.
         // queue read flow_id
      endmethod

      method Action ipv4_fib_set_default_action(Bit#(32) table_name);
         // write default action to action engine
      endmethod

      method Action ipv4_fib_delete_entry(Bit#(32) table_name, Bit#(32) flow_id);
         // invalidate entries in table
      endmethod

      method Action ipv4_fib_modify_entry(Bit#(32) table_name, Bit#(32) flow_id, ActionInput_ipv4_fib actions);
         // enqueue match key
      endmethod

      method Action ipv4_fib_lpm_add_entry(Bit#(32) table_name, MatchInput_ipv4_fib_lpm match_key);
         // write entries to table.
         // queue read flow_id
      endmethod

      method Action ipv4_fib_lpm_set_default_action(Bit#(32) table_name);
         // write default action to action engine
      endmethod

      method Action ipv4_fib_lpm_delete_entry(Bit#(32) table_name, Bit#(32) flow_id);
         // invalidate entries in table
      endmethod

      method Action ipv4_fib_lpm_modify_entry(Bit#(32) table_name, Bit#(32) flow_id, ActionInput_ipv4_fib_lpm actions);
         // enqueue match key
      endmethod

      method Action nexthop_add_entry(Bit#(32) table_name, MatchInput_nexthop match_key);
         // write entries to table.
         // queue read flow_id
      endmethod

      method Action nexthop_set_default_action(Bit#(32) table_name);
         // write default action to action engine
      endmethod

      method Action nexthop_delete_entry(Bit#(32) table_name, Bit#(32) flow_id);
         // invalidate entries in table
      endmethod

      method Action nexthop_modify_entry(Bit#(32) table_name, Bit#(32) flow_id, ActionInput_nexthop actions);
         // enqueue match key
      endmethod

      method Action rewrite_mac_add_entry(Bit#(32) table_name, MatchInput_rewrite_mac match_key);
         // write entries to table.
         // queue read flow_id
      endmethod

      method Action rewrite_mac_set_default_action(Bit#(32) table_name);
         // write default action to action engine
      endmethod

      method Action rewrite_mac_delete_entry(Bit#(32) table_name, Bit#(32) flow_id);
         // invalidate entries in table
      endmethod

      method Action rewrite_mac_modify_entry(Bit#(32) table_name, Bit#(32) flow_id, ActionInput_rewrite_mac actions);
         // enqueue match key
      endmethod

      // -- Internal of API depends on P4 program.
      // -- Use dispatcher to send table info to corresponding table.
      // -- Invalid table tag will result in error.
   endinterface
   interface `PinType pins;
      // Clocks
`ifndef BSIM
      method Action osc_50(Bit#(1) b3d, Bit#(1) b4a, Bit#(1) b4d, Bit#(1) b7a, Bit#(1) b7d, Bit#(1) b8a, Bit#(1) b8d);
         iclock_50.inputclock(b4a);
      endmethod
      method Action sfp(Bit#(1) refclk);
         iclock_644.inputclock(refclk);
      endmethod
      method Action buttons(Bit#(4) v);
         for (Integer i=0; i<valueOf(NumPorts); i=i+1) begin
            buttons[i].ifc.button(v[i]);
         end
      endmethod
//      method Action serial_rx(Bit#(NumPorts) data);
//         phy.serial.rx(data);
//      endmethod
//      method serial_tx_data = phy.serial.tx;
//      method led0 = pci_led.ifc.out;
//      method led1 = eth_tx_led.ifc.out;
//      method led2 = eth_rx_led.ifc.out;
      method led3 = sfp_led.ifc.out;
      method led_bracket = led_xcvr.out;
      interface sfpctrl = (interface SFPCtrl;
         method Action los (Bit#(NumPorts) v); endmethod
         method Action mod0_presnt_n(Bit#(NumPorts) v); endmethod
         method ratesel0 = pack(replicate(1'b1));
         method ratesel1 = pack(replicate(1'b1));
         method txdisable = pack(replicate(1'b0));
         method Action txfault(Bit#(NumPorts) v); endmethod
      endinterface);
      interface deleteme_unused_clock = iclock_50.c;
      interface deleteme_unused_reset = iclock_50.r;
      interface deleteme_unused_clock2 = defaultClock;
      interface deleteme_unused_clock3 = clk_156_25;
`endif
   endinterface
endmodule

// Copyright (c) 2015 Cornell University.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import BRAMFIFO::*;
import BuildVector::*;
import Clocks::*;
import DefaultValue::*;
import FIFO::*;
import FIFOF::*;
import GetPut::*;
import ClientServer::*;
import Vector::*;
import Connectable::*;
import StmtFSM::*;
import Pipe::*;
import HostInterface::*;
import MemTypes::*;
import MemReadEngine::*;
import MemWriteEngine::*;
import MemServerIndication::*;
import MMUIndication::*;
import MatchTable::*;
import MatchTableTypes::*;

import AlteraExtra::*;
import AlteraEthPhy::*;
import EthMac::*;
import ALTERA_SI570_WRAPPER          ::*;
import ALTERA_EDGE_DETECTOR_WRAPPER  ::*;
import ConnectalClocks::*;
import LedController::*;
import Leds::*;
import PushButtonController::*;

import Ethernet::*;
import IngressPipeline::*;
import PacketBuffer::*;
import Parser::*;
import Types::*;
import SharedBuff::*;
import `PinTypeInclude::*;

typedef TDiv#(DataBusWidth, 32) WordsPerBeat;

interface P4TopIndication;
   method Action sonic_read_version_resp(Bit#(32) version);
   method Action matchTableResponse(Bit#(32) key, Bit#(32) value);
endinterface

interface P4TopRequest;
   method Action sonic_read_version();
   method Action writePacketData(Vector#(2, Bit#(64)) data, Bit#(1) sop, Bit#(1) eop);
   method Action readPacketBuffer(Bit#(16) addr);
   method Action writePacketBuffer(Bit#(16) addr, Bit#(64) data);

   method Action matchTableRequest(Bit#(32) key, Bit#(32) value, Bit#(32) op);

   method Action port_mapping_add_entry(Bit#(32) table_name, MatchInput_port_mapping match_key);
   method Action port_mapping_set_default_action(Bit#(32) table_name);
   method Action port_mapping_delete_entry(Bit#(32) table_name, Bit#(32) id);
   method Action port_mapping_modify_entry(Bit#(32) table_name, Bit#(32) id, ActionInput_port_mapping actions);
   method Action bd_add_entry(Bit#(32) table_name, MatchInput_bd match_key);
   method Action bd_set_default_action(Bit#(32) table_name);
   method Action bd_delete_entry(Bit#(32) table_name, Bit#(32) id);
   method Action bd_modify_entry(Bit#(32) table_name, Bit#(32) id, ActionInput_bd actions);
   method Action ipv4_fib_add_entry(Bit#(32) table_name, MatchInput_ipv4_fib match_key);
   method Action ipv4_fib_set_default_action(Bit#(32) table_name);
   method Action ipv4_fib_delete_entry(Bit#(32) table_name, Bit#(32) id);
   method Action ipv4_fib_modify_entry(Bit#(32) table_name, Bit#(32) id, ActionInput_ipv4_fib actions);
   method Action ipv4_fib_lpm_add_entry(Bit#(32) table_name, MatchInput_ipv4_fib_lpm match_key);
   method Action ipv4_fib_lpm_set_default_action(Bit#(32) table_name);
   method Action ipv4_fib_lpm_delete_entry(Bit#(32) table_name, Bit#(32) id);
   method Action ipv4_fib_lpm_modify_entry(Bit#(32) table_name, Bit#(32) id, ActionInput_ipv4_fib_lpm actions);
   method Action nexthop_add_entry(Bit#(32) table_name, MatchInput_nexthop match_key);
   method Action nexthop_set_default_action(Bit#(32) table_name);
   method Action nexthop_delete_entry(Bit#(32) table_name, Bit#(32) id);
   method Action nexthop_modify_entry(Bit#(32) table_name, Bit#(32) id, ActionInput_nexthop actions);
   method Action rewrite_mac_add_entry(Bit#(32) table_name, MatchInput_rewrite_mac match_key);
   method Action rewrite_mac_set_default_action(Bit#(32) table_name);
   method Action rewrite_mac_delete_entry(Bit#(32) table_name, Bit#(32) id);
   method Action rewrite_mac_modify_entry(Bit#(32) table_name, Bit#(32) id, ActionInput_rewrite_mac actions);
endinterface

interface P4Top;
   interface P4TopRequest request;
   interface `PinType pins;
endinterface

//module mkP4Top#(Clock derivedClock, Reset derivedReset, P4TopIndication indication)(P4Top);
module mkP4Top#(P4TopIndication indication)(P4Top);
   Clock defaultClock <- exposeCurrentClock();
   Reset defaultReset <- exposeCurrentReset();

`ifndef BSIM
   B2C iclock_50 <- mkB2C();
   B2C1 iclock_644 <- mkB2C1();
//   B2C iclock_pcie <- mkB2C();

   Vector#(4, PushButtonController) buttons <- replicateM(mkPushButtonController(iclock_50.c, clocked_by iclock_50.c, reset_by iclock_50.r));

   Reset rst_644   <- mkResetInverter(iclock_50.r, clocked_by iclock_644.c);
   Reset rst_644_n <- mkAsyncReset(2, iclock_50.r, iclock_644.c);
   // PLL Input:   SFP REFCLK from SI570
   // PLL Output:  156.25MHz
   // PLL Reset:   Active High, must invert default Reset
   PLL156 pll156 <- mkPLL156(iclock_644.c, rst_644, clocked_by iclock_644.c, reset_by rst_644);
   Clock clk_156_25 = pll156.outclk_0;
   Reset rst_156   <- mkResetInverter(iclock_50.r, clocked_by clk_156_25);
   Reset rst_156_n <- mkAsyncReset(1, rst_156, clk_156_25);
   Si570Wrap si570 <- mkSi570Wrap(iclock_50.c, iclock_50.r, clocked_by iclock_50.c, reset_by iclock_50.r);
   EdgeDetectorWrap edgedetect <- mkEdgeDetectorWrap(iclock_50.c, iclock_50.r, clocked_by iclock_50.c, reset_by iclock_50.r);

   rule si570_connections;
      let ifreq_mode = 3'b110;  //644.53125 MHZ
      si570.ifreq.mode(ifreq_mode);
      si570.istart.go(edgedetect.odebounce.out);
   endrule

   rule button_to_iclock50_r;
      iclock_50.inputreset(pack(buttons[0]));
   endrule

   rule button_to_si570;
      edgedetect.itrigger.in(pack(buttons[1]));
   endrule

//   Reset xcvr_reset_n <- mkAsyncReset(2, iclock_50.r, eth.ifcs.clk_xcvr[0]);

//   LedController pci_led <- mkLedController(False, clocked_by iclock_pcie.c, reset_by iclock_pcie.r);
//   LedController eth_tx_led <- mkLedController(False, clocked_by clk_156_25, reset_by rst_156_n);
//   LedController eth_rx_led <- mkLedController(False, clocked_by eth.ifcs.clk_xcvr[0], reset_by xcvr_reset_n);
   LedController sfp_led <- mkLedController(False, clocked_by iclock_644.c, reset_by rst_644_n);

//   rule led_pcie;
//      pci_led.setPeriod(led_off, 500, led_on_max, 500);
//   endrule
//   rule led_eth_tx;
//      eth_tx_led.setPeriod(led_off, 500, led_on_max, 500);
//   endrule
//   rule led_eth_rx;
//      eth_rx_led.setPeriod(led_off, 500, led_on_max, 500);
//   endrule
   rule led_sfp;
      sfp_led.setPeriod(led_off, 500, led_on_max, 500);
   endrule

   Vector#(4, LedController) led_xcvr_ <- replicateM(mkLedController(False, clocked_by iclock_50.c, reset_by iclock_50.r));
   Led#(4) led_xcvr = combineLeds(led_xcvr_);

   rule led_xcvr_ready;
      for (Integer i = 0; i<4; i=i+1) begin
         led_xcvr_[i].setPeriod(led_off, 500, led_on_max, 500);
      end
   endrule

//   EthPhyIfc phy <- mkAlteraEthPhy(clk_50_b4a_buf.outclk, iclock_644.c, clk_156_25, iclock_50.r, clocked_by clk_156_25, reset_by rst_156_n);
//   Clock xgmii_rx_clk = phy.rx_clkout;
//   EthMacIfc mac <- mkEthMac(clk_50_b4a_buf.outclk, clk_156_25, replicate(xgmii_rx_clk), rst_156_n, clocked_by clk_156_25, reset_by rst_156_n);
//   mapM(uncurry(mkConnection), zip(mac.tx, phy.tx));
//   mapM(uncurry(mkConnection), zip(phy.rx, mac.rx));

`endif

   let verbose = True;
   Reg#(Cycle_t) cycle <- mkReg(defaultValue);
   rule every1 if (verbose);
      cycle.cnt <= cycle.cnt + 1;
   endrule

   Reg#(Standard_metadata_t) standard_metadata <- mkReg(defaultValue);
   Reg#(Ingress_metadata_t) ingress_metadata <- mkReg(defaultValue);

   PacketBuffer rxPktBuff <- mkPacketBuffer();
   Parser parser <- mkParser();
   Pipeline_port_mapping ingress_port_mapping <- mkIngressPipeline_port_mapping();

   /* Match Table Functionalities */
   function RequestType makeRequest(Bit#(32) key, Bit#(32) value, Operation op);
       return RequestType {
           key : key,
           value : value,
           addrIdx : 0,
           op : op
       };
   endfunction

   Server#(RequestType, ResponseType) matchTable <- mkMatchTable();

   // read client interface
   FIFO#(MemRequest) reqFifo <-mkSizedFIFO(4);
   FIFO#(MemData#(DataBusWidth)) dataFifo <- mkSizedFIFO(32);
   MemReadClient#(DataBusWidth) dmaClient = (interface MemReadClient;
      interface Get readReq = toGet(reqFifo);
      interface Put readData = toPut(dataFifo);
   endinterface);

   // write client interface
   FIFO#(MemRequest) writeReqFifo <- mkSizedFIFO(4);
   FIFO#(MemData#(DataBusWidth)) writeDataFifo <- mkSizedFIFO(32);
   FIFO#(Bit#(MemTagSize)) writeDoneFifo <- mkSizedFIFO(4);
   MemWriteClient#(DataBusWidth) dmaWriteClient = (interface MemWriteClient;
      interface Get writeReq = toGet(writeReqFifo);
      interface Get writeData = toGet(writeDataFifo);
      interface Put writeDone = toPut(writeDoneFifo);
   endinterface);

   MemServerIndicationOutput memServerIndication <- mkMemServerIndicationOutput;
   MMUIndicationOutput mmuIndication <- mkMMUIndicationOutput;
//   SharedBuffer#(12, 128, 1) buff <- mkSharedBuffer(vec(dmaClient), vec(dmaWriteClient), memServerIndication.ifc, mmuIndication.ifc);

   Reg#(Bit#(EtherLen)) pktLen <- mkReg(0);
   Reg#(Bit#(9)) rAddr_wires <- mkReg(0);

   rule packetParseStart;
      let pktLen <- rxPktBuff.readServer.readLen.get;
      rxPktBuff.readServer.readReq.put(EtherReq{len: truncate(pktLen)});
      if (verbose) $display(fshow(cycle) + fshow(" read packt ") + fshow(pktLen));
   endrule

   rule packetParseInProgress;
      let v <- rxPktBuff.readServer.readData.get;
      if (verbose) $display(fshow(cycle) + fshow(" load packet ") + fshow(v));
      parser.frameIn.enq(v);
      if (v.eop) begin
         if (verbose) $display(fshow(cycle) + fshow(" eop."));
      end
   endrule

   rule matchTableStart;
      let v <- toGet(parser.parseDone).get;
      $display("Parse Done");
      parser.parserReset();
   endrule

   rule forwardPayload;
      let v <- toGet(parser.payloadOut).get;
   endrule

   rule readData;
      let v <- toGet(dataFifo).get;
      $display("%d: Received data %x", cycle, v.data);
   endrule

   rule writeDone;
      let v <- toGet(writeDoneFifo).get;
      $display("%d: Write done", cycle);
   endrule
   //mkConnection(parser.phvOut, ingress_port_mapping.phvIn);

   rule matchTableRes;
       let res <- matchTable.response.get;
       indication.matchTableResponse(res.key, res.value);
   endrule

   interface P4TopRequest request;
      method Action sonic_read_version();
         let v= `NicVersion;
         indication.sonic_read_version_resp(v);
      endmethod
      method Action writePacketData(Vector#(2, Bit#(64)) data, Bit#(1) sop, Bit#(1) eop);
         EtherData beat = defaultValue;
         beat.data = pack(reverse(data));
         beat.sop = unpack(sop);
         beat.eop = unpack(eop);
         rxPktBuff.writeServer.writeData.put(beat);
      endmethod

      method Action readPacketBuffer(Bit#(16) addr);
         Bit#(ByteEnableSize) firstbe = 'hff;
         Bit#(ByteEnableSize) lastbe = 'hff;
         reqFifo.enq(MemRequest {sglId: 0, offset: 0, burstLen: 16, tag:0, firstbe: firstbe, lastbe: lastbe});
      endmethod

      method Action writePacketBuffer(Bit#(16) addr, Bit#(64) data);
         Bit#(ByteEnableSize) firstbe = 'hff;
         Bit#(ByteEnableSize) lastbe = 'hff;
         writeReqFifo.enq(MemRequest {sglId:0, offset:0, burstLen:16, tag:0, firstbe: firstbe, lastbe: lastbe});

         function Bit#(32) plusi(Integer i); return fromInteger(i); endfunction
         Vector#(WordsPerBeat, Bit#(32)) v = genWith(plusi);
         writeDataFifo.enq(MemData {data: pack(v), tag:0, last:True});
      endmethod

      method Action matchTableRequest(Bit#(32) key, Bit#(32) value, Bit#(32) op);
        if (op == 0)
            matchTable.request.put(makeRequest(key, value, GET));
        else if (op == 1)
            matchTable.request.put(makeRequest(key, value, PUT));
        else if (op == 2)
            matchTable.request.put(makeRequest(key, value, UPDATE));
        else if (op == 3)
            matchTable.request.put(makeRequest(key, value, REMOVE));
      endmethod

      // Generate fixed standard set of API function
      // In software, we should hide the details of match table different behind api
      // expose thrift-server to software to take advantage of existing thrift infra.
      // expose raw connectal cpp generated interface to test application.

      method Action port_mapping_add_entry(Bit#(32) table_name, MatchInput_port_mapping match_key);
         // write entries to table.
         // queue read flow_id
      endmethod

      method Action port_mapping_set_default_action(Bit#(32) table_name);
         // write default action to action engine
      endmethod

      method Action port_mapping_delete_entry(Bit#(32) table_name, Bit#(32) flow_id);
         // invalidate entries in table
      endmethod

      method Action port_mapping_modify_entry(Bit#(32) table_name, Bit#(32) flow_id, ActionInput_port_mapping actions);
         // enqueue match key
      endmethod

      method Action bd_add_entry(Bit#(32) table_name, MatchInput_bd match_key);
         // write entries to table.
         // queue read flow_id
      endmethod

      method Action bd_set_default_action(Bit#(32) table_name);
         // write default action to action engine
      endmethod

      method Action bd_delete_entry(Bit#(32) table_name, Bit#(32) flow_id);
         // invalidate entries in table
      endmethod

      method Action bd_modify_entry(Bit#(32) table_name, Bit#(32) flow_id, ActionInput_bd actions);
         // enqueue match key
      endmethod

      method Action ipv4_fib_add_entry(Bit#(32) table_name, MatchInput_ipv4_fib match_key);
         // write entries to table.
         // queue read flow_id
      endmethod

      method Action ipv4_fib_set_default_action(Bit#(32) table_name);
         // write default action to action engine
      endmethod

      method Action ipv4_fib_delete_entry(Bit#(32) table_name, Bit#(32) flow_id);
         // invalidate entries in table
      endmethod

      method Action ipv4_fib_modify_entry(Bit#(32) table_name, Bit#(32) flow_id, ActionInput_ipv4_fib actions);
         // enqueue match key
      endmethod

      method Action ipv4_fib_lpm_add_entry(Bit#(32) table_name, MatchInput_ipv4_fib_lpm match_key);
         // write entries to table.
         // queue read flow_id
      endmethod

      method Action ipv4_fib_lpm_set_default_action(Bit#(32) table_name);
         // write default action to action engine
      endmethod

      method Action ipv4_fib_lpm_delete_entry(Bit#(32) table_name, Bit#(32) flow_id);
         // invalidate entries in table
      endmethod

      method Action ipv4_fib_lpm_modify_entry(Bit#(32) table_name, Bit#(32) flow_id, ActionInput_ipv4_fib_lpm actions);
         // enqueue match key
      endmethod

      method Action nexthop_add_entry(Bit#(32) table_name, MatchInput_nexthop match_key);
         // write entries to table.
         // queue read flow_id
      endmethod

      method Action nexthop_set_default_action(Bit#(32) table_name);
         // write default action to action engine
      endmethod

      method Action nexthop_delete_entry(Bit#(32) table_name, Bit#(32) flow_id);
         // invalidate entries in table
      endmethod

      method Action nexthop_modify_entry(Bit#(32) table_name, Bit#(32) flow_id, ActionInput_nexthop actions);
         // enqueue match key
      endmethod

      method Action rewrite_mac_add_entry(Bit#(32) table_name, MatchInput_rewrite_mac match_key);
         // write entries to table.
         // queue read flow_id
      endmethod

      method Action rewrite_mac_set_default_action(Bit#(32) table_name);
         // write default action to action engine
      endmethod

      method Action rewrite_mac_delete_entry(Bit#(32) table_name, Bit#(32) flow_id);
         // invalidate entries in table
      endmethod

      method Action rewrite_mac_modify_entry(Bit#(32) table_name, Bit#(32) flow_id, ActionInput_rewrite_mac actions);
         // enqueue match key
      endmethod

      // -- Internal of API depends on P4 program.
      // -- Use dispatcher to send table info to corresponding table.
      // -- Invalid table tag will result in error.
   endinterface
   interface `PinType pins;
      // Clocks
`ifndef BSIM
      method Action osc_50(Bit#(1) b3d, Bit#(1) b4a, Bit#(1) b4d, Bit#(1) b7a, Bit#(1) b7d, Bit#(1) b8a, Bit#(1) b8d);
         iclock_50.inputclock(b4a);
      endmethod
      method Action sfp(Bit#(1) refclk);
         iclock_644.inputclock(refclk);
      endmethod
      method Action buttons(Bit#(4) v);
         for (Integer i=0; i<valueOf(NumPorts); i=i+1) begin
            buttons[i].ifc.button(v[i]);
         end
      endmethod
//      method Action serial_rx(Bit#(NumPorts) data);
//         phy.serial.rx(data);
//      endmethod
//      method serial_tx_data = phy.serial.tx;
//      method led0 = pci_led.ifc.out;
//      method led1 = eth_tx_led.ifc.out;
//      method led2 = eth_rx_led.ifc.out;
      method led3 = sfp_led.ifc.out;
      method led_bracket = led_xcvr.out;
      interface sfpctrl = (interface SFPCtrl;
         method Action los (Bit#(NumPorts) v); endmethod
         method Action mod0_presnt_n(Bit#(NumPorts) v); endmethod
         method ratesel0 = pack(replicate(1'b1));
         method ratesel1 = pack(replicate(1'b1));
         method txdisable = pack(replicate(1'b0));
         method Action txfault(Bit#(NumPorts) v); endmethod
      endinterface);
      interface deleteme_unused_clock = iclock_50.c;
      interface deleteme_unused_reset = iclock_50.r;
      interface deleteme_unused_clock2 = defaultClock;
      interface deleteme_unused_clock3 = clk_156_25;
`endif
   endinterface
endmodule


handle number of clients that is a multiple of 2

Change-Id: I15fbd425521d5b043a3e17adaa6570e63c2c36c4

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import Vector::*;
import FIFOF::*;
import FIFO::*;
import GetPut::*;
import Assert::*;
import ClientServer::*;
import BRAM::*;

import PortalMemory::*;
import Dma::*;
import Pipe::*;

typedef struct {ObjectPointer pointer;
		Bit#(ObjectOffsetSize) base;
		Bit#(8) burstLen;
		Bit#(32) len;
		} MemengineCmd deriving (Eq,Bits);

interface MemreadEngineV#(numeric type dataWidth, numeric type cmdQDepth, numeric type numServers);
   interface Vector#(numServers, Server#(MemengineCmd,Bool)) readServers;
   interface ObjectReadClient#(dataWidth) dmaClient;
   interface Vector#(numServers, PipeOut#(Bit#(dataWidth))) dataPipes;
endinterface

module mkMemreadEngineV(MemreadEngineV#(dataWidth, cmdQDepth, numServers))
   provisos (Div#(dataWidth,8,dataWidthBytes),
	     Mul#(dataWidthBytes,8,dataWidth),
	     Log#(dataWidthBytes,beatShift),
	     Mul#(cmdQDepth,numServers,cmdBuffSz),
	     Log#(cmdBuffSz, cmdBuffAddrSz),
	     Log#(numServers, serverIdxSz),
	     Add#(1,cmdQDepth, outCntSz),
	     Add#(1, c__, numServers),
	     Add#(b__, TLog#(numServers), cmdBuffAddrSz),
	     Add#(a__, serverIdxSz, cmdBuffAddrSz),
	     Min#(2,TLog#(numServers),bpc));

   function Bit#(cmdBuffAddrSz) hf(Integer i) = fromInteger(i*valueOf(cmdQDepth));
   Vector#(numServers, Reg#(Bit#(outCntSz)))     outs1 <- replicateM(mkReg(0));
   Vector#(numServers, Reg#(Bit#(outCntSz)))     outs0 <- replicateM(mkReg(0));
   Vector#(numServers, Reg#(Bit#(cmdBuffAddrSz))) head <- mapM(mkReg, genWith(hf));
   Vector#(numServers, Reg#(Bit#(cmdBuffAddrSz))) tail <- mapM(mkReg, genWith(hf));

   BRAM1Port#(Bit#(cmdBuffAddrSz),MemengineCmd) cmdBuf <- mkBRAM1Server(defaultValue);
   FIFO#(Bit#(serverIdxSz))                      loadf <- mkSizedFIFO(1);
   FIFO#(Tuple3#(Bit#(8),Bit#(serverIdxSz),Bool))workf <- mkSizedFIFO(32); // isthis the right size?

   Vector#(numServers, FIFO#(void))              outfs <- replicateM(mkSizedFIFO(1));
   Vector#(numServers, FIFOF#(Tuple2#(Bit#(serverIdxSz), MemengineCmd))) cmds_in <- replicateM(mkSizedFIFOF(1));
   FunnelPipe#(1, Tuple2#(Bit#(serverIdxSz), MemengineCmd),2) cmds_in_funnel <- mkFunnel1PipesPipelined(map(toPipeOut,cmds_in));
   FIFOF#(Tuple2#(Bit#(TLog#(numServers)), Tuple2#(Bit#(dataWidth),Bool))) read_data <- mkFIFOF;
   FunnelPipe#(numServers, Tuple2#(Bit#(dataWidth),Bool),bpc) read_data_unfunnel <- mkUnFunnel1PipesPipelined(toPipeOut(read_data));
   function PipeOut#(Bit#(dataWidth)) check_out(PipeOut#(Tuple2#(Bit#(dataWidth),Bool)) x, Integer i) =
      (interface PipeOut;
	  method Bit#(dataWidth) first;
	     return tpl_1(x.first);
	  endmethod
	  method Action deq;
	     x.deq;
	     if (tpl_2(x.first))
		outfs[i].enq(?);
	  endmethod
	  method Bool notEmpty = x.notEmpty;
       endinterface);
   Vector#(numServers, PipeOut#(Bit#(dataWidth))) read_data_pipes = zipWith(check_out, read_data_unfunnel, genVector);

   Reg#(Bit#(8))                               respCnt <- mkReg(0);
   Reg#(Bit#(serverIdxSz))                     loadIdx <- mkReg(0);
   let beat_shift = fromInteger(valueOf(beatShift));
   let cmd_q_depth = fromInteger(valueOf(cmdQDepth));

   rule store_cmd;
      match {.idx, .cmd} <- toGet(cmds_in_funnel[0]).get;
      let new_tail = tail[idx]+1;
      if (new_tail >= extend(idx+1)*cmd_q_depth)
	 new_tail = extend(idx)*cmd_q_depth;
      tail[idx] <= new_tail;
      outs1[idx] <= outs1[idx]+1;
      cmdBuf.portA.request.put(BRAMRequest{write:True, responseOnWrite:False, address:tail[idx], datain:cmd});
      //$display("MemreadEngineV.store_cmd: %d %h", idx, tail[idx]);
   endrule

   rule load_ctxt;
      loadIdx <= loadIdx+1;
      if (outs1[loadIdx] > 0) begin
	 cmdBuf.portA.request.put(BRAMRequest{write:False, responseOnWrite:False, address:head[loadIdx], datain:?});
	 loadf.enq(loadIdx);
	 //$display("MemreadEngineV.load_ctxt: %d, %h", loadIdx, head[loadIdx]);
      end
   endrule

   Vector#(numServers, Server#(MemengineCmd,Bool)) rs;
   for(Integer i = 0; i < valueOf(numServers); i=i+1)
      rs[i] =  (interface Server#(MemengineCmd,Bool);
		   interface Put request;
		      method Action put(MemengineCmd c) if (outs0[i] < cmd_q_depth);
			 outs0[i] <= outs0[i]+1;
			 cmds_in[i].enq(tuple2(fromInteger(i),c));
 		      endmethod
		   endinterface
		   interface Get response;
		      method ActionValue#(Bool) get;
			 outfs[i].deq;
	 		 outs0[i] <= outs0[i]-1;
			 return True;
		      endmethod
		   endinterface
		endinterface);
   interface readServers = rs;
   interface ObjectReadClient dmaClient;
      interface Get readReq;
	 method ActionValue#(ObjectRequest) get();
	    let cmd <- cmdBuf.portA.response.get;
	    let idx <- toGet(loadf).get;
	    Bit#(8) bl = cmd.burstLen;
	    let last = False;
	    if (cmd.len <= extend(bl)) begin
	       last = True;
	       bl = truncate(cmd.len);
	       outs1[idx] <= outs1[idx]-1;
	       let new_head = head[idx]+1;
	       if (new_head >= extend(idx+1)*cmd_q_depth)
		  new_head = extend(idx)*cmd_q_depth;
	       head[idx] <= new_head;
	       //$display("new_head %d %d", idx, new_head);
	    end
	    let new_cmd = MemengineCmd{pointer:cmd.pointer, base:cmd.base+extend(bl), burstLen:cmd.burstLen, len:cmd.len-extend(bl)};
	    cmdBuf.portA.request.put(BRAMRequest{write:True, responseOnWrite:False, address:head[idx], datain:new_cmd});
	    workf.enq(tuple3(truncate(bl>>beat_shift), idx, last));
	    //$display("readReq %d, %h %h %h", idx, cmd.base, bl, last);
	    return ObjectRequest { pointer: cmd.pointer, offset: cmd.base, burstLen:bl, tag: 0 };
	 endmethod
      endinterface
      interface Put readData;
	 method Action put(ObjectData#(dataWidth) d);
	    match {.rc, .idx, .last} = workf.first;
	    let new_respCnt = respCnt+1;
	    let l = False;
	    //$display("%h %d", d.data, idx);
	    if (new_respCnt == rc) begin
	       respCnt <= 0;
	       workf.deq;
	       //$display("eob %d", idx);
	       l = last;
	    end
	    else begin
	       respCnt <= new_respCnt;
	    end
	    read_data.enq(tuple2(idx,tuple2(d.data,l)));
	 endmethod
      endinterface
   endinterface
   interface dataPipes = read_data_pipes;
endmodule

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import Vector::*;
import FIFOF::*;
import FIFO::*;
import GetPut::*;
import Assert::*;
import ClientServer::*;
import BRAM::*;

import PortalMemory::*;
import Dma::*;
import Pipe::*;

typedef struct {ObjectPointer pointer;
		Bit#(ObjectOffsetSize) base;
		Bit#(8) burstLen;
		Bit#(32) len;
		} MemengineCmd deriving (Eq,Bits);

interface MemreadEngineV#(numeric type dataWidth, numeric type cmdQDepth, numeric type numServers);
   interface Vector#(numServers, Server#(MemengineCmd,Bool)) readServers;
   interface ObjectReadClient#(dataWidth) dmaClient;
   interface Vector#(numServers, PipeOut#(Bit#(dataWidth))) dataPipes;
endinterface

module mkMemreadEngineV(MemreadEngineV#(dataWidth, cmdQDepth, numServers))
   provisos (Div#(dataWidth,8,dataWidthBytes),
	     Mul#(dataWidthBytes,8,dataWidth),
	     Log#(dataWidthBytes,beatShift),
	     Mul#(cmdQDepth,numServers,cmdBuffSz),
	     Log#(cmdBuffSz, cmdBuffAddrSz),
	     Log#(numServers, serverIdxSz),
	     Add#(1,cmdQDepth, outCntSz),
	     Add#(1, c__, numServers),
	     Add#(b__, TLog#(numServers), cmdBuffAddrSz),
	     Add#(a__, serverIdxSz, cmdBuffAddrSz));

   function Bit#(cmdBuffAddrSz) hf(Integer i) = fromInteger(i*valueOf(cmdQDepth));
   Vector#(numServers, Reg#(Bit#(outCntSz)))     outs1 <- replicateM(mkReg(0));
   Vector#(numServers, Reg#(Bit#(outCntSz)))     outs0 <- replicateM(mkReg(0));
   Vector#(numServers, Reg#(Bit#(cmdBuffAddrSz))) head <- mapM(mkReg, genWith(hf));
   Vector#(numServers, Reg#(Bit#(cmdBuffAddrSz))) tail <- mapM(mkReg, genWith(hf));

   BRAM1Port#(Bit#(cmdBuffAddrSz),MemengineCmd) cmdBuf <- mkBRAM1Server(defaultValue);
   FIFO#(Bit#(serverIdxSz))                      loadf <- mkSizedFIFO(1);
   FIFO#(Tuple3#(Bit#(8),Bit#(serverIdxSz),Bool))workf <- mkSizedFIFO(32); // isthis the right size?

   Vector#(numServers, FIFO#(void))              outfs <- replicateM(mkSizedFIFO(1));
   Vector#(numServers, FIFOF#(Tuple2#(Bit#(serverIdxSz), MemengineCmd))) cmds_in <- replicateM(mkSizedFIFOF(1));
   FunnelPipe#(1, Tuple2#(Bit#(serverIdxSz), MemengineCmd),2) cmds_in_funnel <- mkFunnel1PipesPipelined(map(toPipeOut,cmds_in));
   FIFOF#(Tuple2#(Bit#(TLog#(numServers)), Tuple2#(Bit#(dataWidth),Bool))) read_data <- mkFIFOF;
   FunnelPipe#(numServers, Tuple2#(Bit#(dataWidth),Bool),2) read_data_unfunnel <- mkUnFunnel1PipesPipelined(toPipeOut(read_data));
   function PipeOut#(Bit#(dataWidth)) check_out(PipeOut#(Tuple2#(Bit#(dataWidth),Bool)) x, Integer i) =
      (interface PipeOut;
	  method Bit#(dataWidth) first;
	     return tpl_1(x.first);
	  endmethod
	  method Action deq;
	     x.deq;
	     if (tpl_2(x.first))
		outfs[i].enq(?);
	  endmethod
	  method Bool notEmpty = x.notEmpty;
       endinterface);
   Vector#(numServers, PipeOut#(Bit#(dataWidth))) read_data_pipes = zipWith(check_out, read_data_unfunnel, genVector);

   Reg#(Bit#(8))                               respCnt <- mkReg(0);
   Reg#(Bit#(serverIdxSz))                     loadIdx <- mkReg(0);
   let beat_shift = fromInteger(valueOf(beatShift));
   let cmd_q_depth = fromInteger(valueOf(cmdQDepth));

   rule store_cmd;
      match {.idx, .cmd} <- toGet(cmds_in_funnel[0]).get;
      let new_tail = tail[idx]+1;
      if (new_tail >= extend(idx+1)*cmd_q_depth)
	 new_tail = extend(idx)*cmd_q_depth;
      tail[idx] <= new_tail;
      outs1[idx] <= outs1[idx]+1;
      cmdBuf.portA.request.put(BRAMRequest{write:True, responseOnWrite:False, address:tail[idx], datain:cmd});
      //$display("MemreadEngineV.store_cmd: %d %h", idx, tail[idx]);
   endrule

   rule load_ctxt;
      loadIdx <= loadIdx+1;
      if (outs1[loadIdx] > 0) begin
	 cmdBuf.portA.request.put(BRAMRequest{write:False, responseOnWrite:False, address:head[loadIdx], datain:?});
	 loadf.enq(loadIdx);
	 //$display("MemreadEngineV.load_ctxt: %d, %h", loadIdx, head[loadIdx]);
      end
   endrule

   Vector#(numServers, Server#(MemengineCmd,Bool)) rs;
   for(Integer i = 0; i < valueOf(numServers); i=i+1)
      rs[i] =  (interface Server#(MemengineCmd,Bool);
		   interface Put request;
		      method Action put(MemengineCmd c) if (outs0[i] < cmd_q_depth);
			 outs0[i] <= outs0[i]+1;
			 cmds_in[i].enq(tuple2(fromInteger(i),c));
 		      endmethod
		   endinterface
		   interface Get response;
		      method ActionValue#(Bool) get;
			 outfs[i].deq;
	 		 outs0[i] <= outs0[i]-1;
			 return True;
		      endmethod
		   endinterface
		endinterface);
   interface readServers = rs;
   interface ObjectReadClient dmaClient;
      interface Get readReq;
	 method ActionValue#(ObjectRequest) get();
	    let cmd <- cmdBuf.portA.response.get;
	    let idx <- toGet(loadf).get;
	    Bit#(8) bl = cmd.burstLen;
	    let last = False;
	    if (cmd.len <= extend(bl)) begin
	       last = True;
	       bl = truncate(cmd.len);
	       outs1[idx] <= outs1[idx]-1;
	       let new_head = head[idx]+1;
	       if (new_head >= extend(idx+1)*cmd_q_depth)
		  new_head = extend(idx)*cmd_q_depth;
	       head[idx] <= new_head;
	       //$display("new_head %d %d", idx, new_head);
	    end
	    let new_cmd = MemengineCmd{pointer:cmd.pointer, base:cmd.base+extend(bl), burstLen:cmd.burstLen, len:cmd.len-extend(bl)};
	    cmdBuf.portA.request.put(BRAMRequest{write:True, responseOnWrite:False, address:head[idx], datain:new_cmd});
	    workf.enq(tuple3(truncate(bl>>beat_shift), idx, last));
	    //$display("readReq %d, %h %h %h", idx, cmd.base, bl, last);
	    return ObjectRequest { pointer: cmd.pointer, offset: cmd.base, burstLen:bl, tag: 0 };
	 endmethod
      endinterface
      interface Put readData;
	 method Action put(ObjectData#(dataWidth) d);
	    match {.rc, .idx, .last} = workf.first;
	    let new_respCnt = respCnt+1;
	    let l = False;
	    //$display("%h %d", d.data, idx);
	    if (new_respCnt == rc) begin
	       respCnt <= 0;
	       workf.deq;
	       //$display("eob %d", idx);
	       l = last;
	    end
	    else begin
	       respCnt <= new_respCnt;
	    end
	    read_data.enq(tuple2(idx,tuple2(d.data,l)));
	 endmethod
      endinterface
   endinterface
   interface dataPipes = read_data_pipes;
endmodule


add unspecified fields to BRAMRequest

add unspecified fields to BRAMRequest

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

// BSV Libraries
import BRAMFIFO::*;
import FIFO::*;
import FIFOF::*;
import Vector::*;
import GetPut::*;
import ClientServer::*;
import Assert::*;
import BRAM::*;

// XBSV Libraries
import MemTypes::*;
import PortalMemory::*;
import SGList::*;

typedef 9 SGL_PIPELINE_DEPTH;

interface TagGen#(numeric type numTags);
   method ActionValue#(Bit#(TLog#(numTags))) getTag;
   method Action returnTag(Bit#(TLog#(numTags)) tag);
   method ActionValue#(Bit#(TLog#(numTags))) complete;
endinterface

module mkTagGen(TagGen#(numTags))
   provisos(Log#(numTags,tsz));

   BRAM_Configure cfg = defaultValue;
   cfg.outFIFODepth = 1;
   BRAM2Port#(Bit#(tsz),Bool) tags <- mkBRAM2Server(cfg);
   Reg#(Bit#(tsz))        head_ptr <- mkReg(0);
   Reg#(Bit#(tsz))        tail_ptr <- mkReg(0);
   Reg#(Bool)               inited <- mkReg(False);
   FIFO#(Bit#(tsz))      comp_fifo <- mkFIFO;
   Reg#(Bit#(numTags))  comp_state <- mkReg(0);

   let retFifo <- mkFIFO;

   rule complete_rule0 (comp_state[0] != 0);
      tags.portB.request.put(BRAMRequest{write:False, address:tail_ptr, responseOnWrite: ?, datain: ?});
   endrule

   rule complete_rule1 (comp_state[0] != 0);
      let rv <- tags.portB.response.get;
      if (!rv) begin
	 tail_ptr <= tail_ptr+1;
	 comp_state <= comp_state >> 1;
	 comp_fifo.enq(tail_ptr);
      end
   endrule

   // this used to be in the body of returnTag, but form some reason bsc does not
   // consider access to portA and portB to be conflict free **sigh**
   rule ret;
      let tag <- toGet(retFifo).get;
      tags.portB.request.put(BRAMRequest{write:True, responseOnWrite:False, address:tag, datain:False});
      comp_state <= 1 | (comp_state << 1);
   endrule

   rule init(!inited);
      tags.portA.request.put(BRAMRequest{write:True,address:head_ptr,responseOnWrite:False,datain:False});
      head_ptr <= head_ptr+1;
      inited <= head_ptr+1==0;
   endrule

   method ActionValue#(Bit#(tsz)) getTag() if (inited && (head_ptr+1 != tail_ptr));
      tags.portA.request.put(BRAMRequest{write:True, responseOnWrite:False, address:head_ptr, datain:True});
      head_ptr <= head_ptr+1;
      return head_ptr;
   endmethod

   method Action returnTag(Bit#(tsz) tag) if (inited);
      retFifo.enq(tag);
   endmethod

   method ActionValue#(Bit#(tsz)) complete;
      comp_fifo.deq;
      return comp_fifo.first;
   endmethod

endmodule

interface MemWriteInternal#(numeric type addrWidth, numeric type dataWidth);
   interface DmaDbg dbg;
   interface MemWriteClient#(addrWidth,dataWidth) write_client;
endinterface

interface MemReadInternal#(numeric type addrWidth, numeric type dataWidth);
   interface DmaDbg dbg;
   interface MemReadClient#(addrWidth,dataWidth) read_client;
endinterface

function Bool bad_pointer(ObjectPointer p);
   return ((p[15:0] >>8) > fromInteger(valueOf(MaxNumSGLists)));
endfunction

typedef struct {ObjectRequest req;
		Bit#(TLog#(numClients)) client; } LRec#(numeric type numClients, numeric type addrWidth) deriving(Bits);

typedef struct {ObjectRequest req;
		Bit#(addrWidth) pa;
		Bit#(6) rename_tag;
		Bit#(TLog#(numClients)) client; } RRec#(numeric type numClients, numeric type addrWidth) deriving(Bits);

typedef struct {ObjectRequest req;
		Bit#(6) rename_tag;
		Bit#(TLog#(numClients)) client;
		Bool last; } DRec#(numeric type numClients, numeric type addrWidth) deriving(Bits);

typedef struct {Bit#(6) orig_tag;
		Bit#(TLog#(numClients)) client; } RResp#(numeric type numClients, numeric type addrWidth) deriving(Bits);

typedef struct {DmaErrorType errorType;
		Bit#(32) pref; } DmaError deriving (Bits);

typedef 32 NumTags;

module mkMemReadInternal#(Vector#(numClients, ObjectReadClient#(dataWidth)) readClients,
			  DmaDebugIndication dmaIndication,
			  Vector#(numSGLs,Server#(ReqTup,Bit#(addrWidth))) sgls)
   (MemReadInternal#(addrWidth, dataWidth))

   provisos(Add#(b__, addrWidth, 64),
	    Add#(c__, 12, addrWidth),
	    Add#(1, c__, d__),
	    Div#(dataWidth,8,dataWidthBytes),
	    Mul#(dataWidthBytes,8,dataWidth),
	    Log#(dataWidthBytes,beatShift),
	    Add#(a__, TLog#(numClients), 6),
	    Add#(beatShift, e__, 8)
      );

   // stage 0: address translation (latency = SGL_PIPELINE_DEPTH)
   FIFO#(LRec#(numClients,addrWidth)) lreqFifo <- mkSizedFIFO(valueOf(SGL_PIPELINE_DEPTH));
   // stage 1: address validation (latency = 1)
   FIFO#(RRec#(numClients,addrWidth))  reqFifo <- mkFIFO;
   // stage 2: read commands
   BRAM2Port#(Bit#(TLog#(NumTags)), DRec#(numClients,addrWidth)) dreqFifos <- mkBRAM2Server(defaultValue);
   BRAM2Port#(Bit#(TAdd#(TLog#(NumTags),TSub#(8,beatShift))), ObjectData#(dataWidth)) read_buffer <- mkBRAM2Server(defaultValue);
   // stage 3: read data
   FIFO#(MemData#(dataWidth)) readDataPipelineFifo <- mkFIFO;

   let debug = False;

   Reg#(Bit#(8))           burstReg <- mkReg(0);
   Reg#(Bool)              firstReg <- mkReg(True);
   Reg#(Bool)               lastReg <- mkReg(False);
   Reg#(Bit#(64))  beatCount <- mkReg(0);
   let beat_shift = fromInteger(valueOf(beatShift));
   TagGen#(NumTags) tag_gen <- mkTagGen;

   Reg#(Bit#(8))                 compReg0 <- mkReg(0);
   Reg#(Bit#(TLog#(NumTags)))    compReg1 <- mkReg(0);
   Reg#(Bit#(TLog#(numClients))) compReg2 <- mkReg(0);
   FIFO#(Bit#(TLog#(numClients)))compFifo0 <- mkFIFO;
   FIFO#(Bit#(TLog#(NumTags)))   compFifo1 <- mkFIFO;

   // performance analytics
   Reg#(Bit#(64)) cycle_cnt <- mkReg(0);
   Reg#(Bit#(64)) last_loadClient <- mkReg(0);
   Reg#(Bit#(64)) last_sglResp <- mkReg(0);
   Reg#(Bit#(64)) last_comp <- mkReg(0);
   Reg#(Bit#(64)) last_readReq <- mkReg(0);
   Reg#(Bit#(64)) last_readData <- mkReg(0);
   (* fire_when_enabled *)
   rule cycle;
      cycle_cnt <= cycle_cnt+1;
   endrule

   FIFO#(DmaError) dmaErrorFifo <- mkFIFO();
   rule dmaError;
      let error <- toGet(dmaErrorFifo).get();
      dmaIndication.error(extend(pack(error.errorType)), error.pref, 0, 0);
   endrule

   for (Integer selectReg = 0; selectReg < valueOf(numClients); selectReg = selectReg + 1)
      rule loadClient;
	 last_loadClient <= cycle_cnt;
   	 ObjectRequest req <- readClients[selectReg].readReq.get();
	 let sglsel = req.pointer[31:16];
      	 if (debug) $display("mkMemReadInternal::loadClient %d %d %d", selectReg, sglsel, cycle_cnt-last_loadClient);
   	 if (bad_pointer(req.pointer))
	    dmaErrorFifo.enq(DmaError { errorType: DmaErrorBadPointer4, pref: req.pointer });
   	 else begin
   	    lreqFifo.enq(LRec{req:req, client:fromInteger(selectReg)});
   	    sgls[sglsel].request.put(ReqTup{id:truncate(req.pointer),off:req.offset});
   	 end
      endrule

   rule complete_burst0;
      let tag <- tag_gen.complete;
      dreqFifos.portB.request.put(BRAMRequest{write:False, address:tag, responseOnWrite: ?, datain: ?});
      compFifo1.enq(tag);
   endrule

   rule complete_burst1a if (compReg0==0);
      let drq <- dreqFifos.portB.response.get;
      let cnt = drq.req.burstLen >> beat_shift;
      let cli = drq.client;
      let tag <- toGet(compFifo1).get;
      compFifo0.enq(cli);
      read_buffer.portB.request.put(BRAMRequest{write:False, address:{tag,truncate(cnt)}, responseOnWrite: ?, datain: ?});
      compReg0 <= cnt-1;
      compReg1 <= tag;
      compReg2 <= cli;
   endrule

   rule complete_burst1b if (compReg0 > 0);
      let cnt = compReg0;
      let tag = compReg1;
      let cli = compReg2;
      compFifo0.enq(cli);
      read_buffer.portB.request.put(BRAMRequest{write:False, address:{tag,truncate(cnt)}, responseOnWrite: ?, datain: ?});
      compReg0 <= cnt-1;
   endrule

   rule complete_burst2;
      let client <- toGet(compFifo0).get;
      let data <- read_buffer.portB.response.get;
      readClients[client].readData.put(data);
      if (debug) $display("mkMemReadInternal::comp %d  %x %d", client, data.data, cycle_cnt-last_comp);
      last_comp <= cycle_cnt;
   endrule

   rule checkSglResp;
      let req = lreqFifo.first.req;
      let client = lreqFifo.first.client;
      let physAddr <- sgls[req.pointer[31:16]].response.get;
      let rename_tag <- tag_gen.getTag;
      lreqFifo.deq();
      reqFifo.enq(RRec{req:req, pa:physAddr, client:client, rename_tag:extend(rename_tag)});
      if (debug) $display("checkSglResp: client=%d, rename_tag=%d", client,rename_tag);
      if (debug) $display("mkMemReadInternal::sglResp %d %d", client, cycle_cnt-last_sglResp);
      last_sglResp <= cycle_cnt;
   endrule

   rule read_data;
      let response <- toGet(readDataPipelineFifo).get();
      Bit#(6) response_tag = response.tag;
      let drq <- dreqFifos.portA.response.get;
      let req = drq.req;
      let burstLen = burstReg;
      let first =    firstReg;
      let last  =    lastReg;
      if (first) begin
	 burstLen = drq.req.burstLen >> beat_shift;
	 last = drq.last;
	 dynamicAssert(last == (burstLen==1), "Last incorrect");
	 //$display("burstLen=%d dreqFifo.first.last=%d last=%d\n", burstLen, dreqFifo.first.last, last);
      end
      Bit#(TLog#(NumTags)) tt = truncate(response_tag);
      read_buffer.portA.request.put(BRAMRequest{write:True, responseOnWrite:False, datain:ObjectData{data: response.data, tag: req.tag, last: last}, address:{tt,truncate(burstLen)}});
      if (last) begin
	 tag_gen.returnTag(truncate(response_tag));
      end
      last_readData <= cycle_cnt;
      if (debug) $display("read_data %d", cycle_cnt-last_readData);
      burstReg <= burstLen-1;
      firstReg <= (burstLen-1 == 0);
      lastReg <= (burstLen-1 == 1);
   endrule

   interface MemReadClient read_client;
      interface Get readReq;
	 method ActionValue#(MemRequest#(addrWidth)) get();
	    reqFifo.deq;
	    let req = reqFifo.first.req;
	    let physAddr = reqFifo.first.pa;
	    let client = reqFifo.first.client;
	    let rename_tag = reqFifo.first.rename_tag;
	    if (False && physAddr[31:24] != 0)
	       $display("req_ar: funny physAddr req.pointer=%d req.offset=%h physAddr=%h", req.pointer, req.offset, physAddr);
	    dreqFifos.portB.request.put(BRAMRequest{write:True, responseOnWrite:False, address:truncate(rename_tag), datain:DRec{req:req, client:client, rename_tag:rename_tag, last:(req.burstLen == fromInteger(valueOf(dataWidthBytes)))}});
	    //$display("readReq: client=%d, rename_tag=%d, physAddr=%h req.burstLen=%d beat_shift=%d last=%d", client,rename_tag,physAddr, req.burstLen, beat_shift, req.burstLen == beat_shift);
	    if (debug) $display("read_client.readReq %d", cycle_cnt-last_readReq);
	    last_readReq <= cycle_cnt;
	    return MemRequest{addr:physAddr, burstLen:req.burstLen, tag:rename_tag};
	 endmethod
      endinterface
      interface Put readData;
	 method Action put(MemData#(dataWidth) response);
	    readDataPipelineFifo.enq(response);
	    dreqFifos.portA.request.put(BRAMRequest{write:False, address:truncate(response.tag), responseOnWrite: ?, datain: ?});
	    beatCount <= beatCount+1;
	 endmethod
      endinterface
   endinterface
   interface DmaDbg dbg;
      method ActionValue#(DmaDbgRec) dbg();
	 return DmaDbgRec{x:0, y:0, z:0, w:0};
      endmethod
      method ActionValue#(Bit#(64)) getMemoryTraffic();
	 return beatCount;
      endmethod
   endinterface
endmodule

typedef 64 NumTagsW;

module mkMemWriteInternal#(Vector#(numClients, ObjectWriteClient#(dataWidth)) writeClients,
			   DmaDebugIndication dmaIndication,
			   Vector#(numSGLs,Server#(ReqTup,Bit#(addrWidth))) sgls)
   (MemWriteInternal#(addrWidth, dataWidth))

   provisos(Add#(b__, addrWidth, 64),
	    Add#(c__, 12, addrWidth),
	    Add#(1, c__, d__),
	    Div#(dataWidth,8,dataWidthBytes),
	    Mul#(dataWidthBytes,8,dataWidth),
	    Log#(dataWidthBytes,beatShift),
	    Add#(a__, TLog#(numClients), 6));

   let debug = False;

   // stage 0: address translation (latency = SGL_PIPELINE_DEPTH)
   FIFO#(LRec#(numClients,addrWidth)) lreqFifo <- mkSizedFIFO(valueOf(SGL_PIPELINE_DEPTH));
   // stage 1: address validation (latency = 1)
   FIFO#(RRec#(numClients,addrWidth))  reqFifo <- mkFIFO;
   // stage 2: write commands
   FIFO#(DRec#(numClients, addrWidth)) dreqFifo <- mkSizedBRAMFIFO(valueOf(NumTagsW));
   // stage 3: write data
   BRAM2Port#(Bit#(TLog#(NumTagsW)), RResp#(numClients,addrWidth)) respFifos <- mkBRAM2Server(defaultValue);
   TagGen#(NumTagsW) tag_gen <- mkTagGen;

   Reg#(Bit#(8)) burstReg <- mkReg(0);
   Reg#(Bool)    firstReg <- mkReg(True);
   Reg#(Bool)     lastReg <- mkReg(False);
   Reg#(Bit#(64)) beatCount <- mkReg(0);
   let beat_shift = fromInteger(valueOf(beatShift));

   Reg#(Bit#(64)) cycle_cnt <- mkReg(0);
   Reg#(Bit#(64)) last_loadClient <- mkReg(0);
   Reg#(Bit#(64)) last_sglResp <- mkReg(0);

   (* fire_when_enabled *)
   rule cycle;
      cycle_cnt <= cycle_cnt+1;
   endrule

   FIFO#(DmaError) dmaErrorFifo <- mkFIFO();
   rule dmaError;
      let error <- toGet(dmaErrorFifo).get();
      dmaIndication.error(extend(pack(error.errorType)), error.pref, 0, 0);
   endrule

   for (Integer selectReg = 0; selectReg < valueOf(numClients); selectReg = selectReg + 1)
       rule loadClient;
      	  if (debug) $display("mkMemWriteInternal::loadClient %d %d", selectReg, cycle_cnt-last_loadClient);
	  last_loadClient <= cycle_cnt;
   	  ObjectRequest req <- writeClients[selectReg].writeReq.get();
   	  if (bad_pointer(req.pointer))
	     dmaErrorFifo.enq(DmaError { errorType: DmaErrorBadPointer5, pref: req.pointer });
   	  else begin
   	     lreqFifo.enq(LRec{req:req, client:fromInteger(selectReg)});
   	     sgls[req.pointer[31:16]].request.put(ReqTup{id:truncate(req.pointer),off:req.offset});
   	  end
       endrule

   rule checkSglResp;
      let req = lreqFifo.first.req;
      let client = lreqFifo.first.client;
      let physAddr <- sgls[req.pointer[31:16]].response.get;
      let rename_tag <- tag_gen.getTag;
      lreqFifo.deq();
      reqFifo.enq(RRec{req:req, pa:physAddr, client:client, rename_tag:extend(rename_tag)});
      //if (debug) $display("checkSglResp: client=%d, rename_tag=%d", client,rename_tag);
      if (debug) $display("mkMemWriteInternal::sglResp %d %d", client, cycle_cnt-last_sglResp);
      last_sglResp <= cycle_cnt;
   endrule

   rule writeDoneComp0;
      let tag <- tag_gen.complete;
      respFifos.portB.request.put(BRAMRequest{write:False, address:tag, responseOnWrite: ?, datain: ?});
   endrule

   rule writeDoneComp1;
      let rv <- respFifos.portB.response.get;
      let client = rv.client;
      let orig_tag = rv.orig_tag;
      writeClients[client].writeDone.put(orig_tag);
   endrule

   FIFO#(MemData#(dataWidth)) memDataFifo <- mkFIFO();
   Vector#(numClients, FIFO#(ObjectData#(dataWidth))) clientWriteData <- replicateM(mkFIFO);
   // Pipeline client data:
   // The .get() operation seems to be long latency, so get it into a local FIFO
   for (Integer client = 0; client < valueOf(numClients); client = client + 1)
      rule clientdata;
	 let d <- writeClients[client].writeData.get();
	 clientWriteData[client].enq(d);
      endrule

   rule memdata;
      let client = dreqFifo.first.client;
      let req = dreqFifo.first.req;
      let rename_tag = dreqFifo.first.rename_tag;
      ObjectData#(dataWidth) tagdata <- toGet(clientWriteData[client]).get();
      let burstLen = burstReg;
      let first    = firstReg;
      let last     = lastReg;
      if (first) begin
	 burstLen = req.burstLen >> beat_shift;
	 last     = dreqFifo.first.last;
	 respFifos.portA.request.put(BRAMRequest{write:True,responseOnWrite:False, address:truncate(rename_tag), datain:RResp{orig_tag:req.tag, client:client}});
      end
      burstReg <= burstLen-1;
      firstReg <= (burstLen-1 == 0);
      lastReg  <= (burstLen-1 == 1);
      beatCount <= beatCount+1;
      if (last)
	 dreqFifo.deq();
      //$display("writeData: client=%d, rename_tag=%d", client, rename_tag);
      memDataFifo.enq(MemData { data: tagdata.data,  tag:extend(rename_tag), last: last });
   endrule

   interface MemWriteClient write_client;
      interface Get writeReq;
	 method ActionValue#(MemRequest#(addrWidth)) get();
	    let req = reqFifo.first.req;
	    let physAddr = reqFifo.first.pa;
	    let client = reqFifo.first.client;
	    let rename_tag = reqFifo.first.rename_tag;
	    reqFifo.deq;
	    dreqFifo.enq(DRec{req:req, client:client, rename_tag:rename_tag, last: (req.burstLen == fromInteger(valueOf(dataWidthBytes))) });
	    //$display("writeReq: client=%d, rename_tag=%d", client,rename_tag);
	    return MemRequest{addr:physAddr, burstLen:req.burstLen, tag:extend(rename_tag)};
	 endmethod
      endinterface
      interface Get writeData = toGet(memDataFifo);
      interface Put writeDone;
	 method Action put(Bit#(6) resp);
	    tag_gen.returnTag(truncate(resp));
	    if (debug) $display("writeDone: resp=%d", resp);
	 endmethod
      endinterface
   endinterface
   interface DmaDbg dbg;
      method ActionValue#(DmaDbgRec) dbg();
	 return DmaDbgRec{x:fromInteger(valueOf(numClients)), y:?, z:?, w:?};
      endmethod
      method ActionValue#(Bit#(64)) getMemoryTraffic();
	 return beatCount;
      endmethod
   endinterface
endmodule

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

// BSV Libraries
import BRAMFIFO::*;
import FIFO::*;
import FIFOF::*;
import Vector::*;
import GetPut::*;
import ClientServer::*;
import Assert::*;
import BRAM::*;

// XBSV Libraries
import MemTypes::*;
import PortalMemory::*;
import SGList::*;

typedef 9 SGL_PIPELINE_DEPTH;

interface TagGen#(numeric type numTags);
   method ActionValue#(Bit#(TLog#(numTags))) getTag;
   method Action returnTag(Bit#(TLog#(numTags)) tag);
   method ActionValue#(Bit#(TLog#(numTags))) complete;
endinterface

module mkTagGen(TagGen#(numTags))
   provisos(Log#(numTags,tsz));

   BRAM_Configure cfg = defaultValue;
   cfg.outFIFODepth = 1;
   BRAM2Port#(Bit#(tsz),Bool) tags <- mkBRAM2Server(cfg);
   Reg#(Bit#(tsz))        head_ptr <- mkReg(0);
   Reg#(Bit#(tsz))        tail_ptr <- mkReg(0);
   Reg#(Bool)               inited <- mkReg(False);
   FIFO#(Bit#(tsz))      comp_fifo <- mkFIFO;
   Reg#(Bit#(numTags))  comp_state <- mkReg(0);

   let retFifo <- mkFIFO;

   rule complete_rule0 (comp_state[0] != 0);
      tags.portB.request.put(BRAMRequest{write:False, address:tail_ptr, responseOnWrite: ?, datain: ?});
   endrule

   rule complete_rule1 (comp_state[0] != 0);
      let rv <- tags.portB.response.get;
      if (!rv) begin
	 tail_ptr <= tail_ptr+1;
	 comp_state <= comp_state >> 1;
	 comp_fifo.enq(tail_ptr);
      end
   endrule

   // this used to be in the body of returnTag, but form some reason bsc does not
   // consider access to portA and portB to be conflict free **sigh**
   rule ret;
      let tag <- toGet(retFifo).get;
      tags.portB.request.put(BRAMRequest{write:True, responseOnWrite:False, address:tag, datain:False, responseOnWrite: ?, datain: ?});
      comp_state <= 1 | (comp_state << 1);
   endrule

   rule init(!inited);
      tags.portA.request.put(BRAMRequest{write:True,address:head_ptr,responseOnWrite:False,datain:False, responseOnWrite: ?, datain: ?});
      head_ptr <= head_ptr+1;
      inited <= head_ptr+1==0;
   endrule

   method ActionValue#(Bit#(tsz)) getTag() if (inited && (head_ptr+1 != tail_ptr));
      tags.portA.request.put(BRAMRequest{write:True, responseOnWrite:False, address:head_ptr, datain:True, responseOnWrite: ?, datain: ?});
      head_ptr <= head_ptr+1;
      return head_ptr;
   endmethod

   method Action returnTag(Bit#(tsz) tag) if (inited);
      retFifo.enq(tag);
   endmethod

   method ActionValue#(Bit#(tsz)) complete;
      comp_fifo.deq;
      return comp_fifo.first;
   endmethod

endmodule

interface MemWriteInternal#(numeric type addrWidth, numeric type dataWidth);
   interface DmaDbg dbg;
   interface MemWriteClient#(addrWidth,dataWidth) write_client;
endinterface

interface MemReadInternal#(numeric type addrWidth, numeric type dataWidth);
   interface DmaDbg dbg;
   interface MemReadClient#(addrWidth,dataWidth) read_client;
endinterface

function Bool bad_pointer(ObjectPointer p);
   return ((p[15:0] >>8) > fromInteger(valueOf(MaxNumSGLists)));
endfunction

typedef struct {ObjectRequest req;
		Bit#(TLog#(numClients)) client; } LRec#(numeric type numClients, numeric type addrWidth) deriving(Bits);

typedef struct {ObjectRequest req;
		Bit#(addrWidth) pa;
		Bit#(6) rename_tag;
		Bit#(TLog#(numClients)) client; } RRec#(numeric type numClients, numeric type addrWidth) deriving(Bits);

typedef struct {ObjectRequest req;
		Bit#(6) rename_tag;
		Bit#(TLog#(numClients)) client;
		Bool last; } DRec#(numeric type numClients, numeric type addrWidth) deriving(Bits);

typedef struct {Bit#(6) orig_tag;
		Bit#(TLog#(numClients)) client; } RResp#(numeric type numClients, numeric type addrWidth) deriving(Bits);

typedef struct {DmaErrorType errorType;
		Bit#(32) pref; } DmaError deriving (Bits);

typedef 32 NumTags;

module mkMemReadInternal#(Vector#(numClients, ObjectReadClient#(dataWidth)) readClients,
			  DmaDebugIndication dmaIndication,
			  Vector#(numSGLs,Server#(ReqTup,Bit#(addrWidth))) sgls)
   (MemReadInternal#(addrWidth, dataWidth))

   provisos(Add#(b__, addrWidth, 64),
	    Add#(c__, 12, addrWidth),
	    Add#(1, c__, d__),
	    Div#(dataWidth,8,dataWidthBytes),
	    Mul#(dataWidthBytes,8,dataWidth),
	    Log#(dataWidthBytes,beatShift),
	    Add#(a__, TLog#(numClients), 6),
	    Add#(beatShift, e__, 8)
      );

   // stage 0: address translation (latency = SGL_PIPELINE_DEPTH)
   FIFO#(LRec#(numClients,addrWidth)) lreqFifo <- mkSizedFIFO(valueOf(SGL_PIPELINE_DEPTH));
   // stage 1: address validation (latency = 1)
   FIFO#(RRec#(numClients,addrWidth))  reqFifo <- mkFIFO;
   // stage 2: read commands
   BRAM2Port#(Bit#(TLog#(NumTags)), DRec#(numClients,addrWidth)) dreqFifos <- mkBRAM2Server(defaultValue);
   BRAM2Port#(Bit#(TAdd#(TLog#(NumTags),TSub#(8,beatShift))), ObjectData#(dataWidth)) read_buffer <- mkBRAM2Server(defaultValue);
   // stage 3: read data
   FIFO#(MemData#(dataWidth)) readDataPipelineFifo <- mkFIFO;

   let debug = False;

   Reg#(Bit#(8))           burstReg <- mkReg(0);
   Reg#(Bool)              firstReg <- mkReg(True);
   Reg#(Bool)               lastReg <- mkReg(False);
   Reg#(Bit#(64))  beatCount <- mkReg(0);
   let beat_shift = fromInteger(valueOf(beatShift));
   TagGen#(NumTags) tag_gen <- mkTagGen;

   Reg#(Bit#(8))                 compReg0 <- mkReg(0);
   Reg#(Bit#(TLog#(NumTags)))    compReg1 <- mkReg(0);
   Reg#(Bit#(TLog#(numClients))) compReg2 <- mkReg(0);
   FIFO#(Bit#(TLog#(numClients)))compFifo0 <- mkFIFO;
   FIFO#(Bit#(TLog#(NumTags)))   compFifo1 <- mkFIFO;

   // performance analytics
   Reg#(Bit#(64)) cycle_cnt <- mkReg(0);
   Reg#(Bit#(64)) last_loadClient <- mkReg(0);
   Reg#(Bit#(64)) last_sglResp <- mkReg(0);
   Reg#(Bit#(64)) last_comp <- mkReg(0);
   Reg#(Bit#(64)) last_readReq <- mkReg(0);
   Reg#(Bit#(64)) last_readData <- mkReg(0);
   (* fire_when_enabled *)
   rule cycle;
      cycle_cnt <= cycle_cnt+1;
   endrule

   FIFO#(DmaError) dmaErrorFifo <- mkFIFO();
   rule dmaError;
      let error <- toGet(dmaErrorFifo).get();
      dmaIndication.error(extend(pack(error.errorType)), error.pref, 0, 0);
   endrule

   for (Integer selectReg = 0; selectReg < valueOf(numClients); selectReg = selectReg + 1)
      rule loadClient;
	 last_loadClient <= cycle_cnt;
   	 ObjectRequest req <- readClients[selectReg].readReq.get();
	 let sglsel = req.pointer[31:16];
      	 if (debug) $display("mkMemReadInternal::loadClient %d %d %d", selectReg, sglsel, cycle_cnt-last_loadClient);
   	 if (bad_pointer(req.pointer))
	    dmaErrorFifo.enq(DmaError { errorType: DmaErrorBadPointer4, pref: req.pointer });
   	 else begin
   	    lreqFifo.enq(LRec{req:req, client:fromInteger(selectReg)});
   	    sgls[sglsel].request.put(ReqTup{id:truncate(req.pointer),off:req.offset});
   	 end
      endrule

   rule complete_burst0;
      let tag <- tag_gen.complete;
      dreqFifos.portB.request.put(BRAMRequest{write:False, address:tag});
      compFifo1.enq(tag);
   endrule

   rule complete_burst1a if (compReg0==0);
      let drq <- dreqFifos.portB.response.get;
      let cnt = drq.req.burstLen >> beat_shift;
      let cli = drq.client;
      let tag <- toGet(compFifo1).get;
      compFifo0.enq(cli);
      read_buffer.portB.request.put(BRAMRequest{write:False, address:{tag,truncate(cnt)}});
      compReg0 <= cnt-1;
      compReg1 <= tag;
      compReg2 <= cli;
   endrule

   rule complete_burst1b if (compReg0 > 0);
      let cnt = compReg0;
      let tag = compReg1;
      let cli = compReg2;
      compFifo0.enq(cli);
      read_buffer.portB.request.put(BRAMRequest{write:False, address:{tag,truncate(cnt)}});
      compReg0 <= cnt-1;
   endrule

   rule complete_burst2;
      let client <- toGet(compFifo0).get;
      let data <- read_buffer.portB.response.get;
      readClients[client].readData.put(data);
      if (debug) $display("mkMemReadInternal::comp %d  %x %d", client, data.data, cycle_cnt-last_comp);
      last_comp <= cycle_cnt;
   endrule

   rule checkSglResp;
      let req = lreqFifo.first.req;
      let client = lreqFifo.first.client;
      let physAddr <- sgls[req.pointer[31:16]].response.get;
      let rename_tag <- tag_gen.getTag;
      lreqFifo.deq();
      reqFifo.enq(RRec{req:req, pa:physAddr, client:client, rename_tag:extend(rename_tag)});
      if (debug) $display("checkSglResp: client=%d, rename_tag=%d", client,rename_tag);
      if (debug) $display("mkMemReadInternal::sglResp %d %d", client, cycle_cnt-last_sglResp);
      last_sglResp <= cycle_cnt;
   endrule

   rule read_data;
      let response <- toGet(readDataPipelineFifo).get();
      Bit#(6) response_tag = response.tag;
      let drq <- dreqFifos.portA.response.get;
      let req = drq.req;
      let burstLen = burstReg;
      let first =    firstReg;
      let last  =    lastReg;
      if (first) begin
	 burstLen = drq.req.burstLen >> beat_shift;
	 last = drq.last;
	 dynamicAssert(last == (burstLen==1), "Last incorrect");
	 //$display("burstLen=%d dreqFifo.first.last=%d last=%d\n", burstLen, dreqFifo.first.last, last);
      end
      Bit#(TLog#(NumTags)) tt = truncate(response_tag);
      read_buffer.portA.request.put(BRAMRequest{write:True, responseOnWrite:False, datain:ObjectData{data: response.data, tag: req.tag, last: last}, address:{tt,truncate(burstLen)}});
      if (last) begin
	 tag_gen.returnTag(truncate(response_tag));
      end
      last_readData <= cycle_cnt;
      if (debug) $display("read_data %d", cycle_cnt-last_readData);
      burstReg <= burstLen-1;
      firstReg <= (burstLen-1 == 0);
      lastReg <= (burstLen-1 == 1);
   endrule

   interface MemReadClient read_client;
      interface Get readReq;
	 method ActionValue#(MemRequest#(addrWidth)) get();
	    reqFifo.deq;
	    let req = reqFifo.first.req;
	    let physAddr = reqFifo.first.pa;
	    let client = reqFifo.first.client;
	    let rename_tag = reqFifo.first.rename_tag;
	    if (False && physAddr[31:24] != 0)
	       $display("req_ar: funny physAddr req.pointer=%d req.offset=%h physAddr=%h", req.pointer, req.offset, physAddr);
	    dreqFifos.portB.request.put(BRAMRequest{write:True, responseOnWrite:False, address:truncate(rename_tag), datain:DRec{req:req, client:client, rename_tag:rename_tag, last:(req.burstLen == fromInteger(valueOf(dataWidthBytes)))}});
	    //$display("readReq: client=%d, rename_tag=%d, physAddr=%h req.burstLen=%d beat_shift=%d last=%d", client,rename_tag,physAddr, req.burstLen, beat_shift, req.burstLen == beat_shift);
	    if (debug) $display("read_client.readReq %d", cycle_cnt-last_readReq);
	    last_readReq <= cycle_cnt;
	    return MemRequest{addr:physAddr, burstLen:req.burstLen, tag:rename_tag};
	 endmethod
      endinterface
      interface Put readData;
	 method Action put(MemData#(dataWidth) response);
	    readDataPipelineFifo.enq(response);
	    dreqFifos.portA.request.put(BRAMRequest{write:False, address:truncate(response.tag)});
	    beatCount <= beatCount+1;
	 endmethod
      endinterface
   endinterface
   interface DmaDbg dbg;
      method ActionValue#(DmaDbgRec) dbg();
	 return DmaDbgRec{x:0, y:0, z:0, w:0};
      endmethod
      method ActionValue#(Bit#(64)) getMemoryTraffic();
	 return beatCount;
      endmethod
   endinterface
endmodule

typedef 64 NumTagsW;

module mkMemWriteInternal#(Vector#(numClients, ObjectWriteClient#(dataWidth)) writeClients,
			   DmaDebugIndication dmaIndication,
			   Vector#(numSGLs,Server#(ReqTup,Bit#(addrWidth))) sgls)
   (MemWriteInternal#(addrWidth, dataWidth))

   provisos(Add#(b__, addrWidth, 64),
	    Add#(c__, 12, addrWidth),
	    Add#(1, c__, d__),
	    Div#(dataWidth,8,dataWidthBytes),
	    Mul#(dataWidthBytes,8,dataWidth),
	    Log#(dataWidthBytes,beatShift),
	    Add#(a__, TLog#(numClients), 6));

   let debug = False;

   // stage 0: address translation (latency = SGL_PIPELINE_DEPTH)
   FIFO#(LRec#(numClients,addrWidth)) lreqFifo <- mkSizedFIFO(valueOf(SGL_PIPELINE_DEPTH));
   // stage 1: address validation (latency = 1)
   FIFO#(RRec#(numClients,addrWidth))  reqFifo <- mkFIFO;
   // stage 2: write commands
   FIFO#(DRec#(numClients, addrWidth)) dreqFifo <- mkSizedBRAMFIFO(valueOf(NumTagsW));
   // stage 3: write data
   BRAM2Port#(Bit#(TLog#(NumTagsW)), RResp#(numClients,addrWidth)) respFifos <- mkBRAM2Server(defaultValue);
   TagGen#(NumTagsW) tag_gen <- mkTagGen;

   Reg#(Bit#(8)) burstReg <- mkReg(0);
   Reg#(Bool)    firstReg <- mkReg(True);
   Reg#(Bool)     lastReg <- mkReg(False);
   Reg#(Bit#(64)) beatCount <- mkReg(0);
   let beat_shift = fromInteger(valueOf(beatShift));

   Reg#(Bit#(64)) cycle_cnt <- mkReg(0);
   Reg#(Bit#(64)) last_loadClient <- mkReg(0);
   Reg#(Bit#(64)) last_sglResp <- mkReg(0);

   (* fire_when_enabled *)
   rule cycle;
      cycle_cnt <= cycle_cnt+1;
   endrule

   FIFO#(DmaError) dmaErrorFifo <- mkFIFO();
   rule dmaError;
      let error <- toGet(dmaErrorFifo).get();
      dmaIndication.error(extend(pack(error.errorType)), error.pref, 0, 0);
   endrule

   for (Integer selectReg = 0; selectReg < valueOf(numClients); selectReg = selectReg + 1)
       rule loadClient;
      	  if (debug) $display("mkMemWriteInternal::loadClient %d %d", selectReg, cycle_cnt-last_loadClient);
	  last_loadClient <= cycle_cnt;
   	  ObjectRequest req <- writeClients[selectReg].writeReq.get();
   	  if (bad_pointer(req.pointer))
	     dmaErrorFifo.enq(DmaError { errorType: DmaErrorBadPointer5, pref: req.pointer });
   	  else begin
   	     lreqFifo.enq(LRec{req:req, client:fromInteger(selectReg)});
   	     sgls[req.pointer[31:16]].request.put(ReqTup{id:truncate(req.pointer),off:req.offset});
   	  end
       endrule

   rule checkSglResp;
      let req = lreqFifo.first.req;
      let client = lreqFifo.first.client;
      let physAddr <- sgls[req.pointer[31:16]].response.get;
      let rename_tag <- tag_gen.getTag;
      lreqFifo.deq();
      reqFifo.enq(RRec{req:req, pa:physAddr, client:client, rename_tag:extend(rename_tag)});
      //if (debug) $display("checkSglResp: client=%d, rename_tag=%d", client,rename_tag);
      if (debug) $display("mkMemWriteInternal::sglResp %d %d", client, cycle_cnt-last_sglResp);
      last_sglResp <= cycle_cnt;
   endrule

   rule writeDoneComp0;
      let tag <- tag_gen.complete;
      respFifos.portB.request.put(BRAMRequest{write:False, address:tag});
   endrule

   rule writeDoneComp1;
      let rv <- respFifos.portB.response.get;
      let client = rv.client;
      let orig_tag = rv.orig_tag;
      writeClients[client].writeDone.put(orig_tag);
   endrule

   FIFO#(MemData#(dataWidth)) memDataFifo <- mkFIFO();
   Vector#(numClients, FIFO#(ObjectData#(dataWidth))) clientWriteData <- replicateM(mkFIFO);
   // Pipeline client data:
   // The .get() operation seems to be long latency, so get it into a local FIFO
   for (Integer client = 0; client < valueOf(numClients); client = client + 1)
      rule clientdata;
	 let d <- writeClients[client].writeData.get();
	 clientWriteData[client].enq(d);
      endrule

   rule memdata;
      let client = dreqFifo.first.client;
      let req = dreqFifo.first.req;
      let rename_tag = dreqFifo.first.rename_tag;
      ObjectData#(dataWidth) tagdata <- toGet(clientWriteData[client]).get();
      let burstLen = burstReg;
      let first    = firstReg;
      let last     = lastReg;
      if (first) begin
	 burstLen = req.burstLen >> beat_shift;
	 last     = dreqFifo.first.last;
	 respFifos.portA.request.put(BRAMRequest{write:True,responseOnWrite:False, address:truncate(rename_tag), datain:RResp{orig_tag:req.tag, client:client}});
      end
      burstReg <= burstLen-1;
      firstReg <= (burstLen-1 == 0);
      lastReg  <= (burstLen-1 == 1);
      beatCount <= beatCount+1;
      if (last)
	 dreqFifo.deq();
      //$display("writeData: client=%d, rename_tag=%d", client, rename_tag);
      memDataFifo.enq(MemData { data: tagdata.data,  tag:extend(rename_tag), last: last });
   endrule

   interface MemWriteClient write_client;
      interface Get writeReq;
	 method ActionValue#(MemRequest#(addrWidth)) get();
	    let req = reqFifo.first.req;
	    let physAddr = reqFifo.first.pa;
	    let client = reqFifo.first.client;
	    let rename_tag = reqFifo.first.rename_tag;
	    reqFifo.deq;
	    dreqFifo.enq(DRec{req:req, client:client, rename_tag:rename_tag, last: (req.burstLen == fromInteger(valueOf(dataWidthBytes))) });
	    //$display("writeReq: client=%d, rename_tag=%d", client,rename_tag);
	    return MemRequest{addr:physAddr, burstLen:req.burstLen, tag:extend(rename_tag)};
	 endmethod
      endinterface
      interface Get writeData = toGet(memDataFifo);
      interface Put writeDone;
	 method Action put(Bit#(6) resp);
	    tag_gen.returnTag(truncate(resp));
	    if (debug) $display("writeDone: resp=%d", resp);
	 endmethod
      endinterface
   endinterface
   interface DmaDbg dbg;
      method ActionValue#(DmaDbgRec) dbg();
	 return DmaDbgRec{x:fromInteger(valueOf(numClients)), y:?, z:?, w:?};
      endmethod
      method ActionValue#(Bit#(64)) getMemoryTraffic();
	 return beatCount;
      endmethod
   endinterface
endmodule