fixing DMA from fpga with size not multiple of burstlen

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
import Vector::*;
import BuildVector::*;
import Cntrs::*;
import FIFOF::*;
import FIFO::*;
import GetPut::*;
import Connectable::*;
import BRAMFIFO::*;
import ConfigCounter::*;
import MemTypes::*;
import Pipe::*;
import ConnectalConfig::*;

`include "ConnectalProjectConfig.bsv"

module mkMemWriteEngine(MemWriteEngine#(busWidth, userWidth, cmdQDepth, numServers))
   provisos( Add#(1, d__, busWidth)
	    ,Add#(1, d__, userWidth)
	    ,Add#(e__, TLog#(numServers), MemTagSize)
	    ,FunnelPipesPipelined#(1, numServers, MemData#(userWidth), 2)
	    ,FunnelPipesPipelined#(1, numServers, MemRequest, 2)
	    ,FunnelPipesPipelined#(1, numServers, Bit#(MemTagSize), 2)
	    );
   let rv <- mkMemWriteEngineBuff(valueOf(TExp#(BurstLenSize)));
   return rv;
endmodule

interface MemWriteChannel#(numeric type busWidth, numeric type userWidth, numeric type cmdQDepth);
   interface PipeIn#(Bit#(MemTagSize))        writeGnt; // grants request
   interface PipeOut#(MemRequest)             writeReq;
   interface PipeOut#(MemData#(userWidth))    writeData;
   interface PipeIn#(Bit#(MemTagSize))        writeDone;
   interface MemWriteEngineServer#(userWidth) writeServer;
endinterface

module mkMemWriteChannel#(Integer bufferSizeBytes, Integer channelNumber,
			  PipeOut#(Bit#(MemTagSize)) writeGntPipe, PipeOut#(Bit#(MemTagSize)) writeDonePipe)
   (MemWriteChannel#(busWidth, userWidth, cmdQDepth))
   provisos ( Div#(busWidth,8,busWidthBytes)
	     ,Log#(busWidthBytes,beatShift)
	     ,Add#(1, d__, userWidth)
	     ,Add#(userWidth, 0, busWidth)
	     );

   Integer bufferSizeBeats = bufferSizeBytes/valueOf(busWidthBytes);
   Reg#(Bool) load_in_progress <- mkReg(False);
   FIFO#(Tuple3#(MemengineCmd,Bool,Bool))       serverCond <- mkFIFO1();
   FIFO#(Tuple2#(Bit#(MemTagSize),MemengineCmd)) serverReq <- mkSizedFIFO(valueOf(cmdQDepth));
   FIFO#(Tuple3#(Bit#(8),Bit#(MemTagSize),Bool))inProgress <- mkSizedFIFO(valueOf(cmdQDepth));
   FIFO#(Tuple3#(Bit#(MemTagSize),Bit#(MemTagSize),Bool)) serverDone <- mkSizedFIFO(valueOf(cmdQDepth));
   FIFOF#(MemRequest)           writeReqFifo <- mkFIFOF();
   FIFOF#(MemData#(userWidth)) writeDataFifo <- mkFIFOF();

   Reg#(Bool)              clientInFlight <- mkReg(False);
   Reg#(Bool)              clientBursts <- mkReg(False);
   ConfigCounter#(16)      clientAvail <- mkConfigCounter(0);
   Reg#(MemengineCmd)      clientStart <- mkReg(unpack(0));
   FIFO#(Bool)             clientFinished <- mkSizedFIFO(1);
   FIFOF#(MemengineCmd)    clientCommand <- mkSizedFIFOF(1);
   Count#(Bit#(32))        clientCycles     <- mkCount(0);
   FIFOF#(MemRequestCycles) clientCyclesFifo <- mkFIFOF();
   FIFOF#(Bit#(userWidth)) dataBuffer <- mkSizedBRAMFIFOF(bufferSizeBeats);
   Reg#(Bit#(32)) cycles <- mkReg(0);
   rule rl_cycles;
      cycles <= cycles + 1;
   endrule

   Reg#(Bit#(8))                    respCnt <- mkReg(0);
   let beat_shift = fromInteger(valueOf(beatShift));

   rule store_cmd if (!clientInFlight);
      let cmd <- toGet(clientCommand).get();
      clientInFlight <= True;
      clientBursts <= True;
      clientStart <= cmd;
      $display("cycles %d starting request %d bytes %d", cycles, cmd.tag, cmd.len);
      clientCycles <= 0;
   endrule
   rule rule_request_cycles;
      clientCycles.incr(1);
   endrule

   rule load_ctxt_a if (!load_in_progress);
      if (clientBursts) begin
	 load_in_progress <= True;
	 let cmd = clientStart;
	 let cond1 = cmd.len <= extend(cmd.burstLen);
         Bool cond0 = False;
         if (cond1) begin
            cond0 <- clientAvail.maybeDecrement(unpack(truncate(cmd.len>>beat_shift)));
         end
         else begin
            cond0 <- clientAvail.maybeDecrement(unpack(extend(cmd.burstLen>>beat_shift)));
         end
	 serverCond.enq(tuple3(cmd,cond0,cond1));
      end
   endrule

   rule load_ctxt_b if (load_in_progress);
      load_in_progress <= False;
      match {.cmd,.cond0,.cond1} <- toGet(serverCond).get;
      if  (cond0) begin
	 //$display("load_ctxt_b cycles %d %h", cycles, cmd.base);
	 serverReq.enq(tuple2(0,cmd));
	 if (cond1) begin
	    clientBursts <= False;
	 end
	 else begin
	    clientStart <= MemengineCmd{sglId:cmd.sglId, base:cmd.base+extend(cmd.burstLen),
               burstLen:cmd.burstLen, len:cmd.len-extend(cmd.burstLen), tag:cmd.tag};
	 end
      end
   endrule

   rule rlWriteReq;
      match {.idx, .cmd} <- toGet(serverReq).get;
      Bit#(BurstLenSize) bl = cmd.burstLen;
      Bool last = False;
      if (cmd.len <= extend(bl)) begin
	 last = True;
	 bl = truncate(cmd.len);
      end
      inProgress.enq(tuple3(truncate(bl>>beat_shift), cmd.tag, last));
      //$display("writeReq %d, %h %h %h", channelNumber, cmd.base, bl, last);
      writeReqFifo.enq(MemRequest { sglId: cmd.sglId, offset: extend(cmd.base), burstLen:bl, tag: fromInteger(channelNumber)});
   endrule

   rule rlWriteData;
      match {.rc, .client_tag, .last} = inProgress.first;
      //let gnt = writeGntPipe.first;
      let new_respCnt = respCnt+1;
      let lastBeat = False;
      if (new_respCnt == rc) begin
	 respCnt <= 0;
	 inProgress.deq();
	 //writeGntPipe.deq();
	 serverDone.enq(tuple3(0,client_tag,last));
	 lastBeat = True;
      end
      else begin
	 respCnt <= new_respCnt;
      end
      let wd <- toGet(dataBuffer).get();
      writeDataFifo.enq(MemData{data:wd, tag:fromInteger(channelNumber), last:lastBeat});
   endrule

   rule rlWriteDone;
      let tag <- toGet(writeDonePipe).get();
      match {.idx, .req_tag, .last} <- toGet(serverDone).get;
      if (last) begin
	 clientInFlight <= False;
	 clientFinished.enq(True);
`ifdef MEMENGINE_REQUEST_CYCLES
	 $display("cycles %d req_tag %d clientCycles = %d", cycles, req_tag, clientCycles);
	 clientCyclesFifo.enq(MemRequestCycles { tag: req_tag, cycles: clientCycles });
`endif
      end
      //$display("writeDone %d %d", channelNumber, last);
   endrule

   MemWriteEngineServer#(userWidth) ws = (interface MemWriteEngineServer#(userWidth);
      interface Put request;
	 method Action put(MemengineCmd cmd);
	    Bit#(32) bsb = fromInteger(bufferSizeBytes);
      `ifdef SIMULATION
	    Bit#(32) dw = fromInteger(valueOf(busWidthBytes));
	    Bit#(32) bl = extend(cmd.burstLen);
	    // this is because bsc lifts the divide operation (below)
	    // and on startup the simulator gets a floating-point exception
	    if (bl ==0)
	       bl = 1;
	    let mdw1 = ((cmd.len)/dw)*dw != cmd.len;
	    let bbl = extend(cmd.burstLen) > bsb;
	    if(bbl || mdw1 || cmd.len == 0) begin
	       if (bbl)
					  $display("XXXXXXXXXX mkMemWriteEngineBuff::unsupported burstLen %d %d", bsb, cmd.burstLen);
	       if (mdw1 || cmd.len == 0)
					  $display("XXXXXXXXXX mkMemWriteEngineBuff::unsupported len %h mdw1=%d", cmd.len, mdw1);
	    end
	    else
      `endif
	       begin
		  clientCommand.enq(cmd);
		  $display("(%d) %h %h %h", channelNumber, cmd.base, cmd.len, cmd.burstLen);
	       end
	 endmethod
      endinterface
      interface Get done;
	 method ActionValue#(Bool) get = toGet(clientFinished).get;
      endinterface
      interface PipeIn data = interface PipeIn;
				 method Bool notFull = dataBuffer.notFull;
   				 method Action enq(Bit#(userWidth) v);
				    dataBuffer.enq(v);
				    clientAvail.increment(1);
				 endmethod
			      endinterface;
	 interface PipeOut requestCycles = toPipeOut(clientCyclesFifo);
	 endinterface);
   interface writeServer = ws;
   interface writeReq = toPipeOut(writeReqFifo);
   interface writeData = toPipeOut(writeDataFifo);
endmodule

module mkMemWriteEngineBuff#(Integer bufferSizeBytes)(MemWriteEngine#(busWidth, userWidth, cmdQDepth, numServers))
   provisos ( Div#(busWidth,8,busWidthBytes)
	     ,Log#(busWidthBytes,beatShift)
	     ,Add#(1, a__, userWidth)
	     ,Add#(userWidth, 0, busWidth)
	     ,Add#(b__, TLog#(numServers), MemTagSize)
             ,FunnelPipesPipelined#(1, numServers, MemData#(userWidth), 2)
             ,FunnelPipesPipelined#(1, numServers, MemRequest, 2)
             ,FunnelPipesPipelined#(1, numServers, Bit#(MemTagSize), 2)
	     );

   FIFOF#(Bit#(MemTagSize)) writeDoneFifo <- mkFIFOF();
   function Tuple2#(Bit#(TLog#(numServers)),Bit#(MemTagSize)) tagDone(Bit#(MemTagSize) tag);
      return tuple2(truncate(tag), tag);
   endfunction
`ifdef ARB_FUNNEL
   FIFOF#(Bit#(MemTagSize))       arbFifo <- mkFIFOF1();
   UnFunnelPipe#(1,numServers,Bit#(MemTagSize),2) arbPipes <- mkUnFunnelPipesPipelined(vec(mapPipe(tagDone, toPipeOut(arbFifo))));
   UnFunnelPipe#(1,numServers,Bit#(MemTagSize),2) donePipes <- mkUnFunnelPipesPipelined(vec(mapPipe(tagDone, toPipeOut(writeDoneFifo))));
`else
   Vector#(numServers, FIFOF#(Bit#(MemTagSize)))   arbFifos <- replicateM(mkFIFOF);
   Vector#(numServers, PipeOut#(Bit#(MemTagSize))) arbPipes = map(toPipeOut, arbFifos);
   Vector#(numServers, FIFOF#(Bit#(MemTagSize)))   doneFifos <- replicateM(mkFIFOF);
   Vector#(numServers, PipeOut#(Bit#(MemTagSize))) donePipes = map(toPipeOut, doneFifos);
`endif
   Vector#(numServers, MemWriteChannel#(busWidth,userWidth,cmdQDepth)) writeChannels <- zipWith3M(mkMemWriteChannel(bufferSizeBytes),
												  genVector(),
												  arbPipes,
												  donePipes);
   function PipeOut#(MemRequest) writeChannelDmaWriteReq(Integer i);
      return writeChannels[i].writeReq;
   endfunction
   function PipeOut#(MemData#(userWidth)) writeChannelDmaWriteData(Integer i);
      return writeChannels[i].writeData;
   endfunction
   function MemWriteEngineServer#(userWidth) writeChannelServer(Integer i);
      return writeChannels[i].writeServer;
   endfunction

   Reg#(Bool)         reqInFlight <- mkReg(False);
   Reg#(Bool)         reqNotDone  <- mkReg(False);
   Reg#(Bit#(TLog#(numServers))) currentChannel <- mkReg(0);
   FIFOF#(MemRequest)          writeReqFifo <- mkFIFOF();
   FIFOF#(MemData#(userWidth)) writeDataFifo <- mkSizedFIFOF(16);
   FunnelPipe#(1,numServers,MemRequest,2) reqFunnel <- mkFunnelPipesPipelined(genWith(writeChannelDmaWriteReq));
//   FunnelPipe#(1,numServers,MemData#(userWidth),2) dataFunnel <- mkFunnelPipesPipelined(genWith(writeChannelDmaWriteData));

   rule rl_arbitration if (!reqInFlight && !reqNotDone);
      let req <- toGet(reqFunnel[0]).get();
      // tag is channel number
      currentChannel <= truncate(req.tag);
      reqNotDone  <= True;
      writeReqFifo.enq(req);
      reqInFlight <= True;
   endrule
   rule rl_writeData if (reqInFlight);
      MemData#(userWidth) md <- toGet(writeChannels[currentChannel].writeData).get();
      if (md.last)
	 reqInFlight <= False;
      writeDataFifo.enq(md);
   endrule

   rule rl_writeDone;
      let tag <- toGet(writeDoneFifo).get();
      doneFifos[tag].enq(tag);
      reqNotDone  <= False;
   endrule

   interface writeServers = genWith(writeChannelServer);
   interface MemWriteClient dmaClient;
      interface writeReq = toGet(writeReqFifo);
      interface writeData = toGet(writeDataFifo);
      interface writeDone = toPut(writeDoneFifo);
   endinterface: dmaClient

endmodule

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
import Vector::*;
import BuildVector::*;
import Cntrs::*;
import FIFOF::*;
import FIFO::*;
import GetPut::*;
import Connectable::*;
import BRAMFIFO::*;
import ConfigCounter::*;
import MemTypes::*;
import Pipe::*;
import ConnectalConfig::*;

`include "ConnectalProjectConfig.bsv"

module mkMemWriteEngine(MemWriteEngine#(busWidth, userWidth, cmdQDepth, numServers))
   provisos( Add#(1, d__, busWidth)
	    ,Add#(1, d__, userWidth)
	    ,Add#(e__, TLog#(numServers), MemTagSize)
	    ,FunnelPipesPipelined#(1, numServers, MemData#(userWidth), 2)
	    ,FunnelPipesPipelined#(1, numServers, MemRequest, 2)
	    ,FunnelPipesPipelined#(1, numServers, Bit#(MemTagSize), 2)
	    );
   let rv <- mkMemWriteEngineBuff(valueOf(TExp#(BurstLenSize)));
   return rv;
endmodule

interface MemWriteChannel#(numeric type busWidth, numeric type userWidth, numeric type cmdQDepth);
   interface PipeIn#(Bit#(MemTagSize))        writeGnt; // grants request
   interface PipeOut#(MemRequest)             writeReq;
   interface PipeOut#(MemData#(userWidth))    writeData;
   interface PipeIn#(Bit#(MemTagSize))        writeDone;
   interface MemWriteEngineServer#(userWidth) writeServer;
endinterface

module mkMemWriteChannel#(Integer bufferSizeBytes, Integer channelNumber,
			  PipeOut#(Bit#(MemTagSize)) writeGntPipe, PipeOut#(Bit#(MemTagSize)) writeDonePipe)
   (MemWriteChannel#(busWidth, userWidth, cmdQDepth))
   provisos ( Div#(busWidth,8,busWidthBytes)
	     ,Log#(busWidthBytes,beatShift)
	     ,Add#(1, d__, userWidth)
	     ,Add#(userWidth, 0, busWidth)
	     );

   Integer bufferSizeBeats = bufferSizeBytes/valueOf(busWidthBytes);
   Reg#(Bool) load_in_progress <- mkReg(False);
   FIFO#(Tuple3#(MemengineCmd,Bool,Bool))       serverCond <- mkFIFO1();
   FIFO#(Tuple2#(Bit#(MemTagSize),MemengineCmd)) serverReq <- mkSizedFIFO(valueOf(cmdQDepth));
   FIFO#(Tuple3#(Bit#(8),Bit#(MemTagSize),Bool))inProgress <- mkSizedFIFO(valueOf(cmdQDepth));
   FIFO#(Tuple3#(Bit#(MemTagSize),Bit#(MemTagSize),Bool)) serverDone <- mkSizedFIFO(valueOf(cmdQDepth));
   FIFOF#(MemRequest)           writeReqFifo <- mkFIFOF();
   FIFOF#(MemData#(userWidth)) writeDataFifo <- mkFIFOF();

   Reg#(Bool)              clientInFlight <- mkReg(False);
   Reg#(Bool)              clientBursts <- mkReg(False);
   ConfigCounter#(16)      clientAvail <- mkConfigCounter(0);
   Reg#(MemengineCmd)      clientStart <- mkReg(unpack(0));
   FIFO#(Bool)             clientFinished <- mkSizedFIFO(1);
   FIFOF#(MemengineCmd)    clientCommand <- mkSizedFIFOF(1);
   Count#(Bit#(32))        clientCycles     <- mkCount(0);
   FIFOF#(MemRequestCycles) clientCyclesFifo <- mkFIFOF();
   FIFOF#(Bit#(userWidth)) dataBuffer <- mkSizedBRAMFIFOF(bufferSizeBeats);
   Reg#(Bit#(32)) cycles <- mkReg(0);
   rule rl_cycles;
      cycles <= cycles + 1;
   endrule

   Reg#(Bit#(8))                    respCnt <- mkReg(0);
   let beat_shift = fromInteger(valueOf(beatShift));

   rule store_cmd if (!clientInFlight);
      let cmd <- toGet(clientCommand).get();
      clientInFlight <= True;
      clientBursts <= True;
      clientStart <= cmd;
      $display("cycles %d starting request %d bytes %d", cycles, cmd.tag, cmd.len);
      clientCycles <= 0;
   endrule
   rule rule_request_cycles;
      clientCycles.incr(1);
   endrule

   rule load_ctxt_a if (!load_in_progress);
      if (clientBursts) begin
	 load_in_progress <= True;
	 let cmd = clientStart;
	 let cond0 <- clientAvail.maybeDecrement(unpack(extend(cmd.burstLen>>beat_shift)));
	 let cond1 = cmd.len <= extend(cmd.burstLen);
	 serverCond.enq(tuple3(cmd,cond0,cond1));
      end
   endrule

   rule load_ctxt_b if (load_in_progress);
      load_in_progress <= False;
      match {.cmd,.cond0,.cond1} <- toGet(serverCond).get;
      if  (cond0) begin
	 //$display("load_ctxt_b cycles %d %h", cycles, cmd.base);
	 serverReq.enq(tuple2(0,cmd));
	 if (cond1) begin
	    clientBursts <= False;
	 end
	 else begin
	    clientStart <= MemengineCmd{sglId:cmd.sglId, base:cmd.base+extend(cmd.burstLen),
               burstLen:cmd.burstLen, len:cmd.len-extend(cmd.burstLen), tag:cmd.tag};
	 end
      end
   endrule

   rule rlWriteReq;
      match {.idx, .cmd} <- toGet(serverReq).get;
      Bit#(BurstLenSize) bl = cmd.burstLen;
      Bool last = False;
      if (cmd.len <= extend(bl)) begin
	 last = True;
	 bl = truncate(cmd.len);
      end
      inProgress.enq(tuple3(truncate(bl>>beat_shift), cmd.tag, last));
      //$display("writeReq %d, %h %h %h", channelNumber, cmd.base, bl, last);
      writeReqFifo.enq(MemRequest { sglId: cmd.sglId, offset: extend(cmd.base), burstLen:bl, tag: fromInteger(channelNumber)});
   endrule

   rule rlWriteData;
      match {.rc, .client_tag, .last} = inProgress.first;
      //let gnt = writeGntPipe.first;
      let new_respCnt = respCnt+1;
      let lastBeat = False;
      if (new_respCnt == rc) begin
	 respCnt <= 0;
	 inProgress.deq();
	 //writeGntPipe.deq();
	 serverDone.enq(tuple3(0,client_tag,last));
	 lastBeat = True;
      end
      else begin
	 respCnt <= new_respCnt;
      end
      let wd <- toGet(dataBuffer).get();
      writeDataFifo.enq(MemData{data:wd, tag:fromInteger(channelNumber), last:lastBeat});
   endrule

   rule rlWriteDone;
      let tag <- toGet(writeDonePipe).get();
      match {.idx, .req_tag, .last} <- toGet(serverDone).get;
      if (last) begin
	 clientInFlight <= False;
	 clientFinished.enq(True);
`ifdef MEMENGINE_REQUEST_CYCLES
	 $display("cycles %d req_tag %d clientCycles = %d", cycles, req_tag, clientCycles);
	 clientCyclesFifo.enq(MemRequestCycles { tag: req_tag, cycles: clientCycles });
`endif
      end
      //$display("writeDone %d %d", channelNumber, last);
   endrule

   MemWriteEngineServer#(userWidth) ws = (interface MemWriteEngineServer#(userWidth);
      interface Put request;
	 method Action put(MemengineCmd cmd);
	    Bit#(32) bsb = fromInteger(bufferSizeBytes);
      `ifdef SIMULATION
	    Bit#(32) dw = fromInteger(valueOf(busWidthBytes));
	    Bit#(32) bl = extend(cmd.burstLen);
	    // this is because bsc lifts the divide operation (below)
	    // and on startup the simulator gets a floating-point exception
	    if (bl ==0)
	       bl = 1;
	    let mdw1 = ((cmd.len)/dw)*dw != cmd.len;
	    let bbl = extend(cmd.burstLen) > bsb;
	    if(bbl || mdw1 || cmd.len == 0) begin
	       if (bbl)
					  $display("XXXXXXXXXX mkMemWriteEngineBuff::unsupported burstLen %d %d", bsb, cmd.burstLen);
	       if (mdw1 || cmd.len == 0)
					  $display("XXXXXXXXXX mkMemWriteEngineBuff::unsupported len %h mdw1=%d", cmd.len, mdw1);
	    end
	    else
      `endif
	       begin
		  clientCommand.enq(cmd);
		  $display("(%d) %h %h %h", channelNumber, cmd.base, cmd.len, cmd.burstLen);
	       end
	 endmethod
      endinterface
      interface Get done;
	 method ActionValue#(Bool) get = toGet(clientFinished).get;
      endinterface
      interface PipeIn data = interface PipeIn;
				 method Bool notFull = dataBuffer.notFull;
   				 method Action enq(Bit#(userWidth) v);
				    dataBuffer.enq(v);
				    clientAvail.increment(1);
				 endmethod
			      endinterface;
	 interface PipeOut requestCycles = toPipeOut(clientCyclesFifo);
	 endinterface);
   interface writeServer = ws;
   interface writeReq = toPipeOut(writeReqFifo);
   interface writeData = toPipeOut(writeDataFifo);
endmodule

module mkMemWriteEngineBuff#(Integer bufferSizeBytes)(MemWriteEngine#(busWidth, userWidth, cmdQDepth, numServers))
   provisos ( Div#(busWidth,8,busWidthBytes)
	     ,Log#(busWidthBytes,beatShift)
	     ,Add#(1, a__, userWidth)
	     ,Add#(userWidth, 0, busWidth)
	     ,Add#(b__, TLog#(numServers), MemTagSize)
             ,FunnelPipesPipelined#(1, numServers, MemData#(userWidth), 2)
             ,FunnelPipesPipelined#(1, numServers, MemRequest, 2)
             ,FunnelPipesPipelined#(1, numServers, Bit#(MemTagSize), 2)
	     );

   FIFOF#(Bit#(MemTagSize)) writeDoneFifo <- mkFIFOF();
   function Tuple2#(Bit#(TLog#(numServers)),Bit#(MemTagSize)) tagDone(Bit#(MemTagSize) tag);
      return tuple2(truncate(tag), tag);
   endfunction
`ifdef ARB_FUNNEL
   FIFOF#(Bit#(MemTagSize))       arbFifo <- mkFIFOF1();
   UnFunnelPipe#(1,numServers,Bit#(MemTagSize),2) arbPipes <- mkUnFunnelPipesPipelined(vec(mapPipe(tagDone, toPipeOut(arbFifo))));
   UnFunnelPipe#(1,numServers,Bit#(MemTagSize),2) donePipes <- mkUnFunnelPipesPipelined(vec(mapPipe(tagDone, toPipeOut(writeDoneFifo))));
`else
   Vector#(numServers, FIFOF#(Bit#(MemTagSize)))   arbFifos <- replicateM(mkFIFOF);
   Vector#(numServers, PipeOut#(Bit#(MemTagSize))) arbPipes = map(toPipeOut, arbFifos);
   Vector#(numServers, FIFOF#(Bit#(MemTagSize)))   doneFifos <- replicateM(mkFIFOF);
   Vector#(numServers, PipeOut#(Bit#(MemTagSize))) donePipes = map(toPipeOut, doneFifos);
`endif
   Vector#(numServers, MemWriteChannel#(busWidth,userWidth,cmdQDepth)) writeChannels <- zipWith3M(mkMemWriteChannel(bufferSizeBytes),
												  genVector(),
												  arbPipes,
												  donePipes);
   function PipeOut#(MemRequest) writeChannelDmaWriteReq(Integer i);
      return writeChannels[i].writeReq;
   endfunction
   function PipeOut#(MemData#(userWidth)) writeChannelDmaWriteData(Integer i);
      return writeChannels[i].writeData;
   endfunction
   function MemWriteEngineServer#(userWidth) writeChannelServer(Integer i);
      return writeChannels[i].writeServer;
   endfunction

   Reg#(Bool)         reqInFlight <- mkReg(False);
   Reg#(Bool)         reqNotDone  <- mkReg(False);
   Reg#(Bit#(TLog#(numServers))) currentChannel <- mkReg(0);
   FIFOF#(MemRequest)          writeReqFifo <- mkFIFOF();
   FIFOF#(MemData#(userWidth)) writeDataFifo <- mkSizedFIFOF(16);
   FunnelPipe#(1,numServers,MemRequest,2) reqFunnel <- mkFunnelPipesPipelined(genWith(writeChannelDmaWriteReq));
//   FunnelPipe#(1,numServers,MemData#(userWidth),2) dataFunnel <- mkFunnelPipesPipelined(genWith(writeChannelDmaWriteData));

   rule rl_arbitration if (!reqInFlight && !reqNotDone);
      let req <- toGet(reqFunnel[0]).get();
      // tag is channel number
      currentChannel <= truncate(req.tag);
      reqNotDone  <= True;
      writeReqFifo.enq(req);
      reqInFlight <= True;
   endrule
   rule rl_writeData if (reqInFlight);
      MemData#(userWidth) md <- toGet(writeChannels[currentChannel].writeData).get();
      if (md.last)
	 reqInFlight <= False;
      writeDataFifo.enq(md);
   endrule

   rule rl_writeDone;
      let tag <- toGet(writeDoneFifo).get();
      doneFifos[tag].enq(tag);
      reqNotDone  <= False;
   endrule

   interface writeServers = genWith(writeChannelServer);
   interface MemWriteClient dmaClient;
      interface writeReq = toGet(writeReqFifo);
      interface writeData = toGet(writeDataFifo);
      interface writeDone = toPut(writeDoneFifo);
   endinterface: dmaClient

endmodule


Fixed an issue in CPU_Fetch_C.bsv regarding a mem fault on a 16-bit-aligned FETCH.

The 'lookahead' logic to handle upper 16-bits of a 32 bit instr that
is 16-bit aligned but not 32-bit aligned was not doing the right thing
if the first 16-bits itself raised a mem exception (e.g., page fault).

// Copyright (c) 2016-2020 Bluespec, Inc. All Rights Reserved

package CPU_Fetch_C;

// ================================================================
// mkCPU_Fetch_C is an IMem_IFC->Module#(IMem_IFC) wrapper.

// The input parameter interface is for 32-bit-aligned 32-bit instruction-fetches.

// This module completely encapsulates the logic needed to extend it to
// 16-bit "compressed" instructions ('C' extension)
// which then also includes 32-bit instructions which may be only 16-bit-aligned,
// since 32b and 16b instructions can be freely mixed.

// ================================================================
// Exports

export mkCPU_Fetch_C;

// ================================================================
// BSV lib imports

import GetPut       :: *;
import ClientServer :: *;

// ----------------
// BSV additional libs

import Cur_Cycle :: *;

// ================================================================
// Project imports

import ISA_Decls    :: *;
import Near_Mem_IFC :: *;

// ================================================================
// Functions concerning address-alignment of instructions.
// 'Even': aligned to an even 16b boundary (so, also aligned to 32b boundary)
// 'Odd':  aligned to an odd  16b boundary (so, not  aligned to 32b boundary)

function Bool is_addr_even16 (WordXL pc);
   return (pc [1:0] == 2'b00);
endfunction

function Bool is_addr_odd16 (WordXL pc);
   return (pc [1:0] != 2'b00);
endfunction

function WordXL fn_to_b32_addr (WordXL addr);
   return ((addr >> 2) << 2);
endfunction

// Equality of 32-bit aligned addresses containing a1 and a2 (i.e., ignoring [1:0])
function Bool eq_b32_addr (WordXL a1, WordXL a2) = ((a1 >> 2) == (a2 >> 2));

// ================================================================
// Functions concerning whether an instr is 32b (RV) or 16b (RVC)

function Bool is_32b_instr (Bit #(n) instr);
   return (instr [1:0] == 2'b11);
endfunction

function Bool is_16b_instr (Bit #(n) instr);
   return (instr [1:0] != 2'b11);
endfunction

// ================================================================
// Wrapper: wraps 32-bit aligned IMem to allow 16-bit instrs and 32-bit instrs,
// aligned to 16b or 32b boundaries.

module mkCPU_Fetch_C #(IMem_IFC  imem32) (IMem_IFC);

   // imem32 is the underlying 32-bit-aligned memory.
   // imem32.req is only given 32-bit aligned addrs,
   // and returns the 32b at that addr.
   // imem32.pc (output pc) is always the last requested address, always 32b-aligned.
   // ( Assert: imem32.is_32b_not_16b == True )

   Integer verbosity = 0;

   // The following hold args of the 'req' method.
   Reg #(Bit #(3))  rg_f3          <- mkRegU;
   Reg #(WordXL)    rg_pc          <- mkRegU;
   Reg #(Priv_Mode) rg_priv        <- mkRegU;
   Reg #(Bit #(1))  rg_sstatus_SUM <- mkRegU;
   Reg #(Bit #(1))  rg_mstatus_MXR <- mkRegU;
   Reg #(WordXL)    rg_satp        <- mkRegU;

   // Holds a faulting address
   Reg #(WordXL)    rg_tval        <- mkRegU;

   // These registers caches the last output of imem32 (imem32.pc, and imem32.instr [31:16])
   Reg #(WordXL)    rg_cache_addr <- mkReg ('1);
   Reg #(Bit #(16)) rg_cache_b16  <- mkRegU;

   // ----------------------------------------------------------------
   // Conditions for selecting 16b and 32b instruction

   // Condition: 32b instr from {imem [15:0], rg_cache_b16}
   Bool cond_i32_odd = (   is_addr_odd16 (rg_pc)
			&& eq_b32_addr (rg_pc, rg_cache_addr)
			&& (imem32.pc == rg_pc + 2)
			&& is_32b_instr (rg_cache_b16));

   // Condition: 32b instr from imem [31:0]
   Bool cond_i32_even = (   is_addr_even16 (rg_pc)
			 && eq_b32_addr (rg_pc, imem32.pc)
			 && is_32b_instr (imem32.instr));

   // Condition: 16b instr from imem [31:16]
   Bool cond_i16_odd  = (   is_addr_odd16 (rg_pc)
			 && (   (   eq_b32_addr (rg_pc, imem32.pc)
				 && is_16b_instr (imem32.instr [31:16]))
			     || (   eq_b32_addr (rg_pc, rg_cache_addr)
				 && is_16b_instr (rg_cache_b16))));

   // Condition: 16b instr from imem [15:0]
   Bool cond_i16_even = (   is_addr_even16 (rg_pc)
			 && eq_b32_addr (rg_pc, imem32.pc)
			 && is_16b_instr (imem32.instr));

   // Condition: 32b instr from { ...next imem [15:0], imem [31:16] }
   Bool cond_i32_odd_fetch_next = (   is_addr_odd16 (rg_pc)
				   && imem32.valid
				   && (! imem32.exc)
				   && eq_b32_addr (rg_pc, imem32.pc)
				   && is_32b_instr (imem32.instr [31:16]));

   // ----------------
   // Compose the 32b output 'instr' (either a 32b instr, or { 16'b0, 16b instr })

   function Bit #(32) fn_instr_out ();
      Bit #(32) instr_out = imem32.instr;
      if (cond_i32_odd)
	 instr_out = { imem32.instr [15:0], rg_cache_b16 };

      else if (cond_i16_even)
	 instr_out = { 16'b0, imem32.instr [15:0] };

      else if (cond_i16_odd) begin
	 if (eq_b32_addr (rg_pc, imem32.pc))
	    instr_out = { 16'b0, imem32.instr [31:16] };
	 else
	    instr_out = { 16'b0, rg_cache_b16 };
      end
      return instr_out;
   endfunction

   // ================================================================
   // Assert: The underlying imem32 only knows about 32-bit aligned words

   rule rl_assert_fail (! imem32.is_i32_not_i16);
      $display ("%0d: ERROR: CPU_Fetch_C: imem32.is_i32_not_i16 is False", cur_cycle);
      $finish (1);
   endrule

   rule rl_debug_conds (False && (verbosity != 0) && imem32.valid);
      $display ("%0d: imem32.pc 0x%0h  pc 0x%0h  rg_f3 %0d  imem32.instr 0x%08h  rg_cache_b16 0x%04h",
		cur_cycle, imem32.pc,  rg_pc,  rg_f3,  imem32.instr,  rg_cache_b16);
      $display ("    cond_i32_odd  = ", fshow (cond_i32_odd));
      $display ("    cond_i32_even = ", fshow (cond_i32_even));
      $display ("    cond_i16_odd  = ", fshow (cond_i16_odd));
      $display ("    cond_i16_even = ", fshow (cond_i16_even));
      $display ("    cond_i32_odd_fetch_next             = ", fshow (cond_i32_odd_fetch_next));
      $display ("    eq_b32_addr (rg_pc, imem32.pc)      = ", fshow (eq_b32_addr (rg_pc, imem32.pc)));
      $display ("    is_addr_odd16 (rg_pc)               = ", fshow (is_addr_odd16 (rg_pc)));
      $display ("    is_32b_instr (imem32.instr [31:16]) = ", fshow (is_32b_instr (imem32.instr [31:16])));
   endrule

   // ================================================================
   // When imem32.instr [31:15] has lower half of 32b instr, cache it and fetch next 32 bits

   (* no_implicit_conditions, fire_when_enabled *)
   rule rl_fetch_next_32b (cond_i32_odd_fetch_next);
      Addr next_b32_addr = imem32.pc + 4;
      imem32.req (rg_f3, next_b32_addr, rg_priv, rg_sstatus_SUM, rg_mstatus_MXR, rg_satp);
      rg_cache_addr <= imem32.pc;
      rg_cache_b16  <= imem32.instr [31:16];
      rg_tval       <= next_b32_addr;

      if (verbosity != 0)
	 $display ("%0d: CPU_Fetch_C.rl_fetch_next_32b:  imem32.pc 0x%0h  next_b32_addr 0x%0h",
		   cur_cycle, imem32.pc, next_b32_addr);
   endrule

   // ================================================================
   // INTERFACE

   // CPU side: IMem request
   method Action  req (Bit #(3)   f3,
		       WordXL     addr,
		       // The following  args for VM
		       Priv_Mode  priv,
		       Bit #(1)   sstatus_SUM,
		       Bit #(1)   mstatus_MXR,
		       WordXL     satp               // { VM_Mode, ASID, PPN_for_page_table }
		       ) if (! cond_i32_odd_fetch_next);
      rg_f3          <= f3;
      rg_pc          <= addr;
      rg_priv        <= priv;
      rg_sstatus_SUM <= sstatus_SUM;
      rg_mstatus_MXR <= mstatus_MXR;
      rg_satp        <= satp;
      rg_tval        <= addr;

      // Cache the previous output, if valid
      if (imem32.valid && (! imem32.exc)) begin
	 rg_cache_addr <= imem32.pc;
	 rg_cache_b16  <= imem32.instr [31:16];
      end
      else begin
	 // Invalidate
	 rg_cache_addr <= 'h1;
      end

      WordXL addr_of_b32 = fn_to_b32_addr (addr);

      // Fetch next 32b word if request is odd-16b aligned, we've already got those 16b, and it's a 32b instr
      // (the 16b we've already got is saved in r16_cache_b16).
      // Note: since we know it's a 32b instr, this next fetch is not speculative, so ok if it page faults.
      if (   is_addr_odd16 (addr)
	  && (imem32.valid && (! imem32.exc))
	  && (addr_of_b32 == imem32.pc)
	  && is_32b_instr (imem32.instr [31:16]))
	 begin
	    addr_of_b32 = addr_of_b32 + 4;
	 end

      imem32.req (f3, addr_of_b32, priv, sstatus_SUM, mstatus_MXR, satp);
      if (verbosity > 0) begin
	 $display ("CPU_Fetch_C.req: addr 0x%0h, addr_of_b32 0x%0h", addr, addr_of_b32);
      end
   endmethod

   // CPU side: IMem response
   method Bool     valid    = (imem32.valid && (   imem32.exc
						|| cond_i32_odd
						|| cond_i32_even
						|| cond_i16_odd
						|| cond_i16_even));

   method Bool     is_i32_not_i16 = (cond_i32_odd || cond_i32_even);

   method WordXL   pc       = rg_pc;
   method Instr    instr    = fn_instr_out ();
   method Bool     exc      = imem32.exc;
   method Exc_Code exc_code = imem32.exc_code;
   method WordXL   tval     = rg_tval;        // Can be different from rg_pc

endmodule

// ================================================================

endpackage

// Copyright (c) 2016-2019 Bluespec, Inc. All Rights Reserved

package CPU_Fetch_C;

// ================================================================
// mkCPU_Fetch_C is an IMem_IFC->Module#(IMem_IFC) wrapper.

// The input parameter interface is for 32-bit-aligned 32-bit instruction-fetches.

// This module completely encapsulates the logic needed to extend it to
// 16-bit "compressed" instructions ('C' extension)
// which then also includes 32-bit instructions which may be only 16-bit-aligned,
// since 32b and 16b instructions can be freely mixed.

// ================================================================
// Exports

export mkCPU_Fetch_C;

// ================================================================
// BSV lib imports

import GetPut       :: *;
import ClientServer :: *;

// ----------------
// BSV additional libs

import Cur_Cycle :: *;

// ================================================================
// Project imports

import ISA_Decls    :: *;
import Near_Mem_IFC :: *;

// ================================================================
// Functions concerning address-alignment of instructions.
// 'Even': aligned to an even 16b boundary (so, also aligned to 32b boundary)
// 'Odd':  aligned to an odd  16b boundary (so, not  aligned to 32b boundary)

function Bool is_addr_even16 (WordXL pc);
   return (pc [1:0] == 2'b00);
endfunction

function Bool is_addr_odd16 (WordXL pc);
   return (pc [1:0] != 2'b00);
endfunction

function WordXL fn_to_b32_addr (WordXL addr);
   return ((addr >> 2) << 2);
endfunction

// Equality of 32-bit aligned addresses containing a1 and a2 (i.e., ignoring [1:0])
function Bool eq_b32_addr (WordXL a1, WordXL a2) = ((a1 >> 2) == (a2 >> 2));

// ================================================================
// Functions concerning whether an instr is 32b (RV) or 16b (RVC)

function Bool is_32b_instr (Bit #(n) instr);
   return (instr [1:0] == 2'b11);
endfunction

function Bool is_16b_instr (Bit #(n) instr);
   return (instr [1:0] != 2'b11);
endfunction

// ================================================================
// Wrapper: wraps 32-bit aligned IMem to allow 16-bit instrs and 32-bit instrs,
// aligned to 16b or 32b boundaries.

module mkCPU_Fetch_C #(IMem_IFC  imem32) (IMem_IFC);

   // imem32 is the underlying 32-bit-aligned memory.
   // imem32.req is only given 32-bit aligned addrs,
   // and returns the 32b at that addr.
   // imem32.pc (output pc) is always the last requested address, always 32b-aligned.
   // ( Assert: imem32.is_32b_not_16b == True )

   Integer verbosity = 0;

   // This register holds the actual PC value (which can be +/-2 from
   // the address given to the ICache.
   Reg #(WordXL)  rg_pc  <- mkRegU;

   // These registers caches the last output of imem32 (imem32.pc, and imem32.instr [31:16])
   Reg #(WordXL)    rg_cache_addr <- mkReg ('1);
   Reg #(Bit #(16)) rg_cache_b16  <- mkRegU;

   // The following hold args of the 'req' method.
   Reg #(Bit #(3))  rg_f3          <- mkRegU;
   Reg #(Priv_Mode) rg_priv        <- mkRegU;
   Reg #(Bit #(1))  rg_sstatus_SUM <- mkRegU;
   Reg #(Bit #(1))  rg_mstatus_MXR <- mkRegU;
   Reg #(WordXL)    rg_satp        <- mkRegU;

   // Holds a faulting address
   Reg #(WordXL)    rg_tval        <- mkRegU;

   // ----------------------------------------------------------------
   // Conditions for selecting 16b and 32b instruction

   // Condition: 32b instr from {imem [15:0], rg_cache_b16}
   Bool cond_i32_odd = (   is_addr_odd16 (rg_pc)
			&& eq_b32_addr (rg_pc, rg_cache_addr)
			&& (imem32.pc == rg_pc + 2)
			&& is_32b_instr (rg_cache_b16));

   // Condition: 32b instr from imem [31:0]
   Bool cond_i32_even = (   is_addr_even16 (rg_pc)
			 && eq_b32_addr (rg_pc, imem32.pc)
			 && is_32b_instr (imem32.instr));

   // Condition: 16b instr from imem [31:16]
   Bool cond_i16_odd  = (   is_addr_odd16 (rg_pc)
			 && (   (   eq_b32_addr (rg_pc, imem32.pc)
				 && is_16b_instr (imem32.instr [31:16]))
			     || (   eq_b32_addr (rg_pc, rg_cache_addr)
				 && is_16b_instr (rg_cache_b16))));

   // Condition: 16b instr from imem [15:0]
   Bool cond_i16_even = (   is_addr_even16 (rg_pc)
			 && eq_b32_addr (rg_pc, imem32.pc)
			 && is_16b_instr (imem32.instr));

   // Condition: 32b instr from { ...next imem [15:0], imem [31:16] }
   Bool cond_i32_odd_fetch_next = (   is_addr_odd16 (rg_pc)
				   && imem32.valid
				   && eq_b32_addr (rg_pc, imem32.pc)
				   && is_32b_instr (imem32.instr [31:16]));

   // ----------------
   // Compose the 32b output 'instr' (either a 32b instr, or { 16'b0, 16b instr })

   function Bit #(32) fn_instr_out ();
      Bit #(32) instr_out = imem32.instr;
      if (cond_i32_odd)
	 instr_out = { imem32.instr [15:0], rg_cache_b16 };

      else if (cond_i16_even)
	 instr_out = { 16'b0, imem32.instr [15:0] };

      else if (cond_i16_odd) begin
	 if (eq_b32_addr (rg_pc, imem32.pc))
	    instr_out = { 16'b0, imem32.instr [31:16] };
	 else
	    instr_out = { 16'b0, rg_cache_b16 };
      end
      return instr_out;
   endfunction

   // ================================================================
   // Assert: The underlying imem32 only knows about 32-bit aligned words

   rule rl_assert_fail (! imem32.is_i32_not_i16);
      $display ("%0d: ERROR: CPU_Fetch_C: imem32.is_i32_not_i16 is False", cur_cycle);
      $finish (1);
   endrule

   rule rl_debug_conds (False && (verbosity != 0) && imem32.valid);
      $display ("%0d: imem32.pc 0x%0h  pc 0x%0h  rg_f3 %0d  imem32.instr 0x%08h  rg_cache_b16 0x%04h",
		cur_cycle, imem32.pc,  rg_pc,  rg_f3,  imem32.instr,  rg_cache_b16);
      $display ("    cond_i32_odd  = ", fshow (cond_i32_odd));
      $display ("    cond_i32_even = ", fshow (cond_i32_even));
      $display ("    cond_i16_odd  = ", fshow (cond_i16_odd));
      $display ("    cond_i16_even = ", fshow (cond_i16_even));
      $display ("    cond_i32_odd_fetch_next             = ", fshow (cond_i32_odd_fetch_next));
      $display ("    eq_b32_addr (rg_pc, imem32.pc)      = ", fshow (eq_b32_addr (rg_pc, imem32.pc)));
      $display ("    is_addr_odd16 (rg_pc)               = ", fshow (is_addr_odd16 (rg_pc)));
      $display ("    is_32b_instr (imem32.instr [31:16]) = ", fshow (is_32b_instr (imem32.instr [31:16])));
   endrule

   // ================================================================
   // When imem32.instr [31:15] has lower half of 32b instr, cache it and fetch next 32 bits

   (* no_implicit_conditions, fire_when_enabled *)
   rule rl_fetch_next_32b (imem32.valid && cond_i32_odd_fetch_next);
      Addr next_b32_addr = imem32.pc + 4;
      imem32.req (rg_f3, next_b32_addr, rg_priv, rg_sstatus_SUM, rg_mstatus_MXR, rg_satp);
      rg_cache_addr <= imem32.pc;
      rg_cache_b16  <= imem32.instr [31:16];
      rg_tval       <= next_b32_addr;

      if (verbosity != 0)
	 $display ("%0d: CPU_Fetch_C.rl_fetch_next_32b:  imem32.pc 0x%0h  next_b32_addr 0x%0h",
		   cur_cycle, imem32.pc, next_b32_addr);
   endrule

   // ================================================================
   // INTERFACE

   // CPU side: IMem request
   method Action  req (Bit #(3)   f3,
		       WordXL     addr,
		       // The following  args for VM
		       Priv_Mode  priv,
		       Bit #(1)   sstatus_SUM,
		       Bit #(1)   mstatus_MXR,
		       WordXL     satp               // { VM_Mode, ASID, PPN_for_page_table }
		       ) if (! cond_i32_odd_fetch_next);
      rg_f3          <= f3;
      rg_pc          <= addr;
      rg_priv        <= priv;
      rg_sstatus_SUM <= sstatus_SUM;
      rg_mstatus_MXR <= mstatus_MXR;
      rg_satp        <= satp;
      rg_tval        <= addr;

      // Cache the previous output, if valid
      if (imem32.valid) begin
	 rg_cache_addr     <= imem32.pc;
	 rg_cache_b16 <= imem32.instr [31:16];
      end

      WordXL addr_of_b32 = fn_to_b32_addr (addr);

      // Fetch next 32b word if request is odd-16b aligned, we've already got those 16b, and it's a 32b instr
      // (the 16b we've already got is saved in r16_cache_b16).
      // Note: since we know it's a 32b instr, this next fetch is not speculative, so ok if it page faults.
      if (   is_addr_odd16 (addr)
	  && imem32.valid
	  && (addr_of_b32 == imem32.pc)
	  && is_32b_instr (imem32.instr [31:16]))
	 begin
	    addr_of_b32 = addr_of_b32 + 4;
	 end

      imem32.req (f3, addr_of_b32, priv, sstatus_SUM, mstatus_MXR, satp);
      if (verbosity > 0) begin
	 $display ("CPU_Fetch_C.req: addr 0x%0h, addr_of_b32 0x%0h", addr, addr_of_b32);
      end
   endmethod

   // CPU side: IMem response
   method Bool     valid    = (imem32.valid && (   cond_i32_odd
						|| cond_i32_even
						|| cond_i16_odd
						|| cond_i16_even));

   method Bool     is_i32_not_i16 = (cond_i32_odd || cond_i32_even);

   method WordXL   pc       = rg_pc;
   method Instr    instr    = fn_instr_out ();
   method Bool     exc      = imem32.exc;
   method Exc_Code exc_code = imem32.exc_code;
   method WordXL   tval     = rg_tval;        // Can be different from rg_pc

endmodule

// ================================================================

endpackage