silent debug message in MemMgmt.bsv

silent debug message in MemMgmt.bsv

// Copyright (c) 2016 Cornell University.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

// MemMgmt module handles memory allocation and memory translation

import BuildVector::*;
import Cntrs::*;
import GetPut::*;
import ClientServer::*;
import ConfigCounter::*;
import StmtFSM::*;
import FIFO::*;
import FIFOF::*;
import Pipe::*;
import Vector::*;
import BRAMFIFO::*;
import BRAM::*;
import ConnectalBram::*;
import ConnectalMemory::*;
import MMU::*;
import Ethernet::*;
import SharedBuffMMU::*;
import DbgTypes::*;

typedef enum {
   MemMgmtErrorNone,
   MemMgmtErrorInvalidPacketId,
   MemMgmtErrorOutOfSpace
} MemMgmtErrorType deriving (Bits);

typedef struct {
   MemMgmtErrorType errorType;
   Bit#(32) id;
} MemMgmtError deriving (Bits);

typedef struct {
   Bit#(EtherLen) req;
   Bit#(TLog#(TMax#(1, numClients))) clients;
} MemMgmtAllocReq#(numeric type numClients) deriving (Bits);

typedef struct {
   Maybe#(PktId) id;
   Bit#(TLog#(TMax#(1, numClients))) clients;
} MemMgmtAllocResp#(numeric type numClients) deriving (Bits);

typedef struct {
   PktId id;
   Bit#(TLog#(TMax#(1, numClients))) clients;
} MemMgmtFreeReq#(numeric type numClients) deriving (Bits);

interface MemMgmtIndication;
   method Action memory_allocated(Bit#(32) id);
   method Action packet_committed(Bit#(32) tag);
   method Action packet_freed(Bit#(32) tag);
   method Action error(Bit#(32) errorType, Bit#(32) id);
endinterface

typedef 8  PageAddrLen; // 2^8 = 256 byte page
typedef 16 MemoryAddrLen; // 2^16 = 65536 byte packet buffer
typedef TExp#(PageAddrLen) PageSize; // 256
typedef TSub#(MemoryAddrLen, PageAddrLen) PageIdx; // 8-bit index
typedef TExp#(PageIdx) FreeQueueDepth; // 256 pages

Integer pageIdx = valueOf(PageIdx);
Integer freeQueueDepth = valueOf(FreeQueueDepth);

/* Terminate MMUIndication.idResponse, because sending it to sw is slow*/
interface MMUIndicationProxy;
   interface MMUIndication mmuInd;
   interface Get#(Bit#(32)) idResponse;
endinterface
module mkMMUIndicationProxy
`ifdef DEBUG
                           #(MMUIndication mmuInd)
`endif
                           (MMUIndicationProxy);
   FIFO#(Bit#(32)) idresponse_fifo <- mkFIFO;
   interface MMUIndication mmuInd;
      method Action idResponse(Bit#(32) sglId);
         idresponse_fifo.enq(sglId);
      endmethod
`ifdef DEBUG
      method configResp = mmuInd.configResp;
      method error = mmuInd.error;
`endif
   endinterface
   interface Get idResponse = toGet(idresponse_fifo);
endmodule

typedef struct {
   Bit#(PageIdx) nPages;
   MemMgmtAllocReq#(numAllocClients) request;
   Bit#(EtherLen) mask;
   Bit#(32) id;
} Stage2Params#(numeric type numAllocClients) deriving (Bits);

interface MemMgmt#(numeric type addrWidth, numeric type numAllocClients, numeric type numReadClients);
   method Action init_mem();
   interface MMU#(addrWidth) mmu;
   interface Put#(MemMgmtAllocReq#(numAllocClients)) mallocReq;
   interface Get#(MemMgmtAllocResp#(numAllocClients)) mallocDone;
   interface Put#(MemMgmtFreeReq#(numReadClients)) freeReq;
   interface Get#(Bool) freeDone;
   method MemMgmtDbgRec dbg;
endinterface
module mkMemMgmt
`ifdef DEBUG
                #(MemMgmtIndication indication, MMUIndication mmuInd)
`endif
                (MemMgmt#(addrWidth, numAllocClients, numReadClients))
   provisos(Add#(a__, addrWidth, 44));
   let verbose = False;

   Reg#(Bit#(32)) cycle <- mkReg(0);
   rule cycleRule if (verbose);
      cycle <= cycle + 1;
   endrule

   Reg#(Bit#(64)) allocCnt <- mkReg(0);
   Reg#(Bit#(64)) allocCompleted <- mkReg(0);
   Reg#(Bit#(64)) freeCnt <- mkReg(0);
   Reg#(Bit#(64)) freeCompleted <- mkReg(0);
   Reg#(Bit#(64)) errorCode <- mkReg(0);
   Reg#(Bit#(64)) lastIdFreed <- mkReg(0);
   Reg#(Bit#(64)) lastIdAllocated <- mkReg(0);
   Reg#(Bit#(64)) firstSegment <- mkReg(0);
   Reg#(Bool) invalidSegment <- mkReg(False);

   Reg#(Bool) inited <- mkReg(False);
   FIFOF#(MemMgmtAllocReq#(numAllocClients)) mallcRequestFifo <- mkSizedFIFOF(16);
   FIFO#(MemMgmtAllocReq#(numAllocClients)) currRequestFIfo <- mkFIFO;
   FIFO#(MemMgmtAllocResp#(numAllocClients)) mallocDoneFifo <- mkFIFO;
   FIFOF#(PktId) freeRequestFifo <- mkFIFOF;

   FIFOF#(Bit#(PageIdx)) freePageList <- mkSizedFIFOF(freeQueueDepth);

   BRAM_Configure bramConfig = defaultValue;
   bramConfig.latency        = 2;
   // Store mapping from packetId to first page in linked list
   // PortA
   BRAM2Port#(PktId, Maybe#(Bit#(PageIdx))) idmap <- mkBRAM2Server(bramConfig);
   // Pack linked list of pages into BRAM for free operation
   // BRAM is indiced with pageIdx, and each entry in BRAM stores
   // a Maybe#(pageIdx) for next entry in the same list.
   // Entry is Invalid for last entry in a list
   BRAM2Port#(Bit#(PageIdx), Maybe#(Bit#(PageIdx))) pagemap <- mkBRAM2Server(bramConfig);

   Reg#(Bit#(PageIdx)) reqBurstLen <- mkReg(0);
   Reg#(Bit#(PageIdx)) reqSglIndex<- mkReg(0);
   ConfigCounter#(PageIdx) freePageCount <- mkConfigCounter(0);
   Reg#(Bool) free_started <- mkReg(False);
   Reg#(Maybe#(Bit#(PageIdx))) lastSegment <- mkReg(tagged Invalid);
   Reg#(PktId) idToFree <- mkReg(0);
   Reg#(Bit#(PageIdx)) currSegment <- mkReg(0);

   Reg#(PktId) packetId <- mkReg(0);
   Reg#(Bit#(64)) barr0 <- mkReg(0);

   FIFO#(Bool) freeDoneFifo <- mkFIFO;
   FIFO#(MemMgmtError) memMgmtErrorFifo <- mkFIFO;
   FIFO#(Bit#(PageIdx)) pagePointerFifo <- mkFIFO;

   FIFOF#(Stage2Params#(numAllocClients)) stage2Params <- mkFIFOF;

   MMUIndicationProxy proxy <- mkMMUIndicationProxy(
`ifdef DEBUG
                                                    mmuInd
`endif
                                                   );
   MMU#(addrWidth) iommu <- mkSharedBuffMMU(0, proxy.mmuInd);

   function BRAMServer#(a,b) portsel(BRAM2Port#(a,b) x, Integer i);
      if(i==0) return x.portA;
      else return x.portB;
   endfunction

   rule initialization if (!inited);
      freePageList.enq(pack(freePageCount.read));
      if (freePageCount.read == fromInteger(freeQueueDepth-1)) begin
         inited <= True;
         if (verbose) $display("(%0d) Init: freePageCount %h", $time, freePageCount.read);
      end
      else begin
         // NOTE: maximum freeQueueDepth-1
         freePageCount.increment(1);
      end
   endrule

   // assign available pages to packet id.
   rule handle_alloc_req;
      let v <- toGet(mallcRequestFifo).get;
      let id <- proxy.idResponse.get;
      if (verbose) $display("(%0d) MemMgmt:: %d Allocating pages for packet id %d packet size %d", $time, cycle, id, v.req);
      // Corner case when v is close to 4kb.
      let mask = (1 << valueOf(PageAddrLen)) - 1;
      Bit#(PageIdx) nPages = truncate(((v.req + mask) & (~mask)) >> valueOf(PageAddrLen));
      if (verbose) $display("(%0d) MemMgmt:: %d handle_malloc allocate nPage=%d", $time, cycle, nPages);
      let hasSpace <- freePageCount.maybeDecrement(unpack(nPages));
      if (hasSpace) begin
         stage2Params.enq(Stage2Params{
            nPages : nPages,
            request : v,
            mask : mask,
            id : id });
      end
   endrule

   rule handle_alloc_req2;
      let params <- toGet(stage2Params).get;
      reqBurstLen <= params.nPages;
      reqSglIndex <= 0;
      lastSegment <= tagged Invalid;
      barr0 <= extend(((params.request.req + params.mask) & (~params.mask)) >> valueOf(PageAddrLen));
      packetId <= truncate(params.id); // MaxNumPkts defined in SharedBuffMMU;
      currRequestFIfo.enq(params.request);
   endrule

   rule generate_sglist if (reqBurstLen > 0);
      let segment <- toGet(freePageList).get;
      // assume fixed page size of 256 bytes
      // extend packetId to 32 bit, only lower TLog#(MaxNumPkts) bits are used
      iommu.request.sglist(extend(packetId), extend(reqSglIndex), extend(segment), 256);
      // add current page(segment) to the front of a linked list
      portsel(pagemap, 0).request.put(BRAMRequest{write:True, responseOnWrite:False, address:segment, datain: lastSegment});
      reqBurstLen <= reqBurstLen - 1;
      reqSglIndex <= reqSglIndex + 1;
      lastSegment <= tagged Valid segment;
      if (reqBurstLen == 1) begin
         iommu.request.region(extend(packetId), 0, 0, 0, 0, 0, 0, barr0, 0);
         // map id to linked-list of pages
         portsel(idmap, 0).request.put(BRAMRequest{write:True, responseOnWrite:False, address:packetId, datain:tagged Valid segment});
         mallocDoneFifo.enq(MemMgmtAllocResp{id: tagged Valid packetId, clients: currRequestFIfo.first.clients});
         currRequestFIfo.deq;
         allocCompleted <= allocCompleted + 1;
         lastIdAllocated <= extend(packetId);
`ifdef DEBUG
         indication.memory_allocated(extend(packetId));
`endif
      end
      if (verbose) $display("(%0d) MemMgmt:: %d id=%d, segmentIdx=%x %h", $time, cycle, packetId, segment, lastSegment);
   endrule

   rule handle_free_req if (!free_started);
      let sglId <- toGet(freeRequestFifo).get;
      free_started <= True;
      portsel(idmap, 1).request.put(BRAMRequest{write:False, responseOnWrite:False, address:sglId, datain:?});
      idToFree <= sglId;
      if (verbose) $display("(%0d) MemMgmt:: %d start_free_sglist %h", $time, cycle, sglId);
      lastIdFreed <= extend(sglId);
   endrule

   rule report_error;
      let v <- toGet(memMgmtErrorFifo).get;
`ifdef DEBUG
      indication.error(extend(pack(v.errorType)), v.id);
`endif
      if (verbose) $display("(%0d) MemMgmt:: %d free_error: memMgmt error", $time, cycle);
   endrule

   (* descending_urgency="initialization, del_id_metadata, del_page_metadata, read_next_page_metadata" *)
   rule del_id_metadata if (free_started);
      Maybe#(Bit#(PageIdx)) segment <- portsel(idmap, 1).response.get;
      case (segment) matches
         tagged Valid .page: begin
            portsel(idmap, 1).request.put(BRAMRequest{write:True, responseOnWrite:False, address: idToFree, datain: tagged Invalid});
            portsel(pagemap, 1).request.put(BRAMRequest{write:False, responseOnWrite:False, address: page, datain:?});
            if (verbose) $display("(%0d) MemMgmt:: %d free_idmap ", $time, cycle, fshow(page));
            currSegment <= page;
         end
         tagged Invalid: begin
            memMgmtErrorFifo.enq(MemMgmtError{errorType: MemMgmtErrorInvalidPacketId, id: extend(idToFree)});
            invalidSegment <= True;
         end
      endcase
      firstSegment <= extend(pack(segment));
   endrule

   rule del_page_metadata if (free_started);
      Maybe#(Bit#(PageIdx)) segment <- portsel(pagemap, 1).response.get;
      case (segment) matches
         tagged Valid .page: begin
            currSegment <= page;
            pagePointerFifo.enq(page);
         end
         tagged Invalid: begin
            free_started <= False;
            freeCompleted <= freeCompleted + 1;
`ifdef DEBUG
            indication.packet_freed(extend(idToFree));
`endif
         end
      endcase
      // return current page to free page list
      portsel(pagemap, 1).request.put(BRAMRequest{write:True, responseOnWrite:False, address: currSegment, datain: tagged Invalid});
      freePageList.enq(currSegment);
      freePageCount.increment(1);
      if (verbose) $display("(%0d) MemMgmt:: %d segment ", $time, cycle, fshow(segment));
   endrule

   rule read_next_page_metadata if (free_started);
      let page <- toGet(pagePointerFifo).get;
      portsel(pagemap, 1).request.put(BRAMRequest{write:False, responseOnWrite:False, address:page, datain:?});
      if (verbose) $display("(%0d) MemMgmt:: %d read next page ", $time, cycle, fshow(page));
   endrule

   method Action init_mem();
      freePageList.clear;
      freePageCount.decrement(freePageCount.read);
      inited <= False;
   endmethod
   interface Put mallocReq;
      method Action put(MemMgmtAllocReq#(numAllocClients) req);
         mallcRequestFifo.enq(req);
         iommu.request.idRequest(0); //FIXME
         allocCnt <= allocCnt + 1;
      endmethod
   endinterface
   interface Get mallocDone = toGet(mallocDoneFifo);
   interface Put freeReq;
      method Action put(MemMgmtFreeReq#(numReadClients) req);
         iommu.request.idReturn(extend(req.id));
         freeRequestFifo.enq(req.id);
         freeCnt <= freeCnt + 1;
      endmethod
   endinterface
   interface Get freeDone = toGet(freeDoneFifo);
   interface MMU mmu = iommu;
   method MemMgmtDbgRec dbg;
      return MemMgmtDbgRec { allocCnt: allocCnt
                            ,freeCnt: freeCnt
                            ,allocCompleted: allocCompleted
                            ,freeCompleted: freeCompleted
                            ,errorCode: errorCode
                            ,lastIdFreed: lastIdFreed
                            ,lastIdAllocated: lastIdAllocated
                            ,freeStarted: extend(pack(free_started))
                            ,firstSegment: extend(firstSegment)
                            ,lastSegment: extend(pack(lastSegment))
                            ,currSegment: extend(currSegment)
                            ,invalidSegment: extend(pack(invalidSegment))};
   endmethod
endmodule

// Copyright (c) 2016 Cornell University.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

// MemMgmt module handles memory allocation and memory translation

import BuildVector::*;
import Cntrs::*;
import GetPut::*;
import ClientServer::*;
import ConfigCounter::*;
import StmtFSM::*;
import FIFO::*;
import FIFOF::*;
import Pipe::*;
import Vector::*;
import BRAMFIFO::*;
import BRAM::*;
import ConnectalBram::*;
import ConnectalMemory::*;
import MMU::*;
import Ethernet::*;
import SharedBuffMMU::*;
import DbgTypes::*;

typedef enum {
   MemMgmtErrorNone,
   MemMgmtErrorInvalidPacketId,
   MemMgmtErrorOutOfSpace
} MemMgmtErrorType deriving (Bits);

typedef struct {
   MemMgmtErrorType errorType;
   Bit#(32) id;
} MemMgmtError deriving (Bits);

typedef struct {
   Bit#(EtherLen) req;
   Bit#(TLog#(TMax#(1, numClients))) clients;
} MemMgmtAllocReq#(numeric type numClients) deriving (Bits);

typedef struct {
   Maybe#(PktId) id;
   Bit#(TLog#(TMax#(1, numClients))) clients;
} MemMgmtAllocResp#(numeric type numClients) deriving (Bits);

typedef struct {
   PktId id;
   Bit#(TLog#(TMax#(1, numClients))) clients;
} MemMgmtFreeReq#(numeric type numClients) deriving (Bits);

interface MemMgmtIndication;
   method Action memory_allocated(Bit#(32) id);
   method Action packet_committed(Bit#(32) tag);
   method Action packet_freed(Bit#(32) tag);
   method Action error(Bit#(32) errorType, Bit#(32) id);
endinterface

typedef 8  PageAddrLen; // 2^8 = 256 byte page
typedef 16 MemoryAddrLen; // 2^16 = 65536 byte packet buffer
typedef TExp#(PageAddrLen) PageSize; // 256
typedef TSub#(MemoryAddrLen, PageAddrLen) PageIdx; // 8-bit index
typedef TExp#(PageIdx) FreeQueueDepth; // 256 pages

Integer pageIdx = valueOf(PageIdx);
Integer freeQueueDepth = valueOf(FreeQueueDepth);

/* Terminate MMUIndication.idResponse, because sending it to sw is slow*/
interface MMUIndicationProxy;
   interface MMUIndication mmuInd;
   interface Get#(Bit#(32)) idResponse;
endinterface
module mkMMUIndicationProxy
`ifdef DEBUG
                           #(MMUIndication mmuInd)
`endif
                           (MMUIndicationProxy);
   FIFO#(Bit#(32)) idresponse_fifo <- mkFIFO;
   interface MMUIndication mmuInd;
      method Action idResponse(Bit#(32) sglId);
         idresponse_fifo.enq(sglId);
      endmethod
`ifdef DEBUG
      method configResp = mmuInd.configResp;
      method error = mmuInd.error;
`endif
   endinterface
   interface Get idResponse = toGet(idresponse_fifo);
endmodule

typedef struct {
   Bit#(PageIdx) nPages;
   MemMgmtAllocReq#(numAllocClients) request;
   Bit#(EtherLen) mask;
   Bit#(32) id;
} Stage2Params#(numeric type numAllocClients) deriving (Bits);

interface MemMgmt#(numeric type addrWidth, numeric type numAllocClients, numeric type numReadClients);
   method Action init_mem();
   interface MMU#(addrWidth) mmu;
   interface Put#(MemMgmtAllocReq#(numAllocClients)) mallocReq;
   interface Get#(MemMgmtAllocResp#(numAllocClients)) mallocDone;
   interface Put#(MemMgmtFreeReq#(numReadClients)) freeReq;
   interface Get#(Bool) freeDone;
   method MemMgmtDbgRec dbg;
endinterface
module mkMemMgmt
`ifdef DEBUG
                #(MemMgmtIndication indication, MMUIndication mmuInd)
`endif
                (MemMgmt#(addrWidth, numAllocClients, numReadClients))
   provisos(Add#(a__, addrWidth, 44));
   let verbose = False;

   Reg#(Bit#(32)) cycle <- mkReg(0);
   rule cycleRule if (verbose);
      cycle <= cycle + 1;
   endrule

   Reg#(Bit#(64)) allocCnt <- mkReg(0);
   Reg#(Bit#(64)) allocCompleted <- mkReg(0);
   Reg#(Bit#(64)) freeCnt <- mkReg(0);
   Reg#(Bit#(64)) freeCompleted <- mkReg(0);
   Reg#(Bit#(64)) errorCode <- mkReg(0);
   Reg#(Bit#(64)) lastIdFreed <- mkReg(0);
   Reg#(Bit#(64)) lastIdAllocated <- mkReg(0);
   Reg#(Bit#(64)) firstSegment <- mkReg(0);
   Reg#(Bool) invalidSegment <- mkReg(False);

   Reg#(Bool) inited <- mkReg(False);
   FIFOF#(MemMgmtAllocReq#(numAllocClients)) mallcRequestFifo <- mkSizedFIFOF(16);
   FIFO#(MemMgmtAllocReq#(numAllocClients)) currRequestFIfo <- mkFIFO;
   FIFO#(MemMgmtAllocResp#(numAllocClients)) mallocDoneFifo <- mkFIFO;
   FIFOF#(PktId) freeRequestFifo <- mkFIFOF;

   FIFOF#(Bit#(PageIdx)) freePageList <- mkSizedFIFOF(freeQueueDepth);

   BRAM_Configure bramConfig = defaultValue;
   bramConfig.latency        = 2;
   // Store mapping from packetId to first page in linked list
   // PortA
   BRAM2Port#(PktId, Maybe#(Bit#(PageIdx))) idmap <- mkBRAM2Server(bramConfig);
   // Pack linked list of pages into BRAM for free operation
   // BRAM is indiced with pageIdx, and each entry in BRAM stores
   // a Maybe#(pageIdx) for next entry in the same list.
   // Entry is Invalid for last entry in a list
   BRAM2Port#(Bit#(PageIdx), Maybe#(Bit#(PageIdx))) pagemap <- mkBRAM2Server(bramConfig);

   Reg#(Bit#(PageIdx)) reqBurstLen <- mkReg(0);
   Reg#(Bit#(PageIdx)) reqSglIndex<- mkReg(0);
   ConfigCounter#(PageIdx) freePageCount <- mkConfigCounter(0);
   Reg#(Bool) free_started <- mkReg(False);
   Reg#(Maybe#(Bit#(PageIdx))) lastSegment <- mkReg(tagged Invalid);
   Reg#(PktId) idToFree <- mkReg(0);
   Reg#(Bit#(PageIdx)) currSegment <- mkReg(0);

   Reg#(PktId) packetId <- mkReg(0);
   Reg#(Bit#(64)) barr0 <- mkReg(0);

   FIFO#(Bool) freeDoneFifo <- mkFIFO;
   FIFO#(MemMgmtError) memMgmtErrorFifo <- mkFIFO;
   FIFO#(Bit#(PageIdx)) pagePointerFifo <- mkFIFO;

   FIFOF#(Stage2Params#(numAllocClients)) stage2Params <- mkFIFOF;

   MMUIndicationProxy proxy <- mkMMUIndicationProxy(
`ifdef DEBUG
                                                    mmuInd
`endif
                                                   );
   MMU#(addrWidth) iommu <- mkSharedBuffMMU(0, proxy.mmuInd);

   function BRAMServer#(a,b) portsel(BRAM2Port#(a,b) x, Integer i);
      if(i==0) return x.portA;
      else return x.portB;
   endfunction

   rule initialization if (!inited);
      freePageList.enq(pack(freePageCount.read));
      if (freePageCount.read == fromInteger(freeQueueDepth-1)) begin
         inited <= True;
         $display("Init: freePageCount %h", freePageCount.read);
      end
      else begin
         // NOTE: maximum freeQueueDepth-1
         freePageCount.increment(1);
      end
   endrule

   // assign available pages to packet id.
   rule handle_alloc_req;
      let v <- toGet(mallcRequestFifo).get;
      let id <- proxy.idResponse.get;
      $display("MemMgmt:: %d Allocating pages for packet id %d packet size %d", cycle, id, v.req);
      // Corner case when v is close to 4kb.
      let mask = (1 << valueOf(PageAddrLen)) - 1;
      Bit#(PageIdx) nPages = truncate(((v.req + mask) & (~mask)) >> valueOf(PageAddrLen));
      $display("MemMgmt:: %d handle_malloc allocate nPage=%d", cycle, nPages);
      let hasSpace <- freePageCount.maybeDecrement(unpack(nPages));
      if (hasSpace) begin
         stage2Params.enq(Stage2Params{
            nPages : nPages,
            request : v,
            mask : mask,
            id : id });
      end
   endrule

   rule handle_alloc_req2;
      let params <- toGet(stage2Params).get;
      reqBurstLen <= params.nPages;
      reqSglIndex <= 0;
      lastSegment <= tagged Invalid;
      barr0 <= extend(((params.request.req + params.mask) & (~params.mask)) >> valueOf(PageAddrLen));
      packetId <= truncate(params.id); // MaxNumPkts defined in SharedBuffMMU;
      currRequestFIfo.enq(params.request);
   endrule

   rule generate_sglist if (reqBurstLen > 0);
      let segment <- toGet(freePageList).get;
      // assume fixed page size of 256 bytes
      // extend packetId to 32 bit, only lower TLog#(MaxNumPkts) bits are used
      iommu.request.sglist(extend(packetId), extend(reqSglIndex), extend(segment), 256);
      // add current page(segment) to the front of a linked list
      portsel(pagemap, 0).request.put(BRAMRequest{write:True, responseOnWrite:False, address:segment, datain: lastSegment});
      reqBurstLen <= reqBurstLen - 1;
      reqSglIndex <= reqSglIndex + 1;
      lastSegment <= tagged Valid segment;
      if (reqBurstLen == 1) begin
         iommu.request.region(extend(packetId), 0, 0, 0, 0, 0, 0, barr0, 0);
         // map id to linked-list of pages
         portsel(idmap, 0).request.put(BRAMRequest{write:True, responseOnWrite:False, address:packetId, datain:tagged Valid segment});
         mallocDoneFifo.enq(MemMgmtAllocResp{id: tagged Valid packetId, clients: currRequestFIfo.first.clients});
         currRequestFIfo.deq;
         allocCompleted <= allocCompleted + 1;
         lastIdAllocated <= extend(packetId);
`ifdef DEBUG
         indication.memory_allocated(extend(packetId));
`endif
      end
      $display("MemMgmt:: %d id=%d, segmentIdx=%x %h", cycle, packetId, segment, lastSegment);
   endrule

   rule handle_free_req if (!free_started);
      let sglId <- toGet(freeRequestFifo).get;
      free_started <= True;
      portsel(idmap, 1).request.put(BRAMRequest{write:False, responseOnWrite:False, address:sglId, datain:?});
      idToFree <= sglId;
      if (verbose) $display("MemMgmt:: %d start_free_sglist %h", cycle, sglId);
      lastIdFreed <= extend(sglId);
   endrule

   rule report_error;
      let v <- toGet(memMgmtErrorFifo).get;
`ifdef DEBUG
      indication.error(extend(pack(v.errorType)), v.id);
`endif
      if (verbose) $display("MemMgmt:: %d free_error: memMgmt error", cycle);
   endrule

   (* descending_urgency="initialization, del_id_metadata, del_page_metadata, read_next_page_metadata" *)
   rule del_id_metadata if (free_started);
      Maybe#(Bit#(PageIdx)) segment <- portsel(idmap, 1).response.get;
      case (segment) matches
         tagged Valid .page: begin
            portsel(idmap, 1).request.put(BRAMRequest{write:True, responseOnWrite:False, address: idToFree, datain: tagged Invalid});
            portsel(pagemap, 1).request.put(BRAMRequest{write:False, responseOnWrite:False, address: page, datain:?});
            $display("MemMgmt:: %d free_idmap ", cycle, fshow(page));
            currSegment <= page;
         end
         tagged Invalid: begin
            memMgmtErrorFifo.enq(MemMgmtError{errorType: MemMgmtErrorInvalidPacketId, id: extend(idToFree)});
            invalidSegment <= True;
         end
      endcase
      firstSegment <= extend(pack(segment));
   endrule

   rule del_page_metadata if (free_started);
      Maybe#(Bit#(PageIdx)) segment <- portsel(pagemap, 1).response.get;
      case (segment) matches
         tagged Valid .page: begin
            currSegment <= page;
            pagePointerFifo.enq(page);
         end
         tagged Invalid: begin
            free_started <= False;
            freeCompleted <= freeCompleted + 1;
`ifdef DEBUG
            indication.packet_freed(extend(idToFree));
`endif
         end
      endcase
      // return current page to free page list
      portsel(pagemap, 1).request.put(BRAMRequest{write:True, responseOnWrite:False, address: currSegment, datain: tagged Invalid});
      freePageList.enq(currSegment);
      freePageCount.increment(1);
      if (verbose) $display("MemMgmt:: %d segment ", cycle, fshow(segment));
   endrule

   rule read_next_page_metadata if (free_started);
      let page <- toGet(pagePointerFifo).get;
      portsel(pagemap, 1).request.put(BRAMRequest{write:False, responseOnWrite:False, address:page, datain:?});
      if (verbose) $display("MemMgmt:: %d read next page ", cycle, fshow(page));
   endrule

   method Action init_mem();
      freePageList.clear;
      freePageCount.decrement(freePageCount.read);
      inited <= False;
   endmethod
   interface Put mallocReq;
      method Action put(MemMgmtAllocReq#(numAllocClients) req);
         $display("MemMgmt:: %d req page=%d", cycle, freePageCount.read);
         mallcRequestFifo.enq(req);
         iommu.request.idRequest(0); //FIXME
         allocCnt <= allocCnt + 1;
      endmethod
   endinterface
   interface Get mallocDone = toGet(mallocDoneFifo);
   interface Put freeReq;
      method Action put(MemMgmtFreeReq#(numReadClients) req);
         $display("MemMgmt:: %d free request %h", cycle, req.id);
         iommu.request.idReturn(extend(req.id));
         freeRequestFifo.enq(req.id);
         freeCnt <= freeCnt + 1;
      endmethod
   endinterface
   interface Get freeDone = toGet(freeDoneFifo);
   interface MMU mmu = iommu;
   method MemMgmtDbgRec dbg;
      return MemMgmtDbgRec { allocCnt: allocCnt
                            ,freeCnt: freeCnt
                            ,allocCompleted: allocCompleted
                            ,freeCompleted: freeCompleted
                            ,errorCode: errorCode
                            ,lastIdFreed: lastIdFreed
                            ,lastIdAllocated: lastIdAllocated
                            ,freeStarted: extend(pack(free_started))
                            ,firstSegment: extend(firstSegment)
                            ,lastSegment: extend(pack(lastSegment))
                            ,currSegment: extend(currSegment)
                            ,invalidSegment: extend(pack(invalidSegment))};
   endmethod
endmodule


fix import

fix import

// Copyright (c) 2014 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

// This module implements a fixed point complex multiplication
// that is intended to map smoothly onto the DSP slices in Xilinx
// FPGAs.  To this end, the signal path is 18 bits (2 bits of integer
// and 16 bits of fraction), the coefficient path is 25 bits (2 bits
// of integer and 23 bits of fraction), and the product is 43 bits
// 4 bits of integer and 39 bits of fraction).  The multiplier
// in a DSP slice is actually 18 x 25 -> 45 bits.  The additional two
// product bits are for overflow, since the complex multipler adds
// four intermediate results.

// Signal inputs and outputs are Pipe datatypes, to provide flow control.
// The module is intended for use in environments where there may be
// one result per cycle, but at slow signal rates, clock cycles may
// be skipped.  Because the logic is pipelined, there are valid
// bits that follow the signal path.  The signal is assumed to be provided
// intermittently, while the coefficients are provided on demand up to
// one per cycle.

// Because one intended application is a Channel Select filter with
// downconversion, the coefficient path supplies a "filter phase" boolean
// which indicates the start of a new sample period at the output
// intermediate frequency.

import SpecialFIFOs::*;
import Complex::*;
import FixedPoint::*;
import Pipe::*;

typedef FixedPoint#(2,16) Signal;
typedef FixedPoint#(2,23) Coeff;
typedef FixedPoint#(4,39) Product;

typedef struct {
		Complex#(Coeff) a;
		Bit#(1) filterPhase;
		} CoeffData;

typedef struct {
		Complex#(Product) y;
		Bit#(1) filterPhase;
		} ProductData;

typedef struct {
		Product arxr;
		Product arxr;
		Product arxr;
		Product arxr;
		Bit#(1) filterPhase;
		} MulData;

interface FPCMult;
   interface PipeIn#(Complex#(Signal)) x;
   interface PipeIn#(CoeffData) a;
   interface PipeOut#(ProductData) y;
endinterface

module mkFPCMult(FPCMult)
  provisos(Bits#(CoeffData, a__),
     Bits#(ProductData, b__));
   /* input registers */
   FIFOF#(CoeffData) ain <- mkPipelineFIFOF();
   FIFOF#(Complex#(Signal)) xin <- mkPipelineFIFOF();
   /* pipeline registers at output of multipliers */
   Reg#(MulData) ax <- mkReg(?);
   /* result registers */
   FIFOF#(ProductData) yout <- mkPipelineFIFOF();

   rule work;
      /* compute multiplies */
      Product arxr = fxptMult(ain.a.rel, xin.first.rel);
      Product aixi = fxptMult(ain.a.img, xin.first.img);
      Product arxi = fxptMult(ain.a.rel, xin.first.img);
      Product aixr = fxptMult(ain.a.img, xin.first.rel);
      ain.deq();
      xin.deq();
      ax <= Muldata{arxr: arxr, aixi: aixi, arxi: arxi, aixr: aixr,
	 filterPhase ain.filterPhase};
      /* combine into outputs */
      yout.enq(ProductData{y: Complex{rel: arxr - aixi, img: arxi + aixr}, filterPhase: muloutphase});
   endrule

   interface PipeOut y = toPipeOut(yout);
   interface PipeIn x = toPipeIn(xin);
   interface PipeIn a = toPipeIn(ain);

endmodule

// Copyright (c) 2014 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

// This module implements a fixed point complex multiplication
// that is intended to map smoothly onto the DSP slices in Xilinx
// FPGAs.  To this end, the signal path is 18 bits (2 bits of integer
// and 16 bits of fraction), the coefficient path is 25 bits (2 bits
// of integer and 23 bits of fraction), and the product is 43 bits
// 4 bits of integer and 39 bits of fraction).  The multiplier
// in a DSP slice is actually 18 x 25 -> 45 bits.  The additional two
// product bits are for overflow, since the complex multipler adds
// four intermediate results.

// Signal inputs and outputs are Pipe datatypes, to provide flow control.
// The module is intended for use in environments where there may be
// one result per cycle, but at slow signal rates, clock cycles may
// be skipped.  Because the logic is pipelined, there are valid
// bits that follow the signal path.  The signal is assumed to be provided
// intermittently, while the coefficients are provided on demand up to
// one per cycle.

// Because one intended application is a Channel Select filter with
// downconversion, the coefficient path supplies a "filter phase" boolean
// which indicates the start of a new sample period at the output
// intermediate frequency.

import Complex::*;
import FixedPoint::*;
import Pipe::*;

typedef FixedPoint#(2,16) Signal;
typedef FixedPoint#(2,23) Coeff;
typedef FixedPoint#(4,39) Product;

typedef struct {
		Complex#(Coeff) a;
		Bit#(1) filterPhase;
		} CoeffData;

typedef struct {
		Complex#(Product) y;
		Bit#(1) filterPhase;
		} ProductData;

typedef struct {
		Product arxr;
		Product arxr;
		Product arxr;
		Product arxr;
		Bit#(1) filterPhase;
		} MulData;

interface FPCMult;
   interface PipeIn#(Complex#(Signal)) x;
   interface PipeIn#(CoeffData) a;
   interface PipeOut#(ProductData) y;
endinterface

module mkFPCMult(FPCMult)
  provisos(Bits#(CoeffData, a__),
     Bits#(ProductData, b__));
   /* input registers */
   FIFOF#(CoeffData) ain <- mkPipelineFIFOF();
   FIFOF#(Complex#(Signal)) xin <- mkPipelineFIFOF();
   /* pipeline registers at output of multipliers */
   Reg#(MulData) ax <- mkReg(?);
   /* result registers */
   FIFOF#(ProductData) yout <- mkPipelineFIFOF();

   rule work;
      /* compute multiplies */
      Product arxr = fxptMult(ain.a.rel, xin.first.rel);
      Product aixi = fxptMult(ain.a.img, xin.first.img);
      Product arxi = fxptMult(ain.a.rel, xin.first.img);
      Product aixr = fxptMult(ain.a.img, xin.first.rel);
      ain.deq();
      xin.deq();
      ax <= Muldata{arxr: arxr, aixi: aixi, arxi: arxi, aixr: aixr,
	 filterPhase ain.filterPhase};
      /* combine into outputs */
      yout.enq(ProductData{y: Complex{rel: arxr - aixi, img: arxi + aixr}, filterPhase: muloutphase});
   endrule

   interface PipeOut y = toPipeOut(yout);
   interface PipeIn x = toPipeIn(xin);
   interface PipeIn a = toPipeIn(ain);

endmodule


[RISCV] Configurable Memory baseaddress

[RISCV] Configurable Memory baseaddress

// Copyright (c) 2017 Min Chen.  All Rights Reserved.
// Author: Min Chen

// ================================================================
// BSV library imports

import RegFile       :: *;    // For RISC-V GPRs
import ConfigReg     :: *;
import FIFOF         :: *;
import SpecialFIFOs  :: *;

// ================================================================
// BSV project imports

import ISA_Decls :: *;    // Instruction encodings

// ================================================================
// Memory interface for CPU
// ================================================================

import BRAMCore :: *;
import DReg     :: *;

// ----------------
// IMem responses: either and exception or an instruction

typedef Word IMem_Resp;

// ----------------
// DMem request ops and sizes

typedef enum {
   MEM_OP_LOAD,
   MEM_OP_STORE
} Mem_Op deriving(Eq, Bits, FShow);

// ----------------
// DMem requests

typedef struct {
   Mem_Op               mem_op;
   Addr                 addr;
   Word                 data;          // Only relevant if mem_op == MEM_OP_STORE
   Bit#(Bytes_per_Word) written;
} DMem_Req deriving(Bits);

// ----------------
// DMem responses: either an exception or data
// (data value is only relevant for LOADs, irrelevant for STOREs)

typedef Maybe#(Word) DMem_Resp;

// ----------------
// Memory interface reference design

module mkIMemory#(Reg#(Bit#(64)) cycles, Addr base)(IMemory_IFC#(size))
   provisos(Add#(a__, size, XLEN));
   BRAM_PORT#(Bit#(size), Word)     mem   <- mkBRAMCore1Load(2 ** valueOf(size), False, (genC ? "mem.vmh" : "R:/mem.vmh"), False);

   method Action mem_req(Addr addr);
      let phyAddr = (addr - base);
      mem.put(False, truncate(phyAddr >> 2), ?);
      //$display("[%7d] [IMEM] ReqAddr = 0x%08h", cycles, addr);
   endmethod

   method ActionValue#(IMem_Resp) mem_resp();
      let x = mem.read();
      return x;
   endmethod
endmodule

module mkDMemory#(Reg#(Bit#(64)) cycles, Addr base)(DMemory_IFC#(size))
   provisos(Add#(a__, size, XLEN));
   BRAM_DUAL_PORT_BE#(Bit#(size), Word, 4)   mem      <- mkBRAMCore2BELoad(2 ** valueOf(size), False, (genC ? "mem.vmh.D" : "R:/mem.vmh.D"), False);
   Reg#(Bool)                                mem_rd   <- mkDReg(False);

   method Action mem_req(DMem_Req req);
      let phyAddr = (req.addr - base);
      Bit#(Bits_per_Word_Byte_Index) shift = truncate(req.addr);

      if (shift != 0) begin
         $display("Memory alignment check failed at addr 0x%h", req.addr);
         $finish;
      end

      mem.a.put(req.written, truncate(phyAddr >> 2), req.data);
      mem_rd <= !(req.mem_op == MEM_OP_STORE);
      //$display("[DMEM] Addr = 0x%08h", req.addr);
   endmethod

   method ActionValue#(DMem_Resp) mem_resp;
      Word v = mem.a.read();

      //$display("[DMEM] Data = 0x%08h", v);
      return mem_rd ? tagged Valid v : tagged Invalid;
   endmethod
endmodule

// ----------------------------------------------------------------
// This interface is an argument to the '_mkRISCV' module,
// and is used insided the module to access memory.

interface IMemory_IFC#(numeric type size);
   method Action                  mem_req(Addr addr);
   method ActionValue#(IMem_Resp) mem_resp;
endinterface

interface DMemory_IFC#(numeric type size);
   method Action                  mem_req(DMem_Req req);
   method ActionValue#(DMem_Resp) mem_resp;
endinterface

// ================================================================
// This interfacce is offered by the '_mkRISCV' module to the environment.
// It is not part of the spec, per se, and just has scaffolding that allows
// the environment to control and CPU and probe its state.

interface RISCV_IFC;
   method Action start(Addr initial_pc);

   method ActionValue#(Word) cpuToHost;
endinterface

// ================================================================
// The RISC-V CPU Specification module, 'mkRISCV'

// ----------------
// Default fall-through PC

function Addr fv_fall_through_pc(Addr pc);
   return pc + 4;
endfunction: fv_fall_through_pc

// ----------------

module _mkRISCV#(Bit#(3) cfg_verbose)(RISCV_IFC);

   // CPU state
   Reg#(Bool)  cpu_enabled <- mkReg(False);

   // Program counter
   Reg#(Addr)     rg_FetchPC  <- mkRegU;
   RWire#(Addr)   rw_nxtPC    <- mkRWire;
   RWire#(Addr)   rw_jmpPC    <- mkUnsafeRWire;
   Reg#(Word)     pcEpoch     <- mkConfigRegU;

   // General Purpose Registers
   RegFile#(RegName, Word) rf_GPRs  <- mkRegFileWCF(0, ~0);

   // CSRs
   Reg#(Bit#(64))    csr_cycle   <- mkConfigReg(0);
   Reg#(Bit#(64))    csr_instret <- mkConfigReg(0);

   // internal components
   IMemory_IFC#(14)     imemory  <- mkIMemory(csr_cycle, imemSt);
   DMemory_IFC#(15)     dmemory  <- mkDMemory(csr_cycle, dmemSt);

   // ----------------
   // These CSRs are technically not present in the user-mode ISA.

   Reg#(Word)  csr_mepc       <- mkRegU;
   Reg#(Word)  csr_mcause     <- mkRegU;
   Reg#(Word)  csr_mbadaddr   <- mkRegU;

   Reg#(Maybe#(Word)) csr_dscratch <- mkDReg(tagged Invalid);

   // ----------------------------------------------------------------
   // Non-architectural state, for this model

   Reg#(Maybe#(Addr))      rg_f2d      <- mkDReg(tagged Invalid);
   FIFOF#(Decoded_Instr)   fifo_d2e    <- mkPipelineFIFOF;
   FIFOF#(Exec2Wb_t)       fifo_e2w    <- mkPipelineFIFOF;

   // ----------------------------------------------------------------
   // Scoreboard map
   Reg#(Bool)        rg_scoreGPRs[numRegs];
   for(Integer i = 0; i < numRegs; i = i + 1)
      rg_scoreGPRs[i] <- mkReg(False);
   RWire#(RegName)   rw_scoreGPRsSet   <- mkRWire;
   RWire#(RegName)   rw_scoreGPRsReset <- mkRWire;

   // ----------------------------------------------------------------
   // Read a CSR
   // If the addr is valid, return tagged Valid value
   // else return tagged Invalid

   function Maybe#(Word) fv_read_csr(CSR_Addr csr_addr);
      if      (csr_addr == csr_CYCLE)     return tagged Valid truncate   (csr_cycle);
      else if (csr_addr == csr_INSTRET)   return tagged Valid truncate   (csr_instret);

      else if (csr_addr == csr_CYCLEH  )  return tagged Valid truncateLSB(csr_cycle);
      else if (csr_addr == csr_INSTRETH)  return tagged Valid truncateLSB(csr_instret);

      else if (csr_addr == csr_DSCRATCH)  return tagged Valid 0 ;

      else return tagged Invalid;
   endfunction: fv_read_csr

   // ----------------------------------------------------------------
   // Write a CSR
   // We assume a valid csr_addr, since this is always preceded by a read_csr which performs the check

   function Action fa_write_csr(CSR_Addr csr_addr, Word csr_value);
      action
         if (csr_addr == csr_DSCRATCH) begin
            csr_dscratch <= tagged Valid csr_value;
         end

         else begin
            $display("ERROR: fa_write_csr: (csr_addr 0x%0h, csr_value 0x%0h): illegal csr_addr", csr_addr, csr_value);
            $finish;
         end
      endaction
   endfunction: fa_write_csr

   // ================================================================
   // Instruction execution

   // ----------------------------------------------------------------
   // The following functions are common idioms for finishing an instruction

   // ----------------
   // Finish exception: record exception cause info, go to ENV_CALL state

   function Action fa_finish_with_exception(Bit#(4) exc_code, Addr badaddr);
      action
         if (cfg_verbose > 0) begin
            $display("[%7d] fa_do_exception: epc = 0x%0h, exc_code = 0x%0h, badaddr = 0x%0h", csr_cycle, pcEpoch, exc_code, badaddr);
         end

         csr_mepc     <= pcEpoch;
         csr_mcause   <= { 1'b0, 0, exc_code };
         csr_mbadaddr <= badaddr;

         $finish;
      endaction
   endfunction

   // ----------------
   // Finish instr with no output (no Rd-write): set PC, go to FETCH state

   function Action fa_finish_with_no_output();
      action
      endaction
   endfunction

   // ----------------
   // Finish instr with Rd-write: set Rd, set PC, go to WRITE_BACK state

   function Action fa_finish_with_Rd(RegName rd, Word rd_value);
      action
         fifo_e2w.enq( Exec2Wb_t {rd: rd, rd_value: tagged Value rd_value} );
         fa_finish_with_no_output;
      endaction
   endfunction

   // ----------------
   // Finish instr with Rd-write: set Rd, set PC, go to WRITE_BACK state

   function Action fa_finish_with_Ld(RegName rd, LdFunc op, Bit#(Bits_per_Word_Byte_Index) align);
      action
         fifo_e2w.enq( Exec2Wb_t {rd: rd, rd_value: (tagged MemOp {ld_op: op, align: align})} );
         fa_finish_with_no_output;
      endaction
   endfunction

   // ----------------
   // Finish jump instrs; write Rd, set PC, go to FETCH state

   function Action fa_finish_jump(RegName rd, Word rd_value, Addr next_pc);
      action
         fifo_e2w.enq( Exec2Wb_t {rd: rd, rd_value: tagged Value rd_value} );
         rw_jmpPC.wset(next_pc);
      endaction
   endfunction

   // ----------------
   // Finish conditional branch instr: set PC, go to FETCH state

   function Action fa_finish_cond_branch(Bool condition_taken, Addr next_pc);
      action
         if (condition_taken) begin
            rw_jmpPC.wset(next_pc);
         end
      endaction
   endfunction

   // ----------------------------------------------------------------
   // Instruction execution
   // This function encapsulates ALL the opcodes.
   // It has internal functions that group related sub-opcodes.

   function ActionValue#(Fmt) fa_exec(Decoded_Instr decoded, Decoded_Fields fields);
      actionvalue

         // Values of Rs1 and Rs2 fields of the instr, unsigned
         //Word v1 = decoded.v1;
         //Word v2 = decoded.v2;
         Word  v1  = ((fields.rs1 == 0) ? 0: rf_GPRs.sub(fields.rs1));
         Word  v2  = ((fields.rs2 == 0) ? 0: rf_GPRs.sub(fields.rs2));

         // Values of Rs1 and Rs2 fields of the instr, signed versions
         Word_S  s_v1 = unpack(v1);
         Word_S  s_v2 = unpack(v2);

         // Value of CSR field of instr (if a valid CSR address)
         Maybe #(Word) m_v_csr = fv_read_csr(fields.csr);

         // ----------------------------------------------------------------
         // Instructions for Upper Immediate
         function ActionValue#(Fmt) fa_exec_LUI();
            actionvalue
               Bit#(32)    v32   = { fields.imm20_U, 12'h0 };
               Word_S      iv    = extend(unpack(v32));
               let         value = pack(iv);

               fa_finish_with_Rd(fields.rd, value);
               let msg = $format("lui %s, 0x%h", regNameABI[fields.rd], value[31:12]);
               return msg;
            endactionvalue
         endfunction: fa_exec_LUI

         function ActionValue#(Fmt) fa_exec_AUIPC();
            actionvalue
               Word_S  iv    = extend(unpack({ fields.imm20_U, 12'b0}));
               Word_S  pc_s  = unpack(pcEpoch);
               Word    value = pack(pc_s + iv);

               fa_finish_with_Rd(fields.rd, value);
               let msg = $format("auipc %s, 0x%h", regNameABI[fields.rd], value[31:12]);
               return msg;
            endactionvalue
         endfunction: fa_exec_AUIPC

         // ----------------------------------------------------------------
         // Instructions for control-transfer

         function ActionValue#(Fmt) fa_exec_JAL();
            actionvalue
               Word_S offset  = extend(unpack(fields.imm21_UJ));
               Addr   next_pc = pack(unpack(pcEpoch) + offset);

               fa_finish_jump(fields.rd, fv_fall_through_pc(pcEpoch), next_pc);
               let msg = $format("jal %s, 0x%h", regNameABI[fields.rd], next_pc);
               return msg;
            endactionvalue
         endfunction: fa_exec_JAL

         function ActionValue#(Fmt) fa_exec_JALR();
            actionvalue
               Word_S offset  = extend(unpack(fields.imm12_I));
               Addr   next_pc = {truncateLSB(pack(s_v1 + offset)), 1'b0};

               fa_finish_jump(fields.rd, fv_fall_through_pc(pcEpoch), next_pc);
               let msg = $format("jalr %s, %s, %1d", regNameABI[fields.rd], regNameABI[fields.rs1], offset);
               return msg;
            endactionvalue
         endfunction: fa_exec_JALR

         function ActionValue#(Fmt) fa_exec_BRANCH(BrFunc op);
            actionvalue
               Word_S offset  = extend(unpack(fields.imm13_SB));
               Word   next_pc = pack(unpack(pcEpoch) + offset);

               case(op)
                  Eq    :  fa_finish_cond_branch(v1  == v2,    next_pc);
                  Neq   :  fa_finish_cond_branch(v1  != v2,    next_pc);
                  Lt    :  fa_finish_cond_branch(s_v1 <  s_v2, next_pc);
                  Ge    :  fa_finish_cond_branch(s_v1 >= s_v2, next_pc);
                  Ltu   :  fa_finish_cond_branch(v1  <  v2,    next_pc);
                  /* Geu */
               default  :  fa_finish_cond_branch(v1  >= v2,    next_pc);
               endcase

               let msg =  $format("B")
                        + fshow(op)
                        + $format(" %s, %s, 0x%h", regNameABI[fields.rd], regNameABI[fields.rs1], next_pc);
               return msg;
            endactionvalue
         endfunction: fa_exec_BRANCH

         // ----------------------------------------------------------------
         // LD and ST instructions.
         // Issue request here; will be completed in STATE_EXEC_LD/ST_RESPONSE

         function ActionValue#(Fmt) fa_exec_LD_Req(LdFunc op);
            actionvalue
               Word_S  imm_s    = extend(unpack(fields.imm12_I));
               Word    mem_addr = pack(s_v1 + imm_s);

               function Action fa_LD_Req(Mem_Data_Size sz);
                  action
                     Bit#(Bits_per_Word_Byte_Index) align = truncate(mem_addr);
                     let req = DMem_Req {mem_op:      MEM_OP_LOAD,
                                         addr:        {mem_addr[xlen-1:2], 2'b00},
                                         written:     0,
                                         data:        ?};
                     dmemory.mem_req(req);
                     fa_finish_with_Ld(fields.rd, op, align);
                  endaction
               endfunction

               case(op)
                  Lb    :  fa_LD_Req(BITS8);
                  Lbu   :  fa_LD_Req(BITS8);
                  Lh    :  fa_LD_Req(BITS16);
                  Lhu   :  fa_LD_Req(BITS16);
                  /*Lw*/
               default  :  fa_LD_Req(BITS32);
               endcase

               let msg =  fshow(op)
                        + $format(" %s, %s, %1d", regNameABI[fields.rd], regNameABI[fields.rs1], imm_s);
               return msg;
            endactionvalue
         endfunction: fa_exec_LD_Req

         function ActionValue#(Fmt) fa_exec_ST_Req(StFunc op);
            actionvalue
               Word_S  imm_s    = extend(unpack(fields.imm12_S));
               Word    mem_addr = pack(s_v1 + imm_s);

               function Action fa_ST_req(Mem_Data_Size sz);
                  action
                     Bit#(Bits_per_Word_Byte_Index) align = truncate(mem_addr);
                     Word aligned_data = v2 << {align, 3'b0};
                     Bit#(Bytes_per_Word) write_en = (case(sz)
                                                         BITS8:  ('b0001 << align);
                                                         BITS16: ('b0011 << align);
                                                         default/*BITS32*/: ('b1111);
                                                      endcase);

                     let req = DMem_Req {mem_op:      MEM_OP_STORE,
                                         addr:        {mem_addr[xlen-1:2], 2'b00},
                                         data:        aligned_data,
                                         written:     write_en};
                     dmemory.mem_req(req);
                     fa_finish_with_no_output;
                  endaction
               endfunction

               case(op)
                  Sb    :  fa_ST_req(BITS8);
                  Sh    :  fa_ST_req(BITS16);
                  /*Sw*/
               default  :  fa_ST_req(BITS32);
               endcase

               let msg =  fshow(op)
                        + $format(" %s, %s, %1d", regNameABI[fields.rd], regNameABI[fields.rs1], imm_s);
               return msg;
            endactionvalue
         endfunction: fa_exec_ST_Req

         // ----------------------------------------------------------------
         // Instructios for Register-Immediate alu ops

         function ActionValue#(Fmt) fa_exec_OP_IMM(AluFunc op);
            actionvalue
               Word                v2    = zeroExtend(fields.imm12_I);
               Word_S              s_v2  = signExtend(unpack(fields.imm12_I));
               Bit#(TLog#(XLEN))   shamt = truncate(fields.imm12_I);

               case(op)
                  Add   :  fa_finish_with_Rd(fields.rd, pack(s_v1 + s_v2));
                  Slt   :  fa_finish_with_Rd(fields.rd, ((s_v1 < s_v2) ? 1 : 0));
                  Sltu  :  fa_finish_with_Rd(fields.rd, ((v1  < pack(s_v2))  ? 1 : 0));
                  Xor   :  fa_finish_with_Rd(fields.rd, pack(s_v1 ^ s_v2));
                  Or    :  fa_finish_with_Rd(fields.rd, pack(s_v1 | s_v2));
                  And   :  fa_finish_with_Rd(fields.rd, pack(s_v1 & s_v2));
                  Sll   :  fa_finish_with_Rd(fields.rd, (v1 << shamt));
                  Srl   :  fa_finish_with_Rd(fields.rd, (v1 >> shamt));
                  /*Sra*/
               default  :  fa_finish_with_Rd(fields.rd, pack(s_v1 >> shamt));
               endcase

               let msg =  fshow(op)
                        + $format("i %s, %s, 0x%h", regNameABI[fields.rd], regNameABI[fields.rs1], fields.imm12_I);
               return msg;
            endactionvalue
         endfunction: fa_exec_OP_IMM

         // ----------------------------------------------------------------
         // Instructios for Register-Register alu ops

         function ActionValue#(Fmt) fa_exec_OP(AluFunc op);
            actionvalue
               Bit#(TLog#(XLEN)) shamt = truncate(v2);    // NOTE: upper bits are unspecified in spec

               case(op)
                  Add   :  fa_finish_with_Rd(fields.rd, pack(s_v1 + s_v2));
                  Sub   :  fa_finish_with_Rd(fields.rd, pack(s_v1 - s_v2));
                  Sll   :  fa_finish_with_Rd(fields.rd, (v1 << shamt));
                  Slt   :  fa_finish_with_Rd(fields.rd, ((s_v1 < s_v2) ? 1 : 0));
                  Sltu  :  fa_finish_with_Rd(fields.rd, ((v1  < v2)  ? 1 : 0));
                  Xor   :  fa_finish_with_Rd(fields.rd, pack(s_v1 ^ s_v2));
                  Srl   :  fa_finish_with_Rd(fields.rd, (v1 >> shamt));
                  Sra   :  fa_finish_with_Rd(fields.rd, pack(s_v1 >> shamt));
                  Or    :  fa_finish_with_Rd(fields.rd, pack(s_v1 | s_v2));
                  /*And*/
               default  :  fa_finish_with_Rd(fields.rd, pack(s_v1 & s_v2));
               endcase

               let msg =  fshow(op)
                        + $format(" %s, %s, %s", regNameABI[fields.rd], regNameABI[fields.rs1], regNameABI[fields.rs2]);
               return msg;
            endactionvalue
         endfunction: fa_exec_OP

         // ----------------------------------------------------------------
         // Instrucions for System-level ops

         function ActionValue#(Fmt) fa_exec_SYSTEM(SysFunc op);
            actionvalue
               let csr_old_val = fromMaybe(?, m_v_csr);

               case(op)
                  CSRRW :  begin
                                 fa_write_csr(fields.csr, v1);
                                 fa_finish_with_Rd(fields.rd, csr_old_val);
                              end

                  CSRRS :  begin
                                 if (fields.rs1 != 0) begin
                                    Word csr_new_val = (csr_old_val | v1);
                                    fa_write_csr(fields.csr, csr_new_val);
                                 end
                                 fa_finish_with_Rd(fields.rd, csr_old_val);
                              end

                  /*CSRRC*/
               default  :  begin
                                 if (fields.rs1 != 0) begin
                                    Word csr_new_val = (csr_old_val & (~ v1));
                                    fa_write_csr(fields.csr, csr_new_val);
                                 end
                                 fa_finish_with_Rd(fields.rd, csr_old_val);
                              end
               endcase

               let msg = ?;
               if ( (op == CSRRS) && (fields.csr == csr_CYCLE) )
                  msg = $format("rdcycle %s", regNameABI[fields.rd]);
               else if ( (op == CSRRS) && (fields.csr == csr_INSTRET) )
                  msg = $format("rdinstret %s", regNameABI[fields.rd]);
               else if ( (op == CSRRW) && (fields.csr == csr_DSCRATCH) )
                  msg = $format("csrw dscratch, %s", regNameABI[fields.rs1]);
               else begin
                  msg =  fshow(op)
                       + $format(" %s, 0x%h, %s", regNameABI[fields.rd], fields.csr, regNameABI[fields.rs1]);
               end
               return msg;
            endactionvalue
         endfunction: fa_exec_SYSTEM

         // ----------------------------------------------------------------
         // Main body of fa_exec(), dispatching to the sub functions
         // based on major OPCODE

         let x = ?;
         case(decoded.op.opcode) matches
            tagged Lui        :  x <- fa_exec_LUI();
            tagged Auipc      :  x <- fa_exec_AUIPC();
            tagged Jal        :  x <- fa_exec_JAL();
            tagged Jalr       :  x <- fa_exec_JALR();
            tagged Br   .op   :  x <- fa_exec_BRANCH(op);
            tagged Ld   .op   :  x <- fa_exec_LD_Req(op);
            tagged St   .op   :  x <- fa_exec_ST_Req(op);
            tagged Alui .op   :  x <- fa_exec_OP_IMM(op);
            tagged Alu  .op   :  x <- fa_exec_OP(op);
            tagged Sys  .op   :  x <- fa_exec_SYSTEM(op);
            default           :  fa_finish_with_exception(exc_code_ILLEGAL_INSTRUCTION, ?);
         endcase
         return x;
      endactionvalue
   endfunction: fa_exec

   // ================================================================
   // The CPU's top-level logic

   // Update CPU internal status in every cycle
   (* fire_when_enabled, no_implicit_conditions *)
   rule update_score;
      let x = fromMaybe(0, rw_scoreGPRsSet.wget);
      let y = fromMaybe(0, rw_scoreGPRsReset.wget);

      rg_scoreGPRs[x] <= True;
      if (x != y) begin
         rg_scoreGPRs[y] <= False;
      end
   endrule

   // ---------------- FETCH
   // Issue instruction request and decode

   // ----------------------------------------------------------------
   // Instruction fetch
   (* fire_when_enabled *)
   rule rl_fetch(cpu_enabled);
      let next_pc = fromMaybe(fromMaybe(rg_FetchPC, rw_nxtPC.wget), rw_jmpPC.wget);
      if (cfg_verbose > 1) $display("[%7d] ( |F   ) : Read instruction pc = 0x%08h", csr_cycle, next_pc);
      imemory.mem_req(next_pc);
      rg_f2d      <= tagged Valid next_pc;
      rg_FetchPC  <= next_pc;
   endrule

   // ----------------------------------------------------------------
   // Instruction decode
   (* conflict_free="rl_decode, rl_exec" *)
   rule rl_decode(fifo_d2e.notFull &&& rg_f2d matches tagged Valid .xPC);
      let instr <- imemory.mem_resp;

      if (cfg_verbose > 1) $display("[%7d] ( | D  ) : pc = 0x%08h, instr = %h", csr_cycle, xPC, instr);
      Decoded_Instr  decoded = fv_decode(xPC, instr);

      fifo_d2e.enq( decoded );
      rw_nxtPC.wset(fv_fall_through_pc(xPC));
   endrule

   // ---------------- EXECUTE
   // Receive instruction from IMem; handle exception if any, else execute it;

   //(* no_implicit_conditions *)
   rule rl_exec(fifo_d2e.notEmpty && fifo_e2w.notFull);
      let decoded = fifo_d2e.first;

      // ----------------------------------------------------------------
      // Instruction fields decode
      Decoded_Fields fields   = fv_decode_fields(decoded.instr);

      // Calculate dependency register
      let score1 = False;
      let score2 = False;

      if (decoded.op.rs1 matches tagged Valid .x &&& x != 0)
         score1 = rg_scoreGPRs[x];

      if (decoded.op.rs2 matches tagged Valid .x &&& x != 0)
         score2 = rg_scoreGPRs[x];

      let score_conflict = score1 || score2;

      if (cfg_verbose > 2) begin
         $write("[%7d] ( |    ) : Scoreboard = [", csr_cycle);
         for(Integer i = 0; i < numRegs; i = i + 1) begin
            $write("%1d, ", rg_scoreGPRs[i] ? 1 : 0);
         end
         $write("], dst = %d, rs1 = %d, rs2 = %d\n", fields.rd, fields.rs1, fields.rs2);
      end

      if (decoded.pc != pcEpoch) begin
         if (cfg_verbose > 1) $display("[%7d] (P|  E ) : STALL Ignore pc = 0x%08h, instr = 0x%08h, epoch = 0x%08h", csr_cycle, decoded.pc, decoded.instr, pcEpoch);
         fifo_d2e.deq;
      end
      else if (score_conflict) begin
         if (cfg_verbose > 1) $display("[%7d] (R|  E ) : STALL Conflict pc = 0x%08h, instr = 0x%h, epoch = 0x%08h", csr_cycle, decoded.pc, decoded.instr, pcEpoch);
      end
      else begin
         Bool checkScore = case(decoded.op.opcode) matches
                              tagged Br .*   : False;
                              tagged St .*   : False;
                              default        : True;
                           endcase;

         // Update dependency flag for $rd
         if (checkScore) begin
            rw_scoreGPRsSet.wset(fields.rd);
         end

         let msg <- fa_exec(decoded, fields);
         if (cfg_verbose > 1) $display("[%7d] ( |  E ) : pc = 0x%08h, instr = 0x%08h, epoch = 0x%08h [", csr_cycle, decoded.pc, decoded.instr, pcEpoch, msg, "]");
         fifo_d2e.deq;

         // Update pcEpoch
         pcEpoch <= fromMaybe(fv_fall_through_pc(pcEpoch), rw_jmpPC.wget);

         // ---------------- FINISH: increment csr_instret or record explicit CSRRx update of csr_instret
         csr_instret <= csr_instret + 1;
      end
   endrule

   // ---------------- RegFile & DMem Write Back
   rule rl_write_back(fifo_e2w.notEmpty);
      let x = fifo_e2w.first;
      fifo_e2w.deq;
      let rd = x.rd;
      let rd_value = ?;

      case(x.rd_value) matches
         tagged MemOp .op: begin
            let ld_op = op.ld_op;
            let align = op.align;
            let resp <- dmemory.mem_resp;

            if (cfg_verbose > 0 && !isValid(resp)) begin
               $display("[%7d] ( |   W) : Memory read failed", csr_cycle);
               $finish;
            end

            let u = fromMaybe(?, resp);
            let data = u >> {align, 3'b0};
            let extendFunc = (ld_op == Lbu || ld_op == Lhu) ? zeroExtend : signExtend;
            rd_value = (case(ld_op)
                    Lb, Lbu: extendFunc(data[7:0]);
                    Lh, Lhu: extendFunc(data[15:0]);
                    default: extendFunc(data[31:0]);
                  endcase);
         end
         tagged Value .value: begin
            rd_value = value;
         end
      endcase

      if (cfg_verbose > 1) $display("[%7d] ( |   W) : %s = %h, clear scoreGPRs[%1d] (= %1d)", csr_cycle, regNameABI[rd], rd_value, rd, rg_scoreGPRs[rd] ? 1 : 0);

      // NOTE: DOES NOT check register x0 because set value to Zero when read
      rf_GPRs.upd(rd, rd_value);
      rw_scoreGPRsReset.wset(rd);
   endrule

   // ---------------- Increment csr_cycle according to external oracles

   rule rl_incr_cycle;
      csr_cycle <= csr_cycle + 1;
   endrule

   // ----------------------------------------------------------------
   // INTERFACE

   method Action start(Addr initial_pc) if (!cpu_enabled);
      rg_FetchPC  <= initial_pc;
      pcEpoch     <= initial_pc;
      cpu_enabled <= True;
   endmethod

   method ActionValue#(Word) cpuToHost() if (csr_dscratch matches tagged Valid .ret);
      return ret;
   endmethod
endmodule

// ================================================================

(* synthesize *)
module mkRISCV(RISCV_IFC);
   (* hide *) let _m <- _mkRISCV(0);
   return _m;
endmodule

// ================================================================
`ifdef TEST_BENCH_RISCV
module mkTb();
   Reg#(Bit#(32))    cycles <- mkConfigReg(0);

   let               dut         <- mkRISCV;
   Reg#(Bit#(16))    csr_int_low <- mkRegU;

   rule do_cycle;
      cycles <= cycles + 1;
      //if (cycles > 1200)
      //   $finish;
   endrule

   rule do_cpuToHost;
      let csr_value <- dut.cpuToHost;
      Bit#(16) csrCmd = truncateLSB(csr_value);
      Bit#(16) csrDat = truncate(csr_value);

      case(csrCmd)
         0: begin // Exit
            if (csrDat == 0) begin
               $fdisplay(stderr, "PASSED\n");
            end
            else begin
               $fdisplay(stderr, "FAILED: exit code = %d\n", csrDat);
            end
            $finish;
         end
         1: begin // PrintChar
            $fwrite(stderr, "%c", csrDat[7:0]);
         end
         2: begin // PrintIntLow
            csr_int_low <= csrDat;
         end
         3: begin // PrintIntHigh
            $fwrite(stderr, "%d", {csrDat, csr_int_low});
         end
         default: begin
            $fdisplay(stderr, "Unknown type %d", csrCmd);
            $finish;
         end
      endcase
   endrule

   rule do_start(cycles == 0);
      dut.start('h200);
   endrule
endmodule
`endif // TESTBENCH

// Copyright (c) 2017 Min Chen.  All Rights Reserved.
// Author: Min Chen

// ================================================================
// BSV library imports

import RegFile       :: *;    // For RISC-V GPRs
import ConfigReg     :: *;
import FIFOF         :: *;
import SpecialFIFOs  :: *;

// ================================================================
// BSV project imports

import ISA_Decls :: *;    // Instruction encodings

// ================================================================
// Memory interface for CPU
// ================================================================

import BRAMCore :: *;
import DReg     :: *;

// ----------------
// IMem responses: either and exception or an instruction

typedef Word IMem_Resp;

// ----------------
// DMem request ops and sizes

typedef enum {
   MEM_OP_LOAD,
   MEM_OP_STORE
} Mem_Op deriving(Eq, Bits, FShow);

// ----------------
// DMem requests

typedef struct {
   Mem_Op               mem_op;
   Addr                 addr;
   Word                 data;          // Only relevant if mem_op == MEM_OP_STORE
   Bit#(Bytes_per_Word) written;
} DMem_Req deriving(Bits);

// ----------------
// DMem responses: either an exception or data
// (data value is only relevant for LOADs, irrelevant for STOREs)

typedef Maybe#(Word) DMem_Resp;

// ----------------
// Memory interface reference design

module mkIMemory#(Reg#(Bit#(64)) cycles)(IMemory_IFC#(size))
   provisos(Add#(a__, size, XLEN));
   BRAM_PORT#(Bit#(size), Word)     mem   <- mkBRAMCore1Load(2 ** valueOf(size), False, (genC ? "mem.vmh" : "R:/mem.vmh"), False);

   method Action mem_req(Addr addr);
      let phyAddr = (addr - imemSt);
      mem.put(False, truncate(phyAddr >> 2), ?);
      //$display("[%7d] [IMEM] ReqAddr = 0x%08h", cycles, addr);
   endmethod

   method ActionValue#(IMem_Resp) mem_resp();
      let x = mem.read();
      return x;
   endmethod
endmodule

module mkDMemory#(Reg#(Bit#(64)) cycles)(DMemory_IFC#(size))
   provisos(Add#(a__, size, XLEN));
   BRAM_DUAL_PORT_BE#(Bit#(size), Word, 4)   mem      <- mkBRAMCore2BELoad(2 ** valueOf(size), False, (genC ? "mem.vmh.D" : "R:/mem.vmh.D"), False);
   Reg#(Bool)                                mem_rd   <- mkDReg(False);

   method Action mem_req(DMem_Req req);
      let phyAddr = (req.addr - dmemSt);
      Bit#(Bits_per_Word_Byte_Index) shift = truncate(req.addr);

      if (shift != 0) begin
         $display("Memory alignment check failed at addr 0x%h", req.addr);
         $finish;
      end

      mem.a.put(req.written, truncate(phyAddr >> 2), req.data);
      mem_rd <= !(req.mem_op == MEM_OP_STORE);
      //$display("[DMEM] Addr = 0x%08h", req.addr);
   endmethod

   method ActionValue#(DMem_Resp) mem_resp;
      Word v = mem.a.read();

      //$display("[DMEM] Data = 0x%08h", v);
      return mem_rd ? tagged Valid v : tagged Invalid;
   endmethod
endmodule

// ----------------------------------------------------------------
// This interface is an argument to the '_mkRISCV' module,
// and is used insided the module to access memory.

interface IMemory_IFC#(numeric type size);
   method Action                  mem_req(Addr addr);
   method ActionValue#(IMem_Resp) mem_resp;
endinterface

interface DMemory_IFC#(numeric type size);
   method Action                  mem_req(DMem_Req req);
   method ActionValue#(DMem_Resp) mem_resp;
endinterface

// ================================================================
// This interfacce is offered by the '_mkRISCV' module to the environment.
// It is not part of the spec, per se, and just has scaffolding that allows
// the environment to control and CPU and probe its state.

interface RISCV_IFC;
   method Action start(Addr initial_pc);

   method ActionValue#(Word) cpuToHost;
endinterface

// ================================================================
// The RISC-V CPU Specification module, 'mkRISCV'

// ----------------
// Default fall-through PC

function Addr fv_fall_through_pc(Addr pc);
   return pc + 4;
endfunction: fv_fall_through_pc

// ----------------

module _mkRISCV#(Bit#(3) cfg_verbose)(RISCV_IFC);

   // CPU state
   Reg#(Bool)  cpu_enabled <- mkReg(False);

   // Program counter
   Reg#(Addr)     rg_FetchPC  <- mkRegU;
   RWire#(Addr)   rw_nxtPC    <- mkRWire;
   RWire#(Addr)   rw_jmpPC    <- mkUnsafeRWire;
   Reg#(Word)     pcEpoch     <- mkConfigRegU;

   // General Purpose Registers
   RegFile#(RegName, Word) rf_GPRs  <- mkRegFileWCF(0, ~0);

   // CSRs
   Reg#(Bit#(64))    csr_cycle   <- mkConfigReg(0);
   Reg#(Bit#(64))    csr_instret <- mkConfigReg(0);

   // internal components
   IMemory_IFC#(14)     imemory  <- mkIMemory(csr_cycle);
   DMemory_IFC#(15)     dmemory  <- mkDMemory(csr_cycle);

   // ----------------
   // These CSRs are technically not present in the user-mode ISA.

   Reg#(Word)  csr_mepc       <- mkRegU;
   Reg#(Word)  csr_mcause     <- mkRegU;
   Reg#(Word)  csr_mbadaddr   <- mkRegU;

   Reg#(Maybe#(Word)) csr_dscratch <- mkDReg(tagged Invalid);

   // ----------------------------------------------------------------
   // Non-architectural state, for this model

   Reg#(Maybe#(Addr))      rg_f2d      <- mkDReg(tagged Invalid);
   FIFOF#(Decoded_Instr)   fifo_d2e    <- mkPipelineFIFOF;
   FIFOF#(Exec2Wb_t)       fifo_e2w    <- mkPipelineFIFOF;

   // ----------------------------------------------------------------
   // Scoreboard map
   Reg#(Bool)        rg_scoreGPRs[numRegs];
   for(Integer i = 0; i < numRegs; i = i + 1)
      rg_scoreGPRs[i] <- mkReg(False);
   RWire#(RegName)   rw_scoreGPRsSet   <- mkRWire;
   RWire#(RegName)   rw_scoreGPRsReset <- mkRWire;

   // ----------------------------------------------------------------
   // Read a CSR
   // If the addr is valid, return tagged Valid value
   // else return tagged Invalid

   function Maybe#(Word) fv_read_csr(CSR_Addr csr_addr);
      if      (csr_addr == csr_CYCLE)     return tagged Valid truncate   (csr_cycle);
      else if (csr_addr == csr_INSTRET)   return tagged Valid truncate   (csr_instret);

      else if (csr_addr == csr_CYCLEH  )  return tagged Valid truncateLSB(csr_cycle);
      else if (csr_addr == csr_INSTRETH)  return tagged Valid truncateLSB(csr_instret);

      else if (csr_addr == csr_DSCRATCH)  return tagged Valid 0 ;

      else return tagged Invalid;
   endfunction: fv_read_csr

   // ----------------------------------------------------------------
   // Write a CSR
   // We assume a valid csr_addr, since this is always preceded by a read_csr which performs the check

   function Action fa_write_csr(CSR_Addr csr_addr, Word csr_value);
      action
         if (csr_addr == csr_DSCRATCH) begin
            csr_dscratch <= tagged Valid csr_value;
         end

         else begin
            $display("ERROR: fa_write_csr: (csr_addr 0x%0h, csr_value 0x%0h): illegal csr_addr", csr_addr, csr_value);
            $finish;
         end
      endaction
   endfunction: fa_write_csr

   // ================================================================
   // Instruction execution

   // ----------------------------------------------------------------
   // The following functions are common idioms for finishing an instruction

   // ----------------
   // Finish exception: record exception cause info, go to ENV_CALL state

   function Action fa_finish_with_exception(Bit#(4) exc_code, Addr badaddr);
      action
         if (cfg_verbose > 0) begin
            $display("[%7d] fa_do_exception: epc = 0x%0h, exc_code = 0x%0h, badaddr = 0x%0h", csr_cycle, pcEpoch, exc_code, badaddr);
         end

         csr_mepc     <= pcEpoch;
         csr_mcause   <= { 1'b0, 0, exc_code };
         csr_mbadaddr <= badaddr;

         $finish;
      endaction
   endfunction

   // ----------------
   // Finish instr with no output (no Rd-write): set PC, go to FETCH state

   function Action fa_finish_with_no_output();
      action
      endaction
   endfunction

   // ----------------
   // Finish instr with Rd-write: set Rd, set PC, go to WRITE_BACK state

   function Action fa_finish_with_Rd(RegName rd, Word rd_value);
      action
         fifo_e2w.enq( Exec2Wb_t {rd: rd, rd_value: tagged Value rd_value} );
         fa_finish_with_no_output;
      endaction
   endfunction

   // ----------------
   // Finish instr with Rd-write: set Rd, set PC, go to WRITE_BACK state

   function Action fa_finish_with_Ld(RegName rd, LdFunc op, Bit#(Bits_per_Word_Byte_Index) align);
      action
         fifo_e2w.enq( Exec2Wb_t {rd: rd, rd_value: (tagged MemOp {ld_op: op, align: align})} );
         fa_finish_with_no_output;
      endaction
   endfunction

   // ----------------
   // Finish jump instrs; write Rd, set PC, go to FETCH state

   function Action fa_finish_jump(RegName rd, Word rd_value, Addr next_pc);
      action
         fifo_e2w.enq( Exec2Wb_t {rd: rd, rd_value: tagged Value rd_value} );
         rw_jmpPC.wset(next_pc);
      endaction
   endfunction

   // ----------------
   // Finish conditional branch instr: set PC, go to FETCH state

   function Action fa_finish_cond_branch(Bool condition_taken, Addr next_pc);
      action
         if (condition_taken) begin
            rw_jmpPC.wset(next_pc);
         end
      endaction
   endfunction

   // ----------------------------------------------------------------
   // Instruction execution
   // This function encapsulates ALL the opcodes.
   // It has internal functions that group related sub-opcodes.

   function ActionValue#(Fmt) fa_exec(Decoded_Instr decoded, Decoded_Fields fields);
      actionvalue

         // Values of Rs1 and Rs2 fields of the instr, unsigned
         //Word v1 = decoded.v1;
         //Word v2 = decoded.v2;
         Word  v1  = ((fields.rs1 == 0) ? 0: rf_GPRs.sub(fields.rs1));
         Word  v2  = ((fields.rs2 == 0) ? 0: rf_GPRs.sub(fields.rs2));

         // Values of Rs1 and Rs2 fields of the instr, signed versions
         Word_S  s_v1 = unpack(v1);
         Word_S  s_v2 = unpack(v2);

         // Value of CSR field of instr (if a valid CSR address)
         Maybe #(Word) m_v_csr = fv_read_csr(fields.csr);

         // ----------------------------------------------------------------
         // Instructions for Upper Immediate
         function ActionValue#(Fmt) fa_exec_LUI();
            actionvalue
               Bit#(32)    v32   = { fields.imm20_U, 12'h0 };
               Word_S      iv    = extend(unpack(v32));
               let         value = pack(iv);

               fa_finish_with_Rd(fields.rd, value);
               let msg = $format("lui %s, 0x%h", regNameABI[fields.rd], value[31:12]);
               return msg;
            endactionvalue
         endfunction: fa_exec_LUI

         function ActionValue#(Fmt) fa_exec_AUIPC();
            actionvalue
               Word_S  iv    = extend(unpack({ fields.imm20_U, 12'b0}));
               Word_S  pc_s  = unpack(pcEpoch);
               Word    value = pack(pc_s + iv);

               fa_finish_with_Rd(fields.rd, value);
               let msg = $format("auipc %s, 0x%h", regNameABI[fields.rd], value[31:12]);
               return msg;
            endactionvalue
         endfunction: fa_exec_AUIPC

         // ----------------------------------------------------------------
         // Instructions for control-transfer

         function ActionValue#(Fmt) fa_exec_JAL();
            actionvalue
               Word_S offset  = extend(unpack(fields.imm21_UJ));
               Addr   next_pc = pack(unpack(pcEpoch) + offset);

               fa_finish_jump(fields.rd, fv_fall_through_pc(pcEpoch), next_pc);
               let msg = $format("jal %s, 0x%h", regNameABI[fields.rd], next_pc);
               return msg;
            endactionvalue
         endfunction: fa_exec_JAL

         function ActionValue#(Fmt) fa_exec_JALR();
            actionvalue
               Word_S offset  = extend(unpack(fields.imm12_I));
               Addr   next_pc = {truncateLSB(pack(s_v1 + offset)), 1'b0};

               fa_finish_jump(fields.rd, fv_fall_through_pc(pcEpoch), next_pc);
               let msg = $format("jalr %s, %s, %1d", regNameABI[fields.rd], regNameABI[fields.rs1], offset);
               return msg;
            endactionvalue
         endfunction: fa_exec_JALR

         function ActionValue#(Fmt) fa_exec_BRANCH(BrFunc op);
            actionvalue
               Word_S offset  = extend(unpack(fields.imm13_SB));
               Word   next_pc = pack(unpack(pcEpoch) + offset);

               case(op)
                  Eq    :  fa_finish_cond_branch(v1  == v2,    next_pc);
                  Neq   :  fa_finish_cond_branch(v1  != v2,    next_pc);
                  Lt    :  fa_finish_cond_branch(s_v1 <  s_v2, next_pc);
                  Ge    :  fa_finish_cond_branch(s_v1 >= s_v2, next_pc);
                  Ltu   :  fa_finish_cond_branch(v1  <  v2,    next_pc);
                  /* Geu */
               default  :  fa_finish_cond_branch(v1  >= v2,    next_pc);
               endcase

               let msg =  $format("B")
                        + fshow(op)
                        + $format(" %s, %s, 0x%h", regNameABI[fields.rd], regNameABI[fields.rs1], next_pc);
               return msg;
            endactionvalue
         endfunction: fa_exec_BRANCH

         // ----------------------------------------------------------------
         // LD and ST instructions.
         // Issue request here; will be completed in STATE_EXEC_LD/ST_RESPONSE

         function ActionValue#(Fmt) fa_exec_LD_Req(LdFunc op);
            actionvalue
               Word_S  imm_s    = extend(unpack(fields.imm12_I));
               Word    mem_addr = pack(s_v1 + imm_s);

               function Action fa_LD_Req(Mem_Data_Size sz);
                  action
                     Bit#(Bits_per_Word_Byte_Index) align = truncate(mem_addr);
                     let req = DMem_Req {mem_op:      MEM_OP_LOAD,
                                         addr:        {mem_addr[xlen-1:2], 2'b00},
                                         written:     0,
                                         data:        ?};
                     dmemory.mem_req(req);
                     fa_finish_with_Ld(fields.rd, op, align);
                  endaction
               endfunction

               case(op)
                  Lb    :  fa_LD_Req(BITS8);
                  Lbu   :  fa_LD_Req(BITS8);
                  Lh    :  fa_LD_Req(BITS16);
                  Lhu   :  fa_LD_Req(BITS16);
                  /*Lw*/
               default  :  fa_LD_Req(BITS32);
               endcase

               let msg =  fshow(op)
                        + $format(" %s, %s, %1d", regNameABI[fields.rd], regNameABI[fields.rs1], imm_s);
               return msg;
            endactionvalue
         endfunction: fa_exec_LD_Req

         function ActionValue#(Fmt) fa_exec_ST_Req(StFunc op);
            actionvalue
               Word_S  imm_s    = extend(unpack(fields.imm12_S));
               Word    mem_addr = pack(s_v1 + imm_s);

               function Action fa_ST_req(Mem_Data_Size sz);
                  action
                     Bit#(Bits_per_Word_Byte_Index) align = truncate(mem_addr);
                     Word aligned_data = v2 << {align, 3'b0};
                     Bit#(Bytes_per_Word) write_en = (case(sz)
                                                         BITS8:  ('b0001 << align);
                                                         BITS16: ('b0011 << align);
                                                         default/*BITS32*/: ('b1111);
                                                      endcase);

                     let req = DMem_Req {mem_op:      MEM_OP_STORE,
                                         addr:        {mem_addr[xlen-1:2], 2'b00},
                                         data:        aligned_data,
                                         written:     write_en};
                     dmemory.mem_req(req);
                     fa_finish_with_no_output;
                  endaction
               endfunction

               case(op)
                  Sb    :  fa_ST_req(BITS8);
                  Sh    :  fa_ST_req(BITS16);
                  /*Sw*/
               default  :  fa_ST_req(BITS32);
               endcase

               let msg =  fshow(op)
                        + $format(" %s, %s, %1d", regNameABI[fields.rd], regNameABI[fields.rs1], imm_s);
               return msg;
            endactionvalue
         endfunction: fa_exec_ST_Req

         // ----------------------------------------------------------------
         // Instructios for Register-Immediate alu ops

         function ActionValue#(Fmt) fa_exec_OP_IMM(AluFunc op);
            actionvalue
               Word                v2    = zeroExtend(fields.imm12_I);
               Word_S              s_v2  = signExtend(unpack(fields.imm12_I));
               Bit#(TLog#(XLEN))   shamt = truncate(fields.imm12_I);

               case(op)
                  Add   :  fa_finish_with_Rd(fields.rd, pack(s_v1 + s_v2));
                  Slt   :  fa_finish_with_Rd(fields.rd, ((s_v1 < s_v2) ? 1 : 0));
                  Sltu  :  fa_finish_with_Rd(fields.rd, ((v1  < pack(s_v2))  ? 1 : 0));
                  Xor   :  fa_finish_with_Rd(fields.rd, pack(s_v1 ^ s_v2));
                  Or    :  fa_finish_with_Rd(fields.rd, pack(s_v1 | s_v2));
                  And   :  fa_finish_with_Rd(fields.rd, pack(s_v1 & s_v2));
                  Sll   :  fa_finish_with_Rd(fields.rd, (v1 << shamt));
                  Srl   :  fa_finish_with_Rd(fields.rd, (v1 >> shamt));
                  /*Sra*/
               default  :  fa_finish_with_Rd(fields.rd, pack(s_v1 >> shamt));
               endcase

               let msg =  fshow(op)
                        + $format("i %s, %s, 0x%h", regNameABI[fields.rd], regNameABI[fields.rs1], fields.imm12_I);
               return msg;
            endactionvalue
         endfunction: fa_exec_OP_IMM

         // ----------------------------------------------------------------
         // Instructios for Register-Register alu ops

         function ActionValue#(Fmt) fa_exec_OP(AluFunc op);
            actionvalue
               Bit#(TLog#(XLEN)) shamt = truncate(v2);    // NOTE: upper bits are unspecified in spec

               case(op)
                  Add   :  fa_finish_with_Rd(fields.rd, pack(s_v1 + s_v2));
                  Sub   :  fa_finish_with_Rd(fields.rd, pack(s_v1 - s_v2));
                  Sll   :  fa_finish_with_Rd(fields.rd, (v1 << shamt));
                  Slt   :  fa_finish_with_Rd(fields.rd, ((s_v1 < s_v2) ? 1 : 0));
                  Sltu  :  fa_finish_with_Rd(fields.rd, ((v1  < v2)  ? 1 : 0));
                  Xor   :  fa_finish_with_Rd(fields.rd, pack(s_v1 ^ s_v2));
                  Srl   :  fa_finish_with_Rd(fields.rd, (v1 >> shamt));
                  Sra   :  fa_finish_with_Rd(fields.rd, pack(s_v1 >> shamt));
                  Or    :  fa_finish_with_Rd(fields.rd, pack(s_v1 | s_v2));
                  /*And*/
               default  :  fa_finish_with_Rd(fields.rd, pack(s_v1 & s_v2));
               endcase

               let msg =  fshow(op)
                        + $format(" %s, %s, %s", regNameABI[fields.rd], regNameABI[fields.rs1], regNameABI[fields.rs2]);
               return msg;
            endactionvalue
         endfunction: fa_exec_OP

         // ----------------------------------------------------------------
         // Instrucions for System-level ops

         function ActionValue#(Fmt) fa_exec_SYSTEM(SysFunc op);
            actionvalue
               let csr_old_val = fromMaybe(?, m_v_csr);

               case(op)
                  CSRRW :  begin
                                 fa_write_csr(fields.csr, v1);
                                 fa_finish_with_Rd(fields.rd, csr_old_val);
                              end

                  CSRRS :  begin
                                 if (fields.rs1 != 0) begin
                                    Word csr_new_val = (csr_old_val | v1);
                                    fa_write_csr(fields.csr, csr_new_val);
                                 end
                                 fa_finish_with_Rd(fields.rd, csr_old_val);
                              end

                  /*CSRRC*/
               default  :  begin
                                 if (fields.rs1 != 0) begin
                                    Word csr_new_val = (csr_old_val & (~ v1));
                                    fa_write_csr(fields.csr, csr_new_val);
                                 end
                                 fa_finish_with_Rd(fields.rd, csr_old_val);
                              end
               endcase

               let msg = ?;
               if ( (op == CSRRS) && (fields.csr == csr_CYCLE) )
                  msg = $format("rdcycle %s", regNameABI[fields.rd]);
               else if ( (op == CSRRS) && (fields.csr == csr_INSTRET) )
                  msg = $format("rdinstret %s", regNameABI[fields.rd]);
               else if ( (op == CSRRW) && (fields.csr == csr_DSCRATCH) )
                  msg = $format("csrw dscratch, %s", regNameABI[fields.rs1]);
               else begin
                  msg =  fshow(op)
                       + $format(" %s, 0x%h, %s", regNameABI[fields.rd], fields.csr, regNameABI[fields.rs1]);
               end
               return msg;
            endactionvalue
         endfunction: fa_exec_SYSTEM

         // ----------------------------------------------------------------
         // Main body of fa_exec(), dispatching to the sub functions
         // based on major OPCODE

         let x = ?;
         case(decoded.op.opcode) matches
            tagged Lui        :  x <- fa_exec_LUI();
            tagged Auipc      :  x <- fa_exec_AUIPC();
            tagged Jal        :  x <- fa_exec_JAL();
            tagged Jalr       :  x <- fa_exec_JALR();
            tagged Br   .op   :  x <- fa_exec_BRANCH(op);
            tagged Ld   .op   :  x <- fa_exec_LD_Req(op);
            tagged St   .op   :  x <- fa_exec_ST_Req(op);
            tagged Alui .op   :  x <- fa_exec_OP_IMM(op);
            tagged Alu  .op   :  x <- fa_exec_OP(op);
            tagged Sys  .op   :  x <- fa_exec_SYSTEM(op);
            default           :  fa_finish_with_exception(exc_code_ILLEGAL_INSTRUCTION, ?);
         endcase
         return x;
      endactionvalue
   endfunction: fa_exec

   // ================================================================
   // The CPU's top-level logic

   // Update CPU internal status in every cycle
   (* fire_when_enabled, no_implicit_conditions *)
   rule update_score;
      let x = fromMaybe(0, rw_scoreGPRsSet.wget);
      let y = fromMaybe(0, rw_scoreGPRsReset.wget);

      rg_scoreGPRs[x] <= True;
      if (x != y) begin
         rg_scoreGPRs[y] <= False;
      end
   endrule

   // ---------------- FETCH
   // Issue instruction request and decode

   // ----------------------------------------------------------------
   // Instruction fetch
   (* fire_when_enabled *)
   rule rl_fetch(cpu_enabled);
      let next_pc = fromMaybe(fromMaybe(rg_FetchPC, rw_nxtPC.wget), rw_jmpPC.wget);
      if (cfg_verbose > 1) $display("[%7d] ( |F   ) : Read instruction pc = 0x%08h", csr_cycle, next_pc);
      imemory.mem_req(next_pc);
      rg_f2d      <= tagged Valid next_pc;
      rg_FetchPC  <= next_pc;
   endrule

   // ----------------------------------------------------------------
   // Instruction decode
   (* conflict_free="rl_decode, rl_exec" *)
   rule rl_decode(fifo_d2e.notFull &&& rg_f2d matches tagged Valid .xPC);
      let instr <- imemory.mem_resp;

      if (cfg_verbose > 1) $display("[%7d] ( | D  ) : pc = 0x%08h, instr = %h", csr_cycle, xPC, instr);
      Decoded_Instr  decoded = fv_decode(xPC, instr);

      fifo_d2e.enq( decoded );
      rw_nxtPC.wset(fv_fall_through_pc(xPC));
   endrule

   // ---------------- EXECUTE
   // Receive instruction from IMem; handle exception if any, else execute it;

   //(* no_implicit_conditions *)
   rule rl_exec(fifo_d2e.notEmpty && fifo_e2w.notFull);
      let decoded = fifo_d2e.first;

      // ----------------------------------------------------------------
      // Instruction fields decode
      Decoded_Fields fields   = fv_decode_fields(decoded.instr);

      // Calculate dependency register
      let score1 = False;
      let score2 = False;

      if (decoded.op.rs1 matches tagged Valid .x &&& x != 0)
         score1 = rg_scoreGPRs[x];

      if (decoded.op.rs2 matches tagged Valid .x &&& x != 0)
         score2 = rg_scoreGPRs[x];

      let score_conflict = score1 || score2;

      if (cfg_verbose > 2) begin
         $write("[%7d] ( |    ) : Scoreboard = [", csr_cycle);
         for(Integer i = 0; i < numRegs; i = i + 1) begin
            $write("%1d, ", rg_scoreGPRs[i] ? 1 : 0);
         end
         $write("], dst = %d, rs1 = %d, rs2 = %d\n", fields.rd, fields.rs1, fields.rs2);
      end

      if (decoded.pc != pcEpoch) begin
         if (cfg_verbose > 1) $display("[%7d] (P|  E ) : STALL Ignore pc = 0x%08h, instr = 0x%08h, epoch = 0x%08h", csr_cycle, decoded.pc, decoded.instr, pcEpoch);
         fifo_d2e.deq;
      end
      else if (score_conflict) begin
         if (cfg_verbose > 1) $display("[%7d] (R|  E ) : STALL Conflict pc = 0x%08h, instr = 0x%h, epoch = 0x%08h", csr_cycle, decoded.pc, decoded.instr, pcEpoch);
      end
      else begin
         Bool checkScore = case(decoded.op.opcode) matches
                              tagged Br .*   : False;
                              tagged St .*   : False;
                              default        : True;
                           endcase;

         // Update dependency flag for $rd
         if (checkScore) begin
            rw_scoreGPRsSet.wset(fields.rd);
         end

         let msg <- fa_exec(decoded, fields);
         if (cfg_verbose > 1) $display("[%7d] ( |  E ) : pc = 0x%08h, instr = 0x%08h, epoch = 0x%08h [", csr_cycle, decoded.pc, decoded.instr, pcEpoch, msg, "]");
         fifo_d2e.deq;

         // Update pcEpoch
         pcEpoch <= fromMaybe(fv_fall_through_pc(pcEpoch), rw_jmpPC.wget);

         // ---------------- FINISH: increment csr_instret or record explicit CSRRx update of csr_instret
         csr_instret <= csr_instret + 1;
      end
   endrule

   // ---------------- RegFile & DMem Write Back
   rule rl_write_back(fifo_e2w.notEmpty);
      let x = fifo_e2w.first;
      fifo_e2w.deq;
      let rd = x.rd;
      let rd_value = ?;

      case(x.rd_value) matches
         tagged MemOp .op: begin
            let ld_op = op.ld_op;
            let align = op.align;
            let resp <- dmemory.mem_resp;

            if (cfg_verbose > 0 && !isValid(resp)) begin
               $display("[%7d] ( |   W) : Memory read failed", csr_cycle);
               $finish;
            end

            let u = fromMaybe(?, resp);
            let data = u >> {align, 3'b0};
            let extendFunc = (ld_op == Lbu || ld_op == Lhu) ? zeroExtend : signExtend;
            rd_value = (case(ld_op)
                    Lb, Lbu: extendFunc(data[7:0]);
                    Lh, Lhu: extendFunc(data[15:0]);
                    default: extendFunc(data[31:0]);
                  endcase);
         end
         tagged Value .value: begin
            rd_value = value;
         end
      endcase

      if (cfg_verbose > 1) $display("[%7d] ( |   W) : %s = %h, clear scoreGPRs[%1d] (= %1d)", csr_cycle, regNameABI[rd], rd_value, rd, rg_scoreGPRs[rd] ? 1 : 0);

      // NOTE: DOES NOT check register x0 because set value to Zero when read
      rf_GPRs.upd(rd, rd_value);
      rw_scoreGPRsReset.wset(rd);
   endrule

   // ---------------- Increment csr_cycle according to external oracles

   rule rl_incr_cycle;
      csr_cycle <= csr_cycle + 1;
   endrule

   // ----------------------------------------------------------------
   // INTERFACE

   method Action start(Addr initial_pc) if (!cpu_enabled);
      rg_FetchPC  <= initial_pc;
      pcEpoch     <= initial_pc;
      cpu_enabled <= True;
   endmethod

   method ActionValue#(Word) cpuToHost() if (csr_dscratch matches tagged Valid .ret);
      return ret;
   endmethod
endmodule

// ================================================================

(* synthesize *)
module mkRISCV(RISCV_IFC);
   (* hide *) let _m <- _mkRISCV(0);
   return _m;
endmodule

// ================================================================
`ifdef TEST_BENCH_RISCV
module mkTb();
   Reg#(Bit#(32))    cycles <- mkConfigReg(0);

   let               dut         <- mkRISCV;
   Reg#(Bit#(16))    csr_int_low <- mkRegU;

   rule do_cycle;
      cycles <= cycles + 1;
      //if (cycles > 1200)
      //   $finish;
   endrule

   rule do_cpuToHost;
      let csr_value <- dut.cpuToHost;
      Bit#(16) csrCmd = truncateLSB(csr_value);
      Bit#(16) csrDat = truncate(csr_value);

      case(csrCmd)
         0: begin // Exit
            if (csrDat == 0) begin
               $fdisplay(stderr, "PASSED\n");
            end
            else begin
               $fdisplay(stderr, "FAILED: exit code = %d\n", csrDat);
            end
            $finish;
         end
         1: begin // PrintChar
            $fwrite(stderr, "%c", csrDat[7:0]);
         end
         2: begin // PrintIntLow
            csr_int_low <= csrDat;
         end
         3: begin // PrintIntHigh
            $fwrite(stderr, "%d", {csrDat, csr_int_low});
         end
         default: begin
            $fdisplay(stderr, "Unknown type %d", csrCmd);
            $finish;
         end
      endcase
   endrule

   rule do_start(cycles == 0);
      dut.start('h200);
   endrule
endmodule
`endif // TESTBENCH