[RISCV] debug information on JAR/JALR

[RISCV] debug information on JAR/JALR

// Copyright (c) 2017 Min Chen.  All Rights Reserved.
// Author: Min Chen

// ================================================================
// BSV library imports

import RegFile   :: *;    // For RISC-V GPRs
import ConfigReg :: *;

// ================================================================
// BSV project imports

import ISA_Decls :: *;    // Instruction encodings

// ================================================================
// Memory interface for CPU
// ================================================================

import BRAMCore :: *;

// ----------------
// IMem responses: either and exception or an instruction

typedef Maybe#(Word) IMem_Resp;

// ----------------
// DMem request ops and sizes

typedef enum {
   MEM_OP_LOAD,
   MEM_OP_STORE
} Mem_Op deriving(Eq, Bits, FShow);

// ----------------
// DMem requests

typedef struct {
   Mem_Op         mem_op;
   Mem_Data_Size  mem_data_size;
   Addr           addr;
   Word           data;          // Only relevant if mem_op == MEM_OP_STORE
} DMem_Req deriving(Bits, FShow);

// ----------------
// DMem responses: either an exception or data
// (data value is only relevant for LOADs, irrelevant for STOREs)

typedef union tagged {
   Exc_Code  DMem_Resp_Exception;
   Word      DMem_Resp_Ok;
} DMem_Resp deriving(Bits, FShow);

// ----------------
// Memory interface reference design

module mkMemory(Memory_IFC);
   BRAM_PORT#(Bit#(14), Word)             imem <- mkBRAMCore1Load(valueOf(TExp#(14)), False, "mem.vmh", False);
   BRAM_DUAL_PORT_BE#(Bit#(15), Word, 4)  dmem <- mkBRAMCore2BELoad(valueOf(TExp#(15)), False, "mem.vmh.D", False);
   Reg#(Bit#(2))                          reg_shift <- mkRegU;
   Reg#(Mem_Data_Size)                    reg_size  <- mkRegU;

   method Action imem_req(Addr addr);
      let phyAddr = addr - imemSt;
      imem.put(False, truncate(phyAddr >> 2), ?);
      //$display("[IMEM] Addr = 0x%08h", addr);
   endmethod

   method ActionValue#(IMem_Resp) imem_resp;
      return tagged Valid imem.read();
   endmethod

   method Action dmem_req(DMem_Req req);
      let phyAddr = (req.addr - dmemSt);
      Word val = req.data;
      Bit#(Bits_per_Word_Byte_Index) shift = truncate(req.addr);
      Bit#(Bytes_per_Word) mask = 4'b1111;
      Word pad = ?;

      reg_shift <= shift;
      reg_size <= req.mem_data_size;

      case(req.mem_data_size)
         BITS8: begin
            case(shift)
               0: begin
                  mask = 4'b0001;
               end
               2'd1: begin
                  mask = 4'b0010;
                  val = {pad[15:0], val[7:0], ?};
               end
               2: begin
                  mask = 4'b0100;
                  val = {pad[7:0], val[7:0], ?};
               end
               3: begin
                  mask = 4'b1000;
                  val = {val[7:0], ?};
               end
            endcase
         end
         BITS16: begin
            if (shift[0] != 0) begin
               $display("Unaligned memory access on 16-bits bound");
               $finish;
            end
            if (shift[1] == 0) begin
               mask = 4'b0011;
            end
            else begin
               mask = 4'b1100;
               val = {val[15:0], ?};
            end
         end
      endcase

      dmem.a.put((req.mem_op == MEM_OP_STORE) ? mask : 0, truncate(phyAddr >> 2), val);
      //$display("[DMEM] Addr = 0x%08h", req.addr);
   endmethod

   method ActionValue#(DMem_Resp) dmem_resp;
      Word v = dmem.a.read();

      case(reg_shift)
         0: v = v;
         1: v = {?, v[31: 8]};
         2: v = {?, v[31:16]};
         3: v = {?, v[31:24]};
      endcase

      return tagged DMem_Resp_Ok v;
   endmethod
endmodule

// ----------------------------------------------------------------
// This interface is an argument to the 'mkRISCV_Spec' module,
// and is used insided the module to access memory.
// MMUs, caches etc. are outside this boundary.

interface Memory_IFC;
   method Action                  imem_req(Addr addr);
   method ActionValue#(IMem_Resp) imem_resp;

   method Action                  dmem_req(DMem_Req req);
   method ActionValue#(DMem_Resp) dmem_resp;
endinterface

// ================================================================
// This interfacce is offered by the 'mkRISCV_Spec' module to the environment.
// It is not part of the spec, per se, and just has scaffolding that allows
// the environment to control and CPU and probe its state.

interface RISCV_IFC;
   method Action start(Addr initial_pc);
   method Bool   halted;

   method ActionValue#(Word) cpuToHost;
endinterface

// ================================================================
// The RISC-V CPU Specification module, 'mkRISCV'

// ----------------
// The CPU is initially in the IDLE state.
// It starts running when it is put into the FETCH state.
// When running, it cycles through the following sequences of states:
//
//     FETCH -> EXEC ->                     FINISH    for most instructions
//     FETCH -> EXEC -> EXEC_LD_RESPONSE -> FINISH    for LD instructions
//     FETCH -> EXEC -> EXEC_ST_RESPONSE -> FINISH    for ST instructions
//

typedef enum {STATE_IDLE,
              STATE_FETCH,
              STATE_EXEC,
              STATE_EXEC_LD_RESPONSE,
              STATE_EXEC_ST_RESPONSE,
              STATE_FINISH,
              STATE_HALT
   } CPU_STATE
deriving(Eq, Bits, FShow);

// ----------------
// Default fall-through PC

function Addr fv_fall_through_pc(Addr pc);
   return pc + 4;
endfunction: fv_fall_through_pc

// ----------------

module _mkRISCV#(Bit#(3) cfg_verbose)(RISCV_IFC);

   // internal components
   Memory_IFC           memory   <- mkMemory;

   // Program counter
   Reg#(Word) pc <- mkRegU;

   // General Purpose Registers
   RegFile#(RegName, Word) regfile  <- mkRegFileFull;

   // CSRs
   Reg#(Bit #(64))   csr_cycle   <- mkConfigReg(0);
   Reg#(Bit #(64))   csr_instret <- mkConfigReg(0);

   // ----------------
   // These CSRs are technically not present in the user-mode ISA.

   Reg#(Word)  csr_mepc       <- mkRegU;
   Reg#(Word)  csr_mcause     <- mkRegU;
   Reg#(Word)  csr_mbadaddr   <- mkRegU;

   Reg#(Maybe#(Word)) csr_dcsr <- mkReg(tagged Invalid);

   // ----------------------------------------------------------------
   // Non-architectural state, for this model

   Reg#(CPU_STATE)   cpu_state   <- mkReg(STATE_IDLE);
   Reg#(Addr)        rg_mem_addr <- mkRegU;    // Effective addr in LD/ST
   Reg#(Word)        rg_instr    <- mkRegU;    // Current instruction

   // ----------------------------------------------------------------
   // Read a CSR
   // If the addr is valid, return tagged Valid value
   // else return tagged Invalid

   function Maybe#(Word) fv_read_csr(CSR_Addr csr_addr);
      if      (csr_addr == csr_CYCLE)     return tagged Valid truncate   (csr_cycle);
      else if (csr_addr == csr_INSTRET)   return tagged Valid truncate   (csr_instret);

      else if (csr_addr == csr_CYCLEH  )  return tagged Valid truncateLSB(csr_cycle);
      else if (csr_addr == csr_INSTRETH)  return tagged Valid truncateLSB(csr_instret);

      else if (csr_addr == csr_DCSR)      return tagged Valid 0 ;

      else return tagged Invalid;
   endfunction: fv_read_csr

   // ----------------------------------------------------------------
   // Write a CSR
   // We assume a valid csr_addr, since this is always preceded by a read_csr which performs the check

   function Action fa_write_csr(CSR_Addr csr_addr, Word csr_value);
      action
              if (csr_addr == csr_CYCLE)    csr_cycle   <= zeroExtend(csr_value);

         else if (csr_addr == csr_CYCLEH)
            csr_cycle <= { csr_value[31:0], truncate(csr_cycle) };

         else if (csr_addr == csr_DCSR) begin
            csr_dcsr <= tagged Valid csr_value;
         end

         else begin
            $display("ERROR: RISCV_Spec.fa_write_csr (csr_addr 0x%0h, csr_value 0x%0h): illegal csr_addr", csr_addr, csr_value);
         end
      endaction
   endfunction: fa_write_csr

   // ================================================================
   // Instruction execution

   // ----------------------------------------------------------------
   // The following functions are common idioms for finishing an instruction

   // ----------------
   // Finish exception: record exception cause info, go to ENV_CALL state

   function Action fa_finish_with_exception(Word epc, Exc_Code exc_code, Addr badaddr);
      action
         if ((cfg_verbose != 0) || (exc_code == exc_code_ILLEGAL_INSTRUCTION)) begin
            $display("[%7d] fa_do_exception: epc = 0x%0h, exc_code = 0x%0h, badaddr = 0x%0h", csr_cycle, epc, exc_code, badaddr);
         end

         csr_mepc     <= epc;
         csr_mcause   <= { 1'b0, 0, exc_code };
         csr_mbadaddr <= badaddr;

         cpu_state    <= STATE_HALT;
      endaction
   endfunction

   // ----------------
   // Finish instr with no output (no Rd-write): set PC, go to FETCH state

   function Action fa_finish_with_no_output();
      action
         pc        <= fv_fall_through_pc (pc);
         cpu_state <= STATE_FINISH;
      endaction
   endfunction

   // ----------------
   // Finish instr with Rd-write: write Rd, set PC, go to FETCH state

   function Action fa_finish_with_Rd(RegName rd, Word rd_value);
      action
         if (rd != 0) regfile.upd(rd, rd_value);
         pc        <= fv_fall_through_pc(pc);
         cpu_state <= STATE_FINISH;
      endaction
   endfunction

   // ----------------
   // Finish jump instrs; write Rd, set PC, go to FETCH state

   function Action fa_finish_jump(RegName rd, Word rd_value, Addr next_pc);
      action
         if (rd != 0) regfile.upd(rd, rd_value);
         pc        <= next_pc;
         cpu_state <= STATE_FINISH;
      endaction
   endfunction

   // ----------------
   // Finish conditional branch instr: set PC, go to FETCH state

   function Action fa_finish_cond_branch(Bool condition_taken, Addr next_pc);
      action
         pc        <= (condition_taken ? next_pc : fv_fall_through_pc(pc));
         cpu_state <= STATE_FINISH;
      endaction
   endfunction

   // ----------------------------------------------------------------
   // Instruction execution
   // This function encapsulates ALL the opcodes.
   // It has internal functions that group related sub-opcodes.

   function Action fa_exec(Decoded_Instr decoded);
      action

         // Values of Rs1 and Rs2 fields of the instr, unsigned
         Word    v1   = ((decoded.rs1 == 0) ? 0: regfile.sub(decoded.rs1));
         Word    v2   = ((decoded.rs2 == 0) ? 0: regfile.sub(decoded.rs2));

         // Values of Rs1 and Rs2 fields of the instr, signed versions
         Word_S  s_v1 = unpack(v1);
         Word_S  s_v2 = unpack(v2);

         // Value of CSR field of instr (if a valid CSR address)
         Maybe #(Word) m_v_csr = fv_read_csr(decoded.csr);

         // ----------------------------------------------------------------
         // Instructions for Upper Immediate

         function Action fa_exec_LUI();
            action
               Bit#(32)    v32   = { decoded.imm20_U, 12'h0 };
               Word_S      iv    = extend(unpack(v32));
               let         value = pack(iv);

               fa_finish_with_Rd(decoded.rd, value);
               if (cfg_verbose > 2) $display("[%7d] Decoded: PC = %h, lui %s, 0x%h", csr_cycle, decoded.pc, regNameABI[decoded.rd], value[31:12]);
            endaction
         endfunction: fa_exec_LUI

         function Action fa_exec_AUIPC();
            action
               Word_S  iv    = extend(unpack({ decoded.imm20_U, 12'b0}));
               Word_S  pc_s  = unpack(pc);
               Word    value = pack(pc_s + iv);

               fa_finish_with_Rd(decoded.rd, value);
               if (cfg_verbose > 2) $display("[%7d] Decoded: PC = %h, auipc %s, 0x%h", csr_cycle, decoded.pc, regNameABI[decoded.rd], value[31:12]);
            endaction
         endfunction: fa_exec_AUIPC

         // ----------------------------------------------------------------
         // Instructions for control-transfer

         function Action fa_exec_JAL();
            action
               Word_S offset  = extend(unpack(decoded.imm21_UJ));
               Addr   next_pc = pack(unpack(pc) + offset);

               fa_finish_jump(decoded.rd, fv_fall_through_pc(pc), next_pc);
               if (cfg_verbose > 2) $display("[%7d] Decoded: PC = %h, jal %s, 0x%h", csr_cycle, decoded.pc, regNameABI[decoded.rd], next_pc);
            endaction
         endfunction: fa_exec_JAL

         function Action fa_exec_JALR();
            action
               Word_S offset  = extend(unpack(decoded.imm12_I));
               Addr   next_pc = {truncateLSB(pack(s_v1 + offset)), 1'b0};

               fa_finish_jump(decoded.rd, fv_fall_through_pc(pc), next_pc);
               if (cfg_verbose > 2) $display("[%7d] Decoded: PC = %h, jalr %s, %s, %1d", csr_cycle, decoded.pc, regNameABI[decoded.rd], regNameABI[decoded.rs1], offset);
            endaction
         endfunction: fa_exec_JALR

         function Action fa_exec_BRANCH();
            action
               Word_S offset  = extend(unpack(decoded.imm13_SB));
               Word   next_pc = pack(unpack(pc) + offset);

               if      (decoded.funct3 == f3_BEQ)  fa_finish_cond_branch(v1  == v2,    next_pc);
               else if (decoded.funct3 == f3_BNE)  fa_finish_cond_branch(v1  != v2,    next_pc);
               else if (decoded.funct3 == f3_BLT)  fa_finish_cond_branch(s_v1 <  s_v2, next_pc);
               else if (decoded.funct3 == f3_BGE)  fa_finish_cond_branch(s_v1 >= s_v2, next_pc);
               else if (decoded.funct3 == f3_BLTU) fa_finish_cond_branch(v1  <  v2,    next_pc);
               else if (decoded.funct3 == f3_BGEU) fa_finish_cond_branch(v1  >= v2,    next_pc);
               else fa_finish_with_exception(pc, exc_code_ILLEGAL_INSTRUCTION, ?);
            endaction
         endfunction: fa_exec_BRANCH

         // ----------------------------------------------------------------
         // LD and ST instructions.
         // Issue request here; will be completed in STATE_EXEC_LD/ST_RESPONSE

         function Action fa_exec_LD_Req();
            action
               Word_S  imm_s    = extend(unpack(decoded.imm12_I));
               Word    mem_addr = pack(s_v1 + imm_s);

               if (cfg_verbose > 1) $display("[%7d] fa_exec_LD_Req: Addr = 0x%08h", csr_cycle, mem_addr);

               function Action fa_LD_Req(Mem_Data_Size sz);
                  action
                     let req = DMem_Req {mem_op:        MEM_OP_LOAD,
                                         mem_data_size: sz,
                                         addr:          mem_addr,
                                         data:          ?};
                     memory.dmem_req(req);
                     cpu_state <= STATE_EXEC_LD_RESPONSE;
                  endaction
               endfunction

               rg_mem_addr <= mem_addr;

               if      (decoded.funct3 == f3_LB)      fa_LD_Req(BITS8);
               else if (decoded.funct3 == f3_LBU)     fa_LD_Req(BITS8);
               else if (decoded.funct3 == f3_LH)      fa_LD_Req(BITS16);
               else if (decoded.funct3 == f3_LHU)     fa_LD_Req(BITS16);
               else if (decoded.funct3 == f3_LW)      fa_LD_Req(BITS32);
               else fa_finish_with_exception(pc, exc_code_ILLEGAL_INSTRUCTION, ?);
            endaction
         endfunction: fa_exec_LD_Req

         function Action fa_exec_ST_Req();
            action
               Word_S  imm_s    = extend(unpack(decoded.imm12_S));
               Word    mem_addr = pack(s_v1 + imm_s);

               function Action fa_ST_req(Mem_Data_Size sz);
                  action
                     let req = DMem_Req {mem_op:        MEM_OP_STORE,
                                         mem_data_size: sz,
                                         addr:          mem_addr,
                                         data:          v2};
                     memory.dmem_req(req);
                     cpu_state <= STATE_EXEC_ST_RESPONSE;
                  endaction
               endfunction

               rg_mem_addr <= mem_addr;

               if      (decoded.funct3 == f3_SB)      fa_ST_req(BITS8);
               else if (decoded.funct3 == f3_SH)      fa_ST_req(BITS16);
               else if (decoded.funct3 == f3_SW)      fa_ST_req(BITS32);
               else fa_finish_with_exception(pc, exc_code_ILLEGAL_INSTRUCTION, ?);
            endaction
         endfunction: fa_exec_ST_Req

         // ----------------------------------------------------------------
         // Instructios for Register-Immediate alu ops

         function Action fa_exec_OP_IMM ();
            action
               Word                v2    = zeroExtend(decoded.imm12_I);
               Word_S              s_v2  = signExtend(unpack(decoded.imm12_I));
               Bit#(TLog#(XLEN))   shamt = truncate(decoded.imm12_I);

               if (cfg_verbose > 2) begin
                  $display("[%7d] Decoded: PC = %h, %s %s, %s, 0x%h", csr_cycle, decoded.pc,
                        case(decoded.funct3)
                           f3_ADDI: "addi";
                           f3_SLTI: "slti";
                           f3_SLTIU: "sltiu";
                           f3_XORI: "xori";
                           f3_ANDI: "andi";
                           f3_SLLI: "slli";
                           f3_SRxI: ((decoded.imm12_I[10] == 1'b0) ? "srli" : "srai");
                        endcase,
                        regNameABI[decoded.rd],
                        regNameABI[decoded.rs1],
                        decoded.imm12_I
                  );
               end

               if      (decoded.funct3 == f3_ADDI)  fa_finish_with_Rd(decoded.rd, pack(s_v1 + s_v2));
               else if (decoded.funct3 == f3_SLTI)  fa_finish_with_Rd(decoded.rd, ((s_v1 < s_v2) ? 1 : 0));
               else if (decoded.funct3 == f3_SLTIU) fa_finish_with_Rd(decoded.rd, ((v1  < v2)  ? 1 : 0));
               else if (decoded.funct3 == f3_XORI)  fa_finish_with_Rd(decoded.rd, pack(s_v1 ^ s_v2));
               else if (decoded.funct3 == f3_ORI)   fa_finish_with_Rd(decoded.rd, pack(s_v1 | s_v2));
               else if (decoded.funct3 == f3_ANDI)  fa_finish_with_Rd(decoded.rd, pack(s_v1 & s_v2));

               else if ((decoded.funct3 == f3_SLLI) && (decoded.imm12_I[10] == 1'b0))
                  fa_finish_with_Rd(decoded.rd, (v1 << shamt));

               // SRLI
               else if ((decoded.funct3 == f3_SRxI) && (decoded.imm12_I[10] == 1'b0))
                  fa_finish_with_Rd(decoded.rd, (v1 >> shamt));

               // SRAI
               else if ((decoded.funct3 == f3_SRxI) && (decoded.imm12_I[10] == 1'b1))
                  fa_finish_with_Rd(decoded.rd, pack(s_v1 >> shamt));

               else
                  fa_finish_with_exception(pc, exc_code_ILLEGAL_INSTRUCTION, ?);
            endaction
         endfunction: fa_exec_OP_IMM

         // ----------------------------------------------------------------
         // Instructios for Register-Register alu ops

         function Action fa_exec_OP();
            action
               Bit#(TLog#(XLEN)) shamt = truncate(v2);    // NOTE: upper bits are unspecified in spec

               if      (decoded.funct10 == f10_ADD)  fa_finish_with_Rd(decoded.rd, pack(s_v1 + s_v2));
               else if (decoded.funct10 == f10_SUB)  fa_finish_with_Rd(decoded.rd, pack(s_v1 - s_v2));
               else if (decoded.funct10 == f10_SLL)  fa_finish_with_Rd(decoded.rd, (v1 << shamt));
               else if (decoded.funct10 == f10_SLT)  fa_finish_with_Rd(decoded.rd, ((s_v1 < s_v2) ? 1 : 0));
               else if (decoded.funct10 == f10_SLTU) fa_finish_with_Rd(decoded.rd, ((v1  < v2)  ? 1 : 0));
               else if (decoded.funct10 == f10_XOR)  fa_finish_with_Rd(decoded.rd, pack(s_v1 ^ s_v2));
               else if (decoded.funct10 == f10_SRL)  fa_finish_with_Rd(decoded.rd, (v1 >> shamt));
               else if (decoded.funct10 == f10_SRA)  fa_finish_with_Rd(decoded.rd, pack(s_v1 >> shamt));
               else if (decoded.funct10 == f10_OR)   fa_finish_with_Rd(decoded.rd, pack(s_v1 | s_v2));
               else if (decoded.funct10 == f10_AND)  fa_finish_with_Rd(decoded.rd, pack(s_v1 & s_v2));

               else
                  fa_finish_with_exception(pc, exc_code_ILLEGAL_INSTRUCTION, ?);
            endaction
         endfunction: fa_exec_OP

         // ----------------------------------------------------------------
         // Instructions for MISC-MEM
         // Currently implemented as no-ops (todo: fix)

         function Action fa_exec_MISC_MEM();
            action
               // FENCE
               if (   (decoded.funct3 == f3_FENCE)
                   && (decoded.rd == 0)
                   && (decoded.rs1 == 0)
                   && (truncateLSB(decoded.imm12_I) == 4'b0)) begin   // same as instr[31:28] on Spec v 2.2
                                                                 fa_finish_with_no_output;
                                                              end

               // FENCE.I
               else if (   (decoded.funct3 == f3_FENCE_I)
                        && (decoded.rd == 0)
                        && (decoded.rs1 == 0)
                        && (decoded.imm12_I == 12'b0)) begin
                                                          fa_finish_with_no_output;
                                                       end

               else begin
                  fa_finish_with_exception(pc, exc_code_ILLEGAL_INSTRUCTION, ?);
               end
            endaction
         endfunction: fa_exec_MISC_MEM

         // ----------------------------------------------------------------
         // Instrucions for System-level ops

         function Action fa_exec_SYSTEM();
            action
               // SCALL/ECALL
               if      (   (decoded.funct3 == f3_PRIV)
                        && (decoded.imm12_I == f12_ECALL)
                        && (decoded.rs1 == 0)
                        && (decoded.rd == 0))
                  fa_finish_with_exception(pc, exc_code_ECALL_FROM_U, ?);

               // SBREAK/EBREAK
               else if (   (decoded.funct3 == f3_PRIV)
                        && (decoded.imm12_I == f12_EBREAK)
                        && (decoded.rs1 == 0)
                        && (decoded.rd == 0))
                  fa_finish_with_exception(pc, exc_code_BREAKPOINT, ?);

               // CSRRW
               else if (m_v_csr matches tagged Valid .csr_old_val &&& (decoded.funct3 == f3_CSRRW)) begin
                  fa_write_csr(decoded.csr, v1);
                  fa_finish_with_Rd(decoded.rd, csr_old_val);
               end

               // CSRRS
               else if (m_v_csr matches tagged Valid .csr_old_val &&& (decoded.funct3 == f3_CSRRS)) begin
                  if (decoded.rs1 != 0) begin
                     Word csr_new_val = (csr_old_val | v1);
                     fa_write_csr(decoded.csr, csr_new_val);
                  end
                  fa_finish_with_Rd(decoded.rd, csr_old_val);
               end

               // CSRRC
               else if (m_v_csr matches tagged Valid .csr_old_val &&& (decoded.funct3 == f3_CSRRC)) begin
                  if (decoded.rs1 != 0) begin
                     Word csr_new_val = (csr_old_val & (~ v1));
                     fa_write_csr(decoded.csr, csr_new_val);
                  end
                  fa_finish_with_Rd(decoded.rd, csr_old_val);
               end

               // CSRRWI
               else if (m_v_csr matches tagged Valid .csr_old_val &&& (decoded.funct3 == f3_CSRRW)) begin
                  let v1 = zeroExtend (decoded.rs1);
                  fa_write_csr(decoded.csr, v1);
                  fa_finish_with_Rd(decoded.rd, csr_old_val);
               end

               // CSRRSI
               else if (m_v_csr matches tagged Valid .csr_old_val &&& (decoded.funct3 == f3_CSRRS)) begin
                  Word v1 = zeroExtend (decoded.rs1);
                  if (v1 != 0) begin
                     Word csr_new_val = (csr_old_val | v1);
                     fa_write_csr(decoded.csr, csr_new_val);
                  end
                  fa_finish_with_Rd(decoded.rd, csr_old_val);
               end

               // CSRRCI
               else if (m_v_csr matches tagged Valid .csr_old_val &&& (decoded.funct3 == f3_CSRRC)) begin
                  Word v1 = zeroExtend (decoded.rs1);
                  if (v1 != 0) begin
                     Word csr_new_val = (csr_old_val & (~ v1));
                     fa_write_csr(decoded.csr, csr_new_val);
                  end
                  fa_finish_with_Rd(decoded.rd, csr_old_val);
               end

               else begin
                  fa_finish_with_exception(pc, exc_code_ILLEGAL_INSTRUCTION, ?);
               end
            endaction
         endfunction: fa_exec_SYSTEM

         // ----------------------------------------------------------------
         // Main body of fa_exec(), dispatching to the sub functions
         // based on major OPCODE

         if      (decoded.opcode == op_LUI)      fa_exec_LUI();
         else if (decoded.opcode == op_AUIPC)    fa_exec_AUIPC();
         else if (decoded.opcode == op_JAL)      fa_exec_JAL();
         else if (decoded.opcode == op_JALR)     fa_exec_JALR();
         else if (decoded.opcode == op_BRANCH)   fa_exec_BRANCH();
         else if (decoded.opcode == op_LOAD)     fa_exec_LD_Req();
         else if (decoded.opcode == op_STORE)    fa_exec_ST_Req();
         else if (decoded.opcode == op_OP_IMM)   fa_exec_OP_IMM();
         else if (decoded.opcode == op_OP)       fa_exec_OP();
         else if (decoded.opcode == op_MISC_MEM) fa_exec_MISC_MEM();
         else if (decoded.opcode == op_SYSTEM)   fa_exec_SYSTEM();
         else fa_finish_with_exception(pc, exc_code_ILLEGAL_INSTRUCTION, ?);
      endaction
   endfunction: fa_exec

   // ================================================================
   // The CPU's top-level logic

   // ---------------- FETCH
   // Issue instruction request

   rule rl_fetch(cpu_state == STATE_FETCH);
      if (cfg_verbose > 1) $display("[%7d] rl_fetch: PC = 0x%08h", csr_cycle, pc);

      memory.imem_req(pc);
      cpu_state <= STATE_EXEC;
   endrule

   // ---------------- EXECUTE
   // Receive instruction from IMem; handle exception if any, else execute it;
   // for LD/ST, issue request and move to LD_RESPONSE/ST_RESPONSE state)

   rule rl_exec(cpu_state == STATE_EXEC);
      let imem_resp <- memory.imem_resp;

      if (imem_resp matches tagged Valid .instr) begin
         if (cfg_verbose > 1) $display("[%7d] rl_exec: PC = 0x%08h, instr = 0x%08h", csr_cycle, pc, instr);
         rg_instr <= instr;

         if (cfg_verbose != 0) $display("[%7d] fa_exec: instr 0x%08h", csr_cycle, instr);

         // ----------------------------------------------------------------
         // Instruction decode

         Decoded_Instr decoded = fv_decode(pc, instr);
         fa_exec(decoded);
      end
   endrule

   // ---------------- LD-responses from DMem

   rule rl_exec_LD_response(cpu_state == STATE_EXEC_LD_RESPONSE);
      if (cfg_verbose > 1) $display("[%7d] rl_exec_LD_response: ", csr_cycle);

      Decoded_Instr decoded = fv_decode(?, rg_instr);

      let resp <- memory.dmem_resp;

      case (resp) matches
         tagged DMem_Resp_Exception .exc_code: fa_finish_with_exception(pc, exc_code, rg_mem_addr);
         tagged DMem_Resp_Ok .u: begin
            case (decoded.funct3)
               f3_LB:  begin
                          Int#(8)   s8    = unpack(truncate(u));
                          Word_S    s     = signExtend(s8);
                          Word      value = pack(s);
                          fa_finish_with_Rd(decoded.rd, value);
                       end
               f3_LBU: begin
                          Bit#(8)   u8    = truncate(u);
                          Word      value = zeroExtend(u8);
                          fa_finish_with_Rd(decoded.rd, value);
                       end
               f3_LH:  begin
                          Int#(16)  s16   = unpack(truncate(u));
                          Word_S    s     = signExtend(s16);
                          Word      value = pack(s);
                          fa_finish_with_Rd(decoded.rd, value);
                       end
               f3_LHU: begin
                          Bit#(16)  u16   = truncate(u);
                          Word      value = zeroExtend(u16);
                          fa_finish_with_Rd(decoded.rd, value);
                       end
               f3_LW:  begin
                          Int#(32)  s32   = unpack(truncate(u));
                          Word_S    s     = signExtend(s32);
                          Word      value = pack(s);
                          fa_finish_with_Rd(decoded.rd, value);
                       end
               // Note: request has already checked that 'default' case is impossible
            endcase
            if (cfg_verbose > 1) $display("RISCV_Spec.rl_exec_LD_response: %08h", u);
         end
      endcase
   endrule

   // ---------------- ST-responses from DMem

   rule rl_exec_ST_response(cpu_state == STATE_EXEC_ST_RESPONSE);
      if (cfg_verbose > 1) $display("[%7d] rl_exec_ST_response: ", csr_cycle);

      DMem_Resp resp <- memory.dmem_resp;

      case (resp) matches
         tagged DMem_Resp_Exception .exc_code: fa_finish_with_exception(pc, exc_code, rg_mem_addr);
         tagged DMem_Resp_Ok        .u:        fa_finish_with_no_output;
      endcase
   endrule

   // ---------------- FINISH: increment csr_instret or record explicit CSRRx update of csr_instret

   rule rl_finish(cpu_state == STATE_FINISH);
      if (cfg_verbose > 1) $display("[%7d] rl_finish: ", csr_cycle);

      csr_instret <= csr_instret + 1;
      cpu_state   <= STATE_FETCH;
   endrule

   // ---------------- Increment csr_cycle according to external oracles

   rule rl_incr_cycle;
      csr_cycle <= csr_cycle + 1;
   endrule

   // ----------------------------------------------------------------
   // INTERFACE

   method Action start(Addr initial_pc);
      pc        <= initial_pc;
      cpu_state <= STATE_FETCH;
   endmethod

   method Bool halted;
      return(cpu_state == STATE_HALT);
   endmethod

   method ActionValue#(Word) cpuToHost() if (csr_dcsr matches tagged Valid .ret);
      csr_dcsr <= tagged Invalid;
      return ret;
   endmethod
endmodule

// ================================================================

(* synthesize *)
module mkRISCV(RISCV_IFC);
   (* hide *) let _m <- _mkRISCV(3);
   return _m;
endmodule

// ================================================================
`ifdef TEST_BENCH_RISCV
module mkTb();
   Reg#(Bit#(32))    cycles <- mkConfigReg(0);

   let               dut         <- mkRISCV;
   Reg#(Bit#(16))    csr_int_low <- mkRegU;

   rule do_cycle;
      cycles <= cycles + 1;
   endrule

   rule do_cpuToHost;
      let csr_value <- dut.cpuToHost;
      Bit#(16) csrCmd = truncateLSB(csr_value);
      Bit#(16) csrDat = truncate(csr_value);

      case(csrCmd)
         0: begin // Exit
            if (csrDat == 0) begin
               $fdisplay(stderr, "PASSED\n");
            end
            else begin
               $fdisplay(stderr, "FAILED: exit code = %d\n", csrDat);
            end
            $finish;
         end
         1: begin // PrintChar
            $fwrite(stderr, "%c", csrDat[7:0]);
         end
         2: begin // PrintIntLow
            csr_int_low <= csrDat;
         end
         3: begin // PrintIntHigh
            $fwrite(stderr, "%d", {csrDat, csr_int_low});
         end
         default: begin
            $fdisplay(stderr, "Unknown type %d", csrCmd);
         end
      endcase
   endrule

   rule do_start(cycles == 0);
      dut.start('h200);
   endrule

   rule do_check(cycles > 0);
      let halted = dut.halted;

      if (halted) begin
         $fdisplay(stderr, "CPU Task Finished");
         $finish;
      end
   endrule
endmodule
`endif // TESTBENCH

// Copyright (c) 2017 Min Chen.  All Rights Reserved.
// Author: Min Chen

// ================================================================
// BSV library imports

import RegFile   :: *;    // For RISC-V GPRs
import ConfigReg :: *;

// ================================================================
// BSV project imports

import ISA_Decls :: *;    // Instruction encodings

// ================================================================
// Memory interface for CPU
// ================================================================

import BRAMCore :: *;

// ----------------
// IMem responses: either and exception or an instruction

typedef Maybe#(Word) IMem_Resp;

// ----------------
// DMem request ops and sizes

typedef enum {
   MEM_OP_LOAD,
   MEM_OP_STORE
} Mem_Op deriving(Eq, Bits, FShow);

// ----------------
// DMem requests

typedef struct {
   Mem_Op         mem_op;
   Mem_Data_Size  mem_data_size;
   Addr           addr;
   Word           data;          // Only relevant if mem_op == MEM_OP_STORE
} DMem_Req deriving(Bits, FShow);

// ----------------
// DMem responses: either an exception or data
// (data value is only relevant for LOADs, irrelevant for STOREs)

typedef union tagged {
   Exc_Code  DMem_Resp_Exception;
   Word      DMem_Resp_Ok;
} DMem_Resp deriving(Bits, FShow);

// ----------------
// Memory interface reference design

module mkMemory(Memory_IFC);
   BRAM_PORT#(Bit#(14), Word)             imem <- mkBRAMCore1Load(valueOf(TExp#(14)), False, "mem.vmh", False);
   BRAM_DUAL_PORT_BE#(Bit#(15), Word, 4)  dmem <- mkBRAMCore2BELoad(valueOf(TExp#(15)), False, "mem.vmh.D", False);
   Reg#(Bit#(2))                          reg_shift <- mkRegU;
   Reg#(Mem_Data_Size)                    reg_size  <- mkRegU;

   method Action imem_req(Addr addr);
      let phyAddr = addr - imemSt;
      imem.put(False, truncate(phyAddr >> 2), ?);
      //$display("[IMEM] Addr = 0x%08h", addr);
   endmethod

   method ActionValue#(IMem_Resp) imem_resp;
      return tagged Valid imem.read();
   endmethod

   method Action dmem_req(DMem_Req req);
      let phyAddr = (req.addr - dmemSt);
      Word val = req.data;
      Bit#(Bits_per_Word_Byte_Index) shift = truncate(req.addr);
      Bit#(Bytes_per_Word) mask = 4'b1111;
      Word pad = ?;

      reg_shift <= shift;
      reg_size <= req.mem_data_size;

      case(req.mem_data_size)
         BITS8: begin
            case(shift)
               0: begin
                  mask = 4'b0001;
               end
               2'd1: begin
                  mask = 4'b0010;
                  val = {pad[15:0], val[7:0], ?};
               end
               2: begin
                  mask = 4'b0100;
                  val = {pad[7:0], val[7:0], ?};
               end
               3: begin
                  mask = 4'b1000;
                  val = {val[7:0], ?};
               end
            endcase
         end
         BITS16: begin
            if (shift[0] != 0) begin
               $display("Unaligned memory access on 16-bits bound");
               $finish;
            end
            if (shift[1] == 0) begin
               mask = 4'b0011;
            end
            else begin
               mask = 4'b1100;
               val = {val[15:0], ?};
            end
         end
      endcase

      dmem.a.put((req.mem_op == MEM_OP_STORE) ? mask : 0, truncate(phyAddr >> 2), val);
      //$display("[DMEM] Addr = 0x%08h", req.addr);
   endmethod

   method ActionValue#(DMem_Resp) dmem_resp;
      Word v = dmem.a.read();

      case(reg_shift)
         0: v = v;
         1: v = {?, v[31: 8]};
         2: v = {?, v[31:16]};
         3: v = {?, v[31:24]};
      endcase

      return tagged DMem_Resp_Ok v;
   endmethod
endmodule

// ----------------------------------------------------------------
// This interface is an argument to the 'mkRISCV_Spec' module,
// and is used insided the module to access memory.
// MMUs, caches etc. are outside this boundary.

interface Memory_IFC;
   method Action                  imem_req(Addr addr);
   method ActionValue#(IMem_Resp) imem_resp;

   method Action                  dmem_req(DMem_Req req);
   method ActionValue#(DMem_Resp) dmem_resp;
endinterface

// ================================================================
// This interfacce is offered by the 'mkRISCV_Spec' module to the environment.
// It is not part of the spec, per se, and just has scaffolding that allows
// the environment to control and CPU and probe its state.

interface RISCV_IFC;
   method Action start(Addr initial_pc);
   method Bool   halted;

   method ActionValue#(Word) cpuToHost;
endinterface

// ================================================================
// The RISC-V CPU Specification module, 'mkRISCV'

// ----------------
// The CPU is initially in the IDLE state.
// It starts running when it is put into the FETCH state.
// When running, it cycles through the following sequences of states:
//
//     FETCH -> EXEC ->                     FINISH    for most instructions
//     FETCH -> EXEC -> EXEC_LD_RESPONSE -> FINISH    for LD instructions
//     FETCH -> EXEC -> EXEC_ST_RESPONSE -> FINISH    for ST instructions
//

typedef enum {STATE_IDLE,
              STATE_FETCH,
              STATE_EXEC,
              STATE_EXEC_LD_RESPONSE,
              STATE_EXEC_ST_RESPONSE,
              STATE_FINISH,
              STATE_HALT
   } CPU_STATE
deriving(Eq, Bits, FShow);

// ----------------
// Default fall-through PC

function Addr fv_fall_through_pc(Addr pc);
   return pc + 4;
endfunction: fv_fall_through_pc

// ----------------

module _mkRISCV#(Bit#(3) cfg_verbose)(RISCV_IFC);

   // internal components
   Memory_IFC           memory   <- mkMemory;

   // Program counter
   Reg#(Word) pc <- mkRegU;

   // General Purpose Registers
   RegFile#(RegName, Word) regfile  <- mkRegFileFull;

   // CSRs
   Reg#(Bit #(64))   csr_cycle   <- mkConfigReg(0);
   Reg#(Bit #(64))   csr_instret <- mkConfigReg(0);

   // ----------------
   // These CSRs are technically not present in the user-mode ISA.

   Reg#(Word)  csr_mepc       <- mkRegU;
   Reg#(Word)  csr_mcause     <- mkRegU;
   Reg#(Word)  csr_mbadaddr   <- mkRegU;

   Reg#(Maybe#(Word)) csr_dcsr <- mkReg(tagged Invalid);

   // ----------------------------------------------------------------
   // Non-architectural state, for this model

   Reg#(CPU_STATE)   cpu_state   <- mkReg(STATE_IDLE);
   Reg#(Addr)        rg_mem_addr <- mkRegU;    // Effective addr in LD/ST
   Reg#(Word)        rg_instr    <- mkRegU;    // Current instruction

   // ----------------------------------------------------------------
   // Read a CSR
   // If the addr is valid, return tagged Valid value
   // else return tagged Invalid

   function Maybe#(Word) fv_read_csr(CSR_Addr csr_addr);
      if      (csr_addr == csr_CYCLE)     return tagged Valid truncate   (csr_cycle);
      else if (csr_addr == csr_INSTRET)   return tagged Valid truncate   (csr_instret);

      else if (csr_addr == csr_CYCLEH  )  return tagged Valid truncateLSB(csr_cycle);
      else if (csr_addr == csr_INSTRETH)  return tagged Valid truncateLSB(csr_instret);

      else if (csr_addr == csr_DCSR)      return tagged Valid 0 ;

      else return tagged Invalid;
   endfunction: fv_read_csr

   // ----------------------------------------------------------------
   // Write a CSR
   // We assume a valid csr_addr, since this is always preceded by a read_csr which performs the check

   function Action fa_write_csr(CSR_Addr csr_addr, Word csr_value);
      action
              if (csr_addr == csr_CYCLE)    csr_cycle   <= zeroExtend(csr_value);

         else if (csr_addr == csr_CYCLEH)
            csr_cycle <= { csr_value[31:0], truncate(csr_cycle) };

         else if (csr_addr == csr_DCSR) begin
            csr_dcsr <= tagged Valid csr_value;
         end

         else begin
            $display("ERROR: RISCV_Spec.fa_write_csr (csr_addr 0x%0h, csr_value 0x%0h): illegal csr_addr", csr_addr, csr_value);
         end
      endaction
   endfunction: fa_write_csr

   // ================================================================
   // Instruction execution

   // ----------------------------------------------------------------
   // The following functions are common idioms for finishing an instruction

   // ----------------
   // Finish exception: record exception cause info, go to ENV_CALL state

   function Action fa_finish_with_exception(Word epc, Exc_Code exc_code, Addr badaddr);
      action
         if ((cfg_verbose != 0) || (exc_code == exc_code_ILLEGAL_INSTRUCTION)) begin
            $display("[%7d] fa_do_exception: epc = 0x%0h, exc_code = 0x%0h, badaddr = 0x%0h", csr_cycle, epc, exc_code, badaddr);
         end

         csr_mepc     <= epc;
         csr_mcause   <= { 1'b0, 0, exc_code };
         csr_mbadaddr <= badaddr;

         cpu_state    <= STATE_HALT;
      endaction
   endfunction

   // ----------------
   // Finish instr with no output (no Rd-write): set PC, go to FETCH state

   function Action fa_finish_with_no_output();
      action
         pc        <= fv_fall_through_pc (pc);
         cpu_state <= STATE_FINISH;
      endaction
   endfunction

   // ----------------
   // Finish instr with Rd-write: write Rd, set PC, go to FETCH state

   function Action fa_finish_with_Rd(RegName rd, Word rd_value);
      action
         if (rd != 0) regfile.upd(rd, rd_value);
         pc        <= fv_fall_through_pc(pc);
         cpu_state <= STATE_FINISH;
      endaction
   endfunction

   // ----------------
   // Finish jump instrs; write Rd, set PC, go to FETCH state

   function Action fa_finish_jump(RegName rd, Word rd_value, Addr next_pc);
      action
         if (rd != 0) regfile.upd(rd, rd_value);
         pc        <= next_pc;
         cpu_state <= STATE_FINISH;
      endaction
   endfunction

   // ----------------
   // Finish conditional branch instr: set PC, go to FETCH state

   function Action fa_finish_cond_branch(Bool condition_taken, Addr next_pc);
      action
         pc        <= (condition_taken ? next_pc : fv_fall_through_pc(pc));
         cpu_state <= STATE_FINISH;
      endaction
   endfunction

   // ----------------------------------------------------------------
   // Instruction execution
   // This function encapsulates ALL the opcodes.
   // It has internal functions that group related sub-opcodes.

   function Action fa_exec(Decoded_Instr decoded);
      action

         // Values of Rs1 and Rs2 fields of the instr, unsigned
         Word    v1   = ((decoded.rs1 == 0) ? 0: regfile.sub(decoded.rs1));
         Word    v2   = ((decoded.rs2 == 0) ? 0: regfile.sub(decoded.rs2));

         // Values of Rs1 and Rs2 fields of the instr, signed versions
         Word_S  s_v1 = unpack(v1);
         Word_S  s_v2 = unpack(v2);

         // Value of CSR field of instr (if a valid CSR address)
         Maybe #(Word) m_v_csr = fv_read_csr(decoded.csr);

         // ----------------------------------------------------------------
         // Instructions for Upper Immediate

         function Action fa_exec_LUI();
            action
               Bit#(32)    v32   = { decoded.imm20_U, 12'h0 };
               Word_S      iv    = extend(unpack(v32));
               let         value = pack(iv);

               fa_finish_with_Rd(decoded.rd, value);
               if (cfg_verbose > 2) $display("[%7d] Decoded: PC = %h, lui %s, 0x%h", csr_cycle, decoded.pc, regNameABI[decoded.rd], value[31:12]);
            endaction
         endfunction: fa_exec_LUI

         function Action fa_exec_AUIPC();
            action
               Word_S  iv    = extend(unpack({ decoded.imm20_U, 12'b0}));
               Word_S  pc_s  = unpack(pc);
               Word    value = pack(pc_s + iv);

               fa_finish_with_Rd(decoded.rd, value);
               if (cfg_verbose > 2) $display("[%7d] Decoded: PC = %h, auipc %s, 0x%h", csr_cycle, decoded.pc, regNameABI[decoded.rd], value[31:12]);
            endaction
         endfunction: fa_exec_AUIPC

         // ----------------------------------------------------------------
         // Instructions for control-transfer

         function Action fa_exec_JAL();
            action
               Word_S offset  = extend(unpack(decoded.imm21_UJ));
               Addr   next_pc = pack(unpack(pc) + offset);

               fa_finish_jump(decoded.rd, fv_fall_through_pc(pc), next_pc);
            endaction
         endfunction: fa_exec_JAL

         function Action fa_exec_JALR();
            action
               Word_S offset  = extend(unpack (decoded.imm12_I));
               Addr   next_pc = pack(s_v1 + offset);

               fa_finish_jump(decoded.rd, fv_fall_through_pc(pc), next_pc);
            endaction
         endfunction: fa_exec_JALR

         function Action fa_exec_BRANCH();
            action
               Word_S offset  = extend(unpack(decoded.imm13_SB));
               Word   next_pc = pack(unpack(pc) + offset);

               if      (decoded.funct3 == f3_BEQ)  fa_finish_cond_branch(v1  == v2,    next_pc);
               else if (decoded.funct3 == f3_BNE)  fa_finish_cond_branch(v1  != v2,    next_pc);
               else if (decoded.funct3 == f3_BLT)  fa_finish_cond_branch(s_v1 <  s_v2, next_pc);
               else if (decoded.funct3 == f3_BGE)  fa_finish_cond_branch(s_v1 >= s_v2, next_pc);
               else if (decoded.funct3 == f3_BLTU) fa_finish_cond_branch(v1  <  v2,    next_pc);
               else if (decoded.funct3 == f3_BGEU) fa_finish_cond_branch(v1  >= v2,    next_pc);
               else fa_finish_with_exception(pc, exc_code_ILLEGAL_INSTRUCTION, ?);
            endaction
         endfunction: fa_exec_BRANCH

         // ----------------------------------------------------------------
         // LD and ST instructions.
         // Issue request here; will be completed in STATE_EXEC_LD/ST_RESPONSE

         function Action fa_exec_LD_Req();
            action
               Word_S  imm_s    = extend(unpack(decoded.imm12_I));
               Word    mem_addr = pack(s_v1 + imm_s);

               if (cfg_verbose > 1) $display("[%7d] fa_exec_LD_Req: Addr = 0x%08h", csr_cycle, mem_addr);

               function Action fa_LD_Req(Mem_Data_Size sz);
                  action
                     let req = DMem_Req {mem_op:        MEM_OP_LOAD,
                                         mem_data_size: sz,
                                         addr:          mem_addr,
                                         data:          ?};
                     memory.dmem_req(req);
                     cpu_state <= STATE_EXEC_LD_RESPONSE;
                  endaction
               endfunction

               rg_mem_addr <= mem_addr;

               if      (decoded.funct3 == f3_LB)      fa_LD_Req(BITS8);
               else if (decoded.funct3 == f3_LBU)     fa_LD_Req(BITS8);
               else if (decoded.funct3 == f3_LH)      fa_LD_Req(BITS16);
               else if (decoded.funct3 == f3_LHU)     fa_LD_Req(BITS16);
               else if (decoded.funct3 == f3_LW)      fa_LD_Req(BITS32);
               else fa_finish_with_exception(pc, exc_code_ILLEGAL_INSTRUCTION, ?);
            endaction
         endfunction: fa_exec_LD_Req

         function Action fa_exec_ST_Req();
            action
               Word_S  imm_s    = extend(unpack(decoded.imm12_S));
               Word    mem_addr = pack(s_v1 + imm_s);

               function Action fa_ST_req(Mem_Data_Size sz);
                  action
                     let req = DMem_Req {mem_op:        MEM_OP_STORE,
                                         mem_data_size: sz,
                                         addr:          mem_addr,
                                         data:          v2};
                     memory.dmem_req(req);
                     cpu_state <= STATE_EXEC_ST_RESPONSE;
                  endaction
               endfunction

               rg_mem_addr <= mem_addr;

               if      (decoded.funct3 == f3_SB)      fa_ST_req(BITS8);
               else if (decoded.funct3 == f3_SH)      fa_ST_req(BITS16);
               else if (decoded.funct3 == f3_SW)      fa_ST_req(BITS32);
               else fa_finish_with_exception(pc, exc_code_ILLEGAL_INSTRUCTION, ?);
            endaction
         endfunction: fa_exec_ST_Req

         // ----------------------------------------------------------------
         // Instructios for Register-Immediate alu ops

         function Action fa_exec_OP_IMM ();
            action
               Word                v2    = zeroExtend(decoded.imm12_I);
               Word_S              s_v2  = signExtend(unpack(decoded.imm12_I));
               Bit#(TLog#(XLEN))   shamt = truncate(decoded.imm12_I);

               if (cfg_verbose > 2) begin
                  $display("[%7d] Decoded: PC = %h, %s %s, %s, 0x%h", csr_cycle, decoded.pc,
                        case(decoded.funct3)
                           f3_ADDI: "addi";
                           f3_SLTI: "slti";
                           f3_SLTIU: "sltiu";
                           f3_XORI: "xori";
                           f3_ANDI: "andi";
                           f3_SLLI: "slli";
                           f3_SRxI: ((decoded.imm12_I[10] == 1'b0) ? "srli" : "srai");
                        endcase,
                        regNameABI[decoded.rd],
                        regNameABI[decoded.rs1],
                        decoded.imm12_I
                  );
               end

               if      (decoded.funct3 == f3_ADDI)  fa_finish_with_Rd(decoded.rd, pack(s_v1 + s_v2));
               else if (decoded.funct3 == f3_SLTI)  fa_finish_with_Rd(decoded.rd, ((s_v1 < s_v2) ? 1 : 0));
               else if (decoded.funct3 == f3_SLTIU) fa_finish_with_Rd(decoded.rd, ((v1  < v2)  ? 1 : 0));
               else if (decoded.funct3 == f3_XORI)  fa_finish_with_Rd(decoded.rd, pack(s_v1 ^ s_v2));
               else if (decoded.funct3 == f3_ORI)   fa_finish_with_Rd(decoded.rd, pack(s_v1 | s_v2));
               else if (decoded.funct3 == f3_ANDI)  fa_finish_with_Rd(decoded.rd, pack(s_v1 & s_v2));

               else if ((decoded.funct3 == f3_SLLI) && (decoded.imm12_I[10] == 1'b0))
                  fa_finish_with_Rd(decoded.rd, (v1 << shamt));

               // SRLI
               else if ((decoded.funct3 == f3_SRxI) && (decoded.imm12_I[10] == 1'b0))
                  fa_finish_with_Rd(decoded.rd, (v1 >> shamt));

               // SRAI
               else if ((decoded.funct3 == f3_SRxI) && (decoded.imm12_I[10] == 1'b1))
                  fa_finish_with_Rd(decoded.rd, pack(s_v1 >> shamt));

               else
                  fa_finish_with_exception(pc, exc_code_ILLEGAL_INSTRUCTION, ?);
            endaction
         endfunction: fa_exec_OP_IMM

         // ----------------------------------------------------------------
         // Instructios for Register-Register alu ops

         function Action fa_exec_OP();
            action
               Bit#(TLog#(XLEN)) shamt = truncate(v2);    // NOTE: upper bits are unspecified in spec

               if      (decoded.funct10 == f10_ADD)  fa_finish_with_Rd(decoded.rd, pack(s_v1 + s_v2));
               else if (decoded.funct10 == f10_SUB)  fa_finish_with_Rd(decoded.rd, pack(s_v1 - s_v2));
               else if (decoded.funct10 == f10_SLL)  fa_finish_with_Rd(decoded.rd, (v1 << shamt));
               else if (decoded.funct10 == f10_SLT)  fa_finish_with_Rd(decoded.rd, ((s_v1 < s_v2) ? 1 : 0));
               else if (decoded.funct10 == f10_SLTU) fa_finish_with_Rd(decoded.rd, ((v1  < v2)  ? 1 : 0));
               else if (decoded.funct10 == f10_XOR)  fa_finish_with_Rd(decoded.rd, pack(s_v1 ^ s_v2));
               else if (decoded.funct10 == f10_SRL)  fa_finish_with_Rd(decoded.rd, (v1 >> shamt));
               else if (decoded.funct10 == f10_SRA)  fa_finish_with_Rd(decoded.rd, pack(s_v1 >> shamt));
               else if (decoded.funct10 == f10_OR)   fa_finish_with_Rd(decoded.rd, pack(s_v1 | s_v2));
               else if (decoded.funct10 == f10_AND)  fa_finish_with_Rd(decoded.rd, pack(s_v1 & s_v2));

               else
                  fa_finish_with_exception(pc, exc_code_ILLEGAL_INSTRUCTION, ?);
            endaction
         endfunction: fa_exec_OP

         // ----------------------------------------------------------------
         // Instructions for MISC-MEM
         // Currently implemented as no-ops (todo: fix)

         function Action fa_exec_MISC_MEM();
            action
               // FENCE
               if (   (decoded.funct3 == f3_FENCE)
                   && (decoded.rd == 0)
                   && (decoded.rs1 == 0)
                   && (truncateLSB(decoded.imm12_I) == 4'b0)) begin   // same as instr[31:28] on Spec v 2.2
                                                                 fa_finish_with_no_output;
                                                              end

               // FENCE.I
               else if (   (decoded.funct3 == f3_FENCE_I)
                        && (decoded.rd == 0)
                        && (decoded.rs1 == 0)
                        && (decoded.imm12_I == 12'b0)) begin
                                                          fa_finish_with_no_output;
                                                       end

               else begin
                  fa_finish_with_exception(pc, exc_code_ILLEGAL_INSTRUCTION, ?);
               end
            endaction
         endfunction: fa_exec_MISC_MEM

         // ----------------------------------------------------------------
         // Instrucions for System-level ops

         function Action fa_exec_SYSTEM();
            action
               // SCALL/ECALL
               if      (   (decoded.funct3 == f3_PRIV)
                        && (decoded.imm12_I == f12_ECALL)
                        && (decoded.rs1 == 0)
                        && (decoded.rd == 0))
                  fa_finish_with_exception(pc, exc_code_ECALL_FROM_U, ?);

               // SBREAK/EBREAK
               else if (   (decoded.funct3 == f3_PRIV)
                        && (decoded.imm12_I == f12_EBREAK)
                        && (decoded.rs1 == 0)
                        && (decoded.rd == 0))
                  fa_finish_with_exception(pc, exc_code_BREAKPOINT, ?);

               // CSRRW
               else if (m_v_csr matches tagged Valid .csr_old_val &&& (decoded.funct3 == f3_CSRRW)) begin
                  fa_write_csr(decoded.csr, v1);
                  fa_finish_with_Rd(decoded.rd, csr_old_val);
               end

               // CSRRS
               else if (m_v_csr matches tagged Valid .csr_old_val &&& (decoded.funct3 == f3_CSRRS)) begin
                  if (decoded.rs1 != 0) begin
                     Word csr_new_val = (csr_old_val | v1);
                     fa_write_csr(decoded.csr, csr_new_val);
                  end
                  fa_finish_with_Rd(decoded.rd, csr_old_val);
               end

               // CSRRC
               else if (m_v_csr matches tagged Valid .csr_old_val &&& (decoded.funct3 == f3_CSRRC)) begin
                  if (decoded.rs1 != 0) begin
                     Word csr_new_val = (csr_old_val & (~ v1));
                     fa_write_csr(decoded.csr, csr_new_val);
                  end
                  fa_finish_with_Rd(decoded.rd, csr_old_val);
               end

               // CSRRWI
               else if (m_v_csr matches tagged Valid .csr_old_val &&& (decoded.funct3 == f3_CSRRW)) begin
                  let v1 = zeroExtend (decoded.rs1);
                  fa_write_csr(decoded.csr, v1);
                  fa_finish_with_Rd(decoded.rd, csr_old_val);
               end

               // CSRRSI
               else if (m_v_csr matches tagged Valid .csr_old_val &&& (decoded.funct3 == f3_CSRRS)) begin
                  Word v1 = zeroExtend (decoded.rs1);
                  if (v1 != 0) begin
                     Word csr_new_val = (csr_old_val | v1);
                     fa_write_csr(decoded.csr, csr_new_val);
                  end
                  fa_finish_with_Rd(decoded.rd, csr_old_val);
               end

               // CSRRCI
               else if (m_v_csr matches tagged Valid .csr_old_val &&& (decoded.funct3 == f3_CSRRC)) begin
                  Word v1 = zeroExtend (decoded.rs1);
                  if (v1 != 0) begin
                     Word csr_new_val = (csr_old_val & (~ v1));
                     fa_write_csr(decoded.csr, csr_new_val);
                  end
                  fa_finish_with_Rd(decoded.rd, csr_old_val);
               end

               else begin
                  fa_finish_with_exception(pc, exc_code_ILLEGAL_INSTRUCTION, ?);
               end
            endaction
         endfunction: fa_exec_SYSTEM

         // ----------------------------------------------------------------
         // Main body of fa_exec(), dispatching to the sub functions
         // based on major OPCODE

         if      (decoded.opcode == op_LUI)      fa_exec_LUI();
         else if (decoded.opcode == op_AUIPC)    fa_exec_AUIPC();
         else if (decoded.opcode == op_JAL)      fa_exec_JAL();
         else if (decoded.opcode == op_JALR)     fa_exec_JALR();
         else if (decoded.opcode == op_BRANCH)   fa_exec_BRANCH();
         else if (decoded.opcode == op_LOAD)     fa_exec_LD_Req();
         else if (decoded.opcode == op_STORE)    fa_exec_ST_Req();
         else if (decoded.opcode == op_OP_IMM)   fa_exec_OP_IMM();
         else if (decoded.opcode == op_OP)       fa_exec_OP();
         else if (decoded.opcode == op_MISC_MEM) fa_exec_MISC_MEM();
         else if (decoded.opcode == op_SYSTEM)   fa_exec_SYSTEM();
         else fa_finish_with_exception(pc, exc_code_ILLEGAL_INSTRUCTION, ?);
      endaction
   endfunction: fa_exec

   // ================================================================
   // The CPU's top-level logic

   // ---------------- FETCH
   // Issue instruction request

   rule rl_fetch(cpu_state == STATE_FETCH);
      if (cfg_verbose > 1) $display("[%7d] rl_fetch: PC = 0x%08h", csr_cycle, pc);

      memory.imem_req(pc);
      cpu_state <= STATE_EXEC;
   endrule

   // ---------------- EXECUTE
   // Receive instruction from IMem; handle exception if any, else execute it;
   // for LD/ST, issue request and move to LD_RESPONSE/ST_RESPONSE state)

   rule rl_exec(cpu_state == STATE_EXEC);
      let imem_resp <- memory.imem_resp;

      if (imem_resp matches tagged Valid .instr) begin
         if (cfg_verbose > 1) $display("[%7d] rl_exec: PC = 0x%08h, instr = 0x%08h", csr_cycle, pc, instr);
         rg_instr <= instr;

         if (cfg_verbose != 0) $display("[%7d] fa_exec: instr 0x%08h", csr_cycle, instr);

         // ----------------------------------------------------------------
         // Instruction decode

         Decoded_Instr decoded = fv_decode(pc, instr);
         fa_exec(decoded);
      end
   endrule

   // ---------------- LD-responses from DMem

   rule rl_exec_LD_response(cpu_state == STATE_EXEC_LD_RESPONSE);
      if (cfg_verbose > 1) $display("[%7d] rl_exec_LD_response: ", csr_cycle);

      Decoded_Instr decoded = fv_decode(?, rg_instr);

      let resp <- memory.dmem_resp;

      case (resp) matches
         tagged DMem_Resp_Exception .exc_code: fa_finish_with_exception(pc, exc_code, rg_mem_addr);
         tagged DMem_Resp_Ok .u: begin
            case (decoded.funct3)
               f3_LB:  begin
                          Int#(8)   s8    = unpack(truncate(u));
                          Word_S    s     = signExtend(s8);
                          Word      value = pack(s);
                          fa_finish_with_Rd(decoded.rd, value);
                       end
               f3_LBU: begin
                          Bit#(8)   u8    = truncate(u);
                          Word      value = zeroExtend(u8);
                          fa_finish_with_Rd(decoded.rd, value);
                       end
               f3_LH:  begin
                          Int#(16)  s16   = unpack(truncate(u));
                          Word_S    s     = signExtend(s16);
                          Word      value = pack(s);
                          fa_finish_with_Rd(decoded.rd, value);
                       end
               f3_LHU: begin
                          Bit#(16)  u16   = truncate(u);
                          Word      value = zeroExtend(u16);
                          fa_finish_with_Rd(decoded.rd, value);
                       end
               f3_LW:  begin
                          Int#(32)  s32   = unpack(truncate(u));
                          Word_S    s     = signExtend(s32);
                          Word      value = pack(s);
                          fa_finish_with_Rd(decoded.rd, value);
                       end
               // Note: request has already checked that 'default' case is impossible
            endcase
            if (cfg_verbose > 1) $display("RISCV_Spec.rl_exec_LD_response: %08h", u);
         end
      endcase
   endrule

   // ---------------- ST-responses from DMem

   rule rl_exec_ST_response(cpu_state == STATE_EXEC_ST_RESPONSE);
      if (cfg_verbose > 1) $display("[%7d] rl_exec_ST_response: ", csr_cycle);

      DMem_Resp resp <- memory.dmem_resp;

      case (resp) matches
         tagged DMem_Resp_Exception .exc_code: fa_finish_with_exception(pc, exc_code, rg_mem_addr);
         tagged DMem_Resp_Ok        .u:        fa_finish_with_no_output;
      endcase
   endrule

   // ---------------- FINISH: increment csr_instret or record explicit CSRRx update of csr_instret

   rule rl_finish(cpu_state == STATE_FINISH);
      if (cfg_verbose > 1) $display("[%7d] rl_finish: ", csr_cycle);

      csr_instret <= csr_instret + 1;
      cpu_state   <= STATE_FETCH;
   endrule

   // ---------------- Increment csr_cycle according to external oracles

   rule rl_incr_cycle;
      csr_cycle <= csr_cycle + 1;
   endrule

   // ----------------------------------------------------------------
   // INTERFACE

   method Action start(Addr initial_pc);
      pc        <= initial_pc;
      cpu_state <= STATE_FETCH;
   endmethod

   method Bool halted;
      return(cpu_state == STATE_HALT);
   endmethod

   method ActionValue#(Word) cpuToHost() if (csr_dcsr matches tagged Valid .ret);
      csr_dcsr <= tagged Invalid;
      return ret;
   endmethod
endmodule

// ================================================================

(* synthesize *)
module mkRISCV(RISCV_IFC);
   (* hide *) let _m <- _mkRISCV(3);
   return _m;
endmodule

// ================================================================
`ifdef TEST_BENCH_RISCV
module mkTb();
   Reg#(Bit#(32))    cycles <- mkConfigReg(0);

   let               dut         <- mkRISCV;
   Reg#(Bit#(16))    csr_int_low <- mkRegU;

   rule do_cycle;
      cycles <= cycles + 1;
   endrule

   rule do_cpuToHost;
      let csr_value <- dut.cpuToHost;
      Bit#(16) csrCmd = truncateLSB(csr_value);
      Bit#(16) csrDat = truncate(csr_value);

      case(csrCmd)
         0: begin // Exit
            if (csrDat == 0) begin
               $fdisplay(stderr, "PASSED\n");
            end
            else begin
               $fdisplay(stderr, "FAILED: exit code = %d\n", csrDat);
            end
            $finish;
         end
         1: begin // PrintChar
            $fwrite(stderr, "%c", csrDat[7:0]);
         end
         2: begin // PrintIntLow
            csr_int_low <= csrDat;
         end
         3: begin // PrintIntHigh
            $fwrite(stderr, "%d", {csrDat, csr_int_low});
         end
         default: begin
            $fdisplay(stderr, "Unknown type %d", csrCmd);
         end
      endcase
   endrule

   rule do_start(cycles == 0);
      dut.start('h200);
   endrule

   rule do_check(cycles > 0);
      let halted = dut.halted;

      if (halted) begin
         $fdisplay(stderr, "CPU Task Finished");
         $finish;
      end
   endrule
endmodule
`endif // TESTBENCH


increase size of default MemreadEngine buffer by cmdQDepth

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import Vector::*;
import FIFOF::*;
import FIFO::*;
import GetPut::*;
import Assert::*;
import ClientServer::*;
import BRAM::*;
import BRAMFIFO::*;
import ConfigCounter::*;
import Connectable::*;

import ConnectalMemory::*;
import MemTypes::*;
import Pipe::*;
import MemUtils::*;

module mkMemreadEngine(MemreadEngine#(dataWidth, cmdQDepth, numServers))
   provisos( Mul#(TDiv#(dataWidth, 8), 8, dataWidth)
	    ,Add#(1, a__, numServers)
	    ,Add#(b__, TLog#(numServers), TAdd#(1, TLog#(TMul#(cmdQDepth,numServers))))
	    ,Add#(c__, TLog#(numServers), TLog#(TMul#(cmdQDepth, numServers)))
	    ,Add#(d__, TLog#(numServers), 6)
	    );
   let rv <- mkMemreadEngineBuff(valueOf(cmdQDepth) * valueOf(TExp#(BurstLenSize)));
   return rv;
endmodule

module mkMemreadEngineBuff#(Integer bufferSizeBytes) (MemreadEngine#(dataWidth, cmdQDepth, numServers))
   provisos (Div#(dataWidth,8,dataWidthBytes),
	     Mul#(dataWidthBytes,8,dataWidth),
	     Log#(dataWidthBytes,beatShift),
	     Log#(cmdQDepth,logCmdQDepth),
	     Mul#(cmdQDepth,numServers,cmdBuffSz),
	     Log#(cmdBuffSz, cmdBuffAddrSz),
	     Log#(numServers, serverIdxSz),
	     Add#(1,logCmdQDepth, outCntSz),
	     Add#(1, c__, numServers),
	     Add#(b__, TLog#(numServers), cmdBuffAddrSz),
	     Add#(e__, TLog#(numServers), TAdd#(1, cmdBuffAddrSz)),
	     Add#(a__, serverIdxSz, cmdBuffAddrSz),
	     Min#(2,TLog#(numServers),bpc),
	     Add#(d__, TLog#(numServers), TAdd#(1, serverIdxSz)),
	     Add#(f__, serverIdxSz, 6));

   let verbose = False;

   Integer bufferSizeBeats = bufferSizeBytes/valueOf(dataWidthBytes);
   Vector#(numServers, Reg#(Bool))          clientInFlight <- replicateM(mkReg(False));
   Vector#(numServers, ConfigCounter#(16))  clientAvail <- replicateM(mkConfigCounter(fromInteger(bufferSizeBeats)));
   Vector#(numServers, Reg#(MemengineCmd))  clientCommand <- replicateM(mkReg(unpack(0)));

   Reg#(Bool) load_in_progress <- mkReg(False);
   FIFO#(Tuple4#(MemengineCmd,Bool,Bool,Bit#(BurstLenSize)))              serverCheckAvail <- mkSizedFIFO(1);
   FIFO#(Tuple4#(Bit#(serverIdxSz),MemengineCmd,Bool,Bit#(BurstLenSize))) serverRequest <- mkSizedFIFO(valueOf(cmdQDepth));
   FIFO#(Tuple3#(Bit#(8),Bit#(serverIdxSz),Bool)) serverProcessing <- mkSizedFIFO(valueOf(cmdQDepth));

   Vector#(numServers, FIFO#(void))         clientResponse <- replicateM(mkSizedFIFO(valueOf(cmdQDepth)));
   Vector#(numServers, FIFO#(MemengineCmd)) clientRequest <- replicateM(mkFIFO());

   FIFOF#(MemData#(dataWidth))                       serverDataFifo <- mkFIFOF;
   Vector#(numServers, FIFOF#(MemData#(dataWidth)))  clientDataFifo <- replicateM(mkSizedBRAMFIFOF(bufferSizeBeats));
   function PipeOut#(Bit#(dataWidth)) check_out(PipeOut#(MemData#(dataWidth)) inpipe, Integer i) =
      (interface PipeOut;
	  method Bit#(dataWidth) first;
	     return inpipe.first.data;
	  endmethod
	  method Action deq;
	     if (verbose) $display("mkMemreadEngineBuff::check_out: idx %d data %h clientAvail %d eob %d", i, inpipe.first.data, clientAvail[i].read(), inpipe.first.last);
	     inpipe.deq;
	     clientAvail[i].increment(1);
	  endmethod
	  method Bool notEmpty = inpipe.notEmpty;
       endinterface);
   Vector#(numServers, PipeOut#(Bit#(dataWidth))) clientDataPipes = zipWith(check_out, map(toPipeOut,clientDataFifo), genVector);

   Reg#(Bit#(8))                    respCnt <- mkReg(0);
   Reg#(Bit#(TAdd#(1,serverIdxSz))) loadIdx <- mkReg(0);
   let beat_shift = fromInteger(valueOf(beatShift));
   let cmd_q_depth = fromInteger(valueOf(cmdQDepth));
   Reg#(Bit#(32)) counter <- mkReg(0);

   rule incCounter;
      counter <= counter + 1;
   endrule

   function Action incr_loadIdx =
      (action
       if(loadIdx+1 >= fromInteger(valueOf(numServers)))
	  loadIdx <= 0;
       else
	  loadIdx <= loadIdx+1;
       endaction);

   for (Integer idx = 0; idx < valueOf(numServers); idx = idx + 1)
      rule store_cmd if (!clientInFlight[idx]);
	 let cmd <- toGet(clientRequest[idx]).get();
	 clientInFlight[idx] <= True;
	 clientCommand[idx] <= cmd;
	 if (verbose) $display("mkMemreadEngineBuff::%d store_cmd %d %d", counter, idx, clientAvail[idx].read);
      endrule

   rule load_ctxt_a (!load_in_progress);
      if (clientInFlight[loadIdx]) begin
	 load_in_progress <= True;
	 let cmd = clientCommand[loadIdx];
         let last_burst = cmd.len <= extend(cmd.burstLen);
         Bit#(BurstLenSize) cmd_len = cmd.burstLen;
	 if (last_burst)
             cmd_len = truncate(cmd.len);
	 let cond0 <- clientAvail[loadIdx].maybeDecrement(unpack(extend(cmd_len>>beat_shift)));
	 serverCheckAvail.enq(tuple4(cmd,cond0,last_burst,cmd_len));
	 if (verbose) $display("mkMemreadEngineBuff::%d load_ctxt_b clientAvail[%d] %d burstLen %d cond0 %d last_burst %d", counter, loadIdx, clientAvail[loadIdx].read(), cmd_len>>beat_shift, cond0, last_burst);
      end
      else begin
	 incr_loadIdx;
      end
   endrule

   // should use an EHR for clientInFlight to avoid the need for this pragma
   (* descending_urgency = "load_ctxt_c, store_cmd" *)
   rule load_ctxt_c if (load_in_progress);
      load_in_progress <= False;
      incr_loadIdx;
      match {.cmd,.cond0,.last_burst,.cmd_len} <- toGet(serverCheckAvail).get;
      if  (cond0) begin
	 if (verbose) $display("mkMemreadEngineBuff::%d load_ctxt_c cmd.len %d idx %d cond0 %d last_burst %d", counter, cmd.len, loadIdx, cond0, last_burst);
	 serverRequest.enq(tuple4(truncate(loadIdx),cmd,last_burst,cmd_len));
	 if (last_burst) begin
	    if (verbose) $display("mkMemreadEngineBuff::%d load_ctxt_b last_burst %d", counter, last_burst);
	    clientResponse[loadIdx].enq(?);
	    clientInFlight[loadIdx] <= False;
	 end
	 else begin
	    let new_cmd = MemengineCmd{sglId:cmd.sglId, base:cmd.base+extend(cmd.burstLen),
				       burstLen:cmd.burstLen, len:cmd.len-extend(cmd.burstLen), tag:cmd.tag};
	    clientCommand[loadIdx] <= new_cmd;
	 end
      end
   endrule

   rule read_data_rule;
      let d <- toGet(serverDataFifo).get();
      match {.rc, .idx, .last_burst} = serverProcessing.first;
      let new_respCnt = respCnt+1;
      let l = False;
      if (verbose) $display("mkMemreadEngineBuff::%d data %h new_respCnt %d rc %d last_burst %d idx %d clientInFlight %d eob %d", counter, d.data, new_respCnt, rc, last_burst, idx, clientInFlight[idx], d.last);
      if (new_respCnt == rc) begin
	 respCnt <= 0;
	 serverProcessing.deq;
	 //$display("eob %d", idx);
	 l = last_burst;
      end
      else begin
	 respCnt <= new_respCnt;
      end
      d.last = l;
      clientDataFifo[idx].enq(d);
   endrule

   function MemreadServer#(dataWidth) toMemreadServer(Server#(MemengineCmd,Bool) cs, PipeOut#(Bit#(dataWidth)) p) =
      (interface MemreadServer;
	  interface cmdServer = cs;
	  interface dataPipe  = p;
       endinterface);

   Vector#(numServers, Server#(MemengineCmd,Bool)) rs;
   for(Integer i = 0; i < valueOf(numServers); i=i+1)
      rs[i] = (interface Server#(MemengineCmd,Bool);
		  interface Put request;
		     method Action put(MemengineCmd cmd);
			Bit#(32) bsb = fromInteger(bufferSizeBytes);
`ifdef BSIM
			Bit#(32) dw = fromInteger(valueOf(dataWidthBytes));
			let mdw = ((cmd.len)/dw)*dw != cmd.len;
			let bbl = extend(cmd.burstLen) > bsb;
			if(bbl || mdw) begin
			   if (bbl)
			      $display("XXXXXXXXXX mkMemreadEngineBuff::unsupported burstLen %d %d", bsb, cmd.burstLen);
			   if (mdw)
			      $display("XXXXXXXXXX mkMemreadEngineBuff::unsupported len %d", cmd.len);
			end
			else begin
`endif
			   clientRequest[i].enq(cmd);
`ifdef BSIM
			end
`endif
 		     endmethod
		  endinterface
		  interface Get response;
		     method ActionValue#(Bool) get;
			clientResponse[i].deq;
			return True;
		     endmethod
		  endinterface
	       endinterface);
   interface readServers = rs;
   interface MemReadClient dmaClient;
      interface Get readReq;
	 method ActionValue#(MemRequest) get();
	    match {.idx, .cmd, .last_burst, .bl} <- toGet(serverRequest).get;
	    serverProcessing.enq(tuple3(truncate(bl>>beat_shift), idx, last_burst));
	    if (verbose) $display("MemreadEngine::%d readReq idx %d offset %h burstLenBytes %h last_burst %d", counter, idx, cmd.base, bl, last_burst);
	    return MemRequest { sglId: cmd.sglId, offset: cmd.base, burstLen:bl, tag: (cmd.tag << valueOf(serverIdxSz)) | extend(idx)};
	 endmethod
      endinterface
      interface Put readData = toPut(serverDataFifo);
   endinterface
   interface dataPipes = clientDataPipes;
   interface read_servers = zipWith(toMemreadServer, rs, clientDataPipes);
endmodule

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import Vector::*;
import FIFOF::*;
import FIFO::*;
import GetPut::*;
import Assert::*;
import ClientServer::*;
import BRAM::*;
import BRAMFIFO::*;
import ConfigCounter::*;
import Connectable::*;

import ConnectalMemory::*;
import MemTypes::*;
import Pipe::*;
import MemUtils::*;

module mkMemreadEngine(MemreadEngine#(dataWidth, cmdQDepth, numServers))
   provisos( Mul#(TDiv#(dataWidth, 8), 8, dataWidth)
	    ,Add#(1, a__, numServers)
	    ,Add#(b__, TLog#(numServers), TAdd#(1, TLog#(TMul#(cmdQDepth,numServers))))
	    ,Add#(c__, TLog#(numServers), TLog#(TMul#(cmdQDepth, numServers)))
	    ,Add#(d__, TLog#(numServers), 6)
	    );
   let rv <- mkMemreadEngineBuff(valueOf(TExp#(BurstLenSize)));
   return rv;
endmodule

module mkMemreadEngineBuff#(Integer bufferSizeBytes) (MemreadEngine#(dataWidth, cmdQDepth, numServers))
   provisos (Div#(dataWidth,8,dataWidthBytes),
	     Mul#(dataWidthBytes,8,dataWidth),
	     Log#(dataWidthBytes,beatShift),
	     Log#(cmdQDepth,logCmdQDepth),
	     Mul#(cmdQDepth,numServers,cmdBuffSz),
	     Log#(cmdBuffSz, cmdBuffAddrSz),
	     Log#(numServers, serverIdxSz),
	     Add#(1,logCmdQDepth, outCntSz),
	     Add#(1, c__, numServers),
	     Add#(b__, TLog#(numServers), cmdBuffAddrSz),
	     Add#(e__, TLog#(numServers), TAdd#(1, cmdBuffAddrSz)),
	     Add#(a__, serverIdxSz, cmdBuffAddrSz),
	     Min#(2,TLog#(numServers),bpc),
	     Add#(d__, TLog#(numServers), TAdd#(1, serverIdxSz)),
	     Add#(f__, serverIdxSz, 6));

   let verbose = False;

   Integer bufferSizeBeats = bufferSizeBytes/valueOf(dataWidthBytes);
   Vector#(numServers, Reg#(Bool))          clientInFlight <- replicateM(mkReg(False));
   Vector#(numServers, ConfigCounter#(16))  clientAvail <- replicateM(mkConfigCounter(fromInteger(bufferSizeBeats)));
   Vector#(numServers, Reg#(MemengineCmd))  clientCommand <- replicateM(mkReg(unpack(0)));

   Reg#(Bool) load_in_progress <- mkReg(False);
   FIFO#(Tuple4#(MemengineCmd,Bool,Bool,Bit#(BurstLenSize)))              serverCheckAvail <- mkSizedFIFO(1);
   FIFO#(Tuple4#(Bit#(serverIdxSz),MemengineCmd,Bool,Bit#(BurstLenSize))) serverRequest <- mkSizedFIFO(valueOf(cmdQDepth));
   FIFO#(Tuple3#(Bit#(8),Bit#(serverIdxSz),Bool)) serverProcessing <- mkSizedFIFO(valueOf(cmdQDepth));

   Vector#(numServers, FIFO#(void))         clientResponse <- replicateM(mkSizedFIFO(valueOf(cmdQDepth)));
   Vector#(numServers, FIFO#(MemengineCmd)) clientRequest <- replicateM(mkFIFO());

   FIFOF#(MemData#(dataWidth))                       serverDataFifo <- mkFIFOF;
   Vector#(numServers, FIFOF#(MemData#(dataWidth)))  clientDataFifo <- replicateM(mkSizedBRAMFIFOF(bufferSizeBeats));
   function PipeOut#(Bit#(dataWidth)) check_out(PipeOut#(MemData#(dataWidth)) inpipe, Integer i) =
      (interface PipeOut;
	  method Bit#(dataWidth) first;
	     return inpipe.first.data;
	  endmethod
	  method Action deq;
	     if (verbose) $display("mkMemreadEngineBuff::check_out: idx %d data %h clientAvail %d eob %d", i, inpipe.first.data, clientAvail[i].read(), inpipe.first.last);
	     inpipe.deq;
	     clientAvail[i].increment(1);
	  endmethod
	  method Bool notEmpty = inpipe.notEmpty;
       endinterface);
   Vector#(numServers, PipeOut#(Bit#(dataWidth))) clientDataPipes = zipWith(check_out, map(toPipeOut,clientDataFifo), genVector);

   Reg#(Bit#(8))                    respCnt <- mkReg(0);
   Reg#(Bit#(TAdd#(1,serverIdxSz))) loadIdx <- mkReg(0);
   let beat_shift = fromInteger(valueOf(beatShift));
   let cmd_q_depth = fromInteger(valueOf(cmdQDepth));
   Reg#(Bit#(32)) counter <- mkReg(0);

   rule incCounter;
      counter <= counter + 1;
   endrule

   function Action incr_loadIdx =
      (action
       if(loadIdx+1 >= fromInteger(valueOf(numServers)))
	  loadIdx <= 0;
       else
	  loadIdx <= loadIdx+1;
       endaction);

   for (Integer idx = 0; idx < valueOf(numServers); idx = idx + 1)
      rule store_cmd if (!clientInFlight[idx]);
	 let cmd <- toGet(clientRequest[idx]).get();
	 clientInFlight[idx] <= True;
	 clientCommand[idx] <= cmd;
	 if (verbose) $display("mkMemreadEngineBuff::%d store_cmd %d %d", counter, idx, clientAvail[idx].read);
      endrule

   rule load_ctxt_a (!load_in_progress);
      if (clientInFlight[loadIdx]) begin
	 load_in_progress <= True;
	 let cmd = clientCommand[loadIdx];
         let last_burst = cmd.len <= extend(cmd.burstLen);
         Bit#(BurstLenSize) cmd_len = cmd.burstLen;
	 if (last_burst)
             cmd_len = truncate(cmd.len);
	 let cond0 <- clientAvail[loadIdx].maybeDecrement(unpack(extend(cmd_len>>beat_shift)));
	 serverCheckAvail.enq(tuple4(cmd,cond0,last_burst,cmd_len));
	 if (verbose) $display("mkMemreadEngineBuff::%d load_ctxt_b clientAvail[%d] %d burstLen %d cond0 %d last_burst %d", counter, loadIdx, clientAvail[loadIdx].read(), cmd_len>>beat_shift, cond0, last_burst);
      end
      else begin
	 incr_loadIdx;
      end
   endrule

   // should use an EHR for clientInFlight to avoid the need for this pragma
   (* descending_urgency = "load_ctxt_c, store_cmd" *)
   rule load_ctxt_c if (load_in_progress);
      load_in_progress <= False;
      incr_loadIdx;
      match {.cmd,.cond0,.last_burst,.cmd_len} <- toGet(serverCheckAvail).get;
      if  (cond0) begin
	 if (verbose) $display("mkMemreadEngineBuff::%d load_ctxt_c cmd.len %d idx %d cond0 %d last_burst %d", counter, cmd.len, loadIdx, cond0, last_burst);
	 serverRequest.enq(tuple4(truncate(loadIdx),cmd,last_burst,cmd_len));
	 if (last_burst) begin
	    if (verbose) $display("mkMemreadEngineBuff::%d load_ctxt_b last_burst %d", counter, last_burst);
	    clientResponse[loadIdx].enq(?);
	    clientInFlight[loadIdx] <= False;
	 end
	 else begin
	    let new_cmd = MemengineCmd{sglId:cmd.sglId, base:cmd.base+extend(cmd.burstLen),
				       burstLen:cmd.burstLen, len:cmd.len-extend(cmd.burstLen), tag:cmd.tag};
	    clientCommand[loadIdx] <= new_cmd;
	 end
      end
   endrule

   rule read_data_rule;
      let d <- toGet(serverDataFifo).get();
      match {.rc, .idx, .last_burst} = serverProcessing.first;
      let new_respCnt = respCnt+1;
      let l = False;
      if (verbose) $display("mkMemreadEngineBuff::%d data %h new_respCnt %d rc %d last_burst %d idx %d clientInFlight %d eob %d", counter, d.data, new_respCnt, rc, last_burst, idx, clientInFlight[idx], d.last);
      if (new_respCnt == rc) begin
	 respCnt <= 0;
	 serverProcessing.deq;
	 //$display("eob %d", idx);
	 l = last_burst;
      end
      else begin
	 respCnt <= new_respCnt;
      end
      d.last = l;
      clientDataFifo[idx].enq(d);
   endrule

   function MemreadServer#(dataWidth) toMemreadServer(Server#(MemengineCmd,Bool) cs, PipeOut#(Bit#(dataWidth)) p) =
      (interface MemreadServer;
	  interface cmdServer = cs;
	  interface dataPipe  = p;
       endinterface);

   Vector#(numServers, Server#(MemengineCmd,Bool)) rs;
   for(Integer i = 0; i < valueOf(numServers); i=i+1)
      rs[i] = (interface Server#(MemengineCmd,Bool);
		  interface Put request;
		     method Action put(MemengineCmd cmd);
			Bit#(32) bsb = fromInteger(bufferSizeBytes);
`ifdef BSIM
			Bit#(32) dw = fromInteger(valueOf(dataWidthBytes));
			let mdw = ((cmd.len)/dw)*dw != cmd.len;
			let bbl = extend(cmd.burstLen) > bsb;
			if(bbl || mdw) begin
			   if (bbl)
			      $display("XXXXXXXXXX mkMemreadEngineBuff::unsupported burstLen %d %d", bsb, cmd.burstLen);
			   if (mdw)
			      $display("XXXXXXXXXX mkMemreadEngineBuff::unsupported len %d", cmd.len);
			end
			else begin
`endif
			   clientRequest[i].enq(cmd);
`ifdef BSIM
			end
`endif
 		     endmethod
		  endinterface
		  interface Get response;
		     method ActionValue#(Bool) get;
			clientResponse[i].deq;
			return True;
		     endmethod
		  endinterface
	       endinterface);
   interface readServers = rs;
   interface MemReadClient dmaClient;
      interface Get readReq;
	 method ActionValue#(MemRequest) get();
	    match {.idx, .cmd, .last_burst, .bl} <- toGet(serverRequest).get;
	    serverProcessing.enq(tuple3(truncate(bl>>beat_shift), idx, last_burst));
	    if (verbose) $display("MemreadEngine::%d readReq idx %d offset %h burstLenBytes %h last_burst %d", counter, idx, cmd.base, bl, last_burst);
	    return MemRequest { sglId: cmd.sglId, offset: cmd.base, burstLen:bl, tag: (cmd.tag << valueOf(serverIdxSz)) | extend(idx)};
	 endmethod
      endinterface
      interface Put readData = toPut(serverDataFifo);
   endinterface
   interface dataPipes = clientDataPipes;
   interface read_servers = zipWith(toMemreadServer, rs, clientDataPipes);
endmodule


import Repeat

import Repeat

// Copyright (c) 2013 Nokia, Inc.
// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import Repeat::*;

typedef enum { sp, spl, spll, splll, spllll, spd, spdd, spddd, spdddd } SpitType driving (Bits);

interface PortalPerfRequest;
   method Action swallowl(Bit#(32) v1);
   method Action swallowll(Bit#(32) v1, Bit#(32) v2);
   method Action swallowlll(Bit#(32) v1, Bit#(32) v2, Bit#(32) v3);
   method Action swallowllll(Bit#(32) v1, Bit#(32) v2, Bit#(32) v3, Bit#(32) v4);
   method Action swallowd(Bit#(64) v1);
   method Action swallowdd(Bit#(64) v1, Bit#(64) v2);
   method Action swallowddd(Bit#(64) v1, Bit#(64) v2, Bit#(64) v3);
   method Action swallowdddd(Bit#(64) v1, Bit#(64) v2, Bit#(64) v3, Bit#(64) v4);
   method Action startspit(SpitType spitType, Bit#(16) loops);
endinterface

interface PortalPerfIndication;
   method Action spit();
   method Action spitl(Bit#(32) v1);
   method Action spitll(Bit#(32) v1, Bit#(32) v2);
   method Action spitlll(Bit#(32) v1, Bit#(32) v2, Bit#(32) v3);
   method Action spitllll(Bit#(32) v1, Bit#(32) v2, Bit#(32) v3, Bit#(32) v4);
   method Action spitd(Bit#(64) v1);
   method Action spitdd(Bit#(64) v1, Bit#(64) v2);
   method Action spitddd(Bit#(64) v1, Bit#(64) v2, Bit#(64) v3);
   method Action spitdddd(Bit#(64) v1, Bit#(64) v2, Bit#(64) v3, Bit#(64) v4);
endinterface

module mkPortalPerfRequest#(PortalPerfIndication indication) (PortalPerfRequest);

   Reg#(Bit#(32)) sinkl1 <- mkReg(0);

   Reg#(Bit#(32)) sinkll1 <- mkReg(0);
   Reg#(Bit#(32)) sinkll2 <- mkReg(0);

   Reg#(Bit#(32)) sinklll1 <- mkReg(0);
   Reg#(Bit#(32)) sinklll2 <- mkReg(0);
   Reg#(Bit#(32)) sinklll3 <- mkReg(0);

   Reg#(Bit#(32)) sinkllll1 <- mkReg(0);
   Reg#(Bit#(32)) sinkllll2 <- mkReg(0);
   Reg#(Bit#(32)) sinkllll3 <- mkReg(0);
   Reg#(Bit#(32)) sinkllll4 <- mkReg(0);

   Reg#(Bit#(64)) sinkd1 <- mkReg(0);

   Reg#(Bit#(64)) sinkdd1 <- mkReg(0);
   Reg#(Bit#(64)) sinkdd2 <- mkReg(0);

   Reg#(Bit#(64)) sinkddd1 <- mkReg(0);
   Reg#(Bit#(64)) sinkddd2 <- mkReg(0);
   Reg#(Bit#(64)) sinkddd3 <- mkReg(0);

   Reg#(Bit#(64)) sinkdddd1 <- mkReg(0);
   Reg#(Bit#(64)) sinkdddd2 <- mkReg(0);
   Reg#(Bit#(64)) sinkdddd3 <- mkReg(0);
   Reg#(Bit#(64)) sinkdddd4 <- mkReg(0);

   Vector(9,Repeat) rfns = ?;
   rfns[0] <- mkRepeat(dospit);
   rfns[1] <- mkRepeat(dospitl);
   rfns[2] <- mkRepeat(dospitll);
   rfns[3] <- mkRepeat(dospitlll);
   rfns[4] <- mkRepeat(dospitllll);
   rfns[5] <- mkRepeat(dospitd);
   rfns[6] <- mkRepeat(dospitdd);
   rfns[7] <- mkRepeat(dospitddd);
   rfns[8] <- mkRepeat(dospitdddd);
   Reg#(Bit#(16)) loops <- mkReg(0);

   method Action swallowl(Bit#(32) v1);
      sinkl1 <= v1;
   endmethod

   method Action swallowll(Bit#(32) v1, Bit#(32) v2);
      sinkll1 <= v1;
      sinkll2 <= v2;
   endmethod

   method Action swallowlll(Bit#(32) v1, Bit#(32) v2, Bit#(32) v3);
      sinklll1 <= v1;
      sinklll2 <= v2;
      sinklll3 <= v3;
   endmethod

   method Action swallowllll(Bit#(32) v1, Bit#(32) v2, Bit#(32) v3, Bit#(32) v4);
      sinkllll1 <= v1;
      sinkllll2 <= v2;
      sinkllll3 <= v3;
      sinkllll4 <= v4;
   endmethod

   method Action swallowd(Bit#(64) v1);
      sinkd1 <= v1;
   endmethod

   method Action swallowdd(Bit#(64) v1, Bit#(64) v2);
      sinkdd1 <= v1;
      sinkdd2 <= v2;
   endmethod

   method Action swallowddd(Bit#(64) v1, Bit#(64) v2, Bit#(64) v3);
      sinkddd1 <= v1;
      sinkddd2 <= v2;
      sinkddd3 <= v3;
   endmethod

   method Action swallowdddd(Bit#(64) v1, Bit#(64) v2, Bit#(64) v3, Bit#(64) v4
      );
      sinkdddd1 <= v1;
      sinkdddd2 <= v2;
      sinkdddd3 <= v3;
      sinkdddd4 <= v4;
   endmethod

   method Action dospit();
      indication.spit();
   endmethod

   method Action dospitl();
      indication.spitl(sinkl1);
   endmethod

   method Action dospitll();
      indication.spitll(sinkll1, sinkll2);
   endmethod

   method Action dospitlll();
      indication.spitlll(sinklll1, sinklll2, sinklll3);
   endmethod

   method Action dospitllll();
      indication.spitllll(sinkllll1, sinkllll2, sinkllll3, sinkllll4);
   endmethod

   method Action dospitd();
      indication.spitd(sinkd1);
   endmethod

   method Action dospitdd();
      indication.spitdd(sinkdd1, sinkdd2);
   endmethod

   method Action dospitddd();
      indication.spitddd(sinkddd1, sinkddd2, sinkddd3);
   endmethod
   method Action dospitdddd();
      indication.spitdddd(sinkdddd1, sinkdddd2, sinkdddd3, sinkdddd4);
   endmethod

   method Action startspit(SpitType spitType, Bit#(16) loops);
      dofns[spittype].do(loops);
   endmethod

endmodule

// Copyright (c) 2013 Nokia, Inc.
// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

typedef enum { sp, spl, spll, splll, spllll, spd, spdd, spddd, spdddd } SpitType driving (Bits);

interface PortalPerfRequest;
   method Action swallowl(Bit#(32) v1);
   method Action swallowll(Bit#(32) v1, Bit#(32) v2);
   method Action swallowlll(Bit#(32) v1, Bit#(32) v2, Bit#(32) v3);
   method Action swallowllll(Bit#(32) v1, Bit#(32) v2, Bit#(32) v3, Bit#(32) v4);
   method Action swallowd(Bit#(64) v1);
   method Action swallowdd(Bit#(64) v1, Bit#(64) v2);
   method Action swallowddd(Bit#(64) v1, Bit#(64) v2, Bit#(64) v3);
   method Action swallowdddd(Bit#(64) v1, Bit#(64) v2, Bit#(64) v3, Bit#(64) v4);
   method Action startspit(SpitType spitType, Bit#(16) loops);
endinterface

interface PortalPerfIndication;
   method Action spit();
   method Action spitl(Bit#(32) v1);
   method Action spitll(Bit#(32) v1, Bit#(32) v2);
   method Action spitlll(Bit#(32) v1, Bit#(32) v2, Bit#(32) v3);
   method Action spitllll(Bit#(32) v1, Bit#(32) v2, Bit#(32) v3, Bit#(32) v4);
   method Action spitd(Bit#(64) v1);
   method Action spitdd(Bit#(64) v1, Bit#(64) v2);
   method Action spitddd(Bit#(64) v1, Bit#(64) v2, Bit#(64) v3);
   method Action spitdddd(Bit#(64) v1, Bit#(64) v2, Bit#(64) v3, Bit#(64) v4);
endinterface

module mkPortalPerfRequest#(PortalPerfIndication indication) (PortalPerfRequest);

   Reg#(Bit#(32)) sinkl1 <- mkReg(0);

   Reg#(Bit#(32)) sinkll1 <- mkReg(0);
   Reg#(Bit#(32)) sinkll2 <- mkReg(0);

   Reg#(Bit#(32)) sinklll1 <- mkReg(0);
   Reg#(Bit#(32)) sinklll2 <- mkReg(0);
   Reg#(Bit#(32)) sinklll3 <- mkReg(0);

   Reg#(Bit#(32)) sinkllll1 <- mkReg(0);
   Reg#(Bit#(32)) sinkllll2 <- mkReg(0);
   Reg#(Bit#(32)) sinkllll3 <- mkReg(0);
   Reg#(Bit#(32)) sinkllll4 <- mkReg(0);

   Reg#(Bit#(64)) sinkd1 <- mkReg(0);

   Reg#(Bit#(64)) sinkdd1 <- mkReg(0);
   Reg#(Bit#(64)) sinkdd2 <- mkReg(0);

   Reg#(Bit#(64)) sinkddd1 <- mkReg(0);
   Reg#(Bit#(64)) sinkddd2 <- mkReg(0);
   Reg#(Bit#(64)) sinkddd3 <- mkReg(0);

   Reg#(Bit#(64)) sinkdddd1 <- mkReg(0);
   Reg#(Bit#(64)) sinkdddd2 <- mkReg(0);
   Reg#(Bit#(64)) sinkdddd3 <- mkReg(0);
   Reg#(Bit#(64)) sinkdddd4 <- mkReg(0);

   Vector(9,Repeat) rfns = ?;
   rfns[0] <- mkRepeat(dospit);
   rfns[1] <- mkRepeat(dospitl);
   rfns[2] <- mkRepeat(dospitll);
   rfns[3] <- mkRepeat(dospitlll);
   rfns[4] <- mkRepeat(dospitllll);
   rfns[5] <- mkRepeat(dospitd);
   rfns[6] <- mkRepeat(dospitdd);
   rfns[7] <- mkRepeat(dospitddd);
   rfns[8] <- mkRepeat(dospitdddd);
   Reg#(Bit#(16)) loops <- mkReg(0);

   method Action swallowl(Bit#(32) v1);
      sinkl1 <= v1;
   endmethod

   method Action swallowll(Bit#(32) v1, Bit#(32) v2);
      sinkll1 <= v1;
      sinkll2 <= v2;
   endmethod

   method Action swallowlll(Bit#(32) v1, Bit#(32) v2, Bit#(32) v3);
      sinklll1 <= v1;
      sinklll2 <= v2;
      sinklll3 <= v3;
   endmethod

   method Action swallowllll(Bit#(32) v1, Bit#(32) v2, Bit#(32) v3, Bit#(32) v4);
      sinkllll1 <= v1;
      sinkllll2 <= v2;
      sinkllll3 <= v3;
      sinkllll4 <= v4;
   endmethod

   method Action swallowd(Bit#(64) v1);
      sinkd1 <= v1;
   endmethod

   method Action swallowdd(Bit#(64) v1, Bit#(64) v2);
      sinkdd1 <= v1;
      sinkdd2 <= v2;
   endmethod

   method Action swallowddd(Bit#(64) v1, Bit#(64) v2, Bit#(64) v3);
      sinkddd1 <= v1;
      sinkddd2 <= v2;
      sinkddd3 <= v3;
   endmethod

   method Action swallowdddd(Bit#(64) v1, Bit#(64) v2, Bit#(64) v3, Bit#(64) v4
      );
      sinkdddd1 <= v1;
      sinkdddd2 <= v2;
      sinkdddd3 <= v3;
      sinkdddd4 <= v4;
   endmethod

   method Action dospit();
      indication.spit();
   endmethod

   method Action dospitl();
      indication.spitl(sinkl1);
   endmethod

   method Action dospitll();
      indication.spitll(sinkll1, sinkll2);
   endmethod

   method Action dospitlll();
      indication.spitlll(sinklll1, sinklll2, sinklll3);
   endmethod

   method Action dospitllll();
      indication.spitllll(sinkllll1, sinkllll2, sinkllll3, sinkllll4);
   endmethod

   method Action dospitd();
      indication.spitd(sinkd1);
   endmethod

   method Action dospitdd();
      indication.spitdd(sinkdd1, sinkdd2);
   endmethod

   method Action dospitddd();
      indication.spitddd(sinkddd1, sinkddd2, sinkddd3);
   endmethod
   method Action dospitdddd();
      indication.spitdddd(sinkdddd1, sinkdddd2, sinkdddd3, sinkdddd4);
   endmethod

   method Action startspit(SpitType spitType, Bit#(16) loops);
      dofns[spittype].do(loops);
   endmethod

endmodule