Add 12 bits of padding on the right for enclave check

// Copyright (c) 2017 Massachusetts Institute of Technology
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

`include "ProcConfig.bsv"
import Vector::*;
import DefaultValue::*;
import ClientServer::*;
import GetPut::*;
import Types::*;
import ProcTypes::*;
import TlbTypes::*;
import Performance::*;
import FullAssocTlb::*;
import ConfigReg::*;
import Ehr::*;
import Fifo::*;
import Cntrs::*;
import SafeCounter::*;
import CacheUtils::*;
import SetAssocTlb::*;
import L2SetAssocTlb::*;
import TranslationCache::*;
import LatencyTimer::*;

// for SV39 only

// curretly L2 TLB is just a blocking page walker

// interface with memory (LLC)
typedef L2TlbReqIdx TlbMemReqId;

typedef struct {
    // this is always a load req
    Addr addr;
    TlbMemReqId id;
} TlbMemReq deriving(Bits, Eq, FShow);

typedef struct {
    Data data;
    TlbMemReqId id;
} TlbLdResp deriving(Bits, Eq, FShow);

interface TlbMemClient;
    interface FifoDeq#(TlbMemReq) memReq;
    interface FifoEnq#(TlbLdResp) respLd;
endinterface

// interface with children (I/D TLB)
typedef union tagged {
    void I;
    DTlbReqIdx D;
} TlbChild deriving(Bits, Eq, FShow);
typedef struct {
    TlbChild child;
    Vpn vpn;
} L2TlbRqFromC deriving(Bits, Eq, FShow);

typedef struct {
    TlbChild child;
    Maybe#(TlbEntry) entry;
} L2TlbRsToC deriving(Bits, Eq, FShow);

interface L2TlbToChildren;
    interface Put#(L2TlbRqFromC) rqFromC;
    interface FifoDeq#(L2TlbRsToC) rsToC;
    // flush with I/D TLB
    interface Put#(void) iTlbReqFlush;
    interface Put#(void) dTlbReqFlush;
    interface Get#(void) flushDone;
endinterface

interface L2Tlb;
    // keep update with changes to CSRs
    method Action updateVMInfo(VMInfo vmI, VMInfo vmD);

    // ifc with ITLb & DTLB
    interface L2TlbToChildren toChildren;

    // ifc with memory (LLC)
    interface TlbMemClient toMem;

    // performace
    interface Perf#(L2TlbPerfType) perf;
endinterface

typedef FullAssocTlb#(`L2_TLB_HUGE_SIZE) L2FullAssocTlb;
module mkL2FullAssocTlb(L2FullAssocTlb);
    let m <- mkFullAssocTlb(True); // randomness in replacement
    return m;
endmodule

// a pending tlb req may be in following states
typedef union tagged {
    void None;
    void WaitMem; // wait for page walk resp from memory
    L2TlbReqIdx WaitPeer; // wait for page walk resp from peer req
} L2TlbWait deriving(Bits, Eq, FShow);

// TODO we should raise load access fault if the PTE address is not a DRAM
// address. (trap value is still the virtual address being translated).

(* synthesize *)
module mkL2Tlb(L2Tlb::L2Tlb);
    Bool verbose = True;

    // set associative TLB for 4KB pages
    L2SetAssocTlb tlb4KB <- mkL2SetAssocTlb;
    // fully associative TLB for mega and giga pages
    L2FullAssocTlb tlbMG <- mkL2FullAssocTlb;
    // FIFO in parallel with TLB
    Fifo#(1, L2TlbReqIdx) tlbReqQ <- mkPipelineFifo;

    // MMU translation cache
    TranslationCache transCache <- mkSplitTransCache;
    // FIFO in parallel with MMU cache
    Fifo#(1, L2TlbReqIdx) transCacheReqQ <- mkPipelineFifo;

    // flush
    Reg#(Bool) iFlushReq <- mkReg(False);
    Reg#(Bool) dFlushReq <- mkReg(False);
    Reg#(Bool) waitFlushDone <- mkReg(False);
    Bool flushing = iFlushReq && dFlushReq;
    Fifo#(1, void) flushDoneQ <- mkCFFifo;

    // req/resp with I/D TLBs
    Fifo#(1, L2TlbRqFromC) rqFromCQ <- mkBypassFifo;
    Fifo#(1, L2TlbRsToC) rsToCQ <- mkBypassFifo;

    // pending reqs
    // pendWait should be meaningful even when entry is invalid. pendWait =
    // WaitMem means this entry is waiting for page walk resp from memory;
    // pendWait = WaitPeer means this entry is waiting for a page walk resp
    // resp initiated by another req. Thus, pendWait must be None when entry is
    // invalid.
    Vector#(L2TlbReqNum, Ehr#(2, Bool)) pendValid <- replicateM(mkEhr(False));
    Vector#(L2TlbReqNum, Reg#(L2TlbRqFromC)) pendReq <- replicateM(mkRegU);
    Vector#(L2TlbReqNum, Ehr#(2, L2TlbWait)) pendWait <- replicateM(mkEhr(None));
    Vector#(L2TlbReqNum, Reg#(PageWalkLevel)) pendWalkLevel <- replicateM(mkRegU);
    Vector#(L2TlbReqNum, Reg#(Addr)) pendWalkAddr <- replicateM(mkRegU);
`ifdef SECURITY
    Vector#(L2TlbReqNum, Reg#(Bool)) pendEnclave <- replicateM(mkRegU);
`endif

    // rule ordering:
    // - trans cache resp < tlb resp < tlb req
    // - trans cache resp mutually exclusive with page walk (trans cache resp
    // has precedence). I don't want them to fire together because both rules
    // update walk level, and I don't want a bypass path.
    // - tlb resp mutually exclusive with page walk (tlb resp has precedence).
    // These two rules cannot fire together, because page walk may update trans
    // cache and tlb resp may request trans cache. tlb resp takes precednece
    // because page walk may update tlb and it needs to wait for tlb resp to
    // deq resp.
    // - tlb req mutually exclusive with page walk (page walk has precedence).
    // They cannot fire together, because page walk may udpate tlb and tlb req
    // may request tlb.

    let pendValid_transCacheResp = getVEhrPort(pendValid, 0); // assert
    let pendValid_tlbResp = getVEhrPort(pendValid, 0);
    let pendValid_pageWalk = getVEhrPort(pendValid, 0);
    let pendValid_tlbReq = getVEhrPort(pendValid, 1);

    let pendWait_transCacheResp = getVEhrPort(pendWait, 0);
    let pendWait_pageWalk = getVEhrPort(pendWait, 0);
    let pendWait_tlbResp = getVEhrPort(pendWait, 1); // perf
    let pendWait_tlbReq = getVEhrPort(pendWait, 1); // assert

    // current processor VM information
    Reg#(VMInfo) vm_info_I <- mkReg(defaultValue);
    Reg#(VMInfo) vm_info_D <- mkReg(defaultValue);

    // Memory Queues for page table walks
    Fifo#(2, TlbMemReq) memReqQ <- mkCFFifo;
    Fifo#(2, TlbLdResp) respLdQ <- mkCFFifo;
    // When a mem resp comes, we first process the initiating req, then process
    // other reqs that in WaitPeer.
    Reg#(Maybe#(L2TlbReqIdx)) respForOtherReq <- mkReg(Invalid);

    // FIFO for perf req
    Fifo#(1, L2TlbPerfType) perfReqQ <- mkCFFifo;
`ifdef PERF_COUNT
    Fifo#(1, PerfResp#(L2TlbPerfType)) perfRespQ <- mkCFFifo;
    Reg#(Bool) doStats <- mkConfigReg(False);
    Count#(Data) instMissCnt <- mkCount(0);
    Count#(Data) instMissLat <- mkCount(0);
    Count#(Data) instPageWalks <- mkCount(0);
    Count#(Data) instSavedPageWalks <- mkCount(0);
    Count#(Data) instHugePageHitCnt <- mkCount(0);
    Count#(Data) instHugePageMissCnt <- mkCount(0);
    Count#(Data) dataMissCnt <- mkCount(0);
    Count#(Data) dataMissLat <- mkCount(0);
    Count#(Data) dataPageWalks <- mkCount(0);
    Count#(Data) dataSavedPageWalks <- mkCount(0);
    Count#(Data) dataHugePageHitCnt <- mkCount(0);
    Count#(Data) dataHugePageMissCnt <- mkCount(0);
    Count#(Data) hitUnderMissCnt <- mkCount(0);
    Count#(Data) allMissCycles <- mkCount(0);
    Count#(Data) peerSavedMemReqCnt <- mkCount(0);

    LatencyTimer#(L2TlbReqNum, 12) latTimer <- mkLatencyTimer; // max latency: 4K cycles

    function Action incrMissLat(TlbChild child, L2TlbReqIdx idx);
    action
        let lat <- latTimer.done(idx);
        if(doStats) begin
            if(child == I) begin
                instMissLat.incr(zeroExtend(lat));
            end
            else begin
                dataMissLat.incr(zeroExtend(lat));
            end
        end
    endaction
    endfunction

    rule incrAllMissCycles(doStats);
        function Bool isMiss(L2TlbWait x) = x != None;
        when(all(isMiss, readVEhr(0, pendWait)), allMissCycles.incr(1));
    endrule

    rule doPerf;
        let t <- toGet(perfReqQ).get;
        Data d = (case(t)
            L2TlbInstMissCnt: (instMissCnt);
            L2TlbInstMissLat: (instMissLat);
            L2TlbInstPageWalks: (instPageWalks);
            L2TlbInstSavedPageWalks: (instSavedPageWalks);
            L2TlbInstHugePageHits: (instHugePageHitCnt);
            L2TlbInstHugePageMisses: (instHugePageMissCnt);
            L2TlbDataMissCnt: (dataMissCnt);
            L2TlbDataMissLat: (dataMissLat);
            L2TlbDataPageWalks: (dataPageWalks);
            L2TlbDataSavedPageWalks: (dataSavedPageWalks);
            L2TlbDataHugePageHits: (dataHugePageHitCnt);
            L2TlbDataHugePageMisses: (dataHugePageMissCnt);
            L2TlbHitUnderMissCnt: (hitUnderMissCnt);
            L2TlbAllMissCycles: (allMissCycles);
            L2TlbPeerSavedMemReqs: (peerSavedMemReqCnt);
            default: (0);
        endcase);
        perfRespQ.enq(PerfResp {
            pType: t,
            data: d
        });
    endrule
`endif

    // when flushing is true, since both I and D TLBs have finished flush and
    // is waiting for L2 to flush, all I/D TLB req must have been responded.
    // Thus, there cannot be any req in pendReq or rqFromCQ.
    rule doStartFlush(flushing && !waitFlushDone);
        waitFlushDone <= True;
        tlb4KB.flush;
        tlbMG.flush;
        transCache.flush;
        // check no req
        doAssert(!rqFromCQ.notEmpty, "cannot have new req");
        doAssert(readVEhr(0, pendValid) == replicate(False), "cannot have pending req");
    endrule

    rule doWaitFlush(flushing && waitFlushDone && tlb4KB.flush_done && transCache.flush_done);
        waitFlushDone <= False;
        flushDoneQ.enq(?);
        iFlushReq <= False;
        dFlushReq <= False;
    endrule

    // tlb req rule is preempted by page walk rule, i.e., don't fire when page
    // walk resp is avaiable
    rule doTlbReq(!flushing && !respLdQ.notEmpty);
        // find a slot for the new req
        L2TlbReqIdx idx = ?;
        if(findIndex( \== (False) , readVReg(pendValid_tlbReq) ) matches tagged Valid .i) begin
            idx = pack(i);
        end
        else begin
            when(False, noAction);
        end
        // get new req
        rqFromCQ.deq;
        let r = rqFromCQ.first;
        // req tlb array
        VMInfo vm_info = r.child == I ? vm_info_I : vm_info_D;
        tlb4KB.req(Translate (SetAssocTlbTranslateReq {
            vpn: r.vpn,
            asid: vm_info.asid
        }));
        tlbReqQ.enq(idx);
        // record req
        pendValid_tlbReq[idx] <= True;
        pendReq[idx] <= r;
        doAssert(!pendValid_tlbReq[idx], "entry must be invalid");
        doAssert(pendWait_tlbReq[idx] == None, "cannot be waiting");
        if(verbose) $display("L2TLB new req: ", fshow(r), "; ", fshow(idx));
    endrule

    // process resp from 4KB TLB and mega-giga TLB
    rule doTlbResp(tlbReqQ.notEmpty);
        doAssert(!flushing, "cannot have pending req when flushing");

        // get req in tlb
        tlbReqQ.deq;
        L2TlbReqIdx idx = tlbReqQ.first;
        L2TlbRqFromC cRq = pendReq[idx];
        doAssert(pendWait_tlbResp[idx] == None, "cannot be waiting");

        // get correct VM info
        VMInfo vm_info = cRq.child == I ? vm_info_I : vm_info_D;
        doAssert(vm_info.sv39, "must be in sv39 mode");

        // get resp from 4KB TLB and mega-giga TLB
        let resp4KB = tlb4KB.resp;
        let respMG = tlbMG.translate(cRq.vpn, vm_info.asid);

        if(verbose) begin
            $display("L2TLB resp: ", fshow(vm_info), " ; ", fshow(cRq), " ; ",
                     fshow(resp4KB), " ; ", fshow(respMG));
        end

        // when page hit, resp to child (4KB array is not dequeued)
        function Action pageHit(TlbEntry entry);
        action
            // resp to child
            rsToCQ.enq(L2TlbRsToC {
                child: cRq.child,
                entry: Valid (entry)
            });
            // req is done
            pendValid_tlbResp[idx] <= False;
`ifdef PERF_COUNT
            // perf: hit under miss
            function Bool otherMiss(L2TlbReqIdx i);
                return pendValid_tlbResp[i] && pendWait_tlbResp[i] != None && i != idx;
            endfunction
            Vector#(L2TlbReqNum, L2TlbReqIdx) idxVec = genWith(fromInteger);
            if(any(otherMiss, idxVec)) begin
                hitUnderMissCnt.incr(1);
            end
`endif
        endaction
        endfunction

        if(!vm_info.sv39) begin
            // not in sv39 -> page fault
            // resp with invalid entry
            rsToCQ.enq(L2TlbRsToC {
                child: cRq.child,
                entry: Invalid
            });
            // deq 4KB TLB array
            tlb4KB.deqResp(Invalid);
            // req is done
            pendValid_tlbResp[idx] <= False;
        end
        else if(respMG.hit) begin
            // hit on a mega or giga page
            let entry = respMG.entry;
            doAssert(entry.level > 0 && entry.level <= maxPageWalkLevel,
                     "mega or giga page");
            pageHit(entry);
            tlb4KB.deqResp(Invalid); // just deq 4KB array
            tlbMG.updateRepByHit(respMG.index); // update replacement in MG array
`ifdef PERF_COUNT
            if(doStats) begin
                if(cRq.child == I) begin
                    instHugePageHitCnt.incr(1);
                end
                else begin
                    dataHugePageHitCnt.incr(1);
                end
            end
`endif
        end
        else if(resp4KB.hit) begin
            // hit on 4KB page
            let entry = resp4KB.entry;
            doAssert(entry.level == 0, "must be 4KB page");
            pageHit(entry);
            // update 4KB array replacement, no need to touch MG array
            tlb4KB.deqResp(Valid (resp4KB.way));
        end
        else begin
            // miss, deq resp
            tlb4KB.deqResp(Invalid);
            // check translation cache
            transCache.req(cRq.vpn, vm_info.asid);
            transCacheReqQ.enq(idx);
            // perf: TLB miss
`ifdef PERF_COUNT
            latTimer.start(idx);
            if(doStats) begin
                if(cRq.child == I) begin
                    instMissCnt.incr(1);
                end
                else begin
                    dataMissCnt.incr(1);
                end
            end
`endif
        end
    endrule

    // Find other req doing the same page walk. Although it should be ok to
    // just compare the VPNs, for safety, we compare the PTE addrs.
    function Maybe#(L2TlbReqIdx) otherReqSamePTE(
        L2TlbReqIdx idx, Addr pteAddr,
        Vector#(L2TlbReqNum, L2TlbWait) waitVec
    );
        function Bool samePTE(L2TlbReqIdx i);
            return i != idx && waitVec[i] == WaitMem && pendWalkAddr[i] == pteAddr;
        endfunction
        Vector#(L2TlbReqNum, L2TlbReqIdx) idxVec = genWith(fromInteger);
        return find(samePTE, idxVec);
    endfunction

    rule doTranslationCacheResp(transCacheReqQ.notEmpty);
        // get req in trans cache
        transCacheReqQ.deq;
        L2TlbReqIdx idx = transCacheReqQ.first;
        L2TlbRqFromC cRq = pendReq[idx];
        // get trans cache resp
        transCache.deqResp;
        let resp = transCache.resp;
        // start page walk based on the translation cache resp. Note that if
        // the startLevel in resp is max, then we should use the base ppn in vm
        // info.
        // XXX We believe the PPN stored in translation cache has been
        // sanitized before, so we don't need to check again if the PPN is an
        // enclave PPN or not. (This should follow the same reasoning that TLB
        // hits do not need checks.)
        VMInfo vm_info = cRq.child == I ? vm_info_I : vm_info_D;
        PageWalkLevel level = resp.startLevel;
        Ppn rootPPN = vm_info.basePPN; // root of page table
`ifdef SECURITY
        Bool isEnclave = (zeroExtend({cRq.vpn, PageOffset'0}) & vm_info.sanctum_evmask) == vm_info.sanctum_evbase;
        if(isEnclave) begin
            rootPPN = vm_info.sanctum_ebasePPN; // enclave has its own root of page table
        end
`endif
        Addr baseAddr = getPTBaseAddr(level < maxPageWalkLevel ? resp.ppn : rootPPN);
        Addr pteAddr = getPTEAddr(baseAddr, cRq.vpn, level);
        // record page walk info
        pendWalkLevel[idx] <= level;
        pendWalkAddr[idx] <= pteAddr;
`ifdef SECURITY
        pendEnclave[idx] <= isEnclave;
`endif
        doAssert(pendWait_transCacheResp[idx] == None, "cannot be waiting");
        // don't req memory if someone else is also doing the same page walk.
        if(otherReqSamePTE(idx, pteAddr, readVReg(pendWait_transCacheResp)) matches tagged Valid .i) begin
            // peer entry has already requested, so don't send
            // duplicate req
            pendWait_transCacheResp[idx] <= WaitPeer (i);
            doAssert(pendValid_transCacheResp[i], "peer must be valid");
`ifdef PERF_COUNT
            if(doStats) begin
                peerSavedMemReqCnt.incr(1);
            end
`endif
        end
        else begin
            // no one has requested before, req memory
            memReqQ.enq(TlbMemReq {
                addr: pteAddr,
                id: idx
            });
            pendWait_transCacheResp[idx] <= WaitMem;
        end
        if(verbose) begin
            $display("L2TLB start page walk: ", fshow(cRq), "; ",
                     fshow(vm_info), "; ", fshow(resp), "; ",
                     fshow(level), "; ", fshow(pteAddr));
        end
`ifdef PERF_COUNT
        // perf: saved page walks
        if(doStats) begin
            Data saved = zeroExtend(maxPageWalkLevel - level);
            if(cRq.child == I) begin
                instSavedPageWalks.incr(saved);
            end
            else begin
                dataSavedPageWalks.incr(saved);
            end
        end
`endif
    endrule

    // page walk is preempted by tlb resp rule and trans cache resp rule, i.e.,
    // don't fire when tlb resp or trans cache resp are available
    rule doPageWalk(respLdQ.notEmpty && !tlbReqQ.notEmpty && !transCacheReqQ.notEmpty);
        doAssert(!flushing, "cannot have pending req when flushing");

        // get the resp data from memory (LLC); this resp is for the initiating
        // req and other req that wait on this one, so don't deq right away
        PTESv39 pte = unpack(respLdQ.first.data);
        L2TlbReqIdx idx = fromMaybe(respLdQ.first.id, respForOtherReq);
        L2TlbRqFromC cRq = pendReq[idx];

        // find another req waiting for this resp to process in next cycle
        function Bool waitForResp(L2TlbReqIdx i);
            return pendWait_pageWalk[i] == WaitPeer (respLdQ.first.id) && i != idx;
        endfunction
        Vector#(L2TlbReqNum, L2TlbReqIdx) idxVec = genWith(fromInteger);
        if(find(waitForResp, idxVec) matches tagged Valid .i) begin
            // still have req waiting for this resp, keep processing
            respForOtherReq <= Valid (i);
            doAssert(pendValid_pageWalk[i], "waiting entry must be valid");
        end
        else begin
            // all req done, deq the mem resp
            respForOtherReq <= Invalid;
            respLdQ.deq;
        end

        // handle page fault
        function Action pageFault(String reason);
        action
            // resp with invalid entry
            rsToCQ.enq(L2TlbRsToC {
                child: cRq.child,
                entry: Invalid
            });
            // req is done
            pendValid_pageWalk[idx] <= False;
            pendWait_pageWalk[idx] <= None;
`ifdef PERF_COUNT
            // incr miss latency
            incrMissLat(cRq.child, idx);
`endif
        endaction
        endfunction

        // get correct VM info
        VMInfo vm_info = cRq.child == I ? vm_info_I : vm_info_D;

        // assume we are continuing page walk, update level and pte addr (it
        // causes not harm even if we stop page walk)
        PageWalkLevel walkLevel = pendWalkLevel[idx];
        PageWalkLevel newWalkLevel = walkLevel - 1;
        Addr newPTBase = getPTBaseAddr(pte.ppn);
        Addr newPTEAddr = getPTEAddr(newPTBase, cRq.vpn, newWalkLevel);
        pendWalkLevel[idx] <= newWalkLevel;
        pendWalkAddr[idx] <= newPTEAddr;

        // reach leaf PTE
        Bool leafPTE = isLeafPTE(pte.pteType);

`ifdef SECURITY
        // base/bound/mask to enforce that page walk does not cross enclave
        // boundary
        Addr parbase = pendEnclave[idx] ? vm_info.sanctum_eparbase : vm_info.sanctum_parbase;
        Addr parmask = pendEnclave[idx] ? vm_info.sanctum_eparmask : vm_info.sanctum_parmask;
        Addr mrbm    = pendEnclave[idx] ? vm_info.sanctum_emrbm    : vm_info.sanctum_mrbm;
`endif

        if(verbose) begin
            $display("L2TLB page walk: ", fshow(vm_info), " ; ",
                     fshow(idx), " ; ", fshow(cRq), " ; ",
                     fshow(walkLevel), " ; ", fshow(pte));
        end

        if(!vm_info.sv39) begin
            // no longer in sv39 mode -> page fault
            pageFault("Not in sv39");
        end
        else if(!pte.valid) begin
            // invalid pte -> fault
            pageFault("invalid page");
        end
`ifdef SECURITY
        // TODO: raise page fault or access fault?
        else if((newPTBase & parmask) == parbase) begin
            // access falls into protected address range (that I don't have permission) -> fault
            pageFault("SANCTUM protected address range");
        end
        else if((getAddrRegions(newPTBase, leafPTE, walkLevel) & mrbm) == 0) begin
            // access falls outiside regions belong to me -> fault
            pageFault("SANCTUM protected address range");
        end
`endif
        else begin
            // page is valid, check leaf or not
            if(!leafPTE) begin
                // non-leaf page
                if(walkLevel == 0) begin
                    // page walk end with non-leaf page -> fault
                    pageFault("non-leaf page at end");
                end
                else begin
                    // continue page walk, check if other req is doing the same
                    // walk
                    if(otherReqSamePTE(idx, newPTEAddr, readVReg(pendWait_pageWalk)) matches tagged Valid .i) begin
                        pendWait_pageWalk[idx] <= WaitPeer (i);
`ifdef PERF_COUNT
                        if(doStats) begin
                            peerSavedMemReqCnt.incr(1);
                        end
`endif
                    end
                    else begin
                        memReqQ.enq(TlbMemReq {
                            addr: newPTEAddr,
                            id: idx
                        });
                        pendWait_pageWalk[idx] <= WaitMem;
                    end
                    // add to translation cache
                    transCache.addEntry(cRq.vpn, walkLevel, pte.ppn, vm_info.asid);
                end
            end
            else begin
                // leaf page, get new entry
                Vpn masked_vpn = getMaskedVpn(cRq.vpn, walkLevel);
                Ppn masked_ppn = getMaskedPpn(pte.ppn, walkLevel);
                let entry = TlbEntry {
                    vpn:     masked_vpn,
                    ppn:     masked_ppn,
                    pteType: pte.pteType,
                    level:   walkLevel,
                    asid:    vm_info.asid
                };
                // resp child
                rsToCQ.enq(L2TlbRsToC {
                    child: cRq.child,
                    entry: Valid (entry)
                });
                // update TLB array
                if(entry.level > 0) begin
                    // add to mega/giga page tlb
                    tlbMG.addEntry(entry);
`ifdef PERF_COUNT
                    if(doStats) begin
                        if(cRq.child == I) begin
                            instHugePageMissCnt.incr(1);
                        end
                        else begin
                            dataHugePageMissCnt.incr(1);
                        end
                    end
`endif
                end
                else begin
                    // 4KB page, add to 4KB TLB
                    tlb4KB.req(Refill (entry));
                end
                // req is done
                pendValid_pageWalk[idx] <= False;
                pendWait_pageWalk[idx] <= None;
`ifdef PERF_COUNT
                // incr miss latency
                incrMissLat(cRq.child, idx);
`endif
            end
        end
`ifdef PERF_COUNT
        // perf: page walk done once
        if(doStats) begin
            if(cRq.child == I) begin
                instPageWalks.incr(1);
            end
            else begin
                dataPageWalks.incr(1);
            end
        end
`endif
    endrule

    method Action updateVMInfo(VMInfo vmI, VMInfo vmD); //if(!isValid(pendReq));
        vm_info_I <= vmI;
        vm_info_D <= vmD;
    endmethod

    interface L2TlbToChildren toChildren;
        interface Put rqFromC = toPut(rqFromCQ);
        interface rsToC = toFifoDeq(rsToCQ);

        interface Put iTlbReqFlush;
            method Action put(void x) if(!iFlushReq);
                iFlushReq <= True;
            endmethod
        endinterface
        interface Put dTlbReqFlush;
            method Action put(void x) if(!dFlushReq);
                dFlushReq <= True;
            endmethod
        endinterface
        interface Get flushDone = toGet(flushDoneQ);
    endinterface

    interface TlbMemClient toMem;
        interface FifoDeq memReq = toFifoDeq(memReqQ);
        interface FifoEnq respLd = toFifoEnq(respLdQ);
    endinterface

    interface Perf perf;
        method Action setStatus(Bool stats);
`ifdef PERF_COUNT
            doStats <= stats;
`else
            noAction;
`endif
        endmethod

        method Action req(L2TlbPerfType r);
            perfReqQ.enq(r);
        endmethod

        method ActionValue#(PerfResp#(L2TlbPerfType)) resp;
`ifdef PERF_COUNT
            perfRespQ.deq;
            return perfRespQ.first;
`else
            perfReqQ.deq;
            return PerfResp {
                pType: perfReqQ.first,
                data: 0
            };
`endif
        endmethod

        method Bool respValid;
`ifdef PERF_COUNT
            return perfRespQ.notEmpty;
`else
            return perfReqQ.notEmpty;
`endif
        endmethod
    endinterface
endmodule

// Copyright (c) 2017 Massachusetts Institute of Technology
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

`include "ProcConfig.bsv"
import Vector::*;
import DefaultValue::*;
import ClientServer::*;
import GetPut::*;
import Types::*;
import ProcTypes::*;
import TlbTypes::*;
import Performance::*;
import FullAssocTlb::*;
import ConfigReg::*;
import Ehr::*;
import Fifo::*;
import Cntrs::*;
import SafeCounter::*;
import CacheUtils::*;
import SetAssocTlb::*;
import L2SetAssocTlb::*;
import TranslationCache::*;
import LatencyTimer::*;

// for SV39 only

// curretly L2 TLB is just a blocking page walker

// interface with memory (LLC)
typedef L2TlbReqIdx TlbMemReqId;

typedef struct {
    // this is always a load req
    Addr addr;
    TlbMemReqId id;
} TlbMemReq deriving(Bits, Eq, FShow);

typedef struct {
    Data data;
    TlbMemReqId id;
} TlbLdResp deriving(Bits, Eq, FShow);

interface TlbMemClient;
    interface FifoDeq#(TlbMemReq) memReq;
    interface FifoEnq#(TlbLdResp) respLd;
endinterface

// interface with children (I/D TLB)
typedef union tagged {
    void I;
    DTlbReqIdx D;
} TlbChild deriving(Bits, Eq, FShow);
typedef struct {
    TlbChild child;
    Vpn vpn;
} L2TlbRqFromC deriving(Bits, Eq, FShow);

typedef struct {
    TlbChild child;
    Maybe#(TlbEntry) entry;
} L2TlbRsToC deriving(Bits, Eq, FShow);

interface L2TlbToChildren;
    interface Put#(L2TlbRqFromC) rqFromC;
    interface FifoDeq#(L2TlbRsToC) rsToC;
    // flush with I/D TLB
    interface Put#(void) iTlbReqFlush;
    interface Put#(void) dTlbReqFlush;
    interface Get#(void) flushDone;
endinterface

interface L2Tlb;
    // keep update with changes to CSRs
    method Action updateVMInfo(VMInfo vmI, VMInfo vmD);

    // ifc with ITLb & DTLB
    interface L2TlbToChildren toChildren;

    // ifc with memory (LLC)
    interface TlbMemClient toMem;

    // performace
    interface Perf#(L2TlbPerfType) perf;
endinterface

typedef FullAssocTlb#(`L2_TLB_HUGE_SIZE) L2FullAssocTlb;
module mkL2FullAssocTlb(L2FullAssocTlb);
    let m <- mkFullAssocTlb(True); // randomness in replacement
    return m;
endmodule

// a pending tlb req may be in following states
typedef union tagged {
    void None;
    void WaitMem; // wait for page walk resp from memory
    L2TlbReqIdx WaitPeer; // wait for page walk resp from peer req
} L2TlbWait deriving(Bits, Eq, FShow);

// TODO we should raise load access fault if the PTE address is not a DRAM
// address. (trap value is still the virtual address being translated).

(* synthesize *)
module mkL2Tlb(L2Tlb::L2Tlb);
    Bool verbose = True;

    // set associative TLB for 4KB pages
    L2SetAssocTlb tlb4KB <- mkL2SetAssocTlb;
    // fully associative TLB for mega and giga pages
    L2FullAssocTlb tlbMG <- mkL2FullAssocTlb;
    // FIFO in parallel with TLB
    Fifo#(1, L2TlbReqIdx) tlbReqQ <- mkPipelineFifo;

    // MMU translation cache
    TranslationCache transCache <- mkSplitTransCache;
    // FIFO in parallel with MMU cache
    Fifo#(1, L2TlbReqIdx) transCacheReqQ <- mkPipelineFifo;

    // flush
    Reg#(Bool) iFlushReq <- mkReg(False);
    Reg#(Bool) dFlushReq <- mkReg(False);
    Reg#(Bool) waitFlushDone <- mkReg(False);
    Bool flushing = iFlushReq && dFlushReq;
    Fifo#(1, void) flushDoneQ <- mkCFFifo;

    // req/resp with I/D TLBs
    Fifo#(1, L2TlbRqFromC) rqFromCQ <- mkBypassFifo;
    Fifo#(1, L2TlbRsToC) rsToCQ <- mkBypassFifo;

    // pending reqs
    // pendWait should be meaningful even when entry is invalid. pendWait =
    // WaitMem means this entry is waiting for page walk resp from memory;
    // pendWait = WaitPeer means this entry is waiting for a page walk resp
    // resp initiated by another req. Thus, pendWait must be None when entry is
    // invalid.
    Vector#(L2TlbReqNum, Ehr#(2, Bool)) pendValid <- replicateM(mkEhr(False));
    Vector#(L2TlbReqNum, Reg#(L2TlbRqFromC)) pendReq <- replicateM(mkRegU);
    Vector#(L2TlbReqNum, Ehr#(2, L2TlbWait)) pendWait <- replicateM(mkEhr(None));
    Vector#(L2TlbReqNum, Reg#(PageWalkLevel)) pendWalkLevel <- replicateM(mkRegU);
    Vector#(L2TlbReqNum, Reg#(Addr)) pendWalkAddr <- replicateM(mkRegU);
`ifdef SECURITY
    Vector#(L2TlbReqNum, Reg#(Bool)) pendEnclave <- replicateM(mkRegU);
`endif

    // rule ordering:
    // - trans cache resp < tlb resp < tlb req
    // - trans cache resp mutually exclusive with page walk (trans cache resp
    // has precedence). I don't want them to fire together because both rules
    // update walk level, and I don't want a bypass path.
    // - tlb resp mutually exclusive with page walk (tlb resp has precedence).
    // These two rules cannot fire together, because page walk may update trans
    // cache and tlb resp may request trans cache. tlb resp takes precednece
    // because page walk may update tlb and it needs to wait for tlb resp to
    // deq resp.
    // - tlb req mutually exclusive with page walk (page walk has precedence).
    // They cannot fire together, because page walk may udpate tlb and tlb req
    // may request tlb.

    let pendValid_transCacheResp = getVEhrPort(pendValid, 0); // assert
    let pendValid_tlbResp = getVEhrPort(pendValid, 0);
    let pendValid_pageWalk = getVEhrPort(pendValid, 0);
    let pendValid_tlbReq = getVEhrPort(pendValid, 1);

    let pendWait_transCacheResp = getVEhrPort(pendWait, 0);
    let pendWait_pageWalk = getVEhrPort(pendWait, 0);
    let pendWait_tlbResp = getVEhrPort(pendWait, 1); // perf
    let pendWait_tlbReq = getVEhrPort(pendWait, 1); // assert

    // current processor VM information
    Reg#(VMInfo) vm_info_I <- mkReg(defaultValue);
    Reg#(VMInfo) vm_info_D <- mkReg(defaultValue);

    // Memory Queues for page table walks
    Fifo#(2, TlbMemReq) memReqQ <- mkCFFifo;
    Fifo#(2, TlbLdResp) respLdQ <- mkCFFifo;
    // When a mem resp comes, we first process the initiating req, then process
    // other reqs that in WaitPeer.
    Reg#(Maybe#(L2TlbReqIdx)) respForOtherReq <- mkReg(Invalid);

    // FIFO for perf req
    Fifo#(1, L2TlbPerfType) perfReqQ <- mkCFFifo;
`ifdef PERF_COUNT
    Fifo#(1, PerfResp#(L2TlbPerfType)) perfRespQ <- mkCFFifo;
    Reg#(Bool) doStats <- mkConfigReg(False);
    Count#(Data) instMissCnt <- mkCount(0);
    Count#(Data) instMissLat <- mkCount(0);
    Count#(Data) instPageWalks <- mkCount(0);
    Count#(Data) instSavedPageWalks <- mkCount(0);
    Count#(Data) instHugePageHitCnt <- mkCount(0);
    Count#(Data) instHugePageMissCnt <- mkCount(0);
    Count#(Data) dataMissCnt <- mkCount(0);
    Count#(Data) dataMissLat <- mkCount(0);
    Count#(Data) dataPageWalks <- mkCount(0);
    Count#(Data) dataSavedPageWalks <- mkCount(0);
    Count#(Data) dataHugePageHitCnt <- mkCount(0);
    Count#(Data) dataHugePageMissCnt <- mkCount(0);
    Count#(Data) hitUnderMissCnt <- mkCount(0);
    Count#(Data) allMissCycles <- mkCount(0);
    Count#(Data) peerSavedMemReqCnt <- mkCount(0);

    LatencyTimer#(L2TlbReqNum, 12) latTimer <- mkLatencyTimer; // max latency: 4K cycles

    function Action incrMissLat(TlbChild child, L2TlbReqIdx idx);
    action
        let lat <- latTimer.done(idx);
        if(doStats) begin
            if(child == I) begin
                instMissLat.incr(zeroExtend(lat));
            end
            else begin
                dataMissLat.incr(zeroExtend(lat));
            end
        end
    endaction
    endfunction

    rule incrAllMissCycles(doStats);
        function Bool isMiss(L2TlbWait x) = x != None;
        when(all(isMiss, readVEhr(0, pendWait)), allMissCycles.incr(1));
    endrule

    rule doPerf;
        let t <- toGet(perfReqQ).get;
        Data d = (case(t)
            L2TlbInstMissCnt: (instMissCnt);
            L2TlbInstMissLat: (instMissLat);
            L2TlbInstPageWalks: (instPageWalks);
            L2TlbInstSavedPageWalks: (instSavedPageWalks);
            L2TlbInstHugePageHits: (instHugePageHitCnt);
            L2TlbInstHugePageMisses: (instHugePageMissCnt);
            L2TlbDataMissCnt: (dataMissCnt);
            L2TlbDataMissLat: (dataMissLat);
            L2TlbDataPageWalks: (dataPageWalks);
            L2TlbDataSavedPageWalks: (dataSavedPageWalks);
            L2TlbDataHugePageHits: (dataHugePageHitCnt);
            L2TlbDataHugePageMisses: (dataHugePageMissCnt);
            L2TlbHitUnderMissCnt: (hitUnderMissCnt);
            L2TlbAllMissCycles: (allMissCycles);
            L2TlbPeerSavedMemReqs: (peerSavedMemReqCnt);
            default: (0);
        endcase);
        perfRespQ.enq(PerfResp {
            pType: t,
            data: d
        });
    endrule
`endif

    // when flushing is true, since both I and D TLBs have finished flush and
    // is waiting for L2 to flush, all I/D TLB req must have been responded.
    // Thus, there cannot be any req in pendReq or rqFromCQ.
    rule doStartFlush(flushing && !waitFlushDone);
        waitFlushDone <= True;
        tlb4KB.flush;
        tlbMG.flush;
        transCache.flush;
        // check no req
        doAssert(!rqFromCQ.notEmpty, "cannot have new req");
        doAssert(readVEhr(0, pendValid) == replicate(False), "cannot have pending req");
    endrule

    rule doWaitFlush(flushing && waitFlushDone && tlb4KB.flush_done && transCache.flush_done);
        waitFlushDone <= False;
        flushDoneQ.enq(?);
        iFlushReq <= False;
        dFlushReq <= False;
    endrule

    // tlb req rule is preempted by page walk rule, i.e., don't fire when page
    // walk resp is avaiable
    rule doTlbReq(!flushing && !respLdQ.notEmpty);
        // find a slot for the new req
        L2TlbReqIdx idx = ?;
        if(findIndex( \== (False) , readVReg(pendValid_tlbReq) ) matches tagged Valid .i) begin
            idx = pack(i);
        end
        else begin
            when(False, noAction);
        end
        // get new req
        rqFromCQ.deq;
        let r = rqFromCQ.first;
        // req tlb array
        VMInfo vm_info = r.child == I ? vm_info_I : vm_info_D;
        tlb4KB.req(Translate (SetAssocTlbTranslateReq {
            vpn: r.vpn,
            asid: vm_info.asid
        }));
        tlbReqQ.enq(idx);
        // record req
        pendValid_tlbReq[idx] <= True;
        pendReq[idx] <= r;
        doAssert(!pendValid_tlbReq[idx], "entry must be invalid");
        doAssert(pendWait_tlbReq[idx] == None, "cannot be waiting");
        if(verbose) $display("L2TLB new req: ", fshow(r), "; ", fshow(idx));
    endrule

    // process resp from 4KB TLB and mega-giga TLB
    rule doTlbResp(tlbReqQ.notEmpty);
        doAssert(!flushing, "cannot have pending req when flushing");

        // get req in tlb
        tlbReqQ.deq;
        L2TlbReqIdx idx = tlbReqQ.first;
        L2TlbRqFromC cRq = pendReq[idx];
        doAssert(pendWait_tlbResp[idx] == None, "cannot be waiting");

        // get correct VM info
        VMInfo vm_info = cRq.child == I ? vm_info_I : vm_info_D;
        doAssert(vm_info.sv39, "must be in sv39 mode");

        // get resp from 4KB TLB and mega-giga TLB
        let resp4KB = tlb4KB.resp;
        let respMG = tlbMG.translate(cRq.vpn, vm_info.asid);

        if(verbose) begin
            $display("L2TLB resp: ", fshow(vm_info), " ; ", fshow(cRq), " ; ",
                     fshow(resp4KB), " ; ", fshow(respMG));
        end

        // when page hit, resp to child (4KB array is not dequeued)
        function Action pageHit(TlbEntry entry);
        action
            // resp to child
            rsToCQ.enq(L2TlbRsToC {
                child: cRq.child,
                entry: Valid (entry)
            });
            // req is done
            pendValid_tlbResp[idx] <= False;
`ifdef PERF_COUNT
            // perf: hit under miss
            function Bool otherMiss(L2TlbReqIdx i);
                return pendValid_tlbResp[i] && pendWait_tlbResp[i] != None && i != idx;
            endfunction
            Vector#(L2TlbReqNum, L2TlbReqIdx) idxVec = genWith(fromInteger);
            if(any(otherMiss, idxVec)) begin
                hitUnderMissCnt.incr(1);
            end
`endif
        endaction
        endfunction

        if(!vm_info.sv39) begin
            // not in sv39 -> page fault
            // resp with invalid entry
            rsToCQ.enq(L2TlbRsToC {
                child: cRq.child,
                entry: Invalid
            });
            // deq 4KB TLB array
            tlb4KB.deqResp(Invalid);
            // req is done
            pendValid_tlbResp[idx] <= False;
        end
        else if(respMG.hit) begin
            // hit on a mega or giga page
            let entry = respMG.entry;
            doAssert(entry.level > 0 && entry.level <= maxPageWalkLevel,
                     "mega or giga page");
            pageHit(entry);
            tlb4KB.deqResp(Invalid); // just deq 4KB array
            tlbMG.updateRepByHit(respMG.index); // update replacement in MG array
`ifdef PERF_COUNT
            if(doStats) begin
                if(cRq.child == I) begin
                    instHugePageHitCnt.incr(1);
                end
                else begin
                    dataHugePageHitCnt.incr(1);
                end
            end
`endif
        end
        else if(resp4KB.hit) begin
            // hit on 4KB page
            let entry = resp4KB.entry;
            doAssert(entry.level == 0, "must be 4KB page");
            pageHit(entry);
            // update 4KB array replacement, no need to touch MG array
            tlb4KB.deqResp(Valid (resp4KB.way));
        end
        else begin
            // miss, deq resp
            tlb4KB.deqResp(Invalid);
            // check translation cache
            transCache.req(cRq.vpn, vm_info.asid);
            transCacheReqQ.enq(idx);
            // perf: TLB miss
`ifdef PERF_COUNT
            latTimer.start(idx);
            if(doStats) begin
                if(cRq.child == I) begin
                    instMissCnt.incr(1);
                end
                else begin
                    dataMissCnt.incr(1);
                end
            end
`endif
        end
    endrule

    // Find other req doing the same page walk. Although it should be ok to
    // just compare the VPNs, for safety, we compare the PTE addrs.
    function Maybe#(L2TlbReqIdx) otherReqSamePTE(
        L2TlbReqIdx idx, Addr pteAddr,
        Vector#(L2TlbReqNum, L2TlbWait) waitVec
    );
        function Bool samePTE(L2TlbReqIdx i);
            return i != idx && waitVec[i] == WaitMem && pendWalkAddr[i] == pteAddr;
        endfunction
        Vector#(L2TlbReqNum, L2TlbReqIdx) idxVec = genWith(fromInteger);
        return find(samePTE, idxVec);
    endfunction

    rule doTranslationCacheResp(transCacheReqQ.notEmpty);
        // get req in trans cache
        transCacheReqQ.deq;
        L2TlbReqIdx idx = transCacheReqQ.first;
        L2TlbRqFromC cRq = pendReq[idx];
        // get trans cache resp
        transCache.deqResp;
        let resp = transCache.resp;
        // start page walk based on the translation cache resp. Note that if
        // the startLevel in resp is max, then we should use the base ppn in vm
        // info.
        // XXX We believe the PPN stored in translation cache has been
        // sanitized before, so we don't need to check again if the PPN is an
        // enclave PPN or not. (This should follow the same reasoning that TLB
        // hits do not need checks.)
        VMInfo vm_info = cRq.child == I ? vm_info_I : vm_info_D;
        PageWalkLevel level = resp.startLevel;
        Ppn rootPPN = vm_info.basePPN; // root of page table
`ifdef SECURITY
        Bool isEnclave = (zeroExtend(cRq.vpn) & vm_info.sanctum_evmask) == vm_info.sanctum_evbase;
        if(isEnclave) begin
            rootPPN = vm_info.sanctum_ebasePPN; // enclave has its own root of page table
        end
`endif
        Addr baseAddr = getPTBaseAddr(level < maxPageWalkLevel ? resp.ppn : rootPPN);
        Addr pteAddr = getPTEAddr(baseAddr, cRq.vpn, level);
        // record page walk info
        pendWalkLevel[idx] <= level;
        pendWalkAddr[idx] <= pteAddr;
`ifdef SECURITY
        pendEnclave[idx] <= isEnclave;
`endif
        doAssert(pendWait_transCacheResp[idx] == None, "cannot be waiting");
        // don't req memory if someone else is also doing the same page walk.
        if(otherReqSamePTE(idx, pteAddr, readVReg(pendWait_transCacheResp)) matches tagged Valid .i) begin
            // peer entry has already requested, so don't send
            // duplicate req
            pendWait_transCacheResp[idx] <= WaitPeer (i);
            doAssert(pendValid_transCacheResp[i], "peer must be valid");
`ifdef PERF_COUNT
            if(doStats) begin
                peerSavedMemReqCnt.incr(1);
            end
`endif
        end
        else begin
            // no one has requested before, req memory
            memReqQ.enq(TlbMemReq {
                addr: pteAddr,
                id: idx
            });
            pendWait_transCacheResp[idx] <= WaitMem;
        end
        if(verbose) begin
            $display("L2TLB start page walk: ", fshow(cRq), "; ",
                     fshow(vm_info), "; ", fshow(resp), "; ",
                     fshow(level), "; ", fshow(pteAddr));
        end
`ifdef PERF_COUNT
        // perf: saved page walks
        if(doStats) begin
            Data saved = zeroExtend(maxPageWalkLevel - level);
            if(cRq.child == I) begin
                instSavedPageWalks.incr(saved);
            end
            else begin
                dataSavedPageWalks.incr(saved);
            end
        end
`endif
    endrule

    // page walk is preempted by tlb resp rule and trans cache resp rule, i.e.,
    // don't fire when tlb resp or trans cache resp are available
    rule doPageWalk(respLdQ.notEmpty && !tlbReqQ.notEmpty && !transCacheReqQ.notEmpty);
        doAssert(!flushing, "cannot have pending req when flushing");

        // get the resp data from memory (LLC); this resp is for the initiating
        // req and other req that wait on this one, so don't deq right away
        PTESv39 pte = unpack(respLdQ.first.data);
        L2TlbReqIdx idx = fromMaybe(respLdQ.first.id, respForOtherReq);
        L2TlbRqFromC cRq = pendReq[idx];

        // find another req waiting for this resp to process in next cycle
        function Bool waitForResp(L2TlbReqIdx i);
            return pendWait_pageWalk[i] == WaitPeer (respLdQ.first.id) && i != idx;
        endfunction
        Vector#(L2TlbReqNum, L2TlbReqIdx) idxVec = genWith(fromInteger);
        if(find(waitForResp, idxVec) matches tagged Valid .i) begin
            // still have req waiting for this resp, keep processing
            respForOtherReq <= Valid (i);
            doAssert(pendValid_pageWalk[i], "waiting entry must be valid");
        end
        else begin
            // all req done, deq the mem resp
            respForOtherReq <= Invalid;
            respLdQ.deq;
        end

        // handle page fault
        function Action pageFault(String reason);
        action
            // resp with invalid entry
            rsToCQ.enq(L2TlbRsToC {
                child: cRq.child,
                entry: Invalid
            });
            // req is done
            pendValid_pageWalk[idx] <= False;
            pendWait_pageWalk[idx] <= None;
`ifdef PERF_COUNT
            // incr miss latency
            incrMissLat(cRq.child, idx);
`endif
        endaction
        endfunction

        // get correct VM info
        VMInfo vm_info = cRq.child == I ? vm_info_I : vm_info_D;

        // assume we are continuing page walk, update level and pte addr (it
        // causes not harm even if we stop page walk)
        PageWalkLevel walkLevel = pendWalkLevel[idx];
        PageWalkLevel newWalkLevel = walkLevel - 1;
        Addr newPTBase = getPTBaseAddr(pte.ppn);
        Addr newPTEAddr = getPTEAddr(newPTBase, cRq.vpn, newWalkLevel);
        pendWalkLevel[idx] <= newWalkLevel;
        pendWalkAddr[idx] <= newPTEAddr;

        // reach leaf PTE
        Bool leafPTE = isLeafPTE(pte.pteType);

`ifdef SECURITY
        // base/bound/mask to enforce that page walk does not cross enclave
        // boundary
        Addr parbase = pendEnclave[idx] ? vm_info.sanctum_eparbase : vm_info.sanctum_parbase;
        Addr parmask = pendEnclave[idx] ? vm_info.sanctum_eparmask : vm_info.sanctum_parmask;
        Addr mrbm    = pendEnclave[idx] ? vm_info.sanctum_emrbm    : vm_info.sanctum_mrbm;
`endif

        if(verbose) begin
            $display("L2TLB page walk: ", fshow(vm_info), " ; ",
                     fshow(idx), " ; ", fshow(cRq), " ; ",
                     fshow(walkLevel), " ; ", fshow(pte));
        end

        if(!vm_info.sv39) begin
            // no longer in sv39 mode -> page fault
            pageFault("Not in sv39");
        end
        else if(!pte.valid) begin
            // invalid pte -> fault
            pageFault("invalid page");
        end
`ifdef SECURITY
        // TODO: raise page fault or access fault?
        else if((newPTBase & parmask) == parbase) begin
            // access falls into protected address range (that I don't have permission) -> fault
            pageFault("SANCTUM protected address range");
        end
        else if((getAddrRegions(newPTBase, leafPTE, walkLevel) & mrbm) == 0) begin
            // access falls outiside regions belong to me -> fault
            pageFault("SANCTUM protected address range");
        end
`endif
        else begin
            // page is valid, check leaf or not
            if(!leafPTE) begin
                // non-leaf page
                if(walkLevel == 0) begin
                    // page walk end with non-leaf page -> fault
                    pageFault("non-leaf page at end");
                end
                else begin
                    // continue page walk, check if other req is doing the same
                    // walk
                    if(otherReqSamePTE(idx, newPTEAddr, readVReg(pendWait_pageWalk)) matches tagged Valid .i) begin
                        pendWait_pageWalk[idx] <= WaitPeer (i);
`ifdef PERF_COUNT
                        if(doStats) begin
                            peerSavedMemReqCnt.incr(1);
                        end
`endif
                    end
                    else begin
                        memReqQ.enq(TlbMemReq {
                            addr: newPTEAddr,
                            id: idx
                        });
                        pendWait_pageWalk[idx] <= WaitMem;
                    end
                    // add to translation cache
                    transCache.addEntry(cRq.vpn, walkLevel, pte.ppn, vm_info.asid);
                end
            end
            else begin
                // leaf page, get new entry
                Vpn masked_vpn = getMaskedVpn(cRq.vpn, walkLevel);
                Ppn masked_ppn = getMaskedPpn(pte.ppn, walkLevel);
                let entry = TlbEntry {
                    vpn:     masked_vpn,
                    ppn:     masked_ppn,
                    pteType: pte.pteType,
                    level:   walkLevel,
                    asid:    vm_info.asid
                };
                // resp child
                rsToCQ.enq(L2TlbRsToC {
                    child: cRq.child,
                    entry: Valid (entry)
                });
                // update TLB array
                if(entry.level > 0) begin
                    // add to mega/giga page tlb
                    tlbMG.addEntry(entry);
`ifdef PERF_COUNT
                    if(doStats) begin
                        if(cRq.child == I) begin
                            instHugePageMissCnt.incr(1);
                        end
                        else begin
                            dataHugePageMissCnt.incr(1);
                        end
                    end
`endif
                end
                else begin
                    // 4KB page, add to 4KB TLB
                    tlb4KB.req(Refill (entry));
                end
                // req is done
                pendValid_pageWalk[idx] <= False;
                pendWait_pageWalk[idx] <= None;
`ifdef PERF_COUNT
                // incr miss latency
                incrMissLat(cRq.child, idx);
`endif
            end
        end
`ifdef PERF_COUNT
        // perf: page walk done once
        if(doStats) begin
            if(cRq.child == I) begin
                instPageWalks.incr(1);
            end
            else begin
                dataPageWalks.incr(1);
            end
        end
`endif
    endrule

    method Action updateVMInfo(VMInfo vmI, VMInfo vmD); //if(!isValid(pendReq));
        vm_info_I <= vmI;
        vm_info_D <= vmD;
    endmethod

    interface L2TlbToChildren toChildren;
        interface Put rqFromC = toPut(rqFromCQ);
        interface rsToC = toFifoDeq(rsToCQ);

        interface Put iTlbReqFlush;
            method Action put(void x) if(!iFlushReq);
                iFlushReq <= True;
            endmethod
        endinterface
        interface Put dTlbReqFlush;
            method Action put(void x) if(!dFlushReq);
                dFlushReq <= True;
            endmethod
        endinterface
        interface Get flushDone = toGet(flushDoneQ);
    endinterface

    interface TlbMemClient toMem;
        interface FifoDeq memReq = toFifoDeq(memReqQ);
        interface FifoEnq respLd = toFifoEnq(respLdQ);
    endinterface

    interface Perf perf;
        method Action setStatus(Bool stats);
`ifdef PERF_COUNT
            doStats <= stats;
`else
            noAction;
`endif
        endmethod

        method Action req(L2TlbPerfType r);
            perfReqQ.enq(r);
        endmethod

        method ActionValue#(PerfResp#(L2TlbPerfType)) resp;
`ifdef PERF_COUNT
            perfRespQ.deq;
            return perfRespQ.first;
`else
            perfReqQ.deq;
            return PerfResp {
                pType: perfReqQ.first,
                data: 0
            };
`endif
        endmethod

        method Bool respValid;
`ifdef PERF_COUNT
            return perfRespQ.notEmpty;
`else
            return perfReqQ.notEmpty;
`endif
        endmethod
    endinterface
endmodule


split a step to meet timing on nfsume

split a step to meet timing on nfsume

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

/*
 * Implementation of:
 *    MP algorithm on pages 7-11 from "Pattern Matching Algorithms" by
 *       Alberto Apostolico, Zvi Galil, 1997
 */

`include "ConnectalProjectConfig.bsv"
import FIFO::*;
import FIFOF::*;
import SpecialFIFOs::*;
import Vector::*;
import BRAM::*;
import Gearbox::*;
import Connectable::*;
import ConfigReg::*;
import StmtFSM::*;
import Probe::*;

import MemUtils::*;
import MemTypes::*;
import Dma2BRAM::*;
import Pipe::*;
import EHR::*;

interface MPEngine#(numeric type haystackBusWidth, numeric type configBusWidth);
   interface PipeIn#(Triplet#(Bit#(32))) setsearch;
   interface PipeOut#(Int#(32)) locdone;
endinterface

interface MPStreamEngine#(numeric type haystackBusWidth, numeric type configBusWidth);
   interface PipeIn#(MemDataF#(configBusWidth)) needle;
   interface PipeIn#(MemDataF#(configBusWidth)) mpNext;
   interface PipeIn#(MemDataF#(haystackBusWidth)) haystack;
   interface PipeOut#(Int#(32)) locdone;
   method Action clear();
   method Action start(Bit#(32) needleLen);
endinterface

typedef Bit#(8) Char;
typedef Bit#(64) DWord;
typedef Bit#(32) Word;

typedef 1024 MaxNeedleLen;
typedef TLog#(MaxNeedleLen) NeedleIdxWidth;
typedef Bit#(NeedleIdxWidth) NeedleIdx;

typedef enum {Config_needle, Config_mpNext, Initialized, Search} Stage deriving (Eq, Bits);

module mkMPEngine#(MemReadEngineServer#(haystackBusWidth) haystackReader,
		   MemReadEngineServer#(configBusWidth) configReader) (MPEngine#(haystackBusWidth,configBusWidth))

   provisos(Add#(a__, 8, haystackBusWidth),
	    Div#(haystackBusWidth,8,haystackBusBytes),
	    Mul#(haystackBusBytes,8,haystackBusWidth),
	    Add#(1, b__, haystackBusBytes),
	    Add#(c__, 32, haystackBusWidth),
	    Add#(1, d__, TDiv#(haystackBusWidth, 32)),
	    Mul#(TDiv#(haystackBusWidth, 32), 32, haystackBusWidth),
	    Add#(e__, TLog#(haystackBusBytes), 32),
	    Add#(f__, TLog#(TDiv#(haystackBusWidth, 32)), 32)
	    ,Mul#(TDiv#(configBusWidth, 8), 8, configBusWidth)
	    ,Add#(1, g__, TDiv#(configBusWidth, 8))
	    ,Add#(h__, TLog#(TDiv#(configBusWidth, 8)), 32)
	    ,Add#(i__, TLog#(TDiv#(configBusWidth, 32)), 32)
	    ,Add#(1, j__, TDiv#(configBusWidth, 32))
	    ,Mul#(TDiv#(configBusWidth, 32), 32, configBusWidth)
	    );

   FIFOF#(Int#(32)) locf <- mkFIFOF;
   FIFO#(Bool) conff <- mkSizedFIFO(1);

   let verbose = True;
   let debug = False;

   Clock clk <- exposeCurrentClock;
   Reset rst <- exposeCurrentReset;
   BRAM2Port#(NeedleIdx, Char) needle  <- mkBRAM2Server(defaultValue);
   BRAM2Port#(NeedleIdx, Bit#(32)) mpNext <- mkBRAM2Server(defaultValue);
   Gearbox#(haystackBusBytes,1,Char) haystack <- mkNto1Gearbox(clk,rst,clk,rst);

   Reg#(Bit#(32)) cycleCnt <- mkReg(0);
   Reg#(Bit#(32)) lastHD <- mkReg(0);

   Reg#(Stage)    stage <- mkReg(Config_needle);
   Reg#(Bit#(32)) needleLenReg <- mkReg(0);
   Reg#(Bit#(32)) haystackLenReg <- mkReg(0);
   Reg#(Bit#(32)) haystackBase <- mkReg(0);
   Reg#(Bit#(32)) jReg <- mkReg(0); // offset in haystack
   Reg#(Bit#(32)) iReg <- mkReg(0); // offset in needle
   Reg#(Bit#(2))  epochReg <- mkReg(0);

   BRAMWriter#(NeedleIdxWidth,configBusWidth) n2b <- mkBRAMWriter(0, needle.portB, configReader);
   BRAMWriter#(NeedleIdxWidth,configBusWidth) mp2b <- mkBRAMWriter(1, mpNext.portB, configReader);

   FIFOF#(Tuple2#(Bit#(2),Bit#(32))) efifo <- mkSizedFIFOF(2);
   FIFOF#(Triplet#(Bit#(32))) ssfifo <- mkFIFOF;
   FIFO#(void) doneFifo <- mkFIFO;

   rule countCycles;
      if (debug) $display("******************************************** %d", cycleCnt);
      cycleCnt <= cycleCnt+1;
   endrule

   rule haystackResp;
      if (debug) $display("mkMPEngine::haystackResp");
      let rv <- toGet(haystackReader.data).get;
      haystack.enq(unpack(rv.data));
      if (rv.last)
         conff.deq;
   endrule

   rule haystackDrain(stage != Search);
      if (debug) $display("mkMPEngine::haystackDrain");
      haystack.deq;
   endrule

   rule bramDrain(stage != Search);
      if (debug) $display("mkMPEngine::mpNextDrain");
      let x <- mpNext.portA.response.get;
      let y <- needle.portA.response.get;
      efifo.deq;
   endrule

`define OPTIMIZE_MISMATCH
`ifdef OPTIMIZE_MISMATCH
   rule matchNeedleReq(stage == Search);
      if (debug) $display("mkMPEngine::matchNeedleReq %d %d", epochReg, iReg);
      needle.portA.request.put(BRAMRequest{write:False, address: truncate(iReg-1), datain:?, responseOnWrite:?});
      mpNext.portA.request.put(BRAMRequest{write:False, address: truncate(iReg), datain:?, responseOnWrite:?});
      efifo.enq(tuple2(epochReg,iReg));
      iReg <= 1;
   endrule

   rule matchNeedleResp(stage == Search);
      let nv <- needle.portA.response.get;
      let mp <- mpNext.portA.response.get;
      let epoch = tpl_1(efifo.first);
      efifo.deq;
      if (debug) $display("mkMPEngine::matchNeedleResp %d %d", epochReg, epoch);
      if (epoch == epochReg) begin
	 Bool deq_haystack = False;
	 let n = haystackLenReg;
	 let m = needleLenReg;
	 let hv = haystack.first;
	 let i = tpl_2(efifo.first);
	 let j = jReg;
	 if (debug) $display("mkMPEngine::feck %d %d %d %d %x %x", n, m, i, j, hv[0], nv);
	 if (j > n) begin
	    // jReg points to the end of the haystack; we are done
	    stage <= Config_needle;
	    if (debug) $display("mkMPEngine::end of search %d", j);
	    locf.enq(-1);
	 end
	 else if (i==m+1) begin
	    // iReg points to the end of the needle; we have a match
	    if (debug) $display("mkMPEngine::string match %d", j);
	    locf.enq(unpack(haystackBase+j-i));
	 end
	 else if (nv != hv[0]) begin
	    // mismatch betwen head of haystack and head of needle; rewind iReg
	    if (debug) $display("mkMPEngine::char mismatch %d %d MP_Next[i]=%d", i, j, mp);
	    if (mp == 0) begin
	       iReg <= 1;
	       jReg <= j+1;
	       deq_haystack = True;
	    end
	    else begin
	       epochReg <= epochReg + 1;
	       iReg <= mp;
	    end
	 end
	 else begin
	    // match between head of needle and head of haystack; increment haystack
	    if (debug) $display("mkMPEngine::char match(%d) %d %d", (nv == hv[0]), i, j);
	    deq_haystack = True;
	    jReg <= j+1;
	    epochReg <= epochReg + 1;
	    iReg <= i+1;
	 end
	 if (deq_haystack) begin
	    haystack.deq;
	    lastHD <= cycleCnt;
	    if (debug) $display("mkMPEngine:: deq haystack(%d)", cycleCnt-lastHD);
	 end
      end
      else begin
	 if (debug) $display("mkMPEngine::discard");
	 noAction;
      end
   endrule
`else
   rule matchNeedleReq(stage == Search);
      if (debug) $display("mkMPEngine::matchNeedleReq %d %d", epochReg, iReg);
      needle.portA.request.put(BRAMRequest{write:False, address: truncate(iReg-1), datain:?, responseOnWrite:?});
      mpNext.portA.request.put(BRAMRequest{write:False, address: truncate(iReg), datain:?, responseOnWrite:?});
      efifo.enq(tuple2(epochReg,iReg));
      iReg <= iReg+1;
   endrule

   rule matchNeedleResp(stage == Search);
      let nv <- needle.portA.response.get;
      let mp <- mpNext.portA.response.get;
      let epoch = tpl_1(efifo.first);
      efifo.deq;
      if (debug) $display("mkMPEngine::matchNeedleResp %d %d", epochReg, epoch);
      if (epoch == epochReg) begin
	 Bool deq_haystack = False;
	 let n = haystackLenReg;
	 let m = needleLenReg;
	 let hv = haystack.first;
	 let i = tpl_2(efifo.first);
	 let j = jReg;
	 if (debug) $display("mkMPEngine::feck %d %d %d %d %x %x", n, m, i, j, hv[0], nv);
	 if (j > n) begin
	    // jReg points to the end of the haystack; we are done
	    stage <= Config_needle;
	    if (debug) $display("mkMPEngine::end of search %d", j);
	 end
	 else if (i==m+1) begin
	    // iReg points to the end of the needle; we have a match
	    if (debug) $display("mkMPEngine::string match %d", j);
	    locf.enq(unpack(haystackBase+j-i));
	    epochReg <= epochReg + 1;
	    iReg <= 1;
	 end
	 else if (nv != hv[0]) begin
	    // mismatch betwen head of haystack and head of needle; rewind iReg
	    if (debug) $display("mkMPEngine::char mismatch %d %d MP_Next[i]=%d", i, j, mp);
	    epochReg <= epochReg + 1;
	    if (mp == 0) begin
	       iReg <= 1;
	       jReg <= j+1;
	       deq_haystack = True;
	    end
	    else begin
	       iReg <= mp;
	    end
	 end
	 else begin
	    // match between head of needle and head of haystack; increment haystack
	    if (debug) $display("mkMPEngine::char match(%d) %d %d", (nv == hv[0]), i, j);
	    deq_haystack = True;
	    jReg <= j+1;
	 end
	 if (deq_haystack) begin
	    haystack.deq;
	    lastHD <= cycleCnt;
	    if (debug) $display("mkMPEngine:: deq haystack(%d)", cycleCnt-lastHD);
	 end
      end
      else begin
	 if (debug) $display("mkMPEngine::discard");
	 noAction;
      end
   endrule
`endif

   rule finish_setup_n2b;
      if (verbose) $display("mkMPEngine::finish_setup_n2b");
      let x <- n2b.finish;
      conff.deq;
      stage <= Config_mpNext;
   endrule

   rule finish_setup_mp2b;
      if (verbose) $display("mkMPEngine::finish_setup_mp2b");
      let y <- mp2b.finish;
      conff.deq;
      stage <= Initialized;
   endrule

   rule setup_needle (stage == Config_needle);
      conff.enq(True);
      match {.needle_sglId, .mpNext_sglId, .needle_len} = ssfifo.first;
      needleLenReg <= extend(needle_len);
      if (verbose) $display("mkMPEngine::setup_needle %d %d", needle_sglId, needle_len);
      n2b.start(needle_sglId, 0, 0, pack(truncate(needle_len)));
   endrule

   rule setup_mpNext (stage == Config_mpNext);
      conff.enq(True);
      match {.needle_sglId, .mpNext_sglId, .needle_len} = ssfifo.first;
      needleLenReg <= extend(needle_len);
      if (verbose) $display("mkMPEngine::setup_mpNext %d %d", mpNext_sglId, needle_len);
      mp2b.start(mpNext_sglId, 0, 0, pack(truncate(needle_len)));
      ssfifo.deq;
   endrule

   rule search (stage == Initialized && !efifo.notEmpty && !haystack.notEmpty);
      stage <= Search;
      conff.enq(True);
      match {.haystack_sglId, .haystack_len, .haystack_base} <- toGet(ssfifo).get;
      haystackLenReg <= extend(haystack_len);
      haystackBase <= extend(haystack_base);
      iReg <= 1;
      jReg <= 1;
      epochReg <= 0;
      Bit#(32) haystack_len_ds = haystack_len+fromInteger(valueOf(haystackBusBytes)-1);
      Bit#(TLog#(haystackBusBytes)) zeros = 0;
      Bit#(32) haystack_len_bytes = {zeros,haystack_len_ds[31:valueOf(TLog#(haystackBusBytes))]} * fromInteger(valueOf(haystackBusBytes));
      $display("haystack read offset=%d burstLen=%d", haystack_base, fromInteger(8*valueOf(haystackBusBytes)));
      haystackReader.request.put(MemengineCmd{sglId:haystack_sglId, base:extend(haystack_base), len:haystack_len_bytes, burstLen:fromInteger(8*valueOf(haystackBusBytes)), tag: 0});
      if (verbose) $display("mkMPEngine::search %d %d %d",  haystack_sglId, haystack_base, haystack_len_bytes);
   endrule

   interface PipeIn setsearch = toPipeIn(ssfifo);
   interface PipeOut locdone = toPipeOut(locf);

endmodule

module mkMPStreamEngine(MPStreamEngine#(haystackBusWidth,configBusWidth))

   provisos(Add#(a__, 8, haystackBusWidth),
	    Div#(haystackBusWidth,8,haystackBusBytes),
	    Mul#(haystackBusBytes,8,haystackBusWidth),
	    Add#(1, b__, haystackBusBytes),
	    Add#(c__, 32, haystackBusWidth),
	    Add#(1, d__, TDiv#(haystackBusWidth, 32)),
	    Mul#(TDiv#(haystackBusWidth, 32), 32, haystackBusWidth),
	    Add#(e__, TLog#(haystackBusBytes), 32),
	    Add#(f__, TLog#(TDiv#(haystackBusWidth, 32)), 32)
	    ,Mul#(TDiv#(configBusWidth, 8), 8, configBusWidth)
	    ,Add#(1, g__, TDiv#(configBusWidth, 8))
	    ,Add#(h__, TLog#(TDiv#(configBusWidth, 8)), 32)
	    ,Add#(i__, TLog#(TDiv#(configBusWidth, 32)), 32)
	    ,Add#(1, j__, TDiv#(configBusWidth, 32))
	    ,Mul#(TDiv#(configBusWidth, 32), 32, configBusWidth)
	    );

   FIFOF#(Int#(32)) locf <- mkFIFOF;

   let verbose = True;
   let debug = True;

   Clock clk <- exposeCurrentClock;
   Reset rst <- exposeCurrentReset;
   BRAM2Port#(NeedleIdx, Char) needleBram  <- mkBRAM2Server(defaultValue);
   BRAM2Port#(NeedleIdx, Bit#(32)) mpNextBram <- mkBRAM2Server(defaultValue);
   FIFOF#(MemDataF#(haystackBusWidth)) haystackFifo <- mkFIFOF();
   Gearbox#(haystackBusBytes,1,Char) haystackGb <- mkNto1Gearbox(clk,rst,clk,rst);

   Reg#(Bit#(32)) cycleCnt <- mkReg(0);
   Reg#(Bit#(32)) lastHD <- mkReg(0);

   Reg#(Stage)    stage <- mkReg(Initialized);
   Reg#(Bit#(32)) needleLenReg <- mkReg(0);
   Reg#(Bit#(32)) jReg <- mkReg(1); // offset in haystack
   Reg#(Bit#(32)) iReg <- mkReg(1); // offset in needle

   BRAMPipeIn#(NeedleIdxWidth,configBusWidth) n2b <- mkBRAMPipeIn(0, needleBram.portB);
   BRAMPipeIn#(NeedleIdxWidth,configBusWidth) mp2b <- mkBRAMPipeIn(1, mpNextBram.portB);

   FIFOF#(Tuple2#(Bit#(2),Bit#(32))) efifo <- mkSizedFIFOF(2);
   FIFO#(void) doneFifo <- mkFIFO;

   rule countCycles;
      //if (debug) $display("******************************************** %d", cycleCnt);
      cycleCnt <= cycleCnt+1;
   endrule

   rule haystackResp;
      let rv <- toGet(haystackFifo).get;
      if (debug) $display("mkMPEngine::haystackResp rv=%h", rv.data);
      haystackGb.enq(unpack(rv.data));
   endrule

   let x_i <- mkReg(0);
   let t_j <- mkReg(0);
   let m = needleLenReg;
   let i = iReg;
   let iReg_minus_1 <- mkReg(0);
   let matchFsm <- mkAutoFSM(seq
      while (stage != Search) seq
	 iReg <= 1;
      endseq
      action
	 iReg <= 1;
	 jReg <= 1;
	 let t = haystackGb.first[0]; haystackGb.deq();
	 t_j <= t;
         iReg_minus_1 <= iReg-1;
      endaction
      action
	 needleBram.portA.request.put(BRAMRequest{write:False, address: truncate(iReg_minus_1), datain:?, responseOnWrite:?});
      endaction
      action
	 let xNext <- needleBram.portA.response.get();
	 x_i <= xNext;
      endaction
      while (stage == Search) seq
	 $display("initial i=%d j=%d x_i=%h t_j=%h", iReg, jReg, x_i, t_j);
	 while ((i == m + 1) || (i > 0 && x_i != t_j)) seq
	    action
	       mpNextBram.portA.request.put(BRAMRequest{write:False, address: truncate(iReg), datain:?, responseOnWrite:?});
      	    endaction
	    action
	       Bit#(32) mpNext <- mpNextBram.portA.response.get();
	       let iNext = mpNext[15:0];
	       let xNext = mpNext[31:16];
	       iReg <= extend(iNext);
	       x_i <= truncate(xNext);
	       $display("i=%d iNext=%d m+1=%d x_i=%h t_j=%h mpNext=%h", i, iNext, m+1, xNext, t_j, mpNext);
	    endaction
	 endseq
	 action
	    iReg <= iReg + 1;
	    jReg <= jReg + 1;
	    needleBram.portA.request.put(BRAMRequest{write:False, address: truncate(iReg+1-1), datain:?, responseOnWrite:?});
	    let t = haystackGb.first[0]; haystackGb.deq();
	    t_j <= t;
      endaction
      action
	    if (t_j == 0) begin
	       $display("iReg=%d jReg=%d t==0 locf.enq(-1)", iReg-1, jReg-1, t_j);
	       locf.enq(-1); // this seems to be needed for the strstr example to terminate
	    end
	 endaction
	 action
	    let xNext <- needleBram.portA.response.get();
	    x_i <= xNext;
	    $display("step    i=%d j=%d x_i=%h t_j=%h", iReg, jReg, xNext, t_j);
	    if (iReg == m + 1) begin
	       $display("match at j=%d", jReg - iReg);
	       locf.enq(unpack(jReg - iReg));
	    end
	 endaction
      endseq // stage == Search
      endseq);

   interface PipeIn needle = n2b.pipe;
   interface PipeIn mpNext = mp2b.pipe;
   interface PipeIn haystack = toPipeIn(haystackFifo);
   interface PipeOut locdone = toPipeOut(locf);

   method Action clear();
      stage <= Initialized;
   endmethod
   method Action start(Bit#(32) needleLen);
      stage <= Search;
      jReg <= 1;
      iReg <= 1;
      needleLenReg <= needleLen;
   endmethod

endmodule

// Copyright (c) 2013 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

/*
 * Implementation of:
 *    MP algorithm on pages 7-11 from "Pattern Matching Algorithms" by
 *       Alberto Apostolico, Zvi Galil, 1997
 */

`include "ConnectalProjectConfig.bsv"
import FIFO::*;
import FIFOF::*;
import SpecialFIFOs::*;
import Vector::*;
import BRAM::*;
import Gearbox::*;
import Connectable::*;
import ConfigReg::*;
import StmtFSM::*;
import Probe::*;

import MemUtils::*;
import MemTypes::*;
import Dma2BRAM::*;
import Pipe::*;
import EHR::*;

interface MPEngine#(numeric type haystackBusWidth, numeric type configBusWidth);
   interface PipeIn#(Triplet#(Bit#(32))) setsearch;
   interface PipeOut#(Int#(32)) locdone;
endinterface

interface MPStreamEngine#(numeric type haystackBusWidth, numeric type configBusWidth);
   interface PipeIn#(MemDataF#(configBusWidth)) needle;
   interface PipeIn#(MemDataF#(configBusWidth)) mpNext;
   interface PipeIn#(MemDataF#(haystackBusWidth)) haystack;
   interface PipeOut#(Int#(32)) locdone;
   method Action clear();
   method Action start(Bit#(32) needleLen);
endinterface

typedef Bit#(8) Char;
typedef Bit#(64) DWord;
typedef Bit#(32) Word;

typedef 1024 MaxNeedleLen;
typedef TLog#(MaxNeedleLen) NeedleIdxWidth;
typedef Bit#(NeedleIdxWidth) NeedleIdx;

typedef enum {Config_needle, Config_mpNext, Initialized, Search} Stage deriving (Eq, Bits);

module mkMPEngine#(MemReadEngineServer#(haystackBusWidth) haystackReader,
		   MemReadEngineServer#(configBusWidth) configReader) (MPEngine#(haystackBusWidth,configBusWidth))

   provisos(Add#(a__, 8, haystackBusWidth),
	    Div#(haystackBusWidth,8,haystackBusBytes),
	    Mul#(haystackBusBytes,8,haystackBusWidth),
	    Add#(1, b__, haystackBusBytes),
	    Add#(c__, 32, haystackBusWidth),
	    Add#(1, d__, TDiv#(haystackBusWidth, 32)),
	    Mul#(TDiv#(haystackBusWidth, 32), 32, haystackBusWidth),
	    Add#(e__, TLog#(haystackBusBytes), 32),
	    Add#(f__, TLog#(TDiv#(haystackBusWidth, 32)), 32)
	    ,Mul#(TDiv#(configBusWidth, 8), 8, configBusWidth)
	    ,Add#(1, g__, TDiv#(configBusWidth, 8))
	    ,Add#(h__, TLog#(TDiv#(configBusWidth, 8)), 32)
	    ,Add#(i__, TLog#(TDiv#(configBusWidth, 32)), 32)
	    ,Add#(1, j__, TDiv#(configBusWidth, 32))
	    ,Mul#(TDiv#(configBusWidth, 32), 32, configBusWidth)
	    );

   FIFOF#(Int#(32)) locf <- mkFIFOF;
   FIFO#(Bool) conff <- mkSizedFIFO(1);

   let verbose = True;
   let debug = False;

   Clock clk <- exposeCurrentClock;
   Reset rst <- exposeCurrentReset;
   BRAM2Port#(NeedleIdx, Char) needle  <- mkBRAM2Server(defaultValue);
   BRAM2Port#(NeedleIdx, Bit#(32)) mpNext <- mkBRAM2Server(defaultValue);
   Gearbox#(haystackBusBytes,1,Char) haystack <- mkNto1Gearbox(clk,rst,clk,rst);

   Reg#(Bit#(32)) cycleCnt <- mkReg(0);
   Reg#(Bit#(32)) lastHD <- mkReg(0);

   Reg#(Stage)    stage <- mkReg(Config_needle);
   Reg#(Bit#(32)) needleLenReg <- mkReg(0);
   Reg#(Bit#(32)) haystackLenReg <- mkReg(0);
   Reg#(Bit#(32)) haystackBase <- mkReg(0);
   Reg#(Bit#(32)) jReg <- mkReg(0); // offset in haystack
   Reg#(Bit#(32)) iReg <- mkReg(0); // offset in needle
   Reg#(Bit#(2))  epochReg <- mkReg(0);

   BRAMWriter#(NeedleIdxWidth,configBusWidth) n2b <- mkBRAMWriter(0, needle.portB, configReader);
   BRAMWriter#(NeedleIdxWidth,configBusWidth) mp2b <- mkBRAMWriter(1, mpNext.portB, configReader);

   FIFOF#(Tuple2#(Bit#(2),Bit#(32))) efifo <- mkSizedFIFOF(2);
   FIFOF#(Triplet#(Bit#(32))) ssfifo <- mkFIFOF;
   FIFO#(void) doneFifo <- mkFIFO;

   rule countCycles;
      if (debug) $display("******************************************** %d", cycleCnt);
      cycleCnt <= cycleCnt+1;
   endrule

   rule haystackResp;
      if (debug) $display("mkMPEngine::haystackResp");
      let rv <- toGet(haystackReader.data).get;
      haystack.enq(unpack(rv.data));
      if (rv.last)
         conff.deq;
   endrule

   rule haystackDrain(stage != Search);
      if (debug) $display("mkMPEngine::haystackDrain");
      haystack.deq;
   endrule

   rule bramDrain(stage != Search);
      if (debug) $display("mkMPEngine::mpNextDrain");
      let x <- mpNext.portA.response.get;
      let y <- needle.portA.response.get;
      efifo.deq;
   endrule

`define OPTIMIZE_MISMATCH
`ifdef OPTIMIZE_MISMATCH
   rule matchNeedleReq(stage == Search);
      if (debug) $display("mkMPEngine::matchNeedleReq %d %d", epochReg, iReg);
      needle.portA.request.put(BRAMRequest{write:False, address: truncate(iReg-1), datain:?, responseOnWrite:?});
      mpNext.portA.request.put(BRAMRequest{write:False, address: truncate(iReg), datain:?, responseOnWrite:?});
      efifo.enq(tuple2(epochReg,iReg));
      iReg <= 1;
   endrule

   rule matchNeedleResp(stage == Search);
      let nv <- needle.portA.response.get;
      let mp <- mpNext.portA.response.get;
      let epoch = tpl_1(efifo.first);
      efifo.deq;
      if (debug) $display("mkMPEngine::matchNeedleResp %d %d", epochReg, epoch);
      if (epoch == epochReg) begin
	 Bool deq_haystack = False;
	 let n = haystackLenReg;
	 let m = needleLenReg;
	 let hv = haystack.first;
	 let i = tpl_2(efifo.first);
	 let j = jReg;
	 if (debug) $display("mkMPEngine::feck %d %d %d %d %x %x", n, m, i, j, hv[0], nv);
	 if (j > n) begin
	    // jReg points to the end of the haystack; we are done
	    stage <= Config_needle;
	    if (debug) $display("mkMPEngine::end of search %d", j);
	    locf.enq(-1);
	 end
	 else if (i==m+1) begin
	    // iReg points to the end of the needle; we have a match
	    if (debug) $display("mkMPEngine::string match %d", j);
	    locf.enq(unpack(haystackBase+j-i));
	 end
	 else if (nv != hv[0]) begin
	    // mismatch betwen head of haystack and head of needle; rewind iReg
	    if (debug) $display("mkMPEngine::char mismatch %d %d MP_Next[i]=%d", i, j, mp);
	    if (mp == 0) begin
	       iReg <= 1;
	       jReg <= j+1;
	       deq_haystack = True;
	    end
	    else begin
	       epochReg <= epochReg + 1;
	       iReg <= mp;
	    end
	 end
	 else begin
	    // match between head of needle and head of haystack; increment haystack
	    if (debug) $display("mkMPEngine::char match(%d) %d %d", (nv == hv[0]), i, j);
	    deq_haystack = True;
	    jReg <= j+1;
	    epochReg <= epochReg + 1;
	    iReg <= i+1;
	 end
	 if (deq_haystack) begin
	    haystack.deq;
	    lastHD <= cycleCnt;
	    if (debug) $display("mkMPEngine:: deq haystack(%d)", cycleCnt-lastHD);
	 end
      end
      else begin
	 if (debug) $display("mkMPEngine::discard");
	 noAction;
      end
   endrule
`else
   rule matchNeedleReq(stage == Search);
      if (debug) $display("mkMPEngine::matchNeedleReq %d %d", epochReg, iReg);
      needle.portA.request.put(BRAMRequest{write:False, address: truncate(iReg-1), datain:?, responseOnWrite:?});
      mpNext.portA.request.put(BRAMRequest{write:False, address: truncate(iReg), datain:?, responseOnWrite:?});
      efifo.enq(tuple2(epochReg,iReg));
      iReg <= iReg+1;
   endrule

   rule matchNeedleResp(stage == Search);
      let nv <- needle.portA.response.get;
      let mp <- mpNext.portA.response.get;
      let epoch = tpl_1(efifo.first);
      efifo.deq;
      if (debug) $display("mkMPEngine::matchNeedleResp %d %d", epochReg, epoch);
      if (epoch == epochReg) begin
	 Bool deq_haystack = False;
	 let n = haystackLenReg;
	 let m = needleLenReg;
	 let hv = haystack.first;
	 let i = tpl_2(efifo.first);
	 let j = jReg;
	 if (debug) $display("mkMPEngine::feck %d %d %d %d %x %x", n, m, i, j, hv[0], nv);
	 if (j > n) begin
	    // jReg points to the end of the haystack; we are done
	    stage <= Config_needle;
	    if (debug) $display("mkMPEngine::end of search %d", j);
	 end
	 else if (i==m+1) begin
	    // iReg points to the end of the needle; we have a match
	    if (debug) $display("mkMPEngine::string match %d", j);
	    locf.enq(unpack(haystackBase+j-i));
	    epochReg <= epochReg + 1;
	    iReg <= 1;
	 end
	 else if (nv != hv[0]) begin
	    // mismatch betwen head of haystack and head of needle; rewind iReg
	    if (debug) $display("mkMPEngine::char mismatch %d %d MP_Next[i]=%d", i, j, mp);
	    epochReg <= epochReg + 1;
	    if (mp == 0) begin
	       iReg <= 1;
	       jReg <= j+1;
	       deq_haystack = True;
	    end
	    else begin
	       iReg <= mp;
	    end
	 end
	 else begin
	    // match between head of needle and head of haystack; increment haystack
	    if (debug) $display("mkMPEngine::char match(%d) %d %d", (nv == hv[0]), i, j);
	    deq_haystack = True;
	    jReg <= j+1;
	 end
	 if (deq_haystack) begin
	    haystack.deq;
	    lastHD <= cycleCnt;
	    if (debug) $display("mkMPEngine:: deq haystack(%d)", cycleCnt-lastHD);
	 end
      end
      else begin
	 if (debug) $display("mkMPEngine::discard");
	 noAction;
      end
   endrule
`endif

   rule finish_setup_n2b;
      if (verbose) $display("mkMPEngine::finish_setup_n2b");
      let x <- n2b.finish;
      conff.deq;
      stage <= Config_mpNext;
   endrule

   rule finish_setup_mp2b;
      if (verbose) $display("mkMPEngine::finish_setup_mp2b");
      let y <- mp2b.finish;
      conff.deq;
      stage <= Initialized;
   endrule

   rule setup_needle (stage == Config_needle);
      conff.enq(True);
      match {.needle_sglId, .mpNext_sglId, .needle_len} = ssfifo.first;
      needleLenReg <= extend(needle_len);
      if (verbose) $display("mkMPEngine::setup_needle %d %d", needle_sglId, needle_len);
      n2b.start(needle_sglId, 0, 0, pack(truncate(needle_len)));
   endrule

   rule setup_mpNext (stage == Config_mpNext);
      conff.enq(True);
      match {.needle_sglId, .mpNext_sglId, .needle_len} = ssfifo.first;
      needleLenReg <= extend(needle_len);
      if (verbose) $display("mkMPEngine::setup_mpNext %d %d", mpNext_sglId, needle_len);
      mp2b.start(mpNext_sglId, 0, 0, pack(truncate(needle_len)));
      ssfifo.deq;
   endrule

   rule search (stage == Initialized && !efifo.notEmpty && !haystack.notEmpty);
      stage <= Search;
      conff.enq(True);
      match {.haystack_sglId, .haystack_len, .haystack_base} <- toGet(ssfifo).get;
      haystackLenReg <= extend(haystack_len);
      haystackBase <= extend(haystack_base);
      iReg <= 1;
      jReg <= 1;
      epochReg <= 0;
      Bit#(32) haystack_len_ds = haystack_len+fromInteger(valueOf(haystackBusBytes)-1);
      Bit#(TLog#(haystackBusBytes)) zeros = 0;
      Bit#(32) haystack_len_bytes = {zeros,haystack_len_ds[31:valueOf(TLog#(haystackBusBytes))]} * fromInteger(valueOf(haystackBusBytes));
      $display("haystack read offset=%d burstLen=%d", haystack_base, fromInteger(8*valueOf(haystackBusBytes)));
      haystackReader.request.put(MemengineCmd{sglId:haystack_sglId, base:extend(haystack_base), len:haystack_len_bytes, burstLen:fromInteger(8*valueOf(haystackBusBytes)), tag: 0});
      if (verbose) $display("mkMPEngine::search %d %d %d",  haystack_sglId, haystack_base, haystack_len_bytes);
   endrule

   interface PipeIn setsearch = toPipeIn(ssfifo);
   interface PipeOut locdone = toPipeOut(locf);

endmodule

module mkMPStreamEngine(MPStreamEngine#(haystackBusWidth,configBusWidth))

   provisos(Add#(a__, 8, haystackBusWidth),
	    Div#(haystackBusWidth,8,haystackBusBytes),
	    Mul#(haystackBusBytes,8,haystackBusWidth),
	    Add#(1, b__, haystackBusBytes),
	    Add#(c__, 32, haystackBusWidth),
	    Add#(1, d__, TDiv#(haystackBusWidth, 32)),
	    Mul#(TDiv#(haystackBusWidth, 32), 32, haystackBusWidth),
	    Add#(e__, TLog#(haystackBusBytes), 32),
	    Add#(f__, TLog#(TDiv#(haystackBusWidth, 32)), 32)
	    ,Mul#(TDiv#(configBusWidth, 8), 8, configBusWidth)
	    ,Add#(1, g__, TDiv#(configBusWidth, 8))
	    ,Add#(h__, TLog#(TDiv#(configBusWidth, 8)), 32)
	    ,Add#(i__, TLog#(TDiv#(configBusWidth, 32)), 32)
	    ,Add#(1, j__, TDiv#(configBusWidth, 32))
	    ,Mul#(TDiv#(configBusWidth, 32), 32, configBusWidth)
	    );

   FIFOF#(Int#(32)) locf <- mkFIFOF;

   let verbose = True;
   let debug = True;

   Clock clk <- exposeCurrentClock;
   Reset rst <- exposeCurrentReset;
   BRAM2Port#(NeedleIdx, Char) needleBram  <- mkBRAM2Server(defaultValue);
   BRAM2Port#(NeedleIdx, Bit#(32)) mpNextBram <- mkBRAM2Server(defaultValue);
   FIFOF#(MemDataF#(haystackBusWidth)) haystackFifo <- mkFIFOF();
   Gearbox#(haystackBusBytes,1,Char) haystackGb <- mkNto1Gearbox(clk,rst,clk,rst);

   Reg#(Bit#(32)) cycleCnt <- mkReg(0);
   Reg#(Bit#(32)) lastHD <- mkReg(0);

   Reg#(Stage)    stage <- mkReg(Initialized);
   Reg#(Bit#(32)) needleLenReg <- mkReg(0);
   Reg#(Bit#(32)) jReg <- mkReg(1); // offset in haystack
   Reg#(Bit#(32)) iReg <- mkReg(1); // offset in needle

   BRAMPipeIn#(NeedleIdxWidth,configBusWidth) n2b <- mkBRAMPipeIn(0, needleBram.portB);
   BRAMPipeIn#(NeedleIdxWidth,configBusWidth) mp2b <- mkBRAMPipeIn(1, mpNextBram.portB);

   FIFOF#(Tuple2#(Bit#(2),Bit#(32))) efifo <- mkSizedFIFOF(2);
   FIFO#(void) doneFifo <- mkFIFO;

   rule countCycles;
      //if (debug) $display("******************************************** %d", cycleCnt);
      cycleCnt <= cycleCnt+1;
   endrule

   rule haystackResp;
      let rv <- toGet(haystackFifo).get;
      if (debug) $display("mkMPEngine::haystackResp rv=%h", rv.data);
      haystackGb.enq(unpack(rv.data));
   endrule

   let x_i <- mkReg(0);
   let t_j <- mkReg(0);
   let m = needleLenReg;
   let i = iReg;
   let matchFsm <- mkAutoFSM(seq
      while (stage != Search) seq
	 iReg <= 1;
      endseq
      action
	 iReg <= 1;
	 jReg <= 1;
	 let t = haystackGb.first[0]; haystackGb.deq();
	 t_j <= t;
	 needleBram.portA.request.put(BRAMRequest{write:False, address: truncate(iReg-1), datain:?, responseOnWrite:?});
      endaction
      action
	 let xNext <- needleBram.portA.response.get();
	 x_i <= xNext;
      endaction
      while (stage == Search) seq
	 $display("initial i=%d j=%d x_i=%h t_j=%h", iReg, jReg, x_i, t_j);
	 while ((i == m + 1) || (i > 0 && x_i != t_j)) seq
	    action
	       mpNextBram.portA.request.put(BRAMRequest{write:False, address: truncate(iReg), datain:?, responseOnWrite:?});
      	    endaction
	    action
	       Bit#(32) mpNext <- mpNextBram.portA.response.get();
	       let iNext = mpNext[15:0];
	       let xNext = mpNext[31:16];
	       iReg <= extend(iNext);
	       x_i <= truncate(xNext);
	       $display("i=%d iNext=%d m+1=%d x_i=%h t_j=%h mpNext=%h", i, iNext, m+1, xNext, t_j, mpNext);
	    endaction
	 endseq
	 action
	    iReg <= iReg + 1;
	    jReg <= jReg + 1;
	    needleBram.portA.request.put(BRAMRequest{write:False, address: truncate(iReg+1-1), datain:?, responseOnWrite:?});
	    let t = haystackGb.first[0]; haystackGb.deq();
	    t_j <= t;
	    if (t == 0) begin
	       $display("iReg=%d jReg=%d t==0 locf.enq(-1)", iReg, jReg, t);
	       locf.enq(-1); // this seems to be needed for the strstr example to terminate
	    end
	 endaction
	 action
	    let xNext <- needleBram.portA.response.get();
	    x_i <= xNext;
	    $display("step    i=%d j=%d x_i=%h t_j=%h", iReg, jReg, xNext, t_j);
	    if (iReg == m + 1) begin
	       $display("match at j=%d", jReg - iReg);
	       locf.enq(unpack(jReg - iReg));
	    end
	 endaction
      endseq // stage == Search
      endseq);

   interface PipeIn needle = n2b.pipe;
   interface PipeIn mpNext = mp2b.pipe;
   interface PipeIn haystack = toPipeIn(haystackFifo);
   interface PipeOut locdone = toPipeOut(locf);

   method Action clear();
      stage <= Initialized;
   endmethod
   method Action start(Bit#(32) needleLen);
      stage <= Search;
      jReg <= 1;
      iReg <= 1;
      needleLenReg <= needleLen;
   endmethod

endmodule


added instances of mkConnectionWithClocks for PhysMemMaster and PhysMemSlave

// Copyright (c) 2013,2014 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import GetPut :: *;
import ClientServer :: *;
import Clocks :: *;
import MemTypes :: *;

////////////////////////////////////////////////////////////////////////////////
/// Typeclass Definition
////////////////////////////////////////////////////////////////////////////////
typeclass ConnectableWithClocks#(type a, type b);
   module mkConnectionWithClocks#(a x1, b x2, Clock fastClock, Reset fastReset, Clock slowClock, Reset slowReset)(Empty);
endtypeclass

instance ConnectableWithClocks#(Get#(a), Put#(a)) provisos (Bits#(a, awidth));
   module mkConnectionWithClocks#(Get#(a) in, Put#(a) out,
                                  Clock inClock, Reset inReset,
                                  Clock outClock, Reset outReset)(Empty) provisos (Bits#(a, awidth));
       SyncFIFOIfc#(a) synchronizer <- mkSyncFIFO(1, inClock, inReset, outClock);
       rule doGet;
           let v <- in.get();
	   synchronizer.enq(v);
       endrule
       rule doPut;
           let v = synchronizer.first;
	   synchronizer.deq;
	   out.put(v);
       endrule
   endmodule: mkConnectionWithClocks
endinstance: ConnectableWithClocks

instance ConnectableWithClocks#(Client#(a,b), Server#(a,b)) provisos (Bits#(a, awidth), Bits#(b, bwidth));
   module mkConnectionWithClocks#(Client#(a,b) client, Server#(a,b) server,
                                  Clock sourceClock, Reset sourceReset,
                                  Clock destClock, Reset destReset)(Empty)
      provisos (ConnectableWithClocks#(Get#(a), Put#(a)),
		ConnectableWithClocks#(Get#(b), Put#(b)),
		Bits#(a, awidth),
		Bits#(b, bwidth));
      mkConnectionWithClocks(client.request, server.request, sourceClock, sourceReset, destClock, destReset);
      mkConnectionWithClocks(server.response, client.response, destClock, destReset, sourceClock, sourceReset);
   endmodule
endinstance

instance ConnectableWithClocks#(PhysMemReadClient#(addrWidth, dataWidth),
				PhysMemReadServer#(addrWidth, dataWidth));
   module mkConnectionWithClocks#(PhysMemReadClient#(addrWidth, dataWidth) client,
				  PhysMemReadServer#(addrWidth, dataWidth) server,
                                  Clock sourceClock, Reset sourceReset,
				  Clock destClock, Reset destReset)(Empty);
      mkConnectionWithClocks(client.readReq, server.readReq, sourceClock, sourceReset, destClock, destReset);
      mkConnectionWithClocks(server.readData, client.readData, destClock, destReset, sourceClock, sourceReset);
   endmodule
endinstance

instance ConnectableWithClocks#(PhysMemWriteClient#(addrWidth, dataWidth),
				PhysMemWriteServer#(addrWidth, dataWidth));
   module mkConnectionWithClocks#(PhysMemWriteClient#(addrWidth, dataWidth) client,
				  PhysMemWriteServer#(addrWidth, dataWidth) server,
                                  Clock sourceClock, Reset sourceReset,
				  Clock destClock, Reset destReset)(Empty);
      mkConnectionWithClocks(client.writeReq, server.writeReq, destClock, destReset, sourceClock, sourceReset);
      mkConnectionWithClocks(client.writeData, server.writeData, destClock, destReset, sourceClock, sourceReset);
      mkConnectionWithClocks(server.writeDone, client.writeDone, sourceClock, sourceReset, destClock, destReset);
   endmodule
endinstance

instance ConnectableWithClocks#(PhysMemMaster#(addrWidth, dataWidth), PhysMemSlave#(addrWidth, dataWidth));
   module mkConnectionWithClocks#(PhysMemMaster#(addrWidth, dataWidth) client,
				  PhysMemSlave#(addrWidth, dataWidth) server,
                                  Clock sourceClock, Reset sourceReset,
				  Clock destClock, Reset destReset)(Empty);
      mkConnectionWithClocks(client.read_client, server.read_server, sourceClock, sourceReset, destClock, destReset);
      mkConnectionWithClocks(client.write_client, server.write_server, destClock, destReset, sourceClock, sourceReset);
   endmodule
endinstance

module mkClockBinder#(a ifc) (a);
   return ifc;
endmodule

// Copyright (c) 2013,2014 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import GetPut :: *;
import ClientServer :: *;
import Clocks :: *;

////////////////////////////////////////////////////////////////////////////////
/// Typeclass Definition
////////////////////////////////////////////////////////////////////////////////
typeclass ConnectableWithClocks#(type a, type b);
   module mkConnectionWithClocks#(a x1, b x2, Clock fastClock, Reset fastReset, Clock slowClock, Reset slowReset)(Empty);
endtypeclass

instance ConnectableWithClocks#(Get#(a), Put#(a)) provisos (Bits#(a, awidth));
   module mkConnectionWithClocks#(Get#(a) in, Put#(a) out,
                                  Clock inClock, Reset inReset,
                                  Clock outClock, Reset outReset)(Empty) provisos (Bits#(a, awidth));
       SyncFIFOIfc#(a) synchronizer <- mkSyncFIFO(1, inClock, inReset, outClock);
       rule doGet;
           let v <- in.get();
	   synchronizer.enq(v);
       endrule
       rule doPut;
           let v = synchronizer.first;
	   synchronizer.deq;
	   out.put(v);
       endrule
   endmodule: mkConnectionWithClocks
endinstance: ConnectableWithClocks

instance ConnectableWithClocks#(Client#(a,b), Server#(a,b)) provisos (Bits#(a, awidth), Bits#(b, bwidth));
   module mkConnectionWithClocks#(Client#(a,b) client, Server#(a,b) server,
                                  Clock sourceClock, Reset sourceReset,
                                  Clock destClock, Reset destReset)(Empty)
      provisos (ConnectableWithClocks#(Get#(a), Put#(a)),
		ConnectableWithClocks#(Get#(b), Put#(b)),
		Bits#(a, awidth),
		Bits#(b, bwidth));
      mkConnectionWithClocks(client.request, server.request, sourceClock, sourceReset, destClock, destReset);
      mkConnectionWithClocks(server.response, client.response, destClock, destReset, sourceClock, sourceReset);
   endmodule
endinstance

module mkClockBinder#(a ifc) (a);
   return ifc;
endmodule