TSO passes integrate

TSO passes integrate

import Ehr::*;
import GetPut::*;
import Fifo::*;
import RegFile::*;
import Vector::*;
import BuildVector::*;
import FShow::*;
import Randomizable::*;
import ConfigReg::*;
import StmtFSM::*;

import Types::*;
import MemoryTypes::*;
import ProcTypes::*;
import CCTypes::*;
import TestTypes::*;
import Exec::*;
import SpecFifo::*;
import SplitLSQ::*;
import StoreBuffer::*;
import RefMem::*;
import CCM::*;
import DelayTLB::*;
import SpecTagManager::*;
import HasSpecBits::*;
import GSpecUpdate::*;

// FIFO to begin memory pipeline, needed for an extra stage to update LSQ data
typedef struct {
    TestId testId;
    LdStQTag lsqTag;
    SpecTag specTag;
} MemInput deriving(Bits, Eq, FShow);

typedef SpecFifo_SB_deq_enq_C_deq_enq#(1, MemInput) MemInputQ;
(* synthesize *)
module mkMemInputQ(MemInputQ);
    let m <- mkSpecFifo_SB_deq_enq_C_deq_enq(False);
    return m;
endmodule

typedef enum {Invalid, Lr, Sc} WaitBlockingResp deriving(Bits, Eq, FShow);

typedef struct {
    LineDataOffset offset;
    ByteEn shiftedBE;
    Data shiftedData;
} WaitStResp deriving(Bits, Eq, FShow);

(* synthesize *)
module mkTbIntegrate(Empty);
    // reference & test req generator
    RefMem refMem <- mkRefMem;
    RegFile#(TestId, TestEntry) test <- mkRegFileFull;
    // id of next test of the same type (i.e. same Ld/St type). If id ==
    // TestNum, then no more next test.
    RegFile#(TestId, TestCnt) nextPtr <- mkRegFileFull;

    Randomize#(LineAddr) randLineAddr <- mkConstrainedRandomizer(0, fromInteger(valueof(TestLineNum) - 1));
    Randomize#(LineDataOffset) randDataSel <- mkConstrainedRandomizer(0, fromInteger(valueof(TestDataSelNum) - 1));
    Randomize#(ByteOffset) randByteOff <- mkGenericRandomizer;
    Randomize#(AccessRange) randRange <- mkGenericRandomizer;
    //Randomize#(TestMemFunc) randFunc <- mkGenericRandomizer;
    Randomize#(TestMemFunc) randFunc <- mkConstrainedRandomizer(Ld, St);
    Randomize#(Data) randData <- mkGenericRandomizer;

    // test state & counters
    Reg#(TestState) state <- mkReg(StartInit);
    Reg#(LineAddr) testLineAddr <- mkReg(0);
    Reg#(TestCnt) genPtr <- mkReg(0);
    Reg#(Maybe#(TestId)) lastLdPtr <- mkReg(Invalid);
    Reg#(Maybe#(TestId)) lastStPtr <- mkReg(Invalid);
    Reg#(TestCnt) ldCnt <- mkReg(0);
    Reg#(TestCnt) stCnt <- mkReg(0);
    Reg#(TestCnt) lrCnt <- mkReg(0);
    Reg#(TestCnt) scCnt <- mkReg(0);
    Reg#(Bit#(64)) killCnt <- mkConfigReg(0);
    Reg#(Bit#(64)) stallCnt <- mkConfigReg(0);

    // logs
    Reg#(File) reqLog <- mkReg(InvalidFile);
    Reg#(File) respLdLog <- mkReg(InvalidFile);
    Reg#(File) respStLog <- mkReg(InvalidFile);
    Reg#(File) checkLog <- mkReg(InvalidFile);

    // detect deadlock
    PulseWire commitInst <- mkPulseWire;
    PulseWire deqSB <- mkPulseWire;
    Reg#(TimeOut) timeOut <- mkReg(0);

    // detect test done
    Wire#(Bool) stbEmpty_for_done <- mkBypassWire;
    Wire#(TestCnt) comLdPtr_for_done <- mkBypassWire;
    Wire#(TestCnt) comStPtr_for_done <- mkBypassWire;

    // DUT
    SpecTagManager stm <- mkSpecTagManager;
    MemInputQ inputQ <- mkMemInputQ;
    DelayTLB tlb <- mkDelayTLB;
    SplitLSQ lsq <- mkSplitLSQ;
`ifdef TSO_MM
    StoreBuffer stb <- mkDummyStoreBuffer;
`else
    StoreBuffer stb <- mkStoreBufferEhr;
`endif
    // for Lr/Sc we first need to execution
    Reg#(WaitBlockingResp) waitLrScResp <- mkReg(Invalid);
`ifdef TSO_MM
    Fifo#(1, WaitStResp) waitStRespQ <- mkCFFifo;
`endif
    // req to send to LSQ
    Reg#(TestCnt) sendPtr <- mkReg(0);
    // req to commit
    Reg#(TestCnt) comLdPtr <- mkReg(fromInteger(valueof(TestNum)));
    Reg#(TestCnt) comStPtr <- mkReg(fromInteger(valueof(TestNum)));
    // Test IDs for in-flight loads
    Vector#(LdQSize, Reg#(TestId)) testingLdPtr <- replicateM(mkRegU);
    // load resps
    RegFile#(TestId, Data) ldResp <- mkRegFileFull;

    RWire#(LSQIssueLdInfo) issueLd <- mkRWire; // issue right after TLB resp

    // FIFOs to hold load result
    Fifo#(2, Data) respLrScQ <- mkCFFifo;
    Fifo#(2, Tuple2#(LdQTag, Data)) forwardQ <- mkCFFifo;
    Fifo#(2, Tuple2#(LdQTag, Data)) memRespLdQ <- mkCFFifo;

    // memory
    CCMProcResp memRespIfc = (interface CCMProcResp;
        method Action respLd(CCMReqId id, Data d);
            memRespLdQ.enq(tuple2(id, d));
        endmethod

        method Action respLrScAmo(Data d);
            respLrScQ.enq(d);
        endmethod

        method ActionValue#(Tuple2#(LineByteEn, Line)) respSt(CCMReqId id);
`ifdef TSO_MM
            lsq.deqSt;
            let waitSt <- toGet(waitStRespQ).get;
            Vector#(LineSzData, ByteEn) be = replicate(replicate(False));
            Line data = replicate(0);
            be[waitSt.offset] = waitSt.shiftedBE;
            data[waitSt.offset] = waitSt.shiftedData;
            // incr comptr & set wire to reset timeout
            comStPtr <= nextPtr.sub(comStPtr);
            commitInst.send;
            $display("[respSt] %t: comPtr %d; ", $time, comStPtr, fshow(waitSt));
            return tuple2(unpack(pack(be)), data);
`else
            SBIndex idx = truncate(id);
            // deq SB
            let e <- stb.deq(idx);
            // wake up loads in LSQ
            lsq.wakeupLdStalledBySB(idx);
            // send wire to reset timeout
            deqSB.send;
            // return SB entry
            return tuple2(e.byteEn, unpack(e.data));
`endif
        endmethod
    endinterface);
    CCM ccm <- mkCCM(memRespIfc);

    // speculation related
    GSpecUpdate#(2) globalSpecUpdate <- mkGSpecUpdate(joinSpeculationUpdate(vec(
        stm.specUpdate,
        tlb.specUpdate,
        lsq.specUpdate,
        inputQ.specUpdate
    )));
    Integer tlbCorrectSpecPort = 0;
    Integer lsqCorrectSpecPort = 1;

    function Action redirect(TestId newSendPtr, SpecTag specTag);
    action
        sendPtr <= zeroExtend(newSendPtr);
        globalSpecUpdate.incorrectSpec(specTag);
    endaction
    endfunction

    // =============== Test FSM ===============

    // FSM to init
    Stmt initSeq = (seq
        // init mem
        repeat(fromInteger(valueof(TestLineNum)))
        action
            refMem.initLine({testLineAddr, 0}, replicate(0));
            ccm.initLine({testLineAddr, 0}, replicate(0));
            testLineAddr <= testLineAddr + 1;
        endaction

        // reset test addr & init randomizers & open files
        action
            refMem.initDone;
            ccm.initDone;
            randLineAddr.cntrl.init;
            randDataSel.cntrl.init;
            randByteOff.cntrl.init;
            randRange.cntrl.init;
            randFunc.cntrl.init;
            randData.cntrl.init;
            let f <- $fopen("req.log");
            reqLog <= f;
            f <- $fopen("respLd.log");
            respLdLog <= f;
            f <- $fopen("respSt.log");
            respStLog <= f;
            f <- $fopen("check.log");
            checkLog <= f;
        endaction
        $fdisplay(stderr, "INFO: init mem done");

        // generate req
        while(genPtr < fromInteger(valueof(TestNum)))
        action
            // randomize req
            let lineAddr <- randLineAddr.next;
            let dataSel <- randDataSel.next;
            let offset <- randByteOff.next;
            let range <- randRange.next;
            let func <- randFunc.next;
            let {paddr, memInst} = getAddrMemInst(lineAddr, dataSel, offset, range, func);
            let stData <- randData.next;
            // get resp
            Data resp = ?;
            case(func)
                Ld, Lr: begin
                    let r <- refMem.procReq(MemReq {
                        op: toMemOp(func),
                        addr: paddr,
                        byteEn: replicate(False),
                        data: ?
                    });
                    resp = gatherLoad(paddr, memInst.byteEn, memInst.unsignedLd, r);
                end
                St, Sc: begin
                    let {be, d} = scatterStore(paddr, memInst.byteEn, stData);
                    // directly record Sc resp
                    resp <- refMem.procReq(MemReq {
                        op: toMemOp(func),
                        addr: paddr,
                        byteEn: be,
                        data: d
                    });
                end
                default: doAssert(False, "unsupport func");
            endcase
            // record test entry
            let entry = TestEntry {
                memInst: memInst,
                paddr: paddr,
                stData: stData,
                resp: resp
            };
            test.upd(truncate(genPtr), entry);
            // update genPtr
            genPtr <= genPtr + 1;
            // udpate next id & record first Ld/St ptr into comLd/StPtr
            if(func == Ld || func == Lr) begin
                if(lastLdPtr matches tagged Valid .id) begin
                    nextPtr.upd(id, genPtr);
                end
                lastLdPtr <= Valid (truncate(genPtr));
                if(comLdPtr == fromInteger(valueof(TestNum))) begin
                    comLdPtr <= genPtr;
                end
            end
            else begin
                if(lastStPtr matches tagged Valid .id) begin
                    nextPtr.upd(id, genPtr);
                end
                lastStPtr <= Valid (truncate(genPtr));
                if(comStPtr == fromInteger(valueof(TestNum))) begin
                    comStPtr <= genPtr;
                end
            end
            // log
            $fdisplay(reqLog, "%d: ", genPtr, fshow(entry));
            // stats
            case(func)
                Ld: ldCnt <= ldCnt + 1;
                Lr: lrCnt <= lrCnt + 1;
                St: stCnt <= stCnt + 1;
                Sc: scCnt <= scCnt + 1;
            endcase
        endaction
        // fill in nextPtr for the last Ld and St
        action
            if(lastLdPtr matches tagged Valid .id) begin
                nextPtr.upd(id, fromInteger(valueof(TestNum)));
            end
        endaction
        action
            if(lastStPtr matches tagged Valid .id) begin
                nextPtr.upd(id, fromInteger(valueof(TestNum)));
            end
        endaction
        $fdisplay(stderr, "INFO: gen req done");
    endseq);

    FSM initFSM <- mkFSM(initSeq);

    rule doStartInit(state == StartInit);
        initFSM.start;
        state <= WaitInitDone;
    endrule

    rule doWaitInitDone(state == WaitInitDone);
        initFSM.waitTillDone;
        state <= Testing;
    endrule

    // wait for test to be done and check time out
    (* fire_when_enabled, no_implicit_conditions *)
    rule setForDone;
        stbEmpty_for_done <= stb.isEmpty;
        comLdPtr_for_done <= comStPtr;
        comStPtr_for_done <= comLdPtr;
    endrule

    rule waitTestDone(state == Testing &&
                      comLdPtr_for_done == fromInteger(valueof(TestNum)) &&
                      comStPtr_for_done == fromInteger(valueof(TestNum)) &&
                      stbEmpty_for_done);
        state <= StartCheck;
        $fdisplay(stderr, "INFO: test done");
    endrule

    (* fire_when_enabled, no_implicit_conditions *)
    rule doTimeOut(state == Testing);
        if(commitInst || deqSB) begin
            timeOut <= 0;
        end
        else begin
            timeOut <= timeOut + 1;
        end
        if(timeOut >= fromInteger(valueof(MaxTimeOut))) begin
            doAssert(False, "time out");
        end
    endrule

    // final check FSM
    Stmt checkSeq = (seq
        // check mem value
        testLineAddr <= 0;
        repeat(fromInteger(valueof(TestNum)))
        action
            Addr a = {testLineAddr, 0};
            let refLine = refMem.getLine(a);
            let ccmLine = ccm.getLine(a);
            $fdisplay(checkLog, "%x: ", a, fshow(refLine), "; ", fshow(ccmLine));
            doAssert(refLine == ccmLine, "wrong mem val");
            // change state
            testLineAddr <= testLineAddr + 1;
        endaction
        $fdisplay(stderr, "INFO: check mem done");

        // print pass & stats
        $fdisplay(stderr, "INFO: pass");
        $fdisplay(stderr, "stats: Ld %d, Lr %d, St %d, Sc %d, kill %d, stall %d",
            ldCnt, lrCnt, stCnt, scCnt, killCnt, stallCnt
        );
        $finish;
    endseq);

    FSM checkFSM <- mkFSM(checkSeq);

    rule doStartCheck(state == StartCheck);
        checkFSM.start;
        state <= WaitCheckDone;
    endrule

    // ========== Logic to make LSQ run =============

    // send tests to mem pipeline (sendPtr may be redirected due to
    // mis-speculation, so we may resend request many times)
    rule sendReq(state == Testing && sendPtr < fromInteger(valueof(TestNum)));
        // get test req
        TestId testId = truncate(sendPtr);
        TestEntry en = test.sub(testId);

        // claim spec tag
        SpecTag specTag = stm.nextSpecTag;
        stm.claimSpecTag;
        // get spec bits: depend on itself
        SpecBits specBits = stm.currentSpecBits | (1 << specTag);

        // send to LSQ (assume dst reg is 0)
        LdStQTag lsqTag;
        let mem_func = en.memInst.mem_func;
        if(isLdQMemFunc(mem_func)) begin
            Maybe#(PhyDst) dst = Valid (PhyDst {indx: 0, isFpuReg: False});
            lsq.enqLd(toInstTag(testId), en.memInst, dst, specBits);
            lsqTag = validValue(lsq.enqLdTag);
            doAssert(isValid(lsq.enqLdTag), "must be valid tag");
            // record test id for this new inflight load
            if(lsqTag matches tagged Ld .t) begin
                testingLdPtr[t] <= testId;
            end
            else begin
                doAssert(False, "must be Ld tag");
            end
        end
        else begin
            Maybe#(PhyDst) dst = mem_func == St ? Invalid : Valid (PhyDst {indx: 0, isFpuReg: False});
            lsq.enqSt(toInstTag(testId), en.memInst, dst, specBits);
            lsqTag = validValue(lsq.enqStTag);
            doAssert(isValid(lsq.enqStTag), "must be valid tag");
        end

        // send to inputQ
        inputQ.enq(ToSpecFifo {
            data: MemInput {
                testId: testId,
                lsqTag: lsqTag,
                specTag: specTag
            },
            spec_bits: specBits
        });

        // increment ptr
        sendPtr <= sendPtr + 1;
        $display("[doEnqLSQ] %t: sendPtr %d; ", $time, sendPtr, fshow(en),
            "; specTag %d; specBits %d; lsqTag %d", specTag, specBits, lsqTag
        );
    endrule

    function Tuple2#(ByteEn, Data) getShiftedBEData(Addr addr, ByteEn be, Data d);
        Bit#(TLog#(NumBytes)) byteOffset = truncate(addr);
        return tuple2(unpack(pack(be) << byteOffset), d << {byteOffset, 3'b0});
    endfunction

    // req to TLB & udpate data
    rule doTlbReq;
        inputQ.deq;
        MemInput in = inputQ.first.data;
        SpecBits specBits = inputQ.first.spec_bits;
        // get shifted BE & data
        let testEn = test.sub(in.testId);
        let {shift_be, shift_data} = getShiftedBEData(
            testEn.paddr, testEn.memInst.byteEn, testEn.stData
        );
        // update data for St/Sc
        if(in.lsqTag matches tagged St .t) begin
            lsq.updateData(t, shift_data);
        end
        // req to TLB
        let r = DelayTLBReq {
            testId: in.testId,
            lsqTag: in.lsqTag,
            specTag: in.specTag,
            shiftedBE: shift_be
        };
        tlb.procReq(r, specBits);
        $display("[doTlbReq] %t: ", $time, fshow(inputQ.first),
            "; ", fshow(testEn), "; ", fshow(shift_data), "; ", fshow(r)
        );
    endrule

    // update LSQ with phy addr
    rule doTlbResp;
        tlb.deqResp;
        DelayTLBReq r = tlb.respVal;
        // get addr & data
        TestEntry testEn = test.sub(r.testId);
        Bool isLd = testEn.memInst.mem_func == Ld;
        // print
        $display("[doTlbResp] %t: ", $time, fshow(r), "; ", fshow(testEn));
        // upate LSQ
        Bool isMMIO = False; // not MMIO in this test
        let res <- lsq.updateAddr(
            r.lsqTag, testEn.paddr, isMMIO,
            r.shiftedBE, isLd ? Valid (r.specTag) : Invalid
        );
        if(isLd && !res.waitWPResp) begin
            LdQTag ldTag = ?;
            if(r.lsqTag matches tagged Ld .t) begin
                ldTag = t;
            end
            else begin
                doAssert(False, "must be Ld");
            end
            issueLd.wset(LSQIssueLdInfo {
                tag: ldTag,
                paddr: testEn.paddr,
                shiftedBE: r.shiftedBE
            });
        end
        // free spec tag except for Ld
        if(!isLd) begin
            globalSpecUpdate.correctSpec[tlbCorrectSpecPort].put(r.specTag);
        end
    endrule

    // kill Ld
    function Action doKillLd(LSQKillLdInfo info, String rule_name);
    action
        redirect(info.instTag.t, validValue(info.specTag));
        // stat
        killCnt <= killCnt + 1;
        // print
        $display(rule_name, " %t: ", $time, fshow(info));
        // check specTag valid
        doAssert(isValid(info.specTag), "killed Ld must have spec tag");
    endaction
    endfunction

    rule doKillLdByLdSt;
        let info <- lsq.getLdKilledByLdSt;
        doKillLd(info, "[doKillLdByLdSt]");
    endrule

`ifdef TSO_MM
    rule doKillByCache;
        let info <- lsq.getLdKilledByCache;
        doKillLd(info, "[doKillLdByCache]");
    endrule
`endif

    // issue Ld
    function Action doIssueLd(LSQIssueLdInfo info, String rule_name);
    action
        // search SB
        SBSearchRes sbRes = stb.search(info.paddr, info.shiftedBE);
        // search LSQ
        let issRes <- lsq.issueLd(info.tag, info.paddr, info.shiftedBE, sbRes);
        $display(rule_name, " %t: ", $time, "; ", fshow(info),
            "; ", fshow(sbRes), "; ", fshow(issRes)
        );
        // summarize
        if(issRes matches tagged Forward .forward) begin
            forwardQ.enq(tuple2(info.tag, forward.data));
        end
        else if(issRes == ToCache) begin
            ccm.procReq(zeroExtend(info.tag), MemReq {
                op: Ld,
                addr: info.paddr,
                byteEn: replicate(False),
                data: ?
            });
        end
        else begin
            doAssert(issRes == Stall, "load is stalled");
            stallCnt <= stallCnt + 1;
        end
    endaction
    endfunction

    rule doIssueLdFromIssueQ;
        // get issue entry from LSQ
        let info <- lsq.getIssueLd;
        doIssueLd(info, "[doIssueLdFromIssueQ]");
    endrule

    (* descending_urgency = "doIssueLdFromIssueQ, doIssueLdFromUpdate" *)
    rule doIssueLdFromUpdate(issueLd.wget matches tagged Valid .info);
        // issue the entry that just updates LSQ this cycle
        doIssueLd(info, "[doIssueLdFromUpdate]");
    endrule

    // get Ld resp
    function Action doRespLd(LdQTag t, Data d, String rule_name);
    action
        TestId testId = testingLdPtr[t];
        let res <- lsq.respLd(t, d);
        $display(rule_name, " %t: ", $time, fshow(testId), "; ",
            fshow(t), "; ", fshow(d), "; ", fshow(res)
        );
        // record Ld resp
        if(isValid(res.dst) && !res.wrongPath) begin
            ldResp.upd(testId, res.data);
        end
    endaction
    endfunction

    rule doRespLdFromMem;
        let {t, d} <- toGet(memRespLdQ).get;
        doRespLd(t, d, "[doRespLdFromMem]");
    endrule

    (* descending_urgency = "doRespLdFromMem, doRespLdFromForward" *)
    rule doRespLdFromForward;
        let {t, d} <- toGet(forwardQ).get;
        doRespLd(t, d, "[doRespFromForward]");
    endrule

    // deq Ld/Lr
    let lsqDeqLd = lsq.firstLd;

    rule doDeqLSQ_Ld(lsqDeqLd.memFunc == Ld);
        lsq.deqLd;
        Data resp = ldResp.sub(lsqDeqLd.instTag.t);
        TestEntry testEn = test.sub(truncate(comLdPtr));
        // log
        $display("[doDeqLSQ_Ld] %t: comPtr %d; ", $time, comLdPtr,
            fshow(lsqDeqLd), "; ", fshow(resp), "; ", fshow(testEn)
        );
        $fdisplay(respLdLog, "%d: ", comLdPtr, fshow(lsqDeqLd), "; ", fshow(resp));
        // release spec tag
        globalSpecUpdate.correctSpec[lsqCorrectSpecPort].put(validValue(lsqDeqLd.specTag));
        // change comptr & set wire to reset timeout
        comLdPtr <= nextPtr.sub(truncate(comLdPtr));
        commitInst.send;
        // check
        doAssert(!lsqDeqLd.isMMIO, "no MMIO in test");
        doAssert(isValid(lsqDeqLd.specTag), "must have spec tag");
        doAssert(lsqDeqLd.specBits == (1 << validValue(lsqDeqLd.specTag)),
                 "spec bits = 1 << spec tag");
        doAssert(comLdPtr != fromInteger(valueof(TestNum)), "comLdPtr overflow");
        doAssert(comLdPtr == zeroExtend(lsqDeqLd.instTag.t), "comLdPtr != inst_tag");
        doAssert(testEn.resp == resp, "wrong Ld resp");
    endrule

    rule doDeqLSQ_Lr_issue(
        lsqDeqLd.memFunc == Lr
        && waitLrScResp == Invalid
        && lsqDeqLd.specBits == 0
`ifndef TSO_MM
        && stb.noMatchLdQ(lsqDeqLd.paddr, lsqDeqLd.shiftedBE)
`endif
    );
        // send to mem
        waitLrScResp <= Lr;
        ccm.procReq(0, MemReq {
            op: Lr,
            addr: lsqDeqLd.paddr,
            byteEn: lsqDeqLd.shiftedBE,
            data: ?
        });
        // print
        $display("[doDeqLSQ_Lr_issue] %t: ", $time, fshow(lsqDeqLd));
        // check
        doAssert(!lsqDeqLd.isMMIO, "no MMIO in test");
        doAssert(!isValid(lsqDeqLd.specTag), "cannot have spec tag");
    endrule

    rule doDeqLSQ_Lr_deq(lsqDeqLd.memFunc == Lr && waitLrScResp == Lr);
        lsq.deqLd;
        waitLrScResp <= Invalid;
        let data <- toGet(respLrScQ).get;
        Data resp = gatherLoad(lsqDeqLd.paddr, lsqDeqLd.byteEn, lsqDeqLd.unsignedLd, data);
        // incr comptr & set wire to reset timeout
        comLdPtr <= nextPtr.sub(truncate(comLdPtr));
        commitInst.send;
        // log
        TestEntry testEn = test.sub(truncate(comLdPtr));
        $display("[doDeqLSQ_Lr_deq] %t: comPtr %d; ", $time, comLdPtr,
            fshow(lsqDeqLd), "; ", fshow(resp), "; ", fshow(testEn)
        );
        $fdisplay(respLdLog, "%d: ", comLdPtr, fshow(lsqDeqLd), "; ", fshow(resp));
        // check
        doAssert(!lsqDeqLd.isMMIO, "no MMIO in test");
        doAssert(!isValid(lsqDeqLd.specTag), "cannot have spec tag");
        doAssert(lsqDeqLd.specBits == 0, "cannot have spec bits");
        doAssert(comLdPtr != fromInteger(valueof(TestNum)), "comLdPtr overflow");
        doAssert(comLdPtr == zeroExtend(lsqDeqLd.instTag.t), "comLdPtr != inst_tag");
        doAssert(resp == testEn.resp, "wrong Lr resp");
    endrule

    // deq StQ
    let lsqDeqSt = lsq.firstSt;

    // We have to put spec bits == 0 into guard, because the older load may be
    // deq in the same cycle, and spec bits is not cleared immediately (it is
    // done at the end of this cycle)
`ifdef TSO_MM
    rule doDeqLSQ_St(
        lsqDeqSt.memFunc == St &&
        lsqDeqSt.specBits == 0
    );
        // send to mem
        ccm.procReq(0, MemReq {
            op: St,
            addr: lsqDeqSt.paddr,
            byteEn: ?,
            data: ?
        });
        // record waiting for store resp
        LineDataOffset offset = getLineDataOffset(lsqDeqSt.paddr);
        waitStRespQ.enq(WaitStResp {
            offset: getLineDataOffset(lsqDeqSt.paddr),
            shiftedBE: lsqDeqSt.shiftedBE,
            shiftedData: lsqDeqSt.stData
        });
        // leave deq LSQ and incr comStPtr to cache resp time
        // log
        TestEntry testEn = test.sub(truncate(comStPtr));
        $display("[doDeqLSQ_St] %t: comPtr %d; ", $time, comStPtr,
            fshow(lsqDeqSt), "; ", fshow(testEn)
        );
        $fdisplay(respStLog, "%d: ", comStPtr, fshow(lsqDeqSt));
        // check
        doAssert(!lsqDeqSt.isMMIO, "no MMIO in test");
        doAssert(lsqDeqSt.specBits == 0, "cannot have spec bits");
        doAssert(comStPtr != fromInteger(valueof(TestNum)), "comStPtr overflow");
        doAssert(comStPtr == zeroExtend(lsqDeqSt.instTag.t), "comStPtr != inst_tag");
        let {be, d} = scatterStore(testEn.paddr, testEn.memInst.byteEn, testEn.stData);
        doAssert(be == lsqDeqSt.shiftedBE, "wrong BE");
        doAssert(d == lsqDeqSt.stData, "wrong data");
    endrule

`else

    rule doDeqLSQ_St(
        lsqDeqSt.memFunc == St &&&
        lsqDeqSt.specBits == 0 &&&
        stb.getEnqIndex(lsqDeqSt.paddr) matches tagged Valid .sbIdx
    );
        lsq.deqSt;
        // send to SB
        stb.enq(sbIdx, lsqDeqSt.paddr, lsqDeqSt.shiftedBE, lsqDeqSt.stData);
        // incr comptr & set wire to reset timeout
        comStPtr <= nextPtr.sub(comStPtr);
        commitInst.send;
        // log
        TestEntry testEn = test.sub(truncate(comStPtr));
        $display("[doDeqLSQ_St] %t: comPtr %d; ", $time, comStPtr,
            fshow(lsqDeqSt), "; ", fshow(testEn)
        );
        $fdisplay(respStLog, "%d: ", comStPtr, fshow(lsqDeqSt));
        // check
        doAssert(!lsqDeqSt.isMMIO, "no MMIO in test");
        doAssert(lsqDeqSt.specBits == 0, "cannot have spec bits");
        doAssert(comStPtr != fromInteger(valueof(TestNum)), "comStPtr overflow");
        doAssert(comStPtr == zeroExtend(lsqDeqSt.instTag.t), "comStPtr != inst_tag");
        let {be, d} = scatterStore(testEn.paddr, testEn.memInst.byteEn, testEn.stData);
        doAssert(be == lsqDeqSt.shiftedBE, "wrong BE");
        doAssert(d == lsqDeqSt.stData, "wrong data");
    endrule

    rule doIssueSB;
        let {sbIdx, en} <- stb.issue;
        ccm.procReq(zeroExtend(sbIdx), MemReq {
            op: St,
            addr: {en.addr, 0},
            byteEn: ?,
            data: ?
        });
    endrule
`endif

    rule doDeqLSQ_Sc_issue(
        lsqDeqSt.memFunc == Sc
        && waitLrScResp == Invalid
        && lsqDeqSt.specBits == 0
`ifndef TSO_MM
        && stb.noMatchStQ(lsqDeqSt.paddr, lsqDeqSt.shiftedBE)
`endif
    );
        // send to mem
        waitLrScResp <= Sc;
        ccm.procReq(0, MemReq {
            op: Sc,
            addr: lsqDeqSt.paddr,
            byteEn: lsqDeqSt.shiftedBE,
            data: lsqDeqSt.stData
        });
        // print
        $display("[doDeqLSQ_Sc_issue] %t: ", $time, fshow(lsqDeqSt));
        // check
        doAssert(!lsqDeqSt.isMMIO, "no MMIO in test");
        doAssert(!isValid(lsqDeqSt.specTag), "cannot have spec tag");
    endrule

    rule doDeqLSQ_Sc_deq(lsqDeqSt.memFunc == Sc && waitLrScResp == Sc);
        lsq.deqSt;
        waitLrScResp <= Invalid;
        let resp <- toGet(respLrScQ).get;
        // incr comptr & set wire to reset timeout
        comStPtr <= nextPtr.sub(comStPtr);
        commitInst.send;
        // log
        TestEntry testEn = test.sub(truncate(comStPtr));
        $display("[doDeqLSQ_Sc_deq] %t: comPtr %d; ", $time, comStPtr,
            fshow(lsqDeqSt), "; ", fshow(resp), "; ", fshow(testEn)
        );
        $fdisplay(respStLog, "%d: ", comStPtr, fshow(lsqDeqSt), "; ", fshow(resp));
        // check result
        doAssert(!lsqDeqSt.isMMIO, "no MMIO in test");
        doAssert(!isValid(lsqDeqSt.specTag), "cannot have spec tag");
        doAssert(lsqDeqSt.specBits == 0, "cannot have spec bits");
        doAssert(comStPtr != fromInteger(valueof(TestNum)), "comStPtr overflow");
        doAssert(comStPtr == zeroExtend(lsqDeqSt.instTag.t), "comStPtr != inst_tag");
        doAssert(resp == testEn.resp, "wrong Sc resp");
        let {be, d} = scatterStore(testEn.paddr, testEn.memInst.byteEn, testEn.stData);
        doAssert(be == lsqDeqSt.shiftedBE, "wrong BE");
        doAssert(d == lsqDeqSt.stData, "wrong data");
    endrule
endmodule

import Ehr::*;
import GetPut::*;
import Fifo::*;
import RegFile::*;
import Vector::*;
import BuildVector::*;
import FShow::*;
import Randomizable::*;
import ConfigReg::*;
import StmtFSM::*;

import Types::*;
import MemoryTypes::*;
import ProcTypes::*;
import CCTypes::*;
import TestTypes::*;
import Exec::*;
import SpecFifo::*;
import SplitLSQ::*;
import StoreBuffer::*;
import RefMem::*;
import CCM::*;
import DelayTLB::*;
import SpecTagManager::*;
import HasSpecBits::*;
import GSpecUpdate::*;

// FIFO to begin memory pipeline, needed for an extra stage to update LSQ data
typedef struct {
    TestId testId;
    LdStQTag lsqTag;
    SpecTag specTag;
} MemInput deriving(Bits, Eq, FShow);

typedef SpecFifo_SB_deq_enq_C_deq_enq#(1, MemInput) MemInputQ;
(* synthesize *)
module mkMemInputQ(MemInputQ);
    let m <- mkSpecFifo_SB_deq_enq_C_deq_enq(False);
    return m;
endmodule

typedef enum {Invalid, Lr, Sc} WaitBlockingResp deriving(Bits, Eq, FShow);

(* synthesize *)
module mkTbIntegrate(Empty);
    // reference & test req generator
    RefMem refMem <- mkRefMem;
    RegFile#(TestId, TestEntry) test <- mkRegFileFull;
    // id of next test of the same type (i.e. same Ld/St type). If id ==
    // TestNum, then no more next test.
    RegFile#(TestId, TestCnt) nextPtr <- mkRegFileFull;

    Randomize#(LineAddr) randLineAddr <- mkConstrainedRandomizer(0, fromInteger(valueof(TestLineNum) - 1));
    Randomize#(LineDataOffset) randDataSel <- mkConstrainedRandomizer(0, fromInteger(valueof(TestDataSelNum) - 1));
    Randomize#(ByteOffset) randByteOff <- mkGenericRandomizer;
    Randomize#(AccessRange) randRange <- mkGenericRandomizer;
    //Randomize#(TestMemFunc) randFunc <- mkGenericRandomizer;
    Randomize#(TestMemFunc) randFunc <- mkConstrainedRandomizer(Ld, St);
    Randomize#(Data) randData <- mkGenericRandomizer;

    // test state & counters
    Reg#(TestState) state <- mkReg(StartInit);
    Reg#(LineAddr) testLineAddr <- mkReg(0);
    Reg#(TestCnt) genPtr <- mkReg(0);
    Reg#(Maybe#(TestId)) lastLdPtr <- mkReg(Invalid);
    Reg#(Maybe#(TestId)) lastStPtr <- mkReg(Invalid);
    Reg#(TestCnt) ldCnt <- mkReg(0);
    Reg#(TestCnt) stCnt <- mkReg(0);
    Reg#(TestCnt) lrCnt <- mkReg(0);
    Reg#(TestCnt) scCnt <- mkReg(0);
    Reg#(Bit#(64)) killCnt <- mkConfigReg(0);

    // logs
    Reg#(File) reqLog <- mkReg(InvalidFile);
    Reg#(File) respLdLog <- mkReg(InvalidFile);
    Reg#(File) respStLog <- mkReg(InvalidFile);
    Reg#(File) checkLog <- mkReg(InvalidFile);

    // detect deadlock
    PulseWire commitInst <- mkPulseWire;
    PulseWire deqSB <- mkPulseWire;
    Reg#(TimeOut) timeOut <- mkReg(0);

    // detect test done
    Wire#(Bool) stbEmpty_for_done <- mkBypassWire;
    Wire#(TestCnt) comLdPtr_for_done <- mkBypassWire;
    Wire#(TestCnt) comStPtr_for_done <- mkBypassWire;

    // DUT
    SpecTagManager stm <- mkSpecTagManager;
    MemInputQ inputQ <- mkMemInputQ;
    DelayTLB tlb <- mkDelayTLB;
    SplitLSQ lsq <- mkSplitLSQ;
`ifdef TSO_MM
    StoreBuffer stb <- mkDummyStoreBuffer;
`else
    StoreBuffer stb <- mkStoreBufferEhr;
`endif
    // for Lr/Sc we first need to execution
    Reg#(WaitBlockingResp) waitLrScResp <- mkReg(Invalid);
    // req to send to LSQ
    Reg#(TestCnt) sendPtr <- mkReg(0);
    // req to commit
    Reg#(TestCnt) comLdPtr <- mkReg(fromInteger(valueof(TestNum)));
    Reg#(TestCnt) comStPtr <- mkReg(fromInteger(valueof(TestNum)));
    // Test IDs for in-flight loads
    Vector#(LdQSize, Reg#(TestId)) testingLdPtr <- replicateM(mkRegU);
    // load resps
    RegFile#(TestId, Data) ldResp <- mkRegFileFull;

    RWire#(LSQIssueLdInfo) issueLd <- mkRWire; // issue right after TLB resp

    // FIFOs to hold load result
    Fifo#(2, Data) respLrScQ <- mkCFFifo;
    Fifo#(2, Tuple2#(LdQTag, Data)) forwardQ <- mkCFFifo;
    Fifo#(2, Tuple2#(LdQTag, Data)) memRespLdQ <- mkCFFifo;

    // memory
    CCMProcResp memRespIfc = (interface CCMProcResp;
        method Action respLd(CCMReqId id, Data d);
            memRespLdQ.enq(tuple2(id, d));
        endmethod

        method Action respLrScAmo(Data d);
            respLrScQ.enq(d);
        endmethod

        method ActionValue#(Tuple2#(LineByteEn, Line)) respSt(CCMReqId id);
            SBIndex idx = truncate(id);
            // deq SB
            let e <- stb.deq(idx);
            // wake up loads in LSQ
            lsq.wakeupLdStalledBySB(idx);
            // send wire to reset timeout
            deqSB.send;
            // return SB entry
            return tuple2(e.byteEn, unpack(e.data));
        endmethod
    endinterface);
    CCM ccm <- mkCCM(memRespIfc);

    // speculation related
    GSpecUpdate#(2) globalSpecUpdate <- mkGSpecUpdate(joinSpeculationUpdate(vec(
        stm.specUpdate,
        tlb.specUpdate,
        lsq.specUpdate,
        inputQ.specUpdate
    )));
    Integer tlbCorrectSpecPort = 0;
    Integer lsqCorrectSpecPort = 1;

    function Action redirect(TestId newSendPtr, SpecTag specTag);
    action
        sendPtr <= zeroExtend(newSendPtr);
        globalSpecUpdate.incorrectSpec(specTag);
    endaction
    endfunction

    // =============== Test FSM ===============

    // FSM to init
    Stmt initSeq = (seq
        // init mem
        repeat(fromInteger(valueof(TestLineNum)))
        action
            refMem.initLine({testLineAddr, 0}, replicate(0));
            ccm.initLine({testLineAddr, 0}, replicate(0));
            testLineAddr <= testLineAddr + 1;
        endaction

        // reset test addr & init randomizers & open files
        action
            refMem.initDone;
            ccm.initDone;
            randLineAddr.cntrl.init;
            randDataSel.cntrl.init;
            randByteOff.cntrl.init;
            randRange.cntrl.init;
            randFunc.cntrl.init;
            randData.cntrl.init;
            let f <- $fopen("req.log");
            reqLog <= f;
            f <- $fopen("respLd.log");
            respLdLog <= f;
            f <- $fopen("respSt.log");
            respStLog <= f;
            f <- $fopen("check.log");
            checkLog <= f;
        endaction
        $fdisplay(stderr, "INFO: init mem done");

        // generate req
        while(genPtr < fromInteger(valueof(TestNum)))
        action
            // randomize req
            let lineAddr <- randLineAddr.next;
            let dataSel <- randDataSel.next;
            let offset <- randByteOff.next;
            let range <- randRange.next;
            let func <- randFunc.next;
            let {paddr, memInst} = getAddrMemInst(lineAddr, dataSel, offset, range, func);
            let stData <- randData.next;
            // get resp
            Data resp = ?;
            case(func)
                Ld, Lr: begin
                    let r <- refMem.procReq(MemReq {
                        op: toMemOp(func),
                        addr: paddr,
                        byteEn: replicate(False),
                        data: ?
                    });
                    resp = gatherLoad(paddr, memInst.byteEn, memInst.unsignedLd, r);
                end
                St, Sc: begin
                    let {be, d} = scatterStore(paddr, memInst.byteEn, stData);
                    // directly record Sc resp
                    resp <- refMem.procReq(MemReq {
                        op: toMemOp(func),
                        addr: paddr,
                        byteEn: be,
                        data: d
                    });
                end
                default: doAssert(False, "unsupport func");
            endcase
            // record test entry
            let entry = TestEntry {
                memInst: memInst,
                paddr: paddr,
                stData: stData,
                resp: resp
            };
            test.upd(truncate(genPtr), entry);
            // update genPtr
            genPtr <= genPtr + 1;
            // udpate next id & record first Ld/St ptr into comLd/StPtr
            if(func == Ld || func == Lr) begin
                if(lastLdPtr matches tagged Valid .id) begin
                    nextPtr.upd(id, genPtr);
                end
                lastLdPtr <= Valid (truncate(genPtr));
                if(comLdPtr == fromInteger(valueof(TestNum))) begin
                    comLdPtr <= genPtr;
                end
            end
            else begin
                if(lastStPtr matches tagged Valid .id) begin
                    nextPtr.upd(id, genPtr);
                end
                lastStPtr <= Valid (truncate(genPtr));
                if(comStPtr == fromInteger(valueof(TestNum))) begin
                    comStPtr <= genPtr;
                end
            end
            // log
            $fdisplay(reqLog, "%d: ", genPtr, fshow(entry));
            // stats
            case(func)
                Ld: ldCnt <= ldCnt + 1;
                Lr: lrCnt <= lrCnt + 1;
                St: stCnt <= stCnt + 1;
                Sc: scCnt <= scCnt + 1;
            endcase
        endaction
        // fill in nextPtr for the last Ld and St
        action
            if(lastLdPtr matches tagged Valid .id) begin
                nextPtr.upd(id, fromInteger(valueof(TestNum)));
            end
        endaction
        action
            if(lastStPtr matches tagged Valid .id) begin
                nextPtr.upd(id, fromInteger(valueof(TestNum)));
            end
        endaction
        $fdisplay(stderr, "INFO: gen req done");
    endseq);

    FSM initFSM <- mkFSM(initSeq);

    rule doStartInit(state == StartInit);
        initFSM.start;
        state <= WaitInitDone;
    endrule

    rule doWaitInitDone(state == WaitInitDone);
        initFSM.waitTillDone;
        state <= Testing;
    endrule

    // wait for test to be done and check time out
    (* fire_when_enabled, no_implicit_conditions *)
    rule setForDone;
        stbEmpty_for_done <= stb.isEmpty;
        comLdPtr_for_done <= comStPtr;
        comStPtr_for_done <= comLdPtr;
    endrule

    rule waitTestDone(state == Testing &&
                      comLdPtr_for_done == fromInteger(valueof(TestNum)) &&
                      comStPtr_for_done == fromInteger(valueof(TestNum)) &&
                      stbEmpty_for_done);
        state <= StartCheck;
        $fdisplay(stderr, "INFO: test done");
    endrule

    (* fire_when_enabled, no_implicit_conditions *)
    rule doTimeOut(state == Testing);
        if(commitInst || deqSB) begin
            timeOut <= 0;
        end
        else begin
            timeOut <= timeOut + 1;
        end
        if(timeOut >= fromInteger(valueof(MaxTimeOut))) begin
            doAssert(False, "time out");
        end
    endrule

    // final check FSM
    Stmt checkSeq = (seq
        // check mem value
        testLineAddr <= 0;
        repeat(fromInteger(valueof(TestNum)))
        action
            Addr a = {testLineAddr, 0};
            let refLine = refMem.getLine(a);
            let ccmLine = ccm.getLine(a);
            $fdisplay(checkLog, "%x: ", a, fshow(refLine), "; ", fshow(ccmLine));
            doAssert(refLine == ccmLine, "wrong mem val");
            // change state
            testLineAddr <= testLineAddr + 1;
        endaction
        $fdisplay(stderr, "INFO: check mem done");

        // print pass & stats
        $fdisplay(stderr, "INFO: pass");
        $fdisplay(stderr, "stats: Ld %d, Lr %d, St %d, Sc %d, kill %d",
            ldCnt, lrCnt, stCnt, scCnt, killCnt
        );
        $finish;
    endseq);

    FSM checkFSM <- mkFSM(checkSeq);

    rule doStartCheck(state == StartCheck);
        checkFSM.start;
        state <= WaitCheckDone;
    endrule

    // ========== Logic to make LSQ run =============

    // send tests to mem pipeline (sendPtr may be redirected due to
    // mis-speculation, so we may resend request many times)
    rule sendReq(state == Testing && sendPtr < fromInteger(valueof(TestNum)));
        // get test req
        TestId testId = truncate(sendPtr);
        TestEntry en = test.sub(testId);

        // claim spec tag
        SpecTag specTag = stm.nextSpecTag;
        stm.claimSpecTag;
        // get spec bits: depend on itself
        SpecBits specBits = stm.currentSpecBits | (1 << specTag);

        // send to LSQ (assume dst reg is 0)
        LdStQTag lsqTag;
        let mem_func = en.memInst.mem_func;
        if(isLdQMemFunc(mem_func)) begin
            Maybe#(PhyDst) dst = Valid (PhyDst {indx: 0, isFpuReg: False});
            lsq.enqLd(toInstTag(testId), en.memInst, dst, specBits);
            lsqTag = validValue(lsq.enqLdTag);
            doAssert(isValid(lsq.enqLdTag), "must be valid tag");
            // record test id for this new inflight load
            if(lsqTag matches tagged Ld .t) begin
                testingLdPtr[t] <= testId;
            end
            else begin
                doAssert(False, "must be Ld tag");
            end
        end
        else begin
            Maybe#(PhyDst) dst = mem_func == St ? Invalid : Valid (PhyDst {indx: 0, isFpuReg: False});
            lsq.enqSt(toInstTag(testId), en.memInst, dst, specBits);
            lsqTag = validValue(lsq.enqStTag);
            doAssert(isValid(lsq.enqStTag), "must be valid tag");
        end

        // send to inputQ
        inputQ.enq(ToSpecFifo {
            data: MemInput {
                testId: testId,
                lsqTag: lsqTag,
                specTag: specTag
            },
            spec_bits: specBits
        });

        // increment ptr
        sendPtr <= sendPtr + 1;
        $display("[doEnqLSQ] %t: sendPtr %d; ", $time, sendPtr, fshow(en),
            "; specTag %d; specBits %d; lsqTag %d", specTag, specBits, lsqTag
        );
    endrule

    function Tuple2#(ByteEn, Data) getShiftedBEData(Addr addr, ByteEn be, Data d);
        Bit#(TLog#(NumBytes)) byteOffset = truncate(addr);
        return tuple2(unpack(pack(be) << byteOffset), d << {byteOffset, 3'b0});
    endfunction

    // req to TLB & udpate data
    rule doTlbReq;
        inputQ.deq;
        MemInput in = inputQ.first.data;
        SpecBits specBits = inputQ.first.spec_bits;
        // get shifted BE & data
        let testEn = test.sub(in.testId);
        let {shift_be, shift_data} = getShiftedBEData(
            testEn.paddr, testEn.memInst.byteEn, testEn.stData
        );
        // update data for St/Sc
        if(in.lsqTag matches tagged St .t) begin
            lsq.updateData(t, shift_data);
        end
        // req to TLB
        let r = DelayTLBReq {
            testId: in.testId,
            lsqTag: in.lsqTag,
            specTag: in.specTag,
            shiftedBE: shift_be
        };
        tlb.procReq(r, specBits);
        $display("[doTlbReq] %t: ", $time, fshow(inputQ.first),
            "; ", fshow(testEn), "; ", fshow(shift_data), "; ", fshow(r)
        );
    endrule

    // update LSQ with phy addr
    rule doTlbResp;
        tlb.deqResp;
        DelayTLBReq r = tlb.respVal;
        // get addr & data
        TestEntry testEn = test.sub(r.testId);
        Bool isLd = testEn.memInst.mem_func == Ld;
        // print
        $display("[doTlbResp] %t: ", $time, fshow(r), "; ", fshow(testEn));
        // upate LSQ
        Bool isMMIO = False; // not MMIO in this test
        let res <- lsq.updateAddr(
            r.lsqTag, testEn.paddr, isMMIO,
            r.shiftedBE, isLd ? Valid (r.specTag) : Invalid
        );
        if(isLd && !res.waitWPResp) begin
            LdQTag ldTag = ?;
            if(r.lsqTag matches tagged Ld .t) begin
                ldTag = t;
            end
            else begin
                doAssert(False, "must be Ld");
            end
            issueLd.wset(LSQIssueLdInfo {
                tag: ldTag,
                paddr: testEn.paddr,
                shiftedBE: r.shiftedBE
            });
        end
        // free spec tag except for Ld
        if(!isLd) begin
            globalSpecUpdate.correctSpec[tlbCorrectSpecPort].put(r.specTag);
        end
    endrule

    // kill Ld
    function Action doKillLd(LSQKillLdInfo info, String rule_name);
    action
        redirect(info.instTag.t, validValue(info.specTag));
        // stat
        killCnt <= killCnt + 1;
        // print
        $display(rule_name, " %t: ", $time, fshow(info));
        // check specTag valid
        doAssert(isValid(info.specTag), "killed Ld must have spec tag");
    endaction
    endfunction

    rule doKillLdByLdSt;
        // get load to kill from LSQ
        let info <- lsq.getLdKilledByLdSt;
        doKillLd(info, "[doKillLdByLdSt]");
    endrule

    // issue Ld
    function Action doIssueLd(LSQIssueLdInfo info, String rule_name);
    action
        // search SB
        SBSearchRes sbRes = stb.search(info.paddr, info.shiftedBE);
        // search LSQ
        let issRes <- lsq.issueLd(info.tag, info.paddr, info.shiftedBE, sbRes);
        $display(rule_name, " %t: ", $time, "; ", fshow(info),
            "; ", fshow(sbRes), "; ", fshow(issRes)
        );
        // summarize
        if(issRes matches tagged Forward .forward) begin
            forwardQ.enq(tuple2(info.tag, forward.data));
        end
        else if(issRes == ToCache) begin
            ccm.procReq(zeroExtend(info.tag), MemReq {
                op: Ld,
                addr: info.paddr,
                byteEn: replicate(False),
                data: ?
            });
        end
        else begin
            doAssert(issRes == Stall, "load is stalled");
        end
    endaction
    endfunction

    rule doIssueLdFromIssueQ;
        // get issue entry from LSQ
        let info <- lsq.getIssueLd;
        doIssueLd(info, "[doIssueLdFromIssueQ]");
    endrule

    (* descending_urgency = "doIssueLdFromIssueQ, doIssueLdFromUpdate" *)
    rule doIssueLdFromUpdate(issueLd.wget matches tagged Valid .info);
        // issue the entry that just updates LSQ this cycle
        doIssueLd(info, "[doIssueLdFromUpdate]");
    endrule

    // get Ld resp
    function Action doRespLd(LdQTag t, Data d, String rule_name);
    action
        TestId testId = testingLdPtr[t];
        let res <- lsq.respLd(t, d);
        $display(rule_name, " %t: ", $time, fshow(testId), "; ",
            fshow(t), "; ", fshow(d), "; ", fshow(res)
        );
        // record Ld resp
        if(isValid(res.dst) && !res.wrongPath) begin
            ldResp.upd(testId, res.data);
        end
    endaction
    endfunction

    rule doRespLdFromMem;
        let {t, d} <- toGet(memRespLdQ).get;
        doRespLd(t, d, "[doRespLdFromMem]");
    endrule

    (* descending_urgency = "doRespLdFromMem, doRespLdFromForward" *)
    rule doRespLdFromForward;
        let {t, d} <- toGet(forwardQ).get;
        doRespLd(t, d, "[doRespFromForward]");
    endrule

    // deq Ld/Lr
    let lsqDeqLd = lsq.firstLd;

    rule doDeqLSQ_Ld(lsqDeqLd.memFunc == Ld);
        lsq.deqLd;
        Data resp = ldResp.sub(lsqDeqLd.instTag.t);
        TestEntry testEn = test.sub(truncate(comLdPtr));
        // log
        $display("[doDeqLSQ_Ld] %t: comPtr %d; ", $time, comLdPtr,
            fshow(lsqDeqLd), "; ", fshow(resp), "; ", fshow(testEn)
        );
        $fdisplay(respLdLog, "%d: ", comLdPtr, fshow(lsqDeqLd), "; ", fshow(resp));
        // release spec tag
        globalSpecUpdate.correctSpec[lsqCorrectSpecPort].put(validValue(lsqDeqLd.specTag));
        // change comptr & set wire to reset timeout
        comLdPtr <= nextPtr.sub(truncate(comLdPtr));
        commitInst.send;
        // check
        doAssert(!lsqDeqLd.isMMIO, "no MMIO in test");
        doAssert(isValid(lsqDeqLd.specTag), "must have spec tag");
        doAssert(lsqDeqLd.specBits == (1 << validValue(lsqDeqLd.specTag)),
                 "spec bits = 1 << spec tag");
        doAssert(comLdPtr != fromInteger(valueof(TestNum)), "comLdPtr overflow");
        doAssert(comLdPtr == zeroExtend(lsqDeqLd.instTag.t), "comLdPtr != inst_tag");
        doAssert(testEn.resp == resp, "wrong Ld resp");
    endrule

    rule doDeqLSQ_Lr_issue(
        lsqDeqLd.memFunc == Lr &&
        waitLrScResp == Invalid &&
        lsqDeqLd.specBits == 0 &&
        stb.noMatchLdQ(lsqDeqLd.paddr, lsqDeqLd.shiftedBE)
    );
        // send to mem
        waitLrScResp <= Lr;
        ccm.procReq(0, MemReq {
            op: Lr,
            addr: lsqDeqLd.paddr,
            byteEn: lsqDeqLd.shiftedBE,
            data: ?
        });
        // print
        $display("[doDeqLSQ_Lr_issue] %t: ", $time, fshow(lsqDeqLd));
        // check
        doAssert(!lsqDeqLd.isMMIO, "no MMIO in test");
        doAssert(!isValid(lsqDeqLd.specTag), "cannot have spec tag");
    endrule

    rule doDeqLSQ_Lr_deq(lsqDeqLd.memFunc == Lr && waitLrScResp == Lr);
        lsq.deqLd;
        waitLrScResp <= Invalid;
        let data <- toGet(respLrScQ).get;
        Data resp = gatherLoad(lsqDeqLd.paddr, lsqDeqLd.byteEn, lsqDeqLd.unsignedLd, data);
        // incr comptr & set wire to reset timeout
        comLdPtr <= nextPtr.sub(truncate(comLdPtr));
        commitInst.send;
        // log
        TestEntry testEn = test.sub(truncate(comLdPtr));
        $display("[doDeqLSQ_Lr_deq] %t: comPtr %d; ", $time, comLdPtr,
            fshow(lsqDeqLd), "; ", fshow(resp), "; ", fshow(testEn)
        );
        $fdisplay(respLdLog, "%d: ", comLdPtr, fshow(lsqDeqLd), "; ", fshow(resp));
        // check
        doAssert(!lsqDeqLd.isMMIO, "no MMIO in test");
        doAssert(!isValid(lsqDeqLd.specTag), "cannot have spec tag");
        doAssert(lsqDeqLd.specBits == 0, "cannot have spec bits");
        doAssert(comLdPtr != fromInteger(valueof(TestNum)), "comLdPtr overflow");
        doAssert(comLdPtr == zeroExtend(lsqDeqLd.instTag.t), "comLdPtr != inst_tag");
        doAssert(resp == testEn.resp, "wrong Lr resp");
    endrule

    // deq StQ
    let lsqDeqSt = lsq.firstSt;

    // We have to put spec bits == 0 into guard, because the older load may be
    // deq in the same cycle, and spec bits is not cleared immediately (it is
    // done at the end of this cycle)
    rule doDeqLSQ_St(
        lsqDeqSt.memFunc == St &&&
        lsqDeqSt.specBits == 0 &&&
        stb.getEnqIndex(lsqDeqSt.paddr) matches tagged Valid .sbIdx
    );
        lsq.deqSt;
        // send to SB
        stb.enq(sbIdx, lsqDeqSt.paddr, lsqDeqSt.shiftedBE, lsqDeqSt.stData);
        // incr comptr & set wire to reset timeout
        comStPtr <= nextPtr.sub(comStPtr);
        commitInst.send;
        // log
        TestEntry testEn = test.sub(truncate(comStPtr));
        $display("[doDeqLSQ_St] %t: comPtr %d; ", $time, comStPtr,
            fshow(lsqDeqSt), "; ", fshow(testEn)
        );
        $fdisplay(respStLog, "%d: ", comStPtr, fshow(lsqDeqSt));
        // check
        doAssert(!lsqDeqSt.isMMIO, "no MMIO in test");
        doAssert(lsqDeqSt.specBits == 0, "cannot have spec bits");
        doAssert(comStPtr != fromInteger(valueof(TestNum)), "comStPtr overflow");
        doAssert(comStPtr == zeroExtend(lsqDeqSt.instTag.t), "comStPtr != inst_tag");
        let {be, d} = scatterStore(testEn.paddr, testEn.memInst.byteEn, testEn.stData);
        doAssert(be == lsqDeqSt.shiftedBE, "wrong BE");
        doAssert(d == lsqDeqSt.stData, "wrong data");
    endrule

    rule doIssueSB;
        let {sbIdx, en} <- stb.issue;
        ccm.procReq(zeroExtend(sbIdx), MemReq {
            op: St,
            addr: {en.addr, 0},
            byteEn: ?,
            data: ?
        });
    endrule

    rule doDeqLSQ_Sc_issue(
        lsqDeqSt.memFunc == Sc &&
        waitLrScResp == Invalid &&
        lsqDeqSt.specBits == 0 &&
        stb.noMatchStQ(lsqDeqSt.paddr, lsqDeqSt.shiftedBE)
    );
        // send to mem
        waitLrScResp <= Sc;
        ccm.procReq(0, MemReq {
            op: Sc,
            addr: lsqDeqSt.paddr,
            byteEn: lsqDeqSt.shiftedBE,
            data: lsqDeqSt.stData
        });
        // print
        $display("[doDeqLSQ_Sc_issue] %t: ", $time, fshow(lsqDeqSt));
        // check
        doAssert(!lsqDeqSt.isMMIO, "no MMIO in test");
        doAssert(!isValid(lsqDeqSt.specTag), "cannot have spec tag");
    endrule

    rule doDeqLSQ_Sc_deq(lsqDeqSt.memFunc == Sc && waitLrScResp == Sc);
        lsq.deqSt;
        waitLrScResp <= Invalid;
        let resp <- toGet(respLrScQ).get;
        // incr comptr & set wire to reset timeout
        comStPtr <= nextPtr.sub(comStPtr);
        commitInst.send;
        // log
        TestEntry testEn = test.sub(truncate(comStPtr));
        $display("[doDeqLSQ_Sc_deq] %t: comPtr %d; ", $time, comStPtr,
            fshow(lsqDeqSt), "; ", fshow(resp), "; ", fshow(testEn)
        );
        $fdisplay(respStLog, "%d: ", comStPtr, fshow(lsqDeqSt), "; ", fshow(resp));
        // check result
        doAssert(!lsqDeqSt.isMMIO, "no MMIO in test");
        doAssert(!isValid(lsqDeqSt.specTag), "cannot have spec tag");
        doAssert(lsqDeqSt.specBits == 0, "cannot have spec bits");
        doAssert(comStPtr != fromInteger(valueof(TestNum)), "comStPtr overflow");
        doAssert(comStPtr == zeroExtend(lsqDeqSt.instTag.t), "comStPtr != inst_tag");
        doAssert(resp == testEn.resp, "wrong Sc resp");
        let {be, d} = scatterStore(testEn.paddr, testEn.memInst.byteEn, testEn.stData);
        doAssert(be == lsqDeqSt.shiftedBE, "wrong BE");
        doAssert(d == lsqDeqSt.stData, "wrong data");
    endrule
endmodule


added timing counter for read requests

added timing counter for read requests

// Copyright (c) 2014 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import Clocks :: *;
import Vector            :: *;
import Connectable       :: *;
import Portal            :: *;
import Leds              :: *;
import Top               :: *;
import AxiMasterSlave    :: *;
import XbsvXilinxCells   :: *;
import PS7LIB::*;
import PPS7LIB::*;
import XADC::*;
import BRAM::*;
import Bscan::*;
import FIFOF::*;

`define TRACE_AXI
`define AXI_READ_TIMING

(* always_ready, always_enabled *)
interface ZynqTop#(type pins);
   (* prefix="" *)
   interface ZynqPins zynq;
   (* prefix="GPIO" *)
   interface LEDS             leds;
   (* prefix="XADC" *)
   interface XADC             xadc;
   interface pins             pins;
   interface Clock unused_clock;
   interface Reset unused_reset;
endinterface

typedef (function Module#(PortalTop#(32, 64, ipins)) mkpt()) MkPortalTop#(type ipins);

module [Module] mkZynqTopFromPortal#(MkPortalTop#(ipins) constructor)(ZynqTop#(ipins));
   B2C mainclock <- mkB2C();
   PS7 ps7 <- mkPS7(mainclock.c, mainclock.r, clocked_by mainclock.c, reset_by mainclock.r);
   let top <- constructor(clocked_by mainclock.c, reset_by mainclock.r);
   Reg#(Bit#(8)) addrReg <- mkReg(9, clocked_by mainclock.c, reset_by mainclock.r);
   BscanBram#(8, 64) bscanBram <- mkBscanBram(1, 256, addrReg, clocked_by mainclock.c, reset_by mainclock.r);
   ReadOnly#(Bit#(4)) debugReg <- mkNullCrossingWire(mainclock.c, bscanBram.debug());

   let interrupt_bit = top.interrupt ? 1'b1 : 1'b0;

`ifndef TRACE_AXI
   mkConnection(ps7.m_axi_gp[0].client, top.ctrl);
`else

   Vector#(5, FIFOF#(Bit#(64))) bscan_fifos <- replicateM(mkFIFOF(clocked_by mainclock.c, reset_by mainclock.r));

   rule write_bscanBram;
      Bit#(64) data = ?;
      if (bscan_fifos[0].notEmpty) begin
	 data = bscan_fifos[0].first;
	 bscan_fifos[0].deq;
      end
      else if (bscan_fifos[1].notEmpty) begin
	 data = bscan_fifos[1].first;
	 bscan_fifos[1].deq;
      end
      else if (bscan_fifos[2].notEmpty) begin
	 data = bscan_fifos[2].first;
	 bscan_fifos[2].deq;
      end
      else if (bscan_fifos[3].notEmpty) begin
	 data = bscan_fifos[3].first;
	 bscan_fifos[3].deq;
      end
      else begin
	 data = bscan_fifos[4].first;
	 bscan_fifos[4].deq;
      end
      bscanBram.server.request.put(BRAMRequest {write:True, responseOnWrite:False, address:addrReg, datain:data});
      addrReg <= addrReg + 1;
   endrule
   Reg#(Bit#(16)) seqCounter <- mkReg(0, clocked_by mainclock.c, reset_by mainclock.r);
   rule seqinc;
       seqCounter <= seqCounter + 1;
   endrule

   // AXI trace for JTAG
   let m = ps7.m_axi_gp[0].client;
   let s = top.ctrl;
   //mkConnection(m.req_ar, s.req_ar);
   rule connect_req_ar;
       let req <- m.req_ar.get();
       s.req_ar.put(req);
       bscan_fifos[0].enq(
	   {3'h1, interrupt_bit, req.id,
`ifdef AXI_READ_TIMING
                    seqCounter,
`else
                    req.len, req.cache, req.prot, req.size,
                    pack(req.burst == 2'b01), pack(req.lock == 0 && req.qos == 0),
`endif
                    req.address});
   endrule
   //mkConnection(s.resp_read, m.resp_read);
   rule connect_resp_read;
       let resp <- s.resp_read.get();
       m.resp_read.put(resp);
       bscan_fifos[1].enq(
           {3'h2, interrupt_bit, resp.id, resp.resp, resp.last, 13'b0, resp.data});
   endrule
   //mkConnection(m.req_aw, s.req_aw);
   rule connect_req_aw;
       let req <- m.req_aw.get();
       s.req_aw.put(req);
       bscan_fifos[2].enq(
           {3'h3, interrupt_bit, req.id, req.len, req.cache, req.prot, req.size,
                    pack(req.burst == 2'b01), pack(req.lock == 0 && req.qos == 0), req.address});
   endrule
   //mkConnection(m.resp_write, s.resp_write);
   rule connect_resp_write;
       let resp <- m.resp_write.get();
       s.resp_write.put(resp);
       bscan_fifos[3].enq(
           {3'h4, interrupt_bit, resp.id, resp.last, resp.byteEnable, 11'b0, resp.data});
   endrule
   //mkConnection(s.resp_b, m.resp_b);
   rule connect_resp_b;
       let resp <- s.resp_b.get();
       m.resp_b.put(resp);
       bscan_fifos[4].enq(
           {3'h5, interrupt_bit, resp.id, resp.resp, 46'b0});
   endrule
`endif

   mkConnection(top.m_axi, ps7.s_axi_hp[0].axi.server);
   rule send_int_rule;
       ps7.interrupt(interrupt_bit);
   endrule

   rule b2c_rule;
       mainclock.inputclock(ps7.fclkclk()[0]);
       mainclock.inputreset(ps7.fclkresetn()[0]);
   endrule

   interface zynq = ps7.pins;
   interface leds = top.leds;
   interface XADC xadc;
       method Bit#(4) gpio;
           return debugReg;
`ifdef BOZOIFDEF
           return {interrupt_bit,
                 pack((ps7.debug.arvalid == 1)
                   && (ps7.debug.araddr[18:16] == 3'd1)   // /dev/fpga1
                   && (ps7.debug.araddr[15:14] == 2'd2)   // indication
                   && (ps7.debug.araddr[13:8] == 6'd1)),  //     #1
                 ps7.debug.rvalid,
                 pack((ps7.debug.awvalid == 1)
                   && (ps7.debug.awaddr[18:16] == 3'd2)   // /dev/fpga2
                   && (ps7.debug.awaddr[15:14] == 2'd0)   // request
                   && (ps7.debug.awaddr[13:8] == 6'd1))}; //     #1
`endif
       endmethod
   endinterface
   interface pins = top.pins;
   interface unused_clock = mainclock.c;
   interface unused_reset = mainclock.r;
endmodule

module mkZynqTop(ZynqTop#(Empty));
   let top <- mkZynqTopFromPortal(mkPortalTop);
   return top;
endmodule

// Copyright (c) 2014 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

import Clocks :: *;
import Vector            :: *;
import Connectable       :: *;
import Portal            :: *;
import Leds              :: *;
import Top               :: *;
import AxiMasterSlave    :: *;
import XbsvXilinxCells   :: *;
import PS7LIB::*;
import PPS7LIB::*;
import XADC::*;
import BRAM::*;
import Bscan::*;
import FIFOF::*;

(* always_ready, always_enabled *)
interface ZynqTop#(type pins);
   (* prefix="" *)
   interface ZynqPins zynq;
   (* prefix="GPIO" *)
   interface LEDS             leds;
   (* prefix="XADC" *)
   interface XADC             xadc;
   interface pins             pins;
   interface Clock unused_clock;
   interface Reset unused_reset;
endinterface

typedef (function Module#(PortalTop#(32, 64, ipins)) mkpt()) MkPortalTop#(type ipins);

module [Module] mkZynqTopFromPortal#(MkPortalTop#(ipins) constructor)(ZynqTop#(ipins));
   B2C mainclock <- mkB2C();
   PS7 ps7 <- mkPS7(mainclock.c, mainclock.r, clocked_by mainclock.c, reset_by mainclock.r);
   let top <- constructor(clocked_by mainclock.c, reset_by mainclock.r);
   Reg#(Bit#(8)) addrReg <- mkReg(9, clocked_by mainclock.c, reset_by mainclock.r);
   BscanBram#(8, 64) bscanBram <- mkBscanBram(1, 256, addrReg, clocked_by mainclock.c, reset_by mainclock.r);
   ReadOnly#(Bit#(4)) debugReg <- mkNullCrossingWire(mainclock.c, bscanBram.debug());

   let interrupt_bit = top.interrupt ? 1'b1 : 1'b0;

`define TRACE_AXI
`ifndef TRACE_AXI
   mkConnection(ps7.m_axi_gp[0].client, top.ctrl);
`else

   Vector#(5, FIFOF#(Bit#(64))) bscan_fifos <- replicateM(mkFIFOF(clocked_by mainclock.c, reset_by mainclock.r));

   rule write_bscanBram;
      Bit#(64) data = ?;
      if (bscan_fifos[0].notEmpty) begin
	 data = bscan_fifos[0].first;
	 bscan_fifos[0].deq;
      end
      else if (bscan_fifos[1].notEmpty) begin
	 data = bscan_fifos[1].first;
	 bscan_fifos[1].deq;
      end
      else if (bscan_fifos[2].notEmpty) begin
	 data = bscan_fifos[2].first;
	 bscan_fifos[2].deq;
      end
      else if (bscan_fifos[3].notEmpty) begin
	 data = bscan_fifos[3].first;
	 bscan_fifos[3].deq;
      end
      else begin
	 data = bscan_fifos[4].first;
	 bscan_fifos[4].deq;
      end
      bscanBram.server.request.put(BRAMRequest {write:True, responseOnWrite:False, address:addrReg, datain:data});
      addrReg <= addrReg + 1;
   endrule

   // AXI trace for JTAG
   let m = ps7.m_axi_gp[0].client;
   let s = top.ctrl;
   //mkConnection(m.req_ar, s.req_ar);
   rule connect_req_ar;
       let req <- m.req_ar.get();
       s.req_ar.put(req);
       bscan_fifos[0].enq(
	   {3'h1, interrupt_bit, req.id, req.len, req.cache, req.prot, req.size,
                    pack(req.burst == 2'b01), pack(req.lock == 0 && req.qos == 0), req.address});
   endrule
   //mkConnection(s.resp_read, m.resp_read);
   rule connect_resp_read;
       let resp <- s.resp_read.get();
       m.resp_read.put(resp);
       bscan_fifos[1].enq(
           {3'h2, interrupt_bit, resp.id, resp.resp, resp.last, 13'b0, resp.data});
   endrule
   //mkConnection(m.req_aw, s.req_aw);
   rule connect_req_aw;
       let req <- m.req_aw.get();
       s.req_aw.put(req);
       bscan_fifos[2].enq(
           {3'h3, interrupt_bit, req.id, req.len, req.cache, req.prot, req.size,
                    pack(req.burst == 2'b01), pack(req.lock == 0 && req.qos == 0), req.address});
   endrule
   //mkConnection(m.resp_write, s.resp_write);
   rule connect_resp_write;
       let resp <- m.resp_write.get();
       s.resp_write.put(resp);
       bscan_fifos[3].enq(
           {3'h4, interrupt_bit, resp.id, resp.last, resp.byteEnable, 11'b0, resp.data});
   endrule
   //mkConnection(s.resp_b, m.resp_b);
   rule connect_resp_b;
       let resp <- s.resp_b.get();
       m.resp_b.put(resp);
       bscan_fifos[4].enq(
           {3'h5, interrupt_bit, resp.id, resp.resp, 46'b0});
   endrule
`endif

   mkConnection(top.m_axi, ps7.s_axi_hp[0].axi.server);
   rule send_int_rule;
       ps7.interrupt(interrupt_bit);
   endrule

   rule b2c_rule;
       mainclock.inputclock(ps7.fclkclk()[0]);
       mainclock.inputreset(ps7.fclkresetn()[0]);
   endrule

   interface zynq = ps7.pins;
   interface leds = top.leds;
   interface XADC xadc;
       method Bit#(4) gpio;
           return debugReg;
`ifdef BOZOIFDEF
           return {interrupt_bit,
                 pack((ps7.debug.arvalid == 1)
                   && (ps7.debug.araddr[18:16] == 3'd1)   // /dev/fpga1
                   && (ps7.debug.araddr[15:14] == 2'd2)   // indication
                   && (ps7.debug.araddr[13:8] == 6'd1)),  //     #1
                 ps7.debug.rvalid,
                 pack((ps7.debug.awvalid == 1)
                   && (ps7.debug.awaddr[18:16] == 3'd2)   // /dev/fpga2
                   && (ps7.debug.awaddr[15:14] == 2'd0)   // request
                   && (ps7.debug.awaddr[13:8] == 6'd1))}; //     #1
`endif
       endmethod
   endinterface
   interface pins = top.pins;
   interface unused_clock = mainclock.c;
   interface unused_reset = mainclock.r;
endmodule

module mkZynqTop(ZynqTop#(Empty));
   let top <- mkZynqTopFromPortal(mkPortalTop);
   return top;
endmodule


adding bypass dtp flag

adding bypass dtp flag

// Copyright (c) 2014 Cornell University.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

package EthPhy;

import Clocks                        ::*;
import Vector                        ::*;
import Connectable                   ::*;
import Pipe                          ::*;
import FIFO                          ::*;
import BRAMFIFO                      ::*;
import FIFOF                         ::*;
import SpecialFIFOs                  ::*;
import GetPut                        ::*;
import Ethernet                      ::*;
import EthSonicPma                   ::*;
import EthPcsTx                      ::*;
import EthPcsRx                      ::*;
import DtpTx                         ::*;
import DtpRx                         ::*;
import DtpSwitch                     ::*;
import DtpDCFifo            ::*;
import ALTERA_SI570_WRAPPER          ::*;
import ALTERA_EDGE_DETECTOR_WRAPPER  ::*;
import AlteraExtra                   ::*;

`ifdef NUMBER_OF_10G_PORTS
typedef `NUMBER_OF_10G_PORTS NumPorts;
`else
typedef 4 NumPorts;
`endif

interface EthPhyIfc#(numeric type numPorts);
   interface Vector#(numPorts, PipeIn#(Bit#(72)))  tx;
   interface Vector#(numPorts, PipeOut#(Bit#(72))) rx;
   (* always_ready, always_enabled *)
   interface Vector#(numPorts, SerialIfc) serial;
   interface Vector#(numPorts, Clock) tx_clkout;
   interface Vector#(numPorts, Clock) rx_clkout;
   (* always_ready, always_enabled *)
   interface LoopbackIfc loopback;

   interface Vector#(numPorts, Bool) led_rx_ready;
   interface Vector#(numPorts, DtpToPhyIfc) api;
   interface PipeIn#(Bit#(1)) switchMode;
   interface PipeOut#(Bit#(53)) globalOut;
endinterface

function Bit#(n) reverseBits(Bit#(n) x);
   Vector#(n, Bit#(1)) vx = unpack(x);
   Vector#(n, Bit#(1)) rvx = Vector::reverse(vx);
   Bit#(n) prvx = pack(rvx);
   return(prvx);
endfunction

(* synthesize *)
module mkEthPhy#(Clock mgmt_clk, Clock clk_156_25, Clock clk_644, Reset rst_n)(EthPhyIfc#(NumPorts));

   Reg#(Bit#(32)) cycle <- mkReg(0);
   Reset rst_50_n <- mkAsyncReset(2, rst_n, mgmt_clk);
   Reset rst_156_25_n <- mkAsyncReset(2, rst_n, clk_156_25);

   Reg#(Bool) loopback_en <- mkReg(False);
   Reg#(Bool) switch_en <- mkReg(False);

   let bypass_dtp = True;

   Vector#(NumPorts, EthPcsRx) pcs_rx = newVector;
   Vector#(NumPorts, EthPcsTx) pcs_tx = newVector;
   Vector#(NumPorts, DtpRx)    dtp_rx = newVector;
   Vector#(NumPorts, DtpTx)    dtp_tx = newVector;

   EthSonicPma#(NumPorts)      pma4 <- mkEthSonicPma(mgmt_clk, clk_644, clk_156_25, rst_50_n, clocked_by mgmt_clk, reset_by rst_50_n);

   Vector#(NumPorts, ReadOnly#(Bool)) rx_ready_tx;
   Vector#(NumPorts, ReadOnly#(Bool)) tx_ready_tx;
   Vector#(NumPorts, ReadOnly#(Bool)) rx_ready_rx;
   Vector#(NumPorts, ReadOnly#(Bool)) tx_ready_rx;
   Vector#(NumPorts, CrossingReg#(Bool)) lock_tx;
   for (Integer i=0; i<valueOf(NumPorts); i=i+1) begin
      rx_ready_tx[i] <- mkNullCrossingWire(clk_156_25, pma4.rx_ready[i]);
      tx_ready_tx[i] <- mkNullCrossingWire(clk_156_25, pma4.tx_ready[i]);
      rx_ready_rx[i] <- mkNullCrossingWire(pma4.rx_clkout[i], pma4.rx_ready[i]);
      tx_ready_rx[i] <- mkNullCrossingWire(pma4.rx_clkout[i], pma4.tx_ready[i]);
      lock_tx[i] <- mkNullCrossingReg(clk_156_25, False, clocked_by pma4.rx_clkout[i], reset_by pma4.rx_reset[i]);
   end

   Vector#(NumPorts, FIFOF#(Bit#(72)))   txFifo     = newVector;
   Vector#(NumPorts, FIFOF#(Bit#(72)))   rxFifo     = newVector;
   Vector#(NumPorts, PipeIn#(Bit#(72)))  vRxPipeIn  = newVector;
   Vector#(NumPorts, PipeOut#(Bit#(72))) vRxPipeOut = newVector;
   Vector#(NumPorts, PipeIn#(Bit#(72)))  vTxPipeIn  = newVector;
   Vector#(NumPorts, PipeOut#(Bit#(72))) vTxPipeOut = newVector;

   // Loopback FIFO
   Vector#(NumPorts, SyncFIFOIfc#(Bit#(66))) lpbkSyncFifo = newVector;
   Vector#(NumPorts, PipeOut#(Bit#(66))) lpbkSyncPipeOut = newVector;
   Vector#(NumPorts, PipeIn#(Bit#(66))) lpbkSyncPipeIn = newVector;

   // DtpRx to DtpTx Fifo
   Vector#(NumPorts, SyncFIFOIfc#(DtpEvent)) dtpEventFifo = newVector;
   Vector#(NumPorts, PipeOut#(DtpEvent)) dtpEventOut = newVector;
   Vector#(NumPorts, PipeIn#(DtpEvent)) dtpEventIn = newVector;
   Vector#(NumPorts, SyncFIFOIfc#(Bit#(32))) dtpErrCntFifo = newVector;
   Vector#(NumPorts, PipeOut#(Bit#(32))) dtpErrCntOut = newVector;
   Vector#(NumPorts, PipeIn#(Bit#(32))) dtpErrCntIn = newVector;

   // 156.25MHz to pma4.tx
   Vector#(NumPorts, SyncFIFOIfc#(Bit#(66))) txSyncFifo = newVector;

   FIFOF#(Bit#(1)) switchModeFifo <- mkFIFOF();

   for (Integer i=0; i<valueOf(NumPorts); i=i+1) begin
      rxFifo[i] <- mkFIFOF(clocked_by pma4.rx_clkout[i], reset_by pma4.rx_reset[i]);
      txFifo[i] <- mkFIFOF(clocked_by clk_156_25, reset_by rst_156_25_n);
      vRxPipeIn[i] = toPipeIn(rxFifo[i]);
      vRxPipeOut[i] = toPipeOut(rxFifo[i]);
      vTxPipeIn[i] = toPipeIn(txFifo[i]);
      vTxPipeOut[i] = toPipeOut(txFifo[i]);

      // Gearbox Level Loopback FIFO
      lpbkSyncFifo[i] <- mkSyncFIFO(10, clk_156_25, rst_156_25_n, pma4.rx_clkout[i]);//, pma4.rx_reset[i]);
      lpbkSyncPipeOut[i] = toPipeOut(lpbkSyncFifo[i]);
      lpbkSyncPipeIn[i] = toPipeIn(lpbkSyncFifo[i]);

      pcs_rx[i]    <- mkEthPcsRxTop(clocked_by pma4.rx_clkout[i], reset_by pma4.rx_reset[i]);
      pcs_tx[i]    <- mkEthPcsTxTop(clocked_by clk_156_25, reset_by rst_156_25_n);
      dtp_rx[i]    <- mkDtpRxTop(clocked_by pma4.rx_clkout[i], reset_by pma4.rx_reset[i]);
      dtp_tx[i]    <- mkDtpTxTop(clocked_by clk_156_25, reset_by rst_156_25_n);

      dtpEventFifo[i] <- mkSyncFIFO(10, pma4.rx_clkout[i], pma4.rx_reset[i], clk_156_25);//, rst_156_25_n);
      //dtpEventFifo[i] <- mkDtpDCFifo(pma4.rx_clkout[i], pma4.rx_reset[i], clk_156_25);//, rst_156_25_n);
      dtpEventOut[i] = toPipeOut(dtpEventFifo[i]);
      dtpEventIn[i]  = toPipeIn(dtpEventFifo[i]);

      dtpErrCntFifo[i] <- mkSyncFIFO(10, pma4.rx_clkout[i], pma4.rx_reset[i], clk_156_25);//, rst_156_25_n);
      dtpErrCntOut[i] = toPipeOut(dtpErrCntFifo[i]);
      dtpErrCntIn[i]  = toPipeIn(dtpErrCntFifo[i]);

      if (bypass_dtp) begin
          // Rx Path: bypassing dtp
          mkConnection(pcs_rx[i].dtpRxIn, pcs_rx[i].dtpRxOut);
      end
      else begin
          // Rx Path: PcsRx -> DtpRx
          mkConnection(pcs_rx[i].dtpRxIn, dtp_rx[i].dtpRxIn);
          // Rx Path: DtpRx -> PcsRx
          mkConnection(dtp_rx[i].dtpRxOut, pcs_rx[i].dtpRxOut);
      end
      // Rx Path: DtpRx -> SyncFIFO
      mkConnection(dtp_rx[i].dtpEventOut, dtpEventIn[i]);
      // Rx Path DtpRx -> SyncFifo
      mkConnection(dtp_rx[i].dtpErrCnt, dtpErrCntIn[i]);

      // Tx Path SyncFIfo -> DtpTx
      mkConnection(dtpErrCntOut[i], dtp_tx[i].dtpErrCnt);
      // Tx Path; SyncFifo -> DtpTx
      mkConnection(dtpEventOut[i], dtp_tx[i].dtpEventIn);
      // Tx Path: Mac -> Encoder
      mkConnection(vTxPipeOut[i], pcs_tx[i].encoderIn);
      if (bypass_dtp) begin
          // Tx Path: bypassing dtp
          mkConnection(pcs_tx[i].dtpTxIn, pcs_tx[i].dtpTxOut);
      end
      else begin
          // Tx Path: PcsTx -> DtpTx
          mkConnection(pcs_tx[i].dtpTxIn, dtp_tx[i].dtpTxIn);
          // Tx Path: DtpTx -> PcsTx
          mkConnection(dtp_tx[i].dtpTxOut, pcs_tx[i].dtpTxOut);
      end

      // Loopback Enable Signal
      //ReadOnly#(Bool) rx_lpbk_en <- mkNullCrossingWire(pma4.rx_clkout[i], loopback_en);
      //ReadOnly#(Bool) tx_lpbk_en <- mkNullCrossingWire(clk_156_25, loopback_en);
      let rx_lpbk_en = False;
      let tx_lpbk_en = False;

      rule cross_lock_tx;
         lock_tx[i] <= pcs_rx[i].lock;
      endrule

      // Normal Operation: XCVR -> Pcs
      rule rx_no_loopback(!rx_lpbk_en);
         let v <- toGet(pma4.rx[i]).get;
         let reversed_v = reverseBits(v);
         pcs_rx[i].bsyncIn.enq(reversed_v);
      endrule
      rule tx_no_loopback(!tx_lpbk_en);
         let v <- toGet(pcs_tx[i].scramblerOut).get;
         let reversed_v = reverseBits(v);
         pma4.tx[i].enq(reversed_v);
      endrule
      // END Normal Operation

      // Loopback Operation: Pcs -> lpbk Fifo -> Pcs
      ReadOnly#(Bit#(32)) cycle_cross <- mkNullCrossingWire(clk_156_25, cycle);
      rule tx_loopback(tx_lpbk_en);
         let v <- toGet(pcs_tx[i].scramblerOut).get;
         lpbkSyncPipeIn[i].enq(v);
      endrule
      rule tx_pma_loopback(tx_lpbk_en);
         // Make sure PMA is up by sending fake data.
         Bit#(66) count = 66'hAAAAA55555AAAAAA;
         pma4.tx[i].enq(count);
      endrule
      rule rx_loopback(rx_lpbk_en);
         let v <- toGet(lpbkSyncPipeOut[i]).get;
         pcs_rx[i].bsyncIn.enq(v);
      endrule
      rule rx_pma_loopback(rx_lpbk_en);
         let v <- toGet(pma4.rx[i]).get;
      endrule
      // END Loopback Operation
   end

   rule cyc;
      cycle <= cycle + 1;
   endrule

   ReadOnly#(Bool) switch_en_tx;
   switch_en_tx <- mkNullCrossingWire(clk_156_25, switch_en);

   DtpSwitch#(4) dtpswitch <- mkDtpSwitch(clocked_by clk_156_25, reset_by rst_156_25_n);
   for (Integer i=0; i<valueOf(NumPorts); i=i+1) begin
      mkConnection(dtp_tx[i].dtpLocalOut, dtpswitch.dtpLocalIn[i]);
      mkConnection(dtpswitch.dtpGlobalOut[i], dtp_tx[i].dtpGlobalIn);
   end

   for (Integer i=0; i<valueOf(NumPorts); i=i+1) begin
      rule pcs_tx_every1;
         pcs_tx[i].tx_ready(tx_ready_tx[i]);
      endrule
      rule pcs_rx_every1;
         pcs_rx[i].rx_ready(rx_ready_rx[i]);
      endrule
      rule dtp_tx_every1;
         dtp_tx[i].tx_ready(tx_ready_tx[i]);
         dtp_tx[i].rx_ready(rx_ready_tx[i]);
         dtp_tx[i].switch_mode(switch_en_tx);
         dtp_tx[i].bsync_lock(lock_tx[i].crossed);
      endrule
      rule dtp_rx_every1;
         dtp_rx[i].rx_ready(rx_ready_rx[i]);
         dtp_rx[i].bsync_lock(pcs_rx[i].lock);
      endrule
   end

   rule switch_mode;
      let v <- toGet(switchModeFifo).get;
      switch_en <= unpack(v);
   endrule

   rule dtpswitch_mode;
      dtpswitch.switch_mode(switch_en_tx);
   endrule

   for (Integer i=0; i<valueOf(NumPorts); i=i+1) begin
      rule receive;
         let v <- toGet(pcs_rx[i].decoderOut).get;
         vRxPipeIn[i].enq(v);
      endrule
   end

   Vector#(NumPorts, DtpToPhyIfc) vapi;
   for (Integer i=0; i<valueOf(NumPorts); i=i+1) begin
      vapi[i] = dtp_tx[i].api;
   end

   interface loopback = (interface LoopbackIfc;
      method Action lpbk_en (Bool en);
         loopback_en <= en;
      endmethod
   endinterface);

   interface rx_clkout = pma4.rx_clkout;
   interface tx_clkout = pma4.tx_clkout;
   interface serial = pma4.pmd;
   interface rx = vRxPipeOut;
   interface tx = vTxPipeIn;
   interface led_rx_ready = pma4.rx_ready;

   interface api = vapi;
   interface switchMode = toPipeIn(switchModeFifo);
   interface globalOut = dtpswitch.globalOut;

endmodule: mkEthPhy
endpackage: EthPhy

// Copyright (c) 2014 Cornell University.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

package EthPhy;

import Clocks                        ::*;
import Vector                        ::*;
import Connectable                   ::*;
import Pipe                          ::*;
import FIFO                          ::*;
import BRAMFIFO                      ::*;
import FIFOF                         ::*;
import SpecialFIFOs                  ::*;
import GetPut                        ::*;
import Ethernet                      ::*;
import EthSonicPma                   ::*;
import EthPcsTx                      ::*;
import EthPcsRx                      ::*;
import DtpTx                         ::*;
import DtpRx                         ::*;
import DtpSwitch                     ::*;
import DtpDCFifo            ::*;
import ALTERA_SI570_WRAPPER          ::*;
import ALTERA_EDGE_DETECTOR_WRAPPER  ::*;
import AlteraExtra                   ::*;

`ifdef NUMBER_OF_10G_PORTS
typedef `NUMBER_OF_10G_PORTS NumPorts;
`else
typedef 4 NumPorts;
`endif

interface EthPhyIfc#(numeric type numPorts);
   interface Vector#(numPorts, PipeIn#(Bit#(72)))  tx;
   interface Vector#(numPorts, PipeOut#(Bit#(72))) rx;
   (* always_ready, always_enabled *)
   interface Vector#(numPorts, SerialIfc) serial;
   interface Vector#(numPorts, Clock) tx_clkout;
   interface Vector#(numPorts, Clock) rx_clkout;
   (* always_ready, always_enabled *)
   interface LoopbackIfc loopback;

   interface Vector#(numPorts, Bool) led_rx_ready;
   interface Vector#(numPorts, DtpToPhyIfc) api;
   interface PipeIn#(Bit#(1)) switchMode;
   interface PipeOut#(Bit#(53)) globalOut;
endinterface

function Bit#(n) reverseBits(Bit#(n) x);
   Vector#(n, Bit#(1)) vx = unpack(x);
   Vector#(n, Bit#(1)) rvx = Vector::reverse(vx);
   Bit#(n) prvx = pack(rvx);
   return(prvx);
endfunction

(* synthesize *)
module mkEthPhy#(Clock mgmt_clk, Clock clk_156_25, Clock clk_644, Reset rst_n)(EthPhyIfc#(NumPorts));

   Reg#(Bit#(32)) cycle <- mkReg(0);
   Reset rst_50_n <- mkAsyncReset(2, rst_n, mgmt_clk);
   Reset rst_156_25_n <- mkAsyncReset(2, rst_n, clk_156_25);

   Reg#(Bool) loopback_en <- mkReg(False);
   Reg#(Bool) switch_en <- mkReg(False);

   Vector#(NumPorts, EthPcsRx) pcs_rx = newVector;
   Vector#(NumPorts, EthPcsTx) pcs_tx = newVector;
   Vector#(NumPorts, DtpRx)    dtp_rx = newVector;
   Vector#(NumPorts, DtpTx)    dtp_tx = newVector;

   EthSonicPma#(NumPorts)      pma4 <- mkEthSonicPma(mgmt_clk, clk_644, clk_156_25, rst_50_n, clocked_by mgmt_clk, reset_by rst_50_n);

   Vector#(NumPorts, ReadOnly#(Bool)) rx_ready_tx;
   Vector#(NumPorts, ReadOnly#(Bool)) tx_ready_tx;
   Vector#(NumPorts, ReadOnly#(Bool)) rx_ready_rx;
   Vector#(NumPorts, ReadOnly#(Bool)) tx_ready_rx;
   Vector#(NumPorts, CrossingReg#(Bool)) lock_tx;
   for (Integer i=0; i<valueOf(NumPorts); i=i+1) begin
      rx_ready_tx[i] <- mkNullCrossingWire(clk_156_25, pma4.rx_ready[i]);
      tx_ready_tx[i] <- mkNullCrossingWire(clk_156_25, pma4.tx_ready[i]);
      rx_ready_rx[i] <- mkNullCrossingWire(pma4.rx_clkout[i], pma4.rx_ready[i]);
      tx_ready_rx[i] <- mkNullCrossingWire(pma4.rx_clkout[i], pma4.tx_ready[i]);
      lock_tx[i] <- mkNullCrossingReg(clk_156_25, False, clocked_by pma4.rx_clkout[i], reset_by pma4.rx_reset[i]);
   end

   Vector#(NumPorts, FIFOF#(Bit#(72)))   txFifo     = newVector;
   Vector#(NumPorts, FIFOF#(Bit#(72)))   rxFifo     = newVector;
   Vector#(NumPorts, PipeIn#(Bit#(72)))  vRxPipeIn  = newVector;
   Vector#(NumPorts, PipeOut#(Bit#(72))) vRxPipeOut = newVector;
   Vector#(NumPorts, PipeIn#(Bit#(72)))  vTxPipeIn  = newVector;
   Vector#(NumPorts, PipeOut#(Bit#(72))) vTxPipeOut = newVector;

   // Loopback FIFO
   Vector#(NumPorts, SyncFIFOIfc#(Bit#(66))) lpbkSyncFifo = newVector;
   Vector#(NumPorts, PipeOut#(Bit#(66))) lpbkSyncPipeOut = newVector;
   Vector#(NumPorts, PipeIn#(Bit#(66))) lpbkSyncPipeIn = newVector;

   // DtpRx to DtpTx Fifo
   Vector#(NumPorts, SyncFIFOIfc#(DtpEvent)) dtpEventFifo = newVector;
   Vector#(NumPorts, PipeOut#(DtpEvent)) dtpEventOut = newVector;
   Vector#(NumPorts, PipeIn#(DtpEvent)) dtpEventIn = newVector;
   Vector#(NumPorts, SyncFIFOIfc#(Bit#(32))) dtpErrCntFifo = newVector;
   Vector#(NumPorts, PipeOut#(Bit#(32))) dtpErrCntOut = newVector;
   Vector#(NumPorts, PipeIn#(Bit#(32))) dtpErrCntIn = newVector;

   // 156.25MHz to pma4.tx
   Vector#(NumPorts, SyncFIFOIfc#(Bit#(66))) txSyncFifo = newVector;

   FIFOF#(Bit#(1)) switchModeFifo <- mkFIFOF();

   for (Integer i=0; i<valueOf(NumPorts); i=i+1) begin
      rxFifo[i] <- mkFIFOF(clocked_by pma4.rx_clkout[i], reset_by pma4.rx_reset[i]);
      txFifo[i] <- mkFIFOF(clocked_by clk_156_25, reset_by rst_156_25_n);
      vRxPipeIn[i] = toPipeIn(rxFifo[i]);
      vRxPipeOut[i] = toPipeOut(rxFifo[i]);
      vTxPipeIn[i] = toPipeIn(txFifo[i]);
      vTxPipeOut[i] = toPipeOut(txFifo[i]);

      // Gearbox Level Loopback FIFO
      lpbkSyncFifo[i] <- mkSyncFIFO(10, clk_156_25, rst_156_25_n, pma4.rx_clkout[i]);//, pma4.rx_reset[i]);
      lpbkSyncPipeOut[i] = toPipeOut(lpbkSyncFifo[i]);
      lpbkSyncPipeIn[i] = toPipeIn(lpbkSyncFifo[i]);

      pcs_rx[i]    <- mkEthPcsRxTop(clocked_by pma4.rx_clkout[i], reset_by pma4.rx_reset[i]);
      pcs_tx[i]    <- mkEthPcsTxTop(clocked_by clk_156_25, reset_by rst_156_25_n);
      dtp_rx[i]    <- mkDtpRxTop(clocked_by pma4.rx_clkout[i], reset_by pma4.rx_reset[i]);
      dtp_tx[i]    <- mkDtpTxTop(clocked_by clk_156_25, reset_by rst_156_25_n);

      dtpEventFifo[i] <- mkSyncFIFO(10, pma4.rx_clkout[i], pma4.rx_reset[i], clk_156_25);//, rst_156_25_n);
      //dtpEventFifo[i] <- mkDtpDCFifo(pma4.rx_clkout[i], pma4.rx_reset[i], clk_156_25);//, rst_156_25_n);
      dtpEventOut[i] = toPipeOut(dtpEventFifo[i]);
      dtpEventIn[i]  = toPipeIn(dtpEventFifo[i]);

      dtpErrCntFifo[i] <- mkSyncFIFO(10, pma4.rx_clkout[i], pma4.rx_reset[i], clk_156_25);//, rst_156_25_n);
      dtpErrCntOut[i] = toPipeOut(dtpErrCntFifo[i]);
      dtpErrCntIn[i]  = toPipeIn(dtpErrCntFifo[i]);

      // Rx Path: PcsRx -> DtpRx
      mkConnection(pcs_rx[i].dtpRxIn, dtp_rx[i].dtpRxIn);
      // Rx Path: DtpRx -> PcsRx
      mkConnection(dtp_rx[i].dtpRxOut, pcs_rx[i].dtpRxOut);
      // Rx Path: DtpRx -> SyncFIFO
      mkConnection(dtp_rx[i].dtpEventOut, dtpEventIn[i]);
      // Rx Path DtpRx -> SyncFifo
      mkConnection(dtp_rx[i].dtpErrCnt, dtpErrCntIn[i]);

      // Tx Path SyncFIfo -> DtpTx
      mkConnection(dtpErrCntOut[i], dtp_tx[i].dtpErrCnt);
      // Tx Path; SyncFifo -> DtpTx
      mkConnection(dtpEventOut[i], dtp_tx[i].dtpEventIn);
      // Tx Path: Mac -> Encoder
      mkConnection(vTxPipeOut[i], pcs_tx[i].encoderIn);
      // Tx Path: PcsTx -> DtpTx
      mkConnection(pcs_tx[i].dtpTxIn, dtp_tx[i].dtpTxIn);
      // Tx Path: DtpTx -> PcsTx
      mkConnection(dtp_tx[i].dtpTxOut, pcs_tx[i].dtpTxOut);

      // Loopback Enable Signal
      //ReadOnly#(Bool) rx_lpbk_en <- mkNullCrossingWire(pma4.rx_clkout[i], loopback_en);
      //ReadOnly#(Bool) tx_lpbk_en <- mkNullCrossingWire(clk_156_25, loopback_en);
      let rx_lpbk_en = False;
      let tx_lpbk_en = False;

      rule cross_lock_tx;
         lock_tx[i] <= pcs_rx[i].lock;
      endrule

      // Normal Operation: XCVR -> Pcs
      rule rx_no_loopback(!rx_lpbk_en);
         let v <- toGet(pma4.rx[i]).get;
         let reversed_v = reverseBits(v);
         pcs_rx[i].bsyncIn.enq(reversed_v);
      endrule
      rule tx_no_loopback(!tx_lpbk_en);
         let v <- toGet(pcs_tx[i].scramblerOut).get;
         let reversed_v = reverseBits(v);
         pma4.tx[i].enq(reversed_v);
      endrule
      // END Normal Operation

      // Loopback Operation: Pcs -> lpbk Fifo -> Pcs
      ReadOnly#(Bit#(32)) cycle_cross <- mkNullCrossingWire(clk_156_25, cycle);
      rule tx_loopback(tx_lpbk_en);
         let v <- toGet(pcs_tx[i].scramblerOut).get;
         lpbkSyncPipeIn[i].enq(v);
      endrule
      rule tx_pma_loopback(tx_lpbk_en);
         // Make sure PMA is up by sending fake data.
         Bit#(66) count = 66'hAAAAA55555AAAAAA;
         pma4.tx[i].enq(count);
      endrule
      rule rx_loopback(rx_lpbk_en);
         let v <- toGet(lpbkSyncPipeOut[i]).get;
         pcs_rx[i].bsyncIn.enq(v);
      endrule
      rule rx_pma_loopback(rx_lpbk_en);
         let v <- toGet(pma4.rx[i]).get;
      endrule
      // END Loopback Operation
   end

   rule cyc;
      cycle <= cycle + 1;
   endrule

   ReadOnly#(Bool) switch_en_tx;
   switch_en_tx <- mkNullCrossingWire(clk_156_25, switch_en);

   DtpSwitch#(4) dtpswitch <- mkDtpSwitch(clocked_by clk_156_25, reset_by rst_156_25_n);
   for (Integer i=0; i<valueOf(NumPorts); i=i+1) begin
      mkConnection(dtp_tx[i].dtpLocalOut, dtpswitch.dtpLocalIn[i]);
      mkConnection(dtpswitch.dtpGlobalOut[i], dtp_tx[i].dtpGlobalIn);
   end

   for (Integer i=0; i<valueOf(NumPorts); i=i+1) begin
      rule pcs_tx_every1;
         pcs_tx[i].tx_ready(tx_ready_tx[i]);
      endrule
      rule pcs_rx_every1;
         pcs_rx[i].rx_ready(rx_ready_rx[i]);
      endrule
      rule dtp_tx_every1;
         dtp_tx[i].tx_ready(tx_ready_tx[i]);
         dtp_tx[i].rx_ready(rx_ready_tx[i]);
         dtp_tx[i].switch_mode(switch_en_tx);
         dtp_tx[i].bsync_lock(lock_tx[i].crossed);
      endrule
      rule dtp_rx_every1;
         dtp_rx[i].rx_ready(rx_ready_rx[i]);
         dtp_rx[i].bsync_lock(pcs_rx[i].lock);
      endrule
   end

   rule switch_mode;
      let v <- toGet(switchModeFifo).get;
      switch_en <= unpack(v);
   endrule

   rule dtpswitch_mode;
      dtpswitch.switch_mode(switch_en_tx);
   endrule

   for (Integer i=0; i<valueOf(NumPorts); i=i+1) begin
      rule receive;
         let v <- toGet(pcs_rx[i].decoderOut).get;
         vRxPipeIn[i].enq(v);
      endrule
   end

   Vector#(NumPorts, DtpToPhyIfc) vapi;
   for (Integer i=0; i<valueOf(NumPorts); i=i+1) begin
      vapi[i] = dtp_tx[i].api;
   end

   interface loopback = (interface LoopbackIfc;
      method Action lpbk_en (Bool en);
         loopback_en <= en;
      endmethod
   endinterface);

   interface rx_clkout = pma4.rx_clkout;
   interface tx_clkout = pma4.tx_clkout;
   interface serial = pma4.pmd;
   interface rx = vRxPipeOut;
   interface tx = vTxPipeIn;
   interface led_rx_ready = pma4.rx_ready;

   interface api = vapi;
   interface switchMode = toPipeIn(switchModeFifo);
   interface globalOut = dtpswitch.globalOut;

endmodule: mkEthPhy
endpackage: EthPhy