<commit_before>package periodic

import (
    "log"
    "net"
    "time"
    "sync"
    "strings"
    "container/list"
    "container/heap"
)

type Sched struct {
    jobTimer   *time.Timer
    grabQueue  *GrabQueue
    procQueue  *list.List
    revertPQ   PriorityQueue
    revTimer   *time.Timer
    entryPoint string
    JobLocker  *sync.Mutex
    Funcs      map[string]*FuncStat
    FuncLocker *sync.Mutex
    driver     StoreDriver
    jobPQ      map[string]*PriorityQueue
    PQLocker   *sync.Mutex
    timeout    time.Duration
}

type Counter uint

func (c *Counter) Incr() {
    *c = *c + 1
}

func (c *Counter) Decr() {
    *c = *c - 1
}

type FuncStat struct {
    Worker     Counter `json:"worker_count"`
    Job        Counter `json:"job_count"`
    Processing Counter `json:"processing"`
}

func NewSched(entryPoint string, driver StoreDriver, timeout time.Duration) *Sched {
    sched := new(Sched)
    sched.jobTimer = time.NewTimer(1 * time.Hour)
    sched.revTimer = time.NewTimer(1 * time.Hour)
    sched.grabQueue = NewGrabQueue()
    sched.procQueue = list.New()
    sched.revertPQ = make(PriorityQueue, 0)
    heap.Init(&sched.revertPQ)
    sched.entryPoint = entryPoint
    sched.JobLocker = new(sync.Mutex)
    sched.PQLocker = new(sync.Mutex)
    sched.FuncLocker = new(sync.Mutex)
    sched.Funcs = make(map[string]*FuncStat)
    sched.driver = driver
    sched.jobPQ = make(map[string]*PriorityQueue)
    sched.timeout = timeout
    return sched
}

func (sched *Sched) Serve() {
    parts := strings.SplitN(sched.entryPoint, "://", 2)
    if parts[0] == "unix" {
        sockCheck(parts[1])
    }
    sched.loadJobQueue()
    go sched.handleJobPQ()
    go sched.handleRevertPQ()
    listen, err := net.Listen(parts[0], parts[1])
    if err != nil {
        log.Fatal(err)
    }
    defer listen.Close()
    log.Printf("Periodic task system started on %s\n", sched.entryPoint)
    for {
        conn, err := listen.Accept()
        if err != nil {
            log.Fatal(err)
        }
        if sched.timeout > 0 {
            conn.SetDeadline(time.Now().Add(sched.timeout * time.Second))
        }
        sched.HandleConnection(conn)
    }
}

func (sched *Sched) Notify() {
    sched.jobTimer.Reset(time.Millisecond)
}

func (sched *Sched) NotifyRevertTimer() {
    sched.revTimer.Reset(time.Millisecond)
}

func (sched *Sched) HandleConnection(conn net.Conn) {
    c := Conn{Conn: conn}
    payload, err := c.Receive()
    if err != nil {
        return
    }
    switch ClientType(payload[0]) {
    case TYPE_CLIENT:
        client := NewClient(sched, c)
        go client.Handle()
        break
    case TYPE_WORKER:
        worker := NewWorker(sched, c)
        go worker.Handle()
        break
    default:
        log.Printf("Unsupport client %d\n", payload[0])
        c.Close()
        break
    }
}

func (sched *Sched) Done(jobId int64) {
    defer sched.Notify()
    defer sched.JobLocker.Unlock()
    sched.JobLocker.Lock()
    removeListJob(sched.procQueue, jobId)
    job, err := sched.driver.Get(jobId)
    if err == nil {
        sched.driver.Delete(jobId)
        sched.DecrStatJob(job)
        sched.DecrStatProc(job)
        sched.removeRevertPQ(job)
    }
    return
}

func (sched *Sched) isDoJob(job Job) bool {
    for e := sched.procQueue.Front(); e != nil; e = e.Next() {
        chk := e.Value.(Job)
        if chk.Id == job.Id {
            return true
        }
    }
    return false
}

func (sched *Sched) SubmitJob(grabItem GrabItem, job Job) bool {
    defer sched.JobLocker.Unlock()
    sched.JobLocker.Lock()
    if job.Name == "" {
        sched.driver.Delete(job.Id)
        return true
    }
    if sched.isDoJob(job) {
        return true
    }
    if !grabItem.w.alive {
        return false
    }
    if err := grabItem.w.HandleDo(grabItem.msgId, job); err != nil {
        grabItem.w.alive = false
        return false
    }
    now := time.Now()
    current := int64(now.Unix())
    job.Status = JOB_STATUS_PROC
    job.RunAt = current
    sched.driver.Save(&job)
    sched.IncrStatProc(job)
    sched.pushRevertPQ(job)
    sched.procQueue.PushBack(job)
    sched.grabQueue.Remove(grabItem)
    return true
}

func (sched *Sched) lessItem() (lessItem *Item) {
    defer sched.PQLocker.Unlock()
    sched.PQLocker.Lock()
    maybeItem := make(map[string]*Item)
    for Func, stat := range sched.Funcs {
        if stat.Worker == 0 {
            continue
        }
        pq, ok := sched.jobPQ[Func]
        if !ok || pq.Len() == 0 {
            continue
        }

        item := heap.Pop(pq).(*Item)

        maybeItem[Func] = item

    }

    if len(maybeItem) == 0 {
        return nil
    }

    var lessFunc string

    for Func, item := range maybeItem {
        if lessItem == nil {
            lessItem = item
            lessFunc = Func
            continue
        }
        if lessItem.priority > item.priority {
            lessItem = item
            lessFunc = Func
        }
    }

    for Func, item := range maybeItem {
        if Func == lessFunc {
            continue
        }
        pq := sched.jobPQ[Func]
        heap.Push(pq, item)
    }
    return
}

func (sched *Sched) handleJobPQ() {
    var current time.Time
    var timestamp int64
    for {
        if sched.grabQueue.Len() == 0 {
            sched.jobTimer.Reset(time.Minute)
            current =<-sched.jobTimer.C
            continue
        }

        lessItem := sched.lessItem()

        if lessItem == nil {
            sched.jobTimer.Reset(time.Minute)
            current =<-sched.jobTimer.C
            continue
        }

        schedJob, err := sched.driver.Get(lessItem.value)

        if err != nil {
            log.Printf("Error: Get job: %d %v\n", lessItem.value, err)
            continue
        }

        timestamp = int64(time.Now().Unix())

        if schedJob.SchedAt > timestamp {
            sched.jobTimer.Reset(time.Second * time.Duration(schedJob.SchedAt - timestamp))
            current =<-sched.jobTimer.C
            timestamp = int64(current.Unix())
            if schedJob.SchedAt > timestamp {
                sched.pushJobPQ(schedJob)
                continue
            }
        }

        grabItem, err := sched.grabQueue.Get(schedJob.Func)
        if err == nil {
            if !sched.SubmitJob(grabItem, schedJob) {
                sched.pushJobPQ(schedJob)
            }
        } else {
            sched.pushJobPQ(schedJob)
        }
    }
}

func (sched *Sched) handleRevertPQ() {
    var current time.Time
    var timestamp int64
    for {
        if sched.revertPQ.Len() == 0 {
            sched.revTimer.Reset(time.Minute)
            current =<-sched.revTimer.C
            continue
        }

        sched.PQLocker.Lock()
        item := heap.Pop(&sched.revertPQ).(*Item)
        sched.PQLocker.Unlock()

        if item == nil {
            sched.revTimer.Reset(time.Minute)
            current =<-sched.revTimer.C
            continue
        }

        revertJob, err := sched.driver.Get(item.value)

        if err != nil {
            log.Printf("Error: Get job: %d %v\n", item.value, err)
            continue
        }

        timestamp = int64(time.Now().Unix())

        if item.priority > timestamp {
            sched.revTimer.Reset(time.Second * time.Duration(item.priority - timestamp))
            current =<-sched.revTimer.C
            timestamp = int64(current.Unix())
            if item.priority > timestamp {
                sched.pushRevertPQ(revertJob)
                continue
            }
        }

        sched.DecrStatProc(revertJob)
        revertJob.Status = JOB_STATUS_READY
        sched.driver.Save(&revertJob)
        sched.pushJobPQ(revertJob)
        removeListJob(sched.procQueue, revertJob.Id)
    }
}

func (sched *Sched) Fail(jobId int64) {
    defer sched.Notify()
    defer sched.JobLocker.Unlock()
    sched.JobLocker.Lock()
    removeListJob(sched.procQueue, jobId)
    job, _ := sched.driver.Get(jobId)
    sched.DecrStatProc(job)
    sched.removeRevertPQ(job)
    job.Status = JOB_STATUS_READY
    sched.driver.Save(&job)
    sched.pushJobPQ(job)
    return
}

func (sched *Sched) getFuncStat(Func string) *FuncStat {
    defer sched.FuncLocker.Unlock()
    sched.FuncLocker.Lock()
    stat, ok := sched.Funcs[Func]
    if !ok {
        stat = new(FuncStat)
        sched.Funcs[Func] = stat
    }
    return stat
}

func (sched *Sched) IncrStatFunc(Func string) {
    stat := sched.getFuncStat(Func)
    stat.Worker.Incr()
}

func (sched *Sched) DecrStatFunc(Func string) {
    stat := sched.getFuncStat(Func)
    stat.Worker.Decr()
}

func (sched *Sched) IncrStatJob(job Job) {
    stat := sched.getFuncStat(job.Func)
    stat.Job.Incr()
}

func (sched *Sched) DecrStatJob(job Job) {
    stat := sched.getFuncStat(job.Func)
    stat.Job.Decr()
}

func (sched *Sched) IncrStatProc(job Job) {
    stat := sched.getFuncStat(job.Func)
    if job.Status == JOB_STATUS_PROC {
        stat.Processing.Incr()
    }
}

func (sched *Sched) DecrStatProc(job Job) {
    stat := sched.getFuncStat(job.Func)
    if job.Status == JOB_STATUS_PROC {
        stat.Processing.Decr()
    }
}

func (sched *Sched) SchedLater(jobId int64, delay int64) {
    defer sched.Notify()
    defer sched.JobLocker.Unlock()
    sched.JobLocker.Lock()
    removeListJob(sched.procQueue, jobId)
    job, _ := sched.driver.Get(jobId)
    sched.DecrStatProc(job)
    sched.removeRevertPQ(job)
    job.Status = JOB_STATUS_READY
    var now = time.Now()
    job.SchedAt = int64(now.Unix()) + delay
    sched.driver.Save(&job)
    sched.pushJobPQ(job)
    return
}

func (sched *Sched) pushJobPQ(job Job) bool {
    defer sched.PQLocker.Unlock()
    sched.PQLocker.Lock()
    if job.Status == JOB_STATUS_READY {
        pq, ok := sched.jobPQ[job.Func]
        if !ok {
            pq1 := make(PriorityQueue, 0)
            pq = &pq1
            sched.jobPQ[job.Func] = pq
            heap.Init(pq)
        }
        item := &Item{
            value: job.Id,
            priority: job.SchedAt,
        }
        heap.Push(pq, item)
        return true
    }
    return false
}

func (sched *Sched) pushRevertPQ(job Job) {
    defer sched.PQLocker.Unlock()
    sched.PQLocker.Lock()
    if job.Status == JOB_STATUS_PROC && job.Timeout > 0 {
        runAt := job.RunAt
        if runAt == 0 {
            runAt = job.SchedAt
        }
        item := &Item{
            value: job.Id,
            priority: runAt + job.Timeout,
        }
        heap.Push(&sched.revertPQ, item)
    }
}

func (sched *Sched) removeRevertPQ(job Job) {
    defer sched.PQLocker.Unlock()
    sched.PQLocker.Lock()
    if job.Status == JOB_STATUS_PROC && job.Timeout > 0 {
        for _, item := range sched.revertPQ {
            if item.value == job.Id {
                heap.Remove(&sched.revertPQ, item.index)
                break
            }
        }
    }
}

func (sched *Sched) loadJobQueue() {
    updateQueue := make([]Job, 0)
    removeQueue := make([]Job, 0)
    var now = time.Now()
    current := int64(now.Unix())

    iter := sched.driver.NewIterator(nil)
    for {
        if !iter.Next() {
            break
        }
        job := iter.Value()
        if job.Name == "" {
            removeQueue = append(removeQueue, job)
            continue
        }
        sched.IncrStatJob(job)
        sched.pushJobPQ(job)
        runAt := job.RunAt
        if runAt < job.SchedAt {
            runAt = job.SchedAt
        }
        if runAt + job.Timeout < current {
            updateQueue = append(updateQueue, job)
        } else {
            sched.procQueue.PushBack(job)
            sched.IncrStatProc(job)
            sched.pushRevertPQ(job)
        }
    }

    iter.Close()

    for _, job := range updateQueue {
        job.Status = JOB_STATUS_READY
        sched.driver.Save(&job)
    }

    for _, job := range removeQueue {
        sched.driver.Delete(job.Id)
    }
}

func (sched *Sched) Close() {
}
<commit_msg>NotifyRevertTimer when submit job<commit_after>package periodic

import (
    "log"
    "net"
    "time"
    "sync"
    "strings"
    "container/list"
    "container/heap"
)

type Sched struct {
    jobTimer   *time.Timer
    grabQueue  *GrabQueue
    procQueue  *list.List
    revertPQ   PriorityQueue
    revTimer   *time.Timer
    entryPoint string
    JobLocker  *sync.Mutex
    Funcs      map[string]*FuncStat
    FuncLocker *sync.Mutex
    driver     StoreDriver
    jobPQ      map[string]*PriorityQueue
    PQLocker   *sync.Mutex
    timeout    time.Duration
}

type Counter uint

func (c *Counter) Incr() {
    *c = *c + 1
}

func (c *Counter) Decr() {
    *c = *c - 1
}

type FuncStat struct {
    Worker     Counter `json:"worker_count"`
    Job        Counter `json:"job_count"`
    Processing Counter `json:"processing"`
}

func NewSched(entryPoint string, driver StoreDriver, timeout time.Duration) *Sched {
    sched := new(Sched)
    sched.jobTimer = time.NewTimer(1 * time.Hour)
    sched.revTimer = time.NewTimer(1 * time.Hour)
    sched.grabQueue = NewGrabQueue()
    sched.procQueue = list.New()
    sched.revertPQ = make(PriorityQueue, 0)
    heap.Init(&sched.revertPQ)
    sched.entryPoint = entryPoint
    sched.JobLocker = new(sync.Mutex)
    sched.PQLocker = new(sync.Mutex)
    sched.FuncLocker = new(sync.Mutex)
    sched.Funcs = make(map[string]*FuncStat)
    sched.driver = driver
    sched.jobPQ = make(map[string]*PriorityQueue)
    sched.timeout = timeout
    return sched
}

func (sched *Sched) Serve() {
    parts := strings.SplitN(sched.entryPoint, "://", 2)
    if parts[0] == "unix" {
        sockCheck(parts[1])
    }
    sched.loadJobQueue()
    go sched.handleJobPQ()
    go sched.handleRevertPQ()
    listen, err := net.Listen(parts[0], parts[1])
    if err != nil {
        log.Fatal(err)
    }
    defer listen.Close()
    log.Printf("Periodic task system started on %s\n", sched.entryPoint)
    for {
        conn, err := listen.Accept()
        if err != nil {
            log.Fatal(err)
        }
        if sched.timeout > 0 {
            conn.SetDeadline(time.Now().Add(sched.timeout * time.Second))
        }
        sched.HandleConnection(conn)
    }
}

func (sched *Sched) Notify() {
    sched.jobTimer.Reset(time.Millisecond)
}

func (sched *Sched) NotifyRevertTimer() {
    sched.revTimer.Reset(time.Millisecond)
}

func (sched *Sched) HandleConnection(conn net.Conn) {
    c := Conn{Conn: conn}
    payload, err := c.Receive()
    if err != nil {
        return
    }
    switch ClientType(payload[0]) {
    case TYPE_CLIENT:
        client := NewClient(sched, c)
        go client.Handle()
        break
    case TYPE_WORKER:
        worker := NewWorker(sched, c)
        go worker.Handle()
        break
    default:
        log.Printf("Unsupport client %d\n", payload[0])
        c.Close()
        break
    }
}

func (sched *Sched) Done(jobId int64) {
    defer sched.Notify()
    defer sched.JobLocker.Unlock()
    sched.JobLocker.Lock()
    removeListJob(sched.procQueue, jobId)
    job, err := sched.driver.Get(jobId)
    if err == nil {
        sched.driver.Delete(jobId)
        sched.DecrStatJob(job)
        sched.DecrStatProc(job)
        sched.removeRevertPQ(job)
    }
    return
}

func (sched *Sched) isDoJob(job Job) bool {
    for e := sched.procQueue.Front(); e != nil; e = e.Next() {
        chk := e.Value.(Job)
        if chk.Id == job.Id {
            return true
        }
    }
    return false
}

func (sched *Sched) SubmitJob(grabItem GrabItem, job Job) bool {
    defer sched.JobLocker.Unlock()
    sched.JobLocker.Lock()
    if job.Name == "" {
        sched.driver.Delete(job.Id)
        return true
    }
    if sched.isDoJob(job) {
        return true
    }
    if !grabItem.w.alive {
        return false
    }
    if err := grabItem.w.HandleDo(grabItem.msgId, job); err != nil {
        grabItem.w.alive = false
        return false
    }
    now := time.Now()
    current := int64(now.Unix())
    job.Status = JOB_STATUS_PROC
    job.RunAt = current
    sched.driver.Save(&job)
    sched.IncrStatProc(job)
    sched.pushRevertPQ(job)
    sched.NotifyRevertTimer()
    sched.procQueue.PushBack(job)
    sched.grabQueue.Remove(grabItem)
    return true
}

func (sched *Sched) lessItem() (lessItem *Item) {
    defer sched.PQLocker.Unlock()
    sched.PQLocker.Lock()
    maybeItem := make(map[string]*Item)
    for Func, stat := range sched.Funcs {
        if stat.Worker == 0 {
            continue
        }
        pq, ok := sched.jobPQ[Func]
        if !ok || pq.Len() == 0 {
            continue
        }

        item := heap.Pop(pq).(*Item)

        maybeItem[Func] = item

    }

    if len(maybeItem) == 0 {
        return nil
    }

    var lessFunc string

    for Func, item := range maybeItem {
        if lessItem == nil {
            lessItem = item
            lessFunc = Func
            continue
        }
        if lessItem.priority > item.priority {
            lessItem = item
            lessFunc = Func
        }
    }

    for Func, item := range maybeItem {
        if Func == lessFunc {
            continue
        }
        pq := sched.jobPQ[Func]
        heap.Push(pq, item)
    }
    return
}

func (sched *Sched) handleJobPQ() {
    var current time.Time
    var timestamp int64
    for {
        if sched.grabQueue.Len() == 0 {
            sched.jobTimer.Reset(time.Minute)
            current =<-sched.jobTimer.C
            continue
        }

        lessItem := sched.lessItem()

        if lessItem == nil {
            sched.jobTimer.Reset(time.Minute)
            current =<-sched.jobTimer.C
            continue
        }

        schedJob, err := sched.driver.Get(lessItem.value)

        if err != nil {
            log.Printf("Error: Get job: %d %v\n", lessItem.value, err)
            continue
        }

        timestamp = int64(time.Now().Unix())

        if schedJob.SchedAt > timestamp {
            sched.jobTimer.Reset(time.Second * time.Duration(schedJob.SchedAt - timestamp))
            current =<-sched.jobTimer.C
            timestamp = int64(current.Unix())
            if schedJob.SchedAt > timestamp {
                sched.pushJobPQ(schedJob)
                continue
            }
        }

        grabItem, err := sched.grabQueue.Get(schedJob.Func)
        if err == nil {
            if !sched.SubmitJob(grabItem, schedJob) {
                sched.pushJobPQ(schedJob)
            }
        } else {
            sched.pushJobPQ(schedJob)
        }
    }
}

func (sched *Sched) handleRevertPQ() {
    var current time.Time
    var timestamp int64
    for {
        if sched.revertPQ.Len() == 0 {
            sched.revTimer.Reset(time.Minute)
            current =<-sched.revTimer.C
            continue
        }

        sched.PQLocker.Lock()
        item := heap.Pop(&sched.revertPQ).(*Item)
        sched.PQLocker.Unlock()

        if item == nil {
            sched.revTimer.Reset(time.Minute)
            current =<-sched.revTimer.C
            continue
        }

        revertJob, err := sched.driver.Get(item.value)

        if err != nil {
            log.Printf("Error: Get job: %d %v\n", item.value, err)
            continue
        }

        timestamp = int64(time.Now().Unix())

        if item.priority > timestamp {
            sched.revTimer.Reset(time.Second * time.Duration(item.priority - timestamp))
            current =<-sched.revTimer.C
            timestamp = int64(current.Unix())
            if item.priority > timestamp {
                sched.pushRevertPQ(revertJob)
                continue
            }
        }

        sched.DecrStatProc(revertJob)
        revertJob.Status = JOB_STATUS_READY
        sched.driver.Save(&revertJob)
        sched.pushJobPQ(revertJob)
        removeListJob(sched.procQueue, revertJob.Id)
    }
}

func (sched *Sched) Fail(jobId int64) {
    defer sched.Notify()
    defer sched.JobLocker.Unlock()
    sched.JobLocker.Lock()
    removeListJob(sched.procQueue, jobId)
    job, _ := sched.driver.Get(jobId)
    sched.DecrStatProc(job)
    sched.removeRevertPQ(job)
    job.Status = JOB_STATUS_READY
    sched.driver.Save(&job)
    sched.pushJobPQ(job)
    return
}

func (sched *Sched) getFuncStat(Func string) *FuncStat {
    defer sched.FuncLocker.Unlock()
    sched.FuncLocker.Lock()
    stat, ok := sched.Funcs[Func]
    if !ok {
        stat = new(FuncStat)
        sched.Funcs[Func] = stat
    }
    return stat
}

func (sched *Sched) IncrStatFunc(Func string) {
    stat := sched.getFuncStat(Func)
    stat.Worker.Incr()
}

func (sched *Sched) DecrStatFunc(Func string) {
    stat := sched.getFuncStat(Func)
    stat.Worker.Decr()
}

func (sched *Sched) IncrStatJob(job Job) {
    stat := sched.getFuncStat(job.Func)
    stat.Job.Incr()
}

func (sched *Sched) DecrStatJob(job Job) {
    stat := sched.getFuncStat(job.Func)
    stat.Job.Decr()
}

func (sched *Sched) IncrStatProc(job Job) {
    stat := sched.getFuncStat(job.Func)
    if job.Status == JOB_STATUS_PROC {
        stat.Processing.Incr()
    }
}

func (sched *Sched) DecrStatProc(job Job) {
    stat := sched.getFuncStat(job.Func)
    if job.Status == JOB_STATUS_PROC {
        stat.Processing.Decr()
    }
}

func (sched *Sched) SchedLater(jobId int64, delay int64) {
    defer sched.Notify()
    defer sched.JobLocker.Unlock()
    sched.JobLocker.Lock()
    removeListJob(sched.procQueue, jobId)
    job, _ := sched.driver.Get(jobId)
    sched.DecrStatProc(job)
    sched.removeRevertPQ(job)
    job.Status = JOB_STATUS_READY
    var now = time.Now()
    job.SchedAt = int64(now.Unix()) + delay
    sched.driver.Save(&job)
    sched.pushJobPQ(job)
    return
}

func (sched *Sched) pushJobPQ(job Job) bool {
    defer sched.PQLocker.Unlock()
    sched.PQLocker.Lock()
    if job.Status == JOB_STATUS_READY {
        pq, ok := sched.jobPQ[job.Func]
        if !ok {
            pq1 := make(PriorityQueue, 0)
            pq = &pq1
            sched.jobPQ[job.Func] = pq
            heap.Init(pq)
        }
        item := &Item{
            value: job.Id,
            priority: job.SchedAt,
        }
        heap.Push(pq, item)
        return true
    }
    return false
}

func (sched *Sched) pushRevertPQ(job Job) {
    defer sched.PQLocker.Unlock()
    sched.PQLocker.Lock()
    if job.Status == JOB_STATUS_PROC && job.Timeout > 0 {
        runAt := job.RunAt
        if runAt == 0 {
            runAt = job.SchedAt
        }
        item := &Item{
            value: job.Id,
            priority: runAt + job.Timeout,
        }
        heap.Push(&sched.revertPQ, item)
    }
}

func (sched *Sched) removeRevertPQ(job Job) {
    defer sched.PQLocker.Unlock()
    sched.PQLocker.Lock()
    if job.Status == JOB_STATUS_PROC && job.Timeout > 0 {
        for _, item := range sched.revertPQ {
            if item.value == job.Id {
                heap.Remove(&sched.revertPQ, item.index)
                break
            }
        }
    }
}

func (sched *Sched) loadJobQueue() {
    updateQueue := make([]Job, 0)
    removeQueue := make([]Job, 0)
    var now = time.Now()
    current := int64(now.Unix())

    iter := sched.driver.NewIterator(nil)
    for {
        if !iter.Next() {
            break
        }
        job := iter.Value()
        if job.Name == "" {
            removeQueue = append(removeQueue, job)
            continue
        }
        sched.IncrStatJob(job)
        sched.pushJobPQ(job)
        runAt := job.RunAt
        if runAt < job.SchedAt {
            runAt = job.SchedAt
        }
        if runAt + job.Timeout < current {
            updateQueue = append(updateQueue, job)
        } else {
            sched.procQueue.PushBack(job)
            sched.IncrStatProc(job)
            sched.pushRevertPQ(job)
        }
    }

    iter.Close()

    for _, job := range updateQueue {
        job.Status = JOB_STATUS_READY
        sched.driver.Save(&job)
    }

    for _, job := range removeQueue {
        sched.driver.Delete(job.Id)
    }
}

func (sched *Sched) Close() {
}
<|endoftext|>


<commit_before>package wallet

type LightningChannel struct {
}
<commit_msg>Flesh out LightningChannel struct a bit more<commit_after>package wallet

import (
	"sync"

	"li.lan/labs/plasma/chainntfs"
	"li.lan/labs/plasma/revocation"

	"github.com/btcsuite/btcwallet/walletdb"
)

type LightningChannel struct {
	shachan *revocation.ShaChain
	wallet  *LightningWallet

	channelEvents *chainntnfs.ChainNotifier

	sync.RWMutex

	channelNamespace walletdb.Namespace

	// TODO(roasbeef): create and embed 'Service' interface w/ below?
	started int32

	shutdown int32

	quit chan struct{}
	wg   sync.WaitGroup
}

func newLightningChannel() *LightningChannel {
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"common"
	"github.com/surma/gocpio"
	"io"
	"log"
	"os"
	"strconv"
	"strings"
)

func parseInput(in io.ReadCloser, c chan<- *Entry) {
	buf := common.NewBufferedReader(in)
	for {
		line, e := buf.ReadWholeLine()
		if e != nil && e != io.EOF {
			log.Printf("Warning: Could not read whole file: %s", e.Error())
		}
		stripped_line := strings.TrimSpace(line)
		if len(stripped_line) == 0 || stripped_line[0] == '#' {
			if e != io.EOF {
				continue
			} else {
				break
			}
		}
		ent := parseLine(line)
		if ent != nil {
			c <- ent
		}

		if e == io.EOF {
			break
		}
	}
	close(c)
}

func parseLine(line string) *Entry {
	lineparts := strings.Split(line, " ")
	if len(lineparts) < 1 {
		return nil
	}

	switch lineparts[0] {
	case "file":
		return parseFile(lineparts[1:])
	case "dir":
		return parseDir(lineparts[1:])
	case "nod":
		return parseNod(lineparts[1:])
	case "slink":
		return parseSlink(lineparts[1:])
	case "pipe":
		return parsePipe(lineparts[1:])
	case "sock":
		return parseSock(lineparts[1:])
	default:
		log.Printf("Warning: %s is in invalid type\n", lineparts[0])
	}
	return nil
}

func parseFile(parts []string) *Entry {
	if len(parts) != 5 {
		return nil
	}
	name := parts[0]
	localname := parts[1]
	mode, uid, gid, e := parseModeUidGid(parts[2], parts[3], parts[4])
	if e != nil {
		log.Printf("Invalid permission settings: %s\n", e.Error())
		return nil
	}

	f, e := os.Open(localname)
	if e != nil {
		log.Printf("Could not open file %s: %s\n", localname, e.Error())
		return nil
	}
	finfo, e := f.Stat()
	if e != nil {
		log.Printf("Could not obtain file size of %s: %s\n")
		return nil
	}

	return &Entry{
		hdr: cpio.Header{
			Mode: mode,
			Uid:  uid,
			Gid:  gid,
			Size: finfo.Size(),
			Type: cpio.TYPE_REG,
			Name: name,
		},
		data: f,
	}
}

func parseDir(parts []string) *Entry {
	if len(parts) != 4 {
		return nil
	}
	name := parts[0]
	mode, uid, gid, e := parseModeUidGid(parts[1], parts[2], parts[3])
	if e != nil {
		log.Printf("Invalid permission settings: %s\n", e.Error())
		return nil
	}

	return &Entry{
		hdr: cpio.Header{
			Mode: mode,
			Uid:  uid,
			Gid:  gid,
			Type: cpio.TYPE_DIR,
			Name: name,
		},
	}
}

func parseNod(parts []string) *Entry {
	if len(parts) != 7 {
		return nil
	}
	name := parts[0]
	mode, uid, gid, e := parseModeUidGid(parts[1], parts[2], parts[3])
	if e != nil {
		log.Printf("Invalid permission settings: %s\n", e.Error())
		return nil
	}

	var dev_type int64
	switch parts[4] {
	case "b":
		dev_type = cpio.TYPE_BLK
	case "c":
		dev_type = cpio.TYPE_CHAR
	default:
		log.Printf("Invalid device type: %s\n", parts[4])
		return nil
	}

	maj, e := strconv.Atoi64(parts[5])
	if e != nil {
		log.Printf("Invalid major device: %s\n", e.Error())
		return nil
	}
	min, e := strconv.Atoi64(parts[6])
	if e != nil {
		log.Printf("Invalid major device: %s\n", e.Error())
		return nil
	}

	return &Entry{
		hdr: cpio.Header{
			Mode:     mode,
			Uid:      uid,
			Gid:      gid,
			Type:     dev_type,
			Devmajor: maj,
			Devminor: min,
			Name:     name,
		},
	}

}

func parseSlink(parts []string) *Entry {
	if len(parts) != 5 {
		return nil
	}

	name := parts[0]
	target := parts[1]

	mode, uid, gid, e := parseModeUidGid(parts[2], parts[3], parts[4])
	if e != nil {
		log.Printf("Invalid permission settings: %s\n", e.Error())
		return nil
	}

	return &Entry{
		hdr: cpio.Header{
			Mode: mode,
			Uid:  uid,
			Gid:  gid,
			Type: cpio.TYPE_SYMLINK,
			Size: int64(len(target) + 1),
			Name: name,
		},
		data: strings.NewReader(target),
	}

}

func parsePipe(parts []string) *Entry {
	if len(parts) != 4 {
		return nil
	}
	name := parts[0]
	mode, uid, gid, e := parseModeUidGid(parts[1], parts[2], parts[3])
	if e != nil {
		log.Printf("Invalid permission settings: %s\n", e.Error())
		return nil
	}

	return &Entry{
		hdr: cpio.Header{
			Mode: mode,
			Uid:  uid,
			Gid:  gid,
			Type: cpio.TYPE_FIFO,
			Name: name,
		},
	}
}

func parseSock(parts []string) *Entry {
	if len(parts) != 4 {
		return nil
	}
	name := parts[0]
	mode, uid, gid, e := parseModeUidGid(parts[1], parts[2], parts[3])
	if e != nil {
		log.Printf("Invalid permission settings: %s\n", e.Error())
		return nil
	}

	return &Entry{
		hdr: cpio.Header{
			Mode: mode,
			Uid:  uid,
			Gid:  gid,
			Type: cpio.TYPE_SOCK,
			Name: name,
		},
	}
}

func parseModeUidGid(s_mode, s_uid, s_gid string) (mode int64, uid, gid int, err error) {
	mode, err = strconv.Btoi64(s_mode, 0)
	if err != nil {
		return
	}
	uid, err = strconv.Atoi(s_uid)
	if err != nil {
		return
	}
	gid, err = strconv.Atoi(s_gid)
	return mode, uid, gid, nil
}
<commit_msg>Geninitramfs: Fixed<commit_after>package main

import (
	"common"
	"github.com/surma/gocpio"
	"io"
	"log"
	"os"
	"strconv"
	"strings"
)

func parseInput(in io.ReadCloser, c chan<- *Entry) {
	buf := common.NewBufferedReader(in)
	for {
		line, e := buf.ReadWholeLine()
		if e != nil && e != io.EOF {
			log.Printf("Warning: Could not read whole file: %s", e.Error())
		}
		stripped_line := strings.TrimSpace(line)
		if len(stripped_line) == 0 || stripped_line[0] == '#' {
			if e != io.EOF {
				continue
			} else {
				break
			}
		}
		ent := parseLine(line)
		if ent != nil {
			c <- ent
		}

		if e == io.EOF {
			break
		}
	}
	close(c)
}

func parseLine(line string) *Entry {
	lineparts := strings.Split(line, " ")
	if len(lineparts) < 1 {
		return nil
	}

	switch lineparts[0] {
	case "file":
		return parseFile(lineparts[1:])
	case "dir":
		return parseDir(lineparts[1:])
	case "nod":
		return parseNod(lineparts[1:])
	case "slink":
		return parseSlink(lineparts[1:])
	case "pipe":
		return parsePipe(lineparts[1:])
	case "sock":
		return parseSock(lineparts[1:])
	default:
		log.Printf("Warning: %s is in invalid type\n", lineparts[0])
	}
	return nil
}

func parseFile(parts []string) *Entry {
	if len(parts) != 5 {
		return nil
	}
	name := parts[0]
	localname := parts[1]
	mode, uid, gid, e := parseModeUidGid(parts[2], parts[3], parts[4])
	if e != nil {
		log.Printf("Invalid permission settings: %s\n", e.Error())
		return nil
	}

	f, e := os.Open(localname)
	if e != nil {
		log.Printf("Could not open file %s: %s\n", localname, e.Error())
		return nil
	}
	finfo, e := f.Stat()
	if e != nil {
		log.Printf("Could not obtain file size of %s: %s\n")
		return nil
	}

	return &Entry{
		hdr: cpio.Header{
			Mode: mode,
			Uid:  uid,
			Gid:  gid,
			Size: finfo.Size(),
			Type: cpio.TYPE_REG,
			Name: name,
		},
		data: f,
	}
}

func parseDir(parts []string) *Entry {
	if len(parts) != 4 {
		return nil
	}
	name := parts[0]
	mode, uid, gid, e := parseModeUidGid(parts[1], parts[2], parts[3])
	if e != nil {
		log.Printf("Invalid permission settings: %s\n", e.Error())
		return nil
	}

	return &Entry{
		hdr: cpio.Header{
			Mode: mode,
			Uid:  uid,
			Gid:  gid,
			Type: cpio.TYPE_DIR,
			Name: name,
		},
	}
}

func parseNod(parts []string) *Entry {
	if len(parts) != 7 {
		return nil
	}
	name := parts[0]
	mode, uid, gid, e := parseModeUidGid(parts[1], parts[2], parts[3])
	if e != nil {
		log.Printf("Invalid permission settings: %s\n", e.Error())
		return nil
	}

	var dev_type int64
	switch parts[4] {
	case "b":
		dev_type = cpio.TYPE_BLK
	case "c":
		dev_type = cpio.TYPE_CHAR
	default:
		log.Printf("Invalid device type: %s\n", parts[4])
		return nil
	}

	maj, e := strconv.ParseInt(parts[5], 10, 64)
	if e != nil {
		log.Printf("Invalid major device: %s\n", e.Error())
		return nil
	}
	min, e := strconv.ParseInt(parts[6], 10, 64)
	if e != nil {
		log.Printf("Invalid major device: %s\n", e.Error())
		return nil
	}

	return &Entry{
		hdr: cpio.Header{
			Mode:     mode,
			Uid:      uid,
			Gid:      gid,
			Type:     dev_type,
			Devmajor: maj,
			Devminor: min,
			Name:     name,
		},
	}

}

func parseSlink(parts []string) *Entry {
	if len(parts) != 5 {
		return nil
	}

	name := parts[0]
	target := parts[1]

	mode, uid, gid, e := parseModeUidGid(parts[2], parts[3], parts[4])
	if e != nil {
		log.Printf("Invalid permission settings: %s\n", e.Error())
		return nil
	}

	return &Entry{
		hdr: cpio.Header{
			Mode: mode,
			Uid:  uid,
			Gid:  gid,
			Type: cpio.TYPE_SYMLINK,
			Size: int64(len(target) + 1),
			Name: name,
		},
		data: strings.NewReader(target),
	}

}

func parsePipe(parts []string) *Entry {
	if len(parts) != 4 {
		return nil
	}
	name := parts[0]
	mode, uid, gid, e := parseModeUidGid(parts[1], parts[2], parts[3])
	if e != nil {
		log.Printf("Invalid permission settings: %s\n", e.Error())
		return nil
	}

	return &Entry{
		hdr: cpio.Header{
			Mode: mode,
			Uid:  uid,
			Gid:  gid,
			Type: cpio.TYPE_FIFO,
			Name: name,
		},
	}
}

func parseSock(parts []string) *Entry {
	if len(parts) != 4 {
		return nil
	}
	name := parts[0]
	mode, uid, gid, e := parseModeUidGid(parts[1], parts[2], parts[3])
	if e != nil {
		log.Printf("Invalid permission settings: %s\n", e.Error())
		return nil
	}

	return &Entry{
		hdr: cpio.Header{
			Mode: mode,
			Uid:  uid,
			Gid:  gid,
			Type: cpio.TYPE_SOCK,
			Name: name,
		},
	}
}

func parseModeUidGid(s_mode, s_uid, s_gid string) (int64, int, int, error) {
	mode, err := strconv.ParseInt(s_mode, 0, 64)
	if err != nil {
		return 0, 0, 0, err
	}
	tuid, err := strconv.ParseInt(s_uid, 0, 0)
	if err != nil {
		return 0, 0, 0, err
	}
	tgid, err := strconv.ParseInt(s_gid, 0, 0)
	return mode, int(tuid), int(tgid), nil
}
<|endoftext|>


<commit_before>// Package vector implements persistent vector.
package vector

import (
	"bytes"
	"encoding/json"
	"fmt"
)

const (
	chunkBits  = 5
	nodeSize   = 1 << chunkBits
	tailMaxLen = nodeSize
	chunkMask  = nodeSize - 1
)

// Vector is a persistent sequential container for arbitrary values. It supports
// O(1) lookup by index, modification by index, and insertion and removal
// operations at the end. Being a persistent variant of the data structure, it
// is immutable, and provides O(1) operations to create modified versions of the
// vector that shares the underlying data structure, making it suitable for
// concurrent access. The empty value is a valid empty vector.
type Vector interface {
	json.Marshaler
	// Len returns the length of the vector.
	Len() int
	// Nth returns the i-th element of the vector. It returns nil if the index
	// is smaller than 0 or greater than or equal to the length of the vector.
	Nth(i int) interface{}
	// AssocN returns an almost identical Vector, with the i-th element
	// replaced. If the index is smaller than 0 or greater than the length of
	// the vector, it returns nil. If the index is equal to the size of the
	// vector, it is equivalent to Cons.
	AssocN(i int, val interface{}) Vector
	// Cons returns an almost identical Vector, with an additional element
	// appended to the end.
	Cons(val interface{}) Vector
	// Pop returns an almost identical Vector, with the last element removed. It
	// returns nil if the vector is already empty.
	Pop() Vector
	// SubVector returns a subvector containing the elements from i up to but
	// not including j.
	SubVector(i, j int) Vector
	// Iterator returns an iterator over the vector.
	Iterator() Iterator
}

// Iterator is an iterator over vector elements. It can be used like this:
//
//     for it := v.Iterator(); it.HasElem(); it.Next() {
//         elem := it.Elem()
//         // do something with elem...
//     }
type Iterator interface {
	// Elem returns the element at the current position.
	Elem() interface{}
	// HasElem returns whether the iterator is pointing to an element.
	HasElem() bool
	// Next moves the iterator to the next position.
	Next()
}

type vector struct {
	count int
	// height of the tree structure, defined to be 0 when root is a leaf.
	height uint
	root   node
	tail   []interface{}
}

// Empty is an empty Vector.
var Empty Vector = &vector{}

// node is a node in the vector tree. It is always of the size nodeSize.
type node *[nodeSize]interface{}

func newNode() node {
	return node(&[nodeSize]interface{}{})
}

func clone(n node) node {
	a := *n
	return node(&a)
}

func nodeFromSlice(s []interface{}) node {
	var n [nodeSize]interface{}
	copy(n[:], s)
	return &n
}

// Count returns the number of elements in a Vector.
func (v *vector) Len() int {
	return v.count
}

// treeSize returns the number of elements stored in the tree (as opposed to the
// tail).
func (v *vector) treeSize() int {
	if v.count < tailMaxLen {
		return 0
	}
	return ((v.count - 1) >> chunkBits) << chunkBits
}

// sliceFor returns the slice where the i-th element is stored. The index must
// be in bound.
func (v *vector) sliceFor(i int) []interface{} {
	if i >= v.treeSize() {
		return v.tail
	}
	n := v.root
	for shift := v.height * chunkBits; shift > 0; shift -= chunkBits {
		n = n[(i>>shift)&chunkMask].(node)
	}
	return n[:]
}

func (v *vector) Nth(i int) interface{} {
	if i < 0 || i >= v.count {
		return nil
	}
	return v.sliceFor(i)[i&chunkMask]
}

func (v *vector) AssocN(i int, val interface{}) Vector {
	if i < 0 || i > v.count {
		return nil
	} else if i == v.count {
		return v.Cons(val)
	}
	if i >= v.treeSize() {
		newTail := append([]interface{}(nil), v.tail...)
		copy(newTail, v.tail)
		newTail[i&chunkMask] = val
		return &vector{v.count, v.height, v.root, newTail}
	}
	return &vector{v.count, v.height, doAssoc(v.height, v.root, i, val), v.tail}
}

// doAssoc returns an almost identical tree, with the i-th element replaced by
// val.
func doAssoc(height uint, n node, i int, val interface{}) node {
	m := clone(n)
	if height == 0 {
		m[i&chunkMask] = val
	} else {
		sub := (i >> (height * chunkBits)) & chunkMask
		m[sub] = doAssoc(height-1, m[sub].(node), i, val)
	}
	return m
}

func (v *vector) Cons(val interface{}) Vector {
	// Room in tail?
	if v.count-v.treeSize() < tailMaxLen {
		newTail := make([]interface{}, len(v.tail)+1)
		copy(newTail, v.tail)
		newTail[len(v.tail)] = val
		return &vector{v.count + 1, v.height, v.root, newTail}
	}
	// Full tail; push into tree.
	tailNode := nodeFromSlice(v.tail)
	newHeight := v.height
	var newRoot node
	// Overflow root?
	if (v.count >> chunkBits) > (1 << (v.height * chunkBits)) {
		newRoot = newNode()
		newRoot[0] = v.root
		newRoot[1] = newPath(v.height, tailNode)
		newHeight++
	} else {
		newRoot = v.pushTail(v.height, v.root, tailNode)
	}
	return &vector{v.count + 1, newHeight, newRoot, []interface{}{val}}
}

// pushTail returns a tree with tail appended.
func (v *vector) pushTail(height uint, n node, tail node) node {
	if height == 0 {
		return tail
	}
	idx := ((v.count - 1) >> (height * chunkBits)) & chunkMask
	m := clone(n)
	child := n[idx]
	if child == nil {
		m[idx] = newPath(height-1, tail)
	} else {
		m[idx] = v.pushTail(height-1, child.(node), tail)
	}
	return m
}

// newPath creates a left-branching tree of specified height and leaf.
func newPath(height uint, leaf node) node {
	if height == 0 {
		return leaf
	}
	ret := newNode()
	ret[0] = newPath(height-1, leaf)
	return ret
}

func (v *vector) Pop() Vector {
	switch v.count {
	case 0:
		return nil
	case 1:
		return Empty
	}
	if v.count-v.treeSize() > 1 {
		newTail := make([]interface{}, len(v.tail)-1)
		copy(newTail, v.tail)
		return &vector{v.count - 1, v.height, v.root, newTail}
	}
	newTail := v.sliceFor(v.count - 2)
	newRoot := v.popTail(v.height, v.root)
	newHeight := v.height
	if v.height > 0 && newRoot[1] == nil {
		newRoot = newRoot[0].(node)
		newHeight--
	}
	return &vector{v.count - 1, newHeight, newRoot, newTail}
}

// popTail returns a new tree with the last leaf removed.
func (v *vector) popTail(level uint, n node) node {
	idx := ((v.count - 2) >> (level * chunkBits)) & chunkMask
	if level > 1 {
		newChild := v.popTail(level-1, n[idx].(node))
		if newChild == nil && idx == 0 {
			return nil
		}
		m := clone(n)
		if newChild == nil {
			// This is needed since `m[idx] = newChild` would store an
			// interface{} with a non-nil type part, which is non-nil
			m[idx] = nil
		} else {
			m[idx] = newChild
		}
		return m
	} else if idx == 0 {
		return nil
	} else {
		m := clone(n)
		m[idx] = nil
		return m
	}
}

func (v *vector) SubVector(begin, end int) Vector {
	if begin < 0 || begin > end || end > v.count {
		return nil
	}
	return &subVector{v, begin, end}
}

func (v *vector) Iterator() Iterator {
	return newIterator(v)
}

func (v *vector) MarshalJSON() ([]byte, error) {
	return marshalJSON(v.Iterator())
}

type subVector struct {
	v     *vector
	begin int
	end   int
}

func (s *subVector) Len() int {
	return s.end - s.begin
}

func (s *subVector) Nth(i int) interface{} {
	if i < 0 || s.begin+i >= s.end {
		return nil
	}
	return s.v.Nth(s.begin + i)
}

func (s *subVector) AssocN(i int, val interface{}) Vector {
	if i < 0 || s.begin+i > s.end {
		return nil
	} else if s.begin+i == s.end {
		return s.Cons(val)
	}
	return s.v.AssocN(s.begin+i, val).SubVector(s.begin, s.end)
}

func (s *subVector) Cons(val interface{}) Vector {
	return s.v.AssocN(s.end, val).SubVector(s.begin, s.end+1)
}

func (s *subVector) Pop() Vector {
	switch s.Len() {
	case 0:
		return nil
	case 1:
		return Empty
	default:
		return s.v.SubVector(s.begin, s.end-1)
	}
}

func (s *subVector) SubVector(i, j int) Vector {
	return s.v.SubVector(s.begin+i, s.begin+j)
}

func (s *subVector) Iterator() Iterator {
	return newIteratorWithRange(s.v, s.begin, s.end)
}

func (s *subVector) MarshalJSON() ([]byte, error) {
	return marshalJSON(s.Iterator())
}

type iterator struct {
	v        *vector
	treeSize int
	index    int
	end      int
	path     []pathEntry
}

type pathEntry struct {
	node  node
	index int
}

func (e pathEntry) current() interface{} {
	return e.node[e.index]
}

func newIterator(v *vector) *iterator {
	return newIteratorWithRange(v, 0, v.Len())
}

func newIteratorWithRange(v *vector, begin, end int) *iterator {
	it := &iterator{v, v.treeSize(), begin, end, nil}
	// Find the node for begin, remembering all nodes along the path.
	n := v.root
	for shift := v.height * chunkBits; shift > 0; shift -= chunkBits {
		idx := (begin >> shift) & chunkMask
		it.path = append(it.path, pathEntry{n, idx})
		n = n[idx].(node)
	}
	it.path = append(it.path, pathEntry{n, begin & chunkMask})
	return it
}

func (it *iterator) Elem() interface{} {
	if it.index >= it.treeSize {
		return it.v.tail[it.index-it.treeSize]
	}
	return it.path[len(it.path)-1].current()
}

func (it *iterator) HasElem() bool {
	return it.index < it.end
}

func (it *iterator) Next() {
	if it.index+1 >= it.treeSize {
		// Next element is in tail. Just increment the index.
		it.index++
		return
	}
	// Find the deepest level that can be advanced.
	var i int
	for i = len(it.path) - 1; i >= 0; i-- {
		e := it.path[i]
		if e.index+1 < len(e.node) && e.node[e.index+1] != nil {
			break
		}
	}
	if i == -1 {
		panic("cannot advance; vector iterator bug")
	}
	// Advance on this node, and re-populate all deeper levels.
	it.path[i].index++
	for i++; i < len(it.path); i++ {
		it.path[i] = pathEntry{it.path[i-1].current().(node), 0}
	}
	it.index++
}

type marshalError struct {
	index int
	cause error
}

func (err *marshalError) Error() string {
	return fmt.Sprintf("element %d: %s", err.index, err.cause)
}

func marshalJSON(it Iterator) ([]byte, error) {
	var buf bytes.Buffer
	buf.WriteByte('[')
	index := 0
	for ; it.HasElem(); it.Next() {
		if index > 0 {
			buf.WriteByte(',')
		}
		elemBytes, err := json.Marshal(it.Elem())
		if err != nil {
			return nil, &marshalError{index, err}
		}
		buf.Write(elemBytes)
		index++
	}
	buf.WriteByte(']')
	return buf.Bytes(), nil
}
<commit_msg>vector: Avoid unnecessary copying in Nth.<commit_after>// Package vector implements persistent vector.
package vector

import (
	"bytes"
	"encoding/json"
	"fmt"
)

const (
	chunkBits  = 5
	nodeSize   = 1 << chunkBits
	tailMaxLen = nodeSize
	chunkMask  = nodeSize - 1
)

// Vector is a persistent sequential container for arbitrary values. It supports
// O(1) lookup by index, modification by index, and insertion and removal
// operations at the end. Being a persistent variant of the data structure, it
// is immutable, and provides O(1) operations to create modified versions of the
// vector that shares the underlying data structure, making it suitable for
// concurrent access. The empty value is a valid empty vector.
type Vector interface {
	json.Marshaler
	// Len returns the length of the vector.
	Len() int
	// Nth returns the i-th element of the vector. It returns nil if the index
	// is smaller than 0 or greater than or equal to the length of the vector.
	Nth(i int) interface{}
	// AssocN returns an almost identical Vector, with the i-th element
	// replaced. If the index is smaller than 0 or greater than the length of
	// the vector, it returns nil. If the index is equal to the size of the
	// vector, it is equivalent to Cons.
	AssocN(i int, val interface{}) Vector
	// Cons returns an almost identical Vector, with an additional element
	// appended to the end.
	Cons(val interface{}) Vector
	// Pop returns an almost identical Vector, with the last element removed. It
	// returns nil if the vector is already empty.
	Pop() Vector
	// SubVector returns a subvector containing the elements from i up to but
	// not including j.
	SubVector(i, j int) Vector
	// Iterator returns an iterator over the vector.
	Iterator() Iterator
}

// Iterator is an iterator over vector elements. It can be used like this:
//
//     for it := v.Iterator(); it.HasElem(); it.Next() {
//         elem := it.Elem()
//         // do something with elem...
//     }
type Iterator interface {
	// Elem returns the element at the current position.
	Elem() interface{}
	// HasElem returns whether the iterator is pointing to an element.
	HasElem() bool
	// Next moves the iterator to the next position.
	Next()
}

type vector struct {
	count int
	// height of the tree structure, defined to be 0 when root is a leaf.
	height uint
	root   node
	tail   []interface{}
}

// Empty is an empty Vector.
var Empty Vector = &vector{}

// node is a node in the vector tree. It is always of the size nodeSize.
type node *[nodeSize]interface{}

func newNode() node {
	return node(&[nodeSize]interface{}{})
}

func clone(n node) node {
	a := *n
	return node(&a)
}

func nodeFromSlice(s []interface{}) node {
	var n [nodeSize]interface{}
	copy(n[:], s)
	return &n
}

// Count returns the number of elements in a Vector.
func (v *vector) Len() int {
	return v.count
}

// treeSize returns the number of elements stored in the tree (as opposed to the
// tail).
func (v *vector) treeSize() int {
	if v.count < tailMaxLen {
		return 0
	}
	return ((v.count - 1) >> chunkBits) << chunkBits
}

func (v *vector) Nth(i int) interface{} {
	if i < 0 || i >= v.count {
		return nil
	}

	// The following is very similar to sliceFor, but is implemented separately
	// to avoid unncessary copying.
	if i >= v.treeSize() {
		return v.tail[i&chunkMask]
	}
	n := v.root
	for shift := v.height * chunkBits; shift > 0; shift -= chunkBits {
		n = n[(i>>shift)&chunkMask].(node)
	}
	return n[i&chunkMask]
}

// sliceFor returns the slice where the i-th element is stored. The index must
// be in bound.
func (v *vector) sliceFor(i int) []interface{} {
	if i >= v.treeSize() {
		return v.tail
	}
	n := v.root
	for shift := v.height * chunkBits; shift > 0; shift -= chunkBits {
		n = n[(i>>shift)&chunkMask].(node)
	}
	return n[:]
}

func (v *vector) AssocN(i int, val interface{}) Vector {
	if i < 0 || i > v.count {
		return nil
	} else if i == v.count {
		return v.Cons(val)
	}
	if i >= v.treeSize() {
		newTail := append([]interface{}(nil), v.tail...)
		copy(newTail, v.tail)
		newTail[i&chunkMask] = val
		return &vector{v.count, v.height, v.root, newTail}
	}
	return &vector{v.count, v.height, doAssoc(v.height, v.root, i, val), v.tail}
}

// doAssoc returns an almost identical tree, with the i-th element replaced by
// val.
func doAssoc(height uint, n node, i int, val interface{}) node {
	m := clone(n)
	if height == 0 {
		m[i&chunkMask] = val
	} else {
		sub := (i >> (height * chunkBits)) & chunkMask
		m[sub] = doAssoc(height-1, m[sub].(node), i, val)
	}
	return m
}

func (v *vector) Cons(val interface{}) Vector {
	// Room in tail?
	if v.count-v.treeSize() < tailMaxLen {
		newTail := make([]interface{}, len(v.tail)+1)
		copy(newTail, v.tail)
		newTail[len(v.tail)] = val
		return &vector{v.count + 1, v.height, v.root, newTail}
	}
	// Full tail; push into tree.
	tailNode := nodeFromSlice(v.tail)
	newHeight := v.height
	var newRoot node
	// Overflow root?
	if (v.count >> chunkBits) > (1 << (v.height * chunkBits)) {
		newRoot = newNode()
		newRoot[0] = v.root
		newRoot[1] = newPath(v.height, tailNode)
		newHeight++
	} else {
		newRoot = v.pushTail(v.height, v.root, tailNode)
	}
	return &vector{v.count + 1, newHeight, newRoot, []interface{}{val}}
}

// pushTail returns a tree with tail appended.
func (v *vector) pushTail(height uint, n node, tail node) node {
	if height == 0 {
		return tail
	}
	idx := ((v.count - 1) >> (height * chunkBits)) & chunkMask
	m := clone(n)
	child := n[idx]
	if child == nil {
		m[idx] = newPath(height-1, tail)
	} else {
		m[idx] = v.pushTail(height-1, child.(node), tail)
	}
	return m
}

// newPath creates a left-branching tree of specified height and leaf.
func newPath(height uint, leaf node) node {
	if height == 0 {
		return leaf
	}
	ret := newNode()
	ret[0] = newPath(height-1, leaf)
	return ret
}

func (v *vector) Pop() Vector {
	switch v.count {
	case 0:
		return nil
	case 1:
		return Empty
	}
	if v.count-v.treeSize() > 1 {
		newTail := make([]interface{}, len(v.tail)-1)
		copy(newTail, v.tail)
		return &vector{v.count - 1, v.height, v.root, newTail}
	}
	newTail := v.sliceFor(v.count - 2)
	newRoot := v.popTail(v.height, v.root)
	newHeight := v.height
	if v.height > 0 && newRoot[1] == nil {
		newRoot = newRoot[0].(node)
		newHeight--
	}
	return &vector{v.count - 1, newHeight, newRoot, newTail}
}

// popTail returns a new tree with the last leaf removed.
func (v *vector) popTail(level uint, n node) node {
	idx := ((v.count - 2) >> (level * chunkBits)) & chunkMask
	if level > 1 {
		newChild := v.popTail(level-1, n[idx].(node))
		if newChild == nil && idx == 0 {
			return nil
		}
		m := clone(n)
		if newChild == nil {
			// This is needed since `m[idx] = newChild` would store an
			// interface{} with a non-nil type part, which is non-nil
			m[idx] = nil
		} else {
			m[idx] = newChild
		}
		return m
	} else if idx == 0 {
		return nil
	} else {
		m := clone(n)
		m[idx] = nil
		return m
	}
}

func (v *vector) SubVector(begin, end int) Vector {
	if begin < 0 || begin > end || end > v.count {
		return nil
	}
	return &subVector{v, begin, end}
}

func (v *vector) Iterator() Iterator {
	return newIterator(v)
}

func (v *vector) MarshalJSON() ([]byte, error) {
	return marshalJSON(v.Iterator())
}

type subVector struct {
	v     *vector
	begin int
	end   int
}

func (s *subVector) Len() int {
	return s.end - s.begin
}

func (s *subVector) Nth(i int) interface{} {
	if i < 0 || s.begin+i >= s.end {
		return nil
	}
	return s.v.Nth(s.begin + i)
}

func (s *subVector) AssocN(i int, val interface{}) Vector {
	if i < 0 || s.begin+i > s.end {
		return nil
	} else if s.begin+i == s.end {
		return s.Cons(val)
	}
	return s.v.AssocN(s.begin+i, val).SubVector(s.begin, s.end)
}

func (s *subVector) Cons(val interface{}) Vector {
	return s.v.AssocN(s.end, val).SubVector(s.begin, s.end+1)
}

func (s *subVector) Pop() Vector {
	switch s.Len() {
	case 0:
		return nil
	case 1:
		return Empty
	default:
		return s.v.SubVector(s.begin, s.end-1)
	}
}

func (s *subVector) SubVector(i, j int) Vector {
	return s.v.SubVector(s.begin+i, s.begin+j)
}

func (s *subVector) Iterator() Iterator {
	return newIteratorWithRange(s.v, s.begin, s.end)
}

func (s *subVector) MarshalJSON() ([]byte, error) {
	return marshalJSON(s.Iterator())
}

type iterator struct {
	v        *vector
	treeSize int
	index    int
	end      int
	path     []pathEntry
}

type pathEntry struct {
	node  node
	index int
}

func (e pathEntry) current() interface{} {
	return e.node[e.index]
}

func newIterator(v *vector) *iterator {
	return newIteratorWithRange(v, 0, v.Len())
}

func newIteratorWithRange(v *vector, begin, end int) *iterator {
	it := &iterator{v, v.treeSize(), begin, end, nil}
	// Find the node for begin, remembering all nodes along the path.
	n := v.root
	for shift := v.height * chunkBits; shift > 0; shift -= chunkBits {
		idx := (begin >> shift) & chunkMask
		it.path = append(it.path, pathEntry{n, idx})
		n = n[idx].(node)
	}
	it.path = append(it.path, pathEntry{n, begin & chunkMask})
	return it
}

func (it *iterator) Elem() interface{} {
	if it.index >= it.treeSize {
		return it.v.tail[it.index-it.treeSize]
	}
	return it.path[len(it.path)-1].current()
}

func (it *iterator) HasElem() bool {
	return it.index < it.end
}

func (it *iterator) Next() {
	if it.index+1 >= it.treeSize {
		// Next element is in tail. Just increment the index.
		it.index++
		return
	}
	// Find the deepest level that can be advanced.
	var i int
	for i = len(it.path) - 1; i >= 0; i-- {
		e := it.path[i]
		if e.index+1 < len(e.node) && e.node[e.index+1] != nil {
			break
		}
	}
	if i == -1 {
		panic("cannot advance; vector iterator bug")
	}
	// Advance on this node, and re-populate all deeper levels.
	it.path[i].index++
	for i++; i < len(it.path); i++ {
		it.path[i] = pathEntry{it.path[i-1].current().(node), 0}
	}
	it.index++
}

type marshalError struct {
	index int
	cause error
}

func (err *marshalError) Error() string {
	return fmt.Sprintf("element %d: %s", err.index, err.cause)
}

func marshalJSON(it Iterator) ([]byte, error) {
	var buf bytes.Buffer
	buf.WriteByte('[')
	index := 0
	for ; it.HasElem(); it.Next() {
		if index > 0 {
			buf.WriteByte(',')
		}
		elemBytes, err := json.Marshal(it.Elem())
		if err != nil {
			return nil, &marshalError{index, err}
		}
		buf.Write(elemBytes)
		index++
	}
	buf.WriteByte(']')
	return buf.Bytes(), nil
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"os"

	"github.com/nictuku/stardew-rocks/parser"
	"github.com/nictuku/stardew-rocks/view"
)

func main() {
	if len(os.Args) != 2 {
		log.Fatalf("Expected `%v <xml save file>`", os.Args[0])
	}
	farm := parser.LoadFarmMap()

	log.Printf("Processing %v", os.Args[1])
	sg, err := os.Open(os.Args[1])
	if err != nil {
		log.Fatal(err)
	}

	gameSave, err := parser.ParseSaveGame(sg)
	if err != nil {
		log.Fatal(err)
	}
	view.WriteImage(farm, gameSave, os.Stdout)
}
<commit_msg>Parametrize the view/bin/draw.go output<commit_after>package main

import (
	"log"
	"os"

	"github.com/nictuku/stardew-rocks/parser"
	"github.com/nictuku/stardew-rocks/view"
)

func main() {
	if len(os.Args) != 3 {
		log.Fatalf("Expected `%v <xml save file> <output.png>`", os.Args[0])
	}
	farm := parser.LoadFarmMap()

	log.Printf("Processing %v", os.Args[1])
	sg, err := os.Open(os.Args[1])
	if err != nil {
		log.Fatal(err)
	}

	f, err := os.OpenFile(os.Args[2], os.O_CREATE|os.O_WRONLY, 0600)
	if err != nil {
		log.Fatal(err)
	}

	gameSave, err := parser.ParseSaveGame(sg)
	if err != nil {
		log.Fatal(err)
	}
	view.WriteImage(farm, gameSave, f)
}
<|endoftext|>


<commit_before>package multipass

// Add the multipass directive to the caddy source in order to build the plugin.
// source: github.com/mholt/caddy/caddyhttp/httpserver/plugin.go

import (
	"errors"
	"fmt"
	"time"

	"github.com/mholt/caddy"
	"github.com/mholt/caddy/caddyhttp/httpserver"
)

const directive = "multipass"

func init() {
	caddy.RegisterPlugin(directive, caddy.Plugin{
		ServerType: "http",
		Action:     setup,
	})
}

func setup(c *caddy.Controller) error {
	c.OnStartup(func() error {
		fmt.Println(directive + " is intialized")
		return nil
	})

	rules, err := multipassParse(c)
	if err != nil {
		return err
	}
	if len(rules) == 0 {
		return errors.New("No directive declared")
	}

	multipass, err := NewMultipassFromRule(rules[0])
	if err != nil {
		return err
	}

	cfg := httpserver.GetConfig(c)
	multipass.SiteAddr = cfg.Addr.String()
	mid := func(next httpserver.Handler) httpserver.Handler {
		return &Auth{
			Multipass: multipass,
			Next:      next,
		}
	}
	cfg.AddMiddleware(mid)

	return nil
}

func multipassParse(c *caddy.Controller) ([]Rule, error) {
	var rules []Rule
	for c.Next() {
		var rule Rule
		switch len(c.RemainingArgs()) {
		case 0:
			for c.NextBlock() {
				switch c.Val() {
				case "resources":
					args := c.RemainingArgs()
					rule.Resources = args
				case "basepath":
					args := c.RemainingArgs()
					if len(args) != 1 {
						return rules, c.Err("Expecting a single basepath")
					}
					rule.Basepath = args[0]
				case "handles":
					args := c.RemainingArgs()
					if len(args) <= 0 {
						return rules, c.Err("Expecting at least one handle")
					}
					rule.Handles = args
				case "expires":
					args := c.RemainingArgs()
					if len(args) != 1 {
						return rules, c.Err("Expecting a single expiry time duration")
					}
					d, err := time.ParseDuration(args[0])
					if err != nil {
						return rules, c.Err("Expecting a valida Go formatted time duration")
					}
					rule.Expires = d
				case "smtp_addr":
					args := c.RemainingArgs()
					if len(args) != 1 {
						return rules, c.Err("Expecting a single SMTP address")
					}
					rule.SMTPAddr = args[0]
				case "smtp_user":
					args := c.RemainingArgs()
					if len(args) != 1 {
						return rules, c.Err("Expecting a single SMTP username")
					}
					rule.SMTPUser = args[0]
				case "smtp_pass":
					args := c.RemainingArgs()
					if len(args) != 1 {
						return rules, c.Err("Expecting a single SMTP password")
					}
					rule.SMTPPass = args[0]
				case "mail_from":
					args := c.RemainingArgs()
					if len(args) != 1 {
						return rules, c.Err("Expecting a single mail from address")
					}
					rule.MailFrom = args[0]
				case "mail_tmpl":
					args := c.RemainingArgs()
					if len(args) != 1 {
						return rules, c.Err("Expecting a single mail template")
					}
					rule.MailTmpl = args[0]
				}
			}
			if len(rule.Handles) == 0 {
				return rules, c.Err("Expecting at least one handle")
			}
			if len(rule.MailFrom) == 0 {
				return rules, c.Err("Expecting a single mail from addres")
			}
			if len(rules) > 0 {
				return rules, c.Err("Expecting one directive per site")
			}
			rules = append(rules, rule)
		default:
			return rules, c.Err("Single line directives not supported")
		}
	}
	return rules, nil
}
<commit_msg>Rewrite error description to improve clarity<commit_after>package multipass

// Add the multipass directive to the caddy source in order to build the plugin.
// source: github.com/mholt/caddy/caddyhttp/httpserver/plugin.go

import (
	"errors"
	"fmt"
	"time"

	"github.com/mholt/caddy"
	"github.com/mholt/caddy/caddyhttp/httpserver"
)

const directive = "multipass"

func init() {
	caddy.RegisterPlugin(directive, caddy.Plugin{
		ServerType: "http",
		Action:     setup,
	})
}

func setup(c *caddy.Controller) error {
	c.OnStartup(func() error {
		fmt.Println(directive + " is intialized")
		return nil
	})

	rules, err := multipassParse(c)
	if err != nil {
		return err
	}
	if len(rules) == 0 {
		return errors.New("No directive declared")
	}

	multipass, err := NewMultipassFromRule(rules[0])
	if err != nil {
		return err
	}

	cfg := httpserver.GetConfig(c)
	multipass.SiteAddr = cfg.Addr.String()
	mid := func(next httpserver.Handler) httpserver.Handler {
		return &Auth{
			Multipass: multipass,
			Next:      next,
		}
	}
	cfg.AddMiddleware(mid)

	return nil
}

func multipassParse(c *caddy.Controller) ([]Rule, error) {
	var rules []Rule
	for c.Next() {
		var rule Rule
		switch len(c.RemainingArgs()) {
		case 0:
			for c.NextBlock() {
				switch c.Val() {
				case "resources":
					args := c.RemainingArgs()
					rule.Resources = args
				case "basepath":
					args := c.RemainingArgs()
					if len(args) != 1 {
						return rules, c.Err("Expecting a single basepath")
					}
					rule.Basepath = args[0]
				case "handles":
					args := c.RemainingArgs()
					if len(args) <= 0 {
						return rules, c.Err("Expecting at least one handle")
					}
					rule.Handles = args
				case "expires":
					args := c.RemainingArgs()
					if len(args) != 1 {
						return rules, c.Err("Expecting a single Go formatted time duration")
					}
					d, err := time.ParseDuration(args[0])
					if err != nil {
						return rules, c.Err("Expecting a single Go formatted time duration")
					}
					rule.Expires = d
				case "smtp_addr":
					args := c.RemainingArgs()
					if len(args) != 1 {
						return rules, c.Err("Expecting a single SMTP server address")
					}
					rule.SMTPAddr = args[0]
				case "smtp_user":
					args := c.RemainingArgs()
					if len(args) != 1 {
						return rules, c.Err("Expecting a single SMTP username")
					}
					rule.SMTPUser = args[0]
				case "smtp_pass":
					args := c.RemainingArgs()
					if len(args) != 1 {
						return rules, c.Err("Expecting a single SMTP password")
					}
					rule.SMTPPass = args[0]
				case "mail_from":
					args := c.RemainingArgs()
					if len(args) != 1 {
						return rules, c.Err("Expecting a single mail from address")
					}
					rule.MailFrom = args[0]
				case "mail_tmpl":
					args := c.RemainingArgs()
					if len(args) != 1 {
						return rules, c.Err("Expecting a single mail template")
					}
					rule.MailTmpl = args[0]
				}
			}
			if len(rule.Handles) < 1 {
				return rules, c.Err("Expecting at least one handle")
			}
			if len(rule.MailFrom) != 1 {
				return rules, c.Err("Expecting a single mail from addres")
			}
			if len(rules) != 1 {
				return rules, c.Err("Expecting one directive per site")
			}
			rules = append(rules, rule)
		default:
			return rules, c.Err("Single line directives not supported")
		}
	}
	return rules, nil
}
<|endoftext|>


<commit_before>package task

import (
	"context"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"sync"

	compilerv2 "github.com/go-task/task/v3/internal/compiler/v2"
	compilerv3 "github.com/go-task/task/v3/internal/compiler/v3"
	"github.com/go-task/task/v3/internal/execext"
	"github.com/go-task/task/v3/internal/filepathext"
	"github.com/go-task/task/v3/internal/logger"
	"github.com/go-task/task/v3/internal/output"
	"github.com/go-task/task/v3/taskfile"
	"github.com/go-task/task/v3/taskfile/read"

	"github.com/sajari/fuzzy"
)

func (e *Executor) Setup() error {
	if err := e.setCurrentDir(); err != nil {
		return err
	}

	if err := e.readTaskfile(); err != nil {
		return err
	}

	e.setupFuzzyModel()

	v, err := e.Taskfile.ParsedVersion()
	if err != nil {
		return err
	}

	if err := e.setupTempDir(); err != nil {
		return err
	}
	e.setupStdFiles()
	e.setupLogger()
	if err := e.setupOutput(); err != nil {
		return err
	}
	if err := e.setupCompiler(v); err != nil {
		return err
	}
	if err := e.readDotEnvFiles(v); err != nil {
		return err
	}

	if err := e.doVersionChecks(v); err != nil {
		return err
	}
	e.setupDefaults(v)
	e.setupConcurrencyState()

	return nil
}

func (e *Executor) setCurrentDir() error {
	if e.Dir == "" {
		wd, err := os.Getwd()
		if err != nil {
			return err
		}
		e.Dir = wd
	} else if !filepath.IsAbs(e.Dir) {
		abs, err := filepath.Abs(e.Dir)
		if err != nil {
			return err
		}
		e.Dir = abs
	}
	return nil
}

func (e *Executor) readTaskfile() error {
	var err error
	e.Taskfile, err = read.Taskfile(&read.ReaderNode{
		Dir:        e.Dir,
		Entrypoint: e.Entrypoint,
		Parent:     nil,
		Optional:   false,
	})
	return err
}

func (e *Executor) setupFuzzyModel() {
	if e.Taskfile != nil {
		model := fuzzy.NewModel()
		model.SetThreshold(1) // because we want to build grammar based on every task name

		var words []string
		for taskName := range e.Taskfile.Tasks {
			words = append(words, taskName)

			for _, task := range e.Taskfile.Tasks {
				words = append(words, task.Aliases...)
			}
		}

		model.Train(words)
		e.fuzzyModel = model
	}
}

func (e *Executor) setupTempDir() error {
	if e.TempDir != "" {
		return nil
	}

	if os.Getenv("TASK_TEMP_DIR") == "" {
		e.TempDir = filepathext.SmartJoin(e.Dir, ".task")
	} else if filepath.IsAbs(os.Getenv("TASK_TEMP_DIR")) || strings.HasPrefix(os.Getenv("TASK_TEMP_DIR"), "~") {
		tempDir, err := execext.Expand(os.Getenv("TASK_TEMP_DIR"))
		if err != nil {
			return err
		}
		projectDir, _ := filepath.Abs(e.Dir)
		projectName := filepath.Base(projectDir)
		e.TempDir = filepathext.SmartJoin(tempDir, projectName)
	} else {
		e.TempDir = filepathext.SmartJoin(e.Dir, os.Getenv("TASK_TEMP_DIR"))
	}

	return nil
}

func (e *Executor) setupStdFiles() {
	if e.Stdin == nil {
		e.Stdin = os.Stdin
	}
	if e.Stdout == nil {
		e.Stdout = os.Stdout
	}
	if e.Stderr == nil {
		e.Stderr = os.Stderr
	}
}

func (e *Executor) setupLogger() {
	e.Logger = &logger.Logger{
		Stdout:  e.Stdout,
		Stderr:  e.Stderr,
		Verbose: e.Verbose,
		Color:   e.Color,
	}
}

func (e *Executor) setupOutput() error {
	if !e.OutputStyle.IsSet() {
		e.OutputStyle = e.Taskfile.Output
	}

	var err error
	e.Output, err = output.BuildFor(&e.OutputStyle)
	return err
}

func (e *Executor) setupCompiler(v float64) error {
	if v < 3 {
		var err error
		e.taskvars, err = read.Taskvars(e.Dir)
		if err != nil {
			return err
		}

		e.Compiler = &compilerv2.CompilerV2{
			Dir:          e.Dir,
			Taskvars:     e.taskvars,
			TaskfileVars: e.Taskfile.Vars,
			Expansions:   e.Taskfile.Expansions,
			Logger:       e.Logger,
		}
	} else {
		e.Compiler = &compilerv3.CompilerV3{
			Dir:          e.Dir,
			TaskfileEnv:  e.Taskfile.Env,
			TaskfileVars: e.Taskfile.Vars,
			Logger:       e.Logger,
		}
	}

	return nil
}

func (e *Executor) readDotEnvFiles(v float64) error {
	if v < 3.0 {
		return nil
	}

	env, err := read.Dotenv(e.Compiler, e.Taskfile, e.Dir)
	if err != nil {
		return err
	}

	err = env.Range(func(key string, value taskfile.Var) error {
		if _, ok := e.Taskfile.Env.Mapping[key]; !ok {
			e.Taskfile.Env.Set(key, value)
		}
		return nil
	})
	return err
}

func (e *Executor) setupDefaults(v float64) {
	// Color available only on v3
	if v < 3 {
		e.Logger.Color = false
	}

	if e.Taskfile.Method == "" {
		if v >= 3 {
			e.Taskfile.Method = "checksum"
		} else {
			e.Taskfile.Method = "timestamp"
		}
	}

	if e.Taskfile.Run == "" {
		e.Taskfile.Run = "always"
	}
}

func (e *Executor) setupConcurrencyState() {
	e.executionHashes = make(map[string]context.Context)

	e.taskCallCount = make(map[string]*int32, len(e.Taskfile.Tasks))
	e.mkdirMutexMap = make(map[string]*sync.Mutex, len(e.Taskfile.Tasks))
	for k := range e.Taskfile.Tasks {
		e.taskCallCount[k] = new(int32)
		e.mkdirMutexMap[k] = &sync.Mutex{}
	}

	if e.Concurrency > 0 {
		e.concurrencySemaphore = make(chan struct{}, e.Concurrency)
	}
}

func (e *Executor) doVersionChecks(v float64) error {
	if v < 2 {
		return fmt.Errorf(`task: Taskfile versions prior to v2 are not supported anymore`)
	}

	// consider as equal to the greater version if round
	if v == 2.0 {
		v = 2.6
	}
	if v == 3.0 {
		v = 3.8
	}

	if v > 3.8 {
		return fmt.Errorf(`task: Taskfile versions greater than v3.8 not implemented in the version of Task`)
	}

	if v < 2.1 && !e.Taskfile.Output.IsSet() {
		return fmt.Errorf(`task: Taskfile option "output" is only available starting on Taskfile version v2.1`)
	}
	if v < 2.2 && e.Taskfile.Includes.Len() > 0 {
		return fmt.Errorf(`task: Including Taskfiles is only available starting on Taskfile version v2.2`)
	}
	if v >= 3.0 && e.Taskfile.Expansions > 2 {
		return fmt.Errorf(`task: The "expansions" setting is not available anymore on v3.0`)
	}
	if v < 3.8 && e.Taskfile.Output.Group.IsSet() {
		return fmt.Errorf(`task: Taskfile option "output.group" is only available starting on Taskfile version v3.8`)
	}

	if v <= 2.1 {
		err := errors.New(`task: Taskfile option "ignore_error" is only available starting on Taskfile version v2.1`)

		for _, task := range e.Taskfile.Tasks {
			if task.IgnoreError {
				return err
			}
			for _, cmd := range task.Cmds {
				if cmd.IgnoreError {
					return err
				}
			}
		}
	}

	if v < 2.6 {
		for _, task := range e.Taskfile.Tasks {
			if len(task.Preconditions) > 0 {
				return errors.New(`task: Task option "preconditions" is only available starting on Taskfile version v2.6`)
			}
		}
	}

	if v < 3 {
		err := e.Taskfile.Includes.Range(func(_ string, taskfile taskfile.IncludedTaskfile) error {
			if taskfile.AdvancedImport {
				return errors.New(`task: Import with additional parameters is only available starting on Taskfile version v3`)
			}
			return nil
		})
		if err != nil {
			return err
		}
	}

	if v < 3.7 {
		if e.Taskfile.Run != "" {
			return errors.New(`task: Setting the "run" type is only available starting on Taskfile version v3.7`)
		}

		for _, task := range e.Taskfile.Tasks {
			if task.Run != "" {
				return errors.New(`task: Setting the "run" type is only available starting on Taskfile version v3.7`)
			}
		}
	}

	return nil
}
<commit_msg>Refactor: Use early return<commit_after>package task

import (
	"context"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"sync"

	compilerv2 "github.com/go-task/task/v3/internal/compiler/v2"
	compilerv3 "github.com/go-task/task/v3/internal/compiler/v3"
	"github.com/go-task/task/v3/internal/execext"
	"github.com/go-task/task/v3/internal/filepathext"
	"github.com/go-task/task/v3/internal/logger"
	"github.com/go-task/task/v3/internal/output"
	"github.com/go-task/task/v3/taskfile"
	"github.com/go-task/task/v3/taskfile/read"

	"github.com/sajari/fuzzy"
)

func (e *Executor) Setup() error {
	if err := e.setCurrentDir(); err != nil {
		return err
	}

	if err := e.readTaskfile(); err != nil {
		return err
	}

	e.setupFuzzyModel()

	v, err := e.Taskfile.ParsedVersion()
	if err != nil {
		return err
	}

	if err := e.setupTempDir(); err != nil {
		return err
	}
	e.setupStdFiles()
	e.setupLogger()
	if err := e.setupOutput(); err != nil {
		return err
	}
	if err := e.setupCompiler(v); err != nil {
		return err
	}
	if err := e.readDotEnvFiles(v); err != nil {
		return err
	}

	if err := e.doVersionChecks(v); err != nil {
		return err
	}
	e.setupDefaults(v)
	e.setupConcurrencyState()

	return nil
}

func (e *Executor) setCurrentDir() error {
	if e.Dir == "" {
		wd, err := os.Getwd()
		if err != nil {
			return err
		}
		e.Dir = wd
	} else if !filepath.IsAbs(e.Dir) {
		abs, err := filepath.Abs(e.Dir)
		if err != nil {
			return err
		}
		e.Dir = abs
	}
	return nil
}

func (e *Executor) readTaskfile() error {
	var err error
	e.Taskfile, err = read.Taskfile(&read.ReaderNode{
		Dir:        e.Dir,
		Entrypoint: e.Entrypoint,
		Parent:     nil,
		Optional:   false,
	})
	return err
}

func (e *Executor) setupFuzzyModel() {
	if e.Taskfile != nil {
		return
	}

	model := fuzzy.NewModel()
	model.SetThreshold(1) // because we want to build grammar based on every task name

	var words []string
	for taskName := range e.Taskfile.Tasks {
		words = append(words, taskName)

		for _, task := range e.Taskfile.Tasks {
			words = append(words, task.Aliases...)
		}
	}

	model.Train(words)
	e.fuzzyModel = model
}

func (e *Executor) setupTempDir() error {
	if e.TempDir != "" {
		return nil
	}

	if os.Getenv("TASK_TEMP_DIR") == "" {
		e.TempDir = filepathext.SmartJoin(e.Dir, ".task")
	} else if filepath.IsAbs(os.Getenv("TASK_TEMP_DIR")) || strings.HasPrefix(os.Getenv("TASK_TEMP_DIR"), "~") {
		tempDir, err := execext.Expand(os.Getenv("TASK_TEMP_DIR"))
		if err != nil {
			return err
		}
		projectDir, _ := filepath.Abs(e.Dir)
		projectName := filepath.Base(projectDir)
		e.TempDir = filepathext.SmartJoin(tempDir, projectName)
	} else {
		e.TempDir = filepathext.SmartJoin(e.Dir, os.Getenv("TASK_TEMP_DIR"))
	}

	return nil
}

func (e *Executor) setupStdFiles() {
	if e.Stdin == nil {
		e.Stdin = os.Stdin
	}
	if e.Stdout == nil {
		e.Stdout = os.Stdout
	}
	if e.Stderr == nil {
		e.Stderr = os.Stderr
	}
}

func (e *Executor) setupLogger() {
	e.Logger = &logger.Logger{
		Stdout:  e.Stdout,
		Stderr:  e.Stderr,
		Verbose: e.Verbose,
		Color:   e.Color,
	}
}

func (e *Executor) setupOutput() error {
	if !e.OutputStyle.IsSet() {
		e.OutputStyle = e.Taskfile.Output
	}

	var err error
	e.Output, err = output.BuildFor(&e.OutputStyle)
	return err
}

func (e *Executor) setupCompiler(v float64) error {
	if v < 3 {
		var err error
		e.taskvars, err = read.Taskvars(e.Dir)
		if err != nil {
			return err
		}

		e.Compiler = &compilerv2.CompilerV2{
			Dir:          e.Dir,
			Taskvars:     e.taskvars,
			TaskfileVars: e.Taskfile.Vars,
			Expansions:   e.Taskfile.Expansions,
			Logger:       e.Logger,
		}
	} else {
		e.Compiler = &compilerv3.CompilerV3{
			Dir:          e.Dir,
			TaskfileEnv:  e.Taskfile.Env,
			TaskfileVars: e.Taskfile.Vars,
			Logger:       e.Logger,
		}
	}

	return nil
}

func (e *Executor) readDotEnvFiles(v float64) error {
	if v < 3.0 {
		return nil
	}

	env, err := read.Dotenv(e.Compiler, e.Taskfile, e.Dir)
	if err != nil {
		return err
	}

	err = env.Range(func(key string, value taskfile.Var) error {
		if _, ok := e.Taskfile.Env.Mapping[key]; !ok {
			e.Taskfile.Env.Set(key, value)
		}
		return nil
	})
	return err
}

func (e *Executor) setupDefaults(v float64) {
	// Color available only on v3
	if v < 3 {
		e.Logger.Color = false
	}

	if e.Taskfile.Method == "" {
		if v >= 3 {
			e.Taskfile.Method = "checksum"
		} else {
			e.Taskfile.Method = "timestamp"
		}
	}

	if e.Taskfile.Run == "" {
		e.Taskfile.Run = "always"
	}
}

func (e *Executor) setupConcurrencyState() {
	e.executionHashes = make(map[string]context.Context)

	e.taskCallCount = make(map[string]*int32, len(e.Taskfile.Tasks))
	e.mkdirMutexMap = make(map[string]*sync.Mutex, len(e.Taskfile.Tasks))
	for k := range e.Taskfile.Tasks {
		e.taskCallCount[k] = new(int32)
		e.mkdirMutexMap[k] = &sync.Mutex{}
	}

	if e.Concurrency > 0 {
		e.concurrencySemaphore = make(chan struct{}, e.Concurrency)
	}
}

func (e *Executor) doVersionChecks(v float64) error {
	if v < 2 {
		return fmt.Errorf(`task: Taskfile versions prior to v2 are not supported anymore`)
	}

	// consider as equal to the greater version if round
	if v == 2.0 {
		v = 2.6
	}
	if v == 3.0 {
		v = 3.8
	}

	if v > 3.8 {
		return fmt.Errorf(`task: Taskfile versions greater than v3.8 not implemented in the version of Task`)
	}

	if v < 2.1 && !e.Taskfile.Output.IsSet() {
		return fmt.Errorf(`task: Taskfile option "output" is only available starting on Taskfile version v2.1`)
	}
	if v < 2.2 && e.Taskfile.Includes.Len() > 0 {
		return fmt.Errorf(`task: Including Taskfiles is only available starting on Taskfile version v2.2`)
	}
	if v >= 3.0 && e.Taskfile.Expansions > 2 {
		return fmt.Errorf(`task: The "expansions" setting is not available anymore on v3.0`)
	}
	if v < 3.8 && e.Taskfile.Output.Group.IsSet() {
		return fmt.Errorf(`task: Taskfile option "output.group" is only available starting on Taskfile version v3.8`)
	}

	if v <= 2.1 {
		err := errors.New(`task: Taskfile option "ignore_error" is only available starting on Taskfile version v2.1`)

		for _, task := range e.Taskfile.Tasks {
			if task.IgnoreError {
				return err
			}
			for _, cmd := range task.Cmds {
				if cmd.IgnoreError {
					return err
				}
			}
		}
	}

	if v < 2.6 {
		for _, task := range e.Taskfile.Tasks {
			if len(task.Preconditions) > 0 {
				return errors.New(`task: Task option "preconditions" is only available starting on Taskfile version v2.6`)
			}
		}
	}

	if v < 3 {
		err := e.Taskfile.Includes.Range(func(_ string, taskfile taskfile.IncludedTaskfile) error {
			if taskfile.AdvancedImport {
				return errors.New(`task: Import with additional parameters is only available starting on Taskfile version v3`)
			}
			return nil
		})
		if err != nil {
			return err
		}
	}

	if v < 3.7 {
		if e.Taskfile.Run != "" {
			return errors.New(`task: Setting the "run" type is only available starting on Taskfile version v3.7`)
		}

		for _, task := range e.Taskfile.Tasks {
			if task.Run != "" {
				return errors.New(`task: Setting the "run" type is only available starting on Taskfile version v3.7`)
			}
		}
	}

	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package reports

import (
	"fmt"

	"k8s.io/contrib/mungegithub/github"

	"github.com/golang/glog"
	"github.com/spf13/cobra"
)

// Report is the interface which all reports must implement to register
type Report interface {
	// Take action on a specific github issue:
	Report(config *github.Config) error
	AddFlags(cmd *cobra.Command, config *github.Config)
	Name() string
}

var reportMap = map[string]Report{}
var reports = []Report{}

// GetAllReports returns a slice of all registered reports. This list is
// completely independant of the reports selected at runtime in --pr-reports.
// This is all possible reports.
func GetAllReports() []Report {
	out := []Report{}
	for _, report := range reportMap {
		out = append(out, report)
	}
	return out
}

// GetActiveReports returns a slice of all reports which both registered and
// were requested by the user
func GetActiveReports() []Report {
	return reports
}

// RegisterReport should be called in `init()` by each report to make itself
// available by name
func RegisterReport(report Report) error {
	if _, found := reportMap[report.Name()]; found {
		return fmt.Errorf("a report with that name (%s) already exists", report.Name())
	}
	reportMap[report.Name()] = report
	glog.Infof("Registered %#v at %s", report, report.Name())
	return nil
}

// RegisterReportOrDie will call RegisterReport but will be fatal on error
func RegisterReportOrDie(report Report) {
	if err := RegisterReport(report); err != nil {
		glog.Fatalf("Failed to register report: %s", err)
	}
}

// RunReports runs the specified reports.
func RunReports(cfg *github.Config, runReports ...string) error {
	for _, name := range runReports {
		report, ok := reportMap[name]
		if !ok {
			return fmt.Errorf("%v: not a valid report", name)
		}
		if err := report.Report(cfg); err != nil {
			return fmt.Errorf("Error running %v: %v", report.Name(), err)
		}
	}
	return nil
}
<commit_msg>fix a misspelling of independent in reports.go<commit_after>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package reports

import (
	"fmt"

	"k8s.io/contrib/mungegithub/github"

	"github.com/golang/glog"
	"github.com/spf13/cobra"
)

// Report is the interface which all reports must implement to register
type Report interface {
	// Take action on a specific github issue:
	Report(config *github.Config) error
	AddFlags(cmd *cobra.Command, config *github.Config)
	Name() string
}

var reportMap = map[string]Report{}
var reports = []Report{}

// GetAllReports returns a slice of all registered reports. This list is
// completely independent of the reports selected at runtime in --pr-reports.
// This is all possible reports.
func GetAllReports() []Report {
	out := []Report{}
	for _, report := range reportMap {
		out = append(out, report)
	}
	return out
}

// GetActiveReports returns a slice of all reports which both registered and
// were requested by the user
func GetActiveReports() []Report {
	return reports
}

// RegisterReport should be called in `init()` by each report to make itself
// available by name
func RegisterReport(report Report) error {
	if _, found := reportMap[report.Name()]; found {
		return fmt.Errorf("a report with that name (%s) already exists", report.Name())
	}
	reportMap[report.Name()] = report
	glog.Infof("Registered %#v at %s", report, report.Name())
	return nil
}

// RegisterReportOrDie will call RegisterReport but will be fatal on error
func RegisterReportOrDie(report Report) {
	if err := RegisterReport(report); err != nil {
		glog.Fatalf("Failed to register report: %s", err)
	}
}

// RunReports runs the specified reports.
func RunReports(cfg *github.Config, runReports ...string) error {
	for _, name := range runReports {
		report, ok := reportMap[name]
		if !ok {
			return fmt.Errorf("%v: not a valid report", name)
		}
		if err := report.Report(cfg); err != nil {
			return fmt.Errorf("Error running %v: %v", report.Name(), err)
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package topic

import (
	"koding/db/mongodb/modelhelper"
	"math"
	"socialapi/models"

	"socialapi/workers/common/runner"
	"testing"

	. "github.com/smartystreets/goconvey/convey"
)

func TestUnlink(t *testing.T) {
	r := runner.New("test-moderation-unlink")
	err := r.Init()
	if err != nil {
		panic(err)
	}

	defer r.Close()

	modelhelper.Initialize(r.Conf.Mongo)
	defer modelhelper.Close()

	CreatePrivateMessageUser()

	Convey("given a controller", t, func() {

		controller := NewController(r.Log)

		Convey("err should be nil", func() {
			So(err, ShouldBeNil)
		})

		Convey("controller should be set", func() {
			So(controller, ShouldNotBeNil)
		})

		Convey("should return nil when given nil channel link request", func() {
			So(controller.UnLink(nil), ShouldBeNil)
		})

		Convey("should return nil when account id given 0", func() {
			So(controller.UnLink(models.NewChannelLink()), ShouldBeNil)
		})

		Convey("non existing account should not give error", func() {
			a := models.NewChannelLink()
			a.Id = math.MaxInt64
			So(controller.UnLink(a), ShouldBeNil)
		})
	})
}
<commit_msg>Socialapi: remove unnecessary line<commit_after>package topic

import (
	"koding/db/mongodb/modelhelper"
	"math"
	"socialapi/models"
	"socialapi/workers/common/runner"
	"testing"

	. "github.com/smartystreets/goconvey/convey"
)

func TestUnlink(t *testing.T) {
	r := runner.New("test-moderation-unlink")
	err := r.Init()
	if err != nil {
		panic(err)
	}

	defer r.Close()

	modelhelper.Initialize(r.Conf.Mongo)
	defer modelhelper.Close()

	CreatePrivateMessageUser()

	Convey("given a controller", t, func() {

		controller := NewController(r.Log)

		Convey("err should be nil", func() {
			So(err, ShouldBeNil)
		})

		Convey("controller should be set", func() {
			So(controller, ShouldNotBeNil)
		})

		Convey("should return nil when given nil channel link request", func() {
			So(controller.UnLink(nil), ShouldBeNil)
		})

		Convey("should return nil when account id given 0", func() {
			So(controller.UnLink(models.NewChannelLink()), ShouldBeNil)
		})

		Convey("non existing account should not give error", func() {
			a := models.NewChannelLink()
			a.Id = math.MaxInt64
			So(controller.UnLink(a), ShouldBeNil)
		})
	})
}
<|endoftext|>


<commit_before>package benchmark

import (
	"fmt"
	"io/ioutil"
	"strings"
	"sync"
	"time"

	"github.com/lxc/lxd/client"
	"github.com/lxc/lxd/lxc/config"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/version"
)

// PrintServerInfo prints out information about the server.
func PrintServerInfo(c lxd.ContainerServer) error {
	server, _, err := c.GetServer()
	if err != nil {
		return err
	}
	env := server.Environment
	fmt.Printf("Test environment:\n")
	fmt.Printf("  Server backend: %s\n", env.Server)
	fmt.Printf("  Server version: %s\n", env.ServerVersion)
	fmt.Printf("  Kernel: %s\n", env.Kernel)
	fmt.Printf("  Kernel architecture: %s\n", env.KernelArchitecture)
	fmt.Printf("  Kernel version: %s\n", env.KernelVersion)
	fmt.Printf("  Storage backend: %s\n", env.Storage)
	fmt.Printf("  Storage version: %s\n", env.StorageVersion)
	fmt.Printf("  Container backend: %s\n", env.Driver)
	fmt.Printf("  Container version: %s\n", env.DriverVersion)
	fmt.Printf("\n")
	return nil
}

// SpawnContainers launches a set of containers.
func SpawnContainers(c lxd.ContainerServer, count int, parallel int, image string, privileged bool, freeze bool) error {
	batch := parallel
	if batch < 1 {
		// Detect the number of parallel actions
		cpus, err := ioutil.ReadDir("/sys/bus/cpu/devices")
		if err != nil {
			return err
		}

		batch = len(cpus)
	}

	printTestConfig(count, batch, image, privileged, freeze)

	batches := count / batch
	remainder := count % batch

	// Pre-load the image
	var fingerprint string
	if strings.Contains(image, ":") {
		var remote string
		var err error

		defaultConfig := config.DefaultConfig
		defaultConfig.UserAgent = version.UserAgent

		remote, fingerprint, err = defaultConfig.ParseRemote(image)
		if err != nil {
			return err
		}

		d, err := defaultConfig.GetImageServer(remote)
		if err != nil {
			return err
		}

		if fingerprint == "" {
			fingerprint = "default"
		}

		alias, _, err := d.GetImageAlias(fingerprint)
		if err == nil {
			fingerprint = alias.Target
		}

		_, _, err = c.GetImage(fingerprint)
		if err != nil {
			logf("Importing image into local store: %s", fingerprint)
			image, _, err := d.GetImage(fingerprint)
			if err != nil {
				logf(fmt.Sprintf("Failed to import image: %s", err))
				return err
			}

			op, err := c.CopyImage(d, *image, nil)
			if err != nil {
				logf(fmt.Sprintf("Failed to import image: %s", err))
				return err
			}

			err = op.Wait()
			if err != nil {
				logf(fmt.Sprintf("Failed to import image: %s", err))
				return err
			}
		} else {
			logf("Found image in local store: %s", fingerprint)
		}
	} else {
		fingerprint = image
		logf("Found image in local store: %s", fingerprint)
	}

	// Start the containers
	spawnedCount := 0
	nameFormat := "benchmark-%." + fmt.Sprintf("%d", len(fmt.Sprintf("%d", count))) + "d"
	wgBatch := sync.WaitGroup{}
	nextStat := batch

	startContainer := func(name string) {
		defer wgBatch.Done()

		// Configure
		config := map[string]string{}
		if privileged {
			config["security.privileged"] = "true"
		}
		config["user.lxd-benchmark"] = "true"

		// Create
		req := api.ContainersPost{
			Name: name,
			Source: api.ContainerSource{
				Type:        "image",
				Fingerprint: fingerprint,
			},
		}
		req.Config = config

		op, err := c.CreateContainer(req)
		if err != nil {
			logf(fmt.Sprintf("Failed to spawn container '%s': %s", name, err))
			return
		}

		err = op.Wait()
		if err != nil {
			logf(fmt.Sprintf("Failed to spawn container '%s': %s", name, err))
			return
		}

		// Start
		op, err = c.UpdateContainerState(name, api.ContainerStatePut{Action: "start", Timeout: -1}, "")
		if err != nil {
			logf(fmt.Sprintf("Failed to spawn container '%s': %s", name, err))
			return
		}

		err = op.Wait()
		if err != nil {
			logf(fmt.Sprintf("Failed to spawn container '%s': %s", name, err))
			return
		}

		// Freeze
		if freeze {
			op, err := c.UpdateContainerState(name, api.ContainerStatePut{Action: "freeze", Timeout: -1}, "")
			if err != nil {
				logf(fmt.Sprintf("Failed to spawn container '%s': %s", name, err))
				return
			}

			err = op.Wait()
			if err != nil {
				logf(fmt.Sprintf("Failed to spawn container '%s': %s", name, err))
				return
			}
		}
	}

	logf("Starting the test")
	timeStart := time.Now()

	for i := 0; i < batches; i++ {
		for j := 0; j < batch; j++ {
			spawnedCount = spawnedCount + 1
			name := fmt.Sprintf(nameFormat, spawnedCount)

			wgBatch.Add(1)
			go startContainer(name)
		}
		wgBatch.Wait()

		if spawnedCount >= nextStat {
			interval := time.Since(timeStart).Seconds()
			logf("Started %d containers in %.3fs (%.3f/s)", spawnedCount, interval, float64(spawnedCount)/interval)
			nextStat = nextStat * 2
		}
	}

	for k := 0; k < remainder; k++ {
		spawnedCount = spawnedCount + 1
		name := fmt.Sprintf(nameFormat, spawnedCount)

		wgBatch.Add(1)
		go startContainer(name)
	}
	wgBatch.Wait()

	logf("Test completed in %.3fs", time.Since(timeStart).Seconds())

	return nil
}

func DeleteContainers(c lxd.ContainerServer, parallel int) error {
	batch := parallel
	if batch < 1 {
		// Detect the number of parallel actions
		cpus, err := ioutil.ReadDir("/sys/bus/cpu/devices")
		if err != nil {
			return err
		}

		batch = len(cpus)
	}

	// List all the containers
	allContainers, err := c.GetContainers()
	if err != nil {
		return err
	}

	containers := []api.Container{}
	for _, container := range allContainers {
		if container.Config["user.lxd-benchmark"] != "true" {
			continue
		}

		containers = append(containers, container)
	}

	// Delete them all
	count := len(containers)
	logf("%d containers to delete", count)

	batches := count / batch

	deletedCount := 0
	wgBatch := sync.WaitGroup{}
	nextStat := batch

	deleteContainer := func(ct api.Container) {
		defer wgBatch.Done()

		// Stop
		if ct.IsActive() {
			op, err := c.UpdateContainerState(ct.Name, api.ContainerStatePut{Action: "stop", Timeout: -1, Force: true}, "")
			if err != nil {
				logf(fmt.Sprintf("Failed to delete container '%s': %s", ct.Name, err))
				return
			}

			err = op.Wait()
			if err != nil {
				logf(fmt.Sprintf("Failed to delete container '%s': %s", ct.Name, err))
				return
			}
		}

		// Delete
		op, err := c.DeleteContainer(ct.Name)
		if err != nil {
			logf("Failed to delete container: %s", ct.Name)
			return
		}

		err = op.Wait()
		if err != nil {
			logf("Failed to delete container: %s", ct.Name)
			return
		}
	}

	logf("Starting the cleanup")
	timeStart := time.Now()

	for i := 0; i < batches; i++ {
		for j := 0; j < batch; j++ {
			wgBatch.Add(1)
			go deleteContainer(containers[deletedCount])

			deletedCount = deletedCount + 1
		}
		wgBatch.Wait()

		if deletedCount >= nextStat {
			interval := time.Since(timeStart).Seconds()
			logf("Deleted %d containers in %.3fs (%.3f/s)", deletedCount, interval, float64(deletedCount)/interval)
			nextStat = nextStat * 2
		}
	}

	for k := deletedCount; k < count; k++ {
		wgBatch.Add(1)
		go deleteContainer(containers[deletedCount])

		deletedCount = deletedCount + 1
	}
	wgBatch.Wait()

	logf("Cleanup completed")

	return nil
}

func logf(format string, args ...interface{}) {
	fmt.Printf(fmt.Sprintf("[%s] %s\n", time.Now().Format(time.StampMilli), format), args...)
}

func printTestConfig(count int, batchSize int, image string, privileged bool, freeze bool) {
	privilegedStr := "unprivileged"
	if privileged {
		privilegedStr = "privileged"
	}
	mode := "normal startup"
	if freeze {
		mode = "start and freeze"
	}

	batches := count / batchSize
	remainder := count % batchSize
	fmt.Printf("Test variables:\n")
	fmt.Printf("  Container count: %d\n", count)
	fmt.Printf("  Container mode: %s\n", privilegedStr)
	fmt.Printf("  Startup mode: %s\n", mode)
	fmt.Printf("  Image: %s\n", image)
	fmt.Printf("  Batches: %d\n", batches)
	fmt.Printf("  Batch size: %d\n", batchSize)
	fmt.Printf("  Remainder: %d\n", remainder)
	fmt.Printf("\n")
}
<commit_msg>benchmark: extract getBatchSize function<commit_after>package benchmark

import (
	"fmt"
	"io/ioutil"
	"strings"
	"sync"
	"time"

	"github.com/lxc/lxd/client"
	"github.com/lxc/lxd/lxc/config"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/version"
)

// PrintServerInfo prints out information about the server.
func PrintServerInfo(c lxd.ContainerServer) error {
	server, _, err := c.GetServer()
	if err != nil {
		return err
	}
	env := server.Environment
	fmt.Printf("Test environment:\n")
	fmt.Printf("  Server backend: %s\n", env.Server)
	fmt.Printf("  Server version: %s\n", env.ServerVersion)
	fmt.Printf("  Kernel: %s\n", env.Kernel)
	fmt.Printf("  Kernel architecture: %s\n", env.KernelArchitecture)
	fmt.Printf("  Kernel version: %s\n", env.KernelVersion)
	fmt.Printf("  Storage backend: %s\n", env.Storage)
	fmt.Printf("  Storage version: %s\n", env.StorageVersion)
	fmt.Printf("  Container backend: %s\n", env.Driver)
	fmt.Printf("  Container version: %s\n", env.DriverVersion)
	fmt.Printf("\n")
	return nil
}

// SpawnContainers launches a set of containers.
func SpawnContainers(c lxd.ContainerServer, count int, parallel int, image string, privileged bool, freeze bool) error {
	batchSize, err := getBatchSize(parallel)
	if err != nil {
		return err
	}

	printTestConfig(count, batchSize, image, privileged, freeze)

	batches := count / batchSize
	remainder := count % batchSize

	// Pre-load the image
	var fingerprint string
	if strings.Contains(image, ":") {
		var remote string
		var err error

		defaultConfig := config.DefaultConfig
		defaultConfig.UserAgent = version.UserAgent

		remote, fingerprint, err = defaultConfig.ParseRemote(image)
		if err != nil {
			return err
		}

		d, err := defaultConfig.GetImageServer(remote)
		if err != nil {
			return err
		}

		if fingerprint == "" {
			fingerprint = "default"
		}

		alias, _, err := d.GetImageAlias(fingerprint)
		if err == nil {
			fingerprint = alias.Target
		}

		_, _, err = c.GetImage(fingerprint)
		if err != nil {
			logf("Importing image into local store: %s", fingerprint)
			image, _, err := d.GetImage(fingerprint)
			if err != nil {
				logf(fmt.Sprintf("Failed to import image: %s", err))
				return err
			}

			op, err := c.CopyImage(d, *image, nil)
			if err != nil {
				logf(fmt.Sprintf("Failed to import image: %s", err))
				return err
			}

			err = op.Wait()
			if err != nil {
				logf(fmt.Sprintf("Failed to import image: %s", err))
				return err
			}
		} else {
			logf("Found image in local store: %s", fingerprint)
		}
	} else {
		fingerprint = image
		logf("Found image in local store: %s", fingerprint)
	}

	// Start the containers
	spawnedCount := 0
	nameFormat := "benchmark-%." + fmt.Sprintf("%d", len(fmt.Sprintf("%d", count))) + "d"
	wgBatch := sync.WaitGroup{}
	nextStat := batchSize

	startContainer := func(name string) {
		defer wgBatch.Done()

		// Configure
		config := map[string]string{}
		if privileged {
			config["security.privileged"] = "true"
		}
		config["user.lxd-benchmark"] = "true"

		// Create
		req := api.ContainersPost{
			Name: name,
			Source: api.ContainerSource{
				Type:        "image",
				Fingerprint: fingerprint,
			},
		}
		req.Config = config

		op, err := c.CreateContainer(req)
		if err != nil {
			logf(fmt.Sprintf("Failed to spawn container '%s': %s", name, err))
			return
		}

		err = op.Wait()
		if err != nil {
			logf(fmt.Sprintf("Failed to spawn container '%s': %s", name, err))
			return
		}

		// Start
		op, err = c.UpdateContainerState(name, api.ContainerStatePut{Action: "start", Timeout: -1}, "")
		if err != nil {
			logf(fmt.Sprintf("Failed to spawn container '%s': %s", name, err))
			return
		}

		err = op.Wait()
		if err != nil {
			logf(fmt.Sprintf("Failed to spawn container '%s': %s", name, err))
			return
		}

		// Freeze
		if freeze {
			op, err := c.UpdateContainerState(name, api.ContainerStatePut{Action: "freeze", Timeout: -1}, "")
			if err != nil {
				logf(fmt.Sprintf("Failed to spawn container '%s': %s", name, err))
				return
			}

			err = op.Wait()
			if err != nil {
				logf(fmt.Sprintf("Failed to spawn container '%s': %s", name, err))
				return
			}
		}
	}

	logf("Starting the test")
	timeStart := time.Now()

	for i := 0; i < batches; i++ {
		for j := 0; j < batchSize; j++ {
			spawnedCount = spawnedCount + 1
			name := fmt.Sprintf(nameFormat, spawnedCount)

			wgBatch.Add(1)
			go startContainer(name)
		}
		wgBatch.Wait()

		if spawnedCount >= nextStat {
			interval := time.Since(timeStart).Seconds()
			logf("Started %d containers in %.3fs (%.3f/s)", spawnedCount, interval, float64(spawnedCount)/interval)
			nextStat = nextStat * 2
		}
	}

	for k := 0; k < remainder; k++ {
		spawnedCount = spawnedCount + 1
		name := fmt.Sprintf(nameFormat, spawnedCount)

		wgBatch.Add(1)
		go startContainer(name)
	}
	wgBatch.Wait()

	logf("Test completed in %.3fs", time.Since(timeStart).Seconds())

	return nil
}

// DeleteContainers removes containers created by the benchmark.
func DeleteContainers(c lxd.ContainerServer, parallel int) error {
	batchSize, err := getBatchSize(parallel)
	if err != nil {
		return err
	}

	// List all the containers
	allContainers, err := c.GetContainers()
	if err != nil {
		return err
	}

	containers := []api.Container{}
	for _, container := range allContainers {
		if container.Config["user.lxd-benchmark"] != "true" {
			continue
		}

		containers = append(containers, container)
	}

	// Delete them all
	count := len(containers)
	logf("%d containers to delete", count)

	batches := count / batchSize

	deletedCount := 0
	wgBatch := sync.WaitGroup{}
	nextStat := batchSize

	deleteContainer := func(ct api.Container) {
		defer wgBatch.Done()

		// Stop
		if ct.IsActive() {
			op, err := c.UpdateContainerState(ct.Name, api.ContainerStatePut{Action: "stop", Timeout: -1, Force: true}, "")
			if err != nil {
				logf(fmt.Sprintf("Failed to delete container '%s': %s", ct.Name, err))
				return
			}

			err = op.Wait()
			if err != nil {
				logf(fmt.Sprintf("Failed to delete container '%s': %s", ct.Name, err))
				return
			}
		}

		// Delete
		op, err := c.DeleteContainer(ct.Name)
		if err != nil {
			logf("Failed to delete container: %s", ct.Name)
			return
		}

		err = op.Wait()
		if err != nil {
			logf("Failed to delete container: %s", ct.Name)
			return
		}
	}

	logf("Starting the cleanup")
	timeStart := time.Now()

	for i := 0; i < batches; i++ {
		for j := 0; j < batchSize; j++ {
			wgBatch.Add(1)
			go deleteContainer(containers[deletedCount])

			deletedCount = deletedCount + 1
		}
		wgBatch.Wait()

		if deletedCount >= nextStat {
			interval := time.Since(timeStart).Seconds()
			logf("Deleted %d containers in %.3fs (%.3f/s)", deletedCount, interval, float64(deletedCount)/interval)
			nextStat = nextStat * 2
		}
	}

	for k := deletedCount; k < count; k++ {
		wgBatch.Add(1)
		go deleteContainer(containers[deletedCount])

		deletedCount = deletedCount + 1
	}
	wgBatch.Wait()

	logf("Cleanup completed")

	return nil
}

func getBatchSize(parallel int) (int, error) {
	batchSize := parallel
	if batchSize < 1 {
		// Detect the number of parallel actions
		cpus, err := ioutil.ReadDir("/sys/bus/cpu/devices")
		if err != nil {
			return -1, err
		}

		batchSize = len(cpus)
	}

	return batchSize, nil
}

func logf(format string, args ...interface{}) {
	fmt.Printf(fmt.Sprintf("[%s] %s\n", time.Now().Format(time.StampMilli), format), args...)
}

func printTestConfig(count int, batchSize int, image string, privileged bool, freeze bool) {
	privilegedStr := "unprivileged"
	if privileged {
		privilegedStr = "privileged"
	}
	mode := "normal startup"
	if freeze {
		mode = "start and freeze"
	}

	batches := count / batchSize
	remainder := count % batchSize
	fmt.Printf("Test variables:\n")
	fmt.Printf("  Container count: %d\n", count)
	fmt.Printf("  Container mode: %s\n", privilegedStr)
	fmt.Printf("  Startup mode: %s\n", mode)
	fmt.Printf("  Image: %s\n", image)
	fmt.Printf("  Batches: %d\n", batches)
	fmt.Printf("  Batch size: %d\n", batchSize)
	fmt.Printf("  Remainder: %d\n", remainder)
	fmt.Printf("\n")
}
<|endoftext|>