<commit_before>package main

import (
    "time"
    "log"
    "fmt"
    "strings"
    "sort"
    "text/tabwriter"
    "bytes"
)

var Logger *log.Logger = nil
var ErrLogger *log.Logger = nil

type JobberError struct {
    What  string
    Cause error
}

func (e *JobberError) Error() string {
    if e.Cause == nil {
        return e.What
    } else {
        return e.What + ":" + e.Cause.Error()
    }
}

type RunLogEntry struct {
    Job        *Job
    Time       time.Time
    Succeeded  bool
    Result     JobStatus
}

/* For sorting RunLogEntries: */
type runLogEntrySorter struct {
    entries []RunLogEntry
}

/* For sorting RunLogEntries: */
func (s *runLogEntrySorter) Len() int {
    return len(s.entries)
}

/* For sorting RunLogEntries: */
func (s *runLogEntrySorter) Swap(i, j int) {
    s.entries[i], s.entries[j] = s.entries[j], s.entries[i]
}

/* For sorting RunLogEntries: */
func (s *runLogEntrySorter) Less(i, j int) bool {
    return s.entries[i].Time.After(s.entries[j].Time)
}

type JobManager struct {
    jobs                  []*Job
    loadedJobs            bool
    runLog                []RunLogEntry
    cmdChan               chan ICmd
    mainThreadCtx         *JobberContext
    mainThreadCtl         JobberCtl
    jobRunner             *JobRunnerThread
    Shell                 string
}

func NewJobManager(infoLogger *log.Logger, errLogger *log.Logger) (*JobManager, error) {
    jm := JobManager{Shell: "/bin/sh"}
    Logger = infoLogger
    ErrLogger = errLogger
    jm.loadedJobs = false
    jm.jobRunner = NewJobRunnerThread()
    return &jm, nil
}

func (m *JobManager) jobsForUser(username string) []*Job {
    jobs := make([]*Job, 0)
    for _, job := range m.jobs {
        if username == job.User {
            jobs = append(jobs, job)
        }
    }
    return jobs
}

func (m *JobManager) runLogEntriesForUser(username string) []RunLogEntry {
    entries := make([]RunLogEntry, 0)
    for _, entry := range m.runLog {
        if username == entry.Job.User {
            entries = append(entries, entry)
        }
    }
    return entries
}

func (m *JobManager) Launch() (chan<- ICmd, error) {
    if m.mainThreadCtx != nil {
        return nil, &JobberError{"Already launched.", nil}
    }

    Logger.Println("Launching.")
    if !m.loadedJobs {
        _, err := m.LoadAllJobs()
        if err != nil {
            ErrLogger.Printf("Failed to load jobs: %v.\n", err)
            return nil, err
        }
    }

    // make main thread
    m.cmdChan = make(chan ICmd)
    m.runMainThread()
    return m.cmdChan, nil
}

func (m *JobManager) Cancel() {
    if m.mainThreadCtl.Cancel != nil {
        Logger.Printf("JobManager canceling\n")
        m.mainThreadCtl.Cancel()
    }
}

func (m *JobManager) Wait() {
    if m.mainThreadCtl.Wait != nil {
        m.mainThreadCtl.Wait()
    }
}

func (m *JobManager) handleRunRec(rec *RunRec) {
    if len(rec.Stdout) > 0 {
        Logger.Println(rec.Stdout)
    }
    if len(rec.Stderr) > 0 {
        ErrLogger.Println(rec.Stderr)
    }
    if rec.Err != nil {
        ErrLogger.Panicln(rec.Err)
    }

    m.runLog = append(m.runLog, RunLogEntry{rec.Job, rec.RunTime, rec.Succeeded, rec.NewStatus})

    /* NOTE: error-handler was already applied by the job, if necessary. */

    if (!rec.Succeeded && rec.Job.NotifyOnError) ||
        (rec.Job.NotifyOnFailure && rec.NewStatus == JobFailed) {
        // notify user
        headers := fmt.Sprintf("To: %v\r\nFrom: %v\r\nSubject: \"%v\" failed.", rec.Job.User, rec.Job.User, rec.Job.Name)
        bod := rec.Describe()
        msg := fmt.Sprintf("%s\r\n\r\n%s.\r\n", headers, bod)
        sendmailCmd := fmt.Sprintf("sendmail %v", rec.Job.User)
        sudoResult, err := sudo(rec.Job.User, sendmailCmd, "/bin/sh", &msg)
        if err != nil {
            ErrLogger.Println("Failed to send mail: %v", err)
        } else if !sudoResult.Succeeded {
            ErrLogger.Println("Failed to send mail: %v", sudoResult.Stderr)
        }
    }
}

func (m *JobManager) runMainThread() {
    m.mainThreadCtx, m.mainThreadCtl = NewJobberContext(BackgroundJobberContext())
    Logger.Printf("Main thread context: %v\n", m.mainThreadCtx.Name)

    go func() {
        /*
         All modifications to the job manager's state occur here.
        */

        // start job-runner thread
        m.jobRunner.Start(m.jobs, m.Shell, m.mainThreadCtx)

        Loop: for {
            select {
            case <-m.mainThreadCtx.Done():
                Logger.Printf("Main thread got 'stop'\n")
                break Loop

            case rec, ok := <-m.jobRunner.RunRecChan():
                if ok {
                    m.handleRunRec(rec)
                } else {
                    ErrLogger.Printf("Job-runner thread ended prematurely.\n")
                    break Loop
                }

            case cmd, ok := <-m.cmdChan:
                if ok {
                    //fmt.Printf("JobManager: processing cmd.\n")
                    shouldStop := m.doCmd(cmd)
                    if shouldStop {
                        break Loop
                    }
                } else {
                    ErrLogger.Printf("Command channel was closed.\n")
                    break Loop
                }
            }
        }

        // cancel main thread
        m.mainThreadCtl.Cancel()

        // consume all run-records
        for rec := range m.jobRunner.RunRecChan() {
            m.handleRunRec(rec)
        }

        // finish up (and wait for job-runner thread to finish)
        m.mainThreadCtx.Finish()

        Logger.Printf("Main Thread done.\n")
    }()
}

func (m *JobManager) doCmd(cmd ICmd) bool {  // runs in main thread

    /*
    Security:

    It is jobberd's responsibility to enforce the security policy.

    It does so by assuming that cmd.RequestingUser() is truly the name
    of the requesting user.
    */

    switch cmd.(type) {
    case *ReloadCmd:
        /* Policy: Only root can reload other users' jobfiles. */

        // load jobs
        var err error
        var amt int
        if cmd.(*ReloadCmd).ForAllUsers {
            if cmd.RequestingUser() != "root" {
                cmd.RespChan() <- &ErrorCmdResp{&JobberError{What: "You must be root."}}
                break
            }

            Logger.Printf("Reloading jobs for all users.\n")
            amt, err = m.ReloadAllJobs()
        } else {
            Logger.Printf("Reloading jobs for %v.\n", cmd.RequestingUser())
            amt, err = m.ReloadJobsForUser(cmd.RequestingUser())
        }

        // send response
        if err != nil {
            ErrLogger.Printf("Failed to load jobs: %v.\n", err)
            cmd.RespChan() <- &ErrorCmdResp{err}
        } else {
            cmd.RespChan() <- &SuccessCmdResp{fmt.Sprintf("Loaded %v jobs.", amt)}
        }

        return false

    case *CatCmd:
        /* Policy: Only root can cat other users' jobs. */

        var catCmd *CatCmd = cmd.(*CatCmd)

        // enfore policy
        if catCmd.jobUser != catCmd.RequestingUser() && catCmd.RequestingUser() != "root" {
            cmd.RespChan() <- &ErrorCmdResp{&JobberError{What: "You must be root."}}
            break
        }

        // find job to cat
        var job_p *Job
        for _, job := range m.jobsForUser(catCmd.jobUser) {
            if job.Name == catCmd.job {
                job_p = job
                break
            }
        }
        if job_p == nil {
            msg := fmt.Sprintf("No job named \"%v\".", catCmd.job)
            cmd.RespChan() <- &ErrorCmdResp{&JobberError{What: msg}}
            break
        }

        // make and send response
        cmd.RespChan() <- &SuccessCmdResp{job_p.Cmd}

        return false

    case *ListJobsCmd:
        /* Policy: Only root can list other users' jobs. */

        // get jobs
        var jobs []*Job
        if cmd.(*ListJobsCmd).ForAllUsers {
            if cmd.RequestingUser() != "root" {
                cmd.RespChan() <- &ErrorCmdResp{&JobberError{What: "You must be root."}}
                break
            }

            jobs = m.jobs
        } else {
            jobs = m.jobsForUser(cmd.RequestingUser())
        }

        // make response
        var buffer bytes.Buffer
        var writer *tabwriter.Writer = tabwriter.NewWriter(&buffer, 5, 0, 2, ' ', 0)
        fmt.Fprintf(writer, "NAME\tSTATUS\tSEC/MIN/HR/MDAY/MTH/WDAY\tNEXT RUN TIME\tNOTIFY ON ERR\tNOTIFY ON FAIL\tERR HANDLER\n")
        strs := make([]string, 0, len(m.jobs))
        for _, j := range jobs {
            schedStr := fmt.Sprintf("%v %v %v %v %v %v",
                                    j.FullTimeSpec.Sec,
                                    j.FullTimeSpec.Min,
                                    j.FullTimeSpec.Hour,
                                    j.FullTimeSpec.Mday,
                                    j.FullTimeSpec.Mon,
                                    j.FullTimeSpec.Wday)
            var runTimeStr string = "unknown"
            if j.NextRunTime != nil {
                runTimeStr = j.NextRunTime.Format("Jan _2 15:04:05 2006")
            }
            s := fmt.Sprintf("%v\t%v\t%v\t%v\t%v\t%v\t%v",
                               j.Name,
                               j.Status,
                               schedStr,
                               runTimeStr,
                               j.NotifyOnError,
                               j.NotifyOnFailure,
                               j.ErrorHandler)
            strs = append(strs, s)
        }
        fmt.Fprintf(writer, "%v", strings.Join(strs, "\n"))
        writer.Flush()

        // send response
        cmd.RespChan() <- &SuccessCmdResp{buffer.String()}

        return false

    case *ListHistoryCmd:
        /* Policy: Only root can see the histories of other users' jobs. */

        // get log entries
        var entries []RunLogEntry
        if cmd.(*ListHistoryCmd).ForAllUsers {
            if cmd.RequestingUser() != "root" {
                cmd.RespChan() <- &ErrorCmdResp{&JobberError{What: "You must be root."}}
                break
            }

            entries = m.runLog
        } else {
            entries = m.runLogEntriesForUser(cmd.RequestingUser())
        }
        sort.Sort(&runLogEntrySorter{entries})

        // make response
        var buffer bytes.Buffer
        var writer *tabwriter.Writer = tabwriter.NewWriter(&buffer, 5, 0, 2, ' ', 0)
        fmt.Fprintf(writer, "TIME\tJOB\tUSER\tSUCCEEDED\tRESULT\t\n")
        strs := make([]string, 0, len(m.jobs))
        for _, e := range entries {
            s := fmt.Sprintf("%v\t%v\t%v\t%v\t%v\t", e.Time, e.Job.Name, e.Job.User, e.Succeeded, e.Result)
            strs = append(strs, s)
        }
        fmt.Fprintf(writer, "%v", strings.Join(strs, "\n"))
        writer.Flush()

        // send response
        cmd.RespChan() <- &SuccessCmdResp{buffer.String()}

        return false

    case *StopCmd:
        /* Policy: Only root can stop jobberd. */

        if cmd.RequestingUser() != "root" {
            cmd.RespChan() <- &ErrorCmdResp{&JobberError{What: "You must be root."}}
            break
        }

        Logger.Println("Stopping.")
        return true

    case *TestCmd:
        /* Policy: Only root can test other users' jobs. */

        var testCmd *TestCmd = cmd.(*TestCmd)

        // enfore policy
        if testCmd.jobUser != testCmd.RequestingUser() && testCmd.RequestingUser() != "root" {
            cmd.RespChan() <- &ErrorCmdResp{&JobberError{What: "You must be root."}}
            break
        }

        // find job to test
        var job_p *Job
        for _, job := range m.jobsForUser(testCmd.jobUser) {
            if job.Name == testCmd.job {
                job_p = job
                break
            }
        }
        if job_p == nil {
            msg := fmt.Sprintf("No job named \"%v\".", testCmd.job)
            cmd.RespChan() <- &ErrorCmdResp{&JobberError{What: msg}}
            break
        }

        // run the job in this thread
        runRec := job_p.Run(nil, m.Shell, true)

        // send response
        if runRec.Err != nil {
            cmd.RespChan() <- &ErrorCmdResp{runRec.Err}
            break
        }
        cmd.RespChan() <- &SuccessCmdResp{Details: runRec.Describe()}

        return false

    default:
        cmd.RespChan() <- &ErrorCmdResp{&JobberError{What: "Unknown command."}}
        return false
    }

    return false
}
<commit_msg>Bugfix: #21<commit_after>package main

import (
    "time"
    "log"
    "fmt"
    "strings"
    "sort"
    "text/tabwriter"
    "bytes"
)

var Logger *log.Logger = nil
var ErrLogger *log.Logger = nil

type JobberError struct {
    What  string
    Cause error
}

func (e *JobberError) Error() string {
    if e.Cause == nil {
        return e.What
    } else {
        return e.What + ":" + e.Cause.Error()
    }
}

type RunLogEntry struct {
    Job        *Job
    Time       time.Time
    Succeeded  bool
    Result     JobStatus
}

/* For sorting RunLogEntries: */
type runLogEntrySorter struct {
    entries []RunLogEntry
}

/* For sorting RunLogEntries: */
func (s *runLogEntrySorter) Len() int {
    return len(s.entries)
}

/* For sorting RunLogEntries: */
func (s *runLogEntrySorter) Swap(i, j int) {
    s.entries[i], s.entries[j] = s.entries[j], s.entries[i]
}

/* For sorting RunLogEntries: */
func (s *runLogEntrySorter) Less(i, j int) bool {
    return s.entries[i].Time.After(s.entries[j].Time)
}

type JobManager struct {
    jobs                  []*Job
    loadedJobs            bool
    runLog                []RunLogEntry
    cmdChan               chan ICmd
    mainThreadCtx         *JobberContext
    mainThreadCtl         JobberCtl
    jobRunner             *JobRunnerThread
    Shell                 string
}

func NewJobManager(infoLogger *log.Logger, errLogger *log.Logger) (*JobManager, error) {
    jm := JobManager{Shell: "/bin/sh"}
    Logger = infoLogger
    ErrLogger = errLogger
    jm.loadedJobs = false
    jm.jobRunner = NewJobRunnerThread()
    return &jm, nil
}

func (m *JobManager) jobsForUser(username string) []*Job {
    jobs := make([]*Job, 0)
    for _, job := range m.jobs {
        if username == job.User {
            jobs = append(jobs, job)
        }
    }
    return jobs
}

func (m *JobManager) runLogEntriesForUser(username string) []RunLogEntry {
    entries := make([]RunLogEntry, 0)
    for _, entry := range m.runLog {
        if username == entry.Job.User {
            entries = append(entries, entry)
        }
    }
    return entries
}

func (m *JobManager) Launch() (chan<- ICmd, error) {
    if m.mainThreadCtx != nil {
        return nil, &JobberError{"Already launched.", nil}
    }

    Logger.Println("Launching.")
    if !m.loadedJobs {
        _, err := m.LoadAllJobs()
        if err != nil {
            ErrLogger.Printf("Failed to load jobs: %v.\n", err)
            return nil, err
        }
    }

    // make main thread
    m.cmdChan = make(chan ICmd)
    m.runMainThread()
    return m.cmdChan, nil
}

func (m *JobManager) Cancel() {
    if m.mainThreadCtl.Cancel != nil {
        Logger.Printf("JobManager canceling\n")
        m.mainThreadCtl.Cancel()
    }
}

func (m *JobManager) Wait() {
    if m.mainThreadCtl.Wait != nil {
        m.mainThreadCtl.Wait()
    }
}

func (m *JobManager) handleRunRec(rec *RunRec) {
    if len(rec.Stdout) > 0 {
        Logger.Println(rec.Stdout)
    }
    if len(rec.Stderr) > 0 {
        ErrLogger.Println(rec.Stderr)
    }
    if rec.Err != nil {
        ErrLogger.Panicln(rec.Err)
    }

    m.runLog = append(m.runLog, RunLogEntry{rec.Job, rec.RunTime, rec.Succeeded, rec.NewStatus})

    /* NOTE: error-handler was already applied by the job, if necessary. */

    if (!rec.Succeeded && rec.Job.NotifyOnError) ||
        (rec.Job.NotifyOnFailure && rec.NewStatus == JobFailed) {
        // notify user
        headers := fmt.Sprintf("To: %v\r\nFrom: %v\r\nSubject: \"%v\" failed.", rec.Job.User, rec.Job.User, rec.Job.Name)
        bod := rec.Describe()
        msg := fmt.Sprintf("%s\r\n\r\n%s.\r\n", headers, bod)
        sendmailCmd := fmt.Sprintf("sendmail %v", rec.Job.User)
        sudoResult, err := sudo(rec.Job.User, sendmailCmd, "/bin/sh", &msg)
        if err != nil {
            ErrLogger.Println("Failed to send mail: %v", err)
        } else if !sudoResult.Succeeded {
            ErrLogger.Println("Failed to send mail: %v", sudoResult.Stderr)
        }
    }
}

func (m *JobManager) runMainThread() {
    m.mainThreadCtx, m.mainThreadCtl = NewJobberContext(BackgroundJobberContext())
    Logger.Printf("Main thread context: %v\n", m.mainThreadCtx.Name)

    go func() {
        /*
         All modifications to the job manager's state occur here.
        */

        // start job-runner thread
        m.jobRunner.Start(m.jobs, m.Shell, m.mainThreadCtx)

        Loop: for {
            select {
            case <-m.mainThreadCtx.Done():
                Logger.Printf("Main thread got 'stop'\n")
                break Loop

            case rec, ok := <-m.jobRunner.RunRecChan():
                if ok {
                    m.handleRunRec(rec)
                } else {
                    ErrLogger.Printf("Job-runner thread ended prematurely.\n")
                    break Loop
                }

            case cmd, ok := <-m.cmdChan:
                if ok {
                    //fmt.Printf("JobManager: processing cmd.\n")
                    shouldStop := m.doCmd(cmd)
                    if shouldStop {
                        break Loop
                    }
                } else {
                    ErrLogger.Printf("Command channel was closed.\n")
                    break Loop
                }
            }
        }

        // cancel main thread
        m.mainThreadCtl.Cancel()

        // consume all run-records
        for rec := range m.jobRunner.RunRecChan() {
            m.handleRunRec(rec)
        }

        // finish up (and wait for job-runner thread to finish)
        m.mainThreadCtx.Finish()

        Logger.Printf("Main Thread done.\n")
    }()
}

func (m *JobManager) doCmd(cmd ICmd) bool {  // runs in main thread

    /*
    Security:

    It is jobberd's responsibility to enforce the security policy.

    It does so by assuming that cmd.RequestingUser() is truly the name
    of the requesting user.
    */

    switch cmd.(type) {
    case *ReloadCmd:
        /* Policy: Only root can reload other users' jobfiles. */

        // load jobs
        var err error
        var amt int
        if cmd.(*ReloadCmd).ForAllUsers {
            if cmd.RequestingUser() != "root" {
                cmd.RespChan() <- &ErrorCmdResp{&JobberError{What: "You must be root."}}
                break
            }

            Logger.Printf("Reloading jobs for all users.\n")
            amt, err = m.ReloadAllJobs()
        } else {
            Logger.Printf("Reloading jobs for %v.\n", cmd.RequestingUser())
            amt, err = m.ReloadJobsForUser(cmd.RequestingUser())
        }

        // send response
        if err != nil {
            ErrLogger.Printf("Failed to load jobs: %v.\n", err)
            cmd.RespChan() <- &ErrorCmdResp{err}
        } else {
            cmd.RespChan() <- &SuccessCmdResp{fmt.Sprintf("Loaded %v jobs.", amt)}
        }

        return false

    case *CatCmd:
        /* Policy: Only root can cat other users' jobs. */

        var catCmd *CatCmd = cmd.(*CatCmd)

        // enfore policy
        if catCmd.jobUser != catCmd.RequestingUser() && catCmd.RequestingUser() != "root" {
            cmd.RespChan() <- &ErrorCmdResp{&JobberError{What: "You must be root."}}
            break
        }

        // find job to cat
        var job_p *Job
        for _, job := range m.jobsForUser(catCmd.jobUser) {
            if job.Name == catCmd.job {
                job_p = job
                break
            }
        }
        if job_p == nil {
            msg := fmt.Sprintf("No job named \"%v\".", catCmd.job)
            cmd.RespChan() <- &ErrorCmdResp{&JobberError{What: msg}}
            break
        }

        // make and send response
        cmd.RespChan() <- &SuccessCmdResp{job_p.Cmd}

        return false

    case *ListJobsCmd:
        /* Policy: Only root can list other users' jobs. */

        // get jobs
        var jobs []*Job
        if cmd.(*ListJobsCmd).ForAllUsers {
            if cmd.RequestingUser() != "root" {
                cmd.RespChan() <- &ErrorCmdResp{&JobberError{What: "You must be root."}}
                break
            }

            jobs = m.jobs
        } else {
            jobs = m.jobsForUser(cmd.RequestingUser())
        }

        // make response
        var buffer bytes.Buffer
        var writer *tabwriter.Writer = tabwriter.NewWriter(&buffer, 5, 0, 2, ' ', 0)
        fmt.Fprintf(writer, "NAME\tSTATUS\tSEC/MIN/HR/MDAY/MTH/WDAY\tNEXT RUN TIME\tNOTIFY ON ERR\tNOTIFY ON FAIL\tERR HANDLER\n")
        strs := make([]string, 0, len(m.jobs))
        for _, j := range jobs {
            schedStr := fmt.Sprintf("%v %v %v %v %v %v",
                                    j.FullTimeSpec.Sec,
                                    j.FullTimeSpec.Min,
                                    j.FullTimeSpec.Hour,
                                    j.FullTimeSpec.Mday,
                                    j.FullTimeSpec.Mon,
                                    j.FullTimeSpec.Wday)
            var runTimeStr string = "none"
            if j.NextRunTime != nil {
                runTimeStr = j.NextRunTime.Format("Jan _2 15:04:05 2006")
            }
            s := fmt.Sprintf("%v\t%v\t%v\t%v\t%v\t%v\t%v",
                               j.Name,
                               j.Status,
                               schedStr,
                               runTimeStr,
                               j.NotifyOnError,
                               j.NotifyOnFailure,
                               j.ErrorHandler)
            strs = append(strs, s)
        }
        fmt.Fprintf(writer, "%v", strings.Join(strs, "\n"))
        writer.Flush()

        // send response
        cmd.RespChan() <- &SuccessCmdResp{buffer.String()}

        return false

    case *ListHistoryCmd:
        /* Policy: Only root can see the histories of other users' jobs. */

        // get log entries
        var entries []RunLogEntry
        if cmd.(*ListHistoryCmd).ForAllUsers {
            if cmd.RequestingUser() != "root" {
                cmd.RespChan() <- &ErrorCmdResp{&JobberError{What: "You must be root."}}
                break
            }

            entries = m.runLog
        } else {
            entries = m.runLogEntriesForUser(cmd.RequestingUser())
        }
        sort.Sort(&runLogEntrySorter{entries})

        // make response
        var buffer bytes.Buffer
        var writer *tabwriter.Writer = tabwriter.NewWriter(&buffer, 5, 0, 2, ' ', 0)
        fmt.Fprintf(writer, "TIME\tJOB\tUSER\tSUCCEEDED\tRESULT\t\n")
        strs := make([]string, 0, len(m.jobs))
        for _, e := range entries {
            s := fmt.Sprintf("%v\t%v\t%v\t%v\t%v\t", e.Time, e.Job.Name, e.Job.User, e.Succeeded, e.Result)
            strs = append(strs, s)
        }
        fmt.Fprintf(writer, "%v", strings.Join(strs, "\n"))
        writer.Flush()

        // send response
        cmd.RespChan() <- &SuccessCmdResp{buffer.String()}

        return false

    case *StopCmd:
        /* Policy: Only root can stop jobberd. */

        if cmd.RequestingUser() != "root" {
            cmd.RespChan() <- &ErrorCmdResp{&JobberError{What: "You must be root."}}
            break
        }

        Logger.Println("Stopping.")
        return true

    case *TestCmd:
        /* Policy: Only root can test other users' jobs. */

        var testCmd *TestCmd = cmd.(*TestCmd)

        // enfore policy
        if testCmd.jobUser != testCmd.RequestingUser() && testCmd.RequestingUser() != "root" {
            cmd.RespChan() <- &ErrorCmdResp{&JobberError{What: "You must be root."}}
            break
        }

        // find job to test
        var job_p *Job
        for _, job := range m.jobsForUser(testCmd.jobUser) {
            if job.Name == testCmd.job {
                job_p = job
                break
            }
        }
        if job_p == nil {
            msg := fmt.Sprintf("No job named \"%v\".", testCmd.job)
            cmd.RespChan() <- &ErrorCmdResp{&JobberError{What: msg}}
            break
        }

        // run the job in this thread
        runRec := job_p.Run(nil, m.Shell, true)

        // send response
        if runRec.Err != nil {
            cmd.RespChan() <- &ErrorCmdResp{runRec.Err}
            break
        }
        cmd.RespChan() <- &SuccessCmdResp{Details: runRec.Describe()}

        return false

    default:
        cmd.RespChan() <- &ErrorCmdResp{&JobberError{What: "Unknown command."}}
        return false
    }

    return false
}
<|endoftext|>


<commit_before>// +build darwin

package darwin

import (
	"bufio"
	"bytes"
	"os/exec"
	"strconv"
	"strings"

	"github.com/mackerelio/mackerel-agent/logging"
)

// CPUGenerator Collects CPU specs
type CPUGenerator struct {
}

// Key XXX
func (g *CPUGenerator) Key() string {
	return "cpu"
}

var cpuLogger = logging.GetLogger("spec.cpu")

type cpuSpec map[string]interface{}

var sysCtlKeyMap = map[string]string{
	"core_count":   "cores",
	"brand_string": "model_name",
	"model":        "model",
	"vendor":       "vendor_id",
	"family":       "family",
	"stepping":     "stepping",
}

func (g *CPUGenerator) parseSysCtlBytes(res []byte) (cpuSpec, error) {
	scanner := bufio.NewScanner(bytes.NewBuffer(res))

	results := cpuSpec{}

	for scanner.Scan() {
		line := scanner.Text()
		kv := strings.SplitN(line, ":", 2)
		if len(kv) < 2 {
			continue
		}
		key := strings.TrimPrefix(strings.TrimSpace(kv[0]), "machdep.cpu.")
		val := strings.TrimSpace(kv[1])
		if label, ok := sysCtlKeyMap[key]; ok {
			results[label] = val
		}
	}

	if err := scanner.Err(); err != nil {
		cpuLogger.Errorf("Failed (skip this spec): %s", err)
		return nil, err
	}
	return results, nil
}

func (g *CPUGenerator) getCoreCount() (*int, error) {
	coreCountBytes, err := exec.Command("sysctl", "-n", "hw.logicalcpu").Output()
	if err != nil {
		cpuLogger.Errorf("Failed: %s", err)
		return nil, err
	}
	coreCount, err := strconv.Atoi(strings.TrimSpace(string(coreCountBytes)))
	if err != nil {
		cpuLogger.Errorf("Failed to parse: %s", err)
		return nil, err
	}
	return &coreCount, nil
}

// MEMO: sysctl -a machdep.cpu.brand_string

// Generate collects CPU specs.
// Returns an array of cpuSpec.
// Each spec is expected to have keys below:
// - model_name (used in Web)
// - vendor_id
// - family
// - model
// - stepping
// - physical_id
// - core_id
// - cores
// - mhz
// - cache_size
// - flags
func (g *CPUGenerator) Generate() (interface{}, error) {
	cpuInfoBytes, err := exec.Command("sysctl", "-a", "machdep.cpu").Output()
	coreInfo, err := g.parseSysCtlBytes(cpuInfoBytes)
	if err != nil {
		cpuLogger.Errorf("Failed: %s", err)
		return nil, err
	}
	coreCount, err := g.getCoreCount()
	if err != nil {
		cpuLogger.Errorf("Failed: %s", err)
		return nil, err
	}
	results := make([]cpuSpec, *coreCount)
	for i := 0; i < *coreCount; i++ {
		results[i] = coreInfo
	}
	return results, nil
}
<commit_msg>rename<commit_after>// +build darwin

package darwin

import (
	"bufio"
	"bytes"
	"os/exec"
	"strconv"
	"strings"

	"github.com/mackerelio/mackerel-agent/logging"
)

// CPUGenerator Collects CPU specs
type CPUGenerator struct {
}

// Key XXX
func (g *CPUGenerator) Key() string {
	return "cpu"
}

var cpuLogger = logging.GetLogger("spec.cpu")

type cpuSpec map[string]interface{}

var sysCtlKeyMap = map[string]string{
	"core_count":   "cores",
	"brand_string": "model_name",
	"model":        "model",
	"vendor":       "vendor_id",
	"family":       "family",
	"stepping":     "stepping",
}

func (g *CPUGenerator) parseSysCtlBytes(res []byte) (cpuSpec, error) {
	scanner := bufio.NewScanner(bytes.NewBuffer(res))

	results := cpuSpec{}

	for scanner.Scan() {
		line := scanner.Text()
		kv := strings.SplitN(line, ":", 2)
		if len(kv) < 2 {
			continue
		}
		key := strings.TrimPrefix(strings.TrimSpace(kv[0]), "machdep.cpu.")
		val := strings.TrimSpace(kv[1])
		if label, ok := sysCtlKeyMap[key]; ok {
			results[label] = val
		}
	}

	if err := scanner.Err(); err != nil {
		cpuLogger.Errorf("Failed (skip this spec): %s", err)
		return nil, err
	}
	return results, nil
}

func (g *CPUGenerator) getCpuCount() (*int, error) {
	countBytes, err := exec.Command("sysctl", "-n", "hw.logicalcpu").Output()
	if err != nil {
		cpuLogger.Errorf("Failed: %s", err)
		return nil, err
	}
	count, err := strconv.Atoi(strings.TrimSpace(string(countBytes)))
	if err != nil {
		cpuLogger.Errorf("Failed to parse: %s", err)
		return nil, err
	}
	return &count, nil
}

// MEMO: sysctl -a machdep.cpu

// Generate collects CPU specs.
// Returns an array of cpuSpec.
// Each spec is expected to have keys below:
// - model_name (used in Web)
// - vendor_id
// - family
// - model
// - stepping
// - physical_id
// - core_id
// - cores
// - mhz
// - cache_size
// - flags
func (g *CPUGenerator) Generate() (interface{}, error) {
	cpuInfoBytes, err := exec.Command("sysctl", "-a", "machdep.cpu").Output()
	cpuInfo, err := g.parseSysCtlBytes(cpuInfoBytes)
	if err != nil {
		cpuLogger.Errorf("Failed: %s", err)
		return nil, err
	}
	cpuCount, err := g.getCpuCount()
	if err != nil {
		cpuLogger.Errorf("Failed: %s", err)
		return nil, err
	}
	results := make([]cpuSpec, *cpuCount)
	for i := 0; i < *cpuCount; i++ {
		results[i] = cpuInfo
	}
	return results, nil
}
<|endoftext|>


<commit_before>// Copyright 2018 Google LLC. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cloudspanner

import (
	"bytes"
	"compress/gzip"
	"context"
	"encoding/base64"
	"flag"
	"io/ioutil"
	"sync"
	"time"

	"cloud.google.com/go/spanner"
	"github.com/golang/glog"
	"github.com/google/trillian/monitoring"
	"github.com/google/trillian/storage"
	"google.golang.org/api/option"
)

var (
	csURI                                = flag.String("cloudspanner_uri", "", "Connection URI for CloudSpanner database")
	csNumChannels                        = flag.Int("cloudspanner_num_channels", 0, "Number of gRPC channels to use to talk to CloudSpanner.")
	csSessionMaxOpened                   = flag.Uint64("cloudspanner_max_open_sessions", 0, "Max open sessions.")
	csSessionMinOpened                   = flag.Uint64("cloudspanner_min_open_sessions", 0, "Min open sessions.")
	csSessionMaxIdle                     = flag.Uint64("cloudspanner_max_idle_sessions", 0, "Max idle sessions.")
	csSessionMaxBurst                    = flag.Uint64("cloudspanner_max_burst_sessions", 0, "Max concurrent create session requests.")
	csSessionWriteSessions               = flag.Float64("cloudspanner_write_sessions", 0, "Fraction of write capable sessions to maintain.")
	csSessionHCWorkers                   = flag.Int("cloudspanner_num_healthcheckers", 0, "Number of health check workers for Spanner session pool.")
	csSessionHCInterval                  = flag.Duration("cloudspanner_healthcheck_interval", 0, "Interval betweek pinging sessions.")
	csDequeueAcrossMerkleBucketsFraction = flag.Float64("cloudspanner_dequeue_bucket_fraction", 0.75, "Fraction of merkle keyspace to dequeue from, set to zero to disable.")
	csReadOnlyStaleness                  = flag.Duration("cloudspanner_readonly_staleness", time.Minute, "How far in the past to perform readonly operations. Within limits, raising this should help to increase performance/reduce latency.")

	csMu              sync.RWMutex
	csStorageInstance *cloudSpannerProvider
	warnOnce          sync.Once
)

func init() {
	if err := storage.RegisterProvider("cloud_spanner", newCloudSpannerStorageProvider); err != nil {
		panic(err)
	}
}

func warn() {
	warnOnce.Do(func() {
		w := `H4sIAAAAAAAA/4xUsW7rMAzc8xUE2lE41B2sWlzsZ3TwoKEIgkQZ64mLsxga8vUPlG3FrZ2Hd1Ng3onHE5UDPQOEmVnwjCGhjyLC8RLcPgfhIvmwot8/CaHF9CMdOthdGmKvdSQQET85TqxJtKzjgnd4mYaFilDIlmhsnKql977mZSqzYcLy5K/zCUX66sbtNOAwteTXiVph5m4nigGzzUH7e3+a3XIRf5PFyhQQEV6UXLeY8nL292gyujlMIlIbdcUwet9ieBx/snWIOXkievPenyOMiDjnjOHj+MMJhjZfFBFpHF+AcQkGpr9f1nz02YoKcPXed5nvjHG2DGtB//7gGwHCq69HIPMBGa7hIYi3mPlOBOhf/Z8eMBmAdNVjZKlCFuiQgK19Y1YKrXDT5KWX7ohVC+cArnUKwGAF/rwvk6CrVhJ1DuDDF9igfVtEuFf8U2M0MXW4wf28pBy/4yOuOaLZw2+Qa76m5PpSFy+5N0usbnyr66+AjY7cx3eKz5VHrZpFlqL6nJa82+gI/H3Vh+TKm9Fmib7I5GXSvcStTQrndxwIw4dQvpak00DGpKvbnVgIxXk4kD31oLnTSkgkxchmJ01Vnj7lQLZFXrV532bpfqLJbTzqfygXrLHkh/CoP5Hq13DXJYuV3fD/DcRbm+5f7s1tvNj/RLBD9T6vNbi9dYpT05QTKsV1B+Ut4m8AAAD///IJ0vhIBgAA`

		wd, _ := base64.StdEncoding.DecodeString(w)
		b := bytes.NewReader(wd)
		r, _ := gzip.NewReader(b)
		if err := r.Close(); err != nil {
			// No need to exit, it's an unlikely error and doesn't affect operation.
			glog.Warningf("Close()=%v", err)
		}
		t, _ := ioutil.ReadAll(r)
		glog.Warningf("WARNING\n%s\nCloudspanner is an experimental storage implementation, and only supports Logs currently.", string(t))
	})
}

type cloudSpannerProvider struct {
	client *spanner.Client
}

func configFromFlags() spanner.ClientConfig {
	r := spanner.ClientConfig{}
	setUint64IfNotDefault(&r.SessionPoolConfig.MaxOpened, *csSessionMaxOpened)
	setUint64IfNotDefault(&r.SessionPoolConfig.MinOpened, *csSessionMinOpened)
	setUint64IfNotDefault(&r.SessionPoolConfig.MaxIdle, *csSessionMaxIdle)
	setUint64IfNotDefault(&r.SessionPoolConfig.MaxBurst, *csSessionMaxBurst)
	setFloat64IfNotDefault(&r.SessionPoolConfig.WriteSessions, *csSessionWriteSessions)
	setIntIfNotDefault(&r.SessionPoolConfig.HealthCheckWorkers, *csSessionHCWorkers)
	r.SessionPoolConfig.HealthCheckInterval = *csSessionHCInterval
	return r
}

func optionsFromFlags() []option.ClientOption {
	opts := []option.ClientOption{}
	if numConns := *csNumChannels; numConns != 0 {
		opts = append(opts, option.WithGRPCConnectionPool(numConns))
	}
	return opts
}

func newCloudSpannerStorageProvider(_ monitoring.MetricFactory) (storage.Provider, error) {
	csMu.Lock()
	defer csMu.Unlock()

	if csStorageInstance != nil {
		return csStorageInstance, nil
	}

	client, err := spanner.NewClientWithConfig(context.TODO(), *csURI, configFromFlags(), optionsFromFlags()...)
	if err != nil {
		return nil, err
	}
	csStorageInstance = &cloudSpannerProvider{
		client: client,
	}
	return csStorageInstance, nil
}

// LogStorage builds and returns a new storage.LogStorage using CloudSpanner.
func (s *cloudSpannerProvider) LogStorage() storage.LogStorage {
	warn()
	opts := LogStorageOptions{}
	frac := *csDequeueAcrossMerkleBucketsFraction
	if frac > 1.0 {
		frac = 1.0
	}
	if frac > 0 {
		opts.DequeueAcrossMerkleBuckets = true
		opts.DequeueAcrossMerkleBucketsRangeFraction = frac
	}
	if *csReadOnlyStaleness > 0 {
		opts.ReadOnlyStaleness = *csReadOnlyStaleness
	}
	return NewLogStorageWithOpts(s.client, opts)
}

// MapStorage builds and returns a new storage.MapStorage using CloudSpanner.
func (s *cloudSpannerProvider) MapStorage() storage.MapStorage {
	warn()
	opts := MapStorageOptions{}
	if *csReadOnlyStaleness > 0 {
		opts.ReadOnlyStaleness = *csReadOnlyStaleness
	}
	return NewMapStorageWithOpts(s.client, opts)
}

// AdminStorage builds and returns a new storage.AdminStorage using CloudSpanner.
func (s *cloudSpannerProvider) AdminStorage() storage.AdminStorage {
	warn()
	return NewAdminStorage(s.client)
}

// Close shuts down this provider. Calls to the other methods will fail
// after this.
func (s *cloudSpannerProvider) Close() error {
	s.client.Close()
	return nil
}

func setIntIfNotDefault(t *int, v int) {
	if v != 0 {
		*t = v
	}
}

func setUint64IfNotDefault(t *uint64, v uint64) {
	if v != 0 {
		*t = v
	}
}

func setFloat64IfNotDefault(t *float64, v float64) {
	if v != 0 {
		*t = v
	}
}
<commit_msg>Track session handles (#2185)<commit_after>// Copyright 2018 Google LLC. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cloudspanner

import (
	"bytes"
	"compress/gzip"
	"context"
	"encoding/base64"
	"flag"
	"io/ioutil"
	"sync"
	"time"

	"cloud.google.com/go/spanner"
	"github.com/golang/glog"
	"github.com/google/trillian/monitoring"
	"github.com/google/trillian/storage"
	"google.golang.org/api/option"
)

var (
	csURI                                = flag.String("cloudspanner_uri", "", "Connection URI for CloudSpanner database")
	csNumChannels                        = flag.Int("cloudspanner_num_channels", 0, "Number of gRPC channels to use to talk to CloudSpanner.")
	csSessionMaxOpened                   = flag.Uint64("cloudspanner_max_open_sessions", 0, "Max open sessions.")
	csSessionMinOpened                   = flag.Uint64("cloudspanner_min_open_sessions", 0, "Min open sessions.")
	csSessionMaxIdle                     = flag.Uint64("cloudspanner_max_idle_sessions", 0, "Max idle sessions.")
	csSessionMaxBurst                    = flag.Uint64("cloudspanner_max_burst_sessions", 0, "Max concurrent create session requests.")
	csSessionWriteSessions               = flag.Float64("cloudspanner_write_sessions", 0, "Fraction of write capable sessions to maintain.")
	csSessionHCWorkers                   = flag.Int("cloudspanner_num_healthcheckers", 0, "Number of health check workers for Spanner session pool.")
	csSessionHCInterval                  = flag.Duration("cloudspanner_healthcheck_interval", 0, "Interval betweek pinging sessions.")
	csSessionTrackHandles                = flag.Bool("cloudspanner_track_session_handles", false, "determines whether the session pool will keep track of the stacktrace of the goroutines that take sessions from the pool.")
	csDequeueAcrossMerkleBucketsFraction = flag.Float64("cloudspanner_dequeue_bucket_fraction", 0.75, "Fraction of merkle keyspace to dequeue from, set to zero to disable.")
	csReadOnlyStaleness                  = flag.Duration("cloudspanner_readonly_staleness", time.Minute, "How far in the past to perform readonly operations. Within limits, raising this should help to increase performance/reduce latency.")

	csMu              sync.RWMutex
	csStorageInstance *cloudSpannerProvider
	warnOnce          sync.Once
)

func init() {
	if err := storage.RegisterProvider("cloud_spanner", newCloudSpannerStorageProvider); err != nil {
		panic(err)
	}
}

func warn() {
	warnOnce.Do(func() {
		w := `H4sIAAAAAAAA/4xUsW7rMAzc8xUE2lE41B2sWlzsZ3TwoKEIgkQZ64mLsxga8vUPlG3FrZ2Hd1Ng3onHE5UDPQOEmVnwjCGhjyLC8RLcPgfhIvmwot8/CaHF9CMdOthdGmKvdSQQET85TqxJtKzjgnd4mYaFilDIlmhsnKql977mZSqzYcLy5K/zCUX66sbtNOAwteTXiVph5m4nigGzzUH7e3+a3XIRf5PFyhQQEV6UXLeY8nL292gyujlMIlIbdcUwet9ieBx/snWIOXkievPenyOMiDjnjOHj+MMJhjZfFBFpHF+AcQkGpr9f1nz02YoKcPXed5nvjHG2DGtB//7gGwHCq69HIPMBGa7hIYi3mPlOBOhf/Z8eMBmAdNVjZKlCFuiQgK19Y1YKrXDT5KWX7ohVC+cArnUKwGAF/rwvk6CrVhJ1DuDDF9igfVtEuFf8U2M0MXW4wf28pBy/4yOuOaLZw2+Qa76m5PpSFy+5N0usbnyr66+AjY7cx3eKz5VHrZpFlqL6nJa82+gI/H3Vh+TKm9Fmib7I5GXSvcStTQrndxwIw4dQvpak00DGpKvbnVgIxXk4kD31oLnTSkgkxchmJ01Vnj7lQLZFXrV532bpfqLJbTzqfygXrLHkh/CoP5Hq13DXJYuV3fD/DcRbm+5f7s1tvNj/RLBD9T6vNbi9dYpT05QTKsV1B+Ut4m8AAAD///IJ0vhIBgAA`

		wd, _ := base64.StdEncoding.DecodeString(w)
		b := bytes.NewReader(wd)
		r, _ := gzip.NewReader(b)
		if err := r.Close(); err != nil {
			// No need to exit, it's an unlikely error and doesn't affect operation.
			glog.Warningf("Close()=%v", err)
		}
		t, _ := ioutil.ReadAll(r)
		glog.Warningf("WARNING\n%s\nCloudspanner is an experimental storage implementation, and only supports Logs currently.", string(t))
	})
}

type cloudSpannerProvider struct {
	client *spanner.Client
}

func configFromFlags() spanner.ClientConfig {
	r := spanner.ClientConfig{}
	setUint64IfNotDefault(&r.SessionPoolConfig.MaxOpened, *csSessionMaxOpened)
	setUint64IfNotDefault(&r.SessionPoolConfig.MinOpened, *csSessionMinOpened)
	setUint64IfNotDefault(&r.SessionPoolConfig.MaxIdle, *csSessionMaxIdle)
	setUint64IfNotDefault(&r.SessionPoolConfig.MaxBurst, *csSessionMaxBurst)
	setFloat64IfNotDefault(&r.SessionPoolConfig.WriteSessions, *csSessionWriteSessions)
	setIntIfNotDefault(&r.SessionPoolConfig.HealthCheckWorkers, *csSessionHCWorkers)
	r.SessionPoolConfig.TrackSessionHandles = *csSessionTrackHandles
	r.SessionPoolConfig.HealthCheckInterval = *csSessionHCInterval
	return r
}

func optionsFromFlags() []option.ClientOption {
	opts := []option.ClientOption{}
	if numConns := *csNumChannels; numConns != 0 {
		opts = append(opts, option.WithGRPCConnectionPool(numConns))
	}
	return opts
}

func newCloudSpannerStorageProvider(_ monitoring.MetricFactory) (storage.Provider, error) {
	csMu.Lock()
	defer csMu.Unlock()

	if csStorageInstance != nil {
		return csStorageInstance, nil
	}

	client, err := spanner.NewClientWithConfig(context.TODO(), *csURI, configFromFlags(), optionsFromFlags()...)
	if err != nil {
		return nil, err
	}
	csStorageInstance = &cloudSpannerProvider{
		client: client,
	}
	return csStorageInstance, nil
}

// LogStorage builds and returns a new storage.LogStorage using CloudSpanner.
func (s *cloudSpannerProvider) LogStorage() storage.LogStorage {
	warn()
	opts := LogStorageOptions{}
	frac := *csDequeueAcrossMerkleBucketsFraction
	if frac > 1.0 {
		frac = 1.0
	}
	if frac > 0 {
		opts.DequeueAcrossMerkleBuckets = true
		opts.DequeueAcrossMerkleBucketsRangeFraction = frac
	}
	if *csReadOnlyStaleness > 0 {
		opts.ReadOnlyStaleness = *csReadOnlyStaleness
	}
	return NewLogStorageWithOpts(s.client, opts)
}

// MapStorage builds and returns a new storage.MapStorage using CloudSpanner.
func (s *cloudSpannerProvider) MapStorage() storage.MapStorage {
	warn()
	opts := MapStorageOptions{}
	if *csReadOnlyStaleness > 0 {
		opts.ReadOnlyStaleness = *csReadOnlyStaleness
	}
	return NewMapStorageWithOpts(s.client, opts)
}

// AdminStorage builds and returns a new storage.AdminStorage using CloudSpanner.
func (s *cloudSpannerProvider) AdminStorage() storage.AdminStorage {
	warn()
	return NewAdminStorage(s.client)
}

// Close shuts down this provider. Calls to the other methods will fail
// after this.
func (s *cloudSpannerProvider) Close() error {
	s.client.Close()
	return nil
}

func setIntIfNotDefault(t *int, v int) {
	if v != 0 {
		*t = v
	}
}

func setUint64IfNotDefault(t *uint64, v uint64) {
	if v != 0 {
		*t = v
	}
}

func setFloat64IfNotDefault(t *float64, v float64) {
	if v != 0 {
		*t = v
	}
}
<|endoftext|>


<commit_before>package parse

import (
	"fmt"
	"regexp"
	"strconv"
	"strings"

	"gopkg.in/sqle/sqle.v0/sql"
	"gopkg.in/sqle/sqle.v0/sql/expression"
	"gopkg.in/sqle/sqle.v0/sql/plan"

	"github.com/gitql/vitess/go/vt/sqlparser"
)

const (
	showTables = "SHOW TABLES"
)

func errUnsupported(n sqlparser.SQLNode) error {
	return fmt.Errorf("unsupported syntax: %#v", n)
}

func errUnsupportedFeature(feature string) error {
	return fmt.Errorf("unsupported feature: %s", feature)
}

func Parse(s string) (sql.Node, error) {
	if strings.HasSuffix(s, ";") {
		s = s[:len(s)-1]
	}

	// TODO implement it into the parser
	if strings.ToUpper(s) == showTables {
		return plan.NewShowTables(&sql.UnresolvedDatabase{}), nil
	}

	t := regexp.MustCompile(`^describe\s+table\s+(.*)`).FindStringSubmatch(strings.ToLower(s))
	if len(t) == 2 && t[1] != "" {
		return plan.NewDescribe(plan.NewUnresolvedTable(t[1])), nil
	}

	stmt, err := sqlparser.Parse(s)
	if err != nil {
		return nil, err
	}

	return convert(stmt)
}

func convert(stmt sqlparser.Statement) (sql.Node, error) {
	switch n := stmt.(type) {
	default:
		return nil, errUnsupported(n)
	case *sqlparser.Select:
		return convertSelect(n)
	}
}

func convertSelect(s *sqlparser.Select) (sql.Node, error) {
	var node sql.Node

	node, err := tableExprsToTable(s.From)
	if err != nil {
		return nil, err
	}

	if s.Distinct != "" {
		return nil, errUnsupportedFeature("DISTINCT")
	}

	if s.Having != nil {
		return nil, errUnsupportedFeature("HAVING")
	}

	if s.Where != nil {
		node, err = whereToFilter(s.Where, node)
		if err != nil {
			return nil, err
		}
	}

	if len(s.OrderBy) != 0 {
		node, err = orderByToSort(s.OrderBy, node)
		if err != nil {
			return nil, err
		}
	}

	if s.Limit != nil {
		//TODO: Add support for offset
		node, err = limitToLimit(s.Limit.Rowcount, node)
		if err != nil {
			return nil, err
		}
	}

	return selectToProjectOrGroupBy(s.SelectExprs, s.GroupBy, node)
}

func tableExprsToTable(te sqlparser.TableExprs) (sql.Node, error) {
	if len(te) == 0 {
		return nil, errUnsupportedFeature("zero tables in FROM")
	}

	var nodes []sql.Node
	for _, t := range te {
		n, err := tableExprToTable(t)
		if err != nil {
			return nil, err
		}

		nodes = append(nodes, n)
	}

	if len(nodes) == 1 {
		return nodes[0], nil
	}

	if len(nodes) == 2 {
		return plan.NewCrossJoin(nodes[0], nodes[1]), nil
	}

	//TODO: Support N tables in JOIN.
	return nil, errUnsupportedFeature("more than 2 tables in JOIN")
}

func tableExprToTable(te sqlparser.TableExpr) (sql.Node, error) {
	switch t := (te).(type) {
	default:
		return nil, errUnsupported(te)
	case *sqlparser.AliasedTableExpr:
		//TODO: Add support for table alias.
		//TODO: Add support for qualifier.
		tn, ok := t.Expr.(*sqlparser.TableName)
		if !ok {
			return nil, errUnsupportedFeature("non simple tables")
		}

		return plan.NewUnresolvedTable(tn.Name.String()), nil
	}
}

func whereToFilter(w *sqlparser.Where, child sql.Node) (*plan.Filter, error) {
	c, err := exprToExpression(w.Expr)
	if err != nil {
		return nil, err
	}

	return plan.NewFilter(c, child), nil
}

func orderByToSort(ob sqlparser.OrderBy, child sql.Node) (*plan.Sort, error) {
	var sortFields []plan.SortField
	for _, o := range ob {
		e, err := exprToExpression(o.Expr)
		if err != nil {
			return nil, err
		}

		var so plan.SortOrder
		switch o.Direction {
		default:
			panic(fmt.Errorf("invalid sort order: %s", o.Direction))
		case sqlparser.AscScr:
			so = plan.Ascending
		case sqlparser.DescScr:
			so = plan.Descending
		}

		sf := plan.SortField{Column: e, Order: so}
		sortFields = append(sortFields, sf)
	}

	return plan.NewSort(sortFields, child), nil
}

func limitToLimit(o sqlparser.Expr, child sql.Node) (*plan.Limit, error) {
	e, err := exprToExpression(o)
	if err != nil {
		return nil, err
	}

	nl, ok := e.(*expression.Literal)
	if !ok || nl.Type() != sql.BigInteger {
		return nil, errUnsupportedFeature("LIMIT with non-integer literal")
	}

	n := (nl.Eval(nil)).(int64)
	return plan.NewLimit(n, child), nil
}

func isAggregate(e sql.Expression) bool {
	switch v := e.(type) {
	case *expression.UnresolvedFunction:
		return v.IsAggregate
	case *expression.Alias:
		return isAggregate(v.Child)
	default:
		return false
	}
}

func selectToProjectOrGroupBy(se sqlparser.SelectExprs, g sqlparser.GroupBy, child sql.Node) (sql.Node, error) {
	selectExprs, err := selectExprsToExpressions(se)
	if err != nil {
		return nil, err
	}

	isAgg := len(g) > 0
	if !isAgg {
		for _, e := range selectExprs {
			if isAggregate(e) {
				isAgg = true
				break
			}
		}
	}

	if isAgg {
		groupingExprs, err := groupByToExpressions(g)
		if err != nil {
			return nil, err
		}

		return plan.NewGroupBy(selectExprs, groupingExprs, child), nil
	}

	return plan.NewProject(selectExprs, child), nil
}

func selectExprsToExpressions(se sqlparser.SelectExprs) ([]sql.Expression, error) {
	var exprs []sql.Expression
	for _, e := range se {
		pe, err := selectExprToExpression(e)
		if err != nil {
			return nil, err
		}

		exprs = append(exprs, pe)
	}

	return exprs, nil
}

func exprToExpression(e sqlparser.Expr) (sql.Expression, error) {
	switch v := e.(type) {
	default:
		return nil, errUnsupported(e)
	case *sqlparser.ComparisonExpr:
		return comparisonExprToExpression(v)
	case *sqlparser.NotExpr:
		c, err := exprToExpression(v.Expr)
		if err != nil {
			return nil, err
		}

		return expression.NewNot(c), nil
	case *sqlparser.SQLVal:
		switch v.Type {
		case sqlparser.StrVal:
			return expression.NewLiteral(string(v.Val), sql.String), nil
		case sqlparser.IntVal:
			//TODO: Use smallest integer representation and widen later.
			n, _ := strconv.ParseInt(string(v.Val), 10, 64)
			return expression.NewLiteral(n, sql.BigInteger), nil
		case sqlparser.HexVal:
			//TODO
			return nil, errUnsupported(v)
		default:
			//TODO
			return nil, errUnsupported(v)
		}
	case sqlparser.BoolVal:
		return expression.NewLiteral(bool(v), sql.Boolean), nil
	case *sqlparser.NullVal:
		//TODO
		return expression.NewLiteral(nil, sql.Null), nil
	case *sqlparser.ColName:
		//TODO: add handling of case sensitiveness.
		return expression.NewUnresolvedColumn(v.Name.Lowered()), nil
	case *sqlparser.FuncExpr:
		exprs, err := selectExprsToExpressions(v.Exprs)
		if err != nil {
			return nil, err
		}

		return expression.NewUnresolvedFunction(v.Name.Lowered(),
			v.IsAggregate(), exprs...), nil
	}
}

func comparisonExprToExpression(c *sqlparser.ComparisonExpr) (sql.Expression,
	error) {

	left, err := exprToExpression(c.Left)
	if err != nil {
		return nil, err
	}

	right, err := exprToExpression(c.Right)
	if err != nil {
		return nil, err
	}

	switch c.Operator {
	default:
		return nil, errUnsupportedFeature(c.Operator)
	case sqlparser.RegexpStr:
		return expression.NewRegexp(left, right), nil
	case sqlparser.EqualStr:
		return expression.NewEquals(left, right), nil
	case sqlparser.LessThanStr:
		return expression.NewLessThan(left, right), nil
	case sqlparser.LessEqualStr:
		return expression.NewLessThanOrEqual(left, right), nil
	case sqlparser.GreaterThanStr:
		return expression.NewGreaterThan(left, right), nil
	case sqlparser.GreaterEqualStr:
		return expression.NewGreaterThanOrEqual(left, right), nil
	case sqlparser.NotEqualStr:
		return expression.NewNot(
			expression.NewEquals(left, right),
		), nil
	}
}

func groupByToExpressions(g sqlparser.GroupBy) ([]sql.Expression, error) {
	es := make([]sql.Expression, len(g))
	for i, ve := range g {
		e, err := exprToExpression(ve)
		if err != nil {
			return nil, err
		}

		es[i] = e
	}

	return es, nil
}

func selectExprToExpression(se sqlparser.SelectExpr) (sql.Expression, error) {
	switch e := se.(type) {
	default:
		return nil, errUnsupported(e)
	case *sqlparser.StarExpr:
		//TODO: Add support for qualified start.
		return expression.NewStar(), nil
	case *sqlparser.NonStarExpr:
		expr, err := exprToExpression(e.Expr)
		if err != nil {
			return nil, err
		}

		if e.As.String() == "" {
			return expr, nil
		}

		//TODO: Handle case-sensitiveness when needed.
		return expression.NewAlias(expr, e.As.Lowered()), nil
	}
}
<commit_msg>use gopkg.in/sqle/vitess-go.v1 (#8)<commit_after>package parse

import (
	"fmt"
	"regexp"
	"strconv"
	"strings"

	"gopkg.in/sqle/sqle.v0/sql"
	"gopkg.in/sqle/sqle.v0/sql/expression"
	"gopkg.in/sqle/sqle.v0/sql/plan"

	"gopkg.in/sqle/vitess-go.v1/vt/sqlparser"
)

const (
	showTables = "SHOW TABLES"
)

func errUnsupported(n sqlparser.SQLNode) error {
	return fmt.Errorf("unsupported syntax: %#v", n)
}

func errUnsupportedFeature(feature string) error {
	return fmt.Errorf("unsupported feature: %s", feature)
}

func Parse(s string) (sql.Node, error) {
	if strings.HasSuffix(s, ";") {
		s = s[:len(s)-1]
	}

	// TODO implement it into the parser
	if strings.ToUpper(s) == showTables {
		return plan.NewShowTables(&sql.UnresolvedDatabase{}), nil
	}

	t := regexp.MustCompile(`^describe\s+table\s+(.*)`).FindStringSubmatch(strings.ToLower(s))
	if len(t) == 2 && t[1] != "" {
		return plan.NewDescribe(plan.NewUnresolvedTable(t[1])), nil
	}

	stmt, err := sqlparser.Parse(s)
	if err != nil {
		return nil, err
	}

	return convert(stmt)
}

func convert(stmt sqlparser.Statement) (sql.Node, error) {
	switch n := stmt.(type) {
	default:
		return nil, errUnsupported(n)
	case *sqlparser.Select:
		return convertSelect(n)
	}
}

func convertSelect(s *sqlparser.Select) (sql.Node, error) {
	var node sql.Node

	node, err := tableExprsToTable(s.From)
	if err != nil {
		return nil, err
	}

	if s.Distinct != "" {
		return nil, errUnsupportedFeature("DISTINCT")
	}

	if s.Having != nil {
		return nil, errUnsupportedFeature("HAVING")
	}

	if s.Where != nil {
		node, err = whereToFilter(s.Where, node)
		if err != nil {
			return nil, err
		}
	}

	if len(s.OrderBy) != 0 {
		node, err = orderByToSort(s.OrderBy, node)
		if err != nil {
			return nil, err
		}
	}

	if s.Limit != nil {
		//TODO: Add support for offset
		node, err = limitToLimit(s.Limit.Rowcount, node)
		if err != nil {
			return nil, err
		}
	}

	return selectToProjectOrGroupBy(s.SelectExprs, s.GroupBy, node)
}

func tableExprsToTable(te sqlparser.TableExprs) (sql.Node, error) {
	if len(te) == 0 {
		return nil, errUnsupportedFeature("zero tables in FROM")
	}

	var nodes []sql.Node
	for _, t := range te {
		n, err := tableExprToTable(t)
		if err != nil {
			return nil, err
		}

		nodes = append(nodes, n)
	}

	if len(nodes) == 1 {
		return nodes[0], nil
	}

	if len(nodes) == 2 {
		return plan.NewCrossJoin(nodes[0], nodes[1]), nil
	}

	//TODO: Support N tables in JOIN.
	return nil, errUnsupportedFeature("more than 2 tables in JOIN")
}

func tableExprToTable(te sqlparser.TableExpr) (sql.Node, error) {
	switch t := (te).(type) {
	default:
		return nil, errUnsupported(te)
	case *sqlparser.AliasedTableExpr:
		//TODO: Add support for table alias.
		//TODO: Add support for qualifier.
		tn, ok := t.Expr.(*sqlparser.TableName)
		if !ok {
			return nil, errUnsupportedFeature("non simple tables")
		}

		return plan.NewUnresolvedTable(tn.Name.String()), nil
	}
}

func whereToFilter(w *sqlparser.Where, child sql.Node) (*plan.Filter, error) {
	c, err := exprToExpression(w.Expr)
	if err != nil {
		return nil, err
	}

	return plan.NewFilter(c, child), nil
}

func orderByToSort(ob sqlparser.OrderBy, child sql.Node) (*plan.Sort, error) {
	var sortFields []plan.SortField
	for _, o := range ob {
		e, err := exprToExpression(o.Expr)
		if err != nil {
			return nil, err
		}

		var so plan.SortOrder
		switch o.Direction {
		default:
			panic(fmt.Errorf("invalid sort order: %s", o.Direction))
		case sqlparser.AscScr:
			so = plan.Ascending
		case sqlparser.DescScr:
			so = plan.Descending
		}

		sf := plan.SortField{Column: e, Order: so}
		sortFields = append(sortFields, sf)
	}

	return plan.NewSort(sortFields, child), nil
}

func limitToLimit(o sqlparser.Expr, child sql.Node) (*plan.Limit, error) {
	e, err := exprToExpression(o)
	if err != nil {
		return nil, err
	}

	nl, ok := e.(*expression.Literal)
	if !ok || nl.Type() != sql.BigInteger {
		return nil, errUnsupportedFeature("LIMIT with non-integer literal")
	}

	n := (nl.Eval(nil)).(int64)
	return plan.NewLimit(n, child), nil
}

func isAggregate(e sql.Expression) bool {
	switch v := e.(type) {
	case *expression.UnresolvedFunction:
		return v.IsAggregate
	case *expression.Alias:
		return isAggregate(v.Child)
	default:
		return false
	}
}

func selectToProjectOrGroupBy(se sqlparser.SelectExprs, g sqlparser.GroupBy, child sql.Node) (sql.Node, error) {
	selectExprs, err := selectExprsToExpressions(se)
	if err != nil {
		return nil, err
	}

	isAgg := len(g) > 0
	if !isAgg {
		for _, e := range selectExprs {
			if isAggregate(e) {
				isAgg = true
				break
			}
		}
	}

	if isAgg {
		groupingExprs, err := groupByToExpressions(g)
		if err != nil {
			return nil, err
		}

		return plan.NewGroupBy(selectExprs, groupingExprs, child), nil
	}

	return plan.NewProject(selectExprs, child), nil
}

func selectExprsToExpressions(se sqlparser.SelectExprs) ([]sql.Expression, error) {
	var exprs []sql.Expression
	for _, e := range se {
		pe, err := selectExprToExpression(e)
		if err != nil {
			return nil, err
		}

		exprs = append(exprs, pe)
	}

	return exprs, nil
}

func exprToExpression(e sqlparser.Expr) (sql.Expression, error) {
	switch v := e.(type) {
	default:
		return nil, errUnsupported(e)
	case *sqlparser.ComparisonExpr:
		return comparisonExprToExpression(v)
	case *sqlparser.NotExpr:
		c, err := exprToExpression(v.Expr)
		if err != nil {
			return nil, err
		}

		return expression.NewNot(c), nil
	case *sqlparser.SQLVal:
		switch v.Type {
		case sqlparser.StrVal:
			return expression.NewLiteral(string(v.Val), sql.String), nil
		case sqlparser.IntVal:
			//TODO: Use smallest integer representation and widen later.
			n, _ := strconv.ParseInt(string(v.Val), 10, 64)
			return expression.NewLiteral(n, sql.BigInteger), nil
		case sqlparser.HexVal:
			//TODO
			return nil, errUnsupported(v)
		default:
			//TODO
			return nil, errUnsupported(v)
		}
	case sqlparser.BoolVal:
		return expression.NewLiteral(bool(v), sql.Boolean), nil
	case *sqlparser.NullVal:
		//TODO
		return expression.NewLiteral(nil, sql.Null), nil
	case *sqlparser.ColName:
		//TODO: add handling of case sensitiveness.
		return expression.NewUnresolvedColumn(v.Name.Lowered()), nil
	case *sqlparser.FuncExpr:
		exprs, err := selectExprsToExpressions(v.Exprs)
		if err != nil {
			return nil, err
		}

		return expression.NewUnresolvedFunction(v.Name.Lowered(),
			v.IsAggregate(), exprs...), nil
	}
}

func comparisonExprToExpression(c *sqlparser.ComparisonExpr) (sql.Expression,
	error) {

	left, err := exprToExpression(c.Left)
	if err != nil {
		return nil, err
	}

	right, err := exprToExpression(c.Right)
	if err != nil {
		return nil, err
	}

	switch c.Operator {
	default:
		return nil, errUnsupportedFeature(c.Operator)
	case sqlparser.RegexpStr:
		return expression.NewRegexp(left, right), nil
	case sqlparser.EqualStr:
		return expression.NewEquals(left, right), nil
	case sqlparser.LessThanStr:
		return expression.NewLessThan(left, right), nil
	case sqlparser.LessEqualStr:
		return expression.NewLessThanOrEqual(left, right), nil
	case sqlparser.GreaterThanStr:
		return expression.NewGreaterThan(left, right), nil
	case sqlparser.GreaterEqualStr:
		return expression.NewGreaterThanOrEqual(left, right), nil
	case sqlparser.NotEqualStr:
		return expression.NewNot(
			expression.NewEquals(left, right),
		), nil
	}
}

func groupByToExpressions(g sqlparser.GroupBy) ([]sql.Expression, error) {
	es := make([]sql.Expression, len(g))
	for i, ve := range g {
		e, err := exprToExpression(ve)
		if err != nil {
			return nil, err
		}

		es[i] = e
	}

	return es, nil
}

func selectExprToExpression(se sqlparser.SelectExpr) (sql.Expression, error) {
	switch e := se.(type) {
	default:
		return nil, errUnsupported(e)
	case *sqlparser.StarExpr:
		//TODO: Add support for qualified start.
		return expression.NewStar(), nil
	case *sqlparser.NonStarExpr:
		expr, err := exprToExpression(e.Expr)
		if err != nil {
			return nil, err
		}

		if e.As.String() == "" {
			return expr, nil
		}

		//TODO: Handle case-sensitiveness when needed.
		return expression.NewAlias(expr, e.As.Lowered()), nil
	}
}
<|endoftext|>


<commit_before>package kloud

import (
	"errors"
	"fmt"
	"koding/db/mongodb/modelhelper"
	"koding/kites/kloud/contexthelper/session"
	"koding/kites/kloud/terraformer"
	tf "koding/kites/terraformer"

	"golang.org/x/net/context"

	"github.com/koding/kite"
)

type TerraformPlanRequest struct {
	StackTemplateId string `json:"stackTemplateId"`

	GroupName string `json:"groupName"`
}

func (k *Kloud) Plan(r *kite.Request) (interface{}, error) {
	if r.Args == nil {
		return nil, NewError(ErrNoArguments)
	}

	var args *TerraformPlanRequest
	if err := r.Args.One().Unmarshal(&args); err != nil {
		return nil, err
	}

	if args.StackTemplateId == "" {
		return nil, errors.New("stackIdTemplate is not passed")
	}

	if args.GroupName == "" {
		return nil, errors.New("group name is not passed")
	}

	ctx := k.ContextCreator(context.Background())
	sess, ok := session.FromContext(ctx)
	if !ok {
		return nil, errors.New("session context is not passed")
	}

	k.Log.Debug("Fetching template for id %s", args.StackTemplateId)
	stackTemplate, err := modelhelper.GetStackTemplate(args.StackTemplateId)
	if err != nil {
		return nil, err
	}

	k.Log.Debug("Fetching credentials for id %v", stackTemplate.Credentials)
	data, err := fetchTerraformData(r.Username, args.GroupName, sess.DB, flattenValues(stackTemplate.Credentials))
	if err != nil {
		return nil, err
	}

	// TODO(arslan): make one single persistent connection if needed, for now
	// this is ok.
	tfKite, err := terraformer.Connect(sess.Kite)
	if err != nil {
		return nil, err
	}
	defer tfKite.Close()

	template, err := newTerraformTemplate(stackTemplate.Template.Content)
	if err != nil {
		return nil, err
	}

	if err := template.fillVariables("userInput"); err != nil {
		return nil, err
	}

	var region string
	for _, cred := range data.Creds {
		region, ok = cred.Data["region"]
		if !ok {
			return nil, fmt.Errorf("region for identifer '%s' is not set", cred.Identifier)
		}

		k.Log.Debug("Appending AWS variable for\n%s", stackTemplate.Template.Content)
		if err := template.injectCustomVariables(cred.Provider, cred.Data); err != nil {
			return nil, err
		}
	}

	sess.Log.Debug("Plan: stack template before injecting Koding data")
	sess.Log.Debug("%v", template)
	buildData, err := injectKodingData(ctx, template, r.Username, data)
	if err != nil {
		return nil, err
	}
	stackTemplate.Template.Content = buildData.Template

	k.Log.Debug("Calling plan with content\n%s", stackTemplate.Template.Content)
	plan, err := tfKite.Plan(&tf.TerraformRequest{
		Content:   stackTemplate.Template.Content,
		ContentID: r.Username + "-" + args.StackTemplateId,
		Variables: nil,
	})
	if err != nil {
		return nil, err
	}

	machines, err := machinesFromPlan(plan)
	if err != nil {
		return nil, err
	}
	machines.AppendRegion(region)

	return machines, nil
}
<commit_msg>kloud: fix setting region for plan<commit_after>package kloud

import (
	"errors"
	"fmt"
	"koding/db/mongodb/modelhelper"
	"koding/kites/kloud/contexthelper/session"
	"koding/kites/kloud/terraformer"
	tf "koding/kites/terraformer"

	"golang.org/x/net/context"

	"github.com/koding/kite"
)

type TerraformPlanRequest struct {
	StackTemplateId string `json:"stackTemplateId"`

	GroupName string `json:"groupName"`
}

func (k *Kloud) Plan(r *kite.Request) (interface{}, error) {
	if r.Args == nil {
		return nil, NewError(ErrNoArguments)
	}

	var args *TerraformPlanRequest
	if err := r.Args.One().Unmarshal(&args); err != nil {
		return nil, err
	}

	if args.StackTemplateId == "" {
		return nil, errors.New("stackIdTemplate is not passed")
	}

	if args.GroupName == "" {
		return nil, errors.New("group name is not passed")
	}

	ctx := k.ContextCreator(context.Background())
	sess, ok := session.FromContext(ctx)
	if !ok {
		return nil, errors.New("session context is not passed")
	}

	k.Log.Debug("Fetching template for id %s", args.StackTemplateId)
	stackTemplate, err := modelhelper.GetStackTemplate(args.StackTemplateId)
	if err != nil {
		return nil, err
	}

	k.Log.Debug("Fetching credentials for id %v", stackTemplate.Credentials)
	data, err := fetchTerraformData(r.Username, args.GroupName, sess.DB, flattenValues(stackTemplate.Credentials))
	if err != nil {
		return nil, err
	}

	// TODO(arslan): make one single persistent connection if needed, for now
	// this is ok.
	tfKite, err := terraformer.Connect(sess.Kite)
	if err != nil {
		return nil, err
	}
	defer tfKite.Close()

	template, err := newTerraformTemplate(stackTemplate.Template.Content)
	if err != nil {
		return nil, err
	}

	if err := template.fillVariables("userInput"); err != nil {
		return nil, err
	}

	var region string
	for _, cred := range data.Creds {
		k.Log.Debug("Appending %s provider variables", cred.Provider)
		if err := template.injectCustomVariables(cred.Provider, cred.Data); err != nil {
			return nil, err
		}

		// rest is aws related
		if cred.Provider != "aws" {
			continue
		}

		region, ok = cred.Data["region"]
		if !ok {
			return nil, fmt.Errorf("region for identifer '%s' is not set", cred.Identifier)
		}

		if err := template.setAwsRegion(region); err != nil {
			return nil, err
		}
	}

	sess.Log.Debug("Plan: stack template before injecting Koding data")
	sess.Log.Debug("%v", template)
	buildData, err := injectKodingData(ctx, template, r.Username, data)
	if err != nil {
		return nil, err
	}
	stackTemplate.Template.Content = buildData.Template

	k.Log.Debug("Calling plan with content\n%s", stackTemplate.Template.Content)
	plan, err := tfKite.Plan(&tf.TerraformRequest{
		Content:   stackTemplate.Template.Content,
		ContentID: r.Username + "-" + args.StackTemplateId,
		Variables: nil,
	})
	if err != nil {
		return nil, err
	}

	machines, err := machinesFromPlan(plan)
	if err != nil {
		return nil, err
	}
	machines.AppendRegion(region)

	return machines, nil
}
<|endoftext|>


<commit_before>package models

import (
	"socialapi/request"
	"strings"
	"time"

	"github.com/koding/bongo"
)

// ChannelLink holds the link between two channels
type ChannelLink struct {
	// Id holds the unique id of the link between channels
	Id int64 `json:"id,string"`

	// RootId is the id of the root channel
	RootId int64 `json:"rootId,string"       sql:"NOT NULL"`

	// LeafId is the id of the leaf channel
	LeafId int64 `json:"leafId,string"       sql:"NOT NULL"`

	// CreatedAt holds the creation time of the channel_link
	CreatedAt time.Time `json:"createdAt"    sql:"NOT NULL"`

	// IsFinished holds the linking process
	// we link leaf channel to the root channel
	// and marks channel_link as true
	// if isFinished is false, it means
	// linking the channels process is not done yet.
	IsFinished bool `json:"isFinished"       sql:"NOT NULL"`

	// options for operations

	// DeleteMessages remove the messages of a channel
	DeleteMessages bool `json:"deleteMessages,omitempty" sql:"-"`
}

// List gets the all leaves of a given channel
func (c *ChannelLink) List(q *request.Query) ([]Channel, error) {
	if c.RootId == 0 {
		return nil, ErrRootIsNotSet
	}

	var leafIds []int64

	bq := &bongo.Query{
		Selector: map[string]interface{}{
			"root_id": c.RootId,
		},
		Pluck:      "leaf_id",
		Pagination: *bongo.NewPagination(q.Limit, q.Skip),
	}

	err := c.Some(&leafIds, bq)
	if err != nil {
		return nil, err
	}

	return NewChannel().FetchByIds(leafIds)
}

// FetchRoot fetches the root of a channel if linked
func (c *ChannelLink) FetchRoot() (*Channel, error) {
	if c.LeafId == 0 {
		return nil, ErrLeafIsNotSet
	}

	var rootIds []int64

	bq := &bongo.Query{
		Selector: map[string]interface{}{
			"leaf_id": c.LeafId,
		},
		Pluck: "root_id",
	}

	if err := c.Some(&rootIds, bq); err != nil {
		return nil, err
	}

	if len(rootIds) == 0 {
		return nil, bongo.RecordNotFound
	}

	channel := NewChannel()
	if err := channel.ById(rootIds[0]); err != nil {
		return nil, err
	}

	return channel, nil
}

// ChannelLinksWithRoot fetches the links with root of channel
func (c *ChannelLink) ChannelLinksWithRoot() ([]ChannelLink, error) {
	var cLinks []ChannelLink

	query := bongo.B.DB.Table(c.BongoName())
	query = query.Where("root_id = ? or leaf_id = ?", c.RootId, c.RootId)

	if err := query.Find(&cLinks).Error; err != nil && err != bongo.RecordNotFound {
		return nil, err
	}

	if cLinks == nil {
		return nil, nil
	}

	if len(cLinks) == 0 {
		return nil, nil
	}

	return cLinks, nil
}

// RemoveLinksWithRoot removes the links of the given root channel
func (c *ChannelLink) RemoveLinksWithRoot() error {
	links, err := c.ChannelLinksWithRoot()
	if err != nil && err != bongo.RecordNotFound {
		return err
	}

	var errs error

	for _, link := range links {
		// ignore all not found errors while deleting links
		err := link.Delete()
		if err != nil && err != ErrChannelNotFound {
			errs = err
		}
	}

	if errs != nil {
		return errs
	}

	return nil
}

// Create creates a link between two channels
func (c *ChannelLink) Create() error {
	return c.create()
}

// Delete removes the link between two channels, most probably it wont touch to
// the messages
func (c *ChannelLink) Delete() error {
	if err := c.validate(); err != nil {
		return err
	}

	// first update the leaf node with it's previous channel type constant
	leaf := NewChannel()
	if err := leaf.ById(c.LeafId); err != nil {
		if err == bongo.RecordNotFound {
			return ErrChannelNotFound
		}

		return err
	}

	leaf.TypeConstant = strings.TrimPrefix(
		string(leaf.TypeConstant),
		ChannelLinkedPrefix,
	)

	if err := leaf.Update(); err != nil {
		return err
	}

	toBeDeletedCL := NewChannelLink()
	// then delete the link between two channels
	bq := &bongo.Query{
		Selector: map[string]interface{}{
			"root_id": c.RootId,
			"leaf_id": c.LeafId,
		},
	}

	if err := toBeDeletedCL.One(bq); err != nil {
		return err
	}

	return bongo.B.Delete(toBeDeletedCL)
}

// Blacklist deletes the messages and links the leaf channel to the root while
// removing the participants
func (c *ChannelLink) Blacklist() error {
	c.DeleteMessages = true
	return c.Create()
}

func (c *ChannelLink) create() error {
	if err := c.validate(); err != nil {
		return err
	}

	// it controls the channel linking process, if not finished
	// then dont try to link another channel and return error
	isInProgress, err := c.IsInProgress(c.RootId)
	if err != nil {
		return err
	}
	if isInProgress {
		return ErrLinkingProcessNotDone
	}

	// then delete the link between two channels
	bq := &bongo.Query{
		Selector: map[string]interface{}{
			"root_id": c.RootId,
			"leaf_id": c.LeafId,
		},
	}

	// if there is no error, it means we already have it
	if err := c.One(bq); err == nil {
		c.AfterCreate() // just mimic as if created
		return nil
	}

	// first update the leaf
	leaf := NewChannel()
	if err := leaf.ById(c.LeafId); err != nil {
		if err == bongo.RecordNotFound {
			return ErrChannelNotFound
		}

		return err
	}

	// mark channel as linked
	leaf.TypeConstant = ChannelLinkedPrefix + leaf.TypeConstant

	if err := leaf.Update(); err != nil {
		return err
	}

	return bongo.B.Create(c)
}

// IsInProgress checks the db if linking process is completely done or not
// if true -> linking process is not done
// if false -> linking process is completely done
// We will searh root channel in db as leaf channel,
// Aim of this search, protect linking conflict of root-leaf channels
func (c *ChannelLink) IsInProgress(rootID int64) (bool, error) {
	bq := &bongo.Query{
		Selector: map[string]interface{}{
			"leaf_id":     rootID,
			"is_finished": false,
		},
	}

	// if there is no error, it means we already have it
	newCh := NewChannelLink()
	err := newCh.One(bq)

	if err != nil && err != bongo.RecordNotFound {
		return false, err
	}

	if err == nil {
		return true, nil
	}

	return false, nil
}

func (c *ChannelLink) validate() error {
	if c.LeafId == 0 {
		return ErrLeafIsNotSet
	}

	if c.RootId == 0 {
		return ErrRootIsNotSet
	}

	return nil
}
<commit_msg>socialapi/moderation: remove channel link files<commit_after><|endoftext|>


<commit_before>/*
Copyright 2021 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"os"
	"os/exec"
	"regexp"
	"sort"
	"strings"
	"sync"
	"text/template"
)

type label struct {
	Name string `json:"name"`
}

type prInfo struct {
	Labels []label `json:"labels"`
	Number int     `json:"number"`
	Title  string  `json:"title"`
}

const (
	markdownTemplate = `
{{- range $typeName, $components := . }}
## {{ $typeName }}
{{- range $componentName, $component := $components }}
### {{ $componentName}}
{{- range $prInfo := $component }}
 - {{ $prInfo.Title }} #{{ $prInfo.Number }}
{{- end }}
{{- end }}
{{- end }}
`

	prefixType      = "Type: "
	prefixComponent = "Component: "
)

func loadMergedPRs(from, to string) ([]string, error) {
	cmd := exec.Command("git", "log", "--oneline", fmt.Sprintf("%s..%s", from, to))
	out, err := cmd.Output()
	if err != nil {
		execErr := err.(*exec.ExitError)
		return nil, fmt.Errorf("%s:\nstderr: %s\nstdout: %s", err.Error(), execErr.Stderr, out)
	}

	var prs []string
	rgx := regexp.MustCompile(`Merge pull request #(\d+)`)
	lines := strings.Split(string(out), "\n")
	for _, line := range lines {
		lineInfo := rgx.FindStringSubmatch(line)
		if len(lineInfo) == 2 {
			prs = append(prs, lineInfo[1])
		}
	}

	sort.Strings(prs)
	return prs, nil
}

func loadPRinfo(pr string) (prInfo, error) {
	cmd := exec.Command("gh", "pr", "view", pr, "--json", "title,number,labels")
	out, err := cmd.Output()
	if err != nil {
		execErr := err.(*exec.ExitError)
		return prInfo{}, fmt.Errorf("%s:\nstderr: %s\nstdout: %s", err.Error(), execErr.Stderr, out)
	}
	var prInfo prInfo
	err = json.Unmarshal(out, &prInfo)
	return prInfo, err
}

func loadAllPRs(prs []string) ([]prInfo, error) {
	errChan := make(chan error)
	wgDone := make(chan bool)
	prChan := make(chan string, len(prs))
	// fill the work queue
	for _, s := range prs {
		prChan <- s
	}
	close(prChan)

	var prInfos []prInfo

	wg := sync.WaitGroup{}
	mu := sync.Mutex{}
	for i := 0; i < 10; i++ {
		wg.Add(1)
		go func() {
			// load meta data about PRs
			defer wg.Done()
			for b := range prChan {
				fmt.Print(".")
				prInfo, err := loadPRinfo(b)
				if err != nil {
					errChan <- err
					break
				}
				mu.Lock()
				prInfos = append(prInfos, prInfo)
				mu.Unlock()
			}
		}()
	}

	go func() {
		// wait for the loading to finish
		wg.Wait()
		close(wgDone)
	}()

	var err error
	select {
	case <-wgDone:
		break
	case err = <-errChan:
		break
	}

	fmt.Println()
	return prInfos, err
}

func groupPRs(prInfos []prInfo) map[string]map[string][]prInfo {
	prPerType := map[string]map[string][]prInfo{}

	for _, info := range prInfos {
		var typ, component string
		for _, lbl := range info.Labels {
			switch {
			case strings.HasPrefix(lbl.Name, prefixType):
				typ = strings.TrimPrefix(lbl.Name, prefixType)
			case strings.HasPrefix(lbl.Name, prefixComponent):
				component = strings.TrimPrefix(lbl.Name, prefixComponent)
			}
		}
		if typ == "" {
			typ = "Other"
		}
		if component == "" {
			component = "Other"
		}
		components, exists := prPerType[typ]
		if !exists {
			components = map[string][]prInfo{}
			prPerType[typ] = components
		}

		prsPerComponentAndType := components[component]
		components[component] = append(prsPerComponentAndType, info)
	}
	return prPerType
}

func writePrInfos(fileout string, prPerType map[string]map[string][]prInfo) (err error) {
	writeTo := os.Stdout
	if fileout != "" {
		writeTo, err = os.OpenFile(fileout, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666)
		if err != nil {
			return err
		}
	}

	t := template.Must(template.New("markdownTemplate").Parse(markdownTemplate))
	err = t.ExecuteTemplate(writeTo, "markdownTemplate", prPerType)
	if err != nil {
		return err
	}
	return nil
}

func main() {
	from := flag.String("from", "", "from sha/tag/branch")
	to := flag.String("to", "HEAD", "to sha/tag/branch")
	fileout := flag.String("file", "", "file on which to write release notes, stdout if empty")

	flag.Parse()

	prs, err := loadMergedPRs(*from, *to)
	if err != nil {
		log.Fatal(err)
	}

	prInfos, err := loadAllPRs(prs)
	if err != nil {
		log.Fatal(err)
	}

	prPerType := groupPRs(prInfos)

	err = writePrInfos(*fileout, prPerType)
	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>small beauty fixes<commit_after>/*
Copyright 2021 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"os"
	"os/exec"
	"regexp"
	"sort"
	"strings"
	"sync"
	"text/template"
)

type label struct {
	Name string `json:"name"`
}

type prInfo struct {
	Labels []label `json:"labels"`
	Number int     `json:"number"`
	Title  string  `json:"title"`
}

const (
	markdownTemplate = `
{{- range $typeName, $components := . }}
## {{ $typeName }}
{{- range $componentName, $component := $components }}
### {{ $componentName}}
{{- range $prInfo := $component }}
 - {{ $prInfo.Title }} #{{ $prInfo.Number }}
{{- end }}
{{- end }}
{{- end }}
`

	prefixType      = "Type: "
	prefixComponent = "Component: "
)

func loadMergedPRs(from, to string) ([]string, error) {
	cmd := exec.Command("git", "log", "--oneline", fmt.Sprintf("%s..%s", from, to))
	out, err := cmd.Output()
	if err != nil {
		execErr := err.(*exec.ExitError)
		return nil, fmt.Errorf("%s:\nstderr: %s\nstdout: %s", err.Error(), execErr.Stderr, out)
	}

	var prs []string
	rgx := regexp.MustCompile(`Merge pull request #(\d+)`)
	lines := strings.Split(string(out), "\n")
	for _, line := range lines {
		lineInfo := rgx.FindStringSubmatch(line)
		if len(lineInfo) == 2 {
			prs = append(prs, lineInfo[1])
		}
	}

	sort.Strings(prs)
	return prs, nil
}

func loadPRinfo(pr string) (prInfo, error) {
	cmd := exec.Command("gh", "pr", "view", pr, "--json", "title,number,labels")
	out, err := cmd.Output()
	if err != nil {
		execErr := err.(*exec.ExitError)
		return prInfo{}, fmt.Errorf("%s:\nstderr: %s\nstdout: %s", err.Error(), execErr.Stderr, out)
	}
	var prInfo prInfo
	err = json.Unmarshal(out, &prInfo)
	return prInfo, err
}

func loadAllPRs(prs []string) ([]prInfo, error) {
	errChan := make(chan error)
	wgDone := make(chan bool)
	prChan := make(chan string, len(prs))
	// fill the work queue
	for _, s := range prs {
		prChan <- s
	}
	close(prChan)

	var prInfos []prInfo
	fmt.Printf("Found %d merged PRs. Loading PR info", len(prs))
	wg := sync.WaitGroup{}
	mu := sync.Mutex{}
	for i := 0; i < 10; i++ {
		wg.Add(1)
		go func() {
			// load meta data about PRs
			defer wg.Done()
			for b := range prChan {
				fmt.Print(".")
				prInfo, err := loadPRinfo(b)
				if err != nil {
					errChan <- err
					break
				}
				mu.Lock()
				prInfos = append(prInfos, prInfo)
				mu.Unlock()
			}
		}()
	}

	go func() {
		// wait for the loading to finish
		wg.Wait()
		close(wgDone)
	}()

	var err error
	select {
	case <-wgDone:
		break
	case err = <-errChan:
		break
	}

	fmt.Println()
	return prInfos, err
}

func groupPRs(prInfos []prInfo) map[string]map[string][]prInfo {
	prPerType := map[string]map[string][]prInfo{}

	for _, info := range prInfos {
		var typ, component string
		for _, lbl := range info.Labels {
			switch {
			case strings.HasPrefix(lbl.Name, prefixType):
				typ = strings.TrimPrefix(lbl.Name, prefixType)
			case strings.HasPrefix(lbl.Name, prefixComponent):
				component = strings.TrimPrefix(lbl.Name, prefixComponent)
			}
		}
		switch typ {
		case "":
			typ = "Other"
		case "Bug":
			typ = "Bug fixes"
		}

		if typ == "" {
			typ = "Other"
		}
		if component == "" {
			component = "Other"
		}
		components, exists := prPerType[typ]
		if !exists {
			components = map[string][]prInfo{}
			prPerType[typ] = components
		}

		prsPerComponentAndType := components[component]
		components[component] = append(prsPerComponentAndType, info)
	}
	return prPerType
}

func writePrInfos(fileout string, prPerType map[string]map[string][]prInfo) (err error) {
	writeTo := os.Stdout
	if fileout != "" {
		writeTo, err = os.OpenFile(fileout, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666)
		if err != nil {
			return err
		}
	}

	t := template.Must(template.New("markdownTemplate").Parse(markdownTemplate))
	err = t.ExecuteTemplate(writeTo, "markdownTemplate", prPerType)
	if err != nil {
		return err
	}
	return nil
}

func main() {
	from := flag.String("from", "", "from sha/tag/branch")
	to := flag.String("to", "HEAD", "to sha/tag/branch")
	fileout := flag.String("file", "", "file on which to write release notes, stdout if empty")

	flag.Parse()

	prs, err := loadMergedPRs(*from, *to)
	if err != nil {
		log.Fatal(err)
	}

	prInfos, err := loadAllPRs(prs)
	if err != nil {
		log.Fatal(err)
	}

	prPerType := groupPRs(prInfos)

	err = writePrInfos(*fileout, prPerType)
	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package buildlet

import (
	"context"
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"time"

	"golang.org/x/build/internal/cloud"
	"google.golang.org/api/compute/v1"
)

// VMOpts control how new VMs are started.
type VMOpts struct {
	// Zone is the GCE zone to create the VM in.
	// Optional; defaults to provided build environment's zone.
	Zone string

	// ProjectID is the GCE project ID (e.g. "foo-bar-123", not
	// the numeric ID).
	// Optional; defaults to provided build environment's project ID ("name").
	ProjectID string

	// TLS optionally specifies the TLS keypair to use.
	// If zero, http without auth is used.
	TLS KeyPair

	// Optional description of the VM.
	Description string

	// Optional metadata to put on the instance.
	Meta map[string]string

	// DeleteIn optionally specifies a duration at which
	// to delete the VM.
	// If zero, a reasonable default is used.
	// Negative means no deletion timeout.
	DeleteIn time.Duration

	// OnInstanceRequested optionally specifies a hook to run synchronously
	// after the computeService.Instances.Insert call, but before
	// waiting for its operation to proceed.
	OnInstanceRequested func()

	// OnInstanceCreated optionally specifies a hook to run synchronously
	// after the instance operation succeeds.
	OnInstanceCreated func()

	// OnInstanceCreated optionally specifies a hook to run synchronously
	// after the computeService.Instances.Get call.
	// Only valid for GCE resources.
	OnGotInstanceInfo func(*compute.Instance)

	// OnInstanceCreated optionally specifies a hook to run synchronously
	// after the EC2 instance information is retrieved.
	// Only valid for EC2 resources.
	OnGotEC2InstanceInfo func(*cloud.Instance)

	// OnBeginBuildletProbe optionally specifies a hook to run synchronously
	// before StartNewVM tries to hit buildletURL to see if it's up yet.
	OnBeginBuildletProbe func(buildletURL string)

	// OnEndBuildletProbe optionally specifies a hook to run synchronously
	// after StartNewVM tries to hit the buildlet's URL to see if it's up.
	// The hook parameters are the return values from http.Get.
	OnEndBuildletProbe func(*http.Response, error)

	// SkipEndpointVerification does not verify that the builder is listening
	// on port 80 or 443 before creating a buildlet client.
	SkipEndpointVerification bool
}

// buildletClient returns a buildlet client configured to speak to a VM via the buildlet
// URL. The communication will use TLS if one is provided in the vmopts. This will wait until
// it can connect with the endpoint before returning. The buildletURL is in the form of:
// "https://<ip>". The ipPort field is in the form of "<ip>:<port>". The function
// will attempt to connect to the buildlet for the lesser of: the default timeout period
// (5 minutes) or the timeout set in the passed in context.
func buildletClient(ctx context.Context, buildletURL, ipPort string, opts *VMOpts) (*Client, error) {
	ctx, cancel := context.WithTimeout(ctx, 5*time.Minute)
	defer cancel()
	try := 0
	for !opts.SkipEndpointVerification {
		try++
		if ctx.Err() != nil {
			return nil, fmt.Errorf("unable to probe buildet at %s after %d attempts", buildletURL, try)
		}
		err := probeBuildlet(ctx, buildletURL, opts)
		if err == nil {
			break
		}
		log.Printf("probing buildlet at %s with attempt %d failed: %s", buildletURL, try, err)
		time.Sleep(time.Second)
	}
	return NewClient(ipPort, opts.TLS), nil
}

// probeBuildlet attempts to the connect to a buildlet at the provided URL. An error
// is returned if it unable to connect to the buildlet. Each request is limited by either
// a five second limit or the timeout set in the context.
func probeBuildlet(ctx context.Context, buildletURL string, opts *VMOpts) error {
	cl := &http.Client{
		Transport: &http.Transport{
			Dial:              defaultDialer(),
			DisableKeepAlives: true,
			TLSClientConfig: &tls.Config{
				InsecureSkipVerify: true,
			},
		},
	}
	if fn := opts.OnBeginBuildletProbe; fn != nil {
		fn(buildletURL)
	}
	ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
	defer cancel()
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, buildletURL, nil)
	if err != nil {
		return fmt.Errorf("error creating buildlet probe request: %w", err)
	}
	res, err := cl.Do(req)
	if fn := opts.OnEndBuildletProbe; fn != nil {
		fn(res, err)
	}
	if err != nil {
		return fmt.Errorf("error probe buildlet %s: %w", buildletURL, err)
	}
	ioutil.ReadAll(res.Body)
	res.Body.Close()
	if res.StatusCode != http.StatusOK {
		return fmt.Errorf("buildlet returned HTTP status code %d for %s", res.StatusCode, buildletURL)
	}
	return nil
}
<commit_msg>buildlet: increase timeout period for client<commit_after>// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package buildlet

import (
	"context"
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"time"

	"golang.org/x/build/internal/cloud"
	"google.golang.org/api/compute/v1"
)

// VMOpts control how new VMs are started.
type VMOpts struct {
	// Zone is the GCE zone to create the VM in.
	// Optional; defaults to provided build environment's zone.
	Zone string

	// ProjectID is the GCE project ID (e.g. "foo-bar-123", not
	// the numeric ID).
	// Optional; defaults to provided build environment's project ID ("name").
	ProjectID string

	// TLS optionally specifies the TLS keypair to use.
	// If zero, http without auth is used.
	TLS KeyPair

	// Optional description of the VM.
	Description string

	// Optional metadata to put on the instance.
	Meta map[string]string

	// DeleteIn optionally specifies a duration at which
	// to delete the VM.
	// If zero, a reasonable default is used.
	// Negative means no deletion timeout.
	DeleteIn time.Duration

	// OnInstanceRequested optionally specifies a hook to run synchronously
	// after the computeService.Instances.Insert call, but before
	// waiting for its operation to proceed.
	OnInstanceRequested func()

	// OnInstanceCreated optionally specifies a hook to run synchronously
	// after the instance operation succeeds.
	OnInstanceCreated func()

	// OnInstanceCreated optionally specifies a hook to run synchronously
	// after the computeService.Instances.Get call.
	// Only valid for GCE resources.
	OnGotInstanceInfo func(*compute.Instance)

	// OnInstanceCreated optionally specifies a hook to run synchronously
	// after the EC2 instance information is retrieved.
	// Only valid for EC2 resources.
	OnGotEC2InstanceInfo func(*cloud.Instance)

	// OnBeginBuildletProbe optionally specifies a hook to run synchronously
	// before StartNewVM tries to hit buildletURL to see if it's up yet.
	OnBeginBuildletProbe func(buildletURL string)

	// OnEndBuildletProbe optionally specifies a hook to run synchronously
	// after StartNewVM tries to hit the buildlet's URL to see if it's up.
	// The hook parameters are the return values from http.Get.
	OnEndBuildletProbe func(*http.Response, error)

	// SkipEndpointVerification does not verify that the builder is listening
	// on port 80 or 443 before creating a buildlet client.
	SkipEndpointVerification bool
}

// buildletClient returns a buildlet client configured to speak to a VM via the buildlet
// URL. The communication will use TLS if one is provided in the vmopts. This will wait until
// it can connect with the endpoint before returning. The buildletURL is in the form of:
// "https://<ip>". The ipPort field is in the form of "<ip>:<port>". The function
// will attempt to connect to the buildlet for the lesser of: the default timeout period
// (10 minutes) or the timeout set in the passed in context.
func buildletClient(ctx context.Context, buildletURL, ipPort string, opts *VMOpts) (*Client, error) {
	ctx, cancel := context.WithTimeout(ctx, 10*time.Minute)
	defer cancel()
	try := 0
	for !opts.SkipEndpointVerification {
		try++
		if ctx.Err() != nil {
			return nil, fmt.Errorf("unable to probe buildet at %s after %d attempts", buildletURL, try)
		}
		err := probeBuildlet(ctx, buildletURL, opts)
		if err == nil {
			break
		}
		log.Printf("probing buildlet at %s with attempt %d failed: %s", buildletURL, try, err)
		time.Sleep(time.Second)
	}
	return NewClient(ipPort, opts.TLS), nil
}

// probeBuildlet attempts to the connect to a buildlet at the provided URL. An error
// is returned if it unable to connect to the buildlet. Each request is limited by either
// a five second limit or the timeout set in the context.
func probeBuildlet(ctx context.Context, buildletURL string, opts *VMOpts) error {
	cl := &http.Client{
		Transport: &http.Transport{
			Dial:              defaultDialer(),
			DisableKeepAlives: true,
			TLSClientConfig: &tls.Config{
				InsecureSkipVerify: true,
			},
		},
	}
	if fn := opts.OnBeginBuildletProbe; fn != nil {
		fn(buildletURL)
	}
	ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
	defer cancel()
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, buildletURL, nil)
	if err != nil {
		return fmt.Errorf("error creating buildlet probe request: %w", err)
	}
	res, err := cl.Do(req)
	if fn := opts.OnEndBuildletProbe; fn != nil {
		fn(res, err)
	}
	if err != nil {
		return fmt.Errorf("error probe buildlet %s: %w", buildletURL, err)
	}
	ioutil.ReadAll(res.Body)
	res.Body.Close()
	if res.StatusCode != http.StatusOK {
		return fmt.Errorf("buildlet returned HTTP status code %d for %s", res.StatusCode, buildletURL)
	}
	return nil
}
<|endoftext|>


<commit_before>package layer

import (
	"encoding/gob"
	"errors"
	"gnd.la/cache"
	"gnd.la/cache/codec"
	"gnd.la/log"
	"gnd.la/mux"
	"net/http"
)

var (
	fromLayer = []string{"true"}
)

type cachedResponse struct {
	Header     http.Header
	StatusCode int
	Data       []byte
}

func New(c *cache.Cache, m Mediator) mux.Transformer {
	if c == nil {
		c = cache.NewDefault()
	}
	if m == nil {
		panic(errors.New("nil mediator passed to cache layer"))
	}
	return func(handler mux.Handler) mux.Handler {
		return func(ctx *mux.Context) {
			if m.Skip(ctx) {
				handler(ctx)
				return
			}
			key := m.Key(ctx)
			data := c.GetBytes(key)
			if data != nil {
				// has cached data
				var response *cachedResponse
				err := codec.GobCodec.Decode(data, &response)
				if err == nil {
					ctx.SetServedFromCache(true)
					header := ctx.Header()
					for k, v := range response.Header {
						header[k] = v
					}
					header["X-Gondola-From-Layer"] = fromLayer
					ctx.WriteHeader(response.StatusCode)
					ctx.Write(response.Data)
					return
				}
			}

			rw := ctx.ResponseWriter
			w := newWriter(rw)
			ctx.ResponseWriter = w
			handler(ctx)
			ctx.ResponseWriter = rw
			if m.Cache(ctx, w.statusCode, w.header) {
				response := &cachedResponse{w.header, w.statusCode, w.buf.Bytes()}
				data, err := codec.GobCodec.Encode(response)
				if err == nil {
					ctx.SetCached(true)
					expiration := m.Expires(ctx, w.statusCode, w.header)
					c.SetBytes(key, data, expiration)
				} else {
					log.Errorf("Error encoding cached response: %v", err)
				}
			}
		}
	}
}

func init() {
	gob.Register(&cachedResponse{})
}
<commit_msg>Update cache/layer to work with the latest cache API<commit_after>package layer

import (
	"encoding/gob"
	"errors"
	"gnd.la/cache"
	"gnd.la/cache/codec"
	"gnd.la/log"
	"gnd.la/mux"
	"net/http"
)

var (
	fromLayer  = []string{"true"}
	layerCodec = codec.Get("gob")
)

type cachedResponse struct {
	Header     http.Header
	StatusCode int
	Data       []byte
}

func New(c *cache.Cache, m Mediator) mux.Transformer {
	if c == nil {
		panic(errors.New("nil cache passed to cache layer"))
	}
	if m == nil {
		panic(errors.New("nil mediator passed to cache layer"))
	}
	return func(handler mux.Handler) mux.Handler {
		return func(ctx *mux.Context) {
			if m.Skip(ctx) {
				handler(ctx)
				return
			}
			key := m.Key(ctx)
			data, _ := c.GetBytes(key)
			if data != nil {
				// has cached data
				var response *cachedResponse
				err := layerCodec.Decode(data, &response)
				if err == nil {
					ctx.SetServedFromCache(true)
					header := ctx.Header()
					for k, v := range response.Header {
						header[k] = v
					}
					header["X-Gondola-From-Layer"] = fromLayer
					ctx.WriteHeader(response.StatusCode)
					ctx.Write(response.Data)
					return
				}
			}

			rw := ctx.ResponseWriter
			w := newWriter(rw)
			ctx.ResponseWriter = w
			handler(ctx)
			ctx.ResponseWriter = rw
			if m.Cache(ctx, w.statusCode, w.header) {
				response := &cachedResponse{w.header, w.statusCode, w.buf.Bytes()}
				data, err := layerCodec.Encode(response)
				if err == nil {
					ctx.SetCached(true)
					expiration := m.Expires(ctx, w.statusCode, w.header)
					c.SetBytes(key, data, expiration)
				} else {
					log.Errorf("Error encoding cached response: %v", err)
				}
			}
		}
	}
}

func init() {
	gob.Register(&cachedResponse{})
}
<|endoftext|>


<commit_before>package terraform

import (
	"github.com/hashicorp/terraform/config/module"
	"github.com/hashicorp/terraform/dag"
)

// DestroyApplyGraphBuilder implements GraphBuilder and is responsible for
// applying a pure-destroy plan.
//
// This graph builder is very similar to the ApplyGraphBuilder but
// is slightly simpler.
type DestroyApplyGraphBuilder struct {
	// Module is the root module for the graph to build.
	Module *module.Tree

	// Diff is the diff to apply.
	Diff *Diff

	// State is the current state
	State *State

	// Providers is the list of providers supported.
	Providers []string

	// DisableReduce, if true, will not reduce the graph. Great for testing.
	DisableReduce bool
}

// See GraphBuilder
func (b *DestroyApplyGraphBuilder) Build(path []string) (*Graph, error) {
	return (&BasicGraphBuilder{
		Steps:    b.Steps(),
		Validate: true,
	}).Build(path)
}

// See GraphBuilder
func (b *DestroyApplyGraphBuilder) Steps() []GraphTransformer {
	// Custom factory for creating providers.
	providerFactory := func(name string, path []string) GraphNodeProvider {
		return &NodeApplyableProvider{
			NameValue: name,
			PathValue: path,
		}
	}

	concreteResource := func(a *NodeAbstractResource) dag.Vertex {
		return &NodeApplyableResource{
			NodeAbstractResource: a,
		}
	}

	steps := []GraphTransformer{
		// Creates all the nodes represented in the diff.
		&DiffTransformer{
			Concrete: concreteResource,

			Diff:   b.Diff,
			Module: b.Module,
			State:  b.State,
		},

		// Create orphan output nodes
		&OrphanOutputTransformer{Module: b.Module, State: b.State},

		// Attach the configuration to any resources
		&AttachResourceConfigTransformer{Module: b.Module},

		// Attach the state
		&AttachStateTransformer{State: b.State},

		// Destruction ordering. NOTE: For destroys, we don't need to
		// do any CBD stuff, so that is explicitly not here.
		&DestroyEdgeTransformer{Module: b.Module, State: b.State},

		// Create all the providers
		&MissingProviderTransformer{Providers: b.Providers, Factory: providerFactory},
		&ProviderTransformer{},
		&ParentProviderTransformer{},
		&AttachProviderConfigTransformer{Module: b.Module},

		// Add root variables
		&RootVariableTransformer{Module: b.Module},

		// Add module variables
		&ModuleVariableTransformer{Module: b.Module},

		// Add the outputs
		&OutputTransformer{Module: b.Module},

		// Connect references so ordering is correct
		&ReferenceTransformer{},

		// Add the node to fix the state count boundaries
		&CountBoundaryTransformer{},

		// Single root
		&RootTransformer{},
	}

	if !b.DisableReduce {
		// Perform the transitive reduction to make our graph a bit
		// more sane if possible (it usually is possible).
		steps = append(steps, &TransitiveReductionTransformer{})
	}

	return steps
}
<commit_msg>terraform: destroy apply graph builder should disable providers<commit_after>package terraform

import (
	"github.com/hashicorp/terraform/config/module"
	"github.com/hashicorp/terraform/dag"
)

// DestroyApplyGraphBuilder implements GraphBuilder and is responsible for
// applying a pure-destroy plan.
//
// This graph builder is very similar to the ApplyGraphBuilder but
// is slightly simpler.
type DestroyApplyGraphBuilder struct {
	// Module is the root module for the graph to build.
	Module *module.Tree

	// Diff is the diff to apply.
	Diff *Diff

	// State is the current state
	State *State

	// Providers is the list of providers supported.
	Providers []string

	// DisableReduce, if true, will not reduce the graph. Great for testing.
	DisableReduce bool
}

// See GraphBuilder
func (b *DestroyApplyGraphBuilder) Build(path []string) (*Graph, error) {
	return (&BasicGraphBuilder{
		Steps:    b.Steps(),
		Validate: true,
	}).Build(path)
}

// See GraphBuilder
func (b *DestroyApplyGraphBuilder) Steps() []GraphTransformer {
	// Custom factory for creating providers.
	providerFactory := func(name string, path []string) GraphNodeProvider {
		return &NodeApplyableProvider{
			NameValue: name,
			PathValue: path,
		}
	}

	concreteResource := func(a *NodeAbstractResource) dag.Vertex {
		return &NodeApplyableResource{
			NodeAbstractResource: a,
		}
	}

	steps := []GraphTransformer{
		// Creates all the nodes represented in the diff.
		&DiffTransformer{
			Concrete: concreteResource,

			Diff:   b.Diff,
			Module: b.Module,
			State:  b.State,
		},

		// Create orphan output nodes
		&OrphanOutputTransformer{Module: b.Module, State: b.State},

		// Attach the configuration to any resources
		&AttachResourceConfigTransformer{Module: b.Module},

		// Attach the state
		&AttachStateTransformer{State: b.State},

		// Destruction ordering. NOTE: For destroys, we don't need to
		// do any CBD stuff, so that is explicitly not here.
		&DestroyEdgeTransformer{Module: b.Module, State: b.State},

		// Create all the providers
		&MissingProviderTransformer{Providers: b.Providers, Factory: providerFactory},
		&ProviderTransformer{},
		&DisableProviderTransformer{},
		&ParentProviderTransformer{},
		&AttachProviderConfigTransformer{Module: b.Module},

		// Add root variables
		&RootVariableTransformer{Module: b.Module},

		// Add module variables
		&ModuleVariableTransformer{Module: b.Module},

		// Add the outputs
		&OutputTransformer{Module: b.Module},

		// Connect references so ordering is correct
		&ReferenceTransformer{},

		// Add the node to fix the state count boundaries
		&CountBoundaryTransformer{},

		// Single root
		&RootTransformer{},
	}

	if !b.DisableReduce {
		// Perform the transitive reduction to make our graph a bit
		// more sane if possible (it usually is possible).
		steps = append(steps, &TransitiveReductionTransformer{})
	}

	return steps
}
<|endoftext|>