<commit_before>package gop

import (
    "github.com/gorilla/mux"
    "github.com/gorilla/context"
    "github.com/gorilla/schema"
    "github.com/jbert/timber"

    "fmt"
    "net"
    "time"
    "runtime"
    "net/http"
    "strings"

)

// Stuff we include in both App and Req, for convenience
type common struct {
    timber.Logger
    Cfg             *Config
    Stats           StatsdClient
    Decoder         *schema.Decoder
}

// Represents a managed gop app. Returned by init.
// Embeds logging, provides .Cfg for configuration access.
type App struct {
    common

    AppName         string
    ProjectName     string
    GorillaRouter   *mux.Router
    listener        net.Listener
    wantReq         chan *wantReq
    doneReq         chan *Req
    getReqs         chan chan *Req
    startTime       time.Time
    currentReqs     int
    totalReqs       int
    doingGraceful   bool
}

// Used for RPC to get a new request
type wantReq struct {
    r       *http.Request
    reply   chan *Req
}

// Per request struct. has convenience references to functionality
// in the app singleton. Passed into the request handler.
type Req struct {
    common

    id              int
    startTime       time.Time
    app             *App
    r               *http.Request
    RealRemoteIP    string
    IsHTTPS         bool
}

// The function signature your http handlers need.
type HandlerFunc func(g *Req, w http.ResponseWriter, r *http.Request)

// Set up the application. Reads config. Panic if runtime environment is deficient.
func Init(projectName, appName string) *App {
    app := &App{
        common: common{
            Decoder:        schema.NewDecoder(),
        },
        AppName:        appName,
        ProjectName:    projectName,
        GorillaRouter:  mux.NewRouter(),
        wantReq:        make(chan *wantReq),
        doneReq:        make(chan *Req),
        getReqs:        make(chan chan *Req),
        startTime:      time.Now(),
    }

    app.loadAppConfigFile()

    app.setUserAndGroup()

    app.initLogging()

    maxProcs, _ := app.Cfg.GetInt("gop", "maxprocs", 4 * runtime.NumCPU())
    app.Debug("Seting maxprocs to %d\n", maxProcs)
    runtime.GOMAXPROCS(maxProcs)

    app.initStatsd()

    app.goAgainSetup()

    app.registerGopHandlers()

    go app.watchdog()

    go app.requestMaker()

    return app
}

func (a *App) setUserAndGroup() {
    // We do not have logging set up yet. We just panic() on error.

    desiredUserName, _ := a.Cfg.Get("gop", "user", "")
    // Usernames it's ok to run as, in order of preference
    possibleUserNames := []string{desiredUserName, a.AppName, a.ProjectName}

    doneIt := false
    for _, desiredUserName := range possibleUserNames {
        if runAsUserName(desiredUserName) {
            doneIt = true
            break
        }
    }

    if !doneIt {
        panic(fmt.Sprintf("Can't run as any of these users: %v, please set config and/or create user", possibleUserNames))
    }

// Can't log at this stage :-/
//    a.Info("Running as user %s (%d)", desiredUserName, desiredUser.Uid)
}

// Hands out 'request' objects
func (a *App) requestMaker() {
    nextReqId := 0
    openReqs := make(map[int] *Req)
    for {
        select {
            case wantReq := <- a.wantReq: {
                realRemoteIP := wantReq.r.RemoteAddr
                isHTTPS := false
                useXF, _:= a.Cfg.GetBool("gop", "use_xf_headers", false)
                if useXF {
                    xff := wantReq.r.Header.Get("X-Forwarded-For")
                    if xff != "" {
                        ips := strings.Split(xff, ",")
                        for i, ip := range ips {
                            ips[i] = strings.TrimSpace(ip)
                        }
                        // The only trustworthy component is the *last* (and only if we
                        // are behind nginx or other proxy which is stripping x-f-f from incoming requests)
                        realRemoteIP = ips[len(ips)-1]
                    }
                    xfp := wantReq.r.Header.Get("X-Forwarded-Proto")
                    isHTTPS = strings.ToLower(xfp) == "https"
                }
                req := Req{
                    common: common{
                        Logger:     a.Logger,
                        Cfg:        a.Cfg,
                        Stats:      a.Stats,
                        Decoder:    a.Decoder,
                    },

                    id:         nextReqId,
                    app:        a,
                    startTime:  time.Now(),
                    r:          wantReq.r,
                    RealRemoteIP: realRemoteIP,
                    IsHTTPS:    isHTTPS,
                }
                openReqs[req.id] = &req
                nextReqId++
                a.Stats.Inc("http_reqs", 1)
                a.Stats.Inc("current_http_reqs", 1)
                a.totalReqs++
                a.currentReqs++
                wantReq.reply <- &req
            }
            case doneReq := <- a.doneReq: {
                _, found := openReqs[doneReq.id]
                if !found {
                    a.Error("BUG! Unknown request id [%d] being retired")
                } else {
                    a.Stats.Dec("current_http_reqs", 1)
                    doneReq.finished()
                    a.currentReqs--
                    delete (openReqs, doneReq.id)
                }
            }
            case reply := <- a.getReqs: {
                go func() {
                    for _, req := range openReqs {
                        reply <- req
                    }
                    close(reply)
                }()
            }
        }
    }
}

// Ask requestMaker for a request
func (a *App) getReq(r *http.Request) *Req {
    reply := make(chan *Req)
    a.wantReq <- &wantReq{r: r, reply: reply}
    return <- reply
}

func (g *Req) finished() {
    context.Clear(g.r)      // Cleanup  gorilla stash

    reqDuration := time.Since(g.startTime)
    slowReqSecs, _ := g.Cfg.GetFloat32("gop", "slow_req_secs", 10)
    if reqDuration.Seconds() > float64(slowReqSecs) {
        g.Error("Slow request [%s] took %s", g.r.URL, reqDuration)
    } else {
        g.Info("Request took %s", reqDuration)
    }

    // Tidy up request finalistion (requestMaker, req.finish() method, app.requestFinished())
    restartReqs, _ := g.Cfg.GetInt("gop", "max_requests", 0)
    if restartReqs > 0 && g.app.totalReqs > restartReqs {
        g.Error("Graceful restart after max_requests: %d", restartReqs)
        g.app.StartGracefulRestart("Max requests reached")
    }

    gcEveryReqs, _ := g.Cfg.GetInt("gop", "gc_requests", 0)
    if gcEveryReqs > 0 && g.app.totalReqs % gcEveryReqs == 0 {
        g.Info("Forcing GC after %d reqs", g.app.totalReqs)
        runtime.GC()
    }
}

func (a *App) initLogging() {

    logPattern, _ := a.Cfg.Get("gop", "log_pattern", "[%D %T] [%L] %S %M")

    // If set, hack all logging to stdout for dev
    forceStdout, _:= a.Cfg.GetBool("gop", "stdout_only_logging", false)
    configLogger := timber.ConfigLogger{
        LogWriter:  new(timber.ConsoleWriter),
        Level:      timber.INFO,
        Formatter:  timber.NewPatFormatter(logPattern),
    }

    if !forceStdout {
        defaultLogDir, _ := a.Cfg.Get("gop", "log_dir", "/var/log")
        defaultLogFname := defaultLogDir + "/" + a.ProjectName + "/" + a.AppName + ".log"
        logFname, _ := a.Cfg.Get("gop", "log_file", defaultLogFname)

        newWriter, err := timber.NewFileWriter(logFname)
        if err != nil {
            panic(fmt.Sprintf("Can't open log file: %s", err))
        }
        configLogger.LogWriter = newWriter
    }

    logLevelStr, _ := a.Cfg.Get("gop", "log_level", "INFO")
    logLevelStr = strings.ToUpper(logLevelStr)
    for logLevel, levelStr := range timber.LongLevelStrings {
        if logLevelStr == levelStr {
            configLogger.Level = timber.Level(logLevel)
            break
        }
    }

    l := timber.NewTimber()
    l.AddLogger(configLogger)
    a.Logger = l
}

func (a *App) watchdog() {
    repeat, _ := a.Cfg.GetInt("gop", "watchdog_secs", 300)
    ticker := time.Tick(time.Second * time.Duration(repeat))

    for {
        sysMemBytesLimit, _ := a.Cfg.GetInt64("gop", "sysmem_bytes_limit", 0)
        allocMemBytesLimit, _ := a.Cfg.GetInt64("gop", "allocmem_bytes_limit", 0)
        numFDsLimit, _ := a.Cfg.GetInt64("gop", "numfds_limit", 0)
        numGorosLimit, _ := a.Cfg.GetInt64("gop", "numgoros_limit", 0)

        sysMemBytes, allocMemBytes := getMemInfo()
        numFDs, err := fdsInUse()
        numGoros := int64(runtime.NumGoroutine())
        if err != nil {
            a.Error("Failed to get number of fds in use: %s", err.Error())
            // Continue without
        }

        a.Info("TICK: sys=%d,alloc=%d,fds=%d,current_req=%d,total_req=%d,goros=%d",
            sysMemBytes,
            allocMemBytes,
            numFDs,
            a.currentReqs,
            a.totalReqs,
            numGoros)
        a.Stats.Gauge("mem.sys", sysMemBytes)
        a.Stats.Gauge("mem.alloc", allocMemBytes)
        a.Stats.Gauge("numfds", numFDs)
        a.Stats.Gauge("numgoro", numGoros)

        if sysMemBytesLimit > 0 && sysMemBytes >= sysMemBytesLimit {
            a.Error("SYS MEM LIMIT REACHED [%d >= %d] - starting graceful restart", sysMemBytes, sysMemBytesLimit)
            a.StartGracefulRestart("Sys Memory limit reached")
        }
        if allocMemBytesLimit > 0 && allocMemBytes >= allocMemBytesLimit {
            a.Error("ALLOC MEM LIMIT REACHED [%d >= %d] - starting graceful restart", allocMemBytes, allocMemBytesLimit)
            a.StartGracefulRestart("Alloc Memory limit reached")
        }
        if numFDsLimit > 0 && numFDs >= numFDsLimit {
            a.Error("NUM FDS LIMIT REACHED [%d >= %d] - starting graceful restart", numFDs, numFDsLimit)
            a.StartGracefulRestart("Number of fds limit reached")
        }
        if numGorosLimit > 0 && numGoros >= numGorosLimit {
            a.Error("NUM GOROS LIMIT REACHED [%d >= %d] - starting graceful restart", numGoros, numGorosLimit)
            a.StartGracefulRestart("Number of goros limit reached")
        }

        restartAfterSecs, _ := a.Cfg.GetFloat32("gop", "restart_after_secs", 0)
        appRunTime := time.Since(a.startTime).Seconds()
        if restartAfterSecs > 0 && appRunTime > float64(restartAfterSecs) {
            a.Error("TIME LIMIT REACHED [%f >= %f] - starting graceful restart", appRunTime, restartAfterSecs)
            a.StartGracefulRestart("Run time limit reached")
        }
        <- ticker
    }
}

type responseWriter struct {
    http.ResponseWriter
    app *App
    code int
    notedCode bool
}

// Satisfy the interface
func (w *responseWriter) Header() http.Header {
    return w.ResponseWriter.Header()
}

func (w *responseWriter) Write(buf []byte) (int, error) {
    w.noteCode()
    return w.ResponseWriter.Write(buf)
}

func (w *responseWriter) WriteHeader(code int) {
    w.code = code
    w.noteCode()
    w.ResponseWriter.WriteHeader(code)
}

func (w *responseWriter) noteCode() {
    if !w.notedCode {
        statsKey := fmt.Sprintf("http_status.%d", w.code)
        w.app.Stats.Inc(statsKey, 1)
        w.notedCode = true
    }
}

func (a *App) WrapHandler(h HandlerFunc) http.HandlerFunc {
    // Wrap the handler, so we can do before/after logic
    f := func(w http.ResponseWriter, r *http.Request) {
        gopRequest := a.getReq(r)
        defer func() {
            a.doneReq <- gopRequest
        }()
        gopWriter := responseWriter{code: 200, app: a, ResponseWriter: w}

        err := r.ParseForm()
        if err != nil {
            http.Error(&gopWriter, "Failed to parse form: " + err.Error(), http.StatusInternalServerError)
            return
        }

        // Pass in the gop, for logging, cfg etc
        h(gopRequest, &gopWriter, r)
    }

    return http.HandlerFunc(f)
}

// Register an http handler managed by gop.
// We use Gorilla muxxer, since it is back-compatible and nice to use :-)
func (a *App) HandleFunc(u string, h HandlerFunc) {
    gopHandler := a.WrapHandler(h)

    a.GorillaRouter.HandleFunc(u, gopHandler)
}

func (a *App) Run() {
    listenAddr, _ := a.Cfg.Get("gop", "listen_addr", ":http")
    listenNet, _ := a.Cfg.Get("gop", "listen_net", "tcp")
    a.goAgainListenAndServe(listenNet, listenAddr)
}

func (a *App) Serve(l net.Listener) {
    http.Serve(l, a.GorillaRouter)
}

func getMemInfo() (int64, int64) {
    var memStats runtime.MemStats
    runtime.ReadMemStats(&memStats)
    // There's lots of stuff in here.
    return int64(memStats.Sys), int64(memStats.Alloc)
}
<commit_msg>setup timber to be the standard logger - for 3rd party code (e.g. goamz)<commit_after>package gop

import (
    "github.com/gorilla/mux"
    "github.com/gorilla/context"
    "github.com/gorilla/schema"
    "github.com/jbert/timber"

    "fmt"
    "net"
    "time"
    "runtime"
    "net/http"
    "strings"
    "log"

)

// Stuff we include in both App and Req, for convenience
type common struct {
    timber.Logger
    Cfg             *Config
    Stats           StatsdClient
    Decoder         *schema.Decoder
}

// Represents a managed gop app. Returned by init.
// Embeds logging, provides .Cfg for configuration access.
type App struct {
    common

    AppName         string
    ProjectName     string
    GorillaRouter   *mux.Router
    listener        net.Listener
    wantReq         chan *wantReq
    doneReq         chan *Req
    getReqs         chan chan *Req
    startTime       time.Time
    currentReqs     int
    totalReqs       int
    doingGraceful   bool
}

// Used for RPC to get a new request
type wantReq struct {
    r       *http.Request
    reply   chan *Req
}

// Per request struct. has convenience references to functionality
// in the app singleton. Passed into the request handler.
type Req struct {
    common

    id              int
    startTime       time.Time
    app             *App
    r               *http.Request
    RealRemoteIP    string
    IsHTTPS         bool
}

// The function signature your http handlers need.
type HandlerFunc func(g *Req, w http.ResponseWriter, r *http.Request)

// Set up the application. Reads config. Panic if runtime environment is deficient.
func Init(projectName, appName string) *App {
    app := &App{
        common: common{
            Decoder:        schema.NewDecoder(),
        },
        AppName:        appName,
        ProjectName:    projectName,
        GorillaRouter:  mux.NewRouter(),
        wantReq:        make(chan *wantReq),
        doneReq:        make(chan *Req),
        getReqs:        make(chan chan *Req),
        startTime:      time.Now(),
    }

    app.loadAppConfigFile()

    app.setUserAndGroup()

    app.initLogging()

    maxProcs, _ := app.Cfg.GetInt("gop", "maxprocs", 4 * runtime.NumCPU())
    app.Debug("Seting maxprocs to %d\n", maxProcs)
    runtime.GOMAXPROCS(maxProcs)

    app.initStatsd()

    app.goAgainSetup()

    app.registerGopHandlers()

    go app.watchdog()

    go app.requestMaker()

    return app
}

func (a *App) setUserAndGroup() {
    // We do not have logging set up yet. We just panic() on error.

    desiredUserName, _ := a.Cfg.Get("gop", "user", "")
    // Usernames it's ok to run as, in order of preference
    possibleUserNames := []string{desiredUserName, a.AppName, a.ProjectName}

    doneIt := false
    for _, desiredUserName := range possibleUserNames {
        if runAsUserName(desiredUserName) {
            doneIt = true
            break
        }
    }

    if !doneIt {
        panic(fmt.Sprintf("Can't run as any of these users: %v, please set config and/or create user", possibleUserNames))
    }

// Can't log at this stage :-/
//    a.Info("Running as user %s (%d)", desiredUserName, desiredUser.Uid)
}

// Hands out 'request' objects
func (a *App) requestMaker() {
    nextReqId := 0
    openReqs := make(map[int] *Req)
    for {
        select {
            case wantReq := <- a.wantReq: {
                realRemoteIP := wantReq.r.RemoteAddr
                isHTTPS := false
                useXF, _:= a.Cfg.GetBool("gop", "use_xf_headers", false)
                if useXF {
                    xff := wantReq.r.Header.Get("X-Forwarded-For")
                    if xff != "" {
                        ips := strings.Split(xff, ",")
                        for i, ip := range ips {
                            ips[i] = strings.TrimSpace(ip)
                        }
                        // The only trustworthy component is the *last* (and only if we
                        // are behind nginx or other proxy which is stripping x-f-f from incoming requests)
                        realRemoteIP = ips[len(ips)-1]
                    }
                    xfp := wantReq.r.Header.Get("X-Forwarded-Proto")
                    isHTTPS = strings.ToLower(xfp) == "https"
                }
                req := Req{
                    common: common{
                        Logger:     a.Logger,
                        Cfg:        a.Cfg,
                        Stats:      a.Stats,
                        Decoder:    a.Decoder,
                    },

                    id:         nextReqId,
                    app:        a,
                    startTime:  time.Now(),
                    r:          wantReq.r,
                    RealRemoteIP: realRemoteIP,
                    IsHTTPS:    isHTTPS,
                }
                openReqs[req.id] = &req
                nextReqId++
                a.Stats.Inc("http_reqs", 1)
                a.Stats.Inc("current_http_reqs", 1)
                a.totalReqs++
                a.currentReqs++
                wantReq.reply <- &req
            }
            case doneReq := <- a.doneReq: {
                _, found := openReqs[doneReq.id]
                if !found {
                    a.Error("BUG! Unknown request id [%d] being retired")
                } else {
                    a.Stats.Dec("current_http_reqs", 1)
                    doneReq.finished()
                    a.currentReqs--
                    delete (openReqs, doneReq.id)
                }
            }
            case reply := <- a.getReqs: {
                go func() {
                    for _, req := range openReqs {
                        reply <- req
                    }
                    close(reply)
                }()
            }
        }
    }
}

// Ask requestMaker for a request
func (a *App) getReq(r *http.Request) *Req {
    reply := make(chan *Req)
    a.wantReq <- &wantReq{r: r, reply: reply}
    return <- reply
}

func (g *Req) finished() {
    context.Clear(g.r)      // Cleanup  gorilla stash

    reqDuration := time.Since(g.startTime)
    slowReqSecs, _ := g.Cfg.GetFloat32("gop", "slow_req_secs", 10)
    if reqDuration.Seconds() > float64(slowReqSecs) {
        g.Error("Slow request [%s] took %s", g.r.URL, reqDuration)
    } else {
        g.Info("Request took %s", reqDuration)
    }

    // Tidy up request finalistion (requestMaker, req.finish() method, app.requestFinished())
    restartReqs, _ := g.Cfg.GetInt("gop", "max_requests", 0)
    if restartReqs > 0 && g.app.totalReqs > restartReqs {
        g.Error("Graceful restart after max_requests: %d", restartReqs)
        g.app.StartGracefulRestart("Max requests reached")
    }

    gcEveryReqs, _ := g.Cfg.GetInt("gop", "gc_requests", 0)
    if gcEveryReqs > 0 && g.app.totalReqs % gcEveryReqs == 0 {
        g.Info("Forcing GC after %d reqs", g.app.totalReqs)
        runtime.GC()
    }
}

func (a *App) initLogging() {

    logPattern, _ := a.Cfg.Get("gop", "log_pattern", "[%D %T] [%L] %S %M")

    // If set, hack all logging to stdout for dev
    forceStdout, _:= a.Cfg.GetBool("gop", "stdout_only_logging", false)
    configLogger := timber.ConfigLogger{
        LogWriter:  new(timber.ConsoleWriter),
        Level:      timber.INFO,
        Formatter:  timber.NewPatFormatter(logPattern),
    }

    if !forceStdout {
        defaultLogDir, _ := a.Cfg.Get("gop", "log_dir", "/var/log")
        defaultLogFname := defaultLogDir + "/" + a.ProjectName + "/" + a.AppName + ".log"
        logFname, _ := a.Cfg.Get("gop", "log_file", defaultLogFname)

        newWriter, err := timber.NewFileWriter(logFname)
        if err != nil {
            panic(fmt.Sprintf("Can't open log file: %s", err))
        }
        configLogger.LogWriter = newWriter
    }

    logLevelStr, _ := a.Cfg.Get("gop", "log_level", "INFO")
    logLevelStr = strings.ToUpper(logLevelStr)
    for logLevel, levelStr := range timber.LongLevelStrings {
        if logLevelStr == levelStr {
            configLogger.Level = timber.Level(logLevel)
            break
        }
    }

    l := timber.NewTimber()
    l.AddLogger(configLogger)
    a.Logger = l

    // Set up the default go logger to go here too, so 3rd party
    // module logging plays nicely
    log.SetFlags(0)
    log.SetOutput(timber.Global)
}

func (a *App) watchdog() {
    repeat, _ := a.Cfg.GetInt("gop", "watchdog_secs", 300)
    ticker := time.Tick(time.Second * time.Duration(repeat))

    for {
        sysMemBytesLimit, _ := a.Cfg.GetInt64("gop", "sysmem_bytes_limit", 0)
        allocMemBytesLimit, _ := a.Cfg.GetInt64("gop", "allocmem_bytes_limit", 0)
        numFDsLimit, _ := a.Cfg.GetInt64("gop", "numfds_limit", 0)
        numGorosLimit, _ := a.Cfg.GetInt64("gop", "numgoros_limit", 0)

        sysMemBytes, allocMemBytes := getMemInfo()
        numFDs, err := fdsInUse()
        numGoros := int64(runtime.NumGoroutine())
        if err != nil {
            a.Error("Failed to get number of fds in use: %s", err.Error())
            // Continue without
        }

        a.Info("TICK: sys=%d,alloc=%d,fds=%d,current_req=%d,total_req=%d,goros=%d",
            sysMemBytes,
            allocMemBytes,
            numFDs,
            a.currentReqs,
            a.totalReqs,
            numGoros)
        a.Stats.Gauge("mem.sys", sysMemBytes)
        a.Stats.Gauge("mem.alloc", allocMemBytes)
        a.Stats.Gauge("numfds", numFDs)
        a.Stats.Gauge("numgoro", numGoros)

        if sysMemBytesLimit > 0 && sysMemBytes >= sysMemBytesLimit {
            a.Error("SYS MEM LIMIT REACHED [%d >= %d] - starting graceful restart", sysMemBytes, sysMemBytesLimit)
            a.StartGracefulRestart("Sys Memory limit reached")
        }
        if allocMemBytesLimit > 0 && allocMemBytes >= allocMemBytesLimit {
            a.Error("ALLOC MEM LIMIT REACHED [%d >= %d] - starting graceful restart", allocMemBytes, allocMemBytesLimit)
            a.StartGracefulRestart("Alloc Memory limit reached")
        }
        if numFDsLimit > 0 && numFDs >= numFDsLimit {
            a.Error("NUM FDS LIMIT REACHED [%d >= %d] - starting graceful restart", numFDs, numFDsLimit)
            a.StartGracefulRestart("Number of fds limit reached")
        }
        if numGorosLimit > 0 && numGoros >= numGorosLimit {
            a.Error("NUM GOROS LIMIT REACHED [%d >= %d] - starting graceful restart", numGoros, numGorosLimit)
            a.StartGracefulRestart("Number of goros limit reached")
        }

        restartAfterSecs, _ := a.Cfg.GetFloat32("gop", "restart_after_secs", 0)
        appRunTime := time.Since(a.startTime).Seconds()
        if restartAfterSecs > 0 && appRunTime > float64(restartAfterSecs) {
            a.Error("TIME LIMIT REACHED [%f >= %f] - starting graceful restart", appRunTime, restartAfterSecs)
            a.StartGracefulRestart("Run time limit reached")
        }
        <- ticker
    }
}

type responseWriter struct {
    http.ResponseWriter
    app *App
    code int
    notedCode bool
}

// Satisfy the interface
func (w *responseWriter) Header() http.Header {
    return w.ResponseWriter.Header()
}

func (w *responseWriter) Write(buf []byte) (int, error) {
    w.noteCode()
    return w.ResponseWriter.Write(buf)
}

func (w *responseWriter) WriteHeader(code int) {
    w.code = code
    w.noteCode()
    w.ResponseWriter.WriteHeader(code)
}

func (w *responseWriter) noteCode() {
    if !w.notedCode {
        statsKey := fmt.Sprintf("http_status.%d", w.code)
        w.app.Stats.Inc(statsKey, 1)
        w.notedCode = true
    }
}

func (a *App) WrapHandler(h HandlerFunc) http.HandlerFunc {
    // Wrap the handler, so we can do before/after logic
    f := func(w http.ResponseWriter, r *http.Request) {
        gopRequest := a.getReq(r)
        defer func() {
            a.doneReq <- gopRequest
        }()
        gopWriter := responseWriter{code: 200, app: a, ResponseWriter: w}

        err := r.ParseForm()
        if err != nil {
            http.Error(&gopWriter, "Failed to parse form: " + err.Error(), http.StatusInternalServerError)
            return
        }

        // Pass in the gop, for logging, cfg etc
        h(gopRequest, &gopWriter, r)
    }

    return http.HandlerFunc(f)
}

// Register an http handler managed by gop.
// We use Gorilla muxxer, since it is back-compatible and nice to use :-)
func (a *App) HandleFunc(u string, h HandlerFunc) {
    gopHandler := a.WrapHandler(h)

    a.GorillaRouter.HandleFunc(u, gopHandler)
}

func (a *App) Run() {
    listenAddr, _ := a.Cfg.Get("gop", "listen_addr", ":http")
    listenNet, _ := a.Cfg.Get("gop", "listen_net", "tcp")
    a.goAgainListenAndServe(listenNet, listenAddr)
}

func (a *App) Serve(l net.Listener) {
    http.Serve(l, a.GorillaRouter)
}

func getMemInfo() (int64, int64) {
    var memStats runtime.MemStats
    runtime.ReadMemStats(&memStats)
    // There's lots of stuff in here.
    return int64(memStats.Sys), int64(memStats.Alloc)
}
<|endoftext|>


<commit_before>package tasks

import (
	"context"
	"fmt"
	"log"
	"math"
	"os"
	"runtime/trace"
	"time"

	"pixur.org/pixur/be/schema/db"
	"pixur.org/pixur/be/status"
)

const (
	maxTaskRetries = 5
)

var thetasklogger *log.Logger = log.New(os.Stderr, "", log.LstdFlags)

type TaskRunner struct {
	run    func(context.Context, Task) status.S
	thelog *log.Logger
}

func TestTaskRunner(run func(context.Context, Task) status.S) *TaskRunner {
	return &TaskRunner{
		run: run,
	}
}

func (r *TaskRunner) Run(ctx context.Context, task Task) status.S {
	if trace.IsEnabled() {
		var tracetask *trace.Task
		ctx, tracetask = trace.NewTask(ctx, fmt.Sprintf("%T", task))
		defer tracetask.End()
	}
	if r != nil && r.run != nil {
		return r.run(ctx, task)
	}
	return runTask(ctx, task)
}

func stsSliceToErrSlice(ss []status.S) []error {
	es := make([]error, 0, len(ss))
	for _, s := range ss {
		es = append(es, s)
	}
	return es
}

var (
	taskInitialBackoffDelay      = time.Second
	taskInitialBackoffMultiplier = math.Phi
)

func runTask(ctx context.Context, task Task) (stscap status.S) {
	var failures []status.S
	defer func() {
		if stscap == nil && len(failures) != 0 {
			if thetasklogger != nil {
				warnsts := status.Unknown(nil, "Encountered errors while running")
				warnsts = status.WithSuppressed(warnsts, stsSliceToErrSlice(failures)...)

				thetasklogger.Println(warnsts)
			}
		}
	}()

	backoff := taskInitialBackoffDelay
	deadline, deadlineok := ctx.Deadline()
	for i := 0; i < maxTaskRetries; i++ {
		select {
		case <-ctx.Done():
			failures = append(failures, status.From(ctx.Err()))
			break
		default:
		}
		sts := task.Run(ctx)
		if sts == nil {
			return nil
		}
		failures = append(failures, sts)

		if cause := unwrapTaskStatus(sts); cause != nil {
			if retryable, ok := cause.(db.Retryable); ok {
				if retryable.CanRetry() {
					sleep := backoff
					backoff = time.Duration(float64(backoff) * taskInitialBackoffMultiplier)
					wakeup := time.Now().Add(sleep)
					if deadlineok && wakeup.After(deadline) {
						sts := status.DeadlineExceededf(
							nil, "not enough time to complete %T (%v < %v)", task, deadline, wakeup)
						return status.WithSuppressed(sts, stsSliceToErrSlice(failures)...)
					}
					select {
					case <-time.After(sleep):
					case <-ctx.Done():
						return status.WithSuppressed(status.From(ctx.Err()), stsSliceToErrSlice(failures)...)
					}
					continue
				}
			}
		}
		return sts
	}
	if thetasklogger != nil {
		thetasklogger.Printf("Failed to complete task %T after %d tries", task, maxTaskRetries)
	}

	return status.WithSuppressed(failures[0], stsSliceToErrSlice(failures[1:])...)
}

func unwrapTaskStatus(sts status.S) error {
	var cause error = sts
	for {
		if s, ok := cause.(status.S); ok {
			cause = s.Cause()
		} else {
			break
		}
	}
	return cause
}
<commit_msg>be/tasks: always include supressed errors on taskrunner<commit_after>package tasks

import (
	"context"
	"fmt"
	"log"
	"math"
	"os"
	"runtime/trace"
	"time"

	"pixur.org/pixur/be/schema/db"
	"pixur.org/pixur/be/status"
)

const (
	maxTaskRetries = 5
)

var thetasklogger *log.Logger = log.New(os.Stderr, "", log.LstdFlags)

type TaskRunner struct {
	run    func(context.Context, Task) status.S
	thelog *log.Logger
}

func TestTaskRunner(run func(context.Context, Task) status.S) *TaskRunner {
	return &TaskRunner{
		run: run,
	}
}

func (r *TaskRunner) Run(ctx context.Context, task Task) status.S {
	if trace.IsEnabled() {
		var tracetask *trace.Task
		ctx, tracetask = trace.NewTask(ctx, fmt.Sprintf("%T", task))
		defer tracetask.End()
	}
	if r != nil && r.run != nil {
		return r.run(ctx, task)
	}
	return runTask(ctx, task)
}

func stsSliceToErrSlice(ss []status.S) []error {
	es := make([]error, 0, len(ss))
	for _, s := range ss {
		es = append(es, s)
	}
	return es
}

var (
	taskInitialBackoffDelay      = time.Second
	taskInitialBackoffMultiplier = math.Phi
)

func runTask(ctx context.Context, task Task) (stscap status.S) {
	var failures []status.S
	defer func() {
		if stscap == nil && len(failures) != 0 {
			if thetasklogger != nil {
				warnsts := status.Unknown(nil, "Encountered errors while running")
				warnsts = status.WithSuppressed(warnsts, stsSliceToErrSlice(failures)...)

				thetasklogger.Println(warnsts)
			}
		}
	}()

	backoff := taskInitialBackoffDelay
	deadline, deadlineok := ctx.Deadline()
	for i := 0; i < maxTaskRetries; i++ {
		select {
		case <-ctx.Done():
			failures = append(failures, status.From(ctx.Err()))
			break
		default:
		}
		sts := task.Run(ctx)
		if sts == nil {
			return nil
		}
		failures = append(failures, sts)

		if cause := unwrapTaskStatus(sts); cause != nil {
			if retryable, ok := cause.(db.Retryable); ok {
				if retryable.CanRetry() {
					sleep := backoff
					backoff = time.Duration(float64(backoff) * taskInitialBackoffMultiplier)
					wakeup := time.Now().Add(sleep)
					if deadlineok && wakeup.After(deadline) {
						sts := status.DeadlineExceededf(
							nil, "not enough time to complete %T (%v < %v)", task, deadline, wakeup)
						return status.WithSuppressed(sts, stsSliceToErrSlice(failures)...)
					}
					select {
					case <-time.After(sleep):
					case <-ctx.Done():
						return status.WithSuppressed(status.From(ctx.Err()), stsSliceToErrSlice(failures)...)
					}
					continue
				}
			}
		}
		return status.WithSuppressed(sts, stsSliceToErrSlice(failures[:len(failures)-1])...)
	}
	if thetasklogger != nil {
		thetasklogger.Printf("Failed to complete task %T after %d tries", task, maxTaskRetries)
	}
	return status.WithSuppressed(
		failures[len(failures)-1], stsSliceToErrSlice(failures[:len(failures)-1])...)
}

func unwrapTaskStatus(sts status.S) error {
	var cause error = sts
	for {
		if s, ok := cause.(status.S); ok {
			cause = s.Cause()
		} else {
			break
		}
	}
	return cause
}
<|endoftext|>


<commit_before>package main

import (
	"archive/tar"
	"bytes"
	"fmt"
	"io"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"
	docker "github.com/smashwilson/go-dockerclient"
)

// OutputCollector is an io.Writer that accumulates output from a specified stream in an attached
// Docker container and appends it to the appropriate field within a SubmittedJob.
type OutputCollector struct {
	context  *Context
	job      *SubmittedJob
	isStdout bool
}

// DescribeStream returns "stdout" or "stderr" to indicate which stream this collector is consuming.
func (c OutputCollector) DescribeStream() string {
	if c.isStdout {
		return "stdout"
	}
	return "stderr"
}

// Write appends bytes to the selected stream and updates the SubmittedJob.
func (c OutputCollector) Write(p []byte) (int, error) {
	log.WithFields(log.Fields{
		"length": len(p),
		"bytes":  string(p),
		"stream": c.DescribeStream(),
	}).Debug("Received output from a job")

	if c.isStdout {
		c.job.Stdout += string(p)
	} else {
		c.job.Stderr += string(p)
	}

	if err := c.context.UpdateJob(c.job); err != nil {
		return 0, err
	}

	return len(p), nil
}

// Runner is the main entry point for the job runner goroutine.
func Runner(c *Context) {
	var client *docker.Client
	var err error

	if c.DockerTLS {
		client, err = docker.NewTLSClient(c.DockerHost, c.DockerCert, c.DockerKey, c.DockerCACert)
		if err != nil {
			log.WithFields(log.Fields{
				"docker host":    c.DockerHost,
				"docker cert":    c.DockerCert,
				"docker key":     c.DockerKey,
				"docker CA cert": c.DockerCACert,
			}).Fatal("Unable to connect to Docker with TLS.")
			return
		}
	} else {
		client, err = docker.NewClient(c.DockerHost)
		if err != nil {
			log.WithFields(log.Fields{
				"docker host": c.DockerHost,
				"error":       err,
			}).Fatal("Unable to connect to Docker.")
			return
		}
	}

	for {
		select {
		case <-time.After(time.Duration(c.Poll) * time.Millisecond):
			Claim(c, client)
		}
	}
}

// Claim acquires the oldest single pending job and launches a goroutine to execute its command in
// a new container.
func Claim(c *Context, client *docker.Client) {
	job, err := c.ClaimJob()
	if err != nil {
		log.WithFields(log.Fields{"error": err}).Error("Unable to claim a job.")
		return
	}
	if job == nil {
		// Nothing to claim.
		return
	}
	if err := job.Validate(); err != nil {
		fields := log.Fields{
			"jid":     job.JID,
			"account": job.Account,
			"error":   err,
		}

		log.WithFields(fields).Error("Invalid job in queue.")

		job.Status = StatusError
		if err := c.UpdateJob(job); err != nil {
			fields["error"] = err
			log.WithFields(fields).Error("Unable to update job status.")
		}

		return
	}

	go Execute(c, client, job)
}

// Execute launches a container to process the submitted job. It passes any provided stdin data
// to the container and consumes stdout and stderr, updating Mongo as it runs. Once completed, it
// acquires the job's result from its configured source and marks the job as finished.
func Execute(c *Context, client *docker.Client, job *SubmittedJob) {
	defaultFields := log.Fields{
		"jid":     job.JID,
		"account": job.Account,
	}

	// Logging utility messages.
	debug := func(message string) {
		log.WithFields(defaultFields).Debug(message)
	}
	reportErr := func(message string, err error) {
		fs := log.Fields{}
		for k, v := range defaultFields {
			fs[k] = v
		}
		fs["err"] = err
		log.WithFields(fs).Error(message)
	}
	checkErr := func(message string, err error) bool {
		if err == nil {
			debug(fmt.Sprintf("%s: ok", message))
			return false
		}

		reportErr(fmt.Sprintf("%s: ERROR", message), err)
		return true
	}

	// Update the job model in mongo, reporting any errors along the way.
	updateJob := func(message string) bool {
		if err := c.UpdateJob(job); err != nil {
			reportErr(fmt.Sprintf("Unable to update the job's %s.", message), err)
			return false
		}
		return true
	}

	log.WithFields(defaultFields).Info("Launching a job.")

	job.StartedAt = StoreTime(time.Now())
	job.QueueDelay = job.StartedAt.AsTime().Sub(job.CreatedAt.AsTime()).Nanoseconds()
	updateJob("start timestamp")

	container, err := client.CreateContainer(docker.CreateContainerOptions{
		Name: job.ContainerName(),
		Config: &docker.Config{
			Image:     c.Image,
			Cmd:       []string{"/bin/bash", "-c", job.Command},
			OpenStdin: true,
			StdinOnce: true,
		},
	})
	if checkErr("Created the job's container", err) {
		job.Status = StatusError
		updateJob("status")
		return
	}

	// Include container information in this job's logging messages.
	defaultFields["container id"] = container.ID
	defaultFields["container name"] = container.Name

	// Prepare the input and output streams.
	stdin := bytes.NewReader(job.Stdin)
	stdout := OutputCollector{
		context:  c,
		job:      job,
		isStdout: true,
	}
	stderr := OutputCollector{
		context:  c,
		job:      job,
		isStdout: false,
	}

	go func() {
		err = client.AttachToContainer(docker.AttachToContainerOptions{
			Container:    container.ID,
			Stream:       true,
			InputStream:  stdin,
			OutputStream: stdout,
			ErrorStream:  stderr,
			Stdin:        true,
			Stdout:       true,
			Stderr:       true,
		})
		checkErr("Attached to the container", err)
	}()

	// Start the created container.
	err = client.StartContainer(container.ID, &docker.HostConfig{})
	if checkErr("Started the container", err) {
		job.Status = StatusError
		updateJob("status")
		return
	}

	// Measure the container-launch overhead here.
	overhead := time.Now()
	job.OverheadDelay = overhead.Sub(job.StartedAt.AsTime()).Nanoseconds()
	updateJob("overhead delay")

	status, err := client.WaitContainer(container.ID)
	if checkErr("Waited for the container to complete", err) {
		job.Status = StatusError
		updateJob("status")
		return
	}

	job.FinishedAt = StoreTime(time.Now())
	job.Runtime = job.FinishedAt.AsTime().Sub(overhead).Nanoseconds()
	if status == 0 {
		// Successful termination.
		job.Status = StatusDone
	} else {
		// Something went wrong.
		job.Status = StatusError
	}

	// Extract the result from the job.
	if job.ResultSource == "stdout" {
		job.Result = []byte(job.Stdout)
		debug("Acquired job result from stdout: ok")
	} else if strings.HasPrefix(job.ResultSource, "file:") {
		resultPath := job.ResultSource[len("file:"):len(job.ResultSource)]

		var resultBuffer bytes.Buffer
		err = client.CopyFromContainer(docker.CopyFromContainerOptions{
			Container:    container.ID,
			Resource:     resultPath,
			OutputStream: &resultBuffer,
		})
		if checkErr(fmt.Sprintf("Acquired the job's result from the file [%s]", resultPath), err) {
			job.Status = StatusError
		} else {
			// CopyFromContainer returns the file contents as a tarball.
			var content bytes.Buffer
			r := bytes.NewReader(resultBuffer.Bytes())
			tr := tar.NewReader(r)

			for {
				_, err := tr.Next()
				if err == io.EOF {
					break
				}
				if err != nil {
					reportErr("Read tar-encoded content: ERROR", err)
					job.Status = StatusError
					break
				}

				if _, err = io.Copy(&content, tr); err != nil {
					reportErr("Copy decoded content: ERROR", err)
					job.Status = StatusError
					break
				}
			}

			job.Result = content.Bytes()
		}
	}

	err = client.RemoveContainer(docker.RemoveContainerOptions{ID: container.ID})
	checkErr("Removed the container", err)

	updateJob("status and final result")
	log.WithFields(log.Fields{"jid": job.JID}).Info("Job complete.")
}
<commit_msg>Update account usage stats on job completion.<commit_after>package main

import (
	"archive/tar"
	"bytes"
	"fmt"
	"io"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"
	docker "github.com/smashwilson/go-dockerclient"
)

// OutputCollector is an io.Writer that accumulates output from a specified stream in an attached
// Docker container and appends it to the appropriate field within a SubmittedJob.
type OutputCollector struct {
	context  *Context
	job      *SubmittedJob
	isStdout bool
}

// DescribeStream returns "stdout" or "stderr" to indicate which stream this collector is consuming.
func (c OutputCollector) DescribeStream() string {
	if c.isStdout {
		return "stdout"
	}
	return "stderr"
}

// Write appends bytes to the selected stream and updates the SubmittedJob.
func (c OutputCollector) Write(p []byte) (int, error) {
	log.WithFields(log.Fields{
		"length": len(p),
		"bytes":  string(p),
		"stream": c.DescribeStream(),
	}).Debug("Received output from a job")

	if c.isStdout {
		c.job.Stdout += string(p)
	} else {
		c.job.Stderr += string(p)
	}

	if err := c.context.UpdateJob(c.job); err != nil {
		return 0, err
	}

	return len(p), nil
}

// Runner is the main entry point for the job runner goroutine.
func Runner(c *Context) {
	var client *docker.Client
	var err error

	if c.DockerTLS {
		client, err = docker.NewTLSClient(c.DockerHost, c.DockerCert, c.DockerKey, c.DockerCACert)
		if err != nil {
			log.WithFields(log.Fields{
				"docker host":    c.DockerHost,
				"docker cert":    c.DockerCert,
				"docker key":     c.DockerKey,
				"docker CA cert": c.DockerCACert,
			}).Fatal("Unable to connect to Docker with TLS.")
			return
		}
	} else {
		client, err = docker.NewClient(c.DockerHost)
		if err != nil {
			log.WithFields(log.Fields{
				"docker host": c.DockerHost,
				"error":       err,
			}).Fatal("Unable to connect to Docker.")
			return
		}
	}

	for {
		select {
		case <-time.After(time.Duration(c.Poll) * time.Millisecond):
			Claim(c, client)
		}
	}
}

// Claim acquires the oldest single pending job and launches a goroutine to execute its command in
// a new container.
func Claim(c *Context, client *docker.Client) {
	job, err := c.ClaimJob()
	if err != nil {
		log.WithFields(log.Fields{"error": err}).Error("Unable to claim a job.")
		return
	}
	if job == nil {
		// Nothing to claim.
		return
	}
	if err := job.Validate(); err != nil {
		fields := log.Fields{
			"jid":     job.JID,
			"account": job.Account,
			"error":   err,
		}

		log.WithFields(fields).Error("Invalid job in queue.")

		job.Status = StatusError
		if err := c.UpdateJob(job); err != nil {
			fields["error"] = err
			log.WithFields(fields).Error("Unable to update job status.")
		}

		return
	}

	go Execute(c, client, job)
}

// Execute launches a container to process the submitted job. It passes any provided stdin data
// to the container and consumes stdout and stderr, updating Mongo as it runs. Once completed, it
// acquires the job's result from its configured source and marks the job as finished.
func Execute(c *Context, client *docker.Client, job *SubmittedJob) {
	defaultFields := log.Fields{
		"jid":     job.JID,
		"account": job.Account,
	}

	// Logging utility messages.
	debug := func(message string) {
		log.WithFields(defaultFields).Debug(message)
	}
	reportErr := func(message string, err error) {
		fs := log.Fields{}
		for k, v := range defaultFields {
			fs[k] = v
		}
		fs["err"] = err
		log.WithFields(fs).Error(message)
	}
	checkErr := func(message string, err error) bool {
		if err == nil {
			debug(fmt.Sprintf("%s: ok", message))
			return false
		}

		reportErr(fmt.Sprintf("%s: ERROR", message), err)
		return true
	}

	// Update the job model in mongo, reporting any errors along the way.
	updateJob := func(message string) bool {
		if err := c.UpdateJob(job); err != nil {
			reportErr(fmt.Sprintf("Unable to update the job's %s.", message), err)
			return false
		}
		return true
	}

	log.WithFields(defaultFields).Info("Launching a job.")

	job.StartedAt = StoreTime(time.Now())
	job.QueueDelay = job.StartedAt.AsTime().Sub(job.CreatedAt.AsTime()).Nanoseconds()
	updateJob("start timestamp")

	container, err := client.CreateContainer(docker.CreateContainerOptions{
		Name: job.ContainerName(),
		Config: &docker.Config{
			Image:     c.Image,
			Cmd:       []string{"/bin/bash", "-c", job.Command},
			OpenStdin: true,
			StdinOnce: true,
		},
	})
	if checkErr("Created the job's container", err) {
		job.Status = StatusError
		updateJob("status")
		return
	}

	// Include container information in this job's logging messages.
	defaultFields["container id"] = container.ID
	defaultFields["container name"] = container.Name

	// Prepare the input and output streams.
	stdin := bytes.NewReader(job.Stdin)
	stdout := OutputCollector{
		context:  c,
		job:      job,
		isStdout: true,
	}
	stderr := OutputCollector{
		context:  c,
		job:      job,
		isStdout: false,
	}

	go func() {
		err = client.AttachToContainer(docker.AttachToContainerOptions{
			Container:    container.ID,
			Stream:       true,
			InputStream:  stdin,
			OutputStream: stdout,
			ErrorStream:  stderr,
			Stdin:        true,
			Stdout:       true,
			Stderr:       true,
		})
		checkErr("Attached to the container", err)
	}()

	// Start the created container.
	err = client.StartContainer(container.ID, &docker.HostConfig{})
	if checkErr("Started the container", err) {
		job.Status = StatusError
		updateJob("status")
		return
	}

	// Measure the container-launch overhead here.
	overhead := time.Now()
	job.OverheadDelay = overhead.Sub(job.StartedAt.AsTime()).Nanoseconds()
	updateJob("overhead delay")

	status, err := client.WaitContainer(container.ID)
	if checkErr("Waited for the container to complete", err) {
		job.Status = StatusError
		updateJob("status")
		return
	}

	job.FinishedAt = StoreTime(time.Now())
	job.Runtime = job.FinishedAt.AsTime().Sub(overhead).Nanoseconds()
	if status == 0 {
		// Successful termination.
		job.Status = StatusDone
	} else {
		// Something went wrong.
		job.Status = StatusError
	}

	// Extract the result from the job.
	if job.ResultSource == "stdout" {
		job.Result = []byte(job.Stdout)
		debug("Acquired job result from stdout: ok")
	} else if strings.HasPrefix(job.ResultSource, "file:") {
		resultPath := job.ResultSource[len("file:"):len(job.ResultSource)]

		var resultBuffer bytes.Buffer
		err = client.CopyFromContainer(docker.CopyFromContainerOptions{
			Container:    container.ID,
			Resource:     resultPath,
			OutputStream: &resultBuffer,
		})
		if checkErr(fmt.Sprintf("Acquired the job's result from the file [%s]", resultPath), err) {
			job.Status = StatusError
		} else {
			// CopyFromContainer returns the file contents as a tarball.
			var content bytes.Buffer
			r := bytes.NewReader(resultBuffer.Bytes())
			tr := tar.NewReader(r)

			for {
				_, err := tr.Next()
				if err == io.EOF {
					break
				}
				if err != nil {
					reportErr("Read tar-encoded content: ERROR", err)
					job.Status = StatusError
					break
				}

				if _, err = io.Copy(&content, tr); err != nil {
					reportErr("Copy decoded content: ERROR", err)
					job.Status = StatusError
					break
				}
			}

			job.Result = content.Bytes()
		}
	}

	err = client.RemoveContainer(docker.RemoveContainerOptions{ID: container.ID})
	checkErr("Removed the container", err)

	err = c.UpdateAccountUsage(job.Account, job.Runtime)
	if err != nil {
		reportErr("Update account usage: ERROR", err)
		return
	}
	updateJob("status and final result")

	log.WithFields(log.Fields{"jid": job.JID}).Info("Job complete.")
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"
)

var (
	dirFlag = flag.String("d", ".", "项目根目录")
	cmtFlag = flag.Bool("c", true, "包含注释")
)

const (
	LINE_COMMENT_START_FLAG  string = `//`
	BLOCK_COMMENT_START_FLAG string = `/*`
	BLOCK_COMMENT_END_FLAG   string = `*/`

	GO_FILE_SUFFIX string = ".go"
)

type GoFileLineCount struct {
	CodeCount         int
	LineCommentCount  int
	BlockCommentCount int
}

func (this GoFileLineCount) totalLines() int {
	return this.CodeCount + this.LineCommentCount + this.BlockCommentCount
}

func countProject(projectDir string, withComment bool) {
	fileInfo, err := os.Stat(projectDir)
	if err == nil {
		if fileInfo.IsDir() {
			var totalCodeCount int = 0
			var totalLineCommentCount int = 0
			var totalBlockCommentCount int = 0
			filepath.Walk(projectDir, func(path string, info os.FileInfo, err error) (retError error) {
				fileInfo, fileInfoErr := os.Stat(path)
				if fileInfoErr == nil {
					if !fileInfo.IsDir() && strings.HasSuffix(strings.ToLower(path), GO_FILE_SUFFIX) {

						fileLineCount, countErr := countFile(path)
						if countErr != nil {
							fmt.Println(path, ": Count Error")
						} else {
							totalCodeCount += fileLineCount.CodeCount
							totalLineCommentCount += fileLineCount.LineCommentCount
							totalBlockCommentCount += fileLineCount.BlockCommentCount
						}

					}
				} else {
					retError = fileInfoErr
					fmt.Println(path, ": Access Error")
				}
				return
			})
			fmt.Println()
			if withComment {
				fmt.Println("行注释:", totalLineCommentCount)
				fmt.Println("块注释:", totalBlockCommentCount)
			}
			fmt.Println("代码:", totalCodeCount)
			if withComment {
				totalFileLines := totalCodeCount + totalLineCommentCount + totalBlockCommentCount
				totalCommentLines := totalBlockCommentCount + totalLineCommentCount
				fmt.Println("总计:", totalFileLines)
				var commentPercent = float64(totalCommentLines) / float64(totalFileLines) * 100
				var codePercent = float64(totalCodeCount) / float64(totalFileLines) * 100
				fmt.Printf("注释比例:%.2f%% 代码比例:%.2f%%", commentPercent, codePercent)
				fmt.Println()
			}
		} else {
			fmt.Println("Can not access directory", projectDir)
		}
	} else {
		fmt.Println(err)
	}
}

func countFile(path string) (fileLineCount GoFileLineCount, err error) {
	file, fileError := os.Open(path)
	if fileError == nil {
		fileLineCount = GoFileLineCount{}
		bReader := bufio.NewReader(file)
		var isBlockComment bool = false
		for {
			lineData, readErr := bReader.ReadString('\n')
			lineData = strings.TrimSpace(lineData)

			if strings.HasPrefix(lineData, LINE_COMMENT_START_FLAG) {
				fileLineCount.LineCommentCount++
			} else if strings.HasPrefix(lineData, BLOCK_COMMENT_START_FLAG) {
				isBlockComment = true
				fileLineCount.BlockCommentCount++
			} else if strings.HasPrefix(lineData, BLOCK_COMMENT_END_FLAG) {
				isBlockComment = false
				fileLineCount.BlockCommentCount++
			} else {
				if isBlockComment {
					fileLineCount.BlockCommentCount++
				} else {
					fileLineCount.CodeCount++
				}
			}
			if readErr == io.EOF {
				break
			}
		}
		file.Close()
		fmt.Println(path, fileLineCount.totalLines())
	} else {
		err = fileError
	}

	return
}

func main() {
	flag.Parse()

	var dir = *dirFlag
	var withComment = *cmtFlag
	countProject(dir, withComment)
}
<commit_msg>为gocount添加注释<commit_after>package main

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"
)

//定义gocount命令的参数
// -d 参数 指定项目的根目录，默认为命令启动时的目录
// -c 参数 指定是否需要计算注释的行数，默认为计算
var (
	dirFlag = flag.String("d", ".", "项目根目录")
	cmtFlag = flag.Bool("c", true, "包含注释")
)

//定义go语言的注释标记和文件后缀名
const (
	LINE_COMMENT_START_FLAG  string = `//`
	BLOCK_COMMENT_START_FLAG string = `/*`
	BLOCK_COMMENT_END_FLAG   string = `*/`

	GO_FILE_SUFFIX string = ".go"
)

//为每个文件定义一个GoFileLineCount结构体
type GoFileLineCount struct {
	CodeCount         int //代码行数
	LineCommentCount  int //行注释行数
	BlockCommentCount int //块注释行数
}

//获取该文件的内容总行数
func (this GoFileLineCount) totalLines() int {
	return this.CodeCount + this.LineCommentCount + this.BlockCommentCount
}

func countProject(projectDir string, withComment bool) {
	fileInfo, err := os.Stat(projectDir)
	if err == nil {
		if fileInfo.IsDir() {
			var totalCodeCount int = 0
			var totalLineCommentCount int = 0
			var totalBlockCommentCount int = 0

			filepath.Walk(projectDir, func(path string, info os.FileInfo, err error) (retError error) {
				fileInfo, fileInfoErr := os.Stat(path)
				if fileInfoErr == nil {
					if !fileInfo.IsDir() && strings.HasSuffix(strings.ToLower(path), GO_FILE_SUFFIX) {
						fileLineCount, countErr := countFile(path)
						if countErr != nil {
							fmt.Println(path, ": 计算错误")
						} else {
							totalCodeCount += fileLineCount.CodeCount
							totalLineCommentCount += fileLineCount.LineCommentCount
							totalBlockCommentCount += fileLineCount.BlockCommentCount
						}
					}
				} else {
					retError = fileInfoErr
					fmt.Println(path, ": 无法访问")
				}
				return
			})

			fmt.Println()
			if withComment {
				fmt.Println("行注释:", totalLineCommentCount)
				fmt.Println("块注释:", totalBlockCommentCount)
			}
			fmt.Println("代码:", totalCodeCount)
			if withComment {
				totalFileLines := totalCodeCount + totalLineCommentCount + totalBlockCommentCount
				totalCommentLines := totalBlockCommentCount + totalLineCommentCount
				fmt.Println("总计:", totalFileLines)
				var commentPercent = float64(totalCommentLines) / float64(totalFileLines) * 100
				var codePercent = float64(totalCodeCount) / float64(totalFileLines) * 100
				fmt.Printf("注释比例:%.2f%% 代码比例:%.2f%%", commentPercent, codePercent)
				fmt.Println()
			}
		} else {
			fmt.Println("无法访问目录:", projectDir)
		}
	} else {
		fmt.Println(err)
	}
}

//获取每个文件的内容行数
func countFile(path string) (fileLineCount GoFileLineCount, err error) {
	file, fileError := os.Open(path)
	if fileError == nil {
		fileLineCount = GoFileLineCount{}
		bReader := bufio.NewReader(file)
		var isBlockComment bool = false
		for {
			lineData, readErr := bReader.ReadString('\n')
			lineData = strings.TrimSpace(lineData)
			//分别计算文件中代码，块注释和行注释的总行数
			if strings.HasPrefix(lineData, LINE_COMMENT_START_FLAG) {
				fileLineCount.LineCommentCount++
			} else if strings.HasPrefix(lineData, BLOCK_COMMENT_START_FLAG) {
				isBlockComment = true
				fileLineCount.BlockCommentCount++
			} else if strings.HasPrefix(lineData, BLOCK_COMMENT_END_FLAG) {
				isBlockComment = false
				fileLineCount.BlockCommentCount++
			} else {
				if isBlockComment {
					fileLineCount.BlockCommentCount++
				} else {
					fileLineCount.CodeCount++
				}
			}
			if readErr == io.EOF {
				break
			}
		}
		file.Close()
		fmt.Println(path, fileLineCount.totalLines())
	} else {
		err = fileError
	}

	return
}

func main() {
	flag.Parse()

	var dir = *dirFlag
	var withComment = *cmtFlag
	countProject(dir, withComment)
}
<|endoftext|>


<commit_before>package scenario

import (
	"crypto/md5"
	"fmt"
	"io"
	"io/ioutil"
	"sync"

	"github.com/catatsuy/isucon6-final/bench/action"
	"github.com/catatsuy/isucon6-final/bench/fails"
	"github.com/catatsuy/isucon6-final/bench/session"
)

type Asset struct {
	Path string
	MD5  string
}

func loadAssets(s *session.Session, checkHash bool) bool {
	assets := []Asset{
		Asset{Path: "/css/rc-color-picker.css", MD5: "78055c5c02a2dd66f6207fa19f7ca928"},
		Asset{Path: "/css/sanitize.css", MD5: "7375990d0f1f7d436a952314e3ac7fd0"},
		Asset{Path: "/bundle.js", MD5: "68a787c981bf44f173d0315f3838564f"},
	}
	var wg sync.WaitGroup

	OK := true
	for _, asset := range assets {
		wg.Add(1)
		go func(asset Asset) {
			defer wg.Done()

			ok := action.Get(s, asset.Path, action.OK(func(body io.Reader, l *fails.Logger) bool {
				if checkHash {
					content, err := ioutil.ReadAll(body)
					if err != nil {
						l.Add("ファイルの読み込みに失敗しました", err)
						return false
					}
					actual := fmt.Sprintf("%x", md5.Sum(content))
					if actual != asset.MD5 {
						l.Add("ファイルの内容が正しくありません",
							fmt.Errorf("expected %s, actual %s, content: %s", asset.MD5, actual, string(content[:20])))
						return false
					}
				}
				return true
			}))
			if !ok {
				OK = false
			}
		}(asset)
	}
	wg.Wait()
	return OK
}
<commit_msg>Change hash to that of production bundle.js<commit_after>package scenario

import (
	"crypto/md5"
	"fmt"
	"io"
	"io/ioutil"
	"sync"

	"github.com/catatsuy/isucon6-final/bench/action"
	"github.com/catatsuy/isucon6-final/bench/fails"
	"github.com/catatsuy/isucon6-final/bench/session"
)

type Asset struct {
	Path string
	MD5  string
}

func loadAssets(s *session.Session, checkHash bool) bool {
	assets := []Asset{
		Asset{Path: "/css/rc-color-picker.css", MD5: "78055c5c02a2dd66f6207fa19f7ca928"},
		Asset{Path: "/css/sanitize.css", MD5: "7375990d0f1f7d436a952314e3ac7fd0"},
		Asset{Path: "/bundle.js", MD5: "3d66e4a976da7c915b82188865dc994b"},
	}
	var wg sync.WaitGroup

	OK := true
	for _, asset := range assets {
		wg.Add(1)
		go func(asset Asset) {
			defer wg.Done()

			ok := action.Get(s, asset.Path, action.OK(func(body io.Reader, l *fails.Logger) bool {
				if checkHash {
					content, err := ioutil.ReadAll(body)
					if err != nil {
						l.Add("ファイルの読み込みに失敗しました", err)
						return false
					}
					actual := fmt.Sprintf("%x", md5.Sum(content))
					if actual != asset.MD5 {
						l.Add("ファイルの内容が正しくありません",
							fmt.Errorf("expected %s, actual %s, content: %s", asset.MD5, actual, string(content[:20])))
						return false
					}
				}
				return true
			}))
			if !ok {
				OK = false
			}
		}(asset)
	}
	wg.Wait()
	return OK
}
<|endoftext|>


<commit_before>package goat

type IoWorker interface {
	Lookup(MapDb, Request)
	UpdateLookup(MapDb, Request)
}
type Worker struct {
}

// looks up where in the dbStor the requested data is stored
func (worker Worker) Lookup(db MapDb, id string) []*interface{} {
	return db.MapLookup[id]

}

// reqplaces the pointer in the lookup with a new pointer to the new data
func (worker Worker) UpdateLookup(db MapDb, id string, data interface{}) {
	db.MapLookup[id][0] = &data
}

func (worker Worker) AppendLookup(db MapDb, id string, data interface{}) {
	db.MapLookup[id] = append(db.MapLookup[id], &data)
}
<commit_msg>IoWorker is no longer an interface, and is now a struct which holds a pointer to a MapDb<commit_after>package goat

type IoWorker struct {
	db *MapDb
}

// looks up where in the dbStor the requested data is stored
func (worker IoWorker) Lookup(id string) []*interface{} {
	return worker.db.MapLookup[id]

}

// reqplaces the pointer in the lookup with a new pointer to the new data
func (worker IoWorker) UpdateLookup(id string, data interface{}) {
	worker.db.MapLookup[id][0] = &data
}

// append a pointer on the end of the lookup
func (worker IoWorker) AppendLookup(id string, data interface{}) {
	worker.db.MapLookup[id] = append(worker.db.MapLookup[id], &data)
}

func (worker IoWorker) RemoveLookup(id string) {
	worker.db.MapLookup[id] = nil
}
<|endoftext|>


<commit_before>package sync

import (
	"database/sql"
	"errors"
	"fmt"
	_ "github.com/svdberg/syncmysport-runkeeper/Godeps/_workspace/src/github.com/go-sql-driver/mysql"
	"log"
	"strings"
	"time"
)

//should come from config (file) somewhere...
const default_connection_string = "root:root123@/syncmysport?charset=utf8,parseTime=true"

type DbSync struct {
	ConnectionString string
}

func CreateSyncDbRepo(dbString string) *DbSync {
	if dbString != "" {
		dbString = MakeDbStringHerokuCompliant(dbString)
		appendedConnectionString := fmt.Sprintf("%s?parseTime=true", dbString)
		log.Printf("Connection string was: %s, now is %s", dbString, appendedConnectionString)
		return &DbSync{appendedConnectionString}
	} else {
		return &DbSync{default_connection_string}
	}
}

func (db DbSync) UpdateSyncTask(sync SyncTask) (int, error) {
	if sync.Uid == -1 {
		return 0, errors.New("SyncTask was never stored before, use StoreSyncTask")
	}
	dbCon, _ := sql.Open("mysql", db.ConnectionString)
	defer dbCon.Close()

	stmtOut, err := dbCon.Prepare("UPDATE sync SET rk_key=?, stv_key=?, last_succesfull_retrieve=? WHERE uid = ?")
	if err != nil {
		panic(err.Error()) // proper error handling instead of panic in your app
	}
	defer stmtOut.Close()
	res, err := stmtOut.Exec(sync.RunkeeperToken, sync.StravaToken, createStringOutOfUnixTime(sync.LastSeenTimestamp), sync.Uid)
	if err != nil {
		log.Fatal(err)
	}

	i, err := res.RowsAffected()
	if err != nil {
		log.Fatal(err)
	}
	return int(i), nil
}

/*
* Returns 1) Created Id, 2) Rows changed/added, 3)synctask, 4) error
 */
func (db DbSync) StoreSyncTask(sync SyncTask) (int64, int64, SyncTask, error) {
	dbCon, _ := sql.Open("mysql", db.ConnectionString)
	defer dbCon.Close()

	stmtOut, err := dbCon.Prepare("INSERT INTO sync(rk_key, stv_key, last_succesfull_retrieve) VALUES(?,?,?)")
	if err != nil {
		panic(err.Error()) // proper error handling instead of panic in your app
	}
	defer stmtOut.Close()
	res, err := stmtOut.Exec(sync.RunkeeperToken, sync.StravaToken, createStringOutOfUnixTime(sync.LastSeenTimestamp))
	if err != nil {
		log.Fatal(err)
	}
	lastId, err := res.LastInsertId()
	if err != nil {
		log.Fatal(err)
	}
	rowCnt, err := res.RowsAffected()
	if err != nil {
		log.Fatal(err)
	}
	sync.Uid = lastId
	return lastId, rowCnt, sync, nil
}

func (db DbSync) RetrieveAllSyncTasks() ([]SyncTask, error) {
	dbCon, _ := sql.Open("mysql", db.ConnectionString)
	stmtOut, err := dbCon.Prepare("SELECT * FROM sync")
	if err != nil {
		panic(err.Error()) // proper error handling instead of panic in your app
	}
	defer stmtOut.Close()

	rows, err := stmtOut.Query()
	defer rows.Close()

	result := make([]SyncTask, 0)
	for rows.Next() {
		var rkToken string
		var stvToken string
		var uid int64
		var lastSeenTime string

		rows.Scan(&uid, &rkToken, &stvToken, &lastSeenTime)
		unixTime, err := createUnixTimeOutOfString(lastSeenTime)
		if err != nil {
			panic(err.Error()) // proper error handling instead of panic in your app
		}

		sync := CreateSyncTask(rkToken, stvToken, unixTime)
		sync.Uid = uid
		result = append(result, *sync)
	}
	return result, nil
}

func (db DbSync) RetrieveSyncTaskByToken(token string) (*SyncTask, error) {
	dbCon, _ := sql.Open("mysql", db.ConnectionString)
	stmtOut, err := dbCon.Prepare("SELECT * FROM sync WHERE rk_key = ? OR stv_key = ?")
	if err != nil {
		panic(err.Error()) // proper error handling instead of panic in your app
	}
	defer stmtOut.Close()

	rows, err := stmtOut.Query(token, token)
	defer rows.Close()

	for rows.Next() {
		var uid int64
		var rkToken string
		var stvToken string
		var lastSeen string
		err = rows.Scan(&uid, &rkToken, &stvToken, &lastSeen)
		if err != nil {
			panic(err.Error()) // proper error handling instead of panic in your app
		}
		unixTime, err := createUnixTimeOutOfString(lastSeen)
		if err != nil {
			panic(err.Error()) // proper error handling instead of panic in your app
		}
		task := CreateSyncTask(rkToken, stvToken, unixTime)
		task.Uid = uid
		return task, nil
	}
	return nil, nil
}

func MakeDbStringHerokuCompliant(dbString string) string {
	dbStringWithoutProtocol := strings.Replace(dbString, "mysql://", "", 1)
	parts := strings.Split(dbStringWithoutProtocol, "@")
	userAndPassword := strings.Split(parts[0], ":")

	addr := strings.Split(parts[1], "/")[0]
	dbName := strings.Split(strings.Split(parts[1], "/")[1], "?")[0]

	resultString := fmt.Sprintf("%s:%s@tcp(%s:3306)/%s", userAndPassword[0], userAndPassword[1], addr, dbName)
	return resultString
}

func createUnixTimeOutOfString(lastSeen string) (int, error) {
	timestamp, err := time.Parse("2006-01-02 15:04:05", lastSeen)

	if err != nil {
		return 0, err
	}
	return int(timestamp.Unix()), nil
}

func createStringOutOfUnixTime(t int) string {
	return time.Unix(int64(t), 0).Format("2006-01-02 15:04:05")
}
<commit_msg>Time parsing ??????<commit_after>package sync

import (
	"database/sql"
	"errors"
	"fmt"
	_ "github.com/svdberg/syncmysport-runkeeper/Godeps/_workspace/src/github.com/go-sql-driver/mysql"
	"log"
	"strings"
	"time"
)

//should come from config (file) somewhere...
const default_connection_string = "root:root123@/syncmysport?charset=utf8,parseTime=true"

type DbSync struct {
	ConnectionString string
}

func CreateSyncDbRepo(dbString string) *DbSync {
	if dbString != "" {
		dbString = MakeDbStringHerokuCompliant(dbString)
		//appendedConnectionString := fmt.Sprintf("%s?parseTime=true", dbString)
		appendedConnectionString := fmt.Sprintf("%s", dbString)
		log.Printf("Connection string was: %s, now is %s", dbString, appendedConnectionString)
		return &DbSync{appendedConnectionString}
	} else {
		return &DbSync{default_connection_string}
	}
}

func (db DbSync) UpdateSyncTask(sync SyncTask) (int, error) {
	if sync.Uid == -1 {
		return 0, errors.New("SyncTask was never stored before, use StoreSyncTask")
	}
	dbCon, _ := sql.Open("mysql", db.ConnectionString)
	defer dbCon.Close()

	stmtOut, err := dbCon.Prepare("UPDATE sync SET rk_key=?, stv_key=?, last_succesfull_retrieve=? WHERE uid = ?")
	if err != nil {
		panic(err.Error()) // proper error handling instead of panic in your app
	}
	defer stmtOut.Close()
	res, err := stmtOut.Exec(sync.RunkeeperToken, sync.StravaToken, createStringOutOfUnixTime(sync.LastSeenTimestamp), sync.Uid)
	if err != nil {
		log.Fatal(err)
	}

	i, err := res.RowsAffected()
	if err != nil {
		log.Fatal(err)
	}
	return int(i), nil
}

/*
* Returns 1) Created Id, 2) Rows changed/added, 3)synctask, 4) error
 */
func (db DbSync) StoreSyncTask(sync SyncTask) (int64, int64, SyncTask, error) {
	dbCon, _ := sql.Open("mysql", db.ConnectionString)
	defer dbCon.Close()

	stmtOut, err := dbCon.Prepare("INSERT INTO sync(rk_key, stv_key, last_succesfull_retrieve) VALUES(?,?,?)")
	if err != nil {
		panic(err.Error()) // proper error handling instead of panic in your app
	}
	defer stmtOut.Close()
	res, err := stmtOut.Exec(sync.RunkeeperToken, sync.StravaToken, createStringOutOfUnixTime(sync.LastSeenTimestamp))
	if err != nil {
		log.Fatal(err)
	}
	lastId, err := res.LastInsertId()
	if err != nil {
		log.Fatal(err)
	}
	rowCnt, err := res.RowsAffected()
	if err != nil {
		log.Fatal(err)
	}
	sync.Uid = lastId
	return lastId, rowCnt, sync, nil
}

func (db DbSync) RetrieveAllSyncTasks() ([]SyncTask, error) {
	dbCon, _ := sql.Open("mysql", db.ConnectionString)
	stmtOut, err := dbCon.Prepare("SELECT * FROM sync")
	if err != nil {
		panic(err.Error()) // proper error handling instead of panic in your app
	}
	defer stmtOut.Close()

	rows, err := stmtOut.Query()
	defer rows.Close()

	result := make([]SyncTask, 0)
	for rows.Next() {
		var rkToken string
		var stvToken string
		var uid int64
		var lastSeenTime string

		rows.Scan(&uid, &rkToken, &stvToken, &lastSeenTime)
		unixTime, err := createUnixTimeOutOfString(lastSeenTime)
		if err != nil {
			panic(err.Error()) // proper error handling instead of panic in your app
		}

		sync := CreateSyncTask(rkToken, stvToken, unixTime)
		sync.Uid = uid
		result = append(result, *sync)
	}
	return result, nil
}

func (db DbSync) RetrieveSyncTaskByToken(token string) (*SyncTask, error) {
	dbCon, _ := sql.Open("mysql", db.ConnectionString)
	stmtOut, err := dbCon.Prepare("SELECT * FROM sync WHERE rk_key = ? OR stv_key = ?")
	if err != nil {
		panic(err.Error()) // proper error handling instead of panic in your app
	}
	defer stmtOut.Close()

	rows, err := stmtOut.Query(token, token)
	defer rows.Close()

	for rows.Next() {
		var uid int64
		var rkToken string
		var stvToken string
		var lastSeen string
		err = rows.Scan(&uid, &rkToken, &stvToken, &lastSeen)
		if err != nil {
			panic(err.Error()) // proper error handling instead of panic in your app
		}
		unixTime, err := createUnixTimeOutOfString(lastSeen)
		if err != nil {
			panic(err.Error()) // proper error handling instead of panic in your app
		}
		task := CreateSyncTask(rkToken, stvToken, unixTime)
		task.Uid = uid
		return task, nil
	}
	return nil, nil
}

func MakeDbStringHerokuCompliant(dbString string) string {
	dbStringWithoutProtocol := strings.Replace(dbString, "mysql://", "", 1)
	parts := strings.Split(dbStringWithoutProtocol, "@")
	userAndPassword := strings.Split(parts[0], ":")

	addr := strings.Split(parts[1], "/")[0]
	dbName := strings.Split(strings.Split(parts[1], "/")[1], "?")[0]

	resultString := fmt.Sprintf("%s:%s@tcp(%s:3306)/%s", userAndPassword[0], userAndPassword[1], addr, dbName)
	return resultString
}

func createUnixTimeOutOfString(lastSeen string) (int, error) {
	timestamp, err := time.Parse("2006-01-02 15:04:05", lastSeen)

	if err != nil {
		return 0, err
	}
	return int(timestamp.Unix()), nil
}

func createStringOutOfUnixTime(t int) string {
	return time.Unix(int64(t), 0).Format("2006-01-02 15:04:05")
}
<|endoftext|>


<commit_before>// Copyright 2010 The GoGo Authors. All rights reserved.
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

// This file holds the basic scanning routines that separate a source file
// into the various tokens

package main

import "./libgogo/_obj/libgogo"

//
// Function fetching a new character from the file that is compiled
// Adds counters for line and columns
//
func GetCharWrapped() byte {
    var singleChar byte;
    if tok.nextChar == 0 {
        singleChar = libgogo.GetChar(fileInfo[curFileIndex].fd);
        if (singleChar == 10) {
            fileInfo[curFileIndex].charCounter = 1;
            fileInfo[curFileIndex].lineCounter = fileInfo[curFileIndex].lineCounter + 1;
        } else {
            fileInfo[curFileIndex].charCounter = fileInfo[curFileIndex].charCounter + 1;
        }
    } else {
        singleChar = tok.nextChar;
        tok.nextChar = 0;
    }

    return singleChar;
}

func GetEscapedCharWrapped() byte {
    var singleChar byte;
    var nextChar byte;
    var returnChar byte;
    singleChar = GetCharWrapped();
    if singleChar == '\\' { //Escaped character
        nextChar = GetCharWrapped();
        if nextChar == 'n' { //n => \n
            returnChar = '\n';
        } else {
            if nextChar == '\n' { //Line break in string
                returnChar = GetCharWrapped(); //Return character after line break
            } else {
                if nextChar == '\\' { //\ => \ (single \)
                    returnChar = '\\';
                } else {
                     if nextChar == 'b' { //b => \b
                         returnChar = '\b';
                    } else {
                        ScanErrorString("Unknown escape sequence.");
                    }
                }
            }
        }
    } else { //Normal character
        returnChar = singleChar;
    }
    return returnChar;
}

func GetNextTokenRaw() {
    var singleChar byte; // Byte holding the last read value
    // Flag indicating whether we are in a comment.
    // 0 for no comment
    // 1 for a single line comment
    // 2 for a multi line comment
    var inComment uint64;
    var done uint64; // Flag indicating whether a cycle (Token) is finsihed
    var spaceDone uint64; // Flag indicating whether an abolishment cycle is finished
    var numBuf string;

    // Initialize variables
    done = 0;
    spaceDone = 0;
    inComment = 0;
    tok.strValue = "";

    // If the previous cycle had to read the next char (and stored it), it is
    // now used as first read
    singleChar = GetCharWrapped();

    // check if it is a valid read, or an EOF
    if singleChar == 0 {
        tok.id = TOKEN_EOS;
        done = 1;
        spaceDone = 1;
    }

    //
    // Cleaning Tasks
    // The next part strips out spaces, newlines, tabs, and comments
    // Comments can either be single line with double slashes (//) or multiline
    // using C++ syntax /* */
    //
    for ;spaceDone != 1; {

        // check whether a comment is starting
        if singleChar == '/' {
            // if we are in a comment skip the rest, get the next char otherwise
            if inComment == 0 {
                singleChar = GetCharWrapped();
                if singleChar == '/' {
                    // we are in a single line comment (until newline is found)
                    inComment = 1;
                } else {
                    if singleChar == '*' {
                        // we are in a multiline comment (until ending is found)
                        inComment = 2;
                    } else {
                        tok.nextChar = singleChar;
                        singleChar = '/';
                    }
                }
            }
        }

        if inComment == 2 {
            // check whether a multi-line comment is ending
            if singleChar == '*' {
                singleChar = GetCharWrapped();
                if singleChar == '/' {
                    if inComment == 2 {
                        inComment = 0;
                        singleChar = GetCharWrapped();
                    }
                }
            }
        }

        // if character is a newline:
        //  *) if in a singleline comment, exit the comment
        //  *) skip otherwise
        if singleChar == 10 {
            if inComment == 1 {
                inComment = 0;
            }
        }

        // handle everything that is not a space,tab,newline
        if (singleChar != ' ') && (singleChar != 9) && (singleChar != 10) {
            // if not in a comment we have our current valid char
            if inComment == 0 {
                spaceDone = 1;
            }

            // check if GetChar() returned EOF while skipping
            if singleChar == 0 {
                tok.id = TOKEN_EOS;
                spaceDone = 1;
                done = 1;
            }
        }

        // if we are not done until now, get a new character and start another abolishing cycle
        if spaceDone == 0 {
            singleChar=GetCharWrapped();
        }
    }

    //
    // Actual scanning part starts here
    //

    // Catch identifiers
    // identifier = letter { letter | digit }.
    if (done != 1) && ((singleChar >= 'A') && (singleChar <= 'Z')) || ((singleChar >= 'a') && (singleChar <= 'z')) || (singleChar == '_') { // check for letter or _
        tok.id = TOKEN_IDENTIFIER;
        // preceding characters may be letter,_, or a number
        for ; ((singleChar >= 'A') && (singleChar <= 'Z')) || ((singleChar >= 'a') && (singleChar <= 'z')) || (singleChar == '_') || ((singleChar >= '0') && (singleChar <= '9')); singleChar = GetCharWrapped() {
            tmp_TokAppendStr(singleChar);
        }
        // save the last read character for the next GetNextToken() cycle
        tok.nextChar = singleChar;
        done = 1;
    }

    // string "..."
    if (done != 1) && (singleChar == '"') {
        tok.id = TOKEN_STRING;
        for singleChar = GetCharWrapped(); singleChar != '"';singleChar = GetEscapedCharWrapped() {
            tmp_TokAppendStr(singleChar);
        }
        if singleChar != '"' {
            ScanErrorString("String not closing.");
        }
        done = 1;
    }

    // Single Quoted Character
    if (done != 1) && singleChar == 39 {
        singleChar = GetEscapedCharWrapped();
        if singleChar != 39 {
            tok.id = TOKEN_INTEGER;
            tok.intValue = libgogo.ToIntFromByte(singleChar);
        } else {
            ScanErrorString("Unknown character.");
        }
        singleChar = GetCharWrapped();
        if singleChar != 39 {
            ScanErrorString("Only single characters allowed. Use corresponding integer for special characters.");
        }
        done = 1;
    }

    // left brace (
    if (done != 1) && singleChar == '(' {
        tok.id = TOKEN_LBRAC;
        done = 1;
    }

    // right brace )
    if (done != 1) && singleChar == ')' {
        tok.id = TOKEN_RBRAC;
        done = 1;
    }

    // left square bracket [
    if (done != 1) && singleChar == '[' {
        tok.id = TOKEN_LSBRAC;
        done = 1;
    }

    // right square bracket ]
    if (done != 1) && singleChar == ']' {
        tok.id = TOKEN_RSBRAC;
        done = 1;
    }

    // integer
    if (done != 1) && (singleChar > 47) && (singleChar < 58) {
        numBuf = "";

        for ; (singleChar > 47) && (singleChar < 58) ; singleChar = GetCharWrapped() {
            libgogo.CharAppend(&numBuf, singleChar);
        }

        tok.nextChar = singleChar;
        tok.id = TOKEN_INTEGER;
        tok.intValue = libgogo.StringToInt(numBuf);

        done = 1;
    }

    // Left curly bracket '{'
    if (done != 1) && (singleChar == '{') {
        tok.id = TOKEN_LCBRAC;
        done = 1;
    }

    // Right curly bracket '}'
    if (done != 1) && (singleChar == '}') {
        tok.id = TOKEN_RCBRAC;
        done = 1;
    }

    // Point '.'
    if (done != 1) && (singleChar == '.') {
        tok.id = TOKEN_PT;
        done = 1;
    }

    // Not ('!') or Not Equal ('!=')
    if (done != 1) && (singleChar == '!') {
        singleChar = GetCharWrapped();
        if singleChar == '=' {
            tok.id = TOKEN_NOTEQUAL;
        } else {
            tok.id = TOKEN_NOT;
            tok.nextChar = singleChar;
        }
        done = 1;
    }

    // Semicolon ';'
    if (done != 1) && (singleChar == ';') {
        tok.id = TOKEN_SEMICOLON;
        done = 1;
    }

    // Colon ','
    if (done != 1) && (singleChar == ',') {
        tok.id = TOKEN_COLON;
        done = 1;
    }

    // Assignment '=' or Equals comparison '=='
    if (done != 1) && (singleChar == '=') {
        singleChar = GetCharWrapped();
        if singleChar == '=' {
            tok.id = TOKEN_EQUALS;
        } else {
            tok.id = TOKEN_ASSIGN;
            tok.nextChar = singleChar;
        }
        done = 1;
    }

    // AND Relation '&&'
    if (done != 1) && (singleChar == '&') {
        singleChar = GetCharWrapped();
        if singleChar == '&' {
            tok.id = TOKEN_REL_AND;
        } else {
            tok.id = TOKEN_OP_ADR;
            tok.nextChar = singleChar;
        }
        done = 1;
    }

    // OR Relation '||'
    if (done != 1) && (singleChar == '|') {
        singleChar = GetCharWrapped();
        if singleChar == '|' {
            tok.id = TOKEN_REL_OR;
        } else {
            ScanErrorString("No binary OR (|) supported. Only ||.");
        }
        done = 1;
    }

    // Greater and Greater-Than relation
    if (done != 1) && (singleChar == '>') {
        singleChar = GetCharWrapped();
        if singleChar == '=' {
            tok.id = TOKEN_REL_GTOE;
        } else {
            tok.id = TOKEN_REL_GT;
            tok.nextChar = singleChar;
        }
        done = 1;
    }

    // Less and Less-Than relation
    if (done != 1) && (singleChar == '<') {
        singleChar = GetCharWrapped();
        if singleChar == '=' {
            tok.id = TOKEN_REL_LTOE;
        } else {
            tok.id = TOKEN_REL_LT;
            tok.nextChar = singleChar;
        }
        done = 1;
    }

    if (done != 1) && (singleChar == '+') {
        tok.id = TOKEN_ARITH_PLUS;
        done = 1;
    }

    if (done != 1) && (singleChar == '-') {
        tok.id = TOKEN_ARITH_MINUS;
        done = 1;
    }

    if (done != 1) && (singleChar == '*') {
        tok.id = TOKEN_ARITH_MUL;
        done = 1;
    }

    if (done != 1) && (singleChar == '/') {
        tok.id = TOKEN_ARITH_DIV;
        done = 1;
    }

    if (done != 1) {
        ScanErrorChar(singleChar);
    }
}

//
// GetNextToken should be called by the parser. It bascially fetches the next
// token by calling GetNextTokenRaw() and filters the identifiers for known
// keywords.
//
func GetNextToken() {
    var cmpValue uint64;
    GetNextTokenRaw();

    // Convert identifier to keyworded tokens
    if tok.id == TOKEN_IDENTIFIER {
        cmpValue = libgogo.StringCompare("if",tok.strValue);
        if cmpValue == 0 {
            tok.id = TOKEN_IF;
        }
        cmpValue = libgogo.StringCompare("else",tok.strValue);
        if cmpValue == 0 {
            tok.id = TOKEN_ELSE;
        }
        cmpValue = libgogo.StringCompare("for",tok.strValue);
        if cmpValue == 0 {
            tok.id = TOKEN_FOR;
        }
        cmpValue = libgogo.StringCompare("type",tok.strValue);
        if cmpValue == 0 {
            tok.id = TOKEN_TYPE;
        }
        cmpValue = libgogo.StringCompare("const",tok.strValue);
        if cmpValue == 0 {
            tok.id = TOKEN_CONST;
        }
        cmpValue = libgogo.StringCompare("var",tok.strValue);
        if cmpValue == 0 {
            tok.id = TOKEN_VAR;
        }
        cmpValue = libgogo.StringCompare("struct", tok.strValue);
        if cmpValue == 0 {
            tok.id = TOKEN_STRUCT;
        }
        cmpValue = libgogo.StringCompare("return", tok.strValue);
        if cmpValue == 0 {
            tok.id = TOKEN_RETURN;
        }
        cmpValue = libgogo.StringCompare("func", tok.strValue);
        if cmpValue == 0 {
            tok.id = TOKEN_FUNC;
        }
        cmpValue = libgogo.StringCompare("import", tok.strValue);
        if cmpValue == 0 {
            tok.id = TOKEN_IMPORT;
        }
        cmpValue = libgogo.StringCompare("package", tok.strValue);
        if cmpValue == 0 {
            tok.id = TOKEN_PACKAGE;
        }
        cmpValue = libgogo.StringCompare("break", tok.strValue);
        if cmpValue == 0 {
            tok.id = TOKEN_BREAK;
        }
        cmpValue = libgogo.StringCompare("continue", tok.strValue);
        if cmpValue == 0 {
            tok.id = TOKEN_CONTINUE;
        }
    }

    tok.nextToken = 0;
}

//
// Debugging and temporary functions
//

func debugToken(tok *Token) {
    libgogo.PrintString("---------------------\n");
    libgogo.PrintString("Token Id: ");
    libgogo.PrintNumber(tok.id);
    libgogo.PrintString("\n");
    if (tok.id == TOKEN_IDENTIFIER) || (tok.id == TOKEN_STRING) {
        libgogo.PrintString("Stored string: ");
        libgogo.PrintString(tok.strValue);
        libgogo.PrintString("\n");
    }
    if tok.id == TOKEN_INTEGER {
        libgogo.PrintString("Stored integer: ");
        libgogo.PrintNumber(tok.intValue);
        libgogo.PrintString("\n");
    }
}

func tmp_TokAppendStr(b byte) {
    libgogo.CharAppend(&tok.strValue, b);
}
<commit_msg>scanner: Fix escaping bug<commit_after>// Copyright 2010 The GoGo Authors. All rights reserved.
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

// This file holds the basic scanning routines that separate a source file
// into the various tokens

package main

import "./libgogo/_obj/libgogo"

//
// Function fetching a new character from the file that is compiled
// Adds counters for line and columns
//
func GetCharWrapped() byte {
    var singleChar byte;
    if tok.nextChar == 0 {
        singleChar = libgogo.GetChar(fileInfo[curFileIndex].fd);
        if (singleChar == 10) {
            fileInfo[curFileIndex].charCounter = 1;
            fileInfo[curFileIndex].lineCounter = fileInfo[curFileIndex].lineCounter + 1;
        } else {
            fileInfo[curFileIndex].charCounter = fileInfo[curFileIndex].charCounter + 1;
        }
    } else {
        singleChar = tok.nextChar;
        tok.nextChar = 0;
    }

    return singleChar;
}

func GetEscapedCharWrapped() byte {
    var singleChar byte;
    var nextChar byte;
    var returnChar byte;
    singleChar = GetCharWrapped();
    if singleChar == '\\' { //Escaped character
        nextChar = GetCharWrapped();
        if nextChar == 'n' { //n => \n
            returnChar = '\n';
        } else {
            if nextChar == '\n' { //Line break in string
                returnChar = GetCharWrapped(); //Return character after line break
            } else {
                if nextChar == '\\' { //\ => \ (single \)
                    returnChar = '\\';
                } else {
                     if nextChar == 'b' { //b => \b
                         returnChar = '\b';
                    } else {
                        ScanErrorString("Unknown escape sequence.");
                    }
                }
            }
        }
    } else { //Normal character
        returnChar = singleChar;
    }
    return returnChar;
}

func GetNextTokenRaw() {
    var singleChar byte; // Byte holding the last read value
    // Flag indicating whether we are in a comment.
    // 0 for no comment
    // 1 for a single line comment
    // 2 for a multi line comment
    var inComment uint64;
    var done uint64; // Flag indicating whether a cycle (Token) is finsihed
    var spaceDone uint64; // Flag indicating whether an abolishment cycle is finished
    var numBuf string;

    // Initialize variables
    done = 0;
    spaceDone = 0;
    inComment = 0;
    tok.strValue = "";

    // If the previous cycle had to read the next char (and stored it), it is
    // now used as first read
    singleChar = GetCharWrapped();

    // check if it is a valid read, or an EOF
    if singleChar == 0 {
        tok.id = TOKEN_EOS;
        done = 1;
        spaceDone = 1;
    }

    //
    // Cleaning Tasks
    // The next part strips out spaces, newlines, tabs, and comments
    // Comments can either be single line with double slashes (//) or multiline
    // using C++ syntax /* */
    //
    for ;spaceDone != 1; {

        // check whether a comment is starting
        if singleChar == '/' {
            // if we are in a comment skip the rest, get the next char otherwise
            if inComment == 0 {
                singleChar = GetCharWrapped();
                if singleChar == '/' {
                    // we are in a single line comment (until newline is found)
                    inComment = 1;
                } else {
                    if singleChar == '*' {
                        // we are in a multiline comment (until ending is found)
                        inComment = 2;
                    } else {
                        tok.nextChar = singleChar;
                        singleChar = '/';
                    }
                }
            }
        }

        if inComment == 2 {
            // check whether a multi-line comment is ending
            if singleChar == '*' {
                singleChar = GetCharWrapped();
                if singleChar == '/' {
                    if inComment == 2 {
                        inComment = 0;
                        singleChar = GetCharWrapped();
                    }
                }
            }
        }

        // if character is a newline:
        //  *) if in a singleline comment, exit the comment
        //  *) skip otherwise
        if singleChar == 10 {
            if inComment == 1 {
                inComment = 0;
            }
        }

        // handle everything that is not a space,tab,newline
        if (singleChar != ' ') && (singleChar != 9) && (singleChar != 10) {
            // if not in a comment we have our current valid char
            if inComment == 0 {
                spaceDone = 1;
            }

            // check if GetChar() returned EOF while skipping
            if singleChar == 0 {
                tok.id = TOKEN_EOS;
                spaceDone = 1;
                done = 1;
            }
        }

        // if we are not done until now, get a new character and start another abolishing cycle
        if spaceDone == 0 {
            singleChar=GetCharWrapped();
        }
    }

    //
    // Actual scanning part starts here
    //

    // Catch identifiers
    // identifier = letter { letter | digit }.
    if (done != 1) && ((singleChar >= 'A') && (singleChar <= 'Z')) || ((singleChar >= 'a') && (singleChar <= 'z')) || (singleChar == '_') { // check for letter or _
        tok.id = TOKEN_IDENTIFIER;
        // preceding characters may be letter,_, or a number
        for ; ((singleChar >= 'A') && (singleChar <= 'Z')) || ((singleChar >= 'a') && (singleChar <= 'z')) || (singleChar == '_') || ((singleChar >= '0') && (singleChar <= '9')); singleChar = GetCharWrapped() {
            tmp_TokAppendStr(singleChar);
        }
        // save the last read character for the next GetNextToken() cycle
        tok.nextChar = singleChar;
        done = 1;
    }

    // string "..."
    if (done != 1) && (singleChar == '"') {
        tok.id = TOKEN_STRING;
        for singleChar = GetEscapedCharWrapped(); singleChar != '"';singleChar = GetEscapedCharWrapped() {
            tmp_TokAppendStr(singleChar);
        }
        if singleChar != '"' {
            ScanErrorString("String not closing.");
        }
        done = 1;
    }

    // Single Quoted Character
    if (done != 1) && singleChar == 39 {
        singleChar = GetEscapedCharWrapped();
        if singleChar != 39 {
            tok.id = TOKEN_INTEGER;
            tok.intValue = libgogo.ToIntFromByte(singleChar);
        } else {
            ScanErrorString("Unknown character.");
        }
        singleChar = GetCharWrapped();
        if singleChar != 39 {
            ScanErrorString("Only single characters allowed. Use corresponding integer for special characters.");
        }
        done = 1;
    }

    // left brace (
    if (done != 1) && singleChar == '(' {
        tok.id = TOKEN_LBRAC;
        done = 1;
    }

    // right brace )
    if (done != 1) && singleChar == ')' {
        tok.id = TOKEN_RBRAC;
        done = 1;
    }

    // left square bracket [
    if (done != 1) && singleChar == '[' {
        tok.id = TOKEN_LSBRAC;
        done = 1;
    }

    // right square bracket ]
    if (done != 1) && singleChar == ']' {
        tok.id = TOKEN_RSBRAC;
        done = 1;
    }

    // integer
    if (done != 1) && (singleChar > 47) && (singleChar < 58) {
        numBuf = "";

        for ; (singleChar > 47) && (singleChar < 58) ; singleChar = GetCharWrapped() {
            libgogo.CharAppend(&numBuf, singleChar);
        }

        tok.nextChar = singleChar;
        tok.id = TOKEN_INTEGER;
        tok.intValue = libgogo.StringToInt(numBuf);

        done = 1;
    }

    // Left curly bracket '{'
    if (done != 1) && (singleChar == '{') {
        tok.id = TOKEN_LCBRAC;
        done = 1;
    }

    // Right curly bracket '}'
    if (done != 1) && (singleChar == '}') {
        tok.id = TOKEN_RCBRAC;
        done = 1;
    }

    // Point '.'
    if (done != 1) && (singleChar == '.') {
        tok.id = TOKEN_PT;
        done = 1;
    }

    // Not ('!') or Not Equal ('!=')
    if (done != 1) && (singleChar == '!') {
        singleChar = GetCharWrapped();
        if singleChar == '=' {
            tok.id = TOKEN_NOTEQUAL;
        } else {
            tok.id = TOKEN_NOT;
            tok.nextChar = singleChar;
        }
        done = 1;
    }

    // Semicolon ';'
    if (done != 1) && (singleChar == ';') {
        tok.id = TOKEN_SEMICOLON;
        done = 1;
    }

    // Colon ','
    if (done != 1) && (singleChar == ',') {
        tok.id = TOKEN_COLON;
        done = 1;
    }

    // Assignment '=' or Equals comparison '=='
    if (done != 1) && (singleChar == '=') {
        singleChar = GetCharWrapped();
        if singleChar == '=' {
            tok.id = TOKEN_EQUALS;
        } else {
            tok.id = TOKEN_ASSIGN;
            tok.nextChar = singleChar;
        }
        done = 1;
    }

    // AND Relation '&&'
    if (done != 1) && (singleChar == '&') {
        singleChar = GetCharWrapped();
        if singleChar == '&' {
            tok.id = TOKEN_REL_AND;
        } else {
            tok.id = TOKEN_OP_ADR;
            tok.nextChar = singleChar;
        }
        done = 1;
    }

    // OR Relation '||'
    if (done != 1) && (singleChar == '|') {
        singleChar = GetCharWrapped();
        if singleChar == '|' {
            tok.id = TOKEN_REL_OR;
        } else {
            ScanErrorString("No binary OR (|) supported. Only ||.");
        }
        done = 1;
    }

    // Greater and Greater-Than relation
    if (done != 1) && (singleChar == '>') {
        singleChar = GetCharWrapped();
        if singleChar == '=' {
            tok.id = TOKEN_REL_GTOE;
        } else {
            tok.id = TOKEN_REL_GT;
            tok.nextChar = singleChar;
        }
        done = 1;
    }

    // Less and Less-Than relation
    if (done != 1) && (singleChar == '<') {
        singleChar = GetCharWrapped();
        if singleChar == '=' {
            tok.id = TOKEN_REL_LTOE;
        } else {
            tok.id = TOKEN_REL_LT;
            tok.nextChar = singleChar;
        }
        done = 1;
    }

    if (done != 1) && (singleChar == '+') {
        tok.id = TOKEN_ARITH_PLUS;
        done = 1;
    }

    if (done != 1) && (singleChar == '-') {
        tok.id = TOKEN_ARITH_MINUS;
        done = 1;
    }

    if (done != 1) && (singleChar == '*') {
        tok.id = TOKEN_ARITH_MUL;
        done = 1;
    }

    if (done != 1) && (singleChar == '/') {
        tok.id = TOKEN_ARITH_DIV;
        done = 1;
    }

    if (done != 1) {
        ScanErrorChar(singleChar);
    }
}

//
// GetNextToken should be called by the parser. It bascially fetches the next
// token by calling GetNextTokenRaw() and filters the identifiers for known
// keywords.
//
func GetNextToken() {
    var cmpValue uint64;
    GetNextTokenRaw();

    // Convert identifier to keyworded tokens
    if tok.id == TOKEN_IDENTIFIER {
        cmpValue = libgogo.StringCompare("if",tok.strValue);
        if cmpValue == 0 {
            tok.id = TOKEN_IF;
        }
        cmpValue = libgogo.StringCompare("else",tok.strValue);
        if cmpValue == 0 {
            tok.id = TOKEN_ELSE;
        }
        cmpValue = libgogo.StringCompare("for",tok.strValue);
        if cmpValue == 0 {
            tok.id = TOKEN_FOR;
        }
        cmpValue = libgogo.StringCompare("type",tok.strValue);
        if cmpValue == 0 {
            tok.id = TOKEN_TYPE;
        }
        cmpValue = libgogo.StringCompare("const",tok.strValue);
        if cmpValue == 0 {
            tok.id = TOKEN_CONST;
        }
        cmpValue = libgogo.StringCompare("var",tok.strValue);
        if cmpValue == 0 {
            tok.id = TOKEN_VAR;
        }
        cmpValue = libgogo.StringCompare("struct", tok.strValue);
        if cmpValue == 0 {
            tok.id = TOKEN_STRUCT;
        }
        cmpValue = libgogo.StringCompare("return", tok.strValue);
        if cmpValue == 0 {
            tok.id = TOKEN_RETURN;
        }
        cmpValue = libgogo.StringCompare("func", tok.strValue);
        if cmpValue == 0 {
            tok.id = TOKEN_FUNC;
        }
        cmpValue = libgogo.StringCompare("import", tok.strValue);
        if cmpValue == 0 {
            tok.id = TOKEN_IMPORT;
        }
        cmpValue = libgogo.StringCompare("package", tok.strValue);
        if cmpValue == 0 {
            tok.id = TOKEN_PACKAGE;
        }
        cmpValue = libgogo.StringCompare("break", tok.strValue);
        if cmpValue == 0 {
            tok.id = TOKEN_BREAK;
        }
        cmpValue = libgogo.StringCompare("continue", tok.strValue);
        if cmpValue == 0 {
            tok.id = TOKEN_CONTINUE;
        }
    }

    tok.nextToken = 0;
}

//
// Debugging and temporary functions
//

func debugToken(tok *Token) {
    libgogo.PrintString("---------------------\n");
    libgogo.PrintString("Token Id: ");
    libgogo.PrintNumber(tok.id);
    libgogo.PrintString("\n");
    if (tok.id == TOKEN_IDENTIFIER) || (tok.id == TOKEN_STRING) {
        libgogo.PrintString("Stored string: ");
        libgogo.PrintString(tok.strValue);
        libgogo.PrintString("\n");
    }
    if tok.id == TOKEN_INTEGER {
        libgogo.PrintString("Stored integer: ");
        libgogo.PrintNumber(tok.intValue);
        libgogo.PrintString("\n");
    }
}

func tmp_TokAppendStr(b byte) {
    libgogo.CharAppend(&tok.strValue, b);
}
<|endoftext|>