<commit_before>// Copyright GoFrame Author(https://goframe.org). All Rights Reserved.
//
// This Source Code Form is subject to the terms of the MIT License.
// If a copy of the MIT was not distributed with this file,
// You can obtain one at https://github.com/gogf/gf.

package glog

import (
	"bytes"
	"context"
	"fmt"
	"github.com/fatih/color"
	"github.com/gogf/gf/container/gtype"
	"github.com/gogf/gf/internal/intlog"
	"github.com/gogf/gf/os/gctx"
	"github.com/gogf/gf/os/gfpool"
	"github.com/gogf/gf/os/gmlock"
	"github.com/gogf/gf/os/gtimer"
	"go.opentelemetry.io/otel/trace"
	"io"
	"os"
	"strings"
	"time"

	"github.com/gogf/gf/debug/gdebug"

	"github.com/gogf/gf/os/gfile"
	"github.com/gogf/gf/os/gtime"
	"github.com/gogf/gf/text/gregex"
	"github.com/gogf/gf/util/gconv"
)

// Logger is the struct for logging management.
type Logger struct {
	ctx    context.Context // Context for logging.
	init   *gtype.Bool     // Initialized.
	parent *Logger         // Parent logger, if it is not empty, it means the logger is used in chaining function.
	config Config          // Logger configuration.
}

const (
	defaultFileFormat                 = `{Y-m-d}.log`
	defaultFileFlags                  = os.O_CREATE | os.O_WRONLY | os.O_APPEND
	defaultFilePerm                   = os.FileMode(0666)
	defaultFileExpire                 = time.Minute
	pathFilterKey                     = "/os/glog/glog"
	memoryLockPrefixForPrintingToFile = "glog.printToFile:"
)

const (
	F_ASYNC      = 1 << iota // Print logging content asynchronously。
	F_FILE_LONG              // Print full file name and line number: /a/b/c/d.go:23.
	F_FILE_SHORT             // Print final file name element and line number: d.go:23. overrides F_FILE_LONG.
	F_TIME_DATE              // Print the date in the local time zone: 2009-01-23.
	F_TIME_TIME              // Print the time in the local time zone: 01:23:23.
	F_TIME_MILLI             // Print the time with milliseconds in the local time zone: 01:23:23.675.
	F_CALLER_FN              // Print Caller function name and package: main.main
	F_TIME_STD   = F_TIME_DATE | F_TIME_MILLI
)

// New creates and returns a custom logger.
func New() *Logger {
	logger := &Logger{
		init:   gtype.NewBool(),
		config: DefaultConfig(),
	}
	return logger
}

// NewWithWriter creates and returns a custom logger with io.Writer.
func NewWithWriter(writer io.Writer) *Logger {
	l := New()
	l.SetWriter(writer)
	return l
}

// Clone returns a new logger, which is the clone the current logger.
// It's commonly used for chaining operations.
func (l *Logger) Clone() *Logger {
	newLogger := New()
	newLogger.ctx = l.ctx
	newLogger.config = l.config
	newLogger.parent = l
	return newLogger
}

// getFilePath returns the logging file path.
// The logging file name must have extension name of "log".
func (l *Logger) getFilePath(now time.Time) string {
	// Content containing "{}" in the file name is formatted using gtime.
	file, _ := gregex.ReplaceStringFunc(`{.+?}`, l.config.File, func(s string) string {
		return gtime.New(now).Format(strings.Trim(s, "{}"))
	})
	file = gfile.Join(l.config.Path, file)
	return file
}

// print prints `s` to defined writer, logging file or passed `std`.
func (l *Logger) print(ctx context.Context, level int, values ...interface{}) {
	// Lazy initialize for rotation feature.
	// It uses atomic reading operation to enhance the performance checking.
	// It here uses CAP for performance and concurrent safety.
	p := l
	if p.parent != nil {
		p = p.parent
	}
	// It just initializes once for each logger.
	if p.config.RotateSize > 0 || p.config.RotateExpire > 0 {
		if !p.init.Val() && p.init.Cas(false, true) {
			gtimer.AddOnce(p.config.RotateCheckInterval, p.rotateChecksTimely)
			intlog.Printf(ctx, "logger rotation initialized: every %s", p.config.RotateCheckInterval.String())
		}
	}

	var (
		now   = time.Now()
		input = &HandlerInput{
			Logger:       l,
			Buffer:       bytes.NewBuffer(nil),
			Ctx:          ctx,
			Time:         now,
			Color:        defaultLevelColor[level],
			Level:        level,
			handlerIndex: -1,
		}
	)
	if l.config.HeaderPrint {
		// Time.
		timeFormat := ""
		if l.config.Flags&F_TIME_DATE > 0 {
			timeFormat += "2006-01-02"
		}
		if l.config.Flags&F_TIME_TIME > 0 {
			if timeFormat != "" {
				timeFormat += " "
			}
			timeFormat += "15:04:05"
		}
		if l.config.Flags&F_TIME_MILLI > 0 {
			if timeFormat != "" {
				timeFormat += " "
			}
			timeFormat += "15:04:05.000"
		}
		if len(timeFormat) > 0 {
			input.TimeFormat = now.Format(timeFormat)
		}

		// Level string.
		input.LevelFormat = l.getLevelPrefixWithBrackets(level)

		// Caller path and Fn name.
		if l.config.Flags&(F_FILE_LONG|F_FILE_SHORT|F_CALLER_FN) > 0 {
			callerFnName, path, line := gdebug.CallerWithFilter(pathFilterKey, l.config.StSkip)
			if l.config.Flags&F_CALLER_FN > 0 {
				if len(callerFnName) > 2 {
					input.CallerFunc = fmt.Sprintf(`[%s]`, callerFnName)
				}
			}
			if line >= 0 && len(path) > 1 {
				if l.config.Flags&F_FILE_LONG > 0 {
					input.CallerPath = fmt.Sprintf(`%s:%d:`, path, line)
				}
				if l.config.Flags&F_FILE_SHORT > 0 {
					input.CallerPath = fmt.Sprintf(`%s:%d:`, gfile.Basename(path), line)
				}
			}
		}
		// Prefix.
		if len(l.config.Prefix) > 0 {
			input.Prefix = l.config.Prefix
		}
	}
	// Convert value to string.
	if ctx != nil {
		// Tracing values.
		spanCtx := trace.SpanContextFromContext(ctx)
		if traceId := spanCtx.TraceID(); traceId.IsValid() {
			input.CtxStr = traceId.String()
		}
		// Context values.
		if len(l.config.CtxKeys) > 0 {
			for _, ctxKey := range l.config.CtxKeys {
				var ctxValue interface{}
				if ctxValue = ctx.Value(ctxKey); ctxValue == nil {
					ctxValue = ctx.Value(gctx.StrKey(gconv.String(ctxKey)))
				}
				if ctxValue != nil {
					if input.CtxStr != "" {
						input.CtxStr += ", "
					}
					input.CtxStr += gconv.String(ctxValue)
				}
			}
		}
		if input.CtxStr != "" {
			input.CtxStr = "{" + input.CtxStr + "}"
		}
	}
	var tempStr string
	for _, v := range values {
		tempStr = gconv.String(v)
		if len(input.Content) > 0 {
			if input.Content[len(input.Content)-1] == '\n' {
				// Remove one blank line(\n\n).
				if tempStr[0] == '\n' {
					input.Content += tempStr[1:]
				} else {
					input.Content += tempStr
				}
			} else {
				input.Content += " " + tempStr
			}
		} else {
			input.Content = tempStr
		}
	}
	if l.config.Flags&F_ASYNC > 0 {
		input.IsAsync = true
		err := asyncPool.Add(func() {
			input.Next()
		})
		if err != nil {
			intlog.Error(ctx, err)
		}
	} else {
		input.Next()
	}
}

// doDefaultPrint outputs the logging content according configuration.
func (l *Logger) doDefaultPrint(ctx context.Context, input *HandlerInput) *bytes.Buffer {
	var buffer *bytes.Buffer
	if l.config.Writer == nil {
		// Allow output to stdout?
		if l.config.StdoutPrint {
			if buf := l.printToStdout(ctx, input); buf != nil {
				buffer = buf
			}
		}

		// Output content to disk file.
		if l.config.Path != "" {
			if buf := l.printToFile(ctx, input.Time, input); buf != nil {
				buffer = buf
			}
		}
	} else {
		// Output to custom writer.
		if buf := l.printToWriter(ctx, input); buf != nil {
			buffer = buf
		}
	}
	return buffer
}

// printToWriter writes buffer to writer.
func (l *Logger) printToWriter(ctx context.Context, input *HandlerInput) *bytes.Buffer {
	if l.config.Writer != nil {
		var (
			buffer = input.getRealBuffer(l.config.WriterColorEnable)
		)
		if _, err := l.config.Writer.Write(buffer.Bytes()); err != nil {
			intlog.Error(ctx, err)
		}
		return buffer
	}
	return nil
}

// printToStdout outputs logging content to stdout.
func (l *Logger) printToStdout(ctx context.Context, input *HandlerInput) *bytes.Buffer {
	if l.config.StdoutPrint {
		var (
			buffer = input.getRealBuffer(true)
		)
		// This will lose color in Windows os system.
		// if _, err := os.Stdout.Write(input.getRealBuffer(true).Bytes()); err != nil {

		// This will print color in Windows os system.
		if _, err := fmt.Fprint(color.Output, buffer.String()); err != nil {
			intlog.Error(ctx, err)
		}
		return buffer
	}
	return nil
}

// printToFile outputs logging content to disk file.
func (l *Logger) printToFile(ctx context.Context, t time.Time, in *HandlerInput) *bytes.Buffer {
	var (
		buffer        = in.getRealBuffer(l.config.WriterColorEnable)
		logFilePath   = l.getFilePath(t)
		memoryLockKey = memoryLockPrefixForPrintingToFile + logFilePath
	)
	gmlock.Lock(memoryLockKey)
	defer gmlock.Unlock(memoryLockKey)

	// Rotation file size checks.
	if l.config.RotateSize > 0 {
		if gfile.Size(logFilePath) > l.config.RotateSize {
			l.rotateFileBySize(t)
		}
	}
	// Logging content outputting to disk file.
	if file := l.getFilePointer(ctx, logFilePath); file == nil {
		intlog.Errorf(ctx, `got nil file pointer for: %s`, logFilePath)
	} else {
		if _, err := file.Write(buffer.Bytes()); err != nil {
			intlog.Error(ctx, err)
		}
		if err := file.Close(); err != nil {
			intlog.Error(ctx, err)
		}
	}
	return buffer
}

// getFilePointer retrieves and returns a file pointer from file pool.
func (l *Logger) getFilePointer(ctx context.Context, path string) *gfpool.File {
	file, err := gfpool.Open(
		path,
		defaultFileFlags,
		defaultFilePerm,
		defaultFileExpire,
	)
	if err != nil {
		// panic(err)
		intlog.Error(ctx, err)
	}
	return file
}

// getCtx returns the context which is set through chaining operations.
// It returns an empty context if no context set previously.
func (l *Logger) getCtx() context.Context {
	if l.ctx != nil {
		return l.ctx
	}
	return context.TODO()
}

// printStd prints content `s` without stack.
func (l *Logger) printStd(level int, value ...interface{}) {
	l.print(l.getCtx(), level, value...)
}

// printStd prints content `s` with stack check.
func (l *Logger) printErr(level int, value ...interface{}) {
	if l.config.StStatus == 1 {
		if s := l.GetStack(); s != "" {
			value = append(value, "\nStack:\n"+s)
		}
	}
	// In matter of sequence, do not use stderr here, but use the same stdout.
	l.print(l.getCtx(), level, value...)
}

// format formats `values` using fmt.Sprintf.
func (l *Logger) format(format string, value ...interface{}) string {
	return fmt.Sprintf(format, value...)
}

// PrintStack prints the caller stack,
// the optional parameter `skip` specify the skipped stack offset from the end point.
func (l *Logger) PrintStack(skip ...int) {
	if s := l.GetStack(skip...); s != "" {
		l.Println("Stack:\n" + s)
	} else {
		l.Println()
	}
}

// GetStack returns the caller stack content,
// the optional parameter `skip` specify the skipped stack offset from the end point.
func (l *Logger) GetStack(skip ...int) string {
	stackSkip := l.config.StSkip
	if len(skip) > 0 {
		stackSkip += skip[0]
	}
	filters := []string{pathFilterKey}
	if l.config.StFilter != "" {
		filters = append(filters, l.config.StFilter)
	}
	return gdebug.StackWithFilters(filters, stackSkip)
}

// GetConfig returns the configuration of current Logger.
func (l *Logger) GetConfig() Config {
	return l.config
}
<commit_msg>Fixed: #1421<commit_after>// Copyright GoFrame Author(https://goframe.org). All Rights Reserved.
//
// This Source Code Form is subject to the terms of the MIT License.
// If a copy of the MIT was not distributed with this file,
// You can obtain one at https://github.com/gogf/gf.

package glog

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"os"
	"strings"
	"time"

	"github.com/fatih/color"
	"github.com/gogf/gf/container/gtype"
	"github.com/gogf/gf/debug/gdebug"
	"github.com/gogf/gf/internal/intlog"
	"github.com/gogf/gf/os/gctx"
	"github.com/gogf/gf/os/gfile"
	"github.com/gogf/gf/os/gfpool"
	"github.com/gogf/gf/os/gmlock"
	"github.com/gogf/gf/os/gtime"
	"github.com/gogf/gf/os/gtimer"
	"github.com/gogf/gf/text/gregex"
	"github.com/gogf/gf/util/gconv"
	"go.opentelemetry.io/otel/trace"
)

// Logger is the struct for logging management.
type Logger struct {
	ctx    context.Context // Context for logging.
	init   *gtype.Bool     // Initialized.
	parent *Logger         // Parent logger, if it is not empty, it means the logger is used in chaining function.
	config Config          // Logger configuration.
}

const (
	defaultFileFormat                 = `{Y-m-d}.log`
	defaultFileFlags                  = os.O_CREATE | os.O_WRONLY | os.O_APPEND
	defaultFilePerm                   = os.FileMode(0666)
	defaultFileExpire                 = time.Minute
	pathFilterKey                     = "/os/glog/glog"
	memoryLockPrefixForPrintingToFile = "glog.printToFile:"
)

const (
	F_ASYNC      = 1 << iota // Print logging content asynchronously。
	F_FILE_LONG              // Print full file name and line number: /a/b/c/d.go:23.
	F_FILE_SHORT             // Print final file name element and line number: d.go:23. overrides F_FILE_LONG.
	F_TIME_DATE              // Print the date in the local time zone: 2009-01-23.
	F_TIME_TIME              // Print the time in the local time zone: 01:23:23.
	F_TIME_MILLI             // Print the time with milliseconds in the local time zone: 01:23:23.675.
	F_CALLER_FN              // Print Caller function name and package: main.main
	F_TIME_STD   = F_TIME_DATE | F_TIME_MILLI
)

// New creates and returns a custom logger.
func New() *Logger {
	logger := &Logger{
		init:   gtype.NewBool(),
		config: DefaultConfig(),
	}
	return logger
}

// NewWithWriter creates and returns a custom logger with io.Writer.
func NewWithWriter(writer io.Writer) *Logger {
	l := New()
	l.SetWriter(writer)
	return l
}

// Clone returns a new logger, which is the clone the current logger.
// It's commonly used for chaining operations.
func (l *Logger) Clone() *Logger {
	newLogger := New()
	newLogger.ctx = l.ctx
	newLogger.config = l.config
	newLogger.parent = l
	return newLogger
}

// getFilePath returns the logging file path.
// The logging file name must have extension name of "log".
func (l *Logger) getFilePath(now time.Time) string {
	// Content containing "{}" in the file name is formatted using gtime.
	file, _ := gregex.ReplaceStringFunc(`{.+?}`, l.config.File, func(s string) string {
		return gtime.New(now).Format(strings.Trim(s, "{}"))
	})
	file = gfile.Join(l.config.Path, file)
	return file
}

// print prints `s` to defined writer, logging file or passed `std`.
func (l *Logger) print(ctx context.Context, level int, values ...interface{}) {
	// Lazy initialize for rotation feature.
	// It uses atomic reading operation to enhance the performance checking.
	// It here uses CAP for performance and concurrent safety.
	p := l
	if p.parent != nil {
		p = p.parent
	}
	// It just initializes once for each logger.
	if p.config.RotateSize > 0 || p.config.RotateExpire > 0 {
		if !p.init.Val() && p.init.Cas(false, true) {
			gtimer.AddOnce(p.config.RotateCheckInterval, p.rotateChecksTimely)
			intlog.Printf(ctx, "logger rotation initialized: every %s", p.config.RotateCheckInterval.String())
		}
	}

	var (
		now   = time.Now()
		input = &HandlerInput{
			Logger:       l,
			Buffer:       bytes.NewBuffer(nil),
			Ctx:          ctx,
			Time:         now,
			Color:        defaultLevelColor[level],
			Level:        level,
			handlerIndex: -1,
		}
	)
	if l.config.HeaderPrint {
		// Time.
		timeFormat := ""
		if l.config.Flags&F_TIME_DATE > 0 {
			timeFormat += "2006-01-02"
		}
		if l.config.Flags&F_TIME_TIME > 0 {
			if timeFormat != "" {
				timeFormat += " "
			}
			timeFormat += "15:04:05"
		}
		if l.config.Flags&F_TIME_MILLI > 0 {
			if timeFormat != "" {
				timeFormat += " "
			}
			timeFormat += "15:04:05.000"
		}
		if len(timeFormat) > 0 {
			input.TimeFormat = now.Format(timeFormat)
		}

		// Level string.
		input.LevelFormat = l.getLevelPrefixWithBrackets(level)

		// Caller path and Fn name.
		if l.config.Flags&(F_FILE_LONG|F_FILE_SHORT|F_CALLER_FN) > 0 {
			callerFnName, path, line := gdebug.CallerWithFilter(pathFilterKey, l.config.StSkip)
			if l.config.Flags&F_CALLER_FN > 0 {
				if len(callerFnName) > 2 {
					input.CallerFunc = fmt.Sprintf(`[%s]`, callerFnName)
				}
			}
			if line >= 0 && len(path) > 1 {
				if l.config.Flags&F_FILE_LONG > 0 {
					input.CallerPath = fmt.Sprintf(`%s:%d:`, path, line)
				}
				if l.config.Flags&F_FILE_SHORT > 0 {
					input.CallerPath = fmt.Sprintf(`%s:%d:`, gfile.Basename(path), line)
				}
			}
		}
		// Prefix.
		if len(l.config.Prefix) > 0 {
			input.Prefix = l.config.Prefix
		}
	}
	// Convert value to string.
	if ctx != nil {
		// Tracing values.
		spanCtx := trace.SpanContextFromContext(ctx)
		if traceId := spanCtx.TraceID(); traceId.IsValid() {
			input.CtxStr = traceId.String()
		}
		// Context values.
		if len(l.config.CtxKeys) > 0 {
			for _, ctxKey := range l.config.CtxKeys {
				var ctxValue interface{}
				if ctxValue = ctx.Value(ctxKey); ctxValue == nil {
					ctxValue = ctx.Value(gctx.StrKey(gconv.String(ctxKey)))
				}
				if ctxValue != nil {
					if input.CtxStr != "" {
						input.CtxStr += ", "
					}
					input.CtxStr += gconv.String(ctxValue)
				}
			}
		}
		if input.CtxStr != "" {
			input.CtxStr = "{" + input.CtxStr + "}"
		}
	}
	var tempStr string
	for _, v := range values {
		tempStr = gconv.String(v)
		if len(input.Content) > 0 {
			if input.Content[len(input.Content)-1] == '\n' {
				// Remove one blank line(\n\n).
				if len(tempStr) > 0 && tempStr[0] == '\n' {
					input.Content += tempStr[1:]
				} else {
					input.Content += tempStr
				}
			} else {
				input.Content += " " + tempStr
			}
		} else {
			input.Content = tempStr
		}
	}
	if l.config.Flags&F_ASYNC > 0 {
		input.IsAsync = true
		err := asyncPool.Add(func() {
			input.Next()
		})
		if err != nil {
			intlog.Error(ctx, err)
		}
	} else {
		input.Next()
	}
}

// doDefaultPrint outputs the logging content according configuration.
func (l *Logger) doDefaultPrint(ctx context.Context, input *HandlerInput) *bytes.Buffer {
	var buffer *bytes.Buffer
	if l.config.Writer == nil {
		// Allow output to stdout?
		if l.config.StdoutPrint {
			if buf := l.printToStdout(ctx, input); buf != nil {
				buffer = buf
			}
		}

		// Output content to disk file.
		if l.config.Path != "" {
			if buf := l.printToFile(ctx, input.Time, input); buf != nil {
				buffer = buf
			}
		}
	} else {
		// Output to custom writer.
		if buf := l.printToWriter(ctx, input); buf != nil {
			buffer = buf
		}
	}
	return buffer
}

// printToWriter writes buffer to writer.
func (l *Logger) printToWriter(ctx context.Context, input *HandlerInput) *bytes.Buffer {
	if l.config.Writer != nil {
		var (
			buffer = input.getRealBuffer(l.config.WriterColorEnable)
		)
		if _, err := l.config.Writer.Write(buffer.Bytes()); err != nil {
			intlog.Error(ctx, err)
		}
		return buffer
	}
	return nil
}

// printToStdout outputs logging content to stdout.
func (l *Logger) printToStdout(ctx context.Context, input *HandlerInput) *bytes.Buffer {
	if l.config.StdoutPrint {
		var (
			buffer = input.getRealBuffer(true)
		)
		// This will lose color in Windows os system.
		// if _, err := os.Stdout.Write(input.getRealBuffer(true).Bytes()); err != nil {

		// This will print color in Windows os system.
		if _, err := fmt.Fprint(color.Output, buffer.String()); err != nil {
			intlog.Error(ctx, err)
		}
		return buffer
	}
	return nil
}

// printToFile outputs logging content to disk file.
func (l *Logger) printToFile(ctx context.Context, t time.Time, in *HandlerInput) *bytes.Buffer {
	var (
		buffer        = in.getRealBuffer(l.config.WriterColorEnable)
		logFilePath   = l.getFilePath(t)
		memoryLockKey = memoryLockPrefixForPrintingToFile + logFilePath
	)
	gmlock.Lock(memoryLockKey)
	defer gmlock.Unlock(memoryLockKey)

	// Rotation file size checks.
	if l.config.RotateSize > 0 {
		if gfile.Size(logFilePath) > l.config.RotateSize {
			l.rotateFileBySize(t)
		}
	}
	// Logging content outputting to disk file.
	if file := l.getFilePointer(ctx, logFilePath); file == nil {
		intlog.Errorf(ctx, `got nil file pointer for: %s`, logFilePath)
	} else {
		if _, err := file.Write(buffer.Bytes()); err != nil {
			intlog.Error(ctx, err)
		}
		if err := file.Close(); err != nil {
			intlog.Error(ctx, err)
		}
	}
	return buffer
}

// getFilePointer retrieves and returns a file pointer from file pool.
func (l *Logger) getFilePointer(ctx context.Context, path string) *gfpool.File {
	file, err := gfpool.Open(
		path,
		defaultFileFlags,
		defaultFilePerm,
		defaultFileExpire,
	)
	if err != nil {
		// panic(err)
		intlog.Error(ctx, err)
	}
	return file
}

// getCtx returns the context which is set through chaining operations.
// It returns an empty context if no context set previously.
func (l *Logger) getCtx() context.Context {
	if l.ctx != nil {
		return l.ctx
	}
	return context.TODO()
}

// printStd prints content `s` without stack.
func (l *Logger) printStd(level int, value ...interface{}) {
	l.print(l.getCtx(), level, value...)
}

// printStd prints content `s` with stack check.
func (l *Logger) printErr(level int, value ...interface{}) {
	if l.config.StStatus == 1 {
		if s := l.GetStack(); s != "" {
			value = append(value, "\nStack:\n"+s)
		}
	}
	// In matter of sequence, do not use stderr here, but use the same stdout.
	l.print(l.getCtx(), level, value...)
}

// format formats `values` using fmt.Sprintf.
func (l *Logger) format(format string, value ...interface{}) string {
	return fmt.Sprintf(format, value...)
}

// PrintStack prints the caller stack,
// the optional parameter `skip` specify the skipped stack offset from the end point.
func (l *Logger) PrintStack(skip ...int) {
	if s := l.GetStack(skip...); s != "" {
		l.Println("Stack:\n" + s)
	} else {
		l.Println()
	}
}

// GetStack returns the caller stack content,
// the optional parameter `skip` specify the skipped stack offset from the end point.
func (l *Logger) GetStack(skip ...int) string {
	stackSkip := l.config.StSkip
	if len(skip) > 0 {
		stackSkip += skip[0]
	}
	filters := []string{pathFilterKey}
	if l.config.StFilter != "" {
		filters = append(filters, l.config.StFilter)
	}
	return gdebug.StackWithFilters(filters, stackSkip)
}

// GetConfig returns the configuration of current Logger.
func (l *Logger) GetConfig() Config {
	return l.config
}
<|endoftext|>


<commit_before>// +build test-exec

package main

import (
	"bytes"
	"code.google.com/p/go.net/html"
	"encoding/base64"
	"encoding/gob"
	"flag"
	"fmt"
	"github.com/coreos/go-etcd/etcd"
	"github.com/iron-io/iron_go/mq"
	exec "github.com/mesos/mesos-go/executor"
	mesos "github.com/mesos/mesos-go/mesosproto"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
	"time"
)

type exampleExecutor struct {
	tasksLaunched int
}

type QueueMsg struct {
	URL       string
	ID        string
	QueueName string
}

type dataStore struct {
	URL  string
	HTML string
	Date time.Time
}

func (data *dataStore) String() string {
	return "'url': '" + data.URL + "', 'html': '" + data.HTML + "' , 'date' : '" + data.Date.String() + "'"
}

func newExampleExecutor() *exampleExecutor {
	return &exampleExecutor{tasksLaunched: 0}
}

func (exec *exampleExecutor) Registered(driver exec.ExecutorDriver, execInfo *mesos.ExecutorInfo, fwinfo *mesos.FrameworkInfo, slaveInfo *mesos.SlaveInfo) {
	fmt.Println("Registered Executor on slave ", slaveInfo.GetHostname())
}

func (exec *exampleExecutor) Reregistered(driver exec.ExecutorDriver, slaveInfo *mesos.SlaveInfo) {
	fmt.Println("Re-registered Executor on slave ", slaveInfo.GetHostname())
}

func (exec *exampleExecutor) Disconnected(exec.ExecutorDriver) {
	fmt.Println("Executor disconnected.")
}

func (exec *exampleExecutor) LaunchTask(driver exec.ExecutorDriver, taskInfo *mesos.TaskInfo) {
	fmt.Println("Launching task", taskInfo.GetName(), "with command", taskInfo.Command.GetValue())

	runStatus := &mesos.TaskStatus{
		TaskId: taskInfo.GetTaskId(),
		State:  mesos.TaskState_TASK_RUNNING.Enum(),
	}
	_, err := driver.SendStatusUpdate(runStatus)
	if err != nil {
		fmt.Println("Got error", err)
	}

	exec.tasksLaunched++
	fmt.Println("\n\n\n\nTotal tasks launched ", exec.tasksLaunched)
	//
	// this is where one would perform the requested task
	//
	etcdClient := etcd.NewClient([]string{"127.0.0.1:2379/"})
	ret := etcdClient.SyncCluster()
	if !ret {
		fmt.Println("\n\n\n\n Got problem \n\n\n\n")
	}
	payload := bytes.NewReader(taskInfo.GetData())
	var msgAndID QueueMsg
	dec := gob.NewDecoder(payload)
	err = dec.Decode(&msgAndID)
	if err != nil {
		fmt.Println("decode error:", err)
	}
	queue := mq.New(msgAndID.QueueName)
	resp, err := http.Get(msgAndID.URL)
	if err != nil {
		fmt.Printf("\n\n\n\nError while fetching url: %s, got error: %v\n", msgAndID.URL, err)
		err = queue.ReleaseMessage(msgAndID.ID, 0)
		if err != nil {
			fmt.Printf("Error releasing message id: %s from queue, got: %v\n", msgAndID.ID, err)
		}
		updateStatusDied(driver, taskInfo)
		return
	}
	defer resp.Body.Close()
	parseHtml(resp.Body, msgAndID.URL, queue)
	html, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		fmt.Printf("\n\n\n\nError while reading html for url: %s, got error: %v\n", msgAndID.URL, err)
		err = queue.ReleaseMessage(msgAndID.ID, 0)
		if err != nil {
			fmt.Printf("Error releasing message id: %s from queue, got: %v\n", msgAndID.ID, err)
		}
		updateStatusDied(driver, taskInfo)
		return
	}
	err = queue.DeleteMessage(msgAndID.ID)
	if err != nil {
		fmt.Printf("Error deleting message id: %s from queue, got: %v\n", msgAndID.ID, err)
	}
	encodedURL := base64.StdEncoding.EncodeToString([]byte(msgAndID.URL))
	data := dataStore{
		URL:  msgAndID.URL,
		HTML: string(html[:]),
		Date: time.Now().UTC(),
	}
	_, err = etcdClient.Set(encodedURL, data.String(), 0)
	if err != nil {
		fmt.Printf("Got error adding html to etcd, got: %v\n", err)
	}
	fmt.Printf("\n\n\nhtml url is %s\n\n\n", msgAndID.URL)
	fmt.Printf("\n\n\nhtml encodedURL is %s\n\n\n", encodedURL)
	// finish task
	fmt.Println("Finishing task", taskInfo.GetName())
	finStatus := &mesos.TaskStatus{
		TaskId: taskInfo.GetTaskId(),
		State:  mesos.TaskState_TASK_FINISHED.Enum(),
	}
	_, err = driver.SendStatusUpdate(finStatus)
	if err != nil {
		fmt.Println("Got error", err)
	}
	fmt.Println("Task finished", taskInfo.GetName())
}

func (exec *exampleExecutor) KillTask(exec.ExecutorDriver, *mesos.TaskID) {
	fmt.Println("Kill task")
}

func (exec *exampleExecutor) FrameworkMessage(driver exec.ExecutorDriver, msg string) {
	fmt.Println("Got framework message: ", msg)
}

func (exec *exampleExecutor) Shutdown(exec.ExecutorDriver) {
	fmt.Println("Shutting down the executor")
}

func (exec *exampleExecutor) Error(driver exec.ExecutorDriver, err string) {
	fmt.Println("Got error message:", err)
}

// -------------------------- func inits () ----------------- //
func init() {
	flag.Parse()
}

func main() {
	fmt.Println("Starting Example Executor (Go)")

	driver, err := exec.NewMesosExecutorDriver(newExampleExecutor())

	if err != nil {
		fmt.Println("Unable to create a ExecutorDriver ", err.Error())
	}

	_, err = driver.Start()
	if err != nil {
		fmt.Println("Got error:", err)
		return
	}
	fmt.Println("Executor process has started and running.")
	driver.Join()
}

func updateStatusDied(driver exec.ExecutorDriver, taskInfo *mesos.TaskInfo) {
	runStatus := &mesos.TaskStatus{
		TaskId: taskInfo.GetTaskId(),
		State:  mesos.TaskState_TASK_FAILED.Enum(),
	}
	_, err := driver.SendStatusUpdate(runStatus)
	if err != nil {
		fmt.Printf("Failed to tell mesos that we died, sorry, got: %v", err)
	}

}

func parseHtml(r io.Reader, originalURL string, q *mq.Queue) {

	link, err := url.Parse(originalURL)
	if err != nil {
		fmt.Printf("Error parsing url %s, got: %v", originalURL, err)
	}

	d := html.NewTokenizer(r)

	for {
		tokenType := d.Next()
		if tokenType == html.ErrorToken {
			return
		}
		token := d.Token()
		switch tokenType {
		case html.StartTagToken:
			for _, attribute := range token.Attr {
				if attribute.Key == "href" {
					if strings.HasPrefix(attribute.Val, "//") {
						fmt.Printf("Found url: %s:%s\n", link.Scheme, attribute.Val)
						q.PushString(fmt.Sprintf("%s:%s", link.Scheme, attribute.Val))
					} else if strings.HasPrefix(attribute.Val, "/") {
						fmt.Printf("Found url: %s://%s%s\n", link.Scheme, link.Host, attribute.Val)
						q.PushString(fmt.Sprintf("%s://%s%s", link.Scheme, link.Host, attribute.Val))
					} else {
						fmt.Printf("Not sure what to do with this url: %s\n", attribute.Val)
					}
				}
			}
		}
	}
}
<commit_msg>store html from each page<commit_after>// +build test-exec

package main

import (
	"bytes"
	"code.google.com/p/go.net/html"
	"encoding/base64"
	"encoding/gob"
	"flag"
	"fmt"
	"github.com/coreos/go-etcd/etcd"
	"github.com/iron-io/iron_go/mq"
	exec "github.com/mesos/mesos-go/executor"
	mesos "github.com/mesos/mesos-go/mesosproto"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
	"time"
)

type exampleExecutor struct {
	tasksLaunched int
}

type QueueMsg struct {
	URL       string
	ID        string
	QueueName string
}

type dataStore struct {
	URL  string
	HTML string
	Date time.Time
}

func (data *dataStore) String() string {
	return "'url': '" + data.URL + "', 'html': '" + data.HTML + "' , 'date' : '" + data.Date.String() + "'"
}

func newExampleExecutor() *exampleExecutor {
	return &exampleExecutor{tasksLaunched: 0}
}

func (exec *exampleExecutor) Registered(driver exec.ExecutorDriver, execInfo *mesos.ExecutorInfo, fwinfo *mesos.FrameworkInfo, slaveInfo *mesos.SlaveInfo) {
	fmt.Println("Registered Executor on slave ", slaveInfo.GetHostname())
}

func (exec *exampleExecutor) Reregistered(driver exec.ExecutorDriver, slaveInfo *mesos.SlaveInfo) {
	fmt.Println("Re-registered Executor on slave ", slaveInfo.GetHostname())
}

func (exec *exampleExecutor) Disconnected(exec.ExecutorDriver) {
	fmt.Println("Executor disconnected.")
}

func (exec *exampleExecutor) LaunchTask(driver exec.ExecutorDriver, taskInfo *mesos.TaskInfo) {
	fmt.Println("Launching task", taskInfo.GetName(), "with command", taskInfo.Command.GetValue())

	runStatus := &mesos.TaskStatus{
		TaskId: taskInfo.GetTaskId(),
		State:  mesos.TaskState_TASK_RUNNING.Enum(),
	}
	_, err := driver.SendStatusUpdate(runStatus)
	if err != nil {
		fmt.Println("Got error", err)
	}

	exec.tasksLaunched++
	fmt.Println("\n\n\n\nTotal tasks launched ", exec.tasksLaunched)
	//
	// this is where one would perform the requested task
	//
	etcdClient := etcd.NewClient([]string{"127.0.0.1:2379/"})
	ret := etcdClient.SyncCluster()
	if !ret {
		fmt.Println("\n\n\n\n Got problem \n\n\n\n")
	}
	payload := bytes.NewReader(taskInfo.GetData())
	var msgAndID QueueMsg
	dec := gob.NewDecoder(payload)
	err = dec.Decode(&msgAndID)
	if err != nil {
		fmt.Println("decode error:", err)
	}
	queue := mq.New(msgAndID.QueueName)
	resp, err := http.Get(msgAndID.URL)
	if err != nil {
		fmt.Printf("\n\n\n\nError while fetching url: %s, got error: %v\n", msgAndID.URL, err)
		err = queue.ReleaseMessage(msgAndID.ID, 0)
		if err != nil {
			fmt.Printf("Error releasing message id: %s from queue, got: %v\n", msgAndID.ID, err)
		}
		updateStatusDied(driver, taskInfo)
		return
	}
	defer resp.Body.Close()

	htmlData, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		fmt.Printf("\n\n\n\nError while reading html for url: %s, got error: %v\n", msgAndID.URL, err)
		err = queue.ReleaseMessage(msgAndID.ID, 0)
		if err != nil {
			fmt.Printf("Error releasing message id: %s from queue, got: %v\n", msgAndID.ID, err)
		}
		updateStatusDied(driver, taskInfo)
		return
	}
	err = queue.DeleteMessage(msgAndID.ID)
	if err != nil {
		fmt.Printf("Error deleting message id: %s from queue, got: %v\n", msgAndID.ID, err)
	}
	encodedURL := base64.StdEncoding.EncodeToString([]byte(msgAndID.URL))
	data := dataStore{
		URL:  msgAndID.URL,
		HTML: string(htmlData[:]),
		Date: time.Now().UTC(),
	}
	_, err = etcdClient.Set(encodedURL, data.String(), 0)
	if err != nil {
		fmt.Printf("Got error adding html to etcd, got: %v\n", err)
	}
	fmt.Printf("\n\n\nhtml url is %s\n\n\n", msgAndID.URL)
	fmt.Printf("\n\n\nhtml encodedURL is %s\n\n\n", encodedURL)
	parseHtml(resp.Body, msgAndID.URL, queue)
	// finish task
	fmt.Println("Finishing task", taskInfo.GetName())
	finStatus := &mesos.TaskStatus{
		TaskId: taskInfo.GetTaskId(),
		State:  mesos.TaskState_TASK_FINISHED.Enum(),
	}
	_, err = driver.SendStatusUpdate(finStatus)
	if err != nil {
		fmt.Println("Got error", err)
	}
	fmt.Println("Task finished", taskInfo.GetName())
}

func (exec *exampleExecutor) KillTask(exec.ExecutorDriver, *mesos.TaskID) {
	fmt.Println("Kill task")
}

func (exec *exampleExecutor) FrameworkMessage(driver exec.ExecutorDriver, msg string) {
	fmt.Println("Got framework message: ", msg)
}

func (exec *exampleExecutor) Shutdown(exec.ExecutorDriver) {
	fmt.Println("Shutting down the executor")
}

func (exec *exampleExecutor) Error(driver exec.ExecutorDriver, err string) {
	fmt.Println("Got error message:", err)
}

// -------------------------- func inits () ----------------- //
func init() {
	flag.Parse()
}

func main() {
	fmt.Println("Starting Example Executor (Go)")

	driver, err := exec.NewMesosExecutorDriver(newExampleExecutor())

	if err != nil {
		fmt.Println("Unable to create a ExecutorDriver ", err.Error())
	}

	_, err = driver.Start()
	if err != nil {
		fmt.Println("Got error:", err)
		return
	}
	fmt.Println("Executor process has started and running.")
	driver.Join()
}

func updateStatusDied(driver exec.ExecutorDriver, taskInfo *mesos.TaskInfo) {
	runStatus := &mesos.TaskStatus{
		TaskId: taskInfo.GetTaskId(),
		State:  mesos.TaskState_TASK_FAILED.Enum(),
	}
	_, err := driver.SendStatusUpdate(runStatus)
	if err != nil {
		fmt.Printf("Failed to tell mesos that we died, sorry, got: %v", err)
	}

}

func parseHtml(r io.Reader, originalURL string, q *mq.Queue) {

	link, err := url.Parse(originalURL)
	if err != nil {
		fmt.Printf("Error parsing url %s, got: %v", originalURL, err)
	}

	d := html.NewTokenizer(r)

	for {
		tokenType := d.Next()
		if tokenType == html.ErrorToken {
			return
		}
		token := d.Token()
		switch tokenType {
		case html.StartTagToken:
			for _, attribute := range token.Attr {
				if attribute.Key == "href" {
					if strings.HasPrefix(attribute.Val, "//") {
						fmt.Printf("Found url: %s:%s\n", link.Scheme, attribute.Val)
						q.PushString(fmt.Sprintf("%s:%s", link.Scheme, attribute.Val))
					} else if strings.HasPrefix(attribute.Val, "/") {
						fmt.Printf("Found url: %s://%s%s\n", link.Scheme, link.Host, attribute.Val)
						q.PushString(fmt.Sprintf("%s://%s%s", link.Scheme, link.Host, attribute.Val))
					} else {
						fmt.Printf("Not sure what to do with this url: %s\n", attribute.Val)
					}
				}
			}
		}
	}
}
<|endoftext|>


<commit_before><commit_msg>Fix example after removal of 'error' in Gosl<commit_after><|endoftext|>


<commit_before>// Copyright (c) 2013 The btcsuite developers
// Copyright (c) 2015 The Decred developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package main

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/decred/dcrd/blockchain"
	"github.com/decred/dcrd/chaincfg"
	"github.com/decred/dcrd/chaincfg/chainhash"
	"github.com/decred/dcrd/database"
	_ "github.com/decred/dcrd/database/ldb"
)

const blockDbNamePrefix = "blocks"

var (
	cfg *config
)

// loadBlockDB opens the block database and returns a handle to it.
func loadBlockDB() (database.Db, error) {
	// The database name is based on the database type.
	dbType := cfg.DbType
	dbName := blockDbNamePrefix + "_" + dbType
	if dbType == "sqlite" {
		dbName = dbName + ".db"
	}
	dbPath := filepath.Join(cfg.DataDir, dbName)
	fmt.Printf("Loading block database from '%s'\n", dbPath)
	db, err := database.OpenDB(dbType, dbPath)
	if err != nil {
		return nil, err
	}
	return db, nil
}

// findCandidates searches the chain backwards for checkpoint candidates and
// returns a slice of found candidates, if any.  It also stops searching for
// candidates at the last checkpoint that is already hard coded into chain
// since there is no point in finding candidates before already existing
// checkpoints.
func findCandidates(db database.Db, latestHash *chainhash.Hash) ([]*chaincfg.Checkpoint, error) {
	// Start with the latest block of the main chain.
	block, err := db.FetchBlockBySha(latestHash)
	if err != nil {
		return nil, err
	}

	// Setup chain and get the latest checkpoint.  Ignore notifications
	// since they aren't needed for this util.
	chain := blockchain.New(db, nil, activeNetParams, nil)
	latestCheckpoint := chain.LatestCheckpoint()
	if latestCheckpoint == nil {
		return nil, fmt.Errorf("unable to retrieve latest checkpoint")
	}

	// The latest known block must be at least the last known checkpoint
	// plus required checkpoint confirmations.
	checkpointConfirmations := int64(blockchain.CheckpointConfirmations)
	requiredHeight := latestCheckpoint.Height + checkpointConfirmations
	if block.Height() < requiredHeight {
		return nil, fmt.Errorf("the block database is only at height "+
			"%d which is less than the latest checkpoint height "+
			"of %d plus required confirmations of %d",
			block.Height(), latestCheckpoint.Height,
			checkpointConfirmations)
	}

	// Indeterminate progress setup.
	numBlocksToTest := block.Height() - requiredHeight
	progressInterval := (numBlocksToTest / 100) + 1 // min 1
	fmt.Print("Searching for candidates")
	defer fmt.Println()

	// Loop backwards through the chain to find checkpoint candidates.
	candidates := make([]*chaincfg.Checkpoint, 0, cfg.NumCandidates)
	numTested := int64(0)
	for len(candidates) < cfg.NumCandidates && block.Height() > requiredHeight {
		// Display progress.
		if numTested%progressInterval == 0 {
			fmt.Print(".")
		}

		// Determine if this block is a checkpoint candidate.
		isCandidate, err := chain.IsCheckpointCandidate(block)
		if err != nil {
			return nil, err
		}

		// All checks passed, so this node seems like a reasonable
		// checkpoint candidate.
		if isCandidate {
			checkpoint := chaincfg.Checkpoint{
				Height: block.Height(),
				Hash:   block.Sha(),
			}
			candidates = append(candidates, &checkpoint)
		}

		prevHash := &block.MsgBlock().Header.PrevBlock
		block, err = db.FetchBlockBySha(prevHash)
		if err != nil {
			return nil, err
		}
		numTested++
	}
	return candidates, nil
}

// showCandidate display a checkpoint candidate using and output format
// determined by the configuration parameters.  The Go syntax output
// uses the format the chain code expects for checkpoints added to the list.
func showCandidate(candidateNum int, checkpoint *chaincfg.Checkpoint) {
	if cfg.UseGoOutput {
		fmt.Printf("Candidate %d -- {%d, newShaHashFromStr(\"%v\")},\n",
			candidateNum, checkpoint.Height, checkpoint.Hash)
		return
	}

	fmt.Printf("Candidate %d -- Height: %d, Hash: %v\n", candidateNum,
		checkpoint.Height, checkpoint.Hash)

}

func main() {
	// Load configuration and parse command line.
	tcfg, _, err := loadConfig()
	if err != nil {
		return
	}
	cfg = tcfg

	// Load the block database.
	db, err := loadBlockDB()
	if err != nil {
		fmt.Fprintf(os.Stderr, "failed to load database: %v\n", err)
		return
	}
	defer db.Close()

	// Get the latest block hash and height from the database and report
	// status.
	latestHash, height, err := db.NewestSha()
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		return
	}
	fmt.Printf("Block database loaded with block height %d\n", height)

	// Find checkpoint candidates.
	candidates, err := findCandidates(db, latestHash)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Unable to identify candidates: %v", err)
		return
	}

	// No candidates.
	if len(candidates) == 0 {
		fmt.Println("No candidates found.")
		return
	}

	// Show the candidates.
	for i, checkpoint := range candidates {
		showCandidate(i+1, checkpoint)
	}
}
<commit_msg>findcheckpoint: Update to allow first checkpoint.<commit_after>// Copyright (c) 2013 The btcsuite developers
// Copyright (c) 2015 The Decred developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package main

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/decred/dcrd/blockchain"
	"github.com/decred/dcrd/chaincfg"
	"github.com/decred/dcrd/chaincfg/chainhash"
	"github.com/decred/dcrd/database"
	_ "github.com/decred/dcrd/database/ldb"
)

const blockDbNamePrefix = "blocks"

var (
	cfg *config
)

// loadBlockDB opens the block database and returns a handle to it.
func loadBlockDB() (database.Db, error) {
	// The database name is based on the database type.
	dbType := cfg.DbType
	dbName := blockDbNamePrefix + "_" + dbType
	if dbType == "sqlite" {
		dbName = dbName + ".db"
	}
	dbPath := filepath.Join(cfg.DataDir, dbName)
	fmt.Printf("Loading block database from '%s'\n", dbPath)
	db, err := database.OpenDB(dbType, dbPath)
	if err != nil {
		return nil, err
	}
	return db, nil
}

// findCandidates searches the chain backwards for checkpoint candidates and
// returns a slice of found candidates, if any.  It also stops searching for
// candidates at the last checkpoint that is already hard coded into chain
// since there is no point in finding candidates before already existing
// checkpoints.
func findCandidates(db database.Db, latestHash *chainhash.Hash) ([]*chaincfg.Checkpoint, error) {
	// Start with the latest block of the main chain.
	block, err := db.FetchBlockBySha(latestHash)
	if err != nil {
		return nil, err
	}

	// Setup chain and get the latest checkpoint.  Ignore notifications
	// since they aren't needed for this util.
	chain := blockchain.New(db, nil, activeNetParams, nil)
	latestCheckpoint := chain.LatestCheckpoint()
	if latestCheckpoint == nil {
		// Set the latest checkpoint to the genesis block if there isn't
		// already one.
		latestCheckpoint = &chaincfg.Checkpoint{
			Hash:   activeNetParams.GenesisHash,
			Height: 0,
		}
	}

	// The latest known block must be at least the last known checkpoint
	// plus required checkpoint confirmations.
	checkpointConfirmations := int64(blockchain.CheckpointConfirmations)
	requiredHeight := latestCheckpoint.Height + checkpointConfirmations
	if block.Height() < requiredHeight {
		return nil, fmt.Errorf("the block database is only at height "+
			"%d which is less than the latest checkpoint height "+
			"of %d plus required confirmations of %d",
			block.Height(), latestCheckpoint.Height,
			checkpointConfirmations)
	}

	// For the first checkpoint, the required height is any block after the
	// genesis block, so long as the chain has at least the required number
	// of confirmations (which is enforced above).
	if len(activeNetParams.Checkpoints) == 0 {
		requiredHeight = 1
	}

	// Indeterminate progress setup.
	numBlocksToTest := block.Height() - requiredHeight
	progressInterval := (numBlocksToTest / 100) + 1 // min 1
	fmt.Print("Searching for candidates")
	defer fmt.Println()

	// Loop backwards through the chain to find checkpoint candidates.
	candidates := make([]*chaincfg.Checkpoint, 0, cfg.NumCandidates)
	numTested := int64(0)
	for len(candidates) < cfg.NumCandidates && block.Height() > requiredHeight {
		// Display progress.
		if numTested%progressInterval == 0 {
			fmt.Print(".")
		}

		// Determine if this block is a checkpoint candidate.
		isCandidate, err := chain.IsCheckpointCandidate(block)
		if err != nil {
			return nil, err
		}

		// All checks passed, so this node seems like a reasonable
		// checkpoint candidate.
		if isCandidate {
			checkpoint := chaincfg.Checkpoint{
				Height: block.Height(),
				Hash:   block.Sha(),
			}
			candidates = append(candidates, &checkpoint)
		}

		prevHash := &block.MsgBlock().Header.PrevBlock
		block, err = db.FetchBlockBySha(prevHash)
		if err != nil {
			return nil, err
		}
		numTested++
	}
	return candidates, nil
}

// showCandidate display a checkpoint candidate using and output format
// determined by the configuration parameters.  The Go syntax output
// uses the format the chain code expects for checkpoints added to the list.
func showCandidate(candidateNum int, checkpoint *chaincfg.Checkpoint) {
	if cfg.UseGoOutput {
		fmt.Printf("Candidate %d -- {%d, newShaHashFromStr(\"%v\")},\n",
			candidateNum, checkpoint.Height, checkpoint.Hash)
		return
	}

	fmt.Printf("Candidate %d -- Height: %d, Hash: %v\n", candidateNum,
		checkpoint.Height, checkpoint.Hash)

}

func main() {
	// Load configuration and parse command line.
	tcfg, _, err := loadConfig()
	if err != nil {
		return
	}
	cfg = tcfg

	// Load the block database.
	db, err := loadBlockDB()
	if err != nil {
		fmt.Fprintln(os.Stderr, "failed to load database:", err)
		return
	}
	defer db.Close()

	// Get the latest block hash and height from the database and report
	// status.
	latestHash, height, err := db.NewestSha()
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		return
	}
	fmt.Printf("Block database loaded with block height %d\n", height)

	// Find checkpoint candidates.
	candidates, err := findCandidates(db, latestHash)
	if err != nil {
		fmt.Fprintln(os.Stderr, "Unable to identify candidates:", err)
		return
	}

	// No candidates.
	if len(candidates) == 0 {
		fmt.Println("No candidates found.")
		return
	}

	// Show the candidates.
	for i, checkpoint := range candidates {
		showCandidate(i+1, checkpoint)
	}
}
<|endoftext|>


<commit_before><commit_msg>Delete checkpointerdouble.go<commit_after><|endoftext|>


<commit_before>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package version

// Base version information.
//
// This is the fallback data used when version information from git is not
// provided via go ldflags. It provides an approximation of the Kubernetes
// version for ad-hoc builds (e.g. `go build`) that cannot get the version
// information from git.
//
// If you are looking at these fields in the git tree, they look
// strange. They are modified on the fly by the build process. The
// in-tree values are dummy values used for "git archive", which also
// works for GitHub tar downloads.
//
// When releasing a new Kubernetes version, this file is updated by
// build/mark_new_version.sh to reflect the new version, and then a
// git annotated tag (using format vX.Y where X == Major version and Y
// == Minor version) is created to point to the commit that updates
// pkg/version/base.go
var (
	// TODO: Deprecate gitMajor and gitMinor, use only gitVersion
	// instead. First step in deprecation, keep the fields but make
	// them irrelevant. (Next we'll take it out, which may muck with
	// scripts consuming the kubectl version output - but most of
	// these should be looking at gitVersion already anyways.)
	gitMajor string = "1"  // major version, always numeric
	gitMinor string = "6+" // minor version, numeric possibly followed by "+"

	// semantic version, derived by build scripts (see
	// https://github.com/kubernetes/kubernetes/blob/master/docs/design/versioning.md
	// for a detailed discussion of this field)
	//
	// TODO: This field is still called "gitVersion" for legacy
	// reasons. For prerelease versions, the build metadata on the
	// semantic version is a git hash, but the version itself is no
	// longer the direct output of "git describe", but a slight
	// translation to be semver compliant.
	gitVersion   string = "v1.6.0-beta.2+$Format:%h$"
	gitCommit    string = "$Format:%H$"    // sha1 from git, output of $(git rev-parse HEAD)
	gitTreeState string = "not a git tree" // state of git tree, either "clean" or "dirty"

	buildDate string = "1970-01-01T00:00:00Z" // build date in ISO8601 format, output of $(date -u +'%Y-%m-%dT%H:%M:%SZ')
)
<commit_msg>update-all.sh.<commit_after>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package version

// Base version information.
//
// This is the fallback data used when version information from git is not
// provided via go ldflags. It provides an approximation of the Kubernetes
// version for ad-hoc builds (e.g. `go build`) that cannot get the version
// information from git.
//
// If you are looking at these fields in the git tree, they look
// strange. They are modified on the fly by the build process. The
// in-tree values are dummy values used for "git archive", which also
// works for GitHub tar downloads.
//
// When releasing a new Kubernetes version, this file is updated by
// build/mark_new_version.sh to reflect the new version, and then a
// git annotated tag (using format vX.Y where X == Major version and Y
// == Minor version) is created to point to the commit that updates
// pkg/version/base.go
var (
	// TODO: Deprecate gitMajor and gitMinor, use only gitVersion
	// instead. First step in deprecation, keep the fields but make
	// them irrelevant. (Next we'll take it out, which may muck with
	// scripts consuming the kubectl version output - but most of
	// these should be looking at gitVersion already anyways.)
	gitMajor string = "1"  // major version, always numeric
	gitMinor string = "6+" // minor version, numeric possibly followed by "+"

	// semantic version, derived by build scripts (see
	// https://github.com/kubernetes/kubernetes/blob/master/docs/design/versioning.md
	// for a detailed discussion of this field)
	//
	// TODO: This field is still called "gitVersion" for legacy
	// reasons. For prerelease versions, the build metadata on the
	// semantic version is a git hash, but the version itself is no
	// longer the direct output of "git describe", but a slight
	// translation to be semver compliant.
	gitVersion   string = "v1.6.0-beta.3+$Format:%h$"
	gitCommit    string = "$Format:%H$"    // sha1 from git, output of $(git rev-parse HEAD)
	gitTreeState string = "not a git tree" // state of git tree, either "clean" or "dirty"

	buildDate string = "1970-01-01T00:00:00Z" // build date in ISO8601 format, output of $(date -u +'%Y-%m-%dT%H:%M:%SZ')
)
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package sinks

import (
	"fmt"
	"strings"

	sink_api_old "github.com/GoogleCloudPlatform/heapster/sinks/api"
	sink_api "github.com/GoogleCloudPlatform/heapster/sinks/api/v1"
	source_api "github.com/GoogleCloudPlatform/heapster/sources/api"
	"github.com/golang/glog"
)

type externalSinkManager struct {
	decoder       sink_api_old.Decoder
	externalSinks []sink_api.ExternalSink
}

// NewExternalSinkManager returns an external sink manager that will manage pushing data to all
// the sinks in 'externalSinks', which is a map of sink name to ExternalSink object.
func NewExternalSinkManager(externalSinks []sink_api.ExternalSink) (ExternalSinkManager, error) {
	// Get supported metrics.
	supportedMetrics := sink_api.SupportedStatMetrics()
	for i := range supportedMetrics {
		supportedMetrics[i].Labels = sink_api.SupportedLabels()
	}

	// Create the metrics.
	descriptors := make([]sink_api.MetricDescriptor, 0, len(supportedMetrics))
	for _, supported := range supportedMetrics {
		descriptors = append(descriptors, supported.MetricDescriptor)
	}

	for _, externalSink := range externalSinks {
		err := externalSink.Register(descriptors)
		if err != nil {
			return nil, err
		}
	}
	decoder := sink_api_old.NewDecoder()
	return &externalSinkManager{
		externalSinks: externalSinks,
		decoder:       decoder,
	}, nil
}

// TODO(vmarmol): Paralellize this.
func (self *externalSinkManager) Store(input interface{}) error {
	data, ok := input.(source_api.AggregateData)
	if !ok {
		return fmt.Errorf("unknown input type %T", input)
	}
	// TODO: Store data in cache.
	timeseries, err := self.decoder.Timeseries(data)
	if err != nil {
		return err
	}
	// Format metrics and push them.
	errorsChan := make(chan error, len(self.externalSinks))
	for idx := range self.externalSinks {
		sink := self.externalSinks[idx]
		go func(sink sink_api.ExternalSink) {
			glog.V(2).Infof("Storing Timeseries to %q", sink.Name())
			errorsChan <- sink.StoreTimeseries(timeseries)
		}(sink)
		go func(sink sink_api.ExternalSink) {
			glog.V(2).Infof("Storing Events to %q", sink.Name())
			errorsChan <- sink.StoreEvents(data.Events)
		}(sink)
	}
	var errors []string
	for i := 1; i <= len(errorsChan); i++ {
		if err := <-errorsChan; err != nil {
			errors = append(errors, fmt.Sprintf("%v ", err))
		}
	}
	err = nil
	if len(errors) > 0 {
		err = fmt.Errorf("encountered the following errors: %s", strings.Join(errors, ";\n"))
	}

	return err
}

func (self *externalSinkManager) DebugInfo() string {
	desc := "External Sinks\n"

	// Add metrics being exported.
	desc += "\tExported metrics:\n"
	for _, supported := range sink_api.SupportedStatMetrics() {
		desc += fmt.Sprintf("\t\t%s: %s", supported.Name, supported.Description)
	}

	// Add labels being used.
	desc += "\n\tExported labels:\n"
	for _, label := range sink_api.SupportedLabels() {
		desc += fmt.Sprintf("\t\t%s: %s", label.Key, label.Description)
	}
	desc += "\n\tExternal Sinks:\n"
	for _, externalSink := range self.externalSinks {
		desc += fmt.Sprintf("\n\t\t%s", externalSink.DebugInfo())
	}

	return desc
}
<commit_msg>We're actually getting two errors per sink<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package sinks

import (
	"fmt"
	"strings"

	sink_api_old "github.com/GoogleCloudPlatform/heapster/sinks/api"
	sink_api "github.com/GoogleCloudPlatform/heapster/sinks/api/v1"
	source_api "github.com/GoogleCloudPlatform/heapster/sources/api"
	"github.com/golang/glog"
)

type externalSinkManager struct {
	decoder       sink_api_old.Decoder
	externalSinks []sink_api.ExternalSink
}

// NewExternalSinkManager returns an external sink manager that will manage pushing data to all
// the sinks in 'externalSinks', which is a map of sink name to ExternalSink object.
func NewExternalSinkManager(externalSinks []sink_api.ExternalSink) (ExternalSinkManager, error) {
	// Get supported metrics.
	supportedMetrics := sink_api.SupportedStatMetrics()
	for i := range supportedMetrics {
		supportedMetrics[i].Labels = sink_api.SupportedLabels()
	}

	// Create the metrics.
	descriptors := make([]sink_api.MetricDescriptor, 0, len(supportedMetrics))
	for _, supported := range supportedMetrics {
		descriptors = append(descriptors, supported.MetricDescriptor)
	}

	for _, externalSink := range externalSinks {
		err := externalSink.Register(descriptors)
		if err != nil {
			return nil, err
		}
	}
	decoder := sink_api_old.NewDecoder()
	return &externalSinkManager{
		externalSinks: externalSinks,
		decoder:       decoder,
	}, nil
}

// TODO(vmarmol): Paralellize this.
func (self *externalSinkManager) Store(input interface{}) error {
	data, ok := input.(source_api.AggregateData)
	if !ok {
		return fmt.Errorf("unknown input type %T", input)
	}
	// TODO: Store data in cache.
	timeseries, err := self.decoder.Timeseries(data)
	if err != nil {
		return err
	}
	// Format metrics and push them.
	errorsChan := make(chan error, 2*len(self.externalSinks))
	for idx := range self.externalSinks {
		sink := self.externalSinks[idx]
		go func(sink sink_api.ExternalSink) {
			glog.V(2).Infof("Storing Timeseries to %q", sink.Name())
			errorsChan <- sink.StoreTimeseries(timeseries)
		}(sink)
		go func(sink sink_api.ExternalSink) {
			glog.V(2).Infof("Storing Events to %q", sink.Name())
			errorsChan <- sink.StoreEvents(data.Events)
		}(sink)
	}
	var errors []string
	for i := 1; i <= len(errorsChan); i++ {
		if err := <-errorsChan; err != nil {
			errors = append(errors, fmt.Sprintf("%v ", err))
		}
	}
	err = nil
	if len(errors) > 0 {
		err = fmt.Errorf("encountered the following errors: %s", strings.Join(errors, ";\n"))
	}

	return err
}

func (self *externalSinkManager) DebugInfo() string {
	desc := "External Sinks\n"

	// Add metrics being exported.
	desc += "\tExported metrics:\n"
	for _, supported := range sink_api.SupportedStatMetrics() {
		desc += fmt.Sprintf("\t\t%s: %s", supported.Name, supported.Description)
	}

	// Add labels being used.
	desc += "\n\tExported labels:\n"
	for _, label := range sink_api.SupportedLabels() {
		desc += fmt.Sprintf("\t\t%s: %s", label.Key, label.Description)
	}
	desc += "\n\tExternal Sinks:\n"
	for _, externalSink := range self.externalSinks {
		desc += fmt.Sprintf("\n\t\t%s", externalSink.DebugInfo())
	}

	return desc
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	toml "github.com/BurntSushi/toml"
	"github.com/direnv/direnv/v2/xdg"
)

// Config represents the direnv configuration and state.
type Config struct {
	Env             Env
	WorkDir         string // Current directory
	ConfDir         string
	CacheDir        string
	DataDir         string
	SelfPath        string
	BashPath        string
	RCFile          string
	TomlPath        string
	DisableStdin    bool
	StrictEnv       bool
	LoadDotenv      bool
	WarnTimeout     time.Duration
	WhitelistPrefix []string
	WhitelistExact  map[string]bool
}

type tomlDuration struct {
	time.Duration
}

func (d *tomlDuration) UnmarshalText(text []byte) error {
	var err error
	d.Duration, err = time.ParseDuration(string(text))
	return err
}

type tomlConfig struct {
	*tomlGlobal               // For backward-compatibility
	Global      *tomlGlobal   `toml:"global"`
	Whitelist   tomlWhitelist `toml:"whitelist"`
}

type tomlGlobal struct {
	BashPath     string       `toml:"bash_path"`
	DisableStdin bool         `toml:"disable_stdin"`
	StrictEnv    bool         `toml:"strict_env"`
	SkipDotenv   bool         `toml:"skip_dotenv"` // deprecated, use load_dotenv
	LoadDotenv   bool         `toml:"load_dotenv"`
	WarnTimeout  tomlDuration `toml:"warn_timeout"`
}

type tomlWhitelist struct {
	Prefix []string
	Exact  []string
}

// LoadConfig opens up the direnv configuration from the Env.
func LoadConfig(env Env) (config *Config, err error) {
	config = &Config{
		Env: env,
	}

	config.ConfDir = env[DIRENV_CONFIG]
	if config.ConfDir == "" {
		config.ConfDir = xdg.ConfigDir(env, "direnv")
	}
	if config.ConfDir == "" {
		err = fmt.Errorf("couldn't find a configuration directory for direnv")
		return
	}

	var exePath string
	if exePath, err = os.Executable(); err != nil {
		err = fmt.Errorf("LoadConfig() os.Executable() failed: %w", err)
		return
	}
	// Fix for mingsys
	exePath = strings.Replace(exePath, "\\", "/", -1)
	config.SelfPath = exePath

	if config.WorkDir, err = os.Getwd(); err != nil {
		err = fmt.Errorf("LoadConfig() Getwd failed: %w", err)
		return
	}

	config.RCFile = env[DIRENV_FILE]

	config.WhitelistPrefix = make([]string, 0)
	config.WhitelistExact = make(map[string]bool)

	// Load the TOML config
	config.TomlPath = filepath.Join(config.ConfDir, "direnv.toml")
	if _, statErr := os.Stat(config.TomlPath); statErr != nil {
		config.TomlPath = filepath.Join(config.ConfDir, "config.toml")
		if _, statErr := os.Stat(config.TomlPath); statErr != nil {
			config.TomlPath = ""
		}
	}

	if config.TomlPath != "" {
		// Declare global once and then share it between the top-level and Global
		// keys. The goal here is to let the decoder fill global regardless of if
		// the values are in the [global] section or not. The reason we do that is
		// to keep backward-compatibility with the old top-level notation.
		var global tomlGlobal
		tomlConf := tomlConfig{
			tomlGlobal: &global,
			Global:     &global,
		}
		if _, err = toml.DecodeFile(config.TomlPath, &tomlConf); err != nil {
			err = fmt.Errorf("LoadConfig() failed to parse %s: %w", config.TomlPath, err)
			return
		}

		config.WhitelistPrefix = append(config.WhitelistPrefix, tomlConf.Whitelist.Prefix...)

		for _, path := range tomlConf.Whitelist.Exact {
			if !(strings.HasSuffix(path, "/.envrc") || strings.HasSuffix(path, "/.env")) {
				path = filepath.Join(path, ".envrc")
			}

			config.WhitelistExact[path] = true
		}

		if tomlConf.SkipDotenv {
			logError("skip_dotenv has been inverted to load_dotenv.")
		}

		config.BashPath = tomlConf.BashPath
		config.DisableStdin = tomlConf.DisableStdin
		config.LoadDotenv = tomlConf.LoadDotenv
		config.StrictEnv = tomlConf.StrictEnv
		config.WarnTimeout = tomlConf.WarnTimeout.Duration
	}

	if config.WarnTimeout == 0 {
		timeout, err := time.ParseDuration(env.Fetch("DIRENV_WARN_TIMEOUT", "5s"))
		if err != nil {
			logError("invalid DIRENV_WARN_TIMEOUT: " + err.Error())
			timeout = 5 * time.Second
		}
		config.WarnTimeout = timeout
	}

	if config.BashPath == "" {
		if env[DIRENV_BASH] != "" {
			config.BashPath = env[DIRENV_BASH]
		} else if bashPath != "" {
			config.BashPath = bashPath
		} else if config.BashPath, err = exec.LookPath("bash"); err != nil {
			err = fmt.Errorf("can't find bash: %w", err)
			return
		}
	}

	if config.CacheDir == "" {
		config.CacheDir = xdg.CacheDir(env, "direnv")
	}
	if config.CacheDir == "" {
		err = fmt.Errorf("couldn't find a cache directory for direnv")
		return
	}

	if config.DataDir == "" {
		config.DataDir = xdg.DataDir(env, "direnv")
	}
	if config.DataDir == "" {
		err = fmt.Errorf("couldn't find a data directory for direnv")
		return
	}

	return
}

// AllowDir is the folder where all the "allow" files are stored.
func (config *Config) AllowDir() string {
	return filepath.Join(config.DataDir, "allow")
}

// LoadedRC returns a RC file if any has been loaded
func (config *Config) LoadedRC() *RC {
	if config.RCFile == "" {
		logDebug("RCFile is blank - loadedRC is nil")
		return nil
	}
	rcPath := config.RCFile

	timesString := config.Env[DIRENV_WATCHES]

	return RCFromEnv(rcPath, timesString, config)
}

// EnvFromRC loads an RC from a specified path and returns the new environment
func (config *Config) EnvFromRC(path string, previousEnv Env) (Env, error) {
	rc, err := RCFromPath(path, config)
	if err != nil {
		return nil, err
	}
	return rc.Load(previousEnv)
}

// FindRC looks for a RC file in the config environment
func (config *Config) FindRC() (*RC, error) {
	return FindRC(config.WorkDir, config)
}

// Revert undoes the recorded changes (if any) to the supplied environment,
// returning a new environment
func (config *Config) Revert(env Env) (Env, error) {
	if config.Env[DIRENV_DIFF] == "" {
		return env.Copy(), nil
	}
	diff, err := LoadEnvDiff(config.Env[DIRENV_DIFF])
	if err == nil {
		return diff.Reverse().Patch(env), nil
	}
	return nil, err
}
<commit_msg>Expand ~/ in whitelist paths (#931)<commit_after>package cmd

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	toml "github.com/BurntSushi/toml"
	"github.com/direnv/direnv/v2/xdg"
)

// Config represents the direnv configuration and state.
type Config struct {
	Env             Env
	WorkDir         string // Current directory
	ConfDir         string
	CacheDir        string
	DataDir         string
	SelfPath        string
	BashPath        string
	RCFile          string
	TomlPath        string
	DisableStdin    bool
	StrictEnv       bool
	LoadDotenv      bool
	WarnTimeout     time.Duration
	WhitelistPrefix []string
	WhitelistExact  map[string]bool
}

type tomlDuration struct {
	time.Duration
}

func (d *tomlDuration) UnmarshalText(text []byte) error {
	var err error
	d.Duration, err = time.ParseDuration(string(text))
	return err
}

type tomlConfig struct {
	*tomlGlobal               // For backward-compatibility
	Global      *tomlGlobal   `toml:"global"`
	Whitelist   tomlWhitelist `toml:"whitelist"`
}

type tomlGlobal struct {
	BashPath     string       `toml:"bash_path"`
	DisableStdin bool         `toml:"disable_stdin"`
	StrictEnv    bool         `toml:"strict_env"`
	SkipDotenv   bool         `toml:"skip_dotenv"` // deprecated, use load_dotenv
	LoadDotenv   bool         `toml:"load_dotenv"`
	WarnTimeout  tomlDuration `toml:"warn_timeout"`
}

type tomlWhitelist struct {
	Prefix []string
	Exact  []string
}

// Expand a path string prefixed with ~/ to the current user's home directory.
// Example: if current user is user1 with home directory in /home/user1, then
// ~/project -> /home/user1/project
// It's useful to allow paths with ~/, so that direnv.toml can be reused via
// dotfiles repos across systems with different standard home paths
// (compare Linux /home and macOS /Users).
func expandTildePath(path string) (pathExpanded string) {
	pathExpanded = path
	if strings.HasPrefix(path, "~/") {
		if homedir, homedirErr := os.UserHomeDir(); homedirErr == nil {
			pathExpanded = filepath.Join(homedir, path[2:])
		}
	}
	return pathExpanded
}

// LoadConfig opens up the direnv configuration from the Env.
func LoadConfig(env Env) (config *Config, err error) {
	config = &Config{
		Env: env,
	}

	config.ConfDir = env[DIRENV_CONFIG]
	if config.ConfDir == "" {
		config.ConfDir = xdg.ConfigDir(env, "direnv")
	}
	if config.ConfDir == "" {
		err = fmt.Errorf("couldn't find a configuration directory for direnv")
		return
	}

	var exePath string
	if exePath, err = os.Executable(); err != nil {
		err = fmt.Errorf("LoadConfig() os.Executable() failed: %w", err)
		return
	}
	// Fix for mingsys
	exePath = strings.Replace(exePath, "\\", "/", -1)
	config.SelfPath = exePath

	if config.WorkDir, err = os.Getwd(); err != nil {
		err = fmt.Errorf("LoadConfig() Getwd failed: %w", err)
		return
	}

	config.RCFile = env[DIRENV_FILE]

	config.WhitelistPrefix = make([]string, 0)
	config.WhitelistExact = make(map[string]bool)

	// Load the TOML config
	config.TomlPath = filepath.Join(config.ConfDir, "direnv.toml")
	if _, statErr := os.Stat(config.TomlPath); statErr != nil {
		config.TomlPath = filepath.Join(config.ConfDir, "config.toml")
		if _, statErr := os.Stat(config.TomlPath); statErr != nil {
			config.TomlPath = ""
		}
	}

	if config.TomlPath != "" {
		// Declare global once and then share it between the top-level and Global
		// keys. The goal here is to let the decoder fill global regardless of if
		// the values are in the [global] section or not. The reason we do that is
		// to keep backward-compatibility with the old top-level notation.
		var global tomlGlobal
		tomlConf := tomlConfig{
			tomlGlobal: &global,
			Global:     &global,
		}
		if _, err = toml.DecodeFile(config.TomlPath, &tomlConf); err != nil {
			err = fmt.Errorf("LoadConfig() failed to parse %s: %w", config.TomlPath, err)
			return
		}

		for _, path := range tomlConf.Whitelist.Prefix {
			config.WhitelistPrefix = append(config.WhitelistPrefix, expandTildePath(path))
		}

		for _, path := range tomlConf.Whitelist.Exact {
			if !(strings.HasSuffix(path, "/.envrc") || strings.HasSuffix(path, "/.env")) {
				path = filepath.Join(path, ".envrc")
			}

			config.WhitelistExact[expandTildePath(path)] = true
		}

		if tomlConf.SkipDotenv {
			logError("skip_dotenv has been inverted to load_dotenv.")
		}

		config.BashPath = tomlConf.BashPath
		config.DisableStdin = tomlConf.DisableStdin
		config.LoadDotenv = tomlConf.LoadDotenv
		config.StrictEnv = tomlConf.StrictEnv
		config.WarnTimeout = tomlConf.WarnTimeout.Duration
	}

	if config.WarnTimeout == 0 {
		timeout, err := time.ParseDuration(env.Fetch("DIRENV_WARN_TIMEOUT", "5s"))
		if err != nil {
			logError("invalid DIRENV_WARN_TIMEOUT: " + err.Error())
			timeout = 5 * time.Second
		}
		config.WarnTimeout = timeout
	}

	if config.BashPath == "" {
		if env[DIRENV_BASH] != "" {
			config.BashPath = env[DIRENV_BASH]
		} else if bashPath != "" {
			config.BashPath = bashPath
		} else if config.BashPath, err = exec.LookPath("bash"); err != nil {
			err = fmt.Errorf("can't find bash: %w", err)
			return
		}
	}

	if config.CacheDir == "" {
		config.CacheDir = xdg.CacheDir(env, "direnv")
	}
	if config.CacheDir == "" {
		err = fmt.Errorf("couldn't find a cache directory for direnv")
		return
	}

	if config.DataDir == "" {
		config.DataDir = xdg.DataDir(env, "direnv")
	}
	if config.DataDir == "" {
		err = fmt.Errorf("couldn't find a data directory for direnv")
		return
	}

	return
}

// AllowDir is the folder where all the "allow" files are stored.
func (config *Config) AllowDir() string {
	return filepath.Join(config.DataDir, "allow")
}

// LoadedRC returns a RC file if any has been loaded
func (config *Config) LoadedRC() *RC {
	if config.RCFile == "" {
		logDebug("RCFile is blank - loadedRC is nil")
		return nil
	}
	rcPath := config.RCFile

	timesString := config.Env[DIRENV_WATCHES]

	return RCFromEnv(rcPath, timesString, config)
}

// EnvFromRC loads an RC from a specified path and returns the new environment
func (config *Config) EnvFromRC(path string, previousEnv Env) (Env, error) {
	rc, err := RCFromPath(path, config)
	if err != nil {
		return nil, err
	}
	return rc.Load(previousEnv)
}

// FindRC looks for a RC file in the config environment
func (config *Config) FindRC() (*RC, error) {
	return FindRC(config.WorkDir, config)
}

// Revert undoes the recorded changes (if any) to the supplied environment,
// returning a new environment
func (config *Config) Revert(env Env) (Env, error) {
	if config.Env[DIRENV_DIFF] == "" {
		return env.Copy(), nil
	}
	diff, err := LoadEnvDiff(config.Env[DIRENV_DIFF])
	if err == nil {
		return diff.Reverse().Patch(env), nil
	}
	return nil, err
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Ceph-CSI Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package util

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/hashicorp/vault/api"
	loss "github.com/libopenstorage/secrets"
	"github.com/libopenstorage/secrets/vault"
)

const (
	// path to service account token that will be used to authenticate with Vault
	// #nosec
	serviceAccountTokenPath = "/var/run/secrets/kubernetes.io/serviceaccount/token"

	// vault configuration defaults
	vaultDefaultAuthPath       = "/v1/auth/kubernetes/login"
	vaultDefaultRole           = "csi-kubernetes"
	vaultDefaultNamespace      = ""
	vaultDefaultPassphrasePath = ""
)

/*
VaultKMS represents a Hashicorp Vault KMS configuration

Example JSON structure in the KMS config is,
{
	"local_vault_unique_identifier": {
		"encryptionKMSType": "vault",
		"vaultAddress": "https://127.0.0.1:8500",
		"vaultAuthPath": "/v1/auth/kubernetes/login",
		"vaultRole": "csi-kubernetes",
		"vaultNamespace": "",
		"vaultPassphraseRoot": "/v1/secret",
		"vaultPassphrasePath": "",
		"vaultCAVerify": true,
		"vaultCAFromSecret": "vault-ca"
	},
	...
}.
*/
type VaultKMS struct {
	EncryptionKMSID string

	// vaultPassphrasePath (VPP) used to be added before the "key" of the
	// secret (like /v1/secret/data/<VPP>/key)
	vaultPassphrasePath string

	secrets    loss.Secrets
	keyContext map[string]string
}

// InitVaultKMS returns an interface to HashiCorp Vault KMS.
//
// nolint:gocyclo // this is a long function, as it constructs the Vault config
func InitVaultKMS(kmsID string, config, secrets map[string]string) (EncryptionKMS, error) {
	var (
		ok  bool
		err error
	)

	vaultConfig := make(map[string]interface{})
	keyContext := make(map[string]string)

	kms := &VaultKMS{}
	kms.EncryptionKMSID = kmsID

	vaultAddress, ok := config["vaultAddress"]
	if !ok || vaultAddress == "" {
		return nil, fmt.Errorf("missing 'vaultAddress' for vault KMS %s", kmsID)
	}
	vaultConfig[api.EnvVaultAddress] = vaultAddress

	vaultAuthPath, ok := config["vaultAuthPath"]
	if !ok || vaultAuthPath == "" {
		vaultAuthPath = vaultDefaultAuthPath
	}
	vaultConfig[vault.AuthMountPath], err = detectAuthMountPath(vaultAuthPath)
	if err != nil {
		return nil, fmt.Errorf("failed to set %s in Vault config: %w", vault.AuthMountPath, err)
	}

	vaultRole, ok := config["vaultRole"]
	if !ok || vaultRole == "" {
		vaultRole = vaultDefaultRole
	}
	vaultConfig[vault.AuthKubernetesRole] = vaultRole

	vaultNamespace, ok := config["vaultNamespace"]
	if !ok || vaultNamespace == "" {
		vaultNamespace = vaultDefaultNamespace
	}
	vaultConfig[api.EnvVaultNamespace] = vaultNamespace
	keyContext[loss.KeyVaultNamespace] = vaultNamespace

	// vault.VaultBackendPathKey is "secret/" by default, use vaultPassphraseRoot if configured
	vaultPassphraseRoot, ok := config["vaultPassphraseRoot"]
	if ok && vaultPassphraseRoot != "" {
		// the old example did have "/v1/secret/", convert that format
		if strings.HasPrefix(vaultPassphraseRoot, "/v1/") {
			vaultConfig[vault.VaultBackendPathKey] = strings.TrimPrefix(vaultPassphraseRoot, "/v1/")
		} else {
			vaultConfig[vault.VaultBackendPathKey] = vaultPassphraseRoot
		}
	}

	kms.vaultPassphrasePath, ok = config["vaultPassphrasePath"]
	if !ok || kms.vaultPassphrasePath == "" {
		kms.vaultPassphrasePath = vaultDefaultPassphrasePath
	}

	verifyCA, ok := config["vaultCAVerify"]
	if ok {
		var vaultCAVerify bool
		vaultCAVerify, err = strconv.ParseBool(verifyCA)
		if err != nil {
			return nil, fmt.Errorf("failed to parse 'vaultCAVerify' for vault <%s> kms config: %s",
				kmsID, err)
		}
		vaultConfig[api.EnvVaultInsecure] = !vaultCAVerify
	}

	vaultCAFromSecret, ok := config["vaultCAFromSecret"]
	if ok && vaultCAFromSecret != "" {
		caPEM, ok := secrets[vaultCAFromSecret]
		if !ok {
			return nil, fmt.Errorf("missing vault CA in secret %s", vaultCAFromSecret)
		}
		vaultConfig[api.EnvVaultCACert], err = createTempFile("vault-ca-cert", []byte(caPEM))
		if err != nil {
			return nil, fmt.Errorf("failed to create temporary file for Vault CA: %w", err)
		}
		// TODO: delete f.Name() when VaultKMS is destroyed
	}

	// FIXME: vault.AuthKubernetesTokenPath is not enough? EnvVaultToken needs to be set?
	vaultConfig[vault.AuthMethod] = vault.AuthMethodKubernetes
	vaultConfig[vault.AuthKubernetesTokenPath] = serviceAccountTokenPath

	v, err := vault.New(vaultConfig)
	if err != nil {
		return nil, fmt.Errorf("failed creating new Vault Secrets: %w", err)
	}
	kms.secrets = v

	kms.keyContext = keyContext

	return kms, nil
}

// GetID is returning correlation ID to KMS configuration.
func (kms *VaultKMS) GetID() string {
	return kms.EncryptionKMSID
}

// GetPassphrase returns passphrase from Vault. The passphrase is stored in a
// data.data.passphrase structure.
func (kms *VaultKMS) GetPassphrase(key string) (string, error) {
	s, err := kms.secrets.GetSecret(filepath.Join(kms.vaultPassphrasePath, key), kms.keyContext)
	if errors.Is(err, loss.ErrInvalidSecretId) {
		return "", MissingPassphrase{err}
	} else if err != nil {
		return "", err
	}

	data, ok := s["data"].(map[string]interface{})
	if !ok {
		return "", fmt.Errorf("failed parsing data for get passphrase request for %s", key)
	}
	passphrase, ok := data["passphrase"].(string)
	if !ok {
		return "", fmt.Errorf("failed parsing passphrase for get passphrase request for %s", key)
	}

	return passphrase, nil
}

// SavePassphrase saves new passphrase in Vault.
func (kms *VaultKMS) SavePassphrase(key, value string) error {
	data := map[string]interface{}{
		"data": map[string]string{
			"passphrase": value,
		},
	}

	pathKey := filepath.Join(kms.vaultPassphrasePath, key)
	err := kms.secrets.PutSecret(pathKey, data, kms.keyContext)
	if err != nil {
		return fmt.Errorf("saving passphrase at %s request to vault failed: %w", pathKey, err)
	}

	return nil
}

// DeletePassphrase deletes passphrase from Vault.
func (kms *VaultKMS) DeletePassphrase(key string) error {
	pathKey := filepath.Join(kms.vaultPassphrasePath, key)
	err := kms.secrets.DeleteSecret(pathKey, kms.keyContext)
	if err != nil {
		return fmt.Errorf("delete passphrase at %s request to vault failed: %w", pathKey, err)
	}

	return nil
}

// detectAuthMountPath takes the vaultAuthPath configuration option that
// defaults to "/v1/auth/kubernetes/login" and makes it a vault.AuthMountPath
// like "kubernetes".
func detectAuthMountPath(path string) (string, error) {
	var authMountPath string

	if path == "" {
		return "", errors.New("path is empty")
	}

	// add all components betweed "login" and "auth" to authMountPath
	match := false
	parts := strings.Split(path, "/")
	for _, part := range parts {
		if part == "auth" {
			match = true
			continue
		}
		if part == "login" {
			break
		}
		if match && authMountPath == "" {
			authMountPath = part
		} else if match {
			authMountPath += "/" + part
		}
	}

	// in case authMountPath is empty, return original path as it was
	if authMountPath == "" {
		authMountPath = path
	}

	return authMountPath, nil
}

// createTempFile writes data to a temporary file that contains the pattern in
// the filename (see ioutil.TempFile for details).
func createTempFile(pattern string, data []byte) (string, error) {
	t, err := ioutil.TempFile("", pattern)
	if err != nil {
		return "", fmt.Errorf("failed to create temporary file: %w", err)
	}

	// delete the tmpfile on error
	defer func() {
		if err != nil {
			// ignore error on failure to remove tmpfile (gosec complains)
			_ = os.Remove(t.Name())
		}
	}()

	s, err := t.Write(data)
	if err != nil || s != len(data) {
		return "", fmt.Errorf("failed to write temporary file: %w", err)
	}
	err = t.Close()
	if err != nil {
		return "", fmt.Errorf("failed to close temporary file: %w", err)
	}

	return t.Name(), nil
}
<commit_msg>util: use %w when forwarding an error<commit_after>/*
Copyright 2019 The Ceph-CSI Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package util

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/hashicorp/vault/api"
	loss "github.com/libopenstorage/secrets"
	"github.com/libopenstorage/secrets/vault"
)

const (
	// path to service account token that will be used to authenticate with Vault
	// #nosec
	serviceAccountTokenPath = "/var/run/secrets/kubernetes.io/serviceaccount/token"

	// vault configuration defaults
	vaultDefaultAuthPath       = "/v1/auth/kubernetes/login"
	vaultDefaultRole           = "csi-kubernetes"
	vaultDefaultNamespace      = ""
	vaultDefaultPassphrasePath = ""
)

/*
VaultKMS represents a Hashicorp Vault KMS configuration

Example JSON structure in the KMS config is,
{
	"local_vault_unique_identifier": {
		"encryptionKMSType": "vault",
		"vaultAddress": "https://127.0.0.1:8500",
		"vaultAuthPath": "/v1/auth/kubernetes/login",
		"vaultRole": "csi-kubernetes",
		"vaultNamespace": "",
		"vaultPassphraseRoot": "/v1/secret",
		"vaultPassphrasePath": "",
		"vaultCAVerify": true,
		"vaultCAFromSecret": "vault-ca"
	},
	...
}.
*/
type VaultKMS struct {
	EncryptionKMSID string

	// vaultPassphrasePath (VPP) used to be added before the "key" of the
	// secret (like /v1/secret/data/<VPP>/key)
	vaultPassphrasePath string

	secrets    loss.Secrets
	keyContext map[string]string
}

// InitVaultKMS returns an interface to HashiCorp Vault KMS.
//
// nolint:gocyclo // this is a long function, as it constructs the Vault config
func InitVaultKMS(kmsID string, config, secrets map[string]string) (EncryptionKMS, error) {
	var (
		ok  bool
		err error
	)

	vaultConfig := make(map[string]interface{})
	keyContext := make(map[string]string)

	kms := &VaultKMS{}
	kms.EncryptionKMSID = kmsID

	vaultAddress, ok := config["vaultAddress"]
	if !ok || vaultAddress == "" {
		return nil, fmt.Errorf("missing 'vaultAddress' for vault KMS %s", kmsID)
	}
	vaultConfig[api.EnvVaultAddress] = vaultAddress

	vaultAuthPath, ok := config["vaultAuthPath"]
	if !ok || vaultAuthPath == "" {
		vaultAuthPath = vaultDefaultAuthPath
	}
	vaultConfig[vault.AuthMountPath], err = detectAuthMountPath(vaultAuthPath)
	if err != nil {
		return nil, fmt.Errorf("failed to set %s in Vault config: %w", vault.AuthMountPath, err)
	}

	vaultRole, ok := config["vaultRole"]
	if !ok || vaultRole == "" {
		vaultRole = vaultDefaultRole
	}
	vaultConfig[vault.AuthKubernetesRole] = vaultRole

	vaultNamespace, ok := config["vaultNamespace"]
	if !ok || vaultNamespace == "" {
		vaultNamespace = vaultDefaultNamespace
	}
	vaultConfig[api.EnvVaultNamespace] = vaultNamespace
	keyContext[loss.KeyVaultNamespace] = vaultNamespace

	// vault.VaultBackendPathKey is "secret/" by default, use vaultPassphraseRoot if configured
	vaultPassphraseRoot, ok := config["vaultPassphraseRoot"]
	if ok && vaultPassphraseRoot != "" {
		// the old example did have "/v1/secret/", convert that format
		if strings.HasPrefix(vaultPassphraseRoot, "/v1/") {
			vaultConfig[vault.VaultBackendPathKey] = strings.TrimPrefix(vaultPassphraseRoot, "/v1/")
		} else {
			vaultConfig[vault.VaultBackendPathKey] = vaultPassphraseRoot
		}
	}

	kms.vaultPassphrasePath, ok = config["vaultPassphrasePath"]
	if !ok || kms.vaultPassphrasePath == "" {
		kms.vaultPassphrasePath = vaultDefaultPassphrasePath
	}

	verifyCA, ok := config["vaultCAVerify"]
	if ok {
		var vaultCAVerify bool
		vaultCAVerify, err = strconv.ParseBool(verifyCA)
		if err != nil {
			return nil, fmt.Errorf("failed to parse 'vaultCAVerify' for vault <%s> kms config: %w",
				kmsID, err)
		}
		vaultConfig[api.EnvVaultInsecure] = !vaultCAVerify
	}

	vaultCAFromSecret, ok := config["vaultCAFromSecret"]
	if ok && vaultCAFromSecret != "" {
		caPEM, ok := secrets[vaultCAFromSecret]
		if !ok {
			return nil, fmt.Errorf("missing vault CA in secret %s", vaultCAFromSecret)
		}
		vaultConfig[api.EnvVaultCACert], err = createTempFile("vault-ca-cert", []byte(caPEM))
		if err != nil {
			return nil, fmt.Errorf("failed to create temporary file for Vault CA: %w", err)
		}
		// TODO: delete f.Name() when VaultKMS is destroyed
	}

	// FIXME: vault.AuthKubernetesTokenPath is not enough? EnvVaultToken needs to be set?
	vaultConfig[vault.AuthMethod] = vault.AuthMethodKubernetes
	vaultConfig[vault.AuthKubernetesTokenPath] = serviceAccountTokenPath

	v, err := vault.New(vaultConfig)
	if err != nil {
		return nil, fmt.Errorf("failed creating new Vault Secrets: %w", err)
	}
	kms.secrets = v

	kms.keyContext = keyContext

	return kms, nil
}

// GetID is returning correlation ID to KMS configuration.
func (kms *VaultKMS) GetID() string {
	return kms.EncryptionKMSID
}

// GetPassphrase returns passphrase from Vault. The passphrase is stored in a
// data.data.passphrase structure.
func (kms *VaultKMS) GetPassphrase(key string) (string, error) {
	s, err := kms.secrets.GetSecret(filepath.Join(kms.vaultPassphrasePath, key), kms.keyContext)
	if errors.Is(err, loss.ErrInvalidSecretId) {
		return "", MissingPassphrase{err}
	} else if err != nil {
		return "", err
	}

	data, ok := s["data"].(map[string]interface{})
	if !ok {
		return "", fmt.Errorf("failed parsing data for get passphrase request for %s", key)
	}
	passphrase, ok := data["passphrase"].(string)
	if !ok {
		return "", fmt.Errorf("failed parsing passphrase for get passphrase request for %s", key)
	}

	return passphrase, nil
}

// SavePassphrase saves new passphrase in Vault.
func (kms *VaultKMS) SavePassphrase(key, value string) error {
	data := map[string]interface{}{
		"data": map[string]string{
			"passphrase": value,
		},
	}

	pathKey := filepath.Join(kms.vaultPassphrasePath, key)
	err := kms.secrets.PutSecret(pathKey, data, kms.keyContext)
	if err != nil {
		return fmt.Errorf("saving passphrase at %s request to vault failed: %w", pathKey, err)
	}

	return nil
}

// DeletePassphrase deletes passphrase from Vault.
func (kms *VaultKMS) DeletePassphrase(key string) error {
	pathKey := filepath.Join(kms.vaultPassphrasePath, key)
	err := kms.secrets.DeleteSecret(pathKey, kms.keyContext)
	if err != nil {
		return fmt.Errorf("delete passphrase at %s request to vault failed: %w", pathKey, err)
	}

	return nil
}

// detectAuthMountPath takes the vaultAuthPath configuration option that
// defaults to "/v1/auth/kubernetes/login" and makes it a vault.AuthMountPath
// like "kubernetes".
func detectAuthMountPath(path string) (string, error) {
	var authMountPath string

	if path == "" {
		return "", errors.New("path is empty")
	}

	// add all components betweed "login" and "auth" to authMountPath
	match := false
	parts := strings.Split(path, "/")
	for _, part := range parts {
		if part == "auth" {
			match = true
			continue
		}
		if part == "login" {
			break
		}
		if match && authMountPath == "" {
			authMountPath = part
		} else if match {
			authMountPath += "/" + part
		}
	}

	// in case authMountPath is empty, return original path as it was
	if authMountPath == "" {
		authMountPath = path
	}

	return authMountPath, nil
}

// createTempFile writes data to a temporary file that contains the pattern in
// the filename (see ioutil.TempFile for details).
func createTempFile(pattern string, data []byte) (string, error) {
	t, err := ioutil.TempFile("", pattern)
	if err != nil {
		return "", fmt.Errorf("failed to create temporary file: %w", err)
	}

	// delete the tmpfile on error
	defer func() {
		if err != nil {
			// ignore error on failure to remove tmpfile (gosec complains)
			_ = os.Remove(t.Name())
		}
	}()

	s, err := t.Write(data)
	if err != nil || s != len(data) {
		return "", fmt.Errorf("failed to write temporary file: %w", err)
	}
	err = t.Close()
	if err != nil {
		return "", fmt.Errorf("failed to close temporary file: %w", err)
	}

	return t.Name(), nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package commands

import (
	"cocoon/db"
	"encoding/json"
	"fmt"
)

// RefreshChromebotStatusResult contains chromebot status results.
type RefreshChromebotStatusResult struct {
	Results []*ChromebotResult
}

// ChromebotResult describes a chromebot build result.
type ChromebotResult struct {
	Commit string
	State  db.TaskStatus
}

// RefreshChromebotStatus pulls down the latest chromebot builds and updates the
// corresponding task statuses.
func RefreshChromebotStatus(cocoon *db.Cocoon, _ []byte) (interface{}, error) {
	linuxResults, err := refreshChromebot(cocoon, "linux_bot", "Linux")

	if err != nil {
		return nil, err
	}

	macResults, err := refreshChromebot(cocoon, "mac_bot", "Mac")

	if err != nil {
		return nil, err
	}

	windowsResults, err := refreshChromebot(cocoon, "windows_bot", "Windows")

	if err != nil {
		return nil, err
	}

	var allResults []*ChromebotResult
	allResults = append(allResults, linuxResults...)
	allResults = append(allResults, macResults...)
	allResults = append(allResults, windowsResults...)
	return RefreshChromebotStatusResult{allResults}, nil
}

func refreshChromebot(cocoon *db.Cocoon, taskName string, builderName string) ([]*ChromebotResult, error) {
	tasks, err := cocoon.QueryLatestTasksByName(taskName)

	if err != nil {
		return nil, err
	}

	if len(tasks) == 0 {
		// Short-circuit. Don't bother fetching Chromebot data if there are no tasks to
		// to update.
		return make([]*ChromebotResult, 0), nil
	}

	buildStatuses, err := fetchChromebotBuildStatuses(cocoon, builderName)

	if err != nil {
		return nil, err
	}

	for _, fullTask := range tasks {
		for _, status := range buildStatuses {
			if status.Commit == fullTask.ChecklistEntity.Checklist.Commit.Sha {
				task := fullTask.TaskEntity.Task
				task.Status = status.State
				cocoon.PutTask(fullTask.TaskEntity.Key, task)
			}
		}
	}

	return buildStatuses, nil
}

func fetchChromebotBuildStatuses(cocoon *db.Cocoon, builderName string) ([]*ChromebotResult, error) {
	builderURL := fmt.Sprintf("https://build.chromium.org/p/client.flutter/json/builders/%v", builderName)

	jsonResponse, err := fetchJSON(cocoon, builderURL)

	if err != nil {
		return nil, err
	}

	buildIds := jsonResponse.(map[string]interface{})["cachedBuilds"].([]interface{})

	if len(buildIds) > 10 {
		// Build IDs are sorted in ascending order, to get the latest 10 builds we
		// grab the tail.
		buildIds = buildIds[len(buildIds)-10:]
	}

	var results []*ChromebotResult
	for i := len(buildIds) - 1; i >= 0; i-- {
		buildID := buildIds[i]
		buildJSON, err := fetchJSON(cocoon, fmt.Sprintf("%v/builds/%v", builderURL, buildID))

		if err != nil {
			return nil, err
		}

		results = append(results, &ChromebotResult{
			Commit: getBuildProperty(buildJSON.(map[string]interface{}), "git_revision"),
			State:  getStatus(buildJSON.(map[string]interface{})),
		})
	}

	return results, nil
}

func fetchJSON(cocoon *db.Cocoon, url string) (interface{}, error) {
	body, err := cocoon.FetchURL(url, false)

	if err != nil {
		return nil, err
	}

	var js interface{}
	if json.Unmarshal(body, &js) != nil {
		return nil, err
	}

	return js, nil
}

// Properties are encoded as:
//
//     {
//       "properties": [
//         [
//           "name1",
//           value1,
//           ... things we don't care about ...
//         ],
//         [
//           "name2",
//           value2,
//           ... things we don't care about ...
//         ]
//       ]
//     }
func getBuildProperty(buildJSON map[string]interface{}, propertyName string) string {
	properties := buildJSON["properties"].([]interface{})
	for _, property := range properties {
		if property.([]interface{})[0] == propertyName {
			return property.([]interface{})[1].(string)
		}
	}
	return ""
}

// Parses out whether the build was successful.
//
// Successes are encoded like this:
//
//     "text": [
//       "build",
//       "successful"
//     ]
//
// Exceptions are encoded like this:
//
//     "text": [
//       "exception",
//       "steps",
//       "exception",
//       "flutter build apk material_gallery"
//     ]
//
// Errors are encoded like this:
//
//     "text": [
//       "failed",
//       "steps",
//       "failed",
//       "flutter build ios simulator stocks"
//     ]
//
// In-progress builds are encoded like this:
//
//    "text": []
//
func getStatus(buildJSON map[string]interface{}) db.TaskStatus {
	text := buildJSON["text"].([]interface{})
	if text == nil || len(text) < 2 {
		return db.TaskInProgress
	} else if text[1].(string) == "successful" {
		return db.TaskSucceeded
	} else {
		return db.TaskFailed
	}
}
<commit_msg>fix Chromebot status refresh (#158)<commit_after>// Copyright (c) 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package commands

import (
	"cocoon/db"
	"encoding/json"
	"fmt"
)

// RefreshChromebotStatusResult contains chromebot status results.
type RefreshChromebotStatusResult struct {
	Results []*ChromebotResult
}

// ChromebotResult describes a chromebot build result.
type ChromebotResult struct {
	Commit string
	State  db.TaskStatus
}

// RefreshChromebotStatus pulls down the latest chromebot builds and updates the
// corresponding task statuses.
func RefreshChromebotStatus(cocoon *db.Cocoon, _ []byte) (interface{}, error) {
	linuxResults, err := refreshChromebot(cocoon, "linux_bot", "Linux")

	if err != nil {
		return nil, err
	}

	macResults, err := refreshChromebot(cocoon, "mac_bot", "Mac")

	if err != nil {
		return nil, err
	}

	windowsResults, err := refreshChromebot(cocoon, "windows_bot", "Windows")

	if err != nil {
		return nil, err
	}

	var allResults []*ChromebotResult
	allResults = append(allResults, linuxResults...)
	allResults = append(allResults, macResults...)
	allResults = append(allResults, windowsResults...)
	return RefreshChromebotStatusResult{allResults}, nil
}

func refreshChromebot(cocoon *db.Cocoon, taskName string, builderName string) ([]*ChromebotResult, error) {
	tasks, err := cocoon.QueryLatestTasksByName(taskName)

	if err != nil {
		return nil, err
	}

	if len(tasks) == 0 {
		// Short-circuit. Don't bother fetching Chromebot data if there are no tasks to
		// to update.
		return make([]*ChromebotResult, 0), nil
	}

	buildStatuses, err := fetchChromebotBuildStatuses(cocoon, builderName)

	if err != nil {
		return nil, err
	}

	for _, fullTask := range tasks {
		// Scan build statuses in reverse so the newer result take precedence over old results.
		for i := len(buildStatuses) - 1; i >= 0; i-- {
			status := buildStatuses[i]
			if status.Commit == fullTask.ChecklistEntity.Checklist.Commit.Sha {
				task := fullTask.TaskEntity.Task
				task.Status = status.State
				cocoon.PutTask(fullTask.TaskEntity.Key, task)
			}
		}
	}

	return buildStatuses, nil
}

func fetchChromebotBuildStatuses(cocoon *db.Cocoon, builderName string) ([]*ChromebotResult, error) {
	builderURL := fmt.Sprintf("https://build.chromium.org/p/client.flutter/json/builders/%v", builderName)

	jsonResponse, err := fetchJSON(cocoon, builderURL)

	if err != nil {
		return nil, err
	}

	buildIds := jsonResponse.(map[string]interface{})["cachedBuilds"].([]interface{})

	if len(buildIds) > 10 {
		// Build IDs are sorted in ascending order, to get the latest 10 builds we
		// grab the tail.
		buildIds = buildIds[len(buildIds)-10:]
	}

	var results []*ChromebotResult
	for i := len(buildIds) - 1; i >= 0; i-- {
		buildID := buildIds[i]
		buildJSON, err := fetchJSON(cocoon, fmt.Sprintf("%v/builds/%v", builderURL, buildID))

		if err != nil {
			return nil, err
		}

		results = append(results, &ChromebotResult{
			Commit: getBuildProperty(buildJSON.(map[string]interface{}), "git_revision"),
			State:  getStatus(buildJSON.(map[string]interface{})),
		})
	}

	return results, nil
}

func fetchJSON(cocoon *db.Cocoon, url string) (interface{}, error) {
	body, err := cocoon.FetchURL(url, false)

	if err != nil {
		return nil, err
	}

	var js interface{}
	if json.Unmarshal(body, &js) != nil {
		return nil, err
	}

	return js, nil
}

// Properties are encoded as:
//
//     {
//       "properties": [
//         [
//           "name1",
//           value1,
//           ... things we don't care about ...
//         ],
//         [
//           "name2",
//           value2,
//           ... things we don't care about ...
//         ]
//       ]
//     }
func getBuildProperty(buildJSON map[string]interface{}, propertyName string) string {
	properties := buildJSON["properties"].([]interface{})
	for _, property := range properties {
		if property.([]interface{})[0] == propertyName {
			return property.([]interface{})[1].(string)
		}
	}
	return ""
}

// Parses out whether the build was successful.
//
// Successes are encoded like this:
//
//     "text": [
//       "build",
//       "successful"
//     ]
//
// Exceptions are encoded like this:
//
//     "text": [
//       "exception",
//       "steps",
//       "exception",
//       "flutter build apk material_gallery"
//     ]
//
// Errors are encoded like this:
//
//     "text": [
//       "failed",
//       "steps",
//       "failed",
//       "flutter build ios simulator stocks"
//     ]
//
// In-progress builds are encoded like this:
//
//    "text": []
//
func getStatus(buildJSON map[string]interface{}) db.TaskStatus {
	text := buildJSON["text"].([]interface{})
	if text == nil || len(text) < 2 {
		return db.TaskInProgress
	} else if text[1].(string) == "successful" {
		return db.TaskSucceeded
	} else {
		return db.TaskFailed
	}
}
<|endoftext|>