<commit_before>// Copyright 2016 ~ 2017 AlexStocks(https://github.com/AlexStocks).
// All rights reserved.  Use of this source code is
// governed by Apache License 2.0.

// package gxlog is based on log4go.
// color.go provides colorful terminal log output functions.
package gxlog

import (
	"fmt"
	"os"
)

import (
	"github.com/mattn/go-isatty"
)

var (
	// Normal colors
	// NORMAL   = []byte{'\033', '0', 'm'}
	NORMAL   = []byte{'\033', '0'}
	NBlack   = []byte{'\033', '[', '3', '0', 'm'}
	NRed     = []byte{'\033', '[', '3', '1', 'm'}
	NGreen   = []byte{'\033', '[', '3', '2', 'm'}
	NYellow  = []byte{'\033', '[', '3', '3', 'm'}
	NBlue    = []byte{'\033', '[', '3', '4', 'm'}
	NMagenta = []byte{'\033', '[', '3', '5', 'm'}
	NCyan    = []byte{'\033', '[', '3', '6', 'm'}
	NWhite   = []byte{'\033', '[', '3', '7', 'm'}
	// Bright colors
	BBlack                    = []byte{'\033', '[', '3', '0', ';', '1', 'm'}
	BRed                      = []byte{'\033', '[', '3', '1', ';', '1', 'm'}
	BGreen                    = []byte{'\033', '[', '3', '2', ';', '1', 'm'}
	BYellow                   = []byte{'\033', '[', '3', '3', ';', '1', 'm'}
	BBlue                     = []byte{'\033', '[', '3', '4', ';', '1', 'm'}
	BMagenta                  = []byte{'\033', '[', '3', '5', ';', '1', 'm'}
	BCyan                     = []byte{'\033', '[', '3', '6', ';', '1', 'm'}
	BWhite                    = []byte{'\033', '[', '3', '7', ';', '1', 'm'}
	UnderlineTwinkleHighLight = []byte{'\033', '[', '1', ';', '6', ';', '4', '0', 'm'}

	reset = []byte{'\033', '[', '0', 'm'}
)

func CPrintf(color []byte, format string, args ...interface{}) {
	if isatty.IsTerminal(os.Stdout.Fd()) {
		fmt.Fprintf(os.Stdout, string(color)+funcFileLine()+fmt.Sprintf(format, args...)+string(reset))
	} else {
		fmt.Fprintf(os.Stdout, fmt.Sprintf(format, args...))
	}
}

func CPrintfln(color []byte, format string, args ...interface{}) {
	if isatty.IsTerminal(os.Stdout.Fd()) {
		fmt.Fprintf(os.Stdout, string(color)+funcFileLine()+fmt.Sprintf(format, args...)+string(reset)+"\n")
	} else {
		fmt.Fprintf(os.Stdout, fmt.Sprintf(format, args...)+"\n")
	}
}

func CEPrintf(color []byte, format string, args ...interface{}) {
	if isatty.IsTerminal(os.Stdout.Fd()) {
		fmt.Fprintf(os.Stderr, string(color)+funcFileLine()+fmt.Sprintf(format, args...)+string(reset))
	} else {
		fmt.Fprintf(os.Stderr, fmt.Sprintf(format, args...))
	}
}

func CEPrintfln(color []byte, format string, args ...interface{}) {
	if isatty.IsTerminal(os.Stdout.Fd()) {
		fmt.Fprintf(os.Stderr, string(color)+funcFileLine()+fmt.Sprintf(format, args...)+string(reset)+"\n")
	} else {
		fmt.Fprintf(os.Stderr, fmt.Sprintf(format, args...)+"\n")
	}
}

func CDebug(format string, args ...interface{}) {
	CPrintfln(NORMAL, format, args...)
}

func CInfo(format string, args ...interface{}) {
	CPrintfln(NGreen, format, args...)
}

func CWarn(format string, args ...interface{}) {
	CPrintfln(BYellow, format, args...)
}

func CError(format string, args ...interface{}) {
	CPrintfln(NRed, format, args...)
}

func CFatal(format string, args ...interface{}) {
	CPrintfln(BRed, format, args...)
}
<commit_msg>using manetena<commit_after>// Copyright 2016 ~ 2017 AlexStocks(https://github.com/AlexStocks).
// All rights reserved.  Use of this source code is
// governed by Apache License 2.0.

// package gxlog is based on log4go.
// color.go provides colorful terminal log output functions.
package gxlog

import (
	"fmt"
	"os"
)

import (
	"github.com/mattn/go-isatty"
)

var (
	// Normal colors
	// NORMAL   = []byte{'\033', '0', 'm'}
	NORMAL   = []byte{'\033', '0'}
	NBlack   = []byte{'\033', '[', '3', '0', 'm'}
	NRed     = []byte{'\033', '[', '3', '1', 'm'}
	NGreen   = []byte{'\033', '[', '3', '2', 'm'}
	NYellow  = []byte{'\033', '[', '3', '3', 'm'}
	NBlue    = []byte{'\033', '[', '3', '4', 'm'}
	NMagenta = []byte{'\033', '[', '3', '5', 'm'}
	NCyan    = []byte{'\033', '[', '3', '6', 'm'}
	NWhite   = []byte{'\033', '[', '3', '7', 'm'}
	// Bright colors
	BBlack                    = []byte{'\033', '[', '3', '0', ';', '1', 'm'}
	BRed                      = []byte{'\033', '[', '3', '1', ';', '1', 'm'}
	BGreen                    = []byte{'\033', '[', '3', '2', ';', '1', 'm'}
	BYellow                   = []byte{'\033', '[', '3', '3', ';', '1', 'm'}
	BBlue                     = []byte{'\033', '[', '3', '4', ';', '1', 'm'}
	BMagenta                  = []byte{'\033', '[', '3', '5', ';', '1', 'm'}
	BCyan                     = []byte{'\033', '[', '3', '6', ';', '1', 'm'}
	BWhite                    = []byte{'\033', '[', '3', '7', ';', '1', 'm'}
	UnderlineTwinkleHighLight = []byte{'\033', '[', '1', ';', '6', ';', '4', '0', 'm'}

	reset = []byte{'\033', '[', '0', 'm'}
)

func CPrintf(color []byte, format string, args ...interface{}) {
	if isatty.IsTerminal(os.Stdout.Fd()) {
		fmt.Fprintf(os.Stdout, string(color)+funcFileLine()+fmt.Sprintf(format, args...)+string(reset))
	} else {
		fmt.Fprintf(os.Stdout, fmt.Sprintf(format, args...))
	}
}

func CPrintfln(color []byte, format string, args ...interface{}) {
	if isatty.IsTerminal(os.Stdout.Fd()) {
		fmt.Fprintf(os.Stdout, string(color)+funcFileLine()+fmt.Sprintf(format, args...)+string(reset)+"\n")
	} else {
		fmt.Fprintf(os.Stdout, fmt.Sprintf(format, args...)+"\n")
	}
}

func CEPrintf(color []byte, format string, args ...interface{}) {
	if isatty.IsTerminal(os.Stdout.Fd()) {
		fmt.Fprintf(os.Stderr, string(color)+funcFileLine()+fmt.Sprintf(format, args...)+string(reset))
	} else {
		fmt.Fprintf(os.Stderr, fmt.Sprintf(format, args...))
	}
}

func CEPrintfln(color []byte, format string, args ...interface{}) {
	if isatty.IsTerminal(os.Stdout.Fd()) {
		fmt.Fprintf(os.Stderr, string(color)+funcFileLine()+fmt.Sprintf(format, args...)+string(reset)+"\n")
	} else {
		fmt.Fprintf(os.Stderr, fmt.Sprintf(format, args...)+"\n")
	}
}

func CDebug(format string, args ...interface{}) {
	CPrintfln(NORMAL, format, args...)
}

func CInfo(format string, args ...interface{}) {
	CPrintfln(NGreen, format, args...)
}

func CWarn(format string, args ...interface{}) {
	CPrintfln(BMagenta, format, args...)
}

func CError(format string, args ...interface{}) {
	CPrintfln(NRed, format, args...)
}

func CFatal(format string, args ...interface{}) {
	CPrintfln(BRed, format, args...)
}
<|endoftext|>


<commit_before>// Copyright 2018 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package log

import (
	"fmt"
	"strings"
	"sync"
	"sync/atomic"

	"go.uber.org/zap/zapcore"

	"istio.io/pkg/structured"
)

// Scope constrains logging control to a named scope level. It gives users a fine grained control over output severity
// threshold and stack traces.
//
// Scope supports structured logging using WithLabels:
//
//   s := RegisterScope("MyScope", "Description", 0)
//   s = s.WithLabels("foo", "bar", "baz", 123, "qux", 0.123)
//   s.Info("Hello")                      // <time>   info   MyScope   Hello  foo=bar baz=123 qux=0.123
//
// The output format can be globally configured to be JSON instead, using Options in this package.
//  e.g. <time>   info   MyScope   { "message":"Hello","foo":"bar","baz":123 }
//
// Scope also supports an error dictionary. The caller can pass a *structured.Error object as the first parameter
// to any of the output functions (Fatal*, Error* etc.) and this will append the fields in the object to the output:
//
//   e := &structured.Error{MoreInfo:"See the documentation in istio.io/helpful_link"}
//   s.WithLabels("foo", "bar").Error(e, "Hello")
//     <time>   info   MyScope   Hello  moreInfo=See the documentation in istio.io/helpful_link foo=bar
//
// See structured.Error for additional guidance on defining errors in a dictionary.
type Scope struct {
	// immutable, set at creation
	name        string
	nameToEmit  string
	description string
	callerSkip  int

	// set by the Configure method and adjustable dynamically
	outputLevel     atomic.Value
	stackTraceLevel atomic.Value
	logCallers      atomic.Value

	// labels data - key slice to preserve ordering
	labelKeys []string
	labels    map[string]interface{}
}

var (
	scopes          = make(map[string]*Scope)
	defaultHandlers []scopeHandlerCallbackFunc
	lock            sync.RWMutex
)

// scopeHandlerCallbackFunc is a callback type for the handler called from Fatal*, Error*, Warn*, Info* and Debug*
// function calls.
type scopeHandlerCallbackFunc func(
	level Level,
	scope *Scope,
	ie *structured.Error,
	msg string,
	fields []zapcore.Field)

// registerDefaultHandler registers a scope handler that is called by default from all scopes. It is appended to the
// current list of default scope handlers.
func registerDefaultHandler(callback scopeHandlerCallbackFunc) {
	lock.Lock()
	defer lock.Unlock()
	defaultHandlers = append(defaultHandlers, callback)
}

// RegisterScope registers a new logging scope. If the same name is used multiple times
// for a single process, the same Scope struct is returned.
//
// Scope names cannot include colons, commas, or periods.
func RegisterScope(name string, description string, callerSkip int) *Scope {
	if strings.ContainsAny(name, ":,.") {
		panic(fmt.Sprintf("scope name %s is invalid, it cannot contain colons, commas, or periods", name))
	}

	lock.Lock()
	defer lock.Unlock()

	s, ok := scopes[name]
	if !ok {
		s = &Scope{
			name:        name,
			description: description,
			callerSkip:  callerSkip,
		}
		s.SetOutputLevel(InfoLevel)
		s.SetStackTraceLevel(NoneLevel)
		s.SetLogCallers(false)

		if name != DefaultScopeName {
			s.nameToEmit = name
		}

		scopes[name] = s
	}

	s.labels = make(map[string]interface{})

	return s
}

// FindScope returns a previously registered scope, or nil if the named scope wasn't previously registered
func FindScope(scope string) *Scope {
	lock.RLock()
	defer lock.RUnlock()

	s := scopes[scope]
	return s
}

// Scopes returns a snapshot of the currently defined set of scopes
func Scopes() map[string]*Scope {
	lock.RLock()
	defer lock.RUnlock()

	s := make(map[string]*Scope, len(scopes))
	for k, v := range scopes {
		s[k] = v
	}

	return s
}

// Fatal outputs a message at fatal level.
func (s *Scope) Fatal(fields ...interface{}) {
	if s.GetOutputLevel() >= FatalLevel {
		ie, firstIdx := getErrorStruct(fields)
		s.callHandlers(FatalLevel, s, ie, fields[0].(string), toZapSlice(firstIdx+1, fields))
	}
}

// Fatala uses fmt.Sprint to construct and log a message at fatal level.
func (s *Scope) Fatala(args ...interface{}) {
	if s.GetOutputLevel() >= FatalLevel {
		ie, firstIdx := getErrorStruct(args)
		s.callHandlers(FatalLevel, s, ie, fmt.Sprint(args[firstIdx:]...), nil)
	}
}

// Fatalf uses fmt.Sprintf to construct and log a message at fatal level.
func (s *Scope) Fatalf(args ...interface{}) {
	if s.GetOutputLevel() >= FatalLevel {
		ie, firstIdx := getErrorStruct(args)
		msg := fmt.Sprint(args[firstIdx])
		if len(args) > 1 {
			msg = fmt.Sprintf(msg, args[firstIdx+1:]...)
		}
		s.callHandlers(FatalLevel, s, ie, msg, nil)
	}
}

// FatalEnabled returns whether output of messages using this scope is currently enabled for fatal-level output.
func (s *Scope) FatalEnabled() bool {
	return s.GetOutputLevel() >= FatalLevel
}

// Error outputs a message at error level.
func (s *Scope) Error(fields ...interface{}) {
	if s.GetOutputLevel() >= ErrorLevel {
		ie, firstIdx := getErrorStruct(fields)
		s.callHandlers(ErrorLevel, s, ie, fields[0].(string), toZapSlice(firstIdx+1, fields))
	}
}

// Errora uses fmt.Sprint to construct and log a message at error level.
func (s *Scope) Errora(args ...interface{}) {
	if s.GetOutputLevel() >= ErrorLevel {
		ie, firstIdx := getErrorStruct(args)
		s.callHandlers(ErrorLevel, s, ie, fmt.Sprint(args[firstIdx:]...), nil)
	}
}

// Errorf uses fmt.Sprintf to construct and log a message at error level.
func (s *Scope) Errorf(args ...interface{}) {
	if s.GetOutputLevel() >= ErrorLevel {
		ie, firstIdx := getErrorStruct(args)
		msg := fmt.Sprint(args[firstIdx])
		if len(args) > 1 {
			msg = fmt.Sprintf(msg, args[firstIdx+1:]...)
		}
		s.callHandlers(ErrorLevel, s, ie, msg, nil)
	}
}

// ErrorEnabled returns whether output of messages using this scope is currently enabled for error-level output.
func (s *Scope) ErrorEnabled() bool {
	return s.GetOutputLevel() >= ErrorLevel
}

// Warn outputs a message at warn level.
func (s *Scope) Warn(fields ...interface{}) {
	if s.GetOutputLevel() >= WarnLevel {
		ie, firstIdx := getErrorStruct(fields)
		s.callHandlers(WarnLevel, s, ie, fields[0].(string), toZapSlice(firstIdx+1, fields))
	}
}

// Warna uses fmt.Sprint to construct and log a message at warn level.
func (s *Scope) Warna(args ...interface{}) {
	if s.GetOutputLevel() >= WarnLevel {
		ie, firstIdx := getErrorStruct(args)
		s.callHandlers(WarnLevel, s, ie, fmt.Sprint(args[firstIdx:]...), nil)
	}
}

// Warnf uses fmt.Sprintf to construct and log a message at warn level.
func (s *Scope) Warnf(args ...interface{}) {
	if s.GetOutputLevel() >= WarnLevel {
		ie, firstIdx := getErrorStruct(args)
		msg := fmt.Sprint(args[firstIdx])
		if len(args) > 1 {
			msg = fmt.Sprintf(msg, args[firstIdx+1:]...)
		}
		s.callHandlers(WarnLevel, s, ie, msg, nil)
	}
}

// WarnEnabled returns whether output of messages using this scope is currently enabled for warn-level output.
func (s *Scope) WarnEnabled() bool {
	return s.GetOutputLevel() >= WarnLevel
}

// Info outputs a message at info level.
func (s *Scope) Info(fields ...interface{}) {
	if s.GetOutputLevel() >= InfoLevel {
		ie, firstIdx := getErrorStruct(fields)
		s.callHandlers(InfoLevel, s, ie, fields[0].(string), toZapSlice(firstIdx+1, fields))
	}
}

// Infoa uses fmt.Sprint to construct and log a message at info level.
func (s *Scope) Infoa(args ...interface{}) {
	if s.GetOutputLevel() >= InfoLevel {
		ie, firstIdx := getErrorStruct(args)
		s.callHandlers(InfoLevel, s, ie, fmt.Sprint(args[firstIdx:]...), nil)
	}
}

// Infof uses fmt.Sprintf to construct and log a message at info level.
func (s *Scope) Infof(args ...interface{}) {
	if s.GetOutputLevel() >= InfoLevel {
		ie, firstIdx := getErrorStruct(args)
		msg := fmt.Sprint(args[firstIdx])
		if len(args) > 1 {
			msg = fmt.Sprintf(msg, args[firstIdx+1:]...)
		}
		s.callHandlers(InfoLevel, s, ie, msg, nil)
	}
}

// InfoEnabled returns whether output of messages using this scope is currently enabled for info-level output.
func (s *Scope) InfoEnabled() bool {
	return s.GetOutputLevel() >= InfoLevel
}

// Debug outputs a message at debug level.
func (s *Scope) Debug(fields ...interface{}) {
	if s.GetOutputLevel() >= DebugLevel {
		ie, firstIdx := getErrorStruct(fields)
		s.callHandlers(DebugLevel, s, ie, fields[0].(string), toZapSlice(firstIdx+1, fields))
	}
}

// Debuga uses fmt.Sprint to construct and log a message at debug level.
func (s *Scope) Debuga(args ...interface{}) {
	if s.GetOutputLevel() >= DebugLevel {
		ie, firstIdx := getErrorStruct(args)
		s.callHandlers(DebugLevel, s, ie, fmt.Sprint(args[firstIdx:]...), nil)
	}
}

// Debugf uses fmt.Sprintf to construct and log a message at debug level.
func (s *Scope) Debugf(args ...interface{}) {
	if s.GetOutputLevel() >= DebugLevel {
		ie, firstIdx := getErrorStruct(args)
		msg := fmt.Sprint(args[firstIdx])
		if len(args) > 1 {
			msg = fmt.Sprintf(msg, args[firstIdx+1:]...)
		}
		s.callHandlers(DebugLevel, s, ie, msg, nil)
	}
}

// DebugEnabled returns whether output of messages using this scope is currently enabled for debug-level output.
func (s *Scope) DebugEnabled() bool {
	return s.GetOutputLevel() >= DebugLevel
}

// Name returns this scope's name.
func (s *Scope) Name() string {
	return s.name
}

// Description returns this scope's description
func (s *Scope) Description() string {
	return s.description
}

// SetOutputLevel adjusts the output level associated with the scope.
func (s *Scope) SetOutputLevel(l Level) {
	s.outputLevel.Store(l)
}

// GetOutputLevel returns the output level associated with the scope.
func (s *Scope) GetOutputLevel() Level {
	return s.outputLevel.Load().(Level)
}

// SetStackTraceLevel adjusts the stack tracing level associated with the scope.
func (s *Scope) SetStackTraceLevel(l Level) {
	s.stackTraceLevel.Store(l)
}

// GetStackTraceLevel returns the stack tracing level associated with the scope.
func (s *Scope) GetStackTraceLevel() Level {
	return s.stackTraceLevel.Load().(Level)
}

// SetLogCallers adjusts the output level associated with the scope.
func (s *Scope) SetLogCallers(logCallers bool) {
	s.logCallers.Store(logCallers)
}

// GetLogCallers returns the output level associated with the scope.
func (s *Scope) GetLogCallers() bool {
	return s.logCallers.Load().(bool)
}

// copy makes a copy of s and returns a pointer to it.
func (s *Scope) copy() *Scope {
	out := *s
	out.labels = copyStringInterfaceMap(s.labels)
	return &out
}

// WithLabels adds a key-value pairs to the labels in s. The key must be a string, while the value may be any type.
// It returns a copy of s, with the labels added.
// e.g. newScope := oldScope.WithLabels("foo", "bar", "baz", 123, "qux", 0.123)
func (s *Scope) WithLabels(kvlist ...interface{}) *Scope {
	out := s.copy()
	if len(kvlist)%2 != 0 {
		out.labels["WithLabels error"] = fmt.Sprintf("even number of parameters required, got %d", len(kvlist))
		return out
	}

	for i := 0; i < len(kvlist); i += 2 {
		keyi := kvlist[i]
		key, ok := keyi.(string)
		if !ok {
			out.labels["WithLabels error"] = fmt.Sprintf("label name %v must be a string, got %T ", keyi, keyi)
			return out
		}
		out.labels[key] = kvlist[i+1]
		out.labelKeys = append(out.labelKeys, key)
	}
	return out
}

// callHandlers calls all handlers registered to s.
func (s *Scope) callHandlers(
	severity Level,
	scope *Scope,
	ie *structured.Error,
	msg string,
	fields []zapcore.Field) {
	for _, h := range defaultHandlers {
		h(severity, scope, ie, msg, fields)
	}
}

// getErrorStruct returns (*Error, 1) if it is the first argument in the list is an Error ptr,
// or (nil,0) otherwise. The second return value is the offset to the first non-Error field.
func getErrorStruct(fields ...interface{}) (*structured.Error, int) {
	ief, ok := fields[0].([]interface{})
	if !ok {
		return nil, 0
	}
	ie, ok := ief[0].(*structured.Error)
	if !ok {
		return nil, 0
	}
	// Skip Error, pass remaining fields on as before.
	return ie, 1
}

func copyStringInterfaceMap(m map[string]interface{}) map[string]interface{} {
	out := make(map[string]interface{}, len(m))
	for k, v := range m {
		out[k] = v
	}
	return out
}
<commit_msg>Explicit lock to fix racetest (#201)<commit_after>// Copyright 2018 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package log

import (
	"fmt"
	"strings"
	"sync"
	"sync/atomic"

	"go.uber.org/zap/zapcore"

	"istio.io/pkg/structured"
)

// Scope constrains logging control to a named scope level. It gives users a fine grained control over output severity
// threshold and stack traces.
//
// Scope supports structured logging using WithLabels:
//
//   s := RegisterScope("MyScope", "Description", 0)
//   s = s.WithLabels("foo", "bar", "baz", 123, "qux", 0.123)
//   s.Info("Hello")                      // <time>   info   MyScope   Hello  foo=bar baz=123 qux=0.123
//
// The output format can be globally configured to be JSON instead, using Options in this package.
//  e.g. <time>   info   MyScope   { "message":"Hello","foo":"bar","baz":123 }
//
// Scope also supports an error dictionary. The caller can pass a *structured.Error object as the first parameter
// to any of the output functions (Fatal*, Error* etc.) and this will append the fields in the object to the output:
//
//   e := &structured.Error{MoreInfo:"See the documentation in istio.io/helpful_link"}
//   s.WithLabels("foo", "bar").Error(e, "Hello")
//     <time>   info   MyScope   Hello  moreInfo=See the documentation in istio.io/helpful_link foo=bar
//
// See structured.Error for additional guidance on defining errors in a dictionary.
type Scope struct {
	// immutable, set at creation
	name        string
	nameToEmit  string
	description string
	callerSkip  int

	// set by the Configure method and adjustable dynamically
	outputLevel     atomic.Value
	stackTraceLevel atomic.Value
	logCallers      atomic.Value

	// labels data - key slice to preserve ordering
	labelKeys []string
	labels    map[string]interface{}
}

var (
	scopes = make(map[string]*Scope)
	lock   sync.RWMutex

	defaultHandlers []scopeHandlerCallbackFunc
	// Write lock should only be taken during program startup.
	defaultHandlersMu sync.RWMutex
)

// scopeHandlerCallbackFunc is a callback type for the handler called from Fatal*, Error*, Warn*, Info* and Debug*
// function calls.
type scopeHandlerCallbackFunc func(
	level Level,
	scope *Scope,
	ie *structured.Error,
	msg string,
	fields []zapcore.Field)

// registerDefaultHandler registers a scope handler that is called by default from all scopes. It is appended to the
// current list of default scope handlers.
func registerDefaultHandler(callback scopeHandlerCallbackFunc) {
	defaultHandlersMu.Lock()
	defer defaultHandlersMu.Unlock()
	defaultHandlers = append(defaultHandlers, callback)
}

// RegisterScope registers a new logging scope. If the same name is used multiple times
// for a single process, the same Scope struct is returned.
//
// Scope names cannot include colons, commas, or periods.
func RegisterScope(name string, description string, callerSkip int) *Scope {
	if strings.ContainsAny(name, ":,.") {
		panic(fmt.Sprintf("scope name %s is invalid, it cannot contain colons, commas, or periods", name))
	}

	lock.Lock()
	defer lock.Unlock()

	s, ok := scopes[name]
	if !ok {
		s = &Scope{
			name:        name,
			description: description,
			callerSkip:  callerSkip,
		}
		s.SetOutputLevel(InfoLevel)
		s.SetStackTraceLevel(NoneLevel)
		s.SetLogCallers(false)

		if name != DefaultScopeName {
			s.nameToEmit = name
		}

		scopes[name] = s
	}

	s.labels = make(map[string]interface{})

	return s
}

// FindScope returns a previously registered scope, or nil if the named scope wasn't previously registered
func FindScope(scope string) *Scope {
	lock.RLock()
	defer lock.RUnlock()

	s := scopes[scope]
	return s
}

// Scopes returns a snapshot of the currently defined set of scopes
func Scopes() map[string]*Scope {
	lock.RLock()
	defer lock.RUnlock()

	s := make(map[string]*Scope, len(scopes))
	for k, v := range scopes {
		s[k] = v
	}

	return s
}

// Fatal outputs a message at fatal level.
func (s *Scope) Fatal(fields ...interface{}) {
	if s.GetOutputLevel() >= FatalLevel {
		ie, firstIdx := getErrorStruct(fields)
		s.callHandlers(FatalLevel, s, ie, fields[0].(string), toZapSlice(firstIdx+1, fields))
	}
}

// Fatala uses fmt.Sprint to construct and log a message at fatal level.
func (s *Scope) Fatala(args ...interface{}) {
	if s.GetOutputLevel() >= FatalLevel {
		ie, firstIdx := getErrorStruct(args)
		s.callHandlers(FatalLevel, s, ie, fmt.Sprint(args[firstIdx:]...), nil)
	}
}

// Fatalf uses fmt.Sprintf to construct and log a message at fatal level.
func (s *Scope) Fatalf(args ...interface{}) {
	if s.GetOutputLevel() >= FatalLevel {
		ie, firstIdx := getErrorStruct(args)
		msg := fmt.Sprint(args[firstIdx])
		if len(args) > 1 {
			msg = fmt.Sprintf(msg, args[firstIdx+1:]...)
		}
		s.callHandlers(FatalLevel, s, ie, msg, nil)
	}
}

// FatalEnabled returns whether output of messages using this scope is currently enabled for fatal-level output.
func (s *Scope) FatalEnabled() bool {
	return s.GetOutputLevel() >= FatalLevel
}

// Error outputs a message at error level.
func (s *Scope) Error(fields ...interface{}) {
	if s.GetOutputLevel() >= ErrorLevel {
		ie, firstIdx := getErrorStruct(fields)
		s.callHandlers(ErrorLevel, s, ie, fields[0].(string), toZapSlice(firstIdx+1, fields))
	}
}

// Errora uses fmt.Sprint to construct and log a message at error level.
func (s *Scope) Errora(args ...interface{}) {
	if s.GetOutputLevel() >= ErrorLevel {
		ie, firstIdx := getErrorStruct(args)
		s.callHandlers(ErrorLevel, s, ie, fmt.Sprint(args[firstIdx:]...), nil)
	}
}

// Errorf uses fmt.Sprintf to construct and log a message at error level.
func (s *Scope) Errorf(args ...interface{}) {
	if s.GetOutputLevel() >= ErrorLevel {
		ie, firstIdx := getErrorStruct(args)
		msg := fmt.Sprint(args[firstIdx])
		if len(args) > 1 {
			msg = fmt.Sprintf(msg, args[firstIdx+1:]...)
		}
		s.callHandlers(ErrorLevel, s, ie, msg, nil)
	}
}

// ErrorEnabled returns whether output of messages using this scope is currently enabled for error-level output.
func (s *Scope) ErrorEnabled() bool {
	return s.GetOutputLevel() >= ErrorLevel
}

// Warn outputs a message at warn level.
func (s *Scope) Warn(fields ...interface{}) {
	if s.GetOutputLevel() >= WarnLevel {
		ie, firstIdx := getErrorStruct(fields)
		s.callHandlers(WarnLevel, s, ie, fields[0].(string), toZapSlice(firstIdx+1, fields))
	}
}

// Warna uses fmt.Sprint to construct and log a message at warn level.
func (s *Scope) Warna(args ...interface{}) {
	if s.GetOutputLevel() >= WarnLevel {
		ie, firstIdx := getErrorStruct(args)
		s.callHandlers(WarnLevel, s, ie, fmt.Sprint(args[firstIdx:]...), nil)
	}
}

// Warnf uses fmt.Sprintf to construct and log a message at warn level.
func (s *Scope) Warnf(args ...interface{}) {
	if s.GetOutputLevel() >= WarnLevel {
		ie, firstIdx := getErrorStruct(args)
		msg := fmt.Sprint(args[firstIdx])
		if len(args) > 1 {
			msg = fmt.Sprintf(msg, args[firstIdx+1:]...)
		}
		s.callHandlers(WarnLevel, s, ie, msg, nil)
	}
}

// WarnEnabled returns whether output of messages using this scope is currently enabled for warn-level output.
func (s *Scope) WarnEnabled() bool {
	return s.GetOutputLevel() >= WarnLevel
}

// Info outputs a message at info level.
func (s *Scope) Info(fields ...interface{}) {
	if s.GetOutputLevel() >= InfoLevel {
		ie, firstIdx := getErrorStruct(fields)
		s.callHandlers(InfoLevel, s, ie, fields[0].(string), toZapSlice(firstIdx+1, fields))
	}
}

// Infoa uses fmt.Sprint to construct and log a message at info level.
func (s *Scope) Infoa(args ...interface{}) {
	if s.GetOutputLevel() >= InfoLevel {
		ie, firstIdx := getErrorStruct(args)
		s.callHandlers(InfoLevel, s, ie, fmt.Sprint(args[firstIdx:]...), nil)
	}
}

// Infof uses fmt.Sprintf to construct and log a message at info level.
func (s *Scope) Infof(args ...interface{}) {
	if s.GetOutputLevel() >= InfoLevel {
		ie, firstIdx := getErrorStruct(args)
		msg := fmt.Sprint(args[firstIdx])
		if len(args) > 1 {
			msg = fmt.Sprintf(msg, args[firstIdx+1:]...)
		}
		s.callHandlers(InfoLevel, s, ie, msg, nil)
	}
}

// InfoEnabled returns whether output of messages using this scope is currently enabled for info-level output.
func (s *Scope) InfoEnabled() bool {
	return s.GetOutputLevel() >= InfoLevel
}

// Debug outputs a message at debug level.
func (s *Scope) Debug(fields ...interface{}) {
	if s.GetOutputLevel() >= DebugLevel {
		ie, firstIdx := getErrorStruct(fields)
		s.callHandlers(DebugLevel, s, ie, fields[0].(string), toZapSlice(firstIdx+1, fields))
	}
}

// Debuga uses fmt.Sprint to construct and log a message at debug level.
func (s *Scope) Debuga(args ...interface{}) {
	if s.GetOutputLevel() >= DebugLevel {
		ie, firstIdx := getErrorStruct(args)
		s.callHandlers(DebugLevel, s, ie, fmt.Sprint(args[firstIdx:]...), nil)
	}
}

// Debugf uses fmt.Sprintf to construct and log a message at debug level.
func (s *Scope) Debugf(args ...interface{}) {
	if s.GetOutputLevel() >= DebugLevel {
		ie, firstIdx := getErrorStruct(args)
		msg := fmt.Sprint(args[firstIdx])
		if len(args) > 1 {
			msg = fmt.Sprintf(msg, args[firstIdx+1:]...)
		}
		s.callHandlers(DebugLevel, s, ie, msg, nil)
	}
}

// DebugEnabled returns whether output of messages using this scope is currently enabled for debug-level output.
func (s *Scope) DebugEnabled() bool {
	return s.GetOutputLevel() >= DebugLevel
}

// Name returns this scope's name.
func (s *Scope) Name() string {
	return s.name
}

// Description returns this scope's description
func (s *Scope) Description() string {
	return s.description
}

// SetOutputLevel adjusts the output level associated with the scope.
func (s *Scope) SetOutputLevel(l Level) {
	s.outputLevel.Store(l)
}

// GetOutputLevel returns the output level associated with the scope.
func (s *Scope) GetOutputLevel() Level {
	return s.outputLevel.Load().(Level)
}

// SetStackTraceLevel adjusts the stack tracing level associated with the scope.
func (s *Scope) SetStackTraceLevel(l Level) {
	s.stackTraceLevel.Store(l)
}

// GetStackTraceLevel returns the stack tracing level associated with the scope.
func (s *Scope) GetStackTraceLevel() Level {
	return s.stackTraceLevel.Load().(Level)
}

// SetLogCallers adjusts the output level associated with the scope.
func (s *Scope) SetLogCallers(logCallers bool) {
	s.logCallers.Store(logCallers)
}

// GetLogCallers returns the output level associated with the scope.
func (s *Scope) GetLogCallers() bool {
	return s.logCallers.Load().(bool)
}

// copy makes a copy of s and returns a pointer to it.
func (s *Scope) copy() *Scope {
	out := *s
	out.labels = copyStringInterfaceMap(s.labels)
	return &out
}

// WithLabels adds a key-value pairs to the labels in s. The key must be a string, while the value may be any type.
// It returns a copy of s, with the labels added.
// e.g. newScope := oldScope.WithLabels("foo", "bar", "baz", 123, "qux", 0.123)
func (s *Scope) WithLabels(kvlist ...interface{}) *Scope {
	out := s.copy()
	if len(kvlist)%2 != 0 {
		out.labels["WithLabels error"] = fmt.Sprintf("even number of parameters required, got %d", len(kvlist))
		return out
	}

	for i := 0; i < len(kvlist); i += 2 {
		keyi := kvlist[i]
		key, ok := keyi.(string)
		if !ok {
			out.labels["WithLabels error"] = fmt.Sprintf("label name %v must be a string, got %T ", keyi, keyi)
			return out
		}
		out.labels[key] = kvlist[i+1]
		out.labelKeys = append(out.labelKeys, key)
	}
	return out
}

// callHandlers calls all handlers registered to s.
func (s *Scope) callHandlers(
	severity Level,
	scope *Scope,
	ie *structured.Error,
	msg string,
	fields []zapcore.Field) {

	defaultHandlersMu.RLock()
	defer defaultHandlersMu.RUnlock()
	for _, h := range defaultHandlers {
		h(severity, scope, ie, msg, fields)
	}
}

// getErrorStruct returns (*Error, 1) if it is the first argument in the list is an Error ptr,
// or (nil,0) otherwise. The second return value is the offset to the first non-Error field.
func getErrorStruct(fields ...interface{}) (*structured.Error, int) {
	ief, ok := fields[0].([]interface{})
	if !ok {
		return nil, 0
	}
	ie, ok := ief[0].(*structured.Error)
	if !ok {
		return nil, 0
	}
	// Skip Error, pass remaining fields on as before.
	return ie, 1
}

func copyStringInterfaceMap(m map[string]interface{}) map[string]interface{} {
	out := make(map[string]interface{}, len(m))
	for k, v := range m {
		out[k] = v
	}
	return out
}
<|endoftext|>


<commit_before>package logg

import (
	"fmt"
	"log"

	"github.com/ondevice/ondevice/util"
)

const (
	// DEBUG -- debug log level
	DEBUG = iota
	// INFO -- info log level
	INFO = iota
	// WARNING -- warning log level
	WARNING = iota
	// ERROR -- error log level
	ERROR = iota
	// FATAL -- fatal log level
	FATAL = iota
)

var _level = INFO
var _levels = map[int]string{}

func init() {
	// Debug, Info, Warning, Error, Fatal
	_levels[DEBUG] = "Debug"
	_levels[INFO] = "Info"
	_levels[WARNING] = "Warning"
	_levels[ERROR] = "Error"
	_levels[FATAL] = "Fatal"
}

// Log -- log message with the given log level
func Log(level int, args ...interface{}) {
	prefix := fmt.Sprintf("%s: ", _getName(level))
	args = append([]interface{}{prefix}, args...)
	if level == FATAL {
		log.Fatal(args...)
	}
	if level < _level {
		return
	}

	log.Print(args...)
}

// Logf -- formatted log message with the given log level
func Logf(level int, format string, args ...interface{}) {
	format = fmt.Sprintf("%s: %s", _getName(level), format)

	if level == FATAL {
		log.Fatalf(format, args...)
	}
	if level < _level {
		return
	}

	log.Printf(format, args...)
}

// Fatal -- fail with the given error message
func Fatal(args ...interface{}) {
	Log(FATAL, args...)
}

// Fatalf -- fail with the given error message
func Fatalf(fmt string, args ...interface{}) {
	Logf(FATAL, fmt, args...)
}

// Error -- log error message
func Error(args ...interface{}) {
	Log(ERROR, args...)
}

// Errorf -- log formatted error message
func Errorf(fmt string, args ...interface{}) {
	Logf(ERROR, fmt, args...)
}

// Warning -- log warning message
func Warning(args ...interface{}) {
	Log(WARNING, args...)
}

// Warningf -- log formatted warning message
func Warningf(fmt string, args ...interface{}) {
	Logf(WARNING, fmt, args...)
}

// Info -- log info message
func Info(args ...interface{}) {
	Log(INFO, args...)
}

// Infof -- log formatted info message
func Infof(fmt string, args ...interface{}) {
	Logf(INFO, fmt, args...)
}

// Debug -- log debug message
func Debug(args ...interface{}) {
	Log(DEBUG, args...)
}

// Debugf -- log formatted debug message
func Debugf(fmt string, args ...interface{}) {
	Logf(DEBUG, fmt, args...)
}

// SetLevel -- Set the minimum log level
func SetLevel(level int) {
	_level = level
}

func _getName(level int) string {
	rc, _ := _levels[level]
	return rc
}

// FailWithAPIError -- call Fatal with a nice error message matching the API error we got
func FailWithAPIError(err util.APIError) {
	if err.Code() == util.NotFoundError {
		Fatal("Not found: ", err.Error())
	} else if err.Code() == util.AuthenticationError {
		Fatal("Authentication failed (try running ondevice login)")
	} else if err.Code() == util.ForbiddenError {
		Fatal("Access denied (are you sure your API key has the required roles?)")
	} else {
		Fatalf("Unexpected API error (code %d): %s", err.Code(), err.Error())
	}
}
<commit_msg>Added support for the ONDEVICE_LOG environment variable<commit_after>package logg

import (
	"fmt"
	"log"
	"os"

	"github.com/ondevice/ondevice/util"
)

const (
	// DEBUG -- debug log level
	DEBUG = iota
	// INFO -- info log level
	INFO = iota
	// WARNING -- warning log level
	WARNING = iota
	// ERROR -- error log level
	ERROR = iota
	// FATAL -- fatal log level
	FATAL = iota
)

var _level = INFO
var _levels = map[int]string{}

func init() {
	_levels[DEBUG] = "Debug"
	_levels[INFO] = "Info"
	_levels[WARNING] = "Warning"
	_levels[ERROR] = "Error"
	_levels[FATAL] = "Fatal"

	if level := os.Getenv("ONDEVICE_LOG"); level != "" {
		switch level {
		case "debug":
			_level = DEBUG
			break
		case "info":
			_level = INFO
			break
		case "warning":
			_level = WARNING
			break
		case "error":
			_level = ERROR
			break
		case "fatal":
			_level = FATAL
			break
		default:
			log.Fatalf("Unsupported log level: '%s' - try one of debug/info/warning/error/fatal", level)
		}
	}
}

// Log -- log message with the given log level
func Log(level int, args ...interface{}) {
	prefix := fmt.Sprintf("%s: ", _getName(level))
	args = append([]interface{}{prefix}, args...)
	if level == FATAL {
		log.Fatal(args...)
	}
	if level < _level {
		return
	}

	log.Print(args...)
}

// Logf -- formatted log message with the given log level
func Logf(level int, format string, args ...interface{}) {
	format = fmt.Sprintf("%s: %s", _getName(level), format)

	if level == FATAL {
		log.Fatalf(format, args...)
	}
	if level < _level {
		return
	}

	log.Printf(format, args...)
}

// Fatal -- fail with the given error message
func Fatal(args ...interface{}) {
	Log(FATAL, args...)
}

// Fatalf -- fail with the given error message
func Fatalf(fmt string, args ...interface{}) {
	Logf(FATAL, fmt, args...)
}

// Error -- log error message
func Error(args ...interface{}) {
	Log(ERROR, args...)
}

// Errorf -- log formatted error message
func Errorf(fmt string, args ...interface{}) {
	Logf(ERROR, fmt, args...)
}

// Warning -- log warning message
func Warning(args ...interface{}) {
	Log(WARNING, args...)
}

// Warningf -- log formatted warning message
func Warningf(fmt string, args ...interface{}) {
	Logf(WARNING, fmt, args...)
}

// Info -- log info message
func Info(args ...interface{}) {
	Log(INFO, args...)
}

// Infof -- log formatted info message
func Infof(fmt string, args ...interface{}) {
	Logf(INFO, fmt, args...)
}

// Debug -- log debug message
func Debug(args ...interface{}) {
	Log(DEBUG, args...)
}

// Debugf -- log formatted debug message
func Debugf(fmt string, args ...interface{}) {
	Logf(DEBUG, fmt, args...)
}

// SetLevel -- Set the minimum log level
func SetLevel(level int) {
	_level = level
}

func _getName(level int) string {
	rc, _ := _levels[level]
	return rc
}

// FailWithAPIError -- call Fatal with a nice error message matching the API error we got
func FailWithAPIError(err util.APIError) {
	if err.Code() == util.NotFoundError {
		Fatal("Not found: ", err.Error())
	} else if err.Code() == util.AuthenticationError {
		Fatal("Authentication failed (try running ondevice login)")
	} else if err.Code() == util.ForbiddenError {
		Fatal("Access denied (are you sure your API key has the required roles?)")
	} else {
		Fatalf("Unexpected API error (code %d): %s", err.Code(), err.Error())
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 The Xorm Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package xorm

import (
	"github.com/admpub/log"
	"github.com/coscms/xorm/core"
)

// AdmpubLogger is the default implment of core.ILogger
type AdmpubLogger struct {
	*log.Logger
	level   core.LogLevel
	showSQL bool
}

var _ core.ILogger = &AdmpubLogger{}

func NewAdmpubLogger() *AdmpubLogger {
	l := &AdmpubLogger{
		Logger: log.New(`xorm`),
		level:  DEFAULT_LOG_LEVEL,
	}
	return l
}

// Level implement core.ILogger
func (s *AdmpubLogger) Level() core.LogLevel {
	return s.level
}

// SetLevel implement core.ILogger
func (s *AdmpubLogger) SetLevel(l core.LogLevel) {
	le := ``
	switch l {
	case 0:
		le = `Debug`
	case 1:
		le = `Info`
	case 2:
		le = `Warn`
	case 3:
		le = `Error`
	case 4:
		le = `Fatal`
	default:
		le = `Fatal`
	}
	s.level = l
	s.Logger.SetLevel(le)
	return
}

// ShowSQL implement core.ILogger
func (s *AdmpubLogger) ShowSQL(show ...bool) {
	if len(show) == 0 {
		s.showSQL = true
		return
	}
	s.showSQL = show[0]
}

// IsShowSQL implement core.ILogger
func (s *AdmpubLogger) IsShowSQL() bool {
	return s.showSQL
}
<commit_msg>update<commit_after>// Copyright 2015 The Xorm Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package xorm

import (
	"github.com/admpub/log"
	"github.com/coscms/xorm/core"
)

// AdmpubLogger is the default implment of core.ILogger
type AdmpubLogger struct {
	*log.Logger
	level   core.LogLevel
	showSQL bool
}

var _ core.ILogger = &AdmpubLogger{}

func NewAdmpubLogger(args ...*log.Logger) *AdmpubLogger {
	l := &AdmpubLogger{
		level: DEFAULT_LOG_LEVEL,
	}
	if len(args) > 0 {
		l.Logger = args[0]
	} else {
		l.Logger = log.New(`xorm`)
	}
	return l
}

// Level implement core.ILogger
func (s *AdmpubLogger) Level() core.LogLevel {
	return s.level
}

// SetLevel implement core.ILogger
func (s *AdmpubLogger) SetLevel(l core.LogLevel) {
	le := ``
	switch l {
	case 0:
		le = `Debug`
	case 1:
		le = `Info`
	case 2:
		le = `Warn`
	case 3:
		le = `Error`
	case 4:
		le = `Fatal`
	default:
		le = `Fatal`
	}
	s.level = l
	s.Logger.SetLevel(le)
	return
}

// ShowSQL implement core.ILogger
func (s *AdmpubLogger) ShowSQL(show ...bool) {
	if len(show) == 0 {
		s.showSQL = true
		return
	}
	s.showSQL = show[0]
}

// IsShowSQL implement core.ILogger
func (s *AdmpubLogger) IsShowSQL() bool {
	return s.showSQL
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package codegen_test

import (
	"io/ioutil"
	"os"
	"testing"

	"path/filepath"

	"github.com/stretchr/testify/assert"
	"github.com/uber/zanzibar/codegen"
)

const (
	clientThrift   = "../../examples/example-gateway/idl/github.com/uber/zanzibar/clients/bar/bar.thrift"
	endpointThrift = "../../examples/example-gateway/idl/github.com/uber/zanzibar/endpoints/bar/bar.thrift"
	tmpDir         = "../../.tmp_gen"
)

func cmpGoldenFile(t *testing.T, actualFile string, goldenFileDir string) {
	b, err := ioutil.ReadFile(actualFile)
	if !assert.NoError(t, err, "failed to read genereated file %s", err) {
		return
	}
	goldenFile := filepath.Join(goldenFileDir, filepath.Base(actualFile)+"gen")
	CompareGoldenFile(t, goldenFile, b, *updateGoldenFile)
}

// func newPackageHelper(t *testing.T) *codegen.PackageHelper {
// 	h, err := codegen.NewPackageHelper(
// 		"examples/example-gateway/idl",
// 		"examples/example-gateway/gen-code",
// 		tmpDir,
// 		"examples/example-gateway/idl/github.com/uber/zanzibar",
// 	)
// 	if !assert.NoError(t, err, "failed to create package helper") {
// 		return nil
// 	}
// 	return h
// }

func TestGenerateBar(t *testing.T) {
	// tmpl, err := codegen.NewTemplate("../../codegen/templates/*.tmpl")
	// if !assert.NoError(t, err, "failed to create template %s", err) {
	// 	return
	// }

	assert.NoError(t, os.RemoveAll(tmpDir), "failed to clean temporary directory")
	if err := os.MkdirAll(tmpDir, os.ModePerm); !assert.NoError(t, err, "failed to create temporary directory", err) {
		return
	}

	relativeGatewayPath := "../../examples/example-gateway"
	absGatewayPath, err := filepath.Abs(relativeGatewayPath)
	if !assert.NoError(t, err, "failed to get abs path %s", err) {
		return
	}
	_ = absGatewayPath

	gateway, err := codegen.NewGatewaySpec(
		absGatewayPath,
		filepath.Join(absGatewayPath, "idl"),
		"github.com/uber/zanzibar/examples/example-gateway/gen-code",
		tmpDir,
		filepath.Join(absGatewayPath, "idl/github.com/uber/zanzibar"),
		"./clients-config",
		"./endpoints-config",
		"example-gateway",
	)
	if !assert.NoError(t, err, "failed to create gateway spec %s", err) {
		return
	}

	// pkgHelper := newPackageHelper(t)

	// m, err := codegen.NewModuleSpec(clientThrift, pkgHelper)
	// if !assert.NoError(t, err, "failed to create module spec %s", err) {
	// 	return
	// }

	// clientFiles, err := tmpl.GenerateClientFile(&codegen.ClientSpec{
	// 	ModuleSpec: m,
	// 	GoFileName: path.Join(
	// 		pkgHelper.CodeGenTargetPath(),
	// 		"clients",
	// 		"bar",
	// 		"bar.go",
	// 	),
	// 	GoStructsFileName: path.Join(
	// 		pkgHelper.CodeGenTargetPath(),
	// 		"clients",
	// 		"bar",
	// 		"bar_structs.go",
	// 	),
	// }, pkgHelper)
	// if !assert.NoError(t, err, "failed to generate client code %s", err) {
	// 	return
	// }

	err = gateway.GenerateClients()
	if !assert.NoError(t, err, "failed to create clients %s", err) {
		return
	}

	cmpGoldenFile(
		t,
		filepath.Join(tmpDir, "clients", "bar", "bar.go"),
		"./data/clients",
	)
	cmpGoldenFile(
		t,
		filepath.Join(tmpDir, "clients", "bar", "bar_structs.go"),
		"./data/clients",
	)

	// m, err = codegen.NewModuleSpec(endpointThrift, pkgHelper)
	// if !assert.NoError(t, err, "failed to create module spec %s", err) {
	// 	return
	// }

	// TODO: (jakev) These tests are super painful to run directly because of
	// config parsing, need to add config parsing to these tests
	// Maybe just refactor these tests to drive codegen/runner directly

	// endpointFiles, err := tmpl.GenerateEndpointFile(&codegen.EndpointSpec{
	// 	ModuleSpec: m,
	// 	GoStructsFileName: path.Join(
	// 		pkgHelper.CodeGenTargetPath(),
	// 		"endpoints",
	// 		"bar",
	// 		"bar_structs.go",
	// 	),
	// 	GoFolderName: path.Join(
	// 		pkgHelper.CodeGenTargetPath(),
	// 		"endpoints",
	// 		"bar",
	// 	),
	// }, pkgHelper, "Bar", "argNotStruct")
	// if !assert.NoError(t, err, "failed to generate endpoint code %s", err) {
	// 	return
	// }

	// for _, file := range endpointFiles.HandlerFiles {
	// 	cmpGoldenFile(t, file, "./data/endpoints")
	// }
	// cmpGoldenFile(t, endpointFiles.StructFile, "./data/endpoints")

	// m, err = codegen.NewModuleSpec(endpointThrift, pkgHelper)
	// if !assert.NoError(t, err, "failed to create module spec %s", err) {
	// 	return
	// }

	// testFiles, err := tmpl.GenerateEndpointTestFile(&codegen.EndpointSpec{
	// 	ModuleSpec: m,
	// }, pkgHelper, "Bar", "argNotStruct")
	// if !assert.NoError(t, err, "failed to generate endpoint code %s", err) {
	// 	return
	// }

	// for _, file := range testFiles {
	// 	cmpGoldenFile(t, file, "./data/endpoint_tests")
	// }
}
<commit_msg>test/gencode: update template test to use GatewaySpec<commit_after>// Copyright (c) 2017 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package codegen_test

import (
	"io/ioutil"
	"os"
	"testing"

	"path/filepath"

	"github.com/stretchr/testify/assert"
	"github.com/uber/zanzibar/codegen"
)

const (
	clientThrift   = "../../examples/example-gateway/idl/github.com/uber/zanzibar/clients/bar/bar.thrift"
	endpointThrift = "../../examples/example-gateway/idl/github.com/uber/zanzibar/endpoints/bar/bar.thrift"
	tmpDir         = "../../.tmp_gen"
)

func cmpGoldenFile(t *testing.T, actualFile string, goldenFileDir string) {
	b, err := ioutil.ReadFile(actualFile)
	if !assert.NoError(t, err, "failed to read genereated file %s", err) {
		return
	}
	goldenFile := filepath.Join(goldenFileDir, filepath.Base(actualFile)+"gen")
	CompareGoldenFile(t, goldenFile, b, *updateGoldenFile)
}

func TestGenerateBar(t *testing.T) {
	assert.NoError(t, os.RemoveAll(tmpDir), "failed to clean temporary directory")
	if err := os.MkdirAll(tmpDir, os.ModePerm); !assert.NoError(t, err, "failed to create temporary directory", err) {
		return
	}

	relativeGatewayPath := "../../examples/example-gateway"
	absGatewayPath, err := filepath.Abs(relativeGatewayPath)
	if !assert.NoError(t, err, "failed to get abs path %s", err) {
		return
	}

	gateway, err := codegen.NewGatewaySpec(
		absGatewayPath,
		filepath.Join(absGatewayPath, "idl"),
		"github.com/uber/zanzibar/examples/example-gateway/gen-code",
		tmpDir,
		filepath.Join(absGatewayPath, "idl/github.com/uber/zanzibar"),
		"./clients-config",
		"./endpoints-config",
		"example-gateway",
	)
	if !assert.NoError(t, err, "failed to create gateway spec %s", err) {
		return
	}

	err = gateway.GenerateClients()
	if !assert.NoError(t, err, "failed to create clients %s", err) {
		return
	}

	cmpGoldenFile(
		t,
		filepath.Join(tmpDir, "clients", "bar", "bar.go"),
		"./data/clients",
	)
	cmpGoldenFile(
		t,
		filepath.Join(tmpDir, "clients", "bar", "bar_structs.go"),
		"./data/clients",
	)

	err = gateway.GenerateEndpoints()
	if !assert.NoError(t, err, "failed to create endpoints %s", err) {
		return
	}

	endpoints, err := ioutil.ReadDir(
		filepath.Join(tmpDir, "endpoints", "bar"),
	)
	if !assert.NoError(t, err, "cannot read dir %s", err) {
		return
	}

	for _, file := range endpoints {
		footer := file.Name()[len(file.Name())-8 : len(file.Name())]
		if footer == "_test.go" {
			continue
		}

		cmpGoldenFile(
			t,
			filepath.Join(tmpDir, "endpoints", "bar", file.Name()),
			"./data/endpoints",
		)
	}
	cmpGoldenFile(
		t,
		filepath.Join(tmpDir, "endpoints", "bar", "bar_structs.go"),
		"./data/endpoints",
	)

	for _, file := range endpoints {
		footer := file.Name()[len(file.Name())-8 : len(file.Name())]
		if footer != "_test.go" {
			continue
		}

		cmpGoldenFile(
			t,
			filepath.Join(tmpDir, "endpoints", "bar", file.Name()),
			"./data/endpoint_tests",
		)
	}
}
<|endoftext|>


<commit_before>// Licensed to the Apache Software Foundation (ASF) under one or more
// contributor license agreements.  See the NOTICE file distributed with
// this work for additional information regarding copyright ownership.
// The ASF licenses this file to You under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance with
// the License.  You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// boot is the boot code for the Python SDK harness container. It is responsible
// for retrieving and install staged files and invoking python correctly.
package main

import (
	"context"
	"flag"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/apache/beam/sdks/go/pkg/beam/artifact"
	jobpb "github.com/apache/beam/sdks/go/pkg/beam/model/jobmanagement_v1"
	pipepb "github.com/apache/beam/sdks/go/pkg/beam/model/pipeline_v1"
	"github.com/apache/beam/sdks/go/pkg/beam/provision"
	"github.com/apache/beam/sdks/go/pkg/beam/util/execx"
	"github.com/apache/beam/sdks/go/pkg/beam/util/grpcx"
	"github.com/golang/protobuf/proto"
	"github.com/nightlyone/lockfile"
)

var (
	acceptableWhlSpecs = []string{"cp27-cp27mu-manylinux1_x86_64.whl"}

	// Contract: https://s.apache.org/beam-fn-api-container-contract.

	workerPool        = flag.Bool("worker_pool", false, "Run as worker pool (optional).")
	id                = flag.String("id", "", "Local identifier (required).")
	loggingEndpoint   = flag.String("logging_endpoint", "", "Logging endpoint (required).")
	artifactEndpoint  = flag.String("artifact_endpoint", "", "Artifact endpoint (required).")
	provisionEndpoint = flag.String("provision_endpoint", "", "Provision endpoint (required).")
	controlEndpoint   = flag.String("control_endpoint", "", "Control endpoint (required).")
	semiPersistDir    = flag.String("semi_persist_dir", "/tmp", "Local semi-persistent directory (optional).")
)

const (
	sdkHarnessEntrypoint = "apache_beam.runners.worker.sdk_worker_main"
	// Please keep these names in sync with stager.py
	workflowFile      = "workflow.tar.gz"
	requirementsFile  = "requirements.txt"
	sdkSrcFile        = "dataflow_python_sdk.tar"
	extraPackagesFile = "extra_packages.txt"
	workerPoolIdEnv   = "BEAM_PYTHON_WORKER_POOL_ID"
)

func main() {
	flag.Parse()

	if *workerPool == true {
		workerPoolId := fmt.Sprintf("%d", os.Getpid())
		os.Setenv(workerPoolIdEnv, workerPoolId)
		args := []string{
			"-m",
			"apache_beam.runners.worker.worker_pool_main",
			"--service_port=50000",
			"--container_executable=/opt/apache/beam/boot",
		}
		log.Printf("Starting worker pool %v: python %v", workerPoolId, strings.Join(args, " "))
		log.Fatalf("Python SDK worker pool exited: %v", execx.Execute("python", args...))
	}

	if *id == "" {
		log.Fatal("No id provided.")
	}
	if *provisionEndpoint == "" {
		log.Fatal("No provision endpoint provided.")
	}

	ctx := grpcx.WriteWorkerID(context.Background(), *id)

	info, err := provision.Info(ctx, *provisionEndpoint)
	if err != nil {
		log.Fatalf("Failed to obtain provisioning information: %v", err)
	}
	log.Printf("Provision info:\n%v", info)

	// TODO(BEAM-8201): Simplify once flags are no longer used.
	if info.GetLoggingEndpoint().GetUrl() != "" {
		*loggingEndpoint = info.GetLoggingEndpoint().GetUrl()
	}
	if info.GetArtifactEndpoint().GetUrl() != "" {
		*artifactEndpoint = info.GetArtifactEndpoint().GetUrl()
	}
	if info.GetControlEndpoint().GetUrl() != "" {
		*controlEndpoint = info.GetControlEndpoint().GetUrl()
	}

	if *loggingEndpoint == "" {
		log.Fatal("No logging endpoint provided.")
	}
	if *artifactEndpoint == "" {
		log.Fatal("No artifact endpoint provided.")
	}
	if *controlEndpoint == "" {
		log.Fatal("No control endpoint provided.")
	}

	log.Printf("Initializing python harness: %v", strings.Join(os.Args, " "))

	// (1) Obtain the pipeline options

	options, err := provision.ProtoToJSON(info.GetPipelineOptions())
	if err != nil {
		log.Fatalf("Failed to convert pipeline options: %v", err)
	}

	// (2) Retrieve and install the staged packages.
	//
	// Guard from concurrent artifact retrieval and installation,
	// when called by child processes in a worker pool.

	materializeArtifactsFunc := func() {
		dir := filepath.Join(*semiPersistDir, "staged")

		files, err := artifact.Materialize(ctx, *artifactEndpoint, info.GetDependencies(), info.GetRetrievalToken(), dir)
		if err != nil {
			log.Fatalf("Failed to retrieve staged files: %v", err)
		}

		// TODO(herohde): the packages to install should be specified explicitly. It
		// would also be possible to install the SDK in the Dockerfile.
		if setupErr := installSetupPackages(files, dir); setupErr != nil {
			log.Fatalf("Failed to install required packages: %v", setupErr)
		}
	}

	workerPoolId := os.Getenv(workerPoolIdEnv)
	if workerPoolId != "" {
		multiProcessExactlyOnce(materializeArtifactsFunc, "beam.install.complete."+workerPoolId)
	} else {
		materializeArtifactsFunc()
	}

	// (3) Invoke python

	os.Setenv("WORKER_ID", *id)
	os.Setenv("PIPELINE_OPTIONS", options)
	os.Setenv("SEMI_PERSISTENT_DIRECTORY", *semiPersistDir)
	os.Setenv("LOGGING_API_SERVICE_DESCRIPTOR", proto.MarshalTextString(&pipepb.ApiServiceDescriptor{Url: *loggingEndpoint}))
	os.Setenv("CONTROL_API_SERVICE_DESCRIPTOR", proto.MarshalTextString(&pipepb.ApiServiceDescriptor{Url: *controlEndpoint}))

	if info.GetStatusEndpoint() != nil {
		os.Setenv("STATUS_API_SERVICE_DESCRIPTOR", proto.MarshalTextString(info.GetStatusEndpoint()))
	}

	args := []string{
		"-m",
		sdkHarnessEntrypoint,
	}
	log.Printf("Executing: python %v", strings.Join(args, " "))

	log.Fatalf("Python exited: %v", execx.Execute("python", args...))
}

// installSetupPackages installs Beam SDK and user dependencies.
func installSetupPackages(mds []*jobpb.ArtifactMetadata, workDir string) error {
	log.Printf("Installing setup packages ...")

	files := make([]string, len(mds))
	for i, v := range mds {
		log.Printf("Found artifact: %s", v.Name)
		files[i] = v.Name
	}

	// Install the Dataflow Python SDK and worker packages.
	// We install the extra requirements in case of using the beam sdk. These are ignored by pip
	// if the user is using an SDK that does not provide these.
	if err := installSdk(files, workDir, sdkSrcFile, acceptableWhlSpecs, false); err != nil {
		return fmt.Errorf("failed to install SDK: %v", err)
	}
	// The staged files will not disappear due to restarts because workDir is a
	// folder that is mapped to the host (and therefore survives restarts).
	if err := pipInstallRequirements(files, workDir, requirementsFile); err != nil {
		return fmt.Errorf("failed to install requirements: %v", err)
	}
	if err := installExtraPackages(files, extraPackagesFile, workDir); err != nil {
		return fmt.Errorf("failed to install extra packages: %v", err)
	}
	if err := pipInstallPackage(files, workDir, workflowFile, false, true, nil); err != nil {
		return fmt.Errorf("failed to install workflow: %v", err)
	}

	return nil
}

// joinPaths joins the dir to every artifact path. Each / in the path is
// interpreted as a directory separator.
func joinPaths(dir string, paths ...string) []string {
	var ret []string
	for _, p := range paths {
		ret = append(ret, filepath.Join(dir, filepath.FromSlash(p)))
	}
	return ret
}

// Call the given function exactly once across multiple worker processes.
// The need for multiple processes is specific to the Python SDK due to the GIL.
// Should another SDK require it, this could be separated out as shared utility.
func multiProcessExactlyOnce(actionFunc func(), completeFileName string) {
	installCompleteFile := filepath.Join(os.TempDir(), completeFileName)

	// skip if install already complete, no need to lock
	_, err := os.Stat(installCompleteFile)
	if err == nil {
		return
	}

	lock, err := lockfile.New(filepath.Join(os.TempDir(), completeFileName+".lck"))
	if err != nil {
		log.Fatalf("Cannot init artifact retrieval lock: %v", err)
	}

	for err = lock.TryLock(); err != nil; err = lock.TryLock() {
		if _, ok := err.(lockfile.TemporaryError); ok {
			time.Sleep(5 * time.Second)
			log.Printf("Worker %v waiting for artifact retrieval lock: %v", *id, lock)
		} else {
			log.Fatalf("Worker %v could not obtain artifact retrieval lock: %v", *id, err)
		}
	}
	defer lock.Unlock()

	// skip if install already complete
	_, err = os.Stat(installCompleteFile)
	if err == nil {
		return
	}

	// do the real work
	actionFunc()

	// mark install complete
	os.OpenFile(installCompleteFile, os.O_RDONLY|os.O_CREATE, 0666)

}
<commit_msg>[BEAM-7672] Increase  the set of acceptable Python wheels in Beam Python containers #11970<commit_after>// Licensed to the Apache Software Foundation (ASF) under one or more
// contributor license agreements.  See the NOTICE file distributed with
// this work for additional information regarding copyright ownership.
// The ASF licenses this file to You under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance with
// the License.  You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// boot is the boot code for the Python SDK harness container. It is responsible
// for retrieving and install staged files and invoking python correctly.
package main

import (
	"context"
	"flag"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/apache/beam/sdks/go/pkg/beam/artifact"
	jobpb "github.com/apache/beam/sdks/go/pkg/beam/model/jobmanagement_v1"
	pipepb "github.com/apache/beam/sdks/go/pkg/beam/model/pipeline_v1"
	"github.com/apache/beam/sdks/go/pkg/beam/provision"
	"github.com/apache/beam/sdks/go/pkg/beam/util/execx"
	"github.com/apache/beam/sdks/go/pkg/beam/util/grpcx"
	"github.com/golang/protobuf/proto"
	"github.com/nightlyone/lockfile"
)

var (
	acceptableWhlSpecs = []string{"cp27-cp27mu-manylinux1_x86_64.whl", "cp35-cp35m-manylinux1_x86_64.whl", "cp36-cp36m-manylinux1_x86_64.whl", "cp37-cp37m-manylinux1_x86_64.whl"}

	// Contract: https://s.apache.org/beam-fn-api-container-contract.

	workerPool        = flag.Bool("worker_pool", false, "Run as worker pool (optional).")
	id                = flag.String("id", "", "Local identifier (required).")
	loggingEndpoint   = flag.String("logging_endpoint", "", "Logging endpoint (required).")
	artifactEndpoint  = flag.String("artifact_endpoint", "", "Artifact endpoint (required).")
	provisionEndpoint = flag.String("provision_endpoint", "", "Provision endpoint (required).")
	controlEndpoint   = flag.String("control_endpoint", "", "Control endpoint (required).")
	semiPersistDir    = flag.String("semi_persist_dir", "/tmp", "Local semi-persistent directory (optional).")
)

const (
	sdkHarnessEntrypoint = "apache_beam.runners.worker.sdk_worker_main"
	// Please keep these names in sync with stager.py
	workflowFile      = "workflow.tar.gz"
	requirementsFile  = "requirements.txt"
	sdkSrcFile        = "dataflow_python_sdk.tar"
	extraPackagesFile = "extra_packages.txt"
	workerPoolIdEnv   = "BEAM_PYTHON_WORKER_POOL_ID"
)

func main() {
	flag.Parse()

	if *workerPool == true {
		workerPoolId := fmt.Sprintf("%d", os.Getpid())
		os.Setenv(workerPoolIdEnv, workerPoolId)
		args := []string{
			"-m",
			"apache_beam.runners.worker.worker_pool_main",
			"--service_port=50000",
			"--container_executable=/opt/apache/beam/boot",
		}
		log.Printf("Starting worker pool %v: python %v", workerPoolId, strings.Join(args, " "))
		log.Fatalf("Python SDK worker pool exited: %v", execx.Execute("python", args...))
	}

	if *id == "" {
		log.Fatal("No id provided.")
	}
	if *provisionEndpoint == "" {
		log.Fatal("No provision endpoint provided.")
	}

	ctx := grpcx.WriteWorkerID(context.Background(), *id)

	info, err := provision.Info(ctx, *provisionEndpoint)
	if err != nil {
		log.Fatalf("Failed to obtain provisioning information: %v", err)
	}
	log.Printf("Provision info:\n%v", info)

	// TODO(BEAM-8201): Simplify once flags are no longer used.
	if info.GetLoggingEndpoint().GetUrl() != "" {
		*loggingEndpoint = info.GetLoggingEndpoint().GetUrl()
	}
	if info.GetArtifactEndpoint().GetUrl() != "" {
		*artifactEndpoint = info.GetArtifactEndpoint().GetUrl()
	}
	if info.GetControlEndpoint().GetUrl() != "" {
		*controlEndpoint = info.GetControlEndpoint().GetUrl()
	}

	if *loggingEndpoint == "" {
		log.Fatal("No logging endpoint provided.")
	}
	if *artifactEndpoint == "" {
		log.Fatal("No artifact endpoint provided.")
	}
	if *controlEndpoint == "" {
		log.Fatal("No control endpoint provided.")
	}

	log.Printf("Initializing python harness: %v", strings.Join(os.Args, " "))

	// (1) Obtain the pipeline options

	options, err := provision.ProtoToJSON(info.GetPipelineOptions())
	if err != nil {
		log.Fatalf("Failed to convert pipeline options: %v", err)
	}

	// (2) Retrieve and install the staged packages.
	//
	// Guard from concurrent artifact retrieval and installation,
	// when called by child processes in a worker pool.

	materializeArtifactsFunc := func() {
		dir := filepath.Join(*semiPersistDir, "staged")

		files, err := artifact.Materialize(ctx, *artifactEndpoint, info.GetDependencies(), info.GetRetrievalToken(), dir)
		if err != nil {
			log.Fatalf("Failed to retrieve staged files: %v", err)
		}

		// TODO(herohde): the packages to install should be specified explicitly. It
		// would also be possible to install the SDK in the Dockerfile.
		if setupErr := installSetupPackages(files, dir); setupErr != nil {
			log.Fatalf("Failed to install required packages: %v", setupErr)
		}
	}

	workerPoolId := os.Getenv(workerPoolIdEnv)
	if workerPoolId != "" {
		multiProcessExactlyOnce(materializeArtifactsFunc, "beam.install.complete."+workerPoolId)
	} else {
		materializeArtifactsFunc()
	}

	// (3) Invoke python

	os.Setenv("WORKER_ID", *id)
	os.Setenv("PIPELINE_OPTIONS", options)
	os.Setenv("SEMI_PERSISTENT_DIRECTORY", *semiPersistDir)
	os.Setenv("LOGGING_API_SERVICE_DESCRIPTOR", proto.MarshalTextString(&pipepb.ApiServiceDescriptor{Url: *loggingEndpoint}))
	os.Setenv("CONTROL_API_SERVICE_DESCRIPTOR", proto.MarshalTextString(&pipepb.ApiServiceDescriptor{Url: *controlEndpoint}))

	if info.GetStatusEndpoint() != nil {
		os.Setenv("STATUS_API_SERVICE_DESCRIPTOR", proto.MarshalTextString(info.GetStatusEndpoint()))
	}

	args := []string{
		"-m",
		sdkHarnessEntrypoint,
	}
	log.Printf("Executing: python %v", strings.Join(args, " "))

	log.Fatalf("Python exited: %v", execx.Execute("python", args...))
}

// installSetupPackages installs Beam SDK and user dependencies.
func installSetupPackages(mds []*jobpb.ArtifactMetadata, workDir string) error {
	log.Printf("Installing setup packages ...")

	files := make([]string, len(mds))
	for i, v := range mds {
		log.Printf("Found artifact: %s", v.Name)
		files[i] = v.Name
	}

	// Install the Dataflow Python SDK and worker packages.
	// We install the extra requirements in case of using the beam sdk. These are ignored by pip
	// if the user is using an SDK that does not provide these.
	if err := installSdk(files, workDir, sdkSrcFile, acceptableWhlSpecs, false); err != nil {
		return fmt.Errorf("failed to install SDK: %v", err)
	}
	// The staged files will not disappear due to restarts because workDir is a
	// folder that is mapped to the host (and therefore survives restarts).
	if err := pipInstallRequirements(files, workDir, requirementsFile); err != nil {
		return fmt.Errorf("failed to install requirements: %v", err)
	}
	if err := installExtraPackages(files, extraPackagesFile, workDir); err != nil {
		return fmt.Errorf("failed to install extra packages: %v", err)
	}
	if err := pipInstallPackage(files, workDir, workflowFile, false, true, nil); err != nil {
		return fmt.Errorf("failed to install workflow: %v", err)
	}

	return nil
}

// joinPaths joins the dir to every artifact path. Each / in the path is
// interpreted as a directory separator.
func joinPaths(dir string, paths ...string) []string {
	var ret []string
	for _, p := range paths {
		ret = append(ret, filepath.Join(dir, filepath.FromSlash(p)))
	}
	return ret
}

// Call the given function exactly once across multiple worker processes.
// The need for multiple processes is specific to the Python SDK due to the GIL.
// Should another SDK require it, this could be separated out as shared utility.
func multiProcessExactlyOnce(actionFunc func(), completeFileName string) {
	installCompleteFile := filepath.Join(os.TempDir(), completeFileName)

	// skip if install already complete, no need to lock
	_, err := os.Stat(installCompleteFile)
	if err == nil {
		return
	}

	lock, err := lockfile.New(filepath.Join(os.TempDir(), completeFileName+".lck"))
	if err != nil {
		log.Fatalf("Cannot init artifact retrieval lock: %v", err)
	}

	for err = lock.TryLock(); err != nil; err = lock.TryLock() {
		if _, ok := err.(lockfile.TemporaryError); ok {
			time.Sleep(5 * time.Second)
			log.Printf("Worker %v waiting for artifact retrieval lock: %v", *id, lock)
		} else {
			log.Fatalf("Worker %v could not obtain artifact retrieval lock: %v", *id, err)
		}
	}
	defer lock.Unlock()

	// skip if install already complete
	_, err = os.Stat(installCompleteFile)
	if err == nil {
		return
	}

	// do the real work
	actionFunc()

	// mark install complete
	os.OpenFile(installCompleteFile, os.O_RDONLY|os.O_CREATE, 0666)

}
<|endoftext|>


<commit_before>package git

import (
	"github.com/bmizerany/assert"
	"strings"
	"testing"
)

func TestGitDir(t *testing.T) {
	gitDir, _ := Dir()
	assert.T(t, strings.Contains(gitDir, ".git"))
}

func TestGitEditor(t *testing.T) {
	gitEditor, err := Editor()
	if err == nil {
		assert.NotEqual(t, "", gitEditor)
	}
}

func TestGitLog(t *testing.T) {
	log, err := Log("e357a98a1a580b09d4f1d9bf613a6a51e131ef6e", "49e984e2fe86f68c386aeb133b390d39e4264ec1")
	assert.Equal(t, nil, err)
	assert.NotEqual(t, "", log)
}

func TestGitRef(t *testing.T) {
	gitRef, err := Ref("1c1077c052d32a83aa13a8afaa4a9630d2f28ef6")
	assert.Equal(t, nil, err)
	assert.Equal(t, "1c1077c052d32a83aa13a8afaa4a9630d2f28ef6", gitRef)
}

func TestGitRefList(t *testing.T) {
	refList, err := RefList("e357a98a1a580b09d4f1d9bf613a6a51e131ef6e", "49e984e2fe86f68c386aeb133b390d39e4264ec1")
	assert.Equal(t, nil, err)
	assert.Equal(t, 1, len(refList))

	assert.Equal(t, "49e984e2fe86f68c386aeb133b390d39e4264ec1", refList[0])
}

func TestGitShow(t *testing.T) {
	output, err := Show("8494cd083d5e5817f4aa75fb8a3973ecfd39f2f8")
	assert.Equal(t, nil, err)
	assert.Equal(t, "Fix typo in git show format\n\nIt's possible that comments are multiple lines", output)
}
<commit_msg>Fix test failure due to deleted commit<commit_after>package git

import (
	"github.com/bmizerany/assert"
	"strings"
	"testing"
)

func TestGitDir(t *testing.T) {
	gitDir, _ := Dir()
	assert.T(t, strings.Contains(gitDir, ".git"))
}

func TestGitEditor(t *testing.T) {
	gitEditor, err := Editor()
	if err == nil {
		assert.NotEqual(t, "", gitEditor)
	}
}

func TestGitLog(t *testing.T) {
	log, err := Log("e357a98a1a580b09d4f1d9bf613a6a51e131ef6e", "49e984e2fe86f68c386aeb133b390d39e4264ec1")
	assert.Equal(t, nil, err)
	assert.NotEqual(t, "", log)
}

func TestGitRef(t *testing.T) {
	gitRef, err := Ref("1c1077c052d32a83aa13a8afaa4a9630d2f28ef6")
	assert.Equal(t, nil, err)
	assert.Equal(t, "1c1077c052d32a83aa13a8afaa4a9630d2f28ef6", gitRef)
}

func TestGitRefList(t *testing.T) {
	refList, err := RefList("e357a98a1a580b09d4f1d9bf613a6a51e131ef6e", "49e984e2fe86f68c386aeb133b390d39e4264ec1")
	assert.Equal(t, nil, err)
	assert.Equal(t, 1, len(refList))

	assert.Equal(t, "49e984e2fe86f68c386aeb133b390d39e4264ec1", refList[0])
}

func TestGitShow(t *testing.T) {
	output, err := Show("f6ba909952aea75d69a9b6e1563a84ec3220ab9c")
	assert.Equal(t, nil, err)
	assert.Equal(t, "Fix typo in git show format\n\nIt's possible that comments are multiple lines", output)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"net/http"
	"time"

	c "github.com/flynn/flynn/Godeps/_workspace/src/github.com/flynn/go-check"
	"github.com/flynn/flynn/controller/client"
	ct "github.com/flynn/flynn/controller/types"
)

type DomainMigrationSuite struct {
	Helper
}

var _ = c.Suite(&DomainMigrationSuite{})

func (s *DomainMigrationSuite) migrateDomain(t *c.C, dm *ct.DomainMigration) {
	client := s.controllerClient(t)

	events := make(chan *ct.Event)
	stream, err := client.StreamEvents(controller.StreamEventsOptions{
		ObjectTypes: []ct.EventType{ct.EventTypeDomainMigration},
	}, events)
	t.Assert(err, c.IsNil)
	defer stream.Close()

	prevRouterRelease, err := client.GetAppRelease("router")
	t.Assert(err, c.IsNil)

	err = client.PutDomain(dm)
	t.Assert(err, c.IsNil)

	// created
	var e *ct.Event
	var event ct.DomainMigrationEvent
	select {
	case e = <-events:
	case <-time.After(2 * time.Minute):
		t.Fatal("timed out waiting for initial domain_migration event")
	}
	t.Assert(e.Data, c.Not(c.IsNil))
	t.Assert(json.Unmarshal(e.Data, &event), c.IsNil)
	t.Assert(event.Error, c.Equals, "")
	t.Assert(event.DomainMigration, c.Not(c.IsNil))
	t.Assert(event.DomainMigration.ID, c.Equals, dm.ID)
	t.Assert(event.DomainMigration.OldDomain, c.Equals, dm.OldDomain)
	t.Assert(event.DomainMigration.Domain, c.Equals, dm.Domain)
	t.Assert(event.DomainMigration.TLSCert, c.IsNil)
	t.Assert(event.DomainMigration.OldTLSCert, c.Not(c.IsNil))
	t.Assert(event.DomainMigration.CreatedAt, c.Not(c.IsNil))
	t.Assert(event.DomainMigration.CreatedAt.Equal(*dm.CreatedAt), c.Equals, true)
	t.Assert(event.DomainMigration.FinishedAt, c.IsNil)

	// complete
	select {
	case e = <-events:
	case <-time.After(3 * time.Minute):
		t.Fatal("timed out waiting for final domain_migration event")
	}
	t.Assert(e.Data, c.Not(c.IsNil))
	t.Assert(json.Unmarshal(e.Data, &event), c.IsNil)
	t.Assert(event.Error, c.Equals, "")
	t.Assert(event.DomainMigration, c.Not(c.IsNil))
	t.Assert(event.DomainMigration.ID, c.Equals, dm.ID)
	t.Assert(event.DomainMigration.OldDomain, c.Equals, dm.OldDomain)
	t.Assert(event.DomainMigration.Domain, c.Equals, dm.Domain)
	t.Assert(event.DomainMigration.TLSCert, c.Not(c.IsNil))
	t.Assert(event.DomainMigration.OldTLSCert, c.Not(c.IsNil))
	t.Assert(event.DomainMigration.CreatedAt, c.Not(c.IsNil))
	t.Assert(event.DomainMigration.CreatedAt.Equal(*dm.CreatedAt), c.Equals, true)
	t.Assert(event.DomainMigration.FinishedAt, c.Not(c.IsNil))

	cert := event.DomainMigration.TLSCert

	controllerRelease, err := client.GetAppRelease("controller")
	t.Assert(err, c.IsNil)
	t.Assert(controllerRelease.Env["DEFAULT_ROUTE_DOMAIN"], c.Equals, dm.Domain)
	t.Assert(controllerRelease.Env["CA_CERT"], c.Equals, cert.CACert)

	routerRelease, err := client.GetAppRelease("router")
	t.Assert(err, c.IsNil)
	t.Assert(routerRelease.Env["TLSCERT"], c.Equals, cert.Cert)
	t.Assert(routerRelease.Env["TLSKEY"], c.Not(c.Equals), "")
	t.Assert(routerRelease.Env["TLSKEY"], c.Not(c.Equals), prevRouterRelease.Env["TLSKEY"])

	dashboardRelease, err := client.GetAppRelease("dashboard")
	t.Assert(err, c.IsNil)
	t.Assert(dashboardRelease.Env["DEFAULT_ROUTE_DOMAIN"], c.Equals, dm.Domain)
	t.Assert(dashboardRelease.Env["CONTROLLER_DOMAIN"], c.Equals, fmt.Sprintf("controller.%s", dm.Domain))
	t.Assert(dashboardRelease.Env["URL"], c.Equals, fmt.Sprintf("dashboard.%s", dm.Domain))
	t.Assert(dashboardRelease.Env["CA_CERT"], c.Equals, cert.CACert)

	var doPing func(string, int)
	doPing = func(component string, retriesRemaining int) {
		url := fmt.Sprintf("http://%s.%s/ping", component, dm.Domain)
		res, err := (&http.Client{}).Get(url)
		if (err != nil || res.StatusCode != 200) && retriesRemaining > 0 {
			time.Sleep(100 * time.Millisecond)
			doPing(component, retriesRemaining-1)
			return
		}
		t.Assert(err, c.IsNil)
		t.Assert(res.StatusCode, c.Equals, 200, c.Commentf("failed to ping %s", component))
	}
	doPing("controller", 3)
	doPing("dashboard", 3)
}

func (s *DomainMigrationSuite) TestDomainMigration(t *c.C) {
	release, err := s.controllerClient(t).GetAppRelease("controller")
	t.Assert(err, c.IsNil)
	oldDomain := release.Env["DEFAULT_ROUTE_DOMAIN"]

	// using xip.io to get around modifying /etc/hosts
	dm := &ct.DomainMigration{
		OldDomain: oldDomain,
		Domain:    fmt.Sprintf("%s.xip.io", routerIP),
	}
	s.migrateDomain(t, dm)
	s.migrateDomain(t, &ct.DomainMigration{
		OldDomain: dm.Domain,
		Domain:    dm.OldDomain,
	})
}
<commit_msg>test: Handle closed event stream in TestDomainMigration<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"net/http"
	"time"

	c "github.com/flynn/flynn/Godeps/_workspace/src/github.com/flynn/go-check"
	"github.com/flynn/flynn/controller/client"
	ct "github.com/flynn/flynn/controller/types"
)

type DomainMigrationSuite struct {
	Helper
}

var _ = c.Suite(&DomainMigrationSuite{})

func (s *DomainMigrationSuite) migrateDomain(t *c.C, dm *ct.DomainMigration) {
	debugf(t, "migrating domain from %s to %s", dm.OldDomain, dm.Domain)
	client := s.controllerClient(t)

	events := make(chan *ct.Event)
	stream, err := client.StreamEvents(controller.StreamEventsOptions{
		ObjectTypes: []ct.EventType{ct.EventTypeDomainMigration},
	}, events)
	t.Assert(err, c.IsNil)
	defer stream.Close()

	prevRouterRelease, err := client.GetAppRelease("router")
	t.Assert(err, c.IsNil)

	err = client.PutDomain(dm)
	t.Assert(err, c.IsNil)

	waitEvent := func(typ string, timeout time.Duration) (event ct.DomainMigrationEvent) {
		debugf(t, "waiting for %s domain migration event", typ)
		var e *ct.Event
		var ok bool
		select {
		case e, ok = <-events:
			if !ok {
				t.Fatal("event stream closed unexpectedly")
			}
			debugf(t, "got %s domain migration event", typ)
		case <-time.After(timeout):
			t.Fatalf("timed out waiting for %s domain migration event", typ)
		}
		t.Assert(e.Data, c.NotNil)
		t.Assert(json.Unmarshal(e.Data, &event), c.IsNil)
		return
	}

	// created
	event := waitEvent("initial", 2*time.Minute)
	t.Assert(event.Error, c.Equals, "")
	t.Assert(event.DomainMigration, c.NotNil)
	t.Assert(event.DomainMigration.ID, c.Equals, dm.ID)
	t.Assert(event.DomainMigration.OldDomain, c.Equals, dm.OldDomain)
	t.Assert(event.DomainMigration.Domain, c.Equals, dm.Domain)
	t.Assert(event.DomainMigration.TLSCert, c.IsNil)
	t.Assert(event.DomainMigration.OldTLSCert, c.NotNil)
	t.Assert(event.DomainMigration.CreatedAt, c.NotNil)
	t.Assert(event.DomainMigration.CreatedAt.Equal(*dm.CreatedAt), c.Equals, true)
	t.Assert(event.DomainMigration.FinishedAt, c.IsNil)

	// complete
	event = waitEvent("final", 3*time.Minute)
	t.Assert(event.Error, c.Equals, "")
	t.Assert(event.DomainMigration, c.NotNil)
	t.Assert(event.DomainMigration.ID, c.Equals, dm.ID)
	t.Assert(event.DomainMigration.OldDomain, c.Equals, dm.OldDomain)
	t.Assert(event.DomainMigration.Domain, c.Equals, dm.Domain)
	t.Assert(event.DomainMigration.TLSCert, c.NotNil)
	t.Assert(event.DomainMigration.OldTLSCert, c.NotNil)
	t.Assert(event.DomainMigration.CreatedAt, c.NotNil)
	t.Assert(event.DomainMigration.CreatedAt.Equal(*dm.CreatedAt), c.Equals, true)
	t.Assert(event.DomainMigration.FinishedAt, c.NotNil)

	cert := event.DomainMigration.TLSCert

	controllerRelease, err := client.GetAppRelease("controller")
	t.Assert(err, c.IsNil)
	t.Assert(controllerRelease.Env["DEFAULT_ROUTE_DOMAIN"], c.Equals, dm.Domain)
	t.Assert(controllerRelease.Env["CA_CERT"], c.Equals, cert.CACert)

	routerRelease, err := client.GetAppRelease("router")
	t.Assert(err, c.IsNil)
	t.Assert(routerRelease.Env["TLSCERT"], c.Equals, cert.Cert)
	t.Assert(routerRelease.Env["TLSKEY"], c.Not(c.Equals), "")
	t.Assert(routerRelease.Env["TLSKEY"], c.Not(c.Equals), prevRouterRelease.Env["TLSKEY"])

	dashboardRelease, err := client.GetAppRelease("dashboard")
	t.Assert(err, c.IsNil)
	t.Assert(dashboardRelease.Env["DEFAULT_ROUTE_DOMAIN"], c.Equals, dm.Domain)
	t.Assert(dashboardRelease.Env["CONTROLLER_DOMAIN"], c.Equals, fmt.Sprintf("controller.%s", dm.Domain))
	t.Assert(dashboardRelease.Env["URL"], c.Equals, fmt.Sprintf("dashboard.%s", dm.Domain))
	t.Assert(dashboardRelease.Env["CA_CERT"], c.Equals, cert.CACert)

	var doPing func(string, int)
	doPing = func(component string, retriesRemaining int) {
		url := fmt.Sprintf("http://%s.%s/ping", component, dm.Domain)
		res, err := (&http.Client{}).Get(url)
		if (err != nil || res.StatusCode != 200) && retriesRemaining > 0 {
			time.Sleep(100 * time.Millisecond)
			doPing(component, retriesRemaining-1)
			return
		}
		t.Assert(err, c.IsNil)
		t.Assert(res.StatusCode, c.Equals, 200, c.Commentf("failed to ping %s", component))
	}
	doPing("controller", 3)
	doPing("dashboard", 3)
}

func (s *DomainMigrationSuite) TestDomainMigration(t *c.C) {
	release, err := s.controllerClient(t).GetAppRelease("controller")
	t.Assert(err, c.IsNil)
	oldDomain := release.Env["DEFAULT_ROUTE_DOMAIN"]

	// using xip.io to get around modifying /etc/hosts
	dm := &ct.DomainMigration{
		OldDomain: oldDomain,
		Domain:    fmt.Sprintf("%s.xip.io", routerIP),
	}
	s.migrateDomain(t, dm)
	s.migrateDomain(t, &ct.DomainMigration{
		OldDomain: dm.Domain,
		Domain:    dm.OldDomain,
	})
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
)

func main() {
	a := 42

	fmt.Printf("%v - %T \n", a, a)
	fmt.Printf("%v - %T \n", &a, &a)

	var b *int = &a
	fmt.Printf("%v - %T \n", b, b)
	fmt.Printf("%v - %T \n", *b, *b)

	*b = 27
	fmt.Printf("%v - %T \n", a, a)

	*&a = 12
	fmt.Printf("%v - %T \n", *b, *b)

}
<commit_msg>Passing pointers.<commit_after>package main

import (
	"fmt"
)

func zero(x int) {
	fmt.Printf("%v - %T \n", x, x)
	x = 0
}

func zerop(x *int) {
	fmt.Printf("%v - %T \n", x, x)
	*x = 0
}

func main() {
	x := 5
	fmt.Printf("%v - %T \n", x, x)
	zero(x)
	fmt.Printf("%v - %T \n", x, x)

	zerop(&x)
	fmt.Printf("%v - %T \n", x, x)
}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"fmt"
	"syscall"
	"../parse"
	"../edit"
	"../eval"
	"../async"
)

func lackeol() {
	fmt.Println("\033[7m%\033[m")
}

func main() {
	fmt.Printf("My pid is %d\n", syscall.Getpid())

	tr, err := async.NewTimedReader(os.Stdin)
	if err != nil {
		panic(err)
	}

	ev := eval.NewEvaluator(os.Environ())
	cmd_no := 0

	for {
		cmd_no++
		name := fmt.Sprintf("<tty %d>", cmd_no)

		ed, err := edit.Init(os.Stdin, tr)
		if err != nil {
			panic(err)
		}

		lr := ed.ReadLine("das> ")
		err = ed.Cleanup()
		if err != nil {
			panic(err)
		}

		if lr.Eof {
			lackeol()
			break
		} else if lr.Err != nil {
			panic(lr.Err)
		}

		tree, pe := parse.Parse(name, lr.Line, false)
		if pe != nil {
			fmt.Print(pe.Pprint())
			continue
		}

		updates, err := ev.ExecPipeline(tree.Root.(*parse.ListNode))
		if err != nil {
			fmt.Println(err)
			continue
		}

		for i, update := range updates {
			for up := range update {
				switch up.Msg {
				case "0", "":
				default:
					fmt.Printf("Command #%d update: %s\n", i, up.Msg)
				}
			}
		}
	}
}
<commit_msg>Handle eval errors correctly.<commit_after>package main

import (
	"os"
	"fmt"
	"syscall"
	"../parse"
	"../edit"
	"../eval"
	"../async"
)

func lackeol() {
	fmt.Println("\033[7m%\033[m")
}

func main() {
	fmt.Printf("My pid is %d\n", syscall.Getpid())

	tr, err := async.NewTimedReader(os.Stdin)
	if err != nil {
		panic(err)
	}

	ev := eval.NewEvaluator(os.Environ())
	cmd_no := 0

	for {
		cmd_no++
		name := fmt.Sprintf("<tty %d>", cmd_no)

		ed, err := edit.Init(os.Stdin, tr)
		if err != nil {
			panic(err)
		}

		lr := ed.ReadLine("das> ")
		err = ed.Cleanup()
		if err != nil {
			panic(err)
		}

		if lr.Eof {
			lackeol()
			break
		} else if lr.Err != nil {
			panic(lr.Err)
		}

		tree, pe := parse.Parse(name, lr.Line, false)
		if pe != nil {
			fmt.Print(pe.Pprint())
			continue
		}

		updates, ee := ev.ExecPipeline(tree.Root.(*parse.ListNode))
		if ee != nil {
			fmt.Println(ee)
			continue
		}

		for i, update := range updates {
			for up := range update {
				switch up.Msg {
				case "0", "":
				default:
					fmt.Printf("Command #%d update: %s\n", i, up.Msg)
				}
			}
		}
	}
}
<|endoftext|>


<commit_before>package crypt

import (
	"testing"
)

func TestCryptSHA512(t *testing.T) {
	var (
		pass = "password"
		salt = "$6$saltedsalted64$"
		hash = "$6$saltedsalted64$BG5.9QC/0aBR3V4CX82Fx1Ia244NogxWChRfDH4YyB5tjDVJd.loeTzjMWF4XTfynLPI53mEzp04eKaBaVVPp1"
	)

	enc, err := Crypt(pass, salt)
	if err != nil {
		t.Errorf("unexpected error: %v", err)
	}

	if enc != hash {
		t.Errorf("hashed password mismatch, expected [%s], got [%s], hash, enc")
	}
}

func TestCryptMD5(t *testing.T) {
	var (
		pass = "password"
		salt = "XY"
		hash = "XYGpusIMIT/IM"
	)

	enc, err := Crypt(pass, salt)
	if err != nil {
		t.Errorf("unexpected error: %t, (got %s)", err, enc)
	}

	if enc != hash {
		t.Errorf("hashed password mismatch, expected [%s] but got [%s]", hash, enc)
	}
}

func TestCryptErrors(t *testing.T) {
	tests := [][]string{
		{"no salt", ""},
		{"single char", "/"},
		{"first char bad", "!x"},
		{"second char bad", "Z%"},
		{"both chars bad", ":@"},
		{"un$upported algorithm", "$2$"},
		{"unsupported_algorithm", "_1"},
	}

	for _, test := range tests {
		_, err := Crypt("password", test[1])
		if err == nil {
			t.Error("Expected error when %s", test[0])
		}
	}
}
<commit_msg>Add more logging when tests fail.<commit_after>package crypt

import (
	"testing"
)

func TestCryptSHA512(t *testing.T) {
	var (
		pass = "password"
		salt = "$6$saltedsalted64$"
		hash = "$6$saltedsalted64$BG5.9QC/0aBR3V4CX82Fx1Ia244NogxWChRfDH4YyB5tjDVJd.loeTzjMWF4XTfynLPI53mEzp04eKaBaVVPp1"
	)

	enc, err := Crypt(pass, salt)
	if err != nil {
		t.Errorf("unexpected error: %v", err)
	}

	if enc != hash {
		t.Errorf("hashed password mismatch, expected [%s], got [%s], hash, enc")
	}
}

func TestCryptMD5(t *testing.T) {
	var (
		pass = "password"
		salt = "XY"
		hash = "XYGpusIMIT/IM"
	)

	enc, err := Crypt(pass, salt)
	if err != nil {
		t.Errorf("unexpected error: %t, (got %s)", err, enc)
	}

	if enc != hash {
		t.Errorf("hashed password mismatch, expected [%s] but got [%s]", hash, enc)
	}
}

func TestCryptErrors(t *testing.T) {
	tests := [][]string{
		{"no salt", ""},
		{"single char", "/"},
		{"first char bad", "!x"},
		{"second char bad", "Z%"},
		{"both chars bad", ":@"},
		{"un$upported algorithm", "$2$"},
		{"unsupported_algorithm", "_1"},
	}

	for _, test := range tests {
		enc, err := Crypt("password", test[1])
		if err == nil {
			t.Errorf("Expected error when testing %s, instead got", test[0], enc)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
    "fmt"
	"os"
	"os/signal"
    "github.com/fuzxxl/nfc/2.0/nfc"
    "github.com/fuzxxl/freefare/0.3/freefare"
    "code.google.com/p/go-sqlite/go1/sqlite3"
    "time"
    "github.com/davecheney/gpio"
    "gopkg.in/yaml.v2"
    "io/ioutil"
)

func heartbeat() {
    for {
        time.Sleep(2000 * time.Millisecond)
        fmt.Println("Dunka-dunk")
    }
}

func pulse_gpio(pin gpio.Pin, ms int) {
    pin.Set()
    time.Sleep(time.Duration(ms) * time.Millisecond)
    pin.Clear()
}

func clear_and_close(pin gpio.Pin) {
    pin.Clear()
    pin.Close()
}

func main() {
    d, err := nfc.Open("");
    if err != nil {
        panic(err);
    }

    gpiodata, err := ioutil.ReadFile("gpio.yaml");
    if err != nil {
        panic(err);
    }
    gpiomap := make(map[interface{}]interface{});
    err = yaml.Unmarshal([]byte(gpiodata), &gpiomap);
    if err != nil {
        panic(err);
    }

    c, err := sqlite3.Open("keys.db")
    if err != nil {
        panic(err);
    }
    row := make(sqlite3.RowMap)

    go heartbeat()

	green_led, err := gpio.OpenPin(gpiomap["green_led"].(map[interface{}]interface{})["pin"].(int), gpio.ModeOutput)
	if err != nil {
		fmt.Printf("Error opening green_led! %s\n", err)
		return
	}
	red_led, err := gpio.OpenPin(gpiomap["red_led"].(map[interface{}]interface{})["pin"].(int), gpio.ModeOutput)
	if err != nil {
		fmt.Printf("Error opening green_led! %s\n", err)
		return
	}
	relay, err := gpio.OpenPin(gpiomap["relay"].(map[interface{}]interface{})["pin"].(int), gpio.ModeOutput)
	if err != nil {
		fmt.Printf("Error opening relay! %s\n", err)
		return
	}
	// turn the leds off on exit
	exit_ch := make(chan os.Signal, 1)
	signal.Notify(exit_ch, os.Interrupt)
	signal.Notify(exit_ch, os.Kill)
	go func() {
		for _ = range exit_ch {
			fmt.Printf("\nClearing and unexporting the pins.\n")
			go clear_and_close(green_led)
			go clear_and_close(red_led)
			go clear_and_close(relay)
			os.Exit(0)
		}
	}()

    // mainloop
    for {
        // Poll for tags
        var tags []freefare.Tag
        for {
            tags, err = freefare.GetTags(d);
            if err != nil {
                panic(err);
            }
            if len(tags) > 0 {
                break;
            }
            time.Sleep(100 * time.Millisecond)
            //fmt.Println("...polling")
        }
        valid_found := false
        for i := 0; i < len(tags); i++ {
            tag := tags[i]
            uidstr := tag.UID()
            fmt.Println("Found tag", uidstr);
            sql := "SELECT rowid, * FROM keys where uid=?"
            for s, err := c.Query(sql, uidstr); err == nil; err = s.Next() {
                var rowid int64
                s.Scan(&rowid, row)     // Assigns 1st column to rowid, the rest to row
                fmt.Println(rowid, row)
                valid_found = true
                fmt.Println("Access GRANTED to ", uidstr)
                go pulse_gpio(green_led, gpiomap["green_led"].(map[interface{}]interface{})["time"].(int))
                go pulse_gpio(relay, gpiomap["relay"].(map[interface{}]interface{})["time"].(int))
            }
        }
        if !valid_found {
            fmt.Println("Access DENIED")
            go pulse_gpio(red_led, gpiomap["red_led"].(map[interface{}]interface{})["time"].(int))
        }
        // Wait a moment before continuing with fast polling
        time.Sleep(500 * time.Millisecond)
    }
}
<commit_msg>check ACL on raspi too<commit_after>package main

import (
    "fmt"
	"os"
	"os/signal"
    "github.com/fuzxxl/nfc/2.0/nfc"
    "github.com/fuzxxl/freefare/0.3/freefare"
    "code.google.com/p/go-sqlite/go1/sqlite3"
    "time"
    "github.com/davecheney/gpio"
    "gopkg.in/yaml.v2"
    "io/ioutil"
)

func heartbeat() {
    for {
        time.Sleep(2000 * time.Millisecond)
        fmt.Println("Dunka-dunk")
    }
}

func pulse_gpio(pin gpio.Pin, ms int) {
    pin.Set()
    time.Sleep(time.Duration(ms) * time.Millisecond)
    pin.Clear()
}

func clear_and_close(pin gpio.Pin) {
    pin.Clear()
    pin.Close()
}

func main() {
    d, err := nfc.Open("");
    if err != nil {
        panic(err);
    }

    appdata, err := ioutil.ReadFile("apps.yaml");
    if err != nil {
        panic(err);
    }
    appmap := make(map[interface{}]interface{});
    err = yaml.Unmarshal([]byte(appdata), &appmap);
    if err != nil {
        panic(err);
    }
    required_acl := int64(appmap["aclapp"].(map[interface{}]interface{})["require_acl"].(int))

    gpiodata, err := ioutil.ReadFile("gpio.yaml");
    if err != nil {
        panic(err);
    }
    gpiomap := make(map[interface{}]interface{});
    err = yaml.Unmarshal([]byte(gpiodata), &gpiomap);
    if err != nil {
        panic(err);
    }

    c, err := sqlite3.Open("keys.db")
    if err != nil {
        panic(err);
    }
    row := make(sqlite3.RowMap)

    go heartbeat()

	green_led, err := gpio.OpenPin(gpiomap["green_led"].(map[interface{}]interface{})["pin"].(int), gpio.ModeOutput)
	if err != nil {
		fmt.Printf("Error opening green_led! %s\n", err)
		return
	}
	red_led, err := gpio.OpenPin(gpiomap["red_led"].(map[interface{}]interface{})["pin"].(int), gpio.ModeOutput)
	if err != nil {
		fmt.Printf("Error opening green_led! %s\n", err)
		return
	}
	relay, err := gpio.OpenPin(gpiomap["relay"].(map[interface{}]interface{})["pin"].(int), gpio.ModeOutput)
	if err != nil {
		fmt.Printf("Error opening relay! %s\n", err)
		return
	}
	// turn the leds off on exit
	exit_ch := make(chan os.Signal, 1)
	signal.Notify(exit_ch, os.Interrupt)
	signal.Notify(exit_ch, os.Kill)
	go func() {
		for _ = range exit_ch {
			fmt.Printf("\nClearing and unexporting the pins.\n")
			go clear_and_close(green_led)
			go clear_and_close(red_led)
			go clear_and_close(relay)
			os.Exit(0)
		}
	}()

    // mainloop
    for {
        // Poll for tags
        var tags []freefare.Tag
        for {
            tags, err = freefare.GetTags(d);
            if err != nil {
                panic(err);
            }
            if len(tags) > 0 {
                break;
            }
            time.Sleep(100 * time.Millisecond)
            //fmt.Println("...polling")
        }
        valid_found := false
        for i := 0; i < len(tags); i++ {
            tag := tags[i]
            uidstr := tag.UID()
            fmt.Println("Found tag", uidstr);
            sql := "SELECT rowid, * FROM keys where uid=?"
            for s, err := c.Query(sql, uidstr); err == nil; err = s.Next() {
                var rowid int64
                s.Scan(&rowid, row)     // Assigns 1st column to rowid, the rest to row
                db_acl := row["acl"].(int64)
                if (db_acl & required_acl) == 0 {
                    fmt.Println("Found card ", uidstr , " but ACL not granted")
                    continue
                }
                valid_found = true
                fmt.Println("Access GRANTED to ", uidstr)
                go pulse_gpio(green_led, gpiomap["green_led"].(map[interface{}]interface{})["time"].(int))
                go pulse_gpio(relay, gpiomap["relay"].(map[interface{}]interface{})["time"].(int))
            }
        }
        if !valid_found {
            fmt.Println("Access DENIED")
            go pulse_gpio(red_led, gpiomap["red_led"].(map[interface{}]interface{})["time"].(int))
        }
        // Wait a moment before continuing with fast polling
        time.Sleep(500 * time.Millisecond)
    }
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"sort"
	"strings"
)

func WordCount(s string) map[string]int {
	substrings := strings.Fields(s)
	counts := make(map[string]int)

	for _, word := range substrings {
		if _, ok := counts[word]; ok {
			counts[word] += 1
		} else {
			counts[word] = 1
		}
	}

	return counts
}

func main() {
	if bytes, err := ioutil.ReadAll(os.Stdin); err != nil {
		panic(err)
	} else {
		m := WordCount(string(bytes))
		var words []string
		for word := range m {
			words = append(words, word)
		}
		sort.Strings(words)

		for _, word := range words {
			fmt.Printf("%s %d\n", word, m[word])
		}
	}
}
<commit_msg>Added regex check<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"regexp"
	"sort"
	"strings"
)

func WordCount(s string) map[string]int {
	r := regexp.MustCompile(`[a-z\']+`)
	substrings := strings.Fields(s)
	counts := make(map[string]int)

	for _, word := range substrings {
		if w := r.FindString(word); w != "" {
			if _, ok := counts[w]; ok {
				counts[w] += 1
			} else {
				counts[w] = 1
			}
		}
	}

	return counts
}

func main() {
	if bytes, err := ioutil.ReadAll(os.Stdin); err != nil {
		panic(err)
	} else {
		m := WordCount(string(bytes))
		var words []string
		for word := range m {
			words = append(words, word)
		}
		sort.Strings(words)

		for _, word := range words {
			fmt.Printf("%s %d\n", word, m[word])
		}
	}
}
<|endoftext|>