<commit_before>/*
 *
 * k6 - a next-generation load testing tool
 * Copyright (C) 2016 Load Impact
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 */

package js

import (
	"context"
	"crypto/tls"
	"encoding/json"
	"net"
	"net/http"
	"net/http/cookiejar"
	"strconv"
	"time"

	"github.com/dop251/goja"
	"github.com/loadimpact/k6/js/common"
	"github.com/loadimpact/k6/lib"
	"github.com/loadimpact/k6/lib/metrics"
	"github.com/loadimpact/k6/lib/netext"
	"github.com/loadimpact/k6/stats"
	"github.com/oxtoacart/bpool"
	"github.com/pkg/errors"
	log "github.com/sirupsen/logrus"
	"github.com/spf13/afero"
	"github.com/viki-org/dnscache"
	"golang.org/x/net/http2"
	"golang.org/x/time/rate"
)

var errInterrupt = errors.New("context cancelled")

type Runner struct {
	Bundle       *Bundle
	Logger       *log.Logger
	defaultGroup *lib.Group

	BaseDialer net.Dialer
	Resolver   *dnscache.Resolver
	RPSLimit   *rate.Limiter

	setupData interface{}
}

func New(src *lib.SourceData, fs afero.Fs, rtOpts lib.RuntimeOptions) (*Runner, error) {
	bundle, err := NewBundle(src, fs, rtOpts)
	if err != nil {
		return nil, err
	}
	return NewFromBundle(bundle)
}

func NewFromArchive(arc *lib.Archive, rtOpts lib.RuntimeOptions) (*Runner, error) {
	bundle, err := NewBundleFromArchive(arc, rtOpts)
	if err != nil {
		return nil, err
	}
	return NewFromBundle(bundle)
}

func NewFromBundle(b *Bundle) (*Runner, error) {
	defaultGroup, err := lib.NewGroup("", nil)
	if err != nil {
		return nil, err
	}

	r := &Runner{
		Bundle:       b,
		Logger:       log.StandardLogger(),
		defaultGroup: defaultGroup,
		BaseDialer: net.Dialer{
			Timeout:   30 * time.Second,
			KeepAlive: 30 * time.Second,
			DualStack: true,
		},
		Resolver: dnscache.New(0),
	}
	r.SetOptions(r.Bundle.Options)
	return r, nil
}

func (r *Runner) MakeArchive() *lib.Archive {
	return r.Bundle.MakeArchive()
}

func (r *Runner) NewVU() (lib.VU, error) {
	vu, err := r.newVU()
	if err != nil {
		return nil, err
	}
	return lib.VU(vu), nil
}

func (r *Runner) newVU() (*VU, error) {
	// Instantiate a new bundle, make a VU out of it.
	bi, err := r.Bundle.Instantiate()
	if err != nil {
		return nil, err
	}

	var cipherSuites []uint16
	if r.Bundle.Options.TLSCipherSuites != nil {
		cipherSuites = *r.Bundle.Options.TLSCipherSuites
	}

	var tlsVersions lib.TLSVersions
	if r.Bundle.Options.TLSVersion != nil {
		tlsVersions = *r.Bundle.Options.TLSVersion
	}

	tlsAuth := r.Bundle.Options.TLSAuth
	certs := make([]tls.Certificate, len(tlsAuth))
	nameToCert := make(map[string]*tls.Certificate)
	for i, auth := range tlsAuth {
		for _, name := range auth.Domains {
			cert, err := auth.Certificate()
			if err != nil {
				return nil, err
			}
			certs[i] = *cert
			nameToCert[name] = &certs[i]
		}
	}

	dialer := &netext.Dialer{
		Dialer:    r.BaseDialer,
		Resolver:  r.Resolver,
		Blacklist: r.Bundle.Options.BlacklistIPs,
		Hosts:     r.Bundle.Options.Hosts,
	}
	transport := &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		TLSClientConfig: &tls.Config{
			InsecureSkipVerify: r.Bundle.Options.InsecureSkipTLSVerify.Bool,
			CipherSuites:       cipherSuites,
			MinVersion:         uint16(tlsVersions.Min),
			MaxVersion:         uint16(tlsVersions.Max),
			Certificates:       certs,
			NameToCertificate:  nameToCert,
			Renegotiation:      tls.RenegotiateFreelyAsClient,
		},
		DialContext:        dialer.DialContext,
		DisableCompression: true,
	}
	_ = http2.ConfigureTransport(transport)

	vu := &VU{
		BundleInstance: *bi,
		Runner:         r,
		HTTPTransport:  transport,
		Dialer:         dialer,
		Console:        NewConsole(),
		BPool:          bpool.NewBufferPool(100),
	}
	vu.setupData = vu.Runtime.ToValue(r.setupData)
	vu.Runtime.Set("console", common.Bind(vu.Runtime, vu.Console, vu.Context))

	// Give the VU an initial sense of identity.
	if err := vu.Reconfigure(0); err != nil {
		return nil, err
	}

	return vu, nil
}

func (r *Runner) Setup(ctx context.Context) error {
	v, err := r.runPart(ctx, "setup", nil)
	if err != nil {
		return errors.Wrap(err, "setup")
	}
	data, err := json.Marshal(v.Export())
	if err != nil {
		return errors.Wrap(err, "setup")
	}
	return json.Unmarshal(data, &r.setupData)
}

func (r *Runner) Teardown(ctx context.Context) error {
	_, err := r.runPart(ctx, "teardown", r.setupData)
	return err
}

func (r *Runner) GetDefaultGroup() *lib.Group {
	return r.defaultGroup
}

func (r *Runner) GetOptions() lib.Options {
	return r.Bundle.Options
}

func (r *Runner) SetOptions(opts lib.Options) {
	r.Bundle.Options = opts

	r.RPSLimit = nil
	if rps := opts.RPS; rps.Valid {
		r.RPSLimit = rate.NewLimiter(rate.Limit(rps.Int64), 1)
	}
}

// Runs an exported function in its own temporary VU, optionally with an argument. Execution is
// interrupted if the context expires. No error is returned if the part does not exist.
func (r *Runner) runPart(ctx context.Context, name string, arg interface{}) (goja.Value, error) {
	vu, err := r.newVU()
	if err != nil {
		return goja.Undefined(), err
	}
	exp := vu.Runtime.Get("exports").ToObject(vu.Runtime)
	if exp == nil {
		return goja.Undefined(), nil
	}
	fn, ok := goja.AssertFunction(exp.Get(name))
	if !ok {
		return goja.Undefined(), nil
	}

	ctx, cancel := context.WithCancel(ctx)
	go func() {
		<-ctx.Done()
		vu.Runtime.Interrupt(errInterrupt)
	}()
	v, _, err := vu.runFn(ctx, fn, vu.Runtime.ToValue(arg))
	cancel()
	return v, err
}

type VU struct {
	BundleInstance

	Runner        *Runner
	HTTPTransport *http.Transport
	Dialer        *netext.Dialer
	ID            int64
	Iteration     int64

	Console *Console
	BPool   *bpool.BufferPool

	setupData goja.Value

	// A VU will track the last context it was called with for cancellation.
	// Note that interruptTrackedCtx is the context that is currently being tracked, while
	// interruptCancel cancels an unrelated context that terminates the tracking goroutine
	// without triggering an interrupt (for if the context changes).
	// There are cleaner ways of handling the interruption problem, but this is a hot path that
	// needs to be called thousands of times per second, which rules out anything that spawns a
	// goroutine per call.
	interruptTrackedCtx context.Context
	interruptCancel     context.CancelFunc
}

func (u *VU) Reconfigure(id int64) error {
	u.ID = id
	u.Iteration = 0
	u.Runtime.Set("__VU", u.ID)
	return nil
}

func (u *VU) RunOnce(ctx context.Context) ([]stats.Sample, error) {
	// Track the context and interrupt JS execution if it's cancelled.
	if u.interruptTrackedCtx != ctx {
		interCtx, interCancel := context.WithCancel(context.Background())
		if u.interruptCancel != nil {
			u.interruptCancel()
		}
		u.interruptCancel = interCancel
		u.interruptTrackedCtx = ctx
		go func() {
			select {
			case <-interCtx.Done():
			case <-ctx.Done():
				u.Runtime.Interrupt(errInterrupt)
			}
		}()
	}

	// Call the default function.
	_, state, err := u.runFn(ctx, u.Default, u.setupData)
	if err != nil {
		return nil, err
	}
	return state.Samples, nil
}

func (u *VU) runFn(ctx context.Context, fn goja.Callable, args ...goja.Value) (goja.Value, *common.State, error) {
	cookieJar, err := cookiejar.New(nil)
	if err != nil {
		return goja.Undefined(), nil, err
	}

	state := &common.State{
		Logger:        u.Runner.Logger,
		Options:       u.Runner.Bundle.Options,
		Group:         u.Runner.defaultGroup,
		HTTPTransport: u.HTTPTransport,
		Dialer:        u.Dialer,
		CookieJar:     cookieJar,
		RPSLimit:      u.Runner.RPSLimit,
		BPool:         u.BPool,
		Vu:            u.ID,
		Iteration:     u.Iteration,
	}
	// Zero out the values, since we may be reusing a connection
	u.Dialer.BytesRead = 0
	u.Dialer.BytesWritten = 0

	newctx := common.WithRuntime(ctx, u.Runtime)
	newctx = common.WithState(newctx, state)
	*u.Context = newctx

	u.Runtime.Set("__ITER", u.Iteration)
	iter := u.Iteration
	u.Iteration++

	startTime := time.Now()
	v, err := fn(goja.Undefined(), args...) // Actually run the JS script
	t := time.Now()

	tags := map[string]string{}
	if state.Options.SystemTags["vu"] {
		tags["vu"] = strconv.FormatInt(u.ID, 10)
	}
	if state.Options.SystemTags["iter"] {
		tags["iter"] = strconv.FormatInt(iter, 10)
	}

    state.Samples := append(state.Samples,
		stats.Sample{Time: t, Metric: metrics.DataSent, Value: float64(u.Dialer.BytesWritten), Tags: tags},
		stats.Sample{Time: t, Metric: metrics.DataReceived, Value: float64(u.Dialer.BytesRead), Tags: tags},
		stats.Sample{Time: t, Metric: metrics.IterationDuration, Value: stats.D(t.Sub(startTime)), Tags: tags},
	)

	if u.Runner.Bundle.Options.NoConnectionReuse.Bool {
		u.HTTPTransport.CloseIdleConnections()
	}
	return v, state, err
}
<commit_msg>setupData is now properly populated<commit_after>/*
 *
 * k6 - a next-generation load testing tool
 * Copyright (C) 2016 Load Impact
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 */

package js

import (
	"context"
	"crypto/tls"
	"encoding/json"
	"net"
	"net/http"
	"net/http/cookiejar"
	"strconv"
	"time"

	"github.com/dop251/goja"
	"github.com/loadimpact/k6/js/common"
	"github.com/loadimpact/k6/lib"
	"github.com/loadimpact/k6/lib/metrics"
	"github.com/loadimpact/k6/lib/netext"
	"github.com/loadimpact/k6/stats"
	"github.com/oxtoacart/bpool"
	"github.com/pkg/errors"
	log "github.com/sirupsen/logrus"
	"github.com/spf13/afero"
	"github.com/viki-org/dnscache"
	"golang.org/x/net/http2"
	"golang.org/x/time/rate"
)

var errInterrupt = errors.New("context cancelled")

type Runner struct {
	Bundle       *Bundle
	Logger       *log.Logger
	defaultGroup *lib.Group

	BaseDialer net.Dialer
	Resolver   *dnscache.Resolver
	RPSLimit   *rate.Limiter

	setupData interface{}
}

func New(src *lib.SourceData, fs afero.Fs, rtOpts lib.RuntimeOptions) (*Runner, error) {
	bundle, err := NewBundle(src, fs, rtOpts)
	if err != nil {
		return nil, err
	}
	return NewFromBundle(bundle)
}

func NewFromArchive(arc *lib.Archive, rtOpts lib.RuntimeOptions) (*Runner, error) {
	bundle, err := NewBundleFromArchive(arc, rtOpts)
	if err != nil {
		return nil, err
	}
	return NewFromBundle(bundle)
}

func NewFromBundle(b *Bundle) (*Runner, error) {
	defaultGroup, err := lib.NewGroup("", nil)
	if err != nil {
		return nil, err
	}

	r := &Runner{
		Bundle:       b,
		Logger:       log.StandardLogger(),
		defaultGroup: defaultGroup,
		BaseDialer: net.Dialer{
			Timeout:   30 * time.Second,
			KeepAlive: 30 * time.Second,
			DualStack: true,
		},
		Resolver: dnscache.New(0),
	}
	r.SetOptions(r.Bundle.Options)
	return r, nil
}

func (r *Runner) MakeArchive() *lib.Archive {
	return r.Bundle.MakeArchive()
}

func (r *Runner) NewVU() (lib.VU, error) {
	vu, err := r.newVU()
	if err != nil {
		return nil, err
	}
	return lib.VU(vu), nil
}

func (r *Runner) newVU() (*VU, error) {
	// Instantiate a new bundle, make a VU out of it.
	bi, err := r.Bundle.Instantiate()
	if err != nil {
		return nil, err
	}

	var cipherSuites []uint16
	if r.Bundle.Options.TLSCipherSuites != nil {
		cipherSuites = *r.Bundle.Options.TLSCipherSuites
	}

	var tlsVersions lib.TLSVersions
	if r.Bundle.Options.TLSVersion != nil {
		tlsVersions = *r.Bundle.Options.TLSVersion
	}

	tlsAuth := r.Bundle.Options.TLSAuth
	certs := make([]tls.Certificate, len(tlsAuth))
	nameToCert := make(map[string]*tls.Certificate)
	for i, auth := range tlsAuth {
		for _, name := range auth.Domains {
			cert, err := auth.Certificate()
			if err != nil {
				return nil, err
			}
			certs[i] = *cert
			nameToCert[name] = &certs[i]
		}
	}

	dialer := &netext.Dialer{
		Dialer:    r.BaseDialer,
		Resolver:  r.Resolver,
		Blacklist: r.Bundle.Options.BlacklistIPs,
		Hosts:     r.Bundle.Options.Hosts,
	}
	transport := &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		TLSClientConfig: &tls.Config{
			InsecureSkipVerify: r.Bundle.Options.InsecureSkipTLSVerify.Bool,
			CipherSuites:       cipherSuites,
			MinVersion:         uint16(tlsVersions.Min),
			MaxVersion:         uint16(tlsVersions.Max),
			Certificates:       certs,
			NameToCertificate:  nameToCert,
			Renegotiation:      tls.RenegotiateFreelyAsClient,
		},
		DialContext:        dialer.DialContext,
		DisableCompression: true,
	}
	_ = http2.ConfigureTransport(transport)

	vu := &VU{
		BundleInstance: *bi,
		Runner:         r,
		HTTPTransport:  transport,
		Dialer:         dialer,
		Console:        NewConsole(),
		BPool:          bpool.NewBufferPool(100),
	}
	vu.Runtime.Set("console", common.Bind(vu.Runtime, vu.Console, vu.Context))

	// Give the VU an initial sense of identity.
	if err := vu.Reconfigure(0); err != nil {
		return nil, err
	}

	return vu, nil
}

func (r *Runner) Setup(ctx context.Context) error {
	v, err := r.runPart(ctx, "setup", nil)
	if err != nil {
		return errors.Wrap(err, "setup")
	}
	data, err := json.Marshal(v.Export())
	if err != nil {
		return errors.Wrap(err, "setup")
	}
	return json.Unmarshal(data, &r.setupData)
}

func (r *Runner) Teardown(ctx context.Context) error {
	_, err := r.runPart(ctx, "teardown", r.setupData)
	return err
}

func (r *Runner) GetDefaultGroup() *lib.Group {
	return r.defaultGroup
}

func (r *Runner) GetOptions() lib.Options {
	return r.Bundle.Options
}

func (r *Runner) SetOptions(opts lib.Options) {
	r.Bundle.Options = opts

	r.RPSLimit = nil
	if rps := opts.RPS; rps.Valid {
		r.RPSLimit = rate.NewLimiter(rate.Limit(rps.Int64), 1)
	}
}

// Runs an exported function in its own temporary VU, optionally with an argument. Execution is
// interrupted if the context expires. No error is returned if the part does not exist.
func (r *Runner) runPart(ctx context.Context, name string, arg interface{}) (goja.Value, error) {
	vu, err := r.newVU()
	if err != nil {
		return goja.Undefined(), err
	}
	exp := vu.Runtime.Get("exports").ToObject(vu.Runtime)
	if exp == nil {
		return goja.Undefined(), nil
	}
	fn, ok := goja.AssertFunction(exp.Get(name))
	if !ok {
		return goja.Undefined(), nil
	}

	ctx, cancel := context.WithCancel(ctx)
	go func() {
		<-ctx.Done()
		vu.Runtime.Interrupt(errInterrupt)
	}()
	v, _, err := vu.runFn(ctx, fn, vu.Runtime.ToValue(arg))
	cancel()
	return v, err
}

type VU struct {
	BundleInstance

	Runner        *Runner
	HTTPTransport *http.Transport
	Dialer        *netext.Dialer
	ID            int64
	Iteration     int64

	Console *Console
	BPool   *bpool.BufferPool

	setupData goja.Value

	// A VU will track the last context it was called with for cancellation.
	// Note that interruptTrackedCtx is the context that is currently being tracked, while
	// interruptCancel cancels an unrelated context that terminates the tracking goroutine
	// without triggering an interrupt (for if the context changes).
	// There are cleaner ways of handling the interruption problem, but this is a hot path that
	// needs to be called thousands of times per second, which rules out anything that spawns a
	// goroutine per call.
	interruptTrackedCtx context.Context
	interruptCancel     context.CancelFunc
}

func (u *VU) Reconfigure(id int64) error {
	u.ID = id
	u.Iteration = 0
	u.Runtime.Set("__VU", u.ID)
	return nil
}

func (u *VU) RunOnce(ctx context.Context) ([]stats.Sample, error) {
	// Track the context and interrupt JS execution if it's cancelled.
	if u.interruptTrackedCtx != ctx {
		interCtx, interCancel := context.WithCancel(context.Background())
		if u.interruptCancel != nil {
			u.interruptCancel()
		}
		u.interruptCancel = interCancel
		u.interruptTrackedCtx = ctx
		go func() {
			select {
			case <-interCtx.Done():
			case <-ctx.Done():
				u.Runtime.Interrupt(errInterrupt)
			}
		}()
	}

	// Lazily JS-ify setupData on first run. This is lightweight enough that we can get away with
	// it, and alleviates a problem where setupData wouldn't get populated properly if NewVU() was
	// called before Setup(), which is hard to avoid with how the Executor works w/o complicating
	// the local executor further by deferring SetVUsMax() calls to within the Run() function.
	if u.setupData == nil && u.Runner.setupData != nil {
		u.setupData = u.Runtime.ToValue(u.Runner.setupData)
	}

	// Call the default function.
	_, state, err := u.runFn(ctx, u.Default, u.setupData)
	if err != nil {
		return nil, err
	}
	return state.Samples, nil
}

func (u *VU) runFn(ctx context.Context, fn goja.Callable, args ...goja.Value) (goja.Value, *common.State, error) {
	cookieJar, err := cookiejar.New(nil)
	if err != nil {
		return goja.Undefined(), nil, err
	}

	state := &common.State{
		Logger:        u.Runner.Logger,
		Options:       u.Runner.Bundle.Options,
		Group:         u.Runner.defaultGroup,
		HTTPTransport: u.HTTPTransport,
		Dialer:        u.Dialer,
		CookieJar:     cookieJar,
		RPSLimit:      u.Runner.RPSLimit,
		BPool:         u.BPool,
		Vu:            u.ID,
		Iteration:     u.Iteration,
	}
	// Zero out the values, since we may be reusing a connection
	u.Dialer.BytesRead = 0
	u.Dialer.BytesWritten = 0

	newctx := common.WithRuntime(ctx, u.Runtime)
	newctx = common.WithState(newctx, state)
	*u.Context = newctx

	u.Runtime.Set("__ITER", u.Iteration)
	iter := u.Iteration
	u.Iteration++

	startTime := time.Now()
	v, err := fn(goja.Undefined(), args...) // Actually run the JS script
	t := time.Now()

	tags := map[string]string{}
	if state.Options.SystemTags["vu"] {
		tags["vu"] = strconv.FormatInt(u.ID, 10)
	}
	if state.Options.SystemTags["iter"] {
		tags["iter"] = strconv.FormatInt(iter, 10)
	}

    state.Samples := append(state.Samples,
		stats.Sample{Time: t, Metric: metrics.DataSent, Value: float64(u.Dialer.BytesWritten), Tags: tags},
		stats.Sample{Time: t, Metric: metrics.DataReceived, Value: float64(u.Dialer.BytesRead), Tags: tags},
		stats.Sample{Time: t, Metric: metrics.IterationDuration, Value: stats.D(t.Sub(startTime)), Tags: tags},
	)

	if u.Runner.Bundle.Options.NoConnectionReuse.Bool {
		u.HTTPTransport.CloseIdleConnections()
	}
	return v, state, err
}
<|endoftext|>


<commit_before>package query

import (
	"fmt"
	"math"
	"os"
	"reflect"
	"runtime"
	"strconv"
	"strings"

	"github.com/mithrandie/csvq/lib/cmd"
	"github.com/mithrandie/csvq/lib/parser"
	"github.com/mithrandie/csvq/lib/ternary"
)

func InIntSlice(i int, list []int) bool {
	for _, v := range list {
		if i == v {
			return true
		}
	}
	return false
}

func InStrSliceWithCaseInsensitive(s string, list []string) bool {
	for _, v := range list {
		if strings.EqualFold(s, v) {
			return true
		}
	}
	return false
}

func InRuneSlice(r rune, list []rune) bool {
	for _, v := range list {
		if r == v {
			return true
		}
	}
	return false
}

func Distinguish(list []parser.Primary) []parser.Primary {
	var in = func(list []parser.Primary, item parser.Primary) bool {
		for _, v := range list {
			if EquivalentTo(item, v) == ternary.TRUE {
				return true
			}
		}
		return false
	}

	distinguished := []parser.Primary{}
	for _, v := range list {
		if !in(distinguished, v) {
			distinguished = append(distinguished, v)
		}
	}
	return distinguished
}

func FormatCount(i int, obj string) string {
	var s string
	if i == 0 {
		s = fmt.Sprintf("no %s", obj)
	} else if i == 1 {
		s = fmt.Sprintf("%d %s", i, obj)
	} else {
		s = fmt.Sprintf("%d %ss", i, obj)
	}
	return s
}

func IsReadableFromStdin() bool {
	fi, err := os.Stdin.Stat()
	if err == nil && (fi.Mode()&os.ModeNamedPipe != 0 || 0 < fi.Size()) {
		return true
	}
	return false
}

func FormatString(format string, args []parser.Primary) (string, error) {
	var pad = func(s string, length int, flags []rune) string {
		if length <= len(s) {
			return s
		}

		padchar := " "
		if InRuneSlice('0', flags) {
			padchar = "0"
		}
		padstr := strings.Repeat(padchar, length-len(s))
		if InRuneSlice('-', flags) {
			s = s + padstr
		} else {
			s = padstr + s
		}
		return s
	}

	var numberSign = func(value float64, flags []rune) string {
		sign := ""
		if value < 0 {
			sign = "-"
		} else {
			switch {
			case InRuneSlice('+', flags):
				sign = "+"
			case InRuneSlice(' ', flags):
				sign = " "
			}
		}
		return sign
	}

	str := []rune{}

	escaped := false
	placeholderOrder := 0
	flags := []rune{}
	var length string
	var precision string
	var isPrecision bool
	for _, r := range format {
		if escaped {
			if isPrecision {
				if '0' <= r && r <= '9' {
					precision += string(r)
					continue
				} else {
					isPrecision = false
				}
			}

			if 0 < len(length) && '0' <= r && r <= '9' {
				length += string(r)
				continue
			}

			switch r {
			case '+', '-', ' ', '0':
				flags = append(flags, r)
				continue
			case '1', '2', '3', '4', '5', '6', '7', '8', '9':
				length = string(r)
				continue
			case '.':
				isPrecision = true
				continue
			case 'b', 'o', 'd', 'x', 'X', 'e', 'E', 'f', 's', 'q', 'T':
				if len(args) <= placeholderOrder {
					return "", NewFormatStringLengthNotMatchError()
				}

				switch r {
				case 'b', 'o', 'd', 'x', 'X':
					p := parser.PrimaryToInteger(args[placeholderOrder])
					if !parser.IsNull(p) {
						value := float64(p.(parser.Integer).Value())
						sign := numberSign(value, flags)
						i := int64(math.Abs(value))
						var s string
						switch r {
						case 'b':
							s = strconv.FormatInt(i, 2)
						case 'o':
							s = strconv.FormatInt(i, 8)
						case 'd':
							s = strconv.FormatInt(i, 10)
						case 'x':
							s = strconv.FormatInt(i, 16)
						case 'X':
							s = strings.ToUpper(strconv.FormatInt(i, 16))
						}
						l, _ := strconv.Atoi(length)
						s = sign + pad(s, l-len(sign), flags)
						str = append(str, []rune(s)...)
					}
				case 'e', 'E', 'f':
					p := parser.PrimaryToFloat(args[placeholderOrder])
					if !parser.IsNull(p) {
						value := p.(parser.Float).Value()

						var prec float64
						if 0 < len(precision) {
							prec, _ = strconv.ParseFloat(precision, 64)
							value = round(value, prec)
						}
						sign := numberSign(value, flags)
						f := math.Abs(value)
						s := strconv.FormatFloat(f, byte(r), -1, 64)

						if 0 < prec {
							parts := strings.Split(s, ".")
							intpart := parts[0]
							var dec string
							var en string
							if len(parts) < 2 {
								dec = ""
							} else {
								dec = parts[1]
							}
							if r != 'f' {
								if 0 < len(dec) {
									enidx := strings.Index(dec, string(r))
									en = dec[enidx:]
									dec = dec[:enidx]
								} else {
									enidx := strings.Index(intpart, string(r))
									en = intpart[enidx:]
									intpart = intpart[:enidx]
								}
							}
							if len(dec) < int(prec) {
								dec = dec + strings.Repeat("0", int(prec)-len(dec))
								s = intpart + "." + dec + en
							}
						}

						l, _ := strconv.Atoi(length)
						s = sign + pad(s, l-len(sign), flags)
						str = append(str, []rune(s)...)
					}
				case 's':
					var s string
					switch args[placeholderOrder].(type) {
					case parser.String:
						s = args[placeholderOrder].(parser.String).Value()
					case parser.Integer:
						s = parser.Int64ToStr(args[placeholderOrder].(parser.Integer).Value())
					case parser.Float:
						s = parser.Float64ToStr(args[placeholderOrder].(parser.Float).Value())
					case parser.Boolean:
						s = strconv.FormatBool(args[placeholderOrder].(parser.Boolean).Value())
					case parser.Ternary:
						s = args[placeholderOrder].(parser.Ternary).Ternary().String()
					case parser.Datetime:
						s = args[placeholderOrder].(parser.Datetime).Format()
					case parser.Null:
						s = "NULL"
					}
					l, _ := strconv.Atoi(length)
					s = pad(s, l, flags)
					str = append(str, []rune(s)...)
				case 'q':
					str = append(str, []rune(args[placeholderOrder].String())...)
				case 'T':
					str = append(str, []rune(reflect.TypeOf(args[placeholderOrder]).Name())...)
				}

				placeholderOrder++
			case '%':
				str = append(str, r)
			default:
				str = append(str, '%', r)
			}

			escaped = false
			flags = []rune{}
			length = ""
			precision = ""
			isPrecision = false
			continue
		}

		if r == '%' {
			escaped = true
			continue
		}

		str = append(str, r)
	}
	if escaped {
		str = append(str, '%')
	}

	if placeholderOrder < len(args) {
		return "", NewFormatStringLengthNotMatchError()
	}

	return string(str), nil
}
<commit_msg>Remove unused packages.<commit_after>package query

import (
	"fmt"
	"math"
	"os"
	"reflect"
	"strconv"
	"strings"

	"github.com/mithrandie/csvq/lib/parser"
	"github.com/mithrandie/csvq/lib/ternary"
)

func InIntSlice(i int, list []int) bool {
	for _, v := range list {
		if i == v {
			return true
		}
	}
	return false
}

func InStrSliceWithCaseInsensitive(s string, list []string) bool {
	for _, v := range list {
		if strings.EqualFold(s, v) {
			return true
		}
	}
	return false
}

func InRuneSlice(r rune, list []rune) bool {
	for _, v := range list {
		if r == v {
			return true
		}
	}
	return false
}

func Distinguish(list []parser.Primary) []parser.Primary {
	var in = func(list []parser.Primary, item parser.Primary) bool {
		for _, v := range list {
			if EquivalentTo(item, v) == ternary.TRUE {
				return true
			}
		}
		return false
	}

	distinguished := []parser.Primary{}
	for _, v := range list {
		if !in(distinguished, v) {
			distinguished = append(distinguished, v)
		}
	}
	return distinguished
}

func FormatCount(i int, obj string) string {
	var s string
	if i == 0 {
		s = fmt.Sprintf("no %s", obj)
	} else if i == 1 {
		s = fmt.Sprintf("%d %s", i, obj)
	} else {
		s = fmt.Sprintf("%d %ss", i, obj)
	}
	return s
}

func IsReadableFromStdin() bool {
	fi, err := os.Stdin.Stat()
	if err == nil && (fi.Mode()&os.ModeNamedPipe != 0 || 0 < fi.Size()) {
		return true
	}
	return false
}

func FormatString(format string, args []parser.Primary) (string, error) {
	var pad = func(s string, length int, flags []rune) string {
		if length <= len(s) {
			return s
		}

		padchar := " "
		if InRuneSlice('0', flags) {
			padchar = "0"
		}
		padstr := strings.Repeat(padchar, length-len(s))
		if InRuneSlice('-', flags) {
			s = s + padstr
		} else {
			s = padstr + s
		}
		return s
	}

	var numberSign = func(value float64, flags []rune) string {
		sign := ""
		if value < 0 {
			sign = "-"
		} else {
			switch {
			case InRuneSlice('+', flags):
				sign = "+"
			case InRuneSlice(' ', flags):
				sign = " "
			}
		}
		return sign
	}

	str := []rune{}

	escaped := false
	placeholderOrder := 0
	flags := []rune{}
	var length string
	var precision string
	var isPrecision bool
	for _, r := range format {
		if escaped {
			if isPrecision {
				if '0' <= r && r <= '9' {
					precision += string(r)
					continue
				} else {
					isPrecision = false
				}
			}

			if 0 < len(length) && '0' <= r && r <= '9' {
				length += string(r)
				continue
			}

			switch r {
			case '+', '-', ' ', '0':
				flags = append(flags, r)
				continue
			case '1', '2', '3', '4', '5', '6', '7', '8', '9':
				length = string(r)
				continue
			case '.':
				isPrecision = true
				continue
			case 'b', 'o', 'd', 'x', 'X', 'e', 'E', 'f', 's', 'q', 'T':
				if len(args) <= placeholderOrder {
					return "", NewFormatStringLengthNotMatchError()
				}

				switch r {
				case 'b', 'o', 'd', 'x', 'X':
					p := parser.PrimaryToInteger(args[placeholderOrder])
					if !parser.IsNull(p) {
						value := float64(p.(parser.Integer).Value())
						sign := numberSign(value, flags)
						i := int64(math.Abs(value))
						var s string
						switch r {
						case 'b':
							s = strconv.FormatInt(i, 2)
						case 'o':
							s = strconv.FormatInt(i, 8)
						case 'd':
							s = strconv.FormatInt(i, 10)
						case 'x':
							s = strconv.FormatInt(i, 16)
						case 'X':
							s = strings.ToUpper(strconv.FormatInt(i, 16))
						}
						l, _ := strconv.Atoi(length)
						s = sign + pad(s, l-len(sign), flags)
						str = append(str, []rune(s)...)
					}
				case 'e', 'E', 'f':
					p := parser.PrimaryToFloat(args[placeholderOrder])
					if !parser.IsNull(p) {
						value := p.(parser.Float).Value()

						var prec float64
						if 0 < len(precision) {
							prec, _ = strconv.ParseFloat(precision, 64)
							value = round(value, prec)
						}
						sign := numberSign(value, flags)
						f := math.Abs(value)
						s := strconv.FormatFloat(f, byte(r), -1, 64)

						if 0 < prec {
							parts := strings.Split(s, ".")
							intpart := parts[0]
							var dec string
							var en string
							if len(parts) < 2 {
								dec = ""
							} else {
								dec = parts[1]
							}
							if r != 'f' {
								if 0 < len(dec) {
									enidx := strings.Index(dec, string(r))
									en = dec[enidx:]
									dec = dec[:enidx]
								} else {
									enidx := strings.Index(intpart, string(r))
									en = intpart[enidx:]
									intpart = intpart[:enidx]
								}
							}
							if len(dec) < int(prec) {
								dec = dec + strings.Repeat("0", int(prec)-len(dec))
								s = intpart + "." + dec + en
							}
						}

						l, _ := strconv.Atoi(length)
						s = sign + pad(s, l-len(sign), flags)
						str = append(str, []rune(s)...)
					}
				case 's':
					var s string
					switch args[placeholderOrder].(type) {
					case parser.String:
						s = args[placeholderOrder].(parser.String).Value()
					case parser.Integer:
						s = parser.Int64ToStr(args[placeholderOrder].(parser.Integer).Value())
					case parser.Float:
						s = parser.Float64ToStr(args[placeholderOrder].(parser.Float).Value())
					case parser.Boolean:
						s = strconv.FormatBool(args[placeholderOrder].(parser.Boolean).Value())
					case parser.Ternary:
						s = args[placeholderOrder].(parser.Ternary).Ternary().String()
					case parser.Datetime:
						s = args[placeholderOrder].(parser.Datetime).Format()
					case parser.Null:
						s = "NULL"
					}
					l, _ := strconv.Atoi(length)
					s = pad(s, l, flags)
					str = append(str, []rune(s)...)
				case 'q':
					str = append(str, []rune(args[placeholderOrder].String())...)
				case 'T':
					str = append(str, []rune(reflect.TypeOf(args[placeholderOrder]).Name())...)
				}

				placeholderOrder++
			case '%':
				str = append(str, r)
			default:
				str = append(str, '%', r)
			}

			escaped = false
			flags = []rune{}
			length = ""
			precision = ""
			isPrecision = false
			continue
		}

		if r == '%' {
			escaped = true
			continue
		}

		str = append(str, r)
	}
	if escaped {
		str = append(str, '%')
	}

	if placeholderOrder < len(args) {
		return "", NewFormatStringLengthNotMatchError()
	}

	return string(str), nil
}
<|endoftext|>


<commit_before>package datastore

import (
    "fmt"
    "github.com/alxeg/flibooks/models"
    "github.com/alxeg/flibooks/utils"
    "github.com/jinzhu/gorm"
    _ "github.com/mattn/go-sqlite3"
    "os"
    "strings"
)

type dbStore struct {
    db    gorm.DB
    reset bool
}

func (store *dbStore) PutBook(book *models.Book) (err error) {
    tx := store.db.Begin()

    store.db.FirstOrCreate(&book.Container, book.Container)
    authors := []models.Author{}
    for _, author := range book.Authors {
        filledAuthor := models.Author{}
        store.db.FirstOrCreate(&filledAuthor, author)
        authors = append(authors, filledAuthor)
    }
    book.Authors = authors

    genres := []models.Genre{}
    for _, genre := range book.Genres {
        filledGenre := models.Genre{}
        store.db.FirstOrCreate(&filledGenre, genre)
        genres = append(genres, filledGenre)
    }
    book.Genres = genres

    store.db.Create(&book)

    tx.Commit()

    return err
}

func (store *dbStore) fillBookDetails(book *models.Book) {
    store.db.Select("authors.*").Model(book).Related(&book.Authors, "Authors")
    for j, a := range book.Authors {
        book.Authors[j].Name = utils.UpperInitialAll(a.Name)
    }
    store.db.Select("genres.*").Model(book).Related(&book.Genres, "Genres")
}

func (store *dbStore) fillBooksDetails(books []models.Book) []models.Book {
    for i, _ := range books {
        store.fillBookDetails(&books[i])
    }

    return books
}

func (store *dbStore) FindBooks(title string, authors string, limit int) ([]models.Book, error) {

    result := []models.Book{}
    search := store.db.Select("books.*").Table("books").
        Joins("left join book_authors on books.id=book_authors.book_id left join authors on authors.id=book_authors.author_id")
    for _, term := range utils.SplitBySeparators(strings.ToLower(title)) {
        search = search.Where("title LIKE ?", "%"+term+"%")
    }
    for _, term := range utils.SplitBySeparators(strings.ToLower(authors)) {
        search = search.Where("name LIKE ?", "%"+term+"%")
    }
    if limit > 0 {
        search = search.Limit(limit)
    }
    search.Preload("Container").Order("title").Find(&result)

    result = store.fillBooksDetails(result)
    return result, nil
}

func (store *dbStore) FindAuthors(author string, limit int) ([]models.Author, error) {
    result := []models.Author{}
    search := store.db.Order("name")
    for _, term := range utils.SplitBySeparators(strings.ToLower(author)) {
        search = search.Where("name LIKE ?", "%"+term+"%")
    }
    if limit > 0 {
        search = search.Limit(limit)
    }
    search.Find(&result)
    for i, a := range result {
        result[i].Name = utils.UpperInitialAll(a.Name)
    }
    return result, nil
}

func (store *dbStore) GetAuthor(authorId uint) (*models.Author, error) {
    result := new(models.Author)
    store.db.First(result, authorId)
    if result.ID > 0 {
        result.Name = utils.UpperInitialAll(result.Name)
        return result, nil
    } else {
        return nil, fmt.Errorf("No author found")
    }
}

func (store *dbStore) ListAuthorBooks(authorId uint) ([]models.Book, error) {
    result := []models.Book{}
    search := store.db.Select("books.*").Table("books").
        Joins("left join book_authors on books.id=book_authors.book_id left join authors on authors.id=book_authors.author_id")
    search.Where("authors.ID=?", authorId).Preload("Container").Order("title").Find(&result)
    result = store.fillBooksDetails(result)
    return result, nil
}

func (store *dbStore) GetBook(bookId uint) (*models.Book, error) {
    result := new(models.Book)
    store.db.Preload("Container").First(result, bookId)
    store.fillBookDetails(result)
    if result.ID > 0 {
        return result, nil
    } else {
        return nil, fmt.Errorf("No book found")
    }
}

func (store *dbStore) Close() {
    if store.reset {
    }
}

func NewDBStore(dbPath string, reset bool) (DataStorer, error) {
    dataPath := dbPath + "/fli-data.db"
    if reset {
        os.Remove(dataPath)
    }
    db, err := gorm.Open("sqlite3", dataPath)
    if err == nil {
        db.DB()
        db.AutoMigrate(&models.Author{}, &models.Container{}, &models.Genre{}, &models.Book{})
        // db.LogMode(true)
    }
    result := new(dbStore)
    result.db = db
    result.reset = reset

    return result, err
}
<commit_msg>Add distinct to avoid duplicate entries<commit_after>package datastore

import (
    "fmt"
    "github.com/alxeg/flibooks/models"
    "github.com/alxeg/flibooks/utils"
    "github.com/jinzhu/gorm"
    _ "github.com/mattn/go-sqlite3"
    "os"
    "strings"
)

type dbStore struct {
    db    gorm.DB
    reset bool
}

func (store *dbStore) PutBook(book *models.Book) (err error) {
    tx := store.db.Begin()

    store.db.FirstOrCreate(&book.Container, book.Container)
    authors := []models.Author{}
    for _, author := range book.Authors {
        filledAuthor := models.Author{}
        store.db.FirstOrCreate(&filledAuthor, author)
        authors = append(authors, filledAuthor)
    }
    book.Authors = authors

    genres := []models.Genre{}
    for _, genre := range book.Genres {
        filledGenre := models.Genre{}
        store.db.FirstOrCreate(&filledGenre, genre)
        genres = append(genres, filledGenre)
    }
    book.Genres = genres

    store.db.Create(&book)

    tx.Commit()

    return err
}

func (store *dbStore) fillBookDetails(book *models.Book) {
    store.db.Select("authors.*").Model(book).Related(&book.Authors, "Authors")
    for j, a := range book.Authors {
        book.Authors[j].Name = utils.UpperInitialAll(a.Name)
    }
    store.db.Select("genres.*").Model(book).Related(&book.Genres, "Genres")
}

func (store *dbStore) fillBooksDetails(books []models.Book) []models.Book {
    for i, _ := range books {
        store.fillBookDetails(&books[i])
    }

    return books
}

func (store *dbStore) FindBooks(title string, authors string, limit int) ([]models.Book, error) {

    result := []models.Book{}
    search := store.db.Select("distinct books.*").Table("books").
        Joins("left join book_authors on books.id=book_authors.book_id left join authors on authors.id=book_authors.author_id")
    for _, term := range utils.SplitBySeparators(strings.ToLower(title)) {
        search = search.Where("title LIKE ?", "%"+term+"%")
    }
    for _, term := range utils.SplitBySeparators(strings.ToLower(authors)) {
        search = search.Where("name LIKE ?", "%"+term+"%")
    }
    if limit > 0 {
        search = search.Limit(limit)
    }
    search.Preload("Container").Order("title").Find(&result)

    result = store.fillBooksDetails(result)
    return result, nil
}

func (store *dbStore) FindAuthors(author string, limit int) ([]models.Author, error) {
    result := []models.Author{}
    search := store.db.Order("name")
    for _, term := range utils.SplitBySeparators(strings.ToLower(author)) {
        search = search.Where("name LIKE ?", "%"+term+"%")
    }
    if limit > 0 {
        search = search.Limit(limit)
    }
    search.Find(&result)
    for i, a := range result {
        result[i].Name = utils.UpperInitialAll(a.Name)
    }
    return result, nil
}

func (store *dbStore) GetAuthor(authorId uint) (*models.Author, error) {
    result := new(models.Author)
    store.db.First(result, authorId)
    if result.ID > 0 {
        result.Name = utils.UpperInitialAll(result.Name)
        return result, nil
    } else {
        return nil, fmt.Errorf("No author found")
    }
}

func (store *dbStore) ListAuthorBooks(authorId uint) ([]models.Book, error) {
    result := []models.Book{}
    search := store.db.Select("books.*").Table("books").
        Joins("left join book_authors on books.id=book_authors.book_id left join authors on authors.id=book_authors.author_id")
    search.Where("authors.ID=?", authorId).Preload("Container").Order("title").Find(&result)
    result = store.fillBooksDetails(result)
    return result, nil
}

func (store *dbStore) GetBook(bookId uint) (*models.Book, error) {
    result := new(models.Book)
    store.db.Preload("Container").First(result, bookId)
    store.fillBookDetails(result)
    if result.ID > 0 {
        return result, nil
    } else {
        return nil, fmt.Errorf("No book found")
    }
}

func (store *dbStore) Close() {
    if store.reset {
    }
}

func NewDBStore(dbPath string, reset bool) (DataStorer, error) {
    dataPath := dbPath + "/fli-data.db"
    if reset {
        os.Remove(dataPath)
    }
    db, err := gorm.Open("sqlite3", dataPath)
    if err == nil {
        db.DB()
        db.AutoMigrate(&models.Author{}, &models.Container{}, &models.Genre{}, &models.Book{})
        // db.LogMode(true)
    }
    result := new(dbStore)
    result.db = db
    result.reset = reset

    return result, err
}
<|endoftext|>


<commit_before>package xunyu

import (
	"fmt"
	"sync"

	"github.com/xunyu/common"
	"github.com/xunyu/config"
	"github.com/xunyu/lib/plugins"
)

type Xunyu struct {
	Path    string
	Config  XunyuConfig
	Version string
	Plugins *common.Plugins
}

type XunyuConfig struct {
	Inputs   map[string]*config.Config `config:"inputs"`
	Outputs  map[string]*config.Config `config:"outputs"`
	Channels map[string]*config.Config `config:"channels"`
}

func Run(path string) error {
	xy := newXunyu("")

	if err := xy.configure(path); err != nil {
		return err
	}

	err := xy.init()
	if nil != err {
		return err
	}

	return xy.Run()
}

func newXunyu(version string) *Xunyu {
	if version == "" {
		version = defaultXunyuVersion
	}

	return &Xunyu{
		Version: version,
	}
}

func (xy *Xunyu) init() error {
	p, err := plugins.LoadPlugins(xy.Config.Inputs, xy.Config.Outputs, xy.Config.Channels)

	if nil != err {
		return err
	}

	xy.Plugins = p
	return nil
}

func (xy *Xunyu) configure(path string) error {
	xy.Path = path
	cfg, err := config.Load(xy.Path)
	if nil != err {
		return err
	}

	if err := cfg.Assemble(&xy.Config); nil != err {
		return err
	}

	return nil
}

func (xy *Xunyu) Run() error {
	in := runInput(xy.Plugins.Inputs)
	ch := runChannel(xy.Plugins.Channels, in)
	runOutput(xy.Plugins.Outputs, ch)
	return nil
}

func runInput(inputs []common.Plugin) <-chan common.DataInter {
	fmt.Println("Starting Input")

	out := make(chan common.DataInter, 1)

	var wg sync.WaitGroup
	wg.Add(len(inputs))
	for _, p := range inputs {
		go func(p common.Plugin) {
			defer wg.Done()
			o := p.Plugin.Start()
			for data := range o {
				out <- data
			}
		}(p)
	}

	go func() {
		wg.Wait()
		close(out)
	}()

	return out
}

func runChannel(channels []common.Plugin, in <-chan common.DataInter) <-chan common.DataStr {
	fmt.Println("Starting Channel")

	out := make(chan common.DataStr, 1)
	var wg sync.WaitGroup

	filter := func(p common.Plugin) {
		defer wg.Done()
		p.Plugin.Filter(out)
		for data := range in {
			p.Plugin.GetFilterChannel() <- data
		}
	}

	wg.Add(len(channels))
	for _, p := range channels {
		go filter(p)
	}

	go func() {
		wg.Wait()
		close(out)
	}()

	return out
}

func runOutput(outputs []common.Plugin, cs <-chan common.DataStr) {
	fmt.Println("Starting Output")
	defer fmt.Println("Stopped Output")

	var wg sync.WaitGroup

	output := func(p common.Plugin) {
		defer wg.Done()
		for data := range cs {
			p.Plugin.Output(data)
		}
	}

	wg.Add(len(outputs))
	for _, p := range outputs {
		go output(p)
	}

	wg.Wait()
}
<commit_msg>Use select instead of for-range to block executing<commit_after>package xunyu

import (
	"fmt"
	"sync"

	"github.com/xunyu/common"
	"github.com/xunyu/config"
	"github.com/xunyu/lib/plugins"
)

type Xunyu struct {
	Path    string
	Config  XunyuConfig
	Version string
	Plugins *common.Plugins
}

type XunyuConfig struct {
	Inputs   map[string]*config.Config `config:"inputs"`
	Outputs  map[string]*config.Config `config:"outputs"`
	Channels map[string]*config.Config `config:"channels"`
}

func Run(path string) error {
	xy := newXunyu("")

	if err := xy.configure(path); err != nil {
		return err
	}

	err := xy.init()
	if nil != err {
		return err
	}

	return xy.Run()
}

func newXunyu(version string) *Xunyu {
	if version == "" {
		version = defaultXunyuVersion
	}

	return &Xunyu{
		Version: version,
	}
}

func (xy *Xunyu) init() error {
	p, err := plugins.LoadPlugins(xy.Config.Inputs, xy.Config.Outputs, xy.Config.Channels)

	if nil != err {
		return err
	}

	xy.Plugins = p
	return nil
}

func (xy *Xunyu) configure(path string) error {
	xy.Path = path
	cfg, err := config.Load(xy.Path)
	if nil != err {
		return err
	}

	if err := cfg.Assemble(&xy.Config); nil != err {
		return err
	}

	return nil
}

func (xy *Xunyu) Run() error {
	in := runInput(xy.Plugins.Inputs)
	ch := runChannel(xy.Plugins.Channels, in)
	runOutput(xy.Plugins.Outputs, ch)
	return nil
}

func runInput(inputs []common.Plugin) <-chan common.DataInter {
	fmt.Println("Starting Input")

	out := make(chan common.DataInter, 1)

	var wg sync.WaitGroup
	wg.Add(len(inputs))
	for _, p := range inputs {
		go func(p common.Plugin) {
			defer wg.Done()
			o := p.Plugin.Start()
			for data := range o {
				out <- data
			}
		}(p)
	}

	go func() {
		wg.Wait()
		close(out)
	}()

	return out
}

func runChannel(channels []common.Plugin, in <-chan common.DataInter) <-chan common.DataStr {
	fmt.Println("Starting Channel")

	out := make(chan common.DataStr, 1)
	var wg sync.WaitGroup

	filter := func(p common.Plugin) {
		defer wg.Done()
		p.Plugin.Filter(out)
		for data := range in {
			p.Plugin.GetFilterChannel() <- data
		}
	}

	wg.Add(len(channels))
	for _, p := range channels {
		go filter(p)
	}

	go func() {
		wg.Wait()
		close(out)
	}()

	return out
}

func runOutput(outputs []common.Plugin, cs <-chan common.DataStr) {
	fmt.Println("Starting Output")
	defer fmt.Println("Stopped Output")

	var wg sync.WaitGroup

	output := func(p common.Plugin) {
		defer wg.Done()
		for {
			select {
			case data := <-cs:
				p.Plugin.Output(data)
			}
		}
	}

	wg.Add(len(outputs))
	for _, p := range outputs {
		go output(p)
	}

	wg.Wait()
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"os"
	"strings"

	"github.com/prometheus/client_golang/prometheus"
)

type collector struct {
	sources []*source
}

type source struct {
	URL       string
	Namespace string
	Subsystem string
	Labels    map[string]string
	Keys      map[string]struct {
		Skip           bool
		MapValue       map[string]float64
		UseKeysAsLabel string
	}
}

func main() {
	sourcesJSON := os.Getenv("SOURCES")
	if sourcesJSON == "" {
		log.Print("environemnt variable SOURCES is requred")
		return
	}

	var sources []*source
	if err := json.Unmarshal([]byte(sourcesJSON), &sources); err != nil {
		log.Print(err)
		return
	}

	prometheus.MustRegister(&collector{sources})
	http.Handle("/", http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
		resp.Write([]byte(`<html><head><title>json2prom</title></head><body><h1>json2prom</h1><p><a href="/metrics">Metrics</a></p></body></html>`))
	}))
	http.Handle("/metrics", prometheus.Handler())

	addr := os.Getenv("HTTP_ADDR")
	if addr == "" {
		addr = ":8080"
	}
	fmt.Println("listening on " + addr)
	log.Print(http.ListenAndServe(addr, nil))
}

func (c *collector) Describe(descs chan<- *prometheus.Desc) {
	metrics := make(chan prometheus.Metric)
	done := make(chan struct{})
	go func() {
		for m := range metrics {
			descs <- m.Desc()
		}
		close(done)
	}()
	c.Collect(metrics)
	close(metrics)
	<-done
}

func (c *collector) Collect(metrics chan<- prometheus.Metric) {
	for _, s := range c.sources {
		resp, err := http.Get(s.URL)
		if err != nil {
			log.Print(err)
			continue
		}
		defer resp.Body.Close()

		var value interface{}
		if err := json.NewDecoder(resp.Body).Decode(&value); err != nil {
			log.Print(err)
			continue
		}

		var labelNames []string
		var labelValues []string
		for k, v := range s.Labels {
			labelNames = append(labelNames, k)
			labelValues = append(labelValues, v)
		}
		s.processValue(nil, labelNames, labelValues, value, metrics)
	}
}

func (s *source) processValue(keys []string, labelNames, labelValues []string, value interface{}, metrics chan<- prometheus.Metric) {
	switch value := value.(type) {
	case map[string]interface{}:
		for k2, v2 := range value {
			d := s.Keys[k2]
			if d.Skip {
				continue
			}
			if d.MapValue != nil {
				s.processValue(append(keys, strings.Trim(k2, "_")), labelNames, labelValues, d.MapValue[v2.(string)], metrics)
				continue
			}
			if d.UseKeysAsLabel != "" {
				for k3, v3 := range v2.(map[string]interface{}) {
					s.processValue(append(keys, strings.Trim(k2, "_")), append(labelNames, d.UseKeysAsLabel), append(labelValues, k3), v3, metrics)
				}
				continue
			}
			s.processValue(append(keys, strings.Trim(k2, "_")), labelNames, labelValues, v2, metrics)
		}
	case float64:
		if value == 0 {
			return
		}
		labels := make(prometheus.Labels)
		for i, name := range labelNames {
			labels[name] = labelValues[i]
		}
		g := prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace:   s.Namespace,
			Subsystem:   s.Subsystem,
			Name:        strings.Join(keys, "_"),
			Help:        strings.Join(keys, "."),
			ConstLabels: labels,
		})
		g.Set(value)
		metrics <- g
	}
}
<commit_msg>added option to use sub-key as label<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"os"
	"strings"

	"github.com/prometheus/client_golang/prometheus"
)

type collector struct {
	sources []*source
}

type source struct {
	URL       string
	Namespace string
	Subsystem string
	Labels    map[string]string
	Keys      map[string]struct {
		Skip      bool
		MapValue  map[string]float64
		MakeLabel string
		LabelKey  string
	}
}

func main() {
	sourcesJSON := os.Getenv("SOURCES")
	if sourcesJSON == "" {
		log.Print("environemnt variable SOURCES is requred")
		return
	}

	var sources []*source
	if err := json.Unmarshal([]byte(sourcesJSON), &sources); err != nil {
		log.Print(err)
		return
	}

	prometheus.MustRegister(&collector{sources})
	http.Handle("/", http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
		resp.Write([]byte(`<html><head><title>json2prom</title></head><body><h1>json2prom</h1><p><a href="/metrics">Metrics</a></p></body></html>`))
	}))
	http.Handle("/metrics", prometheus.Handler())

	addr := os.Getenv("HTTP_ADDR")
	if addr == "" {
		addr = ":8080"
	}
	fmt.Println("listening on " + addr)
	log.Print(http.ListenAndServe(addr, nil))
}

func (c *collector) Describe(descs chan<- *prometheus.Desc) {
	metrics := make(chan prometheus.Metric)
	done := make(chan struct{})
	go func() {
		for m := range metrics {
			descs <- m.Desc()
		}
		close(done)
	}()
	c.Collect(metrics)
	close(metrics)
	<-done
}

func (c *collector) Collect(metrics chan<- prometheus.Metric) {
	for _, s := range c.sources {
		resp, err := http.Get(s.URL)
		if err != nil {
			log.Print(err)
			continue
		}
		defer resp.Body.Close()

		var value interface{}
		if err := json.NewDecoder(resp.Body).Decode(&value); err != nil {
			log.Print(err)
			continue
		}

		var labelNames []string
		var labelValues []string
		for k, v := range s.Labels {
			labelNames = append(labelNames, k)
			labelValues = append(labelValues, v)
		}
		s.processValue(nil, labelNames, labelValues, value, metrics)
	}
}

func (s *source) processValue(keys []string, labelNames, labelValues []string, value interface{}, metrics chan<- prometheus.Metric) {
	switch value := value.(type) {
	case map[string]interface{}:
		for k2, v2 := range value {
			d := s.Keys[k2]
			if d.Skip {
				continue
			}
			if d.MapValue != nil {
				s.processValue(append(keys, strings.Trim(k2, "_")), labelNames, labelValues, d.MapValue[v2.(string)], metrics)
				continue
			}
			if d.MakeLabel != "" {
				for k3, v3 := range v2.(map[string]interface{}) {
					labelValue := k3
					if d.LabelKey != "" {
						labelValue = v3.(map[string]interface{})[d.LabelKey].(string)
					}
					s.processValue(append(keys, strings.Trim(k2, "_")), append(labelNames, d.MakeLabel), append(labelValues, labelValue), v3, metrics)
				}
				continue
			}
			s.processValue(append(keys, strings.Trim(k2, "_")), labelNames, labelValues, v2, metrics)
		}
	case float64:
		if value == 0 {
			return
		}
		labels := make(prometheus.Labels)
		for i, name := range labelNames {
			labels[name] = labelValues[i]
		}
		g := prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace:   s.Namespace,
			Subsystem:   s.Subsystem,
			Name:        strings.Join(keys, "_"),
			Help:        strings.Join(keys, "."),
			ConstLabels: labels,
		})
		g.Set(value)
		metrics <- g
	}
}
<|endoftext|>


<commit_before>package kego // import "kego.io/kego"

import (
	"io"

	"kego.io/json"
	_ "kego.io/system"
)

func Unmarshal(data []byte, v *interface{}, path string, imports map[string]string) (unknown bool, err error) {
	return json.Unmarshal(data, v, path, imports)
}

func NewDecoder(r io.Reader, path string, imports map[string]string) *json.Decoder {
	return json.NewDecoder(r, path, imports)
}
<commit_msg>Generating globals working<commit_after>package kego // import "kego.io/kego"

import (
	"io"
	"os"

	"kego.io/json"
	"kego.io/kerr"
	_ "kego.io/system"
)

func Open(filename string, path string, imports map[string]string) (value interface{}, unknown bool, err error) {

	file, err := os.Open(filename)
	if err != nil {
		err = kerr.New("NDJKHCDCIW", err, "kego.Open", "os.Open")
		return
	}
	defer file.Close()

	unknown, err = json.NewDecoder(file, path, imports).Decode(&value)
	return
}

func Unmarshal(data []byte, v *interface{}, path string, imports map[string]string) (unknown bool, err error) {
	return json.Unmarshal(data, v, path, imports)
}

func NewDecoder(r io.Reader, path string, imports map[string]string) *json.Decoder {
	return json.NewDecoder(r, path, imports)
}
<|endoftext|>


<commit_before>package kvdb

import (
	"time"

	"github.com/xiaonanln/goSyncQueue"
	"github.com/xiaonanln/goTimer"
	"github.com/xiaonanln/goworld/config"
	"github.com/xiaonanln/goworld/gwlog"
	"github.com/xiaonanln/goworld/kvdb/backend/mongodb"
	. "github.com/xiaonanln/goworld/kvdb/types"
	"github.com/xiaonanln/goworld/netutil"
	"github.com/xiaonanln/goworld/opmon"
)

var (
	kvdbEngine  KVDBEngine
	kvdbOpQueue sync_queue.SyncQueue
)

type KVDBEngine interface {
	Get(key string) (val string, err error)
	Put(key string, val string) (err error)
	Find(key string) Iterator
}

type KVDBGetCallback func(val string, err error)
type KVDBPutCallback func(err error)
type KVDBGetRangeCallback func(items []KVItem, err error)

func Initialize() {
	var err error
	kvdbCfg := config.GetKVDB()
	if kvdbCfg.Type == "" {
		return // kvdb not enabled
	}

	if kvdbCfg.Type == "mongodb" {
		kvdbEngine, err = kvdb_mongo.OpenMongoKVDB(kvdbCfg.Url, kvdbCfg.DB, kvdbCfg.Collection)
		if err != nil {
			gwlog.Panic(err)
		}
	}

	kvdbOpQueue = sync_queue.NewSyncQueue()
	go netutil.ServeForever(kvdbRoutine)
}

type getReq struct {
	key      string
	callback KVDBGetCallback
}

type putReq struct {
	key      string
	val      string
	callback KVDBPutCallback
}

type getRangeReq struct {
	beginKey string
	endKey   string
	callback KVDBGetRangeCallback
}

func Get(key string, callback KVDBGetCallback) {
	kvdbOpQueue.Push(&getReq{
		key, callback,
	})
	checkOperationQueueLen()
}

func Put(key string, val string, callback KVDBPutCallback) {
	kvdbOpQueue.Push(&putReq{
		key, val, callback,
	})
	checkOperationQueueLen()
}

func GetRange(beginKey string, endKey string, callback KVDBGetRangeCallback) {
	kvdbOpQueue.Push(&getRangeReq{
		beginKey, endKey, callback,
	})
	checkOperationQueueLen()
}

var recentWarnedQueueLen = 0

func checkOperationQueueLen() {
	qlen := kvdbOpQueue.Len()
	if qlen > 100 && qlen%100 == 0 && recentWarnedQueueLen != qlen {
		gwlog.Warn("KVDB operation queue length = %d", qlen)
		recentWarnedQueueLen = qlen
	}
}

func kvdbRoutine() {
	for {
		req := kvdbOpQueue.Pop()
		var op *opmon.Operation
		if getReq, ok := req.(*getReq); ok {
			op = opmon.StartOperation("kvdb.get")
			handleGetReq(getReq)
		} else if putReq, ok := req.(*putReq); ok {
			op = opmon.StartOperation("kvdb.put")
			handlePutReq(putReq)
		} else if getRangeReq, ok := req.(*getRangeReq); ok {
			op = opmon.StartOperation("kvdb.getRange")
			handleGetRangeReq(getRangeReq)
		}
		op.Finish(time.Millisecond * 100)
	}
}

func handleGetReq(getReq *getReq) {
	val, err := kvdbEngine.Get(getReq.key)
	if getReq.callback != nil {
		timer.AddCallback(0, func() {
			getReq.callback(val, err)
		})
	}
}

func handlePutReq(putReq *putReq) {
	err := kvdbEngine.Put(putReq.key, putReq.val)
	if putReq.callback != nil {
		timer.AddCallback(0, func() {
			putReq.callback(err)
		})
	}
}

func handleGetRangeReq(getRangeReq *getRangeReq) {
	it := kvdbEngine.Find(getRangeReq.beginKey)
	var items []KVItem
	endKey := getRangeReq.endKey
	for {
		item, err := it.Next()
		if item.Key >= endKey {
			// it is the end, end is not included
			break
		}

		if err != nil {
			if getRangeReq.callback != nil {
				getRangeReq.callback(nil, err)
			}
			return
		}

		items = append(items, item)
	}

	if getRangeReq.callback != nil {
		getRangeReq.callback(items, nil)
	}
}
<commit_msg>add comments<commit_after>package kvdb

import (
	"time"

	"github.com/xiaonanln/goSyncQueue"
	"github.com/xiaonanln/goTimer"
	"github.com/xiaonanln/goworld/config"
	"github.com/xiaonanln/goworld/gwlog"
	"github.com/xiaonanln/goworld/kvdb/backend/mongodb"
	. "github.com/xiaonanln/goworld/kvdb/types"
	"github.com/xiaonanln/goworld/netutil"
	"github.com/xiaonanln/goworld/opmon"
)

var (
	kvdbEngine  KVDBEngine
	kvdbOpQueue sync_queue.SyncQueue
)

type KVDBEngine interface {
	Get(key string) (val string, err error)
	Put(key string, val string) (err error)
	Find(key string) Iterator
}

type KVDBGetCallback func(val string, err error)
type KVDBPutCallback func(err error)
type KVDBGetRangeCallback func(items []KVItem, err error)

// Initialize the KVDB
//
// Called by game server engine
func Initialize() {
	var err error
	kvdbCfg := config.GetKVDB()
	if kvdbCfg.Type == "" {
		return // kvdb not enabled
	}

	if kvdbCfg.Type == "mongodb" {
		kvdbEngine, err = kvdb_mongo.OpenMongoKVDB(kvdbCfg.Url, kvdbCfg.DB, kvdbCfg.Collection)
		if err != nil {
			gwlog.Panic(err)
		}
	}

	kvdbOpQueue = sync_queue.NewSyncQueue()
	go netutil.ServeForever(kvdbRoutine)
}

type getReq struct {
	key      string
	callback KVDBGetCallback
}

type putReq struct {
	key      string
	val      string
	callback KVDBPutCallback
}

type getRangeReq struct {
	beginKey string
	endKey   string
	callback KVDBGetRangeCallback
}

func Get(key string, callback KVDBGetCallback) {
	kvdbOpQueue.Push(&getReq{
		key, callback,
	})
	checkOperationQueueLen()
}

func Put(key string, val string, callback KVDBPutCallback) {
	kvdbOpQueue.Push(&putReq{
		key, val, callback,
	})
	checkOperationQueueLen()
}

func GetRange(beginKey string, endKey string, callback KVDBGetRangeCallback) {
	kvdbOpQueue.Push(&getRangeReq{
		beginKey, endKey, callback,
	})
	checkOperationQueueLen()
}

var recentWarnedQueueLen = 0

func checkOperationQueueLen() {
	qlen := kvdbOpQueue.Len()
	if qlen > 100 && qlen%100 == 0 && recentWarnedQueueLen != qlen {
		gwlog.Warn("KVDB operation queue length = %d", qlen)
		recentWarnedQueueLen = qlen
	}
}

func kvdbRoutine() {
	for {
		req := kvdbOpQueue.Pop()
		var op *opmon.Operation
		if getReq, ok := req.(*getReq); ok {
			op = opmon.StartOperation("kvdb.get")
			handleGetReq(getReq)
		} else if putReq, ok := req.(*putReq); ok {
			op = opmon.StartOperation("kvdb.put")
			handlePutReq(putReq)
		} else if getRangeReq, ok := req.(*getRangeReq); ok {
			op = opmon.StartOperation("kvdb.getRange")
			handleGetRangeReq(getRangeReq)
		}
		op.Finish(time.Millisecond * 100)
	}
}

func handleGetReq(getReq *getReq) {
	val, err := kvdbEngine.Get(getReq.key)
	if getReq.callback != nil {
		timer.AddCallback(0, func() {
			getReq.callback(val, err)
		})
	}
}

func handlePutReq(putReq *putReq) {
	err := kvdbEngine.Put(putReq.key, putReq.val)
	if putReq.callback != nil {
		timer.AddCallback(0, func() {
			putReq.callback(err)
		})
	}
}

func handleGetRangeReq(getRangeReq *getRangeReq) {
	it := kvdbEngine.Find(getRangeReq.beginKey)
	var items []KVItem
	endKey := getRangeReq.endKey
	for {
		item, err := it.Next()
		if item.Key >= endKey {
			// it is the end, end is not included
			break
		}

		if err != nil {
			if getRangeReq.callback != nil {
				getRangeReq.callback(nil, err)
			}
			return
		}

		items = append(items, item)
	}

	if getRangeReq.callback != nil {
		getRangeReq.callback(items, nil)
	}
}
<|endoftext|>


<commit_before>package logging

import (
	"compress/gzip"
	"io"
	"os/exec"
	"time"
)

type RemoteLog struct {
	Host          string
	User          string
	Pattern       string
	Tail          bool
	Time          time.Time
	Compress      bool
	CustomLogRoot string
	FromBegin     bool // to be used with tail
}

const (
	DEFAULT_LOG_ROOT = "/var/log/hourly"
	HOURLY_PATTERN   = "2006/01/02/2006-01-02T15.log"
)

func NewRemoteLogFromTime(host string, t time.Time, pattern string) *RemoteLog {
	return &RemoteLog{
		Time:    t,
		Host:    host,
		Pattern: pattern,
	}
}

func (rl *RemoteLog) LogRoot() string {
	if rl.CustomLogRoot != "" {
		return rl.CustomLogRoot
	}
	return DEFAULT_LOG_ROOT
}

func (rl *RemoteLog) Current() string {
	return rl.LogRoot() + "/current"
}

func (rl *RemoteLog) Path() string {
	if !rl.Time.IsZero() {
		return rl.Time.UTC().Format(rl.LogRoot() + "/" + HOURLY_PATTERN)
	}
	return rl.Current()
}

func (rl *RemoteLog) GzipPath() string {
	return rl.Path() + ".gz"
}

func (rl *RemoteLog) Command() string {
	cmd := rl.CatCmd()
	if rl.Pattern != "" {
		cmd += " | " + rl.GrepCmd()
	}
	if rl.Compress {
		cmd += " | gzip"
	}
	return cmd
}

func (rl *RemoteLog) GrepCmd() string {
	return "grep " + rl.Pattern
}

func (rl *RemoteLog) CatCmd() string {
	if rl.Tail {
		n := "0"
		if rl.FromBegin {
			n = "+0"
		}
		return "tail -n " + n + " -F " + rl.Current()
	}
	return "{ test -e " + rl.Path() + " && cat " + rl.Path() + "; test -e " + rl.GzipPath() + " && cat " + rl.GzipPath() + " | gunzip; }"
}

func (rl *RemoteLog) Open() (reader io.ReadCloser, e error) {
	c := rl.Command()
	var cmd *exec.Cmd
	if rl.User == "" {
		rl.User = "root"
	}
	if rl.Host != "" {
		cmd = exec.Command("ssh", "-t", "-l", rl.User, rl.Host, c)
	} else {
		cmd = exec.Command("bash", "-c", c)
	}
	dbg.Printf("using cmd %q", cmd)
	reader, e = cmd.StdoutPipe()
	if e != nil {
		return nil, e
	}
	dbg.Print("starting command %q", c)
	e = cmd.Start()
	if e != nil {
		return nil, e
	}
	dbg.Print("command started")
	if rl.Compress {
		reader, e = gzip.NewReader(reader)
		if e != nil {
			return nil, e
		}
	}
	dbg.Print("returning reader")
	return reader, nil
}
<commit_msg>keep original user (in case we need it somewhere else)<commit_after>package logging

import (
	"compress/gzip"
	"io"
	"os/exec"
	"time"
)

type RemoteLog struct {
	Host          string
	User          string
	Pattern       string
	Tail          bool
	Time          time.Time
	Compress      bool
	CustomLogRoot string
	FromBegin     bool // to be used with tail
}

const (
	DEFAULT_LOG_ROOT = "/var/log/hourly"
	HOURLY_PATTERN   = "2006/01/02/2006-01-02T15.log"
)

func NewRemoteLogFromTime(host string, t time.Time, pattern string) *RemoteLog {
	return &RemoteLog{
		Time:    t,
		Host:    host,
		Pattern: pattern,
	}
}

func (rl *RemoteLog) LogRoot() string {
	if rl.CustomLogRoot != "" {
		return rl.CustomLogRoot
	}
	return DEFAULT_LOG_ROOT
}

func (rl *RemoteLog) Current() string {
	return rl.LogRoot() + "/current"
}

func (rl *RemoteLog) Path() string {
	if !rl.Time.IsZero() {
		return rl.Time.UTC().Format(rl.LogRoot() + "/" + HOURLY_PATTERN)
	}
	return rl.Current()
}

func (rl *RemoteLog) GzipPath() string {
	return rl.Path() + ".gz"
}

func (rl *RemoteLog) Command() string {
	cmd := rl.CatCmd()
	if rl.Pattern != "" {
		cmd += " | " + rl.GrepCmd()
	}
	if rl.Compress {
		cmd += " | gzip"
	}
	return cmd
}

func (rl *RemoteLog) GrepCmd() string {
	return "grep " + rl.Pattern
}

func (rl *RemoteLog) CatCmd() string {
	if rl.Tail {
		n := "0"
		if rl.FromBegin {
			n = "+0"
		}
		return "tail -n " + n + " -F " + rl.Current()
	}
	return "{ test -e " + rl.Path() + " && cat " + rl.Path() + "; test -e " + rl.GzipPath() + " && cat " + rl.GzipPath() + " | gunzip; }"
}

func (rl *RemoteLog) Open() (reader io.ReadCloser, e error) {
	c := rl.Command()
	var cmd *exec.Cmd
	user := rl.User
	if user == "" {
		user = "root"
	}
	if rl.Host != "" {
		cmd = exec.Command("ssh", "-t", "-l", user, rl.Host, c)
	} else {
		cmd = exec.Command("bash", "-c", c)
	}
	dbg.Printf("using cmd %q", cmd)
	reader, e = cmd.StdoutPipe()
	if e != nil {
		return nil, e
	}
	dbg.Print("starting command %q", c)
	e = cmd.Start()
	if e != nil {
		return nil, e
	}
	dbg.Print("command started")
	if rl.Compress {
		reader, e = gzip.NewReader(reader)
		if e != nil {
			return nil, e
		}
	}
	dbg.Print("returning reader")
	return reader, nil
}
<|endoftext|>


<commit_before>package logging

import (
	"fmt"
	"io"
	stdlog "log"
	"os"
	"strings"

	isatty "github.com/mattn/go-isatty"
	"github.com/rs/zerolog"
	"github.com/rs/zerolog/log"
	"github.com/sean-/conswriter"
)

var acceptedLogLevels = []string{"DEBUG", "INFO", "WARN", "ERROR", "FATAL"}
var acceptedLogFormat = []string{"HUMAN", "JSON"}

// SetupLogger sets the log level and outout format.
// Accepted levels are panic, fatal, error, warn, info and debug.
// Accepted formats are human or json.
func SetupLogger(level, format string) (err error) {

	if err = setLogFormat(strings.ToUpper(format)); err != nil {
		return err
	}

	if err = setLogLevel(strings.ToUpper(level)); err != nil {
		return err
	}

	return nil
}

func setLogLevel(level string) error {
	switch level {
	case "DEBUG":
		zerolog.SetGlobalLevel(zerolog.DebugLevel)
	case "INFO":
		zerolog.SetGlobalLevel(zerolog.InfoLevel)
	case "WARN":
		zerolog.SetGlobalLevel(zerolog.WarnLevel)
	case "ERROR":
		zerolog.SetGlobalLevel(zerolog.ErrorLevel)
	case "FATAL":
		zerolog.SetGlobalLevel(zerolog.FatalLevel)
	default:
		return fmt.Errorf("unsupported log level: %q (supported levels: %s)", level,
			strings.Join(acceptedLogLevels, " "))
	}
	return nil
}

func setLogFormat(format string) error {

	var logWriter io.Writer
	var zLog zerolog.Logger

	if isatty.IsTerminal(os.Stdout.Fd()) ||
		isatty.IsCygwinTerminal(os.Stdout.Fd()) {
		logWriter = conswriter.GetTerminal()
	} else {
		logWriter = os.Stdout
	}

	switch format {
	case "HUMAN":
		w := zerolog.ConsoleWriter{
			Out:     logWriter,
			NoColor: true,
		}
		zLog = zerolog.New(w).With().Timestamp().Logger()
	case "JSON":
		zLog = zerolog.New(logWriter).With().Timestamp().Logger()
	default:
		return fmt.Errorf("unsupported log format: %q (supported formats: %s)", format,
			strings.Join(acceptedLogFormat, " "))
	}

	log.Logger = zLog
	stdlog.SetFlags(0)
	stdlog.SetOutput(zLog)

	return nil
}
<commit_msg>if stdout isn't a terminal, send logging to stderr<commit_after>package logging

import (
	"fmt"
	"io"
	stdlog "log"
	"os"
	"strings"

	isatty "github.com/mattn/go-isatty"
	"github.com/rs/zerolog"
	"github.com/rs/zerolog/log"
	"github.com/sean-/conswriter"
)

var acceptedLogLevels = []string{"DEBUG", "INFO", "WARN", "ERROR", "FATAL"}
var acceptedLogFormat = []string{"HUMAN", "JSON"}

// SetupLogger sets the log level and outout format.
// Accepted levels are panic, fatal, error, warn, info and debug.
// Accepted formats are human or json.
func SetupLogger(level, format string) (err error) {

	if err = setLogFormat(strings.ToUpper(format)); err != nil {
		return err
	}

	if err = setLogLevel(strings.ToUpper(level)); err != nil {
		return err
	}

	return nil
}

func setLogLevel(level string) error {
	switch level {
	case "DEBUG":
		zerolog.SetGlobalLevel(zerolog.DebugLevel)
	case "INFO":
		zerolog.SetGlobalLevel(zerolog.InfoLevel)
	case "WARN":
		zerolog.SetGlobalLevel(zerolog.WarnLevel)
	case "ERROR":
		zerolog.SetGlobalLevel(zerolog.ErrorLevel)
	case "FATAL":
		zerolog.SetGlobalLevel(zerolog.FatalLevel)
	default:
		return fmt.Errorf("unsupported log level: %q (supported levels: %s)", level,
			strings.Join(acceptedLogLevels, " "))
	}
	return nil
}

func setLogFormat(format string) error {

	var logWriter io.Writer
	var zLog zerolog.Logger

	if isatty.IsTerminal(os.Stdout.Fd()) ||
		isatty.IsCygwinTerminal(os.Stdout.Fd()) {
		logWriter = conswriter.GetTerminal()
	} else {
		logWriter = os.Stderr
	}

	switch format {
	case "HUMAN":
		w := zerolog.ConsoleWriter{
			Out:     logWriter,
			NoColor: true,
		}
		zLog = zerolog.New(w).With().Timestamp().Logger()
	case "JSON":
		zLog = zerolog.New(logWriter).With().Timestamp().Logger()
	default:
		return fmt.Errorf("unsupported log format: %q (supported formats: %s)", format,
			strings.Join(acceptedLogFormat, " "))
	}

	log.Logger = zLog
	stdlog.SetFlags(0)
	stdlog.SetOutput(zLog)

	return nil
}
<|endoftext|>


<commit_before>// +build linux,cgo,!agent

package db

import (
	"fmt"

	deviceConfig "github.com/lxc/lxd/lxd/device/config"
	"github.com/lxc/lxd/shared/api"
	"github.com/pkg/errors"
)

// Code generation directives.
//
//go:generate -command mapper lxd-generate db mapper -t profiles.mapper.go
//go:generate mapper reset
//
//go:generate mapper stmt -p db -e profile names
//go:generate mapper stmt -p db -e profile names-by-Project
//go:generate mapper stmt -p db -e profile names-by-Project-and-Name
//go:generate mapper stmt -p db -e profile objects
//go:generate mapper stmt -p db -e profile objects-by-Project
//go:generate mapper stmt -p db -e profile objects-by-Project-and-Name
//go:generate mapper stmt -p db -e profile config-ref
//go:generate mapper stmt -p db -e profile config-ref-by-Project
//go:generate mapper stmt -p db -e profile config-ref-by-Project-and-Name
//go:generate mapper stmt -p db -e profile devices-ref
//go:generate mapper stmt -p db -e profile devices-ref-by-Project
//go:generate mapper stmt -p db -e profile devices-ref-by-Project-and-Name
//go:generate mapper stmt -p db -e profile used-by-ref
//go:generate mapper stmt -p db -e profile used-by-ref-by-Project
//go:generate mapper stmt -p db -e profile used-by-ref-by-Project-and-Name
//go:generate mapper stmt -p db -e profile id
//go:generate mapper stmt -p db -e profile create struct=Profile
//go:generate mapper stmt -p db -e profile create-config-ref
//go:generate mapper stmt -p db -e profile create-devices-ref
//go:generate mapper stmt -p db -e profile rename
//go:generate mapper stmt -p db -e profile delete
//go:generate mapper stmt -p db -e profile delete-config-ref
//go:generate mapper stmt -p db -e profile delete-devices-ref
//go:generate mapper stmt -p db -e profile update struct=Profile
//
//go:generate mapper method -p db -e profile URIs
//go:generate mapper method -p db -e profile List
//go:generate mapper method -p db -e profile Get
//go:generate mapper method -p db -e profile Exists struct=Profile
//go:generate mapper method -p db -e profile ID struct=Profile
//go:generate mapper method -p db -e profile ConfigRef
//go:generate mapper method -p db -e profile DevicesRef
//go:generate mapper method -p db -e profile UsedByRef
//go:generate mapper method -p db -e profile Create struct=Profile
//go:generate mapper method -p db -e profile Rename
//go:generate mapper method -p db -e profile Delete
//go:generate mapper method -p db -e profile Update struct=Profile

// Profile is a value object holding db-related details about a profile.
type Profile struct {
	ID          int
	Project     string `db:"primary=yes&join=projects.name"`
	Name        string `db:"primary=yes"`
	Description string `db:"coalesce=''"`
	Config      map[string]string
	Devices     map[string]map[string]string
	UsedBy      []string
}

// ProfileToAPI is a convenience to convert a Profile db struct into
// an API profile struct.
func ProfileToAPI(profile *Profile) *api.Profile {
	p := &api.Profile{
		Name:   profile.Name,
		UsedBy: profile.UsedBy,
	}
	p.Description = profile.Description
	p.Config = profile.Config
	p.Devices = profile.Devices

	return p
}

// ProfileFilter can be used to filter results yielded by ProfileList.
type ProfileFilter struct {
	Project string
	Name    string
}

// GetProfileNames returns the names of all profiles in the given project.
func (c *Cluster) GetProfileNames(project string) ([]string, error) {
	err := c.Transaction(func(tx *ClusterTx) error {
		enabled, err := tx.ProjectHasProfiles(project)
		if err != nil {
			return errors.Wrap(err, "Check if project has profiles")
		}
		if !enabled {
			project = "default"
		}
		return nil
	})
	if err != nil {
		return nil, err
	}

	q := fmt.Sprintf(`
SELECT profiles.name
 FROM profiles
 JOIN projects ON projects.id = profiles.project_id
WHERE projects.name = ?
`)
	inargs := []interface{}{project}
	var name string
	outfmt := []interface{}{name}
	result, err := queryScan(c, q, inargs, outfmt)
	if err != nil {
		return []string{}, err
	}

	response := []string{}
	for _, r := range result {
		response = append(response, r[0].(string))
	}

	return response, nil
}

// GetProfile returns the profile with the given name.
func (c *Cluster) GetProfile(project, name string) (int64, *api.Profile, error) {
	var result *api.Profile
	var id int64

	err := c.Transaction(func(tx *ClusterTx) error {
		var err error
		id, result, err = tx.getProfile(project, name)
		return err
	})
	if err != nil {
		return -1, nil, err
	}

	return id, result, nil
}

// Returns the profile with the given name.
func (c *ClusterTx) getProfile(project, name string) (int64, *api.Profile, error) {
	var result *api.Profile
	var id int64

	enabled, err := c.ProjectHasProfiles(project)
	if err != nil {
		return -1, nil, errors.Wrap(err, "Check if project has profiles")
	}
	if !enabled {
		project = "default"
	}

	profile, err := c.GetProfile(project, name)
	if err != nil {
		return -1, nil, err
	}

	result = ProfileToAPI(profile)
	id = int64(profile.ID)

	return id, result, nil
}

// GetProfiles returns the profiles with the given names in the given project.
func (c *Cluster) GetProfiles(project string, names []string) ([]api.Profile, error) {
	profiles := make([]api.Profile, len(names))

	err := c.Transaction(func(tx *ClusterTx) error {
		enabled, err := tx.ProjectHasProfiles(project)
		if err != nil {
			return errors.Wrap(err, "Check if project has profiles")
		}
		if !enabled {
			project = "default"
		}

		for i, name := range names {
			profile, err := tx.GetProfile(project, name)
			if err != nil {
				return errors.Wrapf(err, "Load profile %q", name)
			}
			profiles[i] = *ProfileToAPI(profile)
		}

		return nil
	})
	if err != nil {
		return nil, err
	}

	return profiles, nil
}

// GetInstancesWithProfile gets the names of the instance associated with the
// profile with the given name in the given project.
func (c *Cluster) GetInstancesWithProfile(project, profile string) (map[string][]string, error) {
	err := c.Transaction(func(tx *ClusterTx) error {
		enabled, err := tx.ProjectHasProfiles(project)
		if err != nil {
			return errors.Wrap(err, "Check if project has profiles")
		}
		if !enabled {
			project = "default"
		}
		return nil
	})
	if err != nil {
		return nil, err
	}

	q := `SELECT instances.name, projects.name FROM instances
		JOIN instances_profiles ON instances.id == instances_profiles.instance_id
		JOIN projects ON projects.id == instances.project_id
		WHERE instances_profiles.profile_id ==
		  (SELECT profiles.id FROM profiles
		   JOIN projects ON projects.id == profiles.project_id
		   WHERE profiles.name=? AND projects.name=?)
		AND instances.type == 0`

	results := map[string][]string{}
	inargs := []interface{}{profile, project}
	var name string
	outfmt := []interface{}{name, name}

	output, err := queryScan(c, q, inargs, outfmt)
	if err != nil {
		return nil, err
	}

	for _, r := range output {
		if results[r[1].(string)] == nil {
			results[r[1].(string)] = []string{}
		}

		results[r[1].(string)] = append(results[r[1].(string)], r[0].(string))
	}

	return results, nil
}

// RemoveUnreferencedProfiles removes unreferenced profiles.
func (c *Cluster) RemoveUnreferencedProfiles() error {
	stmt := `
DELETE FROM profiles_config WHERE profile_id NOT IN (SELECT id FROM profiles);
DELETE FROM profiles_devices WHERE profile_id NOT IN (SELECT id FROM profiles);
DELETE FROM profiles_devices_config WHERE profile_device_id NOT IN (SELECT id FROM profiles_devices);
`
	err := exec(c, stmt)
	if err != nil {
		return err
	}

	return nil
}

// ExpandInstanceConfig expands the given instance config with the config
// values of the given profiles.
func ExpandInstanceConfig(config map[string]string, profiles []api.Profile) map[string]string {
	expandedConfig := map[string]string{}

	// Apply all the profiles
	profileConfigs := make([]map[string]string, len(profiles))
	for i, profile := range profiles {
		profileConfigs[i] = profile.Config
	}

	for i := range profileConfigs {
		for k, v := range profileConfigs[i] {
			expandedConfig[k] = v
		}
	}

	// Stick the given config on top
	for k, v := range config {
		expandedConfig[k] = v
	}

	return expandedConfig
}

// ExpandInstanceDevices expands the given instance devices with the devices
// defined in the given profiles.
func ExpandInstanceDevices(devices deviceConfig.Devices, profiles []api.Profile) deviceConfig.Devices {
	expandedDevices := deviceConfig.Devices{}

	// Apply all the profiles
	profileDevices := make([]deviceConfig.Devices, len(profiles))
	for i, profile := range profiles {
		profileDevices[i] = deviceConfig.NewDevices(profile.Devices)
	}
	for i := range profileDevices {
		for k, v := range profileDevices[i] {
			expandedDevices[k] = v
		}
	}

	// Stick the given devices on top
	for k, v := range devices {
		expandedDevices[k] = v
	}

	return expandedDevices
}
<commit_msg>lxd/db/profiles: Updates GetInstancesWithProfile to return all instance types, not just containers<commit_after>// +build linux,cgo,!agent

package db

import (
	"fmt"

	deviceConfig "github.com/lxc/lxd/lxd/device/config"
	"github.com/lxc/lxd/shared/api"
	"github.com/pkg/errors"
)

// Code generation directives.
//
//go:generate -command mapper lxd-generate db mapper -t profiles.mapper.go
//go:generate mapper reset
//
//go:generate mapper stmt -p db -e profile names
//go:generate mapper stmt -p db -e profile names-by-Project
//go:generate mapper stmt -p db -e profile names-by-Project-and-Name
//go:generate mapper stmt -p db -e profile objects
//go:generate mapper stmt -p db -e profile objects-by-Project
//go:generate mapper stmt -p db -e profile objects-by-Project-and-Name
//go:generate mapper stmt -p db -e profile config-ref
//go:generate mapper stmt -p db -e profile config-ref-by-Project
//go:generate mapper stmt -p db -e profile config-ref-by-Project-and-Name
//go:generate mapper stmt -p db -e profile devices-ref
//go:generate mapper stmt -p db -e profile devices-ref-by-Project
//go:generate mapper stmt -p db -e profile devices-ref-by-Project-and-Name
//go:generate mapper stmt -p db -e profile used-by-ref
//go:generate mapper stmt -p db -e profile used-by-ref-by-Project
//go:generate mapper stmt -p db -e profile used-by-ref-by-Project-and-Name
//go:generate mapper stmt -p db -e profile id
//go:generate mapper stmt -p db -e profile create struct=Profile
//go:generate mapper stmt -p db -e profile create-config-ref
//go:generate mapper stmt -p db -e profile create-devices-ref
//go:generate mapper stmt -p db -e profile rename
//go:generate mapper stmt -p db -e profile delete
//go:generate mapper stmt -p db -e profile delete-config-ref
//go:generate mapper stmt -p db -e profile delete-devices-ref
//go:generate mapper stmt -p db -e profile update struct=Profile
//
//go:generate mapper method -p db -e profile URIs
//go:generate mapper method -p db -e profile List
//go:generate mapper method -p db -e profile Get
//go:generate mapper method -p db -e profile Exists struct=Profile
//go:generate mapper method -p db -e profile ID struct=Profile
//go:generate mapper method -p db -e profile ConfigRef
//go:generate mapper method -p db -e profile DevicesRef
//go:generate mapper method -p db -e profile UsedByRef
//go:generate mapper method -p db -e profile Create struct=Profile
//go:generate mapper method -p db -e profile Rename
//go:generate mapper method -p db -e profile Delete
//go:generate mapper method -p db -e profile Update struct=Profile

// Profile is a value object holding db-related details about a profile.
type Profile struct {
	ID          int
	Project     string `db:"primary=yes&join=projects.name"`
	Name        string `db:"primary=yes"`
	Description string `db:"coalesce=''"`
	Config      map[string]string
	Devices     map[string]map[string]string
	UsedBy      []string
}

// ProfileToAPI is a convenience to convert a Profile db struct into
// an API profile struct.
func ProfileToAPI(profile *Profile) *api.Profile {
	p := &api.Profile{
		Name:   profile.Name,
		UsedBy: profile.UsedBy,
	}
	p.Description = profile.Description
	p.Config = profile.Config
	p.Devices = profile.Devices

	return p
}

// ProfileFilter can be used to filter results yielded by ProfileList.
type ProfileFilter struct {
	Project string
	Name    string
}

// GetProfileNames returns the names of all profiles in the given project.
func (c *Cluster) GetProfileNames(project string) ([]string, error) {
	err := c.Transaction(func(tx *ClusterTx) error {
		enabled, err := tx.ProjectHasProfiles(project)
		if err != nil {
			return errors.Wrap(err, "Check if project has profiles")
		}
		if !enabled {
			project = "default"
		}
		return nil
	})
	if err != nil {
		return nil, err
	}

	q := fmt.Sprintf(`
SELECT profiles.name
 FROM profiles
 JOIN projects ON projects.id = profiles.project_id
WHERE projects.name = ?
`)
	inargs := []interface{}{project}
	var name string
	outfmt := []interface{}{name}
	result, err := queryScan(c, q, inargs, outfmt)
	if err != nil {
		return []string{}, err
	}

	response := []string{}
	for _, r := range result {
		response = append(response, r[0].(string))
	}

	return response, nil
}

// GetProfile returns the profile with the given name.
func (c *Cluster) GetProfile(project, name string) (int64, *api.Profile, error) {
	var result *api.Profile
	var id int64

	err := c.Transaction(func(tx *ClusterTx) error {
		var err error
		id, result, err = tx.getProfile(project, name)
		return err
	})
	if err != nil {
		return -1, nil, err
	}

	return id, result, nil
}

// Returns the profile with the given name.
func (c *ClusterTx) getProfile(project, name string) (int64, *api.Profile, error) {
	var result *api.Profile
	var id int64

	enabled, err := c.ProjectHasProfiles(project)
	if err != nil {
		return -1, nil, errors.Wrap(err, "Check if project has profiles")
	}
	if !enabled {
		project = "default"
	}

	profile, err := c.GetProfile(project, name)
	if err != nil {
		return -1, nil, err
	}

	result = ProfileToAPI(profile)
	id = int64(profile.ID)

	return id, result, nil
}

// GetProfiles returns the profiles with the given names in the given project.
func (c *Cluster) GetProfiles(project string, names []string) ([]api.Profile, error) {
	profiles := make([]api.Profile, len(names))

	err := c.Transaction(func(tx *ClusterTx) error {
		enabled, err := tx.ProjectHasProfiles(project)
		if err != nil {
			return errors.Wrap(err, "Check if project has profiles")
		}
		if !enabled {
			project = "default"
		}

		for i, name := range names {
			profile, err := tx.GetProfile(project, name)
			if err != nil {
				return errors.Wrapf(err, "Load profile %q", name)
			}
			profiles[i] = *ProfileToAPI(profile)
		}

		return nil
	})
	if err != nil {
		return nil, err
	}

	return profiles, nil
}

// GetInstancesWithProfile gets the names of the instance associated with the
// profile with the given name in the given project.
func (c *Cluster) GetInstancesWithProfile(project, profile string) (map[string][]string, error) {
	err := c.Transaction(func(tx *ClusterTx) error {
		enabled, err := tx.ProjectHasProfiles(project)
		if err != nil {
			return errors.Wrap(err, "Check if project has profiles")
		}
		if !enabled {
			project = "default"
		}
		return nil
	})
	if err != nil {
		return nil, err
	}

	q := `SELECT instances.name, projects.name FROM instances
		JOIN instances_profiles ON instances.id == instances_profiles.instance_id
		JOIN projects ON projects.id == instances.project_id
		WHERE instances_profiles.profile_id ==
		  (SELECT profiles.id FROM profiles
		   JOIN projects ON projects.id == profiles.project_id
		   WHERE profiles.name=? AND projects.name=?)`

	results := map[string][]string{}
	inargs := []interface{}{profile, project}
	var name string
	outfmt := []interface{}{name, name}

	output, err := queryScan(c, q, inargs, outfmt)
	if err != nil {
		return nil, err
	}

	for _, r := range output {
		if results[r[1].(string)] == nil {
			results[r[1].(string)] = []string{}
		}

		results[r[1].(string)] = append(results[r[1].(string)], r[0].(string))
	}

	return results, nil
}

// RemoveUnreferencedProfiles removes unreferenced profiles.
func (c *Cluster) RemoveUnreferencedProfiles() error {
	stmt := `
DELETE FROM profiles_config WHERE profile_id NOT IN (SELECT id FROM profiles);
DELETE FROM profiles_devices WHERE profile_id NOT IN (SELECT id FROM profiles);
DELETE FROM profiles_devices_config WHERE profile_device_id NOT IN (SELECT id FROM profiles_devices);
`
	err := exec(c, stmt)
	if err != nil {
		return err
	}

	return nil
}

// ExpandInstanceConfig expands the given instance config with the config
// values of the given profiles.
func ExpandInstanceConfig(config map[string]string, profiles []api.Profile) map[string]string {
	expandedConfig := map[string]string{}

	// Apply all the profiles
	profileConfigs := make([]map[string]string, len(profiles))
	for i, profile := range profiles {
		profileConfigs[i] = profile.Config
	}

	for i := range profileConfigs {
		for k, v := range profileConfigs[i] {
			expandedConfig[k] = v
		}
	}

	// Stick the given config on top
	for k, v := range config {
		expandedConfig[k] = v
	}

	return expandedConfig
}

// ExpandInstanceDevices expands the given instance devices with the devices
// defined in the given profiles.
func ExpandInstanceDevices(devices deviceConfig.Devices, profiles []api.Profile) deviceConfig.Devices {
	expandedDevices := deviceConfig.Devices{}

	// Apply all the profiles
	profileDevices := make([]deviceConfig.Devices, len(profiles))
	for i, profile := range profiles {
		profileDevices[i] = deviceConfig.NewDevices(profile.Devices)
	}
	for i := range profileDevices {
		for k, v := range profileDevices[i] {
			expandedDevices[k] = v
		}
	}

	// Stick the given devices on top
	for k, v := range devices {
		expandedDevices[k] = v
	}

	return expandedDevices
}
<|endoftext|>


<commit_before>package main

import (
	"k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/tools/clientcmd"
	"os"
	"path/filepath"
)

var (
	clientset *kubernetes.Clientset
)

func homeDir() string {
	if h := os.Getenv("HOME"); h != "" {
		return h
	}
	return os.Getenv("USERPROFILE") // windows
}

func kubernetesSetup() error {
	kubeconfig := filepath.Join(homeDir(), ".kube", "config")
	config, err := clientcmd.BuildConfigFromFlags("", kubeconfig)
	if err != nil {
		return err
	}

	clientset, err = kubernetes.NewForConfig(config)
	return err
}

func getNamespaces() (*v1.NamespaceList, error) {
	return clientset.CoreV1().Namespaces().List(metav1.ListOptions{})
}
<commit_msg>add KUBECONFIG support<commit_after>package main

import (
	"k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/tools/clientcmd"
	"os"
	"path/filepath"
	"strings"
)

var (
	clientset *kubernetes.Clientset
)

func homeDir() string {
	if h := os.Getenv("HOME"); h != "" {
		return h
	}
	return os.Getenv("USERPROFILE") // windows
}

func kubernetesSetup() error {
	configEnv := strings.Split(os.Getenv("KUBECONFIG"), ":")

	var kubeconfig string
	if len(configEnv) > 0 && configEnv[0] != "" {
		kubeconfig = configEnv[0]
	} else {
		kubeconfig = filepath.Join(homeDir(), ".kube", "config")
	}

	var overrides clientcmd.ConfigOverrides
	if len(configEnv) > 1 && configEnv[1] != "" {
		overrides.CurrentContext = configEnv[1]
	}

	config, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
		&clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfig},
		&overrides,
	).ClientConfig()

	if err != nil {
		return err
	}

	clientset, err = kubernetes.NewForConfig(config)
	return err
}

func getNamespaces() (*v1.NamespaceList, error) {
	return clientset.CoreV1().Namespaces().List(metav1.ListOptions{})
}
<|endoftext|>


<commit_before>// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package syscall

// Nosplit because it is called from forkAndExecInChild.
//
//go:nosplit
func ptrace(request int, pid int, addr uintptr, data uintptr) (err error) {
	panic("unimplemented")
}
<commit_msg>syscall: add explicit ios build tag<commit_after>// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

//go:build ios
// +build ios

package syscall

// Nosplit because it is called from forkAndExecInChild.
//
//go:nosplit
func ptrace(request int, pid int, addr uintptr, data uintptr) (err error) {
	panic("unimplemented")
}
<|endoftext|>


<commit_before>// Copyright 2018 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package apipanic

import (
	"net/http"
	"runtime/debug"

	"github.com/sirupsen/logrus"

	"github.com/cilium/cilium/pkg/logging"
)

// APIPanicHandler recovers from API panics and logs encountered panics
type APIPanicHandler struct {
	Next http.Handler
}

// ServeHTTP implements the http.Handler interface.
// It recovers from panics of all next handlers and logs them
func (h *APIPanicHandler) ServeHTTP(r http.ResponseWriter, req *http.Request) {
	defer func() {
		if r := recover(); r != nil {
			fields := logrus.Fields{
				"panic_message": r,
				"url":           req.URL.String(),
				"method":        req.Method,
				"client":        req.RemoteAddr,
			}
			logging.DefaultLogger.WithFields(fields).Warn("Cilium API handler panicked")
			logging.DefaultLogger.Debug(debug.Stack())
		}
	}()
	h.Next.ServeHTTP(r, req)
}
<commit_msg>apipanic: Log stack as string<commit_after>// Copyright 2018 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package apipanic

import (
	"net/http"
	"runtime/debug"

	"github.com/sirupsen/logrus"

	"github.com/cilium/cilium/pkg/logging"
)

// APIPanicHandler recovers from API panics and logs encountered panics
type APIPanicHandler struct {
	Next http.Handler
}

// ServeHTTP implements the http.Handler interface.
// It recovers from panics of all next handlers and logs them
func (h *APIPanicHandler) ServeHTTP(r http.ResponseWriter, req *http.Request) {
	defer func() {
		if r := recover(); r != nil {
			fields := logrus.Fields{
				"panic_message": r,
				"url":           req.URL.String(),
				"method":        req.Method,
				"client":        req.RemoteAddr,
			}
			logging.DefaultLogger.WithFields(fields).Warn("Cilium API handler panicked")
			logging.DefaultLogger.Debugf("%s", debug.Stack())
		}
	}()
	h.Next.ServeHTTP(r, req)
}
<|endoftext|>


<commit_before>package build

import (
	"fmt"

	"github.com/flant/dapp/pkg/image"
	"github.com/flant/dapp/pkg/lock"
)

func NewBuildPhase() *BuildPhase {
	return &BuildPhase{}
}

type BuildPhase struct{}

func (p *BuildPhase) Run(c *Conveyor) error {
	if debug() {
		fmt.Printf("BuildPhase.Run\n")
	}

	for _, dimg := range c.DimgsInOrder {
		acquiredLocks := []string{}

		unlockLocks := func() {
			locks := acquiredLocks
			for len(acquiredLocks) > 0 {
				var lockName string
				lockName, locks = locks[0], locks[1:]
				lock.Unlock(lockName)
			}
		}

		defer unlockLocks()

		// lock
		for _, stage := range dimg.GetStages() {
			img := stage.GetImage()
			if img.IsExists() {
				continue
			}

			imageLockName := fmt.Sprintf("%s.image.%s", c.ProjectName, img.Name())
			err := lock.Lock(imageLockName, lock.LockOptions{})
			if err != nil {
				return fmt.Errorf("failed to lock %s: %s", imageLockName, err)
			}
		}

		// build
		for _, stage := range dimg.GetStages() {
			img := stage.GetImage()

			err := img.Build2(image.BuildOptions{})
			if err != nil {
				return fmt.Errorf("failed to build %s: %s", img.Name(), err)
			}
		}

		// save in cache
		for _, stage := range dimg.GetStages() {
			img := stage.GetImage()

			err := img.SaveInCache()
			if err != nil {
				return fmt.Errorf("failed to save in cache image %s: %s", img.Name(), err)
			}
		}

		unlockLocks()
	}

	return nil
}
<commit_msg>[go build] ignore built stages on build phase<commit_after>package build

import (
	"fmt"

	"github.com/flant/dapp/pkg/image"
	"github.com/flant/dapp/pkg/lock"
)

func NewBuildPhase() *BuildPhase {
	return &BuildPhase{}
}

type BuildPhase struct{}

func (p *BuildPhase) Run(c *Conveyor) error {
	if debug() {
		fmt.Printf("BuildPhase.Run\n")
	}

	for _, dimg := range c.DimgsInOrder {
		var acquiredLocks []string

		unlockLocks := func() {
			locks := acquiredLocks
			for len(acquiredLocks) > 0 {
				var lockName string
				lockName, locks = locks[0], locks[1:]
				lock.Unlock(lockName)
			}
		}

		defer unlockLocks()

		// lock
		for _, stage := range dimg.GetStages() {
			img := stage.GetImage()
			if img.IsExists() {
				continue
			}

			imageLockName := fmt.Sprintf("%s.image.%s", c.ProjectName, img.Name())
			err := lock.Lock(imageLockName, lock.LockOptions{})
			if err != nil {
				return fmt.Errorf("failed to lock %s: %s", imageLockName, err)
			}
		}

		// build
		for _, stage := range dimg.GetStages() {
			img := stage.GetImage()
			if img.IsExists() {
				continue
			}

			err := img.Build2(image.BuildOptions{})
			if err != nil {
				return fmt.Errorf("failed to build %s: %s", img.Name(), err)
			}
		}

		// save in cache
		for _, stage := range dimg.GetStages() {
			img := stage.GetImage()
			if img.IsExists() {
				continue
			}

			err := img.SaveInCache()
			if err != nil {
				return fmt.Errorf("failed to save in cache image %s: %s", img.Name(), err)
			}
		}

		unlockLocks()
	}

	return nil
}
<|endoftext|>