<commit_before>package main

import (
	"context"
	"flag"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"path/filepath"
	"reflect"
	"strconv"
	"time"

	"github.com/facebookgo/inject"
	"github.com/grafana/grafana/pkg/bus"
	"github.com/grafana/grafana/pkg/middleware"
	"github.com/grafana/grafana/pkg/registry"
	"github.com/grafana/grafana/pkg/services/dashboards"

	"golang.org/x/sync/errgroup"

	"github.com/grafana/grafana/pkg/api"
	"github.com/grafana/grafana/pkg/log"
	"github.com/grafana/grafana/pkg/login"
	"github.com/grafana/grafana/pkg/services/sqlstore"
	"github.com/grafana/grafana/pkg/setting"

	"github.com/grafana/grafana/pkg/social"
	"github.com/grafana/grafana/pkg/tracing"

	// self registering services
	_ "github.com/grafana/grafana/pkg/extensions"
	_ "github.com/grafana/grafana/pkg/metrics"
	_ "github.com/grafana/grafana/pkg/plugins"
	_ "github.com/grafana/grafana/pkg/services/alerting"
	_ "github.com/grafana/grafana/pkg/services/cleanup"
	_ "github.com/grafana/grafana/pkg/services/notifications"
	_ "github.com/grafana/grafana/pkg/services/provisioning"
	_ "github.com/grafana/grafana/pkg/services/search"
)

func NewGrafanaServer() *GrafanaServerImpl {
	rootCtx, shutdownFn := context.WithCancel(context.Background())
	childRoutines, childCtx := errgroup.WithContext(rootCtx)

	return &GrafanaServerImpl{
		context:       childCtx,
		shutdownFn:    shutdownFn,
		childRoutines: childRoutines,
		log:           log.New("server"),
		cfg:           setting.NewCfg(),
	}
}

type GrafanaServerImpl struct {
	context            context.Context
	shutdownFn         context.CancelFunc
	childRoutines      *errgroup.Group
	log                log.Logger
	cfg                *setting.Cfg
	shutdownReason     string
	shutdownInProgress bool

	RouteRegister api.RouteRegister `inject:""`
	HttpServer    *api.HTTPServer   `inject:""`
}

func (g *GrafanaServerImpl) Run() error {
	g.loadConfiguration()
	g.writePIDFile()

	// initSql
	sqlstore.NewEngine() // TODO: this should return an error
	sqlstore.EnsureAdminUser()

	login.Init()
	social.NewOAuthService()

	tracingCloser, err := tracing.Init(g.cfg.Raw)
	if err != nil {
		return fmt.Errorf("Tracing settings is not valid. error: %v", err)
	}
	defer tracingCloser.Close()

	serviceGraph := inject.Graph{}
	serviceGraph.Provide(&inject.Object{Value: bus.GetBus()})
	serviceGraph.Provide(&inject.Object{Value: g.cfg})
	serviceGraph.Provide(&inject.Object{Value: dashboards.NewProvisioningService()})
	serviceGraph.Provide(&inject.Object{Value: api.NewRouteRegister(middleware.RequestMetrics, middleware.RequestTracing)})

	// self registered services
	services := registry.GetServices()

	// Add all services to dependency graph
	for _, service := range services {
		serviceGraph.Provide(&inject.Object{Value: service})
	}

	serviceGraph.Provide(&inject.Object{Value: g})

	// Inject dependencies to services
	if err := serviceGraph.Populate(); err != nil {
		return fmt.Errorf("Failed to populate service dependency: %v", err)
	}

	// Init & start services
	for _, service := range services {
		if registry.IsDisabled(service) {
			continue
		}

		g.log.Info("Initializing " + reflect.TypeOf(service).Elem().Name())

		if err := service.Init(); err != nil {
			return fmt.Errorf("Service init failed: %v", err)
		}
	}

	// Start background services
	for index := range services {
		service, ok := services[index].(registry.BackgroundService)
		if !ok {
			continue
		}

		if registry.IsDisabled(services[index]) {
			continue
		}

		g.childRoutines.Go(func() error {
			// Skip starting new service when shutting down
			// Can happen when service stop/return during startup
			if g.shutdownInProgress {
				return nil
			}

			err := service.Run(g.context)

			// If error is not canceled then the service crashed
			if err != context.Canceled {
				g.log.Error("Stopped "+reflect.TypeOf(service).Elem().Name(), "reason", err)
			} else {
				g.log.Info("Stopped "+reflect.TypeOf(service).Elem().Name(), "reason", err)
			}

			// Mark that we are in shutdown mode
			// So more services are not started
			g.shutdownInProgress = true
			return err
		})
	}

	sendSystemdNotification("READY=1")

	return g.childRoutines.Wait()
}

func (g *GrafanaServerImpl) loadConfiguration() {
	err := g.cfg.Load(&setting.CommandLineArgs{
		Config:   *configFile,
		HomePath: *homePath,
		Args:     flag.Args(),
	})

	if err != nil {
		fmt.Fprintf(os.Stderr, "Failed to start grafana. error: %s\n", err.Error())
		os.Exit(1)
	}

	g.log.Info("Starting "+setting.ApplicationName, "version", version, "commit", commit, "compiled", time.Unix(setting.BuildStamp, 0))
	g.cfg.LogConfigSources()
}

func (g *GrafanaServerImpl) Shutdown(reason string) {
	g.log.Info("Shutdown started", "reason", reason)
	g.shutdownReason = reason
	g.shutdownInProgress = true

	// call cancel func on root context
	g.shutdownFn()

	// wait for child routines
	g.childRoutines.Wait()
}

func (g *GrafanaServerImpl) Exit(reason error) {
	// default exit code is 1
	code := 1

	if reason == context.Canceled && g.shutdownReason != "" {
		reason = fmt.Errorf(g.shutdownReason)
		code = 0
	}

	g.log.Error("Server shutdown", "reason", reason)
	os.Exit(code)
}

func (g *GrafanaServerImpl) writePIDFile() {
	if *pidFile == "" {
		return
	}

	// Ensure the required directory structure exists.
	err := os.MkdirAll(filepath.Dir(*pidFile), 0700)
	if err != nil {
		g.log.Error("Failed to verify pid directory", "error", err)
		os.Exit(1)
	}

	// Retrieve the PID and write it.
	pid := strconv.Itoa(os.Getpid())
	if err := ioutil.WriteFile(*pidFile, []byte(pid), 0644); err != nil {
		g.log.Error("Failed to write pidfile", "error", err)
		os.Exit(1)
	}

	g.log.Info("Writing PID file", "path", *pidFile, "pid", pid)
}

func sendSystemdNotification(state string) error {
	notifySocket := os.Getenv("NOTIFY_SOCKET")

	if notifySocket == "" {
		return fmt.Errorf("NOTIFY_SOCKET environment variable empty or unset.")
	}

	socketAddr := &net.UnixAddr{
		Name: notifySocket,
		Net:  "unixgram",
	}

	conn, err := net.DialUnix(socketAddr.Net, nil, socketAddr)

	if err != nil {
		return err
	}

	_, err = conn.Write([]byte(state))

	conn.Close()

	return err
}
<commit_msg>only error log when err is not nil<commit_after>package main

import (
	"context"
	"flag"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"path/filepath"
	"reflect"
	"strconv"
	"time"

	"github.com/facebookgo/inject"
	"github.com/grafana/grafana/pkg/bus"
	"github.com/grafana/grafana/pkg/middleware"
	"github.com/grafana/grafana/pkg/registry"
	"github.com/grafana/grafana/pkg/services/dashboards"

	"golang.org/x/sync/errgroup"

	"github.com/grafana/grafana/pkg/api"
	"github.com/grafana/grafana/pkg/log"
	"github.com/grafana/grafana/pkg/login"
	"github.com/grafana/grafana/pkg/services/sqlstore"
	"github.com/grafana/grafana/pkg/setting"

	"github.com/grafana/grafana/pkg/social"
	"github.com/grafana/grafana/pkg/tracing"

	// self registering services
	_ "github.com/grafana/grafana/pkg/extensions"
	_ "github.com/grafana/grafana/pkg/metrics"
	_ "github.com/grafana/grafana/pkg/plugins"
	_ "github.com/grafana/grafana/pkg/services/alerting"
	_ "github.com/grafana/grafana/pkg/services/cleanup"
	_ "github.com/grafana/grafana/pkg/services/notifications"
	_ "github.com/grafana/grafana/pkg/services/provisioning"
	_ "github.com/grafana/grafana/pkg/services/search"
)

func NewGrafanaServer() *GrafanaServerImpl {
	rootCtx, shutdownFn := context.WithCancel(context.Background())
	childRoutines, childCtx := errgroup.WithContext(rootCtx)

	return &GrafanaServerImpl{
		context:       childCtx,
		shutdownFn:    shutdownFn,
		childRoutines: childRoutines,
		log:           log.New("server"),
		cfg:           setting.NewCfg(),
	}
}

type GrafanaServerImpl struct {
	context            context.Context
	shutdownFn         context.CancelFunc
	childRoutines      *errgroup.Group
	log                log.Logger
	cfg                *setting.Cfg
	shutdownReason     string
	shutdownInProgress bool

	RouteRegister api.RouteRegister `inject:""`
	HttpServer    *api.HTTPServer   `inject:""`
}

func (g *GrafanaServerImpl) Run() error {
	g.loadConfiguration()
	g.writePIDFile()

	// initSql
	sqlstore.NewEngine() // TODO: this should return an error
	sqlstore.EnsureAdminUser()

	login.Init()
	social.NewOAuthService()

	tracingCloser, err := tracing.Init(g.cfg.Raw)
	if err != nil {
		return fmt.Errorf("Tracing settings is not valid. error: %v", err)
	}
	defer tracingCloser.Close()

	serviceGraph := inject.Graph{}
	serviceGraph.Provide(&inject.Object{Value: bus.GetBus()})
	serviceGraph.Provide(&inject.Object{Value: g.cfg})
	serviceGraph.Provide(&inject.Object{Value: dashboards.NewProvisioningService()})
	serviceGraph.Provide(&inject.Object{Value: api.NewRouteRegister(middleware.RequestMetrics, middleware.RequestTracing)})

	// self registered services
	services := registry.GetServices()

	// Add all services to dependency graph
	for _, service := range services {
		serviceGraph.Provide(&inject.Object{Value: service})
	}

	serviceGraph.Provide(&inject.Object{Value: g})

	// Inject dependencies to services
	if err := serviceGraph.Populate(); err != nil {
		return fmt.Errorf("Failed to populate service dependency: %v", err)
	}

	// Init & start services
	for _, service := range services {
		if registry.IsDisabled(service) {
			continue
		}

		g.log.Info("Initializing " + reflect.TypeOf(service).Elem().Name())

		if err := service.Init(); err != nil {
			return fmt.Errorf("Service init failed: %v", err)
		}
	}

	// Start background services
	for index := range services {
		service, ok := services[index].(registry.BackgroundService)
		if !ok {
			continue
		}

		if registry.IsDisabled(services[index]) {
			continue
		}

		g.childRoutines.Go(func() error {
			// Skip starting new service when shutting down
			// Can happen when service stop/return during startup
			if g.shutdownInProgress {
				return nil
			}

			err := service.Run(g.context)

			// If error is not canceled then the service crashed
			if err != context.Canceled && err != nil {
				g.log.Error("Stopped "+reflect.TypeOf(service).Elem().Name(), "reason", err)
			} else {
				g.log.Info("Stopped "+reflect.TypeOf(service).Elem().Name(), "reason", err)
			}

			// Mark that we are in shutdown mode
			// So more services are not started
			g.shutdownInProgress = true
			return err
		})
	}

	sendSystemdNotification("READY=1")

	return g.childRoutines.Wait()
}

func (g *GrafanaServerImpl) loadConfiguration() {
	err := g.cfg.Load(&setting.CommandLineArgs{
		Config:   *configFile,
		HomePath: *homePath,
		Args:     flag.Args(),
	})

	if err != nil {
		fmt.Fprintf(os.Stderr, "Failed to start grafana. error: %s\n", err.Error())
		os.Exit(1)
	}

	g.log.Info("Starting "+setting.ApplicationName, "version", version, "commit", commit, "compiled", time.Unix(setting.BuildStamp, 0))
	g.cfg.LogConfigSources()
}

func (g *GrafanaServerImpl) Shutdown(reason string) {
	g.log.Info("Shutdown started", "reason", reason)
	g.shutdownReason = reason
	g.shutdownInProgress = true

	// call cancel func on root context
	g.shutdownFn()

	// wait for child routines
	g.childRoutines.Wait()
}

func (g *GrafanaServerImpl) Exit(reason error) {
	// default exit code is 1
	code := 1

	if reason == context.Canceled && g.shutdownReason != "" {
		reason = fmt.Errorf(g.shutdownReason)
		code = 0
	}

	g.log.Error("Server shutdown", "reason", reason)
	os.Exit(code)
}

func (g *GrafanaServerImpl) writePIDFile() {
	if *pidFile == "" {
		return
	}

	// Ensure the required directory structure exists.
	err := os.MkdirAll(filepath.Dir(*pidFile), 0700)
	if err != nil {
		g.log.Error("Failed to verify pid directory", "error", err)
		os.Exit(1)
	}

	// Retrieve the PID and write it.
	pid := strconv.Itoa(os.Getpid())
	if err := ioutil.WriteFile(*pidFile, []byte(pid), 0644); err != nil {
		g.log.Error("Failed to write pidfile", "error", err)
		os.Exit(1)
	}

	g.log.Info("Writing PID file", "path", *pidFile, "pid", pid)
}

func sendSystemdNotification(state string) error {
	notifySocket := os.Getenv("NOTIFY_SOCKET")

	if notifySocket == "" {
		return fmt.Errorf("NOTIFY_SOCKET environment variable empty or unset.")
	}

	socketAddr := &net.UnixAddr{
		Name: notifySocket,
		Net:  "unixgram",
	}

	conn, err := net.DialUnix(socketAddr.Net, nil, socketAddr)

	if err != nil {
		return err
	}

	_, err = conn.Write([]byte(state))

	conn.Close()

	return err
}
<|endoftext|>


<commit_before>package hooks

import (
	"bytes"

	"github.com/deckarep/gosx-notifier"
)
import "text/template"

// NotificationDriver is a driver that notifications some texts to the terminal
type NotificationDriver struct {
	line string
}

// NewNotificationDriver returns a NotificationDriver instance
func NewNotificationDriver(line string) (NotificationDriver, error) {
	return NotificationDriver{
		line: line,
	}, nil
}

// Run notifications a line to the terminal
func (d NotificationDriver) Run(args RunArgs) error {
	// FIXME: handle non OS X hosts

	var buff bytes.Buffer
	tmpl, err := template.New("notification").Parse(d.line + "\n")
	if err != nil {
		return err
	}

	if err := tmpl.Execute(&buff, args); err != nil {
		return err
	}

	note := gosxnotifier.NewNotification(buff.String())
	note.Title = "SSH"
	note.Sound = gosxnotifier.Basso
	return note.Push()
}
<commit_msg>Switch to crossplatform notification library<commit_after>package hooks

import (
	"bytes"

	"github.com/haklop/gnotifier"
)
import "text/template"

// NotificationDriver is a driver that notifications some texts to the terminal
type NotificationDriver struct {
	line string
}

// NewNotificationDriver returns a NotificationDriver instance
func NewNotificationDriver(line string) (NotificationDriver, error) {
	return NotificationDriver{
		line: line,
	}, nil
}

// Run notifications a line to the terminal
func (d NotificationDriver) Run(args RunArgs) error {
	var buff bytes.Buffer
	tmpl, err := template.New("notification").Parse(d.line + "\n")
	if err != nil {
		return err
	}

	if err := tmpl.Execute(&buff, args); err != nil {
		return err
	}

	notification := gnotifier.Notification("ASSH", buff.String())
	notification.GetConfig().Expiration = 3000
	notification.GetConfig().ApplicationName = "assh"

	return notification.Push()
}
<|endoftext|>


<commit_before>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cmd

import (
	"fmt"
	"io"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/kubectl"
	"github.com/spf13/cobra"
)

const (
	updatePeriod = "1m0s"
	timeout      = "5m0s"
	pollInterval = "3s"
)

func (f *Factory) NewCmdRollingUpdate(out io.Writer) *cobra.Command {
	cmd := &cobra.Command{
		Use:   "rollingupdate <old-controller-name> -f <new-controller.json>",
		Short: "Perform a rolling update of the given ReplicationController",
		Long: `Perform a rolling update of the given ReplicationController.

Replaces named controller with new controller, updating one pod at a time to use the
new PodTemplate. The new-controller.json must specify the same namespace as the
existing controller and overwrite at least one (common) label in its replicaSelector.

Examples:
$ kubectl rollingupdate frontend-v1 -f frontend-v2.json
  <update pods of frontend-v1 using new controller data in frontend-v2.json>

$ cat frontend-v2.json | kubectl rollingupdate frontend-v1 -f -
  <update pods of frontend-v1 using json data passed into stdin>`,
		Run: func(cmd *cobra.Command, args []string) {
			filename := GetFlagString(cmd, "filename")
			if len(filename) == 0 {
				usageError(cmd, "Must specify filename for new controller")
			}
			period := GetFlagDuration(cmd, "update-period")
			interval := GetFlagDuration(cmd, "poll-interval")
			timeout := GetFlagDuration(cmd, "timeout")
			if len(args) != 1 {
				usageError(cmd, "Must specify the controller to update")
			}
			oldName := args[0]
			schema, err := f.Validator(cmd)
			checkErr(err)
			mapper, typer := f.Object(cmd)
			mapping, namespace, newName, data := ResourceFromFile(cmd, filename, typer, mapper, schema)
			if mapping.Kind != "ReplicationController" {
				usageError(cmd, "%s does not specify a valid ReplicationController", filename)
			}
			err = CompareNamespaceFromFile(cmd, namespace)
			checkErr(err)

			client, err := f.Client(cmd)
			checkErr(err)

			obj, err := mapping.Codec.Decode(data)
			checkErr(err)
			newRc := obj.(*api.ReplicationController)

			updater := kubectl.NewRollingUpdater(namespace, client)

			// fetch rc
			oldRc, err := client.ReplicationControllers(namespace).Get(oldName)
			checkErr(err)

			var hasLabel bool
			for key, oldValue := range oldRc.Spec.Selector {
				if newValue, ok := newRc.Spec.Selector[key]; ok && newValue != oldValue {
					hasLabel = true
					break
				}
			}
			if !hasLabel {
				usageError(cmd, "%s must specify a matching key with non-equal value in Selector for %s",
					filename, oldName)
			}
			// TODO: handle resizes during rolling update
			if newRc.Spec.Replicas == 0 {
				newRc.Spec.Replicas = oldRc.Spec.Replicas
			}
			err = updater.Update(out, oldRc, newRc, period, interval, timeout)
			checkErr(err)

			fmt.Fprintf(out, "%s\n", newName)
		},
	}
	cmd.Flags().String("update-period", updatePeriod, `Time to wait between updating pods. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".`)
	cmd.Flags().String("poll-interval", pollInterval, `Time delay between polling controller status after update. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".`)
	cmd.Flags().String("timeout", timeout, `Max time to wait for a controller to update before giving up. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".`)
	cmd.Flags().StringP("filename", "f", "", "Filename or URL to file to use to create the new controller")
	return cmd
}
<commit_msg>Fail rollingupdate if replacement rc schema has same name as existing<commit_after>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cmd

import (
	"fmt"
	"io"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/kubectl"
	"github.com/spf13/cobra"
)

const (
	updatePeriod = "1m0s"
	timeout      = "5m0s"
	pollInterval = "3s"
)

func (f *Factory) NewCmdRollingUpdate(out io.Writer) *cobra.Command {
	cmd := &cobra.Command{
		Use:   "rollingupdate <old-controller-name> -f <new-controller.json>",
		Short: "Perform a rolling update of the given ReplicationController",
		Long: `Perform a rolling update of the given ReplicationController.

Replaces named controller with new controller, updating one pod at a time to use the
new PodTemplate. The new-controller.json must specify the same namespace as the
existing controller and overwrite at least one (common) label in its replicaSelector.

Examples:
$ kubectl rollingupdate frontend-v1 -f frontend-v2.json
  <update pods of frontend-v1 using new controller data in frontend-v2.json>

$ cat frontend-v2.json | kubectl rollingupdate frontend-v1 -f -
  <update pods of frontend-v1 using json data passed into stdin>`,
		Run: func(cmd *cobra.Command, args []string) {
			filename := GetFlagString(cmd, "filename")
			if len(filename) == 0 {
				usageError(cmd, "Must specify filename for new controller")
			}
			period := GetFlagDuration(cmd, "update-period")
			interval := GetFlagDuration(cmd, "poll-interval")
			timeout := GetFlagDuration(cmd, "timeout")
			if len(args) != 1 {
				usageError(cmd, "Must specify the controller to update")
			}
			oldName := args[0]
			schema, err := f.Validator(cmd)
			checkErr(err)
			mapper, typer := f.Object(cmd)
			mapping, namespace, newName, data := ResourceFromFile(cmd, filename, typer, mapper, schema)
			if mapping.Kind != "ReplicationController" {
				usageError(cmd, "%s does not specify a valid ReplicationController", filename)
			}
			if oldName == newName {
				usageError(cmd, "%s cannot have the same name as the existing ReplicationController %s",
					filename, oldName)
			}
			err = CompareNamespaceFromFile(cmd, namespace)
			checkErr(err)

			client, err := f.Client(cmd)
			checkErr(err)

			obj, err := mapping.Codec.Decode(data)
			checkErr(err)
			newRc := obj.(*api.ReplicationController)

			updater := kubectl.NewRollingUpdater(namespace, client)

			// fetch rc
			oldRc, err := client.ReplicationControllers(namespace).Get(oldName)
			checkErr(err)

			var hasLabel bool
			for key, oldValue := range oldRc.Spec.Selector {
				if newValue, ok := newRc.Spec.Selector[key]; ok && newValue != oldValue {
					hasLabel = true
					break
				}
			}
			if !hasLabel {
				usageError(cmd, "%s must specify a matching key with non-equal value in Selector for %s",
					filename, oldName)
			}
			// TODO: handle resizes during rolling update
			if newRc.Spec.Replicas == 0 {
				newRc.Spec.Replicas = oldRc.Spec.Replicas
			}
			err = updater.Update(out, oldRc, newRc, period, interval, timeout)
			checkErr(err)

			fmt.Fprintf(out, "%s\n", newName)
		},
	}
	cmd.Flags().String("update-period", updatePeriod, `Time to wait between updating pods. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".`)
	cmd.Flags().String("poll-interval", pollInterval, `Time delay between polling controller status after update. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".`)
	cmd.Flags().String("timeout", timeout, `Max time to wait for a controller to update before giving up. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".`)
	cmd.Flags().StringP("filename", "f", "", "Filename or URL to file to use to create the new controller")
	return cmd
}
<|endoftext|>


<commit_before>package status

import (
	"fmt"

	"github.com/bitfinexcom/bitfinex-api-go/pkg/convert"
)

type DerivativesSnapshot struct {
	Snapshot []*Derivative
}

type Derivative struct {
	Symbol               string
	MTS                  int64
	Price                float64
	SpotPrice            float64
	InsuranceFundBalance float64
	FundingEventMTS      int64
	FundingAccrued       float64
	FundingStep          float64
	CurrentFunding       float64
	MarkPrice            float64
	OpenInterest         float64
	ClampMIN             float64
	ClampMAX             float64
}

func DerivFromRaw(symbol string, raw []interface{}) (*Derivative, error) {
	if len(raw) < 22 {
		return nil, fmt.Errorf("data slice too short for derivative status: %#v", raw)
	}

	return &Derivative{
		Symbol:               symbol,
		MTS:                  convert.I64ValOrZero(raw[0]),
		Price:                convert.F64ValOrZero(raw[2]),
		SpotPrice:            convert.F64ValOrZero(raw[3]),
		InsuranceFundBalance: convert.F64ValOrZero(raw[5]),
		FundingEventMTS:      convert.I64ValOrZero(raw[7]),
		FundingAccrued:       convert.F64ValOrZero(raw[8]),
		FundingStep:          convert.F64ValOrZero(raw[9]),
		CurrentFunding:       convert.F64ValOrZero(raw[11]),
		MarkPrice:            convert.F64ValOrZero(raw[14]),
		OpenInterest:         convert.F64ValOrZero(raw[17]),
		ClampMIN:             convert.F64ValOrZero(raw[21]),
		ClampMAX:             convert.F64ValOrZero(raw[22]),
	}, nil
}

func DerivSnapshotFromRaw(symbol string, raw [][]interface{}) (*DerivativesSnapshot, error) {
	snapshot := make([]*Derivative, len(raw))
	for i, r := range raw {
		d, err := DerivFromRaw(symbol, r)
		if err != nil {
			return nil, err
		}
		snapshot[i] = d
	}
	return &DerivativesSnapshot{Snapshot: snapshot}, nil
}

func DerivFromRestRaw(raw []interface{}) (t *Derivative, err error) {
	if len(raw) < 2 {
		return t, fmt.Errorf("data slice too short for derivatives: %#v", raw)
	}

	return DerivFromRaw(raw[0].(string), raw[1:])
}
<commit_msg>adding extra validation check for DerivSnapshotFromRaw<commit_after>package status

import (
	"fmt"

	"github.com/bitfinexcom/bitfinex-api-go/pkg/convert"
)

type DerivativesSnapshot struct {
	Snapshot []*Derivative
}

type Derivative struct {
	Symbol               string
	MTS                  int64
	Price                float64
	SpotPrice            float64
	InsuranceFundBalance float64
	FundingEventMTS      int64
	FundingAccrued       float64
	FundingStep          float64
	CurrentFunding       float64
	MarkPrice            float64
	OpenInterest         float64
	ClampMIN             float64
	ClampMAX             float64
}

func DerivFromRaw(symbol string, raw []interface{}) (*Derivative, error) {
	if len(raw) < 22 {
		return nil, fmt.Errorf("data slice too short for derivative status: %#v", raw)
	}

	return &Derivative{
		Symbol:               symbol,
		MTS:                  convert.I64ValOrZero(raw[0]),
		Price:                convert.F64ValOrZero(raw[2]),
		SpotPrice:            convert.F64ValOrZero(raw[3]),
		InsuranceFundBalance: convert.F64ValOrZero(raw[5]),
		FundingEventMTS:      convert.I64ValOrZero(raw[7]),
		FundingAccrued:       convert.F64ValOrZero(raw[8]),
		FundingStep:          convert.F64ValOrZero(raw[9]),
		CurrentFunding:       convert.F64ValOrZero(raw[11]),
		MarkPrice:            convert.F64ValOrZero(raw[14]),
		OpenInterest:         convert.F64ValOrZero(raw[17]),
		ClampMIN:             convert.F64ValOrZero(raw[21]),
		ClampMAX:             convert.F64ValOrZero(raw[22]),
	}, nil
}

func DerivSnapshotFromRaw(symbol string, raw [][]interface{}) (*DerivativesSnapshot, error) {
	if len(raw) == 0 {
		return nil, fmt.Errorf("empty data slice")
	}

	snapshot := make([]*Derivative, len(raw))
	for i, r := range raw {
		d, err := DerivFromRaw(symbol, r)
		if err != nil {
			return nil, err
		}
		snapshot[i] = d
	}
	return &DerivativesSnapshot{Snapshot: snapshot}, nil
}

func DerivFromRestRaw(raw []interface{}) (t *Derivative, err error) {
	if len(raw) < 2 {
		return t, fmt.Errorf("data slice too short for derivatives: %#v", raw)
	}

	return DerivFromRaw(raw[0].(string), raw[1:])
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Rook Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package object

import (
	"encoding/json"
	"strconv"
	"strings"
	"syscall"

	"github.com/pkg/errors"
	"github.com/rook/rook/pkg/util/exec"
)

const (
	RGWErrorNone = iota
	RGWErrorUnknown
	RGWErrorNotFound
	RGWErrorBadData
	RGWErrorParse
	ErrorCodeFileExists = 17
)

// An ObjectUser defines the details of an object store user.
type ObjectUser struct {
	UserID      string  `json:"userId"`
	DisplayName *string `json:"displayName"`
	Email       *string `json:"email"`
	AccessKey   *string `json:"accessKey"`
	SecretKey   *string `json:"secretKey"`
}

// ListUsers lists the object pool users.
func ListUsers(c *Context) ([]string, int, error) {
	result, err := runAdminCommand(c, "user", "list")
	if err != nil {
		return nil, RGWErrorUnknown, errors.Wrap(err, "failed to list users")
	}

	var s []string
	if err := json.Unmarshal([]byte(result), &s); err != nil {
		return nil, RGWErrorParse, errors.Wrapf(err, "failed to read users info result=%s", result)
	}

	return s, RGWErrorNone, nil
}

type rgwUserInfo struct {
	UserID      string `json:"user_id"`
	DisplayName string `json:"display_name"`
	Email       string `json:"email"`
	Keys        []struct {
		AccessKey string `json:"access_key"`
		SecretKey string `json:"secret_key"`
	}
}

func decodeUser(data string) (*ObjectUser, int, error) {
	var user rgwUserInfo
	err := json.Unmarshal([]byte(data), &user)
	if err != nil {
		return nil, RGWErrorParse, errors.Wrapf(err, "failed to unmarshal json. %s", data)
	}

	rookUser := ObjectUser{UserID: user.UserID, DisplayName: &user.DisplayName, Email: &user.Email}

	if len(user.Keys) > 0 {
		rookUser.AccessKey = &user.Keys[0].AccessKey
		rookUser.SecretKey = &user.Keys[0].SecretKey
	}

	return &rookUser, RGWErrorNone, nil
}

// GetUser returns the user with the given ID.
func GetUser(c *Context, id string) (*ObjectUser, int, error) {
	logger.Debugf("getting s3 user %q", id)

	// note: err is set for non-existent user but result output is also empty
	result, err := runAdminCommand(c, "user", "info", "--uid", id)
	if strings.Contains(result, "no user info saved") {
		return nil, RGWErrorNotFound, errors.New("warn: s3 user not found")
	}
	if err != nil {
		return nil, RGWErrorUnknown, errors.Wrapf(err, "radosgw-admin command err. %s", result)
	}
	return decodeUser(result)
}

// CreateUser creates a new user with the information given.
func CreateUser(c *Context, user ObjectUser) (*ObjectUser, int, error) {
	logger.Debugf("creating s3 user %q", user.UserID)

	if strings.TrimSpace(user.UserID) == "" {
		return nil, RGWErrorBadData, errors.New("userId cannot be empty")
	}

	if user.DisplayName == nil {
		return nil, RGWErrorBadData, errors.New("displayName is required")
	}

	args := []string{
		"user",
		"create",
		"--uid", user.UserID,
		"--display-name", *user.DisplayName,
	}

	if user.Email != nil {
		args = append(args, "--email", *user.Email)
	}

	result, err := runAdminCommand(c, args...)
	if err != nil {
		if strings.Contains(result, "could not create user: unable to create user, user: ") {
			return nil, ErrorCodeFileExists, errors.New("s3 user already exists")
		}

		if strings.Contains(result, "could not create user: unable to create user, email: ") && strings.Contains(result, " is the email address an existing user") {
			return nil, RGWErrorBadData, errors.New("email already in use")
		}

		// We don't know what happened
		return nil, RGWErrorUnknown, errors.Wrap(err, "failed to create s3 user")
	}

	return decodeUser(result)
}

// UpdateUser updates the user whose ID matches the user.
func UpdateUser(c *Context, user ObjectUser) (*ObjectUser, int, error) {
	logger.Infof("updating s3 user %q", user.UserID)

	args := []string{"user", "modify", "--uid", user.UserID}

	if user.DisplayName != nil {
		args = append(args, "--display-name", *user.DisplayName)
	}
	if user.Email != nil {
		args = append(args, "--email", *user.Email)
	}

	body, err := runAdminCommand(c, args...)
	if err != nil {
		return nil, RGWErrorUnknown, errors.Wrap(err, "failed to update s3 user")
	}

	if body == "could not modify user: unable to modify user, user not found" {
		return nil, RGWErrorNotFound, errors.New("s3 user not found")
	}

	return decodeUser(body)
}

// DeleteUser deletes the user with the given ID.
func DeleteUser(c *Context, id string, opts ...string) (string, error) {
	args := []string{"user", "rm", "--uid", id}
	if opts != nil {
		args = append(args, opts...)
	}
	result, err := runAdminCommand(c, args...)
	if err != nil {
		// If User does not exist return success
		if code, ok := exec.ExitStatus(err); ok && code == int(syscall.ENOENT) {
			return result, nil
		}
	}

	return result, errors.Wrap(err, "failed to delete s3 user")
}

func SetQuotaUserBucketMax(c *Context, id string, max int) (string, error) {
	logger.Infof("Setting user %q max buckets to %d", id, max)
	args := []string{"--quota-scope", "user", "--max-buckets", strconv.Itoa(max)}
	result, err := setUserQuota(c, id, args)
	if err != nil {
		err = errors.Wrap(err, "failed setting bucket max")
	}
	return result, err
}

func setUserQuota(c *Context, id string, args []string) (string, error) {
	args = append([]string{"quota", "set", "--uid", id}, args...)
	result, err := runAdminCommand(c, args...)
	if err != nil {
		err = errors.Wrap(err, "failed to set max buckets for user")
	}
	return result, err
}

func LinkUser(c *Context, id, bucket string) (string, int, error) {
	logger.Infof("Linking (user: %s) (bucket: %s)", id, bucket)
	args := []string{"bucket", "link", "--uid", id, "--bucket", bucket}
	result, err := runAdminCommand(c, args...)
	if err != nil {
		return "", RGWErrorUnknown, err
	}
	if strings.Contains(result, "bucket entry point user mismatch") {
		return "", RGWErrorNotFound, err
	}
	return result, RGWErrorNone, nil
}

func UnlinkUser(c *Context, id, bucket string) (string, int, error) {
	logger.Infof("Unlinking (user: %s) (bucket: %s)", id, bucket)
	args := []string{"bucket", "unlink", "--uid", id, "--bucket", bucket}
	result, err := runAdminCommand(c, args...)
	if err != nil {
		return "", RGWErrorUnknown, err
	}
	if strings.Contains(result, "bucket entry point user mismatch") {
		return "", RGWErrorNotFound, err
	}
	return result, RGWErrorNone, nil
}
<commit_msg>ceph: add missing explaination for apis in user.go<commit_after>/*
Copyright 2016 The Rook Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package object

import (
	"encoding/json"
	"strconv"
	"strings"
	"syscall"

	"github.com/pkg/errors"
	"github.com/rook/rook/pkg/util/exec"
)

const (
	RGWErrorNone = iota
	RGWErrorUnknown
	RGWErrorNotFound
	RGWErrorBadData
	RGWErrorParse
	ErrorCodeFileExists = 17
)

// An ObjectUser defines the details of an object store user.
type ObjectUser struct {
	UserID      string  `json:"userId"`
	DisplayName *string `json:"displayName"`
	Email       *string `json:"email"`
	AccessKey   *string `json:"accessKey"`
	SecretKey   *string `json:"secretKey"`
}

// ListUsers lists the object pool users.
func ListUsers(c *Context) ([]string, int, error) {
	result, err := runAdminCommand(c, "user", "list")
	if err != nil {
		return nil, RGWErrorUnknown, errors.Wrap(err, "failed to list users")
	}

	var s []string
	if err := json.Unmarshal([]byte(result), &s); err != nil {
		return nil, RGWErrorParse, errors.Wrapf(err, "failed to read users info result=%s", result)
	}

	return s, RGWErrorNone, nil
}

type rgwUserInfo struct {
	UserID      string `json:"user_id"`
	DisplayName string `json:"display_name"`
	Email       string `json:"email"`
	Keys        []struct {
		AccessKey string `json:"access_key"`
		SecretKey string `json:"secret_key"`
	}
}

func decodeUser(data string) (*ObjectUser, int, error) {
	var user rgwUserInfo
	err := json.Unmarshal([]byte(data), &user)
	if err != nil {
		return nil, RGWErrorParse, errors.Wrapf(err, "failed to unmarshal json. %s", data)
	}

	rookUser := ObjectUser{UserID: user.UserID, DisplayName: &user.DisplayName, Email: &user.Email}

	if len(user.Keys) > 0 {
		rookUser.AccessKey = &user.Keys[0].AccessKey
		rookUser.SecretKey = &user.Keys[0].SecretKey
	}

	return &rookUser, RGWErrorNone, nil
}

// GetUser returns the user with the given ID.
func GetUser(c *Context, id string) (*ObjectUser, int, error) {
	logger.Debugf("getting s3 user %q", id)

	// note: err is set for non-existent user but result output is also empty
	result, err := runAdminCommand(c, "user", "info", "--uid", id)
	if strings.Contains(result, "no user info saved") {
		return nil, RGWErrorNotFound, errors.New("warn: s3 user not found")
	}
	if err != nil {
		return nil, RGWErrorUnknown, errors.Wrapf(err, "radosgw-admin command err. %s", result)
	}
	return decodeUser(result)
}

// CreateUser creates a new user with the information given.
func CreateUser(c *Context, user ObjectUser) (*ObjectUser, int, error) {
	logger.Debugf("creating s3 user %q", user.UserID)

	if strings.TrimSpace(user.UserID) == "" {
		return nil, RGWErrorBadData, errors.New("userId cannot be empty")
	}

	if user.DisplayName == nil {
		return nil, RGWErrorBadData, errors.New("displayName is required")
	}

	args := []string{
		"user",
		"create",
		"--uid", user.UserID,
		"--display-name", *user.DisplayName,
	}

	if user.Email != nil {
		args = append(args, "--email", *user.Email)
	}

	result, err := runAdminCommand(c, args...)
	if err != nil {
		if strings.Contains(result, "could not create user: unable to create user, user: ") {
			return nil, ErrorCodeFileExists, errors.New("s3 user already exists")
		}

		if strings.Contains(result, "could not create user: unable to create user, email: ") && strings.Contains(result, " is the email address an existing user") {
			return nil, RGWErrorBadData, errors.New("email already in use")
		}

		// We don't know what happened
		return nil, RGWErrorUnknown, errors.Wrap(err, "failed to create s3 user")
	}

	return decodeUser(result)
}

// UpdateUser updates the user whose ID matches the user.
func UpdateUser(c *Context, user ObjectUser) (*ObjectUser, int, error) {
	logger.Infof("updating s3 user %q", user.UserID)

	args := []string{"user", "modify", "--uid", user.UserID}

	if user.DisplayName != nil {
		args = append(args, "--display-name", *user.DisplayName)
	}
	if user.Email != nil {
		args = append(args, "--email", *user.Email)
	}

	body, err := runAdminCommand(c, args...)
	if err != nil {
		return nil, RGWErrorUnknown, errors.Wrap(err, "failed to update s3 user")
	}

	if body == "could not modify user: unable to modify user, user not found" {
		return nil, RGWErrorNotFound, errors.New("s3 user not found")
	}

	return decodeUser(body)
}

// DeleteUser deletes the user with the given ID.
func DeleteUser(c *Context, id string, opts ...string) (string, error) {
	args := []string{"user", "rm", "--uid", id}
	if opts != nil {
		args = append(args, opts...)
	}
	result, err := runAdminCommand(c, args...)
	if err != nil {
		// If User does not exist return success
		if code, ok := exec.ExitStatus(err); ok && code == int(syscall.ENOENT) {
			return result, nil
		}
	}

	return result, errors.Wrap(err, "failed to delete s3 user")
}

// SetQuotaUserBucketMax will set maximum bucket quota for a user
func SetQuotaUserBucketMax(c *Context, id string, max int) (string, error) {
	logger.Infof("Setting user %q max buckets to %d", id, max)
	args := []string{"--quota-scope", "user", "--max-buckets", strconv.Itoa(max)}
	result, err := setUserQuota(c, id, args)
	if err != nil {
		err = errors.Wrap(err, "failed setting bucket max")
	}
	return result, err
}

func setUserQuota(c *Context, id string, args []string) (string, error) {
	args = append([]string{"quota", "set", "--uid", id}, args...)
	result, err := runAdminCommand(c, args...)
	if err != nil {
		err = errors.Wrap(err, "failed to set max buckets for user")
	}
	return result, err
}

// LinkUser will link a user to a bucket
func LinkUser(c *Context, id, bucket string) (string, int, error) {
	logger.Infof("Linking (user: %s) (bucket: %s)", id, bucket)
	args := []string{"bucket", "link", "--uid", id, "--bucket", bucket}
	result, err := runAdminCommand(c, args...)
	if err != nil {
		return "", RGWErrorUnknown, err
	}
	if strings.Contains(result, "bucket entry point user mismatch") {
		return "", RGWErrorNotFound, err
	}
	return result, RGWErrorNone, nil
}

// UnlinkUser will unlink the user from a bucket
func UnlinkUser(c *Context, id, bucket string) (string, int, error) {
	logger.Infof("Unlinking (user: %s) (bucket: %s)", id, bucket)
	args := []string{"bucket", "unlink", "--uid", id, "--bucket", bucket}
	result, err := runAdminCommand(c, args...)
	if err != nil {
		return "", RGWErrorUnknown, err
	}
	if strings.Contains(result, "bucket entry point user mismatch") {
		return "", RGWErrorNotFound, err
	}
	return result, RGWErrorNone, nil
}
<|endoftext|>


<commit_before>package transport

import (
	"crypto/tls"
	"fmt"
	"net"
	"net/http"
	"strings"
	"time"

	"github.com/sirupsen/logrus"
	"golang.org/x/net/http2"
)

const (
	// DefaultDialTimeout when connecting to a backend server.
	DefaultDialTimeout = 30 * time.Second

	// DefaultIdleConnsPerHost the default value set for http.Transport.MaxIdleConnsPerHost.
	DefaultIdleConnsPerHost = 64

	// DefaultIdleConnTimeout is the default value for the the maximum amount of time an idle
	// (keep-alive) connection will remain idle before closing itself.
	DefaultIdleConnTimeout = 90 * time.Second
)

type transport struct {
	// Same as net/http.Transport.MaxIdleConnsPerHost, but the default
	// is 64. This value supports scenarios with relatively few remote
	// hosts. When the routing table contains different hosts in the
	// range of hundreds, it is recommended to set this options to a
	// lower value.
	idleConnectionsPerHost int
	insecureSkipVerify     bool
	dialTimeout            time.Duration
	responseHeaderTimeout  time.Duration
	idleConnTimeout        time.Duration
	idleConnPurgeTicker    *time.Ticker
}

func (t transport) hash() string {
	return strings.Join([]string{
		fmt.Sprintf("idleConnectionsPerHost:%v", t.idleConnectionsPerHost),
		fmt.Sprintf("insecureSkipVerify:%v", t.insecureSkipVerify),
		fmt.Sprintf("dialTimeout:%v", t.dialTimeout),
		fmt.Sprintf("responseHeaderTimeout:%v", t.responseHeaderTimeout),
		fmt.Sprintf("idleConnTimeout:%v", t.idleConnTimeout),
	}, ";")
}

var registryInstance *registry

func init() {
	registryInstance = newRegistry()
}

// New creates a new instance of Transport with the given params
func New(opts ...Option) *http.Transport {
	t := transport{}

	for _, opt := range opts {
		opt(&t)
	}

	if t.dialTimeout <= 0 {
		t.dialTimeout = DefaultDialTimeout
	}

	if t.idleConnectionsPerHost <= 0 {
		t.idleConnectionsPerHost = DefaultIdleConnsPerHost
	}

	if t.idleConnTimeout == 0 {
		t.idleConnTimeout = DefaultIdleConnTimeout
	}

	// let's try to get the cached transport from registry, since there is no need to create lots of
	// transports with the same configuration
	hash := t.hash()
	if tr, ok := registryInstance.get(hash); ok {
		return tr
	}

	tr := &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		DialContext: (&net.Dialer{
			Timeout:   t.dialTimeout,
			KeepAlive: 30 * time.Second,
			DualStack: true,
		}).DialContext,
		MaxIdleConns:          100,
		IdleConnTimeout:       t.idleConnTimeout,
		TLSHandshakeTimeout:   10 * time.Second,
		ExpectContinueTimeout: 1 * time.Second,
		ResponseHeaderTimeout: t.responseHeaderTimeout,
		MaxIdleConnsPerHost:   t.idleConnectionsPerHost,
		TLSClientConfig:       &tls.Config{InsecureSkipVerify: t.insecureSkipVerify},
	}

	http2.ConfigureTransport(tr)

	// Create a channel that listens to idleConnPurgeTicker to periodically purge idle connections
	if t.idleConnPurgeTicker != nil {
		go func(transport *http.Transport) {
			for {
				select {
				case <-t.idleConnPurgeTicker.C:
					logrus.Info("Closing idle connections")
					// hack it, with switching it on and off it flushes all connections
					transport.DisableKeepAlives = true
					transport.CloseIdleConnections()
					transport.DisableKeepAlives = false
				}
			}
		}(tr)
	}

	// save newly created transport in registry, to try to reuse it in the future
	registryInstance.put(hash, tr)

	return tr
}
<commit_msg>Remove log and make hack comment clearer<commit_after>package transport

import (
	"crypto/tls"
	"fmt"
	"net"
	"net/http"
	"strings"
	"time"

	"golang.org/x/net/http2"
)

const (
	// DefaultDialTimeout when connecting to a backend server.
	DefaultDialTimeout = 30 * time.Second

	// DefaultIdleConnsPerHost the default value set for http.Transport.MaxIdleConnsPerHost.
	DefaultIdleConnsPerHost = 64

	// DefaultIdleConnTimeout is the default value for the the maximum amount of time an idle
	// (keep-alive) connection will remain idle before closing itself.
	DefaultIdleConnTimeout = 90 * time.Second
)

type transport struct {
	// Same as net/http.Transport.MaxIdleConnsPerHost, but the default
	// is 64. This value supports scenarios with relatively few remote
	// hosts. When the routing table contains different hosts in the
	// range of hundreds, it is recommended to set this options to a
	// lower value.
	idleConnectionsPerHost int
	insecureSkipVerify     bool
	dialTimeout            time.Duration
	responseHeaderTimeout  time.Duration
	idleConnTimeout        time.Duration
	idleConnPurgeTicker    *time.Ticker
}

func (t transport) hash() string {
	return strings.Join([]string{
		fmt.Sprintf("idleConnectionsPerHost:%v", t.idleConnectionsPerHost),
		fmt.Sprintf("insecureSkipVerify:%v", t.insecureSkipVerify),
		fmt.Sprintf("dialTimeout:%v", t.dialTimeout),
		fmt.Sprintf("responseHeaderTimeout:%v", t.responseHeaderTimeout),
		fmt.Sprintf("idleConnTimeout:%v", t.idleConnTimeout),
	}, ";")
}

var registryInstance *registry

func init() {
	registryInstance = newRegistry()
}

// New creates a new instance of Transport with the given params
func New(opts ...Option) *http.Transport {
	t := transport{}

	for _, opt := range opts {
		opt(&t)
	}

	if t.dialTimeout <= 0 {
		t.dialTimeout = DefaultDialTimeout
	}

	if t.idleConnectionsPerHost <= 0 {
		t.idleConnectionsPerHost = DefaultIdleConnsPerHost
	}

	if t.idleConnTimeout == 0 {
		t.idleConnTimeout = DefaultIdleConnTimeout
	}

	// let's try to get the cached transport from registry, since there is no need to create lots of
	// transports with the same configuration
	hash := t.hash()
	if tr, ok := registryInstance.get(hash); ok {
		return tr
	}

	tr := &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		DialContext: (&net.Dialer{
			Timeout:   t.dialTimeout,
			KeepAlive: 30 * time.Second,
			DualStack: true,
		}).DialContext,
		MaxIdleConns:          100,
		IdleConnTimeout:       t.idleConnTimeout,
		TLSHandshakeTimeout:   10 * time.Second,
		ExpectContinueTimeout: 1 * time.Second,
		ResponseHeaderTimeout: t.responseHeaderTimeout,
		MaxIdleConnsPerHost:   t.idleConnectionsPerHost,
		TLSClientConfig:       &tls.Config{InsecureSkipVerify: t.insecureSkipVerify},
	}

	http2.ConfigureTransport(tr)

	// Create a channel that listens to idleConnPurgeTicker to periodically purge idle connections
	if t.idleConnPurgeTicker != nil {
		go func(transport *http.Transport) {
			for {
				select {
				case <-t.idleConnPurgeTicker.C:
					// This is a hack to prevent (stale) keep-alive connections from being used
					// Toggling DisableKeepAlives flushes all idle keep-alive connections
					transport.DisableKeepAlives = true
					transport.CloseIdleConnections()
					transport.DisableKeepAlives = false
				}
			}
		}(tr)
	}

	// save newly created transport in registry, to try to reuse it in the future
	registryInstance.put(hash, tr)

	return tr
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package event

import (
	"errors"
	"sync/atomic"
	"testing"
	"time"

	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest"
	"github.com/GoogleContainerTools/skaffold/proto"
	"github.com/GoogleContainerTools/skaffold/testutil"
)

func TestGetLogEvents(t *testing.T) {
	for step := 0; step < 10000; step++ {
		ev := &eventHandler{}

		ev.logEvent(proto.LogEntry{Entry: "OLD"})
		go func() {
			ev.logEvent(proto.LogEntry{Entry: "FRESH"})
			ev.logEvent(proto.LogEntry{Entry: "POISON PILL"})
		}()

		var received int32
		ev.forEachEvent(func(e *proto.LogEntry) error {
			if e.Entry == "POISON PILL" {
				return errors.New("Done")
			}

			atomic.AddInt32(&received, 1)
			return nil
		})

		if atomic.LoadInt32(&received) != 2 {
			t.Fatalf("Expected %d events, Got %d (Step: %d)", 2, received, step)
		}
	}
}

func TestGetState(t *testing.T) {
	ev := &eventHandler{
		state: emptyState(latest.BuildConfig{}),
	}

	ev.stateLock.Lock()
	ev.state.BuildState.Artifacts["img"] = Complete
	ev.stateLock.Unlock()

	state := ev.getState()

	testutil.CheckDeepEqual(t, Complete, state.BuildState.Artifacts["img"])
}

func TestDeployInProgress(t *testing.T) {
	defer func() { handler = &eventHandler{} }()

	handler = &eventHandler{
		state: emptyState(latest.BuildConfig{}),
	}

	wait(t, func() bool { return handler.getState().DeployState.Status == NotStarted })
	DeployInProgress()
	wait(t, func() bool { return handler.getState().DeployState.Status == InProgress })
}

func TestDeployFailed(t *testing.T) {
	defer func() { handler = &eventHandler{} }()

	handler = &eventHandler{
		state: emptyState(latest.BuildConfig{}),
	}

	wait(t, func() bool { return handler.getState().DeployState.Status == NotStarted })
	DeployFailed(errors.New("BUG"))
	wait(t, func() bool { return handler.getState().DeployState.Status == Failed })
}

func TestDeployComplete(t *testing.T) {
	defer func() { handler = &eventHandler{} }()

	handler = &eventHandler{
		state: emptyState(latest.BuildConfig{}),
	}

	wait(t, func() bool { return handler.getState().DeployState.Status == NotStarted })
	DeployComplete()
	wait(t, func() bool { return handler.getState().DeployState.Status == Complete })
}

func TestBuildInProgress(t *testing.T) {
	defer func() { handler = &eventHandler{} }()

	handler = &eventHandler{
		state: emptyState(latest.BuildConfig{
			Artifacts: []*latest.Artifact{{
				ImageName: "img",
			}},
		}),
	}

	wait(t, func() bool { return handler.getState().BuildState.Artifacts["img"] == NotStarted })
	BuildInProgress("img")
	wait(t, func() bool { return handler.getState().BuildState.Artifacts["img"] == InProgress })
}

func TestBuildFailed(t *testing.T) {
	defer func() { handler = &eventHandler{} }()

	handler = &eventHandler{
		state: emptyState(latest.BuildConfig{
			Artifacts: []*latest.Artifact{{
				ImageName: "img",
			}},
		}),
	}

	wait(t, func() bool { return handler.getState().BuildState.Artifacts["img"] == NotStarted })
	BuildFailed("img", errors.New("BUG"))
	wait(t, func() bool { return handler.getState().BuildState.Artifacts["img"] == Failed })
}

func TestBuildComplete(t *testing.T) {
	defer func() { handler = &eventHandler{} }()

	handler = &eventHandler{
		state: emptyState(latest.BuildConfig{
			Artifacts: []*latest.Artifact{{
				ImageName: "img",
			}},
		}),
	}

	wait(t, func() bool { return handler.getState().BuildState.Artifacts["img"] == NotStarted })
	BuildComplete("img")
	wait(t, func() bool { return handler.getState().BuildState.Artifacts["img"] == Complete })
}

func TestPortForwarded(t *testing.T) {
	defer func() { handler = &eventHandler{} }()

	handler = &eventHandler{
		state: emptyState(latest.BuildConfig{}),
	}

	wait(t, func() bool { return handler.getState().ForwardedPorts[8080] == nil })
	PortForwarded(8080, 8888, "pod", "container", "ns", "portname", "resourceType", "resourceName")
	wait(t, func() bool { return handler.getState().ForwardedPorts[8080] != nil })
}

func TestStatusCheckEventStarted(t *testing.T) {
	defer func() { handler = &eventHandler{} }()

	handler = &eventHandler{
		state: emptyState(latest.BuildConfig{}),
	}

	wait(t, func() bool { return handler.getState().StatusCheckState.Status == NotStarted })
	StatusCheckEventStarted()
	wait(t, func() bool { return handler.getState().StatusCheckState.Status == Started })
}

func TestStatusCheckEventInProgress(t *testing.T) {
	defer func() { handler = &eventHandler{} }()

	handler = &eventHandler{
		state: emptyState(latest.BuildConfig{}),
	}

	wait(t, func() bool { return handler.getState().StatusCheckState.Status == NotStarted })
	StatusCheckEventInProgress("[2/5 deployment(s) are still pending]")
	wait(t, func() bool { return handler.getState().StatusCheckState.Status == InProgress })
}

func TestStatusCheckEventSucceeded(t *testing.T) {
	defer func() { handler = &eventHandler{} }()

	handler = &eventHandler{
		state: emptyState(latest.BuildConfig{}),
	}

	wait(t, func() bool { return handler.getState().StatusCheckState.Status == NotStarted })
	StatusCheckEventSucceeded()
	wait(t, func() bool { return handler.getState().StatusCheckState.Status == Succeeded })
}

func TestStatusCheckEventFailed(t *testing.T) {
	defer func() { handler = &eventHandler{} }()

	handler = &eventHandler{
		state: emptyState(latest.BuildConfig{}),
	}

	wait(t, func() bool { return handler.getState().StatusCheckState.Status == NotStarted })
	StatusCheckEventFailed(errors.New("one or more deployments failed"))
	wait(t, func() bool { return handler.getState().StatusCheckState.Status == Failed })
}

func TestResourceStatusCheckEventUpdated(t *testing.T) {
	defer func() { handler = &eventHandler{} }()

	handler = &eventHandler{
		state: emptyState(latest.BuildConfig{}),
	}

	wait(t, func() bool { return handler.getState().StatusCheckState.Status == NotStarted })
	ResourceStatusCheckEventUpdated("ns:pod/foo", "img pull error")
	wait(t, func() bool { return handler.getState().StatusCheckState.Resources["ns:pod/foo"] == InProgress })
}

func TestResourceStatusCheckEventSucceeded(t *testing.T) {
	defer func() { handler = &eventHandler{} }()

	handler = &eventHandler{
		state: emptyState(latest.BuildConfig{}),
	}

	wait(t, func() bool { return handler.getState().StatusCheckState.Status == NotStarted })
	ResourceStatusCheckEventSucceeded("ns:pod/foo")
	wait(t, func() bool { return handler.getState().StatusCheckState.Resources["ns:pod/foo"] == Succeeded })
}

func TestResourceStatusCheckEventFailed(t *testing.T) {
	defer func() { handler = &eventHandler{} }()

	handler = &eventHandler{
		state: emptyState(latest.BuildConfig{}),
	}

	wait(t, func() bool { return handler.getState().StatusCheckState.Status == NotStarted })
	ResourceStatusCheckEventFailed("ns:pod/foo", errors.New("one or more deployments failed"))
	wait(t, func() bool { return handler.getState().StatusCheckState.Resources["ns:pod/foo"] == Failed })
}

func wait(t *testing.T, condition func() bool) {
	ticker := time.NewTicker(10 * time.Millisecond)
	defer ticker.Stop()

	timeout := time.NewTimer(5 * time.Second)
	defer timeout.Stop()

	for {
		select {
		case <-ticker.C:
			if condition() {
				return
			}

		case <-timeout.C:
			t.Fatal("Timed out waiting")
		}
	}
}

func TestResetStateOnBuild(t *testing.T) {
	defer func() { handler = &eventHandler{} }()
	handler = &eventHandler{
		state: proto.State{
			BuildState: &proto.BuildState{
				Artifacts: map[string]string{
					"image1": Complete,
				},
			},
			DeployState:      &proto.DeployState{Status: Complete},
			StatusCheckState: &proto.StatusCheckState{Status: Complete},
		},
	}
	ResetStateOnBuild()
	expected := proto.State{
		BuildState: &proto.BuildState{
			Artifacts: map[string]string{
				"image1": NotStarted,
			},
		},
		DeployState:      &proto.DeployState{Status: NotStarted},
		StatusCheckState: &proto.StatusCheckState{Status: NotStarted},
	}
	testutil.CheckDeepEqual(t, expected, handler.getState())
}

func TestResetStateOnDeploy(t *testing.T) {
	defer func() { handler = &eventHandler{} }()
	handler = &eventHandler{
		state: proto.State{
			BuildState: &proto.BuildState{
				Artifacts: map[string]string{
					"image1": Complete,
				},
			},

			DeployState:      &proto.DeployState{Status: Complete},
			StatusCheckState: &proto.StatusCheckState{Status: Complete},
		},
	}
	ResetStateOnDeploy()
	expected := proto.State{
		BuildState: &proto.BuildState{
			Artifacts: map[string]string{
				"image1": NotStarted,
			},
		},
		DeployState:      &proto.DeployState{Status: NotStarted},
		StatusCheckState: &proto.StatusCheckState{Status: NotStarted},
	}
	testutil.CheckDeepEqual(t, expected, handler.getState())
}
<commit_msg>add port forwarding<commit_after>/*
Copyright 2019 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package event

import (
	"errors"
	"sync/atomic"
	"testing"
	"time"

	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest"
	"github.com/GoogleContainerTools/skaffold/proto"
	"github.com/GoogleContainerTools/skaffold/testutil"
)

func TestGetLogEvents(t *testing.T) {
	for step := 0; step < 10000; step++ {
		ev := &eventHandler{}

		ev.logEvent(proto.LogEntry{Entry: "OLD"})
		go func() {
			ev.logEvent(proto.LogEntry{Entry: "FRESH"})
			ev.logEvent(proto.LogEntry{Entry: "POISON PILL"})
		}()

		var received int32
		ev.forEachEvent(func(e *proto.LogEntry) error {
			if e.Entry == "POISON PILL" {
				return errors.New("Done")
			}

			atomic.AddInt32(&received, 1)
			return nil
		})

		if atomic.LoadInt32(&received) != 2 {
			t.Fatalf("Expected %d events, Got %d (Step: %d)", 2, received, step)
		}
	}
}

func TestGetState(t *testing.T) {
	ev := &eventHandler{
		state: emptyState(latest.BuildConfig{}),
	}

	ev.stateLock.Lock()
	ev.state.BuildState.Artifacts["img"] = Complete
	ev.stateLock.Unlock()

	state := ev.getState()

	testutil.CheckDeepEqual(t, Complete, state.BuildState.Artifacts["img"])
}

func TestDeployInProgress(t *testing.T) {
	defer func() { handler = &eventHandler{} }()

	handler = &eventHandler{
		state: emptyState(latest.BuildConfig{}),
	}

	wait(t, func() bool { return handler.getState().DeployState.Status == NotStarted })
	DeployInProgress()
	wait(t, func() bool { return handler.getState().DeployState.Status == InProgress })
}

func TestDeployFailed(t *testing.T) {
	defer func() { handler = &eventHandler{} }()

	handler = &eventHandler{
		state: emptyState(latest.BuildConfig{}),
	}

	wait(t, func() bool { return handler.getState().DeployState.Status == NotStarted })
	DeployFailed(errors.New("BUG"))
	wait(t, func() bool { return handler.getState().DeployState.Status == Failed })
}

func TestDeployComplete(t *testing.T) {
	defer func() { handler = &eventHandler{} }()

	handler = &eventHandler{
		state: emptyState(latest.BuildConfig{}),
	}

	wait(t, func() bool { return handler.getState().DeployState.Status == NotStarted })
	DeployComplete()
	wait(t, func() bool { return handler.getState().DeployState.Status == Complete })
}

func TestBuildInProgress(t *testing.T) {
	defer func() { handler = &eventHandler{} }()

	handler = &eventHandler{
		state: emptyState(latest.BuildConfig{
			Artifacts: []*latest.Artifact{{
				ImageName: "img",
			}},
		}),
	}

	wait(t, func() bool { return handler.getState().BuildState.Artifacts["img"] == NotStarted })
	BuildInProgress("img")
	wait(t, func() bool { return handler.getState().BuildState.Artifacts["img"] == InProgress })
}

func TestBuildFailed(t *testing.T) {
	defer func() { handler = &eventHandler{} }()

	handler = &eventHandler{
		state: emptyState(latest.BuildConfig{
			Artifacts: []*latest.Artifact{{
				ImageName: "img",
			}},
		}),
	}

	wait(t, func() bool { return handler.getState().BuildState.Artifacts["img"] == NotStarted })
	BuildFailed("img", errors.New("BUG"))
	wait(t, func() bool { return handler.getState().BuildState.Artifacts["img"] == Failed })
}

func TestBuildComplete(t *testing.T) {
	defer func() { handler = &eventHandler{} }()

	handler = &eventHandler{
		state: emptyState(latest.BuildConfig{
			Artifacts: []*latest.Artifact{{
				ImageName: "img",
			}},
		}),
	}

	wait(t, func() bool { return handler.getState().BuildState.Artifacts["img"] == NotStarted })
	BuildComplete("img")
	wait(t, func() bool { return handler.getState().BuildState.Artifacts["img"] == Complete })
}

func TestPortForwarded(t *testing.T) {
	defer func() { handler = &eventHandler{} }()

	handler = &eventHandler{
		state: emptyState(latest.BuildConfig{}),
	}

	wait(t, func() bool { return handler.getState().ForwardedPorts[8080] == nil })
	PortForwarded(8080, 8888, "pod", "container", "ns", "portname", "resourceType", "resourceName")
	wait(t, func() bool { return handler.getState().ForwardedPorts[8080] != nil })
}

func TestStatusCheckEventStarted(t *testing.T) {
	defer func() { handler = &eventHandler{} }()

	handler = &eventHandler{
		state: emptyState(latest.BuildConfig{}),
	}

	wait(t, func() bool { return handler.getState().StatusCheckState.Status == NotStarted })
	StatusCheckEventStarted()
	wait(t, func() bool { return handler.getState().StatusCheckState.Status == Started })
}

func TestStatusCheckEventInProgress(t *testing.T) {
	defer func() { handler = &eventHandler{} }()

	handler = &eventHandler{
		state: emptyState(latest.BuildConfig{}),
	}

	wait(t, func() bool { return handler.getState().StatusCheckState.Status == NotStarted })
	StatusCheckEventInProgress("[2/5 deployment(s) are still pending]")
	wait(t, func() bool { return handler.getState().StatusCheckState.Status == InProgress })
}

func TestStatusCheckEventSucceeded(t *testing.T) {
	defer func() { handler = &eventHandler{} }()

	handler = &eventHandler{
		state: emptyState(latest.BuildConfig{}),
	}

	wait(t, func() bool { return handler.getState().StatusCheckState.Status == NotStarted })
	StatusCheckEventSucceeded()
	wait(t, func() bool { return handler.getState().StatusCheckState.Status == Succeeded })
}

func TestStatusCheckEventFailed(t *testing.T) {
	defer func() { handler = &eventHandler{} }()

	handler = &eventHandler{
		state: emptyState(latest.BuildConfig{}),
	}

	wait(t, func() bool { return handler.getState().StatusCheckState.Status == NotStarted })
	StatusCheckEventFailed(errors.New("one or more deployments failed"))
	wait(t, func() bool { return handler.getState().StatusCheckState.Status == Failed })
}

func TestResourceStatusCheckEventUpdated(t *testing.T) {
	defer func() { handler = &eventHandler{} }()

	handler = &eventHandler{
		state: emptyState(latest.BuildConfig{}),
	}

	wait(t, func() bool { return handler.getState().StatusCheckState.Status == NotStarted })
	ResourceStatusCheckEventUpdated("ns:pod/foo", "img pull error")
	wait(t, func() bool { return handler.getState().StatusCheckState.Resources["ns:pod/foo"] == InProgress })
}

func TestResourceStatusCheckEventSucceeded(t *testing.T) {
	defer func() { handler = &eventHandler{} }()

	handler = &eventHandler{
		state: emptyState(latest.BuildConfig{}),
	}

	wait(t, func() bool { return handler.getState().StatusCheckState.Status == NotStarted })
	ResourceStatusCheckEventSucceeded("ns:pod/foo")
	wait(t, func() bool { return handler.getState().StatusCheckState.Resources["ns:pod/foo"] == Succeeded })
}

func TestResourceStatusCheckEventFailed(t *testing.T) {
	defer func() { handler = &eventHandler{} }()

	handler = &eventHandler{
		state: emptyState(latest.BuildConfig{}),
	}

	wait(t, func() bool { return handler.getState().StatusCheckState.Status == NotStarted })
	ResourceStatusCheckEventFailed("ns:pod/foo", errors.New("one or more deployments failed"))
	wait(t, func() bool { return handler.getState().StatusCheckState.Resources["ns:pod/foo"] == Failed })
}

func wait(t *testing.T, condition func() bool) {
	ticker := time.NewTicker(10 * time.Millisecond)
	defer ticker.Stop()

	timeout := time.NewTimer(5 * time.Second)
	defer timeout.Stop()

	for {
		select {
		case <-ticker.C:
			if condition() {
				return
			}

		case <-timeout.C:
			t.Fatal("Timed out waiting")
		}
	}
}

func TestResetStateOnBuild(t *testing.T) {
	defer func() { handler = &eventHandler{} }()
	handler = &eventHandler{
		state: proto.State{
			BuildState: &proto.BuildState{
				Artifacts: map[string]string{
					"image1": Complete,
				},
			},
			DeployState: &proto.DeployState{Status: Complete},
			ForwardedPorts: map[int32]*proto.PortEvent{
				2001: {
					LocalPort:  2000,
					RemotePort: 2001,
					PodName:    "test/pod",
				},
			},
			StatusCheckState: &proto.StatusCheckState{Status: Complete},
		},
	}
	ResetStateOnBuild()
	expected := proto.State{
		BuildState: &proto.BuildState{
			Artifacts: map[string]string{
				"image1": NotStarted,
			},
		},
		DeployState:      &proto.DeployState{Status: NotStarted},
		StatusCheckState: &proto.StatusCheckState{Status: NotStarted},
	}
	testutil.CheckDeepEqual(t, expected, handler.getState())
}

func TestResetStateOnDeploy(t *testing.T) {
	defer func() { handler = &eventHandler{} }()
	handler = &eventHandler{
		state: proto.State{
			BuildState: &proto.BuildState{
				Artifacts: map[string]string{
					"image1": Complete,
				},
			},
			DeployState: &proto.DeployState{Status: Complete},
			ForwardedPorts: map[int32]*proto.PortEvent{
				2001: {
					LocalPort:  2000,
					RemotePort: 2001,
					PodName:    "test/pod",
				},
			},
			StatusCheckState: &proto.StatusCheckState{Status: Complete},
		},
	}
	ResetStateOnDeploy()
	expected := proto.State{
		BuildState: &proto.BuildState{
			Artifacts: map[string]string{
				"image1": Complete,
			},
		},
		DeployState:      &proto.DeployState{Status: NotStarted},
		StatusCheckState: &proto.StatusCheckState{Status: NotStarted},
	}
	testutil.CheckDeepEqual(t, expected, handler.getState())
}
<|endoftext|>


<commit_before><commit_msg>Fix a bug in auth core command.<commit_after><|endoftext|>


<commit_before>package main

import "encoding/json"
import "fmt"
import "net/http"
import log "github.com/Sirupsen/logrus"

type ElasticsearchShipper struct{}

func (hook *ElasticsearchShipper) Ship(logs []log.Fields) error {
	log.Debug(fmt.Sprintf("Shipping %d logs", len(logs)))

	index := Getenv("ELASTICSEARCH_INDEX", "logstash-data")
	metric_type := Getenv("METRIC_TYPE", "metricsd")

	action := ActionMap{
		"index": IndexMap{
			"_index": index,
			"_type":  metric_type,
		},
	}
	serializedAction, err := json.Marshal(action)
	if err != nil {
		fmt.Errorf("Failed to marshal action to JSON, %v", err)
		return nil
	}

	var slice []byte
	newline := []byte("\n")

	for _, item := range logs {
		serialized, err := json.Marshal(item)
		if err != nil {
			fmt.Errorf("Failed to marshal fields to JSON, %v", err)
			return nil
		}
		slice = Extend(slice, serializedAction)
		slice = Extend(slice, newline)
		slice = Extend(slice, serialized)
		slice = Extend(slice, newline)
	}

	status, err := ElasticsearchPost("/_bulk", slice)
	if err != nil {
		log.Warning("Indexing serialized data failed with err: ", err)
	}

	if status != http.StatusOK {
		log.Warning("Indexing serialized data failed with status: ", status)
	}
	return nil
}

func ElasticsearchPost(url string, data []byte) (int, error) {
	elasticsearchUrl := Getenv("ELASTICSEARCH_URL", "http://127.0.0.1:9200")
	req, err := http.NewRequest("POST", fmt.Sprintf("%s%s", elasticsearchUrl, url), bytes.NewBuffer(data))
	req.Header.Set("Content-Type", "application/json")

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		fmt.Errorf("Failed to make request, %v", err)
		return 0, err
	}
	defer resp.Body.Close()

	return resp.StatusCode, nil
}

func SetupTemplate() {
	template := `
{
	"order": 0,
	"template": "logstash-*",
	"settings": {
		"index.refresh_interval": "5s"
	},
	"mappings": {
		"_default_": {
			"dynamic_templates": [
				{
					"string_fields": {
						"mapping": {
							"index": "analyzed",
							"omit_norms": true,
							"type": "string",
							"fields": {
								"raw": {
									"index": "not_analyzed",
									"ignore_above": 256,
									"type": "string"
								}
							}
						},
						"match_mapping_type": "string",
						"match": "*"
					}
				}
			],
			"properties": {
				"geoip": {
					"dynamic": true,
					"path": "full",
					"properties": {
						"location": {
							"type": "geo_point"
						}
					},
					"type": "object"
				},
				"@version": {
					"index": "not_analyzed",
					"type": "string"
				}
			},
			"_all": {
				"enabled": true
			}
		}
	},
	"aliases": {}
}
`
	var data = []byte(template)

	status, err := ElasticsearchPost("/_template/logstash", data)
	if err != nil {
		log.Error("Indexing serialized data failed: ", err)
	}

	if status != http.StatusOK {
		log.Error("Creating index failed")
	}
}
<commit_msg>Remove geoip from being indexed<commit_after>package main

import "encoding/json"
import "fmt"
import "net/http"
import log "github.com/Sirupsen/logrus"

type ElasticsearchShipper struct{}

func (hook *ElasticsearchShipper) Ship(logs []log.Fields) error {
	log.Debug(fmt.Sprintf("Shipping %d logs", len(logs)))

	index := Getenv("ELASTICSEARCH_INDEX", "logstash-data")
	metric_type := Getenv("METRIC_TYPE", "metricsd")

	action := ActionMap{
		"index": IndexMap{
			"_index": index,
			"_type":  metric_type,
		},
	}
	serializedAction, err := json.Marshal(action)
	if err != nil {
		fmt.Errorf("Failed to marshal action to JSON, %v", err)
		return nil
	}

	var slice []byte
	newline := []byte("\n")

	for _, item := range logs {
		serialized, err := json.Marshal(item)
		if err != nil {
			fmt.Errorf("Failed to marshal fields to JSON, %v", err)
			return nil
		}
		slice = Extend(slice, serializedAction)
		slice = Extend(slice, newline)
		slice = Extend(slice, serialized)
		slice = Extend(slice, newline)
	}

	status, err := ElasticsearchPost("/_bulk", slice)
	if err != nil {
		log.Warning("Indexing serialized data failed with err: ", err)
	}

	if status != http.StatusOK {
		log.Warning("Indexing serialized data failed with status: ", status)
	}
	return nil
}

func ElasticsearchPost(url string, data []byte) (int, error) {
	elasticsearchUrl := Getenv("ELASTICSEARCH_URL", "http://127.0.0.1:9200")
	req, err := http.NewRequest("POST", fmt.Sprintf("%s%s", elasticsearchUrl, url), bytes.NewBuffer(data))
	req.Header.Set("Content-Type", "application/json")

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		fmt.Errorf("Failed to make request, %v", err)
		return 0, err
	}
	defer resp.Body.Close()

	return resp.StatusCode, nil
}

func SetupTemplate() {
	template := `
{
	"order": 0,
	"template": "logstash-*",
	"settings": {
		"index.refresh_interval": "5s"
	},
	"mappings": {
		"_default_": {
			"dynamic_templates": [
				{
					"string_fields": {
						"mapping": {
							"index": "analyzed",
							"omit_norms": true,
							"type": "string",
							"fields": {
								"raw": {
									"index": "not_analyzed",
									"ignore_above": 256,
									"type": "string"
								}
							}
						},
						"match_mapping_type": "string",
						"match": "*"
					}
				}
			],
			"properties": {
				"@version": {
					"index": "not_analyzed",
					"type": "string"
				}
			},
			"_all": {
				"enabled": true
			}
		}
	},
	"aliases": {}
}
`
	var data = []byte(template)

	status, err := ElasticsearchPost("/_template/logstash", data)
	if err != nil {
		log.Error("Indexing serialized data failed: ", err)
	}

	if status != http.StatusOK {
		log.Error("Creating index failed")
	}
}
<|endoftext|>


<commit_before><commit_msg>Stop using io.CopyBuffer in ec-split<commit_after><|endoftext|>


<commit_before><commit_msg>Print long usage on usage error<commit_after><|endoftext|>


<commit_before>// Package bip39 is the official Golang implementation of the BIP39 spec.
//
// The official BIP39 spec can be found at
// https://github.com/bitcoin/bips/blob/master/bip-0039.mediawiki
package bip39

import (
	"crypto/rand"
	"crypto/sha256"
	"crypto/sha512"
	"encoding/binary"
	"errors"
	"fmt"
	"math/big"
	"strings"

	"github.com/tyler-smith/go-bip39/wordlists"
	"golang.org/x/crypto/pbkdf2"
)

var (
	// Some bitwise operands for working with big.Ints
	last11BitsMask          = big.NewInt(2047)
	rightShift11BitsDivider = big.NewInt(2048)
	bigOne                  = big.NewInt(1)
	bigTwo                  = big.NewInt(2)

	// used to isolate the checksum bits from the entropy+checksum byte array
	wordLengthChecksumMasksMapping = map[int]*big.Int{
		12: big.NewInt(15),
		15: big.NewInt(31),
		18: big.NewInt(63),
		21: big.NewInt(127),
		24: big.NewInt(255),
	}
	// used to use only the desired x of 8 available checksum bits.
	// 256 bit (word length 24) requires all 8 bits of the checksum,
	// and thus no shifting is needed for it (we would get a divByZero crash if we did)
	wordLengthChecksumShiftMapping = map[int]*big.Int{
		12: big.NewInt(16),
		15: big.NewInt(8),
		18: big.NewInt(4),
		21: big.NewInt(2),
	}

	// wordList is the set of words to use
	wordList []string

	// wordMap is a reverse lookup map for wordList
	wordMap map[string]int
)

var (
	// ErrInvalidMnemonic is returned when trying to use a malformed mnemonic.
	ErrInvalidMnemonic = errors.New("Invalid menomic")

	// ErrEntropyLengthInvalid is returned when trying to use an entropy set with
	// an invalid size.
	ErrEntropyLengthInvalid = errors.New("Entropy length must be [128, 256] and a multiple of 32")

	// ErrValidatedSeedLengthMismatch is returned when a validated seed is not the
	// same size as the given seed. This should never happen is present only as a
	// sanity assertion.
	ErrValidatedSeedLengthMismatch = errors.New("Seed length does not match validated seed length")

	// ErrChecksumIncorrect is returned when entropy has the incorrect checksum.
	ErrChecksumIncorrect = errors.New("Checksum incorrect")
)

func init() {
	SetWordList(wordlists.English)
}

// SetWordList sets the list of words to use for mnemonics. Currently the list
// that is set is used package-wide.
func SetWordList(list []string) {
	wordList = list
	wordMap = map[string]int{}
	for i, v := range wordList {
		wordMap[v] = i
	}
}

// GetWordList gets the list of words to use for mnemonics
func GetWordList() []string {
	return wordList
}

// NewEntropy will create random entropy bytes
// so long as the requested size bitSize is an appropriate size.
//
// bitSize has to be a multiple 32 and be within the inclusive range of {128, 256}
func NewEntropy(bitSize int) ([]byte, error) {
	err := validateEntropyBitSize(bitSize)
	if err != nil {
		return nil, err
	}

	entropy := make([]byte, bitSize/8)
	_, err = rand.Read(entropy)
	return entropy, err
}

// EntropyFromMnemonic takes a mnemonic generated by this library,
// and returns the input entropy used to generate the given mnemonic.
// An error is returned if the given mnemonic is invalid.
func EntropyFromMnemonic(mnemonic string) ([]byte, error) {
	mnemonicSlice, isValid := splitMnemonicWords(mnemonic)
	if !isValid {
		return nil, errors.New("Invalid mnemonic")
	}

	b := big.NewInt(0)
	for _, v := range mnemonicSlice {
		index, found := wordMap[v]
		if found == false {
			return nil, fmt.Errorf("word `%v` not found in reverse map", v)
		}
		var wordBytes [2]byte
		binary.BigEndian.PutUint16(wordBytes[:], uint16(index))
		b = b.Mul(b, rightShift11BitsDivider)
		b = b.Or(b, big.NewInt(0).SetBytes(wordBytes[:]))
	}

	checksum := big.NewInt(0)
	checksumMask := wordLengthChecksumMasksMapping[len(mnemonicSlice)]
	checksum = checksum.And(b, checksumMask)

	b.Div(b, big.NewInt(0).Add(checksumMask, bigOne))
	entropy := b.Bytes()
	// pad entropy if needed
	entropy = padByteSlice(entropy, len(mnemonicSlice)/3*4)

	// generate the checksum once again, mask and ensure it equals the checksum we got from the mneomnic
	entropyChecksumBytes := computeChecksum(entropy)
	entropyChecksum := big.NewInt(int64(entropyChecksumBytes[0]))
	if l := len(mnemonicSlice); l != 24 {
		checksumShift := wordLengthChecksumShiftMapping[l]
		entropyChecksum.Div(entropyChecksum, checksumShift)
	}

	if checksum.Cmp(entropyChecksum) != 0 {
		return nil, errors.New("mnemonic's entropy doesn't match its checksum")
	}

	// return (padded) entropy
	return entropy, nil
}

// NewMnemonic will return a string consisting of the mnemonic words for
// the given entropy.
// If the provide entropy is invalid, an error will be returned.
func NewMnemonic(entropy []byte) (string, error) {
	// Compute some lengths for convenience
	entropyBitLength := len(entropy) * 8
	checksumBitLength := entropyBitLength / 32
	sentenceLength := (entropyBitLength + checksumBitLength) / 11

	err := validateEntropyBitSize(entropyBitLength)
	if err != nil {
		return "", err
	}

	// Add checksum to entropy
	entropy = addChecksum(entropy)

	// Break entropy up into sentenceLength chunks of 11 bits
	// For each word AND mask the rightmost 11 bits and find the word at that index
	// Then bitshift entropy 11 bits right and repeat
	// Add to the last empty slot so we can work with LSBs instead of MSB

	// Entropy as an int so we can bitmask without worrying about bytes slices
	entropyInt := new(big.Int).SetBytes(entropy)

	// Slice to hold words in
	words := make([]string, sentenceLength)

	// Throw away big int for AND masking
	word := big.NewInt(0)

	for i := sentenceLength - 1; i >= 0; i-- {
		// Get 11 right most bits and bitshift 11 to the right for next time
		word.And(entropyInt, last11BitsMask)
		entropyInt.Div(entropyInt, rightShift11BitsDivider)

		// Get the bytes representing the 11 bits as a 2 byte slice
		wordBytes := padByteSlice(word.Bytes(), 2)

		// Convert bytes to an index and add that word to the list
		words[i] = wordList[binary.BigEndian.Uint16(wordBytes)]
	}

	return strings.Join(words, " "), nil
}

// MnemonicToByteArray takes a mnemonic string and turns it into a byte array
// suitable for creating another mnemonic.
// An error is returned if the mnemonic is invalid.
func MnemonicToByteArray(mnemonic string, raw ...bool) ([]byte, error) {
	var (
		mnemonicSlice    = strings.Split(mnemonic, " ")
		entropyBitSize   = len(mnemonicSlice) * 11
		checksumBitSize  = entropyBitSize % 32
		fullByteSize     = (entropyBitSize-checksumBitSize)/8 + 1
		checksumByteSize = fullByteSize - (fullByteSize % 4)
	)

	// Pre validate that the mnemonic is well formed and only contains words that
	// are present in the word list
	if !IsMnemonicValid(mnemonic) {
		return nil, ErrInvalidMnemonic
	}

	// Convert word indices to a `big.Int` representing the entropy
	checksummedEntropy := big.NewInt(0)
	modulo := big.NewInt(2048)
	for _, v := range mnemonicSlice {
		index := big.NewInt(int64(wordMap[v]))
		checksummedEntropy.Mul(checksummedEntropy, modulo)
		checksummedEntropy.Add(checksummedEntropy, index)
	}

	// Calculate the unchecksummed entropy so we can validate that the checksum is
	// correct
	checksumModulo := big.NewInt(0).Exp(bigTwo, big.NewInt(int64(checksumBitSize)), nil)
	rawEntropy := big.NewInt(0).Div(checksummedEntropy, checksumModulo)

	// Convert `big.Int`s to byte padded byte slices
	rawEntropyBytes := padByteSlice(rawEntropy.Bytes(), checksumByteSize)
	checksummedEntropyBytes := padByteSlice(checksummedEntropy.Bytes(), fullByteSize)

	// Validate that the checksum is correct
	newChecksummedEntropyBytes := padByteSlice(addChecksum(rawEntropyBytes), fullByteSize)
	if !compareByteSlices(checksummedEntropyBytes, newChecksummedEntropyBytes) {
		return nil, ErrChecksumIncorrect
	}

	if raw == nil {
		return checksummedEntropyBytes, nil
	}
	if raw[0] == true {
		return rawEntropyBytes, nil
	}
	return checksummedEntropyBytes, nil
}

// NewSeedWithErrorChecking creates a hashed seed output given the mnemonic string and a password.
// An error is returned if the mnemonic is not convertible to a byte array.
func NewSeedWithErrorChecking(mnemonic string, password string) ([]byte, error) {
	_, err := MnemonicToByteArray(mnemonic)
	if err != nil {
		return nil, err
	}
	return NewSeed(mnemonic, password), nil
}

// NewSeed creates a hashed seed output given a provided string and password.
// No checking is performed to validate that the string provided is a valid mnemonic.
func NewSeed(mnemonic string, password string) []byte {
	return pbkdf2.Key([]byte(mnemonic), []byte("mnemonic"+password), 2048, 64, sha512.New)
}

// IsMnemonicValid attempts to verify that the provided mnemonic is valid.
// Validity is determined by both the number of words being appropriate,
// and that all the words in the mnemonic are present in the word list.
func IsMnemonicValid(mnemonic string) bool {
	// Create a list of all the words in the mnemonic sentence
	words := strings.Fields(mnemonic)

	// Get word count
	wordCount := len(words)

	// The number of words should be 12, 15, 18, 21 or 24
	if wordCount%3 != 0 || wordCount < 12 || wordCount > 24 {
		return false
	}

	// Check if all words belong in the wordlist
	for _, word := range words {
		if _, ok := wordMap[word]; !ok {
			return false
		}
	}

	return true
}

// Appends to data the first (len(data) / 32)bits of the result of sha256(data)
// Currently only supports data up to 32 bytes
func addChecksum(data []byte) []byte {
	// Get first byte of sha256
	hash := computeChecksum(data)
	firstChecksumByte := hash[0]

	// len() is in bytes so we divide by 4
	checksumBitLength := uint(len(data) / 4)

	// For each bit of check sum we want we shift the data one the left
	// and then set the (new) right most bit equal to checksum bit at that index
	// staring from the left
	dataBigInt := new(big.Int).SetBytes(data)
	for i := uint(0); i < checksumBitLength; i++ {
		// Bitshift 1 left
		dataBigInt.Mul(dataBigInt, bigTwo)

		// Set rightmost bit if leftmost checksum bit is set
		if uint8(firstChecksumByte&(1<<(7-i))) > 0 {
			dataBigInt.Or(dataBigInt, bigOne)
		}
	}

	return dataBigInt.Bytes()
}

func computeChecksum(data []byte) []byte {
	hasher := sha256.New()
	hasher.Write(data)
	return hasher.Sum(nil)
}

// validateEntropyBitSize ensures that entropy is the correct size for being a
// mnemonic.
func validateEntropyBitSize(bitSize int) error {
	if (bitSize%32) != 0 || bitSize < 128 || bitSize > 256 {
		return ErrEntropyLengthInvalid
	}
	return nil
}

// padByteSlice returns a byte slice of the given size with contents of the
// given slice left padded and any empty spaces filled with 0's.
func padByteSlice(slice []byte, length int) []byte {
	offset := length - len(slice)
	if offset <= 0 {
		return slice
	}
	newSlice := make([]byte, length)
	copy(newSlice[offset:], slice)
	return newSlice
}

// compareByteSlices returns true of the byte slices have equal contents and
// returns false otherwise.
func compareByteSlices(a, b []byte) bool {
	if len(a) != len(b) {
		return false
	}
	for i := range a {
		if a[i] != b[i] {
			return false
		}
	}
	return true
}

func splitMnemonicWords(mnemonic string) ([]string, bool) {
	// Create a list of all the words in the mnemonic sentence
	words := strings.Fields(mnemonic)

	// Get num of words
	numOfWords := len(words)

	// The number of words should be 12, 15, 18, 21 or 24
	if numOfWords%3 != 0 || numOfWords < 12 || numOfWords > 24 {
		return nil, false
	}
	return words, true
}
<commit_msg>Update bip39.go<commit_after>// Package bip39 is the official Golang implementation of the BIP39 spec.
//
// The official BIP39 spec can be found at
// https://github.com/bitcoin/bips/blob/master/bip-0039.mediawiki
package bip39

import (
	"crypto/rand"
	"crypto/sha256"
	"crypto/sha512"
	"encoding/binary"
	"errors"
	"fmt"
	"math/big"
	"strings"

	"github.com/tyler-smith/go-bip39/wordlists"
	"golang.org/x/crypto/pbkdf2"
)

var (
	// Some bitwise operands for working with big.Ints
	last11BitsMask          = big.NewInt(2047)
	rightShift11BitsDivider = big.NewInt(2048)
	bigOne                  = big.NewInt(1)
	bigTwo                  = big.NewInt(2)

	// used to isolate the checksum bits from the entropy+checksum byte array
	wordLengthChecksumMasksMapping = map[int]*big.Int{
		12: big.NewInt(15),
		15: big.NewInt(31),
		18: big.NewInt(63),
		21: big.NewInt(127),
		24: big.NewInt(255),
	}
	// used to use only the desired x of 8 available checksum bits.
	// 256 bit (word length 24) requires all 8 bits of the checksum,
	// and thus no shifting is needed for it (we would get a divByZero crash if we did)
	wordLengthChecksumShiftMapping = map[int]*big.Int{
		12: big.NewInt(16),
		15: big.NewInt(8),
		18: big.NewInt(4),
		21: big.NewInt(2),
	}

	// wordList is the set of words to use
	wordList []string

	// wordMap is a reverse lookup map for wordList
	wordMap map[string]int
)

var (
	// ErrInvalidMnemonic is returned when trying to use a malformed mnemonic.
	ErrInvalidMnemonic = errors.New("Invalid menomic")

	// ErrEntropyLengthInvalid is returned when trying to use an entropy set with
	// an invalid size.
	ErrEntropyLengthInvalid = errors.New("Entropy length must be [128, 256] and a multiple of 32")

	// ErrValidatedSeedLengthMismatch is returned when a validated seed is not the
	// same size as the given seed. This should never happen is present only as a
	// sanity assertion.
	ErrValidatedSeedLengthMismatch = errors.New("Seed length does not match validated seed length")

	// ErrChecksumIncorrect is returned when entropy has the incorrect checksum.
	ErrChecksumIncorrect = errors.New("Checksum incorrect")
)

func init() {
	SetWordList(wordlists.English)
}

// SetWordList sets the list of words to use for mnemonics. Currently the list
// that is set is used package-wide.
func SetWordList(list []string) {
	wordList = list
	wordMap = map[string]int{}
	for i, v := range wordList {
		wordMap[v] = i
	}
}

// GetWordList gets the list of words to use for mnemonics
func GetWordList() []string {
	return wordList
}

// GetWordIndex get word index in wordMap
func GetWordIndex(word string) (int, bool) {
	idx, ok := wordMap[word]
	return idx, ok
}

// NewEntropy will create random entropy bytes
// so long as the requested size bitSize is an appropriate size.
//
// bitSize has to be a multiple 32 and be within the inclusive range of {128, 256}
func NewEntropy(bitSize int) ([]byte, error) {
	err := validateEntropyBitSize(bitSize)
	if err != nil {
		return nil, err
	}

	entropy := make([]byte, bitSize/8)
	_, err = rand.Read(entropy)
	return entropy, err
}

// EntropyFromMnemonic takes a mnemonic generated by this library,
// and returns the input entropy used to generate the given mnemonic.
// An error is returned if the given mnemonic is invalid.
func EntropyFromMnemonic(mnemonic string) ([]byte, error) {
	mnemonicSlice, isValid := splitMnemonicWords(mnemonic)
	if !isValid {
		return nil, errors.New("Invalid mnemonic")
	}

	b := big.NewInt(0)
	for _, v := range mnemonicSlice {
		index, found := wordMap[v]
		if found == false {
			return nil, fmt.Errorf("word `%v` not found in reverse map", v)
		}
		var wordBytes [2]byte
		binary.BigEndian.PutUint16(wordBytes[:], uint16(index))
		b = b.Mul(b, rightShift11BitsDivider)
		b = b.Or(b, big.NewInt(0).SetBytes(wordBytes[:]))
	}

	checksum := big.NewInt(0)
	checksumMask := wordLengthChecksumMasksMapping[len(mnemonicSlice)]
	checksum = checksum.And(b, checksumMask)

	b.Div(b, big.NewInt(0).Add(checksumMask, bigOne))
	entropy := b.Bytes()
	// pad entropy if needed
	entropy = padByteSlice(entropy, len(mnemonicSlice)/3*4)

	// generate the checksum once again, mask and ensure it equals the checksum we got from the mneomnic
	entropyChecksumBytes := computeChecksum(entropy)
	entropyChecksum := big.NewInt(int64(entropyChecksumBytes[0]))
	if l := len(mnemonicSlice); l != 24 {
		checksumShift := wordLengthChecksumShiftMapping[l]
		entropyChecksum.Div(entropyChecksum, checksumShift)
	}

	if checksum.Cmp(entropyChecksum) != 0 {
		return nil, errors.New("mnemonic's entropy doesn't match its checksum")
	}

	// return (padded) entropy
	return entropy, nil
}

// NewMnemonic will return a string consisting of the mnemonic words for
// the given entropy.
// If the provide entropy is invalid, an error will be returned.
func NewMnemonic(entropy []byte) (string, error) {
	// Compute some lengths for convenience
	entropyBitLength := len(entropy) * 8
	checksumBitLength := entropyBitLength / 32
	sentenceLength := (entropyBitLength + checksumBitLength) / 11

	err := validateEntropyBitSize(entropyBitLength)
	if err != nil {
		return "", err
	}

	// Add checksum to entropy
	entropy = addChecksum(entropy)

	// Break entropy up into sentenceLength chunks of 11 bits
	// For each word AND mask the rightmost 11 bits and find the word at that index
	// Then bitshift entropy 11 bits right and repeat
	// Add to the last empty slot so we can work with LSBs instead of MSB

	// Entropy as an int so we can bitmask without worrying about bytes slices
	entropyInt := new(big.Int).SetBytes(entropy)

	// Slice to hold words in
	words := make([]string, sentenceLength)

	// Throw away big int for AND masking
	word := big.NewInt(0)

	for i := sentenceLength - 1; i >= 0; i-- {
		// Get 11 right most bits and bitshift 11 to the right for next time
		word.And(entropyInt, last11BitsMask)
		entropyInt.Div(entropyInt, rightShift11BitsDivider)

		// Get the bytes representing the 11 bits as a 2 byte slice
		wordBytes := padByteSlice(word.Bytes(), 2)

		// Convert bytes to an index and add that word to the list
		words[i] = wordList[binary.BigEndian.Uint16(wordBytes)]
	}

	return strings.Join(words, " "), nil
}

// MnemonicToByteArray takes a mnemonic string and turns it into a byte array
// suitable for creating another mnemonic.
// An error is returned if the mnemonic is invalid.
func MnemonicToByteArray(mnemonic string, raw ...bool) ([]byte, error) {
	var (
		mnemonicSlice    = strings.Split(mnemonic, " ")
		entropyBitSize   = len(mnemonicSlice) * 11
		checksumBitSize  = entropyBitSize % 32
		fullByteSize     = (entropyBitSize-checksumBitSize)/8 + 1
		checksumByteSize = fullByteSize - (fullByteSize % 4)
	)

	// Pre validate that the mnemonic is well formed and only contains words that
	// are present in the word list
	if !IsMnemonicValid(mnemonic) {
		return nil, ErrInvalidMnemonic
	}

	// Convert word indices to a `big.Int` representing the entropy
	checksummedEntropy := big.NewInt(0)
	modulo := big.NewInt(2048)
	for _, v := range mnemonicSlice {
		index := big.NewInt(int64(wordMap[v]))
		checksummedEntropy.Mul(checksummedEntropy, modulo)
		checksummedEntropy.Add(checksummedEntropy, index)
	}

	// Calculate the unchecksummed entropy so we can validate that the checksum is
	// correct
	checksumModulo := big.NewInt(0).Exp(bigTwo, big.NewInt(int64(checksumBitSize)), nil)
	rawEntropy := big.NewInt(0).Div(checksummedEntropy, checksumModulo)

	// Convert `big.Int`s to byte padded byte slices
	rawEntropyBytes := padByteSlice(rawEntropy.Bytes(), checksumByteSize)
	checksummedEntropyBytes := padByteSlice(checksummedEntropy.Bytes(), fullByteSize)

	// Validate that the checksum is correct
	newChecksummedEntropyBytes := padByteSlice(addChecksum(rawEntropyBytes), fullByteSize)
	if !compareByteSlices(checksummedEntropyBytes, newChecksummedEntropyBytes) {
		return nil, ErrChecksumIncorrect
	}

	if raw == nil {
		return checksummedEntropyBytes, nil
	}
	if raw[0] == true {
		return rawEntropyBytes, nil
	}
	return checksummedEntropyBytes, nil
}

// NewSeedWithErrorChecking creates a hashed seed output given the mnemonic string and a password.
// An error is returned if the mnemonic is not convertible to a byte array.
func NewSeedWithErrorChecking(mnemonic string, password string) ([]byte, error) {
	_, err := MnemonicToByteArray(mnemonic)
	if err != nil {
		return nil, err
	}
	return NewSeed(mnemonic, password), nil
}

// NewSeed creates a hashed seed output given a provided string and password.
// No checking is performed to validate that the string provided is a valid mnemonic.
func NewSeed(mnemonic string, password string) []byte {
	return pbkdf2.Key([]byte(mnemonic), []byte("mnemonic"+password), 2048, 64, sha512.New)
}

// IsMnemonicValid attempts to verify that the provided mnemonic is valid.
// Validity is determined by both the number of words being appropriate,
// and that all the words in the mnemonic are present in the word list.
func IsMnemonicValid(mnemonic string) bool {
	// Create a list of all the words in the mnemonic sentence
	words := strings.Fields(mnemonic)

	// Get word count
	wordCount := len(words)

	// The number of words should be 12, 15, 18, 21 or 24
	if wordCount%3 != 0 || wordCount < 12 || wordCount > 24 {
		return false
	}

	// Check if all words belong in the wordlist
	for _, word := range words {
		if _, ok := wordMap[word]; !ok {
			return false
		}
	}

	return true
}

// Appends to data the first (len(data) / 32)bits of the result of sha256(data)
// Currently only supports data up to 32 bytes
func addChecksum(data []byte) []byte {
	// Get first byte of sha256
	hash := computeChecksum(data)
	firstChecksumByte := hash[0]

	// len() is in bytes so we divide by 4
	checksumBitLength := uint(len(data) / 4)

	// For each bit of check sum we want we shift the data one the left
	// and then set the (new) right most bit equal to checksum bit at that index
	// staring from the left
	dataBigInt := new(big.Int).SetBytes(data)
	for i := uint(0); i < checksumBitLength; i++ {
		// Bitshift 1 left
		dataBigInt.Mul(dataBigInt, bigTwo)

		// Set rightmost bit if leftmost checksum bit is set
		if uint8(firstChecksumByte&(1<<(7-i))) > 0 {
			dataBigInt.Or(dataBigInt, bigOne)
		}
	}

	return dataBigInt.Bytes()
}

func computeChecksum(data []byte) []byte {
	hasher := sha256.New()
	hasher.Write(data)
	return hasher.Sum(nil)
}

// validateEntropyBitSize ensures that entropy is the correct size for being a
// mnemonic.
func validateEntropyBitSize(bitSize int) error {
	if (bitSize%32) != 0 || bitSize < 128 || bitSize > 256 {
		return ErrEntropyLengthInvalid
	}
	return nil
}

// padByteSlice returns a byte slice of the given size with contents of the
// given slice left padded and any empty spaces filled with 0's.
func padByteSlice(slice []byte, length int) []byte {
	offset := length - len(slice)
	if offset <= 0 {
		return slice
	}
	newSlice := make([]byte, length)
	copy(newSlice[offset:], slice)
	return newSlice
}

// compareByteSlices returns true of the byte slices have equal contents and
// returns false otherwise.
func compareByteSlices(a, b []byte) bool {
	if len(a) != len(b) {
		return false
	}
	for i := range a {
		if a[i] != b[i] {
			return false
		}
	}
	return true
}

func splitMnemonicWords(mnemonic string) ([]string, bool) {
	// Create a list of all the words in the mnemonic sentence
	words := strings.Fields(mnemonic)

	// Get num of words
	numOfWords := len(words)

	// The number of words should be 12, 15, 18, 21 or 24
	if numOfWords%3 != 0 || numOfWords < 12 || numOfWords > 24 {
		return nil, false
	}
	return words, true
}
<|endoftext|>