<commit_before>/*
Copyright 2018 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kaniko

import (
	"context"
	"fmt"
	"io"

	cstorage "cloud.google.com/go/storage"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/constants"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/v1alpha2"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

func RunKanikoBuild(ctx context.Context, out io.Writer, artifact *v1alpha2.Artifact, cfg *v1alpha2.KanikoBuild) (string, error) {
	dockerfilePath := artifact.DockerArtifact.DockerfilePath

	initialTag := util.RandomID()
	tarName := fmt.Sprintf("context-%s.tar.gz", initialTag)
	if err := docker.UploadContextToGCS(ctx, artifact.Workspace, dockerfilePath, cfg.GCSBucket, tarName); err != nil {
		return "", errors.Wrap(err, "uploading tar to gcs")
	}
	defer gcsDelete(ctx, cfg.GCSBucket, tarName)

	client, err := kubernetes.GetClientset()
	if err != nil {
		return "", errors.Wrap(err, "")
	}

	imageList := kubernetes.NewImageList()
	imageList.Add(constants.DefaultKanikoImage)

	logger := kubernetes.NewLogAggregator(out, imageList, kubernetes.NewColorPicker([]*v1alpha2.Artifact{artifact}))
	if err := logger.Start(ctx); err != nil {
		return "", errors.Wrap(err, "starting log streamer")
	}

	imageDst := fmt.Sprintf("%s:%s", artifact.ImageName, initialTag)
	p, err := client.CoreV1().Pods(cfg.Namespace).Create(&v1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			GenerateName: "kaniko",
			Labels:       map[string]string{"skaffold-kaniko": "skaffold-kaniko"},
			Namespace:    cfg.Namespace,
		},
		Spec: v1.PodSpec{
			Containers: []v1.Container{
				{
					Name:            "kaniko",
					Image:           constants.DefaultKanikoImage,
					ImagePullPolicy: v1.PullIfNotPresent,
					Args: addBuildArgs([]string{
						fmt.Sprintf("--dockerfile=%s", dockerfilePath),
						fmt.Sprintf("--context=gs://%s/%s", cfg.GCSBucket, tarName),
						fmt.Sprintf("--destination=%s", imageDst),
						fmt.Sprintf("-v=%s", logrus.GetLevel().String()),
					}, artifact),
					VolumeMounts: []v1.VolumeMount{
						{
							Name:      constants.DefaultKanikoSecretName,
							MountPath: "/secret",
						},
					},
					Env: []v1.EnvVar{
						{
							Name:  "GOOGLE_APPLICATION_CREDENTIALS",
							Value: "/secret/kaniko-secret",
						},
					},
				},
			},
			Volumes: []v1.Volume{
				{
					Name: constants.DefaultKanikoSecretName,
					VolumeSource: v1.VolumeSource{
						Secret: &v1.SecretVolumeSource{
							SecretName: cfg.PullSecretName,
						},
					},
				},
			},
			RestartPolicy: v1.RestartPolicyNever,
		},
	})
	if err != nil {
		return "", errors.Wrap(err, "creating kaniko pod")
	}

	defer func() {
		imageList.Remove(constants.DefaultKanikoImage)
		if err := client.CoreV1().Pods(cfg.Namespace).Delete(p.Name, &metav1.DeleteOptions{
			GracePeriodSeconds: new(int64),
		}); err != nil {
			logrus.Fatalf("deleting pod: %s", err)
		}
	}()

	if err := kubernetes.WaitForPodComplete(client.CoreV1().Pods(cfg.Namespace), p.Name); err != nil {
		return "", errors.Wrap(err, "waiting for pod to complete")
	}

	return imageDst, nil
}

func gcsDelete(ctx context.Context, bucket, path string) error {
	c, err := cstorage.NewClient(ctx)
	if err != nil {
		return err
	}
	defer c.Close()

	return c.Bucket(bucket).Object(path).Delete(ctx)
}

func addBuildArgs(args []string, artifact *v1alpha2.Artifact) []string {
	if artifact.DockerArtifact == nil {
		return args
	}

	if artifact.DockerArtifact.BuildArgs == nil || len(artifact.DockerArtifact.BuildArgs) == 0 {
		return args
	}

	for k, v := range artifact.DockerArtifact.BuildArgs {
		args = append(args, fmt.Sprintf("--build-arg=%s=%s", k, *v))
	}

	return args
}
<commit_msg>Fix nit (nil args have len == 0, so skip check for nil)<commit_after>/*
Copyright 2018 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kaniko

import (
	"context"
	"fmt"
	"io"

	cstorage "cloud.google.com/go/storage"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/constants"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/v1alpha2"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

func RunKanikoBuild(ctx context.Context, out io.Writer, artifact *v1alpha2.Artifact, cfg *v1alpha2.KanikoBuild) (string, error) {
	dockerfilePath := artifact.DockerArtifact.DockerfilePath

	initialTag := util.RandomID()
	tarName := fmt.Sprintf("context-%s.tar.gz", initialTag)
	if err := docker.UploadContextToGCS(ctx, artifact.Workspace, dockerfilePath, cfg.GCSBucket, tarName); err != nil {
		return "", errors.Wrap(err, "uploading tar to gcs")
	}
	defer gcsDelete(ctx, cfg.GCSBucket, tarName)

	client, err := kubernetes.GetClientset()
	if err != nil {
		return "", errors.Wrap(err, "")
	}

	imageList := kubernetes.NewImageList()
	imageList.Add(constants.DefaultKanikoImage)

	logger := kubernetes.NewLogAggregator(out, imageList, kubernetes.NewColorPicker([]*v1alpha2.Artifact{artifact}))
	if err := logger.Start(ctx); err != nil {
		return "", errors.Wrap(err, "starting log streamer")
	}

	imageDst := fmt.Sprintf("%s:%s", artifact.ImageName, initialTag)
	p, err := client.CoreV1().Pods(cfg.Namespace).Create(&v1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			GenerateName: "kaniko",
			Labels:       map[string]string{"skaffold-kaniko": "skaffold-kaniko"},
			Namespace:    cfg.Namespace,
		},
		Spec: v1.PodSpec{
			Containers: []v1.Container{
				{
					Name:            "kaniko",
					Image:           constants.DefaultKanikoImage,
					ImagePullPolicy: v1.PullIfNotPresent,
					Args: addBuildArgs([]string{
						fmt.Sprintf("--dockerfile=%s", dockerfilePath),
						fmt.Sprintf("--context=gs://%s/%s", cfg.GCSBucket, tarName),
						fmt.Sprintf("--destination=%s", imageDst),
						fmt.Sprintf("-v=%s", logrus.GetLevel().String()),
					}, artifact),
					VolumeMounts: []v1.VolumeMount{
						{
							Name:      constants.DefaultKanikoSecretName,
							MountPath: "/secret",
						},
					},
					Env: []v1.EnvVar{
						{
							Name:  "GOOGLE_APPLICATION_CREDENTIALS",
							Value: "/secret/kaniko-secret",
						},
					},
				},
			},
			Volumes: []v1.Volume{
				{
					Name: constants.DefaultKanikoSecretName,
					VolumeSource: v1.VolumeSource{
						Secret: &v1.SecretVolumeSource{
							SecretName: cfg.PullSecretName,
						},
					},
				},
			},
			RestartPolicy: v1.RestartPolicyNever,
		},
	})
	if err != nil {
		return "", errors.Wrap(err, "creating kaniko pod")
	}

	defer func() {
		imageList.Remove(constants.DefaultKanikoImage)
		if err := client.CoreV1().Pods(cfg.Namespace).Delete(p.Name, &metav1.DeleteOptions{
			GracePeriodSeconds: new(int64),
		}); err != nil {
			logrus.Fatalf("deleting pod: %s", err)
		}
	}()

	if err := kubernetes.WaitForPodComplete(client.CoreV1().Pods(cfg.Namespace), p.Name); err != nil {
		return "", errors.Wrap(err, "waiting for pod to complete")
	}

	return imageDst, nil
}

func gcsDelete(ctx context.Context, bucket, path string) error {
	c, err := cstorage.NewClient(ctx)
	if err != nil {
		return err
	}
	defer c.Close()

	return c.Bucket(bucket).Object(path).Delete(ctx)
}

func addBuildArgs(args []string, artifact *v1alpha2.Artifact) []string {
	if artifact.DockerArtifact == nil {
		return args
	}

	if len(artifact.DockerArtifact.BuildArgs) == 0 {
		return args
	}

	for k, v := range artifact.DockerArtifact.BuildArgs {
		args = append(args, fmt.Sprintf("--build-arg=%s=%s", k, *v))
	}

	return args
}
<|endoftext|>


<commit_before>package collector

import (
	"context"
	"errors"
	"fmt"
	"io"
	"log"
	"net/http"
	"strings"
	"sync"
	"time"

	grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
	"github.com/karimra/gnmic/outputs"
	"github.com/openconfig/gnmi/proto/gnmi"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"google.golang.org/grpc"
)

const (
	defaultTargetReceivebuffer = 1000
)

// Config is the collector config
type Config struct {
	PrometheusAddress   string
	Debug               bool
	Format              string
	TargetReceiveBuffer uint
	RetryTimer          time.Duration
}

// Collector //
type Collector struct {
	Config        *Config
	Subscriptions map[string]*SubscriptionConfig
	Outputs       map[string][]outputs.Output
	DialOpts      []grpc.DialOption
	//
	m          *sync.Mutex
	Targets    map[string]*Target
	Logger     *log.Logger
	httpServer *http.Server
}

// NewCollector //
func NewCollector(
	config *Config,
	targetConfigs map[string]*TargetConfig,
	subscriptions map[string]*SubscriptionConfig,
	outputs map[string][]outputs.Output,
	dialOpts []grpc.DialOption,
	logger *log.Logger,
) *Collector {
	var httpServer *http.Server
	if config.PrometheusAddress != "" {
		grpcMetrics := grpc_prometheus.NewClientMetrics()
		reg := prometheus.NewRegistry()
		reg.MustRegister(prometheus.NewGoCollector())
		reg.MustRegister(prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{}))
		grpcMetrics.EnableClientHandlingTimeHistogram()
		reg.MustRegister(grpcMetrics)
		handler := http.NewServeMux()
		handler.Handle("/metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{}))
		httpServer = &http.Server{
			Handler: handler,
			Addr:    config.PrometheusAddress,
		}
		dialOpts = append(dialOpts, grpc.WithStreamInterceptor(grpcMetrics.StreamClientInterceptor()))
		if config.TargetReceiveBuffer == 0 {
			config.TargetReceiveBuffer = defaultTargetReceivebuffer
		}
		if config.RetryTimer == 0 {
			config.RetryTimer = defaultRetryTimer
		}
	}
	c := &Collector{
		Config:        config,
		Subscriptions: subscriptions,
		Outputs:       outputs,
		DialOpts:      dialOpts,
		m:             new(sync.Mutex),
		Targets:       make(map[string]*Target),
		Logger:        logger,
		httpServer:    httpServer,
	}

	for _, tc := range targetConfigs {
		c.InitTarget(tc)
	}
	return c
}

// InitTarget initializes a target based on *TargetConfig
func (c *Collector) InitTarget(tc *TargetConfig) {
	if tc.BufferSize == 0 {
		tc.BufferSize = c.Config.TargetReceiveBuffer
	}
	if tc.RetryTimer == 0 {
		tc.RetryTimer = c.Config.RetryTimer
	}
	t := NewTarget(tc)
	//
	t.Subscriptions = make([]*SubscriptionConfig, 0, len(tc.Subscriptions))
	for _, subName := range tc.Subscriptions {
		if sub, ok := c.Subscriptions[subName]; ok {
			t.Subscriptions = append(t.Subscriptions, sub)
		}
	}
	if len(t.Subscriptions) == 0 {
		t.Subscriptions = make([]*SubscriptionConfig, 0, len(c.Subscriptions))
		for _, sub := range c.Subscriptions {
			t.Subscriptions = append(t.Subscriptions, sub)
		}
	}
	//
	t.Outputs = make([]outputs.Output, 0, len(tc.Outputs))
	for _, outName := range tc.Outputs {
		if outs, ok := c.Outputs[outName]; ok {
			t.Outputs = append(t.Outputs, outs...)
		}
	}
	if len(t.Outputs) == 0 {
		t.Outputs = make([]outputs.Output, 0, len(c.Outputs))
		for _, o := range c.Outputs {
			t.Outputs = append(t.Outputs, o...)
		}
	}
	c.m.Lock()
	defer c.m.Unlock()
	c.Targets[t.Config.Name] = t
}

// Subscribe //
func (c *Collector) Subscribe(ctx context.Context, tName string) error {
	if t, ok := c.Targets[tName]; ok {
		if err := t.CreateGNMIClient(ctx, c.DialOpts...); err != nil {
			return err
		}
		c.Logger.Printf("target '%s' gNMI client created", t.Config.Name)
		for _, sc := range t.Subscriptions {
			req, err := sc.CreateSubscribeRequest()
			if err != nil {
				return err
			}
			c.Logger.Printf("sending gNMI SubscribeRequest: subscribe='%+v', mode='%+v', encoding='%+v', to %s",
				req, req.GetSubscribe().GetMode(), req.GetSubscribe().GetEncoding(), t.Config.Name)
			go t.Subscribe(ctx, req, sc.Name)
		}
		return nil
	}
	return fmt.Errorf("unknown target name: %s", tName)
}

// Start start the prometheus server as well as a goroutine per target selecting on the response chan, the error chan and the ctx.Done() chan
func (c *Collector) Start(ctx context.Context) {
	if c.httpServer != nil {
		go func() {
			if err := c.httpServer.ListenAndServe(); err != nil {
				c.Logger.Printf("Unable to start prometheus http server: %v", err)
				return
			}
		}()
	}
	defer func() {
		for _, outputs := range c.Outputs {
			for _, o := range outputs {
				o.Close()
			}
		}
	}()
	wg := new(sync.WaitGroup)
	wg.Add(len(c.Targets))
	for _, t := range c.Targets {
		go func(t *Target) {
			defer wg.Done()
			numOnceSubscriptions := t.numberOfOnceSubscriptions()
			remainingOnceSubscriptions := numOnceSubscriptions
			numSubscriptions := len(t.Subscriptions)
			for {
				select {
				case rsp := <-t.SubscribeResponses:
					if c.Config.Debug {
						c.Logger.Printf("received gNMI Subscribe Response: %+v", rsp)
					}
					if c.subscriptionMode(rsp.SubscriptionName) == "ONCE" {
						t.Export(ctx, rsp.Response, outputs.Meta{"source": t.Config.Name, "format": c.Config.Format, "subscription-name": rsp.SubscriptionName})
					} else {
						go t.Export(ctx, rsp.Response, outputs.Meta{"source": t.Config.Name, "format": c.Config.Format, "subscription-name": rsp.SubscriptionName})
					}
					if remainingOnceSubscriptions > 0 {
						if c.subscriptionMode(rsp.SubscriptionName) == "ONCE" {
							switch rsp.Response.Response.(type) {
							case *gnmi.SubscribeResponse_SyncResponse:
								remainingOnceSubscriptions--
							}
						}
					}
					if remainingOnceSubscriptions == 0 && numSubscriptions == numOnceSubscriptions {
						return
					}
				case tErr := <-t.Errors:
					if errors.Is(tErr.Err, io.EOF) {
						c.Logger.Printf("target '%s', subscription %s closed stream(EOF)", t.Config.Name, tErr.SubscriptionName)
					} else {
						c.Logger.Printf("target '%s', subscription %s rcv error: %v", t.Config.Name, tErr.SubscriptionName, tErr.Err)
					}
					if remainingOnceSubscriptions > 0 {
						if c.subscriptionMode(tErr.SubscriptionName) == "ONCE" {
							remainingOnceSubscriptions--
						}
					}
					if remainingOnceSubscriptions == 0 && numSubscriptions == numOnceSubscriptions {
						return
					}
				case <-ctx.Done():
					return
				}
			}
		}(t)
	}
	wg.Wait()
}

// TargetPoll sends a gnmi.SubscribeRequest_Poll to targetName and returns the response and an error,
// it uses the targetName and the subscriptionName strings to find the gnmi.GNMI_SubscribeClient
func (c *Collector) TargetPoll(targetName, subscriptionName string) (*gnmi.SubscribeResponse, error) {
	if sub, ok := c.Subscriptions[subscriptionName]; ok {
		if strings.ToUpper(sub.Mode) != "POLL" {
			return nil, fmt.Errorf("subscription '%s' is not a POLL subscription", subscriptionName)
		}
		if t, ok := c.Targets[targetName]; ok {
			if subClient, ok := t.SubscribeClients[subscriptionName]; ok {
				err := subClient.Send(&gnmi.SubscribeRequest{
					Request: &gnmi.SubscribeRequest_Poll{
						Poll: &gnmi.Poll{},
					},
				})
				if err != nil {
					return nil, err
				}
				return subClient.Recv()
			}
		}
		return nil, fmt.Errorf("unknown target name '%s'", targetName)
	}
	return nil, fmt.Errorf("unknown subscription name '%s'", subscriptionName)
}

// PolledSubscriptionsTargets returns a map of target name to a list of subscription names that have Mode == POLL
func (c *Collector) PolledSubscriptionsTargets() map[string][]string {
	result := make(map[string][]string)
	for tn, target := range c.Targets {
		for _, sub := range target.Subscriptions {
			if strings.ToUpper(sub.Mode) == "POLL" {
				if result[tn] == nil {
					result[tn] = make([]string, 0)
				}
				result[tn] = append(result[tn], sub.Name)
			}
		}
	}
	return result
}

func (c *Collector) subscriptionMode(name string) string {
	if sub, ok := c.Subscriptions[name]; ok {
		return strings.ToUpper(sub.Mode)
	}
	return ""
}
<commit_msg>add a log message for starting prometheus server<commit_after>package collector

import (
	"context"
	"errors"
	"fmt"
	"io"
	"log"
	"net/http"
	"strings"
	"sync"
	"time"

	grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
	"github.com/karimra/gnmic/outputs"
	"github.com/openconfig/gnmi/proto/gnmi"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"google.golang.org/grpc"
)

const (
	defaultTargetReceivebuffer = 1000
)

// Config is the collector config
type Config struct {
	PrometheusAddress   string
	Debug               bool
	Format              string
	TargetReceiveBuffer uint
	RetryTimer          time.Duration
}

// Collector //
type Collector struct {
	Config        *Config
	Subscriptions map[string]*SubscriptionConfig
	Outputs       map[string][]outputs.Output
	DialOpts      []grpc.DialOption
	//
	m          *sync.Mutex
	Targets    map[string]*Target
	Logger     *log.Logger
	httpServer *http.Server
}

// NewCollector //
func NewCollector(
	config *Config,
	targetConfigs map[string]*TargetConfig,
	subscriptions map[string]*SubscriptionConfig,
	outputs map[string][]outputs.Output,
	dialOpts []grpc.DialOption,
	logger *log.Logger,
) *Collector {
	var httpServer *http.Server
	if config.PrometheusAddress != "" {
		grpcMetrics := grpc_prometheus.NewClientMetrics()
		reg := prometheus.NewRegistry()
		reg.MustRegister(prometheus.NewGoCollector())
		reg.MustRegister(prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{}))
		grpcMetrics.EnableClientHandlingTimeHistogram()
		reg.MustRegister(grpcMetrics)
		handler := http.NewServeMux()
		handler.Handle("/metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{}))
		httpServer = &http.Server{
			Handler: handler,
			Addr:    config.PrometheusAddress,
		}
		dialOpts = append(dialOpts, grpc.WithStreamInterceptor(grpcMetrics.StreamClientInterceptor()))
		if config.TargetReceiveBuffer == 0 {
			config.TargetReceiveBuffer = defaultTargetReceivebuffer
		}
		if config.RetryTimer == 0 {
			config.RetryTimer = defaultRetryTimer
		}
	}
	c := &Collector{
		Config:        config,
		Subscriptions: subscriptions,
		Outputs:       outputs,
		DialOpts:      dialOpts,
		m:             new(sync.Mutex),
		Targets:       make(map[string]*Target),
		Logger:        logger,
		httpServer:    httpServer,
	}

	for _, tc := range targetConfigs {
		c.InitTarget(tc)
	}
	return c
}

// InitTarget initializes a target based on *TargetConfig
func (c *Collector) InitTarget(tc *TargetConfig) {
	if tc.BufferSize == 0 {
		tc.BufferSize = c.Config.TargetReceiveBuffer
	}
	if tc.RetryTimer == 0 {
		tc.RetryTimer = c.Config.RetryTimer
	}
	t := NewTarget(tc)
	//
	t.Subscriptions = make([]*SubscriptionConfig, 0, len(tc.Subscriptions))
	for _, subName := range tc.Subscriptions {
		if sub, ok := c.Subscriptions[subName]; ok {
			t.Subscriptions = append(t.Subscriptions, sub)
		}
	}
	if len(t.Subscriptions) == 0 {
		t.Subscriptions = make([]*SubscriptionConfig, 0, len(c.Subscriptions))
		for _, sub := range c.Subscriptions {
			t.Subscriptions = append(t.Subscriptions, sub)
		}
	}
	//
	t.Outputs = make([]outputs.Output, 0, len(tc.Outputs))
	for _, outName := range tc.Outputs {
		if outs, ok := c.Outputs[outName]; ok {
			t.Outputs = append(t.Outputs, outs...)
		}
	}
	if len(t.Outputs) == 0 {
		t.Outputs = make([]outputs.Output, 0, len(c.Outputs))
		for _, o := range c.Outputs {
			t.Outputs = append(t.Outputs, o...)
		}
	}
	c.m.Lock()
	defer c.m.Unlock()
	c.Targets[t.Config.Name] = t
}

// Subscribe //
func (c *Collector) Subscribe(ctx context.Context, tName string) error {
	if t, ok := c.Targets[tName]; ok {
		if err := t.CreateGNMIClient(ctx, c.DialOpts...); err != nil {
			return err
		}
		c.Logger.Printf("target '%s' gNMI client created", t.Config.Name)
		for _, sc := range t.Subscriptions {
			req, err := sc.CreateSubscribeRequest()
			if err != nil {
				return err
			}
			c.Logger.Printf("sending gNMI SubscribeRequest: subscribe='%+v', mode='%+v', encoding='%+v', to %s",
				req, req.GetSubscribe().GetMode(), req.GetSubscribe().GetEncoding(), t.Config.Name)
			go t.Subscribe(ctx, req, sc.Name)
		}
		return nil
	}
	return fmt.Errorf("unknown target name: %s", tName)
}

// Start start the prometheus server as well as a goroutine per target selecting on the response chan, the error chan and the ctx.Done() chan
func (c *Collector) Start(ctx context.Context) {
	if c.httpServer != nil {
		go func() {
			c.Logger.Printf("starting prometheus server on %s", c.httpServer.Addr)
			err := c.httpServer.ListenAndServe()
			if err != nil {
				c.Logger.Printf("Unable to start prometheus http server: %v", err)
				return
			}
		}()
	}
	defer func() {
		for _, outputs := range c.Outputs {
			for _, o := range outputs {
				o.Close()
			}
		}
	}()
	wg := new(sync.WaitGroup)
	wg.Add(len(c.Targets))
	for _, t := range c.Targets {
		go func(t *Target) {
			defer wg.Done()
			numOnceSubscriptions := t.numberOfOnceSubscriptions()
			remainingOnceSubscriptions := numOnceSubscriptions
			numSubscriptions := len(t.Subscriptions)
			for {
				select {
				case rsp := <-t.SubscribeResponses:
					if c.Config.Debug {
						c.Logger.Printf("received gNMI Subscribe Response: %+v", rsp)
					}
					if c.subscriptionMode(rsp.SubscriptionName) == "ONCE" {
						t.Export(ctx, rsp.Response, outputs.Meta{"source": t.Config.Name, "format": c.Config.Format, "subscription-name": rsp.SubscriptionName})
					} else {
						go t.Export(ctx, rsp.Response, outputs.Meta{"source": t.Config.Name, "format": c.Config.Format, "subscription-name": rsp.SubscriptionName})
					}
					if remainingOnceSubscriptions > 0 {
						if c.subscriptionMode(rsp.SubscriptionName) == "ONCE" {
							switch rsp.Response.Response.(type) {
							case *gnmi.SubscribeResponse_SyncResponse:
								remainingOnceSubscriptions--
							}
						}
					}
					if remainingOnceSubscriptions == 0 && numSubscriptions == numOnceSubscriptions {
						return
					}
				case tErr := <-t.Errors:
					if errors.Is(tErr.Err, io.EOF) {
						c.Logger.Printf("target '%s', subscription %s closed stream(EOF)", t.Config.Name, tErr.SubscriptionName)
					} else {
						c.Logger.Printf("target '%s', subscription %s rcv error: %v", t.Config.Name, tErr.SubscriptionName, tErr.Err)
					}
					if remainingOnceSubscriptions > 0 {
						if c.subscriptionMode(tErr.SubscriptionName) == "ONCE" {
							remainingOnceSubscriptions--
						}
					}
					if remainingOnceSubscriptions == 0 && numSubscriptions == numOnceSubscriptions {
						return
					}
				case <-ctx.Done():
					return
				}
			}
		}(t)
	}
	wg.Wait()
}

// TargetPoll sends a gnmi.SubscribeRequest_Poll to targetName and returns the response and an error,
// it uses the targetName and the subscriptionName strings to find the gnmi.GNMI_SubscribeClient
func (c *Collector) TargetPoll(targetName, subscriptionName string) (*gnmi.SubscribeResponse, error) {
	if sub, ok := c.Subscriptions[subscriptionName]; ok {
		if strings.ToUpper(sub.Mode) != "POLL" {
			return nil, fmt.Errorf("subscription '%s' is not a POLL subscription", subscriptionName)
		}
		if t, ok := c.Targets[targetName]; ok {
			if subClient, ok := t.SubscribeClients[subscriptionName]; ok {
				err := subClient.Send(&gnmi.SubscribeRequest{
					Request: &gnmi.SubscribeRequest_Poll{
						Poll: &gnmi.Poll{},
					},
				})
				if err != nil {
					return nil, err
				}
				return subClient.Recv()
			}
		}
		return nil, fmt.Errorf("unknown target name '%s'", targetName)
	}
	return nil, fmt.Errorf("unknown subscription name '%s'", subscriptionName)
}

// PolledSubscriptionsTargets returns a map of target name to a list of subscription names that have Mode == POLL
func (c *Collector) PolledSubscriptionsTargets() map[string][]string {
	result := make(map[string][]string)
	for tn, target := range c.Targets {
		for _, sub := range target.Subscriptions {
			if strings.ToUpper(sub.Mode) == "POLL" {
				if result[tn] == nil {
					result[tn] = make([]string, 0)
				}
				result[tn] = append(result[tn], sub.Name)
			}
		}
	}
	return result
}

func (c *Collector) subscriptionMode(name string) string {
	if sub, ok := c.Subscriptions[name]; ok {
		return strings.ToUpper(sub.Mode)
	}
	return ""
}
<|endoftext|>


<commit_before>package exec

import (
	"archive/tar"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"path"
	"time"

	"github.com/cozy/cozy-stack/pkg/apps"
	"github.com/cozy/cozy-stack/pkg/config"
	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/instance"
	"github.com/cozy/cozy-stack/pkg/jobs"
	"github.com/cozy/cozy-stack/pkg/realtime"
	"github.com/cozy/cozy-stack/pkg/vfs"
	"github.com/sirupsen/logrus"

	"github.com/spf13/afero"
)

type konnectorMsg struct {
	Type    string `json:"type"`
	Message string `json:"message"`
}

type konnectorWorker struct {
	slug     string
	msg      map[string]interface{}
	man      *apps.KonnManifest
	messages []konnectorMsg
}

// konnectorResult stores the result of a konnector execution.
type konnectorResult struct {
	DocID       string    `json:"_id,omitempty"`
	DocRev      string    `json:"_rev,omitempty"`
	CreatedAt   time.Time `json:"last_execution"`
	LastSuccess time.Time `json:"last_success"`
	Account     string    `json:"account"`
	State       string    `json:"state"`
	Error       string    `json:"error"`
}

func (r *konnectorResult) ID() string         { return r.DocID }
func (r *konnectorResult) Rev() string        { return r.DocRev }
func (r *konnectorResult) DocType() string    { return consts.KonnectorResults }
func (r *konnectorResult) Clone() couchdb.Doc { c := *r; return &c }
func (r *konnectorResult) SetID(id string)    { r.DocID = id }
func (r *konnectorResult) SetRev(rev string)  { r.DocRev = rev }

const (
	konnectorMsgTypeDebug    = "debug"
	konnectorMsgTypeWarning  = "warning"
	konnectorMsgTypeError    = "error"
	konnectorMsgTypeCritical = "critical"
)

// const konnectorMsgTypeProgress string = "progress"

func (w *konnectorWorker) PrepareWorkDir(i *instance.Instance, m jobs.Message) (workDir string, err error) {
	var msg map[string]interface{}
	if err = m.Unmarshal(&msg); err != nil {
		return
	}

	slug, _ := msg["konnector"].(string)
	man, err := apps.GetKonnectorBySlug(i, slug)
	if err != nil {
		return
	}

	// TODO: disallow konnectors on state Installed to be run when we define our
	// workflow to accept permissions changes on konnectors.
	if s := man.State(); s != apps.Ready && s != apps.Installed {
		err = errors.New("Konnector is not ready")
		return
	}

	w.slug = slug
	w.msg = msg
	w.man = man

	osFS := afero.NewOsFs()
	workDir, err = afero.TempDir(osFS, "", "konnector-"+slug)
	if err != nil {
		return
	}
	workFS := afero.NewBasePathFs(osFS, workDir)

	fileServer := i.KonnectorsFileServer()
	tarFile, err := fileServer.Open(slug, man.Version(), apps.KonnectorArchiveName)
	if err != nil {
		return
	}

	// Create the folder in which the konnector has the right to write.
	{
		folderToSave, _ := msg["folder_to_save"].(string)
		if folderToSave != "" {
			defaultFolderPath, _ := msg["default_folder_path"].(string)
			if defaultFolderPath == "" {
				defaultFolderPath = fmt.Sprintf("/???/%s", slug)
			}
			fs := i.VFS()
			if _, err = fs.DirByID(folderToSave); os.IsNotExist(err) {
				var dir *vfs.DirDoc
				dir, err = vfs.MkdirAll(fs, defaultFolderPath, nil)
				if err != nil {
					return
				}
				folderToSave = dir.ID()
			}
		}
		msg["folder_to_save"] = folderToSave
	}

	tr := tar.NewReader(tarFile)
	for {
		var hdr *tar.Header
		hdr, err = tr.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return
		}
		dirname := path.Dir(hdr.Name)
		if dirname != "." {
			if err = workFS.MkdirAll(dirname, 0755); err != nil {
				return
			}
		}
		var f afero.File
		f, err = workFS.OpenFile(hdr.Name, os.O_CREATE|os.O_WRONLY, 0640)
		if err != nil {
			return
		}
		_, err = io.Copy(f, tr)
		errc := f.Close()
		if err != nil {
			return
		}
		if errc != nil {
			err = errc
			return
		}
	}

	return workDir, nil
}

func (w *konnectorWorker) PrepareCmdEnv(i *instance.Instance, m jobs.Message) (cmd string, env []string, jobID string, err error) {
	jobID = fmt.Sprintf("konnector/%s/%s", w.slug, i.Domain)

	// Directly pass the job message as fields parameters
	fieldsJSON, err := json.Marshal(w.msg)
	if err != nil {
		return
	}

	paramsJSON, err := json.Marshal(w.man.Parameters)
	if err != nil {
		return
	}

	token := i.BuildKonnectorToken(w.man)

	cmd = config.GetConfig().Konnectors.Cmd
	env = []string{
		"COZY_URL=" + i.PageURL("/", nil),
		"COZY_CREDENTIALS=" + token,
		"COZY_FIELDS=" + string(fieldsJSON),
		"COZY_PARAMETERS=" + string(paramsJSON),
		"COZY_TYPE=" + w.man.Type,
		"COZY_LOCALE=" + i.Locale,
		"COZY_JOB_ID=" + jobID,
	}
	return
}

func (w *konnectorWorker) ScanOuput(i *instance.Instance, log *logrus.Entry, line []byte) error {
	var msg konnectorMsg
	if err := json.Unmarshal(line, &msg); err != nil {
		return fmt.Errorf("Could not parse stdout as JSON: %q", string(line))
	}

	switch msg.Type {
	case konnectorMsgTypeDebug:
		log.Debug(msg.Message)
	case konnectorMsgTypeWarning:
		log.Warn(msg.Message)
	case konnectorMsgTypeError, konnectorMsgTypeCritical:
		log.Error(msg.Message)
	}

	w.messages = append(w.messages, msg)
	realtime.GetHub().Publish(&realtime.Event{
		Verb: realtime.EventCreate,
		Doc: couchdb.JSONDoc{Type: consts.JobEvents, M: map[string]interface{}{
			"type":    msg.Type,
			"message": msg.Message,
		}},
		Domain: i.Domain,
	})
	return nil
}

func (w *konnectorWorker) Error(i *instance.Instance, err error) error {
	// For retro-compatibility, we still use "error" logs as returned error, only
	// in the case that no "critical" message are actually returned. In such
	// case, We use the last "error" log as the returned error.
	var lastErrorMessage error
	for _, msg := range w.messages {
		if msg.Type == konnectorMsgTypeCritical {
			return errors.New(msg.Message)
		}
		if msg.Type == konnectorMsgTypeError {
			lastErrorMessage = errors.New(msg.Message)
		}
	}
	if lastErrorMessage != nil {
		return lastErrorMessage
	}

	return err
}

func (w *konnectorWorker) Commit(ctx context.Context, msg jobs.Message, errjob error) error {
	if w.msg == nil {
		return nil
	}

	accountID, _ := w.msg["account"].(string)
	domain := ctx.Value(jobs.ContextDomainKey).(string)

	inst, err := instance.Get(domain)
	if err != nil {
		return err
	}

	lastResult := &konnectorResult{}
	err = couchdb.GetDoc(inst, consts.KonnectorResults, w.slug, lastResult)
	if err != nil {
		if !couchdb.IsNotFoundError(err) {
			return err
		}
		lastResult = nil
	}

	var state, errstr string
	var lastSuccess time.Time
	if errjob != nil {
		if lastResult != nil {
			lastSuccess = lastResult.LastSuccess
		}
		errstr = errjob.Error()
		state = jobs.Errored
	} else {
		lastSuccess = time.Now()
		state = jobs.Done
	}

	result := &konnectorResult{
		DocID:       w.slug,
		Account:     accountID,
		CreatedAt:   time.Now(),
		LastSuccess: lastSuccess,
		State:       state,
		Error:       errstr,
	}
	if lastResult == nil {
		err = couchdb.CreateNamedDocWithDB(inst, result)
	} else {
		result.SetRev(lastResult.Rev())
		err = couchdb.UpdateDoc(inst, result)
	}
	return err

	// if err != nil {
	// 	return err
	// }

	// // if it is the first try we do not take into account an error, we bail.
	// if lastResult == nil {
	// 	return nil
	// }
	// // if the job has not errored, or the last one was already errored, we bail.
	// if state != jobs.Errored || lastResult.State == jobs.Errored {
	// 	return nil
	// }

	// konnectorURL := inst.SubDomain(consts.CollectSlug)
	// konnectorURL.Fragment = "/category/all/" + slug
	// mail := mails.Options{
	// 	Mode:         mails.ModeNoReply,
	// 	Subject:      inst.Translate("Error Konnector execution", domain),
	// 	TemplateName: "konnector_error_" + inst.Locale,
	// 	TemplateValues: map[string]string{
	// 		"KonnectorName": slug,
	// 		"KonnectorPage": konnectorURL.String(),
	// 	},
	// }
	// msg, err := jobs.NewMessage(&mail)
	// if err != nil {
	// 	return err
	// }
	// log := logger.WithDomain(domain)
	// log.Info("Konnector has failed definitively, should send mail.", mail)
	// _, err = globals.GetBroker().PushJob(&jobs.JobRequest{
	// 	Domain:     domain,
	// 	WorkerType: "sendmail",
	// 	Message:    msg,
	// })
	// return err
}
<commit_msg>Create default folder if no "folder_to_save" specified<commit_after>package exec

import (
	"archive/tar"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"path"
	"time"

	"github.com/cozy/cozy-stack/pkg/apps"
	"github.com/cozy/cozy-stack/pkg/config"
	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/instance"
	"github.com/cozy/cozy-stack/pkg/jobs"
	"github.com/cozy/cozy-stack/pkg/realtime"
	"github.com/cozy/cozy-stack/pkg/vfs"
	"github.com/sirupsen/logrus"

	"github.com/spf13/afero"
)

type konnectorMsg struct {
	Type    string `json:"type"`
	Message string `json:"message"`
}

type konnectorWorker struct {
	slug     string
	msg      map[string]interface{}
	man      *apps.KonnManifest
	messages []konnectorMsg
}

// konnectorResult stores the result of a konnector execution.
type konnectorResult struct {
	DocID       string    `json:"_id,omitempty"`
	DocRev      string    `json:"_rev,omitempty"`
	CreatedAt   time.Time `json:"last_execution"`
	LastSuccess time.Time `json:"last_success"`
	Account     string    `json:"account"`
	State       string    `json:"state"`
	Error       string    `json:"error"`
}

func (r *konnectorResult) ID() string         { return r.DocID }
func (r *konnectorResult) Rev() string        { return r.DocRev }
func (r *konnectorResult) DocType() string    { return consts.KonnectorResults }
func (r *konnectorResult) Clone() couchdb.Doc { c := *r; return &c }
func (r *konnectorResult) SetID(id string)    { r.DocID = id }
func (r *konnectorResult) SetRev(rev string)  { r.DocRev = rev }

const (
	konnectorMsgTypeDebug    = "debug"
	konnectorMsgTypeWarning  = "warning"
	konnectorMsgTypeError    = "error"
	konnectorMsgTypeCritical = "critical"
)

// const konnectorMsgTypeProgress string = "progress"

func (w *konnectorWorker) PrepareWorkDir(i *instance.Instance, m jobs.Message) (workDir string, err error) {
	var msg map[string]interface{}
	if err = m.Unmarshal(&msg); err != nil {
		return
	}

	slug, _ := msg["konnector"].(string)
	man, err := apps.GetKonnectorBySlug(i, slug)
	if err != nil {
		return
	}

	// TODO: disallow konnectors on state Installed to be run when we define our
	// workflow to accept permissions changes on konnectors.
	if s := man.State(); s != apps.Ready && s != apps.Installed {
		err = errors.New("Konnector is not ready")
		return
	}

	w.slug = slug
	w.msg = msg
	w.man = man

	osFS := afero.NewOsFs()
	workDir, err = afero.TempDir(osFS, "", "konnector-"+slug)
	if err != nil {
		return
	}
	workFS := afero.NewBasePathFs(osFS, workDir)

	fileServer := i.KonnectorsFileServer()
	tarFile, err := fileServer.Open(slug, man.Version(), apps.KonnectorArchiveName)
	if err != nil {
		return
	}

	// Create the folder in which the konnector has the right to write.
	{
		fs := i.VFS()
		folderToSave, _ := msg["folder_to_save"].(string)
		defaultFolderPath, _ := msg["default_folder_path"].(string)
		if folderToSave != "" {
			if defaultFolderPath == "" {
				defaultFolderPath = fmt.Sprintf("/???/%s", slug)
			}
			if _, err = fs.DirByID(folderToSave); os.IsNotExist(err) {
				folderToSave = ""
			}
		}
		if folderToSave == "" {
			var dir *vfs.DirDoc
			dir, err = vfs.MkdirAll(fs, defaultFolderPath, nil)
			if err != nil {
				return
			}
			msg["folder_to_save"] = dir.ID()
		}
	}

	tr := tar.NewReader(tarFile)
	for {
		var hdr *tar.Header
		hdr, err = tr.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return
		}
		dirname := path.Dir(hdr.Name)
		if dirname != "." {
			if err = workFS.MkdirAll(dirname, 0755); err != nil {
				return
			}
		}
		var f afero.File
		f, err = workFS.OpenFile(hdr.Name, os.O_CREATE|os.O_WRONLY, 0640)
		if err != nil {
			return
		}
		_, err = io.Copy(f, tr)
		errc := f.Close()
		if err != nil {
			return
		}
		if errc != nil {
			err = errc
			return
		}
	}

	return workDir, nil
}

func (w *konnectorWorker) PrepareCmdEnv(i *instance.Instance, m jobs.Message) (cmd string, env []string, jobID string, err error) {
	jobID = fmt.Sprintf("konnector/%s/%s", w.slug, i.Domain)

	// Directly pass the job message as fields parameters
	fieldsJSON, err := json.Marshal(w.msg)
	if err != nil {
		return
	}

	paramsJSON, err := json.Marshal(w.man.Parameters)
	if err != nil {
		return
	}

	token := i.BuildKonnectorToken(w.man)

	cmd = config.GetConfig().Konnectors.Cmd
	env = []string{
		"COZY_URL=" + i.PageURL("/", nil),
		"COZY_CREDENTIALS=" + token,
		"COZY_FIELDS=" + string(fieldsJSON),
		"COZY_PARAMETERS=" + string(paramsJSON),
		"COZY_TYPE=" + w.man.Type,
		"COZY_LOCALE=" + i.Locale,
		"COZY_JOB_ID=" + jobID,
	}
	return
}

func (w *konnectorWorker) ScanOuput(i *instance.Instance, log *logrus.Entry, line []byte) error {
	var msg konnectorMsg
	if err := json.Unmarshal(line, &msg); err != nil {
		return fmt.Errorf("Could not parse stdout as JSON: %q", string(line))
	}

	switch msg.Type {
	case konnectorMsgTypeDebug:
		log.Debug(msg.Message)
	case konnectorMsgTypeWarning:
		log.Warn(msg.Message)
	case konnectorMsgTypeError, konnectorMsgTypeCritical:
		log.Error(msg.Message)
	}

	w.messages = append(w.messages, msg)
	realtime.GetHub().Publish(&realtime.Event{
		Verb: realtime.EventCreate,
		Doc: couchdb.JSONDoc{Type: consts.JobEvents, M: map[string]interface{}{
			"type":    msg.Type,
			"message": msg.Message,
		}},
		Domain: i.Domain,
	})
	return nil
}

func (w *konnectorWorker) Error(i *instance.Instance, err error) error {
	// For retro-compatibility, we still use "error" logs as returned error, only
	// in the case that no "critical" message are actually returned. In such
	// case, We use the last "error" log as the returned error.
	var lastErrorMessage error
	for _, msg := range w.messages {
		if msg.Type == konnectorMsgTypeCritical {
			return errors.New(msg.Message)
		}
		if msg.Type == konnectorMsgTypeError {
			lastErrorMessage = errors.New(msg.Message)
		}
	}
	if lastErrorMessage != nil {
		return lastErrorMessage
	}

	return err
}

func (w *konnectorWorker) Commit(ctx context.Context, msg jobs.Message, errjob error) error {
	if w.msg == nil {
		return nil
	}

	accountID, _ := w.msg["account"].(string)
	domain := ctx.Value(jobs.ContextDomainKey).(string)

	inst, err := instance.Get(domain)
	if err != nil {
		return err
	}

	lastResult := &konnectorResult{}
	err = couchdb.GetDoc(inst, consts.KonnectorResults, w.slug, lastResult)
	if err != nil {
		if !couchdb.IsNotFoundError(err) {
			return err
		}
		lastResult = nil
	}

	var state, errstr string
	var lastSuccess time.Time
	if errjob != nil {
		if lastResult != nil {
			lastSuccess = lastResult.LastSuccess
		}
		errstr = errjob.Error()
		state = jobs.Errored
	} else {
		lastSuccess = time.Now()
		state = jobs.Done
	}

	result := &konnectorResult{
		DocID:       w.slug,
		Account:     accountID,
		CreatedAt:   time.Now(),
		LastSuccess: lastSuccess,
		State:       state,
		Error:       errstr,
	}
	if lastResult == nil {
		err = couchdb.CreateNamedDocWithDB(inst, result)
	} else {
		result.SetRev(lastResult.Rev())
		err = couchdb.UpdateDoc(inst, result)
	}
	return err

	// if err != nil {
	// 	return err
	// }

	// // if it is the first try we do not take into account an error, we bail.
	// if lastResult == nil {
	// 	return nil
	// }
	// // if the job has not errored, or the last one was already errored, we bail.
	// if state != jobs.Errored || lastResult.State == jobs.Errored {
	// 	return nil
	// }

	// konnectorURL := inst.SubDomain(consts.CollectSlug)
	// konnectorURL.Fragment = "/category/all/" + slug
	// mail := mails.Options{
	// 	Mode:         mails.ModeNoReply,
	// 	Subject:      inst.Translate("Error Konnector execution", domain),
	// 	TemplateName: "konnector_error_" + inst.Locale,
	// 	TemplateValues: map[string]string{
	// 		"KonnectorName": slug,
	// 		"KonnectorPage": konnectorURL.String(),
	// 	},
	// }
	// msg, err := jobs.NewMessage(&mail)
	// if err != nil {
	// 	return err
	// }
	// log := logger.WithDomain(domain)
	// log.Info("Konnector has failed definitively, should send mail.", mail)
	// _, err = globals.GetBroker().PushJob(&jobs.JobRequest{
	// 	Domain:     domain,
	// 	WorkerType: "sendmail",
	// 	Message:    msg,
	// })
	// return err
}
<|endoftext|>


<commit_before>package vptree

import (
	"container/heap"
	"math"
	"math/rand"
)

type node struct {
	Item      interface{}
	Threshold float64
	Left      *node
	Right     *node
}

type heapItem struct {
	Item interface{}
	Dist float64
}

type Metric func(a, b interface{}) float64

type VPTree struct {
	root           *node
	tau            float64
	distanceMetric Metric
}

func New(metric Metric, items []interface{}) (t *VPTree) {
	t = &VPTree{
		distanceMetric: metric,
	}
	t.root = t.buildFromPoints(items)
	return
}

func (vp *VPTree) Search(target interface{}, k int) (results []interface{}, distances []float64) {
	h := make(priorityQueue, 0, k)

	vp.tau = math.MaxFloat64
	vp.search(vp.root, target, k, &h)

	for h.Len() > 0 {
		hi := heap.Pop(&h)
		results = append(results, hi.(*heapItem).Item)
		distances = append(distances, hi.(*heapItem).Dist)
	}

	// Reverse results and distances, because we popped them from the heap
	// in large-to-small order
	for i, j := 0, len(results)-1; i < j; i, j = i+1, j-1 {
		results[i], results[j] = results[j], results[i]
		distances[i], distances[j] = distances[j], distances[i]
	}

	return
}

func (vp *VPTree) buildFromPoints(items []interface{}) (n *node) {
	if len(items) == 0 {
		return nil
	}

	n = &node{}

	// Take a random item out of the items slice and make it this node's item
	idx := rand.Intn(len(items))
	n.Item = items[idx]
	items[idx], items = items[len(items)-1], items[:len(items)-1]

	if len(items) > 0 {
		// Now partition the items into two equal-sized sets, one
		// closer to the node's item than the median, and one farther
		// away.
		median := len(items) / 2
		pivotDist := vp.distanceMetric(items[median], n.Item)
		items[median], items[len(items)-1] = items[len(items)-1], items[median]

		storeIndex := 0
		for i := 0; i < len(items)-1; i++ {
			if vp.distanceMetric(items[i], n.Item) <= pivotDist {
				items[storeIndex], items[i] = items[i], items[storeIndex]
				storeIndex++
			}
		}
		items[len(items)-1], items[storeIndex] = items[storeIndex], items[len(items)-1]
		median = storeIndex

		n.Threshold = vp.distanceMetric(items[median], n.Item)
		n.Left = vp.buildFromPoints(items[:median])
		n.Right = vp.buildFromPoints(items[median:])
	}
	return
}

func (vp *VPTree) search(n *node, target interface{}, k int, h *priorityQueue) {
	if n == nil {
		return
	}

	dist := vp.distanceMetric(n.Item, target)

	if dist < vp.tau {
		if h.Len() == k {
			heap.Pop(h)
		}
		heap.Push(h, &heapItem{n.Item, dist})
		if h.Len() == k {
			vp.tau = h.Top().(*heapItem).Dist
		}
	}

	if n.Left == nil && n.Right == nil {
		return
	}

	if dist < n.Threshold {
		if dist-vp.tau <= n.Threshold {
			vp.search(n.Left, target, k, h)
		}

		if dist+vp.tau >= n.Threshold {
			vp.search(n.Right, target, k, h)
		}
	} else {
		if dist+vp.tau >= n.Threshold {
			vp.search(n.Right, target, k, h)
		}

		if dist-vp.tau <= n.Threshold {
			vp.search(n.Left, target, k, h)
		}
	}
}
<commit_msg>Add comments to the public types and functions<commit_after>package vptree

import (
	"container/heap"
	"math"
	"math/rand"
)

type node struct {
	Item      interface{}
	Threshold float64
	Left      *node
	Right     *node
}

type heapItem struct {
	Item interface{}
	Dist float64
}

// A Metric is a function that measures the distance between two provided
// interface{}-values. The function *must* be a metric in the mathematical
// sense, that is, the metric d must fullfill the following requirements:
//
// * d(x, y) >= 0
// * d(x, y) = 0 if and only if x = y
// * d(x, y) = d(y, x)
// * d(x, z) <= d(x, y) + d(y, z) (triangle inequality)
type Metric func(a, b interface{}) float64

// A VPTree struct represents a Vantage-point tree. Vantage-point trees are
// useful for nearest-neighbour searches in high-dimensional metric spaces.
type VPTree struct {
	root           *node
	tau            float64
	distanceMetric Metric
}

// New creates a new VP-tree using the metric and items provided. The metric
// measures the distance between two items, so that the VP-tree can find the
// nearest neighbour(s) of a target item.
func New(metric Metric, items []interface{}) (t *VPTree) {
	t = &VPTree{
		distanceMetric: metric,
	}
	t.root = t.buildFromPoints(items)
	return
}

// Search searches the VP-tree for the k nearest neighbours of target
func (vp *VPTree) Search(target interface{}, k int) (results []interface{}, distances []float64) {
	h := make(priorityQueue, 0, k)

	vp.tau = math.MaxFloat64
	vp.search(vp.root, target, k, &h)

	for h.Len() > 0 {
		hi := heap.Pop(&h)
		results = append(results, hi.(*heapItem).Item)
		distances = append(distances, hi.(*heapItem).Dist)
	}

	// Reverse results and distances, because we popped them from the heap
	// in large-to-small order
	for i, j := 0, len(results)-1; i < j; i, j = i+1, j-1 {
		results[i], results[j] = results[j], results[i]
		distances[i], distances[j] = distances[j], distances[i]
	}

	return
}

func (vp *VPTree) buildFromPoints(items []interface{}) (n *node) {
	if len(items) == 0 {
		return nil
	}

	n = &node{}

	// Take a random item out of the items slice and make it this node's item
	idx := rand.Intn(len(items))
	n.Item = items[idx]
	items[idx], items = items[len(items)-1], items[:len(items)-1]

	if len(items) > 0 {
		// Now partition the items into two equal-sized sets, one
		// closer to the node's item than the median, and one farther
		// away.
		median := len(items) / 2
		pivotDist := vp.distanceMetric(items[median], n.Item)
		items[median], items[len(items)-1] = items[len(items)-1], items[median]

		storeIndex := 0
		for i := 0; i < len(items)-1; i++ {
			if vp.distanceMetric(items[i], n.Item) <= pivotDist {
				items[storeIndex], items[i] = items[i], items[storeIndex]
				storeIndex++
			}
		}
		items[len(items)-1], items[storeIndex] = items[storeIndex], items[len(items)-1]
		median = storeIndex

		n.Threshold = vp.distanceMetric(items[median], n.Item)
		n.Left = vp.buildFromPoints(items[:median])
		n.Right = vp.buildFromPoints(items[median:])
	}
	return
}

func (vp *VPTree) search(n *node, target interface{}, k int, h *priorityQueue) {
	if n == nil {
		return
	}

	dist := vp.distanceMetric(n.Item, target)

	if dist < vp.tau {
		if h.Len() == k {
			heap.Pop(h)
		}
		heap.Push(h, &heapItem{n.Item, dist})
		if h.Len() == k {
			vp.tau = h.Top().(*heapItem).Dist
		}
	}

	if n.Left == nil && n.Right == nil {
		return
	}

	if dist < n.Threshold {
		if dist-vp.tau <= n.Threshold {
			vp.search(n.Left, target, k, h)
		}

		if dist+vp.tau >= n.Threshold {
			vp.search(n.Right, target, k, h)
		}
	} else {
		if dist+vp.tau >= n.Threshold {
			vp.search(n.Right, target, k, h)
		}

		if dist-vp.tau <= n.Threshold {
			vp.search(n.Left, target, k, h)
		}
	}
}
<|endoftext|>


<commit_before>package stats

import (
	"encoding/json"
	"math"
	"sync"
	"time"
)

// LeaderStats is used by the leader in an etcd cluster, and encapsulates
// statistics about communication with its followers
type LeaderStats struct {
	// TODO(jonboulle): clarify that these are IDs, not names
	Leader    string                    `json:"leader"`
	Followers map[string]*FollowerStats `json:"followers"`

	sync.Mutex
}

// NewLeaderStats generates a new LeaderStats with the given id as leader
func NewLeaderStats(id string) *LeaderStats {
	return &LeaderStats{
		Leader:    id,
		Followers: make(map[string]*FollowerStats),
	}
}

func (ls *LeaderStats) JSON() []byte {
	b, _ := json.Marshal(ls)
	return b
}

func (ls *LeaderStats) Follower(name string) *FollowerStats {
	ls.Lock()
	defer ls.Unlock()
	fs, ok := ls.Followers[name]
	if !ok {
		fs = &FollowerStats{}
		fs.Latency.Minimum = 1 << 63
		ls.Followers[name] = fs
	}
	return fs
}

// FollowerStats encapsulates various statistics about a follower in an etcd cluster
type FollowerStats struct {
	Latency struct {
		Current           float64 `json:"current"`
		Average           float64 `json:"average"`
		averageSquare     float64
		StandardDeviation float64 `json:"standardDeviation"`
		Minimum           float64 `json:"minimum"`
		Maximum           float64 `json:"maximum"`
	} `json:"latency"`

	Counts struct {
		Fail    uint64 `json:"fail"`
		Success uint64 `json:"success"`
	} `json:"counts"`

	sync.Mutex
}

// Succ updates the FollowerStats with a successful send
func (fs *FollowerStats) Succ(d time.Duration) {
	fs.Lock()
	defer fs.Unlock()

	total := float64(fs.Counts.Success) * fs.Latency.Average
	totalSquare := float64(fs.Counts.Success) * fs.Latency.averageSquare

	fs.Counts.Success++

	fs.Latency.Current = float64(d) / (1000000.0)

	if fs.Latency.Current > fs.Latency.Maximum {
		fs.Latency.Maximum = fs.Latency.Current
	}

	if fs.Latency.Current < fs.Latency.Minimum {
		fs.Latency.Minimum = fs.Latency.Current
	}

	fs.Latency.Average = (total + fs.Latency.Current) / float64(fs.Counts.Success)
	fs.Latency.averageSquare = (totalSquare + fs.Latency.Current*fs.Latency.Current) / float64(fs.Counts.Success)

	// sdv = sqrt(avg(x^2) - avg(x)^2)
	fs.Latency.StandardDeviation = math.Sqrt(fs.Latency.averageSquare - fs.Latency.Average*fs.Latency.Average)
}

// Fail updates the FollowerStats with an unsuccessful send
func (fs *FollowerStats) Fail() {
	fs.Lock()
	defer fs.Unlock()
	fs.Counts.Fail++
}
<commit_msg>etcdserver/stats: log any marshaling error<commit_after>package stats

import (
	"encoding/json"
	"log"
	"math"
	"sync"
	"time"
)

// LeaderStats is used by the leader in an etcd cluster, and encapsulates
// statistics about communication with its followers
type LeaderStats struct {
	// TODO(jonboulle): clarify that these are IDs, not names
	Leader    string                    `json:"leader"`
	Followers map[string]*FollowerStats `json:"followers"`

	sync.Mutex
}

// NewLeaderStats generates a new LeaderStats with the given id as leader
func NewLeaderStats(id string) *LeaderStats {
	return &LeaderStats{
		Leader:    id,
		Followers: make(map[string]*FollowerStats),
	}
}

func (ls *LeaderStats) JSON() []byte {
	b, err := json.Marshal(ls)
	// TODO(jonboulle): appropriate error handling?
	if err != nil {
		log.Printf("error marshalling leader stats: %v", err)
	}
	return b
}

func (ls *LeaderStats) Follower(name string) *FollowerStats {
	ls.Lock()
	defer ls.Unlock()
	fs, ok := ls.Followers[name]
	if !ok {
		fs = &FollowerStats{}
		fs.Latency.Minimum = 1 << 63
		ls.Followers[name] = fs
	}
	return fs
}

// FollowerStats encapsulates various statistics about a follower in an etcd cluster
type FollowerStats struct {
	Latency struct {
		Current           float64 `json:"current"`
		Average           float64 `json:"average"`
		averageSquare     float64
		StandardDeviation float64 `json:"standardDeviation"`
		Minimum           float64 `json:"minimum"`
		Maximum           float64 `json:"maximum"`
	} `json:"latency"`

	Counts struct {
		Fail    uint64 `json:"fail"`
		Success uint64 `json:"success"`
	} `json:"counts"`

	sync.Mutex
}

// Succ updates the FollowerStats with a successful send
func (fs *FollowerStats) Succ(d time.Duration) {
	fs.Lock()
	defer fs.Unlock()

	total := float64(fs.Counts.Success) * fs.Latency.Average
	totalSquare := float64(fs.Counts.Success) * fs.Latency.averageSquare

	fs.Counts.Success++

	fs.Latency.Current = float64(d) / (1000000.0)

	if fs.Latency.Current > fs.Latency.Maximum {
		fs.Latency.Maximum = fs.Latency.Current
	}

	if fs.Latency.Current < fs.Latency.Minimum {
		fs.Latency.Minimum = fs.Latency.Current
	}

	fs.Latency.Average = (total + fs.Latency.Current) / float64(fs.Counts.Success)
	fs.Latency.averageSquare = (totalSquare + fs.Latency.Current*fs.Latency.Current) / float64(fs.Counts.Success)

	// sdv = sqrt(avg(x^2) - avg(x)^2)
	fs.Latency.StandardDeviation = math.Sqrt(fs.Latency.averageSquare - fs.Latency.Average*fs.Latency.Average)
}

// Fail updates the FollowerStats with an unsuccessful send
func (fs *FollowerStats) Fail() {
	fs.Lock()
	defer fs.Unlock()
	fs.Counts.Fail++
}
<|endoftext|>


<commit_before>package bitcoincash

import (
	"errors"
	"github.com/OpenBazaar/wallet-interface"
	"github.com/btcsuite/btcd/btcec"
	"github.com/btcsuite/btcd/chaincfg"
	"github.com/btcsuite/btcd/chaincfg/chainhash"
	"github.com/btcsuite/btcd/peer"
	"github.com/btcsuite/btcd/txscript"
	btc "github.com/btcsuite/btcutil"
	hd "github.com/btcsuite/btcutil/hdkeychain"
	"github.com/btcsuite/btcwallet/wallet/txrules"
	"github.com/op/go-logging"
	b39 "github.com/tyler-smith/go-bip39"
	"io"
	"sync"
	"time"
	"github.com/cpacia/bchutil"
)

func setupNetworkParams() {
	chaincfg.MainNetParams.Net = bchutil.MainnetMagic
	chaincfg.TestNet3Params.Net = bchutil.TestnetMagic
	chaincfg.RegressionNetParams.Net = bchutil.Regtestmagic
}

type SPVWallet struct {
	params *chaincfg.Params

	masterPrivateKey *hd.ExtendedKey
	masterPublicKey  *hd.ExtendedKey

	mnemonic string

	feeProvider *FeeProvider

	repoPath string

	blockchain  *Blockchain
	txstore     *TxStore
	peerManager *PeerManager
	keyManager  *KeyManager

	fPositives    chan *peer.Peer
	stopChan      chan int
	fpAccumulator map[int32]int32
	mutex         *sync.RWMutex

	creationDate time.Time

	running bool

	config *PeerManagerConfig
}

var log = logging.MustGetLogger("bitcoin")

const WALLET_VERSION = "0.1.0"

func NewSPVWallet(config *Config) (*SPVWallet, error) {
	setupNetworkParams()

	log.SetBackend(logging.AddModuleLevel(config.Logger))

	if config.Mnemonic == "" {
		ent, err := b39.NewEntropy(128)
		if err != nil {
			return nil, err
		}
		mnemonic, err := b39.NewMnemonic(ent)
		if err != nil {
			return nil, err
		}
		config.Mnemonic = mnemonic
		config.CreationDate = time.Now()
	}
	seed := b39.NewSeed(config.Mnemonic, "")

	mPrivKey, err := hd.NewMaster(seed, config.Params)
	if err != nil {
		return nil, err
	}
	mPubKey, err := mPrivKey.Neuter()
	if err != nil {
		return nil, err
	}
	w := &SPVWallet{
		repoPath:         config.RepoPath,
		masterPrivateKey: mPrivKey,
		masterPublicKey:  mPubKey,
		mnemonic:         config.Mnemonic,
		params:           config.Params,
		creationDate:     config.CreationDate,
		feeProvider: NewFeeProvider(
			config.MaxFee,
			config.HighFee,
			config.MediumFee,
			config.LowFee,
			config.ExchangeRateProvider,
		),
		fPositives:    make(chan *peer.Peer),
		stopChan:      make(chan int),
		fpAccumulator: make(map[int32]int32),
		mutex:         new(sync.RWMutex),
	}

	w.keyManager, err = NewKeyManager(config.DB.Keys(), w.params, w.masterPrivateKey)

	w.txstore, err = NewTxStore(w.params, config.DB, w.keyManager)
	if err != nil {
		return nil, err
	}

	w.blockchain, err = NewBlockchain(w.repoPath, w.creationDate, w.params)
	if err != nil {
		return nil, err
	}

	listeners := &peer.MessageListeners{
		OnMerkleBlock: w.onMerkleBlock,
		OnInv:         w.onInv,
		OnTx:          w.onTx,
		OnGetData:     w.onGetData,
		OnReject:      w.onReject,
	}

	getNewestBlock := func() (*chainhash.Hash, int32, error) {
		storedHeader, err := w.blockchain.db.GetBestHeader()
		if err != nil {
			return nil, 0, err
		}
		height, err := w.blockchain.db.Height()
		if err != nil {
			return nil, 0, err
		}
		hash := storedHeader.header.BlockHash()
		return &hash, int32(height), nil
	}

	w.config = &PeerManagerConfig{
		UserAgentName:      config.UserAgent,
		UserAgentVersion:   WALLET_VERSION,
		Params:             w.params,
		AddressCacheDir:    config.RepoPath,
		GetFilter:          w.txstore.GimmeFilter,
		StartChainDownload: w.startChainDownload,
		GetNewestBlock:     getNewestBlock,
		Listeners:          listeners,
		Proxy:              config.Proxy,
	}

	if config.TrustedPeer != nil {
		w.config.TrustedPeer = config.TrustedPeer
	}

	w.peerManager, err = NewPeerManager(w.config)
	if err != nil {
		return nil, err
	}

	return w, nil
}

func (w *SPVWallet) Start() {
	w.running = true
	go w.peerManager.Start()
	w.fPositiveHandler(w.stopChan)
}

//////////////////////////////////////////////////////////////////////////////////////////////////////////////////
//
// API
//
//////////////

func (w *SPVWallet) CurrencyCode() string {
	if w.params.Name == chaincfg.MainNetParams.Name {
		return "bch"
	} else {
		return "tbch"
	}
}

func (w *SPVWallet) CreationDate() time.Time {
	return w.creationDate
}

func (w *SPVWallet) IsDust(amount int64) bool {
	return txrules.IsDustAmount(btc.Amount(amount), 25, txrules.DefaultRelayFeePerKb)
}

func (w *SPVWallet) MasterPrivateKey() *hd.ExtendedKey {
	return w.masterPrivateKey
}

func (w *SPVWallet) MasterPublicKey() *hd.ExtendedKey {
	return w.masterPublicKey
}

func (w *SPVWallet) Mnemonic() string {
	return w.mnemonic
}

func (w *SPVWallet) ConnectedPeers() []*peer.Peer {
	return w.peerManager.ReadyPeers()
}

func (w *SPVWallet) CurrentAddress(purpose wallet.KeyPurpose) btc.Address {
	key, _ := w.keyManager.GetCurrentKey(purpose)
	addr, _ := key.Address(w.params)
	return btc.Address(addr)
}

func (w *SPVWallet) NewAddress(purpose wallet.KeyPurpose) btc.Address {
	i, _ := w.txstore.Keys().GetUnused(purpose)
	key, _ := w.keyManager.generateChildKey(purpose, uint32(i[1]))
	addr, _ := key.Address(w.params)
	w.txstore.Keys().MarkKeyAsUsed(addr.ScriptAddress())
	w.txstore.PopulateAdrs()
	return btc.Address(addr)
}

func (w *SPVWallet) DecodeAddress(addr string) (btc.Address, error) {
	// Legacy
	decoded, err := btc.DecodeAddress(addr, w.params)
	if err == nil {
		return decoded, nil
	}
	// Cashaddr
	decoded, err = bchutil.DecodeAddress(addr, w.params)
	if err == nil {
		return decoded, nil
	}
	// Bitpay
	decoded, err = bchutil.DecodeBitpay(addr, w.params)
	if err == nil {
		return decoded, nil
	}
	return nil, errors.New("Unrecognized address format")
}

func (w *SPVWallet) ScriptToAddress(script []byte) (btc.Address, error) {
	_, addrs, _, err := txscript.ExtractPkScriptAddrs(script, w.params)
	if err != nil {
		return nil, err
	}
	if len(addrs) == 0 {
		return nil, errors.New("unknown script")
	}
	return addrs[0], nil
}

func (w *SPVWallet) AddressToScript(addr btc.Address) ([]byte, error) {
	return bchutil.PayToAddrScript(addr)
}

func (w *SPVWallet) HasKey(addr btc.Address) bool {
	_, err := w.keyManager.GetKeyForScript(addr.ScriptAddress())
	if err != nil {
		return false
	}
	return true
}

func (w *SPVWallet) GetKey(addr btc.Address) (*btcec.PrivateKey, error) {
	key, err := w.keyManager.GetKeyForScript(addr.ScriptAddress())
	if err != nil {
		return nil, err
	}
	return key.ECPrivKey()
}

func (w *SPVWallet) ListAddresses() []btc.Address {
	keys := w.keyManager.GetKeys()
	addrs := []btc.Address{}
	for _, k := range keys {
		addr, err := k.Address(w.params)
		if err != nil {
			continue
		}
		addrs = append(addrs, addr)
	}
	return addrs
}

func (w *SPVWallet) ListKeys() []btcec.PrivateKey {
	keys := w.keyManager.GetKeys()
	list := []btcec.PrivateKey{}
	for _, k := range keys {
		priv, err := k.ECPrivKey()
		if err != nil {
			continue
		}
		list = append(list, *priv)
	}
	return list
}

func (w *SPVWallet) ImportKey(privKey *btcec.PrivateKey, compress bool) error {
	pub := privKey.PubKey()
	var pubKeyBytes []byte
	if compress {
		pubKeyBytes = pub.SerializeCompressed()
	} else {
		pubKeyBytes = pub.SerializeUncompressed()
	}
	pkHash := btc.Hash160(pubKeyBytes)
	addr, err := btc.NewAddressPubKeyHash(pkHash, w.params)
	if err != nil {
		return err
	}
	return w.keyManager.datastore.ImportKey(addr.ScriptAddress(), privKey)
}

func (w *SPVWallet) Balance() (confirmed, unconfirmed int64) {
	utxos, _ := w.txstore.Utxos().GetAll()
	stxos, _ := w.txstore.Stxos().GetAll()
	for _, utxo := range utxos {
		if !utxo.WatchOnly {
			if utxo.AtHeight > 0 {
				confirmed += utxo.Value
			} else {
				if w.checkIfStxoIsConfirmed(utxo, stxos) {
					confirmed += utxo.Value
				} else {
					unconfirmed += utxo.Value
				}
			}
		}
	}
	return confirmed, unconfirmed
}

func (w *SPVWallet) Transactions() ([]wallet.Txn, error) {
	return w.txstore.Txns().GetAll(false)
}

func (w *SPVWallet) GetTransaction(txid chainhash.Hash) (wallet.Txn, error) {
	_, txn, err := w.txstore.Txns().Get(txid)
	return txn, err
}

func (w *SPVWallet) GetConfirmations(txid chainhash.Hash) (uint32, uint32, error) {
	_, txn, err := w.txstore.Txns().Get(txid)
	if err != nil {
		return 0, 0, err
	}
	if txn.Height == 0 {
		return 0, 0, nil
	}
	chainTip, _ := w.ChainTip()
	return chainTip - uint32(txn.Height) + 1, uint32(txn.Height), nil
}

func (w *SPVWallet) checkIfStxoIsConfirmed(utxo wallet.Utxo, stxos []wallet.Stxo) bool {
	for _, stxo := range stxos {
		if !stxo.Utxo.WatchOnly {
			if stxo.SpendTxid.IsEqual(&utxo.Op.Hash) {
				if stxo.SpendHeight > 0 {
					return true
				} else {
					return w.checkIfStxoIsConfirmed(stxo.Utxo, stxos)
				}
			} else if stxo.Utxo.IsEqual(&utxo) {
				if stxo.Utxo.AtHeight > 0 {
					return true
				} else {
					return false
				}
			}
		}
	}
	return false
}

func (w *SPVWallet) Params() *chaincfg.Params {
	return w.params
}

func (w *SPVWallet) AddTransactionListener(callback func(wallet.TransactionCallback)) {
	w.txstore.listeners = append(w.txstore.listeners, callback)
}

func (w *SPVWallet) ChainTip() (uint32, chainhash.Hash) {
	var ch chainhash.Hash
	sh, err := w.blockchain.db.GetBestHeader()
	if err != nil {
		return 0, ch
	}
	return sh.height, sh.header.BlockHash()
}

func (w *SPVWallet) AddWatchedScript(script []byte) error {
	err := w.txstore.WatchedScripts().Put(script)
	w.txstore.PopulateAdrs()

	for _, peer := range w.peerManager.ReadyPeers() {
		w.updateFilterAndSend(peer)
	}
	return err
}

func (w *SPVWallet) DumpHeaders(writer io.Writer) {
	w.blockchain.db.Print(writer)
}

func (w *SPVWallet) Close() {
	if w.running {
		log.Info("Disconnecting from peers and shutting down")
		w.peerManager.Stop()
		w.blockchain.Close()
		w.stopChan <- 1
		w.running = false
	}
}

func (w *SPVWallet) ReSyncBlockchain(fromDate time.Time) {
	w.peerManager.Stop()
	w.blockchain.Rollback(fromDate)
	w.blockchain.SetChainState(SYNCING)
	w.txstore.PopulateAdrs()
	var err error
	w.peerManager, err = NewPeerManager(w.config)
	if err != nil {
		return
	}
	go w.peerManager.Start()
}
<commit_msg>Switch to using the cashaddr by default<commit_after>package bitcoincash

import (
	"errors"
	"github.com/OpenBazaar/wallet-interface"
	"github.com/btcsuite/btcd/btcec"
	"github.com/btcsuite/btcd/chaincfg"
	"github.com/btcsuite/btcd/chaincfg/chainhash"
	"github.com/btcsuite/btcd/peer"
	"github.com/btcsuite/btcd/txscript"
	btc "github.com/btcsuite/btcutil"
	hd "github.com/btcsuite/btcutil/hdkeychain"
	"github.com/btcsuite/btcwallet/wallet/txrules"
	"github.com/op/go-logging"
	b39 "github.com/tyler-smith/go-bip39"
	"io"
	"sync"
	"time"
	"github.com/cpacia/bchutil"
)

func setupNetworkParams() {
	chaincfg.MainNetParams.Net = bchutil.MainnetMagic
	chaincfg.TestNet3Params.Net = bchutil.TestnetMagic
	chaincfg.RegressionNetParams.Net = bchutil.Regtestmagic
}

type SPVWallet struct {
	params *chaincfg.Params

	masterPrivateKey *hd.ExtendedKey
	masterPublicKey  *hd.ExtendedKey

	mnemonic string

	feeProvider *FeeProvider

	repoPath string

	blockchain  *Blockchain
	txstore     *TxStore
	peerManager *PeerManager
	keyManager  *KeyManager

	fPositives    chan *peer.Peer
	stopChan      chan int
	fpAccumulator map[int32]int32
	mutex         *sync.RWMutex

	creationDate time.Time

	running bool

	config *PeerManagerConfig
}

var log = logging.MustGetLogger("bitcoin")

const WALLET_VERSION = "0.1.0"

func NewSPVWallet(config *Config) (*SPVWallet, error) {
	setupNetworkParams()

	log.SetBackend(logging.AddModuleLevel(config.Logger))

	if config.Mnemonic == "" {
		ent, err := b39.NewEntropy(128)
		if err != nil {
			return nil, err
		}
		mnemonic, err := b39.NewMnemonic(ent)
		if err != nil {
			return nil, err
		}
		config.Mnemonic = mnemonic
		config.CreationDate = time.Now()
	}
	seed := b39.NewSeed(config.Mnemonic, "")

	mPrivKey, err := hd.NewMaster(seed, config.Params)
	if err != nil {
		return nil, err
	}
	mPubKey, err := mPrivKey.Neuter()
	if err != nil {
		return nil, err
	}
	w := &SPVWallet{
		repoPath:         config.RepoPath,
		masterPrivateKey: mPrivKey,
		masterPublicKey:  mPubKey,
		mnemonic:         config.Mnemonic,
		params:           config.Params,
		creationDate:     config.CreationDate,
		feeProvider: NewFeeProvider(
			config.MaxFee,
			config.HighFee,
			config.MediumFee,
			config.LowFee,
			config.ExchangeRateProvider,
		),
		fPositives:    make(chan *peer.Peer),
		stopChan:      make(chan int),
		fpAccumulator: make(map[int32]int32),
		mutex:         new(sync.RWMutex),
	}

	w.keyManager, err = NewKeyManager(config.DB.Keys(), w.params, w.masterPrivateKey)

	w.txstore, err = NewTxStore(w.params, config.DB, w.keyManager)
	if err != nil {
		return nil, err
	}

	w.blockchain, err = NewBlockchain(w.repoPath, w.creationDate, w.params)
	if err != nil {
		return nil, err
	}

	listeners := &peer.MessageListeners{
		OnMerkleBlock: w.onMerkleBlock,
		OnInv:         w.onInv,
		OnTx:          w.onTx,
		OnGetData:     w.onGetData,
		OnReject:      w.onReject,
	}

	getNewestBlock := func() (*chainhash.Hash, int32, error) {
		storedHeader, err := w.blockchain.db.GetBestHeader()
		if err != nil {
			return nil, 0, err
		}
		height, err := w.blockchain.db.Height()
		if err != nil {
			return nil, 0, err
		}
		hash := storedHeader.header.BlockHash()
		return &hash, int32(height), nil
	}

	w.config = &PeerManagerConfig{
		UserAgentName:      config.UserAgent,
		UserAgentVersion:   WALLET_VERSION,
		Params:             w.params,
		AddressCacheDir:    config.RepoPath,
		GetFilter:          w.txstore.GimmeFilter,
		StartChainDownload: w.startChainDownload,
		GetNewestBlock:     getNewestBlock,
		Listeners:          listeners,
		Proxy:              config.Proxy,
	}

	if config.TrustedPeer != nil {
		w.config.TrustedPeer = config.TrustedPeer
	}

	w.peerManager, err = NewPeerManager(w.config)
	if err != nil {
		return nil, err
	}

	return w, nil
}

func (w *SPVWallet) Start() {
	w.running = true
	go w.peerManager.Start()
	w.fPositiveHandler(w.stopChan)
}

//////////////////////////////////////////////////////////////////////////////////////////////////////////////////
//
// API
//
//////////////

func (w *SPVWallet) CurrencyCode() string {
	if w.params.Name == chaincfg.MainNetParams.Name {
		return "bch"
	} else {
		return "tbch"
	}
}

func (w *SPVWallet) CreationDate() time.Time {
	return w.creationDate
}

func (w *SPVWallet) IsDust(amount int64) bool {
	return txrules.IsDustAmount(btc.Amount(amount), 25, txrules.DefaultRelayFeePerKb)
}

func (w *SPVWallet) MasterPrivateKey() *hd.ExtendedKey {
	return w.masterPrivateKey
}

func (w *SPVWallet) MasterPublicKey() *hd.ExtendedKey {
	return w.masterPublicKey
}

func (w *SPVWallet) Mnemonic() string {
	return w.mnemonic
}

func (w *SPVWallet) ConnectedPeers() []*peer.Peer {
	return w.peerManager.ReadyPeers()
}

func (w *SPVWallet) CurrentAddress(purpose wallet.KeyPurpose) btc.Address {
	key, _ := w.keyManager.GetCurrentKey(purpose)
	addr, _ := key.Address(w.params)
	cashaddr, _ := bchutil.NewBitpayAddressPubKeyHash(addr.ScriptAddress(), w.params)
	return btc.Address(cashaddr)
}

func (w *SPVWallet) NewAddress(purpose wallet.KeyPurpose) btc.Address {
	i, _ := w.txstore.Keys().GetUnused(purpose)
	key, _ := w.keyManager.generateChildKey(purpose, uint32(i[1]))
	addr, _ := key.Address(w.params)
	w.txstore.Keys().MarkKeyAsUsed(addr.ScriptAddress())
	w.txstore.PopulateAdrs()
	cashaddr, _ := bchutil.NewBitpayAddressPubKeyHash(addr.ScriptAddress(), w.params)
	return btc.Address(cashaddr)
}

func (w *SPVWallet) DecodeAddress(addr string) (btc.Address, error) {
	// Legacy
	decoded, err := btc.DecodeAddress(addr, w.params)
	if err == nil {
		return decoded, nil
	}
	// Cashaddr
	decoded, err = bchutil.DecodeAddress(addr, w.params)
	if err == nil {
		return decoded, nil
	}
	// Bitpay
	decoded, err = bchutil.DecodeBitpay(addr, w.params)
	if err == nil {
		return decoded, nil
	}
	return nil, errors.New("Unrecognized address format")
}

func (w *SPVWallet) ScriptToAddress(script []byte) (btc.Address, error) {
	addr, err := bchutil.ExtractPkScriptAddrs(script, w.params)
	if err != nil {
		return nil, err
	}
	return btc.Address(addr), nil
}

func (w *SPVWallet) AddressToScript(addr btc.Address) ([]byte, error) {
	return bchutil.PayToAddrScript(addr)
}

func (w *SPVWallet) HasKey(addr btc.Address) bool {
	_, err := w.keyManager.GetKeyForScript(addr.ScriptAddress())
	if err != nil {
		return false
	}
	return true
}

func (w *SPVWallet) GetKey(addr btc.Address) (*btcec.PrivateKey, error) {
	key, err := w.keyManager.GetKeyForScript(addr.ScriptAddress())
	if err != nil {
		return nil, err
	}
	return key.ECPrivKey()
}

func (w *SPVWallet) ListAddresses() []btc.Address {
	keys := w.keyManager.GetKeys()
	addrs := []btc.Address{}
	for _, k := range keys {
		addr, err := k.Address(w.params)
		if err != nil {
			continue
		}
		cashaddr, err := bchutil.NewCashAddressPubKeyHash(addr.ScriptAddress(), w.params)
		if err != nil {
			continue
		}
		addrs = append(addrs, cashaddr)
	}
	return addrs
}

func (w *SPVWallet) ListKeys() []btcec.PrivateKey {
	keys := w.keyManager.GetKeys()
	list := []btcec.PrivateKey{}
	for _, k := range keys {
		priv, err := k.ECPrivKey()
		if err != nil {
			continue
		}
		list = append(list, *priv)
	}
	return list
}

func (w *SPVWallet) ImportKey(privKey *btcec.PrivateKey, compress bool) error {
	pub := privKey.PubKey()
	var pubKeyBytes []byte
	if compress {
		pubKeyBytes = pub.SerializeCompressed()
	} else {
		pubKeyBytes = pub.SerializeUncompressed()
	}
	pkHash := btc.Hash160(pubKeyBytes)
	addr, err := btc.NewAddressPubKeyHash(pkHash, w.params)
	if err != nil {
		return err
	}
	return w.keyManager.datastore.ImportKey(addr.ScriptAddress(), privKey)
}

func (w *SPVWallet) Balance() (confirmed, unconfirmed int64) {
	utxos, _ := w.txstore.Utxos().GetAll()
	stxos, _ := w.txstore.Stxos().GetAll()
	for _, utxo := range utxos {
		if !utxo.WatchOnly {
			if utxo.AtHeight > 0 {
				confirmed += utxo.Value
			} else {
				if w.checkIfStxoIsConfirmed(utxo, stxos) {
					confirmed += utxo.Value
				} else {
					unconfirmed += utxo.Value
				}
			}
		}
	}
	return confirmed, unconfirmed
}

func (w *SPVWallet) Transactions() ([]wallet.Txn, error) {
	return w.txstore.Txns().GetAll(false)
}

func (w *SPVWallet) GetTransaction(txid chainhash.Hash) (wallet.Txn, error) {
	_, txn, err := w.txstore.Txns().Get(txid)
	return txn, err
}

func (w *SPVWallet) GetConfirmations(txid chainhash.Hash) (uint32, uint32, error) {
	_, txn, err := w.txstore.Txns().Get(txid)
	if err != nil {
		return 0, 0, err
	}
	if txn.Height == 0 {
		return 0, 0, nil
	}
	chainTip, _ := w.ChainTip()
	return chainTip - uint32(txn.Height) + 1, uint32(txn.Height), nil
}

func (w *SPVWallet) checkIfStxoIsConfirmed(utxo wallet.Utxo, stxos []wallet.Stxo) bool {
	for _, stxo := range stxos {
		if !stxo.Utxo.WatchOnly {
			if stxo.SpendTxid.IsEqual(&utxo.Op.Hash) {
				if stxo.SpendHeight > 0 {
					return true
				} else {
					return w.checkIfStxoIsConfirmed(stxo.Utxo, stxos)
				}
			} else if stxo.Utxo.IsEqual(&utxo) {
				if stxo.Utxo.AtHeight > 0 {
					return true
				} else {
					return false
				}
			}
		}
	}
	return false
}

func (w *SPVWallet) Params() *chaincfg.Params {
	return w.params
}

func (w *SPVWallet) AddTransactionListener(callback func(wallet.TransactionCallback)) {
	w.txstore.listeners = append(w.txstore.listeners, callback)
}

func (w *SPVWallet) ChainTip() (uint32, chainhash.Hash) {
	var ch chainhash.Hash
	sh, err := w.blockchain.db.GetBestHeader()
	if err != nil {
		return 0, ch
	}
	return sh.height, sh.header.BlockHash()
}

func (w *SPVWallet) AddWatchedScript(script []byte) error {
	err := w.txstore.WatchedScripts().Put(script)
	w.txstore.PopulateAdrs()

	for _, peer := range w.peerManager.ReadyPeers() {
		w.updateFilterAndSend(peer)
	}
	return err
}

func (w *SPVWallet) DumpHeaders(writer io.Writer) {
	w.blockchain.db.Print(writer)
}

func (w *SPVWallet) Close() {
	if w.running {
		log.Info("Disconnecting from peers and shutting down")
		w.peerManager.Stop()
		w.blockchain.Close()
		w.stopChan <- 1
		w.running = false
	}
}

func (w *SPVWallet) ReSyncBlockchain(fromDate time.Time) {
	w.peerManager.Stop()
	w.blockchain.Rollback(fromDate)
	w.blockchain.SetChainState(SYNCING)
	w.txstore.PopulateAdrs()
	var err error
	w.peerManager, err = NewPeerManager(w.config)
	if err != nil {
		return
	}
	go w.peerManager.Start()
}
<|endoftext|>


<commit_before>//
// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package main

import (
	"context"
	"flag"
	"fmt"
	"os"
	"os/signal"
	"path"
	"syscall"
)

const (
	masterWorkloadName = "k8s master"
	workerWorkloadName = "k8s worker"
)

type usageError string

func (e usageError) Error() string {
	return string(e)
}

type vmOptions struct {
	vCPUs   int
	memMiB  int
	diskGiB int
}

type options struct {
	masterVM      vmOptions
	workerVM      vmOptions
	user          string
	publicKeyPath string
	workers       int
	imageUUID     string
	externalIP    string
	keep          bool
	k8sVersion    string
}

type baseConfig struct {
	VCPUs        int
	RAMMiB       int
	DiskGiB      int
	User         string
	ImageUUID    string
	HTTPSProxy   string
	HTTPProxy    string
	NoProxy      string
	Token        string
	PublicKey    string
	UserDataFile string
	Description  string
	K8sVersion   string
}

type proxyConfig struct {
	httpProxy  string
	httpsProxy string
	noProxy    string
}

func init() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage of %s:\n\n", os.Args[0])
		fmt.Fprintf(os.Stderr, "%s create image-uuid [options]\n", os.Args[0])
		fmt.Fprintf(os.Stderr, "%s delete\n\n", os.Args[0])
		fmt.Fprintln(os.Stderr, "- create : creates a k8s cluster")
		fmt.Fprintln(os.Stderr, "- delete : removes kubicle created instances and workloads")
	}
}

func createFlags() (*options, error) {
	opts := options{
		masterVM: vmOptions{
			vCPUs:   1,
			memMiB:  1024,
			diskGiB: 10,
		},
		workerVM: vmOptions{
			vCPUs:   1,
			memMiB:  2048,
			diskGiB: 10,
		},
		workers:    1,
		k8sVersion: "1.6.7",
	}

	opts.user = os.Getenv("USER")
	home := os.Getenv("HOME")
	if home != "" {
		opts.publicKeyPath = path.Join(home, "local", "testkey.pub")
	}

	fs := flag.NewFlagSet("create", flag.ExitOnError)

	fs.IntVar(&opts.masterVM.memMiB, "mmem", opts.masterVM.memMiB,
		"Mebibytes of RAM allocated to master VM")
	fs.IntVar(&opts.masterVM.vCPUs, "mcpus", opts.masterVM.vCPUs, "VCPUs assignged to master VM")
	fs.IntVar(&opts.masterVM.diskGiB, "mdisk", opts.masterVM.diskGiB,
		"Gibibytes of disk allocated to master VM")

	fs.IntVar(&opts.workerVM.memMiB, "wmem", opts.workerVM.memMiB,
		"Mebibytes of RAM allocated to worker VMs")
	fs.IntVar(&opts.workerVM.vCPUs, "wcpus", opts.workerVM.vCPUs, "VCPUs assignged to worker VM")
	fs.IntVar(&opts.workerVM.diskGiB, "wdisk", opts.workerVM.diskGiB,
		"Gibibytes of disk allocated to worker VMs")

	fs.IntVar(&opts.workers, "workers", opts.workers, "Number of worker nodes to create")

	fs.StringVar(&opts.publicKeyPath, "key", opts.publicKeyPath, "Path to public key used to ssh into nodes")
	fs.StringVar(&opts.user, "user", opts.user, "Name of user account to create on the nodes")
	fs.StringVar(&opts.externalIP, "external-ip", opts.externalIP,
		"External-ip to associate with the master node")
	fs.BoolVar(&opts.keep, "keep", false, "Retains workload definition files if set to true")
	fs.StringVar(&opts.k8sVersion, "k8s-version", opts.k8sVersion, "Specifies the version of k8s to install.  Should be either the empty string, meaning the latest, or a version, e.g, 1.6.7")

	if err := fs.Parse(flag.Args()[1:]); err != nil {
		return nil, err
	}

	if len(fs.Args()) < 1 {
		return nil, usageError("No image-uuid specified!")
	}
	opts.imageUUID = fs.Args()[0]

	return &opts, nil
}
func runCommand(signalCh <-chan os.Signal) error {
	var err error

	errCh := make(chan error)
	ctx, cancelFunc := context.WithCancel(context.Background())
	switch os.Args[1] {
	case "create":
		go create(ctx, errCh)
	case "delete":
		go destroy(ctx, errCh)
	}
	select {
	case <-signalCh:
		cancelFunc()
		err = <-errCh
	case err = <-errCh:
		cancelFunc()
	}

	return err
}

func main() {
	flag.Parse()
	if len(flag.Args()) < 1 ||
		!(os.Args[1] == "create" || os.Args[1] == "delete") {
		flag.Usage()
		os.Exit(1)
	}

	signalCh := make(chan os.Signal, 1)
	signal.Notify(signalCh, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)

	if err := runCommand(signalCh); err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		if _, ok := err.(usageError); ok {
			fmt.Fprintln(os.Stderr)
			flag.Usage()
		}
		os.Exit(1)
	}
}
<commit_msg>kubicle: Update the default version of k8s to 1.7.11<commit_after>//
// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package main

import (
	"context"
	"flag"
	"fmt"
	"os"
	"os/signal"
	"path"
	"syscall"
)

const (
	masterWorkloadName = "k8s master"
	workerWorkloadName = "k8s worker"
)

type usageError string

func (e usageError) Error() string {
	return string(e)
}

type vmOptions struct {
	vCPUs   int
	memMiB  int
	diskGiB int
}

type options struct {
	masterVM      vmOptions
	workerVM      vmOptions
	user          string
	publicKeyPath string
	workers       int
	imageUUID     string
	externalIP    string
	keep          bool
	k8sVersion    string
}

type baseConfig struct {
	VCPUs        int
	RAMMiB       int
	DiskGiB      int
	User         string
	ImageUUID    string
	HTTPSProxy   string
	HTTPProxy    string
	NoProxy      string
	Token        string
	PublicKey    string
	UserDataFile string
	Description  string
	K8sVersion   string
}

type proxyConfig struct {
	httpProxy  string
	httpsProxy string
	noProxy    string
}

func init() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage of %s:\n\n", os.Args[0])
		fmt.Fprintf(os.Stderr, "%s create image-uuid [options]\n", os.Args[0])
		fmt.Fprintf(os.Stderr, "%s delete\n\n", os.Args[0])
		fmt.Fprintln(os.Stderr, "- create : creates a k8s cluster")
		fmt.Fprintln(os.Stderr, "- delete : removes kubicle created instances and workloads")
	}
}

func createFlags() (*options, error) {
	opts := options{
		masterVM: vmOptions{
			vCPUs:   1,
			memMiB:  1024,
			diskGiB: 10,
		},
		workerVM: vmOptions{
			vCPUs:   1,
			memMiB:  2048,
			diskGiB: 10,
		},
		workers:    1,
		k8sVersion: "1.7.11",
	}

	opts.user = os.Getenv("USER")
	home := os.Getenv("HOME")
	if home != "" {
		opts.publicKeyPath = path.Join(home, "local", "testkey.pub")
	}

	fs := flag.NewFlagSet("create", flag.ExitOnError)

	fs.IntVar(&opts.masterVM.memMiB, "mmem", opts.masterVM.memMiB,
		"Mebibytes of RAM allocated to master VM")
	fs.IntVar(&opts.masterVM.vCPUs, "mcpus", opts.masterVM.vCPUs, "VCPUs assignged to master VM")
	fs.IntVar(&opts.masterVM.diskGiB, "mdisk", opts.masterVM.diskGiB,
		"Gibibytes of disk allocated to master VM")

	fs.IntVar(&opts.workerVM.memMiB, "wmem", opts.workerVM.memMiB,
		"Mebibytes of RAM allocated to worker VMs")
	fs.IntVar(&opts.workerVM.vCPUs, "wcpus", opts.workerVM.vCPUs, "VCPUs assignged to worker VM")
	fs.IntVar(&opts.workerVM.diskGiB, "wdisk", opts.workerVM.diskGiB,
		"Gibibytes of disk allocated to worker VMs")

	fs.IntVar(&opts.workers, "workers", opts.workers, "Number of worker nodes to create")

	fs.StringVar(&opts.publicKeyPath, "key", opts.publicKeyPath, "Path to public key used to ssh into nodes")
	fs.StringVar(&opts.user, "user", opts.user, "Name of user account to create on the nodes")
	fs.StringVar(&opts.externalIP, "external-ip", opts.externalIP,
		"External-ip to associate with the master node")
	fs.BoolVar(&opts.keep, "keep", false, "Retains workload definition files if set to true")
	fs.StringVar(&opts.k8sVersion, "k8s-version", opts.k8sVersion, "Specifies the version of k8s to install.  Should be either the empty string, meaning the latest, or a version, e.g, 1.6.7")

	if err := fs.Parse(flag.Args()[1:]); err != nil {
		return nil, err
	}

	if len(fs.Args()) < 1 {
		return nil, usageError("No image-uuid specified!")
	}
	opts.imageUUID = fs.Args()[0]

	return &opts, nil
}
func runCommand(signalCh <-chan os.Signal) error {
	var err error

	errCh := make(chan error)
	ctx, cancelFunc := context.WithCancel(context.Background())
	switch os.Args[1] {
	case "create":
		go create(ctx, errCh)
	case "delete":
		go destroy(ctx, errCh)
	}
	select {
	case <-signalCh:
		cancelFunc()
		err = <-errCh
	case err = <-errCh:
		cancelFunc()
	}

	return err
}

func main() {
	flag.Parse()
	if len(flag.Args()) < 1 ||
		!(os.Args[1] == "create" || os.Args[1] == "delete") {
		flag.Usage()
		os.Exit(1)
	}

	signalCh := make(chan os.Signal, 1)
	signal.Notify(signalCh, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)

	if err := runCommand(signalCh); err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		if _, ok := err.(usageError); ok {
			fmt.Fprintln(os.Stderr)
			flag.Usage()
		}
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build !plan9,!solaris

package main

import (
	"bufio"
	"flag"
	"fmt"
	"log"
	"os"
	"path/filepath"

	"github.com/go-fsnotify/fsnotify"
)

func cat(p string) error {
	f, err := os.Open(p)
	if err != nil {
		return err
	}
	defer f.Close()

	var fileInfo os.FileInfo
	fileInfo, err = f.Stat()
	if err != nil {
		return err
	}

	isdir := (fileInfo.Mode() & os.ModeDir) != 0
	if isdir {
		return fmt.Errorf("cannot cat %s", p)
	}

	scanner := bufio.NewScanner(f)

    // Clear the screen first
    fmt.Printf("\033[2J\033[;H");
	for scanner.Scan() {
		fmt.Println(scanner.Text())
	}
	return nil
}

func NewWatcher(p string) {
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		log.Fatal(err)
	}

	defer watcher.Close()
	done := make(chan bool)

    // Initial cat, at least once
    cat(p)

	go func() {
		for {
			select {
			case event := <-watcher.Events:
				if (event.Op & fsnotify.Write) == fsnotify.Write {
					cat(event.Name)
				}
			case err := <-watcher.Errors:
				log.Println("error:", err)
			}
		}
	}()

	err = watcher.Add(p)
	if err != nil {
		log.Fatal(err)
	}
	<-done
}

func main() {
	flag.Parse()
	argv := flag.Args()
	argc := len(argv)
	if argc < 1 {
		fmt.Fprintf(os.Stderr, "expecting a file path at least")
		os.Exit(-1)
	}

	absPath, err := filepath.Abs(argv[0])
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n")
		os.Exit(-2)
	}
	NewWatcher(absPath)
}
<commit_msg>fmt print err on resolve<commit_after>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build !plan9,!solaris

package main

import (
	"bufio"
	"flag"
	"fmt"
	"log"
	"os"
	"path/filepath"

	"github.com/go-fsnotify/fsnotify"
)

func cat(p string) error {
	f, err := os.Open(p)
	if err != nil {
		return err
	}
	defer f.Close()

	var fileInfo os.FileInfo
	fileInfo, err = f.Stat()
	if err != nil {
		return err
	}

	isdir := (fileInfo.Mode() & os.ModeDir) != 0
	if isdir {
		return fmt.Errorf("cat cannot open %s", p)
	}

	scanner := bufio.NewScanner(f)

	// Clear the screen first
	fmt.Printf("\033[2J\033[;H")

	for scanner.Scan() {
		fmt.Println(scanner.Text())
	}
	return nil
}

func NewWatcher(p string) {
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		log.Fatal(err)
	}

	defer watcher.Close()
	done := make(chan bool)

	// Initial cat, at least once
	cat(p)

	go func() {
		for {
			select {
			case event := <-watcher.Events:
				if (event.Op & fsnotify.Write) == fsnotify.Write {
					cat(event.Name)
				}
			case err := <-watcher.Errors:
				log.Println("error:", err)
			}
		}
	}()

	err = watcher.Add(p)
	if err != nil {
		log.Fatal(err)
	}
	<-done
}

func main() {
	flag.Parse()
	argv := flag.Args()
	argc := len(argv)
	if argc < 1 {
		fmt.Fprintf(os.Stderr, "expecting a file path at least")
		os.Exit(-1)
	}

	absPath, err := filepath.Abs(argv[0])
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		os.Exit(-2)
	}
	NewWatcher(absPath)
}
<|endoftext|>


<commit_before>package seed

// ベンチマーカーから参照される
import (
	"encoding/json"
	"math/rand"
)

type Stroke struct {
	Width  int     `json:"width"`
	Red    int     `json:"red"`
	Green  int     `json:"green"`
	Blue   int     `json:"blue"`
	Alpha  float32 `json:"alpha"`
	Points []Point `json:"points"`
}

type Point struct {
	X float32 `json:"x"`
	Y float32 `json:"y"`
}

func GetStrokes(name string) []Stroke {
	data, err := Asset("data/" + name + ".json")
	if err != nil {
		panic(err)
	}
	var s []Stroke
	err = json.Unmarshal(data, &s)
	if err != nil {
		panic(err)
	}
	return s
}

func FluctuateStroke(s Stroke) Stroke {
	points := make([]Point, 0)
	for _, p := range s.Points {
		points = append(points, Point{
			X: p.X + 3.0*rand.Float32() - 1.5,
			Y: p.Y + 3.0*rand.Float32() - 1.5,
		})
	}
	return Stroke{
		Width:  cap(s.Width+rand.Intn(20)-10, 0, 127),
		Red:    rand.Intn(100) + 100,
		Green:  rand.Intn(100) + 100,
		Blue:   rand.Intn(100) + 100,
		Alpha:  s.Alpha,
		Points: points,
	}
}

func cap(i int, min int, max int) int {
	if i < min {
		return min
	} else if i > max {
		return max
	}
	return i
}
<commit_msg>cap collides with internal function<commit_after>package seed

// ベンチマーカーから参照される
import (
	"encoding/json"
	"math/rand"
)

type Stroke struct {
	Width  int     `json:"width"`
	Red    int     `json:"red"`
	Green  int     `json:"green"`
	Blue   int     `json:"blue"`
	Alpha  float32 `json:"alpha"`
	Points []Point `json:"points"`
}

type Point struct {
	X float32 `json:"x"`
	Y float32 `json:"y"`
}

func GetStrokes(name string) []Stroke {
	data, err := Asset("data/" + name + ".json")
	if err != nil {
		panic(err)
	}
	var s []Stroke
	err = json.Unmarshal(data, &s)
	if err != nil {
		panic(err)
	}
	return s
}

func FluctuateStroke(s Stroke) Stroke {
	points := make([]Point, 0)
	for _, p := range s.Points {
		points = append(points, Point{
			X: p.X + 3.0*rand.Float32() - 1.5,
			Y: p.Y + 3.0*rand.Float32() - 1.5,
		})
	}
	return Stroke{
		Width:  bounded(s.Width+rand.Intn(20)-10, 0, 127),
		Red:    rand.Intn(100) + 100,
		Green:  rand.Intn(100) + 100,
		Blue:   rand.Intn(100) + 100,
		Alpha:  s.Alpha,
		Points: points,
	}
}

func bounded(i int, min int, max int) int {
	if i < min {
		return min
	} else if i > max {
		return max
	}
	return i
}
<|endoftext|>