<commit_before>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"net/http"
	"net/url"
	"os"
	"time"

	"k8s.io/contrib/cluster-autoscaler/config"
	"k8s.io/contrib/cluster-autoscaler/simulator"
	"k8s.io/contrib/cluster-autoscaler/utils/gce"
	kube_api "k8s.io/kubernetes/pkg/api"
	kube_record "k8s.io/kubernetes/pkg/client/record"
	kube_client "k8s.io/kubernetes/pkg/client/unversioned"

	"github.com/golang/glog"
	"github.com/prometheus/client_golang/prometheus"
)

var (
	migConfigFlag           config.MigConfigFlag
	address                 = flag.String("address", ":8085", "The address to expose prometheus metrics.")
	kubernetes              = flag.String("kubernetes", "", "Kuberentes master location. Leave blank for default")
	cloudConfig             = flag.String("cloud-config", "", "The path to the cloud provider configuration file.  Empty string for no configuration file.")
	verifyUnschedulablePods = flag.Bool("verify-unschedulable-pods", true,
		"If enabled CA will ensure that each pod marked by Scheduler as unschedulable actually can't be scheduled on any node."+
			"This prevents from adding unnecessary nodes in situation when CA and Scheduler have different configuration.")
	scaleDownEnabled = flag.Bool("scale-down-enabled", true, "Should CA scale down the cluster")
	scaleDownDelay   = flag.Duration("scale-down-delay", 10*time.Minute,
		"Duration from the last scale up to the time when CA starts to check scale down options")
	scaleDownUnderutilizedTime = flag.Duration("scale-down-underutilized-time", 10*time.Minute,
		"How long the node should be underutilized before it is eligible for scale down")
	scaleDownUtilizationThreshold = flag.Float64("scale-down-utilization-threshold", 0.5,
		"Node reservation level below which a node can be considered for scale down")
	scaleDownTrialFrequency = flag.Duration("scale-down-trial-frequency", 10*time.Minute,
		"How often scale down possiblity is check")
)

func main() {
	flag.Var(&migConfigFlag, "nodes", "sets min,max size and url of a MIG to be controlled by Cluster Autoscaler. "+
		"Can be used multiple times. Format: <min>:<max>:<migurl>")
	flag.Parse()

	http.Handle("/metrics", prometheus.Handler())
	err := http.ListenAndServe(*address, nil)
	if err != nil {
		glog.Fatalf("Failed to start http server metrics: %v", err)
	}

	url, err := url.Parse(*kubernetes)
	if err != nil {
		glog.Fatalf("Failed to parse Kuberentes url: %v", err)
	}

	// Configuration
	kubeConfig, err := config.GetKubeClientConfig(url)
	if err != nil {
		glog.Fatalf("Failed to build Kuberentes client configuration: %v", err)
	}
	migConfigs := make([]*config.MigConfig, 0, len(migConfigFlag))
	for i := range migConfigFlag {
		migConfigs = append(migConfigs, &migConfigFlag[i])
	}

	// GCE Manager
	var gceManager *gce.GceManager
	if *cloudConfig != "" {
		config, err := os.Open(*cloudConfig)
		if err != nil {
			glog.Fatalf("Couldn't open cloud provider configuration %s: %#v", *cloudConfig, err)
		}
		defer config.Close()
		gceManager, err = gce.CreateGceManager(migConfigs, config)
	} else {
		gceManager, err = gce.CreateGceManager(migConfigs, nil)
	}
	if err != nil {
		glog.Fatalf("Failed to create GCE Manager: %v", err)
	}

	kubeClient := kube_client.NewOrDie(kubeConfig)

	predicateChecker, err := simulator.NewPredicateChecker(kubeClient)
	if err != nil {
		glog.Fatalf("Failed to create predicate checker: %v", err)
	}
	unschedulablePodLister := NewUnschedulablePodLister(kubeClient)
	scheduledPodLister := NewScheduledPodLister(kubeClient)
	nodeLister := NewNodeLister(kubeClient)

	lastScaleUpTime := time.Now()
	lastScaleDownFailedTrial := time.Now()
	underutilizedNodes := make(map[string]time.Time)

	eventBroadcaster := kube_record.NewBroadcaster()
	eventBroadcaster.StartLogging(glog.Infof)
	eventBroadcaster.StartRecordingToSink(kubeClient.Events(""))
	recorder := eventBroadcaster.NewRecorder(kube_api.EventSource{Component: "cluster-autoscaler"})

	for {
		select {
		case <-time.After(time.Minute):
			{
				loopStart := time.Now()
				updateLastTime("main")

				nodes, err := nodeLister.List()
				if err != nil {
					glog.Errorf("Failed to list nodes: %v", err)
					continue
				}
				if len(nodes) == 0 {
					glog.Errorf("No nodes in the cluster")
					continue
				}

				if err := CheckMigsAndNodes(nodes, gceManager); err != nil {
					glog.Warningf("Cluster is not ready for autoscaling: %v", err)
					continue
				}

				allUnschedulablePods, err := unschedulablePodLister.List()
				if err != nil {
					glog.Errorf("Failed to list unscheduled pods: %v", err)
					continue
				}

				allScheduled, err := scheduledPodLister.List()
				if err != nil {
					glog.Errorf("Failed to list scheduled pods: %v", err)
					continue
				}

				// We need to reset all pods that have been marked as unschedulable not after
				// the newest node became available for the scheduler.
				allNodesAvailableTime := GetAllNodesAvailableTime(nodes)
				podsToReset, unschedulablePodsToHelp := SlicePodsByPodScheduledTime(allUnschedulablePods, allNodesAvailableTime)
				ResetPodScheduledCondition(kubeClient, podsToReset)

				// We need to check whether pods marked as unschedulable are actually unschedulable.
				// This should prevent from adding unnecessary nodes. Example of such situation:
				// - CA and Scheduler has slightly different configuration
				// - Scheduler can't schedule a pod and marks it as unschedulable
				// - CA added a node which should help the pod
				// - Scheduler doesn't schedule the pod on the new node
				//   because according to it logic it doesn't fit there
				// - CA see the pod is still unschedulable, so it adds another node to help it
				//
				// With the check enabled the last point won't happen because CA will ignore a pod
				// which is supposed to schedule on an existing node.
				//
				// Without below check cluster might be unnecessary scaled up to the max allowed size
				// in the describe situation.
				schedulablePodsPresent := false
				if *verifyUnschedulablePods {
					newUnschedulablePodsToHelp := FilterOutSchedulable(unschedulablePodsToHelp, nodes, allScheduled, predicateChecker)

					if len(newUnschedulablePodsToHelp) != len(unschedulablePodsToHelp) {
						schedulablePodsPresent = true
					}
					unschedulablePodsToHelp = newUnschedulablePodsToHelp
				}

				if len(unschedulablePodsToHelp) == 0 {
					glog.V(1).Info("No unschedulable pods")
				} else {
					scaleUpStart := time.Now()
					updateLastTime("scaleup")
					scaledUp, err := ScaleUp(unschedulablePodsToHelp, nodes, migConfigs, gceManager, kubeClient, predicateChecker, recorder)

					updateDuration("scaleup", scaleUpStart)

					if err != nil {
						glog.Errorf("Failed to scale up: %v", err)
						continue
					} else {
						if scaledUp {
							lastScaleUpTime = time.Now()
							// No scale down in this iteration.
							continue
						}
					}
				}

				if *scaleDownEnabled {
					utilizationStart := time.Now()

					// In dry run only utilization is updated
					calculateUtilizationOnly := lastScaleUpTime.Add(*scaleDownDelay).After(time.Now()) ||
						lastScaleDownFailedTrial.Add(*scaleDownTrialFrequency).After(time.Now()) ||
						schedulablePodsPresent

					updateLastTime("utilization")

					underutilizedNodes = CalculateUnderutilizedNodes(
						nodes,
						underutilizedNodes,
						*scaleDownUtilizationThreshold,
						allScheduled,
						predicateChecker)

					updateDuration("utilization", utilizationStart)

					if !calculateUtilizationOnly {
						scaleDownStart := time.Now()
						updateLastTime("scaledown")

						result, err := ScaleDown(
							nodes,
							underutilizedNodes,
							*scaleDownUnderutilizedTime,
							allScheduled,
							gceManager, kubeClient, predicateChecker)

						updateDuration("scaledown", scaleDownStart)

						if err != nil {
							glog.Errorf("Failed to scale down: %v", err)
						} else {
							if result == ScaleDownNodeDeleted {
								// Clean the utilization map to be super sure that the simulated
								// deletions are made in the new context.
								underutilizedNodes = make(map[string]time.Time, len(underutilizedNodes))
							} else {
								lastScaleDownFailedTrial = time.Now()
							}
						}
					}
				}
				updateDuration("main", loopStart)
			}
		}
	}
}

func updateDuration(label string, start time.Time) {
	duration.WithLabelValues(label).Observe(durationToMicro(start))
	lastDuration.WithLabelValues(label).Set(durationToMicro(start))
}

func updateLastTime(label string) {
	lastTimestamp.WithLabelValues(label).Set(float64(time.Now().Unix()))
}
<commit_msg>Cluster-autoscaler: start http server in a separate gorouting<commit_after>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"net/http"
	"net/url"
	"os"
	"time"

	"k8s.io/contrib/cluster-autoscaler/config"
	"k8s.io/contrib/cluster-autoscaler/simulator"
	"k8s.io/contrib/cluster-autoscaler/utils/gce"
	kube_api "k8s.io/kubernetes/pkg/api"
	kube_record "k8s.io/kubernetes/pkg/client/record"
	kube_client "k8s.io/kubernetes/pkg/client/unversioned"

	"github.com/golang/glog"
	"github.com/prometheus/client_golang/prometheus"
)

var (
	migConfigFlag           config.MigConfigFlag
	address                 = flag.String("address", ":8085", "The address to expose prometheus metrics.")
	kubernetes              = flag.String("kubernetes", "", "Kuberentes master location. Leave blank for default")
	cloudConfig             = flag.String("cloud-config", "", "The path to the cloud provider configuration file.  Empty string for no configuration file.")
	verifyUnschedulablePods = flag.Bool("verify-unschedulable-pods", true,
		"If enabled CA will ensure that each pod marked by Scheduler as unschedulable actually can't be scheduled on any node."+
			"This prevents from adding unnecessary nodes in situation when CA and Scheduler have different configuration.")
	scaleDownEnabled = flag.Bool("scale-down-enabled", true, "Should CA scale down the cluster")
	scaleDownDelay   = flag.Duration("scale-down-delay", 10*time.Minute,
		"Duration from the last scale up to the time when CA starts to check scale down options")
	scaleDownUnderutilizedTime = flag.Duration("scale-down-underutilized-time", 10*time.Minute,
		"How long the node should be underutilized before it is eligible for scale down")
	scaleDownUtilizationThreshold = flag.Float64("scale-down-utilization-threshold", 0.5,
		"Node reservation level below which a node can be considered for scale down")
	scaleDownTrialFrequency = flag.Duration("scale-down-trial-frequency", 10*time.Minute,
		"How often scale down possiblity is check")
)

func main() {
	flag.Var(&migConfigFlag, "nodes", "sets min,max size and url of a MIG to be controlled by Cluster Autoscaler. "+
		"Can be used multiple times. Format: <min>:<max>:<migurl>")
	flag.Parse()

	go func() {
		http.Handle("/metrics", prometheus.Handler())
		err := http.ListenAndServe(*address, nil)
		glog.Fatalf("Failed to start metrics: %v", err)
	}()

	url, err := url.Parse(*kubernetes)
	if err != nil {
		glog.Fatalf("Failed to parse Kuberentes url: %v", err)
	}

	// Configuration
	kubeConfig, err := config.GetKubeClientConfig(url)
	if err != nil {
		glog.Fatalf("Failed to build Kuberentes client configuration: %v", err)
	}
	migConfigs := make([]*config.MigConfig, 0, len(migConfigFlag))
	for i := range migConfigFlag {
		migConfigs = append(migConfigs, &migConfigFlag[i])
	}

	// GCE Manager
	var gceManager *gce.GceManager
	if *cloudConfig != "" {
		config, err := os.Open(*cloudConfig)
		if err != nil {
			glog.Fatalf("Couldn't open cloud provider configuration %s: %#v", *cloudConfig, err)
		}
		defer config.Close()
		gceManager, err = gce.CreateGceManager(migConfigs, config)
	} else {
		gceManager, err = gce.CreateGceManager(migConfigs, nil)
	}
	if err != nil {
		glog.Fatalf("Failed to create GCE Manager: %v", err)
	}

	kubeClient := kube_client.NewOrDie(kubeConfig)

	predicateChecker, err := simulator.NewPredicateChecker(kubeClient)
	if err != nil {
		glog.Fatalf("Failed to create predicate checker: %v", err)
	}
	unschedulablePodLister := NewUnschedulablePodLister(kubeClient)
	scheduledPodLister := NewScheduledPodLister(kubeClient)
	nodeLister := NewNodeLister(kubeClient)

	lastScaleUpTime := time.Now()
	lastScaleDownFailedTrial := time.Now()
	underutilizedNodes := make(map[string]time.Time)

	eventBroadcaster := kube_record.NewBroadcaster()
	eventBroadcaster.StartLogging(glog.Infof)
	eventBroadcaster.StartRecordingToSink(kubeClient.Events(""))
	recorder := eventBroadcaster.NewRecorder(kube_api.EventSource{Component: "cluster-autoscaler"})

	for {
		select {
		case <-time.After(time.Minute):
			{
				loopStart := time.Now()
				updateLastTime("main")

				nodes, err := nodeLister.List()
				if err != nil {
					glog.Errorf("Failed to list nodes: %v", err)
					continue
				}
				if len(nodes) == 0 {
					glog.Errorf("No nodes in the cluster")
					continue
				}

				if err := CheckMigsAndNodes(nodes, gceManager); err != nil {
					glog.Warningf("Cluster is not ready for autoscaling: %v", err)
					continue
				}

				allUnschedulablePods, err := unschedulablePodLister.List()
				if err != nil {
					glog.Errorf("Failed to list unscheduled pods: %v", err)
					continue
				}

				allScheduled, err := scheduledPodLister.List()
				if err != nil {
					glog.Errorf("Failed to list scheduled pods: %v", err)
					continue
				}

				// We need to reset all pods that have been marked as unschedulable not after
				// the newest node became available for the scheduler.
				allNodesAvailableTime := GetAllNodesAvailableTime(nodes)
				podsToReset, unschedulablePodsToHelp := SlicePodsByPodScheduledTime(allUnschedulablePods, allNodesAvailableTime)
				ResetPodScheduledCondition(kubeClient, podsToReset)

				// We need to check whether pods marked as unschedulable are actually unschedulable.
				// This should prevent from adding unnecessary nodes. Example of such situation:
				// - CA and Scheduler has slightly different configuration
				// - Scheduler can't schedule a pod and marks it as unschedulable
				// - CA added a node which should help the pod
				// - Scheduler doesn't schedule the pod on the new node
				//   because according to it logic it doesn't fit there
				// - CA see the pod is still unschedulable, so it adds another node to help it
				//
				// With the check enabled the last point won't happen because CA will ignore a pod
				// which is supposed to schedule on an existing node.
				//
				// Without below check cluster might be unnecessary scaled up to the max allowed size
				// in the describe situation.
				schedulablePodsPresent := false
				if *verifyUnschedulablePods {
					newUnschedulablePodsToHelp := FilterOutSchedulable(unschedulablePodsToHelp, nodes, allScheduled, predicateChecker)

					if len(newUnschedulablePodsToHelp) != len(unschedulablePodsToHelp) {
						schedulablePodsPresent = true
					}
					unschedulablePodsToHelp = newUnschedulablePodsToHelp
				}

				if len(unschedulablePodsToHelp) == 0 {
					glog.V(1).Info("No unschedulable pods")
				} else {
					scaleUpStart := time.Now()
					updateLastTime("scaleup")
					scaledUp, err := ScaleUp(unschedulablePodsToHelp, nodes, migConfigs, gceManager, kubeClient, predicateChecker, recorder)

					updateDuration("scaleup", scaleUpStart)

					if err != nil {
						glog.Errorf("Failed to scale up: %v", err)
						continue
					} else {
						if scaledUp {
							lastScaleUpTime = time.Now()
							// No scale down in this iteration.
							continue
						}
					}
				}

				if *scaleDownEnabled {
					utilizationStart := time.Now()

					// In dry run only utilization is updated
					calculateUtilizationOnly := lastScaleUpTime.Add(*scaleDownDelay).After(time.Now()) ||
						lastScaleDownFailedTrial.Add(*scaleDownTrialFrequency).After(time.Now()) ||
						schedulablePodsPresent

					updateLastTime("utilization")

					underutilizedNodes = CalculateUnderutilizedNodes(
						nodes,
						underutilizedNodes,
						*scaleDownUtilizationThreshold,
						allScheduled,
						predicateChecker)

					updateDuration("utilization", utilizationStart)

					if !calculateUtilizationOnly {
						scaleDownStart := time.Now()
						updateLastTime("scaledown")

						result, err := ScaleDown(
							nodes,
							underutilizedNodes,
							*scaleDownUnderutilizedTime,
							allScheduled,
							gceManager, kubeClient, predicateChecker)

						updateDuration("scaledown", scaleDownStart)

						if err != nil {
							glog.Errorf("Failed to scale down: %v", err)
						} else {
							if result == ScaleDownNodeDeleted {
								// Clean the utilization map to be super sure that the simulated
								// deletions are made in the new context.
								underutilizedNodes = make(map[string]time.Time, len(underutilizedNodes))
							} else {
								lastScaleDownFailedTrial = time.Now()
							}
						}
					}
				}
				updateDuration("main", loopStart)
			}
		}
	}
}

func updateDuration(label string, start time.Time) {
	duration.WithLabelValues(label).Observe(durationToMicro(start))
	lastDuration.WithLabelValues(label).Set(durationToMicro(start))
}

func updateLastTime(label string) {
	lastTimestamp.WithLabelValues(label).Set(float64(time.Now().Unix()))
}
<|endoftext|>


<commit_before>/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package main

import (
	"google.golang.org/protobuf/compiler/protogen"
	"google.golang.org/protobuf/reflect/protoreflect"
)

type generator struct {
	out *protogen.GeneratedFile
}

func newGenerator(out *protogen.GeneratedFile) *generator {
	gen := generator{out: out}
	return &gen
}

func (gen *generator) genFieldMethod(m *protogen.Message) {
	p := gen.out

	p.P("// Field returns the value for the given fieldpath as a string, if defined.")
	p.P("// If the value is not defined, the second value will be false.")
	p.P("func (m *", m.GoIdent, ") Field(fieldpath []string) (string, bool) {")

	var (
		fields    []*protogen.Field
		unhandled []*protogen.Field
	)

	for _, f := range m.Fields {
		if f.Desc.Kind() == protoreflect.BoolKind ||
			f.Desc.Kind() == protoreflect.StringKind ||
			isLabelsField(f) || isAnyField(f) || isMessageField(f) {
			fields = append(fields, f)
		} else {
			unhandled = append(unhandled, f)
		}

	}

	if len(fields) > 0 {
		p.P("if len(fieldpath) == 0 {")
		p.P(`return "", false`)
		p.P("}")

		p.P("switch fieldpath[0] {")

		for _, f := range unhandled {
			p.P("// unhandled: ", f.Desc.Name())
		}

		for _, f := range fields {
			p.P(`case "`, f.Desc.Name(), `":`)
			switch {
			case isLabelsField(f):
				stringsJoin := gen.out.QualifiedGoIdent(protogen.GoIdent{
					GoImportPath: "strings",
					GoName:       "Join",
				})

				p.P(`// Labels fields have been special-cased by name. If this breaks,`)
				p.P(`// add better special casing to fieldpath plugin.`)
				p.P("if len(m.", f.GoName, ") == 0 {")
				p.P(`return "", false`)
				p.P("}")
				p.P("value, ok := m.", f.GoName, "[", stringsJoin, `(fieldpath[1:], ".")]`)
				p.P("return value, ok")
			case isAnyField(f):
				typeurlUnmarshalAny := gen.out.QualifiedGoIdent(protogen.GoIdent{
					GoImportPath: "github.com/containerd/typeurl",
					GoName:       "UnmarshalAny",
				})

				p.P("decoded, err := ", typeurlUnmarshalAny, "(m.", f.GoName, ")")
				p.P("if err != nil {")
				p.P(`return "", false`)
				p.P("}")
				p.P("adaptor, ok := decoded.(interface{ Field([]string) (string, bool) })")
				p.P("if !ok {")
				p.P(`return "", false`)
				p.P("}")
				p.P("return adaptor.Field(fieldpath[1:])")
			case isMessageField(f):
				p.P(`// NOTE(stevvooe): This is probably not correct in many cases.`)
				p.P(`// We assume that the target message also implements the Field`)
				p.P(`// method, which isn't likely true in a lot of cases.`)
				p.P(`//`)
				p.P(`// If you have a broken build and have found this comment,`)
				p.P(`// you may be closer to a solution.`)
				p.P("if m.", f.GoName, " == nil {")
				p.P(`return "", false`)
				p.P("}")
				p.P("return m.", f.GoName, ".Field(fieldpath[1:])")
			case f.Desc.Kind() == protoreflect.StringKind:
				p.P("return string(m.", f.GoName, "), len(m.", f.GoName, ") > 0")
			case f.Desc.Kind() == protoreflect.BoolKind:
				fmtSprint := gen.out.QualifiedGoIdent(protogen.GoIdent{
					GoImportPath: "fmt",
					GoName:       "Sprint",
				})

				p.P("return ", fmtSprint, "(m.", f.GoName, "), true")
			}
		}

		p.P("}")
	} else {
		for _, f := range unhandled {
			p.P("// unhandled: ", f.Desc.Name())
		}
	}

	p.P(`return "", false`)
	p.P("}")
}

func isMessageField(f *protogen.Field) bool {
	return f.Desc.Kind() == protoreflect.MessageKind && f.GoIdent.GoName != "Timestamp"
}

func isLabelsField(f *protogen.Field) bool {
	return f.Desc.Kind() == protoreflect.MessageKind && f.Desc.Name() == "labels"
}

func isAnyField(f *protogen.Field) bool {
	return f.Desc.Kind() == protoreflect.MessageKind && f.GoIdent.GoName == "Any"
}

func generate(plugin *protogen.Plugin, input *protogen.File) error {
	file := plugin.NewGeneratedFile(input.GeneratedFilenamePrefix+"_fieldpath.pb.go", input.GoImportPath)
	file.P("// Code generated by protoc-gen-go-fieldpath. DO NOT EDIT.")
	file.P("// source: ", input.Desc.Path())
	file.P("package ", input.GoPackageName)

	gen := newGenerator(file)
	for _, m := range input.Messages {
		gen.genFieldMethod(m)
	}
	return nil
}
<commit_msg>Fix protoc-gen-go-fieldpath<commit_after>/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package main

import (
	"google.golang.org/protobuf/compiler/protogen"
	"google.golang.org/protobuf/reflect/protoreflect"
)

type generator struct {
	out *protogen.GeneratedFile
}

func newGenerator(out *protogen.GeneratedFile) *generator {
	gen := generator{out: out}
	return &gen
}

func (gen *generator) genFieldMethod(m *protogen.Message) {
	p := gen.out

	p.P("// Field returns the value for the given fieldpath as a string, if defined.")
	p.P("// If the value is not defined, the second value will be false.")
	p.P("func (m *", m.GoIdent, ") Field(fieldpath []string) (string, bool) {")

	var (
		fields    []*protogen.Field
		unhandled []*protogen.Field
	)

	for _, f := range m.Fields {
		if f.Desc.Kind() == protoreflect.BoolKind ||
			f.Desc.Kind() == protoreflect.StringKind ||
			isLabelsField(f) || isAnyField(f) || isMessageField(f) {
			fields = append(fields, f)
		} else {
			unhandled = append(unhandled, f)
		}

	}

	if len(fields) > 0 {
		p.P("if len(fieldpath) == 0 {")
		p.P(`return "", false`)
		p.P("}")

		p.P("switch fieldpath[0] {")

		for _, f := range unhandled {
			p.P("// unhandled: ", f.Desc.Name())
		}

		for _, f := range fields {
			p.P(`case "`, f.Desc.Name(), `":`)
			switch {
			case isLabelsField(f):
				stringsJoin := gen.out.QualifiedGoIdent(protogen.GoIdent{
					GoImportPath: "strings",
					GoName:       "Join",
				})

				p.P(`// Labels fields have been special-cased by name. If this breaks,`)
				p.P(`// add better special casing to fieldpath plugin.`)
				p.P("if len(m.", f.GoName, ") == 0 {")
				p.P(`return "", false`)
				p.P("}")
				p.P("value, ok := m.", f.GoName, "[", stringsJoin, `(fieldpath[1:], ".")]`)
				p.P("return value, ok")
			case isAnyField(f):
				typeurlUnmarshalAny := gen.out.QualifiedGoIdent(protogen.GoIdent{
					GoImportPath: "github.com/containerd/typeurl",
					GoName:       "UnmarshalAny",
				})

				p.P("decoded, err := ", typeurlUnmarshalAny, "(m.", f.GoName, ")")
				p.P("if err != nil {")
				p.P(`return "", false`)
				p.P("}")
				p.P("adaptor, ok := decoded.(interface{ Field([]string) (string, bool) })")
				p.P("if !ok {")
				p.P(`return "", false`)
				p.P("}")
				p.P("return adaptor.Field(fieldpath[1:])")
			case isMessageField(f):
				p.P(`// NOTE(stevvooe): This is probably not correct in many cases.`)
				p.P(`// We assume that the target message also implements the Field`)
				p.P(`// method, which isn't likely true in a lot of cases.`)
				p.P(`//`)
				p.P(`// If you have a broken build and have found this comment,`)
				p.P(`// you may be closer to a solution.`)
				p.P("if m.", f.GoName, " == nil {")
				p.P(`return "", false`)
				p.P("}")
				p.P("return m.", f.GoName, ".Field(fieldpath[1:])")
			case f.Desc.Kind() == protoreflect.StringKind:
				p.P("return string(m.", f.GoName, "), len(m.", f.GoName, ") > 0")
			case f.Desc.Kind() == protoreflect.BoolKind:
				fmtSprint := gen.out.QualifiedGoIdent(protogen.GoIdent{
					GoImportPath: "fmt",
					GoName:       "Sprint",
				})

				p.P("return ", fmtSprint, "(m.", f.GoName, "), true")
			}
		}

		p.P("}")
	} else {
		for _, f := range unhandled {
			p.P("// unhandled: ", f.Desc.Name())
		}
	}

	p.P(`return "", false`)
	p.P("}")
}

func isMessageField(f *protogen.Field) bool {
	return f.Desc.Kind() == protoreflect.MessageKind && f.Desc.Cardinality() != protoreflect.Repeated && f.Message.GoIdent.GoName != "Timestamp"
}

func isLabelsField(f *protogen.Field) bool {
	return f.Desc.Kind() == protoreflect.MessageKind && f.Desc.Name() == "labels"
}

func isAnyField(f *protogen.Field) bool {
	return f.Desc.Kind() == protoreflect.MessageKind && f.Message.GoIdent.GoName == "Any"
}

func collectChildlen(parent *protogen.Message) ([]*protogen.Message, error) {
	var children []*protogen.Message
	for _, child := range parent.Messages {
		if child.Desc.IsMapEntry() {
			continue
		}
		children = append(children, child)

		xs, err := collectChildlen(child)
		if err != nil {
			return nil, err
		}
		children = append(children, xs...)
	}
	return children, nil
}

func generate(plugin *protogen.Plugin, input *protogen.File) error {
	file := plugin.NewGeneratedFile(input.GeneratedFilenamePrefix+"_fieldpath.pb.go", input.GoImportPath)
	file.P("// Code generated by protoc-gen-go-fieldpath. DO NOT EDIT.")
	file.P("// source: ", input.Desc.Path())
	file.P("package ", input.GoPackageName)

	gen := newGenerator(file)

	var messages []*protogen.Message
	for _, m := range input.Messages {
		messages = append(messages, m)
		children, err := collectChildlen(m)
		if err != nil {
			return err
		}
		messages = append(messages, children...)
	}

	for _, m := range messages {
		gen.genFieldMethod(m)
	}
	return nil
}
<|endoftext|>


<commit_before>package emojiplugin

import (
	"fmt"
	"net/http"
	"os"
	"regexp"
	"strings"

	"github.com/iopred/bruxism"
)

func emojiFile(base, s string) string {
	found := ""
	filename := ""
	for _, r := range s {
		if filename != "" {
			filename = fmt.Sprintf("%s-%x", filename, r)
		} else {
			filename = fmt.Sprintf("%x", r)
		}

		if _, err := os.Stat(fmt.Sprintf("%s/%s.png", base, filename)); err == nil {
			found = filename
		} else if found != "" {
			return found
		}
	}
	return found
}

func emojiLoadFunc(bot *bruxism.Bot, service bruxism.Service, data []byte) error {
	if service.Name() != bruxism.DiscordServiceName {
		panic("Emoji Plugin only supports Discord.")
	}
	return nil
}

var discordRegex = regexp.MustCompile("<(a?):.*?:(.*?)>")

func emojiMessageFunc(bot *bruxism.Bot, service bruxism.Service, message bruxism.Message) {
	if service.Name() != bruxism.DiscordServiceName {
		return
	}

	if service.IsMe(message) {
		return
	}

	if !bruxism.MatchesCommand(service, "emoji", message) && !bruxism.MatchesCommand(service, "hugemoji", message) && !bruxism.MatchesCommand(service, "hugeemoji", message) {
		return
	}

	base := "emoji/twitter"
	if bruxism.MatchesCommand(service, "hugemoji", message) || bruxism.MatchesCommand(service, "hugeemoji", message) {
		base = "emoji/twitterhuge"
	}
	_, parts := bruxism.ParseCommand(service, message)
	if len(parts) == 1 {
		submatches := discordRegex.FindStringSubmatch(parts[0])
		if len(submatches) != 0 {
			url := discordgo.EndpointEmoji(submatches[2])
			if submatches[1] == "a" {
				url := discordgo.EndpointEmojiAnimated(submatches[2])
			}
			h, err := http.Get(url)
			if err != nil {
				return
			}

			service.SendFile(message.Channel(), "emoji"+fileType, h.Body)
			h.Body.Close()

			return

		}

		s := strings.TrimSpace(parts[0])
		for i := range s {
			filename := emojiFile(base, s[i:])
			if filename != "" {
				if f, err := os.Open(fmt.Sprintf("%s/%s.png", base, filename)); err == nil {
					defer f.Close()
					service.SendFile(message.Channel(), "emoji.png", f)

					return
				}
			}
		}
	}
}

func emojiHelpFunc(bot *bruxism.Bot, service bruxism.Service, message bruxism.Message, detailed bool) []string {
	help := bruxism.CommandHelp(service, "emoji", "<emoji>", "Returns a big version of an emoji.")

	if detailed {
		help = append(help, bruxism.CommandHelp(service, "hugemoji", "<emoji>", "Returns a huge version of an emoji.")[0])
	}

	return help
}

// New creates a new emoji plugin.
func New() bruxism.Plugin {
	p := bruxism.NewSimplePlugin("Emoji")
	p.LoadFunc = emojiLoadFunc
	p.MessageFunc = emojiMessageFunc
	p.HelpFunc = emojiHelpFunc
	return p
}
<commit_msg>Fix emoji plugin<commit_after>package emojiplugin

import (
	"fmt"
	"net/http"
	"os"
	"regexp"
	"strings"

	"github.com/bwmarrin/discordgo"
	"github.com/iopred/bruxism"
)

func emojiFile(base, s string) string {
	found := ""
	filename := ""
	for _, r := range s {
		if filename != "" {
			filename = fmt.Sprintf("%s-%x", filename, r)
		} else {
			filename = fmt.Sprintf("%x", r)
		}

		if _, err := os.Stat(fmt.Sprintf("%s/%s.png", base, filename)); err == nil {
			found = filename
		} else if found != "" {
			return found
		}
	}
	return found
}

func emojiLoadFunc(bot *bruxism.Bot, service bruxism.Service, data []byte) error {
	if service.Name() != bruxism.DiscordServiceName {
		panic("Emoji Plugin only supports Discord.")
	}
	return nil
}

var discordRegex = regexp.MustCompile("<(a?):.*?:(.*?)>")

func emojiMessageFunc(bot *bruxism.Bot, service bruxism.Service, message bruxism.Message) {
	if service.Name() != bruxism.DiscordServiceName {
		return
	}

	if service.IsMe(message) {
		return
	}

	if !bruxism.MatchesCommand(service, "emoji", message) && !bruxism.MatchesCommand(service, "hugemoji", message) && !bruxism.MatchesCommand(service, "hugeemoji", message) {
		return
	}

	base := "emoji/twitter"
	if bruxism.MatchesCommand(service, "hugemoji", message) || bruxism.MatchesCommand(service, "hugeemoji", message) {
		base = "emoji/twitterhuge"
	}
	_, parts := bruxism.ParseCommand(service, message)
	if len(parts) == 1 {
		submatches := discordRegex.FindStringSubmatch(parts[0])
		if len(submatches) != 0 {
			fileType := "png"
			url := discordgo.EndpointEmoji(submatches[2])
			if submatches[1] == "a" {
				fileType := "gif"
				url := discordgo.EndpointEmojiAnimated(submatches[2])
			}
			h, err := http.Get(url)
			if err != nil {
				return
			}

			service.SendFile(message.Channel(), "emoji."+fileType, h.Body)
			h.Body.Close()

			return

		}

		s := strings.TrimSpace(parts[0])
		for i := range s {
			filename := emojiFile(base, s[i:])
			if filename != "" {
				if f, err := os.Open(fmt.Sprintf("%s/%s.png", base, filename)); err == nil {
					defer f.Close()
					service.SendFile(message.Channel(), "emoji.png", f)

					return
				}
			}
		}
	}
}

func emojiHelpFunc(bot *bruxism.Bot, service bruxism.Service, message bruxism.Message, detailed bool) []string {
	help := bruxism.CommandHelp(service, "emoji", "<emoji>", "Returns a big version of an emoji.")

	if detailed {
		help = append(help, bruxism.CommandHelp(service, "hugemoji", "<emoji>", "Returns a huge version of an emoji.")[0])
	}

	return help
}

// New creates a new emoji plugin.
func New() bruxism.Plugin {
	p := bruxism.NewSimplePlugin("Emoji")
	p.LoadFunc = emojiLoadFunc
	p.MessageFunc = emojiMessageFunc
	p.HelpFunc = emojiHelpFunc
	return p
}
<|endoftext|>


<commit_before>package emojiplugin

import (
	"fmt"
	"os"
	"strings"

	"github.com/iopred/bruxism"
)

func emojiFile(base, s string) string {
	found := ""
	filename := ""
	for _, r := range s {
		if filename != "" {
			filename = fmt.Sprintf("%s-%x", filename, r)
		} else {
			filename = fmt.Sprintf("%x", r)
		}

		if _, err := os.Stat(fmt.Sprintf("%s/%s.png", base, filename)); err == nil {
			found = filename
		} else if found != "" {
			return found
		}
	}
	return found
}

func emojiLoadFunc(bot *bruxism.Bot, service bruxism.Service, data []byte) error {
	if service.Name() != bruxism.DiscordServiceName {
		panic("Emoji Plugin only supports Discord.")
	}
	return nil
}

func emojiMessageFunc(bot *bruxism.Bot, service bruxism.Service, message bruxism.Message) {
	if service.Name() == bruxism.DiscordServiceName && !service.IsMe(message) {
		if bruxism.MatchesCommand(service, "emoji", message) || bruxism.MatchesCommand(service, "hugemoji", message) {
			base := "emoji/twitter"
			if bruxism.MatchesCommand(service, "hugemoji", message) {
				base = "emoji/twitterhuge"
			}
			_, parts := bruxism.ParseCommand(service, message)
			if len(parts) == 1 {
				s := strings.TrimSpace(parts[0])
				for i := range s {
					filename := emojiFile(base, s[i:])
					if filename != "" {
						if f, err := os.Open(fmt.Sprintf("%s/%s.png", base, filename)); err == nil {
							defer f.Close()
							service.SendFile(message.Channel(), "emoji.png", f)

							return
						}
					}
				}
			}
		}
	}
}

func emojiHelpFunc(bot *bruxism.Bot, service bruxism.Service, message bruxism.Message, detailed bool) []string {
	help := bruxism.CommandHelp(service, "emoji", "<emoji>", "Returns a big version of an emoji.")

	if detailed {
		help = append(help, bruxism.CommandHelp(service, "hugemoji", "<emoji>", "Returns a huge version of an emoji."))
	}

	return help
}

// New creates a new emoji plugin.
func New() bruxism.Plugin {
	p := bruxism.NewSimplePlugin("Emoji")
	p.LoadFunc = emojiLoadFunc
	p.MessageFunc = emojiMessageFunc
	p.HelpFunc = emojiHelpFunc
	return p
}
<commit_msg>:ok_hand:<commit_after>package emojiplugin

import (
	"fmt"
	"os"
	"strings"

	"github.com/iopred/bruxism"
)

func emojiFile(base, s string) string {
	found := ""
	filename := ""
	for _, r := range s {
		if filename != "" {
			filename = fmt.Sprintf("%s-%x", filename, r)
		} else {
			filename = fmt.Sprintf("%x", r)
		}

		if _, err := os.Stat(fmt.Sprintf("%s/%s.png", base, filename)); err == nil {
			found = filename
		} else if found != "" {
			return found
		}
	}
	return found
}

func emojiLoadFunc(bot *bruxism.Bot, service bruxism.Service, data []byte) error {
	if service.Name() != bruxism.DiscordServiceName {
		panic("Emoji Plugin only supports Discord.")
	}
	return nil
}

func emojiMessageFunc(bot *bruxism.Bot, service bruxism.Service, message bruxism.Message) {
	if service.Name() == bruxism.DiscordServiceName && !service.IsMe(message) {
		if bruxism.MatchesCommand(service, "emoji", message) || bruxism.MatchesCommand(service, "hugemoji", message) {
			base := "emoji/twitter"
			if bruxism.MatchesCommand(service, "hugemoji", message) {
				base = "emoji/twitterhuge"
			}
			_, parts := bruxism.ParseCommand(service, message)
			if len(parts) == 1 {
				s := strings.TrimSpace(parts[0])
				for i := range s {
					filename := emojiFile(base, s[i:])
					if filename != "" {
						if f, err := os.Open(fmt.Sprintf("%s/%s.png", base, filename)); err == nil {
							defer f.Close()
							service.SendFile(message.Channel(), "emoji.png", f)

							return
						}
					}
				}
			}
		}
	}
}

func emojiHelpFunc(bot *bruxism.Bot, service bruxism.Service, message bruxism.Message, detailed bool) []string {
	help := bruxism.CommandHelp(service, "emoji", "<emoji>", "Returns a big version of an emoji.")

	if detailed {
		help = append(help, bruxism.CommandHelp(service, "hugemoji", "<emoji>", "Returns a huge version of an emoji.")[0])
	}

	return help
}

// New creates a new emoji plugin.
func New() bruxism.Plugin {
	p := bruxism.NewSimplePlugin("Emoji")
	p.LoadFunc = emojiLoadFunc
	p.MessageFunc = emojiMessageFunc
	p.HelpFunc = emojiHelpFunc
	return p
}
<|endoftext|>


<commit_before>package docker

import (
	"archive/tar"
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"os"
	"path/filepath"
	"strconv"

	"github.com/docker/go-connections/nat"
	"github.com/docker/go-units"
	"github.com/pkg/errors"

	log "github.com/sirupsen/logrus"

	dockertypes "github.com/docker/docker/api/types"
	dockercontainer "github.com/docker/docker/api/types/container"
	dockernetwork "github.com/docker/docker/api/types/network"
	dockerslice "github.com/docker/docker/api/types/strslice"

	"encoding/json"

	enginetypes "github.com/projecteru2/core/engine/types"
	coretypes "github.com/projecteru2/core/types"
)

const (
	minMemory     = units.MiB * 4
	maxMemory     = math.MaxInt64
	restartAlways = "always"
	root          = "root"
)

type rawArgs struct {
	PidMode    dockercontainer.PidMode `json:"pid_mod"`
	StorageOpt map[string]string       `json:"storage_opt"`
	CapAdd     []string                `json:"cap_add"`
	CapDrop    []string                `json:"cap_drop"`
}

// VirtualizationCreate create a container
func (e *Engine) VirtualizationCreate(ctx context.Context, opts *enginetypes.VirtualizationCreateOptions) (*enginetypes.VirtualizationCreated, error) {
	r := &enginetypes.VirtualizationCreated{}
	// memory should more than 4MiB
	if opts.Memory > 0 && opts.Memory < minMemory || opts.Memory < 0 {
		return r, coretypes.ErrBadMemory
	}
	// set default log driver if lambda
	if opts.Lambda {
		opts.LogType = "json-file"
	}
	// set restart always
	restartRetryCount := 3
	if opts.RestartPolicy == restartAlways {
		restartRetryCount = 0
	}
	// network mode 和 networks 互斥
	// 没有 networks 的时候用 networkmode 的值
	// 有 networks 的时候一律用用 networks 的值作为 mode
	networkMode := dockercontainer.NetworkMode(opts.Network)
	for name := range opts.Networks {
		networkMode = dockercontainer.NetworkMode(name)
		if networkMode.IsHost() {
			opts.Networks[name] = ""
		}
	}
	// 如果没有 network 用默认值替换
	if networkMode == "" {
		networkMode = dockercontainer.NetworkMode(e.config.Docker.NetworkMode)
	}
	// log config
	if opts.LogConfig == nil {
		opts.LogConfig = map[string]string{}
	}
	opts.LogConfig["mode"] = "non-blocking"
	opts.LogConfig["max-buffer-size"] = "4m"
	opts.LogConfig["tag"] = fmt.Sprintf("%s {{.ID}}", opts.Name)
	if opts.Debug {
		opts.LogType = e.config.Docker.Log.Type
		for k, v := range e.config.Docker.Log.Config {
			opts.LogConfig[k] = v
		}
	}
	// add node IP
	hostIP := GetIP(e.client.DaemonHost())
	opts.Env = append(opts.Env, fmt.Sprintf("ERU_NODE_IP=%s", hostIP))
	// 如果有给dns就优先用给定的dns.
	// 没有给出dns的时候, 如果设定是用宿主机IP作为dns, 就会把宿主机IP设置过去.
	// 其他情况就是默认值.
	// 哦对, networkMode如果是host也不给dns.
	if len(opts.DNS) == 0 && e.config.Docker.UseLocalDNS && hostIP != "" {
		opts.DNS = []string{hostIP}
	}
	// mount paths
	binds, volumes := makeMountPaths(opts)
	log.Debugf("[VirtualizationCreate] App %s will bind %v", opts.Name, binds)

	config := &dockercontainer.Config{
		Env:             opts.Env,
		Cmd:             dockerslice.StrSlice(opts.Cmd),
		User:            opts.User,
		Image:           opts.Image,
		Volumes:         volumes,
		WorkingDir:      opts.WorkingDir,
		NetworkDisabled: networkMode == "",
		Labels:          opts.Labels,
		OpenStdin:       opts.Stdin,
		Tty:             opts.Stdin,
	}

	resource := makeResourceSetting(opts.Quota, opts.Memory, opts.CPU, opts.NUMANode, opts.SoftLimit)
	// set ulimits
	resource.Ulimits = []*units.Ulimit{
		&units.Ulimit{Name: "nofile", Soft: 65535, Hard: 65535},
	}
	if networkMode.IsHost() {
		opts.DNS = []string{}
		opts.Sysctl = map[string]string{}
	}
	rArgs := &rawArgs{StorageOpt: map[string]string{}}
	if len(opts.RawArgs) > 0 {
		if err := json.Unmarshal(opts.RawArgs, rArgs); err != nil {
			return r, err
		}
	}
	if opts.Storage > 0 {
		rArgs.StorageOpt["size"] = fmt.Sprintf("%v", opts.Storage)
	}
	// 如果有指定用户，用指定用户
	// 没有指定用户，用镜像自己的
	// CapAdd and Privileged
	capAdds := dockerslice.StrSlice(rArgs.CapAdd)
	if opts.Privileged {
		opts.User = root
		capAdds = append(capAdds, "SYS_ADMIN")
	}
	hostConfig := &dockercontainer.HostConfig{
		Binds: binds,
		DNS:   opts.DNS,
		LogConfig: dockercontainer.LogConfig{
			Type:   opts.LogType,
			Config: opts.LogConfig,
		},
		NetworkMode: networkMode,
		RestartPolicy: dockercontainer.RestartPolicy{
			Name:              opts.RestartPolicy,
			MaximumRetryCount: restartRetryCount,
		},
		CapAdd:     capAdds,
		ExtraHosts: opts.Hosts,
		Privileged: opts.Privileged,
		Resources:  resource,
		Sysctls:    opts.Sysctl,
		PidMode:    rArgs.PidMode,
		StorageOpt: rArgs.StorageOpt,
	}

	if hostConfig.NetworkMode.IsBridge() {
		portMapping := nat.PortMap{}
		exposePorts := nat.PortSet{}
		for _, p := range opts.Publish {
			port, err := nat.NewPort("tcp", p)
			if err != nil {
				return r, err
			}
			exposePorts[port] = struct{}{}
			portMapping[port] = []nat.PortBinding{}
			portMapping[port] = append(portMapping[port], nat.PortBinding{HostPort: p})
		}
		hostConfig.PortBindings = portMapping
		config.ExposedPorts = exposePorts
	}

	networkConfig := &dockernetwork.NetworkingConfig{
		EndpointsConfig: map[string]*dockernetwork.EndpointSettings{},
	}
	for networkID, ipv4 := range opts.Networks {
		endpointSetting, err := e.makeIPV4EndpointSetting(ipv4)
		if err != nil {
			return r, err
		}
		ipForShow := ipv4
		if ipForShow == "" {
			ipForShow = "[AutoAlloc]"
		}
		networkConfig.EndpointsConfig[networkID] = endpointSetting
		log.Infof("[ConnectToNetwork] Connect to %v with IP %v", networkID, ipForShow)
	}

	containerCreated, err := e.client.ContainerCreate(ctx, config, hostConfig, networkConfig, opts.Name)
	r.Name = opts.Name
	r.ID = containerCreated.ID
	return r, err
}

// VirtualizationCopyTo copy things to virtualization
func (e *Engine) VirtualizationCopyTo(ctx context.Context, ID, target string, content io.Reader, AllowOverwriteDirWithFile, CopyUIDGID bool) error {
	return withTarfileDump(target, content, func(target, tarfile string) error {
		content, err := os.Open(tarfile)
		if err != nil {
			return err
		}
		return e.client.CopyToContainer(ctx, ID, filepath.Dir(target), content, dockertypes.CopyToContainerOptions{AllowOverwriteDirWithFile: AllowOverwriteDirWithFile, CopyUIDGID: CopyUIDGID})
	})
}

// VirtualizationStart start virtualization
func (e *Engine) VirtualizationStart(ctx context.Context, ID string) error {
	return e.client.ContainerStart(ctx, ID, dockertypes.ContainerStartOptions{})
}

// VirtualizationStop stop virtualization
func (e *Engine) VirtualizationStop(ctx context.Context, ID string) error {
	return e.client.ContainerStop(ctx, ID, nil)
}

// VirtualizationRemove remove virtualization
func (e *Engine) VirtualizationRemove(ctx context.Context, ID string, removeVolumes, force bool) error {
	return e.client.ContainerRemove(ctx, ID, dockertypes.ContainerRemoveOptions{RemoveVolumes: removeVolumes, Force: force})
}

// VirtualizationInspect get virtualization info
func (e *Engine) VirtualizationInspect(ctx context.Context, ID string) (*enginetypes.VirtualizationInfo, error) {
	if e.client == nil {
		return nil, coretypes.ErrNilEngine
	}

	containerJSON, err := e.client.ContainerInspect(ctx, ID)
	r := &enginetypes.VirtualizationInfo{}
	if err != nil {
		return r, err
	}
	r.ID = containerJSON.ID
	r.User = containerJSON.Config.User
	r.Image = containerJSON.Config.Image
	r.Env = containerJSON.Config.Env
	r.Labels = containerJSON.Config.Labels
	r.Running = containerJSON.State.Running
	r.Networks = map[string]string{}
	for networkName, networkSetting := range containerJSON.NetworkSettings.Networks {
		ip := networkSetting.IPAddress
		if dockercontainer.NetworkMode(networkName).IsHost() {
			ip = GetIP(e.client.DaemonHost())
		}
		r.Networks[networkName] = ip
	}
	return r, nil
}

// VirtualizationLogs show virtualization logs
func (e *Engine) VirtualizationLogs(ctx context.Context, opts *enginetypes.VirtualizationLogStreamOptions) (io.ReadCloser, error) {
	logsOpts := dockertypes.ContainerLogsOptions{Follow: opts.Follow, ShowStdout: opts.Stdout, ShowStderr: opts.Stderr, Tail: opts.Tail}
	resp, err := e.client.ContainerLogs(ctx, opts.ID, logsOpts)
	if err != nil {
		return nil, err
	}
	return ioutil.NopCloser(mergeStream(resp)), nil
}

// VirtualizationAttach attach to a virtualization
func (e *Engine) VirtualizationAttach(ctx context.Context, ID string, stream, stdin bool) (io.ReadCloser, io.WriteCloser, error) {
	opts := dockertypes.ContainerAttachOptions{
		Stream: stream,
		Stdin:  stdin,
		Logs:   true,
		Stdout: true,
		Stderr: true,
	}
	resp, err := e.client.ContainerAttach(ctx, ID, opts)
	if err != nil {
		return nil, nil, err
	}
	return ioutil.NopCloser(resp.Reader), resp.Conn, nil
}

// VirtualizationResize resizes remote terminal
func (e *Engine) VirtualizationResize(ctx context.Context, containerID string, height, width uint) (err error) {
	opts := dockertypes.ResizeOptions{
		Height: height,
		Width:  width,
	}

	return e.client.ContainerResize(ctx, containerID, opts)
}

// VirtualizationWait wait virtualization exit
func (e *Engine) VirtualizationWait(ctx context.Context, ID, state string) (*enginetypes.VirtualizationWaitResult, error) {
	waitBody, errorCh := e.client.ContainerWait(ctx, ID, dockercontainer.WaitConditionNotRunning)
	r := &enginetypes.VirtualizationWaitResult{}
	select {
	case b := <-waitBody:
		if b.Error != nil {
			r.Message = b.Error.Message
		}
		r.Code = b.StatusCode
		return r, nil
	case err := <-errorCh:
		r.Message = err.Error()
		r.Code = -1
		return r, err
	}
}

// VirtualizationUpdateResource update virtualization resource
func (e *Engine) VirtualizationUpdateResource(ctx context.Context, ID string, opts *enginetypes.VirtualizationResource) error {
	if opts.Memory > 0 && opts.Memory < minMemory || opts.Memory < 0 {
		return coretypes.ErrBadMemory
	}
	if opts.VolumeChanged {
		log.Errorf("[VirtualizationUpdateResource] docker engine not support rebinding volume resource: %v", opts.Volumes)
		return coretypes.ErrNotSupport
	}

	memory := opts.Memory
	softLimit := opts.SoftLimit
	// unlimited memory
	if memory == 0 {
		memory = maxMemory
		softLimit = false
	}

	quota := opts.Quota
	cpuMap := opts.CPU
	numaNode := opts.NUMANode
	// unlimited cpu
	if quota == 0 {
		info, err := e.Info(ctx)
		if err != nil {
			return err
		}
		quota = -1
		numaNode = fmt.Sprintf("0-%d", info.NCPU-1)
		cpuMap = map[string]int64{}
		for i := 0; i < info.NCPU; i++ {
			cpuMap[strconv.Itoa(i)] = int64(e.config.Scheduler.ShareBase)
		}
	}

	newResource := makeResourceSetting(quota, memory, cpuMap, numaNode, softLimit)
	updateConfig := dockercontainer.UpdateConfig{Resources: newResource}
	_, err := e.client.ContainerUpdate(ctx, ID, updateConfig)
	return err
}

// VirtualizationCopyFrom copy thing from a virtualization
func (e *Engine) VirtualizationCopyFrom(ctx context.Context, ID, path string) (io.ReadCloser, string, error) {
	resp, stat, err := e.client.CopyFromContainer(ctx, ID, path)
	if err != nil {
		return nil, "", err
	}
	tarReader := tar.NewReader(resp)
	_, err = tarReader.Next()
	return ioutil.NopCloser(tarReader), stat.Name, errors.Wrapf(err, "read tarball from docker API failed: %s", path)
}
<commit_msg>docker engine: numa = "" when quota = 0<commit_after>package docker

import (
	"archive/tar"
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"os"
	"path/filepath"
	"strconv"

	"github.com/docker/go-connections/nat"
	"github.com/docker/go-units"
	"github.com/pkg/errors"

	log "github.com/sirupsen/logrus"

	dockertypes "github.com/docker/docker/api/types"
	dockercontainer "github.com/docker/docker/api/types/container"
	dockernetwork "github.com/docker/docker/api/types/network"
	dockerslice "github.com/docker/docker/api/types/strslice"

	"encoding/json"

	enginetypes "github.com/projecteru2/core/engine/types"
	coretypes "github.com/projecteru2/core/types"
)

const (
	minMemory     = units.MiB * 4
	maxMemory     = math.MaxInt64
	restartAlways = "always"
	root          = "root"
)

type rawArgs struct {
	PidMode    dockercontainer.PidMode `json:"pid_mod"`
	StorageOpt map[string]string       `json:"storage_opt"`
	CapAdd     []string                `json:"cap_add"`
	CapDrop    []string                `json:"cap_drop"`
}

// VirtualizationCreate create a container
func (e *Engine) VirtualizationCreate(ctx context.Context, opts *enginetypes.VirtualizationCreateOptions) (*enginetypes.VirtualizationCreated, error) {
	r := &enginetypes.VirtualizationCreated{}
	// memory should more than 4MiB
	if opts.Memory > 0 && opts.Memory < minMemory || opts.Memory < 0 {
		return r, coretypes.ErrBadMemory
	}
	// set default log driver if lambda
	if opts.Lambda {
		opts.LogType = "json-file"
	}
	// set restart always
	restartRetryCount := 3
	if opts.RestartPolicy == restartAlways {
		restartRetryCount = 0
	}
	// network mode 和 networks 互斥
	// 没有 networks 的时候用 networkmode 的值
	// 有 networks 的时候一律用用 networks 的值作为 mode
	networkMode := dockercontainer.NetworkMode(opts.Network)
	for name := range opts.Networks {
		networkMode = dockercontainer.NetworkMode(name)
		if networkMode.IsHost() {
			opts.Networks[name] = ""
		}
	}
	// 如果没有 network 用默认值替换
	if networkMode == "" {
		networkMode = dockercontainer.NetworkMode(e.config.Docker.NetworkMode)
	}
	// log config
	if opts.LogConfig == nil {
		opts.LogConfig = map[string]string{}
	}
	opts.LogConfig["mode"] = "non-blocking"
	opts.LogConfig["max-buffer-size"] = "4m"
	opts.LogConfig["tag"] = fmt.Sprintf("%s {{.ID}}", opts.Name)
	if opts.Debug {
		opts.LogType = e.config.Docker.Log.Type
		for k, v := range e.config.Docker.Log.Config {
			opts.LogConfig[k] = v
		}
	}
	// add node IP
	hostIP := GetIP(e.client.DaemonHost())
	opts.Env = append(opts.Env, fmt.Sprintf("ERU_NODE_IP=%s", hostIP))
	// 如果有给dns就优先用给定的dns.
	// 没有给出dns的时候, 如果设定是用宿主机IP作为dns, 就会把宿主机IP设置过去.
	// 其他情况就是默认值.
	// 哦对, networkMode如果是host也不给dns.
	if len(opts.DNS) == 0 && e.config.Docker.UseLocalDNS && hostIP != "" {
		opts.DNS = []string{hostIP}
	}
	// mount paths
	binds, volumes := makeMountPaths(opts)
	log.Debugf("[VirtualizationCreate] App %s will bind %v", opts.Name, binds)

	config := &dockercontainer.Config{
		Env:             opts.Env,
		Cmd:             dockerslice.StrSlice(opts.Cmd),
		User:            opts.User,
		Image:           opts.Image,
		Volumes:         volumes,
		WorkingDir:      opts.WorkingDir,
		NetworkDisabled: networkMode == "",
		Labels:          opts.Labels,
		OpenStdin:       opts.Stdin,
		Tty:             opts.Stdin,
	}

	resource := makeResourceSetting(opts.Quota, opts.Memory, opts.CPU, opts.NUMANode, opts.SoftLimit)
	// set ulimits
	resource.Ulimits = []*units.Ulimit{
		&units.Ulimit{Name: "nofile", Soft: 65535, Hard: 65535},
	}
	if networkMode.IsHost() {
		opts.DNS = []string{}
		opts.Sysctl = map[string]string{}
	}
	rArgs := &rawArgs{StorageOpt: map[string]string{}}
	if len(opts.RawArgs) > 0 {
		if err := json.Unmarshal(opts.RawArgs, rArgs); err != nil {
			return r, err
		}
	}
	if opts.Storage > 0 {
		rArgs.StorageOpt["size"] = fmt.Sprintf("%v", opts.Storage)
	}
	// 如果有指定用户，用指定用户
	// 没有指定用户，用镜像自己的
	// CapAdd and Privileged
	capAdds := dockerslice.StrSlice(rArgs.CapAdd)
	if opts.Privileged {
		opts.User = root
		capAdds = append(capAdds, "SYS_ADMIN")
	}
	hostConfig := &dockercontainer.HostConfig{
		Binds: binds,
		DNS:   opts.DNS,
		LogConfig: dockercontainer.LogConfig{
			Type:   opts.LogType,
			Config: opts.LogConfig,
		},
		NetworkMode: networkMode,
		RestartPolicy: dockercontainer.RestartPolicy{
			Name:              opts.RestartPolicy,
			MaximumRetryCount: restartRetryCount,
		},
		CapAdd:     capAdds,
		ExtraHosts: opts.Hosts,
		Privileged: opts.Privileged,
		Resources:  resource,
		Sysctls:    opts.Sysctl,
		PidMode:    rArgs.PidMode,
		StorageOpt: rArgs.StorageOpt,
	}

	if hostConfig.NetworkMode.IsBridge() {
		portMapping := nat.PortMap{}
		exposePorts := nat.PortSet{}
		for _, p := range opts.Publish {
			port, err := nat.NewPort("tcp", p)
			if err != nil {
				return r, err
			}
			exposePorts[port] = struct{}{}
			portMapping[port] = []nat.PortBinding{}
			portMapping[port] = append(portMapping[port], nat.PortBinding{HostPort: p})
		}
		hostConfig.PortBindings = portMapping
		config.ExposedPorts = exposePorts
	}

	networkConfig := &dockernetwork.NetworkingConfig{
		EndpointsConfig: map[string]*dockernetwork.EndpointSettings{},
	}
	for networkID, ipv4 := range opts.Networks {
		endpointSetting, err := e.makeIPV4EndpointSetting(ipv4)
		if err != nil {
			return r, err
		}
		ipForShow := ipv4
		if ipForShow == "" {
			ipForShow = "[AutoAlloc]"
		}
		networkConfig.EndpointsConfig[networkID] = endpointSetting
		log.Infof("[ConnectToNetwork] Connect to %v with IP %v", networkID, ipForShow)
	}

	containerCreated, err := e.client.ContainerCreate(ctx, config, hostConfig, networkConfig, opts.Name)
	r.Name = opts.Name
	r.ID = containerCreated.ID
	return r, err
}

// VirtualizationCopyTo copy things to virtualization
func (e *Engine) VirtualizationCopyTo(ctx context.Context, ID, target string, content io.Reader, AllowOverwriteDirWithFile, CopyUIDGID bool) error {
	return withTarfileDump(target, content, func(target, tarfile string) error {
		content, err := os.Open(tarfile)
		if err != nil {
			return err
		}
		return e.client.CopyToContainer(ctx, ID, filepath.Dir(target), content, dockertypes.CopyToContainerOptions{AllowOverwriteDirWithFile: AllowOverwriteDirWithFile, CopyUIDGID: CopyUIDGID})
	})
}

// VirtualizationStart start virtualization
func (e *Engine) VirtualizationStart(ctx context.Context, ID string) error {
	return e.client.ContainerStart(ctx, ID, dockertypes.ContainerStartOptions{})
}

// VirtualizationStop stop virtualization
func (e *Engine) VirtualizationStop(ctx context.Context, ID string) error {
	return e.client.ContainerStop(ctx, ID, nil)
}

// VirtualizationRemove remove virtualization
func (e *Engine) VirtualizationRemove(ctx context.Context, ID string, removeVolumes, force bool) error {
	return e.client.ContainerRemove(ctx, ID, dockertypes.ContainerRemoveOptions{RemoveVolumes: removeVolumes, Force: force})
}

// VirtualizationInspect get virtualization info
func (e *Engine) VirtualizationInspect(ctx context.Context, ID string) (*enginetypes.VirtualizationInfo, error) {
	if e.client == nil {
		return nil, coretypes.ErrNilEngine
	}

	containerJSON, err := e.client.ContainerInspect(ctx, ID)
	r := &enginetypes.VirtualizationInfo{}
	if err != nil {
		return r, err
	}
	r.ID = containerJSON.ID
	r.User = containerJSON.Config.User
	r.Image = containerJSON.Config.Image
	r.Env = containerJSON.Config.Env
	r.Labels = containerJSON.Config.Labels
	r.Running = containerJSON.State.Running
	r.Networks = map[string]string{}
	for networkName, networkSetting := range containerJSON.NetworkSettings.Networks {
		ip := networkSetting.IPAddress
		if dockercontainer.NetworkMode(networkName).IsHost() {
			ip = GetIP(e.client.DaemonHost())
		}
		r.Networks[networkName] = ip
	}
	return r, nil
}

// VirtualizationLogs show virtualization logs
func (e *Engine) VirtualizationLogs(ctx context.Context, opts *enginetypes.VirtualizationLogStreamOptions) (io.ReadCloser, error) {
	logsOpts := dockertypes.ContainerLogsOptions{Follow: opts.Follow, ShowStdout: opts.Stdout, ShowStderr: opts.Stderr, Tail: opts.Tail}
	resp, err := e.client.ContainerLogs(ctx, opts.ID, logsOpts)
	if err != nil {
		return nil, err
	}
	return ioutil.NopCloser(mergeStream(resp)), nil
}

// VirtualizationAttach attach to a virtualization
func (e *Engine) VirtualizationAttach(ctx context.Context, ID string, stream, stdin bool) (io.ReadCloser, io.WriteCloser, error) {
	opts := dockertypes.ContainerAttachOptions{
		Stream: stream,
		Stdin:  stdin,
		Logs:   true,
		Stdout: true,
		Stderr: true,
	}
	resp, err := e.client.ContainerAttach(ctx, ID, opts)
	if err != nil {
		return nil, nil, err
	}
	return ioutil.NopCloser(resp.Reader), resp.Conn, nil
}

// VirtualizationResize resizes remote terminal
func (e *Engine) VirtualizationResize(ctx context.Context, containerID string, height, width uint) (err error) {
	opts := dockertypes.ResizeOptions{
		Height: height,
		Width:  width,
	}

	return e.client.ContainerResize(ctx, containerID, opts)
}

// VirtualizationWait wait virtualization exit
func (e *Engine) VirtualizationWait(ctx context.Context, ID, state string) (*enginetypes.VirtualizationWaitResult, error) {
	waitBody, errorCh := e.client.ContainerWait(ctx, ID, dockercontainer.WaitConditionNotRunning)
	r := &enginetypes.VirtualizationWaitResult{}
	select {
	case b := <-waitBody:
		if b.Error != nil {
			r.Message = b.Error.Message
		}
		r.Code = b.StatusCode
		return r, nil
	case err := <-errorCh:
		r.Message = err.Error()
		r.Code = -1
		return r, err
	}
}

// VirtualizationUpdateResource update virtualization resource
func (e *Engine) VirtualizationUpdateResource(ctx context.Context, ID string, opts *enginetypes.VirtualizationResource) error {
	if opts.Memory > 0 && opts.Memory < minMemory || opts.Memory < 0 {
		return coretypes.ErrBadMemory
	}
	if opts.VolumeChanged {
		log.Errorf("[VirtualizationUpdateResource] docker engine not support rebinding volume resource: %v", opts.Volumes)
		return coretypes.ErrNotSupport
	}

	memory := opts.Memory
	softLimit := opts.SoftLimit
	// unlimited memory
	if memory == 0 {
		memory = maxMemory
		softLimit = false
	}

	quota := opts.Quota
	cpuMap := opts.CPU
	numaNode := opts.NUMANode
	// unlimited cpu
	if quota == 0 {
		info, err := e.Info(ctx)
		if err != nil {
			return err
		}
		quota = -1
		numaNode = ""
		cpuMap = map[string]int64{}
		for i := 0; i < info.NCPU; i++ {
			cpuMap[strconv.Itoa(i)] = int64(e.config.Scheduler.ShareBase)
		}
	}

	newResource := makeResourceSetting(quota, memory, cpuMap, numaNode, softLimit)
	updateConfig := dockercontainer.UpdateConfig{Resources: newResource}
	_, err := e.client.ContainerUpdate(ctx, ID, updateConfig)
	return err
}

// VirtualizationCopyFrom copy thing from a virtualization
func (e *Engine) VirtualizationCopyFrom(ctx context.Context, ID, path string) (io.ReadCloser, string, error) {
	resp, stat, err := e.client.CopyFromContainer(ctx, ID, path)
	if err != nil {
		return nil, "", err
	}
	tarReader := tar.NewReader(resp)
	_, err = tarReader.Next()
	return ioutil.NopCloser(tarReader), stat.Name, errors.Wrapf(err, "read tarball from docker API failed: %s", path)
}
<|endoftext|>


<commit_before>package lbot

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"log"
	"net/http"
	"strings"
)

func (b *Bot) SetConfig(c Config) {
	b.config = &c
}

func (b *Bot) SendTextMessage(m mid, s string) error {
	if b.config == nil {
		return errors.New("Config have not been set")
	}
	if b.config.Debug {
		log.Println("Start to Set Message")
	}

	var payload Request
	payload.SetDefaults()
	payload.SetText(s)
	payload.AddTargetUser(mid(m))

	out, err := json.Marshal(payload)
	if err != nil {
		return err
	}
	if b.config.Debug {
		log.Println("Output json: " + string(out))
	}

	req, err := http.NewRequest("POST", b.config.ServerHost+"/v1/events", strings.NewReader(string(out)))
	if err != nil {
		return err
	}

	b.addAuthHeader(req)
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}
	result, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}

	if b.config.Debug {
		log.Println("Result: ", string(result))
	}

	return nil
}

func (b *Bot) GetUserProfile(m mid) ([]ProfileInfo, error) {
	if b.config == nil {
		return nil, errors.New("Config have not been set")
	}

	if b.config.Debug {
		log.Println("Start to Set Message")
	}

	req, err := http.NewRequest("GET", b.config.ServerHost+"/v1/profiles?mids="+string(m), nil)
	if err != nil {
		return nil, err
	}

	b.addAuthHeader(req)
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, err
	}
	result, err := ParseProfileResponse(resp)
	if err != nil {
		return nil, err
	}

	if b.config.Debug {
		log.Println("Result: ", string(result))
	}

	return result, nil

}

func (b *Bot) addAuthHeader(r *http.Request) {

	r.Header.Set("Content-type", "application/json; charset=UTF-8")
	r.Header.Set("X-Line-ChannelID", b.config.ChannelID)
	r.Header.Set("X-Line-ChannelSecret", b.config.ChannelSecret)
	r.Header.Set("X-Line-Trusted-User-With-ACL", b.config.MID)

}
<commit_msg>Fix bug<commit_after>package lbot

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"log"
	"net/http"
	"strings"
)

func (b *Bot) SetConfig(c Config) {
	b.config = &c
}

func (b *Bot) SendTextMessage(m mid, s string) error {
	if b.config == nil {
		return errors.New("Config have not been set")
	}
	if b.config.Debug {
		log.Println("Start to Set Message")
	}

	var payload Request
	payload.SetDefaults()
	payload.SetText(s)
	payload.AddTargetUser(mid(m))

	out, err := json.Marshal(payload)
	if err != nil {
		return err
	}
	if b.config.Debug {
		log.Println("Output json: " + string(out))
	}

	req, err := http.NewRequest("POST", b.config.ServerHost+"/v1/events", strings.NewReader(string(out)))
	if err != nil {
		return err
	}

	b.addAuthHeader(req)
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}
	result, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}

	if b.config.Debug {
		log.Println("Result: ", string(result))
	}

	return nil
}

func (b *Bot) GetUserProfile(m mid) ([]ProfileInfo, error) {
	if b.config == nil {
		return nil, errors.New("Config have not been set")
	}

	if b.config.Debug {
		log.Println("Start to Set Message")
	}

	req, err := http.NewRequest("GET", b.config.ServerHost+"/v1/profiles?mids="+string(m), nil)
	if err != nil {
		return nil, err
	}

	b.addAuthHeader(req)
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, err
	}
	result, err := ParseProfileResponse(resp)
	if err != nil {
		return nil, err
	}

	return result.Contacts, nil

}

func (b *Bot) addAuthHeader(r *http.Request) {

	r.Header.Set("Content-type", "application/json; charset=UTF-8")
	r.Header.Set("X-Line-ChannelID", b.config.ChannelID)
	r.Header.Set("X-Line-ChannelSecret", b.config.ChannelSecret)
	r.Header.Set("X-Line-Trusted-User-With-ACL", b.config.MID)

}
<|endoftext|>


<commit_before>package bugsnag

import (
	"fmt"

	"github.com/bugsnag/bugsnag-go/errors"
)

// Notifier sends errors to Bugsnag.
type Notifier struct {
	Config  *Configuration
	RawData []interface{}
}

// New creates a new notifier.
// You can pass an instance of bugsnag.Configuration in rawData to change the configuration.
// Other values of rawData will be passed to Notify.
func New(rawData ...interface{}) *Notifier {
	config := Config.clone()
	for i, datum := range rawData {
		if c, ok := datum.(Configuration); ok {
			config.update(&c)
			rawData[i] = nil
		}
	}

	return &Notifier{
		Config:  config,
		RawData: rawData,
	}
}

// Notify sends an error to Bugsnag. Any rawData you pass here will be sent to
// Bugsnag after being converted to JSON. e.g. bugsnag.SeverityError, bugsnag.Context,
// or bugsnag.MetaData.
func (notifier *Notifier) Notify(rawData ...interface{}) (e error) {
	return notifier.NotifySync(append(rawData, notifier.Config.Synchronous)...)
}

// NotifySync sends an error to Bugsnag. The synchronous parameter specifies
// whether to send the report in the current context. Any rawData you pass here
// will be sent to Bugsnag after being converted to JSON. e.g.
// bugsnag.SeverityError,  bugsnag.Context, or bugsnag.MetaData.
func (notifier *Notifier) NotifySync(rawData ...interface{}) (e error) {
	event, config := newEvent(rawData, notifier)

	// Never block, start throwing away errors if we have too many.
	e = middleware.Run(event, config, func() error {
		config.logf("notifying bugsnag: %s", event.Message)
		if config.notifyInReleaseStage() {
			if config.Synchronous {
				return (&payload{event, config}).deliver()
			}
			// Ensure that any errors are logged if they occur in a goroutine.
			go func(event *Event, config *Configuration) {
				err := (&payload{event, config}).deliver()
				if err != nil {
					config.logf("bugsnag.Notify: %v", err)
				}
			}(event, config)

			return nil
		}
		return fmt.Errorf("not notifying in %s", config.ReleaseStage)
	})

	if e != nil {
		config.logf("bugsnag.Notify: %v", e)
	}
	return e
}

// AutoNotify notifies Bugsnag of any panics, then repanics.
// It sends along any rawData that gets passed in.
// Usage:
//  go func() {
//		defer AutoNotify()
//      // (possibly crashy code)
//  }()
func (notifier *Notifier) AutoNotify(rawData ...interface{}) {
	if err := recover(); err != nil {
		severity := notifier.getDefaultSeverity(rawData, SeverityError)
		state := HandledState{SeverityReasonHandledPanic, severity, true, ""}
		notifier.appendStateIfNeeded(rawData, state)
		notifier.Notify(append(rawData, errors.New(err, 2))...)
		panic(err)
	}
}

// Recover logs any panics, then recovers.
// It sends along any rawData that gets passed in.
// Usage: defer Recover()
func (notifier *Notifier) Recover(rawData ...interface{}) {
	if err := recover(); err != nil {
		severity := notifier.getDefaultSeverity(rawData, SeverityWarning)
		state := HandledState{SeverityReasonHandledPanic, severity, false, ""}
		notifier.appendStateIfNeeded(rawData, state)
		notifier.Notify(append(rawData, errors.New(err, 2))...)
	}
}

func (notifier *Notifier) dontPanic() {
	if err := recover(); err != nil {
		notifier.Config.logf("bugsnag/notifier.Notify: panic! %s", err)
	}
}

// Get defined severity from raw data or a fallback value
func (notifier *Notifier) getDefaultSeverity(rawData []interface{}, s severity) severity {
	allData := append(notifier.RawData, rawData...)
	for _, datum := range allData {
		if _, ok := datum.(severity); ok {
			return datum.(severity)
		}
	}

	for _, datum := range allData {
		if _, ok := datum.(HandledState); ok {
			return datum.(HandledState).OriginalSeverity
		}
	}

	return s
}

func (notifier *Notifier) appendStateIfNeeded(rawData []interface{}, h HandledState) []interface{} {

	for _, datum := range append(notifier.RawData, rawData...) {
		if _, ok := datum.(HandledState); ok {
			return rawData
		}
	}

	return append(rawData, h)
}
<commit_msg>[fix] Ensure rawdata state gets appended<commit_after>package bugsnag

import (
	"fmt"

	"github.com/bugsnag/bugsnag-go/errors"
)

// Notifier sends errors to Bugsnag.
type Notifier struct {
	Config  *Configuration
	RawData []interface{}
}

// New creates a new notifier.
// You can pass an instance of bugsnag.Configuration in rawData to change the configuration.
// Other values of rawData will be passed to Notify.
func New(rawData ...interface{}) *Notifier {
	config := Config.clone()
	for i, datum := range rawData {
		if c, ok := datum.(Configuration); ok {
			config.update(&c)
			rawData[i] = nil
		}
	}

	return &Notifier{
		Config:  config,
		RawData: rawData,
	}
}

// Notify sends an error to Bugsnag. Any rawData you pass here will be sent to
// Bugsnag after being converted to JSON. e.g. bugsnag.SeverityError, bugsnag.Context,
// or bugsnag.MetaData.
func (notifier *Notifier) Notify(rawData ...interface{}) (e error) {
	return notifier.NotifySync(append(rawData, notifier.Config.Synchronous)...)
}

// NotifySync sends an error to Bugsnag. The synchronous parameter specifies
// whether to send the report in the current context. Any rawData you pass here
// will be sent to Bugsnag after being converted to JSON. e.g.
// bugsnag.SeverityError,  bugsnag.Context, or bugsnag.MetaData.
func (notifier *Notifier) NotifySync(rawData ...interface{}) (e error) {
	event, config := newEvent(rawData, notifier)

	// Never block, start throwing away errors if we have too many.
	e = middleware.Run(event, config, func() error {
		config.logf("notifying bugsnag: %s", event.Message)
		if config.notifyInReleaseStage() {
			if config.Synchronous {
				return (&payload{event, config}).deliver()
			}
			// Ensure that any errors are logged if they occur in a goroutine.
			go func(event *Event, config *Configuration) {
				err := (&payload{event, config}).deliver()
				if err != nil {
					config.logf("bugsnag.Notify: %v", err)
				}
			}(event, config)

			return nil
		}
		return fmt.Errorf("not notifying in %s", config.ReleaseStage)
	})

	if e != nil {
		config.logf("bugsnag.Notify: %v", e)
	}
	return e
}

// AutoNotify notifies Bugsnag of any panics, then repanics.
// It sends along any rawData that gets passed in.
// Usage:
//  go func() {
//		defer AutoNotify()
//      // (possibly crashy code)
//  }()
func (notifier *Notifier) AutoNotify(rawData ...interface{}) {
	if err := recover(); err != nil {
		severity := notifier.getDefaultSeverity(rawData, SeverityError)
		state := HandledState{SeverityReasonHandledPanic, severity, true, ""}
		rawData = notifier.appendStateIfNeeded(rawData, state)
		notifier.Notify(append(rawData, errors.New(err, 2))...)
		panic(err)
	}
}

// Recover logs any panics, then recovers.
// It sends along any rawData that gets passed in.
// Usage: defer Recover()
func (notifier *Notifier) Recover(rawData ...interface{}) {
	if err := recover(); err != nil {
		severity := notifier.getDefaultSeverity(rawData, SeverityWarning)
		state := HandledState{SeverityReasonHandledPanic, severity, false, ""}
		rawData = notifier.appendStateIfNeeded(rawData, state)
		notifier.Notify(append(rawData, errors.New(err, 2))...)
	}
}

func (notifier *Notifier) dontPanic() {
	if err := recover(); err != nil {
		notifier.Config.logf("bugsnag/notifier.Notify: panic! %s", err)
	}
}

// Get defined severity from raw data or a fallback value
func (notifier *Notifier) getDefaultSeverity(rawData []interface{}, s severity) severity {
	allData := append(notifier.RawData, rawData...)
	for _, datum := range allData {
		if _, ok := datum.(severity); ok {
			return datum.(severity)
		}
	}

	for _, datum := range allData {
		if _, ok := datum.(HandledState); ok {
			return datum.(HandledState).OriginalSeverity
		}
	}

	return s
}

func (notifier *Notifier) appendStateIfNeeded(rawData []interface{}, h HandledState) []interface{} {

	for _, datum := range append(notifier.RawData, rawData...) {
		if _, ok := datum.(HandledState); ok {
			return rawData
		}
	}

	return append(rawData, h)
}
<|endoftext|>


<commit_before>package etcdserver

import (
	"reflect"
	"testing"
	"time"
	"code.google.com/p/go.net/context"

	pb "github.com/coreos/etcd/etcdserver2/etcdserverpb"
	"github.com/coreos/etcd/raft"
	"github.com/coreos/etcd/raft/raftpb"
	"github.com/coreos/etcd/store"
)

func TestClusterOf1(t *testing.T) { testServer(t, 1) }
func TestClusterOf3(t *testing.T) { testServer(t, 3) }

func testServer(t *testing.T, ns int64) {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	ss := make([]*Server, ns)

	send := func(msgs []raftpb.Message) {
		var m raftpb.Message
		for len(msgs) > 0 {
			m, msgs = msgs[0], msgs[1:]
			t.Logf("sending: %+v", m)
			if err := ss[m.To].Node.Step(ctx, m); err != nil {
				t.Fatal(err)
			}
			rd := raft.RecvReadyNow(ss[m.To].Node)
			msgs = append(msgs, rd.Messages...)
		}
	}

	peers := make([]int64, ns)
	for i := int64(0); i < ns; i++ {
		peers[i] = i
	}

	var srv *Server
	for i := int64(0); i < ns; i++ {
		n := raft.Start(ctx, i, peers)

		srv = &Server{
			Node:  n,
			Store: store.New(),
			Send:  send,
			Save:  func(_ raftpb.State, _ []raftpb.Entry) {},
		}
		Start(srv)

		ss[i] = srv
	}

	if err := srv.Node.Campaign(ctx); err != nil {
		t.Fatal(err)
	}

	r := pb.Request{
		Method: "PUT",
		Id:     1,
		Path:   "/foo",
		Val:    "bar",
	}
	resp, err := srv.Do(ctx, r)
	if err != nil {
		t.Fatal(err)
	}

	g, w := resp.Event.Node, &store.NodeExtern{
		Key:           "/foo",
		ModifiedIndex: 1,
		CreatedIndex:  1,
		Value:         stringp("bar"),
	}

	if !reflect.DeepEqual(g, w) {
		t.Error("value:", *g.Value)
		t.Errorf("g = %+v, w %+v", g, w)
	}

	time.Sleep(10 * time.Millisecond)

	var last interface{}
	for i, sv := range ss {
		sv.Stop()
		g := store.Root(sv.Store)
		if last != nil && !reflect.DeepEqual(last, g) {
			t.Errorf("server %d: Root = %#v, want %#v", i, g, last)
		}
		last = g
	}
}

func stringp(s string) *string { return &s }
<commit_msg>etcdserver: working test<commit_after>package etcdserver

import (
	"fmt"
	"reflect"
	"testing"
	"time"
	"code.google.com/p/go.net/context"

	pb "github.com/coreos/etcd/etcdserver2/etcdserverpb"
	"github.com/coreos/etcd/raft"
	"github.com/coreos/etcd/raft/raftpb"
	"github.com/coreos/etcd/store"
)

func TestClusterOf1(t *testing.T) { testServer(t, 1) }
func TestClusterOf3(t *testing.T) { testServer(t, 3) }

func testServer(t *testing.T, ns int64) {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	ss := make([]*Server, ns)

	send := func(msgs []raftpb.Message) {
		for _, m := range msgs {
			fmt.Printf("sending: %+v\n", m)
			ss[m.To].Node.Step(ctx, m)
		}
	}

	peers := make([]int64, ns)
	for i := int64(0); i < ns; i++ {
		peers[i] = i
	}

	var srv *Server
	for i := int64(0); i < ns; i++ {
		n := raft.Start(ctx, i, peers)

		srv = &Server{
			Node:  n,
			Store: store.New(),
			Send:  send,
			Save:  func(_ raftpb.State, _ []raftpb.Entry) {},
		}
		Start(srv)

		ss[i] = srv
	}

	if err := srv.Node.Campaign(ctx); err != nil {
		t.Fatal(err)
	}

	r := pb.Request{
		Method: "PUT",
		Id:     1,
		Path:   "/foo",
		Val:    "bar",
	}
	resp, err := srv.Do(ctx, r)
	if err != nil {
		t.Fatal(err)
	}

	g, w := resp.Event.Node, &store.NodeExtern{
		Key:           "/foo",
		ModifiedIndex: 1,
		CreatedIndex:  1,
		Value:         stringp("bar"),
	}

	if !reflect.DeepEqual(g, w) {
		t.Error("value:", *g.Value)
		t.Errorf("g = %+v, w %+v", g, w)
	}

	time.Sleep(10 * time.Millisecond)

	var last interface{}
	for i, sv := range ss {
		sv.Stop()
		g := store.Root(sv.Store)
		if last != nil && !reflect.DeepEqual(last, g) {
			t.Errorf("server %d: Root = %#v, want %#v", i, g, last)
		}
		last = g
	}
}

func stringp(s string) *string { return &s }
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"github.com/ethereum/eth-go/ethchain"
	"github.com/ethereum/eth-go/ethpub"
	"github.com/ethereum/eth-go/ethutil"
	"github.com/go-qml/qml"
	"github.com/howeyc/fsnotify"
	"io/ioutil"
	"log"
	"net/url"
	"os"
	"path"
	"path/filepath"
)

type HtmlApplication struct {
	win     *qml.Window
	webView qml.Object
	engine  *qml.Engine
	lib     *UiLib
	path    string
	watcher *fsnotify.Watcher
}

func NewHtmlApplication(path string, lib *UiLib) *HtmlApplication {
	engine := qml.NewEngine()

	return &HtmlApplication{engine: engine, lib: lib, path: path}

}

func (app *HtmlApplication) Create() error {
	component, err := app.engine.LoadFile(app.lib.AssetPath("qml/webapp.qml"))
	if err != nil {
		return err
	}

	if filepath.Ext(app.path) == "eth" {
		return errors.New("Ethereum package not yet supported")

		// TODO
		ethutil.OpenPackage(app.path)
	}

	win := component.CreateWindow(nil)
	win.Set("url", app.path)
	webView := win.ObjectByName("webView")

	app.win = win
	app.webView = webView

	return nil
}

func (app *HtmlApplication) RootFolder() string {
	folder, err := url.Parse(app.path)
	if err != nil {
		return ""
	}
	return path.Dir(folder.RequestURI())
}
func (app *HtmlApplication) RecursiveFolders() []os.FileInfo {
	files, _ := ioutil.ReadDir(app.RootFolder())
	var folders []os.FileInfo
	for _, file := range files {
		if file.IsDir() {
			folders = append(folders, file)
		}
	}
	return folders
}

func (app *HtmlApplication) NewWatcher(quitChan chan bool) {
	var err error

	app.watcher, err = fsnotify.NewWatcher()
	if err != nil {
		return
	}
	err = app.watcher.Watch(app.RootFolder())
	if err != nil {
		log.Fatal(err)
	}
	for _, folder := range app.RecursiveFolders() {
		fullPath := app.RootFolder() + "/" + folder.Name()
		app.watcher.Watch(fullPath)
	}

	go func() {
	out:
		for {
			select {
			case <-quitChan:
				app.watcher.Close()
				break out
			case <-app.watcher.Event:
				//logger.Debugln("Got event:", ev)
				app.webView.Call("reload")
			case err := <-app.watcher.Error:
				// TODO: Do something here
				logger.Infoln("Watcher error:", err)
			}
		}
	}()

}

func (app *HtmlApplication) Engine() *qml.Engine {
	return app.engine
}

func (app *HtmlApplication) Window() *qml.Window {
	return app.win
}

func (app *HtmlApplication) NewBlock(block *ethchain.Block) {
	b := &ethpub.PBlock{Number: int(block.BlockInfo().Number), Hash: ethutil.Bytes2Hex(block.Hash())}
	app.webView.Call("onNewBlockCb", b)
}

func (app *HtmlApplication) ObjectChanged(stateObject *ethchain.StateObject) {
	app.webView.Call("onObjectChangeCb", ethpub.NewPStateObject(stateObject))
}

func (app *HtmlApplication) StorageChanged(storageObject *ethchain.StorageState) {
	app.webView.Call("onStorageChangeCb", ethpub.NewPStorageState(storageObject))
}

func (app *HtmlApplication) Destroy() {
	app.engine.Destroy()
}
<commit_msg>Make the reload watcher use windows-safe paths<commit_after>package main

import (
	"errors"
	"github.com/ethereum/eth-go/ethchain"
	"github.com/ethereum/eth-go/ethpub"
	"github.com/ethereum/eth-go/ethutil"
	"github.com/go-qml/qml"
	"github.com/howeyc/fsnotify"
	"io/ioutil"
	"log"
	"net/url"
	"os"
	"path"
	"path/filepath"
)

type HtmlApplication struct {
	win     *qml.Window
	webView qml.Object
	engine  *qml.Engine
	lib     *UiLib
	path    string
	watcher *fsnotify.Watcher
}

func NewHtmlApplication(path string, lib *UiLib) *HtmlApplication {
	engine := qml.NewEngine()

	return &HtmlApplication{engine: engine, lib: lib, path: path}

}

func (app *HtmlApplication) Create() error {
	component, err := app.engine.LoadFile(app.lib.AssetPath("qml/webapp.qml"))
	if err != nil {
		return err
	}

	if filepath.Ext(app.path) == "eth" {
		return errors.New("Ethereum package not yet supported")

		// TODO
		ethutil.OpenPackage(app.path)
	}

	win := component.CreateWindow(nil)
	win.Set("url", app.path)
	webView := win.ObjectByName("webView")

	app.win = win
	app.webView = webView

	return nil
}

func (app *HtmlApplication) RootFolder() string {
	folder, err := url.Parse(app.path)
	if err != nil {
		return ""
	}
	return path.Dir(ethutil.WindonizePath(folder.RequestURI()))
}
func (app *HtmlApplication) RecursiveFolders() []os.FileInfo {
	files, _ := ioutil.ReadDir(app.RootFolder())
	var folders []os.FileInfo
	for _, file := range files {
		if file.IsDir() {
			folders = append(folders, file)
		}
	}
	return folders
}

func (app *HtmlApplication) NewWatcher(quitChan chan bool) {
	var err error

	app.watcher, err = fsnotify.NewWatcher()
	if err != nil {
		return
	}
	err = app.watcher.Watch(app.RootFolder())
	if err != nil {
		log.Fatal(err)
	}
	for _, folder := range app.RecursiveFolders() {
		fullPath := app.RootFolder() + "/" + folder.Name()
		app.watcher.Watch(fullPath)
	}

	go func() {
	out:
		for {
			select {
			case <-quitChan:
				app.watcher.Close()
				break out
			case <-app.watcher.Event:
				//logger.Debugln("Got event:", ev)
				app.webView.Call("reload")
			case err := <-app.watcher.Error:
				// TODO: Do something here
				logger.Infoln("Watcher error:", err)
			}
		}
	}()

}

func (app *HtmlApplication) Engine() *qml.Engine {
	return app.engine
}

func (app *HtmlApplication) Window() *qml.Window {
	return app.win
}

func (app *HtmlApplication) NewBlock(block *ethchain.Block) {
	b := &ethpub.PBlock{Number: int(block.BlockInfo().Number), Hash: ethutil.Bytes2Hex(block.Hash())}
	app.webView.Call("onNewBlockCb", b)
}

func (app *HtmlApplication) ObjectChanged(stateObject *ethchain.StateObject) {
	app.webView.Call("onObjectChangeCb", ethpub.NewPStateObject(stateObject))
}

func (app *HtmlApplication) StorageChanged(storageObject *ethchain.StorageState) {
	app.webView.Call("onStorageChangeCb", ethpub.NewPStorageState(storageObject))
}

func (app *HtmlApplication) Destroy() {
	app.engine.Destroy()
}
<|endoftext|>


<commit_before>package consensus

import (
	"errors"

	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/encoding"
)

var (
	ErrMissingSignatures = errors.New("transaction has inputs with missing signatures")
)

// Each input has a list of public keys and a required number of signatures.
// inputSignatures keeps track of which public keys have been used and how many
// more signatures are needed.
type inputSignatures struct {
	remainingSignatures uint64
	possibleKeys        []SiaPublicKey
	usedKeys            map[uint64]struct{}
	index               int
}

// sortedUnique checks that 'elems' is sorted, contains no repeats, and that no
// element is an invalid index of 'elems'.
func sortedUnique(elems []uint64) bool {
	if len(elems) == 0 {
		return true
	}

	biggest := elems[0]
	for _, elem := range elems[1:] {
		if elem <= biggest {
			return false
		}
	}
	if biggest >= uint64(len(elems)) {
		return false
	}
	return true
}

// validCoveredFields makes sure that all covered fields objects in the
// signatures follow the rules. This means that if 'WholeTransaction' is set to
// true, all fields except for 'Signatures' must be empty. All fields must be
// sorted numerically, and there can be no repeats.
func (t Transaction) validCoveredFields() error {
	for _, sig := range t.Signatures {
		// convenience variables
		cf := sig.CoveredFields
		fields := [][]uint64{
			cf.SiacoinInputs,
			cf.MinerFees,
			cf.FileContracts,
			cf.FileContractTerminations,
			cf.StorageProofs,
			cf.SiafundInputs,
			cf.SiafundOutputs,
			cf.ArbitraryData,
			cf.Signatures,
		}

		// Check that all fields are empty if 'WholeTransaction' is set.
		if cf.WholeTransaction {
			// 'WholeTransaction' does not check signatures.
			for _, field := range fields[:len(fields)-1] {
				if len(field) != 0 {
					return errors.New("whole transaction flag is set, but not all fields besides signatures are empty")
				}
			}
		}

		// Check that all fields are sorted, and without repeat values, and
		// that all elements point to objects that exists within the
		// transaction.
		for _, field := range fields {
			if !sortedUnique(field) {
				return errors.New("field does not satisfy 'sorted and unique' requirement")
			}
		}
	}

	return nil
}

// validSignatures checks the validaty of all signatures in a transaction.
func (s *State) validSignatures(t Transaction) error {
	// Check that all covered fields objects follow the rules.
	err := t.validCoveredFields()
	if err != nil {
		return err
	}

	// Create the inputSignatures object for each input.
	sigMap := make(map[crypto.Hash]*inputSignatures)
	for i, input := range t.SiacoinInputs {
		id := crypto.Hash(input.ParentID)
		_, exists := sigMap[id]
		if exists {
			return errors.New("siacoin output spent twice in the same transaction")
		}

		sigMap[id] = &inputSignatures{
			remainingSignatures: input.UnlockConditions.NumSignatures,
			possibleKeys:        input.UnlockConditions.PublicKeys,
			index:               i,
		}
	}
	for i, termination := range t.FileContractTerminations {
		id := crypto.Hash(termination.ParentID)
		_, exists := sigMap[id]
		if exists {
			return errors.New("file contract terminated twice in the same transaction")
		}

		sigMap[id] = &inputSignatures{
			remainingSignatures: termination.TerminationConditions.NumSignatures,
			possibleKeys:        termination.TerminationConditions.PublicKeys,
			index:               i,
		}
	}
	for i, input := range t.SiafundInputs {
		id := crypto.Hash(input.ParentID)
		_, exists := sigMap[id]
		if exists {
			return errors.New("siafund output spent twice in the same transaction")
		}

		sigMap[id] = &inputSignatures{
			remainingSignatures: input.UnlockConditions.NumSignatures,
			possibleKeys:        input.UnlockConditions.PublicKeys,
			index:               i,
		}
	}

	// Check all of the signatures for validity.
	for i, sig := range t.Signatures {
		// check that sig corresponds to an entry in sigMap
		inSig, exists := sigMap[crypto.Hash(sig.ParentID)]
		if !exists || inSig.remainingSignatures == 0 {
			return errors.New("frivolous signature in transaction")
		}
		// check that sig's key hasn't already been used
		_, exists = inSig.usedKeys[sig.PublicKeyIndex]
		if exists {
			return errors.New("one public key was used twice while signing an input")
		}
		// Check that the timelock has expired.
		if sig.Timelock > s.height() {
			return errors.New("signature used before timelock expiration")
		}

		// Check that the signature verifies. Multiple signature schemes are
		// supported.
		publicKey := inSig.possibleKeys[sig.PublicKeyIndex]
		switch publicKey.Algorithm {
		case SignatureEntropy:
			return crypto.ErrInvalidSignature

		case SignatureEd25519:
			// Decode the public key and signature.
			var edPK crypto.PublicKey
			err := encoding.Unmarshal([]byte(publicKey.Key), &edPK)
			if err != nil {
				return err
			}
			var edSig [crypto.SignatureSize]byte
			err = encoding.Unmarshal([]byte(sig.Signature), &edSig)
			if err != nil {
				return err
			}
			cryptoSig := crypto.Signature(&edSig)

			sigHash := t.SigHash(i)
			err = crypto.VerifyHash(sigHash, edPK, cryptoSig)
			if err != nil {
				return err
			}

		default:
			// If we don't recognize the identifier, assume that the signature
			// is valid. This allows more signature types to be added via soft
			// forking.
		}

		inSig.remainingSignatures--
	}

	// Check that all inputs have been sufficiently signed.
	for _, reqSigs := range sigMap {
		if reqSigs.remainingSignatures != 0 {
			return ErrMissingSignatures
		}
	}

	return nil
}
<commit_msg>fix validCoveredFields<commit_after>package consensus

import (
	"errors"

	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/encoding"
)

var (
	ErrMissingSignatures = errors.New("transaction has inputs with missing signatures")
)

// Each input has a list of public keys and a required number of signatures.
// inputSignatures keeps track of which public keys have been used and how many
// more signatures are needed.
type inputSignatures struct {
	remainingSignatures uint64
	possibleKeys        []SiaPublicKey
	usedKeys            map[uint64]struct{}
	index               int
}

// sortedUnique checks that 'elems' is sorted, contains no repeats, and that no
// element is larger than or equal to 'max'.
func sortedUnique(elems []uint64, max int) bool {
	if len(elems) == 0 {
		return true
	}

	biggest := elems[0]
	for _, elem := range elems[1:] {
		if elem <= biggest {
			return false
		}
	}
	if biggest >= uint64(max) {
		return false
	}
	return true
}

// validCoveredFields makes sure that all covered fields objects in the
// signatures follow the rules. This means that if 'WholeTransaction' is set to
// true, all fields except for 'Signatures' must be empty. All fields must be
// sorted numerically, and there can be no repeats.
func (t Transaction) validCoveredFields() error {
	for _, sig := range t.Signatures {
		// convenience variables
		cf := sig.CoveredFields
		fieldMaxs := []struct {
			field []uint64
			max   int
		}{
			{cf.SiacoinInputs, len(t.SiacoinInputs)},
			{cf.MinerFees, len(t.MinerFees)},
			{cf.FileContracts, len(t.FileContracts)},
			{cf.FileContractTerminations, len(t.FileContractTerminations)},
			{cf.StorageProofs, len(t.StorageProofs)},
			{cf.SiafundInputs, len(t.SiafundInputs)},
			{cf.SiafundOutputs, len(t.SiafundOutputs)},
			{cf.ArbitraryData, len(t.ArbitraryData)},
			{cf.Signatures, len(t.Signatures)},
		}

		// Check that all fields are empty if 'WholeTransaction' is set.
		if cf.WholeTransaction {
			// 'WholeTransaction' does not check signatures.
			for _, fieldMax := range fieldMaxs[:len(fieldMaxs)-1] {
				if len(fieldMax.field) != 0 {
					return errors.New("whole transaction flag is set, but not all fields besides signatures are empty")
				}
			}
		}

		// Check that all fields are sorted, and without repeat values, and
		// that all elements point to objects that exists within the
		// transaction.
		for _, fieldMax := range fieldMaxs {
			if !sortedUnique(fieldMax.field, fieldMax.max) {
				return errors.New("field does not satisfy 'sorted and unique' requirement")
			}
		}
	}

	return nil
}

// validSignatures checks the validaty of all signatures in a transaction.
func (s *State) validSignatures(t Transaction) error {
	// Check that all covered fields objects follow the rules.
	err := t.validCoveredFields()
	if err != nil {
		return err
	}

	// Create the inputSignatures object for each input.
	sigMap := make(map[crypto.Hash]*inputSignatures)
	for i, input := range t.SiacoinInputs {
		id := crypto.Hash(input.ParentID)
		_, exists := sigMap[id]
		if exists {
			return errors.New("siacoin output spent twice in the same transaction")
		}

		sigMap[id] = &inputSignatures{
			remainingSignatures: input.UnlockConditions.NumSignatures,
			possibleKeys:        input.UnlockConditions.PublicKeys,
			index:               i,
		}
	}
	for i, termination := range t.FileContractTerminations {
		id := crypto.Hash(termination.ParentID)
		_, exists := sigMap[id]
		if exists {
			return errors.New("file contract terminated twice in the same transaction")
		}

		sigMap[id] = &inputSignatures{
			remainingSignatures: termination.TerminationConditions.NumSignatures,
			possibleKeys:        termination.TerminationConditions.PublicKeys,
			index:               i,
		}
	}
	for i, input := range t.SiafundInputs {
		id := crypto.Hash(input.ParentID)
		_, exists := sigMap[id]
		if exists {
			return errors.New("siafund output spent twice in the same transaction")
		}

		sigMap[id] = &inputSignatures{
			remainingSignatures: input.UnlockConditions.NumSignatures,
			possibleKeys:        input.UnlockConditions.PublicKeys,
			index:               i,
		}
	}

	// Check all of the signatures for validity.
	for i, sig := range t.Signatures {
		// check that sig corresponds to an entry in sigMap
		inSig, exists := sigMap[crypto.Hash(sig.ParentID)]
		if !exists || inSig.remainingSignatures == 0 {
			return errors.New("frivolous signature in transaction")
		}
		// check that sig's key hasn't already been used
		_, exists = inSig.usedKeys[sig.PublicKeyIndex]
		if exists {
			return errors.New("one public key was used twice while signing an input")
		}
		// Check that the timelock has expired.
		if sig.Timelock > s.height() {
			return errors.New("signature used before timelock expiration")
		}

		// Check that the signature verifies. Multiple signature schemes are
		// supported.
		publicKey := inSig.possibleKeys[sig.PublicKeyIndex]
		switch publicKey.Algorithm {
		case SignatureEntropy:
			return crypto.ErrInvalidSignature

		case SignatureEd25519:
			// Decode the public key and signature.
			var edPK crypto.PublicKey
			err := encoding.Unmarshal([]byte(publicKey.Key), &edPK)
			if err != nil {
				return err
			}
			var edSig [crypto.SignatureSize]byte
			err = encoding.Unmarshal([]byte(sig.Signature), &edSig)
			if err != nil {
				return err
			}
			cryptoSig := crypto.Signature(&edSig)

			sigHash := t.SigHash(i)
			err = crypto.VerifyHash(sigHash, edPK, cryptoSig)
			if err != nil {
				return err
			}

		default:
			// If we don't recognize the identifier, assume that the signature
			// is valid. This allows more signature types to be added via soft
			// forking.
		}

		inSig.remainingSignatures--
	}

	// Check that all inputs have been sufficiently signed.
	for _, reqSigs := range sigMap {
		if reqSigs.remainingSignatures != 0 {
			return ErrMissingSignatures
		}
	}

	return nil
}
<|endoftext|>