<commit_before>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/resource"
	"k8s.io/kubernetes/pkg/apis/experimental"

	. "github.com/onsi/ginkgo"
)

const (
	kind        = "replicationController"
	subresource = "scale"
)

var _ = Describe("Horizontal pod autoscaling", func() {
	var rc *ResourceConsumer
	f := NewFramework("horizontal-pod-autoscaling")

	// CPU tests
	It("[Skipped][Autoscaling Suite] should scale from 1 pod to 3 pods (scale resource: CPU)", func() {
		rc = NewDynamicResourceConsumer("rc", 1, 700, 0, 800, 100, f)
		defer rc.CleanUp()
		createCPUHorizontalPodAutoscaler(rc, "0.3")
		rc.WaitForReplicas(3)
	})

	It("[Skipped][Horizontal pod autoscaling Suite] should scale from 3 pods to 1 pod (scale resource: CPU)", func() {
		rc = NewDynamicResourceConsumer("rc", 3, 0, 0, 100, 100, f)
		defer rc.CleanUp()
		createCPUHorizontalPodAutoscaler(rc, "0.7")
		rc.WaitForReplicas(1)
	})

	It("[Skipped][Horizontal pod autoscaling Suite] should scale from 1 pod to maximum 5 pods (scale resource: CPU)", func() {
		rc = NewDynamicResourceConsumer("rc", 1, 700, 0, 800, 100, f)
		defer rc.CleanUp()
		createCPUHorizontalPodAutoscaler(rc, "0.1")
		rc.WaitForReplicas(5)
	})

	It("[Skipped][Horizontal pod autoscaling Suite] should scale from 1 pod to 3 pods and from 3 to 1 (scale resource: CPU)", func() {
		rc = NewDynamicResourceConsumer("rc", 1, 700, 0, 800, 100, f)
		defer rc.CleanUp()
		createCPUHorizontalPodAutoscaler(rc, "0.3")
		rc.WaitForReplicas(3)
		rc.ConsumeCPU(300)
		rc.WaitForReplicas(1)
	})

	It("[Skipped][Horizontal pod autoscaling Suite] should scale from 1 pod to 3 pods and from 3 to 5 (scale resource: CPU)", func() {
		rc = NewDynamicResourceConsumer("rc", 1, 250, 0, 400, 100, f)
		defer rc.CleanUp()
		createCPUHorizontalPodAutoscaler(rc, "0.1")
		rc.WaitForReplicas(3)
		rc.ConsumeCPU(700)
		rc.WaitForReplicas(5)
	})

	It("[Skipped][Horizontal pod autoscaling Suite] should scale from 3 pods to 1 pod and from 1 to 3 (scale resource: CPU)", func() {
		rc = NewDynamicResourceConsumer("rc", 3, 0, 0, 800, 100, f)
		defer rc.CleanUp()
		createCPUHorizontalPodAutoscaler(rc, "0.3")
		rc.WaitForReplicas(1)
		rc.ConsumeCPU(700)
		rc.WaitForReplicas(3)
	})

	It("[Skipped][Horizontal pod autoscaling Suite] should scale from 5 pods to 3 pods and from 3 to 1 (scale resource: CPU)", func() {
		rc = NewDynamicResourceConsumer("rc", 5, 700, 0, 200, 100, f)
		defer rc.CleanUp()
		createCPUHorizontalPodAutoscaler(rc, "0.3")
		rc.WaitForReplicas(3)
		rc.ConsumeCPU(100)
		rc.WaitForReplicas(1)
	})

	// Memory tests
	It("[Skipped][Horizontal pod autoscaling Suite] should scale from 1 pod to 3 pods (scale resource: Memory)", func() {
		rc = NewDynamicResourceConsumer("rc", 1, 0, 800, 100, 900, f)
		defer rc.CleanUp()
		createMemoryHorizontalPodAutoscaler(rc, "300")
		rc.WaitForReplicas(3)
	})

	It("[Skipped][Horizontal pod autoscaling Suite] should scale from 3 pods to 1 pod (scale resource: Memory)", func() {
		rc = NewDynamicResourceConsumer("rc", 3, 0, 0, 100, 100, f)
		defer rc.CleanUp()
		createMemoryHorizontalPodAutoscaler(rc, "700")
		rc.WaitForReplicas(1)
	})

	It("[Skipped][Horizontal pod autoscaling Suite] should scale from 1 pod to maximum 5 pods (scale resource: Memory)", func() {
		rc = NewDynamicResourceConsumer("rc", 1, 0, 700, 100, 800, f)
		defer rc.CleanUp()
		createMemoryHorizontalPodAutoscaler(rc, "100")
		rc.WaitForReplicas(5)
	})

	It("[Skipped][Horizontal pod autoscaling Suite] should scale from 1 pod to 3 pods and from 3 to 1 (scale resource: Memory)", func() {
		rc = NewDynamicResourceConsumer("rc", 1, 0, 700, 100, 800, f)
		defer rc.CleanUp()
		createMemoryHorizontalPodAutoscaler(rc, "300")
		rc.WaitForReplicas(3)
		rc.ConsumeMem(100)
		rc.WaitForReplicas(1)
	})

	It("[Skipped][Horizontal pod autoscaling Suite] should scale from 1 pod to 3 pods and from 3 to 5 (scale resource: Memory)", func() {
		rc = NewDynamicResourceConsumer("rc", 1, 0, 500, 100, 1100, f)
		defer rc.CleanUp()
		createMemoryHorizontalPodAutoscaler(rc, "200")
		rc.WaitForReplicas(3)
		rc.ConsumeMem(1000)
		rc.WaitForReplicas(5)
	})

	It("[Skipped][Horizontal pod autoscaling Suite] should scale from 3 pods to 1 pod and from 1 to 3 (scale resource: Memory)", func() {
		rc = NewDynamicResourceConsumer("rc", 3, 0, 0, 100, 800, f)
		defer rc.CleanUp()
		createMemoryHorizontalPodAutoscaler(rc, "300")
		rc.WaitForReplicas(1)
		rc.ConsumeMem(700)
		rc.WaitForReplicas(3)
	})
	It("[Skipped][Autoscaling Suite] should scale from 5 pods to 3 pods and from 3 to 1 (scale resource: Memory)", func() {
		rc = NewDynamicResourceConsumer("rc", 5, 0, 700, 100, 800, f)
		defer rc.CleanUp()
		createMemoryHorizontalPodAutoscaler(rc, "300")
		rc.WaitForReplicas(3)
		rc.ConsumeMem(100)
		rc.WaitForReplicas(1)
	})

})

func createCPUHorizontalPodAutoscaler(rc *ResourceConsumer, cpu string) {
	hpa := &experimental.HorizontalPodAutoscaler{
		ObjectMeta: api.ObjectMeta{
			Name:      rc.name,
			Namespace: rc.framework.Namespace.Name,
		},
		Spec: experimental.HorizontalPodAutoscalerSpec{
			ScaleRef: &experimental.SubresourceReference{
				Kind:        kind,
				Name:        rc.name,
				Namespace:   rc.framework.Namespace.Name,
				Subresource: subresource,
			},
			MinReplicas: 1,
			MaxReplicas: 5,
			Target:      experimental.ResourceConsumption{Resource: api.ResourceCPU, Quantity: resource.MustParse(cpu)},
		},
	}
	_, errHPA := rc.framework.Client.Experimental().HorizontalPodAutoscalers(rc.framework.Namespace.Name).Create(hpa)
	expectNoError(errHPA)
}

// argument memory is in megabytes
func createMemoryHorizontalPodAutoscaler(rc *ResourceConsumer, memory string) {
	hpa := &experimental.HorizontalPodAutoscaler{
		ObjectMeta: api.ObjectMeta{
			Name:      rc.name,
			Namespace: rc.framework.Namespace.Name,
		},
		Spec: experimental.HorizontalPodAutoscalerSpec{
			ScaleRef: &experimental.SubresourceReference{
				Kind:        kind,
				Name:        rc.name,
				Namespace:   rc.framework.Namespace.Name,
				Subresource: subresource,
			},
			MinReplicas: 1,
			MaxReplicas: 5,
			Target:      experimental.ResourceConsumption{Resource: api.ResourceMemory, Quantity: resource.MustParse(memory + "M")},
		},
	}
	_, errHPA := rc.framework.Client.Experimental().HorizontalPodAutoscalers(rc.framework.Namespace.Name).Create(hpa)
	expectNoError(errHPA)
}
<commit_msg>Fixed flakiness of e2e tests for horizontal pod autoscaler.<commit_after>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/resource"
	"k8s.io/kubernetes/pkg/apis/experimental"

	. "github.com/onsi/ginkgo"
)

const (
	kind        = "replicationController"
	subresource = "scale"
)

var _ = Describe("Horizontal pod autoscaling", func() {
	var rc *ResourceConsumer
	f := NewFramework("horizontal-pod-autoscaling")

	// CPU tests
	It("[Skipped][Autoscaling Suite] should scale from 1 pod to 3 pods and from 3 to 5 (scale resource: CPU)", func() {
		rc = NewDynamicResourceConsumer("rc", 1, 250, 0, 400, 100, f)
		defer rc.CleanUp()
		createCPUHorizontalPodAutoscaler(rc, "0.1")
		rc.WaitForReplicas(3)
		rc.ConsumeCPU(700)
		rc.WaitForReplicas(5)
	})

	It("[Skipped][Autoscaling Suite] should scale from 5 pods to 3 pods and from 3 to 1 (scale resource: CPU)", func() {
		rc = NewDynamicResourceConsumer("rc", 5, 700, 0, 200, 100, f)
		defer rc.CleanUp()
		createCPUHorizontalPodAutoscaler(rc, "0.3")
		rc.WaitForReplicas(3)
		rc.ConsumeCPU(100)
		rc.WaitForReplicas(1)
	})

	// Memory tests
	It("[Skipped][Autoscaling Suite] should scale from 1 pod to 3 pods and from 3 to 5 (scale resource: Memory)", func() {
		rc = NewDynamicResourceConsumer("rc", 1, 0, 2200, 100, 2500, f)
		defer rc.CleanUp()
		createMemoryHorizontalPodAutoscaler(rc, "1000")
		rc.WaitForReplicas(3)
		rc.ConsumeMem(4200)
		rc.WaitForReplicas(5)
	})

	It("[Skipped][Autoscaling Suite] should scale from 5 pods to 3 pods and from 3 to 1 (scale resource: Memory)", func() {
		rc = NewDynamicResourceConsumer("rc", 5, 0, 2200, 100, 1000, f)
		defer rc.CleanUp()
		createMemoryHorizontalPodAutoscaler(rc, "1000")
		rc.WaitForReplicas(3)
		rc.ConsumeMem(100)
		rc.WaitForReplicas(1)
	})

	// Backup tests, currently disabled
	It("[Skipped][Autoscaling Suite] should scale from 1 pod to 3 pods (scale resource: CPU)", func() {
		rc = NewDynamicResourceConsumer("rc", 1, 700, 0, 800, 100, f)
		defer rc.CleanUp()
		createCPUHorizontalPodAutoscaler(rc, "0.3")
		rc.WaitForReplicas(3)
	})

	It("[Skipped][Horizontal pod autoscaling Suite] should scale from 3 pods to 1 pod (scale resource: CPU)", func() {
		rc = NewDynamicResourceConsumer("rc", 3, 0, 0, 100, 100, f)
		defer rc.CleanUp()
		createCPUHorizontalPodAutoscaler(rc, "0.7")
		rc.WaitForReplicas(1)
	})

	It("[Skipped][Horizontal pod autoscaling Suite] should scale from 1 pod to maximum 5 pods (scale resource: CPU)", func() {
		rc = NewDynamicResourceConsumer("rc", 1, 700, 0, 800, 100, f)
		defer rc.CleanUp()
		createCPUHorizontalPodAutoscaler(rc, "0.1")
		rc.WaitForReplicas(5)
	})

	It("[Skipped][Horizontal pod autoscaling Suite] should scale from 1 pod to 3 pods and from 3 to 1 (scale resource: CPU)", func() {
		rc = NewDynamicResourceConsumer("rc", 1, 700, 0, 800, 100, f)
		defer rc.CleanUp()
		createCPUHorizontalPodAutoscaler(rc, "0.3")
		rc.WaitForReplicas(3)
		rc.ConsumeCPU(300)
		rc.WaitForReplicas(1)
	})

	It("[Skipped][Horizontal pod autoscaling Suite] should scale from 3 pods to 1 pod and from 1 to 3 (scale resource: CPU)", func() {
		rc = NewDynamicResourceConsumer("rc", 3, 0, 0, 800, 100, f)
		defer rc.CleanUp()
		createCPUHorizontalPodAutoscaler(rc, "0.3")
		rc.WaitForReplicas(1)
		rc.ConsumeCPU(700)
		rc.WaitForReplicas(3)
	})

	It("[Skipped][Horizontal pod autoscaling Suite] should scale from 1 pod to 3 pods (scale resource: Memory)", func() {
		rc = NewDynamicResourceConsumer("rc", 1, 0, 800, 100, 900, f)
		defer rc.CleanUp()
		createMemoryHorizontalPodAutoscaler(rc, "300")
		rc.WaitForReplicas(3)
	})

	It("[Skipped][Horizontal pod autoscaling Suite] should scale from 3 pods to 1 pod (scale resource: Memory)", func() {
		rc = NewDynamicResourceConsumer("rc", 3, 0, 0, 100, 100, f)
		defer rc.CleanUp()
		createMemoryHorizontalPodAutoscaler(rc, "700")
		rc.WaitForReplicas(1)
	})

	It("[Skipped][Horizontal pod autoscaling Suite] should scale from 1 pod to 3 pods and from 3 to 1 (scale resource: Memory)", func() {
		rc = NewDynamicResourceConsumer("rc", 1, 0, 700, 100, 800, f)
		defer rc.CleanUp()
		createMemoryHorizontalPodAutoscaler(rc, "300")
		rc.WaitForReplicas(3)
		rc.ConsumeMem(100)
		rc.WaitForReplicas(1)
	})

	It("[Skipped][Horizontal pod autoscaling Suite] should scale from 3 pods to 1 pod and from 1 to 3 (scale resource: Memory)", func() {
		rc = NewDynamicResourceConsumer("rc", 3, 0, 0, 100, 800, f)
		defer rc.CleanUp()
		createMemoryHorizontalPodAutoscaler(rc, "300")
		rc.WaitForReplicas(1)
		rc.ConsumeMem(700)
		rc.WaitForReplicas(3)
	})

	It("[Skipped][Horizontal pod autoscaling Suite] should scale from 1 pod to maximum 5 pods (scale resource: Memory)", func() {
		rc = NewDynamicResourceConsumer("rc", 1, 0, 700, 100, 800, f)
		defer rc.CleanUp()
		createMemoryHorizontalPodAutoscaler(rc, "100")
		rc.WaitForReplicas(5)
	})
})

func createCPUHorizontalPodAutoscaler(rc *ResourceConsumer, cpu string) {
	hpa := &experimental.HorizontalPodAutoscaler{
		ObjectMeta: api.ObjectMeta{
			Name:      rc.name,
			Namespace: rc.framework.Namespace.Name,
		},
		Spec: experimental.HorizontalPodAutoscalerSpec{
			ScaleRef: &experimental.SubresourceReference{
				Kind:        kind,
				Name:        rc.name,
				Namespace:   rc.framework.Namespace.Name,
				Subresource: subresource,
			},
			MinReplicas: 1,
			MaxReplicas: 5,
			Target:      experimental.ResourceConsumption{Resource: api.ResourceCPU, Quantity: resource.MustParse(cpu)},
		},
	}
	_, errHPA := rc.framework.Client.Experimental().HorizontalPodAutoscalers(rc.framework.Namespace.Name).Create(hpa)
	expectNoError(errHPA)
}

// argument memory is in megabytes
func createMemoryHorizontalPodAutoscaler(rc *ResourceConsumer, memory string) {
	hpa := &experimental.HorizontalPodAutoscaler{
		ObjectMeta: api.ObjectMeta{
			Name:      rc.name,
			Namespace: rc.framework.Namespace.Name,
		},
		Spec: experimental.HorizontalPodAutoscalerSpec{
			ScaleRef: &experimental.SubresourceReference{
				Kind:        kind,
				Name:        rc.name,
				Namespace:   rc.framework.Namespace.Name,
				Subresource: subresource,
			},
			MinReplicas: 1,
			MaxReplicas: 5,
			Target:      experimental.ResourceConsumption{Resource: api.ResourceMemory, Quantity: resource.MustParse(memory + "M")},
		},
	}
	_, errHPA := rc.framework.Client.Experimental().HorizontalPodAutoscalers(rc.framework.Namespace.Name).Create(hpa)
	expectNoError(errHPA)
}
<|endoftext|>


<commit_before>// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gc

import (
	"bufio"
	"internal/testenv"
	"io"
	"os/exec"
	"regexp"
	"runtime"
	"strings"
	"testing"
)

// TestIntendedInlining tests that specific runtime functions are inlined.
// This allows refactoring for code clarity and re-use without fear that
// changes to the compiler will cause silent performance regressions.
func TestIntendedInlining(t *testing.T) {
	if testing.Short() && testenv.Builder() == "" {
		t.Skip("skipping in short mode")
	}
	testenv.MustHaveGoRun(t)
	t.Parallel()

	// want is the list of function names (by package) that should
	// be inlined.
	want := map[string][]string{
		"runtime": {
			"add",
			"addb",
			"adjustpanics",
			"adjustpointer",
			"bucketMask",
			"bucketShift",
			"chanbuf",
			"deferArgs",
			"deferclass",
			"evacuated",
			"fastlog2",
			"fastrand",
			"float64bits",
			"getm",
			"isDirectIface",
			"itabHashFunc",
			"maxSliceCap",
			"noescape",
			"readUnaligned32",
			"readUnaligned64",
			"round",
			"roundupsize",
			"stringStructOf",
			"subtractb",
			"tophash",
			"totaldefersize",
			"(*bmap).keys",
			"(*bmap).overflow",
			"(*waitq).enqueue",

			//"adjustctxt", TODO(mvdan): fix and re-enable
		},
		"runtime/internal/sys": {},
		"unicode/utf8": {
			"FullRune",
			"FullRuneInString",
			"RuneLen",
			"ValidRune",
		},
	}

	if runtime.GOARCH != "386" {
		// nextFreeFast calls sys.Ctz64, which on 386 is implemented in asm and is not inlinable.
		// We currently don't have midstack inlining so nextFreeFast is also not inlinable on 386.
		// So check for it only on non-386 platforms.
		want["runtime"] = append(want["runtime"], "nextFreeFast")
		// As explained above, Ctz64 and Ctz32 are not Go code on 386.
		// The same applies to Bswap32.
		want["runtime/internal/sys"] = append(want["runtime/internal/sys"], "Ctz64")
		want["runtime/internal/sys"] = append(want["runtime/internal/sys"], "Ctz32")
		want["runtime/internal/sys"] = append(want["runtime/internal/sys"], "Bswap32")
	}
	switch runtime.GOARCH {
	case "amd64", "amd64p32", "arm64", "mips64", "mips64le", "ppc64", "ppc64le", "s390x":
		// rotl_31 is only defined on 64-bit architectures
		want["runtime"] = append(want["runtime"], "rotl_31")
	}

	notInlinedReason := make(map[string]string)
	pkgs := make([]string, 0, len(want))
	for pname, fnames := range want {
		pkgs = append(pkgs, pname)
		for _, fname := range fnames {
			fullName := pname + "." + fname
			if _, ok := notInlinedReason[fullName]; ok {
				t.Errorf("duplicate func: %s", fullName)
			}
			notInlinedReason[fullName] = "unknown reason"
		}
	}

	args := append([]string{"build", "-a", "-gcflags=-m -m"}, pkgs...)
	cmd := testenv.CleanCmdEnv(exec.Command(testenv.GoToolPath(t), args...))
	pr, pw := io.Pipe()
	cmd.Stdout = pw
	cmd.Stderr = pw
	cmdErr := make(chan error, 1)
	go func() {
		cmdErr <- cmd.Run()
		pw.Close()
	}()
	scanner := bufio.NewScanner(pr)
	curPkg := ""
	canInline := regexp.MustCompile(`: can inline ([^ ]*)`)
	cannotInline := regexp.MustCompile(`: cannot inline ([^ ]*): (.*)`)
	for scanner.Scan() {
		line := scanner.Text()
		if strings.HasPrefix(line, "# ") {
			curPkg = line[2:]
			continue
		}
		if m := canInline.FindStringSubmatch(line); m != nil {
			fname := m[1]
			delete(notInlinedReason, curPkg+"."+fname)
			continue
		}
		if m := cannotInline.FindStringSubmatch(line); m != nil {
			fname, reason := m[1], m[2]
			fullName := curPkg + "." + fname
			if _, ok := notInlinedReason[fullName]; ok {
				// cmd/compile gave us a reason why
				notInlinedReason[fullName] = reason
			}
			continue
		}
	}
	if err := <-cmdErr; err != nil {
		t.Fatal(err)
	}
	if err := scanner.Err(); err != nil {
		t.Fatal(err)
	}
	for fullName, reason := range notInlinedReason {
		t.Errorf("%s was not inlined: %s", fullName, reason)
	}
}
<commit_msg>cmd/compile: clarify adjustctxt inlining comment<commit_after>// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gc

import (
	"bufio"
	"internal/testenv"
	"io"
	"os/exec"
	"regexp"
	"runtime"
	"strings"
	"testing"
)

// TestIntendedInlining tests that specific runtime functions are inlined.
// This allows refactoring for code clarity and re-use without fear that
// changes to the compiler will cause silent performance regressions.
func TestIntendedInlining(t *testing.T) {
	if testing.Short() && testenv.Builder() == "" {
		t.Skip("skipping in short mode")
	}
	testenv.MustHaveGoRun(t)
	t.Parallel()

	// want is the list of function names (by package) that should
	// be inlined.
	want := map[string][]string{
		"runtime": {
			// TODO(mvdan): enable these once mid-stack
			// inlining is available
			// "adjustctxt",

			"add",
			"addb",
			"adjustpanics",
			"adjustpointer",
			"bucketMask",
			"bucketShift",
			"chanbuf",
			"deferArgs",
			"deferclass",
			"evacuated",
			"fastlog2",
			"fastrand",
			"float64bits",
			"getm",
			"isDirectIface",
			"itabHashFunc",
			"maxSliceCap",
			"noescape",
			"readUnaligned32",
			"readUnaligned64",
			"round",
			"roundupsize",
			"stringStructOf",
			"subtractb",
			"tophash",
			"totaldefersize",
			"(*bmap).keys",
			"(*bmap).overflow",
			"(*waitq).enqueue",
		},
		"runtime/internal/sys": {},
		"unicode/utf8": {
			"FullRune",
			"FullRuneInString",
			"RuneLen",
			"ValidRune",
		},
	}

	if runtime.GOARCH != "386" {
		// nextFreeFast calls sys.Ctz64, which on 386 is implemented in asm and is not inlinable.
		// We currently don't have midstack inlining so nextFreeFast is also not inlinable on 386.
		// So check for it only on non-386 platforms.
		want["runtime"] = append(want["runtime"], "nextFreeFast")
		// As explained above, Ctz64 and Ctz32 are not Go code on 386.
		// The same applies to Bswap32.
		want["runtime/internal/sys"] = append(want["runtime/internal/sys"], "Ctz64")
		want["runtime/internal/sys"] = append(want["runtime/internal/sys"], "Ctz32")
		want["runtime/internal/sys"] = append(want["runtime/internal/sys"], "Bswap32")
	}
	switch runtime.GOARCH {
	case "amd64", "amd64p32", "arm64", "mips64", "mips64le", "ppc64", "ppc64le", "s390x":
		// rotl_31 is only defined on 64-bit architectures
		want["runtime"] = append(want["runtime"], "rotl_31")
	}

	notInlinedReason := make(map[string]string)
	pkgs := make([]string, 0, len(want))
	for pname, fnames := range want {
		pkgs = append(pkgs, pname)
		for _, fname := range fnames {
			fullName := pname + "." + fname
			if _, ok := notInlinedReason[fullName]; ok {
				t.Errorf("duplicate func: %s", fullName)
			}
			notInlinedReason[fullName] = "unknown reason"
		}
	}

	args := append([]string{"build", "-a", "-gcflags=-m -m"}, pkgs...)
	cmd := testenv.CleanCmdEnv(exec.Command(testenv.GoToolPath(t), args...))
	pr, pw := io.Pipe()
	cmd.Stdout = pw
	cmd.Stderr = pw
	cmdErr := make(chan error, 1)
	go func() {
		cmdErr <- cmd.Run()
		pw.Close()
	}()
	scanner := bufio.NewScanner(pr)
	curPkg := ""
	canInline := regexp.MustCompile(`: can inline ([^ ]*)`)
	cannotInline := regexp.MustCompile(`: cannot inline ([^ ]*): (.*)`)
	for scanner.Scan() {
		line := scanner.Text()
		if strings.HasPrefix(line, "# ") {
			curPkg = line[2:]
			continue
		}
		if m := canInline.FindStringSubmatch(line); m != nil {
			fname := m[1]
			delete(notInlinedReason, curPkg+"."+fname)
			continue
		}
		if m := cannotInline.FindStringSubmatch(line); m != nil {
			fname, reason := m[1], m[2]
			fullName := curPkg + "." + fname
			if _, ok := notInlinedReason[fullName]; ok {
				// cmd/compile gave us a reason why
				notInlinedReason[fullName] = reason
			}
			continue
		}
	}
	if err := <-cmdErr; err != nil {
		t.Fatal(err)
	}
	if err := scanner.Err(); err != nil {
		t.Fatal(err)
	}
	for fullName, reason := range notInlinedReason {
		t.Errorf("%s was not inlined: %s", fullName, reason)
	}
}
<|endoftext|>


<commit_before>package db

import (
	"database/sql"
	"time"
)

//DefaultConnMaxLifetimeInSecond default conn max lifetime
const DefaultConnMaxLifetimeInSecond = int64(30)

//PlainDBOption plain database init option interface.
type PlainDBOption interface {
	//Apply init plain database.
	Apply(*PlainDB) error
}

//Config database config
type Config struct {
	//Driver sql driver.
	Driver string
	//Conn sql conn string.
	DataSource string
	//Prefix sql table prefix.
	Prefix string
	//MaxIdleConns max idle conns.
	MaxIdleConns int
	//ConnMaxLifetimeInSecond conn max Lifetime in second.
	ConnMaxLifetimeInSecond int64
	//MaxOpenConns max open conns.
	MaxOpenConns int
}

//Apply init plain database with config
func (c *Config) Apply(d *PlainDB) error {
	db, err := sql.Open(c.Driver, c.DataSource)
	if err != nil {
		return err
	}
	if c.MaxIdleConns > 0 {
		db.SetMaxIdleConns(c.MaxIdleConns)
	}
	if c.ConnMaxLifetimeInSecond > 0 {
		db.SetConnMaxLifetime(time.Duration(c.ConnMaxLifetimeInSecond) * time.Second)
	} else if c.ConnMaxLifetimeInSecond == 0 {
		db.SetConnMaxLifetime(time.Duration(DefaultConnMaxLifetimeInSecond) * time.Second)
	}
	if c.MaxOpenConns > 0 {
		db.SetMaxOpenConns(c.MaxOpenConns)
	}
	d.SetDB(db)
	d.SetPrefix(c.Prefix)
	return nil
}
<commit_msg>update<commit_after>package db

import (
	"database/sql"
	"time"
)

//DefaultConnMaxLifetimeInSecond default conn max lifetime
const DefaultConnMaxLifetimeInSecond = int64(30)

//PlainDBOption plain database init option interface.
type PlainDBOption interface {
	//Apply init plain database.
	Apply(*PlainDB) error
}

//Config database config
type Config struct {
	//Driver sql driver.
	Driver string
	//Type sql database type.
	Type string
	//Conn sql conn string.
	DataSource string
	//Prefix sql table prefix.
	Prefix string
	//MaxIdleConns max idle conns.
	MaxIdleConns int
	//ConnMaxLifetimeInSecond conn max Lifetime in second.
	ConnMaxLifetimeInSecond int64
	//MaxOpenConns max open conns.
	MaxOpenConns int
}

//Apply init plain database with config
func (c *Config) Apply(d *PlainDB) error {
	db, err := sql.Open(c.Driver, c.DataSource)
	if err != nil {
		return err
	}
	if c.MaxIdleConns > 0 {
		db.SetMaxIdleConns(c.MaxIdleConns)
	}
	if c.ConnMaxLifetimeInSecond > 0 {
		db.SetConnMaxLifetime(time.Duration(c.ConnMaxLifetimeInSecond) * time.Second)
	} else if c.ConnMaxLifetimeInSecond == 0 {
		db.SetConnMaxLifetime(time.Duration(DefaultConnMaxLifetimeInSecond) * time.Second)
	}
	if c.MaxOpenConns > 0 {
		db.SetMaxOpenConns(c.MaxOpenConns)
	}
	d.SetDB(db)
	d.SetPrefix(c.Prefix)
	return nil
}
<|endoftext|>


<commit_before>package state

import (
	. "gopkg.in/check.v1"
	"testing"

	"github.com/ethereum/go-ethereum/ethdb"
	"github.com/ethereum/go-ethereum/ethutil"
	"github.com/ethereum/go-ethereum/trie"
)

func Test(t *testing.T) { TestingT(t) }

type StateSuite struct {
	state *State
}

var _ = Suite(&StateSuite{})

const expectedasbytes = "Expected % x Got % x"

// var ZeroHash256 = make([]byte, 32)

func (s *StateSuite) SetUpTest(c *C) {
	db, _ := ethdb.NewMemDatabase()
	ethutil.ReadConfig(".ethtest", "/tmp/ethtest", "")
	ethutil.Config.Db = db
	s.state = New(trie.New(db, ""))
}

func (s *StateSuite) TestSnapshot(c *C) {
	data1 := ethutil.NewValue(42)
	data2 := ethutil.NewValue(43)
	storageaddr := ethutil.Big("0")
	stateobjaddr := []byte("aa")

	stateObject := s.state.GetOrNewStateObject(stateobjaddr)
	stateObject.SetStorage(storageaddr, data1)
	snapshot := s.state.Copy()

	stateObject = s.state.GetStateObject(stateobjaddr)
	stateObject.SetStorage(storageaddr, data2)
	s.state.Set(snapshot)

	stateObject = s.state.GetStateObject(stateobjaddr)
	res := stateObject.GetStorage(storageaddr)

	c.Assert(data1, DeepEquals, res, Commentf(expectedasbytes, data1, res))
}
<commit_msg>Add verbose comments to TestSnapshot<commit_after>package state

import (
	. "gopkg.in/check.v1"
	"testing"

	"github.com/ethereum/go-ethereum/ethdb"
	"github.com/ethereum/go-ethereum/ethutil"
	"github.com/ethereum/go-ethereum/trie"
)

func Test(t *testing.T) { TestingT(t) }

type StateSuite struct {
	state *State
}

var _ = Suite(&StateSuite{})

const expectedasbytes = "Expected % x Got % x"

// var ZeroHash256 = make([]byte, 32)

func (s *StateSuite) SetUpTest(c *C) {
	db, _ := ethdb.NewMemDatabase()
	ethutil.ReadConfig(".ethtest", "/tmp/ethtest", "")
	ethutil.Config.Db = db
	s.state = New(trie.New(db, ""))
}

func (s *StateSuite) TestSnapshot(c *C) {
	stateobjaddr := []byte("aa")
	storageaddr := ethutil.Big("0")
	data1 := ethutil.NewValue(42)
	data2 := ethutil.NewValue(43)

	// get state object
	stateObject := s.state.GetOrNewStateObject(stateobjaddr)
	// set inital state object value
	stateObject.SetStorage(storageaddr, data1)
	// get snapshot of current state
	snapshot := s.state.Copy()

	// get state object. is this strictly necessary?
	stateObject = s.state.GetStateObject(stateobjaddr)
	// set new state object value
	stateObject.SetStorage(storageaddr, data2)
	// restore snapshot
	s.state.Set(snapshot)

	// get state object
	stateObject = s.state.GetStateObject(stateobjaddr)
	// get state storage value
	res := stateObject.GetStorage(storageaddr)

	c.Assert(data1, DeepEquals, res, Commentf(expectedasbytes, data1, res))
}
<|endoftext|>


<commit_before>package mq_test

import (
	"fmt"
	"github.com/manveru/go.iron/mq"
)

var p = fmt.Println

func Example1PushingMessagesToTheQueue() {
	// use the test_queue to push/get messages
	q := mq.New("test_queue")

	q.PushString("Hello, World!")

	// You can also pass multiple messages in a single call.
	q.PushStrings("Message 1", "Message 2")

	// To control parameters like timeout and delay, construct your own message.
	q.PushMessage(&mq.Message{Timeout: 60, Delay: 0, Body: "Hi there"})

	// And finally, all that can be done in bulk as well.
	q.PushMessages(
		&mq.Message{Timeout: 60, Delay: 0, Body: "The first"},
		&mq.Message{Timeout: 60, Delay: 1, Body: "The second"},
		&mq.Message{Timeout: 60, Delay: 2, Body: "The third"},
		&mq.Message{Timeout: 60, Delay: 3, Body: "The fifth"},
	)

	p("all pushed")

	// Output:
	// all pushed
}

func Example2GettingMessagesOffTheQueue() {
	q := mq.New("test_queue")

	// get a single message
	msg, err := q.Get()
	p(err)
	p(msg.Body)

	// get 5 messages
	msgs, err := q.GetN(5)
	p(err)
	p(len(msgs))

	for _, m := range append(msgs, msg) {
		m.Delete()
	}

	// Output:
	// <nil>
	// Hello, World!
	// <nil>
	// 5
}

func Example3DeleteMessagesFromTheQueue() {
	q := mq.New("test_queue")
	msg, err := q.Get()
	p(err)
	msg.Delete()

	// Output:
	// <nil>
}

func Example4ClearQueue() {
	q := mq.New("test_queue")

	for {
		msgs, err := q.GetN(100)
		if err != nil {
			p(err)
			break
		}
		if len(msgs) <= 0 {
			break
		}

		for _, msg := range msgs {
			msg.Delete()
		}
	}

	info, err := q.Info()

	p(err)
	p("Name:", info.Name)
	p("Size:", info.Size)

	// Output:
	// <nil>
	// Name: test_queue
	// Size: 0
}
<commit_msg>add mq clean test<commit_after>package mq_test

import (
	"fmt"
	"github.com/manveru/go.iron/mq"
)

var p = fmt.Println

func Example1PushingMessagesToTheQueue() {
	// use the test_queue to push/get messages
	q := mq.New("test_queue")

	q.PushString("Hello, World!")

	// You can also pass multiple messages in a single call.
	q.PushStrings("Message 1", "Message 2")

	// To control parameters like timeout and delay, construct your own message.
	q.PushMessage(&mq.Message{Timeout: 60, Delay: 0, Body: "Hi there"})

	// And finally, all that can be done in bulk as well.
	q.PushMessages(
		&mq.Message{Timeout: 60, Delay: 0, Body: "The first"},
		&mq.Message{Timeout: 60, Delay: 1, Body: "The second"},
		&mq.Message{Timeout: 60, Delay: 2, Body: "The third"},
		&mq.Message{Timeout: 60, Delay: 3, Body: "The fifth"},
	)

	p("all pushed")

	// Output:
	// all pushed
}

func Example2GettingMessagesOffTheQueue() {
	q := mq.New("test_queue")

	// get a single message
	msg, err := q.Get()
	p(err)
	p(msg.Body)

	// get 5 messages
	msgs, err := q.GetN(5)
	p(err)
	p(len(msgs))

	for _, m := range append(msgs, msg) {
		m.Delete()
	}

	// Output:
	// <nil>
	// Hello, World!
	// <nil>
	// 5
}

func Example3DeleteMessagesFromTheQueue() {
	q := mq.New("test_queue")
	msg, err := q.Get()
	p(err)
	msg.Delete()

	// Output:
	// <nil>
}

func Example4ClearQueue() {
	q := mq.New("test_queue")

	info, err := q.Info()

	p(err)
	p("Name:", info.Name)
	p("Size:", info.Size)

	err = q.Clear()
	p(err)

	info, err = q.Info()

	p(err)
	p("Name:", info.Name)
	p("Size:", info.Size)

	// Output:
	// <nil>
	// Name: test_queue
	// Size: 0
	// <nil>
	// Name: test_queue
	// Size: 0
	// <nil>
	// Name: test_queue
	// Size: 0
}
<|endoftext|>


<commit_before>// Copyright 2016 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package resourceadapters

import (
	"io"
	"time"

	"github.com/juju/errors"
	"github.com/juju/names"
	"github.com/juju/retry"
	"github.com/juju/utils/clock"
	"gopkg.in/juju/charm.v6-unstable"
	charmresource "gopkg.in/juju/charm.v6-unstable/resource"

	"github.com/juju/juju/resource"
	"github.com/juju/juju/resource/charmstore"
	corestate "github.com/juju/juju/state"
)

// charmstoreEntityCache adapts between resource state and charmstore.EntityCache.
type charmstoreEntityCache struct {
	st        corestate.Resources
	userID    names.Tag
	unit      resource.Unit
	serviceID string
}

// GetResource implements charmstore.EntityCache.
func (cache *charmstoreEntityCache) GetResource(name string) (resource.Resource, error) {
	return cache.st.GetResource(cache.serviceID, name)
}

// SetResource implements charmstore.EntityCache.
func (cache *charmstoreEntityCache) SetResource(chRes charmresource.Resource, reader io.Reader) (resource.Resource, error) {
	return cache.st.SetResource(cache.serviceID, cache.userID.Id(), chRes, reader)
}

// OpenResource implements charmstore.EntityCache.
func (cache *charmstoreEntityCache) OpenResource(name string) (resource.Resource, io.ReadCloser, error) {
	if cache.unit != nil {
		return cache.st.OpenResourceForUnit(cache.unit, name)
	}
	return cache.st.OpenResource(cache.serviceID, name)
}

type charmstoreOpener struct {
	// TODO(ericsnow) What do we need?
}

func newCharmstoreOpener(cURL *charm.URL) *charmstoreOpener {
	// TODO(ericsnow) Extract the charm store URL from the charm URL.
	return &charmstoreOpener{}
}

// NewClient opens a new charm store client.
func (cs *charmstoreOpener) NewClient() (charmstore.Client, error) {
	// TODO(ericsnow) Return an actual charm store client.
	client := newFakeCharmStoreClient(nil)
	return newCSRetryClient(client), nil
}

type csRetryClient struct {
	charmstore.Client
	retryArgs retry.CallArgs
}

func newCSRetryClient(client charmstore.Client) *csRetryClient {
	retryArgs := retry.CallArgs{
		// We use errorShouldNotRetry here since errors that should not
		// be retried should cause the retry loop to stop.
		IsFatalError: errorShouldNotRetry,
		// We want to retry until the charm store either gives us the
		// resource (and we cache it) or the resource isn't found in the
		// charm store.
		Attempts: -1, // retry forever...
		// A one minute gives enough time for potential connection
		// issues to sort themselves out without making the caller wait
		// for an exceptional amount of time.
		Delay: 1 * time.Minute,
		Clock: clock.WallClock,
	}
	return &csRetryClient{
		Client:    client,
		retryArgs: retryArgs,
	}
}

// GetResource returns a reader for the resource's data.
func (client csRetryClient) GetResource(cURL *charm.URL, resourceName string, revision int) (io.ReadCloser, error) {
	args := client.retryArgs // a copy

	var reader io.ReadCloser
	args.Func = func() error {
		csReader, err := client.Client.GetResource(cURL, resourceName, revision)
		if err != nil {
			return errors.Trace(err)
		}
		reader = csReader
		return nil
	}

	var lastErr error
	args.NotifyFunc = func(err error, i int) {
		// Remember the error we're hiding and then retry!
		logger.Errorf("(attempt %d) retrying resource download from charm store due to error: %v", i, err)
		lastErr = err
	}

	err := retry.Call(args)
	if retry.IsAttemptsExceeded(err) {
		return nil, errors.Annotate(lastErr, "failed after retrying")
	}
	if err != nil {
		return nil, errors.Trace(err)
	}

	return reader, nil
}

func errorShouldNotRetry(err error) bool {
	if errors.IsNotFound(err) {
		return true
	}
	return false
}
<commit_msg>Drop errorShouldNotRetry().<commit_after>// Copyright 2016 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package resourceadapters

import (
	"io"
	"time"

	"github.com/juju/errors"
	"github.com/juju/names"
	"github.com/juju/retry"
	"github.com/juju/utils/clock"
	"gopkg.in/juju/charm.v6-unstable"
	charmresource "gopkg.in/juju/charm.v6-unstable/resource"

	"github.com/juju/juju/resource"
	"github.com/juju/juju/resource/charmstore"
	corestate "github.com/juju/juju/state"
)

// charmstoreEntityCache adapts between resource state and charmstore.EntityCache.
type charmstoreEntityCache struct {
	st        corestate.Resources
	userID    names.Tag
	unit      resource.Unit
	serviceID string
}

// GetResource implements charmstore.EntityCache.
func (cache *charmstoreEntityCache) GetResource(name string) (resource.Resource, error) {
	return cache.st.GetResource(cache.serviceID, name)
}

// SetResource implements charmstore.EntityCache.
func (cache *charmstoreEntityCache) SetResource(chRes charmresource.Resource, reader io.Reader) (resource.Resource, error) {
	return cache.st.SetResource(cache.serviceID, cache.userID.Id(), chRes, reader)
}

// OpenResource implements charmstore.EntityCache.
func (cache *charmstoreEntityCache) OpenResource(name string) (resource.Resource, io.ReadCloser, error) {
	if cache.unit != nil {
		return cache.st.OpenResourceForUnit(cache.unit, name)
	}
	return cache.st.OpenResource(cache.serviceID, name)
}

type charmstoreOpener struct {
	// TODO(ericsnow) What do we need?
}

func newCharmstoreOpener(cURL *charm.URL) *charmstoreOpener {
	// TODO(ericsnow) Extract the charm store URL from the charm URL.
	return &charmstoreOpener{}
}

// NewClient opens a new charm store client.
func (cs *charmstoreOpener) NewClient() (charmstore.Client, error) {
	// TODO(ericsnow) Return an actual charm store client.
	client := newFakeCharmStoreClient(nil)
	return newCSRetryClient(client), nil
}

type csRetryClient struct {
	charmstore.Client
	retryArgs retry.CallArgs
}

func newCSRetryClient(client charmstore.Client) *csRetryClient {
	retryArgs := retry.CallArgs{
		// The only error that stops the retry loop should be "not found".
		IsFatalError: errors.IsNotFound,
		// We want to retry until the charm store either gives us the
		// resource (and we cache it) or the resource isn't found in the
		// charm store.
		Attempts: -1, // retry forever...
		// A one minute gives enough time for potential connection
		// issues to sort themselves out without making the caller wait
		// for an exceptional amount of time.
		Delay: 1 * time.Minute,
		Clock: clock.WallClock,
	}
	return &csRetryClient{
		Client:    client,
		retryArgs: retryArgs,
	}
}

// GetResource returns a reader for the resource's data.
func (client csRetryClient) GetResource(cURL *charm.URL, resourceName string, revision int) (io.ReadCloser, error) {
	args := client.retryArgs // a copy

	var reader io.ReadCloser
	args.Func = func() error {
		csReader, err := client.Client.GetResource(cURL, resourceName, revision)
		if err != nil {
			return errors.Trace(err)
		}
		reader = csReader
		return nil
	}

	var lastErr error
	args.NotifyFunc = func(err error, i int) {
		// Remember the error we're hiding and then retry!
		logger.Errorf("(attempt %d) retrying resource download from charm store due to error: %v", i, err)
		lastErr = err
	}

	err := retry.Call(args)
	if retry.IsAttemptsExceeded(err) {
		return nil, errors.Annotate(lastErr, "failed after retrying")
	}
	if err != nil {
		return nil, errors.Trace(err)
	}

	return reader, nil
}
<|endoftext|>


<commit_before>package elasti_cache

import (
	"github.com/jagregory/cfval/constraints"
	"github.com/jagregory/cfval/reporting"
	"github.com/jagregory/cfval/resources/common"
	. "github.com/jagregory/cfval/schema"
)

func azModeValidate(value interface{}, ctx PropertyContext) (reporting.ValidateResult, reporting.Reports) {
	if str, ok := value.(string); ok {
		if availabilityZones, ok := ctx.CurrentResource().PropertyValueOrDefault("PreferredAvailabilityZones"); ok {
			if str == "cross-az" && len(availabilityZones.([]interface{})) < 2 {
				return reporting.ValidateOK, reporting.Reports{reporting.NewFailure(ctx, "Cross-AZ clusters must have multiple preferred availability zones")}
			}
		}
	}

	return reporting.ValidateOK, nil
}

func numCacheNodesValidate(value interface{}, ctx PropertyContext) (reporting.ValidateResult, reporting.Reports) {
	if engine, ok := ctx.CurrentResource().PropertyValueOrDefault("Engine"); !ok || engine.(string) == "memcached" {
		return IntegerRangeValidate(1, 20)(value, ctx)
	}

	return SingleValueValidate(float64(1))(value, ctx)
}

// see: http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-elasticache-cache-cluster.html
var CacheCluster = Resource{
	AwsType: "AWS::ElastiCache::CacheCluster",

	Attributes: map[string]Schema{
		"ConfigurationEndpoint.Address": Schema{
			Type: ValueString,
		},

		"ConfigurationEndpoint.Port": Schema{
			Type: ValueString,
		},
	},

	// Name
	ReturnValue: Schema{
		Type: ValueString,
	},

	Properties: Properties{
		"AutoMinorVersionUpgrade": Schema{
			Type: ValueBool,
		},

		"AZMode": Schema{
			Type:         azMode,
			ValidateFunc: azModeValidate,
			Default:      "single-az",
		},

		"CacheNodeType": Schema{
			Type:     cacheNodeType,
			Required: constraints.Always,
		},

		"CacheParameterGroupName": Schema{
			Type: ValueString,
		},

		"CacheSecurityGroupNames": Schema{
			Type: Multiple(cacheSecurityGroupName),
			Conflicts: constraints.Any{
				constraints.PropertyExists("CacheSubnetGroupName"),
				constraints.PropertyExists("VpcSecurityGroupIds"),
			},
		},

		"CacheSubnetGroupName": Schema{
			Type: cacheSecurityGroupName,
			Conflicts: constraints.Any{
				constraints.PropertyExists("CacheSecurityGroupNames"),
				constraints.PropertyExists("VpcSecurityGroupIds"),
			},
		},

		"ClusterName": Schema{
			Type: ValueString,
		},

		"Engine": Schema{
			Type:     engine,
			Required: constraints.Always,
		},

		"EngineVersion": Schema{
			Type: ValueString,
		},

		"NotificationTopicArn": Schema{
			Type: ARN,
		},

		"NumCacheNodes": Schema{
			Type:         ValueNumber,
			Required:     constraints.Always,
			ValidateFunc: numCacheNodesValidate,
		},

		"Port": Schema{
			Type: ValueNumber,
		},

		"PreferredAvailabilityZone": Schema{
			Type: AvailabilityZone,
		},

		"PreferredAvailabilityZones": Schema{
			Type:     Multiple(AvailabilityZone),
			Required: constraints.PropertyIs("AZMode", "cross-az"),
		},

		"PreferredMaintenanceWindow": Schema{
			Type: ValueString,
		},

		"SnapshotArns": Schema{
			Type: Multiple(ARN),
		},

		"SnapshotName": Schema{
			Type: ValueString,
		},

		"SnapshotRetentionLimit": Schema{
			Type: ValueNumber,
		},

		"SnapshotWindow": Schema{
			Type: ValueString,
		},

		"Tags": Schema{
			Type: Multiple(common.ResourceTag),
		},

		"VpcSecurityGroupIds": Schema{
			Type:      Multiple(SecurityGroupID),
			Conflicts: constraints.PropertyExists("CacheSecurityGroupNames"),
		},
	},
}
<commit_msg>Fix for CacheCluster.CacheSubnetGroupName type<commit_after>package elasti_cache

import (
	"github.com/jagregory/cfval/constraints"
	"github.com/jagregory/cfval/reporting"
	"github.com/jagregory/cfval/resources/common"
	. "github.com/jagregory/cfval/schema"
)

func azModeValidate(value interface{}, ctx PropertyContext) (reporting.ValidateResult, reporting.Reports) {
	if str, ok := value.(string); ok {
		if availabilityZones, ok := ctx.CurrentResource().PropertyValueOrDefault("PreferredAvailabilityZones"); ok {
			if str == "cross-az" && len(availabilityZones.([]interface{})) < 2 {
				return reporting.ValidateOK, reporting.Reports{reporting.NewFailure(ctx, "Cross-AZ clusters must have multiple preferred availability zones")}
			}
		}
	}

	return reporting.ValidateOK, nil
}

func numCacheNodesValidate(value interface{}, ctx PropertyContext) (reporting.ValidateResult, reporting.Reports) {
	if engine, ok := ctx.CurrentResource().PropertyValueOrDefault("Engine"); !ok || engine.(string) == "memcached" {
		return IntegerRangeValidate(1, 20)(value, ctx)
	}

	return SingleValueValidate(float64(1))(value, ctx)
}

// see: http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-elasticache-cache-cluster.html
var CacheCluster = Resource{
	AwsType: "AWS::ElastiCache::CacheCluster",

	Attributes: map[string]Schema{
		"ConfigurationEndpoint.Address": Schema{
			Type: ValueString,
		},

		"ConfigurationEndpoint.Port": Schema{
			Type: ValueString,
		},
	},

	// Name
	ReturnValue: Schema{
		Type: ValueString,
	},

	Properties: Properties{
		"AutoMinorVersionUpgrade": Schema{
			Type: ValueBool,
		},

		"AZMode": Schema{
			Type:         azMode,
			ValidateFunc: azModeValidate,
			Default:      "single-az",
		},

		"CacheNodeType": Schema{
			Type:     cacheNodeType,
			Required: constraints.Always,
		},

		"CacheParameterGroupName": Schema{
			Type: ValueString,
		},

		"CacheSecurityGroupNames": Schema{
			Type: Multiple(cacheSecurityGroupName),
			Conflicts: constraints.Any{
				constraints.PropertyExists("CacheSubnetGroupName"),
				constraints.PropertyExists("VpcSecurityGroupIds"),
			},
		},

		"CacheSubnetGroupName": Schema{
			Type: cacheSubnetGroupName,
			Conflicts: constraints.Any{
				constraints.PropertyExists("CacheSecurityGroupNames"),
				constraints.PropertyExists("VpcSecurityGroupIds"),
			},
		},

		"ClusterName": Schema{
			Type: ValueString,
		},

		"Engine": Schema{
			Type:     engine,
			Required: constraints.Always,
		},

		"EngineVersion": Schema{
			Type: ValueString,
		},

		"NotificationTopicArn": Schema{
			Type: ARN,
		},

		"NumCacheNodes": Schema{
			Type:         ValueNumber,
			Required:     constraints.Always,
			ValidateFunc: numCacheNodesValidate,
		},

		"Port": Schema{
			Type: ValueNumber,
		},

		"PreferredAvailabilityZone": Schema{
			Type: AvailabilityZone,
		},

		"PreferredAvailabilityZones": Schema{
			Type:     Multiple(AvailabilityZone),
			Required: constraints.PropertyIs("AZMode", "cross-az"),
		},

		"PreferredMaintenanceWindow": Schema{
			Type: ValueString,
		},

		"SnapshotArns": Schema{
			Type: Multiple(ARN),
		},

		"SnapshotName": Schema{
			Type: ValueString,
		},

		"SnapshotRetentionLimit": Schema{
			Type: ValueNumber,
		},

		"SnapshotWindow": Schema{
			Type: ValueString,
		},

		"Tags": Schema{
			Type: Multiple(common.ResourceTag),
		},

		"VpcSecurityGroupIds": Schema{
			Type:      Multiple(SecurityGroupID),
			Conflicts: constraints.PropertyExists("CacheSecurityGroupNames"),
		},
	},
}
<|endoftext|>


<commit_before>package azurerm

import (
	"fmt"
	"net/http"

	"github.com/Azure/azure-sdk-for-go/arm/network"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceArmLocalNetworkGateway() *schema.Resource {
	return &schema.Resource{
		Create: resourceArmLocalNetworkGatewayCreate,
		Read:   resourceArmLocalNetworkGatewayRead,
		Update: resourceArmLocalNetworkGatewayCreate,
		Delete: resourceArmLocalNetworkGatewayDelete,
		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},

		Schema: map[string]*schema.Schema{
			"name": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"location": locationSchema(),

			"resource_group_name": {
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
			},

			"gateway_address": {
				Type:     schema.TypeString,
				Required: true,
			},

			"address_space": {
				Type:     schema.TypeList,
				Required: true,
				Elem: &schema.Schema{
					Type: schema.TypeString,
				},
			},
		},
	}
}

func resourceArmLocalNetworkGatewayCreate(d *schema.ResourceData, meta interface{}) error {
	lnetClient := meta.(*ArmClient).localNetConnClient

	name := d.Get("name").(string)
	location := d.Get("location").(string)
	resGroup := d.Get("resource_group_name").(string)
	ipAddress := d.Get("gateway_address").(string)

	// fetch the 'address_space_prefixes:
	prefixes := []string{}
	for _, pref := range d.Get("address_space").([]interface{}) {
		prefixes = append(prefixes, pref.(string))
	}

	gateway := network.LocalNetworkGateway{
		Name:     &name,
		Location: &location,
		LocalNetworkGatewayPropertiesFormat: &network.LocalNetworkGatewayPropertiesFormat{
			LocalNetworkAddressSpace: &network.AddressSpace{
				AddressPrefixes: &prefixes,
			},
			GatewayIPAddress: &ipAddress,
		},
	}

	_, err := lnetClient.CreateOrUpdate(resGroup, name, gateway, make(chan struct{}))
	if err != nil {
		return fmt.Errorf("Error creating Azure ARM Local Network Gateway '%s': %s", name, err)
	}

	read, err := lnetClient.Get(resGroup, name)
	if err != nil {
		return err
	}
	if read.ID == nil {
		return fmt.Errorf("Cannot read Virtual Network %s (resource group %s) ID", name, resGroup)
	}

	d.SetId(*read.ID)

	return resourceArmLocalNetworkGatewayRead(d, meta)
}

// resourceArmLocalNetworkGatewayRead goes ahead and reads the state of the corresponding ARM local network gateway.
func resourceArmLocalNetworkGatewayRead(d *schema.ResourceData, meta interface{}) error {
	lnetClient := meta.(*ArmClient).localNetConnClient

	id, err := parseAzureResourceID(d.Id())
	if err != nil {
		return err
	}
	name := id.Path["localNetworkGateways"]
	if name == "" {
		return fmt.Errorf("Cannot find 'localNetworkGateways' in '%s', make sure it is specified in the ID parameter", d.Id())
	}
	resGroup := id.ResourceGroup

	resp, err := lnetClient.Get(resGroup, name)
	if err != nil {
		if resp.StatusCode == http.StatusNotFound {
			d.SetId("")
			return nil
		}
		return fmt.Errorf("Error reading the state of Azure ARM local network gateway '%s': %s", name, err)
	}

	d.Set("resource_group_name", resGroup)
	d.Set("name", resp.Name)
	d.Set("location", resp.Location)
	d.Set("gateway_address", resp.LocalNetworkGatewayPropertiesFormat.GatewayIPAddress)

	prefs := []string{}
	if ps := *resp.LocalNetworkGatewayPropertiesFormat.LocalNetworkAddressSpace.AddressPrefixes; ps != nil {
		prefs = ps
	}
	d.Set("address_space", prefs)

	return nil
}

// resourceArmLocalNetworkGatewayDelete deletes the specified ARM local network gateway.
func resourceArmLocalNetworkGatewayDelete(d *schema.ResourceData, meta interface{}) error {
	lnetClient := meta.(*ArmClient).localNetConnClient

	id, err := parseAzureResourceID(d.Id())
	if err != nil {
		return err
	}
	name := id.Path["localNetworkGateways"]
	resGroup := id.ResourceGroup

	_, err = lnetClient.Delete(resGroup, name, make(chan struct{}))
	if err != nil {
		return fmt.Errorf("Error issuing Azure ARM delete request of local network gateway '%s': %s", name, err)
	}

	return nil
}
<commit_msg>Revert "Even better error message."<commit_after>package azurerm

import (
	"fmt"
	"net/http"

	"github.com/Azure/azure-sdk-for-go/arm/network"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceArmLocalNetworkGateway() *schema.Resource {
	return &schema.Resource{
		Create: resourceArmLocalNetworkGatewayCreate,
		Read:   resourceArmLocalNetworkGatewayRead,
		Update: resourceArmLocalNetworkGatewayCreate,
		Delete: resourceArmLocalNetworkGatewayDelete,
		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},

		Schema: map[string]*schema.Schema{
			"name": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"location": locationSchema(),

			"resource_group_name": {
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
			},

			"gateway_address": {
				Type:     schema.TypeString,
				Required: true,
			},

			"address_space": {
				Type:     schema.TypeList,
				Required: true,
				Elem: &schema.Schema{
					Type: schema.TypeString,
				},
			},
		},
	}
}

func resourceArmLocalNetworkGatewayCreate(d *schema.ResourceData, meta interface{}) error {
	lnetClient := meta.(*ArmClient).localNetConnClient

	name := d.Get("name").(string)
	location := d.Get("location").(string)
	resGroup := d.Get("resource_group_name").(string)
	ipAddress := d.Get("gateway_address").(string)

	// fetch the 'address_space_prefixes:
	prefixes := []string{}
	for _, pref := range d.Get("address_space").([]interface{}) {
		prefixes = append(prefixes, pref.(string))
	}

	gateway := network.LocalNetworkGateway{
		Name:     &name,
		Location: &location,
		LocalNetworkGatewayPropertiesFormat: &network.LocalNetworkGatewayPropertiesFormat{
			LocalNetworkAddressSpace: &network.AddressSpace{
				AddressPrefixes: &prefixes,
			},
			GatewayIPAddress: &ipAddress,
		},
	}

	_, err := lnetClient.CreateOrUpdate(resGroup, name, gateway, make(chan struct{}))
	if err != nil {
		return fmt.Errorf("Error creating Azure ARM Local Network Gateway '%s': %s", name, err)
	}

	read, err := lnetClient.Get(resGroup, name)
	if err != nil {
		return err
	}
	if read.ID == nil {
		return fmt.Errorf("Cannot read Virtual Network %s (resource group %s) ID", name, resGroup)
	}

	d.SetId(*read.ID)

	return resourceArmLocalNetworkGatewayRead(d, meta)
}

// resourceArmLocalNetworkGatewayRead goes ahead and reads the state of the corresponding ARM local network gateway.
func resourceArmLocalNetworkGatewayRead(d *schema.ResourceData, meta interface{}) error {
	lnetClient := meta.(*ArmClient).localNetConnClient

	id, err := parseAzureResourceID(d.Id())
	if err != nil {
		return err
	}
	name := id.Path["localNetworkGateways"]
	if name == "" {
		var pathString, sp string
		for key, value := range id.Path {
			pathString += fmt.Sprintf("%s'%s:%s'", sp, key, value)
			sp = ", "
		}
		return fmt.Errorf("Cannot find 'localNetworkGateways' in [%s], make sure it is specified in the ID parameter", pathString)
	}
	resGroup := id.ResourceGroup

	resp, err := lnetClient.Get(resGroup, name)
	if err != nil {
		if resp.StatusCode == http.StatusNotFound {
			d.SetId("")
			return nil
		}
		return fmt.Errorf("Error reading the state of Azure ARM local network gateway '%s': %s", name, err)
	}

	d.Set("resource_group_name", resGroup)
	d.Set("name", resp.Name)
	d.Set("location", resp.Location)
	d.Set("gateway_address", resp.LocalNetworkGatewayPropertiesFormat.GatewayIPAddress)

	prefs := []string{}
	if ps := *resp.LocalNetworkGatewayPropertiesFormat.LocalNetworkAddressSpace.AddressPrefixes; ps != nil {
		prefs = ps
	}
	d.Set("address_space", prefs)

	return nil
}

// resourceArmLocalNetworkGatewayDelete deletes the specified ARM local network gateway.
func resourceArmLocalNetworkGatewayDelete(d *schema.ResourceData, meta interface{}) error {
	lnetClient := meta.(*ArmClient).localNetConnClient

	id, err := parseAzureResourceID(d.Id())
	if err != nil {
		return err
	}
	name := id.Path["localNetworkGateways"]
	resGroup := id.ResourceGroup

	_, err = lnetClient.Delete(resGroup, name, make(chan struct{}))
	if err != nil {
		return fmt.Errorf("Error issuing Azure ARM delete request of local network gateway '%s': %s", name, err)
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/xml"
	"flag"
	"fmt"
	"net/http"
	"net/url"
	"strconv"

	"code.google.com/p/go-charset/charset"
	_ "code.google.com/p/go-charset/data"
)

type DateTimeType struct {
	Date struct {
		Day   int `xml:"day,attr"`
		Month int `xml:"month,attr"`
		Year  int `xml:"year,attr"`
	} `xml:"itdDate"`

	Time struct {
		Hour   int `xml:"hour,attr"`
		Minute int `xml:"minute,attr"`
	} `xml:"itdTime"`
}

type Line struct {
	Number    string `xml:"number,attr"`
	Direction string `xml:"direction,attr"`
}

type Departure struct {
	Countdown int    `xml:"countdown,attr"`
	Platform  string `xml:"platform,attr"`

	DateTime    DateTimeType `xml:"itdDateTime"`
	ServingLine Line         `xml:"itdServingLine"`
}

type StopInfo struct {
	State string `xml:"state,attr"`

	IdfdStop struct {
		StopName  string `xml:",chardata"`
		MatchQlty int    `xml:"matchQuality,attr"`
		StopID    int    `xml:"stopID,attr"`
	} `xml:"odvNameElem"`
}

type XmlResult struct {
	Stop       StopInfo    `xml:"itdDepartureMonitorRequest>itdOdv>itdOdvName"`
	Departures []Departure `xml:"itdDepartureMonitorRequest>itdDepartureList>itdDeparture"`
}

func fill_width(str string, width int) string {

	orig_len := len(str)

	if orig_len < width {

		for i := 0; i < width-orig_len; i++ {
			str = str + " "
		}
	}

	return str
}

func main() {

	station_id := flag.String("stop", "Königsplatz", "id or (part of the) stop name")
	max_results := flag.Int("results", 5, "how many results to show")
	flag.Parse()

	baseULR := "http://efa.avv-augsburg.de/avv/"
	endpoint := "XML_DM_REQUEST"

	params := url.Values{
		"type_dm":              {"stop"},
		"name_dm":              {*station_id},
		"useRealtime":          {"1"},
		"locationServerActive": {"1"},
		"dmLineSelection":      {"all"},
		"limit":                {strconv.Itoa(*max_results)},
		"mode":                 {"direct"},
	}

	resp, err := http.PostForm(baseULR+endpoint, params)
	defer resp.Body.Close()

	if err != nil {
		return
	}

	var result XmlResult

	decoder := xml.NewDecoder(resp.Body)
	decoder.CharsetReader = charset.NewReader
	err = decoder.Decode(&result)

	if err != nil {
		fmt.Println(err)
		return
	}

	//fmt.Printf("%+v", result)

	if result.Stop.State != "identified" {
		fmt.Println("stop does not exist or name is not unique!")
		return
	}

	fmt.Println("selected stop: " + result.Stop.IdfdStop.StopName + " (" + strconv.Itoa(result.Stop.IdfdStop.StopID) + ")\n")

	for _, departure := range result.Departures {

		plu := ""
		if departure.Countdown != 1 {
			plu = "s"
		}

		fmt.Println("route " + fill_width(departure.ServingLine.Number, 6) + fill_width(" due in "+strconv.Itoa(departure.Countdown)+" minute"+plu, 19) + " --> " + departure.ServingLine.Direction)
	}

}
<commit_msg>* fill_width duplicates printf functionality.<commit_after>package main

import (
	"encoding/xml"
	"flag"
	"fmt"
	"net/http"
	"net/url"
	"strconv"

	"code.google.com/p/go-charset/charset"
	_ "code.google.com/p/go-charset/data"
)

type DateTimeType struct {
	Date struct {
		Day   int `xml:"day,attr"`
		Month int `xml:"month,attr"`
		Year  int `xml:"year,attr"`
	} `xml:"itdDate"`

	Time struct {
		Hour   int `xml:"hour,attr"`
		Minute int `xml:"minute,attr"`
	} `xml:"itdTime"`
}

type Line struct {
	Number    string `xml:"number,attr"`
	Direction string `xml:"direction,attr"`
}

type Departure struct {
	Countdown int    `xml:"countdown,attr"`
	Platform  string `xml:"platform,attr"`

	DateTime    DateTimeType `xml:"itdDateTime"`
	ServingLine Line         `xml:"itdServingLine"`
}

type StopInfo struct {
	State string `xml:"state,attr"`

	IdfdStop struct {
		StopName  string `xml:",chardata"`
		MatchQlty int    `xml:"matchQuality,attr"`
		StopID    int    `xml:"stopID,attr"`
	} `xml:"odvNameElem"`
}

type XmlResult struct {
	Stop       StopInfo    `xml:"itdDepartureMonitorRequest>itdOdv>itdOdvName"`
	Departures []Departure `xml:"itdDepartureMonitorRequest>itdDepartureList>itdDeparture"`
}

func main() {

	station_id := flag.String("stop", "Königsplatz", "id or (part of the) stop name")
	max_results := flag.Int("results", 5, "how many results to show")
	flag.Parse()

	baseULR := "http://efa.avv-augsburg.de/avv/"
	endpoint := "XML_DM_REQUEST"

	params := url.Values{
		"type_dm":              {"stop"},
		"name_dm":              {*station_id},
		"useRealtime":          {"1"},
		"locationServerActive": {"1"},
		"dmLineSelection":      {"all"},
		"limit":                {strconv.Itoa(*max_results)},
		"mode":                 {"direct"},
	}

	resp, err := http.PostForm(baseULR+endpoint, params)
	defer resp.Body.Close()

	if err != nil {
		return
	}

	var result XmlResult

	decoder := xml.NewDecoder(resp.Body)
	decoder.CharsetReader = charset.NewReader
	err = decoder.Decode(&result)

	if err != nil {
		fmt.Println(err)
		return
	}
	//fmt.Printf("%+v", result)

	if result.Stop.State != "identified" {
		fmt.Println("stop does not exist or name is not unique!")
		return
	}

	fmt.Println("selected stop: " + result.Stop.IdfdStop.StopName + " (" + strconv.Itoa(result.Stop.IdfdStop.StopID) + ")\n")

	for _, departure := range result.Departures {

		plu := ""
		if departure.Countdown != 1 {
			plu = "s"
		}

		fmt.Printf("route %-5s due in %-2s minute%s --> %s\n",
				   departure.ServingLine.Number,
				   strconv.Itoa(departure.Countdown),
				   plu,
				   departure.ServingLine.Direction)
	}

}
<|endoftext|>


<commit_before>// Package dequeuer retrieves jobs from the database and does some work.
package dequeuer

import (
	"context"
	"errors"
	"fmt"
	"log"
	"math/rand"
	"sync"
	"time"

	metrics "github.com/kevinburke/go-simple-metrics"
	"github.com/kevinburke/rickover/models/db"
	"github.com/kevinburke/rickover/models/jobs"
	"github.com/kevinburke/rickover/models/queued_jobs"
	"github.com/kevinburke/rickover/newmodels"
	"github.com/kevinburke/rickover/services"
	"github.com/kevinburke/rickover/setup"
	"golang.org/x/sync/errgroup"
)

func init() {
	rand.Seed(time.Now().UnixNano())
}

type WorkServer struct {
	processor       *services.JobProcessor
	stuckJobTimeout time.Duration
}

type Config struct {
	// Database connector, for example db.DatabaseURLConnector. If nil,
	// db.DefaultConnection is used.
	Connector db.Connector
	// Number of open connections to the database
	NumConns        int
	Processor       *services.JobProcessor
	StuckJobTimeout time.Duration
}

// New creates a new WorkServer.
func New(ctx context.Context, cfg Config) (WorkServer, error) {
	if err := setup.DB(ctx, cfg.Connector, cfg.NumConns); err != nil {
		return WorkServer{}, err
	}
	if cfg.StuckJobTimeout == 0 {
		cfg.StuckJobTimeout = 7 * time.Minute
	}
	return WorkServer{processor: cfg.Processor, stuckJobTimeout: cfg.StuckJobTimeout}, nil
}

// How long to wait before marking a job as "stuck"
const DefaultStuckJobTimeout = 7 * time.Minute

// Run starts the WorkServer and several daemons (to measure queue depth,
// process "stuck" jobs)
func (w *WorkServer) Run(ctx context.Context) error {
	group, errctx := errgroup.WithContext(ctx)
	group.Go(func() error {
		pools, err := CreatePools(ctx, w.processor, 200*time.Millisecond)
		if err != nil {
			return err
		}
		select {
		case <-errctx.Done():
			shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
			defer cancel()
			for _, p := range pools {
				go func(p *Pool) {
					p.Shutdown(shutdownCtx)
				}(p)
			}
		}
		return nil
	})
	group.Go(func() error {
		setup.MeasureActiveQueries(errctx, 1*time.Second)
		return nil
	})
	group.Go(func() error {
		setup.MeasureQueueDepth(errctx, 5*time.Second)
		return nil
	})
	group.Go(func() error {
		setup.MeasureInProgressJobs(errctx, 1*time.Second)
		return nil
	})
	group.Go(func() error {
		// Every minute, check for in-progress jobs that haven't been updated for
		// 7 minutes, and mark them as failed.
		services.WatchStuckJobs(errctx, 1*time.Minute, w.stuckJobTimeout)
		return nil
	})
	return group.Wait()
}

func NewPool(ctx context.Context, name string) *Pool {
	tctx, cancel := context.WithCancel(ctx)
	return &Pool{
		Name:   name,
		ctx:    tctx,
		cancel: cancel,
	}
}

type Pools []*Pool

// NumDequeuers returns the total number of dequeuers across all pools.
func (ps Pools) NumDequeuers() int {
	dequeuerCount := 0
	for _, pool := range ps {
		dequeuerCount = dequeuerCount + len(pool.Dequeuers)
	}
	return dequeuerCount
}

// CreatePools creates job pools for all jobs in the database. The provided
// Worker w will be shared between all dequeuers, so it must be thread safe.
func CreatePools(ctx context.Context, w Worker, maxInitialJitter time.Duration) (Pools, error) {
	jobs, err := jobs.GetAll()
	if err != nil {
		return Pools{}, err
	}

	pools := make([]*Pool, len(jobs))
	var g errgroup.Group
	for i, job := range jobs {
		// Copy these so we don't have a concurrency/race problem when the
		// counter iterates
		i := i
		name := job.Name
		concurrency := job.Concurrency
		g.Go(func() error {
			p := NewPool(ctx, name)
			var innerg errgroup.Group
			for j := int16(0); j < concurrency; j++ {
				innerg.Go(func() error {
					time.Sleep(time.Duration(rand.Float64()) * maxInitialJitter)
					err := p.AddDequeuer(ctx, w)
					if err != nil {
						log.Print(err)
					}
					return err
				})
			}
			if err := innerg.Wait(); err != nil {
				return err
			}
			pools[i] = p
			return nil
		})
	}
	if err := g.Wait(); err != nil {
		return nil, err
	}
	return pools, nil
}

// A Pool contains an array of dequeuers, all of which perform work for the
// same models.Job.
type Pool struct {
	Dequeuers []*Dequeuer
	Name      string
	mu        sync.Mutex
	wg        sync.WaitGroup
	ctx       context.Context
	cancel    context.CancelFunc
}

type Dequeuer struct {
	ID     int
	W      Worker
	ctx    context.Context
	cancel context.CancelFunc
}

// A Worker does some work with a QueuedJob. Worker implementations may be
// shared and should be threadsafe.
type Worker interface {
	// DoWork is responsible for performing work and either updating the job
	// status in the database or waiting for the status to be updated by
	// another thread. Success and failure for the job are marked by hitting
	// services.HandleStatusCallback, or POST /v1/jobs/:job-name/:job-id (over
	// HTTP).
	//
	// A good pattern is for DoWork to make a HTTP request to a downstream
	// service, and then for that service to make a HTTP callback to report
	// success or failure.
	//
	// If DoWork is unable to get the work to be done, it should call
	// HandleStatusCallback with a failed callback; errors are logged, but
	// otherwise nothing else is done with them.
	DoWork(context.Context, *newmodels.QueuedJob) error
}

// AddDequeuer adds a Dequeuer to the Pool. w should be the work that the
// Dequeuer will do with a dequeued job.
func (p *Pool) AddDequeuer(ctx context.Context, w Worker) error {
	select {
	case <-ctx.Done():
		return errPoolShutdown
	default:
	}
	tctx, cancel := context.WithCancel(ctx)
	p.mu.Lock()
	defer p.mu.Unlock()
	d := &Dequeuer{
		ID:     len(p.Dequeuers) + 1,
		W:      w,
		ctx:    tctx,
		cancel: cancel,
	}
	p.Dequeuers = append(p.Dequeuers, d)
	p.wg.Add(1)
	go func() {
		d.Work(p.Name, &p.wg)
		// work returned, so it won't do anything more - no point in keeping the
		// dequeuer around
		p.RemoveDequeuer()
	}()
	return nil
}

var errEmptyPool = errors.New("dequeuer: no workers left to dequeue")
var errPoolShutdown = errors.New("dequeuer: cannot add worker because the pool is shutting down")

// RemoveDequeuer removes a dequeuer from the pool and sends that dequeuer
// a shutdown signal.
func (p *Pool) RemoveDequeuer() error {
	p.mu.Lock()
	defer p.mu.Unlock()
	if len(p.Dequeuers) == 0 {
		return errEmptyPool
	}
	dq := p.Dequeuers[0]
	dq.cancel()
	p.Dequeuers = append(p.Dequeuers[:0], p.Dequeuers[1:]...)
	return nil
}

func (p *Pool) Len() int {
	p.mu.Lock()
	defer p.mu.Unlock()
	return len(p.Dequeuers)
}

// Shutdown all workers in the pool.
func (p *Pool) Shutdown(ctx context.Context) error {
	p.cancel()
	l := len(p.Dequeuers)
	for i := 0; i < l; i++ {
		err := p.RemoveDequeuer()
		if err != nil {
			return err
		}
	}
	done := make(chan struct{}, 1)
	go func() {
		p.wg.Wait()
		done <- struct{}{}
	}()
	select {
	case <-done:
		return nil
	case <-ctx.Done():
		return ctx.Err()
	}
}

func (d *Dequeuer) Work(name string, wg *sync.WaitGroup) {
	defer wg.Done()
	failedAcquireCount := uint32(0)
	waitDuration := time.Duration(0)
	for {
		select {
		case <-d.ctx.Done():
			log.Printf("%s worker %d quitting\n", name, d.ID)
			return

		case <-time.After(waitDuration):
			start := time.Now()
			qj, err := queued_jobs.Acquire(context.TODO(), name, d.ID)
			go metrics.Time("acquire.latency", time.Since(start))
			if err == nil {
				failedAcquireCount = 0
				waitDuration = time.Duration(0)
				err = d.W.DoWork(d.ctx, qj)
				if err != nil {
					log.Printf("worker: Error processing job %s: %s", qj.ID.String(), err)
					go metrics.Increment(fmt.Sprintf("dequeue.%s.error", name))
				} else {
					go metrics.Increment(fmt.Sprintf("dequeue.%s.success", name))
				}
			} else {
				failedAcquireCount++
				waitDuration = time.Duration(0)
			}
		}
	}
}
<commit_msg>dequeuer: fix lint error<commit_after>// Package dequeuer retrieves jobs from the database and does some work.
package dequeuer

import (
	"context"
	"errors"
	"fmt"
	"log"
	"math/rand"
	"sync"
	"time"

	metrics "github.com/kevinburke/go-simple-metrics"
	"github.com/kevinburke/rickover/models/db"
	"github.com/kevinburke/rickover/models/jobs"
	"github.com/kevinburke/rickover/models/queued_jobs"
	"github.com/kevinburke/rickover/newmodels"
	"github.com/kevinburke/rickover/services"
	"github.com/kevinburke/rickover/setup"
	"golang.org/x/sync/errgroup"
)

func init() {
	rand.Seed(time.Now().UnixNano())
}

type WorkServer struct {
	processor       *services.JobProcessor
	stuckJobTimeout time.Duration
}

type Config struct {
	// Database connector, for example db.DatabaseURLConnector. If nil,
	// db.DefaultConnection is used.
	Connector db.Connector
	// Number of open connections to the database
	NumConns        int
	Processor       *services.JobProcessor
	StuckJobTimeout time.Duration
}

// New creates a new WorkServer.
func New(ctx context.Context, cfg Config) (WorkServer, error) {
	if err := setup.DB(ctx, cfg.Connector, cfg.NumConns); err != nil {
		return WorkServer{}, err
	}
	if cfg.StuckJobTimeout == 0 {
		cfg.StuckJobTimeout = 7 * time.Minute
	}
	return WorkServer{processor: cfg.Processor, stuckJobTimeout: cfg.StuckJobTimeout}, nil
}

// How long to wait before marking a job as "stuck"
const DefaultStuckJobTimeout = 7 * time.Minute

// Run starts the WorkServer and several daemons (to measure queue depth,
// process "stuck" jobs)
func (w *WorkServer) Run(ctx context.Context) error {
	group, errctx := errgroup.WithContext(ctx)
	group.Go(func() error {
		pools, err := CreatePools(ctx, w.processor, 200*time.Millisecond)
		if err != nil {
			return err
		}
		<-errctx.Done()
		shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
		defer cancel()
		for _, p := range pools {
			go func(p *Pool) {
				p.Shutdown(shutdownCtx)
			}(p)
		}
		return nil
	})
	group.Go(func() error {
		setup.MeasureActiveQueries(errctx, 1*time.Second)
		return nil
	})
	group.Go(func() error {
		setup.MeasureQueueDepth(errctx, 5*time.Second)
		return nil
	})
	group.Go(func() error {
		setup.MeasureInProgressJobs(errctx, 1*time.Second)
		return nil
	})
	group.Go(func() error {
		// Every minute, check for in-progress jobs that haven't been updated for
		// 7 minutes, and mark them as failed.
		services.WatchStuckJobs(errctx, 1*time.Minute, w.stuckJobTimeout)
		return nil
	})
	return group.Wait()
}

func NewPool(ctx context.Context, name string) *Pool {
	tctx, cancel := context.WithCancel(ctx)
	return &Pool{
		Name:   name,
		ctx:    tctx,
		cancel: cancel,
	}
}

type Pools []*Pool

// NumDequeuers returns the total number of dequeuers across all pools.
func (ps Pools) NumDequeuers() int {
	dequeuerCount := 0
	for _, pool := range ps {
		dequeuerCount = dequeuerCount + len(pool.Dequeuers)
	}
	return dequeuerCount
}

// CreatePools creates job pools for all jobs in the database. The provided
// Worker w will be shared between all dequeuers, so it must be thread safe.
func CreatePools(ctx context.Context, w Worker, maxInitialJitter time.Duration) (Pools, error) {
	jobs, err := jobs.GetAll()
	if err != nil {
		return Pools{}, err
	}

	pools := make([]*Pool, len(jobs))
	var g errgroup.Group
	for i, job := range jobs {
		// Copy these so we don't have a concurrency/race problem when the
		// counter iterates
		i := i
		name := job.Name
		concurrency := job.Concurrency
		g.Go(func() error {
			p := NewPool(ctx, name)
			var innerg errgroup.Group
			for j := int16(0); j < concurrency; j++ {
				innerg.Go(func() error {
					time.Sleep(time.Duration(rand.Float64()) * maxInitialJitter)
					err := p.AddDequeuer(ctx, w)
					if err != nil {
						log.Print(err)
					}
					return err
				})
			}
			if err := innerg.Wait(); err != nil {
				return err
			}
			pools[i] = p
			return nil
		})
	}
	if err := g.Wait(); err != nil {
		return nil, err
	}
	return pools, nil
}

// A Pool contains an array of dequeuers, all of which perform work for the
// same models.Job.
type Pool struct {
	Dequeuers []*Dequeuer
	Name      string
	mu        sync.Mutex
	wg        sync.WaitGroup
	ctx       context.Context
	cancel    context.CancelFunc
}

type Dequeuer struct {
	ID     int
	W      Worker
	ctx    context.Context
	cancel context.CancelFunc
}

// A Worker does some work with a QueuedJob. Worker implementations may be
// shared and should be threadsafe.
type Worker interface {
	// DoWork is responsible for performing work and either updating the job
	// status in the database or waiting for the status to be updated by
	// another thread. Success and failure for the job are marked by hitting
	// services.HandleStatusCallback, or POST /v1/jobs/:job-name/:job-id (over
	// HTTP).
	//
	// A good pattern is for DoWork to make a HTTP request to a downstream
	// service, and then for that service to make a HTTP callback to report
	// success or failure.
	//
	// If DoWork is unable to get the work to be done, it should call
	// HandleStatusCallback with a failed callback; errors are logged, but
	// otherwise nothing else is done with them.
	DoWork(context.Context, *newmodels.QueuedJob) error
}

// AddDequeuer adds a Dequeuer to the Pool. w should be the work that the
// Dequeuer will do with a dequeued job.
func (p *Pool) AddDequeuer(ctx context.Context, w Worker) error {
	select {
	case <-ctx.Done():
		return errPoolShutdown
	default:
	}
	tctx, cancel := context.WithCancel(ctx)
	p.mu.Lock()
	defer p.mu.Unlock()
	d := &Dequeuer{
		ID:     len(p.Dequeuers) + 1,
		W:      w,
		ctx:    tctx,
		cancel: cancel,
	}
	p.Dequeuers = append(p.Dequeuers, d)
	p.wg.Add(1)
	go func() {
		d.Work(p.Name, &p.wg)
		// work returned, so it won't do anything more - no point in keeping the
		// dequeuer around
		p.RemoveDequeuer()
	}()
	return nil
}

var errEmptyPool = errors.New("dequeuer: no workers left to dequeue")
var errPoolShutdown = errors.New("dequeuer: cannot add worker because the pool is shutting down")

// RemoveDequeuer removes a dequeuer from the pool and sends that dequeuer
// a shutdown signal.
func (p *Pool) RemoveDequeuer() error {
	p.mu.Lock()
	defer p.mu.Unlock()
	if len(p.Dequeuers) == 0 {
		return errEmptyPool
	}
	dq := p.Dequeuers[0]
	dq.cancel()
	p.Dequeuers = append(p.Dequeuers[:0], p.Dequeuers[1:]...)
	return nil
}

func (p *Pool) Len() int {
	p.mu.Lock()
	defer p.mu.Unlock()
	return len(p.Dequeuers)
}

// Shutdown all workers in the pool.
func (p *Pool) Shutdown(ctx context.Context) error {
	p.cancel()
	l := len(p.Dequeuers)
	for i := 0; i < l; i++ {
		err := p.RemoveDequeuer()
		if err != nil {
			return err
		}
	}
	done := make(chan struct{}, 1)
	go func() {
		p.wg.Wait()
		done <- struct{}{}
	}()
	select {
	case <-done:
		return nil
	case <-ctx.Done():
		return ctx.Err()
	}
}

func (d *Dequeuer) Work(name string, wg *sync.WaitGroup) {
	defer wg.Done()
	failedAcquireCount := uint32(0)
	waitDuration := time.Duration(0)
	for {
		select {
		case <-d.ctx.Done():
			log.Printf("%s worker %d quitting\n", name, d.ID)
			return

		case <-time.After(waitDuration):
			start := time.Now()
			qj, err := queued_jobs.Acquire(context.TODO(), name, d.ID)
			go metrics.Time("acquire.latency", time.Since(start))
			if err == nil {
				failedAcquireCount = 0
				waitDuration = time.Duration(0)
				err = d.W.DoWork(d.ctx, qj)
				if err != nil {
					log.Printf("worker: Error processing job %s: %s", qj.ID.String(), err)
					go metrics.Increment(fmt.Sprintf("dequeue.%s.error", name))
				} else {
					go metrics.Increment(fmt.Sprintf("dequeue.%s.success", name))
				}
			} else {
				failedAcquireCount++
				waitDuration = time.Duration(0)
			}
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ledcmd

import (
	"context"
	"net/http"
	"time"

	"go.chromium.org/luci/auth"
	swarming "go.chromium.org/luci/common/api/swarming/swarming/v1"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/gcloud/googleoauth"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/led/job"
	"go.chromium.org/luci/led/job/jobexport"
)

// LaunchSwarmingOpts are the options for LaunchSwarming.
type LaunchSwarmingOpts struct {
	// If true, just generates the NewTaskRequest but does not send it to swarming
	// (SwarmingRpcsTaskRequestMetadata will be nil).
	DryRun bool

	// Must be a unique user identity string and must not be empty.
	//
	// Picking a bad value here means that generated logdog prefixes will
	// possibly collide, and the swarming task's User field will be misreported.
	//
	// See GetUID to obtain a standardized value here.
	UserID string

	// If launched from within a swarming task, this will be the current swarming
	// task's task id to be attached as the parent of the launched task.
	ParentTaskId string

	// A path, relative to ${ISOLATED_OUTDIR} of where to place the final
	// build.proto from this build. If omitted, the build.proto will not be
	// dumped.
	FinalBuildProto string

	KitchenSupport job.KitchenSupport

	// A flag for swarming/ResultDB integration on the launched task.
	ResultDB job.RDBEnablement
}

// GetUID derives a user id string from the Authenticator for use with
// LaunchSwarming.
//
// If the given authenticator has the userinfo.email scope, this will be the
// email associated with the Authenticator. Otherwise, this will be
// 'uid:<opaque user id>'.
func GetUID(ctx context.Context, authenticator *auth.Authenticator) (string, error) {
	tok, err := authenticator.GetAccessToken(time.Minute)
	if err != nil {
		return "", errors.Annotate(err, "getting access token").Err()
	}
	info, err := googleoauth.GetTokenInfo(ctx, googleoauth.TokenInfoParams{
		AccessToken: tok.AccessToken,
	})
	if info.Email != "" {
		return info.Email, nil
	}
	return "uid:" + info.Sub, nil
}

// LaunchSwarming launches the given job Definition on swarming, returning the
// NewTaskRequest launched, as well as the launch metadata.
func LaunchSwarming(ctx context.Context, authClient *http.Client, jd *job.Definition, opts LaunchSwarmingOpts) (*swarming.SwarmingRpcsNewTaskRequest, *swarming.SwarmingRpcsTaskRequestMetadata, error) {
	if opts.KitchenSupport == nil {
		opts.KitchenSupport = job.NoKitchenSupport()
	}
	if opts.UserID == "" {
		return nil, nil, errors.New("opts.UserID is empty")
	}

	logging.Infof(ctx, "building swarming task")
	if err := jd.FlattenToSwarming(ctx, opts.UserID, opts.ParentTaskId, opts.KitchenSupport, opts.ResultDB); err != nil {
		return nil, nil, errors.Annotate(err, "failed to flatten job definition to swarming").Err()
	}

	st, err := jobexport.ToSwarmingNewTask(jd.GetSwarming(), jd.UserPayload, jd.CasUserPayload)
	if err != nil {
		return nil, nil, err
	}
	logging.Infof(ctx, "building swarming task: done")

	if opts.DryRun {
		return st, nil, nil
	}

	swarm := newSwarmClient(authClient, jd.Info().SwarmingHostname())

	logging.Infof(ctx, "launching swarming task")
	req, err := swarm.Tasks.New(st).Do()
	if err != nil {
		return nil, nil, err
	}
	logging.Infof(ctx, "launching swarming task: done")

	return st, req, nil
}
<commit_msg>[led] Check err return from googleoauth.GetAccessTokenInfo()<commit_after>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ledcmd

import (
	"context"
	"net/http"
	"time"

	"go.chromium.org/luci/auth"
	swarming "go.chromium.org/luci/common/api/swarming/swarming/v1"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/gcloud/googleoauth"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/led/job"
	"go.chromium.org/luci/led/job/jobexport"
)

// LaunchSwarmingOpts are the options for LaunchSwarming.
type LaunchSwarmingOpts struct {
	// If true, just generates the NewTaskRequest but does not send it to swarming
	// (SwarmingRpcsTaskRequestMetadata will be nil).
	DryRun bool

	// Must be a unique user identity string and must not be empty.
	//
	// Picking a bad value here means that generated logdog prefixes will
	// possibly collide, and the swarming task's User field will be misreported.
	//
	// See GetUID to obtain a standardized value here.
	UserID string

	// If launched from within a swarming task, this will be the current swarming
	// task's task id to be attached as the parent of the launched task.
	ParentTaskId string

	// A path, relative to ${ISOLATED_OUTDIR} of where to place the final
	// build.proto from this build. If omitted, the build.proto will not be
	// dumped.
	FinalBuildProto string

	KitchenSupport job.KitchenSupport

	// A flag for swarming/ResultDB integration on the launched task.
	ResultDB job.RDBEnablement
}

// GetUID derives a user id string from the Authenticator for use with
// LaunchSwarming.
//
// If the given authenticator has the userinfo.email scope, this will be the
// email associated with the Authenticator. Otherwise, this will be
// 'uid:<opaque user id>'.
func GetUID(ctx context.Context, authenticator *auth.Authenticator) (string, error) {
	tok, err := authenticator.GetAccessToken(time.Minute)
	if err != nil {
		return "", errors.Annotate(err, "getting access token").Err()
	}
	info, err := googleoauth.GetTokenInfo(ctx, googleoauth.TokenInfoParams{
		AccessToken: tok.AccessToken,
	})
	if err != nil {
		return "", errors.Annotate(err, "getting access token info").Err()
	}
	if info.Email != "" {
		return info.Email, nil
	}
	return "uid:" + info.Sub, nil
}

// LaunchSwarming launches the given job Definition on swarming, returning the
// NewTaskRequest launched, as well as the launch metadata.
func LaunchSwarming(ctx context.Context, authClient *http.Client, jd *job.Definition, opts LaunchSwarmingOpts) (*swarming.SwarmingRpcsNewTaskRequest, *swarming.SwarmingRpcsTaskRequestMetadata, error) {
	if opts.KitchenSupport == nil {
		opts.KitchenSupport = job.NoKitchenSupport()
	}
	if opts.UserID == "" {
		return nil, nil, errors.New("opts.UserID is empty")
	}

	logging.Infof(ctx, "building swarming task")
	if err := jd.FlattenToSwarming(ctx, opts.UserID, opts.ParentTaskId, opts.KitchenSupport, opts.ResultDB); err != nil {
		return nil, nil, errors.Annotate(err, "failed to flatten job definition to swarming").Err()
	}

	st, err := jobexport.ToSwarmingNewTask(jd.GetSwarming(), jd.UserPayload, jd.CasUserPayload)
	if err != nil {
		return nil, nil, err
	}
	logging.Infof(ctx, "building swarming task: done")

	if opts.DryRun {
		return st, nil, nil
	}

	swarm := newSwarmClient(authClient, jd.Info().SwarmingHostname())

	logging.Infof(ctx, "launching swarming task")
	req, err := swarm.Tasks.New(st).Do()
	if err != nil {
		return nil, nil, err
	}
	logging.Infof(ctx, "launching swarming task: done")

	return st, req, nil
}
<|endoftext|>


<commit_before>package cmd

import (
	"encoding/csv"
	"fmt"
	"os"
	"strings"
	"time"
	"unicode/utf8"

	"github.com/howeyc/ledger"
	"github.com/howeyc/ledger/internal/decimal"
	"github.com/jbrukh/bayesian"
	"github.com/spf13/cobra"
)

var csvDateFormat string
var destAccSearch string
var negateAmount bool
var allowMatching bool
var fieldDelimiter string
var scaleFactor float64

// importCmd represents the import command
var importCmd = &cobra.Command{
	Use:   "import <account-substring> <csv-file>",
	Args:  cobra.ExactArgs(2),
	Short: "Import transactions from csv to ledger format",
	Run: func(cmd *cobra.Command, args []string) {
		var accountSubstring, csvFileName string
		accountSubstring = args[0]
		csvFileName = args[1]

		decScale := decimal.NewFromFloat(scaleFactor)

		csvFileReader, err := os.Open(csvFileName)
		if err != nil {
			fmt.Println("CSV: ", err)
			return
		}
		defer csvFileReader.Close()

		generalLedger, parseError := ledger.ParseLedgerFile(ledgerFilePath)
		if parseError != nil {
			fmt.Printf("%s:%s\n", ledgerFilePath, parseError.Error())
			return
		}

		var matchingAccount string
		matchingAccounts := ledger.GetBalances(generalLedger, []string{accountSubstring})
		if len(matchingAccounts) < 1 {
			fmt.Println("Unable to find matching account.")
			return
		}
		matchingAccount = matchingAccounts[len(matchingAccounts)-1].Name

		allAccounts := ledger.GetBalances(generalLedger, []string{})

		csvReader := csv.NewReader(csvFileReader)
		csvReader.Comma, _ = utf8.DecodeRuneInString(fieldDelimiter)
		csvRecords, cerr := csvReader.ReadAll()
		if cerr != nil {
			fmt.Println("CSV parse error:", cerr.Error())
			return
		}

		classes := make([]bayesian.Class, len(allAccounts))
		for i, bal := range allAccounts {
			classes[i] = bayesian.Class(bal.Name)
		}
		classifier := bayesian.NewClassifier(classes...)
		for _, tran := range generalLedger {
			payeeWords := strings.Fields(tran.Payee)
			for _, accChange := range tran.AccountChanges {
				if strings.Contains(accChange.Name, destAccSearch) {
					classifier.Learn(payeeWords, bayesian.Class(accChange.Name))
				}
			}
		}

		// Find columns from header
		var dateColumn, payeeColumn, amountColumn, commentColumn int
		dateColumn, payeeColumn, amountColumn, commentColumn = -1, -1, -1, -1
		for fieldIndex, fieldName := range csvRecords[0] {
			fieldName = strings.ToLower(fieldName)
			if strings.Contains(fieldName, "date") {
				dateColumn = fieldIndex
			} else if strings.Contains(fieldName, "description") {
				payeeColumn = fieldIndex
			} else if strings.Contains(fieldName, "payee") {
				payeeColumn = fieldIndex
			} else if strings.Contains(fieldName, "amount") {
				amountColumn = fieldIndex
			} else if strings.Contains(fieldName, "expense") {
				amountColumn = fieldIndex
			} else if strings.Contains(fieldName, "note") {
				commentColumn = fieldIndex
			} else if strings.Contains(fieldName, "comment") {
				commentColumn = fieldIndex
			}
		}

		if dateColumn < 0 || payeeColumn < 0 || amountColumn < 0 {
			fmt.Println("Unable to find columns required from header field names.")
			return
		}

		expenseAccount := ledger.Account{Name: "unknown:unknown", Balance: decimal.Zero}
		csvAccount := ledger.Account{Name: matchingAccount, Balance: decimal.Zero}
		for _, record := range csvRecords[1:] {
			inputPayeeWords := strings.Fields(record[payeeColumn])
			csvDate, _ := time.Parse(csvDateFormat, record[dateColumn])
			if allowMatching || !existingTransaction(generalLedger, csvDate, inputPayeeWords[0]) {
				// Classify into expense account
				scores, likely, _ := classifier.LogScores(inputPayeeWords)
				if likely >= 0 {
					matchScore := 0.0
					matchIdx := -1
					for j, score := range scores {
						if j == 0 {
							matchScore = score
						}
						if string(classifier.Classes[j]) == csvAccount.Name {
							continue
						}
						if score > matchScore {
							matchScore = score
							matchIdx = j
						}
					}
					if matchIdx >= 0 {
						expenseAccount.Name = string(classifier.Classes[matchIdx])
					} else {
						expenseAccount.Name = string(classifier.Classes[likely])
					}
				}

				// Parse error, set to zero
				if dec, derr := decimal.NewFromString(record[amountColumn]); derr != nil {
					expenseAccount.Balance = decimal.Zero
				} else {
					expenseAccount.Balance = dec
				}

				// Negate amount if required
				if negateAmount {
					expenseAccount.Balance = expenseAccount.Balance.Neg()
				}

				// Apply scale
				expenseAccount.Balance = expenseAccount.Balance.Mul(decScale)

				// Csv amount is the negative of the expense amount
				csvAccount.Balance = expenseAccount.Balance.Neg()

				// Create valid transaction for print in ledger format
				trans := &ledger.Transaction{Date: csvDate, Payee: record[payeeColumn]}
				trans.AccountChanges = []ledger.Account{csvAccount, expenseAccount}

				// Comment
				if commentColumn >= 0 && record[commentColumn] != "" {
					trans.Comments = []string{";" + record[commentColumn]}
				}
				PrintTransaction(trans, 80)
			}
		}

	},
}

func init() {
	rootCmd.AddCommand(importCmd)

	importCmd.Flags().BoolVar(&negateAmount, "neg", false, "Negate amount column value.")
	importCmd.Flags().BoolVar(&allowMatching, "allow-matching", false, "Have output include imported transactions that\nmatch existing ledger transactions.")
	importCmd.Flags().Float64Var(&scaleFactor, "scale", 1.0, "Scale factor to multiply against every imported amount.")
	importCmd.Flags().StringVar(&destAccSearch, "set-search", "Expense", "Search string used to find set of accounts for classification.")
	importCmd.Flags().StringVar(&csvDateFormat, "date-format", "01/02/2006", "Date format.")
	importCmd.Flags().StringVar(&fieldDelimiter, "delimiter", ",", "Field delimiter.")
}

func existingTransaction(generalLedger []*ledger.Transaction, transDate time.Time, payee string) bool {
	for _, trans := range generalLedger {
		if trans.Date == transDate && strings.HasPrefix(trans.Payee, payee) {
			return true
		}
	}
	return false
}
<commit_msg>fix: have import existing match on full payee string<commit_after>package cmd

import (
	"encoding/csv"
	"fmt"
	"os"
	"strings"
	"time"
	"unicode/utf8"

	"github.com/howeyc/ledger"
	"github.com/howeyc/ledger/internal/decimal"
	"github.com/jbrukh/bayesian"
	"github.com/spf13/cobra"
)

var csvDateFormat string
var destAccSearch string
var negateAmount bool
var allowMatching bool
var fieldDelimiter string
var scaleFactor float64

// importCmd represents the import command
var importCmd = &cobra.Command{
	Use:   "import <account-substring> <csv-file>",
	Args:  cobra.ExactArgs(2),
	Short: "Import transactions from csv to ledger format",
	Run: func(cmd *cobra.Command, args []string) {
		var accountSubstring, csvFileName string
		accountSubstring = args[0]
		csvFileName = args[1]

		decScale := decimal.NewFromFloat(scaleFactor)

		csvFileReader, err := os.Open(csvFileName)
		if err != nil {
			fmt.Println("CSV: ", err)
			return
		}
		defer csvFileReader.Close()

		generalLedger, parseError := ledger.ParseLedgerFile(ledgerFilePath)
		if parseError != nil {
			fmt.Printf("%s:%s\n", ledgerFilePath, parseError.Error())
			return
		}

		var matchingAccount string
		matchingAccounts := ledger.GetBalances(generalLedger, []string{accountSubstring})
		if len(matchingAccounts) < 1 {
			fmt.Println("Unable to find matching account.")
			return
		}
		matchingAccount = matchingAccounts[len(matchingAccounts)-1].Name

		allAccounts := ledger.GetBalances(generalLedger, []string{})

		csvReader := csv.NewReader(csvFileReader)
		csvReader.Comma, _ = utf8.DecodeRuneInString(fieldDelimiter)
		csvRecords, cerr := csvReader.ReadAll()
		if cerr != nil {
			fmt.Println("CSV parse error:", cerr.Error())
			return
		}

		classes := make([]bayesian.Class, len(allAccounts))
		for i, bal := range allAccounts {
			classes[i] = bayesian.Class(bal.Name)
		}
		classifier := bayesian.NewClassifier(classes...)
		for _, tran := range generalLedger {
			payeeWords := strings.Fields(tran.Payee)
			for _, accChange := range tran.AccountChanges {
				if strings.Contains(accChange.Name, destAccSearch) {
					classifier.Learn(payeeWords, bayesian.Class(accChange.Name))
				}
			}
		}

		// Find columns from header
		var dateColumn, payeeColumn, amountColumn, commentColumn int
		dateColumn, payeeColumn, amountColumn, commentColumn = -1, -1, -1, -1
		for fieldIndex, fieldName := range csvRecords[0] {
			fieldName = strings.ToLower(fieldName)
			if strings.Contains(fieldName, "date") {
				dateColumn = fieldIndex
			} else if strings.Contains(fieldName, "description") {
				payeeColumn = fieldIndex
			} else if strings.Contains(fieldName, "payee") {
				payeeColumn = fieldIndex
			} else if strings.Contains(fieldName, "amount") {
				amountColumn = fieldIndex
			} else if strings.Contains(fieldName, "expense") {
				amountColumn = fieldIndex
			} else if strings.Contains(fieldName, "note") {
				commentColumn = fieldIndex
			} else if strings.Contains(fieldName, "comment") {
				commentColumn = fieldIndex
			}
		}

		if dateColumn < 0 || payeeColumn < 0 || amountColumn < 0 {
			fmt.Println("Unable to find columns required from header field names.")
			return
		}

		expenseAccount := ledger.Account{Name: "unknown:unknown", Balance: decimal.Zero}
		csvAccount := ledger.Account{Name: matchingAccount, Balance: decimal.Zero}
		for _, record := range csvRecords[1:] {
			inputPayeeWords := strings.Fields(record[payeeColumn])
			csvDate, _ := time.Parse(csvDateFormat, record[dateColumn])
			if allowMatching || !existingTransaction(generalLedger, csvDate, record[payeeColumn]) {
				// Classify into expense account
				scores, likely, _ := classifier.LogScores(inputPayeeWords)
				if likely >= 0 {
					matchScore := 0.0
					matchIdx := -1
					for j, score := range scores {
						if j == 0 {
							matchScore = score
						}
						if string(classifier.Classes[j]) == csvAccount.Name {
							continue
						}
						if score > matchScore {
							matchScore = score
							matchIdx = j
						}
					}
					if matchIdx >= 0 {
						expenseAccount.Name = string(classifier.Classes[matchIdx])
					} else {
						expenseAccount.Name = string(classifier.Classes[likely])
					}
				}

				// Parse error, set to zero
				if dec, derr := decimal.NewFromString(record[amountColumn]); derr != nil {
					expenseAccount.Balance = decimal.Zero
				} else {
					expenseAccount.Balance = dec
				}

				// Negate amount if required
				if negateAmount {
					expenseAccount.Balance = expenseAccount.Balance.Neg()
				}

				// Apply scale
				expenseAccount.Balance = expenseAccount.Balance.Mul(decScale)

				// Csv amount is the negative of the expense amount
				csvAccount.Balance = expenseAccount.Balance.Neg()

				// Create valid transaction for print in ledger format
				trans := &ledger.Transaction{Date: csvDate, Payee: record[payeeColumn]}
				trans.AccountChanges = []ledger.Account{csvAccount, expenseAccount}

				// Comment
				if commentColumn >= 0 && record[commentColumn] != "" {
					trans.Comments = []string{";" + record[commentColumn]}
				}
				PrintTransaction(trans, 80)
			}
		}

	},
}

func init() {
	rootCmd.AddCommand(importCmd)

	importCmd.Flags().BoolVar(&negateAmount, "neg", false, "Negate amount column value.")
	importCmd.Flags().BoolVar(&allowMatching, "allow-matching", false, "Have output include imported transactions that\nmatch existing ledger transactions.")
	importCmd.Flags().Float64Var(&scaleFactor, "scale", 1.0, "Scale factor to multiply against every imported amount.")
	importCmd.Flags().StringVar(&destAccSearch, "set-search", "Expense", "Search string used to find set of accounts for classification.")
	importCmd.Flags().StringVar(&csvDateFormat, "date-format", "01/02/2006", "Date format.")
	importCmd.Flags().StringVar(&fieldDelimiter, "delimiter", ",", "Field delimiter.")
}

func existingTransaction(generalLedger []*ledger.Transaction, transDate time.Time, payee string) bool {
	for _, trans := range generalLedger {
		if trans.Date == transDate && trans.Payee == payee {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>// Package colormap implements a basic color mapping library.
// With givin value, value minimum and value maximum (given range) returns
// RGB color from Blue for minimum value to Red for maximum value.
// Intermediate colors such as green, yellow and more might also be returned for
// values between minimum and maximum (within range).
package colormap

import "fmt"

type Range struct {
	VMin, VMax float64
}

// Correlate returns red, green, blue values represented as float64 in range [0..1].
func (r *Range) Correlate(val float64) (float64, float64, float64) {
	return Correlate(val, r.VMin, r.VMax)
}

func (r *Range) HexStr(val float64) string {
	return HexStr(val, r.VMin, r.VMax)
}

// Correlate returns red, green, blue values represented as float64 in range [0..1].
func Correlate(val, vmin, vmax float64) (float64, float64, float64) {

	var (
		vrange  float64 = vmax - vmin
		r, g, b float64 = 1, 1, 1
	)

	if val < vmin {
		val = vmin
	}
	if val > vmax {
		val = vmax
	}

	if val < (vmin + 0.25*vrange) {
		r = 0
		g = 4 * (val - vmin) / vrange
	} else if val < (vmin + 0.5*vrange) {
		r = 0
		b = 1 + 4*(vmin+0.25*vrange)/vrange
	} else if val < (vmin+0.75*vrange)/vrange {
		b = 0
		r = 4 * (val - vmin - 0.5*vrange) / vrange
	} else {
		b = 0
		g = 1 + 4*(vmin+0.75*vrange-val)/vrange
	}
	return r, g, b
}

// TODO: Think about func which returns three uint8 values.

func HexStr(val, vmin, vmax float64) string {
	const m = 0xFF
	r, g, b := Correlate(val, vmin, vmax)
	return fmt.Sprintf("%X%X%X", uint8(r*m), uint8(g*m), uint8(b*m))
}
<commit_msg>Added hash # for HexStr result.<commit_after>// Package colormap implements a basic color mapping library.
// With givin value, value minimum and value maximum (given range) returns
// RGB color from Blue for minimum value to Red for maximum value.
// Intermediate colors such as green, yellow and more might also be returned for
// values between minimum and maximum (within range).
package colormap

import "fmt"

type Range struct {
	VMin, VMax float64
}

// Correlate returns red, green, blue values represented as float64 in range [0..1].
func (r *Range) Correlate(val float64) (float64, float64, float64) {
	return Correlate(val, r.VMin, r.VMax)
}

// HexStr returns red, green, blue values represented as six hex digits string
// which begins with hash (#).
func (r *Range) HexStr(val float64) string {
	return HexStr(val, r.VMin, r.VMax)
}

// Correlate returns red, green, blue values represented as float64 in range [0..1].
func Correlate(val, vmin, vmax float64) (float64, float64, float64) {

	var (
		vrange  float64 = vmax - vmin
		r, g, b float64 = 1, 1, 1
	)

	if val < vmin {
		val = vmin
	}
	if val > vmax {
		val = vmax
	}

	if val < (vmin + 0.25*vrange) {
		r = 0
		g = 4 * (val - vmin) / vrange
	} else if val < (vmin + 0.5*vrange) {
		r = 0
		b = 1 + 4*(vmin+0.25*vrange)/vrange
	} else if val < (vmin+0.75*vrange)/vrange {
		b = 0
		r = 4 * (val - vmin - 0.5*vrange) / vrange
	} else {
		b = 0
		g = 1 + 4*(vmin+0.75*vrange-val)/vrange
	}
	return r, g, b
}

// HexStr returns red, green, blue values represented as six hex digits string
// which begins with hash (#).
func HexStr(val, vmin, vmax float64) string {
	const m = 0xFF
	r, g, b := Correlate(val, vmin, vmax)
	// TODO: Think about func which returns three uint8 values.
	return fmt.Sprintf("#%X%X%X", uint8(r*m), uint8(g*m), uint8(b*m))
}
<|endoftext|>