<commit_before>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// go-to-protobuf generates a Protobuf IDL from a Go struct, respecting any
// existing IDL tags on the Go struct.
package protobuf

import (
	"bytes"
	"fmt"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"k8s.io/gengo/args"
	"k8s.io/gengo/generator"
	"k8s.io/gengo/namer"
	"k8s.io/gengo/parser"
	"k8s.io/gengo/types"

	flag "github.com/spf13/pflag"
)

type Generator struct {
	Common               args.GeneratorArgs
	APIMachineryPackages string
	Packages             string
	OutputBase           string
	VendorOutputBase     string
	ProtoImport          []string
	Conditional          string
	Clean                bool
	OnlyIDL              bool
	KeepGogoproto        bool
	SkipGeneratedRewrite bool
	DropEmbeddedFields   string
}

func New() *Generator {
	sourceTree := args.DefaultSourceTree()
	common := args.GeneratorArgs{
		OutputBase:       sourceTree,
		GoHeaderFilePath: filepath.Join(sourceTree, "k8s.io/kubernetes/hack/boilerplate/boilerplate.go.txt"),
	}
	defaultProtoImport := filepath.Join(sourceTree, "k8s.io", "kubernetes", "vendor", "github.com", "gogo", "protobuf", "protobuf")
	cwd, err := os.Getwd()
	if err != nil {
		log.Fatalf("Cannot get current directory.")
	}
	return &Generator{
		Common:           common,
		OutputBase:       sourceTree,
		VendorOutputBase: filepath.Join(cwd, "vendor"),
		ProtoImport:      []string{defaultProtoImport},
		APIMachineryPackages: strings.Join([]string{
			`+k8s.io/apimachinery/pkg/util/intstr`,
			`+k8s.io/apimachinery/pkg/api/resource`,
			`+k8s.io/apimachinery/pkg/runtime/schema`,
			`+k8s.io/apimachinery/pkg/runtime`,
			`k8s.io/apimachinery/pkg/apis/meta/v1`,
			`k8s.io/apimachinery/pkg/apis/meta/v1beta1`,
		}, ","),
		Packages:           "",
		DropEmbeddedFields: "k8s.io/apimachinery/pkg/apis/meta/v1.TypeMeta",
	}
}

func (g *Generator) BindFlags(flag *flag.FlagSet) {
	flag.StringVarP(&g.Common.GoHeaderFilePath, "go-header-file", "h", g.Common.GoHeaderFilePath, "File containing boilerplate header text. The string YEAR will be replaced with the current 4-digit year.")
	flag.BoolVar(&g.Common.VerifyOnly, "verify-only", g.Common.VerifyOnly, "If true, only verify existing output, do not write anything.")
	flag.StringVarP(&g.Packages, "packages", "p", g.Packages, "comma-separated list of directories to get input types from. Directories prefixed with '-' are not generated, directories prefixed with '+' only create types with explicit IDL instructions.")
	flag.StringVar(&g.APIMachineryPackages, "apimachinery-packages", g.APIMachineryPackages, "comma-separated list of directories to get apimachinery input types from which are needed by any API. Directories prefixed with '-' are not generated, directories prefixed with '+' only create types with explicit IDL instructions.")
	flag.StringVarP(&g.OutputBase, "output-base", "o", g.OutputBase, "Output base; defaults to $GOPATH/src/")
	flag.StringVar(&g.VendorOutputBase, "vendor-output-base", g.VendorOutputBase, "The vendor/ directory to look for packages in; defaults to $PWD/vendor/.")
	flag.StringSliceVar(&g.ProtoImport, "proto-import", g.ProtoImport, "The search path for the core protobuf .protos, required; defaults $GOPATH/src/k8s.io/kubernetes/vendor/github.com/gogo/protobuf/protobuf.")
	flag.StringVar(&g.Conditional, "conditional", g.Conditional, "An optional Golang build tag condition to add to the generated Go code")
	flag.BoolVar(&g.Clean, "clean", g.Clean, "If true, remove all generated files for the specified Packages.")
	flag.BoolVar(&g.OnlyIDL, "only-idl", g.OnlyIDL, "If true, only generate the IDL for each package.")
	flag.BoolVar(&g.KeepGogoproto, "keep-gogoproto", g.KeepGogoproto, "If true, the generated IDL will contain gogoprotobuf extensions which are normally removed")
	flag.BoolVar(&g.SkipGeneratedRewrite, "skip-generated-rewrite", g.SkipGeneratedRewrite, "If true, skip fixing up the generated.pb.go file (debugging only).")
	flag.StringVar(&g.DropEmbeddedFields, "drop-embedded-fields", g.DropEmbeddedFields, "Comma-delimited list of embedded Go types to omit from generated protobufs")
}

func Run(g *Generator) {
	if g.Common.VerifyOnly {
		g.OnlyIDL = true
		g.Clean = false
	}

	b := parser.New()
	b.AddBuildTags("proto")

	omitTypes := map[types.Name]struct{}{}
	for _, t := range strings.Split(g.DropEmbeddedFields, ",") {
		name := types.Name{}
		if i := strings.LastIndex(t, "."); i != -1 {
			name.Package, name.Name = t[:i], t[i+1:]
		} else {
			name.Name = t
		}
		if len(name.Name) == 0 {
			log.Fatalf("--drop-embedded-types requires names in the form of [GOPACKAGE.]TYPENAME: %v", t)
		}
		omitTypes[name] = struct{}{}
	}

	boilerplate, err := g.Common.LoadGoBoilerplate()
	if err != nil {
		log.Fatalf("Failed loading boilerplate (consider using the go-header-file flag): %v", err)
	}

	protobufNames := NewProtobufNamer()
	outputPackages := generator.Packages{}
	nonOutputPackages := map[string]struct{}{}

	var packages []string
	if len(g.APIMachineryPackages) != 0 {
		packages = append(packages, strings.Split(g.APIMachineryPackages, ",")...)
	}
	if len(g.Packages) != 0 {
		packages = append(packages, strings.Split(g.Packages, ",")...)
	}
	if len(packages) == 0 {
		log.Fatalf("Both apimachinery-packages and packages are empty. At least one package must be specified.")
	}

	for _, d := range packages {
		generateAllTypes, outputPackage := true, true
		switch {
		case strings.HasPrefix(d, "+"):
			d = d[1:]
			generateAllTypes = false
		case strings.HasPrefix(d, "-"):
			d = d[1:]
			outputPackage = false
		}
		name := protoSafePackage(d)
		parts := strings.SplitN(d, "=", 2)
		if len(parts) > 1 {
			d = parts[0]
			name = parts[1]
		}
		p := newProtobufPackage(d, name, generateAllTypes, omitTypes)
		header := append([]byte{}, boilerplate...)
		header = append(header, p.HeaderText...)
		p.HeaderText = header
		protobufNames.Add(p)
		if outputPackage {
			outputPackages = append(outputPackages, p)
		} else {
			nonOutputPackages[name] = struct{}{}
		}
	}

	if !g.Common.VerifyOnly {
		for _, p := range outputPackages {
			if err := p.(*protobufPackage).Clean(g.OutputBase); err != nil {
				log.Fatalf("Unable to clean package %s: %v", p.Name(), err)
			}
		}
	}

	if g.Clean {
		return
	}

	for _, p := range protobufNames.List() {
		if err := b.AddDir(p.Path()); err != nil {
			log.Fatalf("Unable to add directory %q: %v", p.Path(), err)
		}
	}

	c, err := generator.NewContext(
		b,
		namer.NameSystems{
			"public": namer.NewPublicNamer(3),
			"proto":  protobufNames,
		},
		"public",
	)
	if err != nil {
		log.Fatalf("Failed making a context: %v", err)
	}

	c.Verify = g.Common.VerifyOnly
	c.FileTypes["protoidl"] = NewProtoFile()

	var vendoredOutputPackages, localOutputPackages generator.Packages
	for _, p := range protobufNames.packages {
		if _, ok := nonOutputPackages[p.Name()]; ok {
			// if we're not outputting the package, don't include it in either package list
			continue
		}
		p.Vendored = strings.Contains(c.Universe[p.PackagePath].SourcePath, "/vendor/")
		if p.Vendored {
			vendoredOutputPackages = append(vendoredOutputPackages, p)
		} else {
			localOutputPackages = append(localOutputPackages, p)
		}
	}

	if err := protobufNames.AssignTypesToPackages(c); err != nil {
		log.Fatalf("Failed to identify Common types: %v", err)
	}

	if err := c.ExecutePackages(g.VendorOutputBase, vendoredOutputPackages); err != nil {
		log.Fatalf("Failed executing vendor generator: %v", err)
	}
	if err := c.ExecutePackages(g.OutputBase, localOutputPackages); err != nil {
		log.Fatalf("Failed executing local generator: %v", err)
	}

	if g.OnlyIDL {
		return
	}

	if _, err := exec.LookPath("protoc"); err != nil {
		log.Fatalf("Unable to find 'protoc': %v", err)
	}

	searchArgs := []string{"-I", ".", "-I", g.OutputBase}
	if len(g.ProtoImport) != 0 {
		for _, s := range g.ProtoImport {
			searchArgs = append(searchArgs, "-I", s)
		}
	}
	args := append(searchArgs, fmt.Sprintf("--gogo_out=%s", g.OutputBase))

	buf := &bytes.Buffer{}
	if len(g.Conditional) > 0 {
		fmt.Fprintf(buf, "// +build %s\n\n", g.Conditional)
	}
	buf.Write(boilerplate)

	for _, outputPackage := range outputPackages {
		p := outputPackage.(*protobufPackage)

		path := filepath.Join(g.OutputBase, p.ImportPath())
		outputPath := filepath.Join(g.OutputBase, p.OutputPath())
		if p.Vendored {
			path = filepath.Join(g.VendorOutputBase, p.ImportPath())
			outputPath = filepath.Join(g.VendorOutputBase, p.OutputPath())
		}

		// generate the gogoprotobuf protoc
		cmd := exec.Command("protoc", append(args, path)...)
		out, err := cmd.CombinedOutput()
		if len(out) > 0 {
			log.Printf(string(out))
		}
		if err != nil {
			log.Println(strings.Join(cmd.Args, " "))
			log.Fatalf("Unable to generate protoc on %s: %v", p.PackageName, err)
		}

		if g.SkipGeneratedRewrite {
			continue
		}

		// alter the generated protobuf file to remove the generated types (but leave the serializers) and rewrite the
		// package statement to match the desired package name
		if err := RewriteGeneratedGogoProtobufFile(outputPath, p.ExtractGeneratedType, p.OptionalTypeName, buf.Bytes()); err != nil {
			log.Fatalf("Unable to rewrite generated %s: %v", outputPath, err)
		}

		// sort imports
		cmd = exec.Command("goimports", "-w", outputPath)
		out, err = cmd.CombinedOutput()
		if len(out) > 0 {
			log.Printf(string(out))
		}
		if err != nil {
			log.Println(strings.Join(cmd.Args, " "))
			log.Fatalf("Unable to rewrite imports for %s: %v", p.PackageName, err)
		}

		// format and simplify the generated file
		cmd = exec.Command("gofmt", "-s", "-w", outputPath)
		out, err = cmd.CombinedOutput()
		if len(out) > 0 {
			log.Printf(string(out))
		}
		if err != nil {
			log.Println(strings.Join(cmd.Args, " "))
			log.Fatalf("Unable to apply gofmt for %s: %v", p.PackageName, err)
		}
	}

	if g.SkipGeneratedRewrite {
		return
	}

	if !g.KeepGogoproto {
		// generate, but do so without gogoprotobuf extensions
		for _, outputPackage := range outputPackages {
			p := outputPackage.(*protobufPackage)
			p.OmitGogo = true
		}
		if err := c.ExecutePackages(g.VendorOutputBase, vendoredOutputPackages); err != nil {
			log.Fatalf("Failed executing vendor generator: %v", err)
		}
		if err := c.ExecutePackages(g.OutputBase, localOutputPackages); err != nil {
			log.Fatalf("Failed executing local generator: %v", err)
		}
	}

	for _, outputPackage := range outputPackages {
		p := outputPackage.(*protobufPackage)

		if len(p.StructTags) == 0 {
			continue
		}

		pattern := filepath.Join(g.OutputBase, p.PackagePath, "*.go")
		if p.Vendored {
			pattern = filepath.Join(g.VendorOutputBase, p.PackagePath, "*.go")
		}
		files, err := filepath.Glob(pattern)
		if err != nil {
			log.Fatalf("Can't glob pattern %q: %v", pattern, err)
		}

		for _, s := range files {
			if strings.HasSuffix(s, "_test.go") {
				continue
			}
			if err := RewriteTypesWithProtobufStructTags(s, p.StructTags); err != nil {
				log.Fatalf("Unable to rewrite with struct tags %s: %v", s, err)
			}
		}
	}
}
<commit_msg>add testapigroup of apimachinery to go-to-protobuf<commit_after>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// go-to-protobuf generates a Protobuf IDL from a Go struct, respecting any
// existing IDL tags on the Go struct.
package protobuf

import (
	"bytes"
	"fmt"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"k8s.io/gengo/args"
	"k8s.io/gengo/generator"
	"k8s.io/gengo/namer"
	"k8s.io/gengo/parser"
	"k8s.io/gengo/types"

	flag "github.com/spf13/pflag"
)

type Generator struct {
	Common               args.GeneratorArgs
	APIMachineryPackages string
	Packages             string
	OutputBase           string
	VendorOutputBase     string
	ProtoImport          []string
	Conditional          string
	Clean                bool
	OnlyIDL              bool
	KeepGogoproto        bool
	SkipGeneratedRewrite bool
	DropEmbeddedFields   string
}

func New() *Generator {
	sourceTree := args.DefaultSourceTree()
	common := args.GeneratorArgs{
		OutputBase:       sourceTree,
		GoHeaderFilePath: filepath.Join(sourceTree, "k8s.io/kubernetes/hack/boilerplate/boilerplate.go.txt"),
	}
	defaultProtoImport := filepath.Join(sourceTree, "k8s.io", "kubernetes", "vendor", "github.com", "gogo", "protobuf", "protobuf")
	cwd, err := os.Getwd()
	if err != nil {
		log.Fatalf("Cannot get current directory.")
	}
	return &Generator{
		Common:           common,
		OutputBase:       sourceTree,
		VendorOutputBase: filepath.Join(cwd, "vendor"),
		ProtoImport:      []string{defaultProtoImport},
		APIMachineryPackages: strings.Join([]string{
			`+k8s.io/apimachinery/pkg/util/intstr`,
			`+k8s.io/apimachinery/pkg/api/resource`,
			`+k8s.io/apimachinery/pkg/runtime/schema`,
			`+k8s.io/apimachinery/pkg/runtime`,
			`k8s.io/apimachinery/pkg/apis/meta/v1`,
			`k8s.io/apimachinery/pkg/apis/meta/v1beta1`,
			`k8s.io/apimachinery/pkg/apis/testapigroup/v1`,
		}, ","),
		Packages:           "",
		DropEmbeddedFields: "k8s.io/apimachinery/pkg/apis/meta/v1.TypeMeta",
	}
}

func (g *Generator) BindFlags(flag *flag.FlagSet) {
	flag.StringVarP(&g.Common.GoHeaderFilePath, "go-header-file", "h", g.Common.GoHeaderFilePath, "File containing boilerplate header text. The string YEAR will be replaced with the current 4-digit year.")
	flag.BoolVar(&g.Common.VerifyOnly, "verify-only", g.Common.VerifyOnly, "If true, only verify existing output, do not write anything.")
	flag.StringVarP(&g.Packages, "packages", "p", g.Packages, "comma-separated list of directories to get input types from. Directories prefixed with '-' are not generated, directories prefixed with '+' only create types with explicit IDL instructions.")
	flag.StringVar(&g.APIMachineryPackages, "apimachinery-packages", g.APIMachineryPackages, "comma-separated list of directories to get apimachinery input types from which are needed by any API. Directories prefixed with '-' are not generated, directories prefixed with '+' only create types with explicit IDL instructions.")
	flag.StringVarP(&g.OutputBase, "output-base", "o", g.OutputBase, "Output base; defaults to $GOPATH/src/")
	flag.StringVar(&g.VendorOutputBase, "vendor-output-base", g.VendorOutputBase, "The vendor/ directory to look for packages in; defaults to $PWD/vendor/.")
	flag.StringSliceVar(&g.ProtoImport, "proto-import", g.ProtoImport, "The search path for the core protobuf .protos, required; defaults $GOPATH/src/k8s.io/kubernetes/vendor/github.com/gogo/protobuf/protobuf.")
	flag.StringVar(&g.Conditional, "conditional", g.Conditional, "An optional Golang build tag condition to add to the generated Go code")
	flag.BoolVar(&g.Clean, "clean", g.Clean, "If true, remove all generated files for the specified Packages.")
	flag.BoolVar(&g.OnlyIDL, "only-idl", g.OnlyIDL, "If true, only generate the IDL for each package.")
	flag.BoolVar(&g.KeepGogoproto, "keep-gogoproto", g.KeepGogoproto, "If true, the generated IDL will contain gogoprotobuf extensions which are normally removed")
	flag.BoolVar(&g.SkipGeneratedRewrite, "skip-generated-rewrite", g.SkipGeneratedRewrite, "If true, skip fixing up the generated.pb.go file (debugging only).")
	flag.StringVar(&g.DropEmbeddedFields, "drop-embedded-fields", g.DropEmbeddedFields, "Comma-delimited list of embedded Go types to omit from generated protobufs")
}

func Run(g *Generator) {
	if g.Common.VerifyOnly {
		g.OnlyIDL = true
		g.Clean = false
	}

	b := parser.New()
	b.AddBuildTags("proto")

	omitTypes := map[types.Name]struct{}{}
	for _, t := range strings.Split(g.DropEmbeddedFields, ",") {
		name := types.Name{}
		if i := strings.LastIndex(t, "."); i != -1 {
			name.Package, name.Name = t[:i], t[i+1:]
		} else {
			name.Name = t
		}
		if len(name.Name) == 0 {
			log.Fatalf("--drop-embedded-types requires names in the form of [GOPACKAGE.]TYPENAME: %v", t)
		}
		omitTypes[name] = struct{}{}
	}

	boilerplate, err := g.Common.LoadGoBoilerplate()
	if err != nil {
		log.Fatalf("Failed loading boilerplate (consider using the go-header-file flag): %v", err)
	}

	protobufNames := NewProtobufNamer()
	outputPackages := generator.Packages{}
	nonOutputPackages := map[string]struct{}{}

	var packages []string
	if len(g.APIMachineryPackages) != 0 {
		packages = append(packages, strings.Split(g.APIMachineryPackages, ",")...)
	}
	if len(g.Packages) != 0 {
		packages = append(packages, strings.Split(g.Packages, ",")...)
	}
	if len(packages) == 0 {
		log.Fatalf("Both apimachinery-packages and packages are empty. At least one package must be specified.")
	}

	for _, d := range packages {
		generateAllTypes, outputPackage := true, true
		switch {
		case strings.HasPrefix(d, "+"):
			d = d[1:]
			generateAllTypes = false
		case strings.HasPrefix(d, "-"):
			d = d[1:]
			outputPackage = false
		}
		name := protoSafePackage(d)
		parts := strings.SplitN(d, "=", 2)
		if len(parts) > 1 {
			d = parts[0]
			name = parts[1]
		}
		p := newProtobufPackage(d, name, generateAllTypes, omitTypes)
		header := append([]byte{}, boilerplate...)
		header = append(header, p.HeaderText...)
		p.HeaderText = header
		protobufNames.Add(p)
		if outputPackage {
			outputPackages = append(outputPackages, p)
		} else {
			nonOutputPackages[name] = struct{}{}
		}
	}

	if !g.Common.VerifyOnly {
		for _, p := range outputPackages {
			if err := p.(*protobufPackage).Clean(g.OutputBase); err != nil {
				log.Fatalf("Unable to clean package %s: %v", p.Name(), err)
			}
		}
	}

	if g.Clean {
		return
	}

	for _, p := range protobufNames.List() {
		if err := b.AddDir(p.Path()); err != nil {
			log.Fatalf("Unable to add directory %q: %v", p.Path(), err)
		}
	}

	c, err := generator.NewContext(
		b,
		namer.NameSystems{
			"public": namer.NewPublicNamer(3),
			"proto":  protobufNames,
		},
		"public",
	)
	if err != nil {
		log.Fatalf("Failed making a context: %v", err)
	}

	c.Verify = g.Common.VerifyOnly
	c.FileTypes["protoidl"] = NewProtoFile()

	var vendoredOutputPackages, localOutputPackages generator.Packages
	for _, p := range protobufNames.packages {
		if _, ok := nonOutputPackages[p.Name()]; ok {
			// if we're not outputting the package, don't include it in either package list
			continue
		}
		p.Vendored = strings.Contains(c.Universe[p.PackagePath].SourcePath, "/vendor/")
		if p.Vendored {
			vendoredOutputPackages = append(vendoredOutputPackages, p)
		} else {
			localOutputPackages = append(localOutputPackages, p)
		}
	}

	if err := protobufNames.AssignTypesToPackages(c); err != nil {
		log.Fatalf("Failed to identify Common types: %v", err)
	}

	if err := c.ExecutePackages(g.VendorOutputBase, vendoredOutputPackages); err != nil {
		log.Fatalf("Failed executing vendor generator: %v", err)
	}
	if err := c.ExecutePackages(g.OutputBase, localOutputPackages); err != nil {
		log.Fatalf("Failed executing local generator: %v", err)
	}

	if g.OnlyIDL {
		return
	}

	if _, err := exec.LookPath("protoc"); err != nil {
		log.Fatalf("Unable to find 'protoc': %v", err)
	}

	searchArgs := []string{"-I", ".", "-I", g.OutputBase}
	if len(g.ProtoImport) != 0 {
		for _, s := range g.ProtoImport {
			searchArgs = append(searchArgs, "-I", s)
		}
	}
	args := append(searchArgs, fmt.Sprintf("--gogo_out=%s", g.OutputBase))

	buf := &bytes.Buffer{}
	if len(g.Conditional) > 0 {
		fmt.Fprintf(buf, "// +build %s\n\n", g.Conditional)
	}
	buf.Write(boilerplate)

	for _, outputPackage := range outputPackages {
		p := outputPackage.(*protobufPackage)

		path := filepath.Join(g.OutputBase, p.ImportPath())
		outputPath := filepath.Join(g.OutputBase, p.OutputPath())
		if p.Vendored {
			path = filepath.Join(g.VendorOutputBase, p.ImportPath())
			outputPath = filepath.Join(g.VendorOutputBase, p.OutputPath())
		}

		// generate the gogoprotobuf protoc
		cmd := exec.Command("protoc", append(args, path)...)
		out, err := cmd.CombinedOutput()
		if len(out) > 0 {
			log.Printf(string(out))
		}
		if err != nil {
			log.Println(strings.Join(cmd.Args, " "))
			log.Fatalf("Unable to generate protoc on %s: %v", p.PackageName, err)
		}

		if g.SkipGeneratedRewrite {
			continue
		}

		// alter the generated protobuf file to remove the generated types (but leave the serializers) and rewrite the
		// package statement to match the desired package name
		if err := RewriteGeneratedGogoProtobufFile(outputPath, p.ExtractGeneratedType, p.OptionalTypeName, buf.Bytes()); err != nil {
			log.Fatalf("Unable to rewrite generated %s: %v", outputPath, err)
		}

		// sort imports
		cmd = exec.Command("goimports", "-w", outputPath)
		out, err = cmd.CombinedOutput()
		if len(out) > 0 {
			log.Printf(string(out))
		}
		if err != nil {
			log.Println(strings.Join(cmd.Args, " "))
			log.Fatalf("Unable to rewrite imports for %s: %v", p.PackageName, err)
		}

		// format and simplify the generated file
		cmd = exec.Command("gofmt", "-s", "-w", outputPath)
		out, err = cmd.CombinedOutput()
		if len(out) > 0 {
			log.Printf(string(out))
		}
		if err != nil {
			log.Println(strings.Join(cmd.Args, " "))
			log.Fatalf("Unable to apply gofmt for %s: %v", p.PackageName, err)
		}
	}

	if g.SkipGeneratedRewrite {
		return
	}

	if !g.KeepGogoproto {
		// generate, but do so without gogoprotobuf extensions
		for _, outputPackage := range outputPackages {
			p := outputPackage.(*protobufPackage)
			p.OmitGogo = true
		}
		if err := c.ExecutePackages(g.VendorOutputBase, vendoredOutputPackages); err != nil {
			log.Fatalf("Failed executing vendor generator: %v", err)
		}
		if err := c.ExecutePackages(g.OutputBase, localOutputPackages); err != nil {
			log.Fatalf("Failed executing local generator: %v", err)
		}
	}

	for _, outputPackage := range outputPackages {
		p := outputPackage.(*protobufPackage)

		if len(p.StructTags) == 0 {
			continue
		}

		pattern := filepath.Join(g.OutputBase, p.PackagePath, "*.go")
		if p.Vendored {
			pattern = filepath.Join(g.VendorOutputBase, p.PackagePath, "*.go")
		}
		files, err := filepath.Glob(pattern)
		if err != nil {
			log.Fatalf("Can't glob pattern %q: %v", pattern, err)
		}

		for _, s := range files {
			if strings.HasSuffix(s, "_test.go") {
				continue
			}
			if err := RewriteTypesWithProtobufStructTags(s, p.StructTags); err != nil {
				log.Fatalf("Unable to rewrite with struct tags %s: %v", s, err)
			}
		}
	}
}
<|endoftext|>


<commit_before>package kafka

import (
	"sync"
	"time"

	"github.com/Shopify/sarama"
	"github.com/funkygao/gafka/cmd/kguard/watchers"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/go-metrics"
	log "github.com/funkygao/log4go"
)

var _ watchers.Watcher = &WatchTopics{}

// WatchTopics montor kafka total msg count over time.
type WatchTopics struct {
	Zkzone *zk.ZkZone
	Stop   chan struct{}
	Tick   time.Duration
	Wg     *sync.WaitGroup
}

func (this *WatchTopics) Init() {}

func (this *WatchTopics) Run() {
	defer this.Wg.Done()

	ticker := time.NewTicker(this.Tick)
	defer ticker.Stop()

	pubQps := metrics.NewRegisteredMeter("pub.qps", nil)
	offsets := metrics.NewRegisteredGauge("msg.cum", nil)
	topics := metrics.NewRegisteredGauge("topics", nil)
	partitions := metrics.NewRegisteredGauge("partitions", nil)
	brokers := metrics.NewRegisteredGauge("brokers", nil)
	var lastTotalOffsets int64
	for {

		select {
		case <-this.Stop:
			return

		case <-ticker.C:
			o, t, p, b := this.report()
			offsets.Update(o)
			topics.Update(t)
			partitions.Update(p)
			brokers.Update(b)

			if lastTotalOffsets > 0 {
				if o-lastTotalOffsets > 0 {
					pubQps.Mark(o - lastTotalOffsets)
					lastTotalOffsets = o
				} else {
					// e,g. some topics are dead, so the next total offset < lastTotalOffset
					// in this case, we skip this offset metric: only log warning
					log.Warn("offset backwards: %d %d", o, lastTotalOffsets)
				}
			} else {
				// the 1st run inside the loop
				lastTotalOffsets = o
			}

		}
	}

}

func (this *WatchTopics) report() (totalOffsets int64, topicsN int64,
	partitionN int64, brokersN int64) {
	this.Zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
		brokerList := zkcluster.BrokerList()
		kfk, err := sarama.NewClient(brokerList, sarama.NewConfig())
		if err != nil {
			log.Error("cluster[%s] %v", zkcluster.Name(), err)
			return
		}
		defer kfk.Close()

		brokersN += int64(len(brokerList))

		topics, err := kfk.Topics()
		if err != nil {
			log.Error("cluster[%s] %v", zkcluster.Name(), err)
			return
		}

		for _, topic := range topics {
			partions, err := kfk.Partitions(topic)
			if err != nil {
				log.Error("cluster[%s] topic:%s %v", zkcluster.Name(), topic, err)
				continue
			}

			topicsN += 1

			for _, partitionId := range partions {
				latestOffset, err := kfk.GetOffset(topic, partitionId,
					sarama.OffsetNewest)
				if err != nil {
					log.Error("cluster[%s] topic:%s partition:%d %v",
						zkcluster.Name(), topic, partitionId, err)
					continue
				}

				partitionN += 1
				totalOffsets += latestOffset
			}
		}

	})

	return
}
<commit_msg>tweak of the log msg<commit_after>package kafka

import (
	"sync"
	"time"

	"github.com/Shopify/sarama"
	"github.com/funkygao/gafka/cmd/kguard/watchers"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/go-metrics"
	log "github.com/funkygao/log4go"
)

var _ watchers.Watcher = &WatchTopics{}

// WatchTopics montor kafka total msg count over time.
type WatchTopics struct {
	Zkzone *zk.ZkZone
	Stop   chan struct{}
	Tick   time.Duration
	Wg     *sync.WaitGroup
}

func (this *WatchTopics) Init() {}

func (this *WatchTopics) Run() {
	defer this.Wg.Done()

	ticker := time.NewTicker(this.Tick)
	defer ticker.Stop()

	pubQps := metrics.NewRegisteredMeter("pub.qps", nil)
	offsets := metrics.NewRegisteredGauge("msg.cum", nil)
	topics := metrics.NewRegisteredGauge("topics", nil)
	partitions := metrics.NewRegisteredGauge("partitions", nil)
	brokers := metrics.NewRegisteredGauge("brokers", nil)
	var lastTotalOffsets int64
	for {

		select {
		case <-this.Stop:
			return

		case <-ticker.C:
			o, t, p, b := this.report()
			offsets.Update(o)
			topics.Update(t)
			partitions.Update(p)
			brokers.Update(b)

			if lastTotalOffsets > 0 {
				if o-lastTotalOffsets > 0 {
					pubQps.Mark(o - lastTotalOffsets)
					lastTotalOffsets = o
				} else {
					// e,g. some topics are dead, so the next total offset < lastTotalOffset
					// in this case, we skip this offset metric: only log warning
					log.Warn("offset backwards: %d %d", o, lastTotalOffsets)
				}
			} else {
				// the 1st run inside the loop
				lastTotalOffsets = o
			}

		}
	}

}

func (this *WatchTopics) report() (totalOffsets int64, topicsN int64,
	partitionN int64, brokersN int64) {
	this.Zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
		brokerList := zkcluster.BrokerList()
		kfk, err := sarama.NewClient(brokerList, sarama.NewConfig())
		if err != nil {
			log.Error("cluster[%s] %v", zkcluster.Name(), err)
			return
		}
		defer kfk.Close()

		brokersN += int64(len(brokerList))

		topics, err := kfk.Topics()
		if err != nil {
			log.Error("cluster[%s] %v", zkcluster.Name(), err)
			return
		}

		for _, topic := range topics {
			partions, err := kfk.Partitions(topic)
			if err != nil {
				log.Error("cluster[%s] topic:%s %v", zkcluster.Name(), topic, err)
				continue
			}

			topicsN += 1

			for _, partitionId := range partions {
				latestOffset, err := kfk.GetOffset(topic, partitionId,
					sarama.OffsetNewest)
				if err != nil {
					log.Error("cluster[%s] topic[%s/%d]: %v",
						zkcluster.Name(), topic, partitionId, err)
					continue
				}

				partitionN += 1
				totalOffsets += latestOffset
			}
		}

	})

	return
}
<|endoftext|>


<commit_before>package options

import (
	"github.com/spf13/pflag"
)

type Config struct {
	Master                string
	KubeConfig            string
	ProviderName          string
	ClusterName           string
	LoadbalancerImageName string
}

func NewConfig() *Config {
	return &Config{
		Master:                "",
		KubeConfig:            "",
		ProviderName:          "",
		ClusterName:           "",
		LoadbalancerImageName: "appscode/haproxy:1.7.2-k8s",
	}
}

func (s *Config) AddFlags(fs *pflag.FlagSet) {
	fs.StringVar(&s.Master, "master", s.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig)")
	fs.StringVar(&s.KubeConfig, "kubeconfig", s.KubeConfig, "Path to kubeconfig file with authorization information (the master location is set by the master flag).")

	fs.StringVarP(&s.ProviderName, "cloud-provider", "c", s.ProviderName, "Name of cloud provider")
	fs.StringVarP(&s.ClusterName, "cluster-name", "k", s.ClusterName, "Name of Kubernetes cluster")
	fs.StringVarP(&s.LoadbalancerImageName, "haproxy-image", "h", s.LoadbalancerImageName, "haproxy image name to be run")
}
<commit_msg>Use direct configmap reloader. (#2)<commit_after>package options

import (
	"github.com/spf13/pflag"
)

type Config struct {
	Master                string
	KubeConfig            string
	ProviderName          string
	ClusterName           string
	LoadbalancerImageName string
}

func NewConfig() *Config {
	return &Config{
		Master:                "",
		KubeConfig:            "",
		ProviderName:          "",
		ClusterName:           "",
		LoadbalancerImageName: "appscode/haproxy:1.7.2-k8s", // We May Need a Version Update
	}
}

func (s *Config) AddFlags(fs *pflag.FlagSet) {
	fs.StringVar(&s.Master, "master", s.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig)")
	fs.StringVar(&s.KubeConfig, "kubeconfig", s.KubeConfig, "Path to kubeconfig file with authorization information (the master location is set by the master flag).")

	fs.StringVarP(&s.ProviderName, "cloud-provider", "c", s.ProviderName, "Name of cloud provider")
	fs.StringVarP(&s.ClusterName, "cluster-name", "k", s.ClusterName, "Name of Kubernetes cluster")
	fs.StringVarP(&s.LoadbalancerImageName, "haproxy-image", "h", s.LoadbalancerImageName, "haproxy image name to be run")
}
<|endoftext|>


<commit_before>package social

import (
	"fmt"
	"gnd.la/log"
	"gnd.la/mux"
	"gnd.la/tasks"
	"time"
)

const (
	pollInterval = 5 * time.Minute
)

type Sharer struct {
	service  Service
	interval time.Duration
	provider ShareProvider
	config   interface{}
	task     *tasks.Task
}

func (s *Sharer) share(ctx *mux.Context) {
	last, err := s.provider.LastShare(ctx, s.service)
	if err != nil {
		log.Errorf("error finding last share time on %s: %s", s.service, err)
		return
	}
	if last.Before(time.Now().Add(-s.interval)) {
		item, err := s.provider.Item(ctx, s.service)
		if err != nil {
			log.Errorf("error finding next time for sharing on %s: %s", s.service, err)
			return
		}
		if item != nil {
			result, err := Share(s.service, item, s.config)
			if err != nil {
				log.Errorf("error sharing on %s: %s", s.service, err)
			}
			s.provider.Shared(ctx, s.service, item, result, err)
		}
	}
}

func (s *Sharer) Schedule(m *mux.Mux, interval time.Duration) {
	if s.task != nil {
		s.task.Stop()
	}
	s.interval = interval
	s.task = tasks.Schedule(m, pollInterval, &tasks.Options{Unique: true}, s.share)
}

func (s *Sharer) Stop() {
	if s.task != nil {
		s.task.Stop()
		s.task = nil
	}
}

func NewSharer(s Service, provider ShareProvider, config interface{}) *Sharer {
	if provider == nil {
		panic(fmt.Errorf("provider can't be nil"))
	}
	if err := validateConfig(s, config); err != nil {
		panic(err)
	}
	return &Sharer{
		service:  s,
		provider: provider,
		config:   config,
	}
}
<commit_msg>Don't make sharing tasks unique<commit_after>package social

import (
	"fmt"
	"gnd.la/log"
	"gnd.la/mux"
	"gnd.la/tasks"
	"time"
)

const (
	pollInterval = 5 * time.Minute
)

type Sharer struct {
	service  Service
	interval time.Duration
	provider ShareProvider
	config   interface{}
	task     *tasks.Task
}

func (s *Sharer) share(ctx *mux.Context) {
	last, err := s.provider.LastShare(ctx, s.service)
	if err != nil {
		log.Errorf("error finding last share time on %s: %s", s.service, err)
		return
	}
	if last.Before(time.Now().Add(-s.interval)) {
		item, err := s.provider.Item(ctx, s.service)
		if err != nil {
			log.Errorf("error finding next time for sharing on %s: %s", s.service, err)
			return
		}
		if item != nil {
			result, err := Share(s.service, item, s.config)
			if err != nil {
				log.Errorf("error sharing on %s: %s", s.service, err)
			}
			s.provider.Shared(ctx, s.service, item, result, err)
		}
	}
}

func (s *Sharer) Schedule(m *mux.Mux, interval time.Duration) {
	if s.task != nil {
		s.task.Stop()
	}
	s.interval = interval
	s.task = tasks.Schedule(m, pollInterval, nil, s.share)
}

func (s *Sharer) Stop() {
	if s.task != nil {
		s.task.Stop()
		s.task = nil
	}
}

func NewSharer(s Service, provider ShareProvider, config interface{}) *Sharer {
	if provider == nil {
		panic(fmt.Errorf("provider can't be nil"))
	}
	if err := validateConfig(s, config); err != nil {
		panic(err)
	}
	return &Sharer{
		service:  s,
		provider: provider,
		config:   config,
	}
}
<|endoftext|>


<commit_before>package socket

import (
	"fmt"
	"log"
	"os"
	"os/signal"
	"time"

	"github.com/gorilla/websocket"
)

var ops = map[string]string{
	"ping":           "ping",
	"pingBlock":      "ping_block",
	"pingTx":         "ping_tx",
	"unconfirmedTxs": "unconfirmed_sub",
	"newBlocks":      "blocks_sub",
}

const wsURL = "wss://ws.blockchain.info/inv"

func dial() (*websocket.Conn, error) {
	log.Printf("Connecting to %v...", wsURL)
	conn, _, err := websocket.DefaultDialer.Dial(wsURL, nil)

	return conn, err
}

func read(conn *websocket.Conn) {
	for {
		_, message, err := conn.ReadMessage()
		if err != nil {
			log.Println("read: ", err)
			return
		}
		log.Printf("Received:\n%s", message)
	}
}

func monitor(conn *websocket.Conn) {
	done := make(chan struct{})
	interrupt := make(chan os.Signal, 1)

	signal.Notify(interrupt, os.Interrupt)

	for {
		select {
		case <-interrupt:
			log.Println("interrupt caught, closing connection...")
			// To cleanly close a connection, a client should send a close
			// frame and wait for the server to close the connection.
			err := conn.WriteMessage(websocket.CloseMessage,
				websocket.FormatCloseMessage(websocket.CloseNormalClosure, ""))
			if err != nil {
				log.Println("write close:", err)
				return
			}
			select {
			case <-done:
				log.Println("done triggered")
			case <-time.After(time.Second):
			}
			conn.Close()
			return
		}
	}
}

func send(conn *websocket.Conn, op string) {
	msg := "{\"op\": \"" + op + "\"}"
	log.Printf("Sending msg: %v", msg)
	err := conn.WriteMessage(websocket.TextMessage, []byte(msg))
	if err != nil {
		log.Println("send error:", err)
		return
	}
}

func sendWithArgs(conn *websocket.Conn, op string, args map[string]string) {
	var argStr string
	for key, val := range args {
		argStr += "\"" + key + "\": " + "\"" + val + "\","
	}
	fmt.Println(argStr)
}

// ========================================================================

func keepAlive(conn *websocket.Conn) {
	ticker := time.NewTicker(time.Second * 30)
	for {
		select {
		case <-ticker.C:
			log.Println("keepAlive op: Ping")
			send(conn, ops["ping"])
		}
	}
}

func SubUnconfirmedTxs(conn *websocket.Conn) {
	send(conn, ops["unconfirmedTxs"])
}

func SubNewBlocks(conn *websocket.Conn) {
	send(conn, ops["newBlocks"])
}

func SubAddress(conn *websocket.Conn) {

}

func Connect() {
	conn, err := dial()
	if err != nil {
		log.Fatal("dial error: ", err)
	}
	defer conn.Close()

	// sendWithArgs(conn, ops["ping"], map[string]string{"addr": "19LjQtrSw6fKSCmUJR3enuZ8gq3ufCYRNt", "d": "dd"})

	go keepAlive(conn)
	SubNewBlocks(conn)
	go read(conn)
	monitor(conn)
}
<commit_msg>Implements `sendWithArgs()`<commit_after>package socket

import (
	"encoding/json"
	"log"
	"os"
	"os/signal"
	"time"

	"github.com/gorilla/websocket"
)

var ops = map[string]string{
	"ping":           "ping",
	"pingBlock":      "ping_block",
	"pingTx":         "ping_tx",
	"unconfirmedTxs": "unconfirmed_sub",
	"newBlocks":      "blocks_sub",
	"txsOnAddr":      "addr_sub",
}

const wsURL = "wss://ws.blockchain.info/inv"

func dial() (*websocket.Conn, error) {
	log.Printf("Connecting to %v...", wsURL)
	conn, _, err := websocket.DefaultDialer.Dial(wsURL, nil)

	return conn, err
}

func read(conn *websocket.Conn) {
	for {
		_, message, err := conn.ReadMessage()
		if err != nil {
			log.Println("read: ", err)
			return
		}
		log.Printf("Received:\n%s", message)
	}
}

func monitor(conn *websocket.Conn) {
	done := make(chan struct{})
	interrupt := make(chan os.Signal, 1)

	signal.Notify(interrupt, os.Interrupt)

	for {
		select {
		case <-interrupt:
			log.Println("interrupt caught, closing connection...")
			// To cleanly close a connection, a client should send a close
			// frame and wait for the server to close the connection.
			err := conn.WriteMessage(websocket.CloseMessage,
				websocket.FormatCloseMessage(websocket.CloseNormalClosure, ""))
			if err != nil {
				log.Println("write close:", err)
				return
			}
			select {
			case <-done:
				log.Println("done triggered")
			case <-time.After(time.Second):
			}
			conn.Close()
			return
		}
	}
}

func send(conn *websocket.Conn, op string) {
	jsonMsg, err := json.Marshal(map[string]string{"op": op})
	if err != nil {
		log.Println("send .Marshal error:", err)
		return
	}
	msg := string(jsonMsg)
	log.Printf("Sending msg: %v", msg)
	err = conn.WriteMessage(websocket.TextMessage, []byte(msg))
	if err != nil {
		log.Println("send error:", err)
		return
	}
}

func sendWithArgs(conn *websocket.Conn, op string, args map[string]string) {
	args["op"] = op
	jsonArgs, err := json.Marshal(args)
	if err != nil {
		log.Println("sendWithArgs .Marshal error:", err)
		return
	}
	msg := string(jsonArgs)
	log.Printf("Sending msg (with args): %v", msg)
	err = conn.WriteMessage(websocket.TextMessage, []byte(msg))
	if err != nil {
		log.Println("sendWithArgs error:", err)
		return
	}
}

// ========================================================================

func keepAlive(conn *websocket.Conn) {
	const keepAliveInterval = 30
	ticker := time.NewTicker(time.Second * keepAliveInterval)
	for {
		select {
		case <-ticker.C:
			log.Println("keepAlive op: Ping")
			send(conn, ops["ping"])
		}
	}
}

func SubUnconfirmedTxs(conn *websocket.Conn) {
	send(conn, ops["unconfirmedTxs"])
}

func SubNewBlocks(conn *websocket.Conn) {
	send(conn, ops["newBlocks"])
}

func SubAddress(conn *websocket.Conn, addr string) {
	sendWithArgs(conn, ops["txsOnAddr"],
		map[string]string{"addr": addr})
}

func Connect() {
	conn, err := dial()
	if err != nil {
		log.Fatal("dial error: ", err)
	}
	defer conn.Close()

	go keepAlive(conn)
	SubNewBlocks(conn)
	SubAddress(conn, "19LjQtrSw6fKSCmUJR3enuZ8gq3ufCYRNt")
	go read(conn)
	monitor(conn)
}
<|endoftext|>


<commit_before>package e2e

import (
	"github.com/ohsu-comp-bio/funnel/proto/tes"
	"testing"
	"time"
)

func TestHelloWorld(t *testing.T) {
	id := run(`
    --cmd 'echo hello world'
  `)
	task := wait(id)

	if task.Logs[0].Logs[0].Stdout != "hello world\n" {
		t.Fatal("Missing stdout")
	}
}

func TestGetTaskView(t *testing.T) {
	var err error
	var task *tes.Task

	id := run(`
    --cmd 'echo hello world'
    --name 'foo'
  `)
	wait(id)

	task = getView(id, tes.TaskView_MINIMAL)

	if task.Id != id {
		t.Fatal("expected task ID in minimal view")
	}
	if task.State != tes.State_COMPLETE {
		t.Fatal("expected complete state")
	}
	if task.Name != "" {
		t.Fatal("unexpected task name included in minimal view")
	}
	if task.Logs != nil {
		t.Fatal("unexpected task logs included in minimal view")
	}

	task = getView(id, tes.TaskView_BASIC)

	if task.Name != "foo" {
		t.Fatal("expected task name to be included basic view")
	}

	if task.Logs != nil {
		t.Fatal("unexpected task logs included in basic view")
	}

	task = getView(id, tes.TaskView_FULL)

	if task.Logs[0].Logs[0].Stdout != "hello world\n" {
		t.Fatal("Missing stdout in full view")
	}

	// test http proxy
	task, err = hcli.GetTask(id, "MINIMAL")
	if err != nil {
		t.Fatal(err)
	}

	if task.Id != id {
		t.Fatal("expected task ID in minimal view")
	}
	if task.Name != "" {
		t.Fatal("unexpected task name included in minimal view")
	}
	if task.Logs != nil {
		t.Fatal("unexpected task logs included in minimal view")
	}

	task, err = hcli.GetTask(id, "BASIC")
	if err != nil {
		t.Fatal(err)
	}

	if task.Name != "foo" {
		t.Fatal("expected task name to be included basic view")
	}

	if task.Logs != nil {
		t.Fatal("unexpected task logs included in basic view")
	}

	task, err = hcli.GetTask(id, "FULL")
	if err != nil {
		t.Fatal(err)
	}

	if task.Logs[0].Logs[0].Stdout != "hello world\n" {
		t.Fatal("Missing stdout in full view")
	}
}

// TODO this is a bit hacky for now because we're reusing the same
//      server + DB for all the e2e tests, so ListTasks gets the
//      results of all of those. It works for the moment, but
//      should probably run against a clean environment.
func TestListTaskView(t *testing.T) {
	var tasks []*tes.Task
	var task *tes.Task
	var err error

	id := run(`
    --cmd 'echo hello world'
    --name 'foo'
  `)
	wait(id)

	tasks = listView(tes.TaskView_MINIMAL)
	task = tasks[0]

	if task.Id == "" {
		t.Fatal("expected task ID in minimal view")
	}
	if task.State == tes.State_UNKNOWN {
		t.Fatal("expected complete state")
	}
	if task.Name != "" {
		t.Fatal("unexpected task name included in minimal view")
	}
	if task.Logs != nil {
		t.Fatal("unexpected task logs included in minimal view")
	}

	tasks = listView(tes.TaskView_BASIC)
	task = tasks[0]

	if task.Name == "" {
		t.Fatal("expected task name to be included basic view")
	}

	if task.Logs != nil {
		t.Fatal("unexpected task logs included in basic view")
	}

	tasks = listView(tes.TaskView_FULL)
	task = tasks[0]

	if task.Logs[0].Logs[0].Stdout != "hello world\n" {
		t.Fatal("Missing stdout in full view")
	}

	// test http proxy
	var r *tes.ListTasksResponse
	r, err = hcli.ListTasks("MINIMAL")
	if err != nil {
		t.Fatal(err)
	}
	task = r.Tasks[0]

	if task.Id == "" {
		t.Fatal("expected task ID in minimal view")
	}
	if task.State == tes.State_UNKNOWN {
		t.Fatal("expected complete state")
	}
	if task.Name != "" {
		t.Fatal("unexpected task name included in minimal view")
	}
	if task.Logs != nil {
		t.Fatal("unexpected task logs included in minimal view")
	}

	r, err = hcli.ListTasks("BASIC")
	if err != nil {
		t.Fatal(err)
	}
	task = r.Tasks[0]

	if task.Name == "" {
		t.Fatal("expected task name to be included basic view")
	}

	if task.Logs != nil {
		t.Fatal("unexpected task logs included in basic view")
	}

	r, err = hcli.ListTasks("FULL")
	if err != nil {
		t.Fatal(err)
	}
	task = r.Tasks[0]

	if task.Logs[0].Logs[0].Stdout != "hello world\n" {
		t.Fatal("Missing stdout in full view")
	}
}

// Test that the streaming logs pick up a single character.
// This ensures that the streaming works even when a small
// amount of logs are written (which was once a bug).
func TestSingleCharLog(t *testing.T) {
	id := run(`
    --cmd "sh -c 'echo a; sleep 10'"
  `)
	time.Sleep(time.Second * 2)
	task := get(id)
	if task.Logs[0].Logs[0].Stdout != "a\n" {
		t.Fatal("Missing logs")
	}
	cancel(id)
}

// Test that port mappings are being logged.
/* TODO need ports in funnel run
func TestPortLog(t *testing.T) {
  id := run(`
    --cmd 'echo start'
    --cmd 'sleep 10'
  `)
  waitForExec(id, 2)
  task := get(id)
  if task.Logs[0].Logs[0].Ports[0].Host != 5000 {
    t.Fatal("Unexpected port logs")
  }
}
*/

// Test that a completed task cannot change state.
func TestCompleteStateImmutable(t *testing.T) {
	id := run(`
    --cmd 'echo hello'
  `)
	wait(id)
	err := cancel(id)
	if err == nil {
		t.Fatal("expected error")
	}
	task := get(id)
	if task.State != tes.State_COMPLETE {
		t.Fatal("Unexpected state")
	}
}

// Test canceling a task
func TestCancel(t *testing.T) {
	id := run(`
    --cmd 'echo start'
    --cmd 'sleep 1000'
    --cmd 'echo never'
  `)
	waitForExec(id, 1)
	cancel(id)
	// TODO docker details and container ID are very Funnel specific
	//      how could we generalize this to be reusued as TES conformance?
	waitForDockerDestroy(id + "-0")
	task := get(id)
	if task.State != tes.State_CANCELED {
		t.Fatal("Unexpected state")
	}
}

// The task executor logs list should only include entries for steps that
// have been started or completed, i.e. steps that have yet to be started
// won't show up in Task.Logs[0].Logs
func TestExecutorLogLength(t *testing.T) {
	id := run(`
    --cmd 'echo first'
    --cmd 'sleep 10'
    --cmd 'echo done'
  `)
	waitForExec(id, 2)
	task := get(id)
	cancel(id)
	if len(task.Logs[0].Logs) != 2 {
		t.Fatal("Unexpected executor log count")
	}
}

// There was a bug + fix where the task was being marked complete after
// the first step completed, but the correct behavior is to mark the
// task complete after *all* steps have completed.
func TestMarkCompleteBug(t *testing.T) {
	id := run(`
    --cmd 'echo step 1'
    --cmd 'sleep 100'
  `)
	waitForExec(id, 2)
	task := get(id)
	if task.State != tes.State_RUNNING {
		t.Fatal("Unexpected task state")
	}
}
<commit_msg>updated TestSingleCharLog to reduce possibility of failure<commit_after>package e2e

import (
	"github.com/ohsu-comp-bio/funnel/proto/tes"
	"testing"
	"time"
)

func TestHelloWorld(t *testing.T) {
	id := run(`
    --cmd 'echo hello world'
  `)
	task := wait(id)

	if task.Logs[0].Logs[0].Stdout != "hello world\n" {
		t.Fatal("Missing stdout")
	}
}

func TestGetTaskView(t *testing.T) {
	var err error
	var task *tes.Task

	id := run(`
    --cmd 'echo hello world'
    --name 'foo'
  `)
	wait(id)

	task = getView(id, tes.TaskView_MINIMAL)

	if task.Id != id {
		t.Fatal("expected task ID in minimal view")
	}
	if task.State != tes.State_COMPLETE {
		t.Fatal("expected complete state")
	}
	if task.Name != "" {
		t.Fatal("unexpected task name included in minimal view")
	}
	if task.Logs != nil {
		t.Fatal("unexpected task logs included in minimal view")
	}

	task = getView(id, tes.TaskView_BASIC)

	if task.Name != "foo" {
		t.Fatal("expected task name to be included basic view")
	}

	if task.Logs != nil {
		t.Fatal("unexpected task logs included in basic view")
	}

	task = getView(id, tes.TaskView_FULL)

	if task.Logs[0].Logs[0].Stdout != "hello world\n" {
		t.Fatal("Missing stdout in full view")
	}

	// test http proxy
	task, err = hcli.GetTask(id, "MINIMAL")
	if err != nil {
		t.Fatal(err)
	}

	if task.Id != id {
		t.Fatal("expected task ID in minimal view")
	}
	if task.Name != "" {
		t.Fatal("unexpected task name included in minimal view")
	}
	if task.Logs != nil {
		t.Fatal("unexpected task logs included in minimal view")
	}

	task, err = hcli.GetTask(id, "BASIC")
	if err != nil {
		t.Fatal(err)
	}

	if task.Name != "foo" {
		t.Fatal("expected task name to be included basic view")
	}

	if task.Logs != nil {
		t.Fatal("unexpected task logs included in basic view")
	}

	task, err = hcli.GetTask(id, "FULL")
	if err != nil {
		t.Fatal(err)
	}

	if task.Logs[0].Logs[0].Stdout != "hello world\n" {
		t.Fatal("Missing stdout in full view")
	}
}

// TODO this is a bit hacky for now because we're reusing the same
//      server + DB for all the e2e tests, so ListTasks gets the
//      results of all of those. It works for the moment, but
//      should probably run against a clean environment.
func TestListTaskView(t *testing.T) {
	var tasks []*tes.Task
	var task *tes.Task
	var err error

	id := run(`
    --cmd 'echo hello world'
    --name 'foo'
  `)
	wait(id)

	tasks = listView(tes.TaskView_MINIMAL)
	task = tasks[0]

	if task.Id == "" {
		t.Fatal("expected task ID in minimal view")
	}
	if task.State == tes.State_UNKNOWN {
		t.Fatal("expected complete state")
	}
	if task.Name != "" {
		t.Fatal("unexpected task name included in minimal view")
	}
	if task.Logs != nil {
		t.Fatal("unexpected task logs included in minimal view")
	}

	tasks = listView(tes.TaskView_BASIC)
	task = tasks[0]

	if task.Name == "" {
		t.Fatal("expected task name to be included basic view")
	}

	if task.Logs != nil {
		t.Fatal("unexpected task logs included in basic view")
	}

	tasks = listView(tes.TaskView_FULL)
	task = tasks[0]

	if task.Logs[0].Logs[0].Stdout != "hello world\n" {
		t.Fatal("Missing stdout in full view")
	}

	// test http proxy
	var r *tes.ListTasksResponse
	r, err = hcli.ListTasks("MINIMAL")
	if err != nil {
		t.Fatal(err)
	}
	task = r.Tasks[0]

	if task.Id == "" {
		t.Fatal("expected task ID in minimal view")
	}
	if task.State == tes.State_UNKNOWN {
		t.Fatal("expected complete state")
	}
	if task.Name != "" {
		t.Fatal("unexpected task name included in minimal view")
	}
	if task.Logs != nil {
		t.Fatal("unexpected task logs included in minimal view")
	}

	r, err = hcli.ListTasks("BASIC")
	if err != nil {
		t.Fatal(err)
	}
	task = r.Tasks[0]

	if task.Name == "" {
		t.Fatal("expected task name to be included basic view")
	}

	if task.Logs != nil {
		t.Fatal("unexpected task logs included in basic view")
	}

	r, err = hcli.ListTasks("FULL")
	if err != nil {
		t.Fatal(err)
	}
	task = r.Tasks[0]

	if task.Logs[0].Logs[0].Stdout != "hello world\n" {
		t.Fatal("Missing stdout in full view")
	}
}

// Test that the streaming logs pick up a single character.
// This ensures that the streaming works even when a small
// amount of logs are written (which was once a bug).
func TestSingleCharLog(t *testing.T) {
	id := run(`
    --cmd "sh -c 'echo a; sleep 100'"
  `)
	waitForRunning(id)
	time.Sleep(time.Second * 2)
	task := get(id)
	if task.Logs[0].Logs[0].Stdout != "a\n" {
		t.Fatal("Missing logs")
	}
	cancel(id)
}

// Test that port mappings are being logged.
/* TODO need ports in funnel run
func TestPortLog(t *testing.T) {
  id := run(`
    --cmd 'echo start'
    --cmd 'sleep 10'
  `)
  waitForExec(id, 2)
  task := get(id)
  if task.Logs[0].Logs[0].Ports[0].Host != 5000 {
    t.Fatal("Unexpected port logs")
  }
}
*/

// Test that a completed task cannot change state.
func TestCompleteStateImmutable(t *testing.T) {
	id := run(`
    --cmd 'echo hello'
  `)
	wait(id)
	err := cancel(id)
	if err == nil {
		t.Fatal("expected error")
	}
	task := get(id)
	if task.State != tes.State_COMPLETE {
		t.Fatal("Unexpected state")
	}
}

// Test canceling a task
func TestCancel(t *testing.T) {
	id := run(`
    --cmd 'echo start'
    --cmd 'sleep 1000'
    --cmd 'echo never'
  `)
	waitForExec(id, 1)
	cancel(id)
	// TODO docker details and container ID are very Funnel specific
	//      how could we generalize this to be reusued as TES conformance?
	waitForDockerDestroy(id + "-0")
	task := get(id)
	if task.State != tes.State_CANCELED {
		t.Fatal("Unexpected state")
	}
}

// The task executor logs list should only include entries for steps that
// have been started or completed, i.e. steps that have yet to be started
// won't show up in Task.Logs[0].Logs
func TestExecutorLogLength(t *testing.T) {
	id := run(`
    --cmd 'echo first'
    --cmd 'sleep 10'
    --cmd 'echo done'
  `)
	waitForExec(id, 2)
	task := get(id)
	cancel(id)
	if len(task.Logs[0].Logs) != 2 {
		t.Fatal("Unexpected executor log count")
	}
}

// There was a bug + fix where the task was being marked complete after
// the first step completed, but the correct behavior is to mark the
// task complete after *all* steps have completed.
func TestMarkCompleteBug(t *testing.T) {
	id := run(`
    --cmd 'echo step 1'
    --cmd 'sleep 100'
  `)
	waitForExec(id, 2)
	task := get(id)
	if task.State != tes.State_RUNNING {
		t.Fatal("Unexpected task state")
	}
}
<|endoftext|>


<commit_before>package comments

import (
	"log"
	"os"

	"github.com/google/go-github/github"
)

var (
	pendingFeedbackLabel = "pending-feedback"

	HandlerPendingFeedbackLabel = func(client *github.Client, event github.IssueCommentEvent) error {
		// if the comment is from the issue author & issue has the "pending-feedback", remove the label

		if os.Getenv("AUTO_REPLY_DEBUG") == "true" {
			log.Println("[pending_feedback_label]: received event:", event)
		}

		if *event.Sender.ID == *event.Issue.User.ID && hasLabel(event.Issue.Labels, pendingFeedbackLabel) {
			owner, name, number := *event.Repo.Owner.Login, *event.Repo.Name, *event.Issue.Number
			_, err := client.Issues.RemoveLabelForIssue(owner, name, number, pendingFeedbackLabel)
			if err != nil {
				log.Printf("[pending_feedback_label]: error removing label (%s/%s#%d): %v", owner, name, number, err)
				return err
			}
			return nil
		}
		return nil
	}
)

func hasLabel(labels []github.Label, desiredLabel string) bool {
	for _, label := range labels {
		if *label.Name == desiredLabel {
			return true
		}
	}
	return false
}
<commit_msg>Protect against nils in pending feedback labeler thing<commit_after>package comments

import (
	"log"
	"os"

	"github.com/google/go-github/github"
)

var (
	pendingFeedbackLabel = "pending-feedback"

	HandlerPendingFeedbackLabel = func(client *github.Client, event github.IssueCommentEvent) error {
		// if the comment is from the issue author & issue has the "pending-feedback", remove the label

		if os.Getenv("AUTO_REPLY_DEBUG") == "true" {
			log.Println("[pending_feedback_label]: received event:", event)
		}

		if senderAndCreatorEqual(event) && hasLabel(event.Issue.Labels, pendingFeedbackLabel) {
			owner, name, number := *event.Repo.Owner.Login, *event.Repo.Name, *event.Issue.Number
			_, err := client.Issues.RemoveLabelForIssue(owner, name, number, pendingFeedbackLabel)
			if err != nil {
				log.Printf("[pending_feedback_label]: error removing label (%s/%s#%d): %v", owner, name, number, err)
				return err
			}
			return nil
		}
		return nil
	}
)

func senderAndCreatorEqual(event github.IssueCommentEvent) bool {
	return event.Sender != nil && even.Issue != nil && event.Issue.User != nil && *event.Sender.ID == *event.Issue.User.ID
}

func hasLabel(labels []github.Label, desiredLabel string) bool {
	for _, label := range labels {
		if *label.Name == desiredLabel {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package geometry

import (
	"fmt"
	"math"
)

type Line2D struct {
	P1, P2 Point2D
}

func (l Line2D) Angle() float64 {
	dx, dy := l.P2.X-l.P1.X, l.P2.Y-l.P1.Y
	return math.Atan(dy / dx)
}

// Angular difference between the line l and a line with endpoints at line l's
// midpoint and point p.
func (l Line2D) AngularDistanceThroughPoint(p Point2D) float64 {
	rl := Line2D{l.Midpoint(), p}
	a := math.Fabs(rl.Angle() - l.Angle())
	if a > math.Pi*0.5 {
		a = math.Pi - a
	}
	return a
}

// From Dan Sunday,
// http://softsurfer.com/Archive/algorithm_0102/algorithm_0102.htm
func (l Line2D) DistanceToPoint(p Point2D, segment bool) float64 {
	v := l.ToVector()
	w := p.Minus(l.P1)
	c1 := DotProduct2D(w, v)
	if segment && c1 <= 0 {
		return p.DistanceTo(l.P1)
	}
	c2 := DotProduct2D(v, v)
	if segment && c2 <= c1 {
		return p.DistanceTo(l.P2)
	}
	b := c1 / c2
	a := l.P1.Plus(v.Scaled(b))
	return p.DistanceTo(a)
}

func (l Line2D) Dx() float64 {
	return l.P2.X - l.P1.X
}

func (l Line2D) DxDy() (float64, float64) {
	return l.P2.X - l.P1.X, l.P2.Y - l.P1.Y
}

func (l Line2D) Dy() float64 {
	return l.P2.Y - l.P1.Y
}

// Distance that each endpoint moves when l is rotated around it's midpoint so
// that it passes through p.
func (l Line2D) EndpointDistanceSquaredThroughPoint(p Point2D) float64 {
	rl := Line2D{l.Midpoint(), p}
	s := math.Sqrt(l.LengthSquared() * 0.25 / rl.LengthSquared())
	rp := Point2D{rl.P1.X + s*rl.Dx(), rl.P1.Y + s*rl.Dy()}
	d := rp.DistanceToSquared(l.P1)
	if td := rp.DistanceToSquared(l.P2); td < d {
		d = td
	}
	return d
}

// Returns the intersection point and if said point occurs on both lines.
// From Graphics Gems III, Faster Line Segment Intersection.
func (l1 Line2D) Intersection(l2 Line2D) (Point2D, bool) {
	a := l1.P2.Minus(l1.P1)
	b := l2.P1.Minus(l2.P2)
	denominator := a.Y*b.X - a.X*b.Y
	if denominator == 0.0 {
		return Point2D{math.Inf(1), math.Inf(1)}, false
	}
	denominator = 1.0 / denominator
	c := l1.P1.Minus(l2.P1)
	A := (b.Y*c.X - b.X*c.Y) * denominator
	intersection := l1.P1.Plus(a.Scaled(A))
	if A < 0.0 || A > 1.0 {
		return intersection, false
	}
	B := (a.X*c.Y - a.Y*c.X) * denominator
	if B < 0.0 || B > 1.0 {
		return intersection, false
	}
	return intersection, true
}

// From Graphics Gems III, Faster Line Segment Intersection.
func (l1 Line2D) Intersects(l2 Line2D) bool {
	a := l1.P2.Minus(l1.P1)
	b := l2.P1.Minus(l2.P2)
	denominator := a.Y*b.X - a.X*b.Y
	if denominator == 0.0 {
		return false
	}
	c := l1.P1.Minus(l2.P1)
	A := b.Y*c.X - b.X*c.Y
	if denominator > 0 && (A < 0.0 || A > denominator) {
		return false
	}
	B := a.X*c.Y - a.Y*c.X
	if denominator > 0 && (B > 0.0 || B < denominator) {
		return false
	}
	return true
}

func (l Line2D) Length() float64 {
	dx, dy := l.P2.X-l.P1.X, l.P2.Y-l.P1.Y
	return math.Sqrt(dx*dx + dy*dy)
}

func (l Line2D) LengthSquared() float64 {
	dx, dy := l.P2.X-l.P1.X, l.P2.Y-l.P1.Y
	return dx*dx + dy*dy
}

func (l Line2D) Midpoint() Point2D {
	return Point2D{(l.P1.X + l.P2.X) * 0.5, (l.P1.Y + l.P2.Y) * 0.5}
}

// Rotates a line around its midpoint
func (l Line2D) Rotated(t float64) Line2D {
	m := Point2D{(l.P1.X + l.P2.X) * 0.5, (l.P1.Y + l.P2.Y) * 0.5}
	x1 := l.P1.X - m.X
	y1 := l.P1.Y - m.Y
	x2 := l.P2.X - m.X
	y2 := l.P2.Y - m.Y
	cos := math.Cos(t)
	sin := math.Sin(t)
	return Line2D{
		Point2D{x1*cos - y1*sin + m.X, x1*sin + y1*cos + m.Y},
		Point2D{x2*cos - y2*sin + m.X, x2*sin + y2*cos + m.Y}}
}

func (l Line2D) String() string {
	return fmt.Sprintf("{%v %v}", l.P1, l.P2)
}

func (l Line2D) ToVector() Point2D {
	return Point2D{l.P2.X - l.P1.X, l.P2.Y - l.P1.Y}
}
<commit_msg>Renamed some of the long method names.<commit_after>package geometry

import (
	"fmt"
	"math"
)

type Line2D struct {
	P1, P2 Point2D
}

func (l Line2D) Angle() float64 {
	dx, dy := l.P2.X-l.P1.X, l.P2.Y-l.P1.Y
	return math.Atan(dy / dx)
}

// Angular difference between the line l and a line with endpoints at line l's
// midpoint and point p.
func (l Line2D) AngDistPt(p Point2D) float64 {
	rl := Line2D{l.Midpoint(), p}
	a := math.Fabs(rl.Angle() - l.Angle())
	if a > math.Pi*0.5 {
		a = math.Pi - a
	}
	return a
}

// From Dan Sunday,
// http://softsurfer.com/Archive/algorithm_0102/algorithm_0102.htm
func (l Line2D) LinDistPt(p Point2D, segment bool) float64 {
	v := l.ToVector()
	w := p.Minus(l.P1)
	c1 := DotProduct2D(w, v)
	if segment && c1 <= 0 {
		return p.DistanceTo(l.P1)
	}
	c2 := DotProduct2D(v, v)
	if segment && c2 <= c1 {
		return p.DistanceTo(l.P2)
	}
	b := c1 / c2
	a := l.P1.Plus(v.Scaled(b))
	return p.DistanceTo(a)
}

func (l Line2D) Dx() float64 {
	return l.P2.X - l.P1.X
}

func (l Line2D) DxDy() (float64, float64) {
	return l.P2.X - l.P1.X, l.P2.Y - l.P1.Y
}

func (l Line2D) Dy() float64 {
	return l.P2.Y - l.P1.Y
}

// Distance that each endpoint moves when l is rotated around it's midpoint so
// that it passes through p.
func (l Line2D) EndPtDistSqPt(p Point2D) float64 {
	rl := Line2D{l.Midpoint(), p}
	s := math.Sqrt(l.LengthSquared() * 0.25 / rl.LengthSquared())
	rp := Point2D{rl.P1.X + s*rl.Dx(), rl.P1.Y + s*rl.Dy()}
	d := rp.DistanceToSquared(l.P1)
	if td := rp.DistanceToSquared(l.P2); td < d {
		d = td
	}
	return d
}

// Returns the intersection point and if said point occurs on both lines.
// From Graphics Gems III, Faster Line Segment Intersection.
func (l1 Line2D) Intersection(l2 Line2D) (Point2D, bool) {
	a := l1.P2.Minus(l1.P1)
	b := l2.P1.Minus(l2.P2)
	denominator := a.Y*b.X - a.X*b.Y
	if denominator == 0.0 {
		return Point2D{math.Inf(1), math.Inf(1)}, false
	}
	denominator = 1.0 / denominator
	c := l1.P1.Minus(l2.P1)
	A := (b.Y*c.X - b.X*c.Y) * denominator
	intersection := l1.P1.Plus(a.Scaled(A))
	if A < 0.0 || A > 1.0 {
		return intersection, false
	}
	B := (a.X*c.Y - a.Y*c.X) * denominator
	if B < 0.0 || B > 1.0 {
		return intersection, false
	}
	return intersection, true
}

// From Graphics Gems III, Faster Line Segment Intersection.
func (l1 Line2D) Intersects(l2 Line2D) bool {
	a := l1.P2.Minus(l1.P1)
	b := l2.P1.Minus(l2.P2)
	denominator := a.Y*b.X - a.X*b.Y
	if denominator == 0.0 {
		return false
	}
	c := l1.P1.Minus(l2.P1)
	A := b.Y*c.X - b.X*c.Y
	if denominator > 0 && (A < 0.0 || A > denominator) {
		return false
	}
	B := a.X*c.Y - a.Y*c.X
	if denominator > 0 && (B > 0.0 || B < denominator) {
		return false
	}
	return true
}

func (l Line2D) Length() float64 {
	dx, dy := l.P2.X-l.P1.X, l.P2.Y-l.P1.Y
	return math.Sqrt(dx*dx + dy*dy)
}

func (l Line2D) LengthSquared() float64 {
	dx, dy := l.P2.X-l.P1.X, l.P2.Y-l.P1.Y
	return dx*dx + dy*dy
}

func (l Line2D) Midpoint() Point2D {
	return Point2D{(l.P1.X + l.P2.X) * 0.5, (l.P1.Y + l.P2.Y) * 0.5}
}

// Rotates a line around its midpoint
func (l Line2D) Rotated(t float64) Line2D {
	m := Point2D{(l.P1.X + l.P2.X) * 0.5, (l.P1.Y + l.P2.Y) * 0.5}
	x1 := l.P1.X - m.X
	y1 := l.P1.Y - m.Y
	x2 := l.P2.X - m.X
	y2 := l.P2.Y - m.Y
	cos := math.Cos(t)
	sin := math.Sin(t)
	return Line2D{
		Point2D{x1*cos - y1*sin + m.X, x1*sin + y1*cos + m.Y},
		Point2D{x2*cos - y2*sin + m.X, x2*sin + y2*cos + m.Y}}
}

func (l Line2D) String() string {
	return fmt.Sprintf("{%v %v}", l.P1, l.P2)
}

func (l Line2D) ToVector() Point2D {
	return Point2D{l.P2.X - l.P1.X, l.P2.Y - l.P1.Y}
}
<|endoftext|>


<commit_before>package filesystem

import (
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"sort"
	"strings"
	"sync"
	"syscall"

	"golang.org/x/sync/errgroup"
	"golang.org/x/sync/semaphore"
	"golang.org/x/sys/unix"

	"github.com/iafan/cwalk"
	log "github.com/sirupsen/logrus"

	l "github.com/cvmfs/ducc/log"
)

func init() {
	// we need a lot more of goroutines when scanning CVMFS directories
	cwalk.NumWorkers = runtime.GOMAXPROCS(0) * 10
	cwalk.BufferSize = cwalk.NumWorkers
}

func ApplyDirectory(bottom, top string) error {
	bottomDir, err := os.Stat(bottom)
	if err != nil {
		return err
	}
	if !bottomDir.IsDir() {
		return fmt.Errorf("bottom directory '%s' is not a directory", bottom)
	}

	topDir, err := os.Stat(top)
	if err != nil {
		return err
	}
	if !topDir.IsDir() {
		return fmt.Errorf("top directory '%s' is not a directory", top)
	}

	log := l.Log().WithFields(log.Fields{"top layer": top, "bottom": bottom})

	log.Info("Start walking the top layer")

	var opaqueMtx sync.Mutex
	var opaqueWhiteouts []string

	var whiteMtx sync.Mutex
	var whiteOuts []string

	var standardMtx sync.Mutex
	var standards []FilePathAndInfo

	// navigating the filesystem in slow, especially on top of CVMFS
	// for each file we need to go through fuse and to a network call if we are unlucky
	// we make this walk in parallel, so to hide the latency
	err = cwalk.Walk(top, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		path = strings.Replace(path, top, "", 1)
		if isOpaqueWhiteout(path) {
			opaqueMtx.Lock()
			defer opaqueMtx.Unlock()
			opaqueWhiteouts = append(opaqueWhiteouts, path)
		} else if isWhiteout(path) {
			whiteMtx.Lock()
			defer whiteMtx.Unlock()
			whiteOuts = append(whiteOuts, path)
		} else {
			standardMtx.Lock()
			defer standardMtx.Unlock()
			standards = append(standards, FilePathAndInfo{path, info})
		}
		return nil
	})

	if err != nil {
		return err
	}

	log.Info("Done walking the top layer")

	log.Info("Start sorting the files")

	// it is necessary to sort everything since the parallel walk returns unsorted data
	var wg sync.WaitGroup
	wg.Add(3)
	go func() {
		defer wg.Done()
		sort.Strings(opaqueWhiteouts)
	}()
	go func() {
		defer wg.Done()
		sort.Strings(whiteOuts)
	}()
	go func() {
		defer wg.Done()
		sort.Slice(standards,
			func(i, j int) bool { return standards[i].Path < standards[j].Path })
	}()
	wg.Wait()

	log.Info("Done sorting the files")

	log.Info("Start applying the opaque whiteouts")

	// all the call here happens against a local and fast filesystem
	// we are not going to make effort to make this parallel
	for _, opaqueWhiteout := range opaqueWhiteouts {
		// delete all the sibling of the opaque whiteout file file
		opaqueDirPath := filepath.Join(bottom, filepath.Dir(opaqueWhiteout))
		inOpaqueDirectory, err := ioutil.ReadDir(opaqueDirPath)
		if err != nil {
			// it may happen a directory that does not exists in the lower layer
			// is an opaqueWhiteout in the top layer
			continue
		}
		for _, toDelete := range inOpaqueDirectory {
			toDeletePath := filepath.Join(opaqueDirPath, toDelete.Name())
			err = os.RemoveAll(toDeletePath)
			if err != nil {
				return err
			}
		}
	}

	log.Info("Done applying the opaque whiteouts")

	log.Info("Start applying the whiteouts")

	for _, whiteOut := range whiteOuts {
		// delete the file that should be whiteout
		whiteOutBaseFilename := filepath.Base(whiteOut)
		toRemoveBaseFilename := getPathToWhiteout(whiteOutBaseFilename)
		whiteOutPath := filepath.Join(bottom, filepath.Dir(whiteOut), toRemoveBaseFilename)
		err = os.RemoveAll(whiteOutPath)
		if err != nil {
			return err
		}
	}

	log.Info("Done applying the whiteouts")

	log.Info("Start applying the files on top of the bottom dir")

	errGrp := new(errgroup.Group)
	sem := semaphore.NewWeighted(int64(500))

	specialBitsMask := ^(os.ModeSetuid | os.ModeSetgid | os.ModeSticky)

	fmt.Println("specialBitsMask", specialBitsMask)

	for _, file := range standards {
		// add the file or directory
		// remember to set permision and owner
		// If it is a directory, just create the directory with
		//     all the permision and the correct owner
		// If is is a file, just create the file and then copy all the content

		// start by getting the UID and GID of the file
		var UID int
		var GID int
		var rdev int
		if stat, ok := file.Info.Sys().(*syscall.Stat_t); ok {
			UID = int(stat.Uid)
			GID = int(stat.Gid)
			rdev = int(stat.Rdev)
		} else {
			UID = os.Getuid()
			GID = os.Getgid()
			rdev = 0
		}
		// then we get the permission
		filemode := file.Info.Mode()
		// and finally we compose the path of the file
		path := filepath.Join(bottom, file.Path)

		switch {
		case filemode.IsDir():
			// we create a very open directory to avoid permission problem
			// we fix all the permission in the defer statements.
			// the defer are stacked, hence the last directory created will
			// be the first one to have its permission changed.
			err = os.MkdirAll(path, 0700)
			if err != nil {
				return err
			}
			defer func(path string, filemode os.FileMode, UID, GID int) {
				os.Chmod(path, (filemode|0700)&specialBitsMask)
				os.Chown(path, UID, GID)
			}(path, filemode, UID, GID)

		case filemode.IsRegular():
			file := file
			errGrp.Go(func() error {
				sem.Acquire(context.TODO(), 1)
				defer sem.Release(1)
				// we remove the file, it may not exists and this call will return PathError.
				// we just ignore this kind of error
				os.Remove(path)
				newFile, err := os.Create(path)
				if err != nil {
					return err
				}
				defer newFile.Close()
				srcFile, err := os.Open(filepath.Join(top, file.Path))
				if err != nil {
					if err == os.ErrPermission {
						fmt.Printf("Permission error detected! %s", err)
					}
					return err
				}
				defer srcFile.Close()
				_, err = io.Copy(newFile, srcFile)
				if err != nil {
					return err
				}
				newFile.Chown(UID, GID)
				newFile.Chmod((filemode | 0600) & specialBitsMask)
				return nil
			})
		case filemode&os.ModeSymlink != 0:
			os.RemoveAll(path)
			link, err := os.Readlink(filepath.Join(top, file.Path))
			if err != nil {
				return err
			}

			// links can be either absolutes or relative
			// if they are absolute we need to swap the top prefix with the bottom one
			// if they are relative they are good to go
			// even though we should pretty much have alway relative links
			if strings.HasPrefix(link, top) {
				link = strings.Replace(link, top, bottom, 1)
			}

			err = os.Symlink(link, path)
			if err != nil {
				return err
			}
			os.Lchown(path, UID, GID)

		case filemode&os.ModeNamedPipe != 0:
			fallthrough
		case filemode&os.ModeSocket != 0:
			err = unix.Mkfifo(path, uint32(filemode))
			if err != nil {
				return err
			}
			os.Chown(path, UID, GID)
		case filemode&os.ModeDevice != 0:
			err = unix.Mknod(path, uint32(filemode), rdev)
			if err != nil {
				return err
			}
			os.Chown(path, UID, GID)
		}

	}
	if err := errGrp.Wait(); err != nil {
		log.Error("Got error when copying files: ", err)
		return err
	}
	log.Info("Done applying the files on top of the bottom dir")
	return nil
}

type FilePathAndInfo struct {
	Path string
	Info os.FileInfo
}

func isWhiteout(path string) bool {
	base := filepath.Base(path)
	if len(base) <= 3 {
		return false
	}
	return base[0:4] == ".wh."
}

func isOpaqueWhiteout(path string) bool {
	base := filepath.Base(path)
	return base == ".wh..wh..opq"
}

func getPathToWhiteout(path string) string {
	if !isWhiteout(path) {
		return ""
	}
	if isOpaqueWhiteout(path) {
		return ""
	}
	return strings.TrimPrefix(path, ".wh.")
}
<commit_msg>ingest as accurantely as possible<commit_after>package filesystem

import (
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"sort"
	"strings"
	"sync"
	"syscall"

	"golang.org/x/sync/errgroup"
	"golang.org/x/sync/semaphore"
	"golang.org/x/sys/unix"

	"github.com/iafan/cwalk"
	log "github.com/sirupsen/logrus"

	l "github.com/cvmfs/ducc/log"
)

func init() {
	// we need a lot more of goroutines when scanning CVMFS directories
	cwalk.NumWorkers = runtime.GOMAXPROCS(0) * 10
	cwalk.BufferSize = cwalk.NumWorkers
}

func ApplyDirectory(bottom, top string) error {
	bottomDir, err := os.Stat(bottom)
	if err != nil {
		return err
	}
	if !bottomDir.IsDir() {
		return fmt.Errorf("bottom directory '%s' is not a directory", bottom)
	}

	topDir, err := os.Stat(top)
	if err != nil {
		return err
	}
	if !topDir.IsDir() {
		return fmt.Errorf("top directory '%s' is not a directory", top)
	}

	log := l.Log().WithFields(log.Fields{"top layer": top, "bottom": bottom})

	log.Info("Start walking the top layer")

	var opaqueMtx sync.Mutex
	var opaqueWhiteouts []string

	var whiteMtx sync.Mutex
	var whiteOuts []string

	var standardMtx sync.Mutex
	var standards []FilePathAndInfo

	// navigating the filesystem in slow, especially on top of CVMFS
	// for each file we need to go through fuse and to a network call if we are unlucky
	// we make this walk in parallel, so to hide the latency
	err = cwalk.Walk(top, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		path = strings.Replace(path, top, "", 1)
		if isOpaqueWhiteout(path) {
			opaqueMtx.Lock()
			defer opaqueMtx.Unlock()
			opaqueWhiteouts = append(opaqueWhiteouts, path)
		} else if isWhiteout(path) {
			whiteMtx.Lock()
			defer whiteMtx.Unlock()
			whiteOuts = append(whiteOuts, path)
		} else {
			standardMtx.Lock()
			defer standardMtx.Unlock()
			standards = append(standards, FilePathAndInfo{path, info})
		}
		return nil
	})

	if err != nil {
		return err
	}

	log.Info("Done walking the top layer")

	log.Info("Start sorting the files")

	// it is necessary to sort everything since the parallel walk returns unsorted data
	var wg sync.WaitGroup
	wg.Add(3)
	go func() {
		defer wg.Done()
		sort.Strings(opaqueWhiteouts)
	}()
	go func() {
		defer wg.Done()
		sort.Strings(whiteOuts)
	}()
	go func() {
		defer wg.Done()
		sort.Slice(standards,
			func(i, j int) bool { return standards[i].Path < standards[j].Path })
	}()
	wg.Wait()

	log.Info("Done sorting the files")

	log.Info("Start applying the opaque whiteouts")

	// all the call here happens against a local and fast filesystem
	// we are not going to make effort to make this parallel
	for _, opaqueWhiteout := range opaqueWhiteouts {
		// delete all the sibling of the opaque whiteout file file
		opaqueDirPath := filepath.Join(bottom, filepath.Dir(opaqueWhiteout))
		inOpaqueDirectory, err := ioutil.ReadDir(opaqueDirPath)
		if err != nil {
			// it may happen a directory that does not exists in the lower layer
			// is an opaqueWhiteout in the top layer
			continue
		}
		for _, toDelete := range inOpaqueDirectory {
			toDeletePath := filepath.Join(opaqueDirPath, toDelete.Name())
			err = os.RemoveAll(toDeletePath)
			if err != nil {
				return err
			}
		}
	}

	log.Info("Done applying the opaque whiteouts")

	log.Info("Start applying the whiteouts")

	for _, whiteOut := range whiteOuts {
		// delete the file that should be whiteout
		whiteOutBaseFilename := filepath.Base(whiteOut)
		toRemoveBaseFilename := getPathToWhiteout(whiteOutBaseFilename)
		whiteOutPath := filepath.Join(bottom, filepath.Dir(whiteOut), toRemoveBaseFilename)
		err = os.RemoveAll(whiteOutPath)
		if err != nil {
			return err
		}
	}

	log.Info("Done applying the whiteouts")

	log.Info("Start applying the files on top of the bottom dir")

	errGrp := new(errgroup.Group)
	sem := semaphore.NewWeighted(int64(500))

	for _, file := range standards {
		// add the file or directory
		// remember to set permision and owner
		// If it is a directory, just create the directory with
		//     all the permision and the correct owner
		// If is is a file, just create the file and then copy all the content

		// start by getting the UID and GID of the file
		var UID int
		var GID int
		var rdev int
		if stat, ok := file.Info.Sys().(*syscall.Stat_t); ok {
			UID = int(stat.Uid)
			GID = int(stat.Gid)
			rdev = int(stat.Rdev)
		} else {
			UID = os.Getuid()
			GID = os.Getgid()
			rdev = 0
		}
		// then we get the permission
		filemode := file.Info.Mode()
		// and finally we compose the path of the file
		path := filepath.Join(bottom, file.Path)

		switch {
		case filemode.IsDir():
			// we create a very open directory to avoid permission problem
			// we fix all the permission in the defer statements.
			// the defer are stacked, hence the last directory created will
			// be the first one to have its permission changed.
			err = os.MkdirAll(path, 0700)
			if err != nil {
				return err
			}
			defer func(path string, filemode os.FileMode, UID, GID int) {
				os.Chmod(path, filemode|0700)
				os.Chown(path, UID, GID)
			}(path, filemode, UID, GID)

		case filemode.IsRegular():
			file := file
			errGrp.Go(func() error {
				sem.Acquire(context.TODO(), 1)
				defer sem.Release(1)
				// we remove the file, it may not exists and this call will return PathError.
				// we just ignore this kind of error
				os.Remove(path)
				newFile, err := os.Create(path)
				if err != nil {
					return err
				}
				defer newFile.Close()
				srcFile, err := os.Open(filepath.Join(top, file.Path))
				if err != nil {
					if err == os.ErrPermission {
						fmt.Printf("Permission error detected! %s", err)
					}
					return err
				}
				defer srcFile.Close()
				_, err = io.Copy(newFile, srcFile)
				if err != nil {
					return err
				}
				newFile.Chown(UID, GID)
				newFile.Chmod(filemode | 0600)
				return nil
			})
		case filemode&os.ModeSymlink != 0:
			os.RemoveAll(path)
			link, err := os.Readlink(filepath.Join(top, file.Path))
			if err != nil {
				return err
			}

			// links can be either absolutes or relative
			// if they are absolute we need to swap the top prefix with the bottom one
			// if they are relative they are good to go
			// even though we should pretty much have alway relative links
			if strings.HasPrefix(link, top) {
				link = strings.Replace(link, top, bottom, 1)
			}

			err = os.Symlink(link, path)
			if err != nil {
				return err
			}
			os.Lchown(path, UID, GID)

		case filemode&os.ModeNamedPipe != 0:
			fallthrough
		case filemode&os.ModeSocket != 0:
			err = unix.Mkfifo(path, uint32(filemode))
			if err != nil {
				return err
			}
			os.Chown(path, UID, GID)
		case filemode&os.ModeDevice != 0:
			err = unix.Mknod(path, uint32(filemode), rdev)
			if err != nil {
				return err
			}
			os.Chown(path, UID, GID)
		}

	}
	if err := errGrp.Wait(); err != nil {
		log.Error("Got error when copying files: ", err)
		return err
	}
	log.Info("Done applying the files on top of the bottom dir")
	return nil
}

type FilePathAndInfo struct {
	Path string
	Info os.FileInfo
}

func isWhiteout(path string) bool {
	base := filepath.Base(path)
	if len(base) <= 3 {
		return false
	}
	return base[0:4] == ".wh."
}

func isOpaqueWhiteout(path string) bool {
	base := filepath.Base(path)
	return base == ".wh..wh..opq"
}

func getPathToWhiteout(path string) string {
	if !isWhiteout(path) {
		return ""
	}
	if isOpaqueWhiteout(path) {
		return ""
	}
	return strings.TrimPrefix(path, ".wh.")
}
<|endoftext|>


<commit_before>// Copyright 2016 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package v3rpc

import (
	"io"

	"github.com/coreos/etcd/etcdserver"
	pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
	"github.com/coreos/etcd/lease"
	"golang.org/x/net/context"
)

type LeaseServer struct {
	hdr header
	le  etcdserver.Lessor
}

func NewLeaseServer(s *etcdserver.EtcdServer) pb.LeaseServer {
	return &LeaseServer{le: s, hdr: newHeader(s)}
}

func (ls *LeaseServer) LeaseGrant(ctx context.Context, cr *pb.LeaseGrantRequest) (*pb.LeaseGrantResponse, error) {
	resp, err := ls.le.LeaseGrant(ctx, cr)

	if err != nil {
		return nil, togRPCError(err)
	}
	ls.hdr.fill(resp.Header)
	return resp, nil
}

func (ls *LeaseServer) LeaseRevoke(ctx context.Context, rr *pb.LeaseRevokeRequest) (*pb.LeaseRevokeResponse, error) {
	resp, err := ls.le.LeaseRevoke(ctx, rr)
	if err != nil {
		return nil, togRPCError(err)
	}
	ls.hdr.fill(resp.Header)
	return resp, nil
}

func (ls *LeaseServer) LeaseTimeToLive(ctx context.Context, rr *pb.LeaseTimeToLiveRequest) (*pb.LeaseTimeToLiveResponse, error) {
	resp, err := ls.le.LeaseTimeToLive(ctx, rr)
	if err != nil && err != lease.ErrLeaseNotFound {
		return nil, togRPCError(err)
	}
	if err == lease.ErrLeaseNotFound {
		resp = &pb.LeaseTimeToLiveResponse{
			Header: &pb.ResponseHeader{},
			ID:     rr.ID,
			TTL:    -1,
		}
	}
	ls.hdr.fill(resp.Header)
	return resp, nil
}

func (ls *LeaseServer) LeaseKeepAlive(stream pb.Lease_LeaseKeepAliveServer) error {
	for {
		req, err := stream.Recv()
		if err == io.EOF {
			return nil
		}
		if err != nil {
			return err
		}

		// Create header before we sent out the renew request.
		// This can make sure that the revision is strictly smaller or equal to
		// when the keepalive happened at the local server (when the local server is the leader)
		// or remote leader.
		// Without this, a lease might be revoked at rev 3 but client can see the keepalive succeeded
		// at rev 4.
		resp := &pb.LeaseKeepAliveResponse{ID: req.ID, Header: &pb.ResponseHeader{}}
		ls.hdr.fill(resp.Header)

		ttl, err := ls.le.LeaseRenew(stream.Context(), lease.LeaseID(req.ID))
		if err == lease.ErrLeaseNotFound {
			err = nil
			ttl = 0
		}

		if err != nil {
			return togRPCError(err)
		}

		resp.TTL = ttl
		err = stream.Send(resp)
		if err != nil {
			return err
		}
	}
}
<commit_msg>v3rpc: return leader loss error if lease stream is canceled<commit_after>// Copyright 2016 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package v3rpc

import (
	"io"

	"github.com/coreos/etcd/etcdserver"
	"github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes"
	pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
	"github.com/coreos/etcd/lease"
	"golang.org/x/net/context"
)

type LeaseServer struct {
	hdr header
	le  etcdserver.Lessor
}

func NewLeaseServer(s *etcdserver.EtcdServer) pb.LeaseServer {
	return &LeaseServer{le: s, hdr: newHeader(s)}
}

func (ls *LeaseServer) LeaseGrant(ctx context.Context, cr *pb.LeaseGrantRequest) (*pb.LeaseGrantResponse, error) {
	resp, err := ls.le.LeaseGrant(ctx, cr)

	if err != nil {
		return nil, togRPCError(err)
	}
	ls.hdr.fill(resp.Header)
	return resp, nil
}

func (ls *LeaseServer) LeaseRevoke(ctx context.Context, rr *pb.LeaseRevokeRequest) (*pb.LeaseRevokeResponse, error) {
	resp, err := ls.le.LeaseRevoke(ctx, rr)
	if err != nil {
		return nil, togRPCError(err)
	}
	ls.hdr.fill(resp.Header)
	return resp, nil
}

func (ls *LeaseServer) LeaseTimeToLive(ctx context.Context, rr *pb.LeaseTimeToLiveRequest) (*pb.LeaseTimeToLiveResponse, error) {
	resp, err := ls.le.LeaseTimeToLive(ctx, rr)
	if err != nil && err != lease.ErrLeaseNotFound {
		return nil, togRPCError(err)
	}
	if err == lease.ErrLeaseNotFound {
		resp = &pb.LeaseTimeToLiveResponse{
			Header: &pb.ResponseHeader{},
			ID:     rr.ID,
			TTL:    -1,
		}
	}
	ls.hdr.fill(resp.Header)
	return resp, nil
}

func (ls *LeaseServer) LeaseKeepAlive(stream pb.Lease_LeaseKeepAliveServer) (err error) {
	errc := make(chan error, 1)
	go func() {
		errc <- ls.leaseKeepAlive(stream)
	}()
	select {
	case err = <-errc:
	case <-stream.Context().Done():
		// the only server-side cancellation is noleader for now.
		err = stream.Context().Err()
		if err == context.Canceled {
			err = rpctypes.ErrGRPCNoLeader
		}
	}
	return err
}

func (ls *LeaseServer) leaseKeepAlive(stream pb.Lease_LeaseKeepAliveServer) error {
	for {
		req, err := stream.Recv()
		if err == io.EOF {
			return nil
		}
		if err != nil {
			return err
		}

		// Create header before we sent out the renew request.
		// This can make sure that the revision is strictly smaller or equal to
		// when the keepalive happened at the local server (when the local server is the leader)
		// or remote leader.
		// Without this, a lease might be revoked at rev 3 but client can see the keepalive succeeded
		// at rev 4.
		resp := &pb.LeaseKeepAliveResponse{ID: req.ID, Header: &pb.ResponseHeader{}}
		ls.hdr.fill(resp.Header)

		ttl, err := ls.le.LeaseRenew(stream.Context(), lease.LeaseID(req.ID))
		if err == lease.ErrLeaseNotFound {
			err = nil
			ttl = 0
		}

		if err != nil {
			return togRPCError(err)
		}

		resp.TTL = ttl
		err = stream.Send(resp)
		if err != nil {
			return err
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"os"
	"time"

	"github.com/moov-io/ach"
)

func main() {
	// Example transfer to write an ACH PPD file to send/credit a external institutions account
	// Important: All financial instituions are different and will require registration and exact field values.

	// Set originator bank ODFI and destination Operator for the financial institution
	// this is the funding/recieving source of the transfer
	fh := ach.NewFileHeader()
	fh.ImmediateDestination = 9876543210 // Your Bank transit/routing number
	fh.ImmediateOrigin = 1234567890      // Organization or Company federal usually 1 and tax ID
	fh.FileCreationDate = time.Now()
	fh.ImmediateDestinationName = "Federal Reserve Bank"
	fh.ImmediateOriginName = "My Bank Name"

	// BatchHeader identifies the originating entity and the type of transactions contained in the batch
	bh := ach.NewBatchHeader()
	bh.ServiceClassCode = 220                  // ACH credit pushes money out, 225 debits/pulls money in.
	bh.CompanyName = "Name on Bank Account"    // The name of the company/person that has relationship with reciever
	bh.CompanyIdentification = "123456789"     // Identify the originator of the transaction. Usually: 1 followed by 9 digit tax id
	bh.StandardEntryClassCode = "PPD"          // Consumer destination vs Company CCD
	bh.CompanyEntryDescription = "Loan Payoff" // will be on recieving accounts statement
	bh.ODFIIdentification = 12345678           // first 8 digits of your bank account

	// Identifies the recievers account information
	// can be multiple entrys per batch
	entry := ach.NewEntryDetail()
	// Identifies the entry as a debit and credit entry AND to what type of account (Savings, DDA, Loan
	entry.TransactionCode = 22
	entry.SetRDFI(9101298)                            // First 8 digits of recievers bank transit rounting number
	entry.DFIAccountNumber = "12345678"               // recievers bank account number
	entry.Amount = 100000000                          // Amount of transaction with no decimil. One dollar and eleven cents = 111
	entry.IndividualName = "Name on Reciever Account" // Identifies the reciever of the transaction

	// build the batch
	batch := ach.NewBatch().SetHeader(bh)
	batch.AddEntryDetail(entry)
	if err := batch.Build(); err != nil {
		log.Fatalf("Unexpected error building batch: %s\n", err)
	}

	// build the file
	file := ach.NewFile()
	file.SetHeader(fh)
	file.AddBatch(batch)
	if err := file.Build(); err != nil {
		log.Fatalf("Unexpected error building file: %s\n", err)
	}

	// write the file to std out. Anything io.Writer
	w := ach.NewWriter(os.Stdout)
	if err := w.WriteAll([]*ach.File{file}); err != nil {
		log.Fatalf("Unexpected error: %s\n", err)
	}
	w.Flush()
}
<commit_msg>Updated example<commit_after>package main

import (
	"log"
	"os"
	"strconv"
	"time"

	"github.com/moov-io/ach"
)

func main() {
	// Example transfer to write an ACH PPD file to send/credit a external institutions account
	// Important: All financial instituions are different and will require registration and exact field values.

	// Set originator bank ODFI and destination Operator for the financial institution
	// this is the funding/recieving source of the transfer
	fh := ach.NewFileHeader()
	fh.ImmediateDestination = 9876543210 // A blank space followed by your ODFI's transit/routing numbe
	fh.ImmediateOrigin = 1234567890      // Organization or Company FED ID usually 1 and FEIN/SSN. Assigned by your ODFI
	fh.FileCreationDate = time.Now()     // Todays Date
	fh.ImmediateDestinationName = "Federal Reserve Bank"
	fh.ImmediateOriginName = "My Bank Name"

	// BatchHeader identifies the originating entity and the type of transactions contained in the batch
	bh := ach.NewBatchHeader()
	bh.ServiceClassCode = 220          // ACH credit pushes money out, 225 debits/pulls money in.
	bh.CompanyName = "Name on Account" // The name of the company/person that has relationship with reciever
	bh.CompanyIdentification = strconv.Itoa(fh.ImmediateOrigin)
	bh.StandardEntryClassCode = "PPD"         // Consumer destination vs Company CCD
	bh.CompanyEntryDescription = "REG.SALARY" // will be on recieving accounts statement
	bh.EffectiveEntryDate = time.Now().AddDate(0, 0, 1)
	bh.ODFIIdentification = 12345678 // first 8 digits of your bank account

	// Identifies the recievers account information
	// can be multiple entrys per batch
	entry := ach.NewEntryDetail()
	// Identifies the entry as a debit and credit entry AND to what type of account (Savings, DDA, Loan
	entry.TransactionCode = 22                     // Code 22: Credit (deposit) to checking account
	entry.SetRDFI(9101298)                         // Recievers bank transit rounting number
	entry.DFIAccountNumber = "12345678"            // Recievers bank account number
	entry.Amount = 100000000                       // Amount of transaction with no decimil. One dollar and eleven cents = 111
	entry.IndividualName = "Reciever Account Name" // Identifies the reciever of the transaction

	// build the batch
	batch := ach.NewBatch().SetHeader(bh)
	batch.AddEntryDetail(entry)
	if err := batch.Build(); err != nil {
		log.Fatalf("Unexpected error building batch: %s\n", err)
	}

	// build the file
	file := ach.NewFile()
	file.SetHeader(fh)
	file.AddBatch(batch)
	if err := file.Build(); err != nil {
		log.Fatalf("Unexpected error building file: %s\n", err)
	}

	// write the file to std out. Anything io.Writer
	w := ach.NewWriter(os.Stdout)
	if err := w.WriteAll([]*ach.File{file}); err != nil {
		log.Fatalf("Unexpected error: %s\n", err)
	}
	w.Flush()
}
<|endoftext|>


<commit_before>/*
Copyright 2013 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package audio

import (
	"math"
	"math/rand"

	"github.com/nf/sigourney/fast"
)

func sampleToHz(s Sample) float64 {
	return 440 * fast.Exp2(float64(s)*10)
}

func NewSquare() *Square {
	o := &Square{}
	o.inputs("pitch", &o.pitch, "syn", &o.syn)
	return o
}

type Square struct {
	sink
	pitch Processor // 0.1/oct, 0 == 440Hz
	syn   trigger

	pos float64
}

func (o *Square) Process(s []Sample) {
	o.pitch.Process(s)
	t := o.syn.Process()
	p := o.pos
	hz, lastS := sampleToHz(s[0]), s[0]
	for i := range s {
		if o.syn.isTrigger(t[i]) {
			p = 0
		}
		if s[i] != lastS {
			hz = sampleToHz(s[i])
		}
		p += hz
		if p > waveHz {
			p -= waveHz
		}
		if p > waveHz/2 {
			s[i] = -1
		} else {
			s[i] = 1
		}
	}
	o.pos = p
}

func NewSin() *Sin {
	o := &Sin{}
	o.inputs("pitch", &o.pitch, "syn", &o.syn)
	return o
}

type Sin struct {
	sink
	pitch Processor // 0.1/oct, 0 == 440Hz
	syn   trigger

	pos float64
}

func (o *Sin) Process(s []Sample) {
	o.pitch.Process(s)
	t := o.syn.Process()
	p := o.pos
	hz, lastS := sampleToHz(s[0]), s[0]
	for i := range s {
		if o.syn.isTrigger(t[i]) {
			p = 0
		}
		if s[i] != lastS {
			hz = sampleToHz(s[i])
		}
		s[i] = Sample(fast.Sin(p * 2 * math.Pi))
		p += hz / waveHz
		if p > 100 {
			p -= 100
		}
	}
	o.pos = p
}

func NewMul() *Mul {
	a := &Mul{}
	a.inputs("a", &a.a, "b", &a.b)
	return a
}

type Mul struct {
	sink
	a Processor
	b source
}

func (a *Mul) Process(s []Sample) {
	a.a.Process(s)
	m := a.b.Process()
	for i := range s {
		s[i] *= m[i]
	}
}

func NewSum() *Sum {
	s := &Sum{}
	s.inputs("a", &s.a, "b", &s.b)
	return s
}

type Sum struct {
	sink
	a Processor
	b source
}

func (s *Sum) Process(a []Sample) {
	s.a.Process(a)
	b := s.b.Process()
	for i := range a {
		a[i] += b[i]
	}
}

func NewEnv() *Env {
	e := &Env{}
	e.inputs("gate", &e.gate, "trig", &e.trig, "att", &e.att, "dec", &e.dec)
	return e
}

type Env struct {
	sink
	gate     Processor
	trig     trigger
	att, dec source

	v      Sample
	up     bool
	wasLow bool
}

func (e *Env) Process(s []Sample) {
	e.gate.Process(s)
	att, dec, t := e.att.Process(), e.dec.Process(), e.trig.Process()
	v := e.v
	for i := range s {
		trigger := e.trig.isTrigger(t[i])
		if !e.up {
			if trigger || v < s[i] {
				e.up = true
			} else if v > s[i] {
				if d := dec[i]; d > 0 {
					v -= 1 / (d * waveHz * 10)
				}
			}
		}
		if e.up {
			if a := att[i]; a > 0 {
				v += 1 / (a * waveHz * 10)
			}
		}
		if v > 1 {
			v = 1
			e.up = false
		} else if v < 0 {
			v = 0
		}
		s[i] = v
	}
	e.v = v
}

func NewClip() *Clip {
	c := &Clip{}
	c.inputs("in", &c.in)
	return c
}

type Clip struct {
	sink
	in Processor
}

func (c *Clip) Process(s []Sample) {
	c.in.Process(s)
	for i, v := range s {
		if v > 1 {
			s[i] = 1
		} else if v < -1 {
			s[i] = -1
		}
	}
}

type Value Sample

func (v Value) Process(s []Sample) {
	for i := range s {
		s[i] = Sample(v)
	}
}

func NewRand() *Rand {
	r := &Rand{}
	r.inputs("min", &r.min, "max", &r.max, "trig", &r.trig)
	return r
}

type Rand struct {
	sink
	min  Processor
	max  source
	trig trigger

	last Sample
}

func (r *Rand) Process(s []Sample) {
	r.min.Process(s)
	max, t := r.max.Process(), r.trig.Process()
	v := r.last
	for i := range s {
		if r.trig.isTrigger(t[i]) {
			v = s[i] + Sample(rand.Float64())*(max[i]-s[i])
		}
		s[i] = v
	}
	r.last = v
}

func NewDelay() *Delay {
	d := &Delay{buf: make([]Sample, waveHz)}
	d.inputs("in", &d.in, "len", &d.len)
	return d
}

type Delay struct {
	sink
	in  Processor
	len source

	p   int
	buf []Sample
}

func (d *Delay) Process(s []Sample) {
	d.in.Process(s)
	l := d.len.Process()
	p := d.p
	for i := range s {
		max := int(l[i] * waveHz)
		if max <= 0 {
			s[i] = 0
			continue
		}
		if max > waveHz {
			max = waveHz
		}
		if p >= max {
			p = 0
		}
		s[i], d.buf[p] = d.buf[p], s[i]
		p++

	}
	d.p = p
}

func NewDup(src Processor) *Dup {
	d := &Dup{src: src}
	return d
}

type Dup struct {
	src  Processor
	outs []*Output
	buf  []Sample
	done bool
}

func (d *Dup) Tick() {
	d.done = false
}

func (d *Dup) SetSource(p Processor) {
	d.src = p
}

func (d *Dup) Output() *Output {
	o := &Output{d: d}
	d.outs = append(d.outs, o)
	if len(d.outs) > 1 && d.buf == nil {
		d.buf = make([]Sample, nSamples)
	}
	return o
}

type Output struct {
	d *Dup
}

func (o *Output) Process(p []Sample) {
	if !o.d.done {
		o.d.done = true
		o.d.src.Process(p)
		if len(o.d.outs) > 1 {
			copy(o.d.buf, p)
		}
	} else {
		copy(p, o.d.buf)
	}
}

func (o *Output) Close() {
	outs := o.d.outs
	for i, o2 := range outs {
		if o == o2 {
			copy(outs[i:], outs[i+1:])
			o.d.outs = outs[:len(outs)-1]
			break
		}
	}
}
<commit_msg>pass through when delay length <= 0<commit_after>/*
Copyright 2013 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package audio

import (
	"math"
	"math/rand"

	"github.com/nf/sigourney/fast"
)

func sampleToHz(s Sample) float64 {
	return 440 * fast.Exp2(float64(s)*10)
}

func NewSquare() *Square {
	o := &Square{}
	o.inputs("pitch", &o.pitch, "syn", &o.syn)
	return o
}

type Square struct {
	sink
	pitch Processor // 0.1/oct, 0 == 440Hz
	syn   trigger

	pos float64
}

func (o *Square) Process(s []Sample) {
	o.pitch.Process(s)
	t := o.syn.Process()
	p := o.pos
	hz, lastS := sampleToHz(s[0]), s[0]
	for i := range s {
		if o.syn.isTrigger(t[i]) {
			p = 0
		}
		if s[i] != lastS {
			hz = sampleToHz(s[i])
		}
		p += hz
		if p > waveHz {
			p -= waveHz
		}
		if p > waveHz/2 {
			s[i] = -1
		} else {
			s[i] = 1
		}
	}
	o.pos = p
}

func NewSin() *Sin {
	o := &Sin{}
	o.inputs("pitch", &o.pitch, "syn", &o.syn)
	return o
}

type Sin struct {
	sink
	pitch Processor // 0.1/oct, 0 == 440Hz
	syn   trigger

	pos float64
}

func (o *Sin) Process(s []Sample) {
	o.pitch.Process(s)
	t := o.syn.Process()
	p := o.pos
	hz, lastS := sampleToHz(s[0]), s[0]
	for i := range s {
		if o.syn.isTrigger(t[i]) {
			p = 0
		}
		if s[i] != lastS {
			hz = sampleToHz(s[i])
		}
		s[i] = Sample(fast.Sin(p * 2 * math.Pi))
		p += hz / waveHz
		if p > 100 {
			p -= 100
		}
	}
	o.pos = p
}

func NewMul() *Mul {
	a := &Mul{}
	a.inputs("a", &a.a, "b", &a.b)
	return a
}

type Mul struct {
	sink
	a Processor
	b source
}

func (a *Mul) Process(s []Sample) {
	a.a.Process(s)
	m := a.b.Process()
	for i := range s {
		s[i] *= m[i]
	}
}

func NewSum() *Sum {
	s := &Sum{}
	s.inputs("a", &s.a, "b", &s.b)
	return s
}

type Sum struct {
	sink
	a Processor
	b source
}

func (s *Sum) Process(a []Sample) {
	s.a.Process(a)
	b := s.b.Process()
	for i := range a {
		a[i] += b[i]
	}
}

func NewEnv() *Env {
	e := &Env{}
	e.inputs("gate", &e.gate, "trig", &e.trig, "att", &e.att, "dec", &e.dec)
	return e
}

type Env struct {
	sink
	gate     Processor
	trig     trigger
	att, dec source

	v      Sample
	up     bool
	wasLow bool
}

func (e *Env) Process(s []Sample) {
	e.gate.Process(s)
	att, dec, t := e.att.Process(), e.dec.Process(), e.trig.Process()
	v := e.v
	for i := range s {
		trigger := e.trig.isTrigger(t[i])
		if !e.up {
			if trigger || v < s[i] {
				e.up = true
			} else if v > s[i] {
				if d := dec[i]; d > 0 {
					v -= 1 / (d * waveHz * 10)
				}
			}
		}
		if e.up {
			if a := att[i]; a > 0 {
				v += 1 / (a * waveHz * 10)
			}
		}
		if v > 1 {
			v = 1
			e.up = false
		} else if v < 0 {
			v = 0
		}
		s[i] = v
	}
	e.v = v
}

func NewClip() *Clip {
	c := &Clip{}
	c.inputs("in", &c.in)
	return c
}

type Clip struct {
	sink
	in Processor
}

func (c *Clip) Process(s []Sample) {
	c.in.Process(s)
	for i, v := range s {
		if v > 1 {
			s[i] = 1
		} else if v < -1 {
			s[i] = -1
		}
	}
}

type Value Sample

func (v Value) Process(s []Sample) {
	for i := range s {
		s[i] = Sample(v)
	}
}

func NewRand() *Rand {
	r := &Rand{}
	r.inputs("min", &r.min, "max", &r.max, "trig", &r.trig)
	return r
}

type Rand struct {
	sink
	min  Processor
	max  source
	trig trigger

	last Sample
}

func (r *Rand) Process(s []Sample) {
	r.min.Process(s)
	max, t := r.max.Process(), r.trig.Process()
	v := r.last
	for i := range s {
		if r.trig.isTrigger(t[i]) {
			v = s[i] + Sample(rand.Float64())*(max[i]-s[i])
		}
		s[i] = v
	}
	r.last = v
}

func NewDelay() *Delay {
	d := &Delay{buf: make([]Sample, waveHz)}
	d.inputs("in", &d.in, "len", &d.len)
	return d
}

type Delay struct {
	sink
	in  Processor
	len source

	p   int
	buf []Sample
}

func (d *Delay) Process(s []Sample) {
	d.in.Process(s)
	l := d.len.Process()
	p := d.p
	for i := range s {
		max := int(l[i] * waveHz)
		if max <= 0 {
			continue
		}
		if max > waveHz {
			max = waveHz
		}
		if p >= max {
			p = 0
		}
		s[i], d.buf[p] = d.buf[p], s[i]
		p++

	}
	d.p = p
}

func NewDup(src Processor) *Dup {
	d := &Dup{src: src}
	return d
}

type Dup struct {
	src  Processor
	outs []*Output
	buf  []Sample
	done bool
}

func (d *Dup) Tick() {
	d.done = false
}

func (d *Dup) SetSource(p Processor) {
	d.src = p
}

func (d *Dup) Output() *Output {
	o := &Output{d: d}
	d.outs = append(d.outs, o)
	if len(d.outs) > 1 && d.buf == nil {
		d.buf = make([]Sample, nSamples)
	}
	return o
}

type Output struct {
	d *Dup
}

func (o *Output) Process(p []Sample) {
	if !o.d.done {
		o.d.done = true
		o.d.src.Process(p)
		if len(o.d.outs) > 1 {
			copy(o.d.buf, p)
		}
	} else {
		copy(p, o.d.buf)
	}
}

func (o *Output) Close() {
	outs := o.d.outs
	for i, o2 := range outs {
		if o == o2 {
			copy(outs[i:], outs[i+1:])
			o.d.outs = outs[:len(outs)-1]
			break
		}
	}
}
<|endoftext|>