<commit_before>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"go/ast"
	"go/build"
	"go/format"
	"go/parser"
	"go/token"
	"io/ioutil"
	"log"
	"os"
	"reflect"
	"sync"

	"golang.org/x/tools/container/intsets"
	"golang.org/x/tools/go/loader"
	"golang.org/x/tools/go/types"
)

// Unnecessary conversions are identified as the offset of their left parenthesis within a source file.

func apply(file string, edits *intsets.Sparse) {
	if edits.IsEmpty() {
		return
	}

	var fset = token.NewFileSet()

	f, err := parser.ParseFile(fset, file, nil, parser.ParseComments)
	if err != nil {
		log.Fatal(err)
	}

	v := editor{edits: edits, file: fset.File(f.Package)}
	ast.Walk(&v, f)

	// TODO(mdempsky): Write to temporary file and rename.
	var buf bytes.Buffer
	err = format.Node(&buf, fset, f)
	if err != nil {
		log.Fatal(err)
	}

	err = ioutil.WriteFile(file, buf.Bytes(), 0)
	if err != nil {
		log.Fatal(err)
	}
}

type editor struct {
	edits *intsets.Sparse
	file  *token.File
}

func (e *editor) Visit(n ast.Node) ast.Visitor {
	if n == nil {
		return nil
	}
	v := reflect.ValueOf(n).Elem()
	for i, n := 0, v.NumField(); i < n; i++ {
		switch f := v.Field(i).Addr().Interface().(type) {
		case *ast.Expr:
			e.rewrite(f)
		case *[]ast.Expr:
			for i := range *f {
				e.rewrite(&(*f)[i])
			}
		}
	}
	return e
}

func (e *editor) rewrite(f *ast.Expr) {
	if n, ok := (*f).(*ast.CallExpr); ok && e.edits.Has(e.file.Offset(n.Lparen)) {
		*f = n.Args[0]
	}
}

var (
	flagAll   = flag.Bool("all", false, "type check all GOOS and GOARCH combinations")
	flagApply = flag.Bool("apply", false, "apply edits")
)

func main() {
	flag.Parse()

	var m map[string]*intsets.Sparse
	if *flagAll {
		m = mergeEdits()
	} else {
		m = computeEdits(build.Default.GOOS, build.Default.GOARCH)
	}

	if *flagApply {
		var wg sync.WaitGroup
		for f, e := range m {
			wg.Add(1)
			f, e := f, e
			go func() {
				defer wg.Done()
				apply(f, e)
			}()
		}
		wg.Wait()
	} else {
		err := json.NewEncoder(os.Stdout).Encode(m)
		if err != nil {
			log.Fatal(err)
		}
		return
	}
}

var plats = [...]struct {
	goos, goarch string
}{
	{"linux", "386"},
	{"linux", "amd64"},
	{"linux", "arm"},
	{"linux", "arm64"},
	{"linux", "ppc64"},
	{"linux", "ppc64le"},
	{"nacl", "386"},
	{"nacl", "amd64p32"},
	{"nacl", "arm"},
	{"darwin", "386"},
	{"darwin", "amd64"},
	{"dragonfly", "amd64"},
	{"freebsd", "386"},
	{"freebsd", "amd64"},
	{"freebsd", "arm"},
	{"netbsd", "386"},
	{"netbsd", "amd64"},
	{"netbsd", "arm"},
	{"openbsd", "386"},
	{"openbsd", "amd64"},
	{"openbsd", "arm"},
	{"plan9", "386"},
	{"plan9", "amd64"},
	{"solaris", "amd64"},
	{"windows", "386"},
	{"windows", "amd64"},
}

func mergeEdits() map[string]*intsets.Sparse {
	ch := make(chan map[string]*intsets.Sparse)
	var wg sync.WaitGroup
	for _, plat := range plats {
		wg.Add(1)
		go func(goos, goarch string) {
			defer wg.Done()
			ch <- computeEdits(goos, goarch)
		}(plat.goos, plat.goarch)
	}
	go func() {
		wg.Wait()
		close(ch)
	}()

	m := make(map[string]*intsets.Sparse)
	for m1 := range ch {
		for f, e := range m1 {
			if e0, ok := m[f]; ok {
				e0.IntersectionWith(e)
			} else {
				m[f] = e
			}
		}
	}
	return m
}

func noImport(map[string]*types.Package, string) (*types.Package, error) {
	panic("go/loader said this wouldn't be called")
}

func computeEdits(os, arch string) map[string]*intsets.Sparse {
	ctxt := build.Default
	ctxt.GOOS = os
	ctxt.GOARCH = arch
	ctxt.CgoEnabled = false

	var conf loader.Config
	conf.Build = &ctxt
	conf.TypeChecker.Import = noImport
	for _, arg := range flag.Args() {
		conf.Import(arg)
	}
	prog, err := conf.Load()
	if err != nil {
		log.Fatal(err)
	}

	type res struct {
		file  string
		edits *intsets.Sparse
	}
	ch := make(chan res)
	var wg sync.WaitGroup
	for _, pkg := range prog.InitialPackages() {
		for _, file := range pkg.Files {
			wg.Add(1)
			go func(pkg *loader.PackageInfo, file *ast.File) {
				defer wg.Done()
				v := visitor{pkg: pkg, file: conf.Fset.File(file.Package)}
				ast.Walk(&v, file)
				ch <- res{v.file.Name(), &v.edits}
			}(pkg, file)
		}
	}
	go func() {
		wg.Wait()
		close(ch)
	}()

	m := make(map[string]*intsets.Sparse)
	for r := range ch {
		m[r.file] = r.edits
	}
	return m
}

type visitor struct {
	pkg   *loader.PackageInfo
	file  *token.File
	edits intsets.Sparse
}

func (v *visitor) Visit(node ast.Node) ast.Visitor {
	if call, ok := node.(*ast.CallExpr); ok {
		v.unconvert(call)
	}
	return v
}

func (v *visitor) unconvert(call *ast.CallExpr) {
	// TODO(mdempsky): Handle useless multi-conversions.

	// Conversions have exactly one argument.
	if len(call.Args) != 1 || call.Ellipsis != token.NoPos {
		return
	}
	ft, ok := v.pkg.Types[call.Fun]
	if !ok {
		fmt.Println("Missing type for function")
		return
	}
	if !ft.IsType() {
		// Not a conversion.
		return
	}
	at, ok := v.pkg.Types[call.Args[0]]
	if !ok {
		fmt.Println("Missing type for argument")
	}
	if !types.Identical(ft.Type, at.Type) {
		// A real conversion.
		return
	}
	if at.Value != nil || hasUntypedValue(call.Args[0]) {
		// As a workaround for golang.org/issue/13061,
		// skip conversions that contain an untyped value.
		return
	}

	v.edits.Insert(v.file.Offset(call.Lparen))
}

func hasUntypedValue(n ast.Expr) bool {
	var v uvVisitor
	ast.Walk(&v, n)
	return v.found
}

type uvVisitor struct {
	found bool
}

func (v *uvVisitor) Visit(node ast.Node) ast.Visitor {
	// Short circuit.
	if v.found {
		return nil
	}

	switch node := node.(type) {
	case *ast.BinaryExpr:
		switch node.Op {
		case token.SHL, token.SHR, token.EQL, token.NEQ, token.LSS, token.GTR, token.LEQ, token.GEQ:
			// Shifts yield an untyped value if their LHS is untyped.
			// Comparisons yield an untyped boolean value.
			v.found = true
		}
	case *ast.Ident:
		if node.Name == "nil" {
			// Probably the universal untyped zero value.
			v.found = true
		}
	}

	if v.found {
		return nil
	}
	return v
}
<commit_msg>unconvert: sanity checking<commit_after>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"flag"
	"fmt"
	"go/ast"
	"go/build"
	"go/format"
	"go/parser"
	"go/token"
	"io/ioutil"
	"log"
	"os"
	"reflect"
	"sync"

	"golang.org/x/tools/container/intsets"
	"golang.org/x/tools/go/loader"
	"golang.org/x/tools/go/types"
)

// Unnecessary conversions are identified as the offset of their left parenthesis within a source file.

func apply(file string, edits *intsets.Sparse) {
	if edits.IsEmpty() {
		return
	}

	var fset = token.NewFileSet()

	f, err := parser.ParseFile(fset, file, nil, parser.ParseComments)
	if err != nil {
		log.Fatal(err)
	}

	// Note: We modify edits during the walk.
	v := editor{edits: edits, file: fset.File(f.Package)}
	ast.Walk(&v, f)
	if !edits.IsEmpty() {
		log.Printf("%s: missing edits %s", file, edits)
	}

	// TODO(mdempsky): Write to temporary file and rename.
	var buf bytes.Buffer
	err = format.Node(&buf, fset, f)
	if err != nil {
		log.Fatal(err)
	}

	err = ioutil.WriteFile(file, buf.Bytes(), 0)
	if err != nil {
		log.Fatal(err)
	}
}

type editor struct {
	edits *intsets.Sparse
	file  *token.File
}

func (e *editor) Visit(n ast.Node) ast.Visitor {
	if n == nil {
		return nil
	}
	v := reflect.ValueOf(n).Elem()
	for i, n := 0, v.NumField(); i < n; i++ {
		switch f := v.Field(i).Addr().Interface().(type) {
		case *ast.Expr:
			e.rewrite(f)
		case *[]ast.Expr:
			for i := range *f {
				e.rewrite(&(*f)[i])
			}
		}
	}
	return e
}

func (e *editor) rewrite(f *ast.Expr) {
	n, ok := (*f).(*ast.CallExpr)
	if !ok {
		return
	}
	off := e.file.Offset(n.Lparen)
	if !e.edits.Has(off) {
		return
	}
	*f = n.Args[0]
	e.edits.Remove(off)
}

var (
	flagAll   = flag.Bool("all", false, "type check all GOOS and GOARCH combinations")
	flagApply = flag.Bool("apply", false, "apply edits")
)

func main() {
	flag.Parse()

	var m map[string]*intsets.Sparse
	if *flagAll {
		m = mergeEdits()
	} else {
		m = computeEdits(build.Default.GOOS, build.Default.GOARCH)
	}

	if *flagApply {
		var wg sync.WaitGroup
		for f, e := range m {
			wg.Add(1)
			f, e := f, e
			go func() {
				defer wg.Done()
				apply(f, e)
			}()
		}
		wg.Wait()
	} else {
		for f, e := range m {
			if !e.IsEmpty() {
				fmt.Printf("%s: %s\n", f, e)
			}
		}
	}
}

var plats = [...]struct {
	goos, goarch string
}{
	{"linux", "386"},
	{"linux", "amd64"},
	{"linux", "arm"},
	{"linux", "arm64"},
	{"linux", "ppc64"},
	{"linux", "ppc64le"},
	{"nacl", "386"},
	{"nacl", "amd64p32"},
	{"nacl", "arm"},
	{"darwin", "386"},
	{"darwin", "amd64"},
	{"dragonfly", "amd64"},
	{"freebsd", "386"},
	{"freebsd", "amd64"},
	{"freebsd", "arm"},
	{"netbsd", "386"},
	{"netbsd", "amd64"},
	{"netbsd", "arm"},
	{"openbsd", "386"},
	{"openbsd", "amd64"},
	{"openbsd", "arm"},
	{"plan9", "386"},
	{"plan9", "amd64"},
	{"solaris", "amd64"},
	{"windows", "386"},
	{"windows", "amd64"},
}

func mergeEdits() map[string]*intsets.Sparse {
	ch := make(chan map[string]*intsets.Sparse)
	var wg sync.WaitGroup
	for _, plat := range plats {
		wg.Add(1)
		go func(goos, goarch string) {
			defer wg.Done()
			ch <- computeEdits(goos, goarch)
		}(plat.goos, plat.goarch)
	}
	go func() {
		wg.Wait()
		close(ch)
	}()

	m := make(map[string]*intsets.Sparse)
	for m1 := range ch {
		for f, e := range m1 {
			if e0, ok := m[f]; ok {
				e0.IntersectionWith(e)
			} else {
				m[f] = e
			}
		}
	}
	return m
}

func noImport(map[string]*types.Package, string) (*types.Package, error) {
	panic("go/loader said this wouldn't be called")
}

func computeEdits(os, arch string) map[string]*intsets.Sparse {
	ctxt := build.Default
	ctxt.GOOS = os
	ctxt.GOARCH = arch
	ctxt.CgoEnabled = false

	var conf loader.Config
	conf.Build = &ctxt
	conf.TypeChecker.Import = noImport
	for _, arg := range flag.Args() {
		conf.Import(arg)
	}
	prog, err := conf.Load()
	if err != nil {
		log.Fatal(err)
	}

	type res struct {
		file  string
		edits *intsets.Sparse
	}
	ch := make(chan res)
	var wg sync.WaitGroup
	for _, pkg := range prog.InitialPackages() {
		for _, file := range pkg.Files {
			wg.Add(1)
			go func(pkg *loader.PackageInfo, file *ast.File) {
				defer wg.Done()
				v := visitor{pkg: pkg, file: conf.Fset.File(file.Package)}
				ast.Walk(&v, file)
				ch <- res{v.file.Name(), &v.edits}
			}(pkg, file)
		}
	}
	go func() {
		wg.Wait()
		close(ch)
	}()

	m := make(map[string]*intsets.Sparse)
	for r := range ch {
		m[r.file] = r.edits
	}
	return m
}

type visitor struct {
	pkg   *loader.PackageInfo
	file  *token.File
	edits intsets.Sparse
}

func (v *visitor) Visit(node ast.Node) ast.Visitor {
	if call, ok := node.(*ast.CallExpr); ok {
		v.unconvert(call)
	}
	return v
}

func (v *visitor) unconvert(call *ast.CallExpr) {
	// TODO(mdempsky): Handle useless multi-conversions.

	// Conversions have exactly one argument.
	if len(call.Args) != 1 || call.Ellipsis != token.NoPos {
		return
	}
	ft, ok := v.pkg.Types[call.Fun]
	if !ok {
		fmt.Println("Missing type for function")
		return
	}
	if !ft.IsType() {
		// Not a conversion.
		return
	}
	at, ok := v.pkg.Types[call.Args[0]]
	if !ok {
		fmt.Println("Missing type for argument")
	}
	if !types.Identical(ft.Type, at.Type) {
		// A real conversion.
		return
	}
	if at.Value != nil || hasUntypedValue(call.Args[0]) {
		// As a workaround for golang.org/issue/13061,
		// skip conversions that contain an untyped value.
		return
	}

	v.edits.Insert(v.file.Offset(call.Lparen))
}

func hasUntypedValue(n ast.Expr) bool {
	var v uvVisitor
	ast.Walk(&v, n)
	return v.found
}

type uvVisitor struct {
	found bool
}

func (v *uvVisitor) Visit(node ast.Node) ast.Visitor {
	// Short circuit.
	if v.found {
		return nil
	}

	switch node := node.(type) {
	case *ast.BinaryExpr:
		switch node.Op {
		case token.SHL, token.SHR, token.EQL, token.NEQ, token.LSS, token.GTR, token.LEQ, token.GEQ:
			// Shifts yield an untyped value if their LHS is untyped.
			// Comparisons yield an untyped boolean value.
			v.found = true
		}
	case *ast.Ident:
		if node.Name == "nil" {
			// Probably the universal untyped zero value.
			v.found = true
		}
	}

	if v.found {
		return nil
	}
	return v
}
<|endoftext|>


<commit_before>/*
Tensile web stress test tool

Mike Hughes 2014
intermernet AT gmail DOT com

LICENSE BSD 3 Clause
*/
package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"runtime"
	"sync"
	"time"
)

const version = "0.1"

var (
	reqs   int
	max    int
	numCPU int
	maxCPU int

	urlStr string

	flagErr     string
	reqsError   string = "ERROR: -reqs must be greater than 0\n"
	maxError    string = "ERROR: -concurrent must be greater than 0\n"
	urlError    string = "ERROR: URL cannot be blank\n"
	schemeError string = "ERROR: unsupported protocol scheme %s\n"

	cpuWarn       string = "NOTICE: -cpu %d is greater than the number of CPUs on this system\n\tChanging -cpu to %d\n\n"
	maxGTreqsWarn string = "NOTICE: -concurrent is greater than -reqs\n\tChanging -concurrent to -reqs\n\n"

	wg sync.WaitGroup
)

func init() {
	flag.StringVar(&urlStr, "url", "http://localhost/", "Target URL")
	flag.IntVar(&reqs, "reqs", 50, "Total requests")
	flag.IntVar(&max, "concurrent", 5, "Maximum concurrent requests")
	maxCPU = runtime.NumCPU()
	flag.IntVar(&numCPU, "cpu", maxCPU, "Number of CPUs")
}

type Response struct {
	*http.Response
	err error
}

// Dispatcher
func dispatcher(reqChan chan *http.Request) {
	defer close(reqChan)
	for i := 0; i < reqs; i++ {
		req, err := http.NewRequest("GET", urlStr, nil)
		if err != nil {
			log.Println(err)
		}
		reqChan <- req
	}
}

// Worker Pool
func workerPool(reqChan chan *http.Request, respChan chan Response) {
	defer close(respChan)
	t := &http.Transport{}
	defer t.CloseIdleConnections()
	for i := 0; i < max; i++ {
		wg.Add(1)
		go worker(t, reqChan, respChan)
	}
	wg.Wait()
}

// Worker
func worker(t *http.Transport, reqChan chan *http.Request, respChan chan Response) {
	defer wg.Done()
	for req := range reqChan {
		resp, err := t.RoundTrip(req)
		r := Response{resp, err}
		respChan <- r
	}
}

// Consumer
func consumer(respChan chan Response) (int64, int64) {
	var (
		conns int64
		size  int64
	)
	for r := range respChan {
		if r.err != nil {
			log.Println(r.err)
		} else {
			size += r.ContentLength
			if err := r.Body.Close(); err != nil {
				log.Println(r.err)
			}
		}
		conns++
	}
	return conns, size
}

func main() {
	// Flag checks
	flag.Parse()
	fmt.Printf("\n\tTensile web stress test tool v%s\n\n", version)
	if reqs <= 0 {
		flagErr += reqsError
	}
	if max <= 0 {
		flagErr += maxError
	}
	if urlStr == "" {
		flagErr += urlError
	}
	u, err := url.Parse(urlStr)
	if err != nil {
		flagErr += err.Error()
	}
	if u.Scheme != "http" && u.Scheme != "https" {
		flagErr += fmt.Sprintf(schemeError, u.Scheme)
	}
	if flagErr != "" {
		log.Fatal(fmt.Errorf("\n%s", flagErr))
	}
	if numCPU > maxCPU {
		fmt.Printf(cpuWarn, numCPU, maxCPU)
		numCPU = maxCPU
	}
	if max > reqs {
		fmt.Println(maxGTreqsWarn)
		max = reqs
	}
	// Start
	runtime.GOMAXPROCS(runtime.NumCPU())
	reqChan := make(chan *http.Request)
	respChan := make(chan Response)
	fmt.Printf("Sending %d requests to %s with %d concurrent workers.\n\n", reqs, urlStr, max)
	start := time.Now()
	go dispatcher(reqChan)
	go workerPool(reqChan, respChan)
	fmt.Println("Waiting for replies...\n")
	conns, size := consumer(respChan)
	// Calculate stats
	took := time.Since(start)
	ns := took.Nanoseconds()
	var av int64
	if conns != 0 {
		av = ns / conns
	}
	average, err := time.ParseDuration(fmt.Sprintf("%d", av) + "ns")
	if err != nil {
		log.Println(err)
	}
	fmt.Printf("Connections:\t%d\nConcurrent:\t%d\nTotal size:\t%d bytes\nTotal time:\t%s\nAverage time:\t%s\n", conns, max, size, took, average)
}
<commit_msg>Fix use of numCPU variable<commit_after>/*
Tensile web stress test tool

Mike Hughes 2014
intermernet AT gmail DOT com

LICENSE BSD 3 Clause
*/
package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"runtime"
	"sync"
	"time"
)

const version = "0.1"

var (
	reqs   int
	max    int
	numCPU int
	maxCPU int

	urlStr string

	flagErr     string
	reqsError   string = "ERROR: -reqs must be greater than 0\n"
	maxError    string = "ERROR: -concurrent must be greater than 0\n"
	urlError    string = "ERROR: URL cannot be blank\n"
	schemeError string = "ERROR: unsupported protocol scheme %s\n"

	cpuWarn       string = "NOTICE: -cpu %d is greater than the number of CPUs on this system\n\tChanging -cpu to %d\n\n"
	maxGTreqsWarn string = "NOTICE: -concurrent is greater than -reqs\n\tChanging -concurrent to -reqs\n\n"

	wg sync.WaitGroup
)

func init() {
	flag.StringVar(&urlStr, "url", "http://localhost/", "Target URL")
	flag.IntVar(&reqs, "reqs", 50, "Total requests")
	flag.IntVar(&max, "concurrent", 5, "Maximum concurrent requests")
	maxCPU = runtime.NumCPU()
	flag.IntVar(&numCPU, "cpu", maxCPU, "Number of CPUs")
}

type Response struct {
	*http.Response
	err error
}

// Dispatcher
func dispatcher(reqChan chan *http.Request) {
	defer close(reqChan)
	for i := 0; i < reqs; i++ {
		req, err := http.NewRequest("GET", urlStr, nil)
		if err != nil {
			log.Println(err)
		}
		reqChan <- req
	}
}

// Worker Pool
func workerPool(reqChan chan *http.Request, respChan chan Response) {
	defer close(respChan)
	t := &http.Transport{}
	defer t.CloseIdleConnections()
	for i := 0; i < max; i++ {
		wg.Add(1)
		go worker(t, reqChan, respChan)
	}
	wg.Wait()
}

// Worker
func worker(t *http.Transport, reqChan chan *http.Request, respChan chan Response) {
	defer wg.Done()
	for req := range reqChan {
		resp, err := t.RoundTrip(req)
		r := Response{resp, err}
		respChan <- r
	}
}

// Consumer
func consumer(respChan chan Response) (int64, int64) {
	var (
		conns int64
		size  int64
	)
	for r := range respChan {
		if r.err != nil {
			log.Println(r.err)
		} else {
			size += r.ContentLength
			if err := r.Body.Close(); err != nil {
				log.Println(r.err)
			}
		}
		conns++
	}
	return conns, size
}

func main() {
	// Flag checks
	flag.Parse()
	fmt.Printf("\n\tTensile web stress test tool v%s\n\n", version)
	if reqs <= 0 {
		flagErr += reqsError
	}
	if max <= 0 {
		flagErr += maxError
	}
	if urlStr == "" {
		flagErr += urlError
	}
	u, err := url.Parse(urlStr)
	if err != nil {
		flagErr += err.Error()
	}
	if u.Scheme != "http" && u.Scheme != "https" {
		flagErr += fmt.Sprintf(schemeError, u.Scheme)
	}
	if flagErr != "" {
		log.Fatal(fmt.Errorf("\n%s", flagErr))
	}
	if numCPU > maxCPU {
		fmt.Printf(cpuWarn, numCPU, maxCPU)
		numCPU = maxCPU
	}
	if max > reqs {
		fmt.Println(maxGTreqsWarn)
		max = reqs
	}
	// Start
	runtime.GOMAXPROCS(numCPU)
	reqChan := make(chan *http.Request)
	respChan := make(chan Response)
	fmt.Printf("Sending %d requests to %s with %d concurrent workers.\n\n", reqs, urlStr, max)
	start := time.Now()
	go dispatcher(reqChan)
	go workerPool(reqChan, respChan)
	fmt.Println("Waiting for replies...\n")
	conns, size := consumer(respChan)
	// Calculate stats
	took := time.Since(start)
	ns := took.Nanoseconds()
	var av int64
	if conns != 0 {
		av = ns / conns
	}
	average, err := time.ParseDuration(fmt.Sprintf("%d", av) + "ns")
	if err != nil {
		log.Println(err)
	}
	fmt.Printf("Connections:\t%d\nConcurrent:\t%d\nTotal size:\t%d bytes\nTotal time:\t%s\nAverage time:\t%s\n", conns, max, size, took, average)
}
<|endoftext|>


<commit_before>package dropsonde_unmarshaller_test

import (
	"github.com/cloudfoundry/dropsonde/dropsonde_unmarshaller"
	"github.com/cloudfoundry/loggregatorlib/loggertesthelper"

	"fmt"
	"github.com/cloudfoundry/dropsonde/events"
	"github.com/cloudfoundry/dropsonde/factories"
	"github.com/gogo/protobuf/proto"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"runtime"
	"sync"
)

var _ = Describe("DropsondeUnmarshallerCollection", func() {
	var (
		inputChan  chan []byte
		outputChan chan *events.Envelope
		collection dropsonde_unmarshaller.DropsondeUnmarshallerCollection
		waitGroup  sync.WaitGroup
	)
	BeforeEach(func() {
		inputChan = make(chan []byte, 10)
		outputChan = make(chan *events.Envelope, 10)
		collection = dropsonde_unmarshaller.NewDropsondeUnmarshallerCollection(loggertesthelper.Logger(), 5)
		waitGroup = sync.WaitGroup{}
	})

	Context("DropsondeUnmarshallerCollection", func() {
		It("creates the right number of unmarshallers", func() {
			Expect(collection.Size()).To(Equal(5))
		})

	})

	Context("Run", func() {
		It("runs its collection of unmarshallers in separate go routines", func() {
			startingCountGoroutines := runtime.NumGoroutine()
			collection.Run(inputChan, outputChan, waitGroup)
			Expect(startingCountGoroutines + 5).To(Equal(runtime.NumGoroutine()))
		})
	})

	Context("metrics", func() {
		It("emits a total log messages concatenated from the different unmarshallers", func() {
			for n := 0; n < 5; n++ {
				envelope := &events.Envelope{
					Origin:     proto.String("fake-origin-3"),
					EventType:  events.Envelope_LogMessage.Enum(),
					LogMessage: factories.NewLogMessage(events.LogMessage_OUT, "test log message "+string(n), "fake-app-id-1", "DEA"),
				}
				message, _ := proto.Marshal(envelope)

				inputChan <- message
			}

			collection.Run(inputChan, outputChan, waitGroup)

			for n := 0; n < 5; n++ {
				<-outputChan
			}

			metrics := collection.Emit().Metrics

			Expect(metrics).NotTo(BeNil())

			metricsNameMap := make(map[string]int)
			for _, m := range metrics {
				metricsNameMap[m.Name]++
			}

			Expect(metricsNameMap["logMessageTotal"]).To(Equal(1))
			for _, metric := range metrics {
				if metric.Name == "logMessageTotal" {
					Expect(metric.Value.(uint64)).To(Equal(uint64(5)))
				}
			}

			for name, count := range metricsNameMap {
				Expect(count).To(Equal(1), fmt.Sprintf("%v has %v metrics, expected only ONE", name, count))
			}
		})

		It("emits log messages metrics per app concatenated from the different unmarshallers", func() {
			collection.Run(inputChan, outputChan, waitGroup)

			for n := 0; n < 25; n++ {
				envelope := &events.Envelope{
					Origin:     proto.String("fake-origin-3"),
					EventType:  events.Envelope_LogMessage.Enum(),
					LogMessage: factories.NewLogMessage(events.LogMessage_OUT, "test log message "+string(n), "fake-app-id-"+string(n%5), "DEA"),
				}
				message, _ := proto.Marshal(envelope)

				inputChan <- message
			}

			for n := 0; n < 25; n++ {
				<-outputChan
			}

			metrics := collection.Emit().Metrics

			Expect(metrics).NotTo(BeNil())

			metricsNameMap := make(map[string]int)
			for _, m := range metrics {
				metricsNameMap[m.Name]++
			}

			Expect(metricsNameMap["logMessageReceived"]).To(Equal(5))
			for _, metric := range metrics {
				if metric.Name == "logMessageReceived" {
					Expect(metric.Value.(uint64)).To(Equal(uint64(5)))
					Expect(len(metric.Tags)).To(Equal(1))
					Expect(metric.Tags["appId"]).To(ContainSubstring("fake-app-id"))
				}
			}
		})

		It("emits event type metrics concatenated from the different unmarshallers", func() {
			collection.Run(inputChan, outputChan, waitGroup)

			for n := 0; n < 7; n++ {
				envelope := &events.Envelope{
					Origin:    proto.String("fake-origin-1"),
					EventType: events.Envelope_Heartbeat.Enum(),
					Heartbeat: factories.NewHeartbeat(1, 2, 3),
				}
				message, _ := proto.Marshal(envelope)

				inputChan <- message
			}

			for n := 0; n < 7; n++ {
				<-outputChan
			}

			metrics := collection.Emit().Metrics

			Expect(metrics).NotTo(BeNil())

			metricsNameMap := make(map[string]int)
			for _, m := range metrics {
				metricsNameMap[m.Name]++
			}

			Expect(metricsNameMap["heartbeatReceived"]).To(Equal(1))
			for _, metric := range metrics {
				if metric.Name == "heartbeatReceived" {
					Expect(metric.Value.(uint64)).To(Equal(uint64(7)))
				}
			}
		})

		It("emits the correct metrics context", func() {
			Expect(collection.Emit().Name).To(Equal("dropsondeUnmarshaller"))
		})
	})
})
<commit_msg>Make wait group a pointer<commit_after>package dropsonde_unmarshaller_test

import (
	"github.com/cloudfoundry/dropsonde/dropsonde_unmarshaller"
	"github.com/cloudfoundry/loggregatorlib/loggertesthelper"

	"fmt"
	"github.com/cloudfoundry/dropsonde/events"
	"github.com/cloudfoundry/dropsonde/factories"
	"github.com/gogo/protobuf/proto"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"runtime"
	"sync"
)

var _ = Describe("DropsondeUnmarshallerCollection", func() {
	var (
		inputChan  chan []byte
		outputChan chan *events.Envelope
		collection dropsonde_unmarshaller.DropsondeUnmarshallerCollection
		waitGroup  *sync.WaitGroup
	)
	BeforeEach(func() {
		inputChan = make(chan []byte, 10)
		outputChan = make(chan *events.Envelope, 10)
		collection = dropsonde_unmarshaller.NewDropsondeUnmarshallerCollection(loggertesthelper.Logger(), 5)
		waitGroup = &sync.WaitGroup{}
	})

	Context("DropsondeUnmarshallerCollection", func() {
		It("creates the right number of unmarshallers", func() {
			Expect(collection.Size()).To(Equal(5))
		})

	})

	Context("Run", func() {
		It("runs its collection of unmarshallers in separate go routines", func() {
			startingCountGoroutines := runtime.NumGoroutine()
			collection.Run(inputChan, outputChan, waitGroup)
			Expect(startingCountGoroutines + 5).To(Equal(runtime.NumGoroutine()))
		})
	})

	Context("metrics", func() {
		It("emits a total log messages concatenated from the different unmarshallers", func() {
			for n := 0; n < 5; n++ {
				envelope := &events.Envelope{
					Origin:     proto.String("fake-origin-3"),
					EventType:  events.Envelope_LogMessage.Enum(),
					LogMessage: factories.NewLogMessage(events.LogMessage_OUT, "test log message "+string(n), "fake-app-id-1", "DEA"),
				}
				message, _ := proto.Marshal(envelope)

				inputChan <- message
			}

			collection.Run(inputChan, outputChan, waitGroup)

			for n := 0; n < 5; n++ {
				<-outputChan
			}

			metrics := collection.Emit().Metrics

			Expect(metrics).NotTo(BeNil())

			metricsNameMap := make(map[string]int)
			for _, m := range metrics {
				metricsNameMap[m.Name]++
			}

			Expect(metricsNameMap["logMessageTotal"]).To(Equal(1))
			for _, metric := range metrics {
				if metric.Name == "logMessageTotal" {
					Expect(metric.Value.(uint64)).To(Equal(uint64(5)))
				}
			}

			for name, count := range metricsNameMap {
				Expect(count).To(Equal(1), fmt.Sprintf("%v has %v metrics, expected only ONE", name, count))
			}
		})

		It("emits log messages metrics per app concatenated from the different unmarshallers", func() {
			collection.Run(inputChan, outputChan, waitGroup)

			for n := 0; n < 25; n++ {
				envelope := &events.Envelope{
					Origin:     proto.String("fake-origin-3"),
					EventType:  events.Envelope_LogMessage.Enum(),
					LogMessage: factories.NewLogMessage(events.LogMessage_OUT, "test log message "+string(n), "fake-app-id-"+string(n%5), "DEA"),
				}
				message, _ := proto.Marshal(envelope)

				inputChan <- message
			}

			for n := 0; n < 25; n++ {
				<-outputChan
			}

			metrics := collection.Emit().Metrics

			Expect(metrics).NotTo(BeNil())

			metricsNameMap := make(map[string]int)
			for _, m := range metrics {
				metricsNameMap[m.Name]++
			}

			Expect(metricsNameMap["logMessageReceived"]).To(Equal(5))
			for _, metric := range metrics {
				if metric.Name == "logMessageReceived" {
					Expect(metric.Value.(uint64)).To(Equal(uint64(5)))
					Expect(len(metric.Tags)).To(Equal(1))
					Expect(metric.Tags["appId"]).To(ContainSubstring("fake-app-id"))
				}
			}
		})

		It("emits event type metrics concatenated from the different unmarshallers", func() {
			collection.Run(inputChan, outputChan, waitGroup)

			for n := 0; n < 7; n++ {
				envelope := &events.Envelope{
					Origin:    proto.String("fake-origin-1"),
					EventType: events.Envelope_Heartbeat.Enum(),
					Heartbeat: factories.NewHeartbeat(1, 2, 3),
				}
				message, _ := proto.Marshal(envelope)

				inputChan <- message
			}

			for n := 0; n < 7; n++ {
				<-outputChan
			}

			metrics := collection.Emit().Metrics

			Expect(metrics).NotTo(BeNil())

			metricsNameMap := make(map[string]int)
			for _, m := range metrics {
				metricsNameMap[m.Name]++
			}

			Expect(metricsNameMap["heartbeatReceived"]).To(Equal(1))
			for _, metric := range metrics {
				if metric.Name == "heartbeatReceived" {
					Expect(metric.Value.(uint64)).To(Equal(uint64(7)))
				}
			}
		})

		It("emits the correct metrics context", func() {
			Expect(collection.Emit().Name).To(Equal("dropsondeUnmarshaller"))
		})
	})
})
<|endoftext|>


<commit_before>package message

import (
	"bytes"
	"errors"
	"fmt"
	"time"

	"github.com/ishail/m-mail/common"
)

func (msg *Message) ApplySettings(settings []MessageSetting) {
	for _, setting := range settings {
		setting(msg)
	}
}

// Reset resets the message so it can be reused. The message keeps its previous
// settings so it is in the same state that after a call to NewMessage.
func (msg *Message) Reset() {
	for key := range msg.Header {
		delete(msg.Header, key)
	}
	msg.Parts = nil
	msg.Attachments = nil
	msg.Embedded = nil
}

func (msg *Message) SetHeader(field string, value ...string) {
	msg.encodeHeader(value)
	msg.Header[field] = value
}

func (msg *Message) encodeHeader(values []string) {
	for index, val := range values {
		values[index] = msg.encodeString(val)
	}
}

func (msg *Message) encodeString(value string) string {
	return msg.HEncoder.Encode(msg.Charset, value)
}

// SetHeaders sets the message headers.
func (msg *Message) SetHeaders(headers common.Header) {
	for key, val := range headers {
		msg.SetHeader(key, val...)
	}
}

// SetAddressHeader sets an address to the given header field.
func (msg *Message) SetAddressHeader(field, address, name string) {
	msg.Header[field] = []string{msg.FormatAddress(address, name)}
}

// FormatAddress formats an address and a name as a valid RFC 5322 address.
func (msg *Message) FormatAddress(address, name string) string {
	if name == "" {
		return address
	}

	enc := msg.encodeString(name)
	if enc == name {
		msg.Buff.WriteByte('"')
		for _, character := range name {
			if character == '\\' || character == '"' {
				msg.Buff.WriteByte('\\')
			}
			msg.Buff.WriteByte(byte(character))
		}
		msg.Buff.WriteByte('"')
	} else if common.HasSpecials(name) {
		msg.Buff.WriteString(common.BEncoding.Encode(msg.Charset, name))
	} else {
		msg.Buff.WriteString(enc)
	}

	msg.Buff.WriteString(" <")
	msg.Buff.WriteString(address)
	msg.Buff.WriteByte('>')

	addr := msg.Buff.String()
	msg.Buff.Reset()
	return addr
}

// SetDateHeader sets a date to the given header field.
func (msg *Message) SetDateHeader(field string, date time.Time) {
	msg.Header[field] = []string{common.FormatDate(date)}
}

// GetHeader gets a header field.
func (msg *Message) GetHeader(field string) []string {
	return msg.Header[field]
}

//Get From address from Message model
func (msg *Message) GetFrom() (string, error) {
	if from, ok := msg.Header["From"]; ok {
		if len(from) > 0 {
			return common.ParseAddress(from[0])
		}
	}
	return "", errors.New("m-mail: invalid message, 'From' field is missing!")
}

//Get list of recipients(To, Cc, Bcc) from Message object
func (msg *Message) GetRecipients() ([]string, error) {
	recipientLength := 0
	addrHeaderList := []string{"To", "Cc", "Bcc"}

	for _, field := range addrHeaderList {
		if addresses, ok := msg.Header[field]; ok {
			recipientLength += len(addresses)
		}
	}
	recipients := make([]string, recipientLength)
	index := 0

	for _, field := range addrHeaderList {
		if addresses, ok := msg.Header[field]; ok {
			for _, addr := range addresses {
				if addr, err := common.ParseAddress(addr); err != nil {
					return nil, fmt.Errorf(
						"m-mail: Unable to parse address. Address: %s, Error: %v", addr, err)
				} else {
					recipients[index] = addr
					index++
				}
			}
		}
	}

	return recipients, nil
}

//Convert Message object into bytes
func (msg *Message) GetEmailBytes(to string) []byte {
	var msgBytes bytes.Buffer

	msgBytes.WriteString("To: " + to + "\r\n")
	msgBytes.WriteString("Date: " + time.Now().String() + "\r\n")
	msgBytes.WriteString("Subject: " + msg.Subject + "\r\n")
	msgBytes.WriteString("Content-Type: multipart/alternative;\r\n")
	msgBytes.WriteString(`    boundary="boundary-type-1234567892-alt"` + "\r\n")
	msgBytes.WriteString("Mime-Version: 1.0\r\n\r\n")
	msgBytes.WriteString("--boundary-type-1234567892-alt\r\n")
	msgBytes.WriteString("Content-Type: " + msg.Type + `; charset=UTF-8` + "\r\n")
	msgBytes.WriteString("Content-Transfer-Encoding: quoted-printable\r\n\r\n")
	msgBytes.WriteString(msg.Body + "\r\n")

	return msgBytes.Bytes()
}

//Returns headers of message as RFC format
func (msg *Message) getHeadersBytes() []byte {
	var headers bytes.Buffer
	for key, value := range msg.Header {
		headers.Write(getHeaderBytes(key, value...))
	}

	return headers.Bytes()
}
<commit_msg>charset from msg object<commit_after>package message

import (
	"bytes"
	"errors"
	"fmt"
	"time"

	"github.com/ishail/m-mail/common"
)

func (msg *Message) ApplySettings(settings []MessageSetting) {
	for _, setting := range settings {
		setting(msg)
	}
}

// Reset resets the message so it can be reused. The message keeps its previous
// settings so it is in the same state that after a call to NewMessage.
func (msg *Message) Reset() {
	for key := range msg.Header {
		delete(msg.Header, key)
	}
	msg.Parts = nil
	msg.Attachments = nil
	msg.Embedded = nil
}

func (msg *Message) SetHeader(field string, value ...string) {
	msg.encodeHeader(value)
	msg.Header[field] = value
}

func (msg *Message) encodeHeader(values []string) {
	for index, val := range values {
		values[index] = msg.encodeString(val)
	}
}

func (msg *Message) encodeString(value string) string {
	return msg.HEncoder.Encode(msg.Charset, value)
}

// SetHeaders sets the message headers.
func (msg *Message) SetHeaders(headers common.Header) {
	for key, val := range headers {
		msg.SetHeader(key, val...)
	}
}

// SetAddressHeader sets an address to the given header field.
func (msg *Message) SetAddressHeader(field, address, name string) {
	msg.Header[field] = []string{msg.FormatAddress(address, name)}
}

// FormatAddress formats an address and a name as a valid RFC 5322 address.
func (msg *Message) FormatAddress(address, name string) string {
	if name == "" {
		return address
	}

	enc := msg.encodeString(name)
	if enc == name {
		msg.Buff.WriteByte('"')
		for _, character := range name {
			if character == '\\' || character == '"' {
				msg.Buff.WriteByte('\\')
			}
			msg.Buff.WriteByte(byte(character))
		}
		msg.Buff.WriteByte('"')
	} else if common.HasSpecials(name) {
		msg.Buff.WriteString(common.BEncoding.Encode(msg.Charset, name))
	} else {
		msg.Buff.WriteString(enc)
	}

	msg.Buff.WriteString(" <")
	msg.Buff.WriteString(address)
	msg.Buff.WriteByte('>')

	addr := msg.Buff.String()
	msg.Buff.Reset()
	return addr
}

// SetDateHeader sets a date to the given header field.
func (msg *Message) SetDateHeader(field string, date time.Time) {
	msg.Header[field] = []string{common.FormatDate(date)}
}

// GetHeader gets a header field.
func (msg *Message) GetHeader(field string) []string {
	return msg.Header[field]
}

//Get From address from Message model
func (msg *Message) GetFrom() (string, error) {
	if from, ok := msg.Header["From"]; ok {
		if len(from) > 0 {
			return common.ParseAddress(from[0])
		}
	}
	return "", errors.New("m-mail: invalid message, 'From' field is missing!")
}

//Get list of recipients(To, Cc, Bcc) from Message object
func (msg *Message) GetRecipients() ([]string, error) {
	recipientLength := 0
	addrHeaderList := []string{"To", "Cc", "Bcc"}

	for _, field := range addrHeaderList {
		if addresses, ok := msg.Header[field]; ok {
			recipientLength += len(addresses)
		}
	}
	recipients := make([]string, recipientLength)
	index := 0

	for _, field := range addrHeaderList {
		if addresses, ok := msg.Header[field]; ok {
			for _, addr := range addresses {
				if addr, err := common.ParseAddress(addr); err != nil {
					return nil, fmt.Errorf(
						"m-mail: Unable to parse address. Address: %s, Error: %v", addr, err)
				} else {
					recipients[index] = addr
					index++
				}
			}
		}
	}

	return recipients, nil
}

//Convert Message object into bytes
func (msg *Message) GetEmailBytes(to string) []byte {
	var msgBytes bytes.Buffer

	msgBytes.WriteString("To: " + to + "\r\n")
	msgBytes.WriteString("Date: " + time.Now().String() + "\r\n")
	msgBytes.WriteString("Subject: " + msg.Subject + "\r\n")
	msgBytes.WriteString("Content-Type: multipart/alternative;\r\n")
	msgBytes.WriteString(`    boundary="boundary-type-1234567892-alt"` + "\r\n")
	msgBytes.WriteString("Mime-Version: 1.0\r\n\r\n")
	msgBytes.WriteString("--boundary-type-1234567892-alt\r\n")
	msgBytes.WriteString("Content-Type: " + msg.Type + `; charset=` + msg.Charset + "\r\n")
	msgBytes.WriteString("Content-Transfer-Encoding: quoted-printable\r\n\r\n")
	msgBytes.WriteString(msg.Body + "\r\n")

	return msgBytes.Bytes()
}

//Returns headers of message as RFC format
func (msg *Message) getHeadersBytes() []byte {
	var headers bytes.Buffer
	for key, value := range msg.Header {
		headers.Write(getHeaderBytes(key, value...))
	}

	return headers.Bytes()
}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"fmt"
	"crypto/sha256"
	"flag"
	"log"
	"encoding/hex"
	"io"
	"math"
)

const (
	BUFFER_BLOCKS = 1024
	BLOCK_SIZE = 1024
	HASH_SIZE = sha256.Size
	HASHED_BLOCK_SIZE = BLOCK_SIZE + HASH_SIZE

	BUFFER_SIZE = BUFFER_BLOCKS * BLOCK_SIZE
)

var (
	inputFileName = flag.String("i", "", "Specify the input file name.")
	outputFileName = flag.String("o", "", "Specify the output file name.")
	verifyFlag = flag.String("v", "", "Hash0 value in hex")
)

func EncodeAndHash(inputFileName, outputFileName string) ([]byte, error) {
	file, err := os.Open(inputFileName)
	if err != nil {
		return nil, fmt.Errorf("Open input file %s failed with:%v\n", inputFileName, err)
	}

	defer file.Close()

	desFile, err := os.Create(outputFileName)
	if err != nil {
		return nil, fmt.Errorf("Create output file %s failed with: %v\n", outputFileName, err)
	}

	defer  desFile.Close()

	fileInfo, err := file.Stat()
	if err != nil {
		return nil, fmt.Errorf("Get file state failed with: %v\n", err)
	}

	srcLen := fileInfo.Size()
	blockSize := srcLen % BLOCK_SIZE
	bufferSize := BUFFER_SIZE + blockSize
	if srcLen < bufferSize {
		bufferSize = srcLen
	}

	// move pointer to the end
	_, err = file.Seek(-bufferSize, 2)
	if err != nil {
		return nil, fmt.Errorf("Seek file failed with: %v\n", err)
	}
	dataBuff := make([]byte, bufferSize)
	desBuff := make([]byte, (bufferSize-blockSize)/BLOCK_SIZE * (BLOCK_SIZE+HASH_SIZE) + blockSize)
	var hashValue []byte = nil

	if blockSize == 0 {
		blockSize = BLOCK_SIZE
	}

	for {
		readedBytes, err := file.Read(dataBuff)
		if err != nil {
			return nil, fmt.Errorf("Read file failed with: %v\n", err)
		} else if readedBytes != len(dataBuff) {
			return nil, fmt.Errorf("Readed bytes is not enough: %d < %d\n", readedBytes, len(dataBuff))
		}
		//process dataBuff
		hashValue = processBlocks(dataBuff, &desBuff, hashValue, blockSize)
		blockSize = BLOCK_SIZE
		bufferSize = BUFFER_SIZE

		srcLen -= (int64)(readedBytes)

		//write to dist file
		_, err = desFile.Seek((srcLen/BLOCK_SIZE)*(BLOCK_SIZE+HASH_SIZE), 0)
	//	log.Printf("Seek pos:%d, desBuff len:%d\ndes Buff:%v\n", (srcLen/BLOCK_SIZE)*(BLOCK_SIZE+HASH_SIZE),
	//	len(desBuff), desBuff)
		if err != nil {
			return nil, fmt.Errorf("Seek dst file failed with: %v\n", err)
		}

		writedBytes, err := desFile.Write(desBuff)
		if err != nil || writedBytes < len(desBuff) {
			return nil, fmt.Errorf("Write to dst file failed with: %v. Or written bytes are not enough: %d < %d\n", err, writedBytes, len(desBuff))
		}

		//read next buffer
		if srcLen <= 0 {
			break
		} else if srcLen < bufferSize {
			bufferSize = srcLen
		}
		dataBuff = dataBuff[:bufferSize]

		_, err = file.Seek(-(int64)(readedBytes)-bufferSize, 1)
		if err != nil {
			return nil, fmt.Errorf("Seek file failed with: %v\n", err)
		}
	}

	return hashValue, nil
}

func DecodeAndVerify(inputFileName, outputFileName string, hashValue *[HASH_SIZE]byte) error {
	file, err := os.Open(inputFileName)
	if err != nil {
		return fmt.Errorf("Open input file %s failed with:%v\n", inputFileName, err)
	}

	defer file.Close()

	desFile, err := os.Create(outputFileName)
	if err != nil {
		return fmt.Errorf("Create output file %s failed with: %v\n", outputFileName, err)
	}

	defer desFile.Close()

	fileInfo, err := file.Stat()
	if err != nil {
		return fmt.Errorf("Get file state failed with: %v\n", err)
	}

	srcLen := fileInfo.Size()
	srcBlocks := int64(math.Ceil(float64(srcLen) / HASHED_BLOCK_SIZE))

	var bufferBlocks int64 = BUFFER_BLOCKS
	if bufferBlocks > srcBlocks {
		bufferBlocks = srcBlocks
	}

	srcBuff := make([]byte, HASHED_BLOCK_SIZE * bufferBlocks)
	desBuff := make([]byte, BLOCK_SIZE * bufferBlocks)

	var blockIndex int = 0
	for {
		readCount, err := file.Read(srcBuff)
		if err != nil {
			if err == io.EOF && readCount == 0 {
				// read finished
				return nil
			} else {
				return fmt.Errorf("Read from input file failed with: %v\n", err)
			}
		} else if readCount < len(srcBuff) {
			if srcLen >= int64(len(srcBuff)) {
				return fmt.Errorf("Not enough bytes read from file: %d < %d\n", readCount, len(srcBuff))
			} else if int64(readCount) == srcLen {
				srcBuff = srcBuff[:readCount]
			} else {
				return fmt.Errorf("Not enough bytes read from file: %d < %d\n", readCount, srcLen)
			}
		}

		srcLen -= int64(readCount)
		err = verifyBlocks(srcBuff, &desBuff, hashValue, &blockIndex)
		if err != nil {
			return err
		}

		writeCount, err := desFile.Write(desBuff)
		if err != nil || writeCount != len(desBuff) {
			return fmt.Errorf("Write to dst file failed with: %v. Or written bytes are not enough: %d < %d\n", err, writeCount, len(desBuff))
		}
	}
}

func verifyBlocks(srcBuff []byte, desBuff *[]byte, hashValue *[HASH_SIZE]byte, blockIndex *int) error {
	remainedSize := len(srcBuff)
	i := 0
	j := 0
	var verifyBlockSize int
	var lastBlock bool
	for remainedSize > 0 {
		if remainedSize >= HASHED_BLOCK_SIZE {
			verifyBlockSize = HASHED_BLOCK_SIZE
			lastBlock = false
		} else {
			verifyBlockSize = remainedSize
			lastBlock = true
		}

		if sha256.Sum256(srcBuff[i:i+verifyBlockSize]) == *hashValue {
			if lastBlock {
				copy((*desBuff)[j:j+verifyBlockSize], srcBuff[i:i+verifyBlockSize])
				j += verifyBlockSize
			} else {
				copy((*desBuff)[j:j+BLOCK_SIZE], srcBuff[i:i+BLOCK_SIZE])
				copy((*hashValue)[:], srcBuff[i+BLOCK_SIZE:i+HASHED_BLOCK_SIZE])
				j += BLOCK_SIZE
			}

			*blockIndex ++
			remainedSize -= verifyBlockSize
		} else {
			return fmt.Errorf("Verify failed at block index %d\n", *blockIndex)
		}

		i += HASHED_BLOCK_SIZE
	}

	*desBuff = (*desBuff)[:j]

	return nil
}

func processBlocks(srcBuff []byte, desBuff *[]byte, hashValue []byte, blockSize int64) []byte {
//	log.Print(srcBuff, desBuff, hashValue, blockSize)
//	log.Print("len srcBuff=", len(srcBuff), "len desBuff=", len(desBuff))
	srcLen := (int64)(len(srcBuff))
//	log.Print("srcLen=",srcLen)
	desOffset := (srcLen-blockSize)/BLOCK_SIZE * (BLOCK_SIZE+HASH_SIZE)
//	log.Print("desOffset=",desOffset)
	if hashValue == nil {
		*desBuff = (*desBuff)[:desOffset+blockSize]
	} else {
		*desBuff = (*desBuff)[:desOffset+blockSize+HASH_SIZE]
	}
	for i:=srcLen-blockSize; i>=0; i-=BLOCK_SIZE {
//		log.Print("i=",i,",desOffset=",desOffset)
		copy((*desBuff)[desOffset:desOffset+blockSize], srcBuff[i:i+blockSize])
		if hashValue != nil {
			copy((*desBuff)[desOffset+blockSize:desOffset+blockSize+HASH_SIZE],hashValue)
			res := sha256.Sum256((*desBuff)[desOffset:desOffset+blockSize+HASH_SIZE])
			hashValue = res[:]
//			log.Print("desBuff=", desBuff[desOffset:desOffset+blockSize+HASH_SIZE], "len=", len(desBuff[desOffset:desOffset+blockSize+HASH_SIZE]))
//			log.Print("hashValue=",hashValue)
		} else {
			res := sha256.Sum256((*desBuff)[desOffset:desOffset+blockSize])
			hashValue = res[:]
//			log.Print("desBuff=", desBuff[desOffset:desOffset+blockSize], "len=", len(desBuff[desOffset:desOffset+blockSize]))
//			log.Print("hashValue=",hashValue)
		}

		desOffset -= (BLOCK_SIZE+HASH_SIZE)
		blockSize = BLOCK_SIZE
	}

	return hashValue
}

func main() {
	flag.Parse()

	if *inputFileName == "" || *outputFileName == "" {
		fmt.Printf("%s <-i input file name> <-o output file name> [-v hash value]\n", os.Args[0])
		flag.PrintDefaults()
		return
	}

	bVerify := false

	for _,v := range os.Args {
		if v == "-v" {
			bVerify = true
		}
	}

	var hashValue0 [HASH_SIZE]byte

	if bVerify {
		if *verifyFlag != "" {
			hexValue, err := hex.DecodeString(*verifyFlag)
			if err != nil {
				fmt.Printf("Decodex hex string %s failed with: %v\n", *verifyFlag, err)
				return
			} else if len(hexValue) != HASH_SIZE {
				fmt.Printf("The length of hash value is not %d\n", HASH_SIZE)
				return
			}

			copy(hashValue0[:], hexValue)
			bVerify = true
		} else {
			fmt.Print("Hash value can not be empty.\n")
			return
		}
	}

	if bVerify {
		err := DecodeAndVerify(*inputFileName, *outputFileName, &hashValue0)
		if err != nil {
			log.Print(err)
		} else {
			log.Print("Verify and decode succeeded.\n")
		}
	} else {
		hashValue, err := EncodeAndHash(*inputFileName, *outputFileName)
		if err != nil {
			log.Print(err)
		} else {
			log.Print(hex.EncodeToString(hashValue))
		}
	}
}
<commit_msg>go fmt week3<commit_after>package main

import (
	"crypto/sha256"
	"encoding/hex"
	"flag"
	"fmt"
	"io"
	"log"
	"math"
	"os"
)

const (
	BUFFER_BLOCKS     = 1024
	BLOCK_SIZE        = 1024
	HASH_SIZE         = sha256.Size
	HASHED_BLOCK_SIZE = BLOCK_SIZE + HASH_SIZE

	BUFFER_SIZE = BUFFER_BLOCKS * BLOCK_SIZE
)

var (
	inputFileName  = flag.String("i", "", "Specify the input file name.")
	outputFileName = flag.String("o", "", "Specify the output file name.")
	verifyFlag     = flag.String("v", "", "Hash0 value in hex")
)

func EncodeAndHash(inputFileName, outputFileName string) ([]byte, error) {
	file, err := os.Open(inputFileName)
	if err != nil {
		return nil, fmt.Errorf("Open input file %s failed with:%v\n", inputFileName, err)
	}

	defer file.Close()

	desFile, err := os.Create(outputFileName)
	if err != nil {
		return nil, fmt.Errorf("Create output file %s failed with: %v\n", outputFileName, err)
	}

	defer desFile.Close()

	fileInfo, err := file.Stat()
	if err != nil {
		return nil, fmt.Errorf("Get file state failed with: %v\n", err)
	}

	srcLen := fileInfo.Size()
	blockSize := srcLen % BLOCK_SIZE
	bufferSize := BUFFER_SIZE + blockSize
	if srcLen < bufferSize {
		bufferSize = srcLen
	}

	// move pointer to the end
	_, err = file.Seek(-bufferSize, 2)
	if err != nil {
		return nil, fmt.Errorf("Seek file failed with: %v\n", err)
	}
	dataBuff := make([]byte, bufferSize)
	desBuff := make([]byte, (bufferSize-blockSize)/BLOCK_SIZE*(BLOCK_SIZE+HASH_SIZE)+blockSize)
	var hashValue []byte = nil

	if blockSize == 0 {
		blockSize = BLOCK_SIZE
	}

	for {
		readedBytes, err := file.Read(dataBuff)
		if err != nil {
			return nil, fmt.Errorf("Read file failed with: %v\n", err)
		} else if readedBytes != len(dataBuff) {
			return nil, fmt.Errorf("Readed bytes is not enough: %d < %d\n", readedBytes, len(dataBuff))
		}
		//process dataBuff
		hashValue = processBlocks(dataBuff, &desBuff, hashValue, blockSize)
		blockSize = BLOCK_SIZE
		bufferSize = BUFFER_SIZE

		srcLen -= (int64)(readedBytes)

		//write to dist file
		_, err = desFile.Seek((srcLen/BLOCK_SIZE)*(BLOCK_SIZE+HASH_SIZE), 0)
		//	log.Printf("Seek pos:%d, desBuff len:%d\ndes Buff:%v\n", (srcLen/BLOCK_SIZE)*(BLOCK_SIZE+HASH_SIZE),
		//	len(desBuff), desBuff)
		if err != nil {
			return nil, fmt.Errorf("Seek dst file failed with: %v\n", err)
		}

		writedBytes, err := desFile.Write(desBuff)
		if err != nil || writedBytes < len(desBuff) {
			return nil, fmt.Errorf("Write to dst file failed with: %v. Or written bytes are not enough: %d < %d\n", err, writedBytes, len(desBuff))
		}

		//read next buffer
		if srcLen <= 0 {
			break
		} else if srcLen < bufferSize {
			bufferSize = srcLen
		}
		dataBuff = dataBuff[:bufferSize]

		_, err = file.Seek(-(int64)(readedBytes)-bufferSize, 1)
		if err != nil {
			return nil, fmt.Errorf("Seek file failed with: %v\n", err)
		}
	}

	return hashValue, nil
}

func DecodeAndVerify(inputFileName, outputFileName string, hashValue *[HASH_SIZE]byte) error {
	file, err := os.Open(inputFileName)
	if err != nil {
		return fmt.Errorf("Open input file %s failed with:%v\n", inputFileName, err)
	}

	defer file.Close()

	desFile, err := os.Create(outputFileName)
	if err != nil {
		return fmt.Errorf("Create output file %s failed with: %v\n", outputFileName, err)
	}

	defer desFile.Close()

	fileInfo, err := file.Stat()
	if err != nil {
		return fmt.Errorf("Get file state failed with: %v\n", err)
	}

	srcLen := fileInfo.Size()
	srcBlocks := int64(math.Ceil(float64(srcLen) / HASHED_BLOCK_SIZE))

	var bufferBlocks int64 = BUFFER_BLOCKS
	if bufferBlocks > srcBlocks {
		bufferBlocks = srcBlocks
	}

	srcBuff := make([]byte, HASHED_BLOCK_SIZE*bufferBlocks)
	desBuff := make([]byte, BLOCK_SIZE*bufferBlocks)

	var blockIndex int = 0
	for {
		readCount, err := file.Read(srcBuff)
		if err != nil {
			if err == io.EOF && readCount == 0 {
				// read finished
				return nil
			} else {
				return fmt.Errorf("Read from input file failed with: %v\n", err)
			}
		} else if readCount < len(srcBuff) {
			if srcLen >= int64(len(srcBuff)) {
				return fmt.Errorf("Not enough bytes read from file: %d < %d\n", readCount, len(srcBuff))
			} else if int64(readCount) == srcLen {
				srcBuff = srcBuff[:readCount]
			} else {
				return fmt.Errorf("Not enough bytes read from file: %d < %d\n", readCount, srcLen)
			}
		}

		srcLen -= int64(readCount)
		err = verifyBlocks(srcBuff, &desBuff, hashValue, &blockIndex)
		if err != nil {
			return err
		}

		writeCount, err := desFile.Write(desBuff)
		if err != nil || writeCount != len(desBuff) {
			return fmt.Errorf("Write to dst file failed with: %v. Or written bytes are not enough: %d < %d\n", err, writeCount, len(desBuff))
		}
	}
}

func verifyBlocks(srcBuff []byte, desBuff *[]byte, hashValue *[HASH_SIZE]byte, blockIndex *int) error {
	remainedSize := len(srcBuff)
	i := 0
	j := 0
	var verifyBlockSize int
	var lastBlock bool
	for remainedSize > 0 {
		if remainedSize >= HASHED_BLOCK_SIZE {
			verifyBlockSize = HASHED_BLOCK_SIZE
			lastBlock = false
		} else {
			verifyBlockSize = remainedSize
			lastBlock = true
		}

		if sha256.Sum256(srcBuff[i:i+verifyBlockSize]) == *hashValue {
			if lastBlock {
				copy((*desBuff)[j:j+verifyBlockSize], srcBuff[i:i+verifyBlockSize])
				j += verifyBlockSize
			} else {
				copy((*desBuff)[j:j+BLOCK_SIZE], srcBuff[i:i+BLOCK_SIZE])
				copy((*hashValue)[:], srcBuff[i+BLOCK_SIZE:i+HASHED_BLOCK_SIZE])
				j += BLOCK_SIZE
			}

			*blockIndex++
			remainedSize -= verifyBlockSize
		} else {
			return fmt.Errorf("Verify failed at block index %d\n", *blockIndex)
		}

		i += HASHED_BLOCK_SIZE
	}

	*desBuff = (*desBuff)[:j]

	return nil
}

func processBlocks(srcBuff []byte, desBuff *[]byte, hashValue []byte, blockSize int64) []byte {
	//	log.Print(srcBuff, desBuff, hashValue, blockSize)
	//	log.Print("len srcBuff=", len(srcBuff), "len desBuff=", len(desBuff))
	srcLen := (int64)(len(srcBuff))
	//	log.Print("srcLen=",srcLen)
	desOffset := (srcLen - blockSize) / BLOCK_SIZE * (BLOCK_SIZE + HASH_SIZE)
	//	log.Print("desOffset=",desOffset)
	if hashValue == nil {
		*desBuff = (*desBuff)[:desOffset+blockSize]
	} else {
		*desBuff = (*desBuff)[:desOffset+blockSize+HASH_SIZE]
	}
	for i := srcLen - blockSize; i >= 0; i -= BLOCK_SIZE {
		//		log.Print("i=",i,",desOffset=",desOffset)
		copy((*desBuff)[desOffset:desOffset+blockSize], srcBuff[i:i+blockSize])
		if hashValue != nil {
			copy((*desBuff)[desOffset+blockSize:desOffset+blockSize+HASH_SIZE], hashValue)
			res := sha256.Sum256((*desBuff)[desOffset : desOffset+blockSize+HASH_SIZE])
			hashValue = res[:]
			//			log.Print("desBuff=", desBuff[desOffset:desOffset+blockSize+HASH_SIZE], "len=", len(desBuff[desOffset:desOffset+blockSize+HASH_SIZE]))
			//			log.Print("hashValue=",hashValue)
		} else {
			res := sha256.Sum256((*desBuff)[desOffset : desOffset+blockSize])
			hashValue = res[:]
			//			log.Print("desBuff=", desBuff[desOffset:desOffset+blockSize], "len=", len(desBuff[desOffset:desOffset+blockSize]))
			//			log.Print("hashValue=",hashValue)
		}

		desOffset -= (BLOCK_SIZE + HASH_SIZE)
		blockSize = BLOCK_SIZE
	}

	return hashValue
}

func main() {
	flag.Parse()

	if *inputFileName == "" || *outputFileName == "" {
		fmt.Printf("%s <-i input file name> <-o output file name> [-v hash value]\n", os.Args[0])
		flag.PrintDefaults()
		return
	}

	bVerify := false

	for _, v := range os.Args {
		if v == "-v" {
			bVerify = true
		}
	}

	var hashValue0 [HASH_SIZE]byte

	if bVerify {
		if *verifyFlag != "" {
			hexValue, err := hex.DecodeString(*verifyFlag)
			if err != nil {
				fmt.Printf("Decodex hex string %s failed with: %v\n", *verifyFlag, err)
				return
			} else if len(hexValue) != HASH_SIZE {
				fmt.Printf("The length of hash value is not %d\n", HASH_SIZE)
				return
			}

			copy(hashValue0[:], hexValue)
			bVerify = true
		} else {
			fmt.Print("Hash value can not be empty.\n")
			return
		}
	}

	if bVerify {
		err := DecodeAndVerify(*inputFileName, *outputFileName, &hashValue0)
		if err != nil {
			log.Print(err)
		} else {
			log.Print("Verify and decode succeeded.\n")
		}
	} else {
		hashValue, err := EncodeAndHash(*inputFileName, *outputFileName)
		if err != nil {
			log.Print(err)
		} else {
			log.Print(hex.EncodeToString(hashValue))
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"
)

func TestHandlerForDisallowedMethods(t *testing.T) {
	disallowedMethods := []string{"GET", "DELETE", "PUT", "TRACE", "PATCH"}
	randomUrls := []string{"/", "/blah"}

	for _, method := range disallowedMethods {
		for _, url := range randomUrls {
			t.Run(method+url, func(t *testing.T) {
				request, err := http.NewRequest(method, url, nil)
				if err != nil {
					t.Fatalf("failed to create request: %v", err)
				}
				recorder := httptest.NewRecorder()
				handleViolationReport(recorder, request)

				response := recorder.Result()
				defer response.Body.Close()

				if response.StatusCode != http.StatusMethodNotAllowed {
					t.Errorf("expected HTTP status %v; got %v", http.StatusMethodNotAllowed, response.StatusCode)
				}
			})
		}
	}
}

func TestHandlerForAllowingHealthcheck(t *testing.T) {
	request, err := http.NewRequest("GET", "/_healthcheck", nil)
	if err != nil {
		t.Fatalf("failed to create request: %v", err)
	}
	recorder := httptest.NewRecorder()

	handleViolationReport(recorder, request)

	response := recorder.Result()
	defer response.Body.Close()

	if response.StatusCode != http.StatusOK {
		t.Errorf("expected HTTP status %v; got %v", http.StatusOK, response.StatusCode)
	}
}
func TestValidateViolationWithInvalidBlockedURIs(t *testing.T) {
	invalidBlockedURIs := []string{
		"resource://",
		"chromenull://",
		"chrome-extension://",
		"safari-extension://",
		"mxjscall://",
		"webviewprogressproxy://",
		"res://",
		"mx://",
		"safari-resource://",
		"chromeinvoke://",
		"chromeinvokeimmediate://",
		"mbinit://",
		"opera://",
		"localhost",
		"127.0.0.1",
		"none://",
		"about:blank",
		"android-webview",
		"ms-browser-extension",
	}

	for _, blockedURI := range invalidBlockedURIs {
		// Makes the test name more readable for the output.
		testName := strings.Replace(blockedURI, "://", "", -1)

		t.Run(testName, func(t *testing.T) {
			var rawReport = []byte(fmt.Sprintf(`{
				"csp-report": {
					"blocked-uri": "%s"
				}
			}`, blockedURI))

			var report CSPReport
			jsonErr := json.Unmarshal(rawReport, &report)
			if jsonErr != nil {
				fmt.Println("error:", jsonErr)
			}

			validateErr := validateViolation(report)
			if validateErr == nil {
				t.Errorf("expected error to be raised but it didn't")
			}

			if validateErr.Error() != fmt.Sprintf("Blocked URI ('%s') is an invalid resource.", blockedURI) {
				t.Errorf("expected error to include correct message string but it didn't")
			}
		})
	}
}

func TestValidateViolationWithValidBlockedURIs(t *testing.T) {
	var rawReport = []byte(`{
		"csp-report": {
			"blocked-uri": "https://google.com/example.css"
		}
	}`)

	var report CSPReport
	jsonErr := json.Unmarshal(rawReport, &report)
	if jsonErr != nil {
		fmt.Println("error:", jsonErr)
	}

	validateErr := validateViolation(report)
	if validateErr != nil {
		t.Errorf("expected error not be raised")
	}
}
<commit_msg>Adds coverage for ensuring all required keys are outputted<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"
)

func TestHandlerForDisallowedMethods(t *testing.T) {
	disallowedMethods := []string{"GET", "DELETE", "PUT", "TRACE", "PATCH"}
	randomUrls := []string{"/", "/blah"}

	for _, method := range disallowedMethods {
		for _, url := range randomUrls {
			t.Run(method+url, func(t *testing.T) {
				request, err := http.NewRequest(method, url, nil)
				if err != nil {
					t.Fatalf("failed to create request: %v", err)
				}
				recorder := httptest.NewRecorder()
				handleViolationReport(recorder, request)

				response := recorder.Result()
				defer response.Body.Close()

				if response.StatusCode != http.StatusMethodNotAllowed {
					t.Errorf("expected HTTP status %v; got %v", http.StatusMethodNotAllowed, response.StatusCode)
				}
			})
		}
	}
}

func TestHandlerForAllowingHealthcheck(t *testing.T) {
	request, err := http.NewRequest("GET", "/_healthcheck", nil)
	if err != nil {
		t.Fatalf("failed to create request: %v", err)
	}
	recorder := httptest.NewRecorder()

	handleViolationReport(recorder, request)

	response := recorder.Result()
	defer response.Body.Close()

	if response.StatusCode != http.StatusOK {
		t.Errorf("expected HTTP status %v; got %v", http.StatusOK, response.StatusCode)
	}
}
func TestFormattedOutputIncludesEmptyKeysForRequiredValues(t *testing.T) {
	var rawReport = []byte(`{
		"csp-report": {
			"document-uri": "http://example.com/signup.html",
			"referrer": ""
		}
	}`)

	var report CSPReport
	err := json.Unmarshal(rawReport, &report)
	if err != nil {
		fmt.Println("error:", err)
	}

	formattedReportOutput := formatReport(report)

	if !strings.Contains(formattedReportOutput, "referrer=\"\"") {
		t.Errorf("expected to find empty 'referrer' value but did not")
	}
}

func TestValidateViolationWithInvalidBlockedURIs(t *testing.T) {
	invalidBlockedURIs := []string{
		"resource://",
		"chromenull://",
		"chrome-extension://",
		"safari-extension://",
		"mxjscall://",
		"webviewprogressproxy://",
		"res://",
		"mx://",
		"safari-resource://",
		"chromeinvoke://",
		"chromeinvokeimmediate://",
		"mbinit://",
		"opera://",
		"localhost",
		"127.0.0.1",
		"none://",
		"about:blank",
		"android-webview",
		"ms-browser-extension",
	}

	for _, blockedURI := range invalidBlockedURIs {
		// Makes the test name more readable for the output.
		testName := strings.Replace(blockedURI, "://", "", -1)

		t.Run(testName, func(t *testing.T) {
			var rawReport = []byte(fmt.Sprintf(`{
				"csp-report": {
					"blocked-uri": "%s"
				}
			}`, blockedURI))

			var report CSPReport
			jsonErr := json.Unmarshal(rawReport, &report)
			if jsonErr != nil {
				fmt.Println("error:", jsonErr)
			}

			validateErr := validateViolation(report)
			if validateErr == nil {
				t.Errorf("expected error to be raised but it didn't")
			}

			if validateErr.Error() != fmt.Sprintf("Blocked URI ('%s') is an invalid resource.", blockedURI) {
				t.Errorf("expected error to include correct message string but it didn't")
			}
		})
	}
}

func TestValidateViolationWithValidBlockedURIs(t *testing.T) {
	var rawReport = []byte(`{
		"csp-report": {
			"blocked-uri": "https://google.com/example.css"
		}
	}`)

	var report CSPReport
	jsonErr := json.Unmarshal(rawReport, &report)
	if jsonErr != nil {
		fmt.Println("error:", jsonErr)
	}

	validateErr := validateViolation(report)
	if validateErr != nil {
		t.Errorf("expected error not be raised")
	}
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"reflect"
	"testing"

	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
	"github.com/mitchellh/goamz/ec2"
)

func TestAccAWSSecurityGroup_normal(t *testing.T) {
	var group ec2.SecurityGroupInfo

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAWSSecurityGroupDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: testAccAWSSecurityGroupConfig,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSSecurityGroupExists("aws_security_group.web", &group),
					testAccCheckAWSSecurityGroupAttributes(&group),
					resource.TestCheckResourceAttr(
						"aws_security_group.web", "name", "terraform_acceptance_test_example"),
					resource.TestCheckResourceAttr(
						"aws_security_group.web", "description", "Used in the terraform acceptance tests"),
					resource.TestCheckResourceAttr(
						"aws_security_group.web", "ingress.0.protocol", "tcp"),
					resource.TestCheckResourceAttr(
						"aws_security_group.web", "ingress.0.from_port", "80"),
					resource.TestCheckResourceAttr(
						"aws_security_group.web", "ingress.0.to_port", "8000"),
					resource.TestCheckResourceAttr(
						"aws_security_group.web", "ingress.0.cidr_blocks.#", "1"),
					resource.TestCheckResourceAttr(
						"aws_security_group.web", "ingress.0.cidr_blocks.0", "10.0.0.0/0"),
				),
			},
		},
	})
}

func TestAccAWSSecurityGroup_vpc(t *testing.T) {
	var group ec2.SecurityGroupInfo

	testCheck := func(*terraform.State) error {
		if group.VpcId == "" {
			return fmt.Errorf("should have vpc ID")
		}

		return nil
	}

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAWSSecurityGroupDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: testAccAWSSecurityGroupConfigVpc,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSSecurityGroupExists("aws_security_group.web", &group),
					testAccCheckAWSSecurityGroupAttributes(&group),
					resource.TestCheckResourceAttr(
						"aws_security_group.web", "name", "terraform_acceptance_test_example"),
					resource.TestCheckResourceAttr(
						"aws_security_group.web", "description", "Used in the terraform acceptance tests"),
					resource.TestCheckResourceAttr(
						"aws_security_group.web", "ingress.0.protocol", "tcp"),
					resource.TestCheckResourceAttr(
						"aws_security_group.web", "ingress.0.from_port", "80"),
					resource.TestCheckResourceAttr(
						"aws_security_group.web", "ingress.0.to_port", "8000"),
					resource.TestCheckResourceAttr(
						"aws_security_group.web", "ingress.0.cidr_blocks.#", "1"),
					resource.TestCheckResourceAttr(
						"aws_security_group.web", "ingress.0.cidr_blocks.0", "10.0.0.0/0"),
					testCheck,
				),
			},
		},
	})
}

func testAccCheckAWSSecurityGroupDestroy(s *terraform.State) error {
	conn := testAccProvider.ec2conn

	for _, rs := range s.Resources {
		if rs.Type != "aws_security_group" {
			continue
		}

		sgs := []ec2.SecurityGroup{
			ec2.SecurityGroup{
				Id: rs.ID,
			},
		}

		// Retrieve our group
		resp, err := conn.SecurityGroups(sgs, nil)
		if err == nil {
			if len(resp.Groups) > 0 && resp.Groups[0].Id == rs.ID {
				return fmt.Errorf("Security Group (%s) still exists.", rs.ID)
			}

			return nil
		}

		ec2err, ok := err.(*ec2.Error)
		if !ok {
			return err
		}
		// Confirm error code is what we want
		if ec2err.Code != "InvalidGroup.NotFound" {
			return err
		}
	}

	return nil
}

func testAccCheckAWSSecurityGroupExists(n string, group *ec2.SecurityGroupInfo) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.Resources[n]
		if !ok {
			return fmt.Errorf("Not found: %s", n)
		}

		if rs.ID == "" {
			return fmt.Errorf("No Security Group is set")
		}

		conn := testAccProvider.ec2conn
		sgs := []ec2.SecurityGroup{
			ec2.SecurityGroup{
				Id: rs.ID,
			},
		}
		resp, err := conn.SecurityGroups(sgs, nil)
		if err != nil {
			return err
		}

		if len(resp.Groups) > 0 && resp.Groups[0].Id == rs.ID {

			*group = resp.Groups[0]

			return nil
		} else {
			return fmt.Errorf("Security Group not found")
		}

		return nil
	}
}

func testAccCheckAWSSecurityGroupAttributes(group *ec2.SecurityGroupInfo) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		p := ec2.IPPerm{
			FromPort:  80,
			ToPort:    8000,
			Protocol:  "tcp",
			SourceIPs: []string{"10.0.0.0/0"},
		}

		if group.Name != "terraform_acceptance_test_example" {
			return fmt.Errorf("Bad name: %s", group.Name)
		}

		if group.Description != "Used in the terraform acceptance tests" {
			return fmt.Errorf("Bad description: %s", group.Description)
		}

		// Compare our ingress
		if !reflect.DeepEqual(group.IPPerms[0], p) {
			return fmt.Errorf(
				"Got:\n\n%#v\n\nExpected:\n\n%#v\n",
				group.IPPerms[0],
				p)
		}

		return nil
	}
}

const testAccAWSSecurityGroupConfig = `
resource "aws_security_group" "web" {
    name = "terraform_acceptance_test_example"
    description = "Used in the terraform acceptance tests"

    ingress {
        protocol = "tcp"
        from_port = 80
        to_port = 8000
        cidr_blocks = ["10.0.0.0/0"]
    }
}
`

const testAccAWSSecurityGroupConfigVpc = `
resource "aws_vpc" "foo" {
	cidr_block = "10.1.0.0/16"
}

resource "aws_security_group" "web" {
    name = "terraform_acceptance_test_example"
    description = "Used in the terraform acceptance tests"
	vpc_id = "${aws_vpc.foo.id}"

    ingress {
        protocol = "tcp"
        from_port = 80
        to_port = 8000
        cidr_blocks = ["10.0.0.0/0"]
    }
}
`
<commit_msg>provider/aws: fixing security groups test<commit_after>package aws

import (
	"fmt"
	"reflect"
	"testing"

	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
	"github.com/mitchellh/goamz/ec2"
)

func TestAccAWSSecurityGroup_normal(t *testing.T) {
	var group ec2.SecurityGroupInfo

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAWSSecurityGroupDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: testAccAWSSecurityGroupConfig,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSSecurityGroupExists("aws_security_group.web", &group),
					testAccCheckAWSSecurityGroupAttributes(&group),
					resource.TestCheckResourceAttr(
						"aws_security_group.web", "name", "terraform_acceptance_test_example"),
					resource.TestCheckResourceAttr(
						"aws_security_group.web", "description", "Used in the terraform acceptance tests"),
					resource.TestCheckResourceAttr(
						"aws_security_group.web", "ingress.0.protocol", "tcp"),
					resource.TestCheckResourceAttr(
						"aws_security_group.web", "ingress.0.from_port", "80"),
					resource.TestCheckResourceAttr(
						"aws_security_group.web", "ingress.0.to_port", "8000"),
					resource.TestCheckResourceAttr(
						"aws_security_group.web", "ingress.0.cidr_blocks.#", "1"),
					resource.TestCheckResourceAttr(
						"aws_security_group.web", "ingress.0.cidr_blocks.0", "10.0.0.0/8"),
				),
			},
		},
	})
}

func TestAccAWSSecurityGroup_vpc(t *testing.T) {
	var group ec2.SecurityGroupInfo

	testCheck := func(*terraform.State) error {
		if group.VpcId == "" {
			return fmt.Errorf("should have vpc ID")
		}

		return nil
	}

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAWSSecurityGroupDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: testAccAWSSecurityGroupConfigVpc,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSSecurityGroupExists("aws_security_group.web", &group),
					testAccCheckAWSSecurityGroupAttributes(&group),
					resource.TestCheckResourceAttr(
						"aws_security_group.web", "name", "terraform_acceptance_test_example"),
					resource.TestCheckResourceAttr(
						"aws_security_group.web", "description", "Used in the terraform acceptance tests"),
					resource.TestCheckResourceAttr(
						"aws_security_group.web", "ingress.0.protocol", "tcp"),
					resource.TestCheckResourceAttr(
						"aws_security_group.web", "ingress.0.from_port", "80"),
					resource.TestCheckResourceAttr(
						"aws_security_group.web", "ingress.0.to_port", "8000"),
					resource.TestCheckResourceAttr(
						"aws_security_group.web", "ingress.0.cidr_blocks.#", "1"),
					resource.TestCheckResourceAttr(
						"aws_security_group.web", "ingress.0.cidr_blocks.0", "10.0.0.0/8"),
					testCheck,
				),
			},
		},
	})
}

func testAccCheckAWSSecurityGroupDestroy(s *terraform.State) error {
	conn := testAccProvider.ec2conn

	for _, rs := range s.Resources {
		if rs.Type != "aws_security_group" {
			continue
		}

		sgs := []ec2.SecurityGroup{
			ec2.SecurityGroup{
				Id: rs.ID,
			},
		}

		// Retrieve our group
		resp, err := conn.SecurityGroups(sgs, nil)
		if err == nil {
			if len(resp.Groups) > 0 && resp.Groups[0].Id == rs.ID {
				return fmt.Errorf("Security Group (%s) still exists.", rs.ID)
			}

			return nil
		}

		ec2err, ok := err.(*ec2.Error)
		if !ok {
			return err
		}
		// Confirm error code is what we want
		if ec2err.Code != "InvalidGroup.NotFound" {
			return err
		}
	}

	return nil
}

func testAccCheckAWSSecurityGroupExists(n string, group *ec2.SecurityGroupInfo) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.Resources[n]
		if !ok {
			return fmt.Errorf("Not found: %s", n)
		}

		if rs.ID == "" {
			return fmt.Errorf("No Security Group is set")
		}

		conn := testAccProvider.ec2conn
		sgs := []ec2.SecurityGroup{
			ec2.SecurityGroup{
				Id: rs.ID,
			},
		}
		resp, err := conn.SecurityGroups(sgs, nil)
		if err != nil {
			return err
		}

		if len(resp.Groups) > 0 && resp.Groups[0].Id == rs.ID {

			*group = resp.Groups[0]

			return nil
		} else {
			return fmt.Errorf("Security Group not found")
		}

		return nil
	}
}

func testAccCheckAWSSecurityGroupAttributes(group *ec2.SecurityGroupInfo) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		p := ec2.IPPerm{
			FromPort:  80,
			ToPort:    8000,
			Protocol:  "tcp",
			SourceIPs: []string{"10.0.0.0/8"},
		}

		if group.Name != "terraform_acceptance_test_example" {
			return fmt.Errorf("Bad name: %s", group.Name)
		}

		if group.Description != "Used in the terraform acceptance tests" {
			return fmt.Errorf("Bad description: %s", group.Description)
		}

		// Compare our ingress
		if !reflect.DeepEqual(group.IPPerms[0], p) {
			return fmt.Errorf(
				"Got:\n\n%#v\n\nExpected:\n\n%#v\n",
				group.IPPerms[0],
				p)
		}

		return nil
	}
}

const testAccAWSSecurityGroupConfig = `
resource "aws_security_group" "web" {
    name = "terraform_acceptance_test_example"
    description = "Used in the terraform acceptance tests"

    ingress {
        protocol = "tcp"
        from_port = 80
        to_port = 8000
        cidr_blocks = ["10.0.0.0/8"]
    }
}
`

const testAccAWSSecurityGroupConfigVpc = `
resource "aws_vpc" "foo" {
	cidr_block = "10.1.0.0/16"
}

resource "aws_security_group" "web" {
    name = "terraform_acceptance_test_example"
    description = "Used in the terraform acceptance tests"
	vpc_id = "${aws_vpc.foo.id}"

    ingress {
        protocol = "tcp"
        from_port = 80
        to_port = 8000
        cidr_blocks = ["10.0.0.0/8"]
    }
}
`
<|endoftext|>


<commit_before>package topic

import (
	"os"
	"sync"
	"testing"

	"gopkg.in/logex.v1"

	"github.com/chzyer/mmq/mmq"
)

var (
	c *Config
)

func init() {
	c = new(Config)
	c.ChunkBit = 22
	c.Root = "/data/mmq/test/topic"
	os.MkdirAll(c.Root, 0777)
	os.RemoveAll(c.Root)
}

func BenchmarkTopicPut(b *testing.B) {
	topic, err := New("bench-put", c)
	if err != nil {
		b.Fatal(err)
	}
	reply := make(chan []error)
	var wg sync.WaitGroup
	go func() {
		for _ = range reply {
			wg.Done()
		}
	}()
	b.ResetTimer()
	buffer := []*mmq.Message{}
	for i := 0; i < b.N; i++ {
		m, _ := mmq.NewMessage(msg.Bytes(), true)
		buffer = append(buffer, m)
		if len(buffer) >= 100 {
			wg.Add(1)
			topic.Put(buffer, reply)
			buffer = nil
		}
	}
	wg.Wait()
}

func TestTopic(t *testing.T) {
	topic, err := New("topicTest", c)
	if err != nil {
		t.Fatal(err)
	}
	var wg sync.WaitGroup
	var testSource = [][]byte{
		[]byte("hello!"),
		[]byte("who are you"),
		[]byte("oo?"),
	}
	wg.Add(len(testSource))
	go func() {
		incoming := make(chan []*mmq.Message, len(testSource))
		errChan := make(chan error)
		topic.Get(0, len(testSource), incoming, errChan)
		idx := 0
		for {
			select {
			case msg := <-incoming:
				for _, m := range msg {
					if string(m.Data) != string(testSource[idx]) {
						t.Error("result not except", string(m.Data))
					}
					wg.Done()
					idx++
				}
			case err := <-errChan:
				logex.Error("get:", err)
			}
		}
	}()
	go func() {
		for _, m := range testSource {
			msg := mmq.NewMessageByData(m)
			errs := topic.PutSync([]*mmq.Message{msg})
			logex.Error(errs)
		}
	}()
	wg.Wait()

}
<commit_msg>fix test<commit_after>package topic

import (
	"os"
	"sync"
	"testing"

	"gopkg.in/logex.v1"

	"github.com/chzyer/mmq/internal/utils"
	"github.com/chzyer/mmq/mmq"
)

var (
	c *Config
)

func init() {
	c = new(Config)
	c.ChunkBit = 22
	c.Root = "/data/mmq/test/topic"
	os.MkdirAll(c.Root, 0777)
	os.RemoveAll(c.Root)
}

func BenchmarkTopicPut(b *testing.B) {
	topic, err := New("bench-put", c)
	if err != nil {
		b.Fatal(err)
	}
	msg := mmq.NewMessageByData([]byte(utils.RandString(256)))
	reply := make(chan []error)
	var wg sync.WaitGroup
	go func() {
		for _ = range reply {
			wg.Done()
		}
	}()
	b.ResetTimer()
	buffer := []*mmq.Message{}
	for i := 0; i < b.N; i++ {
		m, _ := mmq.NewMessage(msg.Bytes(), true)
		buffer = append(buffer, m)
		if len(buffer) >= 100 {
			wg.Add(1)
			topic.Put(buffer, reply)
			buffer = nil
		}
	}
	wg.Wait()
}

func TestTopic(t *testing.T) {
	topic, err := New("topicTest", c)
	if err != nil {
		t.Fatal(err)
	}
	var wg sync.WaitGroup
	var testSource = [][]byte{
		[]byte("hello!"),
		[]byte("who are you"),
		[]byte("oo?"),
	}
	wg.Add(len(testSource))
	go func() {
		incoming := make(chan []*mmq.Message, len(testSource))
		errChan := make(chan error)
		topic.Get(0, len(testSource), incoming, errChan)
		idx := 0
		for {
			select {
			case msg := <-incoming:
				for _, m := range msg {
					if string(m.Data) != string(testSource[idx]) {
						t.Error("result not except", string(m.Data))
					}
					wg.Done()
					idx++
				}
			case err := <-errChan:
				logex.Error("get:", err)
			}
		}
	}()
	go func() {
		for _, m := range testSource {
			msg := mmq.NewMessageByData(m)
			errs := topic.PutSync([]*mmq.Message{msg})
			logex.Error(errs)
		}
	}()
	wg.Wait()

}
<|endoftext|>


<commit_before>package google

import (
	"fmt"
	"log"
	"time"

	"github.com/hashicorp/terraform-plugin-sdk/helper/schema"
	"google.golang.org/api/compute/v1"
)

func resourceProjectUsageBucket() *schema.Resource {
	return &schema.Resource{
		Create: resourceProjectUsageBucketCreate,
		Read:   resourceProjectUsageBucketRead,
		Delete: resourceProjectUsageBucketDelete,
		Importer: &schema.ResourceImporter{
			State: resourceProjectUsageBucketImportState,
		},

		Timeouts: &schema.ResourceTimeout{
			Create: schema.DefaultTimeout(4 * time.Minute),
			Delete: schema.DefaultTimeout(4 * time.Minute),
		},

		Schema: map[string]*schema.Schema{
			"bucket_name": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"prefix": {
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
			},
			"project": {
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
				ForceNew: true,
			},
		},
	}
}

func resourceProjectUsageBucketRead(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)

	project, err := getProject(d, config)
	if err != nil {
		return err
	}

	p, err := config.clientCompute.Projects.Get(project).Do()
	if err != nil {
		return handleNotFoundError(err, d, fmt.Sprintf("Project data for project %s", project))
	}

	if p.UsageExportLocation == nil {
		log.Printf("[WARN] Removing usage export location resource %s because it's not enabled server-side.", project)
		d.SetId("")
		return nil
	}

	d.Set("project", project)
	d.Set("prefix", p.UsageExportLocation.ReportNamePrefix)
	d.Set("bucket_name", p.UsageExportLocation.BucketName)
	return nil
}

func resourceProjectUsageBucketCreate(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)

	project, err := getProject(d, config)
	if err != nil {
		return err
	}

	op, err := config.clientCompute.Projects.SetUsageExportBucket(project, &compute.UsageExportLocation{
		ReportNamePrefix: d.Get("prefix").(string),
		BucketName:       d.Get("bucket_name").(string),
	}).Do()
	if err != nil {
		return err
	}
	d.SetId(project)
	err = computeOperationWaitTime(config, op, project, "Setting usage export bucket.", d.Timeout(schema.TimeoutCreate))
	if err != nil {
		d.SetId("")
		return err
	}

	d.Set("project", project)

	return resourceProjectUsageBucketRead(d, meta)
}

func resourceProjectUsageBucketDelete(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)

	project, err := getProject(d, config)
	if err != nil {
		return err
	}

	op, err := config.clientCompute.Projects.SetUsageExportBucket(project, nil).Do()
	if err != nil {
		return err
	}

	err = computeOperationWaitTime(config, op, project,
		"Setting usage export bucket to nil, automatically disabling usage export.", d.Timeout(schema.TimeoutDelete))
	if err != nil {
		return err
	}
	d.SetId("")

	return nil
}

func resourceProjectUsageBucketImportState(d *schema.ResourceData, meta interface{}) ([]*schema.ResourceData, error) {
	project := d.Id()
	d.Set("project", project)
	return []*schema.ResourceData{d}, nil
}
<commit_msg>Descriptions usage export bucket (#3688)<commit_after>package google

import (
	"fmt"
	"log"
	"time"

	"github.com/hashicorp/terraform-plugin-sdk/helper/schema"
	"google.golang.org/api/compute/v1"
)

func resourceProjectUsageBucket() *schema.Resource {
	return &schema.Resource{
		Create: resourceProjectUsageBucketCreate,
		Read:   resourceProjectUsageBucketRead,
		Delete: resourceProjectUsageBucketDelete,
		Importer: &schema.ResourceImporter{
			State: resourceProjectUsageBucketImportState,
		},

		Timeouts: &schema.ResourceTimeout{
			Create: schema.DefaultTimeout(4 * time.Minute),
			Delete: schema.DefaultTimeout(4 * time.Minute),
		},

		Schema: map[string]*schema.Schema{
			"bucket_name": {
				Type:        schema.TypeString,
				Required:    true,
				ForceNew:    true,
				Description: `The bucket to store reports in.`,
			},
			"prefix": {
				Type:        schema.TypeString,
				Optional:    true,
				ForceNew:    true,
				Description: `A prefix for the reports, for instance, the project name.`,
			},
			"project": {
				Type:        schema.TypeString,
				Optional:    true,
				Computed:    true,
				ForceNew:    true,
				Description: `The project to set the export bucket on. If it is not provided, the provider project is used.`,
			},
		},
	}
}

func resourceProjectUsageBucketRead(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)

	project, err := getProject(d, config)
	if err != nil {
		return err
	}

	p, err := config.clientCompute.Projects.Get(project).Do()
	if err != nil {
		return handleNotFoundError(err, d, fmt.Sprintf("Project data for project %s", project))
	}

	if p.UsageExportLocation == nil {
		log.Printf("[WARN] Removing usage export location resource %s because it's not enabled server-side.", project)
		d.SetId("")
		return nil
	}

	d.Set("project", project)
	d.Set("prefix", p.UsageExportLocation.ReportNamePrefix)
	d.Set("bucket_name", p.UsageExportLocation.BucketName)
	return nil
}

func resourceProjectUsageBucketCreate(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)

	project, err := getProject(d, config)
	if err != nil {
		return err
	}

	op, err := config.clientCompute.Projects.SetUsageExportBucket(project, &compute.UsageExportLocation{
		ReportNamePrefix: d.Get("prefix").(string),
		BucketName:       d.Get("bucket_name").(string),
	}).Do()
	if err != nil {
		return err
	}
	d.SetId(project)
	err = computeOperationWaitTime(config, op, project, "Setting usage export bucket.", d.Timeout(schema.TimeoutCreate))
	if err != nil {
		d.SetId("")
		return err
	}

	d.Set("project", project)

	return resourceProjectUsageBucketRead(d, meta)
}

func resourceProjectUsageBucketDelete(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)

	project, err := getProject(d, config)
	if err != nil {
		return err
	}

	op, err := config.clientCompute.Projects.SetUsageExportBucket(project, nil).Do()
	if err != nil {
		return err
	}

	err = computeOperationWaitTime(config, op, project,
		"Setting usage export bucket to nil, automatically disabling usage export.", d.Timeout(schema.TimeoutDelete))
	if err != nil {
		return err
	}
	d.SetId("")

	return nil
}

func resourceProjectUsageBucketImportState(d *schema.ResourceData, meta interface{}) ([]*schema.ResourceData, error) {
	project := d.Id()
	d.Set("project", project)
	return []*schema.ResourceData{d}, nil
}
<|endoftext|>


<commit_before>// Package migrate is imported by other Go code.
// It is the entry point to all migration functions.
package migrate

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/signal"
	"path"
	"strconv"
	"strings"

	"github.com/mattes/migrate/driver"
	"github.com/mattes/migrate/file"
	"github.com/mattes/migrate/migrate/direction"
	pipep "github.com/mattes/migrate/pipe"
)

// Up applies all available migrations
func Up(pipe chan interface{}, url, migrationsPath string) {
	d, files, version, err := initDriverAndReadMigrationFilesAndGetVersion(url, migrationsPath)
	if err != nil {
		go pipep.Close(pipe, err)
		return
	}

	applyMigrationFiles, err := files.ToLastFrom(version)
	if err != nil {
		if err2 := d.Close(); err != nil {
			pipe <- err2
		}
		go pipep.Close(pipe, err)
		return
	}

	if len(applyMigrationFiles) > 0 {
		for _, f := range applyMigrationFiles {
			pipe1 := pipep.New()
			go d.Migrate(f, pipe1)
			if ok := pipep.WaitAndRedirect(pipe1, pipe, handleInterrupts()); !ok {
				break
			}
		}
		if err2 := d.Close(); err != nil {
			pipe <- err2
		}
		go pipep.Close(pipe, nil)
		return
	} else {
		if err2 := d.Close(); err != nil {
			pipe <- err2
		}
		go pipep.Close(pipe, nil)
		return
	}
}

// UpSync is synchronous version of Up
func UpSync(url, migrationsPath string) (err []error, ok bool) {
	pipe := pipep.New()
	go Up(pipe, url, migrationsPath)
	err = pipep.ReadErrors(pipe)
	return err, len(err) == 0
}

// Down rolls back all migrations
func Down(pipe chan interface{}, url, migrationsPath string) {
	d, files, version, err := initDriverAndReadMigrationFilesAndGetVersion(url, migrationsPath)
	if err != nil {
		go pipep.Close(pipe, err)
		return
	}

	applyMigrationFiles, err := files.ToFirstFrom(version)
	if err != nil {
		if err2 := d.Close(); err != nil {
			pipe <- err2
		}
		go pipep.Close(pipe, err)
		return
	}

	if len(applyMigrationFiles) > 0 {
		for _, f := range applyMigrationFiles {
			pipe1 := pipep.New()
			go d.Migrate(f, pipe1)
			if ok := pipep.WaitAndRedirect(pipe1, pipe, handleInterrupts()); !ok {
				break
			}
		}
		if err2 := d.Close(); err != nil {
			pipe <- err2
		}
		go pipep.Close(pipe, nil)
		return
	} else {
		if err2 := d.Close(); err != nil {
			pipe <- err2
		}
		go pipep.Close(pipe, nil)
		return
	}
}

// DownSync is synchronous version of Down
func DownSync(url, migrationsPath string) (err []error, ok bool) {
	pipe := pipep.New()
	go Down(pipe, url, migrationsPath)
	err = pipep.ReadErrors(pipe)
	return err, len(err) == 0
}

// Redo rolls back the most recently applied migration, then runs it again.
func Redo(pipe chan interface{}, url, migrationsPath string) {
	pipe1 := pipep.New()
	go Migrate(pipe1, url, migrationsPath, -1)
	if ok := pipep.WaitAndRedirect(pipe1, pipe, handleInterrupts()); !ok {
		go pipep.Close(pipe, nil)
		return
	} else {
		go Migrate(pipe, url, migrationsPath, +1)
	}
}

// RedoSync is synchronous version of Redo
func RedoSync(url, migrationsPath string) (err []error, ok bool) {
	pipe := pipep.New()
	go Redo(pipe, url, migrationsPath)
	err = pipep.ReadErrors(pipe)
	return err, len(err) == 0
}

// Reset runs the down and up migration function
func Reset(pipe chan interface{}, url, migrationsPath string) {
	pipe1 := pipep.New()
	go Down(pipe1, url, migrationsPath)
	if ok := pipep.WaitAndRedirect(pipe1, pipe, handleInterrupts()); !ok {
		go pipep.Close(pipe, nil)
		return
	} else {
		go Up(pipe, url, migrationsPath)
	}
}

// ResetSync is synchronous version of Reset
func ResetSync(url, migrationsPath string) (err []error, ok bool) {
	pipe := pipep.New()
	go Reset(pipe, url, migrationsPath)
	err = pipep.ReadErrors(pipe)
	return err, len(err) == 0
}

// Migrate applies relative +n/-n migrations
func Migrate(pipe chan interface{}, url, migrationsPath string, relativeN int) {
	d, files, version, err := initDriverAndReadMigrationFilesAndGetVersion(url, migrationsPath)
	if err != nil {
		go pipep.Close(pipe, err)
		return
	}

	applyMigrationFiles, err := files.From(version, relativeN)
	if err != nil {
		if err2 := d.Close(); err != nil {
			pipe <- err2
		}
		go pipep.Close(pipe, err)
		return
	}

	if len(applyMigrationFiles) > 0 && relativeN != 0 {
		for _, f := range applyMigrationFiles {
			pipe1 := pipep.New()
			go d.Migrate(f, pipe1)
			if ok := pipep.WaitAndRedirect(pipe1, pipe, handleInterrupts()); !ok {
				break
			}
		}
		if err2 := d.Close(); err != nil {
			pipe <- err2
		}
		go pipep.Close(pipe, nil)
		return
	}
	if err2 := d.Close(); err != nil {
		pipe <- err2
	}
	go pipep.Close(pipe, nil)
	return
}

// MigrateSync is synchronous version of Migrate
func MigrateSync(url, migrationsPath string, relativeN int) (err []error, ok bool) {
	pipe := pipep.New()
	go Migrate(pipe, url, migrationsPath, relativeN)
	err = pipep.ReadErrors(pipe)
	return err, len(err) == 0
}

// Version returns the current migration version
func Version(url, migrationsPath string) (version uint64, err error) {
	d, err := driver.New(url)
	if err != nil {
		return 0, err
	}
	return d.Version()
}

// Create creates new migration files on disk
func Create(url, migrationsPath, name string) (*file.MigrationFile, error) {
	d, err := driver.New(url)
	if err != nil {
		return nil, err
	}
	files, err := file.ReadMigrationFiles(migrationsPath, file.FilenameRegex(d.FilenameExtension()))
	if err != nil {
		return nil, err
	}

	version := uint64(0)
	if len(files) > 0 {
		lastFile := files[len(files)-1]
		version = lastFile.Version
	}
	version += 1
	versionStr := strconv.FormatUint(version, 10)

	length := 4 // TODO(mattes) check existing files and try to guess length
	if len(versionStr)%length != 0 {
		versionStr = strings.Repeat("0", length-len(versionStr)%length) + versionStr
	}

	filenamef := "%s_%s.%s.%s"
	name = strings.Replace(name, " ", "_", -1)

	mfile := &file.MigrationFile{
		Version: version,
		UpFile: &file.File{
			Path:      migrationsPath,
			FileName:  fmt.Sprintf(filenamef, versionStr, name, "up", d.FilenameExtension()),
			Name:      name,
			Content:   []byte(""),
			Direction: direction.Up,
		},
		DownFile: &file.File{
			Path:      migrationsPath,
			FileName:  fmt.Sprintf(filenamef, versionStr, name, "down", d.FilenameExtension()),
			Name:      name,
			Content:   []byte(""),
			Direction: direction.Down,
		},
	}

	if err := ioutil.WriteFile(path.Join(mfile.UpFile.Path, mfile.UpFile.FileName), mfile.UpFile.Content, 0644); err != nil {
		return nil, err
	}
	if err := ioutil.WriteFile(path.Join(mfile.DownFile.Path, mfile.DownFile.FileName), mfile.DownFile.Content, 0644); err != nil {
		return nil, err
	}

	return mfile, nil
}

// initDriverAndReadMigrationFilesAndGetVersion is a small helper
// function that is common to most of the migration funcs
func initDriverAndReadMigrationFilesAndGetVersion(url, migrationsPath string) (driver.Driver, *file.MigrationFiles, uint64, error) {
	d, err := driver.New(url)
	if err != nil {
		return nil, nil, 0, err
	}
	files, err := file.ReadMigrationFiles(migrationsPath, file.FilenameRegex(d.FilenameExtension()))
	if err != nil {
		d.Close() // TODO what happens with errors from this func?
		return nil, nil, 0, err
	}
	version, err := d.Version()
	if err != nil {
		d.Close() // TODO what happens with errors from this func?
		return nil, nil, 0, err
	}
	return d, &files, version, nil
}

// NewPipe is a convenience function for pipe.New().
// This is helpful if the user just wants to import this package and nothing else.
func NewPipe() chan interface{} {
	return pipep.New()
}

// interrupts is an internal variable that holds the state of
// interrupt handling
var interrupts = true

// Graceful enables interrupts checking. Once the first ^C is received
// it will finish the currently running migration and abort execution
// of the next migration. If ^C is received twice, it will stop
// execution immediately.
func Graceful() {
	interrupts = true
}

// NonGraceful disables interrupts checking. The first received ^C will
// stop execution immediately.
func NonGraceful() {
	interrupts = false
}

// interrupts returns a signal channel if interrupts checking is
// enabled. nil otherwise.
func handleInterrupts() chan os.Signal {
	if interrupts {
		c := make(chan os.Signal, 1)
		signal.Notify(c, os.Interrupt)
		return c
	}
	return nil
}
<commit_msg>fix check on err when using err2<commit_after>// Package migrate is imported by other Go code.
// It is the entry point to all migration functions.
package migrate

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/signal"
	"path"
	"strconv"
	"strings"

	"github.com/mattes/migrate/driver"
	"github.com/mattes/migrate/file"
	"github.com/mattes/migrate/migrate/direction"
	pipep "github.com/mattes/migrate/pipe"
)

// Up applies all available migrations
func Up(pipe chan interface{}, url, migrationsPath string) {
	d, files, version, err := initDriverAndReadMigrationFilesAndGetVersion(url, migrationsPath)
	if err != nil {
		go pipep.Close(pipe, err)
		return
	}

	applyMigrationFiles, err := files.ToLastFrom(version)
	if err != nil {
		if err2 := d.Close(); err2 != nil {
			pipe <- err2
		}
		go pipep.Close(pipe, err)
		return
	}

	if len(applyMigrationFiles) > 0 {
		for _, f := range applyMigrationFiles {
			pipe1 := pipep.New()
			go d.Migrate(f, pipe1)
			if ok := pipep.WaitAndRedirect(pipe1, pipe, handleInterrupts()); !ok {
				break
			}
		}
		if err := d.Close(); err != nil {
			pipe <- err
		}
		go pipep.Close(pipe, nil)
		return
	} else {
		if err := d.Close(); err != nil {
			pipe <- err
		}
		go pipep.Close(pipe, nil)
		return
	}
}

// UpSync is synchronous version of Up
func UpSync(url, migrationsPath string) (err []error, ok bool) {
	pipe := pipep.New()
	go Up(pipe, url, migrationsPath)
	err = pipep.ReadErrors(pipe)
	return err, len(err) == 0
}

// Down rolls back all migrations
func Down(pipe chan interface{}, url, migrationsPath string) {
	d, files, version, err := initDriverAndReadMigrationFilesAndGetVersion(url, migrationsPath)
	if err != nil {
		go pipep.Close(pipe, err)
		return
	}

	applyMigrationFiles, err := files.ToFirstFrom(version)
	if err != nil {
		if err2 := d.Close(); err2 != nil {
			pipe <- err2
		}
		go pipep.Close(pipe, err)
		return
	}

	if len(applyMigrationFiles) > 0 {
		for _, f := range applyMigrationFiles {
			pipe1 := pipep.New()
			go d.Migrate(f, pipe1)
			if ok := pipep.WaitAndRedirect(pipe1, pipe, handleInterrupts()); !ok {
				break
			}
		}
		if err2 := d.Close(); err2 != nil {
			pipe <- err2
		}
		go pipep.Close(pipe, nil)
		return
	} else {
		if err2 := d.Close(); err2 != nil {
			pipe <- err2
		}
		go pipep.Close(pipe, nil)
		return
	}
}

// DownSync is synchronous version of Down
func DownSync(url, migrationsPath string) (err []error, ok bool) {
	pipe := pipep.New()
	go Down(pipe, url, migrationsPath)
	err = pipep.ReadErrors(pipe)
	return err, len(err) == 0
}

// Redo rolls back the most recently applied migration, then runs it again.
func Redo(pipe chan interface{}, url, migrationsPath string) {
	pipe1 := pipep.New()
	go Migrate(pipe1, url, migrationsPath, -1)
	if ok := pipep.WaitAndRedirect(pipe1, pipe, handleInterrupts()); !ok {
		go pipep.Close(pipe, nil)
		return
	} else {
		go Migrate(pipe, url, migrationsPath, +1)
	}
}

// RedoSync is synchronous version of Redo
func RedoSync(url, migrationsPath string) (err []error, ok bool) {
	pipe := pipep.New()
	go Redo(pipe, url, migrationsPath)
	err = pipep.ReadErrors(pipe)
	return err, len(err) == 0
}

// Reset runs the down and up migration function
func Reset(pipe chan interface{}, url, migrationsPath string) {
	pipe1 := pipep.New()
	go Down(pipe1, url, migrationsPath)
	if ok := pipep.WaitAndRedirect(pipe1, pipe, handleInterrupts()); !ok {
		go pipep.Close(pipe, nil)
		return
	} else {
		go Up(pipe, url, migrationsPath)
	}
}

// ResetSync is synchronous version of Reset
func ResetSync(url, migrationsPath string) (err []error, ok bool) {
	pipe := pipep.New()
	go Reset(pipe, url, migrationsPath)
	err = pipep.ReadErrors(pipe)
	return err, len(err) == 0
}

// Migrate applies relative +n/-n migrations
func Migrate(pipe chan interface{}, url, migrationsPath string, relativeN int) {
	d, files, version, err := initDriverAndReadMigrationFilesAndGetVersion(url, migrationsPath)
	if err != nil {
		go pipep.Close(pipe, err)
		return
	}

	applyMigrationFiles, err := files.From(version, relativeN)
	if err != nil {
		if err2 := d.Close(); err2 != nil {
			pipe <- err2
		}
		go pipep.Close(pipe, err)
		return
	}

	if len(applyMigrationFiles) > 0 && relativeN != 0 {
		for _, f := range applyMigrationFiles {
			pipe1 := pipep.New()
			go d.Migrate(f, pipe1)
			if ok := pipep.WaitAndRedirect(pipe1, pipe, handleInterrupts()); !ok {
				break
			}
		}
		if err2 := d.Close(); err2 != nil {
			pipe <- err2
		}
		go pipep.Close(pipe, nil)
		return
	}
	if err2 := d.Close(); err2 != nil {
		pipe <- err2
	}
	go pipep.Close(pipe, nil)
	return
}

// MigrateSync is synchronous version of Migrate
func MigrateSync(url, migrationsPath string, relativeN int) (err []error, ok bool) {
	pipe := pipep.New()
	go Migrate(pipe, url, migrationsPath, relativeN)
	err = pipep.ReadErrors(pipe)
	return err, len(err) == 0
}

// Version returns the current migration version
func Version(url, migrationsPath string) (version uint64, err error) {
	d, err := driver.New(url)
	if err != nil {
		return 0, err
	}
	return d.Version()
}

// Create creates new migration files on disk
func Create(url, migrationsPath, name string) (*file.MigrationFile, error) {
	d, err := driver.New(url)
	if err != nil {
		return nil, err
	}
	files, err := file.ReadMigrationFiles(migrationsPath, file.FilenameRegex(d.FilenameExtension()))
	if err != nil {
		return nil, err
	}

	version := uint64(0)
	if len(files) > 0 {
		lastFile := files[len(files)-1]
		version = lastFile.Version
	}
	version += 1
	versionStr := strconv.FormatUint(version, 10)

	length := 4 // TODO(mattes) check existing files and try to guess length
	if len(versionStr)%length != 0 {
		versionStr = strings.Repeat("0", length-len(versionStr)%length) + versionStr
	}

	filenamef := "%s_%s.%s.%s"
	name = strings.Replace(name, " ", "_", -1)

	mfile := &file.MigrationFile{
		Version: version,
		UpFile: &file.File{
			Path:      migrationsPath,
			FileName:  fmt.Sprintf(filenamef, versionStr, name, "up", d.FilenameExtension()),
			Name:      name,
			Content:   []byte(""),
			Direction: direction.Up,
		},
		DownFile: &file.File{
			Path:      migrationsPath,
			FileName:  fmt.Sprintf(filenamef, versionStr, name, "down", d.FilenameExtension()),
			Name:      name,
			Content:   []byte(""),
			Direction: direction.Down,
		},
	}

	if err := ioutil.WriteFile(path.Join(mfile.UpFile.Path, mfile.UpFile.FileName), mfile.UpFile.Content, 0644); err != nil {
		return nil, err
	}
	if err := ioutil.WriteFile(path.Join(mfile.DownFile.Path, mfile.DownFile.FileName), mfile.DownFile.Content, 0644); err != nil {
		return nil, err
	}

	return mfile, nil
}

// initDriverAndReadMigrationFilesAndGetVersion is a small helper
// function that is common to most of the migration funcs
func initDriverAndReadMigrationFilesAndGetVersion(url, migrationsPath string) (driver.Driver, *file.MigrationFiles, uint64, error) {
	d, err := driver.New(url)
	if err != nil {
		return nil, nil, 0, err
	}
	files, err := file.ReadMigrationFiles(migrationsPath, file.FilenameRegex(d.FilenameExtension()))
	if err != nil {
		d.Close() // TODO what happens with errors from this func?
		return nil, nil, 0, err
	}
	version, err := d.Version()
	if err != nil {
		d.Close() // TODO what happens with errors from this func?
		return nil, nil, 0, err
	}
	return d, &files, version, nil
}

// NewPipe is a convenience function for pipe.New().
// This is helpful if the user just wants to import this package and nothing else.
func NewPipe() chan interface{} {
	return pipep.New()
}

// interrupts is an internal variable that holds the state of
// interrupt handling
var interrupts = true

// Graceful enables interrupts checking. Once the first ^C is received
// it will finish the currently running migration and abort execution
// of the next migration. If ^C is received twice, it will stop
// execution immediately.
func Graceful() {
	interrupts = true
}

// NonGraceful disables interrupts checking. The first received ^C will
// stop execution immediately.
func NonGraceful() {
	interrupts = false
}

// interrupts returns a signal channel if interrupts checking is
// enabled. nil otherwise.
func handleInterrupts() chan os.Signal {
	if interrupts {
		c := make(chan os.Signal, 1)
		signal.Notify(c, os.Interrupt)
		return c
	}
	return nil
}
<|endoftext|>