<commit_before>package main

import (
	"crypto/sha1"
	"encoding/hex"
	"flag"
	"fmt"
	"io"
	"log"
	"mime"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"sync"
)

var uploadWg = sync.WaitGroup{}

var uploadFlags = flag.NewFlagSet("upload", flag.ExitOnError)
var uploadVerbose = uploadFlags.Bool("v", false, "Verbose")
var uploadDelete = uploadFlags.Bool("delete", false,
	"Delete locally missing items")

type uploadOpType uint8

const (
	uploadFileOp = uploadOpType(iota)
	removeFileOp
	removeRecurseOp
)

type uploadReq struct {
	src        string
	dest       string
	op         uploadOpType
	remoteHash string
}

func recognizeTypeByName(n, def string) string {
	byname := mime.TypeByExtension(n)
	switch {
	case byname != "":
		return byname
	case strings.HasSuffix(n, ".js"):
		return "application/javascript"
	}
	return def
}

func uploadFile(src, dest string) error {
	if *uploadVerbose {
		log.Printf("Uploading %v -> %v", src, dest)
	}

	f, err := os.Open(src)
	if err != nil {
		return err
	}
	defer f.Close()

	someBytes := make([]byte, 512)
	n, err := f.Read(someBytes)
	if err != nil && err != io.EOF {
		return err
	}
	someBytes = someBytes[:n]
	_, err = f.Seek(0, 0)
	if err != nil {
		return err
	}

	preq, err := http.NewRequest("PUT", dest, f)
	if err != nil {
		return err
	}
	preq.Header.Set("X-CBFS-KeepRevs", strconv.Itoa(*revs))

	ctype := http.DetectContentType(someBytes)
	if strings.HasPrefix(ctype, "text/plain") ||
		strings.HasPrefix(ctype, "application/octet-stream") {
		ctype = recognizeTypeByName(src, ctype)
	}

	preq.Header.Set("Content-Type", ctype)

	resp, err := http.DefaultClient.Do(preq)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 201 {
		return fmt.Errorf("HTTP Error:  %v", resp.Status)
	}
	return nil
}

// This is very similar to rm's version, but uses different channel
// signaling.
func uploadRmDashR(baseUrl string, ch chan uploadReq) ([]string, error) {
	for strings.HasSuffix(baseUrl, "/") {
		baseUrl = baseUrl[:len(baseUrl)-1]
	}

	listing, err := listStuff(baseUrl)
	for err != nil {
		return []string{}, err
	}
	for fn := range listing.Files {
		err = rmFile(baseUrl + "/" + fn)
		if err != nil {
			return []string{}, err
		}
	}
	children := make([]string, 0, len(listing.Dirs))
	for dn := range listing.Dirs {
		children = append(children, baseUrl+"/"+dn)
	}
	return children, nil
}

func localHash(fn string) string {
	f, err := os.Open(fn)
	if err != nil {
		return "unknown"
	}
	defer f.Close()

	h := sha1.New()
	_, err = io.Copy(h, f)
	if err != nil {
		return "unknown"
	}

	return hex.EncodeToString(h.Sum([]byte{}))
}

func uploadWorker(ch chan uploadReq) {
	defer uploadWg.Done()
	for req := range ch {
		retries := 0
		done := false
		for !done {
			var err error
			switch req.op {
			case uploadFileOp:
				if req.remoteHash == "" {
					err = uploadFile(req.src, req.dest)
				} else {
					if localHash(req.src) != req.remoteHash {
						if *uploadVerbose {
							log.Printf("%v has changed, reupping",
								req.src)
						}
						err = uploadFile(req.src, req.dest)
					}
				}
			case removeFileOp:
				if *uploadVerbose {
					log.Printf("Removing file %v", req.dest)
				}
				err = rmFile(req.dest)
			case removeRecurseOp:
				todo := []string{req.dest}
				for err == nil && len(todo) > 0 {
					todo, err = uploadRmDashR(req.dest, ch)
				}
			default:
				log.Fatalf("Unhandled case")
			}
			if err != nil {
				if retries < 3 {
					retries++
					log.Printf("Error uploading file: %v... retrying",
						err)
					time.Sleep(time.Duration(retries) * time.Second)
				} else {
					log.Printf("Error uploading file %v: %v",
						req.src, err)
				}
			} else {
				done = true
			}
		}
	}
}

func syncPath(path, dest string, info os.FileInfo, ch chan<- uploadReq) error {
	f, err := os.Open(path)
	if err != nil {
		return err
	}
	defer f.Close()
	children, err := f.Readdir(0)
	if err != nil {
		return err
	}

	retries := 3
	serverListing, err := listStuff(dest)
	for err != nil && retries > 0 {
		serverListing, err = listStuff(dest)
		time.Sleep(time.Second)
		retries--
	}
	if err != nil {
		// XXX: Stick an error somewhere
		return nil
	}

	localNames := map[string]os.FileInfo{}
	for _, c := range children {
		switch c.Mode() & os.ModeType {
		case os.ModeCharDevice, os.ModeDevice,
			os.ModeNamedPipe, os.ModeSocket, os.ModeSymlink:
			if *uploadVerbose {
				log.Printf("Ignoring special file: %v - %v",
					filepath.Join(path, c.Name()), c.Mode())
			}
		default:
			localNames[c.Name()] = c
		}
	}

	remoteNames := map[string]bool{}
	for n := range serverListing.Files {
		remoteNames[n] = true
	}
	for n := range serverListing.Dirs {
		remoteNames[n] = true
	}

	missingUpstream := []string{}
	for n, fi := range localNames {
		if !(fi.IsDir() || remoteNames[n]) {
			missingUpstream = append(missingUpstream, n)
		} else if !fi.IsDir() {
			if ri, ok := serverListing.Files[n]; ok {
				ch <- uploadReq{filepath.Join(path, n),
					dest + "/" + n, uploadFileOp, ri.OID}
			}
		}
	}

	toRm := []string{}
	for n := range remoteNames {
		if _, ok := localNames[n]; !ok {
			toRm = append(toRm, n)
		}
	}

	if len(missingUpstream) > 0 {
		for _, m := range missingUpstream {
			ch <- uploadReq{filepath.Join(path, m),
				dest + "/" + m, uploadFileOp, ""}
		}
	}

	if *uploadDelete && len(toRm) > 0 {
		for _, m := range toRm {
			ch <- uploadReq{"", dest + "/" + m, removeFileOp, ""}
			ch <- uploadReq{"", dest + "/" + m, removeRecurseOp, ""}
		}
	}

	return nil
}

func syncUp(src, u string, ch chan<- uploadReq) {
	for strings.HasSuffix(u, "/") {
		u = u[:len(u)-1]
	}
	for strings.HasSuffix(src, "/") {
		src = src[:len(src)-1]
	}

	err := filepath.Walk(src,
		func(path string, info os.FileInfo, err error) error {
			if err == nil && info.IsDir() {
				shortPath := path[len(src):]
				err = syncPath(path, u+shortPath, info, ch)
			}
			return err
		})
	if err != nil {
		log.Fatalf("Traversal error: %v", err)
	}
}

func uploadCommand(args []string) {
	uploadFlags.Parse(args)

	if uploadFlags.NArg() < 2 {
		log.Fatalf("src and dest required")
	}

	fi, err := os.Stat(uploadFlags.Arg(0))
	if err != nil {
		log.Fatal(err)
	}

	if fi.IsDir() {
		ch := make(chan uploadReq, 1000)

		for i := 0; i < *workers; i++ {
			uploadWg.Add(1)
			go uploadWorker(ch)
		}

		start := time.Now()
		syncUp(uploadFlags.Arg(0), uploadFlags.Arg(1), ch)

		close(ch)
		log.Printf("Finished traversal in %v", time.Since(start))
		uploadWg.Wait()
		log.Printf("Finished sync in %v", time.Since(start))
	} else {
		err = uploadFile(uploadFlags.Arg(0), uploadFlags.Arg(1))
		if err != nil {
			log.Fatalf("Error uploading file: %v", err)
		}
	}
}
<commit_msg>Ignore "" directory.<commit_after>package main

import (
	"crypto/sha1"
	"encoding/hex"
	"flag"
	"fmt"
	"io"
	"log"
	"mime"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"sync"
)

var uploadWg = sync.WaitGroup{}

var uploadFlags = flag.NewFlagSet("upload", flag.ExitOnError)
var uploadVerbose = uploadFlags.Bool("v", false, "Verbose")
var uploadDelete = uploadFlags.Bool("delete", false,
	"Delete locally missing items")

type uploadOpType uint8

const (
	uploadFileOp = uploadOpType(iota)
	removeFileOp
	removeRecurseOp
)

type uploadReq struct {
	src        string
	dest       string
	op         uploadOpType
	remoteHash string
}

func recognizeTypeByName(n, def string) string {
	byname := mime.TypeByExtension(n)
	switch {
	case byname != "":
		return byname
	case strings.HasSuffix(n, ".js"):
		return "application/javascript"
	}
	return def
}

func uploadFile(src, dest string) error {
	if *uploadVerbose {
		log.Printf("Uploading %v -> %v", src, dest)
	}

	f, err := os.Open(src)
	if err != nil {
		return err
	}
	defer f.Close()

	someBytes := make([]byte, 512)
	n, err := f.Read(someBytes)
	if err != nil && err != io.EOF {
		return err
	}
	someBytes = someBytes[:n]
	_, err = f.Seek(0, 0)
	if err != nil {
		return err
	}

	preq, err := http.NewRequest("PUT", dest, f)
	if err != nil {
		return err
	}
	preq.Header.Set("X-CBFS-KeepRevs", strconv.Itoa(*revs))

	ctype := http.DetectContentType(someBytes)
	if strings.HasPrefix(ctype, "text/plain") ||
		strings.HasPrefix(ctype, "application/octet-stream") {
		ctype = recognizeTypeByName(src, ctype)
	}

	preq.Header.Set("Content-Type", ctype)

	resp, err := http.DefaultClient.Do(preq)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 201 {
		return fmt.Errorf("HTTP Error:  %v", resp.Status)
	}
	return nil
}

// This is very similar to rm's version, but uses different channel
// signaling.
func uploadRmDashR(baseUrl string, ch chan uploadReq) ([]string, error) {
	for strings.HasSuffix(baseUrl, "/") {
		baseUrl = baseUrl[:len(baseUrl)-1]
	}

	listing, err := listStuff(baseUrl)
	for err != nil {
		return []string{}, err
	}
	for fn := range listing.Files {
		err = rmFile(baseUrl + "/" + fn)
		if err != nil {
			return []string{}, err
		}
	}
	children := make([]string, 0, len(listing.Dirs))
	for dn := range listing.Dirs {
		children = append(children, baseUrl+"/"+dn)
	}
	return children, nil
}

func localHash(fn string) string {
	f, err := os.Open(fn)
	if err != nil {
		return "unknown"
	}
	defer f.Close()

	h := sha1.New()
	_, err = io.Copy(h, f)
	if err != nil {
		return "unknown"
	}

	return hex.EncodeToString(h.Sum([]byte{}))
}

func uploadWorker(ch chan uploadReq) {
	defer uploadWg.Done()
	for req := range ch {
		retries := 0
		done := false
		for !done {
			var err error
			switch req.op {
			case uploadFileOp:
				if req.remoteHash == "" {
					err = uploadFile(req.src, req.dest)
				} else {
					if localHash(req.src) != req.remoteHash {
						if *uploadVerbose {
							log.Printf("%v has changed, reupping",
								req.src)
						}
						err = uploadFile(req.src, req.dest)
					}
				}
			case removeFileOp:
				if *uploadVerbose {
					log.Printf("Removing file %v", req.dest)
				}
				err = rmFile(req.dest)
			case removeRecurseOp:
				todo := []string{req.dest}
				for err == nil && len(todo) > 0 {
					todo, err = uploadRmDashR(req.dest, ch)
				}
			default:
				log.Fatalf("Unhandled case")
			}
			if err != nil {
				if retries < 3 {
					retries++
					log.Printf("Error uploading file: %v... retrying",
						err)
					time.Sleep(time.Duration(retries) * time.Second)
				} else {
					log.Printf("Error uploading file %v: %v",
						req.src, err)
				}
			} else {
				done = true
			}
		}
	}
}

func syncPath(path, dest string, info os.FileInfo, ch chan<- uploadReq) error {
	f, err := os.Open(path)
	if err != nil {
		return err
	}
	defer f.Close()
	children, err := f.Readdir(0)
	if err != nil {
		return err
	}

	retries := 3
	serverListing, err := listStuff(dest)
	for err != nil && retries > 0 {
		serverListing, err = listStuff(dest)
		time.Sleep(time.Second)
		retries--
	}
	if err != nil {
		// XXX: Stick an error somewhere
		return nil
	}

	localNames := map[string]os.FileInfo{}
	for _, c := range children {
		switch c.Mode() & os.ModeType {
		case os.ModeCharDevice, os.ModeDevice,
			os.ModeNamedPipe, os.ModeSocket, os.ModeSymlink:
			if *uploadVerbose {
				log.Printf("Ignoring special file: %v - %v",
					filepath.Join(path, c.Name()), c.Mode())
			}
		default:
			localNames[c.Name()] = c
		}
	}

	remoteNames := map[string]bool{}
	for n := range serverListing.Files {
		remoteNames[n] = true
	}
	for n := range serverListing.Dirs {
		if n != "" {
			remoteNames[n] = true
		}
	}

	missingUpstream := []string{}
	for n, fi := range localNames {
		if !(fi.IsDir() || remoteNames[n]) {
			missingUpstream = append(missingUpstream, n)
		} else if !fi.IsDir() {
			if ri, ok := serverListing.Files[n]; ok {
				ch <- uploadReq{filepath.Join(path, n),
					dest + "/" + n, uploadFileOp, ri.OID}
			}
		}
	}

	toRm := []string{}
	for n := range remoteNames {
		if _, ok := localNames[n]; !ok {
			toRm = append(toRm, n)
		}
	}

	if len(missingUpstream) > 0 {
		for _, m := range missingUpstream {
			ch <- uploadReq{filepath.Join(path, m),
				dest + "/" + m, uploadFileOp, ""}
		}
	}

	if *uploadDelete && len(toRm) > 0 {
		for _, m := range toRm {
			ch <- uploadReq{"", dest + "/" + m, removeFileOp, ""}
			ch <- uploadReq{"", dest + "/" + m, removeRecurseOp, ""}
		}
	}

	return nil
}

func syncUp(src, u string, ch chan<- uploadReq) {
	for strings.HasSuffix(u, "/") {
		u = u[:len(u)-1]
	}
	for strings.HasSuffix(src, "/") {
		src = src[:len(src)-1]
	}

	err := filepath.Walk(src,
		func(path string, info os.FileInfo, err error) error {
			if err == nil && info.IsDir() {
				shortPath := path[len(src):]
				err = syncPath(path, u+shortPath, info, ch)
			}
			return err
		})
	if err != nil {
		log.Fatalf("Traversal error: %v", err)
	}
}

func uploadCommand(args []string) {
	uploadFlags.Parse(args)

	if uploadFlags.NArg() < 2 {
		log.Fatalf("src and dest required")
	}

	fi, err := os.Stat(uploadFlags.Arg(0))
	if err != nil {
		log.Fatal(err)
	}

	if fi.IsDir() {
		ch := make(chan uploadReq, 1000)

		for i := 0; i < *workers; i++ {
			uploadWg.Add(1)
			go uploadWorker(ch)
		}

		start := time.Now()
		syncUp(uploadFlags.Arg(0), uploadFlags.Arg(1), ch)

		close(ch)
		log.Printf("Finished traversal in %v", time.Since(start))
		uploadWg.Wait()
		log.Printf("Finished sync in %v", time.Since(start))
	} else {
		err = uploadFile(uploadFlags.Arg(0), uploadFlags.Arg(1))
		if err != nil {
			log.Fatalf("Error uploading file: %v", err)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build ignore

package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"reflect"
	"runtime"
	"strings"
)

func main() {
	flag.Parse()

	args, target := flag.Args(), "peg"
	if len(args) > 0 {
		target = args[0]
	}

	switch target {
	case "peg":
		peg()
	case "clean":
		clean()
	case "test":
		test()
	case "bench":
		bench()
	}
}

var processed = make(map[string]bool)

func done(file string, deps ...interface{}) bool {
	fini := true
	info, err := os.Stat(file)
	if err != nil {
		fini = false
	}
	for _, dep := range deps {
		switch dep := dep.(type) {
		case string:
			if info == nil {
				fini = false
				break
			}

			fileInfo, err := os.Stat(dep)
			if err != nil {
				panic(err)
			}

			if fileInfo.ModTime().After(info.ModTime()) {
				fini = false
			}
		case func() bool:
			name := runtime.FuncForPC(reflect.ValueOf(dep).Pointer()).Name()
			if processed[name] {
				fmt.Printf("%s is done\n", name)
				break
			}
			result := dep()
			fini = fini && result
			fmt.Printf("%s\n", name)
			processed[name] = true
		}
	}

	return fini
}

func chdir(dir string) string {
	working, err := os.Getwd()
	if err != nil {
		panic(err)
	}
	err = os.Chdir(dir)
	if err != nil {
		panic(err)
	}
	fmt.Printf("cd %s\n", dir)
	return working
}

func command(name, inputFile, outputFile string, arg ...string) {
	fmt.Print(name)
	for _, a := range arg {
		fmt.Printf(" %s", a)
	}

	cmd := exec.Command(name, arg...)

	if inputFile != "" {
		fmt.Printf(" < %s", inputFile)
		input, err := ioutil.ReadFile(inputFile)
		if err != nil {
			panic(err)
		}
		writer, err := cmd.StdinPipe()
		if err != nil {
			panic(err)
		}
		go func() {
			defer writer.Close()
			_, err := writer.Write([]byte(input))
			if err != nil {
				panic(err)
			}
		}()
	}

	if outputFile != "" {
		fmt.Printf(" > %s\n", outputFile)
		output, err := cmd.Output()
		if err != nil {
			panic(err)
		}
		err = ioutil.WriteFile(outputFile, output, 0600)
		if err != nil {
			panic(err)
		}
	} else {
		output, err := cmd.CombinedOutput()
		if err != nil {
			panic(err)
		}
		fmt.Printf("\n%s", string(output))
	}
}

func delete(file string) {
	command("rm", "", "", "-f", file)
}

func deleteFilesWithSuffix(suffix string) {
	files, err := ioutil.ReadDir("./")
	if err != nil {
		panic(err)
	}
	for _, file := range files {
		if strings.HasSuffix(file.Name(), suffix) {
			delete(file.Name())
		}
	}
}

func bootstrap() bool {
	if done("bootstrap/bootstrap", "bootstrap/main.go", "tree/peg.go") {
		return true
	}

	wd := chdir("bootstrap")
	defer chdir(wd)

	command("go", "", "", "build")

	return false
}

func peg0() bool {
	if done("cmd/peg-bootstrap/peg0", "cmd/peg-bootstrap/main.go", bootstrap) {
		return true
	}

	wd := chdir("cmd/peg-bootstrap/")
	defer chdir(wd)

	deleteFilesWithSuffix(".peg.go")
	command("../../bootstrap/bootstrap", "", "")
	command("go", "", "", "build", "-o", "peg0")

	return false
}

func peg1() bool {
	if done("cmd/peg-bootstrap/peg1", peg0, "cmd/peg-bootstrap/bootstrap.peg") {
		return true
	}

	wd := chdir("cmd/peg-bootstrap/")
	defer chdir(wd)

	deleteFilesWithSuffix(".peg.go")
	command("./peg0", "bootstrap.peg", "peg1.peg.go")
	command("go", "", "", "build", "-o", "peg1")

	return false
}

func peg2() bool {
	if done("cmd/peg-bootstrap/peg2", peg1, "cmd/peg-bootstrap/peg.bootstrap.peg") {
		return true
	}

	wd := chdir("cmd/peg-bootstrap/")
	defer chdir(wd)

	deleteFilesWithSuffix(".peg.go")
	command("./peg1", "peg.bootstrap.peg", "peg2.peg.go")
	command("go", "", "", "build", "-o", "peg2")

	return false
}

func peg3() bool {
	if done("cmd/peg-bootstrap/peg3", peg2, "peg.peg") {
		return true
	}

	wd := chdir("cmd/peg-bootstrap/")
	defer chdir(wd)

	deleteFilesWithSuffix(".peg.go")
	command("./peg2", "../../peg.peg", "peg3.peg.go")
	command("go", "", "", "build", "-o", "peg3")

	return false
}

func peg_bootstrap() bool {
	if done("cmd/peg-bootstrap/peg-bootstrap", peg3) {
		return true
	}

	wd := chdir("cmd/peg-bootstrap/")
	defer chdir(wd)

	deleteFilesWithSuffix(".peg.go")
	command("./peg3", "../../peg.peg", "peg-bootstrap.peg.go")
	command("go", "", "", "build", "-o", "peg-bootstrap")

	return false
}

func peg_peg_go() bool {
	if done("peg.peg.go", peg_bootstrap) {
		return true
	}

	command("cmd/peg-bootstrap/peg-bootstrap", "peg.peg", "peg.peg.go")
	command("go", "", "", "build")
	command("./peg", "", "", "-inline", "-switch", "peg.peg")

	return false
}

func peg() bool {
	if done("peg", peg_peg_go, "main.go") {
		return true
	}

	command("go", "", "", "build")

	return false
}

func clean() bool {
	delete("bootstrap/bootstrap")

	wd := chdir("cmd/peg-bootstrap/")
	defer chdir(wd)

	deleteFilesWithSuffix(".peg.go")
	delete("peg0")
	delete("peg1")
	delete("peg2")
	delete("peg3")
	delete("peg-bootstrap")

	return false
}

func test() bool {
	peg()

	command("go", "", "", "test")

	return false
}

func bench() bool {
	peg()

	command("go", "", "", "test", "-benchmem", "-bench", ".")

	return false
}
<commit_msg>Support windows<commit_after>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build ignore

package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"reflect"
	"runtime"
	"strings"
)

func main() {
	flag.Parse()

	args, target := flag.Args(), "peg"
	if len(args) > 0 {
		target = args[0]
	}

	switch target {
	case "peg":
		peg()
	case "clean":
		clean()
	case "test":
		test()
	case "bench":
		bench()
	}
}

var processed = make(map[string]bool)

func done(file string, deps ...interface{}) bool {
	fini := true
	file = filepath.FromSlash(file)
	info, err := os.Stat(file)
	if err != nil {
		fini = false
	}
	for _, dep := range deps {
		switch dep := dep.(type) {
		case string:
			if info == nil {
				fini = false
				break
			}
			dep = filepath.FromSlash(dep)
			fileInfo, err := os.Stat(dep)
			if err != nil {
				panic(err)
			}

			if fileInfo.ModTime().After(info.ModTime()) {
				fini = false
			}
		case func() bool:
			name := runtime.FuncForPC(reflect.ValueOf(dep).Pointer()).Name()
			if processed[name] {
				fmt.Printf("%s is done\n", name)
				break
			}
			result := dep()
			fini = fini && result
			fmt.Printf("%s\n", name)
			processed[name] = true
		}
	}

	return fini
}

func chdir(dir string) string {
	dir = filepath.FromSlash(dir)
	working, err := os.Getwd()
	if err != nil {
		panic(err)
	}
	err = os.Chdir(dir)
	if err != nil {
		panic(err)
	}
	fmt.Printf("cd %s\n", dir)
	return working
}

func command(name, inputFile, outputFile string, arg ...string) {
	name = filepath.FromSlash(name)
	inputFile = filepath.FromSlash(inputFile)
	outputFile = filepath.FromSlash(outputFile)
	fmt.Print(name)
	for _, a := range arg {
		fmt.Printf(" %s", a)
	}

	cmd := exec.Command(name, arg...)

	if inputFile != "" {
		fmt.Printf(" < %s", inputFile)
		input, err := ioutil.ReadFile(inputFile)
		if err != nil {
			panic(err)
		}
		writer, err := cmd.StdinPipe()
		if err != nil {
			panic(err)
		}
		go func() {
			defer writer.Close()
			_, err := writer.Write([]byte(input))
			if err != nil {
				panic(err)
			}
		}()
	}

	if outputFile != "" {
		fmt.Printf(" > %s\n", outputFile)
		output, err := cmd.Output()
		if err != nil {
			panic(err)
		}
		err = ioutil.WriteFile(outputFile, output, 0600)
		if err != nil {
			panic(err)
		}
	} else {
		output, err := cmd.CombinedOutput()
		if err != nil {
			panic(err)
		}
		fmt.Printf("\n%s", string(output))
	}
}

func delete(file string) {
	file = filepath.FromSlash(file)
	fmt.Printf("rm -f %s\n", file)
	os.Remove(file)
}

func deleteFilesWithSuffix(suffix string) {
	files, err := ioutil.ReadDir(".")
	if err != nil {
		panic(err)
	}
	for _, file := range files {
		if strings.HasSuffix(file.Name(), suffix) {
			delete(file.Name())
		}
	}
}

func bootstrap() bool {
	if done("bootstrap/bootstrap", "bootstrap/main.go", "tree/peg.go") {
		return true
	}

	wd := chdir("bootstrap")
	defer chdir(wd)

	command("go", "", "", "build")

	return false
}

func peg0() bool {
	if done("cmd/peg-bootstrap/peg0", "cmd/peg-bootstrap/main.go", bootstrap) {
		return true
	}

	wd := chdir("cmd/peg-bootstrap/")
	defer chdir(wd)

	deleteFilesWithSuffix(".peg.go")
	command("../../bootstrap/bootstrap", "", "")
	command("go", "", "", "build", "-o", "peg0")

	return false
}

func peg1() bool {
	if done("cmd/peg-bootstrap/peg1", peg0, "cmd/peg-bootstrap/bootstrap.peg") {
		return true
	}

	wd := chdir("cmd/peg-bootstrap/")
	defer chdir(wd)

	deleteFilesWithSuffix(".peg.go")
	command("./peg0", "bootstrap.peg", "peg1.peg.go")
	command("go", "", "", "build", "-o", "peg1")

	return false
}

func peg2() bool {
	if done("cmd/peg-bootstrap/peg2", peg1, "cmd/peg-bootstrap/peg.bootstrap.peg") {
		return true
	}

	wd := chdir("cmd/peg-bootstrap/")
	defer chdir(wd)

	deleteFilesWithSuffix(".peg.go")
	command("./peg1", "peg.bootstrap.peg", "peg2.peg.go")
	command("go", "", "", "build", "-o", "peg2")

	return false
}

func peg3() bool {
	if done("cmd/peg-bootstrap/peg3", peg2, "peg.peg") {
		return true
	}

	wd := chdir("cmd/peg-bootstrap/")
	defer chdir(wd)

	deleteFilesWithSuffix(".peg.go")
	command("./peg2", "../../peg.peg", "peg3.peg.go")
	command("go", "", "", "build", "-o", "peg3")

	return false
}

func peg_bootstrap() bool {
	if done("cmd/peg-bootstrap/peg-bootstrap", peg3) {
		return true
	}

	wd := chdir("cmd/peg-bootstrap/")
	defer chdir(wd)

	deleteFilesWithSuffix(".peg.go")
	command("./peg3", "../../peg.peg", "peg-bootstrap.peg.go")
	command("go", "", "", "build", "-o", "peg-bootstrap")

	return false
}

func peg_peg_go() bool {
	if done("peg.peg.go", peg_bootstrap) {
		return true
	}

	command("cmd/peg-bootstrap/peg-bootstrap", "peg.peg", "peg.peg.go")
	command("go", "", "", "build")
	command("./peg", "", "", "-inline", "-switch", "peg.peg")

	return false
}

func peg() bool {
	if done("peg", peg_peg_go, "main.go") {
		return true
	}

	command("go", "", "", "build")

	return false
}

func clean() bool {
	delete("bootstrap/bootstrap")

	wd := chdir("cmd/peg-bootstrap/")
	defer chdir(wd)

	deleteFilesWithSuffix(".peg.go")
	delete("peg0")
	delete("peg1")
	delete("peg2")
	delete("peg3")
	delete("peg-bootstrap")

	return false
}

func test() bool {
	peg()

	command("go", "", "", "test")

	return false
}

func bench() bool {
	peg()

	command("go", "", "", "test", "-benchmem", "-bench", ".")

	return false
}
<|endoftext|>


<commit_before>package utils

import (
	"fmt"
	"strings"
)

func Reverse(words []string, phrasing Phrasing) map[string]bool {
	if len(words) > 1 {
		panic("Word must be [1]string: " + fmt.Sprint(words))
	}
	word := strings.ToLower(words[0])
	l := len(word)
	ans := make([]rune, l)
	for i, c := range word {
		ans[l-i-1] = c
	}
	return map[string]bool{string(ans): true}
}

// filter out results with more than one bigram violation. We might allow strings with a bigram violation since they could have letters inserted into them later
func bigram_filter(answers map[string]bool, lengths []int, threshold int) map[string]bool {
	var violations int
	var pass bool
	threshold += len(lengths) - 1 // allow violations across word boundaries

	for ans := range answers {
		violations = 0
		for i := 0; i < len(ans)-1; i++ {
			pass = false
			for _, l := range lengths {
				if NGRAMS[l][ans[i:i+2]] {
					pass = true
					break
				}
			}
			if !pass {
				violations++
				if violations > threshold {
					delete(answers, ans)
					break
				}
			}
		}
	}
	return answers
}

// First 1-3 letters
// last letter
// outside two letters
// inside 1 or 2 letters
// all but first
// all but last
// all but center
// all but edges
func AllLegalSubstrings(words []string, phrasing Phrasing) map[string]bool {
	if len(words) != 1 {
		panic("Word must be [1]string")
	}
	length := Sum(phrasing.Lengths)
	subs := map[string]bool{}
	word := strings.ToLower(words[0])
	if len(word) <= 1 {
		return subs
	}
	if strings.Contains(word, "_") {
		word = strings.Replace(word, "_", "", -1)
	}
	for i := 0; i < len(word)-length+1; i++ {
		s := word[i : i+length]
		if _, ok := (SYNONYMS)[s]; ok {
			subs[s] = true
		}
	}
	for l := 1; l <= min(len(word)-1, length, 3); l++ {
		subs[word[:l]] = true // first l letters
	}
	subs[word[len(word)-1:]] = true // last letter
	if len(word) > 2 {
		subs[word[:1]+word[len(word)-1:]] = true // outside
		if len(word)%2 == 0 {
			subs[word[len(word)/2-1:len(word)/2+1]] = true         // center
			subs[word[:len(word)/2-1]+word[len(word)/2+1:]] = true // all but center
		} else {
			subs[word[len(word)/2:len(word)/2+1]] = true         // center
			subs[word[:len(word)/2]+word[len(word)/2+1:]] = true // all but center
		}
		subs[word[1:]] = true            // all but first
		subs[word[:len(word)-1]] = true  // all but last
		subs[word[1:len(word)-1]] = true // all but edges
	}
	return bigram_filter(subs, phrasing.Lengths, 1)
}

func min(x ...int) int {
	result := x[0]
	for _, y := range x[1:] {
		if y < result {
			result = y
		}
	}
	return result
}

func AllInsertions(words []string, phrasing Phrasing) map[string]bool {
	if len(words) != 2 {
		panic(fmt.Sprintf("Got wrong number of words. Expected 2, got %d", len(words)))
	}
	word1 := strings.Replace(words[0], "_", "", -1)
	word2 := strings.Replace(words[1], "_", "", -1)
	result := map[string]bool{}
	if word1 == "" || word2 == "" || len(word1)+len(word2) > Sum(phrasing.Lengths) {
		return map[string]bool{}
	}
	// if len(words[0])+len(words[1]) > Sum(phrasing.Lengths) {
	// 	return map[string]bool{}
	// }
	w0, w1 := word1, word2
	for j := 1; j < len(w1); j++ {
		result[w1[0:j]+w0+w1[j:]] = true
	}
	w1, w0 = word1, word2
	for j := 0; j < len(w1); j++ {
		result[w1[0:j]+w0+w1[j:]] = true
	}
	return bigram_filter(result, phrasing.Lengths, 0)
}
<commit_msg>Don't take word substrings which are just the entire input word<commit_after>package utils

import (
	"fmt"
	"strings"
)

func Reverse(words []string, phrasing Phrasing) map[string]bool {
	if len(words) > 1 {
		panic("Word must be [1]string: " + fmt.Sprint(words))
	}
	word := strings.ToLower(words[0])
	l := len(word)
	ans := make([]rune, l)
	for i, c := range word {
		ans[l-i-1] = c
	}
	return map[string]bool{string(ans): true}
}

// filter out results with more than one bigram violation. We might allow strings with a bigram violation since they could have letters inserted into them later
func bigram_filter(answers map[string]bool, lengths []int, threshold int) map[string]bool {
	var violations int
	var pass bool
	threshold += len(lengths) - 1 // allow violations across word boundaries

	for ans := range answers {
		violations = 0
		for i := 0; i < len(ans)-1; i++ {
			pass = false
			for _, l := range lengths {
				if NGRAMS[l][ans[i:i+2]] {
					pass = true
					break
				}
			}
			if !pass {
				violations++
				if violations > threshold {
					delete(answers, ans)
					break
				}
			}
		}
	}
	return answers
}

// First 1-3 letters
// last letter
// outside two letters
// inside 1 or 2 letters
// all but first
// all but last
// all but center
// all but edges
func AllLegalSubstrings(words []string, phrasing Phrasing) map[string]bool {
	if len(words) != 1 {
		panic("Word must be [1]string")
	}
	length := Sum(phrasing.Lengths)
	subs := map[string]bool{}
	word := strings.ToLower(words[0])
	if len(word) <= 1 {
		return subs
	}
	if strings.Contains(word, "_") {
		word = strings.Replace(word, "_", "", -1)
	}
	if len(word) > length {
		for i := 0; i < len(word)-length+1; i++ {
			s := word[i : i+length]
			if _, ok := (SYNONYMS)[s]; ok {
				subs[s] = true
			}
		}
	}
	for l := 1; l <= min(len(word)-1, length, 3); l++ {
		subs[word[:l]] = true // first l letters
	}
	subs[word[len(word)-1:]] = true // last letter
	if len(word) > 2 {
		subs[word[:1]+word[len(word)-1:]] = true // outside
		if len(word)%2 == 0 {
			subs[word[len(word)/2-1:len(word)/2+1]] = true         // center
			subs[word[:len(word)/2-1]+word[len(word)/2+1:]] = true // all but center
		} else {
			subs[word[len(word)/2:len(word)/2+1]] = true         // center
			subs[word[:len(word)/2]+word[len(word)/2+1:]] = true // all but center
		}
		subs[word[1:]] = true            // all but first
		subs[word[:len(word)-1]] = true  // all but last
		subs[word[1:len(word)-1]] = true // all but edges
	}
	return bigram_filter(subs, phrasing.Lengths, 1)
}

func min(x ...int) int {
	result := x[0]
	for _, y := range x[1:] {
		if y < result {
			result = y
		}
	}
	return result
}

func AllInsertions(words []string, phrasing Phrasing) map[string]bool {
	if len(words) != 2 {
		panic(fmt.Sprintf("Got wrong number of words. Expected 2, got %d", len(words)))
	}
	word1 := strings.Replace(words[0], "_", "", -1)
	word2 := strings.Replace(words[1], "_", "", -1)
	result := map[string]bool{}
	if word1 == "" || word2 == "" || len(word1)+len(word2) > Sum(phrasing.Lengths) {
		return map[string]bool{}
	}
	// if len(words[0])+len(words[1]) > Sum(phrasing.Lengths) {
	// 	return map[string]bool{}
	// }
	w0, w1 := word1, word2
	for j := 1; j < len(w1); j++ {
		result[w1[0:j]+w0+w1[j:]] = true
	}
	w1, w0 = word1, word2
	for j := 0; j < len(w1); j++ {
		result[w1[0:j]+w0+w1[j:]] = true
	}
	return bigram_filter(result, phrasing.Lengths, 0)
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package vtexplain analyzes a set of sql statements and returns the
// corresponding vtgate and vttablet query plans that will be executed
// on the given statements
package vtexplain

import (
	"context"
	"fmt"

	"vitess.io/vitess/go/cache"
	"vitess.io/vitess/go/vt/topo"
	"vitess.io/vitess/go/vt/topo/memorytopo"

	"vitess.io/vitess/go/vt/vterrors"

	"vitess.io/vitess/go/json2"
	"vitess.io/vitess/go/vt/discovery"
	"vitess.io/vitess/go/vt/key"
	"vitess.io/vitess/go/vt/log"
	"vitess.io/vitess/go/vt/srvtopo"
	"vitess.io/vitess/go/vt/vtgate"
	"vitess.io/vitess/go/vt/vtgate/engine"
	"vitess.io/vitess/go/vt/vttablet/queryservice"

	querypb "vitess.io/vitess/go/vt/proto/query"
	topodatapb "vitess.io/vitess/go/vt/proto/topodata"
	vschemapb "vitess.io/vitess/go/vt/proto/vschema"
	vtgatepb "vitess.io/vitess/go/vt/proto/vtgate"
)

var (
	explainTopo    *ExplainTopo
	vtgateExecutor *vtgate.Executor
	healthCheck    *discovery.FakeHealthCheck

	vtgateSession = &vtgatepb.Session{
		TargetString: "",
		Autocommit:   true,
	}
)

func initVtgateExecutor(vSchemaStr, ksShardMapStr string, opts *Options) error {
	explainTopo = &ExplainTopo{NumShards: opts.NumShards}
	explainTopo.TopoServer = memorytopo.NewServer(vtexplainCell)
	healthCheck = discovery.NewFakeHealthCheck(nil)

	resolver := newFakeResolver(opts, explainTopo, vtexplainCell)

	err := buildTopology(opts, vSchemaStr, ksShardMapStr, opts.NumShards)
	if err != nil {
		return err
	}

	vtgateSession.TargetString = opts.Target

	if opts.PlannerVersion != querypb.ExecuteOptions_DEFAULT_PLANNER {
		if vtgateSession.Options == nil {
			vtgateSession.Options = &querypb.ExecuteOptions{}
		}
		vtgateSession.Options.PlannerVersion = opts.PlannerVersion
	}

	streamSize := 10
	var schemaTracker vtgate.SchemaInfo // no schema tracker for these tests
	vtgateExecutor = vtgate.NewExecutor(context.Background(), explainTopo, vtexplainCell, resolver, opts.Normalize, false /*do not warn for sharded only*/, streamSize, cache.DefaultConfig, schemaTracker, false /*no-scatter*/)

	return nil
}

func newFakeResolver(opts *Options, serv srvtopo.Server, cell string) *vtgate.Resolver {
	ctx := context.Background()
	gw := vtgate.NewTabletGateway(ctx, healthCheck, serv, cell)
	_ = gw.WaitForTablets([]topodatapb.TabletType{topodatapb.TabletType_REPLICA})

	txMode := vtgatepb.TransactionMode_MULTI
	if opts.ExecutionMode == ModeTwoPC {
		txMode = vtgatepb.TransactionMode_TWOPC
	}
	tc := vtgate.NewTxConn(gw, txMode)
	sc := vtgate.NewScatterConn("", tc, gw)
	srvResolver := srvtopo.NewResolver(serv, gw, cell)
	return vtgate.NewResolver(srvResolver, serv, cell, sc)
}

func buildTopology(opts *Options, vschemaStr string, ksShardMapStr string, numShardsPerKeyspace int) error {
	explainTopo.Lock.Lock()
	defer explainTopo.Lock.Unlock()

	// We have to use proto's custom json loader so it can
	// handle string->enum conversion correctly.
	var srvVSchema vschemapb.SrvVSchema
	wrappedStr := fmt.Sprintf(`{"keyspaces": %s}`, vschemaStr)
	err := json2.Unmarshal([]byte(wrappedStr), &srvVSchema)
	if err != nil {
		return err
	}
	explainTopo.Keyspaces = srvVSchema.Keyspaces

	ksShardMap, err := getKeyspaceShardMap(ksShardMapStr)
	if err != nil {
		return err
	}

	explainTopo.TabletConns = make(map[string]*explainTablet)
	explainTopo.KeyspaceShards = make(map[string]map[string]*topodatapb.ShardReference)
	for ks, vschema := range explainTopo.Keyspaces {
		shards, err := getShardRanges(ks, vschema, ksShardMap, numShardsPerKeyspace)
		if err != nil {
			return err
		}

		explainTopo.KeyspaceShards[ks] = make(map[string]*topodatapb.ShardReference)

		for _, shard := range shards {
			hostname := fmt.Sprintf("%s/%s", ks, shard.Name)
			log.Infof("registering test tablet %s for keyspace %s shard %s", hostname, ks, shard.Name)

			tablet := healthCheck.AddFakeTablet(vtexplainCell, hostname, 1, ks, shard.Name, topodatapb.TabletType_PRIMARY, true, 1, nil, func(t *topodatapb.Tablet) queryservice.QueryService {
				return newTablet(opts, t)
			})
			explainTopo.TabletConns[hostname] = tablet.(*explainTablet)
			explainTopo.KeyspaceShards[ks][shard.Name] = shard
		}
	}

	return err
}

func getKeyspaceShardMap(ksShardMapStr string) (map[string]map[string]*topo.ShardInfo, error) {
	if ksShardMapStr == "" {
		return map[string]map[string]*topo.ShardInfo{}, nil
	}

	// keyspace-name -> shard-name -> ShardInfo
	var ksShardMap map[string]map[string]*topo.ShardInfo
	err := json2.Unmarshal([]byte(ksShardMapStr), &ksShardMap)

	return ksShardMap, err
}

func getShardRanges(ks string, vschema *vschemapb.Keyspace, ksShardMap map[string]map[string]*topo.ShardInfo, numShardsPerKeyspace int) ([]*topodatapb.ShardReference, error) {
	shardMap, ok := ksShardMap[ks]
	if ok {
		shards := make([]*topodatapb.ShardReference, 0, len(shardMap))
		for shard, info := range shardMap {
			ref := &topodatapb.ShardReference{
				Name:     shard,
				KeyRange: info.KeyRange,
			}

			shards = append(shards, ref)
		}
		return shards, nil

	}

	numShards := 1
	if vschema.Sharded {
		numShards = numShardsPerKeyspace
	}

	shards := make([]*topodatapb.ShardReference, numShards)

	for i := 0; i < numShards; i++ {
		kr, err := key.EvenShardsKeyRange(i, numShards)
		if err != nil {
			return nil, err
		}

		shards[i] = &topodatapb.ShardReference{
			Name:     key.KeyRangeString(kr),
			KeyRange: kr,
		}
	}

	return shards, nil
}

func vtgateExecute(sql string) ([]*engine.Plan, map[string]*TabletActions, error) {
	// use the plan cache to get the set of plans used for this query, then
	// clear afterwards for the next run
	planCache := vtgateExecutor.Plans()

	_, err := vtgateExecutor.Execute(context.Background(), "VtexplainExecute", vtgate.NewSafeSession(vtgateSession), sql, nil)
	if err != nil {
		for _, tc := range explainTopo.TabletConns {
			tc.tabletQueries = nil
			tc.mysqlQueries = nil
		}
		planCache.Clear()

		return nil, nil, vterrors.Wrapf(err, "vtexplain execute error in '%s'", sql)
	}

	var plans []*engine.Plan
	planCache.ForEach(func(value any) bool {
		plan := value.(*engine.Plan)
		plan.ExecTime = 0
		plans = append(plans, plan)
		return true
	})
	planCache.Clear()

	tabletActions := make(map[string]*TabletActions)
	for shard, tc := range explainTopo.TabletConns {
		if len(tc.tabletQueries) == 0 {
			continue
		}

		func() {
			tc.mu.Lock()
			defer tc.mu.Unlock()

			tabletActions[shard] = &TabletActions{
				TabletQueries: tc.tabletQueries,
				MysqlQueries:  tc.mysqlQueries,
			}

			tc.tabletQueries = nil
			tc.mysqlQueries = nil
		}()
	}

	return plans, tabletActions, nil
}
<commit_msg>fix: vtexplain commit order flaky (#10264)<commit_after>/*
Copyright 2019 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package vtexplain analyzes a set of sql statements and returns the
// corresponding vtgate and vttablet query plans that will be executed
// on the given statements
package vtexplain

import (
	"context"
	"fmt"
	"sort"
	"strings"

	"vitess.io/vitess/go/cache"
	"vitess.io/vitess/go/vt/topo"
	"vitess.io/vitess/go/vt/topo/memorytopo"

	"vitess.io/vitess/go/vt/vterrors"

	"vitess.io/vitess/go/json2"
	"vitess.io/vitess/go/vt/discovery"
	"vitess.io/vitess/go/vt/key"
	"vitess.io/vitess/go/vt/log"
	"vitess.io/vitess/go/vt/srvtopo"
	"vitess.io/vitess/go/vt/vtgate"
	"vitess.io/vitess/go/vt/vtgate/engine"
	"vitess.io/vitess/go/vt/vttablet/queryservice"

	querypb "vitess.io/vitess/go/vt/proto/query"
	topodatapb "vitess.io/vitess/go/vt/proto/topodata"
	vschemapb "vitess.io/vitess/go/vt/proto/vschema"
	vtgatepb "vitess.io/vitess/go/vt/proto/vtgate"
)

var (
	explainTopo    *ExplainTopo
	vtgateExecutor *vtgate.Executor
	healthCheck    *discovery.FakeHealthCheck

	vtgateSession = &vtgatepb.Session{
		TargetString: "",
		Autocommit:   true,
	}
)

func initVtgateExecutor(vSchemaStr, ksShardMapStr string, opts *Options) error {
	explainTopo = &ExplainTopo{NumShards: opts.NumShards}
	explainTopo.TopoServer = memorytopo.NewServer(vtexplainCell)
	healthCheck = discovery.NewFakeHealthCheck(nil)

	resolver := newFakeResolver(opts, explainTopo, vtexplainCell)

	err := buildTopology(opts, vSchemaStr, ksShardMapStr, opts.NumShards)
	if err != nil {
		return err
	}

	vtgateSession.TargetString = opts.Target

	if opts.PlannerVersion != querypb.ExecuteOptions_DEFAULT_PLANNER {
		if vtgateSession.Options == nil {
			vtgateSession.Options = &querypb.ExecuteOptions{}
		}
		vtgateSession.Options.PlannerVersion = opts.PlannerVersion
	}

	streamSize := 10
	var schemaTracker vtgate.SchemaInfo // no schema tracker for these tests
	vtgateExecutor = vtgate.NewExecutor(context.Background(), explainTopo, vtexplainCell, resolver, opts.Normalize, false /*do not warn for sharded only*/, streamSize, cache.DefaultConfig, schemaTracker, false /*no-scatter*/)

	return nil
}

func newFakeResolver(opts *Options, serv srvtopo.Server, cell string) *vtgate.Resolver {
	ctx := context.Background()
	gw := vtgate.NewTabletGateway(ctx, healthCheck, serv, cell)
	_ = gw.WaitForTablets([]topodatapb.TabletType{topodatapb.TabletType_REPLICA})

	txMode := vtgatepb.TransactionMode_MULTI
	if opts.ExecutionMode == ModeTwoPC {
		txMode = vtgatepb.TransactionMode_TWOPC
	}
	tc := vtgate.NewTxConn(gw, txMode)
	sc := vtgate.NewScatterConn("", tc, gw)
	srvResolver := srvtopo.NewResolver(serv, gw, cell)
	return vtgate.NewResolver(srvResolver, serv, cell, sc)
}

func buildTopology(opts *Options, vschemaStr string, ksShardMapStr string, numShardsPerKeyspace int) error {
	explainTopo.Lock.Lock()
	defer explainTopo.Lock.Unlock()

	// We have to use proto's custom json loader so it can
	// handle string->enum conversion correctly.
	var srvVSchema vschemapb.SrvVSchema
	wrappedStr := fmt.Sprintf(`{"keyspaces": %s}`, vschemaStr)
	err := json2.Unmarshal([]byte(wrappedStr), &srvVSchema)
	if err != nil {
		return err
	}
	explainTopo.Keyspaces = srvVSchema.Keyspaces

	ksShardMap, err := getKeyspaceShardMap(ksShardMapStr)
	if err != nil {
		return err
	}

	explainTopo.TabletConns = make(map[string]*explainTablet)
	explainTopo.KeyspaceShards = make(map[string]map[string]*topodatapb.ShardReference)
	for ks, vschema := range explainTopo.Keyspaces {
		shards, err := getShardRanges(ks, vschema, ksShardMap, numShardsPerKeyspace)
		if err != nil {
			return err
		}

		explainTopo.KeyspaceShards[ks] = make(map[string]*topodatapb.ShardReference)

		for _, shard := range shards {
			hostname := fmt.Sprintf("%s/%s", ks, shard.Name)
			log.Infof("registering test tablet %s for keyspace %s shard %s", hostname, ks, shard.Name)

			tablet := healthCheck.AddFakeTablet(vtexplainCell, hostname, 1, ks, shard.Name, topodatapb.TabletType_PRIMARY, true, 1, nil, func(t *topodatapb.Tablet) queryservice.QueryService {
				return newTablet(opts, t)
			})
			explainTopo.TabletConns[hostname] = tablet.(*explainTablet)
			explainTopo.KeyspaceShards[ks][shard.Name] = shard
		}
	}

	return err
}

func getKeyspaceShardMap(ksShardMapStr string) (map[string]map[string]*topo.ShardInfo, error) {
	if ksShardMapStr == "" {
		return map[string]map[string]*topo.ShardInfo{}, nil
	}

	// keyspace-name -> shard-name -> ShardInfo
	var ksShardMap map[string]map[string]*topo.ShardInfo
	err := json2.Unmarshal([]byte(ksShardMapStr), &ksShardMap)

	return ksShardMap, err
}

func getShardRanges(ks string, vschema *vschemapb.Keyspace, ksShardMap map[string]map[string]*topo.ShardInfo, numShardsPerKeyspace int) ([]*topodatapb.ShardReference, error) {
	shardMap, ok := ksShardMap[ks]
	if ok {
		shards := make([]*topodatapb.ShardReference, 0, len(shardMap))
		for shard, info := range shardMap {
			ref := &topodatapb.ShardReference{
				Name:     shard,
				KeyRange: info.KeyRange,
			}

			shards = append(shards, ref)
		}
		return shards, nil

	}

	numShards := 1
	if vschema.Sharded {
		numShards = numShardsPerKeyspace
	}

	shards := make([]*topodatapb.ShardReference, numShards)

	for i := 0; i < numShards; i++ {
		kr, err := key.EvenShardsKeyRange(i, numShards)
		if err != nil {
			return nil, err
		}

		shards[i] = &topodatapb.ShardReference{
			Name:     key.KeyRangeString(kr),
			KeyRange: kr,
		}
	}

	return shards, nil
}

func vtgateExecute(sql string) ([]*engine.Plan, map[string]*TabletActions, error) {
	// This method will sort the shard session lexicographically.
	// This will ensure that the commit/rollback order is predictable.
	sortShardSession()

	// use the plan cache to get the set of plans used for this query, then
	// clear afterwards for the next run
	planCache := vtgateExecutor.Plans()

	_, err := vtgateExecutor.Execute(context.Background(), "VtexplainExecute", vtgate.NewSafeSession(vtgateSession), sql, nil)
	if err != nil {
		for _, tc := range explainTopo.TabletConns {
			tc.tabletQueries = nil
			tc.mysqlQueries = nil
		}
		planCache.Clear()

		return nil, nil, vterrors.Wrapf(err, "vtexplain execute error in '%s'", sql)
	}

	var plans []*engine.Plan
	planCache.ForEach(func(value any) bool {
		plan := value.(*engine.Plan)
		plan.ExecTime = 0
		plans = append(plans, plan)
		return true
	})
	planCache.Clear()

	tabletActions := make(map[string]*TabletActions)
	for shard, tc := range explainTopo.TabletConns {
		if len(tc.tabletQueries) == 0 {
			continue
		}

		func() {
			tc.mu.Lock()
			defer tc.mu.Unlock()

			tabletActions[shard] = &TabletActions{
				TabletQueries: tc.tabletQueries,
				MysqlQueries:  tc.mysqlQueries,
			}

			tc.tabletQueries = nil
			tc.mysqlQueries = nil
		}()
	}

	return plans, tabletActions, nil
}

func sortShardSession() {
	if len(vtgateSession.ShardSessions) > 1 {
		ss := vtgateSession.ShardSessions
		sort.Slice(ss, func(i, j int) bool {
			if ss[i].Target.Keyspace != ss[j].Target.Keyspace {
				return strings.Compare(ss[i].Target.Keyspace, ss[j].Target.Keyspace) <= 0
			}
			return strings.Compare(ss[i].Target.Shard, ss[j].Target.Shard) <= 0
		})
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"strings"
	"sync"
	"syscall"
)

var wg sync.WaitGroup

func main() {
	fmt.Println("running child")
	args := strings.Split(os.Args[1], " ")
	args = append(args, os.Args[2:]...)
	fmt.Println(syscall.Exec(args[0], args, os.Environ()))
}
<commit_msg>If program exits, unmount /app<commit_after>package main

import (
	"fmt"
	"os"
	"strings"
	"sync"
	"syscall"
)

var wg sync.WaitGroup

func main() {
	fmt.Println("running child")
	args := strings.Split(os.Args[1], " ")
	args = append(args, os.Args[2:]...)
	fmt.Println(syscall.Exec(args[0], args, os.Environ()))

	if err := syscall.Unmount("/app", 0); err != nil {
		fmt.Println("Cannot unmount /app", err)
	}
}
<|endoftext|>


<commit_before>package sftp

// sftp integration tests
// enable with -integration

import (
	"flag"
	"io/ioutil"
	"os"
	"os/exec"
	"testing"
)

const (
	READONLY  = true
	READWRITE = false
)

var testIntegration = flag.Bool("integration", false, "perform integration tests against sftp server process")

// testClient returns a *Client connected to a localy running sftp-server
// the *exec.Cmd returned must be defer Wait'd.
func testClient(t *testing.T, readonly bool) (*Client, *exec.Cmd) {
	cmd := exec.Command("/usr/lib/openssh/sftp-server", "-e", "-R") // log to stderr, read only
	if !readonly {
		cmd = exec.Command("/usr/lib/openssh/sftp-server", "-e") // log to stderr
	}
	cmd.Stderr = os.Stdout
	pw, err := cmd.StdinPipe()
	if err != nil {
		t.Fatal(err)
	}
	pr, err := cmd.StdoutPipe()
	if err != nil {
		t.Fatal(err)
	}
	if err := cmd.Start(); err != nil {
		t.Skipf("could not start sftp-server process: %v", err)
	}
	sftp := &Client{
		w: pw,
		r: pr,
	}
	if err := sftp.sendInit(); err != nil {
		defer cmd.Wait()
		t.Fatal(err)
	}
	if err := sftp.recvVersion(); err != nil {
		defer cmd.Wait()
		t.Fatal(err)
	}
	return sftp, cmd
}

func TestNewClient(t *testing.T) {
	sftp, cmd := testClient(t, READONLY)
	defer cmd.Wait()

	if err := sftp.Close(); err != nil {
		t.Fatal(err)
	}
}

func TestClientLstat(t *testing.T) {
	sftp, cmd := testClient(t, READONLY)
	defer cmd.Wait()
	defer sftp.Close()

	f, err := ioutil.TempFile("", "sftptest")
	if err != nil {
		t.Fatal(err)
	}
	defer os.Remove(f.Name())

	want, err := os.Lstat(f.Name())
	if err != nil {
		t.Fatal(err)
	}

	got, err := sftp.Lstat(f.Name())
	if err != nil {
		t.Fatal(err)
	}

	if !sameFile(want, got) {
		t.Fatalf("Lstat(%q): want %#v, got %#v", f.Name(), want, got)
	}
}

func TestClientLstatiMissing(t *testing.T) {
	sftp, cmd := testClient(t, READONLY)
	defer cmd.Wait()
	defer sftp.Close()

	f, err := ioutil.TempFile("", "sftptest")
	if err != nil {
		t.Fatal(err)
	}
	os.Remove(f.Name())

	_, err = sftp.Lstat(f.Name())
	if err1, ok := err.(*StatusError); !ok || err1.Code != ssh_FX_NO_SUCH_FILE {
		t.Fatalf("Lstat: want: %v, got %#v", ssh_FX_NO_SUCH_FILE, err)
	}
}

func TestClientOpen(t *testing.T) {
	sftp, cmd := testClient(t, READONLY)
	defer cmd.Wait()
	defer sftp.Close()

	f, err := ioutil.TempFile("", "sftptest")
	if err != nil {
		t.Fatal(err)
	}
	defer os.Remove(f.Name())

	got, err := sftp.Open(f.Name())
	if err != nil {
		t.Fatal(err)
	}
	if err := got.Close(); err != nil {
		t.Fatal(err)
	}
}

func TestClientRead(t *testing.T) {
	sftp, cmd := testClient(t, READONLY)
	defer cmd.Wait()
	defer sftp.Close()

	f, err := ioutil.TempFile("", "sftptest")
	if err != nil {
		t.Fatal(err)
	}
	defer os.Remove(f.Name())

	if _, err := f.WriteString("Hello world!"); err != nil {
		t.Fatal(err)
	}
	if err := f.Close(); err != nil {
		t.Fatal(err)
	}

	got, err := sftp.Open(f.Name())
	if err != nil {
		t.Fatal(err)
	}
	defer got.Close()

	b, err := ioutil.ReadAll(got)
	if err != nil {
		t.Fatal(err)
	}

	if want, got := "Hello world!", string(b); got != want {
		t.Fatalf("Read(): want %q, got %q", want, got)
	}
}

func TestClientCreate(t *testing.T) {
	sftp, cmd := testClient(t, READWRITE)
	defer cmd.Wait()
	defer sftp.Close()

	f, err := ioutil.TempFile("", "sftptest")
	if err != nil {
		t.Fatal(err)
	}
	defer f.Close()
	defer os.Remove(f.Name())

	f2, err := sftp.Create(f.Name())
	if err != nil {
		t.Fatal(err)
	}
	defer f2.Close()
}

func TestClientCreateFailed(t *testing.T) {
	sftp, cmd := testClient(t, READONLY)
	defer cmd.Wait()
	defer sftp.Close()

	f, err := ioutil.TempFile("", "sftptest")
	if err != nil {
		t.Fatal(err)
	}
	defer f.Close()
	defer os.Remove(f.Name())

	f2, err := sftp.Create(f.Name())
	if err1, ok := err.(*StatusError); !ok || err1.Code != ssh_FX_PERMISSION_DENIED {
		t.Fatalf("Create: want: %v, got %#v", ssh_FX_PERMISSION_DENIED, err)
	}
	if err == nil {
		f2.Close()
	}
}

func TestClientFileStat(t *testing.T) {
	sftp, cmd := testClient(t, READONLY)
	defer cmd.Wait()
	defer sftp.Close()

	f, err := ioutil.TempFile("", "sftptest")
	if err != nil {
		t.Fatal(err)
	}
	defer os.Remove(f.Name())

	want, err := os.Lstat(f.Name())
	if err != nil {
		t.Fatal(err)
	}

	f2, err := sftp.Open(f.Name())
	if err != nil {
		t.Fatal(err)
	}

	got, err := f2.Stat()
	if err != nil {
		t.Fatal(err)
	}

	if !sameFile(want, got) {
		t.Fatalf("Lstat(%q): want %#v, got %#v", f.Name(), want, got)
	}
}

func TestClientWalkdir(t *testing.T) {
	sftp, cmd := testClient(t, READONLY)
	defer cmd.Wait()
	defer sftp.Close()

	d, err := ioutil.TempDir("", "sftptest")
	if err != nil {
		t.Fatal(err)
	}
	defer os.Remove(d)

	w := sftp.Walk(d)
	for w.Step() {
		if err := w.Err(); err != nil {
			t.Error(err)
			continue
		}
		t.Log(w.Path())
	}
}

func sameFile(want, got os.FileInfo) bool {
	return want.Name() == got.Name() &&
		want.Size() == got.Size()
}
<commit_msg>Added Write tests<commit_after>package sftp

// sftp integration tests
// enable with -integration

import (
	"flag"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"testing"
)

const (
	READONLY  = true
	READWRITE = false

	debuglevel = "DEBUG"
)

var testIntegration = flag.Bool("integration", false, "perform integration tests against sftp server process")

// testClient returns a *Client connected to a localy running sftp-server
// the *exec.Cmd returned must be defer Wait'd.
func testClient(t *testing.T, readonly bool) (*Client, *exec.Cmd) {
	if !*testIntegration {
		t.Skip("skipping intergration test")
	}
	cmd := exec.Command("/usr/lib/openssh/sftp-server", "-e", "-R", "-l", debuglevel) // log to stderr, read only
	if !readonly {
		cmd = exec.Command("/usr/lib/openssh/sftp-server", "-e", "-l", debuglevel) // log to stderr
	}
	cmd.Stderr = os.Stdout
	pw, err := cmd.StdinPipe()
	if err != nil {
		t.Fatal(err)
	}
	pr, err := cmd.StdoutPipe()
	if err != nil {
		t.Fatal(err)
	}
	if err := cmd.Start(); err != nil {
		t.Skipf("could not start sftp-server process: %v", err)
	}
	sftp := &Client{
		w: pw,
		r: pr,
	}
	if err := sftp.sendInit(); err != nil {
		defer cmd.Wait()
		t.Fatal(err)
	}
	if err := sftp.recvVersion(); err != nil {
		defer cmd.Wait()
		t.Fatal(err)
	}
	return sftp, cmd
}

func TestNewClient(t *testing.T) {
	sftp, cmd := testClient(t, READONLY)
	defer cmd.Wait()

	if err := sftp.Close(); err != nil {
		t.Fatal(err)
	}
}

func TestClientLstat(t *testing.T) {
	sftp, cmd := testClient(t, READONLY)
	defer cmd.Wait()
	defer sftp.Close()

	f, err := ioutil.TempFile("", "sftptest")
	if err != nil {
		t.Fatal(err)
	}
	defer os.Remove(f.Name())

	want, err := os.Lstat(f.Name())
	if err != nil {
		t.Fatal(err)
	}

	got, err := sftp.Lstat(f.Name())
	if err != nil {
		t.Fatal(err)
	}

	if !sameFile(want, got) {
		t.Fatalf("Lstat(%q): want %#v, got %#v", f.Name(), want, got)
	}
}

func TestClientLstatiMissing(t *testing.T) {
	sftp, cmd := testClient(t, READONLY)
	defer cmd.Wait()
	defer sftp.Close()

	f, err := ioutil.TempFile("", "sftptest")
	if err != nil {
		t.Fatal(err)
	}
	os.Remove(f.Name())

	_, err = sftp.Lstat(f.Name())
	if err1, ok := err.(*StatusError); !ok || err1.Code != ssh_FX_NO_SUCH_FILE {
		t.Fatalf("Lstat: want: %v, got %#v", ssh_FX_NO_SUCH_FILE, err)
	}
}

func TestClientOpen(t *testing.T) {
	sftp, cmd := testClient(t, READONLY)
	defer cmd.Wait()
	defer sftp.Close()

	f, err := ioutil.TempFile("", "sftptest")
	if err != nil {
		t.Fatal(err)
	}
	defer os.Remove(f.Name())

	got, err := sftp.Open(f.Name())
	if err != nil {
		t.Fatal(err)
	}
	if err := got.Close(); err != nil {
		t.Fatal(err)
	}
}

func TestClientRead(t *testing.T) {
	sftp, cmd := testClient(t, READONLY)
	defer cmd.Wait()
	defer sftp.Close()

	f, err := ioutil.TempFile("", "sftptest")
	if err != nil {
		t.Fatal(err)
	}
	defer os.Remove(f.Name())

	if _, err := f.WriteString("Hello world!"); err != nil {
		t.Fatal(err)
	}
	if err := f.Close(); err != nil {
		t.Fatal(err)
	}

	got, err := sftp.Open(f.Name())
	if err != nil {
		t.Fatal(err)
	}
	defer got.Close()

	b, err := ioutil.ReadAll(got)
	if err != nil {
		t.Fatal(err)
	}

	if want, got := "Hello world!", string(b); got != want {
		t.Fatalf("Read(): want %q, got %q", want, got)
	}
}

func TestClientCreate(t *testing.T) {
	sftp, cmd := testClient(t, READWRITE)
	defer cmd.Wait()
	defer sftp.Close()

	f, err := ioutil.TempFile("", "sftptest")
	if err != nil {
		t.Fatal(err)
	}
	defer f.Close()
	defer os.Remove(f.Name())

	f2, err := sftp.Create(f.Name())
	if err != nil {
		t.Fatal(err)
	}
	defer f2.Close()
}

func TestClientCreateFailed(t *testing.T) {
	sftp, cmd := testClient(t, READONLY)
	defer cmd.Wait()
	defer sftp.Close()

	f, err := ioutil.TempFile("", "sftptest")
	if err != nil {
		t.Fatal(err)
	}
	defer f.Close()
	defer os.Remove(f.Name())

	f2, err := sftp.Create(f.Name())
	if err1, ok := err.(*StatusError); !ok || err1.Code != ssh_FX_PERMISSION_DENIED {
		t.Fatalf("Create: want: %v, got %#v", ssh_FX_PERMISSION_DENIED, err)
	}
	if err == nil {
		f2.Close()
	}
}

func TestClientFileStat(t *testing.T) {
	sftp, cmd := testClient(t, READONLY)
	defer cmd.Wait()
	defer sftp.Close()

	f, err := ioutil.TempFile("", "sftptest")
	if err != nil {
		t.Fatal(err)
	}
	defer os.Remove(f.Name())

	want, err := os.Lstat(f.Name())
	if err != nil {
		t.Fatal(err)
	}

	f2, err := sftp.Open(f.Name())
	if err != nil {
		t.Fatal(err)
	}

	got, err := f2.Stat()
	if err != nil {
		t.Fatal(err)
	}

	if !sameFile(want, got) {
		t.Fatalf("Lstat(%q): want %#v, got %#v", f.Name(), want, got)
	}
}

func TestClientWalkdir(t *testing.T) {
	sftp, cmd := testClient(t, READONLY)
	defer cmd.Wait()
	defer sftp.Close()

	d, err := ioutil.TempDir("", "sftptest")
	if err != nil {
		t.Fatal(err)
	}
	defer os.Remove(d)

	w := sftp.Walk(d)
	for w.Step() {
		if err := w.Err(); err != nil {
			t.Error(err)
			continue
		}
		t.Log(w.Path())
	}
}

func sameFile(want, got os.FileInfo) bool {
	return want.Name() == got.Name() &&
		want.Size() == got.Size()
}

var clientWriteTests = []struct {
	n     int
	total int64 // cumulative file size
}{
	{0, 0},
	{1, 1},
	{0, 1},
	{999, 1000},
	{24, 1024},
	{1023, 2047},
	{2048, 4095},
	{1 << 12, 8191},
	{1 << 13, 16383},
	{1 << 14, 32767},
	{1 << 15, 65535},
	{1 << 16, 131071},
	{1 << 17, 262143},
	// TODO(dfc) too large
	// 	{1 << 18, 262143},
}

func TestClientWrite(t *testing.T) {
	sftp, cmd := testClient(t, READWRITE)
	defer cmd.Wait()
	defer sftp.Close()

	d, err := ioutil.TempDir("", "sftptest")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(d)

	f := path.Join(d, "writeTest")
	w, err := sftp.Create(f)
	if err != nil {
		t.Fatal(err)
	}
	defer w.Close()

	for _, tt := range clientWriteTests {
		got, err := w.Write(make([]byte, tt.n))
		if err != nil {
			t.Fatal(err)
		}
		if got != tt.n {
			t.Errorf("Write(%v): wrote: want: %v, got %v", tt.n, tt.n, got)
		}
		fi, err := os.Stat(f)
		if err != nil {
			t.Fatal(err)
		}
		if total := fi.Size(); total != tt.total {
			t.Errorf("Write(%v): size: want: %v, got %v", tt.n, tt.total, total)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	chatwork "github.com/griffin-stewie/go-chatwork"
)

var chatworkExpectedKeys = []string{"type", "check_url", "room_id", "api_token"}

// ChatworkNotifier represents notifier for ChatWork
type ChatworkNotifier struct {
	apiToken string
	roomID   string
}

// NewChatworkNotifier create new ChatworkNotifier instance
func NewChatworkNotifier(apiToken string, roomId string) *ChatworkNotifier {
	c := new(ChatworkNotifier)
	c.apiToken = apiToken
	c.roomID = roomId
	return c
}

// ExpectedKeys returns expected keys for ChatworkNotifier
func (c *ChatworkNotifier) ExpectedKeys() []string {
	return chatworkExpectedKeys
}

// PostStatus perform posting current status for URL
func (c *ChatworkNotifier) PostStatus(param *PostStatusParam) error {
	chatwork := chatwork.NewClient(c.apiToken)

	var statusText string

	successful := IsSuccessfulStatus(param.CurrentStatusCode)

	if successful {
		statusText = "ok (F)"
	} else {
		statusText = "down (devil)"
	}

	title := fmt.Sprintf("%s is %s", param.CheckUrl, statusText)
	format := `statusCode: %d -> %d
responseTime: %f sec`
	body := fmt.Sprintf(format, param.BeforeStatusCode, param.CurrentStatusCode, param.ResponseTime)

	if param.HttpError != nil {
		body += fmt.Sprintf("\nhttpError: %v", param.HttpError)
	}

	message := fmt.Sprintf("[info][title]%s[/title]%s[/info]", title, body)

	_, err := chatwork.PostRoomMessage(c.roomID, message)

	return err
}
<commit_msg>[Resolved] func parameter roomId should be roomID<commit_after>package main

import (
	"fmt"
	chatwork "github.com/griffin-stewie/go-chatwork"
)

var chatworkExpectedKeys = []string{"type", "check_url", "room_id", "api_token"}

// ChatworkNotifier represents notifier for ChatWork
type ChatworkNotifier struct {
	apiToken string
	roomID   string
}

// NewChatworkNotifier create new ChatworkNotifier instance
func NewChatworkNotifier(apiToken string, roomID string) *ChatworkNotifier {
	c := new(ChatworkNotifier)
	c.apiToken = apiToken
	c.roomID = roomID
	return c
}

// ExpectedKeys returns expected keys for ChatworkNotifier
func (c *ChatworkNotifier) ExpectedKeys() []string {
	return chatworkExpectedKeys
}

// PostStatus perform posting current status for URL
func (c *ChatworkNotifier) PostStatus(param *PostStatusParam) error {
	chatwork := chatwork.NewClient(c.apiToken)

	var statusText string

	successful := IsSuccessfulStatus(param.CurrentStatusCode)

	if successful {
		statusText = "ok (F)"
	} else {
		statusText = "down (devil)"
	}

	title := fmt.Sprintf("%s is %s", param.CheckUrl, statusText)
	format := `statusCode: %d -> %d
responseTime: %f sec`
	body := fmt.Sprintf(format, param.BeforeStatusCode, param.CurrentStatusCode, param.ResponseTime)

	if param.HttpError != nil {
		body += fmt.Sprintf("\nhttpError: %v", param.HttpError)
	}

	message := fmt.Sprintf("[info][title]%s[/title]%s[/info]", title, body)

	_, err := chatwork.PostRoomMessage(c.roomID, message)

	return err
}
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package s3

import (
	"encoding/json"
	"errors"
	"fmt"
	"log"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/ernestio/ernestaws/credentials"
)

// Collection ....
type Collection struct {
	UUID               string            `json:"_uuid"`
	BatchID            string            `json:"_batch_id"`
	ProviderType       string            `json:"_type"`
	Service            string            `json:"service"`
	AWSAccessKeyID     string            `json:"aws_access_key_id"`
	AWSSecretAccessKey string            `json:"aws_secret_access_key"`
	DatacenterRegion   string            `json:"datacenter_region"`
	Tags               map[string]string `json:"tags"`
	Results            []interface{}     `json:"components"`
	ErrorMessage       string            `json:"error,omitempty"`
	Subject            string            `json:"-"`
	Body               []byte            `json:"-"`
	CryptoKey          string            `json:"-"`
}

// GetBody : Gets the body for this event
func (col *Collection) GetBody() []byte {
	var err error
	if col.Body, err = json.Marshal(col); err != nil {
		log.Println(err.Error())
	}
	return col.Body
}

// GetSubject : Gets the subject for this event
func (col *Collection) GetSubject() string {
	return col.Subject
}

// Process : starts processing the current message
func (col *Collection) Process() (err error) {
	if err := json.Unmarshal(col.Body, &col); err != nil {
		col.Error(err)
		return err
	}

	if err := col.Validate(); err != nil {
		col.Error(err)
		return err
	}

	return nil
}

// Error : Will respond the current event with an error
func (col *Collection) Error(err error) {
	log.Printf("Error: %s", err.Error())
	col.ErrorMessage = err.Error()

	col.Body, err = json.Marshal(col)
}

// Validate checks if all criteria are met
func (col *Collection) Validate() error {
	if col.AWSAccessKeyID == "" || col.AWSSecretAccessKey == "" {
		return ErrDatacenterCredentialsInvalid
	}

	return nil
}

// Get : Gets a object on aws
func (col *Collection) Get() error {
	return errors.New(col.Subject + " not supported")
}

// Create : Creates an object on aws
func (col *Collection) Create() error {
	return errors.New(col.Subject + " not supported")
}

// Update : Updates an object on aws
func (col *Collection) Update() error {
	return errors.New(col.Subject + " not supported")
}

// Delete : Delete an object on aws
func (col *Collection) Delete() error {
	return errors.New(col.Subject + " not supported")
}

// Find : Find s3 buckets on aws
func (col *Collection) Find() error {
	svc := col.getS3Client()

	resp, err := svc.ListBuckets(nil)
	if err != nil {
		return err
	}

	for _, b := range resp.Buckets {
		location, err := getBucketLocation(svc, b.Name)
		if err != nil {
			return err
		}

		if *location != col.DatacenterRegion {
			continue
		}

		tags, _ := getBucketTagDescriptions(svc, b.Name)

		grants, err := getBucketPermissions(svc, b.Name)
		if err != nil {
			return err
		}

		event := toEvent(b, grants, location, tags)

		if tagsMatch(col.Tags, event.Tags) {
			col.Results = append(col.Results, event)
		}
	}

	return nil
}

func (col *Collection) getS3Client() *s3.S3 {
	creds, _ := credentials.NewStaticCredentials(col.AWSAccessKeyID, col.AWSSecretAccessKey, col.CryptoKey)
	return s3.New(session.New(), &aws.Config{
		Region:      aws.String(col.DatacenterRegion),
		Credentials: creds,
	})
}

func mapS3Tags(input []*s3.Tag) map[string]string {
	t := make(map[string]string)

	for _, tag := range input {
		t[*tag.Key] = *tag.Value
	}

	return t
}

func tagsMatch(qt, rt map[string]string) bool {
	for k, v := range qt {
		if rt[k] != v {
			return false
		}
	}

	return true
}

func getBucketTagDescriptions(svc *s3.S3, name *string) ([]*s3.Tag, error) {
	treq := &s3.GetBucketTaggingInput{
		Bucket: name,
	}

	resp, err := svc.GetBucketTagging(treq)

	return resp.TagSet, err
}

func getBucketPermissions(svc *s3.S3, name *string) ([]*s3.Grant, error) {
	req := &s3.GetBucketAclInput{
		Bucket: name,
	}

	resp, err := svc.GetBucketAcl(req)

	return resp.Grants, err
}

func getBucketLocation(svc *s3.S3, name *string) (*string, error) {
	req := &s3.GetBucketLocationInput{
		Bucket: name,
	}

	resp, err := svc.GetBucketLocation(req)

	return resp.LocationConstraint, err
}

func mapS3Grantees(grantees []*s3.Grant) []Grantee {
	var gs []Grantee

	for _, g := range grantees {
		grantee := Grantee{
			ID:          g.Grantee.ID,
			Type:        g.Grantee.Type,
			Permissions: g.Permission,
		}

		switch *g.Grantee.Type {
		case "id":
			grantee.ID = g.Grantee.ID
		case "emailaddress":
			grantee.ID = g.Grantee.EmailAddress
		case "uri":
			grantee.ID = g.Grantee.URI
		}

		gs = append(gs, grantee)
	}

	return gs
}

// ToEvent converts an s3 bucket object to an ernest event
func toEvent(b *s3.Bucket, grants []*s3.Grant, location *string, tags []*s3.Tag) *Event {
	e := &Event{
		Name:           b.Name,
		Grantees:       mapS3Grantees(grants),
		BucketLocation: location,
		// ACL ???
		BucketURI: fmt.Sprintf("https://%s.s3.amazonaws.com", b.Name),
		Tags:      mapS3Tags(tags),
	}
	return e
}
<commit_msg>fixing s3 uri assignment<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package s3

import (
	"encoding/json"
	"errors"
	"fmt"
	"log"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/ernestio/ernestaws/credentials"
)

// Collection ....
type Collection struct {
	UUID               string            `json:"_uuid"`
	BatchID            string            `json:"_batch_id"`
	ProviderType       string            `json:"_type"`
	Service            string            `json:"service"`
	AWSAccessKeyID     string            `json:"aws_access_key_id"`
	AWSSecretAccessKey string            `json:"aws_secret_access_key"`
	DatacenterRegion   string            `json:"datacenter_region"`
	Tags               map[string]string `json:"tags"`
	Results            []interface{}     `json:"components"`
	ErrorMessage       string            `json:"error,omitempty"`
	Subject            string            `json:"-"`
	Body               []byte            `json:"-"`
	CryptoKey          string            `json:"-"`
}

// GetBody : Gets the body for this event
func (col *Collection) GetBody() []byte {
	var err error
	if col.Body, err = json.Marshal(col); err != nil {
		log.Println(err.Error())
	}
	return col.Body
}

// GetSubject : Gets the subject for this event
func (col *Collection) GetSubject() string {
	return col.Subject
}

// Process : starts processing the current message
func (col *Collection) Process() (err error) {
	if err := json.Unmarshal(col.Body, &col); err != nil {
		col.Error(err)
		return err
	}

	if err := col.Validate(); err != nil {
		col.Error(err)
		return err
	}

	return nil
}

// Error : Will respond the current event with an error
func (col *Collection) Error(err error) {
	log.Printf("Error: %s", err.Error())
	col.ErrorMessage = err.Error()

	col.Body, err = json.Marshal(col)
}

// Validate checks if all criteria are met
func (col *Collection) Validate() error {
	if col.AWSAccessKeyID == "" || col.AWSSecretAccessKey == "" {
		return ErrDatacenterCredentialsInvalid
	}

	return nil
}

// Get : Gets a object on aws
func (col *Collection) Get() error {
	return errors.New(col.Subject + " not supported")
}

// Create : Creates an object on aws
func (col *Collection) Create() error {
	return errors.New(col.Subject + " not supported")
}

// Update : Updates an object on aws
func (col *Collection) Update() error {
	return errors.New(col.Subject + " not supported")
}

// Delete : Delete an object on aws
func (col *Collection) Delete() error {
	return errors.New(col.Subject + " not supported")
}

// Find : Find s3 buckets on aws
func (col *Collection) Find() error {
	svc := col.getS3Client()

	resp, err := svc.ListBuckets(nil)
	if err != nil {
		return err
	}

	for _, b := range resp.Buckets {
		location, err := getBucketLocation(svc, b.Name)
		if err != nil {
			return err
		}

		if *location != col.DatacenterRegion {
			continue
		}

		tags, _ := getBucketTagDescriptions(svc, b.Name)

		grants, err := getBucketPermissions(svc, b.Name)
		if err != nil {
			return err
		}

		event := toEvent(b, grants, location, tags)

		if tagsMatch(col.Tags, event.Tags) {
			col.Results = append(col.Results, event)
		}
	}

	return nil
}

func (col *Collection) getS3Client() *s3.S3 {
	creds, _ := credentials.NewStaticCredentials(col.AWSAccessKeyID, col.AWSSecretAccessKey, col.CryptoKey)
	return s3.New(session.New(), &aws.Config{
		Region:      aws.String(col.DatacenterRegion),
		Credentials: creds,
	})
}

func mapS3Tags(input []*s3.Tag) map[string]string {
	t := make(map[string]string)

	for _, tag := range input {
		t[*tag.Key] = *tag.Value
	}

	return t
}

func tagsMatch(qt, rt map[string]string) bool {
	for k, v := range qt {
		if rt[k] != v {
			return false
		}
	}

	return true
}

func getBucketTagDescriptions(svc *s3.S3, name *string) ([]*s3.Tag, error) {
	treq := &s3.GetBucketTaggingInput{
		Bucket: name,
	}

	resp, err := svc.GetBucketTagging(treq)

	return resp.TagSet, err
}

func getBucketPermissions(svc *s3.S3, name *string) ([]*s3.Grant, error) {
	req := &s3.GetBucketAclInput{
		Bucket: name,
	}

	resp, err := svc.GetBucketAcl(req)

	return resp.Grants, err
}

func getBucketLocation(svc *s3.S3, name *string) (*string, error) {
	req := &s3.GetBucketLocationInput{
		Bucket: name,
	}

	resp, err := svc.GetBucketLocation(req)

	return resp.LocationConstraint, err
}

func mapS3Grantees(grantees []*s3.Grant) []Grantee {
	var gs []Grantee

	for _, g := range grantees {
		grantee := Grantee{
			ID:          g.Grantee.ID,
			Type:        g.Grantee.Type,
			Permissions: g.Permission,
		}

		switch *g.Grantee.Type {
		case "id":
			grantee.ID = g.Grantee.ID
		case "emailaddress":
			grantee.ID = g.Grantee.EmailAddress
		case "uri":
			grantee.ID = g.Grantee.URI
		}

		gs = append(gs, grantee)
	}

	return gs
}

// ToEvent converts an s3 bucket object to an ernest event
func toEvent(b *s3.Bucket, grants []*s3.Grant, location *string, tags []*s3.Tag) *Event {
	uri := fmt.Sprintf("https://%s.s3.amazonaws.com", *b.Name)

	e := &Event{
		Name:           b.Name,
		Grantees:       mapS3Grantees(grants),
		BucketLocation: location,
		// ACL ???
		BucketURI: &uri,
		Tags:      mapS3Tags(tags),
	}
	return e
}
<|endoftext|>


<commit_before>/*
 * Copyright 2012 Nan Deng
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

package cache

import (
	"container/list"
	"fmt"
	"sync"
	"time"
)

type Flusher interface {
	Add(key string, value interface{})
	Remove(key string)
}

type dirtyElement struct {
	modified bool
	removed  bool
	key      string
	value    interface{}
}

type cacheItem struct {
	key   string
	value interface{}
}

type Cache struct {
	mu          sync.Mutex
	flushPeriod time.Duration
	data        map[string]*list.Element
	dirtyList   *list.List
	list        *list.List
	capacity    int
	flusher     Flusher
	maxNrDirty  int
}

func (c *Cache) Flush() {
	c.mu.Lock()
	defer c.mu.Unlock()
	if c.flusher == nil {
		return
	}
	for e := c.dirtyList.Front(); e != nil; e = e.Next() {
		if de, ok := e.Value.(*dirtyElement); ok {
			if de.removed {
				c.flusher.Remove(de.key)
			} else if de.modified {
				c.flusher.Add(de.key, de.value)
			}
		}
	}
	c.dirtyList = list.New()
}

func (c *Cache) debug() {
	fmt.Printf("nr elems %v <= %v, nr dirty elems %v < %v\n",
		c.list.Len(), c.capacity, c.dirtyList.Len(), c.maxNrDirty)
	fmt.Println("-----------------elements------------")
	for e := c.list.Front(); e != nil; e = e.Next() {
		item := e.Value.(*cacheItem)
		fmt.Printf("%v: %v\n", item.key, item.value)
	}
	fmt.Println("-----------dirty elements------------")
	for e := c.dirtyList.Front(); e != nil; e = e.Next() {
		de := e.Value.(*dirtyElement)
		fmt.Printf("%v: %v; modified: %v; removed %v\n",
			de.key, de.value, de.modified, de.removed)
	}
	fmt.Println("-------------------------------------")
}

func (c *Cache) checkAndFlush() {
	c.mu.Lock()
	if c.maxNrDirty >= 0 && c.dirtyList.Len() >= c.maxNrDirty {
		c.mu.Unlock()
		c.Flush()
	} else {
		c.mu.Unlock()
	}
}

// capacity: nr elements in the cache.
//  < 0 means always in memory;
//  = 0 means no cache.
// maxNrDirty: < 0 means no flush.
// flushPeriod:
//  > 1 second means periodically flush;
//  0 second means no periodically flush;
//  undefined in range (0, 1).
func New(capacity int, maxNrDirty int, flushPeriod time.Duration, flusher Flusher) *Cache {
	cache := new(Cache)

	cache.flushPeriod = flushPeriod
	cache.capacity = capacity
	if cache.capacity < 0 {
		cache.data = make(map[string]*list.Element, 1024)
	} else {
		cache.data = make(map[string]*list.Element, cache.capacity)
	}
	cache.list = list.New()
	cache.dirtyList = list.New()
	cache.flusher = flusher
	cache.maxNrDirty = maxNrDirty

	if flushPeriod.Seconds() > 0.9 {
		go func() {
			time.Sleep(flushPeriod)
			cache.Flush()
		}()
	}
	return cache
}

func (c *Cache) Get(key string) interface{} {
	c.mu.Lock()
	defer c.mu.Unlock()
	if elem, ok := c.data[key]; ok {
		item := elem.Value.(*cacheItem)
		c.list.MoveToFront(elem)
		return item.value
	}
	return nil
}

func (c *Cache) Set(key string, value interface{}) {
	c.mu.Lock()
	defer c.checkAndFlush()
	defer c.mu.Unlock()

	de := &dirtyElement{
		modified: true,
		removed:  false,
		key:      key,
		value:    value,
	}

	if e, ok := c.data[key]; ok {
		item := e.Value.(*cacheItem)
		item.value = value
		c.list.MoveToFront(e)
		c.dirtyList.PushBack(de)
	} else {
		elem := c.list.PushFront(&cacheItem{key: key, value: value})
		c.data[key] = elem
		c.dirtyList.PushBack(de)
		if c.capacity >= 0 && len(c.data) > c.capacity {
			last := c.list.Back()
			item := last.Value.(*cacheItem)
			c.list.Remove(last)
			delete(c.data, item.key)
		}
	}
	return
}

func (c *Cache) Delete(key string) interface{} {
	c.mu.Lock()
	defer c.mu.Unlock()

	de := &dirtyElement{
		modified: false,
		removed:  true,
		key:      key,
		value:    nil,
	}
	c.dirtyList.PushBack(de)
	if elem, ok := c.data[key]; ok {
		delete(c.data, key)
		item := elem.Value.(*cacheItem)
		return item.value
	}
	return nil
}
<commit_msg>Added Len() method<commit_after>/*
 * Copyright 2012 Nan Deng
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

package cache

import (
	"container/list"
	"fmt"
	"sync"
	"time"
)

type Flusher interface {
	Add(key string, value interface{})
	Remove(key string)
}

type dirtyElement struct {
	modified bool
	removed  bool
	key      string
	value    interface{}
}

type cacheItem struct {
	key   string
	value interface{}
}

type Cache struct {
	mu          sync.Mutex
	flushPeriod time.Duration
	data        map[string]*list.Element
	dirtyList   *list.List
	list        *list.List
	capacity    int
	flusher     Flusher
	maxNrDirty  int
}

func (c *Cache) Len() int {
	return len(c.data)
}

func (c *Cache) Flush() {
	c.mu.Lock()
	defer c.mu.Unlock()
	if c.flusher == nil {
		return
	}
	for e := c.dirtyList.Front(); e != nil; e = e.Next() {
		if de, ok := e.Value.(*dirtyElement); ok {
			if de.removed {
				c.flusher.Remove(de.key)
			} else if de.modified {
				c.flusher.Add(de.key, de.value)
			}
		}
	}
	c.dirtyList = list.New()
}

func (c *Cache) debug() {
	fmt.Printf("nr elems %v <= %v, nr dirty elems %v < %v\n",
		c.list.Len(), c.capacity, c.dirtyList.Len(), c.maxNrDirty)
	fmt.Println("-----------------elements------------")
	for e := c.list.Front(); e != nil; e = e.Next() {
		item := e.Value.(*cacheItem)
		fmt.Printf("%v: %v\n", item.key, item.value)
	}
	fmt.Println("-----------dirty elements------------")
	for e := c.dirtyList.Front(); e != nil; e = e.Next() {
		de := e.Value.(*dirtyElement)
		fmt.Printf("%v: %v; modified: %v; removed %v\n",
			de.key, de.value, de.modified, de.removed)
	}
	fmt.Println("-------------------------------------")
}

func (c *Cache) checkAndFlush() {
	c.mu.Lock()
	if c.maxNrDirty >= 0 && c.dirtyList.Len() >= c.maxNrDirty {
		c.mu.Unlock()
		c.Flush()
	} else {
		c.mu.Unlock()
	}
}

// capacity: nr elements in the cache.
//  < 0 means always in memory;
//  = 0 means no cache.
// maxNrDirty: < 0 means no flush.
// flushPeriod:
//  > 1 second means periodically flush;
//  0 second means no periodically flush;
//  undefined in range (0, 1).
func New(capacity int, maxNrDirty int, flushPeriod time.Duration, flusher Flusher) *Cache {
	cache := new(Cache)

	cache.flushPeriod = flushPeriod
	cache.capacity = capacity
	if cache.capacity < 0 {
		cache.data = make(map[string]*list.Element, 1024)
	} else {
		cache.data = make(map[string]*list.Element, cache.capacity)
	}
	cache.list = list.New()
	cache.dirtyList = list.New()
	cache.flusher = flusher
	cache.maxNrDirty = maxNrDirty

	if flushPeriod.Seconds() > 0.9 {
		go func() {
			time.Sleep(flushPeriod)
			cache.Flush()
		}()
	}
	return cache
}

func (c *Cache) Get(key string) interface{} {
	c.mu.Lock()
	defer c.mu.Unlock()
	if elem, ok := c.data[key]; ok {
		item := elem.Value.(*cacheItem)
		c.list.MoveToFront(elem)
		return item.value
	}
	return nil
}

func (c *Cache) Set(key string, value interface{}) {
	c.mu.Lock()
	defer c.checkAndFlush()
	defer c.mu.Unlock()

	de := &dirtyElement{
		modified: true,
		removed:  false,
		key:      key,
		value:    value,
	}

	if e, ok := c.data[key]; ok {
		item := e.Value.(*cacheItem)
		item.value = value
		c.list.MoveToFront(e)
		c.dirtyList.PushBack(de)
	} else {
		elem := c.list.PushFront(&cacheItem{key: key, value: value})
		c.data[key] = elem
		c.dirtyList.PushBack(de)
		if c.capacity >= 0 && len(c.data) > c.capacity {
			last := c.list.Back()
			item := last.Value.(*cacheItem)
			c.list.Remove(last)
			delete(c.data, item.key)
		}
	}
	return
}

func (c *Cache) Delete(key string) interface{} {
	c.mu.Lock()
	defer c.mu.Unlock()

	de := &dirtyElement{
		modified: false,
		removed:  true,
		key:      key,
		value:    nil,
	}
	c.dirtyList.PushBack(de)
	if elem, ok := c.data[key]; ok {
		delete(c.data, key)
		item := elem.Value.(*cacheItem)
		return item.value
	}
	return nil
}
<|endoftext|>


<commit_before>package tachymeter

import (
	"math"
	"sort"
	"time"
)

// Satisfy sort for timeSlice.
// Sorts in increasing order of duration.

func (p timeSlice) Len() int {
	return len(p)
}

func (p timeSlice) Less(i, j int) bool {
	return int64(p[i]) < int64(p[j])
}

func (p timeSlice) Swap(i, j int) {
	p[i], p[j] = p[j], p[i]
}

// Calc calcs data held in a *Tachymeter
// and returns a *Metrics.
func (m *Tachymeter) Calc() *Metrics {
	m.Lock()
	defer m.Unlock()

	metrics := &Metrics{}
	if m.Count == 0 {
		return metrics
	}

	times := m.Times[:m.TimesUsed]
	sort.Sort(times)

	metrics.Samples = m.TimesUsed
	metrics.Count = m.Count
	metrics.Time.Total = calcTimeTotal(times)
	metrics.Time.Avg = calcAvg(times, metrics.Samples)
	metrics.Time.Median = times[len(times)/2]
	metrics.Time.p95 = calcp95(times)
	metrics.Time.Long5p = calcLong5p(times)
	metrics.Time.Short5p = calcShort5p(times)
	metrics.Time.Max = times[metrics.Samples-1]
	metrics.Time.Min = times[0]

	var rateTime float64
	if m.WallTime != 0 {
		rateTime = float64(metrics.Count) / float64(m.WallTime)
	} else {
		rateTime = float64(metrics.Samples) / float64(metrics.Time.Total)
	}
	metrics.Rate.Second = rateTime * 1e9

	return metrics
}

// These should be self-explanatory:

func calcTimeTotal(d []time.Duration) time.Duration {
	var total time.Duration
	for _, t := range d {
		total += t
	}

	return total
}

func calcAvg(d []time.Duration, c int) time.Duration {
	var total time.Duration
	for _, t := range d {
		total += t
	}
	return time.Duration(int(total) / c)
}

func calcp95(d []time.Duration) time.Duration {
	return d[int(float64(len(d))*0.95)]
}

func calcLong5p(d []time.Duration) time.Duration {
	set := d[int(float64(len(d))*0.95):]

	var t time.Duration
	var i int
	for _, n := range set {
		t += n
		i++
	}

	return time.Duration(int(t) / i)
}

func calcShort5p(d []time.Duration) time.Duration {
	set := d[:int(math.Ceil(float64(len(d))*0.05))]

	var t time.Duration
	var i int
	for _, n := range set {
		t += n
		i++
	}

	return time.Duration(int(t) / i)
}
<commit_msg>better rounding<commit_after>package tachymeter

import (
	"sort"
	"time"
)

// Satisfy sort for timeSlice.
// Sorts in increasing order of duration.

func (p timeSlice) Len() int {
	return len(p)
}

func (p timeSlice) Less(i, j int) bool {
	return int64(p[i]) < int64(p[j])
}

func (p timeSlice) Swap(i, j int) {
	p[i], p[j] = p[j], p[i]
}

// Calc calcs data held in a *Tachymeter
// and returns a *Metrics.
func (m *Tachymeter) Calc() *Metrics {
	m.Lock()
	defer m.Unlock()

	metrics := &Metrics{}
	if m.Count == 0 {
		return metrics
	}

	times := m.Times[:m.TimesUsed]
	sort.Sort(times)

	metrics.Samples = m.TimesUsed
	metrics.Count = m.Count
	metrics.Time.Total = calcTimeTotal(times)
	metrics.Time.Avg = calcAvg(times, metrics.Samples)
	metrics.Time.Median = times[len(times)/2]
	metrics.Time.p95 = calcp95(times)
	metrics.Time.Long5p = calcLong5p(times)
	metrics.Time.Short5p = calcShort5p(times)
	metrics.Time.Max = times[metrics.Samples-1]
	metrics.Time.Min = times[0]

	var rateTime float64
	if m.WallTime != 0 {
		rateTime = float64(metrics.Count) / float64(m.WallTime)
	} else {
		rateTime = float64(metrics.Samples) / float64(metrics.Time.Total)
	}
	metrics.Rate.Second = rateTime * 1e9

	return metrics
}

// These should be self-explanatory:

func calcTimeTotal(d []time.Duration) time.Duration {
	var total time.Duration
	for _, t := range d {
		total += t
	}

	return total
}

func calcAvg(d []time.Duration, c int) time.Duration {
	var total time.Duration
	for _, t := range d {
		total += t
	}
	return time.Duration(int(total) / c)
}

func calcp95(d []time.Duration) time.Duration {
	return d[int(float64(len(d))*0.95+0.5)]
}

func calcLong5p(d []time.Duration) time.Duration {
	set := d[int(float64(len(d))*0.95+0.5):]

	var t time.Duration
	var i int
	for _, n := range set {
		t += n
		i++
	}

	return time.Duration(int(t) / i)
}

func calcShort5p(d []time.Duration) time.Duration {
	set := d[:int(float64(len(d))*0.05+0.5)]

	var t time.Duration
	var i int
	for _, n := range set {
		t += n
		i++
	}

	return time.Duration(int(t) / i)
}
<|endoftext|>


<commit_before>// Copyright © 2014 Steve Francia <spf@spf13.com>.
//
// Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.

package cast

import (
	"errors"
	"fmt"
	"html/template"
	"reflect"
	"strconv"
	"strings"
	"time"

	jww "github.com/spf13/jwalterweatherman"
)

func ToTimeE(i interface{}) (tim time.Time, err error) {
	i = indirect(i)
	jww.DEBUG.Println("ToTimeE called on type:", reflect.TypeOf(i))

	switch s := i.(type) {
	case time.Time:
		return s, nil
	case string:
		d, e := StringToDate(s)
		if e == nil {
			return d, nil
		}
		return time.Time{}, fmt.Errorf("Could not parse Date/Time format: %v\n", e)
	default:
		return time.Time{}, fmt.Errorf("Unable to Cast %#v to Time\n", i)
	}
}

func ToBoolE(i interface{}) (bool, error) {
	i = indirect(i)
	jww.DEBUG.Println("ToBoolE called on type:", reflect.TypeOf(i))

	switch b := i.(type) {
	case bool:
		return b, nil
	case nil:
		return false, nil
	case int:
		if i.(int) != 0 {
			return true, nil
		}
		return false, nil
	case string:
		return strconv.ParseBool(i.(string))
	default:
		return false, fmt.Errorf("Unable to Cast %#v to bool", i)
	}
}

func ToFloat64E(i interface{}) (float64, error) {
	i = indirect(i)
	jww.DEBUG.Println("ToFloat64E called on type:", reflect.TypeOf(i))

	switch s := i.(type) {
	case float64:
		return s, nil
	case float32:
		return float64(s), nil
	case int64:
		return float64(s), nil
	case int32:
		return float64(s), nil
	case int16:
		return float64(s), nil
	case int8:
		return float64(s), nil
	case int:
		return float64(s), nil
	case string:
		v, err := strconv.ParseFloat(s, 64)
		if err == nil {
			return float64(v), nil
		} else {
			return 0.0, fmt.Errorf("Unable to Cast %#v to float", i)
		}
	default:
		return 0.0, fmt.Errorf("Unable to Cast %#v to float", i)
	}
}

func ToIntE(i interface{}) (int, error) {
	i = indirect(i)
	jww.DEBUG.Println("ToIntE called on type:", reflect.TypeOf(i))

	switch s := i.(type) {
	case int:
		return s, nil
	case int64:
		return int(s), nil
	case int32:
		return int(s), nil
	case int16:
		return int(s), nil
	case int8:
		return int(s), nil
	case string:
		v, err := strconv.ParseInt(s, 0, 0)
		if err == nil {
			return int(v), nil
		} else {
			return 0, fmt.Errorf("Unable to Cast %#v to int", i)
		}
	case float64:
		return int(s), nil
	case bool:
		if bool(s) {
			return 1, nil
		} else {
			return 0, nil
		}
	case nil:
		return 0, nil
	default:
		return 0, fmt.Errorf("Unable to Cast %#v to int", i)
	}
}

// From html/template/content.go
// Copyright 2011 The Go Authors. All rights reserved.
// indirect returns the value, after dereferencing as many times
// as necessary to reach the base type (or nil).
func indirect(a interface{}) interface{} {
	if a == nil {
		return nil
	}
	if t := reflect.TypeOf(a); t.Kind() != reflect.Ptr {
		// Avoid creating a reflect.Value if it's not a pointer.
		return a
	}
	v := reflect.ValueOf(a)
	for v.Kind() == reflect.Ptr && !v.IsNil() {
		v = v.Elem()
	}
	return v.Interface()
}

// From html/template/content.go
// Copyright 2011 The Go Authors. All rights reserved.
// indirectToStringerOrError returns the value, after dereferencing as many times
// as necessary to reach the base type (or nil) or an implementation of fmt.Stringer
// or error,
func indirectToStringerOrError(a interface{}) interface{} {
	if a == nil {
		return nil
	}

	var errorType = reflect.TypeOf((*error)(nil)).Elem()
	var fmtStringerType = reflect.TypeOf((*fmt.Stringer)(nil)).Elem()

	v := reflect.ValueOf(a)
	for !v.Type().Implements(fmtStringerType) && !v.Type().Implements(errorType) && v.Kind() == reflect.Ptr && !v.IsNil() {
		v = v.Elem()
	}
	return v.Interface()
}

func ToStringE(i interface{}) (string, error) {
	i = indirectToStringerOrError(i)
	jww.DEBUG.Println("ToStringE called on type:", reflect.TypeOf(i))

	switch s := i.(type) {
	case string:
		return s, nil
	case float64:
		return strconv.FormatFloat(i.(float64), 'f', -1, 64), nil
	case int:
		return strconv.FormatInt(int64(i.(int)), 10), nil
	case []byte:
		return string(s), nil
	case template.HTML:
		return string(s), nil
	case nil:
		return "", nil
	case fmt.Stringer:
		return s.String(), nil
	case error:
		return s.Error(), nil
	default:
		return "", fmt.Errorf("Unable to Cast %#v to string", i)
	}
}

func ToStringMapStringE(i interface{}) (map[string]string, error) {
	jww.DEBUG.Println("ToStringMapStringE called on type:", reflect.TypeOf(i))

	var m = map[string]string{}

	switch v := i.(type) {
	case map[interface{}]interface{}:
		for k, val := range v {
			m[ToString(k)] = ToString(val)
		}
		return m, nil
	case map[string]interface{}:
		for k, val := range v {
			m[ToString(k)] = ToString(val)
		}
		return m, nil
	case map[string]string:
		return v, nil
	default:
		return m, fmt.Errorf("Unable to Cast %#v to map[string]string", i)
	}
	return m, fmt.Errorf("Unable to Cast %#v to map[string]string", i)
}

func ToStringMapBoolE(i interface{}) (map[string]bool, error) {
	jww.DEBUG.Println("ToStringMapBoolE called on type:", reflect.TypeOf(i))

	var m = map[string]bool{}

	switch v := i.(type) {
	case map[interface{}]interface{}:
		for k, val := range v {
			m[ToString(k)] = ToBool(val)
		}
		return m, nil
	case map[string]interface{}:
		for k, val := range v {
			m[ToString(k)] = ToBool(val)
		}
		return m, nil
	case map[string]bool:
		return v, nil
	default:
		return m, fmt.Errorf("Unable to Cast %#v to map[string]bool", i)
	}
	return m, fmt.Errorf("Unable to Cast %#v to map[string]bool", i)
}

func ToStringMapE(i interface{}) (map[string]interface{}, error) {
	jww.DEBUG.Println("ToStringMapE called on type:", reflect.TypeOf(i))

	var m = map[string]interface{}{}

	switch v := i.(type) {
	case map[interface{}]interface{}:
		for k, val := range v {
			m[ToString(k)] = val
		}
		return m, nil
	case map[string]interface{}:
		return v, nil
	default:
		return m, fmt.Errorf("Unable to Cast %#v to map[string]interface{}", i)
	}

	return m, fmt.Errorf("Unable to Cast %#v to map[string]interface{}", i)
}

func ToSliceE(i interface{}) ([]interface{}, error) {
	jww.DEBUG.Println("ToSliceE called on type:", reflect.TypeOf(i))

	var s []interface{}

	switch v := i.(type) {
	case []interface{}:
		for _, u := range v {
			s = append(s, u)
		}
		return s, nil
	case []map[string]interface{}:
		for _, u := range v {
			s = append(s, u)
		}
		return s, nil
	default:
		return s, fmt.Errorf("Unable to Cast %#v of type %v to []interface{}", i, reflect.TypeOf(i))
	}

	return s, fmt.Errorf("Unable to Cast %#v to []interface{}", i)
}

func ToStringSliceE(i interface{}) ([]string, error) {
	jww.DEBUG.Println("ToStringSliceE called on type:", reflect.TypeOf(i))

	var a []string

	switch v := i.(type) {
	case []interface{}:
		for _, u := range v {
			a = append(a, ToString(u))
		}
		return a, nil
	case []string:
		return v, nil
	case string:
		return strings.Split(v, " "), nil
	default:
		return a, fmt.Errorf("Unable to Cast %#v to []string", i)
	}

	return a, fmt.Errorf("Unable to Cast %#v to []string", i)
}

func StringToDate(s string) (time.Time, error) {
	return parseDateWith(s, []string{
		time.RFC3339,
		"2006-01-02T15:04:05", // iso8601 without timezone
		time.RFC1123Z,
		time.RFC1123,
		time.RFC822Z,
		time.RFC822,
		time.ANSIC,
		time.UnixDate,
		time.RubyDate,
		"2006-01-02 15:04:05Z07:00",
		"02 Jan 06 15:04 MST",
		"2006-01-02",
		"02 Jan 2006",
	})
}

func parseDateWith(s string, dates []string) (d time.Time, e error) {
	for _, dateType := range dates {
		if d, e = time.Parse(dateType, s); e == nil {
			return
		}
	}
	return d, errors.New(fmt.Sprintf("Unable to parse date: %s", s))
}
<commit_msg>Use strings.Fields<commit_after>// Copyright © 2014 Steve Francia <spf@spf13.com>.
//
// Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.

package cast

import (
	"errors"
	"fmt"
	"html/template"
	"reflect"
	"strconv"
	"strings"
	"time"

	jww "github.com/spf13/jwalterweatherman"
)

func ToTimeE(i interface{}) (tim time.Time, err error) {
	i = indirect(i)
	jww.DEBUG.Println("ToTimeE called on type:", reflect.TypeOf(i))

	switch s := i.(type) {
	case time.Time:
		return s, nil
	case string:
		d, e := StringToDate(s)
		if e == nil {
			return d, nil
		}
		return time.Time{}, fmt.Errorf("Could not parse Date/Time format: %v\n", e)
	default:
		return time.Time{}, fmt.Errorf("Unable to Cast %#v to Time\n", i)
	}
}

func ToBoolE(i interface{}) (bool, error) {
	i = indirect(i)
	jww.DEBUG.Println("ToBoolE called on type:", reflect.TypeOf(i))

	switch b := i.(type) {
	case bool:
		return b, nil
	case nil:
		return false, nil
	case int:
		if i.(int) != 0 {
			return true, nil
		}
		return false, nil
	case string:
		return strconv.ParseBool(i.(string))
	default:
		return false, fmt.Errorf("Unable to Cast %#v to bool", i)
	}
}

func ToFloat64E(i interface{}) (float64, error) {
	i = indirect(i)
	jww.DEBUG.Println("ToFloat64E called on type:", reflect.TypeOf(i))

	switch s := i.(type) {
	case float64:
		return s, nil
	case float32:
		return float64(s), nil
	case int64:
		return float64(s), nil
	case int32:
		return float64(s), nil
	case int16:
		return float64(s), nil
	case int8:
		return float64(s), nil
	case int:
		return float64(s), nil
	case string:
		v, err := strconv.ParseFloat(s, 64)
		if err == nil {
			return float64(v), nil
		} else {
			return 0.0, fmt.Errorf("Unable to Cast %#v to float", i)
		}
	default:
		return 0.0, fmt.Errorf("Unable to Cast %#v to float", i)
	}
}

func ToIntE(i interface{}) (int, error) {
	i = indirect(i)
	jww.DEBUG.Println("ToIntE called on type:", reflect.TypeOf(i))

	switch s := i.(type) {
	case int:
		return s, nil
	case int64:
		return int(s), nil
	case int32:
		return int(s), nil
	case int16:
		return int(s), nil
	case int8:
		return int(s), nil
	case string:
		v, err := strconv.ParseInt(s, 0, 0)
		if err == nil {
			return int(v), nil
		} else {
			return 0, fmt.Errorf("Unable to Cast %#v to int", i)
		}
	case float64:
		return int(s), nil
	case bool:
		if bool(s) {
			return 1, nil
		} else {
			return 0, nil
		}
	case nil:
		return 0, nil
	default:
		return 0, fmt.Errorf("Unable to Cast %#v to int", i)
	}
}

// From html/template/content.go
// Copyright 2011 The Go Authors. All rights reserved.
// indirect returns the value, after dereferencing as many times
// as necessary to reach the base type (or nil).
func indirect(a interface{}) interface{} {
	if a == nil {
		return nil
	}
	if t := reflect.TypeOf(a); t.Kind() != reflect.Ptr {
		// Avoid creating a reflect.Value if it's not a pointer.
		return a
	}
	v := reflect.ValueOf(a)
	for v.Kind() == reflect.Ptr && !v.IsNil() {
		v = v.Elem()
	}
	return v.Interface()
}

// From html/template/content.go
// Copyright 2011 The Go Authors. All rights reserved.
// indirectToStringerOrError returns the value, after dereferencing as many times
// as necessary to reach the base type (or nil) or an implementation of fmt.Stringer
// or error,
func indirectToStringerOrError(a interface{}) interface{} {
	if a == nil {
		return nil
	}

	var errorType = reflect.TypeOf((*error)(nil)).Elem()
	var fmtStringerType = reflect.TypeOf((*fmt.Stringer)(nil)).Elem()

	v := reflect.ValueOf(a)
	for !v.Type().Implements(fmtStringerType) && !v.Type().Implements(errorType) && v.Kind() == reflect.Ptr && !v.IsNil() {
		v = v.Elem()
	}
	return v.Interface()
}

func ToStringE(i interface{}) (string, error) {
	i = indirectToStringerOrError(i)
	jww.DEBUG.Println("ToStringE called on type:", reflect.TypeOf(i))

	switch s := i.(type) {
	case string:
		return s, nil
	case float64:
		return strconv.FormatFloat(i.(float64), 'f', -1, 64), nil
	case int:
		return strconv.FormatInt(int64(i.(int)), 10), nil
	case []byte:
		return string(s), nil
	case template.HTML:
		return string(s), nil
	case nil:
		return "", nil
	case fmt.Stringer:
		return s.String(), nil
	case error:
		return s.Error(), nil
	default:
		return "", fmt.Errorf("Unable to Cast %#v to string", i)
	}
}

func ToStringMapStringE(i interface{}) (map[string]string, error) {
	jww.DEBUG.Println("ToStringMapStringE called on type:", reflect.TypeOf(i))

	var m = map[string]string{}

	switch v := i.(type) {
	case map[interface{}]interface{}:
		for k, val := range v {
			m[ToString(k)] = ToString(val)
		}
		return m, nil
	case map[string]interface{}:
		for k, val := range v {
			m[ToString(k)] = ToString(val)
		}
		return m, nil
	case map[string]string:
		return v, nil
	default:
		return m, fmt.Errorf("Unable to Cast %#v to map[string]string", i)
	}
	return m, fmt.Errorf("Unable to Cast %#v to map[string]string", i)
}

func ToStringMapBoolE(i interface{}) (map[string]bool, error) {
	jww.DEBUG.Println("ToStringMapBoolE called on type:", reflect.TypeOf(i))

	var m = map[string]bool{}

	switch v := i.(type) {
	case map[interface{}]interface{}:
		for k, val := range v {
			m[ToString(k)] = ToBool(val)
		}
		return m, nil
	case map[string]interface{}:
		for k, val := range v {
			m[ToString(k)] = ToBool(val)
		}
		return m, nil
	case map[string]bool:
		return v, nil
	default:
		return m, fmt.Errorf("Unable to Cast %#v to map[string]bool", i)
	}
	return m, fmt.Errorf("Unable to Cast %#v to map[string]bool", i)
}

func ToStringMapE(i interface{}) (map[string]interface{}, error) {
	jww.DEBUG.Println("ToStringMapE called on type:", reflect.TypeOf(i))

	var m = map[string]interface{}{}

	switch v := i.(type) {
	case map[interface{}]interface{}:
		for k, val := range v {
			m[ToString(k)] = val
		}
		return m, nil
	case map[string]interface{}:
		return v, nil
	default:
		return m, fmt.Errorf("Unable to Cast %#v to map[string]interface{}", i)
	}

	return m, fmt.Errorf("Unable to Cast %#v to map[string]interface{}", i)
}

func ToSliceE(i interface{}) ([]interface{}, error) {
	jww.DEBUG.Println("ToSliceE called on type:", reflect.TypeOf(i))

	var s []interface{}

	switch v := i.(type) {
	case []interface{}:
		for _, u := range v {
			s = append(s, u)
		}
		return s, nil
	case []map[string]interface{}:
		for _, u := range v {
			s = append(s, u)
		}
		return s, nil
	default:
		return s, fmt.Errorf("Unable to Cast %#v of type %v to []interface{}", i, reflect.TypeOf(i))
	}

	return s, fmt.Errorf("Unable to Cast %#v to []interface{}", i)
}

func ToStringSliceE(i interface{}) ([]string, error) {
	jww.DEBUG.Println("ToStringSliceE called on type:", reflect.TypeOf(i))

	var a []string

	switch v := i.(type) {
	case []interface{}:
		for _, u := range v {
			a = append(a, ToString(u))
		}
		return a, nil
	case []string:
		return v, nil
	case string:
		return strings.Fields(v), nil
	default:
		return a, fmt.Errorf("Unable to Cast %#v to []string", i)
	}

	return a, fmt.Errorf("Unable to Cast %#v to []string", i)
}

func StringToDate(s string) (time.Time, error) {
	return parseDateWith(s, []string{
		time.RFC3339,
		"2006-01-02T15:04:05", // iso8601 without timezone
		time.RFC1123Z,
		time.RFC1123,
		time.RFC822Z,
		time.RFC822,
		time.ANSIC,
		time.UnixDate,
		time.RubyDate,
		"2006-01-02 15:04:05Z07:00",
		"02 Jan 06 15:04 MST",
		"2006-01-02",
		"02 Jan 2006",
	})
}

func parseDateWith(s string, dates []string) (d time.Time, e error) {
	for _, dateType := range dates {
		if d, e = time.Parse(dateType, s); e == nil {
			return
		}
	}
	return d, errors.New(fmt.Sprintf("Unable to parse date: %s", s))
}
<|endoftext|>