<commit_before>package main

import (
	"errors"
	"fmt"
	"os"
	"path"
	"runtime"
	"time"

	. "github.com/fuzzy/gocolor"
	"github.com/kisielk/whisper-go/whisper"
	"github.com/urfave/cli"
)

var (
	Filled  int64
	Overlap int64
)

func ProgressIndicator(c int64, t int64) {
	rtime := (time.Now().Unix() - StartTime)
	fmt.Printf("%s %d of %d (%6.02f%%) runtime: %dsec @ %d/sec\r",
		String("Info").Green().Bold(),
		c,
		t,
		(float64(c)/float64(t))*100.00,
		rtime,
		(Filled / rtime))
}

func fillWorker(d chan map[string]string) {
	for {
		msg := <-d
		if msg["SRC"] == "__EXIT__" && msg["DST"] == "__EXIT__" {
			return
		} else {
			if isFile(msg["SRC"]) && isFile(msg["DST"]) {
				if !backfillFile(msg["SRC"], msg["DST"]) {
					Error.Printf("S:%s D:%s - Backfill operation failed.",
						msg["SRC"],
						msg["DST"])
					os.Exit(1)
				} else {
					Filled++
					ProgressIndicator(Filled, Overlap)
				}
			}
		}
	}
}

func backfillFile(s string, d string) bool {
	// Open our filehandles
	sDb, sErr := whisper.Open(s)
	dDb, dErr := whisper.Open(d)
	if !chkErr(sErr) || !chkErr(dErr) {
		os.Exit(1)
	}
	// Defer their closings
	defer sDb.Close()
	defer dDb.Close()

	// Now for a series of checks, first to ensure that both
	// files have the same number of archives in them.
	if sDb.Header.Metadata.ArchiveCount != dDb.Header.Metadata.ArchiveCount {
		Error.Println("The files have a mismatched set of archives.")
		return false
	}

	// Now we'll start processing the archives, checking as we go to see if they
	// are matched. That way we at least fill in what we can, possibly....
	for i, a := range sDb.Header.Archives {
		// The offset
		if a.Offset == dDb.Header.Archives[i].Offset {
			// and the number of points
			if a.Points == dDb.Header.Archives[i].Points {
				// and finally the interval
				if a.SecondsPerPoint == dDb.Header.Archives[i].SecondsPerPoint {
					// ok, now let's get rolling through the archives
					sp, se := sDb.DumpArchive(i)
					if se != nil {
						Error.Println(se.Error())
						os.Exit(1)
					}
					dp, de := dDb.DumpArchive(i)
					if de != nil {
						Error.Println(de.Error())
						os.Exit(1)
					}
					for idx := 0; idx < len(sp); idx++ {
						if sp[idx].Timestamp != 0 && sp[idx].Value != 0 {
							if dp[idx].Timestamp == 0 || dp[idx].Value == 0 {
								dp[idx].Timestamp = sp[idx].Timestamp
								dp[idx].Value = sp[idx].Value
								dDb.Update(dp[idx])
							}
						}
					}
				}
			}
		}
	}
	return true
}

func Filler(c *cli.Context) error {
	if len(c.Args()) == 2 {
		args := c.Args()
		if isDir(args[0]) && isDir(args[1]) {
			Filled = 0

			// First things first, let's spawn our pool of workers.
			dataCh := make(chan map[string]string, 1)
			for i := 0; i < c.Int("j"); i++ {
				go fillWorker(dataCh)
			}

			// Let's get this dir walking in there then shall we?
			srcFiles := listFiles(args[0])
			dstFiles := listFiles(args[1])

			// Now let's find all our overlap
			overlap := []string{}
			for a := 0; a < len(srcFiles); a++ {
				for b := 0; b < len(dstFiles); b++ {
					if srcFiles[a] == dstFiles[b] {
						overlap = append(overlap, srcFiles[a])
					}
				}
			}

			// And display some stats
			Info.Printf("srcDir: %d files, dstDir: %d files, %d overlap.",
				len(srcFiles),
				len(dstFiles),
				len(overlap))
			Overlap = int64(len(overlap))

			// Now we can push in all our data, and let our workers do their
			// lovely little thing. Ahhhhh concurrency.
			StartTime = time.Now().Unix()
			for i := 0; i < len(overlap); i++ {
				dataCh <- map[string]string{
					"SRC": fmt.Sprintf("%s/%s", args[0], overlap[i]),
					"DST": fmt.Sprintf("%s/%s", args[1], overlap[i]),
				}
			}

			// And while they're off doing that, here we will just sit and watch
			// the return channel and count things. Once we have all our
			// backfill operations accounted for, we can reap all of our workers
			// and carry on.
			for Filled < int64(len(overlap)) {
				time.Sleep(1 * time.Second)
			}
			// And finally let's reap all our children
			for idx := 0; idx < c.Int("j"); idx++ {
				dataCh <- map[string]string{
					"SRC": "__EXIT__",
					"DST": "__EXIT__",
				}
			}

			fmt.Println("")
			Info.Println((Overlap / (time.Now().Unix() - StartTime)),
				"whisper files processed per second.")
		} else {
			if !backfillFile(args[0], args[1]) {
				e := fmt.Sprintf("Error while backfilling.")
				Error.Println(e)
				return errors.New(e)
			}
		}
	} else {
		var e string
		e = fmt.Sprintf("Wrong number of paramters given.")
		Error.Println(e)
		Error.Printf("Try '%s help fill' for more information",
			path.Base(os.Args[0]))
		return errors.New(e)
	}
	return nil
}

func init() {
	Commands = append(Commands, cli.Command{
		Name:        "fill",
		Aliases:     []string{"f"},
		Usage:       "Backfill datapoints in the dst from the src",
		Description: "Backfill datapoints in the dst from the src",
		ArgsUsage:   "<src(File|Dir)> <dst(File|Dir)>",
		Flags: []cli.Flag{
			cli.IntFlag{
				Name:  "j",
				Usage: "Number of workers (for directory recursion)",
				Value: runtime.GOMAXPROCS(0),
			},
			cli.BoolFlag{Name: "c", Usage: "Prevent colors from being used"},
		},
		SkipFlagParsing: false,
		HideHelp:        false,
		Hidden:          false,
		Action:          Filler,
	})
}
<commit_msg>more informative progress output<commit_after>package main

import (
	"errors"
	"fmt"
	"os"
	"path"
	"runtime"
	"time"

	. "github.com/fuzzy/gocolor"
	"github.com/kisielk/whisper-go/whisper"
	"github.com/urfave/cli"
)

var (
	Filled  int64
	Overlap int64
)

func ProgressIndicator(c int64, t int64) {
	rtime := (time.Now().Unix() - StartTime)
	count := Filled
	if rtime > 0 {
		count = (Filled / rtime)
	}
	fmt.Printf("%s %d of %d (%6.02f%%) runtime: %dsec @ %d/sec\r",
		String("Info").Green().Bold(),
		c,
		t,
		(float64(c)/float64(t))*100.00,
		rtime,
		count)
}

func fillWorker(d chan map[string]string) {
	for {
		msg := <-d
		if msg["SRC"] == "__EXIT__" && msg["DST"] == "__EXIT__" {
			return
		} else {
			if isFile(msg["SRC"]) && isFile(msg["DST"]) {
				if !backfillFile(msg["SRC"], msg["DST"]) {
					Error.Printf("S:%s D:%s - Backfill operation failed.",
						msg["SRC"],
						msg["DST"])
					os.Exit(1)
				} else {
					Filled++
					ProgressIndicator(Filled, Overlap)
				}
			}
		}
	}
}

func backfillFile(s string, d string) bool {
	// Open our filehandles
	sDb, sErr := whisper.Open(s)
	dDb, dErr := whisper.Open(d)
	if !chkErr(sErr) || !chkErr(dErr) {
		os.Exit(1)
	}
	// Defer their closings
	defer sDb.Close()
	defer dDb.Close()

	// Now for a series of checks, first to ensure that both
	// files have the same number of archives in them.
	if sDb.Header.Metadata.ArchiveCount != dDb.Header.Metadata.ArchiveCount {
		Error.Println("The files have a mismatched set of archives.")
		return false
	}

	// Now we'll start processing the archives, checking as we go to see if they
	// are matched. That way we at least fill in what we can, possibly....
	for i, a := range sDb.Header.Archives {
		// The offset
		if a.Offset == dDb.Header.Archives[i].Offset {
			// and the number of points
			if a.Points == dDb.Header.Archives[i].Points {
				// and finally the interval
				if a.SecondsPerPoint == dDb.Header.Archives[i].SecondsPerPoint {
					// ok, now let's get rolling through the archives
					sp, se := sDb.DumpArchive(i)
					if se != nil {
						Error.Println(se.Error())
						os.Exit(1)
					}
					dp, de := dDb.DumpArchive(i)
					if de != nil {
						Error.Println(de.Error())
						os.Exit(1)
					}
					for idx := 0; idx < len(sp); idx++ {
						if sp[idx].Timestamp != 0 && sp[idx].Value != 0 {
							if dp[idx].Timestamp == 0 || dp[idx].Value == 0 {
								dp[idx].Timestamp = sp[idx].Timestamp
								dp[idx].Value = sp[idx].Value
								dDb.Update(dp[idx])
							}
						}
					}
				}
			}
		}
	}
	return true
}

func Filler(c *cli.Context) error {
	if len(c.Args()) == 2 {
		args := c.Args()
		if isDir(args[0]) && isDir(args[1]) {
			Filled = 0

			// First things first, let's spawn our pool of workers.
			dataCh := make(chan map[string]string, 1)
			for i := 0; i < c.Int("j"); i++ {
				go fillWorker(dataCh)
			}

			// Let's get this dir walking in there then shall we?
			srcFiles := listFiles(args[0])
			dstFiles := listFiles(args[1])

			// Now let's find all our overlap
			overlap := []string{}
			for a := 0; a < len(srcFiles); a++ {
				for b := 0; b < len(dstFiles); b++ {
					if srcFiles[a] == dstFiles[b] {
						overlap = append(overlap, srcFiles[a])
					}
				}
			}

			// And display some stats
			Info.Printf("srcDir: %d files, dstDir: %d files, %d overlap.",
				len(srcFiles),
				len(dstFiles),
				len(overlap))
			Overlap = int64(len(overlap))

			// Now we can push in all our data, and let our workers do their
			// lovely little thing. Ahhhhh concurrency.
			StartTime = time.Now().Unix()
			for i := 0; i < len(overlap); i++ {
				dataCh <- map[string]string{
					"SRC": fmt.Sprintf("%s/%s", args[0], overlap[i]),
					"DST": fmt.Sprintf("%s/%s", args[1], overlap[i]),
				}
			}

			// And while they're off doing that, here we will just sit and watch
			// the return channel and count things. Once we have all our
			// backfill operations accounted for, we can reap all of our workers
			// and carry on.
			for Filled < int64(len(overlap)) {
				time.Sleep(1 * time.Second)
			}
			// And finally let's reap all our children
			for idx := 0; idx < c.Int("j"); idx++ {
				dataCh <- map[string]string{
					"SRC": "__EXIT__",
					"DST": "__EXIT__",
				}
			}

			fmt.Println("")
			Info.Println((Overlap / (time.Now().Unix() - StartTime)),
				"whisper files processed per second.")
		} else {
			if !backfillFile(args[0], args[1]) {
				e := fmt.Sprintf("Error while backfilling.")
				Error.Println(e)
				return errors.New(e)
			}
		}
	} else {
		var e string
		e = fmt.Sprintf("Wrong number of paramters given.")
		Error.Println(e)
		Error.Printf("Try '%s help fill' for more information",
			path.Base(os.Args[0]))
		return errors.New(e)
	}
	return nil
}

func init() {
	Commands = append(Commands, cli.Command{
		Name:        "fill",
		Aliases:     []string{"f"},
		Usage:       "Backfill datapoints in the dst from the src",
		Description: "Backfill datapoints in the dst from the src",
		ArgsUsage:   "<src(File|Dir)> <dst(File|Dir)>",
		Flags: []cli.Flag{
			cli.IntFlag{
				Name:  "j",
				Usage: "Number of workers (for directory recursion)",
				Value: runtime.GOMAXPROCS(0),
			},
			cli.BoolFlag{Name: "c", Usage: "Prevent colors from being used"},
		},
		SkipFlagParsing: false,
		HideHelp:        false,
		Hidden:          false,
		Action:          Filler,
	})
}
<|endoftext|>


<commit_before>/*
Stest tests reads files from stdin (one per line)
and prints every file that passes all given tests.

Usage:

	stest <option>...

The options are:

	-L
		Matches files that are symbolic links
	-S
		Matches files that are sockets
	-d
		Matches files that are directories
	-e
		Matches files that exist (regardless of type)
	-f
		Matches files that are regular files
	-g
		Matches files whose set group ID flag is set
	-k
		Matches files whose sticky bit is set
	-p
		Matches files that are named pipes (FIFOs)
	-r
		Matches files that are readable
	-s
		Matches files that have a size greater than zero
	-u
		Matches files whose set user ID flag is set
	-w
		Matches files that are writable
	-x
		Matches files that are executable
	-v
		Invert the sense of matching, to select non-matching files
*/
package main

import (
	"bufio"
	"flag"
	"fmt"
	"log"
	"os"
)

type statTests struct {
	stat  func(name string) (info os.FileInfo, err error)
	tests []flagTest
}
type flagTest struct {
	flag *bool
	test testFunc
}
type testFunc func(info os.FileInfo) bool

var (
	this = os.Args[0]
	L    = flag.Bool("L", false, "Matches files that are symbolic links")
	S    = flag.Bool("S", false, "Matches files that are sockets")
	d    = flag.Bool("d", false, "Matches files that are directories")
	e    = flag.Bool("e", false, "Matches files that exist (regardless of type)")
	f    = flag.Bool("f", false, "Matches files that are regular files")
	g    = flag.Bool("g", false, "Matches files whose set group ID flag is set")
	k    = flag.Bool("k", false, "Matches files whose sticky bit is set")
	p    = flag.Bool("p", false, "Matches files that are named pipes (FIFOs)")
	r    = flag.Bool("r", false, "Matches files that are readable")
	s    = flag.Bool("s", false, "Matches files that have a size greater than zero")
	u    = flag.Bool("u", false, "Matches files whose set user ID flag is set")
	v    = flag.Bool("v", false, "Invert the sense of matching, to select non-matching files")
	w    = flag.Bool("w", false, "Matches files that are writable")
	x    = flag.Bool("x", false, "Matches files that are executable")

	tests = []statTests{
		{os.Lstat, []flagTest{
			{L, mode(os.ModeSymlink)}},
		},
		{os.Stat, []flagTest{
			{e, yes},
			{s, size},
			{f, not(mode(os.ModeType))},
			{d, mode(os.ModeDir)},
			{p, mode(os.ModeNamedPipe)},
			{S, mode(os.ModeSocket)},
			{u, mode(os.ModeSetuid)},
			{g, mode(os.ModeSetgid)},
			{k, mode(os.ModeSticky)},
			{r, perm(0444)},
			{w, perm(0222)},
			{x, perm(0111)},
		}},
	}
)

func main() {
	log.SetFlags(0)
	log.SetPrefix("stest: ")
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "usage: %s <option>...\n", this)
		flag.PrintDefaults()
	}
	flag.Parse()
	if flag.NFlag() == 0 {
		flag.Usage()
		os.Exit(2)
	}
	in := bufio.NewScanner(os.Stdin)
	for in.Scan() {
		path := in.Text()
		if *v != test(path) {
			fmt.Println(path)
		}
	}
	if err := in.Err(); err != nil {
		log.Fatal(err)
	}
}

func test(path string) bool {
	for _, st := range tests {
		info, err := st.stat(path)
		if err != nil {
			continue
		}
		for _, ft := range st.tests {
			if *ft.flag && ft.test(info) {
				return true
			}
		}
	}
	return false
}

func yes(info os.FileInfo) bool {
	return true
}

func size(info os.FileInfo) bool {
	return info.Size() > 0
}

func mode(mode os.FileMode) testFunc {
	return func(info os.FileInfo) bool {
		return info.Mode()&mode > 0
	}
}

func perm(perm os.FileMode) testFunc {
	return func(info os.FileInfo) bool {
		return info.Mode().Perm()&perm > 0
	}
}

func not(test testFunc) testFunc {
	return func(info os.FileInfo) bool {
		return !test(info)
	}
}
<commit_msg>stest: Fail when given arguments<commit_after>/*
Stest tests reads files from stdin (one per line)
and prints every file that passes all given tests.

Usage:

	stest <option>...

The options are:

	-L
		Matches files that are symbolic links
	-S
		Matches files that are sockets
	-d
		Matches files that are directories
	-e
		Matches files that exist (regardless of type)
	-f
		Matches files that are regular files
	-g
		Matches files whose set group ID flag is set
	-k
		Matches files whose sticky bit is set
	-p
		Matches files that are named pipes (FIFOs)
	-r
		Matches files that are readable
	-s
		Matches files that have a size greater than zero
	-u
		Matches files whose set user ID flag is set
	-w
		Matches files that are writable
	-x
		Matches files that are executable
	-v
		Invert the sense of matching, to select non-matching files
*/
package main

import (
	"bufio"
	"flag"
	"fmt"
	"log"
	"os"
)

type statTests struct {
	stat  func(name string) (info os.FileInfo, err error)
	tests []flagTest
}
type flagTest struct {
	flag *bool
	test testFunc
}
type testFunc func(info os.FileInfo) bool

var (
	this = os.Args[0]
	L    = flag.Bool("L", false, "Matches files that are symbolic links")
	S    = flag.Bool("S", false, "Matches files that are sockets")
	d    = flag.Bool("d", false, "Matches files that are directories")
	e    = flag.Bool("e", false, "Matches files that exist (regardless of type)")
	f    = flag.Bool("f", false, "Matches files that are regular files")
	g    = flag.Bool("g", false, "Matches files whose set group ID flag is set")
	k    = flag.Bool("k", false, "Matches files whose sticky bit is set")
	p    = flag.Bool("p", false, "Matches files that are named pipes (FIFOs)")
	r    = flag.Bool("r", false, "Matches files that are readable")
	s    = flag.Bool("s", false, "Matches files that have a size greater than zero")
	u    = flag.Bool("u", false, "Matches files whose set user ID flag is set")
	v    = flag.Bool("v", false, "Invert the sense of matching, to select non-matching files")
	w    = flag.Bool("w", false, "Matches files that are writable")
	x    = flag.Bool("x", false, "Matches files that are executable")

	tests = []statTests{
		{os.Lstat, []flagTest{
			{L, mode(os.ModeSymlink)}},
		},
		{os.Stat, []flagTest{
			{e, yes},
			{s, size},
			{f, not(mode(os.ModeType))},
			{d, mode(os.ModeDir)},
			{p, mode(os.ModeNamedPipe)},
			{S, mode(os.ModeSocket)},
			{u, mode(os.ModeSetuid)},
			{g, mode(os.ModeSetgid)},
			{k, mode(os.ModeSticky)},
			{r, perm(0444)},
			{w, perm(0222)},
			{x, perm(0111)},
		}},
	}
)

func main() {
	log.SetFlags(0)
	log.SetPrefix("stest: ")
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "usage: %s <option>...\n", this)
		flag.PrintDefaults()
	}
	flag.Parse()
	if flag.NFlag() == 0 || flag.NArg() > 0 {
		flag.Usage()
		os.Exit(2)
	}
	in := bufio.NewScanner(os.Stdin)
	for in.Scan() {
		path := in.Text()
		if *v != test(path) {
			fmt.Println(path)
		}
	}
	if err := in.Err(); err != nil {
		log.Fatal(err)
	}
}

func test(path string) bool {
	for _, st := range tests {
		info, err := st.stat(path)
		if err != nil {
			continue
		}
		for _, ft := range st.tests {
			if *ft.flag && ft.test(info) {
				return true
			}
		}
	}
	return false
}

func yes(info os.FileInfo) bool {
	return true
}

func size(info os.FileInfo) bool {
	return info.Size() > 0
}

func mode(mode os.FileMode) testFunc {
	return func(info os.FileInfo) bool {
		return info.Mode()&mode > 0
	}
}

func perm(perm os.FileMode) testFunc {
	return func(info os.FileInfo) bool {
		return info.Mode().Perm()&perm > 0
	}
}

func not(test testFunc) testFunc {
	return func(info os.FileInfo) bool {
		return !test(info)
	}
}
<|endoftext|>


<commit_before>package moby

import (
	"archive/tar"
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"path"
	"strings"

	"github.com/containerd/containerd/reference"
	"github.com/opencontainers/runtime-spec/specs-go"
	log "github.com/sirupsen/logrus"
)

type tarWriter interface {
	Close() error
	Flush() error
	Write(b []byte) (n int, err error)
	WriteHeader(hdr *tar.Header) error
}

// This uses Docker to convert a Docker image into a tarball. It would be an improvement if we
// used the containerd libraries to do this instead locally direct from a local image
// cache as it would be much simpler.

// Unfortunately there are some files that Docker always makes appear in a running image and
// export shows them. In particular we have no way for a user to specify their own resolv.conf.
// Even if we were not using docker export to get the image, users of docker build cannot override
// the resolv.conf either, as it is not writeable and bind mounted in.

var exclude = map[string]bool{
	".dockerenv":   true,
	"Dockerfile":   true,
	"dev/console":  true,
	"dev/pts":      true,
	"dev/shm":      true,
	"etc/hostname": true,
}

var replace = map[string]string{
	"etc/hosts": `127.0.0.1       localhost
::1     localhost ip6-localhost ip6-loopback
fe00::0 ip6-localnet
ff00::0 ip6-mcastprefix
ff02::1 ip6-allnodes
ff02::2 ip6-allrouters
`,
	"etc/resolv.conf": `
# no resolv.conf configured
`,
}

// tarPrefix creates the leading directories for a path
func tarPrefix(path string, tw tarWriter) error {
	if path == "" {
		return nil
	}
	if path[len(path)-1] != byte('/') {
		return fmt.Errorf("path does not end with /: %s", path)
	}
	path = path[:len(path)-1]
	if path[0] == byte('/') {
		return fmt.Errorf("path should be relative: %s", path)
	}
	mkdir := ""
	for _, dir := range strings.Split(path, "/") {
		mkdir = mkdir + dir
		hdr := &tar.Header{
			Name:     mkdir,
			Mode:     0755,
			Typeflag: tar.TypeDir,
			Format:   tar.FormatPAX,
		}
		if err := tw.WriteHeader(hdr); err != nil {
			return err
		}
		mkdir = mkdir + "/"
	}
	return nil
}

// ImageTar takes a Docker image and outputs it to a tar stream
func ImageTar(ref *reference.Spec, prefix string, tw tarWriter, trust bool, pull bool, resolv string) (e error) {
	log.Debugf("image tar: %s %s", ref, prefix)
	if prefix != "" && prefix[len(prefix)-1] != byte('/') {
		return fmt.Errorf("prefix does not end with /: %s", prefix)
	}

	err := tarPrefix(prefix, tw)
	if err != nil {
		return err
	}

	if pull || trust {
		err := dockerPull(ref, pull, trust)
		if err != nil {
			return fmt.Errorf("Could not pull image %s: %v", ref, err)
		}
	}
	container, err := dockerCreate(ref.String())
	if err != nil {
		// if the image wasn't found, pull it down.  Bail on other errors.
		if strings.Contains(err.Error(), "No such image") {
			err := dockerPull(ref, true, trust)
			if err != nil {
				return fmt.Errorf("Could not pull image %s: %v", ref, err)
			}
			container, err = dockerCreate(ref.String())
			if err != nil {
				return fmt.Errorf("Failed to docker create image %s: %v", ref, err)
			}
		} else {
			return fmt.Errorf("Failed to create docker image %s: %v", ref, err)
		}
	}
	contents, err := dockerExport(container)
	if err != nil {
		return fmt.Errorf("Failed to docker export container from container %s: %v", container, err)
	}
	defer func() {
		contents.Close()

		if err := dockerRm(container); e == nil && err != nil {
			e = fmt.Errorf("Failed to docker rm container %s: %v", container, err)
		}
	}()

	// now we need to filter out some files from the resulting tar archive

	tr := tar.NewReader(contents)

	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}
		if exclude[hdr.Name] {
			log.Debugf("image tar: %s %s exclude %s", ref, prefix, hdr.Name)
			_, err = io.Copy(ioutil.Discard, tr)
			if err != nil {
				return err
			}
		} else if replace[hdr.Name] != "" {
			if hdr.Name != "etc/resolv.conf" || resolv == "" {
				contents := replace[hdr.Name]
				hdr.Size = int64(len(contents))
				hdr.Name = prefix + hdr.Name
				log.Debugf("image tar: %s %s add %s", ref, prefix, hdr.Name)
				if err := tw.WriteHeader(hdr); err != nil {
					return err
				}
				buf := bytes.NewBufferString(contents)
				_, err = io.Copy(tw, buf)
				if err != nil {
					return err
				}
			} else {
				// replace resolv.conf with specified symlink
				hdr.Name = prefix + hdr.Name
				hdr.Size = 0
				hdr.Typeflag = tar.TypeSymlink
				hdr.Linkname = resolv
				log.Debugf("image tar: %s %s add resolv symlink /etc/resolv.conf -> %s", ref, prefix, resolv)
				if err := tw.WriteHeader(hdr); err != nil {
					return err
				}
			}
			_, err = io.Copy(ioutil.Discard, tr)
			if err != nil {
				return err
			}
		} else {
			log.Debugf("image tar: %s %s add %s", ref, prefix, hdr.Name)
			hdr.Name = prefix + hdr.Name
			if hdr.Typeflag == tar.TypeLink {
				// hard links are referenced by full path so need to be adjusted
				hdr.Linkname = prefix + hdr.Linkname
			}
			if err := tw.WriteHeader(hdr); err != nil {
				return err
			}
			_, err = io.Copy(tw, tr)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

// ImageBundle produces an OCI bundle at the given path in a tarball, given an image and a config.json
func ImageBundle(prefix string, ref *reference.Spec, config []byte, runtime Runtime, tw tarWriter, trust bool, pull bool, readonly bool, dupMap map[string]string) error { // nolint: lll
	// if read only, just unpack in rootfs/ but otherwise set up for overlay
	rootExtract := "rootfs"
	if !readonly {
		rootExtract = "lower"
	}

	// See if we have extracted this image previously
	root := path.Join(prefix, rootExtract)
	var foundElsewhere = dupMap[ref.String()] != ""
	if !foundElsewhere {
		if err := ImageTar(ref, root+"/", tw, trust, pull, ""); err != nil {
			return err
		}
		dupMap[ref.String()] = root
	} else {
		if err := tarPrefix(prefix+"/", tw); err != nil {
			return err
		}
		root = dupMap[ref.String()]
	}

	hdr := &tar.Header{
		Name:   path.Join(prefix, "config.json"),
		Mode:   0644,
		Size:   int64(len(config)),
		Format: tar.FormatPAX,
	}
	if err := tw.WriteHeader(hdr); err != nil {
		return err
	}
	buf := bytes.NewBuffer(config)
	if _, err := io.Copy(tw, buf); err != nil {
		return err
	}

	var rootfsMounts []specs.Mount
	if !readonly {
		// add a tmp directory to be used as a mount point for tmpfs for upper, work
		tmp := path.Join(prefix, "tmp")
		hdr = &tar.Header{
			Name:     tmp,
			Mode:     0755,
			Typeflag: tar.TypeDir,
			Format:   tar.FormatPAX,
		}
		if err := tw.WriteHeader(hdr); err != nil {
			return err
		}
		// add rootfs as merged mount point
		hdr = &tar.Header{
			Name:     path.Join(prefix, "rootfs"),
			Mode:     0755,
			Typeflag: tar.TypeDir,
			Format:   tar.FormatPAX,
		}
		if err := tw.WriteHeader(hdr); err != nil {
			return err
		}
		overlayOptions := []string{"lowerdir=/" + root, "upperdir=/" + path.Join(tmp, "upper"), "workdir=/" + path.Join(tmp, "work")}
		rootfsMounts = []specs.Mount{
			{Source: "tmpfs", Type: "tmpfs", Destination: "/" + tmp},
			// remount private as nothing else should see the temporary layers
			{Destination: "/" + tmp, Options: []string{"remount", "private"}},
			{Source: "overlay", Type: "overlay", Destination: "/" + path.Join(prefix, "rootfs"), Options: overlayOptions},
		}
	} else {
		if foundElsewhere {
			// we need to make the mountpoint at rootfs
			hdr = &tar.Header{
				Name:     path.Join(prefix, "rootfs"),
				Mode:     0755,
				Typeflag: tar.TypeDir,
				Format:   tar.FormatPAX,
			}
			if err := tw.WriteHeader(hdr); err != nil {
				return err
			}
		}
		// either bind from another location, or bind from self to make sure it is a mountpoint as runc prefers this
		rootfsMounts = []specs.Mount{
			{Source: "/" + root, Destination: "/" + path.Join(prefix, "rootfs"), Options: []string{"bind"}},
		}
	}

	// Prepend the rootfs onto the user specified mounts.
	runtimeMounts := append(rootfsMounts, *runtime.Mounts...)
	runtime.Mounts = &runtimeMounts

	// write the runtime config
	runtimeConfig, err := json.MarshalIndent(runtime, "", "    ")
	if err != nil {
		return fmt.Errorf("Failed to create runtime config for %s: %v", ref, err)
	}

	hdr = &tar.Header{
		Name:   path.Join(prefix, "runtime.json"),
		Mode:   0644,
		Size:   int64(len(runtimeConfig)),
		Format: tar.FormatPAX,
	}
	if err := tw.WriteHeader(hdr); err != nil {
		return err
	}
	buf = bytes.NewBuffer(runtimeConfig)
	if _, err := io.Copy(tw, buf); err != nil {
		return err
	}

	log.Debugf("image bundle: %s %s cfg: %s runtime: %s", prefix, ref, string(config), string(runtimeConfig))

	return nil
}
<commit_msg>Fix #3091 enforce all outgoing tar files to use PAX<commit_after>package moby

import (
	"archive/tar"
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"path"
	"strings"

	"github.com/containerd/containerd/reference"
	"github.com/opencontainers/runtime-spec/specs-go"
	log "github.com/sirupsen/logrus"
)

type tarWriter interface {
	Close() error
	Flush() error
	Write(b []byte) (n int, err error)
	WriteHeader(hdr *tar.Header) error
}

// This uses Docker to convert a Docker image into a tarball. It would be an improvement if we
// used the containerd libraries to do this instead locally direct from a local image
// cache as it would be much simpler.

// Unfortunately there are some files that Docker always makes appear in a running image and
// export shows them. In particular we have no way for a user to specify their own resolv.conf.
// Even if we were not using docker export to get the image, users of docker build cannot override
// the resolv.conf either, as it is not writeable and bind mounted in.

var exclude = map[string]bool{
	".dockerenv":   true,
	"Dockerfile":   true,
	"dev/console":  true,
	"dev/pts":      true,
	"dev/shm":      true,
	"etc/hostname": true,
}

var replace = map[string]string{
	"etc/hosts": `127.0.0.1       localhost
::1     localhost ip6-localhost ip6-loopback
fe00::0 ip6-localnet
ff00::0 ip6-mcastprefix
ff02::1 ip6-allnodes
ff02::2 ip6-allrouters
`,
	"etc/resolv.conf": `
# no resolv.conf configured
`,
}

// tarPrefix creates the leading directories for a path
func tarPrefix(path string, tw tarWriter) error {
	if path == "" {
		return nil
	}
	if path[len(path)-1] != byte('/') {
		return fmt.Errorf("path does not end with /: %s", path)
	}
	path = path[:len(path)-1]
	if path[0] == byte('/') {
		return fmt.Errorf("path should be relative: %s", path)
	}
	mkdir := ""
	for _, dir := range strings.Split(path, "/") {
		mkdir = mkdir + dir
		hdr := &tar.Header{
			Name:     mkdir,
			Mode:     0755,
			Typeflag: tar.TypeDir,
			Format:   tar.FormatPAX,
		}
		if err := tw.WriteHeader(hdr); err != nil {
			return err
		}
		mkdir = mkdir + "/"
	}
	return nil
}

// ImageTar takes a Docker image and outputs it to a tar stream
func ImageTar(ref *reference.Spec, prefix string, tw tarWriter, trust bool, pull bool, resolv string) (e error) {
	log.Debugf("image tar: %s %s", ref, prefix)
	if prefix != "" && prefix[len(prefix)-1] != byte('/') {
		return fmt.Errorf("prefix does not end with /: %s", prefix)
	}

	err := tarPrefix(prefix, tw)
	if err != nil {
		return err
	}

	if pull || trust {
		err := dockerPull(ref, pull, trust)
		if err != nil {
			return fmt.Errorf("Could not pull image %s: %v", ref, err)
		}
	}
	container, err := dockerCreate(ref.String())
	if err != nil {
		// if the image wasn't found, pull it down.  Bail on other errors.
		if strings.Contains(err.Error(), "No such image") {
			err := dockerPull(ref, true, trust)
			if err != nil {
				return fmt.Errorf("Could not pull image %s: %v", ref, err)
			}
			container, err = dockerCreate(ref.String())
			if err != nil {
				return fmt.Errorf("Failed to docker create image %s: %v", ref, err)
			}
		} else {
			return fmt.Errorf("Failed to create docker image %s: %v", ref, err)
		}
	}
	contents, err := dockerExport(container)
	if err != nil {
		return fmt.Errorf("Failed to docker export container from container %s: %v", container, err)
	}
	defer func() {
		contents.Close()

		if err := dockerRm(container); e == nil && err != nil {
			e = fmt.Errorf("Failed to docker rm container %s: %v", container, err)
		}
	}()

	// now we need to filter out some files from the resulting tar archive

	tr := tar.NewReader(contents)

	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}
		// force PAX format, since it allows for unlimited Name/Linkname
		// and we move all files below prefix.
		hdr.Format = tar.FormatPAX
		if exclude[hdr.Name] {
			log.Debugf("image tar: %s %s exclude %s", ref, prefix, hdr.Name)
			_, err = io.Copy(ioutil.Discard, tr)
			if err != nil {
				return err
			}
		} else if replace[hdr.Name] != "" {
			if hdr.Name != "etc/resolv.conf" || resolv == "" {
				contents := replace[hdr.Name]
				hdr.Size = int64(len(contents))
				hdr.Name = prefix + hdr.Name
				log.Debugf("image tar: %s %s add %s", ref, prefix, hdr.Name)
				if err := tw.WriteHeader(hdr); err != nil {
					return err
				}
				buf := bytes.NewBufferString(contents)
				_, err = io.Copy(tw, buf)
				if err != nil {
					return err
				}
			} else {
				// replace resolv.conf with specified symlink
				hdr.Name = prefix + hdr.Name
				hdr.Size = 0
				hdr.Typeflag = tar.TypeSymlink
				hdr.Linkname = resolv
				log.Debugf("image tar: %s %s add resolv symlink /etc/resolv.conf -> %s", ref, prefix, resolv)
				if err := tw.WriteHeader(hdr); err != nil {
					return err
				}
			}
			_, err = io.Copy(ioutil.Discard, tr)
			if err != nil {
				return err
			}
		} else {
			log.Debugf("image tar: %s %s add %s", ref, prefix, hdr.Name)
			hdr.Name = prefix + hdr.Name
			if hdr.Typeflag == tar.TypeLink {
				// hard links are referenced by full path so need to be adjusted
				hdr.Linkname = prefix + hdr.Linkname
			}
			if err := tw.WriteHeader(hdr); err != nil {
				return err
			}
			_, err = io.Copy(tw, tr)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

// ImageBundle produces an OCI bundle at the given path in a tarball, given an image and a config.json
func ImageBundle(prefix string, ref *reference.Spec, config []byte, runtime Runtime, tw tarWriter, trust bool, pull bool, readonly bool, dupMap map[string]string) error { // nolint: lll
	// if read only, just unpack in rootfs/ but otherwise set up for overlay
	rootExtract := "rootfs"
	if !readonly {
		rootExtract = "lower"
	}

	// See if we have extracted this image previously
	root := path.Join(prefix, rootExtract)
	var foundElsewhere = dupMap[ref.String()] != ""
	if !foundElsewhere {
		if err := ImageTar(ref, root+"/", tw, trust, pull, ""); err != nil {
			return err
		}
		dupMap[ref.String()] = root
	} else {
		if err := tarPrefix(prefix+"/", tw); err != nil {
			return err
		}
		root = dupMap[ref.String()]
	}

	hdr := &tar.Header{
		Name:   path.Join(prefix, "config.json"),
		Mode:   0644,
		Size:   int64(len(config)),
		Format: tar.FormatPAX,
	}
	if err := tw.WriteHeader(hdr); err != nil {
		return err
	}
	buf := bytes.NewBuffer(config)
	if _, err := io.Copy(tw, buf); err != nil {
		return err
	}

	var rootfsMounts []specs.Mount
	if !readonly {
		// add a tmp directory to be used as a mount point for tmpfs for upper, work
		tmp := path.Join(prefix, "tmp")
		hdr = &tar.Header{
			Name:     tmp,
			Mode:     0755,
			Typeflag: tar.TypeDir,
			Format:   tar.FormatPAX,
		}
		if err := tw.WriteHeader(hdr); err != nil {
			return err
		}
		// add rootfs as merged mount point
		hdr = &tar.Header{
			Name:     path.Join(prefix, "rootfs"),
			Mode:     0755,
			Typeflag: tar.TypeDir,
			Format:   tar.FormatPAX,
		}
		if err := tw.WriteHeader(hdr); err != nil {
			return err
		}
		overlayOptions := []string{"lowerdir=/" + root, "upperdir=/" + path.Join(tmp, "upper"), "workdir=/" + path.Join(tmp, "work")}
		rootfsMounts = []specs.Mount{
			{Source: "tmpfs", Type: "tmpfs", Destination: "/" + tmp},
			// remount private as nothing else should see the temporary layers
			{Destination: "/" + tmp, Options: []string{"remount", "private"}},
			{Source: "overlay", Type: "overlay", Destination: "/" + path.Join(prefix, "rootfs"), Options: overlayOptions},
		}
	} else {
		if foundElsewhere {
			// we need to make the mountpoint at rootfs
			hdr = &tar.Header{
				Name:     path.Join(prefix, "rootfs"),
				Mode:     0755,
				Typeflag: tar.TypeDir,
				Format:   tar.FormatPAX,
			}
			if err := tw.WriteHeader(hdr); err != nil {
				return err
			}
		}
		// either bind from another location, or bind from self to make sure it is a mountpoint as runc prefers this
		rootfsMounts = []specs.Mount{
			{Source: "/" + root, Destination: "/" + path.Join(prefix, "rootfs"), Options: []string{"bind"}},
		}
	}

	// Prepend the rootfs onto the user specified mounts.
	runtimeMounts := append(rootfsMounts, *runtime.Mounts...)
	runtime.Mounts = &runtimeMounts

	// write the runtime config
	runtimeConfig, err := json.MarshalIndent(runtime, "", "    ")
	if err != nil {
		return fmt.Errorf("Failed to create runtime config for %s: %v", ref, err)
	}

	hdr = &tar.Header{
		Name:   path.Join(prefix, "runtime.json"),
		Mode:   0644,
		Size:   int64(len(runtimeConfig)),
		Format: tar.FormatPAX,
	}
	if err := tw.WriteHeader(hdr); err != nil {
		return err
	}
	buf = bytes.NewBuffer(runtimeConfig)
	if _, err := io.Copy(tw, buf); err != nil {
		return err
	}

	log.Debugf("image bundle: %s %s cfg: %s runtime: %s", prefix, ref, string(config), string(runtimeConfig))

	return nil
}
<|endoftext|>


<commit_before>package data

//DB is used for referencing db type systems.
const DB = "db"

//SERVER is used for referencing server type systems.
const SERVER = "server"

//TOOL is used for referencing tool type systems.
const TOOL = "tool"

//WEB is used for referencing web type systems.
const WEB = "web"

// Definition stores information about a system, used for importing data only.
type Definition struct {
	ID          string
	Name        string
	Type        string
	User        string
	Password    string
	URL         string
	Server      string
	DB          string
	Host        string
	Port        string
	SID         string
	Env         string
	Location    []string
	Responsible []string
	Notes       []string
}

// YamlData stores information about all systems, used for importing data only.
type YamlData struct {
	Defs []Definition
}

//ConfigMeta contains properties which are not part of the system information, for internal use.
type ConfigMeta struct {
	IsValid bool
}

//User contains information about login credentials.
type User struct {
	Name     string `json:"name"`
	User     string `json:"user"`
	Password string `json:"password"`
}

// Config stores data about a system, used for managing and exporting data.
type Config struct {
	ID          string     `json:"id"`
	Name        string     `json:"name"`
	Type        string     `json:"type"`
	User        string     `json:"user,omitempty"`
	Password    string     `json:"password,omitempty"`
	URL         string     `json:"url"`
	Server      string     `json:"server,omitempty"`
	DB          string     `json:"db,omitempty"`
	Host        string     `json:"host,omitempty"`
	Port        string     `json:"port,omitempty"`
	SID         string     `json:"sid,omitempty"`
	Env         string     `json:"env,omitempty"`
	Location    []string   `json:"location,omitempty"`
	Responsible []string   `json:"responsible,omitempty"`
	Notes       []string   `json:"notes,omitempty"`
	Meta        ConfigMeta `json:"-"`
}
<commit_msg>Add "Users" to config struct<commit_after>package data

//DB is used for referencing db type systems.
const DB = "db"

//SERVER is used for referencing server type systems.
const SERVER = "server"

//TOOL is used for referencing tool type systems.
const TOOL = "tool"

//WEB is used for referencing web type systems.
const WEB = "web"

// Definition stores information about a system, used for importing data only.
type Definition struct {
	ID          string
	Name        string
	Type        string
	User        string
	Password    string
	URL         string
	Server      string
	DB          string
	Host        string
	Port        string
	SID         string
	Env         string
	Location    []string
	Responsible []string
	Notes       []string
}

// YamlData stores information about all systems, used for importing data only.
type YamlData struct {
	Defs []Definition
}

//ConfigMeta contains properties which are not part of the system information, for internal use.
type ConfigMeta struct {
	IsValid bool
}

//User contains information about login credentials.
type User struct {
	Name     string `json:"name"`
	User     string `json:"user"`
	Password string `json:"password"`
}

// Config stores data about a system, used for managing and exporting data.
type Config struct {
	ID          string     `json:"id"`
	Name        string     `json:"name"`
	Type        string     `json:"type"`
	User        string     `json:"user,omitempty"`
	Password    string     `json:"password,omitempty"`
	URL         string     `json:"url"`
	Server      string     `json:"server,omitempty"`
	DB          string     `json:"db,omitempty"`
	Host        string     `json:"host,omitempty"`
	Port        string     `json:"port,omitempty"`
	SID         string     `json:"sid,omitempty"`
	Env         string     `json:"env,omitempty"`
	Location    []string   `json:"location,omitempty"`
	Responsible []string   `json:"responsible,omitempty"`
	Notes       []string   `json:"notes,omitempty"`
	Users       []User     `json:"users,omitempty"`
	Meta        ConfigMeta `json:"-"`
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"runtime/pprof"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/ASPecherkin/TabletHive/tablet"
)

// Authtokens stores json represent array of tokens
type Authtokens struct {
	Tokens []string `json:"tokens"`
}

// HiveConfig gather all of needed configs
type HiveConfig struct {
	ServerURL    string               `json:"server"`
	TokensPath   string               `json:"token_file_path"`
	SecondsDelay int64                `json:"delay"`
	Endpoints    map[string]Endpoints `json:"endpoints"`
}

// Endpoints stores all urls for requests
type Endpoints struct {
	URL   string `json:"url"`
	Delay int    `json:"delay"`
}

// TabletClient one unit of hive
type TabletClient struct {
	ID         string
	Token      string
	DeviceID   string
	RespObj    tablet.Ride
	Rawresp    string
	StatusCode int
	ch         chan string
}

// HiveResults stores all results of running
type HiveResults struct {
	When          string   `json:"when"`
	ElapsedTime   float64  `json:"elapsed_time"`
	GetResults    []Result `json:"get_results"`
	UpdateResults []Result `json:"update_results"`
	OthersResults []Result `json:"others_result"`
}

// Result store meta info about every request
type Result struct {
	RequestType   string  `json:"type"`
	AuthToken     string  `json:"token"`
	RequestURL    string  `json:"url"`
	Responce      string  `json:"responce"`
	RequestStatus int     `json:"status_code"`
	ProcessedTime float64 `json:"processed_time"`
}

// GetRide create connect amd get ride for that token
func (t *TabletClient) GetRide(wg *sync.WaitGroup, cfg *HiveConfig, res chan Result) error {
	defer wg.Done()
	client := &http.Client{}
	url := strings.Join(append([]string{cfg.ServerURL, cfg.Endpoints["get_rides"].URL, t.DeviceID}), "")
	request, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return err
	}
	request.Header.Add("HTTP-AUTH-TOKEN", t.Token)
	time.Sleep(time.Duration(cfg.Endpoints["get_rides"].Delay))
	start := time.Now()
	responce, err := client.Do(request)
	if err != nil {
		log.Fatal(err)
	}
	jsonData, err := ioutil.ReadAll(responce.Body)
	res <- Result{RequestType: "GET_RIDE", AuthToken: t.Token, RequestURL: url, RequestStatus: responce.StatusCode, ProcessedTime: time.Since(start).Seconds()}
	defer responce.Body.Close()
	if err != nil && err != io.EOF {
		fmt.Println("error reading from responce Body", err)
		return err
	}
	t.StatusCode = responce.StatusCode
	if responce.StatusCode == 404 {
		t.Rawresp = string(jsonData)
		return nil
	} else if responce.StatusCode == 200 {
		var answer tablet.Ride
		err = json.Unmarshal([]byte(jsonData), &answer)
		if err != nil {
			fmt.Printf("err: %s  with token : %s when unmarhal this   \n", err, t.Token)
		}
		t.RespObj, t.Rawresp = answer, string(jsonData)
		return nil
	} else {
		t.Rawresp = string(jsonData)
		return nil
	}
}

// ConsumeResults will store all of results in one HiveResults
func ConsumeResults(input chan Result, cfg *HiveConfig, testResult *HiveResults) {
	for i := range input {
		switch i.RequestType {
		case "CONSUME":
			testResult.UpdateResults = append(testResult.UpdateResults, i)
		case "GET_RIDE":
			testResult.GetResults = append(testResult.GetResults, i)
		default:
			testResult.OthersResults = append(testResult.OthersResults, i)
		}
	}
}

// getTokens read file from path and stores they in tokens
func getTokens(path string) (tokens Authtokens, err error) {
	tokens = Authtokens{}
	content, err := ioutil.ReadFile(path)
	if err != nil {
		fmt.Println("error while open file with tokens", err)
		return tokens, err
	}
	err = json.Unmarshal(content, &tokens)
	return tokens, nil
}

// GetConfigJSON func get full path to config.json and store it in HiveConfig struct
func GetConfigJSON(jsonFile string) (cfg HiveConfig, err error) {
	jsonDoc, err := ioutil.ReadFile(jsonFile)
	if err != nil {
		log.Fatalf("Could not read config file: %s ", err)
		return
	}
	err = json.Unmarshal(jsonDoc, &cfg)
	return cfg, err
}

// ConsumeRidePoints func create serias of request emulates real status updating
func ConsumeRidePoints(authToken string, points []tablet.RidePoint, wg *sync.WaitGroup, cfg *HiveConfig, res chan Result) error {
	defer wg.Done()
	client := &http.Client{}
	requestURL := cfg.ServerURL + cfg.Endpoints["update_status"].URL
	for _, v := range points {
		var jsonStr = []byte(`{"ride_point":{"status":"departure"}}`)
		t := strings.Join(append([]string{requestURL}, strconv.Itoa(int(v.ID))), "")
		req, err := http.NewRequest("PUT", t, bytes.NewBuffer(jsonStr))
		req.Header.Set("HTTP-AUTH-TOKEN", authToken)
		req.Header.Set("Content-Type", "application/json")
		time.Sleep(time.Duration(cfg.Endpoints["update_status"].Delay))
		start := time.Now()
		resp, err := client.Do(req)
        jsonData, err := ioutil.ReadAll(resp.Body)
		resp.Body.Close()
        if err != nil {
            log.Fatalln(err)
        }
		res <- Result{RequestType: "CONSUME", AuthToken: authToken, RequestURL: t, RequestStatus: resp.StatusCode, Responce: string(jsonData), ProcessedTime: time.Since(start).Seconds()}
		if err != nil {
			log.Panicln(err)
			os.Exit(1)
			return err
		}
	}
	return nil
}

var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")
var memprofile = flag.String("memprofile", "", "write mem profile to file")

func main() {
	start := time.Now()
	fmt.Fprintf(os.Stdout, "We start at: %v\n", start)
	cfg, err := GetConfigJSON("./config.json")
	if err != nil {
		log.Fatal(err)
	}
	flag.Parse()
	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			fmt.Println("Error: ", err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}
	if *memprofile != "" {
		f, err := os.Create(*memprofile)
		if err != nil {
			fmt.Println("Error: ", err)
		}
		pprof.WriteHeapProfile(f)
	}
	var wg sync.WaitGroup
	tokens, err := getTokens(cfg.TokensPath)
	hive := make([]TabletClient, 0, 3000)
	if err != nil {
		fmt.Println("error while read tokens.json", err)
	}
	for k, v := range tokens.Tokens[0:] {
		hive = append(hive, TabletClient{ID: v, Token: v, DeviceID: strconv.Itoa(k + 1)})
	}
	fmt.Printf("we have %d tokens \n", len(hive))
	res := make(chan Result, 5)
	testCase := HiveResults{When: start.String()}
	go ConsumeResults(res, &cfg, &testCase)
	for k := range hive {
		wg.Add(1)
		go hive[k].GetRide(&wg, &cfg, res)
	}
	wg.Wait()
	ridePoints := make(map[string][]tablet.RidePoint)
	for _, tablerClient := range hive {
		for _, factRides := range tablerClient.RespObj.FactRides {
			if len(factRides.RidePoints) != 0 {
				ridePoints[tablerClient.Token] = factRides.RidePoints
			}
		}
	}
    fmt.Println("Start updating statuses")
	for k := range ridePoints {
		wg.Add(1)
		go ConsumeRidePoints(k, ridePoints[k], &wg, &cfg, res)
	}
	wg.Wait()
	secs := time.Since(start).Seconds()
	fmt.Printf("we all done with: %.5fs \n", secs)
	testCase.ElapsedTime = secs
	jsondata, err := json.Marshal(testCase)
	if err != nil {
		fmt.Println(err)
	}
	ResultFile, err := os.Create("./TestCase.json")
	defer ResultFile.Close()
	if err != nil {
		fmt.Println(err)
	}
	ResultFile.Write(jsondata)
}
<commit_msg>add spew for more readable obj prints<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"runtime/pprof"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/davecgh/go-spew/spew"

	"github.com/ASPecherkin/TabletHive/tablet"
)

// Authtokens stores json represent array of tokens
type Authtokens struct {
	Tokens []string `json:"tokens"`
}

// HiveConfig gather all of needed configs
type HiveConfig struct {
	ServerURL    string               `json:"server"`
	TokensPath   string               `json:"token_file_path"`
	SecondsDelay int64                `json:"delay"`
	Endpoints    map[string]Endpoints `json:"endpoints"`
}

// Endpoints stores all urls for requests
type Endpoints struct {
	URL   string `json:"url"`
	Delay int    `json:"delay"`
}

// TabletClient one unit of hive
type TabletClient struct {
	ID         string
	Token      string
	DeviceID   string
	RespObj    tablet.Ride
	Rawresp    string
	StatusCode int
	ch         chan string
}

// HiveResults stores all results of running
type HiveResults struct {
	When          string   `json:"when"`
	ElapsedTime   float64  `json:"elapsed_time"`
	GetResults    []Result `json:"get_results"`
	UpdateResults []Result `json:"update_results"`
	OthersResults []Result `json:"others_result"`
}

// Result store meta info about every request
type Result struct {
	RequestType   string  `json:"type"`
	AuthToken     string  `json:"token"`
	RequestURL    string  `json:"url"`
	Responce      string  `json:"responce"`
	RequestStatus int     `json:"status_code"`
	ProcessedTime float64 `json:"processed_time"`
}

// GetRide create connect amd get ride for that token
func (t *TabletClient) GetRide(wg *sync.WaitGroup, cfg *HiveConfig, res chan Result) error {
	defer wg.Done()
	client := &http.Client{}
	url := strings.Join(append([]string{cfg.ServerURL, cfg.Endpoints["get_rides"].URL, t.DeviceID}), "")
	request, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return err
	}
	request.Header.Add("HTTP-AUTH-TOKEN", t.Token)
	time.Sleep(time.Duration(cfg.Endpoints["get_rides"].Delay))
	start := time.Now()
	responce, err := client.Do(request)
	if err != nil {
		log.Fatal(err)
	}
	jsonData, err := ioutil.ReadAll(responce.Body)
	res <- Result{RequestType: "GET_RIDE", AuthToken: t.Token, RequestURL: url, RequestStatus: responce.StatusCode, ProcessedTime: time.Since(start).Seconds()}
	defer responce.Body.Close()
	if err != nil && err != io.EOF {
		fmt.Println("error reading from responce Body", err)
		return err
	}
	t.StatusCode = responce.StatusCode
	if responce.StatusCode == 404 {
		t.Rawresp = string(jsonData)
		return nil
	} else if responce.StatusCode == 200 {
		var answer tablet.Ride
		err = json.Unmarshal([]byte(jsonData), &answer)
		if err != nil {
			spew.Printf("err: %s  with token : %s when unmarhal this   \n", err, t)
		}
		t.RespObj, t.Rawresp = answer, string(jsonData)
		return nil
	} else {
		t.Rawresp = string(jsonData)
		return nil
	}
}

// ConsumeResults will store all of results in one HiveResults
func ConsumeResults(input chan Result, cfg *HiveConfig, testResult *HiveResults) {
	for i := range input {
		switch i.RequestType {
		case "CONSUME":
			testResult.UpdateResults = append(testResult.UpdateResults, i)
		case "GET_RIDE":
			testResult.GetResults = append(testResult.GetResults, i)
		default:
			testResult.OthersResults = append(testResult.OthersResults, i)
		}
	}
}

// getTokens read file from path and stores they in tokens
func getTokens(path string) (tokens Authtokens, err error) {
	tokens = Authtokens{}
	content, err := ioutil.ReadFile(path)
	if err != nil {
		fmt.Println("error while open file with tokens", err)
		return tokens, err
	}
	err = json.Unmarshal(content, &tokens)
	return tokens, nil
}

// GetConfigJSON func get full path to config.json and store it in HiveConfig struct
func GetConfigJSON(jsonFile string) (cfg HiveConfig, err error) {
	jsonDoc, err := ioutil.ReadFile(jsonFile)
	if err != nil {
		log.Fatalf("Could not read config file: %s ", err)
		return
	}
	err = json.Unmarshal(jsonDoc, &cfg)
	return cfg, err
}

// ConsumeRidePoints func create serias of request emulates real status updating
func ConsumeRidePoints(authToken string, points []tablet.RidePoint, wg *sync.WaitGroup, cfg *HiveConfig, res chan Result) error {
	defer wg.Done()
	client := &http.Client{}
	requestURL := cfg.ServerURL + cfg.Endpoints["update_status"].URL
	for _, v := range points {
		var jsonStr = []byte(`{"ride_point":{"status":"departure"}}`)
		t := strings.Join(append([]string{requestURL}, strconv.Itoa(int(v.ID))), "")
		req, err := http.NewRequest("PUT", t, bytes.NewBuffer(jsonStr))
		req.Header.Set("HTTP-AUTH-TOKEN", authToken)
		req.Header.Set("Content-Type", "application/json")
		time.Sleep(time.Duration(cfg.Endpoints["update_status"].Delay))
		start := time.Now()
		resp, err := client.Do(req)
		jsonData, err := ioutil.ReadAll(resp.Body)
		resp.Body.Close()
		if err != nil {
			log.Fatalln(err)
		}
		res <- Result{RequestType: "CONSUME", AuthToken: authToken, RequestURL: t, RequestStatus: resp.StatusCode, Responce: string(jsonData), ProcessedTime: time.Since(start).Seconds()}
		if err != nil {
			log.Panicln(err)
			os.Exit(1)
			return err
		}
	}
	return nil
}

var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")
var memprofile = flag.String("memprofile", "", "write mem profile to file")

func main() {
	start := time.Now()
	fmt.Fprintf(os.Stdout, "We start at: %v\n", start)
	cfg, err := GetConfigJSON("./config.json")
	if err != nil {
		log.Fatal(err)
	}
	flag.Parse()
	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			fmt.Println("Error: ", err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}
	if *memprofile != "" {
		f, err := os.Create(*memprofile)
		if err != nil {
			fmt.Println("Error: ", err)
		}
		pprof.WriteHeapProfile(f)
	}
	var wg sync.WaitGroup
	tokens, err := getTokens(cfg.TokensPath)
	hive := make([]TabletClient, 0, 3000)
	if err != nil {
		fmt.Println("error while read tokens.json", err)
	}
	for k, v := range tokens.Tokens[0:] {
		hive = append(hive, TabletClient{ID: v, Token: v, DeviceID: strconv.Itoa(k + 1)})
	}
	fmt.Printf("we have %d tokens \n", len(hive))
	res := make(chan Result, 5)
	testCase := HiveResults{When: start.String()}
	go ConsumeResults(res, &cfg, &testCase)
	for k := range hive {
		wg.Add(1)
		go hive[k].GetRide(&wg, &cfg, res)
	}
	wg.Wait()
	ridePoints := make(map[string][]tablet.RidePoint)
	for _, tablerClient := range hive {
		for _, factRides := range tablerClient.RespObj.FactRides {
			if len(factRides.RidePoints) != 0 {
				ridePoints[tablerClient.Token] = factRides.RidePoints
			}
		}
	}
	fmt.Println("Start updating statuses")
	for k := range ridePoints {
		wg.Add(1)
		go ConsumeRidePoints(k, ridePoints[k], &wg, &cfg, res)
	}
	wg.Wait()
	secs := time.Since(start).Seconds()
	fmt.Printf("we all done with: %.5fs \n", secs)
	testCase.ElapsedTime = secs
	jsondata, err := json.Marshal(testCase)
	if err != nil {
		fmt.Println(err)
	}
	ResultFile, err := os.Create("./TestCase.json")
	defer ResultFile.Close()
	if err != nil {
		fmt.Println(err)
	}
	ResultFile.Write(jsondata)
}
<|endoftext|>


<commit_before>// Copyright 2018 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package targets

import (
	"github.com/google/syzkaller/prog"
)

// MakePosixMmap creates a "normal" posix mmap call that maps the target data range.
// If exec is set, the mapping is mapped as PROT_EXEC.
// If contain is set, the mapping is surrounded by PROT_NONE pages.
// These flags should be in sync with what executor.
func MakePosixMmap(target *prog.Target, exec, contain bool) func() []*prog.Call {
	meta := target.SyscallMap["mmap"]
	protRW := target.GetConst("PROT_READ") | target.GetConst("PROT_WRITE")
	if exec {
		protRW |= target.GetConst("PROT_EXEC")
	}
	flags := target.GetConst("MAP_ANONYMOUS") | target.GetConst("MAP_PRIVATE") | target.GetConst("MAP_FIXED")
	size := target.NumPages * target.PageSize
	const invalidFD = ^uint64(0)
	makeMmap := func(addr, size, prot uint64) *prog.Call {
		args := []prog.Arg{
			prog.MakeVmaPointerArg(meta.Args[0].Type, prog.DirIn, addr, size),
			prog.MakeConstArg(meta.Args[1].Type, prog.DirIn, size),
			prog.MakeConstArg(meta.Args[2].Type, prog.DirIn, prot),
			prog.MakeConstArg(meta.Args[3].Type, prog.DirIn, flags),
			prog.MakeResultArg(meta.Args[4].Type, prog.DirIn, nil, invalidFD),
		}
		i := len(args)
		// Some targets have a padding argument between fd and offset.
		if len(meta.Args) > 6 {
			args = append(args, prog.MakeConstArg(meta.Args[i].Type, prog.DirIn, 0))
			i++
		}
		args = append(args, prog.MakeConstArg(meta.Args[i].Type, prog.DirIn, 0))
		return &prog.Call{
			Meta: meta,
			Args: args,
			Ret:  prog.MakeReturnArg(meta.Ret),
		}
	}
	return func() []*prog.Call {
		if contain {
			return []*prog.Call{
				makeMmap(^target.PageSize+1, target.PageSize, 0),
				makeMmap(0, size, protRW),
				makeMmap(size, target.PageSize, 0),
			}
		}
		return []*prog.Call{makeMmap(0, size, protRW)}
	}
}

func MakeSyzMmap(target *prog.Target) func() []*prog.Call {
	meta := target.SyscallMap["syz_mmap"]
	size := target.NumPages * target.PageSize
	return func() []*prog.Call {
		return []*prog.Call{
			{
				Meta: meta,
				Args: []prog.Arg{
					prog.MakeVmaPointerArg(meta.Args[0].Type, prog.DirIn, 0, size),
					prog.MakeConstArg(meta.Args[1].Type, prog.DirIn, size),
				},
				Ret: prog.MakeReturnArg(meta.Ret),
			},
		}
	}
}

type UnixNeutralizer struct {
	MAP_FIXED uint64
	S_IFREG   uint64
	S_IFCHR   uint64
	S_IFBLK   uint64
	S_IFIFO   uint64
	S_IFSOCK  uint64
}

func MakeUnixNeutralizer(target *prog.Target) *UnixNeutralizer {
	return &UnixNeutralizer{
		MAP_FIXED: target.GetConst("MAP_FIXED"),
		S_IFREG:   target.GetConst("S_IFREG"),
		S_IFCHR:   target.GetConst("S_IFCHR"),
		S_IFBLK:   target.GetConst("S_IFBLK"),
		S_IFIFO:   target.GetConst("S_IFIFO"),
		S_IFSOCK:  target.GetConst("S_IFSOCK"),
	}
}

func (arch *UnixNeutralizer) Neutralize(c *prog.Call) {
	switch c.Meta.CallName {
	case "mmap":
		if c.Meta.Name == "mmap$bifrost" {
			// Mali bifrost mmap doesn't support MAP_FIXED.
			return
		}
		// Add MAP_FIXED flag, otherwise it produces non-deterministic results.
		c.Args[3].(*prog.ConstArg).Val |= arch.MAP_FIXED
	case "mknod", "mknodat":
		pos := 1
		if c.Meta.CallName == "mknodat" {
			pos = 2
		}
		switch c.Args[pos+1].Type().(type) {
		case *prog.ProcType, *prog.ResourceType:
			return
		}
		mode := c.Args[pos].(*prog.ConstArg)
		dev := c.Args[pos+1].(*prog.ConstArg)
		dev.Val = uint64(uint32(dev.Val))
		// Char and block devices read/write io ports, kernel memory and do other nasty things.
		// TODO: not required if executor drops privileges.
		mask := arch.S_IFREG | arch.S_IFCHR | arch.S_IFBLK | arch.S_IFIFO | arch.S_IFSOCK
		switch mode.Val & mask {
		case arch.S_IFREG, arch.S_IFIFO, arch.S_IFSOCK:
		case arch.S_IFBLK:
			if dev.Val>>8 == 7 {
				break // loop
			}
			mode.Val &^= arch.S_IFBLK
			mode.Val |= arch.S_IFREG
		case arch.S_IFCHR:
			mode.Val &^= arch.S_IFCHR
			mode.Val |= arch.S_IFREG
		}
	case "exit", "exit_group":
		code := c.Args[0].(*prog.ConstArg)
		// This code is reserved by executor.
		if code.Val%128 == 67 {
			code.Val = 1
		}
	}
}
<commit_msg>sys/targets: allow to create /dev/null<commit_after>// Copyright 2018 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package targets

import (
	"github.com/google/syzkaller/prog"
)

// MakePosixMmap creates a "normal" posix mmap call that maps the target data range.
// If exec is set, the mapping is mapped as PROT_EXEC.
// If contain is set, the mapping is surrounded by PROT_NONE pages.
// These flags should be in sync with what executor.
func MakePosixMmap(target *prog.Target, exec, contain bool) func() []*prog.Call {
	meta := target.SyscallMap["mmap"]
	protRW := target.GetConst("PROT_READ") | target.GetConst("PROT_WRITE")
	if exec {
		protRW |= target.GetConst("PROT_EXEC")
	}
	flags := target.GetConst("MAP_ANONYMOUS") | target.GetConst("MAP_PRIVATE") | target.GetConst("MAP_FIXED")
	size := target.NumPages * target.PageSize
	const invalidFD = ^uint64(0)
	makeMmap := func(addr, size, prot uint64) *prog.Call {
		args := []prog.Arg{
			prog.MakeVmaPointerArg(meta.Args[0].Type, prog.DirIn, addr, size),
			prog.MakeConstArg(meta.Args[1].Type, prog.DirIn, size),
			prog.MakeConstArg(meta.Args[2].Type, prog.DirIn, prot),
			prog.MakeConstArg(meta.Args[3].Type, prog.DirIn, flags),
			prog.MakeResultArg(meta.Args[4].Type, prog.DirIn, nil, invalidFD),
		}
		i := len(args)
		// Some targets have a padding argument between fd and offset.
		if len(meta.Args) > 6 {
			args = append(args, prog.MakeConstArg(meta.Args[i].Type, prog.DirIn, 0))
			i++
		}
		args = append(args, prog.MakeConstArg(meta.Args[i].Type, prog.DirIn, 0))
		return &prog.Call{
			Meta: meta,
			Args: args,
			Ret:  prog.MakeReturnArg(meta.Ret),
		}
	}
	return func() []*prog.Call {
		if contain {
			return []*prog.Call{
				makeMmap(^target.PageSize+1, target.PageSize, 0),
				makeMmap(0, size, protRW),
				makeMmap(size, target.PageSize, 0),
			}
		}
		return []*prog.Call{makeMmap(0, size, protRW)}
	}
}

func MakeSyzMmap(target *prog.Target) func() []*prog.Call {
	meta := target.SyscallMap["syz_mmap"]
	size := target.NumPages * target.PageSize
	return func() []*prog.Call {
		return []*prog.Call{
			{
				Meta: meta,
				Args: []prog.Arg{
					prog.MakeVmaPointerArg(meta.Args[0].Type, prog.DirIn, 0, size),
					prog.MakeConstArg(meta.Args[1].Type, prog.DirIn, size),
				},
				Ret: prog.MakeReturnArg(meta.Ret),
			},
		}
	}
}

type UnixNeutralizer struct {
	MAP_FIXED uint64
	S_IFREG   uint64
	S_IFCHR   uint64
	S_IFBLK   uint64
	S_IFIFO   uint64
	S_IFSOCK  uint64
}

func MakeUnixNeutralizer(target *prog.Target) *UnixNeutralizer {
	return &UnixNeutralizer{
		MAP_FIXED: target.GetConst("MAP_FIXED"),
		S_IFREG:   target.GetConst("S_IFREG"),
		S_IFCHR:   target.GetConst("S_IFCHR"),
		S_IFBLK:   target.GetConst("S_IFBLK"),
		S_IFIFO:   target.GetConst("S_IFIFO"),
		S_IFSOCK:  target.GetConst("S_IFSOCK"),
	}
}

func (arch *UnixNeutralizer) Neutralize(c *prog.Call) {
	switch c.Meta.CallName {
	case "mmap":
		if c.Meta.Name == "mmap$bifrost" {
			// Mali bifrost mmap doesn't support MAP_FIXED.
			return
		}
		// Add MAP_FIXED flag, otherwise it produces non-deterministic results.
		c.Args[3].(*prog.ConstArg).Val |= arch.MAP_FIXED
	case "mknod", "mknodat":
		pos := 1
		if c.Meta.CallName == "mknodat" {
			pos = 2
		}
		switch c.Args[pos+1].Type().(type) {
		case *prog.ProcType, *prog.ResourceType:
			return
		}
		mode := c.Args[pos].(*prog.ConstArg)
		dev := c.Args[pos+1].(*prog.ConstArg)
		dev.Val = uint64(uint32(dev.Val))
		// Char and block devices read/write io ports, kernel memory and do other nasty things.
		// TODO: not required if executor drops privileges.
		mask := arch.S_IFREG | arch.S_IFCHR | arch.S_IFBLK | arch.S_IFIFO | arch.S_IFSOCK
		switch mode.Val & mask {
		case arch.S_IFREG, arch.S_IFIFO, arch.S_IFSOCK:
		case arch.S_IFBLK:
			if dev.Val>>8 == 7 {
				break // loop
			}
			mode.Val &^= arch.S_IFBLK
			mode.Val |= arch.S_IFREG
		case arch.S_IFCHR:
			if dev.Val == 0x103 {
				break // /dev/null
			}
			mode.Val &^= arch.S_IFCHR
			mode.Val |= arch.S_IFREG
		}
	case "exit", "exit_group":
		code := c.Args[0].(*prog.ConstArg)
		// This code is reserved by executor.
		if code.Val%128 == 67 {
			code.Val = 1
		}
	}
}
<|endoftext|>


<commit_before>package mongo

import (
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"testing"

	jc "github.com/juju/testing/checkers"
	gc "launchpad.net/gocheck"

	"launchpad.net/juju-core/testing/testbase"
	"launchpad.net/juju-core/upstart"
)

func Test(t *testing.T) { gc.TestingT(t) }

type MongoSuite struct {
	testbase.LoggingSuite
}

var _ = gc.Suite(&MongoSuite{})

func (s *MongoSuite) SetUpSuite(c *gc.C) {
	testpath := c.MkDir()
	s.PatchEnvPathPrepend(testpath)
	// mock out the upstart commands so we can fake install services without sudo
	fakeCmd(filepath.Join(testpath, "start"))
	fakeCmd(filepath.Join(testpath, "stop"))

	s.PatchValue(&upstart.InitDir, c.MkDir())
}

func fakeCmd(path string) {
	err := ioutil.WriteFile(path, []byte("#!/bin/bash --norc\nexit 0"), 0755)
	if err != nil {
		panic(err)
	}
}

func (s *MongoSuite) TestJujuMongodPath(c *gc.C) {
	d := c.MkDir()
	defer os.RemoveAll(d)
	mongoPath := filepath.Join(d, "mongod")
	s.PatchValue(&JujuMongodPath, mongoPath)

	err := ioutil.WriteFile(mongoPath, nil, 0777)
	c.Assert(err, gc.IsNil)

	obtained, err := MongodPath()
	c.Check(err, gc.IsNil)
	c.Check(obtained, gc.Equals, mongoPath)
}

func (s *MongoSuite) TestDefaultMongodPath(c *gc.C) {
	s.PatchValue(&JujuMongodPath, "/not/going/to/exist/mongod")

	dir := c.MkDir()
	s.PatchEnvPathPrepend(dir)
	filename := filepath.Join(dir, "mongod")
	err := ioutil.WriteFile(filename, nil, 0777)
	c.Assert(err, gc.IsNil)

	obtained, err := MongodPath()
	c.Check(err, gc.IsNil)
	c.Check(obtained, gc.Equals, filename)
}

func (s *MongoSuite) TestRemoveOldMongoServices(c *gc.C) {
	namespace := "user-local"

	// Make fake old service.
	// We defer the removes manually just in case the test fails, we don't leave
	// junk behind.
	conf := makeService(ServiceName(namespace), c)

	// now that the old service is installed, change the in-memory expected
	// script to something different.
	conf.Cmd = "echo something else"
	defer conf.Remove()

	// this should remove the old service, because the contents now differ
	err := removeOldService(conf)
	c.Check(err, gc.IsNil)

	c.Check(conf.Installed(), jc.IsFalse)
}

func (s *MongoSuite) TestMakeJournalDirs(c *gc.C) {
	dir := c.MkDir()
	err := makeJournalDirs(dir)
	c.Assert(err, gc.IsNil)

	testJournalDirs(dir, c)
}

func testJournalDirs(dir string, c *gc.C) {
	journalDir := path.Join(dir, "journal")

	c.Check(journalDir, jc.IsDirectory)
	info, err := os.Stat(filepath.Join(journalDir, "prealloc.0"))
	c.Check(err, gc.IsNil)

	size := int64(1024 * 1024)

	c.Check(info.Size(), gc.Equals, size)
	info, err = os.Stat(filepath.Join(journalDir, "prealloc.1"))
	c.Check(err, gc.IsNil)
	c.Check(info.Size(), gc.Equals, size)
	info, err = os.Stat(filepath.Join(journalDir, "prealloc.2"))
	c.Check(err, gc.IsNil)
	c.Check(info.Size(), gc.Equals, size)

}

func (s *MongoSuite) TestEnsureMongoServer(c *gc.C) {
	dir := c.MkDir()
	port := 25252
	namespace := "namespace"
	oldsvc := makeService(ServiceName(namespace), c)
	defer oldsvc.StopAndRemove()

	err := EnsureMongoServer(dir, port, namespace)
	c.Assert(err, gc.IsNil)
	mongodPath := MongodPathForSeries("some-series")
	svc, err := MongoUpstartService(namespace, mongodPath, dir, port)
	c.Assert(err, gc.IsNil)
	defer svc.StopAndRemove()

	testJournalDirs(dir, c)
	c.Assert(svc.Installed(), jc.IsTrue)
	conf, err := svc.ReadConf()
	c.Assert(err, gc.IsNil)
	expected, err := svc.Render()
	c.Assert(err, gc.IsNil)
	c.Assert(string(conf), gc.Equals, string(expected))

	// now check we can call it multiple times without error
	err = EnsureMongoServer(dir, port, namespace)
	c.Assert(err, gc.IsNil)
	c.Assert(svc.Installed(), jc.IsTrue)
	conf, err = svc.ReadConf()
	c.Assert(err, gc.IsNil)
	expected, err = svc.Render()
	c.Assert(err, gc.IsNil)
	c.Assert(string(conf), gc.Equals, string(expected))
}

func (s *MongoSuite) TestServiceName(c *gc.C) {
	name := ServiceName("foo")
	c.Assert(name, gc.Equals, "juju-db-foo")
	name = ServiceName("")
	c.Assert(name, gc.Equals, "juju-db")
}

func makeService(name string, c *gc.C) *upstart.Conf {
	conf := &upstart.Conf{
		Desc:    "foo",
		Service: *upstart.NewService(name),
		Cmd:     "echo hi",
	}
	err := conf.Install()
	c.Assert(err, gc.IsNil)
	return conf
}
<commit_msg>fix the tests<commit_after>package mongo

import (
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"testing"

	jc "github.com/juju/testing/checkers"
	gc "launchpad.net/gocheck"

	"launchpad.net/juju-core/testing/testbase"
	"launchpad.net/juju-core/upstart"
)

func Test(t *testing.T) { gc.TestingT(t) }

type MongoSuite struct {
	testbase.LoggingSuite
}

var _ = gc.Suite(&MongoSuite{})

func (s *MongoSuite) SetUpSuite(c *gc.C) {
	testpath := c.MkDir()
	s.PatchEnvPathPrepend(testpath)
	// mock out the upstart commands so we can fake install services without sudo
	fakeCmd(filepath.Join(testpath, "start"))
	fakeCmd(filepath.Join(testpath, "stop"))

	s.PatchValue(&upstart.InitDir, c.MkDir())
}

func fakeCmd(path string) {
	err := ioutil.WriteFile(path, []byte("#!/bin/bash --norc\nexit 0"), 0755)
	if err != nil {
		panic(err)
	}
}

func (s *MongoSuite) TestJujuMongodPath(c *gc.C) {
	d := c.MkDir()
	defer os.RemoveAll(d)
	mongoPath := filepath.Join(d, "mongod")
	s.PatchValue(&JujuMongodPath, mongoPath)

	err := ioutil.WriteFile(mongoPath, nil, 0777)
	c.Assert(err, gc.IsNil)

	obtained, err := MongodPath()
	c.Check(err, gc.IsNil)
	c.Check(obtained, gc.Equals, mongoPath)
}

func (s *MongoSuite) TestDefaultMongodPath(c *gc.C) {
	s.PatchValue(&JujuMongodPath, "/not/going/to/exist/mongod")

	dir := c.MkDir()
	s.PatchEnvPathPrepend(dir)
	filename := filepath.Join(dir, "mongod")
	err := ioutil.WriteFile(filename, nil, 0777)
	c.Assert(err, gc.IsNil)

	obtained, err := MongodPath()
	c.Check(err, gc.IsNil)
	c.Check(obtained, gc.Equals, filename)
}

func (s *MongoSuite) TestMakeJournalDirs(c *gc.C) {
	dir := c.MkDir()
	err := makeJournalDirs(dir)
	c.Assert(err, gc.IsNil)

	testJournalDirs(dir, c)
}

func testJournalDirs(dir string, c *gc.C) {
	journalDir := path.Join(dir, "journal")

	c.Check(journalDir, jc.IsDirectory)
	info, err := os.Stat(filepath.Join(journalDir, "prealloc.0"))
	c.Check(err, gc.IsNil)

	size := int64(1024 * 1024)

	c.Check(info.Size(), gc.Equals, size)
	info, err = os.Stat(filepath.Join(journalDir, "prealloc.1"))
	c.Check(err, gc.IsNil)
	c.Check(info.Size(), gc.Equals, size)
	info, err = os.Stat(filepath.Join(journalDir, "prealloc.2"))
	c.Check(err, gc.IsNil)
	c.Check(info.Size(), gc.Equals, size)

}

func (s *MongoSuite) TestEnsureMongoServer(c *gc.C) {
	dir := c.MkDir()
	port := 25252
	namespace := "namespace"
	oldsvc := makeService(ServiceName(namespace), c)
	defer oldsvc.StopAndRemove()

	err := EnsureMongoServer(dir, port, namespace)
	c.Assert(err, gc.IsNil)
	mongodPath := MongodPathForSeries("some-series")
	svc, err := MongoUpstartService(namespace, mongodPath, dir, port)
	c.Assert(err, gc.IsNil)
	defer svc.StopAndRemove()

	testJournalDirs(dir, c)
	c.Assert(svc.Installed(), jc.IsTrue)

	// now check we can call it multiple times without error
	err = EnsureMongoServer(dir, port, namespace)
	c.Assert(err, gc.IsNil)
	c.Assert(svc.Installed(), jc.IsTrue)
}

func (s *MongoSuite) TestServiceName(c *gc.C) {
	name := ServiceName("foo")
	c.Assert(name, gc.Equals, "juju-db-foo")
	name = ServiceName("")
	c.Assert(name, gc.Equals, "juju-db")
}

func makeService(name string, c *gc.C) *upstart.Conf {
	conf := &upstart.Conf{
		Desc:    "foo",
		Service: *upstart.NewService(name),
		Cmd:     "echo hi",
	}
	err := conf.Install()
	c.Assert(err, gc.IsNil)
	return conf
}
<|endoftext|>


<commit_before>// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package commitments contains common type definitions and functions used by other
// packages. Types that can cause circular import should be added here.
package commitments

import (
	"bytes"
	"encoding/hex"
	"testing"

	"github.com/google/keytransparency/core/crypto/dev"
)

func TestCommit(t *testing.T) {
	for _, tc := range []struct {
		userID, appID, data    string
		muserID, mappID, mdata string
		mutate                 bool
		want                   error
	}{
		{"foo", "app", "bar", "foo", "app", "bar", false, nil},
		{"foo", "app", "bar", "foo", "app", "bar", true, ErrInvalidCommitment},
		{"foo", "app", "bar", "fooa", "pp", "bar", false, ErrInvalidCommitment},
		{"foo", "app", "bar", "foo", "ap", "pbar", false, ErrInvalidCommitment},
	} {
		k, c, err := Commit(tc.userID, tc.appID, []byte(tc.data))
		if err != nil {
			t.Errorf("Commit(%v, %x): %v", tc.userID, tc.data, err)
		}
		if tc.mutate {
			k[0] ^= 1
		}
		c.Data = []byte(tc.mdata)
		if got := Verify(tc.muserID, tc.mappID, k, c); got != tc.want {
			t.Errorf("Verify(%v, %v, %x, %x): %v, want %v", tc.userID, tc.appID, k, c, err, tc.want)
		}
	}
}

func TestVectors(t *testing.T) {
	// Use a constant key of zero to obtain consistent test vectors.
	// Real commitment library MUST use random keys.
	Rand = dev.Zeros
	zeroKey := []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}
	for _, tc := range []struct {
		userID, appID, data string
		want                []byte
	}{
		{"foo", "app", "bar", dh("064c8933f50f897e8b179065c6b3ec13e9d093337c6d403c77e3ed1701378ed6")},
	} {
		k, c, err := Commit(tc.userID, tc.appID, []byte(tc.data))
		if err != nil {
			t.Errorf("Commit(%v, %x): %v", tc.userID, tc.data, err)
		}
		if got, want := k, tc.want; !bytes.Equal(got, want) {
			t.Errorf("Commit(%v, %x): %x ,want %x", tc.userID, tc.data, got, want)
		}
		if got, want := c.Key, zeroKey; !bytes.Equal(got, want) {
			t.Errorf("Commit(%v, %x).Key: %x ,want %x", tc.userID, tc.data, got, want)
		}
		if got, want := c.Data, []byte(tc.data); !bytes.Equal(got, want) {
			t.Errorf("Commit(%v, %x).Data: %x ,want %x", tc.userID, tc.data, got, want)
		}
	}
}

// Hex to Bytes
func dh(h string) []byte {
	result, err := hex.DecodeString(h)
	if err != nil {
		panic("DecodeString failed")
	}
	return result
}
<commit_msg>Add more test vectors<commit_after>// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package commitments contains common type definitions and functions used by other
// packages. Types that can cause circular import should be added here.
package commitments

import (
	"bytes"
	"encoding/hex"
	"testing"

	"github.com/google/keytransparency/core/crypto/dev"
)

func TestCommit(t *testing.T) {
	for _, tc := range []struct {
		userID, appID, data    string
		muserID, mappID, mdata string
		mutate                 bool
		want                   error
	}{
		{"foo", "app", "bar", "foo", "app", "bar", false, nil},
		{"foo", "app", "bar", "foo", "app", "bar", true, ErrInvalidCommitment},
		{"foo", "app", "bar", "fooa", "pp", "bar", false, ErrInvalidCommitment},
		{"foo", "app", "bar", "foo", "ap", "pbar", false, ErrInvalidCommitment},
	} {
		k, c, err := Commit(tc.userID, tc.appID, []byte(tc.data))
		if err != nil {
			t.Errorf("Commit(%v, %x): %v", tc.userID, tc.data, err)
		}
		if tc.mutate {
			k[0] ^= 1
		}
		c.Data = []byte(tc.mdata)
		if got := Verify(tc.muserID, tc.mappID, k, c); got != tc.want {
			t.Errorf("Verify(%v, %v, %x, %x): %v, want %v", tc.userID, tc.appID, k, c, err, tc.want)
		}
	}
}

func TestVectors(t *testing.T) {
	// Use a constant key of zero to obtain consistent test vectors.
	// Real commitment library MUST use random keys.
	Rand = dev.Zeros
	zeroKey := []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}
	for _, tc := range []struct {
		userID, appID, data string
		want                []byte
	}{
		{"", "", "", dh("0698789c7beed09e93848e4df08be5c911de534d286abcbf69359debe4c62bc2")},
		{"foo", "app", "bar", dh("064c8933f50f897e8b179065c6b3ec13e9d093337c6d403c77e3ed1701378ed6")},
		{"foo1", "app", "bar", dh("77015921f7fe584e1b5866a32ab9f305715c4e0241581d41f66ee34b24cdb566")},
		{"foo", "app1", "bar", dh("e7337229d7747cc2c9a83ee08adbec712f4acafd1b72258bbebf74637de987b7")},
		{"foo", "app", "bar1", dh("0fa2d7d53552e0871564c0e82ad394e72476b75f7fc77f40e2080af7f33d66eb")},
	} {
		k, c, err := Commit(tc.userID, tc.appID, []byte(tc.data))
		if err != nil {
			t.Errorf("Commit(%v, %x): %v", tc.userID, tc.data, err)
		}
		if got, want := k, tc.want; !bytes.Equal(got, want) {
			t.Errorf("Commit(%v, %x): %x ,want %x", tc.userID, tc.data, got, want)
		}
		if got, want := c.Key, zeroKey; !bytes.Equal(got, want) {
			t.Errorf("Commit(%v, %x).Key: %x ,want %x", tc.userID, tc.data, got, want)
		}
		if got, want := c.Data, []byte(tc.data); !bytes.Equal(got, want) {
			t.Errorf("Commit(%v, %x).Data: %x ,want %x", tc.userID, tc.data, got, want)
		}
	}
}

// Hex to Bytes
func dh(h string) []byte {
	result, err := hex.DecodeString(h)
	if err != nil {
		panic("DecodeString failed")
	}
	return result
}
<|endoftext|>


<commit_before>package memdb

import "fmt"
import "testing"
import "time"
import "math/rand"
import "sync"
import "runtime"
import "encoding/binary"

func TestInsert(t *testing.T) {
	db := New()
	w := db.NewWriter()
	for i := 0; i < 2000; i++ {
		w.Put(NewItem([]byte(fmt.Sprintf("%010d", i))))
	}

	for i := 1750; i < 2000; i++ {
		w.Delete(NewItem([]byte(fmt.Sprintf("%010d", i))))
	}
	snap := w.NewSnapshot()

	for i := 2000; i < 5000; i++ {
		w.Put(NewItem([]byte(fmt.Sprintf("%010d", i))))
	}

	_ = w.NewSnapshot()

	itr := w.NewIterator(snap)
	count := 0
	itr.SeekFirst()
	itr.Seek(NewItem([]byte(fmt.Sprintf("%010d", 1500))))
	for ; itr.Valid(); itr.Next() {
		expected := fmt.Sprintf("%010d", count+1500)
		got := string(itr.Get().data)
		count++
		if got != expected {
			t.Errorf("Expected %s, got %v", expected, got)
		}
	}

	if count != 250 {
		t.Errorf("Expected count = 250, got %v", count)
	}
}

func doInsert(db *MemDB, wg *sync.WaitGroup, n int, isRand bool) {
	defer wg.Done()
	w := db.NewWriter()
	rnd := rand.New(rand.NewSource(int64(rand.Int())))
	for i := 0; i < n; i++ {
		var val int
		if isRand {
			val = rnd.Int()
		} else {
			val = i
		}
		if i%100000 == 0 {
			s := w.NewSnapshot()
			s.Close()
		}
		buf := make([]byte, 8)
		binary.BigEndian.PutUint64(buf, uint64(val))
		// w.Put(NewItem([]byte(fmt.Sprintf("%025d", val))))
		w.Put(NewItem(buf))
	}
}

func TestInsertPerf(t *testing.T) {
	var wg sync.WaitGroup
	db := New()
	n := 1000000
	t0 := time.Now()
	total := n * runtime.NumCPU()
	for i := 0; i < runtime.NumCPU(); i++ {
		wg.Add(1)
		go doInsert(db, &wg, n, true)
	}
	wg.Wait()

	dur := time.Since(t0)
	fmt.Printf("%d items took %v -> %v items/s snapshots_created %v live_snapshots %v\n",
		total, dur, float64(total)/float64(dur.Seconds()), db.getCurrSn(), len(db.GetSnapshots()))
}

func doGet(t *testing.T, db *MemDB, snap *Snapshot, wg *sync.WaitGroup, n int) {
	defer wg.Done()
	rnd := rand.New(rand.NewSource(int64(rand.Int())))

	buf := make([]byte, 8)
	itr := db.NewIterator(snap)
	for i := 0; i < n; i++ {
		val := rnd.Int() % n
		binary.BigEndian.PutUint64(buf, uint64(val))
		// itr.Seek(NewItem([]byte(fmt.Sprintf("%025d", val))))
		itr.Seek(NewItem(buf))
		if !itr.Valid() {
			t.Errorf("Expected to find %v", val)
		}
	}
}

func TestGetPerf(t *testing.T) {
	var wg sync.WaitGroup
	db := New()
	n := 1000000
	wg.Add(1)
	go doInsert(db, &wg, n, false)
	wg.Wait()
	snap := db.NewSnapshot()

	t0 := time.Now()
	total := n * runtime.NumCPU()
	for i := 0; i < runtime.NumCPU(); i++ {
		wg.Add(1)
		go doGet(t, db, snap, &wg, n)
	}
	wg.Wait()
	dur := time.Since(t0)
	fmt.Printf("%d items took %v -> %v items/s\n", total, dur, float64(total)/float64(dur.Seconds()))
}
<commit_msg>memdb: Add tests for StoreToDisk() and LoadFromDisk()<commit_after>package memdb

import "fmt"
import "os"
import "testing"
import "time"
import "math/rand"
import "sync"
import "runtime"
import "encoding/binary"

func TestInsert(t *testing.T) {
	db := New()
	w := db.NewWriter()
	for i := 0; i < 2000; i++ {
		w.Put(NewItem([]byte(fmt.Sprintf("%010d", i))))
	}

	for i := 1750; i < 2000; i++ {
		w.Delete(NewItem([]byte(fmt.Sprintf("%010d", i))))
	}
	snap := w.NewSnapshot()

	for i := 2000; i < 5000; i++ {
		w.Put(NewItem([]byte(fmt.Sprintf("%010d", i))))
	}

	_ = w.NewSnapshot()

	itr := w.NewIterator(snap)
	count := 0
	itr.SeekFirst()
	itr.Seek(NewItem([]byte(fmt.Sprintf("%010d", 1500))))
	for ; itr.Valid(); itr.Next() {
		expected := fmt.Sprintf("%010d", count+1500)
		got := string(itr.Get().data)
		count++
		if got != expected {
			t.Errorf("Expected %s, got %v", expected, got)
		}
	}

	if count != 250 {
		t.Errorf("Expected count = 250, got %v", count)
	}
}

func doInsert(db *MemDB, wg *sync.WaitGroup, n int, isRand bool, shouldSnap bool) {
	defer wg.Done()
	w := db.NewWriter()
	rnd := rand.New(rand.NewSource(int64(rand.Int())))
	for i := 0; i < n; i++ {
		var val int
		if isRand {
			val = rnd.Int()
		} else {
			val = i
		}
		if shouldSnap && i%100000 == 0 {
			s := w.NewSnapshot()
			s.Close()
		}
		buf := make([]byte, 8)
		binary.BigEndian.PutUint64(buf, uint64(val))
		// w.Put(NewItem([]byte(fmt.Sprintf("%025d", val))))
		w.Put(NewItem(buf))
	}
}

func TestInsertPerf(t *testing.T) {
	var wg sync.WaitGroup
	db := New()
	n := 1000000
	t0 := time.Now()
	total := n * runtime.NumCPU()
	for i := 0; i < runtime.NumCPU(); i++ {
		wg.Add(1)
		go doInsert(db, &wg, n, true, true)
	}
	wg.Wait()

	dur := time.Since(t0)
	fmt.Printf("%d items took %v -> %v items/s snapshots_created %v live_snapshots %v\n",
		total, dur, float64(total)/float64(dur.Seconds()), db.getCurrSn(), len(db.GetSnapshots()))
}

func doGet(t *testing.T, db *MemDB, snap *Snapshot, wg *sync.WaitGroup, n int) {
	defer wg.Done()
	rnd := rand.New(rand.NewSource(int64(rand.Int())))

	buf := make([]byte, 8)
	itr := db.NewIterator(snap)
	for i := 0; i < n; i++ {
		val := rnd.Int() % n
		binary.BigEndian.PutUint64(buf, uint64(val))
		// itr.Seek(NewItem([]byte(fmt.Sprintf("%025d", val))))
		itr.Seek(NewItem(buf))
		if !itr.Valid() {
			t.Errorf("Expected to find %v", val)
		}
	}
}

func TestGetPerf(t *testing.T) {
	var wg sync.WaitGroup
	db := New()
	n := 1000000
	wg.Add(1)
	go doInsert(db, &wg, n, false, true)
	wg.Wait()
	snap := db.NewSnapshot()

	t0 := time.Now()
	total := n * runtime.NumCPU()
	for i := 0; i < runtime.NumCPU(); i++ {
		wg.Add(1)
		go doGet(t, db, snap, &wg, n)
	}
	wg.Wait()
	dur := time.Since(t0)
	fmt.Printf("%d items took %v -> %v items/s\n", total, dur, float64(total)/float64(dur.Seconds()))
}

func CountItems(db *MemDB, snap *Snapshot) int {
	var count int
	itr := db.NewIterator(snap)
	for itr.SeekFirst(); itr.Valid(); itr.Next() {
		count++
	}
	itr.Close()
	return count
}

func TestLoadStoreDisk(t *testing.T) {
	os.RemoveAll("db.dump")
	var wg sync.WaitGroup
	db := New()
	n := 1000000
	t0 := time.Now()
	wg.Add(1)
	go doInsert(db, &wg, n, false, false)
	wg.Wait()
	fmt.Printf("Inserting %v items took %v\n", n, time.Since(t0))
	snap := db.NewSnapshot()
	snap = db.NewSnapshot()

	t0 = time.Now()
	err := db.DumpToDisk("db.dump", snap)
	if err != nil {
		t.Errorf("Expected no error. got=%v", err)
	}

	fmt.Printf("Dumping to disk took %v\n", time.Since(t0))

	snap.Close()
	db = New()
	t0 = time.Now()
	snap, err = db.LoadFromDisk("db.dump")
	if err != nil {
		t.Errorf("Expected no error. got=%v", err)
	}
	fmt.Printf("Loading from disk took %v\n", time.Since(t0))

	count := CountItems(db, snap)
	if count != n {
		t.Errorf("Expected %v, got %v", n, count)
	}
}
<|endoftext|>


<commit_before>package bootstrap

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"os"
	"strings"

	"github.com/tinzenite/model"
	"github.com/tinzenite/shared"
)

type chaninterface struct {
	// reference back to Bootstrap
	boot *Bootstrap
	// model reference NOTE: once created it means that a bootstrap is in progress! Also it is CORRECT and DESIRED that the model is not stored between runs.
	model *model.Model
	// we need to remember all update messages so that we can apply them when received
	messages map[string]*shared.UpdateMessage
}

func createChanInterface(boot *Bootstrap) *chaninterface {
	return &chaninterface{
		boot:     boot,
		model:    nil,
		messages: make(map[string]*shared.UpdateMessage)}
}

func (c *chaninterface) OnNewConnection(address, message string) {
	log.Println("NewConnection:", address[:8], "ignoring!")
}

func (c *chaninterface) OnMessage(address, message string) {
	log.Println("MSG from", address[:8], ":", message)
}

func (c *chaninterface) OnAllowFile(address, name string) (bool, string) {
	filename := address + "." + name
	return true, c.boot.path + "/" + shared.TINZENITEDIR + "/" + shared.RECEIVINGDIR + "/" + filename
}

func (c *chaninterface) OnFileReceived(address, path, name string) {
	// split filename to get identification
	check := strings.Split(name, ".")[0]
	identification := strings.Split(name, ".")[1]
	if check != address {
		log.Println("Filename is mismatched!")
		return
	}
	// whether we allow accepting a file or model depends on whether we already have a model here...
	if c.model != nil {
		// safe guard
		if len(c.messages) == 0 {
			log.Println("No update messages available! Ignoring file.")
			return
		}
		log.Println("Receiving file!")
		c.onFile(path, identification)
	} else {
		// safe guard
		if identification != shared.IDMODEL {
			log.Println("Excepting model! Ignoring file.")
			return
		}
		log.Println("Receiving model!") // <-- should only be called once!
		// no need to keep sending check in backgronud
		c.boot.stop <- false
		// go due it
		err := c.onModel(address, path)
		if err != nil {
			log.Println("onModel:", err)
		}
	}
}

func (c *chaninterface) OnConnected(address string) {
	log.Println("Connected:", address[:8])
}

// ---------------------- NO CALLBACKS BEYOND THIS POINT -----------------------

func (c *chaninterface) onModel(address, path string) error {
	// read model file and remove it
	data, err := ioutil.ReadFile(path)
	if err != nil {
		return err
	}
	err = os.Remove(path)
	if err != nil {
		log.Println("Failed to remove temp model file:", err)
		// not strictly critical so no return here
	}
	// unmarshal
	foreignModel := &shared.ObjectInfo{}
	err = json.Unmarshal(data, foreignModel)
	if err != nil {
		return err
	}
	// make a model of the local stuff
	m, err := model.Create(c.boot.path, c.boot.peer.Identification)
	if err != nil {
		return err
	}
	c.model = m
	// apply what is already here
	err = c.model.Update()
	if err != nil {
		return err
	}
	// get difference in updates
	var updateLists []*shared.UpdateMessage
	updateLists, err = c.model.BootstrapModel(foreignModel)
	if err != nil {
		return err
	}
	log.Println("Need to fetch", len(updateLists), "updates.")
	// pretend that the updatemessage came from outside here
	for _, um := range updateLists {
		// we have to remember the update messages because we'll need to apply them
		c.messages[um.Object.Identification] = um
		// create & modify must first fetch file
		rm := shared.CreateRequestMessage(shared.ReObject, um.Object.Identification)
		// request file and apply update on success
		c.boot.channel.Send(address, rm.String())
	}
	return nil
}

func (c *chaninterface) onFile(path, identification string) error {
	// see if we have a corresponding update message
	um, exists := c.messages[identification]
	if !exists {
		log.Println("Can not apply file", identification, "!")
		return os.Remove(path)
	}
	// remove from list since we're applying it
	delete(c.messages, identification)
	// rename to correct name for model
	err := os.Rename(path, c.boot.path+"/"+shared.TINZENITEDIR+"/"+shared.TEMPDIR+"/"+identification)
	if err != nil {
		return err
	}
	// apply
	err = c.model.ApplyUpdateMessage(um)
	if err != nil {
		return err
	}
	// detect when done to call success callback
	if len(c.messages) == 0 {
		log.Println("I think I'm done...")
		c.boot.Close()
		// write directory to DIRECTORYLIST because it is now a valid TINZENITEDIR
		err := shared.WriteDirectoryList(c.boot.path)
		if err != nil {
			log.Println("Failed to write path to", shared.DIRECTORYLIST, "!")
			// not a critical error but log in case clients can't find the dir
		}
		if c.boot.onDone != nil {
			c.boot.onDone()
		}
	}
	return nil
}
<commit_msg>debug log<commit_after>package bootstrap

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"os"
	"strings"

	"github.com/tinzenite/model"
	"github.com/tinzenite/shared"
)

type chaninterface struct {
	// reference back to Bootstrap
	boot *Bootstrap
	// model reference NOTE: once created it means that a bootstrap is in progress! Also it is CORRECT and DESIRED that the model is not stored between runs.
	model *model.Model
	// we need to remember all update messages so that we can apply them when received
	messages map[string]*shared.UpdateMessage
}

func createChanInterface(boot *Bootstrap) *chaninterface {
	return &chaninterface{
		boot:     boot,
		model:    nil,
		messages: make(map[string]*shared.UpdateMessage)}
}

func (c *chaninterface) OnNewConnection(address, message string) {
	log.Println("NewConnection:", address[:8], "ignoring!")
}

func (c *chaninterface) OnMessage(address, message string) {
	log.Println("MSG from", address[:8], ":", message)
}

func (c *chaninterface) OnAllowFile(address, name string) (bool, string) {
	filename := address + "." + name
	return true, c.boot.path + "/" + shared.TINZENITEDIR + "/" + shared.RECEIVINGDIR + "/" + filename
}

func (c *chaninterface) OnFileReceived(address, path, name string) {
	// split filename to get identification
	check := strings.Split(name, ".")[0]
	identification := strings.Split(name, ".")[1]
	if check != address {
		log.Println("Filename is mismatched!")
		return
	}
	// whether we allow accepting a file or model depends on whether we already have a model here...
	if c.model != nil {
		// safe guard
		if len(c.messages) == 0 {
			log.Println("No update messages available! Ignoring file.")
			return
		}
		log.Println("Receiving file!")
		c.onFile(path, identification)
	} else {
		// safe guard
		if identification != shared.IDMODEL {
			log.Println("Excepting model! Ignoring file.")
			return
		}
		log.Println("Receiving model!") // <-- should only be called once!
		// no need to keep sending check in backgronud
		c.boot.stop <- false
		// go due it
		err := c.onModel(address, path)
		if err != nil {
			log.Println("onModel:", err)
		}
	}
}

func (c *chaninterface) OnConnected(address string) {
	log.Println("Connected:", address[:8])
}

// ---------------------- NO CALLBACKS BEYOND THIS POINT -----------------------

func (c *chaninterface) onModel(address, path string) error {
	// read model file and remove it
	data, err := ioutil.ReadFile(path)
	if err != nil {
		return err
	}
	err = os.Remove(path)
	if err != nil {
		log.Println("Failed to remove temp model file:", err)
		// not strictly critical so no return here
	}
	// unmarshal
	foreignModel := &shared.ObjectInfo{}
	err = json.Unmarshal(data, foreignModel)
	if err != nil {
		return err
	}
	// make a model of the local stuff
	m, err := model.Create(c.boot.path, c.boot.peer.Identification)
	if err != nil {
		return err
	}
	c.model = m
	// apply what is already here
	err = c.model.Update()
	if err != nil {
		return err
	}
	// get difference in updates
	var updateLists []*shared.UpdateMessage
	updateLists, err = c.model.BootstrapModel(foreignModel)
	if err != nil {
		return err
	}
	log.Println("Need to fetch", len(updateLists), "updates.")
	// pretend that the updatemessage came from outside here
	for _, um := range updateLists {
		// we have to remember the update messages because we'll need to apply them
		c.messages[um.Object.Identification] = um
		// create & modify must first fetch file
		rm := shared.CreateRequestMessage(shared.ReObject, um.Object.Identification)
		// request file and apply update on success
		c.boot.channel.Send(address, rm.String())
	}
	return nil
}

func (c *chaninterface) onFile(path, identification string) error {
	// see if we have a corresponding update message
	um, exists := c.messages[identification]
	if !exists {
		log.Println("Can not apply file", identification, "!")
		return os.Remove(path)
	}
	// remove from list since we're applying it
	delete(c.messages, identification)
	// rename to correct name for model
	err := os.Rename(path, c.boot.path+"/"+shared.TINZENITEDIR+"/"+shared.TEMPDIR+"/"+identification)
	if err != nil {
		return err
	}
	// apply
	err = c.model.ApplyUpdateMessage(um)
	if err != nil {
		return err
	}
	// detect when done to call success callback
	if len(c.messages) == 0 {
		log.Println("I think I'm done...")
		c.boot.Close()
		// write directory to DIRECTORYLIST because it is now a valid TINZENITEDIR
		err := shared.WriteDirectoryList(c.boot.path)
		if err != nil {
			log.Println("Failed to write path to", shared.DIRECTORYLIST, "!")
			// not a critical error but log in case clients can't find the dir
		}
		if c.boot.onDone != nil {
			c.boot.onDone()
		}
	}
	// TODO for some reason bootstrap never finishes, look into why!
	log.Println("Still need to fetch", len(c.messages), "updates!")
	for _, msg := range c.messages {
		log.Println(msg.Object.Path)
	}
	return nil
}
<|endoftext|>


<commit_before>package bitcoin_load_spike

import (
	"fmt"
	_ "math"
)

type timeSeriesPlot struct {
	buckets        []float64
	counts         []int64
	smallestBucket int64
	largestBucket  int64
}

func newTimeSeriesPlot() *timeSeriesPlot {
	return &timeSeriesPlot{
		buckets:        make([]float64, NUM_BUCKETS),
		counts:         make([]int64, NUM_BUCKETS),
		smallestBucket: NUM_BUCKETS,
		largestBucket:  0,
	}
}

func (tsp *timeSeriesPlot) updateBucket(i int64, age float64) {
	total := tsp.buckets[i]*float64(tsp.counts[i]) + age
	tsp.counts[i]++
	tsp.buckets[i] = total / float64(tsp.counts[i])

	if tsp.largestBucket < i {
		tsp.largestBucket = i
	}
	if tsp.smallestBucket > i {
		tsp.smallestBucket = i
	}
}

func (tsp *timeSeriesPlot) output() (fileContents string) {
	for i, avgTxnTime := range tsp.buckets[0 : NUM_BUCKETS-1] {
		fileContents += fmt.Sprintf("%d | %f\n", i, avgTxnTime)
	}
	return
}

type TimeSeriesLogger struct {
	plot          *timeSeriesPlot
	secsPerBucket float64
	filePrefix    string
}

func (tsl TimeSeriesLogger) FilePrefix() string {
	return tsl.filePrefix
}

func (tsl TimeSeriesLogger) FileExtension() string {
	return "tsl-dat"
}

func (tsl *TimeSeriesLogger) Log(blockTimestamp, txnTimestamp float64, spikeNumber int) {
	age := blockTimestamp - txnTimestamp

	b := int64(txnTimestamp / tsl.secsPerBucket)
	// Extend buckets and counts if necessary
	if b >= int64(len(tsl.plot.buckets)) {
		diff := b - int64(len(tsl.plot.buckets)-1)

		// fmt.Print(fmt.Sprintf("<%d>", diff))

		bucketsExtension := make([]float64, diff)
		countsExtension := make([]int64, diff)

		tsl.plot.buckets = append(tsl.plot.buckets, bucketsExtension...)
		tsl.plot.counts = append(tsl.plot.counts, countsExtension...)
	}

	tsl.plot.updateBucket(b, age)
}

func (tsl *TimeSeriesLogger) Outputs() (outputs []string) {
	fmt.Println("[TimeSeriesLogger]: generating time series plot")
	outputs = append(outputs, tsl.plot.output())
	return
}

func (tsl *TimeSeriesLogger) Reset() {
	tsl.plot = newTimeSeriesPlot()
}
<commit_msg>More useful comments<commit_after>package bitcoin_load_spike

import (
	"fmt"
	_ "math"
)

type timeSeriesPlot struct {
	buckets        []float64
	counts         []int64
	smallestBucket int64
	largestBucket  int64
}

func newTimeSeriesPlot() *timeSeriesPlot {
	return &timeSeriesPlot{
		buckets:        make([]float64, NUM_BUCKETS),
		counts:         make([]int64, NUM_BUCKETS),
		smallestBucket: NUM_BUCKETS,
		largestBucket:  0,
	}
}

func (tsp *timeSeriesPlot) updateBucket(i int64, age float64) {
	total := tsp.buckets[i]*float64(tsp.counts[i]) + age
	tsp.counts[i]++
	tsp.buckets[i] = total / float64(tsp.counts[i])

	if tsp.largestBucket < i {
		tsp.largestBucket = i
	}
	if tsp.smallestBucket > i {
		tsp.smallestBucket = i
	}
}

func (tsp *timeSeriesPlot) output() (fileContents string) {
	for i, avgTxnTime := range tsp.buckets[0 : NUM_BUCKETS-1] {
		fileContents += fmt.Sprintf("%d | %f\n", i, avgTxnTime)
	}
	return
}

type TimeSeriesLogger struct {
	plot          *timeSeriesPlot
	secsPerBucket float64
	filePrefix    string
}

func (tsl TimeSeriesLogger) FilePrefix() string {
	return tsl.filePrefix
}

func (tsl TimeSeriesLogger) FileExtension() string {
	return "tsl-dat"
}

func (tsl *TimeSeriesLogger) Log(blockTimestamp, txnTimestamp float64, spikeNumber int) {
	age := blockTimestamp - txnTimestamp

	b := int64(txnTimestamp / tsl.secsPerBucket)
	// Extend buckets and counts if necessary
	if b >= int64(len(tsl.plot.buckets)) {
		diff := b - int64(len(tsl.plot.buckets)-1)
		// Extend buckets
		bucketsExtension := make([]float64, diff)
		tsl.plot.buckets = append(tsl.plot.buckets, bucketsExtension...)
		// Extend counts
		countsExtension := make([]int64, diff)
		tsl.plot.counts = append(tsl.plot.counts, countsExtension...)
	}

	tsl.plot.updateBucket(b, age)
}

func (tsl *TimeSeriesLogger) Outputs() (outputs []string) {
	fmt.Println("[TimeSeriesLogger]: generating time series plot")
	outputs = append(outputs, tsl.plot.output())
	return
}

func (tsl *TimeSeriesLogger) Reset() {
	tsl.plot = newTimeSeriesPlot()
}
<|endoftext|>