<commit_before>package transferarchiver

import (
	"archive/tar"
	"context"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	slashpath "path"

	"github.com/pkg/errors"
)

var (
	//TarArchiverKey configures the one object key returned by the tar Archiver
	TarArchiverKey = "archive.tar"

	//TarArchiverPathSeparator standardizes the header path for cross platform (un)archiving
	TarArchiverPathSeparator = "/"

	//ErrNotADirectory is returned when an archiver was asked to upload a single file without dir
	ErrNotADirectory = errors.New("it is not a directory")

	//ErrNoSuchDirectory is returned when an archiver expected a directory to exist
	ErrNoSuchDirectory = errors.New("directory doesn't exist")

	//ErrEmptyDirectory is returned when the archiver expected the directory to not be empty
	ErrEmptyDirectory = errors.New("directory is empty")

	//ErrDatasetTooLarge is returned when the dataset size is above 1Gb
	ErrDatasetTooLarge = errors.New("dataset is too big, limit is 1Gb")

	//SizeLimit is the maximum size allowed
	//@TODO: Should be based on customer details?
	SizeLimit = int64(1 * 1024 * 1024 * 1024)
)

//TarArchiver will archive a directory into a single tar file
type TarArchiver struct {
	keyPrefix string
	sizeLimit int64
}

//NewTarArchiver will setup the tar archiver
func NewTarArchiver(opts ArchiverOptions) (a *TarArchiver, err error) {
	a = &TarArchiver{keyPrefix: opts.TarArchiverKeyPrefix, sizeLimit: opts.SizeLimit}

	if a.keyPrefix != "" && !strings.HasSuffix(a.keyPrefix, "/") {
		return nil, errors.Errorf("archiver key prefix must end with a forward slash")
	}

	if a.sizeLimit <= 0 {
		a.sizeLimit = SizeLimit
	}

	return a, nil
}

//tempFile will setup a temproary file that can easily be cleaned
func (a *TarArchiver) tempFile() (f *os.File, clean func(), err error) {
	f, err = ioutil.TempFile("", "tar_archiver_")
	if err != nil {
		return nil, nil, errors.Wrap(err, "failed to create temporary file")
	}

	return f, func() {
		_ = f.Close()
		_ = os.Remove(f.Name())
	}, nil
}

func (a *TarArchiver) checkTargetDir(path string) error {
	dir, err := os.Open(path)
	if err != nil {
		if !os.IsNotExist(err) {
			return errors.Wrap(err, "failed to open directory")
		}

		err = os.Mkdir(path, 0777) //@TODO decide on permissions before umask
		if err != nil {
			return errors.Wrap(err, "failed to create directory")
		}

		dir, err = os.Open(path)
		if err != nil {
			return errors.Wrap(err, "failed open created directory")
		}
	}

	fis, err := dir.Readdirnames(1)
	if err != nil && err != io.EOF {
		return errors.Wrap(err, "failed to read directory")
	}

	if len(fis) > 0 {
		return errors.New("directory is not empty")
	}

	return nil
}

//Index calls 'fn' for all object keys that are part of the archive
func (a *TarArchiver) Index(fn func(k string) error) error {
	return fn(slashpath.Join(a.keyPrefix, TarArchiverKey))
}

//@TODO do we want to expose this through the interface?
func (a *TarArchiver) indexFS(path string, fn func(p string, fi os.FileInfo, err error) error) error {
	if err := filepath.Walk(path, func(p string, fi os.FileInfo, err error) error {
		if fi == nil || path == p {
			return nil //this is triggered when a directory doesn't have an executable bit
		}
		if err != nil {
			return err
		}

		return fn(p, fi, nil)
	}); err != nil {
		return err
	}

	return nil
}

//Archive will archive a directory at 'path' into readable objects 'r' and calls 'fn' for each
func (a *TarArchiver) Archive(ctx context.Context, path string, rep Reporter, fn func(k string, r io.ReadSeeker, nbytes int64) error) (err error) {
	f, err := os.Open(path)
	if err != nil {
		if os.IsNotExist(err) {
			return ErrNoSuchDirectory
		}

		return err
	}
	if info, err := os.Stat(path); !info.IsDir() {
		if err != nil {
			return err
		}
		return ErrNotADirectory
	}
	names, err := f.Readdirnames(1)
	if len(names) == 0 {
		return ErrEmptyDirectory
	}
	if err != nil {
		return err
	}

	var totalToTar int64
	if err = a.indexFS(path, func(p string, fi os.FileInfo, err error) error {
		if !fi.Mode().IsRegular() {
			return nil //nothing to write for dirs or symlinks
		}

		totalToTar += fi.Size()
		return nil
	}); err != nil {
		return errors.Wrap(err, "failed to index filesystem")
	}

	if totalToTar > a.sizeLimit {
		return ErrDatasetTooLarge
	}

	tmpf, clean, err := a.tempFile()
	if err != nil {
		return err
	}

	defer clean()
	inc := rep.StartArchivingProgress(tmpf.Name(), totalToTar)

	tw := tar.NewWriter(tmpf)
	defer tw.Close()

	if err = a.indexFS(path, func(p string, fi os.FileInfo, err error) error {
		rel, err := filepath.Rel(path, p)
		if err != nil {
			return errors.Wrap(err, "failed to determine relative path")
		}

		//write header with a filename that standardizes the Separator
		path := strings.Split(rel, string(filepath.Separator))
		hdr, err := tar.FileInfoHeader(fi, "") //@TODO find out how we handle symlinks
		if err != nil {
			return errors.Wrap(err, "failed to convert file info to tar header")
		}

		hdr.Name = strings.Join(path, TarArchiverPathSeparator)
		if err = tw.WriteHeader(hdr); err != nil {
			return errors.Wrap(err, "failed to write tar header")
		}

		if !fi.Mode().IsRegular() {
			return nil //nothing to write for dirs or symlinks
		}

		// open files for taring
		f, err := os.Open(p)
		defer f.Close()
		if err != nil {
			return errors.Wrap(err, "failed to open file for archiving")
		}

		// copy file data into tar writer
		var n int64
		if n, err = Copy(ctx, tw, f); err != nil {
			return errors.Wrap(err, "failed to copy file content to archive")
		}

		inc(n)
		return nil
	}); err != nil {
		return errors.Wrap(err, "failed to perform filesystem walk")
	}
	err = tw.Flush()
	if err != nil {
		return errors.Wrap(err, "failed to flush tar writer to disk")
	}

	_, err = tmpf.Seek(0, 0)
	if err != nil {
		return errors.Wrap(err, "failed to seek to beginning of file")
	}

	//stop progress reporting, we're done
	rep.StopArchivingProgress()
	fi, err := tmpf.Stat()
	if err != nil {
		return errors.Wrap(err, "failed to stat the temporary file")
	}

	return fn(slashpath.Join(a.keyPrefix, TarArchiverKey), tmpf, fi.Size())
}

//Unarchive will take a file system path and call 'fn' for each object that it needs for unarchiving.
//It writes to a temporary directory first and then moves this to the final location
func (a *TarArchiver) Unarchive(ctx context.Context, path string, rep Reporter, fn func(k string, w io.WriterAt) error) error {
	// We need to check the target directory first to avoid downloading data if there is a problem
	err := a.checkTargetDir(path)
	if err != nil {
		return err
	}

	tmpf, clean, err := a.tempFile()
	if err != nil {
		return err
	}

	defer clean()

	err = fn(slashpath.Join(a.keyPrefix, TarArchiverKey), tmpf)
	if err != nil {
		return errors.Wrap(err, "failed to download to temporary file")
	}

	_, err = tmpf.Seek(0, 0)
	if err != nil {
		return errors.Wrap(err, "failed to seek to the beginning of file")
	}

	fi, err := tmpf.Stat()
	if err != nil {
		return errors.Wrap(err, "failed to stat temporary file")
	}

	pr := rep.StartUnarchivingProgress(tmpf.Name(), fi.Size(), tmpf)
	defer rep.StopUnarchivingProgress()

	tr := tar.NewReader(pr)
	for {
		hdr, err := tr.Next()
		switch {
		case err == io.EOF:
			return nil //EOF we're done here
		case err != nil:
			return errors.Wrap(err, "failed to read next header")
		case hdr == nil:
			continue
		}

		// the target location where the dir/file should be created
		parts := []string{path}
		parts = append(parts, strings.Split(hdr.Name, TarArchiverPathSeparator)...)
		target := filepath.Join(parts...)

		switch hdr.Typeflag {
		case tar.TypeDir: //if its a dir and it doesn't exist create it, no-op if it exists already
			err = os.MkdirAll(target, hdr.FileInfo().Mode())
			if err != nil {
				return errors.Wrap(err, "failed to create directory for entry found in tar file")
			}

		case tar.TypeReg: //regular file is written, must not exist yet
			if err = func() (err error) {
				f, err := os.OpenFile(target, os.O_WRONLY|os.O_CREATE|os.O_EXCL, hdr.FileInfo().Mode())
				if err != nil {
					return errors.Wrap(err, "failed to open new file for tar entry ")
				}

				defer f.Close()
				if _, err := io.Copy(f, tr); err != nil {
					return errors.Wrap(err, "failed to copy archived file content")
				}

				return nil
			}(); err != nil {
				return errors.Wrap(err, "failed to extract file")
			}
		}
	}
}
<commit_msg>Update ErrDatasetTooLarge message<commit_after>package transferarchiver

import (
	"archive/tar"
	"context"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	slashpath "path"

	humanize "github.com/dustin/go-humanize"

	"github.com/pkg/errors"
)

var (
	//TarArchiverKey configures the one object key returned by the tar Archiver
	TarArchiverKey = "archive.tar"

	//TarArchiverPathSeparator standardizes the header path for cross platform (un)archiving
	TarArchiverPathSeparator = "/"

	//ErrNotADirectory is returned when an archiver was asked to upload a single file without dir
	ErrNotADirectory = errors.New("it is not a directory")

	//ErrNoSuchDirectory is returned when an archiver expected a directory to exist
	ErrNoSuchDirectory = errors.New("directory doesn't exist")

	//ErrEmptyDirectory is returned when the archiver expected the directory to not be empty
	ErrEmptyDirectory = errors.New("directory is empty")

	//ErrDatasetTooLarge is returned when the dataset size is above the sizelimit set in the dataset.
	ErrDatasetTooLarge = "dataset is too big, limit is %s"

	//SizeLimit is the maximum size allowed
	//@TODO: Should be based on customer details?
	SizeLimit = int64(1 * 1024 * 1024 * 1024)
)

//TarArchiver will archive a directory into a single tar file
type TarArchiver struct {
	keyPrefix string
	sizeLimit int64
}

//NewTarArchiver will setup the tar archiver
func NewTarArchiver(opts ArchiverOptions) (a *TarArchiver, err error) {
	a = &TarArchiver{keyPrefix: opts.TarArchiverKeyPrefix, sizeLimit: opts.SizeLimit}

	if a.keyPrefix != "" && !strings.HasSuffix(a.keyPrefix, "/") {
		return nil, errors.Errorf("archiver key prefix must end with a forward slash")
	}

	if a.sizeLimit <= 0 {
		a.sizeLimit = SizeLimit
	}

	return a, nil
}

//tempFile will setup a temproary file that can easily be cleaned
func (a *TarArchiver) tempFile() (f *os.File, clean func(), err error) {
	f, err = ioutil.TempFile("", "tar_archiver_")
	if err != nil {
		return nil, nil, errors.Wrap(err, "failed to create temporary file")
	}

	return f, func() {
		_ = f.Close()
		_ = os.Remove(f.Name())
	}, nil
}

func (a *TarArchiver) checkTargetDir(path string) error {
	dir, err := os.Open(path)
	if err != nil {
		if !os.IsNotExist(err) {
			return errors.Wrap(err, "failed to open directory")
		}

		err = os.Mkdir(path, 0777) //@TODO decide on permissions before umask
		if err != nil {
			return errors.Wrap(err, "failed to create directory")
		}

		dir, err = os.Open(path)
		if err != nil {
			return errors.Wrap(err, "failed open created directory")
		}
	}

	fis, err := dir.Readdirnames(1)
	if err != nil && err != io.EOF {
		return errors.Wrap(err, "failed to read directory")
	}

	if len(fis) > 0 {
		return errors.New("directory is not empty")
	}

	return nil
}

//Index calls 'fn' for all object keys that are part of the archive
func (a *TarArchiver) Index(fn func(k string) error) error {
	return fn(slashpath.Join(a.keyPrefix, TarArchiverKey))
}

//@TODO do we want to expose this through the interface?
func (a *TarArchiver) indexFS(path string, fn func(p string, fi os.FileInfo, err error) error) error {
	if err := filepath.Walk(path, func(p string, fi os.FileInfo, err error) error {
		if fi == nil || path == p {
			return nil //this is triggered when a directory doesn't have an executable bit
		}
		if err != nil {
			return err
		}

		return fn(p, fi, nil)
	}); err != nil {
		return err
	}

	return nil
}

//Archive will archive a directory at 'path' into readable objects 'r' and calls 'fn' for each
func (a *TarArchiver) Archive(ctx context.Context, path string, rep Reporter, fn func(k string, r io.ReadSeeker, nbytes int64) error) (err error) {
	f, err := os.Open(path)
	if err != nil {
		if os.IsNotExist(err) {
			return ErrNoSuchDirectory
		}

		return err
	}
	if info, err := os.Stat(path); !info.IsDir() {
		if err != nil {
			return err
		}
		return ErrNotADirectory
	}
	names, err := f.Readdirnames(1)
	if len(names) == 0 {
		return ErrEmptyDirectory
	}
	if err != nil {
		return err
	}

	var totalToTar int64
	if err = a.indexFS(path, func(p string, fi os.FileInfo, err error) error {
		if !fi.Mode().IsRegular() {
			return nil //nothing to write for dirs or symlinks
		}

		totalToTar += fi.Size()
		return nil
	}); err != nil {
		return errors.Wrap(err, "failed to index filesystem")
	}

	if totalToTar > a.sizeLimit {
		return errors.Errorf(ErrDatasetTooLarge, humanize.Bytes(uint64(a.sizeLimit)))
	}

	tmpf, clean, err := a.tempFile()
	if err != nil {
		return err
	}

	defer clean()
	inc := rep.StartArchivingProgress(tmpf.Name(), totalToTar)

	tw := tar.NewWriter(tmpf)
	defer tw.Close()

	if err = a.indexFS(path, func(p string, fi os.FileInfo, err error) error {
		rel, err := filepath.Rel(path, p)
		if err != nil {
			return errors.Wrap(err, "failed to determine relative path")
		}

		//write header with a filename that standardizes the Separator
		path := strings.Split(rel, string(filepath.Separator))
		hdr, err := tar.FileInfoHeader(fi, "") //@TODO find out how we handle symlinks
		if err != nil {
			return errors.Wrap(err, "failed to convert file info to tar header")
		}

		hdr.Name = strings.Join(path, TarArchiverPathSeparator)
		if err = tw.WriteHeader(hdr); err != nil {
			return errors.Wrap(err, "failed to write tar header")
		}

		if !fi.Mode().IsRegular() {
			return nil //nothing to write for dirs or symlinks
		}

		// open files for taring
		f, err := os.Open(p)
		defer f.Close()
		if err != nil {
			return errors.Wrap(err, "failed to open file for archiving")
		}

		// copy file data into tar writer
		var n int64
		if n, err = Copy(ctx, tw, f); err != nil {
			return errors.Wrap(err, "failed to copy file content to archive")
		}

		inc(n)
		return nil
	}); err != nil {
		return errors.Wrap(err, "failed to perform filesystem walk")
	}
	err = tw.Flush()
	if err != nil {
		return errors.Wrap(err, "failed to flush tar writer to disk")
	}

	_, err = tmpf.Seek(0, 0)
	if err != nil {
		return errors.Wrap(err, "failed to seek to beginning of file")
	}

	//stop progress reporting, we're done
	rep.StopArchivingProgress()
	fi, err := tmpf.Stat()
	if err != nil {
		return errors.Wrap(err, "failed to stat the temporary file")
	}

	return fn(slashpath.Join(a.keyPrefix, TarArchiverKey), tmpf, fi.Size())
}

//Unarchive will take a file system path and call 'fn' for each object that it needs for unarchiving.
//It writes to a temporary directory first and then moves this to the final location
func (a *TarArchiver) Unarchive(ctx context.Context, path string, rep Reporter, fn func(k string, w io.WriterAt) error) error {
	// We need to check the target directory first to avoid downloading data if there is a problem
	err := a.checkTargetDir(path)
	if err != nil {
		return err
	}

	tmpf, clean, err := a.tempFile()
	if err != nil {
		return err
	}

	defer clean()

	err = fn(slashpath.Join(a.keyPrefix, TarArchiverKey), tmpf)
	if err != nil {
		return errors.Wrap(err, "failed to download to temporary file")
	}

	_, err = tmpf.Seek(0, 0)
	if err != nil {
		return errors.Wrap(err, "failed to seek to the beginning of file")
	}

	fi, err := tmpf.Stat()
	if err != nil {
		return errors.Wrap(err, "failed to stat temporary file")
	}

	pr := rep.StartUnarchivingProgress(tmpf.Name(), fi.Size(), tmpf)
	defer rep.StopUnarchivingProgress()

	tr := tar.NewReader(pr)
	for {
		hdr, err := tr.Next()
		switch {
		case err == io.EOF:
			return nil //EOF we're done here
		case err != nil:
			return errors.Wrap(err, "failed to read next header")
		case hdr == nil:
			continue
		}

		// the target location where the dir/file should be created
		parts := []string{path}
		parts = append(parts, strings.Split(hdr.Name, TarArchiverPathSeparator)...)
		target := filepath.Join(parts...)

		switch hdr.Typeflag {
		case tar.TypeDir: //if its a dir and it doesn't exist create it, no-op if it exists already
			err = os.MkdirAll(target, hdr.FileInfo().Mode())
			if err != nil {
				return errors.Wrap(err, "failed to create directory for entry found in tar file")
			}

		case tar.TypeReg: //regular file is written, must not exist yet
			if err = func() (err error) {
				f, err := os.OpenFile(target, os.O_WRONLY|os.O_CREATE|os.O_EXCL, hdr.FileInfo().Mode())
				if err != nil {
					return errors.Wrap(err, "failed to open new file for tar entry ")
				}

				defer f.Close()
				if _, err := io.Copy(f, tr); err != nil {
					return errors.Wrap(err, "failed to copy archived file content")
				}

				return nil
			}(); err != nil {
				return errors.Wrap(err, "failed to extract file")
			}
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"net/url"
	"os"
)

var initial []string

func breadFirst(f func(string) []string, worklist []string) {
	initial = append(initial, worklist...)
	seen := make(map[string]bool)
	for len(worklist) > 0 {
		items := worklist
		worklist = nil
		for _, item := range items {
			if !seen[item] {
				seen[item] = true
				if len(seen) > 20 {
					return
				}
				worklist = append(worklist, f(item)...)
			}
		}
	}
}

func crawl(u string) []string {
	fmt.Println(u)
	list, err := Extract(u)
	if err != nil {
		log.Panic(err)
	}

	for _, link := range list {
		linkParsed, _ := url.Parse(link)
		for _, r := range os.Args[1:] {
			startParsed, _ := url.Parse(r)
			if linkParsed.Host == startParsed.Host {
				fmt.Println(">>> ", link)

			}
		}
	}
	return list
}

func main() {
	breadFirst(crawl, os.Args[1:])
}
<commit_msg>[5.13] Small refactoring.<commit_after>package main

import (
	"fmt"
	"log"
	"net/url"
	"os"
)

var initial []string

func breadFirst(f func(string) []string, worklist []string) {
	initial = append(initial, worklist...)
	seen := make(map[string]bool)
	for len(worklist) > 0 {
		items := worklist
		worklist = nil
		for _, item := range items {
			if !seen[item] {
				seen[item] = true
				if len(seen) > 20 {
					return
				}
				worklist = append(worklist, f(item)...)
			}
		}
	}
}

func crawl(u string) []string {
	fmt.Println(u)
	list, err := Extract(u)
	if err != nil {
		log.Panic(err)
	}
	forRelative(list, printRelative)
	return list
}

func forRelative(links []string, f func(*url.URL)) {
	for _, link := range links {
		linkParsed, _ := url.Parse(link)
		for _, r := range os.Args[1:] {
			startParsed, _ := url.Parse(r)
			if linkParsed.Host == startParsed.Host {
				f(linkParsed)
			}
		}
	}
}

func printRelative(u *url.URL) {
	fmt.Println(">>> ", u.Path)
}

func main() {
	breadFirst(crawl, os.Args[1:])
}
<|endoftext|>


<commit_before>// Copyright (c) 2015 Uber Technologies, Inc.

// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package tchannel_test

import (
	"bufio"
	"bytes"
	"io"
	"runtime"
	"strings"
	"testing"
	"time"
)

func getStacks() []byte {
	for i := 4096; ; i *= 2 {
		buf := make([]byte, i)
		if n := runtime.Stack(buf, true /* all */); n < i {
			return buf
		}
	}
}

type goroutineStack struct {
	leakLine  string
	fullStack []string
	goState   string
}

func parseGoState(line string) string {
	// parse state from 'goroutine 10 [syscall]:'
	i1 := strings.Index(line, "[")
	i2 := strings.Index(line, "]")
	if i1 < 0 || i2 < 0 {
		return ""
	}
	return line[i1+1 : i2]
}

func getMatchingGoStacks() []goroutineStack {
	badMatches := []string{
		"(*Channel).Serve",
		"(*Connection).readFrames",
		"(*Connection).writeFrames",
	}

	var matching []goroutineStack
	var curStack []string
	var goState string
	stackReader := bufio.NewReader(bytes.NewReader(getStacks()))
	for {
		line, err := stackReader.ReadString('\n')
		if err == io.EOF {
			break
		}
		if err != nil {
			panic("stack reader failed")
		}

		// If we see the goroutine header, start a new stack.
		if strings.HasPrefix(line, "goroutine ") {
			curStack = nil
			goState = parseGoState(line)
		}
		curStack = append(curStack, line)

		// If the line matches any of our badMatches, log it.
		for _, m := range badMatches {
			if strings.Contains(line, m) {
				matching = append(matching, goroutineStack{
					leakLine:  line,
					fullStack: curStack,
					goState:   goState,
				})
			}
		}
	}

	return matching
}

// VerifyNoBlockedGoroutines verifies that there are no goroutines in the global space
// that are stuck inside of readFrames or writeFrames.
// Since some goroutines may still be performing work in the background, we retry the
// checks if any goroutines are fine in a running state a finite number of times.
func VerifyNoBlockedGoroutines(t *testing.T) {
	retryStates := map[string]struct{}{
		"runnable": struct{}{},
		"running":  struct{}{},
		"syscall":  struct{}{},
	}
	const maxAttempts = 10

retry:
	for i := 0; i < maxAttempts; i++ {
		runtime.Gosched()
		if i > maxAttempts/2 {
			time.Sleep(time.Millisecond)
		}

		matching := getMatchingGoStacks()
		for _, v := range matching {
			if _, ok := retryStates[v.goState]; ok {
				continue retry
			}
		}

		for _, v := range matching {
			t.Errorf("Found leaked goroutine in state %q, leakLine:\n  %s  Full stack:\n%s",
				v.goState, v.leakLine, strings.Join(v.fullStack, ""))
		}
		return
	}

	t.Errorf("VerifyNoBlockedGoroutines failed: too many retries")
}
<commit_msg>Rewrite goroutine leak detector<commit_after>// Copyright (c) 2015 Uber Technologies, Inc.

// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package tchannel_test

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"runtime"
	"strconv"
	"strings"
	"testing"
	"time"
)

func getStacks() []byte {
	for i := 4096; ; i *= 2 {
		buf := make([]byte, i)
		if n := runtime.Stack(buf, true /* all */); n < i {
			return buf
		}
	}
}

// parseGoStackHeader parses a stack header that looks like:
// goroutine 643 [runnable]:
// And returns the goroutine ID, and the state.
func parseGoStackHeader(line string) (goroutineID int, state string) {
	parts := strings.SplitN(line, " ", 3)
	if len(parts) != 3 {
		panic(fmt.Sprintf("unexpected stack header format: %v", line))
	}

	id, err := strconv.Atoi(parts[1])
	if err != nil {
		panic(fmt.Sprintf("failed to parse goroutine ID: %v", parts[1]))
	}

	state = strings.TrimSuffix(strings.TrimPrefix(parts[2], "["), "]")
	return id, state
}

type goroutineStack struct {
	id        int
	fullStack *bytes.Buffer
	goState   string
}

func getAllStacks() []goroutineStack {
	var stacks []goroutineStack

	var curStack *goroutineStack
	stackReader := bufio.NewReader(bytes.NewReader(getStacks()))
	for {
		line, err := stackReader.ReadString('\n')
		if err == io.EOF {
			break
		}
		if err != nil {
			panic("stack reader failed")
		}

		// If we see the goroutine header, start a new stack.
		if strings.HasPrefix(line, "goroutine ") {
			// flush any previous stack
			if curStack != nil {
				stacks = append(stacks, *curStack)
			}
			id, goState := parseGoStackHeader(line)
			curStack = &goroutineStack{
				id:        id,
				goState:   goState,
				fullStack: &bytes.Buffer{},
			}
		}
		curStack.fullStack.WriteString(line)
	}

	if curStack != nil {
		stacks = append(stacks, *curStack)
	}
	return stacks
}

// isLeak returns whether the given stack contains a stack frame that is considered a leak.
func (s goroutineStack) isLeak() bool {
	isLeakLine := func(line string) bool {
		return strings.Contains(line, "(*Channel).Serve") ||
			strings.Contains(line, "(*Connection).readFrames") ||
			strings.Contains(line, "(*Connection).writeFrames")
	}

	for {
		s.fullStack.Reset()
		line, err := s.fullStack.ReadString('\n')
		if err == io.EOF {
			return false
		}
		if err != nil {
			panic(err)
		}

		if isLeakLine(line) {
			return true
		}
	}
}

func getLeakStacks(stacks []goroutineStack) []goroutineStack {
	var leakStacks []goroutineStack
	for _, s := range stacks {
		if s.isLeak() {
			leakStacks = append(leakStacks, s)
		}
	}
	return leakStacks
}

// VerifyNoBlockedGoroutines verifies that there are no goroutines in the global space
// that are stuck inside of readFrames or writeFrames.
// Since some goroutines may still be performing work in the background, we retry the
// checks if any goroutines are fine in a running state a finite number of times.
func VerifyNoBlockedGoroutines(t *testing.T) {
	retryStates := map[string]struct{}{
		"runnable": struct{}{},
		"running":  struct{}{},
		"syscall":  struct{}{},
	}
	const maxAttempts = 10

retry:
	for i := 0; i < maxAttempts; i++ {
		// Ignore the first stack which is the current goroutine.
		stacks := getAllStacks()[1:]
		for _, stack := range stacks {
			if _, ok := retryStates[stack.goState]; ok {
				runtime.Gosched()
				if i > maxAttempts/2 {
					time.Sleep(time.Millisecond)
				}
				continue retry
			}
		}

		// There are no running/runnable goroutines, so check for bad leaks.
		leakStacks := getLeakStacks(stacks)
		for _, v := range leakStacks {
			t.Errorf("Found leaked goroutine in state %q Full stack:\n%s\n",
				v.goState, v.fullStack.String())
		}
		return
	}

	t.Errorf("VerifyNoBlockedGoroutines failed: too many retries")
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mungers

import (
	"fmt"
	"strings"
	"time"

	"k8s.io/contrib/mungegithub/features"
	"k8s.io/contrib/mungegithub/github"
	cache "k8s.io/contrib/mungegithub/mungers/flakesync"
	"k8s.io/contrib/test-utils/utils"

	"github.com/golang/glog"
	"github.com/spf13/cobra"
)

// issueFinder finds an issue for a given key.
type issueFinder interface {
	IssueForKey(key string) (int, bool)
	Created(key string, number int)
	Synced() bool
}

// FlakeManager files issues for flaky tests.
type FlakeManager struct {
	finder               issueFinder
	sq                   *SubmitQueue
	config               *github.Config
	googleGCSBucketUtils *utils.Utils

	oldestTime time.Time

	// map of flake to the issue number
	alreadySyncedFlakes map[cache.Flake]int

	// TODO: go backwards fetching statuses until we reach oldestTime
	// The next run number we need to check for each job
	// jobToRunNumber map[string]int
}

func init() {
	RegisterMungerOrDie(&FlakeManager{})
}

// Name is the name usable in --pr-mungers
func (p *FlakeManager) Name() string { return "flake-manager" }

// RequiredFeatures is a slice of 'features' that must be provided
func (p *FlakeManager) RequiredFeatures() []string { return nil }

// Initialize will initialize the munger
func (p *FlakeManager) Initialize(config *github.Config, features *features.Features) error {
	// TODO: don't get the mungers from the global list, they should be passed in...
	for _, m := range GetAllMungers() {
		if m.Name() == "issue-cacher" {
			p.finder = m.(*IssueCacher)
		}
		if m.Name() == "submit-queue" {
			p.sq = m.(*SubmitQueue)
		}
	}
	if p.finder == nil {
		return fmt.Errorf("issue-cacher not found")
	}
	if p.sq == nil {
		return fmt.Errorf("submit-queue not found")
	}
	p.oldestTime = time.Now().Add(-time.Hour * 24)
	p.alreadySyncedFlakes = map[cache.Flake]int{}
	p.config = config
	p.googleGCSBucketUtils = utils.NewUtils(utils.GoogleBucketURL)
	return nil
}

// EachLoop is called at the start of every munge loop
func (p *FlakeManager) EachLoop() error {
	if p.sq.e2e == nil {
		return fmt.Errorf("submit queue not initialized yet")
	}
	if !p.finder.Synced() {
		return nil
	}
	p.sq.e2e.GCSBasedStable()
	for _, f := range p.sq.e2e.Flakes() {
		p.syncFlake(f)
	}
	return nil
}

// AddFlags will add any request flags to the cobra `cmd`
func (p *FlakeManager) AddFlags(cmd *cobra.Command, config *github.Config) {}

// Munge is unused by this munger.
func (p *FlakeManager) Munge(obj *github.MungeObject) {}

func (p *FlakeManager) syncFlake(f cache.Flake) error {
	if _, ok := p.alreadySyncedFlakes[f]; ok {
		return nil
	}
	if n, ok := p.finder.IssueForKey(string(f.Test)); ok {
		if err := p.updateIssue(n, f); err != nil {
			return fmt.Errorf("error updating issue %v for %v: %v", n, f, err)
		}
		p.alreadySyncedFlakes[f] = n
		return nil
	}
	n, err := p.createIssue(f)
	if err != nil {
		return fmt.Errorf("error making issue for %v: %v", f, err)
	}
	p.finder.Created(string(f.Test), n)
	p.alreadySyncedFlakes[f] = n
	return nil
}

// DO NOT CHANGE or it will not recognize previous entries!
func (p *FlakeManager) flakeID(flake cache.Flake) string {
	return p.googleGCSBucketUtils.GetPathToJenkinsGoogleBucket(string(flake.Job), int(flake.Number), "") + "\n"
}

func (p *FlakeManager) flakeExtraInfo(flake cache.Flake) string {
	return fmt.Sprintf("Failed: %v\n\n```\n%v\n```\n\n", flake.Test, flake.Reason)
}

func (p *FlakeManager) updateIssue(issueNumber int, flake cache.Flake) error {
	flakeID := p.flakeID(flake)
	obj, err := p.config.GetObject(issueNumber)
	if err != nil {
		return fmt.Errorf("error getting object for %v: %v", issueNumber, err)
	}
	comments, err := obj.ListComments()
	if err != nil {
		return fmt.Errorf("error getting comments for %v: %v", *obj.Issue.Number, err)
	}
	if obj.Issue.Body != nil && strings.Contains(*obj.Issue.Body, flakeID) {
		// We already wrote this flake
		return nil
	}
	for _, c := range comments {
		if c.Body == nil {
			continue
		}
		if strings.Contains(*c.Body, flakeID) {
			// We already wrote this flake
			return nil
		}
	}
	glog.Infof("Updating issue %v with flake %v", issueNumber, flake)
	return obj.WriteComment(flakeID + "\n" + p.flakeExtraInfo(flake))
}

func (p *FlakeManager) createIssue(flake cache.Flake) (issueNumber int, err error) {
	obj, err := p.config.NewIssue(
		string(flake.Test),                            // title
		p.flakeID(flake)+"\n"+p.flakeExtraInfo(flake), // body
		[]string{"kind/flake"},                        // labels
	)
	if err != nil {
		return 0, err
	}
	glog.Infof("Created issue %v for flake %v", *obj.Issue.Number, flake)
	return *obj.Issue.Number, nil
}
<commit_msg>use link that works for directories<commit_after>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mungers

import (
	"fmt"
	"strings"
	"time"

	"k8s.io/contrib/mungegithub/features"
	"k8s.io/contrib/mungegithub/github"
	cache "k8s.io/contrib/mungegithub/mungers/flakesync"
	"k8s.io/contrib/test-utils/utils"

	"github.com/golang/glog"
	"github.com/spf13/cobra"
)

const (
	// URLTestStorageBucket is a link that works for *directories* and not
	// files, since we want to link people to something that lets them
	// browse all the artifacts.
	URLTestStorageBucket = "https://console.cloud.google.com/storage/kubernetes-jenkins/logs"
)

// issueFinder finds an issue for a given key.
type issueFinder interface {
	IssueForKey(key string) (int, bool)
	Created(key string, number int)
	Synced() bool
}

// FlakeManager files issues for flaky tests.
type FlakeManager struct {
	finder               issueFinder
	sq                   *SubmitQueue
	config               *github.Config
	googleGCSBucketUtils *utils.Utils

	oldestTime time.Time

	// map of flake to the issue number
	alreadySyncedFlakes map[cache.Flake]int

	// TODO: go backwards fetching statuses until we reach oldestTime
	// The next run number we need to check for each job
	// jobToRunNumber map[string]int
}

func init() {
	RegisterMungerOrDie(&FlakeManager{})
}

// Name is the name usable in --pr-mungers
func (p *FlakeManager) Name() string { return "flake-manager" }

// RequiredFeatures is a slice of 'features' that must be provided
func (p *FlakeManager) RequiredFeatures() []string { return nil }

// Initialize will initialize the munger
func (p *FlakeManager) Initialize(config *github.Config, features *features.Features) error {
	// TODO: don't get the mungers from the global list, they should be passed in...
	for _, m := range GetAllMungers() {
		if m.Name() == "issue-cacher" {
			p.finder = m.(*IssueCacher)
		}
		if m.Name() == "submit-queue" {
			p.sq = m.(*SubmitQueue)
		}
	}
	if p.finder == nil {
		return fmt.Errorf("issue-cacher not found")
	}
	if p.sq == nil {
		return fmt.Errorf("submit-queue not found")
	}
	p.oldestTime = time.Now().Add(-time.Hour * 24)
	p.alreadySyncedFlakes = map[cache.Flake]int{}
	p.config = config
	p.googleGCSBucketUtils = utils.NewUtils(URLTestStorageBucket)
	return nil
}

// EachLoop is called at the start of every munge loop
func (p *FlakeManager) EachLoop() error {
	if p.sq.e2e == nil {
		return fmt.Errorf("submit queue not initialized yet")
	}
	if !p.finder.Synced() {
		return nil
	}
	p.sq.e2e.GCSBasedStable()
	for _, f := range p.sq.e2e.Flakes() {
		p.syncFlake(f)
	}
	return nil
}

// AddFlags will add any request flags to the cobra `cmd`
func (p *FlakeManager) AddFlags(cmd *cobra.Command, config *github.Config) {}

// Munge is unused by this munger.
func (p *FlakeManager) Munge(obj *github.MungeObject) {}

func (p *FlakeManager) syncFlake(f cache.Flake) error {
	if _, ok := p.alreadySyncedFlakes[f]; ok {
		return nil
	}
	if n, ok := p.finder.IssueForKey(string(f.Test)); ok {
		if err := p.updateIssue(n, f); err != nil {
			return fmt.Errorf("error updating issue %v for %v: %v", n, f, err)
		}
		p.alreadySyncedFlakes[f] = n
		return nil
	}
	n, err := p.createIssue(f)
	if err != nil {
		return fmt.Errorf("error making issue for %v: %v", f, err)
	}
	p.finder.Created(string(f.Test), n)
	p.alreadySyncedFlakes[f] = n
	return nil
}

// DO NOT CHANGE or it will not recognize previous entries!
func (p *FlakeManager) flakeID(flake cache.Flake) string {
	return p.googleGCSBucketUtils.GetPathToJenkinsGoogleBucket(string(flake.Job), int(flake.Number), "") + "\n"
}

func (p *FlakeManager) flakeExtraInfo(flake cache.Flake) string {
	return fmt.Sprintf("Failed: %v\n\n```\n%v\n```\n\n", flake.Test, flake.Reason)
}

func (p *FlakeManager) updateIssue(issueNumber int, flake cache.Flake) error {
	flakeID := p.flakeID(flake)
	obj, err := p.config.GetObject(issueNumber)
	if err != nil {
		return fmt.Errorf("error getting object for %v: %v", issueNumber, err)
	}
	comments, err := obj.ListComments()
	if err != nil {
		return fmt.Errorf("error getting comments for %v: %v", *obj.Issue.Number, err)
	}
	if obj.Issue.Body != nil && strings.Contains(*obj.Issue.Body, flakeID) {
		// We already wrote this flake
		return nil
	}
	for _, c := range comments {
		if c.Body == nil {
			continue
		}
		if strings.Contains(*c.Body, flakeID) {
			// We already wrote this flake
			return nil
		}
	}
	glog.Infof("Updating issue %v with flake %v", issueNumber, flake)
	return obj.WriteComment(flakeID + "\n" + p.flakeExtraInfo(flake))
}

func (p *FlakeManager) createIssue(flake cache.Flake) (issueNumber int, err error) {
	obj, err := p.config.NewIssue(
		string(flake.Test),                            // title
		p.flakeID(flake)+"\n"+p.flakeExtraInfo(flake), // body
		[]string{"kind/flake"},                        // labels
	)
	if err != nil {
		return 0, err
	}
	glog.Infof("Created issue %v for flake %v", *obj.Issue.Number, flake)
	return *obj.Issue.Number, nil
}
<|endoftext|>


<commit_before>package eval

import "errors"

var NotSupportedOnWindows = errors.New("not supported on Windows")

func execFn(ec *EvalCtx, args []Value, opts map[string]Value) {
	throw(NotSupportedOnWindows)
}

func fg(ec *EvalCtx, args []Value, opts map[string]Value) {
	throw(NotSupportedOnWindows)
}
<commit_msg>Fix build on Windows.<commit_after>package eval

import "errors"

var NotSupportedOnWindows = errors.New("not supported on Windows")

func execFn(ec *Frame, args []Value, opts map[string]Value) {
	throw(NotSupportedOnWindows)
}

func fg(ec *Frame, args []Value, opts map[string]Value) {
	throw(NotSupportedOnWindows)
}
<|endoftext|>


<commit_before>package main

import (
	"cqrs/examples/bank"
	"math/rand"
	"os"
	"time"

	"github.com/oklog/ulid"
)

func main() {
	end := make(chan bool)
	commandBus, err := config()
	if err != nil {
		os.Exit(1)
	}

	//Create Account
	for i := 0; i < 3000; i++ {
		go func() {
			var account bank.CreateAccount
			t := time.Now()
			entropy := rand.New(rand.NewSource(t.UnixNano()))
			uuid := ulid.MustNew(ulid.Timestamp(t), entropy).String()
			//uuid := "0000XSNJG0MQJHBF4QX1EFD6Y1"
			account.AggregateID = uuid
			commandBus.HandleCommand(account)
		}()
	}
	<-end
}
<commit_msg>chore(example): bank example now uses utils.UUID<commit_after>package main

import (
	"cqrs/examples/bank"
	"cqrs/utils"
	"os"
)

func main() {
	end := make(chan bool)
	commandBus, err := config()
	if err != nil {
		os.Exit(1)
	}

	//Create Account
	for i := 0; i < 3000; i++ {
		go func() {
			var account bank.CreateAccount
			uuid, err := utils.UUID()
			if err != nil {
				return
			}
			account.AggregateID = uuid
			commandBus.HandleCommand(account)
		}()
	}
	<-end
}
<|endoftext|>


<commit_before>// Copyright 2017 The go-hep Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package riofs_test

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"reflect"
	"runtime"
	"strings"
	"testing"
	"time"

	"go-hep.org/x/hep/groot"
	"go-hep.org/x/hep/groot/internal/rtests"
	"go-hep.org/x/hep/groot/rbase"
	"go-hep.org/x/hep/groot/rcont"
	"go-hep.org/x/hep/groot/riofs"
	"go-hep.org/x/hep/groot/root"
	"go-hep.org/x/hep/groot/rtree"
)

func TestFileDirectory(t *testing.T) {
	for _, fname := range []string{
		"../testdata/small-flat-tree.root",
		rtests.XrdRemote("testdata/small-flat-tree.root"),
	} {
		t.Run(fname, func(t *testing.T) {
			f, err := groot.Open(fname)
			if err != nil {
				t.Fatal(err.Error())
			}
			defer f.Close()

			for _, table := range []struct {
				test  string
				value string
				want  string
			}{
				{"Name", f.Name(), "test-small.root"}, // name when created
				{"Title", f.Title(), "small event file"},
				{"Class", f.Class(), "TFile"},
			} {
				if table.value != table.want {
					t.Fatalf("%v: got=%q, want=%q", table.test, table.value, table.want)
				}
			}

			for _, table := range []struct {
				name string
				want bool
			}{
				{"tree", true},
				{"tree;0", false},
				{"tree;1", true},
				{"tree;9999", true},
				{"tree_nope", false},
				{"tree_nope;0", false},
				{"tree_nope;1", false},
				{"tree_nope;9999", false},
			} {
				_, err := f.Get(table.name)
				if (err == nil) != table.want {
					t.Fatalf("%s: got key (err=%v). want=%v", table.name, err, table.want)
				}
			}

			for _, table := range []struct {
				name string
				want string
			}{
				{"tree", "TTree"},
				{"tree;1", "TTree"},
			} {
				k, err := f.Get(table.name)
				if err != nil {
					t.Fatalf("%s: expected key to exist! (got %v)", table.name, err)
				}

				if k.Class() != table.want {
					t.Fatalf("%s: got key with class=%s (want=%s)", table.name, k.Class(), table.want)
				}
			}

			for _, table := range []struct {
				name string
				want string
			}{
				{"tree", "tree"},
				{"tree;1", "tree"},
			} {
				o, err := f.Get(table.name)
				if err != nil {
					t.Fatalf("%s: expected key to exist! (got %v)", table.name, err)
				}

				k := o.(root.Named)
				if k.Name() != table.want {
					t.Fatalf("%s: got key with name=%s (want=%v)", table.name, k.Name(), table.want)
				}
			}

			for _, table := range []struct {
				name string
				want string
			}{
				{"tree", "my tree title"},
				{"tree;1", "my tree title"},
			} {
				o, err := f.Get(table.name)
				if err != nil {
					t.Fatalf("%s: expected key to exist! (got %v)", table.name, err)
				}

				k := o.(root.Named)
				if k.Title() != table.want {
					t.Fatalf("%s: got key with title=%s (want=%v)", table.name, k.Title(), table.want)
				}
			}
		})
	}
}

func TestFileOpenStreamerInfo(t *testing.T) {
	for _, fname := range []string{
		"../testdata/small-flat-tree.root",
		"../testdata/simple.root",
		rtests.XrdRemote("testdata/small-flat-tree.root"),
		rtests.XrdRemote("testdata/simple.root"),
	} {
		f, err := groot.Open(fname)
		if err != nil {
			t.Errorf("error opening %q: %v\n", fname, err)
			continue
		}
		defer f.Close()

		_ = f.StreamerInfos()
	}
}

func TestOpenEmptyFile(t *testing.T) {
	f, err := groot.Open("../testdata/uproot/issue70.root")
	if err != nil {
		t.Fatal(err)
	}
	defer f.Close()
	si := f.StreamerInfos()
	if si != nil {
		t.Fatalf("expected no StreamerInfos in empty file")
	}
}

func TestCreate(t *testing.T) {

	rootls := "rootls"
	if runtime.GOOS == "windows" {
		rootls = "rootls.exe"
	}

	rootls, err := exec.LookPath(rootls)
	withROOTCxx := err == nil

	dir, err := ioutil.TempDir("", "riofs-")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(dir)

	for i, tc := range []struct {
		name string
		skip bool
		want []rtests.ROOTer
	}{
		{name: "", want: nil},
		{
			name: "TObjString",
			want: []rtests.ROOTer{rbase.NewObjString("hello")},
		},
		{
			name: "TObjString",
			want: []rtests.ROOTer{rbase.NewObjString("hello"), rbase.NewObjString("world")},
		},
		{
			name: "TObjString",
			want: func() []rtests.ROOTer {
				var out []rtests.ROOTer
				for _, i := range []int{0, 1, 253, 254, 255, 256, 512, 1024} {
					str := strings.Repeat("=", i)
					out = append(out, rbase.NewObjString(str))
				}
				return out
			}(),
		},
		{
			name: "TObject",
			want: []rtests.ROOTer{rbase.NewObject()},
		},
		{
			name: "TNamed",
			want: []rtests.ROOTer{
				rbase.NewNamed("n0", "t0"),
				rbase.NewNamed("n1", "t1"),
				rbase.NewNamed("n2", "t2"),
			},
		},
		{
			name: "TList",
			want: []rtests.ROOTer{rcont.NewList("list-name", []root.Object{
				rbase.NewNamed("n0", "t0"),
				rbase.NewNamed("n1", "t1"),
				rbase.NewNamed("n2", "t2"),
			})},
		},
		{
			name: "TArrayF",
			want: []rtests.ROOTer{
				&rcont.ArrayF{Data: []float32{1, 2, 3, 4, 5, 6}},
			},
		},
		{
			name: "TArrayD",
			want: []rtests.ROOTer{
				&rcont.ArrayD{Data: []float64{1, 2, 3, 4, 5, 6}},
			},
		},
		{
			name: "TArrays",
			want: []rtests.ROOTer{
				&rcont.ArrayF{Data: []float32{1, 2, 3, 4, 5, 6}},
				&rcont.ArrayD{Data: []float64{1, 2, 3, 4, 5, 6}},
			},
		},
	} {
		fname := filepath.Join(dir, fmt.Sprintf("out-%d.root", i))
		t.Run(tc.name, func(t *testing.T) {
			if tc.skip {
				t.Skip()
			}

			w, err := groot.Create(fname)
			if err != nil {
				t.Fatal(err)
			}

			for i := range tc.want {
				var (
					kname = fmt.Sprintf("key-%s-%02d", tc.name, i)
					want  = tc.want[i]
				)

				err = w.Put(kname, want)
				if err != nil {
					t.Fatal(err)
				}
			}

			if got, want := len(w.Keys()), len(tc.want); got != want {
				t.Fatalf("invalid number of keys. got=%d, want=%d", got, want)
			}

			err = w.Close()
			if err != nil {
				t.Fatalf("error closing file: %v", err)
			}

			r, err := groot.Open(fname)
			if err != nil {
				t.Fatal(err)
			}
			defer r.Close()

			if got, want := len(r.Keys()), len(tc.want); got != want {
				t.Fatalf("invalid number of keys. got=%d, want=%d", got, want)
			}

			for i := range tc.want {
				var (
					kname = fmt.Sprintf("key-%s-%02d", tc.name, i)
					want  = tc.want[i]
				)

				rgot, err := r.Get(kname)
				if err != nil {
					t.Fatal(err)
				}

				if got := rgot.(rtests.ROOTer); !reflect.DeepEqual(got, want) {
					t.Fatalf("error reading back value[%d].\ngot = %#v\nwant = %#v", i, got, want)
				}
			}

			err = r.Close()
			if err != nil {
				t.Fatalf("error closing file: %v", err)
			}

			if !withROOTCxx {
				t.Logf("skip test with ROOT/C++")
				return
			}

			cmd := exec.Command(rootls, "-l", fname)
			err = cmd.Run()
			if err != nil {
				t.Fatalf("ROOT/C++ could not open file %q", fname)
			}
		})
	}
}

func TestOpenBigFile(t *testing.T) {
	ch := make(chan int)
	go func() {
		f, err := riofs.Open("root://eospublic.cern.ch//eos/root-eos/cms_opendata_2012_nanoaod/SMHiggsToZZTo4L.root")
		if err != nil {
			t.Fatal(err)
		}
		defer f.Close()

		o, err := f.Get("Events")
		if err != nil {
			t.Fatal(err)
		}

		tree := o.(rtree.Tree)
		if got, want := tree.Entries(), int64(299973); got != want {
			t.Fatalf("invalid entries: got=%d, want=%d", got, want)
		}
		ch <- 1
	}()

	timeout := time.NewTimer(10 * time.Second)
	defer timeout.Stop()
	select {
	case <-ch:
		// ok
	case <-timeout.C:
		t.Fatalf("timeout")
	}
}

func TestReadOnlyFile(t *testing.T) {
	f, err := groot.Open("../testdata/dirs-6.14.00.root")
	if err != nil {
		t.Fatal(err)
	}
	defer f.Close()

	err = f.Put("o1", rbase.NewObjString("v1"))
	if err == nil {
		t.Fatalf("expected an error. got nil")
	}

	o, err := f.Get("dir1")
	if err != nil {
		t.Fatal(err)
	}

	dir1 := o.(riofs.Directory)
	err = dir1.Put("o2", rbase.NewObjString("v2"))
	if err == nil {
		t.Fatalf("expected an error. got nil")
	}

	o, err = dir1.Get("dir11")
	if err != nil {
		t.Fatal(err)
	}

	dir11 := o.(riofs.Directory)
	err = dir11.Put("o3", rbase.NewObjString("v3"))
	if err == nil {
		t.Fatalf("expected an error. got nil")
	}
}
<commit_msg>groot/riofs: increase timeout for eos-root big file test<commit_after>// Copyright 2017 The go-hep Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package riofs_test

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"reflect"
	"runtime"
	"strings"
	"testing"
	"time"

	"go-hep.org/x/hep/groot"
	"go-hep.org/x/hep/groot/internal/rtests"
	"go-hep.org/x/hep/groot/rbase"
	"go-hep.org/x/hep/groot/rcont"
	"go-hep.org/x/hep/groot/riofs"
	"go-hep.org/x/hep/groot/root"
	"go-hep.org/x/hep/groot/rtree"
)

func TestFileDirectory(t *testing.T) {
	for _, fname := range []string{
		"../testdata/small-flat-tree.root",
		rtests.XrdRemote("testdata/small-flat-tree.root"),
	} {
		t.Run(fname, func(t *testing.T) {
			f, err := groot.Open(fname)
			if err != nil {
				t.Fatal(err.Error())
			}
			defer f.Close()

			for _, table := range []struct {
				test  string
				value string
				want  string
			}{
				{"Name", f.Name(), "test-small.root"}, // name when created
				{"Title", f.Title(), "small event file"},
				{"Class", f.Class(), "TFile"},
			} {
				if table.value != table.want {
					t.Fatalf("%v: got=%q, want=%q", table.test, table.value, table.want)
				}
			}

			for _, table := range []struct {
				name string
				want bool
			}{
				{"tree", true},
				{"tree;0", false},
				{"tree;1", true},
				{"tree;9999", true},
				{"tree_nope", false},
				{"tree_nope;0", false},
				{"tree_nope;1", false},
				{"tree_nope;9999", false},
			} {
				_, err := f.Get(table.name)
				if (err == nil) != table.want {
					t.Fatalf("%s: got key (err=%v). want=%v", table.name, err, table.want)
				}
			}

			for _, table := range []struct {
				name string
				want string
			}{
				{"tree", "TTree"},
				{"tree;1", "TTree"},
			} {
				k, err := f.Get(table.name)
				if err != nil {
					t.Fatalf("%s: expected key to exist! (got %v)", table.name, err)
				}

				if k.Class() != table.want {
					t.Fatalf("%s: got key with class=%s (want=%s)", table.name, k.Class(), table.want)
				}
			}

			for _, table := range []struct {
				name string
				want string
			}{
				{"tree", "tree"},
				{"tree;1", "tree"},
			} {
				o, err := f.Get(table.name)
				if err != nil {
					t.Fatalf("%s: expected key to exist! (got %v)", table.name, err)
				}

				k := o.(root.Named)
				if k.Name() != table.want {
					t.Fatalf("%s: got key with name=%s (want=%v)", table.name, k.Name(), table.want)
				}
			}

			for _, table := range []struct {
				name string
				want string
			}{
				{"tree", "my tree title"},
				{"tree;1", "my tree title"},
			} {
				o, err := f.Get(table.name)
				if err != nil {
					t.Fatalf("%s: expected key to exist! (got %v)", table.name, err)
				}

				k := o.(root.Named)
				if k.Title() != table.want {
					t.Fatalf("%s: got key with title=%s (want=%v)", table.name, k.Title(), table.want)
				}
			}
		})
	}
}

func TestFileOpenStreamerInfo(t *testing.T) {
	for _, fname := range []string{
		"../testdata/small-flat-tree.root",
		"../testdata/simple.root",
		rtests.XrdRemote("testdata/small-flat-tree.root"),
		rtests.XrdRemote("testdata/simple.root"),
	} {
		f, err := groot.Open(fname)
		if err != nil {
			t.Errorf("error opening %q: %v\n", fname, err)
			continue
		}
		defer f.Close()

		_ = f.StreamerInfos()
	}
}

func TestOpenEmptyFile(t *testing.T) {
	f, err := groot.Open("../testdata/uproot/issue70.root")
	if err != nil {
		t.Fatal(err)
	}
	defer f.Close()
	si := f.StreamerInfos()
	if si != nil {
		t.Fatalf("expected no StreamerInfos in empty file")
	}
}

func TestCreate(t *testing.T) {

	rootls := "rootls"
	if runtime.GOOS == "windows" {
		rootls = "rootls.exe"
	}

	rootls, err := exec.LookPath(rootls)
	withROOTCxx := err == nil

	dir, err := ioutil.TempDir("", "riofs-")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(dir)

	for i, tc := range []struct {
		name string
		skip bool
		want []rtests.ROOTer
	}{
		{name: "", want: nil},
		{
			name: "TObjString",
			want: []rtests.ROOTer{rbase.NewObjString("hello")},
		},
		{
			name: "TObjString",
			want: []rtests.ROOTer{rbase.NewObjString("hello"), rbase.NewObjString("world")},
		},
		{
			name: "TObjString",
			want: func() []rtests.ROOTer {
				var out []rtests.ROOTer
				for _, i := range []int{0, 1, 253, 254, 255, 256, 512, 1024} {
					str := strings.Repeat("=", i)
					out = append(out, rbase.NewObjString(str))
				}
				return out
			}(),
		},
		{
			name: "TObject",
			want: []rtests.ROOTer{rbase.NewObject()},
		},
		{
			name: "TNamed",
			want: []rtests.ROOTer{
				rbase.NewNamed("n0", "t0"),
				rbase.NewNamed("n1", "t1"),
				rbase.NewNamed("n2", "t2"),
			},
		},
		{
			name: "TList",
			want: []rtests.ROOTer{rcont.NewList("list-name", []root.Object{
				rbase.NewNamed("n0", "t0"),
				rbase.NewNamed("n1", "t1"),
				rbase.NewNamed("n2", "t2"),
			})},
		},
		{
			name: "TArrayF",
			want: []rtests.ROOTer{
				&rcont.ArrayF{Data: []float32{1, 2, 3, 4, 5, 6}},
			},
		},
		{
			name: "TArrayD",
			want: []rtests.ROOTer{
				&rcont.ArrayD{Data: []float64{1, 2, 3, 4, 5, 6}},
			},
		},
		{
			name: "TArrays",
			want: []rtests.ROOTer{
				&rcont.ArrayF{Data: []float32{1, 2, 3, 4, 5, 6}},
				&rcont.ArrayD{Data: []float64{1, 2, 3, 4, 5, 6}},
			},
		},
	} {
		fname := filepath.Join(dir, fmt.Sprintf("out-%d.root", i))
		t.Run(tc.name, func(t *testing.T) {
			if tc.skip {
				t.Skip()
			}

			w, err := groot.Create(fname)
			if err != nil {
				t.Fatal(err)
			}

			for i := range tc.want {
				var (
					kname = fmt.Sprintf("key-%s-%02d", tc.name, i)
					want  = tc.want[i]
				)

				err = w.Put(kname, want)
				if err != nil {
					t.Fatal(err)
				}
			}

			if got, want := len(w.Keys()), len(tc.want); got != want {
				t.Fatalf("invalid number of keys. got=%d, want=%d", got, want)
			}

			err = w.Close()
			if err != nil {
				t.Fatalf("error closing file: %v", err)
			}

			r, err := groot.Open(fname)
			if err != nil {
				t.Fatal(err)
			}
			defer r.Close()

			if got, want := len(r.Keys()), len(tc.want); got != want {
				t.Fatalf("invalid number of keys. got=%d, want=%d", got, want)
			}

			for i := range tc.want {
				var (
					kname = fmt.Sprintf("key-%s-%02d", tc.name, i)
					want  = tc.want[i]
				)

				rgot, err := r.Get(kname)
				if err != nil {
					t.Fatal(err)
				}

				if got := rgot.(rtests.ROOTer); !reflect.DeepEqual(got, want) {
					t.Fatalf("error reading back value[%d].\ngot = %#v\nwant = %#v", i, got, want)
				}
			}

			err = r.Close()
			if err != nil {
				t.Fatalf("error closing file: %v", err)
			}

			if !withROOTCxx {
				t.Logf("skip test with ROOT/C++")
				return
			}

			cmd := exec.Command(rootls, "-l", fname)
			err = cmd.Run()
			if err != nil {
				t.Fatalf("ROOT/C++ could not open file %q", fname)
			}
		})
	}
}

func TestOpenBigFile(t *testing.T) {
	ch := make(chan int)
	go func() {
		f, err := riofs.Open("root://eospublic.cern.ch//eos/root-eos/cms_opendata_2012_nanoaod/SMHiggsToZZTo4L.root")
		if err != nil {
			t.Fatal(err)
		}
		defer f.Close()

		o, err := f.Get("Events")
		if err != nil {
			t.Fatal(err)
		}

		tree := o.(rtree.Tree)
		if got, want := tree.Entries(), int64(299973); got != want {
			t.Fatalf("invalid entries: got=%d, want=%d", got, want)
		}
		ch <- 1
	}()

	timeout := time.NewTimer(30 * time.Second)
	defer timeout.Stop()
	select {
	case <-ch:
		// ok
	case <-timeout.C:
		t.Fatalf("timeout")
	}
}

func TestReadOnlyFile(t *testing.T) {
	f, err := groot.Open("../testdata/dirs-6.14.00.root")
	if err != nil {
		t.Fatal(err)
	}
	defer f.Close()

	err = f.Put("o1", rbase.NewObjString("v1"))
	if err == nil {
		t.Fatalf("expected an error. got nil")
	}

	o, err := f.Get("dir1")
	if err != nil {
		t.Fatal(err)
	}

	dir1 := o.(riofs.Directory)
	err = dir1.Put("o2", rbase.NewObjString("v2"))
	if err == nil {
		t.Fatalf("expected an error. got nil")
	}

	o, err = dir1.Get("dir11")
	if err != nil {
		t.Fatal(err)
	}

	dir11 := o.(riofs.Directory)
	err = dir11.Put("o3", rbase.NewObjString("v3"))
	if err == nil {
		t.Fatalf("expected an error. got nil")
	}
}
<|endoftext|>


<commit_before>package lfs

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/github/git-lfs/vendor/_nuts/github.com/rubyist/tracerx"
)

const (
	Version            = "0.5.3"
	tempDirPerms       = 0755
	localMediaDirPerms = 0755
	localLogDirPerms   = 0755
)

var (
	LargeSizeThreshold = 5 * 1024 * 1024
	TempDir            = filepath.Join(os.TempDir(), "git-lfs")
	GitCommit          string
	UserAgent          string
	LocalWorkingDir    string
	LocalGitDir        string // parent of index / config / hooks etc
	LocalGitStorageDir string // parent of objects/lfs (may be same as LocalGitDir but may not)
	LocalMediaDir      string // root of lfs objects
	LocalLogDir        string
	checkedTempDir     string
)

func TempFile(prefix string) (*os.File, error) {
	if checkedTempDir != TempDir {
		if err := os.MkdirAll(TempDir, tempDirPerms); err != nil {
			return nil, err
		}
		checkedTempDir = TempDir
	}

	return ioutil.TempFile(TempDir, prefix)
}

func ResetTempDir() error {
	checkedTempDir = ""
	return os.RemoveAll(TempDir)
}

func localMediaDirNoCreate(sha string) string {
	return filepath.Join(LocalMediaDir, sha[0:2], sha[2:4])
}
func localMediaPathNoCreate(sha string) string {
	return filepath.Join(localMediaDirNoCreate(sha), sha)
}

func LocalMediaPath(sha string) (string, error) {
	path := localMediaDirNoCreate(sha)
	if err := os.MkdirAll(path, localMediaDirPerms); err != nil {
		return "", fmt.Errorf("Error trying to create local media directory in '%s': %s", path, err)
	}

	return filepath.Join(path, sha), nil
}

func ObjectExistsOfSize(sha string, size int64) bool {
	path := localMediaPathNoCreate(sha)
	return FileExistsOfSize(path, size)
}

func Environ() []string {
	osEnviron := os.Environ()
	env := make([]string, 0, len(osEnviron)+7)
	env = append(env,
		fmt.Sprintf("LocalWorkingDir=%s", LocalWorkingDir),
		fmt.Sprintf("LocalGitDir=%s", LocalGitDir),
		fmt.Sprintf("LocalGitStorageDir=%s", LocalGitStorageDir),
		fmt.Sprintf("LocalMediaDir=%s", LocalMediaDir),
		fmt.Sprintf("TempDir=%s", TempDir),
		fmt.Sprintf("ConcurrentTransfers=%d", Config.ConcurrentTransfers()),
		fmt.Sprintf("BatchTransfer=%v", Config.BatchTransfer()),
	)

	for _, e := range osEnviron {
		if !strings.Contains(e, "GIT_") {
			continue
		}
		env = append(env, e)
	}

	return env
}

func InRepo() bool {
	return LocalWorkingDir != ""
}

func init() {
	var err error

	tracerx.DefaultKey = "GIT"
	tracerx.Prefix = "trace git-lfs: "

	LocalWorkingDir, LocalGitDir, err = resolveGitDir()
	if err == nil {
		LocalGitStorageDir = resolveGitStorageDir(LocalGitDir)
		LocalMediaDir = filepath.Join(LocalGitStorageDir, "lfs", "objects")
		LocalLogDir = filepath.Join(LocalMediaDir, "logs")
		TempDir = filepath.Join(LocalGitDir, "lfs", "tmp") // temp files per worktree

		if err := os.MkdirAll(LocalMediaDir, localMediaDirPerms); err != nil {
			panic(fmt.Errorf("Error trying to create objects directory in '%s': %s", LocalMediaDir, err))
		}

		if err := os.MkdirAll(LocalLogDir, localLogDirPerms); err != nil {
			panic(fmt.Errorf("Error trying to create log directory in '%s': %s", LocalLogDir, err))
		}

		if err := os.MkdirAll(TempDir, tempDirPerms); err != nil {
			panic(fmt.Errorf("Error trying to create temp directory in '%s': %s", TempDir, err))
		}

	}

	gitCommit := ""
	if len(GitCommit) > 0 {
		gitCommit = "; git " + GitCommit
	}
	UserAgent = fmt.Sprintf("git-lfs/%s (GitHub; %s %s; go %s%s)",
		Version,
		runtime.GOOS,
		runtime.GOARCH,
		strings.Replace(runtime.Version(), "go", "", 1),
		gitCommit,
	)
}

func resolveGitDir() (string, string, error) {
	gitDir := Config.Getenv("GIT_DIR")
	workTree := Config.Getenv("GIT_WORK_TREE")

	if gitDir != "" {
		return processGitDirVar(gitDir, workTree)
	}

	wd, err := os.Getwd()
	if err != nil {
		return "", "", err
	}

	workTreeR, gitDirR, err := recursiveResolveGitDir(wd)
	if err != nil {
		return "", "", err
	}

	if workTree != "" {
		return processWorkTreeVar(gitDirR, workTree)
	}

	return workTreeR, gitDirR, nil
}

func processGitDirVar(gitDir, workTree string) (string, string, error) {
	if workTree != "" {
		return processWorkTreeVar(gitDir, workTree)
	}

	// See `core.worktree` in `man git-config`:
	// “If --git-dir or GIT_DIR is specified but none of --work-tree, GIT_WORK_TREE and
	// core.worktree is specified, the current working directory is regarded as the top
	// level of your working tree.”

	wd, err := os.Getwd()
	if err != nil {
		return "", "", err
	}

	return wd, gitDir, nil
}

func processWorkTreeVar(gitDir, workTree string) (string, string, error) {
	// See `core.worktree` in `man git-config`:
	// “The value [of core.worktree, GIT_WORK_TREE, or --work-tree] can be an absolute path
	// or relative to the path to the .git directory, which is either specified
	// by --git-dir or GIT_DIR, or automatically discovered.”

	if filepath.IsAbs(workTree) {
		return workTree, gitDir, nil
	}

	base := filepath.Dir(filepath.Clean(gitDir))
	absWorkTree := filepath.Join(base, workTree)
	return absWorkTree, gitDir, nil
}

func recursiveResolveGitDir(dir string) (string, string, error) {
	var cleanDir = filepath.Clean(dir)
	if cleanDir[len(cleanDir)-1] == os.PathSeparator {
		return "", "", fmt.Errorf("Git repository not found")
	}

	if filepath.Base(dir) == gitExt {
		// We're in the `.git` directory.  Make no assumptions about the working directory.
		return "", dir, nil
	}

	gitDir := filepath.Join(dir, gitExt)
	info, err := os.Stat(gitDir)
	if err != nil {
		// Found neither a directory nor a file named `.git`.
		// Move one directory up.
		return recursiveResolveGitDir(filepath.Dir(dir))
	}

	if !info.IsDir() {
		// Found a file named `.git` (we're in a submodule).
		return resolveDotGitFile(gitDir)
	}

	// Found the `.git` directory.
	return dir, gitDir, nil
}

func resolveDotGitFile(file string) (string, string, error) {
	// The local working directory is the directory the `.git` file is located in.
	wd := filepath.Dir(file)

	// The `.git` file tells us where the submodules `.git` directory is.
	gitDir, err := processDotGitFile(file)
	if err != nil {
		return "", "", err
	}

	return wd, gitDir, nil
}

func processDotGitFile(file string) (string, error) {
	return processGitRedirectFile(file, gitPtrPrefix)
}

func processGitRedirectFile(file, prefix string) (string, error) {
	data, err := ioutil.ReadFile(file)
	if err != nil {
		return "", err
	}

	contents := string(data)
	var dir string
	if len(prefix) > 0 {
		if !strings.HasPrefix(contents, prefix) {
			// Prefix required & not found
			return "", nil
		}
		dir = strings.TrimSpace(contents[len(prefix):])
	} else {
		dir = strings.TrimSpace(contents)
	}

	if !filepath.IsAbs(dir) {
		// The .git file contains a relative path.
		// Create an absolute path based on the directory the .git file is located in.
		dir = filepath.Join(filepath.Dir(file), dir)
	}

	return dir, nil
}

// From a git dir, get the location that objects are to be stored (we will store lfs alongside)
// Sometimes there is an additional level of redirect on the .git folder by way of a commondir file
// before you find object storage, e.g. 'git worktree' uses this. It redirects to gitdir either by GIT_DIR
// (during setup) or .git/git-dir: (during use), but this only contains the index etc, the objects
// are found in another git dir via 'commondir'.
func resolveGitStorageDir(gitDir string) string {
	commondirpath := filepath.Join(gitDir, "commondir")
	if FileExists(commondirpath) && !DirExists(filepath.Join(gitDir, "objects")) {
		// no git-dir: prefix in commondir
		storage, err := processGitRedirectFile(commondirpath, "")
		if err == nil {
			return storage
		}
	}
	return gitDir
}

const (
	gitExt       = ".git"
	gitPtrPrefix = "gitdir: "
)
<commit_msg>ンンー ンンンン ンーンン<commit_after>package lfs

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/github/git-lfs/vendor/_nuts/github.com/rubyist/tracerx"
)

const (
	Version            = "0.6.0-pre"
	tempDirPerms       = 0755
	localMediaDirPerms = 0755
	localLogDirPerms   = 0755
)

var (
	LargeSizeThreshold = 5 * 1024 * 1024
	TempDir            = filepath.Join(os.TempDir(), "git-lfs")
	GitCommit          string
	UserAgent          string
	LocalWorkingDir    string
	LocalGitDir        string // parent of index / config / hooks etc
	LocalGitStorageDir string // parent of objects/lfs (may be same as LocalGitDir but may not)
	LocalMediaDir      string // root of lfs objects
	LocalLogDir        string
	checkedTempDir     string
)

func TempFile(prefix string) (*os.File, error) {
	if checkedTempDir != TempDir {
		if err := os.MkdirAll(TempDir, tempDirPerms); err != nil {
			return nil, err
		}
		checkedTempDir = TempDir
	}

	return ioutil.TempFile(TempDir, prefix)
}

func ResetTempDir() error {
	checkedTempDir = ""
	return os.RemoveAll(TempDir)
}

func localMediaDirNoCreate(sha string) string {
	return filepath.Join(LocalMediaDir, sha[0:2], sha[2:4])
}
func localMediaPathNoCreate(sha string) string {
	return filepath.Join(localMediaDirNoCreate(sha), sha)
}

func LocalMediaPath(sha string) (string, error) {
	path := localMediaDirNoCreate(sha)
	if err := os.MkdirAll(path, localMediaDirPerms); err != nil {
		return "", fmt.Errorf("Error trying to create local media directory in '%s': %s", path, err)
	}

	return filepath.Join(path, sha), nil
}

func ObjectExistsOfSize(sha string, size int64) bool {
	path := localMediaPathNoCreate(sha)
	return FileExistsOfSize(path, size)
}

func Environ() []string {
	osEnviron := os.Environ()
	env := make([]string, 0, len(osEnviron)+7)
	env = append(env,
		fmt.Sprintf("LocalWorkingDir=%s", LocalWorkingDir),
		fmt.Sprintf("LocalGitDir=%s", LocalGitDir),
		fmt.Sprintf("LocalGitStorageDir=%s", LocalGitStorageDir),
		fmt.Sprintf("LocalMediaDir=%s", LocalMediaDir),
		fmt.Sprintf("TempDir=%s", TempDir),
		fmt.Sprintf("ConcurrentTransfers=%d", Config.ConcurrentTransfers()),
		fmt.Sprintf("BatchTransfer=%v", Config.BatchTransfer()),
	)

	for _, e := range osEnviron {
		if !strings.Contains(e, "GIT_") {
			continue
		}
		env = append(env, e)
	}

	return env
}

func InRepo() bool {
	return LocalWorkingDir != ""
}

func init() {
	var err error

	tracerx.DefaultKey = "GIT"
	tracerx.Prefix = "trace git-lfs: "

	LocalWorkingDir, LocalGitDir, err = resolveGitDir()
	if err == nil {
		LocalGitStorageDir = resolveGitStorageDir(LocalGitDir)
		LocalMediaDir = filepath.Join(LocalGitStorageDir, "lfs", "objects")
		LocalLogDir = filepath.Join(LocalMediaDir, "logs")
		TempDir = filepath.Join(LocalGitDir, "lfs", "tmp") // temp files per worktree

		if err := os.MkdirAll(LocalMediaDir, localMediaDirPerms); err != nil {
			panic(fmt.Errorf("Error trying to create objects directory in '%s': %s", LocalMediaDir, err))
		}

		if err := os.MkdirAll(LocalLogDir, localLogDirPerms); err != nil {
			panic(fmt.Errorf("Error trying to create log directory in '%s': %s", LocalLogDir, err))
		}

		if err := os.MkdirAll(TempDir, tempDirPerms); err != nil {
			panic(fmt.Errorf("Error trying to create temp directory in '%s': %s", TempDir, err))
		}

	}

	gitCommit := ""
	if len(GitCommit) > 0 {
		gitCommit = "; git " + GitCommit
	}
	UserAgent = fmt.Sprintf("git-lfs/%s (GitHub; %s %s; go %s%s)",
		Version,
		runtime.GOOS,
		runtime.GOARCH,
		strings.Replace(runtime.Version(), "go", "", 1),
		gitCommit,
	)
}

func resolveGitDir() (string, string, error) {
	gitDir := Config.Getenv("GIT_DIR")
	workTree := Config.Getenv("GIT_WORK_TREE")

	if gitDir != "" {
		return processGitDirVar(gitDir, workTree)
	}

	wd, err := os.Getwd()
	if err != nil {
		return "", "", err
	}

	workTreeR, gitDirR, err := recursiveResolveGitDir(wd)
	if err != nil {
		return "", "", err
	}

	if workTree != "" {
		return processWorkTreeVar(gitDirR, workTree)
	}

	return workTreeR, gitDirR, nil
}

func processGitDirVar(gitDir, workTree string) (string, string, error) {
	if workTree != "" {
		return processWorkTreeVar(gitDir, workTree)
	}

	// See `core.worktree` in `man git-config`:
	// “If --git-dir or GIT_DIR is specified but none of --work-tree, GIT_WORK_TREE and
	// core.worktree is specified, the current working directory is regarded as the top
	// level of your working tree.”

	wd, err := os.Getwd()
	if err != nil {
		return "", "", err
	}

	return wd, gitDir, nil
}

func processWorkTreeVar(gitDir, workTree string) (string, string, error) {
	// See `core.worktree` in `man git-config`:
	// “The value [of core.worktree, GIT_WORK_TREE, or --work-tree] can be an absolute path
	// or relative to the path to the .git directory, which is either specified
	// by --git-dir or GIT_DIR, or automatically discovered.”

	if filepath.IsAbs(workTree) {
		return workTree, gitDir, nil
	}

	base := filepath.Dir(filepath.Clean(gitDir))
	absWorkTree := filepath.Join(base, workTree)
	return absWorkTree, gitDir, nil
}

func recursiveResolveGitDir(dir string) (string, string, error) {
	var cleanDir = filepath.Clean(dir)
	if cleanDir[len(cleanDir)-1] == os.PathSeparator {
		return "", "", fmt.Errorf("Git repository not found")
	}

	if filepath.Base(dir) == gitExt {
		// We're in the `.git` directory.  Make no assumptions about the working directory.
		return "", dir, nil
	}

	gitDir := filepath.Join(dir, gitExt)
	info, err := os.Stat(gitDir)
	if err != nil {
		// Found neither a directory nor a file named `.git`.
		// Move one directory up.
		return recursiveResolveGitDir(filepath.Dir(dir))
	}

	if !info.IsDir() {
		// Found a file named `.git` (we're in a submodule).
		return resolveDotGitFile(gitDir)
	}

	// Found the `.git` directory.
	return dir, gitDir, nil
}

func resolveDotGitFile(file string) (string, string, error) {
	// The local working directory is the directory the `.git` file is located in.
	wd := filepath.Dir(file)

	// The `.git` file tells us where the submodules `.git` directory is.
	gitDir, err := processDotGitFile(file)
	if err != nil {
		return "", "", err
	}

	return wd, gitDir, nil
}

func processDotGitFile(file string) (string, error) {
	return processGitRedirectFile(file, gitPtrPrefix)
}

func processGitRedirectFile(file, prefix string) (string, error) {
	data, err := ioutil.ReadFile(file)
	if err != nil {
		return "", err
	}

	contents := string(data)
	var dir string
	if len(prefix) > 0 {
		if !strings.HasPrefix(contents, prefix) {
			// Prefix required & not found
			return "", nil
		}
		dir = strings.TrimSpace(contents[len(prefix):])
	} else {
		dir = strings.TrimSpace(contents)
	}

	if !filepath.IsAbs(dir) {
		// The .git file contains a relative path.
		// Create an absolute path based on the directory the .git file is located in.
		dir = filepath.Join(filepath.Dir(file), dir)
	}

	return dir, nil
}

// From a git dir, get the location that objects are to be stored (we will store lfs alongside)
// Sometimes there is an additional level of redirect on the .git folder by way of a commondir file
// before you find object storage, e.g. 'git worktree' uses this. It redirects to gitdir either by GIT_DIR
// (during setup) or .git/git-dir: (during use), but this only contains the index etc, the objects
// are found in another git dir via 'commondir'.
func resolveGitStorageDir(gitDir string) string {
	commondirpath := filepath.Join(gitDir, "commondir")
	if FileExists(commondirpath) && !DirExists(filepath.Join(gitDir, "objects")) {
		// no git-dir: prefix in commondir
		storage, err := processGitRedirectFile(commondirpath, "")
		if err == nil {
			return storage
		}
	}
	return gitDir
}

const (
	gitExt       = ".git"
	gitPtrPrefix = "gitdir: "
)
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"os/signal"

	"github.com/Sirupsen/logrus"
	_ "github.com/eliothedeman/bangarang/alarm/console"
	_ "github.com/eliothedeman/bangarang/alarm/email"
	_ "github.com/eliothedeman/bangarang/alarm/grafana-graphite-annotation"
	_ "github.com/eliothedeman/bangarang/alarm/pd"
	"github.com/eliothedeman/bangarang/api"
	"github.com/eliothedeman/bangarang/config"
	"github.com/eliothedeman/bangarang/pipeline"
	_ "github.com/eliothedeman/bangarang/provider/http"
	_ "github.com/eliothedeman/bangarang/provider/tcp"
)

var (
	confFile = flag.String("conf", "/etc/bangarang/conf.json", "path main config file")
	dev      = flag.Bool("dev", false, "puts bangarang in a dev testing mode")
	version  = flag.Bool("version", false, "display the version of this binary")
	confType = flag.String("conf-type", "db", `type of configuration used ["db", "json"]`)
	apiPort  = flag.Int("api-port", 8081, "port to serve the http api on")
)

const (
	versionNumber = "0.9.2"
)

func init() {
	logrus.SetLevel(logrus.InfoLevel)
	tf := &logrus.TextFormatter{}
	tf.FullTimestamp = true
	tf.ForceColors = true
	logrus.SetFormatter(tf)
}

func handleSigs() {
	stop := make(chan os.Signal)
	signal.Notify(stop, os.Kill, os.Interrupt)

	done := <-stop
	logrus.Fatal(done.String())
}

func main() {
	flag.Parse()

	// display the current version and exit
	if *version {
		fmt.Print(versionNumber)
		os.Exit(0)
	}

	// load configuration
	cp := config.GetProvider(*confType, *confFile)
	if cp == nil {
		logrus.Fatalf("Unable to load config of type %s at location %s", *confType, *confFile)
	}
	ac, err := cp.GetCurrent()
	if err != nil {
		logrus.Fatal(err)
	}

	if ac.LogLevel == "" {
		ac.LogLevel = "info"
	}

	ll, err := logrus.ParseLevel(ac.LogLevel)
	if err != nil {
		logrus.Error(err)
	} else {
		logrus.SetLevel(ll)
	}

	logrus.Infof("Starting processing pipeline with %d policie(s)", len(ac.Policies))
	// create and start up a new pipeline
	p := pipeline.NewPipeline(ac)
	p.Start()

	logrus.Infof("Serving the http api on port %d", 8081)
	// create and start a new api server
	apiServer := api.NewServer(*apiPort, p)
	go apiServer.Serve()

	handleSigs()
}
<commit_msg>v0.9.3<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"os/signal"

	"github.com/Sirupsen/logrus"
	_ "github.com/eliothedeman/bangarang/alarm/console"
	_ "github.com/eliothedeman/bangarang/alarm/email"
	_ "github.com/eliothedeman/bangarang/alarm/grafana-graphite-annotation"
	_ "github.com/eliothedeman/bangarang/alarm/pd"
	"github.com/eliothedeman/bangarang/api"
	"github.com/eliothedeman/bangarang/config"
	"github.com/eliothedeman/bangarang/pipeline"
	_ "github.com/eliothedeman/bangarang/provider/http"
	_ "github.com/eliothedeman/bangarang/provider/tcp"
)

var (
	confFile = flag.String("conf", "/etc/bangarang/conf.json", "path main config file")
	dev      = flag.Bool("dev", false, "puts bangarang in a dev testing mode")
	version  = flag.Bool("version", false, "display the version of this binary")
	confType = flag.String("conf-type", "db", `type of configuration used ["db", "json"]`)
	apiPort  = flag.Int("api-port", 8081, "port to serve the http api on")
)

const (
	versionNumber = "0.9.3"
)

func init() {
	logrus.SetLevel(logrus.InfoLevel)
	tf := &logrus.TextFormatter{}
	tf.FullTimestamp = true
	tf.ForceColors = true
	logrus.SetFormatter(tf)
}

func handleSigs() {
	stop := make(chan os.Signal)
	signal.Notify(stop, os.Kill, os.Interrupt)

	done := <-stop
	logrus.Fatal(done.String())
}

func main() {
	flag.Parse()

	// display the current version and exit
	if *version {
		fmt.Print(versionNumber)
		os.Exit(0)
	}

	// load configuration
	cp := config.GetProvider(*confType, *confFile)
	if cp == nil {
		logrus.Fatalf("Unable to load config of type %s at location %s", *confType, *confFile)
	}
	ac, err := cp.GetCurrent()
	if err != nil {
		logrus.Fatal(err)
	}

	if ac.LogLevel == "" {
		ac.LogLevel = "info"
	}

	ll, err := logrus.ParseLevel(ac.LogLevel)
	if err != nil {
		logrus.Error(err)
	} else {
		logrus.SetLevel(ll)
	}

	logrus.Infof("Starting processing pipeline with %d policie(s)", len(ac.Policies))
	// create and start up a new pipeline
	p := pipeline.NewPipeline(ac)
	p.Start()

	logrus.Infof("Serving the http api on port %d", 8081)
	// create and start a new api server
	apiServer := api.NewServer(*apiPort, p)
	go apiServer.Serve()

	handleSigs()
}
<|endoftext|>


<commit_before>package builtin

import (
  "github.com/twitchyliquid64/CNC/plugin/exec"
  "github.com/twitchyliquid64/CNC/logging"
  "github.com/robertkrimen/otto"
  "github.com/twitchyliquid64/CNC/config"
  "github.com/twitchyliquid64/twiliocore"
  "errors"
)

func getTwilioObj()*twiliocore.TwilioClient {
  if config.All().Twilio.Enable {
    return twiliocore.NewClient(config.All().Twilio.AccountSID, config.All().Twilio.AuthToken)
  }
  return nil
}

func sendMsg(fromNum, toNum, msg string)(*twiliocore.Message,error){
  obj := getTwilioObj()
  if obj == nil{
    return nil, errors.New("Configuration error")
  }
  return obj.NewMessage(fromNum, toNum, msg, "")
}

func function_twilio_sendSMS(plugin *exec.Plugin, call otto.FunctionCall)otto.Value{
  from := call.Argument(0).String()
  to   := call.Argument(1).String()
  msg  := call.Argument(2).String()

  resp, err := sendMsg(from, to, msg)
  if err != nil{
    logging.Error("builtin-twilio", err.Error())
    out, err := plugin.VM.ToValue(resp)
    if err != nil {
      logging.Error("builtin-twilio", err.Error())
      out, _ = plugin.VM.ToValue(err.Error())
    }
    return out
  } else {
    out, _ := plugin.VM.ToValue(err.Error())
    return out
  }
}
<commit_msg>Fixed plugin crash when sending messages<commit_after>package builtin

import (
  "github.com/twitchyliquid64/CNC/plugin/exec"
  "github.com/twitchyliquid64/CNC/logging"
  "github.com/robertkrimen/otto"
  "github.com/twitchyliquid64/CNC/config"
  "github.com/twitchyliquid64/twiliocore"
  "errors"
)

func getTwilioObj()*twiliocore.TwilioClient {
  if config.All().Twilio.Enable {
    return twiliocore.NewClient(config.All().Twilio.AccountSID, config.All().Twilio.AuthToken)
  }
  return nil
}

func sendMsg(fromNum, toNum, msg string)(*twiliocore.Message,error){
  obj := getTwilioObj()
  if obj == nil{
    return nil, errors.New("Configuration error")
  }
  return obj.NewMessage(fromNum, toNum, msg, "")
}

func function_twilio_sendSMS(plugin *exec.Plugin, call otto.FunctionCall)otto.Value{
  from := call.Argument(0).String()
  to   := call.Argument(1).String()
  msg  := call.Argument(2).String()

  _, err := sendMsg(from, to, msg)
  if err != nil{
    logging.Error("builtin-twilio", err.Error())
  }
  return otto.Value{}
}
<|endoftext|>