<commit_before>package main

import (
	"bufio"
	"errors"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"sync"

	"github.com/mattn/go-isatty"
	"github.com/motemen/ghq/cmdutil"
	"github.com/motemen/ghq/logger"
	"github.com/urfave/cli/v2"
	"golang.org/x/sync/errgroup"
)

func doGet(c *cli.Context) error {
	var (
		args     = c.Args().Slice()
		andLook  = c.Bool("look")
		parallel = c.Bool("parallel")
	)
	g := &getter{
		update:    c.Bool("update"),
		shallow:   c.Bool("shallow"),
		ssh:       c.Bool("p"),
		vcs:       c.String("vcs"),
		silent:    c.Bool("silent"),
		branch:    c.String("branch"),
		recursive: !c.Bool("no-recursive"),
	}
	if parallel {
		// force silent in parallel import
		g.silent = true
	}

	var (
		firstArg string
		scr      scanner
	)
	if len(args) > 0 {
		scr = &sliceScanner{slice: args}
	} else {
		fd := os.Stdin.Fd()
		if isatty.IsTerminal(fd) || isatty.IsCygwinTerminal(fd) {
			return fmt.Errorf("no target args specified. see `ghq get -h` for more details")
		}
		scr = bufio.NewScanner(os.Stdin)
	}
	eg := &errgroup.Group{}
	sem := make(chan struct{}, 6)
	for scr.Scan() {
		target := scr.Text()
		if firstArg != "" {
			firstArg = target
		}
		if parallel {
			eg.Go(func() error {
				sem <- struct{}{}
				defer func() { <-sem }()
				if err := g.get(target); err != nil {
					logger.Log("error", err.Error())
				}
				return nil
			})
		} else {
			if err := g.get(target); err != nil {
				return err
			}
		}
	}
	if err := scr.Err(); err != nil {
		return fmt.Errorf("While reading input: %s", err)
	}
	if err := eg.Wait(); err != nil {
		return err
	}
	if andLook && firstArg != "" {
		return look(firstArg)
	}
	return nil
}

type sliceScanner struct {
	slice []string
	index int
}

func (s *sliceScanner) Scan() bool {
	s.index++
	return s.index <= len(s.slice)
}

func (s *sliceScanner) Text() string {
	return s.slice[s.index-1]
}

func (s *sliceScanner) Err() error {
	return nil
}

type scanner interface {
	Scan() bool
	Text() string
	Err() error
}

func detectShell() string {
	shell := os.Getenv("SHELL")
	if shell != "" {
		return shell
	}
	if runtime.GOOS == "windows" {
		return os.Getenv("COMSPEC")
	}
	return "/bin/sh"
}

func look(name string) error {
	var (
		reposFound []*LocalRepository
		mu         sync.Mutex
	)
	if err := walkAllLocalRepositories(func(repo *LocalRepository) {
		if repo.Matches(name) {
			mu.Lock()
			reposFound = append(reposFound, repo)
			mu.Unlock()
		}
	}); err != nil {
		return err
	}

	if len(reposFound) == 0 {
		if url, err := newURL(name, false, false); err == nil {
			repo, err := LocalRepositoryFromURL(url)
			if err != nil {
				return err
			}
			_, err = os.Stat(repo.FullPath)

			// if the directory exists
			if err == nil {
				reposFound = append(reposFound, repo)
			}
		}
	}

	switch len(reposFound) {
	case 0:
		return fmt.Errorf("No repository found")
	case 1:
		repo := reposFound[0]
		cmd := exec.Command(detectShell())
		cmd.Stdin = os.Stdin
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		cmd.Dir = repo.FullPath
		cmd.Env = append(os.Environ(), "GHQ_LOOK="+filepath.ToSlash(repo.RelPath))
		return cmdutil.RunCommand(cmd, true)
	default:
		b := &strings.Builder{}
		b.WriteString("More than one repositories are found; Try more precise name\n")
		for _, repo := range reposFound {
			b.WriteString(fmt.Sprintf("       - %s\n", strings.Join(repo.PathParts, "/")))
		}
		return errors.New(b.String())
	}
}
<commit_msg>fix semaphore<commit_after>package main

import (
	"bufio"
	"errors"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"sync"

	"github.com/mattn/go-isatty"
	"github.com/motemen/ghq/cmdutil"
	"github.com/motemen/ghq/logger"
	"github.com/urfave/cli/v2"
	"golang.org/x/sync/errgroup"
)

func doGet(c *cli.Context) error {
	var (
		args     = c.Args().Slice()
		andLook  = c.Bool("look")
		parallel = c.Bool("parallel")
	)
	g := &getter{
		update:    c.Bool("update"),
		shallow:   c.Bool("shallow"),
		ssh:       c.Bool("p"),
		vcs:       c.String("vcs"),
		silent:    c.Bool("silent"),
		branch:    c.String("branch"),
		recursive: !c.Bool("no-recursive"),
	}
	if parallel {
		// force silent in parallel import
		g.silent = true
	}

	var (
		firstArg string
		scr      scanner
	)
	if len(args) > 0 {
		scr = &sliceScanner{slice: args}
	} else {
		fd := os.Stdin.Fd()
		if isatty.IsTerminal(fd) || isatty.IsCygwinTerminal(fd) {
			return fmt.Errorf("no target args specified. see `ghq get -h` for more details")
		}
		scr = bufio.NewScanner(os.Stdin)
	}
	eg := &errgroup.Group{}
	sem := make(chan struct{}, 6)
	for scr.Scan() {
		target := scr.Text()
		if firstArg != "" {
			firstArg = target
		}
		if parallel {
			sem <- struct{}{}
			eg.Go(func() error {
				defer func() { <-sem }()
				if err := g.get(target); err != nil {
					logger.Log("error", err.Error())
				}
				return nil
			})
		} else {
			if err := g.get(target); err != nil {
				return err
			}
		}
	}
	if err := scr.Err(); err != nil {
		return fmt.Errorf("While reading input: %s", err)
	}
	if err := eg.Wait(); err != nil {
		return err
	}
	if andLook && firstArg != "" {
		return look(firstArg)
	}
	return nil
}

type sliceScanner struct {
	slice []string
	index int
}

func (s *sliceScanner) Scan() bool {
	s.index++
	return s.index <= len(s.slice)
}

func (s *sliceScanner) Text() string {
	return s.slice[s.index-1]
}

func (s *sliceScanner) Err() error {
	return nil
}

type scanner interface {
	Scan() bool
	Text() string
	Err() error
}

func detectShell() string {
	shell := os.Getenv("SHELL")
	if shell != "" {
		return shell
	}
	if runtime.GOOS == "windows" {
		return os.Getenv("COMSPEC")
	}
	return "/bin/sh"
}

func look(name string) error {
	var (
		reposFound []*LocalRepository
		mu         sync.Mutex
	)
	if err := walkAllLocalRepositories(func(repo *LocalRepository) {
		if repo.Matches(name) {
			mu.Lock()
			reposFound = append(reposFound, repo)
			mu.Unlock()
		}
	}); err != nil {
		return err
	}

	if len(reposFound) == 0 {
		if url, err := newURL(name, false, false); err == nil {
			repo, err := LocalRepositoryFromURL(url)
			if err != nil {
				return err
			}
			_, err = os.Stat(repo.FullPath)

			// if the directory exists
			if err == nil {
				reposFound = append(reposFound, repo)
			}
		}
	}

	switch len(reposFound) {
	case 0:
		return fmt.Errorf("No repository found")
	case 1:
		repo := reposFound[0]
		cmd := exec.Command(detectShell())
		cmd.Stdin = os.Stdin
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		cmd.Dir = repo.FullPath
		cmd.Env = append(os.Environ(), "GHQ_LOOK="+filepath.ToSlash(repo.RelPath))
		return cmdutil.RunCommand(cmd, true)
	default:
		b := &strings.Builder{}
		b.WriteString("More than one repositories are found; Try more precise name\n")
		for _, repo := range reposFound {
			b.WriteString(fmt.Sprintf("       - %s\n", strings.Join(repo.PathParts, "/")))
		}
		return errors.New(b.String())
	}
}
<|endoftext|>


<commit_before>package combine

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"

	"github.com/kwmt/go-utils"
)

// combine the files under dirName directory
func Combine(dirName string, fileName string) {
	files, err := ioutil.ReadDir(dirName)
	if err != nil {
		log.Fatal(err)
	}
	total := len(files)

	for i, file := range files {
		file, err := os.Open(fmt.Sprintf("%s/%s", dirName, file.Name()))
		if err != nil {
			log.Fatal(err)
		}
		b, err := ioutil.ReadAll(file)
		appendToFile(fileName, string(b))

		fmt.Printf("\r%3.0f%%", utils.Percent(i+1, total))
		file.Close()
	}
	fmt.Println()
}

// append text to filName file
func appendToFile(fileName string, text string) {
	f, err := os.OpenFile(fileName, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600)
	if err != nil {
		panic(err)
	}
	defer f.Close()
	if _, err = f.WriteString(text); err != nil {
		panic(err)
	}
}
<commit_msg>return error<commit_after>package combine

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"

	"github.com/kwmt/go-utils"
)

// combine the files under dirName directory
func Combine(dirName string, fileName string) error {
	files, err := ioutil.ReadDir(dirName)
	if err != nil {
		return err
	}
	total := len(files)

	for i, file := range files {
		file, err := os.Open(fmt.Sprintf("%s/%s", dirName, file.Name()))
		if err != nil {
			return err
		}
		b, err := ioutil.ReadAll(file)
		if err != nil {
			return err
		}
		err = appendToFile(fileName, string(b))
		if err != nil {
			return err
		}

		fmt.Printf("\r%3.0f%%", utils.Percent(i+1, total))
		file.Close()
	}
	fmt.Println()
	return nil
}

// append text to filName file
func appendToFile(fileName string, text string) error {
	f, err := os.OpenFile(fileName, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600)
	if err != nil {
		return err
	}
	defer f.Close()
	if _, err = f.WriteString(text); err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package ebs

import (
	"fmt"

	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/mitchellh/multistep"
	"github.com/mitchellh/packer/builder/amazon/common"
	"github.com/mitchellh/packer/packer"
)

type stepTagEBSVolumes struct {
	VolumeRunTags map[string]string
}

func (s *stepTagEBSVolumes) Run(state multistep.StateBag) multistep.StepAction {
	ec2conn := state.Get("ec2").(*ec2.EC2)
	instance := state.Get("instance").(*ec2.Instance)
	ui := state.Get("ui").(packer.Ui)

	if len(s.VolumeRunTags) == 0 {
		return multistep.ActionContinue
	}

	volumeIds := make([]*string, 0)
	for _, v := range instance.BlockDeviceMappings {
		if ebs := v.Ebs; ebs != nil {
			volumeIds = append(volumeIds, ebs.VolumeId)
		}
	}

	if len(volumeIds) == 0 {
		return multistep.ActionContinue
	}

	ui.Say(fmt.Sprintf("Adding tags to source EBS Volumes:"))
	tags := common.ConvertToEC2Tags(s.VolumeRunTags)

	_, err := ec2conn.CreateTags(&ec2.CreateTagsInput{
		Resources: []*string{
			instance.BlockDeviceMappings[0].Ebs.VolumeId,
		},
		Tags: tags,
	})
	if err != nil {
		err := fmt.Errorf("Error tagging source EBS Volumes on %s: %s", *instance.InstanceId, err)
		state.Put("error", err)
		ui.Error(err.Error())
		return multistep.ActionHalt
	}

	return multistep.ActionContinue
}

func (s *stepTagEBSVolumes) Cleanup(state multistep.StateBag) {
	// No cleanup...
}
<commit_msg>tag all volumes<commit_after>package ebs

import (
	"fmt"

	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/mitchellh/multistep"
	"github.com/mitchellh/packer/builder/amazon/common"
	"github.com/mitchellh/packer/packer"
)

type stepTagEBSVolumes struct {
	VolumeRunTags map[string]string
}

func (s *stepTagEBSVolumes) Run(state multistep.StateBag) multistep.StepAction {
	ec2conn := state.Get("ec2").(*ec2.EC2)
	instance := state.Get("instance").(*ec2.Instance)
	ui := state.Get("ui").(packer.Ui)

	if len(s.VolumeRunTags) == 0 {
		return multistep.ActionContinue
	}

	volumeIds := make([]*string, 0)
	for _, v := range instance.BlockDeviceMappings {
		if ebs := v.Ebs; ebs != nil {
			volumeIds = append(volumeIds, ebs.VolumeId)
		}
	}

	if len(volumeIds) == 0 {
		return multistep.ActionContinue
	}

	ui.Say(fmt.Sprintf("Adding tags to source EBS Volumes:"))
	tags := common.ConvertToEC2Tags(s.VolumeRunTags)

	_, err := ec2conn.CreateTags(&ec2.CreateTagsInput{
		Resources: volumeIds,
		Tags:      tags,
	})
	if err != nil {
		err := fmt.Errorf("Error tagging source EBS Volumes on %s: %s", *instance.InstanceId, err)
		state.Put("error", err)
		ui.Error(err.Error())
		return multistep.ActionHalt
	}

	return multistep.ActionContinue
}

func (s *stepTagEBSVolumes) Cleanup(state multistep.StateBag) {
	// No cleanup...
}
<|endoftext|>


<commit_before>package minify // import "github.com/tdewolff/minify"

import (
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"math/rand"
	"strconv"
	"testing"

	"github.com/tdewolff/test"
)

func TestContentType(t *testing.T) {
	contentTypeTests := []struct {
		contentType string
		expected    string
	}{
		{"text/html", "text/html"},
		{"text/html; charset=UTF-8", "text/html;charset=utf-8"},
		{"text/html; charset=UTF-8 ; param = \" ; \"", "text/html;charset=utf-8;param=\" ; \""},
		{"text/html, text/css", "text/html,text/css"},
	}
	for _, tt := range contentTypeTests {
		test.Minify(t, tt.contentType, nil, string(ContentType([]byte(tt.contentType))), tt.expected)
	}
}

func TestDataURI(t *testing.T) {
	dataURITests := []struct {
		dataURI  string
		expected string
	}{
		{"data:,text", "data:,text"},
		{"data:text/plain;charset=us-ascii,text", "data:,text"},
		{"data:TEXT/PLAIN;CHARSET=US-ASCII,text", "data:,text"},
		{"data:text/plain;charset=us-asciiz,text", "data:;charset=us-asciiz,text"},
		{"data:;base64,dGV4dA==", "data:,text"},
		{"data:text/svg+xml;base64,PT09PT09", "data:text/svg+xml;base64,PT09PT09"},
		{"data:text/xml;version=2.0,content", "data:text/xml;version=2.0,content"},
		{"data:text/xml; version = 2.0,content", "data:text/xml;version=2.0,content"},
		{"data:,=====", "data:,%3D%3D%3D%3D%3D"},
		{"data:,======", "data:;base64,PT09PT09"},
		{"data:text/x,<?x?>", "data:text/x,%3C%3Fx%3F%3E"},
	}
	m := New()
	m.AddFunc("text/x", func(_ *M, w io.Writer, r io.Reader, _ map[string]string) error {
		b, _ := ioutil.ReadAll(r)
		test.String(t, string(b), "<?x?>")
		w.Write(b)
		return nil
	})
	for _, tt := range dataURITests {
		test.Minify(t, tt.dataURI, nil, string(DataURI(m, []byte(tt.dataURI))), tt.expected)
	}
}

func TestNumber(t *testing.T) {
	numberTests := []struct {
		number   string
		expected string
	}{
		{"0", "0"},
		{".0", "0"},
		{"1.0", "1"},
		{"0.1", ".1"},
		{"+1", "1"},
		{"-1", "-1"},
		{"-0.1", "-.1"},
		{"10", "10"},
		{"100", "100"},
		{"1000", "1e3"},
		{"0.001", ".001"},
		{"0.0001", "1e-4"},
		{"100e1", "1e3"},
		{"1.1e+1", "11"},
		{"1.1e6", "11e5"},
		{"0.252", ".252"},
		{"1.252", "1.252"},
		{"-1.252", "-1.252"},
		{"0.075", ".075"},
		{"789012345678901234567890123456789e9234567890123456789", "789012345678901234567890123456789e9234567890123456789"},
		{".000100009", "100009e-9"},
		{".0001000009", ".0001000009"},
		{".0001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000009", ".0001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000009"},
		{"E\x1f", "E\x1f"}, // fuzz
		{"1e9223372036854775807", "1e9223372036854775807"},
		{"11e9223372036854775807", "11e9223372036854775807"},
		{".01e-9223372036854775808", ".01e-9223372036854775808"},
		{".011e-9223372036854775808", ".011e-9223372036854775808"},

		{".12345e8", "12345e3"},
		{".12345e7", "1234500"},
		{".12345e6", "123450"},
		{".12345e5", "12345"},
		{".012345e6", "12345"},
		{".12345e4", "1234.5"},
		{"-.12345e4", "-1234.5"},
		{".12345e0", ".12345"},
		{".12345e-1", ".012345"},
		{".12345e-2", ".0012345"},
		{".12345e-3", "12345e-8"},
		{".12345e-4", "12345e-9"},
		{".12345e-5", "12345e-10"},

		{".123456e-3", "123456e-9"},
		{".123456e-2", ".00123456"},
		{".1234567e-4", "1234567e-11"},
		{".1234567e-3", ".0001234567"},

		{"12345678e-1", "1234567.8"},
		{"72.e-3", ".072"},
		{"7640e-2", "76.4"},
		{"10.e-3", ".01"},
		{".0319e3", "31.9"},
		{"39.7e-2", ".397"},
		{"39.7e-3", ".0397"},
		{".01e1", ".1"},
		{".001e1", ".01"},
		{"39.7e-5", "397e-6"},
	}
	for _, tt := range numberTests {
		test.Minify(t, tt.number, nil, string(Number([]byte(tt.number), -1)), tt.expected)
	}
}

func TestNumberTruncate(t *testing.T) {
	numberTests := []struct {
		number   string
		truncate int
		expected string
	}{
		{"0.1", 1, ".1"},
		{"0.0001", 1, "1e-4"},
		{"0.111", 1, ".1"},
		{"0.111", 0, "0"},
		{"0.075", 1, ".1"},
		{"0.025", 1, "0"},
		{"9.99", 1, "10"},
		{"8.88", 1, "8.9"},
		{"8.88", 0, "9"},
		{"8.00", 0, "8"},
		{".88", 0, "1"},
		{"1.234", 1, "1.2"},
		{"33.33", 0, "33"},
		{"29.666", 0, "30"},
		{"1.51", 1, "1.5"},
	}
	for _, tt := range numberTests {
		test.Minify(t, tt.number, nil, string(Number([]byte(tt.number), tt.truncate)), tt.expected, "truncate to", tt.truncate)
	}
}

func TestNumberRandom(t *testing.T) {
	N := int(1e6)
	if testing.Short() {
		N = 0
	}
	for i := 0; i < N; i++ {
		b := RandNumBytes()
		f, _ := strconv.ParseFloat(string(b), 64)

		b2 := make([]byte, len(b))
		copy(b2, b)
		b2 = Number(b2, -1)
		f2, _ := strconv.ParseFloat(string(b2), 64)
		if math.Abs(f-f2) > 1e-6 {
			fmt.Println("Bad:", f, "!=", f2, "in", string(b), "to", string(b2))
		}
	}
}

////////////////

var n = 100
var numbers [][]byte

func TestMain(t *testing.T) {
	numbers = make([][]byte, 0, n)
	for j := 0; j < n; j++ {
		numbers = append(numbers, RandNumBytes())
	}
}

func RandNumBytes() []byte {
	var b []byte
	n := rand.Int() % 10
	for i := 0; i < n; i++ {
		b = append(b, byte(rand.Int()%10)+'0')
	}
	if rand.Int()%2 == 0 {
		b = append(b, '.')
		n = rand.Int() % 10
		for i := 0; i < n; i++ {
			b = append(b, byte(rand.Int()%10)+'0')
		}
	}
	if rand.Int()%2 == 0 {
		b = append(b, 'e')
		if rand.Int()%2 == 0 {
			b = append(b, '-')
		}
		n = 1 + rand.Int()%4
		for i := 0; i < n; i++ {
			b = append(b, byte(rand.Int()%10)+'0')
		}
	}
	return b
}

func BenchmarkNumber(b *testing.B) {
	for i := 0; i < b.N; i++ {
		for j := 0; j < n; j++ {
			Number(numbers[j], -1)
		}
	}
}

func BenchmarkNumber2(b *testing.B) {
	num := []byte("1.2345e-6")
	for i := 0; i < b.N; i++ {
		Number(num, -1)
	}
}
<commit_msg>Use subtests<commit_after>package minify // import "github.com/tdewolff/minify"

import (
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"math/rand"
	"strconv"
	"testing"

	"github.com/tdewolff/test"
)

func TestContentType(t *testing.T) {
	contentTypeTests := []struct {
		contentType string
		expected    string
	}{
		{"text/html", "text/html"},
		{"text/html; charset=UTF-8", "text/html;charset=utf-8"},
		{"text/html; charset=UTF-8 ; param = \" ; \"", "text/html;charset=utf-8;param=\" ; \""},
		{"text/html, text/css", "text/html,text/css"},
	}
	for _, tt := range contentTypeTests {
		t.Run(tt.contentType, func(t *testing.T) {
			test.Minify(t, tt.contentType, nil, string(ContentType([]byte(tt.contentType))), tt.expected)
		})
	}
}

func TestDataURI(t *testing.T) {
	dataURITests := []struct {
		dataURI  string
		expected string
	}{
		{"data:,text", "data:,text"},
		{"data:text/plain;charset=us-ascii,text", "data:,text"},
		{"data:TEXT/PLAIN;CHARSET=US-ASCII,text", "data:,text"},
		{"data:text/plain;charset=us-asciiz,text", "data:;charset=us-asciiz,text"},
		{"data:;base64,dGV4dA==", "data:,text"},
		{"data:text/svg+xml;base64,PT09PT09", "data:text/svg+xml;base64,PT09PT09"},
		{"data:text/xml;version=2.0,content", "data:text/xml;version=2.0,content"},
		{"data:text/xml; version = 2.0,content", "data:text/xml;version=2.0,content"},
		{"data:,=====", "data:,%3D%3D%3D%3D%3D"},
		{"data:,======", "data:;base64,PT09PT09"},
		{"data:text/x,<?x?>", "data:text/x,%3C%3Fx%3F%3E"},
	}
	m := New()
	m.AddFunc("text/x", func(_ *M, w io.Writer, r io.Reader, _ map[string]string) error {
		b, _ := ioutil.ReadAll(r)
		test.String(t, string(b), "<?x?>")
		w.Write(b)
		return nil
	})
	for _, tt := range dataURITests {
		t.Run(tt.dataURI, func(t *testing.T) {
			test.Minify(t, tt.dataURI, nil, string(DataURI(m, []byte(tt.dataURI))), tt.expected)
		})
	}
}

func TestNumber(t *testing.T) {
	numberTests := []struct {
		number   string
		expected string
	}{
		{"0", "0"},
		{".0", "0"},
		{"1.0", "1"},
		{"0.1", ".1"},
		{"+1", "1"},
		{"-1", "-1"},
		{"-0.1", "-.1"},
		{"10", "10"},
		{"100", "100"},
		{"1000", "1e3"},
		{"0.001", ".001"},
		{"0.0001", "1e-4"},
		{"100e1", "1e3"},
		{"1.1e+1", "11"},
		{"1.1e6", "11e5"},
		{"0.252", ".252"},
		{"1.252", "1.252"},
		{"-1.252", "-1.252"},
		{"0.075", ".075"},
		{"789012345678901234567890123456789e9234567890123456789", "789012345678901234567890123456789e9234567890123456789"},
		{".000100009", "100009e-9"},
		{".0001000009", ".0001000009"},
		{".0001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000009", ".0001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000009"},
		{"E\x1f", "E\x1f"}, // fuzz
		{"1e9223372036854775807", "1e9223372036854775807"},
		{"11e9223372036854775807", "11e9223372036854775807"},
		{".01e-9223372036854775808", ".01e-9223372036854775808"},
		{".011e-9223372036854775808", ".011e-9223372036854775808"},

		{".12345e8", "12345e3"},
		{".12345e7", "1234500"},
		{".12345e6", "123450"},
		{".12345e5", "12345"},
		{".012345e6", "12345"},
		{".12345e4", "1234.5"},
		{"-.12345e4", "-1234.5"},
		{".12345e0", ".12345"},
		{".12345e-1", ".012345"},
		{".12345e-2", ".0012345"},
		{".12345e-3", "12345e-8"},
		{".12345e-4", "12345e-9"},
		{".12345e-5", "12345e-10"},

		{".123456e-3", "123456e-9"},
		{".123456e-2", ".00123456"},
		{".1234567e-4", "1234567e-11"},
		{".1234567e-3", ".0001234567"},

		{"12345678e-1", "1234567.8"},
		{"72.e-3", ".072"},
		{"7640e-2", "76.4"},
		{"10.e-3", ".01"},
		{".0319e3", "31.9"},
		{"39.7e-2", ".397"},
		{"39.7e-3", ".0397"},
		{".01e1", ".1"},
		{".001e1", ".01"},
		{"39.7e-5", "397e-6"},
	}
	for _, tt := range numberTests {
		t.Run(tt.number, func(t *testing.T) {
			test.Minify(t, tt.number, nil, string(Number([]byte(tt.number), -1)), tt.expected)
		})
	}
}

func TestNumberTruncate(t *testing.T) {
	numberTests := []struct {
		number   string
		truncate int
		expected string
	}{
		{"0.1", 1, ".1"},
		{"0.0001", 1, "1e-4"},
		{"0.111", 1, ".1"},
		{"0.111", 0, "0"},
		{"0.075", 1, ".1"},
		{"0.025", 1, "0"},
		{"9.99", 1, "10"},
		{"8.88", 1, "8.9"},
		{"8.88", 0, "9"},
		{"8.00", 0, "8"},
		{".88", 0, "1"},
		{"1.234", 1, "1.2"},
		{"33.33", 0, "33"},
		{"29.666", 0, "30"},
		{"1.51", 1, "1.5"},
	}
	for _, tt := range numberTests {
		t.Run(tt.number, func(t *testing.T) {
			test.Minify(t, tt.number, nil, string(Number([]byte(tt.number), tt.truncate)), tt.expected, "truncate to", tt.truncate)
		})
	}
}

func TestNumberRandom(t *testing.T) {
	N := int(1e6)
	if testing.Short() {
		N = 0
	}
	for i := 0; i < N; i++ {
		b := RandNumBytes()
		f, _ := strconv.ParseFloat(string(b), 64)

		b2 := make([]byte, len(b))
		copy(b2, b)
		b2 = Number(b2, -1)
		f2, _ := strconv.ParseFloat(string(b2), 64)
		if math.Abs(f-f2) > 1e-6 {
			fmt.Println("Bad:", f, "!=", f2, "in", string(b), "to", string(b2))
		}
	}
}

////////////////

var n = 100
var numbers [][]byte

func TestMain(t *testing.T) {
	numbers = make([][]byte, 0, n)
	for j := 0; j < n; j++ {
		numbers = append(numbers, RandNumBytes())
	}
}

func RandNumBytes() []byte {
	var b []byte
	n := rand.Int() % 10
	for i := 0; i < n; i++ {
		b = append(b, byte(rand.Int()%10)+'0')
	}
	if rand.Int()%2 == 0 {
		b = append(b, '.')
		n = rand.Int() % 10
		for i := 0; i < n; i++ {
			b = append(b, byte(rand.Int()%10)+'0')
		}
	}
	if rand.Int()%2 == 0 {
		b = append(b, 'e')
		if rand.Int()%2 == 0 {
			b = append(b, '-')
		}
		n = 1 + rand.Int()%4
		for i := 0; i < n; i++ {
			b = append(b, byte(rand.Int()%10)+'0')
		}
	}
	return b
}

func BenchmarkNumber(b *testing.B) {
	for i := 0; i < b.N; i++ {
		for j := 0; j < n; j++ {
			Number(numbers[j], -1)
		}
	}
}

func BenchmarkNumber2(b *testing.B) {
	num := []byte("1.2345e-6")
	for i := 0; i < b.N; i++ {
		Number(num, -1)
	}
}
<|endoftext|>


<commit_before>package publicip

import (
	"errors"
	"net"
	"strings"
	"sync"
	"time"

	"github.com/attilaolah/publicip/providers"
)

var (
	// How long to cache thet IP?
	Timeout = time.Minute

	cache struct {
		net.IP
		time.Time
		sync.Mutex
	}
)

// IP returns the public IP address.
//
// Calling multiple times will result in a cached value being returned.
// To disable caching, set publicip.Timeout to zero.
func IP() (net.IP, error) {
	cache.Lock()
	defer cache.Unlock()

	if time.Since(cache.Time) < Timeout {
		return cache.IP, nil
	}

	errs := []string{}
	for provider, fn := range providers.All {
		res, err := fn()
		if err == nil {
			cache.IP = res
			cache.Time = time.Now()
			return res, nil
		}
		errs = append(errs, provider+": "+err.Error())
	}
	return nil, errors.New(strings.Join(errs, "; "))
}

// Refresh returns the public IP, regardless of the state of the cache.
func Refresh() (net.IP, error) {
	cache.Lock()
	cache.Time = time.Time{}
	cache.Unlock()

	return IP()
}
<commit_msg>fetch IPs in parallel, returning the fastest available response<commit_after>package publicip

import (
	"errors"
	"net"
	"sync"
	"time"

	"github.com/attilaolah/publicip/providers"
)

var (
	// How long to cache thet IP?
	Timeout = time.Minute

	cache struct {
		net.IP
		time.Time
		sync.Mutex
	}
)

// IP returns the public IP address.
//
// Calling multiple times will result in a cached value being returned.
// To disable caching, set publicip.Timeout to zero.
func IP() (net.IP, error) {
	cache.Lock()
	defer cache.Unlock()

	if time.Since(cache.Time) < Timeout {
		return cache.IP, nil
	}

	ip, err := fastIP()
	if err != nil {
		return nil, err
	}

	cache.IP = ip
	cache.Time = time.Now()
	return ip, nil
}

// Refresh returns the public IP, regardless of the state of the cache.
func Refresh() (net.IP, error) {
	cache.Lock()
	cache.Time = time.Time{}
	cache.Unlock()

	return IP()
}

// fastIP fires up a goroutine for each provider, returning an IP as soon as one is found.
func fastIP() (net.IP, error) {
	ips := make(chan net.IP, 1)
	errs := make(chan error)
	wg := sync.WaitGroup{}

	wg.Add(len(providers.All))
	for provider, fn := range providers.All {
		go func(provider string, fn func() (net.IP, error)) {
			if ip, err := fn(); err == nil {
				ips <- ip
			} else {
				errs <- err
			}
			wg.Done()
		}(provider, fn)
	}

	go func() {
		wg.Wait()
		close(ips)
		close(errs)
	}()

	for ip := range ips {
		return ip, nil
	}

	for err := range errs {
		return nil, err
	}

	return nil, errors.New("no providers")
}
<|endoftext|>


<commit_before>package peer

import (
	"bytes"
	"encoding/binary"
	"io"
	"io/ioutil"
	"net"
	"time"

	"github.com/cenkalti/rain/internal/bitfield"
	"github.com/cenkalti/rain/internal/logger"
	"github.com/cenkalti/rain/internal/messageid"
	"github.com/cenkalti/rain/internal/torrentdata"
)

const connReadTimeout = 3 * time.Minute

// Reject requests larger than this size.
const maxAllowedBlockSize = 32 * 1024

type Peer struct {
	conn          net.Conn
	id            [20]byte
	data          *torrentdata.Data
	messages      *Messages
	FastExtension bool
	log           logger.Logger
}

func New(conn net.Conn, id [20]byte, extensions *bitfield.Bitfield, d *torrentdata.Data, l logger.Logger, messages *Messages) *Peer {
	return &Peer{
		conn:          conn,
		id:            id,
		data:          d,
		messages:      messages,
		FastExtension: extensions.Test(61),
		log:           l,
	}
}

func (p *Peer) ID() [20]byte {
	return p.id
}

func (p *Peer) String() string {
	return p.conn.RemoteAddr().String()
}

func (p *Peer) Close() error {
	return p.conn.Close()
}

// Run reads and processes incoming messages after handshake.
// TODO send keep-alive messages to peers at interval.
func (p *Peer) Run(stopC chan struct{}) {
	p.log.Debugln("Communicating peer", p.conn.RemoteAddr())
	defer func() {
		select {
		case p.messages.Disconnect <- p:
		case <-stopC:
		}
	}()

	if err := p.sendBitfield(p.data.Bitfield()); err != nil {
		p.log.Error(err)
		return
	}

	// TODO remove after implementing uploader
	p.SendUnchoke()
	p.SendInterested()

	select {
	case p.messages.Connect <- p:
	case <-stopC:
		return
	}

	first := true
	for {
		err := p.conn.SetReadDeadline(time.Now().Add(connReadTimeout))
		if err != nil {
			p.log.Error(err)
			return
		}

		var length uint32
		// p.log.Debug("Reading message...")
		err = binary.Read(p.conn, binary.BigEndian, &length)
		if err != nil {
			if err == io.EOF {
				p.log.Debug("Remote peer has closed the connection")
			} else {
				p.log.Error(err)
			}
			return
		}
		// p.log.Debugf("Received message of length: %d", length)

		if length == 0 { // keep-alive message
			p.log.Debug("Received message of type \"keep alive\"")
			continue
		}

		var id messageid.MessageID
		err = binary.Read(p.conn, binary.BigEndian, &id)
		if err != nil {
			p.log.Error(err)
			return
		}
		length--

		p.log.Debugf("Received message of type: %q", id)

		switch id {
		case messageid.Choke:
			select {
			case p.messages.Choke <- p:
			case <-stopC:
				return
			}
		case messageid.Unchoke:
			select {
			case p.messages.Unchoke <- p:
			case <-stopC:
				return
			}
			// TODO implement
		case messageid.Interested:
			// TODO implement
		case messageid.NotInterested:
			// TODO implement
		case messageid.Have:
			var h haveMessage
			err = binary.Read(p.conn, binary.BigEndian, &h)
			if err != nil {
				p.log.Error(err)
				return
			}
			if h.Index >= uint32(len(p.data.Pieces)) {
				p.log.Error("unexpected piece index")
				return
			}
			pi := &p.data.Pieces[h.Index]
			p.log.Debug("Peer ", p.conn.RemoteAddr(), " has piece #", pi.Index)
			select {
			case p.messages.Have <- Have{p, pi}:
			case <-stopC:
				return
			}
		case messageid.Bitfield:
			if !first {
				p.log.Error("bitfield can only be sent after handshake")
				return
			}
			numBytes := uint32(bitfield.NumBytes(uint32(len(p.data.Pieces))))
			if length != numBytes {
				p.log.Error("invalid bitfield length")
				return
			}
			b := make([]byte, numBytes)
			_, err = io.ReadFull(p.conn, b)
			if err != nil {
				p.log.Error(err)
				return
			}
			bf := bitfield.NewBytes(b, uint32(len(p.data.Pieces)))
			p.log.Debugln("Received bitfield:", bf.Hex())
			select {
			case p.messages.Bitfield <- Bitfield{p, bf}:
			case <-stopC:
				return
			}
		case messageid.Request:
			var req requestMessage
			err = binary.Read(p.conn, binary.BigEndian, &req)
			if err != nil {
				p.log.Error(err)
				return
			}
			p.log.Debugf("Received Request: %+v", req)

			if req.Index >= uint32(len(p.data.Pieces)) {
				p.log.Error("invalid request: index")
				return
			}
			if req.Length > maxAllowedBlockSize {
				p.log.Error("received a request with block size larger than allowed")
				return
			}
			if req.Begin+req.Length > p.data.Pieces[req.Index].Length {
				p.log.Error("invalid request: length")
			}

			pi := &p.data.Pieces[req.Index]
			select {
			case p.messages.Request <- Request{p, pi, req.Begin, req.Length}:
			case <-stopC:
				return
			}
		case messageid.Piece:
			var msg pieceMessage
			err = binary.Read(p.conn, binary.BigEndian, &msg)
			if err != nil {
				p.log.Error(err)
				return
			}
			length -= 8

			if msg.Index >= uint32(len(p.data.Pieces)) {
				p.log.Error("invalid piece index")
				return
			}
			piece := &p.data.Pieces[msg.Index]

			block := piece.GetBlock(msg.Begin)
			if block == nil {
				p.log.Error("invalid block begin")
				return
			}
			if length != block.Length {
				p.log.Error("invalid block length")
				return
			}

			pm := Piece{Peer: p, Piece: piece, Block: block}
			pm.Data = make([]byte, length)
			if _, err = io.ReadFull(p.conn, pm.Data); err != nil {
				p.log.Error(err)
				return
			}

			select {
			case p.messages.Piece <- pm:
			case <-stopC:
				return
			}
		case messageid.HaveAll:
			if !p.FastExtension {
				p.log.Error("have_all message received but fast extensions is not enabled")
				return
			}
			if !first {
				p.log.Error("have_all can only be sent after handshake")
				return
			}
			select {
			case p.messages.HaveAll <- p:
			case <-stopC:
				return
			}
		case messageid.HaveNone:
			if !p.FastExtension {
				p.log.Error("have_none message received but fast extensions is not enabled")
				return
			}
			if !first {
				p.log.Error("have_none can only be sent after handshake")
				return
			}
		// TODO handle cancel messages
		// case messageid.Cancel:
		default:
			p.log.Debugf("unhandled message type: %s", id)
			p.log.Debugln("Discarding", length, "bytes...")
			_, err = io.CopyN(ioutil.Discard, p.conn, int64(length))
			if err != nil {
				p.log.Error(err)
				return
			}
		}
		first = false
	}
}

func (p *Peer) sendBitfield(b *bitfield.Bitfield) error {
	// Sending bitfield may be omitted if have no pieces.
	if b.Count() == 0 {
		return nil
	}
	return p.writeMessage(messageid.Bitfield, b.Bytes())
}

func (p *Peer) SendInterested() error {
	return p.writeMessage(messageid.Interested, nil)
}

func (p *Peer) SendNotInterested() error {
	return p.writeMessage(messageid.NotInterested, nil)
}

func (p *Peer) SendChoke() error {
	return p.writeMessage(messageid.Choke, nil)
}

func (p *Peer) SendUnchoke() error {
	return p.writeMessage(messageid.Unchoke, nil)
}

func (p *Peer) SendHave(piece uint32) error {
	req := haveMessage{piece}
	buf := bytes.NewBuffer(make([]byte, 0, 4))
	_ = binary.Write(buf, binary.BigEndian, &req)
	return p.writeMessage(messageid.Have, buf.Bytes())
}

func (p *Peer) SendRequest(piece, begin, length uint32) error {
	req := requestMessage{piece, begin, length}
	p.log.Debugf("Sending Request: %+v", req)
	buf := bytes.NewBuffer(make([]byte, 0, 12))
	_ = binary.Write(buf, binary.BigEndian, &req)
	return p.writeMessage(messageid.Request, buf.Bytes())
}

func (p *Peer) SendPiece(index, begin uint32, block []byte) error {
	msg := pieceMessage{index, begin}
	p.log.Debugf("Sending Piece: %+v", msg)
	buf := bytes.NewBuffer(make([]byte, 0, 8))
	_ = binary.Write(buf, binary.BigEndian, msg)
	buf.Write(block)
	return p.writeMessage(messageid.Piece, buf.Bytes())
}

func (p *Peer) writeMessage(id messageid.MessageID, payload []byte) error {
	p.log.Debugf("Sending message of type: %q", id)
	buf := bytes.NewBuffer(make([]byte, 0, 4+1+len(payload)))
	var header = struct {
		Length uint32
		ID     messageid.MessageID
	}{
		uint32(1 + len(payload)),
		id,
	}
	_ = binary.Write(buf, binary.BigEndian, &header)
	buf.Write(payload)
	_, err := p.conn.Write(buf.Bytes())
	return err
}
<commit_msg>send have all and have none messages<commit_after>package peer

import (
	"bytes"
	"encoding/binary"
	"io"
	"io/ioutil"
	"net"
	"time"

	"github.com/cenkalti/rain/internal/bitfield"
	"github.com/cenkalti/rain/internal/logger"
	"github.com/cenkalti/rain/internal/messageid"
	"github.com/cenkalti/rain/internal/torrentdata"
)

const connReadTimeout = 3 * time.Minute

// Reject requests larger than this size.
const maxAllowedBlockSize = 32 * 1024

type Peer struct {
	conn          net.Conn
	id            [20]byte
	data          *torrentdata.Data
	messages      *Messages
	FastExtension bool
	log           logger.Logger
}

func New(conn net.Conn, id [20]byte, extensions *bitfield.Bitfield, d *torrentdata.Data, l logger.Logger, messages *Messages) *Peer {
	return &Peer{
		conn:          conn,
		id:            id,
		data:          d,
		messages:      messages,
		FastExtension: extensions.Test(61),
		log:           l,
	}
}

func (p *Peer) ID() [20]byte {
	return p.id
}

func (p *Peer) String() string {
	return p.conn.RemoteAddr().String()
}

// Run reads and processes incoming messages after handshake.
// TODO send keep-alive messages to peers at interval.
func (p *Peer) Run(stopC chan struct{}) {
	p.log.Debugln("Communicating peer", p.conn.RemoteAddr())
	defer func() {
		_ = p.conn.Close()
		select {
		case p.messages.Disconnect <- p:
		case <-stopC:
		}
	}()

	var err error
	if p.FastExtension && p.data.Bitfield().All() {
		err = p.sendHaveAll()
	} else if p.FastExtension && p.data.Bitfield().Count() == 0 {
		err = p.sendHaveNone()
	} else {
		err = p.sendBitfield(p.data.Bitfield())
	}
	if err != nil {
		p.log.Error(err)
		return
	}

	// TODO remove after implementing uploader
	p.SendUnchoke()
	p.SendInterested()

	select {
	case p.messages.Connect <- p:
	case <-stopC:
		return
	}

	first := true
	for {
		err := p.conn.SetReadDeadline(time.Now().Add(connReadTimeout))
		if err != nil {
			p.log.Error(err)
			return
		}

		var length uint32
		// p.log.Debug("Reading message...")
		err = binary.Read(p.conn, binary.BigEndian, &length)
		if err != nil {
			if err == io.EOF {
				p.log.Debug("Remote peer has closed the connection")
			} else {
				p.log.Error(err)
			}
			return
		}
		// p.log.Debugf("Received message of length: %d", length)

		if length == 0 { // keep-alive message
			p.log.Debug("Received message of type \"keep alive\"")
			continue
		}

		var id messageid.MessageID
		err = binary.Read(p.conn, binary.BigEndian, &id)
		if err != nil {
			p.log.Error(err)
			return
		}
		length--

		p.log.Debugf("Received message of type: %q", id)

		switch id {
		case messageid.Choke:
			select {
			case p.messages.Choke <- p:
			case <-stopC:
				return
			}
		case messageid.Unchoke:
			select {
			case p.messages.Unchoke <- p:
			case <-stopC:
				return
			}
			// TODO implement
		case messageid.Interested:
			// TODO implement
		case messageid.NotInterested:
			// TODO implement
		case messageid.Have:
			var h haveMessage
			err = binary.Read(p.conn, binary.BigEndian, &h)
			if err != nil {
				p.log.Error(err)
				return
			}
			if h.Index >= uint32(len(p.data.Pieces)) {
				p.log.Error("unexpected piece index")
				return
			}
			pi := &p.data.Pieces[h.Index]
			p.log.Debug("Peer ", p.conn.RemoteAddr(), " has piece #", pi.Index)
			select {
			case p.messages.Have <- Have{p, pi}:
			case <-stopC:
				return
			}
		case messageid.Bitfield:
			if !first {
				p.log.Error("bitfield can only be sent after handshake")
				return
			}
			numBytes := uint32(bitfield.NumBytes(uint32(len(p.data.Pieces))))
			if length != numBytes {
				p.log.Error("invalid bitfield length")
				return
			}
			b := make([]byte, numBytes)
			_, err = io.ReadFull(p.conn, b)
			if err != nil {
				p.log.Error(err)
				return
			}
			bf := bitfield.NewBytes(b, uint32(len(p.data.Pieces)))
			p.log.Debugln("Received bitfield:", bf.Hex())
			select {
			case p.messages.Bitfield <- Bitfield{p, bf}:
			case <-stopC:
				return
			}
		case messageid.Request:
			var req requestMessage
			err = binary.Read(p.conn, binary.BigEndian, &req)
			if err != nil {
				p.log.Error(err)
				return
			}
			p.log.Debugf("Received Request: %+v", req)

			if req.Index >= uint32(len(p.data.Pieces)) {
				p.log.Error("invalid request: index")
				return
			}
			if req.Length > maxAllowedBlockSize {
				p.log.Error("received a request with block size larger than allowed")
				return
			}
			if req.Begin+req.Length > p.data.Pieces[req.Index].Length {
				p.log.Error("invalid request: length")
			}

			pi := &p.data.Pieces[req.Index]
			select {
			case p.messages.Request <- Request{p, pi, req.Begin, req.Length}:
			case <-stopC:
				return
			}
		case messageid.Piece:
			var msg pieceMessage
			err = binary.Read(p.conn, binary.BigEndian, &msg)
			if err != nil {
				p.log.Error(err)
				return
			}
			length -= 8

			if msg.Index >= uint32(len(p.data.Pieces)) {
				p.log.Error("invalid piece index")
				return
			}
			piece := &p.data.Pieces[msg.Index]

			block := piece.GetBlock(msg.Begin)
			if block == nil {
				p.log.Error("invalid block begin")
				return
			}
			if length != block.Length {
				p.log.Error("invalid block length")
				return
			}

			pm := Piece{Peer: p, Piece: piece, Block: block}
			pm.Data = make([]byte, length)
			if _, err = io.ReadFull(p.conn, pm.Data); err != nil {
				p.log.Error(err)
				return
			}

			select {
			case p.messages.Piece <- pm:
			case <-stopC:
				return
			}
		case messageid.HaveAll:
			if !p.FastExtension {
				p.log.Error("have_all message received but fast extensions is not enabled")
				return
			}
			if !first {
				p.log.Error("have_all can only be sent after handshake")
				return
			}
			select {
			case p.messages.HaveAll <- p:
			case <-stopC:
				return
			}
		case messageid.HaveNone:
			if !p.FastExtension {
				p.log.Error("have_none message received but fast extensions is not enabled")
				return
			}
			if !first {
				p.log.Error("have_none can only be sent after handshake")
				return
			}
		// TODO handle cancel messages
		// case messageid.Cancel:
		default:
			p.log.Debugf("unhandled message type: %s", id)
			p.log.Debugln("Discarding", length, "bytes...")
			_, err = io.CopyN(ioutil.Discard, p.conn, int64(length))
			if err != nil {
				p.log.Error(err)
				return
			}
		}
		first = false
	}
}

func (p *Peer) sendBitfield(b *bitfield.Bitfield) error {
	// Sending bitfield may be omitted if have no pieces.
	if b.Count() == 0 {
		return nil
	}
	return p.writeMessage(messageid.Bitfield, b.Bytes())
}

func (p *Peer) SendInterested() error {
	return p.writeMessage(messageid.Interested, nil)
}

func (p *Peer) SendNotInterested() error {
	return p.writeMessage(messageid.NotInterested, nil)
}

func (p *Peer) SendChoke() error {
	return p.writeMessage(messageid.Choke, nil)
}

func (p *Peer) SendUnchoke() error {
	return p.writeMessage(messageid.Unchoke, nil)
}

func (p *Peer) SendHave(piece uint32) error {
	req := haveMessage{piece}
	buf := bytes.NewBuffer(make([]byte, 0, 4))
	_ = binary.Write(buf, binary.BigEndian, &req)
	return p.writeMessage(messageid.Have, buf.Bytes())
}

func (p *Peer) sendHaveAll() error {
	return p.writeMessage(messageid.HaveAll, nil)
}

func (p *Peer) sendHaveNone() error {
	return p.writeMessage(messageid.HaveNone, nil)
}

func (p *Peer) SendRequest(piece, begin, length uint32) error {
	req := requestMessage{piece, begin, length}
	p.log.Debugf("Sending Request: %+v", req)
	buf := bytes.NewBuffer(make([]byte, 0, 12))
	_ = binary.Write(buf, binary.BigEndian, &req)
	return p.writeMessage(messageid.Request, buf.Bytes())
}

func (p *Peer) SendPiece(index, begin uint32, block []byte) error {
	msg := pieceMessage{index, begin}
	p.log.Debugf("Sending Piece: %+v", msg)
	buf := bytes.NewBuffer(make([]byte, 0, 8))
	_ = binary.Write(buf, binary.BigEndian, msg)
	buf.Write(block)
	return p.writeMessage(messageid.Piece, buf.Bytes())
}

func (p *Peer) writeMessage(id messageid.MessageID, payload []byte) error {
	p.log.Debugf("Sending message of type: %q", id)
	buf := bytes.NewBuffer(make([]byte, 0, 4+1+len(payload)))
	var header = struct {
		Length uint32
		ID     messageid.MessageID
	}{
		uint32(1 + len(payload)),
		id,
	}
	_ = binary.Write(buf, binary.BigEndian, &header)
	buf.Write(payload)
	_, err := p.conn.Write(buf.Bytes())
	return err
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Ceph-CSI Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package rbd

import (
	"context"
	"errors"
	"fmt"

	"github.com/ceph/ceph-csi/internal/journal"
	"github.com/ceph/ceph-csi/internal/util"

	librbd "github.com/ceph/go-ceph/rbd"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
)

// checkCloneImage check the cloned image exists, if the cloned image is not
// found it will check the temporary cloned snapshot exists, and again it will
// check the snapshot exists on the temporary cloned image, if yes it will
// create a new cloned and delete the temporary snapshot and adds a task to
// flatten the temp cloned image and return success.
//
// if the temporary snapshot does not exists it creates a temporary snapshot on
// temporary cloned image and creates  a new cloned with user-provided image
// features and delete the temporary snapshot and adds a task to flatten the
// temp cloned image and return success
//
// if the temporary clone does not exist and if there is a temporary snapshot
// present on the parent image it will delete the temporary snapshot and
// returns.
func (rv *rbdVolume) checkCloneImage(ctx context.Context, parentVol *rbdVolume) (bool, error) {
	// generate temp cloned volume
	tempClone := rv.generateTempClone()
	snap := &rbdSnapshot{
		RbdSnapName: rv.RbdImageName,
		Pool:        rv.Pool,
	}
	// check if cloned image exists
	err := rv.getImageInfo()
	if err == nil {
		// check if do we have temporary snapshot on temporary cloned image
		sErr := tempClone.checkSnapExists(snap)
		if sErr != nil {
			if errors.Is(err, ErrSnapNotFound) {
				return true, nil
			}
			return false, err
		}
		err = tempClone.deleteSnapshot(ctx, snap)
		if err == nil {
			return true, nil
		}
		return false, err
	}
	if !errors.Is(err, ErrImageNotFound) {
		// return error if its not image not found
		return false, err
	}

	err = tempClone.checkSnapExists(snap)
	if err != nil {
		if errors.Is(err, ErrSnapNotFound) {
			// check temporary image needs flatten, if yes add task to flatten the
			// temporary clone
			err = tempClone.flattenRbdImage(ctx, rv.conn.Creds, false, rbdHardMaxCloneDepth, rbdSoftMaxCloneDepth)
			if err != nil {
				return false, err
			}
			// as the snapshot is not present, create new snapshot,clone and
			// delete the temporary snapshot
			err = createRBDClone(ctx, tempClone, rv, snap, rv.conn.Creds)
			if err != nil {
				return false, err
			}
			return true, nil
		} else if !errors.Is(err, ErrImageNotFound) {
			// any error other than image not found return error
			return false, err
		}
	} else {
		// snap will be create after we flatten the temporary cloned image,no
		// need to check for flatten here.
		// as the snap exists,create clone image and delete temporary snapshot
		// and add task to flatten temporary cloned image
		err = rv.cloneRbdImageFromSnapshot(ctx, snap)
		if err != nil {
			util.ErrorLog(ctx, "failed to clone rbd image %s from snapshot %s: %v", rv.RbdImageName, snap.RbdSnapName, err)
			err = fmt.Errorf("failed to clone rbd image %s from snapshot %s: %v", rv.RbdImageName, snap.RbdSnapName, err)
			return false, err
		}
		err = tempClone.deleteSnapshot(ctx, snap)
		if err != nil {
			util.ErrorLog(ctx, "failed to delete snapshot: %v", err)
			return false, err
		}
		return true, nil
	}
	// as the temp clone doesnot exists,check snapshot exists on parent volume
	// snapshot name is same as temporary clone image
	snap.RbdImageName = tempClone.RbdImageName
	err = parentVol.checkSnapExists(snap)
	if err == nil {
		// the temp clone exists, delete it lets reserve a new ID and
		// create new resources for clearner approach
		err = parentVol.deleteSnapshot(ctx, snap)
	}
	if errors.Is(err, ErrSnapNotFound) {
		return false, nil
	}
	return false, err
}

func (rv *rbdVolume) generateTempClone() *rbdVolume {
	tempClone := rbdVolume{}
	tempClone.conn = rv.conn
	// The temp clone image need to have deep flatten feature
	f := []string{librbd.FeatureNameLayering, librbd.FeatureNameDeepFlatten}
	tempClone.imageFeatureSet = librbd.FeatureSetFromNames(f)
	tempClone.ClusterID = rv.ClusterID
	tempClone.Monitors = rv.Monitors
	tempClone.Pool = rv.Pool
	// The temp cloned image name will be always (rbd image name + "-temp")
	// this name will be always unique, as cephcsi never creates an image with
	// this format for new rbd images
	tempClone.RbdImageName = rv.RbdImageName + "-temp"
	return &tempClone
}

func (rv *rbdVolume) createCloneFromImage(ctx context.Context, parentVol *rbdVolume) error {
	// generate temp cloned volume
	tempClone := rv.generateTempClone()
	tempSnap := &rbdSnapshot{
		// snapshot name is same as temporary cloned image, This helps to
		// flatten the temporary cloned images as we cannot have more than 510
		// snapshots on an rbd image
		RbdSnapName: tempClone.RbdImageName,
		Pool:        rv.Pool,
	}

	cloneSnap := &rbdSnapshot{
		RbdSnapName: rv.RbdImageName,
		Pool:        rv.Pool,
	}
	var (
		errClone   error
		errFlatten error
		err        error
	)
	var j = &journal.Connection{}

	j, err = volJournal.Connect(rv.Monitors, rv.RadosNamespace, rv.conn.Creds)
	if err != nil {
		return status.Error(codes.Internal, err.Error())
	}
	defer j.Destroy()

	// create snapshot and temporary clone and delete snapshot
	err = createRBDClone(ctx, parentVol, tempClone, tempSnap, rv.conn.Creds)
	if err != nil {
		return err
	}

	defer func() {
		if err != nil || errFlatten != nil {
			if !errors.Is(errFlatten, ErrFlattenInProgress) {
				// cleanup snapshot
				cErr := cleanUpSnapshot(ctx, parentVol, tempSnap, tempClone, rv.conn.Creds)
				if cErr != nil {
					util.ErrorLog(ctx, "failed to cleanup image %s or snapshot %s: %v", tempSnap, tempClone, cErr)
				}
			}
		}
		if err != nil || errClone != nil {
			cErr := cleanUpSnapshot(ctx, tempClone, cloneSnap, rv, rv.conn.Creds)
			if cErr != nil {
				util.ErrorLog(ctx, "failed to cleanup image %s or snapshot %s: %v", cloneSnap, tempClone, cErr)
			}
		}
	}()
	// flatten clone
	errFlatten = tempClone.flattenRbdImage(ctx, rv.conn.Creds, false, rbdHardMaxCloneDepth, rbdSoftMaxCloneDepth)
	if errFlatten != nil {
		return errFlatten
	}
	// create snap of temp clone from temporary cloned image
	// create final clone
	// delete snap of temp clone
	errClone = createRBDClone(ctx, tempClone, rv, cloneSnap, rv.conn.Creds)
	if errClone != nil {
		// set errFlatten error to cleanup temporary snapshot and temporary clone
		errFlatten = errors.New("failed to create user requested cloned image")
		return errClone
	}
	err = rv.getImageID()
	if err != nil {
		util.ErrorLog(ctx, "failed to get volume id %s: %v", rv, err)
		return err
	}

	err = j.StoreImageID(ctx, rv.JournalPool, rv.ReservedID, rv.ImageID, rv.conn.Creds)
	if err != nil {
		util.ErrorLog(ctx, "failed to store volume %s: %v", rv, err)
		return err
	}
	return nil
}

func (rv *rbdVolume) flattenCloneImage(ctx context.Context) error {
	tempClone := rv.generateTempClone()
	// reducing the limit for cloned images to make sure the limit is in range,
	// If the intermediate clone reaches the depth we may need to return ABORT
	// error message as it need to be flatten before continuing, this may leak
	// omap entries and stale temporary snapshots in corner cases, if we reduce
	// the limit and check for the depth of the parent image clain it self we
	// can flatten the parent images before use to avoid the stale omap entries.
	hardLimit := rbdHardMaxCloneDepth
	softLimit := rbdSoftMaxCloneDepth
	// choosing 2 so that we don't need to flatten the image in the request.
	const depthToAvoidFlatten = 2
	if rbdHardMaxCloneDepth < depthToAvoidFlatten {
		hardLimit = rbdHardMaxCloneDepth - depthToAvoidFlatten
	}
	if rbdSoftMaxCloneDepth < depthToAvoidFlatten {
		softLimit = rbdSoftMaxCloneDepth - depthToAvoidFlatten
	}
	err := tempClone.getImageInfo()
	if err == nil {
		return tempClone.flattenRbdImage(ctx, tempClone.conn.Creds, false, hardLimit, softLimit)
	}
	if err != nil {
		if !errors.Is(err, ErrImageNotFound) {
			return err
		}
	}
	return rv.flattenRbdImage(ctx, rv.conn.Creds, false, hardLimit, softLimit)
}
<commit_msg>rbd: rearrange the cleanup order incase clone fails<commit_after>/*
Copyright 2020 The Ceph-CSI Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package rbd

import (
	"context"
	"errors"
	"fmt"

	"github.com/ceph/ceph-csi/internal/journal"
	"github.com/ceph/ceph-csi/internal/util"

	librbd "github.com/ceph/go-ceph/rbd"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
)

// checkCloneImage check the cloned image exists, if the cloned image is not
// found it will check the temporary cloned snapshot exists, and again it will
// check the snapshot exists on the temporary cloned image, if yes it will
// create a new cloned and delete the temporary snapshot and adds a task to
// flatten the temp cloned image and return success.
//
// if the temporary snapshot does not exists it creates a temporary snapshot on
// temporary cloned image and creates  a new cloned with user-provided image
// features and delete the temporary snapshot and adds a task to flatten the
// temp cloned image and return success
//
// if the temporary clone does not exist and if there is a temporary snapshot
// present on the parent image it will delete the temporary snapshot and
// returns.
func (rv *rbdVolume) checkCloneImage(ctx context.Context, parentVol *rbdVolume) (bool, error) {
	// generate temp cloned volume
	tempClone := rv.generateTempClone()
	snap := &rbdSnapshot{
		RbdSnapName: rv.RbdImageName,
		Pool:        rv.Pool,
	}
	// check if cloned image exists
	err := rv.getImageInfo()
	if err == nil {
		// check if do we have temporary snapshot on temporary cloned image
		sErr := tempClone.checkSnapExists(snap)
		if sErr != nil {
			if errors.Is(err, ErrSnapNotFound) {
				return true, nil
			}
			return false, err
		}
		err = tempClone.deleteSnapshot(ctx, snap)
		if err == nil {
			return true, nil
		}
		return false, err
	}
	if !errors.Is(err, ErrImageNotFound) {
		// return error if its not image not found
		return false, err
	}

	err = tempClone.checkSnapExists(snap)
	if err != nil {
		if errors.Is(err, ErrSnapNotFound) {
			// check temporary image needs flatten, if yes add task to flatten the
			// temporary clone
			err = tempClone.flattenRbdImage(ctx, rv.conn.Creds, false, rbdHardMaxCloneDepth, rbdSoftMaxCloneDepth)
			if err != nil {
				return false, err
			}
			// as the snapshot is not present, create new snapshot,clone and
			// delete the temporary snapshot
			err = createRBDClone(ctx, tempClone, rv, snap, rv.conn.Creds)
			if err != nil {
				return false, err
			}
			return true, nil
		} else if !errors.Is(err, ErrImageNotFound) {
			// any error other than image not found return error
			return false, err
		}
	} else {
		// snap will be create after we flatten the temporary cloned image,no
		// need to check for flatten here.
		// as the snap exists,create clone image and delete temporary snapshot
		// and add task to flatten temporary cloned image
		err = rv.cloneRbdImageFromSnapshot(ctx, snap)
		if err != nil {
			util.ErrorLog(ctx, "failed to clone rbd image %s from snapshot %s: %v", rv.RbdImageName, snap.RbdSnapName, err)
			err = fmt.Errorf("failed to clone rbd image %s from snapshot %s: %v", rv.RbdImageName, snap.RbdSnapName, err)
			return false, err
		}
		err = tempClone.deleteSnapshot(ctx, snap)
		if err != nil {
			util.ErrorLog(ctx, "failed to delete snapshot: %v", err)
			return false, err
		}
		return true, nil
	}
	// as the temp clone doesnot exists,check snapshot exists on parent volume
	// snapshot name is same as temporary clone image
	snap.RbdImageName = tempClone.RbdImageName
	err = parentVol.checkSnapExists(snap)
	if err == nil {
		// the temp clone exists, delete it lets reserve a new ID and
		// create new resources for clearner approach
		err = parentVol.deleteSnapshot(ctx, snap)
	}
	if errors.Is(err, ErrSnapNotFound) {
		return false, nil
	}
	return false, err
}

func (rv *rbdVolume) generateTempClone() *rbdVolume {
	tempClone := rbdVolume{}
	tempClone.conn = rv.conn
	// The temp clone image need to have deep flatten feature
	f := []string{librbd.FeatureNameLayering, librbd.FeatureNameDeepFlatten}
	tempClone.imageFeatureSet = librbd.FeatureSetFromNames(f)
	tempClone.ClusterID = rv.ClusterID
	tempClone.Monitors = rv.Monitors
	tempClone.Pool = rv.Pool
	// The temp cloned image name will be always (rbd image name + "-temp")
	// this name will be always unique, as cephcsi never creates an image with
	// this format for new rbd images
	tempClone.RbdImageName = rv.RbdImageName + "-temp"
	return &tempClone
}

func (rv *rbdVolume) createCloneFromImage(ctx context.Context, parentVol *rbdVolume) error {
	// generate temp cloned volume
	tempClone := rv.generateTempClone()
	tempSnap := &rbdSnapshot{
		// snapshot name is same as temporary cloned image, This helps to
		// flatten the temporary cloned images as we cannot have more than 510
		// snapshots on an rbd image
		RbdSnapName: tempClone.RbdImageName,
		Pool:        rv.Pool,
	}

	cloneSnap := &rbdSnapshot{
		RbdSnapName: rv.RbdImageName,
		Pool:        rv.Pool,
	}
	var (
		errClone   error
		errFlatten error
		err        error
	)
	var j = &journal.Connection{}

	j, err = volJournal.Connect(rv.Monitors, rv.RadosNamespace, rv.conn.Creds)
	if err != nil {
		return status.Error(codes.Internal, err.Error())
	}
	defer j.Destroy()

	// create snapshot and temporary clone and delete snapshot
	err = createRBDClone(ctx, parentVol, tempClone, tempSnap, rv.conn.Creds)
	if err != nil {
		return err
	}

	defer func() {
		if err != nil || errClone != nil {
			cErr := cleanUpSnapshot(ctx, tempClone, cloneSnap, rv, rv.conn.Creds)
			if cErr != nil {
				util.ErrorLog(ctx, "failed to cleanup image %s or snapshot %s: %v", cloneSnap, tempClone, cErr)
			}
		}

		if err != nil || errFlatten != nil {
			if !errors.Is(errFlatten, ErrFlattenInProgress) {
				// cleanup snapshot
				cErr := cleanUpSnapshot(ctx, parentVol, tempSnap, tempClone, rv.conn.Creds)
				if cErr != nil {
					util.ErrorLog(ctx, "failed to cleanup image %s or snapshot %s: %v", tempSnap, tempClone, cErr)
				}
			}
		}
	}()
	// flatten clone
	errFlatten = tempClone.flattenRbdImage(ctx, rv.conn.Creds, false, rbdHardMaxCloneDepth, rbdSoftMaxCloneDepth)
	if errFlatten != nil {
		return errFlatten
	}
	// create snap of temp clone from temporary cloned image
	// create final clone
	// delete snap of temp clone
	errClone = createRBDClone(ctx, tempClone, rv, cloneSnap, rv.conn.Creds)
	if errClone != nil {
		// set errFlatten error to cleanup temporary snapshot and temporary clone
		errFlatten = errors.New("failed to create user requested cloned image")
		return errClone
	}
	err = rv.getImageID()
	if err != nil {
		util.ErrorLog(ctx, "failed to get volume id %s: %v", rv, err)
		return err
	}

	err = j.StoreImageID(ctx, rv.JournalPool, rv.ReservedID, rv.ImageID, rv.conn.Creds)
	if err != nil {
		util.ErrorLog(ctx, "failed to store volume %s: %v", rv, err)
		return err
	}
	return nil
}

func (rv *rbdVolume) flattenCloneImage(ctx context.Context) error {
	tempClone := rv.generateTempClone()
	// reducing the limit for cloned images to make sure the limit is in range,
	// If the intermediate clone reaches the depth we may need to return ABORT
	// error message as it need to be flatten before continuing, this may leak
	// omap entries and stale temporary snapshots in corner cases, if we reduce
	// the limit and check for the depth of the parent image clain it self we
	// can flatten the parent images before use to avoid the stale omap entries.
	hardLimit := rbdHardMaxCloneDepth
	softLimit := rbdSoftMaxCloneDepth
	// choosing 2 so that we don't need to flatten the image in the request.
	const depthToAvoidFlatten = 2
	if rbdHardMaxCloneDepth < depthToAvoidFlatten {
		hardLimit = rbdHardMaxCloneDepth - depthToAvoidFlatten
	}
	if rbdSoftMaxCloneDepth < depthToAvoidFlatten {
		softLimit = rbdSoftMaxCloneDepth - depthToAvoidFlatten
	}
	err := tempClone.getImageInfo()
	if err == nil {
		return tempClone.flattenRbdImage(ctx, tempClone.conn.Creds, false, hardLimit, softLimit)
	}
	if err != nil {
		if !errors.Is(err, ErrImageNotFound) {
			return err
		}
	}
	return rv.flattenRbdImage(ctx, rv.conn.Creds, false, hardLimit, softLimit)
}
<|endoftext|>


<commit_before>// Copyright 2015 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package save

import (
	"errors"
	"fmt"
	"regexp"

	"golang.org/x/net/context"

	"github.com/jacobsa/comeback/internal/blob"
	"github.com/jacobsa/comeback/internal/state"
	"github.com/jacobsa/syncutil"
	"github.com/jacobsa/timeutil"
)

// Save a backup of the given directory, applying the supplied exclusions and
// using the supplied score map to avoid reading file content when possible.
// Return a score for the root of the backup.
func Save(
	ctx context.Context,
	dir string,
	exclusions []*regexp.Regexp,
	scoreMap state.ScoreMap,
	blobStore blob.Store,
	clock timeutil.Clock) (score blob.Score, err error) {
	b := syncutil.NewBundle(ctx)

	// List the directory hierarchy, applying exclusions.
	listedNodes := make(chan *fsNode, 100)
	b.Add(func(ctx context.Context) (err error) {
		defer close(listedNodes)
		err = listNodes(ctx, dir, exclusions, listedNodes)
		if err != nil {
			err = fmt.Errorf("listNodes: %v", err)
			return
		}

		return
	})

	// Stat each one.
	stattedNodes := make(chan *fsNode, 100)
	b.Add(func(ctx context.Context) (err error) {
		defer close(stattedNodes)
		err = statNodes(ctx, dir, listedNodes, stattedNodes)
		if err != nil {
			err = fmt.Errorf("statNodes: %v", err)
			return
		}

		return
	})

	// Fill in scores for files that don't appear to have changed since the last
	// run.
	postScoreMap := make(chan *fsNode, 100)
	b.Add(func(ctx context.Context) (err error) {
		defer close(postScoreMap)
		err = consultScoreMap(ctx, scoreMap, clock, stattedNodes, postScoreMap)
		if err != nil {
			err = fmt.Errorf("consultScoreMap: %v", err)
			return
		}

		return
	})

	// Fill in scores for those nodes that didn't hit in the cache. Do this
	// safely, respecting dependency order (children complete before parents
	// start).
	postDAGTraversal := make(chan *fsNode, 100)
	b.Add(func(ctx context.Context) (err error) {
		defer close(postDAGTraversal)
		err = fillInScores(ctx, blobStore, postScoreMap, postDAGTraversal)
		if err != nil {
			err = fmt.Errorf("fillInScores: %v", err)
			return
		}

		return
	})

	// Update the score map with the results of the previous stage.
	{
		// Tap the channel above.
		teeInput := postDAGTraversal
		tmp := make(chan *fsNode, 100)
		postDAGTraversal = make(chan *fsNode, 100)

		b.Add(func(ctx context.Context) (err error) {
			defer close(tmp)
			defer close(postDAGTraversal)

			err = teeNodes(ctx, teeInput, tmp, postDAGTraversal)
			return
		})

		// Run updateScoreMap.
		b.Add(func(ctx context.Context) (err error) {
			err = updateScoreMap(ctx, scoreMap, tmp)
			if err != nil {
				err = fmt.Errorf("updateScoreMap: %v", err)
				return
			}

			return
		})
	}

	// Find the root score.
	b.Add(func(ctx context.Context) (err error) {
		score, err = findRootScore(postDAGTraversal)
		if err != nil {
			err = fmt.Errorf("findRootScore: %v", err)
			return
		}

		return
	})

	err = b.Join()
	return
}

func findRootScore(nodes <-chan *fsNode) (score blob.Score, err error) {
	err = errors.New("TODO")
	return
}

func teeNodes(
	ctx context.Context,
	in <-chan *fsNode,
	out1 chan<- *fsNode,
	out2 chan<- *fsNode) (err error) {
	err = errors.New("TODO")
	return
}
<commit_msg>Fixed a closure bug.<commit_after>// Copyright 2015 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package save

import (
	"errors"
	"fmt"
	"regexp"

	"golang.org/x/net/context"

	"github.com/jacobsa/comeback/internal/blob"
	"github.com/jacobsa/comeback/internal/state"
	"github.com/jacobsa/syncutil"
	"github.com/jacobsa/timeutil"
)

// Save a backup of the given directory, applying the supplied exclusions and
// using the supplied score map to avoid reading file content when possible.
// Return a score for the root of the backup.
func Save(
	ctx context.Context,
	dir string,
	exclusions []*regexp.Regexp,
	scoreMap state.ScoreMap,
	blobStore blob.Store,
	clock timeutil.Clock) (score blob.Score, err error) {
	b := syncutil.NewBundle(ctx)

	// List the directory hierarchy, applying exclusions.
	listedNodes := make(chan *fsNode, 100)
	b.Add(func(ctx context.Context) (err error) {
		defer close(listedNodes)
		err = listNodes(ctx, dir, exclusions, listedNodes)
		if err != nil {
			err = fmt.Errorf("listNodes: %v", err)
			return
		}

		return
	})

	// Stat each one.
	stattedNodes := make(chan *fsNode, 100)
	b.Add(func(ctx context.Context) (err error) {
		defer close(stattedNodes)
		err = statNodes(ctx, dir, listedNodes, stattedNodes)
		if err != nil {
			err = fmt.Errorf("statNodes: %v", err)
			return
		}

		return
	})

	// Fill in scores for files that don't appear to have changed since the last
	// run.
	postScoreMap := make(chan *fsNode, 100)
	b.Add(func(ctx context.Context) (err error) {
		defer close(postScoreMap)
		err = consultScoreMap(ctx, scoreMap, clock, stattedNodes, postScoreMap)
		if err != nil {
			err = fmt.Errorf("consultScoreMap: %v", err)
			return
		}

		return
	})

	// Fill in scores for those nodes that didn't hit in the cache. Do this
	// safely, respecting dependency order (children complete before parents
	// start).
	postDAGTraversal := make(chan *fsNode, 100)
	b.Add(func(ctx context.Context) (err error) {
		defer close(postDAGTraversal)
		err = fillInScores(ctx, blobStore, postScoreMap, postDAGTraversal)
		if err != nil {
			err = fmt.Errorf("fillInScores: %v", err)
			return
		}

		return
	})

	// Update the score map with the results of the previous stage.
	postScoreMapUpdate := make(chan *fsNode, 100)
	{
		// Tee the channel; updateScoreMap doesn't give output, nor does it modify
		// nodes.
		tmp := make(chan *fsNode, 100)
		b.Add(func(ctx context.Context) (err error) {
			defer close(tmp)
			defer close(postScoreMapUpdate)

			err = teeNodes(ctx, postDAGTraversal, tmp, postScoreMapUpdate)
			return
		})

		// Run updateScoreMap.
		b.Add(func(ctx context.Context) (err error) {
			err = updateScoreMap(ctx, scoreMap, tmp)
			if err != nil {
				err = fmt.Errorf("updateScoreMap: %v", err)
				return
			}

			return
		})
	}

	// Find the root score.
	b.Add(func(ctx context.Context) (err error) {
		score, err = findRootScore(postScoreMapUpdate)
		if err != nil {
			err = fmt.Errorf("findRootScore: %v", err)
			return
		}

		return
	})

	err = b.Join()
	return
}

func findRootScore(nodes <-chan *fsNode) (score blob.Score, err error) {
	err = errors.New("TODO")
	return
}

func teeNodes(
	ctx context.Context,
	in <-chan *fsNode,
	out1 chan<- *fsNode,
	out2 chan<- *fsNode) (err error) {
	err = errors.New("TODO")
	return
}
<|endoftext|>


<commit_before>// Copyright 2018 The go-python Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Bytes objects

package py

import (
	"bytes"
	"fmt"
	"strings"
)

var BytesType = ObjectType.NewType("bytes",
	`bytes(iterable_of_ints) -> bytes
bytes(string, encoding[, errors]) -> bytes
bytes(bytes_or_buffer) -> immutable copy of bytes_or_buffer
bytes(int) -> bytes object of size given by the parameter initialized with null bytes
bytes() -> empty bytes object

Construct an immutable array of bytes from:
  - an iterable yielding integers in range(256)
  - a text string encoded using the specified encoding
  - any object implementing the buffer API.
  - an integer`, BytesNew, nil)

type Bytes []byte

// Type of this Bytes object
func (o Bytes) Type() *Type {
	return BytesType
}

// BytesNew
func BytesNew(metatype *Type, args Tuple, kwargs StringDict) (res Object, err error) {
	var x Object
	var encoding Object
	var errors Object
	var New Object
	kwlist := []string{"source", "encoding", "errors"}

	err = ParseTupleAndKeywords(args, kwargs, "|Oss:bytes", kwlist, &x, &encoding, &errors)
	if err != nil {
		return nil, err
	}
	if x == nil {
		if encoding != nil || errors != nil {
			return nil, ExceptionNewf(TypeError, "encoding or errors without sequence argument")
		}
		return Bytes{}, nil
	}

	if s, ok := x.(String); ok {
		// Encode via the codec registry
		if encoding == nil {
			return nil, ExceptionNewf(TypeError, "string argument without an encoding")
		}
		encodingStr := strings.ToLower(string(encoding.(String)))
		if encodingStr == "utf-8" || encodingStr == "utf8" {
			return Bytes([]byte(s)), nil
		}
		// FIXME
		// New = PyUnicode_AsEncodedString(x, encoding, errors)
		// assert(PyBytes_Check(New))
		// return New
		return nil, ExceptionNewf(NotImplementedError, "String decode for %q not implemented", encodingStr)
	}

	// We'd like to call PyObject_Bytes here, but we need to check for an
	// integer argument before deferring to PyBytes_FromObject, something
	// PyObject_Bytes doesn't do.
	var ok bool
	if I, ok := x.(I__bytes__); ok {
		New, err = I.M__bytes__()
		if err != nil {
			return nil, err
		}
	} else if New, ok, err = TypeCall0(x, "__bytes__"); ok {
		if err != nil {
			return nil, err
		}
	} else {
		goto no_bytes_method
	}
	if _, ok = New.(Bytes); !ok {
		return nil, ExceptionNewf(TypeError, "__bytes__ returned non-bytes (type %s)", New.Type().Name)
	}
no_bytes_method:

	// Is it an integer?
	_, isInt := x.(Int)
	_, isBigInt := x.(*BigInt)
	if isInt || isBigInt {
		size, err := MakeGoInt(x)
		if err != nil {
			return nil, err
		}
		if size < 0 {
			return nil, ExceptionNewf(ValueError, "negative count")
		}
		return make(Bytes, size), nil
	}

	// If it's not unicode, there can't be encoding or errors
	if encoding != nil || errors != nil {
		return nil, ExceptionNewf(TypeError, "encoding or errors without a string argument")
	}

	return BytesFromObject(x)
}

// Converts an object into bytes
func BytesFromObject(x Object) (Bytes, error) {
	// Look for special cases
	// FIXME implement converting from any object implementing the buffer API.
	switch z := x.(type) {
	case Bytes:
		// Immutable type so just return what was passed in
		return z, nil
	case String:
		return nil, ExceptionNewf(TypeError, "cannot convert unicode object to bytes")
	}
	// Otherwise iterate through the whatever converting it into ints
	b := Bytes{}
	var loopErr error
	iterErr := Iterate(x, func(item Object) bool {
		var value int
		value, loopErr = IndexInt(item)
		if loopErr != nil {
			return true
		}
		if value < 0 || value >= 256 {
			loopErr = ExceptionNewf(ValueError, "bytes must be in range(0, 256)")
			return true
		}
		b = append(b, byte(value))
		return false
	})
	if iterErr != nil {
		return nil, iterErr
	}
	if loopErr != nil {
		return nil, loopErr
	}
	return b, nil
}

func (a Bytes) M__str__() (Object, error) {
	return a.M__repr__()
}

func (a Bytes) M__repr__() (Object, error) {
	// FIXME combine this with parser/stringescape.go into file in py?
	var out bytes.Buffer
	quote := '\''
	if bytes.IndexByte(a, byte('\'')) >= 0 && !(bytes.IndexByte(a, byte('"')) >= 0) {
		quote = '"'
	}
	out.WriteRune('b')
	out.WriteRune(quote)
	for _, c := range a {
		switch {
		case c < 0x20:
			switch c {
			case '\t':
				out.WriteString(`\t`)
			case '\n':
				out.WriteString(`\n`)
			case '\r':
				out.WriteString(`\r`)
			default:
				fmt.Fprintf(&out, `\x%02x`, c)
			}
		case c < 0x7F:
			if c == '\\' || (quote == '\'' && c == '\'') || (quote == '"' && c == '"') {
				out.WriteRune('\\')
			}
			out.WriteByte(c)
		default:
			fmt.Fprintf(&out, "\\x%02x", c)
		}
	}
	out.WriteRune(quote)
	return String(out.String()), nil
}

// Convert an Object to an Bytes
//
// Retrurns ok as to whether the conversion worked or not
func convertToBytes(other Object) (Bytes, bool) {
	switch b := other.(type) {
	case Bytes:
		return b, true
	}
	return []byte(nil), false
}

// Rich comparison

func (a Bytes) M__lt__(other Object) (Object, error) {
	if b, ok := convertToBytes(other); ok {
		return NewBool(bytes.Compare(a, b) < 0), nil
	}
	return NotImplemented, nil
}

func (a Bytes) M__le__(other Object) (Object, error) {
	if b, ok := convertToBytes(other); ok {
		return NewBool(bytes.Compare(a, b) <= 0), nil
	}
	return NotImplemented, nil
}

func (a Bytes) M__eq__(other Object) (Object, error) {
	if b, ok := convertToBytes(other); ok {
		return NewBool(bytes.Equal(a, b)), nil
	}
	return NotImplemented, nil
}

func (a Bytes) M__ne__(other Object) (Object, error) {
	if b, ok := convertToBytes(other); ok {
		return NewBool(!bytes.Equal(a, b)), nil
	}
	return NotImplemented, nil
}

func (a Bytes) M__gt__(other Object) (Object, error) {
	if b, ok := convertToBytes(other); ok {
		return NewBool(bytes.Compare(a, b) > 0), nil
	}
	return NotImplemented, nil
}

func (a Bytes) M__ge__(other Object) (Object, error) {
	if b, ok := convertToBytes(other); ok {
		return NewBool(bytes.Compare(a, b) >= 0), nil
	}
	return NotImplemented, nil
}

func (a Bytes) M__add__(other Object) (Object, error) {
	if b, ok := convertToBytes(other); ok {
		o := make([]byte, len(a)+len(b))
		copy(o[:len(a)], a)
		copy(o[len(a):], b)
		return Bytes(o), nil
	}
	return NotImplemented, nil
}

func (a Bytes) M__iadd__(other Object) (Object, error) {
	if b, ok := convertToBytes(other); ok {
		a = append(a, b...)
		return a, nil
	}
	return NotImplemented, nil
}

// Check interface is satisfied
var (
	_ richComparison = (Bytes)(nil)
	_ I__add__       = (Bytes)(nil)
	_ I__iadd__      = (Bytes)(nil)
)
<commit_msg>py: add bytes.replace<commit_after>// Copyright 2018 The go-python Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Bytes objects

package py

import (
	"bytes"
	"fmt"
	"strings"
)

var BytesType = ObjectType.NewType("bytes",
	`bytes(iterable_of_ints) -> bytes
bytes(string, encoding[, errors]) -> bytes
bytes(bytes_or_buffer) -> immutable copy of bytes_or_buffer
bytes(int) -> bytes object of size given by the parameter initialized with null bytes
bytes() -> empty bytes object

Construct an immutable array of bytes from:
  - an iterable yielding integers in range(256)
  - a text string encoded using the specified encoding
  - any object implementing the buffer API.
  - an integer`, BytesNew, nil)

type Bytes []byte

// Type of this Bytes object
func (o Bytes) Type() *Type {
	return BytesType
}

// BytesNew
func BytesNew(metatype *Type, args Tuple, kwargs StringDict) (res Object, err error) {
	var x Object
	var encoding Object
	var errors Object
	var New Object
	kwlist := []string{"source", "encoding", "errors"}

	err = ParseTupleAndKeywords(args, kwargs, "|Oss:bytes", kwlist, &x, &encoding, &errors)
	if err != nil {
		return nil, err
	}
	if x == nil {
		if encoding != nil || errors != nil {
			return nil, ExceptionNewf(TypeError, "encoding or errors without sequence argument")
		}
		return Bytes{}, nil
	}

	if s, ok := x.(String); ok {
		// Encode via the codec registry
		if encoding == nil {
			return nil, ExceptionNewf(TypeError, "string argument without an encoding")
		}
		encodingStr := strings.ToLower(string(encoding.(String)))
		if encodingStr == "utf-8" || encodingStr == "utf8" {
			return Bytes([]byte(s)), nil
		}
		// FIXME
		// New = PyUnicode_AsEncodedString(x, encoding, errors)
		// assert(PyBytes_Check(New))
		// return New
		return nil, ExceptionNewf(NotImplementedError, "String decode for %q not implemented", encodingStr)
	}

	// We'd like to call PyObject_Bytes here, but we need to check for an
	// integer argument before deferring to PyBytes_FromObject, something
	// PyObject_Bytes doesn't do.
	var ok bool
	if I, ok := x.(I__bytes__); ok {
		New, err = I.M__bytes__()
		if err != nil {
			return nil, err
		}
	} else if New, ok, err = TypeCall0(x, "__bytes__"); ok {
		if err != nil {
			return nil, err
		}
	} else {
		goto no_bytes_method
	}
	if _, ok = New.(Bytes); !ok {
		return nil, ExceptionNewf(TypeError, "__bytes__ returned non-bytes (type %s)", New.Type().Name)
	}
no_bytes_method:

	// Is it an integer?
	_, isInt := x.(Int)
	_, isBigInt := x.(*BigInt)
	if isInt || isBigInt {
		size, err := MakeGoInt(x)
		if err != nil {
			return nil, err
		}
		if size < 0 {
			return nil, ExceptionNewf(ValueError, "negative count")
		}
		return make(Bytes, size), nil
	}

	// If it's not unicode, there can't be encoding or errors
	if encoding != nil || errors != nil {
		return nil, ExceptionNewf(TypeError, "encoding or errors without a string argument")
	}

	return BytesFromObject(x)
}

// Converts an object into bytes
func BytesFromObject(x Object) (Bytes, error) {
	// Look for special cases
	// FIXME implement converting from any object implementing the buffer API.
	switch z := x.(type) {
	case Bytes:
		// Immutable type so just return what was passed in
		return z, nil
	case String:
		return nil, ExceptionNewf(TypeError, "cannot convert unicode object to bytes")
	}
	// Otherwise iterate through the whatever converting it into ints
	b := Bytes{}
	var loopErr error
	iterErr := Iterate(x, func(item Object) bool {
		var value int
		value, loopErr = IndexInt(item)
		if loopErr != nil {
			return true
		}
		if value < 0 || value >= 256 {
			loopErr = ExceptionNewf(ValueError, "bytes must be in range(0, 256)")
			return true
		}
		b = append(b, byte(value))
		return false
	})
	if iterErr != nil {
		return nil, iterErr
	}
	if loopErr != nil {
		return nil, loopErr
	}
	return b, nil
}

func (a Bytes) M__str__() (Object, error) {
	return a.M__repr__()
}

func (a Bytes) M__repr__() (Object, error) {
	// FIXME combine this with parser/stringescape.go into file in py?
	var out bytes.Buffer
	quote := '\''
	if bytes.IndexByte(a, byte('\'')) >= 0 && !(bytes.IndexByte(a, byte('"')) >= 0) {
		quote = '"'
	}
	out.WriteRune('b')
	out.WriteRune(quote)
	for _, c := range a {
		switch {
		case c < 0x20:
			switch c {
			case '\t':
				out.WriteString(`\t`)
			case '\n':
				out.WriteString(`\n`)
			case '\r':
				out.WriteString(`\r`)
			default:
				fmt.Fprintf(&out, `\x%02x`, c)
			}
		case c < 0x7F:
			if c == '\\' || (quote == '\'' && c == '\'') || (quote == '"' && c == '"') {
				out.WriteRune('\\')
			}
			out.WriteByte(c)
		default:
			fmt.Fprintf(&out, "\\x%02x", c)
		}
	}
	out.WriteRune(quote)
	return String(out.String()), nil
}

// Convert an Object to an Bytes
//
// Retrurns ok as to whether the conversion worked or not
func convertToBytes(other Object) (Bytes, bool) {
	switch b := other.(type) {
	case Bytes:
		return b, true
	}
	return []byte(nil), false
}

// Rich comparison

func (a Bytes) M__lt__(other Object) (Object, error) {
	if b, ok := convertToBytes(other); ok {
		return NewBool(bytes.Compare(a, b) < 0), nil
	}
	return NotImplemented, nil
}

func (a Bytes) M__le__(other Object) (Object, error) {
	if b, ok := convertToBytes(other); ok {
		return NewBool(bytes.Compare(a, b) <= 0), nil
	}
	return NotImplemented, nil
}

func (a Bytes) M__eq__(other Object) (Object, error) {
	if b, ok := convertToBytes(other); ok {
		return NewBool(bytes.Equal(a, b)), nil
	}
	return NotImplemented, nil
}

func (a Bytes) M__ne__(other Object) (Object, error) {
	if b, ok := convertToBytes(other); ok {
		return NewBool(!bytes.Equal(a, b)), nil
	}
	return NotImplemented, nil
}

func (a Bytes) M__gt__(other Object) (Object, error) {
	if b, ok := convertToBytes(other); ok {
		return NewBool(bytes.Compare(a, b) > 0), nil
	}
	return NotImplemented, nil
}

func (a Bytes) M__ge__(other Object) (Object, error) {
	if b, ok := convertToBytes(other); ok {
		return NewBool(bytes.Compare(a, b) >= 0), nil
	}
	return NotImplemented, nil
}

func (a Bytes) M__add__(other Object) (Object, error) {
	if b, ok := convertToBytes(other); ok {
		o := make([]byte, len(a)+len(b))
		copy(o[:len(a)], a)
		copy(o[len(a):], b)
		return Bytes(o), nil
	}
	return NotImplemented, nil
}

func (a Bytes) M__iadd__(other Object) (Object, error) {
	if b, ok := convertToBytes(other); ok {
		a = append(a, b...)
		return a, nil
	}
	return NotImplemented, nil
}

func (a Bytes) Replace(args Tuple) (Object, error) {
	var (
		pyold Object = None
		pynew Object = None
		pycnt Object = Int(-1)
	)
	err := ParseTuple(args, "yy|i:replace", &pyold, &pynew, &pycnt)
	if err != nil {
		return nil, err
	}

	var (
		old = []byte(pyold.(Bytes))
		new = []byte(pynew.(Bytes))
		cnt = int(pycnt.(Int))
	)

	return Bytes(bytes.Replace([]byte(a), old, new, cnt)), nil
}

// Check interface is satisfied
var (
	_ richComparison = (Bytes)(nil)
	_ I__add__       = (Bytes)(nil)
	_ I__iadd__      = (Bytes)(nil)
)

func init() {
	BytesType.Dict["replace"] = MustNewMethod("replace", func(self Object, args Tuple) (Object, error) {
		return self.(Bytes).Replace(args)
	}, 0, `replace(self, old, new, count=-1) -> return a copy with all occurrences of substring old replaced by new.

  count
    Maximum number of occurrences to replace.
    -1 (the default value) means replace all occurrences.

If the optional argument count is given, only the first count occurrences are
replaced.`)

}
<|endoftext|>


<commit_before>package interpreter

import (
	"bytes"
	"errors"
	"os"
	"regexp"
	"strconv"
	"strings"
	"unicode"

	"../dos"
)

type StatementT struct {
	Argv     []string
	Redirect []*Redirecter
	Term     string
}

var prefix []string = []string{" 0<", " 1>", " 2>"}

var PercentFunc = map[string]func() string{
	"CD": func() string {
		wd, err := os.Getwd()
		if err == nil {
			return wd
		} else {
			return ""
		}
	},
	"ERRORLEVEL": func() string {
		return ErrorLevel
	},
}

var rxUnicode = regexp.MustCompile("^[uU]\\+?([0-9a-fA-F]+)$")

func chomp(buffer *bytes.Buffer) {
	original := buffer.String()
	buffer.Reset()
	var lastchar rune
	for i, ch := range original {
		if i > 0 {
			buffer.WriteRune(lastchar)
		}
		lastchar = ch
	}
}

func dequote(source *bytes.Buffer) string {
	var buffer bytes.Buffer

	lastchar := ' '
	quote := false
	for {
		ch, _, err := source.ReadRune()
		if err != nil {
			break
		}
		if ch == '~' && unicode.IsSpace(lastchar) {
			if home := dos.GetHome(); home != "" {
				buffer.WriteString(home)
			} else {
				buffer.WriteRune('~')
			}
			lastchar = '~'
			continue
		}
		if ch == '%' {
			var nameBuf bytes.Buffer
			for {
				ch, _, err = source.ReadRune()
				if err != nil {
					buffer.WriteRune('%')
					buffer.WriteString(nameBuf.String())
					return buffer.String()
				}
				if ch == '%' {
					nameStr := nameBuf.String()
					value := os.Getenv(nameStr)
					if value != "" {
						buffer.WriteString(value)
					} else if m := rxUnicode.FindStringSubmatch(nameStr); m != nil {
						ucode, _ := strconv.ParseInt(m[1], 16, 32)
						buffer.WriteRune(rune(ucode))
					} else if f, ok := PercentFunc[nameStr]; ok {
						buffer.WriteString(f())
					} else {
						buffer.WriteRune('%')
						buffer.WriteString(nameStr)
						buffer.WriteRune('%')
					}
					break
				}
				if !unicode.IsLower(ch) && !unicode.IsUpper(ch) && !unicode.IsNumber(ch) && ch != '_' {
					source.UnreadRune()
					buffer.WriteRune('%')
					buffer.WriteString(nameBuf.String())
					break
				}
				nameBuf.WriteRune(ch)
			}
			continue
		}
		if ch == '"' {
			quote = !quote
			if lastchar == '"' && quote {
				buffer.WriteRune('"')
				lastchar = '\000'
			}
		} else {
			buffer.WriteRune(ch)
		}
		lastchar = ch
	}
	return buffer.String()
}

func terminate(statements *[]StatementT,
	isRedirected *bool,
	redirect *[]*Redirecter,
	buffer *bytes.Buffer,
	argv *[]string,
	term string) {

	var statement1 StatementT
	if buffer.Len() > 0 {
		if *isRedirected && len(*redirect) > 0 {
			(*redirect)[len(*redirect)-1].SetPath(dequote(buffer))
			*isRedirected = false
			statement1.Argv = *argv
		} else {
			statement1.Argv = append(*argv, dequote(buffer))
		}
		buffer.Reset()
	} else if len(*argv) <= 0 {
		return
	} else {
		statement1.Argv = *argv
	}
	statement1.Redirect = *redirect
	*redirect = make([]*Redirecter, 0, 3)
	*argv = make([]string, 0)
	statement1.Term = term
	*statements = append(*statements, statement1)
}

func parse1(text string) ([]StatementT, error) {
	isQuoted := false
	statements := make([]StatementT, 0)
	argv := make([]string, 0)
	lastchar := ' '
	var buffer bytes.Buffer
	isNextRedirect := false
	redirect := make([]*Redirecter, 0, 3)

	reader := strings.NewReader(text)
	for reader.Len() > 0 {
		ch, chSize, chErr := reader.ReadRune()
		if chSize <= 0 {
			break
		}
		if chErr != nil {
			return nil, chErr
		}
		if ch == '"' {
			isQuoted = !isQuoted
		}
		if isQuoted {
			buffer.WriteRune(ch)
		} else if ch == ' ' {
			if buffer.Len() > 0 {
				if isNextRedirect && len(redirect) > 0 {
					redirect[len(redirect)-1].SetPath(dequote(&buffer))
				} else {
					argv = append(argv, dequote(&buffer))
				}
				buffer.Reset()
				isNextRedirect = false
			}
		} else if lastchar == ' ' && ch == ';' {
			terminate(&statements, &isNextRedirect, &redirect, &buffer, &argv, ";")
		} else if ch == '|' {
			if lastchar == '|' {
				statements[len(statements)-1].Term = "||"
			} else {
				terminate(&statements, &isNextRedirect, &redirect, &buffer, &argv, "|")
			}
		} else if ch == '&' {
			switch lastchar {
			case '&':
				statements[len(statements)-1].Term = "&&"
			case '|':
				statements[len(statements)-1].Term = "|&"
			case '>':
				// >&[n]
				ch2, ch2siz, ch2err := reader.ReadRune()
				if ch2err != nil {
					return nil, ch2err
				}
				if ch2siz <= 0 {
					return nil, errors.New("Too Near EOF for >&")
				}
				red := redirect[len(redirect)-1]
				switch ch2 {
				case '1':
					red.DupFrom(1)
				case '2':
					red.DupFrom(2)
				default:
					return nil, errors.New("Syntax error after >&")
				}
				isNextRedirect = false
			default:
				terminate(&statements, &isNextRedirect, &redirect, &buffer, &argv, "&")
			}
		} else if ch == '>' {
			switch lastchar {
			case '1':
				// 1>
				chomp(&buffer)
				redirect = append(redirect, NewRedirecter(1))
			case '2':
				// 2>
				chomp(&buffer)
				redirect = append(redirect, NewRedirecter(2))
			case '>':
				// >>
				if len(redirect) >= 0 {
					redirect[len(redirect)-1].SetAppend()
				}
			default:
				// >
				redirect = append(redirect, NewRedirecter(1))
			}
			isNextRedirect = true
		} else if ch == '<' {
			redirect = append(redirect, NewRedirecter(0))
			isNextRedirect = true
		} else {
			buffer.WriteRune(ch)
		}
		lastchar = ch
	}
	terminate(&statements, &isNextRedirect, &redirect, &buffer, &argv, " ")
	return statements, nil
}

// Make arrays whose elements are pipelines
func parse2(statements []StatementT) [][]StatementT {
	result := make([][]StatementT, 1)
	for _, statement1 := range statements {
		result[len(result)-1] = append(result[len(result)-1], statement1)
		switch statement1.Term {
		case "|", "|&":

		default:
			result = append(result, make([]StatementT, 0))
		}
	}
	if len(result[len(result)-1]) <= 0 {
		result = result[0 : len(result)-1]
	}
	return result
}

func Parse(text string) ([][]StatementT, error) {
	result1, err := parse1(text)
	if err != nil {
		return nil, err
	}
	result2 := parse2(result1)
	return result2, nil
}
<commit_msg>Single quotations like UNIX-Shell<commit_after>package interpreter

import (
	"bytes"
	"errors"
	"os"
	"regexp"
	"strconv"
	"strings"
	"unicode"

	"../dos"
)

type StatementT struct {
	Argv     []string
	Redirect []*Redirecter
	Term     string
}

var prefix []string = []string{" 0<", " 1>", " 2>"}

var PercentFunc = map[string]func() string{
	"CD": func() string {
		wd, err := os.Getwd()
		if err == nil {
			return wd
		} else {
			return ""
		}
	},
	"ERRORLEVEL": func() string {
		return ErrorLevel
	},
}

var rxUnicode = regexp.MustCompile("^[uU]\\+?([0-9a-fA-F]+)$")

func chomp(buffer *bytes.Buffer) {
	original := buffer.String()
	buffer.Reset()
	var lastchar rune
	for i, ch := range original {
		if i > 0 {
			buffer.WriteRune(lastchar)
		}
		lastchar = ch
	}
}

func dequote(source *bytes.Buffer) string {
	var buffer bytes.Buffer

	lastchar := ' '
	lastquote := '\000'
	for {
		ch, _, err := source.ReadRune()
		if err != nil {
			break
		}
		if ch == '~' && unicode.IsSpace(lastchar) {
			if home := dos.GetHome(); home != "" {
				buffer.WriteString(home)
			} else {
				buffer.WriteRune('~')
			}
			lastchar = '~'
			continue
		}
		if ch == '%' {
			var nameBuf bytes.Buffer
			for {
				ch, _, err = source.ReadRune()
				if err != nil {
					buffer.WriteRune('%')
					buffer.WriteString(nameBuf.String())
					return buffer.String()
				}
				if ch == '%' {
					nameStr := nameBuf.String()
					value := os.Getenv(nameStr)
					if value != "" {
						buffer.WriteString(value)
					} else if m := rxUnicode.FindStringSubmatch(nameStr); m != nil {
						ucode, _ := strconv.ParseInt(m[1], 16, 32)
						buffer.WriteRune(rune(ucode))
					} else if f, ok := PercentFunc[nameStr]; ok {
						buffer.WriteString(f())
					} else {
						buffer.WriteRune('%')
						buffer.WriteString(nameStr)
						buffer.WriteRune('%')
					}
					break
				}
				if !unicode.IsLower(ch) && !unicode.IsUpper(ch) && !unicode.IsNumber(ch) && ch != '_' {
					source.UnreadRune()
					buffer.WriteRune('%')
					buffer.WriteString(nameBuf.String())
					break
				}
				nameBuf.WriteRune(ch)
			}
			continue
		}

		if lastquote != '\000' && ch == lastquote {
			lastquote = '\000'
		} else if (ch == '\'' || ch == '"') && lastquote == '\000' {
			lastquote = ch
			if ch == lastchar {
				buffer.WriteRune(ch)
			}
		} else {
			buffer.WriteRune(ch)
		}
		lastchar = ch
	}
	return buffer.String()
}

func terminate(statements *[]StatementT,
	isRedirected *bool,
	redirect *[]*Redirecter,
	buffer *bytes.Buffer,
	argv *[]string,
	term string) {

	var statement1 StatementT
	if buffer.Len() > 0 {
		if *isRedirected && len(*redirect) > 0 {
			(*redirect)[len(*redirect)-1].SetPath(dequote(buffer))
			*isRedirected = false
			statement1.Argv = *argv
		} else {
			statement1.Argv = append(*argv, dequote(buffer))
		}
		buffer.Reset()
	} else if len(*argv) <= 0 {
		return
	} else {
		statement1.Argv = *argv
	}
	statement1.Redirect = *redirect
	*redirect = make([]*Redirecter, 0, 3)
	*argv = make([]string, 0)
	statement1.Term = term
	*statements = append(*statements, statement1)
}

func parse1(text string) ([]StatementT, error) {
	lastquote := '\000'
	statements := make([]StatementT, 0)
	argv := make([]string, 0)
	lastchar := ' '
	var buffer bytes.Buffer
	isNextRedirect := false
	redirect := make([]*Redirecter, 0, 3)

	reader := strings.NewReader(text)
	for reader.Len() > 0 {
		ch, chSize, chErr := reader.ReadRune()
		if chSize <= 0 {
			break
		}
		if chErr != nil {
			return nil, chErr
		}
		if lastquote == '\000' {
			if ch == '"' || ch == '\'' {
				lastquote = ch
			}
		} else if ch == lastquote {
			lastquote = '\000'
		}
		if lastquote != '\000' {
			buffer.WriteRune(ch)
		} else if ch == ' ' {
			if buffer.Len() > 0 {
				if isNextRedirect && len(redirect) > 0 {
					redirect[len(redirect)-1].SetPath(dequote(&buffer))
				} else {
					argv = append(argv, dequote(&buffer))
				}
				buffer.Reset()
				isNextRedirect = false
			}
		} else if lastchar == ' ' && ch == ';' {
			terminate(&statements, &isNextRedirect, &redirect, &buffer, &argv, ";")
		} else if ch == '|' {
			if lastchar == '|' {
				statements[len(statements)-1].Term = "||"
			} else {
				terminate(&statements, &isNextRedirect, &redirect, &buffer, &argv, "|")
			}
		} else if ch == '&' {
			switch lastchar {
			case '&':
				statements[len(statements)-1].Term = "&&"
			case '|':
				statements[len(statements)-1].Term = "|&"
			case '>':
				// >&[n]
				ch2, ch2siz, ch2err := reader.ReadRune()
				if ch2err != nil {
					return nil, ch2err
				}
				if ch2siz <= 0 {
					return nil, errors.New("Too Near EOF for >&")
				}
				red := redirect[len(redirect)-1]
				switch ch2 {
				case '1':
					red.DupFrom(1)
				case '2':
					red.DupFrom(2)
				default:
					return nil, errors.New("Syntax error after >&")
				}
				isNextRedirect = false
			default:
				terminate(&statements, &isNextRedirect, &redirect, &buffer, &argv, "&")
			}
		} else if ch == '>' {
			switch lastchar {
			case '1':
				// 1>
				chomp(&buffer)
				redirect = append(redirect, NewRedirecter(1))
			case '2':
				// 2>
				chomp(&buffer)
				redirect = append(redirect, NewRedirecter(2))
			case '>':
				// >>
				if len(redirect) >= 0 {
					redirect[len(redirect)-1].SetAppend()
				}
			default:
				// >
				redirect = append(redirect, NewRedirecter(1))
			}
			isNextRedirect = true
		} else if ch == '<' {
			redirect = append(redirect, NewRedirecter(0))
			isNextRedirect = true
		} else {
			buffer.WriteRune(ch)
		}
		lastchar = ch
	}
	terminate(&statements, &isNextRedirect, &redirect, &buffer, &argv, " ")
	return statements, nil
}

// Make arrays whose elements are pipelines
func parse2(statements []StatementT) [][]StatementT {
	result := make([][]StatementT, 1)
	for _, statement1 := range statements {
		result[len(result)-1] = append(result[len(result)-1], statement1)
		switch statement1.Term {
		case "|", "|&":

		default:
			result = append(result, make([]StatementT, 0))
		}
	}
	if len(result[len(result)-1]) <= 0 {
		result = result[0 : len(result)-1]
	}
	return result
}

func Parse(text string) ([][]StatementT, error) {
	result1, err := parse1(text)
	if err != nil {
		return nil, err
	}
	result2 := parse2(result1)
	return result2, nil
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strings"
)

var (
	dir = flag.String("dir", ".", "Path to directory with template files to compile. "+
		"Only files with ext extension are compiled. See ext flag for details.\n"+
		"The compiler recursively processes all the subdirectories.\n"+
		"Compiled template files are placed near the original file with .go extension added.")

	ext = flag.String("ext", "qtpl", "Only files with this extension are compiled")
)

var filesCompiled int

func main() {
	flag.Parse()

	if len(*ext) == 0 {
		log.Fatalf("ext cannot be empty")
	}
	if len(*dir) == 0 {
		*dir = "."
	}
	if (*ext)[0] != '.' {
		*ext = "." + *ext
	}

	compileDir(*dir)
	log.Printf("Total files compiled: %d", filesCompiled)
}

func compileDir(path string) {
	fi, err := os.Stat(path)
	if err != nil {
		log.Fatalf("cannot compile files in %q: %s", path, err)
	}
	if !fi.IsDir() {
		log.Fatalf("cannot compile files in %q: it is not directory", path)
	}
	d, err := os.Open(path)
	if err != nil {
		log.Fatalf("cannot compile files in %q: %s", path, err)
	}
	defer d.Close()

	fis, err := d.Readdir(-1)
	if err != nil {
		log.Fatalf("cannot read files in %q: %s", path, err)
	}

	var names []string
	for _, fi = range fis {
		name := fi.Name()
		if name == "." || name == ".." {
			continue
		}
		if !fi.IsDir() {
			names = append(names, name)
		} else {
			subPath := filepath.Join(path, name)
			compileDir(subPath)
		}
	}
	sort.Strings(names)

	for _, name := range names {
		if strings.HasSuffix(name, *ext) {
			filename := filepath.Join(path, name)
			compileFile(filename)
		}
	}
}

func compileFile(infile string) {
	outfile := infile + ".go"
	log.Printf("Compiling %q to %q...", infile, outfile)
	inf, err := os.Open(infile)
	if err != nil {
		log.Fatalf("cannot open file %q: %s", infile, err)
	}
	defer inf.Close()

	outf, err := os.Create(outfile)
	if err != nil {
		log.Fatalf("cannot create file %q: %s", outfile, err)
	}
	defer outf.Close()

	packageName, err := getPackageName(infile)
	if err != nil {
		log.Fatalf("cannot determine package name for %q: %s", infile, err)
	}
	if err = parse(outf, inf, infile, packageName); err != nil {
		log.Fatalf("error when parsing file %q: %s", infile, err)
	}
	filesCompiled++
}
<commit_msg>prefix qtc log messages with 'qtc: '<commit_after>package main

import (
	"flag"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strings"
)

var (
	dir = flag.String("dir", ".", "Path to directory with template files to compile. "+
		"Only files with ext extension are compiled. See ext flag for details.\n"+
		"The compiler recursively processes all the subdirectories.\n"+
		"Compiled template files are placed near the original file with .go extension added.")

	ext = flag.String("ext", "qtpl", "Only files with this extension are compiled")
)

var logger = log.New(os.Stderr, "qtc: ", log.LstdFlags)

var filesCompiled int

func main() {
	flag.Parse()

	if len(*ext) == 0 {
		logger.Fatalf("ext cannot be empty")
	}
	if len(*dir) == 0 {
		*dir = "."
	}
	if (*ext)[0] != '.' {
		*ext = "." + *ext
	}

	logger.Printf("Compiling *%s template files in directory %q", *ext, *dir)
	compileDir(*dir)
	logger.Printf("Total files compiled: %d", filesCompiled)
}

func compileDir(path string) {
	fi, err := os.Stat(path)
	if err != nil {
		logger.Fatalf("cannot compile files in %q: %s", path, err)
	}
	if !fi.IsDir() {
		logger.Fatalf("cannot compile files in %q: it is not directory", path)
	}
	d, err := os.Open(path)
	if err != nil {
		logger.Fatalf("cannot compile files in %q: %s", path, err)
	}
	defer d.Close()

	fis, err := d.Readdir(-1)
	if err != nil {
		logger.Fatalf("cannot read files in %q: %s", path, err)
	}

	var names []string
	for _, fi = range fis {
		name := fi.Name()
		if name == "." || name == ".." {
			continue
		}
		if !fi.IsDir() {
			names = append(names, name)
		} else {
			subPath := filepath.Join(path, name)
			compileDir(subPath)
		}
	}
	sort.Strings(names)

	for _, name := range names {
		if strings.HasSuffix(name, *ext) {
			filename := filepath.Join(path, name)
			compileFile(filename)
		}
	}
}

func compileFile(infile string) {
	outfile := infile + ".go"
	logger.Printf("Compiling %q to %q...", infile, outfile)
	inf, err := os.Open(infile)
	if err != nil {
		logger.Fatalf("cannot open file %q: %s", infile, err)
	}
	defer inf.Close()

	outf, err := os.Create(outfile)
	if err != nil {
		logger.Fatalf("cannot create file %q: %s", outfile, err)
	}
	defer outf.Close()

	packageName, err := getPackageName(infile)
	if err != nil {
		logger.Fatalf("cannot determine package name for %q: %s", infile, err)
	}
	if err = parse(outf, inf, infile, packageName); err != nil {
		logger.Fatalf("error when parsing file %q: %s", infile, err)
	}
	filesCompiled++
}
<|endoftext|>