<commit_before>package tar

import (
	"archive/tar"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"time"

	"github.com/golang/glog"

	"github.com/openshift/source-to-image/pkg/errors"
	"github.com/openshift/source-to-image/pkg/util"
)

// defaultTimeout is the amount of time that the untar will wait for a tar
// stream to extract a single file. A timeout is needed to guard against broken
// connections in which it would wait for a long time to untar and nothing would happen
const defaultTimeout = 30 * time.Second

// DefaultExclusionPattern is the pattern of files that will not be included in a tar
// file when creating one. By default it is any file inside a .git metadata directory
var DefaultExclusionPattern = regexp.MustCompile("((^\\.git\\/)|(\\/.git\\/)|(\\/.git$))")

// Tar can create and extract tar files used in an STI build
type Tar interface {
	// SetExclusionPattern sets the exclusion pattern for tar
	// creation
	SetExclusionPattern(*regexp.Regexp)

	// CreateTarFile creates a tar file in the base directory
	// using the contents of dir directory
	// The name of the new tar file is returned if successful
	CreateTarFile(base, dir string) (string, error)

	// CreateTarStreamWithLogging creates a tar from the given directory
	// and streams it to the given writer.
	// An error is returned if an error occurs during streaming.
	// Archived file names are written to the logger if provided
	CreateTarStreamWithLogging(dir string, includeDirInPath bool, writer io.Writer, logger io.Writer) error

	// CreateTarStream creates a tar from the given directory
	// and streams it to the given writer.
	// An error is returned if an error occurs during streaming.
	CreateTarStream(dir string, includeDirInPath bool, writer io.Writer) error

	// ExtractTarStream extracts files from a given tar stream.
	// Times out if reading from the stream for any given file
	// exceeds the value of timeout
	ExtractTarStream(dir string, reader io.Reader) error

	// ExtractTarStreamWithLogging extracts files from a given tar stream.
	// Times out if reading from the stream for any given file
	// exceeds the value of timeout.
	// Extracted file names are written to the logger if provided.
	ExtractTarStreamWithLogging(dir string, reader io.Reader, logger io.Writer) error

	// StreamFileAsTar streams a single file as a TAR archive into specified
	// writer. The second argument is the file name in archive.
	// The file permissions in tar archive will change to 0666.
	StreamFileAsTar(string, string, io.Writer) error

	// StreamDirAsTar streams a directory as a TAR archive into specified writer.
	// The second argument is the name of the folder in the archive.
	// All files in the source folder will have permissions changed to 0666 in the
	// tar archive.
	StreamDirAsTar(string, string, io.Writer) error
}

// New creates a new Tar
func New() Tar {
	return &stiTar{
		exclude: DefaultExclusionPattern,
		timeout: defaultTimeout,
	}
}

// stiTar is an implementation of the Tar interface
type stiTar struct {
	timeout          time.Duration
	exclude          *regexp.Regexp
	includeDirInPath bool
}

// SetExclusionPattern sets the exclusion pattern for tar creation
func (t *stiTar) SetExclusionPattern(p *regexp.Regexp) {
	t.exclude = p
}

// StreamFileAsTar streams the source file as a tar archive.
// The permissions of the file is changed to 0666.
func (t *stiTar) StreamDirAsTar(source, dest string, writer io.Writer) error {
	f, err := os.Open(source)
	if err != nil {
		return err
	}
	if info, _ := f.Stat(); !info.IsDir() {
		return fmt.Errorf("the source %q has to be directory, not a file", source)
	}
	defer f.Close()
	fs := util.NewFileSystem()
	tmpDir, err := ioutil.TempDir("", "s2i-")
	if err != nil {
		return err
	}
	defer os.RemoveAll(tmpDir)
	if err = fs.Copy(source, tmpDir); err != nil {
		return err
	}
	// Skip chmod if on windows OS
	if runtime.GOOS != "windows" {
		err = filepath.Walk(tmpDir, func(path string, info os.FileInfo, err error) error {
			if err != nil {
				return err
			}
			if info.IsDir() {
				return os.Chmod(path, 0777)
			}
			return os.Chmod(path, 0666)
		})
		if err != nil {
			return err
		}
	}
	return t.CreateTarStream(tmpDir, false, writer)
}

// StreamFileAsTar streams the source file as a tar archive.
// The permissions of all files in archive is changed to 0666.
func (t *stiTar) StreamFileAsTar(source, name string, writer io.Writer) error {
	f, err := os.Open(source)
	if err != nil {
		return err
	}
	if info, _ := f.Stat(); info.IsDir() {
		return fmt.Errorf("the source %q has to be regular file, not directory", source)
	}
	defer f.Close()
	fs := util.NewFileSystem()
	tmpDir, err := ioutil.TempDir("", "s2i-")
	if err != nil {
		return err
	}
	defer os.RemoveAll(tmpDir)
	dst := filepath.Join(tmpDir, name)
	if err := fs.Copy(source, dst); err != nil {
		return err
	}
	if runtime.GOOS != "windows" {
		if err := os.Chmod(dst, 0666); err != nil {
			return err
		}
	}
	return t.CreateTarStream(tmpDir, false, writer)
}

// CreateTarFile creates a tar file from the given directory
// while excluding files that match the given exclusion pattern
// It returns the name of the created file
func (t *stiTar) CreateTarFile(base, dir string) (string, error) {
	tarFile, err := ioutil.TempFile(base, "tar")
	defer tarFile.Close()
	if err != nil {
		return "", err
	}
	if err = t.CreateTarStream(dir, false, tarFile); err != nil {
		return "", err
	}
	return tarFile.Name(), nil
}

func (t *stiTar) shouldExclude(path string) bool {
	return t.exclude != nil && t.exclude.String() != "" && t.exclude.MatchString(path)
}

// CreateTarStream calls CreateTarStreamWithLogging with a nil logger
func (t *stiTar) CreateTarStream(dir string, includeDirInPath bool, writer io.Writer) error {
	return t.CreateTarStreamWithLogging(dir, includeDirInPath, writer, nil)
}

// CreateTarStreamWithLogging creates a tar stream on the given writer from
// the given directory while excluding files that match the given
// exclusion pattern.
// TODO: this should encapsulate the goroutine that generates the stream.
func (t *stiTar) CreateTarStreamWithLogging(dir string, includeDirInPath bool, writer io.Writer, logger io.Writer) error {
	dir = filepath.Clean(dir) // remove relative paths and extraneous slashes
	tarWriter := tar.NewWriter(writer)
	defer tarWriter.Close()
	glog.V(5).Infof("Adding %q to tar ...", dir)
	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if !info.IsDir() && !t.shouldExclude(path) {
			// if file is a link just writing header info is enough
			if info.Mode()&os.ModeSymlink != 0 {
				if dir == path {
					return nil
				}
				if err = t.writeTarHeader(tarWriter, dir, path, info, includeDirInPath, logger); err != nil {
					glog.Errorf("Error writing header for %q: %v", info.Name(), err)
				}
				return err
			}

			// regular files are copied into tar, if accessible
			file, err := os.Open(path)
			if err != nil {
				glog.Errorf("Ignoring file %s: %v", path, err)
				return nil
			}
			defer file.Close()
			if err = t.writeTarHeader(tarWriter, dir, path, info, includeDirInPath, logger); err != nil {
				glog.Errorf("Error writing header for %q: %v", info.Name(), err)
				return err
			}
			if _, err = io.Copy(tarWriter, file); err != nil {
				glog.Errorf("Error copying file %q to tar: %v", path, err)
				return err
			}
		}
		return nil
	})

	if err != nil {
		glog.Errorf("Error writing tar: %v", err)
		return err
	}

	return nil
}

// writeTarHeader writes tar header for given file, returns error if operation fails
func (t *stiTar) writeTarHeader(tarWriter *tar.Writer, dir string, path string, info os.FileInfo, includeDirInPath bool, logger io.Writer) error {
	var (
		link string
		err  error
	)
	if info.Mode()&os.ModeSymlink != 0 {
		link, err = os.Readlink(path)
		if err != nil {
			return err
		}
	}
	header, err := tar.FileInfoHeader(info, link)
	if err != nil {
		return err
	}
	prefix := dir
	if includeDirInPath {
		prefix = filepath.Dir(prefix)
	}
	fileName := path
	if prefix != "." {
		fileName = path[1+len(prefix):]
	}
	header.Name = filepath.ToSlash(fileName)
	logFile(logger, header.Name)
	glog.V(5).Infof("Adding to tar: %s as %s", path, header.Name)
	if err = tarWriter.WriteHeader(header); err != nil {
		return err
	}

	return nil
}

// ExtractTarStream calls ExtractTarStreamWithLogging with a nil logger
func (t *stiTar) ExtractTarStream(dir string, reader io.Reader) error {
	return t.ExtractTarStreamWithLogging(dir, reader, nil)
}

// ExtractTarStreamWithLogging extracts files from a given tar stream.
// Times out if reading from the stream for any given file
// exceeds the value of timeout
func (t *stiTar) ExtractTarStreamWithLogging(dir string, reader io.Reader, logger io.Writer) error {
	tarReader := tar.NewReader(reader)
	errorChannel := make(chan error)
	timeout := t.timeout
	timeoutTimer := time.NewTimer(timeout)
	go func() {
		for {
			header, err := tarReader.Next()
			timeoutTimer.Reset(timeout)
			if err == io.EOF {
				errorChannel <- nil
				break
			}
			if err != nil {
				glog.Errorf("Error reading next tar header: %v", err)
				errorChannel <- err
				break
			}
			if header.FileInfo().IsDir() {
				dirPath := filepath.Join(dir, header.Name)
				glog.V(3).Infof("Creating directory %s", dirPath)
				if err = os.MkdirAll(dirPath, 0700); err != nil {
					glog.Errorf("Error creating dir %q: %v", dirPath, err)
					errorChannel <- err
					break
				}
			} else {
				fileDir := filepath.Dir(header.Name)
				dirPath := filepath.Join(dir, fileDir)
				glog.V(3).Infof("Creating directory %s", dirPath)
				if err = os.MkdirAll(dirPath, 0700); err != nil {
					glog.Errorf("Error creating dir %q: %v", dirPath, err)
					errorChannel <- err
					break
				}
				if header.Typeflag == tar.TypeSymlink {
					if err := extractLink(dir, header, tarReader); err != nil {
						glog.Errorf("Error extracting link %q: %v", header.Name, err)
						errorChannel <- err
						break
					}
					continue
				}
				logFile(logger, header.Name)
				if err := extractFile(dir, header, tarReader); err != nil {
					glog.Errorf("Error extracting file %q: %v", header.Name, err)
					errorChannel <- err
					break
				}
			}
		}
	}()

	for {
		select {
		case err := <-errorChannel:
			if err != nil {
				glog.Error("Error extracting tar stream")
			} else {
				glog.V(2).Info("Done extracting tar stream")
			}
			return err
		case <-timeoutTimer.C:
			return errors.NewTarTimeoutError()
		}
	}
}

func extractLink(dir string, header *tar.Header, tarReader io.Reader) error {
	dest := filepath.Join(dir, header.Name)
	source := header.Linkname

	glog.V(3).Infof("Creating symbolic link from %q to %q", dest, source)

	// TODO: set mtime for symlink (unfortunately we can't use os.Chtimes() and probably should use syscall)
	return os.Symlink(source, dest)
}

func extractFile(dir string, header *tar.Header, tarReader io.Reader) error {
	path := filepath.Join(dir, header.Name)
	glog.V(3).Infof("Creating %s", path)

	file, err := os.Create(path)
	if err != nil {
		return err
	}
	// The file times need to be modified after it's been closed thus this function
	// is deferred after the file close (LIFO order for defer)
	defer os.Chtimes(path, time.Now(), header.FileInfo().ModTime())
	defer file.Close()
	glog.V(3).Infof("Extracting/writing %s", path)
	written, err := io.Copy(file, tarReader)
	if err != nil {
		return err
	}
	if written != header.Size {
		return fmt.Errorf("Wrote %d bytes, expected to write %d", written, header.Size)
	}
	if runtime.GOOS != "windows" { // Skip chmod if on windows OS
		return file.Chmod(header.FileInfo().Mode())
	}
	return nil
}

func logFile(logger io.Writer, name string) {
	if logger == nil {
		return
	}
	fmt.Fprintf(logger, "%s\n", name)
}
<commit_msg>tar(StreamDirAsTar,StreamFileAsTar): fix possible fd leak and handle error from Stat().<commit_after>package tar

import (
	"archive/tar"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"time"

	"github.com/golang/glog"

	"github.com/openshift/source-to-image/pkg/errors"
	"github.com/openshift/source-to-image/pkg/util"
)

// defaultTimeout is the amount of time that the untar will wait for a tar
// stream to extract a single file. A timeout is needed to guard against broken
// connections in which it would wait for a long time to untar and nothing would happen
const defaultTimeout = 30 * time.Second

// DefaultExclusionPattern is the pattern of files that will not be included in a tar
// file when creating one. By default it is any file inside a .git metadata directory
var DefaultExclusionPattern = regexp.MustCompile("((^\\.git\\/)|(\\/.git\\/)|(\\/.git$))")

// Tar can create and extract tar files used in an STI build
type Tar interface {
	// SetExclusionPattern sets the exclusion pattern for tar
	// creation
	SetExclusionPattern(*regexp.Regexp)

	// CreateTarFile creates a tar file in the base directory
	// using the contents of dir directory
	// The name of the new tar file is returned if successful
	CreateTarFile(base, dir string) (string, error)

	// CreateTarStreamWithLogging creates a tar from the given directory
	// and streams it to the given writer.
	// An error is returned if an error occurs during streaming.
	// Archived file names are written to the logger if provided
	CreateTarStreamWithLogging(dir string, includeDirInPath bool, writer io.Writer, logger io.Writer) error

	// CreateTarStream creates a tar from the given directory
	// and streams it to the given writer.
	// An error is returned if an error occurs during streaming.
	CreateTarStream(dir string, includeDirInPath bool, writer io.Writer) error

	// ExtractTarStream extracts files from a given tar stream.
	// Times out if reading from the stream for any given file
	// exceeds the value of timeout
	ExtractTarStream(dir string, reader io.Reader) error

	// ExtractTarStreamWithLogging extracts files from a given tar stream.
	// Times out if reading from the stream for any given file
	// exceeds the value of timeout.
	// Extracted file names are written to the logger if provided.
	ExtractTarStreamWithLogging(dir string, reader io.Reader, logger io.Writer) error

	// StreamFileAsTar streams a single file as a TAR archive into specified
	// writer. The second argument is the file name in archive.
	// The file permissions in tar archive will change to 0666.
	StreamFileAsTar(string, string, io.Writer) error

	// StreamDirAsTar streams a directory as a TAR archive into specified writer.
	// The second argument is the name of the folder in the archive.
	// All files in the source folder will have permissions changed to 0666 in the
	// tar archive.
	StreamDirAsTar(string, string, io.Writer) error
}

// New creates a new Tar
func New() Tar {
	return &stiTar{
		exclude: DefaultExclusionPattern,
		timeout: defaultTimeout,
	}
}

// stiTar is an implementation of the Tar interface
type stiTar struct {
	timeout          time.Duration
	exclude          *regexp.Regexp
	includeDirInPath bool
}

// SetExclusionPattern sets the exclusion pattern for tar creation
func (t *stiTar) SetExclusionPattern(p *regexp.Regexp) {
	t.exclude = p
}

// StreamFileAsTar streams the source directory as a tar archive.
// The permissions of the file is changed to 0666.
func (t *stiTar) StreamDirAsTar(source, dest string, writer io.Writer) error {
	f, err := os.Open(source)
	if err != nil {
		return err
	}
	defer f.Close()
	info, err := f.Stat()
	if err != nil {
		return err
	}
	if !info.IsDir() {
		return fmt.Errorf("the source %q has to be directory, not a file", source)
	}
	fs := util.NewFileSystem()
	tmpDir, err := ioutil.TempDir("", "s2i-")
	if err != nil {
		return err
	}
	defer os.RemoveAll(tmpDir)
	if err = fs.Copy(source, tmpDir); err != nil {
		return err
	}
	// Skip chmod if on windows OS
	if runtime.GOOS != "windows" {
		err = filepath.Walk(tmpDir, func(path string, info os.FileInfo, err error) error {
			if err != nil {
				return err
			}
			if info.IsDir() {
				return os.Chmod(path, 0777)
			}
			return os.Chmod(path, 0666)
		})
		if err != nil {
			return err
		}
	}
	return t.CreateTarStream(tmpDir, false, writer)
}

// StreamFileAsTar streams the source file as a tar archive.
// The permissions of all files in archive is changed to 0666.
func (t *stiTar) StreamFileAsTar(source, name string, writer io.Writer) error {
	f, err := os.Open(source)
	if err != nil {
		return err
	}
	defer f.Close()
	info, err := f.Stat()
	if err != nil {
		return err
	}
	if info.IsDir() {
		return fmt.Errorf("the source %q has to be regular file, not directory", source)
	}
	fs := util.NewFileSystem()
	tmpDir, err := ioutil.TempDir("", "s2i-")
	if err != nil {
		return err
	}
	defer os.RemoveAll(tmpDir)
	dst := filepath.Join(tmpDir, name)
	if err := fs.Copy(source, dst); err != nil {
		return err
	}
	if runtime.GOOS != "windows" {
		if err := os.Chmod(dst, 0666); err != nil {
			return err
		}
	}
	return t.CreateTarStream(tmpDir, false, writer)
}

// CreateTarFile creates a tar file from the given directory
// while excluding files that match the given exclusion pattern
// It returns the name of the created file
func (t *stiTar) CreateTarFile(base, dir string) (string, error) {
	tarFile, err := ioutil.TempFile(base, "tar")
	defer tarFile.Close()
	if err != nil {
		return "", err
	}
	if err = t.CreateTarStream(dir, false, tarFile); err != nil {
		return "", err
	}
	return tarFile.Name(), nil
}

func (t *stiTar) shouldExclude(path string) bool {
	return t.exclude != nil && t.exclude.String() != "" && t.exclude.MatchString(path)
}

// CreateTarStream calls CreateTarStreamWithLogging with a nil logger
func (t *stiTar) CreateTarStream(dir string, includeDirInPath bool, writer io.Writer) error {
	return t.CreateTarStreamWithLogging(dir, includeDirInPath, writer, nil)
}

// CreateTarStreamWithLogging creates a tar stream on the given writer from
// the given directory while excluding files that match the given
// exclusion pattern.
// TODO: this should encapsulate the goroutine that generates the stream.
func (t *stiTar) CreateTarStreamWithLogging(dir string, includeDirInPath bool, writer io.Writer, logger io.Writer) error {
	dir = filepath.Clean(dir) // remove relative paths and extraneous slashes
	tarWriter := tar.NewWriter(writer)
	defer tarWriter.Close()
	glog.V(5).Infof("Adding %q to tar ...", dir)
	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if !info.IsDir() && !t.shouldExclude(path) {
			// if file is a link just writing header info is enough
			if info.Mode()&os.ModeSymlink != 0 {
				if dir == path {
					return nil
				}
				if err = t.writeTarHeader(tarWriter, dir, path, info, includeDirInPath, logger); err != nil {
					glog.Errorf("Error writing header for %q: %v", info.Name(), err)
				}
				return err
			}

			// regular files are copied into tar, if accessible
			file, err := os.Open(path)
			if err != nil {
				glog.Errorf("Ignoring file %s: %v", path, err)
				return nil
			}
			defer file.Close()
			if err = t.writeTarHeader(tarWriter, dir, path, info, includeDirInPath, logger); err != nil {
				glog.Errorf("Error writing header for %q: %v", info.Name(), err)
				return err
			}
			if _, err = io.Copy(tarWriter, file); err != nil {
				glog.Errorf("Error copying file %q to tar: %v", path, err)
				return err
			}
		}
		return nil
	})

	if err != nil {
		glog.Errorf("Error writing tar: %v", err)
		return err
	}

	return nil
}

// writeTarHeader writes tar header for given file, returns error if operation fails
func (t *stiTar) writeTarHeader(tarWriter *tar.Writer, dir string, path string, info os.FileInfo, includeDirInPath bool, logger io.Writer) error {
	var (
		link string
		err  error
	)
	if info.Mode()&os.ModeSymlink != 0 {
		link, err = os.Readlink(path)
		if err != nil {
			return err
		}
	}
	header, err := tar.FileInfoHeader(info, link)
	if err != nil {
		return err
	}
	prefix := dir
	if includeDirInPath {
		prefix = filepath.Dir(prefix)
	}
	fileName := path
	if prefix != "." {
		fileName = path[1+len(prefix):]
	}
	header.Name = filepath.ToSlash(fileName)
	logFile(logger, header.Name)
	glog.V(5).Infof("Adding to tar: %s as %s", path, header.Name)
	if err = tarWriter.WriteHeader(header); err != nil {
		return err
	}

	return nil
}

// ExtractTarStream calls ExtractTarStreamWithLogging with a nil logger
func (t *stiTar) ExtractTarStream(dir string, reader io.Reader) error {
	return t.ExtractTarStreamWithLogging(dir, reader, nil)
}

// ExtractTarStreamWithLogging extracts files from a given tar stream.
// Times out if reading from the stream for any given file
// exceeds the value of timeout
func (t *stiTar) ExtractTarStreamWithLogging(dir string, reader io.Reader, logger io.Writer) error {
	tarReader := tar.NewReader(reader)
	errorChannel := make(chan error)
	timeout := t.timeout
	timeoutTimer := time.NewTimer(timeout)
	go func() {
		for {
			header, err := tarReader.Next()
			timeoutTimer.Reset(timeout)
			if err == io.EOF {
				errorChannel <- nil
				break
			}
			if err != nil {
				glog.Errorf("Error reading next tar header: %v", err)
				errorChannel <- err
				break
			}
			if header.FileInfo().IsDir() {
				dirPath := filepath.Join(dir, header.Name)
				glog.V(3).Infof("Creating directory %s", dirPath)
				if err = os.MkdirAll(dirPath, 0700); err != nil {
					glog.Errorf("Error creating dir %q: %v", dirPath, err)
					errorChannel <- err
					break
				}
			} else {
				fileDir := filepath.Dir(header.Name)
				dirPath := filepath.Join(dir, fileDir)
				glog.V(3).Infof("Creating directory %s", dirPath)
				if err = os.MkdirAll(dirPath, 0700); err != nil {
					glog.Errorf("Error creating dir %q: %v", dirPath, err)
					errorChannel <- err
					break
				}
				if header.Typeflag == tar.TypeSymlink {
					if err := extractLink(dir, header, tarReader); err != nil {
						glog.Errorf("Error extracting link %q: %v", header.Name, err)
						errorChannel <- err
						break
					}
					continue
				}
				logFile(logger, header.Name)
				if err := extractFile(dir, header, tarReader); err != nil {
					glog.Errorf("Error extracting file %q: %v", header.Name, err)
					errorChannel <- err
					break
				}
			}
		}
	}()

	for {
		select {
		case err := <-errorChannel:
			if err != nil {
				glog.Error("Error extracting tar stream")
			} else {
				glog.V(2).Info("Done extracting tar stream")
			}
			return err
		case <-timeoutTimer.C:
			return errors.NewTarTimeoutError()
		}
	}
}

func extractLink(dir string, header *tar.Header, tarReader io.Reader) error {
	dest := filepath.Join(dir, header.Name)
	source := header.Linkname

	glog.V(3).Infof("Creating symbolic link from %q to %q", dest, source)

	// TODO: set mtime for symlink (unfortunately we can't use os.Chtimes() and probably should use syscall)
	return os.Symlink(source, dest)
}

func extractFile(dir string, header *tar.Header, tarReader io.Reader) error {
	path := filepath.Join(dir, header.Name)
	glog.V(3).Infof("Creating %s", path)

	file, err := os.Create(path)
	if err != nil {
		return err
	}
	// The file times need to be modified after it's been closed thus this function
	// is deferred after the file close (LIFO order for defer)
	defer os.Chtimes(path, time.Now(), header.FileInfo().ModTime())
	defer file.Close()
	glog.V(3).Infof("Extracting/writing %s", path)
	written, err := io.Copy(file, tarReader)
	if err != nil {
		return err
	}
	if written != header.Size {
		return fmt.Errorf("Wrote %d bytes, expected to write %d", written, header.Size)
	}
	if runtime.GOOS != "windows" { // Skip chmod if on windows OS
		return file.Chmod(header.FileInfo().Mode())
	}
	return nil
}

func logFile(logger io.Writer, name string) {
	if logger == nil {
		return
	}
	fmt.Fprintf(logger, "%s\n", name)
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package dns

const maxPointer = 2 << 13 // We have 14 bits for the offset

// Function defined in this subpackage work on []byte and but still
// provide some higher level functions.

// RawSetId sets the message ID in buf.
func RawSetId(msg []byte, off int, id uint16) bool {
	msg[off], msg[off+1] = packUint16(id)
	return true
}

type rawlabel struct {
	offset int    // offset where this labels starts in the msg buf
	name   string // the label, not this includes the length at the start
}

type rawmove struct {
	offset int // where to point to
	from   int // where in the buffer set the pointer
	length int // used in calculating how much to shrink the message
}

// Compress performs name comression in the dns message contained in buf.
// It returns the shortened buffer.
func Compress(msg []byte) []byte {

	// First we create a table of domain names to which we
	// can link. This is stored in 'table'
	// Once, for another name, the longest possible link is
	// found we save how we must change msg to perform this
	// compression. This is saved in 'moves'. After we
	// traversed the entire message, we perform all the
	// moves.
	// TODO: Maybe it should be optimized.
	// TODO: put this in the packRR function

	// Map the labels to the offset in the message
	table := make(map[string]int)
	moves := make([]rawmove, 0)
	l := make([]rawlabel, 127) // Max labels?
	i := 0

	// Very much like the loop in msg.go
	off := 12 // Start of the first name in the q section
	question := true
Loop:
	for {
		c := int(msg[off])
		switch c & 0xC0 {
		case 0x00:
			if c == 0x00 {
				// Do all of the bookkeeping

				name := ""
				poffset := 0 // Where to point to
				moffset := 0 // From where to insert the pointer
				for j := i - 1; j >= 0; j-- {
					name = l[j].name + name
					if idx, ok := table[name]; !ok {
						table[name] = l[j].offset
					} else {
						poffset = idx
						moffset = l[j].offset
					}
				}
				if poffset != 0 {
					//println("We kunnen verwijzen naar", poffset, "vanaf", moffset, "voor", name)
					//println("met lengte", len(name)+1)
					// the +1 for the name is for the null byte at the end

					// Discount for previous moves, decrease the poffset counter
					// with moves that occur before it
					for i := len(moves) - 1; i >= 0; i-- {
						if poffset > moves[i].from {
							poffset -= (moves[i].length - 2)
						}
					}
					// Only need to shorten the RR when I'm shortening the rdata
					moves = append(moves, rawmove{offset: poffset, from: moffset, length: len(name) + 1})
				}

				// end of the name
				if question {
					// In question section
					off += 4 // type, class + 1
					question = false
				} else {
					// How to handle well known records here
					// NS, MX, CNAME? eatName() function?
					// In the "body" of the msg
					off += 2 + 2 + 4 + 1 // type, class, ttl + 1
					// we are at the rdlength
					rdlength, _ := unpackUint16(msg, off)
					off += int(rdlength) + 1 // Skip the rdata
				}
				off++
				if off+1 > len(msg) {
					break Loop
				}
				i = 0
				continue Loop
			}

			if off+c+1 > len(msg) {
				break Loop
			}
			// If we are too deep in the message we cannot point to
			// it, so skip this label.
			if off < maxPointer {
				l[i] = rawlabel{offset: off, name: string(msg[off : off+c+1])}
				i++
			}
			// save the new names
			off += c + 1
		default:
			break Loop
		}
	}
	//	fmt.Printf("table %v\n", table)
	//	fmt.Printf("moves %v\n", moves)

	saved := 0
	// Start at the back, easier to move
	for i := len(moves) - 1; i >= 0; i-- {
		//	fmt.Printf("%v\n", moves[i])
		// move the bytes
		copy(msg[moves[i].from+1:], msg[moves[i].from+moves[i].length-1:])
		// Now set the pointer at moves[i].from and moves[i].from+1
		//	fmt.Printf("bits %b\n", moves[i].offset^0xC000)
		msg[moves[i].from], msg[moves[i].from+1] = packUint16(uint16(moves[i].offset ^ 0xC000))
		saved += (moves[i].length - 2) // minus something
	}
	msg = msg[:len(msg)-saved]
	return msg
}
<commit_msg>Some tweaks<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package dns

const maxPointer = 2 << 13 // We have 14 bits for the offset

// Function defined in this subpackage work on []byte and but still
// provide some higher level functions.

// RawSetId sets the message ID in buf.
func RawSetId(msg []byte, off int, id uint16) bool {
	msg[off], msg[off+1] = packUint16(id)
	return true
}

type rawlabel struct {
	offset int    // offset where this labels starts in the msg buf
	name   string // the label, not this includes the length at the start
}

type rawmove struct {
	offset int // where to point to
	from   int // where in the buffer set the pointer
	length int // used in calculating how much to shrink the message
}

// Compress performs name compression in the dns message contained in buf.
// It returns the shortened buffer.
func Compress(msg []byte) []byte {

	// First we create a table of domain names to which we
	// can link. This is stored in 'table'
	// Once, for another name, the longest possible link is
	// found we save how we must change msg to perform this
	// compression. This is saved in 'moves'. After we
	// traversed the entire message, we perform all the
	// moves.
	// TODO: Maybe it should be optimized.
	// TODO: put this in the packRR function
        // TODO: only compresses owner names, not the well known RR
        // These all those defined in 1035, important ones: CNAME, MX, NS, PTR, SOA

	// Map the labels to the offset in the message
	table := make(map[string]int)
	moves := make([]rawmove, 0)
	l := make([]rawlabel, 127) // Max labels?
	i := 0

	// Very much like the loop in msg.go
	off := 12 // Start of the first name in the q section
	question := true
Loop:
	for {
		c := int(msg[off])
		switch c & 0xC0 {
		case 0x00:
			if c == 0x00 {
				// Do all of the bookkeeping

				name := ""
				poffset := 0 // Where to point to
				moffset := 0 // From where to insert the pointer
				for j := i - 1; j >= 0; j-- {
					name = l[j].name + name
					if idx, ok := table[name]; !ok {
						table[name] = l[j].offset
					} else {
						poffset = idx
						moffset = l[j].offset
					}
				}
				if poffset != 0 {
					//println("We kunnen verwijzen naar", poffset, "vanaf", moffset, "voor", name)
					//println("met lengte", len(name)+1)
					// the +1 for the name is for the null byte at the end

					// Discount for previous moves, decrease the poffset counter
					// with moves that occur before it
					for i := len(moves) - 1; i >= 0; i-- {
						if poffset > moves[i].from {
							poffset -= (moves[i].length - 2)
						}
					}
					// Only need to shorten the RR when I'm shortening the rdata
					moves = append(moves, rawmove{offset: poffset, from: moffset, length: len(name) + 1})
				}

				// end of the name
				if question {
					// In question section
					off += 4 // type, class + 1
					question = false
				} else {
					// How to handle well known records here
					// NS, MX, CNAME? eatName() function?
					// In the "body" of the msg
					off += 2 + 2 + 4 + 1 // type, class, ttl + 1
					// we are at the rdlength
					rdlength, _ := unpackUint16(msg, off)
					off += int(rdlength) + 1 // Skip the rdata
				}
				off++
				if off+1 > len(msg) {
					break Loop
				}
				i = 0
				continue Loop
			}

			if off+c+1 > len(msg) {
				break Loop
			}
			// If we are too deep in the message we cannot point to
			// it, otherwise we can just save it.
			if off < maxPointer {
				l[i] = rawlabel{offset: off, name: string(msg[off : off+c+1])}
				i++
			}
			// save the new names
			off += c + 1
		default:
			break Loop
		}
	}
	//	fmt.Printf("table %v\n", table)
	//	fmt.Printf("moves %v\n", moves)

	saved := 0
	// Start at the back, easier to move
	for i := len(moves) - 1; i >= 0; i-- {
		//	fmt.Printf("%v\n", moves[i])
		// move the bytes
		copy(msg[moves[i].from+1:], msg[moves[i].from+moves[i].length-1:])
		// Now set the pointer at moves[i].from and moves[i].from+1
		//	fmt.Printf("bits %b\n", moves[i].offset^0xC000)
		msg[moves[i].from], msg[moves[i].from+1] = packUint16(uint16(moves[i].offset ^ 0xC000))
		saved += (moves[i].length - 2) // minus something
	}
	msg = msg[:len(msg)-saved]
	return msg
}
<|endoftext|>


<commit_before>// Sync files and directories to and from local and remote object stores
//
// Nick Craig-Wood <nick@craig-wood.com>
package main

import (
	"fmt"
	"log"
	"os"
	"runtime"
	"runtime/pprof"
	"strings"
	"time"

	"github.com/ogier/pflag"

	"github.com/ncw/rclone/fs"
	// Active file systems
	_ "github.com/ncw/rclone/drive"
	_ "github.com/ncw/rclone/dropbox"
	_ "github.com/ncw/rclone/googlecloudstorage"
	_ "github.com/ncw/rclone/local"
	_ "github.com/ncw/rclone/s3"
	_ "github.com/ncw/rclone/swift"
)

// Globals
var (
	// Flags
	cpuprofile    = pflag.StringP("cpuprofile", "", "", "Write cpu profile to file")
	statsInterval = pflag.DurationP("stats", "", time.Minute*1, "Interval to print stats")
	version       = pflag.BoolP("version", "V", false, "Print the version number")
)

type Command struct {
	Name     string
	Help     string
	ArgsHelp string
	Run      func(fdst, fsrc fs.Fs)
	MinArgs  int
	MaxArgs  int
	NoStats  bool
}

// checkArgs checks there are enough arguments and prints a message if not
func (cmd *Command) checkArgs(args []string) {
	if len(args) < cmd.MinArgs {
		syntaxError()
		fmt.Fprintf(os.Stderr, "Command %s needs %d arguments mininum\n", cmd.Name, cmd.MinArgs)
		os.Exit(1)
	} else if len(args) > cmd.MaxArgs {
		syntaxError()
		fmt.Fprintf(os.Stderr, "Command %s needs %d arguments maximum\n", cmd.Name, cmd.MaxArgs)
		os.Exit(1)
	}
}

var Commands = []Command{
	{
		Name:     "copy",
		ArgsHelp: "source://path dest://path",
		Help: `
        Copy the source to the destination.  Doesn't transfer
        unchanged files, testing first by modification time then by
        MD5SUM.  Doesn't delete files from the destination.`,
		Run: func(fdst, fsrc fs.Fs) {
			err := fs.Sync(fdst, fsrc, false)
			if err != nil {
				log.Fatalf("Failed to copy: %v", err)
			}
		},
		MinArgs: 2,
		MaxArgs: 2,
	},
	{
		Name:     "sync",
		ArgsHelp: "source://path dest://path",
		Help: `
        Sync the source to the destination.  Doesn't transfer
        unchanged files, testing first by modification time then by
        MD5SUM.  Deletes any files that exist in source that don't
        exist in destination. Since this can cause data loss, test
        first with the --dry-run flag.`,
		Run: func(fdst, fsrc fs.Fs) {
			err := fs.Sync(fdst, fsrc, true)
			if err != nil {
				log.Fatalf("Failed to sync: %v", err)
			}
		},
		MinArgs: 2,
		MaxArgs: 2,
	},
	{
		Name:     "ls",
		ArgsHelp: "[remote://path]",
		Help: `
        List all the objects in the the path with size and path.`,
		Run: func(fdst, fsrc fs.Fs) {
			err := fs.List(fdst, os.Stdout)
			if err != nil {
				log.Fatalf("Failed to list: %v", err)
			}
		},
		MinArgs: 1,
		MaxArgs: 1,
	},
	{
		Name:     "lsd",
		ArgsHelp: "[remote://path]",
		Help: `
        List all directories/containers/buckets in the the path.`,
		Run: func(fdst, fsrc fs.Fs) {
			err := fs.ListDir(fdst, os.Stdout)
			if err != nil {
				log.Fatalf("Failed to listdir: %v", err)
			}
		},
		MinArgs: 1,
		MaxArgs: 1,
	},
	{
		Name:     "lsl",
		ArgsHelp: "[remote://path]",
		Help: `
        List all the objects in the the path with modification time, size and path.`,
		Run: func(fdst, fsrc fs.Fs) {
			err := fs.ListLong(fdst, os.Stdout)
			if err != nil {
				log.Fatalf("Failed to list long: %v", err)
			}
		},
		MinArgs: 1,
		MaxArgs: 1,
	},
	{
		Name:     "md5sum",
		ArgsHelp: "[remote://path]",
		Help: `
        Produces an md5sum file for all the objects in the path.`,
		Run: func(fdst, fsrc fs.Fs) {
			err := fs.Md5sum(fdst, os.Stdout)
			if err != nil {
				log.Fatalf("Failed to list: %v", err)
			}
		},
		MinArgs: 1,
		MaxArgs: 1,
	},
	{
		Name:     "mkdir",
		ArgsHelp: "remote://path",
		Help: `
        Make the path if it doesn't already exist`,
		Run: func(fdst, fsrc fs.Fs) {
			err := fs.Mkdir(fdst)
			if err != nil {
				log.Fatalf("Failed to mkdir: %v", err)
			}
		},
		MinArgs: 1,
		MaxArgs: 1,
	},
	{
		Name:     "rmdir",
		ArgsHelp: "remote://path",
		Help: `
        Remove the path.  Note that you can't remove a path with
        objects in it, use purge for that.`,
		Run: func(fdst, fsrc fs.Fs) {
			err := fs.Rmdir(fdst)
			if err != nil {
				log.Fatalf("Failed to rmdir: %v", err)
			}
		},
		MinArgs: 1,
		MaxArgs: 1,
	},
	{
		Name:     "purge",
		ArgsHelp: "remote://path",
		Help: `
        Remove the path and all of its contents.`,
		Run: func(fdst, fsrc fs.Fs) {
			err := fs.Purge(fdst)
			if err != nil {
				log.Fatalf("Failed to purge: %v", err)
			}
		},
		MinArgs: 1,
		MaxArgs: 1,
	},
	{
		Name:     "check",
		ArgsHelp: "source://path dest://path",
		Help: `
        Checks the files in the source and destination match.  It
        compares sizes and MD5SUMs and prints a report of files which
        don't match.  It doesn't alter the source or destination.`,
		Run: func(fdst, fsrc fs.Fs) {
			err := fs.Check(fdst, fsrc)
			if err != nil {
				log.Fatalf("Failed to check: %v", err)
			}
		},
		MinArgs: 2,
		MaxArgs: 2,
	},
	{
		Name: "config",
		Help: `
        Enter an interactive configuration session.`,
		Run: func(fdst, fsrc fs.Fs) {
			fs.EditConfig()
		},
		NoStats: true,
	},
	{
		Name: "help",
		Help: `
        This help.`,
		NoStats: true,
	},
}

// syntaxError prints the syntax
func syntaxError() {
	fmt.Fprintf(os.Stderr, `Sync files and directories to and from local and remote object stores - %s.

Syntax: [options] subcommand <parameters> <parameters...>

Subcommands:

`, fs.Version)
	for i := range Commands {
		cmd := &Commands[i]
		fmt.Fprintf(os.Stderr, "    %s %s\n", cmd.Name, cmd.ArgsHelp)
		fmt.Fprintf(os.Stderr, "%s\n\n", cmd.Help)
	}

	fmt.Fprintf(os.Stderr, "Options:\n")
	pflag.PrintDefaults()
	fmt.Fprintf(os.Stderr, `
It is only necessary to use a unique prefix of the subcommand, eg 'up' for 'upload'.
`)
}

// Exit with the message
func fatal(message string, args ...interface{}) {
	syntaxError()
	fmt.Fprintf(os.Stderr, message, args...)
	os.Exit(1)
}

// Parse the command line flags
func ParseFlags() {
	pflag.Usage = syntaxError
	pflag.Parse()
	runtime.GOMAXPROCS(runtime.NumCPU())
	fs.LoadConfig()

	// Setup profiling if desired
	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			fs.Stats.Error()
			log.Fatal(err)
		}
		err = pprof.StartCPUProfile(f)
		if err != nil {
			fs.Stats.Error()
			log.Fatal(err)
		}
		defer pprof.StopCPUProfile()
	}
}

// Parse the command from the command line
func ParseCommand() (*Command, []string) {
	args := pflag.Args()
	if len(args) < 1 {
		fatal("No command supplied\n")
	}

	cmd := strings.ToLower(args[0])
	args = args[1:]

	// Find the command doing a prefix match
	var command *Command
	for i := range Commands {
		trialCommand := &Commands[i]
		// exact command name found - use that
		if trialCommand.Name == cmd {
			command = trialCommand
			break
		} else if strings.HasPrefix(trialCommand.Name, cmd) {
			if command != nil {
				fs.Stats.Error()
				log.Fatalf("Not unique - matches multiple commands %q", cmd)
			}
			command = trialCommand
		}
	}
	if command == nil {
		fs.Stats.Error()
		log.Fatalf("Unknown command %q", cmd)
	}
	if command.Run == nil {
		syntaxError()
	}
	command.checkArgs(args)
	return command, args
}

// Create a Fs from a name
func NewFs(remote string) fs.Fs {
	f, err := fs.NewFs(remote)
	if err != nil {
		fs.Stats.Error()
		log.Fatalf("Failed to create file system for %q: %v", remote, err)
	}
	return f
}

// Print the stats every statsInterval
func StartStats() {
	go func() {
		ch := time.Tick(*statsInterval)
		for {
			<-ch
			fs.Stats.Log()
		}
	}()
}

func main() {
	ParseFlags()
	if *version {
		fmt.Printf("rclone %s\n", fs.Version)
		os.Exit(0)
	}
	command, args := ParseCommand()

	// Make source and destination fs
	var fdst, fsrc fs.Fs
	if len(args) >= 1 {
		fdst = NewFs(args[0])
	}
	if len(args) >= 2 {
		fsrc = fdst
		fdst = NewFs(args[1])
	}

	fs.CalculateModifyWindow(fdst, fsrc)

	if !command.NoStats {
		StartStats()
	}

	// Run the actual command
	if command.Run != nil {
		command.Run(fdst, fsrc)
		if !command.NoStats {
			fmt.Println(fs.Stats)
		}
		if fs.Config.Verbose {
			fs.Debug(nil, "Go routines at exit %d\n", runtime.NumGoroutine())
		}
		if fs.Stats.Errored() {
			os.Exit(1)
		}
		os.Exit(0)
	} else {
	}

}
<commit_msg>Make it possible to disable stats printing with --stats=0<commit_after>// Sync files and directories to and from local and remote object stores
//
// Nick Craig-Wood <nick@craig-wood.com>
package main

import (
	"fmt"
	"log"
	"os"
	"runtime"
	"runtime/pprof"
	"strings"
	"time"

	"github.com/ogier/pflag"

	"github.com/ncw/rclone/fs"
	// Active file systems
	_ "github.com/ncw/rclone/drive"
	_ "github.com/ncw/rclone/dropbox"
	_ "github.com/ncw/rclone/googlecloudstorage"
	_ "github.com/ncw/rclone/local"
	_ "github.com/ncw/rclone/s3"
	_ "github.com/ncw/rclone/swift"
)

// Globals
var (
	// Flags
	cpuprofile    = pflag.StringP("cpuprofile", "", "", "Write cpu profile to file")
	statsInterval = pflag.DurationP("stats", "", time.Minute*1, "Interval to print stats (0 to disable)")
	version       = pflag.BoolP("version", "V", false, "Print the version number")
)

type Command struct {
	Name     string
	Help     string
	ArgsHelp string
	Run      func(fdst, fsrc fs.Fs)
	MinArgs  int
	MaxArgs  int
	NoStats  bool
}

// checkArgs checks there are enough arguments and prints a message if not
func (cmd *Command) checkArgs(args []string) {
	if len(args) < cmd.MinArgs {
		syntaxError()
		fmt.Fprintf(os.Stderr, "Command %s needs %d arguments mininum\n", cmd.Name, cmd.MinArgs)
		os.Exit(1)
	} else if len(args) > cmd.MaxArgs {
		syntaxError()
		fmt.Fprintf(os.Stderr, "Command %s needs %d arguments maximum\n", cmd.Name, cmd.MaxArgs)
		os.Exit(1)
	}
}

var Commands = []Command{
	{
		Name:     "copy",
		ArgsHelp: "source://path dest://path",
		Help: `
        Copy the source to the destination.  Doesn't transfer
        unchanged files, testing first by modification time then by
        MD5SUM.  Doesn't delete files from the destination.`,
		Run: func(fdst, fsrc fs.Fs) {
			err := fs.Sync(fdst, fsrc, false)
			if err != nil {
				log.Fatalf("Failed to copy: %v", err)
			}
		},
		MinArgs: 2,
		MaxArgs: 2,
	},
	{
		Name:     "sync",
		ArgsHelp: "source://path dest://path",
		Help: `
        Sync the source to the destination.  Doesn't transfer
        unchanged files, testing first by modification time then by
        MD5SUM.  Deletes any files that exist in source that don't
        exist in destination. Since this can cause data loss, test
        first with the --dry-run flag.`,
		Run: func(fdst, fsrc fs.Fs) {
			err := fs.Sync(fdst, fsrc, true)
			if err != nil {
				log.Fatalf("Failed to sync: %v", err)
			}
		},
		MinArgs: 2,
		MaxArgs: 2,
	},
	{
		Name:     "ls",
		ArgsHelp: "[remote://path]",
		Help: `
        List all the objects in the the path with size and path.`,
		Run: func(fdst, fsrc fs.Fs) {
			err := fs.List(fdst, os.Stdout)
			if err != nil {
				log.Fatalf("Failed to list: %v", err)
			}
		},
		MinArgs: 1,
		MaxArgs: 1,
	},
	{
		Name:     "lsd",
		ArgsHelp: "[remote://path]",
		Help: `
        List all directories/containers/buckets in the the path.`,
		Run: func(fdst, fsrc fs.Fs) {
			err := fs.ListDir(fdst, os.Stdout)
			if err != nil {
				log.Fatalf("Failed to listdir: %v", err)
			}
		},
		MinArgs: 1,
		MaxArgs: 1,
	},
	{
		Name:     "lsl",
		ArgsHelp: "[remote://path]",
		Help: `
        List all the objects in the the path with modification time, size and path.`,
		Run: func(fdst, fsrc fs.Fs) {
			err := fs.ListLong(fdst, os.Stdout)
			if err != nil {
				log.Fatalf("Failed to list long: %v", err)
			}
		},
		MinArgs: 1,
		MaxArgs: 1,
	},
	{
		Name:     "md5sum",
		ArgsHelp: "[remote://path]",
		Help: `
        Produces an md5sum file for all the objects in the path.`,
		Run: func(fdst, fsrc fs.Fs) {
			err := fs.Md5sum(fdst, os.Stdout)
			if err != nil {
				log.Fatalf("Failed to list: %v", err)
			}
		},
		MinArgs: 1,
		MaxArgs: 1,
	},
	{
		Name:     "mkdir",
		ArgsHelp: "remote://path",
		Help: `
        Make the path if it doesn't already exist`,
		Run: func(fdst, fsrc fs.Fs) {
			err := fs.Mkdir(fdst)
			if err != nil {
				log.Fatalf("Failed to mkdir: %v", err)
			}
		},
		MinArgs: 1,
		MaxArgs: 1,
	},
	{
		Name:     "rmdir",
		ArgsHelp: "remote://path",
		Help: `
        Remove the path.  Note that you can't remove a path with
        objects in it, use purge for that.`,
		Run: func(fdst, fsrc fs.Fs) {
			err := fs.Rmdir(fdst)
			if err != nil {
				log.Fatalf("Failed to rmdir: %v", err)
			}
		},
		MinArgs: 1,
		MaxArgs: 1,
	},
	{
		Name:     "purge",
		ArgsHelp: "remote://path",
		Help: `
        Remove the path and all of its contents.`,
		Run: func(fdst, fsrc fs.Fs) {
			err := fs.Purge(fdst)
			if err != nil {
				log.Fatalf("Failed to purge: %v", err)
			}
		},
		MinArgs: 1,
		MaxArgs: 1,
	},
	{
		Name:     "check",
		ArgsHelp: "source://path dest://path",
		Help: `
        Checks the files in the source and destination match.  It
        compares sizes and MD5SUMs and prints a report of files which
        don't match.  It doesn't alter the source or destination.`,
		Run: func(fdst, fsrc fs.Fs) {
			err := fs.Check(fdst, fsrc)
			if err != nil {
				log.Fatalf("Failed to check: %v", err)
			}
		},
		MinArgs: 2,
		MaxArgs: 2,
	},
	{
		Name: "config",
		Help: `
        Enter an interactive configuration session.`,
		Run: func(fdst, fsrc fs.Fs) {
			fs.EditConfig()
		},
		NoStats: true,
	},
	{
		Name: "help",
		Help: `
        This help.`,
		NoStats: true,
	},
}

// syntaxError prints the syntax
func syntaxError() {
	fmt.Fprintf(os.Stderr, `Sync files and directories to and from local and remote object stores - %s.

Syntax: [options] subcommand <parameters> <parameters...>

Subcommands:

`, fs.Version)
	for i := range Commands {
		cmd := &Commands[i]
		fmt.Fprintf(os.Stderr, "    %s %s\n", cmd.Name, cmd.ArgsHelp)
		fmt.Fprintf(os.Stderr, "%s\n\n", cmd.Help)
	}

	fmt.Fprintf(os.Stderr, "Options:\n")
	pflag.PrintDefaults()
	fmt.Fprintf(os.Stderr, `
It is only necessary to use a unique prefix of the subcommand, eg 'up' for 'upload'.
`)
}

// Exit with the message
func fatal(message string, args ...interface{}) {
	syntaxError()
	fmt.Fprintf(os.Stderr, message, args...)
	os.Exit(1)
}

// Parse the command line flags
func ParseFlags() {
	pflag.Usage = syntaxError
	pflag.Parse()
	runtime.GOMAXPROCS(runtime.NumCPU())
	fs.LoadConfig()

	// Setup profiling if desired
	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			fs.Stats.Error()
			log.Fatal(err)
		}
		err = pprof.StartCPUProfile(f)
		if err != nil {
			fs.Stats.Error()
			log.Fatal(err)
		}
		defer pprof.StopCPUProfile()
	}
}

// Parse the command from the command line
func ParseCommand() (*Command, []string) {
	args := pflag.Args()
	if len(args) < 1 {
		fatal("No command supplied\n")
	}

	cmd := strings.ToLower(args[0])
	args = args[1:]

	// Find the command doing a prefix match
	var command *Command
	for i := range Commands {
		trialCommand := &Commands[i]
		// exact command name found - use that
		if trialCommand.Name == cmd {
			command = trialCommand
			break
		} else if strings.HasPrefix(trialCommand.Name, cmd) {
			if command != nil {
				fs.Stats.Error()
				log.Fatalf("Not unique - matches multiple commands %q", cmd)
			}
			command = trialCommand
		}
	}
	if command == nil {
		fs.Stats.Error()
		log.Fatalf("Unknown command %q", cmd)
	}
	if command.Run == nil {
		syntaxError()
	}
	command.checkArgs(args)
	return command, args
}

// Create a Fs from a name
func NewFs(remote string) fs.Fs {
	f, err := fs.NewFs(remote)
	if err != nil {
		fs.Stats.Error()
		log.Fatalf("Failed to create file system for %q: %v", remote, err)
	}
	return f
}

// Print the stats every statsInterval
func StartStats() {
	if *statsInterval <= 0 {
		return
	}
	go func() {
		ch := time.Tick(*statsInterval)
		for {
			<-ch
			fs.Stats.Log()
		}
	}()
}

func main() {
	ParseFlags()
	if *version {
		fmt.Printf("rclone %s\n", fs.Version)
		os.Exit(0)
	}
	command, args := ParseCommand()

	// Make source and destination fs
	var fdst, fsrc fs.Fs
	if len(args) >= 1 {
		fdst = NewFs(args[0])
	}
	if len(args) >= 2 {
		fsrc = fdst
		fdst = NewFs(args[1])
	}

	fs.CalculateModifyWindow(fdst, fsrc)

	if !command.NoStats {
		StartStats()
	}

	// Run the actual command
	if command.Run != nil {
		command.Run(fdst, fsrc)
		if !command.NoStats {
			fmt.Println(fs.Stats)
		}
		if fs.Config.Verbose {
			fs.Debug(nil, "Go routines at exit %d\n", runtime.NumGoroutine())
		}
		if fs.Stats.Errored() {
			os.Exit(1)
		}
		os.Exit(0)
	} else {
	}

}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package os

import (
	"runtime"
	"syscall"
)

// Auxiliary information if the File describes a directory
type dirInfo struct {
	stat         syscall.Stat_t
	usefirststat bool
}

const DevNull = "NUL"

func (file *File) isdir() bool { return file != nil && file.dirinfo != nil }

func openFile(name string, flag int, perm uint32) (file *File, err Error) {
	r, e := syscall.Open(name, flag|syscall.O_CLOEXEC, perm)
	if e != 0 {
		return nil, &PathError{"open", name, Errno(e)}
	}

	// There's a race here with fork/exec, which we are
	// content to live with.  See ../syscall/exec.go
	if syscall.O_CLOEXEC == 0 { // O_CLOEXEC not supported
		syscall.CloseOnExec(r)
	}

	return NewFile(r, name), nil
}

func openDir(name string) (file *File, err Error) {
	d := new(dirInfo)
	r, e := syscall.FindFirstFile(syscall.StringToUTF16Ptr(name+"\\*"), &d.stat.Windata)
	if e != 0 {
		return nil, &PathError{"open", name, Errno(e)}
	}
	f := NewFile(int(r), name)
	d.usefirststat = true
	f.dirinfo = d
	return f, nil
}

// OpenFile is the generalized open call; most users will use Open
// or Create instead.  It opens the named file with specified flag
// (O_RDONLY etc.) and perm, (0666 etc.) if applicable.  If successful,
// methods on the returned File can be used for I/O.
// It returns the File and an Error, if any.
func OpenFile(name string, flag int, perm uint32) (file *File, err Error) {
	// TODO(brainman): not sure about my logic of assuming it is dir first, then fall back to file
	r, e := openDir(name)
	if e == nil {
		if flag&O_WRONLY != 0 || flag&O_RDWR != 0 {
			r.Close()
			return nil, &PathError{"open", name, EISDIR}
		}
		return r, nil
	}
	r, e = openFile(name, flag, perm)
	if e == nil {
		return r, nil
	}
	// Imitating Unix behavior by replacing syscall.ERROR_PATH_NOT_FOUND with
	// os.ENOTDIR. Not sure if we should go into that.
	if e2, ok := e.(*PathError); ok {
		if e3, ok := e2.Error.(Errno); ok {
			if e3 == Errno(syscall.ERROR_PATH_NOT_FOUND) {
				return nil, &PathError{"open", name, ENOTDIR}
			}
		}
	}
	return nil, e
}

// Close closes the File, rendering it unusable for I/O.
// It returns an Error, if any.
func (file *File) Close() Error {
	if file == nil || file.fd < 0 {
		return EINVAL
	}
	var e int
	if file.isdir() {
		e = syscall.FindClose(int32(file.fd))
	} else {
		e = syscall.CloseHandle(int32(file.fd))
	}
	var err Error
	if e != 0 {
		err = &PathError{"close", file.name, Errno(e)}
	}
	file.fd = -1 // so it can't be closed again

	// no need for a finalizer anymore
	runtime.SetFinalizer(file, nil)
	return err
}

func (file *File) statFile(name string) (fi *FileInfo, err Error) {
	var stat syscall.ByHandleFileInformation
	e := syscall.GetFileInformationByHandle(int32(file.fd), &stat)
	if e != 0 {
		return nil, &PathError{"stat", file.name, Errno(e)}
	}
	return fileInfoFromByHandleInfo(new(FileInfo), file.name, &stat), nil
}

// Stat returns the FileInfo structure describing file.
// It returns the FileInfo and an error, if any.
func (file *File) Stat() (fi *FileInfo, err Error) {
	if file == nil || file.fd < 0 {
		return nil, EINVAL
	}
	if file.isdir() {
		// I don't know any better way to do that for directory
		return Stat(file.name)
	}
	return file.statFile(file.name)
}

// Readdir reads the contents of the directory associated with file and
// returns an array of up to count FileInfo structures, as would be returned
// by Lstat, in directory order.  Subsequent calls on the same file will yield
// further FileInfos.
// A negative count means to read until EOF.
// Readdir returns the array and an Error, if any.
func (file *File) Readdir(count int) (fi []FileInfo, err Error) {
	if file == nil || file.fd < 0 {
		return nil, EINVAL
	}
	if !file.isdir() {
		return nil, &PathError{"Readdir", file.name, ENOTDIR}
	}
	di := file.dirinfo
	size := count
	if size < 0 {
		size = 100
	}
	fi = make([]FileInfo, 0, size) // Empty with room to grow.
	for count != 0 {
		if di.usefirststat {
			di.usefirststat = false
		} else {
			e := syscall.FindNextFile(int32(file.fd), &di.stat.Windata)
			if e != 0 {
				if e == syscall.ERROR_NO_MORE_FILES {
					break
				} else {
					return nil, &PathError{"FindNextFile", file.name, Errno(e)}
				}
			}
		}
		var f FileInfo
		fileInfoFromWin32finddata(&f, &di.stat.Windata)
		if f.Name == "." || f.Name == ".." { // Useless names
			continue
		}
		count--
		fi = append(fi, f)
	}
	return fi, nil
}

// Truncate changes the size of the named file.
// If the file is a symbolic link, it changes the size of the link's target.
func Truncate(name string, size int64) Error {
	f, e := Open(name, O_WRONLY|O_CREATE, 0666)
	if e != nil {
		return e
	}
	defer f.Close()
	e1 := f.Truncate(size)
	if e1 != nil {
		return e1
	}
	return nil
}
<commit_msg>os: fix windows build<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package os

import (
	"runtime"
	"syscall"
)

// Auxiliary information if the File describes a directory
type dirInfo struct {
	stat         syscall.Stat_t
	usefirststat bool
}

const DevNull = "NUL"

func (file *File) isdir() bool { return file != nil && file.dirinfo != nil }

func openFile(name string, flag int, perm uint32) (file *File, err Error) {
	r, e := syscall.Open(name, flag|syscall.O_CLOEXEC, perm)
	if e != 0 {
		return nil, &PathError{"open", name, Errno(e)}
	}

	// There's a race here with fork/exec, which we are
	// content to live with.  See ../syscall/exec.go
	if syscall.O_CLOEXEC == 0 { // O_CLOEXEC not supported
		syscall.CloseOnExec(r)
	}

	return NewFile(r, name), nil
}

func openDir(name string) (file *File, err Error) {
	d := new(dirInfo)
	r, e := syscall.FindFirstFile(syscall.StringToUTF16Ptr(name+"\\*"), &d.stat.Windata)
	if e != 0 {
		return nil, &PathError{"open", name, Errno(e)}
	}
	f := NewFile(int(r), name)
	d.usefirststat = true
	f.dirinfo = d
	return f, nil
}

// OpenFile is the generalized open call; most users will use Open
// or Create instead.  It opens the named file with specified flag
// (O_RDONLY etc.) and perm, (0666 etc.) if applicable.  If successful,
// methods on the returned File can be used for I/O.
// It returns the File and an Error, if any.
func OpenFile(name string, flag int, perm uint32) (file *File, err Error) {
	// TODO(brainman): not sure about my logic of assuming it is dir first, then fall back to file
	r, e := openDir(name)
	if e == nil {
		if flag&O_WRONLY != 0 || flag&O_RDWR != 0 {
			r.Close()
			return nil, &PathError{"open", name, EISDIR}
		}
		return r, nil
	}
	r, e = openFile(name, flag, perm)
	if e == nil {
		return r, nil
	}
	// Imitating Unix behavior by replacing syscall.ERROR_PATH_NOT_FOUND with
	// os.ENOTDIR. Not sure if we should go into that.
	if e2, ok := e.(*PathError); ok {
		if e3, ok := e2.Error.(Errno); ok {
			if e3 == Errno(syscall.ERROR_PATH_NOT_FOUND) {
				return nil, &PathError{"open", name, ENOTDIR}
			}
		}
	}
	return nil, e
}

// Close closes the File, rendering it unusable for I/O.
// It returns an Error, if any.
func (file *File) Close() Error {
	if file == nil || file.fd < 0 {
		return EINVAL
	}
	var e int
	if file.isdir() {
		e = syscall.FindClose(int32(file.fd))
	} else {
		e = syscall.CloseHandle(int32(file.fd))
	}
	var err Error
	if e != 0 {
		err = &PathError{"close", file.name, Errno(e)}
	}
	file.fd = -1 // so it can't be closed again

	// no need for a finalizer anymore
	runtime.SetFinalizer(file, nil)
	return err
}

func (file *File) statFile(name string) (fi *FileInfo, err Error) {
	var stat syscall.ByHandleFileInformation
	e := syscall.GetFileInformationByHandle(int32(file.fd), &stat)
	if e != 0 {
		return nil, &PathError{"stat", file.name, Errno(e)}
	}
	return fileInfoFromByHandleInfo(new(FileInfo), file.name, &stat), nil
}

// Stat returns the FileInfo structure describing file.
// It returns the FileInfo and an error, if any.
func (file *File) Stat() (fi *FileInfo, err Error) {
	if file == nil || file.fd < 0 {
		return nil, EINVAL
	}
	if file.isdir() {
		// I don't know any better way to do that for directory
		return Stat(file.name)
	}
	return file.statFile(file.name)
}

// Readdir reads the contents of the directory associated with file and
// returns an array of up to count FileInfo structures, as would be returned
// by Lstat, in directory order.  Subsequent calls on the same file will yield
// further FileInfos.
// A negative count means to read until EOF.
// Readdir returns the array and an Error, if any.
func (file *File) Readdir(count int) (fi []FileInfo, err Error) {
	if file == nil || file.fd < 0 {
		return nil, EINVAL
	}
	if !file.isdir() {
		return nil, &PathError{"Readdir", file.name, ENOTDIR}
	}
	di := file.dirinfo
	size := count
	if size < 0 {
		size = 100
	}
	fi = make([]FileInfo, 0, size) // Empty with room to grow.
	for count != 0 {
		if di.usefirststat {
			di.usefirststat = false
		} else {
			e := syscall.FindNextFile(int32(file.fd), &di.stat.Windata)
			if e != 0 {
				if e == syscall.ERROR_NO_MORE_FILES {
					break
				} else {
					return nil, &PathError{"FindNextFile", file.name, Errno(e)}
				}
			}
		}
		var f FileInfo
		fileInfoFromWin32finddata(&f, &di.stat.Windata)
		if f.Name == "." || f.Name == ".." { // Useless names
			continue
		}
		count--
		fi = append(fi, f)
	}
	return fi, nil
}

// Truncate changes the size of the named file.
// If the file is a symbolic link, it changes the size of the link's target.
func Truncate(name string, size int64) Error {
	f, e := OpenFile(name, O_WRONLY|O_CREATE, 0666)
	if e != nil {
		return e
	}
	defer f.Close()
	e1 := f.Truncate(size)
	if e1 != nil {
		return e1
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build !race

// Package atomic provides low-level atomic memory primitives
// useful for implementing synchronization algorithms.
//
// These functions require great care to be used correctly.
// Except for special, low-level applications, synchronization is better
// done with channels or the facilities of the sync package.
// Share memory by communicating;
// don't communicate by sharing memory.
//
// The swap operation, implemented by the SwapT functions, is the atomic
// equivalent of:
//
//	old = *addr
//	*addr = new
//	return old
//
// The compare-and-swap operation, implemented by the CompareAndSwapT
// functions, is the atomic equivalent of:
//
//	if *addr == old {
//		*addr = new
//		return true
//	}
//	return false
//
// The add operation, implemented by the AddT functions, is the atomic
// equivalent of:
//
//	*addr += delta
//	return *addr
//
// The load and store operations, implemented by the LoadT and StoreT
// functions, are the atomic equivalents of "return *addr" and
// "*addr = val".
//
package atomic

import (
	"unsafe"
)

// BUG(rsc): On x86-32, the 64-bit functions use instructions unavailable before the Pentium MMX.
//
// On both ARM and x86-32, it is the caller's responsibility to arrange for 64-bit
// alignment of 64-bit words accessed atomically. The first word in a global
// variable or in an allocated struct or slice can be relied upon to be
// 64-bit aligned.

// SwapInt32 atomically stores new into *addr and returns the previous *addr value.
func SwapInt32(addr *int32, new int32) (old int32)

// SwapInt64 atomically stores new into *addr and returns the previous *addr value.
func SwapInt64(addr *int64, new int64) (old int64)

// SwapUint32 atomically stores new into *addr and returns the previous *addr value.
func SwapUint32(addr *uint32, new uint32) (old uint32)

// SwapUint64 atomically stores new into *addr and returns the previous *addr value.
func SwapUint64(addr *uint64, new uint64) (old uint64)

// SwapUintptr atomically stores new into *addr and returns the previous *addr value.
func SwapUintptr(addr *uintptr, new uintptr) (old uintptr)

// SwapPointer atomically stores new into *addr and returns the previous *addr value.
func SwapPointer(addr *unsafe.Pointer, new unsafe.Pointer) (old unsafe.Pointer)

// CompareAndSwapInt32 executes the compare-and-swap operation for an int32 value.
func CompareAndSwapInt32(addr *int32, old, new int32) (swapped bool)

// CompareAndSwapInt64 executes the compare-and-swap operation for an int64 value.
func CompareAndSwapInt64(addr *int64, old, new int64) (swapped bool)

// CompareAndSwapUint32 executes the compare-and-swap operation for a uint32 value.
func CompareAndSwapUint32(addr *uint32, old, new uint32) (swapped bool)

// CompareAndSwapUint64 executes the compare-and-swap operation for a uint64 value.
func CompareAndSwapUint64(addr *uint64, old, new uint64) (swapped bool)

// CompareAndSwapUintptr executes the compare-and-swap operation for a uintptr value.
func CompareAndSwapUintptr(addr *uintptr, old, new uintptr) (swapped bool)

// CompareAndSwapPointer executes the compare-and-swap operation for a unsafe.Pointer value.
func CompareAndSwapPointer(addr *unsafe.Pointer, old, new unsafe.Pointer) (swapped bool)

// AddInt32 atomically adds delta to *addr and returns the new value.
func AddInt32(addr *int32, delta int32) (new int32)

// AddUint32 atomically adds delta to *addr and returns the new value.
func AddUint32(addr *uint32, delta uint32) (new uint32)

// AddInt64 atomically adds delta to *addr and returns the new value.
func AddInt64(addr *int64, delta int64) (new int64)

// AddUint64 atomically adds delta to *addr and returns the new value.
func AddUint64(addr *uint64, delta uint64) (new uint64)

// AddUintptr atomically adds delta to *addr and returns the new value.
func AddUintptr(addr *uintptr, delta uintptr) (new uintptr)

// LoadInt32 atomically loads *addr.
func LoadInt32(addr *int32) (val int32)

// LoadInt64 atomically loads *addr.
func LoadInt64(addr *int64) (val int64)

// LoadUint32 atomically loads *addr.
func LoadUint32(addr *uint32) (val uint32)

// LoadUint64 atomically loads *addr.
func LoadUint64(addr *uint64) (val uint64)

// LoadUintptr atomically loads *addr.
func LoadUintptr(addr *uintptr) (val uintptr)

// LoadPointer atomically loads *addr.
func LoadPointer(addr *unsafe.Pointer) (val unsafe.Pointer)

// StoreInt32 atomically stores val into *addr.
func StoreInt32(addr *int32, val int32)

// StoreInt64 atomically stores val into *addr.
func StoreInt64(addr *int64, val int64)

// StoreUint32 atomically stores val into *addr.
func StoreUint32(addr *uint32, val uint32)

// StoreUint64 atomically stores val into *addr.
func StoreUint64(addr *uint64, val uint64)

// StoreUintptr atomically stores val into *addr.
func StoreUintptr(addr *uintptr, val uintptr)

// StorePointer atomically stores val into *addr.
func StorePointer(addr *unsafe.Pointer, val unsafe.Pointer)

// Helper for ARM.  Linker will discard on other systems
func panic64() {
	panic("sync/atomic: broken 64-bit atomic operations (buggy QEMU)")
}
<commit_msg>sync/atomic: update comment for ARM 64-bit atomics<commit_after>// Copyright 2011 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build !race

// Package atomic provides low-level atomic memory primitives
// useful for implementing synchronization algorithms.
//
// These functions require great care to be used correctly.
// Except for special, low-level applications, synchronization is better
// done with channels or the facilities of the sync package.
// Share memory by communicating;
// don't communicate by sharing memory.
//
// The swap operation, implemented by the SwapT functions, is the atomic
// equivalent of:
//
//	old = *addr
//	*addr = new
//	return old
//
// The compare-and-swap operation, implemented by the CompareAndSwapT
// functions, is the atomic equivalent of:
//
//	if *addr == old {
//		*addr = new
//		return true
//	}
//	return false
//
// The add operation, implemented by the AddT functions, is the atomic
// equivalent of:
//
//	*addr += delta
//	return *addr
//
// The load and store operations, implemented by the LoadT and StoreT
// functions, are the atomic equivalents of "return *addr" and
// "*addr = val".
//
package atomic

import (
	"unsafe"
)

// BUG(rsc): On x86-32, the 64-bit functions use instructions unavailable before the Pentium MMX.
//
// On non-Linux ARM, the 64-bit functions use instructions unavailable before the ARMv6k core.
//
// On both ARM and x86-32, it is the caller's responsibility to arrange for 64-bit
// alignment of 64-bit words accessed atomically. The first word in a global
// variable or in an allocated struct or slice can be relied upon to be
// 64-bit aligned.

// SwapInt32 atomically stores new into *addr and returns the previous *addr value.
func SwapInt32(addr *int32, new int32) (old int32)

// SwapInt64 atomically stores new into *addr and returns the previous *addr value.
func SwapInt64(addr *int64, new int64) (old int64)

// SwapUint32 atomically stores new into *addr and returns the previous *addr value.
func SwapUint32(addr *uint32, new uint32) (old uint32)

// SwapUint64 atomically stores new into *addr and returns the previous *addr value.
func SwapUint64(addr *uint64, new uint64) (old uint64)

// SwapUintptr atomically stores new into *addr and returns the previous *addr value.
func SwapUintptr(addr *uintptr, new uintptr) (old uintptr)

// SwapPointer atomically stores new into *addr and returns the previous *addr value.
func SwapPointer(addr *unsafe.Pointer, new unsafe.Pointer) (old unsafe.Pointer)

// CompareAndSwapInt32 executes the compare-and-swap operation for an int32 value.
func CompareAndSwapInt32(addr *int32, old, new int32) (swapped bool)

// CompareAndSwapInt64 executes the compare-and-swap operation for an int64 value.
func CompareAndSwapInt64(addr *int64, old, new int64) (swapped bool)

// CompareAndSwapUint32 executes the compare-and-swap operation for a uint32 value.
func CompareAndSwapUint32(addr *uint32, old, new uint32) (swapped bool)

// CompareAndSwapUint64 executes the compare-and-swap operation for a uint64 value.
func CompareAndSwapUint64(addr *uint64, old, new uint64) (swapped bool)

// CompareAndSwapUintptr executes the compare-and-swap operation for a uintptr value.
func CompareAndSwapUintptr(addr *uintptr, old, new uintptr) (swapped bool)

// CompareAndSwapPointer executes the compare-and-swap operation for a unsafe.Pointer value.
func CompareAndSwapPointer(addr *unsafe.Pointer, old, new unsafe.Pointer) (swapped bool)

// AddInt32 atomically adds delta to *addr and returns the new value.
func AddInt32(addr *int32, delta int32) (new int32)

// AddUint32 atomically adds delta to *addr and returns the new value.
func AddUint32(addr *uint32, delta uint32) (new uint32)

// AddInt64 atomically adds delta to *addr and returns the new value.
func AddInt64(addr *int64, delta int64) (new int64)

// AddUint64 atomically adds delta to *addr and returns the new value.
func AddUint64(addr *uint64, delta uint64) (new uint64)

// AddUintptr atomically adds delta to *addr and returns the new value.
func AddUintptr(addr *uintptr, delta uintptr) (new uintptr)

// LoadInt32 atomically loads *addr.
func LoadInt32(addr *int32) (val int32)

// LoadInt64 atomically loads *addr.
func LoadInt64(addr *int64) (val int64)

// LoadUint32 atomically loads *addr.
func LoadUint32(addr *uint32) (val uint32)

// LoadUint64 atomically loads *addr.
func LoadUint64(addr *uint64) (val uint64)

// LoadUintptr atomically loads *addr.
func LoadUintptr(addr *uintptr) (val uintptr)

// LoadPointer atomically loads *addr.
func LoadPointer(addr *unsafe.Pointer) (val unsafe.Pointer)

// StoreInt32 atomically stores val into *addr.
func StoreInt32(addr *int32, val int32)

// StoreInt64 atomically stores val into *addr.
func StoreInt64(addr *int64, val int64)

// StoreUint32 atomically stores val into *addr.
func StoreUint32(addr *uint32, val uint32)

// StoreUint64 atomically stores val into *addr.
func StoreUint64(addr *uint64, val uint64)

// StoreUintptr atomically stores val into *addr.
func StoreUintptr(addr *uintptr, val uintptr)

// StorePointer atomically stores val into *addr.
func StorePointer(addr *unsafe.Pointer, val unsafe.Pointer)

// Helper for ARM.  Linker will discard on other systems
func panic64() {
	panic("sync/atomic: broken 64-bit atomic operations (buggy QEMU)")
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// The testing package provides support for automated testing of Go packages.
// It is intended to be used in concert with the ``gotest'' utility, which automates
// execution of any function of the form
//     func TestXxx(*testing.T)
// where Xxx can be any alphanumeric string (but the first letter must not be in
// [a-z]) and serves to identify the test routine.
// These TestXxx routines should be declared within the package they are testing.
//
// Functions of the form
//     func BenchmarkXxx(*testing.B)
// are considered benchmarks, and are executed by gotest when the -benchmarks
// flag is provided.
//
// A sample benchmark function looks like this:
//     func BenchmarkHello(b *testing.B) {
//         for i := 0; i < b.N; i++ {
//             fmt.Sprintf("hello")
//         }
//     }
// The benchmark package will vary b.N until the benchmark function lasts
// long enough to be timed reliably.  The output
//     testing.BenchmarkHello	500000	      4076 ns/op
// means that the loop ran 500000 times at a speed of 4076 ns per loop.
//
// If a benchmark needs some expensive setup before running, the timer
// may be stopped:
//     func BenchmarkBigLen(b *testing.B) {
//         b.StopTimer()
//         big := NewBig()
//         b.StartTimer()
//         for i := 0; i < b.N; i++ {
//             big.Len()
//         }
//     }
package testing

import (
	"flag"
	"fmt"
	"os"
	"runtime"
)

// Report as tests are run; default is silent for success.
var chatty = flag.Bool("v", false, "verbose: print additional output")
var match = flag.String("match", "", "regular expression to select tests to run")

// Insert final newline if needed and tabs after internal newlines.
func tabify(s string) string {
	n := len(s)
	if n > 0 && s[n-1] != '\n' {
		s += "\n"
		n++
	}
	for i := 0; i < n-1; i++ { // -1 to avoid final newline
		if s[i] == '\n' {
			return s[0:i+1] + "\t" + tabify(s[i+1:n])
		}
	}
	return s
}

// T is a type passed to Test functions to manage test state and support formatted test logs.
// Logs are accumulated during execution and dumped to standard error when done.
type T struct {
	errors string
	failed bool
	ch     chan *T
}

// Fail marks the Test function as having failed but continues execution.
func (t *T) Fail() { t.failed = true }

// Failed returns whether the Test function has failed.
func (t *T) Failed() bool { return t.failed }

// FailNow marks the Test function as having failed and stops its execution.
// Execution will continue at the next Test.
func (t *T) FailNow() {
	t.Fail()
	t.ch <- t
	runtime.Goexit()
}

// Log formats its arguments using default formatting, analogous to Print(),
// and records the text in the error log.
func (t *T) Log(args ...interface{}) { t.errors += "\t" + tabify(fmt.Sprintln(args...)) }

// Log formats its arguments according to the format, analogous to Printf(),
// and records the text in the error log.
func (t *T) Logf(format string, args ...interface{}) {
	t.errors += "\t" + tabify(fmt.Sprintf(format, args...))
}

// Error is equivalent to Log() followed by Fail().
func (t *T) Error(args ...interface{}) {
	t.Log(args...)
	t.Fail()
}

// Errorf is equivalent to Logf() followed by Fail().
func (t *T) Errorf(format string, args ...interface{}) {
	t.Logf(format, args...)
	t.Fail()
}

// Fatal is equivalent to Log() followed by FailNow().
func (t *T) Fatal(args ...interface{}) {
	t.Log(args...)
	t.FailNow()
}

// Fatalf is equivalent to Logf() followed by FailNow().
func (t *T) Fatalf(format string, args ...interface{}) {
	t.Logf(format, args...)
	t.FailNow()
}

// An internal type but exported because it is cross-package; part of the implementation
// of gotest.
type InternalTest struct {
	Name string
	F    func(*T)
}

func tRunner(t *T, test *InternalTest) {
	test.F(t)
	t.ch <- t
}

// An internal function but exported because it is cross-package; part of the implementation
// of gotest.
func Main(matchString func(pat, str string) (bool, os.Error), tests []InternalTest) {
	flag.Parse()
	ok := true
	if len(tests) == 0 {
		println("testing: warning: no tests to run")
	}
	for i := 0; i < len(tests); i++ {
		matched, err := matchString(*match, tests[i].Name)
		if err != nil {
			println("invalid regexp for -match:", err.String())
			os.Exit(1)
		}
		if !matched {
			continue
		}
		if *chatty {
			println("=== RUN ", tests[i].Name)
		}
		t := new(T)
		t.ch = make(chan *T)
		go tRunner(t, &tests[i])
		<-t.ch
		if t.failed {
			println("--- FAIL:", tests[i].Name)
			print(t.errors)
			ok = false
		} else if *chatty {
			println("--- PASS:", tests[i].Name)
			print(t.errors)
		}
	}
	if !ok {
		println("FAIL")
		os.Exit(1)
	}
	println("PASS")
}
<commit_msg>testing: include elapsed time in output<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// The testing package provides support for automated testing of Go packages.
// It is intended to be used in concert with the ``gotest'' utility, which automates
// execution of any function of the form
//     func TestXxx(*testing.T)
// where Xxx can be any alphanumeric string (but the first letter must not be in
// [a-z]) and serves to identify the test routine.
// These TestXxx routines should be declared within the package they are testing.
//
// Functions of the form
//     func BenchmarkXxx(*testing.B)
// are considered benchmarks, and are executed by gotest when the -benchmarks
// flag is provided.
//
// A sample benchmark function looks like this:
//     func BenchmarkHello(b *testing.B) {
//         for i := 0; i < b.N; i++ {
//             fmt.Sprintf("hello")
//         }
//     }
// The benchmark package will vary b.N until the benchmark function lasts
// long enough to be timed reliably.  The output
//     testing.BenchmarkHello	500000	      4076 ns/op
// means that the loop ran 500000 times at a speed of 4076 ns per loop.
//
// If a benchmark needs some expensive setup before running, the timer
// may be stopped:
//     func BenchmarkBigLen(b *testing.B) {
//         b.StopTimer()
//         big := NewBig()
//         b.StartTimer()
//         for i := 0; i < b.N; i++ {
//             big.Len()
//         }
//     }
package testing

import (
	"flag"
	"fmt"
	"os"
	"runtime"
	"time"
)

// Report as tests are run; default is silent for success.
var chatty = flag.Bool("v", false, "verbose: print additional output")
var match = flag.String("match", "", "regular expression to select tests to run")

// Insert final newline if needed and tabs after internal newlines.
func tabify(s string) string {
	n := len(s)
	if n > 0 && s[n-1] != '\n' {
		s += "\n"
		n++
	}
	for i := 0; i < n-1; i++ { // -1 to avoid final newline
		if s[i] == '\n' {
			return s[0:i+1] + "\t" + tabify(s[i+1:n])
		}
	}
	return s
}

// T is a type passed to Test functions to manage test state and support formatted test logs.
// Logs are accumulated during execution and dumped to standard error when done.
type T struct {
	errors string
	failed bool
	ch     chan *T
}

// Fail marks the Test function as having failed but continues execution.
func (t *T) Fail() { t.failed = true }

// Failed returns whether the Test function has failed.
func (t *T) Failed() bool { return t.failed }

// FailNow marks the Test function as having failed and stops its execution.
// Execution will continue at the next Test.
func (t *T) FailNow() {
	t.Fail()
	t.ch <- t
	runtime.Goexit()
}

// Log formats its arguments using default formatting, analogous to Print(),
// and records the text in the error log.
func (t *T) Log(args ...interface{}) { t.errors += "\t" + tabify(fmt.Sprintln(args...)) }

// Log formats its arguments according to the format, analogous to Printf(),
// and records the text in the error log.
func (t *T) Logf(format string, args ...interface{}) {
	t.errors += "\t" + tabify(fmt.Sprintf(format, args...))
}

// Error is equivalent to Log() followed by Fail().
func (t *T) Error(args ...interface{}) {
	t.Log(args...)
	t.Fail()
}

// Errorf is equivalent to Logf() followed by Fail().
func (t *T) Errorf(format string, args ...interface{}) {
	t.Logf(format, args...)
	t.Fail()
}

// Fatal is equivalent to Log() followed by FailNow().
func (t *T) Fatal(args ...interface{}) {
	t.Log(args...)
	t.FailNow()
}

// Fatalf is equivalent to Logf() followed by FailNow().
func (t *T) Fatalf(format string, args ...interface{}) {
	t.Logf(format, args...)
	t.FailNow()
}

// An internal type but exported because it is cross-package; part of the implementation
// of gotest.
type InternalTest struct {
	Name string
	F    func(*T)
}

func tRunner(t *T, test *InternalTest) {
	test.F(t)
	t.ch <- t
}

// An internal function but exported because it is cross-package; part of the implementation
// of gotest.
func Main(matchString func(pat, str string) (bool, os.Error), tests []InternalTest) {
	flag.Parse()
	ok := true
	if len(tests) == 0 {
		println("testing: warning: no tests to run")
	}
	for i := 0; i < len(tests); i++ {
		matched, err := matchString(*match, tests[i].Name)
		if err != nil {
			println("invalid regexp for -match:", err.String())
			os.Exit(1)
		}
		if !matched {
			continue
		}
		if *chatty {
			println("=== RUN ", tests[i].Name)
		}
		ns := -time.Nanoseconds()
		t := new(T)
		t.ch = make(chan *T)
		go tRunner(t, &tests[i])
		<-t.ch
		ns += time.Nanoseconds()
		tstr := fmt.Sprintf("(%.1f seconds)", float64(ns)/1e9)
		if t.failed {
			println("--- FAIL:", tests[i].Name, tstr)
			print(t.errors)
			ok = false
		} else if *chatty {
			println("--- PASS:", tests[i].Name, tstr)
			print(t.errors)
		}
	}
	if !ok {
		println("FAIL")
		os.Exit(1)
	}
	println("PASS")
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package time_test

import (
	"os"
	"syscall"
	"testing"
	"sort"
	. "time"
)

func TestSleep(t *testing.T) {
	const delay = int64(100e6)
	go func() {
		Sleep(delay / 2)
		syscall.Kill(os.Getpid(), syscall.SIGCHLD)
	}()
	start := Nanoseconds()
	Sleep(delay)
	duration := Nanoseconds() - start
	if duration < delay {
		t.Fatalf("Sleep(%d) slept for only %d ns", delay, duration)
	}
}

// Test the basic function calling behavior. Correct queueing
// behavior is tested elsewhere, since After and AfterFunc share
// the same code.
func TestAfterFunc(t *testing.T) {
	i := 10
	c := make(chan bool)
	var f func()
	f = func() {
		i--
		if i >= 0 {
			AfterFunc(0, f)
			Sleep(1e9)
		} else {
			c <- true
		}
	}

	AfterFunc(0, f)
	<-c
}

func BenchmarkAfterFunc(b *testing.B) {
	i := b.N
	c := make(chan bool)
	var f func()
	f = func() {
		i--
		if i >= 0 {
			AfterFunc(0, f)
		} else {
			c <- true
		}
	}

	AfterFunc(0, f)
	<-c
}

func BenchmarkAfter(b *testing.B) {
	for i := 0; i < b.N; i++ {
		<-After(1)
	}
}

func BenchmarkStop(b *testing.B) {
	for i := 0; i < b.N; i++ {
		NewTimer(1e9).Stop()
	}
}

func TestAfter(t *testing.T) {
	const delay = int64(100e6)
	start := Nanoseconds()
	end := <-After(delay)
	if duration := Nanoseconds() - start; duration < delay {
		t.Fatalf("After(%d) slept for only %d ns", delay, duration)
	}
	if min := start + delay; end < min {
		t.Fatalf("After(%d) expect >= %d, got %d", delay, min, end)
	}
}

func TestAfterTick(t *testing.T) {
	const (
		Delta = 100 * 1e6
		Count = 10
	)
	t0 := Nanoseconds()
	for i := 0; i < Count; i++ {
		<-After(Delta)
	}
	t1 := Nanoseconds()
	ns := t1 - t0
	target := int64(Delta * Count)
	slop := target * 2 / 10
	if ns < target-slop || ns > target+slop {
		t.Fatalf("%d ticks of %g ns took %g ns, expected %g", Count, float64(Delta), float64(ns), float64(target))
	}
}

func TestAfterStop(t *testing.T) {
	const msec = 1e6
	AfterFunc(100*msec, func() {})
	t0 := NewTimer(50 * msec)
	c1 := make(chan bool, 1)
	t1 := AfterFunc(150*msec, func() { c1 <- true })
	c2 := After(200 * msec)
	if !t0.Stop() {
		t.Fatalf("failed to stop event 0")
	}
	if !t1.Stop() {
		t.Fatalf("failed to stop event 1")
	}
	<-c2
	select {
	case <-t0.C:
		t.Fatalf("event 0 was not stopped")
	case <-c1:
		t.Fatalf("event 1 was not stopped")
	default:
	}
	if t1.Stop() {
		t.Fatalf("Stop returned true twice")
	}
}

var slots = []int{5, 3, 6, 6, 6, 1, 1, 2, 7, 9, 4, 8, 0}

type afterResult struct {
	slot int
	t    int64
}

func await(slot int, result chan<- afterResult, ac <-chan int64) {
	result <- afterResult{slot, <-ac}
}

func TestAfterQueuing(t *testing.T) {
	const (
		Delta = 100 * 1e6
	)
	// make the result channel buffered because we don't want
	// to depend on channel queueing semantics that might
	// possibly change in the future.
	result := make(chan afterResult, len(slots))

	t0 := Nanoseconds()
	for _, slot := range slots {
		go await(slot, result, After(int64(slot)*Delta))
	}
	sort.SortInts(slots)
	for _, slot := range slots {
		r := <-result
		if r.slot != slot {
			t.Fatalf("after queue got slot %d, expected %d", r.slot, slot)
		}
		ns := r.t - t0
		target := int64(slot * Delta)
		slop := int64(Delta) / 4
		if ns < target-slop || ns > target+slop {
			t.Fatalf("after queue slot %d arrived at %g, expected [%g,%g]", slot, float64(ns), float64(target-slop), float64(target+slop))
		}
	}
}
<commit_msg>time: make TestAfterQueuing retry 3 times before declaring failure.<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package time_test

import (
	"fmt"
	"os"
	"syscall"
	"testing"
	"sort"
	. "time"
)

func TestSleep(t *testing.T) {
	const delay = int64(100e6)
	go func() {
		Sleep(delay / 2)
		syscall.Kill(os.Getpid(), syscall.SIGCHLD)
	}()
	start := Nanoseconds()
	Sleep(delay)
	duration := Nanoseconds() - start
	if duration < delay {
		t.Fatalf("Sleep(%d) slept for only %d ns", delay, duration)
	}
}

// Test the basic function calling behavior. Correct queueing
// behavior is tested elsewhere, since After and AfterFunc share
// the same code.
func TestAfterFunc(t *testing.T) {
	i := 10
	c := make(chan bool)
	var f func()
	f = func() {
		i--
		if i >= 0 {
			AfterFunc(0, f)
			Sleep(1e9)
		} else {
			c <- true
		}
	}

	AfterFunc(0, f)
	<-c
}

func BenchmarkAfterFunc(b *testing.B) {
	i := b.N
	c := make(chan bool)
	var f func()
	f = func() {
		i--
		if i >= 0 {
			AfterFunc(0, f)
		} else {
			c <- true
		}
	}

	AfterFunc(0, f)
	<-c
}

func BenchmarkAfter(b *testing.B) {
	for i := 0; i < b.N; i++ {
		<-After(1)
	}
}

func BenchmarkStop(b *testing.B) {
	for i := 0; i < b.N; i++ {
		NewTimer(1e9).Stop()
	}
}

func TestAfter(t *testing.T) {
	const delay = int64(100e6)
	start := Nanoseconds()
	end := <-After(delay)
	if duration := Nanoseconds() - start; duration < delay {
		t.Fatalf("After(%d) slept for only %d ns", delay, duration)
	}
	if min := start + delay; end < min {
		t.Fatalf("After(%d) expect >= %d, got %d", delay, min, end)
	}
}

func TestAfterTick(t *testing.T) {
	const (
		Delta = 100 * 1e6
		Count = 10
	)
	t0 := Nanoseconds()
	for i := 0; i < Count; i++ {
		<-After(Delta)
	}
	t1 := Nanoseconds()
	ns := t1 - t0
	target := int64(Delta * Count)
	slop := target * 2 / 10
	if ns < target-slop || ns > target+slop {
		t.Fatalf("%d ticks of %g ns took %g ns, expected %g", Count, float64(Delta), float64(ns), float64(target))
	}
}

func TestAfterStop(t *testing.T) {
	const msec = 1e6
	AfterFunc(100*msec, func() {})
	t0 := NewTimer(50 * msec)
	c1 := make(chan bool, 1)
	t1 := AfterFunc(150*msec, func() { c1 <- true })
	c2 := After(200 * msec)
	if !t0.Stop() {
		t.Fatalf("failed to stop event 0")
	}
	if !t1.Stop() {
		t.Fatalf("failed to stop event 1")
	}
	<-c2
	select {
	case <-t0.C:
		t.Fatalf("event 0 was not stopped")
	case <-c1:
		t.Fatalf("event 1 was not stopped")
	default:
	}
	if t1.Stop() {
		t.Fatalf("Stop returned true twice")
	}
}

func TestAfterQueuing(t *testing.T) {
	// This test flakes out on some systems,
	// so we'll try it a few times before declaring it a failure.
	const attempts = 3
	err := os.NewError("!=nil")
	for i := 0; i < attempts && err != nil; i++ {
		if err = testAfterQueuing(t); err != nil {
			t.Logf("attempt %v failed: %v", i, err)
		}
	}
	if err != nil {
		t.Fatal(err)
	}
}

var slots = []int{5, 3, 6, 6, 6, 1, 1, 2, 7, 9, 4, 8, 0}

type afterResult struct {
	slot int
	t    int64
}

func await(slot int, result chan<- afterResult, ac <-chan int64) {
	result <- afterResult{slot, <-ac}
}

func testAfterQueuing(t *testing.T) os.Error {
	const (
		Delta = 100 * 1e6
	)
	// make the result channel buffered because we don't want
	// to depend on channel queueing semantics that might
	// possibly change in the future.
	result := make(chan afterResult, len(slots))

	t0 := Nanoseconds()
	for _, slot := range slots {
		go await(slot, result, After(int64(slot)*Delta))
	}
	sort.SortInts(slots)
	for _, slot := range slots {
		r := <-result
		if r.slot != slot {
			return fmt.Errorf("after queue got slot %d, expected %d", r.slot, slot)
		}
		ns := r.t - t0
		target := int64(slot * Delta)
		slop := int64(Delta) / 4
		if ns < target-slop || ns > target+slop {
			return fmt.Errorf("after queue slot %d arrived at %g, expected [%g,%g]", slot, float64(ns), float64(target-slop), float64(target+slop))
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
    "gopkg.in/yaml.v2"
    "io/ioutil"
    "fmt"
    "encoding/hex"
    "github.com/fuzxxl/nfc/2.0/nfc"
    "github.com/fuzxxl/freefare/0.3/freefare"
)

func padUID(block []byte) ([]byte, bool) {
	blockLen := len(block)
	if blockLen >= 32 {
		panic("PadBlock input must be less than 32 bytes.")
	}
	if blockLen == 32 {
	   return block, false
	}

	result := make([]byte, 32)
	copy(result, block)
	result[blockLen] = 0x80

	return result, true
}

func main() {
    dat, err := ioutil.ReadFile("keys.yaml")
    if err != nil {
        panic(err)
    }

    m := make(map[interface{}]interface{});
    err = yaml.Unmarshal([]byte(dat), &m);
    if err != nil {
        panic(err)
    }

    appkey_str := m["aclapp"].(map[interface{}]interface{})["read"].(string)
    appkey, err := hex.DecodeString(appkey_str)
    if err != nil {
        panic(err)
    }
    fmt.Println(appkey)

    d, err := nfc.Open("");
    if err != nil {
        panic(err)
    }

    tags, err := freefare.GetTags(d);
    if err != nil {
        panic(err)
    }

    for i := 0; i < len(tags); i++ {
        tag := tags[i]
        uidstr := tag.UID()
        uidbytes, err := hex.DecodeString(uidstr);
        if err != nil {
            panic(err);
        }
        fmt.Println(uidstr)
        fmt.Println(uidbytes)

        fmt.Println("Found tag", uidstr)

        D, padded := padUID(uidbytes)
        fmt.Println("D=", D, " padded=", padded)
    }

}

<commit_msg>set the block limits correcty(?)<commit_after>package main

import (
    "gopkg.in/yaml.v2"
    "io/ioutil"
    "fmt"
    "encoding/hex"
    "github.com/fuzxxl/nfc/2.0/nfc"
    "github.com/fuzxxl/freefare/0.3/freefare"
)

func padUID(block []byte) ([]byte, bool) {
	blockLen := len(block)
	if blockLen >= 32 {
		panic("PadBlock input must be less than 32 bytes.")
	}

	result := make([]byte, 32)
	copy(result, block)
	result[blockLen] = 0x80

	if blockLen == 31 {
	   return result, false
	}

	return result, true
}

func main() {
    dat, err := ioutil.ReadFile("keys.yaml")
    if err != nil {
        panic(err)
    }

    m := make(map[interface{}]interface{});
    err = yaml.Unmarshal([]byte(dat), &m);
    if err != nil {
        panic(err)
    }

    appkey_str := m["aclapp"].(map[interface{}]interface{})["read"].(string)
    appkey, err := hex.DecodeString(appkey_str)
    if err != nil {
        panic(err)
    }
    fmt.Println(appkey)

    d, err := nfc.Open("");
    if err != nil {
        panic(err)
    }

    tags, err := freefare.GetTags(d);
    if err != nil {
        panic(err)
    }

    for i := 0; i < len(tags); i++ {
        tag := tags[i]
        uidstr := tag.UID()
        uidbytes, err := hex.DecodeString(uidstr);
        if err != nil {
            panic(err);
        }
        fmt.Println(uidstr)
        fmt.Println(uidbytes)

        fmt.Println("Found tag", uidstr)

        D, padded := padUID(uidbytes)
        fmt.Println("D=", D, " padded=", padded)
    }

}

<|endoftext|>