<commit_before>package terraform

import (
	"fmt"
	"path/filepath"
	"strings"

	"github.com/hashicorp/hcl/v2"
	"github.com/hashicorp/hcl/v2/hclparse"
	"github.com/spf13/afero"
	"github.com/zclconf/go-cty/cty"
)

// Parser is the main interface to read configuration files and other related
// files from disk.
//
// It retains a cache of all files that are loaded so that they can be used
// to create source code snippets in diagnostics, etc.
type Parser struct {
	fs afero.Afero
	p  *hclparse.Parser
}

// NewParser creates and returns a new Parser that reads files from the given
// filesystem. If a nil filesystem is passed then the system's "real" filesystem
// will be used, via afero.OsFs.
func NewParser(fs afero.Fs) *Parser {
	if fs == nil {
		fs = afero.OsFs{}
	}

	return &Parser{
		fs: afero.Afero{Fs: fs},
		p:  hclparse.NewParser(),
	}
}

// LoadConfigDir reads the .tf and .tf.json files in the given directory and
// then combines these files into a single Module.
//
// If this method returns nil, that indicates that the given directory does not
// exist at all or could not be opened for some reason. Callers may wish to
// detect this case and ignore the returned diagnostics so that they can
// produce a more context-aware error message in that case.
//
// If this method returns a non-nil module while error diagnostics are returned
// then the module may be incomplete but can be used carefully for static
// analysis.
//
// This file does not consider a directory with no files to be an error, and
// will simply return an empty module in that case. Callers should first call
// Parser.IsConfigDir if they wish to recognize that situation.
//
// .tf files are parsed using the HCL native syntax while .tf.json files are
// parsed using the HCL JSON syntax.
func (p *Parser) LoadConfigDir(dir string) (*Module, hcl.Diagnostics) {
	primaries, overrides, diags := p.dirFiles(dir)
	if diags.HasErrors() {
		return nil, diags
	}

	mod := NewEmptyModule()
	mod.primaries = make([]*hcl.File, len(primaries))
	mod.overrides = make([]*hcl.File, len(overrides))

	for i, path := range primaries {
		f, loadDiags := p.loadHCLFile(path)
		diags = diags.Extend(loadDiags)

		mod.primaries[i] = f
		mod.Sources[path] = f.Bytes
		mod.Files[path] = f
	}
	for i, path := range overrides {
		f, loadDiags := p.loadHCLFile(path)
		diags = diags.Extend(loadDiags)

		mod.overrides[i] = f
		mod.Sources[path] = f.Bytes
		mod.Files[path] = f
	}
	if diags.HasErrors() {
		return mod, diags
	}

	mod.SourceDir = dir

	buildDiags := mod.build()
	diags = diags.Extend(buildDiags)

	return mod, diags
}

// LoadValuesFile reads the file at the given path and parses it as a "values
// file", which is an HCL config file whose top-level attributes are treated
// as arbitrary key.value pairs.
//
// If the file cannot be read -- for example, if it does not exist -- then
// a nil map will be returned along with error diagnostics. Callers may wish
// to disregard the returned diagnostics in this case and instead generate
// their own error message(s) with additional context.
//
// If the returned diagnostics has errors when a non-nil map is returned
// then the map may be incomplete but should be valid enough for careful
// static analysis.
//
// This method wraps LoadHCLFile, and so it inherits the syntax selection
// behaviors documented for that method.
func (p *Parser) LoadValuesFile(path string) (map[string]cty.Value, hcl.Diagnostics) {
	f, diags := p.loadHCLFile(path)
	if diags.HasErrors() {
		return nil, diags
	}

	vals := make(map[string]cty.Value)
	if f == nil || f.Body == nil {
		return vals, diags
	}

	attrs, attrDiags := f.Body.JustAttributes()
	diags = diags.Extend(attrDiags)
	if attrs == nil {
		return vals, diags
	}

	for name, attr := range attrs {
		val, valDiags := attr.Expr.Value(nil)
		diags = diags.Extend(valDiags)
		vals[name] = val
	}

	return vals, diags
}

func (p *Parser) loadHCLFile(path string) (*hcl.File, hcl.Diagnostics) {
	src, err := p.fs.ReadFile(path)

	if err != nil {
		return nil, hcl.Diagnostics{
			{
				Severity: hcl.DiagError,
				Summary:  "Failed to read file",
				Detail:   fmt.Sprintf("The file %q could not be read.", path),
			},
		}
	}

	switch {
	case strings.HasSuffix(path, ".json"):
		return p.p.ParseJSON(src, path)
	default:
		return p.p.ParseHCL(src, path)
	}
}

// Sources returns a map of the cached source buffers for all files that
// have been loaded through this parser, with source filenames (as requested
// when each file was opened) as the keys.
func (p *Parser) Sources() map[string][]byte {
	return p.p.Sources()
}

// Files returns a map of the cached HCL file objects for all files that
// have been loaded through this parser, with source filenames (as requested
// when each file was opened) as the keys.
func (p *Parser) Files() map[string]*hcl.File {
	return p.p.Files()
}

// ConfigDirFiles returns lists of the primary and override files configuration
// files in the given directory.
//
// If the given directory does not exist or cannot be read, error diagnostics
// are returned. If errors are returned, the resulting lists may be incomplete.
func (p Parser) ConfigDirFiles(dir string) (primary, override []string, diags hcl.Diagnostics) {
	return p.dirFiles(dir)
}

// IsConfigDir determines whether the given path refers to a directory that
// exists and contains at least one Terraform config file (with a .tf or
// .tf.json extension.)
func (p *Parser) IsConfigDir(path string) bool {
	primaryPaths, overridePaths, _ := p.dirFiles(path)
	return (len(primaryPaths) + len(overridePaths)) > 0
}

func (p *Parser) dirFiles(dir string) (primary, override []string, diags hcl.Diagnostics) {
	infos, err := p.fs.ReadDir(dir)
	if err != nil {
		diags = append(diags, &hcl.Diagnostic{
			Severity: hcl.DiagError,
			Summary:  "Failed to read module directory",
			Detail:   fmt.Sprintf("Module directory %s does not exist or cannot be read.", dir),
		})
		return
	}

	for _, info := range infos {
		if info.IsDir() {
			// We only care about files
			continue
		}

		name := info.Name()
		ext := fileExt(name)
		if ext == "" || isIgnoredFile(name) {
			continue
		}

		baseName := name[:len(name)-len(ext)] // strip extension
		isOverride := baseName == "override" || strings.HasSuffix(baseName, "_override")

		// TODO: path normalization
		fullPath := filepath.Join(dir, name)
		if isOverride {
			override = append(override, fullPath)
		} else {
			primary = append(primary, fullPath)
		}
	}

	return
}

// fileExt returns the Terraform configuration extension of the given
// path, or a blank string if it is not a recognized extension.
func fileExt(path string) string {
	if strings.HasSuffix(path, ".tf") {
		return ".tf"
	} else if strings.HasSuffix(path, ".tf.json") {
		return ".tf.json"
	} else {
		return ""
	}
}

// isIgnoredFile returns true if the given filename (which must not have a
// directory path ahead of it) should be ignored as e.g. an editor swap file.
func isIgnoredFile(name string) bool {
	return strings.HasPrefix(name, ".") || // Unix-like hidden files
		strings.HasSuffix(name, "~") || // vim
		strings.HasPrefix(name, "#") && strings.HasSuffix(name, "#") // emacs
}
<commit_msg>Fix panic when file read fails (#1598)<commit_after>package terraform

import (
	"fmt"
	"path/filepath"
	"strings"

	"github.com/hashicorp/hcl/v2"
	"github.com/hashicorp/hcl/v2/hclparse"
	"github.com/spf13/afero"
	"github.com/zclconf/go-cty/cty"
)

// Parser is the main interface to read configuration files and other related
// files from disk.
//
// It retains a cache of all files that are loaded so that they can be used
// to create source code snippets in diagnostics, etc.
type Parser struct {
	fs afero.Afero
	p  *hclparse.Parser
}

// NewParser creates and returns a new Parser that reads files from the given
// filesystem. If a nil filesystem is passed then the system's "real" filesystem
// will be used, via afero.OsFs.
func NewParser(fs afero.Fs) *Parser {
	if fs == nil {
		fs = afero.OsFs{}
	}

	return &Parser{
		fs: afero.Afero{Fs: fs},
		p:  hclparse.NewParser(),
	}
}

// LoadConfigDir reads the .tf and .tf.json files in the given directory and
// then combines these files into a single Module.
//
// If this method returns nil, that indicates that the given directory does not
// exist at all or could not be opened for some reason. Callers may wish to
// detect this case and ignore the returned diagnostics so that they can
// produce a more context-aware error message in that case.
//
// If this method returns a non-nil module while error diagnostics are returned
// then the module may be incomplete but can be used carefully for static
// analysis.
//
// This file does not consider a directory with no files to be an error, and
// will simply return an empty module in that case. Callers should first call
// Parser.IsConfigDir if they wish to recognize that situation.
//
// .tf files are parsed using the HCL native syntax while .tf.json files are
// parsed using the HCL JSON syntax.
func (p *Parser) LoadConfigDir(dir string) (*Module, hcl.Diagnostics) {
	primaries, overrides, diags := p.dirFiles(dir)
	if diags.HasErrors() {
		return nil, diags
	}

	mod := NewEmptyModule()
	mod.primaries = make([]*hcl.File, len(primaries))
	mod.overrides = make([]*hcl.File, len(overrides))

	for i, path := range primaries {
		f, loadDiags := p.loadHCLFile(path)
		diags = diags.Extend(loadDiags)
		if loadDiags.HasErrors() {
			continue
		}

		mod.primaries[i] = f
		mod.Sources[path] = f.Bytes
		mod.Files[path] = f
	}
	for i, path := range overrides {
		f, loadDiags := p.loadHCLFile(path)
		diags = diags.Extend(loadDiags)
		if loadDiags.HasErrors() {
			continue
		}

		mod.overrides[i] = f
		mod.Sources[path] = f.Bytes
		mod.Files[path] = f
	}
	if diags.HasErrors() {
		return mod, diags
	}

	mod.SourceDir = dir

	buildDiags := mod.build()
	diags = diags.Extend(buildDiags)

	return mod, diags
}

// LoadValuesFile reads the file at the given path and parses it as a "values
// file", which is an HCL config file whose top-level attributes are treated
// as arbitrary key.value pairs.
//
// If the file cannot be read -- for example, if it does not exist -- then
// a nil map will be returned along with error diagnostics. Callers may wish
// to disregard the returned diagnostics in this case and instead generate
// their own error message(s) with additional context.
//
// If the returned diagnostics has errors when a non-nil map is returned
// then the map may be incomplete but should be valid enough for careful
// static analysis.
//
// This method wraps LoadHCLFile, and so it inherits the syntax selection
// behaviors documented for that method.
func (p *Parser) LoadValuesFile(path string) (map[string]cty.Value, hcl.Diagnostics) {
	f, diags := p.loadHCLFile(path)
	if diags.HasErrors() {
		return nil, diags
	}

	vals := make(map[string]cty.Value)
	if f == nil || f.Body == nil {
		return vals, diags
	}

	attrs, attrDiags := f.Body.JustAttributes()
	diags = diags.Extend(attrDiags)
	if attrs == nil {
		return vals, diags
	}

	for name, attr := range attrs {
		val, valDiags := attr.Expr.Value(nil)
		diags = diags.Extend(valDiags)
		vals[name] = val
	}

	return vals, diags
}

func (p *Parser) loadHCLFile(path string) (*hcl.File, hcl.Diagnostics) {
	src, err := p.fs.ReadFile(path)

	if err != nil {
		return nil, hcl.Diagnostics{
			{
				Severity: hcl.DiagError,
				Summary:  "Failed to read file",
				Detail:   fmt.Sprintf("The file %q could not be read.", path),
			},
		}
	}

	switch {
	case strings.HasSuffix(path, ".json"):
		return p.p.ParseJSON(src, path)
	default:
		return p.p.ParseHCL(src, path)
	}
}

// Sources returns a map of the cached source buffers for all files that
// have been loaded through this parser, with source filenames (as requested
// when each file was opened) as the keys.
func (p *Parser) Sources() map[string][]byte {
	return p.p.Sources()
}

// Files returns a map of the cached HCL file objects for all files that
// have been loaded through this parser, with source filenames (as requested
// when each file was opened) as the keys.
func (p *Parser) Files() map[string]*hcl.File {
	return p.p.Files()
}

// ConfigDirFiles returns lists of the primary and override files configuration
// files in the given directory.
//
// If the given directory does not exist or cannot be read, error diagnostics
// are returned. If errors are returned, the resulting lists may be incomplete.
func (p Parser) ConfigDirFiles(dir string) (primary, override []string, diags hcl.Diagnostics) {
	return p.dirFiles(dir)
}

// IsConfigDir determines whether the given path refers to a directory that
// exists and contains at least one Terraform config file (with a .tf or
// .tf.json extension.)
func (p *Parser) IsConfigDir(path string) bool {
	primaryPaths, overridePaths, _ := p.dirFiles(path)
	return (len(primaryPaths) + len(overridePaths)) > 0
}

func (p *Parser) dirFiles(dir string) (primary, override []string, diags hcl.Diagnostics) {
	infos, err := p.fs.ReadDir(dir)
	if err != nil {
		diags = append(diags, &hcl.Diagnostic{
			Severity: hcl.DiagError,
			Summary:  "Failed to read module directory",
			Detail:   fmt.Sprintf("Module directory %s does not exist or cannot be read.", dir),
		})
		return
	}

	for _, info := range infos {
		if info.IsDir() {
			// We only care about files
			continue
		}

		name := info.Name()
		ext := fileExt(name)
		if ext == "" || isIgnoredFile(name) {
			continue
		}

		baseName := name[:len(name)-len(ext)] // strip extension
		isOverride := baseName == "override" || strings.HasSuffix(baseName, "_override")

		// TODO: path normalization
		fullPath := filepath.Join(dir, name)
		if isOverride {
			override = append(override, fullPath)
		} else {
			primary = append(primary, fullPath)
		}
	}

	return
}

// fileExt returns the Terraform configuration extension of the given
// path, or a blank string if it is not a recognized extension.
func fileExt(path string) string {
	if strings.HasSuffix(path, ".tf") {
		return ".tf"
	} else if strings.HasSuffix(path, ".tf.json") {
		return ".tf.json"
	} else {
		return ""
	}
}

// isIgnoredFile returns true if the given filename (which must not have a
// directory path ahead of it) should be ignored as e.g. an editor swap file.
func isIgnoredFile(name string) bool {
	return strings.HasPrefix(name, ".") || // Unix-like hidden files
		strings.HasSuffix(name, "~") || // vim
		strings.HasPrefix(name, "#") && strings.HasSuffix(name, "#") // emacs
}
<|endoftext|>


<commit_before>package fuse

import (
	"log"
	"os"
	"strings"
	"sync"
	"time"
	"unsafe"

	"github.com/hanwen/go-fuse/raw"
)

const (
	// The kernel caps writes at 128k.
	MAX_KERNEL_WRITE = 128 * 1024
)

// MountState contains the logic for reading from the FUSE device and
// translating it to RawFileSystem interface calls.
type MountState struct {
	// Empty if unmounted.
	mountPoint string
	fileSystem RawFileSystem

	// I/O with kernel and daemon.
	mountFile *os.File

	// Dump debug info onto stdout.
	Debug bool

	// For efficient reads and writes.
	buffers *BufferPoolImpl

	latencies *LatencyMap

	opts           *MountOptions
	kernelSettings raw.InitIn

	reqMu   sync.Mutex
	reqPool []*request
	readPool [][]byte
	reqReaders int
	outstandingReadBufs int
}

func (ms *MountState) KernelSettings() raw.InitIn {
	return ms.kernelSettings
}

func (ms *MountState) MountPoint() string {
	return ms.mountPoint
}

// Mount filesystem on mountPoint.
func (ms *MountState) Mount(mountPoint string, opts *MountOptions) error {
	if opts == nil {
		opts = &MountOptions{
			MaxBackground: _DEFAULT_BACKGROUND_TASKS,
		}
	}
	o := *opts
	if o.MaxWrite < 0 {
		o.MaxWrite = 0
	}
	if o.MaxWrite == 0 {
		o.MaxWrite = 1 << 16
	}
	if o.MaxWrite > MAX_KERNEL_WRITE {
		o.MaxWrite = MAX_KERNEL_WRITE
	}
	opts = &o
	ms.opts = &o

	optStrs := opts.Options
	if opts.AllowOther {
		optStrs = append(optStrs, "allow_other")
	}

	file, mp, err := mount(mountPoint, strings.Join(optStrs, ","))
	if err != nil {
		return err
	}
	initParams := RawFsInit{
		InodeNotify: func(n *raw.NotifyInvalInodeOut) Status {
			return ms.writeInodeNotify(n)
		},
		EntryNotify: func(parent uint64, n string) Status {
			return ms.writeEntryNotify(parent, n)
		},
	}
	ms.fileSystem.Init(&initParams)
	ms.mountPoint = mp
	ms.mountFile = file
	return nil
}

func (ms *MountState) SetRecordStatistics(record bool) {
	if record {
		ms.latencies = NewLatencyMap()
	} else {
		ms.latencies = nil
	}
}

func (ms *MountState) Unmount() (err error) {
	if ms.mountPoint == "" {
		return nil
	}
	delay := time.Duration(0)
	for try := 0; try < 5; try++ {
		err = unmount(ms.mountPoint)
		if err == nil {
			break
		}

		// Sleep for a bit. This is not pretty, but there is
		// no way we can be certain that the kernel thinks all
		// open files have already been closed.
		delay = 2*delay + 5*time.Millisecond
		time.Sleep(delay)
	}
	ms.mountPoint = ""
	return err
}

func NewMountState(fs RawFileSystem) *MountState {
	ms := new(MountState)
	ms.mountPoint = ""
	ms.fileSystem = fs
	ms.buffers = NewBufferPool()
	return ms
}

func (ms *MountState) Latencies() map[string]float64 {
	if ms.latencies == nil {
		return nil
	}
	return ms.latencies.Latencies(1e-3)
}

func (ms *MountState) OperationCounts() map[string]int {
	if ms.latencies == nil {
		return nil
	}
	return ms.latencies.Counts()
}

func (ms *MountState) BufferPoolStats() string {
	return ms.buffers.String()
}

const _MAX_READERS = 10
func (ms *MountState) readRequest() (req *request, code Status) {
	var dest []byte

	ms.reqMu.Lock()
	l := len(ms.reqPool)
	if l > 0 {
		req = ms.reqPool[l-1]
		ms.reqPool = ms.reqPool[:l-1]
	} else {
		req = new(request)
	}
	l = len(ms.readPool)
	if l > 0 {
		dest = ms.readPool[l-1]
		ms.readPool = ms.readPool[:l-1]
	} else {
		dest = make([]byte, ms.opts.MaxWrite + 4096)
	}
	ms.outstandingReadBufs++
	ms.reqReaders++
	ms.reqMu.Unlock()

	n, err := ms.mountFile.Read(dest)
	if err != nil {
		code = ToStatus(err)
		ms.reqMu.Lock()
		ms.reqPool = append(ms.reqPool, req)
		ms.reqReaders--
		ms.reqMu.Unlock()
		return nil, code
	}

	gobbled := req.setInput(dest[:n])

	ms.reqMu.Lock()
	if !gobbled {
		ms.outstandingReadBufs--
		ms.readPool = append(ms.readPool, dest)
		dest = nil
	}
	ms.reqReaders--
	ms.reqMu.Unlock()

	return req, OK
}

func (ms *MountState) returnRequest(req *request) {
	ms.recordStats(req)

	if req.bufferPoolOutputBuf != nil {
		ms.buffers.FreeBuffer(req.bufferPoolOutputBuf)
		req.bufferPoolOutputBuf = nil
	}

	req.clear()
	ms.reqMu.Lock()
	if req.bufferPoolOutputBuf != nil {
		ms.readPool = append(ms.readPool, req.bufferPoolInputBuf)
		ms.outstandingReadBufs--
		req.bufferPoolInputBuf = nil
	}
	ms.reqPool = append(ms.reqPool, req)
	ms.reqMu.Unlock()
}

func (ms *MountState) recordStats(req *request) {
	if ms.latencies != nil {
		endNs := time.Now().UnixNano()
		dt := endNs - req.startNs

		opname := operationName(req.inHeader.Opcode)
		ms.latencies.AddMany(
			[]LatencyArg{
				{opname, "", dt},
				{opname + "-write", "", endNs - req.preWriteNs}})
	}
}

// Loop initiates the FUSE loop. Normally, callers should run Loop()
// and wait for it to exit, but tests will want to run this in a
// goroutine.
//
// Each filesystem operation executes in a separate goroutine.
func (ms *MountState) Loop() {
	ms.loop()
	ms.mountFile.Close()
}

func (ms *MountState) loop() {
	exit:
	for {
		req, errNo := ms.readRequest()
		switch errNo {
		case OK:
		case ENOENT:
			continue
		case ENODEV:
			// unmount
			break exit
		default: // some other error?
			log.Printf("Failed to read from fuse conn: %v", errNo)
			break
		}

		if ms.latencies != nil {
			req.startNs = time.Now().UnixNano()
		}
		go ms.handleRequest(req)
	}
}

func (ms *MountState) handleRequest(req *request) {
	defer ms.returnRequest(req)

	req.parse()
	if req.handler == nil {
		req.status = ENOSYS
	}

	if req.status.Ok() && ms.Debug {
		log.Println(req.InputDebug())
	}

	if req.status.Ok() && req.handler.Func == nil {
		log.Printf("Unimplemented opcode %v", operationName(req.inHeader.Opcode))
		req.status = ENOSYS
	}

	if req.status.Ok() {
		req.handler.Func(ms, req)
	}

	errNo := ms.write(req)
	if errNo != 0 {
		log.Printf("writer: Write/Writev failed, err: %v. opcode: %v",
			errNo, operationName(req.inHeader.Opcode))
	}
}

func (ms *MountState) AllocOut(req *request, size uint32) []byte {
	if req.bufferPoolOutputBuf != nil {
		ms.buffers.FreeBuffer(req.bufferPoolOutputBuf)
	}
	req.bufferPoolOutputBuf = ms.buffers.AllocBuffer(size)
	return req.bufferPoolOutputBuf
}

func (ms *MountState) write(req *request) Status {
	// Forget does not wait for reply.
	if req.inHeader.Opcode == _OP_FORGET || req.inHeader.Opcode == _OP_BATCH_FORGET {
		return OK
	}

	header, data := req.serialize()
	if ms.Debug {
		log.Println(req.OutputDebug())
	}

	if ms.latencies != nil {
		req.preWriteNs = time.Now().UnixNano()
	}

	if header == nil {
		return OK
	}
	var err error
	if data == nil {
		_, err = ms.mountFile.Write(header)
	} else {
		_, err = Writev(int(ms.mountFile.Fd()), [][]byte{header, data})
	}

	return ToStatus(err)
}

func (ms *MountState) writeInodeNotify(entry *raw.NotifyInvalInodeOut) Status {
	req := request{
		inHeader: &raw.InHeader{
			Opcode: _OP_NOTIFY_INODE,
		},
		handler: operationHandlers[_OP_NOTIFY_INODE],
		status:  raw.NOTIFY_INVAL_INODE,
	}
	req.outData = unsafe.Pointer(entry)
	result := ms.write(&req)

	if ms.Debug {
		log.Println("Response: INODE_NOTIFY", result)
	}
	return result
}

func (ms *MountState) writeEntryNotify(parent uint64, name string) Status {
	req := request{
		inHeader: &raw.InHeader{
			Opcode: _OP_NOTIFY_ENTRY,
		},
		handler: operationHandlers[_OP_NOTIFY_ENTRY],
		status:  raw.NOTIFY_INVAL_ENTRY,
	}
	entry := &raw.NotifyInvalEntryOut{
		Parent:  parent,
		NameLen: uint32(len(name)),
	}

	// Many versions of FUSE generate stacktraces if the
	// terminating null byte is missing.
	nameBytes := []byte(name + "\000")
	req.outData = unsafe.Pointer(entry)
	req.flatData = nameBytes
	result := ms.write(&req)

	if ms.Debug {
		log.Printf("Response: ENTRY_NOTIFY: %v", result)
	}
	return result
}
<commit_msg>Again: use multiple loops rather than goroutine spawning.  This allows the request to be reused, thus reducing garbage generation.      Back to old speed (10% slower than libfuse).<commit_after>package fuse

import (
	"fmt"
	"log"
	"os"
	"strings"
	"sync"
	"time"
	"unsafe"

	"github.com/hanwen/go-fuse/raw"
)

const (
	// The kernel caps writes at 128k.
	MAX_KERNEL_WRITE = 128 * 1024
)

// MountState contains the logic for reading from the FUSE device and
// translating it to RawFileSystem interface calls.
type MountState struct {
	// Empty if unmounted.
	mountPoint string
	fileSystem RawFileSystem

	// I/O with kernel and daemon.
	mountFile *os.File

	// Dump debug info onto stdout.
	Debug bool

	// For efficient reads and writes.
	buffers *BufferPoolImpl

	latencies *LatencyMap

	opts           *MountOptions
	kernelSettings raw.InitIn

	reqMu   sync.Mutex
	reqPool []*request
	readPool [][]byte
	reqReaders int
	outstandingReadBufs int

	loops sync.WaitGroup
}

func (ms *MountState) KernelSettings() raw.InitIn {
	return ms.kernelSettings
}

func (ms *MountState) MountPoint() string {
	return ms.mountPoint
}

// Mount filesystem on mountPoint.
func (ms *MountState) Mount(mountPoint string, opts *MountOptions) error {
	if opts == nil {
		opts = &MountOptions{
			MaxBackground: _DEFAULT_BACKGROUND_TASKS,
		}
	}
	o := *opts
	if o.MaxWrite < 0 {
		o.MaxWrite = 0
	}
	if o.MaxWrite == 0 {
		o.MaxWrite = 1 << 16
	}
	if o.MaxWrite > MAX_KERNEL_WRITE {
		o.MaxWrite = MAX_KERNEL_WRITE
	}
	opts = &o
	ms.opts = &o

	optStrs := opts.Options
	if opts.AllowOther {
		optStrs = append(optStrs, "allow_other")
	}

	file, mp, err := mount(mountPoint, strings.Join(optStrs, ","))
	if err != nil {
		return err
	}
	initParams := RawFsInit{
		InodeNotify: func(n *raw.NotifyInvalInodeOut) Status {
			return ms.writeInodeNotify(n)
		},
		EntryNotify: func(parent uint64, n string) Status {
			return ms.writeEntryNotify(parent, n)
		},
	}
	ms.fileSystem.Init(&initParams)
	ms.mountPoint = mp
	ms.mountFile = file
	return nil
}

func (ms *MountState) SetRecordStatistics(record bool) {
	if record {
		ms.latencies = NewLatencyMap()
	} else {
		ms.latencies = nil
	}
}

func (ms *MountState) Unmount() (err error) {
	if ms.mountPoint == "" {
		return nil
	}
	delay := time.Duration(0)
	for try := 0; try < 5; try++ {
		err = unmount(ms.mountPoint)
		if err == nil {
			break
		}

		// Sleep for a bit. This is not pretty, but there is
		// no way we can be certain that the kernel thinks all
		// open files have already been closed.
		delay = 2*delay + 5*time.Millisecond
		time.Sleep(delay)
	}
	// Wait for event loops to exit.
	ms.loops.Wait()
	ms.mountPoint = ""
	return err
}

func NewMountState(fs RawFileSystem) *MountState {
	ms := new(MountState)
	ms.mountPoint = ""
	ms.fileSystem = fs
	ms.buffers = NewBufferPool()
	return ms
}

func (ms *MountState) Latencies() map[string]float64 {
	if ms.latencies == nil {
		return nil
	}
	return ms.latencies.Latencies(1e-3)
}

func (ms *MountState) OperationCounts() map[string]int {
	if ms.latencies == nil {
		return nil
	}
	return ms.latencies.Counts()
}

func (ms *MountState) BufferPoolStats() string {
	s := ms.buffers.String()

	var r int
	ms.reqMu.Lock()
	r = len(ms.readPool) + ms.reqReaders
	ms.reqMu.Unlock()

	s += fmt.Sprintf("read buffers: %d (sz %d )",
		r, ms.opts.MaxWrite/PAGESIZE + 1)
	return s
}

// What is a good number?  Maybe the number of CPUs?
const _MAX_READERS = 2

// Returns a new request, or error. In case exitIdle is given, returns
// nil, OK if we have too many readers already.
func (ms *MountState) readRequest(exitIdle bool) (req *request, code Status) {
	var dest []byte

	ms.reqMu.Lock()
	if ms.reqReaders > _MAX_READERS {
		ms.reqMu.Unlock()
		return nil, OK
	}
	l := len(ms.reqPool)
	if l > 0 {
		req = ms.reqPool[l-1]
		ms.reqPool = ms.reqPool[:l-1]
	} else {
		req = new(request)
	}
	l = len(ms.readPool)
	if l > 0 {
		dest = ms.readPool[l-1]
		ms.readPool = ms.readPool[:l-1]
	} else {
		dest = make([]byte, ms.opts.MaxWrite + PAGESIZE)
	}
	ms.outstandingReadBufs++
	ms.reqReaders++
	ms.reqMu.Unlock()

	n, err := ms.mountFile.Read(dest)
	if err != nil {
		code = ToStatus(err)
		ms.reqMu.Lock()
		ms.reqPool = append(ms.reqPool, req)
		ms.reqReaders--
		ms.reqMu.Unlock()
		return nil, code
	}

	gobbled := req.setInput(dest[:n])

	ms.reqMu.Lock()
	if !gobbled {
		ms.outstandingReadBufs--
		ms.readPool = append(ms.readPool, dest)
		dest = nil
	}
	ms.reqReaders--
	if ms.reqReaders <= 0 {
		ms.loops.Add(1)
		go ms.loop(true)
	}
	ms.reqMu.Unlock()

	return req, OK
}

func (ms *MountState) returnRequest(req *request) {
	ms.recordStats(req)

	if req.bufferPoolOutputBuf != nil {
		ms.buffers.FreeBuffer(req.bufferPoolOutputBuf)
		req.bufferPoolOutputBuf = nil
	}

	req.clear()
	ms.reqMu.Lock()
	if req.bufferPoolOutputBuf != nil {
		ms.readPool = append(ms.readPool, req.bufferPoolInputBuf)
		ms.outstandingReadBufs--
		req.bufferPoolInputBuf = nil
	}
	ms.reqPool = append(ms.reqPool, req)
	ms.reqMu.Unlock()
}

func (ms *MountState) recordStats(req *request) {
	if ms.latencies != nil {
		endNs := time.Now().UnixNano()
		dt := endNs - req.startNs

		opname := operationName(req.inHeader.Opcode)
		ms.latencies.AddMany(
			[]LatencyArg{
				{opname, "", dt},
				{opname + "-write", "", endNs - req.preWriteNs}})
	}
}

// Loop initiates the FUSE loop. Normally, callers should run Loop()
// and wait for it to exit, but tests will want to run this in a
// goroutine.
//
// Each filesystem operation executes in a separate goroutine.
func (ms *MountState) Loop() {
	ms.loops.Add(1)
	ms.loop(false)
	ms.loops.Wait()
	ms.mountFile.Close()
}

func (ms *MountState) loop(exitIdle bool) {
	defer ms.loops.Done()
	exit:
	for {
		req, errNo := ms.readRequest(exitIdle)
		switch errNo {
		case OK:
			if req == nil {
				break exit
			}
		case ENOENT:
			continue
		case ENODEV:
			// unmount
			break exit
		default: // some other error?
			log.Printf("Failed to read from fuse conn: %v", errNo)
			break exit
		}

		if ms.latencies != nil {
			req.startNs = time.Now().UnixNano()
		}
		ms.handleRequest(req)
	}
}

func (ms *MountState) handleRequest(req *request) {
	defer ms.returnRequest(req)

	req.parse()
	if req.handler == nil {
		req.status = ENOSYS
	}

	if req.status.Ok() && ms.Debug {
		log.Println(req.InputDebug())
	}

	if req.status.Ok() && req.handler.Func == nil {
		log.Printf("Unimplemented opcode %v", operationName(req.inHeader.Opcode))
		req.status = ENOSYS
	}

	if req.status.Ok() {
		req.handler.Func(ms, req)
	}

	errNo := ms.write(req)
	if errNo != 0 {
		log.Printf("writer: Write/Writev failed, err: %v. opcode: %v",
			errNo, operationName(req.inHeader.Opcode))
	}
}

func (ms *MountState) AllocOut(req *request, size uint32) []byte {
	if req.bufferPoolOutputBuf != nil {
		ms.buffers.FreeBuffer(req.bufferPoolOutputBuf)
	}
	req.bufferPoolOutputBuf = ms.buffers.AllocBuffer(size)
	return req.bufferPoolOutputBuf
}

func (ms *MountState) write(req *request) Status {
	// Forget does not wait for reply.
	if req.inHeader.Opcode == _OP_FORGET || req.inHeader.Opcode == _OP_BATCH_FORGET {
		return OK
	}

	header, data := req.serialize()
	if ms.Debug {
		log.Println(req.OutputDebug())
	}

	if ms.latencies != nil {
		req.preWriteNs = time.Now().UnixNano()
	}

	if header == nil {
		return OK
	}
	var err error
	if data == nil {
		_, err = ms.mountFile.Write(header)
	} else {
		_, err = Writev(int(ms.mountFile.Fd()), [][]byte{header, data})
	}

	return ToStatus(err)
}

func (ms *MountState) writeInodeNotify(entry *raw.NotifyInvalInodeOut) Status {
	req := request{
		inHeader: &raw.InHeader{
			Opcode: _OP_NOTIFY_INODE,
		},
		handler: operationHandlers[_OP_NOTIFY_INODE],
		status:  raw.NOTIFY_INVAL_INODE,
	}
	req.outData = unsafe.Pointer(entry)
	result := ms.write(&req)

	if ms.Debug {
		log.Println("Response: INODE_NOTIFY", result)
	}
	return result
}

func (ms *MountState) writeEntryNotify(parent uint64, name string) Status {
	req := request{
		inHeader: &raw.InHeader{
			Opcode: _OP_NOTIFY_ENTRY,
		},
		handler: operationHandlers[_OP_NOTIFY_ENTRY],
		status:  raw.NOTIFY_INVAL_ENTRY,
	}
	entry := &raw.NotifyInvalEntryOut{
		Parent:  parent,
		NameLen: uint32(len(name)),
	}

	// Many versions of FUSE generate stacktraces if the
	// terminating null byte is missing.
	nameBytes := []byte(name + "\000")
	req.outData = unsafe.Pointer(entry)
	req.flatData = nameBytes
	result := ms.write(&req)

	if ms.Debug {
		log.Printf("Response: ENTRY_NOTIFY: %v", result)
	}
	return result
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fuseops

import (
	"bytes"
	"errors"
	"log"
	"time"
	"unsafe"

	"github.com/jacobsa/fuse/internal/fusekernel"
	"github.com/jacobsa/fuse/internal/fuseshim"
	"golang.org/x/net/context"
)

// This function is an implementation detail of the fuse package, and must not
// be called by anyone else.
//
// Convert the supplied fuse kernel message to an Op. sendReply will be used to
// send the reply back to the kernel once the user calls o.Respond. If the op
// is unknown, a special unexported type will be used.
//
// The debug logging function and error logger may be nil. The caller is
// responsible for arranging for the message to be destroyed.
func Convert(
	opCtx context.Context,
	m *fuseshim.Message,
	protocol fusekernel.Protocol,
	debugLogForOp func(int, string, ...interface{}),
	errorLogger *log.Logger,
	sendReply replyFunc) (o Op, err error) {
	var co *commonOp

	var io internalOp
	switch m.Hdr.Opcode {
	case fusekernel.OpLookup:
		buf := m.Bytes()
		n := len(buf)
		if n == 0 || buf[n-1] != '\x00' {
			err = errors.New("Corrupted OpLookup")
			return
		}

		to := &LookUpInodeOp{
			Parent: InodeID(m.Hdr.Nodeid),
			Name:   string(buf[:n-1]),
		}
		io = to
		co = &to.commonOp

	case fusekernel.OpGetattr:
		to := &GetInodeAttributesOp{
			Inode: InodeID(m.Hdr.Nodeid),
		}
		io = to
		co = &to.commonOp

	case fusekernel.OpSetattr:
		in := (*fusekernel.SetattrIn)(m.Data())
		if m.Len() < unsafe.Sizeof(*in) {
			err = errors.New("Corrupted OpSetattr")
			return
		}

		to := &SetInodeAttributesOp{
			Inode: InodeID(m.Hdr.Nodeid),
		}

		valid := fusekernel.SetattrValid(in.Valid)
		if valid&fusekernel.SetattrSize != 0 {
			to.Size = &in.Size
		}

		if valid&fusekernel.SetattrMode != 0 {
			mode := fuseshim.FileMode(in.Mode)
			to.Mode = &mode
		}

		if valid&fusekernel.SetattrAtime != 0 {
			t := time.Unix(int64(in.Atime), int64(in.AtimeNsec))
			to.Atime = &t
		}

		if valid&fusekernel.SetattrMtime != 0 {
			t := time.Unix(int64(in.Mtime), int64(in.MtimeNsec))
			to.Mtime = &t
		}

		io = to
		co = &to.commonOp

	case fusekernel.OpForget:
		in := (*fusekernel.ForgetIn)(m.Data())
		if m.Len() < unsafe.Sizeof(*in) {
			err = errors.New("Corrupted OpForget")
			return
		}

		to := &ForgetInodeOp{
			Inode: InodeID(m.Hdr.Nodeid),
			N:     in.Nlookup,
		}
		io = to
		co = &to.commonOp

	case fusekernel.OpMkdir:
		size := fusekernel.MkdirInSize(protocol)
		if m.Len() < size {
			err = errors.New("Corrupted OpMkdir")
			return
		}
		in := (*fusekernel.MkdirIn)(m.Data())
		name := m.Bytes()[size:]
		i := bytes.IndexByte(name, '\x00')
		if i < 0 {
			err = errors.New("Corrupted OpMkdir")
			return
		}
		name = name[:i]

		to := &MkDirOp{
			Parent: InodeID(m.Hdr.Nodeid),
			Name:   string(name),
			Mode:   fuseshim.FileMode(in.Mode),
		}
		io = to
		co = &to.commonOp

	case fusekernel.OpCreate:
		size := fusekernel.CreateInSize(protocol)
		if m.Len() < size {
			err = errors.New("Corrupted OpCreate")
			return
		}
		in := (*fusekernel.CreateIn)(m.Data())
		name := m.Bytes()[size:]
		i := bytes.IndexByte(name, '\x00')
		if i < 0 {
			err = errors.New("Corrupted OpCreate")
			return
		}
		name = name[:i]

		to := &CreateFileOp{
			Parent: InodeID(m.Hdr.Nodeid),
			Name:   string(name),
			Mode:   fuseshim.FileMode(in.Mode),
		}
		io = to
		co = &to.commonOp

	case fusekernel.OpSymlink:
		// m.Bytes() is "newName\0target\0"
		names := m.Bytes()
		if len(names) == 0 || names[len(names)-1] != 0 {
			err = errors.New("Corrupted OpSymlink")
			return
		}
		i := bytes.IndexByte(names, '\x00')
		if i < 0 {
			err = errors.New("Corrupted OpSymlink")
			return
		}
		newName, target := names[0:i], names[i+1:len(names)-1]

		to := &CreateSymlinkOp{
			Parent: InodeID(m.Hdr.Nodeid),
			Name:   string(newName),
			Target: string(target),
		}
		io = to
		co = &to.commonOp

	case fusekernel.OpRename:
		in := (*fusekernel.RenameIn)(m.Data())
		if m.Len() < unsafe.Sizeof(*in) {
			err = errors.New("Corrupted OpRename")
			return
		}
		names := m.Bytes()[unsafe.Sizeof(*in):]
		// names should be "old\x00new\x00"
		if len(names) < 4 {
			err = errors.New("Corrupted OpRename")
			return
		}
		if names[len(names)-1] != '\x00' {
			err = errors.New("Corrupted OpRename")
			return
		}
		i := bytes.IndexByte(names, '\x00')
		if i < 0 {
			err = errors.New("Corrupted OpRename")
			return
		}
		oldName, newName := names[:i], names[i+1:len(names)-1]

		to := &RenameOp{
			OldParent: InodeID(m.Hdr.Nodeid),
			OldName:   string(oldName),
			NewParent: InodeID(in.Newdir),
			NewName:   string(newName),
		}
		io = to
		co = &to.commonOp

	case fusekernel.OpUnlink:
		buf := m.Bytes()
		n := len(buf)
		if n == 0 || buf[n-1] != '\x00' {
			err = errors.New("Corrupted OpUnlink")
			return
		}

		to := &UnlinkOp{
			Parent: InodeID(m.Hdr.Nodeid),
			Name:   string(buf[:n-1]),
		}
		io = to
		co = &to.commonOp

	case fusekernel.OpRmdir:
		buf := m.Bytes()
		n := len(buf)
		if n == 0 || buf[n-1] != '\x00' {
			err = errors.New("Corrupted OpRmdir")
			return
		}

		to := &RmDirOp{
			Parent: InodeID(m.Hdr.Nodeid),
			Name:   string(buf[:n-1]),
		}
		io = to
		co = &to.commonOp

	case fusekernel.OpOpen:
		to := &OpenFileOp{
			Inode: InodeID(m.Hdr.Nodeid),
		}
		io = to
		co = &to.commonOp

	case fusekernel.OpOpendir:
		to := &OpenDirOp{
			Inode: InodeID(m.Hdr.Nodeid),
		}
		io = to
		co = &to.commonOp

	case fusekernel.OpRead:
		in := (*fusekernel.ReadIn)(m.Data())
		if m.Len() < fusekernel.ReadInSize(protocol) {
			err = errors.New("Corrupted OpRead")
			return
		}

		to := &ReadFileOp{
			Inode:  InodeID(m.Hdr.Nodeid),
			Handle: HandleID(in.Fh),
			Offset: int64(in.Offset),
			Size:   int(in.Size),
		}
		io = to
		co = &to.commonOp

	case fusekernel.OpReaddir:
		in := (*fusekernel.ReadIn)(m.Data())
		if m.Len() < fusekernel.ReadInSize(protocol) {
			err = errors.New("Corrupted OpReaddir")
			return
		}

		to := &ReadDirOp{
			Inode:  InodeID(m.Hdr.Nodeid),
			Handle: HandleID(in.Fh),
			Offset: DirOffset(in.Offset),
			Size:   int(in.Size),
		}
		io = to
		co = &to.commonOp

	case fusekernel.OpRelease:
		in := (*fusekernel.ReleaseIn)(m.Data())
		if m.Len() < unsafe.Sizeof(*in) {
			err = errors.New("Corrupted OpRelease")
			return
		}

		to := &ReleaseFileHandleOp{
			Handle: HandleID(in.Fh),
		}
		io = to
		co = &to.commonOp

	case fusekernel.OpReleasedir:
		in := (*fusekernel.ReleaseIn)(m.Data())
		if m.Len() < unsafe.Sizeof(*in) {
			err = errors.New("Corrupted OpReleasedir")
			return
		}

		to := &ReleaseDirHandleOp{
			Handle: HandleID(in.Fh),
		}
		io = to
		co = &to.commonOp

	case fusekernel.OpWrite:
		in := (*fusekernel.WriteIn)(m.Data())
		size := fusekernel.WriteInSize(protocol)
		if m.Len() < size {
			err = errors.New("Corrupted OpWrite")
			return
		}

		buf := m.Bytes()[size:]
		if len(buf) < int(in.Size) {
			err = errors.New("Corrupted OpWrite")
			return
		}

		to := &WriteFileOp{
			Inode:  InodeID(m.Hdr.Nodeid),
			Handle: HandleID(in.Fh),
			Data:   buf,
			Offset: int64(in.Offset),
		}
		io = to
		co = &to.commonOp

	case fusekernel.OpFsync:
		in := (*fusekernel.FsyncIn)(m.Data())
		if m.Len() < unsafe.Sizeof(*in) {
			err = errors.New("Corrupted OpFsync")
			return
		}

		to := &SyncFileOp{
			Inode:  InodeID(m.Hdr.Nodeid),
			Handle: HandleID(in.Fh),
		}
		io = to
		co = &to.commonOp

	case fusekernel.OpFlush:
		in := (*fusekernel.FlushIn)(m.Data())
		if m.Len() < unsafe.Sizeof(*in) {
			err = errors.New("Corrupted OpFlush")
			return
		}

		to := &FlushFileOp{
			Inode:  InodeID(m.Hdr.Nodeid),
			Handle: HandleID(in.Fh),
		}
		io = to
		co = &to.commonOp

	case fusekernel.OpReadlink:
		to := &ReadSymlinkOp{
			Inode: InodeID(m.Hdr.Nodeid),
		}
		io = to
		co = &to.commonOp

	default:
		to := &unknownOp{
			opCode: m.Hdr.Opcode,
			inode:  InodeID(m.Hdr.Nodeid),
		}
		io = to
		co = &to.commonOp
	}

	co.init(
		opCtx,
		io,
		m.Hdr.Unique,
		sendReply,
		debugLogForOp,
		errorLogger)

	o = io
	return
}

func convertAttributes(
	inode InodeID,
	attr InodeAttributes,
	expiration time.Time) fuseshim.Attr {
	return fuseshim.Attr{
		Inode:  uint64(inode),
		Size:   attr.Size,
		Mode:   attr.Mode,
		Nlink:  uint32(attr.Nlink),
		Atime:  attr.Atime,
		Mtime:  attr.Mtime,
		Ctime:  attr.Ctime,
		Crtime: attr.Crtime,
		Uid:    attr.Uid,
		Gid:    attr.Gid,
		Valid:  convertExpirationTime(expiration),
	}
}

// Convert an absolute cache expiration time to a relative time from now for
// consumption by fuse.
func convertExpirationTime(t time.Time) (d time.Duration) {
	// Fuse represents durations as unsigned 64-bit counts of seconds and 32-bit
	// counts of nanoseconds (cf. http://goo.gl/EJupJV). The bazil.org/fuse
	// package converts time.Duration values to this form in a straightforward
	// way (cf. http://goo.gl/FJhV8j).
	//
	// So negative durations are right out. There is no need to cap the positive
	// magnitude, because 2^64 seconds is well longer than the 2^63 ns range of
	// time.Duration.
	d = t.Sub(time.Now())
	if d < 0 {
		d = 0
	}

	return
}

func convertChildInodeEntry(
	in *ChildInodeEntry,
	out *fuseshim.LookupResponse) {
	out.Node = fuseshim.NodeID(in.Child)
	out.Generation = uint64(in.Generation)
	out.Attr = convertAttributes(in.Child, in.Attributes, in.AttributesExpiration)
	out.EntryValid = convertExpirationTime(in.EntryExpiration)
}
<commit_msg>Added Convert support for the internal ops.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fuseops

import (
	"bytes"
	"errors"
	"log"
	"time"
	"unsafe"

	"github.com/jacobsa/fuse/internal/fusekernel"
	"github.com/jacobsa/fuse/internal/fuseshim"
	"golang.org/x/net/context"
)

// This function is an implementation detail of the fuse package, and must not
// be called by anyone else.
//
// Convert the supplied fuse kernel message to an Op. sendReply will be used to
// send the reply back to the kernel once the user calls o.Respond. If the op
// is unknown, a special unexported type will be used.
//
// The debug logging function and error logger may be nil. The caller is
// responsible for arranging for the message to be destroyed.
func Convert(
	opCtx context.Context,
	m *fuseshim.Message,
	protocol fusekernel.Protocol,
	debugLogForOp func(int, string, ...interface{}),
	errorLogger *log.Logger,
	sendReply replyFunc) (o Op, err error) {
	var co *commonOp

	var io internalOp
	switch m.Hdr.Opcode {
	case fusekernel.OpLookup:
		buf := m.Bytes()
		n := len(buf)
		if n == 0 || buf[n-1] != '\x00' {
			err = errors.New("Corrupted OpLookup")
			return
		}

		to := &LookUpInodeOp{
			Parent: InodeID(m.Hdr.Nodeid),
			Name:   string(buf[:n-1]),
		}
		io = to
		co = &to.commonOp

	case fusekernel.OpGetattr:
		to := &GetInodeAttributesOp{
			Inode: InodeID(m.Hdr.Nodeid),
		}
		io = to
		co = &to.commonOp

	case fusekernel.OpSetattr:
		in := (*fusekernel.SetattrIn)(m.Data())
		if m.Len() < unsafe.Sizeof(*in) {
			err = errors.New("Corrupted OpSetattr")
			return
		}

		to := &SetInodeAttributesOp{
			Inode: InodeID(m.Hdr.Nodeid),
		}

		valid := fusekernel.SetattrValid(in.Valid)
		if valid&fusekernel.SetattrSize != 0 {
			to.Size = &in.Size
		}

		if valid&fusekernel.SetattrMode != 0 {
			mode := fuseshim.FileMode(in.Mode)
			to.Mode = &mode
		}

		if valid&fusekernel.SetattrAtime != 0 {
			t := time.Unix(int64(in.Atime), int64(in.AtimeNsec))
			to.Atime = &t
		}

		if valid&fusekernel.SetattrMtime != 0 {
			t := time.Unix(int64(in.Mtime), int64(in.MtimeNsec))
			to.Mtime = &t
		}

		io = to
		co = &to.commonOp

	case fusekernel.OpForget:
		in := (*fusekernel.ForgetIn)(m.Data())
		if m.Len() < unsafe.Sizeof(*in) {
			err = errors.New("Corrupted OpForget")
			return
		}

		to := &ForgetInodeOp{
			Inode: InodeID(m.Hdr.Nodeid),
			N:     in.Nlookup,
		}
		io = to
		co = &to.commonOp

	case fusekernel.OpMkdir:
		size := fusekernel.MkdirInSize(protocol)
		if m.Len() < size {
			err = errors.New("Corrupted OpMkdir")
			return
		}
		in := (*fusekernel.MkdirIn)(m.Data())
		name := m.Bytes()[size:]
		i := bytes.IndexByte(name, '\x00')
		if i < 0 {
			err = errors.New("Corrupted OpMkdir")
			return
		}
		name = name[:i]

		to := &MkDirOp{
			Parent: InodeID(m.Hdr.Nodeid),
			Name:   string(name),
			Mode:   fuseshim.FileMode(in.Mode),
		}
		io = to
		co = &to.commonOp

	case fusekernel.OpCreate:
		size := fusekernel.CreateInSize(protocol)
		if m.Len() < size {
			err = errors.New("Corrupted OpCreate")
			return
		}
		in := (*fusekernel.CreateIn)(m.Data())
		name := m.Bytes()[size:]
		i := bytes.IndexByte(name, '\x00')
		if i < 0 {
			err = errors.New("Corrupted OpCreate")
			return
		}
		name = name[:i]

		to := &CreateFileOp{
			Parent: InodeID(m.Hdr.Nodeid),
			Name:   string(name),
			Mode:   fuseshim.FileMode(in.Mode),
		}
		io = to
		co = &to.commonOp

	case fusekernel.OpSymlink:
		// m.Bytes() is "newName\0target\0"
		names := m.Bytes()
		if len(names) == 0 || names[len(names)-1] != 0 {
			err = errors.New("Corrupted OpSymlink")
			return
		}
		i := bytes.IndexByte(names, '\x00')
		if i < 0 {
			err = errors.New("Corrupted OpSymlink")
			return
		}
		newName, target := names[0:i], names[i+1:len(names)-1]

		to := &CreateSymlinkOp{
			Parent: InodeID(m.Hdr.Nodeid),
			Name:   string(newName),
			Target: string(target),
		}
		io = to
		co = &to.commonOp

	case fusekernel.OpRename:
		in := (*fusekernel.RenameIn)(m.Data())
		if m.Len() < unsafe.Sizeof(*in) {
			err = errors.New("Corrupted OpRename")
			return
		}
		names := m.Bytes()[unsafe.Sizeof(*in):]
		// names should be "old\x00new\x00"
		if len(names) < 4 {
			err = errors.New("Corrupted OpRename")
			return
		}
		if names[len(names)-1] != '\x00' {
			err = errors.New("Corrupted OpRename")
			return
		}
		i := bytes.IndexByte(names, '\x00')
		if i < 0 {
			err = errors.New("Corrupted OpRename")
			return
		}
		oldName, newName := names[:i], names[i+1:len(names)-1]

		to := &RenameOp{
			OldParent: InodeID(m.Hdr.Nodeid),
			OldName:   string(oldName),
			NewParent: InodeID(in.Newdir),
			NewName:   string(newName),
		}
		io = to
		co = &to.commonOp

	case fusekernel.OpUnlink:
		buf := m.Bytes()
		n := len(buf)
		if n == 0 || buf[n-1] != '\x00' {
			err = errors.New("Corrupted OpUnlink")
			return
		}

		to := &UnlinkOp{
			Parent: InodeID(m.Hdr.Nodeid),
			Name:   string(buf[:n-1]),
		}
		io = to
		co = &to.commonOp

	case fusekernel.OpRmdir:
		buf := m.Bytes()
		n := len(buf)
		if n == 0 || buf[n-1] != '\x00' {
			err = errors.New("Corrupted OpRmdir")
			return
		}

		to := &RmDirOp{
			Parent: InodeID(m.Hdr.Nodeid),
			Name:   string(buf[:n-1]),
		}
		io = to
		co = &to.commonOp

	case fusekernel.OpOpen:
		to := &OpenFileOp{
			Inode: InodeID(m.Hdr.Nodeid),
		}
		io = to
		co = &to.commonOp

	case fusekernel.OpOpendir:
		to := &OpenDirOp{
			Inode: InodeID(m.Hdr.Nodeid),
		}
		io = to
		co = &to.commonOp

	case fusekernel.OpRead:
		in := (*fusekernel.ReadIn)(m.Data())
		if m.Len() < fusekernel.ReadInSize(protocol) {
			err = errors.New("Corrupted OpRead")
			return
		}

		to := &ReadFileOp{
			Inode:  InodeID(m.Hdr.Nodeid),
			Handle: HandleID(in.Fh),
			Offset: int64(in.Offset),
			Size:   int(in.Size),
		}
		io = to
		co = &to.commonOp

	case fusekernel.OpReaddir:
		in := (*fusekernel.ReadIn)(m.Data())
		if m.Len() < fusekernel.ReadInSize(protocol) {
			err = errors.New("Corrupted OpReaddir")
			return
		}

		to := &ReadDirOp{
			Inode:  InodeID(m.Hdr.Nodeid),
			Handle: HandleID(in.Fh),
			Offset: DirOffset(in.Offset),
			Size:   int(in.Size),
		}
		io = to
		co = &to.commonOp

	case fusekernel.OpRelease:
		in := (*fusekernel.ReleaseIn)(m.Data())
		if m.Len() < unsafe.Sizeof(*in) {
			err = errors.New("Corrupted OpRelease")
			return
		}

		to := &ReleaseFileHandleOp{
			Handle: HandleID(in.Fh),
		}
		io = to
		co = &to.commonOp

	case fusekernel.OpReleasedir:
		in := (*fusekernel.ReleaseIn)(m.Data())
		if m.Len() < unsafe.Sizeof(*in) {
			err = errors.New("Corrupted OpReleasedir")
			return
		}

		to := &ReleaseDirHandleOp{
			Handle: HandleID(in.Fh),
		}
		io = to
		co = &to.commonOp

	case fusekernel.OpWrite:
		in := (*fusekernel.WriteIn)(m.Data())
		size := fusekernel.WriteInSize(protocol)
		if m.Len() < size {
			err = errors.New("Corrupted OpWrite")
			return
		}

		buf := m.Bytes()[size:]
		if len(buf) < int(in.Size) {
			err = errors.New("Corrupted OpWrite")
			return
		}

		to := &WriteFileOp{
			Inode:  InodeID(m.Hdr.Nodeid),
			Handle: HandleID(in.Fh),
			Data:   buf,
			Offset: int64(in.Offset),
		}
		io = to
		co = &to.commonOp

	case fusekernel.OpFsync:
		in := (*fusekernel.FsyncIn)(m.Data())
		if m.Len() < unsafe.Sizeof(*in) {
			err = errors.New("Corrupted OpFsync")
			return
		}

		to := &SyncFileOp{
			Inode:  InodeID(m.Hdr.Nodeid),
			Handle: HandleID(in.Fh),
		}
		io = to
		co = &to.commonOp

	case fusekernel.OpFlush:
		in := (*fusekernel.FlushIn)(m.Data())
		if m.Len() < unsafe.Sizeof(*in) {
			err = errors.New("Corrupted OpFlush")
			return
		}

		to := &FlushFileOp{
			Inode:  InodeID(m.Hdr.Nodeid),
			Handle: HandleID(in.Fh),
		}
		io = to
		co = &to.commonOp

	case fusekernel.OpReadlink:
		to := &ReadSymlinkOp{
			Inode: InodeID(m.Hdr.Nodeid),
		}
		io = to
		co = &to.commonOp

	case fusekernel.OpStatfs:
		to := &InternalStatFSOp{}
		io = to
		co = &to.commonOp

	case fusekernel.OpInterrupt:
		in := (*fusekernel.InterruptIn)(m.Data())
		if m.Len() < unsafe.Sizeof(*in) {
			err = errors.New("Corrupted OpInterrupt")
			return
		}

		to := &InternalInterruptOp{
			FuseID: in.Unique,
		}
		io = to
		co = &to.commonOp

	default:
		to := &unknownOp{
			opCode: m.Hdr.Opcode,
			inode:  InodeID(m.Hdr.Nodeid),
		}
		io = to
		co = &to.commonOp
	}

	co.init(
		opCtx,
		io,
		m.Hdr.Unique,
		sendReply,
		debugLogForOp,
		errorLogger)

	o = io
	return
}

func convertAttributes(
	inode InodeID,
	attr InodeAttributes,
	expiration time.Time) fuseshim.Attr {
	return fuseshim.Attr{
		Inode:  uint64(inode),
		Size:   attr.Size,
		Mode:   attr.Mode,
		Nlink:  uint32(attr.Nlink),
		Atime:  attr.Atime,
		Mtime:  attr.Mtime,
		Ctime:  attr.Ctime,
		Crtime: attr.Crtime,
		Uid:    attr.Uid,
		Gid:    attr.Gid,
		Valid:  convertExpirationTime(expiration),
	}
}

// Convert an absolute cache expiration time to a relative time from now for
// consumption by fuse.
func convertExpirationTime(t time.Time) (d time.Duration) {
	// Fuse represents durations as unsigned 64-bit counts of seconds and 32-bit
	// counts of nanoseconds (cf. http://goo.gl/EJupJV). The bazil.org/fuse
	// package converts time.Duration values to this form in a straightforward
	// way (cf. http://goo.gl/FJhV8j).
	//
	// So negative durations are right out. There is no need to cap the positive
	// magnitude, because 2^64 seconds is well longer than the 2^63 ns range of
	// time.Duration.
	d = t.Sub(time.Now())
	if d < 0 {
		d = 0
	}

	return
}

func convertChildInodeEntry(
	in *ChildInodeEntry,
	out *fuseshim.LookupResponse) {
	out.Node = fuseshim.NodeID(in.Child)
	out.Generation = uint64(in.Generation)
	out.Attr = convertAttributes(in.Child, in.Attributes, in.AttributesExpiration)
	out.EntryValid = convertExpirationTime(in.EntryExpiration)
}
<|endoftext|>


<commit_before>/*
The Venue package exposes the Avid™ VENUE VNC interface as a programmatic API.
*/
package venue

import (
	"flag"
	"fmt"
	"image"
	"log"
	"net"
	"strconv"
	"time"

	vnc "github.com/kward/go-vnc"
	"golang.org/x/net/context"
)

const (
	errPrefix = "Venue error."
	numInputs = 48
)

var (
	refresh = flag.Duration("venue_refresh", 1000*time.Millisecond, "framebuffer refresh period.")
	timeout = flag.Duration("venue_timeout", 10*time.Second, "timeout for Venue connection.")
	debug   = flag.Bool("venue_debug", false, "enable debugging output")
)

// Venue holds information representing the state of the VENUE backend.
type Venue struct {
	host string
	port uint
	cfg  *vnc.ClientConfig
	conn *vnc.ClientConn
	fb   *Framebuffer

	inputs    [numInputs]*Input
	currInput *Input

	outputs    map[string]*Output
	currOutput *Output

	Pages    VenuePages
	currPage int
}

// NewVenue returns a populated Venue struct.
func NewVenue(host string, port uint, passwd string) *Venue {
	cfg := vnc.NewClientConfig(passwd)
	return &Venue{host: host, port: port, cfg: cfg}
}

// Connect to a VENUE console.
func (v *Venue) Connect(ctx context.Context) error {
	if v.conn != nil {
		return fmt.Errorf("%v Already connected.", errPrefix)
	}

	log.Println("Connecting...")
	addr := v.host + ":" + strconv.FormatUint(uint64(v.port), 10)
	netConn, err := net.DialTimeout("tcp", addr, *timeout)
	if err != nil {
		return fmt.Errorf("%v Error connecting to host. %v", errPrefix, err)
	}

	var cancel context.CancelFunc
	if _, ok := ctx.Deadline(); ok {
		ctx, cancel = context.WithCancel(ctx)
	} else {
		ctx, cancel = context.WithTimeout(ctx, *timeout)
	}
	defer cancel()

	log.Println("Establishing...")
	ctx = context.WithValue(ctx, "debug", *debug)
	vncConn, err := vnc.Connect(ctx, netConn, v.cfg)
	if err != nil {
		return fmt.Errorf("%v Could not establish session. %v", errPrefix, err)
	}
	v.conn = vncConn
	return nil
}

// Close a connection to a VENUE console.
func (v *Venue) Close() error {
	return v.conn.Close()
}

// Initialize the in-memory state representation of a VENUE console.
func (v *Venue) Initialize() {
	// Create image to apply framebuffer updates to.
	v.fb = NewFramebuffer(int(v.conn.FramebufferHeight()), int(v.conn.FramebufferHeight()))

	// Setup channel to listen to server messages.
	v.cfg.ServerMessageCh = make(chan vnc.ServerMessage)

	// Initialize pages.
	v.Pages = VenuePages{}
	v.Pages[InputsPage] = NewInputsPage()
	v.Pages[OutputsPage] = NewOutputsPage()
	// Initialize inputs.
	for ch := 0; ch < numInputs; ch++ {
		input := NewInput(v, ch+1, Ichannel)
		v.inputs[ch] = input
	}

	// Choose something besides input page, so that later when the Inputs page is
	// selected, it shows first bank of channels.
	v.SetPage(OptionsPage)
	v.SetInput(1)

	// Clear solo.
	log.Println("Clearing solo.")
	vp := v.Pages[InputsPage]
	e := vp.Elements["solo_clear"]
	e.(*Switch).Update(v)
}

// ListenAndHandle VNC server messages.
func (v *Venue) ListenAndHandle() {
	go v.conn.ListenAndHandle()
	for {
		msg := <-v.cfg.ServerMessageCh
		switch msg.Type() {
		case vnc.FramebufferUpdateMsg:
			//log.Println("ListenAndHandle() FramebufferUpdateMessage")
			for i := uint16(0); i < msg.(*vnc.FramebufferUpdate).NumRect; i++ {
				var colors []vnc.Color
				rect := msg.(*vnc.FramebufferUpdate).Rects[i]
				switch rect.Enc.Type() {
				case vnc.Raw:
					colors = rect.Enc.(*vnc.RawEncoding).Colors
				}
				v.fb.Paint(v, rect, colors)
			}

		default:
			log.Printf("ListenAndHandle() unknown message type:%v msg:%v\n", msg.Type(), msg)
		}
	}
}

// FramebufferRefresh refreshes the local framebuffer image of the VNC server.
func (v *Venue) FramebufferRefresh() {
	screen := image.Rectangle{image.Point{0, 0}, image.Point{v.fb.Width, v.fb.Height}}
	for {
		if err := v.Snapshot(screen); err != nil {
			log.Fatal(err)
		}
		time.Sleep(*refresh)
	}
}

// Snapshot requests updated image info from the VNC server.
func (v *Venue) Snapshot(r image.Rectangle) error {
	//log.Printf("Snapshot(%v)\n", r)
	w, h := uint16(r.Max.X-r.Min.X), uint16(r.Max.Y-r.Min.Y)
	if err := v.conn.FramebufferUpdateRequest(
		vnc.RFBTrue, uint16(r.Min.X), uint16(r.Min.Y), w, h); err != nil {
		log.Println("Snapshot() error:", err)
		return err
	}
	return nil
}

// abs returns the absolute value of an int.
func abs(x int) int {
	switch {
	case x < 0:
		return -x
	case x == 0:
		return 0
	}
	return x
}
<commit_msg>fixed bug in creating new framebuffer; more logging<commit_after>/*
The Venue package exposes the Avid™ VENUE VNC interface as a programmatic API.
*/
package venue

import (
	"flag"
	"fmt"
	"image"
	"log"
	"net"
	"strconv"
	"time"

	vnc "github.com/kward/go-vnc"
	"golang.org/x/net/context"
)

const (
	errPrefix = "Venue error."
	numInputs = 48
)

var (
	refresh = flag.Duration("venue_refresh", 1000*time.Millisecond, "framebuffer refresh period.")
	timeout = flag.Duration("venue_timeout", 10*time.Second, "timeout for Venue connection.")
	debug   = flag.Bool("venue_debug", false, "enable debugging output")
)

// Venue holds information representing the state of the VENUE backend.
type Venue struct {
	host string
	port uint
	cfg  *vnc.ClientConfig
	conn *vnc.ClientConn
	fb   *Framebuffer

	inputs    [numInputs]*Input
	currInput *Input

	outputs    map[string]*Output
	currOutput *Output

	Pages    VenuePages
	currPage int
}

// NewVenue returns a populated Venue struct.
func NewVenue(host string, port uint, passwd string) *Venue {
	cfg := vnc.NewClientConfig(passwd)
	return &Venue{host: host, port: port, cfg: cfg}
}

// Connect to a VENUE console.
func (v *Venue) Connect(ctx context.Context) error {
	if v.conn != nil {
		return fmt.Errorf("%v Already connected.", errPrefix)
	}

	log.Println("Connecting...")
	addr := v.host + ":" + strconv.FormatUint(uint64(v.port), 10)
	netConn, err := net.DialTimeout("tcp", addr, *timeout)
	if err != nil {
		return fmt.Errorf("%v Error connecting to host. %v", errPrefix, err)
	}

	var cancel context.CancelFunc
	if _, ok := ctx.Deadline(); ok {
		ctx, cancel = context.WithCancel(ctx)
	} else {
		ctx, cancel = context.WithTimeout(ctx, *timeout)
	}
	defer cancel()

	log.Println("Establishing...")
	ctx = context.WithValue(ctx, "debug", *debug)
	vncConn, err := vnc.Connect(ctx, netConn, v.cfg)
	if err != nil {
		return fmt.Errorf("%v Could not establish session. %v", errPrefix, err)
	}
	v.conn = vncConn
	return nil
}

// Close a connection to a VENUE console.
func (v *Venue) Close() error {
	return v.conn.Close()
}

// Initialize the in-memory state representation of a VENUE console.
func (v *Venue) Initialize() {
	// Create image to apply framebuffer updates to.
	v.fb = NewFramebuffer(int(v.conn.FramebufferWidth()), int(v.conn.FramebufferHeight()))

	// Setup channel to listen to server messages.
	v.cfg.ServerMessageCh = make(chan vnc.ServerMessage)

	// Initialize pages.
	v.Pages = VenuePages{}
	v.Pages[InputsPage] = NewInputsPage()
	v.Pages[OutputsPage] = NewOutputsPage()
	// Initialize inputs.
	for ch := 0; ch < numInputs; ch++ {
		input := NewInput(v, ch+1, Ichannel)
		v.inputs[ch] = input
	}

	// Choose something besides input page, so that later when the Inputs page is
	// selected, it shows first bank of channels.
	v.SetPage(OptionsPage)
	v.SetInput(1)

	// Clear solo.
	log.Println("Clearing solo.")
	vp := v.Pages[InputsPage]
	e := vp.Elements["solo_clear"]
	e.(*Switch).Update(v)
}

// ListenAndHandle VNC server messages.
func (v *Venue) ListenAndHandle() {
	log.Println("ListenAndHandle()")
	go v.conn.ListenAndHandle()
	for {
		msg := <-v.cfg.ServerMessageCh
		switch msg.Type() {
		case vnc.FramebufferUpdateMsg:
			log.Println("ListenAndHandle() FramebufferUpdateMessage")
			for i := uint16(0); i < msg.(*vnc.FramebufferUpdate).NumRect; i++ {
				var colors []vnc.Color
				rect := msg.(*vnc.FramebufferUpdate).Rects[i]
				switch rect.Enc.Type() {
				case vnc.Raw:
					colors = rect.Enc.(*vnc.RawEncoding).Colors
				}
				v.fb.Paint(v, rect, colors)
			}

		default:
			log.Printf("ListenAndHandle() unknown message type:%v msg:%v\n", msg.Type(), msg)
		}
	}
}

// FramebufferRefresh refreshes the local framebuffer image of the VNC server.
func (v *Venue) FramebufferRefresh() {
	screen := image.Rectangle{image.Point{0, 0}, image.Point{v.fb.Width, v.fb.Height}}
	for {
		if err := v.Snapshot(screen); err != nil {
			log.Fatal(err)
		}
		time.Sleep(*refresh)
	}
}

// Snapshot requests updated image info from the VNC server.
func (v *Venue) Snapshot(r image.Rectangle) error {
	log.Printf("Snapshot(%v)\n", r)
	w, h := uint16(r.Max.X-r.Min.X), uint16(r.Max.Y-r.Min.Y)
	if err := v.conn.FramebufferUpdateRequest(
		vnc.RFBTrue, uint16(r.Min.X), uint16(r.Min.Y), w, h); err != nil {
		log.Println("Snapshot() error:", err)
		return err
	}
	return nil
}

// abs returns the absolute value of an int.
func abs(x int) int {
	switch {
	case x < 0:
		return -x
	case x == 0:
		return 0
	}
	return x
}
<|endoftext|>


<commit_before>package gitbackup

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
)

type repository struct {
	name     string
	cloneURL string
}

// BackupTarget backs up an entity that holds one or more git repositories and
// has an interface to retrieve that list of repositories.
// Examples of entities include:
//   - A GitHub user.
//   - A BitBucket user.
//   - A GitHub organization.
func BackupTarget(target Target, backupDirectory string) error {
	fmt.Printf("########## Backing up target %s ##########\n\n", target.Name)

	// Retrieve a list of all the git repositories available from the target.
	var repoList []repository
	var err error
	switch target.Source {
	case "github":
		repoList, err = getGitHubRepoList(target, backupDirectory)
	case "bitbucket":
		repoList, err = getBitBucketRepoList(target, backupDirectory)
	default:
		err = fmt.Errorf(`"%s" is not a recognized source type`, target.Source)
	}
	if err != nil {
		return err
	}

	// Back up each repository found.
	for _, repo := range repoList {
		fmt.Println(fmt.Sprintf("#> %s", repo.name))
		if includeRepository(repo.name, target) {
			backupRepository(
				target.Name,
				repo.name,
				repo.cloneURL,
				backupDirectory,
			)
			fmt.Println("")
		} else {
			fmt.Print("Skipped.\n\n")
		}
	}

	fmt.Println("")

	return nil
}

// getGitHubRepoList finds all the repositories belonging to a given user or
// organization on GitHub.
func getGitHubRepoList(target Target, backupDirectory string) ([]repository, error) {
	// Create URL to request list of repos.
	requestURL := fmt.Sprintf(
		"https://api.github.com/%s/%s/repos?access_token=%s&per_page=200",
		target.Type,
		target.Entity,
		target.Token,
	)

	// Retrieve list of repositories.
	response, err := http.Get(requestURL)
	if err != nil {
		return nil, fmt.Errorf("Failed to connect with the source to retrieve the list of repositories: %s", err)
	}
	defer response.Body.Close()
	contents, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, fmt.Errorf("Failed to retrieve the list of repositories: %s", err)
	}

	// Parse JSON response.
	var data []map[string]interface{}
	if err := json.Unmarshal(contents, &data); err != nil {
		return nil, fmt.Errorf("Failed to parse JSON: %s", err)
	}

	// Make a list of repositories.
	var repoList []repository
	for _, repo := range data {
		repoName, _ := repo["name"].(string)
		cloneURL, _ := repo["clone_url"].(string)
		cloneURL = strings.Replace(
			cloneURL,
			"https://",
			fmt.Sprintf("https://%s:%s@", target.Entity, target.Token),
			1,
		)
		repo := repository{name: repoName, cloneURL: cloneURL}
		repoList = append(repoList,repo)
	}

	// No errors.
	return repoList, nil
}

// getBitBucketRepoList finds all the repositories belonging to a given user on
// BitBucket.
func getBitBucketRepoList(target Target, backupDirectory string) ([]repository, error) {
	// Create URL to request list of repos.
	// TODO: support pagination.
	requestURL := fmt.Sprintf(
		"https://%s:%s@bitbucket.org/api/2.0/repositories/%s?page=1&pagelen=100",
		target.Entity,
		target.Password,
		target.Entity,
	)

	// Retrieve list of repositories.
	response, err := http.Get(requestURL)
	if err != nil {
		return nil, fmt.Errorf("Failed to connect with the source to retrieve the list of repositories: %s", err)
	}
	defer response.Body.Close()
	contents, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, fmt.Errorf("Failed to retrieve the list of repositories: %s", err)
	}

	// Parse JSON response.
	var metadata map[string]json.RawMessage
	if err := json.Unmarshal(contents, &metadata); err != nil {
		return nil, fmt.Errorf("Failed to parse JSON: %s", err)
	}
	var data []map[string]json.RawMessage
	if err := json.Unmarshal(metadata["values"], &data); err != nil {
		return nil, fmt.Errorf("Failed to parse JSON: %s", err)
	}

	// Make a list of repositories.
	repoList := make([]repository, len(data))
	for i, repo := range data {
		// Parse the remaining JSON message that pertains to this repository.
		var repoName string
		if err := json.Unmarshal(repo["name"], &repoName); err != nil {
			return nil, fmt.Errorf("Failed to parse JSON: %s", err)
		}
		var links map[string]json.RawMessage
		if err := json.Unmarshal(repo["links"], &links); err != nil {
			return nil, fmt.Errorf("Failed to parse JSON: %s", err)
		}
		var cloneLinks []map[string]string
		if err := json.Unmarshal(links["clone"], &cloneLinks); err != nil {
			return nil, fmt.Errorf("Failed to parse JSON: %s", err)
		}

		// Find the https URL to use for cloning.
		var cloneURL string
		for _, link := range cloneLinks {
			if link["name"] == "https" {
				cloneURL = link["href"]
			}
		}
		if cloneURL == "" {
			return nil, fmt.Errorf("Could not determine HTTPS cloning URL: %s", cloneLinks)
		}

		// Determine URL for cloning.
		cloneURL = strings.Replace(
			cloneURL,
			fmt.Sprintf("https://%s@", target.Entity),
			fmt.Sprintf("https://%s:%s@", target.Entity, target.Password),
			1,
		)

		repoList[i] = repository{name: repoName, cloneURL: cloneURL}
	}

	// No errors.
	return repoList, nil
}

// backupRepository takes a remote git repository and backs it up locally.
// Note that this makes a mirror repository - in other words, the backup only
// contains the content of a normal .git repository but no working directory,
// which saves space. You can always get a normal repository from the backup by
// doing a normal git clone of the backup itself.
func backupRepository(targetName string, repoName string, cloneURL string, backupDirectory string) {
	cloneDirectory := filepath.Join(backupDirectory, targetName, repoName)

	if _, err := os.Stat(cloneDirectory); os.IsNotExist(err) {
		// The repo doesn't exist locally, clone it.
		cmd := exec.Command("git", "clone", "--mirror", cloneURL, cloneDirectory)
		cmdOut, err := cmd.CombinedOutput()
		if err != nil {
			fmt.Println("Error cloning the repository:", err)
			return
		}
		if len(cmdOut) > 0 {
			fmt.Printf(string(cmdOut))
		}
		fmt.Println("Cloned repository.")
	} else {
		// The repo already exists, pull updates.
		cmd := exec.Command("git", "remote", "update", "-p")
		cmd.Dir = cloneDirectory
		cmdOut, err := cmd.CombinedOutput()
		if err != nil {
			fmt.Println("Error pulling in the repository:", err)
		} else {
			// Display pulled information.
			if len(cmdOut) > 0 {
				fmt.Printf(string(cmdOut))
			}
			fmt.Println("Pulled latest updates in the repository.")
		}
	}
}

// includeRepository takes a repository name and the information about the
// target it is part of, and determines whether the repository should be backed
// up or not.
func includeRepository(repoName string, target Target) bool {
	if target.Skip != "" {
		r, err := regexp.Compile(target.Skip)
		if err != nil {
			log.Fatalf(`"skip" does not specify a valid regular expression: %s`, err)
		}
		return r.MatchString(repoName) == false
	}
	if target.Only != "" {
		r, err := regexp.Compile(target.Only)
		if err != nil {
			log.Fatalf(`"only" does not specify a valid regular expression: %s`, err)
		}
		return r.MatchString(repoName) == true
	}

	return true
}
<commit_msg>Set origin again in case credentials have changed.<commit_after>package gitbackup

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
)

type repository struct {
	name     string
	cloneURL string
}

// BackupTarget backs up an entity that holds one or more git repositories and
// has an interface to retrieve that list of repositories.
// Examples of entities include:
//   - A GitHub user.
//   - A BitBucket user.
//   - A GitHub organization.
func BackupTarget(target Target, backupDirectory string) error {
	fmt.Printf("########## Backing up target %s ##########\n\n", target.Name)

	// Retrieve a list of all the git repositories available from the target.
	var repoList []repository
	var err error
	switch target.Source {
	case "github":
		repoList, err = getGitHubRepoList(target, backupDirectory)
	case "bitbucket":
		repoList, err = getBitBucketRepoList(target, backupDirectory)
	default:
		err = fmt.Errorf(`"%s" is not a recognized source type`, target.Source)
	}
	if err != nil {
		return err
	}

	// Back up each repository found.
	for _, repo := range repoList {
		fmt.Println(fmt.Sprintf("#> %s", repo.name))
		if includeRepository(repo.name, target) {
			backupRepository(
				target.Name,
				repo.name,
				repo.cloneURL,
				backupDirectory,
			)
			fmt.Println("")
		} else {
			fmt.Print("Skipped.\n\n")
		}
	}

	fmt.Println("")

	return nil
}

// getGitHubRepoList finds all the repositories belonging to a given user or
// organization on GitHub.
func getGitHubRepoList(target Target, backupDirectory string) ([]repository, error) {
	// Create URL to request list of repos.
	requestURL := fmt.Sprintf(
		"https://api.github.com/%s/%s/repos?access_token=%s&per_page=200",
		target.Type,
		target.Entity,
		target.Token,
	)

	// Retrieve list of repositories.
	response, err := http.Get(requestURL)
	if err != nil {
		return nil, fmt.Errorf("Failed to connect with the source to retrieve the list of repositories: %s", err)
	}
	defer response.Body.Close()
	contents, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, fmt.Errorf("Failed to retrieve the list of repositories: %s", err)
	}

	// Parse JSON response.
	var data []map[string]interface{}
	if err := json.Unmarshal(contents, &data); err != nil {
		return nil, fmt.Errorf("Failed to parse JSON: %s", err)
	}

	// Make a list of repositories.
	var repoList []repository
	for _, repo := range data {
		repoName, _ := repo["name"].(string)
		cloneURL, _ := repo["clone_url"].(string)
		cloneURL = strings.Replace(
			cloneURL,
			"https://",
			fmt.Sprintf("https://%s:%s@", target.Entity, target.Token),
			1,
		)
		repo := repository{name: repoName, cloneURL: cloneURL}
		repoList = append(repoList,repo)
	}

	// No errors.
	return repoList, nil
}

// getBitBucketRepoList finds all the repositories belonging to a given user on
// BitBucket.
func getBitBucketRepoList(target Target, backupDirectory string) ([]repository, error) {
	// Create URL to request list of repos.
	// TODO: support pagination.
	requestURL := fmt.Sprintf(
		"https://%s:%s@bitbucket.org/api/2.0/repositories/%s?page=1&pagelen=100",
		target.Entity,
		target.Password,
		target.Entity,
	)

	// Retrieve list of repositories.
	response, err := http.Get(requestURL)
	if err != nil {
		return nil, fmt.Errorf("Failed to connect with the source to retrieve the list of repositories: %s", err)
	}
	defer response.Body.Close()
	contents, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, fmt.Errorf("Failed to retrieve the list of repositories: %s", err)
	}

	// Parse JSON response.
	var metadata map[string]json.RawMessage
	if err := json.Unmarshal(contents, &metadata); err != nil {
		return nil, fmt.Errorf("Failed to parse JSON: %s", err)
	}
	var data []map[string]json.RawMessage
	if err := json.Unmarshal(metadata["values"], &data); err != nil {
		return nil, fmt.Errorf("Failed to parse JSON: %s", err)
	}

	// Make a list of repositories.
	repoList := make([]repository, len(data))
	for i, repo := range data {
		// Parse the remaining JSON message that pertains to this repository.
		var repoName string
		if err := json.Unmarshal(repo["name"], &repoName); err != nil {
			return nil, fmt.Errorf("Failed to parse JSON: %s", err)
		}
		var links map[string]json.RawMessage
		if err := json.Unmarshal(repo["links"], &links); err != nil {
			return nil, fmt.Errorf("Failed to parse JSON: %s", err)
		}
		var cloneLinks []map[string]string
		if err := json.Unmarshal(links["clone"], &cloneLinks); err != nil {
			return nil, fmt.Errorf("Failed to parse JSON: %s", err)
		}

		// Find the https URL to use for cloning.
		var cloneURL string
		for _, link := range cloneLinks {
			if link["name"] == "https" {
				cloneURL = link["href"]
			}
		}
		if cloneURL == "" {
			return nil, fmt.Errorf("Could not determine HTTPS cloning URL: %s", cloneLinks)
		}

		// Determine URL for cloning.
		cloneURL = strings.Replace(
			cloneURL,
			fmt.Sprintf("https://%s@", target.Entity),
			fmt.Sprintf("https://%s:%s@", target.Entity, target.Password),
			1,
		)

		repoList[i] = repository{name: repoName, cloneURL: cloneURL}
	}

	// No errors.
	return repoList, nil
}

// backupRepository takes a remote git repository and backs it up locally.
// Note that this makes a mirror repository - in other words, the backup only
// contains the content of a normal .git repository but no working directory,
// which saves space. You can always get a normal repository from the backup by
// doing a normal git clone of the backup itself.
func backupRepository(targetName string, repoName string, cloneURL string, backupDirectory string) {
	cloneDirectory := filepath.Join(backupDirectory, targetName, repoName)

	if _, err := os.Stat(cloneDirectory); os.IsNotExist(err) {
		// The repo doesn't exist locally, clone it.
		cmd := exec.Command("git", "clone", "--mirror", cloneURL, cloneDirectory)
		cmdOut, err := cmd.CombinedOutput()
		if err != nil {
			fmt.Println("Error cloning the repository:", err)
			return
		}
		if len(cmdOut) > 0 {
			fmt.Printf(string(cmdOut))
		}
		fmt.Println("Cloned repository.")
	} else {
		// The repo already exists.
		// Set the origin again in case the credentials have changed.
		cmd := exec.Command("git", "remote", "set-url", "origin", cloneURL)
		cmd.Dir = cloneDirectory
		cmdOut, err := cmd.CombinedOutput()
		if err != nil {
			fmt.Println("Error setting remote URL:", err)
			return
		}

		// Pull updates.
		cmd = exec.Command("git", "remote", "update", "-p")
		cmd.Dir = cloneDirectory
		cmdOut, err = cmd.CombinedOutput()
		if err != nil {
			fmt.Println("Error pulling in the repository:", err)
			return
		}

		// Display pulled information.
		if len(cmdOut) > 0 {
			fmt.Printf(string(cmdOut))
		}
		fmt.Println("Pulled latest updates in the repository.")
	}
}

// includeRepository takes a repository name and the information about the
// target it is part of, and determines whether the repository should be backed
// up or not.
func includeRepository(repoName string, target Target) bool {
	if target.Skip != "" {
		r, err := regexp.Compile(target.Skip)
		if err != nil {
			log.Fatalf(`"skip" does not specify a valid regular expression: %s`, err)
		}
		return r.MatchString(repoName) == false
	}
	if target.Only != "" {
		r, err := regexp.Compile(target.Only)
		if err != nil {
			log.Fatalf(`"only" does not specify a valid regular expression: %s`, err)
		}
		return r.MatchString(repoName) == true
	}

	return true
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"
	"time"

	"github.com/dynport/dgtk/github"
	"github.com/dynport/gocli"
)

type Status struct {
	WithURLs bool   `cli:"opt --with-urls"`
	Branch   string `cli:"opt --branch"`
}

func (r *Status) Run() error {
	var branches []string
	if r.Branch != "" {
		branches = []string{r.Branch}
	} else {
		branches = []string{"master"}
		if cb, err := currentBranch(); err == nil && cb != "master" {
			branches = append([]string{cb}, "master")
		}
	}
	repo, err := githubRepo()
	if err != nil {
		return err
	}
	cl, err := client()
	if err != nil {
		return err
	}

	t := gocli.NewTable()
	for _, b := range branches {
		var ago, url, status, sha string
		if s, err := loadStatus(cl, repo, b); err != nil {
			if isNotFound(err) {
				status = "not_found"
			} else {
				return err
			}
		} else {
			status = s.State
			sha = s.SHA
			if len(s.Statuses) > 0 {
				ago = strings.Split(time.Since(s.Statuses[0].CreatedAt).String(), ".")[0]
				url = s.Statuses[0].TargetURL
			}
		}
		args := []interface{}{b, colorizeStatus(status), truncate(sha, 8, false), ago}
		if r.WithURLs {
			args = append(args, url)
		}
		t.Add(args...)
	}
	fmt.Println(t)
	return nil
}

func isNotFound(err error) bool {
	return err != nil && strings.Contains(err.Error(), "404 Not Found")
}

func colorizeStatus(in string) string {
	color := gocli.Green
	switch in {
	case "success":
		color = gocli.Green
	case "pending", "not_found":
		color = gocli.Yellow
	default:
		color = gocli.Red
	}
	return color(in)
}

func loadStatus(cl *github.Client, repo, ref string) (res *statusResponse, err error) {
	u := "https://api.github.com/repos/" + repo + "/commits/" + ref + "/status"
	rsp, err := cl.Get(u)
	if err != nil {
		return nil, err
	}
	defer rsp.Body.Close()
	if rsp.Status[0] != '2' {
		b, _ := ioutil.ReadAll(rsp.Body)
		return nil, fmt.Errorf("got status %s but expected 2x. body=%s", rsp.Status, string(b))
	}
	err = json.NewDecoder(rsp.Body).Decode(&res)
	if err != nil {
		return nil, err
	}
	return res, nil
}

func currentBranch() (string, error) {
	b, err := exec.Command("git", "rev-parse", "--abbrev-ref", "HEAD").CombinedOutput()
	if err != nil {
		return "", err
	}
	return strings.TrimSpace(string(b)), nil
}

type statusResponse struct {
	State    string `json:"state"`
	Statuses []*struct {
		URL       string    `json:"url,omitempty"`
		TargetURL string    `json:"target_url,omitempty"`
		CreatedAt time.Time `json:"created_at,omitempty"`
		UpdatedAt time.Time `json:"updated_at,omitempty"`
	} `json:"statuses"`
	SHA string `json:"sha"`
}

// to be used to colorize
func dataOn(f *os.File) bool {
	stat, err := f.Stat()
	if err != nil {
		return false
	}
	return (stat.Mode() & os.ModeCharDevice) == 0
}
<commit_msg>gh status: implement wait flag<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"strings"
	"time"

	"github.com/dynport/dgtk/github"
	"github.com/dynport/gocli"
)

type Status struct {
	WithURLs bool   `cli:"opt --with-urls"`
	Branch   string `cli:"opt --branch"`
	Wait     bool   `cli:"opt --wait"`
}

func (r *Status) Run() error {

	if r.Wait {
	}
	var branches []string
	if r.Branch != "" {
		branches = []string{r.Branch}
	} else {
		branches = []string{"master"}
		if cb, err := currentBranch(); err == nil && cb != "master" {
			branches = append([]string{cb}, "master")
		}
	}
	repo, err := githubRepo()
	if err != nil {
		return err
	}
	cl, err := client()
	if err != nil {
		return err
	}

	l := log.New(os.Stderr, "", 0)

	if r.Wait {
		if r.Branch == "" {
			return fmt.Errorf("wait requires one specific branch")
		}
		var printedURL bool
		for {
			s, err := loadStatus(cl, repo, r.Branch)
			if err != nil {
				l.Printf("error fetching status: %s", err)
			} else {
				if s.State != statePending {
					fmt.Println(s.State)
					if s.State == stateSuccess {
						return nil
					}
					return fmt.Errorf("not successful (%s)", s.State)
				}
				if !printedURL && len(s.Statuses) > 0 {
					l.Printf("url=%s", s.Statuses[0].URL)
					printedURL = true
				}
			}
			time.Sleep(10 * time.Second)
		}
		return nil
	}

	t := gocli.NewTable()
	for _, b := range branches {
		var ago, url, status, sha string
		if s, err := loadStatus(cl, repo, b); err != nil {
			if isNotFound(err) {
				status = "not_found"
			} else {
				return err
			}
		} else {
			status = s.State
			sha = s.SHA
			if len(s.Statuses) > 0 {
				ago = strings.Split(time.Since(s.Statuses[0].CreatedAt).String(), ".")[0]
				url = s.Statuses[0].TargetURL
			}
		}
		args := []interface{}{b, colorizeStatus(status), truncate(sha, 8, false), ago}
		if r.WithURLs {
			args = append(args, url)
		}
		t.Add(args...)
	}
	fmt.Println(t)
	return nil
}

func isNotFound(err error) bool {
	return err != nil && strings.Contains(err.Error(), "404 Not Found")
}

const (
	stateSuccess  = "success"
	statePending  = "pending"
	stateNotFound = "not_found"
)

func colorizeStatus(in string) string {
	color := gocli.Green
	switch in {
	case stateSuccess:
		color = gocli.Green
	case statePending, stateNotFound:
		color = gocli.Yellow
	default:
		color = gocli.Red
	}
	return color(in)
}

func loadStatus(cl *github.Client, repo, ref string) (res *statusResponse, err error) {
	u := "https://api.github.com/repos/" + repo + "/commits/" + ref + "/status"
	rsp, err := cl.Get(u)
	if err != nil {
		return nil, err
	}
	defer rsp.Body.Close()
	if rsp.Status[0] != '2' {
		b, _ := ioutil.ReadAll(rsp.Body)
		return nil, fmt.Errorf("got status %s but expected 2x. body=%s", rsp.Status, string(b))
	}
	err = json.NewDecoder(rsp.Body).Decode(&res)
	if err != nil {
		return nil, err
	}
	return res, nil
}

func currentBranch() (string, error) {
	b, err := exec.Command("git", "rev-parse", "--abbrev-ref", "HEAD").CombinedOutput()
	if err != nil {
		return "", err
	}
	return strings.TrimSpace(string(b)), nil
}

type statusResponse struct {
	State    string `json:"state"`
	Statuses []*struct {
		URL       string    `json:"url,omitempty"`
		TargetURL string    `json:"target_url,omitempty"`
		CreatedAt time.Time `json:"created_at,omitempty"`
		UpdatedAt time.Time `json:"updated_at,omitempty"`
	} `json:"statuses"`
	SHA string `json:"sha"`
}

// to be used to colorize
func dataOn(f *os.File) bool {
	stat, err := f.Stat()
	if err != nil {
		return false
	}
	return (stat.Mode() & os.ModeCharDevice) == 0
}
<|endoftext|>


<commit_before>package glacier

import (
	"crypto/sha256"
	"fmt"
	"hash"
	"io"
)

const (
	KiB = 1024
	MiB = 1024 * KiB
)

type treeHashNode struct {
	hash  [sha256.Size]byte
	left  *treeHashNode
	right *treeHashNode
}

type TreeHash struct {
	whole   hash.Hash
	part    hash.Hash
	hashers io.Writer
	nodes   []treeHashNode
	written int
}

func NewTreeHash() *TreeHash {
	var result TreeHash
	result.whole = sha256.New()
	result.part = sha256.New()
	result.hashers = io.MultiWriter(result.whole, result.part)
	result.nodes = make([]treeHashNode, 0)
	return &result
}

func (th *TreeHash) Write(p []byte) (n int, err error) {
	// check if we can't fill up remaining chunk
	if len(p) < 1024*1024-th.written {
		th.written, _ = th.hashers.Write(p)
		n += th.written
		return
	}

	// fill remaining chunk
	th.written, _ = th.hashers.Write(p[:1024*1024-th.written])
	n += th.written
	p = p[th.written:]
	th.nodes = append(th.nodes, treeHashNode{})
	th.part.Sum(th.nodes[len(th.nodes)-1].hash[:0])
	th.part.Reset()
	th.written = 0

	// write all full chunks
	for len(p) > 1024*1024 {
		th.written, _ = th.hashers.Write(p[:1024*1024-th.written])
		n += th.written
		p = p[th.written:]
		th.nodes = append(th.nodes, treeHashNode{})
		th.part.Sum(th.nodes[len(th.nodes)-1].hash[:0])
		th.part.Reset()
	}

	// write remaining
	th.written, _ = th.hashers.Write(p)
	n += th.written

	return
}

func (th *TreeHash) Close() error {
	// create last node
	if th.written > 0 {
		th.nodes = append(th.nodes, treeHashNode{})
		th.part.Sum(th.nodes[len(th.nodes)-1].hash[:0])
		th.part.Reset()
	}

	// create tree
	outIndex := len(th.nodes)
	childIndex := 0
	added := outIndex
	var remainder *treeHashNode
	for added > 1 || remainder != nil {
		children := added
		added = 0
		// pair up
		for children > 1 {
			th.nodes = append(th.nodes, treeHashNode{})
			th.nodes[outIndex].left = &th.nodes[childIndex]
			th.nodes[outIndex].right = &th.nodes[childIndex+1]
			th.part.Write(th.nodes[childIndex].hash[:])
			th.part.Write(th.nodes[childIndex+1].hash[:])
			th.part.Sum(th.nodes[outIndex].hash[:0])
			th.part.Reset()

			outIndex++
			children -= 2
			childIndex += 2
			added++
		}
		if children == 1 {
			// have a child that couldn't be paired up
			if remainder == nil {
				// hold on to child as remainder for later
				remainder = &th.nodes[childIndex]
				childIndex++
			} else {
				// join with existing remainder
				th.nodes = append(th.nodes, treeHashNode{})
				th.nodes[outIndex].left = &th.nodes[childIndex]
				th.nodes[outIndex].right = remainder
				th.part.Write(th.nodes[childIndex].hash[:])
				th.part.Write(remainder.hash[:])
				th.part.Sum(th.nodes[outIndex].hash[:0])
				th.part.Reset()

				outIndex++
				remainder = nil
				childIndex++
				added++
			}
		}
	}

	return nil
}

func (th *TreeHash) TreeHash() string {
	return string(toHex(th.nodes[len(th.nodes)-1].hash[:]))
}

func (th *TreeHash) Hash() string {
	return string(toHex(th.whole.Sum(nil)))
}

func (th *TreeHash) Reset() {
	th.whole.Reset()
	th.part.Reset()
	th.written = 0
	th.nodes = th.nodes[:0]
}

// TODO hash entire file at the same time
func createTreeHash(r io.Reader) (*treeHashNode, []byte, error) {
	wholeHash := sha256.New()
	partHash := sha256.New()
	hashers := io.MultiWriter(partHash, wholeHash)
	hashes := make([]treeHashNode, 0)
	outIndex := 0

	// generate hashes for 1 MiB chunks
	n, err := io.CopyN(hashers, r, MiB)
	for err == nil {
		hashes = append(hashes, treeHashNode{})
		partHash.Sum(hashes[outIndex].hash[:0])
		partHash.Reset()
		outIndex++
		n, err = io.CopyN(hashers, r, MiB)
	}
	if err != nil && err != io.EOF {
		return nil, nil, err
	}
	if n > 0 {
		hashes = append(hashes, treeHashNode{})
		partHash.Sum(hashes[outIndex].hash[:0])
		partHash.Reset()
		outIndex++
	}

	// build tree
	// TODO calculate levels remaining and grow once
	childIndex := 0
	added := outIndex
	var remainder *treeHashNode
	for added > 1 || remainder != nil {
		children := added
		added = 0
		// pair up
		for children > 1 {
			hashes = append(hashes, treeHashNode{})
			hashes[outIndex].left = &hashes[childIndex]
			hashes[outIndex].right = &hashes[childIndex+1]
			partHash.Write(hashes[childIndex].hash[:])
			partHash.Write(hashes[childIndex+1].hash[:])
			partHash.Sum(hashes[outIndex].hash[:0])
			partHash.Reset()
			outIndex++
			children -= 2
			childIndex += 2
			added++
		}
		if children == 1 {
			// have a remainder that couldn't be paired up
			if remainder == nil {
				// hold on to remainder for later
				remainder = &hashes[childIndex]
				childIndex++
			} else {
				// join with existing remainder
				hashes = append(hashes, treeHashNode{})
				hashes[outIndex].left = &hashes[childIndex]
				hashes[outIndex].right = remainder
				partHash.Write(hashes[childIndex].hash[:])
				partHash.Write(remainder.hash[:])
				partHash.Sum(hashes[outIndex].hash[:0])
				partHash.Reset()
				outIndex++
				remainder = nil
				childIndex++
				added++
			}
		}
	}

	return &hashes[outIndex-1], wholeHash.Sum(nil), nil
}

func (t *treeHashNode) node() string {
	name := fmt.Sprintf("\"%p\"", t)
	label := fmt.Sprintf("\tlabel = \"%s\"\n", string(toHex(t.hash[:4])))

	node := name + " [\n" + label + "];\n"

	var edges string
	if t.left != nil {
		left := fmt.Sprintf("\"%p\"", t.left)
		edges += name + " -> " + left + ";\n"
	}
	if t.right != nil {
		right := fmt.Sprintf("\"%p\"", t.right)
		edges += name + " -> " + right + ";\n"
	}

	return node + edges
}

func (t *treeHashNode) dot() string {
	digraph := "digraph g {\n"
	digraph += "node [\n\tshape = box\n];\n"

	var recurse func(t *treeHashNode)
	recurse = func(t *treeHashNode) {
		if t.left != nil {
			recurse(t.left)
		}
		if t.right != nil {
			recurse(t.right)
		}
		digraph += t.node()
	}
	recurse(t)

	digraph += "{\n\trank=same;\n"
	recurse = func(t *treeHashNode) {
		if t.left == nil && t.right == nil {
			digraph += fmt.Sprintf("\t\"%p\"\n", t)
		} else {
			if t.left != nil {
				recurse(t.left)
			}
			if t.right != nil {
				recurse(t.right)
			}
		}
	}
	recurse(t)
	digraph += "}\n"

	digraph += "}\n"
	return digraph
}
<commit_msg>Fixed bug in tree hash and added method to get raw hash bytes.<commit_after>package glacier

import (
	"crypto/sha256"
	"fmt"
	"hash"
	"io"
)

const (
	KiB = 1024
	MiB = 1024 * KiB
)

type treeHashNode struct {
	hash  [sha256.Size]byte
	left  *treeHashNode
	right *treeHashNode
}

type TreeHash struct {
	whole   hash.Hash
	part    hash.Hash
	hashers io.Writer
	nodes   []treeHashNode
	written int
}

func NewTreeHash() *TreeHash {
	var result TreeHash
	result.whole = sha256.New()
	result.part = sha256.New()
	result.hashers = io.MultiWriter(result.whole, result.part)
	result.nodes = make([]treeHashNode, 0)
	return &result
}

func (th *TreeHash) Write(p []byte) (n int, err error) {
	// check if we can't fill up remaining chunk
	if len(p) < 1024*1024-th.written {
		n, _ = th.hashers.Write(p)
		th.written += n
		return
	}

	// fill remaining chunk
	th.written, _ = th.hashers.Write(p[:1024*1024-th.written])
	n += th.written
	p = p[th.written:]
	th.nodes = append(th.nodes, treeHashNode{})
	th.part.Sum(th.nodes[len(th.nodes)-1].hash[:0])
	th.part.Reset()
	th.written = 0

	// write all full chunks
	for len(p) > 1024*1024 {
		th.written, _ = th.hashers.Write(p[:1024*1024-th.written])
		n += th.written
		p = p[th.written:]
		th.nodes = append(th.nodes, treeHashNode{})
		th.part.Sum(th.nodes[len(th.nodes)-1].hash[:0])
		th.part.Reset()
	}

	// write remaining
	th.written, _ = th.hashers.Write(p)
	n += th.written

	return
}

func (th *TreeHash) Close() error {
	// create last node
	if th.written > 0 {
		th.nodes = append(th.nodes, treeHashNode{})
		th.part.Sum(th.nodes[len(th.nodes)-1].hash[:0])
		th.part.Reset()
	}

	// create tree
	outIndex := len(th.nodes)
	childIndex := 0
	added := outIndex
	var remainder *treeHashNode
	for added > 1 || remainder != nil {
		children := added
		added = 0
		// pair up
		for children > 1 {
			th.nodes = append(th.nodes, treeHashNode{})
			th.nodes[outIndex].left = &th.nodes[childIndex]
			th.nodes[outIndex].right = &th.nodes[childIndex+1]
			th.part.Write(th.nodes[childIndex].hash[:])
			th.part.Write(th.nodes[childIndex+1].hash[:])
			th.part.Sum(th.nodes[outIndex].hash[:0])
			th.part.Reset()

			outIndex++
			children -= 2
			childIndex += 2
			added++
		}
		if children == 1 {
			// have a child that couldn't be paired up
			if remainder == nil {
				// hold on to child as remainder for later
				remainder = &th.nodes[childIndex]
				childIndex++
			} else {
				// join with existing remainder
				th.nodes = append(th.nodes, treeHashNode{})
				th.nodes[outIndex].left = &th.nodes[childIndex]
				th.nodes[outIndex].right = remainder
				th.part.Write(th.nodes[childIndex].hash[:])
				th.part.Write(remainder.hash[:])
				th.part.Sum(th.nodes[outIndex].hash[:0])
				th.part.Reset()

				outIndex++
				remainder = nil
				childIndex++
				added++
			}
		}
	}

	return nil
}

func (th *TreeHash) TreeHash() string {
	return string(toHex(th.nodes[len(th.nodes)-1].hash[:]))
}

func (th *TreeHash) Hash() string {
	return string(toHex(th.whole.Sum(nil)))
}

func (th *TreeHash) HashBytes() []byte {
	return th.whole.Sum(nil)
}

func (th *TreeHash) Reset() {
	th.whole.Reset()
	th.part.Reset()
	th.written = 0
	th.nodes = th.nodes[:0]
}

// TODO hash entire file at the same time
func createTreeHash(r io.Reader) (*treeHashNode, []byte, error) {
	wholeHash := sha256.New()
	partHash := sha256.New()
	hashers := io.MultiWriter(partHash, wholeHash)
	hashes := make([]treeHashNode, 0)
	outIndex := 0

	// generate hashes for 1 MiB chunks
	n, err := io.CopyN(hashers, r, MiB)
	for err == nil {
		hashes = append(hashes, treeHashNode{})
		partHash.Sum(hashes[outIndex].hash[:0])
		partHash.Reset()
		outIndex++
		n, err = io.CopyN(hashers, r, MiB)
	}
	if err != nil && err != io.EOF {
		return nil, nil, err
	}
	if n > 0 {
		hashes = append(hashes, treeHashNode{})
		partHash.Sum(hashes[outIndex].hash[:0])
		partHash.Reset()
		outIndex++
	}

	// build tree
	// TODO calculate levels remaining and grow once
	childIndex := 0
	added := outIndex
	var remainder *treeHashNode
	for added > 1 || remainder != nil {
		children := added
		added = 0
		// pair up
		for children > 1 {
			hashes = append(hashes, treeHashNode{})
			hashes[outIndex].left = &hashes[childIndex]
			hashes[outIndex].right = &hashes[childIndex+1]
			partHash.Write(hashes[childIndex].hash[:])
			partHash.Write(hashes[childIndex+1].hash[:])
			partHash.Sum(hashes[outIndex].hash[:0])
			partHash.Reset()
			outIndex++
			children -= 2
			childIndex += 2
			added++
		}
		if children == 1 {
			// have a remainder that couldn't be paired up
			if remainder == nil {
				// hold on to remainder for later
				remainder = &hashes[childIndex]
				childIndex++
			} else {
				// join with existing remainder
				hashes = append(hashes, treeHashNode{})
				hashes[outIndex].left = &hashes[childIndex]
				hashes[outIndex].right = remainder
				partHash.Write(hashes[childIndex].hash[:])
				partHash.Write(remainder.hash[:])
				partHash.Sum(hashes[outIndex].hash[:0])
				partHash.Reset()
				outIndex++
				remainder = nil
				childIndex++
				added++
			}
		}
	}

	return &hashes[outIndex-1], wholeHash.Sum(nil), nil
}

func (t *treeHashNode) node() string {
	name := fmt.Sprintf("\"%p\"", t)
	label := fmt.Sprintf("\tlabel = \"%s\"\n", string(toHex(t.hash[:4])))

	node := name + " [\n" + label + "];\n"

	var edges string
	if t.left != nil {
		left := fmt.Sprintf("\"%p\"", t.left)
		edges += name + " -> " + left + ";\n"
	}
	if t.right != nil {
		right := fmt.Sprintf("\"%p\"", t.right)
		edges += name + " -> " + right + ";\n"
	}

	return node + edges
}

func (t *treeHashNode) dot() string {
	digraph := "digraph g {\n"
	digraph += "node [\n\tshape = box\n];\n"

	var recurse func(t *treeHashNode)
	recurse = func(t *treeHashNode) {
		if t.left != nil {
			recurse(t.left)
		}
		if t.right != nil {
			recurse(t.right)
		}
		digraph += t.node()
	}
	recurse(t)

	digraph += "{\n\trank=same;\n"
	recurse = func(t *treeHashNode) {
		if t.left == nil && t.right == nil {
			digraph += fmt.Sprintf("\t\"%p\"\n", t)
		} else {
			if t.left != nil {
				recurse(t.left)
			}
			if t.right != nil {
				recurse(t.right)
			}
		}
	}
	recurse(t)
	digraph += "}\n"

	digraph += "}\n"
	return digraph
}
<|endoftext|>