<commit_before>// Copyright 2015 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Compilebench benchmarks the speed of the Go compiler.
//
// Usage:
//
//	compilebench [options]
//
// It times the compilation of various packages and prints results in
// the format used by package testing (and expected by rsc.io/benchstat).
//
// The options are:
//
//	-alloc
//		Report allocations.
//
//	-compile exe
//		Use exe as the path to the cmd/compile binary.
//
//	-compileflags 'list'
//		Pass the space-separated list of flags to the compilation.
//
//	-count n
//		Run each benchmark n times (default 1).
//
//	-cpuprofile file
//		Write a CPU profile of the compiler to file.
//
//	-memprofile file
//		Write a memory profile of the compiler to file.
//
//	-memprofilerate rate
//		Set runtime.MemProfileRate during compilation.
//
//	-run regexp
//		Only run benchmarks with names matching regexp.
//
// Although -cpuprofile and -memprofile are intended to write a
// combined profile for all the executed benchmarks to file,
// today they write only the profile for the last benchmark executed.
//
// The default memory profiling rate is one profile sample per 512 kB
// allocated (see ``go doc runtime.MemProfileRate'').
// Lowering the rate (for example, -memprofilerate 64000) produces
// a more fine-grained and therefore accurate profile, but it also incurs
// execution cost. For benchmark comparisons, never use timings
// obtained with a low -memprofilerate option.
//
// Example
//
// Assuming the base version of the compiler has been saved with
// ``toolstash save,'' this sequence compares the old and new compiler:
//
//	compilebench -count 10 -compile $(toolstash -n compile) >old.txt
//	compilebench -count 10 >new.txt
//	benchstat old.txt new.txt
//
package main

import (
	"flag"
	"fmt"
	"go/build"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strconv"
	"strings"
	"time"
)

var (
	goroot   = runtime.GOROOT()
	compiler string
	runRE    *regexp.Regexp
	is6g     bool
)

var (
	flagAlloc          = flag.Bool("alloc", false, "report allocations")
	flagCompiler       = flag.String("compile", "", "use `exe` as the cmd/compile binary")
	flagCompilerFlags  = flag.String("compileflags", "", "additional `flags` to pass to compile")
	flagRun            = flag.String("run", "", "run benchmarks matching `regexp`")
	flagCount          = flag.Int("count", 1, "run benchmarks `n` times")
	flagCpuprofile     = flag.String("cpuprofile", "", "write CPU profile to `file`")
	flagMemprofile     = flag.String("memprofile", "", "write memory profile to `file`")
	flagMemprofilerate = flag.Int64("memprofilerate", -1, "set memory profile `rate`")
	flagShort          = flag.Bool("short", false, "skip long-running benchmarks")
)

var tests = []struct {
	name string
	dir  string
	long bool
}{
	{"BenchmarkTemplate", "html/template", false},
	{"BenchmarkUnicode", "unicode", false},
	{"BenchmarkGoTypes", "go/types", false},
	{"BenchmarkCompiler", "cmd/compile/internal/gc", false},
	{"BenchmarkSSA", "cmd/compile/internal/ssa", false},
	{"BenchmarkFlate", "compress/flate", false},
	{"BenchmarkGoParser", "go/parser", false},
	{"BenchmarkReflect", "reflect", false},
	{"BenchmarkTar", "archive/tar", false},
	{"BenchmarkXML", "encoding/xml", false},
	{"BenchmarkMakeBash", "", true},
	{"BenchmarkHelloSize", "", false},
	{"BenchmarkCmdGoSize", "", true},
}

func usage() {
	fmt.Fprintf(os.Stderr, "usage: compilebench [options]\n")
	fmt.Fprintf(os.Stderr, "options:\n")
	flag.PrintDefaults()
	os.Exit(2)
}

func main() {
	log.SetFlags(0)
	log.SetPrefix("compilebench: ")
	flag.Usage = usage
	flag.Parse()
	if flag.NArg() != 0 {
		usage()
	}

	compiler = *flagCompiler
	if compiler == "" {
		out, err := exec.Command("go", "tool", "-n", "compile").CombinedOutput()
		if err != nil {
			out, err = exec.Command("go", "tool", "-n", "6g").CombinedOutput()
			is6g = true
			if err != nil {
				out, err = exec.Command("go", "tool", "-n", "compile").CombinedOutput()
				log.Fatalf("go tool -n compiler: %v\n%s", err, out)
			}
		}
		compiler = strings.TrimSpace(string(out))
	}

	if *flagRun != "" {
		r, err := regexp.Compile(*flagRun)
		if err != nil {
			log.Fatalf("invalid -run argument: %v", err)
		}
		runRE = r
	}

	for i := 0; i < *flagCount; i++ {
		for _, tt := range tests {
			if tt.long && *flagShort {
				continue
			}
			if runRE == nil || runRE.MatchString(tt.name) {
				runBuild(tt.name, tt.dir)
			}
		}
	}
}

func runCmd(name string, cmd *exec.Cmd) {
	start := time.Now()
	out, err := cmd.CombinedOutput()
	if err != nil {
		log.Printf("%v: %v\n%s", name, err, out)
		return
	}
	fmt.Printf("%s 1 %d ns/op\n", name, time.Since(start).Nanoseconds())
}

func runMakeBash() {
	cmd := exec.Command("./make.bash")
	cmd.Dir = filepath.Join(runtime.GOROOT(), "src")
	runCmd("BenchmarkMakeBash", cmd)
}

func runCmdGoSize() {
	runSize("BenchmarkCmdGoSize", filepath.Join(runtime.GOROOT(), "bin/go"))
}

func runHelloSize() {
	cmd := exec.Command("go", "build", "-o", "_hello_", filepath.Join(runtime.GOROOT(), "test/helloworld.go"))
	cmd.Stdout = os.Stderr
	cmd.Stderr = os.Stderr
	if err := cmd.Run(); err != nil {
		log.Print(err)
		return
	}
	defer os.Remove("_hello_")
	runSize("BenchmarkHelloSize", "_hello_")
}

func runSize(name, file string) {
	info, err := os.Stat(file)
	if err != nil {
		log.Print(err)
		return
	}
	out, err := exec.Command("size", file).CombinedOutput()
	if err != nil {
		log.Printf("size: %v\n%s", err, out)
		return
	}
	lines := strings.Split(string(out), "\n")
	if len(lines) < 2 {
		log.Printf("not enough output from size: %s", out)
		return
	}
	f := strings.Fields(lines[1])
	if strings.HasPrefix(lines[0], "__TEXT") && len(f) >= 2 { // OS X
		fmt.Printf("%s 1 %s text-bytes %s data-bytes %v exe-bytes\n", name, f[0], f[1], info.Size())
	} else if strings.Contains(lines[0], "bss") && len(f) >= 3 {
		fmt.Printf("%s 1 %s text-bytes %s data-bytes %s bss-bytes %v exe-bytes\n", name, f[0], f[1], f[2], info.Size())
	}
}

func runBuild(name, dir string) {
	switch name {
	case "BenchmarkMakeBash":
		runMakeBash()
		return
	case "BenchmarkCmdGoSize":
		runCmdGoSize()
		return
	case "BenchmarkHelloSize":
		runHelloSize()
		return
	}

	pkg, err := build.Import(dir, ".", 0)
	if err != nil {
		log.Print(err)
		return
	}
	args := []string{"-o", "_compilebench_.o"}
	if is6g {
		*flagMemprofilerate = -1
		*flagAlloc = false
		*flagCpuprofile = ""
		*flagMemprofile = ""
	}
	if *flagMemprofilerate >= 0 {
		args = append(args, "-memprofilerate", fmt.Sprint(*flagMemprofilerate))
	}
	args = append(args, strings.Fields(*flagCompilerFlags)...)
	if *flagAlloc || *flagCpuprofile != "" || *flagMemprofile != "" {
		if *flagAlloc || *flagMemprofile != "" {
			args = append(args, "-memprofile", "_compilebench_.memprof")
		}
		if *flagCpuprofile != "" {
			args = append(args, "-cpuprofile", "_compilebench_.cpuprof")
		}
	}
	args = append(args, pkg.GoFiles...)
	cmd := exec.Command(compiler, args...)
	cmd.Dir = pkg.Dir
	cmd.Stdout = os.Stderr
	cmd.Stderr = os.Stderr
	start := time.Now()
	err = cmd.Run()
	if err != nil {
		log.Printf("%v: %v", name, err)
		return
	}
	end := time.Now()

	var allocs, bytes int64
	if *flagAlloc || *flagMemprofile != "" {
		out, err := ioutil.ReadFile(pkg.Dir + "/_compilebench_.memprof")
		if err != nil {
			log.Print("cannot find memory profile after compilation")
		}
		for _, line := range strings.Split(string(out), "\n") {
			f := strings.Fields(line)
			if len(f) < 4 || f[0] != "#" || f[2] != "=" {
				continue
			}
			val, err := strconv.ParseInt(f[3], 0, 64)
			if err != nil {
				continue
			}
			switch f[1] {
			case "TotalAlloc":
				bytes = val
			case "Mallocs":
				allocs = val
			}
		}

		if *flagMemprofile != "" {
			if err := ioutil.WriteFile(*flagMemprofile, out, 0666); err != nil {
				log.Print(err)
			}
		}
		os.Remove(pkg.Dir + "/_compilebench_.memprof")
	}

	if *flagCpuprofile != "" {
		out, err := ioutil.ReadFile(pkg.Dir + "/_compilebench_.cpuprof")
		if err != nil {
			log.Print(err)
		}
		if err := ioutil.WriteFile(*flagCpuprofile, out, 0666); err != nil {
			log.Print(err)
		}
		os.Remove(pkg.Dir + "/_compilebench_.cpuprof")
	}

	wallns := end.Sub(start).Nanoseconds()
	userns := cmd.ProcessState.UserTime().Nanoseconds()

	if *flagAlloc {
		fmt.Printf("%s 1 %d ns/op %d user-ns/op %d B/op %d allocs/op\n", name, wallns, userns, bytes, allocs)
	} else {
		fmt.Printf("%s 1 %d ns/op %d user-ns/op\n", name, wallns, userns)
	}

	os.Remove(pkg.Dir + "/_compilebench_.o")
}
<commit_msg>cmd/compilebench: change MakeBash to StdCmd<commit_after>// Copyright 2015 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Compilebench benchmarks the speed of the Go compiler.
//
// Usage:
//
//	compilebench [options]
//
// It times the compilation of various packages and prints results in
// the format used by package testing (and expected by rsc.io/benchstat).
//
// The options are:
//
//	-alloc
//		Report allocations.
//
//	-compile exe
//		Use exe as the path to the cmd/compile binary.
//
//	-compileflags 'list'
//		Pass the space-separated list of flags to the compilation.
//
//	-count n
//		Run each benchmark n times (default 1).
//
//	-cpuprofile file
//		Write a CPU profile of the compiler to file.
//
//	-memprofile file
//		Write a memory profile of the compiler to file.
//
//	-memprofilerate rate
//		Set runtime.MemProfileRate during compilation.
//
//	-run regexp
//		Only run benchmarks with names matching regexp.
//
// Although -cpuprofile and -memprofile are intended to write a
// combined profile for all the executed benchmarks to file,
// today they write only the profile for the last benchmark executed.
//
// The default memory profiling rate is one profile sample per 512 kB
// allocated (see ``go doc runtime.MemProfileRate'').
// Lowering the rate (for example, -memprofilerate 64000) produces
// a more fine-grained and therefore accurate profile, but it also incurs
// execution cost. For benchmark comparisons, never use timings
// obtained with a low -memprofilerate option.
//
// Example
//
// Assuming the base version of the compiler has been saved with
// ``toolstash save,'' this sequence compares the old and new compiler:
//
//	compilebench -count 10 -compile $(toolstash -n compile) >old.txt
//	compilebench -count 10 >new.txt
//	benchstat old.txt new.txt
//
package main

import (
	"flag"
	"fmt"
	"go/build"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strconv"
	"strings"
	"time"
)

var (
	goroot   = runtime.GOROOT()
	compiler string
	runRE    *regexp.Regexp
	is6g     bool
)

var (
	flagAlloc          = flag.Bool("alloc", false, "report allocations")
	flagCompiler       = flag.String("compile", "", "use `exe` as the cmd/compile binary")
	flagCompilerFlags  = flag.String("compileflags", "", "additional `flags` to pass to compile")
	flagRun            = flag.String("run", "", "run benchmarks matching `regexp`")
	flagCount          = flag.Int("count", 1, "run benchmarks `n` times")
	flagCpuprofile     = flag.String("cpuprofile", "", "write CPU profile to `file`")
	flagMemprofile     = flag.String("memprofile", "", "write memory profile to `file`")
	flagMemprofilerate = flag.Int64("memprofilerate", -1, "set memory profile `rate`")
	flagShort          = flag.Bool("short", false, "skip long-running benchmarks")
)

var tests = []struct {
	name string
	dir  string
	long bool
}{
	{"BenchmarkTemplate", "html/template", false},
	{"BenchmarkUnicode", "unicode", false},
	{"BenchmarkGoTypes", "go/types", false},
	{"BenchmarkCompiler", "cmd/compile/internal/gc", false},
	{"BenchmarkSSA", "cmd/compile/internal/ssa", false},
	{"BenchmarkFlate", "compress/flate", false},
	{"BenchmarkGoParser", "go/parser", false},
	{"BenchmarkReflect", "reflect", false},
	{"BenchmarkTar", "archive/tar", false},
	{"BenchmarkXML", "encoding/xml", false},
	{"BenchmarkStdCmd", "", true},
	{"BenchmarkHelloSize", "", false},
	{"BenchmarkCmdGoSize", "", true},
}

func usage() {
	fmt.Fprintf(os.Stderr, "usage: compilebench [options]\n")
	fmt.Fprintf(os.Stderr, "options:\n")
	flag.PrintDefaults()
	os.Exit(2)
}

func main() {
	log.SetFlags(0)
	log.SetPrefix("compilebench: ")
	flag.Usage = usage
	flag.Parse()
	if flag.NArg() != 0 {
		usage()
	}

	compiler = *flagCompiler
	if compiler == "" {
		out, err := exec.Command("go", "tool", "-n", "compile").CombinedOutput()
		if err != nil {
			out, err = exec.Command("go", "tool", "-n", "6g").CombinedOutput()
			is6g = true
			if err != nil {
				out, err = exec.Command("go", "tool", "-n", "compile").CombinedOutput()
				log.Fatalf("go tool -n compiler: %v\n%s", err, out)
			}
		}
		compiler = strings.TrimSpace(string(out))
	}

	if *flagRun != "" {
		r, err := regexp.Compile(*flagRun)
		if err != nil {
			log.Fatalf("invalid -run argument: %v", err)
		}
		runRE = r
	}

	for i := 0; i < *flagCount; i++ {
		for _, tt := range tests {
			if tt.long && *flagShort {
				continue
			}
			if runRE == nil || runRE.MatchString(tt.name) {
				runBuild(tt.name, tt.dir)
			}
		}
	}
}

func runCmd(name string, cmd *exec.Cmd) {
	start := time.Now()
	out, err := cmd.CombinedOutput()
	if err != nil {
		log.Printf("%v: %v\n%s", name, err, out)
		return
	}
	fmt.Printf("%s 1 %d ns/op\n", name, time.Since(start).Nanoseconds())
}

func runStdCmd() {
	cmd := exec.Command("go", "build", "-a", "std", "cmd")
	cmd.Dir = filepath.Join(runtime.GOROOT(), "src")
	runCmd("BenchmarkStdCmd", cmd)
}

func runCmdGoSize() {
	runSize("BenchmarkCmdGoSize", filepath.Join(runtime.GOROOT(), "bin/go"))
}

func runHelloSize() {
	cmd := exec.Command("go", "build", "-o", "_hello_", filepath.Join(runtime.GOROOT(), "test/helloworld.go"))
	cmd.Stdout = os.Stderr
	cmd.Stderr = os.Stderr
	if err := cmd.Run(); err != nil {
		log.Print(err)
		return
	}
	defer os.Remove("_hello_")
	runSize("BenchmarkHelloSize", "_hello_")
}

func runSize(name, file string) {
	info, err := os.Stat(file)
	if err != nil {
		log.Print(err)
		return
	}
	out, err := exec.Command("size", file).CombinedOutput()
	if err != nil {
		log.Printf("size: %v\n%s", err, out)
		return
	}
	lines := strings.Split(string(out), "\n")
	if len(lines) < 2 {
		log.Printf("not enough output from size: %s", out)
		return
	}
	f := strings.Fields(lines[1])
	if strings.HasPrefix(lines[0], "__TEXT") && len(f) >= 2 { // OS X
		fmt.Printf("%s 1 %s text-bytes %s data-bytes %v exe-bytes\n", name, f[0], f[1], info.Size())
	} else if strings.Contains(lines[0], "bss") && len(f) >= 3 {
		fmt.Printf("%s 1 %s text-bytes %s data-bytes %s bss-bytes %v exe-bytes\n", name, f[0], f[1], f[2], info.Size())
	}
}

func runBuild(name, dir string) {
	switch name {
	case "BenchmarkStdCmd":
		runStdCmd()
		return
	case "BenchmarkCmdGoSize":
		runCmdGoSize()
		return
	case "BenchmarkHelloSize":
		runHelloSize()
		return
	}

	pkg, err := build.Import(dir, ".", 0)
	if err != nil {
		log.Print(err)
		return
	}
	args := []string{"-o", "_compilebench_.o"}
	if is6g {
		*flagMemprofilerate = -1
		*flagAlloc = false
		*flagCpuprofile = ""
		*flagMemprofile = ""
	}
	if *flagMemprofilerate >= 0 {
		args = append(args, "-memprofilerate", fmt.Sprint(*flagMemprofilerate))
	}
	args = append(args, strings.Fields(*flagCompilerFlags)...)
	if *flagAlloc || *flagCpuprofile != "" || *flagMemprofile != "" {
		if *flagAlloc || *flagMemprofile != "" {
			args = append(args, "-memprofile", "_compilebench_.memprof")
		}
		if *flagCpuprofile != "" {
			args = append(args, "-cpuprofile", "_compilebench_.cpuprof")
		}
	}
	args = append(args, pkg.GoFiles...)
	cmd := exec.Command(compiler, args...)
	cmd.Dir = pkg.Dir
	cmd.Stdout = os.Stderr
	cmd.Stderr = os.Stderr
	start := time.Now()
	err = cmd.Run()
	if err != nil {
		log.Printf("%v: %v", name, err)
		return
	}
	end := time.Now()

	var allocs, bytes int64
	if *flagAlloc || *flagMemprofile != "" {
		out, err := ioutil.ReadFile(pkg.Dir + "/_compilebench_.memprof")
		if err != nil {
			log.Print("cannot find memory profile after compilation")
		}
		for _, line := range strings.Split(string(out), "\n") {
			f := strings.Fields(line)
			if len(f) < 4 || f[0] != "#" || f[2] != "=" {
				continue
			}
			val, err := strconv.ParseInt(f[3], 0, 64)
			if err != nil {
				continue
			}
			switch f[1] {
			case "TotalAlloc":
				bytes = val
			case "Mallocs":
				allocs = val
			}
		}

		if *flagMemprofile != "" {
			if err := ioutil.WriteFile(*flagMemprofile, out, 0666); err != nil {
				log.Print(err)
			}
		}
		os.Remove(pkg.Dir + "/_compilebench_.memprof")
	}

	if *flagCpuprofile != "" {
		out, err := ioutil.ReadFile(pkg.Dir + "/_compilebench_.cpuprof")
		if err != nil {
			log.Print(err)
		}
		if err := ioutil.WriteFile(*flagCpuprofile, out, 0666); err != nil {
			log.Print(err)
		}
		os.Remove(pkg.Dir + "/_compilebench_.cpuprof")
	}

	wallns := end.Sub(start).Nanoseconds()
	userns := cmd.ProcessState.UserTime().Nanoseconds()

	if *flagAlloc {
		fmt.Printf("%s 1 %d ns/op %d user-ns/op %d B/op %d allocs/op\n", name, wallns, userns, bytes, allocs)
	} else {
		fmt.Printf("%s 1 %d ns/op %d user-ns/op\n", name, wallns, userns)
	}

	os.Remove(pkg.Dir + "/_compilebench_.o")
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package leaderelection implements leader election of a set of endpoints.
// It uses an annotation in the endpoints object to store the record of the
// election state.
//
// This implementation does not guarantee that only one client is acting as a
// leader (a.k.a. fencing). A client observes timestamps captured locally to
// infer the state of the leader election. Thus the implementation is tolerant
// to arbitrary clock skew, but is not tolerant to arbitrary clock skew rate.
//
// However the level of tolerance to skew rate can be configured by setting
// RenewDeadline and LeaseDuration appropriately. The tolerance expressed as a
// maximum tolerated ratio of time passed on the fastest node to time passed on
// the slowest node can be approximately achieved with a configuration that sets
// the same ratio of LeaseDuration to RenewDeadline. For example if a user wanted
// to tolerate some nodes progressing forward in time twice as fast as other nodes,
// the user could set LeaseDuration to 60 seconds and RenewDeadline to 30 seconds.
//
// While not required, some method of clock synchronization between nodes in the
// cluster is highly recommended. It's important to keep in mind when configuring
// this client that the tolerance to skew rate varies inversely to master
// availability.
//
// Larger clusters often have a more lenient SLA for API latency. This should be
// taken into account when configuring the client. The rate of leader transitions
// should be monitored and RetryPeriod and LeaseDuration should be increased
// until the rate is stable and acceptably low. It's important to keep in mind
// when configuring this client that the tolerance to API latency varies inversely
// to master availability.
//
// DISCLAIMER: this is an alpha API. This library will likely change significantly
// or even be removed entirely in subsequent releases. Depend on this API at
// your own risk.
package leaderelection

import (
	"context"
	"fmt"
	"reflect"
	"time"

	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/runtime"
	"k8s.io/apimachinery/pkg/util/wait"
	rl "k8s.io/client-go/tools/leaderelection/resourcelock"

	"github.com/golang/glog"
)

const (
	JitterFactor = 1.2
)

// NewLeaderElector creates a LeaderElector from a LeaderElectionConfig
func NewLeaderElector(lec LeaderElectionConfig) (*LeaderElector, error) {
	if lec.LeaseDuration <= lec.RenewDeadline {
		return nil, fmt.Errorf("leaseDuration must be greater than renewDeadline")
	}
	if lec.RenewDeadline <= time.Duration(JitterFactor*float64(lec.RetryPeriod)) {
		return nil, fmt.Errorf("renewDeadline must be greater than retryPeriod*JitterFactor")
	}
	if lec.LeaseDuration < 1 {
		return nil, fmt.Errorf("leaseDuration must be greater than zero")
	}
	if lec.RenewDeadline < 1 {
		return nil, fmt.Errorf("renewDeadline must be greater than zero")
	}
	if lec.RetryPeriod < 1 {
		return nil, fmt.Errorf("retryPeriod must be greater than zero")
	}

	if lec.Lock == nil {
		return nil, fmt.Errorf("Lock must not be nil.")
	}
	return &LeaderElector{
		config: lec,
	}, nil
}

type LeaderElectionConfig struct {
	// Lock is the resource that will be used for locking
	Lock rl.Interface

	// LeaseDuration is the duration that non-leader candidates will
	// wait to force acquire leadership. This is measured against time of
	// last observed ack.
	LeaseDuration time.Duration
	// RenewDeadline is the duration that the acting master will retry
	// refreshing leadership before giving up.
	RenewDeadline time.Duration
	// RetryPeriod is the duration the LeaderElector clients should wait
	// between tries of actions.
	RetryPeriod time.Duration

	// Callbacks are callbacks that are triggered during certain lifecycle
	// events of the LeaderElector
	Callbacks LeaderCallbacks
}

// LeaderCallbacks are callbacks that are triggered during certain
// lifecycle events of the LeaderElector. These are invoked asynchronously.
//
// possible future callbacks:
//  * OnChallenge()
type LeaderCallbacks struct {
	// OnStartedLeading is called when a LeaderElector client starts leading
	OnStartedLeading func(context.Context)
	// OnStoppedLeading is called when a LeaderElector client stops leading
	OnStoppedLeading func()
	// OnNewLeader is called when the client observes a leader that is
	// not the previously observed leader. This includes the first observed
	// leader when the client starts.
	OnNewLeader func(identity string)
}

// LeaderElector is a leader election client.
type LeaderElector struct {
	config LeaderElectionConfig
	// internal bookkeeping
	observedRecord rl.LeaderElectionRecord
	observedTime   time.Time
	// used to implement OnNewLeader(), may lag slightly from the
	// value observedRecord.HolderIdentity if the transition has
	// not yet been reported.
	reportedLeader string
}

// Run starts the leader election loop
func (le *LeaderElector) Run(ctx context.Context) {
	defer func() {
		runtime.HandleCrash()
		le.config.Callbacks.OnStoppedLeading()
	}()
	if !le.acquire(ctx) {
		return // ctx signalled done
	}
	ctx, cancel := context.WithCancel(ctx)
	defer cancel()
	go le.config.Callbacks.OnStartedLeading(ctx)
	le.renew(ctx)
}

// RunOrDie starts a client with the provided config or panics if the config
// fails to validate.
func RunOrDie(ctx context.Context, lec LeaderElectionConfig) {
	le, err := NewLeaderElector(lec)
	if err != nil {
		panic(err)
	}
	le.Run(ctx)
}

// GetLeader returns the identity of the last observed leader or returns the empty string if
// no leader has yet been observed.
func (le *LeaderElector) GetLeader() string {
	return le.observedRecord.HolderIdentity
}

// IsLeader returns true if the last observed leader was this client else returns false.
func (le *LeaderElector) IsLeader() bool {
	return le.observedRecord.HolderIdentity == le.config.Lock.Identity()
}

// acquire loops calling tryAcquireOrRenew and returns true immediately when tryAcquireOrRenew succeeds.
// Returns false if ctx signals done.
func (le *LeaderElector) acquire(ctx context.Context) bool {
	ctx, cancel := context.WithCancel(ctx)
	defer cancel()
	succeeded := false
	desc := le.config.Lock.Describe()
	glog.Infof("attempting to acquire leader lease  %v...", desc)
	wait.JitterUntil(func() {
		succeeded = le.tryAcquireOrRenew()
		le.maybeReportTransition()
		if !succeeded {
			glog.V(4).Infof("failed to acquire lease %v", desc)
			return
		}
		le.config.Lock.RecordEvent("became leader")
		glog.Infof("successfully acquired lease %v", desc)
		cancel()
	}, le.config.RetryPeriod, JitterFactor, true, ctx.Done())
	return succeeded
}

// renew loops calling tryAcquireOrRenew and returns immediately when tryAcquireOrRenew fails or ctx signals done.
func (le *LeaderElector) renew(ctx context.Context) {
	ctx, cancel := context.WithCancel(ctx)
	defer cancel()
	wait.Until(func() {
		timeoutCtx, timeoutCancel := context.WithTimeout(ctx, le.config.RenewDeadline)
		defer timeoutCancel()
		err := wait.PollImmediateUntil(le.config.RetryPeriod, func() (bool, error) {
			done := make(chan bool, 1)
			go func() {
				defer close(done)
				done <- le.tryAcquireOrRenew()
			}()

			select {
			case <-timeoutCtx.Done():
				return false, fmt.Errorf("failed to tryAcquireOrRenew %s", timeoutCtx.Err())
			case result := <-done:
				return result, nil
			}
		}, timeoutCtx.Done())

		le.maybeReportTransition()
		desc := le.config.Lock.Describe()
		if err == nil {
			glog.V(4).Infof("successfully renewed lease %v", desc)
			return
		}
		le.config.Lock.RecordEvent("stopped leading")
		glog.Infof("failed to renew lease %v: %v", desc, err)
		cancel()
	}, 0, ctx.Done())
}

// tryAcquireOrRenew tries to acquire a leader lease if it is not already acquired,
// else it tries to renew the lease if it has already been acquired. Returns true
// on success else returns false.
func (le *LeaderElector) tryAcquireOrRenew() bool {
	now := metav1.Now()
	leaderElectionRecord := rl.LeaderElectionRecord{
		HolderIdentity:       le.config.Lock.Identity(),
		LeaseDurationSeconds: int(le.config.LeaseDuration / time.Second),
		RenewTime:            now,
		AcquireTime:          now,
	}

	// 1. obtain or create the ElectionRecord
	oldLeaderElectionRecord, err := le.config.Lock.Get()
	if err != nil {
		if !errors.IsNotFound(err) {
			glog.Errorf("error retrieving resource lock %v: %v", le.config.Lock.Describe(), err)
			return false
		}
		if err = le.config.Lock.Create(leaderElectionRecord); err != nil {
			glog.Errorf("error initially creating leader election record: %v", err)
			return false
		}
		le.observedRecord = leaderElectionRecord
		le.observedTime = time.Now()
		return true
	}

	// 2. Record obtained, check the Identity & Time
	if !reflect.DeepEqual(le.observedRecord, *oldLeaderElectionRecord) {
		le.observedRecord = *oldLeaderElectionRecord
		le.observedTime = time.Now()
	}
	if le.observedTime.Add(le.config.LeaseDuration).After(now.Time) &&
		!le.IsLeader() {
		glog.V(4).Infof("lock is held by %v and has not yet expired", oldLeaderElectionRecord.HolderIdentity)
		return false
	}

	// 3. We're going to try to update. The leaderElectionRecord is set to it's default
	// here. Let's correct it before updating.
	if le.IsLeader() {
		leaderElectionRecord.AcquireTime = oldLeaderElectionRecord.AcquireTime
		leaderElectionRecord.LeaderTransitions = oldLeaderElectionRecord.LeaderTransitions
	} else {
		leaderElectionRecord.LeaderTransitions = oldLeaderElectionRecord.LeaderTransitions + 1
	}

	// update the lock itself
	if err = le.config.Lock.Update(leaderElectionRecord); err != nil {
		glog.Errorf("Failed to update lock: %v", err)
		return false
	}
	le.observedRecord = leaderElectionRecord
	le.observedTime = time.Now()
	return true
}

func (le *LeaderElector) maybeReportTransition() {
	if le.observedRecord.HolderIdentity == le.reportedLeader {
		return
	}
	le.reportedLeader = le.observedRecord.HolderIdentity
	if le.config.Callbacks.OnNewLeader != nil {
		go le.config.Callbacks.OnNewLeader(le.reportedLeader)
	}
}
<commit_msg>bug fix: dead loop leaderelection<commit_after>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package leaderelection implements leader election of a set of endpoints.
// It uses an annotation in the endpoints object to store the record of the
// election state.
//
// This implementation does not guarantee that only one client is acting as a
// leader (a.k.a. fencing). A client observes timestamps captured locally to
// infer the state of the leader election. Thus the implementation is tolerant
// to arbitrary clock skew, but is not tolerant to arbitrary clock skew rate.
//
// However the level of tolerance to skew rate can be configured by setting
// RenewDeadline and LeaseDuration appropriately. The tolerance expressed as a
// maximum tolerated ratio of time passed on the fastest node to time passed on
// the slowest node can be approximately achieved with a configuration that sets
// the same ratio of LeaseDuration to RenewDeadline. For example if a user wanted
// to tolerate some nodes progressing forward in time twice as fast as other nodes,
// the user could set LeaseDuration to 60 seconds and RenewDeadline to 30 seconds.
//
// While not required, some method of clock synchronization between nodes in the
// cluster is highly recommended. It's important to keep in mind when configuring
// this client that the tolerance to skew rate varies inversely to master
// availability.
//
// Larger clusters often have a more lenient SLA for API latency. This should be
// taken into account when configuring the client. The rate of leader transitions
// should be monitored and RetryPeriod and LeaseDuration should be increased
// until the rate is stable and acceptably low. It's important to keep in mind
// when configuring this client that the tolerance to API latency varies inversely
// to master availability.
//
// DISCLAIMER: this is an alpha API. This library will likely change significantly
// or even be removed entirely in subsequent releases. Depend on this API at
// your own risk.
package leaderelection

import (
	"context"
	"fmt"
	"reflect"
	"time"

	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/runtime"
	"k8s.io/apimachinery/pkg/util/wait"
	rl "k8s.io/client-go/tools/leaderelection/resourcelock"

	"github.com/golang/glog"
)

const (
	JitterFactor = 1.2
)

// NewLeaderElector creates a LeaderElector from a LeaderElectionConfig
func NewLeaderElector(lec LeaderElectionConfig) (*LeaderElector, error) {
	if lec.LeaseDuration <= lec.RenewDeadline {
		return nil, fmt.Errorf("leaseDuration must be greater than renewDeadline")
	}
	if lec.RenewDeadline <= time.Duration(JitterFactor*float64(lec.RetryPeriod)) {
		return nil, fmt.Errorf("renewDeadline must be greater than retryPeriod*JitterFactor")
	}
	if lec.LeaseDuration < 1 {
		return nil, fmt.Errorf("leaseDuration must be greater than zero")
	}
	if lec.RenewDeadline < 1 {
		return nil, fmt.Errorf("renewDeadline must be greater than zero")
	}
	if lec.RetryPeriod < 1 {
		return nil, fmt.Errorf("retryPeriod must be greater than zero")
	}

	if lec.Lock == nil {
		return nil, fmt.Errorf("Lock must not be nil.")
	}
	return &LeaderElector{
		config: lec,
	}, nil
}

type LeaderElectionConfig struct {
	// Lock is the resource that will be used for locking
	Lock rl.Interface

	// LeaseDuration is the duration that non-leader candidates will
	// wait to force acquire leadership. This is measured against time of
	// last observed ack.
	LeaseDuration time.Duration
	// RenewDeadline is the duration that the acting master will retry
	// refreshing leadership before giving up.
	RenewDeadline time.Duration
	// RetryPeriod is the duration the LeaderElector clients should wait
	// between tries of actions.
	RetryPeriod time.Duration

	// Callbacks are callbacks that are triggered during certain lifecycle
	// events of the LeaderElector
	Callbacks LeaderCallbacks
}

// LeaderCallbacks are callbacks that are triggered during certain
// lifecycle events of the LeaderElector. These are invoked asynchronously.
//
// possible future callbacks:
//  * OnChallenge()
type LeaderCallbacks struct {
	// OnStartedLeading is called when a LeaderElector client starts leading
	OnStartedLeading func(context.Context)
	// OnStoppedLeading is called when a LeaderElector client stops leading
	OnStoppedLeading func()
	// OnNewLeader is called when the client observes a leader that is
	// not the previously observed leader. This includes the first observed
	// leader when the client starts.
	OnNewLeader func(identity string)
}

// LeaderElector is a leader election client.
type LeaderElector struct {
	config LeaderElectionConfig
	// internal bookkeeping
	observedRecord rl.LeaderElectionRecord
	observedTime   time.Time
	// used to implement OnNewLeader(), may lag slightly from the
	// value observedRecord.HolderIdentity if the transition has
	// not yet been reported.
	reportedLeader string
}

// Run starts the leader election loop
func (le *LeaderElector) Run(ctx context.Context) {
	defer func() {
		runtime.HandleCrash()
		le.config.Callbacks.OnStoppedLeading()
	}()
	if !le.acquire(ctx) {
		return // ctx signalled done
	}
	ctx, cancel := context.WithCancel(ctx)
	defer cancel()
	go le.config.Callbacks.OnStartedLeading(ctx)
	le.renew(ctx)
}

// RunOrDie starts a client with the provided config or panics if the config
// fails to validate.
func RunOrDie(ctx context.Context, lec LeaderElectionConfig) {
	le, err := NewLeaderElector(lec)
	if err != nil {
		panic(err)
	}
	le.Run(ctx)
}

// GetLeader returns the identity of the last observed leader or returns the empty string if
// no leader has yet been observed.
func (le *LeaderElector) GetLeader() string {
	return le.observedRecord.HolderIdentity
}

// IsLeader returns true if the last observed leader was this client else returns false.
func (le *LeaderElector) IsLeader() bool {
	return le.observedRecord.HolderIdentity == le.config.Lock.Identity()
}

// acquire loops calling tryAcquireOrRenew and returns true immediately when tryAcquireOrRenew succeeds.
// Returns false if ctx signals done.
func (le *LeaderElector) acquire(ctx context.Context) bool {
	ctx, cancel := context.WithCancel(ctx)
	defer cancel()
	succeeded := false
	desc := le.config.Lock.Describe()
	glog.Infof("attempting to acquire leader lease  %v...", desc)
	wait.JitterUntil(func() {
		succeeded = le.tryAcquireOrRenew()
		le.maybeReportTransition()
		if !succeeded {
			glog.V(4).Infof("failed to acquire lease %v", desc)
			return
		}
		le.config.Lock.RecordEvent("became leader")
		glog.Infof("successfully acquired lease %v", desc)
		cancel()
	}, le.config.RetryPeriod, JitterFactor, true, ctx.Done())
	return succeeded
}

// renew loops calling tryAcquireOrRenew and returns immediately when tryAcquireOrRenew fails or ctx signals done.
func (le *LeaderElector) renew(ctx context.Context) {
	ctx, cancel := context.WithCancel(ctx)
	defer cancel()
	wait.Until(func() {
		timeoutCtx, timeoutCancel := context.WithTimeout(ctx, le.config.RenewDeadline)
		defer timeoutCancel()
		err := wait.PollImmediateUntil(le.config.RetryPeriod, func() (bool, error) {
			done := make(chan bool, 1)
			go func() {
				defer close(done)
				done <- le.tryAcquireOrRenew()
			}()

			select {
			case <-timeoutCtx.Done():
				return false, fmt.Errorf("failed to tryAcquireOrRenew %s", timeoutCtx.Err())
			case result := <-done:
				return result, nil
			}
		}, timeoutCtx.Done())

		le.maybeReportTransition()
		desc := le.config.Lock.Describe()
		if err == nil {
			glog.V(4).Infof("successfully renewed lease %v", desc)
			return
		}
		le.config.Lock.RecordEvent("stopped leading")
		glog.Infof("failed to renew lease %v: %v", desc, err)
		cancel()
	}, le.config.RetryPeriod, ctx.Done())
}

// tryAcquireOrRenew tries to acquire a leader lease if it is not already acquired,
// else it tries to renew the lease if it has already been acquired. Returns true
// on success else returns false.
func (le *LeaderElector) tryAcquireOrRenew() bool {
	now := metav1.Now()
	leaderElectionRecord := rl.LeaderElectionRecord{
		HolderIdentity:       le.config.Lock.Identity(),
		LeaseDurationSeconds: int(le.config.LeaseDuration / time.Second),
		RenewTime:            now,
		AcquireTime:          now,
	}

	// 1. obtain or create the ElectionRecord
	oldLeaderElectionRecord, err := le.config.Lock.Get()
	if err != nil {
		if !errors.IsNotFound(err) {
			glog.Errorf("error retrieving resource lock %v: %v", le.config.Lock.Describe(), err)
			return false
		}
		if err = le.config.Lock.Create(leaderElectionRecord); err != nil {
			glog.Errorf("error initially creating leader election record: %v", err)
			return false
		}
		le.observedRecord = leaderElectionRecord
		le.observedTime = time.Now()
		return true
	}

	// 2. Record obtained, check the Identity & Time
	if !reflect.DeepEqual(le.observedRecord, *oldLeaderElectionRecord) {
		le.observedRecord = *oldLeaderElectionRecord
		le.observedTime = time.Now()
	}
	if le.observedTime.Add(le.config.LeaseDuration).After(now.Time) &&
		!le.IsLeader() {
		glog.V(4).Infof("lock is held by %v and has not yet expired", oldLeaderElectionRecord.HolderIdentity)
		return false
	}

	// 3. We're going to try to update. The leaderElectionRecord is set to it's default
	// here. Let's correct it before updating.
	if le.IsLeader() {
		leaderElectionRecord.AcquireTime = oldLeaderElectionRecord.AcquireTime
		leaderElectionRecord.LeaderTransitions = oldLeaderElectionRecord.LeaderTransitions
	} else {
		leaderElectionRecord.LeaderTransitions = oldLeaderElectionRecord.LeaderTransitions + 1
	}

	// update the lock itself
	if err = le.config.Lock.Update(leaderElectionRecord); err != nil {
		glog.Errorf("Failed to update lock: %v", err)
		return false
	}
	le.observedRecord = leaderElectionRecord
	le.observedTime = time.Now()
	return true
}

func (le *LeaderElector) maybeReportTransition() {
	if le.observedRecord.HolderIdentity == le.reportedLeader {
		return
	}
	le.reportedLeader = le.observedRecord.HolderIdentity
	if le.config.Callbacks.OnNewLeader != nil {
		go le.config.Callbacks.OnNewLeader(le.reportedLeader)
	}
}
<|endoftext|>


<commit_before>package contractor

import (
	"sort"
	"time"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/types"
)

// uptimeMinScans is the minimum number of scans required to judge whether a
// host is offline or not.
const uptimeMinScans = 3

// uptimeWindow specifies the duration in which host uptime is checked.
var uptimeWindow = func() time.Duration {
	switch build.Release {
	case "dev":
		return 30 * time.Minute
	case "standard":
		return 7 * 24 * time.Hour // 1 week.
	case "testing":
		return 15 * time.Second
	}
	panic("undefined uptimeWindow")
}()

// IsOffline indicates whether a contract's host should be considered offline,
// based on its scan metrics.
func (c *Contractor) IsOffline(id types.FileContractID) bool {
	c.mu.RLock()
	defer c.mu.RUnlock()
	return c.isOffline(id)
}

// isOffline indicates whether a contract's host should be considered offline,
// based on its scan metrics.
func (c *Contractor) isOffline(id types.FileContractID) bool {
	// Fetch the corresponding contract in the contractor. If the most recent
	// contract is not in the contractors set of active contracts, this contract
	// line is dead, and thus the contract should be considered 'offline'.
	contract, ok := c.contracts[id]
	if !ok {
		return true
	}
	host, ok := c.hdb.Host(contract.HostPublicKey)
	if !ok {
		return true
	}

	// Sanity check - ScanHistory should always be ordered from oldest to
	// newest.
	if build.DEBUG && !sort.IsSorted(host.ScanHistory) {
		sort.Sort(host.ScanHistory)
		build.Critical("host's scan history was not sorted")
	}

	// Consider a host offline if:
	// 1) The host has been scanned at least three times, and
	// 2) The three most recent scans have all failed, and
	// 3) The time between the most recent scan and the last successful scan
	//    (or first scan) is at least uptimeWindow
	numScans := len(host.ScanHistory)
	if numScans < uptimeMinScans {
		// Not enough data to make a fair judgment.
		return false
	}
	recent := host.ScanHistory[numScans-uptimeMinScans:]
	for _, scan := range recent {
		if scan.Success {
			// One of the scans succeeded.
			return false
		}
	}
	// Initialize window bounds.
	windowStart, windowEnd := host.ScanHistory[0].Timestamp, host.ScanHistory[numScans-1].Timestamp
	// Iterate from newest-oldest, seeking to last successful scan.
	for i := numScans - 1; i >= 0; i-- {
		if scan := host.ScanHistory[i]; scan.Success {
			windowStart = scan.Timestamp
			break
		}
	}
	return windowEnd.Sub(windowStart) >= uptimeWindow
}
<commit_msg>simply online code to only consider most recent scan<commit_after>package contractor

import (
	"sort"
	"time"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/types"
)

// uptimeMinScans is the minimum number of scans required to judge whether a
// host is offline or not.
const uptimeMinScans = 3

// uptimeWindow specifies the duration in which host uptime is checked.
var uptimeWindow = func() time.Duration {
	switch build.Release {
	case "dev":
		return 30 * time.Minute
	case "standard":
		return 7 * 24 * time.Hour // 1 week.
	case "testing":
		return 15 * time.Second
	}
	panic("undefined uptimeWindow")
}()

// IsOffline indicates whether a contract's host should be considered offline,
// based on its scan metrics.
func (c *Contractor) IsOffline(id types.FileContractID) bool {
	c.mu.RLock()
	defer c.mu.RUnlock()
	return c.isOffline(id)
}

// isOffline indicates whether a contract's host should be considered offline,
// based on its scan metrics.
func (c *Contractor) isOffline(id types.FileContractID) bool {
	// Fetch the corresponding contract in the contractor. If the most recent
	// contract is not in the contractors set of active contracts, this contract
	// line is dead, and thus the contract should be considered 'offline'.
	contract, ok := c.contracts[id]
	if !ok {
		return true
	}
	host, ok := c.hdb.Host(contract.HostPublicKey)
	if !ok {
		return true
	}
	if len(host.ScanHistory) < 1 {
		return true
	}
	return host.ScanHistory[len(host.ScanHistory)-1].Success
}
<|endoftext|>


<commit_before>package wallet

import (
	"testing"

	"github.com/NebulousLabs/Sia/types"
)

// TestIntegrationTransactions checks that the transaction history is being
// correctly recorded and extended.
func TestIntegrationTransactions(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}
	wt, err := createWalletTester(t.Name())
	if err != nil {
		t.Fatal(err)
	}
	defer wt.closeWt()

	// Creating the wallet tester results in blocks being mined until the miner
	// has money, which means types.MaturityDelay+1 blocks are created, and
	// each block is going to have a transaction (the miner payout) going to
	// the wallet.
	txns, err := wt.wallet.Transactions(0, 100)
	if err != nil {
		t.Fatal(err)
	}
	if len(txns) != int(types.MaturityDelay+1) {
		t.Error("unexpected transaction history length")
	}
	sentValue := types.NewCurrency64(5000)
	_, err = wt.wallet.SendSiacoins(sentValue, types.UnlockHash{})
	if err != nil {
		t.Fatal(err)
	}
	// No more confirmed transactions have been added.
	txns, err = wt.wallet.Transactions(0, 100)
	if err != nil {
		t.Fatal(err)
	}
	if len(txns) != int(types.MaturityDelay+1) {
		t.Error("unexpected transaction history length")
	}
	// Two transactions added to unconfirmed pool - 1 to fund the exact output,
	// and 1 to hold the exact output.
	if len(wt.wallet.UnconfirmedTransactions()) != 2 {
		t.Error("was expecting 4 unconfirmed transactions")
	}

	b, _ := wt.miner.FindBlock()
	err = wt.cs.AcceptBlock(b)
	if err != nil {
		t.Fatal(err)
	}
	// A confirmed transaction was added for the miner payout, and the 2
	// transactions that were previously unconfirmed.
	txns, err = wt.wallet.Transactions(0, 100)
	if err != nil {
		t.Fatal(err)
	}
	if len(txns) != int(types.MaturityDelay+2+2) {
		t.Errorf("unexpected transaction history length: expected %v, got %v", types.MaturityDelay+2+2, len(txns))
	}

	// Try getting a partial history for just the previous block.
	txns, err = wt.wallet.Transactions(types.MaturityDelay+2, types.MaturityDelay+2)
	if err != nil {
		t.Fatal(err)
	}
	// The partial should include one transaction for a block, and 2 for the
	// send that occurred.
	if len(txns) != 3 {
		t.Errorf("unexpected transaction history length: expected %v, got %v", 3, len(txns))
	}
}

// TestIntegrationTransaction checks that individually queried transactions
// contain the correct values.
func TestIntegrationTransaction(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}
	wt, err := createWalletTester(t.Name())
	if err != nil {
		t.Fatal(err)
	}
	defer wt.closeWt()

	_, exists := wt.wallet.Transaction(types.TransactionID{})
	if exists {
		t.Error("able to query a nonexisting transction")
	}

	// test sending siacoins
	sentValue := types.NewCurrency64(5000)
	sendTxns, err := wt.wallet.SendSiacoins(sentValue, types.UnlockHash{})
	if err != nil {
		t.Fatal(err)
	}
	_, err = wt.miner.AddBlock()
	if err != nil {
		t.Fatal(err)
	}

	// sendTxns[0] is the set-up transaction, sendTxns[1] contains the sentValue output
	txn, exists := wt.wallet.Transaction(sendTxns[1].ID())
	if !exists {
		t.Fatal("unable to query transaction")
	}
	if txn.TransactionID != sendTxns[1].ID() {
		t.Error("wrong transaction was fetched")
	} else if len(txn.Inputs) != 1 || len(txn.Outputs) != 2 {
		t.Error("expected 1 input and 2 outputs, got", len(txn.Inputs), len(txn.Outputs))
	} else if !txn.Outputs[0].Value.Equals(sentValue) {
		t.Errorf("expected first output to equal %v, got %v", sentValue, txn.Outputs[0].Value)
	} else if exp := txn.Inputs[0].Value.Sub(sentValue); !txn.Outputs[1].Value.Equals(exp) {
		t.Errorf("expected first output to equal %v, got %v", exp, txn.Outputs[1].Value)
	}

	// test sending siafunds
	err = wt.wallet.LoadSiagKeys(wt.walletMasterKey, []string{"../../types/siag0of1of1.siakey"})
	if err != nil {
		t.Error(err)
	}
	sentValue = types.NewCurrency64(12)
	sendTxns, err = wt.wallet.SendSiafunds(sentValue, types.UnlockHash{})
	if err != nil {
		t.Fatal(err)
	}
	_, err = wt.miner.AddBlock()
	if err != nil {
		t.Fatal(err)
	}

	txn, exists = wt.wallet.Transaction(sendTxns[1].ID())
	if !exists {
		t.Fatal("unable to query transaction")
	}
	if len(txn.Inputs) != 1 || len(txn.Outputs) != 3 {
		t.Error("expected 1 input and 3 outputs, got", len(txn.Inputs), len(txn.Outputs))
	} else if !txn.Outputs[1].Value.Equals(sentValue) {
		t.Errorf("expected first output to equal %v, got %v", sentValue, txn.Outputs[1].Value)
	} else if exp := txn.Inputs[0].Value.Sub(sentValue); !txn.Outputs[2].Value.Equals(exp) {
		t.Errorf("expected first output to equal %v, got %v", exp, txn.Outputs[2].Value)
	}
}

// TestIntegrationAddressTransactions checks grabbing the history for a single
// address.
func TestIntegrationAddressTransactions(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}
	wt, err := createWalletTester(t.Name())
	if err != nil {
		t.Fatal(err)
	}
	defer wt.closeWt()

	// Grab an address and send it money.
	uc, err := wt.wallet.NextAddress()
	addr := uc.UnlockHash()
	if err != nil {
		t.Fatal(err)
	}
	_, err = wt.wallet.SendSiacoins(types.NewCurrency64(5005), addr)
	if err != nil {
		t.Fatal(err)
	}

	// Check the confirmed balance of the address.
	addrHist := wt.wallet.AddressTransactions(addr)
	if len(addrHist) != 0 {
		t.Error("address should be empty - no confirmed transactions")
	}
	if len(wt.wallet.AddressUnconfirmedTransactions(addr)) == 0 {
		t.Error("addresses unconfirmed transactions should not be empty")
	}
	b, _ := wt.miner.FindBlock()
	err = wt.cs.AcceptBlock(b)
	if err != nil {
		t.Fatal(err)
	}
	addrHist = wt.wallet.AddressTransactions(addr)
	if len(addrHist) == 0 {
		t.Error("address history should have some transactions")
	}
	if len(wt.wallet.AddressUnconfirmedTransactions(addr)) != 0 {
		t.Error("addresses unconfirmed transactions should be empty")
	}
}

// TestTransactionInputOutputIDs verifies that ProcessedTransaction's inputs
// and outputs have a valid ID field.
func TestTransactionInputOutputIDs(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}
	wt, err := createWalletTester(t.Name())
	if err != nil {
		t.Fatal(err)
	}
	defer wt.closeWt()

	// mine a few blocks to create miner payouts
	for i := 0; i < 5; i++ {
		_, err = wt.miner.AddBlock()
		if err != nil {
			t.Fatal(err)
		}
	}

	// create some siacoin outputs
	uc, err := wt.wallet.NextAddress()
	addr := uc.UnlockHash()
	if err != nil {
		t.Fatal(err)
	}
	_, err = wt.wallet.SendSiacoins(types.NewCurrency64(5005), addr)
	if err != nil {
		t.Fatal(err)
	}
	_, err = wt.miner.AddBlock()
	if err != nil {
		t.Fatal(err)
	}

	// verify the miner payouts and siacoin outputs/inputs have correct IDs
	txns, err := wt.wallet.Transactions(0, 1000)
	if err != nil {
		t.Fatal(err)
	}

	outputIDs := make(map[types.OutputID]struct{})
	for _, txn := range txns {
		block, _ := wt.cs.BlockAtHeight(txn.ConfirmationHeight)
		for i, output := range txn.Outputs {
			outputIDs[output.ID] = struct{}{}
			if output.FundType == types.SpecifierMinerPayout {
				if output.ID != types.OutputID(block.MinerPayoutID(uint64(i))) {
					t.Fatal("miner payout had incorrect output ID")
				}
			}
			if output.FundType == types.SpecifierSiacoinOutput {
				if output.ID != types.OutputID(txn.Transaction.SiacoinOutputID(uint64(i))) {
					t.Fatal("siacoin output had incorrect output ID")
				}
			}
		}
		for _, input := range txn.Inputs {
			if _, exists := outputIDs[input.ParentID]; !exists {
				t.Fatal("input has ParentID that points to a nonexistent output:", input.ParentID)
			}
		}
	}
}
<commit_msg>add BenchmarkAddressTransactions<commit_after>package wallet

import (
	"testing"

	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

// TestIntegrationTransactions checks that the transaction history is being
// correctly recorded and extended.
func TestIntegrationTransactions(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}
	wt, err := createWalletTester(t.Name())
	if err != nil {
		t.Fatal(err)
	}
	defer wt.closeWt()

	// Creating the wallet tester results in blocks being mined until the miner
	// has money, which means types.MaturityDelay+1 blocks are created, and
	// each block is going to have a transaction (the miner payout) going to
	// the wallet.
	txns, err := wt.wallet.Transactions(0, 100)
	if err != nil {
		t.Fatal(err)
	}
	if len(txns) != int(types.MaturityDelay+1) {
		t.Error("unexpected transaction history length")
	}
	sentValue := types.NewCurrency64(5000)
	_, err = wt.wallet.SendSiacoins(sentValue, types.UnlockHash{})
	if err != nil {
		t.Fatal(err)
	}
	// No more confirmed transactions have been added.
	txns, err = wt.wallet.Transactions(0, 100)
	if err != nil {
		t.Fatal(err)
	}
	if len(txns) != int(types.MaturityDelay+1) {
		t.Error("unexpected transaction history length")
	}
	// Two transactions added to unconfirmed pool - 1 to fund the exact output,
	// and 1 to hold the exact output.
	if len(wt.wallet.UnconfirmedTransactions()) != 2 {
		t.Error("was expecting 4 unconfirmed transactions")
	}

	b, _ := wt.miner.FindBlock()
	err = wt.cs.AcceptBlock(b)
	if err != nil {
		t.Fatal(err)
	}
	// A confirmed transaction was added for the miner payout, and the 2
	// transactions that were previously unconfirmed.
	txns, err = wt.wallet.Transactions(0, 100)
	if err != nil {
		t.Fatal(err)
	}
	if len(txns) != int(types.MaturityDelay+2+2) {
		t.Errorf("unexpected transaction history length: expected %v, got %v", types.MaturityDelay+2+2, len(txns))
	}

	// Try getting a partial history for just the previous block.
	txns, err = wt.wallet.Transactions(types.MaturityDelay+2, types.MaturityDelay+2)
	if err != nil {
		t.Fatal(err)
	}
	// The partial should include one transaction for a block, and 2 for the
	// send that occurred.
	if len(txns) != 3 {
		t.Errorf("unexpected transaction history length: expected %v, got %v", 3, len(txns))
	}
}

// TestIntegrationTransaction checks that individually queried transactions
// contain the correct values.
func TestIntegrationTransaction(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}
	wt, err := createWalletTester(t.Name())
	if err != nil {
		t.Fatal(err)
	}
	defer wt.closeWt()

	_, exists := wt.wallet.Transaction(types.TransactionID{})
	if exists {
		t.Error("able to query a nonexisting transction")
	}

	// test sending siacoins
	sentValue := types.NewCurrency64(5000)
	sendTxns, err := wt.wallet.SendSiacoins(sentValue, types.UnlockHash{})
	if err != nil {
		t.Fatal(err)
	}
	_, err = wt.miner.AddBlock()
	if err != nil {
		t.Fatal(err)
	}

	// sendTxns[0] is the set-up transaction, sendTxns[1] contains the sentValue output
	txn, exists := wt.wallet.Transaction(sendTxns[1].ID())
	if !exists {
		t.Fatal("unable to query transaction")
	}
	if txn.TransactionID != sendTxns[1].ID() {
		t.Error("wrong transaction was fetched")
	} else if len(txn.Inputs) != 1 || len(txn.Outputs) != 2 {
		t.Error("expected 1 input and 2 outputs, got", len(txn.Inputs), len(txn.Outputs))
	} else if !txn.Outputs[0].Value.Equals(sentValue) {
		t.Errorf("expected first output to equal %v, got %v", sentValue, txn.Outputs[0].Value)
	} else if exp := txn.Inputs[0].Value.Sub(sentValue); !txn.Outputs[1].Value.Equals(exp) {
		t.Errorf("expected first output to equal %v, got %v", exp, txn.Outputs[1].Value)
	}

	// test sending siafunds
	err = wt.wallet.LoadSiagKeys(wt.walletMasterKey, []string{"../../types/siag0of1of1.siakey"})
	if err != nil {
		t.Error(err)
	}
	sentValue = types.NewCurrency64(12)
	sendTxns, err = wt.wallet.SendSiafunds(sentValue, types.UnlockHash{})
	if err != nil {
		t.Fatal(err)
	}
	_, err = wt.miner.AddBlock()
	if err != nil {
		t.Fatal(err)
	}

	txn, exists = wt.wallet.Transaction(sendTxns[1].ID())
	if !exists {
		t.Fatal("unable to query transaction")
	}
	if len(txn.Inputs) != 1 || len(txn.Outputs) != 3 {
		t.Error("expected 1 input and 3 outputs, got", len(txn.Inputs), len(txn.Outputs))
	} else if !txn.Outputs[1].Value.Equals(sentValue) {
		t.Errorf("expected first output to equal %v, got %v", sentValue, txn.Outputs[1].Value)
	} else if exp := txn.Inputs[0].Value.Sub(sentValue); !txn.Outputs[2].Value.Equals(exp) {
		t.Errorf("expected first output to equal %v, got %v", exp, txn.Outputs[2].Value)
	}
}

// TestIntegrationAddressTransactions checks grabbing the history for a single
// address.
func TestIntegrationAddressTransactions(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}
	wt, err := createWalletTester(t.Name())
	if err != nil {
		t.Fatal(err)
	}
	defer wt.closeWt()

	// Grab an address and send it money.
	uc, err := wt.wallet.NextAddress()
	addr := uc.UnlockHash()
	if err != nil {
		t.Fatal(err)
	}
	_, err = wt.wallet.SendSiacoins(types.NewCurrency64(5005), addr)
	if err != nil {
		t.Fatal(err)
	}

	// Check the confirmed balance of the address.
	addrHist := wt.wallet.AddressTransactions(addr)
	if len(addrHist) != 0 {
		t.Error("address should be empty - no confirmed transactions")
	}
	if len(wt.wallet.AddressUnconfirmedTransactions(addr)) == 0 {
		t.Error("addresses unconfirmed transactions should not be empty")
	}
	b, _ := wt.miner.FindBlock()
	err = wt.cs.AcceptBlock(b)
	if err != nil {
		t.Fatal(err)
	}
	addrHist = wt.wallet.AddressTransactions(addr)
	if len(addrHist) == 0 {
		t.Error("address history should have some transactions")
	}
	if len(wt.wallet.AddressUnconfirmedTransactions(addr)) != 0 {
		t.Error("addresses unconfirmed transactions should be empty")
	}
}

// TestTransactionInputOutputIDs verifies that ProcessedTransaction's inputs
// and outputs have a valid ID field.
func TestTransactionInputOutputIDs(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}
	wt, err := createWalletTester(t.Name())
	if err != nil {
		t.Fatal(err)
	}
	defer wt.closeWt()

	// mine a few blocks to create miner payouts
	for i := 0; i < 5; i++ {
		_, err = wt.miner.AddBlock()
		if err != nil {
			t.Fatal(err)
		}
	}

	// create some siacoin outputs
	uc, err := wt.wallet.NextAddress()
	addr := uc.UnlockHash()
	if err != nil {
		t.Fatal(err)
	}
	_, err = wt.wallet.SendSiacoins(types.NewCurrency64(5005), addr)
	if err != nil {
		t.Fatal(err)
	}
	_, err = wt.miner.AddBlock()
	if err != nil {
		t.Fatal(err)
	}

	// verify the miner payouts and siacoin outputs/inputs have correct IDs
	txns, err := wt.wallet.Transactions(0, 1000)
	if err != nil {
		t.Fatal(err)
	}

	outputIDs := make(map[types.OutputID]struct{})
	for _, txn := range txns {
		block, _ := wt.cs.BlockAtHeight(txn.ConfirmationHeight)
		for i, output := range txn.Outputs {
			outputIDs[output.ID] = struct{}{}
			if output.FundType == types.SpecifierMinerPayout {
				if output.ID != types.OutputID(block.MinerPayoutID(uint64(i))) {
					t.Fatal("miner payout had incorrect output ID")
				}
			}
			if output.FundType == types.SpecifierSiacoinOutput {
				if output.ID != types.OutputID(txn.Transaction.SiacoinOutputID(uint64(i))) {
					t.Fatal("siacoin output had incorrect output ID")
				}
			}
		}
		for _, input := range txn.Inputs {
			if _, exists := outputIDs[input.ParentID]; !exists {
				t.Fatal("input has ParentID that points to a nonexistent output:", input.ParentID)
			}
		}
	}
}

// BenchmarkAddressTransactions benchmarks the AddressTransactions method,
// using the near-worst-case scenario of 10,000 transactions to search through
// with only a single relevant transaction.
func BenchmarkAddressTransactions(b *testing.B) {
	wt, err := createWalletTester(b.Name())
	if err != nil {
		b.Fatal(err)
	}
	// add a bunch of fake transactions to the db
	//
	// NOTE: this is somewhat brittle, but the alternative (generating
	// authentic transactions) is prohibitively slow.
	wt.wallet.mu.Lock()
	for i := 0; i < 10000; i++ {
		err := dbAppendProcessedTransaction(wt.wallet.dbTx, modules.ProcessedTransaction{
			TransactionID: types.TransactionID{1},
		})
		if err != nil {
			b.Fatal(err)
		}
	}
	// add a single relevant transaction
	searchAddr := types.UnlockHash{1}
	err = dbAppendProcessedTransaction(wt.wallet.dbTx, modules.ProcessedTransaction{
		TransactionID: types.TransactionID{1},
		Inputs: []modules.ProcessedInput{{
			RelatedAddress: searchAddr,
		}},
	})
	if err != nil {
		b.Fatal(err)
	}
	wt.wallet.syncDB()
	wt.wallet.mu.Unlock()

	b.ResetTimer()
	b.Run("indexed", func(b *testing.B) {
		for i := 0; i < b.N; i++ {
			txns := wt.wallet.AddressTransactions(searchAddr)
			if len(txns) != 1 {
				b.Fatal(len(txns))
			}
		}
	})
	b.Run("indexed-nosync", func(b *testing.B) {
		wt.wallet.db.NoSync = true
		for i := 0; i < b.N; i++ {
			txns := wt.wallet.AddressTransactions(searchAddr)
			if len(txns) != 1 {
				b.Fatal(len(txns))
			}
		}
		wt.wallet.db.NoSync = false
	})
	b.Run("unindexed", func(b *testing.B) {
		for i := 0; i < b.N; i++ {
			wt.wallet.mu.Lock()
			wt.wallet.syncDB()
			var pts []modules.ProcessedTransaction
			it := dbProcessedTransactionsIterator(wt.wallet.dbTx)
			for it.next() {
				pt := it.value()
				relevant := false
				for _, input := range pt.Inputs {
					relevant = relevant || input.RelatedAddress == searchAddr
				}
				for _, output := range pt.Outputs {
					relevant = relevant || output.RelatedAddress == searchAddr
				}
				if relevant {
					pts = append(pts, pt)
				}
			}
			_ = pts
			wt.wallet.mu.Unlock()
		}
	})
}
<|endoftext|>


<commit_before>// Copyright 2018 The Periph Authors. All rights reserved.
// Use of this source code is governed under the Apache License, Version 2.0
// that can be found in the LICENSE file.

package fs

import (
	"io/ioutil"
	"net"
	"os"
	"testing"
	"time"
)

func TestEpollEvent_String(t *testing.T) {
	if s := (epollIN | epollOUT).String(); s != "IN|OUT" {
		t.Fatal(s)
	}
	if s := (epollERR | epollEvent(0x1000)).String(); s != "ERR|0x1000" {
		t.Fatal(s)
	}
	if s := epollEvent(0).String(); s != "0" {
		t.Fatal(s)
	}
}

func TestAddFd_Zero(t *testing.T) {
	// We assume this is a bad file descriptor.
	ev, cancelEv := getListener(t)
	defer cancelEv()

	const flags = epollET | epollPRI
	if err := ev.addFd(0xFFFFFFFF, make(chan time.Time), flags); err == nil || err.Error() != "bad file descriptor" {
		t.Fatal("expected failure", err)
	}
}

func TestAddFd_File(t *testing.T) {
	// listen cannot listen to a file.
	ev, cancelEv := getListener(t)
	defer cancelEv()

	f, err := ioutil.TempFile("", "periph_fs")
	if err != nil {
		t.Fatal(err)
	}
	defer func() {
		if err := os.Remove(f.Name()); err != nil {
			t.Fatal(err)
		}
	}()

	const flags = epollET | epollPRI
	if err := ev.addFd(f.Fd(), make(chan time.Time), flags); err == nil || err.Error() != "operation not permitted" {
		t.Fatal("expected failure", err)
	}
}

func TestListen_Pipe(t *testing.T) {
	start := time.Now()
	ev, cancelEv := getListener(t)
	defer cancelEv()

	r, w, err := os.Pipe()
	if err != nil {
		t.Fatal(err)
	}
	c := make(chan time.Time)
	// Pipes do not support epollPRI, so use epollIN instead.
	const flags = epollET | epollIN
	if err := ev.addFd(r.Fd(), c, flags); err != nil {
		t.Fatal(err)
	}

	// Produce a single event.
	if _, err := w.Write([]byte("foo")); err != nil {
		t.Fatal(err)
	}
	expectChan(t, c, start)
	notExpectChan(t, c, "should have produced a single event")

	// Produce one or two events.
	if _, err := w.Write([]byte("bar")); err != nil {
		t.Fatal(err)
	}
	if _, err := w.Write([]byte("baz")); err != nil {
		t.Fatal(err)
	}
	expectChan(t, c, start)
	// It's a race condition between EpollWait() and reading back from the
	// channel.
	select {
	case <-c:
	default:
	}

	if err := ev.removeFd(r.Fd()); err != nil {
		t.Fatal(err)
	}
}

func TestListen_Socket(t *testing.T) {
	start := time.Now()
	ev, _ := getListener(t)

	ln, err := net.ListenTCP("tcp4", nil)
	if err != nil {
		t.Fatal(err)
	}
	defer func() {
		if err := ln.Close(); err != nil {
			t.Fatal(err)
		}
	}()

	conn, err := net.DialTCP("tcp4", nil, ln.Addr().(*net.TCPAddr))
	if err != nil {
		t.Fatal(err)
	}
	defer func() {
		if err := conn.Close(); err != nil {
			t.Fatal(err)
		}
	}()

	recv, err := ln.Accept()
	if err != nil {
		t.Fatal(err)
	}
	defer func() {
		if err := recv.Close(); err != nil {
			t.Fatal(err)
		}
	}()

	f, err := recv.(*net.TCPConn).File()
	if err != nil {
		t.Fatal(err)
	}

	// This channel needs to be buffered since there's going to be an even
	// immediately triggered.
	c := make(chan time.Time, 1)
	// TODO(maruel): Sockets do support epollPRI on out-of-band data. This would
	// make this test a bit more similar to testing a GPIO sysfs file descriptor.
	const flags = epollET | epollIN
	if err := ev.addFd(f.Fd(), c, flags); err != nil {
		t.Fatal(err)
	}
	notExpectChan(t, c, "starting should not produce an event")

	// Produce one or two events.
	// It's a race condition between EpollWait() and reading back from the
	// channel.
	if _, err := conn.Write([]byte("bar\n")); err != nil {
		t.Fatal(err)
	}
	if _, err := conn.Write([]byte("baz\n")); err != nil {
		t.Fatal(err)
	}
	expectChan(t, c, start)
	// It's a race condition between EpollWait() and reading back from the
	// channel.
	select {
	case <-c:
	default:
	}

	// Empty the buffer.
	var buf [16]byte
	expected := "bar\nbaz\n"
	if n, err := recv.Read(buf[:]); n != len(expected) || err != nil {
		t.Fatal(n, err)
	}
	if s := string(buf[:len(expected)]); s != expected {
		t.Fatal(s)
	}

	// Produce one event.
	if _, err := conn.Write([]byte("foo\n")); err != nil {
		t.Fatal(err)
	}
	expectChan(t, c, start)
	notExpectChan(t, c, "should have produced a single event")

	if err := ev.removeFd(f.Fd()); err != nil {
		t.Fatal(err)
	}
}

func TestWakeUpLoop(t *testing.T) {
	// Make sure it doesn't hang when the loop is not running.
	ev := &eventsListener{}
	ev.wakeUpLoop(nil)
}

//

// getListener returns a preinitialized eventsListenenr
func getListener(t *testing.T) (*eventsListener, func()) {
	ev := &eventsListener{}
	if err := ev.init(); err != nil {
		t.Fatal(err)
	}
	return ev, func() {
		if err := ev.stopLoop(); err != nil {
			t.Fatal(err)
		}
	}
}

func expectChan(t *testing.T, c <-chan time.Time, start time.Time) {
	select {
	case v := <-c:
		if v.Before(start) {
			t.Fatal("received an timestamp that was too early", v, start)
		}
	case <-time.After(5 * time.Second):
		t.Fatal("timed out after 5 seconds, waiting for an event")
	}
}

func notExpectChan(t *testing.T, c <-chan time.Time, errmsg string) {
	select {
	case <-c:
		t.Fatal(errmsg)
	default:
	}
}
<commit_msg>fs: workaround a test case that is flaky (#375)<commit_after>// Copyright 2018 The Periph Authors. All rights reserved.
// Use of this source code is governed under the Apache License, Version 2.0
// that can be found in the LICENSE file.

package fs

import (
	"io/ioutil"
	"net"
	"os"
	"testing"
	"time"
)

func TestEpollEvent_String(t *testing.T) {
	if s := (epollIN | epollOUT).String(); s != "IN|OUT" {
		t.Fatal(s)
	}
	if s := (epollERR | epollEvent(0x1000)).String(); s != "ERR|0x1000" {
		t.Fatal(s)
	}
	if s := epollEvent(0).String(); s != "0" {
		t.Fatal(s)
	}
}

func TestAddFd_Zero(t *testing.T) {
	// We assume this is a bad file descriptor.
	ev, cancelEv := getListener(t)
	defer cancelEv()

	const flags = epollET | epollPRI
	if err := ev.addFd(0xFFFFFFFF, make(chan time.Time), flags); err == nil || err.Error() != "bad file descriptor" {
		t.Fatal("expected failure", err)
	}
}

func TestAddFd_File(t *testing.T) {
	// listen cannot listen to a file.
	ev, cancelEv := getListener(t)
	defer cancelEv()

	f, err := ioutil.TempFile("", "periph_fs")
	if err != nil {
		t.Fatal(err)
	}
	defer func() {
		if err := os.Remove(f.Name()); err != nil {
			t.Fatal(err)
		}
	}()

	const flags = epollET | epollPRI
	if err := ev.addFd(f.Fd(), make(chan time.Time), flags); err == nil || err.Error() != "operation not permitted" {
		t.Fatal("expected failure", err)
	}
}

func TestListen_Pipe(t *testing.T) {
	start := time.Now()
	ev, cancelEv := getListener(t)
	defer cancelEv()

	r, w, err := os.Pipe()
	if err != nil {
		t.Fatal(err)
	}
	c := make(chan time.Time)
	// Pipes do not support epollPRI, so use epollIN instead.
	const flags = epollET | epollIN
	if err := ev.addFd(r.Fd(), c, flags); err != nil {
		t.Fatal(err)
	}

	// Produce a single event.
	if _, err := w.Write([]byte("foo")); err != nil {
		t.Fatal(err)
	}
	expectChan(t, c, start)
	notExpectChan(t, c, "should have produced a single event")

	// Produce one or two events.
	if _, err := w.Write([]byte("bar")); err != nil {
		t.Fatal(err)
	}
	if _, err := w.Write([]byte("baz")); err != nil {
		t.Fatal(err)
	}
	expectChan(t, c, start)
	// It's a race condition between EpollWait() and reading back from the
	// channel.
	select {
	case <-c:
	default:
	}

	if err := ev.removeFd(r.Fd()); err != nil {
		t.Fatal(err)
	}
}

func TestListen_Socket(t *testing.T) {
	start := time.Now()
	ev, _ := getListener(t)

	ln, err := net.ListenTCP("tcp4", nil)
	if err != nil {
		t.Fatal(err)
	}
	defer func() {
		if err := ln.Close(); err != nil {
			t.Fatal(err)
		}
	}()

	conn, err := net.DialTCP("tcp4", nil, ln.Addr().(*net.TCPAddr))
	if err != nil {
		t.Fatal(err)
	}
	defer func() {
		if err := conn.Close(); err != nil {
			t.Fatal(err)
		}
	}()

	recv, err := ln.Accept()
	if err != nil {
		t.Fatal(err)
	}
	defer func() {
		if err := recv.Close(); err != nil {
			t.Fatal(err)
		}
	}()

	f, err := recv.(*net.TCPConn).File()
	if err != nil {
		t.Fatal(err)
	}

	// This channel needs to be buffered since there's going to be an even
	// immediately triggered.
	c := make(chan time.Time, 1)
	// TODO(maruel): Sockets do support epollPRI on out-of-band data. This would
	// make this test a bit more similar to testing a GPIO sysfs file descriptor.
	const flags = epollET | epollIN
	if err := ev.addFd(f.Fd(), c, flags); err != nil {
		t.Fatal(err)
	}
	notExpectChan(t, c, "starting should not produce an event")

	// Produce one or two events.
	// It's a race condition between EpollWait() and reading back from the
	// channel.
	if _, err := conn.Write([]byte("bar\n")); err != nil {
		t.Fatal(err)
	}
	if _, err := conn.Write([]byte("baz\n")); err != nil {
		t.Fatal(err)
	}
	expectChan(t, c, start)
	// It's a race condition between EpollWait() and reading back from the
	// channel.
	select {
	case <-c:
	default:
	}

	// Empty the buffer.
	var buf [16]byte
	expected := "bar\nbaz\n"
	if n, err := recv.Read(buf[:]); n != len(expected) || err != nil {
		t.Fatal(n, err)
	}
	if s := string(buf[:len(expected)]); s != expected {
		t.Fatal(s)
	}

	// Produce one event.
	if _, err := conn.Write([]byte("foo\n")); err != nil {
		t.Fatal(err)
	}
	expectChan(t, c, start)
	// This is part of https://github.com/google/periph/issues/323
	//notExpectChan(t, c, "should have produced a single event")
	// Instead consume any extraneous event.
	select {
	case <-c:
	default:
	}

	if err := ev.removeFd(f.Fd()); err != nil {
		t.Fatal(err)
	}
}

func TestWakeUpLoop(t *testing.T) {
	// Make sure it doesn't hang when the loop is not running.
	ev := &eventsListener{}
	ev.wakeUpLoop(nil)
}

//

// getListener returns a preinitialized eventsListenenr
func getListener(t *testing.T) (*eventsListener, func()) {
	ev := &eventsListener{}
	if err := ev.init(); err != nil {
		t.Fatal(err)
	}
	return ev, func() {
		if err := ev.stopLoop(); err != nil {
			t.Fatal(err)
		}
	}
}

func expectChan(t *testing.T, c <-chan time.Time, start time.Time) {
	select {
	case v := <-c:
		if v.Before(start) {
			t.Fatal("received an timestamp that was too early", v, start)
		}
	case <-time.After(5 * time.Second):
		t.Fatal("timed out after 5 seconds, waiting for an event")
	}
}

func notExpectChan(t *testing.T, c <-chan time.Time, errmsg string) {
	select {
	case <-c:
		t.Fatal(errmsg)
	default:
	}
}
<|endoftext|>


<commit_before>package execrunner

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"sync"
	"syscall"

	"github.com/pkg/errors"

	"code.cloudfoundry.org/commandrunner"
	"code.cloudfoundry.org/garden"
	"code.cloudfoundry.org/guardian/logging"
	"code.cloudfoundry.org/guardian/rundmc/depot"
	"code.cloudfoundry.org/guardian/rundmc/goci"
	"code.cloudfoundry.org/lager"
)

type WindowsExecRunner struct {
	runtimePath     string
	commandRunner   commandrunner.CommandRunner
	processes       map[string]*process
	processMux      *sync.Mutex
	bundleSaver     depot.BundleSaver
	bundleLookupper depot.BundleLookupper
	processDepot    ProcessDepot
}

func NewWindowsExecRunner(runtimePath string, commandRunner commandrunner.CommandRunner, bundleSaver depot.BundleSaver, bundleLookupper depot.BundleLookupper, processDepot ProcessDepot) *WindowsExecRunner {
	return &WindowsExecRunner{
		runtimePath:     runtimePath,
		commandRunner:   commandRunner,
		processes:       map[string]*process{},
		processMux:      new(sync.Mutex),
		bundleSaver:     bundleSaver,
		bundleLookupper: bundleLookupper,
		processDepot:    processDepot,
	}
}

type process struct {
	id           string
	exitCode     int
	exitErr      error
	exitMutex    *sync.RWMutex
	cleanup      func() error
	logger       lager.Logger
	stdoutWriter *DynamicMultiWriter
	stderrWriter *DynamicMultiWriter
	stdin        *os.File
	stdout       *os.File
	stderr       *os.File
	outputWg     *sync.WaitGroup
}

func (e *WindowsExecRunner) Run(
	log lager.Logger, processID, sandboxHandle string,
	pio garden.ProcessIO, _ bool, procJSON io.Reader, extraCleanup func() error,
) (garden.Process, error) {
	log = log.Session("execrunner")

	log.Info("start")
	defer log.Info("done")

	processPath, err := e.processDepot.CreateProcessDir(log, sandboxHandle, processID)
	if err != nil {
		return nil, err
	}

	specPath := filepath.Join(processPath, "spec.json")
	if err := writeProcessJSON(procJSON, specPath); err != nil {
		return nil, err
	}
	return e.runProcess(log, "exec", []string{"-p", specPath, sandboxHandle}, processID, processPath, pio,
		extraCleanup)
}

func writeProcessJSON(procJSON io.Reader, specPath string) error {
	specFile, err := os.OpenFile(specPath, os.O_WRONLY|os.O_CREATE, 0600)
	if err != nil {
		return errors.Wrap(err, "opening process spec file for writing")
	}
	defer specFile.Close()
	if _, err := io.Copy(specFile, procJSON); err != nil {
		return errors.Wrap(err, "writing process spec")
	}

	return nil
}

func (e *WindowsExecRunner) RunPea(
	log lager.Logger, processID string, processBundle goci.Bndl, sandboxHandle string,
	pio garden.ProcessIO, tty bool, procJSON io.Reader, extraCleanup func() error,
) (garden.Process, error) {
	log = log.Session("execrunner")

	log.Info("start")
	defer log.Info("done")

	processPath, err := e.processDepot.CreateProcessDir(log, sandboxHandle, processID)
	if err != nil {
		return nil, err
	}

	err = e.bundleSaver.Save(processBundle, processPath)
	if err != nil {
		return nil, err
	}

	return e.runProcess(log, "run", []string{"--bundle", processPath, processID}, processID, processPath, pio, extraCleanup)
}

func (e *WindowsExecRunner) runProcess(
	log lager.Logger, runMode string, runtimeExtraArgs []string, processID, processPath string,
	pio garden.ProcessIO, extraCleanup func() error,
) (garden.Process, error) {
	logR, logW, err := os.Pipe()
	if err != nil {
		return nil, errors.Wrap(err, "creating log pipe")
	}
	defer logW.Close()

	stdinR, stdinW, err := os.Pipe()
	if err != nil {
		return nil, errors.Wrap(err, "creating stdin pipe")
	}
	defer stdinR.Close()

	stdoutR, stdoutW, err := os.Pipe()
	if err != nil {
		return nil, errors.Wrap(err, "creating stdout pipe")
	}
	defer stdoutW.Close()

	stderrR, stderrW, err := os.Pipe()
	if err != nil {
		return nil, errors.Wrap(err, "creating stderr pipe")
	}
	defer stderrW.Close()

	var childLogW syscall.Handle

	// GetCurrentProcess doesn't error
	self, _ := syscall.GetCurrentProcess()
	// duplicate handle so it is inheritable by child process
	err = syscall.DuplicateHandle(self, syscall.Handle(logW.Fd()), self, &childLogW, 0, true, syscall.DUPLICATE_SAME_ACCESS)
	if err != nil {
		return nil, errors.Wrap(err, "duplicating log pipe handle")
	}

	cmd := exec.Command(e.runtimePath, "--debug", "--log-handle", strconv.FormatUint(uint64(childLogW), 10), "--log-format", "json", runMode, "--pid-file", filepath.Join(processPath, "pidfile"))
	cmd.Args = append(cmd.Args, runtimeExtraArgs...)

	cmd.Stdin = stdinR
	cmd.Stdout = stdoutW
	cmd.Stderr = stderrW

	if err := e.commandRunner.Start(cmd); err != nil {
		return nil, errors.Wrap(err, "execing runtime plugin")
	}

	go streamLogs(log, logR)

	cleanup := func() error {
		e.processMux.Lock()
		delete(e.processes, processID)
		e.processMux.Unlock()

		if extraCleanup != nil {
			return extraCleanup()
		}

		return nil
	}

	proc := &process{
		id:           processID,
		cleanup:      cleanup,
		logger:       log,
		stdin:        stdinW,
		stdout:       stdoutR,
		stderr:       stderrR,
		stdoutWriter: NewDynamicMultiWriter(),
		stderrWriter: NewDynamicMultiWriter(),
		outputWg:     &sync.WaitGroup{},
		exitMutex:    new(sync.RWMutex),
	}

	e.processMux.Lock()
	e.processes[processID] = proc
	e.processMux.Unlock()

	proc.stream(pio, false)

	proc.exitMutex.Lock()

	go func() {
		if err := e.commandRunner.Wait(cmd); err != nil {
			if exitErr, ok := err.(*exec.ExitError); ok {
				if status, ok := exitErr.Sys().(syscall.WaitStatus); ok {
					proc.exitCode = status.ExitStatus()
				} else {
					proc.exitCode = 1
					proc.exitErr = errors.New("couldn't get WaitStatus")
				}
			} else {
				proc.exitCode = 1
				proc.exitErr = err
			}
		}
		// the streamLogs go func will only exit once this handle is closed
		syscall.CloseHandle(childLogW)
		proc.exitMutex.Unlock()
	}()

	return proc, nil
}

func (e *WindowsExecRunner) Attach(log lager.Logger, _, processID string, pio garden.ProcessIO) (garden.Process, error) {
	proc, err := e.getProcess(processID)
	if err != nil {
		return nil, err
	}

	proc.stream(pio, true)

	return proc, nil
}

func (e *WindowsExecRunner) getProcess(processID string) (*process, error) {
	e.processMux.Lock()
	defer e.processMux.Unlock()

	proc, ok := e.processes[processID]
	if !ok {
		return nil, garden.ProcessNotFoundError{ProcessID: processID}
	}

	return proc, nil
}

func (p *process) stream(pio garden.ProcessIO, duplicate bool) {
	var procStdin *os.File

	procStdin = p.stdin

	if pio.Stdin != nil {
		if duplicate {
			var dupped syscall.Handle
			self, _ := syscall.GetCurrentProcess()
			err := syscall.DuplicateHandle(self, syscall.Handle(p.stdin.Fd()), self, &dupped, 0, false, syscall.DUPLICATE_SAME_ACCESS)
			if err != nil {
				panic(err)
			}

			procStdin = os.NewFile(uintptr(dupped), fmt.Sprintf("%s.stdin", p.id))
		}

		go func() {
			io.Copy(procStdin, pio.Stdin)
			procStdin.Close()
		}()
	}

	if pio.Stdout != nil {
		count := p.stdoutWriter.Attach(pio.Stdout)
		if count == 1 {
			p.outputWg.Add(1)
			go func() {
				io.Copy(p.stdoutWriter, p.stdout)
				p.stdout.Close()
				p.outputWg.Done()
			}()
		}
	}

	if pio.Stderr != nil {
		count := p.stderrWriter.Attach(pio.Stderr)
		if count == 1 {
			p.outputWg.Add(1)
			go func() {
				io.Copy(p.stderrWriter, p.stderr)
				p.stderr.Close()
				p.outputWg.Done()
			}()
		}
	}
}

func (p *process) ID() string {
	return p.id
}

func (p *process) Wait() (int, error) {
	p.exitMutex.RLock()
	defer p.exitMutex.RUnlock()

	p.outputWg.Wait()

	p.stdin.Close()
	p.stdout.Close()
	p.stderr.Close()

	if p.cleanup != nil {
		if err := p.cleanup(); err != nil {
			p.logger.Error("process-cleanup", err)
		}
	}

	return p.exitCode, p.exitErr
}

func (p *process) SetTTY(ttySpec garden.TTYSpec) error {
	return nil
}

func (p *process) Signal(signal garden.Signal) error {
	return nil
}

func streamLogs(logger lager.Logger, src *os.File) {
	defer src.Close()
	scanner := bufio.NewScanner(src)

	for scanner.Scan() {
		nextLogLine := scanner.Bytes()
		logging.ForwardRuncLogsToLager(logger, "winc", nextLogLine)
	}

	logger.Info("done-streaming-winc-logs")
}
<commit_msg>Always release lock and close handle on process exec<commit_after>package execrunner

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"sync"
	"syscall"

	"github.com/pkg/errors"

	"code.cloudfoundry.org/commandrunner"
	"code.cloudfoundry.org/garden"
	"code.cloudfoundry.org/guardian/logging"
	"code.cloudfoundry.org/guardian/rundmc/depot"
	"code.cloudfoundry.org/guardian/rundmc/goci"
	"code.cloudfoundry.org/lager"
)

type WindowsExecRunner struct {
	runtimePath     string
	commandRunner   commandrunner.CommandRunner
	processes       map[string]*process
	processMux      *sync.Mutex
	bundleSaver     depot.BundleSaver
	bundleLookupper depot.BundleLookupper
	processDepot    ProcessDepot
}

func NewWindowsExecRunner(runtimePath string, commandRunner commandrunner.CommandRunner, bundleSaver depot.BundleSaver, bundleLookupper depot.BundleLookupper, processDepot ProcessDepot) *WindowsExecRunner {
	return &WindowsExecRunner{
		runtimePath:     runtimePath,
		commandRunner:   commandRunner,
		processes:       map[string]*process{},
		processMux:      new(sync.Mutex),
		bundleSaver:     bundleSaver,
		bundleLookupper: bundleLookupper,
		processDepot:    processDepot,
	}
}

type process struct {
	id           string
	exitCode     int
	exitErr      error
	exitMutex    *sync.RWMutex
	cleanup      func() error
	logger       lager.Logger
	stdoutWriter *DynamicMultiWriter
	stderrWriter *DynamicMultiWriter
	stdin        *os.File
	stdout       *os.File
	stderr       *os.File
	outputWg     *sync.WaitGroup
}

func (e *WindowsExecRunner) Run(
	log lager.Logger, processID, sandboxHandle string,
	pio garden.ProcessIO, _ bool, procJSON io.Reader, extraCleanup func() error,
) (garden.Process, error) {
	log = log.Session("execrunner")

	log.Info("start")
	defer log.Info("done")

	processPath, err := e.processDepot.CreateProcessDir(log, sandboxHandle, processID)
	if err != nil {
		return nil, err
	}

	specPath := filepath.Join(processPath, "spec.json")
	if err := writeProcessJSON(procJSON, specPath); err != nil {
		return nil, err
	}
	return e.runProcess(log, "exec", []string{"-p", specPath, sandboxHandle}, processID, processPath, pio,
		extraCleanup)
}

func writeProcessJSON(procJSON io.Reader, specPath string) error {
	specFile, err := os.OpenFile(specPath, os.O_WRONLY|os.O_CREATE, 0600)
	if err != nil {
		return errors.Wrap(err, "opening process spec file for writing")
	}
	defer specFile.Close()
	if _, err := io.Copy(specFile, procJSON); err != nil {
		return errors.Wrap(err, "writing process spec")
	}

	return nil
}

func (e *WindowsExecRunner) RunPea(
	log lager.Logger, processID string, processBundle goci.Bndl, sandboxHandle string,
	pio garden.ProcessIO, tty bool, procJSON io.Reader, extraCleanup func() error,
) (garden.Process, error) {
	log = log.Session("execrunner")

	log.Info("start")
	defer log.Info("done")

	processPath, err := e.processDepot.CreateProcessDir(log, sandboxHandle, processID)
	if err != nil {
		return nil, err
	}

	err = e.bundleSaver.Save(processBundle, processPath)
	if err != nil {
		return nil, err
	}

	return e.runProcess(log, "run", []string{"--bundle", processPath, processID}, processID, processPath, pio, extraCleanup)
}

func (e *WindowsExecRunner) runProcess(
	log lager.Logger, runMode string, runtimeExtraArgs []string, processID, processPath string,
	pio garden.ProcessIO, extraCleanup func() error,
) (garden.Process, error) {
	logR, logW, err := os.Pipe()
	if err != nil {
		return nil, errors.Wrap(err, "creating log pipe")
	}
	defer logW.Close()

	stdinR, stdinW, err := os.Pipe()
	if err != nil {
		return nil, errors.Wrap(err, "creating stdin pipe")
	}
	defer stdinR.Close()

	stdoutR, stdoutW, err := os.Pipe()
	if err != nil {
		return nil, errors.Wrap(err, "creating stdout pipe")
	}
	defer stdoutW.Close()

	stderrR, stderrW, err := os.Pipe()
	if err != nil {
		return nil, errors.Wrap(err, "creating stderr pipe")
	}
	defer stderrW.Close()

	var childLogW syscall.Handle

	// GetCurrentProcess doesn't error
	self, _ := syscall.GetCurrentProcess()
	// duplicate handle so it is inheritable by child process
	err = syscall.DuplicateHandle(self, syscall.Handle(logW.Fd()), self, &childLogW, 0, true, syscall.DUPLICATE_SAME_ACCESS)
	if err != nil {
		return nil, errors.Wrap(err, "duplicating log pipe handle")
	}

	cmd := exec.Command(e.runtimePath, "--debug", "--log-handle", strconv.FormatUint(uint64(childLogW), 10), "--log-format", "json", runMode, "--pid-file", filepath.Join(processPath, "pidfile"))
	cmd.Args = append(cmd.Args, runtimeExtraArgs...)

	cmd.Stdin = stdinR
	cmd.Stdout = stdoutW
	cmd.Stderr = stderrW

	if err := e.commandRunner.Start(cmd); err != nil {
		return nil, errors.Wrap(err, "execing runtime plugin")
	}

	go streamLogs(log, logR)

	cleanup := func() error {
		e.processMux.Lock()
		delete(e.processes, processID)
		e.processMux.Unlock()

		if extraCleanup != nil {
			return extraCleanup()
		}

		return nil
	}

	proc := &process{
		id:           processID,
		cleanup:      cleanup,
		logger:       log,
		stdin:        stdinW,
		stdout:       stdoutR,
		stderr:       stderrR,
		stdoutWriter: NewDynamicMultiWriter(),
		stderrWriter: NewDynamicMultiWriter(),
		outputWg:     &sync.WaitGroup{},
		exitMutex:    new(sync.RWMutex),
	}

	e.processMux.Lock()
	e.processes[processID] = proc
	e.processMux.Unlock()

	proc.stream(pio, false)

	proc.exitMutex.Lock()

	go func() {
		// the streamLogs go func will only exit once this handle is closed
		defer syscall.CloseHandle(childLogW)
		defer proc.exitMutex.Unlock()
		if err := e.commandRunner.Wait(cmd); err != nil {
			if exitErr, ok := err.(*exec.ExitError); ok {
				if status, ok := exitErr.Sys().(syscall.WaitStatus); ok {
					proc.exitCode = status.ExitStatus()
				} else {
					proc.exitCode = 1
					proc.exitErr = errors.New("couldn't get WaitStatus")
				}
			} else {
				proc.exitCode = 1
				proc.exitErr = err
			}
		}
	}()

	return proc, nil
}

func (e *WindowsExecRunner) Attach(log lager.Logger, _, processID string, pio garden.ProcessIO) (garden.Process, error) {
	proc, err := e.getProcess(processID)
	if err != nil {
		return nil, err
	}

	proc.stream(pio, true)

	return proc, nil
}

func (e *WindowsExecRunner) getProcess(processID string) (*process, error) {
	e.processMux.Lock()
	defer e.processMux.Unlock()

	proc, ok := e.processes[processID]
	if !ok {
		return nil, garden.ProcessNotFoundError{ProcessID: processID}
	}

	return proc, nil
}

func (p *process) stream(pio garden.ProcessIO, duplicate bool) {
	var procStdin *os.File

	procStdin = p.stdin

	if pio.Stdin != nil {
		if duplicate {
			var dupped syscall.Handle
			self, _ := syscall.GetCurrentProcess()
			err := syscall.DuplicateHandle(self, syscall.Handle(p.stdin.Fd()), self, &dupped, 0, false, syscall.DUPLICATE_SAME_ACCESS)
			if err != nil {
				panic(err)
			}

			procStdin = os.NewFile(uintptr(dupped), fmt.Sprintf("%s.stdin", p.id))
		}

		go func() {
			io.Copy(procStdin, pio.Stdin)
			procStdin.Close()
		}()
	}

	if pio.Stdout != nil {
		count := p.stdoutWriter.Attach(pio.Stdout)
		if count == 1 {
			p.outputWg.Add(1)
			go func() {
				io.Copy(p.stdoutWriter, p.stdout)
				p.stdout.Close()
				p.outputWg.Done()
			}()
		}
	}

	if pio.Stderr != nil {
		count := p.stderrWriter.Attach(pio.Stderr)
		if count == 1 {
			p.outputWg.Add(1)
			go func() {
				io.Copy(p.stderrWriter, p.stderr)
				p.stderr.Close()
				p.outputWg.Done()
			}()
		}
	}
}

func (p *process) ID() string {
	return p.id
}

func (p *process) Wait() (int, error) {
	p.exitMutex.RLock()
	defer p.exitMutex.RUnlock()

	p.outputWg.Wait()

	p.stdin.Close()
	p.stdout.Close()
	p.stderr.Close()

	if p.cleanup != nil {
		if err := p.cleanup(); err != nil {
			p.logger.Error("process-cleanup", err)
		}
	}

	return p.exitCode, p.exitErr
}

func (p *process) SetTTY(ttySpec garden.TTYSpec) error {
	return nil
}

func (p *process) Signal(signal garden.Signal) error {
	return nil
}

func streamLogs(logger lager.Logger, src *os.File) {
	defer src.Close()
	scanner := bufio.NewScanner(src)

	for scanner.Scan() {
		nextLogLine := scanner.Bytes()
		logging.ForwardRuncLogsToLager(logger, "winc", nextLogLine)
	}

	logger.Info("done-streaming-winc-logs")
}
<|endoftext|>


<commit_before>package file

import (
	"os"

	"github.com/DeedleFake/wdte"
)

type File struct {
	*os.File
}

func (f File) Call(frame wdte.Frame, args ...wdte.Func) wdte.Func {
	return f
}

// Open opens a file and returns it as a reader.
func Open(frame wdte.Frame, args ...wdte.Func) wdte.Func {
	frame = frame.WithID("open")

	if len(args) == 0 {
		return wdte.GoFunc(Open)
	}

	path := args[0].Call(frame).(wdte.String)
	file, err := os.Open(string(path))
	if err != nil {
		return wdte.Error{Err: err, Frame: frame}
	}
	return File{File: file}
}

// Create creates a file, truncating it if it exists, and returns it
// as a writer.
func Create(frame wdte.Frame, args ...wdte.Func) wdte.Func {
	frame = frame.WithID("create")

	if len(args) == 0 {
		return wdte.GoFunc(Create)
	}

	path := args[0].Call(frame).(wdte.String)
	file, err := os.Create(string(path))
	if err != nil {
		return wdte.Error{Err: err, Frame: frame}
	}
	return File{File: file}
}

// Append opens a file for appending as a writer. If it doesn't exist
// already, it is created.
func Append(frame wdte.Frame, args ...wdte.Func) wdte.Func {
	frame = frame.WithID("append")

	if len(args) == 0 {
		return wdte.GoFunc(Append)
	}

	path := args[0].Call(frame).(wdte.String)
	file, err := os.OpenFile(string(path), os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0666)
	if err != nil {
		return wdte.Error{Err: err, Frame: frame}
	}
	return File{File: file}
}

func Module() *wdte.Module {
	return &wdte.Module{
		Funcs: map[wdte.ID]wdte.Func{
			"open":   wdte.GoFunc(Open),
			"create": wdte.GoFunc(Create),
			"append": wdte.GoFunc(Append),
		},
	}
}
<commit_msg>std/io/file: Godoc.<commit_after>// Package file provides functions for dealing with files.
package file

import (
	"os"

	"github.com/DeedleFake/wdte"
)

// File wraps an os.File, allowing it to be used as a WDTE function.
type File struct {
	*os.File
}

func (f File) Call(frame wdte.Frame, args ...wdte.Func) wdte.Func { // nolint
	return f
}

// Open opens a file and returns it as a reader.
func Open(frame wdte.Frame, args ...wdte.Func) wdte.Func {
	frame = frame.WithID("open")

	if len(args) == 0 {
		return wdte.GoFunc(Open)
	}

	path := args[0].Call(frame).(wdte.String)
	file, err := os.Open(string(path))
	if err != nil {
		return wdte.Error{Err: err, Frame: frame}
	}
	return File{File: file}
}

// Create creates a file, truncating it if it exists, and returns it
// as a writer.
func Create(frame wdte.Frame, args ...wdte.Func) wdte.Func {
	frame = frame.WithID("create")

	if len(args) == 0 {
		return wdte.GoFunc(Create)
	}

	path := args[0].Call(frame).(wdte.String)
	file, err := os.Create(string(path))
	if err != nil {
		return wdte.Error{Err: err, Frame: frame}
	}
	return File{File: file}
}

// Append opens a file for appending as a writer. If it doesn't exist
// already, it is created.
func Append(frame wdte.Frame, args ...wdte.Func) wdte.Func {
	frame = frame.WithID("append")

	if len(args) == 0 {
		return wdte.GoFunc(Append)
	}

	path := args[0].Call(frame).(wdte.String)
	file, err := os.OpenFile(string(path), os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0666)
	if err != nil {
		return wdte.Error{Err: err, Frame: frame}
	}
	return File{File: file}
}

// Module returns a module containing the functions in this package.
func Module() *wdte.Module {
	return &wdte.Module{
		Funcs: map[wdte.ID]wdte.Func{
			"open":   wdte.GoFunc(Open),
			"create": wdte.GoFunc(Create),
			"append": wdte.GoFunc(Append),
		},
	}
}
<|endoftext|>


<commit_before>package format

import (
	"fmt"
	"time"
)

// Duration is similar to the time.Duration.String method from the standard
// library but is more readable and shows only 3 digits of precision when
// duration is less than 1 minute.
func Duration(duration time.Duration) string {
	if ns := duration.Nanoseconds(); ns < 1000 {
		return fmt.Sprintf("%dns", ns)
	} else if us := float64(duration) / float64(time.Microsecond); us < 1000 {
		return fmt.Sprintf("%.3gµs", us)
	} else if ms := float64(duration) / float64(time.Millisecond); ms < 1000 {
		return fmt.Sprintf("%.3gms", ms)
	} else if s := float64(duration) / float64(time.Second); s < 60 {
		return fmt.Sprintf("%.3gs", s)
	} else {
		duration -= duration % time.Second
		day := time.Hour * 24
		if duration < day {
			return duration.String()
		}
		days := duration / day
		duration %= day
		return fmt.Sprintf("%dd%s", days, duration)
	}
}

// FormatBytes returns a string with the number of bytes specified converted
// into a human-friendly format with a binary multiplier (i.e. GiB).
func FormatBytes(bytes uint64) string {
	if bytes>>30 > 100 {
		return fmt.Sprintf("%d GiB", bytes>>30)
	} else if bytes>>20 > 100 {
		return fmt.Sprintf("%d MiB", bytes>>20)
	} else if bytes>>10 > 100 {
		return fmt.Sprintf("%d KiB", bytes>>10)
	} else {
		return fmt.Sprintf("%d B", bytes)
	}
}
<commit_msg>Add TiB unit to lib/format.FormatBytes().<commit_after>package format

import (
	"fmt"
	"time"
)

// Duration is similar to the time.Duration.String method from the standard
// library but is more readable and shows only 3 digits of precision when
// duration is less than 1 minute.
func Duration(duration time.Duration) string {
	if ns := duration.Nanoseconds(); ns < 1000 {
		return fmt.Sprintf("%dns", ns)
	} else if us := float64(duration) / float64(time.Microsecond); us < 1000 {
		return fmt.Sprintf("%.3gµs", us)
	} else if ms := float64(duration) / float64(time.Millisecond); ms < 1000 {
		return fmt.Sprintf("%.3gms", ms)
	} else if s := float64(duration) / float64(time.Second); s < 60 {
		return fmt.Sprintf("%.3gs", s)
	} else {
		duration -= duration % time.Second
		day := time.Hour * 24
		if duration < day {
			return duration.String()
		}
		days := duration / day
		duration %= day
		return fmt.Sprintf("%dd%s", days, duration)
	}
}

// FormatBytes returns a string with the number of bytes specified converted
// into a human-friendly format with a binary multiplier (i.e. GiB).
func FormatBytes(bytes uint64) string {
	if bytes>>40 > 100 {
		return fmt.Sprintf("%d TiB", bytes>>40)
	} else if bytes>>30 > 100 {
		return fmt.Sprintf("%d GiB", bytes>>30)
	} else if bytes>>20 > 100 {
		return fmt.Sprintf("%d MiB", bytes>>20)
	} else if bytes>>10 > 100 {
		return fmt.Sprintf("%d KiB", bytes>>10)
	} else {
		return fmt.Sprintf("%d B", bytes)
	}
}
<|endoftext|>