<commit_before>package main

import (
	"bytes"
	"runtime"
	"sort"
	"sync"
)

type intSlice []int

func (p intSlice) Len() int           { return len(p) }
func (p intSlice) Less(i, j int) bool { return p[i] < p[j] }
func (p intSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

type bufPool struct {
	name        string
	defaultSize int
	maxSize     int
	buffers     []*bytes.Buffer

	calls   intSlice
	callInd int

	mutex sync.Mutex
}

func newBufPool(name string, n int, defaultSize int) *bufPool {
	pool := bufPool{
		name:        name,
		defaultSize: defaultSize,
		buffers:     make([]*bytes.Buffer, n),
		calls:       make(intSlice, conf.BufferPoolCalibrationThreshold),
	}

	for i := range pool.buffers {
		pool.buffers[i] = new(bytes.Buffer)
	}

	return &pool
}

func (p *bufPool) calibrateAndClean() {
	sort.Sort(p.calls)

	pos := int(float64(len(p.calls)) * 0.95)
	score := p.calls[pos]

	p.callInd = 0
	p.maxSize = 64

	for {
		if p.maxSize > score {
			break
		}
		p.maxSize <<= 1
	}

	p.defaultSize = maxInt(p.defaultSize, p.calls[0])
	p.maxSize = maxInt(p.defaultSize, p.maxSize)

	cleaned := false

	for i, buf := range p.buffers {
		if buf != nil && buf.Cap() > p.maxSize {
			p.buffers[i] = nil
			cleaned = true
		}
	}

	if cleaned {
		runtime.GC()
	}

	if prometheusEnabled {
		setPrometheusBufferDefaultSize(p.name, p.defaultSize)
		setPrometheusBufferMaxSize(p.name, p.maxSize)
	}
}

func (p *bufPool) Get(size int) *bytes.Buffer {
	p.mutex.Lock()
	defer p.mutex.Unlock()

	minSize, maxSize, minInd, maxInd := -1, -1, -1, -1

	for i := 0; i < len(p.buffers); i++ {
		if p.buffers[i] != nil {
			cap := p.buffers[i].Cap()

			if size > 0 && cap >= size && (minSize > cap || minSize == -1) {
				minSize = cap
				minInd = i
			}

			if cap > maxSize {
				maxSize = cap
				maxInd = i
			}
		}
	}

	var buf *bytes.Buffer

	if minInd >= 0 {
		// We found buffer with the desired size
		buf = p.buffers[minInd]
		p.buffers[minInd] = nil
	} else if maxInd >= 0 {
		// We didn't find buffer with the desired size
		buf = p.buffers[maxInd]
		p.buffers[maxInd] = nil
	} else {
		// We didn't find buffers at all
		buf = new(bytes.Buffer)
	}

	buf.Reset()

	growSize := maxInt(size, p.defaultSize)

	if growSize > buf.Cap() {
		buf.Grow(growSize)
	}

	return buf
}

func (p *bufPool) Put(buf *bytes.Buffer) {
	p.mutex.Lock()
	defer p.mutex.Unlock()

	if buf.Len() > 0 {
		p.calls[p.callInd] = buf.Len()
		p.callInd++

		if p.callInd == len(p.calls) {
			p.calibrateAndClean()
		}
	}

	if p.maxSize > 0 && buf.Cap() > p.maxSize {
		return
	}

	for i, b := range p.buffers {
		if b == nil {
			p.buffers[i] = buf

			if prometheusEnabled && buf.Cap() > 0 {
				observePrometheusBufferSize(p.name, buf.Cap())
			}

			return
		}
	}
}
<commit_msg>Improve download buffer calibration<commit_after>package main

import (
	"bytes"
	"runtime"
	"sort"
	"sync"
)

type intSlice []int

func (p intSlice) Len() int           { return len(p) }
func (p intSlice) Less(i, j int) bool { return p[i] < p[j] }
func (p intSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

type bufPool struct {
	name        string
	defaultSize int
	maxSize     int
	buffers     []*bytes.Buffer

	calls   intSlice
	callInd int

	mutex sync.Mutex
}

func newBufPool(name string, n int, defaultSize int) *bufPool {
	pool := bufPool{
		name:        name,
		defaultSize: defaultSize,
		buffers:     make([]*bytes.Buffer, n),
		calls:       make(intSlice, conf.BufferPoolCalibrationThreshold),
	}

	for i := range pool.buffers {
		pool.buffers[i] = new(bytes.Buffer)
	}

	return &pool
}

func (p *bufPool) calibrateAndClean() {
	sort.Sort(p.calls)

	pos := int(float64(len(p.calls)) * 0.95)
	score := p.calls[pos]

	p.callInd = 0
	p.maxSize = p.normalizeSize(score)

	p.defaultSize = maxInt(p.defaultSize, p.calls[0])
	p.maxSize = maxInt(p.defaultSize, p.maxSize)

	cleaned := false

	for i, buf := range p.buffers {
		if buf != nil && buf.Cap() > p.maxSize {
			p.buffers[i] = nil
			cleaned = true
		}
	}

	if cleaned {
		runtime.GC()
	}

	if prometheusEnabled {
		setPrometheusBufferDefaultSize(p.name, p.defaultSize)
		setPrometheusBufferMaxSize(p.name, p.maxSize)
	}
}

func (p *bufPool) Get(size int) *bytes.Buffer {
	p.mutex.Lock()
	defer p.mutex.Unlock()

	size = p.normalizeSize(size)

	minSize, maxSize, minInd, maxInd := -1, -1, -1, -1

	for i := 0; i < len(p.buffers); i++ {
		if p.buffers[i] != nil {
			cap := p.buffers[i].Cap()

			if size > 0 && cap >= size && (minSize > cap || minSize == -1) {
				minSize = cap
				minInd = i
			}

			if cap > maxSize {
				maxSize = cap
				maxInd = i
			}
		}
	}

	var buf *bytes.Buffer

	if minInd >= 0 {
		// We found buffer with the desired size
		buf = p.buffers[minInd]
		p.buffers[minInd] = nil
	} else if maxInd >= 0 {
		// We didn't find buffer with the desired size
		buf = p.buffers[maxInd]
		p.buffers[maxInd] = nil
	} else {
		// We didn't find buffers at all
		buf = new(bytes.Buffer)
	}

	buf.Reset()

	growSize := maxInt(size, p.defaultSize)

	if growSize > buf.Cap() {
		buf.Grow(growSize)
	}

	return buf
}

func (p *bufPool) Put(buf *bytes.Buffer) {
	p.mutex.Lock()
	defer p.mutex.Unlock()

	if buf.Len() > 0 {
		p.calls[p.callInd] = buf.Len()
		p.callInd++

		if p.callInd == len(p.calls) {
			p.calibrateAndClean()
		}
	}

	if p.maxSize > 0 && buf.Cap() > p.maxSize {
		return
	}

	for i, b := range p.buffers {
		if b == nil {
			p.buffers[i] = buf

			if prometheusEnabled && buf.Cap() > 0 {
				observePrometheusBufferSize(p.name, buf.Cap())
			}

			return
		}
	}
}

func (p *bufPool) normalizeSize(n int) int {
	return (n/bytes.MinRead + 2) * bytes.MinRead
}
<|endoftext|>


<commit_before>// cat -- concatenate and print files
// Part of goutils (https://github.com/trevorparker/goutils)
//
// Copyright (c) 2014 Trevor Parker <trevor@trevorparker.com>
// All rights reserved
//
// Distributed under the terms of the Modified BSD License (see LICENSE)

package main

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"unicode/utf8"
)

type arg struct {
	file                  []string
	nonblank_line_numbers bool
	line_numbers          bool
	show_line_endings     bool
	squeeze_blank         bool
	show_tabs             bool
}

const usage_message string = "usage: cat [OPTION ...] [FILE ...]"
const help_message string = `Concatenate and print FILE or STDIN to STDOUT.

  -b, --number-nonblank     number only non-blank lines
  -E, --show-ends           print $ at the end of each line
  -n, --number              number output lines, starting with 1
  -s, --squeeze-blank       print no more than one consecutive blank line
  -T, --show-tabs           print tab character as ^I
  -h, --help                print this help message and exit
`

const tab rune = 9
const newline rune = 10

func usage(error string) {
	fmt.Fprintf(os.Stderr, "cat: %s\n%s\n", error, usage_message)
	os.Exit(1)
}

func help() {
	fmt.Printf("%s\n%s", usage_message, help_message)
	os.Exit(0)
}

func cat(file io.Reader, args arg) {
	if file == nil {
		file = os.Stdin
	}
	r := bufio.NewReader(file)

	line_number := 0
	newline_next := true

	prev_rune := []rune{0, 0}

	for {
		buf := make([]byte, 16)
		n, err := r.Read(buf)

		if err == io.EOF {
			return
		} else if err != nil {
			panic(err)
		}

		for i := 0; i < n; i++ {
			this_rune, _ := utf8.DecodeRune(buf[i : i+1])

			// Identify and squeeze blank lines
			if args.squeeze_blank && prev_rune[0] == newline {
				if this_rune == newline && prev_rune[0] == prev_rune[1] {
					continue
				}
			}

			prev_rune = []rune{this_rune, prev_rune[0]}

			if args.line_numbers && newline_next == true {
				if this_rune != newline || !args.nonblank_line_numbers {
					line_number++
					fmt.Printf("%6d\t", line_number)
					newline_next = false
				}
			}

			if args.show_tabs && this_rune == tab {
				os.Stdout.Write([]byte("^I"))
				continue
			} else if this_rune == newline {
				newline_next = true
				if args.show_line_endings {
					os.Stdout.Write([]byte("$"))
				}
			}

			os.Stdout.Write(buf[i : i+1])
		}
	}
}

func main() {
	args := arg{}
	reached_files := false

	for i := 1; i < len(os.Args); i++ {
		if reached_files == false {
			if os.Args[i] == "-h" || os.Args[i] == "--help" {
				help()
			}
			if os.Args[i] == "-b" || os.Args[i] == "--number-nonblank" {
				args.nonblank_line_numbers = true
				args.line_numbers = true
				continue
			}
			if os.Args[i] == "-E" || os.Args[i] == "--show-ends" {
				args.show_line_endings = true
				continue
			}
			if os.Args[i] == "-n" || os.Args[i] == "--number" {
				args.line_numbers = true
				continue
			}
			if os.Args[i] == "-s" || os.Args[i] == "--squeeze-blank" {
				args.squeeze_blank = true
				continue
			}
			if os.Args[i] == "-T" || os.Args[i] == "--show-tabs" {
				args.show_tabs = true
				continue
			}
		}
		reached_files = true
		arg_v := os.Args[i]
		args.file = append(args.file, arg_v)
	}

	if len(args.file) == 0 {
		cat(nil, args)
	} else {
		for i := range args.file {
			file, err := os.Open(args.file[i])
			if err != nil {
				panic(err)
			}
			cat(file, args)
		}
	}
}
<commit_msg>Use buffered STDOUT writer for better performance<commit_after>// cat -- concatenate and print files
// Part of goutils (https://github.com/trevorparker/goutils)
//
// Copyright (c) 2014 Trevor Parker <trevor@trevorparker.com>
// All rights reserved
//
// Distributed under the terms of the Modified BSD License (see LICENSE)

package main

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"unicode/utf8"
)

type arg struct {
	file                  []string
	nonblank_line_numbers bool
	line_numbers          bool
	show_line_endings     bool
	squeeze_blank         bool
	show_tabs             bool
}

const usage_message string = "usage: cat [OPTION ...] [FILE ...]"
const help_message string = `Concatenate and print FILE or STDIN to STDOUT.

  -b, --number-nonblank     number only non-blank lines
  -E, --show-ends           print $ at the end of each line
  -n, --number              number output lines, starting with 1
  -s, --squeeze-blank       print no more than one consecutive blank line
  -T, --show-tabs           print tab character as ^I
  -h, --help                print this help message and exit
`

const tab rune = 9
const newline rune = 10

func usage(error string) {
	fmt.Fprintf(os.Stderr, "cat: %s\n%s\n", error, usage_message)
	os.Exit(1)
}

func help() {
	fmt.Printf("%s\n%s", usage_message, help_message)
	os.Exit(0)
}

func cat(file io.Reader, args arg) {
	if file == nil {
		file = os.Stdin
	}
	r := bufio.NewReader(file)
	w := bufio.NewWriterSize(os.Stdout, 512)

	line_number := 0
	newline_next := true

	prev_rune := []rune{0, 0}

	buf := make([]byte, 512)

	for {
		n, err := r.Read(buf)

		if err == io.EOF {
			return
		} else if err != nil {
			panic(err)
		}

		for i := 0; i < n; i++ {
			this_rune, _ := utf8.DecodeRune(buf[i : i+1])

			// Identify and squeeze blank lines
			if args.squeeze_blank && prev_rune[0] == newline {
				if this_rune == newline && prev_rune[0] == prev_rune[1] {
					continue
				}
			}

			prev_rune = []rune{this_rune, prev_rune[0]}

			if args.line_numbers && newline_next == true {
				if this_rune != newline || !args.nonblank_line_numbers {
					line_number++
					fmt.Fprintf(w, "%6d\t", line_number)
					newline_next = false
				}
			}

			if args.show_tabs && this_rune == tab {
				w.Write([]byte("^I"))
				continue
			} else if this_rune == newline {
				newline_next = true
				if args.show_line_endings {
					w.Write([]byte("$"))
				}
			}

			w.Write([]byte(buf[i : i+1]))
		}

		w.Flush()
	}
}

func main() {
	args := arg{}
	reached_files := false

	for i := 1; i < len(os.Args); i++ {
		if reached_files == false {
			if os.Args[i] == "-h" || os.Args[i] == "--help" {
				help()
			}
			if os.Args[i] == "-b" || os.Args[i] == "--number-nonblank" {
				args.nonblank_line_numbers = true
				args.line_numbers = true
				continue
			}
			if os.Args[i] == "-E" || os.Args[i] == "--show-ends" {
				args.show_line_endings = true
				continue
			}
			if os.Args[i] == "-n" || os.Args[i] == "--number" {
				args.line_numbers = true
				continue
			}
			if os.Args[i] == "-s" || os.Args[i] == "--squeeze-blank" {
				args.squeeze_blank = true
				continue
			}
			if os.Args[i] == "-T" || os.Args[i] == "--show-tabs" {
				args.show_tabs = true
				continue
			}
		}
		reached_files = true
		arg_v := os.Args[i]
		args.file = append(args.file, arg_v)
	}

	if len(args.file) == 0 {
		cat(nil, args)
	} else {
		for i := range args.file {
			file, err := os.Open(args.file[i])
			if err != nil {
				panic(err)
			}
			cat(file, args)
		}
	}
}
<|endoftext|>


<commit_before>package writeaheadlog

import (
	"errors"
	"io"
	"io/ioutil"
	"os"
	"sync"

	"github.com/NebulousLabs/fastrand"
)

// These interfaces define the wal's dependencies. Using the smallest
// interface possible makes it easier to mock these dependencies in testing.
type (
	dependencies interface {
		disrupt(string) bool
		readFile(string) ([]byte, error)
		openFile(string, int, os.FileMode) (file, error)
		create(string) (file, error)
	}

	// file implements all of the methods that can be called on an os.File.
	file interface {
		io.ReadWriteCloser
		Name() string
		ReadAt([]byte, int64) (int, error)
		Sync() error
		WriteAt([]byte, int64) (int, error)
		Stat() (os.FileInfo, error)
	}
)

// dependencyCommitFail corrupts the first page of a transaction when it
// is committed
type dependencyCommitFail struct {
	prodDependencies
}

func (dependencyCommitFail) disrupt(s string) bool {
	if s == "CommitFail" {
		return true
	}
	return false
}

// dependencyReleaseFail corrupts the first page of a transaction when it
// is released
type dependencyReleaseFail struct {
	prodDependencies
}

func (dependencyReleaseFail) disrupt(s string) bool {
	if s == "ReleaseFail" {
		return true
	}
	return false
}

// prodDependencies is a passthrough to the standard library calls
type prodDependencies struct{}

func (prodDependencies) disrupt(string) bool { return false }
func (prodDependencies) readFile(path string) ([]byte, error) {
	return ioutil.ReadFile(path)
}
func (prodDependencies) openFile(path string, flag int, perm os.FileMode) (file, error) {
	return os.OpenFile(path, flag, perm)
}
func (prodDependencies) create(path string) (file, error) {
	return os.Create(path)
}

// faultyDiskDependency implements dependencies that simulate a faulty disk.
type faultyDiskDependency struct {
	// failDenominator determines how likely it is that a write will fail, defined
	// as 1/failDenominator. Each write call increments failDenominator, and it
	// starts at 2. This means that the more calls to WriteAt, the less likely
	// the write is to fail. All calls will start automatically failing after
	// 5000 writes.
	failDenominator int
	writeLimit      int
	failed          bool
	mu              *sync.Mutex
}

// newFaultyDiskDependency creates a dependency that can be used to simulate a
// failing disk. writeLimit is the maximum number of writes the disk will
// endure before failing
func newFaultyDiskDependency(writeLimit int) faultyDiskDependency {
	return faultyDiskDependency{
		failDenominator: 3,
		failed:          false,
		writeLimit:      writeLimit,
		mu:              new(sync.Mutex),
	}
}

func (faultyDiskDependency) disrupt(s string) bool {
	return s == "FaultyDisk"
}
func (faultyDiskDependency) readFile(path string) ([]byte, error) {
	return ioutil.ReadFile(path)
}
func (d faultyDiskDependency) openFile(path string, flag int, perm os.FileMode) (file, error) {
	f, err := os.OpenFile(path, flag, perm)
	if err != nil {
		return nil, err
	}
	return d.newFaultyFile(f), nil
}
func (d faultyDiskDependency) create(path string) (file, error) {
	f, err := os.Create(path)
	if err != nil {
		return nil, err
	}
	return d.newFaultyFile(f), nil
}

// faultyFile implements a file that simulates a faulty disk.
type faultyFile struct {
	d    *faultyDiskDependency
	file *os.File
}

func (f *faultyFile) Read(p []byte) (int, error) {
	return f.file.Read(p)
}
func (f *faultyFile) Write(p []byte) (int, error) {
	f.d.mu.Lock()
	defer f.d.mu.Unlock()

	fail := fastrand.Intn(f.d.failDenominator) == 0
	f.d.failDenominator++
	if fail || f.d.failDenominator >= f.d.writeLimit {
		f.d.failed = true
		return len(p), nil
	}
	return f.file.Write(p)
}
func (f *faultyFile) Close() error { return f.file.Close() }
func (f *faultyFile) Name() string {
	return f.file.Name()
}
func (f *faultyFile) ReadAt(p []byte, off int64) (int, error) {
	return f.file.ReadAt(p, off)
}
func (f *faultyFile) WriteAt(p []byte, off int64) (int, error) {
	f.d.mu.Lock()
	defer f.d.mu.Unlock()

	fail := fastrand.Intn(f.d.failDenominator) == 0
	f.d.failDenominator++
	if fail || f.d.failDenominator >= f.d.writeLimit {
		f.d.failed = true
		return len(p), nil
	}
	return f.file.WriteAt(p, off)
}
func (f *faultyFile) Stat() (os.FileInfo, error) {
	return f.file.Stat()
}
func (f *faultyFile) Sync() error {
	f.d.mu.Lock()
	defer f.d.mu.Unlock()

	if f.d.failed {
		return errors.New("could not write to disk (faultyDisk)")
	}
	return f.file.Sync()
}

// newFaultyFile creates a new faulty file around the provided file handle.
func (d *faultyDiskDependency) newFaultyFile(f *os.File) *faultyFile {
	return &faultyFile{d: d, file: f}
}
<commit_msg>fix docstring<commit_after>package writeaheadlog

import (
	"errors"
	"io"
	"io/ioutil"
	"os"
	"sync"

	"github.com/NebulousLabs/fastrand"
)

// These interfaces define the wal's dependencies. Using the smallest
// interface possible makes it easier to mock these dependencies in testing.
type (
	dependencies interface {
		disrupt(string) bool
		readFile(string) ([]byte, error)
		openFile(string, int, os.FileMode) (file, error)
		create(string) (file, error)
	}

	// file implements all of the methods that can be called on an os.File.
	file interface {
		io.ReadWriteCloser
		Name() string
		ReadAt([]byte, int64) (int, error)
		Sync() error
		WriteAt([]byte, int64) (int, error)
		Stat() (os.FileInfo, error)
	}
)

// dependencyCommitFail corrupts the first page of a transaction when it
// is committed
type dependencyCommitFail struct {
	prodDependencies
}

func (dependencyCommitFail) disrupt(s string) bool {
	if s == "CommitFail" {
		return true
	}
	return false
}

// dependencyReleaseFail corrupts the first page of a transaction when it
// is released
type dependencyReleaseFail struct {
	prodDependencies
}

func (dependencyReleaseFail) disrupt(s string) bool {
	if s == "ReleaseFail" {
		return true
	}
	return false
}

// prodDependencies is a passthrough to the standard library calls
type prodDependencies struct{}

func (prodDependencies) disrupt(string) bool { return false }
func (prodDependencies) readFile(path string) ([]byte, error) {
	return ioutil.ReadFile(path)
}
func (prodDependencies) openFile(path string, flag int, perm os.FileMode) (file, error) {
	return os.OpenFile(path, flag, perm)
}
func (prodDependencies) create(path string) (file, error) {
	return os.Create(path)
}

// faultyDiskDependency implements dependencies that simulate a faulty disk.
type faultyDiskDependency struct {
	// failDenominator determines how likely it is that a write will fail,
	// defined as 1/failDenominator. Each write call increments
	// failDenominator, and it starts at 2. This means that the more calls to
	// WriteAt, the less likely the write is to fail. All calls will start
	// automatically failing after writeLimit writes.
	failDenominator int
	writeLimit      int
	failed          bool
	mu              *sync.Mutex
}

// newFaultyDiskDependency creates a dependency that can be used to simulate a
// failing disk. writeLimit is the maximum number of writes the disk will
// endure before failing
func newFaultyDiskDependency(writeLimit int) faultyDiskDependency {
	return faultyDiskDependency{
		failDenominator: 3,
		failed:          false,
		writeLimit:      writeLimit,
		mu:              new(sync.Mutex),
	}
}

func (faultyDiskDependency) disrupt(s string) bool {
	return s == "FaultyDisk"
}
func (faultyDiskDependency) readFile(path string) ([]byte, error) {
	return ioutil.ReadFile(path)
}
func (d faultyDiskDependency) openFile(path string, flag int, perm os.FileMode) (file, error) {
	f, err := os.OpenFile(path, flag, perm)
	if err != nil {
		return nil, err
	}
	return d.newFaultyFile(f), nil
}
func (d faultyDiskDependency) create(path string) (file, error) {
	f, err := os.Create(path)
	if err != nil {
		return nil, err
	}
	return d.newFaultyFile(f), nil
}

// faultyFile implements a file that simulates a faulty disk.
type faultyFile struct {
	d    *faultyDiskDependency
	file *os.File
}

func (f *faultyFile) Read(p []byte) (int, error) {
	return f.file.Read(p)
}
func (f *faultyFile) Write(p []byte) (int, error) {
	f.d.mu.Lock()
	defer f.d.mu.Unlock()

	fail := fastrand.Intn(f.d.failDenominator) == 0
	f.d.failDenominator++
	if fail || f.d.failDenominator >= f.d.writeLimit {
		f.d.failed = true
		return len(p), nil
	}
	return f.file.Write(p)
}
func (f *faultyFile) Close() error { return f.file.Close() }
func (f *faultyFile) Name() string {
	return f.file.Name()
}
func (f *faultyFile) ReadAt(p []byte, off int64) (int, error) {
	return f.file.ReadAt(p, off)
}
func (f *faultyFile) WriteAt(p []byte, off int64) (int, error) {
	f.d.mu.Lock()
	defer f.d.mu.Unlock()

	fail := fastrand.Intn(f.d.failDenominator) == 0
	f.d.failDenominator++
	if fail || f.d.failDenominator >= f.d.writeLimit {
		f.d.failed = true
		return len(p), nil
	}
	return f.file.WriteAt(p, off)
}
func (f *faultyFile) Stat() (os.FileInfo, error) {
	return f.file.Stat()
}
func (f *faultyFile) Sync() error {
	f.d.mu.Lock()
	defer f.d.mu.Unlock()

	if f.d.failed {
		return errors.New("could not write to disk (faultyDisk)")
	}
	return f.file.Sync()
}

// newFaultyFile creates a new faulty file around the provided file handle.
func (d *faultyDiskDependency) newFaultyFile(f *os.File) *faultyFile {
	return &faultyFile{d: d, file: f}
}
<|endoftext|>


<commit_before>package migrations

import (
	"github.com/rubenv/sql-migrate"
)

func addApplicationAndVersionColumnsToEnvironmentsTable() *migrate.Migration {
	return &migrate.Migration{
		Id: "12",
		Up: []string{`
      ALTER TABLE environments
        ADD COLUMN application_uuid varchar(36) NOT NULL AFTER application_id,
        ADD COLUMN version_id int(10) unsigned DEFAULT NULL AFTER application_uuid,
        ADD COLUMN version_uuid varchar(36) NOT NULL AFTER version_id
      ;
    `},
		Down: []string{`
      ALTER TABLE environments
        DROP COLUMN application_uuid,
        DROP COLUMN version_id,
		DROP COLUMN version_uuid
      ;
    `},
	}
}
<commit_msg>set default to null instead of not null<commit_after>package migrations

import (
	"github.com/rubenv/sql-migrate"
)

func addApplicationAndVersionColumnsToEnvironmentsTable() *migrate.Migration {
	return &migrate.Migration{
		Id: "12",
		Up: []string{`
      ALTER TABLE environments
        ADD COLUMN application_uuid varchar(36) DEFAULT NULL AFTER application_id,
        ADD COLUMN version_id int(10) unsigned DEFAULT NULL AFTER application_uuid,
        ADD COLUMN version_uuid varchar(36) NOT NULL AFTER version_id
      ;
    `},
		Down: []string{`
      ALTER TABLE environments
        DROP COLUMN application_uuid,
        DROP COLUMN version_id,
		DROP COLUMN version_uuid
      ;
    `},
	}
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"exec"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"patch"
	"path/filepath"
	"sort"
	"strings"
)

var checkSync = flag.Bool("checksync", true, "check whether repository is out of sync")

func usage() {
	fmt.Fprintf(os.Stderr, "usage: hgpatch [options] [patchfile]\n")
	flag.PrintDefaults()
	os.Exit(2)
}

func main() {
	flag.Usage = usage
	flag.Parse()

	args := flag.Args()
	var data []byte
	var err os.Error
	switch len(args) {
	case 0:
		data, err = ioutil.ReadAll(os.Stdin)
	case 1:
		data, err = ioutil.ReadFile(args[0])
	default:
		usage()
	}
	chk(err)

	pset, err := patch.Parse(data)
	chk(err)

	// Change to hg root directory, because
	// patch paths are relative to root.
	root, err := hgRoot()
	chk(err)
	chk(os.Chdir(root))

	// Make sure there are no pending changes on the server.
	if *checkSync && hgIncoming() {
		fmt.Fprintf(os.Stderr, "incoming changes waiting; run hg sync first\n")
		os.Exit(2)
	}

	// Make sure we won't be editing files with local pending changes.
	dirtylist, err := hgModified()
	chk(err)
	dirty := make(map[string]bool)
	for _, f := range dirtylist {
		dirty[f] = true
	}
	conflict := make(map[string]bool)
	for _, f := range pset.File {
		if f.Verb == patch.Delete || f.Verb == patch.Rename {
			if dirty[f.Src] {
				conflict[f.Src] = true
			}
		}
		if f.Verb != patch.Delete {
			if dirty[f.Dst] {
				conflict[f.Dst] = true
			}
		}
	}
	if len(conflict) > 0 {
		fmt.Fprintf(os.Stderr, "cannot apply patch to locally modified files:\n")
		for name := range conflict {
			fmt.Fprintf(os.Stderr, "\t%s\n", name)
		}
		os.Exit(2)
	}

	// Apply changes in memory.
	op, err := pset.Apply(ioutil.ReadFile)
	chk(err)

	// Write changes to disk copy: order of commands matters.
	// Accumulate undo log as we go, in case there is an error.
	// Also accumulate list of modified files to print at end.
	changed := make(map[string]int)

	// Copy, Rename create the destination file, so they
	// must happen before we write the data out.
	// A single patch may have a Copy and a Rename
	// with the same source, so we have to run all the
	// Copy in one pass, then all the Rename.
	for i := range op {
		o := &op[i]
		if o.Verb == patch.Copy {
			makeParent(o.Dst)
			chk(hgCopy(o.Dst, o.Src))
			undoRevert(o.Dst)
			changed[o.Dst] = 1
		}
	}
	for i := range op {
		o := &op[i]
		if o.Verb == patch.Rename {
			makeParent(o.Dst)
			chk(hgRename(o.Dst, o.Src))
			undoRevert(o.Dst)
			undoRevert(o.Src)
			changed[o.Src] = 1
			changed[o.Dst] = 1
		}
	}

	// Run Delete before writing to files in case one of the
	// deleted paths is becoming a directory.
	for i := range op {
		o := &op[i]
		if o.Verb == patch.Delete {
			chk(hgRemove(o.Src))
			undoRevert(o.Src)
			changed[o.Src] = 1
		}
	}

	// Write files.
	for i := range op {
		o := &op[i]
		if o.Verb == patch.Delete {
			continue
		}
		if o.Verb == patch.Add {
			makeParent(o.Dst)
			changed[o.Dst] = 1
		}
		if o.Data != nil {
			chk(ioutil.WriteFile(o.Dst, o.Data, 0644))
			if o.Verb == patch.Add {
				undoRm(o.Dst)
			} else {
				undoRevert(o.Dst)
			}
			changed[o.Dst] = 1
		}
		if o.Mode != 0 {
			chk(os.Chmod(o.Dst, uint32(o.Mode&0755)))
			undoRevert(o.Dst)
			changed[o.Dst] = 1
		}
	}

	// hg add looks at the destination file, so it must happen
	// after we write the data out.
	for i := range op {
		o := &op[i]
		if o.Verb == patch.Add {
			chk(hgAdd(o.Dst))
			undoRevert(o.Dst)
			changed[o.Dst] = 1
		}
	}

	// Finished editing files.  Write the list of changed files to stdout.
	list := make([]string, len(changed))
	i := 0
	for f := range changed {
		list[i] = f
		i++
	}
	sort.Strings(list)
	for _, f := range list {
		fmt.Printf("%s\n", f)
	}
}

// make parent directory for name, if necessary
func makeParent(name string) {
	parent, _ := filepath.Split(name)
	chk(mkdirAll(parent, 0755))
}

// Copy of os.MkdirAll but adds to undo log after
// creating a directory.
func mkdirAll(path string, perm uint32) os.Error {
	dir, err := os.Lstat(path)
	if err == nil {
		if dir.IsDirectory() {
			return nil
		}
		return &os.PathError{"mkdir", path, os.ENOTDIR}
	}

	i := len(path)
	for i > 0 && path[i-1] == '/' { // Skip trailing slashes.
		i--
	}

	j := i
	for j > 0 && path[j-1] != '/' { // Scan backward over element.
		j--
	}

	if j > 0 {
		err = mkdirAll(path[0:j-1], perm)
		if err != nil {
			return err
		}
	}

	err = os.Mkdir(path, perm)
	if err != nil {
		// Handle arguments like "foo/." by
		// double-checking that directory doesn't exist.
		dir, err1 := os.Lstat(path)
		if err1 == nil && dir.IsDirectory() {
			return nil
		}
		return err
	}
	undoRm(path)
	return nil
}

// If err != nil, process the undo log and exit.
func chk(err os.Error) {
	if err != nil {
		fmt.Fprintf(os.Stderr, "%s\n", err)
		runUndo()
		os.Exit(2)
	}
}

// Undo log
type undo func() os.Error

var undoLog []undo

func undoRevert(name string) {
	undoLog = append(undoLog, undo(func() os.Error { return hgRevert(name) }))
}

func undoRm(name string) { undoLog = append(undoLog, undo(func() os.Error { return os.Remove(name) })) }

func runUndo() {
	for i := len(undoLog) - 1; i >= 0; i-- {
		if err := undoLog[i](); err != nil {
			fmt.Fprintf(os.Stderr, "%s\n", err)
		}
	}
}

// hgRoot returns the root directory of the repository.
func hgRoot() (string, os.Error) {
	out, err := run([]string{"hg", "root"}, nil)
	if err != nil {
		return "", err
	}
	return strings.TrimSpace(out), nil
}

// hgIncoming returns true if hg sync will pull in changes.
func hgIncoming() bool {
	// hg -q incoming exits 0 when there is nothing incoming, 1 otherwise.
	_, err := run([]string{"hg", "-q", "incoming"}, nil)
	return err == nil
}

// hgModified returns a list of the modified files in the
// repository.
func hgModified() ([]string, os.Error) {
	out, err := run([]string{"hg", "status", "-n"}, nil)
	if err != nil {
		return nil, err
	}
	return strings.Split(strings.TrimSpace(out), "\n"), nil
}

// hgAdd adds name to the repository.
func hgAdd(name string) os.Error {
	_, err := run([]string{"hg", "add", name}, nil)
	return err
}

// hgRemove removes name from the repository.
func hgRemove(name string) os.Error {
	_, err := run([]string{"hg", "rm", name}, nil)
	return err
}

// hgRevert reverts name.
func hgRevert(name string) os.Error {
	_, err := run([]string{"hg", "revert", name}, nil)
	return err
}

// hgCopy copies src to dst in the repository.
// Note that the argument order matches io.Copy, not "hg cp".
func hgCopy(dst, src string) os.Error {
	_, err := run([]string{"hg", "cp", src, dst}, nil)
	return err
}

// hgRename renames src to dst in the repository.
// Note that the argument order matches io.Copy, not "hg mv".
func hgRename(dst, src string) os.Error {
	_, err := run([]string{"hg", "mv", src, dst}, nil)
	return err
}

func dup(a []string) []string {
	b := make([]string, len(a))
	copy(b, a)
	return b
}

var lookPathCache = make(map[string]string)

// run runs the command argv, resolving argv[0] if necessary by searching $PATH.
// It provides input on standard input to the command.
func run(argv []string, input []byte) (out string, err os.Error) {
	if len(argv) < 1 {
		return "", &runError{dup(argv), os.EINVAL}
	}

	prog, ok := lookPathCache[argv[0]]
	if !ok {
		prog, err = exec.LookPath(argv[0])
		if err != nil {
			return "", &runError{dup(argv), err}
		}
		lookPathCache[argv[0]] = prog
	}

	cmd := exec.Command(prog, argv[1:]...)
	if len(input) > 0 {
		cmd.Stdin = bytes.NewBuffer(input)
	}
	bs, err := cmd.CombinedOutput()
	if err != nil {
		return "", &runError{dup(argv), err}
	}
	return string(bs), nil
}

// A runError represents an error that occurred while running a command.
type runError struct {
	cmd []string
	err os.Error
}

func (e *runError) String() string { return strings.Join(e.cmd, " ") + ": " + e.err.String() }
<commit_msg>hgpatch: do not use hg exit status<commit_after>// Copyright 2009 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"exec"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"patch"
	"path/filepath"
	"sort"
	"strings"
)

var checkSync = flag.Bool("checksync", true, "check whether repository is out of sync")

func usage() {
	fmt.Fprintf(os.Stderr, "usage: hgpatch [options] [patchfile]\n")
	flag.PrintDefaults()
	os.Exit(2)
}

func main() {
	flag.Usage = usage
	flag.Parse()

	args := flag.Args()
	var data []byte
	var err os.Error
	switch len(args) {
	case 0:
		data, err = ioutil.ReadAll(os.Stdin)
	case 1:
		data, err = ioutil.ReadFile(args[0])
	default:
		usage()
	}
	chk(err)

	pset, err := patch.Parse(data)
	chk(err)

	// Change to hg root directory, because
	// patch paths are relative to root.
	root, err := hgRoot()
	chk(err)
	chk(os.Chdir(root))

	// Make sure there are no pending changes on the server.
	if *checkSync && hgIncoming() {
		fmt.Fprintf(os.Stderr, "incoming changes waiting; run hg sync first\n")
		os.Exit(2)
	}

	// Make sure we won't be editing files with local pending changes.
	dirtylist, err := hgModified()
	chk(err)
	dirty := make(map[string]bool)
	for _, f := range dirtylist {
		dirty[f] = true
	}
	conflict := make(map[string]bool)
	for _, f := range pset.File {
		if f.Verb == patch.Delete || f.Verb == patch.Rename {
			if dirty[f.Src] {
				conflict[f.Src] = true
			}
		}
		if f.Verb != patch.Delete {
			if dirty[f.Dst] {
				conflict[f.Dst] = true
			}
		}
	}
	if len(conflict) > 0 {
		fmt.Fprintf(os.Stderr, "cannot apply patch to locally modified files:\n")
		for name := range conflict {
			fmt.Fprintf(os.Stderr, "\t%s\n", name)
		}
		os.Exit(2)
	}

	// Apply changes in memory.
	op, err := pset.Apply(ioutil.ReadFile)
	chk(err)

	// Write changes to disk copy: order of commands matters.
	// Accumulate undo log as we go, in case there is an error.
	// Also accumulate list of modified files to print at end.
	changed := make(map[string]int)

	// Copy, Rename create the destination file, so they
	// must happen before we write the data out.
	// A single patch may have a Copy and a Rename
	// with the same source, so we have to run all the
	// Copy in one pass, then all the Rename.
	for i := range op {
		o := &op[i]
		if o.Verb == patch.Copy {
			makeParent(o.Dst)
			chk(hgCopy(o.Dst, o.Src))
			undoRevert(o.Dst)
			changed[o.Dst] = 1
		}
	}
	for i := range op {
		o := &op[i]
		if o.Verb == patch.Rename {
			makeParent(o.Dst)
			chk(hgRename(o.Dst, o.Src))
			undoRevert(o.Dst)
			undoRevert(o.Src)
			changed[o.Src] = 1
			changed[o.Dst] = 1
		}
	}

	// Run Delete before writing to files in case one of the
	// deleted paths is becoming a directory.
	for i := range op {
		o := &op[i]
		if o.Verb == patch.Delete {
			chk(hgRemove(o.Src))
			undoRevert(o.Src)
			changed[o.Src] = 1
		}
	}

	// Write files.
	for i := range op {
		o := &op[i]
		if o.Verb == patch.Delete {
			continue
		}
		if o.Verb == patch.Add {
			makeParent(o.Dst)
			changed[o.Dst] = 1
		}
		if o.Data != nil {
			chk(ioutil.WriteFile(o.Dst, o.Data, 0644))
			if o.Verb == patch.Add {
				undoRm(o.Dst)
			} else {
				undoRevert(o.Dst)
			}
			changed[o.Dst] = 1
		}
		if o.Mode != 0 {
			chk(os.Chmod(o.Dst, uint32(o.Mode&0755)))
			undoRevert(o.Dst)
			changed[o.Dst] = 1
		}
	}

	// hg add looks at the destination file, so it must happen
	// after we write the data out.
	for i := range op {
		o := &op[i]
		if o.Verb == patch.Add {
			chk(hgAdd(o.Dst))
			undoRevert(o.Dst)
			changed[o.Dst] = 1
		}
	}

	// Finished editing files.  Write the list of changed files to stdout.
	list := make([]string, len(changed))
	i := 0
	for f := range changed {
		list[i] = f
		i++
	}
	sort.Strings(list)
	for _, f := range list {
		fmt.Printf("%s\n", f)
	}
}

// make parent directory for name, if necessary
func makeParent(name string) {
	parent, _ := filepath.Split(name)
	chk(mkdirAll(parent, 0755))
}

// Copy of os.MkdirAll but adds to undo log after
// creating a directory.
func mkdirAll(path string, perm uint32) os.Error {
	dir, err := os.Lstat(path)
	if err == nil {
		if dir.IsDirectory() {
			return nil
		}
		return &os.PathError{"mkdir", path, os.ENOTDIR}
	}

	i := len(path)
	for i > 0 && path[i-1] == '/' { // Skip trailing slashes.
		i--
	}

	j := i
	for j > 0 && path[j-1] != '/' { // Scan backward over element.
		j--
	}

	if j > 0 {
		err = mkdirAll(path[0:j-1], perm)
		if err != nil {
			return err
		}
	}

	err = os.Mkdir(path, perm)
	if err != nil {
		// Handle arguments like "foo/." by
		// double-checking that directory doesn't exist.
		dir, err1 := os.Lstat(path)
		if err1 == nil && dir.IsDirectory() {
			return nil
		}
		return err
	}
	undoRm(path)
	return nil
}

// If err != nil, process the undo log and exit.
func chk(err os.Error) {
	if err != nil {
		fmt.Fprintf(os.Stderr, "%s\n", err)
		runUndo()
		os.Exit(2)
	}
}

// Undo log
type undo func() os.Error

var undoLog []undo

func undoRevert(name string) {
	undoLog = append(undoLog, undo(func() os.Error { return hgRevert(name) }))
}

func undoRm(name string) { undoLog = append(undoLog, undo(func() os.Error { return os.Remove(name) })) }

func runUndo() {
	for i := len(undoLog) - 1; i >= 0; i-- {
		if err := undoLog[i](); err != nil {
			fmt.Fprintf(os.Stderr, "%s\n", err)
		}
	}
}

// hgRoot returns the root directory of the repository.
func hgRoot() (string, os.Error) {
	out, err := run([]string{"hg", "root"}, nil)
	if err != nil {
		return "", err
	}
	return strings.TrimSpace(out), nil
}

// hgIncoming returns true if hg sync will pull in changes.
func hgIncoming() bool {
	// Cannot trust hg's exit code on Windows,
	// so look at whether hg prints any output.
	out, _ := run([]string{"hg", "-q", "incoming"}, nil)
	return len(out) > 0
}

// hgModified returns a list of the modified files in the
// repository.
func hgModified() ([]string, os.Error) {
	out, err := run([]string{"hg", "status", "-n"}, nil)
	if err != nil {
		return nil, err
	}
	return strings.Split(strings.TrimSpace(out), "\n"), nil
}

// hgAdd adds name to the repository.
func hgAdd(name string) os.Error {
	_, err := run([]string{"hg", "add", name}, nil)
	return err
}

// hgRemove removes name from the repository.
func hgRemove(name string) os.Error {
	_, err := run([]string{"hg", "rm", name}, nil)
	return err
}

// hgRevert reverts name.
func hgRevert(name string) os.Error {
	_, err := run([]string{"hg", "revert", name}, nil)
	return err
}

// hgCopy copies src to dst in the repository.
// Note that the argument order matches io.Copy, not "hg cp".
func hgCopy(dst, src string) os.Error {
	_, err := run([]string{"hg", "cp", src, dst}, nil)
	return err
}

// hgRename renames src to dst in the repository.
// Note that the argument order matches io.Copy, not "hg mv".
func hgRename(dst, src string) os.Error {
	_, err := run([]string{"hg", "mv", src, dst}, nil)
	return err
}

func dup(a []string) []string {
	b := make([]string, len(a))
	copy(b, a)
	return b
}

var lookPathCache = make(map[string]string)

// run runs the command argv, resolving argv[0] if necessary by searching $PATH.
// It provides input on standard input to the command.
func run(argv []string, input []byte) (out string, err os.Error) {
	if len(argv) < 1 {
		return "", &runError{dup(argv), os.EINVAL}
	}

	prog, ok := lookPathCache[argv[0]]
	if !ok {
		prog, err = exec.LookPath(argv[0])
		if err != nil {
			return "", &runError{dup(argv), err}
		}
		lookPathCache[argv[0]] = prog
	}

	cmd := exec.Command(prog, argv[1:]...)
	if len(input) > 0 {
		cmd.Stdin = bytes.NewBuffer(input)
	}
	bs, err := cmd.CombinedOutput()
	if err != nil {
		return "", &runError{dup(argv), err}
	}
	return string(bs), nil
}

// A runError represents an error that occurred while running a command.
type runError struct {
	cmd []string
	err os.Error
}

func (e *runError) String() string { return strings.Join(e.cmd, " ") + ": " + e.err.String() }
<|endoftext|>


<commit_before>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Objdump is a minimal simulation of the GNU objdump tool,
// just enough to support pprof.
//
// Usage:
//	go tool objdump binary start end
//
// Objdump disassembles the binary starting at the start address and
// stopping at the end address. The start and end addresses are program
// counters written in hexadecimal without a leading 0x prefix.
//
// It prints a sequence of stanzas of the form:
//
//	file:line
//	 address: assembly
//	 address: assembly
//	 ...
//
// Each stanza gives the disassembly for a contiguous range of addresses
// all mapped to the same original source file and line number.
//
// The disassembler is missing (golang.org/issue/7452) but will be added
// before the Go 1.3 release.
//
// This tool is intended for use only by pprof; its interface may change or
// it may be deleted entirely in future releases.
package main

import (
	"bufio"
	"debug/elf"
	"debug/gosym"
	"debug/macho"
	"debug/pe"
	"flag"
	"fmt"
	"log"
	"os"
	"strconv"
)

func printUsage(w *os.File) {
	fmt.Fprintf(w, "usage: objdump binary start end\n")
	fmt.Fprintf(w, "disassembles binary from start PC to end PC.\n")
	fmt.Fprintf(w, "start and end are hexadecimal numbers with no 0x prefix.\n")
}

func usage() {
	printUsage(os.Stderr)
	os.Exit(2)
}

func main() {
	log.SetFlags(0)
	log.SetPrefix("objdump: ")

	flag.Usage = usage
	flag.Parse()
	if flag.NArg() != 3 {
		usage()
	}

	f, err := os.Open(flag.Arg(0))
	if err != nil {
		log.Fatal(err)
	}

	textStart, textData, symtab, pclntab, err := loadTables(f)
	if err != nil {
		log.Fatalf("reading %s: %v", flag.Arg(0), err)
	}

	pcln := gosym.NewLineTable(pclntab, textStart)
	tab, err := gosym.NewTable(symtab, pcln)
	if err != nil {
		log.Fatalf("reading %s: %v", flag.Arg(0), err)
	}

	start, err := strconv.ParseUint(flag.Arg(1), 0, 64)
	if err != nil {
		log.Fatalf("invalid start PC: %v", err)
	}
	end, err := strconv.ParseUint(flag.Arg(2), 0, 64)
	if err != nil {
		log.Fatalf("invalid end PC: %v", err)
	}

	stdout := bufio.NewWriter(os.Stdout)

	// For now, find spans of same PC/line/fn and
	// emit them as having dummy instructions.
	var (
		spanPC   uint64
		spanFile string
		spanLine int
		spanFn   *gosym.Func
	)

	flush := func(endPC uint64) {
		if spanPC == 0 {
			return
		}
		fmt.Fprintf(stdout, "%s:%d\n", spanFile, spanLine)
		for pc := spanPC; pc < endPC; pc++ {
			// TODO(rsc): Disassemble instructions here.
			if textStart <= pc && pc-textStart < uint64(len(textData)) {
				fmt.Fprintf(stdout, " %x: byte %#x\n", pc, textData[pc-textStart])
			} else {
				fmt.Fprintf(stdout, " %x: ?\n", pc)
			}
		}
		spanPC = 0
	}

	for pc := start; pc < end; pc++ {
		file, line, fn := tab.PCToLine(pc)
		if file != spanFile || line != spanLine || fn != spanFn {
			flush(pc)
			spanPC, spanFile, spanLine, spanFn = pc, file, line, fn
		}
	}
	flush(end)

	stdout.Flush()
}

func loadTables(f *os.File) (textStart uint64, textData, symtab, pclntab []byte, err error) {
	if obj, err := elf.NewFile(f); err == nil {
		if sect := obj.Section(".text"); sect != nil {
			textStart = sect.Addr
			textData, _ = sect.Data()
		}
		if sect := obj.Section(".gosymtab"); sect != nil {
			if symtab, err = sect.Data(); err != nil {
				return 0, nil, nil, nil, err
			}
		}
		if sect := obj.Section(".gopclntab"); sect != nil {
			if pclntab, err = sect.Data(); err != nil {
				return 0, nil, nil, nil, err
			}
		}
		return textStart, textData, symtab, pclntab, nil
	}

	if obj, err := macho.NewFile(f); err == nil {
		if sect := obj.Section("__text"); sect != nil {
			textStart = sect.Addr
			textData, _ = sect.Data()
		}
		if sect := obj.Section("__gosymtab"); sect != nil {
			if symtab, err = sect.Data(); err != nil {
				return 0, nil, nil, nil, err
			}
		}
		if sect := obj.Section("__gopclntab"); sect != nil {
			if pclntab, err = sect.Data(); err != nil {
				return 0, nil, nil, nil, err
			}
		}
		return textStart, textData, symtab, pclntab, nil
	}

	if obj, err := pe.NewFile(f); err == nil {
		if sect := obj.Section(".text"); sect != nil {
			textStart = uint64(sect.VirtualAddress)
			textData, _ = sect.Data()
		}
		if sect := obj.Section(".gosymtab"); sect != nil {
			if symtab, err = sect.Data(); err != nil {
				return 0, nil, nil, nil, err
			}
		}
		if sect := obj.Section(".gopclntab"); sect != nil {
			if pclntab, err = sect.Data(); err != nil {
				return 0, nil, nil, nil, err
			}
		}
		return textStart, textData, symtab, pclntab, nil
	}

	return 0, nil, nil, nil, fmt.Errorf("unrecognized binary format")
}
<commit_msg>cmd/objdump: actually accept hex address without "0x" prefix. Fixes #7936.<commit_after>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Objdump is a minimal simulation of the GNU objdump tool,
// just enough to support pprof.
//
// Usage:
//	go tool objdump binary start end
//
// Objdump disassembles the binary starting at the start address and
// stopping at the end address. The start and end addresses are program
// counters written in hexadecimal with optional leading 0x prefix.
//
// It prints a sequence of stanzas of the form:
//
//	file:line
//	 address: assembly
//	 address: assembly
//	 ...
//
// Each stanza gives the disassembly for a contiguous range of addresses
// all mapped to the same original source file and line number.
//
// The disassembler is missing (golang.org/issue/7452) but will be added
// before the Go 1.3 release.
//
// This tool is intended for use only by pprof; its interface may change or
// it may be deleted entirely in future releases.
package main

import (
	"bufio"
	"debug/elf"
	"debug/gosym"
	"debug/macho"
	"debug/pe"
	"flag"
	"fmt"
	"log"
	"os"
	"strconv"
	"strings"
)

func printUsage(w *os.File) {
	fmt.Fprintf(w, "usage: objdump binary start end\n")
	fmt.Fprintf(w, "disassembles binary from start PC to end PC.\n")
	fmt.Fprintf(w, "start and end are hexadecimal numbers with optional leading 0x prefix.\n")
}

func usage() {
	printUsage(os.Stderr)
	os.Exit(2)
}

func main() {
	log.SetFlags(0)
	log.SetPrefix("objdump: ")

	flag.Usage = usage
	flag.Parse()
	if flag.NArg() != 3 {
		usage()
	}

	f, err := os.Open(flag.Arg(0))
	if err != nil {
		log.Fatal(err)
	}

	textStart, textData, symtab, pclntab, err := loadTables(f)
	if err != nil {
		log.Fatalf("reading %s: %v", flag.Arg(0), err)
	}

	pcln := gosym.NewLineTable(pclntab, textStart)
	tab, err := gosym.NewTable(symtab, pcln)
	if err != nil {
		log.Fatalf("reading %s: %v", flag.Arg(0), err)
	}

	start, err := strconv.ParseUint(strings.TrimPrefix(flag.Arg(1), "0x"), 16, 64)
	if err != nil {
		log.Fatalf("invalid start PC: %v", err)
	}
	end, err := strconv.ParseUint(strings.TrimPrefix(flag.Arg(2), "0x"), 16, 64)
	if err != nil {
		log.Fatalf("invalid end PC: %v", err)
	}

	stdout := bufio.NewWriter(os.Stdout)

	// For now, find spans of same PC/line/fn and
	// emit them as having dummy instructions.
	var (
		spanPC   uint64
		spanFile string
		spanLine int
		spanFn   *gosym.Func
	)

	flush := func(endPC uint64) {
		if spanPC == 0 {
			return
		}
		fmt.Fprintf(stdout, "%s:%d\n", spanFile, spanLine)
		for pc := spanPC; pc < endPC; pc++ {
			// TODO(rsc): Disassemble instructions here.
			if textStart <= pc && pc-textStart < uint64(len(textData)) {
				fmt.Fprintf(stdout, " %x: byte %#x\n", pc, textData[pc-textStart])
			} else {
				fmt.Fprintf(stdout, " %x: ?\n", pc)
			}
		}
		spanPC = 0
	}

	for pc := start; pc < end; pc++ {
		file, line, fn := tab.PCToLine(pc)
		if file != spanFile || line != spanLine || fn != spanFn {
			flush(pc)
			spanPC, spanFile, spanLine, spanFn = pc, file, line, fn
		}
	}
	flush(end)

	stdout.Flush()
}

func loadTables(f *os.File) (textStart uint64, textData, symtab, pclntab []byte, err error) {
	if obj, err := elf.NewFile(f); err == nil {
		if sect := obj.Section(".text"); sect != nil {
			textStart = sect.Addr
			textData, _ = sect.Data()
		}
		if sect := obj.Section(".gosymtab"); sect != nil {
			if symtab, err = sect.Data(); err != nil {
				return 0, nil, nil, nil, err
			}
		}
		if sect := obj.Section(".gopclntab"); sect != nil {
			if pclntab, err = sect.Data(); err != nil {
				return 0, nil, nil, nil, err
			}
		}
		return textStart, textData, symtab, pclntab, nil
	}

	if obj, err := macho.NewFile(f); err == nil {
		if sect := obj.Section("__text"); sect != nil {
			textStart = sect.Addr
			textData, _ = sect.Data()
		}
		if sect := obj.Section("__gosymtab"); sect != nil {
			if symtab, err = sect.Data(); err != nil {
				return 0, nil, nil, nil, err
			}
		}
		if sect := obj.Section("__gopclntab"); sect != nil {
			if pclntab, err = sect.Data(); err != nil {
				return 0, nil, nil, nil, err
			}
		}
		return textStart, textData, symtab, pclntab, nil
	}

	if obj, err := pe.NewFile(f); err == nil {
		if sect := obj.Section(".text"); sect != nil {
			textStart = uint64(sect.VirtualAddress)
			textData, _ = sect.Data()
		}
		if sect := obj.Section(".gosymtab"); sect != nil {
			if symtab, err = sect.Data(); err != nil {
				return 0, nil, nil, nil, err
			}
		}
		if sect := obj.Section(".gopclntab"); sect != nil {
			if pclntab, err = sect.Data(); err != nil {
				return 0, nil, nil, nil, err
			}
		}
		return textStart, textData, symtab, pclntab, nil
	}

	return 0, nil, nil, nil, fmt.Errorf("unrecognized binary format")
}
<|endoftext|>


<commit_before>// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build ignore

// The vet/all command runs go vet on the standard library and commands.
// It compares the output against a set of whitelists
// maintained in the whitelist directory.
package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"go/build"
	"internal/testenv"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
)

var (
	flagPlatforms = flag.String("p", "", "platform(s) to use e.g. linux/amd64,darwin/386")
	flagAll       = flag.Bool("all", false, "run all platforms")
	flagNoLines   = flag.Bool("n", false, "don't print line numbers")
)

var cmdGoPath string

func main() {
	log.SetPrefix("vet/all: ")
	log.SetFlags(0)

	var err error
	cmdGoPath, err = testenv.GoTool()
	if err != nil {
		log.Print("could not find cmd/go; skipping")
		// We're on a platform that can't run cmd/go.
		// We want this script to be able to run as part of all.bash,
		// so return cleanly rather than with exit code 1.
		return
	}

	flag.Parse()
	switch {
	case *flagAll && *flagPlatforms != "":
		log.Print("-all and -p flags are incompatible")
		flag.Usage()
		os.Exit(2)
	case *flagPlatforms != "":
		vetPlatforms(parseFlagPlatforms())
	case *flagAll:
		vetPlatforms(allPlatforms())
	default:
		host := platform{os: build.Default.GOOS, arch: build.Default.GOARCH}
		host.vet()
	}
}

func allPlatforms() []platform {
	var pp []platform
	cmd := exec.Command(cmdGoPath, "tool", "dist", "list")
	out, err := cmd.Output()
	if err != nil {
		log.Fatal(err)
	}
	lines := bytes.Split(out, []byte{'\n'})
	for _, line := range lines {
		if len(line) == 0 {
			continue
		}
		pp = append(pp, parsePlatform(string(line)))
	}
	return pp
}

func parseFlagPlatforms() []platform {
	var pp []platform
	components := strings.Split(*flagPlatforms, ",")
	for _, c := range components {
		pp = append(pp, parsePlatform(c))
	}
	return pp
}

func parsePlatform(s string) platform {
	vv := strings.Split(s, "/")
	if len(vv) != 2 {
		log.Fatalf("could not parse platform %s, must be of form goos/goarch", s)
	}
	return platform{os: vv[0], arch: vv[1]}
}

type whitelist map[string]int

// load adds entries from the whitelist file, if present, for os/arch to w.
func (w whitelist) load(goos string, goarch string) {
	// Look up whether goarch is a 32-bit or 64-bit architecture.
	archbits, ok := nbits[goarch]
	if !ok {
		log.Fatal("unknown bitwidth for arch %q", goarch)
	}

	// Look up whether goarch has a shared arch suffix,
	// such as mips64x for mips64 and mips64le.
	archsuff := goarch
	if x, ok := archAsmX[goarch]; ok {
		archsuff = x
	}

	// Load whitelists.
	filenames := []string{
		"all.txt",
		goos + ".txt",
		goarch + ".txt",
		goos + "_" + goarch + ".txt",
		fmt.Sprintf("%dbit.txt", archbits),
	}
	if goarch != archsuff {
		filenames = append(filenames,
			archsuff+".txt",
			goos+"_"+archsuff+".txt",
		)
	}

	// We allow error message templates using GOOS and GOARCH.
	if goos == "android" {
		goos = "linux" // so many special cases :(
	}

	// Read whitelists and do template substitution.
	replace := strings.NewReplacer("GOOS", goos, "GOARCH", goarch, "ARCHSUFF", archsuff)

	for _, filename := range filenames {
		path := filepath.Join("whitelist", filename)
		f, err := os.Open(path)
		if err != nil {
			// Allow not-exist errors; not all combinations have whitelists.
			if os.IsNotExist(err) {
				continue
			}
			log.Fatal(err)
		}
		scan := bufio.NewScanner(f)
		for scan.Scan() {
			line := scan.Text()
			if len(line) == 0 || strings.HasPrefix(line, "//") {
				continue
			}
			w[replace.Replace(line)]++
		}
		if err := scan.Err(); err != nil {
			log.Fatal(err)
		}
	}
}

type platform struct {
	os   string
	arch string
}

func (p platform) String() string {
	return p.os + "/" + p.arch
}

// ignorePathPrefixes are file path prefixes that should be ignored wholesale.
var ignorePathPrefixes = [...]string{
	// These testdata dirs have lots of intentionally broken/bad code for tests.
	"cmd/go/testdata/",
	"cmd/vet/testdata/",
	"go/printer/testdata/",
	// cmd/compile/internal/big is a vendored copy of math/big.
	// Ignore it so that we only have to deal with math/big issues once.
	"cmd/compile/internal/big/",
}

func vetPlatforms(pp []platform) {
	for _, p := range pp {
		p.vet()
	}
}

func (p platform) vet() {
	if p.arch == "s390x" {
		// TODO: reinstate when s390x gets vet support (issue 15454)
		return
	}
	fmt.Printf("go run main.go -p %s\n", p)

	// Load whitelist(s).
	w := make(whitelist)
	w.load(p.os, p.arch)

	env := append(os.Environ(), "GOOS="+p.os, "GOARCH="+p.arch)

	// Do 'go install std' before running vet.
	// It is cheap when already installed.
	// Not installing leads to non-obvious failures due to inability to typecheck.
	// TODO: If go/loader ever makes it to the standard library, have vet use it,
	// at which point vet can work off source rather than compiled packages.
	cmd := exec.Command(cmdGoPath, "install", "std")
	cmd.Env = env
	out, err := cmd.CombinedOutput()
	if err != nil {
		log.Fatalf("failed to run GOOS=%s GOARCH=%s 'go install std': %v\n%s", p.os, p.arch, err, out)
	}

	// 'go tool vet .' is considerably faster than 'go vet ./...'
	// TODO: The unsafeptr checks are disabled for now,
	// because there are so many false positives,
	// and no clear way to improve vet to eliminate large chunks of them.
	// And having them in the whitelists will just cause annoyance
	// and churn when working on the runtime.
	cmd = exec.Command(cmdGoPath, "tool", "vet", "-unsafeptr=false", ".")
	cmd.Dir = filepath.Join(runtime.GOROOT(), "src")
	cmd.Env = env
	stderr, err := cmd.StderrPipe()
	if err != nil {
		log.Fatal(err)
	}
	if err := cmd.Start(); err != nil {
		log.Fatal(err)
	}

	// Process vet output.
	scan := bufio.NewScanner(stderr)
NextLine:
	for scan.Scan() {
		line := scan.Text()
		if strings.HasPrefix(line, "vet: ") {
			// Typecheck failure: Malformed syntax or multiple packages or the like.
			// This will yield nicer error messages elsewhere, so ignore them here.
			continue
		}

		fields := strings.SplitN(line, ":", 3)
		var file, lineno, msg string
		switch len(fields) {
		case 2:
			// vet message with no line number
			file, msg = fields[0], fields[1]
		case 3:
			file, lineno, msg = fields[0], fields[1], fields[2]
		default:
			log.Fatalf("could not parse vet output line:\n%s", line)
		}
		msg = strings.TrimSpace(msg)

		for _, ignore := range ignorePathPrefixes {
			if strings.HasPrefix(file, filepath.FromSlash(ignore)) {
				continue NextLine
			}
		}

		// Temporarily ignore unrecognized printf verbs from cmd.
		// The compiler now has several fancy verbs (CL 28339)
		// used with types implementing fmt.Formatters,
		// and I believe gri has plans to add many more.
		// TODO: remove when issue 17057 is fixed.
		if strings.HasPrefix(file, "cmd/") && strings.HasPrefix(msg, "unrecognized printf verb") {
			continue
		}

		key := file + ": " + msg
		if w[key] == 0 {
			// Vet error with no match in the whitelist. Print it.
			if *flagNoLines {
				fmt.Printf("%s: %s\n", file, msg)
			} else {
				fmt.Printf("%s:%s: %s\n", file, lineno, msg)
			}
			continue
		}
		w[key]--
	}
	if scan.Err() != nil {
		log.Fatalf("failed to scan vet output: %v", scan.Err())
	}
	err = cmd.Wait()
	// We expect vet to fail.
	// Make sure it has failed appropriately, though (for example, not a PathError).
	if _, ok := err.(*exec.ExitError); !ok {
		log.Fatalf("unexpected go vet execution failure: %v", err)
	}
	printedHeader := false
	if len(w) > 0 {
		for k, v := range w {
			if v != 0 {
				if !printedHeader {
					fmt.Println("unmatched whitelist entries:")
					printedHeader = true
				}
				for i := 0; i < v; i++ {
					fmt.Println(k)
				}
			}
		}
	}
}

// nbits maps from architecture names to the number of bits in a pointer.
// TODO: figure out a clean way to avoid get this info rather than listing it here yet again.
var nbits = map[string]int{
	"386":      32,
	"amd64":    64,
	"amd64p32": 32,
	"arm":      32,
	"arm64":    64,
	"mips64":   64,
	"mips64le": 64,
	"ppc64":    64,
	"ppc64le":  64,
}

// archAsmX maps architectures to the suffix usually used for their assembly files,
// if different than the arch name itself.
var archAsmX = map[string]string{
	"android":  "linux",
	"mips64":   "mips64x",
	"mips64le": "mips64x",
	"ppc64":    "ppc64x",
	"ppc64le":  "ppc64x",
}
<commit_msg>cmd/vet/all: check platforms concurrently<commit_after>// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build ignore

// The vet/all command runs go vet on the standard library and commands.
// It compares the output against a set of whitelists
// maintained in the whitelist directory.
package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"go/build"
	"internal/testenv"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"sync"
)

var (
	flagPlatforms = flag.String("p", "", "platform(s) to use e.g. linux/amd64,darwin/386")
	flagAll       = flag.Bool("all", false, "run all platforms")
	flagNoLines   = flag.Bool("n", false, "don't print line numbers")
)

var cmdGoPath string

func main() {
	log.SetPrefix("vet/all: ")
	log.SetFlags(0)

	var err error
	cmdGoPath, err = testenv.GoTool()
	if err != nil {
		log.Print("could not find cmd/go; skipping")
		// We're on a platform that can't run cmd/go.
		// We want this script to be able to run as part of all.bash,
		// so return cleanly rather than with exit code 1.
		return
	}

	flag.Parse()
	switch {
	case *flagAll && *flagPlatforms != "":
		log.Print("-all and -p flags are incompatible")
		flag.Usage()
		os.Exit(2)
	case *flagPlatforms != "":
		vetPlatforms(parseFlagPlatforms())
	case *flagAll:
		vetPlatforms(allPlatforms())
	default:
		host := platform{os: build.Default.GOOS, arch: build.Default.GOARCH}
		host.vet(runtime.GOMAXPROCS(-1))
	}
}

func allPlatforms() []platform {
	var pp []platform
	cmd := exec.Command(cmdGoPath, "tool", "dist", "list")
	out, err := cmd.Output()
	if err != nil {
		log.Fatal(err)
	}
	lines := bytes.Split(out, []byte{'\n'})
	for _, line := range lines {
		if len(line) == 0 {
			continue
		}
		pp = append(pp, parsePlatform(string(line)))
	}
	return pp
}

func parseFlagPlatforms() []platform {
	var pp []platform
	components := strings.Split(*flagPlatforms, ",")
	for _, c := range components {
		pp = append(pp, parsePlatform(c))
	}
	return pp
}

func parsePlatform(s string) platform {
	vv := strings.Split(s, "/")
	if len(vv) != 2 {
		log.Fatalf("could not parse platform %s, must be of form goos/goarch", s)
	}
	return platform{os: vv[0], arch: vv[1]}
}

type whitelist map[string]int

// load adds entries from the whitelist file, if present, for os/arch to w.
func (w whitelist) load(goos string, goarch string) {
	// Look up whether goarch is a 32-bit or 64-bit architecture.
	archbits, ok := nbits[goarch]
	if !ok {
		log.Fatal("unknown bitwidth for arch %q", goarch)
	}

	// Look up whether goarch has a shared arch suffix,
	// such as mips64x for mips64 and mips64le.
	archsuff := goarch
	if x, ok := archAsmX[goarch]; ok {
		archsuff = x
	}

	// Load whitelists.
	filenames := []string{
		"all.txt",
		goos + ".txt",
		goarch + ".txt",
		goos + "_" + goarch + ".txt",
		fmt.Sprintf("%dbit.txt", archbits),
	}
	if goarch != archsuff {
		filenames = append(filenames,
			archsuff+".txt",
			goos+"_"+archsuff+".txt",
		)
	}

	// We allow error message templates using GOOS and GOARCH.
	if goos == "android" {
		goos = "linux" // so many special cases :(
	}

	// Read whitelists and do template substitution.
	replace := strings.NewReplacer("GOOS", goos, "GOARCH", goarch, "ARCHSUFF", archsuff)

	for _, filename := range filenames {
		path := filepath.Join("whitelist", filename)
		f, err := os.Open(path)
		if err != nil {
			// Allow not-exist errors; not all combinations have whitelists.
			if os.IsNotExist(err) {
				continue
			}
			log.Fatal(err)
		}
		scan := bufio.NewScanner(f)
		for scan.Scan() {
			line := scan.Text()
			if len(line) == 0 || strings.HasPrefix(line, "//") {
				continue
			}
			w[replace.Replace(line)]++
		}
		if err := scan.Err(); err != nil {
			log.Fatal(err)
		}
	}
}

type platform struct {
	os   string
	arch string
}

func (p platform) String() string {
	return p.os + "/" + p.arch
}

// ignorePathPrefixes are file path prefixes that should be ignored wholesale.
var ignorePathPrefixes = [...]string{
	// These testdata dirs have lots of intentionally broken/bad code for tests.
	"cmd/go/testdata/",
	"cmd/vet/testdata/",
	"go/printer/testdata/",
	// cmd/compile/internal/big is a vendored copy of math/big.
	// Ignore it so that we only have to deal with math/big issues once.
	"cmd/compile/internal/big/",
}

func vetPlatforms(pp []platform) {
	ncpus := runtime.GOMAXPROCS(-1) / len(pp)
	if ncpus < 1 {
		ncpus = 1
	}
	var wg sync.WaitGroup
	wg.Add(len(pp))
	for _, p := range pp {
		p := p
		go func() {
			p.vet(ncpus)
			wg.Done()
		}()
	}
	wg.Wait()
}

func (p platform) vet(ncpus int) {
	if p.arch == "s390x" {
		// TODO: reinstate when s390x gets vet support (issue 15454)
		return
	}
	var buf bytes.Buffer
	fmt.Fprintf(&buf, "go run main.go -p %s\n", p)

	// Load whitelist(s).
	w := make(whitelist)
	w.load(p.os, p.arch)

	env := append(os.Environ(), "GOOS="+p.os, "GOARCH="+p.arch)

	// Do 'go install std' before running vet.
	// It is cheap when already installed.
	// Not installing leads to non-obvious failures due to inability to typecheck.
	// TODO: If go/loader ever makes it to the standard library, have vet use it,
	// at which point vet can work off source rather than compiled packages.
	cmd := exec.Command(cmdGoPath, "install", "-p", strconv.Itoa(ncpus), "std")
	cmd.Env = env
	out, err := cmd.CombinedOutput()
	if err != nil {
		log.Fatalf("failed to run GOOS=%s GOARCH=%s 'go install std': %v\n%s", p.os, p.arch, err, out)
	}

	// 'go tool vet .' is considerably faster than 'go vet ./...'
	// TODO: The unsafeptr checks are disabled for now,
	// because there are so many false positives,
	// and no clear way to improve vet to eliminate large chunks of them.
	// And having them in the whitelists will just cause annoyance
	// and churn when working on the runtime.
	cmd = exec.Command(cmdGoPath, "tool", "vet", "-unsafeptr=false", ".")
	cmd.Dir = filepath.Join(runtime.GOROOT(), "src")
	cmd.Env = env
	stderr, err := cmd.StderrPipe()
	if err != nil {
		log.Fatal(err)
	}
	if err := cmd.Start(); err != nil {
		log.Fatal(err)
	}

	// Process vet output.
	scan := bufio.NewScanner(stderr)
NextLine:
	for scan.Scan() {
		line := scan.Text()
		if strings.HasPrefix(line, "vet: ") {
			// Typecheck failure: Malformed syntax or multiple packages or the like.
			// This will yield nicer error messages elsewhere, so ignore them here.
			continue
		}

		fields := strings.SplitN(line, ":", 3)
		var file, lineno, msg string
		switch len(fields) {
		case 2:
			// vet message with no line number
			file, msg = fields[0], fields[1]
		case 3:
			file, lineno, msg = fields[0], fields[1], fields[2]
		default:
			log.Fatalf("could not parse vet output line:\n%s", line)
		}
		msg = strings.TrimSpace(msg)

		for _, ignore := range ignorePathPrefixes {
			if strings.HasPrefix(file, filepath.FromSlash(ignore)) {
				continue NextLine
			}
		}

		// Temporarily ignore unrecognized printf verbs from cmd.
		// The compiler now has several fancy verbs (CL 28339)
		// used with types implementing fmt.Formatters,
		// and I believe gri has plans to add many more.
		// TODO: remove when issue 17057 is fixed.
		if strings.HasPrefix(file, "cmd/") && strings.HasPrefix(msg, "unrecognized printf verb") {
			continue
		}

		key := file + ": " + msg
		if w[key] == 0 {
			// Vet error with no match in the whitelist. Print it.
			if *flagNoLines {
				fmt.Fprintf(&buf, "%s: %s\n", file, msg)
			} else {
				fmt.Fprintf(&buf, "%s:%s: %s\n", file, lineno, msg)
			}
			continue
		}
		w[key]--
	}
	if scan.Err() != nil {
		log.Fatalf("failed to scan vet output: %v", scan.Err())
	}
	err = cmd.Wait()
	// We expect vet to fail.
	// Make sure it has failed appropriately, though (for example, not a PathError).
	if _, ok := err.(*exec.ExitError); !ok {
		log.Fatalf("unexpected go vet execution failure: %v", err)
	}
	printedHeader := false
	if len(w) > 0 {
		for k, v := range w {
			if v != 0 {
				if !printedHeader {
					fmt.Fprintln(&buf, "unmatched whitelist entries:")
					printedHeader = true
				}
				for i := 0; i < v; i++ {
					fmt.Fprintln(&buf, k)
				}
			}
		}
	}

	os.Stdout.Write(buf.Bytes())
}

// nbits maps from architecture names to the number of bits in a pointer.
// TODO: figure out a clean way to avoid get this info rather than listing it here yet again.
var nbits = map[string]int{
	"386":      32,
	"amd64":    64,
	"amd64p32": 32,
	"arm":      32,
	"arm64":    64,
	"mips64":   64,
	"mips64le": 64,
	"ppc64":    64,
	"ppc64le":  64,
}

// archAsmX maps architectures to the suffix usually used for their assembly files,
// if different than the arch name itself.
var archAsmX = map[string]string{
	"android":  "linux",
	"mips64":   "mips64x",
	"mips64le": "mips64x",
	"ppc64":    "ppc64x",
	"ppc64le":  "ppc64x",
}
<|endoftext|>


<commit_before>package service

import (
	"net/http"
	"path/filepath"

	"github.com/materials-commons/mcstore/pkg/app"
	"github.com/materials-commons/mcstore/pkg/db/schema"
	"github.com/materials-commons/mcstore/pkg/domain"
	"github.com/materials-commons/mcstore/pkg/ws"
)

// dataHandler implements the http.Handler interface. It provides an interface
// to serving up data stored in materials commons.
type dataHandler struct {
	access domain.Access
}

// NewDataHandler creates a new instance of a dataHandler.
func NewDataHandler(access domain.Access) http.Handler {
	return &dataHandler{
		access: access,
	}
}

// ServeHTTP serves data stored in materials commons.
func (h *dataHandler) ServeHTTP(writer http.ResponseWriter, req *http.Request) {
	path, mediaType, err := h.serveData(writer, req)
	switch {
	case err != nil:
		ws.WriteError(err, writer)
	default:
		serveFile(writer, req, path, mediaType)
	}
}

// serveFile serves up the actual file contents. It sets the content-type header to
// the mediatype specified. If the file doesn't exist, or the server doesn't have
// permissions to access the file then a 404 (not found) will be returned by the
// http.ServeFile method.
func serveFile(writer http.ResponseWriter, req *http.Request, path, mediatype string) {
	writer.Header().Set("Content-Type", mediatype)
	app.Log.Debug(app.Logf("Set Content-Type to %s", mediatype))
	http.ServeFile(writer, req, path)
}

// serveData does the actual work of serving the data. It checks the access on each
// request. It will also serve up converted images unless the original flag has been
// specified. The assumption is that without the original flag we are actually trying
// to render an image in a browser. Since browsers do not render all image types we
// convert some types to jpg files. This routine will serve up these jpg conversions
// rather than the original file unless the original flag is specified.
func (h *dataHandler) serveData(writer http.ResponseWriter, req *http.Request) (path string, mediatype string, err error) {
	// All requests require an apikey.
	apikey := req.FormValue("apikey")
	if apikey == "" {
		return path, mediatype, app.ErrNoAccess
	}
	app.Log.Debug(app.Logf("serveData - Request for apikey %s", apikey))

	// Is the original data requested, or can we serve the converted
	// image data (if it exists)?
	original := getOriginalFormValue(req)
	app.Log.Debug(app.Logf("serveData - Original flag %t", original))

	fileID := filepath.Base(req.URL.Path)
	app.Log.Debug(app.Logf("serveData - fileID %s, URL %s", fileID, req.URL.Path))

	// Get the file checking its access.
	file, err := h.access.GetFile(apikey, fileID)
	if err != nil {
		return path, mediatype, err
	}

	path = filePath(file, original)
	app.Log.Debug(app.Logf("serveData - Serving path: %s\n", path))

	mediatype = file.MediaType.Mime

	// The content type is dependent on whether we are
	// serving the original or the converted file.
	if !original && isConvertedImage(file.MediaType.Mime) {
		mediatype = "image/jpeg"
	}

	return path, mediatype, nil
}

// getOriginalFormValue looks for the original argument on the URL. If it exists
// (has a value specified) then return true. The original flag specifies whether
// the user wants the original rather than the converted data.
func getOriginalFormValue(req *http.Request) bool {
	if req.FormValue("original") == "" {
		return false
	}
	return true
}

// filePath determines which file to serve. If there is a converted image we
// will serve that one up unless the original flag is set to true.
func filePath(file *schema.File, original bool) string {
	switch {
	case isConvertedImage(file.MediaType.Mime) && !original:
		return app.MCDir.FilePathImageConversion(file.FileID())
	default:
		return app.MCDir.FilePath(file.FileID())
	}
}

// isConvertedImage checks a name to see if it is an image type we have converted.
func isConvertedImage(mime string) bool {
	switch mime {
	case "image/tiff":
		return true
	case "image/x-ms-bmp":
		return true
	default:
		return false
	}
}
<commit_msg>Fix comment to make it more understandable.<commit_after>package service

import (
	"net/http"
	"path/filepath"

	"github.com/materials-commons/mcstore/pkg/app"
	"github.com/materials-commons/mcstore/pkg/db/schema"
	"github.com/materials-commons/mcstore/pkg/domain"
	"github.com/materials-commons/mcstore/pkg/ws"
)

// dataHandler implements the http.Handler interface. It provides an interface
// to serving up data stored in materials commons.
type dataHandler struct {
	access domain.Access
}

// NewDataHandler creates a new instance of a dataHandler.
func NewDataHandler(access domain.Access) http.Handler {
	return &dataHandler{
		access: access,
	}
}

// ServeHTTP serves data stored in materials commons.
func (h *dataHandler) ServeHTTP(writer http.ResponseWriter, req *http.Request) {
	path, mediaType, err := h.serveData(writer, req)
	switch {
	case err != nil:
		ws.WriteError(err, writer)
	default:
		serveFile(writer, req, path, mediaType)
	}
}

// serveFile serves up the actual file contents. It sets the content-type header to
// the mediatype specified. If the file doesn't exist, or the server doesn't have
// permissions to access the file then a 404 (not found) will be returned by the
// http.ServeFile method.
func serveFile(writer http.ResponseWriter, req *http.Request, path, mediatype string) {
	writer.Header().Set("Content-Type", mediatype)
	app.Log.Debug(app.Logf("Set Content-Type to %s", mediatype))
	http.ServeFile(writer, req, path)
}

// serveData does the actual work of serving the data. It checks the access on each
// request. It will also serve up converted images unless the original flag has been
// specified. The assumption is that without the original flag we are actually trying
// to render an image in a browser. Since browsers do not render all image types we
// convert some types to jpg files. This routine will serve up these jpg conversions
// rather than the original file unless the original flag is specified.
func (h *dataHandler) serveData(writer http.ResponseWriter, req *http.Request) (path string, mediatype string, err error) {
	// All requests require an apikey.
	apikey := req.FormValue("apikey")
	if apikey == "" {
		return path, mediatype, app.ErrNoAccess
	}
	app.Log.Debug(app.Logf("serveData - Request for apikey %s", apikey))

	// Is the original data requested, or can we serve the converted
	// image data (if it exists)?
	original := getOriginalFormValue(req)
	app.Log.Debug(app.Logf("serveData - Original flag %t", original))

	fileID := filepath.Base(req.URL.Path)
	app.Log.Debug(app.Logf("serveData - fileID %s, URL %s", fileID, req.URL.Path))

	// Get the file, checking its access.
	file, err := h.access.GetFile(apikey, fileID)
	if err != nil {
		return path, mediatype, err
	}

	path = filePath(file, original)
	app.Log.Debug(app.Logf("serveData - Serving path: %s\n", path))

	mediatype = file.MediaType.Mime

	// The content type is dependent on whether we are
	// serving the original or the converted file.
	if !original && isConvertedImage(file.MediaType.Mime) {
		mediatype = "image/jpeg"
	}

	return path, mediatype, nil
}

// getOriginalFormValue looks for the original argument on the URL. If it exists
// (has a value specified) then return true. The original flag specifies whether
// the user wants the original rather than the converted data.
func getOriginalFormValue(req *http.Request) bool {
	if req.FormValue("original") == "" {
		return false
	}
	return true
}

// filePath determines which file to serve. If there is a converted image we
// will serve that one up unless the original flag is set to true.
func filePath(file *schema.File, original bool) string {
	switch {
	case isConvertedImage(file.MediaType.Mime) && !original:
		return app.MCDir.FilePathImageConversion(file.FileID())
	default:
		return app.MCDir.FilePath(file.FileID())
	}
}

// isConvertedImage checks a name to see if it is an image type we have converted.
func isConvertedImage(mime string) bool {
	switch mime {
	case "image/tiff":
		return true
	case "image/x-ms-bmp":
		return true
	default:
		return false
	}
}
<|endoftext|>


<commit_before>package cli

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"strings"
	"sync"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/bitrise-io/bitrise-machine/config"
	"github.com/bitrise-io/bitrise-machine/utils"
	"github.com/codegangsta/cli"
)

const (
	logChunkRuneLenght = 1

	// LogFormatJSON ...
	LogFormatJSON = "json"
)

// RunResults ...
type RunResults struct {
	IsTimeoutError bool
	RunError       error
}

// LogBuffer ...
type LogBuffer struct {
	logBytes bytes.Buffer
	rwlock   sync.RWMutex
}

// Write ...
func (buff *LogBuffer) Write(p []byte) (n int, err error) {
	buff.rwlock.Lock()
	defer buff.rwlock.Unlock()
	return buff.logBytes.Write(p)
}

func (buff *LogBuffer) Read(n int) []byte {
	buff.rwlock.Lock()
	defer buff.rwlock.Unlock()
	return buff.logBytes.Next(n)
}

// ReadRunes ...
func (buff *LogBuffer) ReadRunes(n int) (string, bool) {
	buff.rwlock.Lock()
	defer buff.rwlock.Unlock()
	res := ""
	isEOF := false
	for i := 0; i < n; i++ {
		r, _, err := buff.logBytes.ReadRune()
		if err == nil {
			res += string(r)
		} else if err == io.EOF {
			isEOF = true
			break
		} else {
			log.Errorf("Failed to read from LogBuffer: %s", err)
		}
	}
	return res, isEOF
}

// LogChunkModel ...
type LogChunkModel struct {
	Data string `json:"data"`
	Pos  int64  `json:"pos"`
}

func logChunkJSONTransform(logChunkData string) ([]byte, error) {
	logChunk := LogChunkModel{
		Data: logChunkData,
	}
	return json.Marshal(logChunk)
}

func performRun(sshConfig config.SSHConfigModel, commandToRunStr string, timeoutSeconds int64, logFormat string) RunResults {
	logBuff := LogBuffer{}

	// Log processing
	processLogs := func(isFlush bool) {
		for {
			chunkStr, isEOF := logBuff.ReadRunes(logChunkRuneLenght)
			if chunkStr != "" {
				if logFormat == LogFormatJSON {
					logChunkBytes, err := logChunkJSONTransform(chunkStr)
					if err != nil {
						log.Errorf("Failed to convert log chunk. Error: %s", err)
						log.Errorf(" Log chunk was: %s", chunkStr)
					}
					fmt.Printf("%s\n", logChunkBytes)
				} else {
					fmt.Printf("%s", chunkStr)
				}
			}

			if !isFlush || isEOF {
				break
			}
		}
	}

	// Run
	isRunFinished := false
	runRes := RunResults{
		IsTimeoutError: false,
		RunError:       nil,
	}
	c1 := make(chan RunResults, 1)
	go func() {
		if err := utils.RunCommandThroughSSHWithWriters(sshConfig, commandToRunStr, &logBuff, &logBuff); err != nil {
			log.Errorf("Failed to run command, error: %s", err)
			c1 <- RunResults{RunError: err, IsTimeoutError: false}
		} else {
			c1 <- RunResults{RunError: nil, IsTimeoutError: false}
		}
	}()

	var timeoutTriggerred <-chan time.Time
	if timeoutSeconds > 0 {
		log.Infof("Timeout registered with %d seconds from now.", timeoutSeconds)
		timeoutTriggerred = time.After(time.Duration(timeoutSeconds) * time.Second)
	}

	for !isRunFinished {
		select {
		case res := <-c1:
			if res.RunError == nil {
				runRes = res
			} else {
				runRes = res
			}
			isRunFinished = true
		case <-timeoutTriggerred:
			runRes = RunResults{RunError: fmt.Errorf("Timeout after %d seconds", timeoutSeconds), IsTimeoutError: true}
			isRunFinished = true
		case <-time.Tick(100 * time.Millisecond):
			processLogs(true)
		}
	}

	processLogs(true)

	return runRes
}

func run(c *cli.Context) {
	log.Infoln("Run")

	if len(c.Args()) < 1 {
		log.Fatalln("No command to run specified!")
	}

	inCmdArgs := c.Args()
	cmdToRun := inCmdArgs[0]
	cmdToRunArgs := []string{}
	if len(inCmdArgs) > 1 {
		cmdToRunArgs = inCmdArgs[1:]
	}

	sshConfigModel, err := config.ReadSSHConfigFileFromDir(MachineWorkdir)
	if err != nil {
		log.Fatalln("Failed to read SSH configs - you should probably call 'setup' first!")
	}

	fullCmdToRunStr := fmt.Sprintf("%s %s", cmdToRun, strings.Join(cmdToRunArgs, " "))
	log.Infoln("fullCmdToRunStr: ", fullCmdToRunStr)

	timeoutSecs := c.Int(TimeoutFlagKey)
	if timeoutSecs > 0 {
		log.Infof("Timeout parameter: %d seconds", timeoutSecs)
	} else {
		log.Infoln("No timeout defined.")
		timeoutSecs = 0
	}

	logFormat := c.String(LogFormatFlagKey)
	if logFormat == "json" {
		log.Infof("Log format: %s", logFormat)
	} else {
		if logFormat != "" {
			log.Infof("Invalid Log Format - ignoring: %s", logFormat)
		}
		logFormat = ""
	}

	runResult := performRun(sshConfigModel, fullCmdToRunStr, int64(timeoutSecs), logFormat)
	if runResult.RunError != nil {
		if runResult.IsTimeoutError {
			log.Errorf("[!] Timeout: %s", runResult.RunError)
			os.Exit(2)
		} else {
			log.Errorf("Run failed: %s", runResult.RunError)
			os.Exit(1)
		}
	}

	log.Infoln("Run finished - OK")
}
<commit_msg>log chunk position handling & log flushing<commit_after>package cli

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"strings"
	"sync"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/bitrise-io/bitrise-machine/config"
	"github.com/bitrise-io/bitrise-machine/utils"
	"github.com/codegangsta/cli"
)

const (
	logChunkRuneLenght = 100

	// LogFormatJSON ...
	LogFormatJSON = "json"
)

// RunResults ...
type RunResults struct {
	IsTimeoutError bool
	RunError       error
}

// LogBuffer ...
type LogBuffer struct {
	logBytes bytes.Buffer
	rwlock   sync.RWMutex
}

// Write ...
func (buff *LogBuffer) Write(p []byte) (n int, err error) {
	buff.rwlock.Lock()
	defer buff.rwlock.Unlock()
	return buff.logBytes.Write(p)
}

func (buff *LogBuffer) Read(n int) []byte {
	buff.rwlock.Lock()
	defer buff.rwlock.Unlock()
	return buff.logBytes.Next(n)
}

// ReadRunes ...
func (buff *LogBuffer) ReadRunes(n int) (string, bool) {
	buff.rwlock.Lock()
	defer buff.rwlock.Unlock()
	res := ""
	isEOF := false
	for i := 0; i < n; i++ {
		r, _, err := buff.logBytes.ReadRune()
		if err == nil {
			res += string(r)
		} else if err == io.EOF {
			isEOF = true
			break
		} else {
			log.Errorf("Failed to read from LogBuffer: %s", err)
		}
	}
	return res, isEOF
}

// LogChunkModel ...
type LogChunkModel struct {
	Data string `json:"data"`
	Pos  int64  `json:"pos"`
}

func logChunkJSONTransform(logChunkData string, logChunkIdx int64) ([]byte, error) {
	logChunk := LogChunkModel{
		Data: logChunkData,
		Pos:  logChunkIdx,
	}
	return json.Marshal(logChunk)
}

func performRun(sshConfig config.SSHConfigModel, commandToRunStr string, timeoutSeconds int64, logFormat string) RunResults {
	logBuff := LogBuffer{}
	var logChunkIndex int64
	lastLogChunkSentAt := time.Now()

	// Log processing
	processLogs := func(isFlush bool) (isLogChunkGenerated bool) {
		for {
			isChunkDone := false
			if logBuff.logBytes.Len() > logChunkRuneLenght || isFlush {
				chunkStr, isEOF := logBuff.ReadRunes(logChunkRuneLenght)
				if chunkStr != "" {
					if logFormat == LogFormatJSON {
						logChunkBytes, err := logChunkJSONTransform(chunkStr, logChunkIndex)
						logChunkIndex++
						if err != nil {
							log.Errorf("Failed to convert log chunk. Error: %s", err)
							log.Errorf(" Log chunk was: %s", chunkStr)
						}
						fmt.Printf("%s\n", logChunkBytes)
					} else {
						fmt.Printf("%s", chunkStr)
					}
					isLogChunkGenerated = true
				}
				if isEOF {
					isChunkDone = true
				}
			} else {
				isChunkDone = true
			}

			if !isFlush || isChunkDone {
				break
			}
		}
		return
	}

	// Run
	isRunFinished := false
	runRes := RunResults{
		IsTimeoutError: false,
		RunError:       nil,
	}
	c1 := make(chan RunResults, 1)
	go func() {
		if err := utils.RunCommandThroughSSHWithWriters(sshConfig, commandToRunStr, &logBuff, &logBuff); err != nil {
			log.Errorf("Failed to run command, error: %s", err)
			c1 <- RunResults{RunError: err, IsTimeoutError: false}
		} else {
			c1 <- RunResults{RunError: nil, IsTimeoutError: false}
		}
	}()

	var timeoutTriggerred <-chan time.Time
	if timeoutSeconds > 0 {
		log.Infof("Timeout registered with %d seconds from now.", timeoutSeconds)
		timeoutTriggerred = time.After(time.Duration(timeoutSeconds) * time.Second)
	}

	for !isRunFinished {
		select {
		case res := <-c1:
			if res.RunError == nil {
				runRes = res
			} else {
				runRes = res
			}
			isRunFinished = true
		case <-timeoutTriggerred:
			runRes = RunResults{RunError: fmt.Errorf("Timeout after %d seconds", timeoutSeconds), IsTimeoutError: true}
			isRunFinished = true
		case <-time.Tick(100 * time.Millisecond):
			isFlushLogs := false
			// force flush logs if we did not generated log chunks in the last
			//  couple of seconds - if the process did not generate enough logs
			//  to trigger a chunk generation
			timeDiffSec := time.Now().Sub(lastLogChunkSentAt).Seconds()
			if timeDiffSec > 5.0 {
				isFlushLogs = true
			}
			if processLogs(isFlushLogs) || isFlushLogs {
				lastLogChunkSentAt = time.Now()
			}
		}
	}

	processLogs(true)

	return runRes
}

func run(c *cli.Context) {
	log.Infoln("Run")

	if len(c.Args()) < 1 {
		log.Fatalln("No command to run specified!")
	}

	inCmdArgs := c.Args()
	cmdToRun := inCmdArgs[0]
	cmdToRunArgs := []string{}
	if len(inCmdArgs) > 1 {
		cmdToRunArgs = inCmdArgs[1:]
	}

	sshConfigModel, err := config.ReadSSHConfigFileFromDir(MachineWorkdir)
	if err != nil {
		log.Fatalln("Failed to read SSH configs - you should probably call 'setup' first!")
	}

	fullCmdToRunStr := fmt.Sprintf("%s %s", cmdToRun, strings.Join(cmdToRunArgs, " "))
	log.Infoln("fullCmdToRunStr: ", fullCmdToRunStr)

	timeoutSecs := c.Int(TimeoutFlagKey)
	if timeoutSecs > 0 {
		log.Infof("Timeout parameter: %d seconds", timeoutSecs)
	} else {
		log.Infoln("No timeout defined.")
		timeoutSecs = 0
	}

	logFormat := c.String(LogFormatFlagKey)
	if logFormat == "json" {
		log.Infof("Log format: %s", logFormat)
	} else {
		if logFormat != "" {
			log.Infof("Invalid Log Format - ignoring: %s", logFormat)
		}
		logFormat = ""
	}

	runResult := performRun(sshConfigModel, fullCmdToRunStr, int64(timeoutSecs), logFormat)
	if runResult.RunError != nil {
		if runResult.IsTimeoutError {
			log.Errorf("[!] Timeout: %s", runResult.RunError)
			os.Exit(2)
		} else {
			log.Errorf("Run failed: %s", runResult.RunError)
			os.Exit(1)
		}
	}

	log.Infoln("Run finished - OK")
}
<|endoftext|>


<commit_before>// You can edit this code!
// Click here and start typing.
package main

import "fmt"

func main() {
	fmt.Println("Hello, 世界")
}
<commit_msg>Update world.go<commit_after>// You can edit this code!
// Click here and start typing.
package main

import "fmt"

func main() {
	fmt.Println("Hola, World")
}
<|endoftext|>


<commit_before>// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// See https://code.google.com/p/go/source/browse/CONTRIBUTORS
// Licensed under the same terms as Go itself:
// https://code.google.com/p/go/source/browse/LICENSE

package http2

import (
	"crypto/tls"
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/http/httptest"
	"os/exec"
	"strconv"
	"strings"
	"sync/atomic"
	"testing"
	"time"
)

func TestServer(t *testing.T) {
	ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Foo", "Bar")
	}))
	ConfigureServer(ts.Config, &Server{})
	ts.TLS = ts.Config.TLSConfig // the httptest.Server has its own copy of this TLS config
	ts.StartTLS()
	defer ts.Close()

	t.Logf("Running test server at: %s", ts.URL)
	cc, err := tls.Dial("tcp", ts.Listener.Addr().String(), &tls.Config{
		InsecureSkipVerify: true,
		NextProtos:         []string{npnProto},
	})
	if err != nil {
		t.Fatal(err)
	}
	defer cc.Close()

	mustWrite(t, cc, clientPreface)
	fr := NewFramer(cc, cc)
	if err := fr.WriteSettings(); err != nil {
		t.Fatal(err)
	}

	// Expect first Settings frame.
	{
		f, err := fr.ReadFrame()
		if err != nil {
			t.Fatal(err)
		}
		sf, ok := f.(*SettingsFrame)
		if !ok {
			t.Fatalf("Received a %T, not a Settings frame from the server", f)
		}
		sf.ForeachSetting(func(s Setting) {
			t.Logf("Server sent setting %v = %v", s.ID, s.Val)
		})
	}

	// And expect an ACK of our settings.
	{
		f, err := fr.ReadFrame()
		if err != nil {
			t.Fatal(err)
		}
		sf, ok := f.(*SettingsFrame)
		if !ok {
			t.Fatalf("Received a %T, not a Settings ack frame from the server", f)
		}
		if !sf.Header().Flags.Has(FlagSettingsAck) {
			t.Fatal("Settings Frame didn't have ACK set")
		}
	}

	// TODO: table-itize steps, write request (HEADERS frame), read response.
}

func mustWrite(t *testing.T, w io.Writer, p []byte) {
	n, err := w.Write(p)
	const maxLen = 80
	l := len(p)
	if len(p) > maxLen {
		p = p[:maxLen]
	}
	if err != nil {
		t.Fatalf("Error writing %d bytes (%q): %v", l, p, err)
	}
	if n != len(p) {
		t.Fatalf("Only wrote %d of %d bytes (%q)", n, l, p)
	}
}

func TestServerWithCurl(t *testing.T) {
	requireCurl(t)

	ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		// TODO: add a bunch of different tests with different
		// behavior, as a function of r or a table.
		// -- with request body, without.
		// -- no interaction with w.
		// -- panic
		// -- modify Header only, but no writes or writeheader (this test)
		// -- WriteHeader only
		// -- Write only
		// -- WriteString
		// -- both
		// -- huge headers over a frame size so we get continuation headers.
		// Look at net/http's Server tests for inspiration.
		w.Header().Set("Foo", "Bar")
	}))
	ConfigureServer(ts.Config, &Server{})
	ts.TLS = ts.Config.TLSConfig // the httptest.Server has its own copy of this TLS config
	ts.StartTLS()
	defer ts.Close()

	var gotConn int32
	testHookOnConn = func() { atomic.StoreInt32(&gotConn, 1) }

	t.Logf("Running test server for curl to hit at: %s", ts.URL)
	container := curl(t, "--silent", "--http2", "--insecure", "-v", ts.URL)
	defer kill(container)
	resc := make(chan interface{}, 1)
	go func() {
		res, err := dockerLogs(container)
		if err != nil {
			resc <- err
		} else {
			resc <- res
		}
	}()
	select {
	case res := <-resc:
		if err, ok := res.(error); ok {
			t.Fatal(err)
		}
		if !strings.Contains(string(res.([]byte)), "< foo:Bar") {
			t.Errorf("didn't see foo:Bar header")
			t.Logf("Got: %s", res)
		}
	case <-time.After(3 * time.Second):
		t.Errorf("timeout waiting for curl")
	}

	if atomic.LoadInt32(&gotConn) == 0 {
		t.Error("never saw an http2 connection")
	}
}

func dockerLogs(container string) ([]byte, error) {
	out, err := exec.Command("docker", "wait", container).CombinedOutput()
	if err != nil {
		return out, err
	}
	exitStatus, err := strconv.Atoi(strings.TrimSpace(string(out)))
	if err != nil {
		return out, errors.New("unexpected exit status from docker wait")
	}
	out, err = exec.Command("docker", "logs", container).CombinedOutput()
	exec.Command("docker", "rm", container).Run()
	if err == nil && exitStatus != 0 {
		err = fmt.Errorf("exit status %d", exitStatus)
	}
	return out, err
}

func kill(container string) {
	exec.Command("docker", "kill", container).Run()
	exec.Command("docker", "rm", container).Run()
}

// Verify that curl has http2.
func requireCurl(t *testing.T) {
	out, err := dockerLogs(curl(t, "--version"))
	if err != nil {
		t.Skipf("failed to determine curl features; skipping test")
	}
	if !strings.Contains(string(out), "HTTP2") {
		t.Skip("curl doesn't support HTTP2; skipping test")
	}
}

func curl(t *testing.T, args ...string) (container string) {
	out, err := exec.Command("docker", append([]string{"run", "-d", "--net=host", "gohttp2/curl"}, args...)...).CombinedOutput()
	if err != nil {
		t.Skipf("Failed to run curl in docker: %v, %s", err, out)
	}
	return strings.TrimSpace(string(out))
}
<commit_msg>Start of serverTester type<commit_after>// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// See https://code.google.com/p/go/source/browse/CONTRIBUTORS
// Licensed under the same terms as Go itself:
// https://code.google.com/p/go/source/browse/LICENSE

package http2

import (
	"crypto/tls"
	"errors"
	"fmt"
	"io"
	"net"
	"net/http"
	"net/http/httptest"
	"os/exec"
	"strconv"
	"strings"
	"sync/atomic"
	"testing"
	"time"
)

type serverTester struct {
	cc net.Conn // client conn
	t  *testing.T
	ts *httptest.Server
	fr *Framer
}

func newServerTester(t *testing.T, handler http.HandlerFunc) *serverTester {
	ts := httptest.NewUnstartedServer(handler)
	ConfigureServer(ts.Config, &Server{})
	ts.TLS = ts.Config.TLSConfig // the httptest.Server has its own copy of this TLS config
	ts.StartTLS()

	t.Logf("Running test server at: %s", ts.URL)
	cc, err := tls.Dial("tcp", ts.Listener.Addr().String(), &tls.Config{
		InsecureSkipVerify: true,
		NextProtos:         []string{npnProto},
	})
	if err != nil {
		t.Fatal(err)
	}

	return &serverTester{
		t:  t,
		ts: ts,
		cc: cc,
		fr: NewFramer(cc, cc),
	}
}

func (st *serverTester) Close() {
	st.ts.Close()
	st.cc.Close()
}

func (st *serverTester) writePreface() {
	n, err := st.cc.Write(clientPreface)
	if err != nil {
		st.t.Fatalf("Error writing client preface: %v", err)
	}
	if n != len(clientPreface) {
		st.t.Fatalf("Writing client preface, wrote %d bytes; want %d", n, len(clientPreface))
	}
}

func TestServer(t *testing.T) {
	st := newServerTester(t, func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Foo", "Bar")
	})
	defer st.Close()

	st.writePreface()

	fr := st.fr // temporary from here on
	if err := fr.WriteSettings(); err != nil {
		t.Fatal(err)
	}

	// Expect first Settings frame.
	{
		f, err := fr.ReadFrame()
		if err != nil {
			t.Fatal(err)
		}
		sf, ok := f.(*SettingsFrame)
		if !ok {
			t.Fatalf("Received a %T, not a Settings frame from the server", f)
		}
		sf.ForeachSetting(func(s Setting) {
			t.Logf("Server sent setting %v = %v", s.ID, s.Val)
		})
	}

	// And expect an ACK of our settings.
	{
		f, err := fr.ReadFrame()
		if err != nil {
			t.Fatal(err)
		}
		sf, ok := f.(*SettingsFrame)
		if !ok {
			t.Fatalf("Received a %T, not a Settings ack frame from the server", f)
		}
		if !sf.Header().Flags.Has(FlagSettingsAck) {
			t.Fatal("Settings Frame didn't have ACK set")
		}
	}

	// TODO: table-itize steps, write request (HEADERS frame), read response.
}

func mustWrite(t *testing.T, w io.Writer, p []byte) {
	n, err := w.Write(p)
	const maxLen = 80
	l := len(p)
	if len(p) > maxLen {
		p = p[:maxLen]
	}
	if err != nil {
		t.Fatalf("Error writing %d bytes (%q): %v", l, p, err)
	}
	if n != len(p) {
		t.Fatalf("Only wrote %d of %d bytes (%q)", n, l, p)
	}
}

func TestServerWithCurl(t *testing.T) {
	requireCurl(t)

	ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		// TODO: add a bunch of different tests with different
		// behavior, as a function of r or a table.
		// -- with request body, without.
		// -- no interaction with w.
		// -- panic
		// -- modify Header only, but no writes or writeheader (this test)
		// -- WriteHeader only
		// -- Write only
		// -- WriteString
		// -- both
		// -- huge headers over a frame size so we get continuation headers.
		// Look at net/http's Server tests for inspiration.
		w.Header().Set("Foo", "Bar")
	}))
	ConfigureServer(ts.Config, &Server{})
	ts.TLS = ts.Config.TLSConfig // the httptest.Server has its own copy of this TLS config
	ts.StartTLS()
	defer ts.Close()

	var gotConn int32
	testHookOnConn = func() { atomic.StoreInt32(&gotConn, 1) }

	t.Logf("Running test server for curl to hit at: %s", ts.URL)
	container := curl(t, "--silent", "--http2", "--insecure", "-v", ts.URL)
	defer kill(container)
	resc := make(chan interface{}, 1)
	go func() {
		res, err := dockerLogs(container)
		if err != nil {
			resc <- err
		} else {
			resc <- res
		}
	}()
	select {
	case res := <-resc:
		if err, ok := res.(error); ok {
			t.Fatal(err)
		}
		if !strings.Contains(string(res.([]byte)), "< foo:Bar") {
			t.Errorf("didn't see foo:Bar header")
			t.Logf("Got: %s", res)
		}
	case <-time.After(3 * time.Second):
		t.Errorf("timeout waiting for curl")
	}

	if atomic.LoadInt32(&gotConn) == 0 {
		t.Error("never saw an http2 connection")
	}
}

func dockerLogs(container string) ([]byte, error) {
	out, err := exec.Command("docker", "wait", container).CombinedOutput()
	if err != nil {
		return out, err
	}
	exitStatus, err := strconv.Atoi(strings.TrimSpace(string(out)))
	if err != nil {
		return out, errors.New("unexpected exit status from docker wait")
	}
	out, err = exec.Command("docker", "logs", container).CombinedOutput()
	exec.Command("docker", "rm", container).Run()
	if err == nil && exitStatus != 0 {
		err = fmt.Errorf("exit status %d", exitStatus)
	}
	return out, err
}

func kill(container string) {
	exec.Command("docker", "kill", container).Run()
	exec.Command("docker", "rm", container).Run()
}

// Verify that curl has http2.
func requireCurl(t *testing.T) {
	out, err := dockerLogs(curl(t, "--version"))
	if err != nil {
		t.Skipf("failed to determine curl features; skipping test")
	}
	if !strings.Contains(string(out), "HTTP2") {
		t.Skip("curl doesn't support HTTP2; skipping test")
	}
}

func curl(t *testing.T, args ...string) (container string) {
	out, err := exec.Command("docker", append([]string{"run", "-d", "--net=host", "gohttp2/curl"}, args...)...).CombinedOutput()
	if err != nil {
		t.Skipf("Failed to run curl in docker: %v, %s", err, out)
	}
	return strings.TrimSpace(string(out))
}
<|endoftext|>