<commit_before>// Copyright 2014 Jonathan Picques. All rights reserved.
// Use of this source code is governed by a MIT license
// The license can be found in the LICENSE file.

// The GoCSV package aims to provide easy CSV serialization and deserialization to the golang programming language

package gocsv

import (
	"bytes"
	"encoding/csv"
	"fmt"
	"io"
	"os"
	"reflect"
	"strings"
)

// FailIfUnmatchedStructTags indicates whether it is considered an error when there is an unmatched
// struct tag.
var FailIfUnmatchedStructTags = false

// FailIfDoubleHeaderNames indicates whether it is considered an error when a header name is repeated
// in the csv header.
var FailIfDoubleHeaderNames = false

// ShouldAlignDuplicateHeadersWithStructFieldOrder indicates whether we should align duplicate CSV
// headers per their alignment in the struct definition.
var ShouldAlignDuplicateHeadersWithStructFieldOrder = false

// TagSeparator defines seperator string for multiple csv tags in struct fields
var TagSeparator = ","

// Normalizer is a function that takes and returns a string. It is applied to
// struct and header field values before they are compared. It can be used to alter
// names for comparison. For instance, you could allow case insensitive matching
// or convert '-' to '_'.
type Normalizer func(string) string

type ErrorHandler func(*csv.ParseError) bool

// normalizeName function initially set to a nop Normalizer.
var normalizeName = DefaultNameNormalizer()

// DefaultNameNormalizer is a nop Normalizer.
func DefaultNameNormalizer() Normalizer { return func(s string) string { return s } }

// SetHeaderNormalizer sets the normalizer used to normalize struct and header field names.
func SetHeaderNormalizer(f Normalizer) { normalizeName = f }

// --------------------------------------------------------------------------
// CSVWriter used to format CSV

var selfCSVWriter = DefaultCSVWriter

// DefaultCSVWriter is the default SafeCSVWriter used to format CSV (cf. csv.NewWriter)
func DefaultCSVWriter(out io.Writer) *SafeCSVWriter {
	writer := NewSafeCSVWriter(csv.NewWriter(out))

	// As only one rune can be defined as a CSV separator, we are going to trim
	// the custom tag separator and use the first rune.
	if runes := []rune(strings.TrimSpace(TagSeparator)); len(runes) > 0 {
		writer.Comma = runes[0]
	}

	return writer
}

// SetCSVWriter sets the SafeCSVWriter used to format CSV.
func SetCSVWriter(csvWriter func(io.Writer) *SafeCSVWriter) {
	selfCSVWriter = csvWriter
}

func getCSVWriter(out io.Writer) *SafeCSVWriter {
	return selfCSVWriter(out)
}

// --------------------------------------------------------------------------
// CSVReader used to parse CSV

var selfCSVReader = DefaultCSVReader

// DefaultCSVReader is the default CSV reader used to parse CSV (cf. csv.NewReader)
func DefaultCSVReader(in io.Reader) CSVReader {
	return csv.NewReader(in)
}

// LazyCSVReader returns a lazy CSV reader, with LazyQuotes and TrimLeadingSpace.
func LazyCSVReader(in io.Reader) CSVReader {
	csvReader := csv.NewReader(in)
	csvReader.LazyQuotes = true
	csvReader.TrimLeadingSpace = true
	return csvReader
}

// SetCSVReader sets the CSV reader used to parse CSV.
func SetCSVReader(csvReader func(io.Reader) CSVReader) {
	selfCSVReader = csvReader
}

func getCSVReader(in io.Reader) CSVReader {
	return selfCSVReader(in)
}

// --------------------------------------------------------------------------
// Marshal functions

// MarshalFile saves the interface as CSV in the file.
func MarshalFile(in interface{}, file *os.File) (err error) {
	return Marshal(in, file)
}

// MarshalString returns the CSV string from the interface.
func MarshalString(in interface{}) (out string, err error) {
	bufferString := bytes.NewBufferString(out)
	if err := Marshal(in, bufferString); err != nil {
		return "", err
	}
	return bufferString.String(), nil
}

// MarshalBytes returns the CSV bytes from the interface.
func MarshalBytes(in interface{}) (out []byte, err error) {
	bufferString := bytes.NewBuffer(out)
	if err := Marshal(in, bufferString); err != nil {
		return nil, err
	}
	return bufferString.Bytes(), nil
}

// Marshal returns the CSV in writer from the interface.
func Marshal(in interface{}, out io.Writer) (err error) {
	writer := getCSVWriter(out)
	return writeTo(writer, in, false)
}

// MarshalWithoutHeaders returns the CSV in writer from the interface.
func MarshalWithoutHeaders(in interface{}, out io.Writer) (err error) {
	writer := getCSVWriter(out)
	return writeTo(writer, in, true)
}

// MarshalChan returns the CSV read from the channel.
func MarshalChan(c <-chan interface{}, out *SafeCSVWriter) error {
	return writeFromChan(out, c)
}

// MarshalCSV returns the CSV in writer from the interface.
func MarshalCSV(in interface{}, out *SafeCSVWriter) (err error) {
	return writeTo(out, in, false)
}

// MarshalCSVWithoutHeaders returns the CSV in writer from the interface.
func MarshalCSVWithoutHeaders(in interface{}, out *SafeCSVWriter) (err error) {
	return writeTo(out, in, true)
}

// --------------------------------------------------------------------------
// Unmarshal functions

// UnmarshalFile parses the CSV from the file in the interface.
func UnmarshalFile(in *os.File, out interface{}) error {
	return Unmarshal(in, out)
}

// UnmarshalFile parses the CSV from the file in the interface.
func UnmarshalFileWithErrorHandler(in *os.File, errHandler ErrorHandler, out interface{}) error {
	return UnmarshalWithErrorHandler(in, errHandler, out)
}

// UnmarshalString parses the CSV from the string in the interface.
func UnmarshalString(in string, out interface{}) error {
	return Unmarshal(strings.NewReader(in), out)
}

// UnmarshalBytes parses the CSV from the bytes in the interface.
func UnmarshalBytes(in []byte, out interface{}) error {
	return Unmarshal(bytes.NewReader(in), out)
}

// Unmarshal parses the CSV from the reader in the interface.
func Unmarshal(in io.Reader, out interface{}) error {
	return readTo(newSimpleDecoderFromReader(in), out)
}

// Unmarshal parses the CSV from the reader in the interface.
func UnmarshalWithErrorHandler(in io.Reader, errHandle ErrorHandler, out interface{}) error {
	return readToWithErrorHandler(newSimpleDecoderFromReader(in), errHandle, out)
}

// UnmarshalWithoutHeaders parses the CSV from the reader in the interface.
func UnmarshalWithoutHeaders(in io.Reader, out interface{}) error {
	return readToWithoutHeaders(newSimpleDecoderFromReader(in), out)
}

// UnmarshalCSVWithoutHeaders parses a headerless CSV with passed in CSV reader
func UnmarshalCSVWithoutHeaders(in CSVReader, out interface{}) error {
	return readToWithoutHeaders(csvDecoder{in}, out)
}

// UnmarshalDecoder parses the CSV from the decoder in the interface
func UnmarshalDecoder(in Decoder, out interface{}) error {
	return readTo(in, out)
}

// UnmarshalCSV parses the CSV from the reader in the interface.
func UnmarshalCSV(in CSVReader, out interface{}) error {
	return readTo(csvDecoder{in}, out)
}

// UnmarshalToChan parses the CSV from the reader and send each value in the chan c.
// The channel must have a concrete type.
func UnmarshalToChan(in io.Reader, c interface{}) error {
	if c == nil {
		return fmt.Errorf("goscv: channel is %v", c)
	}
	return readEach(newSimpleDecoderFromReader(in), c)
}

// UnmarshalToChanWithoutHeaders parses the CSV from the reader and send each value in the chan c.
// The channel must have a concrete type.
func UnmarshalToChanWithoutHeaders(in io.Reader, c interface{}) error {
	if c == nil {
		return fmt.Errorf("goscv: channel is %v", c)
	}
	return readEachWithoutHeaders(newSimpleDecoderFromReader(in), c)
}

// UnmarshalDecoderToChan parses the CSV from the decoder and send each value in the chan c.
// The channel must have a concrete type.
func UnmarshalDecoderToChan(in SimpleDecoder, c interface{}) error {
	if c == nil {
		return fmt.Errorf("goscv: channel is %v", c)
	}
	return readEach(in, c)
}

// UnmarshalStringToChan parses the CSV from the string and send each value in the chan c.
// The channel must have a concrete type.
func UnmarshalStringToChan(in string, c interface{}) error {
	return UnmarshalToChan(strings.NewReader(in), c)
}

// UnmarshalBytesToChan parses the CSV from the bytes and send each value in the chan c.
// The channel must have a concrete type.
func UnmarshalBytesToChan(in []byte, c interface{}) error {
	return UnmarshalToChan(bytes.NewReader(in), c)
}

// UnmarshalToCallback parses the CSV from the reader and send each value to the given func f.
// The func must look like func(Struct).
func UnmarshalToCallback(in io.Reader, f interface{}) error {
	valueFunc := reflect.ValueOf(f)
	t := reflect.TypeOf(f)
	if t.NumIn() != 1 {
		return fmt.Errorf("the given function must have exactly one parameter")
	}
	cerr := make(chan error)
	c := reflect.MakeChan(reflect.ChanOf(reflect.BothDir, t.In(0)), 0)
	go func() {
		cerr <- UnmarshalToChan(in, c.Interface())
	}()
	for {
		select {
		case err := <-cerr:
			return err
		default:
		}
		v, notClosed := c.Recv()
		if !notClosed || v.Interface() == nil {
			break
		}
		valueFunc.Call([]reflect.Value{v})
	}
	return nil
}

// UnmarshalToCallbackWithError parses the CSV from the reader and
// send each value to the given func f.
//
// If func returns error, it will stop processing, drain the
// parser and propagate the error to caller.
//
// The func must look like func(Struct) error.
func UnmarshalBytesToCallbackWithError(in []byte, f interface{}) error {
	valueFunc := reflect.ValueOf(f)
	t := reflect.TypeOf(f)
	if t.NumIn() != 1 {
		return fmt.Errorf("the given function must have exactly one parameter")
	}
	if t.NumOut() != 1 {
		return fmt.Errorf("the given function must have exactly one return value")
	}
	if !isErrorType(t.Out(0)) {
		return fmt.Errorf("the given function must only return error.")
	}

	cerr := make(chan error)
	c := reflect.MakeChan(reflect.ChanOf(reflect.BothDir, t.In(0)), 0)
	go func() {
		cerr <- UnmarshalBytesToChan(in, c.Interface())
	}()

	var fErr error
	for {
		select {
		case err := <-cerr:
			if err != nil {
				return err
			}
			return fErr
		default:
		}
		v, notClosed := c.Recv()
		if !notClosed || v.Interface() == nil {
			break
		}

		// callback f has already returned an error, stop processing but keep draining the chan c
		if fErr != nil {
			continue
		}

		results := valueFunc.Call([]reflect.Value{v})

		// If the callback f returns an error, stores it and returns it in future.
		errValue := results[0].Interface()
		if errValue != nil {
			fErr = errValue.(error)
		}
	}
	return fErr
}

// UnmarshalDecoderToCallback parses the CSV from the decoder and send each value to the given func f.
// The func must look like func(Struct).
func UnmarshalDecoderToCallback(in SimpleDecoder, f interface{}) error {
	valueFunc := reflect.ValueOf(f)
	t := reflect.TypeOf(f)
	if t.NumIn() != 1 {
		return fmt.Errorf("the given function must have exactly one parameter")
	}
	cerr := make(chan error)
	c := reflect.MakeChan(reflect.ChanOf(reflect.BothDir, t.In(0)), 0)
	go func() {
		cerr <- UnmarshalDecoderToChan(in, c.Interface())
	}()
	for {
		select {
		case err := <-cerr:
			return err
		default:
		}
		v, notClosed := c.Recv()
		if !notClosed || v.Interface() == nil {
			break
		}
		valueFunc.Call([]reflect.Value{v})
	}
	return nil
}

// UnmarshalBytesToCallback parses the CSV from the bytes and send each value to the given func f.
// The func must look like func(Struct).
func UnmarshalBytesToCallback(in []byte, f interface{}) error {
	return UnmarshalToCallback(bytes.NewReader(in), f)
}

// UnmarshalStringToCallback parses the CSV from the string and send each value to the given func f.
// The func must look like func(Struct).
func UnmarshalStringToCallback(in string, c interface{}) (err error) {
	return UnmarshalToCallback(strings.NewReader(in), c)
}

// CSVToMap creates a simple map from a CSV of 2 columns.
func CSVToMap(in io.Reader) (map[string]string, error) {
	decoder := newSimpleDecoderFromReader(in)
	header, err := decoder.getCSVRow()
	if err != nil {
		return nil, err
	}
	if len(header) != 2 {
		return nil, fmt.Errorf("maps can only be created for csv of two columns")
	}
	m := make(map[string]string)
	for {
		line, err := decoder.getCSVRow()
		if err == io.EOF {
			break
		} else if err != nil {
			return nil, err
		}
		m[line[0]] = line[1]
	}
	return m, nil
}

// CSVToMaps takes a reader and returns an array of dictionaries, using the header row as the keys
func CSVToMaps(reader io.Reader) ([]map[string]string, error) {
	r := csv.NewReader(reader)
	rows := []map[string]string{}
	var header []string
	for {
		record, err := r.Read()
		if err == io.EOF {
			break
		}
		if err != nil {
			return nil, err
		}
		if header == nil {
			header = record
		} else {
			dict := map[string]string{}
			for i := range header {
				dict[header[i]] = record[i]
			}
			rows = append(rows, dict)
		}
	}
	return rows, nil
}
<commit_msg>Define the following 3 functions as a group:  - UnmarshalToCallbackWithError  - UnmarshalBytesToCallbackWithError  - UnmarshalStringToCallbackWithError<commit_after>// Copyright 2014 Jonathan Picques. All rights reserved.
// Use of this source code is governed by a MIT license
// The license can be found in the LICENSE file.

// The GoCSV package aims to provide easy CSV serialization and deserialization to the golang programming language

package gocsv

import (
	"bytes"
	"encoding/csv"
	"fmt"
	"io"
	"os"
	"reflect"
	"strings"
)

// FailIfUnmatchedStructTags indicates whether it is considered an error when there is an unmatched
// struct tag.
var FailIfUnmatchedStructTags = false

// FailIfDoubleHeaderNames indicates whether it is considered an error when a header name is repeated
// in the csv header.
var FailIfDoubleHeaderNames = false

// ShouldAlignDuplicateHeadersWithStructFieldOrder indicates whether we should align duplicate CSV
// headers per their alignment in the struct definition.
var ShouldAlignDuplicateHeadersWithStructFieldOrder = false

// TagSeparator defines seperator string for multiple csv tags in struct fields
var TagSeparator = ","

// Normalizer is a function that takes and returns a string. It is applied to
// struct and header field values before they are compared. It can be used to alter
// names for comparison. For instance, you could allow case insensitive matching
// or convert '-' to '_'.
type Normalizer func(string) string

type ErrorHandler func(*csv.ParseError) bool

// normalizeName function initially set to a nop Normalizer.
var normalizeName = DefaultNameNormalizer()

// DefaultNameNormalizer is a nop Normalizer.
func DefaultNameNormalizer() Normalizer { return func(s string) string { return s } }

// SetHeaderNormalizer sets the normalizer used to normalize struct and header field names.
func SetHeaderNormalizer(f Normalizer) { normalizeName = f }

// --------------------------------------------------------------------------
// CSVWriter used to format CSV

var selfCSVWriter = DefaultCSVWriter

// DefaultCSVWriter is the default SafeCSVWriter used to format CSV (cf. csv.NewWriter)
func DefaultCSVWriter(out io.Writer) *SafeCSVWriter {
	writer := NewSafeCSVWriter(csv.NewWriter(out))

	// As only one rune can be defined as a CSV separator, we are going to trim
	// the custom tag separator and use the first rune.
	if runes := []rune(strings.TrimSpace(TagSeparator)); len(runes) > 0 {
		writer.Comma = runes[0]
	}

	return writer
}

// SetCSVWriter sets the SafeCSVWriter used to format CSV.
func SetCSVWriter(csvWriter func(io.Writer) *SafeCSVWriter) {
	selfCSVWriter = csvWriter
}

func getCSVWriter(out io.Writer) *SafeCSVWriter {
	return selfCSVWriter(out)
}

// --------------------------------------------------------------------------
// CSVReader used to parse CSV

var selfCSVReader = DefaultCSVReader

// DefaultCSVReader is the default CSV reader used to parse CSV (cf. csv.NewReader)
func DefaultCSVReader(in io.Reader) CSVReader {
	return csv.NewReader(in)
}

// LazyCSVReader returns a lazy CSV reader, with LazyQuotes and TrimLeadingSpace.
func LazyCSVReader(in io.Reader) CSVReader {
	csvReader := csv.NewReader(in)
	csvReader.LazyQuotes = true
	csvReader.TrimLeadingSpace = true
	return csvReader
}

// SetCSVReader sets the CSV reader used to parse CSV.
func SetCSVReader(csvReader func(io.Reader) CSVReader) {
	selfCSVReader = csvReader
}

func getCSVReader(in io.Reader) CSVReader {
	return selfCSVReader(in)
}

// --------------------------------------------------------------------------
// Marshal functions

// MarshalFile saves the interface as CSV in the file.
func MarshalFile(in interface{}, file *os.File) (err error) {
	return Marshal(in, file)
}

// MarshalString returns the CSV string from the interface.
func MarshalString(in interface{}) (out string, err error) {
	bufferString := bytes.NewBufferString(out)
	if err := Marshal(in, bufferString); err != nil {
		return "", err
	}
	return bufferString.String(), nil
}

// MarshalBytes returns the CSV bytes from the interface.
func MarshalBytes(in interface{}) (out []byte, err error) {
	bufferString := bytes.NewBuffer(out)
	if err := Marshal(in, bufferString); err != nil {
		return nil, err
	}
	return bufferString.Bytes(), nil
}

// Marshal returns the CSV in writer from the interface.
func Marshal(in interface{}, out io.Writer) (err error) {
	writer := getCSVWriter(out)
	return writeTo(writer, in, false)
}

// MarshalWithoutHeaders returns the CSV in writer from the interface.
func MarshalWithoutHeaders(in interface{}, out io.Writer) (err error) {
	writer := getCSVWriter(out)
	return writeTo(writer, in, true)
}

// MarshalChan returns the CSV read from the channel.
func MarshalChan(c <-chan interface{}, out *SafeCSVWriter) error {
	return writeFromChan(out, c)
}

// MarshalCSV returns the CSV in writer from the interface.
func MarshalCSV(in interface{}, out *SafeCSVWriter) (err error) {
	return writeTo(out, in, false)
}

// MarshalCSVWithoutHeaders returns the CSV in writer from the interface.
func MarshalCSVWithoutHeaders(in interface{}, out *SafeCSVWriter) (err error) {
	return writeTo(out, in, true)
}

// --------------------------------------------------------------------------
// Unmarshal functions

// UnmarshalFile parses the CSV from the file in the interface.
func UnmarshalFile(in *os.File, out interface{}) error {
	return Unmarshal(in, out)
}

// UnmarshalFile parses the CSV from the file in the interface.
func UnmarshalFileWithErrorHandler(in *os.File, errHandler ErrorHandler, out interface{}) error {
	return UnmarshalWithErrorHandler(in, errHandler, out)
}

// UnmarshalString parses the CSV from the string in the interface.
func UnmarshalString(in string, out interface{}) error {
	return Unmarshal(strings.NewReader(in), out)
}

// UnmarshalBytes parses the CSV from the bytes in the interface.
func UnmarshalBytes(in []byte, out interface{}) error {
	return Unmarshal(bytes.NewReader(in), out)
}

// Unmarshal parses the CSV from the reader in the interface.
func Unmarshal(in io.Reader, out interface{}) error {
	return readTo(newSimpleDecoderFromReader(in), out)
}

// Unmarshal parses the CSV from the reader in the interface.
func UnmarshalWithErrorHandler(in io.Reader, errHandle ErrorHandler, out interface{}) error {
	return readToWithErrorHandler(newSimpleDecoderFromReader(in), errHandle, out)
}

// UnmarshalWithoutHeaders parses the CSV from the reader in the interface.
func UnmarshalWithoutHeaders(in io.Reader, out interface{}) error {
	return readToWithoutHeaders(newSimpleDecoderFromReader(in), out)
}

// UnmarshalCSVWithoutHeaders parses a headerless CSV with passed in CSV reader
func UnmarshalCSVWithoutHeaders(in CSVReader, out interface{}) error {
	return readToWithoutHeaders(csvDecoder{in}, out)
}

// UnmarshalDecoder parses the CSV from the decoder in the interface
func UnmarshalDecoder(in Decoder, out interface{}) error {
	return readTo(in, out)
}

// UnmarshalCSV parses the CSV from the reader in the interface.
func UnmarshalCSV(in CSVReader, out interface{}) error {
	return readTo(csvDecoder{in}, out)
}

// UnmarshalToChan parses the CSV from the reader and send each value in the chan c.
// The channel must have a concrete type.
func UnmarshalToChan(in io.Reader, c interface{}) error {
	if c == nil {
		return fmt.Errorf("goscv: channel is %v", c)
	}
	return readEach(newSimpleDecoderFromReader(in), c)
}

// UnmarshalToChanWithoutHeaders parses the CSV from the reader and send each value in the chan c.
// The channel must have a concrete type.
func UnmarshalToChanWithoutHeaders(in io.Reader, c interface{}) error {
	if c == nil {
		return fmt.Errorf("goscv: channel is %v", c)
	}
	return readEachWithoutHeaders(newSimpleDecoderFromReader(in), c)
}

// UnmarshalDecoderToChan parses the CSV from the decoder and send each value in the chan c.
// The channel must have a concrete type.
func UnmarshalDecoderToChan(in SimpleDecoder, c interface{}) error {
	if c == nil {
		return fmt.Errorf("goscv: channel is %v", c)
	}
	return readEach(in, c)
}

// UnmarshalStringToChan parses the CSV from the string and send each value in the chan c.
// The channel must have a concrete type.
func UnmarshalStringToChan(in string, c interface{}) error {
	return UnmarshalToChan(strings.NewReader(in), c)
}

// UnmarshalBytesToChan parses the CSV from the bytes and send each value in the chan c.
// The channel must have a concrete type.
func UnmarshalBytesToChan(in []byte, c interface{}) error {
	return UnmarshalToChan(bytes.NewReader(in), c)
}

// UnmarshalToCallback parses the CSV from the reader and send each value to the given func f.
// The func must look like func(Struct).
func UnmarshalToCallback(in io.Reader, f interface{}) error {
	valueFunc := reflect.ValueOf(f)
	t := reflect.TypeOf(f)
	if t.NumIn() != 1 {
		return fmt.Errorf("the given function must have exactly one parameter")
	}
	cerr := make(chan error)
	c := reflect.MakeChan(reflect.ChanOf(reflect.BothDir, t.In(0)), 0)
	go func() {
		cerr <- UnmarshalToChan(in, c.Interface())
	}()
	for {
		select {
		case err := <-cerr:
			return err
		default:
		}
		v, notClosed := c.Recv()
		if !notClosed || v.Interface() == nil {
			break
		}
		valueFunc.Call([]reflect.Value{v})
	}
	return nil
}

// UnmarshalDecoderToCallback parses the CSV from the decoder and send each value to the given func f.
// The func must look like func(Struct).
func UnmarshalDecoderToCallback(in SimpleDecoder, f interface{}) error {
	valueFunc := reflect.ValueOf(f)
	t := reflect.TypeOf(f)
	if t.NumIn() != 1 {
		return fmt.Errorf("the given function must have exactly one parameter")
	}
	cerr := make(chan error)
	c := reflect.MakeChan(reflect.ChanOf(reflect.BothDir, t.In(0)), 0)
	go func() {
		cerr <- UnmarshalDecoderToChan(in, c.Interface())
	}()
	for {
		select {
		case err := <-cerr:
			return err
		default:
		}
		v, notClosed := c.Recv()
		if !notClosed || v.Interface() == nil {
			break
		}
		valueFunc.Call([]reflect.Value{v})
	}
	return nil
}

// UnmarshalBytesToCallback parses the CSV from the bytes and send each value to the given func f.
// The func must look like func(Struct).
func UnmarshalBytesToCallback(in []byte, f interface{}) error {
	return UnmarshalToCallback(bytes.NewReader(in), f)
}

// UnmarshalStringToCallback parses the CSV from the string and send each value to the given func f.
// The func must look like func(Struct).
func UnmarshalStringToCallback(in string, c interface{}) (err error) {
	return UnmarshalToCallback(strings.NewReader(in), c)
}

// UnmarshalToCallbackWithError parses the CSV from the reader and
// send each value to the given func f.
//
// If func returns error, it will stop processing, drain the
// parser and propagate the error to caller.
//
// The func must look like func(Struct) error.
func UnmarshalToCallbackWithError(in io.Reader, f interface{}) error {
	valueFunc := reflect.ValueOf(f)
	t := reflect.TypeOf(f)
	if t.NumIn() != 1 {
		return fmt.Errorf("the given function must have exactly one parameter")
	}
	if t.NumOut() != 1 {
		return fmt.Errorf("the given function must have exactly one return value")
	}
	if !isErrorType(t.Out(0)) {
		return fmt.Errorf("the given function must only return error.")
	}

	cerr := make(chan error)
	c := reflect.MakeChan(reflect.ChanOf(reflect.BothDir, t.In(0)), 0)
	go func() {
		cerr <- UnmarshalToChan(in, c.Interface())
	}()

	var fErr error
	for {
		select {
		case err := <-cerr:
			if err != nil {
				return err
			}
			return fErr
		default:
		}
		v, notClosed := c.Recv()
		if !notClosed || v.Interface() == nil {
			break
		}

		// callback f has already returned an error, stop processing but keep draining the chan c
		if fErr != nil {
			continue
		}

		results := valueFunc.Call([]reflect.Value{v})

		// If the callback f returns an error, stores it and returns it in future.
		errValue := results[0].Interface()
		if errValue != nil {
			fErr = errValue.(error)
		}
	}
	return fErr
}

// UnmarshalBytesToCallbackWithError parses the CSV from the bytes and
// send each value to the given func f.
//
// If func returns error, it will stop processing, drain the
// parser and propagate the error to caller.
//
// The func must look like func(Struct) error.
func UnmarshalBytesToCallbackWithError(in []byte, f interface{}) error {
	return UnmarshalToCallbackWithError(bytes.NewReader(in), f)
}

// UnmarshalStringToCallbackWithError parses the CSV from the string and
// send each value to the given func f.
//
// If func returns error, it will stop processing, drain the
// parser and propagate the error to caller.
//
// The func must look like func(Struct) error.
func UnmarshalStringToCallbackWithError(in string, c interface{}) (err error) {
	return UnmarshalToCallbackWithError(strings.NewReader(in), c)
}

// CSVToMap creates a simple map from a CSV of 2 columns.
func CSVToMap(in io.Reader) (map[string]string, error) {
	decoder := newSimpleDecoderFromReader(in)
	header, err := decoder.getCSVRow()
	if err != nil {
		return nil, err
	}
	if len(header) != 2 {
		return nil, fmt.Errorf("maps can only be created for csv of two columns")
	}
	m := make(map[string]string)
	for {
		line, err := decoder.getCSVRow()
		if err == io.EOF {
			break
		} else if err != nil {
			return nil, err
		}
		m[line[0]] = line[1]
	}
	return m, nil
}

// CSVToMaps takes a reader and returns an array of dictionaries, using the header row as the keys
func CSVToMaps(reader io.Reader) ([]map[string]string, error) {
	r := csv.NewReader(reader)
	rows := []map[string]string{}
	var header []string
	for {
		record, err := r.Read()
		if err == io.EOF {
			break
		}
		if err != nil {
			return nil, err
		}
		if header == nil {
			header = record
		} else {
			dict := map[string]string{}
			for i := range header {
				dict[header[i]] = record[i]
			}
			rows = append(rows, dict)
		}
	}
	return rows, nil
}
<|endoftext|>


<commit_before>package revel

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"mime"
	"mime/multipart"
	"net/http"
	"net/http/cookiejar"
	"net/textproto"
	"net/url"
	"os"
	"path/filepath"
	"regexp"
	"strings"

	"code.google.com/p/go.net/websocket"
)

type TestSuite struct {
	Client       *http.Client
	Response     *http.Response
	ResponseBody []byte
	Session      Session
}

type TestRequest struct {
	*http.Request
	testSuite *TestSuite
}

var TestSuites []interface{} // Array of structs that embed TestSuite

// NewTestSuite returns an initialized TestSuite ready for use. It is invoked
// by the test harness to initialize the embedded field in application tests.
func NewTestSuite() TestSuite {
	jar, _ := cookiejar.New(nil)
	return TestSuite{
		Client:  &http.Client{Jar: jar},
		Session: make(Session),
	}
}

// Return the address and port of the server, e.g. "127.0.0.1:8557"
func (t *TestSuite) Host() string {
	if Server.Addr[0] == ':' {
		return "127.0.0.1" + Server.Addr
	}
	return Server.Addr
}

// Return the base http/https URL of the server, e.g. "http://127.0.0.1:8557".
// The scheme is set to https if http.ssl is set to true in the configuration file.
func (t *TestSuite) BaseUrl() string {
	if HttpSsl {
		return "https://" + t.Host()
	} else {
		return "http://" + t.Host()
	}
}

// Return the base websocket URL of the server, e.g. "ws://127.0.0.1:8557"
func (t *TestSuite) WebSocketUrl() string {
	return "ws://" + t.Host()
}

// Issue a GET request to the given path and store the result in Response and
// ResponseBody.
func (t *TestSuite) Get(path string) {
	t.GetCustom(t.BaseUrl() + path).Send()
}

// Return a GET request to the given uri in a form of its wrapper.
func (t *TestSuite) GetCustom(uri string) *TestRequest {
	req, err := http.NewRequest("GET", uri, nil)
	if err != nil {
		panic(err)
	}
	return &TestRequest{
		Request:   req,
		testSuite: t,
	}
}

// Issue a DELETE request to the given path and store the result in Response and
// ResponseBody.
func (t *TestSuite) Delete(path string) {
	t.DeleteCustom(t.BaseUrl() + path).Send()
}

// Return a DELETE request to the given uri in a form of its wrapper.
func (t *TestSuite) DeleteCustom(uri string) *TestRequest {
	req, err := http.NewRequest("DELETE", uri, nil)
	if err != nil {
		panic(err)
	}
	return &TestRequest{
		Request:   req,
		testSuite: t,
	}
}

// Issue a POST request to the given path, sending the given Content-Type and
// data, and store the result in Response and ResponseBody.  "data" may be nil.
func (t *TestSuite) Post(path string, contentType string, reader io.Reader) {
	t.PostCustom(t.BaseUrl()+path, contentType, reader).Send()
}

// Return a POST request to the given uri with specified Content-Type and data
// in a form of wrapper. "data" may be nil.
func (t *TestSuite) PostCustom(uri string, contentType string, reader io.Reader) *TestRequest {
	req, err := http.NewRequest("POST", uri, reader)
	if err != nil {
		panic(err)
	}
	req.Header.Set("Content-Type", contentType)
	return &TestRequest{
		Request:   req,
		testSuite: t,
	}
}

// Issue a POST request to the given path as a form post of the given key and
// values, and store the result in Response and ResponseBody.
func (t *TestSuite) PostForm(path string, data url.Values) {
	t.PostFormCustom(t.BaseUrl()+path, data).Send()
}

// Return a POST request to the given uri as a form post of the given key and values.
// The request is in a form of TestRequest wrapper.
func (t *TestSuite) PostFormCustom(uri string, data url.Values) *TestRequest {
	return t.PostCustom(uri, "application/x-www-form-urlencoded", strings.NewReader(data.Encode()))
}

// Issue a multipart request to the given path sending given params and files,
// and store the result in Response and ResponseBody.
func (t *TestSuite) PostFile(path string, params url.Values, filePaths url.Values) {
	t.PostFileCustom(t.BaseUrl()+path, params, filePaths).Send()
}

// Return a multipart request to the given uri in a form of its wrapper
// with the given params and files.
func (t *TestSuite) PostFileCustom(uri string, params url.Values, filePaths url.Values) *TestRequest {
	body := &bytes.Buffer{}
	writer := multipart.NewWriter(body)

	for key, values := range filePaths {
		for _, value := range values {
			createFormFile(writer, key, value)
		}
	}

	for key, values := range params {
		for _, value := range values {
			err := writer.WriteField(key, value)
			t.AssertEqual(nil, err)
		}
	}
	err := writer.Close()
	t.AssertEqual(nil, err)

	return t.PostCustom(uri, writer.FormDataContentType(), body)
}

// Issue any request and read the response. If successful, the caller may
// examine the Response and ResponseBody properties. Session data will be
// added to the request cookies for you.
func (r *TestRequest) Send() {
	r.AddCookie(r.testSuite.Session.cookie())
	r.MakeRequest()
}

// Issue any request and read the response. If successful, the caller may
// examine the Response and ResponseBody properties. You will need to
// manage session / cookie data manually
func (r *TestRequest) MakeRequest() {
	var err error
	if r.testSuite.Response, err = r.testSuite.Client.Do(r.Request); err != nil {
		panic(err)
	}
	if r.testSuite.ResponseBody, err = ioutil.ReadAll(r.testSuite.Response.Body); err != nil {
		panic(err)
	}

	// Look for a session cookie in the response and parse it.
	sessionCookieName := r.testSuite.Session.cookie().Name
	for _, cookie := range r.testSuite.Client.Jar.Cookies(r.Request.URL) {
		if cookie.Name == sessionCookieName {
			r.testSuite.Session = getSessionFromCookie(cookie)
			break
		}
	}
}

// Create a websocket connection to the given path and return the connection
func (t *TestSuite) WebSocket(path string) *websocket.Conn {
	origin := t.BaseUrl() + "/"
	url := t.WebSocketUrl() + path
	ws, err := websocket.Dial(url, "", origin)
	if err != nil {
		panic(err)
	}
	return ws
}

func (t *TestSuite) AssertOk() {
	t.AssertStatus(http.StatusOK)
}

func (t *TestSuite) AssertNotFound() {
	t.AssertStatus(http.StatusNotFound)
}

func (t *TestSuite) AssertStatus(status int) {
	if t.Response.StatusCode != status {
		panic(fmt.Errorf("Status: (expected) %d != %d (actual)", status, t.Response.StatusCode))
	}
}

func (t *TestSuite) AssertContentType(contentType string) {
	t.AssertHeader("Content-Type", contentType)
}

func (t *TestSuite) AssertHeader(name, value string) {
	actual := t.Response.Header.Get(name)
	if actual != value {
		panic(fmt.Errorf("Header %s: (expected) %s != %s (actual)", name, value, actual))
	}
}

func (t *TestSuite) AssertEqual(expected, actual interface{}) {
	if !Equal(expected, actual) {
		panic(fmt.Errorf("(expected) %v != %v (actual)", expected, actual))
	}
}

func (t *TestSuite) AssertNotEqual(expected, actual interface{}) {
	if Equal(expected, actual) {
		panic(fmt.Errorf("(expected) %v == %v (actual)", expected, actual))
	}
}

func (t *TestSuite) Assert(exp bool) {
	t.Assertf(exp, "Assertion failed")
}

func (t *TestSuite) Assertf(exp bool, formatStr string, args ...interface{}) {
	if !exp {
		panic(fmt.Errorf(formatStr, args...))
	}
}

// Assert that the response contains the given string.
func (t *TestSuite) AssertContains(s string) {
	if !bytes.Contains(t.ResponseBody, []byte(s)) {
		panic(fmt.Errorf("Assertion failed. Expected response to contain %s", s))
	}
}

// Assert that the response does not contain the given string.
func (t *TestSuite) AssertNotContains(s string) {
	if bytes.Contains(t.ResponseBody, []byte(s)) {
		panic(fmt.Errorf("Assertion failed. Expected response not to contain %s", s))
	}
}

// Assert that the response matches the given regular expression.
func (t *TestSuite) AssertContainsRegex(regex string) {
	r := regexp.MustCompile(regex)

	if !r.Match(t.ResponseBody) {
		panic(fmt.Errorf("Assertion failed. Expected response to match regexp %s", regex))
	}
}

func createFormFile(writer *multipart.Writer, fieldname, filename string) {
	// Try to open the file.
	file, err := os.Open(filename)
	if err != nil {
		panic(err)
	}
	defer file.Close()

	// Create a new form-data header with the provided field name and file name.
	// Determine Content-Type of the file by its extension.
	h := textproto.MIMEHeader{}
	h.Set("Content-Disposition", fmt.Sprintf(
		`form-data; name="%s"; filename="%s"`,
		escapeQuotes(fieldname),
		escapeQuotes(filepath.Base(filename)),
	))
	h.Set("Content-Type", "application/octet-stream")
	if ct := mime.TypeByExtension(filepath.Ext(filename)); ct != "" {
		h.Set("Content-Type", ct)
	}
	part, err := writer.CreatePart(h)
	if err != nil {
		panic(err)
	}

	// Copy the content of the file we have opened not reading the whole
	// file into memory.
	_, err = io.Copy(part, file)
	if err != nil {
		panic(err)
	}
}

var quoteEscaper = strings.NewReplacer("\\", "\\\\", `"`, "\\\"")

// This function was borrowed from mime/multipart package.
func escapeQuotes(s string) string {
	return quoteEscaper.Replace(s)
}
<commit_msg>Add testing support for HTTP PUT & PATCH<commit_after>package revel

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"mime"
	"mime/multipart"
	"net/http"
	"net/http/cookiejar"
	"net/textproto"
	"net/url"
	"os"
	"path/filepath"
	"regexp"
	"strings"

	"code.google.com/p/go.net/websocket"
)

type TestSuite struct {
	Client       *http.Client
	Response     *http.Response
	ResponseBody []byte
	Session      Session
}

type TestRequest struct {
	*http.Request
	testSuite *TestSuite
}

var TestSuites []interface{} // Array of structs that embed TestSuite

// NewTestSuite returns an initialized TestSuite ready for use. It is invoked
// by the test harness to initialize the embedded field in application tests.
func NewTestSuite() TestSuite {
	jar, _ := cookiejar.New(nil)
	return TestSuite{
		Client:  &http.Client{Jar: jar},
		Session: make(Session),
	}
}

// Return the address and port of the server, e.g. "127.0.0.1:8557"
func (t *TestSuite) Host() string {
	if Server.Addr[0] == ':' {
		return "127.0.0.1" + Server.Addr
	}
	return Server.Addr
}

// Return the base http/https URL of the server, e.g. "http://127.0.0.1:8557".
// The scheme is set to https if http.ssl is set to true in the configuration file.
func (t *TestSuite) BaseUrl() string {
	if HttpSsl {
		return "https://" + t.Host()
	} else {
		return "http://" + t.Host()
	}
}

// Return the base websocket URL of the server, e.g. "ws://127.0.0.1:8557"
func (t *TestSuite) WebSocketUrl() string {
	return "ws://" + t.Host()
}

// Issue a GET request to the given path and store the result in Response and
// ResponseBody.
func (t *TestSuite) Get(path string) {
	t.GetCustom(t.BaseUrl() + path).Send()
}

// Return a GET request to the given uri in a form of its wrapper.
func (t *TestSuite) GetCustom(uri string) *TestRequest {
	req, err := http.NewRequest("GET", uri, nil)
	if err != nil {
		panic(err)
	}
	return &TestRequest{
		Request:   req,
		testSuite: t,
	}
}

// Issue a DELETE request to the given path and store the result in Response and
// ResponseBody.
func (t *TestSuite) Delete(path string) {
	t.DeleteCustom(t.BaseUrl() + path).Send()
}

// Return a DELETE request to the given uri in a form of its wrapper.
func (t *TestSuite) DeleteCustom(uri string) *TestRequest {
	req, err := http.NewRequest("DELETE", uri, nil)
	if err != nil {
		panic(err)
	}
	return &TestRequest{
		Request:   req,
		testSuite: t,
	}
}

// Issue a PUT request to the given path, sending the given Content-Type and
// data, and store the result in Response and ResponseBody.  "data" may be nil.
func (t *TestSuite) Put(path string, contentType string, reader io.Reader) {
	t.PutCustom(t.BaseUrl()+path, contentType, reader).Send()
}

// Return a PUT request to the given uri with specified Content-Type and data
// in a form of wrapper. "data" may be nil.
func (t *TestSuite) PutCustom(uri string, contentType string, reader io.Reader) *TestRequest {
	req, err := http.NewRequest("PUT", uri, reader)
	if err != nil {
		panic(err)
	}
	req.Header.Set("Content-Type", contentType)
	return &TestRequest{
		Request:   req,
		testSuite: t,
	}
}

// Issue a PATCH request to the given path, sending the given Content-Type and
// data, and store the result in Response and ResponseBody.  "data" may be nil.
func (t *TestSuite) Patch(path string, contentType string, reader io.Reader) {
	t.PatchCustom(t.BaseUrl()+path, contentType, reader).Send()
}

// Return a PATCH request to the given uri with specified Content-Type and data
// in a form of wrapper. "data" may be nil.
func (t *TestSuite) PatchCustom(uri string, contentType string, reader io.Reader) *TestRequest {
	req, err := http.NewRequest("PATCH", uri, reader)
	if err != nil {
		panic(err)
	}
	req.Header.Set("Content-Type", contentType)
	return &TestRequest{
		Request:   req,
		testSuite: t,
	}
}

// Issue a POST request to the given path, sending the given Content-Type and
// data, and store the result in Response and ResponseBody.  "data" may be nil.
func (t *TestSuite) Post(path string, contentType string, reader io.Reader) {
	t.PostCustom(t.BaseUrl()+path, contentType, reader).Send()
}

// Return a POST request to the given uri with specified Content-Type and data
// in a form of wrapper. "data" may be nil.
func (t *TestSuite) PostCustom(uri string, contentType string, reader io.Reader) *TestRequest {
	req, err := http.NewRequest("POST", uri, reader)
	if err != nil {
		panic(err)
	}
	req.Header.Set("Content-Type", contentType)
	return &TestRequest{
		Request:   req,
		testSuite: t,
	}
}

// Issue a POST request to the given path as a form post of the given key and
// values, and store the result in Response and ResponseBody.
func (t *TestSuite) PostForm(path string, data url.Values) {
	t.PostFormCustom(t.BaseUrl()+path, data).Send()
}

// Return a POST request to the given uri as a form post of the given key and values.
// The request is in a form of TestRequest wrapper.
func (t *TestSuite) PostFormCustom(uri string, data url.Values) *TestRequest {
	return t.PostCustom(uri, "application/x-www-form-urlencoded", strings.NewReader(data.Encode()))
}

// Issue a multipart request to the given path sending given params and files,
// and store the result in Response and ResponseBody.
func (t *TestSuite) PostFile(path string, params url.Values, filePaths url.Values) {
	t.PostFileCustom(t.BaseUrl()+path, params, filePaths).Send()
}

// Return a multipart request to the given uri in a form of its wrapper
// with the given params and files.
func (t *TestSuite) PostFileCustom(uri string, params url.Values, filePaths url.Values) *TestRequest {
	body := &bytes.Buffer{}
	writer := multipart.NewWriter(body)

	for key, values := range filePaths {
		for _, value := range values {
			createFormFile(writer, key, value)
		}
	}

	for key, values := range params {
		for _, value := range values {
			err := writer.WriteField(key, value)
			t.AssertEqual(nil, err)
		}
	}
	err := writer.Close()
	t.AssertEqual(nil, err)

	return t.PostCustom(uri, writer.FormDataContentType(), body)
}

// Issue any request and read the response. If successful, the caller may
// examine the Response and ResponseBody properties. Session data will be
// added to the request cookies for you.
func (r *TestRequest) Send() {
	r.AddCookie(r.testSuite.Session.cookie())
	r.MakeRequest()
}

// Issue any request and read the response. If successful, the caller may
// examine the Response and ResponseBody properties. You will need to
// manage session / cookie data manually
func (r *TestRequest) MakeRequest() {
	var err error
	if r.testSuite.Response, err = r.testSuite.Client.Do(r.Request); err != nil {
		panic(err)
	}
	if r.testSuite.ResponseBody, err = ioutil.ReadAll(r.testSuite.Response.Body); err != nil {
		panic(err)
	}

	// Look for a session cookie in the response and parse it.
	sessionCookieName := r.testSuite.Session.cookie().Name
	for _, cookie := range r.testSuite.Client.Jar.Cookies(r.Request.URL) {
		if cookie.Name == sessionCookieName {
			r.testSuite.Session = getSessionFromCookie(cookie)
			break
		}
	}
}

// Create a websocket connection to the given path and return the connection
func (t *TestSuite) WebSocket(path string) *websocket.Conn {
	origin := t.BaseUrl() + "/"
	url := t.WebSocketUrl() + path
	ws, err := websocket.Dial(url, "", origin)
	if err != nil {
		panic(err)
	}
	return ws
}

func (t *TestSuite) AssertOk() {
	t.AssertStatus(http.StatusOK)
}

func (t *TestSuite) AssertNotFound() {
	t.AssertStatus(http.StatusNotFound)
}

func (t *TestSuite) AssertStatus(status int) {
	if t.Response.StatusCode != status {
		panic(fmt.Errorf("Status: (expected) %d != %d (actual)", status, t.Response.StatusCode))
	}
}

func (t *TestSuite) AssertContentType(contentType string) {
	t.AssertHeader("Content-Type", contentType)
}

func (t *TestSuite) AssertHeader(name, value string) {
	actual := t.Response.Header.Get(name)
	if actual != value {
		panic(fmt.Errorf("Header %s: (expected) %s != %s (actual)", name, value, actual))
	}
}

func (t *TestSuite) AssertEqual(expected, actual interface{}) {
	if !Equal(expected, actual) {
		panic(fmt.Errorf("(expected) %v != %v (actual)", expected, actual))
	}
}

func (t *TestSuite) AssertNotEqual(expected, actual interface{}) {
	if Equal(expected, actual) {
		panic(fmt.Errorf("(expected) %v == %v (actual)", expected, actual))
	}
}

func (t *TestSuite) Assert(exp bool) {
	t.Assertf(exp, "Assertion failed")
}

func (t *TestSuite) Assertf(exp bool, formatStr string, args ...interface{}) {
	if !exp {
		panic(fmt.Errorf(formatStr, args...))
	}
}

// Assert that the response contains the given string.
func (t *TestSuite) AssertContains(s string) {
	if !bytes.Contains(t.ResponseBody, []byte(s)) {
		panic(fmt.Errorf("Assertion failed. Expected response to contain %s", s))
	}
}

// Assert that the response does not contain the given string.
func (t *TestSuite) AssertNotContains(s string) {
	if bytes.Contains(t.ResponseBody, []byte(s)) {
		panic(fmt.Errorf("Assertion failed. Expected response not to contain %s", s))
	}
}

// Assert that the response matches the given regular expression.
func (t *TestSuite) AssertContainsRegex(regex string) {
	r := regexp.MustCompile(regex)

	if !r.Match(t.ResponseBody) {
		panic(fmt.Errorf("Assertion failed. Expected response to match regexp %s", regex))
	}
}

func createFormFile(writer *multipart.Writer, fieldname, filename string) {
	// Try to open the file.
	file, err := os.Open(filename)
	if err != nil {
		panic(err)
	}
	defer file.Close()

	// Create a new form-data header with the provided field name and file name.
	// Determine Content-Type of the file by its extension.
	h := textproto.MIMEHeader{}
	h.Set("Content-Disposition", fmt.Sprintf(
		`form-data; name="%s"; filename="%s"`,
		escapeQuotes(fieldname),
		escapeQuotes(filepath.Base(filename)),
	))
	h.Set("Content-Type", "application/octet-stream")
	if ct := mime.TypeByExtension(filepath.Ext(filename)); ct != "" {
		h.Set("Content-Type", ct)
	}
	part, err := writer.CreatePart(h)
	if err != nil {
		panic(err)
	}

	// Copy the content of the file we have opened not reading the whole
	// file into memory.
	_, err = io.Copy(part, file)
	if err != nil {
		panic(err)
	}
}

var quoteEscaper = strings.NewReplacer("\\", "\\\\", `"`, "\\\"")

// This function was borrowed from mime/multipart package.
func escapeQuotes(s string) string {
	return quoteEscaper.Replace(s)
}
<|endoftext|>


<commit_before>// go:generate go-bindata -o bindata.go template mapconfig
package main

import (
	"bytes"
	"database/sql"
	"fmt"
	"go/format"
	"log"
	"sort"
	"text/template"

	"github.com/BurntSushi/toml"
	"github.com/achiku/varfmt"
	"github.com/pkg/errors"
)

const pgLoadColumnDef = `
SELECT
    a.attnum AS field_ordinal,
    a.attname AS column_name,
    format_type(a.atttypid, a.atttypmod) AS data_type,
    a.attnotnull AS not_null,
    COALESCE(pg_get_expr(ad.adbin, ad.adrelid), '') AS default_value,
    COALESCE(ct.contype = 'p', false) AS  is_primary_key,
    CASE WHEN a.atttypid = ANY ('{int,int8,int2}'::regtype[])
      AND EXISTS (
         SELECT 1 FROM pg_attrdef ad
         WHERE  ad.adrelid = a.attrelid
         AND    ad.adnum   = a.attnum
         AND    ad.adsrc = 'nextval('''
            || (pg_get_serial_sequence (a.attrelid::regclass::text
                                      , a.attname))::regclass
            || '''::regclass)'
         )
    THEN CASE a.atttypid
            WHEN 'int'::regtype  THEN 'serial'
            WHEN 'int8'::regtype THEN 'bigserial'
            WHEN 'int2'::regtype THEN 'smallserial'
         END
    ELSE format_type(a.atttypid, a.atttypmod)
    END AS data_type
FROM pg_attribute a
JOIN ONLY pg_class c ON c.oid = a.attrelid
JOIN ONLY pg_namespace n ON n.oid = c.relnamespace
LEFT JOIN pg_constraint ct ON ct.conrelid = c.oid
AND a.attnum = ANY(ct.conkey) AND ct.contype IN ('p', 'u')
LEFT JOIN pg_attrdef ad ON ad.adrelid = c.oid AND ad.adnum = a.attnum
WHERE a.attisdropped = false
AND n.nspname = $1
AND c.relname = $2
AND a.attnum > 0
ORDER BY a.attnum
`

const pgLoadTableDef = `
SELECT
c.relkind AS type,
c.relname AS table_name
FROM pg_class c
JOIN ONLY pg_namespace n ON n.oid = c.relnamespace
WHERE n.nspname = $1
AND c.relkind = 'r'
`

// TypeMap go/db type map struct
type TypeMap struct {
	DBTypes        []string `toml:"db_types"`
	NotNullGoType  string   `toml:"notnull_go_type"`
	NullableGoType string   `toml:"nullable_go_type"`
}

// AutoKeyMap auto generating key config
type AutoKeyMap struct {
	Types []string `toml:"db_types"`
}

// PgTypeMapConfig go/db type map struct toml config
type PgTypeMapConfig map[string]TypeMap

// PgTable postgres table
type PgTable struct {
	Schema      string
	Name        string
	DataType    string
	AutoGenPk   bool
	PrimaryKeys []*PgColumn
	Columns     []*PgColumn
}

var autoGenKeyCfg = &AutoKeyMap{
	Types: []string{"serial", "bigserial", "UUID"},
}

func (t *PgTable) setPrimaryKeyInfo(cfg *AutoKeyMap) {
	t.AutoGenPk = false
	for _, c := range t.Columns {
		if c.IsPrimaryKey {
			t.PrimaryKeys = append(t.PrimaryKeys, c)
			for _, typ := range cfg.Types {
				if c.DDLType == typ {
					t.AutoGenPk = true
				}
			}
		}
	}
	return
}

// PgColumn postgres columns
type PgColumn struct {
	FieldOrdinal int
	Name         string
	DataType     string
	DDLType      string
	NotNull      bool
	DefaultValue sql.NullString
	IsPrimaryKey bool
}

// Struct go struct
type Struct struct {
	Name    string
	Table   *PgTable
	Comment string
	Fields  []*StructField
}

// StructTmpl go struct passed to template
type StructTmpl struct {
	Struct *Struct
}

// StructField go struct field
type StructField struct {
	Name   string
	Type   string
	Tag    string
	Column *PgColumn
}

// PgLoadTypeMapFromFile load type map from toml file
func PgLoadTypeMapFromFile(filePath string) (*PgTypeMapConfig, error) {
	var conf PgTypeMapConfig
	if _, err := toml.DecodeFile(filePath, &conf); err != nil {
		return nil, errors.Wrap(err, "faild to parse config file")
	}
	return &conf, nil
}

// PgLoadColumnDef load Postgres column definition
func PgLoadColumnDef(db Queryer, schema string, table string) ([]*PgColumn, error) {
	colDefs, err := db.Query(pgLoadColumnDef, schema, table)
	if err != nil {
		return nil, errors.Wrap(err, "failed to load table def")
	}

	cols := []*PgColumn{}
	for colDefs.Next() {
		c := &PgColumn{}
		err := colDefs.Scan(
			&c.FieldOrdinal,
			&c.Name,
			&c.DataType,
			&c.NotNull,
			&c.DefaultValue,
			&c.IsPrimaryKey,
			&c.DDLType,
		)
		if err != nil {
			return nil, errors.Wrap(err, "failed to scan")
		}
		cols = append(cols, c)
	}
	return cols, nil
}

// PgLoadTableDef load Postgres table definition
func PgLoadTableDef(db Queryer, schema string) ([]*PgTable, error) {
	tbDefs, err := db.Query(pgLoadTableDef, schema)
	if err != nil {
		return nil, errors.Wrap(err, "failed to load table def")
	}
	tbs := []*PgTable{}
	for tbDefs.Next() {
		t := &PgTable{Schema: schema}
		err := tbDefs.Scan(
			&t.DataType,
			&t.Name,
		)
		if err != nil {
			return nil, errors.Wrap(err, "failed to scan")
		}
		cols, err := PgLoadColumnDef(db, schema, t.Name)
		if err != nil {
			return nil, errors.Wrap(err, fmt.Sprintf("failed to get columns of %s", t.Name))
		}
		t.Columns = cols
		tbs = append(tbs, t)
	}
	return tbs, nil
}

func contains(v string, l []string) bool {
	sort.Strings(l)
	i := sort.SearchStrings(l, v)
	if i < len(l) && l[i] == v {
		return true
	}
	return false
}

// PgConvertType converts type
func PgConvertType(col *PgColumn, typeCfg *PgTypeMapConfig) string {
	cfg := map[string]TypeMap(*typeCfg)
	typ := cfg["default"].NotNullGoType
	for _, v := range cfg {
		if contains(col.DataType, v.DBTypes) {
			if col.NotNull {
				return v.NotNullGoType
			}
			return v.NullableGoType
		}
	}
	return typ
}

// PgColToField converts pg column to go struct field
func PgColToField(col *PgColumn, typeCfg *PgTypeMapConfig) (*StructField, error) {
	stfType := PgConvertType(col, typeCfg)
	stf := &StructField{
		Name:   varfmt.PublicVarName(col.Name),
		Type:   stfType,
		Column: col,
	}
	return stf, nil
}

// PgTableToStruct converts table def to go struct
func PgTableToStruct(t *PgTable, typeCfg *PgTypeMapConfig, keyConfig *AutoKeyMap) (*Struct, error) {
	t.setPrimaryKeyInfo(keyConfig)
	s := &Struct{
		Name:  varfmt.PublicVarName(t.Name),
		Table: t,
	}
	var fs []*StructField
	for _, c := range t.Columns {
		f, err := PgColToField(c, typeCfg)
		if err != nil {
			return nil, errors.Wrap(err, "faield to convert col to field")
		}
		fs = append(fs, f)
	}
	s.Fields = fs
	return s, nil
}

// PgExecuteStructTmpl execute struct template with *Struct
func PgExecuteStructTmpl(st *StructTmpl, path string) ([]byte, error) {
	var src []byte
	d, err := Asset(path)
	if err != nil {
		return src, errors.Wrap(err, "failed to load asset")
	}
	tpl, err := template.New("struct").Funcs(tmplFuncMap).Parse(string(d))
	if err != nil {
		return src, errors.Wrap(err, "failed to parse template")
	}
	buf := new(bytes.Buffer)
	if err := tpl.Execute(buf, st); err != nil {
		return src, errors.Wrap(err, fmt.Sprintf("failed to execute template:\n%s", src))
	}
	src, err = format.Source(buf.Bytes())
	if err != nil {
		log.Printf("%s", buf)
		return src, errors.Wrap(err, fmt.Sprintf("failed to format code:\n%s", src))
	}
	return src, nil
}

// PgCreateStruct creates struct from given schema
func PgCreateStruct(db Queryer, schema, typeMapPath, pkgName string) ([]byte, error) {
	var src []byte
	pkgDef := []byte(fmt.Sprintf("package %s\n\n", pkgName))
	src = append(src, pkgDef...)

	tbls, err := PgLoadTableDef(db, schema)
	if err != nil {
		return src, errors.Wrap(err, "faield to load table definitions")
	}
	cfg := &PgTypeMapConfig{}
	if typeMapPath == "" {
		if _, err := toml.Decode(typeMap, cfg); err != nil {
			return src, errors.Wrap(err, "faield to read type map")
		}
	} else {
		if _, err := toml.DecodeFile(typeMapPath, cfg); err != nil {
			return src, errors.Wrap(err, fmt.Sprintf("failed to decode type map file %s", typeMapPath))
		}
	}
	for _, tbl := range tbls {
		st, err := PgTableToStruct(tbl, cfg, autoGenKeyCfg)
		if err != nil {
			return src, errors.Wrap(err, "faield to convert table definition to struct")
		}
		s, err := PgExecuteStructTmpl(&StructTmpl{Struct: st}, "template/struct.tmpl")
		if err != nil {
			return src, errors.Wrap(err, "faield to execute template")
		}
		m, err := PgExecuteStructTmpl(&StructTmpl{Struct: st}, "template/method.tmpl")
		if err != nil {
			return src, errors.Wrap(err, "faield to execute template")
		}
		src = append(src, s...)
		src = append(src, m...)
	}
	return src, nil
}
<commit_msg>Order by table name<commit_after>// go:generate go-bindata -o bindata.go template mapconfig
package main

import (
	"bytes"
	"database/sql"
	"fmt"
	"go/format"
	"log"
	"sort"
	"text/template"

	"github.com/BurntSushi/toml"
	"github.com/achiku/varfmt"
	"github.com/pkg/errors"
)

const pgLoadColumnDef = `
SELECT
    a.attnum AS field_ordinal,
    a.attname AS column_name,
    format_type(a.atttypid, a.atttypmod) AS data_type,
    a.attnotnull AS not_null,
    COALESCE(pg_get_expr(ad.adbin, ad.adrelid), '') AS default_value,
    COALESCE(ct.contype = 'p', false) AS  is_primary_key,
    CASE WHEN a.atttypid = ANY ('{int,int8,int2}'::regtype[])
      AND EXISTS (
         SELECT 1 FROM pg_attrdef ad
         WHERE  ad.adrelid = a.attrelid
         AND    ad.adnum   = a.attnum
         AND    ad.adsrc = 'nextval('''
            || (pg_get_serial_sequence (a.attrelid::regclass::text
                                      , a.attname))::regclass
            || '''::regclass)'
         )
    THEN CASE a.atttypid
            WHEN 'int'::regtype  THEN 'serial'
            WHEN 'int8'::regtype THEN 'bigserial'
            WHEN 'int2'::regtype THEN 'smallserial'
         END
    ELSE format_type(a.atttypid, a.atttypmod)
    END AS data_type
FROM pg_attribute a
JOIN ONLY pg_class c ON c.oid = a.attrelid
JOIN ONLY pg_namespace n ON n.oid = c.relnamespace
LEFT JOIN pg_constraint ct ON ct.conrelid = c.oid
AND a.attnum = ANY(ct.conkey) AND ct.contype IN ('p', 'u')
LEFT JOIN pg_attrdef ad ON ad.adrelid = c.oid AND ad.adnum = a.attnum
WHERE a.attisdropped = false
AND n.nspname = $1
AND c.relname = $2
AND a.attnum > 0
ORDER BY a.attnum
`

const pgLoadTableDef = `
SELECT
c.relkind AS type,
c.relname AS table_name
FROM pg_class c
JOIN ONLY pg_namespace n ON n.oid = c.relnamespace
WHERE n.nspname = $1
AND c.relkind = 'r'
ORDER BY c.relname
`

// TypeMap go/db type map struct
type TypeMap struct {
	DBTypes        []string `toml:"db_types"`
	NotNullGoType  string   `toml:"notnull_go_type"`
	NullableGoType string   `toml:"nullable_go_type"`
}

// AutoKeyMap auto generating key config
type AutoKeyMap struct {
	Types []string `toml:"db_types"`
}

// PgTypeMapConfig go/db type map struct toml config
type PgTypeMapConfig map[string]TypeMap

// PgTable postgres table
type PgTable struct {
	Schema      string
	Name        string
	DataType    string
	AutoGenPk   bool
	PrimaryKeys []*PgColumn
	Columns     []*PgColumn
}

var autoGenKeyCfg = &AutoKeyMap{
	Types: []string{"serial", "bigserial", "UUID"},
}

func (t *PgTable) setPrimaryKeyInfo(cfg *AutoKeyMap) {
	t.AutoGenPk = false
	for _, c := range t.Columns {
		if c.IsPrimaryKey {
			t.PrimaryKeys = append(t.PrimaryKeys, c)
			for _, typ := range cfg.Types {
				if c.DDLType == typ {
					t.AutoGenPk = true
				}
			}
		}
	}
	return
}

// PgColumn postgres columns
type PgColumn struct {
	FieldOrdinal int
	Name         string
	DataType     string
	DDLType      string
	NotNull      bool
	DefaultValue sql.NullString
	IsPrimaryKey bool
}

// Struct go struct
type Struct struct {
	Name    string
	Table   *PgTable
	Comment string
	Fields  []*StructField
}

// StructTmpl go struct passed to template
type StructTmpl struct {
	Struct *Struct
}

// StructField go struct field
type StructField struct {
	Name   string
	Type   string
	Tag    string
	Column *PgColumn
}

// PgLoadTypeMapFromFile load type map from toml file
func PgLoadTypeMapFromFile(filePath string) (*PgTypeMapConfig, error) {
	var conf PgTypeMapConfig
	if _, err := toml.DecodeFile(filePath, &conf); err != nil {
		return nil, errors.Wrap(err, "faild to parse config file")
	}
	return &conf, nil
}

// PgLoadColumnDef load Postgres column definition
func PgLoadColumnDef(db Queryer, schema string, table string) ([]*PgColumn, error) {
	colDefs, err := db.Query(pgLoadColumnDef, schema, table)
	if err != nil {
		return nil, errors.Wrap(err, "failed to load table def")
	}

	cols := []*PgColumn{}
	for colDefs.Next() {
		c := &PgColumn{}
		err := colDefs.Scan(
			&c.FieldOrdinal,
			&c.Name,
			&c.DataType,
			&c.NotNull,
			&c.DefaultValue,
			&c.IsPrimaryKey,
			&c.DDLType,
		)
		if err != nil {
			return nil, errors.Wrap(err, "failed to scan")
		}
		cols = append(cols, c)
	}
	return cols, nil
}

// PgLoadTableDef load Postgres table definition
func PgLoadTableDef(db Queryer, schema string) ([]*PgTable, error) {
	tbDefs, err := db.Query(pgLoadTableDef, schema)
	if err != nil {
		return nil, errors.Wrap(err, "failed to load table def")
	}
	tbs := []*PgTable{}
	for tbDefs.Next() {
		t := &PgTable{Schema: schema}
		err := tbDefs.Scan(
			&t.DataType,
			&t.Name,
		)
		if err != nil {
			return nil, errors.Wrap(err, "failed to scan")
		}
		cols, err := PgLoadColumnDef(db, schema, t.Name)
		if err != nil {
			return nil, errors.Wrap(err, fmt.Sprintf("failed to get columns of %s", t.Name))
		}
		t.Columns = cols
		tbs = append(tbs, t)
	}
	return tbs, nil
}

func contains(v string, l []string) bool {
	sort.Strings(l)
	i := sort.SearchStrings(l, v)
	if i < len(l) && l[i] == v {
		return true
	}
	return false
}

// PgConvertType converts type
func PgConvertType(col *PgColumn, typeCfg *PgTypeMapConfig) string {
	cfg := map[string]TypeMap(*typeCfg)
	typ := cfg["default"].NotNullGoType
	for _, v := range cfg {
		if contains(col.DataType, v.DBTypes) {
			if col.NotNull {
				return v.NotNullGoType
			}
			return v.NullableGoType
		}
	}
	return typ
}

// PgColToField converts pg column to go struct field
func PgColToField(col *PgColumn, typeCfg *PgTypeMapConfig) (*StructField, error) {
	stfType := PgConvertType(col, typeCfg)
	stf := &StructField{
		Name:   varfmt.PublicVarName(col.Name),
		Type:   stfType,
		Column: col,
	}
	return stf, nil
}

// PgTableToStruct converts table def to go struct
func PgTableToStruct(t *PgTable, typeCfg *PgTypeMapConfig, keyConfig *AutoKeyMap) (*Struct, error) {
	t.setPrimaryKeyInfo(keyConfig)
	s := &Struct{
		Name:  varfmt.PublicVarName(t.Name),
		Table: t,
	}
	var fs []*StructField
	for _, c := range t.Columns {
		f, err := PgColToField(c, typeCfg)
		if err != nil {
			return nil, errors.Wrap(err, "faield to convert col to field")
		}
		fs = append(fs, f)
	}
	s.Fields = fs
	return s, nil
}

// PgExecuteStructTmpl execute struct template with *Struct
func PgExecuteStructTmpl(st *StructTmpl, path string) ([]byte, error) {
	var src []byte
	d, err := Asset(path)
	if err != nil {
		return src, errors.Wrap(err, "failed to load asset")
	}
	tpl, err := template.New("struct").Funcs(tmplFuncMap).Parse(string(d))
	if err != nil {
		return src, errors.Wrap(err, "failed to parse template")
	}
	buf := new(bytes.Buffer)
	if err := tpl.Execute(buf, st); err != nil {
		return src, errors.Wrap(err, fmt.Sprintf("failed to execute template:\n%s", src))
	}
	src, err = format.Source(buf.Bytes())
	if err != nil {
		log.Printf("%s", buf)
		return src, errors.Wrap(err, fmt.Sprintf("failed to format code:\n%s", src))
	}
	return src, nil
}

// PgCreateStruct creates struct from given schema
func PgCreateStruct(db Queryer, schema, typeMapPath, pkgName string) ([]byte, error) {
	var src []byte
	pkgDef := []byte(fmt.Sprintf("package %s\n\n", pkgName))
	src = append(src, pkgDef...)

	tbls, err := PgLoadTableDef(db, schema)
	if err != nil {
		return src, errors.Wrap(err, "faield to load table definitions")
	}
	cfg := &PgTypeMapConfig{}
	if typeMapPath == "" {
		if _, err := toml.Decode(typeMap, cfg); err != nil {
			return src, errors.Wrap(err, "faield to read type map")
		}
	} else {
		if _, err := toml.DecodeFile(typeMapPath, cfg); err != nil {
			return src, errors.Wrap(err, fmt.Sprintf("failed to decode type map file %s", typeMapPath))
		}
	}
	for _, tbl := range tbls {
		st, err := PgTableToStruct(tbl, cfg, autoGenKeyCfg)
		if err != nil {
			return src, errors.Wrap(err, "faield to convert table definition to struct")
		}
		s, err := PgExecuteStructTmpl(&StructTmpl{Struct: st}, "template/struct.tmpl")
		if err != nil {
			return src, errors.Wrap(err, "faield to execute template")
		}
		m, err := PgExecuteStructTmpl(&StructTmpl{Struct: st}, "template/method.tmpl")
		if err != nil {
			return src, errors.Wrap(err, "faield to execute template")
		}
		src = append(src, s...)
		src = append(src, m...)
	}
	return src, nil
}
<|endoftext|>


<commit_before>package dht

import (
	"crypto"
	_ "crypto/sha1"
	"errors"
	"log"
	"math/rand"
	"net"
	"time"

	"github.com/anacrolix/missinggo"
	"github.com/anacrolix/torrent/iplist"
	"github.com/anacrolix/torrent/metainfo"

	"github.com/anacrolix/dht/krpc"
)

func defaultQueryResendDelay() time.Duration {
	return jitterDuration(5*time.Second, time.Second)
}

// Uniquely identifies a transaction to us.
type transactionKey struct {
	RemoteAddr string // host:port
	T          string // The KRPC transaction ID.
}

// ServerConfig allows to set up a  configuration of the `Server` instance
// to be created with NewServer
type ServerConfig struct {
	// Set NodeId Manually. Caller must ensure that if NodeId does not conform
	// to DHT Security Extensions, that NoSecurity is also set.
	NodeId [20]byte
	Conn   net.PacketConn
	// Don't respond to queries from other nodes.
	Passive       bool
	StartingNodes func() ([]Addr, error)
	// Disable the DHT security extension:
	// http://www.libtorrent.org/dht_sec.html.
	NoSecurity bool
	// Initial IP blocklist to use. Applied before serving and bootstrapping
	// begins.
	IPBlocklist iplist.Ranger
	// Used to secure the server's ID. Defaults to the Conn's LocalAddr(). Set
	// to the IP that remote nodes will see, as that IP is what they'll use to
	// validate our ID.
	PublicIP net.IP

	// Hook received queries. Return false if you don't want to propagate to
	// the default handlers.
	OnQuery func(query *krpc.Msg, source net.Addr) (propagate bool)
	// Called when a peer successfully announces to us.
	OnAnnouncePeer func(infoHash metainfo.Hash, peer Peer)
	// How long to wait before resending queries that haven't received a
	// response. Defaults to a random value between 4.5 and 5.5s.
	QueryResendDelay func() time.Duration
	// TODO: Expose Peers, to return NodeInfo for received get_peers queries.
}

// ServerStats instance is returned by Server.Stats() and stores Server metrics
type ServerStats struct {
	// Count of nodes in the node table that responded to our last query or
	// haven't yet been queried.
	GoodNodes int
	// Count of nodes in the node table.
	Nodes int
	// Transactions awaiting a response.
	OutstandingTransactions int
	// Individual announce_peer requests that got a success response.
	ConfirmedAnnounces int
	// Nodes that have been blocked.
	BadNodes uint
}

func jitterDuration(average time.Duration, plusMinus time.Duration) time.Duration {
	return average - plusMinus/2 + time.Duration(rand.Int63n(int64(plusMinus)))
}

type Peer = krpc.NodeAddr

func GlobalBootstrapAddrs() (addrs []Addr, err error) {
	for _, s := range []string{
		"router.utorrent.com:6881",
		"router.bittorrent.com:6881",
		"dht.transmissionbt.com:6881",
		"dht.aelitis.com:6881", // Vuze
	} {
		host, port, err := net.SplitHostPort(s)
		if err != nil {
			panic(err)
		}
		hostAddrs, err := net.LookupHost(host)
		if err != nil {
			log.Printf("error looking up %q: %v", s, err)
			continue
		}
		for _, a := range hostAddrs {
			ua, err := net.ResolveUDPAddr("udp", net.JoinHostPort(a, port))
			if err != nil {
				log.Printf("error resolving %q: %v", a, err)
				continue
			}
			addrs = append(addrs, NewAddr(ua))
		}
	}
	if len(addrs) == 0 {
		err = errors.New("nothing resolved")
	}
	return
}

func RandomNodeID() (id [20]byte) {
	rand.Read(id[:])
	return
}

func MakeDeterministicNodeID(public net.Addr) (id [20]byte) {
	h := crypto.SHA1.New()
	h.Write([]byte(public.String()))
	h.Sum(id[:0:20])
	SecureNodeId(&id, missinggo.AddrIP(public))
	return
}
<commit_msg>RandomNodeId wasn't cryptographically random<commit_after>package dht

import (
	"crypto"
	crand "crypto/rand"
	_ "crypto/sha1"
	"errors"
	"log"
	"math/rand"
	"net"
	"time"

	"github.com/anacrolix/missinggo"
	"github.com/anacrolix/torrent/iplist"
	"github.com/anacrolix/torrent/metainfo"

	"github.com/anacrolix/dht/krpc"
)

func defaultQueryResendDelay() time.Duration {
	return jitterDuration(5*time.Second, time.Second)
}

// Uniquely identifies a transaction to us.
type transactionKey struct {
	RemoteAddr string // host:port
	T          string // The KRPC transaction ID.
}

// ServerConfig allows to set up a  configuration of the `Server` instance
// to be created with NewServer
type ServerConfig struct {
	// Set NodeId Manually. Caller must ensure that if NodeId does not conform
	// to DHT Security Extensions, that NoSecurity is also set.
	NodeId [20]byte
	Conn   net.PacketConn
	// Don't respond to queries from other nodes.
	Passive       bool
	StartingNodes func() ([]Addr, error)
	// Disable the DHT security extension:
	// http://www.libtorrent.org/dht_sec.html.
	NoSecurity bool
	// Initial IP blocklist to use. Applied before serving and bootstrapping
	// begins.
	IPBlocklist iplist.Ranger
	// Used to secure the server's ID. Defaults to the Conn's LocalAddr(). Set
	// to the IP that remote nodes will see, as that IP is what they'll use to
	// validate our ID.
	PublicIP net.IP

	// Hook received queries. Return false if you don't want to propagate to
	// the default handlers.
	OnQuery func(query *krpc.Msg, source net.Addr) (propagate bool)
	// Called when a peer successfully announces to us.
	OnAnnouncePeer func(infoHash metainfo.Hash, peer Peer)
	// How long to wait before resending queries that haven't received a
	// response. Defaults to a random value between 4.5 and 5.5s.
	QueryResendDelay func() time.Duration
	// TODO: Expose Peers, to return NodeInfo for received get_peers queries.
}

// ServerStats instance is returned by Server.Stats() and stores Server metrics
type ServerStats struct {
	// Count of nodes in the node table that responded to our last query or
	// haven't yet been queried.
	GoodNodes int
	// Count of nodes in the node table.
	Nodes int
	// Transactions awaiting a response.
	OutstandingTransactions int
	// Individual announce_peer requests that got a success response.
	ConfirmedAnnounces int
	// Nodes that have been blocked.
	BadNodes uint
}

func jitterDuration(average time.Duration, plusMinus time.Duration) time.Duration {
	return average - plusMinus/2 + time.Duration(rand.Int63n(int64(plusMinus)))
}

type Peer = krpc.NodeAddr

func GlobalBootstrapAddrs() (addrs []Addr, err error) {
	for _, s := range []string{
		"router.utorrent.com:6881",
		"router.bittorrent.com:6881",
		"dht.transmissionbt.com:6881",
		"dht.aelitis.com:6881", // Vuze
	} {
		host, port, err := net.SplitHostPort(s)
		if err != nil {
			panic(err)
		}
		hostAddrs, err := net.LookupHost(host)
		if err != nil {
			log.Printf("error looking up %q: %v", s, err)
			continue
		}
		for _, a := range hostAddrs {
			ua, err := net.ResolveUDPAddr("udp", net.JoinHostPort(a, port))
			if err != nil {
				log.Printf("error resolving %q: %v", a, err)
				continue
			}
			addrs = append(addrs, NewAddr(ua))
		}
	}
	if len(addrs) == 0 {
		err = errors.New("nothing resolved")
	}
	return
}

func RandomNodeID() (id [20]byte) {
	crand.Read(id[:])
	return
}

func MakeDeterministicNodeID(public net.Addr) (id [20]byte) {
	h := crypto.SHA1.New()
	h.Write([]byte(public.String()))
	h.Sum(id[:0:20])
	SecureNodeId(&id, missinggo.AddrIP(public))
	return
}
<|endoftext|>


<commit_before>/*
Package chess is a go library designed to accomplish the following:
    - chess game / turn management
    - move validation
    - PGN encoding / decoding
    - FEN encoding / decoding

Using Moves
    game := chess.NewGame()
    moves := game.ValidMoves()
    game.Move(moves[0])

Using Algebraic Notation
    game := chess.NewGame()
    game.MoveStr("e4")

Using PGN
    pgn, _ := chess.PGN(pgnReader)
    game := chess.NewGame(pgn)

Using FEN
    fen, _ := chess.FEN("rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR w KQkq - 0 1")
    game := chess.NewGame(fen)

Random Game
    package main

    import (
        "fmt"

        "github.com/notnil/chess"
    )

    func main() {
        game := chess.NewGame()
        // generate moves until game is over
        for game.Outcome() == chess.NoOutcome {
            // select a random move
            moves := game.ValidMoves()
            move := moves[rand.Intn(len(moves))]
            game.Move(move)
        }
        // print outcome and game PGN
        fmt.Println(game.Position().Board().Draw())
        fmt.Printf("Game completed. %s by %s.\n", game.Outcome(), game.Method())
        fmt.Println(game.String())
    }
*/
package chess
<commit_msg>Added a missing dependency in the documentation<commit_after>/*
Package chess is a go library designed to accomplish the following:
    - chess game / turn management
    - move validation
    - PGN encoding / decoding
    - FEN encoding / decoding

Using Moves
    game := chess.NewGame()
    moves := game.ValidMoves()
    game.Move(moves[0])

Using Algebraic Notation
    game := chess.NewGame()
    game.MoveStr("e4")

Using PGN
    pgn, _ := chess.PGN(pgnReader)
    game := chess.NewGame(pgn)

Using FEN
    fen, _ := chess.FEN("rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR w KQkq - 0 1")
    game := chess.NewGame(fen)

Random Game
    package main

    import (
        "fmt"
        "github.com/notnil/chess"
        "math/rand"
    )

    func main() {
        game := chess.NewGame()
        // generate moves until game is over
        for game.Outcome() == chess.NoOutcome {
            // select a random move
            moves := game.ValidMoves()
            move := moves[rand.Intn(len(moves))]
            game.Move(move)
        }
        // print outcome and game PGN
        fmt.Println(game.Position().Board().Draw())
        fmt.Printf("Game completed. %s by %s.\n", game.Outcome(), game.Method())
        fmt.Println(game.String())
    }
*/
package chess
<|endoftext|>


<commit_before>/*
Package fasthttp provides fast HTTP server and client API.

Fasthttp provides the following features:

    * Optimized for speed. Fasthttp is faster than standard net/http package.
    * Optimized for low memory usage.
    * Easily handles more than 1M concurrent keep-alive connections on modern
      hardware.
    * Server supports requests' pipelining. Multiple requests may be read from
      a single network packet and multiple responses may be sent in a single
      network packet. This may be useful for highly loaded REST services.
    * Server is packed with the following anti-DoS limits:

        * The number of concurrent connections.
        * The number of concurrent connections per client IP.
        * The number of requests per connection.
        * Request read timeout.
        * Response write timeout.
        * Maximum request header size.

    * A lot of useful info is exposed to request handler:

        * Server and client address.
        * Per-request logger.
        * Unique request id.
        * Request start time.
        * Connection start time.
        * Request sequence number for the current connection.

    * Client supports automatic retry on idempotent requests' failure.
    * It is quite easy building custom (and fast) client and server
      implementations using fasthttp API.
*/
package fasthttp
<commit_msg>Updated docs<commit_after>/*
Package fasthttp provides fast HTTP server and client API.

Fasthttp provides the following features:

    * Optimized for speed. Easily handles more than 100K qps and more than 1M
      concurrent keep-alive connections on modern hardware.
    * Optimized for low memory usage.
    * Server supports requests' pipelining. Multiple requests may be read from
      a single network packet and multiple responses may be sent in a single
      network packet. This may be useful for highly loaded REST services.
    * Server provides the following anti-DoS limits:

        * The number of concurrent connections.
        * The number of concurrent connections per client IP.
        * The number of requests per connection.
        * Request read timeout.
        * Response write timeout.
        * Maximum request header size.
        * Maximum request execution time.

    * A lot of additional useful info is exposed to request handler:

        * Server and client address.
        * Per-request logger.
        * Unique request id.
        * Request start time.
        * Connection start time.
        * Request sequence number for the current connection.

    * Client supports automatic retry on idempotent requests' failure.
    * Fasthttp API is designed with the ability to extend existing client
      and server implementations or to write custom client and server
      implementations from scratch.
*/
package fasthttp
<|endoftext|>


<commit_before>/*
Package complete provides a tool for bash writing bash completion in go, and bash completion for the go command line.

Writing bash completion scripts is a hard work. This package provides an easy way
to create bash completion scripts for any command, and also an easy way to install/uninstall
the completion of the command.

## go command bash completion

In [gocomplete](./cmd/gocomplete) there is an example for bash completion for the `go` command line.

This is an example that uses the `complete` package on the `go` command - the `complete` package
can also be used to implement any completions, see [Usage](#usage).

### Install

1. Type in your shell:

	go get -u github.com/posener/complete/gocomplete
	gocomplete -install

2. Restart your shell

Uninstall by `gocomplete -uninstall`

### Features

- Complete `go` command, including sub commands and all flags.
- Complete packages names or `.go` files when necessary.
- Complete test names after `-run` flag.

## complete package

Supported shells:

- [x] bash
- [x] zsh
- [x] fish

### Usage

Assuming you have program called `run` and you want to have bash completion
for it, meaning, if you type `run` then space, then press the `Tab` key,
the shell will suggest relevant complete options.

In that case, we will create a program called `runcomplete`, a go program,
with a `func main()` and so, that will make the completion of the `run`
program. Once the `runcomplete` will be in a binary form, we could
`runcomplete -install` and that will add to our shell all the bash completion
options for `run`.

So here it is:

	import "github.com/posener/complete"

	func main() {

		// create a Command object, that represents the command we want
		// to complete.
		run := complete.Command{

			// Sub defines a list of sub commands of the program,
			// this is recursive, since every command is of type command also.
			Sub: complete.Commands{

				// add a build sub command
				"build": complete.Command {

					// define flags of the build sub command
					Flags: complete.Flags{
						// build sub command has a flag '-cpus', which
						// expects number of cpus after it. in that case
						// anything could complete this flag.
						"-cpus": complete.PredictAnything,
					},
				},
			},

			// define flags of the 'run' main command
			Flags: complete.Flags{
				// a flag -o, which expects a file ending with .out after
				// it, the tab completion will auto complete for files matching
				// the given pattern.
				"-o": complete.PredictFiles("*.out"),
			},

			// define global flags of the 'run' main command
			// those will show up also when a sub command was entered in the
			// command line
			GlobalFlags: complete.Flags{

				// a flag '-h' which does not expects anything after it
				"-h": complete.PredictNothing,
			},
		}

		// run the command completion, as part of the main() function.
		// this triggers the autocompletion when needed.
		// name must be exactly as the binary that we want to complete.
		complete.New("run", run).Run()
	}

### Self completing program

In case that the program that we want to complete is written in go we
can make it self completing.

Here is an [example](./example/self/main.go)

*/
package complete
<commit_msg>fix doc<commit_after>/*
Package complete provides a tool for bash writing bash completion in go, and bash completion for the go command line.

Writing bash completion scripts is a hard work. This package provides an easy way
to create bash completion scripts for any command, and also an easy way to install/uninstall
the completion of the command.

go command bash completion

In [gocomplete](./cmd/gocomplete) there is an example for bash completion for the `go` command line.

This is an example that uses the `complete` package on the `go` command - the `complete` package
can also be used to implement any completions, see [Usage](#usage).

Install

1. Type in your shell:

	go get -u github.com/posener/complete/gocomplete
	gocomplete -install

2. Restart your shell

Uninstall by `gocomplete -uninstall`

Features

- Complete `go` command, including sub commands and all flags.
- Complete packages names or `.go` files when necessary.
- Complete test names after `-run` flag.

complete package

Supported shells:

- [x] bash
- [x] zsh
- [x] fish

Usage

Assuming you have program called `run` and you want to have bash completion
for it, meaning, if you type `run` then space, then press the `Tab` key,
the shell will suggest relevant complete options.

In that case, we will create a program called `runcomplete`, a go program,
with a `func main()` and so, that will make the completion of the `run`
program. Once the `runcomplete` will be in a binary form, we could
`runcomplete -install` and that will add to our shell all the bash completion
options for `run`.

So here it is:

	import "github.com/posener/complete"

	func main() {

		// create a Command object, that represents the command we want
		// to complete.
		run := complete.Command{

			// Sub defines a list of sub commands of the program,
			// this is recursive, since every command is of type command also.
			Sub: complete.Commands{

				// add a build sub command
				"build": complete.Command {

					// define flags of the build sub command
					Flags: complete.Flags{
						// build sub command has a flag '-cpus', which
						// expects number of cpus after it. in that case
						// anything could complete this flag.
						"-cpus": complete.PredictAnything,
					},
				},
			},

			// define flags of the 'run' main command
			Flags: complete.Flags{
				// a flag -o, which expects a file ending with .out after
				// it, the tab completion will auto complete for files matching
				// the given pattern.
				"-o": complete.PredictFiles("*.out"),
			},

			// define global flags of the 'run' main command
			// those will show up also when a sub command was entered in the
			// command line
			GlobalFlags: complete.Flags{

				// a flag '-h' which does not expects anything after it
				"-h": complete.PredictNothing,
			},
		}

		// run the command completion, as part of the main() function.
		// this triggers the autocompletion when needed.
		// name must be exactly as the binary that we want to complete.
		complete.New("run", run).Run()
	}

Self completing program

In case that the program that we want to complete is written in go we
can make it self completing.
Here is an [example](./example/self/main.go)

*/
package complete
<|endoftext|>


<commit_before>// Package configeur is an easy to use multi-layer configuration system.
//
// configeur makes use of Checkers, which are used to retrieve values from their respective data sources.
// There are three built in Checkers, Environment, Flag and JSON. Environment retrieves environment variables.
// Flag retrieves variables within the flags of a command. JSON retrieves values from a JSON file/blob. Checkers
// can be essentially thought of as "configuration middlewear", in fact parts of the package API was inspired by
// negroni (https://github.com/codegangsta/negroni, special thanks to codegangsta for the awesome package!) and the
// standard libraries flag package.
//
// It is very easy to create your own Checkers, all they have to do is satisfy the Checker interface.
// That is a, Int method, String method and a Bool method. To retrieve their respective data types.
// If you do create your own Checkers I would be more than happy to add a link to them somewhere in the repository.
//
// A standard use-case can be found in the example folder and a getting started guide can be found on github (http://github.com/paked/configeur).
package configeur
<commit_msg>Update information in README<commit_after>// Package configeur is an easy to use multi-layer configuration system.
//
// Examples can be found in the example folder (http://github.com/paked/configeur/blob/master/examples/)
// as well as a getting started guide in the main README file (http://github.com/paked/configeur).
//
// configeur makes use of Checkers, which are used to retrieve values from their respective data sources.
// There are three built in Checkers, Environment, Flag and JSON. Environment retrieves environment variables.
// Flag retrieves variables within the flags of a command. JSON retrieves values from a JSON file/blob. Checkers
// can be essentially thought of as "middlewear for configuration", in fact parts of the package API was inspired by
// negroni (https://github.com/codegangsta/negroni, the awesome net/http middlewear manager) and the standard library's flag package.
//
// It is very easy to create your own Checkers, all they have to do is satisfy the Checker interface.
//  type Checker interface {
//	  Int(name string) (int, error)
// 	  Bool(name string) (int, error)
// 	  String(name string) (string, error)
//  }
// That is an, Int method, String method and a Bool method. These functions are used to retrieve their respective data types.
// If you do create your own Checkers I would be more than happy to add a link to the README in the github repository.
//
package configeur
<|endoftext|>


<commit_before>// Copyright 2016 Marcel Gotsch. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// Package goserv provides a fast, easy and minimalistic framework for
// web applications in Go.
//
//
package goserv
<commit_msg>Add path documentation and more to doc.go<commit_after>// Copyright 2016 Marcel Gotsch. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// Package goserv provides a fast, easy and minimalistic framework for
// web applications in Go.
//
// Getting Started
//
// Every goserv application starts with a single instance of goserv.Server. This
// instance is the entry point for all incoming requests.
//
//      server := goserv.NewServer()
//
// To start handling paths you must register handlers to paths using one of the Server's
// embedded Router functions, like Get.
//
//      server.Get("/", homeHandler)
//
// The first argument in the Get() call is the path for which the
// handler gets registered. To learn more about paths take a look at the Path section.
// As the name of the function suggests the requests are only getting dispatched to the handler
// if the request method is "GET". There are a lot more methods like this one available, just take
// a look at the documentation of the Router.
//
// At this point it is worth noting that goserv provides it's own Handler type using the
// goserv.ResponseWriter and goserv.Request. This means that all http.Handlers cannot
// be used with goserv without wrapping them in a goserv.Handler. To make this more convenient
// a function called WrapHTTPHandler exists.
//
//      server.Get("/", WrapHTTPHandler(httpHomeHandler))
//
// Sometimes it is useful to have handlers that are invoked all the time, also known as middleware.
// To register middleware use the Use() function.
//
//      server.Use(accessLogger)
//
// Path Syntax
//
// All Routes and Routers are registered under a path. This path is matched against the path
// of incoming requests.
//
// A Route created from "/mypath" will only match the request path if it is exactly
// "/mypath".
//
// In case that the Route should match everything starting with "/mypath" a wildcard
// can be appended to the path, i.e.
//	/mypath*
//
// The wildcard can be at any position in the path. It also possible to just use the wildcard
// as path.
//
// Sometimes it is necessary to capture values from parts of the request path, so called parameters.
// To include parameters in a Route the path must contain a named parameter, e.g.
//	/users/:user_id
//
// Parameters always start with a ":" after a "/". The name (without the leading ":") can contain
// alphanumeric symbols as well as "_" and "-". The number of parameters in a Route is not limited, but
// they must be separated by at least a single "/".
//	/:value1/:value2
//
// Routers allow you to register handlers for each parameter in a Route. Each handler gets invoked
// once per parameter and request. That means even though a Router may invoked multiple Routes the
// parameter handlers are invoked only once.
//
//      router.Get("/users/:user_id", handler)
//      router.Param("user_id", userIDHandler)
//
// When a Route processes a Request it automatically extracts the captured parameter values from the path
// and stores the values under their name in the .Param field of the Request.
//
// Strict vs non-strict Slash
//
// A Route can have either strict slash or non-strict slash behavior. In non-strict mode paths with or
// without a trailing slash are considered to be the same, i.e. a Route registered with "/mypath" in
// non-strict mode matches both "/mypath" and "/mypath/". In strict mode both
// paths are considered to be different.
//
// Order matters
//
// The order in which Handlers are registered does matter. An incoming request goes through in the
// exact same order. After each handler the Router checks wether an error was set on the
// ResponseWriter or if a response was written and ends the processing if necessary. In case of an error
// the Router forwards the request along with the error to its ErrorHandler, but only if one is available.
// All sub Routers have no ErrorHandler by default, so all errors are handled by the top level Server. It
// is possible though to handle errors in a sub Router by setting a custom ErrorHandler.
//
// A small example shows how requests are processed in a Router:
//
//      server.Use(A)
//      server.Get("/", B)
//      server.Use(C)
//      server.SubRouter("/sub", D)
//
// All incoming request will follow the order "ABCD". The
package goserv
<|endoftext|>


<commit_before>// Package try provides retry functionality.
package try
<commit_msg>added example<commit_after>// Package try provides retry functionality.
//     var value string
//     err := try.Do(func(attempt int) (error, bool) {
//       var err error
//       value, err = SomeFunction()
//       return err, attempt < 5 // try 5 times
//     })
//     if err != nil {
//       log.Fatalln("error:", err)
//     }
package try
<|endoftext|>