<commit_before>package sexp

import (
	"exn"
	"lisp/rt"
)

// Cost returns a value that approximates computational
// complexity of passed argument.
// Works only for expressions.
func Cost(form Form) int {
	// This function contains a large amount of magic numbers.
	// Many aspects can be improved.

	switch form := form.(type) {
	case Bool, Int, Float, Str, Symbol:
		return 1

	case *ArrayLit:
		return cost(form.Vals) + 1

	case *SparseArrayLit:
		valuesCost := 0
		for _, val := range form.Vals {
			valuesCost += Cost(val)
		}
		return valuesCost*3 + 1

	case *SliceLit:
		return cost(form.Vals) + 10

	case *ArrayIndex:
		return Cost(form.Index) + Cost(form.Array) + 1

	case *ArraySlice:
		return Cost(form.Array) + spanCost(form.Span) + 4

	case *SliceIndex:
		return Cost(form.Index) + Cost(form.Slice)*2 + 6

	case *Subslice:
		return Cost(form.Slice) + spanCost(form.Span) + 6

	case *Substr:
		return Cost(form.Str) + spanCost(form.Span) + 1

	case *Call:
		// #FIXME: this is not correct way to calculate
		// function call cost. We need to know function
		// implementation in order to approximate cost better.
		const defaultFnComplexity = 5
		return callCost(form.Args) + defaultFnComplexity

	case *LispCall:
		return cost(form.Args) + 1

	case *InstrCall:
		return cost(form.Args)

	case *StructIndex:
		return structIndexCost(form)

	case Var:
		// #FIXME: dynamic scope variables should have higher cost.
		return 1

	case *Let:
		bindCost := len(form.Bindings) * 2
		for _, bind := range form.Bindings {
			bindCost += Cost(bind.Init)
		}
		if form.Expr == nil {
			return Cost(form.Stmt) + bindCost
		}
		return Cost(form.Expr) + bindCost

	case *Return:
		return cost(form.Results) + len(form.Results)

	default:
		panic(exn.Logic("can not evaluate cost of %#v", form))
	}
}

const baseCallCost = 3

var baseOpCost = [...]int{
	OpShl:    1,
	OpShr:    1,
	OpBitOr:  1,
	OpBitAnd: 1,
	OpBitXor: 1,

	OpStrNeq: 4,
	OpStrLte: 5,
	OpStrGt:  baseCallCost,
	OpStrGte: 5 + baseCallCost,
}

func spanCost(span Span) int {
	cost := 0
	if span.Low != nil {
		cost += Cost(span.Low)
	}
	if span.High != nil {
		cost += Cost(span.High)
	}
	return cost
}

// Function invocation cost.
// Does not account called function complexity.
func callCost(args []Form) int {
	return cost(args) + len(args) + baseCallCost
}

func cost(forms []Form) int {
	total := 0
	for _, form := range forms {
		total += Cost(form)
	}
	return total
}

func structIndexCost(form *StructIndex) int {
	switch rt.StructReprOf(form.Typ) {
	case rt.StructAtom:
		return 1 + Cost(form.Struct)
	case rt.StructCons:
		return form.Index + Cost(form.Struct)
	case rt.StructVec:
		return 3 + Cost(form.Struct)
	default:
		return 0
	}
}
<commit_msg>fix sexp.Cost<commit_after>package sexp

import (
	"exn"
	"lisp/rt"
)

// Cost returns a value that approximates computational
// complexity of passed argument.
// Works only for expressions.
func Cost(form Form) int {
	// This function contains a large amount of magic numbers.
	// Many aspects can be improved.

	switch form := form.(type) {
	case Bool, Int, Float, Str, Symbol:
		return 1

	case *ArrayLit:
		return cost(form.Vals) + 2

	case *SparseArrayLit:
		valuesCost := 0
		for _, val := range form.Vals {
			valuesCost += Cost(val)
		}
		return valuesCost*3 + 2

	case *SliceLit:
		return cost(form.Vals) + 10

	case *StructLit:
		return cost(form.Vals) + 2

	case *ArrayIndex:
		return Cost(form.Index) + Cost(form.Array) + 1

	case *ArraySlice:
		return Cost(form.Array) + spanCost(form.Span) + 4

	case *SliceIndex:
		return Cost(form.Index) + Cost(form.Slice)*2 + 6

	case *Subslice:
		return Cost(form.Slice) + spanCost(form.Span) + 6

	case *Substr:
		return Cost(form.Str) + spanCost(form.Span) + 1

	case *Call:
		// #FIXME: this is not correct way to calculate
		// function call cost. We need to know function
		// implementation in order to approximate cost better.
		const defaultFnComplexity = 5
		return callCost(form.Args) + defaultFnComplexity

	case *LispCall:
		return cost(form.Args) + 1

	case *InstrCall:
		return cost(form.Args)

	case *StructIndex:
		return structIndexCost(form)

	case Var:
		// #FIXME: dynamic scope variables should have higher cost.
		return 1

	case *Let:
		bindCost := len(form.Bindings) * 2
		for _, bind := range form.Bindings {
			bindCost += Cost(bind.Init)
		}
		if form.Expr == nil {
			return Cost(form.Stmt) + bindCost
		}
		return Cost(form.Expr) + bindCost

	case *Return:
		return cost(form.Results) + len(form.Results)

	default:
		panic(exn.Logic("can not evaluate cost of %#v", form))
	}
}

const baseCallCost = 3

var baseOpCost = [...]int{
	OpShl:    1,
	OpShr:    1,
	OpBitOr:  1,
	OpBitAnd: 1,
	OpBitXor: 1,

	OpStrNeq: 4,
	OpStrLte: 5,
	OpStrGt:  baseCallCost,
	OpStrGte: 5 + baseCallCost,
}

func spanCost(span Span) int {
	cost := 0
	if span.Low != nil {
		cost += Cost(span.Low)
	}
	if span.High != nil {
		cost += Cost(span.High)
	}
	return cost
}

// Function invocation cost.
// Does not account called function complexity.
func callCost(args []Form) int {
	return cost(args) + len(args) + baseCallCost
}

func cost(forms []Form) int {
	total := 0
	for _, form := range forms {
		total += Cost(form)
	}
	return total
}

func structIndexCost(form *StructIndex) int {
	switch rt.StructReprOf(form.Typ) {
	case rt.StructAtom:
		return 1 + Cost(form.Struct)
	case rt.StructCons:
		return form.Index + Cost(form.Struct)
	case rt.StructVec:
		return 3 + Cost(form.Struct)
	default:
		return 0
	}
}
<|endoftext|>


<commit_before>package herottp

import (
	"crypto/tls"
	"crypto/x509"
	"io"
	"net"
	"net/http"
	"net/url"
	"time"
)

type Client struct {
	*http.Client
}

type Config struct {
	NoFollowRedirect                  bool
	DisableTLSCertificateVerification bool
	RootCAs                           *x509.CertPool
}

func New(config Config) *Client {
	c := &http.Client{}

	if config.NoFollowRedirect {
		c.CheckRedirect = func(req *http.Request, via []*http.Request) error {
			return noFollowRedirect{}
		}
	}

	transport := &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   30 * time.Second,
			KeepAlive: 30 * time.Second,
		}).Dial,
		TLSHandshakeTimeout: 10 * time.Second,
		TLSClientConfig: &tls.Config{
			InsecureSkipVerify: config.DisableTLSCertificateVerification,
			RootCAs:            config.RootCAs,
		},
	}

	c.Transport = transport

	return &Client{
		Client: c,
	}
}

func (c *Client) Do(req *http.Request) (*http.Response, error) {
	resp, err := c.Client.Do(req)
	if e, isURLErr := err.(*url.Error); isURLErr {
		if _, ok := e.Err.(noFollowRedirect); ok {
			return resp, nil
		}
	}

	return resp, err
}

func (c *Client) Get(url string) (resp *http.Response, err error) {
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}

	return c.Do(req)
}

func (c *Client) Head(url string) (resp *http.Response, err error) {
	req, err := http.NewRequest("HEAD", url, nil)
	if err != nil {
		return nil, err
	}

	return c.Do(req)
}

func (c *Client) Post(url string, bodyType string, body io.Reader) (resp *http.Response, err error) {
	req, err := http.NewRequest("POST", url, body)
	if err != nil {
		return nil, err
	}
	req.Header.Set("Content-Type", bodyType)

	return c.Do(req)
}

type noFollowRedirect struct{}

func (noFollowRedirect) Error() string {
	return "This error should not ever be returned!"
}
<commit_msg>add optional timeout<commit_after>package herottp

import (
	"crypto/tls"
	"crypto/x509"
	"io"
	"net"
	"net/http"
	"net/url"
	"time"
)

type Client struct {
	*http.Client
}

type Config struct {
	NoFollowRedirect                  bool
	DisableTLSCertificateVerification bool
	RootCAs                           *x509.CertPool
	Timeout                           time.Duration
}

func New(config Config) *Client {
	c := &http.Client{Timeout: config.Timeout}

	if config.NoFollowRedirect {
		c.CheckRedirect = func(req *http.Request, via []*http.Request) error {
			return noFollowRedirect{}
		}
	}

	transport := &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   30 * time.Second,
			KeepAlive: 30 * time.Second,
		}).Dial,
		TLSHandshakeTimeout: 10 * time.Second,
		TLSClientConfig: &tls.Config{
			InsecureSkipVerify: config.DisableTLSCertificateVerification,
			RootCAs:            config.RootCAs,
		},
	}

	c.Transport = transport

	return &Client{
		Client: c,
	}
}

func (c *Client) Do(req *http.Request) (*http.Response, error) {
	resp, err := c.Client.Do(req)
	if e, isURLErr := err.(*url.Error); isURLErr {
		if _, ok := e.Err.(noFollowRedirect); ok {
			return resp, nil
		}
	}

	return resp, err
}

func (c *Client) Get(url string) (resp *http.Response, err error) {
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}

	return c.Do(req)
}

func (c *Client) Head(url string) (resp *http.Response, err error) {
	req, err := http.NewRequest("HEAD", url, nil)
	if err != nil {
		return nil, err
	}

	return c.Do(req)
}

func (c *Client) Post(url string, bodyType string, body io.Reader) (resp *http.Response, err error) {
	req, err := http.NewRequest("POST", url, body)
	if err != nil {
		return nil, err
	}
	req.Header.Set("Content-Type", bodyType)

	return c.Do(req)
}

type noFollowRedirect struct{}

func (noFollowRedirect) Error() string {
	return "This error should not ever be returned!"
}
<|endoftext|>


<commit_before>package types

import "fmt"

// Aggregate is a type made up of atomic types
type Aggregate interface {
	String() string
	Position(int) Type
	Equal(n Type) bool
}

// arrayType is the type of llvm arrays
type ArrayType struct {
	baseType Type
	count    int
}

// NewArrayType creates a new array type
func NewArrayType(baseType Type, count int) Aggregate {
	return ArrayType{baseType, count}
}

func (t ArrayType) String() string {
	return fmt.Sprintf("[%d x %s]", t.count, t.baseType.String())
}

// Position returns the type at position n
func (t ArrayType) Position(n int) Type {
	return t.baseType
}

// Equal checks if type n equals type t
func (t ArrayType) Equal(n Type) bool {
	arr, ok := n.(ArrayType)
	if !ok {
		return false
	}

	return t.baseType == arr.baseType && t.count == arr.count
}

// StructType represents a collection of types
type StructType struct {
	types []Type
}

// NewStructType creates a new struct type
func NewStructType(types ...Type) Aggregate {
	return StructType{types}
}

func (t StructType) String() string {
	s := "{ "
	for i, structType := range t.types {
		s += structType.String()
		if i < len(t.types)-1 {
			s += ", "
		}
	}

	return s + " }"
}

// Position returns the type at position n
func (t StructType) Position(n int) Type {
	return t.types[n]
}

// Equal checks if type n equals type t
func (t StructType) Equal(n Type) bool {
	str, ok := n.(StructType)
	if !ok {
		return false
	}

	for i := 0; i < len(t.types); i++ {
		if !t.types[i].Equal(str.types[i]) {
			return false
		}
	}

	return true
}
<commit_msg>Added basetype to array type.<commit_after>package types

import "fmt"

// Aggregate is a type made up of atomic types
type Aggregate interface {
	String() string
	Position(int) Type
	Equal(n Type) bool
}

// arrayType is the type of llvm arrays
type ArrayType struct {
	baseType Type
	count    int
}

// NewArrayType creates a new array type
func NewArrayType(baseType Type, count int) Aggregate {
	return ArrayType{baseType, count}
}

// BaseType returns the bast type of the array type
func (t ArrayType) BaseType() Type {
	return t.baseType
}

func (t ArrayType) String() string {
	return fmt.Sprintf("[%d x %s]", t.count, t.baseType.String())
}

// Position returns the type at position n
func (t ArrayType) Position(n int) Type {
	return t.baseType
}

// Equal checks if type n equals type t
func (t ArrayType) Equal(n Type) bool {
	arr, ok := n.(ArrayType)
	if !ok {
		return false
	}

	return t.baseType == arr.baseType && t.count == arr.count
}

// StructType represents a collection of types
type StructType struct {
	types []Type
}

// NewStructType creates a new struct type
func NewStructType(types ...Type) Aggregate {
	return StructType{types}
}

func (t StructType) String() string {
	s := "{ "
	for i, structType := range t.types {
		s += structType.String()
		if i < len(t.types)-1 {
			s += ", "
		}
	}

	return s + " }"
}

// Position returns the type at position n
func (t StructType) Position(n int) Type {
	return t.types[n]
}

// Equal checks if type n equals type t
func (t StructType) Equal(n Type) bool {
	str, ok := n.(StructType)
	if !ok {
		return false
	}

	for i := 0; i < len(t.types); i++ {
		if !t.types[i].Equal(str.types[i]) {
			return false
		}
	}

	return true
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package sha512 implements the SHA384 and SHA512 hash algorithms as defined
// in FIPS 180-2.
package sha512

import (
	"crypto"
	"hash"
)

func init() {
	crypto.RegisterHash(crypto.SHA384, New384)
	crypto.RegisterHash(crypto.SHA512, New)
}

// The size of a SHA512 checksum in bytes.
const Size = 64

// The size of a SHA384 checksum in bytes.
const Size384 = 48

// The blocksize of SHA512 and SHA384 in bytes.
const BlockSize = 128

const (
	chunk     = 128
	init0     = 0x6a09e667f3bcc908
	init1     = 0xbb67ae8584caa73b
	init2     = 0x3c6ef372fe94f82b
	init3     = 0xa54ff53a5f1d36f1
	init4     = 0x510e527fade682d1
	init5     = 0x9b05688c2b3e6c1f
	init6     = 0x1f83d9abfb41bd6b
	init7     = 0x5be0cd19137e2179
	init0_384 = 0xcbbb9d5dc1059ed8
	init1_384 = 0x629a292a367cd507
	init2_384 = 0x9159015a3070dd17
	init3_384 = 0x152fecd8f70e5939
	init4_384 = 0x67332667ffc00b31
	init5_384 = 0x8eb44a8768581511
	init6_384 = 0xdb0c2e0d64f98fa7
	init7_384 = 0x47b5481dbefa4fa4
)

// digest represents the partial evaluation of a checksum.
type digest struct {
	h     [8]uint64
	x     [chunk]byte
	nx    int
	len   uint64
	is384 bool // mark if this digest is SHA-384
}

func (d *digest) Reset() {
	if !d.is384 {
		d.h[0] = init0
		d.h[1] = init1
		d.h[2] = init2
		d.h[3] = init3
		d.h[4] = init4
		d.h[5] = init5
		d.h[6] = init6
		d.h[7] = init7
	} else {
		d.h[0] = init0_384
		d.h[1] = init1_384
		d.h[2] = init2_384
		d.h[3] = init3_384
		d.h[4] = init4_384
		d.h[5] = init5_384
		d.h[6] = init6_384
		d.h[7] = init7_384
	}
	d.nx = 0
	d.len = 0
}

// New returns a new hash.Hash computing the SHA512 checksum.
func New() hash.Hash {
	d := new(digest)
	d.Reset()
	return d
}

// New384 returns a new hash.Hash computing the SHA384 checksum.
func New384() hash.Hash {
	d := new(digest)
	d.is384 = true
	d.Reset()
	return d
}

func (d *digest) Size() int {
	if !d.is384 {
		return Size
	}
	return Size384
}

func (d *digest) BlockSize() int { return BlockSize }

func (d *digest) Write(p []byte) (nn int, err error) {
	nn = len(p)
	d.len += uint64(nn)
	if d.nx > 0 {
		n := len(p)
		if n > chunk-d.nx {
			n = chunk - d.nx
		}
		for i := 0; i < n; i++ {
			d.x[d.nx+i] = p[i]
		}
		d.nx += n
		if d.nx == chunk {
			block(d, d.x[0:])
			d.nx = 0
		}
		p = p[n:]
	}
	if len(p) >= chunk {
		n := len(p) &^ (chunk - 1)
		block(d, p[:n])
		p = p[n:]
	}
	if len(p) > 0 {
		d.nx = copy(d.x[:], p)
	}
	return
}

func (d0 *digest) Sum(in []byte) []byte {
	// Make a copy of d0 so that caller can keep writing and summing.
	d := new(digest)
	*d = *d0
	hash := d.checkSum()
	if d.is384 {
		return append(in, hash[:Size384]...)
	}
	return append(in, hash[:]...)
}

func (d *digest) checkSum() [Size]byte {
	// Padding.  Add a 1 bit and 0 bits until 112 bytes mod 128.
	len := d.len
	var tmp [128]byte
	tmp[0] = 0x80
	if len%128 < 112 {
		d.Write(tmp[0 : 112-len%128])
	} else {
		d.Write(tmp[0 : 128+112-len%128])
	}

	// Length in bits.
	len <<= 3
	for i := uint(0); i < 16; i++ {
		tmp[i] = byte(len >> (120 - 8*i))
	}
	d.Write(tmp[0:16])

	if d.nx != 0 {
		panic("d.nx != 0")
	}

	h := d.h[:]
	if d.is384 {
		h = d.h[:6]
	}

	var digest [Size]byte
	for i, s := range h {
		digest[i*8] = byte(s >> 56)
		digest[i*8+1] = byte(s >> 48)
		digest[i*8+2] = byte(s >> 40)
		digest[i*8+3] = byte(s >> 32)
		digest[i*8+4] = byte(s >> 24)
		digest[i*8+5] = byte(s >> 16)
		digest[i*8+6] = byte(s >> 8)
		digest[i*8+7] = byte(s)
	}

	return digest
}

// Sum returns the SHA512 checksum of the data.
func Sum512(data []byte) [Size]byte {
	var d digest
	d.Reset()
	d.Write(data)
	return d.checkSum()
}

// Sum384 returns the SHA384 checksum of the data.
func Sum384(data []byte) (sum384 [Size384]byte) {
	var d digest
	d.is384 = true
	d.Reset()
	d.Write(data)
	sum := d.checkSum()
	copy(sum384[:], sum[:Size384])
	return
}
<commit_msg>crypto/sha512: update comment for the Sum512 function.<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package sha512 implements the SHA384 and SHA512 hash algorithms as defined
// in FIPS 180-2.
package sha512

import (
	"crypto"
	"hash"
)

func init() {
	crypto.RegisterHash(crypto.SHA384, New384)
	crypto.RegisterHash(crypto.SHA512, New)
}

// The size of a SHA512 checksum in bytes.
const Size = 64

// The size of a SHA384 checksum in bytes.
const Size384 = 48

// The blocksize of SHA512 and SHA384 in bytes.
const BlockSize = 128

const (
	chunk     = 128
	init0     = 0x6a09e667f3bcc908
	init1     = 0xbb67ae8584caa73b
	init2     = 0x3c6ef372fe94f82b
	init3     = 0xa54ff53a5f1d36f1
	init4     = 0x510e527fade682d1
	init5     = 0x9b05688c2b3e6c1f
	init6     = 0x1f83d9abfb41bd6b
	init7     = 0x5be0cd19137e2179
	init0_384 = 0xcbbb9d5dc1059ed8
	init1_384 = 0x629a292a367cd507
	init2_384 = 0x9159015a3070dd17
	init3_384 = 0x152fecd8f70e5939
	init4_384 = 0x67332667ffc00b31
	init5_384 = 0x8eb44a8768581511
	init6_384 = 0xdb0c2e0d64f98fa7
	init7_384 = 0x47b5481dbefa4fa4
)

// digest represents the partial evaluation of a checksum.
type digest struct {
	h     [8]uint64
	x     [chunk]byte
	nx    int
	len   uint64
	is384 bool // mark if this digest is SHA-384
}

func (d *digest) Reset() {
	if !d.is384 {
		d.h[0] = init0
		d.h[1] = init1
		d.h[2] = init2
		d.h[3] = init3
		d.h[4] = init4
		d.h[5] = init5
		d.h[6] = init6
		d.h[7] = init7
	} else {
		d.h[0] = init0_384
		d.h[1] = init1_384
		d.h[2] = init2_384
		d.h[3] = init3_384
		d.h[4] = init4_384
		d.h[5] = init5_384
		d.h[6] = init6_384
		d.h[7] = init7_384
	}
	d.nx = 0
	d.len = 0
}

// New returns a new hash.Hash computing the SHA512 checksum.
func New() hash.Hash {
	d := new(digest)
	d.Reset()
	return d
}

// New384 returns a new hash.Hash computing the SHA384 checksum.
func New384() hash.Hash {
	d := new(digest)
	d.is384 = true
	d.Reset()
	return d
}

func (d *digest) Size() int {
	if !d.is384 {
		return Size
	}
	return Size384
}

func (d *digest) BlockSize() int { return BlockSize }

func (d *digest) Write(p []byte) (nn int, err error) {
	nn = len(p)
	d.len += uint64(nn)
	if d.nx > 0 {
		n := len(p)
		if n > chunk-d.nx {
			n = chunk - d.nx
		}
		for i := 0; i < n; i++ {
			d.x[d.nx+i] = p[i]
		}
		d.nx += n
		if d.nx == chunk {
			block(d, d.x[0:])
			d.nx = 0
		}
		p = p[n:]
	}
	if len(p) >= chunk {
		n := len(p) &^ (chunk - 1)
		block(d, p[:n])
		p = p[n:]
	}
	if len(p) > 0 {
		d.nx = copy(d.x[:], p)
	}
	return
}

func (d0 *digest) Sum(in []byte) []byte {
	// Make a copy of d0 so that caller can keep writing and summing.
	d := new(digest)
	*d = *d0
	hash := d.checkSum()
	if d.is384 {
		return append(in, hash[:Size384]...)
	}
	return append(in, hash[:]...)
}

func (d *digest) checkSum() [Size]byte {
	// Padding.  Add a 1 bit and 0 bits until 112 bytes mod 128.
	len := d.len
	var tmp [128]byte
	tmp[0] = 0x80
	if len%128 < 112 {
		d.Write(tmp[0 : 112-len%128])
	} else {
		d.Write(tmp[0 : 128+112-len%128])
	}

	// Length in bits.
	len <<= 3
	for i := uint(0); i < 16; i++ {
		tmp[i] = byte(len >> (120 - 8*i))
	}
	d.Write(tmp[0:16])

	if d.nx != 0 {
		panic("d.nx != 0")
	}

	h := d.h[:]
	if d.is384 {
		h = d.h[:6]
	}

	var digest [Size]byte
	for i, s := range h {
		digest[i*8] = byte(s >> 56)
		digest[i*8+1] = byte(s >> 48)
		digest[i*8+2] = byte(s >> 40)
		digest[i*8+3] = byte(s >> 32)
		digest[i*8+4] = byte(s >> 24)
		digest[i*8+5] = byte(s >> 16)
		digest[i*8+6] = byte(s >> 8)
		digest[i*8+7] = byte(s)
	}

	return digest
}

// Sum512 returns the SHA512 checksum of the data.
func Sum512(data []byte) [Size]byte {
	var d digest
	d.Reset()
	d.Write(data)
	return d.checkSum()
}

// Sum384 returns the SHA384 checksum of the data.
func Sum384(data []byte) (sum384 [Size384]byte) {
	var d digest
	d.is384 = true
	d.Reset()
	d.Write(data)
	sum := d.checkSum()
	copy(sum384[:], sum[:Size384])
	return
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gob

import (
	"bufio"
	"bytes"
	"errors"
	"io"
	"reflect"
	"sync"
)

// A Decoder manages the receipt of type and data information read from the
// remote side of a connection.
type Decoder struct {
	mutex        sync.Mutex                              // each item must be received atomically
	r            io.Reader                               // source of the data
	buf          bytes.Buffer                            // buffer for more efficient i/o from r
	wireType     map[typeId]*wireType                    // map from remote ID to local description
	decoderCache map[reflect.Type]map[typeId]**decEngine // cache of compiled engines
	ignorerCache map[typeId]**decEngine                  // ditto for ignored objects
	freeList     *decoderState                           // list of free decoderStates; avoids reallocation
	countBuf     []byte                                  // used for decoding integers while parsing messages
	tmp          []byte                                  // temporary storage for i/o; saves reallocating
	err          error
}

// NewDecoder returns a new decoder that reads from the io.Reader.
// If r does not also implement io.ByteReader, it will be wrapped in a
// bufio.Reader.
func NewDecoder(r io.Reader) *Decoder {
	dec := new(Decoder)
	// We use the ability to read bytes as a plausible surrogate for buffering.
	if _, ok := r.(io.ByteReader); !ok {
		r = bufio.NewReader(r)
	}
	dec.r = r
	dec.wireType = make(map[typeId]*wireType)
	dec.decoderCache = make(map[reflect.Type]map[typeId]**decEngine)
	dec.ignorerCache = make(map[typeId]**decEngine)
	dec.countBuf = make([]byte, 9) // counts may be uint64s (unlikely!), require 9 bytes

	return dec
}

// recvType loads the definition of a type.
func (dec *Decoder) recvType(id typeId) {
	// Have we already seen this type?  That's an error
	if id < firstUserId || dec.wireType[id] != nil {
		dec.err = errors.New("gob: duplicate type received")
		return
	}

	// Type:
	wire := new(wireType)
	dec.decodeValue(tWireType, reflect.ValueOf(wire))
	if dec.err != nil {
		return
	}
	// Remember we've seen this type.
	dec.wireType[id] = wire
}

var errBadCount = errors.New("invalid message length")

// recvMessage reads the next count-delimited item from the input. It is the converse
// of Encoder.writeMessage. It returns false on EOF or other error reading the message.
func (dec *Decoder) recvMessage() bool {
	// Read a count.
	nbytes, _, err := decodeUintReader(dec.r, dec.countBuf)
	if err != nil {
		dec.err = err
		return false
	}
	// Upper limit of 1GB, allowing room to grow a little without overflow.
	// TODO: We might want more control over this limit.
	if nbytes >= 1<<30 {
		dec.err = errBadCount
		return false
	}
	dec.readMessage(int(nbytes))
	return dec.err == nil
}

// readMessage reads the next nbytes bytes from the input.
func (dec *Decoder) readMessage(nbytes int) {
	// Allocate the dec.tmp buffer, up to 10KB.
	const maxBuf = 10 * 1024
	nTmp := nbytes
	if nTmp > maxBuf {
		nTmp = maxBuf
	}
	if cap(dec.tmp) < nTmp {
		nAlloc := nTmp + 100 // A little extra for growth.
		if nAlloc > maxBuf {
			nAlloc = maxBuf
		}
		dec.tmp = make([]byte, nAlloc)
	}
	dec.tmp = dec.tmp[:nTmp]

	// Read the data
	dec.buf.Grow(nbytes)
	for nbytes > 0 {
		if nbytes < nTmp {
			dec.tmp = dec.tmp[:nbytes]
		}
		var nRead int
		nRead, dec.err = io.ReadFull(dec.r, dec.tmp)
		if dec.err != nil {
			if dec.err == io.EOF {
				dec.err = io.ErrUnexpectedEOF
			}
			return
		}
		dec.buf.Write(dec.tmp)
		nbytes -= nRead
	}
}

// toInt turns an encoded uint64 into an int, according to the marshaling rules.
func toInt(x uint64) int64 {
	i := int64(x >> 1)
	if x&1 != 0 {
		i = ^i
	}
	return i
}

func (dec *Decoder) nextInt() int64 {
	n, _, err := decodeUintReader(&dec.buf, dec.countBuf)
	if err != nil {
		dec.err = err
	}
	return toInt(n)
}

func (dec *Decoder) nextUint() uint64 {
	n, _, err := decodeUintReader(&dec.buf, dec.countBuf)
	if err != nil {
		dec.err = err
	}
	return n
}

// decodeTypeSequence parses:
// TypeSequence
//	(TypeDefinition DelimitedTypeDefinition*)?
// and returns the type id of the next value.  It returns -1 at
// EOF.  Upon return, the remainder of dec.buf is the value to be
// decoded.  If this is an interface value, it can be ignored by
// resetting that buffer.
func (dec *Decoder) decodeTypeSequence(isInterface bool) typeId {
	for dec.err == nil {
		if dec.buf.Len() == 0 {
			if !dec.recvMessage() {
				break
			}
		}
		// Receive a type id.
		id := typeId(dec.nextInt())
		if id >= 0 {
			// Value follows.
			return id
		}
		// Type definition for (-id) follows.
		dec.recvType(-id)
		// When decoding an interface, after a type there may be a
		// DelimitedValue still in the buffer.  Skip its count.
		// (Alternatively, the buffer is empty and the byte count
		// will be absorbed by recvMessage.)
		if dec.buf.Len() > 0 {
			if !isInterface {
				dec.err = errors.New("extra data in buffer")
				break
			}
			dec.nextUint()
		}
	}
	return -1
}

// Decode reads the next value from the connection and stores
// it in the data represented by the empty interface value.
// If e is nil, the value will be discarded. Otherwise,
// the value underlying e must be a pointer to the
// correct type for the next data item received.
func (dec *Decoder) Decode(e interface{}) error {
	if e == nil {
		return dec.DecodeValue(reflect.Value{})
	}
	value := reflect.ValueOf(e)
	// If e represents a value as opposed to a pointer, the answer won't
	// get back to the caller.  Make sure it's a pointer.
	if value.Type().Kind() != reflect.Ptr {
		dec.err = errors.New("gob: attempt to decode into a non-pointer")
		return dec.err
	}
	return dec.DecodeValue(value)
}

// DecodeValue reads the next value from the connection.
// If v is the zero reflect.Value (v.Kind() == Invalid), DecodeValue discards the value.
// Otherwise, it stores the value into v.  In that case, v must represent
// a non-nil pointer to data or be an assignable reflect.Value (v.CanSet())
func (dec *Decoder) DecodeValue(v reflect.Value) error {
	if v.IsValid() {
		if v.Kind() == reflect.Ptr && !v.IsNil() {
			// That's okay, we'll store through the pointer.
		} else if !v.CanSet() {
			return errors.New("gob: DecodeValue of unassignable value")
		}
	}
	// Make sure we're single-threaded through here.
	dec.mutex.Lock()
	defer dec.mutex.Unlock()

	dec.buf.Reset() // In case data lingers from previous invocation.
	dec.err = nil
	id := dec.decodeTypeSequence(false)
	if dec.err == nil {
		dec.decodeValue(id, v)
	}
	return dec.err
}

// If debug.go is compiled into the program , debugFunc prints a human-readable
// representation of the gob data read from r by calling that file's Debug function.
// Otherwise it is nil.
var debugFunc func(io.Reader)
<commit_msg>encoding/gob: document that Decode returns EOF at EOF<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gob

import (
	"bufio"
	"bytes"
	"errors"
	"io"
	"reflect"
	"sync"
)

// A Decoder manages the receipt of type and data information read from the
// remote side of a connection.
type Decoder struct {
	mutex        sync.Mutex                              // each item must be received atomically
	r            io.Reader                               // source of the data
	buf          bytes.Buffer                            // buffer for more efficient i/o from r
	wireType     map[typeId]*wireType                    // map from remote ID to local description
	decoderCache map[reflect.Type]map[typeId]**decEngine // cache of compiled engines
	ignorerCache map[typeId]**decEngine                  // ditto for ignored objects
	freeList     *decoderState                           // list of free decoderStates; avoids reallocation
	countBuf     []byte                                  // used for decoding integers while parsing messages
	tmp          []byte                                  // temporary storage for i/o; saves reallocating
	err          error
}

// NewDecoder returns a new decoder that reads from the io.Reader.
// If r does not also implement io.ByteReader, it will be wrapped in a
// bufio.Reader.
func NewDecoder(r io.Reader) *Decoder {
	dec := new(Decoder)
	// We use the ability to read bytes as a plausible surrogate for buffering.
	if _, ok := r.(io.ByteReader); !ok {
		r = bufio.NewReader(r)
	}
	dec.r = r
	dec.wireType = make(map[typeId]*wireType)
	dec.decoderCache = make(map[reflect.Type]map[typeId]**decEngine)
	dec.ignorerCache = make(map[typeId]**decEngine)
	dec.countBuf = make([]byte, 9) // counts may be uint64s (unlikely!), require 9 bytes

	return dec
}

// recvType loads the definition of a type.
func (dec *Decoder) recvType(id typeId) {
	// Have we already seen this type?  That's an error
	if id < firstUserId || dec.wireType[id] != nil {
		dec.err = errors.New("gob: duplicate type received")
		return
	}

	// Type:
	wire := new(wireType)
	dec.decodeValue(tWireType, reflect.ValueOf(wire))
	if dec.err != nil {
		return
	}
	// Remember we've seen this type.
	dec.wireType[id] = wire
}

var errBadCount = errors.New("invalid message length")

// recvMessage reads the next count-delimited item from the input. It is the converse
// of Encoder.writeMessage. It returns false on EOF or other error reading the message.
func (dec *Decoder) recvMessage() bool {
	// Read a count.
	nbytes, _, err := decodeUintReader(dec.r, dec.countBuf)
	if err != nil {
		dec.err = err
		return false
	}
	// Upper limit of 1GB, allowing room to grow a little without overflow.
	// TODO: We might want more control over this limit.
	if nbytes >= 1<<30 {
		dec.err = errBadCount
		return false
	}
	dec.readMessage(int(nbytes))
	return dec.err == nil
}

// readMessage reads the next nbytes bytes from the input.
func (dec *Decoder) readMessage(nbytes int) {
	// Allocate the dec.tmp buffer, up to 10KB.
	const maxBuf = 10 * 1024
	nTmp := nbytes
	if nTmp > maxBuf {
		nTmp = maxBuf
	}
	if cap(dec.tmp) < nTmp {
		nAlloc := nTmp + 100 // A little extra for growth.
		if nAlloc > maxBuf {
			nAlloc = maxBuf
		}
		dec.tmp = make([]byte, nAlloc)
	}
	dec.tmp = dec.tmp[:nTmp]

	// Read the data
	dec.buf.Grow(nbytes)
	for nbytes > 0 {
		if nbytes < nTmp {
			dec.tmp = dec.tmp[:nbytes]
		}
		var nRead int
		nRead, dec.err = io.ReadFull(dec.r, dec.tmp)
		if dec.err != nil {
			if dec.err == io.EOF {
				dec.err = io.ErrUnexpectedEOF
			}
			return
		}
		dec.buf.Write(dec.tmp)
		nbytes -= nRead
	}
}

// toInt turns an encoded uint64 into an int, according to the marshaling rules.
func toInt(x uint64) int64 {
	i := int64(x >> 1)
	if x&1 != 0 {
		i = ^i
	}
	return i
}

func (dec *Decoder) nextInt() int64 {
	n, _, err := decodeUintReader(&dec.buf, dec.countBuf)
	if err != nil {
		dec.err = err
	}
	return toInt(n)
}

func (dec *Decoder) nextUint() uint64 {
	n, _, err := decodeUintReader(&dec.buf, dec.countBuf)
	if err != nil {
		dec.err = err
	}
	return n
}

// decodeTypeSequence parses:
// TypeSequence
//	(TypeDefinition DelimitedTypeDefinition*)?
// and returns the type id of the next value.  It returns -1 at
// EOF.  Upon return, the remainder of dec.buf is the value to be
// decoded.  If this is an interface value, it can be ignored by
// resetting that buffer.
func (dec *Decoder) decodeTypeSequence(isInterface bool) typeId {
	for dec.err == nil {
		if dec.buf.Len() == 0 {
			if !dec.recvMessage() {
				break
			}
		}
		// Receive a type id.
		id := typeId(dec.nextInt())
		if id >= 0 {
			// Value follows.
			return id
		}
		// Type definition for (-id) follows.
		dec.recvType(-id)
		// When decoding an interface, after a type there may be a
		// DelimitedValue still in the buffer.  Skip its count.
		// (Alternatively, the buffer is empty and the byte count
		// will be absorbed by recvMessage.)
		if dec.buf.Len() > 0 {
			if !isInterface {
				dec.err = errors.New("extra data in buffer")
				break
			}
			dec.nextUint()
		}
	}
	return -1
}

// Decode reads the next value from the input stream and stores
// it in the data represented by the empty interface value.
// If e is nil, the value will be discarded. Otherwise,
// the value underlying e must be a pointer to the
// correct type for the next data item received.
// If the input is at EOF, Decode returns io.EOF and
// does not modify e.
func (dec *Decoder) Decode(e interface{}) error {
	if e == nil {
		return dec.DecodeValue(reflect.Value{})
	}
	value := reflect.ValueOf(e)
	// If e represents a value as opposed to a pointer, the answer won't
	// get back to the caller.  Make sure it's a pointer.
	if value.Type().Kind() != reflect.Ptr {
		dec.err = errors.New("gob: attempt to decode into a non-pointer")
		return dec.err
	}
	return dec.DecodeValue(value)
}

// DecodeValue reads the next value from the input stream.
// If v is the zero reflect.Value (v.Kind() == Invalid), DecodeValue discards the value.
// Otherwise, it stores the value into v.  In that case, v must represent
// a non-nil pointer to data or be an assignable reflect.Value (v.CanSet())
// If the input is at EOF, DecodeValue returns io.EOF and
// does not modify e.
func (dec *Decoder) DecodeValue(v reflect.Value) error {
	if v.IsValid() {
		if v.Kind() == reflect.Ptr && !v.IsNil() {
			// That's okay, we'll store through the pointer.
		} else if !v.CanSet() {
			return errors.New("gob: DecodeValue of unassignable value")
		}
	}
	// Make sure we're single-threaded through here.
	dec.mutex.Lock()
	defer dec.mutex.Unlock()

	dec.buf.Reset() // In case data lingers from previous invocation.
	dec.err = nil
	id := dec.decodeTypeSequence(false)
	if dec.err == nil {
		dec.decodeValue(id, v)
	}
	return dec.err
}

// If debug.go is compiled into the program , debugFunc prints a human-readable
// representation of the gob data read from r by calling that file's Debug function.
// Otherwise it is nil.
var debugFunc func(io.Reader)
<|endoftext|>


<commit_before>// Copyright 2016 The Upspin Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gcs

import (
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strings"
	"testing"
	"time"

	"upspin.io/cloud/storage"
	"upspin.io/log"
)

const (
	testProjectID  = "upspin"
	testBucketName = "upspin-test"
)

var (
	client      storage.Storage
	testDataStr = fmt.Sprintf("This is test at %v", time.Now())
	testData    = []byte(testDataStr)
	fileName    = "test-file"
)

// This is more of a regression test as it uses the running cloud
// storage in prod. However, since GCP is always available, we accept
// to rely on it.
func TestPutGetAndDownload(t *testing.T) {
	link, err := client.Put(fileName, testData)
	if err != nil {
		t.Fatalf("Can't put: %v", err)
	}
	if !strings.HasPrefix(link, "https://") {
		t.Errorf("Link is not HTTPS")
	}
	retLink, err := client.Get(fileName)
	if err != nil {
		t.Fatalf("Can't get: %v", err)
	}
	if retLink != link {
		t.Errorf("Not the same link as stored: %v vs received: %v", link, retLink)
	}
	resp, err := http.Get(retLink)
	if err != nil {
		t.Errorf("Couldn't get link: %v", err)
	}
	defer resp.Body.Close()
	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		t.Fatalf("Can't read HTTP body: %v", err)
	}
	if string(data) != testDataStr {
		t.Errorf("Expected %q got %q", testDataStr, string(data))
	}
	// Check that Download yields the same data
	bytes, err := client.Download(fileName)
	if err != nil {
		t.Fatal(err)
	}
	if string(bytes) != testDataStr {
		t.Errorf("Expected %q got %q", testDataStr, string(bytes))
	}
}

func TestPutLocalFile(t *testing.T) {
	// Create a temporary local file.
	f, err := ioutil.TempFile("", "test-gcp-")
	if err != nil {
		t.Fatal(err)
	}
	defer f.Close()
	localName := f.Name()
	defer os.Remove(localName)
	n, err := f.Write(testData)
	if err != nil {
		t.Fatal(err)
	}
	if n != len(testData) {
		t.Fatalf("Expected to write %d bytes, got %d", len(testData), n)
	}
	// Put to GCP
	const testFileName = "a-new-name"
	link, err := client.PutLocalFile(localName, testFileName)
	if err != nil {
		t.Fatal(err)
	}
	// Check that we get the same link back from Get.
	retLink, err := client.Get(testFileName)
	if err != nil {
		t.Fatal(err)
	}
	if retLink != link {
		t.Errorf("Not the same link as stored: %v vs received: %v", link, retLink)
	}
	resp, err := http.Get(retLink)
	if err != nil {
		t.Fatal(err)
	}
	// Download contents
	defer resp.Body.Close()
	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		t.Fatalf("Can't read HTTP body: %v", err)
	}
	if string(data) != testDataStr {
		t.Errorf("Expected %q got %q", testDataStr, string(data))
	}
	// Clean up
	err = client.Delete(testFileName)
	if err != nil {
		t.Fatal(err)
	}
}

func Put(t *testing.T, path string) {
	_, err := client.Put(path, testData)
	if err != nil {
		t.Fatal(err)
	}
}

func setupDirectoryTree(t *testing.T) {
	Put(t, "a")
	Put(t, "a/b")
	Put(t, "a/b/c")
	Put(t, "a/b/c/d1")
	Put(t, "a/b/c/d2")
	Put(t, "a/b/c/d1/e11")
	Put(t, "a/b/c/d1/e12")
	Put(t, "a/b/c/d1/e13")
	Put(t, "a/b/c/d2/e21")
	Put(t, "a/b/c/d2/e22")
	Put(t, "a/b/c/d2/e23")
}

func TestList(t *testing.T) {
	setupDirectoryTree(t)
	type testResult struct {
		prefix   string
		depth    int
		expected []string
	}

	tests := []testResult{
		testResult{
			prefix: "a/b",
			depth:  0,
			expected: []string{
				"a/b",
			},
		},
		testResult{
			prefix: "a/b",
			depth:  1,
			expected: []string{
				"a/b",
				"a/b/c",
			},
		},
		testResult{
			prefix: "a/b",
			depth:  2,
			expected: []string{
				"a/b",
				"a/b/c",
				"a/b/c/d1",
				"a/b/c/d2",
			},
		},
		testResult{
			prefix: "a/b",
			depth:  3,
			expected: []string{
				"a/b",
				"a/b/c",
				"a/b/c/d1",
				"a/b/c/d1/e11",
				"a/b/c/d1/e12",
				"a/b/c/d1/e13",
				"a/b/c/d2",
				"a/b/c/d2/e21",
				"a/b/c/d2/e22",
				"a/b/c/d2/e23",
			},
		},
		testResult{
			prefix: "a",
			depth:  4,
			expected: []string{
				"a",
				"a/b",
				"a/b/c",
				"a/b/c/d1",
				"a/b/c/d1/e11",
				"a/b/c/d1/e12",
				"a/b/c/d1/e13",
				"a/b/c/d2",
				"a/b/c/d2/e21",
				"a/b/c/d2/e22",
				"a/b/c/d2/e23",
			},
		},
	}
	for i := range tests {
		testListPrefix(t, tests[i].prefix, tests[i].depth, tests[i].expected)
	}
}

func contains(needle string, haystack []string) bool {
	for _, elem := range haystack {
		if elem == needle {
			return true
		}
	}
	return false
}

func testListPrefix(t *testing.T, prefix string, depth int, expected []string) {
	names, err := client.ListPrefix(prefix, depth)
	if err != nil {
		t.Fatalf("Error in client.List: %v", err)
	}
	t.Logf("Prefix: got: %+v", names)
	if len(names) != len(expected) {
		t.Fatalf("Expected %d results, got %d", len(expected), len(names))
	}
	for _, e := range expected {
		if !contains(e, names) {
			t.Errorf("%s %d: %q not found", prefix, depth, e)
		}
	}
}

func TestListDir(t *testing.T) {
	prefix := "a/b/c/d1/"
	names, err := client.ListDir(prefix)
	if err != nil {
		t.Fatalf("Error in client.List: %v", err)
	}
	expected := []string{prefix + "e11", prefix + "e12", prefix + "e13"}
	if len(names) != len(expected) {
		t.Fatalf("Expected 3 results, got %d", len(names))
	}
	for _, e := range expected {
		if !contains(e, names) {
			t.Errorf("Expected %q, not found", e)
		}
	}
}

func TestDelete(t *testing.T) {
	_, err := client.Put(fileName, testData)
	if err != nil {
		t.Fatal(err)
	}
	err = client.Delete(fileName)
	if err != nil {
		t.Fatalf("Expected no errors, got %v", err)
	}
	// Test the side effect after Delete.
	_, err = client.Get(fileName)
	if err == nil {
		t.Fatal("Expected an error, but got none")
	}
}

func TestMain(m *testing.M) {
	flag.Parse()
	if testing.Short() {
		log.Print("cloud/storage/gcs: skipping network-based tests while -test.short specified")
		os.Exit(0)
	}

	// Create client that writes to test bucket.
	var err error
	client, err = storage.Dial("GCS",
		storage.WithKeyValue("gcpProjectId", testProjectID),
		storage.WithKeyValue("gcpBucketName", testBucketName),
		storage.WithKeyValue("defaultACL", PublicRead))
	if err != nil {
		log.Fatalf("cloud/storage/gcs: couldn't set up client: %v", err)
	}

	code := m.Run()

	// Clean up.
	const verbose = true
	if err := client.(*gcsImpl).EmptyBucket(verbose); err != nil {
		log.Print("cloud/storage/gcs: EmptyBucket failed: %v", err)
	}

	os.Exit(code)
}
<commit_msg>cloud/storage: fix test bucket and project<commit_after>// Copyright 2016 The Upspin Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gcs

import (
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strings"
	"testing"
	"time"

	"upspin.io/cloud/storage"
	"upspin.io/log"
)

const (
	testProjectID  = "upspin-test"
	testBucketName = "upspin-test-scratch"
)

var (
	client      storage.Storage
	testDataStr = fmt.Sprintf("This is test at %v", time.Now())
	testData    = []byte(testDataStr)
	fileName    = "test-file"
)

// This is more of a regression test as it uses the running cloud
// storage in prod. However, since GCP is always available, we accept
// to rely on it.
func TestPutGetAndDownload(t *testing.T) {
	link, err := client.Put(fileName, testData)
	if err != nil {
		t.Fatalf("Can't put: %v", err)
	}
	if !strings.HasPrefix(link, "https://") {
		t.Errorf("Link is not HTTPS")
	}
	retLink, err := client.Get(fileName)
	if err != nil {
		t.Fatalf("Can't get: %v", err)
	}
	if retLink != link {
		t.Errorf("Not the same link as stored: %v vs received: %v", link, retLink)
	}
	resp, err := http.Get(retLink)
	if err != nil {
		t.Errorf("Couldn't get link: %v", err)
	}
	defer resp.Body.Close()
	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		t.Fatalf("Can't read HTTP body: %v", err)
	}
	if string(data) != testDataStr {
		t.Errorf("Expected %q got %q", testDataStr, string(data))
	}
	// Check that Download yields the same data
	bytes, err := client.Download(fileName)
	if err != nil {
		t.Fatal(err)
	}
	if string(bytes) != testDataStr {
		t.Errorf("Expected %q got %q", testDataStr, string(bytes))
	}
}

func TestPutLocalFile(t *testing.T) {
	// Create a temporary local file.
	f, err := ioutil.TempFile("", "test-gcp-")
	if err != nil {
		t.Fatal(err)
	}
	defer f.Close()
	localName := f.Name()
	defer os.Remove(localName)
	n, err := f.Write(testData)
	if err != nil {
		t.Fatal(err)
	}
	if n != len(testData) {
		t.Fatalf("Expected to write %d bytes, got %d", len(testData), n)
	}
	// Put to GCP
	const testFileName = "a-new-name"
	link, err := client.PutLocalFile(localName, testFileName)
	if err != nil {
		t.Fatal(err)
	}
	// Check that we get the same link back from Get.
	retLink, err := client.Get(testFileName)
	if err != nil {
		t.Fatal(err)
	}
	if retLink != link {
		t.Errorf("Not the same link as stored: %v vs received: %v", link, retLink)
	}
	resp, err := http.Get(retLink)
	if err != nil {
		t.Fatal(err)
	}
	// Download contents
	defer resp.Body.Close()
	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		t.Fatalf("Can't read HTTP body: %v", err)
	}
	if string(data) != testDataStr {
		t.Errorf("Expected %q got %q", testDataStr, string(data))
	}
	// Clean up
	err = client.Delete(testFileName)
	if err != nil {
		t.Fatal(err)
	}
}

func Put(t *testing.T, path string) {
	_, err := client.Put(path, testData)
	if err != nil {
		t.Fatal(err)
	}
}

func setupDirectoryTree(t *testing.T) {
	Put(t, "a")
	Put(t, "a/b")
	Put(t, "a/b/c")
	Put(t, "a/b/c/d1")
	Put(t, "a/b/c/d2")
	Put(t, "a/b/c/d1/e11")
	Put(t, "a/b/c/d1/e12")
	Put(t, "a/b/c/d1/e13")
	Put(t, "a/b/c/d2/e21")
	Put(t, "a/b/c/d2/e22")
	Put(t, "a/b/c/d2/e23")
}

func TestList(t *testing.T) {
	setupDirectoryTree(t)
	type testResult struct {
		prefix   string
		depth    int
		expected []string
	}

	tests := []testResult{
		testResult{
			prefix: "a/b",
			depth:  0,
			expected: []string{
				"a/b",
			},
		},
		testResult{
			prefix: "a/b",
			depth:  1,
			expected: []string{
				"a/b",
				"a/b/c",
			},
		},
		testResult{
			prefix: "a/b",
			depth:  2,
			expected: []string{
				"a/b",
				"a/b/c",
				"a/b/c/d1",
				"a/b/c/d2",
			},
		},
		testResult{
			prefix: "a/b",
			depth:  3,
			expected: []string{
				"a/b",
				"a/b/c",
				"a/b/c/d1",
				"a/b/c/d1/e11",
				"a/b/c/d1/e12",
				"a/b/c/d1/e13",
				"a/b/c/d2",
				"a/b/c/d2/e21",
				"a/b/c/d2/e22",
				"a/b/c/d2/e23",
			},
		},
		testResult{
			prefix: "a",
			depth:  4,
			expected: []string{
				"a",
				"a/b",
				"a/b/c",
				"a/b/c/d1",
				"a/b/c/d1/e11",
				"a/b/c/d1/e12",
				"a/b/c/d1/e13",
				"a/b/c/d2",
				"a/b/c/d2/e21",
				"a/b/c/d2/e22",
				"a/b/c/d2/e23",
			},
		},
	}
	for i := range tests {
		testListPrefix(t, tests[i].prefix, tests[i].depth, tests[i].expected)
	}
}

func contains(needle string, haystack []string) bool {
	for _, elem := range haystack {
		if elem == needle {
			return true
		}
	}
	return false
}

func testListPrefix(t *testing.T, prefix string, depth int, expected []string) {
	names, err := client.ListPrefix(prefix, depth)
	if err != nil {
		t.Fatalf("Error in client.List: %v", err)
	}
	t.Logf("Prefix: got: %+v", names)
	if len(names) != len(expected) {
		t.Fatalf("Expected %d results, got %d", len(expected), len(names))
	}
	for _, e := range expected {
		if !contains(e, names) {
			t.Errorf("%s %d: %q not found", prefix, depth, e)
		}
	}
}

func TestListDir(t *testing.T) {
	prefix := "a/b/c/d1/"
	names, err := client.ListDir(prefix)
	if err != nil {
		t.Fatalf("Error in client.List: %v", err)
	}
	expected := []string{prefix + "e11", prefix + "e12", prefix + "e13"}
	if len(names) != len(expected) {
		t.Fatalf("Expected 3 results, got %d", len(names))
	}
	for _, e := range expected {
		if !contains(e, names) {
			t.Errorf("Expected %q, not found", e)
		}
	}
}

func TestDelete(t *testing.T) {
	_, err := client.Put(fileName, testData)
	if err != nil {
		t.Fatal(err)
	}
	err = client.Delete(fileName)
	if err != nil {
		t.Fatalf("Expected no errors, got %v", err)
	}
	// Test the side effect after Delete.
	_, err = client.Get(fileName)
	if err == nil {
		t.Fatal("Expected an error, but got none")
	}
}

func TestMain(m *testing.M) {
	flag.Parse()
	if testing.Short() {
		log.Print("cloud/storage/gcs: skipping network-based tests while -test.short specified")
		os.Exit(0)
	}

	// Create client that writes to test bucket.
	var err error
	client, err = storage.Dial("GCS",
		storage.WithKeyValue("gcpProjectId", testProjectID),
		storage.WithKeyValue("gcpBucketName", testBucketName),
		storage.WithKeyValue("defaultACL", PublicRead))
	if err != nil {
		log.Fatalf("cloud/storage/gcs: couldn't set up client: %v", err)
	}

	code := m.Run()

	// Clean up.
	const verbose = true
	if err := client.(*gcsImpl).EmptyBucket(verbose); err != nil {
		log.Print("cloud/storage/gcs: EmptyBucket failed: %v", err)
	}

	os.Exit(code)
}
<|endoftext|>


<commit_before>// This is a simple example of usage of Grafana client
// for importing dashboards from a bunch of JSON files (current dir used).
// You are can export dashboards with backup-dashboards utitity.
// NOTE: old dashboards with same names will be silently overrided!
//
// Usage:
//   import-dashboards http://sdk host:3000 api-key-string-here
//
// You need get API key with Admin rights from your Grafana!
package main

/*
   Copyright 2016 Alexander I.Grafov <grafov@gmail.com>

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.

   ॐ तारे तुत्तारे तुरे स्व
*/

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"strings"

	"github.com/grafana-tools/sdk"
)

func main() {
	var (
		filesInDir []os.FileInfo
		rawBoard   []byte
		err        error
	)
	if len(os.Args) != 3 {
		fmt.Fprint(os.Stderr, "Usage: import-dashboards http://grafana-host:3000 api-key-string-here\n")
		os.Exit(0)
	}
	c := sdk.NewClient(os.Args[1], os.Args[2], sdk.DefaultHTTPClient)
	filesInDir, err = ioutil.ReadDir(".")
	if err != nil {
		log.Fatal(err)
	}
	for _, file := range filesInDir {
		if strings.HasSuffix(file.Name(), ".json") {
			if rawBoard, err = ioutil.ReadFile(file.Name()); err != nil {
				log.Println(err)
				continue
			}
			var board Board
			if err = json.Unmarshal(rawBoard, &board); err != nil {
				log.Println(err)
				continue
			}
			c.DeleteDashboard(board.UpdateSlug())
			if err = c.SetDashboard(board, false); err != nil {
				log.Printf("error on importing dashboard %s", board.Title)
				continue
			}
		}
	}
}
<commit_msg>Fix object path.<commit_after>// This is a simple example of usage of Grafana client
// for importing dashboards from a bunch of JSON files (current dir used).
// You are can export dashboards with backup-dashboards utitity.
// NOTE: old dashboards with same names will be silently overrided!
//
// Usage:
//   import-dashboards http://sdk host:3000 api-key-string-here
//
// You need get API key with Admin rights from your Grafana!
package main

/*
   Copyright 2016 Alexander I.Grafov <grafov@gmail.com>

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.

   ॐ तारे तुत्तारे तुरे स्व
*/

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"strings"

	"github.com/grafana-tools/sdk"
)

func main() {
	var (
		filesInDir []os.FileInfo
		rawBoard   []byte
		err        error
	)
	if len(os.Args) != 3 {
		fmt.Fprint(os.Stderr, "Usage: import-dashboards http://grafana-host:3000 api-key-string-here\n")
		os.Exit(0)
	}
	c := sdk.NewClient(os.Args[1], os.Args[2], sdk.DefaultHTTPClient)
	filesInDir, err = ioutil.ReadDir(".")
	if err != nil {
		log.Fatal(err)
	}
	for _, file := range filesInDir {
		if strings.HasSuffix(file.Name(), ".json") {
			if rawBoard, err = ioutil.ReadFile(file.Name()); err != nil {
				log.Println(err)
				continue
			}
			var board sdk.Board
			if err = json.Unmarshal(rawBoard, &board); err != nil {
				log.Println(err)
				continue
			}
			c.DeleteDashboard(board.UpdateSlug())
			if err = c.SetDashboard(board, false); err != nil {
				log.Printf("error on importing dashboard %s", board.Title)
				continue
			}
		}
	}
}
<|endoftext|>


<commit_before>package gateway

import (
	"flag"
	"fmt"
	"os"
	"time"

	"github.com/funkygao/gafka/ctx"
)

var (
	Options struct {
		Id                      string
		Zone                    string
		ConfigFile              string
		PubHttpAddr             string
		PubHttpsAddr            string
		SubHttpAddr             string
		SubHttpsAddr            string
		ManHttpAddr             string
		ManHttpsAddr            string
		DebugHttpAddr           string
		Store                   string
		ManagerStore            string
		PidFile                 string
		CertFile                string
		KeyFile                 string
		LogFile                 string
		LogLevel                string
		CrashLogFile            string
		InfluxServer            string
		InfluxDbName            string
		KillFile                string
		ShowVersion             bool
		Ratelimit               bool
		PermitStandbySub        bool
		DisableMetrics          bool
		EnableGzip              bool
		DryRun                  bool
		CpuAffinity             bool
		EnableAccessLog         bool
		EnableHttpPanicRecover  bool
		EnableClientStats       bool
		GolangTrace             bool
		PermitUnregisteredGroup bool
		Debug                   bool
		HttpHeaderMaxBytes      int
		MaxPubSize              int64
		LogRotateSize           int
		MaxMsgTagLen            int
		MinPubSize              int
		MaxPubRetries           int
		PubQpsLimit             int64
		MaxSubBatchSize         int
		MaxClients              int
		MaxRequestPerConn       int // to make load balancer distribute request even for persistent conn
		PubPoolCapcity          int
		PubPoolIdleTimeout      time.Duration
		SubTimeout              time.Duration
		OffsetCommitInterval    time.Duration
		ReporterInterval        time.Duration
		ConsoleMetricsInterval  time.Duration
		MetaRefresh             time.Duration
		ManagerRefresh          time.Duration
		HttpReadTimeout         time.Duration
		HttpWriteTimeout        time.Duration
	}
)

func ParseFlags() {
	ip, err := ctx.LocalIP()
	if err != nil {
		panic(err)
	}

	var (
		defaultPubHttpAddr  = fmt.Sprintf("%s:9191", ip.String())
		defaultPubHttpsAddr = ""
		defaultSubHttpAddr  = fmt.Sprintf("%s:9192", ip.String())
		defaultSubHttpsAddr = ""
		defaultManHttpAddr  = fmt.Sprintf("%s:9193", ip.String())
		defaultManHttpsAddr = ""
	)

	flag.StringVar(&Options.Id, "id", "", "kateway id, the id must be unique within a host")
	flag.StringVar(&Options.Zone, "zone", "", "kafka zone name")
	flag.StringVar(&Options.PubHttpAddr, "pubhttp", defaultPubHttpAddr, "pub http bind addr")
	flag.StringVar(&Options.PubHttpsAddr, "pubhttps", defaultPubHttpsAddr, "pub https bind addr")
	flag.StringVar(&Options.SubHttpAddr, "subhttp", defaultSubHttpAddr, "sub http bind addr")
	flag.StringVar(&Options.SubHttpsAddr, "subhttps", defaultSubHttpsAddr, "sub https bind addr")
	flag.StringVar(&Options.ManHttpAddr, "manhttp", defaultManHttpAddr, "management http bind addr")
	flag.StringVar(&Options.ManHttpsAddr, "manhttps", defaultManHttpsAddr, "management https bind addr")
	flag.StringVar(&Options.LogLevel, "level", "trace", "log level")
	flag.StringVar(&Options.LogFile, "log", "stdout", "log file, default stdout")
	flag.StringVar(&Options.CrashLogFile, "crashlog", "", "crash log")
	flag.StringVar(&Options.CertFile, "certfile", "", "cert file path")
	flag.StringVar(&Options.PidFile, "pid", "", "pid file")
	flag.StringVar(&Options.KeyFile, "keyfile", "", "key file path")
	flag.StringVar(&Options.DebugHttpAddr, "debughttp", "", "debug http bind addr")
	flag.StringVar(&Options.Store, "store", "kafka", "backend store")
	flag.StringVar(&Options.ManagerStore, "mstore", "mysql", "store integration with manager")
	flag.StringVar(&Options.ConfigFile, "conf", "/etc/kateway.cf", "config file")
	flag.StringVar(&Options.KillFile, "kill", "", "kill running kateway by pid file")
	flag.StringVar(&Options.InfluxServer, "influxdbaddr", "http://10.77.144.193:10036", "influxdb server address for the metrics reporter")
	flag.StringVar(&Options.InfluxDbName, "influxdbname", "pubsub", "influxdb db name")
	flag.BoolVar(&Options.ShowVersion, "version", false, "show version and exit")
	flag.BoolVar(&Options.Debug, "debug", false, "enable debug mode")
	flag.BoolVar(&Options.GolangTrace, "gotrace", false, "go tool trace")
	flag.BoolVar(&Options.EnableAccessLog, "accesslog", false, "en(dis)able access log")
	flag.BoolVar(&Options.EnableClientStats, "clientsmap", false, "record online pub/sub clients")
	flag.BoolVar(&Options.DryRun, "dryrun", false, "dry run mode")
	flag.BoolVar(&Options.PermitUnregisteredGroup, "unregrp", false, "permit sub group usage without being registered")
	flag.BoolVar(&Options.PermitStandbySub, "standbysub", false, "permits sub threads exceed partitions")
	flag.BoolVar(&Options.EnableGzip, "gzip", false, "enable http response gzip")
	flag.BoolVar(&Options.CpuAffinity, "cpuaffinity", false, "enable cpu affinity")
	flag.BoolVar(&Options.Ratelimit, "raltelimit", false, "enable rate limit")
	flag.BoolVar(&Options.EnableHttpPanicRecover, "httppanic", false, "enable http handler panic recover")
	flag.BoolVar(&Options.DisableMetrics, "metricsoff", false, "disable metrics reporter")
	flag.IntVar(&Options.HttpHeaderMaxBytes, "maxheader", 4<<10, "http header max size in bytes")
	flag.Int64Var(&Options.MaxPubSize, "maxpub", 512<<10, "max Pub message size")
	flag.IntVar(&Options.MinPubSize, "minpub", 1, "min Pub message size")
	flag.IntVar(&Options.MaxPubRetries, "pubretry", 5, "max retries when Pub fails")
	flag.IntVar(&Options.MaxRequestPerConn, "maxreq", -1, "max request per connection")
	flag.IntVar(&Options.MaxMsgTagLen, "tagsz", 1024, "max message tag length permitted")
	// kafka Fetch maxFetchSize=1MB, so if our msg agv size is 500B, batch size can be 2500
	flag.IntVar(&Options.MaxSubBatchSize, "maxbatch", 2500, "max sub batch size")
	flag.IntVar(&Options.LogRotateSize, "logsize", 10<<30, "max unrotated log file size")
	flag.Int64Var(&Options.PubQpsLimit, "publimit", 60*10000, "pub qps limit per minute per ip")
	flag.IntVar(&Options.PubPoolCapcity, "pubpool", 100, "pub connection pool capacity")
	flag.IntVar(&Options.MaxClients, "maxclient", 100000, "max concurrent connections")
	flag.DurationVar(&Options.OffsetCommitInterval, "offsetcommit", time.Minute, "consumer offset commit interval")
	flag.DurationVar(&Options.HttpReadTimeout, "httprtimeout", time.Minute*5, "http server read timeout")
	flag.DurationVar(&Options.HttpWriteTimeout, "httpwtimeout", time.Minute, "http server write timeout")
	flag.DurationVar(&Options.SubTimeout, "subtimeout", time.Second*30, "sub timeout before send http 204")
	flag.DurationVar(&Options.ReporterInterval, "report", time.Second*10, "reporter flush interval")
	flag.DurationVar(&Options.MetaRefresh, "metarefresh", time.Minute*10, "meta data refresh interval")
	flag.DurationVar(&Options.ManagerRefresh, "manrefresh", time.Minute*5, "manager integration refresh interval")
	flag.DurationVar(&Options.ConsoleMetricsInterval, "consolemetrics", 0, "console metrics report interval")
	flag.DurationVar(&Options.PubPoolIdleTimeout, "pubpoolidle", 0, "pub pool connect idle timeout")

	flag.Parse()
}

func ValidateFlags() {
	if Options.KillFile != "" {
		return
	}

	if Options.Zone == "" {
		fmt.Fprintf(os.Stderr, "-zone required\n")
		os.Exit(1)
	}

	if Options.ManHttpsAddr == "" && Options.ManHttpAddr == "" {
		fmt.Fprintf(os.Stderr, "-manhttp or -manhttps required\n")
	}
}
<commit_msg>increase permitted max sub batch size to 4k<commit_after>package gateway

import (
	"flag"
	"fmt"
	"os"
	"time"

	"github.com/funkygao/gafka/ctx"
)

var (
	Options struct {
		Id                      string
		Zone                    string
		ConfigFile              string
		PubHttpAddr             string
		PubHttpsAddr            string
		SubHttpAddr             string
		SubHttpsAddr            string
		ManHttpAddr             string
		ManHttpsAddr            string
		DebugHttpAddr           string
		Store                   string
		ManagerStore            string
		PidFile                 string
		CertFile                string
		KeyFile                 string
		LogFile                 string
		LogLevel                string
		CrashLogFile            string
		InfluxServer            string
		InfluxDbName            string
		KillFile                string
		ShowVersion             bool
		Ratelimit               bool
		PermitStandbySub        bool
		DisableMetrics          bool
		EnableGzip              bool
		DryRun                  bool
		CpuAffinity             bool
		EnableAccessLog         bool
		EnableHttpPanicRecover  bool
		EnableClientStats       bool
		GolangTrace             bool
		PermitUnregisteredGroup bool
		Debug                   bool
		HttpHeaderMaxBytes      int
		MaxPubSize              int64
		LogRotateSize           int
		MaxMsgTagLen            int
		MinPubSize              int
		MaxPubRetries           int
		PubQpsLimit             int64
		MaxSubBatchSize         int
		MaxClients              int
		MaxRequestPerConn       int // to make load balancer distribute request even for persistent conn
		PubPoolCapcity          int
		PubPoolIdleTimeout      time.Duration
		SubTimeout              time.Duration
		OffsetCommitInterval    time.Duration
		ReporterInterval        time.Duration
		ConsoleMetricsInterval  time.Duration
		MetaRefresh             time.Duration
		ManagerRefresh          time.Duration
		HttpReadTimeout         time.Duration
		HttpWriteTimeout        time.Duration
	}
)

func ParseFlags() {
	ip, err := ctx.LocalIP()
	if err != nil {
		panic(err)
	}

	var (
		defaultPubHttpAddr  = fmt.Sprintf("%s:9191", ip.String())
		defaultPubHttpsAddr = ""
		defaultSubHttpAddr  = fmt.Sprintf("%s:9192", ip.String())
		defaultSubHttpsAddr = ""
		defaultManHttpAddr  = fmt.Sprintf("%s:9193", ip.String())
		defaultManHttpsAddr = ""
	)

	flag.StringVar(&Options.Id, "id", "", "kateway id, the id must be unique within a host")
	flag.StringVar(&Options.Zone, "zone", "", "kafka zone name")
	flag.StringVar(&Options.PubHttpAddr, "pubhttp", defaultPubHttpAddr, "pub http bind addr")
	flag.StringVar(&Options.PubHttpsAddr, "pubhttps", defaultPubHttpsAddr, "pub https bind addr")
	flag.StringVar(&Options.SubHttpAddr, "subhttp", defaultSubHttpAddr, "sub http bind addr")
	flag.StringVar(&Options.SubHttpsAddr, "subhttps", defaultSubHttpsAddr, "sub https bind addr")
	flag.StringVar(&Options.ManHttpAddr, "manhttp", defaultManHttpAddr, "management http bind addr")
	flag.StringVar(&Options.ManHttpsAddr, "manhttps", defaultManHttpsAddr, "management https bind addr")
	flag.StringVar(&Options.LogLevel, "level", "trace", "log level")
	flag.StringVar(&Options.LogFile, "log", "stdout", "log file, default stdout")
	flag.StringVar(&Options.CrashLogFile, "crashlog", "", "crash log")
	flag.StringVar(&Options.CertFile, "certfile", "", "cert file path")
	flag.StringVar(&Options.PidFile, "pid", "", "pid file")
	flag.StringVar(&Options.KeyFile, "keyfile", "", "key file path")
	flag.StringVar(&Options.DebugHttpAddr, "debughttp", "", "debug http bind addr")
	flag.StringVar(&Options.Store, "store", "kafka", "backend store")
	flag.StringVar(&Options.ManagerStore, "mstore", "mysql", "store integration with manager")
	flag.StringVar(&Options.ConfigFile, "conf", "/etc/kateway.cf", "config file")
	flag.StringVar(&Options.KillFile, "kill", "", "kill running kateway by pid file")
	flag.StringVar(&Options.InfluxServer, "influxdbaddr", "http://10.77.144.193:10036", "influxdb server address for the metrics reporter")
	flag.StringVar(&Options.InfluxDbName, "influxdbname", "pubsub", "influxdb db name")
	flag.BoolVar(&Options.ShowVersion, "version", false, "show version and exit")
	flag.BoolVar(&Options.Debug, "debug", false, "enable debug mode")
	flag.BoolVar(&Options.GolangTrace, "gotrace", false, "go tool trace")
	flag.BoolVar(&Options.EnableAccessLog, "accesslog", false, "en(dis)able access log")
	flag.BoolVar(&Options.EnableClientStats, "clientsmap", false, "record online pub/sub clients")
	flag.BoolVar(&Options.DryRun, "dryrun", false, "dry run mode")
	flag.BoolVar(&Options.PermitUnregisteredGroup, "unregrp", false, "permit sub group usage without being registered")
	flag.BoolVar(&Options.PermitStandbySub, "standbysub", false, "permits sub threads exceed partitions")
	flag.BoolVar(&Options.EnableGzip, "gzip", false, "enable http response gzip")
	flag.BoolVar(&Options.CpuAffinity, "cpuaffinity", false, "enable cpu affinity")
	flag.BoolVar(&Options.Ratelimit, "raltelimit", false, "enable rate limit")
	flag.BoolVar(&Options.EnableHttpPanicRecover, "httppanic", false, "enable http handler panic recover")
	flag.BoolVar(&Options.DisableMetrics, "metricsoff", false, "disable metrics reporter")
	flag.IntVar(&Options.HttpHeaderMaxBytes, "maxheader", 4<<10, "http header max size in bytes")
	flag.Int64Var(&Options.MaxPubSize, "maxpub", 512<<10, "max Pub message size")
	flag.IntVar(&Options.MinPubSize, "minpub", 1, "min Pub message size")
	flag.IntVar(&Options.MaxPubRetries, "pubretry", 5, "max retries when Pub fails")
	flag.IntVar(&Options.MaxRequestPerConn, "maxreq", -1, "max request per connection")
	flag.IntVar(&Options.MaxMsgTagLen, "tagsz", 1024, "max message tag length permitted")
	// kafka Fetch maxFetchSize=1MB, so if our msg agv size is 250B, batch size can be 4000
	flag.IntVar(&Options.MaxSubBatchSize, "maxbatch", 4000, "max sub batch size")
	flag.IntVar(&Options.LogRotateSize, "logsize", 10<<30, "max unrotated log file size")
	flag.Int64Var(&Options.PubQpsLimit, "publimit", 60*10000, "pub qps limit per minute per ip")
	flag.IntVar(&Options.PubPoolCapcity, "pubpool", 100, "pub connection pool capacity")
	flag.IntVar(&Options.MaxClients, "maxclient", 100000, "max concurrent connections")
	flag.DurationVar(&Options.OffsetCommitInterval, "offsetcommit", time.Minute, "consumer offset commit interval")
	flag.DurationVar(&Options.HttpReadTimeout, "httprtimeout", time.Minute*5, "http server read timeout")
	flag.DurationVar(&Options.HttpWriteTimeout, "httpwtimeout", time.Minute, "http server write timeout")
	flag.DurationVar(&Options.SubTimeout, "subtimeout", time.Second*30, "sub timeout before send http 204")
	flag.DurationVar(&Options.ReporterInterval, "report", time.Second*10, "reporter flush interval")
	flag.DurationVar(&Options.MetaRefresh, "metarefresh", time.Minute*10, "meta data refresh interval")
	flag.DurationVar(&Options.ManagerRefresh, "manrefresh", time.Minute*5, "manager integration refresh interval")
	flag.DurationVar(&Options.ConsoleMetricsInterval, "consolemetrics", 0, "console metrics report interval")
	flag.DurationVar(&Options.PubPoolIdleTimeout, "pubpoolidle", 0, "pub pool connect idle timeout")

	flag.Parse()
}

func ValidateFlags() {
	if Options.KillFile != "" {
		return
	}

	if Options.Zone == "" {
		fmt.Fprintf(os.Stderr, "-zone required\n")
		os.Exit(1)
	}

	if Options.ManHttpsAddr == "" && Options.ManHttpAddr == "" {
		fmt.Fprintf(os.Stderr, "-manhttp or -manhttps required\n")
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package request

import (
	"fmt"

	appsv1 "k8s.io/api/apps/v1"
	apiv1 "k8s.io/api/core/v1"
	networking "k8s.io/api/networking/v1beta1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/cli-runtime/pkg/genericclioptions"
	appsv1client "k8s.io/client-go/kubernetes/typed/apps/v1"
	corev1 "k8s.io/client-go/kubernetes/typed/core/v1"
	typednetworking "k8s.io/client-go/kubernetes/typed/networking/v1beta1"

	"k8s.io/ingress-nginx/cmd/plugin/util"
)

// ChoosePod finds a pod either by deployment or by name
func ChoosePod(flags *genericclioptions.ConfigFlags, podName string, deployment string, selector string) (apiv1.Pod, error) {
	if podName != "" {
		return GetNamedPod(flags, podName)
	}

	if selector != "" {
		return GetLabeledPod(flags, selector)
	}

	return GetDeploymentPod(flags, deployment)
}

// GetNamedPod finds a pod with the given name
func GetNamedPod(flags *genericclioptions.ConfigFlags, name string) (apiv1.Pod, error) {
	allPods, err := getPods(flags)
	if err != nil {
		return apiv1.Pod{}, err
	}

	for _, pod := range allPods {
		if pod.Name == name {
			return pod, nil
		}
	}

	return apiv1.Pod{}, fmt.Errorf("pod %v not found in namespace %v", name, util.GetNamespace(flags))
}

// GetDeploymentPod finds a pod from a given deployment
func GetDeploymentPod(flags *genericclioptions.ConfigFlags, deployment string) (apiv1.Pod, error) {
	ings, err := getDeploymentPods(flags, deployment)
	if err != nil {
		return apiv1.Pod{}, err
	}

	if len(ings) == 0 {
		return apiv1.Pod{}, fmt.Errorf("no pods for deployment %v found in namespace %v", deployment, util.GetNamespace(flags))
	}

	return ings[0], nil
}

// GetDeploymentPod finds a pod from a given deployment
func GetLabeledPod(flags *genericclioptions.ConfigFlags, label string) (apiv1.Pod, error) {
	ings, err := getLabeledPods(flags, label)
	if err != nil {
		return apiv1.Pod{}, err
	}

	if len(ings) == 0 {
		return apiv1.Pod{}, fmt.Errorf("no pods for label selector %v found in namespace %v", label, util.GetNamespace(flags))
	}

	return ings[0], nil
}

// GetDeployments returns an array of Deployments
func GetDeployments(flags *genericclioptions.ConfigFlags, namespace string) ([]appsv1.Deployment, error) {
	rawConfig, err := flags.ToRESTConfig()
	if err != nil {
		return make([]appsv1.Deployment, 0), err
	}

	api, err := appsv1client.NewForConfig(rawConfig)
	if err != nil {
		return make([]appsv1.Deployment, 0), err
	}

	deployments, err := api.Deployments(namespace).List(metav1.ListOptions{})
	if err != nil {
		return make([]appsv1.Deployment, 0), err
	}

	return deployments.Items, nil
}

// GetIngressDefinitions returns an array of Ingress resource definitions
func GetIngressDefinitions(flags *genericclioptions.ConfigFlags, namespace string) ([]networking.Ingress, error) {
	rawConfig, err := flags.ToRESTConfig()
	if err != nil {
		return make([]networking.Ingress, 0), err
	}

	api, err := typednetworking.NewForConfig(rawConfig)
	if err != nil {
		return make([]networking.Ingress, 0), err
	}

	pods, err := api.Ingresses(namespace).List(metav1.ListOptions{})
	if err != nil {
		return make([]networking.Ingress, 0), err
	}

	return pods.Items, nil
}

// GetNumEndpoints counts the number of endpoints for the service with the given name
func GetNumEndpoints(flags *genericclioptions.ConfigFlags, namespace string, serviceName string) (*int, error) {
	endpoints, err := GetEndpointsByName(flags, namespace, serviceName)
	if err != nil {
		return nil, err
	}

	if endpoints == nil {
		return nil, nil
	}

	ret := 0
	for _, subset := range endpoints.Subsets {
		ret += len(subset.Addresses)
	}
	return &ret, nil
}

// GetEndpointsByName returns the endpoints for the service with the given name
func GetEndpointsByName(flags *genericclioptions.ConfigFlags, namespace string, name string) (*apiv1.Endpoints, error) {
	allEndpoints, err := getEndpoints(flags, namespace)
	if err != nil {
		return nil, err
	}

	for _, endpoints := range allEndpoints {
		if endpoints.Name == name {
			return &endpoints, nil
		}
	}

	return nil, nil
}

var endpointsCache = make(map[string]*[]apiv1.Endpoints)

func getEndpoints(flags *genericclioptions.ConfigFlags, namespace string) ([]apiv1.Endpoints, error) {
	cachedEndpoints, ok := endpointsCache[namespace]
	if ok {
		return *cachedEndpoints, nil
	}

	if namespace != "" {
		tryAllNamespacesEndpointsCache(flags)
	}

	cachedEndpoints = tryFilteringEndpointsFromAllNamespacesCache(flags, namespace)
	if cachedEndpoints != nil {
		return *cachedEndpoints, nil
	}

	rawConfig, err := flags.ToRESTConfig()
	if err != nil {
		return nil, err
	}

	api, err := corev1.NewForConfig(rawConfig)
	if err != nil {
		return nil, err
	}

	endpointsList, err := api.Endpoints(namespace).List(metav1.ListOptions{})
	if err != nil {
		return nil, err
	}
	endpoints := endpointsList.Items

	endpointsCache[namespace] = &endpoints
	return endpoints, nil
}

func tryAllNamespacesEndpointsCache(flags *genericclioptions.ConfigFlags) {
	_, ok := endpointsCache[""]
	if !ok {
		_, err := getEndpoints(flags, "")
		if err != nil {
			endpointsCache[""] = nil
		}
	}
}

func tryFilteringEndpointsFromAllNamespacesCache(flags *genericclioptions.ConfigFlags, namespace string) *[]apiv1.Endpoints {
	allEndpoints := endpointsCache[""]
	if allEndpoints != nil {
		endpoints := make([]apiv1.Endpoints, 0)
		for _, thisEndpoints := range *allEndpoints {
			if thisEndpoints.Namespace == namespace {
				endpoints = append(endpoints, thisEndpoints)
			}
		}
		endpointsCache[namespace] = &endpoints
		return &endpoints
	}
	return nil
}

// GetServiceByName finds and returns the service definition with the given name
func GetServiceByName(flags *genericclioptions.ConfigFlags, name string, services *[]apiv1.Service) (apiv1.Service, error) {
	if services == nil {
		servicesArray, err := getServices(flags)
		if err != nil {
			return apiv1.Service{}, err
		}
		services = &servicesArray
	}

	for _, svc := range *services {
		if svc.Name == name {
			return svc, nil
		}
	}

	return apiv1.Service{}, fmt.Errorf("could not find service %v in namespace %v", name, util.GetNamespace(flags))
}

func getPods(flags *genericclioptions.ConfigFlags) ([]apiv1.Pod, error) {
	namespace := util.GetNamespace(flags)

	rawConfig, err := flags.ToRESTConfig()
	if err != nil {
		return make([]apiv1.Pod, 0), err
	}

	api, err := corev1.NewForConfig(rawConfig)
	if err != nil {
		return make([]apiv1.Pod, 0), err
	}

	pods, err := api.Pods(namespace).List(metav1.ListOptions{})
	if err != nil {
		return make([]apiv1.Pod, 0), err
	}

	return pods.Items, nil
}

func getLabeledPods(flags *genericclioptions.ConfigFlags, label string) ([]apiv1.Pod, error) {
	namespace := util.GetNamespace(flags)

	rawConfig, err := flags.ToRESTConfig()
	if err != nil {
		return make([]apiv1.Pod, 0), err
	}

	api, err := corev1.NewForConfig(rawConfig)
	if err != nil {
		return make([]apiv1.Pod, 0), err
	}

	pods, err := api.Pods(namespace).List(metav1.ListOptions{
		LabelSelector: label,
	})

	if err != nil {
		return make([]apiv1.Pod, 0), err
	}

	return pods.Items, nil
}

func getDeploymentPods(flags *genericclioptions.ConfigFlags, deployment string) ([]apiv1.Pod, error) {
	pods, err := getPods(flags)
	if err != nil {
		return make([]apiv1.Pod, 0), err
	}

	ingressPods := make([]apiv1.Pod, 0)
	for _, pod := range pods {
		if util.PodInDeployment(pod, deployment) {
			ingressPods = append(ingressPods, pod)
		}
	}

	return ingressPods, nil
}

func getServices(flags *genericclioptions.ConfigFlags) ([]apiv1.Service, error) {
	namespace := util.GetNamespace(flags)

	rawConfig, err := flags.ToRESTConfig()
	if err != nil {
		return make([]apiv1.Service, 0), err
	}

	api, err := corev1.NewForConfig(rawConfig)
	if err != nil {
		return make([]apiv1.Service, 0), err
	}

	services, err := api.Services(namespace).List(metav1.ListOptions{})
	if err != nil {
		return make([]apiv1.Service, 0), err
	}

	return services.Items, nil

}
<commit_msg>Fix public function comment (#5181)<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package request

import (
	"fmt"

	appsv1 "k8s.io/api/apps/v1"
	apiv1 "k8s.io/api/core/v1"
	networking "k8s.io/api/networking/v1beta1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/cli-runtime/pkg/genericclioptions"
	appsv1client "k8s.io/client-go/kubernetes/typed/apps/v1"
	corev1 "k8s.io/client-go/kubernetes/typed/core/v1"
	typednetworking "k8s.io/client-go/kubernetes/typed/networking/v1beta1"

	"k8s.io/ingress-nginx/cmd/plugin/util"
)

// ChoosePod finds a pod either by deployment or by name
func ChoosePod(flags *genericclioptions.ConfigFlags, podName string, deployment string, selector string) (apiv1.Pod, error) {
	if podName != "" {
		return GetNamedPod(flags, podName)
	}

	if selector != "" {
		return GetLabeledPod(flags, selector)
	}

	return GetDeploymentPod(flags, deployment)
}

// GetNamedPod finds a pod with the given name
func GetNamedPod(flags *genericclioptions.ConfigFlags, name string) (apiv1.Pod, error) {
	allPods, err := getPods(flags)
	if err != nil {
		return apiv1.Pod{}, err
	}

	for _, pod := range allPods {
		if pod.Name == name {
			return pod, nil
		}
	}

	return apiv1.Pod{}, fmt.Errorf("pod %v not found in namespace %v", name, util.GetNamespace(flags))
}

// GetDeploymentPod finds a pod from a given deployment
func GetDeploymentPod(flags *genericclioptions.ConfigFlags, deployment string) (apiv1.Pod, error) {
	ings, err := getDeploymentPods(flags, deployment)
	if err != nil {
		return apiv1.Pod{}, err
	}

	if len(ings) == 0 {
		return apiv1.Pod{}, fmt.Errorf("no pods for deployment %v found in namespace %v", deployment, util.GetNamespace(flags))
	}

	return ings[0], nil
}

// GetLabeledPod finds a pod from a given label
func GetLabeledPod(flags *genericclioptions.ConfigFlags, label string) (apiv1.Pod, error) {
	ings, err := getLabeledPods(flags, label)
	if err != nil {
		return apiv1.Pod{}, err
	}

	if len(ings) == 0 {
		return apiv1.Pod{}, fmt.Errorf("no pods for label selector %v found in namespace %v", label, util.GetNamespace(flags))
	}

	return ings[0], nil
}

// GetDeployments returns an array of Deployments
func GetDeployments(flags *genericclioptions.ConfigFlags, namespace string) ([]appsv1.Deployment, error) {
	rawConfig, err := flags.ToRESTConfig()
	if err != nil {
		return make([]appsv1.Deployment, 0), err
	}

	api, err := appsv1client.NewForConfig(rawConfig)
	if err != nil {
		return make([]appsv1.Deployment, 0), err
	}

	deployments, err := api.Deployments(namespace).List(metav1.ListOptions{})
	if err != nil {
		return make([]appsv1.Deployment, 0), err
	}

	return deployments.Items, nil
}

// GetIngressDefinitions returns an array of Ingress resource definitions
func GetIngressDefinitions(flags *genericclioptions.ConfigFlags, namespace string) ([]networking.Ingress, error) {
	rawConfig, err := flags.ToRESTConfig()
	if err != nil {
		return make([]networking.Ingress, 0), err
	}

	api, err := typednetworking.NewForConfig(rawConfig)
	if err != nil {
		return make([]networking.Ingress, 0), err
	}

	pods, err := api.Ingresses(namespace).List(metav1.ListOptions{})
	if err != nil {
		return make([]networking.Ingress, 0), err
	}

	return pods.Items, nil
}

// GetNumEndpoints counts the number of endpoints for the service with the given name
func GetNumEndpoints(flags *genericclioptions.ConfigFlags, namespace string, serviceName string) (*int, error) {
	endpoints, err := GetEndpointsByName(flags, namespace, serviceName)
	if err != nil {
		return nil, err
	}

	if endpoints == nil {
		return nil, nil
	}

	ret := 0
	for _, subset := range endpoints.Subsets {
		ret += len(subset.Addresses)
	}
	return &ret, nil
}

// GetEndpointsByName returns the endpoints for the service with the given name
func GetEndpointsByName(flags *genericclioptions.ConfigFlags, namespace string, name string) (*apiv1.Endpoints, error) {
	allEndpoints, err := getEndpoints(flags, namespace)
	if err != nil {
		return nil, err
	}

	for _, endpoints := range allEndpoints {
		if endpoints.Name == name {
			return &endpoints, nil
		}
	}

	return nil, nil
}

var endpointsCache = make(map[string]*[]apiv1.Endpoints)

func getEndpoints(flags *genericclioptions.ConfigFlags, namespace string) ([]apiv1.Endpoints, error) {
	cachedEndpoints, ok := endpointsCache[namespace]
	if ok {
		return *cachedEndpoints, nil
	}

	if namespace != "" {
		tryAllNamespacesEndpointsCache(flags)
	}

	cachedEndpoints = tryFilteringEndpointsFromAllNamespacesCache(flags, namespace)
	if cachedEndpoints != nil {
		return *cachedEndpoints, nil
	}

	rawConfig, err := flags.ToRESTConfig()
	if err != nil {
		return nil, err
	}

	api, err := corev1.NewForConfig(rawConfig)
	if err != nil {
		return nil, err
	}

	endpointsList, err := api.Endpoints(namespace).List(metav1.ListOptions{})
	if err != nil {
		return nil, err
	}
	endpoints := endpointsList.Items

	endpointsCache[namespace] = &endpoints
	return endpoints, nil
}

func tryAllNamespacesEndpointsCache(flags *genericclioptions.ConfigFlags) {
	_, ok := endpointsCache[""]
	if !ok {
		_, err := getEndpoints(flags, "")
		if err != nil {
			endpointsCache[""] = nil
		}
	}
}

func tryFilteringEndpointsFromAllNamespacesCache(flags *genericclioptions.ConfigFlags, namespace string) *[]apiv1.Endpoints {
	allEndpoints := endpointsCache[""]
	if allEndpoints != nil {
		endpoints := make([]apiv1.Endpoints, 0)
		for _, thisEndpoints := range *allEndpoints {
			if thisEndpoints.Namespace == namespace {
				endpoints = append(endpoints, thisEndpoints)
			}
		}
		endpointsCache[namespace] = &endpoints
		return &endpoints
	}
	return nil
}

// GetServiceByName finds and returns the service definition with the given name
func GetServiceByName(flags *genericclioptions.ConfigFlags, name string, services *[]apiv1.Service) (apiv1.Service, error) {
	if services == nil {
		servicesArray, err := getServices(flags)
		if err != nil {
			return apiv1.Service{}, err
		}
		services = &servicesArray
	}

	for _, svc := range *services {
		if svc.Name == name {
			return svc, nil
		}
	}

	return apiv1.Service{}, fmt.Errorf("could not find service %v in namespace %v", name, util.GetNamespace(flags))
}

func getPods(flags *genericclioptions.ConfigFlags) ([]apiv1.Pod, error) {
	namespace := util.GetNamespace(flags)

	rawConfig, err := flags.ToRESTConfig()
	if err != nil {
		return make([]apiv1.Pod, 0), err
	}

	api, err := corev1.NewForConfig(rawConfig)
	if err != nil {
		return make([]apiv1.Pod, 0), err
	}

	pods, err := api.Pods(namespace).List(metav1.ListOptions{})
	if err != nil {
		return make([]apiv1.Pod, 0), err
	}

	return pods.Items, nil
}

func getLabeledPods(flags *genericclioptions.ConfigFlags, label string) ([]apiv1.Pod, error) {
	namespace := util.GetNamespace(flags)

	rawConfig, err := flags.ToRESTConfig()
	if err != nil {
		return make([]apiv1.Pod, 0), err
	}

	api, err := corev1.NewForConfig(rawConfig)
	if err != nil {
		return make([]apiv1.Pod, 0), err
	}

	pods, err := api.Pods(namespace).List(metav1.ListOptions{
		LabelSelector: label,
	})

	if err != nil {
		return make([]apiv1.Pod, 0), err
	}

	return pods.Items, nil
}

func getDeploymentPods(flags *genericclioptions.ConfigFlags, deployment string) ([]apiv1.Pod, error) {
	pods, err := getPods(flags)
	if err != nil {
		return make([]apiv1.Pod, 0), err
	}

	ingressPods := make([]apiv1.Pod, 0)
	for _, pod := range pods {
		if util.PodInDeployment(pod, deployment) {
			ingressPods = append(ingressPods, pod)
		}
	}

	return ingressPods, nil
}

func getServices(flags *genericclioptions.ConfigFlags) ([]apiv1.Service, error) {
	namespace := util.GetNamespace(flags)

	rawConfig, err := flags.ToRESTConfig()
	if err != nil {
		return make([]apiv1.Service, 0), err
	}

	api, err := corev1.NewForConfig(rawConfig)
	if err != nil {
		return make([]apiv1.Service, 0), err
	}

	services, err := api.Services(namespace).List(metav1.ListOptions{})
	if err != nil {
		return make([]apiv1.Service, 0), err
	}

	return services.Items, nil

}
<|endoftext|>


<commit_before>// Build sets scollector version information and should be run from the
// scollector directory.
package main

import (
	"bytes"
	"fmt"
	"go/ast"
	"go/format"
	"go/parser"
	"go/printer"
	"go/token"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"strings"
	"time"
)

func main() {
	const path = "main.go"
	var hash, id string
	fset := token.NewFileSet()
	f, err := parser.ParseFile(fset, path, nil, parser.ParseComments)
	if err != nil {
		log.Fatal(err)
	}
	for _, d := range f.Decls {
		switch d := d.(type) {
		case *ast.GenDecl:
			if d.Tok != token.CONST {
				continue
			}
			for _, spec := range d.Specs {
				switch spec := spec.(type) {
				case *ast.ValueSpec:
					if len(spec.Names) != 1 || len(spec.Values) != 1 {
						continue
					}
					switch spec.Names[0].Name {
					case "VersionDate":
						switch value := spec.Values[0].(type) {
						case *ast.BasicLit:
							id = time.Now().UTC().Format("20060102150405")
							value.Value = id
						}
					case "VersionID":
						switch value := spec.Values[0].(type) {
						case *ast.BasicLit:
							rev, err := exec.Command("git", "rev-parse", "HEAD").Output()
							if err != nil {
								log.Fatal(err)
							}
							hash = fmt.Sprintf(`"%s"`, strings.TrimSpace(string(rev)))
							value.Value = hash
						}
					}
				}
			}
		}
	}
	file, err := os.Open(path)
	if err != nil {
		log.Fatal(err)
	}
	info, err := file.Stat()
	if err != nil {
		log.Fatal(err)
	}
	var buf bytes.Buffer
	if err := printer.Fprint(&buf, fset, f); err != nil {
		log.Fatal(err)
	}
	fb, err := format.Source(buf.Bytes())
	if err != nil {
		log.Fatal(err)
	}
	if err := ioutil.WriteFile(path, fb, info.Mode()); err != nil {
		log.Fatal(err)
	}
	fmt.Printf("version:\n  hash: %s\n  id: %s\n", hash, id)
}
<commit_msg>cmd/scollector: Cleaned up the build/build.go script<commit_after>// Build sets scollector version information and should be run from the
// scollector directory.
package main

import (
	"bytes"
	"fmt"
	"go/ast"
	"go/parser"
	"go/printer"
	"go/token"
	"io"
	"log"
	"os"
	"os/exec"
	"strings"
	"time"
)

func main() {
	const path = "main.go"
	mainfile, err := os.OpenFile(path, os.O_RDWR, 0660)
	if err != nil {
		log.Fatal(err)
	}
	defer mainfile.Close()

	var hash, id string
	fset := token.NewFileSet()
	f, err := parser.ParseFile(fset, path, mainfile, parser.ParseComments)
	if err != nil {
		log.Fatal(err)
	}

	for _, d := range f.Decls {
		d, ok := d.(*ast.GenDecl)
		if !ok {
			continue
		}

		if d.Tok != token.CONST {
			continue
		}
		for _, spec := range d.Specs {
			spec, ok := spec.(*ast.ValueSpec)
			if !ok {
				continue
			}

			if len(spec.Names) != 1 || len(spec.Values) != 1 {
				continue
			}

			value, ok := spec.Values[0].(*ast.BasicLit)
			if !ok {
				continue
			}

			switch spec.Names[0].Name {
			case "VersionDate":
				id = time.Now().UTC().Format("20060102150405")
				value.Value = id
			case "VersionID":
				rev, err := exec.Command("git", "rev-parse", "HEAD").Output()
				if err != nil {
					log.Fatal(err)
				}
				hash = fmt.Sprintf(`"%s"`, strings.TrimSpace(string(rev)))
				value.Value = hash
			}
		}
	}

	var config = printer.Config{Mode: printer.UseSpaces | printer.TabIndent, Tabwidth: 8}
	var buf bytes.Buffer
	if err := config.Fprint(&buf, fset, f); err != nil {
		log.Fatal(err)
	}

	if _, err := mainfile.Seek(0, os.SEEK_SET); err != nil {
		log.Fatal(err)
	}

	if _, err := io.Copy(mainfile, &buf); err != nil {
		log.Fatal(err)
	}

	fmt.Printf("version:\n  hash: %s\n  id: %s\n", hash, id)
}
<|endoftext|>


<commit_before>// Build sets scollector version information and should be run from the
// scollector directory.
package main

import (
	"bytes"
	"fmt"
	"go/ast"
	"go/format"
	"go/parser"
	"go/printer"
	"go/token"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"strings"
	"time"
)

func main() {
	const path = "main.go"
	var hash, id string
	fset := token.NewFileSet()
	f, err := parser.ParseFile(fset, path, nil, parser.ParseComments)
	if err != nil {
		log.Fatal(err)
	}
	for _, d := range f.Decls {
		switch d := d.(type) {
		case *ast.GenDecl:
			if d.Tok != token.CONST {
				continue
			}
			for _, spec := range d.Specs {
				switch spec := spec.(type) {
				case *ast.ValueSpec:
					if len(spec.Names) != 1 || len(spec.Values) != 1 {
						continue
					}
					switch spec.Names[0].Name {
					case "VersionDate":
						switch value := spec.Values[0].(type) {
						case *ast.BasicLit:
							id = time.Now().UTC().Format("20060102150405")
							value.Value = id
						}
					case "VersionID":
						switch value := spec.Values[0].(type) {
						case *ast.BasicLit:
							rev, err := exec.Command("git", "rev-parse", "HEAD").Output()
							if err != nil {
								log.Fatal(err)
							}
							hash = fmt.Sprintf(`"%s"`, strings.TrimSpace(string(rev)))
							value.Value = hash
						}
					}
				}
			}
		}
	}
	file, err := os.Open(path)
	if err != nil {
		log.Fatal(err)
	}
	info, err := file.Stat()
	if err != nil {
		log.Fatal(err)
	}
	var buf bytes.Buffer
	if err := printer.Fprint(&buf, fset, f); err != nil {
		log.Fatal(err)
	}
	fb, err := format.Source(buf.Bytes())
	if err != nil {
		log.Fatal(err)
	}
	if err := ioutil.WriteFile(path, fb, info.Mode()); err != nil {
		log.Fatal(err)
	}
	fmt.Printf("version:\n  hash: %s\n  id: %s\n", hash, id)
}
<commit_msg>cmd/scollector: Merge pull request #134 from Wessie/build-rewrite<commit_after>// Build sets scollector version information and should be run from the
// scollector directory.
package main

import (
	"bytes"
	"fmt"
	"go/ast"
	"go/parser"
	"go/printer"
	"go/token"
	"io"
	"log"
	"os"
	"os/exec"
	"strings"
	"time"
)

func main() {
	const path = "main.go"
	mainfile, err := os.OpenFile(path, os.O_RDWR, 0660)
	if err != nil {
		log.Fatal(err)
	}
	defer mainfile.Close()

	var hash, id string
	fset := token.NewFileSet()
	f, err := parser.ParseFile(fset, path, mainfile, parser.ParseComments)
	if err != nil {
		log.Fatal(err)
	}

	for _, d := range f.Decls {
		d, ok := d.(*ast.GenDecl)
		if !ok {
			continue
		}

		if d.Tok != token.CONST {
			continue
		}
		for _, spec := range d.Specs {
			spec, ok := spec.(*ast.ValueSpec)
			if !ok {
				continue
			}

			if len(spec.Names) != 1 || len(spec.Values) != 1 {
				continue
			}

			value, ok := spec.Values[0].(*ast.BasicLit)
			if !ok {
				continue
			}

			switch spec.Names[0].Name {
			case "VersionDate":
				id = time.Now().UTC().Format("20060102150405")
				value.Value = id
			case "VersionID":
				rev, err := exec.Command("git", "rev-parse", "HEAD").Output()
				if err != nil {
					log.Fatal(err)
				}
				hash = fmt.Sprintf(`"%s"`, strings.TrimSpace(string(rev)))
				value.Value = hash
			}
		}
	}

	var config = printer.Config{Mode: printer.UseSpaces | printer.TabIndent, Tabwidth: 8}
	var buf bytes.Buffer
	if err := config.Fprint(&buf, fset, f); err != nil {
		log.Fatal(err)
	}

	if _, err := mainfile.Seek(0, os.SEEK_SET); err != nil {
		log.Fatal(err)
	}

	if _, err := io.Copy(mainfile, &buf); err != nil {
		log.Fatal(err)
	}

	fmt.Printf("version:\n  hash: %s\n  id: %s\n", hash, id)
}
<|endoftext|>


<commit_before>package main

import (
	"net"

	"github.com/eahydra/socks"
)

type DecorateClient struct {
	forward socks.Dialer
	decorators []ConnDecorator
}

func NewDecorateClient(forward socks.Dialer, ds ...ConnDecorator) *DecorateClient {
	d := &DecorateClient{
		forward: forward,
	}
	d.decorators = append(d.decorators, ds...)
	return d
}

func (d *DecorateClient) Dial(network, address string) (net.Conn, error) {
	conn, err := d.forward.Dial(network, address)
	if err != nil {
		ErrLog.Println("DecorateClient forward.Dial failed, err:", err, address)
		return nil, err
	}
	dconn, err := DecorateConn(conn, d.decorators...)
	if err != nil {
		conn.Close()
		return nil, err
	}
	return dconn, nil
}<commit_msg>remove gofmt warning.<commit_after>package main

import (
	"net"

	"github.com/eahydra/socks"
)

type DecorateClient struct {
	forward    socks.Dialer
	decorators []ConnDecorator
}

func NewDecorateClient(forward socks.Dialer, ds ...ConnDecorator) *DecorateClient {
	d := &DecorateClient{
		forward: forward,
	}
	d.decorators = append(d.decorators, ds...)
	return d
}

func (d *DecorateClient) Dial(network, address string) (net.Conn, error) {
	conn, err := d.forward.Dial(network, address)
	if err != nil {
		ErrLog.Println("DecorateClient forward.Dial failed, err:", err, address)
		return nil, err
	}
	dconn, err := DecorateConn(conn, d.decorators...)
	if err != nil {
		conn.Close()
		return nil, err
	}
	return dconn, nil
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"os/signal"
	"sync"
	"syscall"
	"time"
)

func main() {
	terminationLog := flag.String("termination-log-file", "", "Write logs after SIGTERM to this file (in addition to stderr)")
	terminationLock := flag.String("termination-touch-file", "", "Touch this file on SIGTERM and delete on termination")

	flag.Parse()
	args := flag.CommandLine.Args()

	if len(args) == 0 {
		fmt.Println("Missing command line")
		os.Exit(1)
	}

	// use special tee-like writer when termination log is set
	termCh := make(chan struct{})
	var stderr io.Writer = os.Stderr
	if len(*terminationLog) > 0 {
		stderr = &terminationFileWriter{
			Writer:             os.Stderr,
			fn:                 *terminationLog,
			startFileLoggingCh: termCh,
		}
	}

	cmd := exec.Command(args[0], args[1:]...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = stderr

	// forward SIGTERM and SIGINT to child
	sigCh := make(chan os.Signal, 1)
	signal.Notify(sigCh, syscall.SIGTERM, syscall.SIGINT)
	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		defer wg.Done()
		for s := range sigCh {
			fmt.Fprintf(stderr, "Received signal %s\n", s)

			if len(*terminationLock) > 0 {
				fmt.Fprintln(stderr, "Touching", *terminationLock)
				if err := touch(*terminationLock); err != nil {
					fmt.Fprintln(stderr, fmt.Errorf("error touching %s: %v", *terminationLock, err))
					// keep going
				}
			}

			select {
			case <-termCh:
			default:
				close(termCh)
			}
			cmd.Process.Signal(s)
		}
	}()

	fmt.Printf("Launching %v\n", cmd)
	rc := 0
	if err := cmd.Run(); err != nil {
		if exitError, ok := err.(*exec.ExitError); !ok {
			rc = exitError.ExitCode()
		} else {
			fmt.Fprintf(stderr, "Failed to launch %s: %v\n", args[0], err)
			os.Exit(255)
		}
	}

	// remove signal handling
	signal.Stop(sigCh)
	close(sigCh)
	wg.Wait()

	if len(*terminationLock) > 0 {
		os.Remove(*terminationLock)
	}

	fmt.Fprintf(stderr, "Exit code %d\n", rc)
	os.Exit(rc)
}

// terminationFileWriter forwards everything to the embedded writer. When
// startFileLoggingCh is closed, everything is appended to the given file name
// in addition.
type terminationFileWriter struct {
	io.Writer
	fn                 string
	startFileLoggingCh <-chan struct{}

	f *os.File
}

func (w *terminationFileWriter) Write(bs []byte) (int, error) {
	select {
	case <-w.startFileLoggingCh:
		if w.f == nil {
			f, err := os.OpenFile(w.fn, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
			if err != nil {
				log.Fatal(err)
			}
			w.f = f
			fmt.Println("Starting logging to", w.fn)
		}
		if n, err := w.f.Write(bs); err != nil {
			return n, err
		} else if n != len(bs) {
			return n, io.ErrShortWrite
		}
	default:
	}

	return w.Writer.Write(bs)
}

func touch(fn string) error {
	_, err := os.Stat(fn)
	if os.IsNotExist(err) {
		file, err := os.Create(fn)
		if err != nil {
			return err
		}
		defer file.Close()
		return nil
	}

	currentTime := time.Now().Local()
	return os.Chtimes(fn, currentTime, currentTime)
}
<commit_msg>watch-termination: add log rotation<commit_after>package main

import (
	"flag"
	"fmt"
	"io"
	"os"
	"os/exec"
	"os/signal"
	"sync"
	"syscall"
	"time"

	"gopkg.in/natefinch/lumberjack.v2"
)

func main() {
	terminationLog := flag.String("termination-log-file", "", "Write logs after SIGTERM to this file (in addition to stderr)")
	terminationLock := flag.String("termination-touch-file", "", "Touch this file on SIGTERM and delete on termination")

	flag.Parse()
	args := flag.CommandLine.Args()

	if len(args) == 0 {
		fmt.Println("Missing command line")
		os.Exit(1)
	}

	// use special tee-like writer when termination log is set
	termCh := make(chan struct{})
	var stderr io.Writer = os.Stderr
	if len(*terminationLog) > 0 {
		stderr = &terminationFileWriter{
			Writer:             os.Stderr,
			fn:                 *terminationLog,
			startFileLoggingCh: termCh,
		}
	}

	cmd := exec.Command(args[0], args[1:]...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = stderr

	// forward SIGTERM and SIGINT to child
	sigCh := make(chan os.Signal, 1)
	signal.Notify(sigCh, syscall.SIGTERM, syscall.SIGINT)
	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		defer wg.Done()
		for s := range sigCh {
			select {
			case <-termCh:
			default:
				close(termCh)
			}

			fmt.Fprintf(stderr, "Received signal %s\n", s)

			if len(*terminationLock) > 0 {
				fmt.Fprintln(stderr, "Touching", *terminationLock)
				if err := touch(*terminationLock); err != nil {
					fmt.Fprintln(stderr, fmt.Errorf("error touching %s: %v", *terminationLock, err))
					// keep going
				}
			}

			cmd.Process.Signal(s)
		}
	}()

	fmt.Printf("Launching %v\n", cmd)
	rc := 0
	if err := cmd.Run(); err != nil {
		if exitError, ok := err.(*exec.ExitError); !ok {
			rc = exitError.ExitCode()
		} else {
			fmt.Fprintf(stderr, "Failed to launch %s: %v\n", args[0], err)
			os.Exit(255)
		}
	}

	// remove signal handling
	signal.Stop(sigCh)
	close(sigCh)
	wg.Wait()

	if len(*terminationLock) > 0 {
		os.Remove(*terminationLock)
	}

	fmt.Fprintf(stderr, "Exit code %d\n", rc)
	os.Exit(rc)
}

// terminationFileWriter forwards everything to the embedded writer. When
// startFileLoggingCh is closed, everything is appended to the given file name
// in addition.
type terminationFileWriter struct {
	io.Writer
	fn                 string
	startFileLoggingCh <-chan struct{}

	logger io.Writer
}

func (w *terminationFileWriter) Write(bs []byte) (int, error) {
	select {
	case <-w.startFileLoggingCh:
		if w.logger == nil {
			l := &lumberjack.Logger{
				Filename:   w.fn,
				MaxSize:    100,
				MaxBackups: 3,
				MaxAge:     28,
				Compress:   false,
			}
			w.logger = l
			fmt.Println("Starting logging to", w.fn)
		}
		if n, err := w.logger.Write(bs); err != nil {
			return n, err
		} else if n != len(bs) {
			return n, io.ErrShortWrite
		}
	default:
	}

	return w.Writer.Write(bs)
}

func touch(fn string) error {
	_, err := os.Stat(fn)
	if os.IsNotExist(err) {
		file, err := os.Create(fn)
		if err != nil {
			return err
		}
		defer file.Close()
		return nil
	}

	currentTime := time.Now().Local()
	return os.Chtimes(fn, currentTime, currentTime)
}
<|endoftext|>