<commit_before>// Package util implements small helper function that
// should be included in the stdlib in our opinion.
package util

import (
	"bytes"
	"io"
	"os"
	"sync"
	"sync/atomic"

	log "github.com/Sirupsen/logrus"
)

// Empty is just an empty struct.
// Empty{} reads nicer than struct{}{}
type Empty struct{}

// Min returns the minimum of a and b.
func Min(a, b int) int {
	if a < b {
		return a
	}
	return b
}

func Min64(a, b int64) int64 {
	if a < b {
		return a
	}
	return b
}

// Max returns the maximum of a and b.
func Max(a, b int) int {
	if a < b {
		return b
	}
	return a
}

func Max64(a, b int64) int64 {
	if a < b {
		return b
	}
	return a
}

// Clamp limits x to the range [lo, hi]
func Clamp(x, lo, hi int) int {
	return Max(lo, Min(x, hi))
}

// UMin returns the unsigned minimum of a and b
func UMin(a, b uint) uint {
	if a < b {
		return a
	}
	return b
}

// UMax returns the unsigned minimum of a and b
func UMax(a, b uint) uint {
	if a < b {
		return b
	}
	return a
}

// UClamp limits x to the range [lo, hi]
func UClamp(x, lo, hi uint) uint {
	return UMax(lo, UMin(x, hi))
}

// Closer closes c. If that fails, it will log the error.
// The intended usage is for convinient defer calls only!
// It gives only little knowledge about where the error is,
// but it's slightly better than a bare defer xyz.Close()
func Closer(c io.Closer) {
	if err := c.Close(); err != nil {
		log.Errorf("Error on close `%v`: %v", c, err)
	}
}

// Touch works like the unix touch(1)
func Touch(path string) error {
	fd, err := os.Create(path)
	if err != nil {
		return err
	}

	return fd.Close()
}

// SizeAccumulator is a io.Writer that simply counts
// the amount of bytes that has been written to it.
// It's useful to count the received bytes from a reader
// in conjunction with a io.TeeReader
//
// Example usage without error handling:
//
//   s := &SizeAccumulator{}
//   teeR := io.TeeReader(r, s)
//   io.Copy(os.Stdout, teeR)
//   fmt.Printf("Wrote %d bytes to stdout\n", s.Size())
//
type SizeAccumulator struct {
	size uint64
}

// Write simply increments the internal size count without any IO.
// It can be safely called from any go routine.
func (s *SizeAccumulator) Write(buf []byte) (int, error) {
	atomic.AddUint64(&s.size, uint64(len(buf)))
	return len(buf), nil
}

// Size returns the cumulated written bytes.
// It can be safely called from any go routine.
func (s *SizeAccumulator) Size() uint64 {
	return atomic.LoadUint64(&s.size)
}

// Reset resets the size counter to 0.
func (s *SizeAccumulator) Reset() {
	atomic.StoreUint64(&s.size, 0)
}

// NopCloser returns a WriteCloser with a no-op Close method wrapping
// the provided Writer w.
func NopWriteCloser(w io.Writer) io.WriteCloser {
	return nopCloser{w}
}

type nopCloser struct {
	io.Writer
}

func (nopCloser) Close() error { return nil }

type syncReadWriter struct {
	io.ReadWriter
	sync.Mutex
}

func (s *syncReadWriter) Write(buf []byte) (int, error) {
	s.Lock()
	defer s.Unlock()

	return s.ReadWriter.Write(buf)
}

func (s *syncReadWriter) Read(buf []byte) (int, error) {
	s.Lock()
	defer s.Unlock()

	return s.ReadWriter.Read(buf)
}

// SyncedReadWriter returns a io.ReadWriter that protects each call
// to Read() and Write() with a sync.Mutex.
func SyncedReadWriter(w io.ReadWriter) io.ReadWriter {
	return &syncReadWriter{ReadWriter: w}
}

// SyncBuffer is a bytes.Buffer that protects each call
// to Read() and Write() with a sync.RWMutex, i.e. parallel
// access to Read() is possible, but blocks when doing a Write().
type SyncBuffer struct {
	sync.RWMutex
	buf bytes.Buffer
}

func (b *SyncBuffer) Read(p []byte) (int, error) {
	b.Lock()
	defer b.Unlock()

	return b.buf.Read(p)
}

func (b *SyncBuffer) Write(p []byte) (int, error) {
	b.Lock()
	defer b.Unlock()

	return b.buf.Write(p)
}
<commit_msg>util: add TimeoutWriter<commit_after>// Package util implements small helper function that
// should be included in the stdlib in our opinion.
package util

import (
	"bytes"
	"errors"
	"io"
	"os"
	"sync"
	"sync/atomic"
	"time"

	log "github.com/Sirupsen/logrus"
)

// Empty is just an empty struct.
// Empty{} reads nicer than struct{}{}
type Empty struct{}

// Min returns the minimum of a and b.
func Min(a, b int) int {
	if a < b {
		return a
	}
	return b
}

func Min64(a, b int64) int64 {
	if a < b {
		return a
	}
	return b
}

// Max returns the maximum of a and b.
func Max(a, b int) int {
	if a < b {
		return b
	}
	return a
}

func Max64(a, b int64) int64 {
	if a < b {
		return b
	}
	return a
}

// Clamp limits x to the range [lo, hi]
func Clamp(x, lo, hi int) int {
	return Max(lo, Min(x, hi))
}

// UMin returns the unsigned minimum of a and b
func UMin(a, b uint) uint {
	if a < b {
		return a
	}
	return b
}

// UMax returns the unsigned minimum of a and b
func UMax(a, b uint) uint {
	if a < b {
		return b
	}
	return a
}

// UClamp limits x to the range [lo, hi]
func UClamp(x, lo, hi uint) uint {
	return UMax(lo, UMin(x, hi))
}

// Closer closes c. If that fails, it will log the error.
// The intended usage is for convinient defer calls only!
// It gives only little knowledge about where the error is,
// but it's slightly better than a bare defer xyz.Close()
func Closer(c io.Closer) {
	if err := c.Close(); err != nil {
		log.Errorf("Error on close `%v`: %v", c, err)
	}
}

// Touch works like the unix touch(1)
func Touch(path string) error {
	fd, err := os.Create(path)
	if err != nil {
		return err
	}

	return fd.Close()
}

// SizeAccumulator is a io.Writer that simply counts
// the amount of bytes that has been written to it.
// It's useful to count the received bytes from a reader
// in conjunction with a io.TeeReader
//
// Example usage without error handling:
//
//   s := &SizeAccumulator{}
//   teeR := io.TeeReader(r, s)
//   io.Copy(os.Stdout, teeR)
//   fmt.Printf("Wrote %d bytes to stdout\n", s.Size())
//
type SizeAccumulator struct {
	size uint64
}

// Write simply increments the internal size count without any IO.
// It can be safely called from any go routine.
func (s *SizeAccumulator) Write(buf []byte) (int, error) {
	atomic.AddUint64(&s.size, uint64(len(buf)))
	return len(buf), nil
}

// Size returns the cumulated written bytes.
// It can be safely called from any go routine.
func (s *SizeAccumulator) Size() uint64 {
	return atomic.LoadUint64(&s.size)
}

// Reset resets the size counter to 0.
func (s *SizeAccumulator) Reset() {
	atomic.StoreUint64(&s.size, 0)
}

// NopCloser returns a WriteCloser with a no-op Close method wrapping
// the provided Writer w.
func NopWriteCloser(w io.Writer) io.WriteCloser {
	return nopCloser{w}
}

type nopCloser struct {
	io.Writer
}

func (nopCloser) Close() error { return nil }

type syncReadWriter struct {
	io.ReadWriter
	sync.Mutex
}

func (s *syncReadWriter) Write(buf []byte) (int, error) {
	s.Lock()
	defer s.Unlock()

	return s.ReadWriter.Write(buf)
}

func (s *syncReadWriter) Read(buf []byte) (int, error) {
	s.Lock()
	defer s.Unlock()

	return s.ReadWriter.Read(buf)
}

// SyncedReadWriter returns a io.ReadWriter that protects each call
// to Read() and Write() with a sync.Mutex.
func SyncedReadWriter(w io.ReadWriter) io.ReadWriter {
	return &syncReadWriter{ReadWriter: w}
}

// SyncBuffer is a bytes.Buffer that protects each call
// to Read() and Write() with a sync.RWMutex, i.e. parallel
// access to Read() is possible, but blocks when doing a Write().
type SyncBuffer struct {
	sync.RWMutex
	buf bytes.Buffer
}

func (b *SyncBuffer) Read(p []byte) (int, error) {
	b.Lock()
	defer b.Unlock()

	return b.buf.Read(p)
}

func (b *SyncBuffer) Write(p []byte) (int, error) {
	b.Lock()
	defer b.Unlock()

	return b.buf.Write(p)
}

type timeoutWriter struct {
	io.Writer
	wait time.Duration
}

var ErrTimeout = errors.New("TimeoutWriter: Operation timed out.")

func (w *timeoutWriter) Write(p []byte) (n int, err error) {
	done, deadline := make(chan bool), time.After(w.wait)

	go func() {
		n, err = w.Writer.Write(p)
		done <- true
	}()

	select {
	case <-done:
		return
	case <-deadline:
		return 0, ErrTimeout
	}
}

// TimeoutWriter wraps `w` and returns a io.Writer that times out
// after `d` elapsed with ErrTimeout if `w` didn't succeed in that time.
func TimeoutWriter(w io.Writer, d time.Duration) io.Writer {
	return &timeoutWriter{w, d}
}
<|endoftext|>


<commit_before>// Copyright ©2015 The gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package mat64

import (
	"math"

	"github.com/gonum/blas"
	"github.com/gonum/lapack/lapack64"
)

// Solve solves a minimum-norm solution to a system of linear equations defined
// by the matrices a and b. If a is singular or near-singular a Condition error
// is returned. Please see the documentation for Condition for more information.
//
// The minimization problem solved depends on the input parameters.
//  1. If m >= n and trans == false, find X such that ||a*X - b||_2 is minimized.
//  2. If m < n and trans == false, find the minimum norm solution of a * X = b.
//  3. If m >= n and trans == true, find the minimum norm solution of a^T * X = b.
//  4. If m < n and trans == true, find X such that ||a*X - b||_2 is minimized.
// The solution matrix, X, is stored in place into the receiver.
func (m *Dense) Solve(a, b Matrix) error {
	ar, ac := a.Dims()
	br, bc := b.Dims()
	if ar != br {
		panic(ErrShape)
	}
	m.reuseAs(ac, bc)
	// TODO(btracey): Add a test for the condition number of A.
	// TODO(btracey): Add special cases for TriDense, SymDense, etc.
	switch {
	case ar == ac:
		if a == b {
			// x = I.
			if ar == 1 {
				m.mat.Data[0] = 1
				return nil
			}
			for i := 0; i < ar; i++ {
				v := m.mat.Data[i*m.mat.Stride:]
				zero(v[:i])
				v[i] = 1
				zero(v[i+1 : ac])
			}
			return nil
		}
		bMat, _ := untranspose(b)
		if m == bMat {
			var restore func()
			m, restore = m.isolatedWorkspace(bMat)
			defer restore()
		}
		// Solve using an LU decomposition.
		var lu LU
		lu.Factorize(a)
		if lu.Det() == 0 {
			return Condition(math.Inf(1))
		}
		m.Copy(b)
		lapack64.Getrs(blas.NoTrans, lu.lu.mat, m.mat, lu.pivot)
		return nil
	default:
		// Solve using QR/LQ.

		// Copy a since the corresponding LAPACK argument is modified during
		// the call.
		var aCopy Dense
		aCopy.Clone(a)

		x := getWorkspace(max(ar, ac), bc, false)
		defer putWorkspace(x)
		x.Copy(b)

		work := make([]float64, 1)
		lapack64.Gels(blas.NoTrans, aCopy.mat, x.mat, work, -1)
		work = make([]float64, int(work[0]))
		ok := lapack64.Gels(blas.NoTrans, aCopy.mat, x.mat, work, len(work))
		if !ok {
			return Condition(math.Inf(1))
		}
		m.Copy(x)
		return nil
	}
}

// SolveVec solves a minimum-norm solution to a system of linear equations defined
// by the matrices A and B. If A is singular or near-singular a Condition error
// is returned. Please see the documentation for more information.
func (v *Vector) SolveVec(a Matrix, b *Vector) error {
	_, c := a.Dims()
	// The Solve implementation is non-trivial, so rather than duplicate the code,
	// instead recast the Vectors as Dense and call the matrix code.
	v.reuseAs(c)
	m := vecAsDense(v)
	bm := vecAsDense(b)
	return m.Solve(a, bm)
}
<commit_msg>Only worry about shadowing if b transposed<commit_after>// Copyright ©2015 The gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package mat64

import (
	"math"

	"github.com/gonum/blas"
	"github.com/gonum/lapack/lapack64"
)

// Solve solves a minimum-norm solution to a system of linear equations defined
// by the matrices a and b. If a is singular or near-singular a Condition error
// is returned. Please see the documentation for Condition for more information.
//
// The minimization problem solved depends on the input parameters.
//  1. If m >= n and trans == false, find X such that ||a*X - b||_2 is minimized.
//  2. If m < n and trans == false, find the minimum norm solution of a * X = b.
//  3. If m >= n and trans == true, find the minimum norm solution of a^T * X = b.
//  4. If m < n and trans == true, find X such that ||a*X - b||_2 is minimized.
// The solution matrix, X, is stored in place into the receiver.
func (m *Dense) Solve(a, b Matrix) error {
	ar, ac := a.Dims()
	br, bc := b.Dims()
	if ar != br {
		panic(ErrShape)
	}
	m.reuseAs(ac, bc)
	// TODO(btracey): Add a test for the condition number of A.
	// TODO(btracey): Add special cases for TriDense, SymDense, etc.
	switch {
	case ar == ac:
		if a == b {
			// x = I.
			if ar == 1 {
				m.mat.Data[0] = 1
				return nil
			}
			for i := 0; i < ar; i++ {
				v := m.mat.Data[i*m.mat.Stride:]
				zero(v[:i])
				v[i] = 1
				zero(v[i+1 : ac])
			}
			return nil
		}
		// Solve using an LU decomposition.
		var lu LU
		lu.Factorize(a)
		if lu.Det() == 0 {
			return Condition(math.Inf(1))
		}
		bMat, bTrans := untranspose(b)
		if m == bMat && bTrans {
			var restore func()
			m, restore = m.isolatedWorkspace(bMat)
			defer restore()
		}
		if m != bMat {
			m.Copy(b)
		}
		lapack64.Getrs(blas.NoTrans, lu.lu.mat, m.mat, lu.pivot)
		return nil
	default:
		// Solve using QR/LQ.

		// Copy a since the corresponding LAPACK argument is modified during
		// the call.
		var aCopy Dense
		aCopy.Clone(a)

		x := getWorkspace(max(ar, ac), bc, false)
		defer putWorkspace(x)
		x.Copy(b)

		work := make([]float64, 1)
		lapack64.Gels(blas.NoTrans, aCopy.mat, x.mat, work, -1)
		work = make([]float64, int(work[0]))
		ok := lapack64.Gels(blas.NoTrans, aCopy.mat, x.mat, work, len(work))
		if !ok {
			return Condition(math.Inf(1))
		}
		m.Copy(x)
		return nil
	}
}

// SolveVec solves a minimum-norm solution to a system of linear equations defined
// by the matrices A and B. If A is singular or near-singular a Condition error
// is returned. Please see the documentation for more information.
func (v *Vector) SolveVec(a Matrix, b *Vector) error {
	_, c := a.Dims()
	// The Solve implementation is non-trivial, so rather than duplicate the code,
	// instead recast the Vectors as Dense and call the matrix code.
	v.reuseAs(c)
	m := vecAsDense(v)
	bm := vecAsDense(b)
	return m.Solve(a, bm)
}
<|endoftext|>


<commit_before><commit_msg>minor bug fixed<commit_after><|endoftext|>


<commit_before>// Copyright (c) 2012, Jorge Acereda Maciá. All rights reserved.
//
// Use of this source code is governed by a BSD-style license that can
// be found in the LICENSE file.

// Package ebml decodes EBML data.
//
// EBML is short for Extensible Binary Meta Language. EBML specifies a
// binary and octet (byte) aligned format inspired by the principle of
// XML. EBML itself is a generalized description of the technique of
// binary markup. Like XML, it is completely agnostic to any data that it
// might contain.
// For a specification, see http://ebml.sourceforge.net/specs/
package ebml

import (
	"errors"
	"fmt"
	"io"
	"log"
	"math"
	"reflect"
	"strconv"
)

var Verbose bool = false

// ReachedPayloadError is generated when a field tagged with
// ebmlstop:"1" is reached.
type ReachedPayloadError struct {
	*Element
}

func (r ReachedPayloadError) Error() string {
	return "Reached payload"
}

// Element represents an EBML-encoded chunk of data.
type Element struct {
	*limitedReadSeeker
	Size   int64
	Offset int64
	Id     uint
}

func (e *Element) String() string {
	return fmt.Sprintf("{ReadSeeker: %+v Size: %+v Offset: %+v Id: %x}", e.limitedReadSeeker, e.Size, e.Offset, e.Id)
}

// Creates the root element corresponding to the data available in r.
func RootElement(rs io.ReadSeeker) (*Element, error) {
	e := &Element{newLimitedReadSeeker(rs, math.MaxInt64), 0, 0, 0}
	return e, nil
}

func remaining(x int8) (rem int) {
	for x > 0 {
		rem++
		x += x
	}
	return
}

func parseVint(data []uint8) (val uint64) {
	for i, l := 0, len(data); i < l; i++ {
		val <<= 8
		val += uint64(data[i])
	}
	return
}

func readVintData(r io.Reader) (data []uint8, err error, rem int) {
	var v [16]uint8
	_, err = io.ReadFull(r, v[:1])
	if err == nil {
		rem = remaining(int8(v[0]))
		_, err = io.ReadFull(r, v[1:rem+1])
	}
	if err == nil {
		data = v[0 : rem+1]
	}
	return
}

func readVint(r io.Reader) (val uint64, err error, rem int) {
	var data []uint8
	data, err, rem = readVintData(r)
	if err == nil {
		val = parseVint(data)
	}
	return
}

func readSize(r io.Reader) (int64, error) {
	val, err, rem := readVint(r)
	return int64(val & ^(128 << uint(rem*8-rem))), err
}

// Next returns the next child element in an element.
func (e *Element) Next() (*Element, error) {
	off, err := e.Seek(0, 1)
	if err != nil {
		log.Panic(err)
	}
	id, err, _ := readVint(e)
	if err != nil {
		return nil, err
	}
	sz, err := readSize(e)
	if err != nil {
		return nil, err
	}
	ret := &Element{newLimitedReadSeeker(e, sz), sz, off, uint(id)}
	if Verbose {
		log.Println("next", ret, err)
	}
	return ret, err
}

func (e *Element) readUint64() (uint64, error) {
	d, err := e.ReadData()
	var i int
	sz := len(d)
	var val uint64
	for i = 0; i < sz; i++ {
		val <<= 8
		val += uint64(d[i])
	}
	return val, err
}

func (e *Element) readUint() (uint, error) {
	val, err := e.readUint64()
	return uint(val), err
}

func (e *Element) readString() (string, error) {
	s, err := e.ReadData()
	return string(s), err
}

func (e *Element) ReadData() (d []byte, err error) {
	sz := e.Size
	d = make([]uint8, sz, sz)
	_, err = io.ReadFull(e, d)
	return
}

func (e *Element) readFloat() (val float64, err error) {
	var uval uint64
	sz := e.Size
	uval, err = e.readUint64()
	if sz == 8 {
		val = math.Float64frombits(uval)
	} else {
		val = float64(math.Float32frombits(uint32(uval)))
	}
	return
}

func (e *Element) skip() (err error) {
	_, err = e.ReadData()
	return
}

// Unmarshal reads EBML data from r into data. Data must be a pointer
// to a struct. Fields present in the struct but absent in the stream
// will just keep their zero value.
// Returns an error that can be an io.Error or a ReachedPayloadError.
func (e *Element) Unmarshal(val interface{}) error {
	return e.readStruct(reflect.Indirect(reflect.ValueOf(val)))
}

func getTag(f reflect.StructField, s string) uint {
	sid := f.Tag.Get(s)
	id, _ := strconv.ParseUint(sid, 16, 0)
	return uint(id)
}

func lookup(reqid uint, t reflect.Type) int {
	for i, l := 0, t.NumField(); i < l; i++ {
		f := t.Field(i)
		if getTag(f, "ebml") == reqid {
			return i - 1000000*int(getTag(f, "ebmlstop"))
		}
	}
	return -1
}

func setDefaults(v reflect.Value) {
	t := v.Type()
	for i, l := 0, t.NumField(); i < l; i++ {
		fv := v.Field(i)
		switch fv.Kind() {
		case reflect.Int, reflect.Uint,
			reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
			reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64,
			reflect.Float32, reflect.Float64,
			reflect.String:
			setFieldDefaults(fv, t.Field(i), v)
		case reflect.Array, reflect.Struct, reflect.Slice:
			break
		default:
			log.Panic("Unsupported type")
		}
	}
}

func setFieldDefaults(v reflect.Value, sf reflect.StructField, s reflect.Value) {
	if v.CanInterface() && reflect.DeepEqual(
		v.Interface(), reflect.Zero(v.Type()).Interface()) {
		tag := sf.Tag.Get("ebmldef")
		if tag != "" {
			switch v.Kind() {
			case reflect.Int, reflect.Int64:
				u, _ := strconv.ParseInt(tag, 10, 0)
				v.SetInt(int64(u))
			case reflect.Uint, reflect.Uint64:
				u, _ := strconv.ParseUint(tag, 10, 0)
				v.SetUint(u)
			case reflect.Float32, reflect.Float64:
				f, _ := strconv.ParseFloat(tag, 64)
				v.SetFloat(f)
			case reflect.String:
				v.SetString(tag)
			default:
				log.Panic("Unsupported default value")
			}
		}
		ltag := sf.Tag.Get("ebmldeflink")
		if ltag != "" {
			v.Set(s.FieldByName(ltag))
		}
	}
}

func (e *Element) readStruct(v reflect.Value) (err error) {
	t := v.Type()
	for err == nil {
		var ne *Element
		ne, err = e.Next()
		if err == io.EOF {
			err = nil
			break
		}
		i := lookup(ne.Id, t)
		if i >= 0 {
			err = ne.readField(v.Field(i))
		} else if i == -1 {
			err = ne.skip()
		} else {
			var curr int64
			curr, err = e.Seek(ne.Offset, 0)
			if err != nil || curr != ne.Offset {
				log.Panic(err, curr)
			}
			err = ReachedPayloadError{e}
		}
	}
	setDefaults(v)
	return
}

func (e *Element) readField(v reflect.Value) (err error) {
	switch v.Kind() {
	case reflect.Struct:
		err = e.readStruct(v)
	case reflect.Slice:
		err = e.readSlice(v)
	case reflect.Array:
		for i, l := 0, v.Len(); i < l && err == nil; i++ {
			err = e.readStruct(v.Index(i))
		}
	case reflect.String:
		var s string
		s, err = e.readString()
		v.SetString(s)
	case reflect.Int, reflect.Int64:
		var u uint64
		u, err = e.readUint64()
		v.SetInt(int64(u))
	case reflect.Uint, reflect.Uint64:
		var u uint64
		u, err = e.readUint64()
		v.SetUint(u)
	case reflect.Float32, reflect.Float64:
		var f float64
		f, err = e.readFloat()
		v.SetFloat(f)
	default:
		err = errors.New("Unknown type: " + v.String())
	}
	return
}

func (e *Element) readSlice(v reflect.Value) (err error) {
	switch v.Type().Elem().Kind() {
	case reflect.Uint8:
		var sl []uint8
		sl, err = e.ReadData()
		if err == nil {
			if !v.CanSet() {
				log.Panic("can't set ", v, e)
			}
			v.Set(reflect.ValueOf(sl))
		}
	case reflect.Struct:
		vl := v.Len()
		ne := reflect.New(v.Type().Elem())
		nsl := reflect.Append(v, reflect.Indirect(ne))
		v.Set(nsl)
		err = e.readStruct(v.Index(vl))
	default:
		err = errors.New("Unknown slice type: " + v.String())
	}
	return
}
<commit_msg>Read cues<commit_after>// Copyright (c) 2012, Jorge Acereda Maciá. All rights reserved.
//
// Use of this source code is governed by a BSD-style license that can
// be found in the LICENSE file.

// Package ebml decodes EBML data.
//
// EBML is short for Extensible Binary Meta Language. EBML specifies a
// binary and octet (byte) aligned format inspired by the principle of
// XML. EBML itself is a generalized description of the technique of
// binary markup. Like XML, it is completely agnostic to any data that it
// might contain.
// For a specification, see http://ebml.sourceforge.net/specs/
package ebml

import (
	"errors"
	"fmt"
	"io"
	"log"
	"math"
	"reflect"
	"strconv"
)

var Verbose bool = false

// ReachedPayloadError is generated when a field tagged with
// ebmlstop:"1" is reached.
type ReachedPayloadError struct {
	*Element
}

func (r ReachedPayloadError) Error() string {
	return "Reached payload"
}

// Element represents an EBML-encoded chunk of data.
type Element struct {
	*limitedReadSeeker
	Offset int64
	Id     uint
}

func (e *Element) String() string {
	return fmt.Sprintf("{ReadSeeker: %+v Offset: %+v Id: %x}", e.limitedReadSeeker, e.Offset, e.Id)
}

// Creates the root element corresponding to the data available in r.
func RootElement(rs io.ReadSeeker) (*Element, error) {
	e := &Element{newLimitedReadSeeker(rs, math.MaxInt64), 0, 0}
	return e, nil
}

func remaining(x int8) (rem int) {
	for x > 0 {
		rem++
		x += x
	}
	return
}

func parseVint(data []uint8) (val uint64) {
	for i, l := 0, len(data); i < l; i++ {
		val <<= 8
		val += uint64(data[i])
	}
	return
}

func readVintData(r io.Reader) (data []uint8, err error, rem int) {
	var v [16]uint8
	_, err = io.ReadFull(r, v[:1])
	if err == nil {
		rem = remaining(int8(v[0]))
		_, err = io.ReadFull(r, v[1:rem+1])
	}
	if err == nil {
		data = v[0 : rem+1]
	}
	return
}

func readVint(r io.Reader) (val uint64, err error, rem int) {
	var data []uint8
	data, err, rem = readVintData(r)
	if err == nil {
		val = parseVint(data)
	}
	return
}

func readSize(r io.Reader) (int64, error) {
	val, err, rem := readVint(r)
	return int64(val & ^(128 << uint(rem*8-rem))), err
}

// Next returns the next child element in an element.
func (e *Element) Next() (*Element, error) {
	if Verbose {
		log.Println("next", e)
	}
	off, err := e.Seek(0, 1)
	if err != nil {
		log.Panic(err)
	}
	id, err, _ := readVint(e)
	if err != nil {
		return nil, err
	}
	sz, err := readSize(e)
	if err != nil {
		return nil, err
	}
	ret := &Element{newLimitedReadSeeker(e, sz), off, uint(id)}
	if Verbose {
		log.Println("--->", ret)
	}
	if id == 0x114D9B74 {
		log.Println("sh", ret)
	}
	return ret, err
}

func (e *Element) Size() int64 {
	return e.limitedReadSeeker.N
}

func (e *Element) readUint64() (uint64, error) {
	d, err := e.ReadData()
	var i int
	sz := len(d)
	var val uint64
	for i = 0; i < sz; i++ {
		val <<= 8
		val += uint64(d[i])
	}
	return val, err
}

func (e *Element) readUint() (uint, error) {
	val, err := e.readUint64()
	return uint(val), err
}

func (e *Element) readString() (string, error) {
	s, err := e.ReadData()
	return string(s), err
}

func (e *Element) ReadData() (d []byte, err error) {
	sz := e.Size()
	d = make([]uint8, sz, sz)
	_, err = io.ReadFull(e, d)
	return
}

func (e *Element) readFloat() (val float64, err error) {
	var uval uint64
	sz := e.Size()
	uval, err = e.readUint64()
	if sz == 8 {
		val = math.Float64frombits(uval)
	} else {
		val = float64(math.Float32frombits(uint32(uval)))
	}
	return
}

func (e *Element) skip() (err error) {
	_, err = e.ReadData()
	return
}

// Unmarshal reads EBML data from r into data. Data must be a pointer
// to a struct. Fields present in the struct but absent in the stream
// will just keep their zero value.
// Returns an error that can be an io.Error or a ReachedPayloadError.
func (e *Element) Unmarshal(val interface{}) error {
	return e.readStruct(reflect.Indirect(reflect.ValueOf(val)))
}

func getTag(f reflect.StructField, s string) uint {
	sid := f.Tag.Get(s)
	id, _ := strconv.ParseUint(sid, 16, 0)
	return uint(id)
}

func lookup(reqid uint, t reflect.Type) int {
	for i, l := 0, t.NumField(); i < l; i++ {
		f := t.Field(i)
		if getTag(f, "ebml") == reqid {
			return i - 1000000*int(getTag(f, "ebmlstop"))
		}
	}
	return -1
}

func setDefaults(v reflect.Value) {
	t := v.Type()
	for i, l := 0, t.NumField(); i < l; i++ {
		fv := v.Field(i)
		switch fv.Kind() {
		case reflect.Int, reflect.Uint,
			reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
			reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64,
			reflect.Float32, reflect.Float64,
			reflect.String:
			setFieldDefaults(fv, t.Field(i), v)
		case reflect.Array, reflect.Struct, reflect.Slice:
			break
		default:
			log.Panic("Unsupported type")
		}
	}
}

func setFieldDefaults(v reflect.Value, sf reflect.StructField, s reflect.Value) {
	if v.CanInterface() && reflect.DeepEqual(
		v.Interface(), reflect.Zero(v.Type()).Interface()) {
		tag := sf.Tag.Get("ebmldef")
		if tag != "" {
			switch v.Kind() {
			case reflect.Int, reflect.Int64:
				u, _ := strconv.ParseInt(tag, 10, 0)
				v.SetInt(int64(u))
			case reflect.Uint, reflect.Uint64:
				u, _ := strconv.ParseUint(tag, 10, 0)
				v.SetUint(u)
			case reflect.Float32, reflect.Float64:
				f, _ := strconv.ParseFloat(tag, 64)
				v.SetFloat(f)
			case reflect.String:
				v.SetString(tag)
			default:
				log.Panic("Unsupported default value")
			}
		}
		ltag := sf.Tag.Get("ebmldeflink")
		if ltag != "" {
			v.Set(s.FieldByName(ltag))
		}
	}
}

func (e *Element) readStruct(v reflect.Value) (err error) {
	t := v.Type()
	for err == nil {
		var ne *Element
		ne, err = e.Next()
		if err == io.EOF {
			err = nil
			break
		}
		i := lookup(ne.Id, t)
		if i >= 0 {
			err = ne.readField(v.Field(i))
		} else if i == -1 {
			err = ne.skip()
		} else {
			var curr int64
			curr, err = e.Seek(ne.Offset, 0)
			if err != nil || curr != ne.Offset {
				log.Panic(err, curr)
			}
			err = ReachedPayloadError{e}
		}
	}
	setDefaults(v)
	return
}

func (e *Element) readField(v reflect.Value) (err error) {
	switch v.Kind() {
	case reflect.Struct:
		err = e.readStruct(v)
	case reflect.Slice:
		err = e.readSlice(v)
	case reflect.Array:
		for i, l := 0, v.Len(); i < l && err == nil; i++ {
			err = e.readStruct(v.Index(i))
		}
	case reflect.String:
		var s string
		s, err = e.readString()
		v.SetString(s)
	case reflect.Int, reflect.Int64:
		var u uint64
		u, err = e.readUint64()
		v.SetInt(int64(u))
	case reflect.Uint, reflect.Uint64:
		var u uint64
		u, err = e.readUint64()
		v.SetUint(u)
	case reflect.Float32, reflect.Float64:
		var f float64
		f, err = e.readFloat()
		v.SetFloat(f)
	default:
		err = errors.New("Unknown type: " + v.String())
	}
	return
}

func (e *Element) readSlice(v reflect.Value) (err error) {
	switch v.Type().Elem().Kind() {
	case reflect.Uint8:
		var sl []uint8
		sl, err = e.ReadData()
		if err == nil {
			if !v.CanSet() {
				log.Panic("can't set ", v, e)
			}
			v.Set(reflect.ValueOf(sl))
		}
	case reflect.Struct:
		vl := v.Len()
		ne := reflect.New(v.Type().Elem())
		nsl := reflect.Append(v, reflect.Indirect(ne))
		v.Set(nsl)
		err = e.readStruct(v.Index(vl))
	default:
		err = errors.New("Unknown slice type: " + v.String())
	}
	return
}
<|endoftext|>


<commit_before>package utils

import (
	"errors"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
	"os"

	"github.com/techjanitor/pram-libs/config"
)

// Authenticate to AWS and return handler
func AWSSession() (svc *session.Session, err error) {

	// new credentials from settings
	creds := credentials.NewStaticCredentials(config.Settings.Amazon.Id, config.Settings.Amazon.Key, "")

	// create our session
	svc = session.New(&aws.Config{
		Region:      aws.String(config.Settings.Amazon.Region),
		Credentials: creds,
	})

	return

}

// Upload a file to S3
func UploadS3(filepath, filename, mime string) (err error) {

	session, err := AWSSession()
	if err != nil {
		return
	}

	file, err := os.Open(filepath)
	if err != nil {
		return errors.New("problem opening file for s3")
	}
	defer file.Close()

	uploader := s3manager.NewUploader(session)

	params := &s3manager.UploadInput{
		Bucket:               aws.String(config.Settings.Amazon.Bucket),
		Key:                  aws.String(filename),
		Body:                 file,
		ContentType:          aws.String(mime),
		ServerSideEncryption: aws.String(s3.ServerSideEncryptionAes256),
	}

	_, err = uploader.Upload(params)

	return

}

// Delete a file from S3
func DeleteS3(object string) (err error) {

	session, err := AWSSession()
	if err != nil {
		return
	}

	svc := s3.New(session)

	params := &s3.DeleteObjectInput{
		Bucket: aws.String(config.Settings.Amazon.Bucket),
		Key:    aws.String(object),
	}

	_, err = svc.DeleteObject(params)

	return

}
<commit_msg>add cache-control header to s3 objects<commit_after>package utils

import (
	"errors"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
	"os"

	"github.com/techjanitor/pram-libs/config"
)

// Authenticate to AWS and return handler
func AWSSession() (svc *session.Session, err error) {

	// new credentials from settings
	creds := credentials.NewStaticCredentials(config.Settings.Amazon.Id, config.Settings.Amazon.Key, "")

	// create our session
	svc = session.New(&aws.Config{
		Region:      aws.String(config.Settings.Amazon.Region),
		Credentials: creds,
	})

	return

}

// Upload a file to S3
func UploadS3(filepath, filename, mime string) (err error) {

	session, err := AWSSession()
	if err != nil {
		return
	}

	file, err := os.Open(filepath)
	if err != nil {
		return errors.New("problem opening file for s3")
	}
	defer file.Close()

	uploader := s3manager.NewUploader(session)

	params := &s3manager.UploadInput{
		Bucket:               aws.String(config.Settings.Amazon.Bucket),
		Key:                  aws.String(filename),
		Body:                 file,
		ContentType:          aws.String(mime),
		CacheControl:         aws.String("public, max-age=31536000"),
		ServerSideEncryption: aws.String(s3.ServerSideEncryptionAes256),
	}

	_, err = uploader.Upload(params)

	return

}

// Delete a file from S3
func DeleteS3(object string) (err error) {

	session, err := AWSSession()
	if err != nil {
		return
	}

	svc := s3.New(session)

	params := &s3.DeleteObjectInput{
		Bucket: aws.String(config.Settings.Amazon.Bucket),
		Key:    aws.String(object),
	}

	_, err = svc.DeleteObject(params)

	return

}
<|endoftext|>


<commit_before>package cluster

import (
	"context"
	"fmt"
	"math/rand"
	"net/http"
	"os"
	"time"

	"github.com/canonical/go-dqlite/client"

	"github.com/lxc/lxd/client"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/state"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/log15"
	"github.com/lxc/lxd/shared/logger"
	"github.com/pkg/errors"
)

// NotifyUpgradeCompleted sends a notification to all other nodes in the
// cluster that any possible pending database update has been applied, and any
// nodes which was waiting for this node to be upgraded should re-check if it's
// okay to move forward.
func NotifyUpgradeCompleted(state *state.State, networkCert *shared.CertInfo, serverCert *shared.CertInfo) error {
	notifier, err := NewNotifier(state, networkCert, serverCert, NotifyTryAll)
	if err != nil {
		return err
	}

	return notifier(func(client lxd.InstanceServer) error {
		info, err := client.GetConnectionInfo()
		if err != nil {
			return errors.Wrap(err, "failed to get connection info")
		}

		url := fmt.Sprintf("%s%s", info.Addresses[0], databaseEndpoint)
		request, err := http.NewRequest("PATCH", url, nil)
		if err != nil {
			return errors.Wrap(err, "failed to create database notify upgrade request")
		}
		setDqliteVersionHeader(request)

		httpClient, err := client.GetHTTPClient()
		if err != nil {
			return errors.Wrap(err, "failed to get HTTP client")
		}

		httpClient.Timeout = 5 * time.Second
		response, err := httpClient.Do(request)
		if err != nil {
			return errors.Wrap(err, "failed to notify node about completed upgrade")
		}

		if response.StatusCode != http.StatusOK {
			return fmt.Errorf("database upgrade notification failed: %s", response.Status)
		}

		return nil
	})
}

// MaybeUpdate Check this node's version and possibly run LXD_CLUSTER_UPDATE.
func MaybeUpdate(state *state.State) error {
	shouldUpdate := false

	enabled, err := Enabled(state.Node)
	if err != nil {
		return errors.Wrap(err, "Failed to check clustering is enabled")
	}
	if !enabled {
		return nil
	}

	if state.Cluster == nil {
		return fmt.Errorf("Failed checking cluster update, state not initialised yet")
	}

	err = state.Cluster.Transaction(func(tx *db.ClusterTx) error {
		outdated, err := tx.NodeIsOutdated()
		if err != nil {
			return err
		}
		shouldUpdate = outdated
		return nil
	})

	if err != nil {
		// Just log the error and return.
		return errors.Wrap(err, "Failed to check if this node is out-of-date")
	}

	if !shouldUpdate {
		logger.Debugf("Cluster node is up-to-date")
		return nil
	}

	return triggerUpdate()
}

func triggerUpdate() error {
	logger.Infof("Node is out-of-date with respect to other cluster nodes")

	updateExecutable := os.Getenv("LXD_CLUSTER_UPDATE")
	if updateExecutable == "" {
		logger.Debug("No LXD_CLUSTER_UPDATE variable set, skipping auto-update")
		return nil
	}

	// Wait a random amout of seconds (up to 30) in order to avoid
	// restarting all cluster members at the same time, and make the
	// upgrade more graceful.
	wait := time.Duration(rand.Intn(30)) * time.Second
	logger.Infof("Triggering cluster update in %s using: %s", wait, updateExecutable)
	time.Sleep(wait)

	_, err := shared.RunCommand(updateExecutable)
	if err != nil {
		logger.Errorf("Cluster upgrade failed: '%v'", err.Error())
		return err
	}
	return nil
}

// UpgradeMembersWithoutRole assigns the Spare raft role to all cluster members
// that are not currently part of the raft configuration. It's used for
// upgrading a cluster from a version without roles support.
func UpgradeMembersWithoutRole(gateway *Gateway, members []db.NodeInfo) error {
	nodes, err := gateway.currentRaftNodes()
	if err == ErrNotLeader {
		return nil
	}
	if err != nil {
		return errors.Wrap(err, "Failed to get current raft nodes")
	}

	// Used raft IDs.
	ids := map[uint64]bool{}
	for _, node := range nodes {
		ids[node.ID] = true
	}

	dqliteClient, err := gateway.getClient()
	if err != nil {
		return errors.Wrap(err, "Failed to connect to local dqlite node")
	}
	defer dqliteClient.Close()

	// Check that each member is present in the raft configuration, and add
	// it if not.
	for _, member := range members {
		found := false
		for _, node := range nodes {
			if member.ID == 1 && node.ID == 1 || member.Address == node.Address {
				found = true
				break
			}
		}
		if found {
			continue
		}

		// Try to use the same ID as the node, but it might not be
		// possible if it's use.
		id := uint64(member.ID)
		if _, ok := ids[id]; ok {
			for _, other := range members {
				if _, ok := ids[uint64(other.ID)]; !ok {
					id = uint64(other.ID)
					break
				}
			}

			// This can't really happen since there are always at least as many members as there are
			// nodes, and all of them have different IDs.
			if id == uint64(member.ID) {
				panic("no available ID")
			}
		}
		ids[id] = true

		info := db.RaftNode{
			NodeInfo: client.NodeInfo{
				ID:      id,
				Address: member.Address,
				Role:    db.RaftSpare,
			},
			Name: "",
		}

		logger.Info("Add spare dqlite node", log15.Ctx{"id": info.ID, "address": info.Address})

		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
		defer cancel()
		err = dqliteClient.Add(ctx, info.NodeInfo)
		if err != nil {
			return errors.Wrap(err, "Failed to add dqlite node")
		}

	}

	return nil
}
<commit_msg>lxd/cluster/upgrade: Don't panic in UpgradeMembersWithoutRole in raft ID not found<commit_after>package cluster

import (
	"context"
	"fmt"
	"math/rand"
	"net/http"
	"os"
	"time"

	"github.com/canonical/go-dqlite/client"

	"github.com/lxc/lxd/client"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/state"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/log15"
	"github.com/lxc/lxd/shared/logger"
	"github.com/pkg/errors"
)

// NotifyUpgradeCompleted sends a notification to all other nodes in the
// cluster that any possible pending database update has been applied, and any
// nodes which was waiting for this node to be upgraded should re-check if it's
// okay to move forward.
func NotifyUpgradeCompleted(state *state.State, networkCert *shared.CertInfo, serverCert *shared.CertInfo) error {
	notifier, err := NewNotifier(state, networkCert, serverCert, NotifyTryAll)
	if err != nil {
		return err
	}

	return notifier(func(client lxd.InstanceServer) error {
		info, err := client.GetConnectionInfo()
		if err != nil {
			return errors.Wrap(err, "failed to get connection info")
		}

		url := fmt.Sprintf("%s%s", info.Addresses[0], databaseEndpoint)
		request, err := http.NewRequest("PATCH", url, nil)
		if err != nil {
			return errors.Wrap(err, "failed to create database notify upgrade request")
		}
		setDqliteVersionHeader(request)

		httpClient, err := client.GetHTTPClient()
		if err != nil {
			return errors.Wrap(err, "failed to get HTTP client")
		}

		httpClient.Timeout = 5 * time.Second
		response, err := httpClient.Do(request)
		if err != nil {
			return errors.Wrap(err, "failed to notify node about completed upgrade")
		}

		if response.StatusCode != http.StatusOK {
			return fmt.Errorf("database upgrade notification failed: %s", response.Status)
		}

		return nil
	})
}

// MaybeUpdate Check this node's version and possibly run LXD_CLUSTER_UPDATE.
func MaybeUpdate(state *state.State) error {
	shouldUpdate := false

	enabled, err := Enabled(state.Node)
	if err != nil {
		return errors.Wrap(err, "Failed to check clustering is enabled")
	}
	if !enabled {
		return nil
	}

	if state.Cluster == nil {
		return fmt.Errorf("Failed checking cluster update, state not initialised yet")
	}

	err = state.Cluster.Transaction(func(tx *db.ClusterTx) error {
		outdated, err := tx.NodeIsOutdated()
		if err != nil {
			return err
		}
		shouldUpdate = outdated
		return nil
	})

	if err != nil {
		// Just log the error and return.
		return errors.Wrap(err, "Failed to check if this node is out-of-date")
	}

	if !shouldUpdate {
		logger.Debugf("Cluster node is up-to-date")
		return nil
	}

	return triggerUpdate()
}

func triggerUpdate() error {
	logger.Infof("Node is out-of-date with respect to other cluster nodes")

	updateExecutable := os.Getenv("LXD_CLUSTER_UPDATE")
	if updateExecutable == "" {
		logger.Debug("No LXD_CLUSTER_UPDATE variable set, skipping auto-update")
		return nil
	}

	// Wait a random amout of seconds (up to 30) in order to avoid
	// restarting all cluster members at the same time, and make the
	// upgrade more graceful.
	wait := time.Duration(rand.Intn(30)) * time.Second
	logger.Infof("Triggering cluster update in %s using: %s", wait, updateExecutable)
	time.Sleep(wait)

	_, err := shared.RunCommand(updateExecutable)
	if err != nil {
		logger.Errorf("Cluster upgrade failed: '%v'", err.Error())
		return err
	}
	return nil
}

// UpgradeMembersWithoutRole assigns the Spare raft role to all cluster members
// that are not currently part of the raft configuration. It's used for
// upgrading a cluster from a version without roles support.
func UpgradeMembersWithoutRole(gateway *Gateway, members []db.NodeInfo) error {
	nodes, err := gateway.currentRaftNodes()
	if err == ErrNotLeader {
		return nil
	}
	if err != nil {
		return errors.Wrap(err, "Failed to get current raft nodes")
	}

	// Used raft IDs.
	ids := map[uint64]bool{}
	for _, node := range nodes {
		ids[node.ID] = true
	}

	dqliteClient, err := gateway.getClient()
	if err != nil {
		return errors.Wrap(err, "Failed to connect to local dqlite node")
	}
	defer dqliteClient.Close()

	// Check that each member is present in the raft configuration, and add
	// it if not.
	for _, member := range members {
		found := false
		for _, node := range nodes {
			if member.ID == 1 && node.ID == 1 || member.Address == node.Address {
				found = true
				break
			}
		}
		if found {
			continue
		}

		// Try to use the same ID as the node, but it might not be
		// possible if it's use.
		id := uint64(member.ID)
		if _, ok := ids[id]; ok {
			for _, other := range members {
				if _, ok := ids[uint64(other.ID)]; !ok {
					id = uint64(other.ID)
					break
				}
			}

			// This can't really happen since there are always at least as many members as there are
			// nodes, and all of them have different IDs.
			if id == uint64(member.ID) {
				return fmt.Errorf("No available raft ID for cluster member ID %d", member.ID)
			}
		}
		ids[id] = true

		info := db.RaftNode{
			NodeInfo: client.NodeInfo{
				ID:      id,
				Address: member.Address,
				Role:    db.RaftSpare,
			},
			Name: "",
		}

		logger.Info("Add spare dqlite node", log15.Ctx{"id": info.ID, "address": info.Address})

		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
		defer cancel()
		err = dqliteClient.Add(ctx, info.NodeInfo)
		if err != nil {
			return errors.Wrap(err, "Failed to add dqlite node")
		}

	}

	return nil
}
<|endoftext|>


<commit_before>/*
 *
 * Copyright 2014, Google Inc.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met:
 *
 *     * Redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above
 * copyright notice, this list of conditions and the following disclaimer
 * in the documentation and/or other materials provided with the
 * distribution.
 *     * Neither the name of Google Inc. nor the names of its
 * contributors may be used to endorse or promote products derived from
 * this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 */

package grpc

import (
	"errors"
	"io"
	"sync"
	"time"

	"golang.org/x/net/context"
	"golang.org/x/net/trace"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/metadata"
	"google.golang.org/grpc/transport"
)

type streamHandler func(srv interface{}, stream ServerStream) error

// StreamDesc represents a streaming RPC service's method specification.
type StreamDesc struct {
	StreamName string
	Handler    streamHandler

	// At least one of these is true.
	ServerStreams bool
	ClientStreams bool
}

// Stream defines the common interface a client or server stream has to satisfy.
type Stream interface {
	// Context returns the context for this stream.
	Context() context.Context
	// SendMsg blocks until it sends m, the stream is done or the stream
	// breaks.
	// On error, it aborts the stream and returns an RPC status on client
	// side. On server side, it simply returns the error to the caller.
	// SendMsg is called by generated code.
	SendMsg(m interface{}) error
	// RecvMsg blocks until it receives a message or the stream is
	// done. On client side, it returns io.EOF when the stream is done. On
	// any other error, it aborts the streama nd returns an RPC status. On
	// server side, it simply returns the error to the caller.
	RecvMsg(m interface{}) error
}

// ClientStream defines the interface a client stream has to satify.
type ClientStream interface {
	// Header returns the header metedata received from the server if there
	// is any. It blocks if the metadata is not ready to read.
	Header() (metadata.MD, error)
	// Trailer returns the trailer metadata from the server. It must be called
	// after stream.Recv() returns non-nil error (including io.EOF) for
	// bi-directional streaming and server streaming or stream.CloseAndRecv()
	// returns for client streaming in order to receive trailer metadata if
	// present. Otherwise, it could returns an empty MD even though trailer
	// is present.
	Trailer() metadata.MD
	// CloseSend closes the send direction of the stream. It closes the stream
	// when non-nil error is met.
	CloseSend() error
	Stream
}

// NewClientStream creates a new Stream for the client side. This is called
// by generated code.
func NewClientStream(ctx context.Context, desc *StreamDesc, cc *ClientConn, method string, opts ...CallOption) (ClientStream, error) {
	var (
		t   transport.ClientTransport
		err error
	)
	t, err = cc.dopts.picker.Pick(ctx)
	if err != nil {
		return nil, toRPCErr(err)
	}
	// TODO(zhaoq): CallOption is omitted. Add support when it is needed.
	callHdr := &transport.CallHdr{
		Host:   cc.authority,
		Method: method,
	}
	cs := &clientStream{
		desc:    desc,
		codec:   cc.dopts.codec,
		tracing: EnableTracing,
	}
	if cs.tracing {
		cs.trInfo.tr = trace.New("grpc.Sent."+methodFamily(method), method)
		cs.trInfo.firstLine.client = true
		if deadline, ok := ctx.Deadline(); ok {
			cs.trInfo.firstLine.deadline = deadline.Sub(time.Now())
		}
		cs.trInfo.tr.LazyLog(&cs.trInfo.firstLine, false)
		ctx = trace.NewContext(ctx, cs.trInfo.tr)
	}
	s, err := t.NewStream(ctx, callHdr)
	if err != nil {
		cs.finish(err)
		return nil, toRPCErr(err)
	}
	cs.t = t
	cs.s = s
	cs.p = &parser{s: s}
	// Listen on ctx.Done() to detect cancellation when there is no pending
	// I/O operations on this stream.
	go func() {
		select {
		case <-t.Error():
			// Incur transport error, simply exit.
		case <-s.Context().Done():
			cs.closeTransportStream(transport.ContextErr(s.Context().Err()))
		}
	}()
	return cs, nil
}

// clientStream implements a client side Stream.
type clientStream struct {
	t     transport.ClientTransport
	s     *transport.Stream
	p     *parser
	desc  *StreamDesc
	codec Codec

	tracing bool // set to EnableTracing when the clientStream is created.

	mu     sync.Mutex
	closed bool
	// trInfo.tr is set when the clientStream is created (if EnableTracing is true),
	// and is set to nil when the clientStream's finish method is called.
	trInfo traceInfo
}

func (cs *clientStream) Context() context.Context {
	return cs.s.Context()
}

func (cs *clientStream) Header() (metadata.MD, error) {
	m, err := cs.s.Header()
	if err != nil {
		if _, ok := err.(transport.ConnectionError); !ok {
			cs.closeTransportStream(err)
		}
	}
	return m, err
}

func (cs *clientStream) Trailer() metadata.MD {
	return cs.s.Trailer()
}

func (cs *clientStream) SendMsg(m interface{}) (err error) {
	if cs.tracing {
		cs.mu.Lock()
		if cs.trInfo.tr != nil {
			cs.trInfo.tr.LazyLog(&payload{sent: true, msg: m}, true)
		}
		cs.mu.Unlock()
	}
	defer func() {
		if err == nil || err == io.EOF {
			return
		}
		if _, ok := err.(transport.ConnectionError); !ok {
			cs.closeTransportStream(err)
		}
		err = toRPCErr(err)
	}()
	out, err := encode(cs.codec, m, compressionNone)
	if err != nil {
		return transport.StreamErrorf(codes.Internal, "grpc: %v", err)
	}
	return cs.t.Write(cs.s, out, &transport.Options{Last: false})
}

func (cs *clientStream) RecvMsg(m interface{}) (err error) {
	err = recv(cs.p, cs.codec, m)
	defer func() {
		// err != nil indicates the termination of the stream.
		if err != nil {
			cs.finish(err)
		}
	}()
	if err == nil {
		if cs.tracing {
			cs.mu.Lock()
			if cs.trInfo.tr != nil {
				cs.trInfo.tr.LazyLog(&payload{sent: false, msg: m}, true)
			}
			cs.mu.Unlock()
		}
		if !cs.desc.ClientStreams || cs.desc.ServerStreams {
			return
		}
		// Special handling for client streaming rpc.
		err = recv(cs.p, cs.codec, m)
		cs.closeTransportStream(err)
		if err == nil {
			return toRPCErr(errors.New("grpc: client streaming protocol violation: get <nil>, want <EOF>"))
		}
		if err == io.EOF {
			if cs.s.StatusCode() == codes.OK {
				return nil
			}
			return Errorf(cs.s.StatusCode(), cs.s.StatusDesc())
		}
		return toRPCErr(err)
	}
	if _, ok := err.(transport.ConnectionError); !ok {
		cs.closeTransportStream(err)
	}
	if err == io.EOF {
		if cs.s.StatusCode() == codes.OK {
			// Returns io.EOF to indicate the end of the stream.
			return
		}
		return Errorf(cs.s.StatusCode(), cs.s.StatusDesc())
	}
	return toRPCErr(err)
}

func (cs *clientStream) CloseSend() (err error) {
	err = cs.t.Write(cs.s, nil, &transport.Options{Last: true})
	if err == nil || err == io.EOF {
		return
	}
	if _, ok := err.(transport.ConnectionError); !ok {
		cs.closeTransportStream(err)
	}
	err = toRPCErr(err)
	return
}

func (cs *clientStream) closeTransportStream(err error) {
	cs.mu.Lock()
	if cs.closed {
		cs.mu.Unlock()
		return
	}
	cs.closed = true
	cs.mu.Unlock()
	cs.t.CloseStream(cs.s, err)
}

func (cs *clientStream) finish(err error) {
	if !cs.tracing {
		return
	}
	cs.mu.Lock()
	defer cs.mu.Unlock()
	if cs.trInfo.tr != nil {
		if err == nil || err == io.EOF {
			cs.trInfo.tr.LazyPrintf("RPC: [OK]")
		} else {
			cs.trInfo.tr.LazyPrintf("RPC: [%v]", err)
			cs.trInfo.tr.SetError()
		}
		cs.trInfo.tr.Finish()
		cs.trInfo.tr = nil
	}
}

// ServerStream defines the interface a server stream has to satisfy.
type ServerStream interface {
	// SendHeader sends the header metadata. It should not be called
	// after SendProto. It fails if called multiple times or if
	// called after SendProto.
	SendHeader(metadata.MD) error
	// SetTrailer sets the trailer metadata which will be sent with the
	// RPC status.
	SetTrailer(metadata.MD)
	Stream
}

// serverStream implements a server side Stream.
type serverStream struct {
	t          transport.ServerTransport
	s          *transport.Stream
	p          *parser
	codec      Codec
	statusCode codes.Code
	statusDesc string
	trInfo     *traceInfo

	mu sync.Mutex // protects trInfo.tr after the service handler runs.
}

func (ss *serverStream) Context() context.Context {
	return ss.s.Context()
}

func (ss *serverStream) SendHeader(md metadata.MD) error {
	return ss.t.WriteHeader(ss.s, md)
}

func (ss *serverStream) SetTrailer(md metadata.MD) {
	if md.Len() == 0 {
		return
	}
	ss.s.SetTrailer(md)
	return
}

func (ss *serverStream) SendMsg(m interface{}) (err error) {
	defer func() {
		if ss.trInfo != nil {
			ss.mu.Lock()
			if ss.trInfo.tr != nil {
				if err == nil {
					ss.trInfo.tr.LazyLog(&payload{sent: true, msg: m}, true)
				} else {
					ss.trInfo.tr.LazyLog(&fmtStringer{"%v", []interface{}{err}}, true)
					ss.trInfo.tr.SetError()
				}
			}
			ss.mu.Unlock()
		}
	}()
	out, err := encode(ss.codec, m, compressionNone)
	if err != nil {
		err = transport.StreamErrorf(codes.Internal, "grpc: %v", err)
		return err
	}
	return ss.t.Write(ss.s, out, &transport.Options{Last: false})
}

func (ss *serverStream) RecvMsg(m interface{}) (err error) {
	defer func() {
		if ss.trInfo != nil {
			ss.mu.Lock()
			if ss.trInfo.tr != nil {
				if err == nil {
					ss.trInfo.tr.LazyLog(&payload{sent: false, msg: m}, true)
				} else if err != io.EOF {
					ss.trInfo.tr.LazyLog(&fmtStringer{"%v", []interface{}{err}}, true)
					ss.trInfo.tr.SetError()
				}
			}
			ss.mu.Unlock()
		}
	}()
	return recv(ss.p, ss.codec, m)
}
<commit_msg>Add missing trace finish on an uncommon code path<commit_after>/*
 *
 * Copyright 2014, Google Inc.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met:
 *
 *     * Redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above
 * copyright notice, this list of conditions and the following disclaimer
 * in the documentation and/or other materials provided with the
 * distribution.
 *     * Neither the name of Google Inc. nor the names of its
 * contributors may be used to endorse or promote products derived from
 * this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 */

package grpc

import (
	"errors"
	"io"
	"sync"
	"time"

	"golang.org/x/net/context"
	"golang.org/x/net/trace"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/metadata"
	"google.golang.org/grpc/transport"
)

type streamHandler func(srv interface{}, stream ServerStream) error

// StreamDesc represents a streaming RPC service's method specification.
type StreamDesc struct {
	StreamName string
	Handler    streamHandler

	// At least one of these is true.
	ServerStreams bool
	ClientStreams bool
}

// Stream defines the common interface a client or server stream has to satisfy.
type Stream interface {
	// Context returns the context for this stream.
	Context() context.Context
	// SendMsg blocks until it sends m, the stream is done or the stream
	// breaks.
	// On error, it aborts the stream and returns an RPC status on client
	// side. On server side, it simply returns the error to the caller.
	// SendMsg is called by generated code.
	SendMsg(m interface{}) error
	// RecvMsg blocks until it receives a message or the stream is
	// done. On client side, it returns io.EOF when the stream is done. On
	// any other error, it aborts the streama nd returns an RPC status. On
	// server side, it simply returns the error to the caller.
	RecvMsg(m interface{}) error
}

// ClientStream defines the interface a client stream has to satify.
type ClientStream interface {
	// Header returns the header metedata received from the server if there
	// is any. It blocks if the metadata is not ready to read.
	Header() (metadata.MD, error)
	// Trailer returns the trailer metadata from the server. It must be called
	// after stream.Recv() returns non-nil error (including io.EOF) for
	// bi-directional streaming and server streaming or stream.CloseAndRecv()
	// returns for client streaming in order to receive trailer metadata if
	// present. Otherwise, it could returns an empty MD even though trailer
	// is present.
	Trailer() metadata.MD
	// CloseSend closes the send direction of the stream. It closes the stream
	// when non-nil error is met.
	CloseSend() error
	Stream
}

// NewClientStream creates a new Stream for the client side. This is called
// by generated code.
func NewClientStream(ctx context.Context, desc *StreamDesc, cc *ClientConn, method string, opts ...CallOption) (ClientStream, error) {
	var (
		t   transport.ClientTransport
		err error
	)
	t, err = cc.dopts.picker.Pick(ctx)
	if err != nil {
		return nil, toRPCErr(err)
	}
	// TODO(zhaoq): CallOption is omitted. Add support when it is needed.
	callHdr := &transport.CallHdr{
		Host:   cc.authority,
		Method: method,
	}
	cs := &clientStream{
		desc:    desc,
		codec:   cc.dopts.codec,
		tracing: EnableTracing,
	}
	if cs.tracing {
		cs.trInfo.tr = trace.New("grpc.Sent."+methodFamily(method), method)
		cs.trInfo.firstLine.client = true
		if deadline, ok := ctx.Deadline(); ok {
			cs.trInfo.firstLine.deadline = deadline.Sub(time.Now())
		}
		cs.trInfo.tr.LazyLog(&cs.trInfo.firstLine, false)
		ctx = trace.NewContext(ctx, cs.trInfo.tr)
	}
	s, err := t.NewStream(ctx, callHdr)
	if err != nil {
		cs.finish(err)
		return nil, toRPCErr(err)
	}
	cs.t = t
	cs.s = s
	cs.p = &parser{s: s}
	// Listen on ctx.Done() to detect cancellation when there is no pending
	// I/O operations on this stream.
	go func() {
		select {
		case <-t.Error():
			// Incur transport error, simply exit.
		case <-s.Context().Done():
			err := s.Context().Err()
			cs.finish(err)
			cs.closeTransportStream(transport.ContextErr(err))
		}
	}()
	return cs, nil
}

// clientStream implements a client side Stream.
type clientStream struct {
	t     transport.ClientTransport
	s     *transport.Stream
	p     *parser
	desc  *StreamDesc
	codec Codec

	tracing bool // set to EnableTracing when the clientStream is created.

	mu     sync.Mutex
	closed bool
	// trInfo.tr is set when the clientStream is created (if EnableTracing is true),
	// and is set to nil when the clientStream's finish method is called.
	trInfo traceInfo
}

func (cs *clientStream) Context() context.Context {
	return cs.s.Context()
}

func (cs *clientStream) Header() (metadata.MD, error) {
	m, err := cs.s.Header()
	if err != nil {
		if _, ok := err.(transport.ConnectionError); !ok {
			cs.closeTransportStream(err)
		}
	}
	return m, err
}

func (cs *clientStream) Trailer() metadata.MD {
	return cs.s.Trailer()
}

func (cs *clientStream) SendMsg(m interface{}) (err error) {
	if cs.tracing {
		cs.mu.Lock()
		if cs.trInfo.tr != nil {
			cs.trInfo.tr.LazyLog(&payload{sent: true, msg: m}, true)
		}
		cs.mu.Unlock()
	}
	defer func() {
		if err == nil || err == io.EOF {
			return
		}
		if _, ok := err.(transport.ConnectionError); !ok {
			cs.closeTransportStream(err)
		}
		err = toRPCErr(err)
	}()
	out, err := encode(cs.codec, m, compressionNone)
	if err != nil {
		return transport.StreamErrorf(codes.Internal, "grpc: %v", err)
	}
	return cs.t.Write(cs.s, out, &transport.Options{Last: false})
}

func (cs *clientStream) RecvMsg(m interface{}) (err error) {
	err = recv(cs.p, cs.codec, m)
	defer func() {
		// err != nil indicates the termination of the stream.
		if err != nil {
			cs.finish(err)
		}
	}()
	if err == nil {
		if cs.tracing {
			cs.mu.Lock()
			if cs.trInfo.tr != nil {
				cs.trInfo.tr.LazyLog(&payload{sent: false, msg: m}, true)
			}
			cs.mu.Unlock()
		}
		if !cs.desc.ClientStreams || cs.desc.ServerStreams {
			return
		}
		// Special handling for client streaming rpc.
		err = recv(cs.p, cs.codec, m)
		cs.closeTransportStream(err)
		if err == nil {
			return toRPCErr(errors.New("grpc: client streaming protocol violation: get <nil>, want <EOF>"))
		}
		if err == io.EOF {
			if cs.s.StatusCode() == codes.OK {
				return nil
			}
			return Errorf(cs.s.StatusCode(), cs.s.StatusDesc())
		}
		return toRPCErr(err)
	}
	if _, ok := err.(transport.ConnectionError); !ok {
		cs.closeTransportStream(err)
	}
	if err == io.EOF {
		if cs.s.StatusCode() == codes.OK {
			// Returns io.EOF to indicate the end of the stream.
			return
		}
		return Errorf(cs.s.StatusCode(), cs.s.StatusDesc())
	}
	return toRPCErr(err)
}

func (cs *clientStream) CloseSend() (err error) {
	err = cs.t.Write(cs.s, nil, &transport.Options{Last: true})
	if err == nil || err == io.EOF {
		return
	}
	if _, ok := err.(transport.ConnectionError); !ok {
		cs.closeTransportStream(err)
	}
	err = toRPCErr(err)
	return
}

func (cs *clientStream) closeTransportStream(err error) {
	cs.mu.Lock()
	if cs.closed {
		cs.mu.Unlock()
		return
	}
	cs.closed = true
	cs.mu.Unlock()
	cs.t.CloseStream(cs.s, err)
}

func (cs *clientStream) finish(err error) {
	if !cs.tracing {
		return
	}
	cs.mu.Lock()
	defer cs.mu.Unlock()
	if cs.trInfo.tr != nil {
		if err == nil || err == io.EOF {
			cs.trInfo.tr.LazyPrintf("RPC: [OK]")
		} else {
			cs.trInfo.tr.LazyPrintf("RPC: [%v]", err)
			cs.trInfo.tr.SetError()
		}
		cs.trInfo.tr.Finish()
		cs.trInfo.tr = nil
	}
}

// ServerStream defines the interface a server stream has to satisfy.
type ServerStream interface {
	// SendHeader sends the header metadata. It should not be called
	// after SendProto. It fails if called multiple times or if
	// called after SendProto.
	SendHeader(metadata.MD) error
	// SetTrailer sets the trailer metadata which will be sent with the
	// RPC status.
	SetTrailer(metadata.MD)
	Stream
}

// serverStream implements a server side Stream.
type serverStream struct {
	t          transport.ServerTransport
	s          *transport.Stream
	p          *parser
	codec      Codec
	statusCode codes.Code
	statusDesc string
	trInfo     *traceInfo

	mu sync.Mutex // protects trInfo.tr after the service handler runs.
}

func (ss *serverStream) Context() context.Context {
	return ss.s.Context()
}

func (ss *serverStream) SendHeader(md metadata.MD) error {
	return ss.t.WriteHeader(ss.s, md)
}

func (ss *serverStream) SetTrailer(md metadata.MD) {
	if md.Len() == 0 {
		return
	}
	ss.s.SetTrailer(md)
	return
}

func (ss *serverStream) SendMsg(m interface{}) (err error) {
	defer func() {
		if ss.trInfo != nil {
			ss.mu.Lock()
			if ss.trInfo.tr != nil {
				if err == nil {
					ss.trInfo.tr.LazyLog(&payload{sent: true, msg: m}, true)
				} else {
					ss.trInfo.tr.LazyLog(&fmtStringer{"%v", []interface{}{err}}, true)
					ss.trInfo.tr.SetError()
				}
			}
			ss.mu.Unlock()
		}
	}()
	out, err := encode(ss.codec, m, compressionNone)
	if err != nil {
		err = transport.StreamErrorf(codes.Internal, "grpc: %v", err)
		return err
	}
	return ss.t.Write(ss.s, out, &transport.Options{Last: false})
}

func (ss *serverStream) RecvMsg(m interface{}) (err error) {
	defer func() {
		if ss.trInfo != nil {
			ss.mu.Lock()
			if ss.trInfo.tr != nil {
				if err == nil {
					ss.trInfo.tr.LazyLog(&payload{sent: false, msg: m}, true)
				} else if err != io.EOF {
					ss.trInfo.tr.LazyLog(&fmtStringer{"%v", []interface{}{err}}, true)
					ss.trInfo.tr.SetError()
				}
			}
			ss.mu.Unlock()
		}
	}()
	return recv(ss.p, ss.codec, m)
}
<|endoftext|>


<commit_before>package p4

import (
	"fmt"
	"regexp"
	"strconv"
)

type Describe struct {
	Change      int
	User        string
	Client      string
	Time        string
	Description string
	Status      string
	Files       []DescribeFile
}

type DescribeFile struct {
	Path   string
	Action string
}

type Review struct {
	Change int
	User   string
	Email  string
	Name   string
}

var countersRegexp = regexp.MustCompile("(?m)^(.+) = (.+)$")
var describeRegexp = regexp.MustCompile("\\AChange (\\d+) by (.+)@(.+) on (.+)((?: *pending*)?)\n\n((?:\t.*\n)*)\nAffected files ...\n\n((?:... (?:.+) (?:[\\w/]+)\n)*)\n\\z")
var describeAffectedRegexp = regexp.MustCompile("(?m)^... (.+) ([\\w/]+)$")
var reviewRegexp = regexp.MustCompile("(?m)^Change (\\d+) (.+) <(.+)> \\((.+)\\)$")

func (c *Connection) Counters() (map[string]string, error) {
	counters := map[string]string{}

	if data, err := c.execP4("counters"); err == nil {
		submatch := countersRegexp.FindAllSubmatch(data, 1000000)

		for _, counter := range submatch {
			counters[string(counter[1])] = string(counter[2])
		}

		return counters, nil
	} else {
		return nil, err
	}
}

func (c *Connection) Describe(change int) (Describe, error) {
	var describe Describe

	if data, err := c.execP4("describe", "-s", strconv.Itoa(change)); err == nil {
		submatch := describeRegexp.FindSubmatch(data)
		intChange, err := strconv.Atoi(string(submatch[1]))

		if err != nil {
			return describe, err
		}

		status := "submitted"

		if string(submatch[5]) == " *pending" {
			status = "pending"
		}

		describe = Describe{
			Change:      intChange,
			User:        string(submatch[2]),
			Client:      string(submatch[3]),
			Time:        string(submatch[4]),
			Description: string(submatch[6]),
			Status:      status,
		}

		affectedSubmatch := describeAffectedRegexp.FindAllSubmatch(submatch[7], 10000000)

		for _, m := range affectedSubmatch {
			describe.Files = append(describe.Files, DescribeFile{
				Path:   string(m[1]),
				Action: string(m[2]),
			})
		}

		return describe, nil
	} else {
		return describe, err
	}
}

func (c *Connection) ReviewByCounter(counter string) ([]Review, error) {
	reviews := []Review{}

	if data, err := c.execP4("review", "-t", counter); err == nil {
		submatch := reviewRegexp.FindAllSubmatch(data, 10000000)

		for _, review := range submatch {
			intChange, err := strconv.Atoi(string(review[1]))

			if err != nil {
				return nil, err
			}

			reviews = append(reviews, Review{
				Change: intChange,
				User:   string(review[2]),
				Email:  string(review[3]),
				Name:   string(review[4]),
			})
		}

		return reviews, nil
	} else {
		return nil, err
	}
}

func (c *Connection) SetCounter(counter string, value string) error {
	if _, err := c.execP4("counter", counter, value); err == nil {
		return nil
	} else {
		return err
	}
}

func (c *Connection) Sync(path string, clNumber int) error {
	if _, err := c.execP4("sync", "-s", fmt.Sprintf("%s@%d", path, clNumber)); err == nil {
		return nil
	} else {
		return err
	}
}
<commit_msg>Parse file revision for affected files in describe<commit_after>package p4

import (
	"fmt"
	"regexp"
	"strconv"
)

type Describe struct {
	Change      int
	User        string
	Client      string
	Time        string
	Description string
	Status      string
	Files       []DescribeFile
}

type DescribeFile struct {
	Path    string
	Version int
	Action  string
}

type Review struct {
	Change int
	User   string
	Email  string
	Name   string
}

var countersRegexp = regexp.MustCompile("(?m)^(.+) = (.+)$")
var describeRegexp = regexp.MustCompile("\\AChange (\\d+) by (.+)@(.+) on (.+)((?: *pending*)?)\n\n((?:\t.*\n)*)\nAffected files ...\n\n((?:... (?:.+) (?:[\\w/]+)\n)*)\n\\z")
var describeAffectedRegexp = regexp.MustCompile("(?m)^... (.+)#(\\d+) ([\\w/]+)$")
var reviewRegexp = regexp.MustCompile("(?m)^Change (\\d+) (.+) <(.+)> \\((.+)\\)$")

func (c *Connection) Counters() (map[string]string, error) {
	counters := map[string]string{}

	if data, err := c.execP4("counters"); err == nil {
		submatch := countersRegexp.FindAllSubmatch(data, 1000000)

		for _, counter := range submatch {
			counters[string(counter[1])] = string(counter[2])
		}

		return counters, nil
	} else {
		return nil, err
	}
}

func (c *Connection) Describe(change int) (Describe, error) {
	var describe Describe

	if data, err := c.execP4("describe", "-s", strconv.Itoa(change)); err == nil {
		submatch := describeRegexp.FindSubmatch(data)
		intChange, err := strconv.Atoi(string(submatch[1]))

		if err != nil {
			return describe, err
		}

		status := "submitted"

		if string(submatch[5]) == " *pending" {
			status = "pending"
		}

		describe = Describe{
			Change:      intChange,
			User:        string(submatch[2]),
			Client:      string(submatch[3]),
			Time:        string(submatch[4]),
			Description: string(submatch[6]),
			Status:      status,
		}

		affectedSubmatch := describeAffectedRegexp.FindAllSubmatch(submatch[7], 10000000)

		for _, m := range affectedSubmatch {
			intVersion, err := strconv.Atoi(string(m[2]))

			if err != nil {
				return describe, err
			}

			describe.Files = append(describe.Files, DescribeFile{
				Path:    string(m[1]),
				Version: intVersion,
				Action:  string(m[3]),
			})
		}

		return describe, nil
	} else {
		return describe, err
	}
}

func (c *Connection) ReviewByCounter(counter string) ([]Review, error) {
	reviews := []Review{}

	if data, err := c.execP4("review", "-t", counter); err == nil {
		submatch := reviewRegexp.FindAllSubmatch(data, 10000000)

		for _, review := range submatch {
			intChange, err := strconv.Atoi(string(review[1]))

			if err != nil {
				return nil, err
			}

			reviews = append(reviews, Review{
				Change: intChange,
				User:   string(review[2]),
				Email:  string(review[3]),
				Name:   string(review[4]),
			})
		}

		return reviews, nil
	} else {
		return nil, err
	}
}

func (c *Connection) SetCounter(counter string, value string) error {
	if _, err := c.execP4("counter", counter, value); err == nil {
		return nil
	} else {
		return err
	}
}

func (c *Connection) Sync(path string, clNumber int) error {
	if _, err := c.execP4("sync", "-s", fmt.Sprintf("%s@%d", path, clNumber)); err == nil {
		return nil
	} else {
		return err
	}
}
<|endoftext|>


<commit_before>package proxy

import (
	"context"

	"v2ray.com/core/common/net"
)

type key int

const (
	sourceKey key = iota
	destinationKey
	originalDestinationKey
	inboundDestinationKey
	inboundTagKey
	outboundTagKey
	resolvedIPsKey
	allowPassiveConnKey
	dispatcherKey
)

func ContextWithSource(ctx context.Context, src net.Destination) context.Context {
	return context.WithValue(ctx, sourceKey, src)
}

func SourceFromContext(ctx context.Context) net.Destination {
	v := ctx.Value(sourceKey)
	if v == nil {
		return net.Destination{}
	}
	return v.(net.Destination)
}

func ContextWithOriginalDestination(ctx context.Context, dest net.Destination) context.Context {
	return context.WithValue(ctx, originalDestinationKey, dest)
}

func OriginalDestinationFromContext(ctx context.Context) net.Destination {
	v := ctx.Value(originalDestinationKey)
	if v == nil {
		return net.Destination{}
	}
	return v.(net.Destination)
}

func ContextWithDestination(ctx context.Context, dest net.Destination) context.Context {
	return context.WithValue(ctx, destinationKey, dest)
}

func DestinationFromContext(ctx context.Context) net.Destination {
	v := ctx.Value(destinationKey)
	if v == nil {
		return net.Destination{}
	}
	return v.(net.Destination)
}

func ContextWithInboundDestination(ctx context.Context, dest net.Destination) context.Context {
	return context.WithValue(ctx, inboundDestinationKey, dest)
}

func InboundDestinationFromContext(ctx context.Context) net.Destination {
	v := ctx.Value(inboundDestinationKey)
	if v == nil {
		return net.Destination{}
	}
	return v.(net.Destination)
}

func ContextWithInboundTag(ctx context.Context, tag string) context.Context {
	return context.WithValue(ctx, inboundTagKey, tag)
}

func InboundTagFromContext(ctx context.Context) string {
	v := ctx.Value(inboundTagKey)
	if v == nil {
		return ""
	}
	return v.(string)
}

func ContextWithOutboundTag(ctx context.Context, tag string) context.Context {
	return context.WithValue(ctx, outboundTagKey, tag)
}

func OutboundTagFromContext(ctx context.Context) string {
	v := ctx.Value(outboundTagKey)
	if v == nil {
		return ""
	}
	return v.(string)
}

func ContextWithResolveIPs(ctx context.Context, ips []net.Address) context.Context {
	return context.WithValue(ctx, resolvedIPsKey, ips)
}

func ResolvedIPsFromContext(ctx context.Context) ([]net.Address, bool) {
	ips, ok := ctx.Value(resolvedIPsKey).([]net.Address)
	return ips, ok
}
<commit_msg>remove unused functions<commit_after>package proxy

import (
	"context"

	"v2ray.com/core/common/net"
)

type key int

const (
	sourceKey key = iota
	destinationKey
	originalDestinationKey
	inboundDestinationKey
	inboundTagKey
	resolvedIPsKey
)

func ContextWithSource(ctx context.Context, src net.Destination) context.Context {
	return context.WithValue(ctx, sourceKey, src)
}

func SourceFromContext(ctx context.Context) net.Destination {
	v := ctx.Value(sourceKey)
	if v == nil {
		return net.Destination{}
	}
	return v.(net.Destination)
}

func ContextWithOriginalDestination(ctx context.Context, dest net.Destination) context.Context {
	return context.WithValue(ctx, originalDestinationKey, dest)
}

func OriginalDestinationFromContext(ctx context.Context) net.Destination {
	v := ctx.Value(originalDestinationKey)
	if v == nil {
		return net.Destination{}
	}
	return v.(net.Destination)
}

func ContextWithDestination(ctx context.Context, dest net.Destination) context.Context {
	return context.WithValue(ctx, destinationKey, dest)
}

func DestinationFromContext(ctx context.Context) net.Destination {
	v := ctx.Value(destinationKey)
	if v == nil {
		return net.Destination{}
	}
	return v.(net.Destination)
}

func ContextWithInboundDestination(ctx context.Context, dest net.Destination) context.Context {
	return context.WithValue(ctx, inboundDestinationKey, dest)
}

func InboundDestinationFromContext(ctx context.Context) net.Destination {
	v := ctx.Value(inboundDestinationKey)
	if v == nil {
		return net.Destination{}
	}
	return v.(net.Destination)
}

func ContextWithInboundTag(ctx context.Context, tag string) context.Context {
	return context.WithValue(ctx, inboundTagKey, tag)
}

func InboundTagFromContext(ctx context.Context) string {
	v := ctx.Value(inboundTagKey)
	if v == nil {
		return ""
	}
	return v.(string)
}

func ContextWithResolveIPs(ctx context.Context, ips []net.Address) context.Context {
	return context.WithValue(ctx, resolvedIPsKey, ips)
}

func ResolvedIPsFromContext(ctx context.Context) ([]net.Address, bool) {
	ips, ok := ctx.Value(resolvedIPsKey).([]net.Address)
	return ips, ok
}
<|endoftext|>


<commit_before>package dry

import (
	"bytes"
	"crypto/md5"
	"crypto/sha1"
	"encoding/base64"
	"encoding/csv"
	"encoding/json"
	"fmt"
	"reflect"
	"sort"
	"strconv"
	"strings"
	"time"
	"unicode"
)

func StringListContains(l []string, s string) bool {
	for i := range l {
		if l[i] == s {
			return true
		}
	}
	return false
}

func StringListContainsCaseInsensitive(l []string, s string) bool {
	s = strings.ToLower(s)
	for i := range l {
		if strings.ToLower(l[i]) == s {
			return true
		}
	}
	return false
}

func StringPrettifyJSON(compactJSON string) string {
	var buf bytes.Buffer
	err := json.Indent(&buf, []byte(compactJSON), "", "\t")
	if err != nil {
		return err.Error()
	}
	return buf.String()
}

func StringEscapeJSON(jsonString string) string {
	jsonString = strings.Replace(jsonString, `\`, `\\`, -1)
	jsonString = strings.Replace(jsonString, `"`, `\"`, -1)
	return jsonString
}

// StringStripHTMLTags strips HTML/XML tags from text.
func StringStripHTMLTags(text string) (plainText string) {
	var buf *bytes.Buffer
	tagClose := -1
	tagStart := -1
	for i, char := range text {
		if char == '<' {
			if buf == nil {
				buf = bytes.NewBufferString(text)
				buf.Reset()
			}
			buf.WriteString(text[tagClose+1 : i])
			tagStart = i
		} else if char == '>' && tagStart != -1 {
			tagClose = i
			tagStart = -1
		}
	}
	if buf == nil {
		return text
	}
	buf.WriteString(text[tagClose+1:])
	return buf.String()
}

// StringReplaceHTMLTags replaces HTML/XML tags from text with replacement.
func StringReplaceHTMLTags(text, replacement string) (plainText string) {
	var buf *bytes.Buffer
	tagClose := -1
	tagStart := -1
	for i, char := range text {
		if char == '<' {
			if buf == nil {
				buf = bytes.NewBufferString(text)
				buf.Reset()
			}
			buf.WriteString(text[tagClose+1 : i])
			tagStart = i
		} else if char == '>' && tagStart != -1 {
			buf.WriteString(replacement)
			tagClose = i
			tagStart = -1
		}
	}
	if buf == nil {
		return text
	}
	buf.WriteString(text[tagClose+1:])
	return buf.String()
}

// StringMD5Hex returns the hex encoded MD5 hash of data
func StringMD5Hex(data string) string {
	hash := md5.New()
	hash.Write([]byte(data))
	return fmt.Sprintf("%x", hash.Sum(nil))
}

// StringSHA1Base64 returns the base64 encoded SHA1 hash of data
func StringSHA1Base64(data string) string {
	hash := sha1.Sum([]byte(data))
	return base64.StdEncoding.EncodeToString(hash[:])
}

func StringAddURLParam(url, name, value string) string {
	var separator string
	if strings.IndexRune(url, '?') == -1 {
		separator = "?"
	} else {
		separator = "&"
	}
	return url + separator + name + "=" + value
}

func StringConvertTime(timeString, formatIn, formatOut string) (resultTime string, err error) {
	if timeString == "" {
		return "", nil
	}
	t, err := time.Parse(formatIn, timeString)
	if err != nil {
		return "", err
	}
	return t.Format(formatOut), nil
}

func StringCSV(records [][]string) string {
	var buf bytes.Buffer
	writer := csv.NewWriter(&buf)
	err := writer.WriteAll(records)
	if err != nil {
		return ""
	}
	return buf.String()
}

func StringToInt(s string) int {
	i, _ := strconv.ParseInt(s, 10, 64)
	return int(i)
}

func StringToFloat(s string) float64 {
	f, _ := strconv.ParseFloat(s, 64)
	return f
}

func StringToBool(s string) bool {
	b, _ := strconv.ParseBool(s)
	return b
}

func StringInSlice(s string, slice []string) bool {
	for i := range slice {
		if slice[i] == s {
			return true
		}
	}
	return false
}

// StringJoinFormat formats every value in values with format
// and joins the result with sep as separator.
// values must be a slice of a formatable type
func StringJoinFormat(format string, values interface{}, sep string) string {
	v := reflect.ValueOf(values)
	if v.Kind() != reflect.Slice {
		panic("values is not a slice")
	}
	var buffer bytes.Buffer
	for i := 0; i < v.Len(); i++ {
		if i > 0 {
			buffer.WriteString(sep)
		}
		buffer.WriteString(fmt.Sprintf(format, v.Index(i).Interface()))
	}
	return buffer.String()
}

// StringJoin formats every value in values according to its default formatting
// and joins the result with sep as separator.
// values must be a slice of a formatable type
func StringJoin(values interface{}, sep string) string {
	v := reflect.ValueOf(values)
	if v.Kind() != reflect.Slice {
		panic("values is not a slice")
	}
	var buffer bytes.Buffer
	for i := 0; i < v.Len(); i++ {
		if i > 0 {
			buffer.WriteString(sep)
		}
		buffer.WriteString(fmt.Sprint(v.Index(i).Interface()))
	}
	return buffer.String()
}

func StringFormatBigInt(mem uint64) string {
	switch {
	case mem >= 10e12:
		return fmt.Sprintf("%dT", mem/1e12)
	case mem >= 1e12:
		return strings.TrimSuffix(fmt.Sprintf("%.1fT", float64(mem)/1e12), ".0")

	case mem >= 10e9:
		return fmt.Sprintf("%dG", mem/1e9)
	case mem >= 1e9:
		return strings.TrimSuffix(fmt.Sprintf("%.1fG", float64(mem)/1e9), ".0")

	case mem >= 10e6:
		return fmt.Sprintf("%dM", mem/1e6)
	case mem >= 1e6:
		return strings.TrimSuffix(fmt.Sprintf("%.1fM", float64(mem)/1e6), ".0")

	case mem >= 10e3:
		return fmt.Sprintf("%dk", mem/1e3)
	case mem >= 1e3:
		return strings.TrimSuffix(fmt.Sprintf("%.1fk", float64(mem)/1e3), ".0")
	}
	return fmt.Sprintf("%d", mem)
}

func StringFormatMemory(mem uint64) string {
	return StringFormatBigInt(mem) + "B"
}

func StringReplaceMulti(str string, fromTo ...string) string {
	if len(fromTo)%2 != 0 {
		panic("Need even number of fromTo arguments")
	}
	for i := 0; i < len(fromTo); i += 2 {
		str = strings.Replace(str, fromTo[i], fromTo[i+1], -1)
	}
	return str
}

func StringToUpperCamelCase(str string) string {
	var buf bytes.Buffer
	var last byte = '_'
	for _, c := range []byte(str) {
		if c != '_' {
			if last == '_' {
				c = byte(unicode.ToUpper(rune(c)))
			} else {
				c = byte(unicode.ToLower(rune(c)))
			}
			buf.WriteByte(c)
		}
		last = c
	}
	return buf.String()
}

func StringToLowerCamelCase(str string) string {
	var buf bytes.Buffer
	var last byte
	for _, c := range []byte(str) {
		if c != '_' {
			if last == '_' {
				c = byte(unicode.ToUpper(rune(c)))
			} else {
				c = byte(unicode.ToLower(rune(c)))
			}
			buf.WriteByte(c)
		}
		last = c
	}
	return buf.String()
}

func StringMapSortedKeys(m map[string]string) []string {
	keys := make([]string, 0, len(m))
	for key := range m {
		keys = append(keys, key)
	}
	sort.Strings(keys)
	return keys
}

func StringMapGroupedNumberPostfixSortedKeys(m map[string]string) []string {
	keys := make(StringGroupedNumberPostfixSorter, 0, len(m))
	for key := range m {
		keys = append(keys, key)
	}
	sort.Sort(keys)
	return keys
}

func StringMapGroupedNumberPostfixSortedValues(m map[string]string) []string {
	values := make(StringGroupedNumberPostfixSorter, 0, len(m))
	for _, value := range m {
		values = append(values, value)
	}
	sort.Sort(values)
	return values
}

func StringEndsWithNumber(s string) bool {
	if s == "" {
		return false
	}
	c := s[len(s)-1]
	return c >= '0' && c <= '9'
}

func StringSplitNumberPostfix(s string) (base, number string) {
	if s == "" {
		return "", ""
	}
	for i := len(s) - 1; i >= 0; i-- {
		c := s[i]
		if c < '0' || c > '9' {
			if i == len(s)-1 {
				return s, ""
			}
			return s[:i+1], s[i+1:]
		}
	}
	return "", s
}

func StringSplitOnce(s, sep string) (pre, post string) {
	parts := strings.SplitN(s, sep, 1)
	if len(parts) == 2 {
		return parts[0], parts[1]
	} else {
		return parts[0], ""
	}
}

func StringSplitOnceChar(s string, sep byte) (pre, post string) {
	i := strings.IndexByte(s, sep)
	if i == -1 {
		return s, ""
	}
	return s[:i], s[i+1:]
}

func StringSplitOnceRune(s string, sep rune) (pre, post string) {
	sepIndex := -1
	postSepIndex := -1
	for i, c := range s {
		if sepIndex != -1 {
			postSepIndex = i
			break // we got the index after the sep rune
		}
		if c == sep {
			sepIndex = i
			// continue to get index after the current UTF8 rune
		}
	}
	if sepIndex == -1 {
		return s, ""
	}
	return s[:sepIndex], s[postSepIndex:]
}

type StringGroupedNumberPostfixSorter []string

// Len is the number of elements in the collection.
func (s StringGroupedNumberPostfixSorter) Len() int {
	return len(s)
}

// Less reports whether the element with
// index i should sort before the element with index j.
func (s StringGroupedNumberPostfixSorter) Less(i, j int) bool {
	bi, ni := StringSplitNumberPostfix(s[i])
	bj, nj := StringSplitNumberPostfix(s[j])

	if bi == bj {
		if len(ni) == len(nj) {
			inti, _ := strconv.Atoi(ni)
			intj, _ := strconv.Atoi(nj)
			return inti < intj
		} else {
			return len(ni) < len(nj)
		}
	}

	return bi < bj
}

// Swap swaps the elements with indexes i and j.
func (s StringGroupedNumberPostfixSorter) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}

// Map a function on each element of a slice of strings.
func StringMap(f func(string) string, data []string) []string {
	size := len(data)
	result := make([]string, size, size)
	for i := 0; i < size; i++ {
		result[i] = f(data[i])
	}
	return result
}

// Filter out all strings where the function does not return true.
func StringFilter(f func(string) bool, data []string) []string {
	result := make([]string, 0, 0)
	for _, element := range data {
		if f(element) {
			result = append(result, element)
		}
	}
	return result
}

// StringFindBetween returns the string between the first occurrences of the tokens start and stop.
// The remainder of the string after the stop token will be returned if found.
// If the tokens couldn't be found, then the whole string will be returned as remainder.
func StringFindBetween(s, start, stop string) (between, remainder string, found bool) {
	begin := strings.Index(s, start)
	if begin == -1 {
		return "", s, false
	}
	between = s[begin+len(start):]
	end := strings.Index(between, stop)
	if end == -1 {
		return "", s, false
	}
	return between[:end], s[begin+len(start)+end+len(stop):], true
}

// StringFind returns in found if token has been found in s,
// and returns the remaining string afte token in remainder.
// The whole string s will be returned if found is false.
func StringFind(s, token string) (remainder string, found bool) {
	i := strings.Index(s, token)
	if i == -1 {
		return s, false
	}
	return s[i+len(token):], true
}
<commit_msg>added StringMarshalJSON<commit_after>package dry

import (
	"bytes"
	"crypto/md5"
	"crypto/sha1"
	"encoding/base64"
	"encoding/csv"
	"encoding/json"
	"fmt"
	"reflect"
	"sort"
	"strconv"
	"strings"
	"time"
	"unicode"
)

// StringMarshalJSON marshals data to an indented string.
func StringMarshalJSON(data interface{}, indent string) string {
	buffer, err := json.MarshalIndent(data, "", indent)
	if err != nil {
		return ""
	}
	return string(buffer)
}

func StringListContains(l []string, s string) bool {
	for i := range l {
		if l[i] == s {
			return true
		}
	}
	return false
}

func StringListContainsCaseInsensitive(l []string, s string) bool {
	s = strings.ToLower(s)
	for i := range l {
		if strings.ToLower(l[i]) == s {
			return true
		}
	}
	return false
}

func StringPrettifyJSON(compactJSON string) string {
	var buf bytes.Buffer
	err := json.Indent(&buf, []byte(compactJSON), "", "\t")
	if err != nil {
		return err.Error()
	}
	return buf.String()
}

func StringEscapeJSON(jsonString string) string {
	jsonString = strings.Replace(jsonString, `\`, `\\`, -1)
	jsonString = strings.Replace(jsonString, `"`, `\"`, -1)
	return jsonString
}

// StringStripHTMLTags strips HTML/XML tags from text.
func StringStripHTMLTags(text string) (plainText string) {
	var buf *bytes.Buffer
	tagClose := -1
	tagStart := -1
	for i, char := range text {
		if char == '<' {
			if buf == nil {
				buf = bytes.NewBufferString(text)
				buf.Reset()
			}
			buf.WriteString(text[tagClose+1 : i])
			tagStart = i
		} else if char == '>' && tagStart != -1 {
			tagClose = i
			tagStart = -1
		}
	}
	if buf == nil {
		return text
	}
	buf.WriteString(text[tagClose+1:])
	return buf.String()
}

// StringReplaceHTMLTags replaces HTML/XML tags from text with replacement.
func StringReplaceHTMLTags(text, replacement string) (plainText string) {
	var buf *bytes.Buffer
	tagClose := -1
	tagStart := -1
	for i, char := range text {
		if char == '<' {
			if buf == nil {
				buf = bytes.NewBufferString(text)
				buf.Reset()
			}
			buf.WriteString(text[tagClose+1 : i])
			tagStart = i
		} else if char == '>' && tagStart != -1 {
			buf.WriteString(replacement)
			tagClose = i
			tagStart = -1
		}
	}
	if buf == nil {
		return text
	}
	buf.WriteString(text[tagClose+1:])
	return buf.String()
}

// StringMD5Hex returns the hex encoded MD5 hash of data
func StringMD5Hex(data string) string {
	hash := md5.New()
	hash.Write([]byte(data))
	return fmt.Sprintf("%x", hash.Sum(nil))
}

// StringSHA1Base64 returns the base64 encoded SHA1 hash of data
func StringSHA1Base64(data string) string {
	hash := sha1.Sum([]byte(data))
	return base64.StdEncoding.EncodeToString(hash[:])
}

func StringAddURLParam(url, name, value string) string {
	var separator string
	if strings.IndexRune(url, '?') == -1 {
		separator = "?"
	} else {
		separator = "&"
	}
	return url + separator + name + "=" + value
}

func StringConvertTime(timeString, formatIn, formatOut string) (resultTime string, err error) {
	if timeString == "" {
		return "", nil
	}
	t, err := time.Parse(formatIn, timeString)
	if err != nil {
		return "", err
	}
	return t.Format(formatOut), nil
}

func StringCSV(records [][]string) string {
	var buf bytes.Buffer
	writer := csv.NewWriter(&buf)
	err := writer.WriteAll(records)
	if err != nil {
		return ""
	}
	return buf.String()
}

func StringToInt(s string) int {
	i, _ := strconv.ParseInt(s, 10, 64)
	return int(i)
}

func StringToFloat(s string) float64 {
	f, _ := strconv.ParseFloat(s, 64)
	return f
}

func StringToBool(s string) bool {
	b, _ := strconv.ParseBool(s)
	return b
}

func StringInSlice(s string, slice []string) bool {
	for i := range slice {
		if slice[i] == s {
			return true
		}
	}
	return false
}

// StringJoinFormat formats every value in values with format
// and joins the result with sep as separator.
// values must be a slice of a formatable type
func StringJoinFormat(format string, values interface{}, sep string) string {
	v := reflect.ValueOf(values)
	if v.Kind() != reflect.Slice {
		panic("values is not a slice")
	}
	var buffer bytes.Buffer
	for i := 0; i < v.Len(); i++ {
		if i > 0 {
			buffer.WriteString(sep)
		}
		buffer.WriteString(fmt.Sprintf(format, v.Index(i).Interface()))
	}
	return buffer.String()
}

// StringJoin formats every value in values according to its default formatting
// and joins the result with sep as separator.
// values must be a slice of a formatable type
func StringJoin(values interface{}, sep string) string {
	v := reflect.ValueOf(values)
	if v.Kind() != reflect.Slice {
		panic("values is not a slice")
	}
	var buffer bytes.Buffer
	for i := 0; i < v.Len(); i++ {
		if i > 0 {
			buffer.WriteString(sep)
		}
		buffer.WriteString(fmt.Sprint(v.Index(i).Interface()))
	}
	return buffer.String()
}

func StringFormatBigInt(mem uint64) string {
	switch {
	case mem >= 10e12:
		return fmt.Sprintf("%dT", mem/1e12)
	case mem >= 1e12:
		return strings.TrimSuffix(fmt.Sprintf("%.1fT", float64(mem)/1e12), ".0")

	case mem >= 10e9:
		return fmt.Sprintf("%dG", mem/1e9)
	case mem >= 1e9:
		return strings.TrimSuffix(fmt.Sprintf("%.1fG", float64(mem)/1e9), ".0")

	case mem >= 10e6:
		return fmt.Sprintf("%dM", mem/1e6)
	case mem >= 1e6:
		return strings.TrimSuffix(fmt.Sprintf("%.1fM", float64(mem)/1e6), ".0")

	case mem >= 10e3:
		return fmt.Sprintf("%dk", mem/1e3)
	case mem >= 1e3:
		return strings.TrimSuffix(fmt.Sprintf("%.1fk", float64(mem)/1e3), ".0")
	}
	return fmt.Sprintf("%d", mem)
}

func StringFormatMemory(mem uint64) string {
	return StringFormatBigInt(mem) + "B"
}

func StringReplaceMulti(str string, fromTo ...string) string {
	if len(fromTo)%2 != 0 {
		panic("Need even number of fromTo arguments")
	}
	for i := 0; i < len(fromTo); i += 2 {
		str = strings.Replace(str, fromTo[i], fromTo[i+1], -1)
	}
	return str
}

func StringToUpperCamelCase(str string) string {
	var buf bytes.Buffer
	var last byte = '_'
	for _, c := range []byte(str) {
		if c != '_' {
			if last == '_' {
				c = byte(unicode.ToUpper(rune(c)))
			} else {
				c = byte(unicode.ToLower(rune(c)))
			}
			buf.WriteByte(c)
		}
		last = c
	}
	return buf.String()
}

func StringToLowerCamelCase(str string) string {
	var buf bytes.Buffer
	var last byte
	for _, c := range []byte(str) {
		if c != '_' {
			if last == '_' {
				c = byte(unicode.ToUpper(rune(c)))
			} else {
				c = byte(unicode.ToLower(rune(c)))
			}
			buf.WriteByte(c)
		}
		last = c
	}
	return buf.String()
}

func StringMapSortedKeys(m map[string]string) []string {
	keys := make([]string, 0, len(m))
	for key := range m {
		keys = append(keys, key)
	}
	sort.Strings(keys)
	return keys
}

func StringMapGroupedNumberPostfixSortedKeys(m map[string]string) []string {
	keys := make(StringGroupedNumberPostfixSorter, 0, len(m))
	for key := range m {
		keys = append(keys, key)
	}
	sort.Sort(keys)
	return keys
}

func StringMapGroupedNumberPostfixSortedValues(m map[string]string) []string {
	values := make(StringGroupedNumberPostfixSorter, 0, len(m))
	for _, value := range m {
		values = append(values, value)
	}
	sort.Sort(values)
	return values
}

func StringEndsWithNumber(s string) bool {
	if s == "" {
		return false
	}
	c := s[len(s)-1]
	return c >= '0' && c <= '9'
}

func StringSplitNumberPostfix(s string) (base, number string) {
	if s == "" {
		return "", ""
	}
	for i := len(s) - 1; i >= 0; i-- {
		c := s[i]
		if c < '0' || c > '9' {
			if i == len(s)-1 {
				return s, ""
			}
			return s[:i+1], s[i+1:]
		}
	}
	return "", s
}

func StringSplitOnce(s, sep string) (pre, post string) {
	parts := strings.SplitN(s, sep, 1)
	if len(parts) == 2 {
		return parts[0], parts[1]
	} else {
		return parts[0], ""
	}
}

func StringSplitOnceChar(s string, sep byte) (pre, post string) {
	i := strings.IndexByte(s, sep)
	if i == -1 {
		return s, ""
	}
	return s[:i], s[i+1:]
}

func StringSplitOnceRune(s string, sep rune) (pre, post string) {
	sepIndex := -1
	postSepIndex := -1
	for i, c := range s {
		if sepIndex != -1 {
			postSepIndex = i
			break // we got the index after the sep rune
		}
		if c == sep {
			sepIndex = i
			// continue to get index after the current UTF8 rune
		}
	}
	if sepIndex == -1 {
		return s, ""
	}
	return s[:sepIndex], s[postSepIndex:]
}

type StringGroupedNumberPostfixSorter []string

// Len is the number of elements in the collection.
func (s StringGroupedNumberPostfixSorter) Len() int {
	return len(s)
}

// Less reports whether the element with
// index i should sort before the element with index j.
func (s StringGroupedNumberPostfixSorter) Less(i, j int) bool {
	bi, ni := StringSplitNumberPostfix(s[i])
	bj, nj := StringSplitNumberPostfix(s[j])

	if bi == bj {
		if len(ni) == len(nj) {
			inti, _ := strconv.Atoi(ni)
			intj, _ := strconv.Atoi(nj)
			return inti < intj
		} else {
			return len(ni) < len(nj)
		}
	}

	return bi < bj
}

// Swap swaps the elements with indexes i and j.
func (s StringGroupedNumberPostfixSorter) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}

// Map a function on each element of a slice of strings.
func StringMap(f func(string) string, data []string) []string {
	size := len(data)
	result := make([]string, size, size)
	for i := 0; i < size; i++ {
		result[i] = f(data[i])
	}
	return result
}

// Filter out all strings where the function does not return true.
func StringFilter(f func(string) bool, data []string) []string {
	result := make([]string, 0, 0)
	for _, element := range data {
		if f(element) {
			result = append(result, element)
		}
	}
	return result
}

// StringFindBetween returns the string between the first occurrences of the tokens start and stop.
// The remainder of the string after the stop token will be returned if found.
// If the tokens couldn't be found, then the whole string will be returned as remainder.
func StringFindBetween(s, start, stop string) (between, remainder string, found bool) {
	begin := strings.Index(s, start)
	if begin == -1 {
		return "", s, false
	}
	between = s[begin+len(start):]
	end := strings.Index(between, stop)
	if end == -1 {
		return "", s, false
	}
	return between[:end], s[begin+len(start)+end+len(stop):], true
}

// StringFind returns in found if token has been found in s,
// and returns the remaining string afte token in remainder.
// The whole string s will be returned if found is false.
func StringFind(s, token string) (remainder string, found bool) {
	i := strings.Index(s, token)
	if i == -1 {
		return s, false
	}
	return s[i+len(token):], true
}
<|endoftext|>