<commit_before>package pgx

import (
	"context"
	"fmt"
	"reflect"
	"time"

	"github.com/pkg/errors"

	"github.com/jackc/pgconn"
	"github.com/jackc/pgx/pgtype"
)

// Rows is the result set returned from *Conn.Query. Rows must be closed before
// the *Conn can be used again. Rows are closed by explicitly calling Close(),
// calling Next() until it returns false, or when a fatal error occurs.
type Rows interface {
	// Close closes the rows, making the connection ready for use again. It is safe
	// to call Close after rows is already closed.
	Close()

	Err() error
	FieldDescriptions() []FieldDescription

	// Next prepares the next row for reading. It returns true if there is another
	// row and false if no more rows are available. It automatically closes rows
	// when all rows are read.
	Next() bool

	// Scan reads the values from the current row into dest values positionally.
	// dest can include pointers to core types, values implementing the Scanner
	// interface, []byte, and nil. []byte will skip the decoding process and directly
	// copy the raw bytes received from PostgreSQL. nil will skip the value entirely.
	Scan(dest ...interface{}) error

	// Values returns an array of the row values
	Values() ([]interface{}, error)
}

// Row is a convenience wrapper over Rows that is returned by QueryRow.
type Row interface {
	// Scan works the same as Rows. with the following exceptions. If no
	// rows were found it returns ErrNoRows. If multiple rows are returned it
	// ignores all but the first.
	Scan(dest ...interface{}) error
}

// connRow implements the Row interface for Conn.QueryRow.
type connRow connRows

func (r *connRow) Scan(dest ...interface{}) (err error) {
	rows := (*connRows)(r)

	if rows.Err() != nil {
		return rows.Err()
	}

	if !rows.Next() {
		if rows.Err() == nil {
			return ErrNoRows
		}
		return rows.Err()
	}

	rows.Scan(dest...)
	rows.Close()
	return rows.Err()
}

// connRows implements the Rows interface for Conn.Query.
type connRows struct {
	conn      *Conn
	batch     *Batch
	values    [][]byte
	fields    []FieldDescription
	rowCount  int
	columnIdx int
	err       error
	startTime time.Time
	sql       string
	args      []interface{}
	closed    bool

	resultReader      *pgconn.ResultReader
	multiResultReader *pgconn.MultiResultReader
}

func (rows *connRows) FieldDescriptions() []FieldDescription {
	return rows.fields
}

func (rows *connRows) Close() {
	if rows.closed {
		return
	}

	rows.closed = true

	if rows.resultReader != nil {
		_, closeErr := rows.resultReader.Close()
		if rows.err == nil {
			rows.err = closeErr
		}
	}

	if rows.multiResultReader != nil {
		closeErr := rows.multiResultReader.Close()
		if rows.err == nil {
			rows.err = closeErr
		}
	}

	if rows.err == nil {
		if rows.conn.shouldLog(LogLevelInfo) {
			endTime := time.Now()
			rows.conn.log(LogLevelInfo, "Query", map[string]interface{}{"sql": rows.sql, "args": logQueryArgs(rows.args), "time": endTime.Sub(rows.startTime), "rowCount": rows.rowCount})
		}
	} else if rows.conn.shouldLog(LogLevelError) {
		rows.conn.log(LogLevelError, "Query", map[string]interface{}{"sql": rows.sql, "args": logQueryArgs(rows.args)})
	}

	if rows.batch != nil && rows.err != nil {
		rows.batch.die(rows.err)
	}
}

func (rows *connRows) Err() error {
	return rows.err
}

// fatal signals an error occurred after the query was sent to the server. It
// closes the rows automatically.
func (rows *connRows) fatal(err error) {
	if rows.err != nil {
		return
	}

	rows.err = err
	rows.Close()
}

func (rows *connRows) Next() bool {
	if rows.closed {
		return false
	}

	if rows.resultReader.NextRow() {
		if rows.fields == nil {
			rrFieldDescriptions := rows.resultReader.FieldDescriptions()
			rows.fields = make([]FieldDescription, len(rrFieldDescriptions))
			for i := range rrFieldDescriptions {
				rows.conn.pgproto3FieldDescriptionToPgxFieldDescription(&rrFieldDescriptions[i], &rows.fields[i])
			}
		}
		rows.rowCount++
		rows.columnIdx = 0
		rows.values = rows.resultReader.Values()
		return true
	} else {
		rows.Close()
		return false
	}
}

func (rows *connRows) nextColumn() ([]byte, *FieldDescription, bool) {
	if rows.closed {
		return nil, nil, false
	}
	if len(rows.fields) <= rows.columnIdx {
		rows.fatal(ProtocolError("No next column available"))
		return nil, nil, false
	}

	buf := rows.values[rows.columnIdx]
	fd := &rows.fields[rows.columnIdx]
	rows.columnIdx++
	return buf, fd, true
}

func (rows *connRows) Scan(dest ...interface{}) error {
	if len(rows.fields) != len(dest) {
		err := errors.Errorf("Scan received wrong number of arguments, got %d but expected %d", len(dest), len(rows.fields))
		rows.fatal(err)
		return err
	}

	for i, d := range dest {
		buf, fd, _ := rows.nextColumn()

		if d == nil {
			continue
		}

		err := rows.conn.ConnInfo.Scan(fd.DataType, fd.FormatCode, buf, d)
		if err != nil {
			rows.fatal(scanArgError{col: i, err: err})
			return err
		}
	}

	return nil
}

func (rows *connRows) Values() ([]interface{}, error) {
	if rows.closed {
		return nil, errors.New("rows is closed")
	}

	values := make([]interface{}, 0, len(rows.fields))

	for range rows.fields {
		buf, fd, _ := rows.nextColumn()

		if buf == nil {
			values = append(values, nil)
			continue
		}

		if dt, ok := rows.conn.ConnInfo.DataTypeForOID(fd.DataType); ok {
			value := reflect.New(reflect.ValueOf(dt.Value).Elem().Type()).Interface().(pgtype.Value)

			switch fd.FormatCode {
			case TextFormatCode:
				decoder := value.(pgtype.TextDecoder)
				if decoder == nil {
					decoder = &pgtype.GenericText{}
				}
				err := decoder.DecodeText(rows.conn.ConnInfo, buf)
				if err != nil {
					rows.fatal(err)
				}
				values = append(values, decoder.(pgtype.Value).Get())
			case BinaryFormatCode:
				decoder := value.(pgtype.BinaryDecoder)
				if decoder == nil {
					decoder = &pgtype.GenericBinary{}
				}
				err := decoder.DecodeBinary(rows.conn.ConnInfo, buf)
				if err != nil {
					rows.fatal(err)
				}
				values = append(values, value.Get())
			default:
				rows.fatal(errors.New("Unknown format code"))
			}
		} else {
			rows.fatal(errors.New("Unknown type"))
		}

		if rows.Err() != nil {
			return nil, rows.Err()
		}
	}

	return values, rows.Err()
}

type scanArgError struct {
	col int
	err error
}

func (e scanArgError) Error() string {
	return fmt.Sprintf("can't scan into dest[%d]: %v", e.col, e.err)
}

func (c *Conn) getRows(sql string, args []interface{}) *connRows {
	if len(c.preallocatedRows) == 0 {
		c.preallocatedRows = make([]connRows, 64)
	}

	r := &c.preallocatedRows[len(c.preallocatedRows)-1]
	c.preallocatedRows = c.preallocatedRows[0 : len(c.preallocatedRows)-1]

	r.conn = c
	r.startTime = time.Now()
	r.sql = sql
	r.args = args

	return r
}

type QueryResultFormats []int16

// Query executes sql with args. If there is an error the returned Rows will be returned in an error state. So it is
// allowed to ignore the error returned from Query and handle it in Rows.
func (c *Conn) Query(ctx context.Context, sql string, args ...interface{}) (Rows, error) {
	c.lastStmtSent = false
	// rows = c.getRows(sql, args)

	var resultFormats QueryResultFormats

optionLoop:
	for len(args) > 0 {
		switch arg := args[0].(type) {
		case QueryResultFormats:
			resultFormats = arg
			args = args[1:]
		default:
			break optionLoop
		}
	}

	rows := &connRows{
		conn:      c,
		startTime: time.Now(),
		sql:       sql,
		args:      args,
	}

	// err = c.initContext(ctx)
	// if err != nil {
	// 	rows.fatal(err)
	// 	return rows, rows.err
	// }

	// if options != nil && len(options.ParameterOIDs) > 0 {

	// 	buf, err := c.buildOneRoundTripQueryEx(c.wbuf, sql, options, args)
	// 	if err != nil {
	// 		rows.fatal(err)
	// 		return rows, err
	// 	}

	// 	buf = appendSync(buf)

	// 	n, err := c.pgConn.Conn().Write(buf)
	// 	c.lastStmtSent = true
	// 	if err != nil && fatalWriteErr(n, err) {
	// 		rows.fatal(err)
	// 		c.die(err)
	// 		return rows, err
	// 	}
	// 	c.pendingReadyForQueryCount++

	// 	fieldDescriptions, err := c.readUntilRowDescription()
	// 	if err != nil {
	// 		rows.fatal(err)
	// 		return rows, err
	// 	}

	// 	if len(options.ResultFormatCodes) == 0 {
	// 		for i := range fieldDescriptions {
	// 			fieldDescriptions[i].FormatCode = TextFormatCode
	// 		}
	// 	} else if len(options.ResultFormatCodes) == 1 {
	// 		fc := options.ResultFormatCodes[0]
	// 		for i := range fieldDescriptions {
	// 			fieldDescriptions[i].FormatCode = fc
	// 		}
	// 	} else {
	// 		for i := range options.ResultFormatCodes {
	// 			fieldDescriptions[i].FormatCode = options.ResultFormatCodes[i]
	// 		}
	// 	}

	// 	rows.sql = sql
	// 	rows.fields = fieldDescriptions
	// 	return rows, nil
	// }

	ps, ok := c.preparedStatements[sql]
	if !ok {
		psd, err := c.pgConn.Prepare(ctx, "", sql, nil)
		if err != nil {
			rows.fatal(err)
			return rows, rows.err
		}

		if len(psd.ParamOIDs) != len(args) {
			rows.fatal(errors.Errorf("expected %d arguments, got %d", len(psd.ParamOIDs), len(args)))
			return rows, rows.err
		}

		ps = &PreparedStatement{
			Name:              psd.Name,
			SQL:               psd.SQL,
			ParameterOIDs:     make([]pgtype.OID, len(psd.ParamOIDs)),
			FieldDescriptions: make([]FieldDescription, len(psd.Fields)),
		}

		for i := range ps.ParameterOIDs {
			ps.ParameterOIDs[i] = pgtype.OID(psd.ParamOIDs[i])
		}
		for i := range ps.FieldDescriptions {
			c.pgproto3FieldDescriptionToPgxFieldDescription(&psd.Fields[i], &ps.FieldDescriptions[i])
		}
	}
	rows.sql = ps.SQL

	var err error
	args, err = convertDriverValuers(args)
	if err != nil {
		rows.fatal(err)
		return rows, rows.err
	}

	paramFormats := make([]int16, len(args))
	paramValues := make([][]byte, len(args))
	for i := range args {
		paramFormats[i] = chooseParameterFormatCode(c.ConnInfo, ps.ParameterOIDs[i], args[i])
		paramValues[i], err = newencodePreparedStatementArgument(c.ConnInfo, ps.ParameterOIDs[i], args[i])
		if err != nil {
			rows.fatal(err)
			return rows, rows.err
		}
	}

	if resultFormats == nil {
		resultFormats = make([]int16, len(ps.FieldDescriptions))
		for i := range resultFormats {
			if dt, ok := c.ConnInfo.DataTypeForOID(ps.FieldDescriptions[i].DataType); ok {
				if _, ok := dt.Value.(pgtype.BinaryDecoder); ok {
					resultFormats[i] = BinaryFormatCode
				} else {
					resultFormats[i] = TextFormatCode
				}
			}
		}
	}

	c.lastStmtSent = true
	rows.resultReader = c.pgConn.ExecPrepared(ctx, ps.Name, paramValues, paramFormats, resultFormats)

	return rows, rows.err
}

// QueryRow is a convenience wrapper over Query. Any error that occurs while
// querying is deferred until calling Scan on the returned Row. That Row will
// error with ErrNoRows if no rows are returned.
func (c *Conn) QueryRow(ctx context.Context, sql string, args ...interface{}) Row {
	rows, _ := c.Query(ctx, sql, args...)
	return (*connRow)(rows.(*connRows))
}
<commit_msg>Remove commented out unused one round trip mode<commit_after>package pgx

import (
	"context"
	"fmt"
	"reflect"
	"time"

	"github.com/pkg/errors"

	"github.com/jackc/pgconn"
	"github.com/jackc/pgx/pgtype"
)

// Rows is the result set returned from *Conn.Query. Rows must be closed before
// the *Conn can be used again. Rows are closed by explicitly calling Close(),
// calling Next() until it returns false, or when a fatal error occurs.
type Rows interface {
	// Close closes the rows, making the connection ready for use again. It is safe
	// to call Close after rows is already closed.
	Close()

	Err() error
	FieldDescriptions() []FieldDescription

	// Next prepares the next row for reading. It returns true if there is another
	// row and false if no more rows are available. It automatically closes rows
	// when all rows are read.
	Next() bool

	// Scan reads the values from the current row into dest values positionally.
	// dest can include pointers to core types, values implementing the Scanner
	// interface, []byte, and nil. []byte will skip the decoding process and directly
	// copy the raw bytes received from PostgreSQL. nil will skip the value entirely.
	Scan(dest ...interface{}) error

	// Values returns an array of the row values
	Values() ([]interface{}, error)
}

// Row is a convenience wrapper over Rows that is returned by QueryRow.
type Row interface {
	// Scan works the same as Rows. with the following exceptions. If no
	// rows were found it returns ErrNoRows. If multiple rows are returned it
	// ignores all but the first.
	Scan(dest ...interface{}) error
}

// connRow implements the Row interface for Conn.QueryRow.
type connRow connRows

func (r *connRow) Scan(dest ...interface{}) (err error) {
	rows := (*connRows)(r)

	if rows.Err() != nil {
		return rows.Err()
	}

	if !rows.Next() {
		if rows.Err() == nil {
			return ErrNoRows
		}
		return rows.Err()
	}

	rows.Scan(dest...)
	rows.Close()
	return rows.Err()
}

// connRows implements the Rows interface for Conn.Query.
type connRows struct {
	conn      *Conn
	batch     *Batch
	values    [][]byte
	fields    []FieldDescription
	rowCount  int
	columnIdx int
	err       error
	startTime time.Time
	sql       string
	args      []interface{}
	closed    bool

	resultReader      *pgconn.ResultReader
	multiResultReader *pgconn.MultiResultReader
}

func (rows *connRows) FieldDescriptions() []FieldDescription {
	return rows.fields
}

func (rows *connRows) Close() {
	if rows.closed {
		return
	}

	rows.closed = true

	if rows.resultReader != nil {
		_, closeErr := rows.resultReader.Close()
		if rows.err == nil {
			rows.err = closeErr
		}
	}

	if rows.multiResultReader != nil {
		closeErr := rows.multiResultReader.Close()
		if rows.err == nil {
			rows.err = closeErr
		}
	}

	if rows.err == nil {
		if rows.conn.shouldLog(LogLevelInfo) {
			endTime := time.Now()
			rows.conn.log(LogLevelInfo, "Query", map[string]interface{}{"sql": rows.sql, "args": logQueryArgs(rows.args), "time": endTime.Sub(rows.startTime), "rowCount": rows.rowCount})
		}
	} else if rows.conn.shouldLog(LogLevelError) {
		rows.conn.log(LogLevelError, "Query", map[string]interface{}{"sql": rows.sql, "args": logQueryArgs(rows.args)})
	}

	if rows.batch != nil && rows.err != nil {
		rows.batch.die(rows.err)
	}
}

func (rows *connRows) Err() error {
	return rows.err
}

// fatal signals an error occurred after the query was sent to the server. It
// closes the rows automatically.
func (rows *connRows) fatal(err error) {
	if rows.err != nil {
		return
	}

	rows.err = err
	rows.Close()
}

func (rows *connRows) Next() bool {
	if rows.closed {
		return false
	}

	if rows.resultReader.NextRow() {
		if rows.fields == nil {
			rrFieldDescriptions := rows.resultReader.FieldDescriptions()
			rows.fields = make([]FieldDescription, len(rrFieldDescriptions))
			for i := range rrFieldDescriptions {
				rows.conn.pgproto3FieldDescriptionToPgxFieldDescription(&rrFieldDescriptions[i], &rows.fields[i])
			}
		}
		rows.rowCount++
		rows.columnIdx = 0
		rows.values = rows.resultReader.Values()
		return true
	} else {
		rows.Close()
		return false
	}
}

func (rows *connRows) nextColumn() ([]byte, *FieldDescription, bool) {
	if rows.closed {
		return nil, nil, false
	}
	if len(rows.fields) <= rows.columnIdx {
		rows.fatal(ProtocolError("No next column available"))
		return nil, nil, false
	}

	buf := rows.values[rows.columnIdx]
	fd := &rows.fields[rows.columnIdx]
	rows.columnIdx++
	return buf, fd, true
}

func (rows *connRows) Scan(dest ...interface{}) error {
	if len(rows.fields) != len(dest) {
		err := errors.Errorf("Scan received wrong number of arguments, got %d but expected %d", len(dest), len(rows.fields))
		rows.fatal(err)
		return err
	}

	for i, d := range dest {
		buf, fd, _ := rows.nextColumn()

		if d == nil {
			continue
		}

		err := rows.conn.ConnInfo.Scan(fd.DataType, fd.FormatCode, buf, d)
		if err != nil {
			rows.fatal(scanArgError{col: i, err: err})
			return err
		}
	}

	return nil
}

func (rows *connRows) Values() ([]interface{}, error) {
	if rows.closed {
		return nil, errors.New("rows is closed")
	}

	values := make([]interface{}, 0, len(rows.fields))

	for range rows.fields {
		buf, fd, _ := rows.nextColumn()

		if buf == nil {
			values = append(values, nil)
			continue
		}

		if dt, ok := rows.conn.ConnInfo.DataTypeForOID(fd.DataType); ok {
			value := reflect.New(reflect.ValueOf(dt.Value).Elem().Type()).Interface().(pgtype.Value)

			switch fd.FormatCode {
			case TextFormatCode:
				decoder := value.(pgtype.TextDecoder)
				if decoder == nil {
					decoder = &pgtype.GenericText{}
				}
				err := decoder.DecodeText(rows.conn.ConnInfo, buf)
				if err != nil {
					rows.fatal(err)
				}
				values = append(values, decoder.(pgtype.Value).Get())
			case BinaryFormatCode:
				decoder := value.(pgtype.BinaryDecoder)
				if decoder == nil {
					decoder = &pgtype.GenericBinary{}
				}
				err := decoder.DecodeBinary(rows.conn.ConnInfo, buf)
				if err != nil {
					rows.fatal(err)
				}
				values = append(values, value.Get())
			default:
				rows.fatal(errors.New("Unknown format code"))
			}
		} else {
			rows.fatal(errors.New("Unknown type"))
		}

		if rows.Err() != nil {
			return nil, rows.Err()
		}
	}

	return values, rows.Err()
}

type scanArgError struct {
	col int
	err error
}

func (e scanArgError) Error() string {
	return fmt.Sprintf("can't scan into dest[%d]: %v", e.col, e.err)
}

func (c *Conn) getRows(sql string, args []interface{}) *connRows {
	if len(c.preallocatedRows) == 0 {
		c.preallocatedRows = make([]connRows, 64)
	}

	r := &c.preallocatedRows[len(c.preallocatedRows)-1]
	c.preallocatedRows = c.preallocatedRows[0 : len(c.preallocatedRows)-1]

	r.conn = c
	r.startTime = time.Now()
	r.sql = sql
	r.args = args

	return r
}

type QueryResultFormats []int16

// Query executes sql with args. If there is an error the returned Rows will be returned in an error state. So it is
// allowed to ignore the error returned from Query and handle it in Rows.
func (c *Conn) Query(ctx context.Context, sql string, args ...interface{}) (Rows, error) {
	c.lastStmtSent = false
	// rows = c.getRows(sql, args)

	var resultFormats QueryResultFormats

optionLoop:
	for len(args) > 0 {
		switch arg := args[0].(type) {
		case QueryResultFormats:
			resultFormats = arg
			args = args[1:]
		default:
			break optionLoop
		}
	}

	rows := &connRows{
		conn:      c,
		startTime: time.Now(),
		sql:       sql,
		args:      args,
	}

	ps, ok := c.preparedStatements[sql]
	if !ok {
		psd, err := c.pgConn.Prepare(ctx, "", sql, nil)
		if err != nil {
			rows.fatal(err)
			return rows, rows.err
		}

		if len(psd.ParamOIDs) != len(args) {
			rows.fatal(errors.Errorf("expected %d arguments, got %d", len(psd.ParamOIDs), len(args)))
			return rows, rows.err
		}

		ps = &PreparedStatement{
			Name:              psd.Name,
			SQL:               psd.SQL,
			ParameterOIDs:     make([]pgtype.OID, len(psd.ParamOIDs)),
			FieldDescriptions: make([]FieldDescription, len(psd.Fields)),
		}

		for i := range ps.ParameterOIDs {
			ps.ParameterOIDs[i] = pgtype.OID(psd.ParamOIDs[i])
		}
		for i := range ps.FieldDescriptions {
			c.pgproto3FieldDescriptionToPgxFieldDescription(&psd.Fields[i], &ps.FieldDescriptions[i])
		}
	}
	rows.sql = ps.SQL

	var err error
	args, err = convertDriverValuers(args)
	if err != nil {
		rows.fatal(err)
		return rows, rows.err
	}

	paramFormats := make([]int16, len(args))
	paramValues := make([][]byte, len(args))
	for i := range args {
		paramFormats[i] = chooseParameterFormatCode(c.ConnInfo, ps.ParameterOIDs[i], args[i])
		paramValues[i], err = newencodePreparedStatementArgument(c.ConnInfo, ps.ParameterOIDs[i], args[i])
		if err != nil {
			rows.fatal(err)
			return rows, rows.err
		}
	}

	if resultFormats == nil {
		resultFormats = make([]int16, len(ps.FieldDescriptions))
		for i := range resultFormats {
			if dt, ok := c.ConnInfo.DataTypeForOID(ps.FieldDescriptions[i].DataType); ok {
				if _, ok := dt.Value.(pgtype.BinaryDecoder); ok {
					resultFormats[i] = BinaryFormatCode
				} else {
					resultFormats[i] = TextFormatCode
				}
			}
		}
	}

	c.lastStmtSent = true
	rows.resultReader = c.pgConn.ExecPrepared(ctx, ps.Name, paramValues, paramFormats, resultFormats)

	return rows, rows.err
}

// QueryRow is a convenience wrapper over Query. Any error that occurs while
// querying is deferred until calling Scan on the returned Row. That Row will
// error with ErrNoRows if no rows are returned.
func (c *Conn) QueryRow(ctx context.Context, sql string, args ...interface{}) Row {
	rows, _ := c.Query(ctx, sql, args...)
	return (*connRow)(rows.(*connRows))
}
<|endoftext|>


<commit_before>package varys

import "github.com/garyburd/redigo/redis"

type Queue interface {
	Enqueue(urls ...string) error
	Dequeue() (string, error)

	Repaire() error

	DoneURL(url string) error
	RetryURL(url string) error

	FailedURLs() []string

	Cleanup()
}

type RedisQueue struct {
	pool *redis.Pool

	QueueReady   string
	QueuePending string
	QueueDone    string
	QueueFailed  string
}

func NewRedisQueue() *RedisQueue {
	return &RedisQueue{
		pool: &redis.Pool{
			Dial: func() (redis.Conn, error) {
				return redis.Dial("tcp", "127.0.0.1:6379")
			},
			Wait: true,
		},
		QueueReady:   "queue-ready",
		QueuePending: "queue-pending",
		QueueDone:    "queue-done",
		QueueFailed:  "queue-failed",
	}
}

func (q *RedisQueue) Enqueue(urls ...string) error {
	conn := q.pool.Get()
	defer conn.Close()

	for _, url := range urls {
		if dup, err := redis.Bool(conn.Do("SISMEMBER", q.QueueFailed, url)); err == nil && dup {
			continue
		}
		if dup, err := redis.Bool(conn.Do("SISMEMBER", q.QueueDone, url)); err == nil && dup {
			continue
		}
		if dup, err := redis.Bool(conn.Do("SISMEMBER", q.QueuePending, url)); err == nil && dup {
			continue
		}
		_, err := conn.Do("SADD", q.QueueReady, url)
		if err != nil {
			return err
		}
	}
	return nil
}

func (q *RedisQueue) Dequeue() (url string, err error) {
	conn := q.pool.Get()
	defer conn.Close()

	url, err = redis.String(conn.Do("SPOP", q.QueueReady))
	if err != nil {
		return "", nil
	}
	_, err = conn.Do("SADD", q.QueuePending, url)

	return
}

func (q *RedisQueue) Repaire() error {
	conn := q.pool.Get()
	defer conn.Close()
	_, err := conn.Do("SUNIONSTORE", q.QueueReady, q.QueueReady, q.QueuePending)
	return err
}

func (q *RedisQueue) DoneURL(url string) error {
	conn := q.pool.Get()
	defer conn.Close()
	_, err := conn.Do("SMOVE", q.QueuePending, q.QueueDone, url)
	return err
}

func (q *RedisQueue) RetryURL(url string) error {
	conn := q.pool.Get()
	defer conn.Close()
	_, err := conn.Do("SMOVE", q.QueuePending, q.QueueFailed, url)
	return err
}

func (q *RedisQueue) FailedURLs() []string {
	conn := q.pool.Get()
	defer conn.Close()
	urls, err := redis.Strings(conn.Do("SMEMBERS", q.QueueFailed))
	if err != nil {
		return nil
	}
	return urls
}

func (q *RedisQueue) Cleanup() {

}
<commit_msg>Added redis url & password<commit_after>package varys

import (
	"time"

	"github.com/garyburd/redigo/redis"
)

type Queue interface {
	Enqueue(urls ...string) error
	Dequeue() (string, error)

	Repaire() error

	DoneURL(url string) error
	RetryURL(url string) error

	FailedURLs() []string

	Cleanup()
}

type RedisQueue struct {
	pool *redis.Pool

	QueueReady   string
	QueuePending string
	QueueDone    string
	QueueFailed  string
}

func NewRedisQueue(url, password string) *RedisQueue {
	return &RedisQueue{
		pool: &redis.Pool{
			Dial: func() (redis.Conn, error) {
				conn, err := redis.Dial("tcp", url)
				if err != nil {
					return nil, err
				}
				if len(password) == 0 {
					return conn, err
				}
				if _, err = conn.Do("AUTH", password); err != nil {
					conn.Close()
					return nil, err
				}
				return conn, nil
			},
			TestOnBorrow: func(c redis.Conn, t time.Time) error {
				_, err := c.Do("PING")
				return err
			},
			Wait: true,
		},
		QueueReady:   "queue-ready",
		QueuePending: "queue-pending",
		QueueDone:    "queue-done",
		QueueFailed:  "queue-failed",
	}
}

func (q *RedisQueue) Enqueue(urls ...string) error {
	conn := q.pool.Get()
	defer conn.Close()

	for _, url := range urls {
		if dup, err := redis.Bool(conn.Do("SISMEMBER", q.QueueFailed, url)); err == nil && dup {
			continue
		}
		if dup, err := redis.Bool(conn.Do("SISMEMBER", q.QueueDone, url)); err == nil && dup {
			continue
		}
		if dup, err := redis.Bool(conn.Do("SISMEMBER", q.QueuePending, url)); err == nil && dup {
			continue
		}
		_, err := conn.Do("SADD", q.QueueReady, url)
		if err != nil {
			return err
		}
	}
	return nil
}

func (q *RedisQueue) Dequeue() (url string, err error) {
	conn := q.pool.Get()
	defer conn.Close()

	url, err = redis.String(conn.Do("SPOP", q.QueueReady))
	if err != nil {
		return "", nil
	}
	_, err = conn.Do("SADD", q.QueuePending, url)

	return
}

func (q *RedisQueue) Repaire() error {
	conn := q.pool.Get()
	defer conn.Close()
	_, err := conn.Do("SUNIONSTORE", q.QueueReady, q.QueueReady, q.QueuePending)
	return err
}

func (q *RedisQueue) DoneURL(url string) error {
	conn := q.pool.Get()
	defer conn.Close()
	_, err := conn.Do("SMOVE", q.QueuePending, q.QueueDone, url)
	return err
}

func (q *RedisQueue) RetryURL(url string) error {
	conn := q.pool.Get()
	defer conn.Close()
	_, err := conn.Do("SMOVE", q.QueuePending, q.QueueFailed, url)
	return err
}

func (q *RedisQueue) FailedURLs() []string {
	conn := q.pool.Get()
	defer conn.Close()
	urls, err := redis.Strings(conn.Do("SMEMBERS", q.QueueFailed))
	if err != nil {
		return nil
	}
	return urls
}

func (q *RedisQueue) Cleanup() {

}
<|endoftext|>


<commit_before>package nio

import "github.com/djherbis/buffer"

var empty struct{}

type Packet struct {
	data []byte
	ok   chan struct{}
}

func NewPacket() *Packet {
	return &Packet{
		ok: make(chan struct{}),
	}
}

type BufferQueue struct {
	top  *Packet
	data []byte
	buffer.Buffer
}

func NewBufferQueue(buf buffer.Buffer) *BufferQueue {
	return &BufferQueue{
		top:    NewPacket(),
		data:   make([]byte, 32*1024),
		Buffer: buf,
	}
}

func (q *BufferQueue) Len() (n int64) {
	return q.Buffer.Len()
}

func (q *BufferQueue) Peek() (b interface{}) {
	n, _ := q.Buffer.ReadAt(q.data, 0)
	q.top.data = q.data[:n]
	return q.top
}

func (q *BufferQueue) Pop() (b interface{}) {
	q.top.ok <- empty
	q.Buffer.FFwd(len(q.top.data))
	return q.top
}

func (q *BufferQueue) Push(b interface{}) {
	p := b.(*Packet)
	q.Write(p.data)
	p.ok <- empty
}
<commit_msg>Updated for FFwd change to int64<commit_after>package nio

import "github.com/djherbis/buffer"

var empty struct{}

type Packet struct {
	data []byte
	ok   chan struct{}
}

func NewPacket() *Packet {
	return &Packet{
		ok: make(chan struct{}),
	}
}

type BufferQueue struct {
	top  *Packet
	data []byte
	buffer.Buffer
}

func NewBufferQueue(buf buffer.Buffer) *BufferQueue {
	return &BufferQueue{
		top:    NewPacket(),
		data:   make([]byte, 32*1024),
		Buffer: buf,
	}
}

func (q *BufferQueue) Len() (n int64) {
	return q.Buffer.Len()
}

func (q *BufferQueue) Peek() (b interface{}) {
	n, _ := q.Buffer.ReadAt(q.data, 0)
	q.top.data = q.data[:n]
	return q.top
}

func (q *BufferQueue) Pop() (b interface{}) {
	q.top.ok <- empty
	q.Buffer.FFwd(int64(len(q.top.data)))
	return q.top
}

func (q *BufferQueue) Push(b interface{}) {
	p := b.(*Packet)
	q.Write(p.data)
	p.ok <- empty
}
<|endoftext|>


<commit_before>/*
Package queue provides a fast, ring-buffer queue based on the version suggested by Dariusz Górecki.
Using this instead of other, simpler, queue implementations (slice+append or linked list) provides
substantial memory and time benefits, and fewer GC pauses.

The queue implemented here is as fast as it is for an additional reason: it is *not* thread-safe.
*/
package queue

const minQueueLen = 16

// Queue represents a single instance of the queue data structure.
type Queue struct {
	buf               []interface{}
	head, tail, count int
}

// New constructs and returns a new Queue.
func New() *Queue {
	return &Queue{buf: make([]interface{}, minQueueLen)}
}

// Length returns the number of elements currently stored in the queue.
func (q *Queue) Length() int {
	return q.count
}

func (q *Queue) resize() {
	newBuf := make([]interface{}, q.count*2)

	if q.tail > q.head {
		copy(newBuf, q.buf[q.head:q.tail])
	} else {
		copy(newBuf, q.buf[q.head:len(q.buf)])
		copy(newBuf[len(q.buf)-q.head:], q.buf[:q.tail])
	}

	q.head = 0
	q.tail = q.count
	q.buf = newBuf
}

// Add puts an element on the end of the queue.
func (q *Queue) Add(elem interface{}) {
	if q.count == len(q.buf) {
		q.resize()
	}

	q.buf[q.tail] = elem
	q.tail = (q.tail + 1) % len(q.buf)
	q.count++
}

// Peek returns the element at the head of the queue. This call panics
// if the queue is empty.
func (q *Queue) Peek() interface{} {
	if q.Length() <= 0 {
		panic("queue: empty queue")
	}
	return q.buf[q.head]
}

// Get returns the element at index i in the queue. If the index is
// invalid, the call will panic.
func (q *Queue) Get(i int) interface{} {
	if i >= q.Length() || i < 0 {
		panic("queue: index out of range")
	}
	modi := (q.head + i) % len(q.buf)
	return q.buf[modi]
}

// Remove removes the element from the front of the queue. If you actually
// want the element, call Peek first. This call panics if the queue is empty.
func (q *Queue) Remove() {
	if q.Length() <= 0 {
		panic("queue: empty queue")
	}
	q.buf[q.head] = nil
	q.head = (q.head + 1) % len(q.buf)
	q.count--
	if len(q.buf) > minQueueLen && q.count*4 <= len(q.buf) {
		q.resize()
	}
}
<commit_msg>Minor simplification<commit_after>/*
Package queue provides a fast, ring-buffer queue based on the version suggested by Dariusz Górecki.
Using this instead of other, simpler, queue implementations (slice+append or linked list) provides
substantial memory and time benefits, and fewer GC pauses.

The queue implemented here is as fast as it is for an additional reason: it is *not* thread-safe.
*/
package queue

const minQueueLen = 16

// Queue represents a single instance of the queue data structure.
type Queue struct {
	buf               []interface{}
	head, tail, count int
}

// New constructs and returns a new Queue.
func New() *Queue {
	return &Queue{buf: make([]interface{}, minQueueLen)}
}

// Length returns the number of elements currently stored in the queue.
func (q *Queue) Length() int {
	return q.count
}

func (q *Queue) resize() {
	newBuf := make([]interface{}, q.count*2)

	if q.tail > q.head {
		copy(newBuf, q.buf[q.head:q.tail])
	} else {
		copy(newBuf, q.buf[q.head:len(q.buf)])
		copy(newBuf[len(q.buf)-q.head:], q.buf[:q.tail])
	}

	q.head = 0
	q.tail = q.count
	q.buf = newBuf
}

// Add puts an element on the end of the queue.
func (q *Queue) Add(elem interface{}) {
	if q.count == len(q.buf) {
		q.resize()
	}

	q.buf[q.tail] = elem
	q.tail = (q.tail + 1) % len(q.buf)
	q.count++
}

// Peek returns the element at the head of the queue. This call panics
// if the queue is empty.
func (q *Queue) Peek() interface{} {
	if q.Length() <= 0 {
		panic("queue: empty queue")
	}
	return q.buf[q.head]
}

// Get returns the element at index i in the queue. If the index is
// invalid, the call will panic.
func (q *Queue) Get(i int) interface{} {
	if i >= q.Length() || i < 0 {
		panic("queue: index out of range")
	}
	return q.buf[(q.head+i)%len(q.buf)]
}

// Remove removes the element from the front of the queue. If you actually
// want the element, call Peek first. This call panics if the queue is empty.
func (q *Queue) Remove() {
	if q.Length() <= 0 {
		panic("queue: empty queue")
	}
	q.buf[q.head] = nil
	q.head = (q.head + 1) % len(q.buf)
	q.count--
	if len(q.buf) > minQueueLen && q.count*4 <= len(q.buf) {
		q.resize()
	}
}
<|endoftext|>


<commit_before>package qsim

// A Queue holds Jobs until they're ready for processing.
type Queue struct {
	// The Jobs currently in the queue
	Jobs []*Job

	// Callback lists
	cbBeforeAppend []func(q *Queue, j *Job)
	cbAfterAppend  []func(q *Queue, j *Job)
	cbBeforeShift  []func(q *Queue, j *Job)
	cbAfterShift   []func(q *Queue, j *Job)
}

// Append adds a Job to the tail of the queue.
func (q *Queue) Append(j *Job) {
	q.beforeAppend(j)
	q.Jobs = append(q.Jobs, j)
	q.afterAppend(j)
}

// Shift removes a Job from the head of the queue.
//
// It returns the Job that was removed, as well as the number of Jobs
// still left in the queue after shifting. When Shift is called on an
// empty queue, j will be nil. So an appropriate use of Shift looks like
// this:
//
//  j, nrem := q.Shift()
//	if j != nil {
//	    // Do something with j
//  }
func (q *Queue) Shift() (j *Job, nrem int) {
	if len(q.Jobs) == 0 {
		q.beforeShift(nil)
		q.afterShift(nil)
		return nil, 0
	}
	j = q.Jobs[0]
	q.beforeShift(j)
	q.Jobs = q.Jobs[1:]
	q.afterShift(j)
	return j, len(q.Jobs)
}

// OnBeforeAppend adds a callback to be run immediately before a Job is
// appended to the queue.
//
// The callback will be passed the queue itself and the job that's about
// to be appended.
func (q *Queue) OnBeforeAppend(f func(q *Queue, j *Job)) {
	q.cbBeforeAppend = append(q.cbBeforeAppend, f)
}
func (q *Queue) beforeAppend(j *Job) {
	for _, cb := range q.cbBeforeAppend {
		cb(q, j)
	}
}

// OnAfterAppend adds a callback to be run immediately after a Job is
// appended to the queue.
//
// The callback will be passed the queue itself and the job that was just
// appended.
func (q *Queue) OnAfterAppend(f func(q *Queue, j *Job)) {
	q.cbAfterAppend = append(q.cbAfterAppend, f)
}
func (q *Queue) afterAppend(j *Job) {
	for _, cb := range q.cbAfterAppend {
		cb(q, j)
	}
}

// OnBeforeShift adds a callback to be run immediately before a Job is
// shifted out of the queue.
//
// The callback will be passed the queue itself and the job that's about
// to be shifted. If Shift is called on an empty queue, this callback
// will run but j will be nil.
func (q *Queue) OnBeforeShift(f func(q *Queue, j *Job)) {
	q.cbBeforeShift = append(q.cbBeforeShift, f)
}
func (q *Queue) beforeShift(j *Job) {
	for _, cb := range q.cbBeforeShift {
		cb(q, j)
	}
}

// OnAfterShift adds a callback to be run immediately after a Job is
// shifted out of the queue.
//
// The callback will be passed the queue itself and the job that was
// just shifted. If Shift is called on an empty queue, this callback
// will run but j will be nil.
func (q *Queue) OnAfterShift(f func(q *Queue, j *Job)) {
	q.cbAfterShift = append(q.cbAfterShift, f)
}
func (q *Queue) afterShift(j *Job) {
	for _, cb := range q.cbAfterShift {
		cb(q, j)
	}
}

// NewQueue creates an empty Queue.
func NewQueue() (q *Queue) {
	q = new(Queue)
	q.Jobs = make([]*Job, 0)
	return q
}
<commit_msg>Fixed formatting of godoc I hope<commit_after>package qsim

// A Queue holds Jobs until they're ready for processing.
type Queue struct {
	// The Jobs currently in the queue
	Jobs []*Job

	// Callback lists
	cbBeforeAppend []func(q *Queue, j *Job)
	cbAfterAppend  []func(q *Queue, j *Job)
	cbBeforeShift  []func(q *Queue, j *Job)
	cbAfterShift   []func(q *Queue, j *Job)
}

// Append adds a Job to the tail of the queue.
func (q *Queue) Append(j *Job) {
	q.beforeAppend(j)
	q.Jobs = append(q.Jobs, j)
	q.afterAppend(j)
}

// Shift removes a Job from the head of the queue.
//
// It returns the Job that was removed, as well as the number of Jobs
// still left in the queue after shifting. When Shift is called on an
// empty queue, j will be nil. So an appropriate use of Shift looks like
// this:
//
//  j, nrem := q.Shift()
//	if j != nil {
//    // Do something with j
//  }
func (q *Queue) Shift() (j *Job, nrem int) {
	if len(q.Jobs) == 0 {
		q.beforeShift(nil)
		q.afterShift(nil)
		return nil, 0
	}
	j = q.Jobs[0]
	q.beforeShift(j)
	q.Jobs = q.Jobs[1:]
	q.afterShift(j)
	return j, len(q.Jobs)
}

// OnBeforeAppend adds a callback to be run immediately before a Job is
// appended to the queue.
//
// The callback will be passed the queue itself and the job that's about
// to be appended.
func (q *Queue) OnBeforeAppend(f func(q *Queue, j *Job)) {
	q.cbBeforeAppend = append(q.cbBeforeAppend, f)
}
func (q *Queue) beforeAppend(j *Job) {
	for _, cb := range q.cbBeforeAppend {
		cb(q, j)
	}
}

// OnAfterAppend adds a callback to be run immediately after a Job is
// appended to the queue.
//
// The callback will be passed the queue itself and the job that was just
// appended.
func (q *Queue) OnAfterAppend(f func(q *Queue, j *Job)) {
	q.cbAfterAppend = append(q.cbAfterAppend, f)
}
func (q *Queue) afterAppend(j *Job) {
	for _, cb := range q.cbAfterAppend {
		cb(q, j)
	}
}

// OnBeforeShift adds a callback to be run immediately before a Job is
// shifted out of the queue.
//
// The callback will be passed the queue itself and the job that's about
// to be shifted. If Shift is called on an empty queue, this callback
// will run but j will be nil.
func (q *Queue) OnBeforeShift(f func(q *Queue, j *Job)) {
	q.cbBeforeShift = append(q.cbBeforeShift, f)
}
func (q *Queue) beforeShift(j *Job) {
	for _, cb := range q.cbBeforeShift {
		cb(q, j)
	}
}

// OnAfterShift adds a callback to be run immediately after a Job is
// shifted out of the queue.
//
// The callback will be passed the queue itself and the job that was
// just shifted. If Shift is called on an empty queue, this callback
// will run but j will be nil.
func (q *Queue) OnAfterShift(f func(q *Queue, j *Job)) {
	q.cbAfterShift = append(q.cbAfterShift, f)
}
func (q *Queue) afterShift(j *Job) {
	for _, cb := range q.cbAfterShift {
		cb(q, j)
	}
}

// NewQueue creates an empty Queue.
func NewQueue() (q *Queue) {
	q = new(Queue)
	q.Jobs = make([]*Job, 0)
	return q
}
<|endoftext|>


<commit_before>/* Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements.  See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License.  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

package main

import (
	"flag"
	"fmt"
	"github.com/stealthly/go-avro"
	"io/ioutil"
	"os"
	"strings"
)

type schemas []string

func (i *schemas) String() string {
	return fmt.Sprintf("%s", *i)
}

func (i *schemas) Set(value string) error {
	*i = append(*i, value)
	return nil
}

//var schema = flag.String("schema", "", "Path to avsc schema file.")
var schema schemas
var output = flag.String("out", "", "Output file name.")

func main() {
	parseAndValidateArgs()

	schemas := make([]string, 0)
	for _, schema := range schema {
		contents, err := ioutil.ReadFile(schema)
		checkErr(err)
		schemas = append(schemas, string(contents))
	}

	gen := avro.NewCodeGenerator(schemas)
	code, err := gen.Generate()
	checkErr(err)

	createDirs()
	err = ioutil.WriteFile(*output, []byte(code), 0664)
	checkErr(err)
}

func parseAndValidateArgs() {
	flag.Var(&schema, "schema", "Path to avsc schema file.")
	flag.Parse()

	if len(schema) == 0 {
		fmt.Println("At least one --schema flag is required.")
		os.Exit(1)
	}

	if *output == "" {
		fmt.Println("--out flag is required.")
		os.Exit(1)
	}
}

func createDirs() {
	index := strings.LastIndex(*output, "/")
	if index != -1 {
		path := (*output)[:index]
		err := os.MkdirAll(path, 0777)
		checkErr(err)
	}
}

func checkErr(err error) {
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
}
<commit_msg>removed unnecessary comment. re #13<commit_after>/* Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements.  See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License.  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

package main

import (
	"flag"
	"fmt"
	"github.com/stealthly/go-avro"
	"io/ioutil"
	"os"
	"strings"
)

type schemas []string

func (i *schemas) String() string {
	return fmt.Sprintf("%s", *i)
}

func (i *schemas) Set(value string) error {
	*i = append(*i, value)
	return nil
}

var schema schemas
var output = flag.String("out", "", "Output file name.")

func main() {
	parseAndValidateArgs()

	schemas := make([]string, 0)
	for _, schema := range schema {
		contents, err := ioutil.ReadFile(schema)
		checkErr(err)
		schemas = append(schemas, string(contents))
	}

	gen := avro.NewCodeGenerator(schemas)
	code, err := gen.Generate()
	checkErr(err)

	createDirs()
	err = ioutil.WriteFile(*output, []byte(code), 0664)
	checkErr(err)
}

func parseAndValidateArgs() {
	flag.Var(&schema, "schema", "Path to avsc schema file.")
	flag.Parse()

	if len(schema) == 0 {
		fmt.Println("At least one --schema flag is required.")
		os.Exit(1)
	}

	if *output == "" {
		fmt.Println("--out flag is required.")
		os.Exit(1)
	}
}

func createDirs() {
	index := strings.LastIndex(*output, "/")
	if index != -1 {
		path := (*output)[:index]
		err := os.MkdirAll(path, 0777)
		checkErr(err)
	}
}

func checkErr(err error) {
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"html/template"
	"os"
	"regexp"
	"strings"

	"github.com/spf13/cobra"
	"github.com/spf13/cobra/doc"

	fn "knative.dev/func"
	"knative.dev/func/cmd"
)

var (
	// Helper function for indenting template values correctly
	fm = template.FuncMap{
		"indent": func(i int, c string, v string) string {
			indentation := strings.Repeat(c, i)
			return indentation + strings.Replace(v, "\n", "\n"+indentation, -1)
		},
	}

	rootName  = "func"
	targetDir = "docs/reference"
)

// String substitutions in command help docs
type TemplateOptions struct {
	Name    string
	Options string
	Use     string
}

// This helper application generates markdown help documents
func main() {
	// Create a new client so that we can get builtin repo options
	client := fn.New()

	// Generate options for templates
	opts, err := cmd.RuntimeTemplateOptions(client)
	if err != nil {
		fmt.Printf("%s\n", err)
	}

	// Initialize an options struct
	templateOptions := TemplateOptions{
		Name:    rootName,
		Options: opts,
		Use:     rootName,
	}

	// Create the root command
	var root = cmd.NewRootCmd(cmd.RootCommandConfig{Name: rootName})

	// Write the markdown for the root command
	if err := writeMarkdown(root, rootName, templateOptions); err != nil {
		fmt.Fprintf(os.Stderr, "Error writing help markdown %s", err)
	}

	// Recurse all subcommands and write the markdown for them
	if err := processSubCommands(root, rootName, templateOptions); err != nil {
		fmt.Fprintf(os.Stderr, "Error processing subcommands %s", err)
	}
}

// processSubCommands is a recursive function which writes the markdown text
// for all subcommands of the provided cobra command, prepending the parent
// string to the fiile name, and recursively calls itself for each subcommand.
func processSubCommands(c *cobra.Command, parent string, opts TemplateOptions) error {
	for _, cc := range c.Commands() {
		name := cc.Name()
		if name == "help" {
			continue
		}
		if parent != "" {
			name = parent + "_" + name
		}
		if err := writeMarkdown(cc, name, opts); err != nil {
			return err
		}
		if err := processSubCommands(cc, name, opts); err != nil {
			return err
		}
	}
	return nil
}

// writeMarkdown generates the untemplated markdown string for the given
// command, then does standard template substitution on the generated markdown,
// ultimately writing the markdown file to docs/reference/[command_name].md
func writeMarkdown(c *cobra.Command, name string, opts TemplateOptions) error {
	out := new(bytes.Buffer)
	c.DisableAutoGenTag = true
	err := doc.GenMarkdown(c, out)
	if err != nil {
		return err
	}
	t := template.New("help")
	t.Funcs(fm)
	tpl := template.Must(t.Parse(out.String()))
	out.Reset()
	if err := tpl.Execute(out, opts); err != nil {
		fmt.Fprintf(os.Stderr, "unable to process help text: %v", err)
	}

	re := regexp.MustCompile(`[^\S\r\n]+\n`)
	data := re.ReplaceAll(out.Bytes(), []byte{'\n'}) // trim white spaces before EOL

	if err := os.WriteFile(targetDir+"/"+name+".md", data, 0666); err != nil {
		return err
	}
	return nil
}
<commit_msg>fix: ignore global config on docs gen (#1424)<commit_after>package main

import (
	"bytes"
	"fmt"
	"html/template"
	"os"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/spf13/cobra"
	"github.com/spf13/cobra/doc"

	fn "knative.dev/func"
	"knative.dev/func/cmd"
)

var (
	// Helper function for indenting template values correctly
	fm = template.FuncMap{
		"indent": func(i int, c string, v string) string {
			indentation := strings.Repeat(c, i)
			return indentation + strings.Replace(v, "\n", "\n"+indentation, -1)
		},
	}

	rootName  = "func"
	targetDir = "docs/reference"
)

// String substitutions in command help docs
type TemplateOptions struct {
	Name    string
	Options string
	Use     string
}

// This helper application generates markdown help documents
func main() {
	// Ignore global config.yaml if it exists
	defer ignoreConfigEnv()()

	// Create a new client so that we can get builtin repo options
	client := fn.New()

	// Generate options for templates
	opts, err := cmd.RuntimeTemplateOptions(client)
	if err != nil {
		fmt.Printf("%s\n", err)
	}

	// Initialize an options struct
	templateOptions := TemplateOptions{
		Name:    rootName,
		Options: opts,
		Use:     rootName,
	}

	// Create the root command
	var root = cmd.NewRootCmd(cmd.RootCommandConfig{Name: rootName})

	// Write the markdown for the root command
	if err := writeMarkdown(root, rootName, templateOptions); err != nil {
		fmt.Fprintf(os.Stderr, "Error writing help markdown %s", err)
	}

	// Recurse all subcommands and write the markdown for them
	if err := processSubCommands(root, rootName, templateOptions); err != nil {
		fmt.Fprintf(os.Stderr, "Error processing subcommands %s", err)
	}
}

// ignoreConfigEnv sets FUNC_CONFIG_FILE to a nonexistent path and
// returns a function which undoes this change.
func ignoreConfigEnv() (done func()) {
	var (
		env   = "FUNC_CONFIG_FILE"
		v, ok = os.LookupEnv(env)
	)
	os.Setenv(env, filepath.Join(os.TempDir(), "nonexistent.yaml"))
	return func() {
		if ok {
			os.Setenv(env, v)
		} else {
			os.Unsetenv(env)
		}
	}
}

// processSubCommands is a recursive function which writes the markdown text
// for all subcommands of the provided cobra command, prepending the parent
// string to the fiile name, and recursively calls itself for each subcommand.
func processSubCommands(c *cobra.Command, parent string, opts TemplateOptions) error {
	for _, cc := range c.Commands() {
		name := cc.Name()
		if name == "help" {
			continue
		}
		if parent != "" {
			name = parent + "_" + name
		}
		if err := writeMarkdown(cc, name, opts); err != nil {
			return err
		}
		if err := processSubCommands(cc, name, opts); err != nil {
			return err
		}
	}
	return nil
}

// writeMarkdown generates the untemplated markdown string for the given
// command, then does standard template substitution on the generated markdown,
// ultimately writing the markdown file to docs/reference/[command_name].md
func writeMarkdown(c *cobra.Command, name string, opts TemplateOptions) error {
	out := new(bytes.Buffer)
	c.DisableAutoGenTag = true
	err := doc.GenMarkdown(c, out)
	if err != nil {
		return err
	}
	t := template.New("help")
	t.Funcs(fm)
	tpl := template.Must(t.Parse(out.String()))
	out.Reset()
	if err := tpl.Execute(out, opts); err != nil {
		fmt.Fprintf(os.Stderr, "unable to process help text: %v", err)
	}

	re := regexp.MustCompile(`[^\S\r\n]+\n`)
	data := re.ReplaceAll(out.Bytes(), []byte{'\n'}) // trim white spaces before EOL

	if err := os.WriteFile(targetDir+"/"+name+".md", data, 0666); err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package collector

import (
	"encoding/json"
	"strings"

	flattener "github.com/karimra/go-map-flattener"
	"github.com/openconfig/gnmi/proto/gnmi"
)

// EventMsg //
type EventMsg struct {
	Name      string                 `json:"name,omitempty"` // measurement name
	Timestamp int64                  `json:"timestamp,omitempty"`
	Tags      map[string]string      `json:"tags,omitempty"`
	Values    map[string]interface{} `json:"values,omitempty"`
	Deletes   []string               `json:"deletes,omitempty"`
}

// ResponseToEventMsgs //
func ResponseToEventMsgs(name string, rsp *gnmi.SubscribeResponse, meta map[string]string) ([]*EventMsg, error) {
	if rsp == nil {
		return nil, nil
	}
	evs := make([]*EventMsg, 0)
	switch rsp := rsp.Response.(type) {
	case *gnmi.SubscribeResponse_Update:
		tags := make(map[string]string)
		namePrefix, prefixTags := TagsFromGNMIPath(rsp.Update.Prefix)
		for k, v := range prefixTags {
			if _, ok := tags[k]; ok {
				tags[strings.Join([]string{namePrefix, k}, "/")] = v
				continue
			}
			tags[k] = v
		}

		for _, upd := range rsp.Update.Update {
			e := &EventMsg{
				Tags:   make(map[string]string),
				Values: make(map[string]interface{}),
			}
			e.Timestamp = rsp.Update.Timestamp
			e.Name = name
			e.Tags = tags
			e.Values = make(map[string]interface{})
			pathName, pTags := TagsFromGNMIPath(upd.Path)
			pathName = strings.TrimRight(namePrefix, "/") + "/" + strings.TrimLeft(pathName, "/")
			for k, v := range pTags {
				if _, ok := e.Tags[k]; ok {
					e.Tags[strings.Join([]string{pathName, k}, "/")] = v
					continue
				}
				e.Tags[k] = v
			}
			vs, err := getValueFlat(pathName, upd.GetVal())
			if err != nil {
				return nil, err
			}
			for k, v := range vs {
				e.Values[k] = v
			}
			for k, v := range meta {
				if k == "format" {
					continue
				}
				if _, ok := e.Tags[k]; ok {
					e.Tags["meta:"+k] = v
					continue
				}
				e.Tags[k] = v
			}
			evs = append(evs, e)
		}

		if len(rsp.Update.Delete) > 0 {
			e := &EventMsg{
				Deletes: make([]string, 0, len(rsp.Update.Delete)),
			}
			e.Timestamp = rsp.Update.Timestamp
			e.Name = name
			e.Tags = tags
			for k, v := range meta {
				if k == "format" {
					continue
				}
				if _, ok := e.Tags[k]; ok {
					e.Tags["meta:"+k] = v
					continue
				}
				e.Tags[k] = v
			}
			for _, del := range rsp.Update.Delete {
				e.Deletes = append(e.Deletes, gnmiPathToXPath(del))
			}
			evs = append(evs, e)
		}
	}
	return evs, nil
}

// TagsFromGNMIPath //
func TagsFromGNMIPath(p *gnmi.Path) (string, map[string]string) {
	if p == nil {
		return "", nil
	}
	tags := make(map[string]string)
	sb := strings.Builder{}
	if p.Origin != "" {
		sb.WriteString(p.Origin)
		sb.Write([]byte(":"))
	}
	for _, e := range p.Elem {
		if e.Name != "" {
			sb.Write([]byte("/"))
			sb.WriteString(e.Name)
		}
		if e.Key != nil {
			for k, v := range e.Key {
				tags[k] = v
			}
		}
	}
	return sb.String(), tags
}

func getValueFlat(prefix string, updValue *gnmi.TypedValue) (map[string]interface{}, error) {
	if updValue == nil {
		return nil, nil
	}
	var jsondata []byte
	values := make(map[string]interface{})
	switch updValue.Value.(type) {
	case *gnmi.TypedValue_AsciiVal:
		values[prefix] = updValue.GetAsciiVal()
	case *gnmi.TypedValue_BoolVal:
		values[prefix] = updValue.GetBoolVal()
	case *gnmi.TypedValue_BytesVal:
		values[prefix] = updValue.GetBytesVal()
	case *gnmi.TypedValue_DecimalVal:
		values[prefix] = updValue.GetDecimalVal()
	case *gnmi.TypedValue_FloatVal:
		values[prefix] = updValue.GetFloatVal()
	case *gnmi.TypedValue_IntVal:
		values[prefix] = updValue.GetIntVal()
	case *gnmi.TypedValue_StringVal:
		values[prefix] = updValue.GetStringVal()
	case *gnmi.TypedValue_UintVal:
		values[prefix] = updValue.GetUintVal()
	case *gnmi.TypedValue_LeaflistVal:
		values[prefix] = updValue.GetLeaflistVal()
	case *gnmi.TypedValue_ProtoBytes:
		values[prefix] = updValue.GetProtoBytes()
	case *gnmi.TypedValue_AnyVal:
		values[prefix] = updValue.GetAnyVal()
	case *gnmi.TypedValue_JsonIetfVal:
		jsondata = updValue.GetJsonIetfVal()
	case *gnmi.TypedValue_JsonVal:
		jsondata = updValue.GetJsonVal()
	}
	if len(jsondata) != 0 {
		var value interface{}
		err := json.Unmarshal(jsondata, &value)
		if err != nil {
			return nil, err
		}
		switch value := value.(type) {
		case map[string]interface{}:
			f := flattener.NewFlattener()
			values, err = f.Flatten(value)
		default:
			values[prefix] = value
		}
		if err != nil {
			return nil, err
		}
	}
	return values, nil
}
<commit_msg>simplify ResponseToEventMsgs<commit_after>package collector

import (
	"encoding/json"
	"strings"

	flattener "github.com/karimra/go-map-flattener"
	"github.com/openconfig/gnmi/proto/gnmi"
)

// EventMsg //
type EventMsg struct {
	Name      string                 `json:"name,omitempty"` // measurement name
	Timestamp int64                  `json:"timestamp,omitempty"`
	Tags      map[string]string      `json:"tags,omitempty"`
	Values    map[string]interface{} `json:"values,omitempty"`
	Deletes   []string               `json:"deletes,omitempty"`
}

// ResponseToEventMsgs //
func ResponseToEventMsgs(name string, rsp *gnmi.SubscribeResponse, meta map[string]string) ([]*EventMsg, error) {
	if rsp == nil {
		return nil, nil
	}
	var err error
	evs := make([]*EventMsg, 0)
	switch rsp := rsp.Response.(type) {
	case *gnmi.SubscribeResponse_Update:
		tags := make(map[string]string)
		namePrefix, prefixTags := TagsFromGNMIPath(rsp.Update.Prefix)
		for k, v := range prefixTags {
			if vv, ok := tags[k]; ok {
				if v != vv {
					tags[namePrefix+":::"+k] = v
				}
				continue
			}
			tags[k] = v
		}
		for _, upd := range rsp.Update.Update {
			e := &EventMsg{
				Tags:   make(map[string]string),
				Values: make(map[string]interface{}),
			}
			e.Timestamp = rsp.Update.Timestamp
			e.Name = name
			e.Tags = tags
			pathName, pTags := TagsFromGNMIPath(upd.Path)
			pathName = strings.TrimRight(namePrefix, "/") + "/" + strings.TrimLeft(pathName, "/")
			for k, v := range pTags {
				if vv, ok := e.Tags[k]; ok {
					if v != vv {
						e.Tags[pathName+":::"+k] = v
					}
					continue
				}
				e.Tags[k] = v
			}
			e.Values, err = getValueFlat(pathName, upd.GetVal())
			if err != nil {
				return nil, err
			}
			for k, v := range meta {
				if k == "format" {
					continue
				}
				if _, ok := e.Tags[k]; ok {
					e.Tags["meta:"+k] = v
					continue
				}
				e.Tags[k] = v
			}
			evs = append(evs, e)
		}

		if len(rsp.Update.Delete) > 0 {
			e := &EventMsg{
				Deletes: make([]string, 0, len(rsp.Update.Delete)),
			}
			e.Timestamp = rsp.Update.Timestamp
			e.Name = name
			e.Tags = tags
			for k, v := range meta {
				if k == "format" {
					continue
				}
				if _, ok := e.Tags[k]; ok {
					e.Tags["meta:"+k] = v
					continue
				}
				e.Tags[k] = v
			}
			for _, del := range rsp.Update.Delete {
				e.Deletes = append(e.Deletes, gnmiPathToXPath(del))
			}
			evs = append(evs, e)
		}
	}
	return evs, nil
}

// TagsFromGNMIPath //
func TagsFromGNMIPath(p *gnmi.Path) (string, map[string]string) {
	if p == nil {
		return "", nil
	}
	tags := make(map[string]string)
	sb := strings.Builder{}
	if p.Origin != "" {
		sb.WriteString(p.Origin)
		sb.Write([]byte(":"))
	}
	for _, e := range p.Elem {
		if e.Name != "" {
			sb.Write([]byte("/"))
			sb.WriteString(e.Name)
		}
		if e.Key != nil {
			for k, v := range e.Key {
				tags[k] = v
			}
		}
	}
	return sb.String(), tags
}

func getValueFlat(prefix string, updValue *gnmi.TypedValue) (map[string]interface{}, error) {
	if updValue == nil {
		return nil, nil
	}
	var jsondata []byte
	values := make(map[string]interface{})
	switch updValue.Value.(type) {
	case *gnmi.TypedValue_AsciiVal:
		values[prefix] = updValue.GetAsciiVal()
	case *gnmi.TypedValue_BoolVal:
		values[prefix] = updValue.GetBoolVal()
	case *gnmi.TypedValue_BytesVal:
		values[prefix] = updValue.GetBytesVal()
	case *gnmi.TypedValue_DecimalVal:
		values[prefix] = updValue.GetDecimalVal()
	case *gnmi.TypedValue_FloatVal:
		values[prefix] = updValue.GetFloatVal()
	case *gnmi.TypedValue_IntVal:
		values[prefix] = updValue.GetIntVal()
	case *gnmi.TypedValue_StringVal:
		values[prefix] = updValue.GetStringVal()
	case *gnmi.TypedValue_UintVal:
		values[prefix] = updValue.GetUintVal()
	case *gnmi.TypedValue_LeaflistVal:
		values[prefix] = updValue.GetLeaflistVal()
	case *gnmi.TypedValue_ProtoBytes:
		values[prefix] = updValue.GetProtoBytes()
	case *gnmi.TypedValue_AnyVal:
		values[prefix] = updValue.GetAnyVal()
	case *gnmi.TypedValue_JsonIetfVal:
		jsondata = updValue.GetJsonIetfVal()
	case *gnmi.TypedValue_JsonVal:
		jsondata = updValue.GetJsonVal()
	}
	if len(jsondata) != 0 {
		var value interface{}
		err := json.Unmarshal(jsondata, &value)
		if err != nil {
			return nil, err
		}
		switch value := value.(type) {
		case map[string]interface{}:
			f := flattener.NewFlattener()
			values, err = f.Flatten(value)
		default:
			values[prefix] = value
		}
		if err != nil {
			return nil, err
		}
	}
	return values, nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net/url"
	"strconv"
	"strings"

	"github.com/weirdtales/senor-rosado/slack"
)

// Register for giphy cartridge
func Register() (r string, h string) {
	r = "^(fixer|fx) "
	h = "`fixer eur 99.95 aud` use fixer.io to convert currency"
	return
}

type fixerData struct {
	Base  string             `json:"base"`
	Date  string             `json:"date"`
	Rates map[string]float64 `json:"rates"`
}

type currencyMap map[string]struct {
	Symbol        string  `json:"symbol"`
	SymbolNative  string  `json:"symbol_native"`
	DecimalDigits int     `json:"decimal_digits"`
	Rounding      float64 `json:"rounding"`
	Code          string  `json:"code"`
}

// Respond finds a Gif on Giphy
func Respond(m slack.Message, c slack.Conn, matches []string) {
	// TODO dumplication of problema
	defer slack.PanicSuppress()
	reply := slack.Reply{}
	reply.Channel = m.Channel

	u := "https://api.fixer.io/latest?base=%s"
	u = fmt.Sprintf(u, url.QueryEscape(m.Subcommand))

	var resp fixerData
	if err := slack.GetJSON(u, &resp); err != nil {
		slack.SendError(c, m, err, "")
		return
	}

	u = "http://www.localeplanet.com/api/auto/currencymap.json"
	var cmap currencyMap
	if err := slack.GetJSON(u, &cmap); err != nil {
		slack.SendError(c, m, err, "")
		return
	}

	reply.Text = "hoy no"
	amt, err := strconv.ParseFloat(string(m.Args[1]), 64)
	if err != nil {
		slack.SendError(c, m, err, "")
		return
	}

	t := strings.ToUpper(string(m.Args[2]))
	rate, ok := resp.Rates[t]
	if !ok {
		slack.SendError(c, m, err, fmt.Sprintf("%s :point_left: ¿Qué es esto?", t))
		c.Send(m, reply)
		return
	}

	b := strings.ToUpper(m.Subcommand)
	result := amt * rate
	bsym, ok := cmap[b]
	if !ok {
		slack.SendError(c, m, err, "")
		return
	}
	tsym, ok := cmap[t]
	if !ok {
		slack.SendError(c, m, err, "")
		return
	}
	reply.Text = fmt.Sprintf("%s%.2f :point_right: %s%.2f", bsym.SymbolNative, amt, tsym.SymbolNative, result)
	c.Send(m, reply)
}
<commit_msg>use regexp results instead of args<commit_after>package main

import (
	"fmt"
	"net/url"
	"strconv"
	"strings"

	"github.com/weirdtales/senor-rosado/slack"
)

// Register for giphy cartridge
func Register() (r string, h string) {
	r = `^(fixer|fx) (\w\w\w) ([\d.]+) (\w\w\w)`
	h = "`fixer eur 99.95 aud` use fixer.io to convert currency"
	return
}

type fixerData struct {
	Base  string             `json:"base"`
	Date  string             `json:"date"`
	Rates map[string]float64 `json:"rates"`
}

type currencyMap map[string]struct {
	Symbol        string  `json:"symbol"`
	SymbolNative  string  `json:"symbol_native"`
	DecimalDigits int     `json:"decimal_digits"`
	Rounding      float64 `json:"rounding"`
	Code          string  `json:"code"`
}

// Respond finds a Gif on Giphy
func Respond(m slack.Message, c slack.Conn, matches []string) {
	// TODO dumplication of problema
	defer slack.PanicSuppress()
	reply := slack.Reply{}
	reply.Channel = m.Channel

	u := "https://api.fixer.io/latest?base=%s"
	u = fmt.Sprintf(u, url.QueryEscape(matches[2]))

	var resp fixerData
	if err := slack.GetJSON(u, &resp); err != nil {
		slack.SendError(c, m, err, "")
		return
	}

	u = "http://www.localeplanet.com/api/auto/currencymap.json"
	var cmap currencyMap
	if err := slack.GetJSON(u, &cmap); err != nil {
		slack.SendError(c, m, err, "")
		return
	}

	reply.Text = "hoy no"
	amt, err := strconv.ParseFloat(matches[3], 64)
	if err != nil {
		slack.SendError(c, m, err, "")
		return
	}

	t := strings.ToUpper(matches[4])
	rate, ok := resp.Rates[t]
	if !ok {
		slack.SendError(c, m, err, fmt.Sprintf("%s :point_left: ¿Qué es esto?", t))
		c.Send(m, reply)
		return
	}

	b := strings.ToUpper(m.Subcommand)
	result := amt * rate
	bsym, ok := cmap[b]
	if !ok {
		slack.SendError(c, m, err, "")
		return
	}
	tsym, ok := cmap[t]
	if !ok {
		slack.SendError(c, m, err, "")
		return
	}
	reply.Text = fmt.Sprintf("%s%.2f :point_right: %s%.2f", bsym.SymbolNative, amt, tsym.SymbolNative, result)
	c.Send(m, reply)
}
<|endoftext|>


<commit_before>package gore

import (
	"io"
	"strconv"
	"time"
)

// Reply type, similar to Hiredis
const (
	ReplyString  = 1
	ReplyArray   = 2
	ReplyInteger = 3
	ReplyNil     = 4
	ReplyStatus  = 5
	ReplyError   = 6
)

// Pair holds a pair of value, such as reply from HGETALL or ZRANGE [WITHSCORES]
type Pair struct {
	First  []byte
	Second []byte
}

// Reply holds redis reply
type Reply struct {
	replyType    int
	integerValue int64
	stringValue  []byte
	arrayValue   []*Reply
}

// Type returns reply type
func (r *Reply) Type() int {
	return r.replyType
}

// String returns string value of a reply
func (r *Reply) String() (string, error) {
	if r.Type() == ReplyNil {
		return "", ErrNil
	}
	if r.Type() != ReplyString && r.Type() != ReplyStatus {
		return "", ErrType
	}
	return string(r.stringValue), nil
}

// Bytes returns string value of a reply as byte array
func (r *Reply) Bytes() ([]byte, error) {
	if r.Type() == ReplyNil {
		return nil, ErrNil
	}
	if r.Type() != ReplyString && r.Type() != ReplyStatus {
		return nil, ErrType
	}
	return r.stringValue, nil
}

// Int parses string value to int64. To get the integer value from
// command like INCR, use Integer()
func (r *Reply) Int() (int64, error) {
	s, err := r.String()
	if err != nil {
		return 0, err
	}
	x, err := strconv.ParseInt(s, 10, 64)
	if err != nil {
		return 0, ErrConvert
	}
	return x, nil
}

// Float parses string value to float64
func (r *Reply) Float() (float64, error) {
	s, err := r.String()
	if err != nil {
		return 0, err
	}
	return strconv.ParseFloat(s, 64)
}

// Bool parses string value to boolean.
// String "0" returns false while string "1" returns true. Other values
// will cause ErrConvert
func (r *Reply) Bool() (bool, error) {
	s, err := r.String()
	if err != nil {
		return false, err
	}
	if s == "1" || s == "true"{
		return true, nil
	} else if s == "0" || s == "false" {
		return false, nil
	} else {
		return false, ErrConvert
	}
}

// Integer returns integer value of a reply, such as INCR command
func (r *Reply) Integer() (int64, error) {
	if r.Type() != ReplyInteger {
		return 0, ErrType
	}
	return r.integerValue, nil
}

// Array returns array value of a reply
func (r *Reply) Array() ([]*Reply, error) {
	if r.Type() == ReplyNil {
		return nil, ErrNil
	}
	if r.Type() != ReplyArray {
		return nil, ErrType
	}
	return r.arrayValue, nil
}

// FixInt returns a fixed size int64
func (r *Reply) FixInt() (int64, error) {
	if r.Type() == ReplyNil {
		return 0, ErrNil
	}
	if r.Type() != ReplyString {
		return 0, ErrType
	}
	return ToFixInt(r.stringValue)
}

// VarInt returns a base-128 encoded int64
func (r *Reply) VarInt() (int64, error) {
	if r.Type() == ReplyNil {
		return 0, ErrNil
	}
	if r.Type() != ReplyString {
		return 0, ErrType
	}
	return ToVarInt(r.stringValue)
}

// Slice parses the reply to a slice. The element of the destination slice
// must be integer, float, boolean, string, []byte, FixInt, VarInt, or a Pair
func (r *Reply) Slice(s interface{}) error {
	if r.Type() == ReplyNil {
		return ErrNil
	}
	if r.Type() != ReplyArray {
		return ErrType
	}
	if len(r.arrayValue) == 0 {
		return nil
	}
	switch s := s.(type) {
	case *[]int:
		*s = make([]int, len(r.arrayValue))
		for i := range r.arrayValue {
			x, err := r.arrayValue[i].Int()
			if err != nil && err != ErrNil {
				return err
			}
			(*s)[i] = int(x)
		}
		return nil
	case *[]int64:
		*s = make([]int64, len(r.arrayValue))
		for i := range r.arrayValue {
			x, err := r.arrayValue[i].Int()
			if err != nil && err != ErrNil {
				return err
			}
			(*s)[i] = x
		}
		return nil
	case *[]float64:
		*s = make([]float64, len(r.arrayValue))
		for i := range r.arrayValue {
			x, err := r.arrayValue[i].Float()
			if err != nil && err != ErrNil {
				return err
			}
			(*s)[i] = x
		}
		return nil
	case *[]bool:
		*s = make([]bool, len(r.arrayValue))
		for i := range r.arrayValue {
			x, err := r.arrayValue[i].Bool()
			if err != nil && err != ErrNil {
				return err
			}
			(*s)[i] = x
		}
		return nil
	case *[]string:
		*s = make([]string, len(r.arrayValue))
		for i := range r.arrayValue {
			x, err := r.arrayValue[i].String()
			if err != nil && err != ErrNil {
				return err
			}
			(*s)[i] = x
		}
		return nil
	case *[][]byte:
		*s = make([][]byte, len(r.arrayValue))
		for i := range r.arrayValue {
			x, err := r.arrayValue[i].Bytes()
			if err != nil && err != ErrNil {
				return err
			}
			(*s)[i] = x
		}
		return nil
	case *[]FixInt:
		*s = make([]FixInt, len(r.arrayValue))
		for i := range r.arrayValue {
			x, err := r.arrayValue[i].FixInt()
			if err != nil && err != ErrNil {
				return err
			}
			(*s)[i] = FixInt(x)
		}
		return nil
	case *[]VarInt:
		*s = make([]VarInt, len(r.arrayValue))
		for i := range r.arrayValue {
			x, err := r.arrayValue[i].VarInt()
			if err != nil && err != ErrNil {
				return err
			}
			(*s)[i] = VarInt(x)
		}
		return nil
	case *[]*Pair:
		if len(r.arrayValue)%2 != 0 {
			return ErrType
		}
		*s = make([]*Pair, len(r.arrayValue)/2)
		for i := 0; i < len(r.arrayValue)/2; i++ {
			first, err := r.arrayValue[2*i].Bytes()
			if err != nil && err != ErrNil {
				return err
			}
			second, err := r.arrayValue[2*i+1].Bytes()
			if err != nil && err != ErrNil {
				return err
			}
			(*s)[i] = &Pair{first, second}
		}
		return nil
	default:
		return ErrConvert
	}
	return nil
}

// Map converts the reply into a map[string]string.
// It will return error unless the reply is an array reply from HGETALL,
// or SENTINEL master
func (r *Reply) Map() (map[string]string, error) {
	if r.IsNil() {
		return nil, ErrNil
	}
	if !r.IsArray() {
		return nil, ErrType
	}
	if len(r.arrayValue)%2 != 0 {
		return nil, ErrType
	}
	m := make(map[string]string)
	for i := 0; i < len(r.arrayValue)/2; i++ {
		first, err := r.arrayValue[2*i].String()
		if err != nil {
			continue
		}
		second, err := r.arrayValue[2*i+1].String()
		if err != nil && err != ErrNil {
			continue
		}
		m[first] = second
	}
	return m, nil
}

// Error returns error message
func (r *Reply) Error() (string, error) {
	if r.Type() != ReplyError {
		return "", ErrType
	}
	return string(r.stringValue), nil
}

// IsNil checks if reply is nil or not
func (r *Reply) IsNil() bool {
	return r.Type() == ReplyNil
}

// IsStatus checks if reply is a status or not
func (r *Reply) IsStatus() bool {
	return r.Type() == ReplyStatus
}

// IsOk checks if reply is a status reply with "OK" response
func (r *Reply) IsOk() bool {
	return r == okReply
}

// IsString checks if reply is a string reply or not
func (r *Reply) IsString() bool {
	return r.Type() == ReplyString
}

// IsInteger checks if reply is a integer reply or not
func (r *Reply) IsInteger() bool {
	return r.Type() == ReplyInteger
}

// IsArray checks if reply is an array reply or not
func (r *Reply) IsArray() bool {
	return r.Type() == ReplyArray
}

// IsError checks if reply is an error reply or not
func (r *Reply) IsError() bool {
	return r.Type() == ReplyError
}

// Receive safely read a reply from conn
func Receive(conn *Conn) (r *Reply, err error) {
	conn.Lock()
	defer func() {
		if err != nil {
			conn.state = connStateNotConnected
			conn.Unlock()
                        conn.fail()
                } else {
			conn.Unlock()
                }
	}()
	if conn.RequestTimeout != 0 {
		conn.tcpConn.SetReadDeadline(time.Now().Add(conn.RequestTimeout))
	}
	return readReply(conn)
}

// Motivated by redigo. Good job, man
func readReply(conn *Conn) (*Reply, error) {
	line, err := readLine(conn)
	if err != nil {
		return nil, err
	}
	if len(line) == 0 {
		return nil, ErrRead
	}
	switch line[0] {
	case '+':
		switch {
		case len(line) == 3 && line[1] == 'O' && line[2] == 'K':
			return okReply, nil
		case len(line) == 5 && line[1] == 'P' && line[2] == 'O' && line[3] == 'N' && line[4] == 'G':
			return pongReply, nil
		case len(line) == 7 && line[1] == 'Q' && line[2] == 'U' && line[3] == 'E' &&
			line[4] == 'U' && line[5] == 'E' && line[6] == 'D':
			return queuedReply, nil
		default:
			return &Reply{
				replyType:   ReplyStatus,
				stringValue: line[1:],
			}, nil

		}
	case '-':
		return &Reply{
			replyType:   ReplyError,
			stringValue: line[1:],
		}, nil
	case ':':
		intValue, err := strconv.ParseInt(string(line[1:]), 10, 64)
		if err != nil {
			return nil, ErrRead
		}
		return &Reply{
			replyType:    ReplyInteger,
			integerValue: intValue,
		}, nil
	case '$':
		l, err := strconv.ParseInt(string(line[1:]), 10, 64)
		if err != nil {
			return nil, ErrRead
		}
		if l < 0 {
			return &Reply{
				replyType: ReplyNil,
			}, nil
		}
		b := make([]byte, l)
		_, err = io.ReadFull(conn.rb, b)
		if err != nil {
			return nil, ErrRead
		}
		line, err = readLine(conn)
		if err != nil || len(line) != 0 {
			return nil, ErrRead
		}
		return &Reply{
			replyType:   ReplyString,
			stringValue: b,
		}, nil
	case '*':
		l, err := strconv.ParseInt(string(line[1:]), 10, 64)
		if err != nil {
			return nil, ErrRead
		}
		if l < 0 {
			return &Reply{
				replyType: ReplyNil,
			}, nil
		}
		replyArray := make([]*Reply, l)
		for i := range replyArray {
			replyArray[i], err = readReply(conn)
			if err != nil {
				return nil, err
			}
		}
		return &Reply{
			replyType:  ReplyArray,
			arrayValue: replyArray,
		}, nil
	default:
		return nil, ErrRead
	}
}

func readLine(conn *Conn) ([]byte, error) {
	b, err := conn.rb.ReadSlice('\n')
	if err != nil {
		return nil, ErrRead
	}
	i := len(b) - 2
	if i < 0 || b[i] != '\r' {
		return nil, ErrRead
	}
	return b[:i], nil
}

var (
	okReply = &Reply{
		replyType:   ReplyStatus,
		stringValue: []byte{'O', 'K'},
	}
	pongReply = &Reply{
		replyType:   ReplyStatus,
		stringValue: []byte{'P', 'O', 'N', 'G'},
	}
	queuedReply = &Reply{
		replyType:   ReplyStatus,
		stringValue: []byte{'Q', 'U', 'E', 'U', 'E', 'D'},
	}
)
<commit_msg>Fix Int() and Integer()<commit_after>package gore

import (
	"io"
	"strconv"
	"time"
)

// Reply type, similar to Hiredis
const (
	ReplyString  = 1
	ReplyArray   = 2
	ReplyInteger = 3
	ReplyNil     = 4
	ReplyStatus  = 5
	ReplyError   = 6
)

// Pair holds a pair of value, such as reply from HGETALL or ZRANGE [WITHSCORES]
type Pair struct {
	First  []byte
	Second []byte
}

// Reply holds redis reply
type Reply struct {
	replyType    int
	integerValue int64
	stringValue  []byte
	arrayValue   []*Reply
}

// Type returns reply type
func (r *Reply) Type() int {
	return r.replyType
}

// String returns string value of a reply
func (r *Reply) String() (string, error) {
	if r.Type() == ReplyNil {
		return "", ErrNil
	}
	if r.Type() != ReplyString && r.Type() != ReplyStatus {
		return "", ErrType
	}
	return string(r.stringValue), nil
}

// Bytes returns string value of a reply as byte array
func (r *Reply) Bytes() ([]byte, error) {
	if r.Type() == ReplyNil {
		return nil, ErrNil
	}
	if r.Type() != ReplyString && r.Type() != ReplyStatus {
		return nil, ErrType
	}
	return r.stringValue, nil
}

// Int returns integer value (for example from INCR) or convert string value to integer if possible
func (r *Reply) Int() (int64, error) {
	if r.Type() == ReplyInteger {
		return r.integerValue, nil
	}
	s, err := r.String()
	if err != nil {
		return 0, err
	}
	x, err := strconv.ParseInt(s, 10, 64)
	if err != nil {
		return 0, ErrConvert
	}
	return x, nil
}

// Float parses string value to float64
func (r *Reply) Float() (float64, error) {
	s, err := r.String()
	if err != nil {
		return 0, err
	}
	return strconv.ParseFloat(s, 64)
}

// Bool parses string value to boolean.
// String "0" returns false while string "1" returns true. Other values
// will cause ErrConvert
func (r *Reply) Bool() (bool, error) {
	s, err := r.String()
	if err != nil {
		return false, err
	}
	if s == "1" || s == "true"{
		return true, nil
	} else if s == "0" || s == "false" {
		return false, nil
	} else {
		return false, ErrConvert
	}
}

// Integer returns integer value (for example from INCR) or convert string value to integer if possible
func (r *Reply) Integer() (int64, error) {
	return r.Int()
}

// Array returns array value of a reply
func (r *Reply) Array() ([]*Reply, error) {
	if r.Type() == ReplyNil {
		return nil, ErrNil
	}
	if r.Type() != ReplyArray {
		return nil, ErrType
	}
	return r.arrayValue, nil
}

// FixInt returns a fixed size int64
func (r *Reply) FixInt() (int64, error) {
	if r.Type() == ReplyNil {
		return 0, ErrNil
	}
	if r.Type() != ReplyString {
		return 0, ErrType
	}
	return ToFixInt(r.stringValue)
}

// VarInt returns a base-128 encoded int64
func (r *Reply) VarInt() (int64, error) {
	if r.Type() == ReplyNil {
		return 0, ErrNil
	}
	if r.Type() != ReplyString {
		return 0, ErrType
	}
	return ToVarInt(r.stringValue)
}

// Slice parses the reply to a slice. The element of the destination slice
// must be integer, float, boolean, string, []byte, FixInt, VarInt, or a Pair
func (r *Reply) Slice(s interface{}) error {
	if r.Type() == ReplyNil {
		return ErrNil
	}
	if r.Type() != ReplyArray {
		return ErrType
	}
	if len(r.arrayValue) == 0 {
		return nil
	}
	switch s := s.(type) {
	case *[]int:
		*s = make([]int, len(r.arrayValue))
		for i := range r.arrayValue {
			x, err := r.arrayValue[i].Int()
			if err != nil && err != ErrNil {
				return err
			}
			(*s)[i] = int(x)
		}
		return nil
	case *[]int64:
		*s = make([]int64, len(r.arrayValue))
		for i := range r.arrayValue {
			x, err := r.arrayValue[i].Int()
			if err != nil && err != ErrNil {
				return err
			}
			(*s)[i] = x
		}
		return nil
	case *[]float64:
		*s = make([]float64, len(r.arrayValue))
		for i := range r.arrayValue {
			x, err := r.arrayValue[i].Float()
			if err != nil && err != ErrNil {
				return err
			}
			(*s)[i] = x
		}
		return nil
	case *[]bool:
		*s = make([]bool, len(r.arrayValue))
		for i := range r.arrayValue {
			x, err := r.arrayValue[i].Bool()
			if err != nil && err != ErrNil {
				return err
			}
			(*s)[i] = x
		}
		return nil
	case *[]string:
		*s = make([]string, len(r.arrayValue))
		for i := range r.arrayValue {
			x, err := r.arrayValue[i].String()
			if err != nil && err != ErrNil {
				return err
			}
			(*s)[i] = x
		}
		return nil
	case *[][]byte:
		*s = make([][]byte, len(r.arrayValue))
		for i := range r.arrayValue {
			x, err := r.arrayValue[i].Bytes()
			if err != nil && err != ErrNil {
				return err
			}
			(*s)[i] = x
		}
		return nil
	case *[]FixInt:
		*s = make([]FixInt, len(r.arrayValue))
		for i := range r.arrayValue {
			x, err := r.arrayValue[i].FixInt()
			if err != nil && err != ErrNil {
				return err
			}
			(*s)[i] = FixInt(x)
		}
		return nil
	case *[]VarInt:
		*s = make([]VarInt, len(r.arrayValue))
		for i := range r.arrayValue {
			x, err := r.arrayValue[i].VarInt()
			if err != nil && err != ErrNil {
				return err
			}
			(*s)[i] = VarInt(x)
		}
		return nil
	case *[]*Pair:
		if len(r.arrayValue)%2 != 0 {
			return ErrType
		}
		*s = make([]*Pair, len(r.arrayValue)/2)
		for i := 0; i < len(r.arrayValue)/2; i++ {
			first, err := r.arrayValue[2*i].Bytes()
			if err != nil && err != ErrNil {
				return err
			}
			second, err := r.arrayValue[2*i+1].Bytes()
			if err != nil && err != ErrNil {
				return err
			}
			(*s)[i] = &Pair{first, second}
		}
		return nil
	default:
		return ErrConvert
	}
	return nil
}

// Map converts the reply into a map[string]string.
// It will return error unless the reply is an array reply from HGETALL,
// or SENTINEL master
func (r *Reply) Map() (map[string]string, error) {
	if r.IsNil() {
		return nil, ErrNil
	}
	if !r.IsArray() {
		return nil, ErrType
	}
	if len(r.arrayValue)%2 != 0 {
		return nil, ErrType
	}
	m := make(map[string]string)
	for i := 0; i < len(r.arrayValue)/2; i++ {
		first, err := r.arrayValue[2*i].String()
		if err != nil {
			continue
		}
		second, err := r.arrayValue[2*i+1].String()
		if err != nil && err != ErrNil {
			continue
		}
		m[first] = second
	}
	return m, nil
}

// Error returns error message
func (r *Reply) Error() (string, error) {
	if r.Type() != ReplyError {
		return "", ErrType
	}
	return string(r.stringValue), nil
}

// IsNil checks if reply is nil or not
func (r *Reply) IsNil() bool {
	return r.Type() == ReplyNil
}

// IsStatus checks if reply is a status or not
func (r *Reply) IsStatus() bool {
	return r.Type() == ReplyStatus
}

// IsOk checks if reply is a status reply with "OK" response
func (r *Reply) IsOk() bool {
	return r == okReply
}

// IsString checks if reply is a string reply or not
func (r *Reply) IsString() bool {
	return r.Type() == ReplyString
}

// IsInteger checks if reply is a integer reply or not
func (r *Reply) IsInteger() bool {
	return r.Type() == ReplyInteger
}

// IsArray checks if reply is an array reply or not
func (r *Reply) IsArray() bool {
	return r.Type() == ReplyArray
}

// IsError checks if reply is an error reply or not
func (r *Reply) IsError() bool {
	return r.Type() == ReplyError
}

// Receive safely read a reply from conn
func Receive(conn *Conn) (r *Reply, err error) {
	conn.Lock()
	defer func() {
		if err != nil {
			conn.state = connStateNotConnected
			conn.Unlock()
                        conn.fail()
                } else {
			conn.Unlock()
                }
	}()
	if conn.RequestTimeout != 0 {
		conn.tcpConn.SetReadDeadline(time.Now().Add(conn.RequestTimeout))
	}
	return readReply(conn)
}

// Motivated by redigo. Good job, man
func readReply(conn *Conn) (*Reply, error) {
	line, err := readLine(conn)
	if err != nil {
		return nil, err
	}
	if len(line) == 0 {
		return nil, ErrRead
	}
	switch line[0] {
	case '+':
		switch {
		case len(line) == 3 && line[1] == 'O' && line[2] == 'K':
			return okReply, nil
		case len(line) == 5 && line[1] == 'P' && line[2] == 'O' && line[3] == 'N' && line[4] == 'G':
			return pongReply, nil
		case len(line) == 7 && line[1] == 'Q' && line[2] == 'U' && line[3] == 'E' &&
			line[4] == 'U' && line[5] == 'E' && line[6] == 'D':
			return queuedReply, nil
		default:
			return &Reply{
				replyType:   ReplyStatus,
				stringValue: line[1:],
			}, nil

		}
	case '-':
		return &Reply{
			replyType:   ReplyError,
			stringValue: line[1:],
		}, nil
	case ':':
		intValue, err := strconv.ParseInt(string(line[1:]), 10, 64)
		if err != nil {
			return nil, ErrRead
		}
		return &Reply{
			replyType:    ReplyInteger,
			integerValue: intValue,
		}, nil
	case '$':
		l, err := strconv.ParseInt(string(line[1:]), 10, 64)
		if err != nil {
			return nil, ErrRead
		}
		if l < 0 {
			return &Reply{
				replyType: ReplyNil,
			}, nil
		}
		b := make([]byte, l)
		_, err = io.ReadFull(conn.rb, b)
		if err != nil {
			return nil, ErrRead
		}
		line, err = readLine(conn)
		if err != nil || len(line) != 0 {
			return nil, ErrRead
		}
		return &Reply{
			replyType:   ReplyString,
			stringValue: b,
		}, nil
	case '*':
		l, err := strconv.ParseInt(string(line[1:]), 10, 64)
		if err != nil {
			return nil, ErrRead
		}
		if l < 0 {
			return &Reply{
				replyType: ReplyNil,
			}, nil
		}
		replyArray := make([]*Reply, l)
		for i := range replyArray {
			replyArray[i], err = readReply(conn)
			if err != nil {
				return nil, err
			}
		}
		return &Reply{
			replyType:  ReplyArray,
			arrayValue: replyArray,
		}, nil
	default:
		return nil, ErrRead
	}
}

func readLine(conn *Conn) ([]byte, error) {
	b, err := conn.rb.ReadSlice('\n')
	if err != nil {
		return nil, ErrRead
	}
	i := len(b) - 2
	if i < 0 || b[i] != '\r' {
		return nil, ErrRead
	}
	return b[:i], nil
}

var (
	okReply = &Reply{
		replyType:   ReplyStatus,
		stringValue: []byte{'O', 'K'},
	}
	pongReply = &Reply{
		replyType:   ReplyStatus,
		stringValue: []byte{'P', 'O', 'N', 'G'},
	}
	queuedReply = &Reply{
		replyType:   ReplyStatus,
		stringValue: []byte{'Q', 'U', 'E', 'U', 'E', 'D'},
	}
)
<|endoftext|>


<commit_before>/*
 * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015, 2016 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package minio

import (
	"net"
	"net/http"
	"net/url"
	"strings"
	"time"
)

// MaxRetry is the maximum number of retries before stopping.
var MaxRetry = 5

// MaxJitter will randomize over the full exponential backoff time
const MaxJitter = 1.0

// NoJitter disables the use of jitter for randomizing the exponential backoff time
const NoJitter = 0.0

// newRetryTimer creates a timer with exponentially increasing delays
// until the maximum retry attempts are reached.
func (c Client) newRetryTimer(maxRetry int, unit time.Duration, cap time.Duration, jitter float64, doneCh chan struct{}) <-chan int {
	attemptCh := make(chan int)

	// computes the exponential backoff duration according to
	// https://www.awsarchitectureblog.com/2015/03/backoff.html
	exponentialBackoffWait := func(attempt int) time.Duration {
		// normalize jitter to the range [0, 1.0]
		if jitter < NoJitter {
			jitter = NoJitter
		}
		if jitter > MaxJitter {
			jitter = MaxJitter
		}

		//sleep = random_between(0, min(cap, base * 2 ** attempt))
		sleep := unit * time.Duration(1<<uint(attempt))
		if sleep > cap {
			sleep = cap
		}
		if jitter != NoJitter {
			sleep -= time.Duration(c.random.Float64() * float64(sleep) * jitter)
		}
		return sleep
	}

	go func() {
		defer close(attemptCh)
		for i := 0; i < maxRetry; i++ {
			select {
			// Attempts start from 1.
			case attemptCh <- i + 1:
			case <-doneCh:
				// Stop the routine.
				return
			}
			time.Sleep(exponentialBackoffWait(i))
		}
	}()
	return attemptCh
}

// isNetErrorRetryable - is network error retryable.
func isNetErrorRetryable(err error) bool {
	switch err.(type) {
	case net.Error:
		switch err.(type) {
		case *net.DNSError, *net.OpError, net.UnknownNetworkError:
			return true
		case *url.Error:
			// For a URL error, where it replies back "connection closed"
			// retry again.
			if strings.Contains(err.Error(), "Connection closed by foreign host") {
				return true
			}
		default:
			if strings.Contains(err.Error(), "net/http: TLS handshake timeout") {
				// If error is - tlsHandshakeTimeoutError, retry.
				return true
			} else if strings.Contains(err.Error(), "i/o timeout") {
				// If error is - tcp timeoutError, retry.
				return true
			}
		}
	}
	return false
}

// List of AWS S3 error codes which are retryable.
var retryableS3Codes = map[string]struct{}{
	"RequestError":          {},
	"RequestTimeout":        {},
	"Throttling":            {},
	"ThrottlingException":   {},
	"RequestLimitExceeded":  {},
	"RequestThrottled":      {},
	"InternalError":         {},
	"ExpiredToken":          {},
	"ExpiredTokenException": {},
	// Add more AWS S3 codes here.
}

// isS3CodeRetryable - is s3 error code retryable.
func isS3CodeRetryable(s3Code string) (ok bool) {
	_, ok = retryableS3Codes[s3Code]
	return ok
}

// List of HTTP status codes which are retryable.
var retryableHTTPStatusCodes = map[int]struct{}{
	429: {}, // http.StatusTooManyRequests is not part of the Go 1.5 library, yet
	http.StatusInternalServerError: {},
	http.StatusBadGateway:          {},
	http.StatusServiceUnavailable:  {},
	// Add more HTTP status codes here.
}

// isHTTPStatusRetryable - is HTTP error code retryable.
func isHTTPStatusRetryable(httpStatusCode int) (ok bool) {
	_, ok = retryableHTTPStatusCodes[httpStatusCode]
	return ok
}
<commit_msg>api/retry: Support connection retry for Dial timeout. (#622)<commit_after>/*
 * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015, 2016 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package minio

import (
	"net"
	"net/http"
	"net/url"
	"strings"
	"time"
)

// MaxRetry is the maximum number of retries before stopping.
var MaxRetry = 5

// MaxJitter will randomize over the full exponential backoff time
const MaxJitter = 1.0

// NoJitter disables the use of jitter for randomizing the exponential backoff time
const NoJitter = 0.0

// newRetryTimer creates a timer with exponentially increasing delays
// until the maximum retry attempts are reached.
func (c Client) newRetryTimer(maxRetry int, unit time.Duration, cap time.Duration, jitter float64, doneCh chan struct{}) <-chan int {
	attemptCh := make(chan int)

	// computes the exponential backoff duration according to
	// https://www.awsarchitectureblog.com/2015/03/backoff.html
	exponentialBackoffWait := func(attempt int) time.Duration {
		// normalize jitter to the range [0, 1.0]
		if jitter < NoJitter {
			jitter = NoJitter
		}
		if jitter > MaxJitter {
			jitter = MaxJitter
		}

		//sleep = random_between(0, min(cap, base * 2 ** attempt))
		sleep := unit * time.Duration(1<<uint(attempt))
		if sleep > cap {
			sleep = cap
		}
		if jitter != NoJitter {
			sleep -= time.Duration(c.random.Float64() * float64(sleep) * jitter)
		}
		return sleep
	}

	go func() {
		defer close(attemptCh)
		for i := 0; i < maxRetry; i++ {
			select {
			// Attempts start from 1.
			case attemptCh <- i + 1:
			case <-doneCh:
				// Stop the routine.
				return
			}
			time.Sleep(exponentialBackoffWait(i))
		}
	}()
	return attemptCh
}

// isNetErrorRetryable - is network error retryable.
func isNetErrorRetryable(err error) bool {
	if err == nil {
		return false
	}
	switch err.(type) {
	case net.Error:
		switch err.(type) {
		case *net.DNSError, *net.OpError, net.UnknownNetworkError:
			return true
		case *url.Error:
			// For a URL error, where it replies back "connection closed"
			// retry again.
			if strings.Contains(err.Error(), "Connection closed by foreign host") {
				return true
			}
		default:
			if strings.Contains(err.Error(), "net/http: TLS handshake timeout") {
				// If error is - tlsHandshakeTimeoutError, retry.
				return true
			} else if strings.Contains(err.Error(), "i/o timeout") {
				// If error is - tcp timeoutError, retry.
				return true
			} else if strings.Contains(err.Error(), "connection timed out") {
				// If err is a net.Dial timeout, retry.
				return true
			}
		}
	}
	return false
}

// List of AWS S3 error codes which are retryable.
var retryableS3Codes = map[string]struct{}{
	"RequestError":          {},
	"RequestTimeout":        {},
	"Throttling":            {},
	"ThrottlingException":   {},
	"RequestLimitExceeded":  {},
	"RequestThrottled":      {},
	"InternalError":         {},
	"ExpiredToken":          {},
	"ExpiredTokenException": {},
	// Add more AWS S3 codes here.
}

// isS3CodeRetryable - is s3 error code retryable.
func isS3CodeRetryable(s3Code string) (ok bool) {
	_, ok = retryableS3Codes[s3Code]
	return ok
}

// List of HTTP status codes which are retryable.
var retryableHTTPStatusCodes = map[int]struct{}{
	429: {}, // http.StatusTooManyRequests is not part of the Go 1.5 library, yet
	http.StatusInternalServerError: {},
	http.StatusBadGateway:          {},
	http.StatusServiceUnavailable:  {},
	// Add more HTTP status codes here.
}

// isHTTPStatusRetryable - is HTTP error code retryable.
func isHTTPStatusRetryable(httpStatusCode int) (ok bool) {
	_, ok = retryableHTTPStatusCodes[httpStatusCode]
	return ok
}
<|endoftext|>


<commit_before>/*
 * MinIO Go Library for Amazon S3 Compatible Cloud Storage
 * Copyright 2015-2017 MinIO, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package minio

import (
	"net"
	"net/http"
	"net/url"
	"strings"
	"time"
)

// MaxRetry is the maximum number of retries before stopping.
var MaxRetry = 10

// MaxJitter will randomize over the full exponential backoff time
const MaxJitter = 1.0

// NoJitter disables the use of jitter for randomizing the exponential backoff time
const NoJitter = 0.0

// DefaultRetryUnit - default unit multiplicative per retry.
// defaults to 1 second.
const DefaultRetryUnit = time.Second

// DefaultRetryCap - Each retry attempt never waits no longer than
// this maximum time duration.
const DefaultRetryCap = time.Second * 30

// newRetryTimer creates a timer with exponentially increasing
// delays until the maximum retry attempts are reached.
func (c Client) newRetryTimer(maxRetry int, unit time.Duration, cap time.Duration, jitter float64, doneCh chan struct{}) <-chan int {
	attemptCh := make(chan int)

	// computes the exponential backoff duration according to
	// https://www.awsarchitectureblog.com/2015/03/backoff.html
	exponentialBackoffWait := func(attempt int) time.Duration {
		// normalize jitter to the range [0, 1.0]
		if jitter < NoJitter {
			jitter = NoJitter
		}
		if jitter > MaxJitter {
			jitter = MaxJitter
		}

		//sleep = random_between(0, min(cap, base * 2 ** attempt))
		sleep := unit * time.Duration(1<<uint(attempt))
		if sleep > cap {
			sleep = cap
		}
		if jitter != NoJitter {
			sleep -= time.Duration(c.random.Float64() * float64(sleep) * jitter)
		}
		return sleep
	}

	go func() {
		defer close(attemptCh)
		for i := 0; i < maxRetry; i++ {
			select {
			// Attempts start from 1.
			case attemptCh <- i + 1:
			case <-doneCh:
				// Stop the routine.
				return
			}
			time.Sleep(exponentialBackoffWait(i))
		}
	}()
	return attemptCh
}

// isHTTPReqErrorRetryable - is http requests error retryable, such
// as i/o timeout, connection broken etc..
func isHTTPReqErrorRetryable(err error) bool {
	if err == nil {
		return false
	}
	switch e := err.(type) {
	case *url.Error:
		switch e.Err.(type) {
		case *net.DNSError, *net.OpError, net.UnknownNetworkError:
			return true
		}
		if strings.Contains(err.Error(), "Connection closed by foreign host") {
			return true
		} else if strings.Contains(err.Error(), "net/http: TLS handshake timeout") {
			// If error is - tlsHandshakeTimeoutError, retry.
			return true
		} else if strings.Contains(err.Error(), "i/o timeout") {
			// If error is - tcp timeoutError, retry.
			return true
		} else if strings.Contains(err.Error(), "connection timed out") {
			// If err is a net.Dial timeout, retry.
			return true
		} else if strings.Contains(err.Error(), "net/http: HTTP/1.x transport connection broken") {
			// If error is transport connection broken, retry.
			return true
		}
	}
	return false
}

// List of AWS S3 error codes which are retryable.
var retryableS3Codes = map[string]struct{}{
	"RequestError":          {},
	"RequestTimeout":        {},
	"Throttling":            {},
	"ThrottlingException":   {},
	"RequestLimitExceeded":  {},
	"RequestThrottled":      {},
	"InternalError":         {},
	"ExpiredToken":          {},
	"ExpiredTokenException": {},
	"SlowDown":              {},
	// Add more AWS S3 codes here.
}

// isS3CodeRetryable - is s3 error code retryable.
func isS3CodeRetryable(s3Code string) (ok bool) {
	_, ok = retryableS3Codes[s3Code]
	return ok
}

// List of HTTP status codes which are retryable.
var retryableHTTPStatusCodes = map[int]struct{}{
	429:                            {}, // http.StatusTooManyRequests is not part of the Go 1.5 library, yet
	http.StatusInternalServerError: {},
	http.StatusBadGateway:          {},
	http.StatusServiceUnavailable:  {},
	// Add more HTTP status codes here.
}

// isHTTPStatusRetryable - is HTTP error code retryable.
func isHTTPStatusRetryable(httpStatusCode int) (ok bool) {
	_, ok = retryableHTTPStatusCodes[httpStatusCode]
	return ok
}
<commit_msg>Retry on 504 Gateway timeout http status code. (#1104)<commit_after>/*
 * MinIO Go Library for Amazon S3 Compatible Cloud Storage
 * Copyright 2015-2017 MinIO, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package minio

import (
	"net"
	"net/http"
	"net/url"
	"strings"
	"time"
)

// MaxRetry is the maximum number of retries before stopping.
var MaxRetry = 10

// MaxJitter will randomize over the full exponential backoff time
const MaxJitter = 1.0

// NoJitter disables the use of jitter for randomizing the exponential backoff time
const NoJitter = 0.0

// DefaultRetryUnit - default unit multiplicative per retry.
// defaults to 1 second.
const DefaultRetryUnit = time.Second

// DefaultRetryCap - Each retry attempt never waits no longer than
// this maximum time duration.
const DefaultRetryCap = time.Second * 30

// newRetryTimer creates a timer with exponentially increasing
// delays until the maximum retry attempts are reached.
func (c Client) newRetryTimer(maxRetry int, unit time.Duration, cap time.Duration, jitter float64, doneCh chan struct{}) <-chan int {
	attemptCh := make(chan int)

	// computes the exponential backoff duration according to
	// https://www.awsarchitectureblog.com/2015/03/backoff.html
	exponentialBackoffWait := func(attempt int) time.Duration {
		// normalize jitter to the range [0, 1.0]
		if jitter < NoJitter {
			jitter = NoJitter
		}
		if jitter > MaxJitter {
			jitter = MaxJitter
		}

		//sleep = random_between(0, min(cap, base * 2 ** attempt))
		sleep := unit * time.Duration(1<<uint(attempt))
		if sleep > cap {
			sleep = cap
		}
		if jitter != NoJitter {
			sleep -= time.Duration(c.random.Float64() * float64(sleep) * jitter)
		}
		return sleep
	}

	go func() {
		defer close(attemptCh)
		for i := 0; i < maxRetry; i++ {
			select {
			// Attempts start from 1.
			case attemptCh <- i + 1:
			case <-doneCh:
				// Stop the routine.
				return
			}
			time.Sleep(exponentialBackoffWait(i))
		}
	}()
	return attemptCh
}

// isHTTPReqErrorRetryable - is http requests error retryable, such
// as i/o timeout, connection broken etc..
func isHTTPReqErrorRetryable(err error) bool {
	if err == nil {
		return false
	}
	switch e := err.(type) {
	case *url.Error:
		switch e.Err.(type) {
		case *net.DNSError, *net.OpError, net.UnknownNetworkError:
			return true
		}
		if strings.Contains(err.Error(), "Connection closed by foreign host") {
			return true
		} else if strings.Contains(err.Error(), "net/http: TLS handshake timeout") {
			// If error is - tlsHandshakeTimeoutError, retry.
			return true
		} else if strings.Contains(err.Error(), "i/o timeout") {
			// If error is - tcp timeoutError, retry.
			return true
		} else if strings.Contains(err.Error(), "connection timed out") {
			// If err is a net.Dial timeout, retry.
			return true
		} else if strings.Contains(err.Error(), "net/http: HTTP/1.x transport connection broken") {
			// If error is transport connection broken, retry.
			return true
		}
	}
	return false
}

// List of AWS S3 error codes which are retryable.
var retryableS3Codes = map[string]struct{}{
	"RequestError":          {},
	"RequestTimeout":        {},
	"Throttling":            {},
	"ThrottlingException":   {},
	"RequestLimitExceeded":  {},
	"RequestThrottled":      {},
	"InternalError":         {},
	"ExpiredToken":          {},
	"ExpiredTokenException": {},
	"SlowDown":              {},
	// Add more AWS S3 codes here.
}

// isS3CodeRetryable - is s3 error code retryable.
func isS3CodeRetryable(s3Code string) (ok bool) {
	_, ok = retryableS3Codes[s3Code]
	return ok
}

// List of HTTP status codes which are retryable.
var retryableHTTPStatusCodes = map[int]struct{}{
	429:                            {}, // http.StatusTooManyRequests is not part of the Go 1.5 library, yet
	http.StatusInternalServerError: {},
	http.StatusBadGateway:          {},
	http.StatusServiceUnavailable:  {},
	http.StatusGatewayTimeout:      {},
	// Add more HTTP status codes here.
}

// isHTTPStatusRetryable - is HTTP error code retryable.
func isHTTPStatusRetryable(httpStatusCode int) (ok bool) {
	_, ok = retryableHTTPStatusCodes[httpStatusCode]
	return ok
}
<|endoftext|>