<commit_before>package pgx

import (
	"database/sql/driver"
	"math"
	"reflect"
	"time"

	"github.com/jackc/pgx/pgio"
	"github.com/jackc/pgx/pgtype"
)

const (
	copyData      = 'd'
	copyFail      = 'f'
	copyDone      = 'c'
	varHeaderSize = 4
)

type FieldDescription struct {
	Name            string
	Table           pgtype.OID
	AttributeNumber uint16
	DataType        pgtype.OID
	DataTypeSize    int16
	DataTypeName    string
	Modifier        int32
	FormatCode      int16
}

func (fd FieldDescription) Length() (int64, bool) {
	switch fd.DataType {
	case pgtype.TextOID, pgtype.ByteaOID:
		return math.MaxInt64, true
	case pgtype.VarcharOID, pgtype.BPCharArrayOID:
		return int64(fd.Modifier - varHeaderSize), true
	default:
		return 0, false
	}
}

func (fd FieldDescription) PrecisionScale() (precision, scale int64, ok bool) {
	switch fd.DataType {
	case pgtype.NumericOID:
		mod := fd.Modifier - varHeaderSize
		precision = int64((mod >> 16) & 0xffff)
		scale = int64(mod & 0xffff)
		return precision, scale, true
	default:
		return 0, 0, false
	}
}

func (fd FieldDescription) Type() reflect.Type {
	switch fd.DataType {
	case pgtype.Int8OID:
		return reflect.TypeOf(int64(0))
	case pgtype.Int4OID:
		return reflect.TypeOf(int32(0))
	case pgtype.Int2OID:
		return reflect.TypeOf(int16(0))
	case pgtype.VarcharOID, pgtype.BPCharArrayOID, pgtype.TextOID:
		return reflect.TypeOf("")
	case pgtype.BoolOID:
		return reflect.TypeOf(false)
	case pgtype.NumericOID:
		return reflect.TypeOf(float64(0))
	case pgtype.DateOID, pgtype.TimestampOID, pgtype.TimestamptzOID:
		return reflect.TypeOf(time.Time{})
	case pgtype.ByteaOID:
		return reflect.TypeOf([]byte(nil))
	default:
		return reflect.TypeOf(new(interface{})).Elem()
	}
}

// PgError represents an error reported by the PostgreSQL server. See
// http://www.postgresql.org/docs/9.3/static/protocol-error-fields.html for
// detailed field description.
type PgError struct {
	Severity         string
	Code             string
	Message          string
	Detail           string
	Hint             string
	Position         int32
	InternalPosition int32
	InternalQuery    string
	Where            string
	SchemaName       string
	TableName        string
	ColumnName       string
	DataTypeName     string
	ConstraintName   string
	File             string
	Line             int32
	Routine          string
}

func (pe PgError) Error() string {
	return pe.Severity + ": " + pe.Message + " (SQLSTATE " + pe.Code + ")"
}

// Notice represents a notice response message reported by the PostgreSQL
// server. Be aware that this is distinct from LISTEN/NOTIFY notification.
type Notice PgError

// appendParse appends a PostgreSQL wire protocol parse message to buf and returns it.
func appendParse(buf []byte, name string, query string, parameterOIDs []pgtype.OID) []byte {
	buf = append(buf, 'P')
	sp := len(buf)
	buf = pgio.AppendInt32(buf, -1)
	buf = append(buf, name...)
	buf = append(buf, 0)
	buf = append(buf, query...)
	buf = append(buf, 0)

	buf = pgio.AppendInt16(buf, int16(len(parameterOIDs)))
	for _, oid := range parameterOIDs {
		buf = pgio.AppendUint32(buf, uint32(oid))
	}
	pgio.SetInt32(buf[sp:], int32(len(buf[sp:])))

	return buf
}

// appendDescribe appends a PostgreSQL wire protocol describe message to buf and returns it.
func appendDescribe(buf []byte, objectType byte, name string) []byte {
	buf = append(buf, 'D')
	sp := len(buf)
	buf = pgio.AppendInt32(buf, -1)
	buf = append(buf, objectType)
	buf = append(buf, name...)
	buf = append(buf, 0)
	pgio.SetInt32(buf[sp:], int32(len(buf[sp:])))

	return buf
}

// appendSync appends a PostgreSQL wire protocol sync message to buf and returns it.
func appendSync(buf []byte) []byte {
	buf = append(buf, 'S')
	buf = pgio.AppendInt32(buf, 4)

	return buf
}

// appendBind appends a PostgreSQL wire protocol bind message to buf and returns it.
func appendBind(
	buf []byte,
	destinationPortal,
	preparedStatement string,
	connInfo *pgtype.ConnInfo,
	parameterOIDs []pgtype.OID,
	arguments []interface{},
	resultFormatCodes []int16,
) ([]byte, error) {
	buf = append(buf, 'B')
	sp := len(buf)
	buf = pgio.AppendInt32(buf, -1)
	buf = append(buf, destinationPortal...)
	buf = append(buf, 0)
	buf = append(buf, preparedStatement...)
	buf = append(buf, 0)

	var err error
	arguments, err = convertDriverValuers(arguments)
	if err != nil {
		return nil, err
	}

	buf = pgio.AppendInt16(buf, int16(len(parameterOIDs)))
	for i, oid := range parameterOIDs {
		buf = pgio.AppendInt16(buf, chooseParameterFormatCode(connInfo, oid, arguments[i]))
	}

	buf = pgio.AppendInt16(buf, int16(len(arguments)))
	for i, oid := range parameterOIDs {
		var err error
		buf, err = encodePreparedStatementArgument(connInfo, buf, oid, arguments[i])
		if err != nil {
			return nil, err
		}
	}

	buf = pgio.AppendInt16(buf, int16(len(resultFormatCodes)))
	for _, fc := range resultFormatCodes {
		buf = pgio.AppendInt16(buf, fc)
	}
	pgio.SetInt32(buf[sp:], int32(len(buf[sp:])))

	return buf, nil
}

func convertDriverValuers(args []interface{}) ([]interface{}, error) {
	for i, arg := range args {
		switch arg := arg.(type) {
		case pgtype.BinaryEncoder:
		case pgtype.TextEncoder:
		case driver.Valuer:
			v, err := callValuerValue(arg)
			if err != nil {
				return nil, err
			}
			args[i] = v
		}
	}
	return args, nil
}

// appendExecute appends a PostgreSQL wire protocol execute message to buf and returns it.
func appendExecute(buf []byte, portal string, maxRows uint32) []byte {
	buf = append(buf, 'E')
	sp := len(buf)
	buf = pgio.AppendInt32(buf, -1)

	buf = append(buf, portal...)
	buf = append(buf, 0)
	buf = pgio.AppendUint32(buf, maxRows)

	pgio.SetInt32(buf[sp:], int32(len(buf[sp:])))

	return buf
}

// appendQuery appends a PostgreSQL wire protocol query message to buf and returns it.
func appendQuery(buf []byte, query string) []byte {
	buf = append(buf, 'Q')
	sp := len(buf)
	buf = pgio.AppendInt32(buf, -1)
	buf = append(buf, query...)
	buf = append(buf, 0)
	pgio.SetInt32(buf[sp:], int32(len(buf[sp:])))

	return buf
}
<commit_msg>add float types to database/sql ColumnType ScanType()<commit_after>package pgx

import (
	"database/sql/driver"
	"math"
	"reflect"
	"time"

	"github.com/jackc/pgx/pgio"
	"github.com/jackc/pgx/pgtype"
)

const (
	copyData      = 'd'
	copyFail      = 'f'
	copyDone      = 'c'
	varHeaderSize = 4
)

type FieldDescription struct {
	Name            string
	Table           pgtype.OID
	AttributeNumber uint16
	DataType        pgtype.OID
	DataTypeSize    int16
	DataTypeName    string
	Modifier        int32
	FormatCode      int16
}

func (fd FieldDescription) Length() (int64, bool) {
	switch fd.DataType {
	case pgtype.TextOID, pgtype.ByteaOID:
		return math.MaxInt64, true
	case pgtype.VarcharOID, pgtype.BPCharArrayOID:
		return int64(fd.Modifier - varHeaderSize), true
	default:
		return 0, false
	}
}

func (fd FieldDescription) PrecisionScale() (precision, scale int64, ok bool) {
	switch fd.DataType {
	case pgtype.NumericOID:
		mod := fd.Modifier - varHeaderSize
		precision = int64((mod >> 16) & 0xffff)
		scale = int64(mod & 0xffff)
		return precision, scale, true
	default:
		return 0, 0, false
	}
}

func (fd FieldDescription) Type() reflect.Type {
	switch fd.DataType {
	case pgtype.Float8OID:
		return reflect.TypeOf(float64(0))
	case pgtype.Float4OID:
		return reflect.TypeOf(float32(0))
	case pgtype.Int8OID:
		return reflect.TypeOf(int64(0))
	case pgtype.Int4OID:
		return reflect.TypeOf(int32(0))
	case pgtype.Int2OID:
		return reflect.TypeOf(int16(0))
	case pgtype.VarcharOID, pgtype.BPCharArrayOID, pgtype.TextOID:
		return reflect.TypeOf("")
	case pgtype.BoolOID:
		return reflect.TypeOf(false)
	case pgtype.NumericOID:
		return reflect.TypeOf(float64(0))
	case pgtype.DateOID, pgtype.TimestampOID, pgtype.TimestamptzOID:
		return reflect.TypeOf(time.Time{})
	case pgtype.ByteaOID:
		return reflect.TypeOf([]byte(nil))
	default:
		return reflect.TypeOf(new(interface{})).Elem()
	}
}

// PgError represents an error reported by the PostgreSQL server. See
// http://www.postgresql.org/docs/9.3/static/protocol-error-fields.html for
// detailed field description.
type PgError struct {
	Severity         string
	Code             string
	Message          string
	Detail           string
	Hint             string
	Position         int32
	InternalPosition int32
	InternalQuery    string
	Where            string
	SchemaName       string
	TableName        string
	ColumnName       string
	DataTypeName     string
	ConstraintName   string
	File             string
	Line             int32
	Routine          string
}

func (pe PgError) Error() string {
	return pe.Severity + ": " + pe.Message + " (SQLSTATE " + pe.Code + ")"
}

// Notice represents a notice response message reported by the PostgreSQL
// server. Be aware that this is distinct from LISTEN/NOTIFY notification.
type Notice PgError

// appendParse appends a PostgreSQL wire protocol parse message to buf and returns it.
func appendParse(buf []byte, name string, query string, parameterOIDs []pgtype.OID) []byte {
	buf = append(buf, 'P')
	sp := len(buf)
	buf = pgio.AppendInt32(buf, -1)
	buf = append(buf, name...)
	buf = append(buf, 0)
	buf = append(buf, query...)
	buf = append(buf, 0)

	buf = pgio.AppendInt16(buf, int16(len(parameterOIDs)))
	for _, oid := range parameterOIDs {
		buf = pgio.AppendUint32(buf, uint32(oid))
	}
	pgio.SetInt32(buf[sp:], int32(len(buf[sp:])))

	return buf
}

// appendDescribe appends a PostgreSQL wire protocol describe message to buf and returns it.
func appendDescribe(buf []byte, objectType byte, name string) []byte {
	buf = append(buf, 'D')
	sp := len(buf)
	buf = pgio.AppendInt32(buf, -1)
	buf = append(buf, objectType)
	buf = append(buf, name...)
	buf = append(buf, 0)
	pgio.SetInt32(buf[sp:], int32(len(buf[sp:])))

	return buf
}

// appendSync appends a PostgreSQL wire protocol sync message to buf and returns it.
func appendSync(buf []byte) []byte {
	buf = append(buf, 'S')
	buf = pgio.AppendInt32(buf, 4)

	return buf
}

// appendBind appends a PostgreSQL wire protocol bind message to buf and returns it.
func appendBind(
	buf []byte,
	destinationPortal,
	preparedStatement string,
	connInfo *pgtype.ConnInfo,
	parameterOIDs []pgtype.OID,
	arguments []interface{},
	resultFormatCodes []int16,
) ([]byte, error) {
	buf = append(buf, 'B')
	sp := len(buf)
	buf = pgio.AppendInt32(buf, -1)
	buf = append(buf, destinationPortal...)
	buf = append(buf, 0)
	buf = append(buf, preparedStatement...)
	buf = append(buf, 0)

	var err error
	arguments, err = convertDriverValuers(arguments)
	if err != nil {
		return nil, err
	}

	buf = pgio.AppendInt16(buf, int16(len(parameterOIDs)))
	for i, oid := range parameterOIDs {
		buf = pgio.AppendInt16(buf, chooseParameterFormatCode(connInfo, oid, arguments[i]))
	}

	buf = pgio.AppendInt16(buf, int16(len(arguments)))
	for i, oid := range parameterOIDs {
		var err error
		buf, err = encodePreparedStatementArgument(connInfo, buf, oid, arguments[i])
		if err != nil {
			return nil, err
		}
	}

	buf = pgio.AppendInt16(buf, int16(len(resultFormatCodes)))
	for _, fc := range resultFormatCodes {
		buf = pgio.AppendInt16(buf, fc)
	}
	pgio.SetInt32(buf[sp:], int32(len(buf[sp:])))

	return buf, nil
}

func convertDriverValuers(args []interface{}) ([]interface{}, error) {
	for i, arg := range args {
		switch arg := arg.(type) {
		case pgtype.BinaryEncoder:
		case pgtype.TextEncoder:
		case driver.Valuer:
			v, err := callValuerValue(arg)
			if err != nil {
				return nil, err
			}
			args[i] = v
		}
	}
	return args, nil
}

// appendExecute appends a PostgreSQL wire protocol execute message to buf and returns it.
func appendExecute(buf []byte, portal string, maxRows uint32) []byte {
	buf = append(buf, 'E')
	sp := len(buf)
	buf = pgio.AppendInt32(buf, -1)

	buf = append(buf, portal...)
	buf = append(buf, 0)
	buf = pgio.AppendUint32(buf, maxRows)

	pgio.SetInt32(buf[sp:], int32(len(buf[sp:])))

	return buf
}

// appendQuery appends a PostgreSQL wire protocol query message to buf and returns it.
func appendQuery(buf []byte, query string) []byte {
	buf = append(buf, 'Q')
	sp := len(buf)
	buf = pgio.AppendInt32(buf, -1)
	buf = append(buf, query...)
	buf = append(buf, 0)
	pgio.SetInt32(buf[sp:], int32(len(buf[sp:])))

	return buf
}
<|endoftext|>


<commit_before>package connector

import (
	"bytes"
	"io"

	"io/ioutil"

	"time"

	logging "github.com/op/go-logging"
)

var logStreamLogger = logging.MustGetLogger("stream_logger")

// LogStreamer defines a structure for collecting
// data into a writer and writing to a logger.
type LogStreamer struct {
	writer   io.Writer
	logger   *logging.Logger
	stopRead bool
}

// NewLogStreamer reads from a stream and logs to stdout
func NewLogStreamer() *LogStreamer {
	return &LogStreamer{
		writer: bytes.NewBuffer(nil),
		logger: logStreamLogger,
	}
}

// GetWriter returns a writer
func (ls *LogStreamer) GetWriter() io.Writer {
	return ls.writer
}

// SetLogger sets the logger
func (ls *LogStreamer) SetLogger(logger *logging.Logger) {
	ls.logger = logger
}

// Stop sets the stop read flag to true which effectively stops the
// the writer from being read and logged.
func (ls *LogStreamer) Stop() {
	ls.stopRead = true
}

// Start starts reading the writer and logging its data
func (ls *LogStreamer) Start() error {
	for ls.stopRead == false {
		bs, err := ioutil.ReadAll(ls.writer.(*bytes.Buffer))
		if err != nil {
			ls.mutex.Unlock()
			return err
		}

		// reset the writer
		ls.writer.(*bytes.Buffer).Reset()

		if len(bs) > 0 {
			ls.logger.Info(string(bs))
		}
		time.Sleep(100 * time.Millisecond)
	}
	return nil
}
<commit_msg>remove use of mutex<commit_after>package connector

import (
	"bytes"
	"io"

	"io/ioutil"

	"time"

	logging "github.com/op/go-logging"
)

var logStreamLogger = logging.MustGetLogger("stream_logger")

// LogStreamer defines a structure for collecting
// data into a writer and writing to a logger.
type LogStreamer struct {
	writer   io.Writer
	logger   *logging.Logger
	stopRead bool
}

// NewLogStreamer reads from a stream and logs to stdout
func NewLogStreamer() *LogStreamer {
	return &LogStreamer{
		writer: bytes.NewBuffer(nil),
		logger: logStreamLogger,
	}
}

// GetWriter returns a writer
func (ls *LogStreamer) GetWriter() io.Writer {
	return ls.writer
}

// SetLogger sets the logger
func (ls *LogStreamer) SetLogger(logger *logging.Logger) {
	ls.logger = logger
}

// Stop sets the stop read flag to true which effectively stops the
// the writer from being read and logged.
func (ls *LogStreamer) Stop() {
	ls.stopRead = true
}

// Start starts reading the writer and logging its data
func (ls *LogStreamer) Start() error {
	for ls.stopRead == false {
		bs, err := ioutil.ReadAll(ls.writer.(*bytes.Buffer))
		if err != nil {
			return err
		}

		// reset the writer
		ls.writer.(*bytes.Buffer).Reset()

		if len(bs) > 0 {
			ls.logger.Info(string(bs))
		}
		time.Sleep(100 * time.Millisecond)
	}
	return nil
}
<|endoftext|>


<commit_before>package mole

import (
	"gopkg.in/mgo.v2/bson"
	"time"
)

type Log struct {
	Id                 bson.ObjectId      `bson:"_id" json:"id"`
	Timestamp          string             `json:"timestamp" binding:"required"`
	CreatedAt          time.Time          `bson:"created_at" json:"-"`
	Location           Location           `json:"location" binding:"required"`
	Error              Error              `json:"error" binding:"required"`
	ActionStateHistory ActionStateHistory `json:"action_state_history"`
}

type Location struct {
	Host     string `json:"host"`
	Href     string `json:"href"`
	Hash     string `json:"hash"`
	Pathname string `json:"pathname"`
	Port     string `json:"port"`
	Protocol string `json:"protocol"`
	Search   string `json:"search"`
}

type Error struct {
	Message    string            `json:"message"`
	Stacktrace []StracktraceLine `json:"stacktrace"`
}

type StracktraceLine struct {
	Function string `json:"function"`
	File     string `json:"file"`
	Line     string `json:"line"`
	Column   string `json:"column"`
}

type JSONObjectArray []map[string]interface{}

type ActionStateHistory struct {
	Action JSONObjectArray `json:"action"`
	State  JSONObjectArray `json:"state"`
}
<commit_msg>Fix bug<commit_after>package mole

import (
	"gopkg.in/mgo.v2/bson"
	"time"
)

type Log struct {
	Id                 bson.ObjectId      `bson:"_id" json:"id"`
	Timestamp          string             `json:"timestamp" binding:"required"`
	CreatedAt          time.Time          `bson:"created_at" json:"-"`
	Location           Location           `json:"location" binding:"required"`
	Error              Error              `json:"error" binding:"required"`
	ActionStateHistory []ActionStateHistory `json:"action_state_history"`
}

type Location struct {
	Host     string `json:"host"`
	Href     string `json:"href"`
	Hash     string `json:"hash"`
	Pathname string `json:"pathname"`
	Port     string `json:"port"`
	Protocol string `json:"protocol"`
	Search   string `json:"search"`
}

type Error struct {
	Message    string            `json:"message"`
	Stacktrace []StracktraceLine `json:"stacktrace"`
}

type StracktraceLine struct {
	Function string `json:"function"`
	File     string `json:"file"`
	Line     int `json:"line"`
	Column   int `json:"column"`
}

type JSONObjectArray []map[string]interface{}

type ActionStateHistory struct {
	Action JSONObjectArray `json:"action"`
	State  JSONObjectArray `json:"state"`
}
<|endoftext|>


<commit_before>package metafora

import (
	"math/rand"
	"sort"
	"sync"
	"time"
)

var (

	// balance calls are randomized and this is the upper bound of the random
	// amount
	balanceJitterMax = 10 * int64(time.Second)

	//FIXME should probably be improved, see usage in Run()
	consumerRetryDelay = 10 * time.Second
)

type runningTask struct {
	h Handler
	c chan struct{}
}

// Consumer is the core Metafora task runner.
type Consumer struct {
	// Func to create new handlers
	handler HandlerFunc

	// Map of task:Handler
	running map[string]runningTask

	// Mutex to protect access to running
	runL sync.Mutex

	// WaitGroup for running handlers and consumer goroutines
	hwg sync.WaitGroup

	bal      Balancer
	balEvery time.Duration
	coord    Coordinator
	logger   *logger
	stop     chan struct{} // closed by Shutdown to cause Run to exit

	// ticked on each loop of the main loop to enforce sequential interaction
	// with coordinator and balancer
	tick chan int

	watch chan string // channel for watcher to send tasks to main loop

	// Set by command handler, read anywhere via Consumer.frozen()
	freezeL sync.Mutex
	freeze  bool
}

// NewConsumer returns a new consumer and calls Init on the Balancer and Coordinator.
func NewConsumer(coord Coordinator, h HandlerFunc, b Balancer) (*Consumer, error) {
	c := &Consumer{
		running:  make(map[string]runningTask),
		handler:  h,
		bal:      b,
		balEvery: 15 * time.Minute, //TODO make balance wait configurable
		coord:    coord,
		logger:   stdoutLogger(),
		stop:     make(chan struct{}),
		tick:     make(chan int),
		watch:    make(chan string),
	}

	// initialize balancer with the consumer and a prefixed logger
	b.Init(&struct {
		*Consumer
		Logger
	}{Consumer: c, Logger: c.logger})

	if err := coord.Init(&coordinatorContext{Consumer: c, Logger: c.logger}); err != nil {
		return nil, err
	}
	return c, nil
}

// SetLogger assigns the logger to use as well as a level
//
// The logger parameter is an interface that requires the following
// method to be implemented (such as the the stdlib log.Logger):
//
//    Output(calldepth int, s string)
//
func (c *Consumer) SetLogger(l logOutputter, lvl LogLevel) {
	c.logger.l = l
	c.logger.lvl = lvl
}

// Run is the core run loop of Metafora. It is responsible for calling into the
// Coordinator to claim work and Balancer to rebalance work.
//
// Run blocks until Shutdown is called or an internal error occurs.
func (c *Consumer) Run() {
	c.logger.Log(LogLevelDebug, "Starting consumer")

	// chans for core goroutines to communicate with main loop
	balance := make(chan bool)
	cmdChan := make(chan Command)

	// Balance is called by the main loop when the balance channel is ticked
	c.hwg.Add(1)
	go func() {
		defer c.hwg.Done()
		randInt := rand.New(rand.NewSource(time.Now().UnixNano())).Int63n
		for {
			select {
			case <-c.stop:
				// Shutdown has been called.
				return
			case <-time.After(c.balEvery + time.Duration(randInt(balanceJitterMax))):
				c.logger.Log(LogLevelInfo, "Balancing")
				select {
				case balance <- true:
					// Ticked balance
				case <-c.stop:
					// Shutdown has been called.
					return
				}
			}
			// Wait for main loop to signal balancing is done
			select {
			case <-c.stop:
				return
			case <-c.tick:
			}
		}
	}()

	// Watch for new tasks in a goroutine
	c.hwg.Add(1)
	go c.watcher()

	// Watch for new commands in a goroutine
	c.hwg.Add(1)
	go func() {
		defer func() {
			close(cmdChan)
			c.hwg.Done()
		}()
		for {
			cmd, err := c.coord.Command()
			if err != nil {
				//FIXME add more sophisticated error handling
				c.logger.Log(LogLevelError, "Coordinator returned an error during command, waiting and retrying. %v", err)
				select {
				case <-c.stop:
					return
				case <-time.After(consumerRetryDelay):
				}
				continue
			}
			if cmd == nil {
				c.logger.Log(LogLevelDebug, "Command coordinator exited")
				return
			}
			// Send command to watcher (or shutdown)
			select {
			case <-c.stop:
				return
			case cmdChan <- cmd:
			}
			// Wait for main loop to signal command has been handled
			select {
			case <-c.stop:
				return
			case <-c.tick:
			}
		}
	}()

	// Make sure Run() cleans up on exit (stops coordinator, releases tasks, etc)
	defer c.shutdown()

	// Main Loop ensures events are processed synchronously
	for {
		if c.frozen() {
			// Only recv commands while frozen
			select {
			case <-c.stop:
				// Shutdown has been called.
				return
			case cmd, ok := <-cmdChan:
				if !ok {
					c.logger.Log(LogLevelDebug, "Command channel closed. Exiting main loop.")
					return
				}
				c.logger.Log(LogLevelDebug, "Received command: %s", cmd)
				c.handleCommand(cmd)
			}
			// Must send tick whenever main loop restarts
			c.tick <- 1
			continue
		}

		select {
		case <-c.stop:
			// Shutdown has been called.
			return
		case <-balance:
			c.balance()
		case task, ok := <-c.watch:
			if !ok {
				c.logger.Log(LogLevelDebug, "Watch channel closed. Exiting main loop.")
				return
			}
			if !c.bal.CanClaim(task) {
				c.logger.Log(LogLevelInfo, "Balancer rejected task %s", task)
				break
			}
			if !c.coord.Claim(task) {
				c.logger.Log(LogLevelInfo, "Coordinator unable to claim task %s", task)
				break
			}
			c.claimed(task)
		case cmd, ok := <-cmdChan:
			if !ok {
				c.logger.Log(LogLevelDebug, "Command channel closed. Exiting main loop.")
				return
			}
			c.handleCommand(cmd)
		}
		// Signal that main loop is restarting after handling an event
		c.tick <- 1
	}
}

func (c *Consumer) watcher() {
	defer func() {
		close(c.watch)
		c.hwg.Done()
	}()
	c.logger.Log(LogLevelDebug, "Consumer watching")

	for {
		task, err := c.coord.Watch()
		if err != nil {
			//FIXME add more sophisticated error handling
			c.logger.Log(LogLevelError, "Coordinator returned an error during watch, waiting and retrying: %v", err)
			select {
			case <-c.stop:
				return
			case <-time.After(consumerRetryDelay):
			}
			continue
		}
		if task == "" {
			c.logger.Log(LogLevelInfo, "Coordinator has closed, no longer watching for tasks.")
			return
		}
		// Send task to watcher (or shutdown)
		select {
		case <-c.stop:
			return
		case c.watch <- task:
		}
		// Wait for main loop to signal task has been handled
		select {
		case <-c.stop:
			return
		case <-c.tick:
		}
	}
}

func (c *Consumer) balance() {
	for _, task := range c.bal.Balance() {
		//TODO Release tasks asynchronously as their shutdown might be slow?
		c.release(task)
	}
}

// shutdown is the actual shutdown logic called when Run() exits.
func (c *Consumer) shutdown() {
	c.logger.Log(LogLevelDebug, "Closing Coordinator")
	c.coord.Close()

	// Build list of of currently running tasks
	tasks := c.Tasks()
	c.logger.Log(LogLevelInfo, "Sending stop signal to %d handler(s)", len(tasks))

	// Concurrently shutdown handlers as they may take a while to shutdown
	for _, id := range tasks {
		go c.release(id)
	}

	c.logger.Log(LogLevelInfo, "Waiting for handlers to exit")
	c.hwg.Wait()
}

// Shutdown stops the main Run loop, calls Stop on all handlers, and calls
// Close on the Coordinator. Running tasks will be released for other nodes to
// claim.
func (c *Consumer) Shutdown() {
	select {
	case <-c.stop:
		// already stopped
	default:
		c.logger.Log(LogLevelDebug, "Stopping Run loop")
		close(c.stop)
	}
	c.hwg.Wait()
}

// Tasks returns a sorted list of running Task IDs.
func (c *Consumer) Tasks() []string {
	c.runL.Lock()
	defer c.runL.Unlock()
	t := make([]string, len(c.running))
	i := 0
	for id, _ := range c.running {
		t[i] = id
		i++
	}
	sort.Strings(t)
	return t
}

// claimed starts a handler for a claimed task. It is the only method to
// manipulate c.running and closes the runningTask channel when a handler's Run
// method exits.
func (c *Consumer) claimed(taskID string) {
	h := c.handler()

	c.logger.Log(LogLevelDebug, "Attempting to start task "+taskID)
	// Associate handler with taskID
	// **This is the only place tasks should be added to c.running**
	c.runL.Lock()
	c.running[taskID] = runningTask{h: h, c: make(chan struct{})}
	c.runL.Unlock()

	c.hwg.Add(1)
	// Start handler in its own goroutine
	go func() {
		c.logger.Log(LogLevelInfo, "Task started: %s", taskID)
		defer c.logger.Log(LogLevelInfo, "Task exited: %s", taskID)
		defer func() {
			if err := recover(); err != nil {
				c.logger.Log(LogLevelError, "Handler %s panic()'d: %v", taskID, err)
				// panics are considered fatal errors. Make sure the task isn't
				// rescheduled.
				c.coord.Done(taskID)
			}
			// **This is the only place tasks should be removed from c.running**
			c.runL.Lock()
			close(c.running[taskID].c)
			delete(c.running, taskID)
			c.runL.Unlock()
			c.hwg.Done()
		}()

		// Run the task
		c.logger.Log(LogLevelDebug, "Calling run for task %s", taskID)
		if err := h.Run(taskID); err != nil {
			if ferr, ok := err.(FatalError); ok && ferr.Fatal() {
				c.logger.Log(LogLevelError, "Handler for %s exited with fatal error: %v", taskID, err)
			} else {
				c.logger.Log(LogLevelError, "Handler for %s exited with error: %v", taskID, err)
				// error was non-fatal, release and let another node try
				c.coord.Release(taskID)
				return
			}
		}
		c.coord.Done(taskID)
	}()
}

// release stops and Coordinator.Release()s a task if it's running.
//
// release blocks until the task handler stops running.
func (c *Consumer) release(taskID string) {
	// Stop task...
	if c.stopTask(taskID) {
		// ...instruct the coordinator to release it
		c.coord.Release(taskID)
	}
}

// stopTask returns true if the task was running and stopped successfully.
func (c *Consumer) stopTask(taskID string) bool {
	c.runL.Lock()
	task, ok := c.running[taskID]
	c.runL.Unlock()

	if !ok {
		// This can happen if a task completes during Balance() and is not an error.
		c.logger.Log(LogLevelWarn, "Tried to release a non-running task: %s", taskID)
		return false
	}

	// all handler methods must be wrapped in a recover to prevent a misbehaving
	// handler from crashing the entire consumer
	func() {
		defer func() {
			if err := recover(); err != nil {
				c.logger.Log(LogLevelError, "Handler %s panic()'d on Stop: %v", taskID, err)
			}
		}()

		task.h.Stop()
	}()

	// Once the handler is stopped...
	//FIXME should there be a timeout here?
	<-task.c
	return true
}

func (c *Consumer) frozen() bool {
	c.freezeL.Lock()
	r := c.freeze
	c.freezeL.Unlock()
	return r
}

func (c *Consumer) handleCommand(cmd Command) {
	switch cmd.Name() {
	case cmdFreeze:
		if c.frozen() {
			c.logger.Log(LogLevelInfo, "Ignoring freeze command: already frozen")
			return
		}
		c.logger.Log(LogLevelInfo, "Freezing")
		c.freezeL.Lock()
		c.freeze = true
		c.freezeL.Unlock()
	case cmdUnfreeze:
		if !c.frozen() {
			c.logger.Log(LogLevelInfo, "Ignoring unfreeze command: not frozen")
			return
		}
		c.logger.Log(LogLevelInfo, "Unfreezing")
		c.freezeL.Lock()
		c.freeze = false
		c.freezeL.Unlock()
	case cmdBalance:
		c.logger.Log(LogLevelInfo, "Balancing due to command")
		c.balance()
		c.logger.Log(LogLevelDebug, "Finished balancing due to command")
	case cmdReleaseTask:
		taskI, ok := cmd.Parameters()["task"]
		task, ok2 := taskI.(string)
		if !ok || !ok2 {
			c.logger.Log(LogLevelError, "Release task command didn't contain a valid task")
			return
		}
		c.logger.Log(LogLevelInfo, "Releasing task %s due to command", task)
		c.release(task)
	case cmdStopTask:
		taskI, ok := cmd.Parameters()["task"]
		task, ok2 := taskI.(string)
		if !ok || !ok2 {
			c.logger.Log(LogLevelError, "Stop task command didn't contain a valid task")
			return
		}
		c.logger.Log(LogLevelInfo, "Stopping task %s due to command", task)
		c.stopTask(task)
	default:
		c.logger.Log(LogLevelWarn, "Discarding unknown command: %s", cmd.Name())
	}
}
<commit_msg>Fix tick can block forever during shutdown<commit_after>package metafora

import (
	"math/rand"
	"sort"
	"sync"
	"time"
)

var (

	// balance calls are randomized and this is the upper bound of the random
	// amount
	balanceJitterMax = 10 * int64(time.Second)

	//FIXME should probably be improved, see usage in Run()
	consumerRetryDelay = 10 * time.Second
)

type runningTask struct {
	h Handler
	c chan struct{}
}

// Consumer is the core Metafora task runner.
type Consumer struct {
	// Func to create new handlers
	handler HandlerFunc

	// Map of task:Handler
	running map[string]runningTask

	// Mutex to protect access to running
	runL sync.Mutex

	// WaitGroup for running handlers and consumer goroutines
	hwg sync.WaitGroup

	bal      Balancer
	balEvery time.Duration
	coord    Coordinator
	logger   *logger
	stop     chan struct{} // closed by Shutdown to cause Run to exit

	// ticked on each loop of the main loop to enforce sequential interaction
	// with coordinator and balancer
	tick chan int

	watch chan string // channel for watcher to send tasks to main loop

	// Set by command handler, read anywhere via Consumer.frozen()
	freezeL sync.Mutex
	freeze  bool
}

// NewConsumer returns a new consumer and calls Init on the Balancer and Coordinator.
func NewConsumer(coord Coordinator, h HandlerFunc, b Balancer) (*Consumer, error) {
	c := &Consumer{
		running:  make(map[string]runningTask),
		handler:  h,
		bal:      b,
		balEvery: 15 * time.Minute, //TODO make balance wait configurable
		coord:    coord,
		logger:   stdoutLogger(),
		stop:     make(chan struct{}),
		tick:     make(chan int),
		watch:    make(chan string),
	}

	// initialize balancer with the consumer and a prefixed logger
	b.Init(&struct {
		*Consumer
		Logger
	}{Consumer: c, Logger: c.logger})

	if err := coord.Init(&coordinatorContext{Consumer: c, Logger: c.logger}); err != nil {
		return nil, err
	}
	return c, nil
}

// SetLogger assigns the logger to use as well as a level
//
// The logger parameter is an interface that requires the following
// method to be implemented (such as the the stdlib log.Logger):
//
//    Output(calldepth int, s string)
//
func (c *Consumer) SetLogger(l logOutputter, lvl LogLevel) {
	c.logger.l = l
	c.logger.lvl = lvl
}

// Run is the core run loop of Metafora. It is responsible for calling into the
// Coordinator to claim work and Balancer to rebalance work.
//
// Run blocks until Shutdown is called or an internal error occurs.
func (c *Consumer) Run() {
	c.logger.Log(LogLevelDebug, "Starting consumer")

	// chans for core goroutines to communicate with main loop
	balance := make(chan bool)
	cmdChan := make(chan Command)

	// Balance is called by the main loop when the balance channel is ticked
	c.hwg.Add(1)
	go func() {
		defer c.hwg.Done()
		randInt := rand.New(rand.NewSource(time.Now().UnixNano())).Int63n
		for {
			select {
			case <-c.stop:
				// Shutdown has been called.
				return
			case <-time.After(c.balEvery + time.Duration(randInt(balanceJitterMax))):
				c.logger.Log(LogLevelInfo, "Balancing")
				select {
				case balance <- true:
					// Ticked balance
				case <-c.stop:
					// Shutdown has been called.
					return
				}
			}
			// Wait for main loop to signal balancing is done
			select {
			case <-c.stop:
				return
			case <-c.tick:
			}
		}
	}()

	// Watch for new tasks in a goroutine
	c.hwg.Add(1)
	go c.watcher()

	// Watch for new commands in a goroutine
	c.hwg.Add(1)
	go func() {
		defer func() {
			close(cmdChan)
			c.hwg.Done()
		}()
		for {
			cmd, err := c.coord.Command()
			if err != nil {
				//FIXME add more sophisticated error handling
				c.logger.Log(LogLevelError, "Coordinator returned an error during command, waiting and retrying. %v", err)
				select {
				case <-c.stop:
					return
				case <-time.After(consumerRetryDelay):
				}
				continue
			}
			if cmd == nil {
				c.logger.Log(LogLevelDebug, "Command coordinator exited")
				return
			}
			// Send command to watcher (or shutdown)
			select {
			case <-c.stop:
				return
			case cmdChan <- cmd:
			}
			// Wait for main loop to signal command has been handled
			select {
			case <-c.stop:
				return
			case <-c.tick:
			}
		}
	}()

	// Make sure Run() cleans up on exit (stops coordinator, releases tasks, etc)
	defer c.shutdown()

	// Main Loop ensures events are processed synchronously
	for {
		if c.frozen() {
			// Only recv commands while frozen
			select {
			case <-c.stop:
				// Shutdown has been called.
				return
			case cmd, ok := <-cmdChan:
				if !ok {
					c.logger.Log(LogLevelDebug, "Command channel closed. Exiting main loop.")
					return
				}
				c.logger.Log(LogLevelDebug, "Received command: %s", cmd)
				c.handleCommand(cmd)
			}
			// Must send tick whenever main loop restarts
			select {
			case <-c.stop:
				return
			case c.tick <- 1:
			}
			continue
		}

		select {
		case <-c.stop:
			// Shutdown has been called.
			return
		case <-balance:
			c.balance()
		case task, ok := <-c.watch:
			if !ok {
				c.logger.Log(LogLevelDebug, "Watch channel closed. Exiting main loop.")
				return
			}
			if !c.bal.CanClaim(task) {
				c.logger.Log(LogLevelInfo, "Balancer rejected task %s", task)
				break
			}
			if !c.coord.Claim(task) {
				c.logger.Log(LogLevelInfo, "Coordinator unable to claim task %s", task)
				break
			}
			c.claimed(task)
		case cmd, ok := <-cmdChan:
			if !ok {
				c.logger.Log(LogLevelDebug, "Command channel closed. Exiting main loop.")
				return
			}
			c.handleCommand(cmd)
		}
		// Signal that main loop is restarting after handling an event
		select {
		case <-c.stop:
			return
		case c.tick <- 1:
		}
	}
}

func (c *Consumer) watcher() {
	defer func() {
		close(c.watch)
		c.hwg.Done()
	}()
	c.logger.Log(LogLevelDebug, "Consumer watching")

	for {
		task, err := c.coord.Watch()
		if err != nil {
			//FIXME add more sophisticated error handling
			c.logger.Log(LogLevelError, "Coordinator returned an error during watch, waiting and retrying: %v", err)
			select {
			case <-c.stop:
				return
			case <-time.After(consumerRetryDelay):
			}
			continue
		}
		if task == "" {
			c.logger.Log(LogLevelInfo, "Coordinator has closed, no longer watching for tasks.")
			return
		}
		// Send task to watcher (or shutdown)
		select {
		case <-c.stop:
			return
		case c.watch <- task:
		}
		// Wait for main loop to signal task has been handled
		select {
		case <-c.stop:
			return
		case <-c.tick:
		}
	}
}

func (c *Consumer) balance() {
	for _, task := range c.bal.Balance() {
		//TODO Release tasks asynchronously as their shutdown might be slow?
		c.release(task)
	}
}

// shutdown is the actual shutdown logic called when Run() exits.
func (c *Consumer) shutdown() {
	c.logger.Log(LogLevelDebug, "Closing Coordinator")
	c.coord.Close()

	// Build list of of currently running tasks
	tasks := c.Tasks()
	c.logger.Log(LogLevelInfo, "Sending stop signal to %d handler(s)", len(tasks))

	// Concurrently shutdown handlers as they may take a while to shutdown
	for _, id := range tasks {
		go c.release(id)
	}

	c.logger.Log(LogLevelInfo, "Waiting for handlers to exit")
	c.hwg.Wait()
}

// Shutdown stops the main Run loop, calls Stop on all handlers, and calls
// Close on the Coordinator. Running tasks will be released for other nodes to
// claim.
func (c *Consumer) Shutdown() {
	select {
	case <-c.stop:
		// already stopped
	default:
		c.logger.Log(LogLevelDebug, "Stopping Run loop")
		close(c.stop)
	}
	c.hwg.Wait()
}

// Tasks returns a sorted list of running Task IDs.
func (c *Consumer) Tasks() []string {
	c.runL.Lock()
	defer c.runL.Unlock()
	t := make([]string, len(c.running))
	i := 0
	for id, _ := range c.running {
		t[i] = id
		i++
	}
	sort.Strings(t)
	return t
}

// claimed starts a handler for a claimed task. It is the only method to
// manipulate c.running and closes the runningTask channel when a handler's Run
// method exits.
func (c *Consumer) claimed(taskID string) {
	h := c.handler()

	c.logger.Log(LogLevelDebug, "Attempting to start task "+taskID)
	// Associate handler with taskID
	// **This is the only place tasks should be added to c.running**
	c.runL.Lock()
	c.running[taskID] = runningTask{h: h, c: make(chan struct{})}
	c.runL.Unlock()

	c.hwg.Add(1)
	// Start handler in its own goroutine
	go func() {
		c.logger.Log(LogLevelInfo, "Task started: %s", taskID)
		defer c.logger.Log(LogLevelInfo, "Task exited: %s", taskID)
		defer func() {
			if err := recover(); err != nil {
				c.logger.Log(LogLevelError, "Handler %s panic()'d: %v", taskID, err)
				// panics are considered fatal errors. Make sure the task isn't
				// rescheduled.
				c.coord.Done(taskID)
			}
			// **This is the only place tasks should be removed from c.running**
			c.runL.Lock()
			close(c.running[taskID].c)
			delete(c.running, taskID)
			c.runL.Unlock()
			c.hwg.Done()
		}()

		// Run the task
		c.logger.Log(LogLevelDebug, "Calling run for task %s", taskID)
		if err := h.Run(taskID); err != nil {
			if ferr, ok := err.(FatalError); ok && ferr.Fatal() {
				c.logger.Log(LogLevelError, "Handler for %s exited with fatal error: %v", taskID, err)
			} else {
				c.logger.Log(LogLevelError, "Handler for %s exited with error: %v", taskID, err)
				// error was non-fatal, release and let another node try
				c.coord.Release(taskID)
				return
			}
		}
		c.coord.Done(taskID)
	}()
}

// release stops and Coordinator.Release()s a task if it's running.
//
// release blocks until the task handler stops running.
func (c *Consumer) release(taskID string) {
	// Stop task...
	if c.stopTask(taskID) {
		// ...instruct the coordinator to release it
		c.coord.Release(taskID)
	}
}

// stopTask returns true if the task was running and stopped successfully.
func (c *Consumer) stopTask(taskID string) bool {
	c.runL.Lock()
	task, ok := c.running[taskID]
	c.runL.Unlock()

	if !ok {
		// This can happen if a task completes during Balance() and is not an error.
		c.logger.Log(LogLevelWarn, "Tried to release a non-running task: %s", taskID)
		return false
	}

	// all handler methods must be wrapped in a recover to prevent a misbehaving
	// handler from crashing the entire consumer
	func() {
		defer func() {
			if err := recover(); err != nil {
				c.logger.Log(LogLevelError, "Handler %s panic()'d on Stop: %v", taskID, err)
			}
		}()

		task.h.Stop()
	}()

	// Once the handler is stopped...
	//FIXME should there be a timeout here?
	<-task.c
	return true
}

func (c *Consumer) frozen() bool {
	c.freezeL.Lock()
	r := c.freeze
	c.freezeL.Unlock()
	return r
}

func (c *Consumer) handleCommand(cmd Command) {
	switch cmd.Name() {
	case cmdFreeze:
		if c.frozen() {
			c.logger.Log(LogLevelInfo, "Ignoring freeze command: already frozen")
			return
		}
		c.logger.Log(LogLevelInfo, "Freezing")
		c.freezeL.Lock()
		c.freeze = true
		c.freezeL.Unlock()
	case cmdUnfreeze:
		if !c.frozen() {
			c.logger.Log(LogLevelInfo, "Ignoring unfreeze command: not frozen")
			return
		}
		c.logger.Log(LogLevelInfo, "Unfreezing")
		c.freezeL.Lock()
		c.freeze = false
		c.freezeL.Unlock()
	case cmdBalance:
		c.logger.Log(LogLevelInfo, "Balancing due to command")
		c.balance()
		c.logger.Log(LogLevelDebug, "Finished balancing due to command")
	case cmdReleaseTask:
		taskI, ok := cmd.Parameters()["task"]
		task, ok2 := taskI.(string)
		if !ok || !ok2 {
			c.logger.Log(LogLevelError, "Release task command didn't contain a valid task")
			return
		}
		c.logger.Log(LogLevelInfo, "Releasing task %s due to command", task)
		c.release(task)
	case cmdStopTask:
		taskI, ok := cmd.Parameters()["task"]
		task, ok2 := taskI.(string)
		if !ok || !ok2 {
			c.logger.Log(LogLevelError, "Stop task command didn't contain a valid task")
			return
		}
		c.logger.Log(LogLevelInfo, "Stopping task %s due to command", task)
		c.stopTask(task)
	default:
		c.logger.Log(LogLevelWarn, "Discarding unknown command: %s", cmd.Name())
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"net/http"
	"os"
	"path"
	"strings"
)

var (
	rootDirectory = flag.String("path", ".", "Server root directory.")
	listenPort    = flag.Int("port", 8080, "Server listen port.")
)

func expandDirectory(s string) string {
	if ok := strings.HasPrefix(s, "~"); ok {
		s = path.Join(os.Getenv("HOME"), strings.SplitAfter(s, "~")[1])
	}
	return path.Clean(s)
}

func main() {
	flag.Parse()

	root := expandDirectory(*rootDirectory)
	fmt.Printf("Serving HTTP on 0.0.0.0 port %d ...\n", *listenPort)

	http.Handle("/", http.FileServer(http.Dir(root)))
	http.ListenAndServe(fmt.Sprintf(":%d", *listenPort), nil)
}
<commit_msg>Add better logging to Go HTTP file server.<commit_after>package main

import (
	"flag"
	"fmt"
	"net"
	"net/http"
	"os"
	"os/signal"
	"path"
	"strings"
	"syscall"
	"time"
)

var (
	rootDirectory = flag.String("path", ".", "Server root directory.")
	listenHost    = flag.String("host", "0.0.0.0", "Server listen address.")
	listenPort    = flag.Int("port", 8080, "Server listen port.")
)

type ResponseWriterWrapper struct {
	status int
	bytes  int
	http.ResponseWriter
}

func (rww *ResponseWriterWrapper) Write(bytes []byte) (int, error) {
	if rww.status == 0 {
		rww.status = http.StatusOK
	}

	n, err := rww.ResponseWriter.Write(bytes)
	rww.bytes += n

	return n, err
}

func (rww *ResponseWriterWrapper) WriteHeader(code int) {
	rww.status = code
	rww.ResponseWriter.WriteHeader(code)
}

type HandlerWrapper struct {
	http.Handler
}

func (hw *HandlerWrapper) RecordRequest(r *http.Request, w *ResponseWriterWrapper, duration time.Duration) {
	var host string
	user := "-"

	if i := strings.LastIndex(r.RemoteAddr, ":"); i != -1 {
		host = r.RemoteAddr[:i]
	}

	if r.URL.User != nil {
		if u := r.URL.User.Username(); u != "" {
			user = u
		}
	}

	fmt.Printf("%s - %s [%s] \"%s %s %s\" %d %d %f\n",
		host,
		user,
		time.Now().Format("02/Jan/2006:15:04:05 -0700"),
		r.Method,
		r.URL,
		r.Proto,
		w.status,
		w.bytes,
		duration.Seconds())
}

func (hw *HandlerWrapper) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	start := time.Now()
	rww := &ResponseWriterWrapper{0, 0, w}
	hw.Handler.ServeHTTP(rww, r)
	go hw.RecordRequest(r, rww, time.Since(start))
}

func expandPath(p string) string {
	if ok := strings.HasPrefix(p, "~"); ok {
		p = path.Join(os.Getenv("HOME"), strings.SplitAfter(p, "~")[1])
	}
	return path.Clean(p)
}

func main() {
	var l net.Listener
	var err error

	os.Stdout = os.Stderr

	flag.Parse()

	s := make(chan os.Signal)
	go func() {
		signal.Notify(s, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
		<-s
		l.Close()
		signal.Stop(s)
		os.Exit(0)
	}()

	mux := http.NewServeMux()
	handler := &HandlerWrapper{http.FileServer(http.Dir(expandPath(*rootDirectory)))}

	mux.Handle("/", handler)

	*listenHost = strings.TrimSpace(*listenHost)

	l, err = net.Listen("tcp4", fmt.Sprintf("%s:%d", *listenHost, *listenPort))
	if err != nil {
		fmt.Printf("An error occurred: %s\n", err.Error())
		os.Exit(1)
	}
	fmt.Printf("Serving HTTP on %s port %d...\n", *listenHost, *listenPort)

	err = http.Serve(l, mux)
	if !strings.Contains(err.Error(), "use of closed network connection") {
		fmt.Printf("An error occurred: %s\n", err.Error())
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package lumber

type MultiLogger struct {
	loggers []Logger
}

func NewMultiLogger() (l *MultiLogger) {
	return &MultiLogger{}
}

func (p *MultiLogger) AddLoggers(newLogs ...Logger) {
	for _, l := range newLogs {
		p.loggers = append(p.loggers, l)
	}
}

// All of these implement the Logger interface and distribute calls to it over
// all of the member Logger objects.
func (p *MultiLogger) Fatal(s string, v ...interface{}) {
	for _, logger := range p.loggers {
		logger.Fatal(s, v...)
	}
}

func (p *MultiLogger) Error(s string, v ...interface{}) {
	for _, logger := range p.loggers {
		logger.Error(s, v...)
	}
}

func (p *MultiLogger) Warn(s string, v ...interface{}) {
	for _, logger := range p.loggers {
		logger.Warn(s, v...)
	}
}

func (p *MultiLogger) Info(s string, v ...interface{}) {
	for _, logger := range p.loggers {
		logger.Info(s, v...)
	}
}

func (p *MultiLogger) Debug(s string, v ...interface{}) {
	for _, logger := range p.loggers {
		logger.Debug(s, v...)
	}
}

func (p *MultiLogger) Trace(s string, v ...interface{}) {
	for _, logger := range p.loggers {
		logger.Trace(s, v...)
	}
}

func (p *MultiLogger) Level(i int) {
	for _, logger := range p.loggers {
		logger.Level(i)
	}
}

func (p *MultiLogger) Prefix(s string) {
	for _, logger := range p.loggers {
		logger.Prefix(s)
	}
}

func (p *MultiLogger) TimeFormat(s string) {
	for _, logger := range p.loggers {
		logger.TimeFormat(s)
	}
}

func (p *MultiLogger) Close() {
	for _, logger := range p.loggers {
		logger.Close()
	}
}

func (p *MultiLogger) output(m *Message) {
	for _, logger := range p.loggers {
		logger.output(m)
	}
}
<commit_msg>Fix multilog so it implements the Logger interface<commit_after>package lumber

import (
	"fmt"
	"time"
)

type MultiLogger struct {
	loggers []Logger
}

func NewMultiLogger() (l *MultiLogger) {
	return &MultiLogger{}
}

func (p *MultiLogger) AddLoggers(newLogs ...Logger) {
	for _, l := range newLogs {
		p.loggers = append(p.loggers, l)
	}
}

// All of these implement the Logger interface and distribute calls to it over
// all of the member Logger objects.
func (p *MultiLogger) Fatal(s string, v ...interface{}) {
	for _, logger := range p.loggers {
		logger.Fatal(s, v...)
	}
}

func (p *MultiLogger) Error(s string, v ...interface{}) {
	for _, logger := range p.loggers {
		logger.Error(s, v...)
	}
}

func (p *MultiLogger) Warn(s string, v ...interface{}) {
	for _, logger := range p.loggers {
		logger.Warn(s, v...)
	}
}

func (p *MultiLogger) Info(s string, v ...interface{}) {
	for _, logger := range p.loggers {
		logger.Info(s, v...)
	}
}

func (p *MultiLogger) Debug(s string, v ...interface{}) {
	for _, logger := range p.loggers {
		logger.Debug(s, v...)
	}
}

func (p *MultiLogger) Trace(s string, v ...interface{}) {
	for _, logger := range p.loggers {
		logger.Trace(s, v...)
	}
}

func (p *MultiLogger) Level(i int) {
	for _, logger := range p.loggers {
		logger.Level(i)
	}
}

func (p *MultiLogger) Prefix(s string) {
	for _, logger := range p.loggers {
		logger.Prefix(s)
	}
}

func (p *MultiLogger) TimeFormat(s string) {
	for _, logger := range p.loggers {
		logger.TimeFormat(s)
	}
}

func (p *MultiLogger) Close() {
	for _, logger := range p.loggers {
		logger.Close()
	}
}

func (p *MultiLogger) output(m *Message) {
	for _, logger := range p.loggers {
		logger.output(m)
	}
}

func (p *MultiLogger) Print(lvl int, v ...interface{}) {
	p.output(&Message{lvl, fmt.Sprint(v...), time.Now()})
}

func (p *MultiLogger) Printf(lvl int, format string, v ...interface{}) {
	p.output(&Message{lvl, fmt.Sprintf(format, v...), time.Now()})
}
<|endoftext|>


<commit_before>/*
renamer.go

Renaming files script on various patterns

Author: Gianluca Fiore <forod.g@gmail.com> © 2013
*/

package main

import (
	"flag"
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"errors"
	"strings"
	"io"
)

var usageMessage string = `
renamer.go (-p <prefix>|-s <suffix>|-i <indexname> -I <num>|-l|-u) [-x <regexp>] [-t <target_dir>] -[cn]

renamer.go will rename all files matching a regexp or all files in the
given directory (and optionally in all its subdirectories) by the flag
chosen. It can add a prefix/suffix, rename according to a
<name><numeric_index> pattern, make all low/uppercase and copy files
somewhere else leaving the originals untouched.

Arguments:

	-prefix|-p <prefix>
		Renames matching files by adding a prefix

	-suffix|-s <suffix>
		Renames matching files by adding a suffix

	-index|-i <name>
		Renames matching files to <name><num>. Requires '-I'

	-startnumber|-I <num>
		The <num> to start renaming in index mode. Requires '-i'

	-regexp|-x <regexp>
		The regular expression for matching files. Use double quotes
		around it

	-target-dir|-t <path>
		The directory where to rename/copy the files. Default is the
		current directory

	-copy|-c
		Copy instead of renaming

	-lowercase|-l
		Lowercase the new filename. It is mutually exclusive with
		'-u'

	-uppercase|-u
		Uppercase the new filename. It is mutually exclusive with
		'-l'

	-dry-run|-n
		List files but don't actually perform any action

	-force|-f
		Overwrite existing files. The default is to not copy/rename
		if the target file already exists
`

var regexpArg string // the regexp argument
var fileRegex *regexp.Regexp // the files matching the regexp
var prefixArg string // the prefix
var suffixArg string // the suffix
var indexArg string  // the <name> in the <name><num> pattern
var numArg int // the <num> in the <name><num> pattern
var targetArg string // the target directory
var lowerArg bool // the lowercase switch
var upperArg bool // the uppercase switch
var copyArg bool // the copy switch
var dryrunArg bool // the dry-run switch
var forceArg bool  // the force switch

var operationSuccessful int // numeric flag to keep trace of what went
			// wrong during the renaming

// Print a message and the usage instructions
func printUsage(msg string) {
	fmt.Fprintf(os.Stderr, msg + "\n")
	fmt.Fprintf(os.Stderr, usageMessage)
	os.Exit(2)
}

// Parse flags ("arguments")
func flagsInit() {
	// Use our custom usageMessage to print usage instructions
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, usageMessage)
	}

	const (
		def_regexp		= ""
		def_prefix		= ""
		def_suffix		= ""
		def_index		= ""
		def_num			= 1
		def_target		= ""
		def_lowercase	= false
		def_uppercase	= false
		def_copy		= false
		def_dryrun		= false
		def_force		= false
	)

	flag.StringVar(&regexpArg, "regexp", def_regexp, "")
	flag.StringVar(&regexpArg, "x", def_regexp, "")
	flag.StringVar(&prefixArg, "prefix", def_prefix, "")
	flag.StringVar(&prefixArg, "p", def_prefix, "")
	flag.StringVar(&suffixArg, "suffix", def_suffix, "")
	flag.StringVar(&suffixArg, "s", def_suffix, "")
	flag.StringVar(&indexArg, "index", def_index, "")
	flag.StringVar(&indexArg, "i", def_index, "")
	flag.IntVar(&numArg, "startnumber", def_num, "")
	flag.IntVar(&numArg, "I", def_num, "")
	flag.StringVar(&targetArg, "target", def_target, "")
	flag.StringVar(&targetArg, "t", def_target, "")
	flag.BoolVar(&lowerArg, "lowercase", def_lowercase, "")
	flag.BoolVar(&lowerArg, "l", def_lowercase, "")
	flag.BoolVar(&upperArg, "uppercase", def_uppercase, "")
	flag.BoolVar(&upperArg, "u", def_uppercase, "")
	flag.BoolVar(&copyArg, "copy", def_copy, "")
	flag.BoolVar(&copyArg, "c", def_copy, "")
	flag.BoolVar(&dryrunArg, "dry-run", def_dryrun, "")
	flag.BoolVar(&dryrunArg, "n", def_dryrun, "")
	flag.BoolVar(&forceArg, "force", def_force, "")
	flag.BoolVar(&forceArg, "f", def_force, "")

	flag.Parse()

	if regexpArg == "" && prefixArg == "" && suffixArg == "" && indexArg == "" && lowerArg == false && upperArg == false {
		printUsage("At least a flag must be given, nothing to do...")
	}
}

// Write a renamed or a copy of a file to disk
func writeFile(oldname, newname string) {
	newname = filepath.Join(targetArg, newname)
	oldname = filepath.Join(targetArg, oldname)
	// check if the new filename is already present
	_, lstat_err := os.Lstat(newname)
	if lstat_err == nil && forceArg == false {
		fmt.Fprintf(os.Stderr, "File %s already exist! Use -force to override it\n", newname)
		operationSuccessful = operationSuccessful + 1
	}
	if dryrunArg {
		// if dry-run was given, just output the renaming operation
		fmt.Fprintf(os.Stdout, "Renaming %s to %s (dry-run)\n", oldname, newname)
		operationSuccessful = operationSuccessful + 0
	} else {
		if copyArg {
			copyf, create_err := os.Create(newname)
			if create_err != nil {
				fmt.Fprintf(os.Stderr, create_err.Error())
				operationSuccessful = operationSuccessful + 1
			}
			originalf, open_err := os.Open(oldname)
			if open_err != nil {
				fmt.Fprintf(os.Stderr, open_err.Error())
				operationSuccessful = operationSuccessful + 1
			}
			_, copy_err := io.Copy(copyf, originalf)
			if copy_err != nil {
				fmt.Fprintf(os.Stderr, "An error occurred during the copy of %s to %s\n", oldname, newname)
				fmt.Fprintf(os.Stderr, copy_err.Error())
				operationSuccessful = operationSuccessful + 1
			}
		} else {
			rename_err := os.Rename(oldname, newname)
			if rename_err != nil {
				fmt.Fprintf(os.Stderr, "An error occurred during the renaming of %s to %s\n", oldname, newname)
				fmt.Fprintf(os.Stderr, rename_err.Error())
				operationSuccessful = operationSuccessful + 1
			}
		}
	}
}

// Add a prefix string to a name
func addPrefix(names []string, prefix string) int {
	var finalname string
	for _, f := range names {
		finalname = prefix + filepath.Base(f)
		writeFile(f, finalname)
	}
	return 0
}

// Add a suffix string to a name
func addSuffix(names []string, suffix string) int {
	var finalname string
	for _, f := range names {
		ext := filepath.Ext(f)
		justname := strings.TrimSuffix(filepath.Base(f), ext)
		finalname = justname + suffix + ext
		writeFile(f, finalname)
	}
	return 0
}

// Rename a slice of filenames to <newname><count>.<extension>
func indexName(names []string, newname string, count int) int {
	var finalname string
	for _, f := range names {
		ext := filepath.Ext(f)
		finalname = fmt.Sprintf("%s%03d%s", newname, count, ext)
		writeFile(f, finalname)
		count++
	}
	return 0
}

// Make filenames all lowercase
func lowercaseFiles(names []string) int {
	var finalname string
	for _, f := range names {
		finalname = strings.ToLower(filepath.Base(f))
		writeFile(f, finalname)
	}
	return 0
}

// Make filenames all uppercase
func uppercaseFiles(names []string) int {
	var finalname string
	for _, f := range names {
		finalname = strings.ToUpper(filepath.Base(f))
		writeFile(f, finalname)
	}
	return 0
}

// Get all files and directories
func getFilesFromDir(dirname string) ([]string, []string) {
	var complete_path string // final, absolute, path
	var filesindir = make([]os.FileInfo, 0) // files & directories found in path
	var allfiles = make([]string, 0)
	var alldirectories = make([]string, 0)

	dirinfo, lerr := os.Lstat(dirname)
	if lerr != nil {
		fmt.Fprintf(os.Stderr, lerr.Error())
		return alldirectories, allfiles
	}

	// check whether targetArg is an absolute path AND a directory
	if filepath.IsAbs(dirname) && dirinfo.IsDir() {
		complete_path = dirname
	} else {
		abs_path, err := filepath.Abs(dirname)
		if err != nil {
			fmt.Fprintf(os.Stderr, err.Error())
		}
		complete_path = abs_path
	}
	dir, err := os.Open(complete_path)
	defer dir.Close()
	if err != nil {
		err = errors.New(fmt.Sprintf("Target directory %s is not a directory or can't be accessed\n", complete_path))
		fmt.Fprintf(os.Stderr, err.Error())
		return alldirectories, allfiles
	}

	// scan for files/directories in path
	filesindir, read_err := dir.Readdir(0)
	if read_err != nil {
		fmt.Fprintf(os.Stderr, read_err.Error())
		return alldirectories, allfiles
	}

	// check in filesindir slice and separate directories from files in
	// 2 different slices
	for _, f := range filesindir {
		if f.IsDir() {
			alldirectories = append(alldirectories, f.Name())
		} else {
			allfiles = append(allfiles, f.Name())
		}
	}

	return alldirectories, allfiles
}

func renameFiles(dir, files []string) int {
	var basename string
	var matchingfiles []string // a slice containing only the files
					// matching the regexp passed as argument (if)
	var result int // the integer returned by each functions, signaling
					// success or failure

	//TODO: add a check for the `dir` slice and a recursiveArg to act
	//recursively on all subdirectories of targetDir. Save each files
	//with its full path or each one with the subdirectories' so it's
	//possible to rename/copy it with os.Rename later in writeFile()

	// check if the files should match a given regexp
	if regexpArg != "" {
		for _, f := range files {
		basename = filepath.Base(f)
			// Compile and check if the regexp is a valid one
			compRegexp, err := regexp.Compile(regexpArg)
			if err != nil {
				fmt.Fprintf(os.Stderr, "Invalid regexp: %s\n", regexpArg)
				printUsage("You must give a valid regexp (or none, to operate on all files). Alternatively, add -force to force renaming all files, whether they match the regexp or not")
				return 1
			} else {
				if compRegexp.MatchString(basename) == false {
					continue
				} else {
					matchingfiles = append(matchingfiles, f)
				}
			}
		}
	}

	// if matchingfiles contains something, iterate over it.
	// Otherwise, we simply use the generic files slice
	if len(matchingfiles) >= 1 {
		files = matchingfiles
	} else {
		// different messages whether we have forceArg == true or false
		if regexpArg != "" {
			if forceArg {
				fmt.Fprintf(os.Stdout, "No files matched, including all files anyway (-force enabled)\n")
			} else {
				fmt.Fprintf(os.Stderr, "No files matched, check the correctness of your regexp\n")
				return 1
			}
		}
	}

	if prefixArg != "" {
		result = addPrefix(files, prefixArg)
	}
	if suffixArg != "" {
		result = addSuffix(files, suffixArg)
	}
	if indexArg != "" {
		result = indexName(files, indexArg, numArg)
	}
	if lowerArg == true && upperArg == true {
		// can't use both
		printUsage("Can't use both lowercase and uppercase, choose one only!")
		return 1
	}
	if lowerArg == true {
		result = lowercaseFiles(files)
	}
	if upperArg == true {
		result = uppercaseFiles(files)
	}

	return result
}

func main() {
	var success_rename int
	var directories, files []string

	flagsInit()

	directories, files = getFilesFromDir(targetArg)

	success_rename = renameFiles(directories, files)

	// check that everything went smoothly
	if success_rename == 0  && operationSuccessful == 0 {
		fmt.Fprintf(os.Stdout, "\nRenaming complete\n")
	} else {
		fmt.Fprintf(os.Stdout, "\nNot all files were correctly renamed, check the previous error messages")
	}
}
<commit_msg>Print operational info on renaming and copying<commit_after>/*
renamer.go

Renaming files script on various patterns

Author: Gianluca Fiore <forod.g@gmail.com> © 2013
*/

package main

import (
	"flag"
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"errors"
	"strings"
	"io"
)

var usageMessage string = `
renamer.go (-p <prefix>|-s <suffix>|-i <indexname> -I <num>|-l|-u) [-x <regexp>] [-t <target_dir>] -[cn]

renamer.go will rename all files matching a regexp or all files in the
given directory (and optionally in all its subdirectories) by the flag
chosen. It can add a prefix/suffix, rename according to a
<name><numeric_index> pattern, make all low/uppercase and copy files
somewhere else leaving the originals untouched.

Arguments:

	-prefix|-p <prefix>
		Renames matching files by adding a prefix

	-suffix|-s <suffix>
		Renames matching files by adding a suffix

	-index|-i <name>
		Renames matching files to <name><num>. Requires '-I'

	-startnumber|-I <num>
		The <num> to start renaming in index mode. Requires '-i'

	-regexp|-x <regexp>
		The regular expression for matching files. Use double quotes
		around it

	-target-dir|-t <path>
		The directory where to rename/copy the files. Default is the
		current directory

	-copy|-c
		Copy instead of renaming

	-lowercase|-l
		Lowercase the new filename. It is mutually exclusive with
		'-u'

	-uppercase|-u
		Uppercase the new filename. It is mutually exclusive with
		'-l'

	-dry-run|-n
		List files but don't actually perform any action

	-force|-f
		Overwrite existing files. The default is to not copy/rename
		if the target file already exists
`

var regexpArg string // the regexp argument
var fileRegex *regexp.Regexp // the files matching the regexp
var prefixArg string // the prefix
var suffixArg string // the suffix
var indexArg string  // the <name> in the <name><num> pattern
var numArg int // the <num> in the <name><num> pattern
var targetArg string // the target directory
var lowerArg bool // the lowercase switch
var upperArg bool // the uppercase switch
var copyArg bool // the copy switch
var dryrunArg bool // the dry-run switch
var forceArg bool  // the force switch

var operationSuccessful int // numeric flag to keep trace of what went
			// wrong during the renaming

// Print a message and the usage instructions
func printUsage(msg string) {
	fmt.Fprintf(os.Stderr, msg + "\n")
	fmt.Fprintf(os.Stderr, usageMessage)
	os.Exit(2)
}

// Parse flags ("arguments")
func flagsInit() {
	// Use our custom usageMessage to print usage instructions
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, usageMessage)
	}

	const (
		def_regexp		= ""
		def_prefix		= ""
		def_suffix		= ""
		def_index		= ""
		def_num			= 1
		def_target		= ""
		def_lowercase	= false
		def_uppercase	= false
		def_copy		= false
		def_dryrun		= false
		def_force		= false
	)

	flag.StringVar(&regexpArg, "regexp", def_regexp, "")
	flag.StringVar(&regexpArg, "x", def_regexp, "")
	flag.StringVar(&prefixArg, "prefix", def_prefix, "")
	flag.StringVar(&prefixArg, "p", def_prefix, "")
	flag.StringVar(&suffixArg, "suffix", def_suffix, "")
	flag.StringVar(&suffixArg, "s", def_suffix, "")
	flag.StringVar(&indexArg, "index", def_index, "")
	flag.StringVar(&indexArg, "i", def_index, "")
	flag.IntVar(&numArg, "startnumber", def_num, "")
	flag.IntVar(&numArg, "I", def_num, "")
	flag.StringVar(&targetArg, "target", def_target, "")
	flag.StringVar(&targetArg, "t", def_target, "")
	flag.BoolVar(&lowerArg, "lowercase", def_lowercase, "")
	flag.BoolVar(&lowerArg, "l", def_lowercase, "")
	flag.BoolVar(&upperArg, "uppercase", def_uppercase, "")
	flag.BoolVar(&upperArg, "u", def_uppercase, "")
	flag.BoolVar(&copyArg, "copy", def_copy, "")
	flag.BoolVar(&copyArg, "c", def_copy, "")
	flag.BoolVar(&dryrunArg, "dry-run", def_dryrun, "")
	flag.BoolVar(&dryrunArg, "n", def_dryrun, "")
	flag.BoolVar(&forceArg, "force", def_force, "")
	flag.BoolVar(&forceArg, "f", def_force, "")

	flag.Parse()

	if regexpArg == "" && prefixArg == "" && suffixArg == "" && indexArg == "" && lowerArg == false && upperArg == false {
		printUsage("At least a flag must be given, nothing to do...")
	}
}

// Write a renamed or a copy of a file to disk
func writeFile(oldname, newname string) {
	newname = filepath.Join(targetArg, newname)
	oldname = filepath.Join(targetArg, oldname)
	// check if the new filename is already present
	_, lstat_err := os.Lstat(newname)
	if lstat_err == nil && forceArg == false {
		fmt.Fprintf(os.Stderr, "File %s already exist! Use -force to override it\n", newname)
		operationSuccessful = operationSuccessful + 1
	}
	if dryrunArg {
		// if dry-run was given, just output the renaming operation
		fmt.Fprintf(os.Stdout, "Renaming %s to %s (dry-run)\n", oldname, newname)
		operationSuccessful = operationSuccessful + 0
	} else {
		if copyArg {
			copyf, create_err := os.Create(newname)
			if create_err != nil {
				fmt.Fprintf(os.Stderr, create_err.Error())
				operationSuccessful = operationSuccessful + 1
			}
			originalf, open_err := os.Open(oldname)
			if open_err != nil {
				fmt.Fprintf(os.Stderr, open_err.Error())
				operationSuccessful = operationSuccessful + 1
			}
			_, copy_err := io.Copy(copyf, originalf)
			if copy_err != nil {
				fmt.Fprintf(os.Stderr, "An error occurred during the copy of %s to %s\n", oldname, newname)
				fmt.Fprintf(os.Stderr, copy_err.Error())
				operationSuccessful = operationSuccessful + 1
			} else {
				fmt.Fprintf(os.Stdout, "Copying %s to %s\n", oldname, newname)
			}
		} else {
			rename_err := os.Rename(oldname, newname)
			if rename_err != nil {
				fmt.Fprintf(os.Stderr, "An error occurred during the renaming of %s to %s\n", oldname, newname)
				fmt.Fprintf(os.Stderr, rename_err.Error())
				operationSuccessful = operationSuccessful + 1
			} else {
				fmt.Fprintf(os.Stdout, "Renaming %s to %s\n", oldname, newname)
			}
		}
	}
}

// Add a prefix string to a name
func addPrefix(names []string, prefix string) int {
	var finalname string
	for _, f := range names {
		finalname = prefix + filepath.Base(f)
		writeFile(f, finalname)
	}
	return 0
}

// Add a suffix string to a name
func addSuffix(names []string, suffix string) int {
	var finalname string
	for _, f := range names {
		ext := filepath.Ext(f)
		justname := strings.TrimSuffix(filepath.Base(f), ext)
		finalname = justname + suffix + ext
		writeFile(f, finalname)
	}
	return 0
}

// Rename a slice of filenames to <newname><count>.<extension>
func indexName(names []string, newname string, count int) int {
	var finalname string
	for _, f := range names {
		ext := filepath.Ext(f)
		finalname = fmt.Sprintf("%s%03d%s", newname, count, ext)
		writeFile(f, finalname)
		count++
	}
	return 0
}

// Make filenames all lowercase
func lowercaseFiles(names []string) int {
	var finalname string
	for _, f := range names {
		finalname = strings.ToLower(filepath.Base(f))
		writeFile(f, finalname)
	}
	return 0
}

// Make filenames all uppercase
func uppercaseFiles(names []string) int {
	var finalname string
	for _, f := range names {
		finalname = strings.ToUpper(filepath.Base(f))
		writeFile(f, finalname)
	}
	return 0
}

// Get all files and directories
func getFilesFromDir(dirname string) ([]string, []string) {
	var complete_path string // final, absolute, path
	var filesindir = make([]os.FileInfo, 0) // files & directories found in path
	var allfiles = make([]string, 0)
	var alldirectories = make([]string, 0)

	dirinfo, lerr := os.Lstat(dirname)
	if lerr != nil {
		fmt.Fprintf(os.Stderr, lerr.Error())
		return alldirectories, allfiles
	}

	// check whether targetArg is an absolute path AND a directory
	if filepath.IsAbs(dirname) && dirinfo.IsDir() {
		complete_path = dirname
	} else {
		abs_path, err := filepath.Abs(dirname)
		if err != nil {
			fmt.Fprintf(os.Stderr, err.Error())
		}
		complete_path = abs_path
	}
	dir, err := os.Open(complete_path)
	defer dir.Close()
	if err != nil {
		err = errors.New(fmt.Sprintf("Target directory %s is not a directory or can't be accessed\n", complete_path))
		fmt.Fprintf(os.Stderr, err.Error())
		return alldirectories, allfiles
	}

	// scan for files/directories in path
	filesindir, read_err := dir.Readdir(0)
	if read_err != nil {
		fmt.Fprintf(os.Stderr, read_err.Error())
		return alldirectories, allfiles
	}

	// check in filesindir slice and separate directories from files in
	// 2 different slices
	for _, f := range filesindir {
		if f.IsDir() {
			alldirectories = append(alldirectories, f.Name())
		} else {
			allfiles = append(allfiles, f.Name())
		}
	}

	return alldirectories, allfiles
}

func renameFiles(dir, files []string) int {
	var basename string
	var matchingfiles []string // a slice containing only the files
					// matching the regexp passed as argument (if)
	var result int // the integer returned by each functions, signaling
					// success or failure

	//TODO: add a check for the `dir` slice and a recursiveArg to act
	//recursively on all subdirectories of targetDir. Save each files
	//with its full path or each one with the subdirectories' so it's
	//possible to rename/copy it with os.Rename later in writeFile()

	// check if the files should match a given regexp
	if regexpArg != "" {
		for _, f := range files {
		basename = filepath.Base(f)
			// Compile and check if the regexp is a valid one
			compRegexp, err := regexp.Compile(regexpArg)
			if err != nil {
				fmt.Fprintf(os.Stderr, "Invalid regexp: %s\n", regexpArg)
				printUsage("You must give a valid regexp (or none, to operate on all files). Alternatively, add -force to force renaming all files, whether they match the regexp or not")
				return 1
			} else {
				if compRegexp.MatchString(basename) == false {
					continue
				} else {
					matchingfiles = append(matchingfiles, f)
				}
			}
		}
	}

	// if matchingfiles contains something, iterate over it.
	// Otherwise, we simply use the generic files slice
	if len(matchingfiles) >= 1 {
		files = matchingfiles
	} else {
		// different messages whether we have forceArg == true or false
		if regexpArg != "" {
			if forceArg {
				fmt.Fprintf(os.Stdout, "No files matched, including all files anyway (-force enabled)\n")
			} else {
				fmt.Fprintf(os.Stderr, "No files matched, check the correctness of your regexp\n")
				return 1
			}
		}
	}

	if prefixArg != "" {
		result = addPrefix(files, prefixArg)
	}
	if suffixArg != "" {
		result = addSuffix(files, suffixArg)
	}
	if indexArg != "" {
		result = indexName(files, indexArg, numArg)
	}
	if lowerArg == true && upperArg == true {
		// can't use both
		printUsage("Can't use both lowercase and uppercase, choose one only!")
		return 1
	}
	if lowerArg == true {
		result = lowercaseFiles(files)
	}
	if upperArg == true {
		result = uppercaseFiles(files)
	}

	return result
}

func main() {
	var success_rename int
	var directories, files []string

	flagsInit()

	directories, files = getFilesFromDir(targetArg)

	success_rename = renameFiles(directories, files)

	// check that everything went smoothly
	if success_rename == 0  && operationSuccessful == 0 {
		fmt.Fprintf(os.Stdout, "\nRenaming complete\n")
	} else {
		fmt.Fprintf(os.Stdout, "\nNot all files were correctly renamed, check the previous error messages")
	}
}
<|endoftext|>


<commit_before>package godo

// Version is the current version
var Version = "1.3.0"
<commit_msg>bump version<commit_after>package godo

// Version is the current version
var Version = "1.3.1"
<|endoftext|>


<commit_before>package command

import (
	"github.com/chrislusf/seaweedfs/weed/security"
	"github.com/chrislusf/seaweedfs/weed/shell"
	"github.com/chrislusf/seaweedfs/weed/util"
	"github.com/spf13/viper"
)

var (
	shellOptions shell.ShellOptions
)

func init() {
	cmdShell.Run = runShell // break init cycle
	shellOptions.Masters = cmdShell.Flag.String("master", "localhost:9333", "comma-separated master servers")
}

var cmdShell = &Command{
	UsageLine: "shell",
	Short:     "run interactive administrative commands",
	Long: `run interactive administrative commands.

  `,
}

var ()

func runShell(command *Command, args []string) bool {

	util.LoadConfiguration("security", false)
	shellOptions.GrpcDialOption = security.LoadClientTLS(viper.Sub("grpc"), "client")

	shellOptions.FilerHost = "localhost"
	shellOptions.FilerPort = 8888
	shellOptions.Directory = "/"

	shell.RunShell(shellOptions)

	return true

}
<commit_msg> fix: weed shell add filer runtime attributes<commit_after>package command

import (
	"github.com/chrislusf/seaweedfs/weed/security"
	"github.com/chrislusf/seaweedfs/weed/shell"
	"github.com/chrislusf/seaweedfs/weed/util"
	"github.com/spf13/viper"
)

var (
	shellOptions shell.ShellOptions
)

func init() {
	cmdShell.Run = runShell // break init cycle
	shellOptions.Masters = cmdShell.Flag.String("master", "localhost:9333", "comma-separated master servers")
	filerHost := cmdShell.Flag.String("filer.host", "localhost", "comma-separated filer server host")
	flierPort := cmdShell.Flag.Int64("filer.port", 9333, "comma-separated filer server port")
	directory := cmdShell.Flag.String("filer.dir", "/", "comma-separated filer server directory")
	shellOptions.FilerHost = *filerHost
	shellOptions.FilerPort = *flierPort
	shellOptions.Directory = *directory
}

var cmdShell = &Command{
	UsageLine: "shell",
	Short:     "run interactive administrative commands",
	Long: `run interactive administrative commands.

  `,
}

func runShell(command *Command, args []string) bool {

	util.LoadConfiguration("security", false)
	shellOptions.GrpcDialOption = security.LoadClientTLS(viper.Sub("grpc"), "client")

	shell.RunShell(shellOptions)

	return true

}<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package populator implements interfaces that monitor and keep the states of the
// desired_state_of_word in sync with the "ground truth" from informer.
package populator

import (
	"time"

	"github.com/golang/glog"

	"k8s.io/kubernetes/pkg/api"
	kcache "k8s.io/kubernetes/pkg/client/cache"
	"k8s.io/kubernetes/pkg/controller/framework"
	"k8s.io/kubernetes/pkg/controller/volume/attachdetach/cache"
	"k8s.io/kubernetes/pkg/util/wait"
	"k8s.io/kubernetes/pkg/volume/util/volumehelper"
)

// DesiredStateOfWorldPopulator periodically verifies that the pods in the
// desired state of th world still exist, if not, it removes them.
// TODO: it also loops through the list of active pods and ensures that
// each one exists in the desired state of the world cache
// if it has volumes.
type DesiredStateOfWorldPopulator interface {
	Run(stopCh <-chan struct{})
}

// NewDesiredStateOfWorldPopulator returns a new instance of DesiredStateOfWorldPopulator.
// loopSleepDuration - the amount of time the populator loop sleeps between
//     successive executions
// podManager - the kubelet podManager that is the source of truth for the pods
//     that exist on this host
// desiredStateOfWorld - the cache to populate
func NewDesiredStateOfWorldPopulator(
	loopSleepDuration time.Duration,
	podInformer framework.SharedInformer,
	desiredStateOfWorld cache.DesiredStateOfWorld) DesiredStateOfWorldPopulator {
	return &desiredStateOfWorldPopulator{
		loopSleepDuration:   loopSleepDuration,
		podInformer:         podInformer,
		desiredStateOfWorld: desiredStateOfWorld,
	}
}

type desiredStateOfWorldPopulator struct {
	loopSleepDuration   time.Duration
	podInformer         framework.SharedInformer
	desiredStateOfWorld cache.DesiredStateOfWorld
}

func (dswp *desiredStateOfWorldPopulator) Run(stopCh <-chan struct{}) {
	wait.Until(dswp.populatorLoopFunc(), dswp.loopSleepDuration, stopCh)
}

func (dswp *desiredStateOfWorldPopulator) populatorLoopFunc() func() {
	return func() {
		dswp.findAndRemoveDeletedPods()
	}
}

// Iterate through all pods in desired state of world, and remove if they no
// longer exist in the informer
func (dswp *desiredStateOfWorldPopulator) findAndRemoveDeletedPods() {
	for dswPodUID, dswPodToAdd := range dswp.desiredStateOfWorld.GetPodToAdd() {
		dswPodKey, err := kcache.MetaNamespaceKeyFunc(dswPodToAdd.Pod)
		if err != nil {
			glog.Errorf("MetaNamespaceKeyFunc failed for pod %q (UID %q) with: %v", dswPodKey, dswPodUID, err)
			continue
		}
		// retrieve the pod object from pod informer with the namespace key
		informerPodObj, exists, err := dswp.podInformer.GetStore().GetByKey(dswPodKey)
		if err != nil || informerPodObj == nil {
			glog.Errorf("podInformer GetByKey failed for pod %q (UID %q) with %v", dswPodKey, dswPodUID, err)
			continue
		}
		if exists {
			informerPod, ok := informerPodObj.(*api.Pod)
			if !ok {
				glog.Errorf("Failed to cast obj %#v to pod object for pod %q (UID %q)", informerPod, dswPodKey, dswPodUID)
				continue
			}
			informerPodUID := volumehelper.GetUniquePodName(informerPod)
			// Check whether the unique idenfier of the pod from dsw matches the one retrived from pod informer
			if informerPodUID == dswPodUID {
				glog.V(10).Infof(
					"Verified pod %q (UID %q) from dsw exists in pod informer.", dswPodKey, dswPodUID)
				continue

			}
		}
		// the pod from dsw does not exist in pod informer, or it does not match the unique idenfier retrieved
		// from the informer, delete it from dsw
		glog.V(1).Infof(
			"Removing pod %q (UID %q) from dsw because it does not exist in pod informer.", dswPodKey, dswPodUID)
		dswp.desiredStateOfWorld.DeletePod(dswPodUID, dswPodToAdd.VolumeName, dswPodToAdd.NodeName)
	}
}
<commit_msg>Check pod exist in attach controller DSW populator<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package populator implements interfaces that monitor and keep the states of the
// desired_state_of_word in sync with the "ground truth" from informer.
package populator

import (
	"time"

	"github.com/golang/glog"

	"k8s.io/kubernetes/pkg/api"
	kcache "k8s.io/kubernetes/pkg/client/cache"
	"k8s.io/kubernetes/pkg/controller/framework"
	"k8s.io/kubernetes/pkg/controller/volume/attachdetach/cache"
	"k8s.io/kubernetes/pkg/util/wait"
	"k8s.io/kubernetes/pkg/volume/util/volumehelper"
)

// DesiredStateOfWorldPopulator periodically verifies that the pods in the
// desired state of th world still exist, if not, it removes them.
// TODO: it also loops through the list of active pods and ensures that
// each one exists in the desired state of the world cache
// if it has volumes.
type DesiredStateOfWorldPopulator interface {
	Run(stopCh <-chan struct{})
}

// NewDesiredStateOfWorldPopulator returns a new instance of DesiredStateOfWorldPopulator.
// loopSleepDuration - the amount of time the populator loop sleeps between
//     successive executions
// podManager - the kubelet podManager that is the source of truth for the pods
//     that exist on this host
// desiredStateOfWorld - the cache to populate
func NewDesiredStateOfWorldPopulator(
	loopSleepDuration time.Duration,
	podInformer framework.SharedInformer,
	desiredStateOfWorld cache.DesiredStateOfWorld) DesiredStateOfWorldPopulator {
	return &desiredStateOfWorldPopulator{
		loopSleepDuration:   loopSleepDuration,
		podInformer:         podInformer,
		desiredStateOfWorld: desiredStateOfWorld,
	}
}

type desiredStateOfWorldPopulator struct {
	loopSleepDuration   time.Duration
	podInformer         framework.SharedInformer
	desiredStateOfWorld cache.DesiredStateOfWorld
}

func (dswp *desiredStateOfWorldPopulator) Run(stopCh <-chan struct{}) {
	wait.Until(dswp.populatorLoopFunc(), dswp.loopSleepDuration, stopCh)
}

func (dswp *desiredStateOfWorldPopulator) populatorLoopFunc() func() {
	return func() {
		dswp.findAndRemoveDeletedPods()
	}
}

// Iterate through all pods in desired state of world, and remove if they no
// longer exist in the informer
func (dswp *desiredStateOfWorldPopulator) findAndRemoveDeletedPods() {
	for dswPodUID, dswPodToAdd := range dswp.desiredStateOfWorld.GetPodToAdd() {
		dswPodKey, err := kcache.MetaNamespaceKeyFunc(dswPodToAdd.Pod)
		if err != nil {
			glog.Errorf("MetaNamespaceKeyFunc failed for pod %q (UID %q) with: %v", dswPodKey, dswPodUID, err)
			continue
		}

		// Retrieve the pod object from pod informer with the namespace key
		informerPodObj, exists, err :=
			dswp.podInformer.GetStore().GetByKey(dswPodKey)
		if err != nil {
			glog.Errorf(
				"podInformer GetByKey failed for pod %q (UID %q) with %v",
				dswPodKey,
				dswPodUID,
				err)
			continue
		}
		if exists && informerPodObj == nil {
			glog.Info(
				"podInformer GetByKey found pod, but informerPodObj is nil for pod %q (UID %q)",
				dswPodKey,
				dswPodUID)
			continue
		}

		if exists {
			informerPod, ok := informerPodObj.(*api.Pod)
			if !ok {
				glog.Errorf("Failed to cast obj %#v to pod object for pod %q (UID %q)", informerPod, dswPodKey, dswPodUID)
				continue
			}
			informerPodUID := volumehelper.GetUniquePodName(informerPod)
			// Check whether the unique identifier of the pod from dsw matches the one retrieved from pod informer
			if informerPodUID == dswPodUID {
				glog.V(10).Infof(
					"Verified pod %q (UID %q) from dsw exists in pod informer.", dswPodKey, dswPodUID)
				continue

			}
		}
		// the pod from dsw does not exist in pod informer, or it does not match the unique identifer retrieved
		// from the informer, delete it from dsw
		glog.V(1).Infof(
			"Removing pod %q (UID %q) from dsw because it does not exist in pod informer.", dswPodKey, dswPodUID)
		dswp.desiredStateOfWorld.DeletePod(dswPodUID, dswPodToAdd.VolumeName, dswPodToAdd.NodeName)
	}
}
<|endoftext|>


<commit_before>package datastoreapi

// TODO: Support other request/response formats besides JSON (e.g., xml, gob)
// TODO: Figure out if PropertyList can support nested objects, or fail if they are detected.
// TODO: Add rudimentary single-property queries, pagination, sorting, etc.
// TODO: Allow clients to specify the Kind? Namespace datastore by user identity (and maintain user identity)?

import (
	"appengine"
	"appengine/datastore"
	"appengine/urlfetch"
	"encoding/json"
	"io"
	"net/http"
	"strconv"
	"time"
)

const (
	kind         = "JsonObject"
	idKey        = "_id"
	createdKey   = "_created"
	defaultLimit = 10
)

func init() {
	http.HandleFunc("/datastore/v1dev/objects", datastoreApi)
	http.HandleFunc("/datastore/v1dev/objects/", datastoreApi)
}

type UserQuery struct {
	Limit, Offset                      int
	FilterKey, FilterType, FilterValue string
	Cursor                             string
}

type UserInfo struct {
	Id string
}

func getUserId(accessToken string, client http.Client) (string, error) {
	resp, err := client.Get("https://www.googleapis.com/oauth2/v1/userinfo?access_token=" + accessToken)
	if err != nil {
		return "", err
	}
	var info UserInfo
	if err := json.NewDecoder(resp.Body).Decode(&info); err != nil {
		return "", err
	}
	return info.Id, nil
}

// datastoreApi dispatches requests to the relevant API method and arranges certain common state
func datastoreApi(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	w.Header().Add("Access-Control-Allow-Origin", "*")

	r.ParseForm()
	client := urlfetch.Client(c)

	// Get the access_token from the request and turn it into a user ID with which we will namespace Kinds in the datastore.
	accessToken := r.Form.Get("access_token")
	if accessToken == "" {
		http.Error(w, "Unauthorized", http.StatusUnauthorized)
		return
	}
	_ /*userId*/, err := getUserId(accessToken, *client)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	if r.URL.Path == "/datastore/v1dev/objects" {
		switch r.Method {
		case "POST":
			insert(w, r.Body, c)
			return
		case "GET":
			// TODO: Parse user request into UserQuery and pass to list method
			list(w, UserQuery{}, c)
			return
		}
	} else {
		sid := r.URL.Path[len("/datastore/v1dev/objects/"):]
		if sid == "" {
			http.Error(w, "Must specify ID", http.StatusBadRequest)
			return
		}
		id, err := strconv.ParseInt(sid, 10, 64)
		if err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}

		switch r.Method {
		case "GET":
			get(w, id, c)
			return
		case "DELETE":
			delete(w, id, c)
			return
		case "POST":
			// This is strictly "replace all properties/values", not "add new properties, update existing"
			update(w, id, r.Body, c)
			return
		}
	}
	http.Error(w, "Unsupported Method", http.StatusMethodNotAllowed)
}

func delete(w http.ResponseWriter, id int64, c appengine.Context) {
	k := datastore.NewKey(c, kind, "", id, nil)
	if err := datastore.Delete(c, k); err != nil {
		if err == datastore.ErrNoSuchEntity {
			http.Error(w, "Not Found", http.StatusNotFound)
		} else {
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}
		return
	}
}

func get(w http.ResponseWriter, id int64, c appengine.Context) {
	k := datastore.NewKey(c, kind, "", id, nil)
	var plist datastore.PropertyList
	if err := datastore.Get(c, k, &plist); err != nil {
		if err == datastore.ErrNoSuchEntity {
			http.Error(w, "Not Found", http.StatusNotFound)
		} else {
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}
		return
	}
	m := plistToMap(plist, k)
	json.NewEncoder(w).Encode(m)
}

func insert(w http.ResponseWriter, r io.Reader, c appengine.Context) {
	var m map[string]interface{}
	if err := json.NewDecoder(r).Decode(&m); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	m[createdKey] = time.Now()

	plist := mapToPlist(m)

	k := datastore.NewIncompleteKey(c, kind, nil)
	k, err := datastore.Put(c, k, &plist)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	m[idKey] = k.IntID()
	json.NewEncoder(w).Encode(m)
}

// plistToMap transforms a PropertyList such as you would get from the datastore into a map[string]interface{} suitable for JSON-encoding.
func plistToMap(plist datastore.PropertyList, k *datastore.Key) map[string]interface{} {
	m := make(map[string]interface{})
	for _, p := range plist {
		if _, exists := m[p.Name]; exists {
			if _, isArr := m[p.Name].([]interface{}); isArr {
				m[p.Name] = append(m[p.Name].([]interface{}), p.Value)
			} else {
				m[p.Name] = []interface{}{m[p.Name], p.Value}
			}
		} else {
			m[p.Name] = p.Value
		}
	}
	m[idKey] = k.IntID()
	return m
}

// mapToPlist transforms a map[string]interface{} such as you would get from decoding JSON into a PropertyList to store in the datastore.
func mapToPlist(m map[string]interface{}) datastore.PropertyList {
	plist := make(datastore.PropertyList, 0, len(m))
	for k, v := range m {
		if _, mult := v.([]interface{}); mult {
			for _, mv := range v.([]interface{}) {
				plist = append(plist, datastore.Property{
					Name:     k,
					Value:    mv,
					Multiple: true,
				})
			}
		} else {
			plist = append(plist, datastore.Property{
				Name:  k,
				Value: v,
			})
		}
	}
	return plist
}

func list(w http.ResponseWriter, uq UserQuery, c appengine.Context) {
	limit := 3
	q := datastore.NewQuery(kind).Limit(limit)

	items := make([]map[string]interface{}, 0, limit)

	var crs datastore.Cursor
	for t := q.Run(c); ; {
		var plist datastore.PropertyList
		k, err := t.Next(&plist)
		if err == datastore.Done {
			break
		}
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		m := plistToMap(plist, k)
		items = append(items, m)
		if crs, err = t.Cursor(); err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
	}
	r := map[string]interface{}{
		"items":          items,
		"nextStartToken": crs.String(),
	}
	json.NewEncoder(w).Encode(r)
}

func update(w http.ResponseWriter, id int64, r io.Reader, c appengine.Context) {
	var m map[string]interface{}
	if err := json.NewDecoder(r).Decode(&m); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	plist := mapToPlist(m)

	k := datastore.NewKey(c, kind, "", id, nil)
	if _, err := datastore.Put(c, k, &plist); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	m[idKey] = id
	json.NewEncoder(w).Encode(m)
}
<commit_msg>Parse kind/ID from path -- changes made via Cloud9, nothing has been tested<commit_after>package datastoreapi

// TODO: Support other request/response formats besides JSON (e.g., xml, gob)
// TODO: Figure out if PropertyList can support nested objects, or fail if they are detected.
// TODO: Add rudimentary single-property queries, pagination, sorting, etc.
// TODO: Allow clients to specify the Kind? Namespace datastore by user identity (and maintain user identity)?

import (
	"appengine"
	"appengine/datastore"
	"appengine/urlfetch"
	"encoding/json"
	"io"
	"net/http"
	"strconv"
	"time"
)

const (
	kind         = "JsonObject"
	idKey        = "_id"
	createdKey   = "_created"
	defaultLimit = 10
)

func init() {
	http.HandleFunc("/datastore/v1dev/objects/", datastoreApi)
}

type UserQuery struct {
	Limit, Offset                      int
	FilterKey, FilterType, FilterValue string
	Cursor                             string
}

type UserInfo struct {
	Id string
}

func getUserId(accessToken string, client http.Client) (id string, err error) {
	resp, err := client.Get("https://www.googleapis.com/oauth2/v1/userinfo?access_token=" + accessToken)
	if err != nil {
		return
	}
	var info UserInfo
	if err = json.NewDecoder(resp.Body).Decode(&info); err != nil {
		return
	}
    id = info.Id
    return
}

func getKind(path string) string {
	return path[len("/datastore/v1dev/objects/"):]
}

func getKindAndId(path string) (kind string, id int64, err error) {
    kind = path[len("/datastore/v1dev/objects/"):strings.LastIndex(path, "/")]
	idStr := path[strings.LastIndex(path, "/")+1:]
	id, err = strconv.ParseInt(idStr, 10, 64)
	if err != nil {
		return
	}
	return
}

// datastoreApi dispatches requests to the relevant API method and arranges certain common state
func datastoreApi(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	w.Header().Add("Access-Control-Allow-Origin", "*")

	r.ParseForm()
	client := urlfetch.Client(c)

	// Get the access_token from the request and turn it into a user ID with which we will namespace Kinds in the datastore.
	accessToken := r.Form.Get("access_token")
	if accessToken == "" {
		http.Error(w, "Unauthorized", http.StatusUnauthorized)
		return
	}
	_ /*userId*/, err := getUserId(accessToken, *client)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	if r.URL.Path == "/datastore/v1dev/objects" {
		switch r.Method {
		case "POST":
			insert(w, r.Body, c)
			return
		case "GET":
			// TODO: Parse user request into UserQuery and pass to list method
			list(w, UserQuery{}, c)
			return
		}
	} else {
		sid := r.URL.Path[len("/datastore/v1dev/objects/"):]
		if sid == "" {
			http.Error(w, "Must specify ID", http.StatusBadRequest)
			return
		}
		id, err := strconv.ParseInt(sid, 10, 64)
		if err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}

		switch r.Method {
		case "GET":
			get(w, id, c)
			return
		case "DELETE":
			delete(w, id, c)
			return
		case "POST":
			// This is strictly "replace all properties/values", not "add new properties, update existing"
			update(w, id, r.Body, c)
			return
		}
	}
	http.Error(w, "Unsupported Method", http.StatusMethodNotAllowed)
}

func delete(w http.ResponseWriter, id int64, c appengine.Context) {
	k := datastore.NewKey(c, kind, "", id, nil)
	if err := datastore.Delete(c, k); err != nil {
		if err == datastore.ErrNoSuchEntity {
			http.Error(w, "Not Found", http.StatusNotFound)
		} else {
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}
		return
	}
}

func get(w http.ResponseWriter, id int64, c appengine.Context) {
	k := datastore.NewKey(c, kind, "", id, nil)
	var plist datastore.PropertyList
	if err := datastore.Get(c, k, &plist); err != nil {
		if err == datastore.ErrNoSuchEntity {
			http.Error(w, "Not Found", http.StatusNotFound)
		} else {
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}
		return
	}
	m := plistToMap(plist, k)
	json.NewEncoder(w).Encode(m)
}

func insert(w http.ResponseWriter, r io.Reader, c appengine.Context) {
	var m map[string]interface{}
	if err := json.NewDecoder(r).Decode(&m); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	m[createdKey] = time.Now()

	plist := mapToPlist(m)

	k := datastore.NewIncompleteKey(c, kind, nil)
	k, err := datastore.Put(c, k, &plist)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	m[idKey] = k.IntID()
	json.NewEncoder(w).Encode(m)
}

// plistToMap transforms a PropertyList such as you would get from the datastore into a map[string]interface{} suitable for JSON-encoding.
func plistToMap(plist datastore.PropertyList, k *datastore.Key) map[string]interface{} {
	m := make(map[string]interface{})
	for _, p := range plist {
		if _, exists := m[p.Name]; exists {
			if _, isArr := m[p.Name].([]interface{}); isArr {
				m[p.Name] = append(m[p.Name].([]interface{}), p.Value)
			} else {
				m[p.Name] = []interface{}{m[p.Name], p.Value}
			}
		} else {
			m[p.Name] = p.Value
		}
	}
	m[idKey] = k.IntID()
	return m
}

// mapToPlist transforms a map[string]interface{} such as you would get from decoding JSON into a PropertyList to store in the datastore.
func mapToPlist(m map[string]interface{}) datastore.PropertyList {
	plist := make(datastore.PropertyList, 0, len(m))
	for k, v := range m {
		if _, mult := v.([]interface{}); mult {
			for _, mv := range v.([]interface{}) {
				plist = append(plist, datastore.Property{
					Name:     k,
					Value:    mv,
					Multiple: true,
				})
			}
		} else {
			plist = append(plist, datastore.Property{
				Name:  k,
				Value: v,
			})
		}
	}
	return plist
}

func list(w http.ResponseWriter, uq UserQuery, c appengine.Context) {
	limit := 3
	q := datastore.NewQuery(kind).Limit(limit)

	items := make([]map[string]interface{}, 0, limit)

	var crs datastore.Cursor
	for t := q.Run(c); ; {
		var plist datastore.PropertyList
		k, err := t.Next(&plist)
		if err == datastore.Done {
			break
		}
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		m := plistToMap(plist, k)
		items = append(items, m)
		if crs, err = t.Cursor(); err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
	}
	r := map[string]interface{}{
		"items":          items,
		"nextStartToken": crs.String(),
	}
	json.NewEncoder(w).Encode(r)
}

func update(w http.ResponseWriter, id int64, r io.Reader, c appengine.Context) {
	var m map[string]interface{}
	if err := json.NewDecoder(r).Decode(&m); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	plist := mapToPlist(m)

	k := datastore.NewKey(c, kind, "", id, nil)
	if _, err := datastore.Put(c, k, &plist); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	m[idKey] = id
	json.NewEncoder(w).Encode(m)
}
<|endoftext|>