<commit_before>// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package service handles the command-line, configuration, and runs the
// OpenTelemetry Collector.
package service

import (
	"context"
	"errors"
	"flag"
	"fmt"
	"os"
	"os/signal"
	"runtime"
	"syscall"

	"github.com/spf13/cobra"
	"go.uber.org/zap"

	"go.opentelemetry.io/collector/component"
	"go.opentelemetry.io/collector/config/configcheck"
	"go.opentelemetry.io/collector/config/configloader"
	"go.opentelemetry.io/collector/config/configtelemetry"
	"go.opentelemetry.io/collector/config/experimental/configsource"
	"go.opentelemetry.io/collector/consumer/consumererror"
	"go.opentelemetry.io/collector/internal/collector/telemetry"
	"go.opentelemetry.io/collector/service/internal/builder"
	"go.opentelemetry.io/collector/service/parserprovider"
)

const (
	servicezPath   = "servicez"
	pipelinezPath  = "pipelinez"
	extensionzPath = "extensionz"
)

// State defines Collector's state.
type State int

const (
	Starting State = iota
	Running
	Closing
	Closed
)

// Collector represents a server providing the OpenTelemetry Collector service.
type Collector struct {
	info    component.BuildInfo
	rootCmd *cobra.Command
	logger  *zap.Logger

	service      *service
	stateChannel chan State

	factories component.Factories

	parserProvider parserprovider.ParserProvider

	// stopTestChan is used to terminate the collector server in end to end tests.
	stopTestChan chan struct{}

	// signalsChannel is used to receive termination signals from the OS.
	signalsChannel chan os.Signal

	// asyncErrorChannel is used to signal a fatal error from any component.
	asyncErrorChannel chan error
}

// New creates and returns a new instance of Collector.
func New(set CollectorSettings) (*Collector, error) {
	if err := configcheck.ValidateConfigFromFactories(set.Factories); err != nil {
		return nil, err
	}

	col := &Collector{
		info:         set.BuildInfo,
		factories:    set.Factories,
		stateChannel: make(chan State, Closed+1),
	}

	rootCmd := &cobra.Command{
		Use:     set.BuildInfo.Command,
		Version: set.BuildInfo.Version,
		RunE: func(cmd *cobra.Command, args []string) error {
			var err error
			if col.logger, err = newLogger(set.LoggingOptions); err != nil {
				return fmt.Errorf("failed to get logger: %w", err)
			}

			return col.execute(cmd.Context())
		},
	}

	// TODO: coalesce this code and expose this information to other components.
	flagSet := new(flag.FlagSet)
	addFlagsFns := []func(*flag.FlagSet){
		configtelemetry.Flags,
		parserprovider.Flags,
		telemetry.Flags,
		builder.Flags,
		loggerFlags,
	}
	for _, addFlags := range addFlagsFns {
		addFlags(flagSet)
	}
	rootCmd.Flags().AddGoFlagSet(flagSet)
	col.rootCmd = rootCmd

	parserProvider := set.ParserProvider
	if parserProvider == nil {
		// use default provider.
		parserProvider = parserprovider.Default()
	}
	col.parserProvider = parserProvider

	return col, nil
}

// Run starts the collector according to the command and configuration
// given by the user, and waits for it to complete.
func (col *Collector) Run() error {
	// From this point on do not show usage in case of error.
	col.rootCmd.SilenceUsage = true

	return col.rootCmd.Execute()
}

// GetStateChannel returns state channel of the collector server.
func (col *Collector) GetStateChannel() chan State {
	return col.stateChannel
}

// Command returns Collector's root command.
func (col *Collector) Command() *cobra.Command {
	return col.rootCmd
}

// GetLogger returns logger used by the Collector.
// The logger is initialized after collector server start.
func (col *Collector) GetLogger() *zap.Logger {
	return col.logger
}

// Shutdown shuts down the collector server.
func (col *Collector) Shutdown() {
	// TODO: Implement a proper shutdown with graceful draining of the pipeline.
	// See https://github.com/open-telemetry/opentelemetry-collector/issues/483.
	defer func() {
		if r := recover(); r != nil {
			col.logger.Info("stopTestChan already closed")
		}
	}()
	close(col.stopTestChan)
}

func (col *Collector) setupTelemetry(ballastSizeBytes uint64) error {
	col.logger.Info("Setting up own telemetry...")

	err := applicationTelemetry.init(col.asyncErrorChannel, ballastSizeBytes, col.logger)
	if err != nil {
		return fmt.Errorf("failed to initialize telemetry: %w", err)
	}

	return nil
}

// runAndWaitForShutdownEvent waits for one of the shutdown events that can happen.
func (col *Collector) runAndWaitForShutdownEvent() {
	col.logger.Info("Everything is ready. Begin running and processing data.")

	// plug SIGTERM signal into a channel.
	col.signalsChannel = make(chan os.Signal, 1)
	signal.Notify(col.signalsChannel, os.Interrupt, syscall.SIGTERM)

	// set the channel to stop testing.
	col.stopTestChan = make(chan struct{})
	col.stateChannel <- Running
	select {
	case err := <-col.asyncErrorChannel:
		col.logger.Error("Asynchronous error received, terminating process", zap.Error(err))
	case s := <-col.signalsChannel:
		col.logger.Info("Received signal from OS", zap.String("signal", s.String()))
	case <-col.stopTestChan:
		col.logger.Info("Received stop test request")
	}
	col.stateChannel <- Closing
}

// setupConfigurationComponents loads the config and starts the components. If all the steps succeeds it
// sets the col.service with the service currently running.
func (col *Collector) setupConfigurationComponents(ctx context.Context) error {
	col.logger.Info("Loading configuration...")

	cp, err := col.parserProvider.Get()
	if err != nil {
		return fmt.Errorf("cannot load configuration's parser: %w", err)
	}

	cfg, err := configloader.Load(cp, col.factories)
	if err != nil {
		return fmt.Errorf("cannot load configuration: %w", err)
	}

	if err = cfg.Validate(); err != nil {
		return fmt.Errorf("invalid configuration: %w", err)
	}

	col.logger.Info("Applying configuration...")

	service, err := newService(&svcSettings{
		BuildInfo:         col.info,
		Factories:         col.factories,
		Config:            cfg,
		Logger:            col.logger,
		AsyncErrorChannel: col.asyncErrorChannel,
	})
	if err != nil {
		return err
	}

	err = service.Start(ctx)
	if err != nil {
		return err
	}

	col.service = service

	// If provider is watchable start a goroutine watching for updates.
	if watchable, ok := col.parserProvider.(parserprovider.Watchable); ok {
		go func() {
			err := watchable.WatchForUpdate()
			switch {
			// TODO: Move configsource.ErrSessionClosed to providerparser package to avoid depending on configsource.
			case errors.Is(err, configsource.ErrSessionClosed):
				// This is the case of shutdown of the whole collector server, nothing to do.
				col.logger.Info("Config WatchForUpdate closed", zap.Error(err))
				return
			default:
				col.logger.Warn("Config WatchForUpdated exited", zap.Error(err))
				col.reloadService(context.Background())
			}
		}()
	}

	return nil
}

func (col *Collector) execute(ctx context.Context) error {
	col.logger.Info("Starting "+col.info.Command+"...",
		zap.String("Version", col.info.Version),
		zap.Int("NumCPU", runtime.NumCPU()),
	)
	col.stateChannel <- Starting

	// Set memory ballast
	ballast, ballastSizeBytes := col.createMemoryBallast()

	col.asyncErrorChannel = make(chan error)

	// Setup everything.
	err := col.setupTelemetry(ballastSizeBytes)
	if err != nil {
		return err
	}

	err = col.setupConfigurationComponents(ctx)
	if err != nil {
		return err
	}

	// Everything is ready, now run until an event requiring shutdown happens.
	col.runAndWaitForShutdownEvent()

	// Accumulate errors and proceed with shutting down remaining components.
	var errs []error

	// Begin shutdown sequence.
	runtime.KeepAlive(ballast)
	col.logger.Info("Starting shutdown...")

	if closable, ok := col.parserProvider.(parserprovider.Closeable); ok {
		if err := closable.Close(ctx); err != nil {
			errs = append(errs, fmt.Errorf("failed to close config: %w", err))
		}
	}

	if col.service != nil {
		if err := col.service.Shutdown(ctx); err != nil {
			errs = append(errs, fmt.Errorf("failed to shutdown service: %w", err))
		}
	}

	if err := applicationTelemetry.shutdown(); err != nil {
		errs = append(errs, fmt.Errorf("failed to shutdown application telemetry: %w", err))
	}

	col.logger.Info("Shutdown complete.")
	col.stateChannel <- Closed
	close(col.stateChannel)

	return consumererror.Combine(errs)
}

func (col *Collector) createMemoryBallast() ([]byte, uint64) {
	ballastSizeMiB := builder.MemBallastSize()
	if ballastSizeMiB > 0 {
		ballastSizeBytes := uint64(ballastSizeMiB) * 1024 * 1024
		ballast := make([]byte, ballastSizeBytes)
		col.logger.Info("Using memory ballast", zap.Int("MiBs", ballastSizeMiB))
		return ballast, ballastSizeBytes
	}
	return nil, 0
}

// reloadService shutdowns the current col.service and setups a new one according
// to the latest configuration. It requires that col.parserProvider and col.factories
// are properly populated to finish successfully.
func (col *Collector) reloadService(ctx context.Context) error {
	if closeable, ok := col.parserProvider.(parserprovider.Closeable); ok {
		if err := closeable.Close(ctx); err != nil {
			return fmt.Errorf("failed close current config provider: %w", err)
		}
	}

	if col.service != nil {
		retiringService := col.service
		col.service = nil
		if err := retiringService.Shutdown(ctx); err != nil {
			return fmt.Errorf("failed to shutdown the retiring config: %w", err)
		}
	}

	if err := col.setupConfigurationComponents(ctx); err != nil {
		return fmt.Errorf("failed to setup configuration components: %w", err)
	}

	return nil
}
<commit_msg>Rename the shutdown channel (#3578)<commit_after>// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package service handles the command-line, configuration, and runs the
// OpenTelemetry Collector.
package service

import (
	"context"
	"errors"
	"flag"
	"fmt"
	"os"
	"os/signal"
	"runtime"
	"syscall"

	"github.com/spf13/cobra"
	"go.uber.org/zap"

	"go.opentelemetry.io/collector/component"
	"go.opentelemetry.io/collector/config/configcheck"
	"go.opentelemetry.io/collector/config/configloader"
	"go.opentelemetry.io/collector/config/configtelemetry"
	"go.opentelemetry.io/collector/config/experimental/configsource"
	"go.opentelemetry.io/collector/consumer/consumererror"
	"go.opentelemetry.io/collector/internal/collector/telemetry"
	"go.opentelemetry.io/collector/service/internal/builder"
	"go.opentelemetry.io/collector/service/parserprovider"
)

const (
	servicezPath   = "servicez"
	pipelinezPath  = "pipelinez"
	extensionzPath = "extensionz"
)

// State defines Collector's state.
type State int

const (
	Starting State = iota
	Running
	Closing
	Closed
)

// Collector represents a server providing the OpenTelemetry Collector service.
type Collector struct {
	info    component.BuildInfo
	rootCmd *cobra.Command
	logger  *zap.Logger

	service      *service
	stateChannel chan State

	factories component.Factories

	parserProvider parserprovider.ParserProvider

	// shutdownChan is used to terminate the collector.
	shutdownChan chan struct{}

	// signalsChannel is used to receive termination signals from the OS.
	signalsChannel chan os.Signal

	// asyncErrorChannel is used to signal a fatal error from any component.
	asyncErrorChannel chan error
}

// New creates and returns a new instance of Collector.
func New(set CollectorSettings) (*Collector, error) {
	if err := configcheck.ValidateConfigFromFactories(set.Factories); err != nil {
		return nil, err
	}

	col := &Collector{
		info:         set.BuildInfo,
		factories:    set.Factories,
		stateChannel: make(chan State, Closed+1),
	}

	rootCmd := &cobra.Command{
		Use:     set.BuildInfo.Command,
		Version: set.BuildInfo.Version,
		RunE: func(cmd *cobra.Command, args []string) error {
			var err error
			if col.logger, err = newLogger(set.LoggingOptions); err != nil {
				return fmt.Errorf("failed to get logger: %w", err)
			}

			return col.execute(cmd.Context())
		},
	}

	// TODO: coalesce this code and expose this information to other components.
	flagSet := new(flag.FlagSet)
	addFlagsFns := []func(*flag.FlagSet){
		configtelemetry.Flags,
		parserprovider.Flags,
		telemetry.Flags,
		builder.Flags,
		loggerFlags,
	}
	for _, addFlags := range addFlagsFns {
		addFlags(flagSet)
	}
	rootCmd.Flags().AddGoFlagSet(flagSet)
	col.rootCmd = rootCmd

	parserProvider := set.ParserProvider
	if parserProvider == nil {
		// use default provider.
		parserProvider = parserprovider.Default()
	}
	col.parserProvider = parserProvider

	return col, nil
}

// Run starts the collector according to the command and configuration
// given by the user, and waits for it to complete.
func (col *Collector) Run() error {
	// From this point on do not show usage in case of error.
	col.rootCmd.SilenceUsage = true

	return col.rootCmd.Execute()
}

// GetStateChannel returns state channel of the collector server.
func (col *Collector) GetStateChannel() chan State {
	return col.stateChannel
}

// Command returns Collector's root command.
func (col *Collector) Command() *cobra.Command {
	return col.rootCmd
}

// GetLogger returns logger used by the Collector.
// The logger is initialized after collector server start.
func (col *Collector) GetLogger() *zap.Logger {
	return col.logger
}

// Shutdown shuts down the collector server.
func (col *Collector) Shutdown() {
	// TODO: Implement a proper shutdown with graceful draining of the pipeline.
	// See https://github.com/open-telemetry/opentelemetry-collector/issues/483.
	defer func() {
		if r := recover(); r != nil {
			col.logger.Info("shutdownChan already closed")
		}
	}()
	close(col.shutdownChan)
}

func (col *Collector) setupTelemetry(ballastSizeBytes uint64) error {
	col.logger.Info("Setting up own telemetry...")

	err := applicationTelemetry.init(col.asyncErrorChannel, ballastSizeBytes, col.logger)
	if err != nil {
		return fmt.Errorf("failed to initialize telemetry: %w", err)
	}

	return nil
}

// runAndWaitForShutdownEvent waits for one of the shutdown events that can happen.
func (col *Collector) runAndWaitForShutdownEvent() {
	col.logger.Info("Everything is ready. Begin running and processing data.")

	// plug SIGTERM signal into a channel.
	col.signalsChannel = make(chan os.Signal, 1)
	signal.Notify(col.signalsChannel, os.Interrupt, syscall.SIGTERM)

	col.shutdownChan = make(chan struct{})
	col.stateChannel <- Running
	select {
	case err := <-col.asyncErrorChannel:
		col.logger.Error("Asynchronous error received, terminating process", zap.Error(err))
	case s := <-col.signalsChannel:
		col.logger.Info("Received signal from OS", zap.String("signal", s.String()))
	case <-col.shutdownChan:
		col.logger.Info("Received stop test request")
	}
	col.stateChannel <- Closing
}

// setupConfigurationComponents loads the config and starts the components. If all the steps succeeds it
// sets the col.service with the service currently running.
func (col *Collector) setupConfigurationComponents(ctx context.Context) error {
	col.logger.Info("Loading configuration...")

	cp, err := col.parserProvider.Get()
	if err != nil {
		return fmt.Errorf("cannot load configuration's parser: %w", err)
	}

	cfg, err := configloader.Load(cp, col.factories)
	if err != nil {
		return fmt.Errorf("cannot load configuration: %w", err)
	}

	if err = cfg.Validate(); err != nil {
		return fmt.Errorf("invalid configuration: %w", err)
	}

	col.logger.Info("Applying configuration...")

	service, err := newService(&svcSettings{
		BuildInfo:         col.info,
		Factories:         col.factories,
		Config:            cfg,
		Logger:            col.logger,
		AsyncErrorChannel: col.asyncErrorChannel,
	})
	if err != nil {
		return err
	}

	err = service.Start(ctx)
	if err != nil {
		return err
	}

	col.service = service

	// If provider is watchable start a goroutine watching for updates.
	if watchable, ok := col.parserProvider.(parserprovider.Watchable); ok {
		go func() {
			err := watchable.WatchForUpdate()
			switch {
			// TODO: Move configsource.ErrSessionClosed to providerparser package to avoid depending on configsource.
			case errors.Is(err, configsource.ErrSessionClosed):
				// This is the case of shutdown of the whole collector server, nothing to do.
				col.logger.Info("Config WatchForUpdate closed", zap.Error(err))
				return
			default:
				col.logger.Warn("Config WatchForUpdated exited", zap.Error(err))
				col.reloadService(context.Background())
			}
		}()
	}

	return nil
}

func (col *Collector) execute(ctx context.Context) error {
	col.logger.Info("Starting "+col.info.Command+"...",
		zap.String("Version", col.info.Version),
		zap.Int("NumCPU", runtime.NumCPU()),
	)
	col.stateChannel <- Starting

	// Set memory ballast
	ballast, ballastSizeBytes := col.createMemoryBallast()

	col.asyncErrorChannel = make(chan error)

	// Setup everything.
	err := col.setupTelemetry(ballastSizeBytes)
	if err != nil {
		return err
	}

	err = col.setupConfigurationComponents(ctx)
	if err != nil {
		return err
	}

	// Everything is ready, now run until an event requiring shutdown happens.
	col.runAndWaitForShutdownEvent()

	// Accumulate errors and proceed with shutting down remaining components.
	var errs []error

	// Begin shutdown sequence.
	runtime.KeepAlive(ballast)
	col.logger.Info("Starting shutdown...")

	if closable, ok := col.parserProvider.(parserprovider.Closeable); ok {
		if err := closable.Close(ctx); err != nil {
			errs = append(errs, fmt.Errorf("failed to close config: %w", err))
		}
	}

	if col.service != nil {
		if err := col.service.Shutdown(ctx); err != nil {
			errs = append(errs, fmt.Errorf("failed to shutdown service: %w", err))
		}
	}

	if err := applicationTelemetry.shutdown(); err != nil {
		errs = append(errs, fmt.Errorf("failed to shutdown application telemetry: %w", err))
	}

	col.logger.Info("Shutdown complete.")
	col.stateChannel <- Closed
	close(col.stateChannel)

	return consumererror.Combine(errs)
}

func (col *Collector) createMemoryBallast() ([]byte, uint64) {
	ballastSizeMiB := builder.MemBallastSize()
	if ballastSizeMiB > 0 {
		ballastSizeBytes := uint64(ballastSizeMiB) * 1024 * 1024
		ballast := make([]byte, ballastSizeBytes)
		col.logger.Info("Using memory ballast", zap.Int("MiBs", ballastSizeMiB))
		return ballast, ballastSizeBytes
	}
	return nil, 0
}

// reloadService shutdowns the current col.service and setups a new one according
// to the latest configuration. It requires that col.parserProvider and col.factories
// are properly populated to finish successfully.
func (col *Collector) reloadService(ctx context.Context) error {
	if closeable, ok := col.parserProvider.(parserprovider.Closeable); ok {
		if err := closeable.Close(ctx); err != nil {
			return fmt.Errorf("failed close current config provider: %w", err)
		}
	}

	if col.service != nil {
		retiringService := col.service
		col.service = nil
		if err := retiringService.Shutdown(ctx); err != nil {
			return fmt.Errorf("failed to shutdown the retiring config: %w", err)
		}
	}

	if err := col.setupConfigurationComponents(ctx); err != nil {
		return fmt.Errorf("failed to setup configuration components: %w", err)
	}

	return nil
}
<|endoftext|>


<commit_before>package mysql

import (
	"bufio"
	"encoding/binary"
	"os"
	"fmt"
	"bytes"
	"strconv"
)

type MySQLStatement struct {
	StatementId uint32
	Columns     uint16
	Parameters  uint16
	Warnings    uint16
	FieldCount  uint8

	ResultSet *MySQLResultSet
	mysql     *MySQLInstance
}

// Encode values fro each field
func encodeParamValues(a []interface{}) ([]byte, int) {
	var b []byte
	for i := 0; i < len(a); i++ {
		f := a[i];
		switch t := f.(type) {
		case string:
			b = bytes.Add(b, packString(string(t)))
		case int:
			b = bytes.Add(b, packString(strconv.Itoa(int(t))))
		case float:
			b = bytes.Add(b, packString(strconv.Ftoa(float(t), 'f', -1)))
		}
	}
	return b, len(b)
}

func putuint16(b []byte, v uint16) {
	b[0] = byte(v)
	b[1] = byte(v >> 8)
}

// For each field encode 2 byte type code. First bit is signed/unsigned
// Cheats and only bind parameters as strings
func encodeParamTypes(a []interface{}) ([]byte, int) {
	buf := make([]byte, len(a)*2)
	off := 0
	for i := 0; i < len(a); i++ {
		f := a[i];
		if f == nil {
			putuint16(buf[off:off+2], uint16(MYSQL_TYPE_NULL))
			continue
		}
		putuint16(buf[off:off+2], uint16(MYSQL_TYPE_STRING))
		off += 2
	}
	return buf, off
}

func readPrepareInit(br *bufio.Reader) (*MySQLStatement, os.Error) {
	ph, err := readHeader(br)
	if err != nil {
		return nil, err
	}
	s := new(MySQLStatement)
	err = binary.Read(br, binary.LittleEndian, &s.FieldCount)
	if s.FieldCount == uint8(0xff) {
		return nil, readErrorPacket(br)
	}
	err = binary.Read(br, binary.LittleEndian, &s.StatementId)
	err = binary.Read(br, binary.LittleEndian, &s.Columns)
	err = binary.Read(br, binary.LittleEndian, &s.Parameters)
	if ph.Len >= 12 {
		ignoreBytes(br, 1)
		err = binary.Read(br, binary.LittleEndian, &s.Warnings)
	}
	return s, err
}

//Currently just skips the pakets as I'm not sure if they are useful.
func readPrepareParameters(br *bufio.Reader, s *MySQLStatement) os.Error {
	for i := uint16(0); i < s.Parameters; i++ {
		ph, err := readHeader(br)
		if err != nil {
			return err
		}
		ignoreBytes(br, ph.Len)
	}
	return nil
}

func (sth *MySQLStatement) execute(va []interface{}) (*MySQLResponse, os.Error) {
	if int(sth.Parameters) != len(va) {
		return nil, os.ErrorString(fmt.Sprintf("Parameter count mismatch. %d != %d", sth.Parameters, len(va)))
	}
	type_parm, tn := encodeParamTypes(va)
	value_parm, vn := encodeParamValues(va)
	bitmap_len := (len(va) + 7) / 8
	mysql := sth.mysql
	packUint24(mysql.writer, uint32(11+bitmap_len+tn+vn))
	packUint8(mysql.writer, uint8(0))
	packUint8(mysql.writer, uint8(COM_STMT_EXECUTE))
	packUint32(mysql.writer, uint32(sth.StatementId))
	packUint8(mysql.writer, uint8(0))
	packUint32(mysql.writer, uint32(1))
	b := make([]byte, bitmap_len)
	mysql.writer.Write(b) //TODO: Support null params.
	packUint8(mysql.writer, uint8(1))
	mysql.writer.Write(type_parm)
	mysql.writer.Write(value_parm)
	mysql.writer.Flush()
	res, err := mysql.readResult()
	res.Prepared = true
	return res, err
}

func (mysql *MySQLInstance) prepare(arg string) (*MySQLStatement, os.Error) {
	plen := len(arg) + 1
	var head [5]byte
	head[0] = byte(plen)
	head[1] = byte(plen >> 8)
	head[2] = byte(plen >> 16)
	head[3] = 0
	head[4] = uint8(COM_STMT_PREPARE)
	_, err := mysql.writer.Write(&head)
	_, err = mysql.writer.WriteString(arg)
	err = mysql.writer.Flush()
	if err != nil {
		fmt.Printf("%s\n", err)
		return nil, err
	}
	sth, err := readPrepareInit(mysql.reader)
	if err != nil {
		return nil, err
	}
	if sth.Parameters > 0 {
		readPrepareParameters(mysql.reader, sth)
		readEOFPacket(mysql.reader)
	}
	if sth.Columns > 0 {
		rs, _ := mysql.readResultSet(uint64(sth.Columns))
		sth.ResultSet = rs
	}
	sth.mysql = mysql
	return sth, nil
}
<commit_msg>Small crashfix<commit_after>package mysql

import (
	"bufio"
	"encoding/binary"
	"os"
	"fmt"
	"bytes"
	"strconv"
)

type MySQLStatement struct {
	StatementId uint32
	Columns     uint16
	Parameters  uint16
	Warnings    uint16
	FieldCount  uint8

	ResultSet *MySQLResultSet
	mysql     *MySQLInstance
}

// Encode values fro each field
func encodeParamValues(a []interface{}) ([]byte, int) {
	var b []byte
	for i := 0; i < len(a); i++ {
		f := a[i];
		switch t := f.(type) {
		case string:
			b = bytes.Add(b, packString(string(t)))
		case int:
			b = bytes.Add(b, packString(strconv.Itoa(int(t))))
		case float:
			b = bytes.Add(b, packString(strconv.Ftoa(float(t), 'f', -1)))
		}
	}
	return b, len(b)
}

func putuint16(b []byte, v uint16) {
	b[0] = byte(v)
	b[1] = byte(v >> 8)
}

// For each field encode 2 byte type code. First bit is signed/unsigned
// Cheats and only bind parameters as strings
func encodeParamTypes(a []interface{}) ([]byte, int) {
	buf := make([]byte, len(a)*2)
	off := 0
	for i := 0; i < len(a); i++ {
		f := a[i];
		if f == nil {
			putuint16(buf[off:off+2], uint16(MYSQL_TYPE_NULL))
			continue
		}
		putuint16(buf[off:off+2], uint16(MYSQL_TYPE_STRING))
		off += 2
	}
	return buf, off
}

func readPrepareInit(br *bufio.Reader) (*MySQLStatement, os.Error) {
	ph, err := readHeader(br)
	if err != nil {
		return nil, err
	}
	s := new(MySQLStatement)
	err = binary.Read(br, binary.LittleEndian, &s.FieldCount)
	if s.FieldCount == uint8(0xff) {
		return nil, readErrorPacket(br)
	}
	err = binary.Read(br, binary.LittleEndian, &s.StatementId)
	err = binary.Read(br, binary.LittleEndian, &s.Columns)
	err = binary.Read(br, binary.LittleEndian, &s.Parameters)
	if ph.Len >= 12 {
		ignoreBytes(br, 1)
		err = binary.Read(br, binary.LittleEndian, &s.Warnings)
	}
	return s, err
}

//Currently just skips the pakets as I'm not sure if they are useful.
func readPrepareParameters(br *bufio.Reader, s *MySQLStatement) os.Error {
	for i := uint16(0); i < s.Parameters; i++ {
		ph, err := readHeader(br)
		if err != nil {
			return err
		}
		ignoreBytes(br, ph.Len)
	}
	return nil
}

func (sth *MySQLStatement) execute(va []interface{}) (*MySQLResponse, os.Error) {
	if va == nil || int(sth.Parameters) != len(va) {
		return nil, os.ErrorString(fmt.Sprintf("Parameter count mismatch. %d != %d", sth.Parameters, len(va)))
	}
	type_parm, tn := encodeParamTypes(va)
	value_parm, vn := encodeParamValues(va)
	bitmap_len := (len(va) + 7) / 8
	mysql := sth.mysql
	packUint24(mysql.writer, uint32(11+bitmap_len+tn+vn))
	packUint8(mysql.writer, uint8(0))
	packUint8(mysql.writer, uint8(COM_STMT_EXECUTE))
	packUint32(mysql.writer, uint32(sth.StatementId))
	packUint8(mysql.writer, uint8(0))
	packUint32(mysql.writer, uint32(1))
	b := make([]byte, bitmap_len)
	mysql.writer.Write(b) //TODO: Support null params.
	packUint8(mysql.writer, uint8(1))
	mysql.writer.Write(type_parm)
	mysql.writer.Write(value_parm)
	mysql.writer.Flush()
	res, err := mysql.readResult()
	res.Prepared = true
	return res, err
}

func (mysql *MySQLInstance) prepare(arg string) (*MySQLStatement, os.Error) {
	plen := len(arg) + 1
	var head [5]byte
	head[0] = byte(plen)
	head[1] = byte(plen >> 8)
	head[2] = byte(plen >> 16)
	head[3] = 0
	head[4] = uint8(COM_STMT_PREPARE)
	_, err := mysql.writer.Write(&head)
	_, err = mysql.writer.WriteString(arg)
	err = mysql.writer.Flush()
	if err != nil {
		fmt.Printf("%s\n", err)
		return nil, err
	}
	sth, err := readPrepareInit(mysql.reader)
	if err != nil {
		return nil, err
	}
	if sth.Parameters > 0 {
		readPrepareParameters(mysql.reader, sth)
		readEOFPacket(mysql.reader)
	}
	if sth.Columns > 0 {
		rs, _ := mysql.readResultSet(uint64(sth.Columns))
		sth.ResultSet = rs
	}
	sth.mysql = mysql
	return sth, nil
}
<|endoftext|>


<commit_before>package settings

import (
	"flag"
	"strings"
)

// Customize settings by passing proper parameters. See ./upendo -h for details.
var (
	// can be set to change application starting directory
	StartDir string

	// determines on which port upendo will run
	ServicePort string

	// if set to yes all templates will be reloaded on each request
	ReloadTemplates bool

	// should sessions be stored in file when terminating upendo
	ArchiveSessions bool

	// should sessions be restored from file when launching upendo
	RestoreSessions bool

	// limits chain of routing calls to specified value
	RoutingChainMax int
)

func init() {
	flag.StringVar(&StartDir, "start-dir", "", "app start directory, defaults to \".\"")
	flag.StringVar(&ServicePort, "port", "8080", "port for service to listen on")
	flag.BoolVar(&ReloadTemplates, "reload-templates", false, "if \"true\" then on each request page templates are reloaded")
	flag.BoolVar(&ArchiveSessions, "archive-sessions", true, "if \"true\" upon closing active sessions are archived to file")
	flag.BoolVar(&RestoreSessions, "restore-sessions", true, "if \"true\" restores previously active sessions")
	flag.IntVar(&RoutingChainMax, "routing-chain-max", 4, "limits maximum routing calls to specified value")
	flag.Parse()
	if len(StartDir) > 0 && strings.HasSuffix(StartDir, "/") == false {
		StartDir = StartDir + "/"
	}
}
<commit_msg>'map' files are now ignored by default<commit_after>package settings

import (
	"flag"
	"strings"
)

// Customize settings by passing proper parameters. See ./upendo -h for details.
var (
	// can be set to change application starting directory
	StartDir string

	// determines on which port upendo will run
	ServicePort string

	// if set to yes all templates will be reloaded on each request
	ReloadTemplates bool

	// should sessions be stored in file when terminating upendo
	ArchiveSessions bool

	// should sessions be restored from file when launching upendo
	RestoreSessions bool

	// limits chain of routing calls to specified value
	RoutingChainMax int

	// if map files are ignored (js.map, css.map)
	IgnoreMapFiles bool
)

func init() {
	flag.StringVar(&StartDir, "start-dir", "", "app start directory, defaults to \".\"")
	flag.StringVar(&ServicePort, "port", "8080", "port for service to listen on")
	flag.BoolVar(&ReloadTemplates, "reload-templates", false, "if \"true\" then on each request page templates are reloaded")
	flag.BoolVar(&ArchiveSessions, "archive-sessions", true, "if \"true\" upon closing active sessions are archived to file")
	flag.BoolVar(&RestoreSessions, "restore-sessions", true, "if \"true\" restores previously active sessions")
	flag.BoolVar(&IgnoreMapFiles, "ignore-map-files", true, "if \"true\" \"file not found\" errors for .map files will be ignored")
	flag.IntVar(&RoutingChainMax, "routing-chain-max", 4, "limits maximum routing calls to specified value")
	flag.Parse()
	if len(StartDir) > 0 && strings.HasSuffix(StartDir, "/") == false {
		StartDir = StartDir + "/"
	}
}
<|endoftext|>


<commit_before>package negotiator

import (
	"net/http"
	"strings"
)

const MimeJSON = "application/json"
const MimeXML = "application/xml"

// Encoder is an interface for a struct that can encode data into []byte
type Encoder interface {
	Encode(data interface{}) ([]byte, error)
	ContentType() string
}

// A Negotiator can Negotiate to determine what content type to convert
// a struct into for client consumption
type Negotiator interface {
	Negotiate(req *http.Request, data interface{}) ([]byte, error)
}

// ContentNegotiator is a Neotiator that supports a fallback/default
// encoder as well as dynamically adding encoders
type ContentNegotiator struct {
	DefaultEncoder Encoder
	ResponseWriter http.ResponseWriter
	encoderMap     map[string]Encoder
}

// NewContentNegotiator creates a basic ContentNegotiator with out any attached
// encoders
func NewContentNegotiator(defaultEncoder Encoder, responseWriter http.ResponseWriter) ContentNegotiator {
	result := ContentNegotiator{}
	result.DefaultEncoder = defaultEncoder
	result.ResponseWriter = responseWriter
	return result
}

// NewJsonXmlContentNegotiator creates a basic ContentNegotiator and attaches
// a JSON and an XML encoder to it.
func NewJsonXmlContentNegotiator(defaultEncoder Encoder, responseWriter http.ResponseWriter, prettyPrint bool) ContentNegotiator {
	result := NewContentNegotiator(defaultEncoder, responseWriter)
	result.AddEncoder(MimeJSON, JsonEncoder{prettyPrint})
	result.AddEncoder(MimeXML, XmlEncoder{prettyPrint})
	return result
}

// Negotiate inspects the request for the accept header and
// encodes the response appropriately.
func (cn ContentNegotiator) Negotiate(req *http.Request, data interface{}) ([]byte, error) {
	if len(cn.encoderMap) <= 0 {
		panic("No Encoders present. Please add them using ContentNegotiator.AddEncoder()")
	}
	var e = cn.getEncoder(req)
	cn.ResponseWriter.Header().Set("Content-Type", e.ContentType())
	return e.Encode(data)
}

// AddEncoder registers a mimetype and its encoder to be used if a client
// requests that mimetype
func (cn ContentNegotiator) AddEncoder(mimeType string, enc Encoder) {
	if cn.encoderMap == nil {
		cn.encoderMap = make(map[string]Encoder)
	}
	cn.encoderMap[mimeType] = enc
}

// getEncoder parses the Accept header an returns the appropriate encoder to use
func (cn ContentNegotiator) getEncoder(req *http.Request) Encoder {
	var result = cn.DefaultEncoder
	accept := req.Header.Get("Accept")

	for k, v := range cn.encoderMap {
		if strings.Contains(accept, k) {
			result = v
			break
		}
	}
	return result
}

// Check for an error and panic
func Must(data []byte, err error) []byte {
	if err != nil {
		panic(err)
	}
	return data
}
<commit_msg>fixing method bindings<commit_after>package negotiator

import (
	"net/http"
	"strings"
)

const MimeJSON = "application/json"
const MimeXML = "application/xml"

// Encoder is an interface for a struct that can encode data into []byte
type Encoder interface {
	Encode(data interface{}) ([]byte, error)
	ContentType() string
}

// A Negotiator can Negotiate to determine what content type to convert
// a struct into for client consumption
type Negotiator interface {
	Negotiate(req *http.Request, data interface{}) ([]byte, error)
}

// ContentNegotiator is a Neotiator that supports a fallback/default
// encoder as well as dynamically adding encoders
type ContentNegotiator struct {
	DefaultEncoder Encoder
	ResponseWriter http.ResponseWriter
	encoderMap     map[string]Encoder
}

// NewContentNegotiator creates a basic ContentNegotiator with out any attached
// encoders
func NewContentNegotiator(defaultEncoder Encoder, responseWriter http.ResponseWriter) *ContentNegotiator {
	result := &ContentNegotiator{}
	result.DefaultEncoder = defaultEncoder
	result.ResponseWriter = responseWriter
	return result
}

// NewJsonXmlContentNegotiator creates a basic ContentNegotiator and attaches
// a JSON and an XML encoder to it.
func NewJsonXmlContentNegotiator(defaultEncoder Encoder, responseWriter http.ResponseWriter, prettyPrint bool) *ContentNegotiator {
	result := NewContentNegotiator(defaultEncoder, responseWriter)
	result.AddEncoder(MimeJSON, JsonEncoder{prettyPrint})
	result.AddEncoder(MimeXML, XmlEncoder{prettyPrint})
	return result
}

// Negotiate inspects the request for the accept header and
// encodes the response appropriately.
func (cn *ContentNegotiator) Negotiate(req *http.Request, data interface{}) ([]byte, error) {
	if len(cn.encoderMap) <= 0 {
		panic("No Encoders present. Please add them using ContentNegotiator.AddEncoder()")
	}
	var e = cn.getEncoder(req)
	cn.ResponseWriter.Header().Set("Content-Type", e.ContentType())
	return e.Encode(data)
}

// AddEncoder registers a mimetype and its encoder to be used if a client
// requests that mimetype
func (cn *ContentNegotiator) AddEncoder(mimeType string, enc Encoder) {
	if cn.encoderMap == nil {
		cn.encoderMap = make(map[string]Encoder)
	}
	cn.encoderMap[mimeType] = enc
}

// getEncoder parses the Accept header an returns the appropriate encoder to use
func (cn *ContentNegotiator) getEncoder(req *http.Request) Encoder {
	var result = cn.DefaultEncoder
	accept := req.Header.Get("Accept")

	for k, v := range cn.encoderMap {
		if strings.Contains(accept, k) {
			result = v
			break
		}
	}
	return result
}

// Check for an error and panic
func Must(data []byte, err error) []byte {
	if err != nil {
		panic(err)
	}
	return data
}
<|endoftext|>


<commit_before>// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
//   http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.

package indexer

import (
	"errors"
	"github.com/couchbase/cbauth/metakv"
	"github.com/couchbase/indexing/secondary/common"
	"io/ioutil"
	"net/http"
)

const (
	indexerMetaDir          = "/indexer/"
	indexerSettingsMetaPath = indexerMetaDir + "settings"
)

// Implements dynamic settings management for indexer
type settingsManager struct {
	supvCmdch MsgChannel
	supvMsgch MsgChannel
	config    common.Config
	cancelCh  chan struct{}
}

func NewSettingsManager(supvCmdch MsgChannel,
	supvMsgch MsgChannel, config common.Config) (settingsManager, Message) {
	s := settingsManager{
		supvCmdch: supvCmdch,
		supvMsgch: supvMsgch,
		config:    config,
		cancelCh:  make(chan struct{}),
	}

	http.HandleFunc("/settings", s.handleSettingsReq)
	go func() {
		for {
			err := metakv.RunObserveChildren("/", s.metaKVCallback, s.cancelCh)
			if err == nil {
				return
			} else {
				common.Errorf("IndexerSettingsManager: metakv notifier failed (%v)..Restarting", err)
			}
		}
	}()
	return s, &MsgSuccess{}
}

func (s *settingsManager) writeOk(w http.ResponseWriter) {
	w.WriteHeader(200)
	w.Write([]byte("OK\n"))
}

func (s *settingsManager) writeError(w http.ResponseWriter, err error) {
	w.WriteHeader(400)
	w.Write([]byte(err.Error() + "\n"))
}

func (s *settingsManager) writeJson(w http.ResponseWriter, json []byte) {
	header := w.Header()
	header["Content-Type"] = []string{"application/json"}
	w.WriteHeader(200)
	w.Write(json)
	w.Write([]byte("\n"))
}

func (s *settingsManager) handleSettingsReq(w http.ResponseWriter, r *http.Request) {
	if r.Method == "POST" {
		bytes, _ := ioutil.ReadAll(r.Body)

		config := s.config.Clone()
		current, rev, err := metakv.Get(indexerSettingsMetaPath)
		if err == nil {
			if len(current) > 0 {
				config.Update(current)
			}
			err = config.Update(bytes)
		}

		if err != nil {
			s.writeError(w, err)
			return
		}

		settingsConfig := config.SectionConfig("settings.", false)
		newSettingsBytes := settingsConfig.Json()
		if err = metakv.Set(indexerSettingsMetaPath, newSettingsBytes, rev); err != nil {
			s.writeError(w, err)
			return
		}
		s.writeOk(w)
	} else if r.Method == "GET" {
		settingsConfig, err := getSettingsConfig(s.config)
		if err != nil {
			s.writeError(w, err)
			return
		}
		s.writeJson(w, settingsConfig.Json())
	} else {
		s.writeError(w, errors.New("Unsupported method"))
		return
	}
}

func (s *settingsManager) run() {
loop:
	for {
		select {
		case cmd, ok := <-s.supvCmdch:
			if ok {
				if cmd.GetMsgType() == STORAGE_MGR_SHUTDOWN {
					common.Infof("SettingsManager::run Shutting Down")
					close(s.cancelCh)
					s.supvCmdch <- &MsgSuccess{}
					break loop
				}
			} else {
				break loop
			}
		}
	}
}

func (s *settingsManager) metaKVCallback(path string, value []byte, rev interface{}) error {
	if path == indexerSettingsMetaPath {
		common.Infof("New settings received: \n%s", string(value))
		config := s.config.Clone()
		config.Update(value)
		s.config = config
		s.supvMsgch <- &MsgConfigUpdate{
			cfg: s.config,
		}
	}

	return nil
}

func getSettingsConfig(cfg common.Config) (common.Config, error) {
	settingsConfig := cfg.SectionConfig("settings.", false)
	current, _, err := metakv.Get(indexerSettingsMetaPath)
	if err == nil {
		if len(current) > 0 {
			settingsConfig.Update(current)
		}
	}
	return settingsConfig, err
}
<commit_msg>indexer: Implement manual compaction trigger<commit_after>// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
//   http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.

package indexer

import (
	"bytes"
	"errors"
	"github.com/couchbase/cbauth/metakv"
	"github.com/couchbase/indexing/secondary/common"
	"io/ioutil"
	"net/http"
	"time"
)

const (
	indexerMetaDir          = "/indexer/"
	indexerSettingsMetaPath = indexerMetaDir + "settings"
	indexCompactonMetaPath  = indexerMetaDir + "triggerCompaction"
)

// Implements dynamic settings management for indexer
type settingsManager struct {
	supvCmdch       MsgChannel
	supvMsgch       MsgChannel
	config          common.Config
	cancelCh        chan struct{}
	compactionToken []byte
}

func NewSettingsManager(supvCmdch MsgChannel,
	supvMsgch MsgChannel, config common.Config) (settingsManager, Message) {
	s := settingsManager{
		supvCmdch: supvCmdch,
		supvMsgch: supvMsgch,
		config:    config,
		cancelCh:  make(chan struct{}),
	}

	http.HandleFunc("/settings", s.handleSettingsReq)
	http.HandleFunc("/triggerCompaction", s.handleCompactionTrigger)
	go func() {
		for {
			err := metakv.RunObserveChildren("/", s.metaKVCallback, s.cancelCh)
			if err == nil {
				return
			} else {
				common.Errorf("IndexerSettingsManager: metakv notifier failed (%v)..Restarting", err)
			}
		}
	}()
	return s, &MsgSuccess{}
}

func (s *settingsManager) writeOk(w http.ResponseWriter) {
	w.WriteHeader(200)
	w.Write([]byte("OK\n"))
}

func (s *settingsManager) writeError(w http.ResponseWriter, err error) {
	w.WriteHeader(400)
	w.Write([]byte(err.Error() + "\n"))
}

func (s *settingsManager) writeJson(w http.ResponseWriter, json []byte) {
	header := w.Header()
	header["Content-Type"] = []string{"application/json"}
	w.WriteHeader(200)
	w.Write(json)
	w.Write([]byte("\n"))
}

func (s *settingsManager) handleSettingsReq(w http.ResponseWriter, r *http.Request) {
	if r.Method == "POST" {
		bytes, _ := ioutil.ReadAll(r.Body)

		config := s.config.Clone()
		current, rev, err := metakv.Get(indexerSettingsMetaPath)
		if err == nil {
			if len(current) > 0 {
				config.Update(current)
			}
			err = config.Update(bytes)
		}

		if err != nil {
			s.writeError(w, err)
			return
		}

		settingsConfig := config.SectionConfig("settings.", false)
		newSettingsBytes := settingsConfig.Json()
		if err = metakv.Set(indexerSettingsMetaPath, newSettingsBytes, rev); err != nil {
			s.writeError(w, err)
			return
		}
		s.writeOk(w)
	} else if r.Method == "GET" {
		settingsConfig, err := getSettingsConfig(s.config)
		if err != nil {
			s.writeError(w, err)
			return
		}
		s.writeJson(w, settingsConfig.Json())
	} else {
		s.writeError(w, errors.New("Unsupported method"))
		return
	}
}

func (s *settingsManager) handleCompactionTrigger(w http.ResponseWriter, r *http.Request) {
	_, rev, err := metakv.Get(indexCompactonMetaPath)
	if err != nil {
		s.writeError(w, err)
		return
	}

	newToken := time.Now().String()
	if err = metakv.Set(indexCompactonMetaPath, []byte(newToken), rev); err != nil {
		s.writeError(w, err)
		return
	}

	s.writeOk(w)
}

func (s *settingsManager) run() {
loop:
	for {
		select {
		case cmd, ok := <-s.supvCmdch:
			if ok {
				if cmd.GetMsgType() == STORAGE_MGR_SHUTDOWN {
					common.Infof("SettingsManager::run Shutting Down")
					close(s.cancelCh)
					s.supvCmdch <- &MsgSuccess{}
					break loop
				}
			} else {
				break loop
			}
		}
	}
}

func (s *settingsManager) metaKVCallback(path string, value []byte, rev interface{}) error {
	if path == indexerSettingsMetaPath {
		common.Infof("New settings received: \n%s", string(value))
		config := s.config.Clone()
		config.Update(value)
		s.config = config
		s.supvMsgch <- &MsgConfigUpdate{
			cfg: s.config,
		}
	} else if path == indexCompactonMetaPath {
		currentToken := s.compactionToken
		s.compactionToken = value
		if currentToken == nil || bytes.Equal(currentToken, value) {
			return nil
		}

		common.Infof("Manual compaction trigger requested")
		replych := make(chan []IndexStorageStats)
		statReq := &MsgIndexStorageStats{respch: replych}
		s.supvMsgch <- statReq
		stats := <-replych
		// XXX: minFile size check can be applied
		go func() {
			for _, is := range stats {
				errch := make(chan error)
				compactReq := &MsgIndexCompact{
					instId: is.InstId,
					errch:  errch,
				}
				common.Infof("ManualCompaction: Compacting index instance:%v", is.InstId)
				s.supvMsgch <- compactReq
				err := <-errch
				if err == nil {
					common.Infof("ManualCompaction: Finished compacting index instance:%v", is.InstId)
				} else {
					common.Errorf("ManualCompaction: Index instance:%v Compaction failed with reason - %v", is.InstId, err)
				}
			}
		}()
	}

	return nil
}

func getSettingsConfig(cfg common.Config) (common.Config, error) {
	settingsConfig := cfg.SectionConfig("settings.", false)
	current, _, err := metakv.Get(indexerSettingsMetaPath)
	if err == nil {
		if len(current) > 0 {
			settingsConfig.Update(current)
		}
	}
	return settingsConfig, err
}
<|endoftext|>


<commit_before>// Copyright 2015 xeipuuv ( https://github.com/xeipuuv )
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// author  			xeipuuv
// author-github 	https://github.com/xeipuuv
// author-mail		xeipuuv@gmail.com
//
// repository-name	gojsonpointer
// repository-desc	An implementation of JSON Pointer - Go language
//
// description		Main and unique file.
//
// created      	25-02-2013

package gojsonpointer

import (
	"errors"
	"fmt"
	"reflect"
	"strconv"
	"strings"
)

const (
	const_empty_pointer     = ``
	const_pointer_separator = `/`

	const_invalid_start = `JSON pointer must be empty or start with a "` + const_pointer_separator + `"`
)

type implStruct struct {
	mode string // "SET" or "GET"

	inDocument interface{}

	setInValue interface{}

	getOutNode interface{}
	getOutKind reflect.Kind
	outError   error
}

func NewJsonPointer(jsonPointerString string) (JsonPointer, error) {

	var p JsonPointer
	err := p.parse(jsonPointerString)
	return p, err

}

type JsonPointer struct {
	referenceTokens []string
}

// "Constructor", parses the given string JSON pointer
func (p *JsonPointer) parse(jsonPointerString string) error {

	var err error

	if jsonPointerString != const_empty_pointer {
		if !strings.HasPrefix(jsonPointerString, const_pointer_separator) {
			err = errors.New(const_invalid_start)
		} else {
			referenceTokens := strings.Split(jsonPointerString, const_pointer_separator)
			for _, referenceToken := range referenceTokens[1:] {
				p.referenceTokens = append(p.referenceTokens, referenceToken)
			}
		}
	}

	return err
}

// Uses the pointer to retrieve a value from a JSON document
func (p *JsonPointer) Get(document interface{}) (interface{}, reflect.Kind, error) {

	is := &implStruct{mode: "GET", inDocument: document}
	p.implementation(is)
	return is.getOutNode, is.getOutKind, is.outError

}

// Uses the pointer to update a value from a JSON document
func (p *JsonPointer) Set(document interface{}, value interface{}) (interface{}, error) {

	is := &implStruct{mode: "SET", inDocument: document, setInValue: value}
	p.implementation(is)
	return document, is.outError

}

// Both Get and Set functions use the same implementation to avoid code duplication
func (p *JsonPointer) implementation(i *implStruct) {

	kind := reflect.Invalid

	// Full document when empty
	if len(p.referenceTokens) == 0 {
		i.getOutNode = i.inDocument
		i.outError = nil
		i.getOutKind = kind
		i.outError = nil
		return
	}

	node := i.inDocument

	for ti, token := range p.referenceTokens {

		decodedToken := decodeReferenceToken(token)
		isLastToken := ti == len(p.referenceTokens)-1

		switch v := node.(type) {

		case map[string]interface{}:
			if _, ok := v[decodedToken]; ok {
				node = v[decodedToken]
				if isLastToken && i.mode == "SET" {
					v[decodedToken] = i.setInValue
				}
			} else {
				i.outError = errors.New(fmt.Sprintf("Object has no key '%s'", token))
				i.getOutKind = reflect.Map
				i.getOutNode = nil
				return
			}

		case []interface{}:
			tokenIndex, err := strconv.Atoi(token)
			if err != nil {
				i.outError = errors.New(fmt.Sprintf("Invalid array index '%s'", token))
				i.getOutKind = reflect.Slice
				i.getOutNode = nil
				return
			}
			if tokenIndex < 0 || tokenIndex >= len(v) {
				i.outError = errors.New(fmt.Sprintf("Out of bound array[0,%d] index '%d'", len(v), tokenIndex))
				i.getOutKind = reflect.Slice
				i.getOutNode = nil
				return
			}

			node = v[tokenIndex]
			if isLastToken && i.mode == "SET" {
				v[tokenIndex] = i.setInValue
			}

		default:
			i.outError = errors.New(fmt.Sprintf("Invalid token reference '%s'", token))
			i.getOutKind = reflect.ValueOf(node).Kind()
			i.getOutNode = nil
			return
		}

	}

	i.getOutNode = node
	i.getOutKind = reflect.ValueOf(node).Kind()
	i.outError = nil
}

// Pointer to string representation function
func (p *JsonPointer) String() string {

	if len(p.referenceTokens) == 0 {
		return const_empty_pointer
	}

	pointerString := const_pointer_separator + strings.Join(p.referenceTokens, const_pointer_separator)

	return pointerString
}

// Specific JSON pointer encoding here
// ~0 => ~
// ~1 => /
// ... and vice versa

const (
	const_encoded_reference_token_0 = `~0`
	const_encoded_reference_token_1 = `~1`
	const_decoded_reference_token_0 = `~`
	const_decoded_reference_token_1 = `/`
)

func decodeReferenceToken(token string) string {
	step1 := strings.Replace(token, const_encoded_reference_token_1, const_decoded_reference_token_1, -1)
	step2 := strings.Replace(step1, const_encoded_reference_token_0, const_decoded_reference_token_0, -1)
	return step2
}

func encodeReferenceToken(token string) string {
	step1 := strings.Replace(token, const_decoded_reference_token_1, const_encoded_reference_token_1, -1)
	step2 := strings.Replace(step1, const_decoded_reference_token_0, const_encoded_reference_token_0, -1)
	return step2
}
<commit_msg>Decode tokens only when dereferencing object property name<commit_after>// Copyright 2015 xeipuuv ( https://github.com/xeipuuv )
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// author  			xeipuuv
// author-github 	https://github.com/xeipuuv
// author-mail		xeipuuv@gmail.com
//
// repository-name	gojsonpointer
// repository-desc	An implementation of JSON Pointer - Go language
//
// description		Main and unique file.
//
// created      	25-02-2013

package gojsonpointer

import (
	"errors"
	"fmt"
	"reflect"
	"strconv"
	"strings"
)

const (
	const_empty_pointer     = ``
	const_pointer_separator = `/`

	const_invalid_start = `JSON pointer must be empty or start with a "` + const_pointer_separator + `"`
)

type implStruct struct {
	mode string // "SET" or "GET"

	inDocument interface{}

	setInValue interface{}

	getOutNode interface{}
	getOutKind reflect.Kind
	outError   error
}

func NewJsonPointer(jsonPointerString string) (JsonPointer, error) {

	var p JsonPointer
	err := p.parse(jsonPointerString)
	return p, err

}

type JsonPointer struct {
	referenceTokens []string
}

// "Constructor", parses the given string JSON pointer
func (p *JsonPointer) parse(jsonPointerString string) error {

	var err error

	if jsonPointerString != const_empty_pointer {
		if !strings.HasPrefix(jsonPointerString, const_pointer_separator) {
			err = errors.New(const_invalid_start)
		} else {
			referenceTokens := strings.Split(jsonPointerString, const_pointer_separator)
			for _, referenceToken := range referenceTokens[1:] {
				p.referenceTokens = append(p.referenceTokens, referenceToken)
			}
		}
	}

	return err
}

// Uses the pointer to retrieve a value from a JSON document
func (p *JsonPointer) Get(document interface{}) (interface{}, reflect.Kind, error) {

	is := &implStruct{mode: "GET", inDocument: document}
	p.implementation(is)
	return is.getOutNode, is.getOutKind, is.outError

}

// Uses the pointer to update a value from a JSON document
func (p *JsonPointer) Set(document interface{}, value interface{}) (interface{}, error) {

	is := &implStruct{mode: "SET", inDocument: document, setInValue: value}
	p.implementation(is)
	return document, is.outError

}

// Both Get and Set functions use the same implementation to avoid code duplication
func (p *JsonPointer) implementation(i *implStruct) {

	kind := reflect.Invalid

	// Full document when empty
	if len(p.referenceTokens) == 0 {
		i.getOutNode = i.inDocument
		i.outError = nil
		i.getOutKind = kind
		i.outError = nil
		return
	}

	node := i.inDocument

	for ti, token := range p.referenceTokens {

		isLastToken := ti == len(p.referenceTokens)-1

		switch v := node.(type) {

		case map[string]interface{}:
			decodedToken := decodeReferenceToken(token)
			if _, ok := v[decodedToken]; ok {
				node = v[decodedToken]
				if isLastToken && i.mode == "SET" {
					v[decodedToken] = i.setInValue
				}
			} else {
				i.outError = errors.New(fmt.Sprintf("Object has no key '%s'", decodedToken))
				i.getOutKind = reflect.Map
				i.getOutNode = nil
				return
			}

		case []interface{}:
			tokenIndex, err := strconv.Atoi(token)
			if err != nil {
				i.outError = errors.New(fmt.Sprintf("Invalid array index '%s'", token))
				i.getOutKind = reflect.Slice
				i.getOutNode = nil
				return
			}
			if tokenIndex < 0 || tokenIndex >= len(v) {
				i.outError = errors.New(fmt.Sprintf("Out of bound array[0,%d] index '%d'", len(v), tokenIndex))
				i.getOutKind = reflect.Slice
				i.getOutNode = nil
				return
			}

			node = v[tokenIndex]
			if isLastToken && i.mode == "SET" {
				v[tokenIndex] = i.setInValue
			}

		default:
			i.outError = errors.New(fmt.Sprintf("Invalid token reference '%s'", token))
			i.getOutKind = reflect.ValueOf(node).Kind()
			i.getOutNode = nil
			return
		}

	}

	i.getOutNode = node
	i.getOutKind = reflect.ValueOf(node).Kind()
	i.outError = nil
}

// Pointer to string representation function
func (p *JsonPointer) String() string {

	if len(p.referenceTokens) == 0 {
		return const_empty_pointer
	}

	pointerString := const_pointer_separator + strings.Join(p.referenceTokens, const_pointer_separator)

	return pointerString
}

// Specific JSON pointer encoding here
// ~0 => ~
// ~1 => /
// ... and vice versa

const (
	const_encoded_reference_token_0 = `~0`
	const_encoded_reference_token_1 = `~1`
	const_decoded_reference_token_0 = `~`
	const_decoded_reference_token_1 = `/`
)

func decodeReferenceToken(token string) string {
	step1 := strings.Replace(token, const_encoded_reference_token_1, const_decoded_reference_token_1, -1)
	step2 := strings.Replace(step1, const_encoded_reference_token_0, const_decoded_reference_token_0, -1)
	return step2
}

func encodeReferenceToken(token string) string {
	step1 := strings.Replace(token, const_decoded_reference_token_1, const_encoded_reference_token_1, -1)
	step2 := strings.Replace(step1, const_decoded_reference_token_0, const_encoded_reference_token_0, -1)
	return step2
}
<|endoftext|>


<commit_before>// Copyright 2016-2020 Olivier Mengué. All rights reserved.
// Use of this source code is governed by the Apache 2.0 license that
// can be found in the LICENSE file.

package jsonptr

import (
	"bytes"
	"encoding/json"
	"fmt"
	"strconv"
	"strings"
)

// Pointer represents a mutable parsed JSON Pointer.
//
// The Go representation is a array of non-encoded path elements. This
// allows to use type conversion from/to a []string.
type Pointer []string

// Parse parses a JSON pointer from its text representation.
func Parse(pointer string) (Pointer, error) {
	if pointer == "" {
		return nil, nil
	}
	if pointer[0] != '/' {
		return nil, ErrSyntax
	}
	ptr := strings.Split(pointer[1:], "/")
	// Optimize for the common case
	if strings.IndexByte(pointer, '~') == -1 {
		return ptr, nil
	}
	for i, part := range ptr {
		var err error
		if ptr[i], err = UnescapeString(part); err != nil {
			// TODO return the full prefix
			return nil, err
		}
	}
	return ptr, nil
}

// MustParse wraps Parse and panics in case of error.
func MustParse(pointer string) Pointer {
	ptr, err := Parse(pointer)
	if err != nil {
		panic(fmt.Errorf("%q: %v", pointer, err))
	}
	return ptr
}

// MarshalText() implements encoding.TextUnmarshaler
func (ptr *Pointer) UnmarshalText(text []byte) error {
	if len(text) == 0 {
		*ptr = nil
		return nil
	}
	if text[0] != '/' {
		return ErrSyntax
	}

	var p Pointer
	t := text[1:]
	for {
		i := bytes.IndexByte(t, '/')
		if i < 0 {
			break
		}
		part, err := Unescape(t[:i])
		if err != nil {
			return syntaxError(string(text[:len(text)-len(t)-i-1]))
		}
		p = append(p, string(part))
		t = t[i+1:]
	}
	part, err := Unescape(t)
	if err != nil {
		return syntaxError(string(text))
	}
	*ptr = append(p, string(part))
	return nil
}

// String returns a JSON Pointer string, escaping components when necessary:
// '~' is replaced by "~0", '/' by "~1"
func (ptr Pointer) String() string {
	if len(ptr) == 0 {
		return ""
	}

	dst := make([]byte, 0, 8*len(ptr))
	for _, part := range ptr {
		dst = AppendEscape(append(dst, '/'), part)
	}
	return string(dst)
}

// MarshalText() implements encoding.TextMarshaler
func (ptr Pointer) MarshalText() (text []byte, err error) {
	if len(ptr) == 0 {
		return nil, nil
	}
	dst := make([]byte, 0, 8*len(ptr))
	for _, part := range ptr {
		dst = AppendEscape(append(dst, '/'), part)
	}
	return dst, nil
}

// Grow allows to prepare space for growth (before use of Property/Index).
func (ptr *Pointer) Grow(n int) {
	if cap(*ptr) >= len(*ptr)+n {
		return
	}
	*ptr = append(make(Pointer, 0, len(*ptr)+n), *ptr...)
}

// Copy returns a new, independant, copy of the pointer.
func (ptr Pointer) Copy() Pointer {
	return append(Pointer(nil), ptr...)
}

// IsRoot returns true if the pointer is at root (empty).
func (ptr Pointer) IsRoot() bool {
	return len(ptr) == 0
}

// Up removes the last element of the pointer.
// The pointer is returned for chaining.
//
// Panics if already at root.
func (ptr *Pointer) Up() *Pointer {
	if ptr.IsRoot() {
		panic(ErrRoot)
	}
	*ptr = (*ptr)[:len(*ptr)-1]
	return ptr
}

// Pop removes the last element of the pointer and returns it.
//
// Panics if already at root.
func (ptr *Pointer) Pop() string {
	last := len(*ptr) - 1
	if last < 0 {
		panic(ErrRoot)
	}
	prop := (*ptr)[last]
	*ptr = (*ptr)[:last]
	return prop
}

// Property moves the pointer deeper, following a property name.
// The pointer is returned for chaining.
func (ptr *Pointer) Property(name string) *Pointer {
	*ptr = append(*ptr, name)
	return ptr
}

// Index moves the pointer deeper, following an array index.
// The pointer is returned for chaining.
func (ptr *Pointer) Index(index int) *Pointer {
	var prop string
	if index < 0 {
		prop = "-"
	} else {
		prop = strconv.Itoa(index)
	}
	return ptr.Property(prop)
}

// LeafName returns the name of the last part of the pointer.
func (ptr Pointer) LeafName() string {
	return ptr[len(ptr)-1]
}

// LeafIndex returns the last part of the pointer as an array index.
// -1 is returned for "-".
func (ptr Pointer) LeafIndex() (int, error) {
	return arrayIndex(ptr.LeafName())
}

// In returns the value from doc pointed by ptr.
//
// doc may be a deserialized document, or a [encoding/json.RawMessage].
func (ptr Pointer) In(doc interface{}) (interface{}, error) {
	for i, key := range ptr {
		switch here := (doc).(type) {
		case map[string]interface{}:
			var ok bool
			if doc, ok = here[key]; !ok {
				return nil, propertyError(ptr[:i+1].String())
			}
		case []interface{}:
			n, err := arrayIndex(key)
			if err != nil || n < 0 || n >= len(here) {
				return nil, indexError(ptr[:i+1].String())
			}
			doc = here[n]
		case JSONDecoder:
			v, err := getJSON(here, ptr[i:].String())
			if err != nil {
				err.rebase(ptr[:i].String())
			}
			return v, err
		case json.RawMessage:
			v, err := getRaw(here, ptr[i:].String())
			if err != nil {
				err.rebase(ptr[:i].String())
			}
			return v, err
		default:
			// We report the error at the upper level
			return nil, docError(ptr[:i-1].String(), doc)
		}
	}

	doc, err := getLeaf(doc)
	if err != nil {
		err.rebase(ptr.String())
	}
	return doc, err
}

// Set changes a value in document pdoc at location pointed by ptr.
func (ptr Pointer) Set(pdoc *interface{}, value interface{}) error {
	// TODO Make an optimised implementation
	return Set(pdoc, ptr.String(), value)
}

// Delete removes an object property or an array element (and shifts remaining ones).
// It can't be applied on root.
func (ptr Pointer) Delete(pdoc *interface{}) (interface{}, error) {
	// TODO Make an optimised implementation
	return Delete(pdoc, ptr.String())
}
<commit_msg>doc: add more doc links<commit_after>// Copyright 2016-2020 Olivier Mengué. All rights reserved.
// Use of this source code is governed by the Apache 2.0 license that
// can be found in the LICENSE file.

package jsonptr

import (
	"bytes"
	"encoding/json"
	"fmt"
	"strconv"
	"strings"
)

// Pointer represents a mutable parsed JSON Pointer.
//
// The Go representation is a array of non-encoded path elements. This
// allows to use type conversion from/to a []string.
type Pointer []string

// Parse parses a JSON pointer from its text representation.
func Parse(pointer string) (Pointer, error) {
	if pointer == "" {
		return nil, nil
	}
	if pointer[0] != '/' {
		return nil, ErrSyntax
	}
	ptr := strings.Split(pointer[1:], "/")
	// Optimize for the common case
	if strings.IndexByte(pointer, '~') == -1 {
		return ptr, nil
	}
	for i, part := range ptr {
		var err error
		if ptr[i], err = UnescapeString(part); err != nil {
			// TODO return the full prefix
			return nil, err
		}
	}
	return ptr, nil
}

// MustParse wraps Parse and panics in case of error.
func MustParse(pointer string) Pointer {
	ptr, err := Parse(pointer)
	if err != nil {
		panic(fmt.Errorf("%q: %v", pointer, err))
	}
	return ptr
}

// MarshalText implements [encoding.TextUnmarshaler].
func (ptr *Pointer) UnmarshalText(text []byte) error {
	if len(text) == 0 {
		*ptr = nil
		return nil
	}
	if text[0] != '/' {
		return ErrSyntax
	}

	var p Pointer
	t := text[1:]
	for {
		i := bytes.IndexByte(t, '/')
		if i < 0 {
			break
		}
		part, err := Unescape(t[:i])
		if err != nil {
			return syntaxError(string(text[:len(text)-len(t)-i-1]))
		}
		p = append(p, string(part))
		t = t[i+1:]
	}
	part, err := Unescape(t)
	if err != nil {
		return syntaxError(string(text))
	}
	*ptr = append(p, string(part))
	return nil
}

// String returns a JSON Pointer string, escaping components when necessary:
// '~' is replaced by "~0", '/' by "~1"
func (ptr Pointer) String() string {
	if len(ptr) == 0 {
		return ""
	}

	dst := make([]byte, 0, 8*len(ptr))
	for _, part := range ptr {
		dst = AppendEscape(append(dst, '/'), part)
	}
	return string(dst)
}

// MarshalText implements [encoding.TextMarshaler].
func (ptr Pointer) MarshalText() (text []byte, err error) {
	if len(ptr) == 0 {
		return nil, nil
	}
	dst := make([]byte, 0, 8*len(ptr))
	for _, part := range ptr {
		dst = AppendEscape(append(dst, '/'), part)
	}
	return dst, nil
}

// Grow allows to prepare space for growth (before use of [Pointer.Property]/[Pointer.Index]).
func (ptr *Pointer) Grow(n int) {
	if cap(*ptr) >= len(*ptr)+n {
		return
	}
	*ptr = append(make(Pointer, 0, len(*ptr)+n), *ptr...)
}

// Copy returns a new, independant, copy of the pointer.
func (ptr Pointer) Copy() Pointer {
	return append(Pointer(nil), ptr...)
}

// IsRoot returns true if the pointer is at root (empty).
func (ptr Pointer) IsRoot() bool {
	return len(ptr) == 0
}

// Up removes the last element of the pointer.
// The pointer is returned for chaining.
//
// Panics if already at root.
func (ptr *Pointer) Up() *Pointer {
	if ptr.IsRoot() {
		panic(ErrRoot)
	}
	*ptr = (*ptr)[:len(*ptr)-1]
	return ptr
}

// Pop removes the last element of the pointer and returns it.
//
// Panics if already at root.
func (ptr *Pointer) Pop() string {
	last := len(*ptr) - 1
	if last < 0 {
		panic(ErrRoot)
	}
	prop := (*ptr)[last]
	*ptr = (*ptr)[:last]
	return prop
}

// Property moves the pointer deeper, following a property name.
// The pointer is returned for chaining.
func (ptr *Pointer) Property(name string) *Pointer {
	*ptr = append(*ptr, name)
	return ptr
}

// Index moves the pointer deeper, following an array index.
// The pointer is returned for chaining.
func (ptr *Pointer) Index(index int) *Pointer {
	var prop string
	if index < 0 {
		prop = "-"
	} else {
		prop = strconv.Itoa(index)
	}
	return ptr.Property(prop)
}

// LeafName returns the name of the last part of the pointer.
func (ptr Pointer) LeafName() string {
	return ptr[len(ptr)-1]
}

// LeafIndex returns the last part of the pointer as an array index.
// -1 is returned for "-".
func (ptr Pointer) LeafIndex() (int, error) {
	return arrayIndex(ptr.LeafName())
}

// In returns the value from doc pointed by ptr.
//
// doc may be a deserialized document, or a [encoding/json.RawMessage].
func (ptr Pointer) In(doc interface{}) (interface{}, error) {
	for i, key := range ptr {
		switch here := (doc).(type) {
		case map[string]interface{}:
			var ok bool
			if doc, ok = here[key]; !ok {
				return nil, propertyError(ptr[:i+1].String())
			}
		case []interface{}:
			n, err := arrayIndex(key)
			if err != nil || n < 0 || n >= len(here) {
				return nil, indexError(ptr[:i+1].String())
			}
			doc = here[n]
		case JSONDecoder:
			v, err := getJSON(here, ptr[i:].String())
			if err != nil {
				err.rebase(ptr[:i].String())
			}
			return v, err
		case json.RawMessage:
			v, err := getRaw(here, ptr[i:].String())
			if err != nil {
				err.rebase(ptr[:i].String())
			}
			return v, err
		default:
			// We report the error at the upper level
			return nil, docError(ptr[:i-1].String(), doc)
		}
	}

	doc, err := getLeaf(doc)
	if err != nil {
		err.rebase(ptr.String())
	}
	return doc, err
}

// Set changes a value in document pdoc at location pointed by ptr.
func (ptr Pointer) Set(pdoc *interface{}, value interface{}) error {
	// TODO Make an optimised implementation
	return Set(pdoc, ptr.String(), value)
}

// Delete removes an object property or an array element (and shifts remaining ones).
// It can't be applied on root.
func (ptr Pointer) Delete(pdoc *interface{}) (interface{}, error) {
	// TODO Make an optimised implementation
	return Delete(pdoc, ptr.String())
}
<|endoftext|>


<commit_before>// Copyright 2015 Reborndb Org. All Rights Reserved.
// Licensed under the MIT (MIT-LICENSE.txt) license.

package main

import (
	"fmt"
	"net/url"
	"os"
	"os/exec"
	"strings"
	"testing"
	"time"

	"github.com/ngaut/zkhelper"
	. "github.com/reborndb/go/gocheck2"
	"github.com/reborndb/reborn/pkg/env"
	"github.com/reborndb/reborn/pkg/models"
	"github.com/reborndb/reborn/pkg/utils"

	. "gopkg.in/check.v1"
)

func TestT(t *testing.T) {
	TestingT(t)
}

type testAgentInfo struct {
	cmd  *exec.Cmd
	addr string
}

func (a testAgentInfo) apiURL(action string, query string) string {
	if len(query) > 0 {
		return fmt.Sprintf("http://%s/api/%s?%s", a.addr, action, query)
	} else {
		return fmt.Sprintf("http://%s/api/%s", a.addr, action)
	}
}

func (a testAgentInfo) httpCall(c *C, objPrt interface{}, action string, query string, method string) {
	err := httpCall(objPrt, a.apiURL(action, query), method, nil)
	c.Assert(err, IsNil)
}

type testAgentSuite struct {
	agentDashboard   testAgentInfo
	agentProxy       testAgentInfo
	agentStoreMaster testAgentInfo
	agentStoreSlave  testAgentInfo
}

var _ = Suite(&testAgentSuite{})

func (s *testAgentSuite) SetUpSuite(c *C) {
	// initial whole test environment
	configFile = "config.ini"
	resetAbsPath(&configFile)
	cfg, err := utils.InitConfigFromFile(configFile)
	c.Assert(err, IsNil)

	globalEnv = env.LoadRebornEnv(cfg)
	globalConn, err = globalEnv.NewCoordConn()
	c.Assert(err, IsNil)

	s.testSetExecPath(c)

	// remove all infos in coordinator first
	err = zkhelper.DeleteRecursive(globalConn, fmt.Sprintf("/zk/reborn/db_%s", globalEnv.ProductName()), -1)
	c.Assert(err, IsNil)

	s.agentDashboard = s.testStartAgent(c, "127.0.0.1:39001")
	s.agentProxy = s.testStartAgent(c, "127.0.0.1:39002")
	s.agentStoreMaster = s.testStartAgent(c, "127.0.0.1:39003")
	s.agentStoreSlave = s.testStartAgent(c, "127.0.0.1:39004")

	s.testDashboard(c)

	s.testInitGroup(c, 1)
	s.testInitSlots(c)
}

func (s *testAgentSuite) TearDownSuite(c *C) {
	if globalConn != nil {
		globalConn.Close()
	}

	stopAgent := func(agent testAgentInfo) {
		s.testStopAllProcs(c, agent)
		s.testStopAgent(c, agent)
	}

	stopAgent(s.agentProxy)
	stopAgent(s.agentStoreSlave)
	stopAgent(s.agentStoreMaster)
	stopAgent(s.agentDashboard)
}

func (s *testAgentSuite) testSetExecPath(c *C) {
	// the executer is in ../../bin path, we will add this path in global $PATH
	execPath := "../../bin"
	resetAbsPath(&execPath)
	c.Assert(strings.HasSuffix(execPath, "/src/github.com/reborndb/reborn/bin"), IsTrue)

	os.Setenv("PATH", os.ExpandEnv(fmt.Sprintf("${PATH}:%s", execPath)))

	_, err := exec.LookPath("reborn-config")
	c.Assert(err, IsNil)

	_, err = exec.LookPath("reborn-proxy")
	c.Assert(err, IsNil)

	_, err = exec.LookPath("reborn-server")
	c.Assert(err, IsNil)

	_, err = exec.LookPath("reborn-agent")
	c.Assert(err, IsNil)
}

func (s *testAgentSuite) testStartAgent(c *C, addr string) testAgentInfo {
	dataDir := fmt.Sprintf("./var/%s/data", addr)
	logDir := fmt.Sprintf("./var/%s/log", addr)

	os.RemoveAll(dataDir)
	os.RemoveAll(logDir)

	os.MkdirAll(dataDir, 0755)
	os.MkdirAll(logDir, 0755)

	cmd := exec.Command("reborn-agent", "--addr", addr, "--data-dir", dataDir, "--log-dir",
		logDir, "-L", fmt.Sprintf("./var/%s/agent.log", addr))
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	err := cmd.Start()
	c.Assert(err, IsNil)

	var info testAgentInfo
	info.cmd = cmd
	info.addr = addr

	time.Sleep(time.Second)

	return info
}

func (s *testAgentSuite) testStopAgent(c *C, agent testAgentInfo) {
	c.Assert(agent.cmd, NotNil)
	agent.cmd.Process.Signal(os.Interrupt)
	agent.cmd.Wait()
}

func (s *testAgentSuite) testStopAllProcs(c *C, agent testAgentInfo) {
	procs := s.testGetProcs(c, agent)

	for _, proc := range procs {
		agent.httpCall(c, nil, "stop", fmt.Sprintf("id=%s", proc.ID), "POST")
	}
}

func (s *testAgentSuite) testKillAllProcs(c *C, agent testAgentInfo) {
	procs := s.testGetProcs(c, agent)

	for _, proc := range procs {
		p, err := os.FindProcess(proc.Pid)
		c.Assert(err, IsNil)
		p.Kill()
		p.Wait()
	}
}

func (s *testAgentSuite) testGetProcs(c *C, agent testAgentInfo) []procStatus {
	var procs []procStatus
	agent.httpCall(c, &procs, "procs", "", "GET")
	return procs
}

func (s *testAgentSuite) callDashboardAPI(c *C, retVal interface{}, apiPath string, method string, arg interface{}) {
	err := httpCall(retVal, fmt.Sprintf("http://%s%s", globalEnv.DashboardAddr(), apiPath), method, arg)
	c.Assert(err, IsNil)
}

func (s *testAgentSuite) testDashboard(c *C) {
	agent := s.agentDashboard

	agent.httpCall(c, nil, "start_dashboard", "", "POST")

	time.Sleep(2 * time.Second)
	s.callDashboardAPI(c, nil, "/ping", "GET", nil)

	// if we kill dashboard, the restart one will wait a coorindator session timeout time
	// for old session expired, this will be a long time, so we don't test this case here.
}

func (s *testAgentSuite) testStore(c *C, agent testAgentInfo, port int) {
	addr := fmt.Sprintf("127.0.0.1:%d", port)
	agent.httpCall(c, nil, "start_redis", fmt.Sprintf("addr=%s", url.QueryEscape(addr)), "POST")

	err := utils.Ping(addr, globalEnv.StoreAuth())
	c.Assert(err, IsNil)

	// kill store and then wait 2s for restart
	s.testKillAllProcs(c, agent)

	time.Sleep(2 * time.Second)

	err = utils.Ping(addr, globalEnv.StoreAuth())
	c.Assert(err, IsNil)
}

func (s *testAgentSuite) testStoreAddServer(c *C) {
	s.testStore(c, s.agentStoreMaster, 6381)
	s.testStore(c, s.agentStoreSlave, 6382)

	groupID := 1

	s.testAddStoreToGroup(c, 6381, models.SERVER_TYPE_MASTER, groupID)
	s.testAddStoreToGroup(c, 6382, models.SERVER_TYPE_SLAVE, groupID)
}

func (s *testAgentSuite) testInitGroup(c *C, groupID int) {
	serverGroup := models.NewServerGroup(globalEnv.ProductName(), groupID)

	s.callDashboardAPI(c, nil, "/api/server_groups", "PUT", serverGroup)
}

func (s *testAgentSuite) testAddStoreToGroup(c *C, port int, role string, groupID int) {
	server := models.NewServer(role, fmt.Sprintf("127.0.0.1:%d", port))
	s.callDashboardAPI(c, nil, fmt.Sprintf("/api/server_group/%d/addServer", groupID), "PUT", server)
}

func (s *testAgentSuite) testInitSlots(c *C) {
	s.callDashboardAPI(c, nil, "/api/slots/init", "POST", nil)

	v := struct {
		FromSlot   int    `json:"from"`
		ToSlot     int    `json:"to"`
		NewGroupID int    `json:"new_group"`
		Status     string `json:"status"`
	}{
		FromSlot:   0,
		ToSlot:     1023,
		NewGroupID: 1,
		Status:     "online",
	}

	s.callDashboardAPI(c, nil, "/api/slot", "POST", v)
}

func (s *testAgentSuite) testProxy(c *C) {
	agent := s.agentProxy
	proxyAddr := "127.0.0.1:19000"
	proxyHTTPAddr := "127.0.0.1:29000"

	args := make(url.Values)
	args.Set("addr", proxyAddr)
	args.Set("http_addr", proxyHTTPAddr)

	agent.httpCall(c, nil, "start_proxy", args.Encode(), "POST")

	// now the proxy will wait 3s for online, this is very long for test
	// maybe later we will change it.
	var err error
	for i := 0; i < 3; i++ {
		time.Sleep(2 * time.Second)
		if err = utils.Ping(proxyAddr, ""); err == nil {
			break
		}
	}
	c.Assert(err, IsNil)

	// kill proxy and then wait for restart
	s.testKillAllProcs(c, agent)

	for i := 0; i < 3; i++ {
		time.Sleep(2 * time.Second)
		if err = utils.Ping(proxyAddr, ""); err == nil {
			break
		}
	}
	c.Assert(err, IsNil)
}

func (s *testAgentSuite) TestSimple(c *C) {
	s.testStoreAddServer(c)
	s.testProxy(c)
}
<commit_msg>add ha test<commit_after>// Copyright 2015 Reborndb Org. All Rights Reserved.
// Licensed under the MIT (MIT-LICENSE.txt) license.

package main

import (
	"fmt"
	"net/url"
	"os"
	"os/exec"
	"strings"
	"testing"
	"time"

	"github.com/ngaut/zkhelper"
	. "github.com/reborndb/go/gocheck2"
	"github.com/reborndb/reborn/pkg/env"
	"github.com/reborndb/reborn/pkg/models"
	"github.com/reborndb/reborn/pkg/utils"

	. "gopkg.in/check.v1"
)

func TestT(t *testing.T) {
	TestingT(t)
}

type testAgentInfo struct {
	cmd  *exec.Cmd
	addr string
}

func (a testAgentInfo) apiURL(action string, query string) string {
	if len(query) > 0 {
		return fmt.Sprintf("http://%s/api/%s?%s", a.addr, action, query)
	} else {
		return fmt.Sprintf("http://%s/api/%s", a.addr, action)
	}
}

func (a testAgentInfo) httpCall(c *C, objPrt interface{}, action string, query string, method string) {
	err := httpCall(objPrt, a.apiURL(action, query), method, nil)
	c.Assert(err, IsNil)
}

type testAgentSuite struct {
	agentDashboard   testAgentInfo
	agentProxy       testAgentInfo
	agentStoreMaster testAgentInfo
	agentStoreSlave  testAgentInfo
	agentHA          testAgentInfo
}

var _ = Suite(&testAgentSuite{})

func (s *testAgentSuite) SetUpSuite(c *C) {
	// initial whole test environment
	configFile = "config.ini"
	resetAbsPath(&configFile)
	cfg, err := utils.InitConfigFromFile(configFile)
	c.Assert(err, IsNil)

	globalEnv = env.LoadRebornEnv(cfg)
	globalConn, err = globalEnv.NewCoordConn()
	c.Assert(err, IsNil)

	s.testSetExecPath(c)

	// remove all infos in coordinator first
	err = zkhelper.DeleteRecursive(globalConn, fmt.Sprintf("/zk/reborn/db_%s", globalEnv.ProductName()), -1)
	c.Assert(err, IsNil)

	s.agentDashboard = s.testStartAgent(c, "127.0.0.1:39001", false)
	s.agentProxy = s.testStartAgent(c, "127.0.0.1:39002", false)
	s.agentStoreMaster = s.testStartAgent(c, "127.0.0.1:39003", false)
	s.agentStoreSlave = s.testStartAgent(c, "127.0.0.1:39004", false)

	s.testDashboard(c)

	s.testInitGroup(c)
	s.testInitSlots(c)
	s.testStoreAddServer(c)
}

func (s *testAgentSuite) TearDownSuite(c *C) {
	if globalConn != nil {
		globalConn.Close()
	}

	stopAgent := func(agent testAgentInfo) {
		s.testStopAllProcs(c, agent)
		s.testStopAgent(c, agent)
	}

	stopAgent(s.agentProxy)
	stopAgent(s.agentStoreSlave)
	stopAgent(s.agentStoreMaster)
	stopAgent(s.agentDashboard)
}

func (s *testAgentSuite) testSetExecPath(c *C) {
	// the executer is in ../../bin path, we will add this path in global $PATH
	execPath := "../../bin"
	resetAbsPath(&execPath)
	c.Assert(strings.HasSuffix(execPath, "/src/github.com/reborndb/reborn/bin"), IsTrue)

	os.Setenv("PATH", os.ExpandEnv(fmt.Sprintf("${PATH}:%s", execPath)))

	_, err := exec.LookPath("reborn-config")
	c.Assert(err, IsNil)

	_, err = exec.LookPath("reborn-proxy")
	c.Assert(err, IsNil)

	_, err = exec.LookPath("reborn-server")
	c.Assert(err, IsNil)

	_, err = exec.LookPath("reborn-agent")
	c.Assert(err, IsNil)
}

func (s *testAgentSuite) testStartAgent(c *C, addr string, ha bool) testAgentInfo {
	dataDir := fmt.Sprintf("./var/%s/data", addr)
	logDir := fmt.Sprintf("./var/%s/log", addr)

	os.RemoveAll(dataDir)
	os.RemoveAll(logDir)

	os.MkdirAll(dataDir, 0755)
	os.MkdirAll(logDir, 0755)

	args := []string{
		"--addr", addr, "--data-dir", dataDir, "--log-dir",
		logDir, "-L", fmt.Sprintf("./var/%s/agent.log", addr)}

	if ha {
		args = append(args, "--ha", "--ha-max-retry-num", "1", "--ha-retry-delay", "1")
	}

	cmd := exec.Command("reborn-agent", args...)

	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	err := cmd.Start()
	c.Assert(err, IsNil)

	var info testAgentInfo
	info.cmd = cmd
	info.addr = addr

	time.Sleep(time.Second)

	return info
}

func (s *testAgentSuite) testStopAgent(c *C, agent testAgentInfo) {
	c.Assert(agent.cmd, NotNil)
	agent.cmd.Process.Signal(os.Interrupt)
	agent.cmd.Wait()
}

func (s *testAgentSuite) testStopAllProcs(c *C, agent testAgentInfo) {
	procs := s.testGetProcs(c, agent)

	for _, proc := range procs {
		agent.httpCall(c, nil, "stop", fmt.Sprintf("id=%s", proc.ID), "POST")
	}
}

func (s *testAgentSuite) testKillAllProcs(c *C, agent testAgentInfo) {
	procs := s.testGetProcs(c, agent)
	s.testKillProcs(c, procs)
}

func (s *testAgentSuite) testKillProcs(c *C, procs []procStatus) {
	for _, proc := range procs {
		p, err := os.FindProcess(proc.Pid)
		c.Assert(err, IsNil)
		p.Kill()
		p.Wait()
	}
}

func (s *testAgentSuite) testGetProcs(c *C, agent testAgentInfo) []procStatus {
	var procs []procStatus
	agent.httpCall(c, &procs, "procs", "", "GET")
	return procs
}

func (s *testAgentSuite) callDashboardAPI(c *C, retVal interface{}, apiPath string, method string, arg interface{}) {
	err := httpCall(retVal, fmt.Sprintf("http://%s%s", globalEnv.DashboardAddr(), apiPath), method, arg)
	c.Assert(err, IsNil)
}

func (s *testAgentSuite) testDashboard(c *C) {
	agent := s.agentDashboard

	agent.httpCall(c, nil, "start_dashboard", "", "POST")

	time.Sleep(2 * time.Second)
	s.callDashboardAPI(c, nil, "/ping", "GET", nil)

	// if we kill dashboard, the restart one will wait a coorindator session timeout time
	// for old session expired, this will be a long time, so we don't test this case here.
}

func (s *testAgentSuite) testStore(c *C, agent testAgentInfo, port int) {
	addr := fmt.Sprintf("127.0.0.1:%d", port)
	agent.httpCall(c, nil, "start_redis", fmt.Sprintf("addr=%s", url.QueryEscape(addr)), "POST")

	err := utils.Ping(addr, globalEnv.StoreAuth())
	c.Assert(err, IsNil)

	// kill store and then wait 2s for restart
	s.testKillAllProcs(c, agent)

	time.Sleep(2 * time.Second)

	err = utils.Ping(addr, globalEnv.StoreAuth())
	c.Assert(err, IsNil)
}

func (s *testAgentSuite) testStoreAddServer(c *C) {
	s.testStore(c, s.agentStoreMaster, 6381)
	s.testStore(c, s.agentStoreSlave, 6382)

	s.testAddStoreToGroup(c, 6381, models.SERVER_TYPE_MASTER)
	s.testAddStoreToGroup(c, 6382, models.SERVER_TYPE_SLAVE)

	s.checkStoreServerType(c, "127.0.0.1:6381", models.SERVER_TYPE_MASTER)
	s.checkStoreServerType(c, "127.0.0.1:6382", models.SERVER_TYPE_SLAVE)
}

func (s *testAgentSuite) testInitGroup(c *C) {
	serverGroup := models.NewServerGroup(globalEnv.ProductName(), 1)

	s.callDashboardAPI(c, nil, "/api/server_groups", "PUT", serverGroup)
}

func (s *testAgentSuite) testAddStoreToGroup(c *C, port int, role string) {
	server := models.NewServer(role, fmt.Sprintf("127.0.0.1:%d", port))
	s.callDashboardAPI(c, nil, fmt.Sprintf("/api/server_group/%d/addServer", 1), "PUT", server)
}

func (s *testAgentSuite) testInitSlots(c *C) {
	s.callDashboardAPI(c, nil, "/api/slots/init", "POST", nil)

	v := struct {
		FromSlot   int    `json:"from"`
		ToSlot     int    `json:"to"`
		NewGroupID int    `json:"new_group"`
		Status     string `json:"status"`
	}{
		FromSlot:   0,
		ToSlot:     1023,
		NewGroupID: 1,
		Status:     "online",
	}

	s.callDashboardAPI(c, nil, "/api/slot", "POST", v)
}

func (s *testAgentSuite) testProxy(c *C) {
	agent := s.agentProxy
	proxyAddr := "127.0.0.1:19000"
	proxyHTTPAddr := "127.0.0.1:29000"

	args := make(url.Values)
	args.Set("addr", proxyAddr)
	args.Set("http_addr", proxyHTTPAddr)

	agent.httpCall(c, nil, "start_proxy", args.Encode(), "POST")

	// now the proxy will wait 3s for online, this is very long for test
	// maybe later we will change it.
	var err error
	for i := 0; i < 3; i++ {
		time.Sleep(2 * time.Second)
		if err = utils.Ping(proxyAddr, ""); err == nil {
			break
		}
	}
	c.Assert(err, IsNil)

	// kill proxy and then wait for restart
	s.testKillAllProcs(c, agent)

	for i := 0; i < 3; i++ {
		time.Sleep(2 * time.Second)
		if err = utils.Ping(proxyAddr, ""); err == nil {
			break
		}
	}
	c.Assert(err, IsNil)
}

func (s *testAgentSuite) TestProxy(c *C) {
	s.testProxy(c)
}

func (s *testAgentSuite) checkStoreServerType(c *C, addr string, tp string) {
	group := models.NewServerGroup(globalEnv.ProductName(), 1)
	servers, err := group.GetServers(globalConn)
	c.Assert(err, IsNil)

	for _, server := range servers {
		if server.Addr == addr {
			c.Assert(server.Type, Equals, tp)
			return
		}
	}
	c.Fatalf("addr %s is not in group servers %v", addr, servers)
}

func (s *testAgentSuite) TestHA(c *C) {
	s.agentHA = s.testStartAgent(c, "127.0.0.1:39005", true)
	defer s.testStopAgent(c, s.agentHA)

	time.Sleep(3 * time.Second)

	// first test slave ha
	procs := s.testGetProcs(c, s.agentStoreSlave)
	// first stop agent then kill procs

	s.testStopAgent(c, s.agentStoreSlave)
	s.testKillProcs(c, procs)

	time.Sleep(3 * time.Second)

	s.checkStoreServerType(c, "127.0.0.1:6382", models.SERVER_TYPE_OFFLINE)

	s.agentStoreSlave = s.testStartAgent(c, "127.0.0.1:39004", false)
	s.testStore(c, s.agentStoreSlave, 6382)
	s.testAddStoreToGroup(c, 6382, models.SERVER_TYPE_SLAVE)

	s.checkStoreServerType(c, "127.0.0.1:6382", models.SERVER_TYPE_SLAVE)

	// test master ha
	procs = s.testGetProcs(c, s.agentStoreMaster)

	s.testStopAgent(c, s.agentStoreMaster)
	s.testKillProcs(c, procs)

	time.Sleep(3 * time.Second)
	// now 6382 is slave, and 6381 is offline
	s.checkStoreServerType(c, "127.0.0.1:6382", models.SERVER_TYPE_MASTER)
	s.checkStoreServerType(c, "127.0.0.1:6381", models.SERVER_TYPE_OFFLINE)

	s.agentStoreMaster = s.testStartAgent(c, "127.0.0.1:39003", false)
	s.testStore(c, s.agentStoreMaster, 6381)
	s.testAddStoreToGroup(c, 6381, models.SERVER_TYPE_SLAVE)

	s.checkStoreServerType(c, "127.0.0.1:6381", models.SERVER_TYPE_SLAVE)

}
<|endoftext|>


<commit_before>// Copyright 2013 bee authors
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.
package run

import (
	"io/ioutil"
	"os"
	path "path/filepath"
	"runtime"
	"strings"

	"github.com/beego/bee/cmd/commands"
	"github.com/beego/bee/cmd/commands/version"
	"github.com/beego/bee/config"
	beeLogger "github.com/beego/bee/logger"
	"github.com/beego/bee/utils"
)

var CmdRun = &commands.Command{
	UsageLine: "run [appname] [watchall] [-main=*.go] [-downdoc=true]  [-gendoc=true] [-vendor=true] [-e=folderToExclude] [-ex=extraPackageToWatch] [-tags=goBuildTags] [-runmode=BEEGO_RUNMODE]",
	Short:     "Run the application by starting a local development server",
	Long: `
Run command will supervise the filesystem of the application for any changes, and recompile/restart it.

`,
	PreRun: func(cmd *commands.Command, args []string) { version.ShowShortVersionBanner() },
	Run:    RunApp,
}

var (
	mainFiles utils.ListOpts
	downdoc   utils.DocValue
	gendoc    utils.DocValue
	// The flags list of the paths excluded from watching
	excludedPaths utils.StrFlags
	// Pass through to -tags arg of "go build"
	buildTags string
	// Application path
	currpath string
	// Application name
	appname string
	// Channel to signal an Exit
	exit chan bool
	// Flag to watch the vendor folder
	vendorWatch bool
	// Current user workspace
	currentGoPath string
	// Current runmode
	runmode string
	// Extra directories
	extraPackages utils.StrFlags
)
var started = make(chan bool)

func init() {
	CmdRun.Flag.Var(&mainFiles, "main", "Specify main go files.")
	CmdRun.Flag.Var(&gendoc, "gendoc", "Enable auto-generate the docs.")
	CmdRun.Flag.Var(&downdoc, "downdoc", "Enable auto-download of the swagger file if it does not exist.")
	CmdRun.Flag.Var(&excludedPaths, "e", "List of paths to exclude.")
	CmdRun.Flag.BoolVar(&vendorWatch, "vendor", false, "Enable watch vendor folder.")
	CmdRun.Flag.StringVar(&buildTags, "tags", "", "Set the build tags. See: https://golang.org/pkg/go/build/")
	CmdRun.Flag.StringVar(&runmode, "runmode", "", "Set the Beego run mode.")
	CmdRun.Flag.Var(&extraPackages, "ex", "List of extra package to watch.")
	exit = make(chan bool)
	commands.AvailableCommands = append(commands.AvailableCommands, CmdRun)
}

func RunApp(cmd *commands.Command, args []string) int {
	if len(args) == 0 || args[0] == "watchall" {
		currpath, _ = os.Getwd()
		if found, _gopath, _ := utils.SearchGOPATHs(currpath); found {
			appname = path.Base(currpath)
			currentGoPath = _gopath
		} else {
			beeLogger.Log.Fatalf("No application '%s' found in your GOPATH", currpath)
		}
	} else {
		// Check if passed Bee application path/name exists in the GOPATH(s)
		if found, _gopath, _path := utils.SearchGOPATHs(args[0]); found {
			currpath = _path
			currentGoPath = _gopath
			appname = path.Base(currpath)
		} else {
			beeLogger.Log.Fatalf("No application '%s' found in your GOPATH", args[0])
		}

		if strings.HasSuffix(appname, ".go") && utils.IsExist(currpath) {
			beeLogger.Log.Warnf("The appname is in conflict with file's current path. Do you want to build appname as '%s'", appname)
			beeLogger.Log.Info("Do you want to overwrite it? [yes|no] ")
			if !utils.AskForConfirmation() {
				return 0
			}
		}
	}

	beeLogger.Log.Infof("Using '%s' as 'appname'", appname)

	beeLogger.Log.Debugf("Current path: %s", utils.FILE(), utils.LINE(), currpath)

	if runmode == "prod" || runmode == "dev" {
		os.Setenv("BEEGO_RUNMODE", runmode)
		beeLogger.Log.Infof("Using '%s' as 'runmode'", os.Getenv("BEEGO_RUNMODE"))
	} else if runmode != "" {
		os.Setenv("BEEGO_RUNMODE", runmode)
		beeLogger.Log.Warnf("Using '%s' as 'runmode'", os.Getenv("BEEGO_RUNMODE"))
	} else if os.Getenv("BEEGO_RUNMODE") != "" {
		beeLogger.Log.Warnf("Using '%s' as 'runmode'", os.Getenv("BEEGO_RUNMODE"))
	}

	var paths []string
	readAppDirectories(currpath, &paths)

	// Because monitor files has some issues, we watch current directory
	// and ignore non-go files.
	for _, p := range config.Conf.DirStruct.Others {
		paths = append(paths, strings.Replace(p, "$GOPATH", currentGoPath, -1))
	}

	if len(extraPackages) > 0 {
		// get the full path
		for _, packagePath := range extraPackages {
			if found, _, _fullPath := utils.SearchGOPATHs(packagePath); found {
				readAppDirectories(_fullPath, &paths)
			} else {
				beeLogger.Log.Warnf("No extra package '%s' found in your GOPATH", packagePath)
			}
		}
		// let paths unique
		strSet := make(map[string]struct{})
		for _, p := range paths {
			strSet[p] = struct{}{}
		}
		paths = make([]string, len(strSet))
		index := 0
		for i := range strSet {
			paths[index] = i
			index++
		}
	}

	files := []string{}
	for _, arg := range mainFiles {
		if len(arg) > 0 {
			files = append(files, arg)
		}
	}
	if downdoc == "true" {
		if _, err := os.Stat(path.Join(currpath, "swagger", "index.html")); err != nil {
			if os.IsNotExist(err) {
				downloadFromURL(swaggerlink, "swagger.zip")
				unzipAndDelete("swagger.zip")
			}
		}
	}

	// Start the Reload server (if enabled)
	if config.Conf.EnableReload {
		startReloadServer()
	}
	if gendoc == "true" {
		NewWatcher(paths, files, true)
		AutoBuild(files, true)
	} else {
		NewWatcher(paths, files, false)
		AutoBuild(files, false)
	}

	for {
		<-exit
		runtime.Goexit()
	}
}

func readAppDirectories(directory string, paths *[]string) {
	fileInfos, err := ioutil.ReadDir(directory)
	if err != nil {
		return
	}

	useDirectory := false
	for _, fileInfo := range fileInfos {
		if strings.HasSuffix(fileInfo.Name(), "docs") {
			continue
		}
		if strings.HasSuffix(fileInfo.Name(), "swagger") {
			continue
		}

		if !vendorWatch && strings.HasSuffix(fileInfo.Name(), "vendor") {
			continue
		}

		if isExcluded(path.Join(directory, fileInfo.Name())) {
			continue
		}

		if fileInfo.IsDir() && fileInfo.Name()[0] != '.' {
			readAppDirectories(directory+"/"+fileInfo.Name(), paths)
			continue
		}

		if useDirectory {
			continue
		}

		if path.Ext(fileInfo.Name()) == ".go" || (ifStaticFile(fileInfo.Name()) && config.Conf.EnableReload) {
			*paths = append(*paths, directory)
			useDirectory = true
		}
	}
	return
}

// If a file is excluded
func isExcluded(filePath string) bool {
	for _, p := range excludedPaths {
		absP, err := path.Abs(p)
		if err != nil {
			beeLogger.Log.Errorf("Cannot get absolute path of '%s'", p)
			continue
		}
		absFilePath, err := path.Abs(filePath)
		if err != nil {
			beeLogger.Log.Errorf("Cannot get absolute path of '%s'", filePath)
			break
		}
		if strings.HasPrefix(absFilePath, absP) {
			beeLogger.Log.Infof("'%s' is not being watched", filePath)
			return true
		}
	}
	return false
}
<commit_msg>fix gosimple<commit_after>// Copyright 2013 bee authors
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.
package run

import (
	"io/ioutil"
	"os"
	path "path/filepath"
	"runtime"
	"strings"

	"github.com/beego/bee/cmd/commands"
	"github.com/beego/bee/cmd/commands/version"
	"github.com/beego/bee/config"
	beeLogger "github.com/beego/bee/logger"
	"github.com/beego/bee/utils"
)

var CmdRun = &commands.Command{
	UsageLine: "run [appname] [watchall] [-main=*.go] [-downdoc=true]  [-gendoc=true] [-vendor=true] [-e=folderToExclude] [-ex=extraPackageToWatch] [-tags=goBuildTags] [-runmode=BEEGO_RUNMODE]",
	Short:     "Run the application by starting a local development server",
	Long: `
Run command will supervise the filesystem of the application for any changes, and recompile/restart it.

`,
	PreRun: func(cmd *commands.Command, args []string) { version.ShowShortVersionBanner() },
	Run:    RunApp,
}

var (
	mainFiles utils.ListOpts
	downdoc   utils.DocValue
	gendoc    utils.DocValue
	// The flags list of the paths excluded from watching
	excludedPaths utils.StrFlags
	// Pass through to -tags arg of "go build"
	buildTags string
	// Application path
	currpath string
	// Application name
	appname string
	// Channel to signal an Exit
	exit chan bool
	// Flag to watch the vendor folder
	vendorWatch bool
	// Current user workspace
	currentGoPath string
	// Current runmode
	runmode string
	// Extra directories
	extraPackages utils.StrFlags
)
var started = make(chan bool)

func init() {
	CmdRun.Flag.Var(&mainFiles, "main", "Specify main go files.")
	CmdRun.Flag.Var(&gendoc, "gendoc", "Enable auto-generate the docs.")
	CmdRun.Flag.Var(&downdoc, "downdoc", "Enable auto-download of the swagger file if it does not exist.")
	CmdRun.Flag.Var(&excludedPaths, "e", "List of paths to exclude.")
	CmdRun.Flag.BoolVar(&vendorWatch, "vendor", false, "Enable watch vendor folder.")
	CmdRun.Flag.StringVar(&buildTags, "tags", "", "Set the build tags. See: https://golang.org/pkg/go/build/")
	CmdRun.Flag.StringVar(&runmode, "runmode", "", "Set the Beego run mode.")
	CmdRun.Flag.Var(&extraPackages, "ex", "List of extra package to watch.")
	exit = make(chan bool)
	commands.AvailableCommands = append(commands.AvailableCommands, CmdRun)
}

func RunApp(cmd *commands.Command, args []string) int {
	if len(args) == 0 || args[0] == "watchall" {
		currpath, _ = os.Getwd()
		if found, _gopath, _ := utils.SearchGOPATHs(currpath); found {
			appname = path.Base(currpath)
			currentGoPath = _gopath
		} else {
			beeLogger.Log.Fatalf("No application '%s' found in your GOPATH", currpath)
		}
	} else {
		// Check if passed Bee application path/name exists in the GOPATH(s)
		if found, _gopath, _path := utils.SearchGOPATHs(args[0]); found {
			currpath = _path
			currentGoPath = _gopath
			appname = path.Base(currpath)
		} else {
			beeLogger.Log.Fatalf("No application '%s' found in your GOPATH", args[0])
		}

		if strings.HasSuffix(appname, ".go") && utils.IsExist(currpath) {
			beeLogger.Log.Warnf("The appname is in conflict with file's current path. Do you want to build appname as '%s'", appname)
			beeLogger.Log.Info("Do you want to overwrite it? [yes|no] ")
			if !utils.AskForConfirmation() {
				return 0
			}
		}
	}

	beeLogger.Log.Infof("Using '%s' as 'appname'", appname)

	beeLogger.Log.Debugf("Current path: %s", utils.FILE(), utils.LINE(), currpath)

	if runmode == "prod" || runmode == "dev" {
		os.Setenv("BEEGO_RUNMODE", runmode)
		beeLogger.Log.Infof("Using '%s' as 'runmode'", os.Getenv("BEEGO_RUNMODE"))
	} else if runmode != "" {
		os.Setenv("BEEGO_RUNMODE", runmode)
		beeLogger.Log.Warnf("Using '%s' as 'runmode'", os.Getenv("BEEGO_RUNMODE"))
	} else if os.Getenv("BEEGO_RUNMODE") != "" {
		beeLogger.Log.Warnf("Using '%s' as 'runmode'", os.Getenv("BEEGO_RUNMODE"))
	}

	var paths []string
	readAppDirectories(currpath, &paths)

	// Because monitor files has some issues, we watch current directory
	// and ignore non-go files.
	for _, p := range config.Conf.DirStruct.Others {
		paths = append(paths, strings.Replace(p, "$GOPATH", currentGoPath, -1))
	}

	if len(extraPackages) > 0 {
		// get the full path
		for _, packagePath := range extraPackages {
			if found, _, _fullPath := utils.SearchGOPATHs(packagePath); found {
				readAppDirectories(_fullPath, &paths)
			} else {
				beeLogger.Log.Warnf("No extra package '%s' found in your GOPATH", packagePath)
			}
		}
		// let paths unique
		strSet := make(map[string]struct{})
		for _, p := range paths {
			strSet[p] = struct{}{}
		}
		paths = make([]string, len(strSet))
		index := 0
		for i := range strSet {
			paths[index] = i
			index++
		}
	}

	files := []string{}
	for _, arg := range mainFiles {
		if len(arg) > 0 {
			files = append(files, arg)
		}
	}
	if downdoc == "true" {
		if _, err := os.Stat(path.Join(currpath, "swagger", "index.html")); err != nil {
			if os.IsNotExist(err) {
				downloadFromURL(swaggerlink, "swagger.zip")
				unzipAndDelete("swagger.zip")
			}
		}
	}

	// Start the Reload server (if enabled)
	if config.Conf.EnableReload {
		startReloadServer()
	}
	if gendoc == "true" {
		NewWatcher(paths, files, true)
		AutoBuild(files, true)
	} else {
		NewWatcher(paths, files, false)
		AutoBuild(files, false)
	}

	for {
		<-exit
		runtime.Goexit()
	}
}

func readAppDirectories(directory string, paths *[]string) {
	fileInfos, err := ioutil.ReadDir(directory)
	if err != nil {
		return
	}

	useDirectory := false
	for _, fileInfo := range fileInfos {
		if strings.HasSuffix(fileInfo.Name(), "docs") {
			continue
		}
		if strings.HasSuffix(fileInfo.Name(), "swagger") {
			continue
		}

		if !vendorWatch && strings.HasSuffix(fileInfo.Name(), "vendor") {
			continue
		}

		if isExcluded(path.Join(directory, fileInfo.Name())) {
			continue
		}

		if fileInfo.IsDir() && fileInfo.Name()[0] != '.' {
			readAppDirectories(directory+"/"+fileInfo.Name(), paths)
			continue
		}

		if useDirectory {
			continue
		}

		if path.Ext(fileInfo.Name()) == ".go" || (ifStaticFile(fileInfo.Name()) && config.Conf.EnableReload) {
			*paths = append(*paths, directory)
			useDirectory = true
		}
	}
}

// If a file is excluded
func isExcluded(filePath string) bool {
	for _, p := range excludedPaths {
		absP, err := path.Abs(p)
		if err != nil {
			beeLogger.Log.Errorf("Cannot get absolute path of '%s'", p)
			continue
		}
		absFilePath, err := path.Abs(filePath)
		if err != nil {
			beeLogger.Log.Errorf("Cannot get absolute path of '%s'", filePath)
			break
		}
		if strings.HasPrefix(absFilePath, absP) {
			beeLogger.Log.Infof("'%s' is not being watched", filePath)
			return true
		}
	}
	return false
}
<|endoftext|>