<commit_before>// Copyright (c) 2017 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package service

import (
	"fmt"
	"os"
	"os/signal"
	"sync"
	"syscall"
	"time"

	"go.uber.org/fx/config"
	"go.uber.org/zap"

	"github.com/pkg/errors"
)

const (
	defaultStartupWait = time.Second
)

// A ExitCallback is a function to handle a service shutdown and provide
// an exit code
type ExitCallback func(shutdown Exit) int

// Implements Manager interface
type manager struct {
	serviceCore
	locked         bool
	observer       Observer
	moduleWrappers []*moduleWrapper
	roles          map[string]bool
	stateMu        sync.Mutex

	// Shutdown fields.
	shutdownMu     sync.Mutex
	inShutdown     bool  // Protected by shutdownMu
	shutdownReason *Exit // Protected by shutdownMu
	closeChan      chan Exit
	started        bool
}

// TODO(glib): manager is both an Manager and a Host?
var _ Host = &manager{}
var _ Manager = &manager{}

// newManager creates a service Manager from a Builder.
func newManager(builder *Builder) (Manager, error) {
	m := &manager{
		// TODO: get these out of config struct instead
		moduleWrappers: []*moduleWrapper{},
		serviceCore:    serviceCore{},
	}
	m.roles = map[string]bool{}
	for _, r := range m.standardConfig.Roles {
		m.roles[r] = true
	}
	for _, opt := range builder.options {
		if optionErr := opt(m); optionErr != nil {
			return nil, errors.Wrap(optionErr, "option failed to apply")
		}
	}
	if m.configProvider == nil {
		// If the user didn't pass in a configuration provider, load the standard.
		// Bypassing standard config load is pretty much only used for tests, although it could be
		// useful in certain circumstances.
		m.configProvider = config.Load()
	}
	if err := m.setupStandardConfig(); err != nil {
		return nil, err
	}
	// Initialize metrics. If no metrics reporters were Registered, do nop
	// TODO(glib): add a logging reporter and use it by default, rather than nop
	m.setupMetrics()
	if err := m.setupLogging(); err != nil {
		return nil, err
	}
	m.setupAuthClient()
	if err := m.setupRuntimeMetricsCollector(); err != nil {
		return nil, err
	}
	m.setupVersionMetricsEmitter()
	if err := m.setupTracer(); err != nil {
		return nil, err
	}
	// if we have an observer, look for a property called "config" and load the service
	// node into that config.
	m.setupObserver()
	m.transitionState(Initialized)
	m.Metrics().Counter("boot").Inc(1)
	for _, moduleInfo := range builder.moduleInfos {
		if err := m.addModule(moduleInfo.provider, moduleInfo.options...); err != nil {
			return nil, err
		}
	}
	return m, nil
}

func (m *manager) IsRunning() bool {
	return m.closeChan != nil
}

func (m *manager) OnCriticalError(err error) {
	shutdown := true
	if m.observer == nil {
		zap.L().Warn(
			"No observer set to handle lifecycle events. Shutting down.",
			zap.String("event", "OnCriticalError"),
		)
	} else {
		shutdown = !m.observer.OnCriticalError(err)
	}

	if shutdown {
		if ok, err := m.shutdown(err, "", nil); !ok || err != nil {
			// TODO(ai) verify we flush logs
			zap.L().Info("Problem shutting down module",
				zap.Bool("success", ok),
				zap.Error(err),
			)
		}
	}
}

// StartAsync service is used as a non-blocking the call on service start. StartAsync will
// return the call to the caller with a Control to listen on channels
// and trigger manual shutdowns.
func (m *manager) StartAsync() Control {
	return m.start()
}

// Start service is used for blocking the call on service start. Start will block the
// call and yield the control to the service lifecyce manager. Start will not yield back
// the control to the caller, so no code will be executed after calling Start()
func (m *manager) Start() {
	m.start()
	// block until forced exit
	m.WaitForShutdown(nil)
}

// Stop shuts down the service.
func (m *manager) Stop(reason string, exitCode int) error {
	ec := &exitCode
	_, err := m.shutdown(nil, reason, ec)
	return err
}

func (m *manager) WaitForShutdown(exitCallback ExitCallback) {
	shutdown := <-m.closeChan
	zap.L().Info("Shutting down", zap.String("reason", shutdown.Reason))

	exit := 0
	if exitCallback != nil {
		exit = exitCallback(shutdown)
	} else if shutdown.Error != nil {
		exit = 1
	}
	os.Exit(exit)
}

func (m *manager) supportsRole(roles ...string) bool {
	if len(m.roles) == 0 || len(roles) == 0 {
		return true
	}

	for _, role := range roles {
		if found, ok := m.roles[role]; found && ok {
			return true
		}
	}
	return false
}

func (m *manager) shutdown(err error, reason string, exitCode *int) (bool, error) {
	m.shutdownMu.Lock()
	m.inShutdown = true
	defer func() {
		m.inShutdown = false
		m.shutdownMu.Unlock()
	}()

	if m.shutdownReason != nil || !m.IsRunning() {
		return false, nil
	}

	m.transitionState(Stopping)

	m.shutdownReason = &Exit{
		Reason:   reason,
		Error:    err,
		ExitCode: 0,
	}

	if err != nil {
		if reason != "" {
			m.shutdownReason.Reason = err.Error()
		}
		m.shutdownReason.ExitCode = 1
	}

	if exitCode != nil {
		m.shutdownReason.ExitCode = *exitCode
	}

	// Log the module shutdown errors
	errs := m.stopModules()
	if len(errs) > 0 {
		for _, err := range errs {
			zap.L().Error("Failure to shut down module", zap.Error(err))
		}
	}

	// Stop runtime metrics collection. Uses scope, should be closed before scope is closed.
	if m.runtimeCollector != nil {
		m.runtimeCollector.Close()
	}

	if m.versionEmitter != nil {
		m.versionEmitter.close()
	}

	// Stop the metrics reporting
	if m.metricsCloser != nil {
		if err = m.metricsCloser.Close(); err != nil {
			zap.L().Error("Failure to close metrics", zap.Error(err))
		}
	}

	// Flush tracing buffers
	if m.tracerCloser != nil {
		zap.L().Debug("Closing tracer")
		if err = m.tracerCloser.Close(); err != nil {
			zap.L().Error("Failure to close tracer", zap.Error(err))
		}
	}

	// report that we shutdown.
	m.closeChan <- *m.shutdownReason
	close(m.closeChan)

	if m.observer != nil {
		m.observer.OnShutdown(*m.shutdownReason)
	}

	m.transitionState(Stopped)

	return true, err
}

func (m *manager) addModule(provider ModuleProvider, options ...ModuleOption) error {
	if m.locked {
		return fmt.Errorf("can't add module: service already started")
	}
	moduleWrapper, err := newModuleWrapper(m, provider, options...)
	if err != nil {
		return err
	}
	if moduleWrapper == nil {
		return nil
	}
	if !m.supportsRole(moduleWrapper.scopedHost.Roles()...) {
		zap.L().Info(
			"module will not be added due to selected roles",
			zap.Any("roles", moduleWrapper.scopedHost.Roles()),
		)
	}
	m.moduleWrappers = append(m.moduleWrappers, moduleWrapper)
	return nil
}

func (m *manager) start() Control {
	m.locked = true
	m.shutdownMu.Lock()
	m.transitionState(Starting)

	readyCh := make(chan struct{}, 1)
	defer func() {
		readyCh <- struct{}{}
	}()
	if m.inShutdown {
		m.shutdownMu.Unlock()
		return Control{
			ReadyChan:    readyCh,
			ServiceError: errors.New("shutting down the service"),
		}
	} else if m.IsRunning() {
		m.shutdownMu.Unlock()
		return Control{
			ExitChan:     m.closeChan,
			ReadyChan:    readyCh,
			ServiceError: errors.New("service is already running"),
		}
	} else {
		if m.observer != nil {
			if err := m.observer.OnInit(m); err != nil {
				m.shutdownMu.Unlock()
				return Control{
					ReadyChan:    readyCh,
					ServiceError: errors.Wrap(err, "failed to initialize the observer"),
				}
			}
		}
		m.shutdownReason = nil
		m.closeChan = make(chan Exit, 1)
		errs := m.startModules()
		m.registerSignalHandlers()
		if len(errs) > 0 {
			var serviceErr error
			errChan := make(chan Exit, 1)
			// grab the first error, shut down the service and return the error
			for _, e := range errs {
				errChan <- Exit{
					Error:    e,
					Reason:   "Module start failed",
					ExitCode: 4,
				}

				m.shutdownMu.Unlock()
				if _, err := m.shutdown(e, "", nil); err != nil {
					zap.L().Error("Unable to shut down modules",
						zap.NamedError("initialError", e),
						zap.NamedError("shutdownError", err),
					)
				}
				zap.L().Error("Error starting the module", zap.Error(e))
				// return first service error
				if serviceErr == nil {
					serviceErr = e
				}
			}
			return Control{
				ExitChan:     errChan,
				ReadyChan:    readyCh,
				ServiceError: serviceErr,
			}
		}
	}

	m.transitionState(Running)
	m.shutdownMu.Unlock()

	return Control{
		ExitChan:  m.closeChan,
		ReadyChan: readyCh,
	}
}

func (m *manager) registerSignalHandlers() {
	ch := make(chan os.Signal)
	signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM)
	go func() {
		sig := <-ch
		zap.L().Warn("Received shutdown signal", zap.String("signal", sig.String()))
		if err := m.Stop("Received syscall", 0); err != nil {
			zap.L().Error("Error shutting down", zap.Error(err))
		}
	}()
}

func (m *manager) startModules() []error {
	var results []error
	var lock sync.Mutex
	wg := sync.WaitGroup{}

	// make sure we wait for all the start
	// calls to return
	wg.Add(len(m.moduleWrappers))
	for _, mod := range m.moduleWrappers {
		go func(m *moduleWrapper) {
			if !m.IsRunning() {
				errC := make(chan error, 1)
				go func() { errC <- m.Start() }()
				select {
				case err := <-errC:
					if err != nil {
						zap.L().Error("Error received while starting module", zap.String("module", m.Name()), zap.Error(err))
						lock.Lock()
						results = append(results, err)
						lock.Unlock()
					} else {
						zap.L().Info("Module started up cleanly", zap.String("module", m.Name()))
					}
				case <-time.After(defaultStartupWait):
					lock.Lock()
					results = append(results, fmt.Errorf("module didn't start after %v", defaultStartupWait))
					lock.Unlock()
				}
			}
			wg.Done()
		}(mod)
	}

	// wait for the modules to all start
	wg.Wait()
	return results
}

func (m *manager) stopModules() []error {
	var results []error
	var lock sync.Mutex
	wg := sync.WaitGroup{}
	wg.Add(len(m.moduleWrappers))
	for _, mod := range m.moduleWrappers {
		go func(m *moduleWrapper) {
			if !m.IsRunning() {
				// TODO: have a timeout here so a bad shutdown
				// doesn't block everyone
				if err := m.Stop(); err != nil {
					lock.Lock()
					results = append(results, err)
					lock.Unlock()
				}
			}
			wg.Done()
		}(mod)
	}
	wg.Wait()
	return results
}

func (m *manager) transitionState(to State) {
	m.stateMu.Lock()
	defer m.stateMu.Unlock()

	// TODO(ai) this isn't used yet
	if to < m.state {
		zap.L().Fatal("Can't down from state",
			zap.Any("from", m.state),
			zap.Any("to", to),
			zap.String("service", m.Name()),
		)
	}

	for m.state < to {
		old := m.state
		newState := m.state
		switch m.state {
		case Uninitialized:
			newState = Initialized
		case Initialized:
			newState = Starting
		case Starting:
			newState = Running
		case Running:
			newState = Stopping
		case Stopping:
			newState = Stopped
		case Stopped:
		}
		m.state = newState
		if m.observer != nil {
			m.observer.OnStateChange(old, newState)
		}
	}
}
<commit_msg>Increase module startup timeout to 10 seconds (#370)<commit_after>// Copyright (c) 2017 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package service

import (
	"fmt"
	"os"
	"os/signal"
	"sync"
	"syscall"
	"time"

	"go.uber.org/fx/config"
	"go.uber.org/zap"

	"github.com/pkg/errors"
)

const (
	defaultStartupWait = 10 * time.Second
)

// A ExitCallback is a function to handle a service shutdown and provide
// an exit code
type ExitCallback func(shutdown Exit) int

// Implements Manager interface
type manager struct {
	serviceCore
	locked         bool
	observer       Observer
	moduleWrappers []*moduleWrapper
	roles          map[string]bool
	stateMu        sync.Mutex

	// Shutdown fields.
	shutdownMu     sync.Mutex
	inShutdown     bool  // Protected by shutdownMu
	shutdownReason *Exit // Protected by shutdownMu
	closeChan      chan Exit
	started        bool
}

// TODO(glib): manager is both an Manager and a Host?
var _ Host = &manager{}
var _ Manager = &manager{}

// newManager creates a service Manager from a Builder.
func newManager(builder *Builder) (Manager, error) {
	m := &manager{
		// TODO: get these out of config struct instead
		moduleWrappers: []*moduleWrapper{},
		serviceCore:    serviceCore{},
	}
	m.roles = map[string]bool{}
	for _, r := range m.standardConfig.Roles {
		m.roles[r] = true
	}
	for _, opt := range builder.options {
		if optionErr := opt(m); optionErr != nil {
			return nil, errors.Wrap(optionErr, "option failed to apply")
		}
	}
	if m.configProvider == nil {
		// If the user didn't pass in a configuration provider, load the standard.
		// Bypassing standard config load is pretty much only used for tests, although it could be
		// useful in certain circumstances.
		m.configProvider = config.Load()
	}
	if err := m.setupStandardConfig(); err != nil {
		return nil, err
	}
	// Initialize metrics. If no metrics reporters were Registered, do nop
	// TODO(glib): add a logging reporter and use it by default, rather than nop
	m.setupMetrics()
	if err := m.setupLogging(); err != nil {
		return nil, err
	}
	m.setupAuthClient()
	if err := m.setupRuntimeMetricsCollector(); err != nil {
		return nil, err
	}
	m.setupVersionMetricsEmitter()
	if err := m.setupTracer(); err != nil {
		return nil, err
	}
	// if we have an observer, look for a property called "config" and load the service
	// node into that config.
	m.setupObserver()
	m.transitionState(Initialized)
	m.Metrics().Counter("boot").Inc(1)
	for _, moduleInfo := range builder.moduleInfos {
		if err := m.addModule(moduleInfo.provider, moduleInfo.options...); err != nil {
			return nil, err
		}
	}
	return m, nil
}

func (m *manager) IsRunning() bool {
	return m.closeChan != nil
}

func (m *manager) OnCriticalError(err error) {
	shutdown := true
	if m.observer == nil {
		zap.L().Warn(
			"No observer set to handle lifecycle events. Shutting down.",
			zap.String("event", "OnCriticalError"),
		)
	} else {
		shutdown = !m.observer.OnCriticalError(err)
	}

	if shutdown {
		if ok, err := m.shutdown(err, "", nil); !ok || err != nil {
			// TODO(ai) verify we flush logs
			zap.L().Info("Problem shutting down module",
				zap.Bool("success", ok),
				zap.Error(err),
			)
		}
	}
}

// StartAsync service is used as a non-blocking the call on service start. StartAsync will
// return the call to the caller with a Control to listen on channels
// and trigger manual shutdowns.
func (m *manager) StartAsync() Control {
	return m.start()
}

// Start service is used for blocking the call on service start. Start will block the
// call and yield the control to the service lifecyce manager. Start will not yield back
// the control to the caller, so no code will be executed after calling Start()
func (m *manager) Start() {
	m.start()
	// block until forced exit
	m.WaitForShutdown(nil)
}

// Stop shuts down the service.
func (m *manager) Stop(reason string, exitCode int) error {
	ec := &exitCode
	_, err := m.shutdown(nil, reason, ec)
	return err
}

func (m *manager) WaitForShutdown(exitCallback ExitCallback) {
	shutdown := <-m.closeChan
	zap.L().Info("Shutting down", zap.String("reason", shutdown.Reason))

	exit := 0
	if exitCallback != nil {
		exit = exitCallback(shutdown)
	} else if shutdown.Error != nil {
		exit = 1
	}
	os.Exit(exit)
}

func (m *manager) supportsRole(roles ...string) bool {
	if len(m.roles) == 0 || len(roles) == 0 {
		return true
	}

	for _, role := range roles {
		if found, ok := m.roles[role]; found && ok {
			return true
		}
	}
	return false
}

func (m *manager) shutdown(err error, reason string, exitCode *int) (bool, error) {
	m.shutdownMu.Lock()
	m.inShutdown = true
	defer func() {
		m.inShutdown = false
		m.shutdownMu.Unlock()
	}()

	if m.shutdownReason != nil || !m.IsRunning() {
		return false, nil
	}

	m.transitionState(Stopping)

	m.shutdownReason = &Exit{
		Reason:   reason,
		Error:    err,
		ExitCode: 0,
	}

	if err != nil {
		if reason != "" {
			m.shutdownReason.Reason = err.Error()
		}
		m.shutdownReason.ExitCode = 1
	}

	if exitCode != nil {
		m.shutdownReason.ExitCode = *exitCode
	}

	// Log the module shutdown errors
	errs := m.stopModules()
	if len(errs) > 0 {
		for _, err := range errs {
			zap.L().Error("Failure to shut down module", zap.Error(err))
		}
	}

	// Stop runtime metrics collection. Uses scope, should be closed before scope is closed.
	if m.runtimeCollector != nil {
		m.runtimeCollector.Close()
	}

	if m.versionEmitter != nil {
		m.versionEmitter.close()
	}

	// Stop the metrics reporting
	if m.metricsCloser != nil {
		if err = m.metricsCloser.Close(); err != nil {
			zap.L().Error("Failure to close metrics", zap.Error(err))
		}
	}

	// Flush tracing buffers
	if m.tracerCloser != nil {
		zap.L().Debug("Closing tracer")
		if err = m.tracerCloser.Close(); err != nil {
			zap.L().Error("Failure to close tracer", zap.Error(err))
		}
	}

	// report that we shutdown.
	m.closeChan <- *m.shutdownReason
	close(m.closeChan)

	if m.observer != nil {
		m.observer.OnShutdown(*m.shutdownReason)
	}

	m.transitionState(Stopped)

	return true, err
}

func (m *manager) addModule(provider ModuleProvider, options ...ModuleOption) error {
	if m.locked {
		return fmt.Errorf("can't add module: service already started")
	}
	moduleWrapper, err := newModuleWrapper(m, provider, options...)
	if err != nil {
		return err
	}
	if moduleWrapper == nil {
		return nil
	}
	if !m.supportsRole(moduleWrapper.scopedHost.Roles()...) {
		zap.L().Info(
			"module will not be added due to selected roles",
			zap.Any("roles", moduleWrapper.scopedHost.Roles()),
		)
	}
	m.moduleWrappers = append(m.moduleWrappers, moduleWrapper)
	return nil
}

func (m *manager) start() Control {
	m.locked = true
	m.shutdownMu.Lock()
	m.transitionState(Starting)

	readyCh := make(chan struct{}, 1)
	defer func() {
		readyCh <- struct{}{}
	}()
	if m.inShutdown {
		m.shutdownMu.Unlock()
		return Control{
			ReadyChan:    readyCh,
			ServiceError: errors.New("shutting down the service"),
		}
	} else if m.IsRunning() {
		m.shutdownMu.Unlock()
		return Control{
			ExitChan:     m.closeChan,
			ReadyChan:    readyCh,
			ServiceError: errors.New("service is already running"),
		}
	} else {
		if m.observer != nil {
			if err := m.observer.OnInit(m); err != nil {
				m.shutdownMu.Unlock()
				return Control{
					ReadyChan:    readyCh,
					ServiceError: errors.Wrap(err, "failed to initialize the observer"),
				}
			}
		}
		m.shutdownReason = nil
		m.closeChan = make(chan Exit, 1)
		errs := m.startModules()
		m.registerSignalHandlers()
		if len(errs) > 0 {
			var serviceErr error
			errChan := make(chan Exit, 1)
			// grab the first error, shut down the service and return the error
			for _, e := range errs {
				errChan <- Exit{
					Error:    e,
					Reason:   "Module start failed",
					ExitCode: 4,
				}

				m.shutdownMu.Unlock()
				if _, err := m.shutdown(e, "", nil); err != nil {
					zap.L().Error("Unable to shut down modules",
						zap.NamedError("initialError", e),
						zap.NamedError("shutdownError", err),
					)
				}
				zap.L().Error("Error starting the module", zap.Error(e))
				// return first service error
				if serviceErr == nil {
					serviceErr = e
				}
			}
			return Control{
				ExitChan:     errChan,
				ReadyChan:    readyCh,
				ServiceError: serviceErr,
			}
		}
	}

	m.transitionState(Running)
	m.shutdownMu.Unlock()

	return Control{
		ExitChan:  m.closeChan,
		ReadyChan: readyCh,
	}
}

func (m *manager) registerSignalHandlers() {
	ch := make(chan os.Signal)
	signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM)
	go func() {
		sig := <-ch
		zap.L().Warn("Received shutdown signal", zap.String("signal", sig.String()))
		if err := m.Stop("Received syscall", 0); err != nil {
			zap.L().Error("Error shutting down", zap.Error(err))
		}
	}()
}

func (m *manager) startModules() []error {
	var results []error
	var lock sync.Mutex
	wg := sync.WaitGroup{}

	// make sure we wait for all the start
	// calls to return
	wg.Add(len(m.moduleWrappers))
	for _, mod := range m.moduleWrappers {
		go func(m *moduleWrapper) {
			if !m.IsRunning() {
				errC := make(chan error, 1)
				go func() { errC <- m.Start() }()
				select {
				case err := <-errC:
					if err != nil {
						zap.L().Error("Error received while starting module", zap.String("module", m.Name()), zap.Error(err))
						lock.Lock()
						results = append(results, err)
						lock.Unlock()
					} else {
						zap.L().Info("Module started up cleanly", zap.String("module", m.Name()))
					}
				case <-time.After(defaultStartupWait):
					lock.Lock()
					results = append(
						results,
						fmt.Errorf("module: %s didn't start after %v", m.Name(), defaultStartupWait),
					)
					lock.Unlock()
				}
			}
			wg.Done()
		}(mod)
	}

	// wait for the modules to all start
	wg.Wait()
	return results
}

func (m *manager) stopModules() []error {
	var results []error
	var lock sync.Mutex
	wg := sync.WaitGroup{}
	wg.Add(len(m.moduleWrappers))
	for _, mod := range m.moduleWrappers {
		go func(m *moduleWrapper) {
			if !m.IsRunning() {
				// TODO: have a timeout here so a bad shutdown
				// doesn't block everyone
				if err := m.Stop(); err != nil {
					lock.Lock()
					results = append(results, err)
					lock.Unlock()
				}
			}
			wg.Done()
		}(mod)
	}
	wg.Wait()
	return results
}

func (m *manager) transitionState(to State) {
	m.stateMu.Lock()
	defer m.stateMu.Unlock()

	// TODO(ai) this isn't used yet
	if to < m.state {
		zap.L().Fatal("Can't down from state",
			zap.Any("from", m.state),
			zap.Any("to", to),
			zap.String("service", m.Name()),
		)
	}

	for m.state < to {
		old := m.state
		newState := m.state
		switch m.state {
		case Uninitialized:
			newState = Initialized
		case Initialized:
			newState = Starting
		case Starting:
			newState = Running
		case Running:
			newState = Stopping
		case Stopping:
			newState = Stopped
		case Stopped:
		}
		m.state = newState
		if m.observer != nil {
			m.observer.OnStateChange(old, newState)
		}
	}
}
<|endoftext|>


<commit_before>package session

import (
	"github.com/gin-gonic/gin"
	"github.com/gorilla/sessions"
	"github.com/iiinsomnia/yiigo"
)

const gosessid = "GOSESSID"

var store = sessions.NewCookieStore([]byte(yiigo.EnvString("session", "secret", "N0awmAuS2OziVFu^9!*0LY7MeCRgQ&z0")))

// Get get session key - value
func Get(c *gin.Context, key string, defaultValule ...interface{}) (interface{}, error) {
	session, err := store.Get(c.Request, gosessid)

	if err != nil {
		yiigo.Err(err.Error())
		return nil, err
	}

	// Get some session values.
	v, ok := session.Values[key]

	if !ok {
		if len(defaultValule) > 0 {
			return defaultValule, nil
		}

		return nil, nil
	}

	return v, nil
}

// Set set session key - value, duration: seconds
func Set(c *gin.Context, key string, data interface{}, duration ...int) error {
	session, err := store.Get(c.Request, gosessid)

	if err != nil {
		yiigo.Err(err.Error())
		return err
	}

	if len(duration) > 0 {
		session.Options = &sessions.Options{
			Path:   "/",
			MaxAge: duration[0],
		}
	}

	// Set some session values.
	session.Values[key] = data
	// Save it before we write to the response/return from the handler.
	err = session.Save(c.Request, c.Writer)

	if err != nil {
		yiigo.Err(err.Error())
		return err
	}

	return nil
}

// Delete delete session key
func Delete(c *gin.Context, key string) error {
	session, err := store.Get(c.Request, gosessid)

	if err != nil {
		yiigo.Err(err.Error())
		return err
	}

	delete(session.Values, key)

	err = session.Save(c.Request, c.Writer)

	if err != nil {
		yiigo.Err(err.Error())
		return err
	}

	return nil
}

// Destroy destroy session
func Destroy(c *gin.Context) error {
	session, err := store.Get(c.Request, gosessid)

	if err != nil {
		yiigo.Err(err.Error())
		return err
	}

	session.Options = &sessions.Options{
		Path:   "/",
		MaxAge: -1,
	}

	err = session.Save(c.Request, c.Writer)

	if err != nil {
		yiigo.Err(err.Error())
		return err
	}

	return nil
}
<commit_msg>update<commit_after>package session

import (
	"github.com/gin-gonic/gin"
	"github.com/gorilla/sessions"
	"github.com/iiinsomnia/yiigo"
)

const gosessid = "GOSESSID"

var store *sessions.CookieStore

// Start start session
func Start() {
	store = sessions.NewCookieStore([]byte(yiigo.EnvString("session", "secret", "N0awmAuS2OziVFu^9!*0LY7MeCRgQ&z0")))
}

// Get get session key - value
func Get(c *gin.Context, key string, defaultValule ...interface{}) (interface{}, error) {
	session, err := store.Get(c.Request, gosessid)

	if err != nil {
		yiigo.Err(err.Error())
		return nil, err
	}

	// Get some session values.
	v, ok := session.Values[key]

	if !ok {
		if len(defaultValule) > 0 {
			return defaultValule, nil
		}

		return nil, nil
	}

	return v, nil
}

// Set set session key - value, duration: seconds
func Set(c *gin.Context, key string, data interface{}, duration ...int) error {
	session, err := store.Get(c.Request, gosessid)

	if err != nil {
		yiigo.Err(err.Error())
		return err
	}

	if len(duration) > 0 {
		session.Options = &sessions.Options{
			Path:   "/",
			MaxAge: duration[0],
		}
	}

	// Set some session values.
	session.Values[key] = data
	// Save it before we write to the response/return from the handler.
	err = session.Save(c.Request, c.Writer)

	if err != nil {
		yiigo.Err(err.Error())
		return err
	}

	return nil
}

// Delete delete session key
func Delete(c *gin.Context, key string) error {
	session, err := store.Get(c.Request, gosessid)

	if err != nil {
		yiigo.Err(err.Error())
		return err
	}

	delete(session.Values, key)

	err = session.Save(c.Request, c.Writer)

	if err != nil {
		yiigo.Err(err.Error())
		return err
	}

	return nil
}

// Destroy destroy session
func Destroy(c *gin.Context) error {
	session, err := store.Get(c.Request, gosessid)

	if err != nil {
		yiigo.Err(err.Error())
		return err
	}

	session.Options = &sessions.Options{
		Path:   "/",
		MaxAge: -1,
	}

	err = session.Save(c.Request, c.Writer)

	if err != nil {
		yiigo.Err(err.Error())
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package session

import (
	"github.com/devfeel/dotweb/framework/crypto"
	"github.com/devfeel/dotweb/framework/log"
	"net/http"
	"net/url"
	"strconv"
	"time"
)

const (
	DefaultSessionGCLifeTime  = 60      //second
	DefaultSessionMaxLifeTime = 20 * 60 //second
	DefaultSessionCookieName  = "dotweb_sessionId"
	DefaultSessionLength      = 20
	SessionMode_Runtime       = "runtime"
	SessionMode_Redis         = "redis"

	LogTarget_Session = "dotweb_session"
)

type (
	SessionStore interface {
		SessionRead(sessionId string) (*SessionState, error)
		SessionExist(sessionId string) bool
		SessionUpdate(state *SessionState) bool
		SessionRemove(sessionId string) error
		SessionCount() int //get all active session length
		SessionGC() int    //gc session and return out of date state num
	}

	//session config info
	StoreConfig struct {
		StoreName   string
		Maxlifetime int64
		ServerIP    string
	}

	SessionManager struct {
		store       SessionStore
		CookieName  string `json:"cookieName"`
		GCLifetime  int64  `json:"gclifetime"`
		storeConfig *StoreConfig
	}
)

//create new session store with store config
func GetSessionStore(config *StoreConfig) SessionStore {
	switch config.StoreName {
	case SessionMode_Runtime:
		return NewRuntimeStore(config)
	case SessionMode_Redis:
		return NewRedisStore(config)
	default:
		panic("not support session store -> " + config.StoreName)
	}
	return nil

}

//create new store with default config and use runtime store
func NewDefaultRuntimeConfig() *StoreConfig {
	return NewStoreConfig(SessionMode_Runtime, DefaultSessionMaxLifeTime, "")
}

//create new store with default config and use redis store
func NewDefaultRedisConfig(serverIp string) *StoreConfig {
	return NewStoreConfig(SessionMode_Redis, DefaultSessionMaxLifeTime, serverIp)
}

//create new store config
func NewStoreConfig(storeName string, maxlifetime int64, serverIp string) *StoreConfig {
	return &StoreConfig{
		StoreName:   storeName,
		Maxlifetime: maxlifetime,
		ServerIP:    serverIp,
	}
}

//create new session manager with default config info
func NewDefaultSessionManager(config *StoreConfig) (*SessionManager, error) {
	return NewSessionManager(DefaultSessionCookieName, DefaultSessionGCLifeTime, config)
}

//create new seesion manager
func NewSessionManager(cookieName string, gcLifetime int64, config *StoreConfig) (*SessionManager, error) {
	if gcLifetime <= 0 {
		gcLifetime = DefaultSessionGCLifeTime
	}
	if cookieName == "" {
		cookieName = DefaultSessionCookieName
	}
	manager := &SessionManager{
		store:       GetSessionStore(config),
		GCLifetime:  gcLifetime,
		storeConfig: config,
		CookieName:  cookieName,
	}
	//开启GC
	go func() {
		time.AfterFunc(time.Duration(manager.GCLifetime)*time.Second, func() { manager.GC() })
	}()
	return manager, nil
}

//create new session id with DefaultSessionLength
func (manager *SessionManager) NewSessionID() string {
	val := cryptos.GetRandString(DefaultSessionLength)
	return val
}

//get session id from client
//default mode is from cookie
func (manager *SessionManager) GetClientSessionID(req *http.Request) (string, error) {
	cookie, err := req.Cookie(manager.CookieName)
	if err != nil {
		return "", err
	}
	if cookie.Value == "" {
		return "", nil
	}
	//TODO: check client validity
	//check ip & agent
	return url.QueryUnescape(cookie.Value)
}

func (manager *SessionManager) GetSessionState(sessionId string) (session *SessionState, err error) {
	session, err = manager.store.SessionRead(sessionId)
}

//GC loop gc session data
func (manager *SessionManager) GC() {
	num := manager.store.SessionGC()
	if num > 0 {
		logger.Debug("SessionManger.GC => "+strconv.Itoa(num), LogTarget_Session)
	}
	time.AfterFunc(time.Duration(manager.GCLifetime)*time.Second, func() { manager.GC() })
}
<commit_msg>extension function and fixed bugs: 1、HttpContext.Redirect: add set nocache and support set statuscode 2、HrrpContext add Write(code, content), you can set code to write bytes 3、SessionStoreConfig remove UserName&Password field 4、add des\tripdes code<commit_after>package session

import (
	"github.com/devfeel/dotweb/framework/crypto"
	"github.com/devfeel/dotweb/framework/log"
	"net/http"
	"net/url"
	"strconv"
	"time"
)

const (
	DefaultSessionGCLifeTime  = 60      //second
	DefaultSessionMaxLifeTime = 20 * 60 //second
	DefaultSessionCookieName  = "dotweb_sessionId"
	DefaultSessionLength      = 20
	SessionMode_Runtime       = "runtime"
	SessionMode_Redis         = "redis"

	LogTarget_Session = "dotweb_session"
)

type (
	SessionStore interface {
		SessionRead(sessionId string) (*SessionState, error)
		SessionExist(sessionId string) bool
		SessionUpdate(state *SessionState) bool
		SessionRemove(sessionId string) error
		SessionCount() int //get all active session length
		SessionGC() int    //gc session and return out of date state num
	}

	//session config info
	StoreConfig struct {
		StoreName   string
		Maxlifetime int64
		ServerIP    string
	}

	SessionManager struct {
		store       SessionStore
		CookieName  string `json:"cookieName"`
		GCLifetime  int64  `json:"gclifetime"`
		storeConfig *StoreConfig
	}
)

//create new session store with store config
func GetSessionStore(config *StoreConfig) SessionStore {
	switch config.StoreName {
	case SessionMode_Runtime:
		return NewRuntimeStore(config)
	case SessionMode_Redis:
		return NewRedisStore(config)
	default:
		panic("not support session store -> " + config.StoreName)
	}
	return nil

}

//create new store with default config and use runtime store
func NewDefaultRuntimeConfig() *StoreConfig {
	return NewStoreConfig(SessionMode_Runtime, DefaultSessionMaxLifeTime, "")
}

//create new store with default config and use redis store
func NewDefaultRedisConfig(serverIp string) *StoreConfig {
	return NewStoreConfig(SessionMode_Redis, DefaultSessionMaxLifeTime, serverIp)
}

//create new store config
func NewStoreConfig(storeName string, maxlifetime int64, serverIp string) *StoreConfig {
	return &StoreConfig{
		StoreName:   storeName,
		Maxlifetime: maxlifetime,
		ServerIP:    serverIp,
	}
}

//create new session manager with default config info
func NewDefaultSessionManager(config *StoreConfig) (*SessionManager, error) {
	return NewSessionManager(DefaultSessionCookieName, DefaultSessionGCLifeTime, config)
}

//create new seesion manager
func NewSessionManager(cookieName string, gcLifetime int64, config *StoreConfig) (*SessionManager, error) {
	if gcLifetime <= 0 {
		gcLifetime = DefaultSessionGCLifeTime
	}
	if cookieName == "" {
		cookieName = DefaultSessionCookieName
	}
	manager := &SessionManager{
		store:       GetSessionStore(config),
		GCLifetime:  gcLifetime,
		storeConfig: config,
		CookieName:  cookieName,
	}
	//开启GC
	go func() {
		time.AfterFunc(time.Duration(manager.GCLifetime)*time.Second, func() { manager.GC() })
	}()
	return manager, nil
}

//create new session id with DefaultSessionLength
func (manager *SessionManager) NewSessionID() string {
	val := cryptos.GetRandString(DefaultSessionLength)
	return val
}

//get session id from client
//default mode is from cookie
func (manager *SessionManager) GetClientSessionID(req *http.Request) (string, error) {
	cookie, err := req.Cookie(manager.CookieName)
	if err != nil {
		return "", err
	}
	if cookie.Value == "" {
		return "", nil
	}
	//TODO: check client validity
	//check ip & agent
	return url.QueryUnescape(cookie.Value)
}

func (manager *SessionManager) GetSessionState(sessionId string) (session *SessionState, err error) {
	session, err = manager.store.SessionRead(sessionId)
	return
}

//GC loop gc session data
func (manager *SessionManager) GC() {
	num := manager.store.SessionGC()
	if num > 0 {
		logger.Debug("SessionManger.GC => "+strconv.Itoa(num), LogTarget_Session)
	}
	time.AfterFunc(time.Duration(manager.GCLifetime)*time.Second, func() { manager.GC() })
}
<|endoftext|>


<commit_before>package session

import (
	"hash"
	"io"
	"io/ioutil"
	"os"
	"syscall"

	"github.com/pkg/errors"

	"github.com/havoc-io/mutagen/sync"
)

const (
	// byteMax is the maximum value a byte can take.
	byteMax = 1<<8 - 1
)

type stagingSink struct {
	root string
	// path is the path that is being staged. It is not the path to the storage
	// or the staging destination.
	path     string
	storage  *os.File
	digester hash.Hash
}

func (s *stagingSink) Write(data []byte) (int, error) {
	// Write to the underlying storage.
	n, err := s.storage.Write(data)

	// Write as much to the digester as we wrote to the underlying storage. This
	// can't fail.
	s.digester.Write(data[:n])

	// Done.
	return n, err
}

func (s *stagingSink) Close() error {
	// Close the underlying storage.
	if err := s.storage.Close(); err != nil {
		return errors.Wrap(err, "unable to close underlying storage")
	}

	// Compute the final digest.
	digest := s.digester.Sum(nil)

	// Compute where the file should be relocated.
	destination, err := pathForStaging(s.root, s.path, digest)
	if err != nil {
		os.Remove(s.storage.Name())
		return errors.Wrap(err, "unable to compute staging destination")
	}

	// Relocate the file to the destination.
	// HACK: Just use syscall.Rename to avoid an os.Lstat call made by os.Rename
	// that can add significant overhead (~10% of total synchronization time) on
	// systems with slow stat implementations (e.g. macOS). We lose out on long
	// path fixes on Windows, but those shouldn't be necessary for paths inside
	// the staging directory (they are ~160 characters and up to 248 characters
	// is safe - see fixLongPath implementation in Go runtime on Windows). Even
	// on Linux this shaves about 2% of the time off.
	if err = syscall.Rename(s.storage.Name(), destination); err != nil {
		os.Remove(s.storage.Name())
		return errors.Wrap(err, "unable to relocate file")
	}

	// Success.
	return nil
}

// stagingCoordinator coordinates the reception of files via rsync (by
// implementing rsync.Sinker) and the provision of those files to transitions
// (by implementing sync.Provider).
type stagingCoordinator struct {
	version Version
	root    string
}

func newStagingCoordinator(session string, version Version, alpha bool) (*stagingCoordinator, error) {
	// Compute the staging root.
	root, err := pathForStagingRoot(session, alpha)
	if err != nil {
		return nil, errors.Wrap(err, "unable to compute staging root")
	}

	// Success.
	return &stagingCoordinator{
		version: version,
		root:    root,
	}, nil
}

func (c *stagingCoordinator) prepare() error {
	// Create the staging root and all of its prefix directories. We keep this
	// functionality in with the other path functionality to keep all of that
	// logic together.
	return createStagingRootWithPrefixes(c.root)
}

func (c *stagingCoordinator) wipe() error {
	return errors.Wrap(os.RemoveAll(c.root), "unable to remove staging directory")
}

func (c *stagingCoordinator) Sink(path string) (io.WriteCloser, error) {
	// Create a temporary storage file in the staging root.
	storage, err := ioutil.TempFile(c.root, "staging")
	if err != nil {
		return nil, errors.Wrap(err, "unable to create temporary storage file")
	}

	// Success.
	return &stagingSink{
		root:     c.root,
		path:     path,
		storage:  storage,
		digester: c.version.hasher(),
	}, nil
}

func (c *stagingCoordinator) Provide(path string, entry *sync.Entry) (string, error) {
	// Compute the expected location of the file.
	expectedLocation, err := pathForStaging(c.root, path, entry.Digest)
	if err != nil {
		return "", errors.Wrap(err, "unable to compute staging path")
	}

	// Ensure that it has the correct permissions. This will fail if the file
	// doens't exist.
	permissions := os.FileMode(0600)
	if entry.Executable {
		permissions = os.FileMode(0700)
	}
	if err = os.Chmod(expectedLocation, permissions); err != nil {
		return "", errors.Wrap(err, "unable to set file permissions")
	}

	// Success.
	return expectedLocation, nil
}
<commit_msg>Revert "Switched to syscall.Rename instead of os.Rename to avoid os.Lstat."<commit_after>package session

import (
	"hash"
	"io"
	"io/ioutil"
	"os"

	"github.com/pkg/errors"

	"github.com/havoc-io/mutagen/sync"
)

const (
	// byteMax is the maximum value a byte can take.
	byteMax = 1<<8 - 1
)

type stagingSink struct {
	root string
	// path is the path that is being staged. It is not the path to the storage
	// or the staging destination.
	path     string
	storage  *os.File
	digester hash.Hash
}

func (s *stagingSink) Write(data []byte) (int, error) {
	// Write to the underlying storage.
	n, err := s.storage.Write(data)

	// Write as much to the digester as we wrote to the underlying storage. This
	// can't fail.
	s.digester.Write(data[:n])

	// Done.
	return n, err
}

func (s *stagingSink) Close() error {
	// Close the underlying storage.
	if err := s.storage.Close(); err != nil {
		return errors.Wrap(err, "unable to close underlying storage")
	}

	// Compute the final digest.
	digest := s.digester.Sum(nil)

	// Compute where the file should be relocated.
	destination, err := pathForStaging(s.root, s.path, digest)
	if err != nil {
		os.Remove(s.storage.Name())
		return errors.Wrap(err, "unable to compute staging destination")
	}

	// Relocate the file to the destination.
	if err = os.Rename(s.storage.Name(), destination); err != nil {
		os.Remove(s.storage.Name())
		return errors.Wrap(err, "unable to relocate file")
	}

	// Success.
	return nil
}

// stagingCoordinator coordinates the reception of files via rsync (by
// implementing rsync.Sinker) and the provision of those files to transitions
// (by implementing sync.Provider).
type stagingCoordinator struct {
	version Version
	root    string
}

func newStagingCoordinator(session string, version Version, alpha bool) (*stagingCoordinator, error) {
	// Compute the staging root.
	root, err := pathForStagingRoot(session, alpha)
	if err != nil {
		return nil, errors.Wrap(err, "unable to compute staging root")
	}

	// Success.
	return &stagingCoordinator{
		version: version,
		root:    root,
	}, nil
}

func (c *stagingCoordinator) prepare() error {
	// Create the staging root and all of its prefix directories. We keep this
	// functionality in with the other path functionality to keep all of that
	// logic together.
	return createStagingRootWithPrefixes(c.root)
}

func (c *stagingCoordinator) wipe() error {
	return errors.Wrap(os.RemoveAll(c.root), "unable to remove staging directory")
}

func (c *stagingCoordinator) Sink(path string) (io.WriteCloser, error) {
	// Create a temporary storage file in the staging root.
	storage, err := ioutil.TempFile(c.root, "staging")
	if err != nil {
		return nil, errors.Wrap(err, "unable to create temporary storage file")
	}

	// Success.
	return &stagingSink{
		root:     c.root,
		path:     path,
		storage:  storage,
		digester: c.version.hasher(),
	}, nil
}

func (c *stagingCoordinator) Provide(path string, entry *sync.Entry) (string, error) {
	// Compute the expected location of the file.
	expectedLocation, err := pathForStaging(c.root, path, entry.Digest)
	if err != nil {
		return "", errors.Wrap(err, "unable to compute staging path")
	}

	// Ensure that it has the correct permissions. This will fail if the file
	// doens't exist.
	permissions := os.FileMode(0600)
	if entry.Executable {
		permissions = os.FileMode(0700)
	}
	if err = os.Chmod(expectedLocation, permissions); err != nil {
		return "", errors.Wrap(err, "unable to set file permissions")
	}

	// Success.
	return expectedLocation, nil
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"strconv"
	"strings"
	"time"
)

// HTTPVer is an HTTP version this server supports.
const HTTPVer = "HTTP/1.0"

// headerDelim is an HTTP request header delimiter.
const headerDelim = ": "

const maxRequestBodyLength = 1024 * 1024

const (
	serverName = "LittleHTTP"
	serverVer  = "1.0"
)

// HTTPHeaderField represents an HTTP header field.
type HTTPHeaderField map[string]string

// HTTPRequest represents an HTTP request.
type HTTPRequest struct {
	method string
	path   string
	ver    float64
	header HTTPHeaderField
	body   []byte
	length int64
}

func main() {
	args := os.Args

	if len(args) != 2 {
		fmt.Fprintf(os.Stderr, "Usage: %v <docroot>\n", args[0])
		return
	}

	// TODO: installSignalHandlers()

	docroot := args[1]
	service(os.Stdin, os.Stdout, docroot)
}

func service(in, out *os.File, docroot string) {
	req, err := readRequest(in)
	if err != nil {
		perror(err)
		return
	}

	err = respondTo(req, out, docroot)
	if err != nil {
		perror(err)
		return
	}
}

func readRequest(in *os.File) (*HTTPRequest, error) {
	r := bufio.NewReader(in)

	// read request line
	line, err := r.ReadString('\n')
	if err == io.EOF {
		return nil, fmt.Errorf("failed to read request: EOF")
	}

	// parse request line to HTTPRequest
	req, err := parseRequestLine(line)
	if err != nil {
		return nil, fmt.Errorf("failed to read request: %v", err)
	}

	// iterate to read request header fields
	h := make(HTTPHeaderField)
	for {
		line, err := r.ReadString('\n')
		// finish reading if EOF or empty line
		if line == "\n" || line == "\r\n" {
			break
		}

		// parse header field
		key, val, err := parseHeaderField(line)
		if err != nil {
			return nil, fmt.Errorf("failed to read request: %v", err)
		}

		h[key] = val
	}
	req.header = h

	// set content length
	// TODO: do this in constructor
	l, err := req.contentLength()
	if err != nil {
		return nil, fmt.Errorf("failed to parse content length: %v", err)
	}
	req.length = l

	if req.length == 0 {
		return req, nil
	}

	if req.length > maxRequestBodyLength {
		return nil, fmt.Errorf("request body too long: %v", req.length)
	}

	// set content body
	// TODO: do this in constructor
	b := make([]byte, req.length)
	if _, e := r.Read(b); e != nil {
		return nil, fmt.Errorf("failed to read request body: %v", b)
	}
	req.body = b

	return req, err
}

func parseRequestLine(line string) (*HTTPRequest, error) {
	// trim trailing \r, \n
	line = strings.Trim(line, "\r\n")

	// split request line
	p := strings.Split(line, " ")
	if len(p) != 3 { // METHOD Request-URI HTTP-Version (RFC2616)
		return nil, fmt.Errorf("parse error on request line: %v", line)
	}

	// check supported HTTP version
	if p[2] != HTTPVer {
		return nil, fmt.Errorf("not supported HTTP version: %v", p[2])
	}

	ver, err := parseHTTPVer(p[2])
	if err != nil {
		return nil, fmt.Errorf("invalid HTTP version: %v", p[2])
	}

	// p consists of ["METHOD", "/path/to/file", "HTTP/ver"]
	req := &HTTPRequest{
		method: strings.ToUpper(p[0]), // HTTP method is upper case
		path:   p[1],
		ver:    ver,
	}

	return req, nil
}

func parseHTTPVer(ver string) (float64, error) {
	l := len("HTTP/")
	return strconv.ParseFloat(ver[l:], 64)
}

// parseHeaderField parses request header to (key, val, err).
func parseHeaderField(line string) (key, val string, err error) {
	// trim trailing \r, \n
	line = strings.Trim(line, "\r\n")

	// split header field
	p := strings.Split(line, headerDelim)
	if len(p) != 2 { // key:value style
		err = fmt.Errorf("parse error on request header field: %v", line)
		return
	}

	key, val = p[0], p[1]
	return
}

func (req *HTTPRequest) contentLength() (int64, error) {
	key := "Content-Length"

	val, err := req.headerValue(key)
	if err != nil {
		return 0, nil
	}

	l, e := strconv.ParseInt(val, 10, 64)
	if e != nil {
		return 0, fmt.Errorf("parse error on ``%v: %v`: %v", key, val, e)
	}

	return l, nil
}

func (req *HTTPRequest) headerValue(name string) (string, error) {
	val, exists := req.header[name]
	if !exists {
		return "", fmt.Errorf("header `%v` not exist", name)
	}

	return val, nil
}

func respondTo(req *HTTPRequest, out *os.File, docroot string) error {
	var err error

	switch req.method {
	case "GET", "HEAD":
		err = doFileResponse(req, out, docroot)
	case "POST":
		methodNotAllowed(req, out)
	default:
		notImplemented(req, out)
	}

	return err
}

func doFileResponse(req *HTTPRequest, out *os.File, docroot string) error {
	fi, err := getFileInfo(docroot, req.path)
	if err != nil {
		notFound(req, out)
		return fmt.Errorf("getFileInfo(%s, %s) failed: %v", docroot, req.path, err)
	}

	outputCommonHeaderFields(req, out, "200 OK")
	fmt.Fprintf(out, "Content-Length: %d\r\n", fi.Size())
	fmt.Fprintf(out, "Content-Type: %s\r\n\r\n", guessContentType(fi))

	// HEAD method responses only headers
	if req.method == "HEAD" {
		return nil
	}

	// read file contents
	fspath := buildFSPath(docroot, req.path)
	err = printFileContents(fspath, out)
	if err != nil {
		return fmt.Errorf("doFileResponse() failed: %v", err)
	}

	return nil
}

func printFileContents(path string, out *os.File) error {
	// Note: suppose the size of the requested file is small
	buf, err := ioutil.ReadFile(path)
	if err != nil {
		return fmt.Errorf("failed to read file %s: %v", path, err)
	}

	if _, e := out.Write(buf); e != nil {
		return fmt.Errorf("failed to write: %v", err)
	}

	return nil
}

func notFound(req *HTTPRequest, out *os.File) {
	outputCommonHeaderFields(req, out, "404 Not Found")
	fmt.Fprintf(out, "Content-Type: text/plain\r\n\r\n")
	fmt.Fprintf(out, "File not found\r\n")
}

func methodNotAllowed(req *HTTPRequest, out *os.File) {
	outputCommonHeaderFields(req, out, "405 Method Not Allowed")
	fmt.Fprintf(out, "Content-Type: text/plain\r\n\r\n")
	fmt.Fprintf(out, "The requested method %s is not allowed\r\n", req.method)
}

func notImplemented(req *HTTPRequest, out *os.File) {
	outputCommonHeaderFields(req, out, "501 Not Implemented")
	fmt.Fprintf(out, "Content-Type: text/plain\r\n\r\n")
	fmt.Fprintf(out, "The requested method %s is not implemented\r\n", req.method)
}

func outputCommonHeaderFields(req *HTTPRequest, out *os.File, status string) {
	fmt.Fprintf(out, "%s %s\r\n", HTTPVer, status)
	fmt.Fprintf(out, "Date: %s\r\n", time.Now().Format(time.RFC1123))
	fmt.Fprintf(out, "Server: %s/%s\r\n", serverName, serverVer)
	fmt.Fprintf(out, "Connection: close\r\n")
}

func guessContentType(fi os.FileInfo) string {
	return "text/plain"
}

func getFileInfo(docroot, urlpath string) (os.FileInfo, error) {
	path := buildFSPath(docroot, urlpath)
	return os.Lstat(path)
}

func buildFSPath(docroot, urlpath string) string {
	return fmt.Sprintf("%s/%s", docroot, urlpath)
}

func installSignalHandlers() {
	// TODO
	return
}

func perror(err error) {
	fmt.Fprintf(os.Stderr, "[error] %v\n", err)
}
<commit_msg>httpd.go: format constants<commit_after>package main

import (
	"bufio"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"strconv"
	"strings"
	"time"
)

const (
	httpVer     = "HTTP/1.0"
	headerDelim = ": "
	maxBodyLen  = 1024 * 1024
	serverName  = "LittleHTTP"
	serverVer   = "1.0"
)

// HTTPHeaderField represents an HTTP header field.
type HTTPHeaderField map[string]string

// HTTPRequest represents an HTTP request.
type HTTPRequest struct {
	method string
	path   string
	ver    float64
	header HTTPHeaderField
	body   []byte
	length int64
}

func main() {
	args := os.Args

	if len(args) != 2 {
		fmt.Fprintf(os.Stderr, "Usage: %v <docroot>\n", args[0])
		return
	}

	// TODO: installSignalHandlers()

	docroot := args[1]
	service(os.Stdin, os.Stdout, docroot)
}

func service(in, out *os.File, docroot string) {
	req, err := readRequest(in)
	if err != nil {
		perror(err)
		return
	}

	err = respondTo(req, out, docroot)
	if err != nil {
		perror(err)
		return
	}
}

func readRequest(in *os.File) (*HTTPRequest, error) {
	r := bufio.NewReader(in)

	// read request line
	line, err := r.ReadString('\n')
	if err == io.EOF {
		return nil, fmt.Errorf("failed to read request: EOF")
	}

	// parse request line to HTTPRequest
	req, err := parseRequestLine(line)
	if err != nil {
		return nil, fmt.Errorf("failed to read request: %v", err)
	}

	// iterate to read request header fields
	h := make(HTTPHeaderField)
	for {
		line, err := r.ReadString('\n')
		// finish reading if EOF or empty line
		if line == "\n" || line == "\r\n" {
			break
		}

		// parse header field
		key, val, err := parseHeaderField(line)
		if err != nil {
			return nil, fmt.Errorf("failed to read request: %v", err)
		}

		h[key] = val
	}
	req.header = h

	// set content length
	// TODO: do this in constructor
	l, err := req.contentLength()
	if err != nil {
		return nil, fmt.Errorf("failed to parse content length: %v", err)
	}
	req.length = l

	if req.length == 0 {
		return req, nil
	}

	if req.length > maxBodyLen {
		return nil, fmt.Errorf("request body too long: %v", req.length)
	}

	// set content body
	// TODO: do this in constructor
	b := make([]byte, req.length)
	if _, e := r.Read(b); e != nil {
		return nil, fmt.Errorf("failed to read request body: %v", b)
	}
	req.body = b

	return req, err
}

func parseRequestLine(line string) (*HTTPRequest, error) {
	// trim trailing \r, \n
	line = strings.Trim(line, "\r\n")

	// split request line
	p := strings.Split(line, " ")
	if len(p) != 3 { // METHOD Request-URI HTTP-Version (RFC2616)
		return nil, fmt.Errorf("parse error on request line: %v", line)
	}

	// check supported HTTP version
	if p[2] != httpVer {
		return nil, fmt.Errorf("not supported HTTP version: %v", p[2])
	}

	ver, err := parseHTTPVer(p[2])
	if err != nil {
		return nil, fmt.Errorf("invalid HTTP version: %v", p[2])
	}

	// p consists of ["METHOD", "/path/to/file", "HTTP/ver"]
	req := &HTTPRequest{
		method: strings.ToUpper(p[0]), // HTTP method is upper case
		path:   p[1],
		ver:    ver,
	}

	return req, nil
}

func parseHTTPVer(ver string) (float64, error) {
	l := len("HTTP/")
	return strconv.ParseFloat(ver[l:], 64)
}

// parseHeaderField parses request header to (key, val, err).
func parseHeaderField(line string) (key, val string, err error) {
	// trim trailing \r, \n
	line = strings.Trim(line, "\r\n")

	// split header field
	p := strings.Split(line, headerDelim)
	if len(p) != 2 { // key:value style
		err = fmt.Errorf("parse error on request header field: %v", line)
		return
	}

	key, val = p[0], p[1]
	return
}

func (req *HTTPRequest) contentLength() (int64, error) {
	key := "Content-Length"

	val, err := req.headerValue(key)
	if err != nil {
		return 0, nil
	}

	l, e := strconv.ParseInt(val, 10, 64)
	if e != nil {
		return 0, fmt.Errorf("parse error on ``%v: %v`: %v", key, val, e)
	}

	return l, nil
}

func (req *HTTPRequest) headerValue(name string) (string, error) {
	val, exists := req.header[name]
	if !exists {
		return "", fmt.Errorf("header `%v` not exist", name)
	}

	return val, nil
}

func respondTo(req *HTTPRequest, out *os.File, docroot string) error {
	var err error

	switch req.method {
	case "GET", "HEAD":
		err = doFileResponse(req, out, docroot)
	case "POST":
		methodNotAllowed(req, out)
	default:
		notImplemented(req, out)
	}

	return err
}

func doFileResponse(req *HTTPRequest, out *os.File, docroot string) error {
	fi, err := getFileInfo(docroot, req.path)
	if err != nil {
		notFound(req, out)
		return fmt.Errorf("getFileInfo(%s, %s) failed: %v", docroot, req.path, err)
	}

	outputCommonHeaderFields(req, out, "200 OK")
	fmt.Fprintf(out, "Content-Length: %d\r\n", fi.Size())
	fmt.Fprintf(out, "Content-Type: %s\r\n\r\n", guessContentType(fi))

	// HEAD method responses only headers
	if req.method == "HEAD" {
		return nil
	}

	// read file contents
	fspath := buildFSPath(docroot, req.path)
	err = printFileContents(fspath, out)
	if err != nil {
		return fmt.Errorf("doFileResponse() failed: %v", err)
	}

	return nil
}

func printFileContents(path string, out *os.File) error {
	// Note: suppose the size of the requested file is small
	buf, err := ioutil.ReadFile(path)
	if err != nil {
		return fmt.Errorf("failed to read file %s: %v", path, err)
	}

	if _, e := out.Write(buf); e != nil {
		return fmt.Errorf("failed to write: %v", err)
	}

	return nil
}

func notFound(req *HTTPRequest, out *os.File) {
	outputCommonHeaderFields(req, out, "404 Not Found")
	fmt.Fprintf(out, "Content-Type: text/plain\r\n\r\n")
	fmt.Fprintf(out, "File not found\r\n")
}

func methodNotAllowed(req *HTTPRequest, out *os.File) {
	outputCommonHeaderFields(req, out, "405 Method Not Allowed")
	fmt.Fprintf(out, "Content-Type: text/plain\r\n\r\n")
	fmt.Fprintf(out, "The requested method %s is not allowed\r\n", req.method)
}

func notImplemented(req *HTTPRequest, out *os.File) {
	outputCommonHeaderFields(req, out, "501 Not Implemented")
	fmt.Fprintf(out, "Content-Type: text/plain\r\n\r\n")
	fmt.Fprintf(out, "The requested method %s is not implemented\r\n", req.method)
}

func outputCommonHeaderFields(req *HTTPRequest, out *os.File, status string) {
	fmt.Fprintf(out, "%s %s\r\n", httpVer, status)
	fmt.Fprintf(out, "Date: %s\r\n", time.Now().Format(time.RFC1123))
	fmt.Fprintf(out, "Server: %s/%s\r\n", serverName, serverVer)
	fmt.Fprintf(out, "Connection: close\r\n")
}

func guessContentType(fi os.FileInfo) string {
	return "text/plain"
}

func getFileInfo(docroot, urlpath string) (os.FileInfo, error) {
	path := buildFSPath(docroot, urlpath)
	return os.Lstat(path)
}

func buildFSPath(docroot, urlpath string) string {
	return fmt.Sprintf("%s/%s", docroot, urlpath)
}

func installSignalHandlers() {
	// TODO
	return
}

func perror(err error) {
	fmt.Fprintf(os.Stderr, "[error] %v\n", err)
}
<|endoftext|>


<commit_before>package resolv

import (
	"fmt"
	"log"
	"net"
	"strings"

	"github.com/miekg/dns"
	"golang.org/x/net/context"
)

// Iterator represents an iterative DNS query.
type Iterator struct {
	Debug bool
	rs    *Resolver
}

// NewIterator initializes an Iterator structure.
func NewIterator(r *Resolver) *Iterator {
	return &Iterator{rs: r}
}

// LookupIPv4 looks up the IPv4 addresses for the host.
func (it *Iterator) LookupIPv4(ctx context.Context, host string) ([]net.IP, error) {
	fqdn := dns.Fqdn(strings.ToLower(host))

	var last *Response
	for r := range it.Resolve(ctx, fqdn, dns.TypeA) {
		if r.Err != nil {
			return nil, r.Err
		}
		last = r
	}

	var ips []net.IP
	for _, i := range last.Msg.Answer {
		switch i.(type) {
		case (*dns.A):
			rr := i.(*dns.A)
			ips = append(ips, rr.A)
		default:
			return nil, fmt.Errorf("iterator: unexpected record: %v", i)
		}
	}

	return ips, nil
}

// Resolve looks up the name starting from root servers following referals.
func (it *Iterator) Resolve(ctx context.Context, name string, type_ uint16) <-chan *Response {
	return it.run(ctx, name, type_, 0, 0, map[string]bool{}, RootServers...)
}

func (it *Iterator) run(ctx context.Context, name string, type_ uint16, depth, i int, skip map[string]bool, nss ...string) <-chan *Response {
	out := make(chan *Response, MaxDepth)
	defer close(out)

	if depth > MaxDepth {
		out <- &Response{Err: fmt.Errorf("iterator: max depth reached")}
		return out
	}

	for len(nss) > 0 {
		var ns string

		i++
		if i > MaxIterations {
			out <- &Response{Err: fmt.Errorf("iterator: max iterations reached")}
			return out
		}

		// Peek random name server and mark it as used.
		ns, nss = peekRandom(nss)
		if _, ok := skip[ns]; ok {
			continue
		}
		skip[ns] = true

		// Issue DNS query.
		fqdn := dns.Fqdn(strings.ToLower(name))
		c := it.rs.Resolve(
			NewRequest(ns, fqdn, type_),
		)

		select {
		case resp := <-c:
			if it.Debug {
				log.Println("iterator: servers=", nss)
				log.Println("iterator: ===>", resp.Addr(), resp)
			}

			if resp.Err != nil {
				// Ignore DNS errors, stop if error is NXDOMAIN.
				if err, ok := resp.Err.(*DNSError); ok {
					switch {
					case err.NameError():
						out <- &Response{Err: err}
						return out
					default:
						continue
					}
				}
				out <- resp
				return out
			}

			if referals, cname, ok := it.lookup(resp.Msg, fqdn); ok {
				if cname != "" {
					return it.run(ctx, cname, type_, depth+1, i, map[string]bool{}, RootServers...)
				} else {
					out <- resp
					return out
				}
			} else {
				if len(referals) > 0 {
					return it.run(ctx, name, type_, depth+1, i, skip, referals...)
				}
			}
		case <-ctx.Done():
			out <- &Response{Err: ctx.Err()}
			return out
		}
	}

	out <- &Response{Err: fmt.Errorf("iterator: no more servers to try")}
	return out
}

func (it *Iterator) lookup(msg *dns.Msg, fqdn string) ([]string, string, bool) {
	cname := ""
	found := false

	// Look for fqdn in the Answer section.
	for _, i := range msg.Answer {
		nm := strings.ToLower(i.Header().Name)
		if nm == fqdn {
			found = true
		}

		switch i.(type) {
		case (*dns.CNAME):
			rr := i.(*dns.CNAME)
			cname = dns.Fqdn(strings.ToLower(rr.Target))
		}
	}

	if found {
		return nil, cname, found
	}

	// If fqdn is not found then collect and return referals.
	var nss []string
	for _, i := range msg.Ns {
		switch i.(type) {
		case *dns.NS:
			rr := i.(*dns.NS)
			nm := strings.ToLower(rr.Header().Name)

			if strings.HasSuffix(fqdn, nm) {
				nss = append(nss, strings.ToLower(rr.Ns))
			}
		default:
			log.Println("iterator: bad RR type", i, "for", fqdn)
		}
	}

	return nss, cname, found
}
<commit_msg>Increment the iterations using for<commit_after>package resolv

import (
	"fmt"
	"log"
	"net"
	"strings"

	"github.com/miekg/dns"
	"golang.org/x/net/context"
)

// Iterator represents an iterative DNS query.
type Iterator struct {
	Debug bool
	rs    *Resolver
}

// NewIterator initializes an Iterator structure.
func NewIterator(r *Resolver) *Iterator {
	return &Iterator{rs: r}
}

// LookupIPv4 looks up the IPv4 addresses for the host.
func (it *Iterator) LookupIPv4(ctx context.Context, host string) ([]net.IP, error) {
	fqdn := dns.Fqdn(strings.ToLower(host))

	var last *Response
	for r := range it.Resolve(ctx, fqdn, dns.TypeA) {
		if r.Err != nil {
			return nil, r.Err
		}
		last = r
	}

	var ips []net.IP
	for _, i := range last.Msg.Answer {
		switch i.(type) {
		case (*dns.A):
			rr := i.(*dns.A)
			ips = append(ips, rr.A)
		default:
			return nil, fmt.Errorf("iterator: unexpected record: %v", i)
		}
	}

	return ips, nil
}

// Resolve looks up the name starting from root servers following referals.
func (it *Iterator) Resolve(ctx context.Context, name string, type_ uint16) <-chan *Response {
	return it.run(ctx, name, type_, 0, 0, map[string]bool{}, RootServers...)
}

func (it *Iterator) run(ctx context.Context, name string, type_ uint16, depth, i int, skip map[string]bool, nss ...string) <-chan *Response {
	out := make(chan *Response, MaxDepth)
	defer close(out)

	if depth > MaxDepth {
		out <- &Response{Err: fmt.Errorf("iterator: max depth reached")}
		return out
	}

	for ; len(nss) > 0; i++ {
		var ns string

		if i > MaxIterations {
			out <- &Response{Err: fmt.Errorf("iterator: max iterations reached")}
			return out
		}

		// Peek random name server and mark it as used.
		ns, nss = peekRandom(nss)
		if _, ok := skip[ns]; ok {
			continue
		}
		skip[ns] = true

		// Issue DNS query.
		fqdn := dns.Fqdn(strings.ToLower(name))
		c := it.rs.Resolve(
			NewRequest(ns, fqdn, type_),
		)

		select {
		case resp := <-c:
			if it.Debug {
				log.Println("iterator: servers=", nss)
				log.Println("iterator: ===>", resp.Addr(), resp)
			}

			if resp.Err != nil {
				// Ignore DNS errors, stop if error is NXDOMAIN.
				if err, ok := resp.Err.(*DNSError); ok {
					switch {
					case err.NameError():
						out <- &Response{Err: err}
						return out
					default:
						continue
					}
				}
				out <- resp
				return out
			}

			if referals, cname, ok := it.lookup(resp.Msg, fqdn); ok {
				if cname != "" {
					return it.run(ctx, cname, type_, depth+1, i+1, map[string]bool{}, RootServers...)
				} else {
					out <- resp
					return out
				}
			} else {
				if len(referals) > 0 {
					return it.run(ctx, name, type_, depth+1, i+1, skip, referals...)
				}
			}
		case <-ctx.Done():
			out <- &Response{Err: ctx.Err()}
			return out
		}
	}

	out <- &Response{Err: fmt.Errorf("iterator: no more servers to try")}
	return out
}

func (it *Iterator) lookup(msg *dns.Msg, fqdn string) ([]string, string, bool) {
	cname := ""
	found := false

	// Look for fqdn in the Answer section.
	for _, i := range msg.Answer {
		nm := strings.ToLower(i.Header().Name)
		if nm == fqdn {
			found = true
		}

		switch i.(type) {
		case (*dns.CNAME):
			rr := i.(*dns.CNAME)
			cname = dns.Fqdn(strings.ToLower(rr.Target))
		}
	}

	if found {
		return nil, cname, found
	}

	// If fqdn is not found then collect and return referals.
	var nss []string
	for _, i := range msg.Ns {
		switch i.(type) {
		case *dns.NS:
			rr := i.(*dns.NS)
			nm := strings.ToLower(rr.Header().Name)

			if strings.HasSuffix(fqdn, nm) {
				nss = append(nss, strings.ToLower(rr.Ns))
			}
		default:
			log.Println("iterator: bad RR type", i, "for", fqdn)
		}
	}

	return nss, cname, found
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/sha256"
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"

	"github.com/fsnotify/fsnotify"

	"github.com/NebulousLabs/Sia/api"
)

type apiClient interface {
	Get(string, interface{}) error
	Post(string, string, interface{}) error
}

// SiaFolder is a folder that is synchronized to a Sia node.
type SiaFolder struct {
	path    string
	client  apiClient
	watcher *fsnotify.Watcher

	files map[string]string // files is a map of file paths to SHA256 checksums, used to reconcile file changes

	closeChan chan struct{}
}

// NewSiafolder creates a new SiaFolder using the provided path and api
// address.
func NewSiafolder(path string, client apiClient) (*SiaFolder, error) {
	sf := &SiaFolder{}
	abspath, err := filepath.Abs(path)
	if err != nil {
		return nil, err
	}
	sf.path = abspath
	sf.files = make(map[string]string)
	sf.closeChan = make(chan struct{})

	sf.client = client
	var contracts api.RenterContracts
	err = sf.client.Get("/renter/contracts", &contracts)
	if err != nil {
		return nil, err
	}
	if len(contracts.Contracts) == 0 {
		return nil, errors.New("you must have formed contracts to upload to Sia")
	}

	// watch for file changes
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		return nil, err
	}
	err = watcher.Add(abspath)
	if err != nil {
		return nil, err
	}

	sf.watcher = watcher

	// walk the provided path, accumulating a slice of files to potentially
	// upload and adding any subdirectories to the watcher.
	err = filepath.Walk(abspath, func(walkpath string, f os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if walkpath == path {
			return nil
		}

		if f.IsDir() {
			// subdirectories must be added to the watcher.
			watcher.Add(walkpath)
		} else {
			checksum, err := checksumFile(walkpath)
			if err != nil {
				return err
			}
			sf.files[walkpath] = checksum
		}
		return nil
	})
	if err != nil {
		return nil, err
	}

	err = sf.uploadNonExisting()
	if err != nil {
		return nil, err
	}

	go sf.eventWatcher()

	return sf, nil
}

// checksumFile returns a sha256 checksum of a given file on disk.
func checksumFile(path string) (string, error) {
	f, err := os.Open(path)
	if err != nil {
		return "", err
	}
	defer f.Close()

	h := sha256.New()
	if _, err := io.Copy(h, f); err != nil {
		return "", err
	}

	return string(h.Sum(nil)), nil
}

// eventWatcher continuously listens on the SiaFolder's watcher channels and
// performs the necessary upload/delete operations.
func (sf *SiaFolder) eventWatcher() {
	for {
		select {
		case <-sf.closeChan:
			return
		case event := <-sf.watcher.Events:
			filename := filepath.Clean(event.Name)
			f, err := os.Stat(filename)
			if err == nil && f.IsDir() {
				sf.watcher.Add(filename)
				continue
			}

			// WRITE event, checksum the file and re-upload it if it has changed
			if event.Op&fsnotify.Write == fsnotify.Write {
				err = sf.handleFileWrite(filename)
				if err != nil {
					log.Println(err)
				}
			}

			// REMOVE event
			if event.Op&fsnotify.Remove == fsnotify.Remove {
				log.Println("file removal detected, removing", filename)
				err = sf.handleRemove(filename)
				if err != nil {
					log.Println(err)
				}
			}

			// CREATE event
			if event.Op&fsnotify.Create == fsnotify.Create {
				log.Println("file creation detected, uploading", filename)
				err = sf.handleCreate(filename)
				if err != nil {
					log.Println(err)
				}
			}

		case err := <-sf.watcher.Errors:
			if err != nil {
				log.Println("fsevents error:", err)
			}
		}
	}
}

// handleFileWrite handles a WRITE fsevent.
func (sf *SiaFolder) handleFileWrite(file string) error {
	checksum, err := checksumFile(file)
	if err != nil {
		return err
	}

	oldChecksum, exists := sf.files[file]
	if exists && oldChecksum != checksum {
		log.Printf("change in %v detected, reuploading..\n", file)
		sf.files[file] = checksum
		err = sf.handleRemove(file)
		if err != nil {
			return err
		}
		err = sf.handleCreate(file)
		if err != nil {
			return err
		}
	}

	return nil
}

// Close releases any resources allocated by a SiaFolder.
func (sf *SiaFolder) Close() error {
	close(sf.closeChan)
	return sf.watcher.Close()
}

// handleCreate handles a file creation event. `file` is a relative path to the
// file on disk.
func (sf *SiaFolder) handleCreate(file string) error {
	abspath, err := filepath.Abs(file)
	if err != nil {
		return fmt.Errorf("error getting absolute path to upload: %v\n:", err)
	}
	relpath, err := filepath.Rel(sf.path, file)
	if err != nil {
		return fmt.Errorf("error getting relative path to upload: %v\n", err)
	}
	err = sf.client.Post(fmt.Sprintf("/renter/upload/%v", relpath), fmt.Sprintf("source=%v", abspath), nil)
	if err != nil {
		return fmt.Errorf("error uploading %v: %v\n", file, err)
	}
	checksum, err := checksumFile(file)
	if err != nil {
		return err
	}
	sf.files[file] = checksum
	return nil
}

// handleRemove handles a file removal event.
func (sf *SiaFolder) handleRemove(file string) error {
	relpath, err := filepath.Rel(sf.path, file)
	if err != nil {
		return fmt.Errorf("error getting relative path to remove: %v\n", err)
	}
	err = sf.client.Post(fmt.Sprintf("/renter/delete/%v", relpath), "", nil)
	if err != nil {
		return fmt.Errorf("error removing %v: %v\n", file, err)
	}
	delete(sf.files, file)
	return nil
}

// uploadNonExisting runs once and performs any uploads required to ensure
// every file in files is uploaded to the Sia node.
func (sf *SiaFolder) uploadNonExisting() error {
	var renterFiles api.RenterFiles
	err := sf.client.Get("/renter/files", &renterFiles)
	if err != nil {
		return err
	}

	for file := range sf.files {
		relpath, err := filepath.Rel(sf.path, file)
		if err != nil {
			return err
		}
		exists := false
		for _, siafile := range renterFiles.Files {
			if siafile.SiaPath == relpath {
				exists = true
			}
		}

		if !exists {
			sf.handleCreate(file)
		}
	}

	return nil
}
<commit_msg>newlines cleanup<commit_after>package main

import (
	"crypto/sha256"
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"

	"github.com/fsnotify/fsnotify"

	"github.com/NebulousLabs/Sia/api"
)

type apiClient interface {
	Get(string, interface{}) error
	Post(string, string, interface{}) error
}

// SiaFolder is a folder that is synchronized to a Sia node.
type SiaFolder struct {
	path    string
	client  apiClient
	watcher *fsnotify.Watcher

	files map[string]string // files is a map of file paths to SHA256 checksums, used to reconcile file changes

	closeChan chan struct{}
}

// NewSiafolder creates a new SiaFolder using the provided path and api
// address.
func NewSiafolder(path string, client apiClient) (*SiaFolder, error) {
	sf := &SiaFolder{}

	abspath, err := filepath.Abs(path)
	if err != nil {
		return nil, err
	}

	sf.path = abspath
	sf.files = make(map[string]string)
	sf.closeChan = make(chan struct{})
	sf.client = client

	var contracts api.RenterContracts
	err = sf.client.Get("/renter/contracts", &contracts)
	if err != nil {
		return nil, err
	}
	if len(contracts.Contracts) == 0 {
		return nil, errors.New("you must have formed contracts to upload to Sia")
	}

	// watch for file changes
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		return nil, err
	}
	err = watcher.Add(abspath)
	if err != nil {
		return nil, err
	}

	sf.watcher = watcher

	// walk the provided path, accumulating a slice of files to potentially
	// upload and adding any subdirectories to the watcher.
	err = filepath.Walk(abspath, func(walkpath string, f os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if walkpath == path {
			return nil
		}

		if f.IsDir() {
			// subdirectories must be added to the watcher.
			watcher.Add(walkpath)
		} else {
			checksum, err := checksumFile(walkpath)
			if err != nil {
				return err
			}
			sf.files[walkpath] = checksum
		}
		return nil
	})
	if err != nil {
		return nil, err
	}

	err = sf.uploadNonExisting()
	if err != nil {
		return nil, err
	}

	go sf.eventWatcher()

	return sf, nil
}

// checksumFile returns a sha256 checksum of a given file on disk.
func checksumFile(path string) (string, error) {
	f, err := os.Open(path)
	if err != nil {
		return "", err
	}
	defer f.Close()

	h := sha256.New()
	if _, err := io.Copy(h, f); err != nil {
		return "", err
	}

	return string(h.Sum(nil)), nil
}

// eventWatcher continuously listens on the SiaFolder's watcher channels and
// performs the necessary upload/delete operations.
func (sf *SiaFolder) eventWatcher() {
	for {
		select {
		case <-sf.closeChan:
			return
		case event := <-sf.watcher.Events:
			filename := filepath.Clean(event.Name)
			f, err := os.Stat(filename)
			if err == nil && f.IsDir() {
				sf.watcher.Add(filename)
				continue
			}

			// WRITE event, checksum the file and re-upload it if it has changed
			if event.Op&fsnotify.Write == fsnotify.Write {
				err = sf.handleFileWrite(filename)
				if err != nil {
					log.Println(err)
				}
			}

			// REMOVE event
			if event.Op&fsnotify.Remove == fsnotify.Remove {
				log.Println("file removal detected, removing", filename)
				err = sf.handleRemove(filename)
				if err != nil {
					log.Println(err)
				}
			}

			// CREATE event
			if event.Op&fsnotify.Create == fsnotify.Create {
				log.Println("file creation detected, uploading", filename)
				err = sf.handleCreate(filename)
				if err != nil {
					log.Println(err)
				}
			}

		case err := <-sf.watcher.Errors:
			if err != nil {
				log.Println("fsevents error:", err)
			}
		}
	}
}

// handleFileWrite handles a WRITE fsevent.
func (sf *SiaFolder) handleFileWrite(file string) error {
	checksum, err := checksumFile(file)
	if err != nil {
		return err
	}

	oldChecksum, exists := sf.files[file]
	if exists && oldChecksum != checksum {
		log.Printf("change in %v detected, reuploading..\n", file)
		sf.files[file] = checksum
		err = sf.handleRemove(file)
		if err != nil {
			return err
		}
		err = sf.handleCreate(file)
		if err != nil {
			return err
		}
	}

	return nil
}

// Close releases any resources allocated by a SiaFolder.
func (sf *SiaFolder) Close() error {
	close(sf.closeChan)
	return sf.watcher.Close()
}

// handleCreate handles a file creation event. `file` is a relative path to the
// file on disk.
func (sf *SiaFolder) handleCreate(file string) error {
	abspath, err := filepath.Abs(file)
	if err != nil {
		return fmt.Errorf("error getting absolute path to upload: %v\n:", err)
	}
	relpath, err := filepath.Rel(sf.path, file)
	if err != nil {
		return fmt.Errorf("error getting relative path to upload: %v\n", err)
	}
	err = sf.client.Post(fmt.Sprintf("/renter/upload/%v", relpath), fmt.Sprintf("source=%v", abspath), nil)
	if err != nil {
		return fmt.Errorf("error uploading %v: %v\n", file, err)
	}
	checksum, err := checksumFile(file)
	if err != nil {
		return err
	}
	sf.files[file] = checksum
	return nil
}

// handleRemove handles a file removal event.
func (sf *SiaFolder) handleRemove(file string) error {
	relpath, err := filepath.Rel(sf.path, file)
	if err != nil {
		return fmt.Errorf("error getting relative path to remove: %v\n", err)
	}
	err = sf.client.Post(fmt.Sprintf("/renter/delete/%v", relpath), "", nil)
	if err != nil {
		return fmt.Errorf("error removing %v: %v\n", file, err)
	}
	delete(sf.files, file)
	return nil
}

// uploadNonExisting runs once and performs any uploads required to ensure
// every file in files is uploaded to the Sia node.
func (sf *SiaFolder) uploadNonExisting() error {
	var renterFiles api.RenterFiles
	err := sf.client.Get("/renter/files", &renterFiles)
	if err != nil {
		return err
	}

	for file := range sf.files {
		relpath, err := filepath.Rel(sf.path, file)
		if err != nil {
			return err
		}
		exists := false
		for _, siafile := range renterFiles.Files {
			if siafile.SiaPath == relpath {
				exists = true
			}
		}

		if !exists {
			sf.handleCreate(file)
		}
	}

	return nil
}
<|endoftext|>


<commit_before>// Alexandria
//
// Copyright (C) 2015  Colin Benner
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package alexandria

import (
	"io/ioutil"
	"strings"

	"github.com/blevesearch/bleve"
)

// Run index++ to generate a (new) swish++ index file.
func GenerateIndex() error {
	// Try to open an existing index or create a new one if none exists.
	index, err := openIndex()
	if err != nil {
		mapping := bleve.NewIndexMapping()
		index, err = bleve.New(Config.AlexandriaDirectory+"bleve", mapping)
		if err != nil {
			return err
		}
	}
	defer index.Close()

	files, err := ioutil.ReadDir(Config.KnowledgeDirectory)
	if err != nil {
		return err
	}

	for _, f := range files {
		id := strings.TrimSuffix(f.Name(), ".tex")
		contentBytes, err := ioutil.ReadFile(Config.KnowledgeDirectory + f.Name())
		TryLogError(err)
		content := string(contentBytes)
		metadata := ParseMetadata(content)
		scroll := Scroll{metadata, Id(id), content}

		index.Index(id, scroll)
	}

	return nil
}

// Open the bleve index
func openIndex() (bleve.Index, error) {
	return bleve.Open(Config.AlexandriaDirectory + "bleve")
}

// Search the swish index for a given query.
func searchBleve(queryString string) ([]Id, error) {
	index, err := openIndex()
	if err != nil {
		LogError(err)
		return nil, err
	}
	defer index.Close()

	query := bleve.NewQueryStringQuery(queryString)
	search := bleve.NewSearchRequest(query)
	searchResults, err := index.Search(search)
	if err != nil {
		LogError(err)
		return nil, err
	}

	results := make([]Id, Config.MaxResults)
	for i, match := range searchResults.Hits {
		results[i] = Id(match.ID)
	}

	return results[:len(searchResults.Hits)], nil
}

// Get a list of scrolls matching the query.
func FindScrolls(query string) ([]Id, error) {
	ids, err := searchBleve(query)
	if err != nil {
		return nil, err
	}
	ProcessScrolls(ids)

	return ids, nil
}

func ComputeStatistics() Statistics {
	index, err := openIndex()
	if err != nil {
		LogError(err)
	}
	defer index.Close()

	num, size := getDirSize(Config.KnowledgeDirectory)
	if err == nil {
		tmp, err := index.DocCount()
		if err != nil {
			LogError(err)
		} else {
			num = int(tmp)
		}
	}

	return Stats{num, size}
}
<commit_msg>Only update the index for new scrolls<commit_after>// Alexandria
//
// Copyright (C) 2015  Colin Benner
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package alexandria

import (
	"io/ioutil"
	"os"
	"strings"
	"time"

	"github.com/blevesearch/bleve"
)

func touch(file string) error {
	now := time.Now()
	return os.Chtimes(file, now, now)
}

// Run index++ to generate a (new) swish++ index file.
func GenerateIndex() error {
	// Try to open an existing index or create a new one if none exists.
	index, err := openIndex()
	if err != nil {
		mapping := bleve.NewIndexMapping()
		index, err = bleve.New(Config.AlexandriaDirectory+"bleve", mapping)
		if err != nil {
			return err
		}
	}
	defer index.Close()

	files, err := ioutil.ReadDir(Config.KnowledgeDirectory)
	if err != nil {
		return err
	}

	indexUpdateFile := Config.AlexandriaDirectory + "index_updated"
	indexUpdateTime, err := getModTime(indexUpdateFile)
	if err != nil {
		LogError(err)
		return nil
	}
	// Save the time of this indexing operation
	_ = touch(Config.AlexandriaDirectory + "index_updated")
	for _, file := range files {
		// Check whether the scroll is newer than the index.
		modTime, err := getModTime(Config.KnowledgeDirectory + file.Name())
		if err != nil {
			LogError(err)
			continue
		}
		id := strings.TrimSuffix(file.Name(), ".tex")
		if modTime < indexUpdateTime {
			continue
		}

		// Load and parse the scroll content
		contentBytes, err := ioutil.ReadFile(Config.KnowledgeDirectory + file.Name())
		TryLogError(err)
		content := string(contentBytes)
		metadata := ParseMetadata(content)
		scroll := Scroll{metadata, Id(id), content}

		index.Index(id, scroll)
	}

	return nil
}

// Open the bleve index
func openIndex() (bleve.Index, error) {
	return bleve.Open(Config.AlexandriaDirectory + "bleve")
}

// Search the swish index for a given query.
func searchBleve(queryString string) ([]Id, error) {
	index, err := openIndex()
	if err != nil {
		LogError(err)
		return nil, err
	}
	defer index.Close()

	query := bleve.NewQueryStringQuery(queryString)
	search := bleve.NewSearchRequest(query)
	searchResults, err := index.Search(search)
	if err != nil {
		LogError(err)
		return nil, err
	}

	results := make([]Id, Config.MaxResults)
	for i, match := range searchResults.Hits {
		results[i] = Id(match.ID)
	}

	return results[:len(searchResults.Hits)], nil
}

// Get a list of scrolls matching the query.
func FindScrolls(query string) ([]Id, error) {
	ids, err := searchBleve(query)
	if err != nil {
		return nil, err
	}
	ProcessScrolls(ids)

	return ids, nil
}

func ComputeStatistics() Statistics {
	index, err := openIndex()
	if err != nil {
		LogError(err)
	}
	defer index.Close()

	num, size := getDirSize(Config.KnowledgeDirectory)
	if err == nil {
		tmp, err := index.DocCount()
		if err != nil {
			LogError(err)
		} else {
			num = int(tmp)
		}
	}

	return Stats{num, size}
}
<|endoftext|>