<commit_before>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package factory

import (
	"fmt"
	"regexp"
	"sort"
	"strings"
	"sync"

	"github.com/golang/glog"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/kubernetes/plugin/pkg/scheduler/algorithm"
	"k8s.io/kubernetes/plugin/pkg/scheduler/algorithm/predicates"
	"k8s.io/kubernetes/plugin/pkg/scheduler/algorithm/priorities"
	schedulerapi "k8s.io/kubernetes/plugin/pkg/scheduler/api"
)

// PluginFactoryArgs are passed to all plugin factory functions.
type PluginFactoryArgs struct {
	PodLister                      algorithm.PodLister
	ServiceLister                  algorithm.ServiceLister
	ControllerLister               algorithm.ControllerLister
	ReplicaSetLister               algorithm.ReplicaSetLister
	StatefulSetLister              algorithm.StatefulSetLister
	NodeLister                     algorithm.NodeLister
	NodeInfo                       predicates.NodeInfo
	PVInfo                         predicates.PersistentVolumeInfo
	PVCInfo                        predicates.PersistentVolumeClaimInfo
	HardPodAffinitySymmetricWeight int
}

// MetadataProducerFactory produces MetadataProducer from the given args.
type MetadataProducerFactory func(PluginFactoryArgs) algorithm.MetadataProducer

// A FitPredicateFactory produces a FitPredicate from the given args.
type FitPredicateFactory func(PluginFactoryArgs) algorithm.FitPredicate

// DEPRECATED
// Use Map-Reduce pattern for priority functions.
// A PriorityFunctionFactory produces a PriorityConfig from the given args.
type PriorityFunctionFactory func(PluginFactoryArgs) algorithm.PriorityFunction

// A PriorityFunctionFactory produces map & reduce priority functions
// from a given args.
// FIXME: Rename to PriorityFunctionFactory.
type PriorityFunctionFactory2 func(PluginFactoryArgs) (algorithm.PriorityMapFunction, algorithm.PriorityReduceFunction)

// A PriorityConfigFactory produces a PriorityConfig from the given function and weight
type PriorityConfigFactory struct {
	Function          PriorityFunctionFactory
	MapReduceFunction PriorityFunctionFactory2
	Weight            int
}

var (
	schedulerFactoryMutex sync.Mutex

	// maps that hold registered algorithm types
	fitPredicateMap        = make(map[string]FitPredicateFactory)
	mandatoryFitPredicates = make(map[string]bool)
	priorityFunctionMap    = make(map[string]PriorityConfigFactory)
	algorithmProviderMap   = make(map[string]AlgorithmProviderConfig)

	// Registered metadata producers
	priorityMetadataProducer  MetadataProducerFactory
	predicateMetadataProducer MetadataProducerFactory

	// get equivalence pod function
	getEquivalencePodFunc algorithm.GetEquivalencePodFunc
)

const (
	DefaultProvider = "DefaultProvider"
)

type AlgorithmProviderConfig struct {
	FitPredicateKeys     sets.String
	PriorityFunctionKeys sets.String
}

// RegisterFitPredicate registers a fit predicate with the algorithm
// registry. Returns the name with which the predicate was registered.
func RegisterFitPredicate(name string, predicate algorithm.FitPredicate) string {
	return RegisterFitPredicateFactory(name, func(PluginFactoryArgs) algorithm.FitPredicate { return predicate })
}

// RegisterMandatoryFitPredicate registers a fit predicate with the algorithm registry, the predicate is used by
// kubelet, DaemonSet; it is always included in configuration. Returns the name with which the predicate was
// registered.
func RegisterMandatoryFitPredicate(name string, predicate algorithm.FitPredicate) string {
	schedulerFactoryMutex.Lock()
	defer schedulerFactoryMutex.Unlock()
	validateAlgorithmNameOrDie(name)
	fitPredicateMap[name] = func(PluginFactoryArgs) algorithm.FitPredicate { return predicate }
	mandatoryFitPredicates[name] = true
	return name
}

// RegisterFitPredicateFactory registers a fit predicate factory with the
// algorithm registry. Returns the name with which the predicate was registered.
func RegisterFitPredicateFactory(name string, predicateFactory FitPredicateFactory) string {
	schedulerFactoryMutex.Lock()
	defer schedulerFactoryMutex.Unlock()
	validateAlgorithmNameOrDie(name)
	fitPredicateMap[name] = predicateFactory
	return name
}

// RegisterCustomFitPredicate registers a custom fit predicate with the algorithm registry.
// Returns the name, with which the predicate was registered.
func RegisterCustomFitPredicate(policy schedulerapi.PredicatePolicy) string {
	var predicateFactory FitPredicateFactory
	var ok bool

	validatePredicateOrDie(policy)

	// generate the predicate function, if a custom type is requested
	if policy.Argument != nil {
		if policy.Argument.ServiceAffinity != nil {
			predicateFactory = func(args PluginFactoryArgs) algorithm.FitPredicate {
				predicate, precomputationFunction := predicates.NewServiceAffinityPredicate(
					args.PodLister,
					args.ServiceLister,
					args.NodeInfo,
					policy.Argument.ServiceAffinity.Labels,
				)

				// Once we generate the predicate we should also Register the Precomputation
				predicates.RegisterPredicatePrecomputation(policy.Name, precomputationFunction)
				return predicate
			}
		} else if policy.Argument.LabelsPresence != nil {
			predicateFactory = func(args PluginFactoryArgs) algorithm.FitPredicate {
				return predicates.NewNodeLabelPredicate(
					policy.Argument.LabelsPresence.Labels,
					policy.Argument.LabelsPresence.Presence,
				)
			}
		}
	} else if predicateFactory, ok = fitPredicateMap[policy.Name]; ok {
		// checking to see if a pre-defined predicate is requested
		glog.V(2).Infof("Predicate type %s already registered, reusing.", policy.Name)
		return policy.Name
	}

	if predicateFactory == nil {
		glog.Fatalf("Invalid configuration: Predicate type not found for %s", policy.Name)
	}

	return RegisterFitPredicateFactory(policy.Name, predicateFactory)
}

// IsFitPredicateRegistered is useful for testing providers.
func IsFitPredicateRegistered(name string) bool {
	schedulerFactoryMutex.Lock()
	defer schedulerFactoryMutex.Unlock()
	_, ok := fitPredicateMap[name]
	return ok
}

func RegisterPriorityMetadataProducerFactory(factory MetadataProducerFactory) {
	schedulerFactoryMutex.Lock()
	defer schedulerFactoryMutex.Unlock()
	priorityMetadataProducer = factory
}

func RegisterPredicateMetadataProducerFactory(factory MetadataProducerFactory) {
	schedulerFactoryMutex.Lock()
	defer schedulerFactoryMutex.Unlock()
	predicateMetadataProducer = factory
}

// DEPRECATED
// Use Map-Reduce pattern for priority functions.
// Registers a priority function with the algorithm registry. Returns the name,
// with which the function was registered.
func RegisterPriorityFunction(name string, function algorithm.PriorityFunction, weight int) string {
	return RegisterPriorityConfigFactory(name, PriorityConfigFactory{
		Function: func(PluginFactoryArgs) algorithm.PriorityFunction {
			return function
		},
		Weight: weight,
	})
}

// RegisterPriorityFunction2 registers a priority function with the algorithm registry. Returns the name,
// with which the function was registered.
// FIXME: Rename to PriorityFunctionFactory.
func RegisterPriorityFunction2(
	name string,
	mapFunction algorithm.PriorityMapFunction,
	reduceFunction algorithm.PriorityReduceFunction,
	weight int) string {
	return RegisterPriorityConfigFactory(name, PriorityConfigFactory{
		MapReduceFunction: func(PluginFactoryArgs) (algorithm.PriorityMapFunction, algorithm.PriorityReduceFunction) {
			return mapFunction, reduceFunction
		},
		Weight: weight,
	})
}

func RegisterPriorityConfigFactory(name string, pcf PriorityConfigFactory) string {
	schedulerFactoryMutex.Lock()
	defer schedulerFactoryMutex.Unlock()
	validateAlgorithmNameOrDie(name)
	priorityFunctionMap[name] = pcf
	return name
}

// RegisterCustomPriorityFunction registers a custom priority function with the algorithm registry.
// Returns the name, with which the priority function was registered.
func RegisterCustomPriorityFunction(policy schedulerapi.PriorityPolicy) string {
	var pcf *PriorityConfigFactory

	validatePriorityOrDie(policy)

	// generate the priority function, if a custom priority is requested
	if policy.Argument != nil {
		if policy.Argument.ServiceAntiAffinity != nil {
			pcf = &PriorityConfigFactory{
				Function: func(args PluginFactoryArgs) algorithm.PriorityFunction {
					return priorities.NewServiceAntiAffinityPriority(
						args.PodLister,
						args.ServiceLister,
						policy.Argument.ServiceAntiAffinity.Label,
					)
				},
				Weight: policy.Weight,
			}
		} else if policy.Argument.LabelPreference != nil {
			pcf = &PriorityConfigFactory{
				MapReduceFunction: func(args PluginFactoryArgs) (algorithm.PriorityMapFunction, algorithm.PriorityReduceFunction) {
					return priorities.NewNodeLabelPriority(
						policy.Argument.LabelPreference.Label,
						policy.Argument.LabelPreference.Presence,
					)
				},
				Weight: policy.Weight,
			}
		}
	} else if existingPcf, ok := priorityFunctionMap[policy.Name]; ok {
		glog.V(2).Infof("Priority type %s already registered, reusing.", policy.Name)
		// set/update the weight based on the policy
		pcf = &PriorityConfigFactory{
			Function:          existingPcf.Function,
			MapReduceFunction: existingPcf.MapReduceFunction,
			Weight:            policy.Weight,
		}
	}

	if pcf == nil {
		glog.Fatalf("Invalid configuration: Priority type not found for %s", policy.Name)
	}

	return RegisterPriorityConfigFactory(policy.Name, *pcf)
}

func RegisterGetEquivalencePodFunction(equivalenceFunc algorithm.GetEquivalencePodFunc) {
	getEquivalencePodFunc = equivalenceFunc
}

// IsPriorityFunctionRegistered is useful for testing providers.
func IsPriorityFunctionRegistered(name string) bool {
	schedulerFactoryMutex.Lock()
	defer schedulerFactoryMutex.Unlock()
	_, ok := priorityFunctionMap[name]
	return ok
}

// RegisterAlgorithmProvider registers a new algorithm provider with the algorithm registry. This should
// be called from the init function in a provider plugin.
func RegisterAlgorithmProvider(name string, predicateKeys, priorityKeys sets.String) string {
	schedulerFactoryMutex.Lock()
	defer schedulerFactoryMutex.Unlock()
	validateAlgorithmNameOrDie(name)
	algorithmProviderMap[name] = AlgorithmProviderConfig{
		FitPredicateKeys:     predicateKeys,
		PriorityFunctionKeys: priorityKeys,
	}
	return name
}

// GetAlgorithmProvider should not be used to modify providers. It is publicly visible for testing.
func GetAlgorithmProvider(name string) (*AlgorithmProviderConfig, error) {
	schedulerFactoryMutex.Lock()
	defer schedulerFactoryMutex.Unlock()

	provider, ok := algorithmProviderMap[name]
	if !ok {
		return nil, fmt.Errorf("plugin %q has not been registered", name)
	}

	return &provider, nil
}

func getFitPredicateFunctions(names sets.String, args PluginFactoryArgs) (map[string]algorithm.FitPredicate, error) {
	schedulerFactoryMutex.Lock()
	defer schedulerFactoryMutex.Unlock()

	predicates := map[string]algorithm.FitPredicate{}
	for _, name := range names.List() {
		factory, ok := fitPredicateMap[name]
		if !ok {
			return nil, fmt.Errorf("Invalid predicate name %q specified - no corresponding function found", name)
		}
		predicates[name] = factory(args)
	}

	// Always include mandatory fit predicates.
	for name, mandatory := range mandatoryFitPredicates {
		if mandatory {
			if factory, found := fitPredicateMap[name]; found {
				predicates[name] = factory(args)
			}
		}
	}

	return predicates, nil
}

func getPriorityMetadataProducer(args PluginFactoryArgs) (algorithm.MetadataProducer, error) {
	schedulerFactoryMutex.Lock()
	defer schedulerFactoryMutex.Unlock()

	if priorityMetadataProducer == nil {
		return algorithm.EmptyMetadataProducer, nil
	}
	return priorityMetadataProducer(args), nil
}

func getPredicateMetadataProducer(args PluginFactoryArgs) (algorithm.MetadataProducer, error) {
	schedulerFactoryMutex.Lock()
	defer schedulerFactoryMutex.Unlock()

	if predicateMetadataProducer == nil {
		return algorithm.EmptyMetadataProducer, nil
	}
	return predicateMetadataProducer(args), nil
}

func getPriorityFunctionConfigs(names sets.String, args PluginFactoryArgs) ([]algorithm.PriorityConfig, error) {
	schedulerFactoryMutex.Lock()
	defer schedulerFactoryMutex.Unlock()

	configs := []algorithm.PriorityConfig{}
	for _, name := range names.List() {
		factory, ok := priorityFunctionMap[name]
		if !ok {
			return nil, fmt.Errorf("Invalid priority name %s specified - no corresponding function found", name)
		}
		if factory.Function != nil {
			configs = append(configs, algorithm.PriorityConfig{
				Function: factory.Function(args),
				Weight:   factory.Weight,
			})
		} else {
			mapFunction, reduceFunction := factory.MapReduceFunction(args)
			configs = append(configs, algorithm.PriorityConfig{
				Map:    mapFunction,
				Reduce: reduceFunction,
				Weight: factory.Weight,
			})
		}
	}
	if err := validateSelectedConfigs(configs); err != nil {
		return nil, err
	}
	return configs, nil
}

// validateSelectedConfigs validates the config weights to avoid the overflow.
func validateSelectedConfigs(configs []algorithm.PriorityConfig) error {
	var totalPriority int
	for _, config := range configs {
		// Checks totalPriority against MaxTotalPriority to avoid overflow
		if config.Weight*schedulerapi.MaxPriority > schedulerapi.MaxTotalPriority-totalPriority {
			return fmt.Errorf("Total priority of priority functions has overflown")
		}
		totalPriority += config.Weight * schedulerapi.MaxPriority
	}
	return nil
}

var validName = regexp.MustCompile("^[a-zA-Z0-9]([-a-zA-Z0-9]*[a-zA-Z0-9])$")

func validateAlgorithmNameOrDie(name string) {
	if !validName.MatchString(name) {
		glog.Fatalf("Algorithm name %v does not match the name validation regexp \"%v\".", name, validName)
	}
}

func validatePredicateOrDie(predicate schedulerapi.PredicatePolicy) {
	if predicate.Argument != nil {
		numArgs := 0
		if predicate.Argument.ServiceAffinity != nil {
			numArgs++
		}
		if predicate.Argument.LabelsPresence != nil {
			numArgs++
		}
		if numArgs != 1 {
			glog.Fatalf("Exactly 1 predicate argument is required, numArgs: %v, Predicate: %s", numArgs, predicate.Name)
		}
	}
}

func validatePriorityOrDie(priority schedulerapi.PriorityPolicy) {
	if priority.Argument != nil {
		numArgs := 0
		if priority.Argument.ServiceAntiAffinity != nil {
			numArgs++
		}
		if priority.Argument.LabelPreference != nil {
			numArgs++
		}
		if numArgs != 1 {
			glog.Fatalf("Exactly 1 priority argument is required, numArgs: %v, Priority: %s", numArgs, priority.Name)
		}
	}
}

func ListRegisteredFitPredicates() []string {
	schedulerFactoryMutex.Lock()
	defer schedulerFactoryMutex.Unlock()

	names := []string{}
	for name := range fitPredicateMap {
		names = append(names, name)
	}
	return names
}

func ListRegisteredPriorityFunctions() []string {
	schedulerFactoryMutex.Lock()
	defer schedulerFactoryMutex.Unlock()

	names := []string{}
	for name := range priorityFunctionMap {
		names = append(names, name)
	}
	return names
}

// ListAlgorithmProviders is called when listing all available algorithm providers in `kube-scheduler --help`
func ListAlgorithmProviders() string {
	var availableAlgorithmProviders []string
	for name := range algorithmProviderMap {
		availableAlgorithmProviders = append(availableAlgorithmProviders, name)
	}
	sort.Strings(availableAlgorithmProviders)
	return strings.Join(availableAlgorithmProviders, " | ")
}
<commit_msg>Replaced bool map to string set.<commit_after>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package factory

import (
	"fmt"
	"regexp"
	"sort"
	"strings"
	"sync"

	"github.com/golang/glog"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/kubernetes/plugin/pkg/scheduler/algorithm"
	"k8s.io/kubernetes/plugin/pkg/scheduler/algorithm/predicates"
	"k8s.io/kubernetes/plugin/pkg/scheduler/algorithm/priorities"
	schedulerapi "k8s.io/kubernetes/plugin/pkg/scheduler/api"
)

// PluginFactoryArgs are passed to all plugin factory functions.
type PluginFactoryArgs struct {
	PodLister                      algorithm.PodLister
	ServiceLister                  algorithm.ServiceLister
	ControllerLister               algorithm.ControllerLister
	ReplicaSetLister               algorithm.ReplicaSetLister
	StatefulSetLister              algorithm.StatefulSetLister
	NodeLister                     algorithm.NodeLister
	NodeInfo                       predicates.NodeInfo
	PVInfo                         predicates.PersistentVolumeInfo
	PVCInfo                        predicates.PersistentVolumeClaimInfo
	HardPodAffinitySymmetricWeight int
}

// MetadataProducerFactory produces MetadataProducer from the given args.
type MetadataProducerFactory func(PluginFactoryArgs) algorithm.MetadataProducer

// A FitPredicateFactory produces a FitPredicate from the given args.
type FitPredicateFactory func(PluginFactoryArgs) algorithm.FitPredicate

// DEPRECATED
// Use Map-Reduce pattern for priority functions.
// A PriorityFunctionFactory produces a PriorityConfig from the given args.
type PriorityFunctionFactory func(PluginFactoryArgs) algorithm.PriorityFunction

// A PriorityFunctionFactory produces map & reduce priority functions
// from a given args.
// FIXME: Rename to PriorityFunctionFactory.
type PriorityFunctionFactory2 func(PluginFactoryArgs) (algorithm.PriorityMapFunction, algorithm.PriorityReduceFunction)

// A PriorityConfigFactory produces a PriorityConfig from the given function and weight
type PriorityConfigFactory struct {
	Function          PriorityFunctionFactory
	MapReduceFunction PriorityFunctionFactory2
	Weight            int
}

var (
	schedulerFactoryMutex sync.Mutex

	// maps that hold registered algorithm types
	fitPredicateMap        = make(map[string]FitPredicateFactory)
	mandatoryFitPredicates = sets.NewString()
	priorityFunctionMap    = make(map[string]PriorityConfigFactory)
	algorithmProviderMap   = make(map[string]AlgorithmProviderConfig)

	// Registered metadata producers
	priorityMetadataProducer  MetadataProducerFactory
	predicateMetadataProducer MetadataProducerFactory

	// get equivalence pod function
	getEquivalencePodFunc algorithm.GetEquivalencePodFunc
)

const (
	DefaultProvider = "DefaultProvider"
)

type AlgorithmProviderConfig struct {
	FitPredicateKeys     sets.String
	PriorityFunctionKeys sets.String
}

// RegisterFitPredicate registers a fit predicate with the algorithm
// registry. Returns the name with which the predicate was registered.
func RegisterFitPredicate(name string, predicate algorithm.FitPredicate) string {
	return RegisterFitPredicateFactory(name, func(PluginFactoryArgs) algorithm.FitPredicate { return predicate })
}

// RegisterMandatoryFitPredicate registers a fit predicate with the algorithm registry, the predicate is used by
// kubelet, DaemonSet; it is always included in configuration. Returns the name with which the predicate was
// registered.
func RegisterMandatoryFitPredicate(name string, predicate algorithm.FitPredicate) string {
	schedulerFactoryMutex.Lock()
	defer schedulerFactoryMutex.Unlock()
	validateAlgorithmNameOrDie(name)
	fitPredicateMap[name] = func(PluginFactoryArgs) algorithm.FitPredicate { return predicate }
	mandatoryFitPredicates.Insert(name)
	return name
}

// RegisterFitPredicateFactory registers a fit predicate factory with the
// algorithm registry. Returns the name with which the predicate was registered.
func RegisterFitPredicateFactory(name string, predicateFactory FitPredicateFactory) string {
	schedulerFactoryMutex.Lock()
	defer schedulerFactoryMutex.Unlock()
	validateAlgorithmNameOrDie(name)
	fitPredicateMap[name] = predicateFactory
	return name
}

// RegisterCustomFitPredicate registers a custom fit predicate with the algorithm registry.
// Returns the name, with which the predicate was registered.
func RegisterCustomFitPredicate(policy schedulerapi.PredicatePolicy) string {
	var predicateFactory FitPredicateFactory
	var ok bool

	validatePredicateOrDie(policy)

	// generate the predicate function, if a custom type is requested
	if policy.Argument != nil {
		if policy.Argument.ServiceAffinity != nil {
			predicateFactory = func(args PluginFactoryArgs) algorithm.FitPredicate {
				predicate, precomputationFunction := predicates.NewServiceAffinityPredicate(
					args.PodLister,
					args.ServiceLister,
					args.NodeInfo,
					policy.Argument.ServiceAffinity.Labels,
				)

				// Once we generate the predicate we should also Register the Precomputation
				predicates.RegisterPredicatePrecomputation(policy.Name, precomputationFunction)
				return predicate
			}
		} else if policy.Argument.LabelsPresence != nil {
			predicateFactory = func(args PluginFactoryArgs) algorithm.FitPredicate {
				return predicates.NewNodeLabelPredicate(
					policy.Argument.LabelsPresence.Labels,
					policy.Argument.LabelsPresence.Presence,
				)
			}
		}
	} else if predicateFactory, ok = fitPredicateMap[policy.Name]; ok {
		// checking to see if a pre-defined predicate is requested
		glog.V(2).Infof("Predicate type %s already registered, reusing.", policy.Name)
		return policy.Name
	}

	if predicateFactory == nil {
		glog.Fatalf("Invalid configuration: Predicate type not found for %s", policy.Name)
	}

	return RegisterFitPredicateFactory(policy.Name, predicateFactory)
}

// IsFitPredicateRegistered is useful for testing providers.
func IsFitPredicateRegistered(name string) bool {
	schedulerFactoryMutex.Lock()
	defer schedulerFactoryMutex.Unlock()
	_, ok := fitPredicateMap[name]
	return ok
}

func RegisterPriorityMetadataProducerFactory(factory MetadataProducerFactory) {
	schedulerFactoryMutex.Lock()
	defer schedulerFactoryMutex.Unlock()
	priorityMetadataProducer = factory
}

func RegisterPredicateMetadataProducerFactory(factory MetadataProducerFactory) {
	schedulerFactoryMutex.Lock()
	defer schedulerFactoryMutex.Unlock()
	predicateMetadataProducer = factory
}

// DEPRECATED
// Use Map-Reduce pattern for priority functions.
// Registers a priority function with the algorithm registry. Returns the name,
// with which the function was registered.
func RegisterPriorityFunction(name string, function algorithm.PriorityFunction, weight int) string {
	return RegisterPriorityConfigFactory(name, PriorityConfigFactory{
		Function: func(PluginFactoryArgs) algorithm.PriorityFunction {
			return function
		},
		Weight: weight,
	})
}

// RegisterPriorityFunction2 registers a priority function with the algorithm registry. Returns the name,
// with which the function was registered.
// FIXME: Rename to PriorityFunctionFactory.
func RegisterPriorityFunction2(
	name string,
	mapFunction algorithm.PriorityMapFunction,
	reduceFunction algorithm.PriorityReduceFunction,
	weight int) string {
	return RegisterPriorityConfigFactory(name, PriorityConfigFactory{
		MapReduceFunction: func(PluginFactoryArgs) (algorithm.PriorityMapFunction, algorithm.PriorityReduceFunction) {
			return mapFunction, reduceFunction
		},
		Weight: weight,
	})
}

func RegisterPriorityConfigFactory(name string, pcf PriorityConfigFactory) string {
	schedulerFactoryMutex.Lock()
	defer schedulerFactoryMutex.Unlock()
	validateAlgorithmNameOrDie(name)
	priorityFunctionMap[name] = pcf
	return name
}

// RegisterCustomPriorityFunction registers a custom priority function with the algorithm registry.
// Returns the name, with which the priority function was registered.
func RegisterCustomPriorityFunction(policy schedulerapi.PriorityPolicy) string {
	var pcf *PriorityConfigFactory

	validatePriorityOrDie(policy)

	// generate the priority function, if a custom priority is requested
	if policy.Argument != nil {
		if policy.Argument.ServiceAntiAffinity != nil {
			pcf = &PriorityConfigFactory{
				Function: func(args PluginFactoryArgs) algorithm.PriorityFunction {
					return priorities.NewServiceAntiAffinityPriority(
						args.PodLister,
						args.ServiceLister,
						policy.Argument.ServiceAntiAffinity.Label,
					)
				},
				Weight: policy.Weight,
			}
		} else if policy.Argument.LabelPreference != nil {
			pcf = &PriorityConfigFactory{
				MapReduceFunction: func(args PluginFactoryArgs) (algorithm.PriorityMapFunction, algorithm.PriorityReduceFunction) {
					return priorities.NewNodeLabelPriority(
						policy.Argument.LabelPreference.Label,
						policy.Argument.LabelPreference.Presence,
					)
				},
				Weight: policy.Weight,
			}
		}
	} else if existingPcf, ok := priorityFunctionMap[policy.Name]; ok {
		glog.V(2).Infof("Priority type %s already registered, reusing.", policy.Name)
		// set/update the weight based on the policy
		pcf = &PriorityConfigFactory{
			Function:          existingPcf.Function,
			MapReduceFunction: existingPcf.MapReduceFunction,
			Weight:            policy.Weight,
		}
	}

	if pcf == nil {
		glog.Fatalf("Invalid configuration: Priority type not found for %s", policy.Name)
	}

	return RegisterPriorityConfigFactory(policy.Name, *pcf)
}

func RegisterGetEquivalencePodFunction(equivalenceFunc algorithm.GetEquivalencePodFunc) {
	getEquivalencePodFunc = equivalenceFunc
}

// IsPriorityFunctionRegistered is useful for testing providers.
func IsPriorityFunctionRegistered(name string) bool {
	schedulerFactoryMutex.Lock()
	defer schedulerFactoryMutex.Unlock()
	_, ok := priorityFunctionMap[name]
	return ok
}

// RegisterAlgorithmProvider registers a new algorithm provider with the algorithm registry. This should
// be called from the init function in a provider plugin.
func RegisterAlgorithmProvider(name string, predicateKeys, priorityKeys sets.String) string {
	schedulerFactoryMutex.Lock()
	defer schedulerFactoryMutex.Unlock()
	validateAlgorithmNameOrDie(name)
	algorithmProviderMap[name] = AlgorithmProviderConfig{
		FitPredicateKeys:     predicateKeys,
		PriorityFunctionKeys: priorityKeys,
	}
	return name
}

// GetAlgorithmProvider should not be used to modify providers. It is publicly visible for testing.
func GetAlgorithmProvider(name string) (*AlgorithmProviderConfig, error) {
	schedulerFactoryMutex.Lock()
	defer schedulerFactoryMutex.Unlock()

	provider, ok := algorithmProviderMap[name]
	if !ok {
		return nil, fmt.Errorf("plugin %q has not been registered", name)
	}

	return &provider, nil
}

func getFitPredicateFunctions(names sets.String, args PluginFactoryArgs) (map[string]algorithm.FitPredicate, error) {
	schedulerFactoryMutex.Lock()
	defer schedulerFactoryMutex.Unlock()

	predicates := map[string]algorithm.FitPredicate{}
	for _, name := range names.List() {
		factory, ok := fitPredicateMap[name]
		if !ok {
			return nil, fmt.Errorf("Invalid predicate name %q specified - no corresponding function found", name)
		}
		predicates[name] = factory(args)
	}

	// Always include mandatory fit predicates.
	for _, name := range mandatoryFitPredicates.List() {
		if factory, found := fitPredicateMap[name]; found {
			predicates[name] = factory(args)
		}
	}

	return predicates, nil
}

func getPriorityMetadataProducer(args PluginFactoryArgs) (algorithm.MetadataProducer, error) {
	schedulerFactoryMutex.Lock()
	defer schedulerFactoryMutex.Unlock()

	if priorityMetadataProducer == nil {
		return algorithm.EmptyMetadataProducer, nil
	}
	return priorityMetadataProducer(args), nil
}

func getPredicateMetadataProducer(args PluginFactoryArgs) (algorithm.MetadataProducer, error) {
	schedulerFactoryMutex.Lock()
	defer schedulerFactoryMutex.Unlock()

	if predicateMetadataProducer == nil {
		return algorithm.EmptyMetadataProducer, nil
	}
	return predicateMetadataProducer(args), nil
}

func getPriorityFunctionConfigs(names sets.String, args PluginFactoryArgs) ([]algorithm.PriorityConfig, error) {
	schedulerFactoryMutex.Lock()
	defer schedulerFactoryMutex.Unlock()

	configs := []algorithm.PriorityConfig{}
	for _, name := range names.List() {
		factory, ok := priorityFunctionMap[name]
		if !ok {
			return nil, fmt.Errorf("Invalid priority name %s specified - no corresponding function found", name)
		}
		if factory.Function != nil {
			configs = append(configs, algorithm.PriorityConfig{
				Function: factory.Function(args),
				Weight:   factory.Weight,
			})
		} else {
			mapFunction, reduceFunction := factory.MapReduceFunction(args)
			configs = append(configs, algorithm.PriorityConfig{
				Map:    mapFunction,
				Reduce: reduceFunction,
				Weight: factory.Weight,
			})
		}
	}
	if err := validateSelectedConfigs(configs); err != nil {
		return nil, err
	}
	return configs, nil
}

// validateSelectedConfigs validates the config weights to avoid the overflow.
func validateSelectedConfigs(configs []algorithm.PriorityConfig) error {
	var totalPriority int
	for _, config := range configs {
		// Checks totalPriority against MaxTotalPriority to avoid overflow
		if config.Weight*schedulerapi.MaxPriority > schedulerapi.MaxTotalPriority-totalPriority {
			return fmt.Errorf("Total priority of priority functions has overflown")
		}
		totalPriority += config.Weight * schedulerapi.MaxPriority
	}
	return nil
}

var validName = regexp.MustCompile("^[a-zA-Z0-9]([-a-zA-Z0-9]*[a-zA-Z0-9])$")

func validateAlgorithmNameOrDie(name string) {
	if !validName.MatchString(name) {
		glog.Fatalf("Algorithm name %v does not match the name validation regexp \"%v\".", name, validName)
	}
}

func validatePredicateOrDie(predicate schedulerapi.PredicatePolicy) {
	if predicate.Argument != nil {
		numArgs := 0
		if predicate.Argument.ServiceAffinity != nil {
			numArgs++
		}
		if predicate.Argument.LabelsPresence != nil {
			numArgs++
		}
		if numArgs != 1 {
			glog.Fatalf("Exactly 1 predicate argument is required, numArgs: %v, Predicate: %s", numArgs, predicate.Name)
		}
	}
}

func validatePriorityOrDie(priority schedulerapi.PriorityPolicy) {
	if priority.Argument != nil {
		numArgs := 0
		if priority.Argument.ServiceAntiAffinity != nil {
			numArgs++
		}
		if priority.Argument.LabelPreference != nil {
			numArgs++
		}
		if numArgs != 1 {
			glog.Fatalf("Exactly 1 priority argument is required, numArgs: %v, Priority: %s", numArgs, priority.Name)
		}
	}
}

func ListRegisteredFitPredicates() []string {
	schedulerFactoryMutex.Lock()
	defer schedulerFactoryMutex.Unlock()

	names := []string{}
	for name := range fitPredicateMap {
		names = append(names, name)
	}
	return names
}

func ListRegisteredPriorityFunctions() []string {
	schedulerFactoryMutex.Lock()
	defer schedulerFactoryMutex.Unlock()

	names := []string{}
	for name := range priorityFunctionMap {
		names = append(names, name)
	}
	return names
}

// ListAlgorithmProviders is called when listing all available algorithm providers in `kube-scheduler --help`
func ListAlgorithmProviders() string {
	var availableAlgorithmProviders []string
	for name := range algorithmProviderMap {
		availableAlgorithmProviders = append(availableAlgorithmProviders, name)
	}
	sort.Strings(availableAlgorithmProviders)
	return strings.Join(availableAlgorithmProviders, " | ")
}
<|endoftext|>


<commit_before><commit_msg>remove duplicated update of MinikubeISO<commit_after><|endoftext|>


<commit_before>// Copyright © 2016 Abcum Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package orbit

import (
	"io/ioutil"
	"path/filepath"
	"strings"
)

func addModule(name string, item module) {
	modules[name] = item
}

func addSource(name string, file string) {

	if strings.Contains(file, "*") {

		files, _ := filepath.Glob(file)

		for i, file := range files {

			vers := filepath.Base(file)

			data, err := ioutil.ReadFile(file)
			if err != nil {
				modules[name+"@"+vers] = null()
				return
			}
			modules[name+"@"+vers] = exec(data, file)

			if i == len(files)-1 {
				modules[name] = modules[name+"@"+vers]
				modules[name+"@latest"] = modules[name+"@"+vers]
			}

		}

	} else {

		data, err := ioutil.ReadFile(file)
		if err != nil {
			modules[name] = null()
			return
		}
		modules[name] = exec(data, file)

	}

}
<commit_msg>Don't include file extensions in module names<commit_after>// Copyright © 2016 Abcum Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package orbit

import (
	"io/ioutil"
	"path/filepath"
	"strings"
)

func addModule(name string, item module) {
	modules[name] = item
}

func addSource(name string, file string) {

	if strings.Contains(file, "*") {

		files, _ := filepath.Glob(file)

		for i, file := range files {

			extn := filepath.Ext(file)
			full := filepath.Base(file)
			vers := full[0 : len(full)-len(extn)]

			data, err := ioutil.ReadFile(file)
			if err != nil {
				modules[name+"@"+vers] = null()
				return
			}
			modules[name+"@"+vers] = exec(data, file)

			if i == len(files)-1 {
				modules[name] = modules[name+"@"+vers]
				modules[name+"@latest"] = modules[name+"@"+vers]
			}

		}

	} else {

		data, err := ioutil.ReadFile(file)
		if err != nil {
			modules[name] = null()
			return
		}
		modules[name] = exec(data, file)

	}

}
<|endoftext|>


<commit_before>/* Package go140 implements interaction with the Twitter API */
package go140

import (
	"github.com/alloy-d/goauth"
	"fmt"
	"io/ioutil"
	"json"
	"os"
)

type API struct {
	Root string
	oauth.OAuth
}

type Status struct {
	Date string "created_at"
	Text string
	Location string "place"
}

type User struct {
	ScreenName string "screen_name"
	Name string
	Location string
	Description string
	Status *Status
}

func (api *API) UserByID(id uint) (*User, os.Error) {
	url := api.Root + "/1/users/show.json"
	params := map[string]string{
		"id": fmt.Sprintf("%d", id),
	}

	resp, err := api.Get(url, params)
	if err != nil {
		return nil, err
	}

	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	var user User
	err = json.Unmarshal(data, &user)
	if err != nil {
		return nil, err
	}
	return &user, nil
}

func (api *API) Status() (*Status, os.Error) {
	user, err := api.UserByID(api.UserID())
	if err != nil {
		return nil, err
	}

	return user.Status, nil
}

func (api *API) Update(s string) (string, os.Error) {
	if len(s) > 140 {
		return "", tweetError{"Tweet too long!"}
	}

	url := api.Root + "/1/statuses/update.json"
	params := map[string]string{"status": s}

	_, err := api.Post(url, params)
	if err != nil {
		return "", err
	}
	return "<not yet implemented>", nil
}
<commit_msg>Allow unauthenticated requests; allow getting users by screen name.<commit_after>/* Package go140 implements interaction with the Twitter API */
package go140

import (
	"github.com/alloy-d/goauth"
	"fmt"
	"http"
	"io/ioutil"
	"json"
	"os"
	"url"
)

type API struct {
	Root string
	oauth.OAuth
}

// TODO: this is SILLY.
func addQueryParams(resource string, params map[string]string) string {
	str := resource

	first := true
	for k, v := range params {
		if first {
			str += "?"
			first = false
		} else {
			str += "&"
		}

		rawv, err := url.QueryUnescape(v)
		if err == nil {
			v = rawv
		}
		str += k + "=" + url.QueryEscape(v)
	}
	return str
}

func (api *API) Get(resource string, params map[string]string) (*http.Response, os.Error) {
	if api.Authorized() {
		return api.OAuth.Get(resource, params)
	}

	fullURL := addQueryParams(resource, params)
	return http.Get(fullURL)
}

type Status struct {
	Date string "created_at"
	Text string
	Location string "place"
}

type User struct {
	ScreenName string "screen_name"
	Name string
	Location string
	Description string
	Status *Status
}

func (api *API) UserByID(id uint) (*User, os.Error) {
	return api.user(map[string]string{
		"id": fmt.Sprintf("%d", id),
	})
}

func (api *API) User(screen_name string) (*User, os.Error) {
	return api.user(map[string]string{
		"screen_name": screen_name,
	})
}

func (api *API) user(params map[string]string) (*User, os.Error) {
	url := api.Root + "/1/users/show.json"

	resp, err := api.Get(url, params)
	if err != nil {
		return nil, err
	}

	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	var user User
	err = json.Unmarshal(data, &user)
	if err != nil {
		return nil, err
	}
	return &user, nil
}

func (api *API) Status() (*Status, os.Error) {
	user, err := api.UserByID(api.UserID())
	if err != nil {
		return nil, err
	}

	return user.Status, nil
}

func (api *API) Update(s string) (string, os.Error) {
	if len(s) > 140 {
		return "", tweetError{"Tweet too long!"}
	}

	url := api.Root + "/1/statuses/update.json"
	params := map[string]string{"status": s}

	_, err := api.Post(url, params)
	if err != nil {
		return "", err
	}
	return "<not yet implemented>", nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"net/http"
	"strconv"

	"strings"

	log "github.com/Sirupsen/logrus"
	"github.com/blevesearch/bleve"
	"github.com/spf13/viper"
)

// DefaultResponse Error response structure
type DefaultResponse struct {
	Message string `json:"message"`
}

// SeachResultItem is a response structure for search result item
type SeachResultItem struct {
	ID     string      `json:"id"`
	Score  float64     `json:"score"`
	Fields interface{} `json:"fields"`
}

// SearchResultsResponse is a response structure for final search results
type SearchResultsResponse struct {
	TotalResultsPages uint64            `json:"total_results_pages"`
	MoreResults       bool              `json:"more_results"`
	Page              int               `json:"page"`
	Time              string            `json:"took"`
	Results           []SeachResultItem `json:"results"`
}

// Adapter type
type Adapter func(http.Handler) http.Handler

// Adapt wraps http handlers with middlewares
func Adapt(h http.Handler, adapters ...Adapter) http.Handler {
	for _, adapter := range adapters {
		h = adapter(h)
	}

	return h
}

// Log all requests
func HttpLogger() Adapter {
	return func(h http.Handler) http.Handler {
		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			defer log.Infof("%s %s", r.Method, r.RequestURI)
			h.ServeHTTP(w, r)
		})
	}
}

// Write response as a JSON formt
func writeJSONResponse(w http.ResponseWriter, i interface{}, status int) {
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(status)

	if err := json.NewEncoder(w).Encode(i); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

// Validate search query
func sanatizeSearchQuery(query string) (string, error) {
	if len(query) < 3 {
		return query, errors.New("Search query should be of minimum 3 characters")
	}

	return strings.ToLower(query), nil
}

// Index page handler
func indexHandler(w http.ResponseWriter, r *http.Request) {
	writeJSONResponse(w, DefaultResponse{"Bankr API v3"}, http.StatusOK)
}

func getGeocodeAddressHandler(w http.ResponseWriter, r *http.Request) {
	latitude := r.URL.Query().Get("latitude")
	longitude := r.URL.Query().Get("longitude")
	geocodeApiKey := viper.GetString("geocode_api_key")
	geocodeAPIURI := viper.GetString("geocode_api_uri")

	client := &http.Client{}
	request, err := http.NewRequest("GET", geocodeAPIURI, nil)

	if err != nil {
		log.Errorf("Error while getting location: %v", err)
		writeJSONResponse(w, DefaultResponse{"Error while getting location"}, http.StatusBadGateway)
	}

	// Add query params to the request
	q := request.URL.Query()
	q.Add("latlng", latitude+","+longitude)
	q.Add("key", geocodeApiKey)
	request.URL.RawQuery = q.Encode()

	resp, err := client.Do(request)
	responseData, err := ioutil.ReadAll(resp.Body)

	if err != nil {
		log.Errorf("Error while parsing location response: %v", err)
		writeJSONResponse(w, DefaultResponse{"Error while getting location"}, http.StatusBadGateway)
	}

	var response map[string]interface{}
	err = json.Unmarshal(responseData, &response)

	if err != nil {
		log.Errorf("Error while parsing unmarshaling response: %v", err)
		writeJSONResponse(w, DefaultResponse{"Error while getting location"}, http.StatusBadGateway)
	}

	writeJSONResponse(w, response, http.StatusOK)
}

// Query search handler
func searchHandler(w http.ResponseWriter, r *http.Request) {
	query := r.URL.Query().Get("q")
	page := r.URL.Query().Get("p")

	var (
		errorRespose         DefaultResponse
		searchResults        *bleve.SearchResult
		searchResultItems    []SeachResultItem
		resultsSize          = 10
		pageNumber           = 1
		moreResultsAvailable = false
	)

	// Validate search query
	query, err := sanatizeSearchQuery(query)
	if err != nil {
		errorRespose.Message = err.Error()
		writeJSONResponse(w, errorRespose, http.StatusBadRequest)
		return
	}

	// Validate page number
	if page == "" {
		pageNumber = 1
	} else {
		pageNumber, err = strconv.Atoi(page)
		if err != nil {
			errorRespose.Message = "Invalid page number."
			writeJSONResponse(w, errorRespose, http.StatusBadRequest)
			return
		}
	}

	// Search for give query and result size (startIndex + size). Start index is (pageNum - 1)
	searchResults, err = querySearch(query, resultsSize, pageNumber-1)
	if err != nil {
		log.Errorf("Error while searching query: %v", err)
		errorRespose.Message = "Something went wrong. Please report to admin."
		writeJSONResponse(w, errorRespose, http.StatusInternalServerError)
		return
	}

	// Create list for search items response
	for _, result := range searchResults.Hits {
		searchResultItems = append(searchResultItems, SeachResultItem{
			ID:     result.ID,
			Score:  result.Score,
			Fields: result.Fields,
		})
	}

	// Check if more available
	if searchResults.Total > uint64(pageNumber+resultsSize) {
		moreResultsAvailable = true
	}

	// Final search response
	searchResultsResponse := SearchResultsResponse{
		TotalResultsPages: searchResults.Total - 1,
		MoreResults:       moreResultsAvailable,
		Page:              pageNumber,
		Time:              searchResults.Took.String(),
		Results:           searchResultItems,
	}

	// Write the output
	writeJSONResponse(w, searchResultsResponse, http.StatusOK)
}

func initServer(address string) {
	// Server static files
	http.Handle("/", http.FileServer(http.Dir("./frontend/dist/")))

	// API handlers
	http.Handle("/api", Adapt(http.HandlerFunc(indexHandler)))
	http.Handle("/api/search", Adapt(http.HandlerFunc(searchHandler), HttpLogger()))
	http.Handle("/api/location", Adapt(http.HandlerFunc(getGeocodeAddressHandler), HttpLogger()))

	// Start the server
	log.Infof("Starting server: http://%s", address)
	if err := http.ListenAndServe(address, nil); err != nil {
		log.Error("Error starting the server: ", err)
	}
}
<commit_msg>Fixed typo in a struct<commit_after>package main

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"net/http"
	"strconv"

	"strings"

	log "github.com/Sirupsen/logrus"
	"github.com/blevesearch/bleve"
	"github.com/spf13/viper"
)

// DefaultResponse Error response structure
type DefaultResponse struct {
	Message string `json:"message"`
}

// SeachResultItem is a response structure for search result item
type SeachResultItem struct {
	ID     string      `json:"id"`
	Score  float64     `json:"score"`
	Fields interface{} `json:"fields"`
}

// SearchResultsResponse is a response structure for final search results
type SearchResultsResponse struct {
	TotalResultsPages uint64            `json:"total_results_pages"`
	MoreResults       bool              `json:"more_results"`
	Page              int               `json:"page"`
	Time              string            `json:"took"`
	Results           []SeachResultItem `json:"results"`
}

// Adapter type
type Adapter func(http.Handler) http.Handler

// Adapt wraps http handlers with middlewares
func Adapt(h http.Handler, adapters ...Adapter) http.Handler {
	for _, adapter := range adapters {
		h = adapter(h)
	}

	return h
}

// Log all requests
func HttpLogger() Adapter {
	return func(h http.Handler) http.Handler {
		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			defer log.Infof("%s %s", r.Method, r.RequestURI)
			h.ServeHTTP(w, r)
		})
	}
}

// Write response as a JSON formt
func writeJSONResponse(w http.ResponseWriter, i interface{}, status int) {
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(status)

	if err := json.NewEncoder(w).Encode(i); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

// Validate search query
func sanatizeSearchQuery(query string) (string, error) {
	if len(query) < 3 {
		return query, errors.New("Search query should be of minimum 3 characters")
	}

	return strings.ToLower(query), nil
}

// Index page handler
func indexHandler(w http.ResponseWriter, r *http.Request) {
	writeJSONResponse(w, DefaultResponse{"Bankr API v3"}, http.StatusOK)
}

func getGeocodeAddressHandler(w http.ResponseWriter, r *http.Request) {
	latitude := r.URL.Query().Get("latitude")
	longitude := r.URL.Query().Get("longitude")
	geocodeApiKey := viper.GetString("geocode_api_key")
	geocodeAPIURI := viper.GetString("geocode_api_uri")

	client := &http.Client{}
	request, err := http.NewRequest("GET", geocodeAPIURI, nil)

	if err != nil {
		log.Errorf("Error while getting location: %v", err)
		writeJSONResponse(w, DefaultResponse{"Error while getting location"}, http.StatusBadGateway)
	}

	// Add query params to the request
	q := request.URL.Query()
	q.Add("latlng", latitude+","+longitude)
	q.Add("key", geocodeApiKey)
	request.URL.RawQuery = q.Encode()

	resp, err := client.Do(request)
	responseData, err := ioutil.ReadAll(resp.Body)

	if err != nil {
		log.Errorf("Error while parsing location response: %v", err)
		writeJSONResponse(w, DefaultResponse{"Error while getting location"}, http.StatusBadGateway)
	}

	var response map[string]interface{}
	err = json.Unmarshal(responseData, &response)

	if err != nil {
		log.Errorf("Error while parsing unmarshaling response: %v", err)
		writeJSONResponse(w, DefaultResponse{"Error while getting location"}, http.StatusBadGateway)
	}

	writeJSONResponse(w, response, http.StatusOK)
}

// Query search handler
func searchHandler(w http.ResponseWriter, r *http.Request) {
	query := r.URL.Query().Get("q")
	page := r.URL.Query().Get("p")

	var (
		errorResponse        DefaultResponse
		searchResults        *bleve.SearchResult
		searchResultItems    []SeachResultItem
		resultsSize          = 10
		pageNumber           = 1
		moreResultsAvailable = false
	)

	// Validate search query
	query, err := sanatizeSearchQuery(query)
	if err != nil {
		errorResponse.Message = err.Error()
		writeJSONResponse(w, errorResponse, http.StatusBadRequest)
		return
	}

	// Validate page number
	if page == "" {
		pageNumber = 1
	} else {
		pageNumber, err = strconv.Atoi(page)
		if err != nil {
			errorResponse.Message = "Invalid page number."
			writeJSONResponse(w, errorResponse, http.StatusBadRequest)
			return
		}
	}

	// Search for give query and result size (startIndex + size). Start index is (pageNum - 1)
	searchResults, err = querySearch(query, resultsSize, pageNumber-1)
	if err != nil {
		log.Errorf("Error while searching query: %v", err)
		errorResponse.Message = "Something went wrong. Please report to admin."
		writeJSONResponse(w, errorResponse, http.StatusInternalServerError)
		return
	}

	// Create list for search items response
	for _, result := range searchResults.Hits {
		searchResultItems = append(searchResultItems, SeachResultItem{
			ID:     result.ID,
			Score:  result.Score,
			Fields: result.Fields,
		})
	}

	// Check if more available
	if searchResults.Total > uint64(pageNumber+resultsSize) {
		moreResultsAvailable = true
	}

	// Final search response
	searchResultsResponse := SearchResultsResponse{
		TotalResultsPages: searchResults.Total - 1,
		MoreResults:       moreResultsAvailable,
		Page:              pageNumber,
		Time:              searchResults.Took.String(),
		Results:           searchResultItems,
	}

	log.Infof("Searched for term q=%v - %v results generated in %v nanoseconds", query, searchResults.Total, searchResults.Took.Nanoseconds())

	// Write the output
	writeJSONResponse(w, searchResultsResponse, http.StatusOK)
}

func initServer(address string) {
	// Server static files
	http.Handle("/", http.FileServer(http.Dir("./frontend/dist/")))

	// API handlers
	http.Handle("/api", Adapt(http.HandlerFunc(indexHandler)))
	http.Handle("/api/search", Adapt(http.HandlerFunc(searchHandler), HttpLogger()))
	http.Handle("/api/location", Adapt(http.HandlerFunc(getGeocodeAddressHandler), HttpLogger()))

	// Start the server
	log.Infof("Starting server: http://%s", address)
	if err := http.ListenAndServe(address, nil); err != nil {
		log.Error("Error starting the server: ", err)
	}
}
<|endoftext|>


<commit_before>// Package stockfighter provides a simple wrapper for the Stockfighter API:
//
// https://www.stockfighter.io/
//
// https://starfighter.readme.io/
package stockfighter

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/http/httputil"

	"github.com/gorilla/websocket"
)

func apiUrl(path string, args ...interface{}) string {
	return fmt.Sprintf("https://api.stockfighter.io/ob/api/"+path, args...)
}

func gmUrl(path string, args ...interface{}) string {
	return fmt.Sprintf("https://api.stockfighter.io/gm/"+path, args...)
}

func wsUrl(path string, args ...interface{}) string {
	return fmt.Sprintf("wss://api.stockfighter.io/ob/api/ws/"+path, args...)
}

type apiCall interface {
	Err() error
}

type response struct {
	Ok    bool
	Error string
}

func (r response) Err() error {
	if len(r.Error) > 0 {
		return fmt.Errorf(r.Error)
	}
	return nil
}

type venueResponse struct {
	response
	Venue string
}

type stocksResponse struct {
	response
	Symbols []Symbol
}

type orderBookResponse struct {
	response
	OrderBook
}

type quoteResponse struct {
	response
	Quote
}

type orderResponse struct {
	response
	OrderState
}

type bulkOrderResponse struct {
	response
	Venue  string
	Orders []OrderState
}

type quoteMessage struct {
	Ok    bool
	Quote Quote
}

type executionMessage struct {
	Ok bool
	Execution
}

type gameResponse struct {
	response
	Game
}

type gameStateResponse struct {
	response
	GameState
}

type Stockfighter struct {
	apiKey string
	debug  bool
}

// Create new Stockfighter API instance.
// If debug is true, log all HTTP requests and responses.
func NewStockfighter(apiKey string, debug bool) *Stockfighter {
	return &Stockfighter{
		apiKey: apiKey,
		debug:  debug,
	}
}

// Check the API Is Up. If venue is a non-empty string, then check that venue.
// Returns nil if ok, otherwise the error indicates the problem.
func (sf *Stockfighter) Heartbeat(venue string) error {
	var resp response
	url := apiUrl("heartbeat")
	if len(venue) > 0 {
		url = apiUrl("venues/%s/heartbeat", venue)
	}
	return sf.do("GET", url, nil, &resp)
}

// Get the stocks available for trading on a venue.
func (sf *Stockfighter) Stocks(venue string) ([]Symbol, error) {
	var resp stocksResponse
	url := apiUrl("venues/%s/stocks", venue)
	if err := sf.do("GET", url, nil, &resp); err != nil {
		return nil, err
	}
	return resp.Symbols, nil
}

// Get the orderbook for a particular stock.
func (sf *Stockfighter) OrderBook(venue, stock string) (*OrderBook, error) {
	var resp orderBookResponse
	url := apiUrl("venues/%s/stocks/%s", venue, stock)
	if err := sf.do("GET", url, nil, &resp); err != nil {
		return nil, err
	}
	return &resp.OrderBook, nil
}

// Get a quick look at the most recent trade information for a stock.
func (sf *Stockfighter) Quote(venue, stock string) (*Quote, error) {
	var resp quoteResponse
	url := apiUrl("venues/%s/stocks/%s/quote", venue, stock)
	if err := sf.do("GET", url, nil, &resp); err != nil {
		return nil, err
	}
	return &resp.Quote, nil
}

// Place an order
func (sf *Stockfighter) Place(order *Order) (*OrderState, error) {
	body, err := encodeJson(order)
	if err != nil {
		return nil, err
	}
	var resp orderResponse
	url := apiUrl("venues/%s/stocks/%s/orders", order.Venue, order.Stock)
	if err := sf.do("POST", url, body, &resp); err != nil {
		return nil, err
	}
	return &resp.OrderState, nil
}

// Get the status for an existing order.
func (sf *Stockfighter) Status(venue, stock string, id uint64) (*OrderState, error) {
	var resp orderResponse
	url := apiUrl("venues/%s/stocks/%s/orders/%d", venue, stock, id)
	if err := sf.do("GET", url, nil, &resp); err != nil {
		return nil, err
	}
	return &resp.OrderState, nil
}

// Get the statuses for all an account's orders of a stock on a venue.
// If stock is a non-empty string, only statuses for that stock are returned
func (sf *Stockfighter) StockStatus(account, venue, stock string) ([]OrderState, error) {
	url := apiUrl("venues/%s/accounts/%s/orders", venue, account)
	if len(stock) > 0 {
		url = apiUrl("venues/%s/accounts/%s/stocks/%s/orders", venue, account, stock)
	}
	var resp bulkOrderResponse
	if err := sf.do("GET", url, nil, &resp); err != nil {
		return nil, err
	}
	return resp.Orders, nil
}

// Cancel an existing order
func (sf *Stockfighter) Cancel(venue, stock string, id uint64) error {
	var resp response
	url := apiUrl("venues/%s/stocks/%s/orders/%d", venue, stock, id)
	return sf.do("DELETE", url, nil, &resp)
}

// Subscribe to a stream of quotes for a venue.
// If stock is a non-empy string, only quotes for that stock are returned.
func (sf *Stockfighter) Quotes(account, venue, stock string) (chan *Quote, error) {
	url := wsUrl("%s/venues/%s/tickertape", account, venue)
	if len(stock) > 0 {
		url = wsUrl("%s/venues/%s/tickertape/stocks/%s", account, venue, stock)
	}
	c := make(chan *Quote)
	return c, sf.pump(url, func(conn *websocket.Conn) error {
		var quote quoteMessage
		if err := conn.ReadJSON(&quote); err != nil {
			close(c)
			return err
		}
		if quote.Ok {
			c <- &quote.Quote
		}
		return nil
	})
}

// Subscribe to a stream of executions for a venue.
// If stock is a non-empy string, only executions for that stock are returned.
func (sf *Stockfighter) Executions(account, venue, stock string) (chan *Execution, error) {
	url := wsUrl("%s/venues/%s/executions", account, venue)
	if len(stock) > 0 {
		url = wsUrl("%s/venues/%s/executions/stocks/%s", account, venue, stock)
	}
	c := make(chan *Execution)
	return c, sf.pump(url, func(conn *websocket.Conn) error {
		var execution executionMessage
		if err := conn.ReadJSON(&execution); err != nil {
			close(c)
			return err
		}
		if execution.Ok {
			c <- &execution.Execution
		}
		return nil
	})
}

// Start a new level.
func (sf *Stockfighter) Start(level string) (*Game, error) {
	var resp gameResponse
	url := gmUrl("levels/%s", level)
	if err := sf.do("POST", url, nil, &resp); err != nil {
		return nil, err
	}
	return &resp.Game, nil
}

// Restart a level using the instance id from a previously started Game.
func (sf *Stockfighter) Restart(id uint64) error {
	var resp response
	url := gmUrl("instances/%d/restart", id)
	return sf.do("POST", url, nil, &resp)
}

// Resume a level using the instance id from a previously started Game.
func (sf *Stockfighter) Resume(id uint64) error {
	var resp response
	url := gmUrl("instances/%d/resume", id)
	return sf.do("POST", url, nil, &resp)
}

// Stop a level using the instance id from a previously started Game.
func (sf *Stockfighter) Stop(id uint64) error {
	var resp response
	url := gmUrl("instances/%d/stop", id)
	return sf.do("POST", url, nil, &resp)
}

// Get the GameState using the instance id from a previously started Game.
func (sf *Stockfighter) GameStatus(id uint64) (*GameState, error) {
	var resp gameStateResponse
	url := gmUrl("instances/%d", id)
	if err := sf.do("GET", url, nil, &resp); err != nil {
		return nil, err
	}
	return &resp.GameState, nil
}

func (sf *Stockfighter) Judge(id uint64) error {
	var resp response
	url := gmUrl("instances/%d/judge", id)
	test := map[string]interface{}{"test": "test"}
	body, err := encodeJson(test)
	if err != nil {
		return err
	}
	return sf.do("POST", url, body, &resp)
}

func encodeJson(v interface{}) (io.Reader, error) {
	var buf bytes.Buffer
	if err := json.NewEncoder(&buf).Encode(v); err != nil {
		return nil, err
	}
	return &buf, nil
}

func (sf *Stockfighter) do(method, url string, body io.Reader, value apiCall) error {
	req, err := http.NewRequest(method, url, body)
	if err != nil {
		return err
	}
	req.Header.Add("X-Starfighter-Authorization", sf.apiKey)
	if sf.debug {
		out, _ := httputil.DumpRequest(req, true)
		log.Println(string(out))
	}
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if sf.debug {
		out, _ := httputil.DumpResponse(resp, true)
		log.Println(string(out))
	}
	if err := json.NewDecoder(resp.Body).Decode(value); err != nil {
		if resp.StatusCode >= 500 {
			return fmt.Errorf(resp.Status)
		}
		return err
	}
	return value.Err()
}

func (sf *Stockfighter) pump(url string, f func(*websocket.Conn) error) error {
	conn, _, err := websocket.DefaultDialer.Dial(url, nil)
	if err != nil {
		return err
	}
	go func() {
		defer conn.Close()
		for err := f(conn); err == nil; err = f(conn) {
		}
	}()
	return nil
}
<commit_msg>add debug to web sockets<commit_after>// Package stockfighter provides a simple wrapper for the Stockfighter API:
//
// https://www.stockfighter.io/
//
// https://starfighter.readme.io/
package stockfighter

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/http/httputil"

	"github.com/gorilla/websocket"
)

func apiUrl(path string, args ...interface{}) string {
	return fmt.Sprintf("https://api.stockfighter.io/ob/api/"+path, args...)
}

func gmUrl(path string, args ...interface{}) string {
	return fmt.Sprintf("https://api.stockfighter.io/gm/"+path, args...)
}

func wsUrl(path string, args ...interface{}) string {
	return fmt.Sprintf("wss://api.stockfighter.io/ob/api/ws/"+path, args...)
}

type apiCall interface {
	Err() error
}

type response struct {
	Ok    bool
	Error string
}

func (r response) Err() error {
	if len(r.Error) > 0 {
		return fmt.Errorf(r.Error)
	}
	return nil
}

type venueResponse struct {
	response
	Venue string
}

type stocksResponse struct {
	response
	Symbols []Symbol
}

type orderBookResponse struct {
	response
	OrderBook
}

type quoteResponse struct {
	response
	Quote
}

type orderResponse struct {
	response
	OrderState
}

type bulkOrderResponse struct {
	response
	Venue  string
	Orders []OrderState
}

type quoteMessage struct {
	Ok    bool
	Quote Quote
}

type executionMessage struct {
	Ok bool
	Execution
}

type gameResponse struct {
	response
	Game
}

type gameStateResponse struct {
	response
	GameState
}

type Stockfighter struct {
	apiKey string
	debug  bool
}

// Create new Stockfighter API instance.
// If debug is true, log all HTTP requests and responses.
func NewStockfighter(apiKey string, debug bool) *Stockfighter {
	return &Stockfighter{
		apiKey: apiKey,
		debug:  debug,
	}
}

// Check the API Is Up. If venue is a non-empty string, then check that venue.
// Returns nil if ok, otherwise the error indicates the problem.
func (sf *Stockfighter) Heartbeat(venue string) error {
	var resp response
	url := apiUrl("heartbeat")
	if len(venue) > 0 {
		url = apiUrl("venues/%s/heartbeat", venue)
	}
	return sf.do("GET", url, nil, &resp)
}

// Get the stocks available for trading on a venue.
func (sf *Stockfighter) Stocks(venue string) ([]Symbol, error) {
	var resp stocksResponse
	url := apiUrl("venues/%s/stocks", venue)
	if err := sf.do("GET", url, nil, &resp); err != nil {
		return nil, err
	}
	return resp.Symbols, nil
}

// Get the orderbook for a particular stock.
func (sf *Stockfighter) OrderBook(venue, stock string) (*OrderBook, error) {
	var resp orderBookResponse
	url := apiUrl("venues/%s/stocks/%s", venue, stock)
	if err := sf.do("GET", url, nil, &resp); err != nil {
		return nil, err
	}
	return &resp.OrderBook, nil
}

// Get a quick look at the most recent trade information for a stock.
func (sf *Stockfighter) Quote(venue, stock string) (*Quote, error) {
	var resp quoteResponse
	url := apiUrl("venues/%s/stocks/%s/quote", venue, stock)
	if err := sf.do("GET", url, nil, &resp); err != nil {
		return nil, err
	}
	return &resp.Quote, nil
}

// Place an order
func (sf *Stockfighter) Place(order *Order) (*OrderState, error) {
	body, err := encodeJson(order)
	if err != nil {
		return nil, err
	}
	var resp orderResponse
	url := apiUrl("venues/%s/stocks/%s/orders", order.Venue, order.Stock)
	if err := sf.do("POST", url, body, &resp); err != nil {
		return nil, err
	}
	return &resp.OrderState, nil
}

// Get the status for an existing order.
func (sf *Stockfighter) Status(venue, stock string, id uint64) (*OrderState, error) {
	var resp orderResponse
	url := apiUrl("venues/%s/stocks/%s/orders/%d", venue, stock, id)
	if err := sf.do("GET", url, nil, &resp); err != nil {
		return nil, err
	}
	return &resp.OrderState, nil
}

// Get the statuses for all an account's orders of a stock on a venue.
// If stock is a non-empty string, only statuses for that stock are returned
func (sf *Stockfighter) StockStatus(account, venue, stock string) ([]OrderState, error) {
	url := apiUrl("venues/%s/accounts/%s/orders", venue, account)
	if len(stock) > 0 {
		url = apiUrl("venues/%s/accounts/%s/stocks/%s/orders", venue, account, stock)
	}
	var resp bulkOrderResponse
	if err := sf.do("GET", url, nil, &resp); err != nil {
		return nil, err
	}
	return resp.Orders, nil
}

// Cancel an existing order
func (sf *Stockfighter) Cancel(venue, stock string, id uint64) error {
	var resp response
	url := apiUrl("venues/%s/stocks/%s/orders/%d", venue, stock, id)
	return sf.do("DELETE", url, nil, &resp)
}

// Subscribe to a stream of quotes for a venue.
// If stock is a non-empy string, only quotes for that stock are returned.
func (sf *Stockfighter) Quotes(account, venue, stock string) (chan *Quote, error) {
	url := wsUrl("%s/venues/%s/tickertape", account, venue)
	if len(stock) > 0 {
		url = wsUrl("%s/venues/%s/tickertape/stocks/%s", account, venue, stock)
	}
	c := make(chan *Quote)
	return c, sf.pump(url, func(conn *websocket.Conn) error {
		var quote quoteMessage
		if err := sf.decodeMessage(conn, &quote); err != nil {
			close(c)
			return err
		}
		if quote.Ok {
			c <- &quote.Quote
		}
		return nil
	})
}

// Subscribe to a stream of executions for a venue.
// If stock is a non-empy string, only executions for that stock are returned.
func (sf *Stockfighter) Executions(account, venue, stock string) (chan *Execution, error) {
	url := wsUrl("%s/venues/%s/executions", account, venue)
	if len(stock) > 0 {
		url = wsUrl("%s/venues/%s/executions/stocks/%s", account, venue, stock)
	}
	c := make(chan *Execution)
	return c, sf.pump(url, func(conn *websocket.Conn) error {
		var execution executionMessage
		if err := sf.decodeMessage(conn, &execution); err != nil {
			close(c)
			return err
		}
		if execution.Ok {
			c <- &execution.Execution
		}
		return nil
	})
}

func (sf *Stockfighter) decodeMessage(conn *websocket.Conn, v interface{}) error {
	_, msg, err := conn.ReadMessage()
	if err != nil {
		return err
	}
	if sf.debug {
		log.Println(string(msg))
	}
	return json.Unmarshal(msg, v)
}

// Start a new level.
func (sf *Stockfighter) Start(level string) (*Game, error) {
	var resp gameResponse
	url := gmUrl("levels/%s", level)
	if err := sf.do("POST", url, nil, &resp); err != nil {
		return nil, err
	}
	return &resp.Game, nil
}

// Restart a level using the instance id from a previously started Game.
func (sf *Stockfighter) Restart(id uint64) error {
	var resp response
	url := gmUrl("instances/%d/restart", id)
	return sf.do("POST", url, nil, &resp)
}

// Resume a level using the instance id from a previously started Game.
func (sf *Stockfighter) Resume(id uint64) error {
	var resp response
	url := gmUrl("instances/%d/resume", id)
	return sf.do("POST", url, nil, &resp)
}

// Stop a level using the instance id from a previously started Game.
func (sf *Stockfighter) Stop(id uint64) error {
	var resp response
	url := gmUrl("instances/%d/stop", id)
	return sf.do("POST", url, nil, &resp)
}

// Get the GameState using the instance id from a previously started Game.
func (sf *Stockfighter) GameStatus(id uint64) (*GameState, error) {
	var resp gameStateResponse
	url := gmUrl("instances/%d", id)
	if err := sf.do("GET", url, nil, &resp); err != nil {
		return nil, err
	}
	return &resp.GameState, nil
}

func (sf *Stockfighter) Judge(id uint64) error {
	var resp response
	url := gmUrl("instances/%d/judge", id)
	test := map[string]interface{}{"test": "test"}
	body, err := encodeJson(test)
	if err != nil {
		return err
	}
	return sf.do("POST", url, body, &resp)
}

func encodeJson(v interface{}) (io.Reader, error) {
	var buf bytes.Buffer
	if err := json.NewEncoder(&buf).Encode(v); err != nil {
		return nil, err
	}
	return &buf, nil
}

func (sf *Stockfighter) do(method, url string, body io.Reader, value apiCall) error {
	req, err := http.NewRequest(method, url, body)
	if err != nil {
		return err
	}
	req.Header.Add("X-Starfighter-Authorization", sf.apiKey)
	if sf.debug {
		out, _ := httputil.DumpRequest(req, true)
		log.Println(string(out))
	}
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if sf.debug {
		out, _ := httputil.DumpResponse(resp, true)
		log.Println(string(out))
	}
	if err := json.NewDecoder(resp.Body).Decode(value); err != nil {
		if resp.StatusCode >= 500 {
			return fmt.Errorf(resp.Status)
		}
		return err
	}
	return value.Err()
}

func (sf *Stockfighter) pump(url string, f func(*websocket.Conn) error) error {
	conn, _, err := websocket.DefaultDialer.Dial(url, nil)
	if err != nil {
		return err
	}
	go func() {
		defer conn.Close()
		for err := f(conn); err == nil; err = f(conn) {
		}
	}()
	return nil
}
<|endoftext|>


<commit_before>/*
 * Minimal object storage library (C) 2015 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package objectstorage

import (
	"errors"
	"fmt"
	"io"
	"runtime"
	"sort"
	"time"
)

// API - object storage API interface
type API interface {
	// Bucket Read/Write/Stat operations
	BucketAPI

	// Object Read/Write/Stat operations
	ObjectAPI
}

// BucketAPI - bucket specific Read/Write/Stat interface
type BucketAPI interface {
	CreateBucket(bucket, acl, location string) error
	SetBucketACL(bucket, acl string) error
	StatBucket(bucket string) error
	DeleteBucket(bucket string) error

	ListObjects(bucket, prefix string, recursive bool) <-chan ObjectOnChannel
	ListBuckets() <-chan BucketOnChannel
}

// ObjectAPI - object specific Read/Write/Stat interface
type ObjectAPI interface {
	GetObject(bucket, object string, offset, length uint64) (io.ReadCloser, *ObjectMetadata, error)
	CreateObject(bucket, object string, size uint64, data io.Reader) (string, error)
	StatObject(bucket, object string) (*ObjectMetadata, error)
	DeleteObject(bucket, object string) error
}

// BucketOnChannel - bucket metadata over read channel
type BucketOnChannel struct {
	Data *BucketMetadata
	Err  error
}

// ObjectOnChannel - object metadata over read channel
type ObjectOnChannel struct {
	Data *ObjectMetadata
	Err  error
}

// BucketMetadata container for bucket metadata
type BucketMetadata struct {
	// The name of the bucket.
	Name string
	// Date the bucket was created.
	CreationDate time.Time
}

// ObjectMetadata container for object metadata
type ObjectMetadata struct {
	ETag         string
	Key          string
	LastModified time.Time
	Size         int64

	Owner struct {
		DisplayName string
		ID          string
	}

	// The class of storage used to store the object.
	StorageClass string
}

// Regions s3 region map used by bucket location constraint
var Regions = map[string]string{
	"us-gov-west-1":  "https://s3-fips-us-gov-west-1.amazonaws.com",
	"us-east-1":      "https://s3.amazonaws.com",
	"us-west-1":      "https://s3-us-west-1.amazonaws.com",
	"us-west-2":      "https://s3-us-west-2.amazonaws.com",
	"eu-west-1":      "https://s3-eu-west-1.amazonaws.com",
	"eu-central-1":   "https://s3-eu-central-1.amazonaws.com",
	"ap-southeast-1": "https://s3-ap-southeast-1.amazonaws.com",
	"ap-southeast-2": "https://s3-ap-southeast-2.amazonaws.com",
	"ap-northeast-1": "https://s3-ap-northeast-1.amazonaws.com",
	"sa-east-1":      "https://s3-sa-east-1.amazonaws.com",
	"cn-north-1":     "https://s3.cn-north-1.amazonaws.com.cn",
}

type api struct {
	*lowLevelAPI
}

// Config - main configuration struct used by all to set endpoint, credentials, and other options for requests.
type Config struct {
	AccessKeyID     string
	SecretAccessKey string
	Endpoint        string
	ContentType     string
	// not exported internal usage only
	userAgent string
}

// Global constants
const (
	LibraryName    = "objectstorage-go/"
	LibraryVersion = "0.1"
)

// New - instantiate a new minio api client
func New(config *Config) API {
	// Not configurable at the moment, but we will relook on this in future
	config.userAgent = LibraryName + " (" + LibraryVersion + "; " + runtime.GOOS + "; " + runtime.GOARCH + ")"
	return &api{&lowLevelAPI{config}}
}

/// Object operations

// GetObject retrieve object
//
// Additionally it also takes range arguments to download the specified range bytes of an object.
// For more information about the HTTP Range header, go to http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35.
func (a *api) GetObject(bucket, object string, offset, length uint64) (io.ReadCloser, *ObjectMetadata, error) {
	// get the the object
	// NOTE : returned md5sum could be the md5sum of the partial object itself
	// not the whole object depending on if offset range was requested or not
	body, objectMetadata, err := a.getObject(bucket, object, offset, length)
	if err != nil {
		return nil, nil, err
	}
	return body, objectMetadata, nil
}

// completedParts is a wrapper to make parts sortable by their part number
// multi part completion requires list of multi parts to be sorted
type completedParts []*completePart

func (a completedParts) Len() int           { return len(a) }
func (a completedParts) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a completedParts) Less(i, j int) bool { return a[i].PartNumber < a[j].PartNumber }

// DefaultPartSize - default size per object after which PutObject becomes multipart
var DefaultPartSize uint64 = 1024 * 1024 * 5

// CreateObject create an object in a bucket
//
// You must have WRITE permissions on a bucket to create an object
//
// This version of CreateObject automatically does multipart for more than 5MB worth of data
// This default part size is not configurable currently but can be configurable in future
func (a *api) CreateObject(bucket, object string, size uint64, data io.Reader) (string, error) {
	switch {
	case size < DefaultPartSize:
		// Single Part use case, use PutObject directly
		for part := range MultiPart(data, DefaultPartSize) {
			if part.Err != nil {
				return "", part.Err
			}
			return "", a.putObject(bucket, object, part.Len, part.Data)
		}
	default:
		initiateMultipartUploadResult, err := a.initiateMultipartUpload(bucket, object)
		if err != nil {
			return "", err
		}
		uploadID := initiateMultipartUploadResult.UploadID
		completeMultipartUpload := new(completeMultipartUpload)
		for part := range MultiPart(data, DefaultPartSize) {
			if part.Err != nil {
				return "", part.Err
			}
			completePart, err := a.uploadPart(bucket, object, uploadID, part.Num, part.Len, part.Data)
			fmt.Println(err)
			if err != nil {
				return "", a.abortMultipartUpload(bucket, object, uploadID)
			}
			completeMultipartUpload.Part = append(completeMultipartUpload.Part, completePart)
		}
		sort.Sort(completedParts(completeMultipartUpload.Part))
		completeMultipartUploadResult, err := a.completeMultipartUpload(bucket, object, uploadID, completeMultipartUpload)
		if err != nil {
			return "", a.abortMultipartUpload(bucket, object, uploadID)
		}
		return completeMultipartUploadResult.ETag, nil
	}
	return "", errors.New("Unexpected control flow")
}

// StatObject verify if object exists and you have permission to access it
func (a *api) StatObject(bucket, object string) (*ObjectMetadata, error) {
	return a.headObject(bucket, object)
}

// DeleteObject remove the object from a bucket
func (a *api) DeleteObject(bucket, object string) error {
	return a.deleteObject(bucket, object)
}

/// Bucket operations

// CreateBucket create a new bucket
//
// optional arguments are acl and location - by default all buckets are created
// with ``private`` acl and location set to US Standard if one wishes to set
// different ACLs and Location one can set them properly.
//
// ACL valid values
// ------------------
// private - owner gets full access [DEFAULT]
// public-read - owner gets full access, others get read access
// public-read-write - owner gets full access, others get full access too
// ------------------
//
// Location valid values
// ------------------
// [ us-west-1 | us-west-2 | eu-west-1 | eu-central-1 | ap-southeast-1 | ap-northeast-1 | ap-southeast-2 | sa-east-1 ]
// Default - US standard
func (a *api) CreateBucket(bucket, acl, location string) error {
	return a.putBucket(bucket, acl, location)
}

// SetBucketACL set the permissions on an existing bucket using access control lists (ACL)
//
// Currently supported are:
// ------------------
// private - owner gets full access
// public-read - owner gets full access, others get read access
// public-read-write - owner gets full access, others get full access too
// ------------------
func (a *api) SetBucketACL(bucket, acl string) error {
	return a.putBucketACL(bucket, acl)
}

// StatBucket verify if bucket exists and you have permission to access it
func (a *api) StatBucket(bucket string) error {
	return a.headBucket(bucket)
}

// DeleteBucket deletes the bucket named in the URI
// NOTE: -
//  All objects (including all object versions and delete markers)
//  in the bucket must be deleted before successfully attempting this request
func (a *api) DeleteBucket(bucket string) error {
	return a.deleteBucket(bucket)
}

// listObjectsInRoutine is an internal goroutine function called for listing objects
// This function feeds data into channel
func (a *api) listObjectsInRoutine(bucket, prefix string, recursive bool, ch chan ObjectOnChannel) {
	defer close(ch)
	switch {
	case recursive == true:
		listBucketResult, err := a.listObjects(bucket, 1000, "", prefix, "")
		if err != nil {
			ch <- ObjectOnChannel{
				Data: nil,
				Err:  err,
			}
			return
		}
		for _, object := range listBucketResult.Contents {
			ch <- ObjectOnChannel{
				Data: object,
				Err:  nil,
			}
		}
		for {
			if !listBucketResult.IsTruncated {
				break
			}
			listBucketResult, err = a.listObjects(bucket, 1000, listBucketResult.Marker, prefix, "")
			if err != nil {
				ch <- ObjectOnChannel{
					Data: nil,
					Err:  err,
				}
				return
			}
			for _, object := range listBucketResult.Contents {
				ch <- ObjectOnChannel{
					Data: object,
					Err:  nil,
				}
				listBucketResult.Marker = object.Key
			}
		}
	default:
		listBucketResult, err := a.listObjects(bucket, 1000, "", prefix, "/")
		if err != nil {
			ch <- ObjectOnChannel{
				Data: nil,
				Err:  err,
			}
			return
		}
		for _, object := range listBucketResult.Contents {
			ch <- ObjectOnChannel{
				Data: object,
				Err:  nil,
			}
		}
	}
}

// ListObjects - (List Objects) - List some objects or all recursively
//
// ListObjects is a channel based API implemented to facilitate ease of usage of S3 API ListObjects()
// by automatically recursively traversing all objects on a given bucket if specified.
//
// Your input paramters are just bucket, prefix and recursive
//
// If you enable recursive as 'true' this function will return back all the objects in a given bucket
//
//  eg:-
//         api := objectstorage.New(....)
//         for message := range api.ListObjects("mytestbucket", "starthere", true) {
//                 fmt.Println(message.Data)
//         }
//
func (a *api) ListObjects(bucket string, prefix string, recursive bool) <-chan ObjectOnChannel {
	ch := make(chan ObjectOnChannel)
	go a.listObjectsInRoutine(bucket, prefix, recursive, ch)
	return ch
}

// listBucketsInRoutine is an internal go routine function called for listing buckets
// This function feeds data into channel
func (a *api) listBucketsInRoutine(ch chan BucketOnChannel) {
	defer close(ch)
	listAllMyBucketListResults, err := a.listBuckets()
	if err != nil {
		ch <- BucketOnChannel{
			Data: nil,
			Err:  err,
		}
		return
	}
	for _, bucket := range listAllMyBucketListResults.Buckets.Bucket {
		ch <- BucketOnChannel{
			Data: bucket,
			Err:  nil,
		}
	}

}

// ListBuckets list of all buckets owned by the authenticated sender of the request
//
// NOTE:
//     This call requires explicit authentication, no anonymous
//     requests are allowed for listing buckets
//
//  eg:-
//         api := objectstorage.New(....)
//         for message := range api.ListBuckets() {
//                 fmt.Println(message.Data)
//         }
//
func (a *api) ListBuckets() <-chan BucketOnChannel {
	ch := make(chan BucketOnChannel)
	go a.listBucketsInRoutine(ch)
	return ch
}
<commit_msg>Fix a typo, push it in<commit_after>/*
 * Minimal object storage library (C) 2015 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package objectstorage

import (
	"errors"
	"io"
	"runtime"
	"sort"
	"time"
)

// API - object storage API interface
type API interface {
	// Bucket Read/Write/Stat operations
	BucketAPI

	// Object Read/Write/Stat operations
	ObjectAPI
}

// BucketAPI - bucket specific Read/Write/Stat interface
type BucketAPI interface {
	CreateBucket(bucket, acl, location string) error
	SetBucketACL(bucket, acl string) error
	StatBucket(bucket string) error
	DeleteBucket(bucket string) error

	ListObjects(bucket, prefix string, recursive bool) <-chan ObjectOnChannel
	ListBuckets() <-chan BucketOnChannel
}

// ObjectAPI - object specific Read/Write/Stat interface
type ObjectAPI interface {
	GetObject(bucket, object string, offset, length uint64) (io.ReadCloser, *ObjectMetadata, error)
	CreateObject(bucket, object string, size uint64, data io.Reader) (string, error)
	StatObject(bucket, object string) (*ObjectMetadata, error)
	DeleteObject(bucket, object string) error
}

// BucketOnChannel - bucket metadata over read channel
type BucketOnChannel struct {
	Data *BucketMetadata
	Err  error
}

// ObjectOnChannel - object metadata over read channel
type ObjectOnChannel struct {
	Data *ObjectMetadata
	Err  error
}

// BucketMetadata container for bucket metadata
type BucketMetadata struct {
	// The name of the bucket.
	Name string
	// Date the bucket was created.
	CreationDate time.Time
}

// ObjectMetadata container for object metadata
type ObjectMetadata struct {
	ETag         string
	Key          string
	LastModified time.Time
	Size         int64

	Owner struct {
		DisplayName string
		ID          string
	}

	// The class of storage used to store the object.
	StorageClass string
}

// Regions s3 region map used by bucket location constraint
var Regions = map[string]string{
	"us-gov-west-1":  "https://s3-fips-us-gov-west-1.amazonaws.com",
	"us-east-1":      "https://s3.amazonaws.com",
	"us-west-1":      "https://s3-us-west-1.amazonaws.com",
	"us-west-2":      "https://s3-us-west-2.amazonaws.com",
	"eu-west-1":      "https://s3-eu-west-1.amazonaws.com",
	"eu-central-1":   "https://s3-eu-central-1.amazonaws.com",
	"ap-southeast-1": "https://s3-ap-southeast-1.amazonaws.com",
	"ap-southeast-2": "https://s3-ap-southeast-2.amazonaws.com",
	"ap-northeast-1": "https://s3-ap-northeast-1.amazonaws.com",
	"sa-east-1":      "https://s3-sa-east-1.amazonaws.com",
	"cn-north-1":     "https://s3.cn-north-1.amazonaws.com.cn",
}

type api struct {
	*lowLevelAPI
}

// Config - main configuration struct used by all to set endpoint, credentials, and other options for requests.
type Config struct {
	AccessKeyID     string
	SecretAccessKey string
	Endpoint        string
	ContentType     string
	// not exported internal usage only
	userAgent string
}

// Global constants
const (
	LibraryName    = "objectstorage-go/"
	LibraryVersion = "0.1"
)

// New - instantiate a new minio api client
func New(config *Config) API {
	// Not configurable at the moment, but we will relook on this in future
	config.userAgent = LibraryName + " (" + LibraryVersion + "; " + runtime.GOOS + "; " + runtime.GOARCH + ")"
	return &api{&lowLevelAPI{config}}
}

/// Object operations

// GetObject retrieve object
//
// Additionally it also takes range arguments to download the specified range bytes of an object.
// For more information about the HTTP Range header, go to http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35.
func (a *api) GetObject(bucket, object string, offset, length uint64) (io.ReadCloser, *ObjectMetadata, error) {
	// get the the object
	// NOTE : returned md5sum could be the md5sum of the partial object itself
	// not the whole object depending on if offset range was requested or not
	body, objectMetadata, err := a.getObject(bucket, object, offset, length)
	if err != nil {
		return nil, nil, err
	}
	return body, objectMetadata, nil
}

// completedParts is a wrapper to make parts sortable by their part number
// multi part completion requires list of multi parts to be sorted
type completedParts []*completePart

func (a completedParts) Len() int           { return len(a) }
func (a completedParts) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a completedParts) Less(i, j int) bool { return a[i].PartNumber < a[j].PartNumber }

// DefaultPartSize - default size per object after which PutObject becomes multipart
var DefaultPartSize uint64 = 1024 * 1024 * 5

// CreateObject create an object in a bucket
//
// You must have WRITE permissions on a bucket to create an object
//
// This version of CreateObject automatically does multipart for more than 5MB worth of data
// This default part size is not configurable currently but can be configurable in future
func (a *api) CreateObject(bucket, object string, size uint64, data io.Reader) (string, error) {
	switch {
	case size < DefaultPartSize:
		// Single Part use case, use PutObject directly
		for part := range MultiPart(data, DefaultPartSize) {
			if part.Err != nil {
				return "", part.Err
			}
			return "", a.putObject(bucket, object, part.Len, part.Data)
		}
	default:
		initiateMultipartUploadResult, err := a.initiateMultipartUpload(bucket, object)
		if err != nil {
			return "", err
		}
		uploadID := initiateMultipartUploadResult.UploadID
		completeMultipartUpload := new(completeMultipartUpload)
		for part := range MultiPart(data, DefaultPartSize) {
			if part.Err != nil {
				return "", part.Err
			}
			completePart, err := a.uploadPart(bucket, object, uploadID, part.Num, part.Len, part.Data)
			if err != nil {
				return "", a.abortMultipartUpload(bucket, object, uploadID)
			}
			completeMultipartUpload.Part = append(completeMultipartUpload.Part, completePart)
		}
		sort.Sort(completedParts(completeMultipartUpload.Part))
		completeMultipartUploadResult, err := a.completeMultipartUpload(bucket, object, uploadID, completeMultipartUpload)
		if err != nil {
			return "", a.abortMultipartUpload(bucket, object, uploadID)
		}
		return completeMultipartUploadResult.ETag, nil
	}
	return "", errors.New("Unexpected control flow")
}

// StatObject verify if object exists and you have permission to access it
func (a *api) StatObject(bucket, object string) (*ObjectMetadata, error) {
	return a.headObject(bucket, object)
}

// DeleteObject remove the object from a bucket
func (a *api) DeleteObject(bucket, object string) error {
	return a.deleteObject(bucket, object)
}

/// Bucket operations

// CreateBucket create a new bucket
//
// optional arguments are acl and location - by default all buckets are created
// with ``private`` acl and location set to US Standard if one wishes to set
// different ACLs and Location one can set them properly.
//
// ACL valid values
// ------------------
// private - owner gets full access [DEFAULT]
// public-read - owner gets full access, others get read access
// public-read-write - owner gets full access, others get full access too
// ------------------
//
// Location valid values
// ------------------
// [ us-west-1 | us-west-2 | eu-west-1 | eu-central-1 | ap-southeast-1 | ap-northeast-1 | ap-southeast-2 | sa-east-1 ]
// Default - US standard
func (a *api) CreateBucket(bucket, acl, location string) error {
	return a.putBucket(bucket, acl, location)
}

// SetBucketACL set the permissions on an existing bucket using access control lists (ACL)
//
// Currently supported are:
// ------------------
// private - owner gets full access
// public-read - owner gets full access, others get read access
// public-read-write - owner gets full access, others get full access too
// ------------------
func (a *api) SetBucketACL(bucket, acl string) error {
	return a.putBucketACL(bucket, acl)
}

// StatBucket verify if bucket exists and you have permission to access it
func (a *api) StatBucket(bucket string) error {
	return a.headBucket(bucket)
}

// DeleteBucket deletes the bucket named in the URI
// NOTE: -
//  All objects (including all object versions and delete markers)
//  in the bucket must be deleted before successfully attempting this request
func (a *api) DeleteBucket(bucket string) error {
	return a.deleteBucket(bucket)
}

// listObjectsInRoutine is an internal goroutine function called for listing objects
// This function feeds data into channel
func (a *api) listObjectsInRoutine(bucket, prefix string, recursive bool, ch chan ObjectOnChannel) {
	defer close(ch)
	switch {
	case recursive == true:
		listBucketResult, err := a.listObjects(bucket, 1000, "", prefix, "")
		if err != nil {
			ch <- ObjectOnChannel{
				Data: nil,
				Err:  err,
			}
			return
		}
		for _, object := range listBucketResult.Contents {
			ch <- ObjectOnChannel{
				Data: object,
				Err:  nil,
			}
		}
		for {
			if !listBucketResult.IsTruncated {
				break
			}
			listBucketResult, err = a.listObjects(bucket, 1000, listBucketResult.Marker, prefix, "")
			if err != nil {
				ch <- ObjectOnChannel{
					Data: nil,
					Err:  err,
				}
				return
			}
			for _, object := range listBucketResult.Contents {
				ch <- ObjectOnChannel{
					Data: object,
					Err:  nil,
				}
				listBucketResult.Marker = object.Key
			}
		}
	default:
		listBucketResult, err := a.listObjects(bucket, 1000, "", prefix, "/")
		if err != nil {
			ch <- ObjectOnChannel{
				Data: nil,
				Err:  err,
			}
			return
		}
		for _, object := range listBucketResult.Contents {
			ch <- ObjectOnChannel{
				Data: object,
				Err:  nil,
			}
		}
	}
}

// ListObjects - (List Objects) - List some objects or all recursively
//
// ListObjects is a channel based API implemented to facilitate ease of usage of S3 API ListObjects()
// by automatically recursively traversing all objects on a given bucket if specified.
//
// Your input paramters are just bucket, prefix and recursive
//
// If you enable recursive as 'true' this function will return back all the objects in a given bucket
//
//  eg:-
//         api := objectstorage.New(....)
//         for message := range api.ListObjects("mytestbucket", "starthere", true) {
//                 fmt.Println(message.Data)
//         }
//
func (a *api) ListObjects(bucket string, prefix string, recursive bool) <-chan ObjectOnChannel {
	ch := make(chan ObjectOnChannel)
	go a.listObjectsInRoutine(bucket, prefix, recursive, ch)
	return ch
}

// listBucketsInRoutine is an internal go routine function called for listing buckets
// This function feeds data into channel
func (a *api) listBucketsInRoutine(ch chan BucketOnChannel) {
	defer close(ch)
	listAllMyBucketListResults, err := a.listBuckets()
	if err != nil {
		ch <- BucketOnChannel{
			Data: nil,
			Err:  err,
		}
		return
	}
	for _, bucket := range listAllMyBucketListResults.Buckets.Bucket {
		ch <- BucketOnChannel{
			Data: bucket,
			Err:  nil,
		}
	}

}

// ListBuckets list of all buckets owned by the authenticated sender of the request
//
// NOTE:
//     This call requires explicit authentication, no anonymous
//     requests are allowed for listing buckets
//
//  eg:-
//         api := objectstorage.New(....)
//         for message := range api.ListBuckets() {
//                 fmt.Println(message.Data)
//         }
//
func (a *api) ListBuckets() <-chan BucketOnChannel {
	ch := make(chan BucketOnChannel)
	go a.listBucketsInRoutine(ch)
	return ch
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"strings"

	"github.com/bitly/go-nsq"
	"github.com/garyburd/redigo/redis"
	"github.com/zenazn/goji/web"
)

type ScriptRequest struct {
	ID          int
	Script      string            `json:"script"`
	Args        []string          `json:"args"`
	Files       map[string]string `json:"files"`
	CallbackURL string            `json:"callback_url"`
}

// Handle the root route, also useful as a heartbeat.
func ServiceRoot(w http.ResponseWriter, r *http.Request) {
	if r.Method == "POST" {
		body, err := ioutil.ReadAll(r.Body)
		if err == nil {
			log.Info("Callback: %s", body)
		}
	}
	w.Write([]byte("."))
}

// Get a list of all the scripts in script folder.
func GetAllScripts(w http.ResponseWriter, r *http.Request) {

	// Open and parse whitelist
	p := path.Join(config.Worker.ScriptDir, config.Worker.WhiteList)
	file, err := os.Open(p)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)
	scanner.Scan()
	list := make([]string, 0)
	for scanner.Scan() {
		list = append(list, fmt.Sprintf("\"%s\"", scanner.Text()))
	}
	var buf bytes.Buffer
	buf.WriteString("[")
	buf.WriteString(strings.Join(list, ", "))
	buf.WriteString("]")

	w.Header().Set("Content-Type", "application/json")
	w.Write(buf.Bytes())
}

// Reload the whitelist of scripts.
func ReloadScripts(w http.ResponseWriter, r *http.Request) {
	p := path.Join(config.Worker.ScriptDir, config.Worker.WhiteList)

	doneChan := make(chan *nsq.ProducerTransaction)
	err := producer.PublishAsync("reload", []byte(p), doneChan)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	<-doneChan

	var buf bytes.Buffer
	buf.WriteString("Reload request sent")

	w.Header().Set("Content-Type", "application/json")
	w.Write(buf.Bytes())
}

// Send details to queue for execution.
func RunScript(c web.C, w http.ResponseWriter, r *http.Request) {
	// Parse the request
	body, err := ioutil.ReadAll(r.Body)
	if err != nil {
		log.Error(err.Error())
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	var sr ScriptRequest
	err = json.Unmarshal(body, &sr)
	if err != nil {
		log.Error(err.Error())
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	id, err := getRedisID()
	if err != nil {
		log.Error(err.Error())
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	sr.ID = id
	sr.Script = c.URLParams["name"]

	// Queue up the request
	doneChan := make(chan *nsq.ProducerTransaction)
	data, err := json.Marshal(sr)
	if err != nil {
		log.Error(err.Error())
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	err = producer.PublishAsync(config.Topic, data, doneChan)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	<-doneChan
	log.Debug("Request queued as %d", sr.ID)

	w.Header().Set("Content-Type", "application/json")
	w.Write(data)
}

// Retrieve all logs for a specific script.
func GetAllLogs(c web.C, w http.ResponseWriter, r *http.Request) {
	conn := redisDB.Get()
	defer conn.Close()

	// LRANGE returns an array of json strings
	reply, err := redis.Strings(conn.Do("ZRANGE", c.URLParams["name"], 0, -1))
	if err != nil {
		log.Error(err.Error())
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	var buf bytes.Buffer
	buf.WriteString("[")

	length := len(reply)
	if length > 0 {
		buf.WriteString(reply[length-1])
		for i := length - 2; i > 0; i-- {
			buf.WriteString(", ")
			buf.WriteString(reply[i])
		}
	}
	buf.WriteString("]")
	w.Header().Set("Content-Type", "application/json")
	w.Write(buf.Bytes())
}

// Retrieve a specific log of a specific script.
func GetLog(c web.C, w http.ResponseWriter, r *http.Request) {
	conn := redisDB.Get()
	defer conn.Close()

	script := c.URLParams["name"]
	id := c.URLParams["id"]

	reply, err := redis.Strings(conn.Do("ZRANGEBYSCORE", script, id, id))
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	w.Header().Set("Content-Type", "application/json")
	w.Write([]byte(reply[0]))
}
<commit_msg>Match log level of enqueue with dequeue.<commit_after>package main

import (
	"bufio"
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"strings"

	"github.com/bitly/go-nsq"
	"github.com/garyburd/redigo/redis"
	"github.com/zenazn/goji/web"
)

type ScriptRequest struct {
	ID          int
	Script      string            `json:"script"`
	Args        []string          `json:"args"`
	Files       map[string]string `json:"files"`
	CallbackURL string            `json:"callback_url"`
}

// Handle the root route, also useful as a heartbeat.
func ServiceRoot(w http.ResponseWriter, r *http.Request) {
	if r.Method == "POST" {
		body, err := ioutil.ReadAll(r.Body)
		if err == nil {
			log.Info("Callback: %s", body)
		}
	}
	w.Write([]byte("."))
}

// Get a list of all the scripts in script folder.
func GetAllScripts(w http.ResponseWriter, r *http.Request) {

	// Open and parse whitelist
	p := path.Join(config.Worker.ScriptDir, config.Worker.WhiteList)
	file, err := os.Open(p)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)
	scanner.Scan()
	list := make([]string, 0)
	for scanner.Scan() {
		list = append(list, fmt.Sprintf("\"%s\"", scanner.Text()))
	}
	var buf bytes.Buffer
	buf.WriteString("[")
	buf.WriteString(strings.Join(list, ", "))
	buf.WriteString("]")

	w.Header().Set("Content-Type", "application/json")
	w.Write(buf.Bytes())
}

// Reload the whitelist of scripts.
func ReloadScripts(w http.ResponseWriter, r *http.Request) {
	p := path.Join(config.Worker.ScriptDir, config.Worker.WhiteList)

	doneChan := make(chan *nsq.ProducerTransaction)
	err := producer.PublishAsync("reload", []byte(p), doneChan)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	<-doneChan

	var buf bytes.Buffer
	buf.WriteString("Reload request sent")

	w.Header().Set("Content-Type", "application/json")
	w.Write(buf.Bytes())
}

// Send details to queue for execution.
func RunScript(c web.C, w http.ResponseWriter, r *http.Request) {
	// Parse the request
	body, err := ioutil.ReadAll(r.Body)
	if err != nil {
		log.Error(err.Error())
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	var sr ScriptRequest
	err = json.Unmarshal(body, &sr)
	if err != nil {
		log.Error(err.Error())
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	id, err := getRedisID()
	if err != nil {
		log.Error(err.Error())
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	sr.ID = id
	sr.Script = c.URLParams["name"]

	// Queue up the request
	doneChan := make(chan *nsq.ProducerTransaction)
	data, err := json.Marshal(sr)
	if err != nil {
		log.Error(err.Error())
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	err = producer.PublishAsync(config.Topic, data, doneChan)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	<-doneChan
	log.Info("Request queued as %d", sr.ID)

	w.Header().Set("Content-Type", "application/json")
	w.Write(data)
}

// Retrieve all logs for a specific script.
func GetAllLogs(c web.C, w http.ResponseWriter, r *http.Request) {
	conn := redisDB.Get()
	defer conn.Close()

	// LRANGE returns an array of json strings
	reply, err := redis.Strings(conn.Do("ZRANGE", c.URLParams["name"], 0, -1))
	if err != nil {
		log.Error(err.Error())
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	var buf bytes.Buffer
	buf.WriteString("[")

	length := len(reply)
	if length > 0 {
		buf.WriteString(reply[length-1])
		for i := length - 2; i > 0; i-- {
			buf.WriteString(", ")
			buf.WriteString(reply[i])
		}
	}
	buf.WriteString("]")
	w.Header().Set("Content-Type", "application/json")
	w.Write(buf.Bytes())
}

// Retrieve a specific log of a specific script.
func GetLog(c web.C, w http.ResponseWriter, r *http.Request) {
	conn := redisDB.Get()
	defer conn.Close()

	script := c.URLParams["name"]
	id := c.URLParams["id"]

	reply, err := redis.Strings(conn.Do("ZRANGEBYSCORE", script, id, id))
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	w.Header().Set("Content-Type", "application/json")
	w.Write([]byte(reply[0]))
}
<|endoftext|>