<commit_before>/*
Copyright 2017 The Nuclio Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package controller

import (
	"context"
	"fmt"
	"strings"
	"time"

	"github.com/nuclio/nuclio/pkg/common"
	"github.com/nuclio/nuclio/pkg/functionconfig"
	nuclioio "github.com/nuclio/nuclio/pkg/platform/kube/apis/nuclio.io/v1beta1"
	"github.com/nuclio/nuclio/pkg/platform/kube/client"
	"github.com/nuclio/nuclio/pkg/platform/kube/functionres"
	"github.com/nuclio/nuclio/pkg/platform/kube/operator"

	"github.com/nuclio/errors"
	"github.com/nuclio/logger"
	"github.com/v3io/scaler/pkg/scalertypes"
	"k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/validation"
	"k8s.io/apimachinery/pkg/watch"
	"k8s.io/client-go/tools/cache"
)

type functionOperator struct {
	logger            logger.Logger
	controller        *Controller
	operator          operator.Operator
	imagePullSecrets  string
	functionresClient functionres.Client
}

func newFunctionOperator(parentLogger logger.Logger,
	controller *Controller,
	resyncInterval *time.Duration,
	imagePullSecrets string,
	functionresClient functionres.Client,
	numWorkers int) (*functionOperator, error) {
	var err error

	loggerInstance := parentLogger.GetChild("function")

	newFunctionOperator := &functionOperator{
		logger:            loggerInstance,
		controller:        controller,
		imagePullSecrets:  imagePullSecrets,
		functionresClient: functionresClient,
	}

	// create a function operator
	newFunctionOperator.operator, err = operator.NewMultiWorker(loggerInstance,
		numWorkers,
		newFunctionOperator.getListWatcher(controller.namespace),
		&nuclioio.NuclioFunction{},
		resyncInterval,
		newFunctionOperator)

	if err != nil {
		return nil, errors.Wrap(err, "Failed to create function operator")
	}

	parentLogger.DebugWith("Created function operator",
		"numWorkers", numWorkers,
		"resyncInterval", resyncInterval)

	return newFunctionOperator, nil
}

// CreateOrUpdate handles creation/update of an object
func (fo *functionOperator) CreateOrUpdate(ctx context.Context, object runtime.Object) error {
	function, objectIsFunction := object.(*nuclioio.NuclioFunction)
	if !objectIsFunction {
		return fo.setFunctionError(nil,
			functionconfig.FunctionStateError,
			errors.New("Received unexpected object, expected function"))
	}

	defer common.CatchAndLogPanicWithOptions(ctx, // nolint: errcheck
		fo.logger,
		"nucliofunction.CreateOrUpdate",
		&common.CatchAndLogPanicOptions{
			Args: []interface{}{
				"function", function,
			},
			CustomHandler: func(panicError error) {
				fo.setFunctionError(function, // nolint: errcheck
					functionconfig.FunctionStateError,
					errors.Wrap(panicError, "Failed to create/update function"))
			},
		})

	// validate function name is according to k8s convention
	errorMessages := validation.IsQualifiedName(function.Name)
	if len(errorMessages) != 0 {
		joinedErrorMessage := strings.Join(errorMessages, ", ")
		return errors.New("Function name doesn't conform to k8s naming convention. Errors: " + joinedErrorMessage)
	}

	// ready functions as part of controller resyncs, where we verify that a given function CRD has its resources
	// properly configured
	statesToRespond := []functionconfig.FunctionState{

		// monitor provisioning states, we need to create / update function resources
		functionconfig.FunctionStateWaitingForResourceConfiguration,
		functionconfig.FunctionStateWaitingForScaleResourcesFromZero,
		functionconfig.FunctionStateWaitingForScaleResourcesToZero,

		// to know when to scale a function to zero
		functionconfig.FunctionStateReady,

		// to know when to scale a function from zero
		functionconfig.FunctionStateScaledToZero,
	}
	if !functionconfig.FunctionStateInSlice(function.Status.State, statesToRespond) {
		fo.logger.DebugWith("NuclioFunction is not waiting for resource creation or ready, skipping create/update",
			"name", function.Name,
			"state", function.Status.State,
			"namespace", function.Namespace)

		return nil
	}

	// imported functions have skip deploy annotation, set its state and bail
	if functionconfig.ShouldSkipDeploy(function.Annotations) {
		fo.logger.InfoWith("Skipping function deploy",
			"name", function.Name,
			"state", function.Status.State,
			"namespace", function.Namespace)
		return fo.setFunctionStatus(function, &functionconfig.Status{
			State: functionconfig.FunctionStateImported,
		})
	}

	// wait for up to the default readiness timeout or whatever was set in the spec
	readinessTimeout := function.Spec.ReadinessTimeoutSeconds
	if readinessTimeout == 0 {
		readinessTimeout = int(fo.
			controller.
			GetPlatformConfiguration().
			GetDefaultFunctionReadinessTimeout().Seconds())
	}

	fo.logger.DebugWith("Ensuring function resources",
		"functionNamespace", function.Namespace,
		"readinessTimeout", readinessTimeout,
		"functionName", function.Name)

	// ensure function resources (deployment, ingress, configmap, etc ...)
	resources, err := fo.functionresClient.CreateOrUpdate(ctx, function, fo.imagePullSecrets)
	if err != nil {
		return fo.setFunctionError(function,
			functionconfig.FunctionStateError,
			errors.Wrap(err, "Failed to create/update function"))
	}

	// readinessTimeout would be zero when
	// - not defined on function spec
	// - defined 0 on platform-config
	if readinessTimeout != 0 {
		waitContext, cancel := context.WithDeadline(ctx, time.Now().Add(time.Duration(readinessTimeout)*time.Second))
		defer cancel()

		// wait until the function resources are ready
		if err = fo.functionresClient.WaitAvailable(waitContext, function.Namespace, function.Name); err != nil {
			return fo.setFunctionError(function,
				functionconfig.FunctionStateUnhealthy,
				errors.Wrap(err, "Failed to wait for function resources to be available"))
		}
	}

	waitingStates := []functionconfig.FunctionState{
		functionconfig.FunctionStateWaitingForResourceConfiguration,
		functionconfig.FunctionStateWaitingForScaleResourcesFromZero,
		functionconfig.FunctionStateWaitingForScaleResourcesToZero,
	}

	if functionconfig.FunctionStateInSlice(function.Status.State, waitingStates) {

		var scaleEvent scalertypes.ScaleEvent
		var finalState functionconfig.FunctionState
		switch function.Status.State {
		case functionconfig.FunctionStateWaitingForScaleResourcesToZero:
			scaleEvent = scalertypes.ScaleToZeroCompletedScaleEvent
			finalState = functionconfig.FunctionStateScaledToZero
		case functionconfig.FunctionStateWaitingForScaleResourcesFromZero:
			scaleEvent = scalertypes.ScaleFromZeroCompletedScaleEvent
			finalState = functionconfig.FunctionStateReady
		case functionconfig.FunctionStateWaitingForResourceConfiguration:
			scaleEvent = scalertypes.ResourceUpdatedScaleEvent
			finalState = functionconfig.FunctionStateReady
		}

		// NOTE: this reconstructs function status and hence omits all other function status fields
		// ... such as message and logs.
		functionStatus := &functionconfig.Status{
			State: finalState,
		}

		if err := fo.populateFunctionInvocationStatus(function, functionStatus, resources); err != nil {
			return errors.Wrap(err, "Failed to populate function invocation status")
		}

		if err := fo.setFunctionScaleToZeroStatus(ctx, functionStatus, scaleEvent); err != nil {
			return errors.Wrap(err, "Failed setting function scale to zero status")
		}

		return fo.setFunctionStatus(function, functionStatus)
	}

	return nil
}

// Delete handles delete of an object
func (fo *functionOperator) Delete(ctx context.Context, namespace string, name string) error {
	fo.logger.DebugWith("Deleting function",
		"name", name,
		"namespace", namespace)

	return fo.functionresClient.Delete(ctx, namespace, name)
}

func (fo *functionOperator) setFunctionScaleToZeroStatus(ctx context.Context,
	functionStatus *functionconfig.Status,
	scaleToZeroEvent scalertypes.ScaleEvent) error {

	fo.logger.DebugWith("Setting scale to zero status",
		"LastScaleEvent", scaleToZeroEvent)
	now := time.Now()
	functionStatus.ScaleToZero = &functionconfig.ScaleToZeroStatus{
		LastScaleEvent:     scaleToZeroEvent,
		LastScaleEventTime: &now,
	}
	return nil
}

func (fo *functionOperator) start() error {
	go fo.operator.Start() // nolint: errcheck

	return nil
}

func (fo *functionOperator) setFunctionError(function *nuclioio.NuclioFunction,
	functionErrorState functionconfig.FunctionState,
	err error) error {

	// whatever the error, try to update the function CR
	fo.logger.WarnWith("Setting function error",
		"functionErrorState", functionErrorState,
		"functionName", function.Name,
		"err", err)

	if setStatusErr := fo.setFunctionStatus(function, &functionconfig.Status{
		Logs:    function.Status.Logs,
		State:   functionErrorState,
		Message: errors.GetErrorStackString(err, 10),
	}); setStatusErr != nil {
		fo.logger.WarnWith("Failed to update function on error",
			"setStatusErr", errors.Cause(setStatusErr))
	}

	return err
}

func (fo *functionOperator) setFunctionStatus(function *nuclioio.NuclioFunction,
	status *functionconfig.Status) error {

	fo.logger.DebugWith("Setting function state", "name", function.Name, "status", status)

	// indicate error state
	function.Status = *status

	// try to update the function
	_, err := fo.controller.nuclioClientSet.NuclioV1beta1().NuclioFunctions(function.Namespace).Update(function)
	return err
}

func (fo *functionOperator) getListWatcher(namespace string) cache.ListerWatcher {
	return &cache.ListWatch{
		ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
			return fo.controller.nuclioClientSet.NuclioV1beta1().NuclioFunctions(namespace).List(options)
		},
		WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
			return fo.controller.nuclioClientSet.NuclioV1beta1().NuclioFunctions(namespace).Watch(options)
		},
	}
}

func (fo *functionOperator) getFunctionHTTPPort(functionResources functionres.Resources) (int, error) {
	var httpPort int

	service, err := functionResources.Service()
	if err != nil {
		return 0, errors.Wrap(err, "Failed to get function service")
	}

	if service != nil && len(service.Spec.Ports) != 0 {
		for _, port := range service.Spec.Ports {
			if port.Name == functionres.ContainerHTTPPortName {
				httpPort = int(port.NodePort)
				break
			}
		}
	}
	return httpPort, nil
}

func (fo *functionOperator) populateFunctionInvocationStatus(function *nuclioio.NuclioFunction,
	functionStatus *functionconfig.Status,
	functionResources functionres.Resources) error {

	// get function http port
	httpPort, err := fo.getFunctionHTTPPort(functionResources)
	if err != nil {
		return errors.Wrap(err, "Failed to get function http port")
	}

	service, err := functionResources.Service()
	if err != nil {
		return errors.Wrap(err, "Failed to get function service")
	}

	ingress, err := functionResources.Ingress()
	if err != nil {
		return errors.Wrap(err, "Failed to get function ingress")
	}

	functionStatus.HTTPPort = httpPort

	// add internal invocation urls
	functionStatus.InternalInvocationURLs = []string{}
	if service != nil {
		serviceHost, servicePort := client.GetDomainNameInvokeURL(service.GetName(), service.GetNamespace())
		functionStatus.InternalInvocationURLs = append(functionStatus.InternalInvocationURLs,
			fmt.Sprintf("%s:%d", serviceHost, servicePort))
	}

	// TODO: move the information on platformConfig and share with controller?
	// add external invocation url in form of "external-ip:nodeport"
	// first item is being filled by nuclio-dashboard to holds the information regarding the external ip address
	if len(function.Status.ExternalInvocationURLs) > 0 && service.Spec.Type == v1.ServiceTypeNodePort {
		hostPort := strings.Split(function.Status.ExternalInvocationURLs[0], ":")
		functionStatus.ExternalInvocationURLs = []string{fmt.Sprintf("%s:%d", hostPort[0], httpPort)}
	} else {
		functionStatus.ExternalInvocationURLs = []string{}
	}

	// add ingresses to external invocation urls
	if ingress != nil {
		for _, rule := range ingress.Spec.Rules {
			host := rule.Host
			path := "/"
			if rule.HTTP != nil {
				if len(rule.HTTP.Paths) > 0 {
					path = rule.HTTP.Paths[0].Path
				}
			}
			functionStatus.ExternalInvocationURLs = append(functionStatus.ExternalInvocationURLs,
				fmt.Sprintf("%s%s", host, path))
		}
	}
	return nil

}
<commit_msg>Preserve build logs - update (#2281)<commit_after>/*
Copyright 2017 The Nuclio Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package controller

import (
	"context"
	"fmt"
	"strings"
	"time"

	"github.com/nuclio/nuclio/pkg/common"
	"github.com/nuclio/nuclio/pkg/functionconfig"
	nuclioio "github.com/nuclio/nuclio/pkg/platform/kube/apis/nuclio.io/v1beta1"
	"github.com/nuclio/nuclio/pkg/platform/kube/client"
	"github.com/nuclio/nuclio/pkg/platform/kube/functionres"
	"github.com/nuclio/nuclio/pkg/platform/kube/operator"

	"github.com/nuclio/errors"
	"github.com/nuclio/logger"
	"github.com/v3io/scaler/pkg/scalertypes"
	"k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/validation"
	"k8s.io/apimachinery/pkg/watch"
	"k8s.io/client-go/tools/cache"
)

type functionOperator struct {
	logger            logger.Logger
	controller        *Controller
	operator          operator.Operator
	imagePullSecrets  string
	functionresClient functionres.Client
}

func newFunctionOperator(parentLogger logger.Logger,
	controller *Controller,
	resyncInterval *time.Duration,
	imagePullSecrets string,
	functionresClient functionres.Client,
	numWorkers int) (*functionOperator, error) {
	var err error

	loggerInstance := parentLogger.GetChild("function")

	newFunctionOperator := &functionOperator{
		logger:            loggerInstance,
		controller:        controller,
		imagePullSecrets:  imagePullSecrets,
		functionresClient: functionresClient,
	}

	// create a function operator
	newFunctionOperator.operator, err = operator.NewMultiWorker(loggerInstance,
		numWorkers,
		newFunctionOperator.getListWatcher(controller.namespace),
		&nuclioio.NuclioFunction{},
		resyncInterval,
		newFunctionOperator)

	if err != nil {
		return nil, errors.Wrap(err, "Failed to create function operator")
	}

	parentLogger.DebugWith("Created function operator",
		"numWorkers", numWorkers,
		"resyncInterval", resyncInterval)

	return newFunctionOperator, nil
}

// CreateOrUpdate handles creation/update of an object
func (fo *functionOperator) CreateOrUpdate(ctx context.Context, object runtime.Object) error {
	function, objectIsFunction := object.(*nuclioio.NuclioFunction)
	if !objectIsFunction {
		return fo.setFunctionError(nil,
			functionconfig.FunctionStateError,
			errors.New("Received unexpected object, expected function"))
	}

	defer common.CatchAndLogPanicWithOptions(ctx, // nolint: errcheck
		fo.logger,
		"nucliofunction.CreateOrUpdate",
		&common.CatchAndLogPanicOptions{
			Args: []interface{}{
				"function", function,
			},
			CustomHandler: func(panicError error) {
				fo.setFunctionError(function, // nolint: errcheck
					functionconfig.FunctionStateError,
					errors.Wrap(panicError, "Failed to create/update function"))
			},
		})

	// validate function name is according to k8s convention
	errorMessages := validation.IsQualifiedName(function.Name)
	if len(errorMessages) != 0 {
		joinedErrorMessage := strings.Join(errorMessages, ", ")
		return errors.New("Function name doesn't conform to k8s naming convention. Errors: " + joinedErrorMessage)
	}

	// ready functions as part of controller resyncs, where we verify that a given function CRD has its resources
	// properly configured
	statesToRespond := []functionconfig.FunctionState{

		// monitor provisioning states, we need to create / update function resources
		functionconfig.FunctionStateWaitingForResourceConfiguration,
		functionconfig.FunctionStateWaitingForScaleResourcesFromZero,
		functionconfig.FunctionStateWaitingForScaleResourcesToZero,

		// to know when to scale a function to zero
		functionconfig.FunctionStateReady,

		// to know when to scale a function from zero
		functionconfig.FunctionStateScaledToZero,
	}
	if !functionconfig.FunctionStateInSlice(function.Status.State, statesToRespond) {
		fo.logger.DebugWith("NuclioFunction is not waiting for resource creation or ready, skipping create/update",
			"name", function.Name,
			"state", function.Status.State,
			"namespace", function.Namespace)

		return nil
	}

	// imported functions have skip deploy annotation, set its state and bail
	if functionconfig.ShouldSkipDeploy(function.Annotations) {
		fo.logger.InfoWith("Skipping function deploy",
			"name", function.Name,
			"state", function.Status.State,
			"namespace", function.Namespace)
		return fo.setFunctionStatus(function, &functionconfig.Status{
			State: functionconfig.FunctionStateImported,
		})
	}

	// wait for up to the default readiness timeout or whatever was set in the spec
	readinessTimeout := function.Spec.ReadinessTimeoutSeconds
	if readinessTimeout == 0 {
		readinessTimeout = int(fo.
			controller.
			GetPlatformConfiguration().
			GetDefaultFunctionReadinessTimeout().Seconds())
	}

	fo.logger.DebugWith("Ensuring function resources",
		"functionNamespace", function.Namespace,
		"readinessTimeout", readinessTimeout,
		"functionName", function.Name)

	// ensure function resources (deployment, ingress, configmap, etc ...)
	resources, err := fo.functionresClient.CreateOrUpdate(ctx, function, fo.imagePullSecrets)
	if err != nil {
		return fo.setFunctionError(function,
			functionconfig.FunctionStateError,
			errors.Wrap(err, "Failed to create/update function"))
	}

	// readinessTimeout would be zero when
	// - not defined on function spec
	// - defined 0 on platform-config
	if readinessTimeout != 0 {
		waitContext, cancel := context.WithDeadline(ctx, time.Now().Add(time.Duration(readinessTimeout)*time.Second))
		defer cancel()

		// wait until the function resources are ready
		if err = fo.functionresClient.WaitAvailable(waitContext, function.Namespace, function.Name); err != nil {
			return fo.setFunctionError(function,
				functionconfig.FunctionStateUnhealthy,
				errors.Wrap(err, "Failed to wait for function resources to be available"))
		}
	}

	waitingStates := []functionconfig.FunctionState{
		functionconfig.FunctionStateWaitingForResourceConfiguration,
		functionconfig.FunctionStateWaitingForScaleResourcesFromZero,
		functionconfig.FunctionStateWaitingForScaleResourcesToZero,
	}

	if functionconfig.FunctionStateInSlice(function.Status.State, waitingStates) {

		var scaleEvent scalertypes.ScaleEvent
		var finalState functionconfig.FunctionState
		switch function.Status.State {
		case functionconfig.FunctionStateWaitingForScaleResourcesToZero:
			scaleEvent = scalertypes.ScaleToZeroCompletedScaleEvent
			finalState = functionconfig.FunctionStateScaledToZero
		case functionconfig.FunctionStateWaitingForScaleResourcesFromZero:
			scaleEvent = scalertypes.ScaleFromZeroCompletedScaleEvent
			finalState = functionconfig.FunctionStateReady
		case functionconfig.FunctionStateWaitingForResourceConfiguration:
			scaleEvent = scalertypes.ResourceUpdatedScaleEvent
			finalState = functionconfig.FunctionStateReady
		}

		// NOTE: this reconstructs function status and hence omits all other function status fields
		// ... such as message and logs.
		functionStatus := &functionconfig.Status{
			State: finalState,
			Logs:  function.Status.Logs,
		}

		if err := fo.populateFunctionInvocationStatus(function, functionStatus, resources); err != nil {
			return errors.Wrap(err, "Failed to populate function invocation status")
		}

		if err := fo.setFunctionScaleToZeroStatus(ctx, functionStatus, scaleEvent); err != nil {
			return errors.Wrap(err, "Failed setting function scale to zero status")
		}

		return fo.setFunctionStatus(function, functionStatus)
	}

	return nil
}

// Delete handles delete of an object
func (fo *functionOperator) Delete(ctx context.Context, namespace string, name string) error {
	fo.logger.DebugWith("Deleting function",
		"name", name,
		"namespace", namespace)

	return fo.functionresClient.Delete(ctx, namespace, name)
}

func (fo *functionOperator) setFunctionScaleToZeroStatus(ctx context.Context,
	functionStatus *functionconfig.Status,
	scaleToZeroEvent scalertypes.ScaleEvent) error {

	fo.logger.DebugWith("Setting scale to zero status",
		"LastScaleEvent", scaleToZeroEvent)
	now := time.Now()
	functionStatus.ScaleToZero = &functionconfig.ScaleToZeroStatus{
		LastScaleEvent:     scaleToZeroEvent,
		LastScaleEventTime: &now,
	}
	return nil
}

func (fo *functionOperator) start() error {
	go fo.operator.Start() // nolint: errcheck

	return nil
}

func (fo *functionOperator) setFunctionError(function *nuclioio.NuclioFunction,
	functionErrorState functionconfig.FunctionState,
	err error) error {

	// whatever the error, try to update the function CR
	fo.logger.WarnWith("Setting function error",
		"functionErrorState", functionErrorState,
		"functionName", function.Name,
		"err", err)

	if setStatusErr := fo.setFunctionStatus(function, &functionconfig.Status{
		Logs:    function.Status.Logs,
		State:   functionErrorState,
		Message: errors.GetErrorStackString(err, 10),
	}); setStatusErr != nil {
		fo.logger.WarnWith("Failed to update function on error",
			"setStatusErr", errors.Cause(setStatusErr))
	}

	return err
}

func (fo *functionOperator) setFunctionStatus(function *nuclioio.NuclioFunction,
	status *functionconfig.Status) error {

	fo.logger.DebugWith("Setting function state", "name", function.Name, "status", status)

	// indicate error state
	function.Status = *status

	// try to update the function
	_, err := fo.controller.nuclioClientSet.NuclioV1beta1().NuclioFunctions(function.Namespace).Update(function)
	return err
}

func (fo *functionOperator) getListWatcher(namespace string) cache.ListerWatcher {
	return &cache.ListWatch{
		ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
			return fo.controller.nuclioClientSet.NuclioV1beta1().NuclioFunctions(namespace).List(options)
		},
		WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
			return fo.controller.nuclioClientSet.NuclioV1beta1().NuclioFunctions(namespace).Watch(options)
		},
	}
}

func (fo *functionOperator) getFunctionHTTPPort(functionResources functionres.Resources) (int, error) {
	var httpPort int

	service, err := functionResources.Service()
	if err != nil {
		return 0, errors.Wrap(err, "Failed to get function service")
	}

	if service != nil && len(service.Spec.Ports) != 0 {
		for _, port := range service.Spec.Ports {
			if port.Name == functionres.ContainerHTTPPortName {
				httpPort = int(port.NodePort)
				break
			}
		}
	}
	return httpPort, nil
}

func (fo *functionOperator) populateFunctionInvocationStatus(function *nuclioio.NuclioFunction,
	functionStatus *functionconfig.Status,
	functionResources functionres.Resources) error {

	// get function http port
	httpPort, err := fo.getFunctionHTTPPort(functionResources)
	if err != nil {
		return errors.Wrap(err, "Failed to get function http port")
	}

	service, err := functionResources.Service()
	if err != nil {
		return errors.Wrap(err, "Failed to get function service")
	}

	ingress, err := functionResources.Ingress()
	if err != nil {
		return errors.Wrap(err, "Failed to get function ingress")
	}

	functionStatus.HTTPPort = httpPort

	// add internal invocation urls
	functionStatus.InternalInvocationURLs = []string{}
	if service != nil {
		serviceHost, servicePort := client.GetDomainNameInvokeURL(service.GetName(), service.GetNamespace())
		functionStatus.InternalInvocationURLs = append(functionStatus.InternalInvocationURLs,
			fmt.Sprintf("%s:%d", serviceHost, servicePort))
	}

	// TODO: move the information on platformConfig and share with controller?
	// add external invocation url in form of "external-ip:nodeport"
	// first item is being filled by nuclio-dashboard to holds the information regarding the external ip address
	if len(function.Status.ExternalInvocationURLs) > 0 && service.Spec.Type == v1.ServiceTypeNodePort {
		hostPort := strings.Split(function.Status.ExternalInvocationURLs[0], ":")
		functionStatus.ExternalInvocationURLs = []string{fmt.Sprintf("%s:%d", hostPort[0], httpPort)}
	} else {
		functionStatus.ExternalInvocationURLs = []string{}
	}

	// add ingresses to external invocation urls
	if ingress != nil {
		for _, rule := range ingress.Spec.Rules {
			host := rule.Host
			path := "/"
			if rule.HTTP != nil {
				if len(rule.HTTP.Paths) > 0 {
					path = rule.HTTP.Paths[0].Path
				}
			}
			functionStatus.ExternalInvocationURLs = append(functionStatus.ExternalInvocationURLs,
				fmt.Sprintf("%s%s", host, path))
		}
	}
	return nil

}
<|endoftext|>


<commit_before>// Copyright (c) 2013-2014 by Michael Dvorkin. All Rights Reserved.
// Use of this source code is governed by a MIT-style license that can
// be found in the LICENSE file.

package donna

// Returns a bitmask of possible Bishop moves from the given square.
func (p *Position) bishopMoves(square int) Bitmask {
	return p.bishopMovesAt(square, p.board)
}

// Returns a bitmask of possible Rook moves from the given square.
func (p *Position) rookMoves(square int) Bitmask {
	return p.rookMovesAt(square, p.board)
}

// Returns a bitmask of possible Bishop moves from the given square wherees
// other pieces on the board are represented by the explicit parameter.
func (p *Position) bishopMovesAt(square int, board Bitmask) Bitmask {
	magic := ((bishopMagic[square].mask & board) * bishopMagic[square].magic) >> 55
	return bishopMagicMoves[square][magic]
}

// Returns a bitmask of possible Rook moves from the given square wherees other
// pieces on the board are represented by the explicit parameter.
func (p *Position) rookMovesAt(square int, board Bitmask) Bitmask {
	magic := ((rookMagic[square].mask & board) * rookMagic[square].magic) >> 52
	return rookMagicMoves[square][magic]
}

func (p *Position) targets(square int) Bitmask {
	return p.targetsFor(square, p.pieces[square])
}

func (p *Position) targetsFor(square int, piece Piece) (bitmask Bitmask) {
	switch kind, color := piece.kind(), piece.color(); kind {
	case Pawn:
		bitmask = pawnMoves[color][square] & p.outposts[color^1]
		//
		// If the square in front of the pawn is empty then add it as possible
		// target.
		//
		if target := square + eight[color]; p.board.isClear(target) {
			bitmask.set(target)
			//
			// If the pawn is in its initial position and two squares in front of
			// the pawn are empty then add the second square as possible target.
			//
			if RelRow(square, color) == 1 {
				if target += eight[color]; p.board.isClear(target) {
					bitmask.set(target)
				}
			}
		}
		//
		// If the last move set the en-passant square and it is diagonally adjacent
		// to the current pawn, then add en-passant to the pawn's attack targets.
		//
		if p.enpassant != 0 && maskPawn[color][p.enpassant].isSet(square) {
			bitmask.set(p.enpassant)
		}
	case Knight:
		bitmask = knightMoves[square] & ^p.outposts[color]
	case Bishop:
		bitmask = p.bishopMoves(square) & ^p.outposts[color]
	case Rook:
		bitmask = p.rookMoves(square) & ^p.outposts[color]
	case Queen:
		bitmask = (p.bishopMoves(square) | p.rookMoves(square)) & ^p.outposts[color]
	case King:
		bitmask = kingMoves[square] & ^p.outposts[color]
	}
	return
}

func (p *Position) xrayTargets(square int) Bitmask {
	return p.xrayTargetsFor(square, p.pieces[square])
}

func (p *Position) xrayTargetsFor(square int, piece Piece) (bitmask Bitmask) {
	switch kind, color := piece.kind(), piece.color(); kind {
	case Bishop:
		board := p.board ^ p.outposts[queen(color)]
		bitmask = p.bishopMovesAt(square, board) & ^p.outposts[color]
	case Rook:
		board := p.board ^ p.outposts[queen(color)]
		bitmask = p.rookMovesAt(square, board) & ^p.outposts[color]
	}
	return p.targetsFor(square, piece)
}

func (p *Position) attacks(square int) Bitmask {
	return p.attacksFor(square, p.pieces[square])
}

func (p *Position) attacksFor(square int, piece Piece) (bitmask Bitmask) {
	switch kind, color := piece.kind(), piece.color(); kind {
	case Pawn:
		return pawnMoves[color][square]
	case Knight:
		return knightMoves[square]
	case Bishop:
		return p.bishopMoves(square)
	case Rook:
		return p.rookMoves(square)
	case Queen:
		return p.bishopMoves(square) | p.rookMoves(square)
	case King:
		return kingMoves[square]
	}
	return
}

func (p *Position) xrayAttacks(square int) Bitmask {
	return p.xrayAttacksFor(square, p.pieces[square])
}

func (p *Position) xrayAttacksFor(square int, piece Piece) (bitmask Bitmask) {
	switch kind, color := piece.kind(), piece.color(); kind {
	case Bishop:
		board := p.board ^ p.outposts[queen(color)]
		return p.bishopMovesAt(square, board)
	case Rook:
		board := p.board ^ p.outposts[queen(color)]
		return p.rookMovesAt(square, board)
	}
	return p.attacksFor(square, piece)
}

func (p *Position) allAttacks(color int) (bitmask Bitmask) {
	bitmask = p.pawnAttacks(color) | p.knightAttacks(color) | p.kingAttacks(color)

	outposts := p.outposts[bishop(color)] | p.outposts[queen(color)]
	for outposts != 0 {
		bitmask |= p.bishopMoves(outposts.pop())
	}

	outposts = p.outposts[rook(color)] | p.outposts[queen(color)]
	for outposts != 0 {
		bitmask |= p.rookMoves(outposts.pop())
	}
	return
}

// Returns a bitmask of pieces that attack given square. The resulting bitmask
// only counts pieces of requested color.
//
// This method is used in static exchange evaluation so instead of using current
// board bitmask (p.board) we pass the one that gets continuously updated during
// the evaluation.
func (p *Position) attackers(square, color int, board Bitmask) Bitmask {
	attackers := knightMoves[square] & p.outposts[knight(color)]
	attackers |= maskPawn[color][square] & p.outposts[pawn(color)]
	attackers |= kingMoves[square] & p.outposts[king(color)]
	attackers |= p.rookMovesAt(square, board) & (p.outposts[rook(color)] | p.outposts[queen(color)])
	attackers |= p.bishopMovesAt(square, board) & (p.outposts[bishop(color)] | p.outposts[queen(color)])

	return attackers
}

func (p *Position) isAttacked(square, color int) bool {
	return (knightMoves[square] & p.outposts[knight(color)]) != 0 ||
		(maskPawn[color][square] & p.outposts[pawn(color)]) != 0 ||
		(kingMoves[square] & p.outposts[king(color)]) != 0 ||
		(p.rookMoves(square) & (p.outposts[rook(color)]|p.outposts[queen(color)])) != 0 ||
		(p.bishopMoves(square) & (p.outposts[bishop(color)]|p.outposts[queen(color)])) != 0
}

func (p *Position) pawnAttacks(color int) (bitmask Bitmask) {
	if color == White {
		bitmask = (p.outposts[Pawn] & ^maskFile[0]) << 7
		bitmask |= (p.outposts[Pawn] & ^maskFile[7]) << 9
	} else {
		bitmask = (p.outposts[BlackPawn] & ^maskFile[0]) >> 9
		bitmask |= (p.outposts[BlackPawn] & ^maskFile[7]) >> 7
	}
	return
}

func (p *Position) knightAttacks(color int) (bitmask Bitmask) {
	outposts := p.outposts[knight(color)]
	for outposts != 0 {
		bitmask |= knightMoves[outposts.pop()]
	}
	return
}

func (p *Position) bishopAttacks(color int) (bitmask Bitmask) {
	outposts := p.outposts[bishop(color)]
	for outposts != 0 {
		bitmask |= p.bishopMoves(outposts.pop())
	}
	return
}

func (p *Position) rookAttacks(color int) (bitmask Bitmask) {
	outposts := p.outposts[rook(color)]
	for outposts != 0 {
		bitmask |= p.rookMoves(outposts.pop())
	}
	return
}

func (p *Position) queenAttacks(color int) (bitmask Bitmask) {
	outposts := p.outposts[queen(color)]
	for outposts != 0 {
		square := outposts.pop()
		bitmask |= p.rookMoves(square) | p.bishopMoves(square)
	}
	return
}

func (p *Position) kingAttacks(color int) Bitmask {
	return kingMoves[p.king[color]]
}

func (p *Position) strongestPiece(color int, targets Bitmask) Piece {
	if targets & p.outposts[queen(color)] != 0 {
		return queen(color)
	}
	if targets & p.outposts[rook(color)] != 0 {
		return rook(color)
	}
	if targets & p.outposts[bishop(color)] != 0 {
		return bishop(color)
	}
	if targets & p.outposts[knight(color)] != 0 {
		return knight(color)
	}
	if targets & p.outposts[pawn(color)] != 0 {
		return pawn(color)
	}
	return Piece(0)
}<commit_msg>Minor cleanup<commit_after>// Copyright (c) 2013-2014 by Michael Dvorkin. All Rights Reserved.
// Use of this source code is governed by a MIT-style license that can
// be found in the LICENSE file.

package donna

// Returns a bitmask of possible Bishop moves from the given square.
func (p *Position) bishopMoves(square int) Bitmask {
	return p.bishopMovesAt(square, p.board)
}

// Returns a bitmask of possible Rook moves from the given square.
func (p *Position) rookMoves(square int) Bitmask {
	return p.rookMovesAt(square, p.board)
}

// Returns a bitmask of possible Bishop moves from the given square wherees
// other pieces on the board are represented by the explicit parameter.
func (p *Position) bishopMovesAt(square int, board Bitmask) Bitmask {
	magic := ((bishopMagic[square].mask & board) * bishopMagic[square].magic) >> 55
	return bishopMagicMoves[square][magic]
}

// Returns a bitmask of possible Rook moves from the given square wherees other
// pieces on the board are represented by the explicit parameter.
func (p *Position) rookMovesAt(square int, board Bitmask) Bitmask {
	magic := ((rookMagic[square].mask & board) * rookMagic[square].magic) >> 52
	return rookMagicMoves[square][magic]
}

func (p *Position) targets(square int) Bitmask {
	return p.targetsFor(square, p.pieces[square])
}

func (p *Position) targetsFor(square int, piece Piece) (bitmask Bitmask) {
	switch kind, color := piece.kind(), piece.color(); kind {
	case Pawn:
		bitmask = pawnMoves[color][square] & p.outposts[color^1]
		//
		// If the square in front of the pawn is empty then add it as possible
		// target.
		//
		if target := square + eight[color]; p.board.isClear(target) {
			bitmask.set(target)
			//
			// If the pawn is in its initial position and two squares in front of
			// the pawn are empty then add the second square as possible target.
			//
			if RelRow(square, color) == 1 {
				if target += eight[color]; p.board.isClear(target) {
					bitmask.set(target)
				}
			}
		}
		//
		// If the last move set the en-passant square and it is diagonally adjacent
		// to the current pawn, then add en-passant to the pawn's attack targets.
		//
		if p.enpassant != 0 && maskPawn[color][p.enpassant].isSet(square) {
			bitmask.set(p.enpassant)
		}
	case Knight:
		bitmask = knightMoves[square] & ^p.outposts[color]
	case Bishop:
		bitmask = p.bishopMoves(square) & ^p.outposts[color]
	case Rook:
		bitmask = p.rookMoves(square) & ^p.outposts[color]
	case Queen:
		bitmask = (p.bishopMoves(square) | p.rookMoves(square)) & ^p.outposts[color]
	case King:
		bitmask = kingMoves[square] & ^p.outposts[color]
	}
	return
}

func (p *Position) attacks(square int) Bitmask {
	return p.attacksFor(square, p.pieces[square])
}

func (p *Position) attacksFor(square int, piece Piece) (bitmask Bitmask) {
	switch kind, color := piece.kind(), piece.color(); kind {
	case Pawn:
		return pawnMoves[color][square]
	case Knight:
		return knightMoves[square]
	case Bishop:
		return p.bishopMoves(square)
	case Rook:
		return p.rookMoves(square)
	case Queen:
		return p.bishopMoves(square) | p.rookMoves(square)
	case King:
		return kingMoves[square]
	}
	return
}

func (p *Position) xrayAttacks(square int) Bitmask {
	return p.xrayAttacksFor(square, p.pieces[square])
}

func (p *Position) xrayAttacksFor(square int, piece Piece) (bitmask Bitmask) {
	switch kind, color := piece.kind(), piece.color(); kind {
	case Bishop:
		board := p.board ^ p.outposts[queen(color)]
		return p.bishopMovesAt(square, board)
	case Rook:
		board := p.board ^ p.outposts[queen(color)]
		return p.rookMovesAt(square, board)
	}
	return p.attacksFor(square, piece)
}

func (p *Position) allAttacks(color int) (bitmask Bitmask) {
	bitmask = p.pawnAttacks(color) | p.knightAttacks(color) | p.kingAttacks(color)

	outposts := p.outposts[bishop(color)] | p.outposts[queen(color)]
	for outposts != 0 {
		bitmask |= p.bishopMoves(outposts.pop())
	}

	outposts = p.outposts[rook(color)] | p.outposts[queen(color)]
	for outposts != 0 {
		bitmask |= p.rookMoves(outposts.pop())
	}
	return
}

// Returns a bitmask of pieces that attack given square. The resulting bitmask
// only counts pieces of requested color.
//
// This method is used in static exchange evaluation so instead of using current
// board bitmask (p.board) we pass the one that gets continuously updated during
// the evaluation.
func (p *Position) attackers(square, color int, board Bitmask) Bitmask {
	attackers := knightMoves[square] & p.outposts[knight(color)]
	attackers |= maskPawn[color][square] & p.outposts[pawn(color)]
	attackers |= kingMoves[square] & p.outposts[king(color)]
	attackers |= p.rookMovesAt(square, board) & (p.outposts[rook(color)] | p.outposts[queen(color)])
	attackers |= p.bishopMovesAt(square, board) & (p.outposts[bishop(color)] | p.outposts[queen(color)])

	return attackers
}

func (p *Position) isAttacked(square, color int) bool {
	return (knightMoves[square] & p.outposts[knight(color)]) != 0 ||
		(maskPawn[color][square] & p.outposts[pawn(color)]) != 0 ||
		(kingMoves[square] & p.outposts[king(color)]) != 0 ||
		(p.rookMoves(square) & (p.outposts[rook(color)]|p.outposts[queen(color)])) != 0 ||
		(p.bishopMoves(square) & (p.outposts[bishop(color)]|p.outposts[queen(color)])) != 0
}

func (p *Position) pawnAttacks(color int) (bitmask Bitmask) {
	if color == White {
		bitmask = (p.outposts[Pawn] & ^maskFile[0]) << 7
		bitmask |= (p.outposts[Pawn] & ^maskFile[7]) << 9
	} else {
		bitmask = (p.outposts[BlackPawn] & ^maskFile[0]) >> 9
		bitmask |= (p.outposts[BlackPawn] & ^maskFile[7]) >> 7
	}
	return
}

func (p *Position) knightAttacks(color int) (bitmask Bitmask) {
	outposts := p.outposts[knight(color)]
	for outposts != 0 {
		bitmask |= knightMoves[outposts.pop()]
	}
	return
}

func (p *Position) bishopAttacks(color int) (bitmask Bitmask) {
	outposts := p.outposts[bishop(color)]
	for outposts != 0 {
		bitmask |= p.bishopMoves(outposts.pop())
	}
	return
}

func (p *Position) rookAttacks(color int) (bitmask Bitmask) {
	outposts := p.outposts[rook(color)]
	for outposts != 0 {
		bitmask |= p.rookMoves(outposts.pop())
	}
	return
}

func (p *Position) queenAttacks(color int) (bitmask Bitmask) {
	outposts := p.outposts[queen(color)]
	for outposts != 0 {
		square := outposts.pop()
		bitmask |= p.rookMoves(square) | p.bishopMoves(square)
	}
	return
}

func (p *Position) kingAttacks(color int) Bitmask {
	return kingMoves[p.king[color]]
}

func (p *Position) strongestPiece(color int, targets Bitmask) Piece {
	if targets & p.outposts[queen(color)] != 0 {
		return queen(color)
	}
	if targets & p.outposts[rook(color)] != 0 {
		return rook(color)
	}
	if targets & p.outposts[bishop(color)] != 0 {
		return bishop(color)
	}
	if targets & p.outposts[knight(color)] != 0 {
		return knight(color)
	}
	if targets & p.outposts[pawn(color)] != 0 {
		return pawn(color)
	}
	return Piece(0)
}<|endoftext|>


<commit_before>// Copyright (c) 2017 Cisco and/or its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at:
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package l4plugin

import "github.com/ligato/vpp-agent/plugins/defaultplugins/l4plugin/model/l4"

// Resync configures app namespaces to the empty VPP
func (plugin *L4Configurator) ResyncAppNs(appNamespaces []*l4.AppNamespaces_AppNamespace) error {
	return nil
}

// Resync configures app namespaces to the empty VPP
func (plugin *L4Configurator) ResyncFeatures(l4Features *l4.L4Features) error {
	return nil
}<commit_msg>implement resync<commit_after>// Copyright (c) 2017 Cisco and/or its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at:
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package l4plugin

import (
	"github.com/ligato/vpp-agent/plugins/defaultplugins/l4plugin/model/l4"
)

// ResyncAppNs configures app namespaces to the empty VPP
func (plugin *L4Configurator) ResyncAppNs(appNamespaces []*l4.AppNamespaces_AppNamespace) error {
	plugin.Log.WithField("cfg", plugin).Debug("RESYNC application namespaces begin. ")
	// Calculate and log application namespaces resync
	defer func() {
		if plugin.Stopwatch != nil {
			plugin.Stopwatch.PrintLog()
		}
	}()

	var wasError error
	if len(appNamespaces) > 0 {
		for _, appNs := range appNamespaces {
			wasError = plugin.ConfigureAppNamespace(appNs)
		}
	}
	plugin.Log.WithField("cfg", plugin).Debug("RESYNC application namespaces end. ", wasError)

	return nil
}

// ResyncFeatures sets L4Features flag
func (plugin *L4Configurator) ResyncFeatures(l4Features *l4.L4Features) error {
	plugin.Log.WithField("cfg", plugin).Debug("RESYNC L4Features begin. ")
	// Calculate and log L4Features resync
	defer func() {
		if plugin.Stopwatch != nil {
			plugin.Stopwatch.PrintLog()
		}
	}()

	var wasError error
	if l4Features != nil {
		wasError = plugin.ConfigureL4FeatureFlag(l4Features)
	}
	plugin.Log.WithField("cfg", plugin).Debug("RESYNC L4Features end. ", wasError)

	return nil
}
<|endoftext|>


<commit_before>package leveldb

import (
	"bytes"
	"context"
	"crypto/md5"
	"fmt"
	"github.com/syndtr/goleveldb/leveldb"
	leveldb_errors "github.com/syndtr/goleveldb/leveldb/errors"
	"github.com/syndtr/goleveldb/leveldb/opt"
	leveldb_util "github.com/syndtr/goleveldb/leveldb/util"
	"io"
	"os"
	"strings"
	"sync"

	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	weed_util "github.com/chrislusf/seaweedfs/weed/util"
)

const (
	DEFAULT = "_main"
)

func init() {
	filer.Stores = append(filer.Stores, &LevelDB3Store{})
}

type LevelDB3Store struct {
	dir     string
	dbs     map[string]*leveldb.DB
	dbsLock sync.RWMutex
}

func (store *LevelDB3Store) GetName() string {
	return "leveldb3"
}

func (store *LevelDB3Store) Initialize(configuration weed_util.Configuration, prefix string) (err error) {
	dir := configuration.GetString(prefix + "dir")
	return store.initialize(dir)
}

func (store *LevelDB3Store) initialize(dir string) (err error) {
	glog.Infof("filer store leveldb3 dir: %s", dir)
	if err := weed_util.TestFolderWritable(dir); err != nil {
		return fmt.Errorf("Check Level Folder %s Writable: %s", dir, err)
	}
	store.dir = dir

	db, loadDbErr := store.loadDB(DEFAULT)
	if loadDbErr != nil {
		return loadDbErr
	}
	store.dbs = make(map[string]*leveldb.DB)
	store.dbs[DEFAULT] = db

	return
}

func (store *LevelDB3Store) loadDB(name string) (*leveldb.DB, error) {

	opts := &opt.Options{
		BlockCacheCapacity:            32 * 1024 * 1024, // default value is 8MiB
		WriteBuffer:                   16 * 1024 * 1024, // default value is 4MiB
		CompactionTableSizeMultiplier: 4,
	}
	if name != DEFAULT {
		opts = &opt.Options{
			BlockCacheCapacity:            4 * 1024 * 1024, // default value is 8MiB
			WriteBuffer:                   2 * 1024 * 1024, // default value is 4MiB
			CompactionTableSizeMultiplier: 4,
		}
	}

	dbFolder := fmt.Sprintf("%s/%s", store.dir, name)
	os.MkdirAll(dbFolder, 0755)
	db, dbErr := leveldb.OpenFile(dbFolder, opts)
	if leveldb_errors.IsCorrupted(dbErr) {
		db, dbErr = leveldb.RecoverFile(dbFolder, opts)
	}
	if dbErr != nil {
		glog.Errorf("filer store open dir %s: %v", dbFolder, dbErr)
		return nil, dbErr
	}
	return db, nil
}

func (store *LevelDB3Store) findDB(fullpath weed_util.FullPath, isForChildren bool) (*leveldb.DB, string, weed_util.FullPath, error) {

	store.dbsLock.RLock()

	defaultDB := store.dbs[DEFAULT]
	if !strings.HasPrefix(string(fullpath), "/buckets/") {
		store.dbsLock.RUnlock()
		return defaultDB, DEFAULT, fullpath, nil
	}

	// detect bucket
	bucketAndObjectKey := string(fullpath)[len("/buckets/"):]
	t := strings.Index(bucketAndObjectKey, "/")
	if t < 0 && !isForChildren {
		store.dbsLock.RUnlock()
		return defaultDB, DEFAULT, fullpath, nil
	}
	bucket := bucketAndObjectKey
	shortPath := weed_util.FullPath("/")
	if t > 0 {
		bucket = bucketAndObjectKey[:t]
		shortPath = weed_util.FullPath(bucketAndObjectKey[t:])
	}

	println("bucket:", bucket, "shortPath", shortPath)

	if db, found := store.dbs[bucket]; found {
		store.dbsLock.RUnlock()
		return db, bucket, shortPath, nil
	}

	store.dbsLock.RUnlock()
	// upgrade to write lock
	store.dbsLock.Lock()
	defer store.dbsLock.Unlock()

	// double check after getting the write lock
	if db, found := store.dbs[bucket]; found {
		return db, bucket, shortPath, nil
	}

	// create db
	db, err := store.loadDB(bucket)
	if err != nil {
		return nil, bucket, shortPath, err
	}
	store.dbs[bucket] = db

	return db, bucket, shortPath, nil
}

func (store *LevelDB3Store) BeginTransaction(ctx context.Context) (context.Context, error) {
	return ctx, nil
}
func (store *LevelDB3Store) CommitTransaction(ctx context.Context) error {
	return nil
}
func (store *LevelDB3Store) RollbackTransaction(ctx context.Context) error {
	return nil
}

func (store *LevelDB3Store) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) {

	db, _, shortPath, err := store.findDB(entry.FullPath, false)
	if err != nil {
		return fmt.Errorf("findDB %s : %v", entry.FullPath, err)
	}

	dir, name := shortPath.DirAndName()
	key := genKey(dir, name)

	value, err := entry.EncodeAttributesAndChunks()
	if err != nil {
		return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err)
	}

	if len(entry.Chunks) > 50 {
		value = weed_util.MaybeGzipData(value)
	}

	err = db.Put(key, value, nil)

	if err != nil {
		return fmt.Errorf("persisting %s : %v", entry.FullPath, err)
	}

	// println("saved", entry.FullPath, "chunks", len(entry.Chunks))

	return nil
}

func (store *LevelDB3Store) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) {

	return store.InsertEntry(ctx, entry)
}

func (store *LevelDB3Store) FindEntry(ctx context.Context, fullpath weed_util.FullPath) (entry *filer.Entry, err error) {

	db, _, shortPath, err := store.findDB(fullpath, false)
	if err != nil {
		return nil, fmt.Errorf("findDB %s : %v", fullpath, err)
	}

	dir, name := shortPath.DirAndName()
	key := genKey(dir, name)

	data, err := db.Get(key, nil)

	if err == leveldb.ErrNotFound {
		return nil, filer_pb.ErrNotFound
	}
	if err != nil {
		return nil, fmt.Errorf("get %s : %v", fullpath, err)
	}

	entry = &filer.Entry{
		FullPath: fullpath,
	}
	err = entry.DecodeAttributesAndChunks(weed_util.MaybeDecompressData(data))
	if err != nil {
		return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err)
	}

	// println("read", entry.FullPath, "chunks", len(entry.Chunks), "data", len(data), string(data))

	return entry, nil
}

func (store *LevelDB3Store) DeleteEntry(ctx context.Context, fullpath weed_util.FullPath) (err error) {

	db, _, shortPath, err := store.findDB(fullpath, false)
	if err != nil {
		return fmt.Errorf("findDB %s : %v", fullpath, err)
	}

	dir, name := shortPath.DirAndName()
	key := genKey(dir, name)

	err = db.Delete(key, nil)
	if err != nil {
		return fmt.Errorf("delete %s : %v", fullpath, err)
	}

	return nil
}

func (store *LevelDB3Store) DeleteFolderChildren(ctx context.Context, fullpath weed_util.FullPath) (err error) {

	db, bucket, shortPath, err := store.findDB(fullpath, true)
	if err != nil {
		return fmt.Errorf("findDB %s : %v", fullpath, err)
	}

	if bucket != DEFAULT && shortPath == "/" {
		db.Close()
		if bucket != "" { // just to make sure
			os.RemoveAll(store.dir + "/" + bucket)
		}
		return nil
	}

	directoryPrefix := genDirectoryKeyPrefix(shortPath, "")

	batch := new(leveldb.Batch)

	iter := db.NewIterator(&leveldb_util.Range{Start: directoryPrefix}, nil)
	for iter.Next() {
		key := iter.Key()
		if !bytes.HasPrefix(key, directoryPrefix) {
			break
		}
		fileName := getNameFromKey(key)
		if fileName == "" {
			continue
		}
		batch.Delete(append(directoryPrefix, []byte(fileName)...))
	}
	iter.Release()

	err = db.Write(batch, nil)

	if err != nil {
		return fmt.Errorf("delete %s : %v", fullpath, err)
	}

	return nil
}

func (store *LevelDB3Store) ListDirectoryEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool,
	limit int) (entries []*filer.Entry, err error) {
	return store.ListDirectoryPrefixedEntries(ctx, fullpath, startFileName, inclusive, limit, "")
}

func (store *LevelDB3Store) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer.Entry, err error) {

	db, _, shortPath, err := store.findDB(fullpath, true)
	if err != nil {
		return nil, fmt.Errorf("findDB %s : %v", fullpath, err)
	}

	directoryPrefix := genDirectoryKeyPrefix(shortPath, prefix)
	lastFileStart := directoryPrefix
	if startFileName != "" {
		lastFileStart = genDirectoryKeyPrefix(shortPath, startFileName)
	}

	iter := db.NewIterator(&leveldb_util.Range{Start: lastFileStart}, nil)
	for iter.Next() {
		key := iter.Key()
		if !bytes.HasPrefix(key, directoryPrefix) {
			break
		}
		fileName := getNameFromKey(key)
		if fileName == "" {
			continue
		}
		if fileName == startFileName && !inclusive {
			continue
		}
		limit--
		if limit < 0 {
			break
		}
		entry := &filer.Entry{
			FullPath: weed_util.NewFullPath(string(fullpath), fileName),
		}

		// println("list", entry.FullPath, "chunks", len(entry.Chunks))
		if decodeErr := entry.DecodeAttributesAndChunks(weed_util.MaybeDecompressData(iter.Value())); decodeErr != nil {
			err = decodeErr
			glog.V(0).Infof("list %s : %v", entry.FullPath, err)
			break
		}
		entries = append(entries, entry)
	}
	iter.Release()

	return entries, err
}

func genKey(dirPath, fileName string) (key []byte) {
	key = hashToBytes(dirPath)
	key = append(key, []byte(fileName)...)
	return key
}

func genDirectoryKeyPrefix(fullpath weed_util.FullPath, startFileName string) (keyPrefix []byte) {
	keyPrefix = hashToBytes(string(fullpath))
	if len(startFileName) > 0 {
		keyPrefix = append(keyPrefix, []byte(startFileName)...)
	}
	return keyPrefix
}

func getNameFromKey(key []byte) string {

	return string(key[md5.Size:])

}

// hash directory
func hashToBytes(dir string) []byte {
	h := md5.New()
	io.WriteString(h, dir)
	b := h.Sum(nil)
	return b
}

func (store *LevelDB3Store) Shutdown() {
	for _, db := range store.dbs {
		db.Close()
	}
}
<commit_msg>remove println<commit_after>package leveldb

import (
	"bytes"
	"context"
	"crypto/md5"
	"fmt"
	"github.com/syndtr/goleveldb/leveldb"
	leveldb_errors "github.com/syndtr/goleveldb/leveldb/errors"
	"github.com/syndtr/goleveldb/leveldb/opt"
	leveldb_util "github.com/syndtr/goleveldb/leveldb/util"
	"io"
	"os"
	"strings"
	"sync"

	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	weed_util "github.com/chrislusf/seaweedfs/weed/util"
)

const (
	DEFAULT = "_main"
)

func init() {
	filer.Stores = append(filer.Stores, &LevelDB3Store{})
}

type LevelDB3Store struct {
	dir     string
	dbs     map[string]*leveldb.DB
	dbsLock sync.RWMutex
}

func (store *LevelDB3Store) GetName() string {
	return "leveldb3"
}

func (store *LevelDB3Store) Initialize(configuration weed_util.Configuration, prefix string) (err error) {
	dir := configuration.GetString(prefix + "dir")
	return store.initialize(dir)
}

func (store *LevelDB3Store) initialize(dir string) (err error) {
	glog.Infof("filer store leveldb3 dir: %s", dir)
	if err := weed_util.TestFolderWritable(dir); err != nil {
		return fmt.Errorf("Check Level Folder %s Writable: %s", dir, err)
	}
	store.dir = dir

	db, loadDbErr := store.loadDB(DEFAULT)
	if loadDbErr != nil {
		return loadDbErr
	}
	store.dbs = make(map[string]*leveldb.DB)
	store.dbs[DEFAULT] = db

	return
}

func (store *LevelDB3Store) loadDB(name string) (*leveldb.DB, error) {

	opts := &opt.Options{
		BlockCacheCapacity:            32 * 1024 * 1024, // default value is 8MiB
		WriteBuffer:                   16 * 1024 * 1024, // default value is 4MiB
		CompactionTableSizeMultiplier: 4,
	}
	if name != DEFAULT {
		opts = &opt.Options{
			BlockCacheCapacity:            4 * 1024 * 1024, // default value is 8MiB
			WriteBuffer:                   2 * 1024 * 1024, // default value is 4MiB
			CompactionTableSizeMultiplier: 4,
		}
	}

	dbFolder := fmt.Sprintf("%s/%s", store.dir, name)
	os.MkdirAll(dbFolder, 0755)
	db, dbErr := leveldb.OpenFile(dbFolder, opts)
	if leveldb_errors.IsCorrupted(dbErr) {
		db, dbErr = leveldb.RecoverFile(dbFolder, opts)
	}
	if dbErr != nil {
		glog.Errorf("filer store open dir %s: %v", dbFolder, dbErr)
		return nil, dbErr
	}
	return db, nil
}

func (store *LevelDB3Store) findDB(fullpath weed_util.FullPath, isForChildren bool) (*leveldb.DB, string, weed_util.FullPath, error) {

	store.dbsLock.RLock()

	defaultDB := store.dbs[DEFAULT]
	if !strings.HasPrefix(string(fullpath), "/buckets/") {
		store.dbsLock.RUnlock()
		return defaultDB, DEFAULT, fullpath, nil
	}

	// detect bucket
	bucketAndObjectKey := string(fullpath)[len("/buckets/"):]
	t := strings.Index(bucketAndObjectKey, "/")
	if t < 0 && !isForChildren {
		store.dbsLock.RUnlock()
		return defaultDB, DEFAULT, fullpath, nil
	}
	bucket := bucketAndObjectKey
	shortPath := weed_util.FullPath("/")
	if t > 0 {
		bucket = bucketAndObjectKey[:t]
		shortPath = weed_util.FullPath(bucketAndObjectKey[t:])
	}

	if db, found := store.dbs[bucket]; found {
		store.dbsLock.RUnlock()
		return db, bucket, shortPath, nil
	}

	store.dbsLock.RUnlock()
	// upgrade to write lock
	store.dbsLock.Lock()
	defer store.dbsLock.Unlock()

	// double check after getting the write lock
	if db, found := store.dbs[bucket]; found {
		return db, bucket, shortPath, nil
	}

	// create db
	db, err := store.loadDB(bucket)
	if err != nil {
		return nil, bucket, shortPath, err
	}
	store.dbs[bucket] = db

	return db, bucket, shortPath, nil
}

func (store *LevelDB3Store) BeginTransaction(ctx context.Context) (context.Context, error) {
	return ctx, nil
}
func (store *LevelDB3Store) CommitTransaction(ctx context.Context) error {
	return nil
}
func (store *LevelDB3Store) RollbackTransaction(ctx context.Context) error {
	return nil
}

func (store *LevelDB3Store) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) {

	db, _, shortPath, err := store.findDB(entry.FullPath, false)
	if err != nil {
		return fmt.Errorf("findDB %s : %v", entry.FullPath, err)
	}

	dir, name := shortPath.DirAndName()
	key := genKey(dir, name)

	value, err := entry.EncodeAttributesAndChunks()
	if err != nil {
		return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err)
	}

	if len(entry.Chunks) > 50 {
		value = weed_util.MaybeGzipData(value)
	}

	err = db.Put(key, value, nil)

	if err != nil {
		return fmt.Errorf("persisting %s : %v", entry.FullPath, err)
	}

	// println("saved", entry.FullPath, "chunks", len(entry.Chunks))

	return nil
}

func (store *LevelDB3Store) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) {

	return store.InsertEntry(ctx, entry)
}

func (store *LevelDB3Store) FindEntry(ctx context.Context, fullpath weed_util.FullPath) (entry *filer.Entry, err error) {

	db, _, shortPath, err := store.findDB(fullpath, false)
	if err != nil {
		return nil, fmt.Errorf("findDB %s : %v", fullpath, err)
	}

	dir, name := shortPath.DirAndName()
	key := genKey(dir, name)

	data, err := db.Get(key, nil)

	if err == leveldb.ErrNotFound {
		return nil, filer_pb.ErrNotFound
	}
	if err != nil {
		return nil, fmt.Errorf("get %s : %v", fullpath, err)
	}

	entry = &filer.Entry{
		FullPath: fullpath,
	}
	err = entry.DecodeAttributesAndChunks(weed_util.MaybeDecompressData(data))
	if err != nil {
		return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err)
	}

	// println("read", entry.FullPath, "chunks", len(entry.Chunks), "data", len(data), string(data))

	return entry, nil
}

func (store *LevelDB3Store) DeleteEntry(ctx context.Context, fullpath weed_util.FullPath) (err error) {

	db, _, shortPath, err := store.findDB(fullpath, false)
	if err != nil {
		return fmt.Errorf("findDB %s : %v", fullpath, err)
	}

	dir, name := shortPath.DirAndName()
	key := genKey(dir, name)

	err = db.Delete(key, nil)
	if err != nil {
		return fmt.Errorf("delete %s : %v", fullpath, err)
	}

	return nil
}

func (store *LevelDB3Store) DeleteFolderChildren(ctx context.Context, fullpath weed_util.FullPath) (err error) {

	db, bucket, shortPath, err := store.findDB(fullpath, true)
	if err != nil {
		return fmt.Errorf("findDB %s : %v", fullpath, err)
	}

	if bucket != DEFAULT && shortPath == "/" {
		db.Close()
		if bucket != "" { // just to make sure
			os.RemoveAll(store.dir + "/" + bucket)
		}
		return nil
	}

	directoryPrefix := genDirectoryKeyPrefix(shortPath, "")

	batch := new(leveldb.Batch)

	iter := db.NewIterator(&leveldb_util.Range{Start: directoryPrefix}, nil)
	for iter.Next() {
		key := iter.Key()
		if !bytes.HasPrefix(key, directoryPrefix) {
			break
		}
		fileName := getNameFromKey(key)
		if fileName == "" {
			continue
		}
		batch.Delete(append(directoryPrefix, []byte(fileName)...))
	}
	iter.Release()

	err = db.Write(batch, nil)

	if err != nil {
		return fmt.Errorf("delete %s : %v", fullpath, err)
	}

	return nil
}

func (store *LevelDB3Store) ListDirectoryEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool,
	limit int) (entries []*filer.Entry, err error) {
	return store.ListDirectoryPrefixedEntries(ctx, fullpath, startFileName, inclusive, limit, "")
}

func (store *LevelDB3Store) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer.Entry, err error) {

	db, _, shortPath, err := store.findDB(fullpath, true)
	if err != nil {
		return nil, fmt.Errorf("findDB %s : %v", fullpath, err)
	}

	directoryPrefix := genDirectoryKeyPrefix(shortPath, prefix)
	lastFileStart := directoryPrefix
	if startFileName != "" {
		lastFileStart = genDirectoryKeyPrefix(shortPath, startFileName)
	}

	iter := db.NewIterator(&leveldb_util.Range{Start: lastFileStart}, nil)
	for iter.Next() {
		key := iter.Key()
		if !bytes.HasPrefix(key, directoryPrefix) {
			break
		}
		fileName := getNameFromKey(key)
		if fileName == "" {
			continue
		}
		if fileName == startFileName && !inclusive {
			continue
		}
		limit--
		if limit < 0 {
			break
		}
		entry := &filer.Entry{
			FullPath: weed_util.NewFullPath(string(fullpath), fileName),
		}

		// println("list", entry.FullPath, "chunks", len(entry.Chunks))
		if decodeErr := entry.DecodeAttributesAndChunks(weed_util.MaybeDecompressData(iter.Value())); decodeErr != nil {
			err = decodeErr
			glog.V(0).Infof("list %s : %v", entry.FullPath, err)
			break
		}
		entries = append(entries, entry)
	}
	iter.Release()

	return entries, err
}

func genKey(dirPath, fileName string) (key []byte) {
	key = hashToBytes(dirPath)
	key = append(key, []byte(fileName)...)
	return key
}

func genDirectoryKeyPrefix(fullpath weed_util.FullPath, startFileName string) (keyPrefix []byte) {
	keyPrefix = hashToBytes(string(fullpath))
	if len(startFileName) > 0 {
		keyPrefix = append(keyPrefix, []byte(startFileName)...)
	}
	return keyPrefix
}

func getNameFromKey(key []byte) string {

	return string(key[md5.Size:])

}

// hash directory
func hashToBytes(dir string) []byte {
	h := md5.New()
	io.WriteString(h, dir)
	b := h.Sum(nil)
	return b
}

func (store *LevelDB3Store) Shutdown() {
	for _, db := range store.dbs {
		db.Close()
	}
}
<|endoftext|>


<commit_before>package memdb

import (
	"testing"
	"github.com/chrislusf/seaweedfs/weed/filer2"
)

func TestCreateAndFind(t *testing.T) {
	filer := filer2.NewFiler("")
	store := &MemDbStore{}
	store.Initialize(nil)
	filer.SetStore(store)
	filer.DisableDirectoryCache()

	fullpath := filer2.FullPath("/home/chris/this/is/one/file1.jpg")

	entry1 := &filer2.Entry{
		FullPath: fullpath,
		Attr: filer2.Attr{
			Mode: 0440,
			Uid:  1234,
			Gid:  5678,
		},
	}

	if err := filer.CreateEntry(entry1); err != nil {
		t.Errorf("create entry %v: %v", entry1.FullPath, err)
		return
	}

	found, entry, err := filer.FindEntry(fullpath)

	if err != nil {
		t.Errorf("find entry: %v", err)
		return
	}

	if !found {
		t.Errorf("Failed to find newly created file")
		return
	}

	if entry.FullPath != entry1.FullPath {
		t.Errorf("find wrong entry: %v", entry.FullPath)
		return
	}

}

func TestCreateFileAndList(t *testing.T) {
	filer := filer2.NewFiler("")
	store := &MemDbStore{}
	store.Initialize(nil)
	filer.SetStore(store)
	filer.DisableDirectoryCache()

	entry1 := &filer2.Entry{
		FullPath: filer2.FullPath("/home/chris/this/is/one/file1.jpg"),
		Attr: filer2.Attr{
			Mode: 0440,
			Uid:  1234,
			Gid:  5678,
		},
	}

	entry2 := &filer2.Entry{
		FullPath: filer2.FullPath("/home/chris/this/is/one/file2.jpg"),
		Attr: filer2.Attr{
			Mode: 0440,
			Uid:  1234,
			Gid:  5678,
		},
	}

	filer.CreateEntry(entry1)
	filer.CreateEntry(entry2)

	// checking the 2 files
	entries, err := filer.ListDirectoryEntries(filer2.FullPath("/home/chris/this/is/one/"), "", false, 100)

	if err != nil {
		t.Errorf("list entries: %v", err)
		return
	}

	if len(entries) != 2 {
		t.Errorf("list entries count: %v", len(entries))
		return
	}

	if entries[0].FullPath != entry1.FullPath {
		t.Errorf("find wrong entry 1: %v", entries[0].FullPath)
		return
	}

	if entries[1].FullPath != entry2.FullPath {
		t.Errorf("find wrong entry 2: %v", entries[1].FullPath)
		return
	}

	// checking the offset
	entries, err = filer.ListDirectoryEntries(filer2.FullPath("/home/chris/this/is/one/"), "file1.jpg", false, 100)
	if len(entries) != 1 {
		t.Errorf("list entries count: %v", len(entries))
		return
	}

	// checking one upper directory
	entries, _ = filer.ListDirectoryEntries(filer2.FullPath("/home/chris/this/is"), "", false, 100)
	if len(entries) != 1 {
		t.Errorf("list entries count: %v", len(entries))
		return
	}

	// checking root directory
	entries, _ = filer.ListDirectoryEntries(filer2.FullPath("/"), "", false, 100)
	if len(entries) != 1 {
		t.Errorf("list entries count: %v", len(entries))
		return
	}

	// add file3
	file3Path := filer2.FullPath("/home/chris/this/is/file3.jpg")
	entry3 := &filer2.Entry{
		FullPath: file3Path,
		Attr: filer2.Attr{
			Mode: 0440,
			Uid:  1234,
			Gid:  5678,
		},
	}
	filer.CreateEntry(entry3)

	// checking one upper directory
	entries, _ = filer.ListDirectoryEntries(filer2.FullPath("/home/chris/this/is"), "", false, 100)
	if len(entries) != 2 {
		t.Errorf("list entries count: %v", len(entries))
		return
	}

	// delete file and count
	filer.DeleteEntry(file3Path)
	entries, _ = filer.ListDirectoryEntries(filer2.FullPath("/home/chris/this/is"), "", false, 100)
	if len(entries) != 1 {
		t.Errorf("list entries count: %v", len(entries))
		return
	}

}
<commit_msg>fix test error<commit_after>package memdb

import (
	"testing"
	"github.com/chrislusf/seaweedfs/weed/filer2"
)

func TestCreateAndFind(t *testing.T) {
	filer := filer2.NewFiler("")
	store := &MemDbStore{}
	store.Initialize(nil)
	filer.SetStore(store)
	filer.DisableDirectoryCache()

	fullpath := filer2.FullPath("/home/chris/this/is/one/file1.jpg")

	entry1 := &filer2.Entry{
		FullPath: fullpath,
		Attr: filer2.Attr{
			Mode: 0440,
			Uid:  1234,
			Gid:  5678,
		},
	}

	if err := filer.CreateEntry(entry1); err != nil {
		t.Errorf("create entry %v: %v", entry1.FullPath, err)
		return
	}

	entry, err := filer.FindEntry(fullpath)

	if err != nil {
		t.Errorf("find entry: %v", err)
		return
	}

	if entry.FullPath != entry1.FullPath {
		t.Errorf("find wrong entry: %v", entry.FullPath)
		return
	}

}

func TestCreateFileAndList(t *testing.T) {
	filer := filer2.NewFiler("")
	store := &MemDbStore{}
	store.Initialize(nil)
	filer.SetStore(store)
	filer.DisableDirectoryCache()

	entry1 := &filer2.Entry{
		FullPath: filer2.FullPath("/home/chris/this/is/one/file1.jpg"),
		Attr: filer2.Attr{
			Mode: 0440,
			Uid:  1234,
			Gid:  5678,
		},
	}

	entry2 := &filer2.Entry{
		FullPath: filer2.FullPath("/home/chris/this/is/one/file2.jpg"),
		Attr: filer2.Attr{
			Mode: 0440,
			Uid:  1234,
			Gid:  5678,
		},
	}

	filer.CreateEntry(entry1)
	filer.CreateEntry(entry2)

	// checking the 2 files
	entries, err := filer.ListDirectoryEntries(filer2.FullPath("/home/chris/this/is/one/"), "", false, 100)

	if err != nil {
		t.Errorf("list entries: %v", err)
		return
	}

	if len(entries) != 2 {
		t.Errorf("list entries count: %v", len(entries))
		return
	}

	if entries[0].FullPath != entry1.FullPath {
		t.Errorf("find wrong entry 1: %v", entries[0].FullPath)
		return
	}

	if entries[1].FullPath != entry2.FullPath {
		t.Errorf("find wrong entry 2: %v", entries[1].FullPath)
		return
	}

	// checking the offset
	entries, err = filer.ListDirectoryEntries(filer2.FullPath("/home/chris/this/is/one/"), "file1.jpg", false, 100)
	if len(entries) != 1 {
		t.Errorf("list entries count: %v", len(entries))
		return
	}

	// checking one upper directory
	entries, _ = filer.ListDirectoryEntries(filer2.FullPath("/home/chris/this/is"), "", false, 100)
	if len(entries) != 1 {
		t.Errorf("list entries count: %v", len(entries))
		return
	}

	// checking root directory
	entries, _ = filer.ListDirectoryEntries(filer2.FullPath("/"), "", false, 100)
	if len(entries) != 1 {
		t.Errorf("list entries count: %v", len(entries))
		return
	}

	// add file3
	file3Path := filer2.FullPath("/home/chris/this/is/file3.jpg")
	entry3 := &filer2.Entry{
		FullPath: file3Path,
		Attr: filer2.Attr{
			Mode: 0440,
			Uid:  1234,
			Gid:  5678,
		},
	}
	filer.CreateEntry(entry3)

	// checking one upper directory
	entries, _ = filer.ListDirectoryEntries(filer2.FullPath("/home/chris/this/is"), "", false, 100)
	if len(entries) != 2 {
		t.Errorf("list entries count: %v", len(entries))
		return
	}

	// delete file and count
	filer.DeleteEntry(file3Path)
	entries, _ = filer.ListDirectoryEntries(filer2.FullPath("/home/chris/this/is"), "", false, 100)
	if len(entries) != 1 {
		t.Errorf("list entries count: %v", len(entries))
		return
	}

}
<|endoftext|>


<commit_before>package meta_cache

import (
	"context"
	"os"
	"sync"

	"github.com/chrislusf/seaweedfs/weed/filer2"
	"github.com/chrislusf/seaweedfs/weed/filer2/leveldb"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util"
	"github.com/chrislusf/seaweedfs/weed/util/bounded_tree"
)

// need to have logic similar to FilerStoreWrapper
// e.g. fill fileId field for chunks

type MetaCache struct {
	actualStore filer2.FilerStore
	sync.RWMutex
	visitedBoundary *bounded_tree.BoundedTree
}

func NewMetaCache(dbFolder string) *MetaCache {
	return &MetaCache{
		actualStore:     openMetaStore(dbFolder),
		visitedBoundary: bounded_tree.NewBoundedTree(),
	}
}

func openMetaStore(dbFolder string) filer2.FilerStore {

	os.RemoveAll(dbFolder)
	os.MkdirAll(dbFolder, 0755)

	store := &leveldb.LevelDBStore{}
	config := &cacheConfig{
		dir: dbFolder,
	}

	if err := store.Initialize(config, ""); err != nil {
		glog.Fatalf("Failed to initialize metadata cache store for %s: %+v", store.GetName(), err)
	}

	return store

}

func (mc *MetaCache) InsertEntry(ctx context.Context, entry *filer2.Entry) error {
	mc.Lock()
	defer mc.Unlock()
	filer_pb.BeforeEntrySerialization(entry.Chunks)
	return mc.actualStore.InsertEntry(ctx, entry)
}

func (mc *MetaCache) AtomicUpdateEntry(ctx context.Context, oldPath util.FullPath, newEntry *filer2.Entry) error {
	mc.Lock()
	defer mc.Unlock()

	oldDir, _ := oldPath.DirAndName()
	if mc.visitedBoundary.HasVisited(util.FullPath(oldDir)) {
		if oldPath != "" {
			if oldPath == newEntry.FullPath {
				// skip the unnecessary deletion
				// leave the update to the following InsertEntry operation
			} else {
				if err := mc.actualStore.DeleteEntry(ctx, oldPath); err != nil {
					return err
				}
			}
		}
	} else {
		// println("unknown old directory:", oldDir)
	}

	if newEntry != nil {
		newDir, _ := newEntry.DirAndName()
		if mc.visitedBoundary.HasVisited(util.FullPath(newDir)) {
			if err := mc.actualStore.InsertEntry(ctx, newEntry); err != nil {
				return err
			}
		}
	}
	return nil
}

func (mc *MetaCache) UpdateEntry(ctx context.Context, entry *filer2.Entry) error {
	mc.Lock()
	defer mc.Unlock()
	filer_pb.BeforeEntrySerialization(entry.Chunks)
	return mc.actualStore.UpdateEntry(ctx, entry)
}

func (mc *MetaCache) FindEntry(ctx context.Context, fp util.FullPath) (entry *filer2.Entry, err error) {
	mc.RLock()
	defer mc.RUnlock()
	entry, err = mc.actualStore.FindEntry(ctx, fp)
	if err != nil {
		return nil, err
	}
	filer_pb.AfterEntryDeserialization(entry.Chunks)
	return
}

func (mc *MetaCache) DeleteEntry(ctx context.Context, fp util.FullPath) (err error) {
	mc.Lock()
	defer mc.Unlock()
	return mc.actualStore.DeleteEntry(ctx, fp)
}

func (mc *MetaCache) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int) ([]*filer2.Entry, error) {
	mc.RLock()
	defer mc.RUnlock()

	entries, err := mc.actualStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit)
	if err != nil {
		return nil, err
	}
	for _, entry := range entries {
		filer_pb.AfterEntryDeserialization(entry.Chunks)
	}
	return entries, err
}

func (mc *MetaCache) Shutdown() {
	mc.Lock()
	defer mc.Unlock()
	mc.actualStore.Shutdown()
}
<commit_msg>prevent nil<commit_after>package meta_cache

import (
	"context"
	"os"
	"sync"

	"github.com/chrislusf/seaweedfs/weed/filer2"
	"github.com/chrislusf/seaweedfs/weed/filer2/leveldb"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util"
	"github.com/chrislusf/seaweedfs/weed/util/bounded_tree"
)

// need to have logic similar to FilerStoreWrapper
// e.g. fill fileId field for chunks

type MetaCache struct {
	actualStore filer2.FilerStore
	sync.RWMutex
	visitedBoundary *bounded_tree.BoundedTree
}

func NewMetaCache(dbFolder string) *MetaCache {
	return &MetaCache{
		actualStore:     openMetaStore(dbFolder),
		visitedBoundary: bounded_tree.NewBoundedTree(),
	}
}

func openMetaStore(dbFolder string) filer2.FilerStore {

	os.RemoveAll(dbFolder)
	os.MkdirAll(dbFolder, 0755)

	store := &leveldb.LevelDBStore{}
	config := &cacheConfig{
		dir: dbFolder,
	}

	if err := store.Initialize(config, ""); err != nil {
		glog.Fatalf("Failed to initialize metadata cache store for %s: %+v", store.GetName(), err)
	}

	return store

}

func (mc *MetaCache) InsertEntry(ctx context.Context, entry *filer2.Entry) error {
	mc.Lock()
	defer mc.Unlock()
	filer_pb.BeforeEntrySerialization(entry.Chunks)
	return mc.actualStore.InsertEntry(ctx, entry)
}

func (mc *MetaCache) AtomicUpdateEntry(ctx context.Context, oldPath util.FullPath, newEntry *filer2.Entry) error {
	mc.Lock()
	defer mc.Unlock()

	oldDir, _ := oldPath.DirAndName()
	if mc.visitedBoundary.HasVisited(util.FullPath(oldDir)) {
		if oldPath != "" {
			if newEntry != nil && oldPath == newEntry.FullPath {
				// skip the unnecessary deletion
				// leave the update to the following InsertEntry operation
			} else {
				if err := mc.actualStore.DeleteEntry(ctx, oldPath); err != nil {
					return err
				}
			}
		}
	} else {
		// println("unknown old directory:", oldDir)
	}

	if newEntry != nil {
		newDir, _ := newEntry.DirAndName()
		if mc.visitedBoundary.HasVisited(util.FullPath(newDir)) {
			if err := mc.actualStore.InsertEntry(ctx, newEntry); err != nil {
				return err
			}
		}
	}
	return nil
}

func (mc *MetaCache) UpdateEntry(ctx context.Context, entry *filer2.Entry) error {
	mc.Lock()
	defer mc.Unlock()
	filer_pb.BeforeEntrySerialization(entry.Chunks)
	return mc.actualStore.UpdateEntry(ctx, entry)
}

func (mc *MetaCache) FindEntry(ctx context.Context, fp util.FullPath) (entry *filer2.Entry, err error) {
	mc.RLock()
	defer mc.RUnlock()
	entry, err = mc.actualStore.FindEntry(ctx, fp)
	if err != nil {
		return nil, err
	}
	filer_pb.AfterEntryDeserialization(entry.Chunks)
	return
}

func (mc *MetaCache) DeleteEntry(ctx context.Context, fp util.FullPath) (err error) {
	mc.Lock()
	defer mc.Unlock()
	return mc.actualStore.DeleteEntry(ctx, fp)
}

func (mc *MetaCache) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int) ([]*filer2.Entry, error) {
	mc.RLock()
	defer mc.RUnlock()

	entries, err := mc.actualStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit)
	if err != nil {
		return nil, err
	}
	for _, entry := range entries {
		filer_pb.AfterEntryDeserialization(entry.Chunks)
	}
	return entries, err
}

func (mc *MetaCache) Shutdown() {
	mc.Lock()
	defer mc.Unlock()
	mc.actualStore.Shutdown()
}
<|endoftext|>


<commit_before>package about

import (
	"encoding/json"
	"fmt"
	"os"

	"github.com/ncw/rclone/cmd"
	"github.com/ncw/rclone/fs"
	"github.com/pkg/errors"
	"github.com/spf13/cobra"
)

var (
	jsonOutput bool
	fullOutput bool
)

func init() {
	cmd.Root.AddCommand(commandDefinition)
	commandDefinition.Flags().BoolVar(&jsonOutput, "json", false, "Format output as JSON")
	commandDefinition.Flags().BoolVar(&fullOutput, "full", false, "Full numbers instead of SI units")
}

// printValue formats uv to be output
func printValue(what string, uv *int64) {
	what += ":"
	if uv == nil {
		return
	}
	var val string
	if fullOutput {
		val = fmt.Sprintf("%d", *uv)
	} else {
		val = fs.SizeSuffix(*uv).String()
	}
	fmt.Printf("%-9s%v\n", what, val)
}

var commandDefinition = &cobra.Command{
	Use:   "about remote:",
	Short: `Get quota information from the remote.`,
	Long: `
Get quota information from the remote, like bytes used/free/quota and bytes
used in the trash. Not supported by all remotes.

This will print to stdout something like this:

    Total:   17G
    Used:    7.444G
    Free:    1.315G
    Trashed: 100.000M
    Other:   8.241G

Where the fields are:

  * Total: total size available.
  * Used: total size used
  * Free: total amount this user could upload.
  * Trashed: total amount in the trash
  * Other: total amount in other storage (eg Gmail, Google Photos)
  * Objects: total number of objects in the storage

Note that not all the backends provide all the fields - they will be
missing if they are not known for that backend.  Where it is known
that the value is unlimited the value will also be omitted.

Use the --full flag to see the numbers written out in full, eg

    Total:   18253611008
    Used:    7993453766
    Free:    1411001220
    Trashed: 104857602
    Other:   8849156022

Use the --json flag for a computer readable output, eg

    {
        "total": 18253611008,
        "used": 7993453766,
        "trashed": 104857602,
        "other": 8849156022,
        "free": 1411001220
    }
`,
	Run: func(command *cobra.Command, args []string) {
		cmd.CheckArgs(1, 1, command, args)
		f := cmd.NewFsSrc(args)
		cmd.Run(false, false, command, func() error {
			doAbout := f.Features().About
			if doAbout == nil {
				return errors.Errorf("%v doesn't support about", f)
			}
			u, err := doAbout()
			if err != nil {
				return errors.Wrap(err, "About call failed")
			}
			if jsonOutput {
				out := json.NewEncoder(os.Stdout)
				out.SetIndent("", "\t")
				return out.Encode(u)
			}
			printValue("Total", u.Total)
			printValue("Used", u.Used)
			printValue("Free", u.Free)
			printValue("Trashed", u.Trashed)
			printValue("Other", u.Other)
			printValue("Objects", u.Objects)
			return nil
		})
	},
}
<commit_msg>about: fix crash if backend returns a nil usage<commit_after>package about

import (
	"encoding/json"
	"fmt"
	"os"

	"github.com/ncw/rclone/cmd"
	"github.com/ncw/rclone/fs"
	"github.com/pkg/errors"
	"github.com/spf13/cobra"
)

var (
	jsonOutput bool
	fullOutput bool
)

func init() {
	cmd.Root.AddCommand(commandDefinition)
	commandDefinition.Flags().BoolVar(&jsonOutput, "json", false, "Format output as JSON")
	commandDefinition.Flags().BoolVar(&fullOutput, "full", false, "Full numbers instead of SI units")
}

// printValue formats uv to be output
func printValue(what string, uv *int64) {
	what += ":"
	if uv == nil {
		return
	}
	var val string
	if fullOutput {
		val = fmt.Sprintf("%d", *uv)
	} else {
		val = fs.SizeSuffix(*uv).String()
	}
	fmt.Printf("%-9s%v\n", what, val)
}

var commandDefinition = &cobra.Command{
	Use:   "about remote:",
	Short: `Get quota information from the remote.`,
	Long: `
Get quota information from the remote, like bytes used/free/quota and bytes
used in the trash. Not supported by all remotes.

This will print to stdout something like this:

    Total:   17G
    Used:    7.444G
    Free:    1.315G
    Trashed: 100.000M
    Other:   8.241G

Where the fields are:

  * Total: total size available.
  * Used: total size used
  * Free: total amount this user could upload.
  * Trashed: total amount in the trash
  * Other: total amount in other storage (eg Gmail, Google Photos)
  * Objects: total number of objects in the storage

Note that not all the backends provide all the fields - they will be
missing if they are not known for that backend.  Where it is known
that the value is unlimited the value will also be omitted.

Use the --full flag to see the numbers written out in full, eg

    Total:   18253611008
    Used:    7993453766
    Free:    1411001220
    Trashed: 104857602
    Other:   8849156022

Use the --json flag for a computer readable output, eg

    {
        "total": 18253611008,
        "used": 7993453766,
        "trashed": 104857602,
        "other": 8849156022,
        "free": 1411001220
    }
`,
	Run: func(command *cobra.Command, args []string) {
		cmd.CheckArgs(1, 1, command, args)
		f := cmd.NewFsSrc(args)
		cmd.Run(false, false, command, func() error {
			doAbout := f.Features().About
			if doAbout == nil {
				return errors.Errorf("%v doesn't support about", f)
			}
			u, err := doAbout()
			if err != nil {
				return errors.Wrap(err, "About call failed")
			}
			if u == nil {
				return errors.New("nil usage returned")
			}
			if jsonOutput {
				out := json.NewEncoder(os.Stdout)
				out.SetIndent("", "\t")
				return out.Encode(u)
			}
			printValue("Total", u.Total)
			printValue("Used", u.Used)
			printValue("Free", u.Free)
			printValue("Trashed", u.Trashed)
			printValue("Other", u.Other)
			printValue("Objects", u.Objects)
			return nil
		})
	},
}
<|endoftext|>


<commit_before><commit_msg>moved WAYF specifics to wayfhybrid<commit_after><|endoftext|>


<commit_before>package main

//go:generate gocat -o licenses.go xzLicense:github.com/uli-go/xz/LICENSE pflagLicense:github.com/ogier/pflag/LICENSE

import (
	"bytes"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"strings"
	"text/template"

	"github.com/ogier/pflag"
)

const (
	lzmaExt  = ".lzma"
	usageStr = `Usage: lzmago [OPTION]... [FILE]...
Compress or uncompress FILEs in the .lzma format (by default, compress FILES
in place).

  -c, --stdout      write to standard output and don't delete input files
  -d, --decompress  force decompression
  -f, --force       force overwrite of output file and compress links
  -h, --help        give this help
  -k, --keep        keep (don't delete) input files
  -L, --license     display software license
  -q, --quiet       suppress all warnings
  -v, --verbose     verbose mode
  -V, --version     display version string
  -z, --compress    force compression
  -0 ... -9         compression preset; default is 6

With no file, or when FILE is -, read standard input.

Report bugs using <https://github.com/uli-go/xz/issues>.
`
)

type Preset int

const defaultPreset Preset = 6

func (p *Preset) filterArg(arg string) string {
	if len(arg) < 2 || arg[0] != '-' || arg[1] == '-' {
		return arg
	}
	buf := new(bytes.Buffer)
	buf.Grow(len(arg))
	for _, c := range arg {
		if '0' <= c && c <= '9' {
			*p = Preset(c - '0')
			continue
		}
		buf.WriteRune(c)
	}
	return buf.String()
}

func (p *Preset) filter() {
	args := make([]string, 1, len(os.Args))
	args[0] = os.Args[0]
	for i, arg := range os.Args[1:] {
		if arg == "--" {
			args = append(args, os.Args[1+i:]...)
			break

		}
		arg = p.filterArg(arg)
		if arg != "-" {
			args = append(args, arg)
		}
	}
	os.Args = args
}

func usage(w io.Writer) {
	fmt.Fprint(w, usageStr)
}

func licenses(w io.Writer) {
	out := `
github.com/uli-go/xz -- xz for Go
=================================

{{.xz}}

pflag -- Posix flag package
===========================

{{.pflag}}
`
	out = strings.TrimLeft(out, " \n")
	tmpl, err := template.New("licenses").Parse(out)
	if err != nil {
		log.Panicf("error %s parsing licenses template", err)
	}
	lmap := map[string]string{
		"xz":    strings.TrimSpace(xzLicense),
		"pflag": strings.TrimSpace(pflagLicense),
	}
	if err = tmpl.Execute(w, lmap); err != nil {
		log.Fatalf("error %s writing licenses template", err)
	}
}

func main() {
	// setup logger
	cmdName := filepath.Base(os.Args[0])
	log.SetPrefix(fmt.Sprintf("%s: ", cmdName))
	log.SetFlags(0)

	// initialize flags
	pflag.CommandLine = pflag.NewFlagSet(cmdName, pflag.ExitOnError)
	pflag.SetInterspersed(true)
	pflag.Usage = func() { usage(os.Stderr); os.Exit(1) }
	var (
		help       = pflag.BoolP("help", "h", false, "")
		stdout     = pflag.BoolP("stdout", "c", false, "")
		decompress = pflag.BoolP("decompress", "d", false, "")
		force      = pflag.BoolP("force", "f", false, "")
		keep       = pflag.BoolP("keep", "k", false, "")
		license    = pflag.BoolP("license", "L", false, "")
		preset     = defaultPreset
	)

	// process arguments
	preset.filter()
	log.Printf("filtered args %v", os.Args)
	pflag.Parse()

	if pflag.NArg() == 0 {
		log.Fatal("for help, type lzmago -h")
	}

	if *help {
		usage(os.Stdout)
		os.Exit(0)
	}
	if *license {
		licenses(os.Stdout)
		os.Exit(0)
	}

	log.Printf("decompress %t", *decompress)
	log.Printf("force %t", *force)
	log.Printf("keep %t", *keep)
	log.Printf("keep %t", *keep)
	log.Printf("preset %d", preset)
	log.Printf("stdout %t", *stdout)
}
<commit_msg>lzmago/main.go: fixed location of argument number check<commit_after>package main

//go:generate gocat -o licenses.go xzLicense:github.com/uli-go/xz/LICENSE pflagLicense:github.com/ogier/pflag/LICENSE

import (
	"bytes"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"strings"
	"text/template"

	"github.com/ogier/pflag"
)

const (
	lzmaExt  = ".lzma"
	usageStr = `Usage: lzmago [OPTION]... [FILE]...
Compress or uncompress FILEs in the .lzma format (by default, compress FILES
in place).

  -c, --stdout      write to standard output and don't delete input files
  -d, --decompress  force decompression
  -f, --force       force overwrite of output file and compress links
  -h, --help        give this help
  -k, --keep        keep (don't delete) input files
  -L, --license     display software license
  -q, --quiet       suppress all warnings
  -v, --verbose     verbose mode
  -V, --version     display version string
  -z, --compress    force compression
  -0 ... -9         compression preset; default is 6

With no file, or when FILE is -, read standard input.

Report bugs using <https://github.com/uli-go/xz/issues>.
`
)

type Preset int

const defaultPreset Preset = 6

func (p *Preset) filterArg(arg string) string {
	if len(arg) < 2 || arg[0] != '-' || arg[1] == '-' {
		return arg
	}
	buf := new(bytes.Buffer)
	buf.Grow(len(arg))
	for _, c := range arg {
		if '0' <= c && c <= '9' {
			*p = Preset(c - '0')
			continue
		}
		buf.WriteRune(c)
	}
	return buf.String()
}

func (p *Preset) filter() {
	args := make([]string, 1, len(os.Args))
	args[0] = os.Args[0]
	for i, arg := range os.Args[1:] {
		if arg == "--" {
			args = append(args, os.Args[1+i:]...)
			break

		}
		arg = p.filterArg(arg)
		if arg != "-" {
			args = append(args, arg)
		}
	}
	os.Args = args
}

func usage(w io.Writer) {
	fmt.Fprint(w, usageStr)
}

func licenses(w io.Writer) {
	out := `
github.com/uli-go/xz -- xz for Go
=================================

{{.xz}}

pflag -- Posix flag package
===========================

{{.pflag}}
`
	out = strings.TrimLeft(out, " \n")
	tmpl, err := template.New("licenses").Parse(out)
	if err != nil {
		log.Panicf("error %s parsing licenses template", err)
	}
	lmap := map[string]string{
		"xz":    strings.TrimSpace(xzLicense),
		"pflag": strings.TrimSpace(pflagLicense),
	}
	if err = tmpl.Execute(w, lmap); err != nil {
		log.Fatalf("error %s writing licenses template", err)
	}
}

func main() {
	// setup logger
	cmdName := filepath.Base(os.Args[0])
	log.SetPrefix(fmt.Sprintf("%s: ", cmdName))
	log.SetFlags(0)

	// initialize flags
	pflag.CommandLine = pflag.NewFlagSet(cmdName, pflag.ExitOnError)
	pflag.SetInterspersed(true)
	pflag.Usage = func() { usage(os.Stderr); os.Exit(1) }
	var (
		help       = pflag.BoolP("help", "h", false, "")
		stdout     = pflag.BoolP("stdout", "c", false, "")
		decompress = pflag.BoolP("decompress", "d", false, "")
		force      = pflag.BoolP("force", "f", false, "")
		keep       = pflag.BoolP("keep", "k", false, "")
		license    = pflag.BoolP("license", "L", false, "")
		preset     = defaultPreset
	)

	// process arguments
	preset.filter()
	log.Printf("filtered args %v", os.Args)
	pflag.Parse()

	if *help {
		usage(os.Stdout)
		os.Exit(0)
	}
	if *license {
		licenses(os.Stdout)
		os.Exit(0)
	}
	if pflag.NArg() == 0 {
		log.Fatal("for help, type lzmago -h")
	}

	log.Printf("decompress %t", *decompress)
	log.Printf("force %t", *force)
	log.Printf("keep %t", *keep)
	log.Printf("keep %t", *keep)
	log.Printf("preset %d", preset)
	log.Printf("stdout %t", *stdout)
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"io"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/codegangsta/cli"
	"github.com/itchyny/maze"
	"github.com/mattn/go-isatty"
	"github.com/nsf/termbox-go"
)

// Config is the command configuration
type Config struct {
	Width       int
	Height      int
	Start       *maze.Point
	Goal        *maze.Point
	Interactive bool
	Image       bool
	Scale       int
	Solution    bool
	Format      *maze.Format
	Seed        int64
	Output      io.Writer
}

func makeConfig(ctx *cli.Context) (*Config, []error) {
	var errs []error

	if ctx.GlobalBool("help") {
		errs = append(errs, errors.New(""))
		return nil, errs
	}

	termWidth, termHeight := termbox.Size()

	width := ctx.GlobalInt("width")
	if width <= 0 {
		width = (termWidth - 4) / 4
	}

	height := ctx.GlobalInt("height")
	if height <= 0 {
		height = (termHeight - 5) / 2
	}

	start := &maze.Point{0, 0}
	starts := strings.Split(ctx.GlobalString("start"), ",")
	if len(starts) > 0 {
		if value, err := strconv.Atoi(starts[0]); err == nil {
			if 0 <= value && value < height {
				start.X = value
			}
		}
	}
	if len(starts) > 1 {
		if value, err := strconv.Atoi(starts[1]); err == nil {
			if 0 <= value && value < width {
				start.Y = value
			}
		}
	}

	goal := &maze.Point{height - 1, width - 1}
	goals := strings.Split(ctx.GlobalString("goal"), ",")
	if len(goals) > 0 {
		if value, err := strconv.Atoi(goals[0]); err == nil {
			if 0 <= value && value < height {
				goal.X = value
			}
		}
	}
	if len(goals) > 1 {
		if value, err := strconv.Atoi(goals[1]); err == nil {
			if 0 <= value && value < width {
				goal.Y = value
			}
		}
	}

	interactive := ctx.GlobalBool("interactive")

	solution := ctx.GlobalBool("solution")

	format := maze.Default
	if ctx.GlobalString("format") == "color" {
		format = maze.Color
	}

	output := ctx.App.Writer
	outfile := ctx.GlobalString("output")
	if outfile != "" {
		file, err := os.Create(outfile)
		if err != nil {
			errs = append(errs, errors.New("cannot create the output file: "+outfile))
		} else {
			output = file
		}
	}

	image := ctx.GlobalBool("image")
	if image {
		if file, ok := output.(*os.File); ok && isatty.IsTerminal(file.Fd()) {
			errs = append(errs, errors.New("cannot write binary data into the terminal\nuse -output flag"))
		}
	}

	scale := ctx.GlobalInt("scale")

	seed := int64(ctx.GlobalInt("seed"))
	if !ctx.IsSet("seed") {
		seed = time.Now().UnixNano()
	}

	if len(errs) > 0 {
		return nil, errs
	}

	return &Config{
		Width:       width,
		Height:      height,
		Start:       start,
		Goal:        goal,
		Interactive: interactive,
		Image:       image,
		Scale:       scale,
		Solution:    solution,
		Format:      format,
		Seed:        seed,
		Output:      output,
	}, nil
}
<commit_msg>fix go vet<commit_after>package main

import (
	"errors"
	"io"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/codegangsta/cli"
	"github.com/itchyny/maze"
	"github.com/mattn/go-isatty"
	"github.com/nsf/termbox-go"
)

// Config is the command configuration
type Config struct {
	Width       int
	Height      int
	Start       *maze.Point
	Goal        *maze.Point
	Interactive bool
	Image       bool
	Scale       int
	Solution    bool
	Format      *maze.Format
	Seed        int64
	Output      io.Writer
}

func makeConfig(ctx *cli.Context) (*Config, []error) {
	var errs []error

	if ctx.GlobalBool("help") {
		errs = append(errs, errors.New(""))
		return nil, errs
	}

	termWidth, termHeight := termbox.Size()

	width := ctx.GlobalInt("width")
	if width <= 0 {
		width = (termWidth - 4) / 4
	}

	height := ctx.GlobalInt("height")
	if height <= 0 {
		height = (termHeight - 5) / 2
	}

	start := &maze.Point{X: 0, Y: 0}
	starts := strings.Split(ctx.GlobalString("start"), ",")
	if len(starts) > 0 {
		if value, err := strconv.Atoi(starts[0]); err == nil {
			if 0 <= value && value < height {
				start.X = value
			}
		}
	}
	if len(starts) > 1 {
		if value, err := strconv.Atoi(starts[1]); err == nil {
			if 0 <= value && value < width {
				start.Y = value
			}
		}
	}

	goal := &maze.Point{X: height - 1, Y: width - 1}
	goals := strings.Split(ctx.GlobalString("goal"), ",")
	if len(goals) > 0 {
		if value, err := strconv.Atoi(goals[0]); err == nil {
			if 0 <= value && value < height {
				goal.X = value
			}
		}
	}
	if len(goals) > 1 {
		if value, err := strconv.Atoi(goals[1]); err == nil {
			if 0 <= value && value < width {
				goal.Y = value
			}
		}
	}

	interactive := ctx.GlobalBool("interactive")

	solution := ctx.GlobalBool("solution")

	format := maze.Default
	if ctx.GlobalString("format") == "color" {
		format = maze.Color
	}

	output := ctx.App.Writer
	outfile := ctx.GlobalString("output")
	if outfile != "" {
		file, err := os.Create(outfile)
		if err != nil {
			errs = append(errs, errors.New("cannot create the output file: "+outfile))
		} else {
			output = file
		}
	}

	image := ctx.GlobalBool("image")
	if image {
		if file, ok := output.(*os.File); ok && isatty.IsTerminal(file.Fd()) {
			errs = append(errs, errors.New("cannot write binary data into the terminal\nuse -output flag"))
		}
	}

	scale := ctx.GlobalInt("scale")

	seed := int64(ctx.GlobalInt("seed"))
	if !ctx.IsSet("seed") {
		seed = time.Now().UnixNano()
	}

	if len(errs) > 0 {
		return nil, errs
	}

	return &Config{
		Width:       width,
		Height:      height,
		Start:       start,
		Goal:        goal,
		Interactive: interactive,
		Image:       image,
		Scale:       scale,
		Solution:    solution,
		Format:      format,
		Seed:        seed,
		Output:      output,
	}, nil
}
<|endoftext|>


<commit_before>//
// Copyright (c) 2014 The pblcache Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package spc

import (
	"github.com/lpabon/goioworkload/spc1"
	"github.com/pblcache/pblcache/tests"
	"strings"
	"testing"
	"time"
)

func TestIoMeterColletc(t *testing.T) {

	spc1info := &spc1.Spc1Io{
		Asu:    1,
		Blocks: 1,
		Stream: 2,
		Offset: 123456,
		When:   time.Second * 2,
	}

	stat := &IoStats{
		Io:      spc1info,
		Start:   time.Now(),
		Latency: time.Millisecond * 3,
	}

	// Inster first data point
	meter := &IoMeter{}

	meter.Collect(stat)
	tests.Assert(t, meter.blocks == 1)
	tests.Assert(t, meter.ios == 1)
	tests.Assert(t, meter.latency.MeanTimeUsecs() == 3000.0)

	// Inster second data point
	spc1info.Blocks = 4
	stat.Latency = time.Millisecond
	meter.Collect(stat)
	tests.Assert(t, meter.blocks == 5)
	tests.Assert(t, meter.ios == 2)
	tests.Assert(t, meter.latency.MeanTimeUsecs() == 2000.0)

}

func TestIoMeterCsvDelta(t *testing.T) {
	spc1info := &spc1.Spc1Io{
		Asu:    1,
		Blocks: 1,
		Stream: 2,
		Offset: 123456,
		When:   time.Second * 2,
	}

	stat := &IoStats{
		Io:      spc1info,
		Start:   time.Now(),
		Latency: time.Millisecond * 3,
	}

	// Inster first data point
	meter := &IoMeter{}

	meter.Collect(stat)
	tests.Assert(t, meter.blocks == 1)
	tests.Assert(t, meter.ios == 1)
	tests.Assert(t, meter.latency.MeanTimeUsecs() == 3000.0)

	prev := &IoMeter{}
	s := meter.CsvDelta(prev, time.Millisecond)
	split := strings.Split(s, ",")
	tests.Assert(t, split[0] == "1")
	tests.Assert(t, split[1] == "4096")
	tests.Assert(t, strings.Contains(split[2], "3.9"))
	tests.Assert(t, strings.Contains(split[3], "3000"))
}

func TestIoMeterDeltas(t *testing.T) {

	prev := &IoMeter{
		ios:    1000,
		blocks: 5000,
	}
	prev.latency.Add(time.Millisecond)
	prev.latency.Add(time.Millisecond)
	prev.latency.Add(time.Millisecond)
	prev.latency.Add(time.Millisecond)

	meter := &IoMeter{}
	*meter = *prev
	meter.ios += 500
	meter.blocks += 500
	meter.latency.Add(time.Millisecond * 2)
	meter.latency.Add(time.Millisecond * 2)
	meter.latency.Add(time.Millisecond * 2)
	meter.latency.Add(time.Millisecond * 2)

	// Test Accessors
	tests.Assert(t, meter.MeanLatencyDeltaUsecs(prev) == 2000.0)
	tests.Assert(t, meter.IosDelta(prev) == 500)
	tests.Assert(t, meter.MeanLatencyUsecs() == 1500.0)
}
<commit_msg>pblio: Misspelled test name<commit_after>//
// Copyright (c) 2014 The pblcache Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package spc

import (
	"github.com/lpabon/goioworkload/spc1"
	"github.com/pblcache/pblcache/tests"
	"strings"
	"testing"
	"time"
)

func TestIoMeterCollect(t *testing.T) {

	spc1info := &spc1.Spc1Io{
		Asu:    1,
		Blocks: 1,
		Stream: 2,
		Offset: 123456,
		When:   time.Second * 2,
	}

	stat := &IoStats{
		Io:      spc1info,
		Start:   time.Now(),
		Latency: time.Millisecond * 3,
	}

	// Inster first data point
	meter := &IoMeter{}

	meter.Collect(stat)
	tests.Assert(t, meter.blocks == 1)
	tests.Assert(t, meter.ios == 1)
	tests.Assert(t, meter.latency.MeanTimeUsecs() == 3000.0)

	// Inster second data point
	spc1info.Blocks = 4
	stat.Latency = time.Millisecond
	meter.Collect(stat)
	tests.Assert(t, meter.blocks == 5)
	tests.Assert(t, meter.ios == 2)
	tests.Assert(t, meter.latency.MeanTimeUsecs() == 2000.0)

}

func TestIoMeterCsvDelta(t *testing.T) {
	spc1info := &spc1.Spc1Io{
		Asu:    1,
		Blocks: 1,
		Stream: 2,
		Offset: 123456,
		When:   time.Second * 2,
	}

	stat := &IoStats{
		Io:      spc1info,
		Start:   time.Now(),
		Latency: time.Millisecond * 3,
	}

	// Inster first data point
	meter := &IoMeter{}

	meter.Collect(stat)
	tests.Assert(t, meter.blocks == 1)
	tests.Assert(t, meter.ios == 1)
	tests.Assert(t, meter.latency.MeanTimeUsecs() == 3000.0)

	prev := &IoMeter{}
	s := meter.CsvDelta(prev, time.Millisecond)
	split := strings.Split(s, ",")
	tests.Assert(t, split[0] == "1")
	tests.Assert(t, split[1] == "4096")
	tests.Assert(t, strings.Contains(split[2], "3.9"))
	tests.Assert(t, strings.Contains(split[3], "3000"))
}

func TestIoMeterDeltas(t *testing.T) {

	prev := &IoMeter{
		ios:    1000,
		blocks: 5000,
	}
	prev.latency.Add(time.Millisecond)
	prev.latency.Add(time.Millisecond)
	prev.latency.Add(time.Millisecond)
	prev.latency.Add(time.Millisecond)

	meter := &IoMeter{}
	*meter = *prev
	meter.ios += 500
	meter.blocks += 500
	meter.latency.Add(time.Millisecond * 2)
	meter.latency.Add(time.Millisecond * 2)
	meter.latency.Add(time.Millisecond * 2)
	meter.latency.Add(time.Millisecond * 2)

	// Test Accessors
	tests.Assert(t, meter.MeanLatencyDeltaUsecs(prev) == 2000.0)
	tests.Assert(t, meter.IosDelta(prev) == 500)
	tests.Assert(t, meter.MeanLatencyUsecs() == 1500.0)
}
<|endoftext|>