<commit_before>package setpipelinehelpers

import (
	"fmt"
	"io/ioutil"
	"os"

	yaml "gopkg.in/yaml.v2"

	"github.com/cloudfoundry/bosh-cli/director/template"
	"github.com/concourse/atc"
	"github.com/concourse/atc/web"
	"github.com/concourse/fly/commands/internal/displayhelpers"
	"github.com/concourse/fly/commands/internal/flaghelpers"
	temp "github.com/concourse/fly/template"
	"github.com/concourse/fly/ui"
	"github.com/concourse/go-concourse/concourse"
	"github.com/onsi/gomega/gexec"
	"github.com/tedsuo/rata"
	"github.com/vito/go-interact/interact"
)

type ATCConfig struct {
	PipelineName        string
	Team                concourse.Team
	WebRequestGenerator *rata.RequestGenerator
	SkipInteraction     bool
}

func (atcConfig ATCConfig) ApplyConfigInteraction() bool {
	if atcConfig.SkipInteraction {
		return true
	}

	confirm := false
	err := interact.NewInteraction("apply configuration?").Resolve(&confirm)
	if err != nil {
		return false
	}

	return confirm
}

func (atcConfig ATCConfig) Set(configPath atc.PathFlag, templateVariables []flaghelpers.VariablePairFlag, yamlTemplateVariables []flaghelpers.YAMLVariablePairFlag, templateVariablesFiles []atc.PathFlag) error {
	newConfig := atcConfig.newConfig(configPath, templateVariablesFiles, templateVariables, yamlTemplateVariables)
	existingConfig, _, existingConfigVersion, _, err := atcConfig.Team.PipelineConfig(atcConfig.PipelineName)
	errorMessages := []string{}
	if err != nil {
		if configError, ok := err.(concourse.PipelineConfigError); ok {
			errorMessages = configError.ErrorMessages
		} else {
			return err
		}
	}

	var new atc.Config
	err = yaml.Unmarshal([]byte(newConfig), &new)
	if err != nil {
		return err
	}

	diff(existingConfig, new)

	if len(errorMessages) > 0 {
		atcConfig.showPipelineConfigErrors(errorMessages)
	}

	if !atcConfig.ApplyConfigInteraction() {
		fmt.Println("bailing out")
		return nil
	}

	created, updated, warnings, err := atcConfig.Team.CreateOrUpdatePipelineConfig(
		atcConfig.PipelineName,
		existingConfigVersion,
		newConfig,
	)
	if err != nil {
		return err
	}

	if len(warnings) > 0 {
		atcConfig.showWarnings(warnings)
	}

	atcConfig.showHelpfulMessage(created, updated)
	return nil
}

func (atcConfig ATCConfig) newConfig(configPath atc.PathFlag, templateVariablesFiles []atc.PathFlag, templateVariables []flaghelpers.VariablePairFlag, yamlTemplateVariables []flaghelpers.YAMLVariablePairFlag) []byte {
	evaluatedConfig, err := ioutil.ReadFile(string(configPath))
	if err != nil {
		displayhelpers.FailWithErrorf("could not read config file", err)
	}

	var paramPayloads [][]byte
	for _, path := range templateVariablesFiles {
		templateVars, err := ioutil.ReadFile(string(path))
		if err != nil {
			displayhelpers.FailWithErrorf("could not read template variables file (%s)", err, string(path))
		}

		paramPayloads = append(paramPayloads, templateVars)
	}

	if temp.Present(evaluatedConfig) {
		evaluatedConfig, err = atcConfig.resolveDeprecatedTemplateStyle(evaluatedConfig, paramPayloads, templateVariables, yamlTemplateVariables)
		if err != nil {
			displayhelpers.FailWithErrorf("could not resolve old-style template vars", err)
		}
	}

	evaluatedConfig, err = atcConfig.resolveTemplates(evaluatedConfig, paramPayloads, templateVariables, yamlTemplateVariables)
	if err != nil {
		displayhelpers.Failf("could not resolve template vars", err)
	}

	return evaluatedConfig
}

func (atcConfig ATCConfig) resolveTemplates(configPayload []byte, paramPayloads [][]byte, variables []flaghelpers.VariablePairFlag, yamlVariables []flaghelpers.YAMLVariablePairFlag) ([]byte, error) {
	tpl := template.NewTemplate(configPayload)

	flagVars := template.StaticVariables{}
	for _, f := range variables {
		flagVars[f.Name] = f.Value
	}

	for _, f := range yamlVariables {
		flagVars[f.Name] = f.Value
	}

	vars := []template.Variables{flagVars}
	for i := len(paramPayloads) - 1; i >= 0; i-- {
		payload := paramPayloads[i]

		var staticVars template.StaticVariables
		err := yaml.Unmarshal(payload, &staticVars)
		if err != nil {
			return nil, err
		}

		vars = append(vars, staticVars)
	}

	bytes, err := tpl.Evaluate(template.NewMultiVars(vars), nil, template.EvaluateOpts{
		ExpectAllKeys: true,
	})
	if err != nil {
		return nil, err
	}

	return bytes, nil
}

func (atcConfig ATCConfig) resolveDeprecatedTemplateStyle(configPayload []byte, paramPayloads [][]byte, variables []flaghelpers.VariablePairFlag, yamlVariables []flaghelpers.YAMLVariablePairFlag) ([]byte, error) {
	vars := temp.Variables{}
	for _, payload := range paramPayloads {
		var payloadVars temp.Variables
		err := yaml.Unmarshal(payload, &payloadVars)
		if err != nil {
			return nil, err
		}

		vars = vars.Merge(payloadVars)
	}

	flagVars := temp.Variables{}
	for _, flag := range variables {
		flagVars[flag.Name] = flag.Value
	}

	vars = vars.Merge(flagVars)

	return temp.Evaluate(configPayload, vars)
}

func (atcConfig ATCConfig) showPipelineConfigErrors(errorMessages []string) {
	fmt.Fprintln(ui.Stderr, "")
	displayhelpers.PrintWarningHeader()

	fmt.Fprintln(ui.Stderr, "Error loading existing config:")
	for _, errorMessage := range errorMessages {
		fmt.Fprintf(ui.Stderr, "  - %s\n", errorMessage)
	}

	fmt.Fprintln(ui.Stderr, "")
}

func (atcConfig ATCConfig) showWarnings(warnings []concourse.ConfigWarning) {
	fmt.Fprintln(ui.Stderr, "")
	displayhelpers.PrintDeprecationWarningHeader()

	for _, warning := range warnings {
		fmt.Fprintf(ui.Stderr, "  - %s\n", warning.Message)
	}

	fmt.Fprintln(ui.Stderr, "")
}

func (atcConfig ATCConfig) showHelpfulMessage(created bool, updated bool) {
	if updated {
		fmt.Println("configuration updated")
	} else if created {
		pipelineWebReq, _ := atcConfig.WebRequestGenerator.CreateRequest(
			web.Pipeline,
			rata.Params{
				"pipeline":  atcConfig.PipelineName,
				"team_name": atcConfig.Team.Name(),
			},
			nil,
		)

		fmt.Println("pipeline created!")

		pipelineURL := pipelineWebReq.URL
		// don't show username and password
		pipelineURL.User = nil

		fmt.Printf("you can view your pipeline here: %s\n", pipelineURL.String())

		fmt.Println("")
		fmt.Println("the pipeline is currently paused. to unpause, either:")
		fmt.Println("  - run the unpause-pipeline command")
		fmt.Println("  - click play next to the pipeline in the web ui")
	} else {
		panic("Something really went wrong!")
	}
}

func diff(existingConfig atc.Config, newConfig atc.Config) {
	indent := gexec.NewPrefixedWriter("  ", os.Stdout)

	groupDiffs := diffIndices(GroupIndex(existingConfig.Groups), GroupIndex(newConfig.Groups))
	if len(groupDiffs) > 0 {
		fmt.Println("groups:")

		for _, diff := range groupDiffs {
			diff.Render(indent, "group")
		}
	}

	resourceDiffs := diffIndices(ResourceIndex(existingConfig.Resources), ResourceIndex(newConfig.Resources))
	if len(resourceDiffs) > 0 {
		fmt.Println("resources:")

		for _, diff := range resourceDiffs {
			diff.Render(indent, "resource")
		}
	}

	resourceTypeDiffs := diffIndices(ResourceTypeIndex(existingConfig.ResourceTypes), ResourceTypeIndex(newConfig.ResourceTypes))
	if len(resourceTypeDiffs) > 0 {
		fmt.Println("resource types:")

		for _, diff := range resourceTypeDiffs {
			diff.Render(indent, "resource type")
		}
	}

	jobDiffs := diffIndices(JobIndex(existingConfig.Jobs), JobIndex(newConfig.Jobs))
	if len(jobDiffs) > 0 {
		fmt.Println("jobs:")

		for _, diff := range jobDiffs {
			diff.Render(indent, "job")
		}
	}
}
<commit_msg>don't expect all keys client-side<commit_after>package setpipelinehelpers

import (
	"fmt"
	"io/ioutil"
	"os"

	yaml "gopkg.in/yaml.v2"

	"github.com/cloudfoundry/bosh-cli/director/template"
	"github.com/concourse/atc"
	"github.com/concourse/atc/web"
	"github.com/concourse/fly/commands/internal/displayhelpers"
	"github.com/concourse/fly/commands/internal/flaghelpers"
	temp "github.com/concourse/fly/template"
	"github.com/concourse/fly/ui"
	"github.com/concourse/go-concourse/concourse"
	"github.com/onsi/gomega/gexec"
	"github.com/tedsuo/rata"
	"github.com/vito/go-interact/interact"
)

type ATCConfig struct {
	PipelineName        string
	Team                concourse.Team
	WebRequestGenerator *rata.RequestGenerator
	SkipInteraction     bool
}

func (atcConfig ATCConfig) ApplyConfigInteraction() bool {
	if atcConfig.SkipInteraction {
		return true
	}

	confirm := false
	err := interact.NewInteraction("apply configuration?").Resolve(&confirm)
	if err != nil {
		return false
	}

	return confirm
}

func (atcConfig ATCConfig) Set(configPath atc.PathFlag, templateVariables []flaghelpers.VariablePairFlag, yamlTemplateVariables []flaghelpers.YAMLVariablePairFlag, templateVariablesFiles []atc.PathFlag) error {
	newConfig := atcConfig.newConfig(configPath, templateVariablesFiles, templateVariables, yamlTemplateVariables)
	existingConfig, _, existingConfigVersion, _, err := atcConfig.Team.PipelineConfig(atcConfig.PipelineName)
	errorMessages := []string{}
	if err != nil {
		if configError, ok := err.(concourse.PipelineConfigError); ok {
			errorMessages = configError.ErrorMessages
		} else {
			return err
		}
	}

	var new atc.Config
	err = yaml.Unmarshal([]byte(newConfig), &new)
	if err != nil {
		return err
	}

	diff(existingConfig, new)

	if len(errorMessages) > 0 {
		atcConfig.showPipelineConfigErrors(errorMessages)
	}

	if !atcConfig.ApplyConfigInteraction() {
		fmt.Println("bailing out")
		return nil
	}

	created, updated, warnings, err := atcConfig.Team.CreateOrUpdatePipelineConfig(
		atcConfig.PipelineName,
		existingConfigVersion,
		newConfig,
	)
	if err != nil {
		return err
	}

	if len(warnings) > 0 {
		atcConfig.showWarnings(warnings)
	}

	atcConfig.showHelpfulMessage(created, updated)
	return nil
}

func (atcConfig ATCConfig) newConfig(configPath atc.PathFlag, templateVariablesFiles []atc.PathFlag, templateVariables []flaghelpers.VariablePairFlag, yamlTemplateVariables []flaghelpers.YAMLVariablePairFlag) []byte {
	evaluatedConfig, err := ioutil.ReadFile(string(configPath))
	if err != nil {
		displayhelpers.FailWithErrorf("could not read config file", err)
	}

	var paramPayloads [][]byte
	for _, path := range templateVariablesFiles {
		templateVars, err := ioutil.ReadFile(string(path))
		if err != nil {
			displayhelpers.FailWithErrorf("could not read template variables file (%s)", err, string(path))
		}

		paramPayloads = append(paramPayloads, templateVars)
	}

	if temp.Present(evaluatedConfig) {
		evaluatedConfig, err = atcConfig.resolveDeprecatedTemplateStyle(evaluatedConfig, paramPayloads, templateVariables, yamlTemplateVariables)
		if err != nil {
			displayhelpers.FailWithErrorf("could not resolve old-style template vars: %s", err)
		}
	}

	evaluatedConfig, err = atcConfig.resolveTemplates(evaluatedConfig, paramPayloads, templateVariables, yamlTemplateVariables)
	if err != nil {
		displayhelpers.Failf("could not resolve template vars: %s", err)
	}

	return evaluatedConfig
}

func (atcConfig ATCConfig) resolveTemplates(configPayload []byte, paramPayloads [][]byte, variables []flaghelpers.VariablePairFlag, yamlVariables []flaghelpers.YAMLVariablePairFlag) ([]byte, error) {
	tpl := template.NewTemplate(configPayload)

	flagVars := template.StaticVariables{}
	for _, f := range variables {
		flagVars[f.Name] = f.Value
	}

	for _, f := range yamlVariables {
		flagVars[f.Name] = f.Value
	}

	vars := []template.Variables{flagVars}
	for i := len(paramPayloads) - 1; i >= 0; i-- {
		payload := paramPayloads[i]

		var staticVars template.StaticVariables
		err := yaml.Unmarshal(payload, &staticVars)
		if err != nil {
			return nil, err
		}

		vars = append(vars, staticVars)
	}

	bytes, err := tpl.Evaluate(template.NewMultiVars(vars), nil, template.EvaluateOpts{})
	if err != nil {
		return nil, err
	}

	return bytes, nil
}

func (atcConfig ATCConfig) resolveDeprecatedTemplateStyle(configPayload []byte, paramPayloads [][]byte, variables []flaghelpers.VariablePairFlag, yamlVariables []flaghelpers.YAMLVariablePairFlag) ([]byte, error) {
	vars := temp.Variables{}
	for _, payload := range paramPayloads {
		var payloadVars temp.Variables
		err := yaml.Unmarshal(payload, &payloadVars)
		if err != nil {
			return nil, err
		}

		vars = vars.Merge(payloadVars)
	}

	flagVars := temp.Variables{}
	for _, flag := range variables {
		flagVars[flag.Name] = flag.Value
	}

	vars = vars.Merge(flagVars)

	return temp.Evaluate(configPayload, vars)
}

func (atcConfig ATCConfig) showPipelineConfigErrors(errorMessages []string) {
	fmt.Fprintln(ui.Stderr, "")
	displayhelpers.PrintWarningHeader()

	fmt.Fprintln(ui.Stderr, "Error loading existing config:")
	for _, errorMessage := range errorMessages {
		fmt.Fprintf(ui.Stderr, "  - %s\n", errorMessage)
	}

	fmt.Fprintln(ui.Stderr, "")
}

func (atcConfig ATCConfig) showWarnings(warnings []concourse.ConfigWarning) {
	fmt.Fprintln(ui.Stderr, "")
	displayhelpers.PrintDeprecationWarningHeader()

	for _, warning := range warnings {
		fmt.Fprintf(ui.Stderr, "  - %s\n", warning.Message)
	}

	fmt.Fprintln(ui.Stderr, "")
}

func (atcConfig ATCConfig) showHelpfulMessage(created bool, updated bool) {
	if updated {
		fmt.Println("configuration updated")
	} else if created {
		pipelineWebReq, _ := atcConfig.WebRequestGenerator.CreateRequest(
			web.Pipeline,
			rata.Params{
				"pipeline":  atcConfig.PipelineName,
				"team_name": atcConfig.Team.Name(),
			},
			nil,
		)

		fmt.Println("pipeline created!")

		pipelineURL := pipelineWebReq.URL
		// don't show username and password
		pipelineURL.User = nil

		fmt.Printf("you can view your pipeline here: %s\n", pipelineURL.String())

		fmt.Println("")
		fmt.Println("the pipeline is currently paused. to unpause, either:")
		fmt.Println("  - run the unpause-pipeline command")
		fmt.Println("  - click play next to the pipeline in the web ui")
	} else {
		panic("Something really went wrong!")
	}
}

func diff(existingConfig atc.Config, newConfig atc.Config) {
	indent := gexec.NewPrefixedWriter("  ", os.Stdout)

	groupDiffs := diffIndices(GroupIndex(existingConfig.Groups), GroupIndex(newConfig.Groups))
	if len(groupDiffs) > 0 {
		fmt.Println("groups:")

		for _, diff := range groupDiffs {
			diff.Render(indent, "group")
		}
	}

	resourceDiffs := diffIndices(ResourceIndex(existingConfig.Resources), ResourceIndex(newConfig.Resources))
	if len(resourceDiffs) > 0 {
		fmt.Println("resources:")

		for _, diff := range resourceDiffs {
			diff.Render(indent, "resource")
		}
	}

	resourceTypeDiffs := diffIndices(ResourceTypeIndex(existingConfig.ResourceTypes), ResourceTypeIndex(newConfig.ResourceTypes))
	if len(resourceTypeDiffs) > 0 {
		fmt.Println("resource types:")

		for _, diff := range resourceTypeDiffs {
			diff.Render(indent, "resource type")
		}
	}

	jobDiffs := diffIndices(JobIndex(existingConfig.Jobs), JobIndex(newConfig.Jobs))
	if len(jobDiffs) > 0 {
		fmt.Println("jobs:")

		for _, diff := range jobDiffs {
			diff.Render(indent, "job")
		}
	}
}
<|endoftext|>


<commit_before>package v1

import (
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/util/managedfields"
	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager"
	"k8s.io/client-go/discovery"
	"k8s.io/kube-openapi/pkg/util/proto"
	"sigs.k8s.io/structured-merge-diff/v4/typed"
)

type UnstructuredExtractor interface {
	ExtractUnstructured(object *unstructured.Unstructured, fieldManager string) (*unstructured.Unstructured, error)
	ExtractUnstructuredStatus(object *unstructured.Unstructured, fieldManager string) (*unstructured.Unstructured, error)
}

type parserCache interface {
	parserForGVK(gvk schema.GroupVersionKind) (*typed.ParseableType, error)
}

type nonCachingParserCache struct {
	discoveryClient *discovery.DiscoveryClient
}

func (c *nonCachingParserCache) parserForGVK(gvk schema.GroupVersionKind) (*typed.ParseableType, error) {
	doc, err := c.discoveryClient.OpenAPISchema()
	if err != nil {
		return nil, err
	}

	models, err := proto.NewOpenAPIData(doc)
	if err != nil {
		return nil, err
	}

	gvkParser, err := fieldmanager.NewGVKParser(models, false)
	if err != nil {
		return nil, err
	}

	return gvkParser.Type(gvk), nil
}

type extractor struct {
	cache parserCache
}

func NewUnstructuredExtractor(dc *discovery.DiscoveryClient) UnstructuredExtractor {
	return &extractor{
		cache: &nonCachingParserCache{dc},
	}
}

func (e *extractor) ExtractUnstructured(object *unstructured.Unstructured, fieldManager string) (*unstructured.Unstructured, error) {
	return e.extractUnstructured(object, fieldManager, "")
}

func (e *extractor) ExtractUnstructuredStatus(object *unstructured.Unstructured, fieldManager string) (*unstructured.Unstructured, error) {
	return e.extractUnstructured(object, fieldManager, "status")
}

func (e *extractor) extractUnstructured(object *unstructured.Unstructured, fieldManager string, subresource string) (*unstructured.Unstructured, error) {
	gvk := object.GetObjectKind().GroupVersionKind()
	parser, err := e.cache.parserForGVK(gvk)
	if err != nil {
		return nil, err
	}
	result := &unstructured.Unstructured{}
	err = managedfields.ExtractInto(object, *parser, fieldManager, result, subresource)
	if err != nil {
		return nil, err
	}
	result.SetName(object.GetName())
	result.SetNamespace(object.GetNamespace())
	result.SetKind(object.GetKind())
	result.SetAPIVersion(object.GetAPIVersion())
	return result, nil
}
<commit_msg>Manually set GVK in extract, add commentary to extractor<commit_after>package v1

import (
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/util/managedfields"
	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager"
	"k8s.io/client-go/discovery"
	"k8s.io/kube-openapi/pkg/util/proto"
	"sigs.k8s.io/structured-merge-diff/v4/typed"
)

// UnstructuredExtractor enables extracting the applied configuration state from object for fieldManager into an
// unstructured object type.
type UnstructuredExtractor interface {
	ExtractUnstructured(object *unstructured.Unstructured, fieldManager string) (*unstructured.Unstructured, error)
	ExtractUnstructuredStatus(object *unstructured.Unstructured, fieldManager string) (*unstructured.Unstructured, error)
}

// objectTypeCache is a cache of typed.ParseableTypes
type objectTypeCache interface {
	objectTypeForGVK(gvk schema.GroupVersionKind) (*typed.ParseableType, error)
}

// nonCachingObjectTypeCache is a objectTypeCache that does no caching
// (i.e. it downloads the OpenAPISchema every time)
// Useful during the proof-of-concept stage until we agree on a caching solution.
type nonCachingObjectTypeCache struct {
	discoveryClient *discovery.DiscoveryClient
}

// objectTypeForGVK retrieves the typed.ParseableType for a given gvk from the cache
func (c *nonCachingObjectTypeCache) objectTypeForGVK(gvk schema.GroupVersionKind) (*typed.ParseableType, error) {
	doc, err := c.discoveryClient.OpenAPISchema()
	if err != nil {
		return nil, err
	}

	models, err := proto.NewOpenAPIData(doc)
	if err != nil {
		return nil, err
	}

	gvkParser, err := fieldmanager.NewGVKParser(models, false)
	if err != nil {
		return nil, err
	}

	return gvkParser.Type(gvk), nil
}

type extractor struct {
	cache objectTypeCache
}

// NewUnstructuredExtractor creates the extractor with which you can extract the applied configuration
// for a given manager from an unstructured object.
func NewUnstructuredExtractor(dc *discovery.DiscoveryClient) UnstructuredExtractor {
	return &extractor{
		cache: &nonCachingObjectTypeCache{dc},
	}
}

// ExtractUnstructured extracts the applied configuration owned by fiieldManager from an unstructured object.
// Note that the apply configuration itself is also an unstructured object.
func (e *extractor) ExtractUnstructured(object *unstructured.Unstructured, fieldManager string) (*unstructured.Unstructured, error) {
	return e.extractUnstructured(object, fieldManager, "")
}

// ExtractUnstructuredStatus is the same as ExtractUnstructured except
// that it extracts the status subresource applied configuration.
// Experimental!
func (e *extractor) ExtractUnstructuredStatus(object *unstructured.Unstructured, fieldManager string) (*unstructured.Unstructured, error) {
	return e.extractUnstructured(object, fieldManager, "status")
}

func (e *extractor) extractUnstructured(object *unstructured.Unstructured, fieldManager string, subresource string) (*unstructured.Unstructured, error) {
	gvk := object.GetObjectKind().GroupVersionKind()
	objectType, err := e.cache.objectTypeForGVK(gvk)
	if err != nil {
		return nil, err
	}
	result := &unstructured.Unstructured{}
	err = managedfields.ExtractInto(object, *objectType, fieldManager, result, subresource)
	if err != nil {
		return nil, err
	}
	result.SetName(object.GetName())
	result.SetNamespace(object.GetNamespace())
	result.SetKind(object.GetKind())
	result.SetAPIVersion(object.GetAPIVersion())
	return result, nil
}
<|endoftext|>


<commit_before>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package highavailability

import (
	"sort"
	"strconv"

	"github.com/juju/errors"
	"github.com/juju/loggo"
	"gopkg.in/juju/names.v2"

	"github.com/juju/juju/apiserver/common"
	"github.com/juju/juju/apiserver/facade"
	"github.com/juju/juju/apiserver/params"
	"github.com/juju/juju/constraints"
	"github.com/juju/juju/mongo"
	"github.com/juju/juju/permission"
	"github.com/juju/juju/state"
)

var logger = loggo.GetLogger("juju.apiserver.highavailability")

func init() {
	common.RegisterStandardFacade("HighAvailability", 2, NewHighAvailabilityAPI)
}

// HighAvailability defines the methods on the highavailability API end point.
type HighAvailability interface {
	EnableHA(args params.ControllersSpecs) (params.ControllersChangeResults, error)
}

// HighAvailabilityAPI implements the HighAvailability interface and is the concrete
// implementation of the api end point.
type HighAvailabilityAPI struct {
	state      *state.State
	resources  facade.Resources
	authorizer facade.Authorizer
}

var _ HighAvailability = (*HighAvailabilityAPI)(nil)

// NewHighAvailabilityAPI creates a new server-side highavailability API end point.
func NewHighAvailabilityAPI(st *state.State, resources facade.Resources, authorizer facade.Authorizer) (*HighAvailabilityAPI, error) {
	// Only clients and environment managers can access the high availability service.
	if !authorizer.AuthClient() && !authorizer.AuthModelManager() {
		return nil, common.ErrPerm
	}
	return &HighAvailabilityAPI{
		state:      st,
		resources:  resources,
		authorizer: authorizer,
	}, nil
}

func (api *HighAvailabilityAPI) EnableHA(args params.ControllersSpecs) (params.ControllersChangeResults, error) {
	results := params.ControllersChangeResults{Results: make([]params.ControllersChangeResult, len(args.Specs))}
	for i, controllersServersSpec := range args.Specs {
		if api.authorizer.AuthClient() {
			admin, err := api.authorizer.HasPermission(permission.SuperuserAccess, api.state.ControllerTag())
			if err != nil && !errors.IsNotFound(err) {
				return results, errors.Trace(err)
			}
			if !admin {
				return results, common.ServerError(common.ErrPerm)
			}

		}
		result, err := EnableHASingle(api.state, controllersServersSpec)
		results.Results[i].Result = result
		results.Results[i].Error = common.ServerError(err)
	}
	return results, nil
}

// Convert machine ids to tags.
func machineIdsToTags(ids ...string) []string {
	var result []string
	for _, id := range ids {
		result = append(result, names.NewMachineTag(id).String())
	}
	return result
}

// Generate a ControllersChanges structure.
func controllersChanges(change state.ControllersChanges) params.ControllersChanges {
	return params.ControllersChanges{
		Added:      machineIdsToTags(change.Added...),
		Maintained: machineIdsToTags(change.Maintained...),
		Removed:    machineIdsToTags(change.Removed...),
		Promoted:   machineIdsToTags(change.Promoted...),
		Demoted:    machineIdsToTags(change.Demoted...),
		Converted:  machineIdsToTags(change.Converted...),
	}
}

// EnableHASingle applies a single ControllersServersSpec specification to the current environment.
// Exported so it can be called by the legacy client API in the client package.
func EnableHASingle(st *state.State, spec params.ControllersSpec) (params.ControllersChanges, error) {
	if !st.IsController() {
		return params.ControllersChanges{}, errors.New("unsupported with hosted models")
	}
	// Check if changes are allowed and the command may proceed.
	blockChecker := common.NewBlockChecker(st)
	if err := blockChecker.ChangeAllowed(); err != nil {
		return params.ControllersChanges{}, errors.Trace(err)
	}
	// Validate the environment tag if present.
	if spec.ModelTag != "" {
		tag, err := names.ParseModelTag(spec.ModelTag)
		if err != nil {
			return params.ControllersChanges{}, errors.Errorf("invalid model tag: %v", err)
		}
		if _, err := st.FindEntity(tag); err != nil {
			return params.ControllersChanges{}, err
		}
	}

	series := spec.Series
	if series == "" {
		ssi, err := st.ControllerInfo()
		if err != nil {
			return params.ControllersChanges{}, err
		}

		// We should always have at least one voting machine
		// If we *really* wanted we could just pick whatever series is
		// in the majority, but really, if we always copy the value of
		// the first one, then they'll stay in sync.
		if len(ssi.VotingMachineIds) == 0 {
			// Better than a panic()?
			return params.ControllersChanges{}, errors.Errorf("internal error, failed to find any voting machines")
		}
		templateMachine, err := st.Machine(ssi.VotingMachineIds[0])
		if err != nil {
			return params.ControllersChanges{}, err
		}
		series = templateMachine.Series()
	}
	if constraints.IsEmpty(&spec.Constraints) {
		// No constraints specified, so we'll use the constraints off
		// a running controller.
		controllerInfo, err := st.ControllerInfo()
		if err != nil {
			return params.ControllersChanges{}, err
		}
		// We'll sort the controller ids to find the smallest.
		// This will typically give the initial bootstrap machine.
		var controllerIds []int
		for _, id := range controllerInfo.MachineIds {
			idNum, err := strconv.Atoi(id)
			if err != nil {
				logger.Warningf("ignoring non numeric controller id %v", id)
				continue
			}
			controllerIds = append(controllerIds, idNum)
		}
		if len(controllerIds) == 0 {
			errors.Errorf("internal error, failed to find any controllers")
		}
		sort.Ints(controllerIds)

		// Load the controller machine and get its constraints.
		controllerId := controllerIds[0]
		controller, err := st.Machine(strconv.Itoa(controllerId))
		if err != nil {
			return params.ControllersChanges{}, errors.Annotatef(err, "reading controller id %v", controllerId)
		}
		spec.Constraints, err = controller.Constraints()
		if err != nil {
			return params.ControllersChanges{}, errors.Annotatef(err, "reading constraints for controller id %v", controllerId)
		}
	}

	changes, err := st.EnableHA(spec.NumControllers, spec.Constraints, series, spec.Placement)
	if err != nil {
		return params.ControllersChanges{}, err
	}
	return controllersChanges(changes), nil
}

// StopHAReplicationForUpgrade will prompt the HA cluster to enter upgrade
// mongo mode.
func (api *HighAvailabilityAPI) StopHAReplicationForUpgrade(args params.UpgradeMongoParams) (params.MongoUpgradeResults, error) {
	ha, err := api.state.SetUpgradeMongoMode(mongo.Version{
		Major:         args.Target.Major,
		Minor:         args.Target.Minor,
		Patch:         args.Target.Patch,
		StorageEngine: mongo.StorageEngine(args.Target.StorageEngine),
	})
	if err != nil {
		return params.MongoUpgradeResults{}, errors.Annotate(err, "cannot stop HA for ugprade")
	}
	members := make([]params.HAMember, len(ha.Members))
	for i, m := range ha.Members {
		members[i] = params.HAMember{
			Tag:           m.Tag,
			PublicAddress: m.PublicAddress,
			Series:        m.Series,
		}
	}
	return params.MongoUpgradeResults{
		Master: params.HAMember{
			Tag:           ha.Master.Tag,
			PublicAddress: ha.Master.PublicAddress,
			Series:        ha.Master.Series,
		},
		Members:   members,
		RsMembers: ha.RsMembers,
	}, nil
}

// ResumeHAReplicationAfterUpgrade will add the upgraded members of HA
// cluster to the upgraded master.
func (api *HighAvailabilityAPI) ResumeHAReplicationAfterUpgrade(args params.ResumeReplicationParams) error {
	return api.state.ResumeReplication(args.Members)
}
<commit_msg>apiserver/highavailability: Hide EnableHASingle<commit_after>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package highavailability

import (
	"sort"
	"strconv"

	"github.com/juju/errors"
	"github.com/juju/loggo"
	"gopkg.in/juju/names.v2"

	"github.com/juju/juju/apiserver/common"
	"github.com/juju/juju/apiserver/facade"
	"github.com/juju/juju/apiserver/params"
	"github.com/juju/juju/constraints"
	"github.com/juju/juju/mongo"
	"github.com/juju/juju/permission"
	"github.com/juju/juju/state"
)

var logger = loggo.GetLogger("juju.apiserver.highavailability")

func init() {
	common.RegisterStandardFacade("HighAvailability", 2, NewHighAvailabilityAPI)
}

// HighAvailability defines the methods on the highavailability API end point.
type HighAvailability interface {
	EnableHA(args params.ControllersSpecs) (params.ControllersChangeResults, error)
}

// HighAvailabilityAPI implements the HighAvailability interface and is the concrete
// implementation of the api end point.
type HighAvailabilityAPI struct {
	state      *state.State
	resources  facade.Resources
	authorizer facade.Authorizer
}

var _ HighAvailability = (*HighAvailabilityAPI)(nil)

// NewHighAvailabilityAPI creates a new server-side highavailability API end point.
func NewHighAvailabilityAPI(st *state.State, resources facade.Resources, authorizer facade.Authorizer) (*HighAvailabilityAPI, error) {
	// Only clients and environment managers can access the high availability service.
	if !authorizer.AuthClient() && !authorizer.AuthModelManager() {
		return nil, common.ErrPerm
	}
	return &HighAvailabilityAPI{
		state:      st,
		resources:  resources,
		authorizer: authorizer,
	}, nil
}

// EnableHA adds controller machines as necessary to ensure the
// controller has the number of machines specified.
func (api *HighAvailabilityAPI) EnableHA(args params.ControllersSpecs) (params.ControllersChangeResults, error) {
	results := params.ControllersChangeResults{Results: make([]params.ControllersChangeResult, len(args.Specs))}
	for i, controllersServersSpec := range args.Specs {
		if api.authorizer.AuthClient() {
			admin, err := api.authorizer.HasPermission(permission.SuperuserAccess, api.state.ControllerTag())
			if err != nil && !errors.IsNotFound(err) {
				return results, errors.Trace(err)
			}
			if !admin {
				return results, common.ServerError(common.ErrPerm)
			}

		}
		result, err := enableHASingle(api.state, controllersServersSpec)
		results.Results[i].Result = result
		results.Results[i].Error = common.ServerError(err)
	}
	return results, nil
}

// Convert machine ids to tags.
func machineIdsToTags(ids ...string) []string {
	var result []string
	for _, id := range ids {
		result = append(result, names.NewMachineTag(id).String())
	}
	return result
}

// Generate a ControllersChanges structure.
func controllersChanges(change state.ControllersChanges) params.ControllersChanges {
	return params.ControllersChanges{
		Added:      machineIdsToTags(change.Added...),
		Maintained: machineIdsToTags(change.Maintained...),
		Removed:    machineIdsToTags(change.Removed...),
		Promoted:   machineIdsToTags(change.Promoted...),
		Demoted:    machineIdsToTags(change.Demoted...),
		Converted:  machineIdsToTags(change.Converted...),
	}
}

func enableHASingle(st *state.State, spec params.ControllersSpec) (params.ControllersChanges, error) {
	if !st.IsController() {
		return params.ControllersChanges{}, errors.New("unsupported with hosted models")
	}
	// Check if changes are allowed and the command may proceed.
	blockChecker := common.NewBlockChecker(st)
	if err := blockChecker.ChangeAllowed(); err != nil {
		return params.ControllersChanges{}, errors.Trace(err)
	}
	// Validate the environment tag if present.
	if spec.ModelTag != "" {
		tag, err := names.ParseModelTag(spec.ModelTag)
		if err != nil {
			return params.ControllersChanges{}, errors.Errorf("invalid model tag: %v", err)
		}
		if _, err := st.FindEntity(tag); err != nil {
			return params.ControllersChanges{}, err
		}
	}

	series := spec.Series
	if series == "" {
		ssi, err := st.ControllerInfo()
		if err != nil {
			return params.ControllersChanges{}, err
		}

		// We should always have at least one voting machine
		// If we *really* wanted we could just pick whatever series is
		// in the majority, but really, if we always copy the value of
		// the first one, then they'll stay in sync.
		if len(ssi.VotingMachineIds) == 0 {
			// Better than a panic()?
			return params.ControllersChanges{}, errors.Errorf("internal error, failed to find any voting machines")
		}
		templateMachine, err := st.Machine(ssi.VotingMachineIds[0])
		if err != nil {
			return params.ControllersChanges{}, err
		}
		series = templateMachine.Series()
	}
	if constraints.IsEmpty(&spec.Constraints) {
		// No constraints specified, so we'll use the constraints off
		// a running controller.
		controllerInfo, err := st.ControllerInfo()
		if err != nil {
			return params.ControllersChanges{}, err
		}
		// We'll sort the controller ids to find the smallest.
		// This will typically give the initial bootstrap machine.
		var controllerIds []int
		for _, id := range controllerInfo.MachineIds {
			idNum, err := strconv.Atoi(id)
			if err != nil {
				logger.Warningf("ignoring non numeric controller id %v", id)
				continue
			}
			controllerIds = append(controllerIds, idNum)
		}
		if len(controllerIds) == 0 {
			errors.Errorf("internal error, failed to find any controllers")
		}
		sort.Ints(controllerIds)

		// Load the controller machine and get its constraints.
		controllerId := controllerIds[0]
		controller, err := st.Machine(strconv.Itoa(controllerId))
		if err != nil {
			return params.ControllersChanges{}, errors.Annotatef(err, "reading controller id %v", controllerId)
		}
		spec.Constraints, err = controller.Constraints()
		if err != nil {
			return params.ControllersChanges{}, errors.Annotatef(err, "reading constraints for controller id %v", controllerId)
		}
	}

	changes, err := st.EnableHA(spec.NumControllers, spec.Constraints, series, spec.Placement)
	if err != nil {
		return params.ControllersChanges{}, err
	}
	return controllersChanges(changes), nil
}

// StopHAReplicationForUpgrade will prompt the HA cluster to enter upgrade
// mongo mode.
func (api *HighAvailabilityAPI) StopHAReplicationForUpgrade(args params.UpgradeMongoParams) (params.MongoUpgradeResults, error) {
	ha, err := api.state.SetUpgradeMongoMode(mongo.Version{
		Major:         args.Target.Major,
		Minor:         args.Target.Minor,
		Patch:         args.Target.Patch,
		StorageEngine: mongo.StorageEngine(args.Target.StorageEngine),
	})
	if err != nil {
		return params.MongoUpgradeResults{}, errors.Annotate(err, "cannot stop HA for ugprade")
	}
	members := make([]params.HAMember, len(ha.Members))
	for i, m := range ha.Members {
		members[i] = params.HAMember{
			Tag:           m.Tag,
			PublicAddress: m.PublicAddress,
			Series:        m.Series,
		}
	}
	return params.MongoUpgradeResults{
		Master: params.HAMember{
			Tag:           ha.Master.Tag,
			PublicAddress: ha.Master.PublicAddress,
			Series:        ha.Master.Series,
		},
		Members:   members,
		RsMembers: ha.RsMembers,
	}, nil
}

// ResumeHAReplicationAfterUpgrade will add the upgraded members of HA
// cluster to the upgraded master.
func (api *HighAvailabilityAPI) ResumeHAReplicationAfterUpgrade(args params.ResumeReplicationParams) error {
	return api.state.ResumeReplication(args.Members)
}
<|endoftext|>


<commit_before>package fleet

import (
	"golang.org/x/net/context"

	"github.com/giantswarm/inago/logging"
)

// DummyConfig holds configuration for the DummyFleet struct.
type DummyConfig struct {
	Logger logging.Logger
}

// DummyFleet is an implementation of the Fleet interface,
// that is primarily intended to be used for testing.
type DummyFleet struct {
	Config DummyConfig
	Units  map[string]UnitStatus
}

// DefaultDummyConfig returns a best-effort configuration for the DummyFleet struct.
func DefaultDummyConfig() DummyConfig {
	return DummyConfig{
		Logger: logging.NewLogger(logging.DefaultConfig()),
	}
}

// NewDummyFleet returns a DummyFleet, given a DummyConfig.
func NewDummyFleet(DummyConfig) *DummyFleet {
	return &DummyFleet{
		Config: DefaultDummyConfig(),
		Units:  make(map[string]UnitStatus),
	}
}

// Submit creates a UnitStatus, and stores it.
func (f *DummyFleet) Submit(ctx context.Context, name, content string) error {
	f.Config.Logger.Debug(ctx, "dummy fleet: submit %v %v", name, content)

	f.Units[name] = UnitStatus{
		Current: unitStateLoaded,
		Desired: unitStateLoaded,
		Name:    name,
	}

	return nil
}

// Start sets the Current and Desired state of the stored UnitStatus
// to unitStateLaunched.
func (f *DummyFleet) Start(ctx context.Context, name string) error {
	f.Config.Logger.Debug(ctx, "dummy fleet: start %v", name)

	if _, ok := f.Units[name]; !ok {
		return maskAny(unitNotFoundError)
	}

	unitStatus, _ := f.Units[name]

	unitStatus.Current = unitStateLaunched
	unitStatus.Desired = unitStateLaunched

	f.Units[name] = unitStatus

	return nil
}

// Stop sets the Current and Desired state of the stored UnitStatus
// to unitStateLoaded.
func (f *DummyFleet) Stop(ctx context.Context, name string) error {
	f.Config.Logger.Debug(ctx, "dummy fleet: stop %v", name)

	if _, ok := f.Units[name]; !ok {
		return maskAny(unitNotFoundError)
	}

	unitStatus, _ := f.Units[name]

	unitStatus.Current = unitStateLoaded
	unitStatus.Desired = unitStateLoaded

	f.Units[name] = unitStatus

	return nil
}

// Destroy remove the UnitStatus from the internal store.
func (f *DummyFleet) Destroy(ctx context.Context, name string) error {
	f.Config.Logger.Debug(ctx, "dummy fleet: destroy %v", name)

	if _, ok := f.Units[name]; !ok {
		return maskAny(unitNotFoundError)
	}

	delete(f.Units, name)

	return nil
}

// GetStatus returns the UnitStatus for the given name.
func (f *DummyFleet) GetStatus(ctx context.Context, name string) (UnitStatus, error) {
	f.Config.Logger.Debug(ctx, "dummy fleet: get status %v", name)

	unitStatus, ok := f.Units[name]
	if !ok {
		return UnitStatus{}, maskAny(unitNotFoundError)
	}

	return unitStatus, nil
}

// GetStatusWithMatcher returns all UnitStatus that match.
func (f *DummyFleet) GetStatusWithMatcher(m func(string) bool) ([]UnitStatus, error) {
	f.Config.Logger.Debug(context.Background(), "dummy fleet: get status with matcher")

	if len(f.Units) == 0 {
		return []UnitStatus{}, maskAny(unitNotFoundError)
	}

	unitStatusList := []UnitStatus{}
	for _, unitStatus := range f.Units {
		if m(unitStatus.Name) {
			unitStatusList = append(unitStatusList, unitStatus)
		}
	}

	return unitStatusList, nil
}
<commit_msg>Add mutex to dummy fleet<commit_after>package fleet

import (
	"sync"

	"golang.org/x/net/context"

	"github.com/giantswarm/inago/logging"
)

// DummyConfig holds configuration for the DummyFleet struct.
type DummyConfig struct {
	Logger logging.Logger
}

// DummyFleet is an implementation of the Fleet interface,
// that is primarily intended to be used for testing.
type DummyFleet struct {
	Config DummyConfig
	Units  map[string]UnitStatus
	Mutex  sync.Mutex
}

// DefaultDummyConfig returns a best-effort configuration for the DummyFleet struct.
func DefaultDummyConfig() DummyConfig {
	return DummyConfig{
		Logger: logging.NewLogger(logging.DefaultConfig()),
	}
}

// NewDummyFleet returns a DummyFleet, given a DummyConfig.
func NewDummyFleet(DummyConfig) *DummyFleet {
	return &DummyFleet{
		Config: DefaultDummyConfig(),
		Units:  make(map[string]UnitStatus),
	}
}

// Submit creates a UnitStatus, and stores it.
func (f *DummyFleet) Submit(ctx context.Context, name, content string) error {
	f.Config.Logger.Debug(ctx, "dummy fleet: submit %v %v", name, content)

	f.Mutex.Lock()
	defer f.Mutex.Unlock()

	f.Units[name] = UnitStatus{
		Current: unitStateLoaded,
		Desired: unitStateLoaded,
		Name:    name,
	}

	return nil
}

// Start sets the Current and Desired state of the stored UnitStatus
// to unitStateLaunched.
func (f *DummyFleet) Start(ctx context.Context, name string) error {
	f.Config.Logger.Debug(ctx, "dummy fleet: start %v", name)

	f.Mutex.Lock()
	defer f.Mutex.Unlock()

	if _, ok := f.Units[name]; !ok {
		return maskAny(unitNotFoundError)
	}

	unitStatus, _ := f.Units[name]

	unitStatus.Current = unitStateLaunched
	unitStatus.Desired = unitStateLaunched

	f.Units[name] = unitStatus

	return nil
}

// Stop sets the Current and Desired state of the stored UnitStatus
// to unitStateLoaded.
func (f *DummyFleet) Stop(ctx context.Context, name string) error {
	f.Config.Logger.Debug(ctx, "dummy fleet: stop %v", name)

	f.Mutex.Lock()
	defer f.Mutex.Unlock()

	if _, ok := f.Units[name]; !ok {
		return maskAny(unitNotFoundError)
	}

	unitStatus, _ := f.Units[name]

	unitStatus.Current = unitStateLoaded
	unitStatus.Desired = unitStateLoaded

	f.Units[name] = unitStatus

	return nil
}

// Destroy remove the UnitStatus from the internal store.
func (f *DummyFleet) Destroy(ctx context.Context, name string) error {
	f.Config.Logger.Debug(ctx, "dummy fleet: destroy %v", name)

	f.Mutex.Lock()
	defer f.Mutex.Unlock()

	if _, ok := f.Units[name]; !ok {
		return maskAny(unitNotFoundError)
	}

	delete(f.Units, name)

	return nil
}

// GetStatus returns the UnitStatus for the given name.
func (f *DummyFleet) GetStatus(ctx context.Context, name string) (UnitStatus, error) {
	f.Config.Logger.Debug(ctx, "dummy fleet: get status %v", name)

	f.Mutex.Lock()
	defer f.Mutex.Unlock()

	unitStatus, ok := f.Units[name]
	if !ok {
		return UnitStatus{}, maskAny(unitNotFoundError)
	}

	return unitStatus, nil
}

// GetStatusWithMatcher returns all UnitStatus that match.
func (f *DummyFleet) GetStatusWithMatcher(m func(string) bool) ([]UnitStatus, error) {
	f.Config.Logger.Debug(context.Background(), "dummy fleet: get status with matcher")

	f.Mutex.Lock()
	defer f.Mutex.Unlock()

	if len(f.Units) == 0 {
		return []UnitStatus{}, maskAny(unitNotFoundError)
	}

	unitStatusList := []UnitStatus{}
	for _, unitStatus := range f.Units {
		if m(unitStatus.Name) {
			unitStatusList = append(unitStatusList, unitStatus)
		}
	}

	return unitStatusList, nil
}
<|endoftext|>


<commit_before>package storage

import (
	"context"
	"fmt"
	"io"
	"sync"
	"time"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/operation"
	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
	"github.com/chrislusf/seaweedfs/weed/storage/erasure_coding"
	"github.com/chrislusf/seaweedfs/weed/storage/needle"
	"github.com/klauspost/reedsolomon"
)

func (s *Store) CollectErasureCodingHeartbeat() *master_pb.Heartbeat {
	var ecShardMessages []*master_pb.VolumeEcShardInformationMessage
	for _, location := range s.Locations {
		location.ecVolumesLock.RLock()
		for _, ecShards := range location.ecVolumes {
			ecShardMessages = append(ecShardMessages, ecShards.ToVolumeEcShardInformationMessage()...)
		}
		location.ecVolumesLock.RUnlock()
	}

	return &master_pb.Heartbeat{
		EcShards: ecShardMessages,
	}

}

func (s *Store) MountEcShards(collection string, vid needle.VolumeId, shardId erasure_coding.ShardId) error {
	for _, location := range s.Locations {
		if err := location.LoadEcShard(collection, vid, shardId); err == nil {
			glog.V(0).Infof("MountEcShards %d.%d", vid, shardId)

			var shardBits erasure_coding.ShardBits

			s.NewEcShardsChan <- master_pb.VolumeEcShardInformationMessage{
				Id:          uint32(vid),
				Collection:  collection,
				EcIndexBits: uint32(shardBits.AddShardId(shardId)),
			}
			return nil
		}
	}

	return fmt.Errorf("MountEcShards %d.%d not found on disk", vid, shardId)
}

func (s *Store) UnmountEcShards(vid needle.VolumeId, shardId erasure_coding.ShardId) error {

	ecShard, found := s.findEcShard(vid, shardId)
	if !found {
		return nil
	}

	var shardBits erasure_coding.ShardBits
	message := master_pb.VolumeEcShardInformationMessage{
		Id:          uint32(vid),
		Collection:  ecShard.Collection,
		EcIndexBits: uint32(shardBits.AddShardId(shardId)),
	}

	for _, location := range s.Locations {
		if deleted := location.UnloadEcShard(vid, shardId); deleted {
			glog.V(0).Infof("UnmountEcShards %d.%d", vid, shardId)
			s.DeletedEcShardsChan <- message
			return nil
		}
	}

	return fmt.Errorf("UnmountEcShards %d.%d not found on disk", vid, shardId)
}

func (s *Store) findEcShard(vid needle.VolumeId, shardId erasure_coding.ShardId) (*erasure_coding.EcVolumeShard, bool) {
	for _, location := range s.Locations {
		if v, found := location.FindEcShard(vid, shardId); found {
			return v, found
		}
	}
	return nil, false
}

func (s *Store) FindEcVolume(vid needle.VolumeId) (*erasure_coding.EcVolume, bool) {
	for _, location := range s.Locations {
		if s, found := location.FindEcVolume(vid); found {
			return s, true
		}
	}
	return nil, false
}

func (s *Store) ReadEcShardNeedle(ctx context.Context, vid needle.VolumeId, n *needle.Needle) (int, error) {
	for _, location := range s.Locations {
		if localEcVolume, found := location.FindEcVolume(vid); found {

			// TODO need to read the version
			version := needle.CurrentVersion

			offset, size, intervals, err := localEcVolume.LocateEcShardNeedle(n, version)
			if err != nil {
				return 0, err
			}

			glog.V(4).Infof("read ec volume %d offset %d size %d intervals:%+v", vid, offset.ToAcutalOffset(), size, intervals)

			if len(intervals) > 1 {
				glog.V(4).Infof("ReadEcShardNeedle needle id %s intervals:%+v", n.String(), intervals)
			}
			bytes, err := s.readEcShardIntervals(ctx, vid, localEcVolume, intervals)
			if err != nil {
				return 0, fmt.Errorf("ReadEcShardIntervals: %v", err)
			}

			err = n.ReadBytes(bytes, offset.ToAcutalOffset(), size, version)
			if err != nil {
				return 0, fmt.Errorf("readbytes: %v", err)
			}

			return len(bytes), nil
		}
	}
	return 0, fmt.Errorf("ec shard %d not found", vid)
}

func (s *Store) readEcShardIntervals(ctx context.Context, vid needle.VolumeId, ecVolume *erasure_coding.EcVolume, intervals []erasure_coding.Interval) (data []byte, err error) {

	if err = s.cachedLookupEcShardLocations(ctx, ecVolume); err != nil {
		return nil, fmt.Errorf("failed to locate shard via master grpc %s: %v", s.MasterAddress, err)
	}

	for i, interval := range intervals {
		if d, e := s.readOneEcShardInterval(ctx, ecVolume, interval); e != nil {
			return nil, e
		} else {
			if i == 0 {
				data = d
			} else {
				data = append(data, d...)
			}
		}
	}
	return
}

func (s *Store) readOneEcShardInterval(ctx context.Context, ecVolume *erasure_coding.EcVolume, interval erasure_coding.Interval) (data []byte, err error) {
	shardId, actualOffset := interval.ToShardIdAndOffset(erasure_coding.ErasureCodingLargeBlockSize, erasure_coding.ErasureCodingSmallBlockSize)
	data = make([]byte, interval.Size)
	if shard, found := ecVolume.FindEcVolumeShard(shardId); found {
		if _, err = shard.ReadAt(data, actualOffset); err != nil {
			glog.V(0).Infof("read local ec shard %d.%d: %v", ecVolume.VolumeId, shardId, err)
			return
		}
	} else {
		ecVolume.ShardLocationsLock.RLock()
		sourceDataNodes, _ := ecVolume.ShardLocations[shardId]
		ecVolume.ShardLocationsLock.RUnlock()

		// try reading directly
		_, err = s.readRemoteEcShardInterval(ctx, sourceDataNodes, ecVolume.VolumeId, shardId, data, actualOffset)
		if err == nil {
			return
		}

		// try reading by recovering from other shards
		_, err = s.recoverOneRemoteEcShardInterval(ctx, ecVolume, shardId, data, actualOffset)
		if err == nil {
			return
		}
		glog.V(0).Infof("recover ec shard %d.%d : %v", ecVolume.VolumeId, shardId, err)
	}
	return
}

func (s *Store) cachedLookupEcShardLocations(ctx context.Context, ecVolume *erasure_coding.EcVolume) (err error) {

	if ecVolume.ShardLocationsRefreshTime.Add(10 * time.Minute).After(time.Now()) {
		// still fresh
		return nil
	}

	glog.V(3).Infof("lookup and cache ec volume %d locations", ecVolume.VolumeId)

	err = operation.WithMasterServerClient(s.MasterAddress, s.grpcDialOption, func(masterClient master_pb.SeaweedClient) error {
		req := &master_pb.LookupEcVolumeRequest{
			VolumeId: uint32(ecVolume.VolumeId),
		}
		resp, err := masterClient.LookupEcVolume(ctx, req)
		if err != nil {
			return fmt.Errorf("lookup ec volume %d: %v", ecVolume.VolumeId, err)
		}

		ecVolume.ShardLocationsLock.Lock()
		for _, shardIdLocations := range resp.ShardIdLocations {
			shardId := erasure_coding.ShardId(shardIdLocations.ShardId)
			delete(ecVolume.ShardLocations, shardId)
			for _, loc := range shardIdLocations.Locations {
				ecVolume.ShardLocations[shardId] = append(ecVolume.ShardLocations[shardId], loc.Url)
			}
		}
		ecVolume.ShardLocationsRefreshTime = time.Now()
		ecVolume.ShardLocationsLock.Unlock()

		return nil
	})
	return
}

func (s *Store) readRemoteEcShardInterval(ctx context.Context, sourceDataNodes []string, vid needle.VolumeId, shardId erasure_coding.ShardId, buf []byte, offset int64) (n int, err error) {

	if len(sourceDataNodes) == 0 {
		return 0, fmt.Errorf("failed to find ec shard %d.%d", vid, shardId)
	}

	for _, sourceDataNode := range sourceDataNodes {
		glog.V(4).Infof("read remote ec shard %d.%d from %s", vid, shardId, sourceDataNode)
		n, err = s.doReadRemoteEcShardInterval(ctx, sourceDataNode, vid, shardId, buf, offset)
		if err == nil {
			return
		}
		glog.V(1).Infof("read remote ec shard %d.%d from %s: %v", vid, shardId, sourceDataNode, err)
	}

	return
}

func (s *Store) doReadRemoteEcShardInterval(ctx context.Context, sourceDataNode string, vid needle.VolumeId, shardId erasure_coding.ShardId, buf []byte, offset int64) (n int, err error) {

	err = operation.WithVolumeServerClient(sourceDataNode, s.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {

		// copy data slice
		shardReadClient, err := client.VolumeEcShardRead(ctx, &volume_server_pb.VolumeEcShardReadRequest{
			VolumeId: uint32(vid),
			ShardId:  uint32(shardId),
			Offset:   offset,
			Size:     int64(len(buf)),
		})
		if err != nil {
			return fmt.Errorf("failed to start reading ec shard %d.%d from %s: %v", vid, shardId, sourceDataNode, err)
		}

		for {
			resp, receiveErr := shardReadClient.Recv()
			if receiveErr == io.EOF {
				break
			}
			if receiveErr != nil {
				return fmt.Errorf("receiving ec shard %d.%d from %s: %v", vid, shardId, sourceDataNode, err)
			}
			copy(buf[n:n+len(resp.Data)], resp.Data)
			n += len(resp.Data)
		}

		return nil
	})
	if err != nil {
		return 0, fmt.Errorf("read ec shard %d.%d from %s: %v", vid, shardId, sourceDataNode, err)
	}

	return
}

func (s *Store) recoverOneRemoteEcShardInterval(ctx context.Context, ecVolume *erasure_coding.EcVolume, shardIdToRecover erasure_coding.ShardId, buf []byte, offset int64) (n int, err error) {
	glog.V(1).Infof("recover ec shard %d.%d from other locations", ecVolume.VolumeId, shardIdToRecover)

	enc, err := reedsolomon.New(erasure_coding.DataShardsCount, erasure_coding.ParityShardsCount)
	if err != nil {
		return 0, fmt.Errorf("failed to create encoder: %v", err)
	}

	bufs := make([][]byte, erasure_coding.TotalShardsCount)

	var wg sync.WaitGroup
	ecVolume.ShardLocationsLock.RLock()
	for shardId, locations := range ecVolume.ShardLocations {

		// skip currnent shard or empty shard
		if shardId == shardIdToRecover {
			continue
		}
		if len(locations) == 0 {
			glog.V(3).Infof("readRemoteEcShardInterval missing %d.%d from %+v", ecVolume.VolumeId, shardId, locations)
			continue
		}

		// read from remote locations
		wg.Add(1)
		go func(shardId erasure_coding.ShardId, locations []string) {
			defer wg.Done()
			data := make([]byte, len(buf))
			nRead, readErr := s.readRemoteEcShardInterval(ctx, locations, ecVolume.VolumeId, shardId, data, offset)
			if readErr != nil {
				glog.V(4).Infof("recover: readRemoteEcShardInterval %d.%d %d bytes from %+v: %v", ecVolume.VolumeId, shardId, nRead, locations, readErr)
			}
			if nRead == len(buf) {
				bufs[shardId] = data
			}
		}(shardId, locations)
	}
	ecVolume.ShardLocationsLock.RUnlock()

	wg.Wait()

	if err = enc.ReconstructData(bufs); err != nil {
		glog.V(3).Infof("recovered ec shard %d.%d failed: %v", ecVolume.VolumeId, shardIdToRecover, err)
		return 0, err
	}
	glog.V(3).Infof("recovered ec shard %d.%d from other locations", ecVolume.VolumeId, shardIdToRecover)

	copy(buf, bufs[shardIdToRecover])

	return len(buf), nil
}
<commit_msg>forget shards that are broken<commit_after>package storage

import (
	"context"
	"fmt"
	"io"
	"sync"
	"time"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/operation"
	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
	"github.com/chrislusf/seaweedfs/weed/storage/erasure_coding"
	"github.com/chrislusf/seaweedfs/weed/storage/needle"
	"github.com/klauspost/reedsolomon"
)

func (s *Store) CollectErasureCodingHeartbeat() *master_pb.Heartbeat {
	var ecShardMessages []*master_pb.VolumeEcShardInformationMessage
	for _, location := range s.Locations {
		location.ecVolumesLock.RLock()
		for _, ecShards := range location.ecVolumes {
			ecShardMessages = append(ecShardMessages, ecShards.ToVolumeEcShardInformationMessage()...)
		}
		location.ecVolumesLock.RUnlock()
	}

	return &master_pb.Heartbeat{
		EcShards: ecShardMessages,
	}

}

func (s *Store) MountEcShards(collection string, vid needle.VolumeId, shardId erasure_coding.ShardId) error {
	for _, location := range s.Locations {
		if err := location.LoadEcShard(collection, vid, shardId); err == nil {
			glog.V(0).Infof("MountEcShards %d.%d", vid, shardId)

			var shardBits erasure_coding.ShardBits

			s.NewEcShardsChan <- master_pb.VolumeEcShardInformationMessage{
				Id:          uint32(vid),
				Collection:  collection,
				EcIndexBits: uint32(shardBits.AddShardId(shardId)),
			}
			return nil
		}
	}

	return fmt.Errorf("MountEcShards %d.%d not found on disk", vid, shardId)
}

func (s *Store) UnmountEcShards(vid needle.VolumeId, shardId erasure_coding.ShardId) error {

	ecShard, found := s.findEcShard(vid, shardId)
	if !found {
		return nil
	}

	var shardBits erasure_coding.ShardBits
	message := master_pb.VolumeEcShardInformationMessage{
		Id:          uint32(vid),
		Collection:  ecShard.Collection,
		EcIndexBits: uint32(shardBits.AddShardId(shardId)),
	}

	for _, location := range s.Locations {
		if deleted := location.UnloadEcShard(vid, shardId); deleted {
			glog.V(0).Infof("UnmountEcShards %d.%d", vid, shardId)
			s.DeletedEcShardsChan <- message
			return nil
		}
	}

	return fmt.Errorf("UnmountEcShards %d.%d not found on disk", vid, shardId)
}

func (s *Store) findEcShard(vid needle.VolumeId, shardId erasure_coding.ShardId) (*erasure_coding.EcVolumeShard, bool) {
	for _, location := range s.Locations {
		if v, found := location.FindEcShard(vid, shardId); found {
			return v, found
		}
	}
	return nil, false
}

func (s *Store) FindEcVolume(vid needle.VolumeId) (*erasure_coding.EcVolume, bool) {
	for _, location := range s.Locations {
		if s, found := location.FindEcVolume(vid); found {
			return s, true
		}
	}
	return nil, false
}

func (s *Store) ReadEcShardNeedle(ctx context.Context, vid needle.VolumeId, n *needle.Needle) (int, error) {
	for _, location := range s.Locations {
		if localEcVolume, found := location.FindEcVolume(vid); found {

			// TODO need to read the version
			version := needle.CurrentVersion

			offset, size, intervals, err := localEcVolume.LocateEcShardNeedle(n, version)
			if err != nil {
				return 0, err
			}

			glog.V(4).Infof("read ec volume %d offset %d size %d intervals:%+v", vid, offset.ToAcutalOffset(), size, intervals)

			if len(intervals) > 1 {
				glog.V(4).Infof("ReadEcShardNeedle needle id %s intervals:%+v", n.String(), intervals)
			}
			bytes, err := s.readEcShardIntervals(ctx, vid, localEcVolume, intervals)
			if err != nil {
				return 0, fmt.Errorf("ReadEcShardIntervals: %v", err)
			}

			err = n.ReadBytes(bytes, offset.ToAcutalOffset(), size, version)
			if err != nil {
				return 0, fmt.Errorf("readbytes: %v", err)
			}

			return len(bytes), nil
		}
	}
	return 0, fmt.Errorf("ec shard %d not found", vid)
}

func (s *Store) readEcShardIntervals(ctx context.Context, vid needle.VolumeId, ecVolume *erasure_coding.EcVolume, intervals []erasure_coding.Interval) (data []byte, err error) {

	if err = s.cachedLookupEcShardLocations(ctx, ecVolume); err != nil {
		return nil, fmt.Errorf("failed to locate shard via master grpc %s: %v", s.MasterAddress, err)
	}

	for i, interval := range intervals {
		if d, e := s.readOneEcShardInterval(ctx, ecVolume, interval); e != nil {
			return nil, e
		} else {
			if i == 0 {
				data = d
			} else {
				data = append(data, d...)
			}
		}
	}
	return
}

func (s *Store) readOneEcShardInterval(ctx context.Context, ecVolume *erasure_coding.EcVolume, interval erasure_coding.Interval) (data []byte, err error) {
	shardId, actualOffset := interval.ToShardIdAndOffset(erasure_coding.ErasureCodingLargeBlockSize, erasure_coding.ErasureCodingSmallBlockSize)
	data = make([]byte, interval.Size)
	if shard, found := ecVolume.FindEcVolumeShard(shardId); found {
		if _, err = shard.ReadAt(data, actualOffset); err != nil {
			glog.V(0).Infof("read local ec shard %d.%d: %v", ecVolume.VolumeId, shardId, err)
			return
		}
	} else {
		ecVolume.ShardLocationsLock.RLock()
		sourceDataNodes, hasShardIdLocation := ecVolume.ShardLocations[shardId]
		ecVolume.ShardLocationsLock.RUnlock()

		// try reading directly
		if hasShardIdLocation {
			_, err = s.readRemoteEcShardInterval(ctx, sourceDataNodes, ecVolume.VolumeId, shardId, data, actualOffset)
			if err == nil {
				return
			}
			glog.V(0).Infof("clearing ec shard %d.%d locations: %v", ecVolume.VolumeId, shardId, err)
			forgetShardId(ecVolume, shardId)
		}

		// try reading by recovering from other shards
		_, err = s.recoverOneRemoteEcShardInterval(ctx, ecVolume, shardId, data, actualOffset)
		if err == nil {
			return
		}
		glog.V(0).Infof("recover ec shard %d.%d : %v", ecVolume.VolumeId, shardId, err)
	}
	return
}

func forgetShardId(ecVolume *erasure_coding.EcVolume, shardId erasure_coding.ShardId) {
	// failed to access the source data nodes, clear it up
	ecVolume.ShardLocationsLock.Lock()
	delete(ecVolume.ShardLocations, shardId)
	ecVolume.ShardLocationsLock.Unlock()
}

func (s *Store) cachedLookupEcShardLocations(ctx context.Context, ecVolume *erasure_coding.EcVolume) (err error) {

	if ecVolume.ShardLocationsRefreshTime.Add(10 * time.Minute).After(time.Now()) {
		// still fresh
		return nil
	}

	glog.V(3).Infof("lookup and cache ec volume %d locations", ecVolume.VolumeId)

	err = operation.WithMasterServerClient(s.MasterAddress, s.grpcDialOption, func(masterClient master_pb.SeaweedClient) error {
		req := &master_pb.LookupEcVolumeRequest{
			VolumeId: uint32(ecVolume.VolumeId),
		}
		resp, err := masterClient.LookupEcVolume(ctx, req)
		if err != nil {
			return fmt.Errorf("lookup ec volume %d: %v", ecVolume.VolumeId, err)
		}

		ecVolume.ShardLocationsLock.Lock()
		for _, shardIdLocations := range resp.ShardIdLocations {
			shardId := erasure_coding.ShardId(shardIdLocations.ShardId)
			delete(ecVolume.ShardLocations, shardId)
			for _, loc := range shardIdLocations.Locations {
				ecVolume.ShardLocations[shardId] = append(ecVolume.ShardLocations[shardId], loc.Url)
			}
		}
		ecVolume.ShardLocationsRefreshTime = time.Now()
		ecVolume.ShardLocationsLock.Unlock()

		return nil
	})
	return
}

func (s *Store) readRemoteEcShardInterval(ctx context.Context, sourceDataNodes []string, vid needle.VolumeId, shardId erasure_coding.ShardId, buf []byte, offset int64) (n int, err error) {

	if len(sourceDataNodes) == 0 {
		return 0, fmt.Errorf("failed to find ec shard %d.%d", vid, shardId)
	}

	for _, sourceDataNode := range sourceDataNodes {
		glog.V(4).Infof("read remote ec shard %d.%d from %s", vid, shardId, sourceDataNode)
		n, err = s.doReadRemoteEcShardInterval(ctx, sourceDataNode, vid, shardId, buf, offset)
		if err == nil {
			return
		}
		glog.V(1).Infof("read remote ec shard %d.%d from %s: %v", vid, shardId, sourceDataNode, err)
	}

	return
}

func (s *Store) doReadRemoteEcShardInterval(ctx context.Context, sourceDataNode string, vid needle.VolumeId, shardId erasure_coding.ShardId, buf []byte, offset int64) (n int, err error) {

	err = operation.WithVolumeServerClient(sourceDataNode, s.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {

		// copy data slice
		shardReadClient, err := client.VolumeEcShardRead(ctx, &volume_server_pb.VolumeEcShardReadRequest{
			VolumeId: uint32(vid),
			ShardId:  uint32(shardId),
			Offset:   offset,
			Size:     int64(len(buf)),
		})
		if err != nil {
			return fmt.Errorf("failed to start reading ec shard %d.%d from %s: %v", vid, shardId, sourceDataNode, err)
		}

		for {
			resp, receiveErr := shardReadClient.Recv()
			if receiveErr == io.EOF {
				break
			}
			if receiveErr != nil {
				return fmt.Errorf("receiving ec shard %d.%d from %s: %v", vid, shardId, sourceDataNode, err)
			}
			copy(buf[n:n+len(resp.Data)], resp.Data)
			n += len(resp.Data)
		}

		return nil
	})
	if err != nil {
		return 0, fmt.Errorf("read ec shard %d.%d from %s: %v", vid, shardId, sourceDataNode, err)
	}

	return
}

func (s *Store) recoverOneRemoteEcShardInterval(ctx context.Context, ecVolume *erasure_coding.EcVolume, shardIdToRecover erasure_coding.ShardId, buf []byte, offset int64) (n int, err error) {
	glog.V(4).Infof("recover ec shard %d.%d from other locations", ecVolume.VolumeId, shardIdToRecover)

	enc, err := reedsolomon.New(erasure_coding.DataShardsCount, erasure_coding.ParityShardsCount)
	if err != nil {
		return 0, fmt.Errorf("failed to create encoder: %v", err)
	}

	bufs := make([][]byte, erasure_coding.TotalShardsCount)

	var wg sync.WaitGroup
	ecVolume.ShardLocationsLock.RLock()
	for shardId, locations := range ecVolume.ShardLocations {

		// skip currnent shard or empty shard
		if shardId == shardIdToRecover {
			continue
		}
		if len(locations) == 0 {
			glog.V(3).Infof("readRemoteEcShardInterval missing %d.%d from %+v", ecVolume.VolumeId, shardId, locations)
			continue
		}

		// read from remote locations
		wg.Add(1)
		go func(shardId erasure_coding.ShardId, locations []string) {
			defer wg.Done()
			data := make([]byte, len(buf))
			nRead, readErr := s.readRemoteEcShardInterval(ctx, locations, ecVolume.VolumeId, shardId, data, offset)
			if readErr != nil {
				glog.V(3).Infof("recover: readRemoteEcShardInterval %d.%d %d bytes from %+v: %v", ecVolume.VolumeId, shardId, nRead, locations, readErr)
				forgetShardId(ecVolume, shardId)
			}
			if nRead == len(buf) {
				bufs[shardId] = data
			}
		}(shardId, locations)
	}
	ecVolume.ShardLocationsLock.RUnlock()

	wg.Wait()

	if err = enc.ReconstructData(bufs); err != nil {
		glog.V(3).Infof("recovered ec shard %d.%d failed: %v", ecVolume.VolumeId, shardIdToRecover, err)
		return 0, err
	}
	glog.V(4).Infof("recovered ec shard %d.%d from other locations", ecVolume.VolumeId, shardIdToRecover)

	copy(buf, bufs[shardIdToRecover])

	return len(buf), nil
}
<|endoftext|>


<commit_before>package cli

import (
	"fmt"
	"os"
	"text/tabwriter"

	"github.com/codegangsta/cli"

	"github.com/libopenstorage/gossip/types"
	"github.com/libopenstorage/openstorage/api"
	"github.com/libopenstorage/openstorage/api/client"
	"github.com/libopenstorage/openstorage/cluster"
)

type clusterClient struct {
	manager cluster.Cluster
}

func (c *clusterClient) clusterOptions(context *cli.Context) {
	clnt, err := client.NewClusterClient()
	if err != nil {
		fmt.Printf("Failed to initialize client library: %v\n", err)
		os.Exit(1)
	}
	c.manager = clnt.ClusterManager()
}

func (c *clusterClient) status(context *cli.Context) {
	c.clusterOptions(context)
	jsonOut := context.GlobalBool("json")
	outFd := os.Stdout
	fn := "status"

	cluster, err := c.manager.Enumerate()
	if err != nil {
		cmdError(context, fn, err)
		return
	}

	if jsonOut {
		fmtOutput(context, &Format{Cluster: &cluster})
	} else {
		fmt.Fprintf(outFd, "ID %s: Status: %v\n",
			cluster.Id, cluster.Status)

		w := new(tabwriter.Writer)
		w.Init(outFd, 12, 12, 1, ' ', 0)

		fmt.Fprintln(w, "ID\t IP\t STATUS\t CPU\t MEMORY\t CONTAINERS")
		for _, n := range cluster.Nodes {
			status := ""
			if n.Status == api.StatusInit {
				status = "Initializing"
			} else if n.Status == api.StatusOk {
				status = "OK"
			} else if n.Status == api.StatusOffline {
				status = "Off Line"
			} else {
				status = "Error"
			}

			fmt.Fprintln(w, n.Id, "\t", n.Ip, "\t", status, "\t",
				n.Cpu, "\t", n.Memory, "\t", len(n.Containers))
		}

		fmt.Fprintln(w)
		w.Flush()
	}
}

func (c *clusterClient) load(context *cli.Context) {
	c.clusterOptions(context)
	jsonOut := context.GlobalBool("json")
	outFd := os.Stdout
	fn := "load"

	cluster, err := c.manager.Enumerate()
	if err != nil {
		cmdError(context, fn, err)
		return
	}

	if jsonOut {
		fmtOutput(context, &Format{Cluster: &cluster})
	} else {
		w := new(tabwriter.Writer)
		w.Init(outFd, 12, 12, 1, ' ', 0)

		fmt.Fprintln(w, "ID\t IMAGE\t STATUS\t NAMES\t NODE")
		for _, n := range cluster.Nodes {
			for _, c := range n.Containers {
				fmt.Fprintln(w, c.ID, "\t", c.Image, "\t", c.Status, "\t",
					c.Names, "\t", n.Ip)
			}
		}

		fmt.Fprintln(w)
		w.Flush()
	}
}

func (c *clusterClient) remove(context *cli.Context) {
}

func (c *clusterClient) shutdown(context *cli.Context) {
}

func (c *clusterClient) disableGossip(context *cli.Context) {
	c.clusterOptions(context)
	c.manager.DisableUpdates()
}

func (c *clusterClient) enableGossip(context *cli.Context) {
	c.clusterOptions(context)
	c.manager.EnableUpdates()
}

func (c *clusterClient) gossipStatus(context *cli.Context) {
	c.clusterOptions(context)
	jsonOut := context.GlobalBool("json")
	outFd := os.Stdout
	fn := "gossipstatus"

	s := c.manager.GetState()
	if s == nil {
		cmdError(context, fn, fmt.Errorf("Failed to get status"))
		return
	}

	if jsonOut {
		fmtOutput(context, &Format{Result: s})
	} else {
		w := new(tabwriter.Writer)
		w.Init(outFd, 12, 12, 1, ' ', 0)

		fmt.Fprintln(w, "ID\t LAST CONTACT TS\t DIR\t Errors")
		for _, n := range s.History {
			dirStr := "From Peer"
			if n.Dir == types.GD_ME_TO_PEER {
				dirStr = "To Peer"
			}
			fmt.Fprintln(w, n.Node, "\t", n.Ts, "\t", dirStr, "\t", n.Err)
		}

		fmt.Fprintln(w)
		w.Flush()

		fmt.Println("Individual Node Status")
		w = new(tabwriter.Writer)
		w.Init(outFd, 12, 12, 1, ' ', 0)

		fmt.Fprintln(w, "ID\t LAST UPDATE TS\t STATUS")
		for _, n := range s.NodeStatus {
			statusStr := "Up"
			switch {
			case n.Status == types.NODE_STATUS_DOWN,
				n.Status == types.NODE_STATUS_DOWN_WAITING_FOR_NEW_UPDATE:
				statusStr = "Down"
			case n.Status == types.NODE_STATUS_INVALID:
				statusStr = "Invalid"
			case n.Status == types.NODE_STATUS_NEVER_GOSSIPED:
				statusStr = "Node not yet gossiped"
			case n.Status == types.NODE_STATUS_WAITING_FOR_NEW_UPDATE:
				statusStr = "Waiting for new data with new generation"
			}
			fmt.Fprintln(w, n.Id, "\t", n.LastUpdateTs, "\t", statusStr)
		}

		fmt.Fprintln(w)
		w.Flush()
	}
}

// ClusterCommands exports CLI comamnds for File VolumeDriver
func ClusterCommands() []cli.Command {
	c := &clusterClient{}

	commands := []cli.Command{
		{
			Name:    "status",
			Aliases: []string{"s"},
			Usage:   "Inspect the cluster",
			Action:  c.status,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "machine,m",
					Usage: "Comma separated machine ids, e.g uuid1,uuid2",
					Value: "",
				},
			},
		},
		{
			Name:    "load",
			Aliases: []string{"l"},
			Usage:   "List containers in the cluster",
			Action:  c.load,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "machine,m",
					Usage: "Comma separated machine ids, e.g uuid1,uuid2",
					Value: "",
				},
			},
		},
		{
			Name:    "disable-gossip",
			Aliases: []string{"dg"},
			Usage:   "Disable gossip updates",
			Action:  c.disableGossip,
		},
		{
			Name:    "enable-gossip",
			Aliases: []string{"eg"},
			Usage:   "Enable gossip updates",
			Action:  c.enableGossip,
		},
		{
			Name:    "gossip-status",
			Aliases: []string{"gs"},
			Usage:   "Display gossip status",
			Action:  c.gossipStatus,
		},
		{
			Name:    "remove",
			Aliases: []string{"r"},
			Usage:   "Remove a machine from the cluster",
			Action:  c.remove,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "machine,m",
					Usage: "Comma separated machine ids, e.g uuid1,uuid2",
					Value: "",
				},
			},
		},
		{
			Name:   "shutdown",
			Usage:  "Shutdown a cluster or a specific machine",
			Action: c.shutdown,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "machine,m",
					Usage: "Comma separated machine ids, e.g uuid1,uuid2",
					Value: "",
				},
			},
		},
	}
	return commands
}
<commit_msg>Cleanup on CLI output<commit_after>package cli

import (
	"fmt"
	"os"
	"text/tabwriter"

	"github.com/codegangsta/cli"

	"github.com/libopenstorage/gossip/types"
	"github.com/libopenstorage/openstorage/api"
	"github.com/libopenstorage/openstorage/api/client"
	"github.com/libopenstorage/openstorage/cluster"
)

type clusterClient struct {
	manager cluster.Cluster
}

func (c *clusterClient) clusterOptions(context *cli.Context) {
	clnt, err := client.NewClusterClient()
	if err != nil {
		fmt.Printf("Failed to initialize client library: %v\n", err)
		os.Exit(1)
	}
	c.manager = clnt.ClusterManager()
}

func (c *clusterClient) status(context *cli.Context) {
	c.clusterOptions(context)
	jsonOut := context.GlobalBool("json")
	outFd := os.Stdout
	fn := "status"

	cluster, err := c.manager.Enumerate()
	if err != nil {
		cmdError(context, fn, err)
		return
	}

	if jsonOut {
		fmtOutput(context, &Format{Cluster: &cluster})
	} else {
		fmt.Fprintf(outFd, "Cluster Information:\nCluster ID %s: Status: %v\n\n",
			cluster.Id, cluster.Status)

		fmt.Fprintf(outFd, "Load Information:\n")
		w := new(tabwriter.Writer)
		w.Init(outFd, 12, 12, 1, ' ', 0)

		fmt.Fprintln(w, "ID\t IP\t STATUS\t CPU\t MEMORY\t CONTAINERS")
		for _, n := range cluster.Nodes {
			status := ""
			if n.Status == api.StatusInit {
				status = "Initializing"
			} else if n.Status == api.StatusOk {
				status = "OK"
			} else if n.Status == api.StatusOffline {
				status = "Off Line"
			} else {
				status = "Error"
			}

			fmt.Fprintln(w, n.Id, "\t", n.Ip, "\t", status, "\t",
				n.Cpu, "\t", n.Memory, "\t", len(n.Containers))
		}

		fmt.Fprintln(w)
		w.Flush()
	}
}

func (c *clusterClient) load(context *cli.Context) {
	c.clusterOptions(context)
	jsonOut := context.GlobalBool("json")
	outFd := os.Stdout
	fn := "load"

	cluster, err := c.manager.Enumerate()
	if err != nil {
		cmdError(context, fn, err)
		return
	}

	if jsonOut {
		fmtOutput(context, &Format{Cluster: &cluster})
	} else {
		w := new(tabwriter.Writer)
		w.Init(outFd, 12, 12, 1, ' ', 0)

		fmt.Fprintln(w, "ID\t IMAGE\t STATUS\t NAMES\t NODE")
		for _, n := range cluster.Nodes {
			for _, c := range n.Containers {
				fmt.Fprintln(w, c.ID, "\t", c.Image, "\t", c.Status, "\t",
					c.Names, "\t", n.Ip)
			}
		}

		fmt.Fprintln(w)
		w.Flush()
	}
}

func (c *clusterClient) remove(context *cli.Context) {
}

func (c *clusterClient) shutdown(context *cli.Context) {
}

func (c *clusterClient) disableGossip(context *cli.Context) {
	c.clusterOptions(context)
	c.manager.DisableUpdates()
}

func (c *clusterClient) enableGossip(context *cli.Context) {
	c.clusterOptions(context)
	c.manager.EnableUpdates()
}

func (c *clusterClient) gossipStatus(context *cli.Context) {
	c.clusterOptions(context)
	jsonOut := context.GlobalBool("json")
	outFd := os.Stdout
	fn := "gossipstatus"

	s := c.manager.GetState()
	if s == nil {
		cmdError(context, fn, fmt.Errorf("Failed to get status"))
		return
	}

	if jsonOut {
		fmtOutput(context, &Format{Result: s})
	} else {
		w := new(tabwriter.Writer)
		w.Init(outFd, 12, 12, 1, ' ', 0)

		fmt.Fprintln(w, "ID\t LAST CONTACT TS\t DIR\t Errors")
		for _, n := range s.History {
			dirStr := "From Peer"
			if n.Dir == types.GD_ME_TO_PEER {
				dirStr = "To Peer"
			}
			fmt.Fprintln(w, n.Node, "\t", n.Ts, "\t", dirStr, "\t", n.Err)
		}

		fmt.Fprintln(w)
		w.Flush()

		fmt.Println("Individual Node Status")
		w = new(tabwriter.Writer)
		w.Init(outFd, 12, 12, 1, ' ', 0)

		fmt.Fprintln(w, "ID\t LAST UPDATE TS\t STATUS")
		for _, n := range s.NodeStatus {
			statusStr := "Up"
			switch {
			case n.Status == types.NODE_STATUS_DOWN,
				n.Status == types.NODE_STATUS_DOWN_WAITING_FOR_NEW_UPDATE:
				statusStr = "Down"
			case n.Status == types.NODE_STATUS_INVALID:
				statusStr = "Invalid"
			case n.Status == types.NODE_STATUS_NEVER_GOSSIPED:
				statusStr = "Node not yet gossiped"
			case n.Status == types.NODE_STATUS_WAITING_FOR_NEW_UPDATE:
				statusStr = "Waiting for new data with new generation"
			}
			fmt.Fprintln(w, n.Id, "\t", n.LastUpdateTs, "\t", statusStr)
		}

		fmt.Fprintln(w)
		w.Flush()
	}
}

// ClusterCommands exports CLI comamnds for File VolumeDriver
func ClusterCommands() []cli.Command {
	c := &clusterClient{}

	commands := []cli.Command{
		{
			Name:    "status",
			Aliases: []string{"s"},
			Usage:   "Inspect the cluster",
			Action:  c.status,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "machine,m",
					Usage: "Comma separated machine ids, e.g uuid1,uuid2",
					Value: "",
				},
			},
		},
		{
			Name:    "load",
			Aliases: []string{"l"},
			Usage:   "List containers in the cluster",
			Action:  c.load,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "machine,m",
					Usage: "Comma separated machine ids, e.g uuid1,uuid2",
					Value: "",
				},
			},
		},
		{
			Name:    "disable-gossip",
			Aliases: []string{"dg"},
			Usage:   "Disable gossip updates",
			Action:  c.disableGossip,
		},
		{
			Name:    "enable-gossip",
			Aliases: []string{"eg"},
			Usage:   "Enable gossip updates",
			Action:  c.enableGossip,
		},
		{
			Name:    "gossip-status",
			Aliases: []string{"gs"},
			Usage:   "Display gossip status",
			Action:  c.gossipStatus,
		},
		{
			Name:    "remove",
			Aliases: []string{"r"},
			Usage:   "Remove a machine from the cluster",
			Action:  c.remove,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "machine,m",
					Usage: "Comma separated machine ids, e.g uuid1,uuid2",
					Value: "",
				},
			},
		},
		{
			Name:   "shutdown",
			Usage:  "Shutdown a cluster or a specific machine",
			Action: c.shutdown,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "machine,m",
					Usage: "Comma separated machine ids, e.g uuid1,uuid2",
					Value: "",
				},
			},
		},
	}
	return commands
}
<|endoftext|>


<commit_before>// Forex data API

// http://finance.yahoo.com/webservice/v1/symbols/CNY=X/quote?format=json

package forex

import (
	"encoding/json"
	"fmt"
	// "github.com/davecgh/go-spew/spew"
	"io/ioutil"
	"net/http"
	"time"
)

// Forex data API URL
const (
	DATAURL = "http://finance.yahoo.com/webservice/v1/symbols/"
)

// Quote forex info
type Quote struct {
	Price  float64
	Symbol string
	Error  error
}

// CommunicateFX sends the latest FX quote to the supplied channel
func CommunicateFX(symbol string, fxChan chan<- Quote, doneChan <-chan bool) error {
	// Check connection is ok to start
	quote := getQuote(symbol)
	if quote.Error != nil {
		return quote.Error
	}
	if quote.Price == 0 {
		return fmt.Errorf("Price is zero")
	}

	// Run read loop in new goroutine
	go runLoop(symbol, fxChan, doneChan)
	return nil
}

// HTTP read loop
func runLoop(symbol string, fxChan chan<- Quote, doneChan <-chan bool) {
	ticker := time.NewTicker(15 * time.Second)

	for {
		select {
		case <-doneChan:
			close(fxChan)
			return
		case <-ticker.C:
			fxChan <- getQuote(symbol)
		}
	}
}

// Returns quote for requested instrument
func getQuote(symbol string) Quote {
	tmp := struct {
		List struct {
			Resources []struct {
				Resource struct {
					Fields struct {
						Price float64 `json:"price,string"`
					} `json:"fields"`
				} `json:"resource"`
			} `json:"resources"`
		} `json:"list"`
	}{}

	url := fmt.Sprintf("%s=x/quote?format=json", symbol)

	data, err := get(url)
	if err != nil {
		return Quote{Error: fmt.Errorf("Forex error %s", err)}
	}

	if err = json.Unmarshal(data, &tmp); err != nil {
		return Quote{Error: fmt.Errorf("Forex error %s", err)}
	}

	return Quote{
		Price:  tmp.List.Resources[0].Resource.Fields.Price,
		Symbol: symbol,
		Error:  nil,
	}
}

// unauthenticated GET
func get(url string) ([]byte, error) {
	resp, err := http.Get(DATAURL + url)
	if err != nil {
		return []byte{}, err
	}
	defer resp.Body.Close()

	return ioutil.ReadAll(resp.Body)
}
<commit_msg>stop ticker on exit<commit_after>// Forex data API

// http://finance.yahoo.com/webservice/v1/symbols/CNY=X/quote?format=json

package forex

import (
	"encoding/json"
	"fmt"
	// "github.com/davecgh/go-spew/spew"
	"io/ioutil"
	"net/http"
	"time"
)

// Forex data API URL
const (
	DATAURL = "http://finance.yahoo.com/webservice/v1/symbols/"
)

// Quote forex info
type Quote struct {
	Price  float64
	Symbol string
	Error  error
}

// CommunicateFX sends the latest FX quote to the supplied channel
func CommunicateFX(symbol string, fxChan chan<- Quote, doneChan <-chan bool) error {
	// Check connection is ok to start
	quote := getQuote(symbol)
	if quote.Error != nil {
		return quote.Error
	}
	if quote.Price == 0 {
		return fmt.Errorf("Price is zero")
	}

	// Run read loop in new goroutine
	go runLoop(symbol, fxChan, doneChan)
	return nil
}

// HTTP read loop
func runLoop(symbol string, fxChan chan<- Quote, doneChan <-chan bool) {
	ticker := time.NewTicker(15 * time.Second)

	for {
		select {
		case <-doneChan:
			ticker.Stop()
			close(fxChan)
			return
		case <-ticker.C:
			fxChan <- getQuote(symbol)
		}
	}
}

// Returns quote for requested instrument
func getQuote(symbol string) Quote {
	tmp := struct {
		List struct {
			Resources []struct {
				Resource struct {
					Fields struct {
						Price float64 `json:"price,string"`
					} `json:"fields"`
				} `json:"resource"`
			} `json:"resources"`
		} `json:"list"`
	}{}

	url := fmt.Sprintf("%s=x/quote?format=json", symbol)

	data, err := get(url)
	if err != nil {
		return Quote{Error: fmt.Errorf("Forex error %s", err)}
	}

	if err = json.Unmarshal(data, &tmp); err != nil {
		return Quote{Error: fmt.Errorf("Forex error %s", err)}
	}

	return Quote{
		Price:  tmp.List.Resources[0].Resource.Fields.Price,
		Symbol: symbol,
		Error:  nil,
	}
}

// unauthenticated GET
func get(url string) ([]byte, error) {
	resp, err := http.Get(DATAURL + url)
	if err != nil {
		return []byte{}, err
	}
	defer resp.Body.Close()

	return ioutil.ReadAll(resp.Body)
}
<|endoftext|>


<commit_before>package goCompany

import "testing"

func TestGetCompanyInfo(t *testing.T) {
	okOGRN := 5990130
	resultOGRN, err := GetCompanyInfo("7736002426")
	if err != nil {
		t.Fatal(err)
	}
	if resultOGRN[0].ID != okOGRN {
		t.Fatalf("Want %v, but got %v", resultOGRN, okOGRN)
	}

	okNAME := 212722
	resultNAME, err := GetCompanyInfo("СТРОЙПРОЕКТ")
	if err != nil {
		t.Fatal(err)
	}
	if resultNAME[0].ID != okNAME {
		t.Fatalf("Want %v, but got %v", resultNAME, okNAME)
	}

	result, err := GetCompanyInfo("1658064460")
	if err != nil {
		t.Fatal(err)
	}
	if result[0].ID != okNAME {
		t.Fatalf("Want %v, but got %v", result, okNAME)
	}
}

func TestGetEmployees(t *testing.T) {
	ok := 149735376
	result, err := GetEmployees("32357")
	if err != nil {
		t.Fatal(err)
	}
	if result[0].ID != ok {
		t.Fatalf("Want %v, but got %v", result, ok)
	}
}

func TestGetIndivEntrep(t *testing.T) {
	ok := 1
	result, _ := GetIndivEntrep("7528374")
	if result[0].ID != ok {
		t.Fatalf("Want %v, but got %v", result, ok)
	}
}

func TestGetCompFounder(t *testing.T) {
	ok := 5545071
	result, _ := GetCompFounder("2191023")
	if result[0].ID != ok {
		t.Fatalf("Want %v, but got %v", result, ok)
	}
}

func TestGetDepComp(t *testing.T) {
	ok := 1425227
	result, _ := GetDepComp("7030")
	if result[0].ID != ok {
		t.Fatalf("Want %v, but got %v", result, ok)
	}
}

func TestGetPerson(t *testing.T) {
	ok := "АЛЕКСЕЙ"
	result, _ := GetPerson("2191023")
	if result.FirstName != ok {
		t.Fatalf("Want %v, but got %v", result, ok)
	}
}

func TestGetPositions(t *testing.T) {
	ok := 147863776
	result, _ := GetPositions("2191023")
	if result[0].ID != ok {
		t.Fatalf("Want %v, but got %v", result, ok)
	}
}

func TestGetIDData(t *testing.T) {
	ok := 7030
	result, _ := GetIDData("7030")
	if result.ID != ok {
		t.Fatalf("Want %v, but got %v", result, ok)
	}
}

func TestGetFounders(t *testing.T) {
	ok := 253175464
	result, _ := GetFounders("7030")
	if result[0].ID != ok {
		t.Fatalf("Want %v, but got %v", result, ok)
	}
}

func TestResponse(t *testing.T) {
	ok := []byte("[ ]")
	result := response("https://ru.rus.company/интеграция/компании/")
	if result[0] != ok[0] {
		t.Fatalf("Want %v, but got %v", result[0], ok[0])
	}
}
<commit_msg>test change<commit_after>package goCompany

import "testing"

func TestGetCompanyInfo(t *testing.T) {
	okOGRN := 5990130
	resultOGRN, err := GetCompanyInfo("7736002426")
	if err != nil {
		t.Fatal(err)
	}
	if resultOGRN[0].ID != okOGRN {
		t.Fatalf("Want %v, but got %v", resultOGRN, okOGRN)
	}

	okNAME := 731085
	resultNAME, err := GetCompanyInfo("СТРОЙПРОЕКТ")
	if err != nil {
		t.Fatal(err)
	}
	if resultNAME[0].ID != okNAME {
		t.Fatalf("Want %v, but got %v", resultNAME[0].ID, okNAME)
	}

	ok := 212722
	result, err := GetCompanyInfo("1658064460")
	if err != nil {
		t.Fatal(err)
	}
	if result[0].ID != ok {
		t.Fatalf("Want %v, but got %v", result[0].ID, ok)
	}
}

func TestGetEmployees(t *testing.T) {
	ok := 149735376
	result, err := GetEmployees("32357")
	if err != nil {
		t.Fatal(err)
	}
	if result[0].ID != ok {
		t.Fatalf("Want %v, but got %v", result, ok)
	}
}

func TestGetIndivEntrep(t *testing.T) {
	ok := 1
	result, _ := GetIndivEntrep("7528374")
	if result[0].ID != ok {
		t.Fatalf("Want %v, but got %v", result, ok)
	}
}

func TestGetCompFounder(t *testing.T) {
	ok := 5545071
	result, _ := GetCompFounder("2191023")
	if result[0].ID != ok {
		t.Fatalf("Want %v, but got %v", result, ok)
	}
}

func TestGetDepComp(t *testing.T) {
	ok := 1425227
	result, _ := GetDepComp("7030")
	if result[0].ID != ok {
		t.Fatalf("Want %v, but got %v", result, ok)
	}
}

func TestGetPerson(t *testing.T) {
	ok := "АЛЕКСЕЙ"
	result, _ := GetPerson("2191023")
	if result.FirstName != ok {
		t.Fatalf("Want %v, but got %v", result, ok)
	}
}

func TestGetPositions(t *testing.T) {
	ok := 147863776
	result, _ := GetPositions("2191023")
	if result[0].ID != ok {
		t.Fatalf("Want %v, but got %v", result, ok)
	}
}

func TestGetIDData(t *testing.T) {
	ok := 7030
	result, _ := GetIDData("7030")
	if result.ID != ok {
		t.Fatalf("Want %v, but got %v", result, ok)
	}
}

func TestGetFounders(t *testing.T) {
	ok := 253175464
	result, _ := GetFounders("7030")
	if result[0].ID != ok {
		t.Fatalf("Want %v, but got %v", result, ok)
	}
}

func TestResponse(t *testing.T) {
	ok := []byte("[ ]")
	result := response("https://ru.rus.company/интеграция/компании/")
	if result[0] != ok[0] {
		t.Fatalf("Want %v, but got %v", result[0], ok[0])
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net/url"
	"os"

	functions "github.com/funcy/functions_go"
	"github.com/urfave/cli"
)

// Version of Functions CLI
var Version = "0.3.35"

func version() cli.Command {
	r := versionCmd{VersionApi: functions.NewVersionApi()}
	return cli.Command{
		Name:   "version",
		Usage:  "displays fn and functions daemon versions",
		Action: r.version,
	}
}

type versionCmd struct {
	*functions.VersionApi
}

func (r *versionCmd) version(c *cli.Context) error {
	apiURL := os.Getenv("API_URL")
	if apiURL == "" {
		apiURL = "http://localhost:8080"
	}

	u, err := url.Parse(apiURL)
	if err != nil {
		return err
	}
	r.Configuration.BasePath = u.String()

	fmt.Println("Client version:", Version)
	v, _, err := r.VersionGet()
	if err != nil {
		return err
	}
	fmt.Println("Server version", v.Version)
	return nil
}
<commit_msg>fn tool: 0.3.36 release [skip ci]<commit_after>package main

import (
	"fmt"
	"net/url"
	"os"

	functions "github.com/funcy/functions_go"
	"github.com/urfave/cli"
)

// Version of Functions CLI
var Version = "0.3.36"

func version() cli.Command {
	r := versionCmd{VersionApi: functions.NewVersionApi()}
	return cli.Command{
		Name:   "version",
		Usage:  "displays fn and functions daemon versions",
		Action: r.version,
	}
}

type versionCmd struct {
	*functions.VersionApi
}

func (r *versionCmd) version(c *cli.Context) error {
	apiURL := os.Getenv("API_URL")
	if apiURL == "" {
		apiURL = "http://localhost:8080"
	}

	u, err := url.Parse(apiURL)
	if err != nil {
		return err
	}
	r.Configuration.BasePath = u.String()

	fmt.Println("Client version:", Version)
	v, _, err := r.VersionGet()
	if err != nil {
		return err
	}
	fmt.Println("Server version", v.Version)
	return nil
}
<|endoftext|>