<commit_before>// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// report is a demo application that displays information about an
// OpenAPI description.
package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"

	"github.com/golang/protobuf/proto"
	"github.com/googleapis/gnostic/printer"

	pb "github.com/googleapis/gnostic/OpenAPIv2"
)

func readDocumentFromFileWithName(filename string) *pb.Document {
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		fmt.Printf("File error: %v\n", err)
		os.Exit(1)
	}
	document := &pb.Document{}
	err = proto.Unmarshal(data, document)
	if err != nil {
		panic(err)
	}
	return document
}

func printDocument(code *printer.Code, document *pb.Document) {
	code.Print("BasePath: %+v", document.BasePath)
	code.Print("Consumes: %+v", document.Consumes)
	code.Print("Definitions:")
	code.Indent()
	if document.Definitions != nil && document.Definitions.AdditionalProperties != nil {
		for _, pair := range document.Definitions.AdditionalProperties {
			code.Print("%s", pair.Name)
			code.Indent()
			printSchema(code, pair.Value)
			code.Outdent()
		}
	}
	code.Outdent()
	code.Print("ExternalDocs: %+v", document.ExternalDocs)
	code.Print("Host: %+v", document.Host)
	if document.Info != nil {
		code.Print("Info:")
		code.Indent()
		code.Print("Title: %s", document.Info.Title)
		code.Print("Description: %s", document.Info.Description)
		code.Print("Version: %s", document.Info.Version)
		code.Print("TermsOfService: %s", document.Info.TermsOfService)
		if document.Info.Contact != nil {
			code.Print("Contact Email: %s", document.Info.Contact.Email)
		}
		if document.Info.License != nil {
			code.Print("License Name: %s", document.Info.License.Name)
			code.Print("License URL: %s", document.Info.License.Url)
		}
		code.Outdent()
	}
	code.Print("Parameters: %+v", document.Parameters)
	code.Print("Paths:")
	code.Indent()
	for _, pair := range document.Paths.Path {
		code.Print("%+v", pair.Name)
		code.Indent()
		v := pair.Value
		if v.Get != nil {
			code.Print("GET")
			code.Indent()
			printOperation(code, v.Get)
			code.Outdent()
		}
		if v.Post != nil {
			code.Print("POST")
			code.Indent()
			printOperation(code, v.Post)
			code.Outdent()
		}
		code.Outdent()
	}
	code.Outdent()
	code.Print("Produces: %+v", document.Produces)
	code.Print("Responses: %+v", document.Responses)
	code.Print("Schemes: %+v", document.Schemes)
	code.Print("Security: %+v", document.Security)
	if document.SecurityDefinitions != nil {
		code.Print("SecurityDefinitions:")
		code.Indent()
		for _, pair := range document.SecurityDefinitions.AdditionalProperties {
			code.Print("%s", pair.Name)
			code.Indent()
			v := pair.Value
			switch t := v.Oneof.(type) {
			default:
				code.Print("unexpected type %T", t) // %T prints whatever type t has
			case *pb.SecurityDefinitionsItem_ApiKeySecurity:
				code.Print("ApiKeySecurity: %+v", t)
			case *pb.SecurityDefinitionsItem_BasicAuthenticationSecurity:
				code.Print("BasicAuthenticationSecurity: %+v", t)
			case *pb.SecurityDefinitionsItem_Oauth2AccessCodeSecurity:
				code.Print("Oauth2AccessCodeSecurity: %+v", t)
			case *pb.SecurityDefinitionsItem_Oauth2ApplicationSecurity:
				code.Print("Oauth2ApplicationSecurity: %+v", t)
			case *pb.SecurityDefinitionsItem_Oauth2ImplicitSecurity:
				code.Print("Oauth2ImplicitSecurity")
				code.Indent()
				code.Print("AuthorizationUrl: %+v", t.Oauth2ImplicitSecurity.AuthorizationUrl)
				code.Print("Flow: %+v", t.Oauth2ImplicitSecurity.Flow)
				code.Print("Scopes:")
				code.Indent()
				for _, pair := range t.Oauth2ImplicitSecurity.Scopes.AdditionalProperties {
					code.Print("%s -> %s", pair.Name, pair.Value)
				}
				code.Outdent()
				code.Outdent()
			case *pb.SecurityDefinitionsItem_Oauth2PasswordSecurity:
				code.Print("Oauth2PasswordSecurity: %+v", t)
			}
			code.Outdent()
		}
		code.Outdent()
	}
	code.Print("Swagger: %+v", document.Swagger)
	code.Print("Tags:")
	code.Indent()
	for _, tag := range document.Tags {
		code.Print("Tag:")
		code.Indent()
		code.Print("Name: %s", tag.Name)
		code.Print("Description: %s", tag.Description)
		code.Print("ExternalDocs: %s", tag.ExternalDocs)
		printVendorExtension(code, tag.VendorExtension)
		code.Outdent()
	}
	code.Outdent()
}

func printOperation(code *printer.Code, operation *pb.Operation) {
	code.Print("Consumes: %+v", operation.Consumes)
	code.Print("Deprecated: %+v", operation.Deprecated)
	code.Print("Description: %+v", operation.Description)
	code.Print("ExternalDocs: %+v", operation.ExternalDocs)
	code.Print("OperationId: %+v", operation.OperationId)
	code.Print("Parameters:")
	code.Indent()
	for _, item := range operation.Parameters {
		switch t := item.Oneof.(type) {
		default:
			code.Print("unexpected type %T", t) // %T prints whatever type t has
		case *pb.ParametersItem_JsonReference:
			code.Print("JsonReference: %+v", t)
		case *pb.ParametersItem_Parameter:
			code.Print("Parameter: %+v", t)
		}
	}
	code.Outdent()
	code.Print("Produces: %+v", operation.Produces)
	code.Print("Responses:")
	code.Indent()
	code.Print("ResponseCode:")
	code.Indent()
	for _, pair := range operation.Responses.ResponseCode {
		code.Print("%s %s", pair.Name, pair.Value)
	}
	code.Outdent()
	printVendorExtension(code, operation.Responses.VendorExtension)
	code.Outdent()
	code.Print("Schemes: %+v", operation.Schemes)
	code.Print("Security: %+v", operation.Security)
	code.Print("Summary: %+v", operation.Summary)
	code.Print("Tags: %+v", operation.Tags)
	printVendorExtension(code, operation.VendorExtension)
}

func printSchema(code *printer.Code, schema *pb.Schema) {
	//code.Print("%+v", schema)
	if schema.Format != "" {
		code.Print("Format: %+v", schema.Format)
	}
	if schema.Properties != nil {
		code.Print("Properties")
		code.Indent()
		for _, pair := range schema.Properties.AdditionalProperties {
			code.Print("%s", pair.Name)
			code.Indent()
			printSchema(code, pair.Value)
			code.Outdent()
		}
		code.Outdent()
	}
	if schema.Type != nil {
		code.Print("Type: %+v", schema.Type)
	}
	if schema.Xml != nil {
		code.Print("Xml: %+v", schema.Xml)
	}
	printVendorExtension(code, schema.VendorExtension)
}

func printVendorExtension(code *printer.Code, vendorExtension []*pb.NamedAny) {
	if len(vendorExtension) > 0 {
		code.Print("VendorExtension: %+v", vendorExtension)
	}
}

func main() {
	flag.Parse()
	args := flag.Args()

	if len(args) != 1 {
		fmt.Printf("Usage: report <file.pb>\n")
		return
	}

	document := readDocumentFromFileWithName(args[0])

	code := &printer.Code{}
	code.Print("API REPORT")
	code.Print("----------")
	printDocument(code, document)
	fmt.Printf("%s", code)
}
<commit_msg>Update apps/report sample to report better errors for non-OpenAPIv2 input.<commit_after>// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// report is a demo application that displays information about an
// OpenAPI description.
package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"

	"github.com/golang/protobuf/proto"
	"github.com/googleapis/gnostic/printer"

	pb "github.com/googleapis/gnostic/OpenAPIv2"
)

func readDocumentFromFileWithName(filename string) (*pb.Document, error) {
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}
	document := &pb.Document{}
	err = proto.Unmarshal(data, document)
	if err != nil {
		return nil, err
	}
	return document, nil
}

func printDocument(code *printer.Code, document *pb.Document) {
	code.Print("BasePath: %+v", document.BasePath)
	code.Print("Consumes: %+v", document.Consumes)
	code.Print("Definitions:")
	code.Indent()
	if document.Definitions != nil && document.Definitions.AdditionalProperties != nil {
		for _, pair := range document.Definitions.AdditionalProperties {
			code.Print("%s", pair.Name)
			code.Indent()
			printSchema(code, pair.Value)
			code.Outdent()
		}
	}
	code.Outdent()
	code.Print("ExternalDocs: %+v", document.ExternalDocs)
	code.Print("Host: %+v", document.Host)
	if document.Info != nil {
		code.Print("Info:")
		code.Indent()
		code.Print("Title: %s", document.Info.Title)
		code.Print("Description: %s", document.Info.Description)
		code.Print("Version: %s", document.Info.Version)
		code.Print("TermsOfService: %s", document.Info.TermsOfService)
		if document.Info.Contact != nil {
			code.Print("Contact Email: %s", document.Info.Contact.Email)
		}
		if document.Info.License != nil {
			code.Print("License Name: %s", document.Info.License.Name)
			code.Print("License URL: %s", document.Info.License.Url)
		}
		code.Outdent()
	}
	code.Print("Parameters: %+v", document.Parameters)
	code.Print("Paths:")
	code.Indent()
	for _, pair := range document.Paths.Path {
		code.Print("%+v", pair.Name)
		code.Indent()
		v := pair.Value
		if v.Get != nil {
			code.Print("GET")
			code.Indent()
			printOperation(code, v.Get)
			code.Outdent()
		}
		if v.Post != nil {
			code.Print("POST")
			code.Indent()
			printOperation(code, v.Post)
			code.Outdent()
		}
		code.Outdent()
	}
	code.Outdent()
	code.Print("Produces: %+v", document.Produces)
	code.Print("Responses: %+v", document.Responses)
	code.Print("Schemes: %+v", document.Schemes)
	code.Print("Security: %+v", document.Security)
	if document.SecurityDefinitions != nil {
		code.Print("SecurityDefinitions:")
		code.Indent()
		for _, pair := range document.SecurityDefinitions.AdditionalProperties {
			code.Print("%s", pair.Name)
			code.Indent()
			v := pair.Value
			switch t := v.Oneof.(type) {
			default:
				code.Print("unexpected type %T", t) // %T prints whatever type t has
			case *pb.SecurityDefinitionsItem_ApiKeySecurity:
				code.Print("ApiKeySecurity: %+v", t)
			case *pb.SecurityDefinitionsItem_BasicAuthenticationSecurity:
				code.Print("BasicAuthenticationSecurity: %+v", t)
			case *pb.SecurityDefinitionsItem_Oauth2AccessCodeSecurity:
				code.Print("Oauth2AccessCodeSecurity: %+v", t)
			case *pb.SecurityDefinitionsItem_Oauth2ApplicationSecurity:
				code.Print("Oauth2ApplicationSecurity: %+v", t)
			case *pb.SecurityDefinitionsItem_Oauth2ImplicitSecurity:
				code.Print("Oauth2ImplicitSecurity")
				code.Indent()
				code.Print("AuthorizationUrl: %+v", t.Oauth2ImplicitSecurity.AuthorizationUrl)
				code.Print("Flow: %+v", t.Oauth2ImplicitSecurity.Flow)
				code.Print("Scopes:")
				code.Indent()
				for _, pair := range t.Oauth2ImplicitSecurity.Scopes.AdditionalProperties {
					code.Print("%s -> %s", pair.Name, pair.Value)
				}
				code.Outdent()
				code.Outdent()
			case *pb.SecurityDefinitionsItem_Oauth2PasswordSecurity:
				code.Print("Oauth2PasswordSecurity: %+v", t)
			}
			code.Outdent()
		}
		code.Outdent()
	}
	code.Print("Swagger: %+v", document.Swagger)
	code.Print("Tags:")
	code.Indent()
	for _, tag := range document.Tags {
		code.Print("Tag:")
		code.Indent()
		code.Print("Name: %s", tag.Name)
		code.Print("Description: %s", tag.Description)
		code.Print("ExternalDocs: %s", tag.ExternalDocs)
		printVendorExtension(code, tag.VendorExtension)
		code.Outdent()
	}
	code.Outdent()
}

func printOperation(code *printer.Code, operation *pb.Operation) {
	code.Print("Consumes: %+v", operation.Consumes)
	code.Print("Deprecated: %+v", operation.Deprecated)
	code.Print("Description: %+v", operation.Description)
	code.Print("ExternalDocs: %+v", operation.ExternalDocs)
	code.Print("OperationId: %+v", operation.OperationId)
	code.Print("Parameters:")
	code.Indent()
	for _, item := range operation.Parameters {
		switch t := item.Oneof.(type) {
		default:
			code.Print("unexpected type %T", t) // %T prints whatever type t has
		case *pb.ParametersItem_JsonReference:
			code.Print("JsonReference: %+v", t)
		case *pb.ParametersItem_Parameter:
			code.Print("Parameter: %+v", t)
		}
	}
	code.Outdent()
	code.Print("Produces: %+v", operation.Produces)
	code.Print("Responses:")
	code.Indent()
	code.Print("ResponseCode:")
	code.Indent()
	for _, pair := range operation.Responses.ResponseCode {
		code.Print("%s %s", pair.Name, pair.Value)
	}
	code.Outdent()
	printVendorExtension(code, operation.Responses.VendorExtension)
	code.Outdent()
	code.Print("Schemes: %+v", operation.Schemes)
	code.Print("Security: %+v", operation.Security)
	code.Print("Summary: %+v", operation.Summary)
	code.Print("Tags: %+v", operation.Tags)
	printVendorExtension(code, operation.VendorExtension)
}

func printSchema(code *printer.Code, schema *pb.Schema) {
	//code.Print("%+v", schema)
	if schema.Format != "" {
		code.Print("Format: %+v", schema.Format)
	}
	if schema.Properties != nil {
		code.Print("Properties")
		code.Indent()
		for _, pair := range schema.Properties.AdditionalProperties {
			code.Print("%s", pair.Name)
			code.Indent()
			printSchema(code, pair.Value)
			code.Outdent()
		}
		code.Outdent()
	}
	if schema.Type != nil {
		code.Print("Type: %+v", schema.Type)
	}
	if schema.Xml != nil {
		code.Print("Xml: %+v", schema.Xml)
	}
	printVendorExtension(code, schema.VendorExtension)
}

func printVendorExtension(code *printer.Code, vendorExtension []*pb.NamedAny) {
	if len(vendorExtension) > 0 {
		code.Print("VendorExtension: %+v", vendorExtension)
	}
}

func main() {
	flag.Parse()
	args := flag.Args()

	if len(args) != 1 {
		fmt.Printf("Usage: report <file.pb>\n")
		return
	}

	document, err := readDocumentFromFileWithName(args[0])

	if err != nil {
		log.Printf("Error reading %s. This sample expects OpenAPI v2.", args[0])
		os.Exit(-1)
	}
	code := &printer.Code{}
	code.Print("API REPORT")
	code.Print("----------")
	printDocument(code, document)
	fmt.Printf("%s", code)
}
<|endoftext|>


<commit_before>package aviator

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/JulzDiverse/aviator/spruce"

	"gopkg.in/yaml.v2"
)

var Warnings []string
var Silent bool
var Verbose bool

type Aviator struct {
	Spruce  []SpruceConfig `yaml:"spruce"`
	Fly     FlyConfig      `yaml:"fly"`
	Aviator AviatorConfig  `yaml:"aviator"`
}

type AviatorConfig struct {
	Verbose bool `yaml:"verbose`
	Silent  bool `yaml:"silent"`
}

type SpruceConfig struct {
	Base           string   `yaml:"base"`
	Prune          []string `yaml:"prune"`
	Chain          []Chain  `yaml:"merge"`
	WithIn         string   `yaml:"with_in"`
	Folder         string   `yaml:"dir"`
	ForEach        []string `yaml:"for_each"`
	ForEachIn      string   `yaml:"for_each_in"`
	Walk           string   `yaml:"walk_through"`
	IncludeAllIn   string   `yaml:"include_all_in"`
	ForAll         string   `yaml:"for_all"`
	CopyParents    bool     `yaml:"copy_parents"`
	EnableMatching bool     `yaml:"enable_matching"`
	SkipEval       bool     `yaml:"skip_eval"`
	CherryPicks    []string `yaml:"cherry_pick"`
	DestFile       string   `yaml:"to"`
	DestDir        string   `yaml:"to_dir"`
	Except         []string `yaml:"except"`
	Regexp         string   `yaml:"regexp"`
}

type Chain struct {
	With   With     `yaml:"with"`
	WithIn string   `yaml:"with_in"`
	Except []string `yaml:"except"`
	Regexp string   `yaml:"regexp"`
}

type With struct {
	Files    []string `yaml:"files"`
	InDir    string   `yaml:"in_dir"`
	Existing bool     `yaml:"skip_non_existing"`
}

type FlyConfig struct {
	Name   string   `yaml:"name"`
	Target string   `yaml:"target"`
	Config string   `yaml:"config"`
	Vars   []string `yaml:"vars"`
}

func ReadYaml(ymlBytes []byte) Aviator {
	var yml Aviator

	ymlBytes = quoteBraces(ymlBytes)
	err := yaml.Unmarshal(ymlBytes, &yml)
	if err != nil {
		panic(err)
	}

	return yml
}

var quoteRegex = `\{\{([-\w\p{L}]+)\}\}`
var re = regexp.MustCompile("(" + quoteRegex + ")")

func quoteBraces(input []byte) []byte {
	return re.ReplaceAll(input, []byte("\"$1\""))
}

func FlyPipeline(fly FlyConfig) {

	flyCmd := []string{"-t", fly.Target, "set-pipeline", "-p", fly.Name, "-c", fly.Config}
	for _, val := range fly.Vars {
		flyCmd = append(flyCmd, "-l", val)
	}

	cmd := exec.Command("fly", flyCmd...)

	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Stdin = os.Stdin

	err := cmd.Run()
	if err != nil {
		fmt.Printf("Failed to run fly. %s\n", err.Error())
		os.Exit(1)
	}
}

func ProcessSprucePlan(spruce []SpruceConfig, verbose bool, silent bool) error {
	Silent = silent
	Verbose = verbose

	for _, conf := range spruce {

		verifySpruceConfig(conf)

		if conf.ForEachIn == "" && len(conf.ForEach) == 0 && conf.Walk == "" {
			err := simpleMerge(conf)
			if err != nil {
				return err
			}
		}
		if len(conf.ForEach) != 0 {
			err := ForEachFile(conf)
			if err != nil {
				return err
			}
		}
		if conf.ForEachIn != "" {
			err := ForEachIn(conf)
			if err != nil {
				return err
			}
		}
		if conf.Walk != "" {
			if conf.ForAll != "" {
				err := ForAll(conf)
				if err != nil {
					return err
				}
			} else {
				err := Walk(conf, "")
				if err != nil {
					return err
				}
			}
		}
		if conf.IncludeAllIn != "" {
			err := WalkInclude(conf)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

func simpleMerge(conf SpruceConfig) error {
	files := collectFiles(conf)
	mergeConf := spruce.MergeOpts{
		Files:       files,
		Prune:       conf.Prune,
		SkipEval:    conf.SkipEval,
		CherryPicks: conf.CherryPicks,
	}
	err := spruceToFile(mergeConf, conf.DestFile)
	if err != nil {
		return err
	}
	return nil
}

func collectFiles(conf SpruceConfig) []string {
	files := []string{conf.Base}
	for _, val := range conf.Chain {
		tmp := collectFromMergeSection(val)
		for _, str := range tmp {
			files = append(files, str)
		}
	}
	return files
}

func ForEachFile(conf SpruceConfig) error {
	for _, val := range conf.ForEach {
		files := collectFiles(conf)
		fileName, _ := ConcatFileName(val)
		files = append(files, val)
		mergeConf := spruce.MergeOpts{
			Files:       files,
			Prune:       conf.Prune,
			SkipEval:    conf.SkipEval,
			CherryPicks: conf.CherryPicks,
		}
		err := spruceToFile(mergeConf, conf.DestDir+fileName)
		if err != nil {
			return err
		}

	}
	return nil
}

func ForEachIn(conf SpruceConfig) error {
	filePaths, _ := ioutil.ReadDir(conf.ForEachIn)
	regex := getRegexp(conf)
	files := collectFiles(conf)
	for _, f := range filePaths {
		if except(conf.Except, f.Name()) {
			Warnings = append(Warnings, "SKIPPED: "+f.Name())
			continue
		}
		matched, _ := regexp.MatchString(regex, f.Name())
		if matched {
			prefix := Chunk(conf.ForEachIn)
			filesTmp := append(files, conf.ForEachIn+f.Name())
			mergeConf := spruce.MergeOpts{
				Files:       filesTmp,
				SkipEval:    conf.SkipEval,
				Prune:       conf.Prune,
				CherryPicks: conf.CherryPicks,
			}
			CreateDir(conf.DestDir)
			err := spruceToFile(mergeConf, conf.DestDir+prefix+"_"+f.Name())
			if err != nil {
				return err
			}
		} else {
			Warnings = append(Warnings, "EXCLUDED BY REGEXP "+regex+": "+conf.ForEachIn+f.Name())
		}
	}
	return nil
}

func ForEachInner(conf SpruceConfig, outer string) error {
	filePaths, _ := ioutil.ReadDir(conf.ForEachIn)
	regex := getRegexp(conf)
	for _, f := range filePaths {
		files := collectFiles(conf)
		matched, _ := regexp.MatchString(regex, f.Name())
		if matched {
			prefix := Chunk(conf.ForEachIn)
			files = append(files, conf.ForEachIn+f.Name())
			files = append(files, outer)
			mergeConf := spruce.MergeOpts{
				Files:       files,
				Prune:       conf.Prune,
				SkipEval:    conf.SkipEval,
				CherryPicks: conf.CherryPicks,
			}
			err := spruceToFile(mergeConf, conf.DestDir+prefix+"_"+f.Name())
			if err != nil {
				return err
			}
		}
	}
	return nil
}

func ForAll(conf SpruceConfig) error {
	if conf.ForAll != "" {
		files, _ := ioutil.ReadDir(conf.ForAll)
		for _, f := range files {
			err := Walk(conf, conf.ForAll+f.Name())
			if err != nil {
				return err
			}
		}
	}
	return nil
}

func Walk(conf SpruceConfig, outer string) error {
	sl := getAllFilesInSubDirs(conf.Walk)
	regex := getRegexp(conf)

	for _, f := range sl {
		filename, parent := ConcatFileName(f)
		match := isMatchingEnabled(conf, parent)
		if strings.Contains(outer, match) {
			matched, _ := regexp.MatchString(regex, filename)
			if matched {
				files := collectFiles(conf)
				files = append(files, f)
				files = append(files, outer)
				if conf.CopyParents {
					CreateDir(conf.DestDir + parent)
				} else {
					parent = ""
				}
				mergeConf := spruce.MergeOpts{
					Files:       files,
					Prune:       conf.Prune,
					SkipEval:    conf.SkipEval,
					CherryPicks: conf.CherryPicks,
				}
				err := spruceToFile(mergeConf, conf.DestDir+parent+"/"+filename)
				if err != nil {
					return err
				}

			}
		}
	}
	return nil
}

func WalkInclude(conf SpruceConfig) error {
	allFiles := getAllFilesInSubDirs(conf.IncludeAllIn)
	regex := getRegexp(conf)
	files := []string{}
	for _, file := range allFiles {
		filename, _ := ConcatFileName(file)
		matched, _ := regexp.MatchString(regex, filename)
		if matched {
			files = append(files, file)
		}
	}
	mergeConf := spruce.MergeOpts{
		Files:       files,
		Prune:       conf.Prune,
		SkipEval:    conf.SkipEval,
		CherryPicks: conf.CherryPicks,
	}
	err := spruceToFile(mergeConf, conf.DestFile)
	if err != nil {
		return err
	}

	return nil
}

func collectFromMergeSection(chain Chain) []string {
	var result []string
	for _, file := range chain.With.Files {
		if chain.With.InDir != "" {
			dir := chain.With.InDir
			file = dir + file
		}
		if !chain.With.Existing || fileExists(file) {
			result = append(result, file)
		}
	}

	if chain.WithIn != "" {
		within := chain.WithIn
		files, _ := ioutil.ReadDir(within)
		regex := getChainRegexp(chain)
		for _, f := range files {
			if except(chain.Except, f.Name()) {
				continue
			}
			matched, _ := regexp.MatchString(regex, f.Name())
			if matched {
				result = append(result, within+f.Name())
			} else {
				Warnings = append(Warnings, "EXCLUDED BY REGEXP "+regex+": "+chain.WithIn+f.Name())
			}
		}
	}

	return result
}

func except(except []string, file string) bool {
	for _, f := range except {
		if f == file {
			return true
		}
	}
	return false
}

func spruceToFile(opts spruce.MergeOpts, fileName string) error {
	if !Silent {
		beautifyPrint(opts, fileName)
	}
	Warnings = []string{}

	rawYml, err := spruce.CmdMergeEval(opts)
	if err != nil {
		return err
	}

	resultYml, err := yaml.Marshal(rawYml)
	if err != nil {
		return err
	}

	spruce.WriteYamlToPathOrStore(fileName, resultYml)
	return nil
}

func Cleanup(path string) {
	d, err := os.Open(path)
	if err != nil {
		panic(err)
	}
	defer d.Close()
	names, err := d.Readdirnames(-1)
	if err != nil {
		panic(err)
	}
	for _, name := range names {
		err = os.RemoveAll(filepath.Join(path, name))
		if err != nil {
			panic(err)
		}
	}
}
<commit_msg>adding fix to walkinclude merging feature<commit_after>package aviator

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/JulzDiverse/aviator/spruce"

	"gopkg.in/yaml.v2"
)

var Warnings []string
var Silent bool
var Verbose bool

type Aviator struct {
	Spruce  []SpruceConfig `yaml:"spruce"`
	Fly     FlyConfig      `yaml:"fly"`
	Aviator AviatorConfig  `yaml:"aviator"`
}

type AviatorConfig struct {
	Verbose bool `yaml:"verbose`
	Silent  bool `yaml:"silent"`
}

type SpruceConfig struct {
	Base           string   `yaml:"base"`
	Prune          []string `yaml:"prune"`
	Chain          []Chain  `yaml:"merge"`
	WithIn         string   `yaml:"with_in"`
	Folder         string   `yaml:"dir"`
	ForEach        []string `yaml:"for_each"`
	ForEachIn      string   `yaml:"for_each_in"`
	Walk           string   `yaml:"walk_through"`
	IncludeAllIn   string   `yaml:"include_all_in"`
	ForAll         string   `yaml:"for_all"`
	CopyParents    bool     `yaml:"copy_parents"`
	EnableMatching bool     `yaml:"enable_matching"`
	SkipEval       bool     `yaml:"skip_eval"`
	CherryPicks    []string `yaml:"cherry_pick"`
	DestFile       string   `yaml:"to"`
	DestDir        string   `yaml:"to_dir"`
	Except         []string `yaml:"except"`
	Regexp         string   `yaml:"regexp"`
}

type Chain struct {
	With   With     `yaml:"with"`
	WithIn string   `yaml:"with_in"`
	Except []string `yaml:"except"`
	Regexp string   `yaml:"regexp"`
}

type With struct {
	Files    []string `yaml:"files"`
	InDir    string   `yaml:"in_dir"`
	Existing bool     `yaml:"skip_non_existing"`
}

type FlyConfig struct {
	Name   string   `yaml:"name"`
	Target string   `yaml:"target"`
	Config string   `yaml:"config"`
	Vars   []string `yaml:"vars"`
}

func ReadYaml(ymlBytes []byte) Aviator {
	var yml Aviator

	ymlBytes = quoteBraces(ymlBytes)
	err := yaml.Unmarshal(ymlBytes, &yml)
	if err != nil {
		panic(err)
	}

	return yml
}

var quoteRegex = `\{\{([-\w\p{L}]+)\}\}`
var re = regexp.MustCompile("(" + quoteRegex + ")")

func quoteBraces(input []byte) []byte {
	return re.ReplaceAll(input, []byte("\"$1\""))
}

func FlyPipeline(fly FlyConfig) {

	flyCmd := []string{"-t", fly.Target, "set-pipeline", "-p", fly.Name, "-c", fly.Config}
	for _, val := range fly.Vars {
		flyCmd = append(flyCmd, "-l", val)
	}

	cmd := exec.Command("fly", flyCmd...)

	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Stdin = os.Stdin

	err := cmd.Run()
	if err != nil {
		fmt.Printf("Failed to run fly. %s\n", err.Error())
		os.Exit(1)
	}
}

func ProcessSprucePlan(spruce []SpruceConfig, verbose bool, silent bool) error {
	Silent = silent
	Verbose = verbose

	for _, conf := range spruce {

		verifySpruceConfig(conf)

		if conf.ForEachIn == "" && len(conf.ForEach) == 0 && conf.Walk == ""  && conf.IncludeAllIn == "" {
			err := simpleMerge(conf)
			if err != nil {
				return err
			}
		}
		if len(conf.ForEach) != 0 {
			err := ForEachFile(conf)
			if err != nil {
				return err
			}
		}
		if conf.ForEachIn != "" {
			err := ForEachIn(conf)
			if err != nil {
				return err
			}
		}
		if conf.Walk != "" {
			if conf.ForAll != "" {
				err := ForAll(conf)
				if err != nil {
					return err
				}
			} else {
				err := Walk(conf, "")
				if err != nil {
					return err
				}
			}
		}
		if conf.IncludeAllIn != "" {
			err := WalkInclude(conf)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

func simpleMerge(conf SpruceConfig) error {
	files := collectFiles(conf)
	mergeConf := spruce.MergeOpts{
		Files:       files,
		Prune:       conf.Prune,
		SkipEval:    conf.SkipEval,
		CherryPicks: conf.CherryPicks,
	}
	err := spruceToFile(mergeConf, conf.DestFile)
	if err != nil {
		return err
	}
	return nil
}

func collectFiles(conf SpruceConfig) []string {
	files := []string{conf.Base}
	for _, val := range conf.Chain {
		tmp := collectFromMergeSection(val)
		for _, str := range tmp {
			files = append(files, str)
		}
	}
	return files
}

func ForEachFile(conf SpruceConfig) error {
	for _, val := range conf.ForEach {
		files := collectFiles(conf)
		fileName, _ := ConcatFileName(val)
		files = append(files, val)
		mergeConf := spruce.MergeOpts{
			Files:       files,
			Prune:       conf.Prune,
			SkipEval:    conf.SkipEval,
			CherryPicks: conf.CherryPicks,
		}
		err := spruceToFile(mergeConf, conf.DestDir+fileName)
		if err != nil {
			return err
		}

	}
	return nil
}

func ForEachIn(conf SpruceConfig) error {
	filePaths, _ := ioutil.ReadDir(conf.ForEachIn)
	regex := getRegexp(conf)
	files := collectFiles(conf)
	for _, f := range filePaths {
		if except(conf.Except, f.Name()) {
			Warnings = append(Warnings, "SKIPPED: "+f.Name())
			continue
		}
		matched, _ := regexp.MatchString(regex, f.Name())
		if matched {
			prefix := Chunk(conf.ForEachIn)
			filesTmp := append(files, conf.ForEachIn+f.Name())
			mergeConf := spruce.MergeOpts{
				Files:       filesTmp,
				SkipEval:    conf.SkipEval,
				Prune:       conf.Prune,
				CherryPicks: conf.CherryPicks,
			}
			CreateDir(conf.DestDir)
			err := spruceToFile(mergeConf, conf.DestDir+prefix+"_"+f.Name())
			if err != nil {
				return err
			}
		} else {
			Warnings = append(Warnings, "EXCLUDED BY REGEXP "+regex+": "+conf.ForEachIn+f.Name())
		}
	}
	return nil
}

func ForEachInner(conf SpruceConfig, outer string) error {
	filePaths, _ := ioutil.ReadDir(conf.ForEachIn)
	regex := getRegexp(conf)
	for _, f := range filePaths {
		files := collectFiles(conf)
		matched, _ := regexp.MatchString(regex, f.Name())
		if matched {
			prefix := Chunk(conf.ForEachIn)
			files = append(files, conf.ForEachIn+f.Name())
			files = append(files, outer)
			mergeConf := spruce.MergeOpts{
				Files:       files,
				Prune:       conf.Prune,
				SkipEval:    conf.SkipEval,
				CherryPicks: conf.CherryPicks,
			}
			err := spruceToFile(mergeConf, conf.DestDir+prefix+"_"+f.Name())
			if err != nil {
				return err
			}
		}
	}
	return nil
}

func ForAll(conf SpruceConfig) error {
	if conf.ForAll != "" {
		files, _ := ioutil.ReadDir(conf.ForAll)
		for _, f := range files {
			err := Walk(conf, conf.ForAll+f.Name())
			if err != nil {
				return err
			}
		}
	}
	return nil
}

func Walk(conf SpruceConfig, outer string) error {
	sl := getAllFilesInSubDirs(conf.Walk)
	regex := getRegexp(conf)

	for _, f := range sl {
		filename, parent := ConcatFileName(f)
		match := isMatchingEnabled(conf, parent)
		if strings.Contains(outer, match) {
			matched, _ := regexp.MatchString(regex, filename)
			if matched {
				files := collectFiles(conf)
				files = append(files, f)
				files = append(files, outer)
				if conf.CopyParents {
					CreateDir(conf.DestDir + parent)
				} else {
					parent = ""
				}
				mergeConf := spruce.MergeOpts{
					Files:       files,
					Prune:       conf.Prune,
					SkipEval:    conf.SkipEval,
					CherryPicks: conf.CherryPicks,
				}
				err := spruceToFile(mergeConf, conf.DestDir+parent+"/"+filename)
				if err != nil {
					return err
				}

			}
		}
	}
	return nil
}

func WalkInclude(conf SpruceConfig) error {
	allFiles := getAllFilesInSubDirs(conf.IncludeAllIn)
	regex := getRegexp(conf)
	files := []string{}
	for _, file := range allFiles {
		matched, _ := regexp.MatchString(regex, file)
		if matched {
			files = append(files, file)
		}
	}
	mergeConf := spruce.MergeOpts{
		Files:       files,
		Prune:       conf.Prune,
		SkipEval:    conf.SkipEval,
		CherryPicks: conf.CherryPicks,
	}
	err := spruceToFile(mergeConf, conf.DestFile)
	if err != nil {
		return err
	}

	return nil
}

func collectFromMergeSection(chain Chain) []string {
	var result []string
	for _, file := range chain.With.Files {
		if chain.With.InDir != "" {
			dir := chain.With.InDir
			file = dir + file
		}
		if !chain.With.Existing || fileExists(file) {
			result = append(result, file)
		}
	}

	if chain.WithIn != "" {
		within := chain.WithIn
		files, _ := ioutil.ReadDir(within)
		regex := getChainRegexp(chain)
		for _, f := range files {
			if except(chain.Except, f.Name()) {
				continue
			}
			matched, _ := regexp.MatchString(regex, f.Name())
			if matched {
				result = append(result, within+f.Name())
			} else {
				Warnings = append(Warnings, "EXCLUDED BY REGEXP "+regex+": "+chain.WithIn+f.Name())
			}
		}
	}

	return result
}

func except(except []string, file string) bool {
	for _, f := range except {
		if f == file {
			return true
		}
	}
	return false
}

func spruceToFile(opts spruce.MergeOpts, fileName string) error {
	if !Silent {
		beautifyPrint(opts, fileName)
	}
	Warnings = []string{}

	rawYml, err := spruce.CmdMergeEval(opts)
	if err != nil {
		return err
	}

	resultYml, err := yaml.Marshal(rawYml)
	if err != nil {
		return err
	}

	spruce.WriteYamlToPathOrStore(fileName, resultYml)
	return nil
}

func Cleanup(path string) {
	d, err := os.Open(path)
	if err != nil {
		panic(err)
	}
	defer d.Close()
	names, err := d.Readdirnames(-1)
	if err != nil {
		panic(err)
	}
	for _, name := range names {
		err = os.RemoveAll(filepath.Join(path, name))
		if err != nil {
			panic(err)
		}
	}
}
<|endoftext|>


<commit_before>/*
   Copyright 2014 CoreOS, Inc.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package etcdhttp

import (
	"errors"
	"log"
	"net/http"
	"strings"
	"time"

	etcdErr "github.com/coreos/etcd/error"
	"github.com/coreos/etcd/etcdserver/etcdhttp/httptypes"
)

const (
	// time to wait for response from EtcdServer requests
	defaultServerTimeout = 5 * time.Minute

	// time to wait for a Watch request
	defaultWatchTimeout = 5 * time.Minute
)

var errClosed = errors.New("etcdhttp: client closed connection")

// writeError logs and writes the given Error to the ResponseWriter
// If Error is an etcdErr, it is rendered to the ResponseWriter
// Otherwise, it is assumed to be an InternalServerError
func writeError(w http.ResponseWriter, err error) {
	if err == nil {
		return
	}
	switch e := err.(type) {
	case *etcdErr.Error:
		e.WriteTo(w)
	case *httptypes.HTTPError:
		e.WriteTo(w)
	default:
		log.Printf("etcdhttp: unexpected error: %v", err)
		herr := httptypes.NewHTTPError(http.StatusInternalServerError, "Internal Server Error")
		herr.WriteTo(w)
	}
}

// allowMethod verifies that the given method is one of the allowed methods,
// and if not, it writes an error to w.  A boolean is returned indicating
// whether or not the method is allowed.
func allowMethod(w http.ResponseWriter, m string, ms ...string) bool {
	for _, meth := range ms {
		if m == meth {
			return true
		}
	}
	w.Header().Set("Allow", strings.Join(ms, ","))
	http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed)
	return false
}
<commit_msg>etcdhttp: reset serve and watch timeout<commit_after>/*
   Copyright 2014 CoreOS, Inc.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package etcdhttp

import (
	"errors"
	"log"
	"math"
	"net/http"
	"strings"
	"time"

	etcdErr "github.com/coreos/etcd/error"
	"github.com/coreos/etcd/etcdserver/etcdhttp/httptypes"
)

const (
	// time to wait for response from EtcdServer requests
	// 5s for disk and network delay + 10*heartbeat for commit and possible
	// leader switch
	// TODO: use heartbeat set in etcdserver
	defaultServerTimeout = 5*time.Second + 10*(100*time.Millisecond)

	// time to wait for a Watch request
	defaultWatchTimeout = time.Duration(math.MaxInt64)
)

var errClosed = errors.New("etcdhttp: client closed connection")

// writeError logs and writes the given Error to the ResponseWriter
// If Error is an etcdErr, it is rendered to the ResponseWriter
// Otherwise, it is assumed to be an InternalServerError
func writeError(w http.ResponseWriter, err error) {
	if err == nil {
		return
	}
	switch e := err.(type) {
	case *etcdErr.Error:
		e.WriteTo(w)
	case *httptypes.HTTPError:
		e.WriteTo(w)
	default:
		log.Printf("etcdhttp: unexpected error: %v", err)
		herr := httptypes.NewHTTPError(http.StatusInternalServerError, "Internal Server Error")
		herr.WriteTo(w)
	}
}

// allowMethod verifies that the given method is one of the allowed methods,
// and if not, it writes an error to w.  A boolean is returned indicating
// whether or not the method is allowed.
func allowMethod(w http.ResponseWriter, m string, ms ...string) bool {
	for _, meth := range ms {
		if m == meth {
			return true
		}
	}
	w.Header().Set("Allow", strings.Join(ms, ","))
	http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed)
	return false
}
<|endoftext|>


<commit_before>package Handlers

import (
	"Gate/Manager"
)

type UserLoginHander struct {
}

func (hander *UserLoginHander) GetMessage() {
	return nil
}

func (hander *UserLoginHander) Action() {
	var msg = hander.GetMessage()
	Manager.GetInstance().UserManager.UserLogin(msg.GetSession(), msg.GetUserName(), msg.GetUserPass())
}
<commit_msg>Modify UserLoginHandler<commit_after>package Handlers

import (
	"Gate"
	"Gate/Manager"
)

type UserLoginHander struct {
	message *Gate.Messager
}

func (hander *UserLoginHander) GetMessage() *Gate.Messager {
	return hander.message
}

func (hander *UserLoginHander) Action() {
	var msg = hander.GetMessage()
	Manager.GetInstance().UserManager.UserLogin(msg.GetSession(), msg.GetUserName(), msg.GetUserPass())
}
<|endoftext|>


<commit_before>// File deduplication
package main

import (
	"crypto/sha256"
	"hash"
	"io"
	"os"
	"strings"
)

// File attributes.
type FileAttr struct {
	Path       string   // Full path.
	Name       string   // Name.
	ModTime    int64    // the number of nanoseconds elapsed since January 1, 1970 UTC
	Size       int64    // File size, in bytes.
	SHA256     [32]byte // SHA256 checksum.
	StillExist bool     // Indicates if the file still exists.
}

// Update status.
type Updater interface {
	Error() error                          // Any error happened or job was cancelled.
	SetError(err error)                    // Set error code.
	Print(format string, a ...interface{}) // Print status message.
}

// Scan a path (could be file or folder).
func ScanPath(path string, files map[string]*FileAttr,
	updater Updater) error {

	// Check if it's file or folder.
	info, err := os.Stat(path)
	if err != nil {
		updater.Print("Path %v might not exist. Error:%v", path, err)
		updater.SetError(err)
		return err
	}

	// Create hash engine and allocate buffer to read file.
	hash := sha256.New()
	buffer := make([]byte, 16*1024)

	if info.IsDir() {
		if err = scanFolder_i(path, files,
			updater, hash, buffer); err != nil {
			return err
		}
	} else {
		if err = scanFile_i(path, files,
			updater, hash, buffer, info); err != nil {
			return err
		}
	}

	// Some files do not exist in disk any more,
	// let's remove them from the map.
	removeNonExistFiles(files)

	return nil
}

// Scan a folder and its sub-folders recursively.
func scanFolder_i(path string, files map[string]*FileAttr,
	updater Updater, hash hash.Hash, buffer []byte) error {
	fp, err := os.Open(path)
	if err != nil {
		updater.Print("Could not open folder %v. Error:%v", path, err)
		updater.SetError(err)
		return err
	}
	defer fp.Close()

	for {
		list, err := fp.Readdir(256)
		if err != nil && err != io.EOF {
			updater.Print("Could not enumerate folder %v. Error:%v", path, err)
			updater.SetError(err)
			return err
		}

		for i := 0; i < len(list); i++ {
			// Check if job was cancelled or an error ever happened.
			if err := updater.Error(); err != nil {
				return err
			}

			// Get full path.
			var fullPath string

			if len(path) == 1 && path == "/" {
				fullPath = path + list[i].Name()
			} else {
				fullPath = path + string(os.PathSeparator) + list[i].Name()
			}

			if list[i].IsDir() {
				if tmp := scanFolder_i(fullPath, files,
					updater, hash, buffer); tmp != nil {
					return tmp
				}
			} else {
				if tmp := scanFile_i(fullPath, files,
					updater, hash, buffer, list[i]); tmp != nil {
					return tmp
				}
			}
		}

		// If reaching end of the folder, then break.
		if err == io.EOF {
			break
		}
	}

	return nil
}

// Calculate single file checksum.
func scanFile_i(path string, files map[string]*FileAttr,
	updater Updater, hash hash.Hash, buffer []byte,
	info os.FileInfo) error {

	var key string = path

	// Case insensitive on Windows.
	if os.PathSeparator != '/' {
		key = strings.ToLower(key)
	}

	// If the file already exists in the map,
	// and file size & last modification time are the same,
	// then skip to read it to enhance performance.
	if value, found := files[key]; found {
		if value.Size == info.Size() && value.ModTime == info.ModTime().UnixNano() {
			value.StillExist = true
			return nil
		}
	}

	// Open file.
	fp, err := os.Open(path)
	if err != nil {
		updater.Print("Could not open file %v. Error:%v", path, err)
		return err
	}
	defer fp.Close()

	// Reset hash engine
	hash.Reset()

	// Read file content
	for {
		// Check if job was cancelled or an error ever happened.
		if err := updater.Error(); err != nil {
			return err
		}

		n, err := fp.Read(buffer)
		if err != nil && err != io.EOF {
			updater.Print("Could not read file %v. Error:%v", path, err)
			return err
		}
		hash.Write(buffer[0:n])

		if err == io.EOF {
			break
		}
	}

	// Create a new object.
	newValue := &FileAttr{
		Path:       path,
		ModTime:    info.ModTime().UnixNano(),
		Size:       info.Size(),
		StillExist: true,
	}
	copy(newValue.SHA256[:], hash.Sum(nil))

	// Add the new object to map.
	files[key] = newValue

	return nil
}

// Some files do not exist in disk any more, let's remove them from the map.
func removeNonExistFiles(files map[string]*FileAttr) {
	for key, value := range files {
		if !value.StillExist {
			delete(files, key)
		}
	}
}
<commit_msg>Save file name in FileAttr<commit_after>// File deduplication
package main

import (
	"crypto/sha256"
	"hash"
	"io"
	"os"
	"strings"
)

// File attributes.
type FileAttr struct {
	Path       string   // Full path.
	Name       string   // Name.
	ModTime    int64    // the number of nanoseconds elapsed since January 1, 1970 UTC
	Size       int64    // File size, in bytes.
	SHA256     [32]byte // SHA256 checksum.
	StillExist bool     // Indicates if the file still exists.
}

// Update status.
type Updater interface {
	Error() error                          // Any error happened or job was cancelled.
	SetError(err error)                    // Set error code.
	Print(format string, a ...interface{}) // Print status message.
}

// Scan a path (could be file or folder).
func ScanPath(path string, files map[string]*FileAttr,
	updater Updater) error {

	// Check if it's file or folder.
	info, err := os.Stat(path)
	if err != nil {
		updater.Print("Path %v might not exist. Error:%v", path, err)
		updater.SetError(err)
		return err
	}

	// Create hash engine and allocate buffer to read file.
	hash := sha256.New()
	buffer := make([]byte, 16*1024)

	if info.IsDir() {
		if err = scanFolder_i(path, files,
			updater, hash, buffer); err != nil {
			return err
		}
	} else {
		if err = scanFile_i(path, files,
			updater, hash, buffer, info); err != nil {
			return err
		}
	}

	// Some files do not exist in disk any more,
	// let's remove them from the map.
	removeNonExistFiles(files)

	return nil
}

// Scan a folder and its sub-folders recursively.
func scanFolder_i(path string, files map[string]*FileAttr,
	updater Updater, hash hash.Hash, buffer []byte) error {
	fp, err := os.Open(path)
	if err != nil {
		updater.Print("Could not open folder %v. Error:%v", path, err)
		updater.SetError(err)
		return err
	}
	defer fp.Close()

	for {
		list, err := fp.Readdir(256)
		if err != nil && err != io.EOF {
			updater.Print("Could not enumerate folder %v. Error:%v", path, err)
			updater.SetError(err)
			return err
		}

		for i := 0; i < len(list); i++ {
			// Check if job was cancelled or an error ever happened.
			if err := updater.Error(); err != nil {
				return err
			}

			// Get full path.
			var fullPath string

			if len(path) == 1 && path == "/" {
				fullPath = path + list[i].Name()
			} else {
				fullPath = path + string(os.PathSeparator) + list[i].Name()
			}

			if list[i].IsDir() {
				if tmp := scanFolder_i(fullPath, files,
					updater, hash, buffer); tmp != nil {
					return tmp
				}
			} else {
				if tmp := scanFile_i(fullPath, files,
					updater, hash, buffer, list[i]); tmp != nil {
					return tmp
				}
			}
		}

		// If reaching end of the folder, then break.
		if err == io.EOF {
			break
		}
	}

	return nil
}

// Calculate single file checksum.
func scanFile_i(path string, files map[string]*FileAttr,
	updater Updater, hash hash.Hash, buffer []byte,
	info os.FileInfo) error {

	var key string = path

	// Case insensitive on Windows.
	if os.PathSeparator != '/' {
		key = strings.ToLower(key)
	}

	// If the file already exists in the map,
	// and file size & last modification time are the same,
	// then skip to read it to enhance performance.
	if value, found := files[key]; found {
		if value.Size == info.Size() && value.ModTime == info.ModTime().UnixNano() {
			value.StillExist = true
			return nil
		}
	}

	// Open file.
	fp, err := os.Open(path)
	if err != nil {
		updater.Print("Could not open file %v. Error:%v", path, err)
		return err
	}
	defer fp.Close()

	// Reset hash engine
	hash.Reset()

	// Read file content
	for {
		// Check if job was cancelled or an error ever happened.
		if err := updater.Error(); err != nil {
			return err
		}

		n, err := fp.Read(buffer)
		if err != nil && err != io.EOF {
			updater.Print("Could not read file %v. Error:%v", path, err)
			return err
		}
		hash.Write(buffer[0:n])

		if err == io.EOF {
			break
		}
	}

	// Create a new object.
	newValue := &FileAttr{
		Path:       path,
		Name:       info.Name(),
		ModTime:    info.ModTime().UnixNano(),
		Size:       info.Size(),
		StillExist: true,
	}
	copy(newValue.SHA256[:], hash.Sum(nil))

	// Add the new object to map.
	files[key] = newValue

	return nil
}

// Some files do not exist in disk any more, let's remove them from the map.
func removeNonExistFiles(files map[string]*FileAttr) {
	for key, value := range files {
		if !value.StillExist {
			delete(files, key)
		}
	}
}
<|endoftext|>


<commit_before>package apixu

import (
	"os"
	"testing"
	"time"
)

const InvalidCityName = "INVALID-CITY-NAME"

var cities = []string{
	"Cairo",
	"London",
	"Paris",
	"Berlin",
	"New York",
}

func getAPIKey() string {
	return os.Getenv("APIXU_KEY")
}

func TestInvalidAPIKey(t *testing.T) {
	client := NewClient("Invalid_Key")
	_, err := client.Current("Paris")

	if err == nil {
		t.Error("Worked with invalid key")
	}
}

func TestCurrentWeatherValidCities(t *testing.T) {
	client := NewClient(getAPIKey())

	for _, city := range cities {
		_, err := client.Current(city)

		if err != nil {
			t.Errorf("There was an error getting current weather of %s: %v", city, err)
		}
	}
}

func TestCurrentWeatherInValidCity(t *testing.T) {
	client := NewClient(getAPIKey())
	_, err := client.Current(InvalidCityName)

	if err == nil {
		t.Error("No errors getting current weather of invalid city name")
	}
}

func TestForecastWeatherValidCities(t *testing.T) {
	days := []int{1, 5, 10}
	client := NewClient(getAPIKey())

	for _, day := range days {
		for _, city := range cities {
			_, err := client.Forecast(city, day)

			if err != nil {
				t.Errorf("There was an error getting forecast weather of %s days %d: %v", city, day, err)
			}
		}
	}
}

func TestForecastWeatherInValidCity(t *testing.T) {
	days := []int{1, 5, 10}
	client := NewClient(getAPIKey())

	for _, day := range days {
		_, err := client.Forecast(InvalidCityName, day)

		if err == nil {
			t.Error("No errors getting forecast weather of invalid city name")
		}
	}
}

func TestHistoryWeatherValidCities(t *testing.T) {
	yesterday := time.Now().AddDate(0, 0, -1)
	client := NewClient(getAPIKey())

	for _, city := range cities {
		_, err := client.History(city, yesterday.Format("2006-01-2"))

		if err != nil {
			t.Errorf("There was an error getting current weather of %s: %v", city, err)
		}
	}
}

func TestHistoryWeatherInValidCity(t *testing.T) {
	yesterday := time.Now().AddDate(0, 0, -1)
	client := NewClient(getAPIKey())

	_, err := client.History(InvalidCityName, yesterday.Format("2006-01-2"))

	if err == nil {
		t.Error("No errors getting history weather of invalid city name")
	}

}

func TestSearchValidCities(t *testing.T) {
	client := NewClient(getAPIKey())

	for _, city := range cities {
		_, err := client.Search(city)

		if err != nil {
			t.Errorf("There was an error getting current weather of %s: %v", city, err)
		}
	}
}

func TestSearchInValidCity(t *testing.T) {
	client := NewClient(getAPIKey())

	matchedCities, err := client.Search(InvalidCityName)

	if err != nil {
		t.Error(err)
	}

	if len(*matchedCities) != 0 {
		t.Error("Non-Empty array of matched cities for invalid city name")
	}
}
<commit_msg>test: update InvalidCityName<commit_after>package apixu

import (
	"os"
	"testing"
	"time"
)

const InvalidCityName = "INVALIDCITYNAME"

var cities = []string{
	"Cairo",
	"London",
	"Paris",
	"Berlin",
	"New York",
}

func getAPIKey() string {
	return os.Getenv("APIXU_KEY")
}

func TestInvalidAPIKey(t *testing.T) {
	client := NewClient("Invalid_Key")
	_, err := client.Current("Paris")

	if err == nil {
		t.Error("Worked with invalid key")
	}
}

func TestCurrentWeatherValidCities(t *testing.T) {
	client := NewClient(getAPIKey())

	for _, city := range cities {
		_, err := client.Current(city)

		if err != nil {
			t.Errorf("There was an error getting current weather of %s: %v", city, err)
		}
	}
}

func TestCurrentWeatherInValidCity(t *testing.T) {
	client := NewClient(getAPIKey())
	_, err := client.Current(InvalidCityName)

	if err == nil {
		t.Error("No errors getting current weather of invalid city name")
	}
}

func TestForecastWeatherValidCities(t *testing.T) {
	days := []int{1, 5, 10}
	client := NewClient(getAPIKey())

	for _, day := range days {
		for _, city := range cities {
			_, err := client.Forecast(city, day)

			if err != nil {
				t.Errorf("There was an error getting forecast weather of %s days %d: %v", city, day, err)
			}
		}
	}
}

func TestForecastWeatherInValidCity(t *testing.T) {
	days := []int{1, 5, 10}
	client := NewClient(getAPIKey())

	for _, day := range days {
		_, err := client.Forecast(InvalidCityName, day)

		if err == nil {
			t.Error("No errors getting forecast weather of invalid city name")
		}
	}
}

func TestHistoryWeatherValidCities(t *testing.T) {
	yesterday := time.Now().AddDate(0, 0, -1)
	client := NewClient(getAPIKey())

	for _, city := range cities {
		_, err := client.History(city, yesterday.Format("2006-01-2"))

		if err != nil {
			t.Errorf("There was an error getting current weather of %s: %v", city, err)
		}
	}
}

func TestHistoryWeatherInValidCity(t *testing.T) {
	yesterday := time.Now().AddDate(0, 0, -1)
	client := NewClient(getAPIKey())

	_, err := client.History(InvalidCityName, yesterday.Format("2006-01-2"))

	if err == nil {
		t.Error("No errors getting history weather of invalid city name")
	}

}

func TestSearchValidCities(t *testing.T) {
	client := NewClient(getAPIKey())

	for _, city := range cities {
		_, err := client.Search(city)

		if err != nil {
			t.Errorf("There was an error getting current weather of %s: %v", city, err)
		}
	}
}

func TestSearchInValidCity(t *testing.T) {
	client := NewClient(getAPIKey())

	matchedCities, err := client.Search(InvalidCityName)

	if err != nil {
		t.Error(err)
	}

	if len(*matchedCities) != 0 {
		t.Error("Non-Empty array of matched cities for invalid city name")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"runtime"
	"time"

	"github.com/smartystreets/go-disruptor"
)

const (
	BufferSize = 1024 * 64
	BufferMask = BufferSize - 1
	Iterations = 1000000 * 100
)

var ringBuffer = [BufferSize]int64{}

func main() {
	runtime.GOMAXPROCS(2)

	written, read := disruptor.NewCursor(), disruptor.NewCursor()
	reader := disruptor.NewReader(read, written, written, SampleConsumer{})

	started := time.Now()
	reader.Start()
	publish(written, read)
	// publish(disruptor.NewWriter(written, read, BufferSize))
	reader.Stop()
	finished := time.Now()
	fmt.Println(Iterations, finished.Sub(started))
}

// func publish(writer *disruptor.Writer) {
// 	for sequence := disruptor.InitialSequenceValue; sequence <= Iterations; {
// 		sequence = writer.Reserve()
// 		// ringBuffer[sequence&BufferMask] = sequence
// 		writer.Commit(sequence)
// 	}
// }

func publish(written, read *disruptor.Cursor) {
	previous := disruptor.InitialSequenceValue
	gate := disruptor.InitialSequenceValue

	for previous <= Iterations {
		next := previous + 1
		wrap := next - BufferSize

		for wrap > gate {
			gate = read.Sequence
		}

		// ringBuffer[next&BufferMask] = next
		written.Sequence = next
		previous = next
	}
}

type SampleConsumer struct{}

func (this SampleConsumer) Consume(lower, upper int64) {
	for lower <= upper {
		// message := ringBuffer[lower&BufferMask]
		// if message != lower {
		// 	fmt.Println("Race condition", message, lower)
		// 	panic("Race condition")
		// }
		lower++
	}
}
<commit_msg>4.59ms per operation when writing to ring buffer and using "cleaner" API.<commit_after>package main

import (
	"fmt"
	"runtime"
	"time"

	"github.com/smartystreets/go-disruptor"
)

const (
	BufferSize = 1024 * 64
	BufferMask = BufferSize - 1
	Iterations = 1000000 * 100
)

var ringBuffer = [BufferSize]int64{}

func main() {
	runtime.GOMAXPROCS(2)

	written, read := disruptor.NewCursor(), disruptor.NewCursor()
	reader := disruptor.NewReader(read, written, written, SampleConsumer{})

	started := time.Now()
	reader.Start()
	// publish(written, read)
	publish(disruptor.NewWriter(written, read, BufferSize))
	reader.Stop()
	finished := time.Now()
	fmt.Println(Iterations, finished.Sub(started))
}

func publish(writer *disruptor.Writer) {
	for sequence := disruptor.InitialSequenceValue; sequence <= Iterations; {
		sequence = writer.Reserve()
		ringBuffer[sequence&BufferMask] = sequence
		writer.Commit(sequence)
	}
}

// func publish(written, read *disruptor.Cursor) {
// 	previous := disruptor.InitialSequenceValue
// 	gate := disruptor.InitialSequenceValue

// 	for previous <= Iterations {
// 		next := previous + 1
// 		wrap := next - BufferSize

// 		for wrap > gate {
// 			gate = read.Sequence
// 		}

// 		ringBuffer[next&BufferMask] = next
// 		written.Sequence = next
// 		previous = next
// 	}
// }

type SampleConsumer struct{}

func (this SampleConsumer) Consume(lower, upper int64) {
	for lower <= upper {
		message := ringBuffer[lower&BufferMask]
		if message != lower {
			fmt.Println("Race condition", message, lower)
			panic("Race condition")
		}
		lower++
	}
}
<|endoftext|>


<commit_before>package app

import (
	"errors"
	"github.com/Sirupsen/logrus"
	"github.com/Tapjoy/lane"
	"github.com/hashicorp/memberlist"
	"math"
	"math/rand"
	"sort"
	"sync"
	"time"
)

type Partitions struct {
	partitions     *lane.PQueue
	partitionCount int
	sync.RWMutex
}

type Partition struct {
	Id       int
	LastUsed time.Time
}

func InitPartitions(cfg *Config, queueName string) *Partitions {
	part := &Partitions{
		partitions:     lane.NewPQueue(lane.MINPQ),
		partitionCount: 0,
	}
	// We'll initially allocate the minimum amount
	minPartitions, _ := cfg.GetMinPartitions(queueName)
	part.Lock()
	part.makePartitions(cfg, queueName, minPartitions)
	part.Unlock()
	return part
}

func (part *Partitions) PartitionCount() int {
	return part.partitionCount
}
func (part *Partitions) GetPartition(cfg *Config, queueName string, list *memberlist.Memberlist) (int, int, *Partition, error) {

	//get the node position and the node count
	nodePosition, nodeCount := getNodePosition(list)

	//calculate the range that our node is responsible for
	step := math.MaxInt64 / nodeCount
	nodeBottom := nodePosition * step
	nodeTop := (nodePosition + 1) * step
	myPartition, partition, totalPartitions, err := part.getPartitionPosition(cfg, queueName)
	if err != nil {
		logrus.Println(err)
	}

	// calculate my range for the given number
	node_range := nodeTop - nodeBottom
	nodeStep := node_range / totalPartitions
	partitionBottom := nodeStep*myPartition + nodeBottom
	partitionTop := nodeStep*(myPartition+1) + nodeBottom
	return partitionBottom, partitionTop, partition, err
}

//helper method to get the node position
func getNodePosition(list *memberlist.Memberlist) (int, int) {
	// figure out which node we are
	// grab and sort the node names
	nodes := list.Members()
	var nodeNames []string
	for _, node := range nodes {
		nodeNames = append(nodeNames, node.Name)
	}
	// sort our nodes so that we have a canonical ordering
	// node failure will cause more dupes
	sort.Strings(nodeNames)
	// find our index position
	nodePosition := sort.SearchStrings(nodeNames, list.LocalNode().Name)
	nodeCount := len(nodeNames)
	return nodePosition, nodeCount
}

func (part *Partitions) getPartitionPosition(cfg *Config, queueName string) (int, *Partition, int, error) {
	//iterate over the partitions and then increase or decrease the number of partitions

	//TODO move loging out of the sync operation for better throughput
	myPartition := -1

	var err error
	poppedPartition, _ := part.partitions.Pop()
	var workingPartition *Partition
	if poppedPartition != nil {
		workingPartition = poppedPartition.(*Partition)
	} else {
		// this seems a little scary
		return myPartition, workingPartition, part.partitionCount, errors.New("no available partitions")
	}
	visTimeout, _ := cfg.GetVisibilityTimeout(queueName)
	if time.Since(workingPartition.LastUsed).Seconds() > visTimeout {
		myPartition = workingPartition.Id
	} else {
		part.partitions.Push(workingPartition, workingPartition.LastUsed.UnixNano())
		part.Lock()
		maxPartitions, _ := cfg.GetMaxPartitions(queueName)
		if part.partitionCount < maxPartitions {
			workingPartition := new(Partition)
			workingPartition.Id = part.partitionCount
			myPartition = workingPartition.Id
			part.partitionCount = part.partitionCount + 1
		} else {
			err = errors.New("no available partitions")
		}
		part.Unlock()
	}
	return myPartition, workingPartition, part.partitionCount, err
}
func (part *Partitions) PushPartition(cfg *Config, queueName string, partition *Partition, lock bool) {
	if lock {
		partition.LastUsed = time.Now()
		part.partitions.Push(partition, partition.LastUsed.UnixNano())
	} else {
		visTimeout, _ := cfg.GetVisibilityTimeout(queueName)
		unlockTime := int(visTimeout)
		partition.LastUsed = time.Now().Add(-(time.Duration(unlockTime) * time.Second))
		part.partitions.Push(partition, partition.LastUsed.UnixNano())
	}
}

func (part *Partitions) makePartitions(cfg *Config, queueName string, partitionsToMake int) {
	var initialTime time.Time
	maxPartitions, _ := cfg.GetMaxPartitions(queueName)
	offset := part.partitionCount
	for partitionId := offset; partitionId < offset+partitionsToMake; partitionId++ {
		if maxPartitions > partitionId {
			partition := new(Partition)
			partition.Id = partitionId
			partition.LastUsed = initialTime
			part.partitions.Push(partition, rand.Int63n(100000))
			part.partitionCount = part.partitionCount + 1
		}
	}
}
func (part *Partitions) syncPartitions(cfg *Config, queueName string) {

	part.Lock()
	maxPartitions, _ := cfg.GetMaxPartitions(queueName)
	minPartitions, _ := cfg.GetMinPartitions(queueName)
	maxPartitionAge, _ := cfg.GetMaxPartitionAge(queueName)

	var partsRemoved int
	for partsRemoved = 0; maxPartitions < part.partitionCount; partsRemoved++ {
		_, _ = part.partitions.Pop()
	}
	part.partitionCount = part.partitionCount - partsRemoved

	if part.partitionCount < minPartitions {
		part.makePartitions(cfg, queueName, minPartitions-part.partitionCount)
	}

	// Partition Aging logic
	// pop a partition
	var workingPartition *Partition
	poppedPartition, _ := part.partitions.Pop()
	if poppedPartition != nil {
		workingPartition = poppedPartition.(*Partition)
	} else {
		// this seems a little scary. we do a similiar thing in getPartitionPosition
		return
	}
	part.partitionCount = part.partitionCount - 1

	// check if the partition is older than the max age ( but not a fresh partition )
	// if true pop the next partition, continue until this condition
	for time.Since(workingPartition.LastUsed).Seconds() > maxPartitionAge && part.partitionCount >= minPartitions {
		poppedPartition, _ = part.partitions.Pop()
		if poppedPartition != nil {
			workingPartition = poppedPartition.(*Partition)
		}
		part.partitionCount = part.partitionCount - 1
	}
	//when false push the last popped partition
	part.partitions.Push(workingPartition, workingPartition.LastUsed.UnixNano())
	part.partitionCount = part.partitionCount + 1
	part.Unlock()
}
<commit_msg>Fixing logrus print statement<commit_after>package app

import (
	"errors"
	"github.com/Sirupsen/logrus"
	"github.com/Tapjoy/lane"
	"github.com/hashicorp/memberlist"
	"math"
	"math/rand"
	"sort"
	"sync"
	"time"
)

type Partitions struct {
	partitions     *lane.PQueue
	partitionCount int
	sync.RWMutex
}

type Partition struct {
	Id       int
	LastUsed time.Time
}

func InitPartitions(cfg *Config, queueName string) *Partitions {
	part := &Partitions{
		partitions:     lane.NewPQueue(lane.MINPQ),
		partitionCount: 0,
	}
	// We'll initially allocate the minimum amount
	minPartitions, _ := cfg.GetMinPartitions(queueName)
	part.Lock()
	part.makePartitions(cfg, queueName, minPartitions)
	part.Unlock()
	return part
}

func (part *Partitions) PartitionCount() int {
	return part.partitionCount
}
func (part *Partitions) GetPartition(cfg *Config, queueName string, list *memberlist.Memberlist) (int, int, *Partition, error) {

	//get the node position and the node count
	nodePosition, nodeCount := getNodePosition(list)

	//calculate the range that our node is responsible for
	step := math.MaxInt64 / nodeCount
	nodeBottom := nodePosition * step
	nodeTop := (nodePosition + 1) * step
	myPartition, partition, totalPartitions, err := part.getPartitionPosition(cfg, queueName)
	if err != nil {
		logrus.Error(err)
	}

	// calculate my range for the given number
	node_range := nodeTop - nodeBottom
	nodeStep := node_range / totalPartitions
	partitionBottom := nodeStep*myPartition + nodeBottom
	partitionTop := nodeStep*(myPartition+1) + nodeBottom
	return partitionBottom, partitionTop, partition, err
}

//helper method to get the node position
func getNodePosition(list *memberlist.Memberlist) (int, int) {
	// figure out which node we are
	// grab and sort the node names
	nodes := list.Members()
	var nodeNames []string
	for _, node := range nodes {
		nodeNames = append(nodeNames, node.Name)
	}
	// sort our nodes so that we have a canonical ordering
	// node failure will cause more dupes
	sort.Strings(nodeNames)
	// find our index position
	nodePosition := sort.SearchStrings(nodeNames, list.LocalNode().Name)
	nodeCount := len(nodeNames)
	return nodePosition, nodeCount
}

func (part *Partitions) getPartitionPosition(cfg *Config, queueName string) (int, *Partition, int, error) {
	//iterate over the partitions and then increase or decrease the number of partitions

	//TODO move loging out of the sync operation for better throughput
	myPartition := -1

	var err error
	poppedPartition, _ := part.partitions.Pop()
	var workingPartition *Partition
	if poppedPartition != nil {
		workingPartition = poppedPartition.(*Partition)
	} else {
		// this seems a little scary
		return myPartition, workingPartition, part.partitionCount, errors.New("no available partitions")
	}
	visTimeout, _ := cfg.GetVisibilityTimeout(queueName)
	if time.Since(workingPartition.LastUsed).Seconds() > visTimeout {
		myPartition = workingPartition.Id
	} else {
		part.partitions.Push(workingPartition, workingPartition.LastUsed.UnixNano())
		part.Lock()
		maxPartitions, _ := cfg.GetMaxPartitions(queueName)
		if part.partitionCount < maxPartitions {
			workingPartition := new(Partition)
			workingPartition.Id = part.partitionCount
			myPartition = workingPartition.Id
			part.partitionCount = part.partitionCount + 1
		} else {
			err = errors.New("no available partitions")
		}
		part.Unlock()
	}
	return myPartition, workingPartition, part.partitionCount, err
}
func (part *Partitions) PushPartition(cfg *Config, queueName string, partition *Partition, lock bool) {
	if lock {
		partition.LastUsed = time.Now()
		part.partitions.Push(partition, partition.LastUsed.UnixNano())
	} else {
		visTimeout, _ := cfg.GetVisibilityTimeout(queueName)
		unlockTime := int(visTimeout)
		partition.LastUsed = time.Now().Add(-(time.Duration(unlockTime) * time.Second))
		part.partitions.Push(partition, partition.LastUsed.UnixNano())
	}
}

func (part *Partitions) makePartitions(cfg *Config, queueName string, partitionsToMake int) {
	var initialTime time.Time
	maxPartitions, _ := cfg.GetMaxPartitions(queueName)
	offset := part.partitionCount
	for partitionId := offset; partitionId < offset+partitionsToMake; partitionId++ {
		if maxPartitions > partitionId {
			partition := new(Partition)
			partition.Id = partitionId
			partition.LastUsed = initialTime
			part.partitions.Push(partition, rand.Int63n(100000))
			part.partitionCount = part.partitionCount + 1
		}
	}
}
func (part *Partitions) syncPartitions(cfg *Config, queueName string) {

	part.Lock()
	maxPartitions, _ := cfg.GetMaxPartitions(queueName)
	minPartitions, _ := cfg.GetMinPartitions(queueName)
	maxPartitionAge, _ := cfg.GetMaxPartitionAge(queueName)

	var partsRemoved int
	for partsRemoved = 0; maxPartitions < part.partitionCount; partsRemoved++ {
		_, _ = part.partitions.Pop()
	}
	part.partitionCount = part.partitionCount - partsRemoved

	if part.partitionCount < minPartitions {
		part.makePartitions(cfg, queueName, minPartitions-part.partitionCount)
	}

	// Partition Aging logic
	// pop a partition
	var workingPartition *Partition
	poppedPartition, _ := part.partitions.Pop()
	if poppedPartition != nil {
		workingPartition = poppedPartition.(*Partition)
	} else {
		// this seems a little scary. we do a similiar thing in getPartitionPosition
		return
	}
	part.partitionCount = part.partitionCount - 1

	// check if the partition is older than the max age ( but not a fresh partition )
	// if true pop the next partition, continue until this condition
	for time.Since(workingPartition.LastUsed).Seconds() > maxPartitionAge && part.partitionCount >= minPartitions {
		poppedPartition, _ = part.partitions.Pop()
		if poppedPartition != nil {
			workingPartition = poppedPartition.(*Partition)
		}
		part.partitionCount = part.partitionCount - 1
	}
	//when false push the last popped partition
	part.partitions.Push(workingPartition, workingPartition.LastUsed.UnixNano())
	part.partitionCount = part.partitionCount + 1
	part.Unlock()
}
<|endoftext|>


<commit_before>package providers

import (
	"os"
	"strings"

	log "github.com/Sirupsen/logrus"
	"github.com/camptocamp/conplicity/handler"
	"github.com/fgrehm/go-dockerpty"
	"github.com/fsouza/go-dockerclient"
)

type Provider interface {
	GetName() string
	GetHandler() *handler.Conplicity
	GetBackupDir() string
	PrepareBackup() error
}

func GetProvider(c *handler.Conplicity, v *docker.Volume) Provider {
	log.Infof("Detecting provider for volume %v", v.Name)
	if f, err := os.Stat(v.Mountpoint + "/PG_VERSION"); err == nil && f.Mode().IsRegular() {
		log.Infof("PG_VERSION file found, this should be a PostgreSQL datadir")
		return &PostgreSQLProvider{
			handler: c,
			vol:     v,
		}
	} else if f, err := os.Stat(v.Mountpoint + "/mysql"); err == nil && f.Mode().IsDir() {
		log.Infof("mysql directory found, this should be MySQL datadir")
		return &MySQLProvider{
			handler: c,
			vol:     v,
		}
	} else if f, err := os.Stat(v.Mountpoint + "/DB_CONFIG"); err == nil && f.Mode().IsRegular() {
		log.Infof("DB_CONFIG file found, this should be and OpenLDAP datadir")
		return &OpenLDAPProvider{
			handler: c,
			vol:     v,
		}
	} else {
		return &DefaultProvider{
			handler: c,
			vol:     v,
		}
	}
}

func BackupVolume(p Provider, vol *docker.Volume) (err error) {
	log.Infof("ID: " + vol.Name)
	log.Infof("Driver: " + vol.Driver)
	log.Infof("Mountpoint: " + vol.Mountpoint)

	log.Infof("Creating duplicity container...")

	c := p.GetHandler()

	fullIfOlderThan := getVolumeLabel(vol, ".full_if_older_than")
	if fullIfOlderThan == "" {
		fullIfOlderThan = c.FullIfOlderThan
	}

	pathSeparator := "/"
	if strings.HasPrefix(c.DuplicityTargetURL, "swift://") {
		// Looks like I'm not the one to fall on this issue: http://stackoverflow.com/questions/27991960/upload-to-swift-pseudo-folders-using-duplicity
		pathSeparator = "_"
	}

	backupDir := p.GetBackupDir()

	container, err := c.CreateContainer(
		docker.CreateContainerOptions{
			Config: &docker.Config{
				Cmd: []string{
					"--full-if-older-than", fullIfOlderThan,
					"--s3-use-new-style",
					"--no-encryption",
					"--allow-source-mismatch",
					vol.Mountpoint + "/" + backupDir,
					c.DuplicityTargetURL + pathSeparator + c.Hostname + pathSeparator + vol.Name,
				},
				Env: []string{
					"AWS_ACCESS_KEY_ID=" + c.AWSAccessKeyID,
					"AWS_SECRET_ACCESS_KEY=" + c.AWSSecretAccessKey,
					"SWIFT_USERNAME=" + c.SwiftUsername,
					"SWIFT_PASSWORD=" + c.SwiftPassword,
					"SWIFT_AUTHURL=" + c.SwiftAuthURL,
					"SWIFT_TENANTNAME=" + c.SwiftTenantName,
					"SWIFT_REGIONNAME=" + c.SwiftRegionName,
					"SWIFT_AUTHVERSION=2",
				},
				Image:        c.Image,
				OpenStdin:    true,
				StdinOnce:    true,
				AttachStdin:  true,
				AttachStdout: true,
				AttachStderr: true,
				Tty:          true,
			},
		},
	)

	checkErr(err, "Failed to create container for volume "+vol.Name+": %v", 1)

	defer func() {
		c.RemoveContainer(docker.RemoveContainerOptions{
			ID:    container.ID,
			Force: true,
		})
	}()

	binds := []string{
		vol.Name + ":" + vol.Mountpoint + ":ro",
	}

	err = dockerpty.Start(c.Client, container, &docker.HostConfig{
		Binds: binds,
	})
	checkErr(err, "Failed to start container for volume "+vol.Name+": %v", -1)
	return
	return nil
}

func checkErr(err error, msg string, exit int) {
	if err != nil {
		log.Errorf(msg, err)

		if exit != -1 {
			os.Exit(exit)
		}
	}
}

func getVolumeLabel(vol *docker.Volume, key string) (value string) {
	value = vol.Labels[labelPrefix+key]
	return
}
<commit_msg>Persist duplicity cache (Fixes #12)<commit_after>package providers

import (
	"os"
	"strings"

	log "github.com/Sirupsen/logrus"
	"github.com/camptocamp/conplicity/handler"
	"github.com/fgrehm/go-dockerpty"
	"github.com/fsouza/go-dockerclient"
)

type Provider interface {
	GetName() string
	GetHandler() *handler.Conplicity
	GetBackupDir() string
	PrepareBackup() error
}

func GetProvider(c *handler.Conplicity, v *docker.Volume) Provider {
	log.Infof("Detecting provider for volume %v", v.Name)
	if f, err := os.Stat(v.Mountpoint + "/PG_VERSION"); err == nil && f.Mode().IsRegular() {
		log.Infof("PG_VERSION file found, this should be a PostgreSQL datadir")
		return &PostgreSQLProvider{
			handler: c,
			vol:     v,
		}
	} else if f, err := os.Stat(v.Mountpoint + "/mysql"); err == nil && f.Mode().IsDir() {
		log.Infof("mysql directory found, this should be MySQL datadir")
		return &MySQLProvider{
			handler: c,
			vol:     v,
		}
	} else if f, err := os.Stat(v.Mountpoint + "/DB_CONFIG"); err == nil && f.Mode().IsRegular() {
		log.Infof("DB_CONFIG file found, this should be and OpenLDAP datadir")
		return &OpenLDAPProvider{
			handler: c,
			vol:     v,
		}
	} else {
		return &DefaultProvider{
			handler: c,
			vol:     v,
		}
	}
}

func BackupVolume(p Provider, vol *docker.Volume) (err error) {
	log.Infof("ID: " + vol.Name)
	log.Infof("Driver: " + vol.Driver)
	log.Infof("Mountpoint: " + vol.Mountpoint)

	log.Infof("Creating duplicity container...")

	c := p.GetHandler()

	fullIfOlderThan := getVolumeLabel(vol, ".full_if_older_than")
	if fullIfOlderThan == "" {
		fullIfOlderThan = c.FullIfOlderThan
	}

	pathSeparator := "/"
	if strings.HasPrefix(c.DuplicityTargetURL, "swift://") {
		// Looks like I'm not the one to fall on this issue: http://stackoverflow.com/questions/27991960/upload-to-swift-pseudo-folders-using-duplicity
		pathSeparator = "_"
	}

	backupDir := p.GetBackupDir()

	container, err := c.CreateContainer(
		docker.CreateContainerOptions{
			Config: &docker.Config{
				Cmd: []string{
					"--full-if-older-than", fullIfOlderThan,
					"--s3-use-new-style",
					"--no-encryption",
					"--allow-source-mismatch",
					vol.Mountpoint + "/" + backupDir,
					c.DuplicityTargetURL + pathSeparator + c.Hostname + pathSeparator + vol.Name,
				},
				Env: []string{
					"AWS_ACCESS_KEY_ID=" + c.AWSAccessKeyID,
					"AWS_SECRET_ACCESS_KEY=" + c.AWSSecretAccessKey,
					"SWIFT_USERNAME=" + c.SwiftUsername,
					"SWIFT_PASSWORD=" + c.SwiftPassword,
					"SWIFT_AUTHURL=" + c.SwiftAuthURL,
					"SWIFT_TENANTNAME=" + c.SwiftTenantName,
					"SWIFT_REGIONNAME=" + c.SwiftRegionName,
					"SWIFT_AUTHVERSION=2",
				},
				Image:        c.Image,
				OpenStdin:    true,
				StdinOnce:    true,
				AttachStdin:  true,
				AttachStdout: true,
				AttachStderr: true,
				Tty:          true,
			},
		},
	)

	checkErr(err, "Failed to create container for volume "+vol.Name+": %v", 1)

	defer func() {
		c.RemoveContainer(docker.RemoveContainerOptions{
			ID:    container.ID,
			Force: true,
		})
	}()

	binds := []string{
		vol.Name + ":" + vol.Mountpoint + ":ro",
		"duplicity_cache:/root/.cache/duplicity",
	}

	err = dockerpty.Start(c.Client, container, &docker.HostConfig{
		Binds: binds,
	})
	checkErr(err, "Failed to start container for volume "+vol.Name+": %v", -1)
	return
	return nil
}

func checkErr(err error, msg string, exit int) {
	if err != nil {
		log.Errorf(msg, err)

		if exit != -1 {
			os.Exit(exit)
		}
	}
}

func getVolumeLabel(vol *docker.Volume, key string) (value string) {
	value = vol.Labels[labelPrefix+key]
	return
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package patch

import (
	"encoding/json"
	"fmt"

	"github.com/evanphx/json-patch"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/util/strategicpatch"
	"k8s.io/client-go/kubernetes/scheme"
	"sigs.k8s.io/kustomize/pkg/gvk"
	"sigs.k8s.io/kustomize/pkg/resmap"
	"sigs.k8s.io/kustomize/pkg/resource"
	"sigs.k8s.io/kustomize/pkg/transformers"
)

// patchTransformer applies patches.
type patchTransformer struct {
	patches []*resource.Resource
	rf      *resource.Factory
}

var _ transformers.Transformer = &patchTransformer{}

// NewPatchTransformer constructs a patchTransformer.
func NewPatchTransformer(
	slice []*resource.Resource, rf *resource.Factory) (transformers.Transformer, error) {
	if len(slice) == 0 {
		return transformers.NewNoOpTransformer(), nil
	}
	return &patchTransformer{patches: slice, rf: rf}, nil
}

// Transform apply the patches on top of the base resources.
func (pt *patchTransformer) Transform(baseResourceMap resmap.ResMap) error {
	// Merge and then index the patches by Id.
	patches, err := pt.mergePatches()
	if err != nil {
		return err
	}

	// Strategic merge the resources exist in both base and patches.
	for _, patch := range patches {
		// Merge patches with base resource.
		id := patch.Id()
		matchedIds := baseResourceMap.FindByGVKN(id)
		if len(matchedIds) == 0 {
			return fmt.Errorf("failed to find an object with %#v to apply the patch", id.Gvk())
		}
		if len(matchedIds) > 1 {
			return fmt.Errorf("found multiple objects %#v targeted by patch %#v (ambiguous)", matchedIds, id)
		}
		id = matchedIds[0]
		base := baseResourceMap[id]
		merged := map[string]interface{}{}
		versionedObj, err := scheme.Scheme.New(toSchemaGvk(id.Gvk()))
		baseName := base.GetName()
		switch {
		case runtime.IsNotRegisteredError(err):
			// Use JSON merge patch to handle types w/o schema
			baseBytes, err := json.Marshal(base.Map())
			if err != nil {
				return err
			}
			patchBytes, err := json.Marshal(patch.Map())
			if err != nil {
				return err
			}
			mergedBytes, err := jsonpatch.MergePatch(baseBytes, patchBytes)
			if err != nil {
				return err
			}
			err = json.Unmarshal(mergedBytes, &merged)
			if err != nil {
				return err
			}
		case err != nil:
			return err
		default:
			// Use Strategic-Merge-Patch to handle types w/ schema
			// TODO: Change this to use the new Merge package.
			// Store the name of the base object, because this name may have been munged.
			// Apply this name to the patched object.
			lookupPatchMeta, err := strategicpatch.NewPatchMetaFromStruct(versionedObj)
			if err != nil {
				return err
			}
			merged, err = strategicpatch.StrategicMergeMapPatchUsingLookupPatchMeta(
				base.Map(),
				patch.Map(),
				lookupPatchMeta)
			if err != nil {
				return err
			}
		}
		base.SetName(baseName)
		baseResourceMap[id].SetMap(merged)
	}
	return nil
}

// mergePatches merge and index patches by Id.
// It errors out if there is conflict between patches.
func (pt *patchTransformer) mergePatches() (resmap.ResMap, error) {
	rc := resmap.ResMap{}
	for ix, patch := range pt.patches {
		id := patch.Id()
		existing, found := rc[id]
		if !found {
			rc[id] = patch
			continue
		}

		versionedObj, err := scheme.Scheme.New(toSchemaGvk(id.Gvk()))
		if err != nil && !runtime.IsNotRegisteredError(err) {
			return nil, err
		}
		var cd conflictDetector
		if err != nil {
			cd = newJMPConflictDetector(pt.rf)
		} else {
			cd, err = newSMPConflictDetector(versionedObj, pt.rf)
			if err != nil {
				return nil, err
			}
		}

		conflict, err := cd.hasConflict(existing, patch)
		if err != nil {
			return nil, err
		}
		if conflict {
			conflictingPatch, err := cd.findConflict(ix, pt.patches)
			if err != nil {
				return nil, err
			}
			return nil, fmt.Errorf(
				"conflict between %#v and %#v",
				conflictingPatch.Map(), patch.Map())
		}
		merged, err := cd.mergePatches(existing, patch)
		if err != nil {
			return nil, err
		}
		rc[id] = merged
	}
	return rc, nil
}

// toSchemaGvk converts to a schema.GroupVersionKind.
func toSchemaGvk(x gvk.Gvk) schema.GroupVersionKind {
	return schema.GroupVersionKind{
		Group:   x.Group,
		Version: x.Version,
		Kind:    x.Kind,
	}
}
<commit_msg>improve error message when failing to find an object to patch<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package patch

import (
	"encoding/json"
	"fmt"

	"github.com/evanphx/json-patch"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/util/strategicpatch"
	"k8s.io/client-go/kubernetes/scheme"
	"sigs.k8s.io/kustomize/pkg/gvk"
	"sigs.k8s.io/kustomize/pkg/resmap"
	"sigs.k8s.io/kustomize/pkg/resource"
	"sigs.k8s.io/kustomize/pkg/transformers"
)

// patchTransformer applies patches.
type patchTransformer struct {
	patches []*resource.Resource
	rf      *resource.Factory
}

var _ transformers.Transformer = &patchTransformer{}

// NewPatchTransformer constructs a patchTransformer.
func NewPatchTransformer(
	slice []*resource.Resource, rf *resource.Factory) (transformers.Transformer, error) {
	if len(slice) == 0 {
		return transformers.NewNoOpTransformer(), nil
	}
	return &patchTransformer{patches: slice, rf: rf}, nil
}

// Transform apply the patches on top of the base resources.
func (pt *patchTransformer) Transform(baseResourceMap resmap.ResMap) error {
	// Merge and then index the patches by Id.
	patches, err := pt.mergePatches()
	if err != nil {
		return err
	}

	// Strategic merge the resources exist in both base and patches.
	for _, patch := range patches {
		// Merge patches with base resource.
		id := patch.Id()
		matchedIds := baseResourceMap.FindByGVKN(id)
		if len(matchedIds) == 0 {
			return fmt.Errorf("failed to find an object with %s to apply the patch", id.GvknString())
		}
		if len(matchedIds) > 1 {
			return fmt.Errorf("found multiple objects %#v targeted by patch %#v (ambiguous)", matchedIds, id)
		}
		id = matchedIds[0]
		base := baseResourceMap[id]
		merged := map[string]interface{}{}
		versionedObj, err := scheme.Scheme.New(toSchemaGvk(id.Gvk()))
		baseName := base.GetName()
		switch {
		case runtime.IsNotRegisteredError(err):
			// Use JSON merge patch to handle types w/o schema
			baseBytes, err := json.Marshal(base.Map())
			if err != nil {
				return err
			}
			patchBytes, err := json.Marshal(patch.Map())
			if err != nil {
				return err
			}
			mergedBytes, err := jsonpatch.MergePatch(baseBytes, patchBytes)
			if err != nil {
				return err
			}
			err = json.Unmarshal(mergedBytes, &merged)
			if err != nil {
				return err
			}
		case err != nil:
			return err
		default:
			// Use Strategic-Merge-Patch to handle types w/ schema
			// TODO: Change this to use the new Merge package.
			// Store the name of the base object, because this name may have been munged.
			// Apply this name to the patched object.
			lookupPatchMeta, err := strategicpatch.NewPatchMetaFromStruct(versionedObj)
			if err != nil {
				return err
			}
			merged, err = strategicpatch.StrategicMergeMapPatchUsingLookupPatchMeta(
				base.Map(),
				patch.Map(),
				lookupPatchMeta)
			if err != nil {
				return err
			}
		}
		base.SetName(baseName)
		baseResourceMap[id].SetMap(merged)
	}
	return nil
}

// mergePatches merge and index patches by Id.
// It errors out if there is conflict between patches.
func (pt *patchTransformer) mergePatches() (resmap.ResMap, error) {
	rc := resmap.ResMap{}
	for ix, patch := range pt.patches {
		id := patch.Id()
		existing, found := rc[id]
		if !found {
			rc[id] = patch
			continue
		}

		versionedObj, err := scheme.Scheme.New(toSchemaGvk(id.Gvk()))
		if err != nil && !runtime.IsNotRegisteredError(err) {
			return nil, err
		}
		var cd conflictDetector
		if err != nil {
			cd = newJMPConflictDetector(pt.rf)
		} else {
			cd, err = newSMPConflictDetector(versionedObj, pt.rf)
			if err != nil {
				return nil, err
			}
		}

		conflict, err := cd.hasConflict(existing, patch)
		if err != nil {
			return nil, err
		}
		if conflict {
			conflictingPatch, err := cd.findConflict(ix, pt.patches)
			if err != nil {
				return nil, err
			}
			return nil, fmt.Errorf(
				"conflict between %#v and %#v",
				conflictingPatch.Map(), patch.Map())
		}
		merged, err := cd.mergePatches(existing, patch)
		if err != nil {
			return nil, err
		}
		rc[id] = merged
	}
	return rc, nil
}

// toSchemaGvk converts to a schema.GroupVersionKind.
func toSchemaGvk(x gvk.Gvk) schema.GroupVersionKind {
	return schema.GroupVersionKind{
		Group:   x.Group,
		Version: x.Version,
		Kind:    x.Kind,
	}
}
<|endoftext|>


<commit_before>package command

import (
	"bufio"
	"flag"
	"fmt"
	"net"
	"sort"
	"strings"

	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/gocli"
	"github.com/funkygao/golib/color"
	"github.com/funkygao/golib/pipestream"
	consulapi "github.com/hashicorp/consul/api"
	"github.com/ryanuber/columnize"
)

// consul members will include:
// - zk cluster as server
// - agents
//   - brokers
//   - kateway
type Members struct {
	Ui  cli.Ui
	Cmd string

	brokerHosts, zkHosts, katewayHosts map[string]struct{}
	nodeHostMap                        map[string]string // consul members node->ip
}

func (this *Members) Run(args []string) (exitCode int) {
	var (
		zone        string
		showLoadAvg bool
		exec        string
		node        string
		role        string
	)
	cmdFlags := flag.NewFlagSet("members", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&zone, "z", ctx.ZkDefaultZone(), "")
	cmdFlags.BoolVar(&showLoadAvg, "l", false, "")
	cmdFlags.StringVar(&exec, "exec", "", "")
	cmdFlags.StringVar(&node, "n", "", "")
	cmdFlags.StringVar(&role, "r", "", "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	if validateArgs(this, this.Ui).
		requireAdminRights("-exec").
		invalid(args) {
		return 2
	}

	zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone)))
	this.fetchAllRunningHostsFromZk(zkzone)

	members := this.consulMembers()
	consulLiveNode := make([]string, 0, len(members))
	for _, member := range members {
		if member.Status == 1 {
			consulLiveNode = append(consulLiveNode, member.Addr)
		} else {
			this.Ui.Warn(fmt.Sprintf("%s %s status:%d", member.Name, member.Addr, member.Status))
		}
	}

	consulLiveMap := make(map[string]struct{})
	brokerN, zkN, katewayN, unknownN := 0, 0, 0, 0
	for _, node := range consulLiveNode {
		_, presentInBroker := this.brokerHosts[node]
		_, presentInZk := this.zkHosts[node]
		_, presentInKateway := this.katewayHosts[node]
		if presentInBroker {
			brokerN++
		}
		if presentInZk {
			zkN++
		}
		if presentInKateway {
			katewayN++
		}

		if !presentInBroker && !presentInZk && !presentInKateway {
			unknownN++
		}

		consulLiveMap[node] = struct{}{}
	}

	// all brokers should run consul
	for broker, _ := range this.brokerHosts {
		if _, present := consulLiveMap[broker]; !present {
			this.Ui.Warn(fmt.Sprintf("- %s", broker))
		}
	}

	switch {
	case showLoadAvg:
		this.displayLoadAvg(role)

	case exec != "":
		this.executeOnAll(exec, role, node)

	default:
		this.displayMembers(members, role)
	}

	// summary
	this.Ui.Output(fmt.Sprintf("Zk:%d Broker:%d Kateway:%d ?:%s => %d",
		zkN, brokerN, katewayN, color.Yellow("%d", unknownN), zkN+brokerN+katewayN+unknownN))

	return
}

func (this *Members) fetchAllRunningHostsFromZk(zkzone *zk.ZkZone) {
	this.brokerHosts = make(map[string]struct{})
	zkzone.ForSortedBrokers(func(cluster string, brokers map[string]*zk.BrokerZnode) {
		for _, brokerInfo := range brokers {
			this.brokerHosts[brokerInfo.Host] = struct{}{}
		}
	})

	this.zkHosts = make(map[string]struct{})
	for _, addr := range zkzone.ZkAddrList() {
		zkNode, _, err := net.SplitHostPort(addr)
		swallow(err)
		this.zkHosts[zkNode] = struct{}{}
	}

	this.katewayHosts = make(map[string]struct{})
	kws, err := zkzone.KatewayInfos()
	swallow(err)
	for _, kw := range kws {
		host, _, err := net.SplitHostPort(kw.PubAddr)
		swallow(err)
		this.katewayHosts[host] = struct{}{}
	}
}

// TODO role is ignored
func (this *Members) executeOnAll(execCmd string, role string, node string) {
	args := []string{"exec"}
	if node != "" {
		args = append(args, fmt.Sprintf("-node=%s", node))
	}
	args = append(args, strings.Split(execCmd, " ")...)
	cmd := pipestream.New("consul", args...)
	err := cmd.Open()
	swallow(err)
	defer cmd.Close()

	this.Ui.Info(fmt.Sprintf("%s ...", execCmd))

	lines := make([]string, 0)
	header := "Node|Host|Role|Result"
	lines = append(lines, header)

	scanner := bufio.NewScanner(cmd.Reader())
	scanner.Split(bufio.ScanLines)
	for scanner.Scan() {
		line := scanner.Text()
		if strings.Contains(line, "finished with exit code 0") ||
			strings.Contains(line, "completed / acknowledged") {
			continue
		}
		fields := strings.Fields(line)
		if len(fields) == 1 {
			continue
		}

		node := fields[0]
		if strings.HasSuffix(node, ":") {
			node = strings.TrimRight(node, ":")
		}

		host := this.nodeHostMap[node]
		lines = append(lines, fmt.Sprintf("%s|%s|%s|%s", node, host, this.roleOfHost(host),
			strings.Join(fields[1:], " ")))
	}

	if len(lines) > 1 {
		sort.Strings(lines[1:])
		this.Ui.Output(columnize.SimpleFormat(lines))
	}
}

func (this *Members) displayMembers(members []*consulapi.AgentMember, role string) {
	lines := make([]string, 0)
	header := "Node|Host|Role"
	lines = append(lines, header)
	for _, member := range members {
		hostRole := this.roleOfHost(member.Addr)
		if role != "" && role != hostRole {
			continue
		}

		lines = append(lines, fmt.Sprintf("%s|%s|%s", member.Name, member.Addr, hostRole))
	}

	if len(lines) > 1 {
		sort.Strings(lines[1:])
		this.Ui.Output(columnize.SimpleFormat(lines))
	}
}

func (this *Members) displayLoadAvg(role string) {
	cmd := pipestream.New("consul", "exec",
		"uptime", "|", "grep", "load")
	err := cmd.Open()
	swallow(err)
	defer cmd.Close()

	lines := make([]string, 0)
	header := "Node|Host|Role|Load Avg"
	lines = append(lines, header)

	scanner := bufio.NewScanner(cmd.Reader())
	scanner.Split(bufio.ScanLines)
	for scanner.Scan() {
		line := scanner.Text()
		fields := strings.Fields(line)
		node := fields[0]
		parts := strings.Split(line, "load average:")
		if len(parts) < 2 {
			continue
		}
		if strings.HasSuffix(node, ":") {
			node = strings.TrimRight(node, ":")
		}

		loadAvg := strings.TrimSpace(parts[1])
		if loadAvg[0] > '0' {
			loadAvg += " !"
			if loadAvg[0] > '1' {
				loadAvg += "!"
			}
		}

		host := this.nodeHostMap[node]
		hostRole := this.roleOfHost(host)
		if role != "" && hostRole != role {
			continue
		}

		lines = append(lines, fmt.Sprintf("%s|%s|%s|%s", node, host, hostRole, loadAvg))
	}

	if len(lines) > 1 {
		sort.Strings(lines[1:])
		this.Ui.Output(columnize.SimpleFormat(lines))
	}
}

func (this *Members) roleOfHost(host string) string {
	if _, present := this.brokerHosts[host]; present {
		return "B"
	}
	if _, present := this.zkHosts[host]; present {
		return "Z"
	}
	if _, present := this.katewayHosts[host]; present {
		return "K"
	}
	return "?"
}

func (this *Members) consulMembers() []*consulapi.AgentMember {
	cf := consulapi.DefaultConfig()
	client, err := consulapi.NewClient(cf)
	swallow(err)
	members, err := client.Agent().Members(false)
	swallow(err)

	m := make(map[string]*consulapi.AgentMember, len(members))
	sortedName := make([]string, 0, len(members))
	this.nodeHostMap = make(map[string]string, len(members))
	for _, member := range members {
		m[member.Name] = member
		this.nodeHostMap[member.Name] = member.Addr

		sortedName = append(sortedName, member.Name)
	}
	sort.Strings(sortedName)

	r := make([]*consulapi.AgentMember, 0, len(sortedName))
	for _, name := range sortedName {
		r = append(r, m[name])
	}

	return r
}

func (*Members) Synopsis() string {
	return "Verify consul members match kafka zone"
}

func (this *Members) Help() string {
	help := fmt.Sprintf(`
Usage: %s members [options]

    Verify consul members match kafka zone

    -z zone

    -r role

    -l
      Display each member load average

    -exec <cmd>
      Execute cmd on all members and print the result by host
      e,g. gk members -exec "ifconfig bond0 | grep 'TX bytes'"

    -n node
      Execute cmd on a single node

`, this.Cmd)
	return strings.TrimSpace(help)
}
<commit_msg>tweak of the load avg highlight<commit_after>package command

import (
	"bufio"
	"flag"
	"fmt"
	"net"
	"sort"
	"strings"

	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/gocli"
	"github.com/funkygao/golib/color"
	"github.com/funkygao/golib/pipestream"
	consulapi "github.com/hashicorp/consul/api"
	"github.com/ryanuber/columnize"
)

// consul members will include:
// - zk cluster as server
// - agents
//   - brokers
//   - kateway
type Members struct {
	Ui  cli.Ui
	Cmd string

	brokerHosts, zkHosts, katewayHosts map[string]struct{}
	nodeHostMap                        map[string]string // consul members node->ip
}

func (this *Members) Run(args []string) (exitCode int) {
	var (
		zone        string
		showLoadAvg bool
		exec        string
		node        string
		role        string
	)
	cmdFlags := flag.NewFlagSet("members", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&zone, "z", ctx.ZkDefaultZone(), "")
	cmdFlags.BoolVar(&showLoadAvg, "l", false, "")
	cmdFlags.StringVar(&exec, "exec", "", "")
	cmdFlags.StringVar(&node, "n", "", "")
	cmdFlags.StringVar(&role, "r", "", "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	if validateArgs(this, this.Ui).
		requireAdminRights("-exec").
		invalid(args) {
		return 2
	}

	zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone)))
	this.fetchAllRunningHostsFromZk(zkzone)

	members := this.consulMembers()
	consulLiveNode := make([]string, 0, len(members))
	for _, member := range members {
		if member.Status == 1 {
			consulLiveNode = append(consulLiveNode, member.Addr)
		} else {
			this.Ui.Warn(fmt.Sprintf("%s %s status:%d", member.Name, member.Addr, member.Status))
		}
	}

	consulLiveMap := make(map[string]struct{})
	brokerN, zkN, katewayN, unknownN := 0, 0, 0, 0
	for _, node := range consulLiveNode {
		_, presentInBroker := this.brokerHosts[node]
		_, presentInZk := this.zkHosts[node]
		_, presentInKateway := this.katewayHosts[node]
		if presentInBroker {
			brokerN++
		}
		if presentInZk {
			zkN++
		}
		if presentInKateway {
			katewayN++
		}

		if !presentInBroker && !presentInZk && !presentInKateway {
			unknownN++
		}

		consulLiveMap[node] = struct{}{}
	}

	// all brokers should run consul
	for broker, _ := range this.brokerHosts {
		if _, present := consulLiveMap[broker]; !present {
			this.Ui.Warn(fmt.Sprintf("- %s", broker))
		}
	}

	switch {
	case showLoadAvg:
		this.displayLoadAvg(role)

	case exec != "":
		this.executeOnAll(exec, role, node)

	default:
		this.displayMembers(members, role)
	}

	// summary
	this.Ui.Output(fmt.Sprintf("Zk:%d Broker:%d Kateway:%d ?:%s => %d",
		zkN, brokerN, katewayN, color.Yellow("%d", unknownN), zkN+brokerN+katewayN+unknownN))

	return
}

func (this *Members) fetchAllRunningHostsFromZk(zkzone *zk.ZkZone) {
	this.brokerHosts = make(map[string]struct{})
	zkzone.ForSortedBrokers(func(cluster string, brokers map[string]*zk.BrokerZnode) {
		for _, brokerInfo := range brokers {
			this.brokerHosts[brokerInfo.Host] = struct{}{}
		}
	})

	this.zkHosts = make(map[string]struct{})
	for _, addr := range zkzone.ZkAddrList() {
		zkNode, _, err := net.SplitHostPort(addr)
		swallow(err)
		this.zkHosts[zkNode] = struct{}{}
	}

	this.katewayHosts = make(map[string]struct{})
	kws, err := zkzone.KatewayInfos()
	swallow(err)
	for _, kw := range kws {
		host, _, err := net.SplitHostPort(kw.PubAddr)
		swallow(err)
		this.katewayHosts[host] = struct{}{}
	}
}

// TODO role is ignored
func (this *Members) executeOnAll(execCmd string, role string, node string) {
	args := []string{"exec"}
	if node != "" {
		args = append(args, fmt.Sprintf("-node=%s", node))
	}
	args = append(args, strings.Split(execCmd, " ")...)
	cmd := pipestream.New("consul", args...)
	err := cmd.Open()
	swallow(err)
	defer cmd.Close()

	this.Ui.Info(fmt.Sprintf("%s ...", execCmd))

	lines := make([]string, 0)
	header := "Node|Host|Role|Result"
	lines = append(lines, header)

	scanner := bufio.NewScanner(cmd.Reader())
	scanner.Split(bufio.ScanLines)
	for scanner.Scan() {
		line := scanner.Text()
		if strings.Contains(line, "finished with exit code 0") ||
			strings.Contains(line, "completed / acknowledged") {
			continue
		}
		fields := strings.Fields(line)
		if len(fields) == 1 {
			continue
		}

		node := fields[0]
		if strings.HasSuffix(node, ":") {
			node = strings.TrimRight(node, ":")
		}

		host := this.nodeHostMap[node]
		lines = append(lines, fmt.Sprintf("%s|%s|%s|%s", node, host, this.roleOfHost(host),
			strings.Join(fields[1:], " ")))
	}

	if len(lines) > 1 {
		sort.Strings(lines[1:])
		this.Ui.Output(columnize.SimpleFormat(lines))
	}
}

func (this *Members) displayMembers(members []*consulapi.AgentMember, role string) {
	lines := make([]string, 0)
	header := "Node|Host|Role"
	lines = append(lines, header)
	for _, member := range members {
		hostRole := this.roleOfHost(member.Addr)
		if role != "" && role != hostRole {
			continue
		}

		lines = append(lines, fmt.Sprintf("%s|%s|%s", member.Name, member.Addr, hostRole))
	}

	if len(lines) > 1 {
		sort.Strings(lines[1:])
		this.Ui.Output(columnize.SimpleFormat(lines))
	}
}

func (this *Members) displayLoadAvg(role string) {
	cmd := pipestream.New("consul", "exec",
		"uptime", "|", "grep", "load")
	err := cmd.Open()
	swallow(err)
	defer cmd.Close()

	lines := make([]string, 0)
	header := "Node|Host|Role|Load Avg"
	lines = append(lines, header)

	scanner := bufio.NewScanner(cmd.Reader())
	scanner.Split(bufio.ScanLines)
	for scanner.Scan() {
		line := scanner.Text()
		fields := strings.Fields(line)
		node := fields[0]
		parts := strings.Split(line, "load average:")
		if len(parts) < 2 {
			continue
		}
		if strings.HasSuffix(node, ":") {
			node = strings.TrimRight(node, ":")
		}

		loadAvg := strings.TrimSpace(parts[1])
		if loadAvg[0] > '0' {
			loadAvg += " !"
			if loadAvg[0] > '2' {
				loadAvg += "!"
			}
			if loadAvg[0] > '4' {
				loadAvg += "!"
			}
		}

		host := this.nodeHostMap[node]
		hostRole := this.roleOfHost(host)
		if role != "" && hostRole != role {
			continue
		}

		lines = append(lines, fmt.Sprintf("%s|%s|%s|%s", node, host, hostRole, loadAvg))
	}

	if len(lines) > 1 {
		sort.Strings(lines[1:])
		this.Ui.Output(columnize.SimpleFormat(lines))
	}
}

func (this *Members) roleOfHost(host string) string {
	if _, present := this.brokerHosts[host]; present {
		return "B"
	}
	if _, present := this.zkHosts[host]; present {
		return "Z"
	}
	if _, present := this.katewayHosts[host]; present {
		return "K"
	}
	return "?"
}

func (this *Members) consulMembers() []*consulapi.AgentMember {
	cf := consulapi.DefaultConfig()
	client, err := consulapi.NewClient(cf)
	swallow(err)
	members, err := client.Agent().Members(false)
	swallow(err)

	m := make(map[string]*consulapi.AgentMember, len(members))
	sortedName := make([]string, 0, len(members))
	this.nodeHostMap = make(map[string]string, len(members))
	for _, member := range members {
		m[member.Name] = member
		this.nodeHostMap[member.Name] = member.Addr

		sortedName = append(sortedName, member.Name)
	}
	sort.Strings(sortedName)

	r := make([]*consulapi.AgentMember, 0, len(sortedName))
	for _, name := range sortedName {
		r = append(r, m[name])
	}

	return r
}

func (*Members) Synopsis() string {
	return "Verify consul members match kafka zone"
}

func (this *Members) Help() string {
	help := fmt.Sprintf(`
Usage: %s members [options]

    Verify consul members match kafka zone

    -z zone

    -r role

    -l
      Display each member load average

    -exec <cmd>
      Execute cmd on all members and print the result by host
      e,g. gk members -exec "ifconfig bond0 | grep 'TX bytes'"

    -n node
      Execute cmd on a single node

`, this.Cmd)
	return strings.TrimSpace(help)
}
<|endoftext|>