<commit_before>/*
Copyright 2018 The Doctl Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package commands

import (
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"sort"
	"strings"

	"github.com/apache/openwhisk-client-go/whisk"
	"github.com/digitalocean/doctl"
	"github.com/digitalocean/doctl/commands/displayers"
	"github.com/digitalocean/doctl/do"
	"github.com/spf13/cobra"
)

// Functions generates the serverless 'functions' subtree for addition to the doctl command
func Functions() *Command {
	cmd := &Command{
		Command: &cobra.Command{
			Use:   "functions",
			Short: "Work with the functions in your namespace",
			Long: `The subcommands of ` + "`" + `doctl serverless functions` + "`" + ` operate on your functions namespace.
You are able to inspect and list these functions to know what is deployed.  You can also invoke functions to test them.`,
			Aliases: []string{"fn"},
		},
	}

	get := CmdBuilder(cmd, RunFunctionsGet, "get <functionName>", "Retrieves the deployed copy of a function (code or metadata)",
		`Use `+"`"+`doctl serverless functions get`+"`"+` to obtain the code or metadata of a deployed function.
This allows you to inspect the deployed copy and ascertain whether it corresponds to what
is in your functions project in the local file system.`,
		Writer)
	AddBoolFlag(get, "url", "r", false, "get function url")
	AddBoolFlag(get, "code", "", false, "show function code (only works if code is not a zip file)")
	AddStringFlag(get, "save-env", "E", "", "save environment variables to FILE as key-value pairs")
	AddStringFlag(get, "save-env-json", "J", "", "save environment variables to FILE as JSON")
	AddBoolFlag(get, "save", "", false, "save function code to file corresponding to the function name")
	AddStringFlag(get, "save-as", "", "", "file to save function code to")

	invoke := CmdBuilder(cmd, RunFunctionsInvoke, "invoke <functionName>", "Invokes a function",
		`Use `+"`"+`doctl serverless functions invoke`+"`"+` to invoke a function in your functions namespace.
You can provide inputs and inspect outputs.`,
		Writer)
	AddBoolFlag(invoke, "web", "", false, "Invoke as a web function, show result as web page")
	AddStringSliceFlag(invoke, "param", "p", []string{}, "parameter values in KEY:VALUE format, list allowed")
	AddStringFlag(invoke, "param-file", "P", "", "FILE containing parameter values in JSON format")
	AddBoolFlag(invoke, "full", "f", false, "wait for full activation record")
	AddBoolFlag(invoke, "no-wait", "n", false, "fire and forget (asynchronous invoke, does not wait for the result)")

	list := CmdBuilder(cmd, RunFunctionsList, "list [<packageName>]", "Lists the functions in your functions namespace",
		`Use `+"`"+`doctl serverless functions list`+"`"+` to list the functions in your functions namespace.`,
		Writer, aliasOpt("ls"), displayerType(&displayers.Functions{}))
	AddStringFlag(list, "limit", "l", "", "only return LIMIT number of functions (default 30, max 200)")
	AddStringFlag(list, "skip", "s", "", "exclude the first SKIP number of functions from the result")
	AddBoolFlag(list, "count", "", false, "show only the total number of functions")
	AddBoolFlag(list, "name-sort", "", false, "sort results by name")
	AddBoolFlag(list, "name", "n", false, "sort results by name")

	return cmd
}

// RunFunctionsGet supports the 'serverless functions get' command
func RunFunctionsGet(c *CmdConfig) error {
	err := ensureOneArg(c)
	if err != nil {
		return err
	}
	urlFlag, _ := c.Doit.GetBool(c.NS, flagURL)
	codeFlag, _ := c.Doit.GetBool(c.NS, flagCode)
	saveFlag, _ := c.Doit.GetBool(c.NS, flagSave)
	saveAsFlag, _ := c.Doit.GetString(c.NS, flagSaveAs)
	saveEnvFlag, _ := c.Doit.GetString(c.NS, flagSaveEnv)
	saveEnvJSONFlag, _ := c.Doit.GetString(c.NS, flagSaveEnvJSON)
	fetchCode := codeFlag || saveFlag || saveAsFlag != ""

	sls := c.Serverless()
	action, parms, err := sls.GetFunction(c.Args[0], fetchCode)
	if err != nil {
		return err
	}

	if urlFlag {
		host, err := sls.GetConnectedAPIHost()
		if err != nil {
			return err
		}
		_, err = fmt.Fprintln(c.Out, computeURL(action, host))
		return err
	}

	if saveFlag || saveAsFlag != "" {
		return doSaveFunctionCode(action, saveFlag, saveAsFlag)
	}

	if saveEnvFlag != "" || saveEnvJSONFlag != "" {
		return doSaveFunctionEnvironment(saveEnvFlag, saveEnvJSONFlag, parms)
	}

	if codeFlag {
		if !*action.Exec.Binary {
			_, err = fmt.Fprintln(c.Out, *action.Exec.Code)
			return err
		}
		return errors.New("Binary code cannot be displayed on the console")
	}

	output := do.ServerlessOutput{Entity: action}
	return c.PrintServerlessTextOutput(output)
}

// doSaveFunctionCode performs the save operations for code
func doSaveFunctionCode(action whisk.Action, save bool, saveAs string) error {
	var extension string // used only when save and !saveAs
	var data []byte
	if *action.Exec.Binary {
		extension = ".zip"
		decoded, err := base64.StdEncoding.DecodeString(*action.Exec.Code)
		if err != nil {
			return err
		}
		data = decoded
	} else {
		extension = fileExtensionForKind(action.Exec.Kind) // find equivalent
		data = []byte(*action.Exec.Code)
	}
	if save && saveAs == "" {
		saveAs = action.Name + extension
	}
	if saveAs != "" {
		err := os.WriteFile(saveAs, data, 0666)
		if err != nil {
			return err
		}
	}

	return nil
}

// doSaveFunctionEnvironment saves the environment variables for a function to file,
// either as key-value pairs or JSON.  Could do both if both are specified.
func doSaveFunctionEnvironment(saveEnv string, saveEnvJSON string, parms []do.FunctionParameter) error {
	keyVals := []string{}
	envMap := map[string]string{}
	for _, parm := range parms {
		if parm.Init {
			keyVal := parm.Key + "=" + parm.Value
			keyVals = append(keyVals, keyVal)
			envMap[parm.Key] = parm.Value
		}
	}

	if saveEnv != "" {
		data := []byte(strings.Join(keyVals, "\n"))
		err := os.WriteFile(saveEnv, data, 0666)
		if err != nil {
			return err
		}
	}

	if saveEnvJSON != "" {
		data, err := json.MarshalIndent(&envMap, "", "  ")
		if err != nil {
			return err
		}
		err = os.WriteFile(saveEnvJSON, data, 0666)
		if err != nil {
			return err
		}
	}

	return nil
}

// fileExtensionforKind finds the right file extension for a given runtime 'kind'.
// This code will require modification when the repertoire of runtimes is extended.
func fileExtensionForKind(kind string) string {
	lang := strings.Split(kind, ":")[0]
	switch strings.ToLower(lang) {
	case "go":
		return ".go"
	case "nodejs":
		return ".js"
	case "php":
		return ".php"
	case "python":
		return ".py"
	}
	return ""
}

// computeURL determines the URL string based on the action get output.
// Based on code in aio-cli-plugin-runtime, src/commands/runtime/action/get.js
func computeURL(action whisk.Action, host string) string {
	nameParts := strings.Split(action.Namespace, "/")
	namespace := nameParts[0]
	var packageName string
	if len(nameParts) > 1 {
		packageName = nameParts[1]
	}
	if action.WebAction() {
		if packageName == "" {
			packageName = "default"
		}
		return fmt.Sprintf("%s/api/v1/web/%s/%s/%s", host, namespace, packageName, action.Name)
	}
	if packageName != "" {
		packageName += "/"
	}
	return fmt.Sprintf("%s/api/v1/namespaces/%s/actions/%s%s", host, namespace, packageName, action.Name)
}

// RunFunctionsInvoke supports the 'serverless functions invoke' command
func RunFunctionsInvoke(c *CmdConfig) error {
	err := ensureOneArg(c)
	if err != nil {
		return err
	}
	paramFile, _ := c.Doit.GetString(c.NS, flagParamFile)
	paramFlags, _ := c.Doit.GetStringSlice(c.NS, flagParam)
	params, err := consolidateParams(paramFile, paramFlags)
	if err != nil {
		return err
	}
	web, _ := c.Doit.GetBool(c.NS, flagWeb)
	if web {
		var mapParams map[string]interface{} = nil
		if params != nil {
			p, ok := params.(map[string]interface{})
			if !ok {
				return fmt.Errorf("cannot invoke via web: parameters do not form a dictionary")
			}
			mapParams = p
		}
		return c.Serverless().InvokeFunctionViaWeb(c.Args[0], mapParams)
	}
	full, _ := c.Doit.GetBool(c.NS, flagFull)
	noWait, _ := c.Doit.GetBool(c.NS, flagNoWait)
	blocking := !noWait
	result := blocking && !full
	response, err := c.Serverless().InvokeFunction(c.Args[0], params, blocking, result)
	if err != nil {
		return err
	}
	output := do.ServerlessOutput{Entity: response}
	return c.PrintServerlessTextOutput(output)
}

// RunFunctionsList supports the 'serverless functions list' command
func RunFunctionsList(c *CmdConfig) error {
	argCount := len(c.Args)
	if argCount > 1 {
		return doctl.NewTooManyArgsErr(c.NS)
	}
	var pkg string
	if argCount == 1 {
		pkg = c.Args[0]
	}
	// Determine if '--count' is requested since we will use simple text output in that case.
	// Count is mutually exclusive with the global format flag.
	count, _ := c.Doit.GetBool(c.NS, flagCount)
	if count && c.Doit.IsSet("format") {
		return errors.New("the --count and --format flags are mutually exclusive")
	}
	// Retrieve other flags
	skip, _ := c.Doit.GetInt(c.NS, flagSkip)
	limit, _ := c.Doit.GetInt(c.NS, flagLimit)
	nameSort, _ := c.Doit.GetBool(c.NS, flagNameSort)
	nameName, _ := c.Doit.GetBool(c.NS, flagNameName)
	// Get information from backend
	list, err := c.Serverless().ListFunctions(pkg, skip, limit)
	if err != nil {
		return err
	}
	if count {
		plural := "s"
		are := "are"
		if len(list) == 1 {
			plural = ""
			are = "is"
		}
		fmt.Fprintf(c.Out, "There %s %d function%s in this namespace.\n", are, len(list), plural)
		return nil
	}
	if nameSort || nameName {
		sortFunctionList(list)
	}
	return c.Display(&displayers.Functions{Info: list})
}

// sortFunctionList performs a sort of a function list (by name)
func sortFunctionList(list []whisk.Action) {
	isLess := func(i, j int) bool {
		return list[i].Name < list[j].Name
	}
	sort.Slice(list, isLess)
}

// consolidateParams accepts parameters from a file, the command line, or both, and consolidates all
// such parameters into a simple dictionary.
func consolidateParams(paramFile string, params []string) (interface{}, error) {
	consolidated := map[string]interface{}{}
	if len(paramFile) > 0 {
		contents, err := os.ReadFile(paramFile)
		if err != nil {
			return nil, err
		}
		err = json.Unmarshal(contents, &consolidated)
		if err != nil {
			return nil, err
		}
	}
	for _, param := range params {
		parts := strings.Split(param, ":")
		if len(parts) < 2 {
			return nil, fmt.Errorf("values for --params must have KEY:VALUE form")
		}
		parts1 := strings.Join(parts[1:], ":")
		consolidated[parts[0]] = parts1
	}
	if len(consolidated) > 0 {
		return consolidated, nil
	}
	return nil, nil
}
<commit_msg>Adds a more descriptive output of what happens when long running functions are invoked synchronously (#1298)<commit_after>/*
Copyright 2018 The Doctl Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package commands

import (
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"sort"
	"strings"

	"github.com/apache/openwhisk-client-go/whisk"
	"github.com/digitalocean/doctl"
	"github.com/digitalocean/doctl/commands/charm/template"
	"github.com/digitalocean/doctl/commands/displayers"
	"github.com/digitalocean/doctl/do"
	"github.com/spf13/cobra"
)

// Functions generates the serverless 'functions' subtree for addition to the doctl command
func Functions() *Command {
	cmd := &Command{
		Command: &cobra.Command{
			Use:   "functions",
			Short: "Work with the functions in your namespace",
			Long: `The subcommands of ` + "`" + `doctl serverless functions` + "`" + ` operate on your functions namespace.
You are able to inspect and list these functions to know what is deployed.  You can also invoke functions to test them.`,
			Aliases: []string{"fn"},
		},
	}

	get := CmdBuilder(cmd, RunFunctionsGet, "get <functionName>", "Retrieves the deployed copy of a function (code or metadata)",
		`Use `+"`"+`doctl serverless functions get`+"`"+` to obtain the code or metadata of a deployed function.
This allows you to inspect the deployed copy and ascertain whether it corresponds to what
is in your functions project in the local file system.`,
		Writer)
	AddBoolFlag(get, "url", "r", false, "get function url")
	AddBoolFlag(get, "code", "", false, "show function code (only works if code is not a zip file)")
	AddStringFlag(get, "save-env", "E", "", "save environment variables to FILE as key-value pairs")
	AddStringFlag(get, "save-env-json", "J", "", "save environment variables to FILE as JSON")
	AddBoolFlag(get, "save", "", false, "save function code to file corresponding to the function name")
	AddStringFlag(get, "save-as", "", "", "file to save function code to")

	invoke := CmdBuilder(cmd, RunFunctionsInvoke, "invoke <functionName>", "Invokes a function",
		`Use `+"`"+`doctl serverless functions invoke`+"`"+` to invoke a function in your functions namespace.
You can provide inputs and inspect outputs.`,
		Writer)
	AddBoolFlag(invoke, "web", "", false, "Invoke as a web function, show result as web page")
	AddStringSliceFlag(invoke, "param", "p", []string{}, "parameter values in KEY:VALUE format, list allowed")
	AddStringFlag(invoke, "param-file", "P", "", "FILE containing parameter values in JSON format")
	AddBoolFlag(invoke, "full", "f", false, "wait for full activation record")
	AddBoolFlag(invoke, "no-wait", "n", false, "fire and forget (asynchronous invoke, does not wait for the result)")

	list := CmdBuilder(cmd, RunFunctionsList, "list [<packageName>]", "Lists the functions in your functions namespace",
		`Use `+"`"+`doctl serverless functions list`+"`"+` to list the functions in your functions namespace.`,
		Writer, aliasOpt("ls"), displayerType(&displayers.Functions{}))
	AddStringFlag(list, "limit", "l", "", "only return LIMIT number of functions (default 30, max 200)")
	AddStringFlag(list, "skip", "s", "", "exclude the first SKIP number of functions from the result")
	AddBoolFlag(list, "count", "", false, "show only the total number of functions")
	AddBoolFlag(list, "name-sort", "", false, "sort results by name")
	AddBoolFlag(list, "name", "n", false, "sort results by name")

	return cmd
}

// RunFunctionsGet supports the 'serverless functions get' command
func RunFunctionsGet(c *CmdConfig) error {
	err := ensureOneArg(c)
	if err != nil {
		return err
	}
	urlFlag, _ := c.Doit.GetBool(c.NS, flagURL)
	codeFlag, _ := c.Doit.GetBool(c.NS, flagCode)
	saveFlag, _ := c.Doit.GetBool(c.NS, flagSave)
	saveAsFlag, _ := c.Doit.GetString(c.NS, flagSaveAs)
	saveEnvFlag, _ := c.Doit.GetString(c.NS, flagSaveEnv)
	saveEnvJSONFlag, _ := c.Doit.GetString(c.NS, flagSaveEnvJSON)
	fetchCode := codeFlag || saveFlag || saveAsFlag != ""

	sls := c.Serverless()
	action, parms, err := sls.GetFunction(c.Args[0], fetchCode)
	if err != nil {
		return err
	}

	if urlFlag {
		host, err := sls.GetConnectedAPIHost()
		if err != nil {
			return err
		}
		_, err = fmt.Fprintln(c.Out, computeURL(action, host))
		return err
	}

	if saveFlag || saveAsFlag != "" {
		return doSaveFunctionCode(action, saveFlag, saveAsFlag)
	}

	if saveEnvFlag != "" || saveEnvJSONFlag != "" {
		return doSaveFunctionEnvironment(saveEnvFlag, saveEnvJSONFlag, parms)
	}

	if codeFlag {
		if !*action.Exec.Binary {
			_, err = fmt.Fprintln(c.Out, *action.Exec.Code)
			return err
		}
		return errors.New("Binary code cannot be displayed on the console")
	}

	output := do.ServerlessOutput{Entity: action}
	return c.PrintServerlessTextOutput(output)
}

// doSaveFunctionCode performs the save operations for code
func doSaveFunctionCode(action whisk.Action, save bool, saveAs string) error {
	var extension string // used only when save and !saveAs
	var data []byte
	if *action.Exec.Binary {
		extension = ".zip"
		decoded, err := base64.StdEncoding.DecodeString(*action.Exec.Code)
		if err != nil {
			return err
		}
		data = decoded
	} else {
		extension = fileExtensionForKind(action.Exec.Kind) // find equivalent
		data = []byte(*action.Exec.Code)
	}
	if save && saveAs == "" {
		saveAs = action.Name + extension
	}
	if saveAs != "" {
		err := os.WriteFile(saveAs, data, 0666)
		if err != nil {
			return err
		}
	}

	return nil
}

// doSaveFunctionEnvironment saves the environment variables for a function to file,
// either as key-value pairs or JSON.  Could do both if both are specified.
func doSaveFunctionEnvironment(saveEnv string, saveEnvJSON string, parms []do.FunctionParameter) error {
	keyVals := []string{}
	envMap := map[string]string{}
	for _, parm := range parms {
		if parm.Init {
			keyVal := parm.Key + "=" + parm.Value
			keyVals = append(keyVals, keyVal)
			envMap[parm.Key] = parm.Value
		}
	}

	if saveEnv != "" {
		data := []byte(strings.Join(keyVals, "\n"))
		err := os.WriteFile(saveEnv, data, 0666)
		if err != nil {
			return err
		}
	}

	if saveEnvJSON != "" {
		data, err := json.MarshalIndent(&envMap, "", "  ")
		if err != nil {
			return err
		}
		err = os.WriteFile(saveEnvJSON, data, 0666)
		if err != nil {
			return err
		}
	}

	return nil
}

// fileExtensionforKind finds the right file extension for a given runtime 'kind'.
// This code will require modification when the repertoire of runtimes is extended.
func fileExtensionForKind(kind string) string {
	lang := strings.Split(kind, ":")[0]
	switch strings.ToLower(lang) {
	case "go":
		return ".go"
	case "nodejs":
		return ".js"
	case "php":
		return ".php"
	case "python":
		return ".py"
	}
	return ""
}

// computeURL determines the URL string based on the action get output.
// Based on code in aio-cli-plugin-runtime, src/commands/runtime/action/get.js
func computeURL(action whisk.Action, host string) string {
	nameParts := strings.Split(action.Namespace, "/")
	namespace := nameParts[0]
	var packageName string
	if len(nameParts) > 1 {
		packageName = nameParts[1]
	}
	if action.WebAction() {
		if packageName == "" {
			packageName = "default"
		}
		return fmt.Sprintf("%s/api/v1/web/%s/%s/%s", host, namespace, packageName, action.Name)
	}
	if packageName != "" {
		packageName += "/"
	}
	return fmt.Sprintf("%s/api/v1/namespaces/%s/actions/%s%s", host, namespace, packageName, action.Name)
}

// RunFunctionsInvoke supports the 'serverless functions invoke' command
func RunFunctionsInvoke(c *CmdConfig) error {
	err := ensureOneArg(c)
	if err != nil {
		return err
	}
	paramFile, _ := c.Doit.GetString(c.NS, flagParamFile)
	paramFlags, _ := c.Doit.GetStringSlice(c.NS, flagParam)
	params, err := consolidateParams(paramFile, paramFlags)
	if err != nil {
		return err
	}
	web, _ := c.Doit.GetBool(c.NS, flagWeb)
	if web {
		var mapParams map[string]interface{} = nil
		if params != nil {
			p, ok := params.(map[string]interface{})
			if !ok {
				return fmt.Errorf("cannot invoke via web: parameters do not form a dictionary")
			}
			mapParams = p
		}
		return c.Serverless().InvokeFunctionViaWeb(c.Args[0], mapParams)
	}
	full, _ := c.Doit.GetBool(c.NS, flagFull)
	noWait, _ := c.Doit.GetBool(c.NS, flagNoWait)
	blocking := !noWait
	result := blocking && !full
	response, err := c.Serverless().InvokeFunction(c.Args[0], params, blocking, result)

	if err != nil {
		if response != nil {
			activationResponse := response.(map[string]interface{})
			template.Print(`Request accepted, but processing not completed yet. {{nl}}All functions invocation >= 30s will get demoted to an asynchronous invocation. Use {{highlight "--no-wait"}} flag to immediately return the activation id. {{nl}}
Use this command to view the results.
{{bold "doctl sls activations result" }} {{bold .}} {{nl 2}}`, activationResponse["activationId"])
			return nil
		}
		return err
	}

	output := do.ServerlessOutput{Entity: response}
	return c.PrintServerlessTextOutput(output)
}

// RunFunctionsList supports the 'serverless functions list' command
func RunFunctionsList(c *CmdConfig) error {
	argCount := len(c.Args)
	if argCount > 1 {
		return doctl.NewTooManyArgsErr(c.NS)
	}
	var pkg string
	if argCount == 1 {
		pkg = c.Args[0]
	}
	// Determine if '--count' is requested since we will use simple text output in that case.
	// Count is mutually exclusive with the global format flag.
	count, _ := c.Doit.GetBool(c.NS, flagCount)
	if count && c.Doit.IsSet("format") {
		return errors.New("the --count and --format flags are mutually exclusive")
	}
	// Retrieve other flags
	skip, _ := c.Doit.GetInt(c.NS, flagSkip)
	limit, _ := c.Doit.GetInt(c.NS, flagLimit)
	nameSort, _ := c.Doit.GetBool(c.NS, flagNameSort)
	nameName, _ := c.Doit.GetBool(c.NS, flagNameName)
	// Get information from backend
	list, err := c.Serverless().ListFunctions(pkg, skip, limit)
	if err != nil {
		return err
	}
	if count {
		plural := "s"
		are := "are"
		if len(list) == 1 {
			plural = ""
			are = "is"
		}
		fmt.Fprintf(c.Out, "There %s %d function%s in this namespace.\n", are, len(list), plural)
		return nil
	}
	if nameSort || nameName {
		sortFunctionList(list)
	}
	return c.Display(&displayers.Functions{Info: list})
}

// sortFunctionList performs a sort of a function list (by name)
func sortFunctionList(list []whisk.Action) {
	isLess := func(i, j int) bool {
		return list[i].Name < list[j].Name
	}
	sort.Slice(list, isLess)
}

// consolidateParams accepts parameters from a file, the command line, or both, and consolidates all
// such parameters into a simple dictionary.
func consolidateParams(paramFile string, params []string) (interface{}, error) {
	consolidated := map[string]interface{}{}
	if len(paramFile) > 0 {
		contents, err := os.ReadFile(paramFile)
		if err != nil {
			return nil, err
		}
		err = json.Unmarshal(contents, &consolidated)
		if err != nil {
			return nil, err
		}
	}
	for _, param := range params {
		parts := strings.Split(param, ":")
		if len(parts) < 2 {
			return nil, fmt.Errorf("values for --params must have KEY:VALUE form")
		}
		parts1 := strings.Join(parts[1:], ":")
		consolidated[parts[0]] = parts1
	}
	if len(consolidated) > 0 {
		return consolidated, nil
	}
	return nil, nil
}
<|endoftext|>


<commit_before>package crypto

import (
	"crypto/cipher"
	"io"

	"v2ray.com/core/common"
	"v2ray.com/core/common/buf"
	"v2ray.com/core/common/protocol"
)

type BytesGenerator interface {
	Next() []byte
}

type NoOpBytesGenerator struct {
	buffer [1]byte
}

func (v NoOpBytesGenerator) Next() []byte {
	return v.buffer[:0]
}

type StaticBytesGenerator struct {
	Content []byte
}

func (v StaticBytesGenerator) Next() []byte {
	return v.Content
}

type Authenticator interface {
	NonceSize() int
	Overhead() int
	Open(dst, cipherText []byte) ([]byte, error)
	Seal(dst, plainText []byte) ([]byte, error)
}

type AEADAuthenticator struct {
	cipher.AEAD
	NonceGenerator          BytesGenerator
	AdditionalDataGenerator BytesGenerator
}

func (v *AEADAuthenticator) Open(dst, cipherText []byte) ([]byte, error) {
	iv := v.NonceGenerator.Next()
	if len(iv) != v.AEAD.NonceSize() {
		return nil, newError("invalid AEAD nonce size: ", len(iv))
	}

	additionalData := v.AdditionalDataGenerator.Next()
	return v.AEAD.Open(dst, iv, cipherText, additionalData)
}

func (v *AEADAuthenticator) Seal(dst, plainText []byte) ([]byte, error) {
	iv := v.NonceGenerator.Next()
	if len(iv) != v.AEAD.NonceSize() {
		return nil, newError("invalid AEAD nonce size: ", len(iv))
	}

	additionalData := v.AdditionalDataGenerator.Next()
	return v.AEAD.Seal(dst, iv, plainText, additionalData), nil
}

type AuthenticationReader struct {
	auth         Authenticator
	buffer       *buf.Buffer
	reader       io.Reader
	sizeParser   ChunkSizeDecoder
	size         int
	transferType protocol.TransferType
}

const (
	readerBufferSize = 32 * 1024
)

func NewAuthenticationReader(auth Authenticator, sizeParser ChunkSizeDecoder, reader io.Reader, transferType protocol.TransferType) *AuthenticationReader {
	return &AuthenticationReader{
		auth:         auth,
		buffer:       buf.NewLocal(readerBufferSize),
		reader:       reader,
		sizeParser:   sizeParser,
		size:         -1,
		transferType: transferType,
	}
}

func (r *AuthenticationReader) readSize() error {
	if r.size >= 0 {
		return nil
	}

	sizeBytes := r.sizeParser.SizeBytes()
	if r.buffer.Len() < sizeBytes {
		if r.buffer.IsEmpty() {
			r.buffer.Clear()
		} else {
			common.Must(r.buffer.Reset(buf.ReadFrom(r.buffer)))
		}

		delta := sizeBytes - r.buffer.Len()
		if err := r.buffer.AppendSupplier(buf.ReadAtLeastFrom(r.reader, delta)); err != nil {
			return err
		}
	}
	size, err := r.sizeParser.Decode(r.buffer.BytesTo(sizeBytes))
	if err != nil {
		return err
	}
	r.size = int(size)
	r.buffer.SliceFrom(sizeBytes)
	return nil
}

func (r *AuthenticationReader) readChunk(waitForData bool) ([]byte, error) {
	if err := r.readSize(); err != nil {
		return nil, err
	}
	if r.size > readerBufferSize-r.sizeParser.SizeBytes() {
		return nil, newError("size too large ", r.size).AtWarning()
	}

	if r.size == r.auth.Overhead() {
		return nil, io.EOF
	}

	if r.buffer.Len() < r.size {
		if !waitForData {
			return nil, io.ErrNoProgress
		}

		if r.buffer.IsEmpty() {
			r.buffer.Clear()
		} else {
			common.Must(r.buffer.Reset(buf.ReadFrom(r.buffer)))
		}

		delta := r.size - r.buffer.Len()
		if err := r.buffer.AppendSupplier(buf.ReadAtLeastFrom(r.reader, delta)); err != nil {
			return nil, err
		}
	}

	b, err := r.auth.Open(r.buffer.BytesTo(0), r.buffer.BytesTo(r.size))
	if err != nil {
		return nil, err
	}
	r.buffer.SliceFrom(r.size)
	r.size = -1
	return b, nil
}

func (r *AuthenticationReader) ReadMultiBuffer() (buf.MultiBuffer, error) {
	b, err := r.readChunk(true)
	if err != nil {
		return nil, err
	}

	var mb buf.MultiBuffer
	if r.transferType == protocol.TransferTypeStream {
		mb.Write(b)
	} else {
		var bb *buf.Buffer
		if len(b) < buf.Size {
			bb = buf.New()
		} else {
			bb = buf.NewLocal(len(b))
		}
		bb.Append(b)
		mb.Append(bb)
	}

	for r.buffer.Len() >= r.sizeParser.SizeBytes() {
		b, err := r.readChunk(false)
		if err != nil {
			break
		}
		if r.transferType == protocol.TransferTypeStream {
			mb.Write(b)
		} else {
			var bb *buf.Buffer
			if len(b) < buf.Size {
				bb = buf.New()
			} else {
				bb = buf.NewLocal(len(b))
			}
			bb.Append(b)
			mb.Append(bb)
		}
	}

	return mb, nil
}

const (
	WriteSize = 1024
)

type AuthenticationWriter struct {
	auth         Authenticator
	writer       buf.Writer
	sizeParser   ChunkSizeEncoder
	transferType protocol.TransferType
}

func NewAuthenticationWriter(auth Authenticator, sizeParser ChunkSizeEncoder, writer io.Writer, transferType protocol.TransferType) *AuthenticationWriter {
	return &AuthenticationWriter{
		auth:         auth,
		writer:       buf.NewWriter(writer),
		sizeParser:   sizeParser,
		transferType: transferType,
	}
}

func (w *AuthenticationWriter) seal(b *buf.Buffer) (*buf.Buffer, error) {
	encryptedSize := b.Len() + w.auth.Overhead()

	eb := buf.New()
	common.Must(eb.Reset(func(bb []byte) (int, error) {
		w.sizeParser.Encode(uint16(encryptedSize), bb[:0])
		return w.sizeParser.SizeBytes(), nil
	}))
	if err := eb.AppendSupplier(func(bb []byte) (int, error) {
		_, err := w.auth.Seal(bb[:0], b.Bytes())
		return encryptedSize, err
	}); err != nil {
		eb.Release()
		return nil, err
	}

	return eb, nil
}

func (w *AuthenticationWriter) writeStream(mb buf.MultiBuffer) error {
	defer mb.Release()

	mb2Write := buf.NewMultiBufferCap(len(mb) + 10)

	for {
		b := buf.New()
		common.Must(b.Reset(func(bb []byte) (int, error) {
			return mb.Read(bb[:WriteSize])
		}))
		eb, err := w.seal(b)
		b.Release()

		if err != nil {
			mb2Write.Release()
			return err
		}
		mb2Write.Append(eb)
		if mb.IsEmpty() {
			break
		}
	}

	return w.writer.WriteMultiBuffer(mb2Write)
}

func (w *AuthenticationWriter) writePacket(mb buf.MultiBuffer) error {
	defer mb.Release()

	mb2Write := buf.NewMultiBufferCap(len(mb) * 2)

	for {
		b := mb.SplitFirst()
		if b == nil {
			b = buf.New()
		}
		eb, err := w.seal(b)
		b.Release()
		if err != nil {
			mb2Write.Release()
			return err
		}
		mb2Write.Append(eb)
		if mb.IsEmpty() {
			break
		}
	}

	return w.writer.WriteMultiBuffer(mb2Write)
}

func (w *AuthenticationWriter) WriteMultiBuffer(mb buf.MultiBuffer) error {
	if w.transferType == protocol.TransferTypeStream {
		return w.writeStream(mb)
	}

	return w.writePacket(mb)
}
<commit_msg>fix length check in auth reader<commit_after>package crypto

import (
	"crypto/cipher"
	"io"

	"v2ray.com/core/common"
	"v2ray.com/core/common/buf"
	"v2ray.com/core/common/protocol"
)

type BytesGenerator interface {
	Next() []byte
}

type NoOpBytesGenerator struct {
	buffer [1]byte
}

func (v NoOpBytesGenerator) Next() []byte {
	return v.buffer[:0]
}

type StaticBytesGenerator struct {
	Content []byte
}

func (v StaticBytesGenerator) Next() []byte {
	return v.Content
}

type Authenticator interface {
	NonceSize() int
	Overhead() int
	Open(dst, cipherText []byte) ([]byte, error)
	Seal(dst, plainText []byte) ([]byte, error)
}

type AEADAuthenticator struct {
	cipher.AEAD
	NonceGenerator          BytesGenerator
	AdditionalDataGenerator BytesGenerator
}

func (v *AEADAuthenticator) Open(dst, cipherText []byte) ([]byte, error) {
	iv := v.NonceGenerator.Next()
	if len(iv) != v.AEAD.NonceSize() {
		return nil, newError("invalid AEAD nonce size: ", len(iv))
	}

	additionalData := v.AdditionalDataGenerator.Next()
	return v.AEAD.Open(dst, iv, cipherText, additionalData)
}

func (v *AEADAuthenticator) Seal(dst, plainText []byte) ([]byte, error) {
	iv := v.NonceGenerator.Next()
	if len(iv) != v.AEAD.NonceSize() {
		return nil, newError("invalid AEAD nonce size: ", len(iv))
	}

	additionalData := v.AdditionalDataGenerator.Next()
	return v.AEAD.Seal(dst, iv, plainText, additionalData), nil
}

type AuthenticationReader struct {
	auth         Authenticator
	buffer       *buf.Buffer
	reader       io.Reader
	sizeParser   ChunkSizeDecoder
	size         int
	transferType protocol.TransferType
}

const (
	readerBufferSize = 32 * 1024
)

func NewAuthenticationReader(auth Authenticator, sizeParser ChunkSizeDecoder, reader io.Reader, transferType protocol.TransferType) *AuthenticationReader {
	return &AuthenticationReader{
		auth:         auth,
		buffer:       buf.NewLocal(readerBufferSize),
		reader:       reader,
		sizeParser:   sizeParser,
		size:         -1,
		transferType: transferType,
	}
}

func (r *AuthenticationReader) readSize() error {
	if r.size >= 0 {
		return nil
	}

	sizeBytes := r.sizeParser.SizeBytes()
	if r.buffer.Len() < sizeBytes {
		if r.buffer.IsEmpty() {
			r.buffer.Clear()
		} else {
			common.Must(r.buffer.Reset(buf.ReadFrom(r.buffer)))
		}

		delta := sizeBytes - r.buffer.Len()
		if err := r.buffer.AppendSupplier(buf.ReadAtLeastFrom(r.reader, delta)); err != nil {
			return err
		}
	}
	size, err := r.sizeParser.Decode(r.buffer.BytesTo(sizeBytes))
	if err != nil {
		return err
	}
	r.size = int(size)
	r.buffer.SliceFrom(sizeBytes)
	return nil
}

func (r *AuthenticationReader) readChunk(waitForData bool) ([]byte, error) {
	if err := r.readSize(); err != nil {
		return nil, err
	}
	if r.size > readerBufferSize-r.sizeParser.SizeBytes() {
		return nil, newError("size too large ", r.size).AtWarning()
	}

	if r.size == r.auth.Overhead() {
		return nil, io.EOF
	}

	if r.buffer.Len() < r.size {
		if !waitForData {
			return nil, io.ErrNoProgress
		}

		if r.buffer.IsEmpty() {
			r.buffer.Clear()
		} else {
			common.Must(r.buffer.Reset(buf.ReadFrom(r.buffer)))
		}

		delta := r.size - r.buffer.Len()
		if err := r.buffer.AppendSupplier(buf.ReadAtLeastFrom(r.reader, delta)); err != nil {
			return nil, err
		}
	}

	b, err := r.auth.Open(r.buffer.BytesTo(0), r.buffer.BytesTo(r.size))
	if err != nil {
		return nil, err
	}
	r.buffer.SliceFrom(r.size)
	r.size = -1
	return b, nil
}

func (r *AuthenticationReader) ReadMultiBuffer() (buf.MultiBuffer, error) {
	b, err := r.readChunk(true)
	if err != nil {
		return nil, err
	}

	var mb buf.MultiBuffer
	if r.transferType == protocol.TransferTypeStream {
		mb.Write(b)
	} else {
		var bb *buf.Buffer
		if len(b) <= buf.Size {
			bb = buf.New()
		} else {
			bb = buf.NewLocal(len(b))
		}
		bb.Append(b)
		mb.Append(bb)
	}

	for r.buffer.Len() >= r.sizeParser.SizeBytes() {
		b, err := r.readChunk(false)
		if err != nil {
			break
		}
		if r.transferType == protocol.TransferTypeStream {
			mb.Write(b)
		} else {
			var bb *buf.Buffer
			if len(b) <= buf.Size {
				bb = buf.New()
			} else {
				bb = buf.NewLocal(len(b))
			}
			bb.Append(b)
			mb.Append(bb)
		}
	}

	return mb, nil
}

const (
	WriteSize = 1024
)

type AuthenticationWriter struct {
	auth         Authenticator
	writer       buf.Writer
	sizeParser   ChunkSizeEncoder
	transferType protocol.TransferType
}

func NewAuthenticationWriter(auth Authenticator, sizeParser ChunkSizeEncoder, writer io.Writer, transferType protocol.TransferType) *AuthenticationWriter {
	return &AuthenticationWriter{
		auth:         auth,
		writer:       buf.NewWriter(writer),
		sizeParser:   sizeParser,
		transferType: transferType,
	}
}

func (w *AuthenticationWriter) seal(b *buf.Buffer) (*buf.Buffer, error) {
	encryptedSize := b.Len() + w.auth.Overhead()

	eb := buf.New()
	common.Must(eb.Reset(func(bb []byte) (int, error) {
		w.sizeParser.Encode(uint16(encryptedSize), bb[:0])
		return w.sizeParser.SizeBytes(), nil
	}))
	if err := eb.AppendSupplier(func(bb []byte) (int, error) {
		_, err := w.auth.Seal(bb[:0], b.Bytes())
		return encryptedSize, err
	}); err != nil {
		eb.Release()
		return nil, err
	}

	return eb, nil
}

func (w *AuthenticationWriter) writeStream(mb buf.MultiBuffer) error {
	defer mb.Release()

	mb2Write := buf.NewMultiBufferCap(len(mb) + 10)

	for {
		b := buf.New()
		common.Must(b.Reset(func(bb []byte) (int, error) {
			return mb.Read(bb[:WriteSize])
		}))
		eb, err := w.seal(b)
		b.Release()

		if err != nil {
			mb2Write.Release()
			return err
		}
		mb2Write.Append(eb)
		if mb.IsEmpty() {
			break
		}
	}

	return w.writer.WriteMultiBuffer(mb2Write)
}

func (w *AuthenticationWriter) writePacket(mb buf.MultiBuffer) error {
	defer mb.Release()

	mb2Write := buf.NewMultiBufferCap(len(mb) * 2)

	for {
		b := mb.SplitFirst()
		if b == nil {
			b = buf.New()
		}
		eb, err := w.seal(b)
		b.Release()
		if err != nil {
			mb2Write.Release()
			return err
		}
		mb2Write.Append(eb)
		if mb.IsEmpty() {
			break
		}
	}

	return w.writer.WriteMultiBuffer(mb2Write)
}

func (w *AuthenticationWriter) WriteMultiBuffer(mb buf.MultiBuffer) error {
	if w.transferType == protocol.TransferTypeStream {
		return w.writeStream(mb)
	}

	return w.writePacket(mb)
}
<|endoftext|>


<commit_before>package gonavitia

import (
	"fmt"
	"github.com/aabizri/gonavitia/types"
	"net/url"
)

// PlacesResults doesn't have pagination
type PlacesResults struct {
	Places []types.Place

	Logging
	session *Session
}

func (res PlacesResults) String() string {
	var msg string
	for i, place := range res.Places {
		msg += fmt.Sprintf("Place #%d (%s): %v\n", i, place.PlaceType(), place) //TODO: When types implements a String() method on its Place type, use it
	}
	return msg
}

type PlacesRequest struct {
	Query string // The search item

	// Types are the type of objects to query
	// It can either be a stop_area, an address, a poi or an administrative_region
	Types []string

	AdminURI []string // If given it will filter the search by specific admin uris

	DisableGeoJson bool

	Around types.Coordinates // If given, it will prioritize objects around these coordinates
}

// toURL formats a Places request to url
func (req PlacesRequest) toURL() (url.Values, error) {
	params := url.Values{
		"q": []string{req.Query},
	}

	if len(req.Types) != 0 {
		params["type[]"] = req.Types
	}

	if len(req.AdminURI) != 0 {
		params["admin_uri[]"] = req.AdminURI
	}

	if req.DisableGeoJson {
		params["disable_geojson"] = []string{"true"}
	}

	return params, nil
}

// places is the internal function used by Places functions
func (s *Session) places(url string, params PlacesRequest) (*PlacesResults, error) {
	var results = &PlacesResults{session: s}
	err := s.request(url, params, results)
	return results, err
}

const placesEndpoint = "places"

// Places search in all geographical objects using their names, returning a list of corresponding places.
func (s *Session) Places(params PlacesRequest) (*PlacesResults, error) {
	// Create the URL
	url := s.APIURL + "/" + placesEndpoint

	// Call
	return s.places(url, params)
}

// Places search in all geographical objects within a given region using their names, returning a list of places.
func (s *Session) PlacesR(params PlacesRequest, regionID string) (*PlacesResults, error) {
	// Create the URL
	url := s.APIURL + "/coverage/" + regionID + "/" + placesEndpoint

	// Call
	return s.places(url, params)
}
<commit_msg>Add Count to PlacesRequest<commit_after>package gonavitia

import (
	"fmt"
	"github.com/aabizri/gonavitia/types"
	"net/url"
	"strconv"
)

// PlacesResults doesn't have pagination
type PlacesResults struct {
	Places []types.Place

	Logging
	session *Session
}

func (res PlacesResults) String() string {
	var msg string
	for i, place := range res.Places {
		msg += fmt.Sprintf("Place #%d (%s): %s\n", i, place.PlaceType(), place.String())
	}
	return msg
}

type PlacesRequest struct {
	Query string // The search item

	// Types are the type of objects to query
	// It can either be a stop_area, an address, a poi or an administrative_region
	Types []string

	AdminURI []string // If given it will filter the search by specific admin uris

	DisableGeoJson bool

	Around types.Coordinates // If given, it will prioritize objects around these coordinates

	// Maximum amount of results
	Count uint
}

// toURL formats a Places request to url
func (req PlacesRequest) toURL() (url.Values, error) {
	params := url.Values{
		"q": []string{req.Query},
	}

	if len(req.Types) != 0 {
		params["type[]"] = req.Types
	}

	if len(req.AdminURI) != 0 {
		params["admin_uri[]"] = req.AdminURI
	}

	if req.DisableGeoJson {
		params["disable_geojson"] = []string{"true"}
	}

	if req.Count != 0 {
		countStr := strconv.FormatUint(uint64(req.Count), 10)
		params["count"] = []string{countStr}
	}
	return params, nil
}

// places is the internal function used by Places functions
func (s *Session) places(url string, params PlacesRequest) (*PlacesResults, error) {
	var results = &PlacesResults{session: s}
	err := s.request(url, params, results)
	return results, err
}

const placesEndpoint = "places"

// Places search in all geographical objects using their names, returning a list of corresponding places.
func (s *Session) Places(params PlacesRequest) (*PlacesResults, error) {
	// Create the URL
	url := s.APIURL + "/" + placesEndpoint

	// Call
	return s.places(url, params)
}

// Places search in all geographical objects within a given region using their names, returning a list of places.
func (s *Session) PlacesR(params PlacesRequest, regionID string) (*PlacesResults, error) {
	// Create the URL
	url := s.APIURL + "/coverage/" + regionID + "/" + placesEndpoint

	// Call
	return s.places(url, params)
}
<|endoftext|>


<commit_before>// Copyright 2014 Jesper Brodersen. All rights reserved.
// This code is BSD-licensed, see LICENSE file.

// pm-get is a package manger written in Go Language
// this is the user application for running daily tasks on packages,
// like browsing, installing, uninstalling and updating
package main

import (
	"github.com/broeman/gopack/cmd" // using CLI command args
	"github.com/codegangsta/cli"
	"os"
	"runtime"
)

const APP_VER = "0.1 Alpha"

func init() {
	runtime.GOMAXPROCS(runtime.NumCPU())
}

func main() {
	app := cli.NewApp()
	app.Name = "pm-get"
	app.Usage = "Package Manager in Go"
	app.Version = APP_VER
	app.Commands = []cli.Command{
		cmd.Install,   // install a package
		cmd.UnInstall, // uninstall a package
		cmd.Show,      // show package
		cmd.Installed, // shows current installed packages, placeholder
		//cmd.Update,	// update packages
		cmd.Init, // placeholder initialization
	}
	app.Run(os.Args)

}
<commit_msg>linked to local cmd, as library doesn't provide any longer<commit_after>// Copyright 2014 Jesper Brodersen. All rights reserved.
// This code is BSD-licensed, see LICENSE file.

// pm-get is a package manger written in Go Language
// this is the user application for running daily tasks on packages,
// like browsing, installing, uninstalling and updating
package main

import (
	"github.com/broeman/pm-get/cmd" // using CLI command args
	"github.com/codegangsta/cli"
	"os"
	"runtime"
)

const APP_VER = "0.1 Alpha"

func init() {
	runtime.GOMAXPROCS(runtime.NumCPU())
}

func main() {
	app := cli.NewApp()
	app.Name = "pm-get"
	app.Usage = "Package Manager in Go"
	app.Version = APP_VER
	app.Commands = []cli.Command{
		cmd.Install,   // install a package
		cmd.UnInstall, // uninstall a package
		cmd.Show,      // show package
		cmd.Installed, // shows current installed packages, placeholder
		//cmd.Update,	// update packages
		cmd.Init, // placeholder initialization
	}
	app.Run(os.Args)

}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)

package syncutil

import (
	"flag"
	"sync"
)

var fCheckInvariants = flag.Bool("syncutil.check_invariants", false, "Crash when registered invariants are violated.")

// A reader/writer mutex like sync.RWMutex that additionally runs a check for
// registered invariants at times when invariants should hold, when enabled.
// This can aid debugging subtle code by crashing early as soon as something
// unexpected happens.
//
// Must be created with NewInvariantMutex. See that function for more details.
//
// A typical use looks like this:
//
//     type myStruct struct {
//       mu InvariantMutex
//
//       // INVARIANT: nextGeneration == currentGeneration + 1
//       currentGeneration int  // GUARDED_BY(mu)
//       nextGeneration    int  // GUARDED_BY(mu)
//     }
//
//     // The constructor function for myStruct sets up the mutex to
//     // call the checkInvariants method.
//     func newMyStruct() *myStruct {
//       s := &myStruct{
//         currentGeneration: 1,
//         nextGeneration:    2,
//       }
//
//       s.mu = NewInvariantMutex(func() { s.checkInvariants() })
//       return s
//     }
//
//     type (s *myStruct) checkInvariants() {
//       if s.nextGeneration != s.currentGeneration + 1 {
//         panic(
//           fmt.Sprintf("%v != %v + 1", s.nextGeneration, s.currentGeneration))
//       }
//     }
//
//     // When the flag is set, invariants will be checked at entry to and exit
//     // from this function.
//     func (s *myStruct) setGeneration(n int) {
//       s.mu.Lock()
//       defer s.mu.Unlock()
//
//       currentGeneration = n
//       nextGeneration = n + 1
//     }
//
type InvariantMutex struct {
	mu    sync.RWMutex
	check func()
}

func (i *InvariantMutex) Lock() {
	i.mu.Lock()
	i.checkIfEnabled()
}

func (i *InvariantMutex) Unlock() {
	i.checkIfEnabled()
	i.mu.Unlock()
}

func (i *InvariantMutex) RLock() {
	i.mu.RLock()
	i.checkIfEnabled()
}

func (i *InvariantMutex) RUnlock() {
	i.checkIfEnabled()
	i.mu.RUnlock()
}

func (i *InvariantMutex) checkIfEnabled() {
	if *fCheckInvariants {
		i.check()
	}
}

// Create a reader/writer mutex which, when the flag -syncutil.check_invariants
// is set, will call the supplied function at moments when invariants protected
// by the mutex should hold (e.g. just after acquiring the lock). The function
// should crash if an invariant is violated. It should not have side effects,
// as there are no guarantees that it will run.
func NewInvariantMutex(check func()) InvariantMutex {
	if check == nil {
		panic("check must be non-nil.")
	}

	return InvariantMutex{
		check: check,
	}
}
<commit_msg>Fixed errors in the example.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)

package syncutil

import (
	"flag"
	"sync"
)

var fCheckInvariants = flag.Bool("syncutil.check_invariants", false, "Crash when registered invariants are violated.")

// A reader/writer mutex like sync.RWMutex that additionally runs a check for
// registered invariants at times when invariants should hold, when enabled.
// This can aid debugging subtle code by crashing early as soon as something
// unexpected happens.
//
// Must be created with NewInvariantMutex. See that function for more details.
//
// A typical use looks like this:
//
//     type myStruct struct {
//       mu syncutil.InvariantMutex
//
//       // INVARIANT: nextGeneration == currentGeneration + 1
//       currentGeneration int // GUARDED_BY(mu)
//       nextGeneration    int // GUARDED_BY(mu)
//     }
//
//     // The constructor function for myStruct sets up the mutex to
//     // call the checkInvariants method.
//     func newMyStruct() *myStruct {
//       s := &myStruct{
//         currentGeneration: 1,
//         nextGeneration:    2,
//       }
//
//       s.mu = syncutil.NewInvariantMutex(func() { s.checkInvariants() })
//       return s
//     }
//
//     func (s *myStruct) checkInvariants() {
//       if s.nextGeneration != s.currentGeneration+1 {
//         panic(
//           fmt.Sprintf("%v != %v + 1", s.nextGeneration, s.currentGeneration))
//       }
//     }
//
//     // When the flag is set, invariants will be checked at entry to and exit
//     // from this function.
//     func (s *myStruct) setGeneration(n int) {
//       s.mu.Lock()
//       defer s.mu.Unlock()
//
//       s.currentGeneration = n
//       s.nextGeneration = n + 1
//     }
//
type InvariantMutex struct {
	mu    sync.RWMutex
	check func()
}

func (i *InvariantMutex) Lock() {
	i.mu.Lock()
	i.checkIfEnabled()
}

func (i *InvariantMutex) Unlock() {
	i.checkIfEnabled()
	i.mu.Unlock()
}

func (i *InvariantMutex) RLock() {
	i.mu.RLock()
	i.checkIfEnabled()
}

func (i *InvariantMutex) RUnlock() {
	i.checkIfEnabled()
	i.mu.RUnlock()
}

func (i *InvariantMutex) checkIfEnabled() {
	if *fCheckInvariants {
		i.check()
	}
}

// Create a reader/writer mutex which, when the flag -syncutil.check_invariants
// is set, will call the supplied function at moments when invariants protected
// by the mutex should hold (e.g. just after acquiring the lock). The function
// should crash if an invariant is violated. It should not have side effects,
// as there are no guarantees that it will run.
func NewInvariantMutex(check func()) InvariantMutex {
	if check == nil {
		panic("check must be non-nil.")
	}

	return InvariantMutex{
		check: check,
	}
}
<|endoftext|>


<commit_before>package s3gof3r

import (
	"bytes"
	"crypto/md5"
	"encoding/base64"
	"encoding/xml"
	"fmt"
	"hash"
	"io"
	"math"
	"net/http"
	"net/url"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"time"
)

// defined by amazon
const (
	minPartSize = 5 * mb
	maxPartSize = 5 * gb
	maxObjSize  = 5 * tb
	maxNPart    = 10000
	md5Header   = "content-md5"
)

type part struct {
	r   io.ReadSeeker
	len int64
	b   []byte

	// read by xml encoder
	PartNumber int
	ETag       string

	// Used for checksum of checksums on completion
	contentMd5 string
}

type putter struct {
	url url.URL
	b   *Bucket
	c   *Config

	bufsz      int64
	buf        []byte
	bufbytes   int // bytes written to current buffer
	ch         chan *part
	part       int
	closed     bool
	err        error
	wg         sync.WaitGroup
	md5OfParts hash.Hash
	md5        hash.Hash
	ETag       string

	sp *bp

	makes    int
	UploadId string // casing matches s3 xml
	xml      struct {
		XMLName string `xml:"CompleteMultipartUpload"`
		Part    []*part
	}
	putsz int64
}

// Sends an S3 multipart upload initiation request.
// See http://docs.amazonwebservices.com/AmazonS3/latest/dev/mpuoverview.html.
// The initial request returns an UploadId that we use to identify
// subsequent PUT requests.
func newPutter(url url.URL, h http.Header, c *Config, b *Bucket) (p *putter, err error) {
	p = new(putter)
	p.url = url
	p.b = b
	p.c = c
	p.c.Concurrency = max(c.Concurrency, 1)
	p.c.NTry = max(c.NTry, 1)
	p.bufsz = max64(minPartSize, c.PartSize)
	resp, err := p.retryRequest("POST", url.String()+"?uploads", nil, h)
	if err != nil {
		return nil, err
	}
	defer checkClose(resp.Body, &err)
	if resp.StatusCode != 200 {
		return nil, newRespError(resp)
	}
	err = xml.NewDecoder(resp.Body).Decode(p)
	if err != nil {
		return nil, err
	}
	p.ch = make(chan *part)
	for i := 0; i < p.c.Concurrency; i++ {
		go p.worker()
	}
	p.md5OfParts = md5.New()
	p.md5 = md5.New()

	p.sp = bufferPool(p.bufsz)

	return p, nil
}

func (p *putter) Write(b []byte) (int, error) {
	if p.closed {
		p.abort()
		return 0, syscall.EINVAL
	}
	if p.err != nil {
		p.abort()
		return 0, p.err
	}
	nw := 0
	for nw < len(b) {
		if p.buf == nil {
			p.buf = <-p.sp.get
			if int64(cap(p.buf)) < p.bufsz {
				p.buf = make([]byte, p.bufsz)
				runtime.GC()
			}
		}
		n := copy(p.buf[p.bufbytes:], b[nw:])
		p.bufbytes += n
		nw += n

		if len(p.buf) == p.bufbytes {
			p.flush()
		}
	}
	return nw, nil
}

func (p *putter) flush() {
	p.wg.Add(1)
	p.part++
	p.putsz += int64(p.bufbytes)
	part := &part{bytes.NewReader(p.buf[:p.bufbytes]), int64(p.bufbytes), p.buf, p.part, "", ""}
	var err error
	part.contentMd5, part.ETag, err = p.md5Content(part.r)
	if err != nil {
		p.err = err
	}

	p.xml.Part = append(p.xml.Part, part)
	p.ch <- part
	p.buf, p.bufbytes = nil, 0

	// if necessary, double buffer size every 2000 parts due to the 10000-part AWS limit
	// to reach the 5 Terabyte max object size, initial part size must be ~85 MB
	if p.part%2000 == 0 && p.part < maxNPart && growPartSize(p.part, p.bufsz, p.putsz) {
		p.bufsz = min64(p.bufsz*2, maxPartSize)
		p.sp.sizech <- p.bufsz // update pool buffer size
		logger.debugPrintf("part size doubled to %d", p.bufsz)
	}
}

func (p *putter) worker() {
	for part := range p.ch {
		p.retryPutPart(part)
	}
}

// Calls putPart up to nTry times to recover from transient errors.
func (p *putter) retryPutPart(part *part) {
	defer p.wg.Done()
	var err error
	for i := 0; i < p.c.NTry; i++ {
		time.Sleep(time.Duration(math.Exp2(float64(i))) * 100 * time.Millisecond) // exponential back-off
		err = p.putPart(part)
		if err == nil {
			p.sp.give <- part.b
			part.b = nil
			return
		}
		logger.debugPrintf("Error on attempt %d: Retrying part: %d, Error: %s", i, part.PartNumber, err)
	}
	p.err = err
}

// uploads a part, checking the etag against the calculated value
func (p *putter) putPart(part *part) error {
	v := url.Values{}
	v.Set("partNumber", strconv.Itoa(part.PartNumber))
	v.Set("uploadId", p.UploadId)
	if _, err := part.r.Seek(0, 0); err != nil { // move back to beginning, if retrying
		return err
	}
	req, err := http.NewRequest("PUT", p.url.String()+"?"+v.Encode(), part.r)
	if err != nil {
		return err
	}
	req.ContentLength = part.len
	req.Header.Set(md5Header, part.contentMd5)
	p.b.Sign(req)
	resp, err := p.c.Client.Do(req)
	if err != nil {
		return err
	}
	defer checkClose(resp.Body, &err)
	if resp.StatusCode != 200 {
		return newRespError(resp)
	}
	s := resp.Header.Get("etag")
	s = s[1 : len(s)-1] // includes quote chars for some reason
	if part.ETag != s {
		return fmt.Errorf("Response etag does not match. Remote:%s Calculated:%s", s, p.ETag)
	}
	return nil
}

func (p *putter) Close() (err error) {
	if p.closed {
		p.abort()
		return syscall.EINVAL
	}
	if p.err != nil {
		p.abort()
		return p.err
	}
	if p.bufbytes > 0 || // partial part
		p.part == 0 { // 0 length file
		p.flush()
	}
	p.wg.Wait()
	close(p.ch)
	p.closed = true
	close(p.sp.quit)

	// check p.err before completing
	if p.err != nil {
		p.abort()
		return p.err
	}
	// Complete Multipart upload
	body, err := xml.Marshal(p.xml)
	if err != nil {
		p.abort()
		return
	}
	b := bytes.NewReader(body)
	v := url.Values{}
	v.Set("uploadId", p.UploadId)
	resp, err := p.retryRequest("POST", p.url.String()+"?"+v.Encode(), b, nil)
	if err != nil {
		p.abort()
		return
	}
	defer checkClose(resp.Body, &err)
	if resp.StatusCode != 200 {
		p.abort()
		return newRespError(resp)
	}
	// Check md5 hash of concatenated part md5 hashes against ETag
	// more info: https://forums.aws.amazon.com/thread.jspa?messageID=456442&#456442
	calculatedMd5ofParts := fmt.Sprintf("%x", p.md5OfParts.Sum(nil))
	// Parse etag from body of response
	err = xml.NewDecoder(resp.Body).Decode(p)
	if err != nil {
		return
	}
	// strip part count from end and '"' from front.
	remoteMd5ofParts := strings.Split(p.ETag, "-")[0]
	remoteMd5ofParts = remoteMd5ofParts[1:len(remoteMd5ofParts)]
	if calculatedMd5ofParts != remoteMd5ofParts {
		if err != nil {
			return err
		}
		return fmt.Errorf("MD5 hash of part hashes comparison failed. Hash from multipart complete header: %s."+
			" Calculated multipart hash: %s.", remoteMd5ofParts, calculatedMd5ofParts)
	}
	if p.c.Md5Check {
		for i := 0; i < p.c.NTry; i++ {
			if err = p.putMd5(); err == nil {
				break
			}
		}
	}
	return
}

// Try to abort multipart upload. Do not error on failure.
func (p *putter) abort() {
	v := url.Values{}
	v.Set("uploadId", p.UploadId)
	s := p.url.String() + "?" + v.Encode()
	resp, err := p.retryRequest("DELETE", s, nil, nil)
	if err != nil {
		logger.Printf("Error aborting multipart upload: %v\n", err)
		return
	}
	defer checkClose(resp.Body, &err)
	if resp.StatusCode != 204 {
		logger.Printf("Error aborting multipart upload: %v", newRespError(resp))
	}
	return
}

// Md5 functions
func (p *putter) md5Content(r io.ReadSeeker) (string, string, error) {
	h := md5.New()
	mw := io.MultiWriter(h, p.md5)
	if _, err := io.Copy(mw, r); err != nil {
		return "", "", err
	}
	sum := h.Sum(nil)
	hexSum := fmt.Sprintf("%x", sum)
	// add to checksum of all parts for verification on upload completion
	if _, err := p.md5OfParts.Write(sum); err != nil {
		return "", "", err
	}
	return base64.StdEncoding.EncodeToString(sum), hexSum, nil
}

// Put md5 file in .md5 subdirectory of bucket  where the file is stored
// e.g. the md5 for https://mybucket.s3.amazonaws.com/gof3r will be stored in
// https://mybucket.s3.amazonaws.com/.md5/gof3r.md5
func (p *putter) putMd5() (err error) {
	calcMd5 := fmt.Sprintf("%x", p.md5.Sum(nil))
	md5Reader := strings.NewReader(calcMd5)
	md5Path := fmt.Sprint(".md5", p.url.Path, ".md5")
	md5Url, err := p.b.url(md5Path, p.c)
	if err != nil {
		return err
	}
	logger.debugPrintln("md5: ", calcMd5)
	logger.debugPrintln("md5Path: ", md5Path)
	r, err := http.NewRequest("PUT", md5Url.String(), md5Reader)
	if err != nil {
		return
	}
	p.b.Sign(r)
	resp, err := p.c.Client.Do(r)
	if err != nil {
		return
	}
	defer checkClose(resp.Body, &err)
	if resp.StatusCode != 200 {
		return newRespError(resp)
	}
	return
}

func (p *putter) retryRequest(method, urlStr string, body io.ReadSeeker, h http.Header) (resp *http.Response, err error) {
	for i := 0; i < p.c.NTry; i++ {
		var req *http.Request
		req, err = http.NewRequest(method, urlStr, body)
		if err != nil {
			return
		}
		for k := range h {
			for _, v := range h[k] {
				req.Header.Add(k, v)
			}
		}

		p.b.Sign(req)
		resp, err = p.c.Client.Do(req)
		if err == nil {
			return
		}
		logger.debugPrintln(err)
		if body != nil {
			if _, err = body.Seek(0, 0); err != nil {
				return
			}
		}
	}
	return
}

// returns true unless partSize is large enough
// to achieve maxObjSize with remaining parts
func growPartSize(partIndex int, partSize, putsz int64) bool {
	return (maxObjSize-putsz)/(maxNPart-int64(partIndex)) > partSize
}
<commit_msg>Handle cases where etag is missing gracefully.<commit_after>package s3gof3r

import (
	"bytes"
	"crypto/md5"
	"encoding/base64"
	"encoding/xml"
	"fmt"
	"hash"
	"io"
	"math"
	"net/http"
	"net/url"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"time"
)

// defined by amazon
const (
	minPartSize = 5 * mb
	maxPartSize = 5 * gb
	maxObjSize  = 5 * tb
	maxNPart    = 10000
	md5Header   = "content-md5"
)

type part struct {
	r   io.ReadSeeker
	len int64
	b   []byte

	// read by xml encoder
	PartNumber int
	ETag       string

	// Used for checksum of checksums on completion
	contentMd5 string
}

type putter struct {
	url url.URL
	b   *Bucket
	c   *Config

	bufsz      int64
	buf        []byte
	bufbytes   int // bytes written to current buffer
	ch         chan *part
	part       int
	closed     bool
	err        error
	wg         sync.WaitGroup
	md5OfParts hash.Hash
	md5        hash.Hash
	ETag       string

	sp *bp

	makes    int
	UploadId string // casing matches s3 xml
	xml      struct {
		XMLName string `xml:"CompleteMultipartUpload"`
		Part    []*part
	}
	putsz int64
}

// Sends an S3 multipart upload initiation request.
// See http://docs.amazonwebservices.com/AmazonS3/latest/dev/mpuoverview.html.
// The initial request returns an UploadId that we use to identify
// subsequent PUT requests.
func newPutter(url url.URL, h http.Header, c *Config, b *Bucket) (p *putter, err error) {
	p = new(putter)
	p.url = url
	p.b = b
	p.c = c
	p.c.Concurrency = max(c.Concurrency, 1)
	p.c.NTry = max(c.NTry, 1)
	p.bufsz = max64(minPartSize, c.PartSize)
	resp, err := p.retryRequest("POST", url.String()+"?uploads", nil, h)
	if err != nil {
		return nil, err
	}
	defer checkClose(resp.Body, &err)
	if resp.StatusCode != 200 {
		return nil, newRespError(resp)
	}
	err = xml.NewDecoder(resp.Body).Decode(p)
	if err != nil {
		return nil, err
	}
	p.ch = make(chan *part)
	for i := 0; i < p.c.Concurrency; i++ {
		go p.worker()
	}
	p.md5OfParts = md5.New()
	p.md5 = md5.New()

	p.sp = bufferPool(p.bufsz)

	return p, nil
}

func (p *putter) Write(b []byte) (int, error) {
	if p.closed {
		p.abort()
		return 0, syscall.EINVAL
	}
	if p.err != nil {
		p.abort()
		return 0, p.err
	}
	nw := 0
	for nw < len(b) {
		if p.buf == nil {
			p.buf = <-p.sp.get
			if int64(cap(p.buf)) < p.bufsz {
				p.buf = make([]byte, p.bufsz)
				runtime.GC()
			}
		}
		n := copy(p.buf[p.bufbytes:], b[nw:])
		p.bufbytes += n
		nw += n

		if len(p.buf) == p.bufbytes {
			p.flush()
		}
	}
	return nw, nil
}

func (p *putter) flush() {
	p.wg.Add(1)
	p.part++
	p.putsz += int64(p.bufbytes)
	part := &part{bytes.NewReader(p.buf[:p.bufbytes]), int64(p.bufbytes), p.buf, p.part, "", ""}
	var err error
	part.contentMd5, part.ETag, err = p.md5Content(part.r)
	if err != nil {
		p.err = err
	}

	p.xml.Part = append(p.xml.Part, part)
	p.ch <- part
	p.buf, p.bufbytes = nil, 0

	// if necessary, double buffer size every 2000 parts due to the 10000-part AWS limit
	// to reach the 5 Terabyte max object size, initial part size must be ~85 MB
	if p.part%2000 == 0 && p.part < maxNPart && growPartSize(p.part, p.bufsz, p.putsz) {
		p.bufsz = min64(p.bufsz*2, maxPartSize)
		p.sp.sizech <- p.bufsz // update pool buffer size
		logger.debugPrintf("part size doubled to %d", p.bufsz)
	}
}

func (p *putter) worker() {
	for part := range p.ch {
		p.retryPutPart(part)
	}
}

// Calls putPart up to nTry times to recover from transient errors.
func (p *putter) retryPutPart(part *part) {
	defer p.wg.Done()
	var err error
	for i := 0; i < p.c.NTry; i++ {
		time.Sleep(time.Duration(math.Exp2(float64(i))) * 100 * time.Millisecond) // exponential back-off
		err = p.putPart(part)
		if err == nil {
			p.sp.give <- part.b
			part.b = nil
			return
		}
		logger.debugPrintf("Error on attempt %d: Retrying part: %d, Error: %s", i, part.PartNumber, err)
	}
	p.err = err
}

// uploads a part, checking the etag against the calculated value
func (p *putter) putPart(part *part) error {
	v := url.Values{}
	v.Set("partNumber", strconv.Itoa(part.PartNumber))
	v.Set("uploadId", p.UploadId)
	if _, err := part.r.Seek(0, 0); err != nil { // move back to beginning, if retrying
		return err
	}
	req, err := http.NewRequest("PUT", p.url.String()+"?"+v.Encode(), part.r)
	if err != nil {
		return err
	}
	req.ContentLength = part.len
	req.Header.Set(md5Header, part.contentMd5)
	p.b.Sign(req)
	resp, err := p.c.Client.Do(req)
	if err != nil {
		return err
	}
	defer checkClose(resp.Body, &err)
	if resp.StatusCode != 200 {
		return newRespError(resp)
	}
	s := resp.Header.Get("etag")
	if len(s) < 2 {
		return fmt.Errorf("Got Bad etag:%s", s)
	}
	s = s[1 : len(s)-1] // includes quote chars for some reason
	if part.ETag != s {
		return fmt.Errorf("Response etag does not match. Remote:%s Calculated:%s", s, p.ETag)
	}
	return nil
}

func (p *putter) Close() (err error) {
	if p.closed {
		p.abort()
		return syscall.EINVAL
	}
	if p.err != nil {
		p.abort()
		return p.err
	}
	if p.bufbytes > 0 || // partial part
		p.part == 0 { // 0 length file
		p.flush()
	}
	p.wg.Wait()
	close(p.ch)
	p.closed = true
	close(p.sp.quit)

	// check p.err before completing
	if p.err != nil {
		p.abort()
		return p.err
	}
	// Complete Multipart upload
	body, err := xml.Marshal(p.xml)
	if err != nil {
		p.abort()
		return
	}
	b := bytes.NewReader(body)
	v := url.Values{}
	v.Set("uploadId", p.UploadId)
	resp, err := p.retryRequest("POST", p.url.String()+"?"+v.Encode(), b, nil)
	if err != nil {
		p.abort()
		return
	}
	defer checkClose(resp.Body, &err)
	if resp.StatusCode != 200 {
		p.abort()
		return newRespError(resp)
	}
	// Check md5 hash of concatenated part md5 hashes against ETag
	// more info: https://forums.aws.amazon.com/thread.jspa?messageID=456442&#456442
	calculatedMd5ofParts := fmt.Sprintf("%x", p.md5OfParts.Sum(nil))
	// Parse etag from body of response
	err = xml.NewDecoder(resp.Body).Decode(p)
	if err != nil {
		return
	}
	// strip part count from end and '"' from front.
	remoteMd5ofParts := strings.Split(p.ETag, "-")[0]
	if len(remoteMd5ofParts) == 0 {
		return fmt.Errorf("Nil ETag")
	}
	remoteMd5ofParts = remoteMd5ofParts[1:len(remoteMd5ofParts)]
	if calculatedMd5ofParts != remoteMd5ofParts {
		if err != nil {
			return err
		}
		return fmt.Errorf("MD5 hash of part hashes comparison failed. Hash from multipart complete header: %s."+
			" Calculated multipart hash: %s.", remoteMd5ofParts, calculatedMd5ofParts)
	}
	if p.c.Md5Check {
		for i := 0; i < p.c.NTry; i++ {
			if err = p.putMd5(); err == nil {
				break
			}
		}
	}
	return
}

// Try to abort multipart upload. Do not error on failure.
func (p *putter) abort() {
	v := url.Values{}
	v.Set("uploadId", p.UploadId)
	s := p.url.String() + "?" + v.Encode()
	resp, err := p.retryRequest("DELETE", s, nil, nil)
	if err != nil {
		logger.Printf("Error aborting multipart upload: %v\n", err)
		return
	}
	defer checkClose(resp.Body, &err)
	if resp.StatusCode != 204 {
		logger.Printf("Error aborting multipart upload: %v", newRespError(resp))
	}
	return
}

// Md5 functions
func (p *putter) md5Content(r io.ReadSeeker) (string, string, error) {
	h := md5.New()
	mw := io.MultiWriter(h, p.md5)
	if _, err := io.Copy(mw, r); err != nil {
		return "", "", err
	}
	sum := h.Sum(nil)
	hexSum := fmt.Sprintf("%x", sum)
	// add to checksum of all parts for verification on upload completion
	if _, err := p.md5OfParts.Write(sum); err != nil {
		return "", "", err
	}
	return base64.StdEncoding.EncodeToString(sum), hexSum, nil
}

// Put md5 file in .md5 subdirectory of bucket  where the file is stored
// e.g. the md5 for https://mybucket.s3.amazonaws.com/gof3r will be stored in
// https://mybucket.s3.amazonaws.com/.md5/gof3r.md5
func (p *putter) putMd5() (err error) {
	calcMd5 := fmt.Sprintf("%x", p.md5.Sum(nil))
	md5Reader := strings.NewReader(calcMd5)
	md5Path := fmt.Sprint(".md5", p.url.Path, ".md5")
	md5Url, err := p.b.url(md5Path, p.c)
	if err != nil {
		return err
	}
	logger.debugPrintln("md5: ", calcMd5)
	logger.debugPrintln("md5Path: ", md5Path)
	r, err := http.NewRequest("PUT", md5Url.String(), md5Reader)
	if err != nil {
		return
	}
	p.b.Sign(r)
	resp, err := p.c.Client.Do(r)
	if err != nil {
		return
	}
	defer checkClose(resp.Body, &err)
	if resp.StatusCode != 200 {
		return newRespError(resp)
	}
	return
}

func (p *putter) retryRequest(method, urlStr string, body io.ReadSeeker, h http.Header) (resp *http.Response, err error) {
	for i := 0; i < p.c.NTry; i++ {
		var req *http.Request
		req, err = http.NewRequest(method, urlStr, body)
		if err != nil {
			return
		}
		for k := range h {
			for _, v := range h[k] {
				req.Header.Add(k, v)
			}
		}

		p.b.Sign(req)
		resp, err = p.c.Client.Do(req)
		if err == nil {
			return
		}
		logger.debugPrintln(err)
		if body != nil {
			if _, err = body.Seek(0, 0); err != nil {
				return
			}
		}
	}
	return
}

// returns true unless partSize is large enough
// to achieve maxObjSize with remaining parts
func growPartSize(partIndex int, partSize, putsz int64) bool {
	return (maxObjSize-putsz)/(maxNPart-int64(partIndex)) > partSize
}
<|endoftext|>


<commit_before>package integration_test

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os/exec"
	"strings"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/ginkgo/extensions/table"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
)

var _ = Describe("Garden External Networker errors", func() {
	var (
		command            *exec.Cmd
		fakeConfigFilePath string
		defaultConfig      map[string]interface{}
	)

	var writeConfig = func(configHash map[string]interface{}) {
		configBytes, err := json.Marshal(configHash)
		Expect(err).NotTo(HaveOccurred())
		err = ioutil.WriteFile(fakeConfigFilePath, configBytes, 0600)
		Expect(err).NotTo(HaveOccurred())
	}

	BeforeEach(func() {
		var err error
		configFile, err := ioutil.TempFile("", "adapter-config-")
		Expect(err).NotTo(HaveOccurred())
		Expect(configFile.Close()).To(Succeed())

		dir, err := ioutil.TempDir("", "fake-cni-dir")
		Expect(err).ToNot(HaveOccurred())

		stateFilePath, err := ioutil.TempFile("", "external-networker-state.json")
		Expect(err).NotTo(HaveOccurred())

		fakeConfigFilePath = configFile.Name()
		defaultConfig = map[string]interface{}{
			"cni_plugin_dir": dir,
			"cni_config_dir": dir,
			"bind_mount_dir": dir,
			"state_file":     stateFilePath.Name(),
			"start_port":     1234,
			"total_ports":    56,
		}
		writeConfig(defaultConfig)

		command = exec.Command(paths.PathToAdapter)
		command.Args = []string{paths.PathToAdapter,
			"--action=up",
			"--handle=some-container-handle",
			"--configFile=" + fakeConfigFilePath,
		}
		command.Env = []string{"PATH=/sbin"}

		command.Stdin = strings.NewReader(fmt.Sprintf(`{ "pid": %d }`, GinkgoParallelNode()))
	})

	Context("when inputs are invalid", func() {
		Context("when there's a generic error in main", func() {
			It("prints the error to stderr with the lager logger", func() {
				command.Args = []string{
					paths.PathToAdapter,
					"invalidArg",
				}
				session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
				Expect(err).NotTo(HaveOccurred())

				Eventually(session).Should(gexec.Exit(1))
				Expect(session.Err.Contents()).To(ContainSubstring("error: parse args: unexpected extra args: [invalidArg]"))
			})
		})
		Context("when stdin is not valid JSON", func() {
			It("should exit status 1 and print an error to stderr", func() {
				command.Stdin = strings.NewReader("{{{bad")
				session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
				Expect(err).NotTo(HaveOccurred())

				Eventually(session).Should(gexec.Exit(1))
				Expect(session.Out.Contents()).To(BeEmpty())
				By("checking that the error was logged to stderr")
				Expect(session.Err.Contents()).To(ContainSubstring("invalid character"))
			})
		})

		Context("when the stdin JSON is missing a pid field", func() {
			It("should exit status 1 and print an error to stderr", func() {
				command.Stdin = strings.NewReader(`{ "something": 12 }`)
				session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
				Expect(err).NotTo(HaveOccurred())

				Eventually(session, "2s").Should(gexec.Exit(1))
				Expect(session.Out.Contents()).To(BeEmpty())
				Expect(session.Err.Contents()).To(ContainSubstring("missing pid"))
			})
		})

		Context("when the provided pid is not an integer", func() {
			It("should exit status 1 and print an error to stderr", func() {
				command.Stdin = strings.NewReader(`{ "pid": "not-a-number"  }`)
				session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
				Expect(err).NotTo(HaveOccurred())

				Eventually(session).Should(gexec.Exit(1))
				Expect(session.Out.Contents()).To(BeEmpty())
				Expect(session.Err.Contents()).To(ContainSubstring(`cannot unmarshal string into Go value of type int`))
			})
		})

		Context("when the action is incorrect", func() {
			It("should return an error", func() {
				command.Args[1] = "--action=some-invalid-action"

				session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
				Expect(err).NotTo(HaveOccurred())

				Eventually(session).Should(gexec.Exit(1))
				Expect(session.Out.Contents()).To(BeEmpty())
				Expect(session.Err.Contents()).To(ContainSubstring(`unrecognized action: some-invalid-action`))
			})
		})

		Context("when an unknown flag is provided", func() {
			It("should return an error", func() {
				command.Args = append(command.Args, "--banana")

				session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
				Expect(err).NotTo(HaveOccurred())

				Eventually(session).Should(gexec.Exit(1))
				Expect(session.Out.Contents()).To(BeEmpty())
				Expect(session.Err.Contents()).To(ContainSubstring(`flag provided but not defined: -banana`))
			})
		})

		Context("when an unknown positional arg is provided", func() {
			It("should return an error", func() {
				command.Args = append(command.Args, "something-else")

				session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
				Expect(err).NotTo(HaveOccurred())

				Eventually(session).Should(gexec.Exit(1))
				Expect(session.Out.Contents()).To(BeEmpty())
				Expect(session.Err.Contents()).To(ContainSubstring(`unexpected extra args: [something-else]`))
			})
		})

		var removeArrayElement = func(src []string, elementToRemove string) []string {
			reduced := []string{}
			for _, element := range src {
				if !strings.HasPrefix(element, elementToRemove) {
					reduced = append(reduced, element)
				}
			}

			return reduced
		}

		DescribeTable("missing required arguments",
			func(missingFlag string) {
				command.Args = removeArrayElement(command.Args, "--"+missingFlag)

				session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
				Expect(err).NotTo(HaveOccurred())

				By("checking that process exits with an err")
				Eventually(session).Should(gexec.Exit(1))

				By("checking that the error was logged to stderr")
				Expect(session.Out.Contents()).To(BeEmpty())
				expectedErrorString := fmt.Sprintf("missing required flag '%s'", missingFlag)
				Expect(session.Err.Contents()).To(ContainSubstring(expectedErrorString))
			},
			Entry("action", "action"),
			Entry("handle", "handle"),
			Entry("configFile", "configFile"),
		)

		DescribeTable("missing required config",
			func(missingKey string) {
				delete(defaultConfig, missingKey)
				writeConfig(defaultConfig)

				session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
				Expect(err).NotTo(HaveOccurred())

				By("checking that process exits with an err")
				Eventually(session).Should(gexec.Exit(1))

				By("checking that the error was logged to stderr")
				Expect(session.Out.Contents()).To(BeEmpty())
				expectedErrorString := fmt.Sprintf("missing required config '%s'", missingKey)
				Expect(session.Err.Contents()).To(ContainSubstring(expectedErrorString))
			},
			Entry("cni_plugin_dir", "cni_plugin_dir"),
			Entry("cni_config_dir", "cni_config_dir"),
			Entry("bind_mount_dir", "bind_mount_dir"),
		)

		Context("when the user doesn't know what to do", func() {
			DescribeTable("arguments that indicate ignorance",
				func(args []string) {
					command.Args = args
					command.Stdin = strings.NewReader("invalid json")

					session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
					Expect(err).NotTo(HaveOccurred())

					Eventually(session).Should(gexec.Exit(1))
					Expect(session.Out.Contents()).To(BeEmpty())
					Expect(session.Err.Contents()).To(ContainSubstring(`this is a plugin for Garden-runC.  Don't run it directly.`))
				},
				Entry("no args", []string{paths.PathToAdapter}),
				Entry("short help", []string{paths.PathToAdapter, "-h"}),
				Entry("long help", []string{paths.PathToAdapter, "--help"}),
			)
		})
	})
})
<commit_msg>Relax assertion on error string<commit_after>package integration_test

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os/exec"
	"strings"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/ginkgo/extensions/table"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
)

var _ = Describe("Garden External Networker errors", func() {
	var (
		command            *exec.Cmd
		fakeConfigFilePath string
		defaultConfig      map[string]interface{}
	)

	var writeConfig = func(configHash map[string]interface{}) {
		configBytes, err := json.Marshal(configHash)
		Expect(err).NotTo(HaveOccurred())
		err = ioutil.WriteFile(fakeConfigFilePath, configBytes, 0600)
		Expect(err).NotTo(HaveOccurred())
	}

	BeforeEach(func() {
		var err error
		configFile, err := ioutil.TempFile("", "adapter-config-")
		Expect(err).NotTo(HaveOccurred())
		Expect(configFile.Close()).To(Succeed())

		dir, err := ioutil.TempDir("", "fake-cni-dir")
		Expect(err).ToNot(HaveOccurred())

		stateFilePath, err := ioutil.TempFile("", "external-networker-state.json")
		Expect(err).NotTo(HaveOccurred())

		fakeConfigFilePath = configFile.Name()
		defaultConfig = map[string]interface{}{
			"cni_plugin_dir": dir,
			"cni_config_dir": dir,
			"bind_mount_dir": dir,
			"state_file":     stateFilePath.Name(),
			"start_port":     1234,
			"total_ports":    56,
		}
		writeConfig(defaultConfig)

		command = exec.Command(paths.PathToAdapter)
		command.Args = []string{paths.PathToAdapter,
			"--action=up",
			"--handle=some-container-handle",
			"--configFile=" + fakeConfigFilePath,
		}
		command.Env = []string{"PATH=/sbin"}

		command.Stdin = strings.NewReader(fmt.Sprintf(`{ "pid": %d }`, GinkgoParallelNode()))
	})

	Context("when inputs are invalid", func() {
		Context("when there's a generic error in main", func() {
			It("prints the error to stderr with the lager logger", func() {
				command.Args = []string{
					paths.PathToAdapter,
					"invalidArg",
				}
				session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
				Expect(err).NotTo(HaveOccurred())

				Eventually(session).Should(gexec.Exit(1))
				Expect(session.Err.Contents()).To(ContainSubstring("error: parse args: unexpected extra args: [invalidArg]"))
			})
		})
		Context("when stdin is not valid JSON", func() {
			It("should exit status 1 and print an error to stderr", func() {
				command.Stdin = strings.NewReader("{{{bad")
				session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
				Expect(err).NotTo(HaveOccurred())

				Eventually(session).Should(gexec.Exit(1))
				Expect(session.Out.Contents()).To(BeEmpty())
				By("checking that the error was logged to stderr")
				Expect(session.Err.Contents()).To(ContainSubstring("invalid character"))
			})
		})

		Context("when the stdin JSON is missing a pid field", func() {
			It("should exit status 1 and print an error to stderr", func() {
				command.Stdin = strings.NewReader(`{ "something": 12 }`)
				session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
				Expect(err).NotTo(HaveOccurred())

				Eventually(session, "2s").Should(gexec.Exit(1))
				Expect(session.Out.Contents()).To(BeEmpty())
				Expect(session.Err.Contents()).To(ContainSubstring("missing pid"))
			})
		})

		Context("when the provided pid is not an integer", func() {
			It("should exit status 1 and print an error to stderr", func() {
				command.Stdin = strings.NewReader(`{ "pid": "not-a-number"  }`)
				session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
				Expect(err).NotTo(HaveOccurred())

				Eventually(session).Should(gexec.Exit(1))
				Expect(session.Out.Contents()).To(BeEmpty())
				Expect(session.Err.Contents()).To(MatchRegexp(`cannot unmarshal string into Go.*type int`))
			})
		})

		Context("when the action is incorrect", func() {
			It("should return an error", func() {
				command.Args[1] = "--action=some-invalid-action"

				session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
				Expect(err).NotTo(HaveOccurred())

				Eventually(session).Should(gexec.Exit(1))
				Expect(session.Out.Contents()).To(BeEmpty())
				Expect(session.Err.Contents()).To(ContainSubstring(`unrecognized action: some-invalid-action`))
			})
		})

		Context("when an unknown flag is provided", func() {
			It("should return an error", func() {
				command.Args = append(command.Args, "--banana")

				session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
				Expect(err).NotTo(HaveOccurred())

				Eventually(session).Should(gexec.Exit(1))
				Expect(session.Out.Contents()).To(BeEmpty())
				Expect(session.Err.Contents()).To(ContainSubstring(`flag provided but not defined: -banana`))
			})
		})

		Context("when an unknown positional arg is provided", func() {
			It("should return an error", func() {
				command.Args = append(command.Args, "something-else")

				session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
				Expect(err).NotTo(HaveOccurred())

				Eventually(session).Should(gexec.Exit(1))
				Expect(session.Out.Contents()).To(BeEmpty())
				Expect(session.Err.Contents()).To(ContainSubstring(`unexpected extra args: [something-else]`))
			})
		})

		var removeArrayElement = func(src []string, elementToRemove string) []string {
			reduced := []string{}
			for _, element := range src {
				if !strings.HasPrefix(element, elementToRemove) {
					reduced = append(reduced, element)
				}
			}

			return reduced
		}

		DescribeTable("missing required arguments",
			func(missingFlag string) {
				command.Args = removeArrayElement(command.Args, "--"+missingFlag)

				session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
				Expect(err).NotTo(HaveOccurred())

				By("checking that process exits with an err")
				Eventually(session).Should(gexec.Exit(1))

				By("checking that the error was logged to stderr")
				Expect(session.Out.Contents()).To(BeEmpty())
				expectedErrorString := fmt.Sprintf("missing required flag '%s'", missingFlag)
				Expect(session.Err.Contents()).To(ContainSubstring(expectedErrorString))
			},
			Entry("action", "action"),
			Entry("handle", "handle"),
			Entry("configFile", "configFile"),
		)

		DescribeTable("missing required config",
			func(missingKey string) {
				delete(defaultConfig, missingKey)
				writeConfig(defaultConfig)

				session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
				Expect(err).NotTo(HaveOccurred())

				By("checking that process exits with an err")
				Eventually(session).Should(gexec.Exit(1))

				By("checking that the error was logged to stderr")
				Expect(session.Out.Contents()).To(BeEmpty())
				expectedErrorString := fmt.Sprintf("missing required config '%s'", missingKey)
				Expect(session.Err.Contents()).To(ContainSubstring(expectedErrorString))
			},
			Entry("cni_plugin_dir", "cni_plugin_dir"),
			Entry("cni_config_dir", "cni_config_dir"),
			Entry("bind_mount_dir", "bind_mount_dir"),
		)

		Context("when the user doesn't know what to do", func() {
			DescribeTable("arguments that indicate ignorance",
				func(args []string) {
					command.Args = args
					command.Stdin = strings.NewReader("invalid json")

					session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
					Expect(err).NotTo(HaveOccurred())

					Eventually(session).Should(gexec.Exit(1))
					Expect(session.Out.Contents()).To(BeEmpty())
					Expect(session.Err.Contents()).To(ContainSubstring(`this is a plugin for Garden-runC.  Don't run it directly.`))
				},
				Entry("no args", []string{paths.PathToAdapter}),
				Entry("short help", []string{paths.PathToAdapter, "-h"}),
				Entry("long help", []string{paths.PathToAdapter, "--help"}),
			)
		})
	})
})
<|endoftext|>


<commit_before>package matching

import (
	"encoding/json"
	"fmt"
	"reflect"

	"github.com/SpectoLabs/hoverfly/core/util"
)

func JsonMatch(matchingString string, toMatch string) bool {
	minifiedMatchingString, err := util.MinifyJson(matchingString)
	if err != nil {
		return false
	}

	minifiedToMatch, err := util.MinifyJson(toMatch)
	if err != nil {
		return false
	}

	var matchingJson, toMatchJson map[string]interface{}

	err = json.Unmarshal([]byte(minifiedMatchingString), &matchingJson)
	if err != nil {
		return false
	}

	err = json.Unmarshal([]byte(minifiedToMatch), &toMatchJson)
	if err != nil {
		return false
	}

	fmt.Println(matchingJson)
	fmt.Println(toMatchJson)
	return reflect.DeepEqual(matchingJson, toMatchJson)
}
<commit_msg>Removing fmt calls<commit_after>package matching

import (
	"encoding/json"
	"reflect"

	"github.com/SpectoLabs/hoverfly/core/util"
)

func JsonMatch(matchingString string, toMatch string) bool {
	minifiedMatchingString, err := util.MinifyJson(matchingString)
	if err != nil {
		return false
	}

	minifiedToMatch, err := util.MinifyJson(toMatch)
	if err != nil {
		return false
	}

	var matchingJson, toMatchJson map[string]interface{}

	err = json.Unmarshal([]byte(minifiedMatchingString), &matchingJson)
	if err != nil {
		return false
	}

	err = json.Unmarshal([]byte(minifiedToMatch), &toMatchJson)
	if err != nil {
		return false
	}

	return reflect.DeepEqual(matchingJson, toMatchJson)
}
<|endoftext|>


<commit_before>// syncwatcher.go
package main

import (
	"code.google.com/p/go.exp/fsnotify"
	"os"
	"bufio"
	"io/ioutil"
	"net/http"
	"net/url"
	"crypto/tls"
	"encoding/json"
	"log"
	"fmt"
	"time"
	"flag"
	"runtime"
	"path/filepath"
	"strings"
	"sort"
)

type Configuration struct {
	Version			int
	Repositories	[]RepositoryConfiguration
}

type RepositoryConfiguration struct {
	ID					string
	Directory			string
	ReadOnly			bool
	RescanIntervalS		int
}

// HTTP Authentication
var (
	target		string
	authUser	string
	authPass	string
	csrfToken	string
	csrfFile	string
	apiKey		string
)

// HTTP Debounce
var (
	debounceTimeout = 300*time.Millisecond
	dirVsFiles = 10
)

// Main
var (
	stop = make(chan int)
)

func init() {
	flag.StringVar(&target, "target", "localhost:8080", "Target")
	flag.StringVar(&authUser, "user", "", "Username")
	flag.StringVar(&authPass, "pass", "", "Password")
	flag.StringVar(&csrfFile, "csrf", "", "CSRF token file")
	flag.StringVar(&apiKey, "api", "", "API key")
	flag.Parse()
	if !strings.Contains(target, "://") { target = "http://" + target }
	if len(csrfFile) > 0 {
		fd, err := os.Open(csrfFile)
		if err != nil {
			log.Fatal(err)
		}
		s := bufio.NewScanner(fd)
		for s.Scan() {
			csrfToken = s.Text()
		}
		fd.Close()
	}
}

func main() {

	testWebGuiPost()

	repos := getRepos()
	for i := range repos {
		repo := repos[i]
		repodir := repo.Directory
		repodir = expandTilde(repo.Directory)
		go watchRepo(repo.ID, repodir)
	}

	code := <-stop
	println("Exiting")
	os.Exit(code)

}

func getRepos() []RepositoryConfiguration {
	r, err := http.NewRequest("GET", target+"/rest/config", nil)
	if err != nil {
		log.Fatal(err)
	}
	if len(csrfToken) > 0 {
		r.Header.Set("X-CSRF-Token", csrfToken)
	}
	if len(authUser) > 0 {
		r.SetBasicAuth(authUser, authPass)
	}
	if len(apiKey) > 0 {
		r.Header.Set("X-API-Key", apiKey)
	}
	tr := &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify : true} }
	client := &http.Client{Transport: tr, Timeout: 5*time.Second}
	res, err := client.Do(r)
	if err != nil {
		log.Fatal(err)
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		log.Fatalf("Status %d != 200 for GET", res.StatusCode)
	}
	bs, err := ioutil.ReadAll(res.Body)
	if err != nil {
		log.Fatal(err)
	}
	var cfg Configuration
	err = json.Unmarshal(bs, &cfg)
	if err != nil {
		log.Fatal(err)
	}
	return cfg.Repositories
}

func watchRepo(repo string, directory string) {
	expandedDirectory := expandTilde(directory)
	sw, err := NewSyncWatcher()
	if sw == nil || err != nil {
		log.Println(err)
		return
	}
	defer sw.Close()
	err = sw.Watch(expandedDirectory)
	if err != nil {
		log.Println(err)
		return
	}
	informChangeDebounced := informChangeDebounce(debounceTimeout, repo, directory, dirVsFiles, informChange)
	log.Println("Watching " + repo + ": " + directory)
	for {
		ev := waitForEvent(sw)
		if ev == nil {
			log.Println("Error: fsnotify event is nil")
			continue
		}
		log.Println("Change detected in " + ev.Name)
		informChangeDebounced(ev.Name)
	}
}

func waitForEvent(sw *SyncWatcher) (ev *fsnotify.FileEvent) {
	var ok bool
	select {
		case ev, ok = <-sw.Event:
			if !ok {
				log.Println("Error: channel closed")
			}
		case err, eok := <-sw.Error:
			log.Println(err, eok)
	}
	return
}

func testWebGuiPost() {
	r, err := http.NewRequest("POST", target+"/rest/404", nil)
	if err != nil {
		log.Fatal(err)
	}
	if len(csrfToken) > 0 {
		r.Header.Set("X-CSRF-Token", csrfToken)
	}
	if len(authUser) > 0 {
		r.SetBasicAuth(authUser, authPass)
	}
	if len(apiKey) > 0 {
		r.Header.Set("X-API-Key", apiKey)
	}
	tr := &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify : true} }
	client := &http.Client{Transport: tr, Timeout: 5*time.Second}
	res, err := client.Do(r)
	if err != nil {
		log.Fatal(err)
	}
	defer res.Body.Close()
	if res.StatusCode != 404 {
		log.Fatalf("Status %d != 404 for POST\n", res.StatusCode)
	}
}

func informChange(repo string, sub string) {
	data := url.Values {}
	data.Set("repo", repo)
	data.Set("sub", sub)
	r, err := http.NewRequest("POST", target+"/rest/scan?"+data.Encode(), nil)
	if err != nil {
		log.Println(err)
		return
	}
	if len(csrfToken) > 0 {
		r.Header.Set("X-CSRF-Token", csrfToken)
	}
	if len(authUser) > 0 {
		r.SetBasicAuth(authUser, authPass)
	}
	if len(apiKey) > 0 {
		r.Header.Set("X-API-Key", apiKey)
	}
	tr := &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify : true} }
	client := &http.Client{Transport: tr, Timeout: 5*time.Second}
	res, err := client.Do(r)
	if err != nil {
		log.Println(err)
		return
	}
	if res.StatusCode != 200 {
		log.Printf("Error: Status %d != 200 for POST.\n" + repo + ": " + sub, res.StatusCode)
		return
	} else {
		log.Println("Syncthing is indexing change in " + repo + ": " + sub)
	}
}

func informChangeDebounce(interval time.Duration, repo string, repoDirectory string, dirVsFiles int, callback func(repo string, sub string)) func(string) {
	debounce := func(f func(paths []string)) func(string) {
		timer := &time.Timer{}
		subs := make([]string, 0)
		return func(sub string) {
			timer.Stop()
			subs = append(subs, sub)
			timer = time.AfterFunc(interval, func() {
				f(subs)
				subs = make([]string, 0)
			})
		}
	}

	return debounce(func(paths []string) {
		// Do not inform Syncthing immediately but wait for debounce
		// Therefore, we need to keep track of the paths that were changed
		// This function optimises tracking in two ways:
		//	- If there are more than `dirVsFiles` changes in a directory, we inform Syncthing to scan the entire directory
		//	- Directories with parent directory changes are aggregated. If A/B has 3 changes and A/C has 8, A will have 11 changes and if this is bigger than dirVsFiles we will scan A.
		if (len(paths) == 0) { return }
		trackedPaths := make(map[string]int) // Map directories to scores; if score == -1 the path is a filename
		sort.Strings(paths) // Make sure parent paths are processed first
		for i := range paths {
			path := paths[i]
			dir := filepath.Dir(path)
			score := 1 // File change counts for 1 per directory
			if dir == filepath.Clean(path) {
				score = dirVsFiles // Is directory itself, should definitely inform
			}
			// Search for existing parent directory relations in the map
			for trackedPath, _ := range trackedPaths {
				if strings.Contains(dir, trackedPath) {
					// Increment score of tracked current/parent directory
					trackedPaths[trackedPath] += score
				}
			}
			_, exists := trackedPaths[dir]
			if !exists {
				trackedPaths[dir] = score
			}
			trackedPaths[path] = -1
		}
		var keys []string
		for k := range trackedPaths {
			keys = append(keys, k)
		}
		sort.Strings(keys) // Sort directories before their own files
		previousDone, previousPath := false, ""
		for i := range keys {
			trackedPath := keys[i]
			trackedPathScore, _ := trackedPaths[trackedPath]
			if previousDone && strings.Contains(trackedPath, previousPath) { continue } // Already informed parent directory change
			if trackedPathScore < dirVsFiles && trackedPathScore != -1 { continue } // Not enough files for this directory or it is a file
			previousDone = trackedPathScore != -1
			previousPath = trackedPath
			sub := strings.TrimPrefix(trackedPath, repoDirectory)
			sub = strings.TrimPrefix(sub, string(os.PathSeparator))
			callback(repo, sub)
		}
	})
}

func getHomeDir() string {
	var home string
	switch runtime.GOOS {
		case "windows":
		home = filepath.Join(os.Getenv("HomeDrive"), os.Getenv("HomePath"))
		if home == "" {
			home = os.Getenv("UserProfile")
		}
		default:
			home = os.Getenv("HOME")
	}
	if home == "" {
		log.Fatal("No home directory found - set $HOME (or the platform equivalent).")
	}
	return home
}

func expandTilde(p string) string {
	if p == "~" {
		return getHomeDir()
	}
	p = filepath.FromSlash(p)
	if !strings.HasPrefix(p, fmt.Sprintf("~%c", os.PathSeparator)) {
		return p
	}
	return filepath.Join(getHomeDir(), p[2:])
}
<commit_msg>Update compatibility with Syncthing wrt Repo vs Folder naming<commit_after>// syncwatcher.go
package main

import (
	"code.google.com/p/go.exp/fsnotify"
	"os"
	"bufio"
	"io/ioutil"
	"net/http"
	"net/url"
	"crypto/tls"
	"encoding/json"
	"log"
	"fmt"
	"time"
	"flag"
	"runtime"
	"path/filepath"
	"strings"
	"sort"
)

type Configuration struct {
	Version			int
	Folders			[]FolderConfiguration
}

type FolderConfiguration struct {
	ID			string
	Path			string
	ReadOnly		bool
	RescanIntervalS		int
}

// HTTP Authentication
var (
	target		string
	authUser	string
	authPass	string
	csrfToken	string
	csrfFile	string
	apiKey		string
)

// HTTP Debounce
var (
	debounceTimeout = 300*time.Millisecond
	dirVsFiles = 10
)

// Main
var (
	stop = make(chan int)
)

func init() {
	flag.StringVar(&target, "target", "localhost:8080", "Target")
	flag.StringVar(&authUser, "user", "", "Username")
	flag.StringVar(&authPass, "pass", "", "Password")
	flag.StringVar(&csrfFile, "csrf", "", "CSRF token file")
	flag.StringVar(&apiKey, "api", "", "API key")
	flag.Parse()
	if !strings.Contains(target, "://") { target = "http://" + target }
	if len(csrfFile) > 0 {
		fd, err := os.Open(csrfFile)
		if err != nil {
			log.Fatal(err)
		}
		s := bufio.NewScanner(fd)
		for s.Scan() {
			csrfToken = s.Text()
		}
		fd.Close()
	}
}

func main() {

	testWebGuiPost()

	folders := getFolders()
	for i := range folders {
		folder := folders[i]
		folderdir := folder.Path
		folderdir = expandTilde(folder.Path)
		go watchRepo(folder.ID, folderdir)
	}

	code := <-stop
	println("Exiting")
	os.Exit(code)

}

func getFolders() []FolderConfiguration {
	r, err := http.NewRequest("GET", target+"/rest/config", nil)
	if err != nil {
		log.Fatal(err)
	}
	if len(csrfToken) > 0 {
		r.Header.Set("X-CSRF-Token", csrfToken)
	}
	if len(authUser) > 0 {
		r.SetBasicAuth(authUser, authPass)
	}
	if len(apiKey) > 0 {
		r.Header.Set("X-API-Key", apiKey)
	}
	tr := &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify : true} }
	client := &http.Client{Transport: tr, Timeout: 5*time.Second}
	res, err := client.Do(r)
	if err != nil {
		log.Fatal(err)
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		log.Fatalf("Status %d != 200 for GET", res.StatusCode)
	}
	bs, err := ioutil.ReadAll(res.Body)
	if err != nil {
		log.Fatal(err)
	}
	var cfg Configuration
	err = json.Unmarshal(bs, &cfg)
	if err != nil {
		log.Fatal(err)
	}
	return cfg.Folders
}

func watchRepo(folder string, path string) {
	expandedPath := expandTilde(path)
	sw, err := NewSyncWatcher()
	if sw == nil || err != nil {
		log.Println(err)
		return
	}
	defer sw.Close()
	err = sw.Watch(expandedPath)
	if err != nil {
		log.Println(err)
		return
	}
	informChangeDebounced := informChangeDebounce(debounceTimeout, folder, path, dirVsFiles, informChange)
	log.Println("Watching " + folder + ": " + path)
	for {
		ev := waitForEvent(sw)
		if ev == nil {
			log.Println("Error: fsnotify event is nil")
			continue
		}
		log.Println("Change detected in " + ev.Name)
		informChangeDebounced(ev.Name)
	}
}

func waitForEvent(sw *SyncWatcher) (ev *fsnotify.FileEvent) {
	var ok bool
	select {
		case ev, ok = <-sw.Event:
			if !ok {
				log.Println("Error: channel closed")
			}
		case err, eok := <-sw.Error:
			log.Println(err, eok)
	}
	return
}

func testWebGuiPost() {
	r, err := http.NewRequest("POST", target+"/rest/404", nil)
	if err != nil {
		log.Fatal(err)
	}
	if len(csrfToken) > 0 {
		r.Header.Set("X-CSRF-Token", csrfToken)
	}
	if len(authUser) > 0 {
		r.SetBasicAuth(authUser, authPass)
	}
	if len(apiKey) > 0 {
		r.Header.Set("X-API-Key", apiKey)
	}
	tr := &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify : true} }
	client := &http.Client{Transport: tr, Timeout: 5*time.Second}
	res, err := client.Do(r)
	if err != nil {
		log.Fatal(err)
	}
	defer res.Body.Close()
	if res.StatusCode != 404 {
		log.Fatalf("Status %d != 404 for POST\n", res.StatusCode)
	}
}

func informChange(folder string, sub string) {
	data := url.Values {}
	data.Set("folder", folder)
	data.Set("sub", sub)
	r, err := http.NewRequest("POST", target+"/rest/scan?"+data.Encode(), nil)
	if err != nil {
		log.Println(err)
		return
	}
	if len(csrfToken) > 0 {
		r.Header.Set("X-CSRF-Token", csrfToken)
	}
	if len(authUser) > 0 {
		r.SetBasicAuth(authUser, authPass)
	}
	if len(apiKey) > 0 {
		r.Header.Set("X-API-Key", apiKey)
	}
	tr := &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify : true} }
	client := &http.Client{Transport: tr, Timeout: 5*time.Second}
	res, err := client.Do(r)
	if err != nil {
		log.Println(err)
		return
	}
	if res.StatusCode != 200 {
		log.Printf("Error: Status %d != 200 for POST.\n" + folder + ": " + sub, res.StatusCode)
		return
	} else {
		log.Println("Syncthing is indexing change in " + folder + ": " + sub)
	}
}

func informChangeDebounce(interval time.Duration, folder string, folderPath string, dirVsFiles int, callback func(folder string, sub string)) func(string) {
	debounce := func(f func(paths []string)) func(string) {
		timer := &time.Timer{}
		subs := make([]string, 0)
		return func(sub string) {
			timer.Stop()
			subs = append(subs, sub)
			timer = time.AfterFunc(interval, func() {
				f(subs)
				subs = make([]string, 0)
			})
		}
	}

	return debounce(func(paths []string) {
		// Do not inform Syncthing immediately but wait for debounce
		// Therefore, we need to keep track of the paths that were changed
		// This function optimises tracking in two ways:
		//	- If there are more than `dirVsFiles` changes in a directory, we inform Syncthing to scan the entire directory
		//	- Directories with parent directory changes are aggregated. If A/B has 3 changes and A/C has 8, A will have 11 changes and if this is bigger than dirVsFiles we will scan A.
		if (len(paths) == 0) { return }
		trackedPaths := make(map[string]int) // Map directories to scores; if score == -1 the path is a filename
		sort.Strings(paths) // Make sure parent paths are processed first
		for i := range paths {
			path := paths[i]
			dir := filepath.Dir(path)
			score := 1 // File change counts for 1 per directory
			if dir == filepath.Clean(path) {
				score = dirVsFiles // Is directory itself, should definitely inform
			}
			// Search for existing parent directory relations in the map
			for trackedPath, _ := range trackedPaths {
				if strings.Contains(dir, trackedPath) {
					// Increment score of tracked current/parent directory
					trackedPaths[trackedPath] += score
				}
			}
			_, exists := trackedPaths[dir]
			if !exists {
				trackedPaths[dir] = score
			}
			trackedPaths[path] = -1
		}
		var keys []string
		for k := range trackedPaths {
			keys = append(keys, k)
		}
		sort.Strings(keys) // Sort directories before their own files
		previousDone, previousPath := false, ""
		for i := range keys {
			trackedPath := keys[i]
			trackedPathScore, _ := trackedPaths[trackedPath]
			if previousDone && strings.Contains(trackedPath, previousPath) { continue } // Already informed parent directory change
			if trackedPathScore < dirVsFiles && trackedPathScore != -1 { continue } // Not enough files for this directory or it is a file
			previousDone = trackedPathScore != -1
			previousPath = trackedPath
			sub := strings.TrimPrefix(trackedPath, folderPath)
			sub = strings.TrimPrefix(sub, string(os.PathSeparator))
			callback(folder, sub)
		}
	})
}

func getHomeDir() string {
	var home string
	switch runtime.GOOS {
		case "windows":
		home = filepath.Join(os.Getenv("HomeDrive"), os.Getenv("HomePath"))
		if home == "" {
			home = os.Getenv("UserProfile")
		}
		default:
			home = os.Getenv("HOME")
	}
	if home == "" {
		log.Fatal("No home path found - set $HOME (or the platform equivalent).")
	}
	return home
}

func expandTilde(p string) string {
	if p == "~" {
		return getHomeDir()
	}
	p = filepath.FromSlash(p)
	if !strings.HasPrefix(p, fmt.Sprintf("~%c", os.PathSeparator)) {
		return p
	}
	return filepath.Join(getHomeDir(), p[2:])
}
<|endoftext|>