<commit_before>// Copyright 2019 The Go Cloud Development Kit Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package docstore provides a portable implementation of a document store.
// A document store is a service that stores data in recursively defined
// documents (like, but not necessarily the same as, JSON). Like other
// NoSQL databases, document stores are schemaless.
// See https://en.wikipedia.org/wiki/Document-oriented_database for more information.
//
// In docstore, documents are grouped into collections, and each document has a key
// that is unique in its collection. You can add, delete, modify and retrieve
// documents by key, and you can query a collection to retrieve documents that match
// certain criteria.
//
// Subpackages contain distinct implementations ("drivers") of docstore for various
// providers, including MongoDB, Google Cloud Firestore, and Amazon DynamoDB. You can
// use Azure Cosmos and Amazon DocumentDB via the MongoDB driver. There is also
// memdocstore, an in-process, in-memory implementation suitable for testing and
// development.
//
// Your application should import one of these provider-specific subpackages and use
// its exported functions to create a *Collection; do not use the NewCollection
// function in this package. For example:
//
//   coll, err := memdocstore.OpenCollection("SSN", nil)
//   if err != nil {
//     return fmt.Errorf("opening collection: %v", err)
//   }
//   defer coll.Close()
//   // coll is a *docstore.Collection
//
// Then, write your application code using the *Collection type. You can easily
// reconfigure your initialization code to choose a different provider.
// You can develop your application locally using memdocstore, or deploy it to
// multiple Cloud providers. You may find http://github.com/google/wire useful
// for managing your initialization code.
//
// Alternatively, you can construct a *Collection via a URL and the OpenCollection
// function in this package. See https://gocloud.dev/concepts/urls for more
// information.
//
//
// Documents
//
// Docstore allows you to represent documents as either map[string]interface{} or
// struct pointers. Using structs is recommended, because it enforces some structure
// on your data. Docstore mimics the encoding/json package in its treatment of
// structs: by default, a struct's exported fields are the fields of the document.
// You can alter this default mapping by using a struct tag beginning with
// "docstore:". Docstore struct tags support renaming, omitting fields
// unconditionally, or omitting them only when they are empty, exactly like
// encoding/json. Docstore also honors a "json" struct tag if there is no "docstore"
// tag on the field.
//
//
// Representing Data
//
// The value stored in a document field can be any of a wide range of types. All
// primitive types except for complex numbers are supported, as well as slices and
// maps (the map key type must be a string, an integer, or a type that implements
// encoding.TextMarshaler). In addition, any type that implements
// encoding.BinaryMarshaler or encoding.TextMarshaler is permitted. This set of types
// closely matches the encoding/json package.
//
// Times deserve special mention. Docstore can store and retrieve values of type
// time.Time, with two caveats. First, the timezone will not be preserved. Second,
// Docstore guarantees only that time.Time values are represented to millisecond
// precision. Many providers will do better, but if you need to be sure that times
// are stored with nanosecond precision, convert the time.Time to another type before
// storing, and re-create when you retrieve it. For instance, you could call the
// time's UnixNano method to get an int64, and get the original time back (in the
// local timezone) with the time.Unix function.
//
//
// Keys
//
// The key of a docstore document is some function of its contents, usually a field.
// Keys never appear alone in the docstore API, only as part of a document. For
// instance, to retrieve a document by key, you pass the Get method a document whose
// key field is populated, and docstore populates the rest of that argument with the
// stored document.
//
// When you open a collection using one of the subpackages' OpenCollection methods or
// a URL, you specify how to extract the key from a document.
// Usually, you provide the name of the key field, as in the example above:
//
//   coll, err := memdocstore.OpenBucket("SSN", nil)
//
// Here, the "SSN" field of the document is used as the key. Some providers let you
// supply a function to extract the key from the document, which can be useful if the
// key is composed of more than field.
//
//
// Actions
//
// Docstore supports six actions on documents:
//   - Get retrieves a document.
//   - Create creates a new document.
//   - Replace replaces an existing document.
//   - Put puts a document into a collection whether or not it is already present.
//   - Update applies a set of modifications to a document.
//   - Delete deletes a document.
//
// Each action acts atomically on a single document. You can execute actions
// individually--the Collection type has a method for each one--or you can group them
// into a list, like so:
//
//   err := coll.Actions().Put(doc1).Replace(doc2).Get(doc3).Do(ctx)
//
// When you use an action list, docstore will try to optimize the execution of the
// actions. For example, multiple Get actions may be combined into a single "batch
// get" RPC. For the most part, actions in a list execute in an undefined order
// (perhaps concurrently) and independently, but a Get and a write on the same
// document are ordered with respect to each other. See the documentation of
// ActionList for details.
//
//
// Revisions
//
// Docstore supports document revisions to distinguish different versions of a
// document and enable optimistic locking. By default, Docstore stores the revision
// in the field named "DocstoreRevision" (stored in the constant
// docstore.DefaultRevisionField). Providers give you the option of changing
// that field name.
//
// Docstore gives every document a revision when it is created, and changes that
// revision whenever the document is modified. If a struct doesn't have a revision
// field, then no revision logic is performed. Otherwise, whenever you pass a
// document with a revision to Put, Replace, Update or Delete, Docstore will return
// an error with code FailedPrecondition if the stored document's revision does not
// match the given document's. (See the gocloud.dev/gcerrors package for information
// about error codes.) If you call Get to retrieve a document, then later perform a
// write action with that same document, it will fail if the document was changed
// since the Get. Create, Put, Replace and Update all populate their argument
// document with the new revision of the document.
//
// Since different providers use different types for revisions, the type of the
// revision field is unspecified. When defining a struct for storing docstore data,
// define the field to be of type interface{}. For example,
//
//    type User {
//      Name             string
//      DocstoreRevision interface{}
//    }
//
//
// Queries
//
// Docstore supports querying within a collection. Call the Query method on
// Collection to obtain a Query value, then build your query by calling Query methods
// like Where, Limit and so on. Finally, call the Get method on the query to execute it.
// The result is an iterator, whose use is described below.
//
//     iter := coll.Query().Where("size", ">", 10).Limit(5).Get(ctx)
//
// The Where methods defines a filter condition, much like a WHERE clause in SQL.
// Conditions are of the form "field op value", where field is any document field
// path (including dot-separated paths), op is one of "=", ">", "<", ">=" or "<=",
// and value can be any value.
//
// You can make multiple Where calls. In some cases, parts of a Where clause may be
// processed on the client rather than natively by the provider, which may have
// performance implications for large result sets. See the provider-specific package
// documentation for details.
//
// Use the DocumentIterator returned from Query.Get by repeatedly calling its Next
// method until it returns io.EOF. Always call Stop when you are finished with an
// iterator. It is wise to use a defer statement for this.
//
//     iter := coll.Query().Where("size", ">", 10).Limit(5).Get(ctx)
//     defer iter.Stop()
//     for {
//         m := map[string]interface{}{}
//         err := iter.Next(ctx, m)
//         if err == io.EOF {
//             break
//         }
//         if err != nil {
//             return err
//         }
//         fmt.Println(m)
//     }
//
//
// Errors
//
// The errors returned from this package can be inspected in several ways:
//
// The Code function from gocloud.dev/gcerrors will return an error code, also
// defined in that package, when invoked on an error.
//
// The Collection.ErrorAs method can retrieve the driver error underlying the returned
// error.
// TODO(shantuo): implement ErrorAs.
//
//
// OpenCensus Integration
// TODO(jba): implement
package docstore // import "gocloud.dev/docstore"
<commit_msg>docstore: fix some errors in the package doc (#2433)<commit_after>// Copyright 2019 The Go Cloud Development Kit Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package docstore provides a portable implementation of a document store.
// A document store is a service that stores data in recursively defined
// documents (like, but not necessarily the same as, JSON). Like other
// NoSQL databases, document stores are schemaless.
// See https://en.wikipedia.org/wiki/Document-oriented_database for more information.
//
// In docstore, documents are grouped into collections, and each document has a key
// that is unique in its collection. You can add, delete, modify and retrieve
// documents by key, and you can query a collection to retrieve documents that match
// certain criteria.
//
// Subpackages contain distinct implementations ("drivers") of docstore for various
// providers, including MongoDB, Google Cloud Firestore, and Amazon DynamoDB.
// You can use Azure Cosmos DB and Amazon DocumentDB via the MongoDB driver.
// There is also memdocstore, an in-process, in-memory implementation suitable
// for testing and development.
//
// Your application should import one of these provider-specific subpackages and use
// its exported functions to create a *Collection; do not use the NewCollection
// function in this package. For example:
//
//   coll, err := memdocstore.OpenCollection("SSN", nil)
//   if err != nil {
//     return fmt.Errorf("opening collection: %v", err)
//   }
//   defer coll.Close()
//   // coll is a *docstore.Collection
//
// Then, write your application code using the *Collection type. You can easily
// reconfigure your initialization code to choose a different provider.
// You can develop your application locally using memdocstore, or deploy it to
// multiple Cloud providers. You may find http://github.com/google/wire useful
// for managing your initialization code.
//
// Alternatively, you can construct a *Collection via a URL and the OpenCollection
// function in this package. See https://gocloud.dev/concepts/urls for more
// information.
//
//
// Documents
//
// Docstore allows you to represent documents as either map[string]interface{} or
// struct pointers. Using structs is recommended, because it enforces some structure
// on your data. Docstore mimics the encoding/json package in its treatment of
// structs: by default, a struct's exported fields are the fields of the document.
// You can alter this default mapping by using a struct tag beginning with
// "docstore:". Docstore struct tags support renaming, omitting fields
// unconditionally, or omitting them only when they are empty, exactly like
// encoding/json. Docstore also honors a "json" struct tag if there is no "docstore"
// tag on the field.
//
//
// Representing Data
//
// The value stored in a document field can be any of a wide range of types. All
// primitive types except for complex numbers are supported, as well as slices and
// maps (the map key type must be a string, an integer, or a type that implements
// encoding.TextMarshaler). In addition, any type that implements
// encoding.BinaryMarshaler or encoding.TextMarshaler is permitted. This set of types
// closely matches the encoding/json package.
//
// Times deserve special mention. Docstore can store and retrieve values of type
// time.Time, with two caveats. First, the timezone will not be preserved. Second,
// Docstore guarantees only that time.Time values are represented to millisecond
// precision. Many providers will do better, but if you need to be sure that times
// are stored with nanosecond precision, convert the time.Time to another type before
// storing, and re-create when you retrieve it. For instance, you could call the
// time's UnixNano method to get an int64, and get the original time back (in the
// local timezone) with the time.Unix function.
//
//
// Keys
//
// The key of a docstore document is some function of its contents, usually a field.
// Keys never appear alone in the docstore API, only as part of a document. For
// instance, to retrieve a document by key, you pass the Get method a document whose
// key field is populated, and docstore populates the rest of that argument with the
// stored document.
//
// When you open a collection using one of the subpackages' OpenCollection methods or
// a URL, you specify how to extract the key from a document.
// Usually, you provide the name of the key field, as in the example below:
//
//   coll, err := memdocstore.OpenCollection("SSN", nil)
//
// Here, the "SSN" field of the document is used as the key. Some providers let you
// supply a function to extract the key from the document, which can be useful if the
// key is composed of more than one field.
//
//
// Actions
//
// Docstore supports six actions on documents:
//   - Get retrieves a document.
//   - Create creates a new document.
//   - Replace replaces an existing document.
//   - Put puts a document into a collection whether or not it is already present.
//   - Update applies a set of modifications to a document.
//   - Delete deletes a document.
//
// Each action acts atomically on a single document. You can execute actions
// individually--the Collection type has a method for each one--or you can group them
// into a list, like so:
//
//   err := coll.Actions().Put(doc1).Replace(doc2).Get(doc3).Do(ctx)
//
// When you use an action list, docstore will try to optimize the execution of the
// actions. For example, multiple Get actions may be combined into a single "batch
// get" RPC. For the most part, actions in a list execute in an undefined order
// (perhaps concurrently) and independently, but a Get and a write on the same
// document are ordered with respect to each other. See the documentation of
// ActionList for details.
//
//
// Revisions
//
// Docstore supports document revisions to distinguish different versions of a
// document and enable optimistic locking. By default, Docstore stores the revision
// in the field named "DocstoreRevision" (stored in the constant
// DefaultRevisionField). Providers give you the option of changing that field
// name.
//
// Docstore gives every document a revision when it is created, and changes that
// revision whenever the document is modified. If a struct doesn't have a revision
// field, then no revision logic is performed. Otherwise, whenever you pass a
// document with a revision to Put, Replace, Update or Delete, Docstore will return
// an error with code FailedPrecondition if the stored document's revision does not
// match the given document's. (See the gocloud.dev/gcerrors package for information
// about error codes.) If you call Get to retrieve a document, then later perform a
// write action with that same document, it will fail if the document was changed
// since the Get. Create, Put, Replace and Update all populate their argument
// document with the new revision of the document.
//
// Since different providers use different types for revisions, the type of the
// revision field is unspecified. When defining a struct for storing docstore data,
// define the field to be of type interface{}. For example,
//
//    type User {
//      Name             string
//      DocstoreRevision interface{}
//    }
//
//
// Queries
//
// Docstore supports querying within a collection. Call the Query method on
// Collection to obtain a Query value, then build your query by calling Query methods
// like Where, Limit and so on. Finally, call the Get method on the query to execute it.
// The result is an iterator, whose use is described below.
//
//     iter := coll.Query().Where("size", ">", 10).Limit(5).Get(ctx)
//
// The Where methods defines a filter condition, much like a WHERE clause in SQL.
// Conditions are of the form "field op value", where field is any document field
// path (including dot-separated paths), op is one of "=", ">", "<", ">=" or "<=",
// and value can be any value.
//
// You can make multiple Where calls. In some cases, parts of a Where clause may be
// processed on the client rather than natively by the provider, which may have
// performance implications for large result sets. See the provider-specific package
// documentation for details.
//
// Use the DocumentIterator returned from Query.Get by repeatedly calling its Next
// method until it returns io.EOF. Always call Stop when you are finished with an
// iterator. It is wise to use a defer statement for this.
//
//     iter := coll.Query().Where("size", ">", 10).Limit(5).Get(ctx)
//     defer iter.Stop()
//     for {
//         m := map[string]interface{}{}
//         err := iter.Next(ctx, m)
//         if err == io.EOF {
//             break
//         }
//         if err != nil {
//             return err
//         }
//         fmt.Println(m)
//     }
//
//
// Errors
//
// The errors returned from this package can be inspected in several ways:
//
// The Code function from gocloud.dev/gcerrors will return an error code, also
// defined in that package, when invoked on an error.
//
// The Collection.ErrorAs method can retrieve the underlying driver error from
// the returned error. See the specific driver's package doc for the supported
// types.
//
//
// OpenCensus Integration
// TODO(jba): implement
package docstore // import "gocloud.dev/docstore"
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"
	"text/template"
)

type OutputTemplateData struct {
	Dir  string
	OS   string
	Arch string
}

type CompileOpts struct {
	PackagePath string
	Platform    Platform
	OutputTpl   string
	Ldflags     string
	Gcflags     string
	Tags        string
	Cgo         bool
	Rebuild     bool
}

// GoCrossCompile
func GoCrossCompile(opts *CompileOpts) error {
	env := append(os.Environ(),
		"GOOS="+opts.Platform.OS,
		"GOARCH="+opts.Platform.Arch)
	if opts.Cgo {
		env = append(env, "CGO_ENABLED=1")
	}

	var outputPath bytes.Buffer
	tpl, err := template.New("output").Parse(opts.OutputTpl)
	if err != nil {
		return err
	}
	tplData := OutputTemplateData{
		Dir:  filepath.Base(opts.PackagePath),
		OS:   opts.Platform.OS,
		Arch: opts.Platform.Arch,
	}
	if err := tpl.Execute(&outputPath, &tplData); err != nil {
		return err
	}

	if opts.Platform.OS == "windows" {
		outputPath.WriteString(".exe")
	}

	// Determine the full path to the output so that we can change our
	// working directory when executing go build.
	outputPathReal := outputPath.String()
	outputPathReal, err = filepath.Abs(outputPathReal)
	if err != nil {
		return err
	}

	// Go prefixes the import directory with '_' when it is outside
	// the GOPATH.For this, we just drop it since we move to that
	// directory to build.
	chdir := ""
	if opts.PackagePath[0] == '_' {
		if runtime.GOOS == "windows" {
			// We have to replace weird paths like this:
			//
			//   _/c_/Users
			//
			// With:
			//
			//   c:\Users
			//
			re := regexp.MustCompile("^/([a-zA-Z])_/")
			chdir = re.ReplaceAllString(opts.PackagePath[1:], "$1:\\")
			chdir = strings.Replace(chdir, "/", "\\", -1)
		} else {
			chdir = opts.PackagePath[1:]
		}

		opts.PackagePath = ""
	}

	args := []string{"build"}
	if opts.Rebuild {
		args = append(args, "-a")
	}
	args = append(args,
		"-gcflags", opts.Gcflags,
		"-ldflags", opts.Ldflags,
		"-tags", opts.Tags,
		"-o", outputPathReal,
		opts.PackagePath)

	_, err = execGo(env, chdir, args...)
	return err
}

// GoMainDirs returns the file paths to the packages that are "main"
// packages, from the list of packages given. The list of packages can
// include relative paths, the special "..." Go keyword, etc.
func GoMainDirs(packages []string) ([]string, error) {
	args := make([]string, 0, len(packages)+3)
	args = append(args, "list", "-f", "{{.Name}}|{{.ImportPath}}")
	args = append(args, packages...)

	output, err := execGo(nil, "", args...)
	if err != nil {
		return nil, err
	}

	results := make([]string, 0, len(output))
	for _, line := range strings.Split(output, "\n") {
		if line == "" {
			continue
		}

		parts := strings.SplitN(line, "|", 2)
		if len(parts) != 2 {
			log.Printf("Bad line reading packages: %s", line)
			continue
		}

		if parts[0] == "main" {
			results = append(results, parts[1])
		}
	}

	return results, nil
}

// GoRoot returns the GOROOT value for the compiled `go` binary.
func GoRoot() (string, error) {
	output, err := execGo(nil, "", "env", "GOROOT")
	if err != nil {
		return "", err
	}

	return strings.TrimSpace(output), nil
}

// GoVersion reads the version of `go` that is on the PATH. This is done
// instead of `runtime.Version()` because it is possible to run gox against
// another Go version.
func GoVersion() (string, error) {
	// NOTE: We use `go run` instead of `go version` because the output
	// of `go version` might change whereas the source is guaranteed to run
	// for some time thanks to Go's compatibility guarantee.

	td, err := ioutil.TempDir("", "gox")
	if err != nil {
		return "", err
	}
	defer os.RemoveAll(td)

	// Write the source code for the program that will generate the version
	sourcePath := filepath.Join(td, "version.go")
	if err := ioutil.WriteFile(sourcePath, []byte(versionSource), 0644); err != nil {
		return "", err
	}

	// Execute and read the version, which will be the only thing on stdout.
	return execGo(nil, "", "run", sourcePath)
}

// GoVersionParts parses the version numbers from the version itself
// into major and minor: 1.5, 1.4, etc.
func GoVersionParts() (result [2]int, err error) {
	version, err := GoVersion()
	if err != nil {
		return
	}

	_, err = fmt.Sscanf(version, "go%d.%d", &result[0], &result[1])
	return
}

func execGo(env []string, dir string, args ...string) (string, error) {
	var stderr, stdout bytes.Buffer
	cmd := exec.Command("go", args...)
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr
	if env != nil {
		cmd.Env = env
	}
	if dir != "" {
		cmd.Dir = dir
	}
	if err := cmd.Run(); err != nil {
		err = fmt.Errorf("%s\nStderr: %s", err, stderr.String())
		return "", err
	}

	return stdout.String(), nil
}

const versionSource = `package main

import (
	"fmt"
	"runtime"
)

func main() {
	fmt.Print(runtime.Version())
}`
<commit_msg>disable cgo by default explicitly<commit_after>package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"
	"text/template"
)

type OutputTemplateData struct {
	Dir  string
	OS   string
	Arch string
}

type CompileOpts struct {
	PackagePath string
	Platform    Platform
	OutputTpl   string
	Ldflags     string
	Gcflags     string
	Tags        string
	Cgo         bool
	Rebuild     bool
}

// GoCrossCompile
func GoCrossCompile(opts *CompileOpts) error {
	env := append(os.Environ(),
		"GOOS="+opts.Platform.OS,
		"GOARCH="+opts.Platform.Arch)
	if opts.Cgo {
		env = append(env, "CGO_ENABLED=1")
	} else {
		env = append(env, "CGO_ENABLED=0")
	}

	var outputPath bytes.Buffer
	tpl, err := template.New("output").Parse(opts.OutputTpl)
	if err != nil {
		return err
	}
	tplData := OutputTemplateData{
		Dir:  filepath.Base(opts.PackagePath),
		OS:   opts.Platform.OS,
		Arch: opts.Platform.Arch,
	}
	if err := tpl.Execute(&outputPath, &tplData); err != nil {
		return err
	}

	if opts.Platform.OS == "windows" {
		outputPath.WriteString(".exe")
	}

	// Determine the full path to the output so that we can change our
	// working directory when executing go build.
	outputPathReal := outputPath.String()
	outputPathReal, err = filepath.Abs(outputPathReal)
	if err != nil {
		return err
	}

	// Go prefixes the import directory with '_' when it is outside
	// the GOPATH.For this, we just drop it since we move to that
	// directory to build.
	chdir := ""
	if opts.PackagePath[0] == '_' {
		if runtime.GOOS == "windows" {
			// We have to replace weird paths like this:
			//
			//   _/c_/Users
			//
			// With:
			//
			//   c:\Users
			//
			re := regexp.MustCompile("^/([a-zA-Z])_/")
			chdir = re.ReplaceAllString(opts.PackagePath[1:], "$1:\\")
			chdir = strings.Replace(chdir, "/", "\\", -1)
		} else {
			chdir = opts.PackagePath[1:]
		}

		opts.PackagePath = ""
	}

	args := []string{"build"}
	if opts.Rebuild {
		args = append(args, "-a")
	}
	args = append(args,
		"-gcflags", opts.Gcflags,
		"-ldflags", opts.Ldflags,
		"-tags", opts.Tags,
		"-o", outputPathReal,
		opts.PackagePath)

	_, err = execGo(env, chdir, args...)
	return err
}

// GoMainDirs returns the file paths to the packages that are "main"
// packages, from the list of packages given. The list of packages can
// include relative paths, the special "..." Go keyword, etc.
func GoMainDirs(packages []string) ([]string, error) {
	args := make([]string, 0, len(packages)+3)
	args = append(args, "list", "-f", "{{.Name}}|{{.ImportPath}}")
	args = append(args, packages...)

	output, err := execGo(nil, "", args...)
	if err != nil {
		return nil, err
	}

	results := make([]string, 0, len(output))
	for _, line := range strings.Split(output, "\n") {
		if line == "" {
			continue
		}

		parts := strings.SplitN(line, "|", 2)
		if len(parts) != 2 {
			log.Printf("Bad line reading packages: %s", line)
			continue
		}

		if parts[0] == "main" {
			results = append(results, parts[1])
		}
	}

	return results, nil
}

// GoRoot returns the GOROOT value for the compiled `go` binary.
func GoRoot() (string, error) {
	output, err := execGo(nil, "", "env", "GOROOT")
	if err != nil {
		return "", err
	}

	return strings.TrimSpace(output), nil
}

// GoVersion reads the version of `go` that is on the PATH. This is done
// instead of `runtime.Version()` because it is possible to run gox against
// another Go version.
func GoVersion() (string, error) {
	// NOTE: We use `go run` instead of `go version` because the output
	// of `go version` might change whereas the source is guaranteed to run
	// for some time thanks to Go's compatibility guarantee.

	td, err := ioutil.TempDir("", "gox")
	if err != nil {
		return "", err
	}
	defer os.RemoveAll(td)

	// Write the source code for the program that will generate the version
	sourcePath := filepath.Join(td, "version.go")
	if err := ioutil.WriteFile(sourcePath, []byte(versionSource), 0644); err != nil {
		return "", err
	}

	// Execute and read the version, which will be the only thing on stdout.
	return execGo(nil, "", "run", sourcePath)
}

// GoVersionParts parses the version numbers from the version itself
// into major and minor: 1.5, 1.4, etc.
func GoVersionParts() (result [2]int, err error) {
	version, err := GoVersion()
	if err != nil {
		return
	}

	_, err = fmt.Sscanf(version, "go%d.%d", &result[0], &result[1])
	return
}

func execGo(env []string, dir string, args ...string) (string, error) {
	var stderr, stdout bytes.Buffer
	cmd := exec.Command("go", args...)
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr
	if env != nil {
		cmd.Env = env
	}
	if dir != "" {
		cmd.Dir = dir
	}
	if err := cmd.Run(); err != nil {
		err = fmt.Errorf("%s\nStderr: %s", err, stderr.String())
		return "", err
	}

	return stdout.String(), nil
}

const versionSource = `package main

import (
	"fmt"
	"runtime"
)

func main() {
	fmt.Print(runtime.Version())
}`
<|endoftext|>


<commit_before>/*
Copyright 2018 Google Inc. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1

import (
	"errors"
	"reflect"
	"testing"

	"github.com/GoogleCloudPlatform/runtimes-common/tuf/metadata"
	"github.com/GoogleCloudPlatform/runtimes-common/tuf/testutil"
	"github.com/GoogleCloudPlatform/runtimes-common/tuf/types"
)

func TestRootMetadata(t *testing.T) {
	tuf := TUF{testutil.TestTUFConfig,
		[]types.Scheme{},
		[]types.Scheme{},
		[]types.Scheme{}, []types.Scheme{}}
	rootMetadata, err := tuf.PopulateRootMetadata()
	if err != nil {
		t.Fatalf("Did not expect error %v", err)
	}
	expectedBaseSigned := &metadata.BaseSigned{
		Type:        "root",
		SpecVersion: 1,
		Version:     1,
	}
	// TODO (tejaldesai): Fix this by reading time form config.
	expectedBaseSigned.Expires = rootMetadata.Signed.BaseSigned.Expires
	if !reflect.DeepEqual(rootMetadata.Signed.BaseSigned, expectedBaseSigned) {
		t.Fatalf("Expected %v\n Got %v", rootMetadata.Signed.BaseSigned, expectedBaseSigned)
	}
}

var tuf = TUF{
	TufConfig: testutil.TestTUFConfig,
	RootSecrets: []types.Scheme{
		&testutil.TestKey{
			PrivateKey: "secret1",
			KeyType:    "test-scheme1",
			PublicKey:  "public1",
			KeyId:      "idroot1",
			SignStr:    "secret1",
		},
		&testutil.TestKey{
			PrivateKey: "secret2",
			KeyType:    "test-scheme2",
			PublicKey:  "public2",
			KeyId:      "idroot2",
			SignStr:    "secret2",
		},
	},
	TargetSecrets: []types.Scheme{
		&testutil.TestKey{
			PrivateKey: "target",
			KeyType:    "test-scheme1",
			PublicKey:  "targetpublic",
			KeyId:      "idtarget",
			SignStr:    "targetsecret",
		},
	},
	SnapshotSecrets: []types.Scheme{
		&testutil.TestKey{
			PrivateKey: "snapshot",
			KeyType:    "test-scheme1",
			PublicKey:  "snapshotpublic",
			KeyId:      "idsnapshot",
			SignStr:    "snapshotsecret",
		},
	},
}

func TestPopulateRoles(t *testing.T) {
	expectedResult := map[types.RoleType]Role{
		"root": {
			Keyids: []types.KeyId{
				types.KeyId("idroot1"),
				types.KeyId("idroot2"),
			},
			Threshold: 1,
		},
		"target": {
			Keyids: []types.KeyId{
				types.KeyId("idtarget"),
			},
			Threshold: 1,
		},
		"snapshot": {
			Keyids: []types.KeyId{
				types.KeyId("idsnapshot"),
			},
			Threshold: 1,
		},
	}
	actual := tuf.populateRoles()
	if !reflect.DeepEqual(expectedResult, actual) {
		t.Fatalf("Expected %v\n Got : %v", expectedResult, actual)
	}
}

func TestPopulateKeys(t *testing.T) {
	expectedResult := map[types.KeyId]KeysDef{
		types.KeyId("idroot1"): {
			KeyidHashAlgorithms: []types.HashAlgo{types.HashAlgo("sha256")},
			Val: map[string]string{
				"public": "public1",
			},
			Keytype: types.KeyType("test-scheme1"),
			Scheme:  types.KeyScheme("test-scheme1"),
		},
		types.KeyId("idroot2"): {
			KeyidHashAlgorithms: []types.HashAlgo{types.HashAlgo("sha256")},
			Val: map[string]string{
				"public": "public2",
			},
			Keytype: types.KeyType("test-scheme2"),
			Scheme:  types.KeyScheme("test-scheme2"),
		},
	}
	actual := populateKeys(tuf.RootSecrets)
	if !reflect.DeepEqual(expectedResult, actual) {
		t.Fatalf("Expected:\n%v\n Got :\n%v", expectedResult, actual)
	}
}

var signTC = []struct {
	name         string
	signMetadata interface{}
	secrets      string
	expectedErr  error
	expectedRes  []metadata.Signature
}{
	{"signRootSuccess", "str", "root", nil,
		[]metadata.Signature{
			{
				KeyId: types.KeyId("idroot1"),
				Sig:   "secret1str",
			},
			{
				KeyId: types.KeyId("idroot2"),
				Sig:   "secret2str",
			},
		},
	},
	{"signTargetSuccess", "str", "target", nil,
		[]metadata.Signature{
			{
				KeyId: types.KeyId("idtarget"),
				Sig:   "targetsecretstr",
			},
		},
	},
	{"signSnapshotSuccess", "str", "snapshot", nil,
		[]metadata.Signature{
			{
				KeyId: types.KeyId("idsnapshot"),
				Sig:   "snapshotsecretstr",
			},
		},
	},
	{"singErr", 1, "snapshot", errors.New("Signing Error"), nil},
}

func TestSign(t *testing.T) {
	for _, tc := range signTC {
		t.Run(tc.name, func(t *testing.T) {
			var actual []metadata.Signature
			var err error
			switch tc.secrets {
			case "root":
				actual, err = tuf.SignRootMetadata(tc.signMetadata)
			case "target":
				actual, err = tuf.SignTargetMetadata(tc.signMetadata)
			default:
				actual, err = tuf.SignSnapshotMetadata(tc.signMetadata)

			}
			if !reflect.DeepEqual(tc.expectedErr, err) {
				t.Fatalf("Expected Error:\n%v\n Got :\n%v", tc.expectedErr, err)
			}
			if !reflect.DeepEqual(tc.expectedRes, actual) {
				t.Fatalf("Expected:\n%v\n Got :\n%v", tc.expectedRes, actual)
			}
		})
	}
}

var populateTargetsTC = []struct {
	name        string
	targets     []string
	expectedRes map[string]Target
	err         error
}{
	{"success", []string{"file1.txt", "file22.txt"},
		map[string]Target{
			"file1.txt": {
				Custom: map[string]string{
					"name":       "file1.txt",
					"permission": "0644",
				},
				Length: 9,
				Hashes: []string{"sha256"},
			},
			"file22.txt": {
				Custom: map[string]string{
					"name":       "file22.txt",
					"permission": "0644",
				},
				Length: 10,
				Hashes: []string{"sha256"},
			},
		}, nil},
	{"fetcherror", []string{""}, nil, errors.New("Could not fetch file")},
	{"attrerror", []string{"AttrError"}, nil, errors.New("Could not get attributes")},
	{"hasherror", []string{"HashError"}, nil, errors.New("hash error")},
}

func TestPopulateTargets(t *testing.T) {
	for _, tc := range populateTargetsTC {
		t.Run(tc.name, func(t *testing.T) {
			tf := TestTargetMetadataFetcher{}
			actual, err := populateTargets(&tf, tc.targets)
			if !reflect.DeepEqual(tc.err, err) {
				t.Fatalf("Expected Error:\n%v\n Got :\n%v", tc.err, err)
			}
			if tc.err == nil {
				if !reflect.DeepEqual(tc.expectedRes, actual) {
					t.Fatalf("Expected:\n%v\n Got :\n%v", tc.expectedRes, actual)
				}
			}
		})
	}
}

var metaTC = []struct {
	name        string
	version     int
	expectedRes map[string]MetaVersionInfo
}{
	{"one", 1, map[string]MetaVersionInfo{
		"root1.json":   {Version: 1},
		"target1.json": {Version: 1},
	}},
	{"two", 2, map[string]MetaVersionInfo{
		"root2.json":   {Version: 2},
		"target2.json": {Version: 2},
	}},
}

func TestMeta(t *testing.T) {
	for _, tc := range metaTC {
		t.Run(tc.name, func(t *testing.T) {
			actual := populateMeta(tc.version)
			if !reflect.DeepEqual(tc.expectedRes, actual) {
				t.Fatalf("Expected:\n%v\n Got :\n%v", tc.expectedRes, actual)
			}
		})
	}
}
<commit_msg>add the code review comment back<commit_after>/*
Copyright 2018 Google Inc. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1

import (
	"errors"
	"reflect"
	"testing"

	"github.com/GoogleCloudPlatform/runtimes-common/tuf/metadata"
	"github.com/GoogleCloudPlatform/runtimes-common/tuf/testutil"
	"github.com/GoogleCloudPlatform/runtimes-common/tuf/types"
)

func TestRootMetadata(t *testing.T) {
	tuf := TUF{
		TufConfig:       testutil.TestTUFConfig,
		OldRootSecrets:  []types.Scheme{},
		RootSecrets:     []types.Scheme{},
		TargetSecrets:   []types.Scheme{},
		SnapshotSecrets: []types.Scheme{},
	}
	rootMetadata, err := tuf.PopulateRootMetadata()
	if err != nil {
		t.Fatalf("Did not expect error %v", err)
	}
	expectedBaseSigned := &metadata.BaseSigned{
		Type:        "root",
		SpecVersion: 1,
		Version:     1,
	}
	// TODO (tejaldesai): Fix this by reading time form config.
	expectedBaseSigned.Expires = rootMetadata.Signed.BaseSigned.Expires
	if !reflect.DeepEqual(rootMetadata.Signed.BaseSigned, expectedBaseSigned) {
		t.Fatalf("Expected %v\n Got %v", rootMetadata.Signed.BaseSigned, expectedBaseSigned)
	}
}

var tuf = TUF{
	TufConfig: testutil.TestTUFConfig,
	RootSecrets: []types.Scheme{
		&testutil.TestKey{
			PrivateKey: "secret1",
			KeyType:    "test-scheme1",
			PublicKey:  "public1",
			KeyId:      "idroot1",
			SignStr:    "secret1",
		},
		&testutil.TestKey{
			PrivateKey: "secret2",
			KeyType:    "test-scheme2",
			PublicKey:  "public2",
			KeyId:      "idroot2",
			SignStr:    "secret2",
		},
	},
	TargetSecrets: []types.Scheme{
		&testutil.TestKey{
			PrivateKey: "target",
			KeyType:    "test-scheme1",
			PublicKey:  "targetpublic",
			KeyId:      "idtarget",
			SignStr:    "targetsecret",
		},
	},
	SnapshotSecrets: []types.Scheme{
		&testutil.TestKey{
			PrivateKey: "snapshot",
			KeyType:    "test-scheme1",
			PublicKey:  "snapshotpublic",
			KeyId:      "idsnapshot",
			SignStr:    "snapshotsecret",
		},
	},
}

func TestPopulateRoles(t *testing.T) {
	expectedResult := map[types.RoleType]Role{
		"root": {
			Keyids: []types.KeyId{
				types.KeyId("idroot1"),
				types.KeyId("idroot2"),
			},
			Threshold: 1,
		},
		"target": {
			Keyids: []types.KeyId{
				types.KeyId("idtarget"),
			},
			Threshold: 1,
		},
		"snapshot": {
			Keyids: []types.KeyId{
				types.KeyId("idsnapshot"),
			},
			Threshold: 1,
		},
	}
	actual := tuf.populateRoles()
	if !reflect.DeepEqual(expectedResult, actual) {
		t.Fatalf("Expected %v\n Got : %v", expectedResult, actual)
	}
}

func TestPopulateKeys(t *testing.T) {
	expectedResult := map[types.KeyId]KeysDef{
		types.KeyId("idroot1"): {
			KeyidHashAlgorithms: []types.HashAlgo{types.HashAlgo("sha256")},
			Val: map[string]string{
				"public": "public1",
			},
			Keytype: types.KeyType("test-scheme1"),
			Scheme:  types.KeyScheme("test-scheme1"),
		},
		types.KeyId("idroot2"): {
			KeyidHashAlgorithms: []types.HashAlgo{types.HashAlgo("sha256")},
			Val: map[string]string{
				"public": "public2",
			},
			Keytype: types.KeyType("test-scheme2"),
			Scheme:  types.KeyScheme("test-scheme2"),
		},
	}
	actual := populateKeys(tuf.RootSecrets)
	if !reflect.DeepEqual(expectedResult, actual) {
		t.Fatalf("Expected:\n%v\n Got :\n%v", expectedResult, actual)
	}
}

var signTC = []struct {
	name         string
	signMetadata interface{}
	secrets      string
	expectedErr  error
	expectedRes  []metadata.Signature
}{
	{"signRootSuccess", "str", "root", nil,
		[]metadata.Signature{
			{
				KeyId: types.KeyId("idroot1"),
				Sig:   "secret1str",
			},
			{
				KeyId: types.KeyId("idroot2"),
				Sig:   "secret2str",
			},
		},
	},
	{"signTargetSuccess", "str", "target", nil,
		[]metadata.Signature{
			{
				KeyId: types.KeyId("idtarget"),
				Sig:   "targetsecretstr",
			},
		},
	},
	{"signSnapshotSuccess", "str", "snapshot", nil,
		[]metadata.Signature{
			{
				KeyId: types.KeyId("idsnapshot"),
				Sig:   "snapshotsecretstr",
			},
		},
	},
	{"singErr", 1, "snapshot", errors.New("Signing Error"), nil},
}

func TestSign(t *testing.T) {
	for _, tc := range signTC {
		t.Run(tc.name, func(t *testing.T) {
			var actual []metadata.Signature
			var err error
			switch tc.secrets {
			case "root":
				actual, err = tuf.SignRootMetadata(tc.signMetadata)
			case "target":
				actual, err = tuf.SignTargetMetadata(tc.signMetadata)
			default:
				actual, err = tuf.SignSnapshotMetadata(tc.signMetadata)

			}
			if !reflect.DeepEqual(tc.expectedErr, err) {
				t.Fatalf("Expected Error:\n%v\n Got :\n%v", tc.expectedErr, err)
			}
			if !reflect.DeepEqual(tc.expectedRes, actual) {
				t.Fatalf("Expected:\n%v\n Got :\n%v", tc.expectedRes, actual)
			}
		})
	}
}

var populateTargetsTC = []struct {
	name        string
	targets     []string
	expectedRes map[string]Target
	err         error
}{
	{"success", []string{"file1.txt", "file22.txt"},
		map[string]Target{
			"file1.txt": {
				Custom: map[string]string{
					"name":       "file1.txt",
					"permission": "0644",
				},
				Length: 9,
				Hashes: []string{"sha256"},
			},
			"file22.txt": {
				Custom: map[string]string{
					"name":       "file22.txt",
					"permission": "0644",
				},
				Length: 10,
				Hashes: []string{"sha256"},
			},
		}, nil},
	{"fetcherror", []string{""}, nil, errors.New("Could not fetch file")},
	{"attrerror", []string{"AttrError"}, nil, errors.New("Could not get attributes")},
	{"hasherror", []string{"HashError"}, nil, errors.New("hash error")},
}

func TestPopulateTargets(t *testing.T) {
	for _, tc := range populateTargetsTC {
		t.Run(tc.name, func(t *testing.T) {
			tf := TestTargetMetadataFetcher{}
			actual, err := populateTargets(&tf, tc.targets)
			if !reflect.DeepEqual(tc.err, err) {
				t.Fatalf("Expected Error:\n%v\n Got :\n%v", tc.err, err)
			}
			if tc.err == nil {
				if !reflect.DeepEqual(tc.expectedRes, actual) {
					t.Fatalf("Expected:\n%v\n Got :\n%v", tc.expectedRes, actual)
				}
			}
		})
	}
}

var metaTC = []struct {
	name        string
	version     int
	expectedRes map[string]MetaVersionInfo
}{
	{"one", 1, map[string]MetaVersionInfo{
		"root1.json":   {Version: 1},
		"target1.json": {Version: 1},
	}},
	{"two", 2, map[string]MetaVersionInfo{
		"root2.json":   {Version: 2},
		"target2.json": {Version: 2},
	}},
}

func TestMeta(t *testing.T) {
	for _, tc := range metaTC {
		t.Run(tc.name, func(t *testing.T) {
			actual := populateMeta(tc.version)
			if !reflect.DeepEqual(tc.expectedRes, actual) {
				t.Fatalf("Expected:\n%v\n Got :\n%v", tc.expectedRes, actual)
			}
		})
	}
}
<|endoftext|>


<commit_before>package multibar

import (
	"fmt"
	"reflect"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/sethgrid/curse"
)

type progressFunc func(progress int)

type BarContainer struct {
	Bars []*ProgressBar

	screenLines   int
	screenWidth   int
	startingLine  int
	totalNewlines int

	history map[int]string
	sync.Mutex
}

type ProgressBar struct {
	Width           int
	Total           int
	LeftEnd         byte
	RightEnd        byte
	Fill            byte
	Head            byte
	Empty           byte
	ShowPercent     bool
	ShowTimeElapsed bool
	StartTime       time.Time
	Line            int
	Prepend         string

	progressChan chan int
}

func New() (*BarContainer, error) {
	// can swallow err because sensible defaults are returned from curse
	width, lines, _ := curse.GetScreenDimensions()
	_, line, _ := curse.GetCursorPosition()

	history := make(map[int]string)

	b := &BarContainer{screenWidth: width, screenLines: lines, startingLine: line, history: history}
	// todo: need to figure out a way to deal with additional progressbars while the listener
	// is listening. for the time being, the calling app will have to call listen after
	// all bars are declared
	//go b.Listen()
	return b, nil
}

func (b *BarContainer) Listen() {
	for len(b.Bars) == 0 {
		// wait until we have some bars to work with
		time.Sleep(time.Millisecond * 100)
	}
	cases := make([]reflect.SelectCase, len(b.Bars))
	for i, bar := range b.Bars {
		cases[i] = reflect.SelectCase{Dir: reflect.SelectRecv, Chan: reflect.ValueOf(bar.progressChan)}
	}

	remaining := len(cases)
	for remaining > 0 {
		chosen, value, ok := reflect.Select(cases)
		if !ok {
			// The chosen channel has been closed, so zero out the channel to disable the case
			cases[chosen].Chan = reflect.ValueOf(nil)
			remaining -= 1
			continue
		}

		b.Bars[chosen].Update(int(value.Int()))
	}
	b.Println()
}

func (b *BarContainer) MakeBar(total int, prepend string) progressFunc {
	ch := make(chan int)
	bar := &ProgressBar{
		Width:           b.screenWidth - len(prepend) - 20,
		Total:           total,
		Prepend:         prepend,
		LeftEnd:         '[',
		RightEnd:        ']',
		Fill:            '=',
		Head:            '>',
		Empty:           '-',
		ShowPercent:     true,
		ShowTimeElapsed: true,
		StartTime:       time.Now(),
		progressChan:    ch,
	}

	b.Bars = append(b.Bars, bar)
	bar.Line = b.startingLine + b.totalNewlines
	b.history[bar.Line] = ""
	bar.Update(0)
	b.Println()

	return func(progress int) { bar.progressChan <- progress }
}

func (p *ProgressBar) AddPrepend(str string) {
	p.Prepend = str
}

func (p *ProgressBar) Update(progress int) {
	bar := make([]string, p.Width)

	// avoid division by zero errors on non-properly constructed progressbars
	if p.Width == 0 {
		p.Width = 1
	}
	if p.Total == 0 {
		p.Total = 1
	}
	justGotToFirstEmptySpace := true
	for i, _ := range bar {
		if float32(progress)/float32(p.Total) > float32(i)/float32(p.Width) {
			bar[i] = string(p.Fill)
		} else {
			bar[i] = string(p.Empty)
			if justGotToFirstEmptySpace {
				bar[i] = string(p.Head)
				justGotToFirstEmptySpace = false
			}
		}
	}

	percent := ""
	if p.ShowPercent {
		asInt := int(100 * (float32(progress) / float32(p.Total)))
		padding := ""
		if asInt < 10 {
			padding = "  "
		} else if asInt < 99 {
			padding = " "
		}
		percent = padding + strconv.Itoa(asInt) + "% "
	}

	timeElapsed := ""
	if p.ShowTimeElapsed {
		timeElapsed = " " + prettyTime(time.Since(p.StartTime))
	}

	// record where we are, jump to the progress bar, update it, jump back
	c, _ := curse.New()
	c.Move(1, p.Line)
	c.EraseCurrentLine()
	fmt.Printf("\r%s %s%c%s%c%s", p.Prepend, percent, p.LeftEnd, strings.Join(bar, ""), p.RightEnd, timeElapsed)
	c.Move(c.StartingPosition.X, c.StartingPosition.Y)
}

func prettyTime(t time.Duration) string {
	re, err := regexp.Compile(`(\d+).(\d+)(\w+)`)
	if err != nil {
		return err.Error()
	}
	parts := re.FindSubmatch([]byte(t.String()))
	if len(parts) != 4 {
		return "---"
	}
	return string(parts[1]) + string(parts[3])
}

func (b *BarContainer) addedNewlines(count int) {
	b.totalNewlines += count

	// if we hit the bottom of the screen, we "scroll" our bar displays by pushing
	// them up count lines (closer to line 0)
	if b.startingLine+b.totalNewlines > b.screenLines {
		b.totalNewlines -= count
		for _, bar := range b.Bars {
			bar.Line -= count
		}
		b.redrawAll(count)
	}
}

func (b *BarContainer) redrawAll(moveUp int) {
	c, _ := curse.New()

	newHistory := make(map[int]string)
	for line, printed := range b.history {
		newHistory[line+moveUp] = printed
		c.Move(1, line)
		c.EraseCurrentLine()
		c.Move(1, line+moveUp)
		c.EraseCurrentLine()
		fmt.Print(printed)
	}
	b.history = newHistory
	c.Move(c.StartingPosition.X, c.StartingPosition.Y)
}

// print wrappers to capture newlines to adjust line positions on bars

func (b *BarContainer) Print(a ...interface{}) (n int, err error) {
	b.Lock()
	defer b.Unlock()
	newlines := countAllNewlines(a...)
	b.addedNewlines(newlines)
	thisLine := b.startingLine + b.totalNewlines
	b.history[thisLine] = fmt.Sprint(a...)
	return fmt.Print(a...)
}

func (b *BarContainer) Printf(format string, a ...interface{}) (n int, err error) {
	b.Lock()
	defer b.Unlock()
	newlines := strings.Count(format, "\n")
	newlines += countAllNewlines(a...)
	b.addedNewlines(newlines)
	thisLine := b.startingLine + b.totalNewlines
	b.history[thisLine] = fmt.Sprintf(format, a...)
	return fmt.Printf(format, a...)
}

func (b *BarContainer) Println(a ...interface{}) (n int, err error) {
	b.Lock()
	defer b.Unlock()
	newlines := countAllNewlines(a...) + 1
	b.addedNewlines(newlines)
	thisLine := b.startingLine + b.totalNewlines
	b.history[thisLine] = fmt.Sprint(a...)
	return fmt.Println(a...)
}

func countAllNewlines(interfaces ...interface{}) int {
	count := 0
	for _, iface := range interfaces {
		switch s := iface.(type) {
		case string:
			count += strings.Count(s, "\n")
		}
	}
	return count
}
<commit_msg>Export progressFunc type<commit_after>package multibar

import (
	"fmt"
	"reflect"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/sethgrid/curse"
)

type ProgressFunc func(progress int)

type BarContainer struct {
	Bars []*ProgressBar

	screenLines   int
	screenWidth   int
	startingLine  int
	totalNewlines int

	history map[int]string
	sync.Mutex
}

type ProgressBar struct {
	Width           int
	Total           int
	LeftEnd         byte
	RightEnd        byte
	Fill            byte
	Head            byte
	Empty           byte
	ShowPercent     bool
	ShowTimeElapsed bool
	StartTime       time.Time
	Line            int
	Prepend         string

	progressChan chan int
}

func New() (*BarContainer, error) {
	// can swallow err because sensible defaults are returned from curse
	width, lines, _ := curse.GetScreenDimensions()
	_, line, _ := curse.GetCursorPosition()

	history := make(map[int]string)

	b := &BarContainer{screenWidth: width, screenLines: lines, startingLine: line, history: history}
	// todo: need to figure out a way to deal with additional progressbars while the listener
	// is listening. for the time being, the calling app will have to call listen after
	// all bars are declared
	//go b.Listen()
	return b, nil
}

func (b *BarContainer) Listen() {
	for len(b.Bars) == 0 {
		// wait until we have some bars to work with
		time.Sleep(time.Millisecond * 100)
	}
	cases := make([]reflect.SelectCase, len(b.Bars))
	for i, bar := range b.Bars {
		cases[i] = reflect.SelectCase{Dir: reflect.SelectRecv, Chan: reflect.ValueOf(bar.progressChan)}
	}

	remaining := len(cases)
	for remaining > 0 {
		chosen, value, ok := reflect.Select(cases)
		if !ok {
			// The chosen channel has been closed, so zero out the channel to disable the case
			cases[chosen].Chan = reflect.ValueOf(nil)
			remaining -= 1
			continue
		}

		b.Bars[chosen].Update(int(value.Int()))
	}
	b.Println()
}

func (b *BarContainer) MakeBar(total int, prepend string) ProgressFunc {
	ch := make(chan int)
	bar := &ProgressBar{
		Width:           b.screenWidth - len(prepend) - 20,
		Total:           total,
		Prepend:         prepend,
		LeftEnd:         '[',
		RightEnd:        ']',
		Fill:            '=',
		Head:            '>',
		Empty:           '-',
		ShowPercent:     true,
		ShowTimeElapsed: true,
		StartTime:       time.Now(),
		progressChan:    ch,
	}

	b.Bars = append(b.Bars, bar)
	bar.Line = b.startingLine + b.totalNewlines
	b.history[bar.Line] = ""
	bar.Update(0)
	b.Println()

	return func(progress int) { bar.progressChan <- progress }
}

func (p *ProgressBar) AddPrepend(str string) {
	p.Prepend = str
}

func (p *ProgressBar) Update(progress int) {
	bar := make([]string, p.Width)

	// avoid division by zero errors on non-properly constructed progressbars
	if p.Width == 0 {
		p.Width = 1
	}
	if p.Total == 0 {
		p.Total = 1
	}
	justGotToFirstEmptySpace := true
	for i, _ := range bar {
		if float32(progress)/float32(p.Total) > float32(i)/float32(p.Width) {
			bar[i] = string(p.Fill)
		} else {
			bar[i] = string(p.Empty)
			if justGotToFirstEmptySpace {
				bar[i] = string(p.Head)
				justGotToFirstEmptySpace = false
			}
		}
	}

	percent := ""
	if p.ShowPercent {
		asInt := int(100 * (float32(progress) / float32(p.Total)))
		padding := ""
		if asInt < 10 {
			padding = "  "
		} else if asInt < 99 {
			padding = " "
		}
		percent = padding + strconv.Itoa(asInt) + "% "
	}

	timeElapsed := ""
	if p.ShowTimeElapsed {
		timeElapsed = " " + prettyTime(time.Since(p.StartTime))
	}

	// record where we are, jump to the progress bar, update it, jump back
	c, _ := curse.New()
	c.Move(1, p.Line)
	c.EraseCurrentLine()
	fmt.Printf("\r%s %s%c%s%c%s", p.Prepend, percent, p.LeftEnd, strings.Join(bar, ""), p.RightEnd, timeElapsed)
	c.Move(c.StartingPosition.X, c.StartingPosition.Y)
}

func prettyTime(t time.Duration) string {
	re, err := regexp.Compile(`(\d+).(\d+)(\w+)`)
	if err != nil {
		return err.Error()
	}
	parts := re.FindSubmatch([]byte(t.String()))
	if len(parts) != 4 {
		return "---"
	}
	return string(parts[1]) + string(parts[3])
}

func (b *BarContainer) addedNewlines(count int) {
	b.totalNewlines += count

	// if we hit the bottom of the screen, we "scroll" our bar displays by pushing
	// them up count lines (closer to line 0)
	if b.startingLine+b.totalNewlines > b.screenLines {
		b.totalNewlines -= count
		for _, bar := range b.Bars {
			bar.Line -= count
		}
		b.redrawAll(count)
	}
}

func (b *BarContainer) redrawAll(moveUp int) {
	c, _ := curse.New()

	newHistory := make(map[int]string)
	for line, printed := range b.history {
		newHistory[line+moveUp] = printed
		c.Move(1, line)
		c.EraseCurrentLine()
		c.Move(1, line+moveUp)
		c.EraseCurrentLine()
		fmt.Print(printed)
	}
	b.history = newHistory
	c.Move(c.StartingPosition.X, c.StartingPosition.Y)
}

// print wrappers to capture newlines to adjust line positions on bars

func (b *BarContainer) Print(a ...interface{}) (n int, err error) {
	b.Lock()
	defer b.Unlock()
	newlines := countAllNewlines(a...)
	b.addedNewlines(newlines)
	thisLine := b.startingLine + b.totalNewlines
	b.history[thisLine] = fmt.Sprint(a...)
	return fmt.Print(a...)
}

func (b *BarContainer) Printf(format string, a ...interface{}) (n int, err error) {
	b.Lock()
	defer b.Unlock()
	newlines := strings.Count(format, "\n")
	newlines += countAllNewlines(a...)
	b.addedNewlines(newlines)
	thisLine := b.startingLine + b.totalNewlines
	b.history[thisLine] = fmt.Sprintf(format, a...)
	return fmt.Printf(format, a...)
}

func (b *BarContainer) Println(a ...interface{}) (n int, err error) {
	b.Lock()
	defer b.Unlock()
	newlines := countAllNewlines(a...) + 1
	b.addedNewlines(newlines)
	thisLine := b.startingLine + b.totalNewlines
	b.history[thisLine] = fmt.Sprint(a...)
	return fmt.Println(a...)
}

func countAllNewlines(interfaces ...interface{}) int {
	count := 0
	for _, iface := range interfaces {
		switch s := iface.(type) {
		case string:
			count += strings.Count(s, "\n")
		}
	}
	return count
}
<|endoftext|>


<commit_before>package editor

import (
	"image"
	"time"

	"sigint.ca/graphics/editor/internal/text"

	"golang.org/x/mobile/event/mouse"
)

const dClickPause = 500 * time.Millisecond

func (ed *Editor) handleMouseEvent(e mouse.Event) {
	if e.Direction == mouse.DirRelease {
		return
	}

	pos := image.Pt(int(e.X), int(e.Y)).Add(ed.visible().Min) // adjust for scrolling
	a := ed.getAddress(pos)

	ed.commitTransformation()

	if e.Direction == mouse.DirPress {
		ed.dirty = true
		if e.Button == mouse.ButtonLeft {
			ed.sweepOrigin = a
			ed.click(a, e.Button)
		} else if e.Button != mouse.ButtonNone {
			ed.dot = ed.buf.SelWord(a)
		}
	} else if e.Direction == mouse.DirNone {
		vis := ed.visible()
		if a == ed.sweepLast && pos.In(vis) {
			return
		}
		if pos.Y <= vis.Min.Y && vis.Min.Y > 0 {
			ed.scroll(image.Pt(0, ed.font.height))
		} else if pos.Y >= vis.Max.Y && vis.Max.Y < (len(ed.buf.Lines)-1)*ed.font.height {
			ed.scroll(image.Pt(0, -ed.font.height))
		}

		ed.dirty = true
		ed.sweepLast = a

		ed.sweep(a)
	}
}

func (ed *Editor) click(a text.Address, button mouse.Button) {
	switch button {
	case mouse.ButtonLeft:
		prev := ed.dot
		ed.dot.From, ed.dot.To = a, a

		if time.Since(ed.lastClickTime) < dClickPause && ed.dot == prev {
			// double click
			ed.dot = ed.buf.AutoSelect(a)
			ed.lastClickTime = time.Time{}
		} else {
			ed.lastClickTime = time.Now()
		}
	}
}

func (ed *Editor) sweep(to text.Address) {
	if to.LessThan(ed.sweepOrigin) {
		ed.dot = text.Selection{to, ed.sweepOrigin}
	} else if to != ed.sweepOrigin {
		ed.dot = text.Selection{ed.sweepOrigin, to}
	} else {
		ed.dot = text.Selection{ed.sweepOrigin, ed.sweepOrigin}
	}
}
<commit_msg>editor: fix mouse B2 and B3 sweeping<commit_after>package editor

import (
	"image"
	"time"

	"sigint.ca/graphics/editor/internal/text"

	"golang.org/x/mobile/event/mouse"
)

const dClickPause = 500 * time.Millisecond

func (ed *Editor) handleMouseEvent(e mouse.Event) {
	if e.Direction == mouse.DirRelease {
		return
	}

	pos := image.Pt(int(e.X), int(e.Y)).Add(ed.visible().Min) // adjust for scrolling
	a := ed.getAddress(pos)

	ed.commitTransformation()

	if e.Direction == mouse.DirPress {
		ed.dirty = true
		ed.sweepOrigin = a
		ed.click(a, e.Button)
	} else if e.Direction == mouse.DirNone {
		vis := ed.visible()
		if a == ed.sweepLast && pos.In(vis) {
			return
		}
		if pos.Y <= vis.Min.Y && vis.Min.Y > 0 {
			ed.scroll(image.Pt(0, ed.font.height))
		} else if pos.Y >= vis.Max.Y && vis.Max.Y < (len(ed.buf.Lines)-1)*ed.font.height {
			ed.scroll(image.Pt(0, -ed.font.height))
		}

		ed.dirty = true
		ed.sweepLast = a

		ed.sweep(a)
	}
}

func (ed *Editor) click(a text.Address, button mouse.Button) {
	switch button {
	case mouse.ButtonLeft:
		prev := ed.dot
		ed.dot.From, ed.dot.To = a, a

		if time.Since(ed.lastClickTime) < dClickPause && ed.dot == prev {
			// double click
			ed.dot = ed.buf.AutoSelect(a)
			ed.lastClickTime = time.Time{}
		} else {
			ed.lastClickTime = time.Now()
		}
	case mouse.ButtonMiddle, mouse.ButtonRight:
		ed.dot = ed.buf.SelWord(a)
	}
}

func (ed *Editor) sweep(to text.Address) {
	if to.LessThan(ed.sweepOrigin) {
		ed.dot = text.Selection{to, ed.sweepOrigin}
	} else if to != ed.sweepOrigin {
		ed.dot = text.Selection{ed.sweepOrigin, to}
	} else {
		ed.dot = text.Selection{ed.sweepOrigin, ed.sweepOrigin}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"strconv"
	"strings"

	"github.com/ajm188/advent_of_code/pkg/cli"
)

// offsetSumsWhere returns the count of offset sums in xs meeting the condition
// function.
//
// offset sums for a slice are a partition of xs into two slices.
// LHS: [sum(xs[0:n]), sum(xs[1:n+1]), ..., sum(xs[len(xs)-n-1:len(xs)-1])]
// RHS: [sum(xs[1:n]), sum(xs[2:n+2]), ..., sum(xs[len(xs)-n:])]
func offsetSumsWhere(xs []int, n int, f func(l, r int) bool) (count int) {
	l := make([]int, len(xs)-n)
	r := make([]int, len(xs)-n)

	for i, j := 0, 1; i < len(xs)-n && j < len(xs); i, j = i+1, j+1 {
		for k := 0; k < n; k++ {
			l[i] += xs[i+k]
			r[i] += xs[j+k]
		}
	}

	for i := 0; i < len(l); i++ {
		if f(l[i], r[i]) {
			count++
		}
	}

	return count
}

func main() {
	path := flag.String("path", "input.txt", "")
	flag.Parse()

	data, err := cli.GetInput(*path)
	cli.ExitOnError(err)

	lines := strings.Split(string(data), "\n")
	depths := make([]int, 0, len(lines)-1)

	for i, line := range lines {
		if line == "" {
			continue
		}

		val, err := strconv.ParseInt(line, 10, 64)
		if err != nil {
			cli.ExitOnError(fmt.Errorf("input line %d: %w", i, err))
		}

		depths = append(depths, int(val))
	}

	for _, n := range []int{1, 3} {
		fmt.Println(offsetSumsWhere(depths, n, func(l, r int) bool {
			return l < r
		}))
	}
}
<commit_msg>even more configurable<commit_after>package main

import (
	"flag"
	"fmt"
	"strconv"
	"strings"

	"github.com/ajm188/advent_of_code/pkg/cli"
)

// offsetSumsWhere returns the count of offset sums in xs meeting the condition
// function.
//
// offset sums for a slice are a partition of xs into two slices.
// LHS: [sum(xs[0:n]), sum(xs[1:n+1]), ..., sum(xs[len(xs)-n-1:len(xs)-1])]
// RHS: [sum(xs[1:n]), sum(xs[2:n+2]), ..., sum(xs[len(xs)-n:])]
func offsetSumsWhere(xs []int, n int, f func(l, r int) bool) (count int) {
	l := make([]int, len(xs)-n)
	r := make([]int, len(xs)-n)

	for i, j := 0, 1; i < len(xs)-n && j < len(xs); i, j = i+1, j+1 {
		for k := 0; k < n; k++ {
			l[i] += xs[i+k]
			r[i] += xs[j+k]
		}
	}

	for i := 0; i < len(l); i++ {
		if f(l[i], r[i]) {
			count++
		}
	}

	return count
}

func main() {
	path := flag.String("path", "input.txt", "")
	offsetCSV := flag.String("offsets", "1,3", "csv of integer offsets to sum")
	flag.Parse()

	data, err := cli.GetInput(*path)
	cli.ExitOnError(err)

	lines := strings.Split(string(data), "\n")
	depths := make([]int, 0, len(lines)-1)

	for i, line := range lines {
		if line == "" {
			continue
		}

		val, err := strconv.ParseInt(line, 10, 64)
		if err != nil {
			cli.ExitOnError(fmt.Errorf("input line %d: %w", i, err))
		}

		depths = append(depths, int(val))
	}

	var offsets []int
	for _, offsetStr := range strings.Split(*offsetCSV, ",") {
		offset, err := strconv.ParseInt(offsetStr, 10, 64)
		if err != nil {
			cli.ExitOnError(fmt.Errorf("could not parse offset csv (%s): %w", *offsetCSV, err))
		}

		offsets = append(offsets, int(offset))
	}

	for _, n := range offsets {
		fmt.Printf("n=%d\t%d\n", n, offsetSumsWhere(depths, n, func(l, r int) bool {
			return l < r
		}))
	}
}
<|endoftext|>


<commit_before>package gremgo

import (
	"log"
	"net/http"

	"github.com/gorilla/websocket"
)

type dialer interface {
	connect() error
	write([]byte) error
	read() ([]byte, error)
}

/////
/*
WebSocket Connection
*/
/////

// Ws is the dialer for a WebSocket connection
type Ws struct {
	host string
	conn *websocket.Conn
}

func (ws *Ws) connect() (err error) {
	d := websocket.Dialer{
		WriteBufferSize: 8192,
		ReadBufferSize:  8192,
	}
	ws.conn, _, err = d.Dial(ws.host, http.Header{})
	return
}

func (ws *Ws) write(msg []byte) (err error) {
	err = ws.conn.WriteMessage(2, msg)
	return
}

func (ws *Ws) read() (msg []byte, err error) {
	_, msg, err = ws.conn.ReadMessage()
	return
}

/////

func (c *Client) writeWorker() { // writeWorker works on a loop and dispatches messages as soon as it recieves them
	for {
		select {
		case msg := <-c.requests: // Wait for message send request
			err := c.conn.write(msg) // Write message
			if err != nil {          // TODO: Fix error handling here
				log.Fatal(err)
			}
		default:
		}
	}
}

func (c *Client) readWorker() { // readWorker works on a loop and sorts messages as soon as it recieves them
	for {
		msg, err := c.conn.read()
		if err != nil {
			log.Fatal(err)
		}
		if msg != nil {
			// TODO: Make this multithreaded
			c.handleResponse(msg) // Send message for sorting and retrieval on a separate thread
		}
	}
}
<commit_msg>Fixed 100 percent CPU utilization with write worker<commit_after>package gremgo

import (
	"log"
	"net/http"

	"github.com/gorilla/websocket"
)

type dialer interface {
	connect() error
	write([]byte) error
	read() ([]byte, error)
}

/////
/*
WebSocket Connection
*/
/////

// Ws is the dialer for a WebSocket connection
type Ws struct {
	host string
	conn *websocket.Conn
}

func (ws *Ws) connect() (err error) {
	d := websocket.Dialer{
		WriteBufferSize: 8192,
		ReadBufferSize:  8192,
	}
	ws.conn, _, err = d.Dial(ws.host, http.Header{})
	return
}

func (ws *Ws) write(msg []byte) (err error) {
	err = ws.conn.WriteMessage(2, msg)
	return
}

func (ws *Ws) read() (msg []byte, err error) {
	_, msg, err = ws.conn.ReadMessage()
	return
}

/////

func (c *Client) writeWorker() { // writeWorker works on a loop and dispatches messages as soon as it recieves them
	for {
		select {
		case msg := <-c.requests: // Wait for message send request
			err := c.conn.write(msg) // Write message
			if err != nil {          // TODO: Fix error handling here
				log.Fatal(err)
			}
		}
	}
}

func (c *Client) readWorker() { // readWorker works on a loop and sorts messages as soon as it recieves them
	for {
		msg, err := c.conn.read()
		if err != nil {
			log.Fatal(err)
		}
		if msg != nil {
			// TODO: Make this multithreaded
			c.handleResponse(msg) // Send message for sorting and retrieval on a separate thread
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"crypto/hmac"
	"crypto/sha1"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/Comcast/webpa-common/logging"
	"github.com/Comcast/webpa-common/webhook"
	"hash"
	"net/http"
	"net/url"
	"regexp"
	"sync"
	"time"
)

// failureText is human readable text for the failure message
const failureText = `Unfortunately, your endpoint is not able to keep up with the ` +
	`traffic being sent to it.  Due to this circumstance, all notification traffic ` +
	`is being cut off and dropped for a period of time.  Please increase your ` +
	`capacity to handle notifications, or reduce the number of notifications ` +
	`you have requested.`

// outboundRequest stores the outgoing request and assorted data that has been
// collected so far (to save on processing the information again).
type outboundRequest struct {
	req         CaduceusRequest
	event       string
	transID     string
	deviceID    string
	contentType string
}

// FailureMessage is a helper that lets us easily create a json struct to send
// when we have to cut and endpoint off.
type FailureMessage struct {
	Text         string    `json:"text"`
	Original     webhook.W `json:"webhook_registration"`
	CutOffPeriod string    `json:"cut_off_period"`
	QueueSize    int       `json:"queue_size"`
	Workers      int       `json:"worker_count"`
}

// OutboundSenderFactory is a configurable factory for OutboundSender objects.
type OutboundSenderFactory struct {
	// The WebHookListener to service
	Listener webhook.W

	// The http client to use for requests.
	Client *http.Client

	// The number of delivery workers to create and use.
	NumWorkers int

	// The queue depth to buffer events before we declare overflow, shut
	// off the message delivery, and basically put the endpoint in "timeout."
	QueueSize int

	// The amount of time to cut off the consumer if they don't keep up.
	// Must be greater then 0 seconds
	CutOffPeriod time.Duration

	// The factory that we'll use to make new ServerProfilers on a per
	// outboundSender basis
	ProfilerFactory ServerProfilerFactory

	// The logger to use.
	Logger logging.Logger
}

type OutboundSender interface {
	Extend(time.Time)
	Shutdown(bool)
	RetiredSince() time.Time
	QueueJSON(CaduceusRequest, string, string, string)
	QueueWrp(CaduceusRequest, map[string]string, string, string, string)
}

// CaduceusOutboundSender is the outbound sender object.
type CaduceusOutboundSender struct {
	listener     webhook.W
	deliverUntil time.Time
	dropUntil    time.Time
	client       *http.Client
	secret       []byte
	events       []*regexp.Regexp
	matcher      []*regexp.Regexp
	queueSize    int
	queue        chan outboundRequest
	profiler     ServerProfiler
	wg           sync.WaitGroup
	cutOffPeriod time.Duration
	failureMsg   FailureMessage
	logger       logging.Logger
	mutex        sync.RWMutex
}

// New creates a new OutboundSender object from the factory, or returns an error.
func (osf OutboundSenderFactory) New() (obs OutboundSender, err error) {
	if _, err = url.ParseRequestURI(osf.Listener.Config.URL); nil != err {
		return
	}

	if nil == osf.Client {
		err = errors.New("nil http.Client")
		return
	}

	if 0 == osf.CutOffPeriod.Nanoseconds() {
		err = errors.New("Invalid CutOffPeriod")
		return
	}

	if nil == osf.Logger {
		err = errors.New("Logger required")
		return
	}

	caduceusOutboundSender := &CaduceusOutboundSender{
		listener:     osf.Listener,
		client:       osf.Client,
		queueSize:    osf.QueueSize,
		cutOffPeriod: osf.CutOffPeriod,
		deliverUntil: osf.Listener.Until,
		logger:       osf.Logger,
		failureMsg: FailureMessage{
			Original:     osf.Listener,
			Text:         failureText,
			CutOffPeriod: osf.CutOffPeriod.String(),
			QueueSize:    osf.QueueSize,
			Workers:      osf.NumWorkers,
		},
	}

	if "" != osf.Listener.Config.Secret {
		caduceusOutboundSender.secret = []byte(osf.Listener.Config.Secret)
	}

	if "" != osf.Listener.FailureURL {
		if _, err = url.ParseRequestURI(osf.Listener.FailureURL); nil != err {
			return
		}
	}

	caduceusOutboundSender.profiler, err = osf.ProfilerFactory.New()
	if err != nil {
		return
	}

	// Give us some head room so that we don't block when we get near the
	// completely full point.
	caduceusOutboundSender.queue = make(chan outboundRequest, osf.QueueSize)

	// Create the event regex objects
	for _, event := range osf.Listener.Events {
		var re *regexp.Regexp
		if re, err = regexp.Compile(event); nil != err {
			return
		}

		caduceusOutboundSender.events = append(caduceusOutboundSender.events, re)
	}
	if nil == caduceusOutboundSender.events {
		err = errors.New("Events must not be empty.")
		return
	}

	// Create the matcher regex objects
	for _, item := range osf.Listener.Matcher.DeviceId {
		if ".*" == item {
			// Match everything - skip the filtering
			caduceusOutboundSender.matcher = nil
			break
		}

		var re *regexp.Regexp
		if re, err = regexp.Compile(item); nil != err {
			err = fmt.Errorf("Invalid matcher item: '%s'", item)
			return
		}
		caduceusOutboundSender.matcher = append(caduceusOutboundSender.matcher, re)
	}

	caduceusOutboundSender.wg.Add(osf.NumWorkers)
	for i := 0; i < osf.NumWorkers; i++ {
		go caduceusOutboundSender.worker(i)
	}

	obs = caduceusOutboundSender
	return
}

// Extend extends the time the CaduceusOutboundSender will deliver messages to the
// specified time.  The new delivery cutoff time must be after the previously
// set delivery cutoff time.
func (obs *CaduceusOutboundSender) Extend(until time.Time) {
	obs.mutex.Lock()
	if until.After(obs.deliverUntil) {
		obs.deliverUntil = until
	}
	obs.mutex.Unlock()
}

// Shutdown causes the CaduceusOutboundSender to stop it's activities either gently or
// abruptly based on the gentle parameter.  If gentle is false, all queued
// messages will be dropped without an attempt to send made.
func (obs *CaduceusOutboundSender) Shutdown(gentle bool) {
	close(obs.queue)
	obs.mutex.Lock()
	if false == gentle {
		obs.deliverUntil = time.Time{}
	}
	obs.mutex.Unlock()
	obs.wg.Wait()

	obs.mutex.Lock()
	obs.deliverUntil = time.Time{}
	obs.mutex.Unlock()
}

// RetiredSince returns the time the CaduceusOutboundSender retired (which could be in
// the future).
func (obs *CaduceusOutboundSender) RetiredSince() time.Time {
	obs.mutex.RLock()
	deliverUntil := obs.deliverUntil
	obs.mutex.RUnlock()

	return deliverUntil
}

// QueueJSON is given a request to evaluate and optionally enqueue in the list
// of messages to deliver.  The request is checked to see if it matches the
// criteria before being accepted or silently dropped.
func (obs *CaduceusOutboundSender) QueueJSON(req CaduceusRequest,
	eventType, deviceID, transID string) {

	obs.mutex.RLock()
	deliverUntil := obs.deliverUntil
	dropUntil := obs.dropUntil
	obs.mutex.RUnlock()

	now := time.Now()

	if now.Before(deliverUntil) && now.After(dropUntil) {
		for _, eventRegex := range obs.events {
			if eventRegex.MatchString(eventType) {
				matchDevice := (nil == obs.matcher)
				if nil != obs.matcher {
					for _, deviceRegex := range obs.matcher {
						if deviceRegex.MatchString(deviceID) {
							matchDevice = true
							break
						}
					}
				}
				if matchDevice {
					if len(obs.queue) < obs.queueSize {
						outboundReq := outboundRequest{req: req,
							event:       eventType,
							transID:     transID,
							deviceID:    deviceID,
							contentType: "application/json",
						}
						obs.queue <- outboundReq
					} else {
						obs.queueOverflow()
					}
				}
			}
		}
	} else {
		// Report drop
	}
}

// QueueWrp is given a request to evaluate and optionally enqueue in the list
// of messages to deliver.  The request is checked to see if it matches the
// criteria before being accepted or silently dropped.
func (obs *CaduceusOutboundSender) QueueWrp(req CaduceusRequest, metaData map[string]string,
	eventType, deviceID, transID string) {
	obs.mutex.RLock()
	deliverUntil := obs.deliverUntil
	dropUntil := obs.dropUntil
	obs.mutex.RUnlock()

	now := time.Now()

	if now.Before(deliverUntil) && now.After(dropUntil) {
		for _, eventRegex := range obs.events {
			if eventRegex.MatchString(eventType) {
				matchDevice := (nil == obs.matcher)
				if nil != obs.matcher {
					for _, deviceRegex := range obs.matcher {
						if deviceRegex.MatchString(deviceID) {
							matchDevice = true
							break
						}
					}
				}
				/*
					// if the device id matches then we want to look through all the metadata
					// and make sure that the obs metadata matches the metadata provided
					if matchDevice {
						for key, val := range metaData {
							if matchers, ok := obs.matcher[key]; ok {
								for _, deviceRegex := range matchers {
									matchDevice = false
									if deviceRegex.MatchString(val) {
										matchDevice = true
										break
									}
								}

								// metadata was provided but did not match our expectations,
								// so it is time to drop the message
								if !matchDevice {
									break
								}
							}
						}
					}
				*/
				if matchDevice {
					if len(obs.queue) < obs.queueSize {
						outboundReq := outboundRequest{req: req,
							event:       eventType,
							transID:     transID,
							deviceID:    deviceID,
							contentType: "application/wrp",
						}
						obs.queue <- outboundReq
					} else {
						obs.queueOverflow()
					}
				}
			}
		}
	} else {
		// Report drop
	}
}

// worker is the routine that actually takes the queued messages and delivers
// them to the listeners outside webpa
func (obs *CaduceusOutboundSender) worker(id int) {
	defer obs.wg.Done()

	// Make a local copy of the hmac
	var h hash.Hash

	// Create the base sha1 hash object for each thread
	if nil != obs.secret {
		h = hmac.New(sha1.New, obs.secret)
	}

	for work := range obs.queue {
		obs.mutex.RLock()
		deliverUntil := obs.deliverUntil
		dropUntil := obs.dropUntil
		obs.mutex.RUnlock()

		now := time.Now()
		if now.Before(deliverUntil) && now.After(dropUntil) {
			payload := bytes.NewReader(work.req.Payload)
			req, err := http.NewRequest("POST", obs.listener.Config.URL, payload)
			if nil != err {
				// Report drop
				obs.logger.Error("http.NewRequest(\"POST\", '%s', payload) failed: %s", obs.listener.Config.URL, err)
			} else {
				req.Header.Set("Content-Type", work.contentType)
				req.Header.Set("X-Webpa-Event", work.event)
				req.Header.Set("X-Webpa-Transaction-Id", work.transID)
				req.Header.Set("X-Webpa-Device-Id", work.deviceID)

				if nil != h {
					h.Reset()
					h.Write(work.req.Payload)
					sig := fmt.Sprintf("sha1=%s", hex.EncodeToString(h.Sum(nil)))
					req.Header.Set("X-Webpa-Signature", sig)
				}

				// Send it
				resp, err := obs.client.Do(req)
				if nil != err {
					// Report failure
				} else {
					if (200 <= resp.StatusCode) && (resp.StatusCode <= 204) {
						// Report success
						obs.logger.Error("Success sending to '%s'", obs.listener.Config.URL)
						obs.profiler.Send(work.req)
					} else {
						// Report partial success
						obs.profiler.Send(work.req)
					}
				}
			}

		} else {
			// Report drop
		}
	}
}

// queueOverflow handles the logic of what to do when a queue overflows
func (obs *CaduceusOutboundSender) queueOverflow() {
	obs.mutex.Lock()
	obs.dropUntil = time.Now().Add(obs.cutOffPeriod)
	obs.mutex.Unlock()

	// Send a "you've been cut off" warning message
	if "" != obs.listener.FailureURL {
		msg, err := json.Marshal(obs.failureMsg)

		if nil != err {
			obs.logger.Error("json.Marshal( %v ) failed: %s", obs.failureMsg, err)
		} else {

			payload := bytes.NewReader(msg)
			req, err := http.NewRequest("POST", obs.listener.FailureURL, payload)
			req.Header.Set("Content-Type", "application/json")

			if nil != obs.secret {
				h := hmac.New(sha1.New, obs.secret)
				h.Write(msg)
				sig := fmt.Sprintf("sha1=%s", hex.EncodeToString(h.Sum(nil)))
				req.Header.Set("X-Webpa-Signature", sig)
			}

			resp, err := obs.client.Do(req)
			if nil != err {
				// Failure
				obs.logger.Error("Unable to send cut-off notification (%s) err: %s", obs.listener.FailureURL, err)
			} else {
				if nil == resp {
					// Failure
					obs.logger.Error("Unable to send cut-off notification (%s) nil response", obs.listener.FailureURL)
				} else {
					// Success
					obs.logger.Error("Able to send cut-off notification (%s) status: %s", obs.listener.FailureURL, resp.Status)
				}
			}
		}
	} else {
		obs.logger.Error("No cut-off notification URL specified.")
	}
}
<commit_msg>Revert undesireable log entry.<commit_after>package main

import (
	"bytes"
	"crypto/hmac"
	"crypto/sha1"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/Comcast/webpa-common/logging"
	"github.com/Comcast/webpa-common/webhook"
	"hash"
	"net/http"
	"net/url"
	"regexp"
	"sync"
	"time"
)

// failureText is human readable text for the failure message
const failureText = `Unfortunately, your endpoint is not able to keep up with the ` +
	`traffic being sent to it.  Due to this circumstance, all notification traffic ` +
	`is being cut off and dropped for a period of time.  Please increase your ` +
	`capacity to handle notifications, or reduce the number of notifications ` +
	`you have requested.`

// outboundRequest stores the outgoing request and assorted data that has been
// collected so far (to save on processing the information again).
type outboundRequest struct {
	req         CaduceusRequest
	event       string
	transID     string
	deviceID    string
	contentType string
}

// FailureMessage is a helper that lets us easily create a json struct to send
// when we have to cut and endpoint off.
type FailureMessage struct {
	Text         string    `json:"text"`
	Original     webhook.W `json:"webhook_registration"`
	CutOffPeriod string    `json:"cut_off_period"`
	QueueSize    int       `json:"queue_size"`
	Workers      int       `json:"worker_count"`
}

// OutboundSenderFactory is a configurable factory for OutboundSender objects.
type OutboundSenderFactory struct {
	// The WebHookListener to service
	Listener webhook.W

	// The http client to use for requests.
	Client *http.Client

	// The number of delivery workers to create and use.
	NumWorkers int

	// The queue depth to buffer events before we declare overflow, shut
	// off the message delivery, and basically put the endpoint in "timeout."
	QueueSize int

	// The amount of time to cut off the consumer if they don't keep up.
	// Must be greater then 0 seconds
	CutOffPeriod time.Duration

	// The factory that we'll use to make new ServerProfilers on a per
	// outboundSender basis
	ProfilerFactory ServerProfilerFactory

	// The logger to use.
	Logger logging.Logger
}

type OutboundSender interface {
	Extend(time.Time)
	Shutdown(bool)
	RetiredSince() time.Time
	QueueJSON(CaduceusRequest, string, string, string)
	QueueWrp(CaduceusRequest, map[string]string, string, string, string)
}

// CaduceusOutboundSender is the outbound sender object.
type CaduceusOutboundSender struct {
	listener     webhook.W
	deliverUntil time.Time
	dropUntil    time.Time
	client       *http.Client
	secret       []byte
	events       []*regexp.Regexp
	matcher      []*regexp.Regexp
	queueSize    int
	queue        chan outboundRequest
	profiler     ServerProfiler
	wg           sync.WaitGroup
	cutOffPeriod time.Duration
	failureMsg   FailureMessage
	logger       logging.Logger
	mutex        sync.RWMutex
}

// New creates a new OutboundSender object from the factory, or returns an error.
func (osf OutboundSenderFactory) New() (obs OutboundSender, err error) {
	if _, err = url.ParseRequestURI(osf.Listener.Config.URL); nil != err {
		return
	}

	if nil == osf.Client {
		err = errors.New("nil http.Client")
		return
	}

	if 0 == osf.CutOffPeriod.Nanoseconds() {
		err = errors.New("Invalid CutOffPeriod")
		return
	}

	if nil == osf.Logger {
		err = errors.New("Logger required")
		return
	}

	caduceusOutboundSender := &CaduceusOutboundSender{
		listener:     osf.Listener,
		client:       osf.Client,
		queueSize:    osf.QueueSize,
		cutOffPeriod: osf.CutOffPeriod,
		deliverUntil: osf.Listener.Until,
		logger:       osf.Logger,
		failureMsg: FailureMessage{
			Original:     osf.Listener,
			Text:         failureText,
			CutOffPeriod: osf.CutOffPeriod.String(),
			QueueSize:    osf.QueueSize,
			Workers:      osf.NumWorkers,
		},
	}

	if "" != osf.Listener.Config.Secret {
		caduceusOutboundSender.secret = []byte(osf.Listener.Config.Secret)
	}

	if "" != osf.Listener.FailureURL {
		if _, err = url.ParseRequestURI(osf.Listener.FailureURL); nil != err {
			return
		}
	}

	caduceusOutboundSender.profiler, err = osf.ProfilerFactory.New()
	if err != nil {
		return
	}

	// Give us some head room so that we don't block when we get near the
	// completely full point.
	caduceusOutboundSender.queue = make(chan outboundRequest, osf.QueueSize)

	// Create the event regex objects
	for _, event := range osf.Listener.Events {
		var re *regexp.Regexp
		if re, err = regexp.Compile(event); nil != err {
			return
		}

		caduceusOutboundSender.events = append(caduceusOutboundSender.events, re)
	}
	if nil == caduceusOutboundSender.events {
		err = errors.New("Events must not be empty.")
		return
	}

	// Create the matcher regex objects
	for _, item := range osf.Listener.Matcher.DeviceId {
		if ".*" == item {
			// Match everything - skip the filtering
			caduceusOutboundSender.matcher = nil
			break
		}

		var re *regexp.Regexp
		if re, err = regexp.Compile(item); nil != err {
			err = fmt.Errorf("Invalid matcher item: '%s'", item)
			return
		}
		caduceusOutboundSender.matcher = append(caduceusOutboundSender.matcher, re)
	}

	caduceusOutboundSender.wg.Add(osf.NumWorkers)
	for i := 0; i < osf.NumWorkers; i++ {
		go caduceusOutboundSender.worker(i)
	}

	obs = caduceusOutboundSender
	return
}

// Extend extends the time the CaduceusOutboundSender will deliver messages to the
// specified time.  The new delivery cutoff time must be after the previously
// set delivery cutoff time.
func (obs *CaduceusOutboundSender) Extend(until time.Time) {
	obs.mutex.Lock()
	if until.After(obs.deliverUntil) {
		obs.deliverUntil = until
	}
	obs.mutex.Unlock()
}

// Shutdown causes the CaduceusOutboundSender to stop it's activities either gently or
// abruptly based on the gentle parameter.  If gentle is false, all queued
// messages will be dropped without an attempt to send made.
func (obs *CaduceusOutboundSender) Shutdown(gentle bool) {
	close(obs.queue)
	obs.mutex.Lock()
	if false == gentle {
		obs.deliverUntil = time.Time{}
	}
	obs.mutex.Unlock()
	obs.wg.Wait()

	obs.mutex.Lock()
	obs.deliverUntil = time.Time{}
	obs.mutex.Unlock()
}

// RetiredSince returns the time the CaduceusOutboundSender retired (which could be in
// the future).
func (obs *CaduceusOutboundSender) RetiredSince() time.Time {
	obs.mutex.RLock()
	deliverUntil := obs.deliverUntil
	obs.mutex.RUnlock()

	return deliverUntil
}

// QueueJSON is given a request to evaluate and optionally enqueue in the list
// of messages to deliver.  The request is checked to see if it matches the
// criteria before being accepted or silently dropped.
func (obs *CaduceusOutboundSender) QueueJSON(req CaduceusRequest,
	eventType, deviceID, transID string) {

	obs.mutex.RLock()
	deliverUntil := obs.deliverUntil
	dropUntil := obs.dropUntil
	obs.mutex.RUnlock()

	now := time.Now()

	if now.Before(deliverUntil) && now.After(dropUntil) {
		for _, eventRegex := range obs.events {
			if eventRegex.MatchString(eventType) {
				matchDevice := (nil == obs.matcher)
				if nil != obs.matcher {
					for _, deviceRegex := range obs.matcher {
						if deviceRegex.MatchString(deviceID) {
							matchDevice = true
							break
						}
					}
				}
				if matchDevice {
					if len(obs.queue) < obs.queueSize {
						outboundReq := outboundRequest{req: req,
							event:       eventType,
							transID:     transID,
							deviceID:    deviceID,
							contentType: "application/json",
						}
						obs.queue <- outboundReq
					} else {
						obs.queueOverflow()
					}
				}
			}
		}
	} else {
		// Report drop
	}
}

// QueueWrp is given a request to evaluate and optionally enqueue in the list
// of messages to deliver.  The request is checked to see if it matches the
// criteria before being accepted or silently dropped.
func (obs *CaduceusOutboundSender) QueueWrp(req CaduceusRequest, metaData map[string]string,
	eventType, deviceID, transID string) {
	obs.mutex.RLock()
	deliverUntil := obs.deliverUntil
	dropUntil := obs.dropUntil
	obs.mutex.RUnlock()

	now := time.Now()

	if now.Before(deliverUntil) && now.After(dropUntil) {
		for _, eventRegex := range obs.events {
			if eventRegex.MatchString(eventType) {
				matchDevice := (nil == obs.matcher)
				if nil != obs.matcher {
					for _, deviceRegex := range obs.matcher {
						if deviceRegex.MatchString(deviceID) {
							matchDevice = true
							break
						}
					}
				}
				/*
					// if the device id matches then we want to look through all the metadata
					// and make sure that the obs metadata matches the metadata provided
					if matchDevice {
						for key, val := range metaData {
							if matchers, ok := obs.matcher[key]; ok {
								for _, deviceRegex := range matchers {
									matchDevice = false
									if deviceRegex.MatchString(val) {
										matchDevice = true
										break
									}
								}

								// metadata was provided but did not match our expectations,
								// so it is time to drop the message
								if !matchDevice {
									break
								}
							}
						}
					}
				*/
				if matchDevice {
					if len(obs.queue) < obs.queueSize {
						outboundReq := outboundRequest{req: req,
							event:       eventType,
							transID:     transID,
							deviceID:    deviceID,
							contentType: "application/wrp",
						}
						obs.queue <- outboundReq
					} else {
						obs.queueOverflow()
					}
				}
			}
		}
	} else {
		// Report drop
	}
}

// worker is the routine that actually takes the queued messages and delivers
// them to the listeners outside webpa
func (obs *CaduceusOutboundSender) worker(id int) {
	defer obs.wg.Done()

	// Make a local copy of the hmac
	var h hash.Hash

	// Create the base sha1 hash object for each thread
	if nil != obs.secret {
		h = hmac.New(sha1.New, obs.secret)
	}

	for work := range obs.queue {
		obs.mutex.RLock()
		deliverUntil := obs.deliverUntil
		dropUntil := obs.dropUntil
		obs.mutex.RUnlock()

		now := time.Now()
		if now.Before(deliverUntil) && now.After(dropUntil) {
			payload := bytes.NewReader(work.req.Payload)
			req, err := http.NewRequest("POST", obs.listener.Config.URL, payload)
			if nil != err {
				// Report drop
				obs.logger.Error("http.NewRequest(\"POST\", '%s', payload) failed: %s", obs.listener.Config.URL, err)
			} else {
				req.Header.Set("Content-Type", work.contentType)
				req.Header.Set("X-Webpa-Event", work.event)
				req.Header.Set("X-Webpa-Transaction-Id", work.transID)
				req.Header.Set("X-Webpa-Device-Id", work.deviceID)

				if nil != h {
					h.Reset()
					h.Write(work.req.Payload)
					sig := fmt.Sprintf("sha1=%s", hex.EncodeToString(h.Sum(nil)))
					req.Header.Set("X-Webpa-Signature", sig)
				}

				// Send it
				resp, err := obs.client.Do(req)
				if nil != err {
					// Report failure
				} else {
					if (200 <= resp.StatusCode) && (resp.StatusCode <= 204) {
						// Report success
						obs.profiler.Send(work.req)
					} else {
						// Report partial success
						obs.profiler.Send(work.req)
					}
				}
			}

		} else {
			// Report drop
		}
	}
}

// queueOverflow handles the logic of what to do when a queue overflows
func (obs *CaduceusOutboundSender) queueOverflow() {
	obs.mutex.Lock()
	obs.dropUntil = time.Now().Add(obs.cutOffPeriod)
	obs.mutex.Unlock()

	// Send a "you've been cut off" warning message
	if "" != obs.listener.FailureURL {
		msg, err := json.Marshal(obs.failureMsg)

		if nil != err {
			obs.logger.Error("json.Marshal( %v ) failed: %s", obs.failureMsg, err)
		} else {

			payload := bytes.NewReader(msg)
			req, err := http.NewRequest("POST", obs.listener.FailureURL, payload)
			req.Header.Set("Content-Type", "application/json")

			if nil != obs.secret {
				h := hmac.New(sha1.New, obs.secret)
				h.Write(msg)
				sig := fmt.Sprintf("sha1=%s", hex.EncodeToString(h.Sum(nil)))
				req.Header.Set("X-Webpa-Signature", sig)
			}

			resp, err := obs.client.Do(req)
			if nil != err {
				// Failure
				obs.logger.Error("Unable to send cut-off notification (%s) err: %s", obs.listener.FailureURL, err)
			} else {
				if nil == resp {
					// Failure
					obs.logger.Error("Unable to send cut-off notification (%s) nil response", obs.listener.FailureURL)
				} else {
					// Success
					obs.logger.Error("Able to send cut-off notification (%s) status: %s", obs.listener.FailureURL, resp.Status)
				}
			}
		}
	} else {
		obs.logger.Error("No cut-off notification URL specified.")
	}
}
<|endoftext|>