<commit_before>package goprocessctx

import (
	"context"

	goprocess "github.com/jbenet/goprocess"
)

// WithContext constructs and returns a Process that respects
// given context. It is the equivalent of:
//
//   func ProcessWithContext(ctx context.Context) goprocess.Process {
//     p := goprocess.WithParent(goprocess.Background())
//     CloseAfterContext(p, ctx)
//     return p
//   }
//
func WithContext(ctx context.Context) goprocess.Process {
	p := goprocess.WithParent(goprocess.Background())
	CloseAfterContext(p, ctx)
	return p
}

// WithContextAndTeardown is a helper function to set teardown at initiation
// of WithContext
func WithContextAndTeardown(ctx context.Context, tf goprocess.TeardownFunc) goprocess.Process {
	p := goprocess.WithTeardown(tf)
	CloseAfterContext(p, ctx)
	return p
}

// WaitForContext makes p WaitFor ctx. When Closing, p waits for
// ctx.Done(), before being Closed(). It is simply:
//
//   p.WaitFor(goprocess.WithContext(ctx))
//
func WaitForContext(ctx context.Context, p goprocess.Process) {
	p.WaitFor(WithContext(ctx))
}

// CloseAfterContext schedules the process to close after the given
// context is done. It is the equivalent of:
//
//   func CloseAfterContext(p goprocess.Process, ctx context.Context) {
//     go func() {
//       <-ctx.Done()
//       p.Close()
//     }()
//   }
//
func CloseAfterContext(p goprocess.Process, ctx context.Context) {
	if p == nil {
		panic("nil Process")
	}
	if ctx == nil {
		panic("nil Context")
	}

	// context.Background(). if ctx.Done() is nil, it will never be done.
	// we check for this to avoid wasting a goroutine forever.
	if ctx.Done() == nil {
		return
	}

	go func() {
		select {
		case <-ctx.Done():
			p.Close()
		case <-p.Closed():
		}
	}()
}

// WithProcessClosing returns a context.Context derived from ctx that
// is cancelled as p is Closing (after: <-p.Closing()). It is simply:
//
//   func WithProcessClosing(ctx context.Context, p goprocess.Process) context.Context {
//     ctx, cancel := context.WithCancel(ctx)
//     go func() {
//       <-p.Closing()
//       cancel()
//     }()
//     return ctx
//   }
//
func WithProcessClosing(ctx context.Context, p goprocess.Process) context.Context {
	ctx, cancel := context.WithCancel(ctx)
	p.AddChildNoWait(goprocess.WithTeardown(func() error {
		cancel()
		return nil
	}))
	return ctx
}

// WithProcessClosed returns a context.Context that is cancelled
// after Process p is Closed. It is the equivalent of:
//
//   func WithProcessClosed(ctx context.Context, p goprocess.Process) context.Context {
//     ctx, cancel := context.WithCancel(ctx)
//     go func() {
//       <-p.Closed()
//       cancel()
//     }()
//     return ctx
//   }
//
func WithProcessClosed(ctx context.Context, p goprocess.Process) context.Context {
	ctx, cancel := context.WithCancel(ctx)
	p.AddChildNoWait(goprocess.WithTeardown(func() error {
		select {
		case <-p.Closed():
		case <-ctx.Done():
		}
		cancel()
		return nil
	}))
	return ctx
}
<commit_msg>avoid a goroutine when a process can't close<commit_after>package goprocessctx

import (
	"context"

	goprocess "github.com/jbenet/goprocess"
)

// WithContext constructs and returns a Process that respects
// given context. It is the equivalent of:
//
//   func ProcessWithContext(ctx context.Context) goprocess.Process {
//     p := goprocess.WithParent(goprocess.Background())
//     CloseAfterContext(p, ctx)
//     return p
//   }
//
func WithContext(ctx context.Context) goprocess.Process {
	p := goprocess.WithParent(goprocess.Background())
	CloseAfterContext(p, ctx)
	return p
}

// WithContextAndTeardown is a helper function to set teardown at initiation
// of WithContext
func WithContextAndTeardown(ctx context.Context, tf goprocess.TeardownFunc) goprocess.Process {
	p := goprocess.WithTeardown(tf)
	CloseAfterContext(p, ctx)
	return p
}

// WaitForContext makes p WaitFor ctx. When Closing, p waits for
// ctx.Done(), before being Closed(). It is simply:
//
//   p.WaitFor(goprocess.WithContext(ctx))
//
func WaitForContext(ctx context.Context, p goprocess.Process) {
	p.WaitFor(WithContext(ctx))
}

// CloseAfterContext schedules the process to close after the given
// context is done. It is the equivalent of:
//
//   func CloseAfterContext(p goprocess.Process, ctx context.Context) {
//     go func() {
//       <-ctx.Done()
//       p.Close()
//     }()
//   }
//
func CloseAfterContext(p goprocess.Process, ctx context.Context) {
	if p == nil {
		panic("nil Process")
	}
	if ctx == nil {
		panic("nil Context")
	}

	// Avoid a goroutine for both context.Background() and goprocess.Background().
	if ctx.Done() == nil || p.Closed() == nil {
		return
	}

	go func() {
		select {
		case <-ctx.Done():
			p.Close()
		case <-p.Closed():
		}
	}()
}

// WithProcessClosing returns a context.Context derived from ctx that
// is cancelled as p is Closing (after: <-p.Closing()). It is simply:
//
//   func WithProcessClosing(ctx context.Context, p goprocess.Process) context.Context {
//     ctx, cancel := context.WithCancel(ctx)
//     go func() {
//       <-p.Closing()
//       cancel()
//     }()
//     return ctx
//   }
//
func WithProcessClosing(ctx context.Context, p goprocess.Process) context.Context {
	ctx, cancel := context.WithCancel(ctx)
	p.AddChildNoWait(goprocess.WithTeardown(func() error {
		cancel()
		return nil
	}))
	return ctx
}

// WithProcessClosed returns a context.Context that is cancelled
// after Process p is Closed. It is the equivalent of:
//
//   func WithProcessClosed(ctx context.Context, p goprocess.Process) context.Context {
//     ctx, cancel := context.WithCancel(ctx)
//     go func() {
//       <-p.Closed()
//       cancel()
//     }()
//     return ctx
//   }
//
func WithProcessClosed(ctx context.Context, p goprocess.Process) context.Context {
	ctx, cancel := context.WithCancel(ctx)
	p.AddChildNoWait(goprocess.WithTeardown(func() error {
		select {
		case <-p.Closed():
		case <-ctx.Done():
		}
		cancel()
		return nil
	}))
	return ctx
}
<|endoftext|>


<commit_before>package diffiehellman

import (
	"crypto/rand"
	"math/big"
)

// Diffie-Hellman-Merkle key exchange
// Private keys should be generated randomly.

func PrivateKey(p *big.Int) *big.Int {
	max := big.NewInt(0)
	two := big.NewInt(2)
	max.Sub(p, two)
	n, err := rand.Int(rand.Reader, max)
	if err != nil {
		panic(err)
	}
	return two.Add(two, n)
}

func PublicKey(a *big.Int, p *big.Int, g int64) *big.Int {
	temp := big.NewInt(g)
	return temp.Exp(temp, a, p)
}

func SecretKey(a *big.Int, B *big.Int, p *big.Int) *big.Int {
	return B.Exp(B, a, p)

}

func NewPair(p *big.Int, g int64) (private *big.Int, public *big.Int) {
	private = PrivateKey(p)
	public = PublicKey(private, p, g)
	return private, public
}
<commit_msg>Avoid overwriting public2 when calculating secret key<commit_after>package diffiehellman

import (
	"crypto/rand"
	"math/big"
)

// Diffie-Hellman-Merkle key exchange
// Private keys should be generated randomly.

func PrivateKey(p *big.Int) *big.Int {
	max := big.NewInt(0)
	two := big.NewInt(2)
	max.Sub(p, two)
	n, err := rand.Int(rand.Reader, max)
	if err != nil {
		panic(err)
	}
	return two.Add(two, n)
}

func PublicKey(a *big.Int, p *big.Int, g int64) *big.Int {
	temp := big.NewInt(g)
	return temp.Exp(temp, a, p)
}

func SecretKey(private1 *big.Int, public2 *big.Int, p *big.Int) *big.Int {
	return new(big.Int).Exp(public2, private1, p)

}

func NewPair(p *big.Int, g int64) (private *big.Int, public *big.Int) {
	private = PrivateKey(p)
	public = PublicKey(private, p, g)
	return private, public
}
<|endoftext|>


<commit_before>// +build go1.8

/*******************************************************************************
The MIT License (MIT)

Copyright (c) 2016 Hajime Nakagami

Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*******************************************************************************/

package firebirdsql

import (
	"context"
	"database/sql"
	//	"reflect"
	"testing"
	"time"
)

func TestGo18(t *testing.T) {
	conn, err := sql.Open("firebirdsql_createdb", "sysdba:masterkey@localhost:3050/tmp/go_test_go18.fdb")
	defer conn.Close()

	if err != nil {
		t.Fatalf("Error connecting: %v", err)
	}

	conn.Exec(`
        CREATE TABLE foo (
            a INTEGER NOT NULL,
            b VARCHAR(30) NOT NULL UNIQUE,
            c VARCHAR(1024),
            d DECIMAL(16,3) DEFAULT -0.123,
            e DATE DEFAULT '1967-08-11',
            f TIMESTAMP DEFAULT '1967-08-11 23:45:01',
            g TIME DEFAULT '23:45:01',
            h BLOB SUB_TYPE 1,
            i DOUBLE PRECISION DEFAULT 0.0,
            j FLOAT DEFAULT 0.0,
            PRIMARY KEY (a),
            CONSTRAINT CHECK_A CHECK (a <> 0)
        )
    `)
	conn.Exec("insert into foo(a, b, c, h) values (1, 'a', 'b','This is a memo')")
	conn.Exec("insert into foo(a, b, c, e, g, i, j) values (2, 'A', 'B', '1999-01-25', '00:00:01', 0.1, 0.1)")

	ctx := context.Background()
	opts := &sql.TxOptions{sql.LevelDefault, true}
	tx, err := conn.BeginTx(ctx, opts)
	if err != nil {
		t.Fatalf("Error BeginTx(): %v", err)
	}

	_, err = tx.Exec("insert into foo(a, b, c, e, g, i, j) values (3, 'X', 'Y', '2001-07-05', '00:01:02', 0.2, 0.2)")
	if err == nil {
		t.Fatalf("Need read-only transaction error")
	}

	var n int
	err = tx.QueryRow("select count(*) cnt from foo").Scan(&n)
	if err != nil {
		t.Fatalf("Error QueryRow: %v", err)
	}
	if n != 2 {
		t.Fatalf("Error bad record count: %v", n)
	}

	rows, err := tx.QueryContext(ctx, "select a, b, c, d, e, f, g, h, i, j from foo")
	var a int
	var b, c string
	var d float64
	var e time.Time
	var f time.Time
	var g time.Time
	var h []byte
	var i float64
	var j float32

	for rows.Next() {
		rows.Scan(&a, &b, &c, &d, &e, &f, &g, &h, &i, &j)
	}
}
<commit_msg>add rows.Columns() test<commit_after>// +build go1.8

/*******************************************************************************
The MIT License (MIT)

Copyright (c) 2016 Hajime Nakagami

Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*******************************************************************************/

package firebirdsql

import (
	"context"
	"database/sql"
	"reflect"
	"testing"
	"time"
)

func TestGo18(t *testing.T) {
	conn, err := sql.Open("firebirdsql_createdb", "sysdba:masterkey@localhost:3050/tmp/go_test_go18.fdb")
	defer conn.Close()

	if err != nil {
		t.Fatalf("Error connecting: %v", err)
	}

	conn.Exec(`
        CREATE TABLE foo (
            a INTEGER NOT NULL,
            b VARCHAR(30) NOT NULL UNIQUE,
            c VARCHAR(1024),
            d DECIMAL(16,3) DEFAULT -0.123,
            e DATE DEFAULT '1967-08-11',
            f TIMESTAMP DEFAULT '1967-08-11 23:45:01',
            g TIME DEFAULT '23:45:01',
            h BLOB SUB_TYPE 1,
            i DOUBLE PRECISION DEFAULT 0.0,
            j FLOAT DEFAULT 0.0,
            PRIMARY KEY (a),
            CONSTRAINT CHECK_A CHECK (a <> 0)
        )
    `)
	conn.Exec("insert into foo(a, b, c, h) values (1, 'a', 'b','This is a memo')")
	conn.Exec("insert into foo(a, b, c, e, g, i, j) values (2, 'A', 'B', '1999-01-25', '00:00:01', 0.1, 0.1)")

	ctx := context.Background()
	opts := &sql.TxOptions{sql.LevelDefault, true}
	tx, err := conn.BeginTx(ctx, opts)
	if err != nil {
		t.Fatalf("Error BeginTx(): %v", err)
	}

	_, err = tx.Exec("insert into foo(a, b, c, e, g, i, j) values (3, 'X', 'Y', '2001-07-05', '00:01:02', 0.2, 0.2)")
	if err == nil {
		t.Fatalf("Need read-only transaction error")
	}

	var n int
	err = tx.QueryRow("select count(*) cnt from foo").Scan(&n)
	if err != nil {
		t.Fatalf("Error QueryRow: %v", err)
	}
	if n != 2 {
		t.Fatalf("Error bad record count: %v", n)
	}

	rows, err := tx.QueryContext(ctx, "select a, b, c, d, e, f, g, h, i, j from foo")
	columns, err := rows.Columns()
	if !reflect.DeepEqual(columns, []string{"A", "B", "C", "D", "E", "F", "G", "H", "I", "J"}) {
		t.Fatalf("Columns() mismatch: %v", columns)
	}

	var a int
	var b, c string
	var d float64
	var e time.Time
	var f time.Time
	var g time.Time
	var h []byte
	var i float64
	var j float32

	for rows.Next() {
		rows.Scan(&a, &b, &c, &d, &e, &f, &g, &h, &i, &j)
	}
}
<|endoftext|>


<commit_before>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package backups

import (
	"bytes"
	"encoding/json"
	"io"
	"io/ioutil"
	"net/http"

	"github.com/juju/errors"

	"github.com/juju/juju/api/base"
	"github.com/juju/juju/apiserver/params"
)

// Download implements the API method.
func (c *Client) Download(id string) (io.ReadCloser, error) {
	// Initialize the HTTP request.
	req, err := c.http.NewHTTPRequest("GET", "backups")
	if err != nil {
		return nil, errors.Trace(err)
	}

	// Serialize the args.
	req.Header.Set("Content-Type", "application/json")
	args := params.BackupsDownloadArgs{
		ID: id,
	}
	data, err := json.Marshal(&args)
	if err != nil {
		return nil, errors.Annotate(err, "while serializing args")
	}
	req.Body = ioutil.NopCloser(bytes.NewBuffer(data))

	// Send the request.
	resp, err := c.http.SendHTTPRequest(req)
	if err != nil {
		return nil, errors.Annotate(err, "while sending HTTP request")
	}

	// Handle the response.
	if resp.StatusCode != http.StatusOK {
		failure, err := base.HandleHTTPFailure(resp)
		if err != nil {
			return nil, errors.Trace(err)
		}
		return nil, errors.Trace(failure)
	}

	return resp.Body, nil
}
<commit_msg>Simplify error handling.<commit_after>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package backups

import (
	"bytes"
	"encoding/json"
	"io"
	"io/ioutil"
	"net/http"

	"github.com/juju/errors"

	"github.com/juju/juju/api/base"
	"github.com/juju/juju/apiserver/params"
)

// Download implements the API method.
func (c *Client) Download(id string) (io.ReadCloser, error) {
	// Initialize the HTTP request.
	req, err := c.http.NewHTTPRequest("GET", "backups")
	if err != nil {
		return nil, errors.Trace(err)
	}

	// Serialize the args.
	req.Header.Set("Content-Type", "application/json")
	args := params.BackupsDownloadArgs{
		ID: id,
	}
	data, err := json.Marshal(&args)
	if err != nil {
		return nil, errors.Annotate(err, "while serializing args")
	}
	req.Body = ioutil.NopCloser(bytes.NewBuffer(data))

	// Send the request.
	resp, err := c.http.SendHTTPRequest(req)
	if err != nil {
		return nil, errors.Annotate(err, "while sending HTTP request")
	}

	// Handle the response.
	if resp.StatusCode != http.StatusOK {
		return nil, errors.Trace(base.HandleHTTPFailure(resp))
	}

	return resp.Body, nil
}
<|endoftext|>


<commit_before>package upload

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"

	"github.com/materials-commons/mcstore/pkg/app"
	"github.com/materials-commons/mcstore/pkg/app/flow"
)

type RequestWriter interface {
	Write(req *flow.Request) error
}

type RequestPath interface {
	Path(req *flow.Request) string
	Dir(req *flow.Request) string
}

type fileRequestWriter struct {
	RequestPath
}

func (r *fileRequestWriter) Write(req *flow.Request) error {
	path := r.Path(req)
	err := r.validateWrite(path, req)
	switch {
	case err == nil:
		return ioutil.WriteFile(path, req.Chunk, 0700)
	case err == app.ErrExists:
		return nil
	default:
		return err
	}
}

// validateWrite determines if a particular chunk can be written.
// If the size of the ondisk chunk is smaller than the request
// chunk then that chunk is incomplete and we allow a write to it.
func (r *fileRequestWriter) validateWrite(path string, req *flow.Request) error {
	if err := os.MkdirAll(path, 0700); err != nil {
		return err
	}
	finfo, err := os.Stat(path)
	switch {
	case os.IsNotExist(err):
		return nil
	case err != nil:
		return app.ErrInvalid
	case finfo.Size() < int64(req.FlowChunkSize):
		return nil
	case finfo.Size() == int64(req.FlowChunkSize):
		return app.ErrExists
	default:
		return app.ErrInvalid
	}
}

type mcdirRequestPath struct{}

func (p *mcdirRequestPath) Path(req *flow.Request) string {
	return filepath.Join(p.Dir(req), fmt.Sprintf("%d", req.FlowChunkNumber))
}

func (p *mcdirRequestPath) Dir(req *flow.Request) string {
	mcdir := app.MCDir.Path()
	uploadPath := filepath.Join(mcdir, "upload", req.ProjectID, req.DirectoryID, req.FileID)
	return uploadPath
}

type nopRequestWriter struct {
	err error
}

func (r *nopRequestWriter) Write(req *flow.Request) error {
	return r.err
}
<commit_msg>Add new* methods.<commit_after>package upload

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"

	"github.com/materials-commons/mcstore/pkg/app"
	"github.com/materials-commons/mcstore/pkg/app/flow"
)

type RequestWriter interface {
	Write(req *flow.Request) error
}

type RequestPath interface {
	Path(req *flow.Request) string
	Dir(req *flow.Request) string
}

type fileRequestWriter struct {
	RequestPath
}

func newFileRequestWriter(requestPath RequestPath) *fileRequestWriter {
	return &fileRequestWriter{
		RequestPath: requestPath,
	}
}

func (r *fileRequestWriter) Write(req *flow.Request) error {
	path := r.Path(req)
	err := r.validateWrite(path, req)
	switch {
	case err == nil:
		return ioutil.WriteFile(path, req.Chunk, 0700)
	case err == app.ErrExists:
		return nil
	default:
		return err
	}
}

// validateWrite determines if a particular chunk can be written.
// If the size of the ondisk chunk is smaller than the request
// chunk then that chunk is incomplete and we allow a write to it.
func (r *fileRequestWriter) validateWrite(path string, req *flow.Request) error {
	if err := os.MkdirAll(path, 0700); err != nil {
		return err
	}
	finfo, err := os.Stat(path)
	switch {
	case os.IsNotExist(err):
		return nil
	case err != nil:
		return app.ErrInvalid
	case finfo.Size() < int64(req.FlowChunkSize):
		return nil
	case finfo.Size() == int64(req.FlowChunkSize):
		return app.ErrExists
	default:
		return app.ErrInvalid
	}
}

type mcdirRequestPath struct{}

func newMCDirRequestPath() *mcdirRequestPath {
	return &mcdirRequestPath{}
}

func (p *mcdirRequestPath) Path(req *flow.Request) string {
	return filepath.Join(p.Dir(req), fmt.Sprintf("%d", req.FlowChunkNumber))
}

func (p *mcdirRequestPath) Dir(req *flow.Request) string {
	mcdir := app.MCDir.Path()
	uploadPath := filepath.Join(mcdir, "upload", req.ProjectID, req.DirectoryID, req.FileID)
	return uploadPath
}

type nopRequestWriter struct {
	err error
}

func (r *nopRequestWriter) Write(req *flow.Request) error {
	return r.err
}
<|endoftext|>


<commit_before>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package protoimpl

import (
	"fmt"
	"strings"
)

// These constants determine the current version of this module.
//
//
// For our release process, we enforce the following rules:
//	* Tagged releases use a tag that is identical to VersionString.
//	* Tagged releases never reference a commit where the VersionString
//	contains "devel".
//	* The set of all commits in this repository where VersionString
//	does not contain "devel" must have a unique VersionString.
//
//
// Steps for tagging a new release:
//	1. Create a new CL.
//
//	2. Update versionMinor, versionPatch, and/or versionPreRelease as necessary.
//	versionPreRelease must not contain the string "devel".
//
//	3. Since the last released minor version, have there been any changes to
//	generator that relies on new functionality in the runtime?
//	If yes, then increment GenVersion.
//
//	4. Since the last released minor version, have there been any changes to
//	the runtime that removes support for old .pb.go source code?
//	If yes, then increment MinVersion.
//
//	5. Send out the CL for review and submit it.
//	Note that the next CL in step 8 must be submitted after this CL
//	without any other CLs in-between.
//
//	6. Tag a new version, where the tag is is the current VersionString.
//
//	7. Write release notes for all notable changes
//	between this release and the last release.
//
//	8. Create a new CL.
//
//	9. Update versionPreRelease to include the string "devel".
//	For example: "" -> "devel" or "rc.1" -> "rc.1.devel"
//
//	10. Send out the CL for review and submit it.
const (
	versionMajor      = 1
	versionMinor      = 20
	versionPatch      = 0
	versionPreRelease = "devel"
)

// VersionString formats the version string for this module in semver format.
//
// Examples:
//	v1.20.1
//	v1.21.0-rc.1
func VersionString() string {
	v := fmt.Sprintf("v%d.%d.%d", versionMajor, versionMinor, versionPatch)
	if versionPreRelease != "" {
		v += "-" + versionPreRelease

		// TODO: Add metadata about the commit or build hash.
		// See https://golang.org/issue/29814
		// See https://golang.org/issue/33533
		var versionMetadata string
		if strings.Contains(versionPreRelease, "devel") && versionMetadata != "" {
			v += "+" + versionMetadata
		}
	}
	return v
}

const (
	// MaxVersion is the maximum supported version for generated .pb.go files.
	// It is always the current version of the module.
	MaxVersion = versionMinor

	// GenVersion is the runtime version required by generated .pb.go files.
	// This is incremented when generated code relies on new functionality
	// in the runtime.
	GenVersion = 20

	// MinVersion is the minimum supported version for generated .pb.go files.
	// This is incremented when the runtime drops support for old code.
	MinVersion = 0
)

// EnforceVersion is used by code generated by protoc-gen-go
// to statically enforce minimum and maximum versions of this package.
// A compilation failure implies either that:
//	* the runtime package is too old and needs to be updated OR
//	* the generated code is too old and needs to be regenerated.
//
// The runtime package can be upgraded by running:
//	go get google.golang.org/protobuf
//
// The generated code can be regenerated by running:
//	protoc --go_out=${PROTOC_GEN_GO_ARGS} ${PROTO_FILES}
//
// Example usage by generated code:
//	const (
//		// Verify that this generated code is sufficiently up-to-date.
//		_ = protoimpl.EnforceVersion(genVersion - protoimpl.MinVersion)
//		// Verify that runtime/protoimpl is sufficiently up-to-date.
//		_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - genVersion)
//	)
//
// The genVersion is the current minor version used to generated the code.
// This compile-time check relies on negative integer overflow of a uint
// being a compilation failure (guaranteed by the Go specification).
type EnforceVersion uint

// This enforces the following invariant:
//	MinVersion ≤ GenVersion ≤ MaxVersion
const (
	_ = EnforceVersion(GenVersion - MinVersion)
	_ = EnforceVersion(MaxVersion - GenVersion)
)
<commit_msg>all: release v1.20.1<commit_after>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package protoimpl

import (
	"fmt"
	"strings"
)

// These constants determine the current version of this module.
//
//
// For our release process, we enforce the following rules:
//	* Tagged releases use a tag that is identical to VersionString.
//	* Tagged releases never reference a commit where the VersionString
//	contains "devel".
//	* The set of all commits in this repository where VersionString
//	does not contain "devel" must have a unique VersionString.
//
//
// Steps for tagging a new release:
//	1. Create a new CL.
//
//	2. Update versionMinor, versionPatch, and/or versionPreRelease as necessary.
//	versionPreRelease must not contain the string "devel".
//
//	3. Since the last released minor version, have there been any changes to
//	generator that relies on new functionality in the runtime?
//	If yes, then increment GenVersion.
//
//	4. Since the last released minor version, have there been any changes to
//	the runtime that removes support for old .pb.go source code?
//	If yes, then increment MinVersion.
//
//	5. Send out the CL for review and submit it.
//	Note that the next CL in step 8 must be submitted after this CL
//	without any other CLs in-between.
//
//	6. Tag a new version, where the tag is is the current VersionString.
//
//	7. Write release notes for all notable changes
//	between this release and the last release.
//
//	8. Create a new CL.
//
//	9. Update versionPreRelease to include the string "devel".
//	For example: "" -> "devel" or "rc.1" -> "rc.1.devel"
//
//	10. Send out the CL for review and submit it.
const (
	versionMajor      = 1
	versionMinor      = 20
	versionPatch      = 1
	versionPreRelease = ""
)

// VersionString formats the version string for this module in semver format.
//
// Examples:
//	v1.20.1
//	v1.21.0-rc.1
func VersionString() string {
	v := fmt.Sprintf("v%d.%d.%d", versionMajor, versionMinor, versionPatch)
	if versionPreRelease != "" {
		v += "-" + versionPreRelease

		// TODO: Add metadata about the commit or build hash.
		// See https://golang.org/issue/29814
		// See https://golang.org/issue/33533
		var versionMetadata string
		if strings.Contains(versionPreRelease, "devel") && versionMetadata != "" {
			v += "+" + versionMetadata
		}
	}
	return v
}

const (
	// MaxVersion is the maximum supported version for generated .pb.go files.
	// It is always the current version of the module.
	MaxVersion = versionMinor

	// GenVersion is the runtime version required by generated .pb.go files.
	// This is incremented when generated code relies on new functionality
	// in the runtime.
	GenVersion = 20

	// MinVersion is the minimum supported version for generated .pb.go files.
	// This is incremented when the runtime drops support for old code.
	MinVersion = 0
)

// EnforceVersion is used by code generated by protoc-gen-go
// to statically enforce minimum and maximum versions of this package.
// A compilation failure implies either that:
//	* the runtime package is too old and needs to be updated OR
//	* the generated code is too old and needs to be regenerated.
//
// The runtime package can be upgraded by running:
//	go get google.golang.org/protobuf
//
// The generated code can be regenerated by running:
//	protoc --go_out=${PROTOC_GEN_GO_ARGS} ${PROTO_FILES}
//
// Example usage by generated code:
//	const (
//		// Verify that this generated code is sufficiently up-to-date.
//		_ = protoimpl.EnforceVersion(genVersion - protoimpl.MinVersion)
//		// Verify that runtime/protoimpl is sufficiently up-to-date.
//		_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - genVersion)
//	)
//
// The genVersion is the current minor version used to generated the code.
// This compile-time check relies on negative integer overflow of a uint
// being a compilation failure (guaranteed by the Go specification).
type EnforceVersion uint

// This enforces the following invariant:
//	MinVersion ≤ GenVersion ≤ MaxVersion
const (
	_ = EnforceVersion(GenVersion - MinVersion)
	_ = EnforceVersion(MaxVersion - GenVersion)
)
<|endoftext|>


<commit_before>package redis

import (
	"github.com/Clever/leakybucket"
	"sync"
	"testing"
	"time"
)

func getLocalStorage() *Storage {
	storage, err := New("tcp", "localhost:6379")
	if err != nil {
		panic(err)
	}
	return storage
}

func flushDb() {
	storage := getLocalStorage()
	conn := storage.pool.Get()
	defer conn.Close()
	_, err := conn.Do("FLUSHDB")
	if err != nil {
		panic(err)
	}
}

func TestInvalidHost(t *testing.T) {
	_, err := New("tcp", "localhost:6378")
	if err == nil {
		t.Fatalf("expected error connecting to invalid host")
	}
}

func TestCreate(t *testing.T) {
	flushDb()
	leakybucket.CreateTest(getLocalStorage())(t)
}

func TestAdd(t *testing.T) {
	flushDb()
	leakybucket.AddTest(getLocalStorage())(t)
}

func TestThreadSafeAdd(t *testing.T) {
	// Redis Add is not thread safe. If you run this, the test should fail because it never received
	// ErrorFull. It's not thread safe because we don't atomically check the state of the bucket and
	// increment.
	t.Skip()
	flushDb()
	leakybucket.ThreadSafeAddTest(getLocalStorage())(t)
}

func TestReset(t *testing.T) {
	flushDb()
	leakybucket.AddResetTest(getLocalStorage())(t)
}

func TestFindOrCreate(t *testing.T) {
	flushDb()
	leakybucket.FindOrCreateTest(getLocalStorage())(t)
}

func TestBucketInstanceConsistencyTest(t *testing.T) {
	flushDb()
	leakybucket.BucketInstanceConsistencyTest(getLocalStorage())(t)
}

// One implementation of redis leaky bucket had a bug where very fast access could result in us
// creating buckets without a TTL on them. This test was reliably able to reproduce this bug.
func TestFastAccess(t *testing.T) {
	flushDb()
	s := getLocalStorage()
	bucket, err := s.Create("testbucket", 10, time.Millisecond)
	if err != nil {
		t.Fatal(err)
	}
	hold := make(chan struct{})
	wg := sync.WaitGroup{}
	for i := 0; i < 1000; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			<-hold
			if _, err := bucket.Add(1); err != nil && err != leakybucket.ErrorFull {
				t.Fatal(err)
			}
		}()
	}
	close(hold) // Let all concurrent requests start
	wg.Wait()   // Wait for all concurrent requests to finish

	pool := s.pool
	conn := pool.Get()
	defer conn.Close()

	if exists, err := conn.Do("GET", "testbucket"); err != nil {
		t.Fatal(err)
	} else if exists == nil {
		return
	}
	ttl, err := conn.Do("PTTL", "testbucket")
	if err != nil {
		t.Fatal(err)
	}
	if ttl.(int64) == -1 {
		t.Fatal("no ttl set on bucket")
	}

}
<commit_msg>redis: style<commit_after>package redis

import (
	"github.com/Clever/leakybucket"
	"sync"
	"testing"
	"time"
)

func getLocalStorage() *Storage {
	storage, err := New("tcp", "localhost:6379")
	if err != nil {
		panic(err)
	}
	return storage
}

func flushDb() {
	storage := getLocalStorage()
	conn := storage.pool.Get()
	defer conn.Close()
	_, err := conn.Do("FLUSHDB")
	if err != nil {
		panic(err)
	}
}

func TestInvalidHost(t *testing.T) {
	_, err := New("tcp", "localhost:6378")
	if err == nil {
		t.Fatalf("expected error connecting to invalid host")
	}
}

func TestCreate(t *testing.T) {
	flushDb()
	leakybucket.CreateTest(getLocalStorage())(t)
}

func TestAdd(t *testing.T) {
	flushDb()
	leakybucket.AddTest(getLocalStorage())(t)
}

func TestThreadSafeAdd(t *testing.T) {
	// Redis Add is not thread safe. If you run this, the test should fail because it never received
	// ErrorFull. It's not thread safe because we don't atomically check the state of the bucket and
	// increment.
	t.Skip()
	flushDb()
	leakybucket.ThreadSafeAddTest(getLocalStorage())(t)
}

func TestReset(t *testing.T) {
	flushDb()
	leakybucket.AddResetTest(getLocalStorage())(t)
}

func TestFindOrCreate(t *testing.T) {
	flushDb()
	leakybucket.FindOrCreateTest(getLocalStorage())(t)
}

func TestBucketInstanceConsistencyTest(t *testing.T) {
	flushDb()
	leakybucket.BucketInstanceConsistencyTest(getLocalStorage())(t)
}

// One implementation of redis leaky bucket had a bug where very fast access could result in us
// creating buckets without a TTL on them. This test was reliably able to reproduce this bug.
func TestFastAccess(t *testing.T) {
	flushDb()
	s := getLocalStorage()
	bucket, err := s.Create("testbucket", 10, time.Millisecond)
	if err != nil {
		t.Fatal(err)
	}
	hold := make(chan struct{})
	wg := sync.WaitGroup{}
	for i := 0; i < 1000; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			<-hold
			if _, err := bucket.Add(1); err != nil && err != leakybucket.ErrorFull {
				t.Fatal(err)
			}
		}()
	}
	close(hold) // Let all concurrent requests start
	wg.Wait()   // Wait for all concurrent requests to finish

	conn := s.pool.Get()
	defer conn.Close()

	if exists, err := conn.Do("GET", "testbucket"); err != nil {
		t.Fatal(err)
	} else if exists == nil {
		return
	}
	ttl, err := conn.Do("PTTL", "testbucket")
	if err != nil {
		t.Fatal(err)
	}
	if ttl.(int64) == -1 {
		t.Fatal("no ttl set on bucket")
	}

}
<|endoftext|>


<commit_before>package containers

import (
	"bufio"
	"encoding/gob"
	"log"
	"os"
	"path"
)

func save() {
	saveObject(ContainersFile, containers)
	saveObject(PortsFile, ports)
}

// Use gob to save an object to a file
func saveObject(file string, object interface{}) {
	gob.Register(object)
	fo, err := os.Create(path.Join(SaveDir, file))
	if err != nil {
		log.Printf("Could not save %s: %s", file, err)
		// hope everything works out.
		// TODO[jigish] email error
		return
	}
	defer fo.Close()
	w := bufio.NewWriter(fo)
	e := gob.NewEncoder(w)
	e.Encode(object)
	w.Flush()
}

// Use gob to retrieve an object from a file
func retrieveObject(file string, object interface{}) bool {
	fi, err := os.Open(path.Join(SaveDir, file))
	if err != nil {
		log.Printf("Could not retrieve %s: %s", file, err)
		return false
	}
	r := bufio.NewReader(fi)
	d := gob.NewDecoder(r)
	d.Decode(object)
	log.Printf("Retrieved %s: %#v", file, object)
	return true
}
<commit_msg>return false if object retrieved was nil<commit_after>package containers

import (
	"bufio"
	"encoding/gob"
	"log"
	"os"
	"path"
)

func save() {
	saveObject(ContainersFile, containers)
	saveObject(PortsFile, ports)
}

// Use gob to save an object to a file
func saveObject(file string, object interface{}) {
	gob.Register(object)
	fo, err := os.Create(path.Join(SaveDir, file))
	if err != nil {
		log.Printf("Could not save %s: %s", file, err)
		// hope everything works out.
		// TODO[jigish] email error
		return
	}
	defer fo.Close()
	w := bufio.NewWriter(fo)
	e := gob.NewEncoder(w)
	e.Encode(object)
	w.Flush()
}

// Use gob to retrieve an object from a file
func retrieveObject(file string, object interface{}) bool {
	fi, err := os.Open(path.Join(SaveDir, file))
	if err != nil {
		log.Printf("Could not retrieve %s: %s", file, err)
		return false
	}
	r := bufio.NewReader(fi)
	d := gob.NewDecoder(r)
	d.Decode(object)
	log.Printf("Retrieved %s: %#v", file, object)
	if object == nil {
		log.Println("Object retrieved was nil.")
		return false
	}
	return true
}
<|endoftext|>


<commit_before>package openstack

import (
	"bytes"
	"fmt"
	"log"

	"github.com/hashicorp/terraform/helper/hashcode"
	"github.com/hashicorp/terraform/helper/schema"
	"github.com/rackspace/gophercloud/openstack/compute/v2/extensions/secgroups"
)

func resourceComputeSecGroupV2() *schema.Resource {
	return &schema.Resource{
		Create: resourceComputeSecGroupV2Create,
		Read:   resourceComputeSecGroupV2Read,
		Update: resourceComputeSecGroupV2Update,
		Delete: resourceComputeSecGroupV2Delete,

		Schema: map[string]*schema.Schema{
			"region": &schema.Schema{
				Type:        schema.TypeString,
				Required:    true,
				ForceNew:    true,
				DefaultFunc: envDefaultFunc("OS_REGION_NAME"),
			},
			"name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: false,
			},
			"description": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: false,
			},
			"rules": &schema.Schema{
				Type:     schema.TypeSet,
				Optional: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"from_port": &schema.Schema{
							Type:     schema.TypeInt,
							Required: true,
							ForceNew: true,
						},
						"to_port": &schema.Schema{
							Type:     schema.TypeInt,
							Required: true,
							ForceNew: true,
						},
						"ip_protocol": &schema.Schema{
							Type:     schema.TypeString,
							Required: true,
							ForceNew: true,
						},
						"cidr": &schema.Schema{
							Type:     schema.TypeString,
							Optional: true,
							ForceNew: true,
						},
						"from_group_id": &schema.Schema{
							Type:     schema.TypeString,
							Optional: true,
							ForceNew: true,
						},
					},
				},
				Set: resourceSecGroupRuleHash,
			},
		},
	}
}

func resourceComputeSecGroupV2Create(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)
	computeClient, err := config.computeV2Client(d.Get("region").(string))
	if err != nil {
		return fmt.Errorf("Error creating OpenStack compute client: %s", err)
	}

	createOpts := secgroups.CreateOpts{
		Name:        d.Get("name").(string),
		Description: d.Get("description").(string),
	}

	sg, err := secgroups.Create(computeClient, createOpts).Extract()
	if err != nil {
		return fmt.Errorf("Error creating OpenStack security group: %s", err)
	}

	d.SetId(sg.ID)

	createRuleOptsList := resourceSecGroupRulesV2(d)
	for _, createRuleOpts := range createRuleOptsList {
		_, err := secgroups.CreateRule(computeClient, createRuleOpts).Extract()
		if err != nil {
			return fmt.Errorf("Error creating OpenStack security group rule: %s", err)
		}
	}

	return resourceComputeSecGroupV2Read(d, meta)
}

func resourceComputeSecGroupV2Read(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)
	computeClient, err := config.computeV2Client(d.Get("region").(string))
	if err != nil {
		return fmt.Errorf("Error creating OpenStack compute client: %s", err)
	}

	sg, err := secgroups.Get(computeClient, d.Id()).Extract()
	if err != nil {
		return fmt.Errorf("Error retrieving OpenStack security group: %s", err)
	}

	d.Set("region", d.Get("region").(string))
	d.Set("name", sg.Name)
	d.Set("description", sg.Description)
	d.Set("rules", sg.Rules)

	return nil
}

func resourceComputeSecGroupV2Update(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)
	computeClient, err := config.computeV2Client(d.Get("region").(string))
	if err != nil {
		return fmt.Errorf("Error creating OpenStack compute client: %s", err)
	}

	updateOpts := secgroups.UpdateOpts{
		Name:        d.Get("name").(string),
		Description: d.Get("description").(string),
	}

	log.Printf("[DEBUG] Updating Security Group (%s) with options: %+v", d.Id(), updateOpts)

	_, err = secgroups.Update(computeClient, d.Id(), updateOpts).Extract()
	if err != nil {
		return fmt.Errorf("Error updating OpenStack security group (%s): %s", d.Id(), err)
	}

	if d.HasChange("rules") {
		oldSGRaw, newSGRaw := d.GetChange("rules")
		oldSGRSet, newSGRSet := oldSGRaw.(*schema.Set), newSGRaw.(*schema.Set)
		secgrouprulesToAdd := newSGRSet.Difference(oldSGRSet)
		secgrouprulesToRemove := oldSGRSet.Difference(newSGRSet)

		log.Printf("[DEBUG] Security group rules to add: %v", secgrouprulesToAdd)

		log.Printf("[DEBUG] Security groups to remove: %v", secgrouprulesToRemove)

		for _, rawRule := range secgrouprulesToAdd.List() {
			createRuleOpts := resourceSecGroupRuleV2(d, rawRule)
			rule, err := secgroups.CreateRule(computeClient, createRuleOpts).Extract()
			if err != nil {
				return fmt.Errorf("Error adding rule to OpenStack security group (%s): %s", d.Id(), err)
			}
			log.Printf("[DEBUG] Added rule (%s) to OpenStack security group (%s) ", rule.ID, d.Id())
		}

		for _, r := range secgrouprulesToRemove.List() {
			rule := r.(secgroups.Rule)
			err := secgroups.DeleteRule(computeClient, "").ExtractErr()
			if err != nil {
				return fmt.Errorf("Error removing rule (%s) from OpenStack security group (%s): %s", rule.ID, d.Id(), err)
			}
			log.Printf("[DEBUG] Removed rule (%s) from OpenStack security group (%s)", rule.ID, d.Id())
		}
	}

	return resourceComputeSecGroupV2Read(d, meta)
}

func resourceComputeSecGroupV2Delete(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)
	computeClient, err := config.computeV2Client(d.Get("region").(string))
	if err != nil {
		return fmt.Errorf("Error creating OpenStack compute client: %s", err)
	}

	err = secgroups.Delete(computeClient, d.Id()).ExtractErr()
	if err != nil {
		return fmt.Errorf("Error deleting OpenStack security group: %s", err)
	}
	d.SetId("")
	return nil
}

func resourceSecGroupRuleHash(v interface{}) int {
	var buf bytes.Buffer
	m := v.(map[string]interface{})
	buf.WriteString(fmt.Sprintf("%d-", m["from_port"].(int)))
	buf.WriteString(fmt.Sprintf("%d-", m["to_port"].(int)))
	buf.WriteString(fmt.Sprintf("%s-", m["ip_protocol"].(string)))
	buf.WriteString(fmt.Sprintf("%s-", m["cidr"].(string)))
	buf.WriteString(fmt.Sprintf("%s-", m["from_group_id"].(string)))

	return hashcode.String(buf.String())
}

func resourceSecGroupRulesV2(d *schema.ResourceData) []secgroups.CreateRuleOpts {
	rawRules := (d.Get("rules")).(*schema.Set)
	createRuleOptsList := make([]secgroups.CreateRuleOpts, rawRules.Len())
	for i, raw := range rawRules.List() {
		rawMap := raw.(map[string]interface{})
		createRuleOptsList[i] = secgroups.CreateRuleOpts{
			ParentGroupID: d.Id(),
			FromPort:      rawMap["from_port"].(int),
			ToPort:        rawMap["to_port"].(int),
			IPProtocol:    rawMap["ip_protocol"].(string),
			CIDR:          rawMap["cidr"].(string),
			FromGroupID:   rawMap["from_group_id"].(string),
		}
	}
	return createRuleOptsList
}

func resourceSecGroupRuleV2(d *schema.ResourceData, raw interface{}) secgroups.CreateRuleOpts {
	rawMap := raw.(map[string]interface{})
	createRuleOpts := secgroups.CreateRuleOpts{
		ParentGroupID: d.Id(),
		FromPort:      rawMap["from_port"].(int),
		ToPort:        rawMap["to_port"].(int),
		IPProtocol:    rawMap["ip_protocol"].(string),
		CIDR:          rawMap["cidr"].(string),
		FromGroupID:   rawMap["from_group_id"].(string),
	}

	return createRuleOpts
}
<commit_msg>'rules' -> 'rule'<commit_after>package openstack

import (
	"bytes"
	"fmt"
	"log"

	"github.com/hashicorp/terraform/helper/hashcode"
	"github.com/hashicorp/terraform/helper/schema"
	"github.com/rackspace/gophercloud/openstack/compute/v2/extensions/secgroups"
)

func resourceComputeSecGroupV2() *schema.Resource {
	return &schema.Resource{
		Create: resourceComputeSecGroupV2Create,
		Read:   resourceComputeSecGroupV2Read,
		Update: resourceComputeSecGroupV2Update,
		Delete: resourceComputeSecGroupV2Delete,

		Schema: map[string]*schema.Schema{
			"region": &schema.Schema{
				Type:        schema.TypeString,
				Required:    true,
				ForceNew:    true,
				DefaultFunc: envDefaultFunc("OS_REGION_NAME"),
			},
			"name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: false,
			},
			"description": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: false,
			},
			"rule": &schema.Schema{
				Type:     schema.TypeSet,
				Optional: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"from_port": &schema.Schema{
							Type:     schema.TypeInt,
							Required: true,
							ForceNew: true,
						},
						"to_port": &schema.Schema{
							Type:     schema.TypeInt,
							Required: true,
							ForceNew: true,
						},
						"ip_protocol": &schema.Schema{
							Type:     schema.TypeString,
							Required: true,
							ForceNew: true,
						},
						"cidr": &schema.Schema{
							Type:     schema.TypeString,
							Optional: true,
							ForceNew: true,
						},
						"from_group_id": &schema.Schema{
							Type:     schema.TypeString,
							Optional: true,
							ForceNew: true,
						},
					},
				},
				Set: resourceSecGroupRuleV2Hash,
			},
		},
	}
}

func resourceComputeSecGroupV2Create(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)
	computeClient, err := config.computeV2Client(d.Get("region").(string))
	if err != nil {
		return fmt.Errorf("Error creating OpenStack compute client: %s", err)
	}

	createOpts := secgroups.CreateOpts{
		Name:        d.Get("name").(string),
		Description: d.Get("description").(string),
	}

	sg, err := secgroups.Create(computeClient, createOpts).Extract()
	if err != nil {
		return fmt.Errorf("Error creating OpenStack security group: %s", err)
	}

	d.SetId(sg.ID)

	createRuleOptsList := resourceSecGroupRulesV2(d)
	for _, createRuleOpts := range createRuleOptsList {
		_, err := secgroups.CreateRule(computeClient, createRuleOpts).Extract()
		if err != nil {
			return fmt.Errorf("Error creating OpenStack security group rule: %s", err)
		}
	}

	return resourceComputeSecGroupV2Read(d, meta)
}

func resourceComputeSecGroupV2Read(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)
	computeClient, err := config.computeV2Client(d.Get("region").(string))
	if err != nil {
		return fmt.Errorf("Error creating OpenStack compute client: %s", err)
	}

	sg, err := secgroups.Get(computeClient, d.Id()).Extract()
	if err != nil {
		return fmt.Errorf("Error retrieving OpenStack security group: %s", err)
	}

	d.Set("region", d.Get("region").(string))
	d.Set("name", sg.Name)
	d.Set("description", sg.Description)
	d.Set("rule", sg.Rules)

	return nil
}

func resourceComputeSecGroupV2Update(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)
	computeClient, err := config.computeV2Client(d.Get("region").(string))
	if err != nil {
		return fmt.Errorf("Error creating OpenStack compute client: %s", err)
	}

	updateOpts := secgroups.UpdateOpts{
		Name:        d.Get("name").(string),
		Description: d.Get("description").(string),
	}

	log.Printf("[DEBUG] Updating Security Group (%s) with options: %+v", d.Id(), updateOpts)

	_, err = secgroups.Update(computeClient, d.Id(), updateOpts).Extract()
	if err != nil {
		return fmt.Errorf("Error updating OpenStack security group (%s): %s", d.Id(), err)
	}

	if d.HasChange("rule") {
		oldSGRaw, newSGRaw := d.GetChange("rule")
		oldSGRSet, newSGRSet := oldSGRaw.(*schema.Set), newSGRaw.(*schema.Set)
		secgrouprulesToAdd := newSGRSet.Difference(oldSGRSet)
		secgrouprulesToRemove := oldSGRSet.Difference(newSGRSet)

		log.Printf("[DEBUG] Security group rules to add: %v", secgrouprulesToAdd)

		log.Printf("[DEBUG] Security groups to remove: %v", secgrouprulesToRemove)

		for _, rawRule := range secgrouprulesToAdd.List() {
			createRuleOpts := resourceSecGroupRuleV2(d, rawRule)
			rule, err := secgroups.CreateRule(computeClient, createRuleOpts).Extract()
			if err != nil {
				return fmt.Errorf("Error adding rule to OpenStack security group (%s): %s", d.Id(), err)
			}
			log.Printf("[DEBUG] Added rule (%s) to OpenStack security group (%s) ", rule.ID, d.Id())
		}

		for _, r := range secgrouprulesToRemove.List() {
			rule := r.(secgroups.Rule)
			err := secgroups.DeleteRule(computeClient, "").ExtractErr()
			if err != nil {
				return fmt.Errorf("Error removing rule (%s) from OpenStack security group (%s): %s", rule.ID, d.Id(), err)
			}
			log.Printf("[DEBUG] Removed rule (%s) from OpenStack security group (%s)", rule.ID, d.Id())
		}
	}

	return resourceComputeSecGroupV2Read(d, meta)
}

func resourceComputeSecGroupV2Delete(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)
	computeClient, err := config.computeV2Client(d.Get("region").(string))
	if err != nil {
		return fmt.Errorf("Error creating OpenStack compute client: %s", err)
	}

	err = secgroups.Delete(computeClient, d.Id()).ExtractErr()
	if err != nil {
		return fmt.Errorf("Error deleting OpenStack security group: %s", err)
	}
	d.SetId("")
	return nil
}

func resourceSecGroupRuleV2Hash(v interface{}) int {
	var buf bytes.Buffer
	m := v.(map[string]interface{})
	buf.WriteString(fmt.Sprintf("%d-", m["from_port"].(int)))
	buf.WriteString(fmt.Sprintf("%d-", m["to_port"].(int)))
	buf.WriteString(fmt.Sprintf("%s-", m["ip_protocol"].(string)))
	buf.WriteString(fmt.Sprintf("%s-", m["cidr"].(string)))
	buf.WriteString(fmt.Sprintf("%s-", m["from_group_id"].(string)))

	return hashcode.String(buf.String())
}

func resourceSecGroupRulesV2(d *schema.ResourceData) []secgroups.CreateRuleOpts {
	rawRules := (d.Get("rule")).(*schema.Set)
	createRuleOptsList := make([]secgroups.CreateRuleOpts, rawRules.Len())
	for i, raw := range rawRules.List() {
		rawMap := raw.(map[string]interface{})
		createRuleOptsList[i] = secgroups.CreateRuleOpts{
			ParentGroupID: d.Id(),
			FromPort:      rawMap["from_port"].(int),
			ToPort:        rawMap["to_port"].(int),
			IPProtocol:    rawMap["ip_protocol"].(string),
			CIDR:          rawMap["cidr"].(string),
			FromGroupID:   rawMap["from_group_id"].(string),
		}
	}
	return createRuleOptsList
}

func resourceSecGroupRuleV2(d *schema.ResourceData, raw interface{}) secgroups.CreateRuleOpts {
	rawMap := raw.(map[string]interface{})
	return secgroups.CreateRuleOpts{
		ParentGroupID: d.Id(),
		FromPort:      rawMap["from_port"].(int),
		ToPort:        rawMap["to_port"].(int),
		IPProtocol:    rawMap["ip_protocol"].(string),
		CIDR:          rawMap["cidr"].(string),
		FromGroupID:   rawMap["from_group_id"].(string),
	}
}
<|endoftext|>


<commit_before>package aws

import (
	"math"
	"net"
	"net/http"
	"time"
)

type RetryableFunc func(*http.Request, *http.Response, error) bool
type WaitFunc func(try int)
type DeadlineFunc func() time.Time

type ResilientTransport struct {
	// Timeout is the maximum amount of time a dial will wait for
	// a connect to complete.
	//
	// The default is no timeout.
	//
	// With or without a timeout, the operating system may impose
	// its own earlier timeout. For instance, TCP timeouts are
	// often around 3 minutes.
	DialTimeout time.Duration

	// MaxTries, if non-zero, specifies the number of times we will retry on
	// failure. Retries are only attempted for temporary network errors or known
	// safe failures.
	MaxTries    int
	Deadline    DeadlineFunc
	ShouldRetry RetryableFunc
	Wait        WaitFunc
	transport   *http.Transport
}

// Convenience method for creating an http client
func NewClient(rt *ResilientTransport) *http.Client {
	rt.transport = &http.Transport{
		Dial: func(netw, addr string) (net.Conn, error) {
			c, err := net.DialTimeout(netw, addr, rt.DialTimeout)
			if err != nil {
				return nil, err
			}
			c.SetDeadline(rt.Deadline())
			return c, nil
		},
		Proxy: http.ProxyFromEnvironment,
	}
	// TODO: Would be nice is ResilientTransport allowed clients to initialize
	// with http.Transport attributes.
	return &http.Client{
		Transport: rt,
	}
}

var retryingTransport = &ResilientTransport{
	Deadline: func() time.Time {
		return time.Now().Add(5 * time.Second)
	},
	DialTimeout: 10 * time.Second,
	MaxTries:    3,
	ShouldRetry: awsRetry,
	Wait:        ExpBackoff,
}

// Exported default client
var RetryingClient = NewClient(retryingTransport)

func (t *ResilientTransport) RoundTrip(req *http.Request) (*http.Response, error) {
	return t.tries(req)
}

// Retry a request a maximum of t.MaxTries times.
// We'll only retry if the proper criteria are met.
// If a wait function is specified, wait that amount of time
// In between requests.
func (t *ResilientTransport) tries(req *http.Request) (res *http.Response, err error) {
	for try := 0; try < t.MaxTries; try += 1 {
		res, err = t.transport.RoundTrip(req)

		if !t.ShouldRetry(req, res, err) {
			break
		}
		if res != nil {
			res.Body.Close()
		}
		if t.Wait != nil {
			t.Wait(try)
		}
	}

	return
}

func ExpBackoff(try int) {
	time.Sleep(100 * time.Millisecond *
		time.Duration(math.Exp2(float64(try))))
}

func LinearBackoff(try int) {
	time.Sleep(time.Duration(try*100) * time.Millisecond)
}

// Decide if we should retry a request.
// In general, the criteria for retrying a request is described here
// http://docs.aws.amazon.com/general/latest/gr/api-retries.html
func awsRetry(req *http.Request, res *http.Response, err error) bool {
	retry := false

	// Retry if there's a temporary network error.
	if neterr, ok := err.(net.Error); ok {
		if neterr.Temporary() {
			retry = true
		}
	}

	// Retry if we get a 5xx series error.
	if res != nil {
		if res.StatusCode >= 500 && res.StatusCode < 600 {
			retry = true
		}
	}

	return retry
}
<commit_msg>client: increase the host to have more idle connection due our high client connections<commit_after>package aws

import (
	"math"
	"net"
	"net/http"
	"time"
)

type RetryableFunc func(*http.Request, *http.Response, error) bool
type WaitFunc func(try int)
type DeadlineFunc func() time.Time

type ResilientTransport struct {
	// Timeout is the maximum amount of time a dial will wait for
	// a connect to complete.
	//
	// The default is no timeout.
	//
	// With or without a timeout, the operating system may impose
	// its own earlier timeout. For instance, TCP timeouts are
	// often around 3 minutes.
	DialTimeout time.Duration

	// MaxTries, if non-zero, specifies the number of times we will retry on
	// failure. Retries are only attempted for temporary network errors or known
	// safe failures.
	MaxTries    int
	Deadline    DeadlineFunc
	ShouldRetry RetryableFunc
	Wait        WaitFunc
	transport   *http.Transport
}

// Convenience method for creating an http client
func NewClient(rt *ResilientTransport) *http.Client {
	rt.transport = &http.Transport{
		Dial: func(netw, addr string) (net.Conn, error) {
			c, err := net.DialTimeout(netw, addr, rt.DialTimeout)
			if err != nil {
				return nil, err
			}
			c.SetDeadline(rt.Deadline())
			return c, nil
		},
		Proxy:               http.ProxyFromEnvironment,
		MaxIdleConnsPerHost: 15,
	}
	// TODO: Would be nice is ResilientTransport allowed clients to initialize
	// with http.Transport attributes.
	return &http.Client{
		Transport: rt,
	}
}

var retryingTransport = &ResilientTransport{
	Deadline: func() time.Time {
		return time.Now().Add(5 * time.Second)
	},
	DialTimeout: 10 * time.Second,
	MaxTries:    3,
	ShouldRetry: awsRetry,
	Wait:        ExpBackoff,
}

// Exported default client
var RetryingClient = NewClient(retryingTransport)

func (t *ResilientTransport) RoundTrip(req *http.Request) (*http.Response, error) {
	return t.tries(req)
}

// Retry a request a maximum of t.MaxTries times.
// We'll only retry if the proper criteria are met.
// If a wait function is specified, wait that amount of time
// In between requests.
func (t *ResilientTransport) tries(req *http.Request) (res *http.Response, err error) {
	for try := 0; try < t.MaxTries; try += 1 {
		res, err = t.transport.RoundTrip(req)

		if !t.ShouldRetry(req, res, err) {
			break
		}
		if res != nil {
			res.Body.Close()
		}
		if t.Wait != nil {
			t.Wait(try)
		}
	}

	return
}

func ExpBackoff(try int) {
	time.Sleep(100 * time.Millisecond *
		time.Duration(math.Exp2(float64(try))))
}

func LinearBackoff(try int) {
	time.Sleep(time.Duration(try*100) * time.Millisecond)
}

// Decide if we should retry a request.
// In general, the criteria for retrying a request is described here
// http://docs.aws.amazon.com/general/latest/gr/api-retries.html
func awsRetry(req *http.Request, res *http.Response, err error) bool {
	retry := false

	// Retry if there's a temporary network error.
	if neterr, ok := err.(net.Error); ok {
		if neterr.Temporary() {
			retry = true
		}
	}

	// Retry if we get a 5xx series error.
	if res != nil {
		if res.StatusCode >= 500 && res.StatusCode < 600 {
			retry = true
		}
	}

	return retry
}
<|endoftext|>


<commit_before>package api

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"socialapi/workers/common/handler"
	"socialapi/workers/common/response"
	"socialapi/workers/realtime/models"
	"time"

	"github.com/koding/logging"
)

type Handler struct {
	pubnub *models.PubNub
	logger logging.Logger
}

func NewHandler(p *models.PubNub, l logging.Logger) *Handler {
	return &Handler{
		pubnub: p,
		logger: l,
	}
}

// SubscribeChannel checks users channel accessability and regarding to that
// grants channel access for them
func (h *Handler) SubscribeChannel(u *url.URL, header http.Header, req *models.Channel) (int, http.Header, interface{}, error) {
	res, err := checkParticipation(u, header, req)
	if err != nil {
		return response.NewAccessDenied(err)
	}

	// user has access permission, now authenticate user to channel via pubnub
	a := new(models.Authenticate)
	a.Channel = models.NewPrivateMessageChannel(*res.Channel)
	a.Account = res.Account

	err = h.pubnub.Authenticate(a)
	if err != nil {
		return response.NewBadRequest(err)
	}

	return responseWithCookie(req, a.Account.Token)
}

// SubscribeNotification grants notification channel access for user. User information is
// fetched from session
func (h *Handler) SubscribeNotification(u *url.URL, header http.Header, temp *models.Account) (int, http.Header, interface{}, error) {

	// fetch account information from session
	account, err := getAccountInfo(u, header)
	if err != nil {
		return response.NewBadRequest(err)
	}

	// authenticate user to their notification channel
	a := new(models.Authenticate)
	a.Channel = models.NewNotificationChannel(account)
	a.Account = account

	// TODO need async requests. Re-try in case of an error
	err = h.pubnub.Authenticate(a)
	if err != nil {
		return response.NewBadRequest(err)
	}

	return responseWithCookie(temp, account.Token)
}

func (h *Handler) SubscribeMessage(u *url.URL, header http.Header, um *models.UpdateInstanceMessage) (int, http.Header, interface{}, error) {
	if um.Token == "" {
		return response.NewBadRequest(models.ErrTokenNotSet)
	}

	a := new(models.Authenticate)
	a.Channel = models.NewMessageUpdateChannel(*um)
	err := h.pubnub.Authenticate(a)
	if err != nil {
		return response.NewBadRequest(err)
	}

	return response.NewOK(um)
}

func responseWithCookie(req interface{}, token string) (int, http.Header, interface{}, error) {
	expires := time.Now().AddDate(5, 0, 0)
	cookie := &http.Cookie{
		Name:       "realtimeToken",
		Value:      token,
		Path:       "/",
		Expires:    expires,
		RawExpires: expires.Format(time.UnixDate),
		Raw:        "realtimeToken=" + token,
		Unparsed:   []string{"realtimeToken=" + token},
	}

	return response.NewOKWithCookie(req, []*http.Cookie{cookie})
}

// TODO needs a better request handler
func checkParticipation(u *url.URL, header http.Header, cr *models.Channel) (*models.CheckParticipationResponse, error) {
	// relay the cookie to other endpoint
	cookie := header.Get("Cookie")
	request := &handler.Request{
		Type:     "GET",
		Endpoint: "/api/social/channel/checkparticipation",
		Params: map[string]string{
			"name":  cr.Name,
			"group": cr.Group,
			"type":  cr.Type,
		},
		Cookie: cookie,
	}

	// TODO update this requester
	resp, err := handler.MakeRequest(request)
	if err != nil {
		return nil, err
	}

	// Need a better response
	if resp.StatusCode != 200 {
		return nil, fmt.Errorf(resp.Status)
	}

	var cpr models.CheckParticipationResponse
	err = json.NewDecoder(resp.Body).Decode(&cpr)
	if err != nil {
		return nil, err
	}

	return &cpr, nil
}

func getAccountInfo(u *url.URL, header http.Header) (*models.Account, error) {
	cookie := header.Get("Cookie")
	request := &handler.Request{
		Type:     "GET",
		Endpoint: "/api/social/account",
		Cookie:   cookie,
	}

	// TODO update this requester
	resp, err := handler.MakeRequest(request)
	if err != nil {
		return nil, err
	}

	// Need a better response
	if resp.StatusCode != 200 {
		return nil, fmt.Errorf(resp.Status)
	}

	var a models.Account
	err = json.NewDecoder(resp.Body).Decode(&a)
	if err != nil {
		return nil, err
	}

	return &a, nil
}
<commit_msg>realtime: remove raw cookie part from request<commit_after>package api

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"socialapi/workers/common/handler"
	"socialapi/workers/common/response"
	"socialapi/workers/realtime/models"
	"time"

	"github.com/koding/logging"
)

type Handler struct {
	pubnub *models.PubNub
	logger logging.Logger
}

func NewHandler(p *models.PubNub, l logging.Logger) *Handler {
	return &Handler{
		pubnub: p,
		logger: l,
	}
}

// SubscribeChannel checks users channel accessability and regarding to that
// grants channel access for them
func (h *Handler) SubscribeChannel(u *url.URL, header http.Header, req *models.Channel) (int, http.Header, interface{}, error) {
	res, err := checkParticipation(u, header, req)
	if err != nil {
		return response.NewAccessDenied(err)
	}

	// user has access permission, now authenticate user to channel via pubnub
	a := new(models.Authenticate)
	a.Channel = models.NewPrivateMessageChannel(*res.Channel)
	a.Account = res.Account

	err = h.pubnub.Authenticate(a)
	if err != nil {
		return response.NewBadRequest(err)
	}

	return responseWithCookie(req, a.Account.Token)
}

// SubscribeNotification grants notification channel access for user. User information is
// fetched from session
func (h *Handler) SubscribeNotification(u *url.URL, header http.Header, temp *models.Account) (int, http.Header, interface{}, error) {

	// fetch account information from session
	account, err := getAccountInfo(u, header)
	if err != nil {
		return response.NewBadRequest(err)
	}

	// authenticate user to their notification channel
	a := new(models.Authenticate)
	a.Channel = models.NewNotificationChannel(account)
	a.Account = account

	// TODO need async requests. Re-try in case of an error
	err = h.pubnub.Authenticate(a)
	if err != nil {
		return response.NewBadRequest(err)
	}

	return responseWithCookie(temp, account.Token)
}

func (h *Handler) SubscribeMessage(u *url.URL, header http.Header, um *models.UpdateInstanceMessage) (int, http.Header, interface{}, error) {
	if um.Token == "" {
		return response.NewBadRequest(models.ErrTokenNotSet)
	}

	a := new(models.Authenticate)
	a.Channel = models.NewMessageUpdateChannel(*um)
	err := h.pubnub.Authenticate(a)
	if err != nil {
		return response.NewBadRequest(err)
	}

	return response.NewOK(um)
}

func responseWithCookie(req interface{}, token string) (int, http.Header, interface{}, error) {
	expires := time.Now().AddDate(5, 0, 0)
	cookie := &http.Cookie{
		Name:    "realtimeToken",
		Value:   token,
		Path:    "/",
		Expires: expires,
	}

	return response.NewOKWithCookie(req, []*http.Cookie{cookie})
}

// TODO needs a better request handler
func checkParticipation(u *url.URL, header http.Header, cr *models.Channel) (*models.CheckParticipationResponse, error) {
	// relay the cookie to other endpoint
	cookie := header.Get("Cookie")
	request := &handler.Request{
		Type:     "GET",
		Endpoint: "/api/social/channel/checkparticipation",
		Params: map[string]string{
			"name":  cr.Name,
			"group": cr.Group,
			"type":  cr.Type,
		},
		Cookie: cookie,
	}

	// TODO update this requester
	resp, err := handler.MakeRequest(request)
	if err != nil {
		return nil, err
	}

	// Need a better response
	if resp.StatusCode != 200 {
		return nil, fmt.Errorf(resp.Status)
	}

	var cpr models.CheckParticipationResponse
	err = json.NewDecoder(resp.Body).Decode(&cpr)
	if err != nil {
		return nil, err
	}

	return &cpr, nil
}

func getAccountInfo(u *url.URL, header http.Header) (*models.Account, error) {
	cookie := header.Get("Cookie")
	request := &handler.Request{
		Type:     "GET",
		Endpoint: "/api/social/account",
		Cookie:   cookie,
	}

	// TODO update this requester
	resp, err := handler.MakeRequest(request)
	if err != nil {
		return nil, err
	}

	// Need a better response
	if resp.StatusCode != 200 {
		return nil, fmt.Errorf(resp.Status)
	}

	var a models.Account
	err = json.NewDecoder(resp.Body).Decode(&a)
	if err != nil {
		return nil, err
	}

	return &a, nil
}
<|endoftext|>


<commit_before>package batchproducer

import (
	"errors"
	"log"
	"os"
	"sync"
	"time"

	"github.com/timehop/go-kinesis"
)

// Producer collects records individually and then sends them to Kinesis in
// batches in the background using PutRecords, with retries.
// A Producer will do nothing until Start is called.
type Producer interface {
	// Start starts the main goroutine. No need to call it using `go`.
	Start() error

	// Stop signals the main goroutine to finish. Once this is called, Add will immediately start
	// returning errors (unless and until Start is called again). Stop will block until
	// all remaining records in the buffer have been sent.
	Stop() error

	// Add might block if the BatchProducer has a buffer and the buffer is full.
	// In order to prevent filling the buffer and eventually blocking indefinitely,
	// Add will fail and return an error if the BatchProducer is stopped or stopping. Note
	// that it’s critical to check the return value because the BatchProducer could have
	// died in the background due to a panic (or something).
	Add(data []byte, partitionKey string) error
}

// StatReceiver defines an object that can accept stats.
type StatReceiver interface {
	// Receive will be called by the main Producer goroutine so it will block all batches from being
	// sent, so make sure it is either very fast or never blocks at all!
	Receive(StatsBatch)
}

// StatsBatch is a kind of a snapshot of activity and happenings. Some of its fields represent
// "moment-in-time" values e.g. BufferSize is the size of the buffer at the moment the StatsBatch
// is sent. Other fields are cumulative since the last StatsBatch, i.e. ErrorsSinceLastStat.
type StatsBatch struct {
	// Moment-in-time stats
	BufferSize int

	// Cumulative stats
	KinesisErrorsSinceLastStat           int
	RecordsSentSuccessfullySinceLastStat int
	RecordsDroppedSinceLastStat          int
}

// BatchingKinesisClient is a subset of KinesisClient to ease mocking.
type BatchingKinesisClient interface {
	PutRecords(args *kinesis.RequestArgs) (resp *kinesis.PutRecordsResp, err error)
}

// Config is a collection of config values for a Producer
type Config struct {
	// AddBlocksWhenBufferFull controls the behavior of Add when the buffer is full. If true, Add
	// will block. If false, Add will return an error. This enables integrating applications to
	// decide how they want to handle a full buffer e.g. so they can discard records if there’s
	// a problem.
	AddBlocksWhenBufferFull bool

	// BatchSize controls the maximum size of the batches sent to Kinesis. If the number of records
	// in the buffer hits this size, a batch of this size will be sent at that time, regardless of
	// whether FlushInterval has a value or not.
	BatchSize int

	// BufferSize is the size of the buffer that stores records before they are sent to the Kinesis
	// stream. If when Add is called the number of records in the buffer is >= bufferSize then
	// Add will either block or return an error, depending on the value of AddBlocksWhenBufferFull.
	BufferSize int

	// FlushInterval controls how often the buffer is flushed to Kinesis. If nonzero, then every
	// time this interval occurs, if there are any records in the buffer, they will be flushed,
	// no matter how few there are. The size of the batch that’s flushed may be as small as 1 but
	// will be no larger than BatchSize.
	FlushInterval time.Duration

	// The logger used by the Producer.
	Logger *log.Logger

	// MaxAttemptsPerRecord defines how many attempts should be made for each record before it is
	// dropped. You probably want this higher than the init default of 0.
	MaxAttemptsPerRecord int

	// StatInterval will be used to make a *best effort* attempt to send stats *approximately*
	// when this interval elapses. There’s no guarantee, however, since the main goroutine is
	// used to send the stats and therefore there may be some skew.
	StatInterval time.Duration

	// StatReceiver will have its Receive method called approximately every StatInterval.
	StatReceiver StatReceiver
}

// DefaultConfig is provided for convenience; if you have no specific preferences on how you’d
// like to configure your Producer you can pass this into New.
var DefaultConfig = Config{
	AddBlocksWhenBufferFull: false,
	BufferSize:              10000,
	FlushInterval:           1 * time.Second,
	BatchSize:               10,
	MaxAttemptsPerRecord:    10,
	StatInterval:            1 * time.Second,
	Logger:                  log.New(os.Stdout, "", log.LstdFlags),
}

// New creates and returns a BatchProducer that will do nothing until its Start method is called.
// Once it is started, it will flush a batch to Kinesis whenever either
// the flushInterval occurs (if flushInterval > 0) or the batchSize is reached,
// whichever happens first.
func New(
	client BatchingKinesisClient,
	streamName string,
	config Config,
) (Producer, error) {
	if config.BatchSize < 1 || config.BatchSize > 500 {
		return nil, errors.New("BatchSize must be between 1 and 500 inclusive")
	}

	if config.BufferSize < config.BatchSize && config.FlushInterval <= 0 {
		return nil, errors.New("If BufferSize < BatchSize && FlushInterval <= 0 then the buffer will eventually fill up and Add will block forever.")
	}

	if config.FlushInterval > 0 && config.FlushInterval < 50*time.Millisecond {
		return nil, errors.New("Are you crazy?")
	}

	batchProducer := batchProducer{
		client:      client,
		streamName:  streamName,
		config:      config,
		logger:      config.Logger,
		currentStat: new(StatsBatch),
		records:     make(chan batchRecord, config.BufferSize),
		stop:        make(chan interface{}),
	}

	return &batchProducer, nil
}

type batchProducer struct {
	client            BatchingKinesisClient
	streamName        string
	config            Config
	logger            *log.Logger
	running           bool
	runningMu         sync.Mutex
	consecutiveErrors int
	currentDelay      time.Duration
	currentStat       *StatsBatch
	records           chan batchRecord
	stop              chan interface{}
}

type batchRecord struct {
	data         []byte
	partitionKey string
	sendAttempts int
}

// from/for interface BatchProducer
func (b *batchProducer) Add(data []byte, partitionKey string) error {
	if !b.isRunning() {
		return errors.New("Cannot call Add when BatchProducer is not running (to prevent the buffer filling up and Add blocking indefinitely).")
	}
	if b.isBufferFull() && !b.config.AddBlocksWhenBufferFull {
		return errors.New("Buffer is full")
	}
	b.records <- batchRecord{data: data, partitionKey: partitionKey}
	return nil
}

// from/for interface BatchProducer
func (b *batchProducer) Start() error {
	if b.isRunning() {
		return nil
	}

	go b.run()

	for !b.isRunning() {
		time.Sleep(1 * time.Millisecond)
	}

	return nil
}

func (b *batchProducer) run() {
	flushTicker := &time.Ticker{}
	if b.config.FlushInterval > 0 {
		flushTicker = time.NewTicker(b.config.FlushInterval)
		defer flushTicker.Stop()
	}

	statTicker := &time.Ticker{}
	if b.config.StatReceiver != nil && b.config.StatInterval > 0 {
		statTicker = time.NewTicker(b.config.StatInterval)
		defer statTicker.Stop()
	}

	b.setRunning(true)
	defer b.setRunning(false)

	for {
		select {
		case <-flushTicker.C:
			b.sendBatch()
		case <-statTicker.C:
			b.sendStats()
		case <-b.stop:
			break
		default:
			if len(b.records) >= b.config.BatchSize {
				b.sendBatch()
			} else {
				time.Sleep(1 * time.Millisecond)
			}
		}
	}
}

// from/for interface BatchProducer
func (b *batchProducer) Stop() error {
	// TODO: Immediately stop accepting new records by Add, then block until all the records in the buffer have been sent
	if b.isRunning() {
		b.stop <- true
	}
	return nil
}

func (b *batchProducer) setRunning(running bool) {
	b.runningMu.Lock()
	defer b.runningMu.Unlock()
	b.running = running
}

func (b *batchProducer) isRunning() bool {
	b.runningMu.Lock()
	defer b.runningMu.Unlock()
	return b.running
}

func (b *batchProducer) sendBatch() {
	if len(b.records) == 0 {
		return
	}

	// In the future, maybe this could be a RetryPolicy or something
	if b.consecutiveErrors == 1 {
		b.currentDelay = 50 * time.Millisecond
	} else if b.consecutiveErrors > 1 {
		b.currentDelay *= 2
	}

	if b.currentDelay > 0 {
		b.logger.Printf("Delaying the batch by %v because of %v consecutive errors", b.currentDelay, b.consecutiveErrors)
		time.Sleep(b.currentDelay)
	}

	records := b.takeRecordsFromBuffer()
	res, err := b.client.PutRecords(b.recordsToArgs(records))

	if err != nil {
		b.consecutiveErrors++
		b.currentStat.KinesisErrorsSinceLastStat++
		b.logger.Printf("Error occurred when sending PutRecords request to Kinesis stream %v: %v", b.streamName, err)

		if b.consecutiveErrors >= 5 && b.isBufferFullOrNearlyFull() {
			// In order to prevent Add from hanging indefinitely, we start dropping records
			b.logger.Printf("DROPPING %v records because buffer is full or nearly full and there have been %v consecutive errors from Kinesis", len(records), b.consecutiveErrors)
		} else {
			b.logger.Printf("Returning %v records to buffer (%v consecutive errors)", len(records), b.consecutiveErrors)
			b.returnRecordsToBuffer(records)
		}

		return
	}

	b.consecutiveErrors = 0
	b.currentDelay = 0
	succeeded := len(records) - res.FailedRecordCount

	b.currentStat.RecordsSentSuccessfullySinceLastStat += succeeded

	if res.FailedRecordCount == 0 {
		b.logger.Printf("PutRecords request succeeded: sent %v records to Kinesis stream %v", succeeded, b.streamName)
	} else {
		b.logger.Printf("Partial success when sending a PutRecords request to Kinesis stream %v: %v succeeded, %v failed. Re-enqueueing failed records.", b.streamName, succeeded, res.FailedRecordCount)
		b.returnSomeFailedRecordsToBuffer(res, records)
	}
}

func (b *batchProducer) isBufferFullOrNearlyFull() bool {
	return float32(len(b.records))/float32(cap(b.records)) >= 0.95
}

func (b *batchProducer) isBufferFull() bool {
	// Treating 99% as full because IIRC, len(chan) has a margin of error
	return float32(len(b.records))/float32(cap(b.records)) >= 0.99
}

func (b *batchProducer) takeRecordsFromBuffer() []batchRecord {
	var size int
	bufferLen := len(b.records)
	if bufferLen >= b.config.BatchSize {
		size = b.config.BatchSize
	} else {
		size = bufferLen
	}

	result := make([]batchRecord, size)
	for i := 0; i < size; i++ {
		result[i] = <-b.records
	}
	return result
}

func (b *batchProducer) recordsToArgs(records []batchRecord) *kinesis.RequestArgs {
	args := kinesis.NewArgs()
	args.Add("StreamName", b.streamName)
	for _, record := range records {
		args.AddRecord(record.data, record.partitionKey)
	}
	return args
}

// TODO: perhaps we should use a deque internally as the buffer so we can return records to
// the front of the queue.
func (b *batchProducer) returnRecordsToBuffer(records []batchRecord) {
	for _, record := range records {
		// Not using b.Add because we want to preserve the value of record.sendAttempts.
		b.records <- record
	}
}

func (b *batchProducer) returnSomeFailedRecordsToBuffer(res *kinesis.PutRecordsResp, records []batchRecord) {
	for i, result := range res.Records {
		record := records[i]
		if result.ErrorCode != "" {
			record.sendAttempts++

			if record.sendAttempts < b.config.MaxAttemptsPerRecord {
				b.logger.Printf("Re-enqueueing failed record to buffer for retry. Error code was: '%v' and message was '%v'", result.ErrorCode, result.ErrorMessage)
				// Not using b.Add because we want to preserve the value of record.sendAttempts.
				b.records <- record
			} else {
				b.currentStat.RecordsDroppedSinceLastStat++
				msg := "Dropping failed record; it has hit %v attempts " +
					"which is the maximum. Error code was: '%v' and message was '%v'."
				b.logger.Printf(msg, record.sendAttempts, result.ErrorCode, result.ErrorMessage)
			}
		}
	}
}

func (b *batchProducer) sendStats() {
	b.currentStat.BufferSize = len(b.records)

	// I considered running this as a goroutine, but I’m concerned about leaks. So instead, for now,
	// the provider of the BatchStatReceiver must ensure that it is either very fast or non-blocking.
	b.config.StatReceiver.Receive(*b.currentStat)

	b.currentStat = new(StatsBatch)
}
<commit_msg>Bit of cleanup on 8b80fc9<commit_after>package batchproducer

import (
	"errors"
	"log"
	"os"
	"sync"
	"time"

	"github.com/timehop/go-kinesis"
)

// Producer collects records individually and then sends them to Kinesis in
// batches in the background using PutRecords, with retries.
// A Producer will do nothing until Start is called.
type Producer interface {
	// Start starts the main goroutine. No need to call it using `go`.
	Start() error

	// Stop signals the main goroutine to finish. Once this is called, Add will immediately start
	// returning errors (unless and until Start is called again). Stop will block until
	// all remaining records in the buffer have been sent.
	Stop() error

	// Add might block if the BatchProducer has a buffer and the buffer is full.
	// In order to prevent filling the buffer and eventually blocking indefinitely,
	// Add will fail and return an error if the BatchProducer is stopped or stopping. Note
	// that it’s critical to check the return value because the BatchProducer could have
	// died in the background due to a panic (or something).
	Add(data []byte, partitionKey string) error
}

// StatReceiver defines an object that can accept stats.
type StatReceiver interface {
	// Receive will be called by the main Producer goroutine so it will block all batches from being
	// sent, so make sure it is either very fast or never blocks at all!
	Receive(StatsBatch)
}

// StatsBatch is a kind of a snapshot of activity and happenings. Some of its fields represent
// "moment-in-time" values e.g. BufferSize is the size of the buffer at the moment the StatsBatch
// is sent. Other fields are cumulative since the last StatsBatch, i.e. ErrorsSinceLastStat.
type StatsBatch struct {
	// Moment-in-time stats
	BufferSize int

	// Cumulative stats
	KinesisErrorsSinceLastStat           int
	RecordsSentSuccessfullySinceLastStat int
	RecordsDroppedSinceLastStat          int
}

// BatchingKinesisClient is a subset of KinesisClient to ease mocking.
type BatchingKinesisClient interface {
	PutRecords(args *kinesis.RequestArgs) (resp *kinesis.PutRecordsResp, err error)
}

// Config is a collection of config values for a Producer
type Config struct {
	// AddBlocksWhenBufferFull controls the behavior of Add when the buffer is full. If true, Add
	// will block. If false, Add will return an error. This enables integrating applications to
	// decide how they want to handle a full buffer e.g. so they can discard records if there’s
	// a problem.
	AddBlocksWhenBufferFull bool

	// BatchSize controls the maximum size of the batches sent to Kinesis. If the number of records
	// in the buffer hits this size, a batch of this size will be sent at that time, regardless of
	// whether FlushInterval has a value or not.
	BatchSize int

	// BufferSize is the size of the buffer that stores records before they are sent to the Kinesis
	// stream. If when Add is called the number of records in the buffer is >= bufferSize then
	// Add will either block or return an error, depending on the value of AddBlocksWhenBufferFull.
	BufferSize int

	// FlushInterval controls how often the buffer is flushed to Kinesis. If nonzero, then every
	// time this interval occurs, if there are any records in the buffer, they will be flushed,
	// no matter how few there are. The size of the batch that’s flushed may be as small as 1 but
	// will be no larger than BatchSize.
	FlushInterval time.Duration

	// The logger used by the Producer.
	Logger *log.Logger

	// MaxAttemptsPerRecord defines how many attempts should be made for each record before it is
	// dropped. You probably want this higher than the init default of 0.
	MaxAttemptsPerRecord int

	// StatInterval will be used to make a *best effort* attempt to send stats *approximately*
	// when this interval elapses. There’s no guarantee, however, since the main goroutine is
	// used to send the stats and therefore there may be some skew.
	StatInterval time.Duration

	// StatReceiver will have its Receive method called approximately every StatInterval.
	StatReceiver StatReceiver
}

// DefaultConfig is provided for convenience; if you have no specific preferences on how you’d
// like to configure your Producer you can pass this into New. The default value of Logger is
// the same as the standard logger in "log" : `log.New(os.Stderr, "", log.LstdFlags)`.
var DefaultConfig = Config{
	AddBlocksWhenBufferFull: false,
	BufferSize:              10000,
	FlushInterval:           1 * time.Second,
	BatchSize:               10,
	MaxAttemptsPerRecord:    10,
	StatInterval:            1 * time.Second,
	Logger:                  log.New(os.Stderr, "", log.LstdFlags),
}

// New creates and returns a BatchProducer that will do nothing until its Start method is called.
// Once it is started, it will flush a batch to Kinesis whenever either
// the flushInterval occurs (if flushInterval > 0) or the batchSize is reached,
// whichever happens first.
func New(
	client BatchingKinesisClient,
	streamName string,
	config Config,
) (Producer, error) {
	if config.BatchSize < 1 || config.BatchSize > 500 {
		return nil, errors.New("BatchSize must be between 1 and 500 inclusive")
	}

	if config.BufferSize < config.BatchSize && config.FlushInterval <= 0 {
		return nil, errors.New("if BufferSize < BatchSize && FlushInterval <= 0 then the buffer will eventually fill up and Add will block forever")
	}

	if config.FlushInterval > 0 && config.FlushInterval < 50*time.Millisecond {
		return nil, errors.New("are you crazy")
	}

	batchProducer := batchProducer{
		client:      client,
		streamName:  streamName,
		config:      config,
		logger:      config.Logger,
		currentStat: new(StatsBatch),
		records:     make(chan batchRecord, config.BufferSize),
		stop:        make(chan interface{}),
	}

	return &batchProducer, nil
}

type batchProducer struct {
	client            BatchingKinesisClient
	streamName        string
	config            Config
	logger            *log.Logger
	running           bool
	runningMu         sync.Mutex
	consecutiveErrors int
	currentDelay      time.Duration
	currentStat       *StatsBatch
	records           chan batchRecord
	stop              chan interface{}
}

type batchRecord struct {
	data         []byte
	partitionKey string
	sendAttempts int
}

// from/for interface BatchProducer
func (b *batchProducer) Add(data []byte, partitionKey string) error {
	if !b.isRunning() {
		return errors.New("Cannot call Add when BatchProducer is not running (to prevent the buffer filling up and Add blocking indefinitely).")
	}
	if b.isBufferFull() && !b.config.AddBlocksWhenBufferFull {
		return errors.New("Buffer is full")
	}
	b.records <- batchRecord{data: data, partitionKey: partitionKey}
	return nil
}

// from/for interface BatchProducer
func (b *batchProducer) Start() error {
	if b.isRunning() {
		return nil
	}

	go b.run()

	for !b.isRunning() {
		time.Sleep(1 * time.Millisecond)
	}

	return nil
}

func (b *batchProducer) run() {
	flushTicker := &time.Ticker{}
	if b.config.FlushInterval > 0 {
		flushTicker = time.NewTicker(b.config.FlushInterval)
		defer flushTicker.Stop()
	}

	statTicker := &time.Ticker{}
	if b.config.StatReceiver != nil && b.config.StatInterval > 0 {
		statTicker = time.NewTicker(b.config.StatInterval)
		defer statTicker.Stop()
	}

	b.setRunning(true)
	defer b.setRunning(false)

	for {
		select {
		case <-flushTicker.C:
			b.sendBatch()
		case <-statTicker.C:
			b.sendStats()
		case <-b.stop:
			break
		default:
			if len(b.records) >= b.config.BatchSize {
				b.sendBatch()
			} else {
				time.Sleep(1 * time.Millisecond)
			}
		}
	}
}

// from/for interface BatchProducer
func (b *batchProducer) Stop() error {
	// TODO: Immediately stop accepting new records by Add, then block until all the records in the buffer have been sent
	if b.isRunning() {
		b.stop <- true
	}
	return nil
}

func (b *batchProducer) setRunning(running bool) {
	b.runningMu.Lock()
	defer b.runningMu.Unlock()
	b.running = running
}

func (b *batchProducer) isRunning() bool {
	b.runningMu.Lock()
	defer b.runningMu.Unlock()
	return b.running
}

func (b *batchProducer) sendBatch() {
	if len(b.records) == 0 {
		return
	}

	// In the future, maybe this could be a RetryPolicy or something
	if b.consecutiveErrors == 1 {
		b.currentDelay = 50 * time.Millisecond
	} else if b.consecutiveErrors > 1 {
		b.currentDelay *= 2
	}

	if b.currentDelay > 0 {
		b.logger.Printf("Delaying the batch by %v because of %v consecutive errors", b.currentDelay, b.consecutiveErrors)
		time.Sleep(b.currentDelay)
	}

	records := b.takeRecordsFromBuffer()
	res, err := b.client.PutRecords(b.recordsToArgs(records))

	if err != nil {
		b.consecutiveErrors++
		b.currentStat.KinesisErrorsSinceLastStat++
		b.logger.Printf("Error occurred when sending PutRecords request to Kinesis stream %v: %v", b.streamName, err)

		if b.consecutiveErrors >= 5 && b.isBufferFullOrNearlyFull() {
			// In order to prevent Add from hanging indefinitely, we start dropping records
			b.logger.Printf("DROPPING %v records because buffer is full or nearly full and there have been %v consecutive errors from Kinesis", len(records), b.consecutiveErrors)
		} else {
			b.logger.Printf("Returning %v records to buffer (%v consecutive errors)", len(records), b.consecutiveErrors)
			b.returnRecordsToBuffer(records)
		}

		return
	}

	b.consecutiveErrors = 0
	b.currentDelay = 0
	succeeded := len(records) - res.FailedRecordCount

	b.currentStat.RecordsSentSuccessfullySinceLastStat += succeeded

	if res.FailedRecordCount == 0 {
		b.logger.Printf("PutRecords request succeeded: sent %v records to Kinesis stream %v", succeeded, b.streamName)
	} else {
		b.logger.Printf("Partial success when sending a PutRecords request to Kinesis stream %v: %v succeeded, %v failed. Re-enqueueing failed records.", b.streamName, succeeded, res.FailedRecordCount)
		b.returnSomeFailedRecordsToBuffer(res, records)
	}
}

func (b *batchProducer) isBufferFullOrNearlyFull() bool {
	return float32(len(b.records))/float32(cap(b.records)) >= 0.95
}

func (b *batchProducer) isBufferFull() bool {
	// Treating 99% as full because IIRC, len(chan) has a margin of error
	return float32(len(b.records))/float32(cap(b.records)) >= 0.99
}

func (b *batchProducer) takeRecordsFromBuffer() []batchRecord {
	var size int
	bufferLen := len(b.records)
	if bufferLen >= b.config.BatchSize {
		size = b.config.BatchSize
	} else {
		size = bufferLen
	}

	result := make([]batchRecord, size)
	for i := 0; i < size; i++ {
		result[i] = <-b.records
	}
	return result
}

func (b *batchProducer) recordsToArgs(records []batchRecord) *kinesis.RequestArgs {
	args := kinesis.NewArgs()
	args.Add("StreamName", b.streamName)
	for _, record := range records {
		args.AddRecord(record.data, record.partitionKey)
	}
	return args
}

// TODO: perhaps we should use a deque internally as the buffer so we can return records to
// the front of the queue.
func (b *batchProducer) returnRecordsToBuffer(records []batchRecord) {
	for _, record := range records {
		// Not using b.Add because we want to preserve the value of record.sendAttempts.
		b.records <- record
	}
}

func (b *batchProducer) returnSomeFailedRecordsToBuffer(res *kinesis.PutRecordsResp, records []batchRecord) {
	for i, result := range res.Records {
		record := records[i]
		if result.ErrorCode != "" {
			record.sendAttempts++

			if record.sendAttempts < b.config.MaxAttemptsPerRecord {
				b.logger.Printf("Re-enqueueing failed record to buffer for retry. Error code was: '%v' and message was '%v'", result.ErrorCode, result.ErrorMessage)
				// Not using b.Add because we want to preserve the value of record.sendAttempts.
				b.records <- record
			} else {
				b.currentStat.RecordsDroppedSinceLastStat++
				msg := "Dropping failed record; it has hit %v attempts " +
					"which is the maximum. Error code was: '%v' and message was '%v'."
				b.logger.Printf(msg, record.sendAttempts, result.ErrorCode, result.ErrorMessage)
			}
		}
	}
}

func (b *batchProducer) sendStats() {
	b.currentStat.BufferSize = len(b.records)

	// I considered running this as a goroutine, but I’m concerned about leaks. So instead, for now,
	// the provider of the BatchStatReceiver must ensure that it is either very fast or non-blocking.
	b.config.StatReceiver.Receive(*b.currentStat)

	b.currentStat = new(StatsBatch)
}
<|endoftext|>


<commit_before>// cloudhealthtestserver
//
// This application is a fake cloudhealth endpoint. It accepts requests
// for the *real* cloud health endpoint and writes the data to lmm.
//
// cloudhealthtestserver accepts incoming cloudhealth requests at
// http://localhost:7776/endpoint
//
// For this service to work, create a config file at
// /etc/cloudhealthtestserver/lmm.yaml that describes how to connect to lmm.
// the format of this file looks like:
//
// 	endpoints:
//	- "some-kafka-endpoint-for-lmm.net:9092"
//	topic: "awsMetricTopic"
//	apiKey: "your lmm api key goes here"
//	tenantId: "your lmm tenant id goes here"
// 	clientId: "scotty"
//
package main

import (
	"bufio"
	"flag"
	"fmt"
	"github.com/Symantec/Dominator/lib/html"
	"github.com/Symantec/Dominator/lib/logbuf"
	"github.com/Symantec/scotty/lib/dynconfig"
	"github.com/Symantec/scotty/lib/trimetrics"
	"github.com/Symantec/tricorder/go/tricorder"
	"io"
	"log"
	"net/http"
	"time"
)

var (
	fPort = flag.Int(
		"portNum",
		7776,
		"Port number for cloudhealthtestserver.")
	fConfigDir = flag.String(
		"configDir",
		"/etc/cloudhealthtestserver",
		"Config directory location.")
)

type endpointHandler struct {
	Logger  *log.Logger
	Lmm     *dynconfig.DynConfig
	Metrics *trimetrics.WriterMetrics
}

func (h *endpointHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	r.ParseForm()
	dryRun := r.Form.Get("dryrun") != ""
	metrics, err := extractMetricsFromBody(r.Body)
	if err != nil {
		w.WriteHeader(400)
		fmt.Fprintf(w, `{ "error": "%v" }`, err)
		return
	}
	if !dryRun {
		start := time.Now()
		if err := h.Lmm.Get().(*lmmWriterType).Write(metrics); err != nil {
			h.Logger.Println("Error writing to LMM: ", err)
			h.Metrics.LogError(time.Since(start), uint64(len(metrics)), err)
		} else {
			h.Metrics.LogSuccess(time.Since(start), uint64(len(metrics)))
		}

	}
	fmt.Fprintln(w, "{")
	fmt.Fprintf(w, "  succeeded: %d,", len(metrics))
	fmt.Fprintln(w, "  failed: 0,")
	fmt.Fprintln(w, "  errors: 0")
	fmt.Fprintln(w, "}")
}

type htmlWriter interface {
	WriteHtml(writer io.Writer)
}

type splashHandler struct {
	Log htmlWriter
}

func (h *splashHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	writer := bufio.NewWriter(w)
	defer writer.Flush()
	w.Header().Set("Content-Type", "text/html")
	fmt.Fprintln(writer, "")
	fmt.Fprintln(writer, "<title>CloudHealthTestServer status page</title>")
	fmt.Fprintln(writer, "")
	fmt.Fprintln(writer, "")
	fmt.Fprintln(writer, "
CloudHealthTestServer status page
")
	fmt.Fprintln(writer, "")
	html.WriteHeaderNoGC(writer)
	fmt.Fprintln(writer, "
")
	h.Log.WriteHtml(writer)
	fmt.Fprintln(writer, "")
	fmt.Fprintln(writer, "")

}

func main() {
	tricorder.RegisterFlags()
	flag.Parse()
	circularBuffer := logbuf.New()
	logger := log.New(circularBuffer, "", log.LstdFlags)

	lmm, err := newLmmConfig(*fConfigDir, logger)
	if err != nil {
		log.Fatal(err)
	}

	http.Handle(
		"/",
		&splashHandler{
			Log: circularBuffer,
		})

	metrics, err := trimetrics.NewWriterMetrics("/lmm")
	if err != nil {
		log.Fatal(err)
	}

	http.Handle(
		"/endpoint",
		&endpointHandler{
			Logger:  logger,
			Lmm:     lmm,
			Metrics: metrics,
		})

	if err := http.ListenAndServe(fmt.Sprintf(":%d", *fPort), nil); err != nil {
		log.Fatal(err)
	}
}
<commit_msg>mock cloudhealth server forwards requests onto real cloudhealth server.<commit_after>// cloudhealthtestserver
//
// This application is a fake cloudhealth endpoint. It accepts requests
// for the *real* cloud health endpoint and writes the data to lmm. Finally,
// it forwards requests onto the *real* cloud health endpoint.
//
// cloudhealthtestserver accepts incoming cloudhealth requests at
// http://localhost:7776/endpoint
//
// For this service to work, create a config file at
// /etc/cloudhealthtestserver/lmm.yaml that describes how to connect to lmm.
// the format of this file looks like:
//
// 	endpoints:
//	- "some-kafka-endpoint-for-lmm.net:9092"
//	topic: "awsMetricTopic"
//	apiKey: "your lmm api key goes here"
//	tenantId: "your lmm tenant id goes here"
// 	clientId: "scotty"
//
package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"github.com/Symantec/Dominator/lib/html"
	"github.com/Symantec/Dominator/lib/logbuf"
	"github.com/Symantec/scotty/cloudhealth"
	"github.com/Symantec/scotty/lib/dynconfig"
	"github.com/Symantec/scotty/lib/trimetrics"
	"github.com/Symantec/tricorder/go/tricorder"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/http/httputil"
	"net/url"
	"time"
)

var (
	fPort = flag.Int(
		"portNum",
		7776,
		"Port number for cloudhealthtestserver.")
	fConfigDir = flag.String(
		"configDir",
		"/etc/cloudhealthtestserver",
		"Config directory location.")
)

var (
	kReverseProxy = newReverseProxy(cloudhealth.DefaultEndpoint)
)

func newReverseProxy(URL string) *httputil.ReverseProxy {
	pURL, err := url.Parse(URL)
	if err != nil {
		log.Fatal(err)
	}
	return httputil.NewSingleHostReverseProxy(pURL)
}

type endpointHandler struct {
	Logger  *log.Logger
	Lmm     *dynconfig.DynConfig
	Metrics *trimetrics.WriterMetrics
}

func extractAsBytes(r io.Reader) []byte {
	var buffer bytes.Buffer
	buffer.ReadFrom(r)
	return buffer.Bytes()
}

func (h *endpointHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	r.ParseForm()
	dryRun := r.Form.Get("dryrun") != ""

	// Ultimately, we have to proxy this request onto the real cloudhealth
	// service, but when we read the request body we exhaust it so that it
	// won't get passed on as we want. To get around this, we extract the
	// request body as a slice of bytes and then create a new request body
	// off that slice of bytes.
	bodyAsBytes := extractAsBytes(r.Body)

	// Close the original body as our transport layer won't be able to do
	// this for us.
	r.Body.Close()

	// Now set the body to a byte buffer of the original body so that it
	// can get read again.
	r.Body = ioutil.NopCloser(bytes.NewBuffer(bodyAsBytes))

	// Here we have to promote our content to a stream to call this function
	metrics, err := extractMetricsFromBody(bytes.NewBuffer(bodyAsBytes))
	if err != nil {
		h.Logger.Println(err)
	}
	if !dryRun {
		start := time.Now()
		if err := h.Lmm.Get().(*lmmWriterType).Write(metrics); err != nil {
			h.Logger.Println("Error writing to LMM: ", err)
			h.Metrics.LogError(time.Since(start), uint64(len(metrics)), err)
		} else {
			h.Metrics.LogSuccess(time.Since(start), uint64(len(metrics)))
		}
	}

	// Because of how the reverse proxy works, we have to make sure the
	// path of the request we send to the proxy is empty without changing
	// the original request

	// Make defensive copy to prevent changing original request
	newReq := *r
	{
		// URL is a pointer field so we have to make another defensive copy
		newUrl := *r.URL
		newReq.URL = &newUrl
	}
	// zero out the path
	newReq.URL.Path = ""
	kReverseProxy.ServeHTTP(w, &newReq)
}

type htmlWriter interface {
	WriteHtml(writer io.Writer)
}

type splashHandler struct {
	Log htmlWriter
}

func (h *splashHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	writer := bufio.NewWriter(w)
	defer writer.Flush()
	w.Header().Set("Content-Type", "text/html")
	fmt.Fprintln(writer, "")
	fmt.Fprintln(writer, "<title>CloudHealthTestServer status page</title>")
	fmt.Fprintln(writer, "")
	fmt.Fprintln(writer, "")
	fmt.Fprintln(writer, "
CloudHealthTestServer status page
")
	fmt.Fprintln(writer, "")
	html.WriteHeaderNoGC(writer)
	fmt.Fprintln(writer, "
")
	h.Log.WriteHtml(writer)
	fmt.Fprintln(writer, "")
	fmt.Fprintln(writer, "")

}

func main() {
	tricorder.RegisterFlags()
	flag.Parse()
	circularBuffer := logbuf.New()
	logger := log.New(circularBuffer, "", log.LstdFlags)

	lmm, err := newLmmConfig(*fConfigDir, logger)
	if err != nil {
		log.Fatal(err)
	}

	http.Handle(
		"/",
		&splashHandler{
			Log: circularBuffer,
		})

	metrics, err := trimetrics.NewWriterMetrics("/lmm")
	if err != nil {
		log.Fatal(err)
	}

	http.Handle(
		"/endpoint",
		&endpointHandler{
			Logger:  logger,
			Lmm:     lmm,
			Metrics: metrics,
		})

	if err := http.ListenAndServe(fmt.Sprintf(":%d", *fPort), nil); err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2017-2022 The Usacloud Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package version

import (
	"fmt"
	"runtime"
)

var (
	// Version app version
	Version = "1.7.1"
	// Revision git commit short commithash
	Revision = "xxxxxx" // set on build time

	// CopyrightYear .
	CopyrightYear = "2017-2022"
)

// FullVersion return usacloud full version text
func FullVersion() string {
	return fmt.Sprintf("%s %s/%s, build %s", Version, runtime.GOOS, runtime.GOARCH, Revision)
}
<commit_msg>Bump to v1.8.0<commit_after>// Copyright 2017-2022 The Usacloud Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package version

import (
	"fmt"
	"runtime"
)

var (
	// Version app version
	Version = "1.8.0"
	// Revision git commit short commithash
	Revision = "xxxxxx" // set on build time

	// CopyrightYear .
	CopyrightYear = "2017-2022"
)

// FullVersion return usacloud full version text
func FullVersion() string {
	return fmt.Sprintf("%s %s/%s, build %s", Version, runtime.GOOS, runtime.GOARCH, Revision)
}
<|endoftext|>


<commit_before>// Copyright (c) 2017-2019 Snowflake Computing Inc. All right reserved.

package gosnowflake

// SnowflakeGoDriverVersion is the version of Go Snowflake Driver.
const SnowflakeGoDriverVersion = "1.3.7"
<commit_msg>Did Version Bump to 1.3.8<commit_after>// Copyright (c) 2017-2019 Snowflake Computing Inc. All right reserved.

package gosnowflake

// SnowflakeGoDriverVersion is the version of Go Snowflake Driver.
const SnowflakeGoDriverVersion = "1.3.8"
<|endoftext|>