<commit_before>package auth0_test

import (
	"os"
	"testing"

	"github.com/markbates/goth"
	"github.com/markbates/goth/providers/auth0"
	"github.com/stretchr/testify/assert"
)

func Test_New(t *testing.T) {
	t.Parallel()
	a := assert.New(t)
	p := provider()

	a.Equal(p.ClientKey, os.Getenv("AUTH0_KEY"))
	a.Equal(p.Secret, os.Getenv("AUTH0_SECRET"))
	a.Equal(p.Domain, os.Getenv("AUTH0_DOMAIN"))
	a.Equal(p.CallbackURL, "/foo")
}

func Test_Implements_Provider(t *testing.T) {
	t.Parallel()
	a := assert.New(t)
	a.Implements((*goth.Provider)(nil), provider())
}

func Test_BeginAuth(t *testing.T) {
	t.Parallel()
	a := assert.New(t)
	p := provider()
	session, err := p.BeginAuth("test_state")
	s := session.(*auth0.Session)
	a.NoError(err)
	expectedAuthURL := "https://" + os.Getenv("AUTH0_DOMAIN") + "/authorize"
	a.Contains(s.AuthURL, expectedAuthURL)
}

func Test_SessionFromJSON(t *testing.T) {
	t.Parallel()
	a := assert.New(t)

	p := provider()
	sessionResp := `{"AuthURL":"https://` + p.Domain + `/oauth/authorize","AccessToken":"1234567890"}`
	session, err := p.UnmarshalSession(sessionResp)
	a.NoError(err)

	s := session.(*auth0.Session)
	expectedAuthURL := "https://" + os.Getenv("AUTH0_DOMAIN") + "/oauth/authorize"
	a.Equal(s.AuthURL, expectedAuthURL)
	a.Equal(s.AccessToken, "1234567890")
}

func Test_FetchUser(t *testing.T) {
	//t.Parallel()
	a := assert.New(t)

	httpmock.Activate()
	defer httpmock.DeactivateAndReset()

	sampleResp := `{
  		"email_verified": false,
  		"email": "test.account@userinfo.com",
  		"clientID": "q2hnj2iu...",
  		"updated_at": "2016-12-05T15:15:40.545Z",
  		"name": "test.account@userinfo.com",
  		"picture": "https://s.gravatar.com/avatar/dummy.png",
  		"user_id": "auth0|58454...",
  		"nickname": "test.account",
  		"identities": [
  		  {
      			"user_id": "58454...",
      			"provider": "auth0",
      			"connection": "Username-Password-Authentication",
      			"isSocial": false
    		}],
  		"created_at": "2016-12-05T11:16:59.640Z",
  		"sub": "auth0|58454..."
	}`

	httpmock.RegisterResponder("GET", "https://"+os.Getenv("AUTH0_DOMAIN")+"/userinfo",
		httpmock.NewStringResponder(200, sampleResp))

	p := provider()
	session, err := p.BeginAuth("test_state")
	s := session.(*auth0.Session)
	s.AccessToken = "token"
	u, err := p.FetchUser(s)
	a.Nil(err)
	a.Equal(u.Email, "test.account@userinfo.com")
	a.Equal(u.UserID, "auth0|58454...")
	a.Equal(u.NickName, "test.account")
	a.Equal(u.Name, "test.account@userinfo.com")
	a.Equal("token", u.AccessToken)

}

func provider() *auth0.Provider {
	return auth0.New(os.Getenv("AUTH0_KEY"), os.Getenv("AUTH0_SECRET"), "/foo", os.Getenv("AUTH0_DOMAIN"))
}
<commit_msg>fixing tests<commit_after>package auth0_test

import (
	"os"
	"testing"

	"github.com/jarcoal/httpmock"
	"github.com/markbates/goth"
	"github.com/markbates/goth/providers/auth0"
	"github.com/stretchr/testify/assert"
)

func Test_New(t *testing.T) {
	t.Parallel()
	a := assert.New(t)
	p := provider()

	a.Equal(p.ClientKey, os.Getenv("AUTH0_KEY"))
	a.Equal(p.Secret, os.Getenv("AUTH0_SECRET"))
	a.Equal(p.Domain, os.Getenv("AUTH0_DOMAIN"))
	a.Equal(p.CallbackURL, "/foo")
}

func Test_Implements_Provider(t *testing.T) {
	t.Parallel()
	a := assert.New(t)
	a.Implements((*goth.Provider)(nil), provider())
}

func Test_BeginAuth(t *testing.T) {
	t.Parallel()
	a := assert.New(t)
	p := provider()
	session, err := p.BeginAuth("test_state")
	s := session.(*auth0.Session)
	a.NoError(err)
	expectedAuthURL := "https://" + os.Getenv("AUTH0_DOMAIN") + "/authorize"
	a.Contains(s.AuthURL, expectedAuthURL)
}

func Test_SessionFromJSON(t *testing.T) {
	t.Parallel()
	a := assert.New(t)

	p := provider()
	sessionResp := `{"AuthURL":"https://` + p.Domain + `/oauth/authorize","AccessToken":"1234567890"}`
	session, err := p.UnmarshalSession(sessionResp)
	a.NoError(err)

	s := session.(*auth0.Session)
	expectedAuthURL := "https://" + os.Getenv("AUTH0_DOMAIN") + "/oauth/authorize"
	a.Equal(s.AuthURL, expectedAuthURL)
	a.Equal(s.AccessToken, "1234567890")
}

func Test_FetchUser(t *testing.T) {
	//t.Parallel()
	a := assert.New(t)

	httpmock.Activate()
	defer httpmock.DeactivateAndReset()

	sampleResp := `{
  		"email_verified": false,
  		"email": "test.account@userinfo.com",
  		"clientID": "q2hnj2iu...",
  		"updated_at": "2016-12-05T15:15:40.545Z",
  		"name": "test.account@userinfo.com",
  		"picture": "https://s.gravatar.com/avatar/dummy.png",
  		"user_id": "auth0|58454...",
  		"nickname": "test.account",
  		"identities": [
  		  {
      			"user_id": "58454...",
      			"provider": "auth0",
      			"connection": "Username-Password-Authentication",
      			"isSocial": false
    		}],
  		"created_at": "2016-12-05T11:16:59.640Z",
  		"sub": "auth0|58454..."
	}`

	httpmock.RegisterResponder("GET", "https://"+os.Getenv("AUTH0_DOMAIN")+"/userinfo", httpmock.NewStringResponder(200, sampleResp))

	p := provider()
	session, err := p.BeginAuth("test_state")
	s := session.(*auth0.Session)
	s.AccessToken = "token"
	u, err := p.FetchUser(s)
	a.Nil(err)
	a.Equal(u.Email, "test.account@userinfo.com")
	a.Equal(u.UserID, "auth0|58454...")
	a.Equal(u.NickName, "test.account")
	a.Equal(u.Name, "test.account@userinfo.com")
	a.Equal("token", u.AccessToken)

}

func provider() *auth0.Provider {
	return auth0.New(os.Getenv("AUTH0_KEY"), os.Getenv("AUTH0_SECRET"), "/foo", os.Getenv("AUTH0_DOMAIN"))
}
<|endoftext|>


<commit_before>// Copyright 2012 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package juju

import (
	"bytes"
	"fmt"
	"github.com/globocom/tsuru/provision"
	"io"
	"launchpad.net/goyaml"
	"os/exec"
	"regexp"
	"strconv"
	"strings"
	"time"
)

// JujuProvisioner is an implementation for the Provisioner interface. For more
// details on how a provisioner work, check the documentation of the provision
// package.
type JujuProvisioner struct{}

func (p *JujuProvisioner) Provision(app provision.App) error {
	var buf bytes.Buffer
	args := []string{
		"deploy", "--repository", "/home/charms",
		"local:" + app.GetFramework(), app.GetName(),
	}
	err := runCmd(&buf, &buf, args...)
	out := buf.String()
	if err != nil {
		app.Log("Failed to create machine: "+out, "tsuru")
		return &provision.Error{Reason: out, Err: err}
	}
	return nil
}

func (p *JujuProvisioner) Destroy(app provision.App) error {
	var buf bytes.Buffer
	err := runCmd(&buf, &buf, "destroy-service", app.GetName())
	out := buf.String()
	if err != nil {
		app.Log("Failed to destroy machine: "+out, "tsuru")
		return &provision.Error{Reason: out, Err: err}
	}
	for _, u := range app.ProvisionUnits() {
		buf.Reset()
		err = runCmd(&buf, &buf, "terminate-machine", strconv.Itoa(u.GetMachine()))
		out = buf.String()
		if err != nil {
			app.Log("Failed to destroy machine: "+out, "tsuru")
			return &provision.Error{Reason: out, Err: err}
		}
	}
	return nil
}

func (p *JujuProvisioner) AddUnits(app provision.App, n uint) ([]provision.Unit, error) {
	return nil, nil
}

func (p *JujuProvisioner) RemoveUnit(app provision.App, name string) error {
	return nil
}

func (p *JujuProvisioner) RemoveUnits(app provision.App, n uint) error {
	return nil
}

func (p *JujuProvisioner) ExecuteCommand(stdout, stderr io.Writer, app provision.App, cmd string, args ...string) error {
	arguments := []string{"ssh", "-o", "StrictHostKeyChecking no", "-q"}
	units := app.ProvisionUnits()
	length := len(units)
	for i, unit := range units {
		if length > 1 {
			if i > 0 {
				fmt.Fprintln(stdout)
			}
			fmt.Fprintf(stdout, "Output from unit %q:\n\n", unit.GetName())
			if status := unit.GetStatus(); status != provision.StatusStarted {
				fmt.Fprintf(stdout, "Unit state is %q, it must be %q for running commands.\n",
					status, provision.StatusStarted)
				continue
			}
		}
		var cmdargs []string
		cmdargs = append(cmdargs, arguments...)
		cmdargs = append(cmdargs, strconv.Itoa(unit.GetMachine()), cmd)
		cmdargs = append(cmdargs, args...)
		err := runCmd(stdout, stderr, cmdargs...)
		fmt.Fprintln(stdout)
		if err != nil {
			return err
		}
	}
	return nil
}

func (p *JujuProvisioner) CollectStatus() ([]provision.Unit, error) {
	output, err := execWithTimeout(30e9, "juju", "status")
	if err != nil {
		return nil, &provision.Error{Reason: string(output), Err: err}
	}
	var out jujuOutput
	err = goyaml.Unmarshal(output, &out)
	if err != nil {
		return nil, &provision.Error{Reason: `"juju status" returned invalid data`, Err: err}
	}
	var units []provision.Unit
	for name, service := range out.Services {
		for unitName, u := range service.Units {
			machine := out.Machines[u.Machine]
			unit := provision.Unit{
				Name:    unitName,
				AppName: name,
				Machine: u.Machine,
				Ip:      machine.IpAddress,
			}
			typeRegexp := regexp.MustCompile(`^(local:)?(\w+)/(\w+)-\d+$`)
			matchs := typeRegexp.FindStringSubmatch(service.Charm)
			if len(matchs) > 3 {
				unit.Type = matchs[3]
			}
			unit.Status = unitStatus(machine.InstanceState, u.AgentState, machine.AgentState)
			units = append(units, unit)
		}
	}
	return units, nil
}

type unit struct {
	AgentState string `yaml:"agent-state"`
	Machine    int
}

type service struct {
	Units map[string]unit
	Charm string
}

type machine struct {
	AgentState    string `yaml:"agent-state"`
	IpAddress     string `yaml:"dns-name"`
	InstanceId    string `yaml:"instance-id"`
	InstanceState string `yaml:"instance-state"`
}

type jujuOutput struct {
	Services map[string]service
	Machines map[int]machine
}

func init() {
	provision.Register("juju", &JujuProvisioner{})
}

func runCmd(stdout, stderr io.Writer, cmd ...string) error {
	stdout = &Writer{stdout}
	stderr = &Writer{stderr}
	command := exec.Command("juju", cmd...)
	command.Stdout = stdout
	command.Stderr = stderr
	return command.Run()
}

func execWithTimeout(timeout time.Duration, cmd string, args ...string) (output []byte, err error) {
	var buf bytes.Buffer
	ch := make(chan []byte, 1)
	errCh := make(chan error, 1)
	command := exec.Command(cmd, args...)
	command.Stdout = &Writer{&buf}
	command.Stderr = &Writer{&buf}
	if err = command.Start(); err != nil {
		return nil, err
	}
	go func() {
		if err := command.Wait(); err == nil {
			ch <- buf.Bytes()
		} else {
			errCh <- err
			ch <- buf.Bytes()
		}
	}()
	select {
	case output = <-ch:
		select {
		case err = <-errCh:
		case <-time.After(1e9):
		}
	case err = <-errCh:
		output = <-ch
	case <-time.After(timeout):
		argsStr := strings.Join(args, " ")
		err = fmt.Errorf("%q ran for more than %s.", cmd+" "+argsStr, timeout)
		command.Process.Kill()
	}
	return output, err
}

func unitStatus(instanceState, agentState, machineAgentState string) provision.Status {
	if instanceState == "error" || agentState == "install-error" || machineAgentState == "start-error" {
		return provision.StatusError
	}
	if machineAgentState == "pending" || machineAgentState == "not-started" || machineAgentState == "" {
		return provision.StatusCreating
	}
	if instanceState == "pending" || instanceState == "" {
		return provision.StatusCreating
	}
	if agentState == "down" {
		return provision.StatusDown
	}
	if machineAgentState == "running" && agentState == "not-started" {
		return provision.StatusCreating
	}
	if machineAgentState == "running" && instanceState == "running" && agentState == "pending" {
		return provision.StatusInstalling
	}
	if machineAgentState == "running" && agentState == "started" && instanceState == "running" {
		return provision.StatusStarted
	}
	return provision.StatusPending
}
<commit_msg>provision/juju: added parameter in runCmd to enable or disable filtering<commit_after>// Copyright 2012 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package juju

import (
	"bytes"
	"fmt"
	"github.com/globocom/tsuru/provision"
	"io"
	"launchpad.net/goyaml"
	"os/exec"
	"regexp"
	"strconv"
	"strings"
	"time"
)

// JujuProvisioner is an implementation for the Provisioner interface. For more
// details on how a provisioner work, check the documentation of the provision
// package.
type JujuProvisioner struct{}

func (p *JujuProvisioner) Provision(app provision.App) error {
	var buf bytes.Buffer
	args := []string{
		"deploy", "--repository", "/home/charms",
		"local:" + app.GetFramework(), app.GetName(),
	}
	err := runCmd(true, &buf, &buf, args...)
	out := buf.String()
	if err != nil {
		app.Log("Failed to create machine: "+out, "tsuru")
		return &provision.Error{Reason: out, Err: err}
	}
	return nil
}

func (p *JujuProvisioner) Destroy(app provision.App) error {
	var buf bytes.Buffer
	err := runCmd(true, &buf, &buf, "destroy-service", app.GetName())
	out := buf.String()
	if err != nil {
		app.Log("Failed to destroy machine: "+out, "tsuru")
		return &provision.Error{Reason: out, Err: err}
	}
	for _, u := range app.ProvisionUnits() {
		buf.Reset()
		err = runCmd(true, &buf, &buf, "terminate-machine", strconv.Itoa(u.GetMachine()))
		out = buf.String()
		if err != nil {
			app.Log("Failed to destroy machine: "+out, "tsuru")
			return &provision.Error{Reason: out, Err: err}
		}
	}
	return nil
}

func (p *JujuProvisioner) AddUnits(app provision.App, n uint) ([]provision.Unit, error) {
	return nil, nil
}

func (p *JujuProvisioner) RemoveUnit(app provision.App, name string) error {
	return nil
}

func (p *JujuProvisioner) RemoveUnits(app provision.App, n uint) error {
	return nil
}

func (p *JujuProvisioner) ExecuteCommand(stdout, stderr io.Writer, app provision.App, cmd string, args ...string) error {
	arguments := []string{"ssh", "-o", "StrictHostKeyChecking no", "-q"}
	units := app.ProvisionUnits()
	length := len(units)
	for i, unit := range units {
		if length > 1 {
			if i > 0 {
				fmt.Fprintln(stdout)
			}
			fmt.Fprintf(stdout, "Output from unit %q:\n\n", unit.GetName())
			if status := unit.GetStatus(); status != provision.StatusStarted {
				fmt.Fprintf(stdout, "Unit state is %q, it must be %q for running commands.\n",
					status, provision.StatusStarted)
				continue
			}
		}
		var cmdargs []string
		cmdargs = append(cmdargs, arguments...)
		cmdargs = append(cmdargs, strconv.Itoa(unit.GetMachine()), cmd)
		cmdargs = append(cmdargs, args...)
		err := runCmd(true, stdout, stderr, cmdargs...)
		fmt.Fprintln(stdout)
		if err != nil {
			return err
		}
	}
	return nil
}

func (p *JujuProvisioner) CollectStatus() ([]provision.Unit, error) {
	output, err := execWithTimeout(30e9, "juju", "status")
	if err != nil {
		return nil, &provision.Error{Reason: string(output), Err: err}
	}
	var out jujuOutput
	err = goyaml.Unmarshal(output, &out)
	if err != nil {
		return nil, &provision.Error{Reason: `"juju status" returned invalid data`, Err: err}
	}
	var units []provision.Unit
	for name, service := range out.Services {
		for unitName, u := range service.Units {
			machine := out.Machines[u.Machine]
			unit := provision.Unit{
				Name:    unitName,
				AppName: name,
				Machine: u.Machine,
				Ip:      machine.IpAddress,
			}
			typeRegexp := regexp.MustCompile(`^(local:)?(\w+)/(\w+)-\d+$`)
			matchs := typeRegexp.FindStringSubmatch(service.Charm)
			if len(matchs) > 3 {
				unit.Type = matchs[3]
			}
			unit.Status = unitStatus(machine.InstanceState, u.AgentState, machine.AgentState)
			units = append(units, unit)
		}
	}
	return units, nil
}

type unit struct {
	AgentState string `yaml:"agent-state"`
	Machine    int
}

type service struct {
	Units map[string]unit
	Charm string
}

type machine struct {
	AgentState    string `yaml:"agent-state"`
	IpAddress     string `yaml:"dns-name"`
	InstanceId    string `yaml:"instance-id"`
	InstanceState string `yaml:"instance-state"`
}

type jujuOutput struct {
	Services map[string]service
	Machines map[int]machine
}

func init() {
	provision.Register("juju", &JujuProvisioner{})
}

func runCmd(filter bool, stdout, stderr io.Writer, cmd ...string) error {
	if filter {
		stdout = &Writer{stdout}
		stderr = &Writer{stderr}
	}
	command := exec.Command("juju", cmd...)
	command.Stdout = stdout
	command.Stderr = stderr
	return command.Run()
}

func execWithTimeout(timeout time.Duration, cmd string, args ...string) (output []byte, err error) {
	var buf bytes.Buffer
	ch := make(chan []byte, 1)
	errCh := make(chan error, 1)
	command := exec.Command(cmd, args...)
	command.Stdout = &Writer{&buf}
	command.Stderr = &Writer{&buf}
	if err = command.Start(); err != nil {
		return nil, err
	}
	go func() {
		if err := command.Wait(); err == nil {
			ch <- buf.Bytes()
		} else {
			errCh <- err
			ch <- buf.Bytes()
		}
	}()
	select {
	case output = <-ch:
		select {
		case err = <-errCh:
		case <-time.After(1e9):
		}
	case err = <-errCh:
		output = <-ch
	case <-time.After(timeout):
		argsStr := strings.Join(args, " ")
		err = fmt.Errorf("%q ran for more than %s.", cmd+" "+argsStr, timeout)
		command.Process.Kill()
	}
	return output, err
}

func unitStatus(instanceState, agentState, machineAgentState string) provision.Status {
	if instanceState == "error" || agentState == "install-error" || machineAgentState == "start-error" {
		return provision.StatusError
	}
	if machineAgentState == "pending" || machineAgentState == "not-started" || machineAgentState == "" {
		return provision.StatusCreating
	}
	if instanceState == "pending" || instanceState == "" {
		return provision.StatusCreating
	}
	if agentState == "down" {
		return provision.StatusDown
	}
	if machineAgentState == "running" && agentState == "not-started" {
		return provision.StatusCreating
	}
	if machineAgentState == "running" && instanceState == "running" && agentState == "pending" {
		return provision.StatusInstalling
	}
	if machineAgentState == "running" && agentState == "started" && instanceState == "running" {
		return provision.StatusStarted
	}
	return provision.StatusPending
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package assign

import (
	"fmt"
	"regexp"
	"strings"

	"github.com/sirupsen/logrus"

	"k8s.io/test-infra/prow/github"
	"k8s.io/test-infra/prow/pluginhelp"
	"k8s.io/test-infra/prow/plugins"
)

const pluginName = "assign"

var (
	assignRe = regexp.MustCompile(`(?mi)^/(un)?assign(( @?[-\w]+?)*)\s*$`)
	// CCRegexp parses and validates /cc commands, also used by blunderbuss
	CCRegexp = regexp.MustCompile(`(?mi)^/(un)?cc(( +@?[-/\w]+?)*)\s*$`)
)

func init() {
	plugins.RegisterGenericCommentHandler(pluginName, handleGenericComment, helpProvider)
}

func helpProvider(config *plugins.Configuration, enabledRepos []string) (*pluginhelp.PluginHelp, error) {
	// The Config field is omitted because this plugin is not configurable.
	pluginHelp := &pluginhelp.PluginHelp{
		Description: "The assign plugin assigns or requests reviews from users. Specific users can be assigned with the command '/assign @user1' or have reviews requested of them with the command '/cc @user1'. If no user is specified the commands default to targeting the user who created the command. Assignments and requested reviews can be removed in the same way that they are added by prefixing the commands with 'un'.",
	}
	pluginHelp.AddCommand(pluginhelp.Command{
		Usage:       "/[un]assign [[@]<username>...]",
		Description: "Assigns an assignee to the PR",
		Featured:    true,
		WhoCanUse:   "Anyone can use the command, but the target user must be a member of the org that owns the repository.",
		Examples:    []string{"/assign", "/unassign", "/assign @k8s-ci-robot"},
	})
	pluginHelp.AddCommand(pluginhelp.Command{
		Usage:       "/[un]cc [[@]<username>...]",
		Description: "Requests a review from the user(s).",
		Featured:    true,
		WhoCanUse:   "Anyone can use the command, but the target user must be a member of the org that owns the repository.",
		Examples:    []string{"/cc", "/uncc", "/cc @k8s-ci-robot"},
	})
	return pluginHelp, nil
}

type githubClient interface {
	AssignIssue(owner, repo string, number int, logins []string) error
	UnassignIssue(owner, repo string, number int, logins []string) error

	RequestReview(org, repo string, number int, logins []string) error
	UnrequestReview(org, repo string, number int, logins []string) error

	CreateComment(owner, repo string, number int, comment string) error
}

func handleGenericComment(pc plugins.PluginClient, e github.GenericCommentEvent) error {
	if e.Action != github.GenericCommentActionCreated {
		return nil
	}
	err := handle(newAssignHandler(e, pc.GitHubClient, pc.Logger))
	if e.IsPR {
		err = combineErrors(err, handle(newReviewHandler(e, pc.GitHubClient, pc.Logger)))
	}
	return err
}

func parseLogins(text string) []string {
	var parts []string
	for _, p := range strings.Split(text, " ") {
		t := strings.Trim(p, "@ ")
		if t == "" {
			continue
		}
		parts = append(parts, t)
	}
	return parts
}

func combineErrors(err1, err2 error) error {
	if err1 != nil && err2 != nil {
		return fmt.Errorf("two errors: 1) %v 2) %v", err1, err2)
	} else if err1 != nil {
		return err1
	} else {
		return err2
	}
}

// handle is the generic handler for the assign plugin. It uses the handler's regexp and affectedLogins
// functions to identify the users to add and/or remove and then passes the appropriate users to the
// handler's add and remove functions. If add fails to add some of the users, a response comment is
// created where the body of the response is generated by the handler's addFailureResponse function.
func handle(h *handler) error {
	e := h.event
	org := e.Repo.Owner.Login
	repo := e.Repo.Name
	matches := h.regexp.FindAllStringSubmatch(e.Body, -1)
	if matches == nil {
		return nil
	}
	users := make(map[string]bool)
	for _, re := range matches {
		add := re[1] != "un" // un<cmd> == !add
		if re[2] == "" {
			users[e.User.Login] = add
		} else {
			for _, login := range parseLogins(re[2]) {
				users[login] = add
			}
		}
	}
	var toAdd, toRemove []string
	for login, add := range users {
		if add {
			toAdd = append(toAdd, login)
		} else {
			toRemove = append(toRemove, login)
		}
	}

	if len(toRemove) > 0 {
		h.log.Printf("Removing %s from %s/%s#%d: %v", h.userType, org, repo, e.Number, toRemove)
		if err := h.remove(org, repo, e.Number, toRemove); err != nil {
			return err
		}
	}
	if len(toAdd) > 0 {
		h.log.Printf("Adding %s to %s/%s#%d: %v", h.userType, org, repo, e.Number, toAdd)
		if err := h.add(org, repo, e.Number, toAdd); err != nil {
			if mu, ok := err.(github.MissingUsers); ok {
				msg := h.addFailureResponse(mu)
				if len(msg) == 0 {
					return nil
				}
				if err := h.gc.CreateComment(org, repo, e.Number, plugins.FormatResponseRaw(e.Body, e.HTMLURL, e.User.Login, msg)); err != nil {
					return fmt.Errorf("comment err: %v", err)
				}
				return nil
			}
			return err
		}
	}
	return nil
}

// handler is a struct that contains data about a github event and provides functions to help handle it.
type handler struct {
	// addFailureResponse generates the body of a response comment in the event that the add function fails.
	addFailureResponse func(mu github.MissingUsers) string
	// remove is the function that is called on the affected logins for a command prefixed with 'un'.
	remove func(org, repo string, number int, users []string) error
	// add is the function that is called on the affected logins for a command with no 'un' prefix.
	add func(org, repo string, number int, users []string) error

	// event is a pointer to the github.GenericCommentEvent struct that triggered the handler.
	event *github.GenericCommentEvent
	// regexp is the regular expression describing the command. It must have an optional 'un' prefix
	// as the first subgroup and the arguments to the command as the second subgroup.
	regexp *regexp.Regexp
	// gc is the githubClient to use for creating response comments in the event of a failure.
	gc githubClient

	// log is a logrus.Entry used to record actions the handler takes.
	log *logrus.Entry
	// userType is a string that represents the type of users affected by this handler. (e.g. 'assignees')
	userType string
}

func newAssignHandler(e github.GenericCommentEvent, gc githubClient, log *logrus.Entry) *handler {
	org := e.Repo.Owner.Login
	addFailureResponse := func(mu github.MissingUsers) string {
		return fmt.Sprintf("GitHub didn't allow me to assign the following users: %s.\n\nNote that only [%s members](https://github.com/orgs/%s/people) and repo collaborators can be assigned.", strings.Join(mu.Users, ", "), org, org)
	}

	return &handler{
		addFailureResponse: addFailureResponse,
		remove:             gc.UnassignIssue,
		add:                gc.AssignIssue,
		event:              &e,
		regexp:             assignRe,
		gc:                 gc,
		log:                log,
		userType:           "assignee(s)",
	}
}

func newReviewHandler(e github.GenericCommentEvent, gc githubClient, log *logrus.Entry) *handler {
	org := e.Repo.Owner.Login
	addFailureResponse := func(mu github.MissingUsers) string {
		return fmt.Sprintf("GitHub didn't allow me to request PR reviews from the following users: %s.\n\nNote that only [%s members](https://github.com/orgs/%s/people) and repo collaborators can review this PR, and authors cannot review their own PRs.", strings.Join(mu.Users, ", "), org, org)
	}

	return &handler{
		addFailureResponse: addFailureResponse,
		remove:             gc.UnrequestReview,
		add:                gc.RequestReview,
		event:              &e,
		regexp:             CCRegexp,
		gc:                 gc,
		log:                log,
		userType:           "reviewer(s)",
	}
}
<commit_msg>Adds link about assign limitations to bot response text<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package assign

import (
	"fmt"
	"regexp"
	"strings"

	"github.com/sirupsen/logrus"

	"k8s.io/test-infra/prow/github"
	"k8s.io/test-infra/prow/pluginhelp"
	"k8s.io/test-infra/prow/plugins"
)

const pluginName = "assign"

var (
	assignRe = regexp.MustCompile(`(?mi)^/(un)?assign(( @?[-\w]+?)*)\s*$`)
	// CCRegexp parses and validates /cc commands, also used by blunderbuss
	CCRegexp = regexp.MustCompile(`(?mi)^/(un)?cc(( +@?[-/\w]+?)*)\s*$`)
)

func init() {
	plugins.RegisterGenericCommentHandler(pluginName, handleGenericComment, helpProvider)
}

func helpProvider(config *plugins.Configuration, enabledRepos []string) (*pluginhelp.PluginHelp, error) {
	// The Config field is omitted because this plugin is not configurable.
	pluginHelp := &pluginhelp.PluginHelp{
		Description: "The assign plugin assigns or requests reviews from users. Specific users can be assigned with the command '/assign @user1' or have reviews requested of them with the command '/cc @user1'. If no user is specified the commands default to targeting the user who created the command. Assignments and requested reviews can be removed in the same way that they are added by prefixing the commands with 'un'.",
	}
	pluginHelp.AddCommand(pluginhelp.Command{
		Usage:       "/[un]assign [[@]<username>...]",
		Description: "Assigns an assignee to the PR",
		Featured:    true,
		WhoCanUse:   "Anyone can use the command, but the target user must be a member of the org that owns the repository.",
		Examples:    []string{"/assign", "/unassign", "/assign @k8s-ci-robot"},
	})
	pluginHelp.AddCommand(pluginhelp.Command{
		Usage:       "/[un]cc [[@]<username>...]",
		Description: "Requests a review from the user(s).",
		Featured:    true,
		WhoCanUse:   "Anyone can use the command, but the target user must be a member of the org that owns the repository.",
		Examples:    []string{"/cc", "/uncc", "/cc @k8s-ci-robot"},
	})
	return pluginHelp, nil
}

type githubClient interface {
	AssignIssue(owner, repo string, number int, logins []string) error
	UnassignIssue(owner, repo string, number int, logins []string) error

	RequestReview(org, repo string, number int, logins []string) error
	UnrequestReview(org, repo string, number int, logins []string) error

	CreateComment(owner, repo string, number int, comment string) error
}

func handleGenericComment(pc plugins.PluginClient, e github.GenericCommentEvent) error {
	if e.Action != github.GenericCommentActionCreated {
		return nil
	}
	err := handle(newAssignHandler(e, pc.GitHubClient, pc.Logger))
	if e.IsPR {
		err = combineErrors(err, handle(newReviewHandler(e, pc.GitHubClient, pc.Logger)))
	}
	return err
}

func parseLogins(text string) []string {
	var parts []string
	for _, p := range strings.Split(text, " ") {
		t := strings.Trim(p, "@ ")
		if t == "" {
			continue
		}
		parts = append(parts, t)
	}
	return parts
}

func combineErrors(err1, err2 error) error {
	if err1 != nil && err2 != nil {
		return fmt.Errorf("two errors: 1) %v 2) %v", err1, err2)
	} else if err1 != nil {
		return err1
	} else {
		return err2
	}
}

// handle is the generic handler for the assign plugin. It uses the handler's regexp and affectedLogins
// functions to identify the users to add and/or remove and then passes the appropriate users to the
// handler's add and remove functions. If add fails to add some of the users, a response comment is
// created where the body of the response is generated by the handler's addFailureResponse function.
func handle(h *handler) error {
	e := h.event
	org := e.Repo.Owner.Login
	repo := e.Repo.Name
	matches := h.regexp.FindAllStringSubmatch(e.Body, -1)
	if matches == nil {
		return nil
	}
	users := make(map[string]bool)
	for _, re := range matches {
		add := re[1] != "un" // un<cmd> == !add
		if re[2] == "" {
			users[e.User.Login] = add
		} else {
			for _, login := range parseLogins(re[2]) {
				users[login] = add
			}
		}
	}
	var toAdd, toRemove []string
	for login, add := range users {
		if add {
			toAdd = append(toAdd, login)
		} else {
			toRemove = append(toRemove, login)
		}
	}

	if len(toRemove) > 0 {
		h.log.Printf("Removing %s from %s/%s#%d: %v", h.userType, org, repo, e.Number, toRemove)
		if err := h.remove(org, repo, e.Number, toRemove); err != nil {
			return err
		}
	}
	if len(toAdd) > 0 {
		h.log.Printf("Adding %s to %s/%s#%d: %v", h.userType, org, repo, e.Number, toAdd)
		if err := h.add(org, repo, e.Number, toAdd); err != nil {
			if mu, ok := err.(github.MissingUsers); ok {
				msg := h.addFailureResponse(mu)
				if len(msg) == 0 {
					return nil
				}
				if err := h.gc.CreateComment(org, repo, e.Number, plugins.FormatResponseRaw(e.Body, e.HTMLURL, e.User.Login, msg)); err != nil {
					return fmt.Errorf("comment err: %v", err)
				}
				return nil
			}
			return err
		}
	}
	return nil
}

// handler is a struct that contains data about a github event and provides functions to help handle it.
type handler struct {
	// addFailureResponse generates the body of a response comment in the event that the add function fails.
	addFailureResponse func(mu github.MissingUsers) string
	// remove is the function that is called on the affected logins for a command prefixed with 'un'.
	remove func(org, repo string, number int, users []string) error
	// add is the function that is called on the affected logins for a command with no 'un' prefix.
	add func(org, repo string, number int, users []string) error

	// event is a pointer to the github.GenericCommentEvent struct that triggered the handler.
	event *github.GenericCommentEvent
	// regexp is the regular expression describing the command. It must have an optional 'un' prefix
	// as the first subgroup and the arguments to the command as the second subgroup.
	regexp *regexp.Regexp
	// gc is the githubClient to use for creating response comments in the event of a failure.
	gc githubClient

	// log is a logrus.Entry used to record actions the handler takes.
	log *logrus.Entry
	// userType is a string that represents the type of users affected by this handler. (e.g. 'assignees')
	userType string
}

func newAssignHandler(e github.GenericCommentEvent, gc githubClient, log *logrus.Entry) *handler {
	org := e.Repo.Owner.Login
	addFailureResponse := func(mu github.MissingUsers) string {
		return fmt.Sprintf("GitHub didn't allow me to assign the following users: %s.\n\nNote that only [%s members](https://github.com/orgs/%s/people) and repo collaborators can be assigned. \nFor more information please see [the contributor guide](https://git.k8s.io/community/contributors/guide/#issue-assignment-in-github)", strings.Join(mu.Users, ", "), org, org)
	}

	return &handler{
		addFailureResponse: addFailureResponse,
		remove:             gc.UnassignIssue,
		add:                gc.AssignIssue,
		event:              &e,
		regexp:             assignRe,
		gc:                 gc,
		log:                log,
		userType:           "assignee(s)",
	}
}

func newReviewHandler(e github.GenericCommentEvent, gc githubClient, log *logrus.Entry) *handler {
	org := e.Repo.Owner.Login
	addFailureResponse := func(mu github.MissingUsers) string {
		return fmt.Sprintf("GitHub didn't allow me to request PR reviews from the following users: %s.\n\nNote that only [%s members](https://github.com/orgs/%s/people) and repo collaborators can review this PR, and authors cannot review their own PRs.", strings.Join(mu.Users, ", "), org, org)
	}

	return &handler{
		addFailureResponse: addFailureResponse,
		remove:             gc.UnrequestReview,
		add:                gc.RequestReview,
		event:              &e,
		regexp:             CCRegexp,
		gc:                 gc,
		log:                log,
		userType:           "reviewer(s)",
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"os/signal"

	"github.com/coreos/go-etcd/etcd"
	"github.com/lytics/etcdlog"
)

func main() {
	host := flag.String("host", "http://localhost:2379", "URL to etcd")
	path := flag.String("path", "/", "path to watch (recursively)")
	index := flag.Uint64("index", 0, "index to start from")
	jsonout := flag.String("json", "", "file to write json output to; '-' for stdout, '' for nowhere")
	humout := flag.String("log", "-", "file to write human output to; '-' for stdout, '' for nowhere")

	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
		flag.PrintDefaults()
		fmt.Fprintf(os.Stderr, `
Log format:
  - Timestamp (format: 2006-01-02 15:04:05.999999999)
  - Modified Index
  - Action
  - Key
  - Value
  - Created Index
`)

	}

	flag.Parse()

	if *jsonout == "" && *humout == "" {
		log.Fatal("expected one of -json or -log")
	}

	c, err := etcdlog.NewEtcdClient([]string{*host})
	if err != nil {
		log.Fatal("error connecting to etcd: ", err)
	}

	// JSON output encoder
	var enc *json.Encoder
	switch *jsonout {
	case "-":
		enc = json.NewEncoder(os.Stdout)
	case "":
		// don't even set it
	default:
		jfd, err := os.Create(*jsonout)
		if err != nil {
			log.Fatalf("error creating json output file %s: %v", *jsonout, err)
		}
		defer jfd.Close()
		enc = json.NewEncoder(jfd)
	}

	// Human output encoder
	var hlog io.Writer
	switch *humout {
	case "-":
		hlog = os.Stderr
	case "":
		// don't even set it
	default:
		hfd, err := os.Create(*humout)
		if err != nil {
			log.Fatalf("error creating log output file %s: %v", *humout, err)
		}
		defer hfd.Close()
		hlog = hfd
	}

	// Signal handler
	sigs := make(chan os.Signal, 1)
	signal.Notify(sigs, os.Interrupt, os.Kill)

	// etcd watcher
	w := etcdlog.NewWatcher(c, *path, *index)
	resps := w.Watch()

	newline := []byte("\n")

mainloop:
	for {
		select {
		case resp, ok := <-resps:
			if !ok {
				break mainloop
			}
			// Write human readable log if writer exists
			if hlog != nil {
				if _, err := hlog.Write([]byte(resp.String())); err != nil {
					log.Fatal("error writing log output: ", err)
				}
				hlog.Write(newline)
			}

			// Write JSON if encoder exists
			if enc != nil {
				if err := enc.Encode(resp); err != nil {
					log.Fatal("error encoding response: ", err)
				}
			}
		case <-sigs:
			w.Close()
		}
	}

	if err := w.Err(); err != nil {
		if ee, ok := err.(*etcd.EtcdError); ok {
			log.Println(ee.Index)
		}
		log.Fatal("error when closing: ", err)
	}
}
<commit_msg>Remove debugging code<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"os/signal"

	"github.com/lytics/etcdlog"
)

func main() {
	host := flag.String("host", "http://localhost:2379", "URL to etcd")
	path := flag.String("path", "/", "path to watch (recursively)")
	index := flag.Uint64("index", 0, "index to start from")
	jsonout := flag.String("json", "", "file to write json output to; '-' for stdout, '' for nowhere")
	humout := flag.String("log", "-", "file to write human output to; '-' for stdout, '' for nowhere")

	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
		flag.PrintDefaults()
		fmt.Fprintf(os.Stderr, `
Log format:
  - Timestamp (format: 2006-01-02 15:04:05.999999999)
  - Modified Index
  - Action
  - Key
  - Value
  - Created Index
`)

	}

	flag.Parse()

	if *jsonout == "" && *humout == "" {
		log.Fatal("expected one of -json or -log")
	}

	c, err := etcdlog.NewEtcdClient([]string{*host})
	if err != nil {
		log.Fatal("error connecting to etcd: ", err)
	}

	// JSON output encoder
	var enc *json.Encoder
	switch *jsonout {
	case "-":
		enc = json.NewEncoder(os.Stdout)
	case "":
		// don't even set it
	default:
		jfd, err := os.Create(*jsonout)
		if err != nil {
			log.Fatalf("error creating json output file %s: %v", *jsonout, err)
		}
		defer jfd.Close()
		enc = json.NewEncoder(jfd)
	}

	// Human output encoder
	var hlog io.Writer
	switch *humout {
	case "-":
		hlog = os.Stderr
	case "":
		// don't even set it
	default:
		hfd, err := os.Create(*humout)
		if err != nil {
			log.Fatalf("error creating log output file %s: %v", *humout, err)
		}
		defer hfd.Close()
		hlog = hfd
	}

	// Signal handler
	sigs := make(chan os.Signal, 1)
	signal.Notify(sigs, os.Interrupt, os.Kill)

	// etcd watcher
	w := etcdlog.NewWatcher(c, *path, *index)
	resps := w.Watch()

	newline := []byte("\n")

mainloop:
	for {
		select {
		case resp, ok := <-resps:
			if !ok {
				break mainloop
			}
			// Write human readable log if writer exists
			if hlog != nil {
				if _, err := hlog.Write([]byte(resp.String())); err != nil {
					log.Fatal("error writing log output: ", err)
				}
				hlog.Write(newline)
			}

			// Write JSON if encoder exists
			if enc != nil {
				if err := enc.Encode(resp); err != nil {
					log.Fatal("error encoding response: ", err)
				}
			}
		case <-sigs:
			w.Close()
		}
	}

	if err := w.Err(); err != nil {
		log.Fatal("error when closing: ", err)
	}
}
<|endoftext|>


<commit_before>package eth

import (
	"bytes"
	"math"
	"math/big"

	"github.com/ethereum/go-ethereum/core/types"
	"github.com/ethereum/go-ethereum/ethutil"
	"github.com/ethereum/go-ethereum/p2p"
	"github.com/ethereum/go-ethereum/rlp"
)

// ethProtocol represents the ethereum wire protocol
// instance is running on each peer
type ethProtocol struct {
	eth  backend
	peer *p2p.Peer
	id   string
	rw   p2p.MsgReadWriter
}

// backend is the interface the ethereum protocol backend should implement
// used as an argument to EthProtocol
type backend interface {
	GetTransactions() (txs []*types.Transaction)
	AddTransactions([]*types.Transaction)
	GetBlockHashes(hash []byte, amount uint32) (hashes [][]byte)
	AddBlockHashes(next func() ([]byte, bool), peerId string)
	GetBlock(hash []byte) (block *types.Block)
	AddBlock(block *types.Block, peerId string) (err error)
	AddPeer(td *big.Int, currentBlock []byte, peerId string, requestHashes func([]byte) error, requestBlocks func([][]byte) error, invalidBlock func(error)) (best bool)
	RemovePeer(peerId string)
	Status() (td *big.Int, currentBlock []byte, genesisBlock []byte)
}

const (
	ProtocolVersion    = 43
	NetworkId          = 0
	ProtocolLength     = uint64(8)
	ProtocolMaxMsgSize = 10 * 1024 * 1024
)

// eth protocol message codes
const (
	StatusMsg = iota
	GetTxMsg  // unused
	TxMsg
	GetBlockHashesMsg
	BlockHashesMsg
	GetBlocksMsg
	BlocksMsg
	NewBlockMsg
)

// message structs used for rlp decoding
type newBlockMsgData struct {
	Block *types.Block
	TD    *big.Int
}

type getBlockHashesMsgData struct {
	Hash   []byte
	Amount uint32
}

// main entrypoint, wrappers starting a server running the eth protocol
// use this constructor to attach the protocol ("class") to server caps
// the Dev p2p layer then runs the protocol instance on each peer
func EthProtocol(eth backend) *p2p.Protocol {
	return &p2p.Protocol{
		Name:    "eth",
		Version: ProtocolVersion,
		Length:  ProtocolLength,
		Run: func(peer *p2p.Peer, rw p2p.MsgReadWriter) error {
			return runEthProtocol(eth, peer, rw)
		},
	}
}

// the main loop that handles incoming messages
// note RemovePeer in the post-disconnect hook
func runEthProtocol(eth backend, peer *p2p.Peer, rw p2p.MsgReadWriter) (err error) {
	self := &ethProtocol{
		eth:  eth,
		rw:   rw,
		peer: peer,
		id:   (string)(peer.Identity().Pubkey()),
	}
	err = self.handleStatus()
	if err == nil {
		go func() {
			for {
				err = self.handle()
				if err != nil {
					self.eth.RemovePeer(self.id)
					break
				}
			}
		}()
	}
	return
}

func (self *ethProtocol) handle() error {
	msg, err := self.rw.ReadMsg()
	if err != nil {
		return err
	}
	if msg.Size > ProtocolMaxMsgSize {
		return ProtocolError(ErrMsgTooLarge, "%v > %v", msg.Size, ProtocolMaxMsgSize)
	}
	// make sure that the payload has been fully consumed
	defer msg.Discard()

	switch msg.Code {

	case StatusMsg:
		return ProtocolError(ErrExtraStatusMsg, "")

	case GetTxMsg:
		txs := self.eth.GetTransactions()
		// TODO: rewrite using rlp flat
		txsInterface := make([]interface{}, len(txs))
		for i, tx := range txs {
			txsInterface[i] = tx.RlpData()
		}
		return self.rw.EncodeMsg(TxMsg, txsInterface...)

	case TxMsg:
		// TODO: rework using lazy RLP stream
		var txs []*types.Transaction
		if err := msg.Decode(&txs); err != nil {
			return ProtocolError(ErrDecode, "%v", err)
		}
		self.eth.AddTransactions(txs)

	case GetBlockHashesMsg:
		var request getBlockHashesMsgData
		if err := msg.Decode(&request); err != nil {
			return ProtocolError(ErrDecode, "%v", err)
		}
		hashes := self.eth.GetBlockHashes(request.Hash, request.Amount)
		return self.rw.EncodeMsg(BlockHashesMsg, ethutil.ByteSliceToInterface(hashes)...)

	case BlockHashesMsg:
		// TODO: redo using lazy decode , this way very inefficient on known chains
		msgStream := rlp.NewListStream(msg.Payload, uint64(msg.Size))
		var err error
		iter := func() (hash []byte, ok bool) {
			hash, err = msgStream.Bytes()
			if err == nil {
				ok = true
			}
			return
		}
		self.eth.AddBlockHashes(iter, self.id)
		if err != nil && err != rlp.EOL {
			return ProtocolError(ErrDecode, "%v", err)
		}

	case GetBlocksMsg:
		var blockHashes [][]byte
		if err := msg.Decode(&blockHashes); err != nil {
			return ProtocolError(ErrDecode, "%v", err)
		}
		max := int(math.Min(float64(len(blockHashes)), blockHashesBatchSize))
		var blocks []interface{}
		for i, hash := range blockHashes {
			if i >= max {
				break
			}
			block := self.eth.GetBlock(hash)
			if block != nil {
				blocks = append(blocks, block.Value().Raw())
			}
		}
		return self.rw.EncodeMsg(BlocksMsg, blocks...)

	case BlocksMsg:
		msgStream := rlp.NewListStream(msg.Payload, uint64(msg.Size))
		for {
			var block *types.Block
			if err := msgStream.Decode(&block); err != nil {
				if err == rlp.EOL {
					break
				} else {
					return ProtocolError(ErrDecode, "%v", err)
				}
			}
			if err := self.eth.AddBlock(block, self.id); err != nil {
				return ProtocolError(ErrInvalidBlock, "%v", err)
			}
		}

	case NewBlockMsg:
		var request newBlockMsgData
		if err := msg.Decode(&request); err != nil {
			return ProtocolError(ErrDecode, "%v", err)
		}
		hash := request.Block.Hash()
		// to simplify backend interface adding a new block
		// uses AddPeer followed by AddHashes, AddBlock only if peer is the best peer
		// (or selected as new best peer)
		if self.eth.AddPeer(request.TD, hash, self.id, self.requestBlockHashes, self.requestBlocks, self.invalidBlock) {
			called := true
			iter := func() (hash []byte, ok bool) {
				if called {
					called = false
					return hash, true
				} else {
					return
				}
			}
			self.eth.AddBlockHashes(iter, self.id)
			if err := self.eth.AddBlock(request.Block, self.id); err != nil {
				return ProtocolError(ErrInvalidBlock, "%v", err)
			}
		}

	default:
		return ProtocolError(ErrInvalidMsgCode, "%v", msg.Code)
	}
	return nil
}

type statusMsgData struct {
	ProtocolVersion uint
	NetworkId       uint
	TD              *big.Int
	CurrentBlock    []byte
	GenesisBlock    []byte
}

func (self *ethProtocol) statusMsg() p2p.Msg {
	td, currentBlock, genesisBlock := self.eth.Status()

	return p2p.NewMsg(StatusMsg,
		uint32(ProtocolVersion),
		uint32(NetworkId),
		td,
		currentBlock,
		genesisBlock,
	)
}

func (self *ethProtocol) handleStatus() error {
	// send precanned status message
	if err := self.rw.WriteMsg(self.statusMsg()); err != nil {
		return err
	}

	// read and handle remote status
	msg, err := self.rw.ReadMsg()
	if err != nil {
		return err
	}

	if msg.Code != StatusMsg {
		return ProtocolError(ErrNoStatusMsg, "first msg has code %x (!= %x)", msg.Code, StatusMsg)
	}

	if msg.Size > ProtocolMaxMsgSize {
		return ProtocolError(ErrMsgTooLarge, "%v > %v", msg.Size, ProtocolMaxMsgSize)
	}

	var status statusMsgData
	if err := msg.Decode(&status); err != nil {
		return ProtocolError(ErrDecode, "%v", err)
	}

	_, _, genesisBlock := self.eth.Status()

	if bytes.Compare(status.GenesisBlock, genesisBlock) != 0 {
		return ProtocolError(ErrGenesisBlockMismatch, "%x (!= %x)", status.GenesisBlock, genesisBlock)
	}

	if status.NetworkId != NetworkId {
		return ProtocolError(ErrNetworkIdMismatch, "%d (!= %d)", status.NetworkId, NetworkId)
	}

	if ProtocolVersion != status.ProtocolVersion {
		return ProtocolError(ErrProtocolVersionMismatch, "%d (!= %d)", status.ProtocolVersion, ProtocolVersion)
	}

	self.peer.Infof("Peer is [eth] capable (%d/%d). TD = %v ~ %x", status.ProtocolVersion, status.NetworkId, status.CurrentBlock)

	self.eth.AddPeer(status.TD, status.CurrentBlock, self.id, self.requestBlockHashes, self.requestBlocks, self.invalidBlock)

	return nil
}

func (self *ethProtocol) requestBlockHashes(from []byte) error {
	self.peer.Debugf("fetching hashes (%d) %x...\n", blockHashesBatchSize, from[0:4])
	return self.rw.EncodeMsg(GetBlockHashesMsg, from, blockHashesBatchSize)
}

func (self *ethProtocol) requestBlocks(hashes [][]byte) error {
	self.peer.Debugf("fetching %v blocks", len(hashes))
	return self.rw.EncodeMsg(GetBlocksMsg, ethutil.ByteSliceToInterface(hashes))
}

func (self *ethProtocol) invalidBlock(err error) {
	ProtocolError(ErrInvalidBlock, "%v", err)
	self.peer.Disconnect(p2p.DiscSubprotocolError)
}

func (self *ethProtocol) protoError(code int, format string, params ...interface{}) (err *protocolError) {
	err = ProtocolError(code, format, params...)
	if err.Fatal() {
		self.peer.Errorln(err)
	} else {
		self.peer.Debugln(err)
	}
	return
}
<commit_msg>protocol - new interface explicit backend components txPool, chainManager, blockPool - added protoErrorDisconnect for blockpool callback (FIXME: handling peer disconnects)<commit_after>package eth

import (
	"bytes"
	"math"
	"math/big"

	"github.com/ethereum/go-ethereum/core/types"
	"github.com/ethereum/go-ethereum/ethutil"
	"github.com/ethereum/go-ethereum/p2p"
	"github.com/ethereum/go-ethereum/rlp"
)

// ethProtocol represents the ethereum wire protocol
// instance is running on each peer
type ethProtocol struct {
	txPool       txPool
	chainManager chainManager
	blockPool    blockPool
	peer         *p2p.Peer
	id           string
	rw           p2p.MsgReadWriter
}

// backend is the interface the ethereum protocol backend should implement
// used as an argument to EthProtocol
type txPool interface {
	AddTransactions([]*types.Transaction)
}

type chainManager interface {
	GetBlockHashesFromHash(hash []byte, amount uint64) (hashes [][]byte)
	GetBlock(hash []byte) (block *types.Block)
	Status() (td *big.Int, currentBlock []byte, genesisBlock []byte)
}

type blockPool interface {
	AddBlockHashes(next func() ([]byte, bool), peerId string)
	AddBlock(block *types.Block, peerId string)
	AddPeer(td *big.Int, currentBlock []byte, peerId string, requestHashes func([]byte) error, requestBlocks func([][]byte) error, peerError func(int, string, ...interface{})) (best bool)
	RemovePeer(peerId string)
}

const (
	ProtocolVersion    = 43
	NetworkId          = 0
	ProtocolLength     = uint64(8)
	ProtocolMaxMsgSize = 10 * 1024 * 1024
)

// eth protocol message codes
const (
	StatusMsg = iota
	GetTxMsg  // unused
	TxMsg
	GetBlockHashesMsg
	BlockHashesMsg
	GetBlocksMsg
	BlocksMsg
	NewBlockMsg
)

// message structs used for rlp decoding
type newBlockMsgData struct {
	Block *types.Block
	TD    *big.Int
}

type getBlockHashesMsgData struct {
	Hash   []byte
	Amount uint64
}

// main entrypoint, wrappers starting a server running the eth protocol
// use this constructor to attach the protocol ("class") to server caps
// the Dev p2p layer then runs the protocol instance on each peer
func EthProtocol(txPool txPool, chainManager chainManager, blockPool blockPool) p2p.Protocol {
	return p2p.Protocol{
		Name:    "eth",
		Version: ProtocolVersion,
		Length:  ProtocolLength,
		Run: func(peer *p2p.Peer, rw p2p.MsgReadWriter) error {
			return runEthProtocol(txPool, chainManager, blockPool, peer, rw)
		},
	}
}

// the main loop that handles incoming messages
// note RemovePeer in the post-disconnect hook
func runEthProtocol(txPool txPool, chainManager chainManager, blockPool blockPool, peer *p2p.Peer, rw p2p.MsgReadWriter) (err error) {
	self := &ethProtocol{
		txPool:       txPool,
		chainManager: chainManager,
		blockPool:    blockPool,
		rw:           rw,
		peer:         peer,
		id:           (string)(peer.Identity().Pubkey()),
	}
	err = self.handleStatus()
	if err == nil {
		go func() {
			for {
				err = self.handle()
				if err != nil {
					self.blockPool.RemovePeer(self.id)
					break
				}
			}
		}()
	}
	return
}

func (self *ethProtocol) handle() error {
	msg, err := self.rw.ReadMsg()
	if err != nil {
		return err
	}
	if msg.Size > ProtocolMaxMsgSize {
		return ProtocolError(ErrMsgTooLarge, "%v > %v", msg.Size, ProtocolMaxMsgSize)
	}
	// make sure that the payload has been fully consumed
	defer msg.Discard()

	switch msg.Code {

	case StatusMsg:
		return ProtocolError(ErrExtraStatusMsg, "")

	case TxMsg:
		// TODO: rework using lazy RLP stream
		var txs []*types.Transaction
		if err := msg.Decode(&txs); err != nil {
			return ProtocolError(ErrDecode, "%v", err)
		}
		self.txPool.AddTransactions(txs)

	case GetBlockHashesMsg:
		var request getBlockHashesMsgData
		if err := msg.Decode(&request); err != nil {
			return ProtocolError(ErrDecode, "%v", err)
		}
		hashes := self.chainManager.GetBlockHashesFromHash(request.Hash, request.Amount)
		return self.rw.EncodeMsg(BlockHashesMsg, ethutil.ByteSliceToInterface(hashes)...)

	case BlockHashesMsg:
		// TODO: redo using lazy decode , this way very inefficient on known chains
		msgStream := rlp.NewListStream(msg.Payload, uint64(msg.Size))
		var err error
		iter := func() (hash []byte, ok bool) {
			hash, err = msgStream.Bytes()
			if err == nil {
				ok = true
			}
			return
		}
		self.blockPool.AddBlockHashes(iter, self.id)
		if err != nil && err != rlp.EOL {
			return ProtocolError(ErrDecode, "%v", err)
		}

	case GetBlocksMsg:
		var blockHashes [][]byte
		if err := msg.Decode(&blockHashes); err != nil {
			return ProtocolError(ErrDecode, "%v", err)
		}
		max := int(math.Min(float64(len(blockHashes)), blockHashesBatchSize))
		var blocks []interface{}
		for i, hash := range blockHashes {
			if i >= max {
				break
			}
			block := self.chainManager.GetBlock(hash)
			if block != nil {
				blocks = append(blocks, block.Value().Raw())
			}
		}
		return self.rw.EncodeMsg(BlocksMsg, blocks...)

	case BlocksMsg:
		msgStream := rlp.NewListStream(msg.Payload, uint64(msg.Size))
		for {
			var block *types.Block
			if err := msgStream.Decode(&block); err != nil {
				if err == rlp.EOL {
					break
				} else {
					return ProtocolError(ErrDecode, "%v", err)
				}
			}
			self.blockPool.AddBlock(block, self.id)
		}

	case NewBlockMsg:
		var request newBlockMsgData
		if err := msg.Decode(&request); err != nil {
			return ProtocolError(ErrDecode, "%v", err)
		}
		hash := request.Block.Hash()
		// to simplify backend interface adding a new block
		// uses AddPeer followed by AddHashes, AddBlock only if peer is the best peer
		// (or selected as new best peer)
		if self.blockPool.AddPeer(request.TD, hash, self.id, self.requestBlockHashes, self.requestBlocks, self.protoErrorDisconnect) {
			called := true
			iter := func() (hash []byte, ok bool) {
				if called {
					called = false
					return hash, true
				} else {
					return
				}
			}
			self.blockPool.AddBlockHashes(iter, self.id)
			self.blockPool.AddBlock(request.Block, self.id)
		}

	default:
		return ProtocolError(ErrInvalidMsgCode, "%v", msg.Code)
	}
	return nil
}

type statusMsgData struct {
	ProtocolVersion uint
	NetworkId       uint
	TD              *big.Int
	CurrentBlock    []byte
	GenesisBlock    []byte
}

func (self *ethProtocol) statusMsg() p2p.Msg {
	td, currentBlock, genesisBlock := self.chainManager.Status()

	return p2p.NewMsg(StatusMsg,
		uint32(ProtocolVersion),
		uint32(NetworkId),
		td,
		currentBlock,
		genesisBlock,
	)
}

func (self *ethProtocol) handleStatus() error {
	// send precanned status message
	if err := self.rw.WriteMsg(self.statusMsg()); err != nil {
		return err
	}

	// read and handle remote status
	msg, err := self.rw.ReadMsg()
	if err != nil {
		return err
	}

	if msg.Code != StatusMsg {
		return ProtocolError(ErrNoStatusMsg, "first msg has code %x (!= %x)", msg.Code, StatusMsg)
	}

	if msg.Size > ProtocolMaxMsgSize {
		return ProtocolError(ErrMsgTooLarge, "%v > %v", msg.Size, ProtocolMaxMsgSize)
	}

	var status statusMsgData
	if err := msg.Decode(&status); err != nil {
		return ProtocolError(ErrDecode, "%v", err)
	}

	_, _, genesisBlock := self.chainManager.Status()

	if bytes.Compare(status.GenesisBlock, genesisBlock) != 0 {
		return ProtocolError(ErrGenesisBlockMismatch, "%x (!= %x)", status.GenesisBlock, genesisBlock)
	}

	if status.NetworkId != NetworkId {
		return ProtocolError(ErrNetworkIdMismatch, "%d (!= %d)", status.NetworkId, NetworkId)
	}

	if ProtocolVersion != status.ProtocolVersion {
		return ProtocolError(ErrProtocolVersionMismatch, "%d (!= %d)", status.ProtocolVersion, ProtocolVersion)
	}

	self.peer.Infof("Peer is [eth] capable (%d/%d). TD = %v ~ %x", status.ProtocolVersion, status.NetworkId, status.CurrentBlock)

	self.blockPool.AddPeer(status.TD, status.CurrentBlock, self.id, self.requestBlockHashes, self.requestBlocks, self.protoErrorDisconnect)

	return nil
}

func (self *ethProtocol) requestBlockHashes(from []byte) error {
	self.peer.Debugf("fetching hashes (%d) %x...\n", blockHashesBatchSize, from[0:4])
	return self.rw.EncodeMsg(GetBlockHashesMsg, from, blockHashesBatchSize)
}

func (self *ethProtocol) requestBlocks(hashes [][]byte) error {
	self.peer.Debugf("fetching %v blocks", len(hashes))
	return self.rw.EncodeMsg(GetBlocksMsg, ethutil.ByteSliceToInterface(hashes))
}

func (self *ethProtocol) protoError(code int, format string, params ...interface{}) (err *protocolError) {
	err = ProtocolError(code, format, params...)
	if err.Fatal() {
		self.peer.Errorln(err)
	} else {
		self.peer.Debugln(err)
	}
	return
}

func (self *ethProtocol) protoErrorDisconnect(code int, format string, params ...interface{}) {
	err := ProtocolError(code, format, params...)
	if err.Fatal() {
		self.peer.Errorln(err)
		// disconnect
	} else {
		self.peer.Debugln(err)
	}

}
<|endoftext|>


<commit_before>package batch

import (
	"bufio"
	"bytes"
	"errors"
	"io"
	"io/ioutil"
	"log"
	"mime"
	"mime/multipart"
	"net/http"
	"net/textproto"
	"strings"
	"time"

	"github.com/8legd/RRP/processors"
)

// MultipartMixed handles a batch of HTTP requests in `multipart/mixed` format.
// Each part contains `application/http` content representing an individual request.
// Once processed, HTTP responses are returned as `application/http` content in
// the same sequence as the corresponding requests.
func MultipartMixed(w http.ResponseWriter, r *http.Request) {
	var batch []*http.Request
	ct, params, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
	if err != nil {
		log.Println("Error:", err)
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	if ct != "multipart/mixed" {
		err = errors.New("unsupported content type, expected multipart/mixed")
		log.Println("Error:", err)
		http.Error(w, err.Error(), http.StatusUnsupportedMediaType)
		return
	}
	// check for optional timeout header
	tm := r.Header.Get("x-rrp-timeout")
	var timeout time.Duration
	if tm != "" {
		timeout, err = time.ParseDuration(tm + "s")
		if err != nil {
			log.Println("Error:", err)
			http.Error(w, "invalid value for x-rrp-timeout header, expected number of seconds", http.StatusBadRequest)
			return
		}
		log.Println("With specified timeout", timeout)
	} else {
		timeout = time.Duration(20) * time.Second // Default timeout is 20 seconds
		log.Println("With default timeout", timeout)
	}
	boundary, ok := params["boundary"]
	if !ok {
		err = errors.New("missing multipart boundary")
		log.Println("Error:", err)
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	mr := multipart.NewReader(r.Body, boundary)
	var urls []string
	for {
		p, err := mr.NextPart()
		if err == io.EOF {
			if len(batch) < 1 {
				err = errors.New("invalid multipart content")
				log.Println("Error:", err)
				http.Error(w, err.Error(), http.StatusBadRequest)
				return
			}
			break // finished reading multpart parts
		}
		if err != nil {
			log.Println("Error:", err)
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		// check next part's contet type
		pct, _, err := mime.ParseMediaType(p.Header.Get("Content-Type"))
		if err != nil {
			log.Println("Error:", err)
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		if pct != "application/http" {
			err = errors.New("unsupported content type for multipart/mixed content, expected each part to be application/http")
			log.Println("Error:", err)
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}
		pr, err := http.ReadRequest(bufio.NewReader(p))
		// we need to get the protocol from a header in the part's request
		protocol := pr.Header.Get("Forwarded")
		if protocol == "" || !strings.Contains(protocol, "proto=http") { // proto must be `http` or `https`
			err = errors.New("missing header in multipart/mixed content, expected each part to contain a Forwarded header with a valid proto value (proto=http or proto=https)")
			log.Println("Error:", err)
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}
		parts := strings.Split(protocol, "proto=")
		if len(parts) < 2 || (parts[1] != "http" && parts[1] != "https") {
			err = errors.New("invalid proto value in Forwarded header, expected proto=http or proto=https")
			log.Println("Error:", err)
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}
		protocol = parts[1]
		url := protocol + "://" + pr.Host + pr.RequestURI
		urls = append(urls, url)
		// read part's body
		// NOTE: if there is no Content-Length header the body will not have been read (will be empty)
		pb, err := ioutil.ReadAll(pr.Body)
		if err != nil {
			log.Println("Error:", err)
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		request, err := http.NewRequest(r.Method, url, bytes.NewBuffer(pb))
		if err != nil {
			log.Println("Error:", err)
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		// add headers
		request.Header = pr.Header
		batch = append(batch, request)
	}
	responses, err := processors.ProcessBatch(batch, timeout)
	if err != nil {
		log.Println("Error:", err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	// create some variables to keep track of the indvidual responses as we process them
	// (these are used for reporting via log output e.g. should a runtime panic occur)
	var nextIndex int
	var nextResponse *http.Response

	// send a multipart response back
	// (We need to buffer this in case there is an error. If we didn't and wrote
	// directly to the response stream it would implicitly set a status of 200 OK)

	var buf bytes.Buffer
	mw := multipart.NewWriter(&buf)

	defer func() {
		mw.Close()

		// Report state on any panic
		if r := recover(); r != nil {
			log.Println("Reporting state on panic", r)
			log.Println("Request", batch[nextIndex])
			log.Println("Response", nextResponse)
			err = errors.New("panic while processing " + urls[nextIndex])
			log.Println("Error:", err)
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
	}()

	w.Header().Set("Content-Type", "multipart/mixed; boundary="+mw.Boundary())

	var pw io.Writer
	var pb []byte

	// the individual response are sent as `application/http` as per requests
	for nextIndex, nextResponse = range responses {
		ph := make(textproto.MIMEHeader)
		ph.Set("Content-Type", "application/http")
		pw, err = mw.CreatePart(ph)
		if err != nil {
			log.Println("Error:", err)
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		if nextResponse != nil {
			log.Println("Received ", nextResponse.Status, "from", urls[nextIndex])
			io.WriteString(pw, nextResponse.Proto+" "+nextResponse.Status+"\r\n")
			if nextResponse.Header != nil {
				nextResponse.Header.Write(pw)
				io.WriteString(pw, "\r\n")
			}
			if nextResponse.Body != nil {
				// Close the underlying connection so it can be re-used
				defer nextResponse.Body.Close()
				pb, err = ioutil.ReadAll(nextResponse.Body)
				if err != nil {
					log.Println("Error:", err)
					http.Error(w, err.Error(), http.StatusInternalServerError)
					return
				}
				pw.Write(pb)
				io.WriteString(pw, "\r\n")
			}
		} else {
			err = errors.New("missing response for " + urls[nextIndex])
			log.Println("Error:", err)
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
	}

	// write response
	w.Write(buf.Bytes())
	return
}
<commit_msg>started work on improved timeout handling<commit_after>package batch

import (
	"bufio"
	"bytes"
	"errors"
	"io"
	"io/ioutil"
	"log"
	"mime"
	"mime/multipart"
	"net/http"
	"net/textproto"
	"strings"
	"time"

	"github.com/8legd/RRP/processors"
)

func writeResponse(nextURL string, nextResponse *http.Response, partWriter io.Writer, timeout time.Duration) error {

	totalBytesRead := 0
	startedReading := time.Now()

	// Defer closing of underlying connection so it can be re-used
	defer func() {
		nextResponse.Body.Close()
		log.Println("Read", totalBytesRead, "bytes from", nextURL, "in", time.Since(startedReading))
	}()

	for {
		chunk := make([]byte, 8192)
		lastReadLength, err := nextResponse.Body.Read(chunk)
		totalBytesRead = totalBytesRead + lastReadLength
		if lastReadLength > 0 && lastReadLength < 8192 {
			chunk = chunk[0:lastReadLength]
		}

		if lastReadLength < 1 || err == io.EOF {
			if lastReadLength > 0 {
				partWriter.Write(chunk)
			}
			io.WriteString(partWriter, "\r\n")
			return nil
		}
		if err != nil {
			return err
		}
		partWriter.Write(chunk)
	}
}

// MultipartMixed handles a batch of HTTP requests in `multipart/mixed` format.
// Each part contains `application/http` content representing an individual request.
// Once processed, HTTP responses are returned as `application/http` content in
// the same sequence as the corresponding requests.
func MultipartMixed(w http.ResponseWriter, r *http.Request) {
	var batch []*http.Request
	ct, params, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
	if err != nil {
		log.Println("Error:", err)
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	if ct != "multipart/mixed" {
		err = errors.New("unsupported content type, expected multipart/mixed")
		log.Println("Error:", err)
		http.Error(w, err.Error(), http.StatusUnsupportedMediaType)
		return
	}
	// check for optional timeout header
	tm := r.Header.Get("x-rrp-timeout")
	var timeout time.Duration
	if tm != "" {
		timeout, err = time.ParseDuration(tm + "s")
		if err != nil {
			log.Println("Error:", err)
			http.Error(w, "invalid value for x-rrp-timeout header, expected number of seconds", http.StatusBadRequest)
			return
		}
		log.Println("With specified timeout", timeout)
	} else {
		timeout = time.Duration(20) * time.Second // Default timeout is 20 seconds
		log.Println("With default timeout", timeout)
	}
	boundary, ok := params["boundary"]
	if !ok {
		err = errors.New("missing multipart boundary")
		log.Println("Error:", err)
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	mr := multipart.NewReader(r.Body, boundary)
	var urls []string
	for {
		p, err := mr.NextPart()
		if err == io.EOF {
			if len(batch) < 1 {
				err = errors.New("invalid multipart content")
				log.Println("Error:", err)
				http.Error(w, err.Error(), http.StatusBadRequest)
				return
			}
			break // finished reading multpart parts
		}
		if err != nil {
			log.Println("Error:", err)
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		// check next part's contet type
		pct, _, err := mime.ParseMediaType(p.Header.Get("Content-Type"))
		if err != nil {
			log.Println("Error:", err)
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		if pct != "application/http" {
			err = errors.New("unsupported content type for multipart/mixed content, expected each part to be application/http")
			log.Println("Error:", err)
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}
		pr, err := http.ReadRequest(bufio.NewReader(p))
		// we need to get the protocol from a header in the part's request
		protocol := pr.Header.Get("Forwarded")
		if protocol == "" || !strings.Contains(protocol, "proto=http") { // proto must be `http` or `https`
			err = errors.New("missing header in multipart/mixed content, expected each part to contain a Forwarded header with a valid proto value (proto=http or proto=https)")
			log.Println("Error:", err)
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}
		parts := strings.Split(protocol, "proto=")
		if len(parts) < 2 || (parts[1] != "http" && parts[1] != "https") {
			err = errors.New("invalid proto value in Forwarded header, expected proto=http or proto=https")
			log.Println("Error:", err)
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}
		protocol = parts[1]
		url := protocol + "://" + pr.Host + pr.RequestURI
		urls = append(urls, url)
		// read part's body
		// NOTE: if there is no Content-Length header the body will not have been read (will be empty)
		// TODO ditch ReadAll ()
		pb, err := ioutil.ReadAll(pr.Body)
		if err != nil {
			log.Println("Error:", err)
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		request, err := http.NewRequest(r.Method, url, bytes.NewBuffer(pb))
		if err != nil {
			log.Println("Error:", err)
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		// add headers
		request.Header = pr.Header
		batch = append(batch, request)
	}

	startedProcessing := time.Now()
	responses, err := processors.ProcessBatch(batch, timeout)
	if err != nil {
		log.Println("Error:", err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	// create some variables to keep track of the indvidual responses as we process them
	// (these are used for reporting via log output e.g. should a runtime panic occur)
	var nextIndex int
	var nextResponse *http.Response

	// send a multipart response back
	// (We need to buffer this in case there is an error. If we didn't and wrote
	// directly to the response stream it would implicitly set a status of 200 OK)

	var buf bytes.Buffer
	mw := multipart.NewWriter(&buf)

	defer func() {
		mw.Close()

		// Report state on any panic
		if r := recover(); r != nil {
			log.Println("Reporting state on panic", r)
			log.Println("Request", batch[nextIndex])
			log.Println("Response", nextResponse)
			err = errors.New("panic while processing " + urls[nextIndex])
			log.Println("Error:", err)
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
	}()

	w.Header().Set("Content-Type", "multipart/mixed; boundary="+mw.Boundary())

	var pw io.Writer

	// the individual response are sent as `application/http` as per requests
	for nextIndex, nextResponse = range responses {
		ph := make(textproto.MIMEHeader)
		ph.Set("Content-Type", "application/http")
		pw, err = mw.CreatePart(ph)
		if err != nil {
			log.Println("Error:", err)
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		if nextResponse != nil {
			log.Println("Received", nextResponse.Status, "from", urls[nextIndex], "in", time.Since(startedProcessing))
			io.WriteString(pw, nextResponse.Proto+" "+nextResponse.Status+"\r\n")
			if nextResponse.Header != nil {
				nextResponse.Header.Write(pw)
				io.WriteString(pw, "\r\n")
			}
			if nextResponse.Body != nil {
				err = writeResponse(urls[nextIndex], nextResponse, pw, timeout-time.Since(startedProcessing))
				if err != nil {
					log.Println("Error:", err)
					http.Error(w, err.Error(), http.StatusInternalServerError)
					return
				}
			}
		} else {
			err = errors.New("missing response for " + urls[nextIndex])
			log.Println("Error:", err)
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
	}

	// write response
	w.Write(buf.Bytes())
	return
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"io"
	"log"
	"net/http"
	"time"

	. "github.com/Bluek404/go2html5"
)

func main() {
	html := Html(Attr{"lang": "cn"},
		Head(nil,
			Title(nil, "Go2HTML5 Example"),
		),
		Body(nil,
			P(nil,
				"现在是：",
				func() string {
					var s bytes.Buffer
					t := time.Now().Hour()
					switch {
					case t >= 0 && t <= 4:
						s.WriteString("凌晨")
					case t >= 5 && t <= 7:
						s.WriteString("早上")
					case t >= 8 && t < 10:
						s.WriteString("上午")
					case t >= 11 && t <= 13:
						s.WriteString("中午")
					case t >= 14 && t <= 19:
						s.WriteString("下午")
					case t >= 20 && t <= 22:
						s.WriteString("晚上")
					case t >= 23 && t <= 24:
						s.WriteString("深夜")
					}
					s.WriteString(time.Now().Format("3点4分5秒"))
					return s.String()
				}(),
			),
			Hr(nil),
			P(nil,
				"Written in ", A(Attr{"href": "http://golang.org"}, "Go!"),
			),
		),
	)

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		io.WriteString(w, html)
	})

	err := http.ListenAndServe("127.0.0.1:8080", nil)
	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>modify: 完善例子<commit_after>package main

import (
	"bytes"
	"io"
	"log"
	"net/http"
	"time"

	. "github.com/Bluek404/go2html5"
)

func main() {
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		t := time.Now().UnixNano()
		io.WriteString(w, index())
		log.Println(r.RemoteAddr, r.Method, r.URL, time.Now().UnixNano()-t, "ns")
	})

	err := http.ListenAndServe("127.0.0.1:8080", nil)
	if err != nil {
		log.Fatal(err)
	}
}

func index() string {
	html := Html(Attr{"lang": "cn"},
		Head(nil,
			Title(nil, "Go2HTML5 Example"),
		),
		Body(nil,
			P(nil,
				"现在是：",
				func() string {
					var s bytes.Buffer
					t := time.Now().Hour()
					switch {
					case t >= 0 && t <= 4:
						s.WriteString("凌晨")
					case t >= 5 && t <= 7:
						s.WriteString("早上")
					case t >= 8 && t < 10:
						s.WriteString("上午")
					case t >= 11 && t <= 13:
						s.WriteString("中午")
					case t >= 14 && t <= 19:
						s.WriteString("下午")
					case t >= 20 && t <= 22:
						s.WriteString("晚上")
					case t >= 23 && t <= 24:
						s.WriteString("深夜")
					}
					s.WriteString(time.Now().Format("3点4分5秒"))
					return s.String()
				}(),
			),
			Hr(nil),
			P(nil,
				"Written in ", A(Attr{"href": "http://golang.org"}, "Go!"),
			),
		),
	)
	return html
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"errors"
	"fmt"
	"net"

	"github.com/lucas-clemente/quic-go"
	"github.com/lucas-clemente/quic-go/crypto"
)

const (
	// QuicVersion32 is Q032
	QuicVersion32 uint32 = 'Q' + '0'<<8 + '3'<<16 + '2'<<24
)

func main() {
	path := "/Users/lucas/src/go/src/github.com/lucas-clemente/quic-go/example/"
	keyData, err := crypto.LoadKeyData(path+"cert.der", path+"key.der")
	if err != nil {
		panic(err)
	}

	addr, err := net.ResolveUDPAddr("udp", "localhost:6121")
	if err != nil {
		panic(err)
	}

	conn, err := net.ListenUDP("udp", addr)
	if err != nil {
		panic(err)
	}

	data := make([]byte, 0x10000)
	n, remoteAddr, err := conn.ReadFromUDP(data)
	if err != nil {
		panic(err)
	}
	data = data[:n]
	r := bytes.NewReader(data)

	fmt.Printf("Number of bytes: %d\n", n)
	fmt.Printf("Remote addr: %v\n", remoteAddr)

	publicHeader, err := quic.ParsePublicHeader(r)
	if err != nil {
		panic(err)
	}

	if publicHeader.VersionFlag && publicHeader.QuicVersion < QuicVersion32 {
		println(publicHeader.QuicVersion)
		panic("only versions >= Q032 supported")
	}

	nullAEAD := &crypto.NullAEAD{}
	r, err = nullAEAD.Open(data[0:int(r.Size())-r.Len()], r)
	if err != nil {
		panic(err)
	}

	privateFlag, err := r.ReadByte()
	if err != nil {
		panic(err)
	}

	if privateFlag&0x02 > 0 || privateFlag&0x04 > 0 {
		panic(errors.New("FEC packets are not implemented"))
	}

	frame, err := quic.ParseStreamFrame(r)
	if err != nil {
		panic(err)
	}

	messageTag, cryptoData, err := quic.ParseCryptoMessage(frame.Data)
	if err != nil {
		panic(err)
	}

	fmt.Printf("Tag: %d\n", messageTag)
	fmt.Printf("Talking to: %q\n", cryptoData[quic.TagUAID])

	serverConfig := &bytes.Buffer{}
	quic.WriteCryptoMessage(serverConfig, quic.TagSCFG, map[quic.Tag][]byte{
		quic.TagSCID: []byte{0xC5, 0x1C, 0x73, 0x6B, 0x8F, 0x48, 0x49, 0xAE, 0xB3, 0x00, 0xA2, 0xD4, 0x4B, 0xA0, 0xCF, 0xDF},
		quic.TagKEXS: []byte("C255"),
		quic.TagAEAD: []byte("AESG"),
		quic.TagPUBS: []byte{0x1, 0x2, 0x3},
		quic.TagORBT: []byte{0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7},
		quic.TagEXPY: []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff},
		quic.TagVER:  []byte("Q032"),
	})

	proof, err := keyData.SignServerProof(frame.Data, serverConfig.Bytes())
	if err != nil {
		panic(err)
	}
	serverReply := &bytes.Buffer{}
	quic.WriteCryptoMessage(serverReply, quic.TagREJ, map[quic.Tag][]byte{
		quic.TagSCFG: serverConfig.Bytes(),
		quic.TagCERT: keyData.GetCERTdata(),
		quic.TagPROF: proof,
	})

	replyFrame := &bytes.Buffer{}
	replyFrame.WriteByte(0) // Private header
	quic.WriteAckFrame(replyFrame, &quic.AckFrame{
		LargestObserved: 1,
	})
	quic.WriteStreamFrame(replyFrame, &quic.StreamFrame{
		StreamID: 1,
		Data:     serverReply.Bytes(),
	})

	fullReply := &bytes.Buffer{}
	quic.WritePublicHeader(fullReply, &quic.PublicHeader{
		ConnectionID: publicHeader.ConnectionID,
		PacketNumber: 1,
	})

	nullAEAD.Seal(fullReply, fullReply.Bytes(), replyFrame.Bytes())

	conn.WriteToUDP(fullReply.Bytes(), remoteAddr)

	n, _, err = conn.ReadFromUDP(data)
	if err != nil {
		panic(err)
	}
	data = data[:n]
	r = bytes.NewReader(data)

	fmt.Printf("%v\n", data)

	publicHeader, err = quic.ParsePublicHeader(r)
	if err != nil {
		panic(err)
	}
	fmt.Printf("%#v\n", publicHeader)
}
<commit_msg>read certificate path from $GOPATH in example server<commit_after>package main

import (
	"bytes"
	"errors"
	"fmt"
	"net"
	"os"

	"github.com/lucas-clemente/quic-go"
	"github.com/lucas-clemente/quic-go/crypto"
)

const (
	// QuicVersion32 is Q032
	QuicVersion32 uint32 = 'Q' + '0'<<8 + '3'<<16 + '2'<<24
)

func main() {
	path := os.Getenv("GOPATH") + "/src/github.com/lucas-clemente/quic-go/example/"
	keyData, err := crypto.LoadKeyData(path+"cert.der", path+"key.der")
	if err != nil {
		panic(err)
	}

	addr, err := net.ResolveUDPAddr("udp", "localhost:6121")
	if err != nil {
		panic(err)
	}

	conn, err := net.ListenUDP("udp", addr)
	if err != nil {
		panic(err)
	}

	data := make([]byte, 0x10000)
	n, remoteAddr, err := conn.ReadFromUDP(data)
	if err != nil {
		panic(err)
	}
	data = data[:n]
	r := bytes.NewReader(data)

	fmt.Printf("Number of bytes: %d\n", n)
	fmt.Printf("Remote addr: %v\n", remoteAddr)

	publicHeader, err := quic.ParsePublicHeader(r)
	if err != nil {
		panic(err)
	}

	if publicHeader.VersionFlag && publicHeader.QuicVersion < QuicVersion32 {
		println(publicHeader.QuicVersion)
		panic("only versions >= Q032 supported")
	}

	nullAEAD := &crypto.NullAEAD{}
	r, err = nullAEAD.Open(data[0:int(r.Size())-r.Len()], r)
	if err != nil {
		panic(err)
	}

	privateFlag, err := r.ReadByte()
	if err != nil {
		panic(err)
	}

	if privateFlag&0x02 > 0 || privateFlag&0x04 > 0 {
		panic(errors.New("FEC packets are not implemented"))
	}

	frame, err := quic.ParseStreamFrame(r)
	if err != nil {
		panic(err)
	}

	messageTag, cryptoData, err := quic.ParseCryptoMessage(frame.Data)
	if err != nil {
		panic(err)
	}

	fmt.Printf("Tag: %d\n", messageTag)
	fmt.Printf("Talking to: %q\n", cryptoData[quic.TagUAID])

	serverConfig := &bytes.Buffer{}
	quic.WriteCryptoMessage(serverConfig, quic.TagSCFG, map[quic.Tag][]byte{
		quic.TagSCID: []byte{0xC5, 0x1C, 0x73, 0x6B, 0x8F, 0x48, 0x49, 0xAE, 0xB3, 0x00, 0xA2, 0xD4, 0x4B, 0xA0, 0xCF, 0xDF},
		quic.TagKEXS: []byte("C255"),
		quic.TagAEAD: []byte("AESG"),
		quic.TagPUBS: []byte{0x1, 0x2, 0x3},
		quic.TagORBT: []byte{0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7},
		quic.TagEXPY: []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff},
		quic.TagVER:  []byte("Q032"),
	})

	proof, err := keyData.SignServerProof(frame.Data, serverConfig.Bytes())
	if err != nil {
		panic(err)
	}
	serverReply := &bytes.Buffer{}
	quic.WriteCryptoMessage(serverReply, quic.TagREJ, map[quic.Tag][]byte{
		quic.TagSCFG: serverConfig.Bytes(),
		quic.TagCERT: keyData.GetCERTdata(),
		quic.TagPROF: proof,
	})

	replyFrame := &bytes.Buffer{}
	replyFrame.WriteByte(0) // Private header
	quic.WriteAckFrame(replyFrame, &quic.AckFrame{
		LargestObserved: 1,
	})
	quic.WriteStreamFrame(replyFrame, &quic.StreamFrame{
		StreamID: 1,
		Data:     serverReply.Bytes(),
	})

	fullReply := &bytes.Buffer{}
	quic.WritePublicHeader(fullReply, &quic.PublicHeader{
		ConnectionID: publicHeader.ConnectionID,
		PacketNumber: 1,
	})

	nullAEAD.Seal(fullReply, fullReply.Bytes(), replyFrame.Bytes())

	conn.WriteToUDP(fullReply.Bytes(), remoteAddr)

	n, _, err = conn.ReadFromUDP(data)
	if err != nil {
		panic(err)
	}
	data = data[:n]
	r = bytes.NewReader(data)

	fmt.Printf("%v\n", data)

	publicHeader, err = quic.ParsePublicHeader(r)
	if err != nil {
		panic(err)
	}
	fmt.Printf("%#v\n", publicHeader)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"runtime"
	"time"

	"github.com/smartystreets/go-disruptor"
)

const (
	BufferSize = 1024 * 64
	BufferMask = BufferSize - 1
	Iterations = 1000000 * 100
)

var ringBuffer = [BufferSize]int64{}

func main() {
	runtime.GOMAXPROCS(2)

	written, read := disruptor.NewCursor(), disruptor.NewCursor()
	reader := disruptor.NewReader(read, written, written, SampleConsumer{})

	started := time.Now()
	reader.Start()
	// publish(written, read)
	publish(disruptor.NewWriter(written, read, BufferSize))
	reader.Stop()
	finished := time.Now()
	fmt.Println(Iterations, finished.Sub(started))
}

func publish(writer *disruptor.Writer) {
	for sequence := disruptor.InitialSequenceValue; sequence <= Iterations; {
		sequence = writer.Reserve()
		ringBuffer[sequence&BufferMask] = sequence
		writer.Commit(sequence)
	}
}

// func publish(written, read *disruptor.Cursor) {
// 	previous := disruptor.InitialSequenceValue
// 	gate := disruptor.InitialSequenceValue

// 	for previous <= Iterations {
// 		next := previous + 1
// 		wrap := next - BufferSize

// 		for wrap > gate {
// 			gate = read.Sequence
// 		}

// 		ringBuffer[next&BufferMask] = next
// 		written.Sequence = next
// 		previous = next
// 	}
// }

type SampleConsumer struct{}

func (this SampleConsumer) Consume(lower, upper int64) {
	for lower <= upper {
		message := ringBuffer[lower&BufferMask]
		if message != lower {
			fmt.Println("Race condition", message, lower)
			panic("Race condition")
		}
		lower++
	}
}
<commit_msg>1.5ns per operation when using correct Barrier abstraction on stack-based writer.<commit_after>package main

import (
	"fmt"
	"runtime"
	"time"

	"github.com/smartystreets/go-disruptor"
)

const (
	BufferSize = 1024 * 64
	BufferMask = BufferSize - 1
	Iterations = 1000000 * 100
)

var ringBuffer = [BufferSize]int64{}

func main() {
	runtime.GOMAXPROCS(2)

	written, read := disruptor.NewCursor(), disruptor.NewCursor()
	reader := disruptor.NewReader(read, written, written, SampleConsumer{})

	started := time.Now()
	reader.Start()
	publish(written, read)
	// publish(disruptor.NewWriter(written, read, BufferSize))
	reader.Stop()
	finished := time.Now()
	fmt.Println(Iterations, finished.Sub(started))
}

// func publish(writer *disruptor.Writer) {
// 	for sequence := disruptor.InitialSequenceValue; sequence <= Iterations; {
// 		sequence = writer.Reserve()
// 		ringBuffer[sequence&BufferMask] = sequence
// 		writer.Commit(sequence)
// 	}
// }

func publish(written *disruptor.Cursor, upstream disruptor.Barrier) {
	previous := disruptor.InitialSequenceValue
	gate := disruptor.InitialSequenceValue

	for previous <= Iterations {
		next := previous + 1
		wrap := next - BufferSize

		for wrap > gate {
			gate = upstream.Read(next)
		}

		ringBuffer[next&BufferMask] = next
		written.Store(next)
		previous = next
	}
}

type SampleConsumer struct{}

func (this SampleConsumer) Consume(lower, upper int64) {
	for lower <= upper {
		message := ringBuffer[lower&BufferMask]
		if message != lower {
			fmt.Println("Race condition", message, lower)
			panic("Race condition")
		}
		lower++
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"runtime"
	"time"

	"github.com/smartystreets/go-disruptor"
)

const (
	BufferSize = 1024 * 64
	BufferMask = BufferSize - 1
	Iterations = 1000000 * 100
)

var ringBuffer = [BufferSize]int64{}

func main() {
	runtime.GOMAXPROCS(2)

	written, read := disruptor.NewCursor(), disruptor.NewCursor()
	reader := disruptor.NewReader(read, written, written, SampleConsumer{})

	started := time.Now()
	reader.Start()
	publish(written, read)
	reader.Stop()
	finished := time.Now()
	fmt.Println(Iterations, finished.Sub(started))

	time.Sleep(time.Millisecond * 10)
}

func publish(written, read *disruptor.Cursor) {
	previous := disruptor.InitialSequenceValue
	gate := disruptor.InitialSequenceValue

	for previous <= Iterations {
		next := previous + 1
		wrap := next - BufferSize

		for wrap > gate {
			gate = read.Sequence
		}

		ringBuffer[next&BufferMask] = next
		written.Sequence = next
		previous = next
	}
}

type SampleConsumer struct{}

func (this SampleConsumer) Consume(lower, upper int64) {
	for lower <= upper {
		message := ringBuffer[lower&BufferMask]
		if message != lower {
			fmt.Println("Race condition", message, lower)
			panic("Race condition")
		}
		lower++
	}
}
<commit_msg>Experimenting with different writing techniques.<commit_after>package main

import (
	"fmt"
	"runtime"
	"time"

	"github.com/smartystreets/go-disruptor"
)

const (
	BufferSize = 1024 * 64
	BufferMask = BufferSize - 1
	Iterations = 1000000 * 100
)

var ringBuffer = [BufferSize]int64{}

func main() {
	runtime.GOMAXPROCS(2)

	written, read := disruptor.NewCursor(), disruptor.NewCursor()
	reader := disruptor.NewReader(read, written, written, SampleConsumer{})

	started := time.Now()
	reader.Start()
	publish(written, read)
	reader.Stop()
	finished := time.Now()
	fmt.Println(Iterations, finished.Sub(started))

	time.Sleep(time.Millisecond * 10)
}

func publish(written, read *disruptor.Cursor) {

	// sequence := disruptor.InitialSequenceValue
	// writer := &disruptor.Writer2{}

	// // writer := disruptor.NewWriter(written, read, BufferSize)
	// for sequence < Iterations {
	// 	sequence = writer.Reserve()
	// }

	// sequence := disruptor.InitialSequenceValue
	// writer := disruptor.NewWriter(written, read, BufferSize)

	// for sequence <= Iterations {
	// 	sequence = writer.Reserve()
	// 	ringBuffer[sequence&BufferMask] = sequence
	// 	written.Sequence = sequence
	// 	// writer.Commit(sequence)
	// }

	// fmt.Println(writer.Gating())

	gating := 0

	previous := disruptor.InitialSequenceValue
	gate := disruptor.InitialSequenceValue

	for previous <= Iterations {
		next := previous + 1
		wrap := next - BufferSize

		for wrap > gate {
			gate = read.Sequence
			gating++
		}

		ringBuffer[next&BufferMask] = next
		written.Sequence = next
		previous = next
	}

	fmt.Println("Gating", gating)
}

type SampleConsumer struct{}

func (this SampleConsumer) Consume(lower, upper int64) {
	for lower <= upper {
		message := ringBuffer[lower&BufferMask]
		if message != lower {
			fmt.Println("Race condition", message, lower)
			panic("Race condition")
		}
		lower++
	}
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package binary

// This file implements "varint" encoding of 64-bit integers.
// The encoding is:
// - unsigned integers are serialized 7 bits at a time, starting with the
//   least significant bits
// - the most significant bit (msb) in each output byte indicates if there
//   is a continuation byte (msb = 1)
// - signed integers are mapped to unsigned integers using "zig-zag"
//   encoding: Positive values x are written as 2*x + 0, negative values
//   are written as 2*(^x) + 1; that is, negative numbers are complemented
//   and whether to complement is encoded in bit 0.
//
// Design note:
// At most 10 bytes are needed for 64-bit values. The encoding could
// be more dense: a full 64-bit value needs an extra byte just to hold bit 63.
// Instead, the msb of the previous byte could be used to hold bit 63 since we
// know there can't be more than 64 bits. This is a trivial improvement and
// would reduce the maximum encoding length to 9 bytes. However, it breaks the
// invariant that the msb is always the "continuation bit" and thus makes the
// format incompatible with a varint encoding for larger numbers (say 128-bit).

import (
	"errors"
	"io"
)

// MaxVarintLenN is the maximum length of a varint-encoded N-bit integer.
const (
	MaxVarintLen16 = 3
	MaxVarintLen32 = 5
	MaxVarintLen64 = 10
)

// PutUvarint encodes a uint64 into buf and returns the number of bytes written.
// If the buffer is too small, PutUvarint will panic.
func PutUvarint(buf []byte, x uint64) int {
	i := 0
	for x >= 0x80 {
		buf[i] = byte(x) | 0x80
		x >>= 7
		i++
	}
	buf[i] = byte(x)
	return i + 1
}

// Uvarint decodes a uint64 from buf and returns that value and the
// number of bytes read (> 0). If an error occurred, the value is 0
// and the number of bytes n is <= 0 meaning:
//
//	n == 0: buf too small
//	n  < 0: value larger than 64 bits (overflow)
//              and -n is the number of bytes read
//
func Uvarint(buf []byte) (uint64, int) {
	var x uint64
	var s uint
	for i, b := range buf {
		if b < 0x80 {
			if i > 9 || i == 9 && b > 1 {
				return 0, -(i + 1) // overflow
			}
			return x | uint64(b)<<s, i + 1
		}
		x |= uint64(b&0x7f) << s
		s += 7
	}
	return 0, 0
}

// PutVarint encodes an int64 into buf and returns the number of bytes written.
// If the buffer is too small, PutVarint will panic.
func PutVarint(buf []byte, x int64) int {
	ux := uint64(x) << 1
	if x < 0 {
		ux = ^ux
	}
	return PutUvarint(buf, ux)
}

// Varint decodes an int64 from buf and returns that value and the
// number of bytes read (> 0). If an error occurred, the value is 0
// and the number of bytes n is <= 0 with the following meaning:
//
//	n == 0: buf too small
//	n  < 0: value larger than 64 bits (overflow)
//              and -n is the number of bytes read
//
func Varint(buf []byte) (int64, int) {
	ux, n := Uvarint(buf) // ok to continue in presence of error
	x := int64(ux >> 1)
	if ux&1 != 0 {
		x = ^x
	}
	return x, n
}

var overflow = errors.New("binary: varint overflows a 64-bit integer")

// ReadUvarint reads an encoded unsigned integer from r and returns it as a uint64.
func ReadUvarint(r io.ByteReader) (uint64, error) {
	var x uint64
	var s uint
	for i := 0; ; i++ {
		b, err := r.ReadByte()
		if err != nil {
			return x, err
		}
		if b < 0x80 {
			if i > 9 || i == 9 && b > 1 {
				return x, overflow
			}
			return x | uint64(b)<<s, nil
		}
		x |= uint64(b&0x7f) << s
		s += 7
	}
	panic("unreachable")
}

// ReadVarint reads an encoded unsigned integer from r and returns it as a uint64.
func ReadVarint(r io.ByteReader) (int64, error) {
	ux, err := ReadUvarint(r) // ok to continue in presence of error
	x := int64(ux >> 1)
	if ux&1 != 0 {
		x = ^x
	}
	return x, err
}
<commit_msg>encoding/binary: fix typo: ReadVarint returns an int64, not a uint64.<commit_after>// Copyright 2011 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package binary

// This file implements "varint" encoding of 64-bit integers.
// The encoding is:
// - unsigned integers are serialized 7 bits at a time, starting with the
//   least significant bits
// - the most significant bit (msb) in each output byte indicates if there
//   is a continuation byte (msb = 1)
// - signed integers are mapped to unsigned integers using "zig-zag"
//   encoding: Positive values x are written as 2*x + 0, negative values
//   are written as 2*(^x) + 1; that is, negative numbers are complemented
//   and whether to complement is encoded in bit 0.
//
// Design note:
// At most 10 bytes are needed for 64-bit values. The encoding could
// be more dense: a full 64-bit value needs an extra byte just to hold bit 63.
// Instead, the msb of the previous byte could be used to hold bit 63 since we
// know there can't be more than 64 bits. This is a trivial improvement and
// would reduce the maximum encoding length to 9 bytes. However, it breaks the
// invariant that the msb is always the "continuation bit" and thus makes the
// format incompatible with a varint encoding for larger numbers (say 128-bit).

import (
	"errors"
	"io"
)

// MaxVarintLenN is the maximum length of a varint-encoded N-bit integer.
const (
	MaxVarintLen16 = 3
	MaxVarintLen32 = 5
	MaxVarintLen64 = 10
)

// PutUvarint encodes a uint64 into buf and returns the number of bytes written.
// If the buffer is too small, PutUvarint will panic.
func PutUvarint(buf []byte, x uint64) int {
	i := 0
	for x >= 0x80 {
		buf[i] = byte(x) | 0x80
		x >>= 7
		i++
	}
	buf[i] = byte(x)
	return i + 1
}

// Uvarint decodes a uint64 from buf and returns that value and the
// number of bytes read (> 0). If an error occurred, the value is 0
// and the number of bytes n is <= 0 meaning:
//
//	n == 0: buf too small
//	n  < 0: value larger than 64 bits (overflow)
//              and -n is the number of bytes read
//
func Uvarint(buf []byte) (uint64, int) {
	var x uint64
	var s uint
	for i, b := range buf {
		if b < 0x80 {
			if i > 9 || i == 9 && b > 1 {
				return 0, -(i + 1) // overflow
			}
			return x | uint64(b)<<s, i + 1
		}
		x |= uint64(b&0x7f) << s
		s += 7
	}
	return 0, 0
}

// PutVarint encodes an int64 into buf and returns the number of bytes written.
// If the buffer is too small, PutVarint will panic.
func PutVarint(buf []byte, x int64) int {
	ux := uint64(x) << 1
	if x < 0 {
		ux = ^ux
	}
	return PutUvarint(buf, ux)
}

// Varint decodes an int64 from buf and returns that value and the
// number of bytes read (> 0). If an error occurred, the value is 0
// and the number of bytes n is <= 0 with the following meaning:
//
//	n == 0: buf too small
//	n  < 0: value larger than 64 bits (overflow)
//              and -n is the number of bytes read
//
func Varint(buf []byte) (int64, int) {
	ux, n := Uvarint(buf) // ok to continue in presence of error
	x := int64(ux >> 1)
	if ux&1 != 0 {
		x = ^x
	}
	return x, n
}

var overflow = errors.New("binary: varint overflows a 64-bit integer")

// ReadUvarint reads an encoded unsigned integer from r and returns it as a uint64.
func ReadUvarint(r io.ByteReader) (uint64, error) {
	var x uint64
	var s uint
	for i := 0; ; i++ {
		b, err := r.ReadByte()
		if err != nil {
			return x, err
		}
		if b < 0x80 {
			if i > 9 || i == 9 && b > 1 {
				return x, overflow
			}
			return x | uint64(b)<<s, nil
		}
		x |= uint64(b&0x7f) << s
		s += 7
	}
	panic("unreachable")
}

// ReadVarint reads an encoded unsigned integer from r and returns it as an int64.
func ReadVarint(r io.ByteReader) (int64, error) {
	ux, err := ReadUvarint(r) // ok to continue in presence of error
	x := int64(ux >> 1)
	if ux&1 != 0 {
		x = ^x
	}
	return x, err
}
<|endoftext|>