<commit_before>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package main

import (
	"fmt"
	"launchpad.net/gnuflag"
	"launchpad.net/juju-core/charm"
	"launchpad.net/juju-core/cmd"
	"launchpad.net/juju-core/log"
	"launchpad.net/juju-core/state"
	"launchpad.net/juju-core/state/api"
	"launchpad.net/juju-core/state/api/machineagent"
	"launchpad.net/juju-core/state/api/params"
	"launchpad.net/juju-core/state/apiserver"
	"launchpad.net/juju-core/worker"
	"launchpad.net/juju-core/worker/firewaller"
	"launchpad.net/juju-core/worker/machiner"
	"launchpad.net/juju-core/worker/provisioner"
	"launchpad.net/tomb"
	"path/filepath"
	"time"
)

var retryDelay = 3 * time.Second

// MachineAgent is a cmd.Command responsible for running a machine agent.
type MachineAgent struct {
	cmd.CommandBase
	tomb      tomb.Tomb
	Conf      AgentConf
	MachineId string
	runner    *worker.Runner
}

// Info returns usage information for the command.
func (a *MachineAgent) Info() *cmd.Info {
	return &cmd.Info{
		Name:    "machine",
		Purpose: "run a juju machine agent",
	}
}

func (a *MachineAgent) SetFlags(f *gnuflag.FlagSet) {
	a.Conf.addFlags(f)
	f.StringVar(&a.MachineId, "machine-id", "", "id of the machine to run")
}

// Init initializes the command for running.
func (a *MachineAgent) Init(args []string) error {
	if !state.IsMachineId(a.MachineId) {
		return fmt.Errorf("--machine-id option must be set, and expects a non-negative integer")
	}
	if err := a.Conf.checkArgs(args); err != nil {
		return err
	}
	a.runner = worker.NewRunner(isFatal, moreImportant)
	return nil
}

// Wait waits for the machine agent to finish.
func (a *MachineAgent) Wait() error {
	return a.tomb.Wait()
}

// Stop stops the machine agent.
func (a *MachineAgent) Stop() error {
	a.runner.Kill()
	return a.tomb.Wait()
}

// Run runs a machine agent.
func (a *MachineAgent) Run(_ *cmd.Context) error {
	defer a.tomb.Done()
	log.Infof("machine agent %v start", a.Tag())
	if err := a.Conf.read(a.Tag()); err != nil {
		return err
	}
	charm.CacheDir = filepath.Join(a.Conf.DataDir, "charmcache")

	// ensureStateWorker ensures that there is a worker that
	// connects to the state that runs within itself all the workers
	// that need a state connection Unless we're bootstrapping, we
	// need to connect to the API server to find out if we need to
	// call this, so we make the APIWorker call it when necessary if
	// the machine requires it.  Note that ensureStateWorker can be
	// called many times - StartWorker does nothing if there is
	// already a worker started with the given name.
	ensureStateWorker := func() {
		a.runner.StartWorker("state", func() (worker.Worker, error) {
			// TODO(rog) go1.1: use method expression
			return a.StateWorker()
		})
	}
	if a.MachineId == "0" {
		// If we're bootstrapping, we don't have an API
		// server to connect to, so start the state worker regardless.

		// TODO(rog) When we have HA, we only want to do this
		// when we really are bootstrapping - once other
		// instances of the API server have been started, we
		// should follow the normal course of things and ignore
		// the fact that this was once the bootstrap machine.
		ensureStateWorker()
	}
	a.runner.StartWorker("api", func() (worker.Worker, error) {
		// TODO(rog) go1.1: use method expression
		return a.APIWorker(ensureStateWorker)
	})
	err := agentDone(a.runner.Wait())
	a.tomb.Kill(err)
	return err
}

func allFatal(error) bool {
	return true
}

var stateJobs = map[params.MachineJob]bool{
	params.JobHostUnits:     true,
	params.JobManageEnviron: true,
	params.JobManageState:   true,
}

// APIWorker returns a Worker that connects to the API and starts any
// workers that need an API connection.
//
// If a state worker is necessary, APIWorker calls ensureStateWorker.
func (a *MachineAgent) APIWorker(ensureStateWorker func()) (worker.Worker, error) {
	st, entity, err := openAPIState(a.Conf.Conf, a)
	if err != nil {
		return nil, err
	}
	m := entity.(*machineagent.Machine)
	needsStateWorker := false
	for _, job := range m.Jobs() {
		needsStateWorker = needsStateWorker || stateJobs[job]
	}
	if needsStateWorker {
		ensureStateWorker()
	}
	runner := worker.NewRunner(allFatal, moreImportant)
	// No agents currently connect to the API, so just
	// return the runner running nothing.
	return newCloseWorker(runner, st), nil // Note: a worker.Runner is itself a worker.Worker.
}

// StateJobs returns a worker running all the workers that require
// a *state.State connection.
func (a *MachineAgent) StateWorker() (worker.Worker, error) {
	st, entity, err := openState(a.Conf.Conf, a)
	if err != nil {
		return nil, err
	}
	m := entity.(*state.Machine)
	// TODO(rog) use more discriminating test for errors
	// rather than taking everything down indiscriminately.
	dataDir := a.Conf.DataDir
	runner := worker.NewRunner(allFatal, moreImportant)
	runner.StartWorker("upgrader", func() (worker.Worker, error) {
		// TODO(rog) use id instead of *Machine (or introduce Clone method)
		return NewUpgrader(st, m, dataDir), nil
	})
	runner.StartWorker("machiner", func() (worker.Worker, error) {
		return machiner.NewMachiner(st, m.Id()), nil
	})
	// At this stage, since we don't embed lxc containers, just start an lxc
	// provisioner task for non-lxc containers.
	if m.ContainerType() != state.LXC {
		workerName := fmt.Sprintf("%s-provisioner", provisioner.LXC)
		runner.StartWorker(workerName, func() (worker.Worker, error) {
			return provisioner.NewProvisioner(provisioner.LXC, st, a.MachineId, dataDir), nil
		})
	}
	for _, job := range m.Jobs() {
		switch job {
		case state.JobHostUnits:
			runner.StartWorker("deployer", func() (worker.Worker, error) {
				return newDeployer(st, m.WatchPrincipalUnits(), dataDir), nil
			})
		case state.JobManageEnviron:
			runner.StartWorker("environ-provisioner", func() (worker.Worker, error) {
				return provisioner.NewProvisioner(provisioner.ENVIRON, st, a.MachineId, dataDir), nil
			})
			runner.StartWorker("firewaller", func() (worker.Worker, error) {
				return firewaller.NewFirewaller(st), nil
			})
		case state.JobManageState:
			runner.StartWorker("apiserver", func() (worker.Worker, error) {
				// If the configuration does not have the required information,
				// it is currently not a recoverable error, so we kill the whole
				// agent, potentially enabling human intervention to fix
				// the agent's configuration file. In the future, we may retrieve
				// the state server certificate and key from the state, and
				// this should then change.
				if len(a.Conf.StateServerCert) == 0 || len(a.Conf.StateServerKey) == 0 {
					return nil, &fatalError{"configuration does not have state server cert/key"}
				}
				return apiserver.NewServer(st, fmt.Sprintf(":%d", a.Conf.APIPort), a.Conf.StateServerCert, a.Conf.StateServerKey)
			})
		default:
			log.Warningf("ignoring unknown job %q", job)
		}
	}
	return newCloseWorker(runner, st), nil
}

func (a *MachineAgent) Entity(st *state.State) (AgentState, error) {
	m, err := st.Machine(a.MachineId)
	if err != nil {
		return nil, err
	}
	// Check the machine nonce as provisioned matches the agent.Conf value.
	if !m.CheckProvisioned(a.Conf.MachineNonce) {
		// The agent is running on a different machine to the one it
		// should be according to state. It must stop immediately.
		log.Errorf("running machine %v agent on inappropriate instance", m)
		return nil, worker.ErrTerminateAgent
	}
	return m, nil
}

func (a *MachineAgent) APIEntity(st *api.State) (AgentAPIState, error) {
	m, err := st.MachineAgent().Machine(a.MachineId)
	if err != nil {
		return nil, err
	}
	// TODO(rog) move the CheckProvisioned test into
	// this method when it's implemented in the API
	return m, nil
}

func (a *MachineAgent) Tag() string {
	return state.MachineTag(a.MachineId)
}
<commit_msg>Extend the comment.<commit_after>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package main

import (
	"fmt"
	"launchpad.net/gnuflag"
	"launchpad.net/juju-core/charm"
	"launchpad.net/juju-core/cmd"
	"launchpad.net/juju-core/log"
	"launchpad.net/juju-core/state"
	"launchpad.net/juju-core/state/api"
	"launchpad.net/juju-core/state/api/machineagent"
	"launchpad.net/juju-core/state/api/params"
	"launchpad.net/juju-core/state/apiserver"
	"launchpad.net/juju-core/worker"
	"launchpad.net/juju-core/worker/firewaller"
	"launchpad.net/juju-core/worker/machiner"
	"launchpad.net/juju-core/worker/provisioner"
	"launchpad.net/tomb"
	"path/filepath"
	"time"
)

var retryDelay = 3 * time.Second

// MachineAgent is a cmd.Command responsible for running a machine agent.
type MachineAgent struct {
	cmd.CommandBase
	tomb      tomb.Tomb
	Conf      AgentConf
	MachineId string
	runner    *worker.Runner
}

// Info returns usage information for the command.
func (a *MachineAgent) Info() *cmd.Info {
	return &cmd.Info{
		Name:    "machine",
		Purpose: "run a juju machine agent",
	}
}

func (a *MachineAgent) SetFlags(f *gnuflag.FlagSet) {
	a.Conf.addFlags(f)
	f.StringVar(&a.MachineId, "machine-id", "", "id of the machine to run")
}

// Init initializes the command for running.
func (a *MachineAgent) Init(args []string) error {
	if !state.IsMachineId(a.MachineId) {
		return fmt.Errorf("--machine-id option must be set, and expects a non-negative integer")
	}
	if err := a.Conf.checkArgs(args); err != nil {
		return err
	}
	a.runner = worker.NewRunner(isFatal, moreImportant)
	return nil
}

// Wait waits for the machine agent to finish.
func (a *MachineAgent) Wait() error {
	return a.tomb.Wait()
}

// Stop stops the machine agent.
func (a *MachineAgent) Stop() error {
	a.runner.Kill()
	return a.tomb.Wait()
}

// Run runs a machine agent.
func (a *MachineAgent) Run(_ *cmd.Context) error {
	defer a.tomb.Done()
	log.Infof("machine agent %v start", a.Tag())
	if err := a.Conf.read(a.Tag()); err != nil {
		return err
	}
	charm.CacheDir = filepath.Join(a.Conf.DataDir, "charmcache")

	// ensureStateWorker ensures that there is a worker that
	// connects to the state that runs within itself all the workers
	// that need a state connection Unless we're bootstrapping, we
	// need to connect to the API server to find out if we need to
	// call this, so we make the APIWorker call it when necessary if
	// the machine requires it.  Note that ensureStateWorker can be
	// called many times - StartWorker does nothing if there is
	// already a worker started with the given name.
	ensureStateWorker := func() {
		a.runner.StartWorker("state", func() (worker.Worker, error) {
			// TODO(rog) go1.1: use method expression
			return a.StateWorker()
		})
	}
	if a.MachineId == "0" {
		// If we're bootstrapping, we don't have an API
		// server to connect to, so start the state worker regardless.

		// TODO(rog) When we have HA, we only want to do this
		// when we really are bootstrapping - once other
		// instances of the API server have been started, we
		// should follow the normal course of things and ignore
		// the fact that this was once the bootstrap machine.
		ensureStateWorker()
	}
	a.runner.StartWorker("api", func() (worker.Worker, error) {
		// TODO(rog) go1.1: use method expression
		return a.APIWorker(ensureStateWorker)
	})
	err := agentDone(a.runner.Wait())
	a.tomb.Kill(err)
	return err
}

func allFatal(error) bool {
	return true
}

var stateJobs = map[params.MachineJob]bool{
	params.JobHostUnits:     true,
	params.JobManageEnviron: true,
	params.JobManageState:   true,
}

// APIWorker returns a Worker that connects to the API and starts any
// workers that need an API connection.
//
// If a state worker is necessary, APIWorker calls ensureStateWorker.
func (a *MachineAgent) APIWorker(ensureStateWorker func()) (worker.Worker, error) {
	st, entity, err := openAPIState(a.Conf.Conf, a)
	if err != nil {
		return nil, err
	}
	m := entity.(*machineagent.Machine)
	needsStateWorker := false
	for _, job := range m.Jobs() {
		needsStateWorker = needsStateWorker || stateJobs[job]
	}
	if needsStateWorker {
		ensureStateWorker()
	}
	runner := worker.NewRunner(allFatal, moreImportant)
	// No agents currently connect to the API, so just
	// return the runner running nothing.
	return newCloseWorker(runner, st), nil // Note: a worker.Runner is itself a worker.Worker.
}

// StateJobs returns a worker running all the workers that require
// a *state.State connection.
func (a *MachineAgent) StateWorker() (worker.Worker, error) {
	st, entity, err := openState(a.Conf.Conf, a)
	if err != nil {
		return nil, err
	}
	m := entity.(*state.Machine)
	// TODO(rog) use more discriminating test for errors
	// rather than taking everything down indiscriminately.
	dataDir := a.Conf.DataDir
	runner := worker.NewRunner(allFatal, moreImportant)
	runner.StartWorker("upgrader", func() (worker.Worker, error) {
		// TODO(rog) use id instead of *Machine (or introduce Clone method)
		return NewUpgrader(st, m, dataDir), nil
	})
	runner.StartWorker("machiner", func() (worker.Worker, error) {
		return machiner.NewMachiner(st, m.Id()), nil
	})
	// At this stage, since we don't embed lxc containers, just start an lxc
	// provisioner task for non-lxc containers.  Since we have only LXC
	// containers and normal machines, this effectively means that we only
	// have an LXC provisioner when we have a normally provisioned machine
	// (through the environ-provisioner).  With the upcoming advent of KVM
	// containers, it is likely that we will want an LXC provisioner on a KVM
	// machine, and once we get nested LXC containers, we can remove this
	// check.
	if m.ContainerType() != state.LXC {
		workerName := fmt.Sprintf("%s-provisioner", provisioner.LXC)
		runner.StartWorker(workerName, func() (worker.Worker, error) {
			return provisioner.NewProvisioner(provisioner.LXC, st, a.MachineId, dataDir), nil
		})
	}
	for _, job := range m.Jobs() {
		switch job {
		case state.JobHostUnits:
			runner.StartWorker("deployer", func() (worker.Worker, error) {
				return newDeployer(st, m.WatchPrincipalUnits(), dataDir), nil
			})
		case state.JobManageEnviron:
			runner.StartWorker("environ-provisioner", func() (worker.Worker, error) {
				return provisioner.NewProvisioner(provisioner.ENVIRON, st, a.MachineId, dataDir), nil
			})
			runner.StartWorker("firewaller", func() (worker.Worker, error) {
				return firewaller.NewFirewaller(st), nil
			})
		case state.JobManageState:
			runner.StartWorker("apiserver", func() (worker.Worker, error) {
				// If the configuration does not have the required information,
				// it is currently not a recoverable error, so we kill the whole
				// agent, potentially enabling human intervention to fix
				// the agent's configuration file. In the future, we may retrieve
				// the state server certificate and key from the state, and
				// this should then change.
				if len(a.Conf.StateServerCert) == 0 || len(a.Conf.StateServerKey) == 0 {
					return nil, &fatalError{"configuration does not have state server cert/key"}
				}
				return apiserver.NewServer(st, fmt.Sprintf(":%d", a.Conf.APIPort), a.Conf.StateServerCert, a.Conf.StateServerKey)
			})
		default:
			log.Warningf("ignoring unknown job %q", job)
		}
	}
	return newCloseWorker(runner, st), nil
}

func (a *MachineAgent) Entity(st *state.State) (AgentState, error) {
	m, err := st.Machine(a.MachineId)
	if err != nil {
		return nil, err
	}
	// Check the machine nonce as provisioned matches the agent.Conf value.
	if !m.CheckProvisioned(a.Conf.MachineNonce) {
		// The agent is running on a different machine to the one it
		// should be according to state. It must stop immediately.
		log.Errorf("running machine %v agent on inappropriate instance", m)
		return nil, worker.ErrTerminateAgent
	}
	return m, nil
}

func (a *MachineAgent) APIEntity(st *api.State) (AgentAPIState, error) {
	m, err := st.MachineAgent().Machine(a.MachineId)
	if err != nil {
		return nil, err
	}
	// TODO(rog) move the CheckProvisioned test into
	// this method when it's implemented in the API
	return m, nil
}

func (a *MachineAgent) Tag() string {
	return state.MachineTag(a.MachineId)
}
<|endoftext|>


<commit_before>package upload

import (
	"crypto/tls"
	"fmt"
	"os"

	"github.com/codegangsta/cli"
	"github.com/materials-commons/mcstore/pkg/app"
	"github.com/materials-commons/mcstore/pkg/files"
	"github.com/materials-commons/mcstore/server/mcstored/service/rest/upload"
	"github.com/parnurzeal/gorequest"
)

// Command contains the arguments and functions for the cli upload command.
var Command = cli.Command{
	Name:    "upload",
	Aliases: []string{"up", "u"},
	Usage:   "Upload data to MaterialsCommons",
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "project, p, proj",
			Usage: "Project id to upload to",
		},
		cli.IntFlag{
			Name:  "parallel, n",
			Value: 5,
			Usage: "Number of simultaneous uploads to perform, defaults to 5",
		},
	},
	Action: uploadCLI,
}

const oneMeg = 1024 * 1024
const twoMeg = oneMeg * 2
const largeFileSize = oneMeg * 25

var project string

//var pbPool = &pb.Pool{}

// uploadCLI implements the cli command upload.
func uploadCLI(c *cli.Context) {
	fmt.Println("upload: ", c.Args())
	if len(c.Args()) != 1 {
		fmt.Println("You must give the directory to walk")
		os.Exit(1)
	}
	dir := c.Args()[0]
	project = c.String("project")
	numThreads := c.Int("parallel")

	_, errc := files.PWalk(dir, numThreads, processFiles)
	if err := <-errc; err != nil {
		fmt.Println("Got error: ", err)
	}
}

func processFiles(done <-chan struct{}, entries <-chan files.TreeEntry, result chan<- string) {
	u := &uploader{
		client: gorequest.New().TLSClientConfig(&tls.Config{InsecureSkipVerify: true}),
	}
	for entry := range entries {
		select {
		case result <- u.sendFile(entry):
		case <-done:
			// Received done, so stop processing requests.
			return
		}
	}
}

type uploader struct {
	client *gorequest.SuperAgent
}

func (u *uploader) sendFile(fileEntry files.TreeEntry) string {
	u.createUploadRequest()
	// buf := make([]byte, twoMeg)
	// f, err := os.Open(fileEntry.Path)
	// if err != nil {
	// 	return ""
	// }
	// fileHash := ""
	// // For small files just transfer the bytes. For
	// // large files compute the file hash.
	// if fileEntry.Finfo.Size() > largeFileSize {
	// 	fileHash, _ = file.HashStr(md5.New(), fileEntry.Path)
	// }
	// for {
	// 	read, err := f.Read(buf)
	// 	sendFlowChunk(buf)
	// }
	return fileEntry.Path
}

func (u *uploader) createUploadRequest() {
	req := upload.CreateRequest{
		ProjectID:   "9ead5bbf-f7eb-4010-bc1f-e4a063f56226",
		DirectoryID: "c54a77d6-cd6d-4cd1-8f19-44facc761da6",
		FileName:    "abc.txt",
		FileSize:    10,
		FileMTime:   "Thu, 30 Apr 2015 13:10:04 EST",
	}

	var resp upload.CreateResponse
	r, body, errs := u.client.Post(app.MCApi.APIUrl("/upload")).Send(&req).End()
	if err := app.MCApi.APIError(r, errs); err != nil {
		return
	}
	app.MCApi.ToJSON(body, &resp)
	fmt.Printf("%#v\n", resp)
}

func sendFlowChunk(buf []byte) {

}
<commit_msg>Add checking numThreads to ensure its in a valid range.<commit_after>package upload

import (
	"crypto/tls"
	"fmt"
	"os"

	"github.com/codegangsta/cli"
	"github.com/materials-commons/mcstore/pkg/app"
	"github.com/materials-commons/mcstore/pkg/files"
	"github.com/materials-commons/mcstore/server/mcstored/service/rest/upload"
	"github.com/parnurzeal/gorequest"
)

// Command contains the arguments and functions for the cli upload command.
var Command = cli.Command{
	Name:    "upload",
	Aliases: []string{"up", "u"},
	Usage:   "Upload data to MaterialsCommons",
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "project, p, proj",
			Usage: "Project id to upload to",
		},
		cli.IntFlag{
			Name:  "parallel, n",
			Value: 3,
			Usage: "Number of simultaneous uploads to perform, defaults to 3",
		},
	},
	Action: uploadCLI,
}

const oneMeg = 1024 * 1024
const twoMeg = oneMeg * 2
const largeFileSize = oneMeg * 25
const maxSimultaneous = 5

var project string

//var pbPool = &pb.Pool{}

// uploadCLI implements the cli command upload.
func uploadCLI(c *cli.Context) {
	fmt.Println("upload: ", c.Args())
	if len(c.Args()) != 1 {
		fmt.Println("You must give the directory to walk")
		os.Exit(1)
	}
	dir := c.Args()[0]
	project = c.String("project")
	numThreads := getNumThreads(c)

	_, errc := files.PWalk(dir, numThreads, processFiles)
	if err := <-errc; err != nil {
		fmt.Println("Got error: ", err)
	}
}

// getNumThreads ensures that the number of parallel downloads is valid.
func getNumThreads(c *cli.Context) int {
	numThreads := c.Int("parallel")

	if numThreads < 1 {
		fmt.Println("Simultaneous downloads must be positive: ", numThreads)
		os.Exit(1)
	} else if numThreads > maxSimultaneous {
		fmt.Printf("You may not set simultaneous downloads greater than %d: %d\n", maxSimultaneous, numThreads)
		os.Exit(1)
	}

	return numThreads
}

// processFiles is the callback passed into PWalk. It processes each file, determines
// if it should be uploaded, and if so uploads the file. There can be a maxSimultaneous
// processFiles routines running.
func processFiles(done <-chan struct{}, entries <-chan files.TreeEntry, result chan<- string) {
	u := &uploader{
		client: gorequest.New().TLSClientConfig(&tls.Config{InsecureSkipVerify: true}),
	}
	for entry := range entries {
		select {
		case result <- u.sendFile(entry):
		case <-done:
			// Received done, so stop processing requests.
			return
		}
	}
}

type uploader struct {
	client *gorequest.SuperAgent
}

func (u *uploader) sendFile(fileEntry files.TreeEntry) string {
	u.createUploadRequest()
	// buf := make([]byte, twoMeg)
	// f, err := os.Open(fileEntry.Path)
	// if err != nil {
	// 	return ""
	// }
	// fileHash := ""
	// // For small files just transfer the bytes. For
	// // large files compute the file hash.
	// if fileEntry.Finfo.Size() > largeFileSize {
	// 	fileHash, _ = file.HashStr(md5.New(), fileEntry.Path)
	// }
	// for {
	// 	read, err := f.Read(buf)
	// 	sendFlowChunk(buf)
	// }
	return fileEntry.Path
}

func (u *uploader) createUploadRequest() {
	req := upload.CreateRequest{
		ProjectID:   "9ead5bbf-f7eb-4010-bc1f-e4a063f56226",
		DirectoryID: "c54a77d6-cd6d-4cd1-8f19-44facc761da6",
		FileName:    "abc.txt",
		FileSize:    10,
		FileMTime:   "Thu, 30 Apr 2015 13:10:04 EST",
	}

	var resp upload.CreateResponse
	r, body, errs := u.client.Post(app.MCApi.APIUrl("/upload")).Send(&req).End()
	if err := app.MCApi.APIError(r, errs); err != nil {
		return
	}
	app.MCApi.ToJSON(body, &resp)
	fmt.Printf("%#v\n", resp)
}

func sendFlowChunk(buf []byte) {

}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main // import "k8s.io/helm/cmd/rudder"

import (
	"bytes"
	"fmt"
	"net"

	"golang.org/x/net/context"
	"google.golang.org/grpc"
	"google.golang.org/grpc/grpclog"

	"k8s.io/helm/pkg/kube"
	"k8s.io/helm/pkg/proto/hapi/release"
	"k8s.io/helm/pkg/rudder"
)

var kubeClient = kube.New(nil)

func main() {
	lis, err := net.Listen("tcp", fmt.Sprintf(":%d", rudder.GrpcAddr))
	if err != nil {
		grpclog.Fatalf("failed to listen: %v", err)
	}
	grpcServer := grpc.NewServer()
	release.RegisterReleaseModuleServiceServer(grpcServer, &ReleaseModuleServiceServer{})

	grpclog.Print("Server starting")
	grpcServer.Serve(lis)
	grpclog.Print("Server started")
}

// ReleaseModuleServiceServer provides implementation for release.ReleaseModuleServiceServer
type ReleaseModuleServiceServer struct{}

// Version is not yet implemented
func (r *ReleaseModuleServiceServer) Version(ctx context.Context, in *release.VersionReleaseRequest) (*release.VersionReleaseResponse, error) {
	grpclog.Print("version")
	return &release.VersionReleaseResponse{}, nil
}

// InstallRelease creates a release using kubeClient.Create
func (r *ReleaseModuleServiceServer) InstallRelease(ctx context.Context, in *release.InstallReleaseRequest) (*release.InstallReleaseResponse, error) {
	grpclog.Print("install")
	b := bytes.NewBufferString(in.Release.Manifest)
	err := kubeClient.Create(in.Release.Namespace, b, 500, false)
	if err != nil {
		grpclog.Printf("error when creating release: %s", err)
	}
	return &release.InstallReleaseResponse{}, err
}

// DeleteRelease is not implemented
func (r *ReleaseModuleServiceServer) DeleteRelease(ctx context.Context, in *release.DeleteReleaseRequest) (*release.DeleteReleaseResponse, error) {
	grpclog.Print("delete")
	return nil, nil
}

// RollbackRelease is not implemented
func (r *ReleaseModuleServiceServer) RollbackRelease(ctx context.Context, in *release.RollbackReleaseRequest) (*release.RollbackReleaseResponse, error) {
	grpclog.Print("rollback")
	return nil, nil
}

// UpgradeRelease is not implemented
func (r *ReleaseModuleServiceServer) UpgradeRelease(ctx context.Context, in *release.UpgradeReleaseRequest) (*release.UpgradeReleaseResponse, error) {
	grpclog.Print("upgrade")
	return nil, nil
}
<commit_msg>Use helm version as rudder version<commit_after>/*
Copyright 2017 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main // import "k8s.io/helm/cmd/rudder"

import (
	"bytes"
	"fmt"
	"net"

	"golang.org/x/net/context"
	"google.golang.org/grpc"
	"google.golang.org/grpc/grpclog"

	"k8s.io/helm/pkg/kube"
	"k8s.io/helm/pkg/proto/hapi/release"
	"k8s.io/helm/pkg/rudder"
	"k8s.io/helm/pkg/version"
)

var kubeClient = kube.New(nil)

func main() {
	lis, err := net.Listen("tcp", fmt.Sprintf(":%d", rudder.GrpcAddr))
	if err != nil {
		grpclog.Fatalf("failed to listen: %v", err)
	}
	grpcServer := grpc.NewServer()
	release.RegisterReleaseModuleServiceServer(grpcServer, &ReleaseModuleServiceServer{})

	grpclog.Print("Server starting")
	grpcServer.Serve(lis)
	grpclog.Print("Server started")
}

// ReleaseModuleServiceServer provides implementation for release.ReleaseModuleServiceServer
type ReleaseModuleServiceServer struct{}

// Version is not yet implemented
func (r *ReleaseModuleServiceServer) Version(ctx context.Context, in *release.VersionReleaseRequest) (*release.VersionReleaseResponse, error) {
	grpclog.Print("version")
	return &release.VersionReleaseResponse{
		Name:    "helm-rudder-native",
		Version: version.Version,
	}, nil
}

// InstallRelease creates a release using kubeClient.Create
func (r *ReleaseModuleServiceServer) InstallRelease(ctx context.Context, in *release.InstallReleaseRequest) (*release.InstallReleaseResponse, error) {
	grpclog.Print("install")
	b := bytes.NewBufferString(in.Release.Manifest)
	err := kubeClient.Create(in.Release.Namespace, b, 500, false)
	if err != nil {
		grpclog.Printf("error when creating release: %s", err)
	}
	return &release.InstallReleaseResponse{}, err
}

// DeleteRelease is not implemented
func (r *ReleaseModuleServiceServer) DeleteRelease(ctx context.Context, in *release.DeleteReleaseRequest) (*release.DeleteReleaseResponse, error) {
	grpclog.Print("delete")
	return nil, nil
}

// RollbackRelease is not implemented
func (r *ReleaseModuleServiceServer) RollbackRelease(ctx context.Context, in *release.RollbackReleaseRequest) (*release.RollbackReleaseResponse, error) {
	grpclog.Print("rollback")
	return nil, nil
}

// UpgradeRelease is not implemented
func (r *ReleaseModuleServiceServer) UpgradeRelease(ctx context.Context, in *release.UpgradeReleaseRequest) (*release.UpgradeReleaseResponse, error) {
	grpclog.Print("upgrade")
	return nil, nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Richard Lehane. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"io"
	"path/filepath"
	"strings"
	"time"

	"archive/tar"
	"archive/zip"
	"compress/gzip"

	"github.com/richardlehane/characterize"
	"github.com/richardlehane/webarchive"

	"github.com/richardlehane/siegfried/pkg/core/siegreader"
)

type decompressor interface {
	next() error // when finished, should return io.EOF
	reader() io.Reader
	path() string
	mime() string
	size() int64
	mod() string
	dirs() []string
}

type zipD struct {
	idx     int
	p       string
	rdr     *zip.Reader
	rc      io.ReadCloser
	written map[string]bool
}

func newZip(ra io.ReaderAt, path string, sz int64) (decompressor, error) {
	zr, err := zip.NewReader(ra, sz)
	return &zipD{idx: -1, p: path, rdr: zr}, err
}

func (z *zipD) close() {
	if z.rc == nil {
		return
	}
	z.rc.Close()
}

func (z *zipD) next() error {
	z.close() // close the previous entry, if any
	// proceed
	z.idx++
	// scan past directories
	for ; z.idx < len(z.rdr.File) && z.rdr.File[z.idx].FileInfo().IsDir(); z.idx++ {
	}
	if z.idx >= len(z.rdr.File) {
		return io.EOF
	}
	var err error
	z.rc, err = z.rdr.File[z.idx].Open()
	return err
}

func (z *zipD) reader() io.Reader {
	return z.rc
}

func (z *zipD) path() string {
	return arcpath(z.p, filepath.FromSlash(characterize.ZipName(z.rdr.File[z.idx].Name)))
}

func (z *zipD) mime() string {
	return ""
}

func (z *zipD) size() int64 {
	return int64(z.rdr.File[z.idx].UncompressedSize64)
}

func (z *zipD) mod() string {
	return z.rdr.File[z.idx].ModTime().Format(time.RFC3339)
}

func (z *zipD) dirs() []string {
	if z.written == nil {
		z.written = make(map[string]bool)
	}
	return dirs(z.p, characterize.ZipName(z.rdr.File[z.idx].Name), z.written)
}

type tarD struct {
	p       string
	hdr     *tar.Header
	rdr     *tar.Reader
	written map[string]bool
}

func newTar(r io.Reader, path string) (decompressor, error) {
	return &tarD{p: path, rdr: tar.NewReader(r)}, nil
}

func (t *tarD) next() error {
	var err error
	// scan past directories
	for t.hdr, err = t.rdr.Next(); err == nil && t.hdr.FileInfo().IsDir(); t.hdr, err = t.rdr.Next() {
	}
	return err
}

func (t *tarD) reader() io.Reader {
	return t.rdr
}

func (t *tarD) path() string {
	return arcpath(t.p, filepath.FromSlash(t.hdr.Name))
}

func (t *tarD) mime() string {
	return ""
}

func (t *tarD) size() int64 {
	return t.hdr.Size
}

func (t *tarD) mod() string {
	return t.hdr.ModTime.Format(time.RFC3339)
}

func (t *tarD) dirs() []string {
	if t.written == nil {
		t.written = make(map[string]bool)
	}
	return dirs(t.p, t.hdr.Name, t.written)
}

type gzipD struct {
	sz   int64
	p    string
	read bool
	rdr  *gzip.Reader
}

func newGzip(b *siegreader.Buffer, path string) (decompressor, error) {
	_ = b.SizeNow()              // in case of a stream, force full read
	buf, err := b.EofSlice(0, 4) // gzip stores uncompressed size in last 4 bytes of the stream
	if err != nil {
		return nil, err
	}
	sz := int64(uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24)
	g, err := gzip.NewReader(siegreader.ReaderFrom(b))
	return &gzipD{sz: sz, p: path, rdr: g}, err
}

func (g *gzipD) next() error {
	if g.read {
		g.rdr.Close()
		return io.EOF
	}
	g.read = true
	return nil
}

func (g *gzipD) reader() io.Reader {
	return g.rdr
}

func (g *gzipD) path() string {
	name := g.rdr.Name
	if len(name) == 0 {
		switch filepath.Ext(g.p) {
		case ".gz", ".z", ".gzip", ".zip":
			name = strings.TrimSuffix(filepath.Base(g.p), filepath.Ext(g.p))
		default:
			name = filepath.Base(g.p)
		}
	}
	return arcpath(g.p, name)
}

func (g *gzipD) mime() string {
	return ""
}

func (g *gzipD) size() int64 {
	return g.sz
}

func (g *gzipD) mod() string {
	return g.rdr.ModTime.Format(time.RFC3339)
}

func (t *gzipD) dirs() []string {
	return nil
}

func trimWebPath(p string) string {
	d, f := filepath.Split(p)
	clean := strings.TrimSuffix(d, string(filepath.Separator))
	_, f1 := filepath.Split(clean)
	if f == strings.TrimSuffix(f1, filepath.Ext(clean)) {
		return clean
	}
	return p
}

var (
	arcReader  *webarchive.ARCReader
	warcReader *webarchive.WARCReader
)

type wa struct {
	p   string
	rec webarchive.Record
	rdr webarchive.Reader
}

func newARC(r io.Reader, path string) (decompressor, error) {
	var err error
	if arcReader == nil {
		arcReader, err = webarchive.NewARCReader(r)
	} else {
		err = arcReader.Reset(r)
	}
	return &wa{p: trimWebPath(path), rdr: arcReader}, err
}

func newWARC(r io.Reader, path string) (decompressor, error) {
	var err error
	if warcReader == nil {
		warcReader, err = webarchive.NewWARCReader(r)
	} else {
		err = warcReader.Reset(r)
	}
	return &wa{p: trimWebPath(path), rdr: warcReader}, err
}

func (w *wa) next() error {
	var err error
	w.rec, err = w.rdr.NextPayload()
	return err
}

func (w *wa) reader() io.Reader {
	return webarchive.DecodePayload(w.rec)
}

func (w *wa) path() string {
	return arcpath(w.p, filepath.Join(w.rec.Date().Format(webarchive.ARCTime), w.rec.URL()))
}

func (w *wa) mime() string {
	return w.rec.MIME()
}

func (w *wa) size() int64 {
	return w.rec.Size()
}

func (w *wa) mod() string {
	return w.rec.Date().Format(time.RFC3339)
}

func (w *wa) dirs() []string {
	return nil
}

func dirs(path, name string, written map[string]bool) []string {
	ds := strings.Split(filepath.ToSlash(name), "/")
	if len(ds) > 1 {
		var ret []string
		for _, p := range ds[:len(ds)-1] {
			path = path + string(filepath.Separator) + p
			if !written[path] {
				ret = append(ret, path)
				written[path] = true
			}
		}
		return ret
	}
	return nil
}

// per https://github.com/richardlehane/siegfried/issues/81
// construct paths for compressed objects acc. to KDE hash notation
func arcpath(base, path string) string {
	if *droido {
		return base + string(filepath.Separator) + path
	}
	return base + "#" + path
}
<commit_msg>webarchive use unix slash<commit_after>// Copyright 2015 Richard Lehane. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"io"
	"path/filepath"
	"strings"
	"time"

	"archive/tar"
	"archive/zip"
	"compress/gzip"

	"github.com/richardlehane/characterize"
	"github.com/richardlehane/webarchive"

	"github.com/richardlehane/siegfried/pkg/core/siegreader"
)

type decompressor interface {
	next() error // when finished, should return io.EOF
	reader() io.Reader
	path() string
	mime() string
	size() int64
	mod() string
	dirs() []string
}

type zipD struct {
	idx     int
	p       string
	rdr     *zip.Reader
	rc      io.ReadCloser
	written map[string]bool
}

func newZip(ra io.ReaderAt, path string, sz int64) (decompressor, error) {
	zr, err := zip.NewReader(ra, sz)
	return &zipD{idx: -1, p: path, rdr: zr}, err
}

func (z *zipD) close() {
	if z.rc == nil {
		return
	}
	z.rc.Close()
}

func (z *zipD) next() error {
	z.close() // close the previous entry, if any
	// proceed
	z.idx++
	// scan past directories
	for ; z.idx < len(z.rdr.File) && z.rdr.File[z.idx].FileInfo().IsDir(); z.idx++ {
	}
	if z.idx >= len(z.rdr.File) {
		return io.EOF
	}
	var err error
	z.rc, err = z.rdr.File[z.idx].Open()
	return err
}

func (z *zipD) reader() io.Reader {
	return z.rc
}

func (z *zipD) path() string {
	return arcpath(z.p, filepath.FromSlash(characterize.ZipName(z.rdr.File[z.idx].Name)))
}

func (z *zipD) mime() string {
	return ""
}

func (z *zipD) size() int64 {
	return int64(z.rdr.File[z.idx].UncompressedSize64)
}

func (z *zipD) mod() string {
	return z.rdr.File[z.idx].ModTime().Format(time.RFC3339)
}

func (z *zipD) dirs() []string {
	if z.written == nil {
		z.written = make(map[string]bool)
	}
	return dirs(z.p, characterize.ZipName(z.rdr.File[z.idx].Name), z.written)
}

type tarD struct {
	p       string
	hdr     *tar.Header
	rdr     *tar.Reader
	written map[string]bool
}

func newTar(r io.Reader, path string) (decompressor, error) {
	return &tarD{p: path, rdr: tar.NewReader(r)}, nil
}

func (t *tarD) next() error {
	var err error
	// scan past directories
	for t.hdr, err = t.rdr.Next(); err == nil && t.hdr.FileInfo().IsDir(); t.hdr, err = t.rdr.Next() {
	}
	return err
}

func (t *tarD) reader() io.Reader {
	return t.rdr
}

func (t *tarD) path() string {
	return arcpath(t.p, filepath.FromSlash(t.hdr.Name))
}

func (t *tarD) mime() string {
	return ""
}

func (t *tarD) size() int64 {
	return t.hdr.Size
}

func (t *tarD) mod() string {
	return t.hdr.ModTime.Format(time.RFC3339)
}

func (t *tarD) dirs() []string {
	if t.written == nil {
		t.written = make(map[string]bool)
	}
	return dirs(t.p, t.hdr.Name, t.written)
}

type gzipD struct {
	sz   int64
	p    string
	read bool
	rdr  *gzip.Reader
}

func newGzip(b *siegreader.Buffer, path string) (decompressor, error) {
	_ = b.SizeNow()              // in case of a stream, force full read
	buf, err := b.EofSlice(0, 4) // gzip stores uncompressed size in last 4 bytes of the stream
	if err != nil {
		return nil, err
	}
	sz := int64(uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24)
	g, err := gzip.NewReader(siegreader.ReaderFrom(b))
	return &gzipD{sz: sz, p: path, rdr: g}, err
}

func (g *gzipD) next() error {
	if g.read {
		g.rdr.Close()
		return io.EOF
	}
	g.read = true
	return nil
}

func (g *gzipD) reader() io.Reader {
	return g.rdr
}

func (g *gzipD) path() string {
	name := g.rdr.Name
	if len(name) == 0 {
		switch filepath.Ext(g.p) {
		case ".gz", ".z", ".gzip", ".zip":
			name = strings.TrimSuffix(filepath.Base(g.p), filepath.Ext(g.p))
		default:
			name = filepath.Base(g.p)
		}
	}
	return arcpath(g.p, name)
}

func (g *gzipD) mime() string {
	return ""
}

func (g *gzipD) size() int64 {
	return g.sz
}

func (g *gzipD) mod() string {
	return g.rdr.ModTime.Format(time.RFC3339)
}

func (t *gzipD) dirs() []string {
	return nil
}

func trimWebPath(p string) string {
	d, f := filepath.Split(p)
	clean := strings.TrimSuffix(d, string(filepath.Separator))
	_, f1 := filepath.Split(clean)
	if f == strings.TrimSuffix(f1, filepath.Ext(clean)) {
		return clean
	}
	return p
}

var (
	arcReader  *webarchive.ARCReader
	warcReader *webarchive.WARCReader
)

type wa struct {
	p   string
	rec webarchive.Record
	rdr webarchive.Reader
}

func newARC(r io.Reader, path string) (decompressor, error) {
	var err error
	if arcReader == nil {
		arcReader, err = webarchive.NewARCReader(r)
	} else {
		err = arcReader.Reset(r)
	}
	return &wa{p: trimWebPath(path), rdr: arcReader}, err
}

func newWARC(r io.Reader, path string) (decompressor, error) {
	var err error
	if warcReader == nil {
		warcReader, err = webarchive.NewWARCReader(r)
	} else {
		err = warcReader.Reset(r)
	}
	return &wa{p: trimWebPath(path), rdr: warcReader}, err
}

func (w *wa) next() error {
	var err error
	w.rec, err = w.rdr.NextPayload()
	return err
}

func (w *wa) reader() io.Reader {
	return webarchive.DecodePayload(w.rec)
}

func (w *wa) path() string {
	return arcpath(w.p, w.rec.Date().Format(webarchive.ARCTime)+'/'+w.rec.URL())
}

func (w *wa) mime() string {
	return w.rec.MIME()
}

func (w *wa) size() int64 {
	return w.rec.Size()
}

func (w *wa) mod() string {
	return w.rec.Date().Format(time.RFC3339)
}

func (w *wa) dirs() []string {
	return nil
}

func dirs(path, name string, written map[string]bool) []string {
	ds := strings.Split(filepath.ToSlash(name), "/")
	if len(ds) > 1 {
		var ret []string
		for _, p := range ds[:len(ds)-1] {
			path = path + string(filepath.Separator) + p
			if !written[path] {
				ret = append(ret, path)
				written[path] = true
			}
		}
		return ret
	}
	return nil
}

// per https://github.com/richardlehane/siegfried/issues/81
// construct paths for compressed objects acc. to KDE hash notation
func arcpath(base, path string) string {
	if *droido {
		return base + string(filepath.Separator) + path
	}
	return base + "#" + path
}
<|endoftext|>


<commit_before>// Copyright 2016 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.

// Package utils contains internal helper functions for go-ethereum commands.
package utils

import (
	"fmt"
	"runtime"

	"github.com/ethereum/go-ethereum/logger"
	"github.com/ethereum/go-ethereum/logger/glog"
	"github.com/ethereum/go-ethereum/params"
	"github.com/ethereum/go-ethereum/rlp"
)

const (
	VersionMajor = 1          // Major version component of the current release
	VersionMinor = 5          // Minor version component of the current release
	VersionPatch = 3          // Patch version component of the current release
	VersionMeta  = "unstable" // Version metadata to append to the version string
)

// Version holds the textual version string.
var Version = func() string {
	v := fmt.Sprintf("%d.%d.%d", VersionMajor, VersionMinor, VersionPatch)
	if VersionMeta != "" {
		v += "-" + VersionMeta
	}
	return v
}()

// MakeDefaultExtraData returns the default Ethereum block extra data blob.
func MakeDefaultExtraData(clientIdentifier string) []byte {
	var clientInfo = struct {
		Version   uint
		Name      string
		GoVersion string
		Os        string
	}{uint(VersionMajor<<16 | VersionMinor<<8 | VersionPatch), clientIdentifier, runtime.Version(), runtime.GOOS}
	extra, err := rlp.EncodeToBytes(clientInfo)
	if err != nil {
		glog.V(logger.Warn).Infoln("error setting canonical miner information:", err)
	}
	if uint64(len(extra)) > params.MaximumExtraDataSize.Uint64() {
		glog.V(logger.Warn).Infoln("error setting canonical miner information: extra exceeds", params.MaximumExtraDataSize)
		glog.V(logger.Debug).Infof("extra: %x\n", extra)
		return nil
	}
	return extra
}
<commit_msg>cmd/utils: 1.5.3 stable<commit_after>// Copyright 2016 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.

// Package utils contains internal helper functions for go-ethereum commands.
package utils

import (
	"fmt"
	"runtime"

	"github.com/ethereum/go-ethereum/logger"
	"github.com/ethereum/go-ethereum/logger/glog"
	"github.com/ethereum/go-ethereum/params"
	"github.com/ethereum/go-ethereum/rlp"
)

const (
	VersionMajor = 1        // Major version component of the current release
	VersionMinor = 5        // Minor version component of the current release
	VersionPatch = 3        // Patch version component of the current release
	VersionMeta  = "stable" // Version metadata to append to the version string
)

// Version holds the textual version string.
var Version = func() string {
	v := fmt.Sprintf("%d.%d.%d", VersionMajor, VersionMinor, VersionPatch)
	if VersionMeta != "" {
		v += "-" + VersionMeta
	}
	return v
}()

// MakeDefaultExtraData returns the default Ethereum block extra data blob.
func MakeDefaultExtraData(clientIdentifier string) []byte {
	var clientInfo = struct {
		Version   uint
		Name      string
		GoVersion string
		Os        string
	}{uint(VersionMajor<<16 | VersionMinor<<8 | VersionPatch), clientIdentifier, runtime.Version(), runtime.GOOS}
	extra, err := rlp.EncodeToBytes(clientInfo)
	if err != nil {
		glog.V(logger.Warn).Infoln("error setting canonical miner information:", err)
	}
	if uint64(len(extra)) > params.MaximumExtraDataSize.Uint64() {
		glog.V(logger.Warn).Infoln("error setting canonical miner information: extra exceeds", params.MaximumExtraDataSize)
		glog.V(logger.Debug).Infof("extra: %x\n", extra)
		return nil
	}
	return extra
}
<|endoftext|>


<commit_before>package topology

import (
	"fmt"
	"math/rand"
	"pkg/storage"
	"time"
)

func (t *Topology) StartRefreshWritableVolumes() {
	go func() {
		for {
			freshThreshHold := time.Now().Unix() - 3*t.pulse //5 times of sleep interval
			t.CollectDeadNodeAndFullVolumes(freshThreshHold, t.volumeSizeLimit)
			time.Sleep(time.Duration(float32(t.pulse*1e3)*(1+rand.Float32())) * time.Millisecond)
		}
	}()
	go func() {
		for {
			select {
			case v := <-t.chanFullVolumes:
				t.SetVolumeCapacityFull(v)
				fmt.Println("Volume", v, "is full!")
			case dn := <-t.chanRecoveredDataNodes:
				t.RegisterRecoveredDataNode(dn)
				fmt.Println("DataNode", dn, "is back alive!")
			case dn := <-t.chanDeadDataNodes:
				t.UnRegisterDataNode(dn)
				fmt.Println("DataNode", dn, "is dead!")
			}
		}
	}()
}
func (t *Topology) SetVolumeCapacityFull(volumeInfo *storage.VolumeInfo) {
	vl := t.GetVolumeLayout(volumeInfo.RepType)
	vl.SetVolumeCapacityFull(volumeInfo.Id)
	for _, dn := range vl.vid2location[volumeInfo.Id].list {
		dn.UpAdjustActiveVolumeCountDelta(-1)
	}
}
func (t *Topology) UnRegisterDataNode(dn *DataNode) {
	for _, v := range dn.volumes {
		fmt.Println("Removing Volume", v.Id, "from the dead volume server", dn)
		vl := t.GetVolumeLayout(v.RepType)
		vl.SetVolumeUnavailable(dn, v.Id)
	}
	dn.UpAdjustActiveVolumeCountDelta(-dn.GetActiveVolumeCount())
	dn.UpAdjustMaxVolumeCountDelta(-dn.GetMaxVolumeCount())
	dn.Parent().UnlinkChildNode(dn.Id())
}
func (t *Topology) RegisterRecoveredDataNode(dn *DataNode) {
	for _, v := range dn.volumes {
		if uint64(v.Size) < t.volumeSizeLimit {
			vl := t.GetVolumeLayout(v.RepType)
			vl.SetVolumeAvailable(dn, v.Id)
		}
	}
}
<commit_msg>comment change: 3 times of the pulse<commit_after>package topology

import (
	"fmt"
	"math/rand"
	"pkg/storage"
	"time"
)

func (t *Topology) StartRefreshWritableVolumes() {
	go func() {
		for {
			freshThreshHold := time.Now().Unix() - 3*t.pulse //3 times of sleep interval
			t.CollectDeadNodeAndFullVolumes(freshThreshHold, t.volumeSizeLimit)
			time.Sleep(time.Duration(float32(t.pulse*1e3)*(1+rand.Float32())) * time.Millisecond)
		}
	}()
	go func() {
		for {
			select {
			case v := <-t.chanFullVolumes:
				t.SetVolumeCapacityFull(v)
				fmt.Println("Volume", v, "is full!")
			case dn := <-t.chanRecoveredDataNodes:
				t.RegisterRecoveredDataNode(dn)
				fmt.Println("DataNode", dn, "is back alive!")
			case dn := <-t.chanDeadDataNodes:
				t.UnRegisterDataNode(dn)
				fmt.Println("DataNode", dn, "is dead!")
			}
		}
	}()
}
func (t *Topology) SetVolumeCapacityFull(volumeInfo *storage.VolumeInfo) {
	vl := t.GetVolumeLayout(volumeInfo.RepType)
	vl.SetVolumeCapacityFull(volumeInfo.Id)
	for _, dn := range vl.vid2location[volumeInfo.Id].list {
		dn.UpAdjustActiveVolumeCountDelta(-1)
	}
}
func (t *Topology) UnRegisterDataNode(dn *DataNode) {
	for _, v := range dn.volumes {
		fmt.Println("Removing Volume", v.Id, "from the dead volume server", dn)
		vl := t.GetVolumeLayout(v.RepType)
		vl.SetVolumeUnavailable(dn, v.Id)
	}
	dn.UpAdjustActiveVolumeCountDelta(-dn.GetActiveVolumeCount())
	dn.UpAdjustMaxVolumeCountDelta(-dn.GetMaxVolumeCount())
	dn.Parent().UnlinkChildNode(dn.Id())
}
func (t *Topology) RegisterRecoveredDataNode(dn *DataNode) {
	for _, v := range dn.volumes {
		if uint64(v.Size) < t.volumeSizeLimit {
			vl := t.GetVolumeLayout(v.RepType)
			vl.SetVolumeAvailable(dn, v.Id)
		}
	}
}
<|endoftext|>


<commit_before>package benchlist

import (
	"container/list"
	"math/rand"
	"sync"
	"time"

	"github.com/ava-labs/avalanchego/ids"
	"github.com/ava-labs/avalanchego/snow"
	"github.com/ava-labs/avalanchego/snow/validators"
	"github.com/ava-labs/avalanchego/utils/constants"
	"github.com/ava-labs/avalanchego/utils/timer"

	safemath "github.com/ava-labs/avalanchego/utils/math"
)

// If a peer consistently does not respond to queries, it will
// increase latencies on the network whenever that peer is polled.
// If we cannot terminate the poll early, then the poll will wait
// the full timeout before finalizing the poll and making progress.
// This can increase network latencies to an undesirable level.

// Therefore, nodes that consistently fail are "benched" such that
// queries to that node fail immediately to avoid waiting up to
// the full network timeout for a response.

// QueryBenchlist ...
type QueryBenchlist interface {
	// RegisterQuery registers a sent query and returns whether the query is subject to benchlist
	RegisterQuery(validatorID ids.ShortID, requestID uint32, msgType constants.MsgType) bool
	// RegisterResponse registers the response to a query message
	RegisterResponse(validatorID ids.ShortID, requstID uint32)
	// QueryFailed registers that a query did not receive a response within our synchrony bound
	QueryFailed(validatorID ids.ShortID, requestID uint32)
	// IsBenched returns true if messages to [validatorID]
	// should not be sent over the network and should immediately fail.
	IsBenched(validatorID ids.ShortID) bool
}

type queryBenchlist struct {
	vdrs validators.Set
	// Validator ID --> Request ID --> non-empty iff
	// there is an outstanding request to this validator
	// with the corresponding requestID
	pendingQueries map[ids.ShortID]map[uint32]pendingQuery
	// Map of consecutive query failures
	consecutiveFailures map[ids.ShortID]failureStreak

	// Maintain benchlist
	benchlistTimes map[ids.ShortID]time.Time
	benchlistOrder *list.List
	benchlistSet   ids.ShortSet

	threshold              int
	minimumFailingDuration time.Duration
	duration               time.Duration
	maxPortion             float64

	clock timer.Clock

	metrics *metrics
	ctx     *snow.Context

	lock sync.Mutex
}

type pendingQuery struct {
	registered time.Time
	msgType    constants.MsgType
}

type failureStreak struct {
	firstFailure time.Time
	consecutive  int
}

// NewQueryBenchlist ...
func NewQueryBenchlist(
	validators validators.Set,
	ctx *snow.Context,
	threshold int,
	minimumFailingDuration,
	duration time.Duration,
	maxPortion float64,
	summaryEnabled bool,
	namespace string,
) (QueryBenchlist, error) {
	metrics := &metrics{}
	return &queryBenchlist{
		pendingQueries:         make(map[ids.ShortID]map[uint32]pendingQuery),
		consecutiveFailures:    make(map[ids.ShortID]failureStreak),
		benchlistTimes:         make(map[ids.ShortID]time.Time),
		benchlistOrder:         list.New(),
		benchlistSet:           ids.ShortSet{},
		vdrs:                   validators,
		threshold:              threshold,
		minimumFailingDuration: minimumFailingDuration,
		duration:               duration,
		maxPortion:             maxPortion,
		ctx:                    ctx,
		metrics:                metrics,
	}, metrics.Initialize(ctx, namespace, summaryEnabled)
}

// IsBenched returns true if messages to [validatorID]
// should not be sent over the network and should immediately fail.
func (b *queryBenchlist) IsBenched(validatorID ids.ShortID) bool {
	b.lock.Lock()
	isBenched := b.isBenched(validatorID)
	b.lock.Unlock()
	return isBenched
}

// isBenched checks if [validatorID] is currently benched
// and calls cleanup if its benching period has elapsed
func (b *queryBenchlist) isBenched(validatorID ids.ShortID) bool {
	end, ok := b.benchlistTimes[validatorID]
	if !ok {
		return false
	}

	if b.clock.Time().Before(end) {
		return true
	}

	// If a benched item has expired, cleanup the benchlist
	b.cleanup()
	return false
}

// RegisterQuery attempts to register a query from [validatorID] and returns true
// if that request should be made (not subject to benchlisting)
func (b *queryBenchlist) RegisterQuery(validatorID ids.ShortID, requestID uint32, msgType constants.MsgType) bool {
	b.lock.Lock()
	defer b.lock.Unlock()

	if benched := b.isBenched(validatorID); benched {
		return false
	}

	validatorRequests, ok := b.pendingQueries[validatorID]
	if !ok {
		validatorRequests = make(map[uint32]pendingQuery)
		b.pendingQueries[validatorID] = validatorRequests
	}

	validatorRequests[requestID] = pendingQuery{
		registered: b.clock.Time(),
		msgType:    msgType,
	}
	return true
}

// RegisterResponse removes the query from pending
func (b *queryBenchlist) RegisterResponse(validatorID ids.ShortID, requestID uint32) {
	b.lock.Lock()
	defer b.lock.Unlock()

	if ok := b.removeQuery(validatorID, requestID); !ok {
		return
	}

	// Reset consecutive failures on success
	delete(b.consecutiveFailures, validatorID)
}

// QueryFailed notes a failure and benchlists [validatorID] if necessary
func (b *queryBenchlist) QueryFailed(validatorID ids.ShortID, requestID uint32) {
	b.lock.Lock()
	defer b.lock.Unlock()

	if ok := b.removeQuery(validatorID, requestID); !ok {
		return
	}

	// Track the message failure and bench [validatorID] if it has
	// surpassed the threshold
	currentTime := b.clock.Time()
	failureStreak := b.consecutiveFailures[validatorID]
	if failureStreak.consecutive == 0 {
		failureStreak.firstFailure = currentTime
	}
	failureStreak.consecutive++
	b.consecutiveFailures[validatorID] = failureStreak

	if failureStreak.consecutive >= b.threshold && !currentTime.Before(failureStreak.firstFailure.Add(b.minimumFailingDuration)) {
		b.bench(validatorID)
	}
}

func (b *queryBenchlist) bench(validatorID ids.ShortID) {
	if b.benchlistSet.Contains(validatorID) {
		return
	}

	// Goal:
	// Random end time in the range:
	// [max(lastEndTime, (currentTime + (duration/2)): currentTime + duration]
	// This maintains the invariant that validators in benchlistOrder are
	// ordered by the time that they should be unbenched
	currTime := b.clock.Time()
	minEndTime := currTime.Add(b.duration / 2)
	if elem := b.benchlistOrder.Back(); elem != nil {
		lastValidator := elem.Value.(ids.ShortID)
		lastEndTime := b.benchlistTimes[lastValidator]
		if lastEndTime.After(minEndTime) {
			minEndTime = lastEndTime
		}
	}
	maxEndTime := currTime.Add(b.duration)
	// Since maxEndTime is at least [duration] in the future and every element
	// added to benchlist was added in the past with an end time at most [duration]
	// in the future, this should never produce a negative duration.
	diff := maxEndTime.Sub(minEndTime)
	randomizedEndTime := minEndTime.Add(time.Duration(rand.Float64() * float64(diff))) // #nosec G404

	// Add to benchlist times with randomized delay
	b.benchlistTimes[validatorID] = randomizedEndTime
	b.benchlistOrder.PushBack(validatorID)
	b.benchlistSet.Add(validatorID)
	delete(b.consecutiveFailures, validatorID)
	b.ctx.Log.Debug(
		"benching validator %s after %d consecutive failed queries for %s",
		validatorID,
		b.threshold,
		randomizedEndTime.Sub(currTime),
	)

	// Note: there could be a memory leak if a large number of
	// validators were added, sampled, benched, and never sampled
	// again. Due to the minimum staking amount and durations this
	// is not a realistic concern.
	b.cleanup()
}

// cleanup ensures that we have not benched too much stake
// and removes anything from the benchlist whose time has expired
func (b *queryBenchlist) cleanup() {
	currentWeight, err := b.vdrs.SubsetWeight(b.benchlistSet)
	if err != nil {
		// Add log for this, should never happen
		b.ctx.Log.Error("failed to calculate subset weight due to: %s... Resetting benchlist", err)
		b.reset()
		return
	}

	currentTime := b.clock.Time()

	benchLen := b.benchlistSet.Len()
	updatedWeight := currentWeight
	totalWeight := b.vdrs.Weight()
	maxBenchlistWeight := uint64(float64(totalWeight) * b.maxPortion)
	// Iterate over elements of the benchlist in order of expiration
	for b.benchlistOrder.Len() > 0 {
		e := b.benchlistOrder.Front()

		validatorID := e.Value.(ids.ShortID)
		end := b.benchlistTimes[validatorID]
		// Remove elements with the next expiration until the next item has not
		// expired and the bench has less than the maximum weight
		// Note: this creates an edge case where benching a validator
		// with a sufficient stake may clear the bench if the benchlist is
		// not parameterized correctly.
		if currentTime.Before(end) && updatedWeight < maxBenchlistWeight {
			break
		}

		removeWeight, ok := b.vdrs.GetWeight(validatorID)
		if ok {
			newWeight, err := safemath.Sub64(updatedWeight, removeWeight)
			if err != nil {
				b.ctx.Log.Error("failed to calculate new subset weight due to: %s... Resetting benchlist", err)
				b.reset()
				return
			}
			updatedWeight = newWeight
		}

		b.ctx.Log.Debug("Removed Validator: (%s, %d). EndTime: %s. CurrentTime: %s)", validatorID, removeWeight, end, currentTime)
		b.benchlistOrder.Remove(e)
		delete(b.benchlistTimes, validatorID)
		b.benchlistSet.Remove(validatorID)
	}

	updatedBenchLen := b.benchlistSet.Len()
	b.ctx.Log.Debug("Maximum Benchable Weight: %d. Benched Weight: (%d/%d) -> (%d/%d). Benched Validators: %d -> %d.",
		maxBenchlistWeight,
		currentWeight,
		totalWeight,
		updatedWeight,
		totalWeight,
		benchLen,
		updatedBenchLen,
	)
	b.metrics.weightBenched.Set(float64(updatedWeight))
	b.metrics.numBenched.Set(float64(updatedBenchLen))
}

func (b *queryBenchlist) reset() {
	b.pendingQueries = make(map[ids.ShortID]map[uint32]pendingQuery)
	b.consecutiveFailures = make(map[ids.ShortID]failureStreak)
	b.benchlistTimes = make(map[ids.ShortID]time.Time)
	b.benchlistOrder.Init()
	b.benchlistSet.Clear()
	b.metrics.weightBenched.Set(0)
	b.metrics.numBenched.Set(0)
}

// removeQuery returns true if the query was present
func (b *queryBenchlist) removeQuery(validatorID ids.ShortID, requestID uint32) bool {
	validatorRequests, ok := b.pendingQueries[validatorID]
	if !ok {
		return false
	}

	query, ok := validatorRequests[requestID]
	if !ok {
		return false
	}

	delete(validatorRequests, requestID)
	if len(validatorRequests) == 0 {
		delete(b.pendingQueries, validatorID)
	}
	b.metrics.observe(validatorID, query.msgType, b.clock.Time().Sub(query.registered))
	return true
}
<commit_msg>add benchlist comments<commit_after>package benchlist

import (
	"container/list"
	"fmt"
	"math/rand"
	"sync"
	"time"

	"github.com/ava-labs/avalanchego/ids"
	"github.com/ava-labs/avalanchego/snow"
	"github.com/ava-labs/avalanchego/snow/validators"
	"github.com/ava-labs/avalanchego/utils/constants"
	"github.com/ava-labs/avalanchego/utils/timer"

	safemath "github.com/ava-labs/avalanchego/utils/math"
)

// If a peer consistently does not respond to queries, it will
// increase latencies on the network whenever that peer is polled.
// If we cannot terminate the poll early, then the poll will wait
// the full timeout before finalizing the poll and making progress.
// This can increase network latencies to an undesirable level.

// Therefore, nodes that consistently fail are "benched" such that
// queries to that node fail immediately to avoid waiting up to
// the full network timeout for a response.

// QueryBenchlist ...
type QueryBenchlist interface {
	// RegisterQuery registers a sent query and returns whether the query is subject to benchlist
	RegisterQuery(validatorID ids.ShortID, requestID uint32, msgType constants.MsgType) bool
	// RegisterResponse registers the response to a query message
	RegisterResponse(validatorID ids.ShortID, requstID uint32)
	// QueryFailed registers that a query did not receive a response within our synchrony bound
	QueryFailed(validatorID ids.ShortID, requestID uint32)
	// IsBenched returns true if messages to [validatorID]
	// should not be sent over the network and should immediately fail.
	IsBenched(validatorID ids.ShortID) bool
}

type queryBenchlist struct {
	lock    sync.Mutex
	ctx     *snow.Context
	metrics *metrics
	// Tells the time. Can be faked for testing.
	clock timer.Clock

	// Validator set of the network
	vdrs validators.Set
	// Validator ID --> Request ID --> non-empty iff
	// there is an outstanding request to this validator
	// with the corresponding requestID
	pendingQueries map[ids.ShortID]map[uint32]pendingQuery
	// Validator ID --> Consecutive failure information
	consecutiveFailures map[ids.ShortID]failureStreak

	// Validator ID --> Time they are benched until
	// If a validator is not benched, their ID is not a key in this map
	benchlistTimes map[ids.ShortID]time.Time
	benchlistOrder *list.List
	// IDs of validators that are currently benched
	benchlistSet ids.ShortSet

	// A validator will be benched if [threshold] messages in a row
	// to them time out and the first of those messages was more than
	// [minimumFailingDuration] ago
	threshold              int
	minimumFailingDuration time.Duration
	// A benched validator will be benched for between [duration/2] and [duration]
	duration time.Duration
	// The maximum percentage of total network stake that may be benched
	// Must be in [0,1)
	maxPortion float64
}

type pendingQuery struct {
	registered time.Time
	msgType    constants.MsgType
}

type failureStreak struct {
	// Time of first consecutive timeout
	firstFailure time.Time
	// Number of consecutive message timeouts
	consecutive int
}

// NewQueryBenchlist ...
func NewQueryBenchlist(
	validators validators.Set,
	ctx *snow.Context,
	threshold int,
	minimumFailingDuration,
	duration time.Duration,
	maxPortion float64,
	summaryEnabled bool,
	namespace string,
) (QueryBenchlist, error) {
	if maxPortion < 0 || maxPortion >= 1 {
		return nil, fmt.Errorf("max portion of benched stake must be in [0,1) but got %f", maxPortion)
	}
	metrics := &metrics{}
	return &queryBenchlist{
		pendingQueries:         make(map[ids.ShortID]map[uint32]pendingQuery),
		consecutiveFailures:    make(map[ids.ShortID]failureStreak),
		benchlistTimes:         make(map[ids.ShortID]time.Time),
		benchlistOrder:         list.New(),
		benchlistSet:           ids.ShortSet{},
		vdrs:                   validators,
		threshold:              threshold,
		minimumFailingDuration: minimumFailingDuration,
		duration:               duration,
		maxPortion:             maxPortion,
		ctx:                    ctx,
		metrics:                metrics,
	}, metrics.Initialize(ctx, namespace, summaryEnabled)
}

// IsBenched returns true if messages to [validatorID]
// should not be sent over the network and should immediately fail.
func (b *queryBenchlist) IsBenched(validatorID ids.ShortID) bool {
	b.lock.Lock()
	isBenched := b.isBenched(validatorID)
	b.lock.Unlock()
	return isBenched
}

// isBenched checks if [validatorID] is currently benched
// and calls cleanup if its benching period has elapsed
func (b *queryBenchlist) isBenched(validatorID ids.ShortID) bool {
	end, ok := b.benchlistTimes[validatorID]
	if !ok {
		return false
	}

	if b.clock.Time().Before(end) {
		return true
	}

	// If a benched item has expired, cleanup the benchlist
	b.cleanup()
	return false
}

// RegisterQuery attempts to register a query from [validatorID] and returns true
// if that request should be made (not subject to benchlisting)
func (b *queryBenchlist) RegisterQuery(validatorID ids.ShortID, requestID uint32, msgType constants.MsgType) bool {
	b.lock.Lock()
	defer b.lock.Unlock()

	if benched := b.isBenched(validatorID); benched {
		return false
	}

	validatorRequests, ok := b.pendingQueries[validatorID]
	if !ok {
		validatorRequests = make(map[uint32]pendingQuery)
		b.pendingQueries[validatorID] = validatorRequests
	}

	validatorRequests[requestID] = pendingQuery{
		registered: b.clock.Time(),
		msgType:    msgType,
	}
	return true
}

// RegisterResponse removes the query from pending
func (b *queryBenchlist) RegisterResponse(validatorID ids.ShortID, requestID uint32) {
	b.lock.Lock()
	defer b.lock.Unlock()

	if ok := b.removeQuery(validatorID, requestID); !ok {
		return
	}

	// Reset consecutive failures on success
	delete(b.consecutiveFailures, validatorID)
}

// QueryFailed notes a failure and benchlists [validatorID] if necessary
func (b *queryBenchlist) QueryFailed(validatorID ids.ShortID, requestID uint32) {
	b.lock.Lock()
	defer b.lock.Unlock()

	if ok := b.removeQuery(validatorID, requestID); !ok {
		return
	}

	// Track the message failure and bench [validatorID] if it has
	// surpassed the threshold
	currentTime := b.clock.Time()
	failureStreak := b.consecutiveFailures[validatorID]
	if failureStreak.consecutive == 0 {
		failureStreak.firstFailure = currentTime
	}
	failureStreak.consecutive++
	b.consecutiveFailures[validatorID] = failureStreak

	if failureStreak.consecutive >= b.threshold && !currentTime.Before(failureStreak.firstFailure.Add(b.minimumFailingDuration)) {
		b.bench(validatorID)
	}
}

func (b *queryBenchlist) bench(validatorID ids.ShortID) {
	if b.benchlistSet.Contains(validatorID) {
		return
	}

	// Goal:
	// Random end time in the range:
	// [max(lastEndTime, (currentTime + (duration/2)): currentTime + duration]
	// This maintains the invariant that validators in benchlistOrder are
	// ordered by the time that they should be unbenched
	currTime := b.clock.Time()
	minEndTime := currTime.Add(b.duration / 2)
	if elem := b.benchlistOrder.Back(); elem != nil {
		lastValidator := elem.Value.(ids.ShortID)
		lastEndTime := b.benchlistTimes[lastValidator]
		if lastEndTime.After(minEndTime) {
			minEndTime = lastEndTime
		}
	}
	maxEndTime := currTime.Add(b.duration)
	// Since maxEndTime is at least [duration] in the future and every element
	// added to benchlist was added in the past with an end time at most [duration]
	// in the future, this should never produce a negative duration.
	diff := maxEndTime.Sub(minEndTime)
	randomizedEndTime := minEndTime.Add(time.Duration(rand.Float64() * float64(diff))) // #nosec G404

	// Add to benchlist times with randomized delay
	b.benchlistTimes[validatorID] = randomizedEndTime
	b.benchlistOrder.PushBack(validatorID)
	b.benchlistSet.Add(validatorID)
	delete(b.consecutiveFailures, validatorID)
	b.ctx.Log.Debug(
		"benching validator %s after %d consecutive failed queries for %s",
		validatorID,
		b.threshold,
		randomizedEndTime.Sub(currTime),
	)

	// Note: there could be a memory leak if a large number of
	// validators were added, sampled, benched, and never sampled
	// again. Due to the minimum staking amount and durations this
	// is not a realistic concern.
	b.cleanup()
}

// cleanup ensures that we have not benched too much stake
// and removes anything from the benchlist whose time has expired
func (b *queryBenchlist) cleanup() {
	currentWeight, err := b.vdrs.SubsetWeight(b.benchlistSet)
	if err != nil {
		// Add log for this, should never happen
		b.ctx.Log.Error("failed to calculate subset weight due to: %w. Resetting benchlist", err)
		b.reset()
		return
	}

	currentTime := b.clock.Time()

	benchLen := b.benchlistSet.Len()
	updatedWeight := currentWeight
	totalWeight := b.vdrs.Weight()
	maxBenchlistWeight := uint64(float64(totalWeight) * b.maxPortion)
	// Iterate over elements of the benchlist in order of expiration
	for b.benchlistOrder.Len() > 0 {
		e := b.benchlistOrder.Front()

		validatorID := e.Value.(ids.ShortID)
		end := b.benchlistTimes[validatorID]
		// Remove elements with the next expiration until the next item has not
		// expired and the bench has less than the maximum weight
		// Note: this creates an edge case where benching a validator
		// with a sufficient stake may clear the bench if the benchlist is
		// not parameterized correctly.
		if currentTime.Before(end) && updatedWeight < maxBenchlistWeight {
			break
		}

		removeWeight, ok := b.vdrs.GetWeight(validatorID)
		if ok {
			newWeight, err := safemath.Sub64(updatedWeight, removeWeight)
			if err != nil {
				b.ctx.Log.Error("failed to calculate new subset weight due to: %w. Resetting benchlist", err)
				b.reset()
				return
			}
			updatedWeight = newWeight
		}

		b.ctx.Log.Debug("Removed Validator: (%s, %d). EndTime: %s. CurrentTime: %s)", validatorID, removeWeight, end, currentTime)
		b.benchlistOrder.Remove(e)
		delete(b.benchlistTimes, validatorID)
		b.benchlistSet.Remove(validatorID)
	}

	updatedBenchLen := b.benchlistSet.Len()
	b.ctx.Log.Debug("Maximum Benchable Weight: %d. Benched Weight: (%d/%d) -> (%d/%d). Benched Validators: %d -> %d.",
		maxBenchlistWeight,
		currentWeight,
		totalWeight,
		updatedWeight,
		totalWeight,
		benchLen,
		updatedBenchLen,
	)
	b.metrics.weightBenched.Set(float64(updatedWeight))
	b.metrics.numBenched.Set(float64(updatedBenchLen))
}

func (b *queryBenchlist) reset() {
	b.pendingQueries = make(map[ids.ShortID]map[uint32]pendingQuery)
	b.consecutiveFailures = make(map[ids.ShortID]failureStreak)
	b.benchlistTimes = make(map[ids.ShortID]time.Time)
	b.benchlistOrder.Init()
	b.benchlistSet.Clear()
	b.metrics.weightBenched.Set(0)
	b.metrics.numBenched.Set(0)
}

// removeQuery returns true if the query was present
func (b *queryBenchlist) removeQuery(validatorID ids.ShortID, requestID uint32) bool {
	validatorRequests, ok := b.pendingQueries[validatorID]
	if !ok {
		return false
	}

	query, ok := validatorRequests[requestID]
	if !ok {
		return false
	}

	delete(validatorRequests, requestID)
	if len(validatorRequests) == 0 {
		delete(b.pendingQueries, validatorID)
	}
	b.metrics.observe(validatorID, query.msgType, b.clock.Time().Sub(query.registered))
	return true
}
<|endoftext|>


<commit_before>package apns

import (
	"bytes"
	"encoding/binary"
	"encoding/hex"
	"encoding/json"
	"errors"
	"math/rand"
	"strconv"
	"time"
)

// Push commands always start with command value 2.
const pushCommandValue = 2

// Your total notification payload cannot exceed 2 KB.
const MaxPayloadSizeBytes = 2048

// Every push notification gets a pseudo-unique identifier;
// this establishes the upper boundary for it. Apple will return
// this identifier if there is an issue sending your notification.
const IdentifierUbound = 9999

// Constants related to the payload fields and their lengths.
const (
	deviceTokenItemid            = 1
	payloadItemid                = 2
	notificationIdentifierItemid = 3
	expirationDateItemid         = 4
	priorityItemid               = 5
	deviceTokenItemLength            = 64
	notificationIdentifierLength = 4
	expirationDateLength         = 4
	priorityLength               = 1
)

// Payload contains the notification data for your request.
//
// Alert is an interface here because it supports either a string
// or a dictionary, represented within by an AlertDictionary struct.
type Payload struct {
	Alert            interface{} `json:"alert,omitempty"`
	Badge            int         `json:"badge,omitempty"`
	Sound            string      `json:"sound,omitempty"`
	ContentAvailable int         `json:"content-available,omitempty"`
	Category         string      `json:"category,omitempty"`
}

// NewPayload creates and returns a Payload structure.
func NewPayload() *Payload {
	return new(Payload)
}

// AlertDictionary is a more complex notification payload.
//
// From the APN docs: "Use the ... alert dictionary in general only if you absolutely need to."
// The AlertDictionary is suitable for specific localization needs.
type AlertDictionary struct {
	Body         string   `json:"body,omitempty"`
	ActionLocKey string   `json:"action-loc-key,omitempty"`
	LocKey       string   `json:"loc-key,omitempty"`
	LocArgs      []string `json:"loc-args,omitempty"`
	LaunchImage  string   `json:"launch-image,omitempty"`
}

// NewAlertDictionary creates and returns an AlertDictionary structure.
func NewAlertDictionary() *AlertDictionary {
	return new(AlertDictionary)
}

// PushNotification is the wrapper for the Payload.
// The length fields are computed in ToBytes() and aren't represented here.
type PushNotification struct {
	Identifier  uint32 `json:"identifier"`
	Expiry      uint32 `json:"expiry"`
	DeviceToken string `json:"device_token"`
	Payload     map[string]interface{} `json:"payload"`
	Priority    uint8 `json:"priority"`
}

// NewPushNotification creates and returns a PushNotification structure.
// It also initializes the pseudo-random identifier.
func NewPushNotification() (pn *PushNotification) {
	pn = new(PushNotification)
	pn.Payload = make(map[string]interface{})
	pn.Identifier = uint32(rand.New(rand.NewSource(time.Now().UnixNano())).Int31n(9999))
	pn.Priority = 10
	return
}

// AddPayload sets the "aps" Payload section of the request. It also
// has a hack described within to deal with specific zero values.
func (pn *PushNotification) AddPayload(p *Payload) {
	// This deserves some explanation.
	//
	// Setting an exported field of type int to 0
	// triggers the omitempty behavior if you've set it.
	// Since the badge is optional, we should omit it if
	// it's not set. However, we want to include it if the
	// value is 0, so there's a hack in push_notification.go
	// that exploits the fact that Apple treats -1 for a
	// badge value as though it were 0 (i.e. it clears the
	// badge but doesn't stop the notification from going
	// through successfully.)
	//
	// Still a hack though :)
	if p.Badge == 0 {
		p.Badge = -1
	}
	pn.Set("aps", p)
}

// Get returns the value of a Payload key, if it exists.
func (pn *PushNotification) Get(key string) interface{} {
	return pn.Payload[key]
}

// Set defines the value of a Payload key.
func (pn *PushNotification) Set(key string, value interface{}) {
	pn.Payload[key] = value
}

// PayloadJSON returns the current Payload in JSON format.
func (pn *PushNotification) PayloadJSON() ([]byte, error) {
	return json.Marshal(pn.Payload)
}

// PayloadString returns the current Payload in string format.
func (pn *PushNotification) PayloadString() (string, error) {
	j, err := pn.PayloadJSON()
	return string(j), err
}

// ToBytes returns a byte array of the complete PushNotification
// struct. This array is what should be transmitted to the APN Service.
func (pn *PushNotification) ToBytes() ([]byte, error) {
	token, err := hex.DecodeString(pn.DeviceToken)
	if err != nil {
		return nil, err
	}
	Payload, err := pn.PayloadJSON()
	if err != nil {
		return nil, err
	}
	if len(Payload) > MaxPayloadSizeBytes {
		return nil, errors.New("Payload is larger than the " + strconv.Itoa(MaxPayloadSizeBytes) + " byte limit")
	}

	frameBuffer := new(bytes.Buffer)

	binary.Write(frameBuffer, binary.BigEndian, uint8(deviceTokenItemid))
	binary.Write(frameBuffer, binary.BigEndian, uint16(len(token)))
	binary.Write(frameBuffer, binary.BigEndian, token)
	binary.Write(frameBuffer, binary.BigEndian, uint8(payloadItemid))
	binary.Write(frameBuffer, binary.BigEndian, uint16(len(Payload)))
	binary.Write(frameBuffer, binary.BigEndian, Payload)
	binary.Write(frameBuffer, binary.BigEndian, uint8(notificationIdentifierItemid))
	binary.Write(frameBuffer, binary.BigEndian, uint16(notificationIdentifierLength))
	binary.Write(frameBuffer, binary.BigEndian, uint32(pn.Identifier))

	binary.Write(frameBuffer, binary.BigEndian, uint8(expirationDateItemid))
	binary.Write(frameBuffer, binary.BigEndian, uint16(expirationDateLength))

	binary.Write(frameBuffer, binary.BigEndian, pn.Expiry)

	binary.Write(frameBuffer, binary.BigEndian, uint8(priorityItemid))
	binary.Write(frameBuffer, binary.BigEndian, uint16(priorityLength))
	binary.Write(frameBuffer, binary.BigEndian, uint8(pn.Priority))

	buffer := bytes.NewBuffer([]byte{})
	binary.Write(buffer, binary.BigEndian, pushCommandValue)
	binary.Write(buffer, binary.BigEndian, frameBuffer.Len())
	binary.Write(buffer, binary.BigEndian, frameBuffer.Bytes())

	return buffer.Bytes(), nil
}
<commit_msg>Going back to 32 size on device token length<commit_after>package apns

import (
	"bytes"
	"encoding/binary"
	"encoding/hex"
	"encoding/json"
	"errors"
	"math/rand"
	"strconv"
	"time"
)

// Push commands always start with command value 2.
const pushCommandValue = 2

// Your total notification payload cannot exceed 2 KB.
const MaxPayloadSizeBytes = 2048

// Every push notification gets a pseudo-unique identifier;
// this establishes the upper boundary for it. Apple will return
// this identifier if there is an issue sending your notification.
const IdentifierUbound = 9999

// Constants related to the payload fields and their lengths.
const (
	deviceTokenItemid            = 1
	payloadItemid                = 2
	notificationIdentifierItemid = 3
	expirationDateItemid         = 4
	priorityItemid               = 5
	deviceTokenItemLength            = 32
	notificationIdentifierLength = 4
	expirationDateLength         = 4
	priorityLength               = 1
)

// Payload contains the notification data for your request.
//
// Alert is an interface here because it supports either a string
// or a dictionary, represented within by an AlertDictionary struct.
type Payload struct {
	Alert            interface{} `json:"alert,omitempty"`
	Badge            int         `json:"badge,omitempty"`
	Sound            string      `json:"sound,omitempty"`
	ContentAvailable int         `json:"content-available,omitempty"`
	Category         string      `json:"category,omitempty"`
}

// NewPayload creates and returns a Payload structure.
func NewPayload() *Payload {
	return new(Payload)
}

// AlertDictionary is a more complex notification payload.
//
// From the APN docs: "Use the ... alert dictionary in general only if you absolutely need to."
// The AlertDictionary is suitable for specific localization needs.
type AlertDictionary struct {
	Body         string   `json:"body,omitempty"`
	ActionLocKey string   `json:"action-loc-key,omitempty"`
	LocKey       string   `json:"loc-key,omitempty"`
	LocArgs      []string `json:"loc-args,omitempty"`
	LaunchImage  string   `json:"launch-image,omitempty"`
}

// NewAlertDictionary creates and returns an AlertDictionary structure.
func NewAlertDictionary() *AlertDictionary {
	return new(AlertDictionary)
}

// PushNotification is the wrapper for the Payload.
// The length fields are computed in ToBytes() and aren't represented here.
type PushNotification struct {
	Identifier  uint32 `json:"identifier"`
	Expiry      uint32 `json:"expiry"`
	DeviceToken string `json:"device_token"`
	Payload     map[string]interface{} `json:"payload"`
	Priority    uint8 `json:"priority"`
}

// NewPushNotification creates and returns a PushNotification structure.
// It also initializes the pseudo-random identifier.
func NewPushNotification() (pn *PushNotification) {
	pn = new(PushNotification)
	pn.Payload = make(map[string]interface{})
	pn.Identifier = uint32(rand.New(rand.NewSource(time.Now().UnixNano())).Int31n(9999))
	pn.Priority = 10
	return
}

// AddPayload sets the "aps" Payload section of the request. It also
// has a hack described within to deal with specific zero values.
func (pn *PushNotification) AddPayload(p *Payload) {
	// This deserves some explanation.
	//
	// Setting an exported field of type int to 0
	// triggers the omitempty behavior if you've set it.
	// Since the badge is optional, we should omit it if
	// it's not set. However, we want to include it if the
	// value is 0, so there's a hack in push_notification.go
	// that exploits the fact that Apple treats -1 for a
	// badge value as though it were 0 (i.e. it clears the
	// badge but doesn't stop the notification from going
	// through successfully.)
	//
	// Still a hack though :)
	if p.Badge == 0 {
		p.Badge = -1
	}
	pn.Set("aps", p)
}

// Get returns the value of a Payload key, if it exists.
func (pn *PushNotification) Get(key string) interface{} {
	return pn.Payload[key]
}

// Set defines the value of a Payload key.
func (pn *PushNotification) Set(key string, value interface{}) {
	pn.Payload[key] = value
}

// PayloadJSON returns the current Payload in JSON format.
func (pn *PushNotification) PayloadJSON() ([]byte, error) {
	return json.Marshal(pn.Payload)
}

// PayloadString returns the current Payload in string format.
func (pn *PushNotification) PayloadString() (string, error) {
	j, err := pn.PayloadJSON()
	return string(j), err
}

// ToBytes returns a byte array of the complete PushNotification
// struct. This array is what should be transmitted to the APN Service.
func (pn *PushNotification) ToBytes() ([]byte, error) {
	token, err := hex.DecodeString(pn.DeviceToken)
	if err != nil {
		return nil, err
	}
	Payload, err := pn.PayloadJSON()
	if err != nil {
		return nil, err
	}
	if len(Payload) > MaxPayloadSizeBytes {
		return nil, errors.New("Payload is larger than the " + strconv.Itoa(MaxPayloadSizeBytes) + " byte limit")
	}

	frameBuffer := new(bytes.Buffer)

	binary.Write(frameBuffer, binary.BigEndian, uint8(deviceTokenItemid))
	binary.Write(frameBuffer, binary.BigEndian, uint16(deviceTokenItemLength))
	binary.Write(frameBuffer, binary.BigEndian, token)
	binary.Write(frameBuffer, binary.BigEndian, uint8(payloadItemid))
	binary.Write(frameBuffer, binary.BigEndian, uint16(len(Payload)))
	binary.Write(frameBuffer, binary.BigEndian, Payload)
	binary.Write(frameBuffer, binary.BigEndian, uint8(notificationIdentifierItemid))
	binary.Write(frameBuffer, binary.BigEndian, uint16(notificationIdentifierLength))
	binary.Write(frameBuffer, binary.BigEndian, uint32(pn.Identifier))

	binary.Write(frameBuffer, binary.BigEndian, uint8(expirationDateItemid))
	binary.Write(frameBuffer, binary.BigEndian, uint16(expirationDateLength))

	binary.Write(frameBuffer, binary.BigEndian, pn.Expiry)

	binary.Write(frameBuffer, binary.BigEndian, uint8(priorityItemid))
	binary.Write(frameBuffer, binary.BigEndian, uint16(priorityLength))
	binary.Write(frameBuffer, binary.BigEndian, uint8(pn.Priority))

	buffer := bytes.NewBuffer([]byte{})
	binary.Write(buffer, binary.BigEndian, pushCommandValue)
	binary.Write(buffer, binary.BigEndian, frameBuffer.Len())
	binary.Write(buffer, binary.BigEndian, frameBuffer.Bytes())

	return buffer.Bytes(), nil
}
<|endoftext|>


<commit_before>package manta

import (
	"encoding/json"

	"github.com/dotabuff/manta/dota"
	"github.com/golang/protobuf/proto"
)

// Field is always filled, table only for sub-tables
type dt_property struct {
	Field *dt_field
	Table *dt
}

// A datatable field
type dt_field struct {
	Name  string
	Type  string
	Index int32

	Flags     *int32
	BitCount  *int32
	LowValue  *float32
	HighValue *float32

	Version    *int32
	Serializer *PropertySerializer `json:"-"`
}

// A single datatable
type dt struct {
	Name       string
	Flags      *int32
	Version    int32
	Properties []*dt_property
}

// The flattened serializers object
type flattened_serializers struct {
	Serializers map[string]map[int32]*dt // serializer name -> [versions]
	proto       *dota.CSVCMsg_FlattenedSerializer
	pst         *PropertySerializerTable
}

// Dumps a flattened table as json
func (sers *flattened_serializers) dump_json(name string) string {
	// Can't marshal map[int32]x
	type jContainer struct {
		Version int32
		Data    *dt
	}

	j := make([]jContainer, 0)
	for i, o := range sers.Serializers[name] {
		j = append(j, jContainer{i, o})
	}

	str, _ := json.MarshalIndent(j, "", "  ") // two space ident
	return string(str)
}

// Fills properties for a data table
func (sers *flattened_serializers) recurse_table(cur *dota.ProtoFlattenedSerializerT) *dt {
	// Basic table structure
	table := &dt{
		Name:       sers.proto.GetSymbols()[cur.GetSerializerNameSym()],
		Version:    cur.GetSerializerVersion(),
		Properties: make([]*dt_property, 0),
	}

	props := sers.proto.GetFields()

	// Append all the properties
	for _, idx := range cur.GetFieldsIndex() {
		pField := props[idx]
		prop := &dt_property{nil, nil}

		// Field can always be set
		prop.Field = &dt_field{
			Name:  sers.proto.GetSymbols()[pField.GetVarNameSym()],
			Index: -1,

			Flags:     pField.EncodeFlags,
			BitCount:  pField.BitCount,
			LowValue:  pField.LowValue,
			HighValue: pField.HighValue,

			Type:       (sers.proto.GetSymbols()[pField.GetVarTypeSym()]),
			Version:    pField.FieldSerializerVersion,
			Serializer: sers.pst.GetPropertySerializerByName(sers.proto.GetSymbols()[pField.GetVarTypeSym()]),
		}

		// Optional: Attach the serializer version for the property if applicable
		if pField.FieldSerializerNameSym != nil {
			pFieldName := sers.proto.GetSymbols()[pField.GetFieldSerializerNameSym()]
			pFieldVersion := pField.GetFieldSerializerVersion()
			pSerializer := sers.Serializers[pFieldName][pFieldVersion]

			if pSerializer == nil {
				_panicf("Error: Serializer version %d for %s hasn't been added yet.", pFieldVersion, pFieldName)
			}

			prop.Table = pSerializer
		}

		// Optional: Adjust array fields
		if prop.Field.Serializer.IsArray {
			// Add our own temp table for the array
			tmpDt := &dt{
				Name:       prop.Field.Name,
				Flags:      nil,
				Version:    0,
				Properties: make([]*dt_property, 0),
			}

			// Add each array field to the table
			for i := uint32(0); i < prop.Field.Serializer.Length; i++ {
				tmpDt.Properties = append(tmpDt.Properties, &dt_property{
					Field: &dt_field{
						Name:       _sprintf("%s.%04d", prop.Field.Name, i),
						Type:       prop.Field.Serializer.Name,
						Index:      int32(i),
						Flags:      prop.Field.Flags,
						BitCount:   prop.Field.BitCount,
						LowValue:   prop.Field.LowValue,
						HighValue:  prop.Field.HighValue,
						Version:    prop.Field.Version,
						Serializer: prop.Field.Serializer.ArraySerializer,
					},
					Table: prop.Table, // This carries on the actual table instead of overriding it
				})
			}

			prop.Table = tmpDt
		}

		table.Properties = append(
			table.Properties,
			prop,
		)
	}

	return table
}

// Parses a CDemoSendTables packet
func ParseSendTablesNew(m *dota.CDemoSendTables, pst *PropertySerializerTable) *flattened_serializers {
	// This packet just contains a single large buffer
	r := NewReader(m.GetData())

	// The buffer starts with a varint encoded length
	size := int(r.readVarUint32())
	if size != r.remBytes() {
		_panicf("expected %d additional bytes, got %d", size, r.remBytes())
	}

	// Read the rest of the buffer as a CSVCMsg_FlattenedSerializer.
	buf := r.readBytes(size)
	msg := &dota.CSVCMsg_FlattenedSerializer{}
	if err := proto.Unmarshal(buf, msg); err != nil {
		_panicf("cannot decode proto: %s", err)
	}

	// Create the flattened_serializers object and fill it
	fs := &flattened_serializers{
		Serializers: make(map[string]map[int32]*dt),
		proto:       msg,
		pst:         pst,
	}

	// Iterate through all flattened serializers and fill their properties
	for _, o := range msg.GetSerializers() {
		sName := msg.GetSymbols()[o.GetSerializerNameSym()]
		sVer := o.GetSerializerVersion()

		if fs.Serializers[sName] == nil {
			fs.Serializers[sName] = make(map[int32]*dt)
		}

		fs.Serializers[sName][sVer] = fs.recurse_table(o)
	}

	return fs
}

// Internal callback for OnCDemoSendTables.
func (p *Parser) onCDemoSendTablesNew(m *dota.CDemoSendTables) error {
	// Temporary: Also parse things the old way
	p.onCDemoSendTables(m)

	p.Serializers = ParseSendTablesNew(m, GetDefaultPropertySerializerTable()).Serializers
	return nil
}
<commit_msg>Carry table index over<commit_after>package manta

import (
	"encoding/json"

	"github.com/dotabuff/manta/dota"
	"github.com/golang/protobuf/proto"
)

// Field is always filled, table only for sub-tables
type dt_property struct {
	Field *dt_field
	Table *dt
}

// A datatable field
type dt_field struct {
	Name  string
	Type  string
	Index int32

	Flags     *int32
	BitCount  *int32
	LowValue  *float32
	HighValue *float32

	Version    *int32
	Serializer *PropertySerializer `json:"-"`
}

// A single datatable
type dt struct {
	Name       string
	Flags      *int32
	Version    int32
	Properties []*dt_property
}

// The flattened serializers object
type flattened_serializers struct {
	Serializers map[string]map[int32]*dt // serializer name -> [versions]
	proto       *dota.CSVCMsg_FlattenedSerializer
	pst         *PropertySerializerTable
}

// Dumps a flattened table as json
func (sers *flattened_serializers) dump_json(name string) string {
	// Can't marshal map[int32]x
	type jContainer struct {
		Version int32
		Data    *dt
	}

	j := make([]jContainer, 0)
	for i, o := range sers.Serializers[name] {
		j = append(j, jContainer{i, o})
	}

	str, _ := json.MarshalIndent(j, "", "  ") // two space ident
	return string(str)
}

// Fills properties for a data table
func (sers *flattened_serializers) recurse_table(cur *dota.ProtoFlattenedSerializerT) *dt {
	// Basic table structure
	table := &dt{
		Name:       sers.proto.GetSymbols()[cur.GetSerializerNameSym()],
		Version:    cur.GetSerializerVersion(),
		Properties: make([]*dt_property, 0),
	}

	props := sers.proto.GetFields()

	// Append all the properties
	for _, idx := range cur.GetFieldsIndex() {
		pField := props[idx]
		prop := &dt_property{nil, nil}

		// Field can always be set
		prop.Field = &dt_field{
			Name:  sers.proto.GetSymbols()[pField.GetVarNameSym()],
			Index: -1,

			Flags:     pField.EncodeFlags,
			BitCount:  pField.BitCount,
			LowValue:  pField.LowValue,
			HighValue: pField.HighValue,

			Type:       (sers.proto.GetSymbols()[pField.GetVarTypeSym()]),
			Version:    pField.FieldSerializerVersion,
			Serializer: sers.pst.GetPropertySerializerByName(sers.proto.GetSymbols()[pField.GetVarTypeSym()]),
		}

		// Optional: Attach the serializer version for the property if applicable
		if pField.FieldSerializerNameSym != nil {
			pFieldName := sers.proto.GetSymbols()[pField.GetFieldSerializerNameSym()]
			pFieldVersion := pField.GetFieldSerializerVersion()
			pSerializer := sers.Serializers[pFieldName][pFieldVersion]

			if pSerializer == nil {
				_panicf("Error: Serializer version %d for %s hasn't been added yet.", pFieldVersion, pFieldName)
			}

			prop.Table = pSerializer
		}

		// Optional: Adjust array fields
		if prop.Field.Serializer.IsArray {
			// Add our own temp table for the array
			tmpDt := &dt{
				Name:       prop.Field.Name,
				Flags:      nil,
				Version:    0,
				Properties: make([]*dt_property, 0),
			}

			// Add each array field to the table
			for i := uint32(0); i < prop.Field.Serializer.Length; i++ {
				tmpDt.Properties = append(tmpDt.Properties, &dt_property{
					Field: &dt_field{
						Name:       _sprintf("%s.%04d", prop.Field.Name, i),
						Type:       prop.Field.Serializer.Name,
						Index:      int32(i),
						Flags:      prop.Field.Flags,
						BitCount:   prop.Field.BitCount,
						LowValue:   prop.Field.LowValue,
						HighValue:  prop.Field.HighValue,
						Version:    prop.Field.Version,
						Serializer: prop.Field.Serializer.ArraySerializer,
					},
					Table: prop.Table, // This carries on the actual table instead of overriding it
				})

				// Copy parent prop to rename it's name according to the array index
				if prop.Table != nil {
					nTable := *prop.Table
					nTable.Name = _sprintf("%s.%04d", nTable.Name, i)
					tmpDt.Properties[len(tmpDt.Properties)-1].Table = &nTable
				}
			}

			prop.Table = tmpDt
		}

		table.Properties = append(
			table.Properties,
			prop,
		)
	}

	return table
}

// Parses a CDemoSendTables packet
func ParseSendTablesNew(m *dota.CDemoSendTables, pst *PropertySerializerTable) *flattened_serializers {
	// This packet just contains a single large buffer
	r := NewReader(m.GetData())

	// The buffer starts with a varint encoded length
	size := int(r.readVarUint32())
	if size != r.remBytes() {
		_panicf("expected %d additional bytes, got %d", size, r.remBytes())
	}

	// Read the rest of the buffer as a CSVCMsg_FlattenedSerializer.
	buf := r.readBytes(size)
	msg := &dota.CSVCMsg_FlattenedSerializer{}
	if err := proto.Unmarshal(buf, msg); err != nil {
		_panicf("cannot decode proto: %s", err)
	}

	// Create the flattened_serializers object and fill it
	fs := &flattened_serializers{
		Serializers: make(map[string]map[int32]*dt),
		proto:       msg,
		pst:         pst,
	}

	// Iterate through all flattened serializers and fill their properties
	for _, o := range msg.GetSerializers() {
		sName := msg.GetSymbols()[o.GetSerializerNameSym()]
		sVer := o.GetSerializerVersion()

		if fs.Serializers[sName] == nil {
			fs.Serializers[sName] = make(map[int32]*dt)
		}

		fs.Serializers[sName][sVer] = fs.recurse_table(o)
	}

	return fs
}

// Internal callback for OnCDemoSendTables.
func (p *Parser) onCDemoSendTablesNew(m *dota.CDemoSendTables) error {
	// Temporary: Also parse things the old way
	p.onCDemoSendTables(m)

	p.Serializers = ParseSendTablesNew(m, GetDefaultPropertySerializerTable()).Serializers
	return nil
}
<|endoftext|>


<commit_before>// Package waitmap implements a simple thread-safe map.
package waitmap

import (
	"sync"
)

// An entry in a WaitMap. Note that mutx and cond may legally be nil - this
// avoids heavy allocations at the cost of some code complexity.
type entry struct {
	mutx *sync.Mutex
	cond *sync.Cond
	data interface{}
	ok   bool
}

type WaitMap struct {
	lock *sync.Mutex
	ents map[interface{}]*entry
}

func New() *WaitMap {
	return &WaitMap{
		lock: new(sync.Mutex),
		ents: make(map[interface{}]*entry),
	}
}

// Retrieves the value mapped to by k. If no such value is yet available, waits
// until one is, and then returns that. Otherwise, it returns the value
// available at the time Get is called.
func (m *WaitMap) Get(k interface{}) interface{} {
	m.lock.Lock()
	e, ok := m.ents[k]
	if !ok {
		mutx := new(sync.Mutex)
		e = &entry{
			mutx: mutx,
			cond: sync.NewCond(mutx),
			data: nil,
			ok: false,
		}
		m.ents[k] = e
	}
	m.lock.Unlock()

	// If e.ok is true, e.data exists and can never cease to exist. We need
	// this check to avoid using a nil e.mutx. We could also actually check e.mutx to see if it's nil, but this accomplishes the same thing and will be slightly faster on average (since we will often avoid unnecessarily messing with the mutex).
	if e.ok { return e.data }
	e.mutx.Lock()
	defer e.mutx.Unlock()
	e.cond.Wait()
	return e.data
}

// Maps the given key and value, waking any waiting calls to Get.
func (m *WaitMap) Set(k interface{}, v interface{}) {
	m.lock.Lock()
	defer m.lock.Unlock()
	e, ok := m.ents[k]
	if !ok {
		e := &entry{nil, nil, v, true}
		m.ents[k] = e
		return
	}
	e.data = v
	e.ok = true
	e.cond.Broadcast()
}

// Returns true if k is a key in the map.
func (m *WaitMap) Check(k interface{}) bool {
	m.lock.Lock()
	defer m.lock.Unlock()
	e, ok := m.ents[k]
	if !ok { return false }
	return e.ok
}

<commit_msg>Fixing wrapping<commit_after>// Package waitmap implements a simple thread-safe map.
package waitmap

import (
	"sync"
)

// An entry in a WaitMap. Note that mutx and cond may legally be nil - this
// avoids heavy allocations at the cost of some code complexity.
type entry struct {
	mutx *sync.Mutex
	cond *sync.Cond
	data interface{}
	ok   bool
}

type WaitMap struct {
	lock *sync.Mutex
	ents map[interface{}]*entry
}

func New() *WaitMap {
	return &WaitMap{
		lock: new(sync.Mutex),
		ents: make(map[interface{}]*entry),
	}
}

// Retrieves the value mapped to by k. If no such value is yet available, waits
// until one is, and then returns that. Otherwise, it returns the value
// available at the time Get is called.
func (m *WaitMap) Get(k interface{}) interface{} {
	m.lock.Lock()
	e, ok := m.ents[k]
	if !ok {
		mutx := new(sync.Mutex)
		e = &entry{
			mutx: mutx,
			cond: sync.NewCond(mutx),
			data: nil,
			ok: false,
		}
		m.ents[k] = e
	}
	m.lock.Unlock()

	// If e.ok is true, e.data exists and can never cease to exist. We need
	// this check to avoid using a nil e.mutx. We could also actually check
	// e.mutx to see if it's nil, but this accomplishes the same thing and
	// will be slightly faster on average (since we will often avoid
	// unnecessarily messing with the mutex).
	if e.ok { return e.data }
	e.mutx.Lock()
	defer e.mutx.Unlock()
	e.cond.Wait()
	return e.data
}

// Maps the given key and value, waking any waiting calls to Get.
func (m *WaitMap) Set(k interface{}, v interface{}) {
	m.lock.Lock()
	defer m.lock.Unlock()
	e, ok := m.ents[k]
	if !ok {
		e := &entry{nil, nil, v, true}
		m.ents[k] = e
		return
	}
	e.data = v
	e.ok = true
	e.cond.Broadcast()
}

// Returns true if k is a key in the map.
func (m *WaitMap) Check(k interface{}) bool {
	m.lock.Lock()
	defer m.lock.Unlock()
	e, ok := m.ents[k]
	if !ok { return false }
	return e.ok
}

<|endoftext|>