<commit_before>package tokauth

import (
	"code.google.com/p/go-uuid/uuid"
	"encoding/hex"
	mgo "gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
	"time"
)

type accessData struct {
	ExpiresAt    time.Time `json:"expiresAt,omitempty" bson:"createdAt"`
	AccessToken  string    `json:"accessToken,omitempty" bson:"accessToken"`
	RefreshToken string    `json:"refreshToken,omitempty" bson:"refreshToken"`
}

// newAccessData creates new AccessData from the ID and insert it in the database.
func newAccessData(refreshToken string) (data *accessData, err error) {
	data = &accessData{
		ExpiresAt:    time.Now().Add(tokenExpiration),
		AccessToken:  hex.EncodeToString([]byte(uuid.New())),
		RefreshToken: refreshToken,
	}
	_, err = accessCollection.UpsertId(data.AccessToken, data)
	return
}

// Register a new client. The client must exists in the collection given via the function SetClientCollection.
// The ID parameter is the ID of the client that needs to be registered.
func Register(id interface{}) (refreshToken, accessToken string, err error) {
	refreshToken = hex.EncodeToString([]byte(uuid.New()))
	data, err := newAccessData(refreshToken)
	if err != nil {
		return
	}
	accessToken = data.AccessToken
	if err = clientCollection.Update(bson.M{"_id": id}, bson.M{"$set": bson.M{"refreshToken": refreshToken}}); err != nil {
		return
	}
	return
}

// SetRemainingTime set the new date at which the token will expires.
func SetRemainingTime(accessToken string, date time.Time) (err error) {
	return accessCollection.Update(bson.M{"accessToken": accessToken}, bson.M{"expiresAt": date})
}

// Refresh the AccessData of the user.
func Refresh(refreshToken string) (accessToken string, err error) {
	if err = clientCollection.Find(bson.M{"refreshToken": refreshToken}).One(&Client{}); err != nil {
		return
	}
	data, err := newAccessData(refreshToken)
	if err != nil {
		return
	}
	accessToken = data.AccessToken
	return
}

// Authorize check if the given token match with the AccessToken of the client.
func Authorize(token string) bool {
	var data *accessData
	if err := accessCollection.FindId(token).One(&data); err == mgo.ErrNotFound {
		return false
	}
	return true
}

// GetRefreshTokenFromAccessToken returns the refreshToken of the user associated by this accessToken.
func GetRefreshTokenFromAccessToken(accessToken string) (refreshToken string, err error) {
	client := &Client{}
	if err = accessCollection.FindId(accessToken).One(&client); err != nil {
		return
	}
	refreshToken = client.RefreshToken
	return
}

// GetAccessTokenFromRefreshToken returns the accessToken of the user associated by this refreshToken.
func GetAccessTokenFromRefreshToken(refreshToken string) (accessToken string, err error) {
	data := &accessData{}
	if err = accessCollection.Find(bson.M{"refreshToken": refreshToken}).One(&data); err != nil {
		return
	}
	accessToken = data.AccessToken
	return
}
<commit_msg>Added function to refresh and returns the client<commit_after>package tokauth

import (
	"code.google.com/p/go-uuid/uuid"
	"encoding/hex"
	mgo "gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
	"time"
)

type accessData struct {
	ExpiresAt    time.Time `json:"expiresAt,omitempty" bson:"createdAt"`
	AccessToken  string    `json:"accessToken,omitempty" bson:"accessToken"`
	RefreshToken string    `json:"refreshToken,omitempty" bson:"refreshToken"`
}

// newAccessData creates new AccessData from the ID and insert it in the database.
func newAccessData(refreshToken string) (data *accessData, err error) {
	data = &accessData{
		ExpiresAt:    time.Now().Add(tokenExpiration),
		AccessToken:  hex.EncodeToString([]byte(uuid.New())),
		RefreshToken: refreshToken,
	}
	_, err = accessCollection.UpsertId(data.AccessToken, data)
	return
}

// Register a new client. The client must exists in the collection given via the function SetClientCollection.
// The ID parameter is the ID of the client that needs to be registered.
func Register(id interface{}) (refreshToken, accessToken string, err error) {
	refreshToken = hex.EncodeToString([]byte(uuid.New()))
	data, err := newAccessData(refreshToken)
	if err != nil {
		return
	}
	accessToken = data.AccessToken
	if err = clientCollection.Update(bson.M{"_id": id}, bson.M{"$set": bson.M{"refreshToken": refreshToken}}); err != nil {
		return
	}
	return
}

// SetRemainingTime set the new date at which the token will expires.
func SetRemainingTime(accessToken string, date time.Time) (err error) {
	return accessCollection.Update(bson.M{"accessToken": accessToken}, bson.M{"expiresAt": date})
}

// Refresh the AccessData of the user.
func Refresh(refreshToken string) (accessToken string, err error) {
	if err = clientCollection.Find(bson.M{"refreshToken": refreshToken}).One(&Client{}); err != nil {
		return
	}
	data, err := newAccessData(refreshToken)
	if err != nil {
		return
	}
	accessToken = data.AccessToken
	return
}

// RefreshAndGetClient refreshes the user and returns it.
func RefreshAndGetClient(refreshToken string, object interface{}) (err error) {
	if err = clientCollection.Find(bson.M{"refreshToken": refreshToken}).One(object); err != nil {
		return
	}
	_, err = newAccessData(refreshToken)
	return err
}

// Authorize check if the given token match with the AccessToken of the client.
func Authorize(token string) bool {
	var data *accessData
	if err := accessCollection.FindId(token).One(&data); err == mgo.ErrNotFound {
		return false
	}
	return true
}

// GetRefreshTokenFromAccessToken returns the refreshToken of the user associated by this accessToken.
func GetRefreshTokenFromAccessToken(accessToken string) (refreshToken string, err error) {
	client := &Client{}
	if err = accessCollection.FindId(accessToken).One(&client); err != nil {
		return
	}
	refreshToken = client.RefreshToken
	return
}

// GetAccessTokenFromRefreshToken returns the accessToken of the user associated by this refreshToken.
func GetAccessTokenFromRefreshToken(refreshToken string) (accessToken string, err error) {
	data := &accessData{}
	if err = accessCollection.Find(bson.M{"refreshToken": refreshToken}).One(&data); err != nil {
		return
	}
	accessToken = data.AccessToken
	return
}
<|endoftext|>


<commit_before>/*
 *
 * Copyright 2018 gRPC authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

// Binary client is an example client.
package main

import (
	"context"
	"flag"
	"fmt"
	"io"
	"log"
	"time"

	"golang.org/x/oauth2"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/credentials/oauth"
	ecpb "google.golang.org/grpc/examples/features/proto/echo"
	"google.golang.org/grpc/testdata"
)

var addr = flag.String("addr", "localhost:50051", "the address to connect to")

const fallbackToken = "some-secret-token"

// logger is to mock a sophisticated logging system. To simplify the example, we just print out the content.
func logger(format string, a ...interface{}) {
	fmt.Printf("LOG:\t"+format+"\n", a...)
}

// unaryInterceptor is an example unary interceptor.
func unaryInterceptor(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
	var credsConfigured bool
	for _, o := range opts {
		_, ok := o.(grpc.PerRPCCredsCallOption)
		if ok {
			credsConfigured = true
			break
		}
	}
	if !credsConfigured {
		opts = append(opts, grpc.PerRPCCredentials(oauth.NewOauthAccess(&oauth2.Token{
			AccessToken: fallbackToken,
		})))
	}
	start := time.Now()
	err := invoker(ctx, method, req, reply, cc, opts...)
	end := time.Now()
	logger("RPC: %s, start time: %s, end time: %s, err: %v", method, start.Format("Basic"), end.Format(time.RFC3339), err)
	return err
}

// wrappedStream  wraps around the embedded grpc.ClientStream, and intercepts the RecvMsg and
// SendMsg method call.
type wrappedStream struct {
	grpc.ClientStream
}

func (w *wrappedStream) RecvMsg(m interface{}) error {
	logger("Receive a message (Type: %T) at %v", m, time.Now().Format(time.RFC3339))
	return w.ClientStream.RecvMsg(m)
}

func (w *wrappedStream) SendMsg(m interface{}) error {
	logger("Send a message (Type: %T) at %v", m, time.Now().Format(time.RFC3339))
	return w.ClientStream.SendMsg(m)
}

func newWrappedStream(s grpc.ClientStream) grpc.ClientStream {
	return &wrappedStream{s}
}

// streamInterceptor is an example stream interceptor.
func streamInterceptor(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) {
	var credsConfigured bool
	for _, o := range opts {
		_, ok := o.(*grpc.PerRPCCredsCallOption)
		if ok {
			credsConfigured = true
		}
	}
	if !credsConfigured {
		opts = append(opts, grpc.PerRPCCredentials(oauth.NewOauthAccess(&oauth2.Token{
			AccessToken: fallbackToken,
		})))
	}
	s, err := streamer(ctx, desc, cc, method, opts...)
	if err != nil {
		return nil, err
	}
	return newWrappedStream(s), nil
}

func callUnaryEcho(client ecpb.EchoClient, message string) {
	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()
	resp, err := client.UnaryEcho(ctx, &ecpb.EchoRequest{Message: message})
	if err != nil {
		log.Fatalf("client.UnaryEcho(_) = _, %v: ", err)
	}
	fmt.Println("UnaryEcho: ", resp.Message)
}

func callBidiStreamingEcho(client ecpb.EchoClient) {
	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()
	c, err := client.BidirectionalStreamingEcho(ctx)
	if err != nil {
		return
	}
	for i := 0; i < 5; i++ {
		if err := c.Send(&ecpb.EchoRequest{Message: fmt.Sprintf("Request %d", i+1)}); err != nil {
			log.Fatalf("failed to send request due to error: %v", err)
		}
	}
	c.CloseSend()
	for {
		resp, err := c.Recv()
		if err == io.EOF {
			break
		}
		if err != nil {
			log.Fatalf("failed to receive response due to error: %v", err)
		}
		fmt.Println("BidiStreaming Echo: ", resp.Message)
	}
}

func main() {
	flag.Parse()

	// Create tls based credential.
	creds, err := credentials.NewClientTLSFromFile(testdata.Path("ca.pem"), "x.test.youtube.com")
	if err != nil {
		log.Fatalf("failed to load credentials: %v", err)
	}

	// Set up a connection to the server.
	conn, err := grpc.Dial(*addr, grpc.WithTransportCredentials(creds), grpc.WithUnaryInterceptor(unaryInterceptor), grpc.WithStreamInterceptor(streamInterceptor), grpc.WithBlock())
	if err != nil {
		log.Fatalf("did not connect: %v", err)
	}
	defer conn.Close()

	// Make a echo client and send RPCs.
	rgc := ecpb.NewEchoClient(conn)
	callUnaryEcho(rgc, "hello world")
	callBidiStreamingEcho(rgc)
}
<commit_msg>examples: break from 'PerRPCCredsCallOption' status checking loop (#3772)<commit_after>/*
 *
 * Copyright 2018 gRPC authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

// Binary client is an example client.
package main

import (
	"context"
	"flag"
	"fmt"
	"io"
	"log"
	"time"

	"golang.org/x/oauth2"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/credentials/oauth"
	ecpb "google.golang.org/grpc/examples/features/proto/echo"
	"google.golang.org/grpc/testdata"
)

var addr = flag.String("addr", "localhost:50051", "the address to connect to")

const fallbackToken = "some-secret-token"

// logger is to mock a sophisticated logging system. To simplify the example, we just print out the content.
func logger(format string, a ...interface{}) {
	fmt.Printf("LOG:\t"+format+"\n", a...)
}

// unaryInterceptor is an example unary interceptor.
func unaryInterceptor(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
	var credsConfigured bool
	for _, o := range opts {
		_, ok := o.(grpc.PerRPCCredsCallOption)
		if ok {
			credsConfigured = true
			break
		}
	}
	if !credsConfigured {
		opts = append(opts, grpc.PerRPCCredentials(oauth.NewOauthAccess(&oauth2.Token{
			AccessToken: fallbackToken,
		})))
	}
	start := time.Now()
	err := invoker(ctx, method, req, reply, cc, opts...)
	end := time.Now()
	logger("RPC: %s, start time: %s, end time: %s, err: %v", method, start.Format("Basic"), end.Format(time.RFC3339), err)
	return err
}

// wrappedStream  wraps around the embedded grpc.ClientStream, and intercepts the RecvMsg and
// SendMsg method call.
type wrappedStream struct {
	grpc.ClientStream
}

func (w *wrappedStream) RecvMsg(m interface{}) error {
	logger("Receive a message (Type: %T) at %v", m, time.Now().Format(time.RFC3339))
	return w.ClientStream.RecvMsg(m)
}

func (w *wrappedStream) SendMsg(m interface{}) error {
	logger("Send a message (Type: %T) at %v", m, time.Now().Format(time.RFC3339))
	return w.ClientStream.SendMsg(m)
}

func newWrappedStream(s grpc.ClientStream) grpc.ClientStream {
	return &wrappedStream{s}
}

// streamInterceptor is an example stream interceptor.
func streamInterceptor(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) {
	var credsConfigured bool
	for _, o := range opts {
		_, ok := o.(*grpc.PerRPCCredsCallOption)
		if ok {
			credsConfigured = true
			break
		}
	}
	if !credsConfigured {
		opts = append(opts, grpc.PerRPCCredentials(oauth.NewOauthAccess(&oauth2.Token{
			AccessToken: fallbackToken,
		})))
	}
	s, err := streamer(ctx, desc, cc, method, opts...)
	if err != nil {
		return nil, err
	}
	return newWrappedStream(s), nil
}

func callUnaryEcho(client ecpb.EchoClient, message string) {
	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()
	resp, err := client.UnaryEcho(ctx, &ecpb.EchoRequest{Message: message})
	if err != nil {
		log.Fatalf("client.UnaryEcho(_) = _, %v: ", err)
	}
	fmt.Println("UnaryEcho: ", resp.Message)
}

func callBidiStreamingEcho(client ecpb.EchoClient) {
	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()
	c, err := client.BidirectionalStreamingEcho(ctx)
	if err != nil {
		return
	}
	for i := 0; i < 5; i++ {
		if err := c.Send(&ecpb.EchoRequest{Message: fmt.Sprintf("Request %d", i+1)}); err != nil {
			log.Fatalf("failed to send request due to error: %v", err)
		}
	}
	c.CloseSend()
	for {
		resp, err := c.Recv()
		if err == io.EOF {
			break
		}
		if err != nil {
			log.Fatalf("failed to receive response due to error: %v", err)
		}
		fmt.Println("BidiStreaming Echo: ", resp.Message)
	}
}

func main() {
	flag.Parse()

	// Create tls based credential.
	creds, err := credentials.NewClientTLSFromFile(testdata.Path("ca.pem"), "x.test.youtube.com")
	if err != nil {
		log.Fatalf("failed to load credentials: %v", err)
	}

	// Set up a connection to the server.
	conn, err := grpc.Dial(*addr, grpc.WithTransportCredentials(creds), grpc.WithUnaryInterceptor(unaryInterceptor), grpc.WithStreamInterceptor(streamInterceptor), grpc.WithBlock())
	if err != nil {
		log.Fatalf("did not connect: %v", err)
	}
	defer conn.Close()

	// Make a echo client and send RPCs.
	rgc := ecpb.NewEchoClient(conn)
	callUnaryEcho(rgc, "hello world")
	callBidiStreamingEcho(rgc)
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"strings"

	"github.com/codegangsta/cli"
	"github.com/pivotalservices/cfops/tileregistry"
	"github.com/xchapter7x/lo"
)

//CreateBURACliCommand - this will create a cli command object for backup / restore
func CreateBURACliCommand(name string, usage string, eh *ErrorHandler) (command cli.Command) {
	desc := fmt.Sprintf("%s --opsmanagerhost <host> --adminuser <usr> --adminpass <pass> --opsmanageruser <opsuser> --opsmanagerpass <opspass> -d <dir> --tile elastic-runtime", name)
	command = cli.Command{
		Name:        name,
		Usage:       usage,
		Description: desc,
		Flags:       buraFlags,
		Action:      buraAction(name, eh),
	}
	return
}

func buraAction(commandName string, eh *ErrorHandler) (action func(*cli.Context)) {
	action = func(c *cli.Context) {
		var (
			fs = &flagSet{
				host:           c.String(flagList[opsManagerHost].Flag[0]),
				adminUser:      c.String(flagList[adminUser].Flag[0]),
				adminPass:      c.String(flagList[adminPass].Flag[0]),
				opsManagerUser: c.String(flagList[opsManagerUser].Flag[0]),
				opsManagerPass: c.String(flagList[opsManagerPass].Flag[0]),
				dest:           c.String(flagList[dest].Flag[0]),
				tile:           c.String(flagList[tile].Flag[0]),
			}
		)

		if tile, err := getTileFromRegistry(fs, commandName); err == nil {
			if err = runTileAction(commandName, tile); err != nil {
				lo.G.Errorf("there was an error: %s running %s on %s tile:%v", err.Error(), commandName, fs.Tile(), tile)
				exitOnError(eh, c, commandName, err)
				return
			}
		} else {
			lo.G.Errorf("there was an error getting tile from registry: %s", err.Error())
			exitOnError(eh, c, commandName, err)
			return
		}
		lo.G.Debug("Tile action completed successfully")
	}
	return
}

func exitOnError(eh *ErrorHandler, c *cli.Context, commandName string, err error) {
	cli.ShowCommandHelp(c, commandName)
	eh.ExitCode = helpExitCode
	eh.Error = err
}

func runTileAction(commandName string, tile tileregistry.Tile) (err error) {
	lo.G.Debug("Running %s for tile: %+v", commandName, tile)
	switch commandName {
	case "backup":
		err = tile.Backup()
	case "restore":
		err = tile.Restore()
	}
	return
}

func getTileFromRegistry(fs *flagSet, commandName string) (tile tileregistry.Tile, err error) {
	lo.G.Debug("checking registry for '%s' tile", fs.Tile())

	if tileBuilder, ok := tileregistry.GetRegistry()[fs.Tile()]; ok {
		lo.G.Debug("found tile in registry")

		if hasValidBackupRestoreFlags(fs) {
			lo.G.Debug("we have all required flags and a proper builder")
			tile, err = tileBuilder.New(tileregistry.TileSpec{
				OpsManagerHost:   fs.Host(),
				AdminUser:        fs.AdminUser(),
				AdminPass:        fs.AdminPass(),
				OpsManagerUser:   fs.OpsManagerUser(),
				OpsManagerPass:   fs.OpsManagerPass(),
				ArchiveDirectory: fs.Dest(),
			})
			if err != nil {
				return nil, fmt.Errorf("failure to connect to ops manager host: %s", err.Error())
			}

		} else {
			err = ErrInvalidFlagArgs
		}

	} else {
		lo.G.Errorf("tile '%s' not found", fs.Tile())
		err = ErrInvalidTileSelection
	}
	return
}

var buraFlags = func() (flags []cli.Flag) {
	for _, v := range flagList {
		flags = append(flags, cli.StringFlag{
			Name:   strings.Join(v.Flag, ", "),
			Value:  "",
			Usage:  v.Desc,
			EnvVar: v.EnvVar,
		})
	}
	return
}()

const (
	errExitCode           = 1
	helpExitCode          = 2
	cleanExitCode         = 0
	opsManagerHost string = "opsmanagerHost"
	adminUser      string = "adminUser"
	adminPass      string = "adminPass"
	opsManagerUser string = "opsManagerUser"
	opsManagerPass string = "opsManagerPass"
	dest           string = "destination"
	tile           string = "tile"
)

var (
	//ErrInvalidFlagArgs - error for invalid flags
	ErrInvalidFlagArgs = errors.New("invalid cli flag args")
	//ErrInvalidTileSelection - error for invalid tile
	ErrInvalidTileSelection = errors.New("invalid tile selected. try the 'list-tiles' option to see a list of available tiles")
	flagList                = map[string]flagBucket{
		opsManagerHost: flagBucket{
			Flag:   []string{"opsmanagerhost", "omh"},
			Desc:   "hostname for Ops Manager",
			EnvVar: "CFOPS_HOST",
		},
		adminUser: flagBucket{
			Flag:   []string{"adminuser", "du"},
			Desc:   "username for Ops Mgr admin (Ops Manager WebConsole Credentials)",
			EnvVar: "CFOPS_ADMIN_USER",
		},
		adminPass: flagBucket{
			Flag:   []string{"adminpass", "dp"},
			Desc:   "password for Ops Mgr admin (Ops Manager WebConsole Credentials)",
			EnvVar: "CFOPS_ADMIN_PASS",
		},
		opsManagerUser: flagBucket{
			Flag:   []string{"opsmanageruser", "omu"},
			Desc:   "username for Ops Manager VM Access (used for ssh connections)",
			EnvVar: "CFOPS_OM_USER",
		},
		opsManagerPass: flagBucket{
			Flag:   []string{"opsmanagerpass", "omp"},
			Desc:   "password for Ops Manager VM Access (used for ssh connections)",
			EnvVar: "CFOPS_OM_PASS",
		},
		dest: flagBucket{
			Flag:   []string{"destination", "d"},
			Desc:   "path of the Cloud Foundry archive",
			EnvVar: "CFOPS_DEST_PATH",
		},
		tile: flagBucket{
			Flag:   []string{"tile", "t"},
			Desc:   "a tile you would like to run the operation on",
			EnvVar: "CFOPS_TILE",
		},
	}
)

type (
	flagSet struct {
		host           string
		adminUser      string
		adminPass      string
		opsManagerUser string
		opsManagerPass string
		dest           string
		tile           string
	}

	flagBucket struct {
		Flag   []string
		Desc   string
		EnvVar string
	}
)

func (s *flagSet) Host() string {
	return s.host
}

func (s *flagSet) AdminUser() string {
	return s.adminUser
}

func (s *flagSet) AdminPass() string {
	return s.adminPass
}

func (s *flagSet) OpsManagerUser() string {
	return s.opsManagerUser
}

func (s *flagSet) OpsManagerPass() string {
	return s.opsManagerPass
}

func (s *flagSet) Dest() string {
	return s.dest
}

func (s *flagSet) Tile() string {
	return s.tile
}

func hasValidBackupRestoreFlags(fs *flagSet) bool {
	res := (fs.Host() != "" &&
		fs.AdminUser() != "" &&
		fs.AdminPass() != "" &&
		fs.OpsManagerUser() != "" &&
		fs.Dest() != "" &&
		fs.Tile() != "")

	if res == false {
		lo.G.Debug("OpsManagerHost: ", fs.Host())
		lo.G.Debug("AdminUser: ", fs.AdminUser())
		lo.G.Debug("AdminPass: ", fs.AdminPass())
		lo.G.Debug("OpsManagerUser: ", fs.OpsManagerUser())
		lo.G.Debug("OpsManagerPass: ", fs.OpsManagerPass())
		lo.G.Debug("Destination: ", fs.Dest())
	}
	return res
}
<commit_msg>[#114884611] adding encrpytionkey flag to cfops cli<commit_after>package main

import (
	"errors"
	"fmt"
	"strings"

	"github.com/codegangsta/cli"
	"github.com/pivotalservices/cfops/tileregistry"
	"github.com/xchapter7x/lo"
)

//CreateBURACliCommand - this will create a cli command object for backup / restore
func CreateBURACliCommand(name string, usage string, eh *ErrorHandler) (command cli.Command) {
	desc := fmt.Sprintf("%s --opsmanagerhost <host> --adminuser <usr> --adminpass <pass> --opsmanageruser <opsuser> --opsmanagerpass <opspass> -d <dir> --tile elastic-runtime", name)
	command = cli.Command{
		Name:        name,
		Usage:       usage,
		Description: desc,
		Flags:       buraFlags,
		Action:      buraAction(name, eh),
	}
	return
}

func buraAction(commandName string, eh *ErrorHandler) (action func(*cli.Context)) {
	action = func(c *cli.Context) {
		var (
			fs = &flagSet{
				host:           c.String(flagList[opsManagerHost].Flag[0]),
				adminUser:      c.String(flagList[adminUser].Flag[0]),
				adminPass:      c.String(flagList[adminPass].Flag[0]),
				opsManagerUser: c.String(flagList[opsManagerUser].Flag[0]),
				opsManagerPass: c.String(flagList[opsManagerPass].Flag[0]),
				dest:           c.String(flagList[dest].Flag[0]),
				tile:           c.String(flagList[tile].Flag[0]),
			}
		)

		if tile, err := getTileFromRegistry(fs, commandName); err == nil {
			if err = runTileAction(commandName, tile); err != nil {
				lo.G.Errorf("there was an error: %s running %s on %s tile:%v", err.Error(), commandName, fs.Tile(), tile)
				exitOnError(eh, c, commandName, err)
				return
			}
		} else {
			lo.G.Errorf("there was an error getting tile from registry: %s", err.Error())
			exitOnError(eh, c, commandName, err)
			return
		}
		lo.G.Debug("Tile action completed successfully")
	}
	return
}

func exitOnError(eh *ErrorHandler, c *cli.Context, commandName string, err error) {
	cli.ShowCommandHelp(c, commandName)
	eh.ExitCode = helpExitCode
	eh.Error = err
}

func runTileAction(commandName string, tile tileregistry.Tile) (err error) {
	lo.G.Debug("Running %s for tile: %+v", commandName, tile)
	switch commandName {
	case "backup":
		err = tile.Backup()
	case "restore":
		err = tile.Restore()
	}
	return
}

func getTileFromRegistry(fs *flagSet, commandName string) (tile tileregistry.Tile, err error) {
	lo.G.Debug("checking registry for '%s' tile", fs.Tile())

	if tileBuilder, ok := tileregistry.GetRegistry()[fs.Tile()]; ok {
		lo.G.Debug("found tile in registry")

		if hasValidBackupRestoreFlags(fs) {
			lo.G.Debug("we have all required flags and a proper builder")
			tile, err = tileBuilder.New(tileregistry.TileSpec{
				OpsManagerHost:   fs.Host(),
				AdminUser:        fs.AdminUser(),
				AdminPass:        fs.AdminPass(),
				OpsManagerUser:   fs.OpsManagerUser(),
				OpsManagerPass:   fs.OpsManagerPass(),
				ArchiveDirectory: fs.Dest(),
				CryptKey:         fs.EncryptionKey(),
			})
			if err != nil {
				return nil, fmt.Errorf("failure to connect to ops manager host: %s", err.Error())
			}

		} else {
			err = ErrInvalidFlagArgs
		}

	} else {
		lo.G.Errorf("tile '%s' not found", fs.Tile())
		err = ErrInvalidTileSelection
	}
	return
}

var buraFlags = func() (flags []cli.Flag) {
	for _, v := range flagList {
		flags = append(flags, cli.StringFlag{
			Name:   strings.Join(v.Flag, ", "),
			Value:  "",
			Usage:  v.Desc,
			EnvVar: v.EnvVar,
		})
	}
	return
}()

const (
	errExitCode           = 1
	helpExitCode          = 2
	cleanExitCode         = 0
	opsManagerHost string = "opsmanagerHost"
	adminUser      string = "adminUser"
	adminPass      string = "adminPass"
	opsManagerUser string = "opsManagerUser"
	opsManagerPass string = "opsManagerPass"
	dest           string = "destination"
	tile           string = "tile"
	encryptionKey  string = "encryptionKey"
)

var (
	//ErrInvalidFlagArgs - error for invalid flags
	ErrInvalidFlagArgs = errors.New("invalid cli flag args")
	//ErrInvalidTileSelection - error for invalid tile
	ErrInvalidTileSelection = errors.New("invalid tile selected. try the 'list-tiles' option to see a list of available tiles")
	flagList                = map[string]flagBucket{
		opsManagerHost: flagBucket{
			Flag:   []string{"opsmanagerhost", "omh"},
			Desc:   "hostname for Ops Manager",
			EnvVar: "CFOPS_HOST",
		},
		adminUser: flagBucket{
			Flag:   []string{"adminuser", "du"},
			Desc:   "username for Ops Mgr admin (Ops Manager WebConsole Credentials)",
			EnvVar: "CFOPS_ADMIN_USER",
		},
		adminPass: flagBucket{
			Flag:   []string{"adminpass", "dp"},
			Desc:   "password for Ops Mgr admin (Ops Manager WebConsole Credentials)",
			EnvVar: "CFOPS_ADMIN_PASS",
		},
		opsManagerUser: flagBucket{
			Flag:   []string{"opsmanageruser", "omu"},
			Desc:   "username for Ops Manager VM Access (used for ssh connections)",
			EnvVar: "CFOPS_OM_USER",
		},
		opsManagerPass: flagBucket{
			Flag:   []string{"opsmanagerpass", "omp"},
			Desc:   "password for Ops Manager VM Access (used for ssh connections)",
			EnvVar: "CFOPS_OM_PASS",
		},
		dest: flagBucket{
			Flag:   []string{"destination", "d"},
			Desc:   "path of the Cloud Foundry archive",
			EnvVar: "CFOPS_DEST_PATH",
		},
		tile: flagBucket{
			Flag:   []string{"tile", "t"},
			Desc:   "a tile you would like to run the operation on",
			EnvVar: "CFOPS_TILE",
		},
		encryptionKey: flagBucket{
			Flag:   []string{"encryptionkey", "k"},
			Desc:   "encryption key to encrypt/decrypt your archive (key lengths supported are 16, 24, 32 for AES-128, AES-192, or AES-256)",
			EnvVar: "CFOPS_ENCRYPTION_KEY",
		},
	}
)

type (
	flagSet struct {
		host           string
		adminUser      string
		adminPass      string
		opsManagerUser string
		opsManagerPass string
		dest           string
		tile           string
		encryptionKey  string
	}

	flagBucket struct {
		Flag   []string
		Desc   string
		EnvVar string
	}
)

func (s *flagSet) Host() string {
	return s.host
}

func (s *flagSet) AdminUser() string {
	return s.adminUser
}

func (s *flagSet) AdminPass() string {
	return s.adminPass
}

func (s *flagSet) OpsManagerUser() string {
	return s.opsManagerUser
}

func (s *flagSet) OpsManagerPass() string {
	return s.opsManagerPass
}

func (s *flagSet) Dest() string {
	return s.dest
}

func (s *flagSet) Tile() string {
	return s.tile
}

func (s *flagSet) EncryptionKey() string {
	return s.encryptionKey
}

func hasValidBackupRestoreFlags(fs *flagSet) bool {
	res := (fs.Host() != "" &&
		fs.AdminUser() != "" &&
		fs.AdminPass() != "" &&
		fs.OpsManagerUser() != "" &&
		fs.Dest() != "" &&
		fs.Tile() != "")

	if res == false {
		lo.G.Debug("OpsManagerHost: ", fs.Host())
		lo.G.Debug("AdminUser: ", fs.AdminUser())
		lo.G.Debug("AdminPass: ", fs.AdminPass())
		lo.G.Debug("OpsManagerUser: ", fs.OpsManagerUser())
		lo.G.Debug("OpsManagerPass: ", fs.OpsManagerPass())
		lo.G.Debug("Destination: ", fs.Dest())
	}
	return res
}
<|endoftext|>


<commit_before>package zwave

import "net"

const CommandClassZIPGateway uint8 = 0x61
const GatewayConfigurationSet uint8 = 0x01

func MakeConfigurationSetPacket() []byte {
	bytes := make([]byte, 0)

	bytes = append(bytes, CommandClassZIPGateway)
	bytes = append(bytes, GatewayConfigurationSet)
	bytes = append(bytes, (make([]byte, 16))...)     // LAN IP Address
	bytes = append(bytes, byte(0x0))                 // LAN IP Prefix Prefix Length
	bytes = append(bytes, net.ParseIP("3000::1")...) // Portal IPv6 Prefix
	bytes = append(bytes, byte(0x40))                // Portal IPv6 Prefix length
	bytes = append(bytes, net.ParseIP("3000::1")...) // Gateway IPv6 Address
	bytes = append(bytes, (make([]byte, 16))...)     // PAN IPv6 Address

	return bytes
}

const CommandClassNetworkManagementProxy uint8 = 0x52
const NodeInfoCachedGet uint8 = 0x03

func MakeNodeInfoCachedGetPacket() []byte {
	bytes := make([]byte, 0)

	bytes = append(bytes, CommandClassNetworkManagementProxy)
	bytes = append(bytes, NodeInfoCachedGet)
	bytes = append(bytes, uint8(0x20))
	bytes = append(bytes, uint8(0x02))
	bytes = append(bytes, uint8(0x00))

	return bytes
}
<commit_msg>remove unneeded zwave command<commit_after>package zwave

const CommandClassNetworkManagementProxy uint8 = 0x52
const NodeInfoCachedGet uint8 = 0x03

func MakeNodeInfoCachedGetPacket() []byte {
	bytes := make([]byte, 0)

	bytes = append(bytes, CommandClassNetworkManagementProxy)
	bytes = append(bytes, NodeInfoCachedGet)
	bytes = append(bytes, uint8(0x20))
	bytes = append(bytes, uint8(0x02))
	bytes = append(bytes, uint8(0x00))

	return bytes
}
<|endoftext|>


<commit_before>package expr

import (
	"bytes"
	"math"
	"strconv"
	"time"

	pb "github.com/dgryski/carbonzipper/carbonzipperpb"
	pickle "github.com/kisielk/og-rek"
)

type MetricData struct {
	pb.FetchResponse

	// extra options
	xStep          float64
	valuesPerPoint int
	color          string
	alpha          float64
	lineWidth      float64

	drawAsInfinite bool
	secondYAxis    bool
	dashed         bool // TODO (ikruglov) smth like lineType would be better
	hasAlpha       bool
	stacked        bool
	stackName      string

	aggregatedValues  []float64
	aggregatedAbsent  []bool
	aggregateFunction func([]float64, []bool) (float64, bool)
}

func MarshalCSV(results []*MetricData) []byte {

	var b []byte

	for _, r := range results {

		step := r.GetStepTime()
		t := r.GetStartTime()
		for i, v := range r.Values {
			b = append(b, '"')
			b = append(b, r.GetName()...)
			b = append(b, '"')
			b = append(b, ',')
			b = append(b, time.Unix(int64(t), 0).Format("2006-01-02 15:04:05")...)
			b = append(b, ',')
			if !r.IsAbsent[i] {
				b = strconv.AppendFloat(b, v, 'f', -1, 64)
			}
			b = append(b, '\n')
			t += step
		}
	}
	return b
}

func ConsolidateJSON(maxDataPoints int, results []*MetricData) {
	var startTime int32 = -1
	var endTime int32 = -1

	for _, r := range results {
		t := r.GetStartTime()
		if startTime == -1 || startTime > t {
			startTime = t
		}
		t = r.GetStopTime()
		if endTime == -1 || endTime < t {
			endTime = t
		}
	}

	timeRange := endTime - startTime

	if timeRange <= 0 {
		return
	}

	for _, r := range results {
		numberOfDataPoints := math.Floor(float64(timeRange / r.GetStepTime()))
		if numberOfDataPoints > float64(maxDataPoints) {
			valuesPerPoint := math.Ceil(numberOfDataPoints / float64(maxDataPoints))
			r.valuesPerPoint = int(valuesPerPoint)
		}
	}
}

func MarshalJSON(results []*MetricData) []byte {
	var b []byte
	b = append(b, '[')

	var topComma bool
	for _, r := range results {
		if r == nil {
			continue
		}

		if topComma {
			b = append(b, ',')
		}
		topComma = true

		b = append(b, `{"target":`...)
		b = strconv.AppendQuoteToASCII(b, r.GetName())
		b = append(b, `,"datapoints":[`...)

		var innerComma bool
		t := r.GetStartTime()
		absent := r.AggregatedAbsent()
		for i, v := range r.AggregatedValues() {
			if innerComma {
				b = append(b, ',')
			}
			innerComma = true

			b = append(b, '[')

			if absent[i] || math.IsInf(v, 0) || math.IsNaN(v) {
				b = append(b, "null"...)
			} else {
				b = strconv.AppendFloat(b, v, 'f', -1, 64)
			}

			b = append(b, ',')

			b = strconv.AppendInt(b, int64(t), 10)

			b = append(b, ']')

			t += r.AggregatedTimeStep()
		}

		b = append(b, `]}`...)
	}

	b = append(b, ']')

	return b
}

func MarshalPickle(results []*MetricData) []byte {

	var p []map[string]interface{}

	for _, r := range results {
		values := make([]interface{}, len(r.Values))
		for i, v := range r.Values {
			if r.IsAbsent[i] {
				values[i] = pickle.None{}
			} else {
				values[i] = v
			}

		}
		p = append(p, map[string]interface{}{
			"name":   r.GetName(),
			"start":  r.GetStartTime(),
			"end":    r.GetStopTime(),
			"step":   r.GetStepTime(),
			"values": values,
		})
	}

	var buf bytes.Buffer

	penc := pickle.NewEncoder(&buf)
	penc.Encode(p)

	return buf.Bytes()
}

func MarshalProtobuf(results []*MetricData) ([]byte, error) {
	response := pb.MultiFetchResponse{}
	for _, metric := range results {
		response.Metrics = append(response.Metrics, &((*metric).FetchResponse))
	}
	b, err := response.Marshal()
	if err != nil {
		return nil, err
	}

	return b, nil
}

func MarshalRaw(results []*MetricData) []byte {

	var b []byte

	for _, r := range results {

		b = append(b, r.GetName()...)

		b = append(b, ',')
		b = strconv.AppendInt(b, int64(r.GetStartTime()), 10)
		b = append(b, ',')
		b = strconv.AppendInt(b, int64(r.GetStopTime()), 10)
		b = append(b, ',')
		b = strconv.AppendInt(b, int64(r.GetStepTime()), 10)
		b = append(b, '|')

		var comma bool
		for i, v := range r.Values {
			if comma {
				b = append(b, ',')
			}
			comma = true
			if r.IsAbsent[i] {
				b = append(b, "None"...)
			} else {
				b = strconv.AppendFloat(b, v, 'f', -1, 64)
			}
		}

		b = append(b, '\n')
	}
	return b
}

func (r *MetricData) AggregatedTimeStep() int32 {
	if r.valuesPerPoint == 1 || r.valuesPerPoint == 0 {
		return r.GetStepTime()
	}

	return r.GetStepTime() * int32(r.valuesPerPoint)
}

func (r *MetricData) AggregatedValues() []float64 {
	if r.aggregatedValues == nil {
		r.AggregateValues()
	}
	return r.aggregatedValues
}

func (r *MetricData) AggregatedAbsent() []bool {
	if r.aggregatedAbsent == nil {
		r.AggregateValues()
	}
	return r.aggregatedAbsent
}

func (r *MetricData) AggregateValues() {
	if r.valuesPerPoint == 1 || r.valuesPerPoint == 0 {
		v := make([]float64, len(r.Values))
		a := make([]bool, len(r.Values))
		for i, _ := range r.Values {
			a[i] = r.IsAbsent[i]
			v[i] = r.Values[i]
		}

		r.aggregatedValues = v
		r.aggregatedAbsent = a
		return
	}

	if r.aggregateFunction == nil {
		r.aggregateFunction = aggMean
	}

	n := len(r.Values)/r.valuesPerPoint + 1
	aggV := make([]float64, 0, n)
	aggA := make([]bool, 0, n)

	v := r.Values
	absent := r.IsAbsent

	for len(v) >= r.valuesPerPoint {
		val, abs := r.aggregateFunction(v[:r.valuesPerPoint], absent[:r.valuesPerPoint])
		aggV = append(aggV, val)
		aggA = append(aggA, abs)
		v = v[r.valuesPerPoint:]
		absent = absent[r.valuesPerPoint:]
	}

	if len(v) > 0 {
		val, abs := r.aggregateFunction(v, absent)
		aggV = append(aggV, val)
		aggA = append(aggA, abs)
	}

	r.aggregatedValues = aggV
	r.aggregatedAbsent = aggA
}

func aggMean(v []float64, absent []bool) (float64, bool) {
	var sum float64
	var n int
	for i, vv := range v {
		if !math.IsNaN(vv) && !absent[i] {
			sum += vv
			n++
		}
	}
	return sum / float64(n), n == 0
}

func aggMax(v []float64, absent []bool) (float64, bool) {
	var m float64 = math.Inf(-1)
	var abs bool = true
	for i, vv := range v {
		if !absent[i] && !math.IsNaN(vv) {
			abs = false
			if m < vv {
				m = vv
			}
		}
	}
	return m, abs
}

func aggMin(v []float64, absent []bool) (float64, bool) {
	var m float64 = math.Inf(1)
	var abs bool = true
	for i, vv := range v {
		if !absent[i] && !math.IsNaN(vv) {
			abs = false
			if m > vv {
				m = vv
			}
		}
	}
	return m, abs
}

func aggSum(v []float64, absent []bool) (float64, bool) {
	var sum float64
	var abs bool = true
	for i, vv := range v {
		if !math.IsNaN(vv) && !absent[i] {
			sum += vv
			abs = false
		}
	}
	return sum, abs
}
<commit_msg>expr: replace for loop with copy()<commit_after>package expr

import (
	"bytes"
	"math"
	"strconv"
	"time"

	pb "github.com/dgryski/carbonzipper/carbonzipperpb"
	pickle "github.com/kisielk/og-rek"
)

type MetricData struct {
	pb.FetchResponse

	// extra options
	xStep          float64
	valuesPerPoint int
	color          string
	alpha          float64
	lineWidth      float64

	drawAsInfinite bool
	secondYAxis    bool
	dashed         bool // TODO (ikruglov) smth like lineType would be better
	hasAlpha       bool
	stacked        bool
	stackName      string

	aggregatedValues  []float64
	aggregatedAbsent  []bool
	aggregateFunction func([]float64, []bool) (float64, bool)
}

func MarshalCSV(results []*MetricData) []byte {

	var b []byte

	for _, r := range results {

		step := r.GetStepTime()
		t := r.GetStartTime()
		for i, v := range r.Values {
			b = append(b, '"')
			b = append(b, r.GetName()...)
			b = append(b, '"')
			b = append(b, ',')
			b = append(b, time.Unix(int64(t), 0).Format("2006-01-02 15:04:05")...)
			b = append(b, ',')
			if !r.IsAbsent[i] {
				b = strconv.AppendFloat(b, v, 'f', -1, 64)
			}
			b = append(b, '\n')
			t += step
		}
	}
	return b
}

func ConsolidateJSON(maxDataPoints int, results []*MetricData) {
	var startTime int32 = -1
	var endTime int32 = -1

	for _, r := range results {
		t := r.GetStartTime()
		if startTime == -1 || startTime > t {
			startTime = t
		}
		t = r.GetStopTime()
		if endTime == -1 || endTime < t {
			endTime = t
		}
	}

	timeRange := endTime - startTime

	if timeRange <= 0 {
		return
	}

	for _, r := range results {
		numberOfDataPoints := math.Floor(float64(timeRange / r.GetStepTime()))
		if numberOfDataPoints > float64(maxDataPoints) {
			valuesPerPoint := math.Ceil(numberOfDataPoints / float64(maxDataPoints))
			r.valuesPerPoint = int(valuesPerPoint)
		}
	}
}

func MarshalJSON(results []*MetricData) []byte {
	var b []byte
	b = append(b, '[')

	var topComma bool
	for _, r := range results {
		if r == nil {
			continue
		}

		if topComma {
			b = append(b, ',')
		}
		topComma = true

		b = append(b, `{"target":`...)
		b = strconv.AppendQuoteToASCII(b, r.GetName())
		b = append(b, `,"datapoints":[`...)

		var innerComma bool
		t := r.GetStartTime()
		absent := r.AggregatedAbsent()
		for i, v := range r.AggregatedValues() {
			if innerComma {
				b = append(b, ',')
			}
			innerComma = true

			b = append(b, '[')

			if absent[i] || math.IsInf(v, 0) || math.IsNaN(v) {
				b = append(b, "null"...)
			} else {
				b = strconv.AppendFloat(b, v, 'f', -1, 64)
			}

			b = append(b, ',')

			b = strconv.AppendInt(b, int64(t), 10)

			b = append(b, ']')

			t += r.AggregatedTimeStep()
		}

		b = append(b, `]}`...)
	}

	b = append(b, ']')

	return b
}

func MarshalPickle(results []*MetricData) []byte {

	var p []map[string]interface{}

	for _, r := range results {
		values := make([]interface{}, len(r.Values))
		for i, v := range r.Values {
			if r.IsAbsent[i] {
				values[i] = pickle.None{}
			} else {
				values[i] = v
			}

		}
		p = append(p, map[string]interface{}{
			"name":   r.GetName(),
			"start":  r.GetStartTime(),
			"end":    r.GetStopTime(),
			"step":   r.GetStepTime(),
			"values": values,
		})
	}

	var buf bytes.Buffer

	penc := pickle.NewEncoder(&buf)
	penc.Encode(p)

	return buf.Bytes()
}

func MarshalProtobuf(results []*MetricData) ([]byte, error) {
	response := pb.MultiFetchResponse{}
	for _, metric := range results {
		response.Metrics = append(response.Metrics, &((*metric).FetchResponse))
	}
	b, err := response.Marshal()
	if err != nil {
		return nil, err
	}

	return b, nil
}

func MarshalRaw(results []*MetricData) []byte {

	var b []byte

	for _, r := range results {

		b = append(b, r.GetName()...)

		b = append(b, ',')
		b = strconv.AppendInt(b, int64(r.GetStartTime()), 10)
		b = append(b, ',')
		b = strconv.AppendInt(b, int64(r.GetStopTime()), 10)
		b = append(b, ',')
		b = strconv.AppendInt(b, int64(r.GetStepTime()), 10)
		b = append(b, '|')

		var comma bool
		for i, v := range r.Values {
			if comma {
				b = append(b, ',')
			}
			comma = true
			if r.IsAbsent[i] {
				b = append(b, "None"...)
			} else {
				b = strconv.AppendFloat(b, v, 'f', -1, 64)
			}
		}

		b = append(b, '\n')
	}
	return b
}

func (r *MetricData) AggregatedTimeStep() int32 {
	if r.valuesPerPoint == 1 || r.valuesPerPoint == 0 {
		return r.GetStepTime()
	}

	return r.GetStepTime() * int32(r.valuesPerPoint)
}

func (r *MetricData) AggregatedValues() []float64 {
	if r.aggregatedValues == nil {
		r.AggregateValues()
	}
	return r.aggregatedValues
}

func (r *MetricData) AggregatedAbsent() []bool {
	if r.aggregatedAbsent == nil {
		r.AggregateValues()
	}
	return r.aggregatedAbsent
}
func (r *MetricData) AggregateValues() {
	if r.valuesPerPoint == 1 || r.valuesPerPoint == 0 {
		r.aggregatedValues = make([]float64, len(r.Values))
		r.aggregatedAbsent = make([]bool, len(r.Values))
		copy(r.aggregatedValues, r.Values)
		copy(r.aggregatedAbsent, r.IsAbsent)
		return
	}

	if r.aggregateFunction == nil {
		r.aggregateFunction = aggMean
	}

	n := len(r.Values)/r.valuesPerPoint + 1
	aggV := make([]float64, 0, n)
	aggA := make([]bool, 0, n)

	v := r.Values
	absent := r.IsAbsent

	for len(v) >= r.valuesPerPoint {
		val, abs := r.aggregateFunction(v[:r.valuesPerPoint], absent[:r.valuesPerPoint])
		aggV = append(aggV, val)
		aggA = append(aggA, abs)
		v = v[r.valuesPerPoint:]
		absent = absent[r.valuesPerPoint:]
	}

	if len(v) > 0 {
		val, abs := r.aggregateFunction(v, absent)
		aggV = append(aggV, val)
		aggA = append(aggA, abs)
	}

	r.aggregatedValues = aggV
	r.aggregatedAbsent = aggA
}

func aggMean(v []float64, absent []bool) (float64, bool) {
	var sum float64
	var n int
	for i, vv := range v {
		if !math.IsNaN(vv) && !absent[i] {
			sum += vv
			n++
		}
	}
	return sum / float64(n), n == 0
}

func aggMax(v []float64, absent []bool) (float64, bool) {
	var m float64 = math.Inf(-1)
	var abs bool = true
	for i, vv := range v {
		if !absent[i] && !math.IsNaN(vv) {
			abs = false
			if m < vv {
				m = vv
			}
		}
	}
	return m, abs
}

func aggMin(v []float64, absent []bool) (float64, bool) {
	var m float64 = math.Inf(1)
	var abs bool = true
	for i, vv := range v {
		if !absent[i] && !math.IsNaN(vv) {
			abs = false
			if m > vv {
				m = vv
			}
		}
	}
	return m, abs
}

func aggSum(v []float64, absent []bool) (float64, bool) {
	var sum float64
	var abs bool = true
	for i, vv := range v {
		if !math.IsNaN(vv) && !absent[i] {
			sum += vv
			abs = false
		}
	}
	return sum, abs
}
<|endoftext|>


<commit_before>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package store

import (
	"fmt"
	"path"
	"strings"
	"sync"

	etcdErr "github.com/coreos/etcd/error"
)

type EventHistory struct {
	Queue      eventQueue
	StartIndex uint64
	LastIndex  uint64
	rwl        sync.RWMutex
}

func newEventHistory(capacity int) *EventHistory {
	return &EventHistory{
		Queue: eventQueue{
			Capacity: capacity,
			Events:   make([]*Event, capacity),
		},
	}
}

// addEvent function adds event into the eventHistory
func (eh *EventHistory) addEvent(e *Event) *Event {
	eh.rwl.Lock()
	defer eh.rwl.Unlock()

	eh.Queue.insert(e)

	eh.LastIndex = e.Index()

	eh.StartIndex = eh.Queue.Events[eh.Queue.Front].Index()

	return e
}

// scan enumerates events from the index history and stops at the first point
// where the key matches.
func (eh *EventHistory) scan(key string, recursive bool, index uint64) (*Event, *etcdErr.Error) {
	eh.rwl.RLock()
	defer eh.rwl.RUnlock()

	// index should be after the event history's StartIndex
	if index < eh.StartIndex {
		return nil,
			etcdErr.NewError(etcdErr.EcodeEventIndexCleared,
				fmt.Sprintf("the requested history has been cleared [%v/%v]",
					eh.StartIndex, index), 0)
	}

	// the index should come before the size of the queue minus the duplicate count
	if index > eh.LastIndex { // future index
		return nil, nil
	}

	offset := index - eh.StartIndex
	i := (eh.Queue.Front + int(offset)) % eh.Queue.Capacity

	for {
		e := eh.Queue.Events[i]

		ok := (e.Node.Key == key)

		if recursive {
			// add tailing slash
			key := path.Clean(key)
			if key[len(key)-1] != '/' {
				key = key + "/"
			}

			ok = ok || strings.HasPrefix(e.Node.Key, key)
		}

		if ok {
			return e, nil
		}

		i = (i + 1) % eh.Queue.Capacity

		if i == eh.Queue.Back {
			return nil, nil
		}
	}
}

// clone will be protected by a stop-world lock
// do not need to obtain internal lock
func (eh *EventHistory) clone() *EventHistory {
	clonedQueue := eventQueue{
		Capacity: eh.Queue.Capacity,
		Events:   make([]*Event, eh.Queue.Capacity),
		Size:     eh.Queue.Size,
		Front:    eh.Queue.Front,
		Back:     eh.Queue.Back,
	}

	for i, e := range eh.Queue.Events {
		clonedQueue.Events[i] = e
	}

	return &EventHistory{
		StartIndex: eh.StartIndex,
		Queue:      clonedQueue,
		LastIndex:  eh.LastIndex,
	}

}
<commit_msg>store: fixes shadowed variables with go tool vet.<commit_after>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package store

import (
	"fmt"
	"path"
	"strings"
	"sync"

	etcdErr "github.com/coreos/etcd/error"
)

type EventHistory struct {
	Queue      eventQueue
	StartIndex uint64
	LastIndex  uint64
	rwl        sync.RWMutex
}

func newEventHistory(capacity int) *EventHistory {
	return &EventHistory{
		Queue: eventQueue{
			Capacity: capacity,
			Events:   make([]*Event, capacity),
		},
	}
}

// addEvent function adds event into the eventHistory
func (eh *EventHistory) addEvent(e *Event) *Event {
	eh.rwl.Lock()
	defer eh.rwl.Unlock()

	eh.Queue.insert(e)

	eh.LastIndex = e.Index()

	eh.StartIndex = eh.Queue.Events[eh.Queue.Front].Index()

	return e
}

// scan enumerates events from the index history and stops at the first point
// where the key matches.
func (eh *EventHistory) scan(key string, recursive bool, index uint64) (*Event, *etcdErr.Error) {
	eh.rwl.RLock()
	defer eh.rwl.RUnlock()

	// index should be after the event history's StartIndex
	if index < eh.StartIndex {
		return nil,
			etcdErr.NewError(etcdErr.EcodeEventIndexCleared,
				fmt.Sprintf("the requested history has been cleared [%v/%v]",
					eh.StartIndex, index), 0)
	}

	// the index should come before the size of the queue minus the duplicate count
	if index > eh.LastIndex { // future index
		return nil, nil
	}

	offset := index - eh.StartIndex
	i := (eh.Queue.Front + int(offset)) % eh.Queue.Capacity

	for {
		e := eh.Queue.Events[i]

		ok := (e.Node.Key == key)

		if recursive {
			// add tailing slash
			key = path.Clean(key)
			if key[len(key)-1] != '/' {
				key = key + "/"
			}

			ok = ok || strings.HasPrefix(e.Node.Key, key)
		}

		if ok {
			return e, nil
		}

		i = (i + 1) % eh.Queue.Capacity

		if i == eh.Queue.Back {
			return nil, nil
		}
	}
}

// clone will be protected by a stop-world lock
// do not need to obtain internal lock
func (eh *EventHistory) clone() *EventHistory {
	clonedQueue := eventQueue{
		Capacity: eh.Queue.Capacity,
		Events:   make([]*Event, eh.Queue.Capacity),
		Size:     eh.Queue.Size,
		Front:    eh.Queue.Front,
		Back:     eh.Queue.Back,
	}

	for i, e := range eh.Queue.Events {
		clonedQueue.Events[i] = e
	}

	return &EventHistory{
		StartIndex: eh.StartIndex,
		Queue:      clonedQueue,
		LastIndex:  eh.LastIndex,
	}

}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/fujiwara/fluent-agent-hydra/fluent"
	"github.com/fujiwara/fluent-agent-hydra/ltsv"
)

var (
	MessageCh        = make(map[string]map[int64]chan fluent.FluentRecordSet)
	MessageBufferLen = 100
	MessageChMutex   sync.Mutex
	ConnectionId     int64
	Counter          int64
)

type OutputOption struct {
	IncludeTime bool
	IncludeTag  bool
}

type Encoder interface {
	Encode(interface{}) error
}

func main() {
	var (
		forwardPort   int
		httpPort      int
		countInterval int
	)
	flag.IntVar(&forwardPort, "forward-port", 24224, "fluentd forward listen port")
	flag.IntVar(&httpPort, "http-port", 24225, "http listen port")
	flag.IntVar(&countInterval, "count-interval", 60, "log counter output interval(sec)")
	flag.Parse()

	go runReporter(time.Duration(countInterval) * time.Second)

	var err error
	err = runForwardServer(forwardPort)
	if err != nil {
		log.Fatal(err)
	}
	err = runHTTPServer(httpPort)
	if err != nil {
		log.Fatal(err)
	}
}

func runReporter(t time.Duration) {
	ticker := time.Tick(t)
	for _ = range ticker {
		c := atomic.SwapInt64(&Counter, 0)
		if c > 0 {
			log.Println("count:", c, "cps:", float64(c)/float64(t/time.Second))
		}
	}
}

func runForwardServer(port int) error {
	addr := fmt.Sprintf(":%d", port)
	l, err := net.Listen("tcp", addr)
	if err != nil {
		return err
	}
	log.Println("[info] forward server listing", l.Addr())
	go func() {
		for {
			conn, err := l.Accept()
			if err != nil {
				log.Println("[error] accept error", err)
			}
			go handleForwardConn(conn)
		}
	}()
	return nil
}

func handleForwardConn(conn net.Conn) {
	defer conn.Close()
	for {
		recordSets, err := fluent.DecodeEntries(conn)
		if err == io.EOF {
			return
		} else if err != nil {
			log.Println("decode entries failed", err, conn.LocalAddr())
			return
		}
		for _, recordSet := range recordSets {
			atomic.AddInt64(&Counter, int64(len(recordSet.Records)))
			for tag, channels := range MessageCh {
				if !matchTag(tag, recordSet.Tag) {
					continue
				}
				for _, ch := range channels {
					select {
					case ch <- recordSet:
					default:
						log.Printf("[warn] %d records dropped for http client.", len(recordSet.Records))
					}
				}
			}
		}
	}
}

func runHTTPServer(port int) error {
	http.HandleFunc("/", httpHandler)
	addr := fmt.Sprintf(":%d", port)
	log.Println("[info] http server listing", addr)
	return http.ListenAndServe(addr, nil)
}

func httpHandler(w http.ResponseWriter, r *http.Request) {
	tag := strings.Trim(r.URL.Path, "/")
	if tag == "" {
		http.NotFound(w, r)
		return
	}

	option := OutputOption{false, false}
	option.IncludeTime, _ = strconv.ParseBool(r.FormValue("time"))
	option.IncludeTag, _ = strconv.ParseBool(r.FormValue("tag"))
	var encoder Encoder
	switch t, _ := strconv.ParseBool(r.FormValue("ltsv")); t {
	case true:
		encoder = ltsv.NewEncoder(w)
	default:
		encoder = json.NewEncoder(w)
	}

	id := atomic.AddInt64(&ConnectionId, 1)
	ch := subscribe(tag, id)
	defer unsubscribe(tag, id)

	log.Printf(
		"[info] client %s tag:%s",
		r.RemoteAddr,
		tag,
	)

	w.WriteHeader(http.StatusOK)
	if f, ok := w.(http.Flusher); ok {
		f.Flush()
	}
	for {
		recordSet := <-ch
		err := writeResponse(encoder, w, recordSet, option)
		if err != nil {
			return
		}
		if f, ok := w.(http.Flusher); ok {
			f.Flush()
		}
	}
	return
}

func matchTag(matcher, tag string) bool {
	matcher = strings.TrimRight(matcher, ".*")
	return strings.HasPrefix(tag, matcher)
}

func subscribe(tag string, id int64) chan fluent.FluentRecordSet {
	MessageChMutex.Lock()
	defer MessageChMutex.Unlock()

	if m := MessageCh[tag]; m == nil {
		MessageCh[tag] = make(map[int64]chan fluent.FluentRecordSet)
	}
	ch := make(chan fluent.FluentRecordSet, MessageBufferLen)
	MessageCh[tag][id] = ch
	return ch
}

func unsubscribe(tag string, id int64) {
	MessageChMutex.Lock()
	defer MessageChMutex.Unlock()

	delete(MessageCh[tag], id)
}

func writeResponse(encoder Encoder, w http.ResponseWriter, rs fluent.FluentRecordSet, option OutputOption) error {
	for _, record := range rs.Records {
		record, _ := record.(*fluent.TinyFluentRecord)
		data := record.GetAllData()
		if option.IncludeTime {
			fmt.Fprint(w, time.Unix(record.Timestamp, 0).Format(time.RFC3339), "\t")
		}
		if option.IncludeTag {
			fmt.Fprint(w, rs.Tag, "\t")
		}
		err := encoder.Encode(data)
		if err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>add file dumper<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/fujiwara/fluent-agent-hydra/fluent"
	"github.com/fujiwara/fluent-agent-hydra/ltsv"
)

var (
	MessageCh        = make(map[string]map[int64]chan fluent.FluentRecordSet)
	MessageBufferLen = 100
	MessageChMutex   sync.Mutex
	ConnectionId     int64
	Counter          int64
	DumpCh           chan fluent.FluentRecordSet
)

type OutputOption struct {
	IncludeTime bool
	IncludeTag  bool
}

type Encoder interface {
	Encode(interface{}) error
}

func main() {
	var (
		forwardPort     int
		httpPort        int
		countInterval   int
		dir             string
		dumpIncludeTag  bool
		dumpIncludeTime bool
		dumpFileFormat  string
	)
	flag.IntVar(&forwardPort, "forward-port", 24224, "fluentd forward listen port")
	flag.IntVar(&httpPort, "http-port", 24225, "http listen port")
	flag.IntVar(&countInterval, "count-interval", 60, "log counter output interval(sec)")
	flag.StringVar(&dir, "dump-file-dir", "", "dump file directory")
	flag.BoolVar(&dumpIncludeTag, "dump-include-tag", false, "dump file include tag")
	flag.BoolVar(&dumpIncludeTime, "dump-include-time", false, "dump file include time")
	flag.StringVar(&dumpFileFormat, "dump-file-format", "json", "dump file format")
	flag.Parse()

	go runReporter(time.Duration(countInterval) * time.Second)
	if dir != "" {
		DumpCh = make(chan fluent.FluentRecordSet, MessageBufferLen)
		option := OutputOption{dumpIncludeTime, dumpIncludeTag}
		go runDumper(dir, dumpFileFormat, option)
	}
	var err error
	err = runForwardServer(forwardPort)
	if err != nil {
		log.Fatal(err)
	}
	err = runHTTPServer(httpPort)
	if err != nil {
		log.Fatal(err)
	}
}

func runDumper(dir string, format string, option OutputOption) {
	var filename string
	var fh io.WriteCloser
	var err error
	for {
		rs := <-DumpCh
		now := time.Now().Format("2006-01-02-15")
		_filename := filepath.Join(dir, rs.Tag+"."+now)
		if filename != _filename {
			if fh != nil {
				fh.Close()
			}
			filename = _filename
			fh, err = os.OpenFile(filename, os.O_CREATE|os.O_APPEND|os.O_RDWR, 0644)
			defer fh.Close()
			if err != nil {
				log.Println("[error]", err)
				continue
			}
		}
		if fh == nil {
			continue
		}
		var encoder Encoder
		switch format {
		case "ltsv":
			encoder = ltsv.NewEncoder(fh)
		default:
			encoder = json.NewEncoder(fh)
		}
		for _, record := range rs.Records {
			record, _ := record.(*fluent.TinyFluentRecord)
			if option.IncludeTime {
				fmt.Fprint(fh, time.Unix(record.Timestamp, 0).Format(time.RFC3339), "\t")
			}
			if option.IncludeTag {
				fmt.Fprint(fh, rs.Tag, "\t")
			}
			err = encoder.Encode(record.GetAllData())
			if err != nil {
				continue
			}
		}
	}
}

func runReporter(t time.Duration) {
	ticker := time.Tick(t)
	for _ = range ticker {
		c := atomic.SwapInt64(&Counter, 0)
		if c > 0 {
			log.Println("count:", c, "cps:", float64(c)/float64(t/time.Second))
		}
	}
}

func runForwardServer(port int) error {
	addr := fmt.Sprintf(":%d", port)
	l, err := net.Listen("tcp", addr)
	if err != nil {
		return err
	}
	log.Println("[info] forward server listing", l.Addr())
	go func() {
		for {
			conn, err := l.Accept()
			if err != nil {
				log.Println("[error] accept error", err)
			}
			go handleForwardConn(conn)
		}
	}()
	return nil
}

func handleForwardConn(conn net.Conn) {
	defer conn.Close()
	for {
		recordSets, err := fluent.DecodeEntries(conn)
		if err == io.EOF {
			return
		} else if err != nil {
			log.Println("decode entries failed", err, conn.LocalAddr())
			return
		}
		for _, recordSet := range recordSets {
			atomic.AddInt64(&Counter, int64(len(recordSet.Records)))
			if DumpCh != nil {
				select {
				case DumpCh <- recordSet:
				default:
					log.Printf("[warn] %d records dropped for dump file. tag: %s", len(recordSet.Records), recordSet.Tag)
				}
			}
			for tag, channels := range MessageCh {
				if !matchTag(tag, recordSet.Tag) {
					continue
				}
				for _, ch := range channels {
					select {
					case ch <- recordSet:
					default:
						log.Printf("[warn] %d records dropped for http client.", len(recordSet.Records))
					}
				}
			}
		}
	}
}

func runHTTPServer(port int) error {
	http.HandleFunc("/", httpHandler)
	addr := fmt.Sprintf(":%d", port)
	log.Println("[info] http server listing", addr)
	return http.ListenAndServe(addr, nil)
}

func httpHandler(w http.ResponseWriter, r *http.Request) {
	tag := strings.Trim(r.URL.Path, "/")
	if tag == "" {
		http.NotFound(w, r)
		return
	}

	option := OutputOption{false, false}
	option.IncludeTime, _ = strconv.ParseBool(r.FormValue("time"))
	option.IncludeTag, _ = strconv.ParseBool(r.FormValue("tag"))
	var encoder Encoder
	switch t, _ := strconv.ParseBool(r.FormValue("ltsv")); t {
	case true:
		encoder = ltsv.NewEncoder(w)
	default:
		encoder = json.NewEncoder(w)
	}

	id := atomic.AddInt64(&ConnectionId, 1)
	ch := subscribe(tag, id)
	defer unsubscribe(tag, id)

	log.Printf(
		"[info] client %s tag:%s",
		r.RemoteAddr,
		tag,
	)

	w.WriteHeader(http.StatusOK)
	if f, ok := w.(http.Flusher); ok {
		f.Flush()
	}
	for {
		recordSet := <-ch
		err := writeResponse(encoder, w, recordSet, option)
		if err != nil {
			return
		}
		if f, ok := w.(http.Flusher); ok {
			f.Flush()
		}
	}
	return
}

func matchTag(matcher, tag string) bool {
	matcher = strings.TrimRight(matcher, ".*")
	return strings.HasPrefix(tag, matcher)
}

func subscribe(tag string, id int64) chan fluent.FluentRecordSet {
	MessageChMutex.Lock()
	defer MessageChMutex.Unlock()

	if m := MessageCh[tag]; m == nil {
		MessageCh[tag] = make(map[int64]chan fluent.FluentRecordSet)
	}
	ch := make(chan fluent.FluentRecordSet, MessageBufferLen)
	MessageCh[tag][id] = ch
	return ch
}

func unsubscribe(tag string, id int64) {
	MessageChMutex.Lock()
	defer MessageChMutex.Unlock()

	delete(MessageCh[tag], id)
}

func writeResponse(encoder Encoder, w http.ResponseWriter, rs fluent.FluentRecordSet, option OutputOption) error {
	for _, record := range rs.Records {
		record, _ := record.(*fluent.TinyFluentRecord)
		data := record.GetAllData()
		if option.IncludeTime {
			fmt.Fprint(w, time.Unix(record.Timestamp, 0).Format(time.RFC3339), "\t")
		}
		if option.IncludeTag {
			fmt.Fprint(w, rs.Tag, "\t")
		}
		err := encoder.Encode(data)
		if err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright (c) nano Author. All Rights Reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

package codec

import (
	"bytes"
	"errors"

	"github.com/lonnng/nano/internal/packet"
)

// Codec constants.
const (
	HeadLength    = 4
	MaxPacketSize = 64 * 1024
)

// ErrPacketSizeExcced is the error used for encode/decode.
var ErrPacketSizeExcced = errors.New("codec: packet size exceed")

// A Decoder reads and decodes network data slice
type Decoder struct {
	buf  *bytes.Buffer
	size int  // last packet length
	typ  byte // last packet type
}

// NewDecoder returns a new decoder that used for decode network data slice.
func NewDecoder() *Decoder {
	return &Decoder{
		buf:  bytes.NewBuffer(nil),
		size: -1,
	}
}

// Decode decode the network data slice to packet.Packet(s)
// TODO(Warning): shared slice
func (c *Decoder) Decode(data []byte) ([]*packet.Packet, error) {
	c.buf.Write(data)

	var (
		packets []*packet.Packet
		err     error
	)
	// check length
	if c.buf.Len() < HeadLength {
		return nil, err
	}

	f := func() error {
		header := c.buf.Next(HeadLength)
		c.typ = header[0]
		if c.typ < packet.Handshake || c.typ > packet.Kick {
			return packet.ErrWrongPacketType
		}
		c.size = bytesToInt(header[1:])

		return nil

	}

	// first time
	if c.size < 0 {
		err = f()
		if err != nil {
			return nil, err
		}
	}

	// packet length limitation
	if c.size > MaxPacketSize {
		return packets, ErrPacketSizeExcced
	}

	for c.size <= c.buf.Len() {
		p := &packet.Packet{Type: packet.Type(c.typ), Length: c.size, Data: c.buf.Next(c.size)}
		packets = append(packets, p)

		// more packet
		if c.buf.Len() < HeadLength {
			c.size = -1
			break
		}

		err = f()
		if err != nil {
			return nil, err
		}

		if c.size > MaxPacketSize {
			return packets, ErrPacketSizeExcced
		}

	}

	return packets, nil
}

// Encode create a packet.Packet from  the raw data and then encode to network data slice
// Protocol refs: https://github.com/NetEase/pomelo/wiki/Communication-Protocol
//
// -<type>-|--------<length>--------|-<data>-
// --------|------------------------|--------
// 1 byte packet type, 3 bytes packet data length(big end), and data segment
func Encode(typ packet.Type, data []byte) ([]byte, error) {
	if typ < packet.Handshake || typ > packet.Kick {
		return nil, packet.ErrWrongPacketType
	}

	p := &packet.Packet{Type: typ, Length: len(data)}
	buf := make([]byte, p.Length+HeadLength)
	buf[0] = byte(p.Type)

	copy(buf[1:HeadLength], intToBytes(p.Length))
	copy(buf[HeadLength:], data)

	return buf, nil
}

// Decode packet data length byte to int(Big end)
func bytesToInt(b []byte) int {
	result := 0
	for _, v := range b {
		result = result<<8 + int(v)
	}
	return result
}

// Encode packet data length to bytes(Big end)
func intToBytes(n int) []byte {
	buf := make([]byte, 3)
	buf[0] = byte((n >> 16) & 0xFF)
	buf[1] = byte((n >> 8) & 0xFF)
	buf[2] = byte(n & 0xFF)
	return buf
}
<commit_msg>refactor: tweak code style<commit_after>// Copyright (c) nano Author. All Rights Reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

package codec

import (
	"bytes"
	"errors"

	"github.com/lonnng/nano/internal/packet"
)

// Codec constants.
const (
	HeadLength    = 4
	MaxPacketSize = 64 * 1024
)

// ErrPacketSizeExcced is the error used for encode/decode.
var ErrPacketSizeExcced = errors.New("codec: packet size exceed")

// A Decoder reads and decodes network data slice
type Decoder struct {
	buf  *bytes.Buffer
	size int  // last packet length
	typ  byte // last packet type
}

// NewDecoder returns a new decoder that used for decode network data slice.
func NewDecoder() *Decoder {
	return &Decoder{
		buf:  bytes.NewBuffer(nil),
		size: -1,
	}
}

// Decode decode the network data slice to packet.Packet(s)
// TODO(Warning): shared slice
func (c *Decoder) Decode(data []byte) ([]*packet.Packet, error) {
	c.buf.Write(data)

	var (
		packets []*packet.Packet
		err     error
	)
	// check length
	if c.buf.Len() < HeadLength {
		return nil, err
	}

	f := func() error {
		header := c.buf.Next(HeadLength)
		c.typ = header[0]
		if c.typ < packet.Handshake || c.typ > packet.Kick {
			return packet.ErrWrongPacketType
		}
		c.size = bytesToInt(header[1:])

		return nil

	}

	// first time
	if c.size < 0 {
		if err = f(); err != nil {
			return nil, err
		}
	}

	// packet length limitation
	if c.size > MaxPacketSize {
		return packets, ErrPacketSizeExcced
	}

	for c.size <= c.buf.Len() {
		p := &packet.Packet{Type: packet.Type(c.typ), Length: c.size, Data: c.buf.Next(c.size)}
		packets = append(packets, p)

		// more packet
		if c.buf.Len() < HeadLength {
			c.size = -1
			break
		}

		if err = f(); err != nil {
			return nil, err
		}

		if c.size > MaxPacketSize {
			return packets, ErrPacketSizeExcced
		}

	}

	return packets, nil
}

// Encode create a packet.Packet from  the raw data and then encode to network data slice
// Protocol refs: https://github.com/NetEase/pomelo/wiki/Communication-Protocol
//
// -<type>-|--------<length>--------|-<data>-
// --------|------------------------|--------
// 1 byte packet type, 3 bytes packet data length(big end), and data segment
func Encode(typ packet.Type, data []byte) ([]byte, error) {
	if typ < packet.Handshake || typ > packet.Kick {
		return nil, packet.ErrWrongPacketType
	}

	p := &packet.Packet{Type: typ, Length: len(data)}
	buf := make([]byte, p.Length+HeadLength)
	buf[0] = byte(p.Type)

	copy(buf[1:HeadLength], intToBytes(p.Length))
	copy(buf[HeadLength:], data)

	return buf, nil
}

// Decode packet data length byte to int(Big end)
func bytesToInt(b []byte) int {
	result := 0
	for _, v := range b {
		result = result<<8 + int(v)
	}
	return result
}

// Encode packet data length to bytes(Big end)
func intToBytes(n int) []byte {
	buf := make([]byte, 3)
	buf[0] = byte((n >> 16) & 0xFF)
	buf[1] = byte((n >> 8) & 0xFF)
	buf[2] = byte(n & 0xFF)
	return buf
}
<|endoftext|>


<commit_before>package piglowservice

import (
	"log"

	"fmt"

	pb "github.com/didrocks/grpc-piglow/proto"
	context "golang.org/x/net/context"
)

func (s *service) SetLED(ctx context.Context, in *pb.LedRequest) (*pb.Ack, error) {
	var err error

	n, err := ensureNumLed(in.Num)
	if err != nil {
		return nil, err
	}
	b, err := ensureBrightness(in.Brightness)
	if err != nil {
		return nil, err
	}

	s.p.SetLED(n, b)
	if err = s.p.Apply(); err != nil { // Apply the changes
		log.Println("Couldn't apply changes: ", err)
	}
	return &pb.Ack{Ok: true}, err
}

// convert and ensure num led is valid
func ensureNumLed(n int32) (int8, error) {
	if n < 0 || n > 17 {
		return 0, fmt.Errorf("invalid led number: %d", n)
	}
	return int8(n), nil
}

// convert and ensure brightness is valid
func ensureBrightness(b uint32) (uint8, error) {
	if b > 255 {
		return 0, fmt.Errorf("invalid brightness value: %d", b)
	}
	return uint8(b), nil
}
<commit_msg>Factorize apply function<commit_after>package piglowservice

import (
	"log"

	"fmt"

	pb "github.com/didrocks/grpc-piglow/proto"
	context "golang.org/x/net/context"
)

// Set LED n to brightness
func (s *service) SetLED(ctx context.Context, in *pb.LedRequest) (*pb.Ack, error) {
	var err error

	n, err := ensureNumLed(in.Num)
	if err != nil {
		return nil, err
	}
	b, err := ensureBrightness(in.Brightness)
	if err != nil {
		return nil, err
	}

	s.p.SetLED(n, b)
	return s.apply()
}

// convert and ensure num led is valid
func ensureNumLed(n int32) (int8, error) {
	if n < 0 || n > 17 {
		return 0, fmt.Errorf("invalid led number: %d", n)
	}
	return int8(n), nil
}

// convert and ensure brightness is valid
func ensureBrightness(b uint32) (uint8, error) {
	if b > 255 {
		return 0, fmt.Errorf("invalid brightness value: %d", b)
	}
	return uint8(b), nil
}

// internal apply correct changes functions
func (s *service) apply() (ack *pb.Ack, err error) {
	ack = &pb.Ack{Ok: true}
	if err = s.p.Apply(); err != nil {
		// server side logging
		log.Println("Couldn't apply changes: ", err)
	}
	return ack, nil
}
<|endoftext|>


<commit_before>package scanner_test

import (
	. "github.com/afeld/tangle/scanner"

	"net/http"
	"net/url"

	. "github.com/afeld/tangle/helpers"
	. "github.com/afeld/tangle/models"
	"github.com/jarcoal/httpmock"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

func registerResponse(dest string, status int) {
	responder := httpmock.NewStringResponder(status, "")
	httpmock.RegisterResponder("HEAD", dest, responder)
}

func registerLink(dest string, status int) Link {
	registerResponse(dest, status)
	return CreateLink(dest)
}

var _ = Describe("Scanner", func() {
	Describe("ScanLinks", func() {
		It("returns the number of broken links", func() {
			dest1 := registerLink("http://ok.com", 200)
			dest2 := registerLink("http://not-ok.com", 404)
			links := []Link{dest1, dest2}

			resultByLink := ScanLinks(links)

			Expect(len(resultByLink)).To(Equal(2))
			Expect(resultByLink[dest1]).To(BeTrue())
			Expect(resultByLink[dest2]).To(BeFalse())
		})

		It("only checks each URL once", func() {
			hits := 0
			responder := func(req *http.Request) (*http.Response, error) {
				hits++
				return httpmock.NewStringResponse(200, ""), nil
			}
			httpmock.RegisterResponder("HEAD", "http://ok.com", responder)

			// two identical links that are different tags
			dest1 := CreateLink("http://ok.com")
			dest2 := CreateLink("http://ok.com")
			Expect(dest1).ToNot(Equal(dest2))
			links := []Link{dest1, dest2}

			resultByLink := ScanLinks(links)

			Expect(len(resultByLink)).To(Equal(2))
			Expect(resultByLink[dest1]).To(BeTrue())
			Expect(resultByLink[dest2]).To(BeTrue())
			Expect(hits).To(Equal(1))
		})
	})

	Describe("ScanPage", func() {
		It("returns the result for each link", func() {
			registerResponse("http://ok.com", 200)
			registerResponse("http://not-ok.com", 404)

			source, _ := url.Parse("http://source.com")
			responder := httpmock.NewStringResponder(200, `

			`)
			httpmock.RegisterResponder("GET", "http://source.com", responder)

			resultByLink, err := ScanPage(source)
			Expect(err).ToNot(HaveOccurred())
			Expect(len(resultByLink)).To(Equal(3))
		})
	})
})
<commit_msg>add checks for all results from ScanPage in the test<commit_after>package scanner_test

import (
	. "github.com/afeld/tangle/scanner"

	"net/http"
	"net/url"

	. "github.com/afeld/tangle/helpers"
	. "github.com/afeld/tangle/models"
	"github.com/jarcoal/httpmock"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

func registerResponse(dest string, status int) {
	responder := httpmock.NewStringResponder(status, "")
	httpmock.RegisterResponder("HEAD", dest, responder)
}

func registerLink(dest string, status int) Link {
	registerResponse(dest, status)
	return CreateLink(dest)
}

var _ = Describe("Scanner", func() {
	Describe("ScanLinks", func() {
		It("returns the number of broken links", func() {
			dest1 := registerLink("http://ok.com", 200)
			dest2 := registerLink("http://not-ok.com", 404)
			links := []Link{dest1, dest2}

			resultByLink := ScanLinks(links)

			Expect(len(resultByLink)).To(Equal(2))
			Expect(resultByLink[dest1]).To(BeTrue())
			Expect(resultByLink[dest2]).To(BeFalse())
		})

		It("only checks each URL once", func() {
			hits := 0
			responder := func(req *http.Request) (*http.Response, error) {
				hits++
				return httpmock.NewStringResponse(200, ""), nil
			}
			httpmock.RegisterResponder("HEAD", "http://ok.com", responder)

			// two identical links that are different tags
			dest1 := CreateLink("http://ok.com")
			dest2 := CreateLink("http://ok.com")
			Expect(dest1).ToNot(Equal(dest2))
			links := []Link{dest1, dest2}

			resultByLink := ScanLinks(links)

			Expect(len(resultByLink)).To(Equal(2))
			Expect(resultByLink[dest1]).To(BeTrue())
			Expect(resultByLink[dest2]).To(BeTrue())
			Expect(hits).To(Equal(1))
		})
	})

	Describe("ScanPage", func() {
		It("returns the result for each link", func() {
			registerResponse("http://ok.com", 200)
			registerResponse("http://not-ok.com", 404)

			sourceStr := "http://source.com"
			source, _ := url.Parse(sourceStr)
			responder := httpmock.NewStringResponder(200, `

			`)
			httpmock.RegisterResponder("GET", sourceStr, responder)

			resultByLink, err := ScanPage(source)

			Expect(err).ToNot(HaveOccurred())
			Expect(len(resultByLink)).To(Equal(3))
			for link, result := range resultByLink {
				dest, _ := link.DestURL()
				if dest.Host == "ok.com" {
					Expect(result).To(BeTrue())
				} else {
					Expect(result).To(BeFalse())
				}
			}
		})
	})
})
<|endoftext|>


<commit_before>// Copyright 2020 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package tikv

import (
	"bytes"
	"math"
	"sync/atomic"
	"time"

	"github.com/opentracing/opentracing-go"
	"github.com/pingcap/errors"
	"github.com/pingcap/failpoint"
	pb "github.com/pingcap/kvproto/pkg/kvrpcpb"
	"github.com/pingcap/tidb/config"
	"github.com/pingcap/tidb/metrics"
	"github.com/pingcap/tidb/store/tikv/tikvrpc"
	"github.com/pingcap/tidb/util/logutil"
	"github.com/prometheus/client_golang/prometheus"
	"go.uber.org/zap"
)

type actionPrewrite struct{}

var _ twoPhaseCommitAction = actionPrewrite{}
var tiKVTxnRegionsNumHistogramPrewrite = metrics.TiKVTxnRegionsNumHistogram.WithLabelValues(metricsTag("prewrite"))

func (actionPrewrite) String() string {
	return "prewrite"
}

func (actionPrewrite) tiKVTxnRegionsNumHistogram() prometheus.Observer {
	return tiKVTxnRegionsNumHistogramPrewrite
}

func (c *twoPhaseCommitter) buildPrewriteRequest(batch batchMutations, txnSize uint64) *tikvrpc.Request {
	m := &batch.mutations
	mutations := make([]*pb.Mutation, m.len())
	for i := range m.keys {
		mutations[i] = &pb.Mutation{
			Op:    m.ops[i],
			Key:   m.keys[i],
			Value: m.values[i],
		}
	}
	var minCommitTS uint64
	if c.forUpdateTS > 0 {
		minCommitTS = c.forUpdateTS + 1
	} else {
		minCommitTS = c.startTS + 1
	}

	failpoint.Inject("mockZeroCommitTS", func(val failpoint.Value) {
		// Should be val.(uint64) but failpoint doesn't support that.
		if tmp, ok := val.(int); ok && uint64(tmp) == c.startTS {
			minCommitTS = 0
		}
	})

	req := &pb.PrewriteRequest{
		Mutations:         mutations,
		PrimaryLock:       c.primary(),
		StartVersion:      c.startTS,
		LockTtl:           c.lockTTL,
		IsPessimisticLock: m.isPessimisticLock,
		ForUpdateTs:       c.forUpdateTS,
		TxnSize:           txnSize,
		MinCommitTs:       minCommitTS,
	}

	if c.isAsyncCommit() {
		if batch.isPrimary {
			req.Secondaries = c.asyncSecondaries()
		}
		req.UseAsyncCommit = true
	}

	return tikvrpc.NewRequest(tikvrpc.CmdPrewrite, req, pb.Context{Priority: c.priority, SyncLog: c.syncLog})
}

func (action actionPrewrite) handleSingleBatch(c *twoPhaseCommitter, bo *Backoffer, batch batchMutations) error {
	txnSize := uint64(c.regionTxnSize[batch.region.id])
	// When we retry because of a region miss, we don't know the transaction size. We set the transaction size here
	// to MaxUint64 to avoid unexpected "resolve lock lite".
	if len(bo.errors) > 0 {
		txnSize = math.MaxUint64
	}

	req := c.buildPrewriteRequest(batch, txnSize)
	for {
		resp, err := c.store.SendReq(bo, req, batch.region, readTimeoutShort)
		if err != nil {
			return errors.Trace(err)
		}
		regionErr, err := resp.GetRegionError()
		if err != nil {
			return errors.Trace(err)
		}
		if regionErr != nil {
			err = bo.Backoff(BoRegionMiss, errors.New(regionErr.String()))
			if err != nil {
				return errors.Trace(err)
			}
			err = c.prewriteMutations(bo, batch.mutations)
			return errors.Trace(err)
		}
		if resp.Resp == nil {
			return errors.Trace(ErrBodyMissing)
		}
		prewriteResp := resp.Resp.(*pb.PrewriteResponse)
		keyErrs := prewriteResp.GetErrors()
		if len(keyErrs) == 0 {
			if bytes.Equal(c.primary(), batch.mutations.keys[0]) {
				// After writing the primary key, if the size of the transaction is large than 32M,
				// start the ttlManager. The ttlManager will be closed in tikvTxn.Commit().
				if int64(c.txnSize) > config.GetGlobalConfig().TiKVClient.TTLRefreshedTxnSize {
					c.run(c, nil)
				}
			}
			if c.isAsyncCommit() {
				// 0 if the min_commit_ts is not ready or any other reason that async
				// commit cannot proceed. The client can then fallback to normal way to
				// continue committing the transaction if prewrite are all finished.
				if prewriteResp.MinCommitTs == 0 {
					logutil.Logger(bo.ctx).Warn("async commit cannot proceed since the returned minCommitTS is zero, "+
						"fallback to normal path", zap.Uint64("startTS", c.startTS))
					c.setAsyncCommit(false)
				} else {
					c.mu.Lock()
					if prewriteResp.MinCommitTs > c.minCommitTS {
						c.minCommitTS = prewriteResp.MinCommitTs
					}
					c.mu.Unlock()
				}
			}
			return nil
		}
		var locks []*Lock
		for _, keyErr := range keyErrs {
			// Check already exists error
			if alreadyExist := keyErr.GetAlreadyExist(); alreadyExist != nil {
				key := alreadyExist.GetKey()
				return c.extractKeyExistsErr(key)
			}

			// Extract lock from key error
			lock, err1 := extractLockFromKeyErr(keyErr)
			if err1 != nil {
				return errors.Trace(err1)
			}
			logutil.BgLogger().Info("prewrite encounters lock",
				zap.Uint64("conn", c.connID),
				zap.Stringer("lock", lock))
			locks = append(locks, lock)
		}
		start := time.Now()
		msBeforeExpired, err := c.store.lockResolver.resolveLocksForWrite(bo, c.startTS, locks)
		if err != nil {
			return errors.Trace(err)
		}
		atomic.AddInt64(&c.getDetail().ResolveLockTime, int64(time.Since(start)))
		if msBeforeExpired > 0 {
			err = bo.BackoffWithMaxSleep(BoTxnLock, int(msBeforeExpired), errors.Errorf("2PC prewrite lockedKeys: %d", len(locks)))
			if err != nil {
				return errors.Trace(err)
			}
		}
	}
}

func (c *twoPhaseCommitter) prewriteMutations(bo *Backoffer, mutations committerMutations) error {
	if span := opentracing.SpanFromContext(bo.ctx); span != nil && span.Tracer() != nil {
		span1 := span.Tracer().StartSpan("twoPhaseCommitter.prewriteMutations", opentracing.ChildOf(span.Context()))
		defer span1.Finish()
		bo.ctx = opentracing.ContextWithSpan(bo.ctx, span1)
	}

	return c.doActionOnMutations(bo, actionPrewrite{}, mutations)
}
<commit_msg>txn:set min commit ts for prewrite build for async commit protocol (#18759)<commit_after>// Copyright 2020 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package tikv

import (
	"bytes"
	"math"
	"sync/atomic"
	"time"

	"github.com/opentracing/opentracing-go"
	"github.com/pingcap/errors"
	"github.com/pingcap/failpoint"
	pb "github.com/pingcap/kvproto/pkg/kvrpcpb"
	"github.com/pingcap/tidb/config"
	"github.com/pingcap/tidb/metrics"
	"github.com/pingcap/tidb/store/tikv/tikvrpc"
	"github.com/pingcap/tidb/util/logutil"
	"github.com/prometheus/client_golang/prometheus"
	"go.uber.org/zap"
)

type actionPrewrite struct{}

var _ twoPhaseCommitAction = actionPrewrite{}
var tiKVTxnRegionsNumHistogramPrewrite = metrics.TiKVTxnRegionsNumHistogram.WithLabelValues(metricsTag("prewrite"))

func (actionPrewrite) String() string {
	return "prewrite"
}

func (actionPrewrite) tiKVTxnRegionsNumHistogram() prometheus.Observer {
	return tiKVTxnRegionsNumHistogramPrewrite
}

func (c *twoPhaseCommitter) buildPrewriteRequest(batch batchMutations, txnSize uint64) *tikvrpc.Request {
	m := &batch.mutations
	mutations := make([]*pb.Mutation, m.len())
	for i := range m.keys {
		mutations[i] = &pb.Mutation{
			Op:    m.ops[i],
			Key:   m.keys[i],
			Value: m.values[i],
		}
	}
	var minCommitTS uint64
	if c.forUpdateTS > 0 {
		minCommitTS = c.forUpdateTS + 1
	} else {
		minCommitTS = c.startTS + 1
	}

	failpoint.Inject("mockZeroCommitTS", func(val failpoint.Value) {
		// Should be val.(uint64) but failpoint doesn't support that.
		if tmp, ok := val.(int); ok && uint64(tmp) == c.startTS {
			minCommitTS = 0
		}
	})

	req := &pb.PrewriteRequest{
		Mutations:         mutations,
		PrimaryLock:       c.primary(),
		StartVersion:      c.startTS,
		LockTtl:           c.lockTTL,
		IsPessimisticLock: m.isPessimisticLock,
		ForUpdateTs:       c.forUpdateTS,
		TxnSize:           txnSize,
		MinCommitTs:       minCommitTS,
	}

	if c.isAsyncCommit() {
		if batch.isPrimary {
			req.Secondaries = c.asyncSecondaries()
		}
		req.UseAsyncCommit = true
		// The async commit can not be used for large transactions, and the commit ts can't be pushed.
		req.MinCommitTs = 0
	}

	return tikvrpc.NewRequest(tikvrpc.CmdPrewrite, req, pb.Context{Priority: c.priority, SyncLog: c.syncLog})
}

func (action actionPrewrite) handleSingleBatch(c *twoPhaseCommitter, bo *Backoffer, batch batchMutations) error {
	txnSize := uint64(c.regionTxnSize[batch.region.id])
	// When we retry because of a region miss, we don't know the transaction size. We set the transaction size here
	// to MaxUint64 to avoid unexpected "resolve lock lite".
	if len(bo.errors) > 0 {
		txnSize = math.MaxUint64
	}

	req := c.buildPrewriteRequest(batch, txnSize)
	for {
		resp, err := c.store.SendReq(bo, req, batch.region, readTimeoutShort)
		if err != nil {
			return errors.Trace(err)
		}
		regionErr, err := resp.GetRegionError()
		if err != nil {
			return errors.Trace(err)
		}
		if regionErr != nil {
			err = bo.Backoff(BoRegionMiss, errors.New(regionErr.String()))
			if err != nil {
				return errors.Trace(err)
			}
			err = c.prewriteMutations(bo, batch.mutations)
			return errors.Trace(err)
		}
		if resp.Resp == nil {
			return errors.Trace(ErrBodyMissing)
		}
		prewriteResp := resp.Resp.(*pb.PrewriteResponse)
		keyErrs := prewriteResp.GetErrors()
		if len(keyErrs) == 0 {
			if bytes.Equal(c.primary(), batch.mutations.keys[0]) {
				// After writing the primary key, if the size of the transaction is large than 32M,
				// start the ttlManager. The ttlManager will be closed in tikvTxn.Commit().
				if int64(c.txnSize) > config.GetGlobalConfig().TiKVClient.TTLRefreshedTxnSize {
					c.run(c, nil)
				}
			}
			if c.isAsyncCommit() {
				// 0 if the min_commit_ts is not ready or any other reason that async
				// commit cannot proceed. The client can then fallback to normal way to
				// continue committing the transaction if prewrite are all finished.
				if prewriteResp.MinCommitTs == 0 {
					logutil.Logger(bo.ctx).Warn("async commit cannot proceed since the returned minCommitTS is zero, "+
						"fallback to normal path", zap.Uint64("startTS", c.startTS))
					c.setAsyncCommit(false)
				} else {
					c.mu.Lock()
					if prewriteResp.MinCommitTs > c.minCommitTS {
						c.minCommitTS = prewriteResp.MinCommitTs
					}
					c.mu.Unlock()
				}
			}
			return nil
		}
		var locks []*Lock
		for _, keyErr := range keyErrs {
			// Check already exists error
			if alreadyExist := keyErr.GetAlreadyExist(); alreadyExist != nil {
				key := alreadyExist.GetKey()
				return c.extractKeyExistsErr(key)
			}

			// Extract lock from key error
			lock, err1 := extractLockFromKeyErr(keyErr)
			if err1 != nil {
				return errors.Trace(err1)
			}
			logutil.BgLogger().Info("prewrite encounters lock",
				zap.Uint64("conn", c.connID),
				zap.Stringer("lock", lock))
			locks = append(locks, lock)
		}
		start := time.Now()
		msBeforeExpired, err := c.store.lockResolver.resolveLocksForWrite(bo, c.startTS, locks)
		if err != nil {
			return errors.Trace(err)
		}
		atomic.AddInt64(&c.getDetail().ResolveLockTime, int64(time.Since(start)))
		if msBeforeExpired > 0 {
			err = bo.BackoffWithMaxSleep(BoTxnLock, int(msBeforeExpired), errors.Errorf("2PC prewrite lockedKeys: %d", len(locks)))
			if err != nil {
				return errors.Trace(err)
			}
		}
	}
}

func (c *twoPhaseCommitter) prewriteMutations(bo *Backoffer, mutations committerMutations) error {
	if span := opentracing.SpanFromContext(bo.ctx); span != nil && span.Tracer() != nil {
		span1 := span.Tracer().StartSpan("twoPhaseCommitter.prewriteMutations", opentracing.ChildOf(span.Context()))
		defer span1.Finish()
		bo.ctx = opentracing.ContextWithSpan(bo.ctx, span1)
	}

	return c.doActionOnMutations(bo, actionPrewrite{}, mutations)
}
<|endoftext|>


<commit_before>package app

import (
	"net"
	"net/http"

	"github.com/coredns/coredns/plugin"
	clog "github.com/coredns/coredns/plugin/pkg/log"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
)

var (
	setupErrCount = prometheus.NewCounterVec(prometheus.CounterOpts{
		Namespace: plugin.Namespace,
		Subsystem: "nodecache",
		Name:      "setup_errors",
		Help:      "The number of errors during periodic network setup for node-cache",
	}, []string{"errortype"})
)

func initMetrics(ipport string) {
	if err := serveMetrics(ipport); err != nil {
		clog.Errorf("Failed to start metrics handler: %s", err)
		return
	}
	registerMetrics()
}

func registerMetrics() {
	prometheus.MustRegister(setupErrCount)
}

func serveMetrics(ipport string) error {
	ln, err := net.Listen("tcp", ipport)
	if err != nil {
		return err
	}

	mux := http.NewServeMux()
	mux.Handle("/metrics", promhttp.Handler())
	srv := &http.Server{Handler: mux}
	go func() {
		srv.Serve(ln)
	}()
	return nil

}
<commit_msg>Improve Prometheus metrics for node-cache<commit_after>package app

import (
	"net"
	"net/http"

	"github.com/coredns/coredns/plugin"
	clog "github.com/coredns/coredns/plugin/pkg/log"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
)

var setupErrCount = prometheus.NewCounterVec(prometheus.CounterOpts{
	Namespace: plugin.Namespace,
	Subsystem: "nodecache",
	Name:      "setup_errors_total",
	Help:      "The number of errors during periodic network setup for node-cache",
}, []string{"errortype"})

func initMetrics(ipport string) {
	if err := serveMetrics(ipport); err != nil {
		clog.Errorf("Failed to start metrics handler: %s", err)
		return
	}
	registerMetrics()
}

func registerMetrics() {
	prometheus.MustRegister(setupErrCount)
	setupErrCount.WithLabelValues("iptables").Add(0)
	setupErrCount.WithLabelValues("iptables_lock").Add(0)
	setupErrCount.WithLabelValues("interface_add").Add(0)
	setupErrCount.WithLabelValues("interface_check").Add(0)
}

func serveMetrics(ipport string) error {
	ln, err := net.Listen("tcp", ipport)
	if err != nil {
		return err
	}

	mux := http.NewServeMux()
	mux.Handle("/metrics", promhttp.Handler())
	srv := &http.Server{Handler: mux}
	go func() {
		srv.Serve(ln)
	}()
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package backup

import (
	"errors"
	"fmt"
	"github.com/jacobsa/comeback/blob"
	"github.com/jacobsa/comeback/fs"
	"github.com/jacobsa/comeback/repr"
	"path"
)

// An object that knows how to save directories to some underlying storage.
type DirectorySaver interface {
	// Recursively save the contents of the supplied directory (defined by a base
	// path of a backup and a relative path within the backup) to the underlying
	// storage, returning the score of a blob representing the directory's
	// listing in a format that can be recovered with repr.Unmarshal.
	Save(basePath, relPath string) (score blob.Score, err error)
}

// A directory saver that creates a new directory saver for each call to Save.
// This breaks a self-dependency that would be needed to make use of
// NewNonRecursiveDirectorySaver.
type onDemandDirSaver struct {
	createSaver func(wrapped DirectorySaver) DirectorySaver
}

func (s *onDemandDirSaver) Save(basePath, relPath string) (score blob.Score, err error) {
	return s.createSaver(s).Save(basePath, relPath)
}

// Return a directory saver that makes use of the supplied dependencies.
func NewDirectorySaver(
	blobStore blob.Store,
	fileSystem fs.FileSystem,
	fileSaver FileSaver) (DirectorySaver, error) {
	linkResolver := NewLinkResolver()
	createSaver := func(wrapped DirectorySaver) DirectorySaver {
		saver, err := NewNonRecursiveDirectorySaver(
			blobStore,
			fileSystem,
			fileSaver,
			wrapped,
			linkResolver)

		if err != nil {
			panic(err)
		}

		return saver
	}

	return &onDemandDirSaver{createSaver}, nil
}

// Equivalent to NewDirectorySaver, but with an injectable wrapped directory
// saver and link resolver to aid with testability. You should not use this
// function.
func NewNonRecursiveDirectorySaver(
	store blob.Store,
	fileSystem fs.FileSystem,
	fileSaver FileSaver,
	wrapped DirectorySaver,
	linkResolver LinkResolver) (DirectorySaver, error) {
	return &dirSaver{
		blobStore:    store,
		fileSystem:   fileSystem,
		fileSaver:    fileSaver,
		wrapped:      wrapped,
		linkResolver: linkResolver,
	}, nil
}

type dirSaver struct {
	blobStore    blob.Store
	fileSystem   fs.FileSystem
	fileSaver    FileSaver
	wrapped      DirectorySaver
	linkResolver LinkResolver
}

func (s *dirSaver) saveDir(basePath string, relPath string, entry *fs.DirectoryEntry) ([]blob.Score, error) {
	// Recurse.
	score, err := s.wrapped.Save(basePath, path.Join(relPath, entry.Name))
	if err != nil {
		return nil, err
	}

	return []blob.Score{score}, nil
}

func (s *dirSaver) saveFile(parent string, entry *fs.DirectoryEntry) ([]blob.Score, error) {
	// Open the file.
	f, err := s.fileSystem.OpenForReading(path.Join(parent, entry.Name))
	if err != nil {
		return nil, fmt.Errorf("Opening file: %v", err)
	}
	defer f.Close()

	// Defer to the file saver.
	return s.fileSaver.Save(f)
}

func (s *dirSaver) Save(basePath, relPath string) (score blob.Score, err error) {
	dirpath := path.Join(basePath, relPath)

	// Grab a listing for the directory.
	entries, err := s.fileSystem.ReadDir(dirpath)
	if err != nil {
		return nil, fmt.Errorf("Listing directory: %v", err)
	}

	// Save the data for each entry.
	for _, entry := range entries {
		// Call the appropriate method based on this entry's type.
		switch entry.Type {
		case fs.TypeFile:
			// Make sure this isn't a hard link to something we've already saved.
			entry.HardLinkTarget = s.linkResolver.Register(
				entry.ContainingDevice,
				entry.Inode,
				path.Join(relPath, entry.Name))

			if entry.HardLinkTarget == nil {
				entry.Scores, err = s.saveFile(dirpath, entry)
			}

		case fs.TypeDirectory:
			entry.Scores, err = s.saveDir(basePath, relPath, entry)
		case fs.TypeSymlink:
		case fs.TypeBlockDevice:
		case fs.TypeCharDevice:
		case fs.TypeNamedPipe:
		default:
			err = fmt.Errorf("Unhandled type: %v", entry.Type)
		}

		if err != nil {
			return nil, err
		}
	}

	// Create a serialized version of this information.
	data, err := repr.Marshal(entries)
	if err != nil {
		return nil, fmt.Errorf("Marshaling: %v", err)
	}

	// Store that serialized version.
	score, err = s.blobStore.Store(data)
	if err != nil {
		err = errors.New("Storing dir blob: " + err.Error())
	}

	return
}
<commit_msg>Declared new interface for dir saver.<commit_after>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package backup

import (
	"errors"
	"fmt"
	"github.com/jacobsa/comeback/blob"
	"github.com/jacobsa/comeback/fs"
	"github.com/jacobsa/comeback/repr"
	"path"
	"regexp"
)

// An object that knows how to save directories to some underlying storage.
type DirectorySaver interface {
	// Recursively save the contents of the supplied directory (defined by a base
	// path of a backup and a relative path within the backup) to the underlying
	// storage, returning the score of a blob representing the directory's
	// listing in a format that can be recovered with repr.Unmarshal.
	//
	// Recursively exclude from backup relative paths that match any of the
	// supplied exclusion regexps.
	Save(basePath, relPath string, exclusions []regexp.Regexp) (blob.Score, error)
}

// A directory saver that creates a new directory saver for each call to Save.
// This breaks a self-dependency that would be needed to make use of
// NewNonRecursiveDirectorySaver.
type onDemandDirSaver struct {
	createSaver func(wrapped DirectorySaver) DirectorySaver
}

func (s *onDemandDirSaver) Save(basePath, relPath string) (score blob.Score, err error) {
	return s.createSaver(s).Save(basePath, relPath)
}

// Return a directory saver that makes use of the supplied dependencies.
func NewDirectorySaver(
	blobStore blob.Store,
	fileSystem fs.FileSystem,
	fileSaver FileSaver) (DirectorySaver, error) {
	linkResolver := NewLinkResolver()
	createSaver := func(wrapped DirectorySaver) DirectorySaver {
		saver, err := NewNonRecursiveDirectorySaver(
			blobStore,
			fileSystem,
			fileSaver,
			wrapped,
			linkResolver)

		if err != nil {
			panic(err)
		}

		return saver
	}

	return &onDemandDirSaver{createSaver}, nil
}

// Equivalent to NewDirectorySaver, but with an injectable wrapped directory
// saver and link resolver to aid with testability. You should not use this
// function.
func NewNonRecursiveDirectorySaver(
	store blob.Store,
	fileSystem fs.FileSystem,
	fileSaver FileSaver,
	wrapped DirectorySaver,
	linkResolver LinkResolver) (DirectorySaver, error) {
	return &dirSaver{
		blobStore:    store,
		fileSystem:   fileSystem,
		fileSaver:    fileSaver,
		wrapped:      wrapped,
		linkResolver: linkResolver,
	}, nil
}

type dirSaver struct {
	blobStore    blob.Store
	fileSystem   fs.FileSystem
	fileSaver    FileSaver
	wrapped      DirectorySaver
	linkResolver LinkResolver
}

func (s *dirSaver) saveDir(basePath string, relPath string, entry *fs.DirectoryEntry) ([]blob.Score, error) {
	// Recurse.
	score, err := s.wrapped.Save(basePath, path.Join(relPath, entry.Name))
	if err != nil {
		return nil, err
	}

	return []blob.Score{score}, nil
}

func (s *dirSaver) saveFile(parent string, entry *fs.DirectoryEntry) ([]blob.Score, error) {
	// Open the file.
	f, err := s.fileSystem.OpenForReading(path.Join(parent, entry.Name))
	if err != nil {
		return nil, fmt.Errorf("Opening file: %v", err)
	}
	defer f.Close()

	// Defer to the file saver.
	return s.fileSaver.Save(f)
}

func (s *dirSaver) Save(basePath, relPath string) (score blob.Score, err error) {
	dirpath := path.Join(basePath, relPath)

	// Grab a listing for the directory.
	entries, err := s.fileSystem.ReadDir(dirpath)
	if err != nil {
		return nil, fmt.Errorf("Listing directory: %v", err)
	}

	// Save the data for each entry.
	for _, entry := range entries {
		// Call the appropriate method based on this entry's type.
		switch entry.Type {
		case fs.TypeFile:
			// Make sure this isn't a hard link to something we've already saved.
			entry.HardLinkTarget = s.linkResolver.Register(
				entry.ContainingDevice,
				entry.Inode,
				path.Join(relPath, entry.Name))

			if entry.HardLinkTarget == nil {
				entry.Scores, err = s.saveFile(dirpath, entry)
			}

		case fs.TypeDirectory:
			entry.Scores, err = s.saveDir(basePath, relPath, entry)
		case fs.TypeSymlink:
		case fs.TypeBlockDevice:
		case fs.TypeCharDevice:
		case fs.TypeNamedPipe:
		default:
			err = fmt.Errorf("Unhandled type: %v", entry.Type)
		}

		if err != nil {
			return nil, err
		}
	}

	// Create a serialized version of this information.
	data, err := repr.Marshal(entries)
	if err != nil {
		return nil, fmt.Errorf("Marshaling: %v", err)
	}

	// Store that serialized version.
	score, err = s.blobStore.Store(data)
	if err != nil {
		err = errors.New("Storing dir blob: " + err.Error())
	}

	return
}
<|endoftext|>