<commit_before>package main

import (
	"fmt"
	"time"

	"github.com/micro/go-micro/client"
	"github.com/micro/go-micro/cmd"
	c "github.com/micro/go-micro/context"
	example "github.com/micro/go-micro/examples/server/proto/example"
	"golang.org/x/net/context"
)

// wrapper example code

// log wrapper logs every time a request is made
type logWrapper struct {
	client.Client
}

func (l *logWrapper) Call(ctx context.Context, req client.Request, rsp interface{}) error {
	md, _ := c.GetMetadata(ctx)
	fmt.Printf("[Log Wrapper] ctx: %v service: %s method: %s\n", md, req.Service(), req.Method())
	return l.Client.Call(ctx, req, rsp)
}

// trace wrapper attaches a unique trace ID - timestamp
type traceWrapper struct {
	client.Client
}

func (t *traceWrapper) Call(ctx context.Context, req client.Request, rsp interface{}) error {
	ctx = c.WithMetadata(ctx, map[string]string{
		"X-Trace-Id": fmt.Sprintf("%d", time.Now().Unix()),
	})
	return t.Client.Call(ctx, req, rsp)
}

// Implements client.Wrapper as logWrapper
func logWrap(c client.Client) client.Client {
	return &logWrapper{c}
}

// Implements client.Wrapper as traceWrapper
func traceWrap(c client.Client) client.Client {
	return &traceWrapper{c}
}

// publishes a message
func pub() {
	msg := client.NewPublication("topic.go.micro.srv.example", &example.Message{
		Say: "This is a publication",
	})

	// create context with metadata
	ctx := c.WithMetadata(context.Background(), map[string]string{
		"X-User-Id": "john",
		"X-From-Id": "script",
	})

	// publish message
	if err := client.Publish(ctx, msg); err != nil {
		fmt.Println("pub err: ", err)
		return
	}

	fmt.Printf("Published: %v\n", msg)
}

func call(i int) {
	// Create new request to service go.micro.srv.example, method Example.Call
	req := client.NewRequest("go.micro.srv.example", "Example.Call", &example.Request{
		Name: "John",
	})

	// create context with metadata
	ctx := c.WithMetadata(context.Background(), map[string]string{
		"X-User-Id": "john",
		"X-From-Id": "script",
	})

	rsp := &example.Response{}

	// Call service
	if err := client.Call(ctx, req, rsp); err != nil {
		fmt.Println("call err: ", err, rsp)
		return
	}

	fmt.Println("Call:", i, "rsp:", rsp.Msg)
}

func stream() {
	// Create new request to service go.micro.srv.example, method Example.Call
	req := client.NewRequest("go.micro.srv.example", "Example.Stream", &example.StreamingRequest{
		Count: int64(10),
	})

	rspChan := make(chan *example.StreamingResponse, 10)

	stream, err := client.Stream(context.Background(), req, rspChan)
	if err != nil {
		fmt.Println("err:", err)
		return
	}

	for rsp := range rspChan {
		fmt.Println("Stream: rsp:", rsp.Count)
	}

	if stream.Error() != nil {
		fmt.Println("stream err:", err)
		return
	}

	if err := stream.Close(); err != nil {
		fmt.Println("stream close err:", err)
	}
}

func main() {
	cmd.Init()

	//	client.DefaultClient = client.NewClient(
	//		client.Codec("application/pb", pb.Codec),
	//		client.ContentType("application/pb"),
	//	)
	for {
		fmt.Println("\n--- Call example ---\n")
		for i := 0; i < 10; i++ {
			call(i)
		}

		fmt.Println("\n--- Streamer example ---\n")
		stream()

		fmt.Println("\n--- Publisher example ---\n")
		pub()

		fmt.Println("\n--- Wrapper example ---\n")

		// Wrap the default client
		client.DefaultClient = logWrap(client.DefaultClient)

		call(0)

		// Wrap using client.Wrap option
		client.DefaultClient = client.NewClient(
			client.Wrap(traceWrap),
			client.Wrap(logWrap),
		)

		call(1)
		time.Sleep(time.Millisecond * 100)
	}
}
<commit_msg>Don't loop the example forever<commit_after>package main

import (
	"fmt"
	"time"

	"github.com/micro/go-micro/client"
	"github.com/micro/go-micro/cmd"
	c "github.com/micro/go-micro/context"
	example "github.com/micro/go-micro/examples/server/proto/example"
	"golang.org/x/net/context"
)

// wrapper example code

// log wrapper logs every time a request is made
type logWrapper struct {
	client.Client
}

func (l *logWrapper) Call(ctx context.Context, req client.Request, rsp interface{}) error {
	md, _ := c.GetMetadata(ctx)
	fmt.Printf("[Log Wrapper] ctx: %v service: %s method: %s\n", md, req.Service(), req.Method())
	return l.Client.Call(ctx, req, rsp)
}

// trace wrapper attaches a unique trace ID - timestamp
type traceWrapper struct {
	client.Client
}

func (t *traceWrapper) Call(ctx context.Context, req client.Request, rsp interface{}) error {
	ctx = c.WithMetadata(ctx, map[string]string{
		"X-Trace-Id": fmt.Sprintf("%d", time.Now().Unix()),
	})
	return t.Client.Call(ctx, req, rsp)
}

// Implements client.Wrapper as logWrapper
func logWrap(c client.Client) client.Client {
	return &logWrapper{c}
}

// Implements client.Wrapper as traceWrapper
func traceWrap(c client.Client) client.Client {
	return &traceWrapper{c}
}

// publishes a message
func pub() {
	msg := client.NewPublication("topic.go.micro.srv.example", &example.Message{
		Say: "This is a publication",
	})

	// create context with metadata
	ctx := c.WithMetadata(context.Background(), map[string]string{
		"X-User-Id": "john",
		"X-From-Id": "script",
	})

	// publish message
	if err := client.Publish(ctx, msg); err != nil {
		fmt.Println("pub err: ", err)
		return
	}

	fmt.Printf("Published: %v\n", msg)
}

func call(i int) {
	// Create new request to service go.micro.srv.example, method Example.Call
	req := client.NewRequest("go.micro.srv.example", "Example.Call", &example.Request{
		Name: "John",
	})

	// create context with metadata
	ctx := c.WithMetadata(context.Background(), map[string]string{
		"X-User-Id": "john",
		"X-From-Id": "script",
	})

	rsp := &example.Response{}

	// Call service
	if err := client.Call(ctx, req, rsp); err != nil {
		fmt.Println("call err: ", err, rsp)
		return
	}

	fmt.Println("Call:", i, "rsp:", rsp.Msg)
}

func stream() {
	// Create new request to service go.micro.srv.example, method Example.Call
	req := client.NewRequest("go.micro.srv.example", "Example.Stream", &example.StreamingRequest{
		Count: int64(10),
	})

	rspChan := make(chan *example.StreamingResponse, 10)

	stream, err := client.Stream(context.Background(), req, rspChan)
	if err != nil {
		fmt.Println("err:", err)
		return
	}

	for rsp := range rspChan {
		fmt.Println("Stream: rsp:", rsp.Count)
	}

	if stream.Error() != nil {
		fmt.Println("stream err:", err)
		return
	}

	if err := stream.Close(); err != nil {
		fmt.Println("stream close err:", err)
	}
}

func main() {
	cmd.Init()

	fmt.Println("\n--- Call example ---\n")
	for i := 0; i < 10; i++ {
		call(i)
	}

	fmt.Println("\n--- Streamer example ---\n")
	stream()

	fmt.Println("\n--- Publisher example ---\n")
	pub()

	fmt.Println("\n--- Wrapper example ---\n")

	// Wrap the default client
	client.DefaultClient = logWrap(client.DefaultClient)

	call(0)

	// Wrap using client.Wrap option
	client.DefaultClient = client.NewClient(
		client.Wrap(traceWrap),
		client.Wrap(logWrap),
	)

	call(1)
}
<|endoftext|>


<commit_before>// Copyright (C) 2017 Damon Revoe. All rights reserved.
// Use of this source code is governed by the MIT
// license, which can be found in the LICENSE file.

package main

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"gopkg.in/yaml.v2"
)

type packageDefinition struct {
	packageName string
	packageType string
	pathname    string
	requires    []string
	params      templateParams
}

func getRequiredField(pd *packageDefinition,
	fieldName string) (interface{}, error) {
	if value := pd.params[fieldName]; value != nil {
		return value, nil
	}
	return nil, errors.New(pd.pathname +
		": missing required field '" + fieldName + "'")
}

func getRequiredStringField(pd *packageDefinition,
	fieldName string) (string, error) {
	if value, err := getRequiredField(pd, fieldName); err != nil {
		return "", err
	} else if stringValue, ok := value.(string); ok {
		return stringValue, nil
	} else {
		return "", errors.New(pd.pathname +
			": '" + fieldName + "' field must be a string")
	}
}

func loadPackageDefinition(pathname string) (pd packageDefinition, err error) {
	data, err := ioutil.ReadFile(pathname)

	if err != nil {
		return
	}

	if err = yaml.Unmarshal(data, &pd.params); err != nil {
		errMessage := strings.TrimPrefix(err.Error(), "yaml: ")
		err = errors.New(pathname + ": " + errMessage)
		return
	}

	pd.packageName, err = getRequiredStringField(&pd, "name")
	if err != nil {
		return
	}

	pd.packageType, err = getRequiredStringField(&pd, "type")
	if err != nil {
		return
	}

	if requiredPackages := pd.params["requires"]; requiredPackages != nil {
		pkgList, ok := requiredPackages.([]interface{})
		if !ok {
			err = errors.New(pathname +
				": 'requires' must be a list")
			return
		}
		for _, pkgName := range pkgList {
			pd.requires = append(pd.requires, pkgName.(string))
		}
	}

	return
}

type packageIndex struct {
	packageByName   map[string]packageDefinition
	orderedPackages []packageDefinition
}

func buildPackageIndex(pkgpath string) (packageIndex, error) {
	var pi packageIndex

	if len(pkgpath) == 0 {
		pkgpath = os.Getenv(pkgPathEnvVar)

		if len(pkgpath) == 0 {
			return pi, errors.New("--pkgpath is not given and $" +
				pkgPathEnvVar + " is not defined")
		}
	}

	pkgpathDirs := append(strings.Split(pkgpath, ":"),
		filepath.Join(filepath.Dir(os.Args[0]), "templates"))

	for _, pkgpathDir := range pkgpathDirs {
		dirEntries, _ := ioutil.ReadDir(pkgpathDir)

		for _, dirEntry := range dirEntries {
			dirEntryPathname := filepath.Join(pkgpathDir,
				dirEntry.Name(), dirEntry.Name()+".yaml")

			fileInfo, err := os.Stat(dirEntryPathname)
			if err != nil || !fileInfo.Mode().IsRegular() {
				continue
			}

			pd, err := loadPackageDefinition(dirEntryPathname)

			if err != nil {
				return packageIndex{}, err
			}

			pi.orderedPackages = append(pi.orderedPackages, pd)
		}
	}

	return pi, nil
}

func (index *packageIndex) printListOfPackages() {
	fmt.Println("List of packages:")

	for _, pd := range index.orderedPackages {
		fmt.Println(pd.packageName)
		fmt.Println(pd.packageType)
		for _, rp := range pd.requires {
			fmt.Println("-", rp)
		}
		fmt.Println()
	}
}
<commit_msg>Use idiomatic construct to check for empty strings<commit_after>// Copyright (C) 2017 Damon Revoe. All rights reserved.
// Use of this source code is governed by the MIT
// license, which can be found in the LICENSE file.

package main

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"gopkg.in/yaml.v2"
)

type packageDefinition struct {
	packageName string
	packageType string
	pathname    string
	requires    []string
	params      templateParams
}

func getRequiredField(pd *packageDefinition,
	fieldName string) (interface{}, error) {
	if value := pd.params[fieldName]; value != nil {
		return value, nil
	}
	return nil, errors.New(pd.pathname +
		": missing required field '" + fieldName + "'")
}

func getRequiredStringField(pd *packageDefinition,
	fieldName string) (string, error) {
	if value, err := getRequiredField(pd, fieldName); err != nil {
		return "", err
	} else if stringValue, ok := value.(string); ok {
		return stringValue, nil
	} else {
		return "", errors.New(pd.pathname +
			": '" + fieldName + "' field must be a string")
	}
}

func loadPackageDefinition(pathname string) (pd packageDefinition, err error) {
	data, err := ioutil.ReadFile(pathname)

	if err != nil {
		return
	}

	if err = yaml.Unmarshal(data, &pd.params); err != nil {
		errMessage := strings.TrimPrefix(err.Error(), "yaml: ")
		err = errors.New(pathname + ": " + errMessage)
		return
	}

	pd.packageName, err = getRequiredStringField(&pd, "name")
	if err != nil {
		return
	}

	pd.packageType, err = getRequiredStringField(&pd, "type")
	if err != nil {
		return
	}

	if requiredPackages := pd.params["requires"]; requiredPackages != nil {
		pkgList, ok := requiredPackages.([]interface{})
		if !ok {
			err = errors.New(pathname +
				": 'requires' must be a list")
			return
		}
		for _, pkgName := range pkgList {
			pd.requires = append(pd.requires, pkgName.(string))
		}
	}

	return
}

type packageIndex struct {
	packageByName   map[string]packageDefinition
	orderedPackages []packageDefinition
}

func buildPackageIndex(pkgpath string) (packageIndex, error) {
	var pi packageIndex

	if pkgpath == "" {
		pkgpath = os.Getenv(pkgPathEnvVar)

		if pkgpath == "" {
			return pi, errors.New("--pkgpath is not given and $" +
				pkgPathEnvVar + " is not defined")
		}
	}

	pkgpathDirs := append(strings.Split(pkgpath, ":"),
		filepath.Join(filepath.Dir(os.Args[0]), "templates"))

	for _, pkgpathDir := range pkgpathDirs {
		dirEntries, _ := ioutil.ReadDir(pkgpathDir)

		for _, dirEntry := range dirEntries {
			dirEntryPathname := filepath.Join(pkgpathDir,
				dirEntry.Name(), dirEntry.Name()+".yaml")

			fileInfo, err := os.Stat(dirEntryPathname)
			if err != nil || !fileInfo.Mode().IsRegular() {
				continue
			}

			pd, err := loadPackageDefinition(dirEntryPathname)

			if err != nil {
				return packageIndex{}, err
			}

			pi.orderedPackages = append(pi.orderedPackages, pd)
		}
	}

	return pi, nil
}

func (index *packageIndex) printListOfPackages() {
	fmt.Println("List of packages:")

	for _, pd := range index.orderedPackages {
		fmt.Println(pd.packageName)
		fmt.Println(pd.packageType)
		for _, rp := range pd.requires {
			fmt.Println("-", rp)
		}
		fmt.Println()
	}
}
<|endoftext|>


<commit_before>package leafveingo

import (
	"fmt"
	"net/url"
	"reflect"
	"testing"
	"time"
)

type Hobby struct {
	Name  string
	Names []string
}

type UserType struct {
	TypeName string
}

type User struct {
	UserName string
	UserPwd  string
	Type     UserType
	TypeP    *UserType
	Hobbys   []Hobby
	HobbysP  []*Hobby
	Content  []byte
}

type Params struct {
	Users    []User
	UsersP   []*User
	OneUser  User
	OneUserP *User
}

//	测试单个设值测试，此函数主要用户调试测试
func TestStringNameSetStructFieldValue(t *testing.T) {
	leafvein := &sfLeafvein{}

	keys := url.Values{}
	// keys.Add("users[0].hobbys[0].name", "hobbys0-0")
	// keys.Add("users[0].hobbys[1].name", "hobbys0-1")
	// keys.Add("users[0].hobbys[2].name", "hobbys0-2")
	// keys.Add("users[1].hobbys[0].name", "hobbys1-0")
	// keys.Add("users[1].hobbys[1].name", "hobbys1-1")
	// keys.Add("users[0].typeP.typeName", "0-type-1")
	// keys.Add("users[0].type.typeName", "0-type-2")
	// keys.Add("users[1].hobbysP[0].name", "hobbysP1-0")
	// keys.Add("users[1].hobbysP[1].name", "hobbysP1-1")
	// keys.Add("oneUser.userName", "oneName")
	// keys.Add("oneUser.userPwd", "onePwd")
	// keys.Add("oneUser.hobbys[0].names[0]", "one-0-bobys-0-names-0")
	// keys.Add("oneUser.hobbys[1].names[0]", "one-0-bobys-1-names-0")
	keys.Add("usersP[0].hobbys[0].name", "userP0-hobbys0-0")
	keys.Add("usersP[0].hobbys[1].name", "userP0-hobbys0-1")
	keys.Add("usersP[1].hobbys[0].name", "userP1-hobbys1-0")

	refValue := leafvein.newStructPtr(reflect.TypeOf(Params{}), keys)

	// str := "userP0-hobbys0-0"
	// leafvein.setStructFieldValue(refValue, "usersP[0].hobbys[0].name", &str)

	//	转换类型然后进行比较判断
	params := refValue.Interface().(*Params)

	fmt.Println(len(params.UsersP[0].Hobbys))
}

//	设置结构字段测试
func funcSetStructFieldValue() *Params {
	leafvein := &sfLeafvein{}

	keys := url.Values{}
	keys.Add("users[0].hobbys[0].name", "hobbys0-0")
	keys.Add("users[0].hobbys[1].name", "hobbys0-1")
	keys.Add("users[0].hobbys[2].name", "hobbys0-2")
	keys.Add("users[1].hobbys[0].name", "hobbys1-0")
	keys.Add("users[1].hobbys[1].name", "hobbys1-1")
	keys.Add("users[0].typeP.typeName", "0-type-1")
	keys.Add("users[0].type.typeName", "0-type-2")
	keys.Add("users[1].hobbysP[0].name", "hobbys1-0")
	keys.Add("users[1].hobbysP[1].name", "hobbys1-1")
	keys.Add("oneUser.userName", "oneName")
	keys.Add("oneUser.userPwd", "onePwd")

	start := time.Now()
	refValue := leafvein.newStructPtr(reflect.TypeOf(Params{}), keys)
	fmt.Println("newStructPtr time :", time.Now().Sub(start))
	start = time.Now()
	for k, v := range keys {
		leafvein.setStructFieldValue(refValue, k, v[0])
	}
	fmt.Println("setStructFieldValue time :", time.Now().Sub(start))
	return refValue.Interface().(*Params)
}

//	测试设置结构字段直
func TestSetStructFieldValue(t *testing.T) {
	start := time.Now()
	params := funcSetStructFieldValue()
	fmt.Println("time :", time.Now().Sub(start))

	if nil == params.OneUserP {
		t.Error("nil != params.OneUserP 初始化错误")
		return
	}

	if 2 != len(params.Users) {
		t.Error("2 != len(params.Users) 初始化的用户数量不正确")
		return
	}

	if 3 != len(params.Users[0].Hobbys) {
		t.Error("3 != len(params.Users[0].Hobbys) 初始化数量不正确")
		return
	}

	if 2 != len(params.Users[1].HobbysP) {
		t.Error("2 != len(params.Users[1].HobbysP 初始化数量不正确")
		return
	}

	if "oneName" != params.OneUser.UserName {
		t.Error("oneName != params.OneUser.UserName 设值不正确")
		return
	}

	if "onePwd" != params.OneUser.UserPwd {
		t.Error("onePwd != params.OneUser.UserPwd 设值不正确")
		return
	}

	if "hobbys0-0" != params.Users[0].Hobbys[0].Name {
		t.Error(`"hobbys0-0" != params.Users[0].Hobbys[0].Name`)
		return
	}

	if "hobbys0-1" != params.Users[0].Hobbys[1].Name {
		t.Error(`"hobbys0-1" != params.Users[0].Hobbys[1].Name`)
		return
	}

	if "hobbys0-2" != params.Users[0].Hobbys[2].Name {
		t.Error(`"hobbys0-2" != params.Users[0].Hobbys[2].Name`)
		return
	}

	if "0-type-1" != params.Users[0].TypeP.TypeName {
		t.Error(`""0-type-1" != params.Users[0].TypeP.TypeName`)
		return
	}

	if "0-type-2" != params.Users[0].Type.TypeName {
		t.Error(`""0-type-2" != params.Users[0].Type.TypeName`)
		return
	}

	if "hobbys1-0" != params.Users[1].Hobbys[0].Name {
		t.Error(`"hobbys1-0" != params.Users[1].Hobbys[0].Name`)
		return
	}

	if "hobbys1-1" != params.Users[1].Hobbys[1].Name {
		t.Error(`"hobbys1-1" != params.Users[1].Hobbys[1].Name`)
		return
	}
}

//	提供TestNewStructPtr和Benchmark_NewStructPtr进行测试
func funcNewStructPtr() interface{} {

	leafvein := &sfLeafvein{}

	keys := url.Values{}

	keys.Add("oneUser.hobbys[0].names[2]", "one-0-hobbys-0-names-2")
	keys.Add("users[0].hobbys[1].name", "hobbys1")
	keys.Add("oneUser.hobbys[0].names[1]", "one-0-hobbys-0-names-1")
	keys.Add("users[1].hobbys[0].name", "hobbys1")
	keys.Add("oneUser.hobbys[0].names[0]", "one-0-hobbys-0-names-0")
	keys.Add("users[1].hobbysP[0].name", "hobbys1")
	keys.Add("users[1].hobbysP[1].name", "hobbys1")
	keys.Add("users[0].hobbys[0].name", "hobbys1")
	keys.Add("oneUser.userName", "oneName")
	keys.Add("oneUser.userPwd", "onePwd")
	keys.Add("users[0].hobbys[2].name", "hobbys1")
	keys.Add("users[0].hobbysP[0].name", "hobbys1")
	keys.Add("oneUser.hobbys[1].names[0]", "one-0-hobbys-1-names-1")
	keys.Add("users[0].hobbys[0].name", "hobbys1")

	rt := reflect.TypeOf(Params{})
	rv := leafvein.newStructPtr(rt, keys)
	return rv.Interface()
}

//	测试创建结构里的嵌套内容
func TestNewStructPtr(t *testing.T) {
	start := time.Now()

	params := funcNewStructPtr().(*Params)

	fmt.Println("执行newStructPtr速度:", time.Now().Sub(start))

	if nil == params.OneUserP {
		t.Error("nil != params.OneUserP 初始化错误")
		return
	}

	if 2 != len(params.Users) {
		t.Error("2 != len(params.Users) 初始化的用户数量不正确")
		return
	}

	if 3 != len(params.Users[0].Hobbys) {
		t.Error("3 != len(params.Users[0].Hobbys) 初始化数量不正确")
		return
	}

	if 1 != len(params.Users[0].HobbysP) {
		t.Error("1 != len(params.Users[0].HobbysP 初始化数量不正确")
		return
	}

	if 2 != len(params.Users[1].HobbysP) {
		t.Error("2 != len(params.Users[1].HobbysP 初始化数量不正确")
		return
	}

	if 1 != len(params.Users[1].Hobbys) {
		t.Error("1 != len(params.Users[1].Hobbys 初始化数量不正确")
		return
	}

	if 2 != len(params.OneUser.Hobbys) {
		t.Error("2 != len(params.OneUser.Hobbys) 初始化数量不正确")
		return
	}

	if 3 != len(params.OneUser.Hobbys[0].Names) {
		t.Error("3 != len(params.OneUser.Hobbys[0].Names) 初始化数量不正确")
		return
	}

	if 1 != len(params.OneUser.Hobbys[1].Names) {
		t.Error("1 != len(params.OneUser.Hobbys[1].Names) 初始化数量不正确")
		return
	}
}

//	速度测试new struct, 包含里面的指针结构类型和切片大小分配
func Benchmark_NewStructPtr(b *testing.B) {
	for i := 0; i < b.N; i++ {
		params := funcNewStructPtr().(*Params)

		if nil == params.OneUserP {
			b.Error("nil != params.OneUserP 初始化错误")
			return
		}

		if 2 != len(params.Users) {
			b.Error("2 != len(params.Users) 初始化的用户数量不正确")
			return
		}

		if 3 != len(params.Users[0].Hobbys) {
			b.Error("3 != len(params.Users[0].Hobbys) 初始化数量不正确")
			return
		}

		if 1 != len(params.Users[0].HobbysP) {
			b.Error("1 != len(params.Users[0].HobbysP 初始化数量不正确")
			return
		}

		if 2 != len(params.Users[1].HobbysP) {
			b.Error("2 != len(params.Users[1].HobbysP 初始化数量不正确")
			return
		}

		if 1 != len(params.Users[1].Hobbys) {
			b.Error("1 != len(params.Users[1].Hobbys 初始化数量不正确")
			return
		}

		if 2 != len(params.OneUser.Hobbys) {
			b.Error("2 != len(params.OneUser.Hobbys) 初始化数量不正确")
			return
		}

		if 3 != len(params.OneUser.Hobbys[0].Names) {
			b.Error("3 != len(params.OneUser.Hobbys[0].Names) 初始化数量不正确")
			return
		}

		if 1 != len(params.OneUser.Hobbys[1].Names) {
			b.Error("1 != len(params.OneUser.Hobbys[1].Names) 初始化数量不正确")
			return
		}

	}
}
<commit_msg>整改后修改封装参数的测试<commit_after>package leafveingo

import (
	"fmt"
	"net/url"
	"reflect"
	"testing"
	"time"
)

type Hobby struct {
	Name  string
	Names []string
}

type UserType struct {
	TypeName string
}

type User struct {
	UserName string
	UserPwd  string
	Type     UserType
	TypeP    *UserType
	Hobbys   []Hobby
	HobbysP  []*Hobby
	Content  []byte
}

type Params struct {
	Users    []User
	UsersP   []*User
	OneUser  User
	OneUserP *User
}

//	测试单个设值测试，此函数主要用户调试测试
func TestStringNameSetStructFieldValue(t *testing.T) {

	keys := url.Values{}
	// keys.Add("users[0].hobbys[0].name", "hobbys0-0")
	// keys.Add("users[0].hobbys[1].name", "hobbys0-1")
	// keys.Add("users[0].hobbys[2].name", "hobbys0-2")
	// keys.Add("users[1].hobbys[0].name", "hobbys1-0")
	// keys.Add("users[1].hobbys[1].name", "hobbys1-1")
	// keys.Add("users[0].typeP.typeName", "0-type-1")
	// keys.Add("users[0].type.typeName", "0-type-2")
	// keys.Add("users[1].hobbysP[0].name", "hobbysP1-0")
	// keys.Add("users[1].hobbysP[1].name", "hobbysP1-1")
	// keys.Add("oneUser.userName", "oneName")
	// keys.Add("oneUser.userPwd", "onePwd")
	// keys.Add("oneUser.hobbys[0].names[0]", "one-0-bobys-0-names-0")
	// keys.Add("oneUser.hobbys[1].names[0]", "one-0-bobys-1-names-0")
	keys.Add("usersP[0].hobbys[0].name", "userP0-hobbys0-0")
	keys.Add("usersP[0].hobbys[1].name", "userP0-hobbys0-1")
	keys.Add("usersP[1].hobbys[0].name", "userP1-hobbys1-0")

	refValue := parampackNewStructPtr(reflect.TypeOf(Params{}), keys)

	// str := "userP0-hobbys0-0"
	// leafvein.setStructFieldValue(refValue, "usersP[0].hobbys[0].name", &str)

	//	转换类型然后进行比较判断
	params := refValue.Interface().(*Params)

	fmt.Println(len(params.UsersP[0].Hobbys))
}

//	设置结构字段测试
func funcSetStructFieldValue() *Params {

	keys := url.Values{}
	keys.Add("users[0].hobbys[0].name", "hobbys0-0")
	keys.Add("users[0].hobbys[1].name", "hobbys0-1")
	keys.Add("users[0].hobbys[2].name", "hobbys0-2")
	keys.Add("users[1].hobbys[0].name", "hobbys1-0")
	keys.Add("users[1].hobbys[1].name", "hobbys1-1")
	keys.Add("users[0].typeP.typeName", "0-type-1")
	keys.Add("users[0].type.typeName", "0-type-2")
	keys.Add("users[1].hobbysP[0].name", "hobbys1-0")
	keys.Add("users[1].hobbysP[1].name", "hobbys1-1")
	keys.Add("oneUser.userName", "oneName")
	keys.Add("oneUser.userPwd", "onePwd")

	start := time.Now()
	refValue := parampackNewStructPtr(reflect.TypeOf(Params{}), keys)
	fmt.Println("newStructPtr time :", time.Now().Sub(start))
	start = time.Now()
	for k, v := range keys {
		parampackSetStructFieldValue(refValue, k, v[0])
	}
	fmt.Println("setStructFieldValue time :", time.Now().Sub(start))
	return refValue.Interface().(*Params)
}

//	测试设置结构字段直
func TestSetStructFieldValue(t *testing.T) {
	start := time.Now()
	params := funcSetStructFieldValue()
	fmt.Println("time :", time.Now().Sub(start))

	if nil == params.OneUserP {
		t.Error("nil != params.OneUserP 初始化错误")
		return
	}

	if 2 != len(params.Users) {
		t.Error("2 != len(params.Users) 初始化的用户数量不正确")
		return
	}

	if 3 != len(params.Users[0].Hobbys) {
		t.Error("3 != len(params.Users[0].Hobbys) 初始化数量不正确")
		return
	}

	if 2 != len(params.Users[1].HobbysP) {
		t.Error("2 != len(params.Users[1].HobbysP 初始化数量不正确")
		return
	}

	if "oneName" != params.OneUser.UserName {
		t.Error("oneName != params.OneUser.UserName 设值不正确 " + params.OneUser.UserName)
		return
	}

	if "onePwd" != params.OneUser.UserPwd {
		t.Error("onePwd != params.OneUser.UserPwd 设值不正确")
		return
	}

	if "hobbys0-0" != params.Users[0].Hobbys[0].Name {
		t.Error(`"hobbys0-0" != params.Users[0].Hobbys[0].Name`)
		return
	}

	if "hobbys0-1" != params.Users[0].Hobbys[1].Name {
		t.Error(`"hobbys0-1" != params.Users[0].Hobbys[1].Name`)
		return
	}

	if "hobbys0-2" != params.Users[0].Hobbys[2].Name {
		t.Error(`"hobbys0-2" != params.Users[0].Hobbys[2].Name`)
		return
	}

	if "0-type-1" != params.Users[0].TypeP.TypeName {
		t.Error(`""0-type-1" != params.Users[0].TypeP.TypeName`)
		return
	}

	if "0-type-2" != params.Users[0].Type.TypeName {
		t.Error(`""0-type-2" != params.Users[0].Type.TypeName`)
		return
	}

	if "hobbys1-0" != params.Users[1].Hobbys[0].Name {
		t.Error(`"hobbys1-0" != params.Users[1].Hobbys[0].Name`)
		return
	}

	if "hobbys1-1" != params.Users[1].Hobbys[1].Name {
		t.Error(`"hobbys1-1" != params.Users[1].Hobbys[1].Name`)
		return
	}
}

//	提供TestNewStructPtr和Benchmark_NewStructPtr进行测试
func funcNewStructPtr() interface{} {

	keys := url.Values{}

	keys.Add("oneUser.hobbys[0].names[2]", "one-0-hobbys-0-names-2")
	keys.Add("users[0].hobbys[1].name", "hobbys1")
	keys.Add("oneUser.hobbys[0].names[1]", "one-0-hobbys-0-names-1")
	keys.Add("users[1].hobbys[0].name", "hobbys1")
	keys.Add("oneUser.hobbys[0].names[0]", "one-0-hobbys-0-names-0")
	keys.Add("users[1].hobbysP[0].name", "hobbys1")
	keys.Add("users[1].hobbysP[1].name", "hobbys1")
	keys.Add("users[0].hobbys[0].name", "hobbys1")
	keys.Add("oneUser.userName", "oneName")
	keys.Add("oneUser.userPwd", "onePwd")
	keys.Add("users[0].hobbys[2].name", "hobbys1")
	keys.Add("users[0].hobbysP[0].name", "hobbys1")
	keys.Add("oneUser.hobbys[1].names[0]", "one-0-hobbys-1-names-1")
	keys.Add("users[0].hobbys[0].name", "hobbys1")

	rt := reflect.TypeOf(Params{})
	rv := parampackNewStructPtr(rt, keys)
	return rv.Interface()
}

//	测试创建结构里的嵌套内容
func TestNewStructPtr(t *testing.T) {
	start := time.Now()

	params := funcNewStructPtr().(*Params)

	fmt.Println("执行newStructPtr速度:", time.Now().Sub(start))

	if nil == params.OneUserP {
		t.Error("nil != params.OneUserP 初始化错误")
		return
	}

	if 2 != len(params.Users) {
		t.Error("2 != len(params.Users) 初始化的用户数量不正确")
		return
	}

	if 3 != len(params.Users[0].Hobbys) {
		t.Error("3 != len(params.Users[0].Hobbys) 初始化数量不正确")
		return
	}

	if 1 != len(params.Users[0].HobbysP) {
		t.Error("1 != len(params.Users[0].HobbysP 初始化数量不正确")
		return
	}

	if 2 != len(params.Users[1].HobbysP) {
		t.Error("2 != len(params.Users[1].HobbysP 初始化数量不正确")
		return
	}

	if 1 != len(params.Users[1].Hobbys) {
		t.Error("1 != len(params.Users[1].Hobbys 初始化数量不正确")
		return
	}

	if 2 != len(params.OneUser.Hobbys) {
		t.Error("2 != len(params.OneUser.Hobbys) 初始化数量不正确")
		return
	}

	if 3 != len(params.OneUser.Hobbys[0].Names) {
		t.Error("3 != len(params.OneUser.Hobbys[0].Names) 初始化数量不正确")
		return
	}

	if 1 != len(params.OneUser.Hobbys[1].Names) {
		t.Error("1 != len(params.OneUser.Hobbys[1].Names) 初始化数量不正确")
		return
	}
}

//	速度测试new struct, 包含里面的指针结构类型和切片大小分配
func Benchmark_NewStructPtr(b *testing.B) {
	for i := 0; i < b.N; i++ {
		params := funcNewStructPtr().(*Params)

		if nil == params.OneUserP {
			b.Error("nil != params.OneUserP 初始化错误")
			return
		}

		if 2 != len(params.Users) {
			b.Error("2 != len(params.Users) 初始化的用户数量不正确")
			return
		}

		if 3 != len(params.Users[0].Hobbys) {
			b.Error("3 != len(params.Users[0].Hobbys) 初始化数量不正确")
			return
		}

		if 1 != len(params.Users[0].HobbysP) {
			b.Error("1 != len(params.Users[0].HobbysP 初始化数量不正确")
			return
		}

		if 2 != len(params.Users[1].HobbysP) {
			b.Error("2 != len(params.Users[1].HobbysP 初始化数量不正确")
			return
		}

		if 1 != len(params.Users[1].Hobbys) {
			b.Error("1 != len(params.Users[1].Hobbys 初始化数量不正确")
			return
		}

		if 2 != len(params.OneUser.Hobbys) {
			b.Error("2 != len(params.OneUser.Hobbys) 初始化数量不正确")
			return
		}

		if 3 != len(params.OneUser.Hobbys[0].Names) {
			b.Error("3 != len(params.OneUser.Hobbys[0].Names) 初始化数量不正确")
			return
		}

		if 1 != len(params.OneUser.Hobbys[1].Names) {
			b.Error("1 != len(params.OneUser.Hobbys[1].Names) 初始化数量不正确")
			return
		}

	}
}
<|endoftext|>


<commit_before>package main

import (
        "fmt"
        "strconv"
        "github.com/fzzy/radix/redis"
        "strings"
)

func calculateLimit(page int, limit int, size int) int {
        end := page * limit + limit
        if end > size {
                return size
        } else {
                return end
        }
}

func Paginate(configuration *Configuration, reply *redis.Reply, page int, limit int) []Actor {
        connection := connect(configuration.Redis.Host, configuration.Redis.Port)

        var actors []Actor
        start := page * limit
        end := calculateLimit(page, limit, len(reply.Elems))
        for i := start; i < end; i++ {
                actor, _ := reply.Elems[i].Str()
                reason := connection.Cmd("GET", actor)
                reasonString, _ := reason.Str()
                actors = append(actors, Actor{Id: strings.Split(actor, ":")[0], Reason: reasonString})
        }

        return actors
}

func GeneratePaginationLinks(actors *redis.Reply, limit int, offset int, uri string) map[string]string {
        links := make(map[string]string)
        last_page := len(actors.Elems) / limit
        for i := 1; i < last_page; i++ {
                link := fmt.Sprintf("%s?page=%d", uri, i)
                links[strconv.Itoa(i)] = link
        }
        links[strconv.Itoa(last_page)] = fmt.Sprintf("%s?page=%d", uri, last_page)
        return links
}
<commit_msg>Fix pagination logic<commit_after>package main

import (
        "fmt"
        "strconv"
        "github.com/fzzy/radix/redis"
        "strings"
        "math"
)

func calculateEnd(start int, limit int, size int) int {
        end := start + limit
        if end > size {
                return size
        } else {
                return end
        }
}

func calculateStart(page int, limit int, size int) int {
        if page == 0 || page == 1 {
                return 0
        } else {
                start := page * limit - limit
                if start > size {
                        return 0
                } else {
                        return start
                }
        }
}

func Paginate(configuration *Configuration, reply *redis.Reply, page int, limit int) []Actor {
        connection := connect(configuration.Redis.Host, configuration.Redis.Port)

        var actors []Actor
        start := calculateStart(page, limit, len(reply.Elems))
        end   := calculateEnd(start, limit, len(reply.Elems))
        for i := start; i < end; i++ {
                actor, _ := reply.Elems[i].Str()
                reason := connection.Cmd("GET", actor)
                reasonString, _ := reason.Str()
                actors = append(actors, Actor{Id: strings.Split(actor, ":")[0], Reason: reasonString})
        }

        return actors
}

func GeneratePaginationLinks(actors *redis.Reply, limit int, offset int, uri string) map[string]string {
        links := make(map[string]string)
        last_page := int(math.Ceil(float64(len(actors.Elems)) / float64(limit)))
        for i := 1; i < last_page; i++ {
                link := fmt.Sprintf("%s?page=%d", uri, i)
                links[strconv.Itoa(i)] = link
        }
        links[strconv.Itoa(last_page)] = fmt.Sprintf("%s?page=%d", uri, last_page)
        return links
}
<|endoftext|>


<commit_before>package lib

import (
	"fmt"
)

type AutorizationError string
type NotImplementedByRemoteError string
type MalformedResponseError string
type InvalidStatusCodeError string
type NotFoundError string
type InvalidRequestError string

var genericAuthorizationError AutorizationError = "autorization rejected by registry"
var genericMalformedResponseError MalformedResponseError = "malformed response"

func (e AutorizationError) Error() string {
	return string(e)
}

func (e NotImplementedByRemoteError) Error() string {
	return string(e)
}

func (e MalformedResponseError) Error() string {
	return string(e)
}

func (e InvalidStatusCodeError) Error() string {
	return string(e)
}

func (e NotFoundError) Error() string {
	return string(e)
}

func (e InvalidRequestError) Error() string {
	return string(e)
}

func newInvalidStatusCodeError(code int) error {
	return InvalidStatusCodeError(fmt.Sprintf("invalid API response status %d", code))
}

func newNotFoundError(description string) error {
	return NotFoundError(description)
}

func newInvalidRequestError(description string) error {
	return InvalidRequestError(description)
}
<commit_msg>Typo.<commit_after>package lib

import (
	"fmt"
)

type AutorizationError string
type NotImplementedByRemoteError string
type MalformedResponseError string
type InvalidStatusCodeError string
type NotFoundError string
type InvalidRequestError string

var genericAuthorizationError AutorizationError = "authorization rejected by registry"
var genericMalformedResponseError MalformedResponseError = "malformed response"

func (e AutorizationError) Error() string {
	return string(e)
}

func (e NotImplementedByRemoteError) Error() string {
	return string(e)
}

func (e MalformedResponseError) Error() string {
	return string(e)
}

func (e InvalidStatusCodeError) Error() string {
	return string(e)
}

func (e NotFoundError) Error() string {
	return string(e)
}

func (e InvalidRequestError) Error() string {
	return string(e)
}

func newInvalidStatusCodeError(code int) error {
	return InvalidStatusCodeError(fmt.Sprintf("invalid API response status %d", code))
}

func newNotFoundError(description string) error {
	return NotFoundError(description)
}

func newInvalidRequestError(description string) error {
	return InvalidRequestError(description)
}
<|endoftext|>


<commit_before>/*
Copyright 2016 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Command agent forwards requests from an inverting proxy to a backend server.
//
// To build, run:
//
//    $ make
//
// And to use, run:
//
//    $ $(GOPATH)/bin/proxy-forwarding-agent -proxy <proxy-url> -backend <backend-ID>

package main

import (
	"context"
	"flag"
	"fmt"
	"log"
	"net/http"
	"net/http/cookiejar"
	"net/http/httputil"
	"net/url"
	"strings"
	"time"

	"github.com/golang/groupcache/lru"
	"golang.org/x/net/publicsuffix"
	"golang.org/x/oauth2/google"
	compute "google.golang.org/api/compute/v1"

	"github.com/google/inverting-proxy/agent/banner"
	"github.com/google/inverting-proxy/agent/metrics"
	"github.com/google/inverting-proxy/agent/sessions"
	"github.com/google/inverting-proxy/agent/utils"
	"github.com/google/inverting-proxy/agent/websockets"
)

const (
	requestCacheLimit   = 1000
	emailScope          = "email"
	headerAuthorization = "Authorization"
)

var (
	proxy                     = flag.String("proxy", "", "URL (including scheme) of the inverting proxy")
	proxyTimeout              = flag.Duration("proxy-timeout", 60*time.Second, "Client timeout when sending requests to the inverting proxy")
	host                      = flag.String("host", "localhost:8080", "Hostname (including port) of the backend server")
	backendID                 = flag.String("backend", "", "Unique ID for this backend.")
	debug                     = flag.Bool("debug", false, "Whether or not to print debug log messages")
	disableSSLForTest         = flag.Bool("disable-ssl-for-test", false, "Disable requirements for SSL when running tests locally")
	favIconURL                = flag.String("favicon-url", "", "URL of the favicon")
	forwardUserID             = flag.Bool("forward-user-id", false, "Whether or not to include the ID (email address) of the end user in requests to the backend")
	injectBanner              = flag.String("inject-banner", "", "HTML snippet to inject in served webpages")
	bannerHeight              = flag.String("banner-height", "40px", "Height of the injected banner. This is ignored if no banner is set.")
	shimWebsockets            = flag.Bool("shim-websockets", false, "Whether or not to replace websockets with a shim")
	shimPath                  = flag.String("shim-path", "", "Path under which to handle websocket shim requests")
	healthCheckPath           = flag.String("health-check-path", "/", "Path on backend host to issue health checks against.  Defaults to the root.")
	healthCheckFreq           = flag.Int("health-check-interval-seconds", 0, "Wait time in seconds between health checks.  Set to zero to disable health checks.  Checks disabled by default.")
	healthCheckUnhealthy      = flag.Int("health-check-unhealthy-threshold", 2, "A so-far healthy backend will be marked unhealthy after this many consecutive failures. The minimum value is 1.")
	disableGCEVM              = flag.Bool("disable-gce-vm-header", false, "Disable the agent from adding a GCE VM header.")
	enableWebsocketsInjection = flag.Bool("enable-websockets-injection", false, "Enables the injection of HTTP headers into websocket messages. "+
		"Websocket message injection will inject all headers from the HTTP request to /data and inject them "+
		"into JSON-serialized websocket messages at the JSONPath `resource.headers`")

	sessionCookieName       = flag.String("session-cookie-name", "", "Name of the session cookie; an empty value disables agent-based session tracking")
	sessionCookieTimeout    = flag.Duration("session-cookie-timeout", 12*time.Hour, "Expiration flag for the session cookie")
	sessionCookieCacheLimit = flag.Int("session-cookie-cache-limit", 1000, "Upper bound on the number of concurrent sessions that can be tracked by the agent")
	rewriteWebsocketHost    = flag.Bool("rewrite-websocket-host", false, "Whether to rewrite the Host header to the original request when shimming a websocket connection")
	stripCredentials        = flag.Bool("strip-credentials", false, "Whether to strip the Authorization header from all requests.")

	projectID                = flag.String("monitoring-project-id", "", "Name of the GCP project id")
	metricDomain             = flag.String("metric-domain", "", "Domain under which to write metrics eg. notebooks.googleapis.com")
	monitoringEndpoint       = flag.String("monitoring-endpoint", "monitoring.googleapis.com:443", "The endpoint to which to write metrics. Eg: monitoring.googleapis.com corresponds to Cloud Monarch")
	monitoringResourceType   = flag.String("monitoring-resource-type", "gce_instance", "The monitoring resource type. Eg: gce_instance")
	monitoringResourceLabels = flag.String("monitoring-resource-labels", "", "Comma separated key value pairs specifying the resource labels. Eg: 'instance-id=12345678901234,instance-zone=us-west1-a")

	sessionLRU    *sessions.Cache
	metricHandler *metrics.MetricHandler
)

func hostProxy(ctx context.Context, host, shimPath string, injectShimCode bool) (http.Handler, error) {
	hostProxy := httputil.NewSingleHostReverseProxy(&url.URL{
		Scheme: "http",
		Host:   host,
	})
	hostProxy.FlushInterval = 100 * time.Millisecond
	var h http.Handler = hostProxy
	h = sessionLRU.SessionHandler(h, metricHandler)
	if shimPath != "" {
		var err error
		// Note that we pass in the sessionHandler to the websocket proxy twice (h and sessionLRU.SessionHandler)
		// h is the wrapped handler that will handle all non-websocket-shim requests
		// sessionLRU.SessionHandler will be called for websocket open requests
		// This is necessary to handle an edge case in session handling. Websocket open requests arrive with a path
		// of `/$shimPath/open`. The original target URL and path are encoded in the request body, which is
		// restored in the websocket handler. This means that attempting to restore session cookies that are
		// restricted to a path prefix not equal to "/" will fail for websocket open requests. Passing in the
		// sessionHandler twice allows the websocket handler to ensure that cookies are applied based on the
		// correct, restored path.
		h, err = websockets.Proxy(ctx, h, host, shimPath, *rewriteWebsocketHost, *enableWebsocketsInjection, sessionLRU.SessionHandler, metricHandler)
		if injectShimCode {
			shimFunc, err := websockets.ShimBody(shimPath)
			if err != nil {
				return nil, err
			}
			hostProxy.ModifyResponse = shimFunc
		}
		if err != nil {
			return nil, err
		}
	}
	if *injectBanner == "" {
		return h, nil
	}
	return banner.Proxy(ctx, h, *injectBanner, *bannerHeight, *favIconURL, metricHandler)
}

// forwardRequest forwards the given request from the proxy to
// the backend server and reports the response back to the proxy.
func forwardRequest(client *http.Client, hostProxy http.Handler, request *utils.ForwardedRequest) error {
	httpRequest := request.Contents
	if *forwardUserID {
		httpRequest.Header.Add(utils.HeaderUserID, request.User)
	}
	if *stripCredentials {
		httpRequest.Header.Del(headerAuthorization)
	}
	responseForwarder, err := utils.NewResponseForwarder(client, *proxy, request.BackendID, request.RequestID)
	if err != nil {
		return fmt.Errorf("failed to create the response forwarder: %v", err)
	}
	hostProxy.ServeHTTP(responseForwarder, httpRequest)
	if *debug {
		log.Printf("Backend latency for request %s: %s\n", request.RequestID, time.Since(request.StartTime).String())
	}
	if err := responseForwarder.Close(); err != nil {
		return fmt.Errorf("failed to close the response forwarder: %v", err)
	}
	return nil
}

// healthCheck issues a health check against the backend server
// and returns the result.
func healthCheck() error {
	resp, err := http.Get("http://" + *host + *healthCheckPath)
	if err != nil {
		log.Printf("Health Check request failed: %s", err.Error())
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		log.Printf("Health Check request had non-200 status code: %d", resp.StatusCode)
		return fmt.Errorf("Bad Health Check Response Code: %s", resp.Status)
	}
	return nil
}

// processOneRequest reads a single request from the proxy and forwards it to the backend server.
func processOneRequest(client *http.Client, hostProxy http.Handler, backendID string, requestID string) {
	requestForwarder := func(client *http.Client, request *utils.ForwardedRequest) error {
		if err := forwardRequest(client, hostProxy, request); err != nil {
			log.Printf("Failure forwarding a request: [%s] %q\n", requestID, err.Error())
			return fmt.Errorf("failed to forward the request %q: %v", requestID, err)
		}
		return nil
	}
	if err := utils.ReadRequest(client, *proxy, backendID, requestID, requestForwarder, metricHandler); err != nil {
		log.Printf("Failed to forward a request: [%s] %q\n", requestID, err.Error())
	}
}

// pollForNewRequests repeatedly reaches out to the proxy server to ask if any pending are available, and then
// processes any newly-seen ones.
func pollForNewRequests(client *http.Client, hostProxy http.Handler, backendID string) {
	previouslySeenRequests := lru.New(requestCacheLimit)
	var retryCount uint
	for {
		if requests, err := utils.ListPendingRequests(client, *proxy, backendID, metricHandler); err != nil {
			log.Printf("Failed to read pending requests: %q\n", err.Error())
			time.Sleep(utils.ExponentialBackoffDuration(retryCount))
			retryCount++
		} else {
			retryCount = 0
			for _, requestID := range requests {
				if _, ok := previouslySeenRequests.Get(requestID); !ok {
					previouslySeenRequests.Add(requestID, requestID)
					go processOneRequest(client, hostProxy, backendID, requestID)
				}
			}
		}
	}
}

func getGoogleClient(ctx context.Context) (*http.Client, error) {
	sdkConfig, err := google.NewSDKConfig("")
	if err == nil {
		return sdkConfig.Client(ctx), nil
	}

	client, err := google.DefaultClient(ctx, compute.CloudPlatformScope, emailScope)
	if err != nil {
		return nil, err
	}
	client.Transport = utils.RoundTripperWithVMIdentity(ctx, client.Transport, *proxy, *disableGCEVM)
	client.Jar, err = cookiejar.New(&cookiejar.Options{
		PublicSuffixList: publicsuffix.List,
	})
	if err != nil {
		return nil, err
	}
	return client, nil
}

// waitForHealthy runs health checks against the backend and returns
// the first time it sees a healthy check.
func waitForHealthy() {
	if *healthCheckFreq <= 0 {
		return
	}
	if healthCheck() == nil {
		return
	}
	ticker := time.NewTicker(time.Duration(*healthCheckFreq) * time.Second)
	for _ = range ticker.C {
		if healthCheck() == nil {
			ticker.Stop()
			return
		}
	}
}

// runHealthChecks runs health checks against the backend and shuts down
// the proxy if the backend is unhealthy.
func runHealthChecks() {
	if *healthCheckFreq <= 0 {
		return
	}
	if *healthCheckUnhealthy < 1 {
		*healthCheckUnhealthy = 1
	}
	// Always start in the unhealthy state, but only require a single positive
	// health check to become healthy for the first time, and do the first check
	// immediately.
	ticker := time.NewTicker(time.Duration(*healthCheckFreq) * time.Second)
	badHealthChecks := 0
	for _ = range ticker.C {
		if healthCheck() != nil {
			badHealthChecks++
		} else {
			badHealthChecks = 0
		}
		if badHealthChecks >= *healthCheckUnhealthy {
			ticker.Stop()
			log.Fatal("Too many unhealthy checks")
		}
	}
}

// runAdapter sets up the HTTP client for the agent to use (including OAuth credentials),
// and then does the actual work of forwarding requests and responses.
func runAdapter(ctx context.Context) error {
	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	client, err := getGoogleClient(ctx)
	if err != nil {
		return err
	}
	client.Timeout = *proxyTimeout

	hostProxy, err := hostProxy(ctx, *host, *shimPath, *shimWebsockets)
	if err != nil {
		return err
	}
	pollForNewRequests(client, hostProxy, *backendID)
	return nil
}

func main() {
	flag.Parse()
	ctx := context.Background()

	if *proxy == "" {
		log.Fatal("You must specify the address of the proxy")
	}
	if *backendID == "" {
		log.Fatal("You must specify a backend ID")
	}
	if !strings.HasPrefix(*healthCheckPath, "/") {
		*healthCheckPath = "/" + *healthCheckPath
	}
	if *sessionCookieName != "" {
		sessionLRU = sessions.NewCache(*sessionCookieName, *sessionCookieTimeout, *sessionCookieCacheLimit, *disableSSLForTest)
	}
	mh, err := metrics.NewMetricHandler(ctx, *projectID, *monitoringResourceType, *monitoringResourceLabels, *metricDomain, *monitoringEndpoint)
	metricHandler = mh
	if err != nil {
		log.Printf("Unable to create metric handler: %v", err)
	}

	waitForHealthy()
	go runHealthChecks()

	if err := runAdapter(ctx); err != nil {
		log.Fatal(err.Error())
	}
}
<commit_msg>Update agent.go<commit_after>/*
Copyright 2016 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Command agent forwards requests from an inverting proxy to a backend server.
//
// To build, run:
//
//    $ make
//
// And to use, run:
//
//    $ $(GOPATH)/bin/proxy-forwarding-agent -proxy <proxy-url> -backend <backend-ID>

package main

import (
	"context"
	"flag"
	"fmt"
	"log"
	"net/http"
	"net/http/cookiejar"
	"net/http/httputil"
	"net/url"
	"strings"
	"time"

	"github.com/golang/groupcache/lru"
	"golang.org/x/net/publicsuffix"
	"golang.org/x/oauth2/google"
	compute "google.golang.org/api/compute/v1"

	"github.com/google/inverting-proxy/agent/banner"
	"github.com/google/inverting-proxy/agent/metrics"
	"github.com/google/inverting-proxy/agent/sessions"
	"github.com/google/inverting-proxy/agent/utils"
	"github.com/google/inverting-proxy/agent/websockets"
)

const (
	requestCacheLimit   = 1000
	emailScope          = "email"
	headerAuthorization = "Authorization"
)

var (
	proxy                     = flag.String("proxy", "", "URL (including scheme) of the inverting proxy")
	proxyTimeout              = flag.Duration("proxy-timeout", 60*time.Second, "Client timeout when sending requests to the inverting proxy")
	host                      = flag.String("host", "localhost:8080", "Hostname (including port) of the backend server")
	backendID                 = flag.String("backend", "", "Unique ID for this backend.")
	debug                     = flag.Bool("debug", false, "Whether or not to print debug log messages")
	disableSSLForTest         = flag.Bool("disable-ssl-for-test", false, "Disable requirements for SSL when running tests locally")
	favIconURL                = flag.String("favicon-url", "", "URL of the favicon")
	forwardUserID             = flag.Bool("forward-user-id", false, "Whether or not to include the ID (email address) of the end user in requests to the backend")
	injectBanner              = flag.String("inject-banner", "", "HTML snippet to inject in served webpages")
	bannerHeight              = flag.String("banner-height", "40px", "Height of the injected banner. This is ignored if no banner is set.")
	shimWebsockets            = flag.Bool("shim-websockets", false, "Whether or not to replace websockets with a shim")
	shimPath                  = flag.String("shim-path", "", "Path under which to handle websocket shim requests")
	healthCheckPath           = flag.String("health-check-path", "/", "Path on backend host to issue health checks against.  Defaults to the root.")
	healthCheckFreq           = flag.Int("health-check-interval-seconds", 0, "Wait time in seconds between health checks.  Set to zero to disable health checks.  Checks disabled by default.")
	healthCheckUnhealthy      = flag.Int("health-check-unhealthy-threshold", 2, "A so-far healthy backend will be marked unhealthy after this many consecutive failures. The minimum value is 1.")
	disableGCEVM              = flag.Bool("disable-gce-vm-header", false, "Disable the agent from adding a GCE VM header.")
	enableWebsocketsInjection = flag.Bool("enable-websockets-injection", false, "Enables the injection of HTTP headers into websocket messages. "+
		"Websocket message injection will inject all headers from the HTTP request to /data and inject them "+
		"into JSON-serialized websocket messages at the JSONPath `resource.headers`")

	sessionCookieName       = flag.String("session-cookie-name", "", "Name of the session cookie; an empty value disables agent-based session tracking")
	sessionCookieTimeout    = flag.Duration("session-cookie-timeout", 12*time.Hour, "Expiration flag for the session cookie")
	sessionCookieCacheLimit = flag.Int("session-cookie-cache-limit", 1000, "Upper bound on the number of concurrent sessions that can be tracked by the agent")
	rewriteWebsocketHost    = flag.Bool("rewrite-websocket-host", false, "Whether to rewrite the Host header to the original request when shimming a websocket connection")
	stripCredentials        = flag.Bool("strip-credentials", false, "Whether to strip the Authorization header from all requests.")

	projectID                = flag.String("monitoring-project-id", "", "Name of the GCP project id")
	metricDomain             = flag.String("metric-domain", "", "Domain under which to write metrics eg. notebooks.googleapis.com")
	monitoringEndpoint       = flag.String("monitoring-endpoint", "monitoring.googleapis.com:443", "The endpoint to which to write metrics. Eg: monitoring.googleapis.com corresponds to Cloud Monarch")
	monitoringResourceType   = flag.String("monitoring-resource-type", "gce_instance", "The monitoring resource type. Eg: gce_instance")
	monitoringResourceLabels = flag.String("monitoring-resource-labels", "", "Comma separated key value pairs specifying the resource labels. Eg: 'instance-id=12345678901234,instance-zone=us-west1-a")

	sessionLRU    *sessions.Cache
	metricHandler *metrics.MetricHandler
)

func hostProxy(ctx context.Context, host, shimPath string, injectShimCode bool) (http.Handler, error) {
	hostProxy := httputil.NewSingleHostReverseProxy(&url.URL{
		Scheme: "http",
		Host:   host,
	})
	hostProxy.FlushInterval = 100 * time.Millisecond
	var h http.Handler = hostProxy
	h = sessionLRU.SessionHandler(h, metricHandler)
	if shimPath != "" {
		var err error
		// Note that we pass in the sessionHandler to the websocket proxy twice (h and sessionLRU.SessionHandler)
		// h is the wrapped handler that will handle all non-websocket-shim requests
		// sessionLRU.SessionHandler will be called for websocket open requests
		// This is necessary to handle an edge case in session handling. Websocket open requests arrive with a path
		// of `/$shimPath/open`. The original target URL and path are encoded in the request body, which is
		// restored in the websocket handler. This means that attempting to restore session cookies that are
		// restricted to a path prefix not equal to "/" will fail for websocket open requests. Passing in the
		// sessionHandler twice allows the websocket handler to ensure that cookies are applied based on the
		// correct, restored path.
		h, err = websockets.Proxy(ctx, h, host, shimPath, *rewriteWebsocketHost, *enableWebsocketsInjection, sessionLRU.SessionHandler, metricHandler)
		if injectShimCode {
			shimFunc, err := websockets.ShimBody(shimPath)
			if err != nil {
				return nil, err
			}
			hostProxy.ModifyResponse = shimFunc
		}
		if err != nil {
			return nil, err
		}
	}
	if *injectBanner == "" {
		return h, nil
	}
	return banner.Proxy(ctx, h, *injectBanner, *bannerHeight, *favIconURL, metricHandler)
}

// forwardRequest forwards the given request from the proxy to
// the backend server and reports the response back to the proxy.
func forwardRequest(client *http.Client, hostProxy http.Handler, request *utils.ForwardedRequest) error {
	httpRequest := request.Contents
	if *debug {
		log.Printf("Request %s: %s %s %v\n", request.RequestID, request.Contents.Method, request.Contents.Host, request.Contents.ContentLength)
	}
	if *forwardUserID {
		httpRequest.Header.Add(utils.HeaderUserID, request.User)
	}
	if *stripCredentials {
		httpRequest.Header.Del(headerAuthorization)
	}
	responseForwarder, err := utils.NewResponseForwarder(client, *proxy, request.BackendID, request.RequestID)
	if err != nil {
		return fmt.Errorf("failed to create the response forwarder: %v", err)
	}
	hostProxy.ServeHTTP(responseForwarder, httpRequest)
	if *debug {
		log.Printf("Backend latency for request %s: %s\n", request.RequestID, time.Since(request.StartTime).String())
	}
	if err := responseForwarder.Close(); err != nil {
		return fmt.Errorf("failed to close the response forwarder: %v", err)
	}
	return nil
}

// healthCheck issues a health check against the backend server
// and returns the result.
func healthCheck() error {
	resp, err := http.Get("http://" + *host + *healthCheckPath)
	if err != nil {
		log.Printf("Health Check request failed: %s", err.Error())
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		log.Printf("Health Check request had non-200 status code: %d", resp.StatusCode)
		return fmt.Errorf("Bad Health Check Response Code: %s", resp.Status)
	}
	return nil
}

// processOneRequest reads a single request from the proxy and forwards it to the backend server.
func processOneRequest(client *http.Client, hostProxy http.Handler, backendID string, requestID string) {
	requestForwarder := func(client *http.Client, request *utils.ForwardedRequest) error {
		if err := forwardRequest(client, hostProxy, request); err != nil {
			log.Printf("Failure forwarding a request: [%s] %q\n", requestID, err.Error())
			return fmt.Errorf("failed to forward the request %q: %v", requestID, err)
		}
		return nil
	}
	if err := utils.ReadRequest(client, *proxy, backendID, requestID, requestForwarder, metricHandler); err != nil {
		log.Printf("Failed to forward a request: [%s] %q\n", requestID, err.Error())
	}
}

// pollForNewRequests repeatedly reaches out to the proxy server to ask if any pending are available, and then
// processes any newly-seen ones.
func pollForNewRequests(client *http.Client, hostProxy http.Handler, backendID string) {
	previouslySeenRequests := lru.New(requestCacheLimit)
	var retryCount uint
	for {
		if requests, err := utils.ListPendingRequests(client, *proxy, backendID, metricHandler); err != nil {
			log.Printf("Failed to read pending requests: %q\n", err.Error())
			time.Sleep(utils.ExponentialBackoffDuration(retryCount))
			retryCount++
		} else {
			retryCount = 0
			for _, requestID := range requests {
				if _, ok := previouslySeenRequests.Get(requestID); !ok {
					previouslySeenRequests.Add(requestID, requestID)
					go processOneRequest(client, hostProxy, backendID, requestID)
				}
			}
		}
	}
}

func getGoogleClient(ctx context.Context) (*http.Client, error) {
	sdkConfig, err := google.NewSDKConfig("")
	if err == nil {
		return sdkConfig.Client(ctx), nil
	}

	client, err := google.DefaultClient(ctx, compute.CloudPlatformScope, emailScope)
	if err != nil {
		return nil, err
	}
	client.Transport = utils.RoundTripperWithVMIdentity(ctx, client.Transport, *proxy, *disableGCEVM)
	client.Jar, err = cookiejar.New(&cookiejar.Options{
		PublicSuffixList: publicsuffix.List,
	})
	if err != nil {
		return nil, err
	}
	return client, nil
}

// waitForHealthy runs health checks against the backend and returns
// the first time it sees a healthy check.
func waitForHealthy() {
	if *healthCheckFreq <= 0 {
		return
	}
	if healthCheck() == nil {
		return
	}
	ticker := time.NewTicker(time.Duration(*healthCheckFreq) * time.Second)
	for _ = range ticker.C {
		if healthCheck() == nil {
			ticker.Stop()
			return
		}
	}
}

// runHealthChecks runs health checks against the backend and shuts down
// the proxy if the backend is unhealthy.
func runHealthChecks() {
	if *healthCheckFreq <= 0 {
		return
	}
	if *healthCheckUnhealthy < 1 {
		*healthCheckUnhealthy = 1
	}
	// Always start in the unhealthy state, but only require a single positive
	// health check to become healthy for the first time, and do the first check
	// immediately.
	ticker := time.NewTicker(time.Duration(*healthCheckFreq) * time.Second)
	badHealthChecks := 0
	for _ = range ticker.C {
		if healthCheck() != nil {
			badHealthChecks++
		} else {
			badHealthChecks = 0
		}
		if badHealthChecks >= *healthCheckUnhealthy {
			ticker.Stop()
			log.Fatal("Too many unhealthy checks")
		}
	}
}

// runAdapter sets up the HTTP client for the agent to use (including OAuth credentials),
// and then does the actual work of forwarding requests and responses.
func runAdapter(ctx context.Context) error {
	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	client, err := getGoogleClient(ctx)
	if err != nil {
		return err
	}
	client.Timeout = *proxyTimeout

	hostProxy, err := hostProxy(ctx, *host, *shimPath, *shimWebsockets)
	if err != nil {
		return err
	}
	pollForNewRequests(client, hostProxy, *backendID)
	return nil
}

func main() {
	flag.Parse()
	ctx := context.Background()

	if *proxy == "" {
		log.Fatal("You must specify the address of the proxy")
	}
	if *backendID == "" {
		log.Fatal("You must specify a backend ID")
	}
	if !strings.HasPrefix(*healthCheckPath, "/") {
		*healthCheckPath = "/" + *healthCheckPath
	}
	if *sessionCookieName != "" {
		sessionLRU = sessions.NewCache(*sessionCookieName, *sessionCookieTimeout, *sessionCookieCacheLimit, *disableSSLForTest)
	}
	mh, err := metrics.NewMetricHandler(ctx, *projectID, *monitoringResourceType, *monitoringResourceLabels, *metricDomain, *monitoringEndpoint)
	metricHandler = mh
	if err != nil {
		log.Printf("Unable to create metric handler: %v", err)
	}

	waitForHealthy()
	go runHealthChecks()

	if err := runAdapter(ctx); err != nil {
		log.Fatal(err.Error())
	}
}
<|endoftext|>


<commit_before>package main

import (
    "os"
    "bufio"
    "fmt"
    "io"
    "math"
    "regexp"
    "runtime/pprof"
    "sort"
    "strconv"
    "strings"
)

const (
   useProfile = false
   // for Nginx($request_time)
   SCALE = 0
   EFFECTIVE_DIGIT = 3
   // for Apache(%D)
   // SCALE = -6
   // EFFECTIVE_DIGIT = 6
)

var (
    topCount = 10
    urlNormalizes = []string{
      "^GET /memo/[0-9]+$",
      "^GET /stylesheets/",
      "^GET /images/",
    }
)

type Measure struct {
  Url string
  Count int
  Total float64
  Min float64
  Mean float64
  Median float64
  P90 float64
  P95 float64
  P99 float64
  Max float64
}

type By func(a, b *Measure) bool

func (by By) Sort(measures []*Measure) {
  ms := &measureSorter{
    measures: measures,
    by: by,
  }
  sort.Sort(ms)
}

type measureSorter struct {
  measures []*Measure
  by func(a, b *Measure) bool
}

func (s *measureSorter) Len() int {
  return len(s.measures)
}

func (s *measureSorter) Swap(i, j int) {
  s.measures[i], s.measures[j] = s.measures[j], s.measures[i]
}

func (s *measureSorter) Less(i, j int) bool {
  return s.by(s.measures[i], s.measures[j])
}

type Column struct {
  Name string
  Summary string
  Sort By
}

var (
    columns = []*Column{
      &Column{ Name: "Count", Summary: "Count", Sort: func(a, b *Measure) bool { return a.Count > b.Count } },
      &Column{ Name: "Total", Summary: "Total", Sort: func(a, b *Measure) bool { return a.Total > b.Total } },
      &Column{ Name: "Mean", Summary: "Mean", Sort: func(a, b *Measure) bool { return a.Mean > b.Mean } },
      &Column{ Name: "Min", Summary: "Minimum(0 Percentile)", Sort: func(a, b *Measure) bool { return a.Min > b.Min } },
      &Column{ Name: "Median", Summary: "Median(50 Percentile)", Sort: func(a, b *Measure) bool { return a.Median > b.Median } },
      &Column{ Name: "P90", Summary: "90 Percentile", Sort: func(a, b *Measure) bool { return a.P90 > b.P90 } },
      &Column{ Name: "P95", Summary: "95 Percentile", Sort: func(a, b *Measure) bool { return a.P95 > b.P95 } },
      &Column{ Name: "P99", Summary: "99 Percentile", Sort: func(a, b *Measure) bool { return a.P99 > b.P99 } },
      &Column{ Name: "Max", Summary: "Maximum(100 Percentile)", Sort: func(a, b *Measure) bool { return a.Max > b.Max } },
    }
)

func showMeasures(measures []*Measure) {
  countWidth := 5 // for title
  totalWidth := 2 + EFFECTIVE_DIGIT
  meanWidth := 2 + EFFECTIVE_DIGIT * 2
  maxWidth := 2 + EFFECTIVE_DIGIT

  for i := 0; i < topCount; i++ {
    if countWidth < int(math.Log10(float64(measures[i].Count)) + 1) {
      countWidth = int(math.Log10(float64(measures[i].Count)) + 1)
    }
    if totalWidth < int(math.Log10(measures[i].Total) + 1 + EFFECTIVE_DIGIT + 1) {
      totalWidth = int(math.Log10(measures[i].Total) + 1 + EFFECTIVE_DIGIT + 1)
    }
    if meanWidth < int(math.Log10(measures[i].Max) + 1 + EFFECTIVE_DIGIT * 2 + 1) {
      meanWidth = int(math.Log10(measures[i].Max) + 1 + EFFECTIVE_DIGIT * 2 + 1)
    }
    if maxWidth < int(math.Log10(measures[i].Max) + 1 + EFFECTIVE_DIGIT + 1) {
      maxWidth = int(math.Log10(measures[i].Max) + 1 + EFFECTIVE_DIGIT + 1)
    }
  }

  var format string
  for _, column := range columns {
    switch column.Name {
    case "Count":
      fmt.Printf(fmt.Sprintf("%%%ds  ", countWidth), column.Name)
      format += fmt.Sprintf("%%%dd  ", countWidth)
    case "Total":
      fmt.Printf(fmt.Sprintf("%%%ds  ", totalWidth), column.Name)
      format += fmt.Sprintf("%%%d.%df  ", totalWidth, EFFECTIVE_DIGIT)
    case "Mean":
      fmt.Printf(fmt.Sprintf("%%%ds  ", meanWidth), column.Name)
      format += fmt.Sprintf("%%%d.%df  ", meanWidth, EFFECTIVE_DIGIT * 2)
    default:
      fmt.Printf(fmt.Sprintf("%%%ds  ", maxWidth), column.Name)
      format += fmt.Sprintf("%%%d.%df  ", maxWidth, EFFECTIVE_DIGIT)
    }
  }
  fmt.Printf("url\n")
  format += "%s\n"

  for i := 0; i < topCount; i++ {
    m := measures[i]
    fmt.Printf(format, m.Count, m.Total, m.Mean, m.Min, m.Median, m.P90, m.P95, m.P99, m.Max, m.Url)
  }
}

func main() {
    if (useProfile) {
        f, err := os.Create("/tmp/parse_access_log.prof")
        if err != nil {
            panic(err)
        }
        pprof.StartCPUProfile(f)
        defer pprof.StopCPUProfile()
    }
    reader := bufio.NewReaderSize(os.Stdin, 4096)
    scale := math.Pow10(SCALE)

    var urlNormalizeRegexps []*regexp.Regexp
    for _, str := range urlNormalizes {
	re := regexp.MustCompile(str)
        urlNormalizeRegexps = append(urlNormalizeRegexps, re)
    }

    totals := make(map[string]float64)
    times := make(map[string][]float64)
    for {
        line, err := reader.ReadString('\n')
        if err == io.EOF {
          break
        } else if err != nil {
          panic(err)
        }
        s := strings.Split(line, " ")
        if len(s) >= 7 {
          url := strings.TrimLeft(strings.Join(s[5:7], " "), "\"")
          for _, re := range urlNormalizeRegexps {
            if re.MatchString(url) {
              url = re.String()
              break
            }
          }
          time, err := strconv.ParseFloat(strings.Trim(s[len(s)-1], "\r\n"), 10)
          if err == nil {
            time = time * scale
          } else {
            time = 0.000
          }
          totals[url] += time
          times[url] = append(times[url], time)
        }
    }

    var measures []*Measure
    for url, total := range totals {
      sorted := times[url]
      sort.Float64s(sorted)
      count := len(sorted)
      measure := &Measure{
        Url: url,
        Count: count,
        Total: total,
        Min: sorted[0],
        Mean: totals[url]/float64(count),
        Median: sorted[int(count*50/100)],
        P90: sorted[int(count*90/100)],
        P95: sorted[int(count*95/100)],
        P99: sorted[int(count*99/100)],
        Max: sorted[count-1],
      }
      measures = append(measures, measure)
    }
    if len(measures) < topCount {
      topCount = len(measures)
    }

    for _, column := range columns {
      fmt.Printf("Sort By %s\n", column.Summary)
      By(column.Sort).Sort(measures)
      showMeasures(measures)
      fmt.Println()
    }
}
<commit_msg>goroutineを用いて高速化<commit_after>package main

import (
    "os"
    "bufio"
    "fmt"
    "io"
    "math"
    "regexp"
    "runtime"
    "runtime/pprof"
    "sort"
    "strconv"
    "strings"
    "sync"
)

const (
   useProfile = false
   // for Nginx($request_time)
   SCALE = 0
   EFFECTIVE_DIGIT = 3
   // for Apache(%D)
   // SCALE = -6
   // EFFECTIVE_DIGIT = 6
)

var (
    topCount = 10
    urlNormalizes = []string{
      "^GET /memo/[0-9]+$",
      "^GET /stylesheets/",
      "^GET /images/",
    }
)

type Measure struct {
  Url string
  Count int
  Total float64
  Min float64
  Mean float64
  Median float64
  P90 float64
  P95 float64
  P99 float64
  Max float64
}

type By func(a, b *Measure) bool

func (by By) Sort(measures []*Measure) {
  ms := &measureSorter{
    measures: measures,
    by: by,
  }
  sort.Sort(ms)
}

type measureSorter struct {
  measures []*Measure
  by func(a, b *Measure) bool
}

func (s *measureSorter) Len() int {
  return len(s.measures)
}

func (s *measureSorter) Swap(i, j int) {
  s.measures[i], s.measures[j] = s.measures[j], s.measures[i]
}

func (s *measureSorter) Less(i, j int) bool {
  return s.by(s.measures[i], s.measures[j])
}

type Column struct {
  Name string
  Summary string
  Sort By
}

var (
    columns = []*Column{
      &Column{ Name: "Count", Summary: "Count", Sort: func(a, b *Measure) bool { return a.Count > b.Count } },
      &Column{ Name: "Total", Summary: "Total", Sort: func(a, b *Measure) bool { return a.Total > b.Total } },
      &Column{ Name: "Mean", Summary: "Mean", Sort: func(a, b *Measure) bool { return a.Mean > b.Mean } },
      &Column{ Name: "Min", Summary: "Minimum(0 Percentile)", Sort: func(a, b *Measure) bool { return a.Min > b.Min } },
      &Column{ Name: "Median", Summary: "Median(50 Percentile)", Sort: func(a, b *Measure) bool { return a.Median > b.Median } },
      &Column{ Name: "P90", Summary: "90 Percentile", Sort: func(a, b *Measure) bool { return a.P90 > b.P90 } },
      &Column{ Name: "P95", Summary: "95 Percentile", Sort: func(a, b *Measure) bool { return a.P95 > b.P95 } },
      &Column{ Name: "P99", Summary: "99 Percentile", Sort: func(a, b *Measure) bool { return a.P99 > b.P99 } },
      &Column{ Name: "Max", Summary: "Maximum(100 Percentile)", Sort: func(a, b *Measure) bool { return a.Max > b.Max } },
    }
)

type Time struct {
  Url string
  Time float64
}

func showMeasures(measures []*Measure) {
  countWidth := 5 // for title
  totalWidth := 2 + EFFECTIVE_DIGIT
  meanWidth := 2 + EFFECTIVE_DIGIT * 2
  maxWidth := 2 + EFFECTIVE_DIGIT

  for i := 0; i < topCount; i++ {
    if countWidth < int(math.Log10(float64(measures[i].Count)) + 1) {
      countWidth = int(math.Log10(float64(measures[i].Count)) + 1)
    }
    if totalWidth < int(math.Log10(measures[i].Total) + 1 + EFFECTIVE_DIGIT + 1) {
      totalWidth = int(math.Log10(measures[i].Total) + 1 + EFFECTIVE_DIGIT + 1)
    }
    if meanWidth < int(math.Log10(measures[i].Max) + 1 + EFFECTIVE_DIGIT * 2 + 1) {
      meanWidth = int(math.Log10(measures[i].Max) + 1 + EFFECTIVE_DIGIT * 2 + 1)
    }
    if maxWidth < int(math.Log10(measures[i].Max) + 1 + EFFECTIVE_DIGIT + 1) {
      maxWidth = int(math.Log10(measures[i].Max) + 1 + EFFECTIVE_DIGIT + 1)
    }
  }

  var format string
  for _, column := range columns {
    switch column.Name {
    case "Count":
      fmt.Printf(fmt.Sprintf("%%%ds  ", countWidth), column.Name)
      format += fmt.Sprintf("%%%dd  ", countWidth)
    case "Total":
      fmt.Printf(fmt.Sprintf("%%%ds  ", totalWidth), column.Name)
      format += fmt.Sprintf("%%%d.%df  ", totalWidth, EFFECTIVE_DIGIT)
    case "Mean":
      fmt.Printf(fmt.Sprintf("%%%ds  ", meanWidth), column.Name)
      format += fmt.Sprintf("%%%d.%df  ", meanWidth, EFFECTIVE_DIGIT * 2)
    default:
      fmt.Printf(fmt.Sprintf("%%%ds  ", maxWidth), column.Name)
      format += fmt.Sprintf("%%%d.%df  ", maxWidth, EFFECTIVE_DIGIT)
    }
  }
  fmt.Printf("url\n")
  format += "%s\n"

  for i := 0; i < topCount; i++ {
    m := measures[i]
    fmt.Printf(format, m.Count, m.Total, m.Mean, m.Min, m.Median, m.P90, m.P95, m.P99, m.Max, m.Url)
  }
}

func main() {
    runtime.GOMAXPROCS(runtime.NumCPU())
    if (useProfile) {
        f, err := os.Create("/tmp/parse_access_log.prof")
        if err != nil {
            panic(err)
        }
        pprof.StartCPUProfile(f)
        defer pprof.StopCPUProfile()
    }
    reader := bufio.NewReaderSize(os.Stdin, 4096)
    scale := math.Pow10(SCALE)

    var urlNormalizeRegexps []*regexp.Regexp
    for _, str := range urlNormalizes {
	re := regexp.MustCompile(str)
        urlNormalizeRegexps = append(urlNormalizeRegexps, re)
    }

    ch := make(chan *Time)
    totals := make(map[string]float64)
    times := make(map[string][]float64)
    go func() {
      for time := range ch {
        totals[time.Url] += time.Time
        times[time.Url] = append(times[time.Url], time.Time)
      }
    }()

    var wg sync.WaitGroup
    for {
        line, err := reader.ReadString('\n')
        if err == io.EOF {
          wg.Wait()
          close(ch)
          break
        } else if err != nil {
          panic(err)
        }
        wg.Add(1)
        go func(line string) {
          defer wg.Done()
          s := strings.Split(line, " ")
          if len(s) >= 7 {
            url := strings.TrimLeft(strings.Join(s[5:7], " "), "\"")
            for _, re := range urlNormalizeRegexps {
              if re.MatchString(url) {
                url = re.String()
                break
              }
            }
            time, err := strconv.ParseFloat(strings.Trim(s[len(s)-1], "\r\n"), 10)
            if err == nil {
              time = time * scale
            } else {
              time = 0.000
            }
            ch <- &Time{Url: url, Time: time}
          }
        }(line)
    }

    var measures []*Measure
    for url, total := range totals {
      sorted := times[url]
      sort.Float64s(sorted)
      count := len(sorted)
      measure := &Measure{
        Url: url,
        Count: count,
        Total: total,
        Min: sorted[0],
        Mean: totals[url]/float64(count),
        Median: sorted[int(count*50/100)],
        P90: sorted[int(count*90/100)],
        P95: sorted[int(count*95/100)],
        P99: sorted[int(count*99/100)],
        Max: sorted[count-1],
      }
      measures = append(measures, measure)
    }
    if len(measures) < topCount {
      topCount = len(measures)
    }

    for _, column := range columns {
      fmt.Printf("Sort By %s\n", column.Summary)
      By(column.Sort).Sort(measures)
      showMeasures(measures)
      fmt.Println()
    }
}
<|endoftext|>


<commit_before>package agent

import (
	"fmt"
	"io"
	"net"
	"net/http"
	"sync"
	"time"

	"github.com/armon/go-metrics/prometheus"
	"github.com/hashicorp/go-hclog"
	"google.golang.org/grpc/grpclog"
	grpcresolver "google.golang.org/grpc/resolver"

	autoconf "github.com/hashicorp/consul/agent/auto-config"
	"github.com/hashicorp/consul/agent/cache"
	"github.com/hashicorp/consul/agent/config"
	"github.com/hashicorp/consul/agent/consul"
	"github.com/hashicorp/consul/agent/consul/fsm"
	"github.com/hashicorp/consul/agent/consul/usagemetrics"
	"github.com/hashicorp/consul/agent/grpc"
	"github.com/hashicorp/consul/agent/grpc/resolver"
	"github.com/hashicorp/consul/agent/local"
	"github.com/hashicorp/consul/agent/pool"
	"github.com/hashicorp/consul/agent/router"
	"github.com/hashicorp/consul/agent/submatview"
	"github.com/hashicorp/consul/agent/token"
	"github.com/hashicorp/consul/ipaddr"
	"github.com/hashicorp/consul/lib"
	"github.com/hashicorp/consul/logging"
	"github.com/hashicorp/consul/tlsutil"
)

// TODO: BaseDeps should be renamed in the future once more of Agent.Start
// has been moved out in front of Agent.New, and we can better see the setup
// dependencies.
type BaseDeps struct {
	consul.Deps // TODO: un-embed

	RuntimeConfig  *config.RuntimeConfig
	MetricsHandler MetricsHandler
	AutoConfig     *autoconf.AutoConfig // TODO: use an interface
	Cache          *cache.Cache
	ViewStore      *submatview.Store
}

// MetricsHandler provides an http.Handler for displaying metrics.
type MetricsHandler interface {
	DisplayMetrics(resp http.ResponseWriter, req *http.Request) (interface{}, error)
}

type ConfigLoader func(source config.Source) (config.LoadResult, error)

func NewBaseDeps(configLoader ConfigLoader, logOut io.Writer) (BaseDeps, error) {
	d := BaseDeps{}
	result, err := configLoader(nil)
	if err != nil {
		return d, err
	}

	cfg := result.RuntimeConfig
	logConf := cfg.Logging
	logConf.Name = logging.Agent
	d.Logger, err = logging.Setup(logConf, logOut)
	if err != nil {
		return d, err
	}
	grpclog.SetLoggerV2(logging.NewGRPCLogger(cfg.Logging.LogLevel, d.Logger))

	for _, w := range result.Warnings {
		d.Logger.Warn(w)
	}

	cfg.NodeID, err = newNodeIDFromConfig(cfg, d.Logger)
	if err != nil {
		return d, fmt.Errorf("failed to setup node ID: %w", err)
	}

	gauges, counters, summaries := getPrometheusDefs(cfg.Telemetry)
	cfg.Telemetry.PrometheusOpts.GaugeDefinitions = gauges
	cfg.Telemetry.PrometheusOpts.CounterDefinitions = counters
	cfg.Telemetry.PrometheusOpts.SummaryDefinitions = summaries
	d.MetricsHandler, err = lib.InitTelemetry(cfg.Telemetry)
	if err != nil {
		return d, fmt.Errorf("failed to initialize telemetry: %w", err)
	}

	d.TLSConfigurator, err = tlsutil.NewConfigurator(cfg.ToTLSUtilConfig(), d.Logger)
	if err != nil {
		return d, err
	}

	d.RuntimeConfig = cfg
	d.Tokens = new(token.Store)

	cfg.Cache.Logger = d.Logger.Named("cache")
	// cache-types are not registered yet, but they won't be used until the components are started.
	d.Cache = cache.New(cfg.Cache)
	d.ViewStore = submatview.NewStore(d.Logger.Named("viewstore"))
	d.ConnPool = newConnPool(cfg, d.Logger, d.TLSConfigurator)

	builder := resolver.NewServerResolverBuilder(resolver.Config{})
	registerWithGRPC(builder)
	d.GRPCConnPool = grpc.NewClientConnPool(builder, grpc.TLSWrapper(d.TLSConfigurator.OutgoingRPCWrapper()), d.TLSConfigurator.UseTLS)

	d.Router = router.NewRouter(d.Logger, cfg.Datacenter, fmt.Sprintf("%s.%s", cfg.NodeName, cfg.Datacenter), builder)

	acConf := autoconf.Config{
		DirectRPC:       d.ConnPool,
		Logger:          d.Logger,
		Loader:          configLoader,
		ServerProvider:  d.Router,
		TLSConfigurator: d.TLSConfigurator,
		Cache:           d.Cache,
		Tokens:          d.Tokens,
	}
	d.AutoConfig, err = autoconf.New(acConf)
	if err != nil {
		return d, err
	}

	return d, nil
}

func newConnPool(config *config.RuntimeConfig, logger hclog.Logger, tls *tlsutil.Configurator) *pool.ConnPool {
	var rpcSrcAddr *net.TCPAddr
	if !ipaddr.IsAny(config.RPCBindAddr) {
		rpcSrcAddr = &net.TCPAddr{IP: config.RPCBindAddr.IP}
	}

	pool := &pool.ConnPool{
		Server:          config.ServerMode,
		SrcAddr:         rpcSrcAddr,
		Logger:          logger.StandardLogger(&hclog.StandardLoggerOptions{InferLevels: true}),
		TLSConfigurator: tls,
		Datacenter:      config.Datacenter,
	}
	if config.ServerMode {
		pool.MaxTime = 2 * time.Minute
		pool.MaxStreams = 64
	} else {
		// MaxTime controls how long we keep an idle connection open to a server.
		// 127s was chosen as the first prime above 120s
		// (arbitrarily chose to use a prime) with the intent of reusing
		// connections who are used by once-a-minute cron(8) jobs *and* who
		// use a 60s jitter window (e.g. in vixie cron job execution can
		// drift by up to 59s per job, or 119s for a once-a-minute cron job).
		pool.MaxTime = 127 * time.Second
		pool.MaxStreams = 32
	}
	return pool
}

var registerLock sync.Mutex

// registerWithGRPC registers the grpc/resolver.Builder as a grpc/resolver.
// This function exists to synchronize registrations with a lock.
// grpc/resolver.Register expects all registration to happen at init and does
// not allow for concurrent registration. This function exists to support
// parallel testing.
func registerWithGRPC(b grpcresolver.Builder) {
	registerLock.Lock()
	defer registerLock.Unlock()
	grpcresolver.Register(b)
}

// getPrometheusDefs reaches into every slice of prometheus defs we've defined in each part of the agent, and appends
//  all of our slices into one nice slice of definitions per metric type for the Consul agent to pass to go-metrics.
func getPrometheusDefs(cfg lib.TelemetryConfig) ([]prometheus.GaugeDefinition, []prometheus.CounterDefinition, []prometheus.SummaryDefinition) {
	// Build slice of slices for all gauge definitions
	var gauges = [][]prometheus.GaugeDefinition{
		cache.Gauges,
		consul.AutopilotGauges,
		consul.RPCGauges,
		consul.SessionGauges,
		grpc.StatsGauges,
		usagemetrics.Gauges,
		consul.ReplicationGauges,
		Gauges,
	}
	// Flatten definitions
	// NOTE(kit): Do we actually want to create a set here so we can ensure definition names are unique?
	var gaugeDefs []prometheus.GaugeDefinition
	for _, g := range gauges {
		// Set Consul to each definition's namespace
		// TODO(kit): Prepending the service to each definition should be handled by go-metrics
		var withService []prometheus.GaugeDefinition
		for _, gauge := range g {
			gauge.Name = append([]string{cfg.MetricsPrefix}, gauge.Name...)
			withService = append(withService, gauge)
		}
		gaugeDefs = append(gaugeDefs, withService...)
	}

	raftCounters := []prometheus.CounterDefinition{
		// TODO(kit): "raft..." metrics come from the raft lib and we should migrate these to a telemetry
		//  package within. In the mean time, we're going to define a few here because they're key to monitoring Consul.
		{
			Name: []string{"raft", "apply"},
			Help: "This counts the number of Raft transactions occurring over the interval.",
		},
		{
			Name: []string{"raft", "state", "candidate"},
			Help: "This increments whenever a Consul server starts an election.",
		},
		{
			Name: []string{"raft", "state", "leader"},
			Help: "This increments whenever a Consul server becomes a leader.",
		},
	}

	var counters = [][]prometheus.CounterDefinition{
		CatalogCounters,
		cache.Counters,
		consul.ACLCounters,
		consul.CatalogCounters,
		consul.ClientCounters,
		consul.RPCCounters,
		grpc.StatsCounters,
		local.StateCounters,
		raftCounters,
	}
	// Flatten definitions
	// NOTE(kit): Do we actually want to create a set here so we can ensure definition names are unique?
	var counterDefs []prometheus.CounterDefinition
	for _, c := range counters {
		// TODO(kit): Prepending the service to each definition should be handled by go-metrics
		var withService []prometheus.CounterDefinition
		for _, counter := range c {
			counter.Name = append([]string{cfg.MetricsPrefix}, counter.Name...)
			withService = append(withService, counter)
		}
		counterDefs = append(counterDefs, withService...)
	}

	raftSummaries := []prometheus.SummaryDefinition{
		// TODO(kit): "raft..." metrics come from the raft lib and we should migrate these to a telemetry
		//  package within. In the mean time, we're going to define a few here because they're key to monitoring Consul.
		{
			Name: []string{"raft", "commitTime"},
			Help: "This measures the time it takes to commit a new entry to the Raft log on the leader.",
		},
		{
			Name: []string{"raft", "leader", "lastContact"},
			Help: "Measures the time since the leader was last able to contact the follower nodes when checking its leader lease.",
		},
	}

	var summaries = [][]prometheus.SummaryDefinition{
		HTTPSummaries,
		consul.ACLSummaries,
		consul.ACLEndpointSummaries,
		consul.ACLEndpointLegacySummaries,
		consul.CatalogSummaries,
		consul.FederationStateSummaries,
		consul.IntentionSummaries,
		consul.KVSummaries,
		consul.LeaderSummaries,
		consul.PreparedQuerySummaries,
		consul.RPCSummaries,
		consul.SegmentOSSSummaries,
		consul.SessionSummaries,
		consul.SessionEndpointSummaries,
		consul.TxnSummaries,
		fsm.CommandsSummaries,
		fsm.SnapshotSummaries,
		raftSummaries,
	}
	// Flatten definitions
	// NOTE(kit): Do we actually want to create a set here so we can ensure definition names are unique?
	var summaryDefs []prometheus.SummaryDefinition
	for _, s := range summaries {
		// TODO(kit): Prepending the service to each definition should be handled by go-metrics
		var withService []prometheus.SummaryDefinition
		for _, summary := range s {
			summary.Name = append([]string{cfg.MetricsPrefix}, summary.Name...)
			withService = append(withService, summary)
		}
		summaryDefs = append(summaryDefs, withService...)
	}

	return gaugeDefs, counterDefs, summaryDefs
}
<commit_msg>agent: fix data race in tests caused by grpc log init<commit_after>package agent

import (
	"fmt"
	"io"
	"net"
	"net/http"
	"sync"
	"time"

	"github.com/armon/go-metrics/prometheus"
	"github.com/hashicorp/go-hclog"
	"google.golang.org/grpc/grpclog"
	grpcresolver "google.golang.org/grpc/resolver"

	autoconf "github.com/hashicorp/consul/agent/auto-config"
	"github.com/hashicorp/consul/agent/cache"
	"github.com/hashicorp/consul/agent/config"
	"github.com/hashicorp/consul/agent/consul"
	"github.com/hashicorp/consul/agent/consul/fsm"
	"github.com/hashicorp/consul/agent/consul/usagemetrics"
	"github.com/hashicorp/consul/agent/grpc"
	"github.com/hashicorp/consul/agent/grpc/resolver"
	"github.com/hashicorp/consul/agent/local"
	"github.com/hashicorp/consul/agent/pool"
	"github.com/hashicorp/consul/agent/router"
	"github.com/hashicorp/consul/agent/submatview"
	"github.com/hashicorp/consul/agent/token"
	"github.com/hashicorp/consul/ipaddr"
	"github.com/hashicorp/consul/lib"
	"github.com/hashicorp/consul/logging"
	"github.com/hashicorp/consul/tlsutil"
)

// TODO: BaseDeps should be renamed in the future once more of Agent.Start
// has been moved out in front of Agent.New, and we can better see the setup
// dependencies.
type BaseDeps struct {
	consul.Deps // TODO: un-embed

	RuntimeConfig  *config.RuntimeConfig
	MetricsHandler MetricsHandler
	AutoConfig     *autoconf.AutoConfig // TODO: use an interface
	Cache          *cache.Cache
	ViewStore      *submatview.Store
}

// MetricsHandler provides an http.Handler for displaying metrics.
type MetricsHandler interface {
	DisplayMetrics(resp http.ResponseWriter, req *http.Request) (interface{}, error)
}

type ConfigLoader func(source config.Source) (config.LoadResult, error)

func NewBaseDeps(configLoader ConfigLoader, logOut io.Writer) (BaseDeps, error) {
	d := BaseDeps{}
	result, err := configLoader(nil)
	if err != nil {
		return d, err
	}

	cfg := result.RuntimeConfig
	logConf := cfg.Logging
	logConf.Name = logging.Agent
	d.Logger, err = logging.Setup(logConf, logOut)
	if err != nil {
		return d, err
	}

	grpcLogInitOnce.Do(func() {
		grpclog.SetLoggerV2(logging.NewGRPCLogger(cfg.Logging.LogLevel, d.Logger))
	})

	for _, w := range result.Warnings {
		d.Logger.Warn(w)
	}

	cfg.NodeID, err = newNodeIDFromConfig(cfg, d.Logger)
	if err != nil {
		return d, fmt.Errorf("failed to setup node ID: %w", err)
	}

	gauges, counters, summaries := getPrometheusDefs(cfg.Telemetry)
	cfg.Telemetry.PrometheusOpts.GaugeDefinitions = gauges
	cfg.Telemetry.PrometheusOpts.CounterDefinitions = counters
	cfg.Telemetry.PrometheusOpts.SummaryDefinitions = summaries
	d.MetricsHandler, err = lib.InitTelemetry(cfg.Telemetry)
	if err != nil {
		return d, fmt.Errorf("failed to initialize telemetry: %w", err)
	}

	d.TLSConfigurator, err = tlsutil.NewConfigurator(cfg.ToTLSUtilConfig(), d.Logger)
	if err != nil {
		return d, err
	}

	d.RuntimeConfig = cfg
	d.Tokens = new(token.Store)

	cfg.Cache.Logger = d.Logger.Named("cache")
	// cache-types are not registered yet, but they won't be used until the components are started.
	d.Cache = cache.New(cfg.Cache)
	d.ViewStore = submatview.NewStore(d.Logger.Named("viewstore"))
	d.ConnPool = newConnPool(cfg, d.Logger, d.TLSConfigurator)

	builder := resolver.NewServerResolverBuilder(resolver.Config{})
	registerWithGRPC(builder)
	d.GRPCConnPool = grpc.NewClientConnPool(builder, grpc.TLSWrapper(d.TLSConfigurator.OutgoingRPCWrapper()), d.TLSConfigurator.UseTLS)

	d.Router = router.NewRouter(d.Logger, cfg.Datacenter, fmt.Sprintf("%s.%s", cfg.NodeName, cfg.Datacenter), builder)

	acConf := autoconf.Config{
		DirectRPC:       d.ConnPool,
		Logger:          d.Logger,
		Loader:          configLoader,
		ServerProvider:  d.Router,
		TLSConfigurator: d.TLSConfigurator,
		Cache:           d.Cache,
		Tokens:          d.Tokens,
	}
	d.AutoConfig, err = autoconf.New(acConf)
	if err != nil {
		return d, err
	}

	return d, nil
}

// grpcLogInitOnce because the test suite will call NewBaseDeps in many tests and
// causes data races when it is re-initialized.
var grpcLogInitOnce sync.Once

func newConnPool(config *config.RuntimeConfig, logger hclog.Logger, tls *tlsutil.Configurator) *pool.ConnPool {
	var rpcSrcAddr *net.TCPAddr
	if !ipaddr.IsAny(config.RPCBindAddr) {
		rpcSrcAddr = &net.TCPAddr{IP: config.RPCBindAddr.IP}
	}

	pool := &pool.ConnPool{
		Server:          config.ServerMode,
		SrcAddr:         rpcSrcAddr,
		Logger:          logger.StandardLogger(&hclog.StandardLoggerOptions{InferLevels: true}),
		TLSConfigurator: tls,
		Datacenter:      config.Datacenter,
	}
	if config.ServerMode {
		pool.MaxTime = 2 * time.Minute
		pool.MaxStreams = 64
	} else {
		// MaxTime controls how long we keep an idle connection open to a server.
		// 127s was chosen as the first prime above 120s
		// (arbitrarily chose to use a prime) with the intent of reusing
		// connections who are used by once-a-minute cron(8) jobs *and* who
		// use a 60s jitter window (e.g. in vixie cron job execution can
		// drift by up to 59s per job, or 119s for a once-a-minute cron job).
		pool.MaxTime = 127 * time.Second
		pool.MaxStreams = 32
	}
	return pool
}

var registerLock sync.Mutex

// registerWithGRPC registers the grpc/resolver.Builder as a grpc/resolver.
// This function exists to synchronize registrations with a lock.
// grpc/resolver.Register expects all registration to happen at init and does
// not allow for concurrent registration. This function exists to support
// parallel testing.
func registerWithGRPC(b grpcresolver.Builder) {
	registerLock.Lock()
	defer registerLock.Unlock()
	grpcresolver.Register(b)
}

// getPrometheusDefs reaches into every slice of prometheus defs we've defined in each part of the agent, and appends
//  all of our slices into one nice slice of definitions per metric type for the Consul agent to pass to go-metrics.
func getPrometheusDefs(cfg lib.TelemetryConfig) ([]prometheus.GaugeDefinition, []prometheus.CounterDefinition, []prometheus.SummaryDefinition) {
	// Build slice of slices for all gauge definitions
	var gauges = [][]prometheus.GaugeDefinition{
		cache.Gauges,
		consul.AutopilotGauges,
		consul.RPCGauges,
		consul.SessionGauges,
		grpc.StatsGauges,
		usagemetrics.Gauges,
		consul.ReplicationGauges,
		Gauges,
	}
	// Flatten definitions
	// NOTE(kit): Do we actually want to create a set here so we can ensure definition names are unique?
	var gaugeDefs []prometheus.GaugeDefinition
	for _, g := range gauges {
		// Set Consul to each definition's namespace
		// TODO(kit): Prepending the service to each definition should be handled by go-metrics
		var withService []prometheus.GaugeDefinition
		for _, gauge := range g {
			gauge.Name = append([]string{cfg.MetricsPrefix}, gauge.Name...)
			withService = append(withService, gauge)
		}
		gaugeDefs = append(gaugeDefs, withService...)
	}

	raftCounters := []prometheus.CounterDefinition{
		// TODO(kit): "raft..." metrics come from the raft lib and we should migrate these to a telemetry
		//  package within. In the mean time, we're going to define a few here because they're key to monitoring Consul.
		{
			Name: []string{"raft", "apply"},
			Help: "This counts the number of Raft transactions occurring over the interval.",
		},
		{
			Name: []string{"raft", "state", "candidate"},
			Help: "This increments whenever a Consul server starts an election.",
		},
		{
			Name: []string{"raft", "state", "leader"},
			Help: "This increments whenever a Consul server becomes a leader.",
		},
	}

	var counters = [][]prometheus.CounterDefinition{
		CatalogCounters,
		cache.Counters,
		consul.ACLCounters,
		consul.CatalogCounters,
		consul.ClientCounters,
		consul.RPCCounters,
		grpc.StatsCounters,
		local.StateCounters,
		raftCounters,
	}
	// Flatten definitions
	// NOTE(kit): Do we actually want to create a set here so we can ensure definition names are unique?
	var counterDefs []prometheus.CounterDefinition
	for _, c := range counters {
		// TODO(kit): Prepending the service to each definition should be handled by go-metrics
		var withService []prometheus.CounterDefinition
		for _, counter := range c {
			counter.Name = append([]string{cfg.MetricsPrefix}, counter.Name...)
			withService = append(withService, counter)
		}
		counterDefs = append(counterDefs, withService...)
	}

	raftSummaries := []prometheus.SummaryDefinition{
		// TODO(kit): "raft..." metrics come from the raft lib and we should migrate these to a telemetry
		//  package within. In the mean time, we're going to define a few here because they're key to monitoring Consul.
		{
			Name: []string{"raft", "commitTime"},
			Help: "This measures the time it takes to commit a new entry to the Raft log on the leader.",
		},
		{
			Name: []string{"raft", "leader", "lastContact"},
			Help: "Measures the time since the leader was last able to contact the follower nodes when checking its leader lease.",
		},
	}

	var summaries = [][]prometheus.SummaryDefinition{
		HTTPSummaries,
		consul.ACLSummaries,
		consul.ACLEndpointSummaries,
		consul.ACLEndpointLegacySummaries,
		consul.CatalogSummaries,
		consul.FederationStateSummaries,
		consul.IntentionSummaries,
		consul.KVSummaries,
		consul.LeaderSummaries,
		consul.PreparedQuerySummaries,
		consul.RPCSummaries,
		consul.SegmentOSSSummaries,
		consul.SessionSummaries,
		consul.SessionEndpointSummaries,
		consul.TxnSummaries,
		fsm.CommandsSummaries,
		fsm.SnapshotSummaries,
		raftSummaries,
	}
	// Flatten definitions
	// NOTE(kit): Do we actually want to create a set here so we can ensure definition names are unique?
	var summaryDefs []prometheus.SummaryDefinition
	for _, s := range summaries {
		// TODO(kit): Prepending the service to each definition should be handled by go-metrics
		var withService []prometheus.SummaryDefinition
		for _, summary := range s {
			summary.Name = append([]string{cfg.MetricsPrefix}, summary.Name...)
			withService = append(withService, summary)
		}
		summaryDefs = append(summaryDefs, withService...)
	}

	return gaugeDefs, counterDefs, summaryDefs
}
<|endoftext|>


<commit_before>package ai

import (
	"github.com/nelhage/taktician/bitboard"
	"github.com/nelhage/taktician/tak"
)

const (
	v3WeightFlat       = 200
	v3WeightCaptured   = 100
	v3WeightControlled = 300
	v3WeightCapstone   = -150
	v3WeightGroup      = 4
)

func V3evaluate(m *MinimaxAI, p *tak.Position) int64 {
	if over, winner := p.GameOver(); over {
		switch winner {
		case tak.NoColor:
			return 0
		case p.ToMove():
			return maxEval - int64(p.MoveNumber())
		default:
			return minEval + int64(p.MoveNumber())
		}
	}
	mine, theirs := 0, 0
	me := p.ToMove()
	addw := func(c tak.Color, w int) {
		if c == me {
			mine += w
		} else {
			theirs += w
		}
	}
	weightGroup := v3WeightGroup
	maxStack := 0
	analysis := p.Analysis()
	for x := 0; x < p.Size(); x++ {
		for y := 0; y < p.Size(); y++ {
			sq := p.At(x, y)
			if len(sq) == 0 {
				continue
			}
			if len(sq) > maxStack {
				maxStack = len(sq)
			}
			if sq[0].Kind() != tak.Standing {
				addw(sq[0].Color(), v3WeightControlled)
			}
			if sq[0].Kind() == tak.Capstone {
				addw(sq[0].Color(), v3WeightCapstone)
			}

			for i, stone := range sq {
				if i > 0 && i < p.Size() {
					addw(sq[0].Color(), v3WeightCaptured)
				}
				if stone.Kind() == tak.Flat {
					addw(stone.Color(), v3WeightFlat)
				}
			}
		}
	}

	empty := ^(analysis.White | analysis.Black)
	if maxStack >= 3 {
		weightGroup--
	}
	if bitboard.Popcount(empty) < (p.Size()*p.Size())/2 {
		weightGroup--
	}
	addw(tak.White, m.scoreGroups(analysis.WhiteGroups, empty, weightGroup))
	addw(tak.Black, m.scoreGroups(analysis.BlackGroups, empty, weightGroup))

	return int64(mine - theirs)
}

func (ai *MinimaxAI) scoreGroups(gs []uint64, empty uint64, weight int) int {
	sc := 0
	for _, g := range gs {
		w, h := bitboard.Dimensions(&ai.c, g)
		sz := bitboard.Popcount(g)
		libs := bitboard.Popcount(bitboard.Grow(&ai.c, g|empty, g) &^ g)

		sp := w
		if h > sp {
			sp = h
		}
		sc += (sp*sp + (2 * sz) + libs) * weight
	}

	return sc
}
<commit_msg>weight for standing stones<commit_after>package ai

import (
	"github.com/nelhage/taktician/bitboard"
	"github.com/nelhage/taktician/tak"
)

const (
	v3WeightFlat       = 200
	v3WeightStanding   = 100
	v3WeightCaptured   = 100
	v3WeightControlled = 300
	v3WeightCapstone   = -150
	v3WeightGroup      = 4
)

func V3evaluate(m *MinimaxAI, p *tak.Position) int64 {
	if over, winner := p.GameOver(); over {
		switch winner {
		case tak.NoColor:
			return 0
		case p.ToMove():
			return maxEval - int64(p.MoveNumber())
		default:
			return minEval + int64(p.MoveNumber())
		}
	}
	mine, theirs := 0, 0
	me := p.ToMove()
	addw := func(c tak.Color, w int) {
		if c == me {
			mine += w
		} else {
			theirs += w
		}
	}
	weightGroup := v3WeightGroup
	maxStack := 0
	analysis := p.Analysis()
	for x := 0; x < p.Size(); x++ {
		for y := 0; y < p.Size(); y++ {
			sq := p.At(x, y)
			if len(sq) == 0 {
				continue
			}
			if len(sq) > maxStack {
				maxStack = len(sq)
			}
			if sq[0].Kind() == tak.Standing {
				addw(sq[0].Color(), v3WeightStanding)
			} else {
				addw(sq[0].Color(), v3WeightControlled)
			}
			if sq[0].Kind() == tak.Capstone {
				addw(sq[0].Color(), v3WeightCapstone)
			}

			for i, stone := range sq {
				if i > 0 && i < p.Size() {
					addw(sq[0].Color(), v3WeightCaptured)
				}
				if stone.Kind() == tak.Flat {
					addw(stone.Color(), v3WeightFlat)
				}
			}
		}
	}

	empty := ^(analysis.White | analysis.Black)
	if maxStack >= 3 {
		weightGroup--
	}
	if bitboard.Popcount(empty) < (p.Size()*p.Size())/2 {
		weightGroup--
	}
	addw(tak.White, m.scoreGroups(analysis.WhiteGroups, empty, weightGroup))
	addw(tak.Black, m.scoreGroups(analysis.BlackGroups, empty, weightGroup))

	return int64(mine - theirs)
}

func (ai *MinimaxAI) scoreGroups(gs []uint64, empty uint64, weight int) int {
	sc := 0
	for _, g := range gs {
		w, h := bitboard.Dimensions(&ai.c, g)
		sz := bitboard.Popcount(g)
		libs := bitboard.Popcount(bitboard.Grow(&ai.c, g|empty, g) &^ g)

		sp := w
		if h > sp {
			sp = h
		}
		sc += (sp*sp + (2 * sz) + libs) * weight
	}

	return sc
}
<|endoftext|>