<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/golang/glog"
	"golang.org/x/net/context"

	"github.com/cloudstax/firecamp/common"
	"github.com/cloudstax/firecamp/containersvc"
	"github.com/cloudstax/firecamp/containersvc/awsecs"
	"github.com/cloudstax/firecamp/containersvc/k8s"
	"github.com/cloudstax/firecamp/containersvc/swarm"
	"github.com/cloudstax/firecamp/db"
	"github.com/cloudstax/firecamp/db/awsdynamodb"
	"github.com/cloudstax/firecamp/db/controldb/client"
	"github.com/cloudstax/firecamp/db/k8sconfigdb"
	"github.com/cloudstax/firecamp/dns"
	"github.com/cloudstax/firecamp/dns/awsroute53"
	"github.com/cloudstax/firecamp/log"
	"github.com/cloudstax/firecamp/log/awscloudwatch"
	"github.com/cloudstax/firecamp/manage/server"
	"github.com/cloudstax/firecamp/server"
	"github.com/cloudstax/firecamp/server/awsec2"
	"github.com/cloudstax/firecamp/utils"
)

var (
	platform      = flag.String("container-platform", common.ContainerPlatformECS, "The underline container platform: ecs or swarm, default: ecs")
	dbtype        = flag.String("dbtype", common.DBTypeCloudDB, "The db type, such as the AWS DynamoDB or the embedded controldb")
	zones         = flag.String("availability-zones", "", "The availability zones for the system, example: us-east-1a,us-east-1b,us-east-1c")
	manageDNSName = flag.String("dnsname", "", "the dns name of the management service. Default: "+dns.GetDefaultManageServiceDNSName("cluster"))
	managePort    = flag.Int("port", common.ManageHTTPServerPort, "port that the manage http service listens on")
	tlsEnabled    = flag.Bool("tlsverify", false, "whether TLS is enabled")
	caFile        = flag.String("tlscacert", "", "The CA file")
	certFile      = flag.String("tlscert", "", "The TLS server certificate file")
	keyFile       = flag.String("tlskey", "", "The TLS server key file")
)

const azSep = ","

func main() {
	flag.Parse()

	// log to std, the manageserver container will send the logs to such as AWS CloudWatch.
	utils.SetLogToStd()
	//utils.SetLogDir()

	if *tlsEnabled && (*certFile == "" || *keyFile == "") {
		fmt.Println("invalid command, please pass cert file and key file for tls", *certFile, *keyFile)
		os.Exit(-1)
	}

	if *platform != common.ContainerPlatformECS &&
		*platform != common.ContainerPlatformSwarm &&
		*platform != common.ContainerPlatformK8s {
		fmt.Println("not supported container platform", *platform)
		os.Exit(-1)
	}

	if *zones == "" {
		fmt.Println("Invalid command, please specify the availability zones for the system")
		os.Exit(-1)
	}

	k8snamespace := ""
	if *platform == common.ContainerPlatformK8s {
		k8snamespace := os.Getenv(common.ENV_K8S_NAMESPACE)
		if len(k8snamespace) == 0 {
			glog.Infoln("k8s namespace is not set. set to default")
			k8snamespace = common.DefaultK8sNamespace
		}
	}

	region, err := awsec2.GetLocalEc2Region()
	if err != nil {
		glog.Fatalln("awsec2 GetLocalEc2Region error", err)
	}

	config := aws.NewConfig().WithRegion(region)
	sess, err := session.NewSession(config)
	if err != nil {
		glog.Fatalln("failed to create session, error", err)
	}

	serverIns := awsec2.NewAWSEc2(sess)
	serverInfo, err := awsec2.NewEc2Info(sess)
	if err != nil {
		glog.Fatalln("NewEc2Info error", err)
	}

	regionAZs := serverInfo.GetLocalRegionAZs()
	azs := strings.Split(*zones, azSep)
	for _, az := range azs {
		find := false
		for _, raz := range regionAZs {
			if az == raz {
				find = true
				break
			}
		}
		if !find {
			glog.Fatalln("The specified availability zone is not in the region's availability zones", *zones, regionAZs)
		}
	}

	dnsIns := awsroute53.NewAWSRoute53(sess)
	logIns := awscloudwatch.NewLog(sess, region, *platform, k8snamespace)

	cluster := ""
	var containersvcIns containersvc.ContainerSvc
	switch *platform {
	case common.ContainerPlatformECS:
		info, err := awsecs.NewEcsInfo()
		if err != nil {
			glog.Fatalln("NewEcsInfo error", err)
		}

		cluster = info.GetContainerClusterID()
		containersvcIns = awsecs.NewAWSEcs(sess)

	case common.ContainerPlatformSwarm:
		cluster = os.Getenv(common.ENV_CLUSTER)
		if len(cluster) == 0 {
			glog.Fatalln("Swarm cluster name is not set")
		}

		info, err := swarmsvc.NewSwarmInfo(cluster)
		if err != nil {
			glog.Fatalln("NewSwarmInfo error", err)
		}

		cluster = info.GetContainerClusterID()
		containersvcIns, err = swarmsvc.NewSwarmSvc(azs)
		if err != nil {
			glog.Fatalln("NewSwarmSvc error", err)
		}

	case common.ContainerPlatformK8s:
		cluster = os.Getenv(common.ENV_CLUSTER)
		if len(cluster) == 0 {
			glog.Fatalln("K8s cluster name is not set")
		}

		fullhostname, err := awsec2.GetLocalEc2Hostname()
		if err != nil {
			glog.Fatalln("get local ec2 hostname error", err)
		}
		info := k8ssvc.NewK8sInfo(cluster, fullhostname)
		cluster = info.GetContainerClusterID()

		containersvcIns, err = k8ssvc.NewK8sSvc(cluster, common.CloudPlatformAWS, *dbtype, k8snamespace)
		if err != nil {
			glog.Fatalln("NewK8sSvc error", err)
		}

	default:
		glog.Fatalln("unsupport container platform", *platform)
	}

	ctx := context.Background()
	var dbIns db.DB
	switch *dbtype {
	case common.DBTypeCloudDB:
		dbIns = awsdynamodb.NewDynamoDB(sess, cluster)

		tableStatus, ready, err := dbIns.SystemTablesReady(ctx)
		if err != nil && err != db.ErrDBResourceNotFound {
			glog.Fatalln("check dynamodb SystemTablesReady error", err, tableStatus)
		}
		if !ready {
			// create system tables
			err = dbIns.CreateSystemTables(ctx)
			if err != nil {
				glog.Fatalln("dynamodb CreateSystemTables error", err)
			}

			// wait the system tables ready
			waitSystemTablesReady(ctx, dbIns)
		}

	case common.DBTypeControlDB:
		addr := dns.GetDefaultControlDBAddr(cluster)
		dbIns = controldbcli.NewControlDBCli(addr)

		createControlDB(ctx, region, cluster, logIns, containersvcIns, serverIns, serverInfo)
		waitControlDBReady(ctx, cluster, containersvcIns)

	case common.DBTypeK8sDB:
		dbIns, err = k8sconfigdb.NewK8sConfigDB(k8snamespace)
		if err != nil {
			glog.Fatalln("NewK8sConfigDB error", err)
		}

	default:
		glog.Fatalln("unknown db type", *dbtype)
	}

	err = manageserver.StartServer(*platform, cluster, azs, *manageDNSName, *managePort, containersvcIns,
		dbIns, dnsIns, logIns, serverInfo, serverIns, *tlsEnabled, *caFile, *certFile, *keyFile)
	if err != nil {
		glog.Fatalln("StartServer error", err)
	}
}

func createControlDB(ctx context.Context, region string, cluster string, logIns cloudlog.CloudLog,
	containersvcIns containersvc.ContainerSvc, serverIns server.Server, serverInfo server.Info) {
	// check if the controldb service exists.
	exist, err := containersvcIns.IsServiceExist(ctx, cluster, common.ControlDBServiceName)
	if err != nil {
		glog.Fatalln("check the controlDB service exist error", err, common.ControlDBServiceName)
	}
	if exist {
		glog.Infoln("The controlDB service is already created")
		return
	}

	// create the controldb volume
	volOpts := &server.CreateVolumeOptions{
		AvailabilityZone: serverInfo.GetLocalAvailabilityZone(),
		VolumeType:       common.VolumeTypeGPSSD,
		VolumeSizeGB:     common.ControlDBVolumeSizeGB,
		TagSpecs: []common.KeyValuePair{
			common.KeyValuePair{
				Key:   "Name",
				Value: common.SystemName + common.NameSeparator + cluster + common.NameSeparator + common.ControlDBServiceName,
			},
		},
	}
	// TODO if some step fails, the volume may not be deleted.
	//      add tag to EBS volume, so the old volume could be deleted.
	volID, err := serverIns.CreateVolume(ctx, volOpts)
	if err != nil {
		glog.Fatalln("failed to create the controldb volume", err)
	}

	glog.Infoln("create the controldb volume", volID)

	err = serverIns.WaitVolumeCreated(ctx, volID)
	if err != nil {
		serverIns.DeleteVolume(ctx, volID)
		glog.Fatalln("volume is not at available status", err, volID)
	}

	// create the controldb service
	serviceUUID := utils.GenControlDBServiceUUID(volID)
	logConfig := logIns.CreateServiceLogConfig(ctx, cluster, common.ControlDBServiceName, serviceUUID)
	logConfig.Options[common.LogServiceMemberKey] = common.ControlDBServiceName

	commonOpts := &containersvc.CommonOptions{
		Cluster:        cluster,
		ServiceName:    common.ControlDBServiceName,
		ServiceUUID:    serviceUUID,
		ContainerImage: common.ControlDBContainerImage,
		Resource: &common.Resources{
			MaxCPUUnits:     common.DefaultMaxCPUUnits,
			ReserveCPUUnits: common.ControlDBReserveCPUUnits,
			MaxMemMB:        common.ControlDBMaxMemMB,
			ReserveMemMB:    common.ControlDBReserveMemMB,
		},
		LogConfig: logConfig,
	}

	kv := &common.EnvKeyValuePair{
		Name:  common.ENV_CONTAINER_PLATFORM,
		Value: *platform,
	}

	p := common.PortMapping{
		ContainerPort: common.ControlDBServerPort,
		HostPort:      common.ControlDBServerPort,
	}

	createOpts := &containersvc.CreateServiceOptions{
		Common: commonOpts,
		DataVolume: &containersvc.VolumeOptions{
			MountPath:  common.ControlDBDefaultDir,
			VolumeType: common.VolumeTypeGPSSD,
			SizeGB:     common.ControlDBVolumeSizeGB,
		},
		PortMappings: []common.PortMapping{p},
		Replicas:     int64(1),
		Envkvs:       []*common.EnvKeyValuePair{kv},
	}

	err = containersvcIns.CreateService(ctx, createOpts)
	if err != nil {
		glog.Fatalln("create controlDB service error", err, common.ControlDBServiceName, "serviceUUID", serviceUUID)
	}

	glog.Infoln("created the controlDB service")
}

func waitControlDBReady(ctx context.Context, cluster string, containersvcIns containersvc.ContainerSvc) {
	// wait the controldb service is running
	for sec := int64(0); sec < common.DefaultServiceWaitSeconds; sec += common.DefaultRetryWaitSeconds {
		status, err := containersvcIns.GetServiceStatus(ctx, cluster, common.ControlDBServiceName)
		if err != nil {
			// The service is successfully created. It may be possible there are some
			// temporary error, such as network error. For example, ECS may return MISSING
			// for the GET right after the service creation.
			// Here just log the GetServiceStatus error and retry.
			glog.Errorln("GetServiceStatus error", err)
		} else {
			if status.RunningCount == status.DesiredCount {
				glog.Infoln("The controldb service is ready")
				return
			}
		}

		time.Sleep(time.Duration(common.DefaultRetryWaitSeconds) * time.Second)
	}

	glog.Fatalln("Wait the controldb service ready timeout")
}

func waitSystemTablesReady(ctx context.Context, dbIns db.DB) {
	for sec := int64(0); sec < common.DefaultServiceWaitSeconds; sec += common.DefaultRetryWaitSeconds {
		tableStatus, ready, err := dbIns.SystemTablesReady(ctx)
		if err != nil {
			glog.Errorln("SystemTablesReady check failed", err)
		} else {
			if ready {
				glog.Infoln("system tables are ready")
				return
			}

			if tableStatus != db.TableStatusCreating {
				glog.Fatalln("unexpected table status", tableStatus)
			}
		}

		glog.Infoln("tables are not ready, sleep and check again")
		time.Sleep(time.Duration(common.DefaultRetryWaitSeconds) * time.Second)
	}

	glog.Fatalln("Wait the SystemTablesReady timeout")
}
<commit_msg>fix k8s namespace in the manage server start<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/golang/glog"
	"golang.org/x/net/context"

	"github.com/cloudstax/firecamp/common"
	"github.com/cloudstax/firecamp/containersvc"
	"github.com/cloudstax/firecamp/containersvc/awsecs"
	"github.com/cloudstax/firecamp/containersvc/k8s"
	"github.com/cloudstax/firecamp/containersvc/swarm"
	"github.com/cloudstax/firecamp/db"
	"github.com/cloudstax/firecamp/db/awsdynamodb"
	"github.com/cloudstax/firecamp/db/controldb/client"
	"github.com/cloudstax/firecamp/db/k8sconfigdb"
	"github.com/cloudstax/firecamp/dns"
	"github.com/cloudstax/firecamp/dns/awsroute53"
	"github.com/cloudstax/firecamp/log"
	"github.com/cloudstax/firecamp/log/awscloudwatch"
	"github.com/cloudstax/firecamp/manage/server"
	"github.com/cloudstax/firecamp/server"
	"github.com/cloudstax/firecamp/server/awsec2"
	"github.com/cloudstax/firecamp/utils"
)

var (
	platform      = flag.String("container-platform", common.ContainerPlatformECS, "The underline container platform: ecs or swarm, default: ecs")
	dbtype        = flag.String("dbtype", common.DBTypeCloudDB, "The db type, such as the AWS DynamoDB or the embedded controldb")
	zones         = flag.String("availability-zones", "", "The availability zones for the system, example: us-east-1a,us-east-1b,us-east-1c")
	manageDNSName = flag.String("dnsname", "", "the dns name of the management service. Default: "+dns.GetDefaultManageServiceDNSName("cluster"))
	managePort    = flag.Int("port", common.ManageHTTPServerPort, "port that the manage http service listens on")
	tlsEnabled    = flag.Bool("tlsverify", false, "whether TLS is enabled")
	caFile        = flag.String("tlscacert", "", "The CA file")
	certFile      = flag.String("tlscert", "", "The TLS server certificate file")
	keyFile       = flag.String("tlskey", "", "The TLS server key file")
)

const azSep = ","

func main() {
	flag.Parse()

	// log to std, the manageserver container will send the logs to such as AWS CloudWatch.
	utils.SetLogToStd()
	//utils.SetLogDir()

	if *tlsEnabled && (*certFile == "" || *keyFile == "") {
		fmt.Println("invalid command, please pass cert file and key file for tls", *certFile, *keyFile)
		os.Exit(-1)
	}

	if *platform != common.ContainerPlatformECS &&
		*platform != common.ContainerPlatformSwarm &&
		*platform != common.ContainerPlatformK8s {
		fmt.Println("not supported container platform", *platform)
		os.Exit(-1)
	}

	if *zones == "" {
		fmt.Println("Invalid command, please specify the availability zones for the system")
		os.Exit(-1)
	}

	k8snamespace := ""
	if *platform == common.ContainerPlatformK8s {
		k8snamespace = os.Getenv(common.ENV_K8S_NAMESPACE)
		if len(k8snamespace) == 0 {
			glog.Infoln("k8s namespace is not set. set to default")
			k8snamespace = common.DefaultK8sNamespace
		}
	}

	region, err := awsec2.GetLocalEc2Region()
	if err != nil {
		glog.Fatalln("awsec2 GetLocalEc2Region error", err)
	}

	config := aws.NewConfig().WithRegion(region)
	sess, err := session.NewSession(config)
	if err != nil {
		glog.Fatalln("failed to create session, error", err)
	}

	serverIns := awsec2.NewAWSEc2(sess)
	serverInfo, err := awsec2.NewEc2Info(sess)
	if err != nil {
		glog.Fatalln("NewEc2Info error", err)
	}

	regionAZs := serverInfo.GetLocalRegionAZs()
	azs := strings.Split(*zones, azSep)
	for _, az := range azs {
		find := false
		for _, raz := range regionAZs {
			if az == raz {
				find = true
				break
			}
		}
		if !find {
			glog.Fatalln("The specified availability zone is not in the region's availability zones", *zones, regionAZs)
		}
	}

	glog.Infoln("create manageserver, container platform", *platform, ", dbtype", *dbtype, ", availability zones", azs, ", k8snamespace", k8snamespace)

	dnsIns := awsroute53.NewAWSRoute53(sess)
	logIns := awscloudwatch.NewLog(sess, region, *platform, k8snamespace)

	cluster := ""
	var containersvcIns containersvc.ContainerSvc
	switch *platform {
	case common.ContainerPlatformECS:
		info, err := awsecs.NewEcsInfo()
		if err != nil {
			glog.Fatalln("NewEcsInfo error", err)
		}

		cluster = info.GetContainerClusterID()
		containersvcIns = awsecs.NewAWSEcs(sess)

	case common.ContainerPlatformSwarm:
		cluster = os.Getenv(common.ENV_CLUSTER)
		if len(cluster) == 0 {
			glog.Fatalln("Swarm cluster name is not set")
		}

		info, err := swarmsvc.NewSwarmInfo(cluster)
		if err != nil {
			glog.Fatalln("NewSwarmInfo error", err)
		}

		cluster = info.GetContainerClusterID()
		containersvcIns, err = swarmsvc.NewSwarmSvc(azs)
		if err != nil {
			glog.Fatalln("NewSwarmSvc error", err)
		}

	case common.ContainerPlatformK8s:
		cluster = os.Getenv(common.ENV_CLUSTER)
		if len(cluster) == 0 {
			glog.Fatalln("K8s cluster name is not set")
		}

		fullhostname, err := awsec2.GetLocalEc2Hostname()
		if err != nil {
			glog.Fatalln("get local ec2 hostname error", err)
		}
		info := k8ssvc.NewK8sInfo(cluster, fullhostname)
		cluster = info.GetContainerClusterID()

		containersvcIns, err = k8ssvc.NewK8sSvc(cluster, common.CloudPlatformAWS, *dbtype, k8snamespace)
		if err != nil {
			glog.Fatalln("NewK8sSvc error", err)
		}

	default:
		glog.Fatalln("unsupport container platform", *platform)
	}

	ctx := context.Background()
	var dbIns db.DB
	switch *dbtype {
	case common.DBTypeCloudDB:
		dbIns = awsdynamodb.NewDynamoDB(sess, cluster)

		tableStatus, ready, err := dbIns.SystemTablesReady(ctx)
		if err != nil && err != db.ErrDBResourceNotFound {
			glog.Fatalln("check dynamodb SystemTablesReady error", err, tableStatus)
		}
		if !ready {
			// create system tables
			err = dbIns.CreateSystemTables(ctx)
			if err != nil {
				glog.Fatalln("dynamodb CreateSystemTables error", err)
			}

			// wait the system tables ready
			waitSystemTablesReady(ctx, dbIns)
		}

	case common.DBTypeControlDB:
		addr := dns.GetDefaultControlDBAddr(cluster)
		dbIns = controldbcli.NewControlDBCli(addr)

		createControlDB(ctx, region, cluster, logIns, containersvcIns, serverIns, serverInfo)
		waitControlDBReady(ctx, cluster, containersvcIns)

	case common.DBTypeK8sDB:
		dbIns, err = k8sconfigdb.NewK8sConfigDB(k8snamespace)
		if err != nil {
			glog.Fatalln("NewK8sConfigDB error", err)
		}

	default:
		glog.Fatalln("unknown db type", *dbtype)
	}

	err = manageserver.StartServer(*platform, cluster, azs, *manageDNSName, *managePort, containersvcIns,
		dbIns, dnsIns, logIns, serverInfo, serverIns, *tlsEnabled, *caFile, *certFile, *keyFile)
	if err != nil {
		glog.Fatalln("StartServer error", err)
	}
}

func createControlDB(ctx context.Context, region string, cluster string, logIns cloudlog.CloudLog,
	containersvcIns containersvc.ContainerSvc, serverIns server.Server, serverInfo server.Info) {
	// check if the controldb service exists.
	exist, err := containersvcIns.IsServiceExist(ctx, cluster, common.ControlDBServiceName)
	if err != nil {
		glog.Fatalln("check the controlDB service exist error", err, common.ControlDBServiceName)
	}
	if exist {
		glog.Infoln("The controlDB service is already created")
		return
	}

	// create the controldb volume
	volOpts := &server.CreateVolumeOptions{
		AvailabilityZone: serverInfo.GetLocalAvailabilityZone(),
		VolumeType:       common.VolumeTypeGPSSD,
		VolumeSizeGB:     common.ControlDBVolumeSizeGB,
		TagSpecs: []common.KeyValuePair{
			common.KeyValuePair{
				Key:   "Name",
				Value: common.SystemName + common.NameSeparator + cluster + common.NameSeparator + common.ControlDBServiceName,
			},
		},
	}
	// TODO if some step fails, the volume may not be deleted.
	//      add tag to EBS volume, so the old volume could be deleted.
	volID, err := serverIns.CreateVolume(ctx, volOpts)
	if err != nil {
		glog.Fatalln("failed to create the controldb volume", err)
	}

	glog.Infoln("create the controldb volume", volID)

	err = serverIns.WaitVolumeCreated(ctx, volID)
	if err != nil {
		serverIns.DeleteVolume(ctx, volID)
		glog.Fatalln("volume is not at available status", err, volID)
	}

	// create the controldb service
	serviceUUID := utils.GenControlDBServiceUUID(volID)
	logConfig := logIns.CreateServiceLogConfig(ctx, cluster, common.ControlDBServiceName, serviceUUID)
	logConfig.Options[common.LogServiceMemberKey] = common.ControlDBServiceName

	commonOpts := &containersvc.CommonOptions{
		Cluster:        cluster,
		ServiceName:    common.ControlDBServiceName,
		ServiceUUID:    serviceUUID,
		ContainerImage: common.ControlDBContainerImage,
		Resource: &common.Resources{
			MaxCPUUnits:     common.DefaultMaxCPUUnits,
			ReserveCPUUnits: common.ControlDBReserveCPUUnits,
			MaxMemMB:        common.ControlDBMaxMemMB,
			ReserveMemMB:    common.ControlDBReserveMemMB,
		},
		LogConfig: logConfig,
	}

	kv := &common.EnvKeyValuePair{
		Name:  common.ENV_CONTAINER_PLATFORM,
		Value: *platform,
	}

	p := common.PortMapping{
		ContainerPort: common.ControlDBServerPort,
		HostPort:      common.ControlDBServerPort,
	}

	createOpts := &containersvc.CreateServiceOptions{
		Common: commonOpts,
		DataVolume: &containersvc.VolumeOptions{
			MountPath:  common.ControlDBDefaultDir,
			VolumeType: common.VolumeTypeGPSSD,
			SizeGB:     common.ControlDBVolumeSizeGB,
		},
		PortMappings: []common.PortMapping{p},
		Replicas:     int64(1),
		Envkvs:       []*common.EnvKeyValuePair{kv},
	}

	err = containersvcIns.CreateService(ctx, createOpts)
	if err != nil {
		glog.Fatalln("create controlDB service error", err, common.ControlDBServiceName, "serviceUUID", serviceUUID)
	}

	glog.Infoln("created the controlDB service")
}

func waitControlDBReady(ctx context.Context, cluster string, containersvcIns containersvc.ContainerSvc) {
	// wait the controldb service is running
	for sec := int64(0); sec < common.DefaultServiceWaitSeconds; sec += common.DefaultRetryWaitSeconds {
		status, err := containersvcIns.GetServiceStatus(ctx, cluster, common.ControlDBServiceName)
		if err != nil {
			// The service is successfully created. It may be possible there are some
			// temporary error, such as network error. For example, ECS may return MISSING
			// for the GET right after the service creation.
			// Here just log the GetServiceStatus error and retry.
			glog.Errorln("GetServiceStatus error", err)
		} else {
			if status.RunningCount == status.DesiredCount {
				glog.Infoln("The controldb service is ready")
				return
			}
		}

		time.Sleep(time.Duration(common.DefaultRetryWaitSeconds) * time.Second)
	}

	glog.Fatalln("Wait the controldb service ready timeout")
}

func waitSystemTablesReady(ctx context.Context, dbIns db.DB) {
	for sec := int64(0); sec < common.DefaultServiceWaitSeconds; sec += common.DefaultRetryWaitSeconds {
		tableStatus, ready, err := dbIns.SystemTablesReady(ctx)
		if err != nil {
			glog.Errorln("SystemTablesReady check failed", err)
		} else {
			if ready {
				glog.Infoln("system tables are ready")
				return
			}

			if tableStatus != db.TableStatusCreating {
				glog.Fatalln("unexpected table status", tableStatus)
			}
		}

		glog.Infoln("tables are not ready, sleep and check again")
		time.Sleep(time.Duration(common.DefaultRetryWaitSeconds) * time.Second)
	}

	glog.Fatalln("Wait the SystemTablesReady timeout")
}
<|endoftext|>


<commit_before>package internal

import (
	"math/rand"
	"reflect"
	"testing"
	"testing/quick"

	"github.com/kylelemons/godebug/pretty"
	"github.com/nochso/bolster/codec"
)

type testStruct struct {
	A int
	B uint
	C int8
	D int16
	E int32
	F int64
	G uint8
	H uint16
	I uint32
	J uint64
	K bool
	L float32
	M float64
	N string
}

func Roundtrip(t *testing.T, c codec.Interface) {
	r := rand.New(rand.NewSource(0))
	for i := 0; i < 1000; i++ {
		expr, ok := quick.Value(reflect.TypeOf(testStruct{}), r)
		if !ok {
			t.Fatal("unable to create random struct")
		}
		exp := expr.Addr().Interface()
		b, err := c.Marshal(exp)
		if err != nil {
			t.Fatal(err)
		}
		var act testStruct
		err = c.Unmarshal(b, &act)
		if err != nil {
			t.Fatal(err)
		}
		if !reflect.DeepEqual(act, exp) {
			pretty.Compare(act, exp)
		}
	}
}
<commit_msg>Improve codec roundtrip test<commit_after>package internal

import (
	"math/rand"
	"reflect"
	"testing"
	"testing/quick"

	"github.com/kylelemons/godebug/pretty"
	"github.com/nochso/bolster/codec"
)

type testStruct struct {
	A int
	B uint
	C int8
	D int16
	E int32
	F int64
	G uint8
	H uint16
	I uint32
	J uint64
	K bool
	L float32
	M float64
	N string
	O testStructChild
	P *testStructChild
}

type testStructChild struct {
	A int
	B bool
}

func Roundtrip(t *testing.T, c codec.Interface) {
	r := rand.New(rand.NewSource(0))
	for i := 0; i < 1000; i++ {
		// Create random struct
		expr, ok := quick.Value(reflect.TypeOf(testStruct{}), r)
		if !ok {
			t.Fatal("unable to create random struct")
		}
		exp := expr.Addr().Interface()
		// Encode to bytes
		b, err := c.Marshal(exp)
		if err != nil {
			t.Fatal(err)
		}
		// Decode it back to a new struct
		act := &testStruct{}
		err = c.Unmarshal(b, act)
		if err != nil {
			t.Fatal(err)
		}
		if !reflect.DeepEqual(act, exp) {
			t.Error(pretty.Compare(act, exp))
		}
		// Encode to bytes again
		b2, err := c.Marshal(act)
		if err != nil {
			t.Error(err)
		}
		if !reflect.DeepEqual(b, b2) {
			t.Error(pretty.Compare(b2, b))
		}
	}
}
<|endoftext|>


<commit_before>package stats

import (
	"math/bits"
	"reflect"
	"testing"
	"time"
)

func TestWriteReadIO(t *testing.T) {
	apps := []string{"test0", "test1"}
	testSetup(t, apps)
	verifySetup(t, apps)

	StoreToFile("/tmp/coBenchTest.result.json")
	ReadFromFile("/tmp/coBenchTest.result.json")

	verifySetup(t, apps)
}

func testSetup(t *testing.T, apps []string) {
	for _, app := range apps {
		r := make([]time.Duration, 10)
		for i := 0; i < 10; i++ {
			r[i] = time.Duration(i)
		}
		AddReferenceRuntime(app, r)

		for _, CAT := range getCATMasks() {
			r := make([]time.Duration, 10)
			for i := 0; i < 10; i++ {
				r[i] = time.Duration(CAT) * time.Duration(i)
			}
			AddCATRuntime(app, CAT, r)
		}
	}

	r := make([]time.Duration, 10)
	for i := 0; i < 10; i++ {
		r[i] = time.Duration(i * i)
	}
	AddCoSchedRuntime(apps[0], apps[1], r)

	SetCommandline(false, 2, []string{"/tmp", "/tmp2"}, [2]string{"0-2", "3-5"}, apps, false, "/sys/fs/res/", 15, "3", 0.002)
}

func verifySetup(t *testing.T, apps []string) {
	for _, app := range apps {
		r := make([]time.Duration, 10)
		for i := 0; i < 10; i++ {
			r[i] = time.Duration(i)
		}
		runtime := ComputeRuntimeStats(r, NoCATMask, RuntimeT{})

		ref := GetReferenceRuntime(app)
		if !reflect.DeepEqual(*ref, runtime) {
			t.Errorf("Comparision failure with GetReferenceRuntime for app %v.", app)
		}

		catRs := GetIndvCATRuntimes(app)

		for _, CAT := range getCATMasks() {
			r := make([]time.Duration, 10)
			for i := 0; i < 10; i++ {
				r[i] = time.Duration(CAT) * time.Duration(i)
			}
			runtime := ComputeRuntimeStats(r, CAT, RuntimeT{})

			c := (*catRs)[bits.OnesCount64(CAT)]
			if !reflect.DeepEqual(c, runtime) {
				t.Errorf("Comparision failure with GetIndvCATRuntimes for app %v.", app)
			}
		}
	}

	r := make([]time.Duration, 10)
	for i := 0; i < 10; i++ {
		r[i] = time.Duration(i * i)
	}
	runtime := ComputeRuntimeStats(r, NoCATMask, RuntimeT{})
	co := GetCoSchedRuntimes(apps[0], apps[1])
	if !reflect.DeepEqual(*co, runtime) {
		t.Errorf("Comparision failure with GetCoSchedRuntimes for apps %v.", apps)
	}

	if !reflect.DeepEqual(apps, GetAllApplications()) {
		t.Errorf("Comparision failure with GetAllApplications for apps %v - %v.", apps, GetAllApplications())
	}
}

func getCATMasks() []uint64 {
	return []uint64{1, 3, 7}
}
<commit_msg>Fixed test.<commit_after>package stats

import (
	"math/bits"
	"reflect"
	"testing"
	"time"
)

func TestWriteReadIO(t *testing.T) {
	apps := []string{"test0", "test1"}
	testSetup(t, apps)
	verifySetup(t, apps)

	StoreToFile("/tmp/coBenchTest.result.json")
	ReadFromFile("/tmp/coBenchTest.result.json")

	verifySetup(t, apps)
}

func testSetup(t *testing.T, apps []string) {
	for _, app := range apps {
		r := make([]time.Duration, 10)
		for i := 0; i < 10; i++ {
			r[i] = time.Duration(i)
		}
		AddReferenceRuntime(app, r)

		for _, CAT := range getCATMasks() {
			r := make([]time.Duration, 10)
			for i := 0; i < 10; i++ {
				r[i] = time.Duration(CAT) * time.Duration(i)
			}
			AddCATRuntime(app, CAT, r)
		}
	}

	r := make([]time.Duration, 10)
	for i := 0; i < 10; i++ {
		r[i] = time.Duration(i * i)
	}
	AddCoSchedRuntime(apps[0], apps[1], r)

	SetCommandline(false, 2, []string{"/tmp", "/tmp2"}, [2]string{"0-2", "3-5"}, apps, false, "/sys/fs/res/", 15, "3", 0.002)
}

func verifySetup(t *testing.T, apps []string) {
	for _, app := range apps {
		r := make([]time.Duration, 10)
		for i := 0; i < 10; i++ {
			r[i] = time.Duration(i)
		}
		runtime := ComputeRuntimeStats(r, NoCATMask, RuntimeT{})

		ref := GetReferenceRuntime(app)
		if !reflect.DeepEqual(*ref, runtime) {
			t.Errorf("Comparision failure with GetReferenceRuntime for app %v.", app)
		}

		catRs := GetCATRuntimes(app)

		for _, CAT := range getCATMasks() {
			r := make([]time.Duration, 10)
			for i := 0; i < 10; i++ {
				r[i] = time.Duration(CAT) * time.Duration(i)
			}
			runtime := ComputeRuntimeStats(r, CAT, RuntimeT{})

			c := (*catRs)[bits.OnesCount64(CAT)]
			if !reflect.DeepEqual(c, runtime) {
				t.Errorf("Comparision failure with GetIndvCATRuntimes for app %v.", app)
			}
		}
	}

	r := make([]time.Duration, 10)
	for i := 0; i < 10; i++ {
		r[i] = time.Duration(i * i)
	}
	runtime := ComputeRuntimeStats(r, NoCATMask, RuntimeT{})
	co := GetCoSchedRuntimes(apps[0], apps[1])
	if !reflect.DeepEqual(*co, runtime) {
		t.Errorf("Comparision failure with GetCoSchedRuntimes for apps %v.", apps)
	}

	if !reflect.DeepEqual(apps, GetAllApplications()) {
		t.Errorf("Comparision failure with GetAllApplications for apps %v - %v.", apps, GetAllApplications())
	}
}

func getCATMasks() []uint64 {
	return []uint64{1, 3, 7}
}
<|endoftext|>


<commit_before>// +build integration

package riak

import (
	"io"
	"net"
	"reflect"
	"testing"
	"time"
)

func TestSuccessfulConnection(t *testing.T) {
	ln, err := net.Listen("tcp4", "127.0.0.1:1337")
	if err != nil {
		t.Error(err)
	}
	defer ln.Close()

	sawConnection := false

	go func() {
		c, err := ln.Accept()
		defer c.Close()
		if err != nil {
			t.Error(err.Error())
		}
		sawConnection = true
	}()

	addr, err := net.ResolveTCPAddr("tcp4", "127.0.0.1:1337")
	if err != nil {
		t.Error(err.Error())
	}

	opts := &connectionOptions{
		remoteAddress: addr,
	}

	conn, err := newConnection(opts)
	if err != nil {
		t.Error(err)
	}

	if err := conn.connect(); err != nil {
		t.Error(err)
	}

	if err := conn.close(); err != nil {
		t.Error(err)
	}

	if !sawConnection {
		t.Error("did not connect")
	}
}

func TestConnectionClosed(t *testing.T) {
	ln, err := net.Listen("tcp", "127.0.0.1:1338")
	if err != nil {
		t.Error(err)
	}
	defer ln.Close()

	go func() {
		c, err := ln.Accept()
		if err != nil {
			t.Error(err)
		}
		if err := c.Close(); err != nil {
			t.Error(err)
		}
	}()

	addr, err := net.ResolveTCPAddr("tcp4", "127.0.0.1:1338")
	if err != nil {
		t.Error(err.Error())
	}

	opts := &connectionOptions{
		remoteAddress: addr,
	}

	conn, err := newConnection(opts)
	if err != nil {
		t.Error(err)
	}

	if err := conn.connect(); err != nil {
		t.Error("unexpected error in connect", err)
	} else {
		if err := ln.Close(); err != nil {
			t.Error(err)
		}
		cmd := &PingCommand{}
		if err := conn.execute(cmd); err != nil {
			if operr, ok := err.(*net.OpError); ok {
				t.Log("op error", operr, operr.Op)
			} else if err == io.EOF {
				t.Log("saw EOF")
			} else {
				t.Errorf("expected to see net.OpError or io.EOF, but got '%s' (type: %v)", err.Error(), reflect.TypeOf(err))
			}
		} else {
			t.Error("expected error in execute")
		}
	}
}

func TestConnectionTimeout(t *testing.T) {
	addr, err := net.ResolveTCPAddr("tcp4", "10.255.255.1:65535")
	if err != nil {
		t.Error(err.Error())
	}

	opts := &connectionOptions{
		remoteAddress:  addr,
		connectTimeout: time.Millisecond * 150,
	}

	if conn, err := newConnection(opts); err == nil {
		if err := conn.connect(); err == nil {
			t.Error("expected to see timeout error")
		} else {
			if neterr, ok := err.(net.Error); ok && neterr.Timeout() {
				t.Log("timeout error", neterr)
			} else if operr, ok := err.(*net.OpError); ok {
				t.Log("op error", operr)
			} else {
				t.Errorf("expected to see timeout error, but got '%s' (type: %v)", err.Error(), reflect.TypeOf(err))
			}
		}
	} else {
		t.Error(err)
	}
}

func TestHealthCheckFail(t *testing.T) {
	ln, err := net.Listen("tcp", "127.0.0.1:1339")
	if err != nil {
		t.Error(err)
	}
	defer ln.Close()

	go func() {
		for {
			conn, err := ln.Accept()
			if err != nil {
				if _, ok := err.(*net.OpError); !ok {
					t.Error(err)
				}
				return
			}
			go handleClientMessageWithRiakError(t, conn, nil)
		}
	}()

	addr, err := net.ResolveTCPAddr("tcp4", "127.0.0.1:1339")
	if err != nil {
		t.Error(err.Error())
	}

	opts := &connectionOptions{
		remoteAddress:  addr,
		connectTimeout: thirtySeconds,
		healthCheck:    &PingCommand{},
	}

	if conn, err := newConnection(opts); err == nil {
		if err := conn.connect(); err == nil {
			t.Error("expected to see error")
		} else {
			if riakError, ok := err.(Error); ok == true {
				if expected, actual := "riak error - errcode '1', errmsg 'this is an error'", riakError.Error(); expected != actual {
					t.Errorf("expected %v, got %v", expected, actual)
				}
			} else {
				t.Error("expected to see Riak error, got:", err)
			}
		}
	} else {
		t.Error(err)
	}
}

func TestHealthCheckSuccess(t *testing.T) {
	ln, err := net.Listen("tcp", "127.0.0.1:1340")
	if err != nil {
		t.Error(err)
	}
	defer ln.Close()

	go func() {
		for {
			conn, err := ln.Accept()
			if err != nil {
				if _, ok := err.(*net.OpError); !ok {
					t.Error(err)
				}
				return
			}
			go readWritePingResp(t, conn)
		}
	}()

	addr, err := net.ResolveTCPAddr("tcp4", "127.0.0.1:1340")
	if err != nil {
		t.Error(err.Error())
	}

	opts := &connectionOptions{
		remoteAddress:  addr,
		connectTimeout: thirtySeconds,
		healthCheck:    &PingCommand{},
	}

	if conn, err := newConnection(opts); err == nil {
		if err := conn.connect(); err != nil {
			t.Error("unexpected error:", err)
		}
	} else {
		t.Error("unexpected error:", err)
	}
}
<commit_msg>Fix connection integration test.<commit_after>// +build integration

package riak

import (
	"io"
	"net"
	"reflect"
	"testing"
	"time"
)

func TestSuccessfulConnection(t *testing.T) {
	ln, err := net.Listen("tcp4", "127.0.0.1:1337")
	if err != nil {
		t.Error(err)
	}
	defer ln.Close()

	connChan := make(chan bool)
	go func() {
		c, err := ln.Accept()
		defer c.Close()
		if err != nil {
			t.Log(err.Error())
		}
		connChan <- true
	}()

	addr, err := net.ResolveTCPAddr("tcp4", "127.0.0.1:1337")
	if err != nil {
		t.Error(err.Error())
	}

	opts := &connectionOptions{
		remoteAddress: addr,
	}

	conn, err := newConnection(opts)
	if err != nil {
		t.Error(err)
	}

	if err := conn.connect(); err != nil {
		t.Error(err)
	}

	sawConnection := <-connChan

	if err := conn.close(); err != nil {
		t.Error(err)
	}

	if !sawConnection {
		t.Error("did not connect")
	}
}

func TestConnectionClosed(t *testing.T) {
	ln, err := net.Listen("tcp", "127.0.0.1:1338")
	if err != nil {
		t.Error(err)
	}
	defer ln.Close()

	go func() {
		c, err := ln.Accept()
		if err != nil {
			t.Error(err)
		}
		if err := c.Close(); err != nil {
			t.Error(err)
		}
	}()

	addr, err := net.ResolveTCPAddr("tcp4", "127.0.0.1:1338")
	if err != nil {
		t.Error(err.Error())
	}

	opts := &connectionOptions{
		remoteAddress: addr,
	}

	conn, err := newConnection(opts)
	if err != nil {
		t.Error(err)
	}

	if err := conn.connect(); err != nil {
		t.Error("unexpected error in connect", err)
	} else {
		if err := ln.Close(); err != nil {
			t.Error(err)
		}
		cmd := &PingCommand{}
		if err := conn.execute(cmd); err != nil {
			if operr, ok := err.(*net.OpError); ok {
				t.Log("op error", operr, operr.Op)
			} else if err == io.EOF {
				t.Log("saw EOF")
			} else {
				t.Errorf("expected to see net.OpError or io.EOF, but got '%s' (type: %v)", err.Error(), reflect.TypeOf(err))
			}
		} else {
			t.Error("expected error in execute")
		}
	}
}

func TestConnectionTimeout(t *testing.T) {
	addr, err := net.ResolveTCPAddr("tcp4", "10.255.255.1:65535")
	if err != nil {
		t.Error(err.Error())
	}

	opts := &connectionOptions{
		remoteAddress:  addr,
		connectTimeout: time.Millisecond * 150,
	}

	if conn, err := newConnection(opts); err == nil {
		if err := conn.connect(); err == nil {
			t.Error("expected to see timeout error")
		} else {
			if neterr, ok := err.(net.Error); ok && neterr.Timeout() {
				t.Log("timeout error", neterr)
			} else if operr, ok := err.(*net.OpError); ok {
				t.Log("op error", operr)
			} else {
				t.Errorf("expected to see timeout error, but got '%s' (type: %v)", err.Error(), reflect.TypeOf(err))
			}
		}
	} else {
		t.Error(err)
	}
}

func TestHealthCheckFail(t *testing.T) {
	ln, err := net.Listen("tcp", "127.0.0.1:1339")
	if err != nil {
		t.Error(err)
	}
	defer ln.Close()

	go func() {
		for {
			conn, err := ln.Accept()
			if err != nil {
				if _, ok := err.(*net.OpError); !ok {
					t.Error(err)
				}
				return
			}
			go handleClientMessageWithRiakError(t, conn, nil)
		}
	}()

	addr, err := net.ResolveTCPAddr("tcp4", "127.0.0.1:1339")
	if err != nil {
		t.Error(err.Error())
	}

	opts := &connectionOptions{
		remoteAddress:  addr,
		connectTimeout: thirtySeconds,
		healthCheck:    &PingCommand{},
	}

	if conn, err := newConnection(opts); err == nil {
		if err := conn.connect(); err == nil {
			t.Error("expected to see error")
		} else {
			if riakError, ok := err.(Error); ok == true {
				if expected, actual := "riak error - errcode '1', errmsg 'this is an error'", riakError.Error(); expected != actual {
					t.Errorf("expected %v, got %v", expected, actual)
				}
			} else {
				t.Error("expected to see Riak error, got:", err)
			}
		}
	} else {
		t.Error(err)
	}
}

func TestHealthCheckSuccess(t *testing.T) {
	ln, err := net.Listen("tcp", "127.0.0.1:1340")
	if err != nil {
		t.Error(err)
	}
	defer ln.Close()

	go func() {
		for {
			conn, err := ln.Accept()
			if err != nil {
				if _, ok := err.(*net.OpError); !ok {
					t.Error(err)
				}
				return
			}
			go readWritePingResp(t, conn)
		}
	}()

	addr, err := net.ResolveTCPAddr("tcp4", "127.0.0.1:1340")
	if err != nil {
		t.Error(err.Error())
	}

	opts := &connectionOptions{
		remoteAddress:  addr,
		connectTimeout: thirtySeconds,
		healthCheck:    &PingCommand{},
	}

	if conn, err := newConnection(opts); err == nil {
		if err := conn.connect(); err != nil {
			t.Error("unexpected error:", err)
		}
	} else {
		t.Error("unexpected error:", err)
	}
}
<|endoftext|>


<commit_before>/*Package nikto parses Nikto XML data into a similary formed struct.*/
package nikto

import (
	"encoding/xml"
)

// NiktoRun contains all the data from a single nikto scan.
type NiktoRun struct {
	HostsTest        string       `xml:"hoststest,attr"`
	Options          string       `xml:"options,attr"`
	Version          string       `xml:"version,attr"`
	ScanStart        string       `xml:"scanstart,attr"`
	ScanEnd          string       `xml:"scanend,attr"`
	ScanElapsed      string       `xml:"scanelapsed,attr"`
	XMLOutputVersion string       `xml:"nxmlversion,attr"`
	ScanDetails      []ScanDetail `xml:"scandetails"`
}

// ScanDetails contains all the information for a single host scan.
type ScanDetail struct {
	SSL        SSL        `xml:"ssl"`
	Items      []Item     `xml:"item"`
	Statistics Statistics `xml:"statistics"`
}

// SSL contains the SSL cipher information
type SSL struct {
	Ciphers string `xml:"ciphers,attr"`
	Issuers string `xml:"issuers,attr"`
	Info    string `xml:"info,attr"`
}

// Item contains the nikto finding results
type Item struct {
	Description string `xml:"description"`
	URI         string `xml:"uri"`
	NameLink    string `xml:"namelink"`
	IPLink      string `xml:"iplink"`
}

// Statistics contains the final scan statistics
type Statistics struct {
	Elapsed    string `xml:"elapsed,attr"`
	ItemsFound string `xml:"itemsfound,attr"`
	EndTime    string `xml:"endtime,attr"`
}

// Parse takes a byte array of nikto xml data and unmarshals it into an
// NiktoRun struct. All elements are returned as strings, it is up to the caller
// to check and cast them to the proper type.
func Parse(content []byte) (*NiktoRun, error) {
	r := &NiktoRun{}
	err := xml.Unmarshal(content, r)
	if err != nil {
		return r, err
	}
	return r, nil
}
<commit_msg>Delete nikto.go<commit_after><|endoftext|>


<commit_before>package connectors

import (
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"strings"

	"github.com/projectjane/jane/models"
)

// Twilio Struct for manipulating the webhook connector
type Twilio struct {
}

// Listen Twilio listener
func (x Twilio) Listen(commandMsgs chan<- models.Message, connector models.Connector) {
	defer Recovery(connector)
	return
}

// Command Twilio command parser
func (x Twilio) Command(message models.Message, publishMsgs chan<- models.Message, connector models.Connector) {
	return
}

// Publish Twilio publisher
func (x Twilio) Publish(connector models.Connector, message models.Message, target string) {
	client := &http.Client{}

	req, err := buildRequest(target, message.Out.Text, connector)
	if err != nil {
		log.Println(err)
	}

	resp, err := client.Do(req)
	if err != nil {
		log.Println(err)
	}
	defer resp.Body.Close()

	body, _ := ioutil.ReadAll(resp.Body)

	// if connector.Debug {
	log.Println(string(body))
	// }
}

// Help Twilio help information
func (x Twilio) Help(connector models.Connector) (help string) {
	help += fmt.Sprintf("Twilio messages sent from %s\n", connector.From)
	return help
}

func buildRequest(toNumber, body string, connector models.Connector) (*http.Request, error) {
	accountSid := connector.Key
	authToken := connector.Pass
	urlStr := "https://api.twilio.com/2010-04-01/Accounts/" + accountSid + "/Messages.json"

	values := url.Values{}
	values.Set("To", toNumber)
	values.Set("From", connector.From)
	values.Set("Body", body)

	req, err := http.NewRequest("POST", urlStr, strings.NewReader(values.Encode()))
	req.SetBasicAuth(accountSid, authToken)
	req.Header.Add("Accept", "application/json")
	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")

	return req, err
}
<commit_msg>Forgot to readd some commented out code<commit_after>package connectors

import (
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"strings"

	"github.com/projectjane/jane/models"
)

// Twilio Struct for manipulating the webhook connector
type Twilio struct {
}

// Listen Twilio listener
func (x Twilio) Listen(commandMsgs chan<- models.Message, connector models.Connector) {
	defer Recovery(connector)
	return
}

// Command Twilio command parser
func (x Twilio) Command(message models.Message, publishMsgs chan<- models.Message, connector models.Connector) {
	return
}

// Publish Twilio publisher
func (x Twilio) Publish(connector models.Connector, message models.Message, target string) {
	client := &http.Client{}

	req, err := buildRequest(target, message.Out.Text, connector)
	if err != nil {
		log.Println(err)
	}

	resp, err := client.Do(req)
	if err != nil {
		log.Println(err)
	}
	defer resp.Body.Close()

	body, _ := ioutil.ReadAll(resp.Body)

	if connector.Debug {
		log.Println(string(body))
	}
}

// Help Twilio help information
func (x Twilio) Help(connector models.Connector) (help string) {
	help += fmt.Sprintf("Twilio messages sent from %s\n", connector.From)
	return help
}

func buildRequest(toNumber, body string, connector models.Connector) (*http.Request, error) {
	accountSid := connector.Key
	authToken := connector.Pass
	urlStr := "https://api.twilio.com/2010-04-01/Accounts/" + accountSid + "/Messages.json"

	values := url.Values{}
	values.Set("To", toNumber)
	values.Set("From", connector.From)
	values.Set("Body", body)

	req, err := http.NewRequest("POST", urlStr, strings.NewReader(values.Encode()))
	req.SetBasicAuth(accountSid, authToken)
	req.Header.Add("Accept", "application/json")
	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")

	return req, err
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package pkix contains shared, low level structures used for ASN.1 parsing
// and serialization of X.509 certificates, CRL and OCSP.
package pkix

import (
	"encoding/asn1"
	"encoding/hex"
	"fmt"
	"math/big"
	"time"
)

// AlgorithmIdentifier represents the ASN.1 structure of the same name. See RFC
// 5280, section 4.1.1.2.
type AlgorithmIdentifier struct {
	Algorithm  asn1.ObjectIdentifier
	Parameters asn1.RawValue `asn1:"optional"`
}

type RDNSequence []RelativeDistinguishedNameSET

var attributeTypeNames = map[string]string{
	"2.5.4.6":  "C",
	"2.5.4.10": "O",
	"2.5.4.11": "OU",
	"2.5.4.3":  "CN",
	"2.5.4.5":  "SERIALNUMBER",
	"2.5.4.7":  "L",
	"2.5.4.8":  "ST",
	"2.5.4.9":  "STREET",
	"2.5.4.17": "POSTALCODE",
}

// String returns a string representation of the sequence r,
// roughly following the RFC 2253 Distinguished Names syntax.
func (r RDNSequence) String() string {
	s := ""
	for i := 0; i < len(r); i++ {
		rdn := r[len(r)-1-i]
		if i > 0 {
			s += ","
		}
		for j, tv := range rdn {
			if j > 0 {
				s += "+"
			}

			oidString := tv.Type.String()
			typeName, ok := attributeTypeNames[oidString]
			if !ok {
				derBytes, err := asn1.Marshal(tv.Value)
				if err == nil {
					s += oidString + "=#" + hex.EncodeToString(derBytes)
					continue // No value escaping necessary.
				}

				typeName = oidString
			}

			valueString := fmt.Sprint(tv.Value)
			escaped := make([]rune, 0, len(valueString))

			for k, c := range valueString {
				escape := false

				switch c {
				case ',', '+', '"', '\\', '<', '>', ';':
					escape = true

				case ' ':
					escape = k == 0 || k == len(valueString)-1

				case '#':
					escape = k == 0
				}

				if escape {
					escaped = append(escaped, '\\', c)
				} else {
					escaped = append(escaped, c)
				}
			}

			s += typeName + "=" + string(escaped)
		}
	}

	return s
}

type RelativeDistinguishedNameSET []AttributeTypeAndValue

// AttributeTypeAndValue mirrors the ASN.1 structure of the same name in
// RFC 5280, Section 4.1.2.4.
type AttributeTypeAndValue struct {
	Type  asn1.ObjectIdentifier
	Value any
}

// AttributeTypeAndValueSET represents a set of ASN.1 sequences of
// AttributeTypeAndValue sequences from RFC 2986 (PKCS #10).
type AttributeTypeAndValueSET struct {
	Type  asn1.ObjectIdentifier
	Value [][]AttributeTypeAndValue `asn1:"set"`
}

// Extension represents the ASN.1 structure of the same name. See RFC
// 5280, section 4.2.
type Extension struct {
	Id       asn1.ObjectIdentifier
	Critical bool `asn1:"optional"`
	Value    []byte
}

// Name represents an X.509 distinguished name. This only includes the common
// elements of a DN. Note that Name is only an approximation of the X.509
// structure. If an accurate representation is needed, asn1.Unmarshal the raw
// subject or issuer as an RDNSequence.
type Name struct {
	Country, Organization, OrganizationalUnit []string
	Locality, Province                        []string
	StreetAddress, PostalCode                 []string
	SerialNumber, CommonName                  string

	// Names contains all parsed attributes. When parsing distinguished names,
	// this can be used to extract non-standard attributes that are not parsed
	// by this package. When marshaling to RDNSequences, the Names field is
	// ignored, see ExtraNames.
	Names []AttributeTypeAndValue

	// ExtraNames contains attributes to be copied, raw, into any marshaled
	// distinguished names. Values override any attributes with the same OID.
	// The ExtraNames field is not populated when parsing, see Names.
	ExtraNames []AttributeTypeAndValue
}

// FillFromRDNSequence populates n from the provided RDNSequence.
// Multi-entry RDNs are flattened, all entries are added to the
// relevant n fields, and the grouping is not preserved.
func (n *Name) FillFromRDNSequence(rdns *RDNSequence) {
	for _, rdn := range *rdns {
		if len(rdn) == 0 {
			continue
		}

		for _, atv := range rdn {
			n.Names = append(n.Names, atv)
			value, ok := atv.Value.(string)
			if !ok {
				continue
			}

			t := atv.Type
			if len(t) == 4 && t[0] == 2 && t[1] == 5 && t[2] == 4 {
				switch t[3] {
				case 3:
					n.CommonName = value
				case 5:
					n.SerialNumber = value
				case 6:
					n.Country = append(n.Country, value)
				case 7:
					n.Locality = append(n.Locality, value)
				case 8:
					n.Province = append(n.Province, value)
				case 9:
					n.StreetAddress = append(n.StreetAddress, value)
				case 10:
					n.Organization = append(n.Organization, value)
				case 11:
					n.OrganizationalUnit = append(n.OrganizationalUnit, value)
				case 17:
					n.PostalCode = append(n.PostalCode, value)
				}
			}
		}
	}
}

var (
	oidCountry            = []int{2, 5, 4, 6}
	oidOrganization       = []int{2, 5, 4, 10}
	oidOrganizationalUnit = []int{2, 5, 4, 11}
	oidCommonName         = []int{2, 5, 4, 3}
	oidSerialNumber       = []int{2, 5, 4, 5}
	oidLocality           = []int{2, 5, 4, 7}
	oidProvince           = []int{2, 5, 4, 8}
	oidStreetAddress      = []int{2, 5, 4, 9}
	oidPostalCode         = []int{2, 5, 4, 17}
)

// appendRDNs appends a relativeDistinguishedNameSET to the given RDNSequence
// and returns the new value. The relativeDistinguishedNameSET contains an
// attributeTypeAndValue for each of the given values. See RFC 5280, A.1, and
// search for AttributeTypeAndValue.
func (n Name) appendRDNs(in RDNSequence, values []string, oid asn1.ObjectIdentifier) RDNSequence {
	if len(values) == 0 || oidInAttributeTypeAndValue(oid, n.ExtraNames) {
		return in
	}

	s := make([]AttributeTypeAndValue, len(values))
	for i, value := range values {
		s[i].Type = oid
		s[i].Value = value
	}

	return append(in, s)
}

// ToRDNSequence converts n into a single RDNSequence. The following
// attributes are encoded as multi-value RDNs:
//
//   - Country
//   - Organization
//   - OrganizationalUnit
//   - Locality
//   - Province
//   - StreetAddress
//   - PostalCode
//
// Each ExtraNames entry is encoded as an individual RDN.
func (n Name) ToRDNSequence() (ret RDNSequence) {
	ret = n.appendRDNs(ret, n.Country, oidCountry)
	ret = n.appendRDNs(ret, n.Province, oidProvince)
	ret = n.appendRDNs(ret, n.Locality, oidLocality)
	ret = n.appendRDNs(ret, n.StreetAddress, oidStreetAddress)
	ret = n.appendRDNs(ret, n.PostalCode, oidPostalCode)
	ret = n.appendRDNs(ret, n.Organization, oidOrganization)
	ret = n.appendRDNs(ret, n.OrganizationalUnit, oidOrganizationalUnit)
	if len(n.CommonName) > 0 {
		ret = n.appendRDNs(ret, []string{n.CommonName}, oidCommonName)
	}
	if len(n.SerialNumber) > 0 {
		ret = n.appendRDNs(ret, []string{n.SerialNumber}, oidSerialNumber)
	}
	for _, atv := range n.ExtraNames {
		ret = append(ret, []AttributeTypeAndValue{atv})
	}

	return ret
}

// String returns the string form of n, roughly following
// the RFC 2253 Distinguished Names syntax.
func (n Name) String() string {
	var rdns RDNSequence
	// If there are no ExtraNames, surface the parsed value (all entries in
	// Names) instead.
	if n.ExtraNames == nil {
		for _, atv := range n.Names {
			t := atv.Type
			if len(t) == 4 && t[0] == 2 && t[1] == 5 && t[2] == 4 {
				switch t[3] {
				case 3, 5, 6, 7, 8, 9, 10, 11, 17:
					// These attributes were already parsed into named fields.
					continue
				}
			}
			// Place non-standard parsed values at the beginning of the sequence
			// so they will be at the end of the string. See Issue 39924.
			rdns = append(rdns, []AttributeTypeAndValue{atv})
		}
	}
	rdns = append(rdns, n.ToRDNSequence()...)
	return rdns.String()
}

// oidInAttributeTypeAndValue reports whether a type with the given OID exists
// in atv.
func oidInAttributeTypeAndValue(oid asn1.ObjectIdentifier, atv []AttributeTypeAndValue) bool {
	for _, a := range atv {
		if a.Type.Equal(oid) {
			return true
		}
	}
	return false
}

// CertificateList represents the ASN.1 structure of the same name. See RFC
// 5280, section 5.1. Use Certificate.CheckCRLSignature to verify the
// signature.
type CertificateList struct {
	TBSCertList        TBSCertificateList
	SignatureAlgorithm AlgorithmIdentifier
	SignatureValue     asn1.BitString
}

// HasExpired reports whether certList should have been updated by now.
func (certList *CertificateList) HasExpired(now time.Time) bool {
	return !now.Before(certList.TBSCertList.NextUpdate)
}

// TBSCertificateList represents the ASN.1 structure of the same name. See RFC
// 5280, section 5.1.
//
// Deprecated: x509.RevocationList should be used instead.
type TBSCertificateList struct {
	Raw                 asn1.RawContent
	Version             int `asn1:"optional,default:0"`
	Signature           AlgorithmIdentifier
	Issuer              RDNSequence
	ThisUpdate          time.Time
	NextUpdate          time.Time            `asn1:"optional"`
	RevokedCertificates []RevokedCertificate `asn1:"optional"`
	Extensions          []Extension          `asn1:"tag:0,optional,explicit"`
}

// RevokedCertificate represents the ASN.1 structure of the same name. See RFC
// 5280, section 5.1.
//
// Deprecated: x509.RevocationList should be used instead.
type RevokedCertificate struct {
	SerialNumber   *big.Int
	RevocationTime time.Time
	Extensions     []Extension `asn1:"optional"`
}
<commit_msg>crypto/x509/pkix: move crl deprecation message<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package pkix contains shared, low level structures used for ASN.1 parsing
// and serialization of X.509 certificates, CRL and OCSP.
package pkix

import (
	"encoding/asn1"
	"encoding/hex"
	"fmt"
	"math/big"
	"time"
)

// AlgorithmIdentifier represents the ASN.1 structure of the same name. See RFC
// 5280, section 4.1.1.2.
type AlgorithmIdentifier struct {
	Algorithm  asn1.ObjectIdentifier
	Parameters asn1.RawValue `asn1:"optional"`
}

type RDNSequence []RelativeDistinguishedNameSET

var attributeTypeNames = map[string]string{
	"2.5.4.6":  "C",
	"2.5.4.10": "O",
	"2.5.4.11": "OU",
	"2.5.4.3":  "CN",
	"2.5.4.5":  "SERIALNUMBER",
	"2.5.4.7":  "L",
	"2.5.4.8":  "ST",
	"2.5.4.9":  "STREET",
	"2.5.4.17": "POSTALCODE",
}

// String returns a string representation of the sequence r,
// roughly following the RFC 2253 Distinguished Names syntax.
func (r RDNSequence) String() string {
	s := ""
	for i := 0; i < len(r); i++ {
		rdn := r[len(r)-1-i]
		if i > 0 {
			s += ","
		}
		for j, tv := range rdn {
			if j > 0 {
				s += "+"
			}

			oidString := tv.Type.String()
			typeName, ok := attributeTypeNames[oidString]
			if !ok {
				derBytes, err := asn1.Marshal(tv.Value)
				if err == nil {
					s += oidString + "=#" + hex.EncodeToString(derBytes)
					continue // No value escaping necessary.
				}

				typeName = oidString
			}

			valueString := fmt.Sprint(tv.Value)
			escaped := make([]rune, 0, len(valueString))

			for k, c := range valueString {
				escape := false

				switch c {
				case ',', '+', '"', '\\', '<', '>', ';':
					escape = true

				case ' ':
					escape = k == 0 || k == len(valueString)-1

				case '#':
					escape = k == 0
				}

				if escape {
					escaped = append(escaped, '\\', c)
				} else {
					escaped = append(escaped, c)
				}
			}

			s += typeName + "=" + string(escaped)
		}
	}

	return s
}

type RelativeDistinguishedNameSET []AttributeTypeAndValue

// AttributeTypeAndValue mirrors the ASN.1 structure of the same name in
// RFC 5280, Section 4.1.2.4.
type AttributeTypeAndValue struct {
	Type  asn1.ObjectIdentifier
	Value any
}

// AttributeTypeAndValueSET represents a set of ASN.1 sequences of
// AttributeTypeAndValue sequences from RFC 2986 (PKCS #10).
type AttributeTypeAndValueSET struct {
	Type  asn1.ObjectIdentifier
	Value [][]AttributeTypeAndValue `asn1:"set"`
}

// Extension represents the ASN.1 structure of the same name. See RFC
// 5280, section 4.2.
type Extension struct {
	Id       asn1.ObjectIdentifier
	Critical bool `asn1:"optional"`
	Value    []byte
}

// Name represents an X.509 distinguished name. This only includes the common
// elements of a DN. Note that Name is only an approximation of the X.509
// structure. If an accurate representation is needed, asn1.Unmarshal the raw
// subject or issuer as an RDNSequence.
type Name struct {
	Country, Organization, OrganizationalUnit []string
	Locality, Province                        []string
	StreetAddress, PostalCode                 []string
	SerialNumber, CommonName                  string

	// Names contains all parsed attributes. When parsing distinguished names,
	// this can be used to extract non-standard attributes that are not parsed
	// by this package. When marshaling to RDNSequences, the Names field is
	// ignored, see ExtraNames.
	Names []AttributeTypeAndValue

	// ExtraNames contains attributes to be copied, raw, into any marshaled
	// distinguished names. Values override any attributes with the same OID.
	// The ExtraNames field is not populated when parsing, see Names.
	ExtraNames []AttributeTypeAndValue
}

// FillFromRDNSequence populates n from the provided RDNSequence.
// Multi-entry RDNs are flattened, all entries are added to the
// relevant n fields, and the grouping is not preserved.
func (n *Name) FillFromRDNSequence(rdns *RDNSequence) {
	for _, rdn := range *rdns {
		if len(rdn) == 0 {
			continue
		}

		for _, atv := range rdn {
			n.Names = append(n.Names, atv)
			value, ok := atv.Value.(string)
			if !ok {
				continue
			}

			t := atv.Type
			if len(t) == 4 && t[0] == 2 && t[1] == 5 && t[2] == 4 {
				switch t[3] {
				case 3:
					n.CommonName = value
				case 5:
					n.SerialNumber = value
				case 6:
					n.Country = append(n.Country, value)
				case 7:
					n.Locality = append(n.Locality, value)
				case 8:
					n.Province = append(n.Province, value)
				case 9:
					n.StreetAddress = append(n.StreetAddress, value)
				case 10:
					n.Organization = append(n.Organization, value)
				case 11:
					n.OrganizationalUnit = append(n.OrganizationalUnit, value)
				case 17:
					n.PostalCode = append(n.PostalCode, value)
				}
			}
		}
	}
}

var (
	oidCountry            = []int{2, 5, 4, 6}
	oidOrganization       = []int{2, 5, 4, 10}
	oidOrganizationalUnit = []int{2, 5, 4, 11}
	oidCommonName         = []int{2, 5, 4, 3}
	oidSerialNumber       = []int{2, 5, 4, 5}
	oidLocality           = []int{2, 5, 4, 7}
	oidProvince           = []int{2, 5, 4, 8}
	oidStreetAddress      = []int{2, 5, 4, 9}
	oidPostalCode         = []int{2, 5, 4, 17}
)

// appendRDNs appends a relativeDistinguishedNameSET to the given RDNSequence
// and returns the new value. The relativeDistinguishedNameSET contains an
// attributeTypeAndValue for each of the given values. See RFC 5280, A.1, and
// search for AttributeTypeAndValue.
func (n Name) appendRDNs(in RDNSequence, values []string, oid asn1.ObjectIdentifier) RDNSequence {
	if len(values) == 0 || oidInAttributeTypeAndValue(oid, n.ExtraNames) {
		return in
	}

	s := make([]AttributeTypeAndValue, len(values))
	for i, value := range values {
		s[i].Type = oid
		s[i].Value = value
	}

	return append(in, s)
}

// ToRDNSequence converts n into a single RDNSequence. The following
// attributes are encoded as multi-value RDNs:
//
//   - Country
//   - Organization
//   - OrganizationalUnit
//   - Locality
//   - Province
//   - StreetAddress
//   - PostalCode
//
// Each ExtraNames entry is encoded as an individual RDN.
func (n Name) ToRDNSequence() (ret RDNSequence) {
	ret = n.appendRDNs(ret, n.Country, oidCountry)
	ret = n.appendRDNs(ret, n.Province, oidProvince)
	ret = n.appendRDNs(ret, n.Locality, oidLocality)
	ret = n.appendRDNs(ret, n.StreetAddress, oidStreetAddress)
	ret = n.appendRDNs(ret, n.PostalCode, oidPostalCode)
	ret = n.appendRDNs(ret, n.Organization, oidOrganization)
	ret = n.appendRDNs(ret, n.OrganizationalUnit, oidOrganizationalUnit)
	if len(n.CommonName) > 0 {
		ret = n.appendRDNs(ret, []string{n.CommonName}, oidCommonName)
	}
	if len(n.SerialNumber) > 0 {
		ret = n.appendRDNs(ret, []string{n.SerialNumber}, oidSerialNumber)
	}
	for _, atv := range n.ExtraNames {
		ret = append(ret, []AttributeTypeAndValue{atv})
	}

	return ret
}

// String returns the string form of n, roughly following
// the RFC 2253 Distinguished Names syntax.
func (n Name) String() string {
	var rdns RDNSequence
	// If there are no ExtraNames, surface the parsed value (all entries in
	// Names) instead.
	if n.ExtraNames == nil {
		for _, atv := range n.Names {
			t := atv.Type
			if len(t) == 4 && t[0] == 2 && t[1] == 5 && t[2] == 4 {
				switch t[3] {
				case 3, 5, 6, 7, 8, 9, 10, 11, 17:
					// These attributes were already parsed into named fields.
					continue
				}
			}
			// Place non-standard parsed values at the beginning of the sequence
			// so they will be at the end of the string. See Issue 39924.
			rdns = append(rdns, []AttributeTypeAndValue{atv})
		}
	}
	rdns = append(rdns, n.ToRDNSequence()...)
	return rdns.String()
}

// oidInAttributeTypeAndValue reports whether a type with the given OID exists
// in atv.
func oidInAttributeTypeAndValue(oid asn1.ObjectIdentifier, atv []AttributeTypeAndValue) bool {
	for _, a := range atv {
		if a.Type.Equal(oid) {
			return true
		}
	}
	return false
}

// CertificateList represents the ASN.1 structure of the same name. See RFC
// 5280, section 5.1. Use Certificate.CheckCRLSignature to verify the
// signature.
//
// Deprecated: x509.RevocationList should be used instead.
type CertificateList struct {
	TBSCertList        TBSCertificateList
	SignatureAlgorithm AlgorithmIdentifier
	SignatureValue     asn1.BitString
}

// HasExpired reports whether certList should have been updated by now.
func (certList *CertificateList) HasExpired(now time.Time) bool {
	return !now.Before(certList.TBSCertList.NextUpdate)
}

// TBSCertificateList represents the ASN.1 structure of the same name. See RFC
// 5280, section 5.1.
//
// Deprecated: x509.RevocationList should be used instead.
type TBSCertificateList struct {
	Raw                 asn1.RawContent
	Version             int `asn1:"optional,default:0"`
	Signature           AlgorithmIdentifier
	Issuer              RDNSequence
	ThisUpdate          time.Time
	NextUpdate          time.Time            `asn1:"optional"`
	RevokedCertificates []RevokedCertificate `asn1:"optional"`
	Extensions          []Extension          `asn1:"tag:0,optional,explicit"`
}

// RevokedCertificate represents the ASN.1 structure of the same name. See RFC
// 5280, section 5.1.
type RevokedCertificate struct {
	SerialNumber   *big.Int
	RevocationTime time.Time
	Extensions     []Extension `asn1:"optional"`
}
<|endoftext|>


<commit_before>package letsencrypt

import (
	"errors"
	"net/http"
	"sync"
)

// All valid responses by the ACME server are required to have
// a Replay-Nonce header.
// https://letsencrypt.github.io/acme-spec/#replay-protection
var errNoNonce = errors.New("no Replay-Nonce header in HTTP response")

// nonceRoundTripper is a round tripper which
type nonceRoundTripper struct {
	rt http.RoundTripper

	// a cache of unused nonce tokens for JWS objects
	mu     *sync.Mutex
	nonces []string
}

func newNonceRoundTripper(rt http.RoundTripper) *nonceRoundTripper {
	return &nonceRoundTripper{rt: rt, mu: new(sync.Mutex)}
}

func (nrt *nonceRoundTripper) Nonce() (string, error) {
	nrt.mu.Lock()
	defer nrt.mu.Unlock()

	n := len(nrt.nonces)
	if n == 0 {
		// TODO: determine a good strategy for replenishing the nonce cache.
		return "", errors.New("acme: nonce cache depleted")
	}
	// pop a nonce from the stack
	nonce := nrt.nonces[n-1]
	nrt.nonces = nrt.nonces[:n-1]
	return nonce, nil
}

func (nrt *nonceRoundTripper) RoundTrip(req *http.Request) (resp *http.Response, err error) {
	if nrt.rt == nil {
		resp, err = http.DefaultTransport.RoundTrip(req)
	} else {
		resp, err = nrt.rt.RoundTrip(req)
	}
	if err != nil {
		return
	}
	tok := resp.Header.Get("Replay-Nonce")
	if tok == "" {
		return
	}
	nrt.mu.Lock()
	defer nrt.mu.Unlock()

	// cap the size of the nonce cache
	if len(nrt.nonces) < 2048 {
		nrt.nonces = append(nrt.nonces, tok)
	}
	return
}
<commit_msg>Set http.DefaultTransport if one is not provided to newNonceRoundTripper<commit_after>package letsencrypt

import (
	"errors"
	"net/http"
	"sync"
)

// All valid responses by the ACME server are required to have
// a Replay-Nonce header.
// https://letsencrypt.github.io/acme-spec/#replay-protection
var errNoNonce = errors.New("no Replay-Nonce header in HTTP response")

// nonceRoundTripper is a round tripper which
type nonceRoundTripper struct {
	rt http.RoundTripper

	// a cache of unused nonce tokens for JWS objects
	mu     *sync.Mutex
	nonces []string
}

func newNonceRoundTripper(rt http.RoundTripper) *nonceRoundTripper {
	if rt == nil {
		rt = http.DefaultTransport
	}
	return &nonceRoundTripper{rt: rt, mu: new(sync.Mutex)}
}

func (nrt *nonceRoundTripper) Nonce() (string, error) {
	nrt.mu.Lock()
	defer nrt.mu.Unlock()

	n := len(nrt.nonces)
	if n == 0 {
		// TODO: determine a good strategy for replenishing the nonce cache.
		return "", errors.New("acme: nonce cache depleted")
	}
	// pop a nonce from the stack
	nonce := nrt.nonces[n-1]
	nrt.nonces = nrt.nonces[:n-1]
	return nonce, nil
}

func (nrt *nonceRoundTripper) RoundTrip(req *http.Request) (resp *http.Response, err error) {
	resp, err = nrt.rt.RoundTrip(req)
	if err != nil {
		return
	}
	tok := resp.Header.Get("Replay-Nonce")
	if tok == "" {
		return
	}
	nrt.mu.Lock()
	defer nrt.mu.Unlock()

	// cap the size of the nonce cache
	if len(nrt.nonces) < 2048 {
		nrt.nonces = append(nrt.nonces, tok)
	}

	return
}
<|endoftext|>


<commit_before>/*=============================================================================
#     FileName: rwstream.go
#       Author: sunminghong, allen.fantasy@gmail.com, http://weibo.com/5d13
#         Team: http://1201.us
#   LastChange: 2015-08-25 16:09:39
#      History:
=============================================================================*/

/*

*/

package gotools

import (
    "encoding/binary"
    "errors"
)

const (
    BigEndian    = 0
    LittleEndian = 1
)

func GetEndianer(endian int) IEndianer {
    if endian == BigEndian {
        return binary.BigEndian
    } else {
        return binary.LittleEndian
    }
}

//switch bigEndianer or littleEndianer
type IEndianer interface {
    Uint16(b []byte) uint16
    PutUint16(b []byte, v uint16)

    Uint32(b []byte) uint32
    PutUint32(b []byte, v uint32)

    Uint64(b []byte) uint64
    PutUint64(b []byte, v uint64)
}

// A Buffer is a variable-sized buffer of bytes with Read and Write methods.
// The zero value for Buffer is an empty buffer ready to use.
type RWStream struct {
    buffSize int

    //Endian   int //default to false, means that is littleEdian
    Endianer IEndianer

    buf []byte // contents are the bytes buf[off:len(buf)]

    off  int // read at &buf[off], write at &buf[len(buf)]
    end  int // data end pos, data = buf[off,end]
    last int // last read operation, so that Unread* can work correctly.
}

func NewRWStream(buf interface{}, endianer IEndianer) *RWStream {
    b := &RWStream{Endianer: endianer}

    b.Endianer = endianer
    /*
    b.Endian = endian
    if endian == BigEndian {
        b.Endianer = binary.BigEndian
    } else {
        b.Endianer = binary.LittleEndian
    }
    */

    b.Init(buf)
    return b
}

// ErrTooLarge is passed to panic if memory cannot be allocated to store data in a buffer.
var ErrTooLarge = errors.New("net.RWStream: too large")
var ErrIndex = errors.New("net.RWStream: index over range")

func (b *RWStream) DebugOut() (string,int,[]byte,int,int,int) { return "rwstream.buf:",len(b.buf),b.buf[:],b.off,b.last,b.end}
func (b *RWStream) Bytes() []byte { return b.buf[b.off:b.end] }

func (b *RWStream) Len() int { return b.end - b.off }

func (b *RWStream) Init(params ...interface{}) {
    if len(params) > 0 {
        buf := params[0]

        switch tmp := buf.(type) {
        case int:
            b.buffSize = tmp
            b.buf = make([]byte, b.buffSize)
            b.last = 0
            b.off = 0
            b.end = 0
        case []byte:
            b.buf = tmp[:]
            b.buffSize = len(tmp)
            b.last = 0
            b.off = 0
            b.end = len(tmp)
        default:
            b.buffSize = 1024
            b.buf = make([]byte, b.buffSize)
            b.last = 0
            b.end = 0
            b.off = 0
        }
    } else {
        b.last = 0
        b.end = 0
        b.off = 0
    }
}

//call Reset before each use this Buffer
func (b *RWStream) Reset() {
    b.off = b.end
    b.last = b.off
}

// makeSlice allocates a slice of size n. If the allocation fails, it panics
// with ErrTooLarge.
func makeSlice(n int) []byte {
    // If the make fails, give a known error.
    defer func() {
        if recover() != nil {
            panic(ErrTooLarge)
        }
    }()
    return make([]byte, n*2)
}

// grow grows the buffer to guarantee space for n more bytes.
// It returns the index where bytes should be written.
// If the buffer can't grow it will panic with ErrTooLarge.
func (b *RWStream) grow(n int) int {
    m := b.Len()
    x := len(b.buf)

    if b.end+n > x {
        if m+n > x {
            var buf []byte
            // not enough space anywhere
            buf = makeSlice(m + n)
            copy(buf, b.buf[b.off:])
            b.buf = buf
        } else {
            copy(b.buf[0:], b.buf[b.off:b.off+m])
        }
        b.last -= b.off
        b.off = 0
        b.end = m
    //} else {
        //if x > b.buffSize {
            //b.buf = b.buf[b.off : b.off+m]
            //b.last -= b.off
            //b.off = 0
            //b.end = m
        //}
    }
    return b.end
}

// Write appends the contents of p to the buffer.  The return
// value n is the length of p; err is always nil.
// If the buffer becomes too large, Write will panic with
// ErrTooLarge.
func (b *RWStream) Write(p []byte) (n int) {
    nn := len(p)
    m := b.grow(nn)
    b.end += nn

    return copy(b.buf[m:], p)
}

func (b *RWStream) GetPos() int {
    return b.last - b.off
}

func (b *RWStream) SetPos(pos int) {
    if pos < 0 {
        b.last += pos
        if b.last < b.off {
            b.last = b.off
        }
        return
    }

    if pos+b.off > b.end {
        b.last = b.end
    } else {
        b.last = pos + b.off
    }
}

func (b *RWStream) Read(n int) ([]byte,int) {
    if b.last+n > b.end {
        return nil,0
        //n = b.end - b.last
    }
    //if n<0 {
    //    return 0,nil
    //}
    p := b.buf[b.last : b.last+n]
    b.last += n
    return p, n
}

// WriteString appends the contents of s to the buffer.  The return
// value n is the length of s; err is always nil.
// If the buffer becomes too large, WriteString will panic with
// ErrTooLarge.
func (b *RWStream) WriteString(s string) int {
    b.WriteUint(uint(len(s)))
    return b.Write([]byte(s))
}

func (b *RWStream) WriteStringU32(s string) int {
    b.WriteUint32(uint32(len(s)))
    return b.Write([]byte(s))
}

func (b *RWStream) WriteByte(c byte) int {
    m := b.grow(1)
    b.buf[m] = c
    b.end += 1
    return 1
}

func (b *RWStream) WriteUint16(x uint16) int {
    var buf = make([]byte, 2)
    b.Endianer.PutUint16(buf, x)
    return b.Write(buf)
}

func (b *RWStream) WriteUint32(x uint32) int {
    var buf = make([]byte, 4)
    b.Endianer.PutUint32(buf, x)
    return b.Write(buf)
}

func (b *RWStream) WriteUint64(x uint64) int {
    var buf = make([]byte, 8)
    b.Endianer.PutUint64(buf, x)
    return b.Write(buf)
}

func (b *RWStream) ReadByte() (byte, error) {
    buf, n := b.Read(1)
    if n < 1 {
        return 0, ErrIndex
    }
    return buf[0], nil
}

func (b *RWStream) ReadUint16() (uint16, error) {
    buf, n := b.Read(2)
    if n < 2 {
        return 0, ErrIndex
    }
    x := b.Endianer.Uint16(buf)
    return x, nil
}

func (b *RWStream) ReadUint32() (uint32, error) {
    buf, n := b.Read(4)
    if n < 4 {
        return 0, ErrIndex
    }
    x := b.Endianer.Uint32(buf)
    return x, nil
}

func (b *RWStream) ReadUint64() (uint64, error) {
    buf, n := b.Read(8)
    if n < 8 {
        return 0, ErrIndex
    }
    x := b.Endianer.Uint64(buf)
    return x, nil
}

func (b *RWStream) ReadUint() (uint, error) {
    if b.last >= b.end {
        return 0, ErrIndex
    }

    var x uint
    var s uint
    for {
        i := 0
        b, err := b.ReadByte()
        if err != nil {
            break
        }

        if b < 0x80 {
            if i > 9 || i == 9 && b > 1 {
                return 0, ErrTooLarge
            }
            return x | uint(b)<<s, nil
        }
        x |= uint(b&0x7f) << s
        s += 7
        i += 1
    }
    return 0, ErrTooLarge
}

func (b *RWStream) ReadInt() (int, error) {
    ux, err := b.ReadUint() // ok to continue in presence of error
    if err != nil {
        return 0, err
    }

    x := int(ux >> 1)
    if ux&1 != 0 {
        x = ^x
    }
    return x, nil
}

func (b *RWStream) WriteUint(x uint) int {
    buf := [8]byte{}
    i := 0
    for x >= 0x80 {
        buf[i] = byte(x) | 0x80
        x >>= 7
        i++
    }
    buf[i] = byte(x)

    b.Write(buf[0 : i+1])
    return i + 1
}

func (b *RWStream) WriteInt(x int) int {
    ux := uint(x) << 1
    if x < 0 {
        ux = ^ux
    }
    return b.WriteUint(ux)
}

func (b *RWStream) ReadStringU32() (string, error) {
    l, err := b.ReadUint32()
    if err != nil {
        return "", err
    }

    ll := int(l)
    buf, n := b.Read(ll)
    if n < ll {
        return "", ErrIndex
    }

    return string(buf), nil
}

func (b *RWStream) ReadString() (string, error) {
    l, err := b.ReadUint()
    if err != nil {
        return "", err
    }

    ll := int(l)
    buf, n := b.Read(ll)
    if n < ll {
        return "", ErrIndex
    }

    return string(buf), nil
}
<commit_msg>finish to add ringbuffer to rwstream<commit_after>/*=============================================================================
#     FileName: rwstream.go
#       Author: sunminghong, allen.fantasy@gmail.com, http://weibo.com/5d13
#         Team: http://1201.us
#   LastChange: 2015-08-25 16:09:39
#      History:
=============================================================================*/

/*

*/

package gotools

import (
    "encoding/binary"
    "errors"
    "sync"

    "fmt"
)

const (
    BigEndian    = 0
    LittleEndian = 1
)

func GetEndianer(endian int) IEndianer {
    if endian == BigEndian {
        return binary.BigEndian
    } else {
        return binary.LittleEndian
    }
}

//switch bigEndianer or littleEndianer
type IEndianer interface {
    Uint16(b []byte) uint16
    PutUint16(b []byte, v uint16)

    Uint32(b []byte) uint32
    PutUint32(b []byte, v uint32)

    Uint64(b []byte) uint64
    PutUint64(b []byte, v uint64)
}

// A Buffer is a variable-sized buffer of bytes with Read and Write methods.
// The zero value for Buffer is an empty buffer ready to use.
type RWStream struct {
    buffSize int

    //Endian   int //default to false, means that is littleEdian
    Endianer IEndianer

    buf []byte
    rlock *sync.RWMutex

    end  int
    last int
}

func NewRWStream(buf interface{}, endianer IEndianer) *RWStream {
    b := &RWStream{Endianer: endianer, rlock:new(sync.RWMutex)}

    b.Endianer = endianer
    /*
    b.Endian = endian
    if endian == BigEndian {
        b.Endianer = binary.BigEndian
    } else {
        b.Endianer = binary.LittleEndian
    }
    */

    b.Init(buf)
    return b
}

// ErrTooLarge is passed to panic if memory cannot be allocated to store data in a buffer.
var ErrTooLarge = errors.New("net.RWStream: too large")
var ErrIndex = errors.New("net.RWStream: index over range")

func (b *RWStream) DebugOut() (string,int,[]byte,int,int) { return "rwstream.buf:",b.buffSize,b.buf[:],b.last,b.end}

func (b *RWStream) Bytes() []byte {
    if b.last <= b.end {
        p := b.buf[b.last : b.end]
        return p
    }

    //分段读取
    buf := make([]byte, b.Len())
    copy(buf, b.buf[b.last:])
    copy(buf[b.buffSize - b.last:], b.buf[:b.end])

    return buf
}

func (b *RWStream) Len() int {
    if b.end > b.last {
        return b.end - b.last
    } else {
        return b.buffSize + b.end - b.last
    }
}

func (b *RWStream) Init(params ...interface{}) {
    if len(params) > 0 {
        buf := params[0]

        switch tmp := buf.(type) {
        case int:
            b.buffSize = tmp
            b.buf = make([]byte, b.buffSize)
            b.last = 0
            b.end = 0
        case []byte:
            b.buf = tmp[:]
            b.buffSize = len(tmp)
            b.last = 0
            b.end = len(tmp)
        default:
            b.buffSize = 1024
            b.buf = make([]byte, b.buffSize)
            b.last = 0
            b.end = 0
        }
    } else {
        b.last = 0
        b.end = 0
        b.buffSize = 1024
        b.buf = make([]byte, b.buffSize)
    }
}

//call Reset before each use this Buffer
func (b *RWStream) Reset() {
    b.end = 0
    b.last = 0
    b.buffSize = 1024
    b.buf = make([]byte, b.buffSize)
}

func (b *RWStream) Write(p []byte) (i int) {
    n := len(p)
    m := b.Len()
    max := b.buffSize

    if b.end + n <= max {
        copy(b.buf[b.end:], p)
        b.end = (b.end + n) % max
        return n
    }

    if m + n > max {
        b.rlock.Lock()
        // not enough space anywhere
        //icap := m + n - max
        icap := m + n
        if icap < 10240 {
            icap = icap * 2
        }
        fmt.Printf("makeslice makeslice makeslice", b.last,b.end, icap,n, "\n")
        tmp := make([]byte, icap)
        //tmp := make([]byte, icap + max)
        if b.last < b.end {
            copy(tmp, b.buf[b.last:b.end])
        } else {
            copy(tmp, b.buf[b.last:b.buffSize])
            copy(tmp[b.buffSize - b.last:], b.buf[:b.end])
        }
        copy(tmp[m:], p)

        fmt.Printf("|||% X\n",b.buf)
        fmt.Printf("|||% X\n",tmp)

        b.buffSize = len(tmp)
        b.last = 0
        b.end = n + m
        b.buf = tmp
        b.rlock.Unlock()

        return n
    }

    if b.last > b.end {
        copy(b.buf[b.end:], p)
        b.end = (b.end + n) % max
        return n
    }

    //分段写入
    copy(b.buf[b.end:], p[:max - b.end])
    copy(b.buf[0:], p[max - b.end:])
    b.end = n + b.end - max

    return n
}

func (b *RWStream) Read(n int) ([]byte,int) {
    end := b.end
    if b.last <= b.end {
        if b.last + n <= end {
            p := b.buf[b.last : b.last+n]
            b.last += n
            return p, n
        } else {
            return nil,0
        }
    }

    max := b.buffSize
    if b.last + n < max {
        p := b.buf[b.last : b.last+n]
        b.last += n
        return p, n
    }

    if b.last + n > b.end + max {
        return nil,0
    }

    //分段读取
    buf := make([]byte, n)
    copy(buf, b.buf[b.last:])
    copy(buf[max - b.last:], b.buf[:n + b.last - max])
    b.last = n - max + b.last

    return buf, n
}

func (b *RWStream) GetPos() int {
    return b.last
}

func (b *RWStream) SetPos(pos int) {
    if pos < 0 {
        b.last += pos
        if b.last < 0 {
            b.last += b.buffSize
        }
        return
    }

    if b.end < b.last {
        la := (b.last + pos) % b.buffSize
        if la > b.end {
            b.last = b.end
        } else {
            b.last = la
        }
    } else if b.last + pos > b.end {
        b.last = b.end
    } else {
        b.last += pos
    }
}

// WriteString appends the contents of s to the buffer.  The return
// value n is the length of s; err is always nil.
// If the buffer becomes too large, WriteString will panic with
// ErrTooLarge.
func (b *RWStream) WriteString(s string) int {
    b.WriteUint(uint(len(s)))
    return b.Write([]byte(s))
}

func (b *RWStream) WriteStringU32(s string) int {
    b.WriteUint32(uint32(len(s)))
    return b.Write([]byte(s))
}

func (b *RWStream) WriteByte(c byte) int {
    b.Write([]byte{c})
    return 1
}

func (b *RWStream) WriteUint16(x uint16) int {
    var buf = make([]byte, 2)
    b.Endianer.PutUint16(buf, x)
    return b.Write(buf)
}

func (b *RWStream) WriteUint32(x uint32) int {
    var buf = make([]byte, 4)
    b.Endianer.PutUint32(buf, x)
    return b.Write(buf)
}

func (b *RWStream) WriteUint64(x uint64) int {
    var buf = make([]byte, 8)
    b.Endianer.PutUint64(buf, x)
    return b.Write(buf)
}

func (b *RWStream) ReadByte() (byte, error) {
    buf, n := b.Read(1)
    if n < 1 {
        return 0, ErrIndex
    }
    return buf[0], nil
}

func (b *RWStream) ReadUint16() (uint16, error) {
    buf, n := b.Read(2)
    if n < 2 {
        return 0, ErrIndex
    }
    x := b.Endianer.Uint16(buf)
    return x, nil
}

func (b *RWStream) ReadUint32() (uint32, error) {
    buf, n := b.Read(4)
    if n < 4 {
        return 0, ErrIndex
    }
    x := b.Endianer.Uint32(buf)
    return x, nil
}

func (b *RWStream) ReadUint64() (uint64, error) {
    buf, n := b.Read(8)
    if n < 8 {
        return 0, ErrIndex
    }
    x := b.Endianer.Uint64(buf)
    return x, nil
}

func (b *RWStream) ReadUint() (uint, error) {
    if b.Len() < 1 {
        return 0, ErrIndex
    }

    var x uint
    var s uint
    for {
        i := 0
        b, err := b.ReadByte()
        if err != nil {
            break
        }

        if b < 0x80 {
            if i > 9 || i == 9 && b > 1 {
                return 0, ErrTooLarge
            }
            return x | uint(b)<<s, nil
        }
        x |= uint(b & 0x7f) << s
        s += 7
        i += 1
    }
    return 0, ErrTooLarge
}

func (b *RWStream) ReadInt() (int, error) {
    ux, err := b.ReadUint() // ok to continue in presence of error
    if err != nil {
        return 0, err
    }

    x := int(ux >> 1)
    if ux&1 != 0 {
        x = ^x
    }
    return x, nil
}

func (b *RWStream) WriteUint(x uint) int {
    buf := [8]byte{}
    i := 0
    for x >= 0x80 {
        buf[i] = byte(x) | 0x80
        x >>= 7
        i++
    }
    buf[i] = byte(x)

    b.Write(buf[0 : i+1])
    return i + 1
}

func (b *RWStream) WriteInt(x int) int {
    ux := uint(x) << 1
    if x < 0 {
        ux = ^ux
    }
    return b.WriteUint(ux)
}

func (b *RWStream) ReadStringU32() (string, error) {
    l, err := b.ReadUint32()
    if err != nil {
        return "", err
    }

    ll := int(l)
    buf, n := b.Read(ll)
    if n < ll {
        return "", ErrIndex
    }

    return string(buf), nil
}

func (b *RWStream) ReadString() (string, error) {
    l, err := b.ReadUint()
    if err != nil {
        return "", err
    }

    ll := int(l)
    buf, n := b.Read(ll)
    if n < ll {
        return "", ErrIndex
    }

    return string(buf), nil
}
<|endoftext|>


<commit_before>package etcd

import (
	"errors"
	"fmt"
	"io/ioutil"
	"math/rand"
	"net/http"
	"net/url"
	"path"
	"strings"
	"sync"
	"time"
)

// Errors introduced by handling requests
var (
	ErrRequestCancelled = errors.New("sending request is cancelled")
)

type RawRequest struct {
	Method       string
	RelativePath string
	Values       url.Values
	Cancel       <-chan bool
}

// NewRawRequest returns a new RawRequest
func NewRawRequest(method, relativePath string, values url.Values, cancel <-chan bool) *RawRequest {
	return &RawRequest{
		Method:       method,
		RelativePath: relativePath,
		Values:       values,
		Cancel:       cancel,
	}
}

// getCancelable issues a cancelable GET request
func (c *Client) getCancelable(key string, options Options,
	cancel <-chan bool) (*RawResponse, error) {
	logger.Debugf("get %s [%s]", key, c.cluster.Leader)
	p := keyToPath(key)

	// If consistency level is set to STRONG, append
	// the `consistent` query string.
	if c.config.Consistency == STRONG_CONSISTENCY {
		options["consistent"] = true
	}

	str, err := options.toParameters(VALID_GET_OPTIONS)
	if err != nil {
		return nil, err
	}
	p += str

	req := NewRawRequest("GET", p, nil, cancel)
	resp, err := c.SendRequest(req)

	if err != nil {
		return nil, err
	}

	return resp, nil
}

// get issues a GET request
func (c *Client) get(key string, options Options) (*RawResponse, error) {
	return c.getCancelable(key, options, nil)
}

// put issues a PUT request
func (c *Client) put(key string, value string, ttl uint64,
	options Options) (*RawResponse, error) {

	logger.Debugf("put %s, %s, ttl: %d, [%s]", key, value, ttl, c.cluster.Leader)
	p := keyToPath(key)

	str, err := options.toParameters(VALID_PUT_OPTIONS)
	if err != nil {
		return nil, err
	}
	p += str

	req := NewRawRequest("PUT", p, buildValues(value, ttl), nil)
	resp, err := c.SendRequest(req)

	if err != nil {
		return nil, err
	}

	return resp, nil
}

// post issues a POST request
func (c *Client) post(key string, value string, ttl uint64) (*RawResponse, error) {
	logger.Debugf("post %s, %s, ttl: %d, [%s]", key, value, ttl, c.cluster.Leader)
	p := keyToPath(key)

	req := NewRawRequest("POST", p, buildValues(value, ttl), nil)
	resp, err := c.SendRequest(req)

	if err != nil {
		return nil, err
	}

	return resp, nil
}

// delete issues a DELETE request
func (c *Client) delete(key string, options Options) (*RawResponse, error) {
	logger.Debugf("delete %s [%s]", key, c.cluster.Leader)
	p := keyToPath(key)

	str, err := options.toParameters(VALID_DELETE_OPTIONS)
	if err != nil {
		return nil, err
	}
	p += str

	req := NewRawRequest("DELETE", p, nil, nil)
	resp, err := c.SendRequest(req)

	if err != nil {
		return nil, err
	}

	return resp, nil
}

// SendRequest sends a HTTP request and returns a Response as defined by etcd
func (c *Client) SendRequest(rr *RawRequest) (*RawResponse, error) {

	var req *http.Request
	var resp *http.Response
	var httpPath string
	var err error
	var respBody []byte

	reqs := make([]http.Request, 0)
	resps := make([]http.Response, 0)

	checkRetry := c.CheckRetry
	if checkRetry == nil {
		checkRetry = DefaultCheckRetry
	}

	cancelled := make(chan bool, 1)
	reqLock := new(sync.Mutex)

	if rr.Cancel != nil {
		cancelRoutine := make(chan bool)
		defer close(cancelRoutine)

		go func() {
			select {
			case <-rr.Cancel:
				cancelled <-true
				logger.Debug("send.request is cancelled")
			case <-cancelRoutine:
				return
			}

			// Repeat canceling request until this thread is stopped
			// because we have no idea about whether it succeeds.
			for {
				reqLock.Lock()
				c.httpClient.Transport.(*http.Transport).CancelRequest(req)
				reqLock.Unlock()

				select {
				case <-time.After(100 * time.Millisecond):
				case <-cancelRoutine:
					return
				}
			}
		}()
	}

	// if we connect to a follower, we will retry until we find a leader
	for attempt := 0; ; attempt++ {
		select {
		case <-cancelled:
			return nil, ErrRequestCancelled
		default:
		}

		logger.Debug("begin attempt", attempt, "for", rr.RelativePath)

		if rr.Method == "GET" && c.config.Consistency == WEAK_CONSISTENCY {
			// If it's a GET and consistency level is set to WEAK,
			// then use a random machine.
			httpPath = c.getHttpPath(true, rr.RelativePath)
		} else {
			// Else use the leader.
			httpPath = c.getHttpPath(false, rr.RelativePath)
		}

		// Return a cURL command if curlChan is set
		if c.cURLch != nil {
			command := fmt.Sprintf("curl -X %s %s", rr.Method, httpPath)
			for key, value := range rr.Values {
				command += fmt.Sprintf(" -d %s=%s", key, value[0])
			}
			c.sendCURL(command)
		}

		logger.Debug("send.request.to ", httpPath, " | method ", rr.Method)

		reqLock.Lock()
		if rr.Values == nil {
			req, err = http.NewRequest(rr.Method, httpPath, nil)
		} else {
			req, err = http.NewRequest(rr.Method, httpPath,
				strings.NewReader(rr.Values.Encode()))

			req.Header.Set("Content-Type",
				"application/x-www-form-urlencoded; param=value")
		}
		reqLock.Unlock()

		if err != nil {
			return nil, err
		}

		resp, err = c.httpClient.Do(req)
		// If the request was cancelled, return ErrRequestCancelled directly
		select {
		case <-cancelled:
			return nil, ErrRequestCancelled
		default:
		}

		reqs = append(reqs, *req)

		// network error, change a machine!
		if err != nil {
			logger.Debug("network error:", err.Error())
			resps = append(resps, http.Response{})
			if checkErr := checkRetry(c.cluster, reqs, resps, err); checkErr != nil {
				return nil, checkErr
			}

			c.cluster.switchLeader(attempt % len(c.cluster.Machines))
			continue
		}

		// if there is no error, it should receive response
		resps = append(resps, *resp)
		defer resp.Body.Close()
		logger.Debug("recv.response.from", httpPath)

		if validHttpStatusCode[resp.StatusCode] {
			// try to read byte code and break the loop
			respBody, err = ioutil.ReadAll(resp.Body)
			if err == nil {
				logger.Debug("recv.success.", httpPath)
				break
			}
		}

		// if resp is TemporaryRedirect, set the new leader and retry
		if resp.StatusCode == http.StatusTemporaryRedirect {
			u, err := resp.Location()

			if err != nil {
				logger.Warning(err)
			} else {
				// Update cluster leader based on redirect location
				// because it should point to the leader address
				c.cluster.updateLeaderFromURL(u)
				logger.Debug("recv.response.relocate", u.String())
			}
			continue
		}

		if checkErr := checkRetry(c.cluster, reqs, resps,
			errors.New("Unexpected HTTP status code")); checkErr != nil {
			return nil, checkErr
		}
	}

	r := &RawResponse{
		StatusCode: resp.StatusCode,
		Body:       respBody,
		Header:     resp.Header,
	}

	return r, nil
}

// DefaultCheckRetry checks retry cases
// If it has retried 2 * machine number, stop to retry it anymore
// If resp is nil, sleep for 200ms
// If status code is InternalServerError, sleep for 200ms.
func DefaultCheckRetry(cluster *Cluster, reqs []http.Request,
	resps []http.Response, err error) error {

	if len(reqs) >= 2*len(cluster.Machines) {
		return newError(ErrCodeEtcdNotReachable,
			"Tried to connect to each peer twice and failed", 0)
	}

	resp := &resps[len(resps)-1]

	if resp == nil {
		time.Sleep(time.Millisecond * 200)
		return nil
	}

	code := resp.StatusCode
	if code == http.StatusInternalServerError {
		time.Sleep(time.Millisecond * 200)

	}

	logger.Warning("bad response status code", code)
	return nil
}

func (c *Client) getHttpPath(random bool, s ...string) string {
	var machine string
	if random {
		machine = c.cluster.Machines[rand.Intn(len(c.cluster.Machines))]
	} else {
		machine = c.cluster.Leader
	}

	fullPath := machine + "/" + version
	for _, seg := range s {
		fullPath = fullPath + "/" + seg
	}

	return fullPath
}

// buildValues builds a url.Values map according to the given value and ttl
func buildValues(value string, ttl uint64) url.Values {
	v := url.Values{}

	if value != "" {
		v.Set("value", value)
	}

	if ttl > 0 {
		v.Set("ttl", fmt.Sprintf("%v", ttl))
	}

	return v
}

// convert key string to http path exclude version
// for example: key[foo] -> path[keys/foo]
// key[/] -> path[keys/]
func keyToPath(key string) string {
	p := path.Join("keys", key)

	// corner case: if key is "/" or "//" ect
	// path join will clear the tailing "/"
	// we need to add it back
	if p == "keys" {
		p = "keys/"
	}

	return p
}
<commit_msg>Bubble up errors from http.NewRequest before calling Header<commit_after>package etcd

import (
	"errors"
	"fmt"
	"io/ioutil"
	"math/rand"
	"net/http"
	"net/url"
	"path"
	"strings"
	"sync"
	"time"
)

// Errors introduced by handling requests
var (
	ErrRequestCancelled = errors.New("sending request is cancelled")
)

type RawRequest struct {
	Method       string
	RelativePath string
	Values       url.Values
	Cancel       <-chan bool
}

// NewRawRequest returns a new RawRequest
func NewRawRequest(method, relativePath string, values url.Values, cancel <-chan bool) *RawRequest {
	return &RawRequest{
		Method:       method,
		RelativePath: relativePath,
		Values:       values,
		Cancel:       cancel,
	}
}

// getCancelable issues a cancelable GET request
func (c *Client) getCancelable(key string, options Options,
	cancel <-chan bool) (*RawResponse, error) {
	logger.Debugf("get %s [%s]", key, c.cluster.Leader)
	p := keyToPath(key)

	// If consistency level is set to STRONG, append
	// the `consistent` query string.
	if c.config.Consistency == STRONG_CONSISTENCY {
		options["consistent"] = true
	}

	str, err := options.toParameters(VALID_GET_OPTIONS)
	if err != nil {
		return nil, err
	}
	p += str

	req := NewRawRequest("GET", p, nil, cancel)
	resp, err := c.SendRequest(req)

	if err != nil {
		return nil, err
	}

	return resp, nil
}

// get issues a GET request
func (c *Client) get(key string, options Options) (*RawResponse, error) {
	return c.getCancelable(key, options, nil)
}

// put issues a PUT request
func (c *Client) put(key string, value string, ttl uint64,
	options Options) (*RawResponse, error) {

	logger.Debugf("put %s, %s, ttl: %d, [%s]", key, value, ttl, c.cluster.Leader)
	p := keyToPath(key)

	str, err := options.toParameters(VALID_PUT_OPTIONS)
	if err != nil {
		return nil, err
	}
	p += str

	req := NewRawRequest("PUT", p, buildValues(value, ttl), nil)
	resp, err := c.SendRequest(req)

	if err != nil {
		return nil, err
	}

	return resp, nil
}

// post issues a POST request
func (c *Client) post(key string, value string, ttl uint64) (*RawResponse, error) {
	logger.Debugf("post %s, %s, ttl: %d, [%s]", key, value, ttl, c.cluster.Leader)
	p := keyToPath(key)

	req := NewRawRequest("POST", p, buildValues(value, ttl), nil)
	resp, err := c.SendRequest(req)

	if err != nil {
		return nil, err
	}

	return resp, nil
}

// delete issues a DELETE request
func (c *Client) delete(key string, options Options) (*RawResponse, error) {
	logger.Debugf("delete %s [%s]", key, c.cluster.Leader)
	p := keyToPath(key)

	str, err := options.toParameters(VALID_DELETE_OPTIONS)
	if err != nil {
		return nil, err
	}
	p += str

	req := NewRawRequest("DELETE", p, nil, nil)
	resp, err := c.SendRequest(req)

	if err != nil {
		return nil, err
	}

	return resp, nil
}

// SendRequest sends a HTTP request and returns a Response as defined by etcd
func (c *Client) SendRequest(rr *RawRequest) (*RawResponse, error) {

	var req *http.Request
	var resp *http.Response
	var httpPath string
	var err error
	var respBody []byte

	reqs := make([]http.Request, 0)
	resps := make([]http.Response, 0)

	checkRetry := c.CheckRetry
	if checkRetry == nil {
		checkRetry = DefaultCheckRetry
	}

	cancelled := make(chan bool, 1)
	reqLock := new(sync.Mutex)

	if rr.Cancel != nil {
		cancelRoutine := make(chan bool)
		defer close(cancelRoutine)

		go func() {
			select {
			case <-rr.Cancel:
				cancelled <- true
				logger.Debug("send.request is cancelled")
			case <-cancelRoutine:
				return
			}

			// Repeat canceling request until this thread is stopped
			// because we have no idea about whether it succeeds.
			for {
				reqLock.Lock()
				c.httpClient.Transport.(*http.Transport).CancelRequest(req)
				reqLock.Unlock()

				select {
				case <-time.After(100 * time.Millisecond):
				case <-cancelRoutine:
					return
				}
			}
		}()
	}

	// if we connect to a follower, we will retry until we find a leader
	for attempt := 0; ; attempt++ {
		select {
		case <-cancelled:
			return nil, ErrRequestCancelled
		default:
		}

		logger.Debug("begin attempt", attempt, "for", rr.RelativePath)

		if rr.Method == "GET" && c.config.Consistency == WEAK_CONSISTENCY {
			// If it's a GET and consistency level is set to WEAK,
			// then use a random machine.
			httpPath = c.getHttpPath(true, rr.RelativePath)
		} else {
			// Else use the leader.
			httpPath = c.getHttpPath(false, rr.RelativePath)
		}

		// Return a cURL command if curlChan is set
		if c.cURLch != nil {
			command := fmt.Sprintf("curl -X %s %s", rr.Method, httpPath)
			for key, value := range rr.Values {
				command += fmt.Sprintf(" -d %s=%s", key, value[0])
			}
			c.sendCURL(command)
		}

		logger.Debug("send.request.to ", httpPath, " | method ", rr.Method)

		reqLock.Lock()
		if rr.Values == nil {
			if req, err = http.NewRequest(rr.Method, httpPath, nil); err != nil {
				return nil, err
			}
		} else {
			body := strings.NewReader(rr.Values.Encode())
			if req, err = http.NewRequest(rr.Method, httpPath, body); err != nil {
				return nil, err
			}

			req.Header.Set("Content-Type",
				"application/x-www-form-urlencoded; param=value")
		}
		reqLock.Unlock()

		resp, err = c.httpClient.Do(req)
		// If the request was cancelled, return ErrRequestCancelled directly
		select {
		case <-cancelled:
			return nil, ErrRequestCancelled
		default:
		}

		reqs = append(reqs, *req)

		// network error, change a machine!
		if err != nil {
			logger.Debug("network error:", err.Error())
			resps = append(resps, http.Response{})
			if checkErr := checkRetry(c.cluster, reqs, resps, err); checkErr != nil {
				return nil, checkErr
			}

			c.cluster.switchLeader(attempt % len(c.cluster.Machines))
			continue
		}

		// if there is no error, it should receive response
		resps = append(resps, *resp)
		defer resp.Body.Close()
		logger.Debug("recv.response.from", httpPath)

		if validHttpStatusCode[resp.StatusCode] {
			// try to read byte code and break the loop
			respBody, err = ioutil.ReadAll(resp.Body)
			if err == nil {
				logger.Debug("recv.success.", httpPath)
				break
			}
		}

		// if resp is TemporaryRedirect, set the new leader and retry
		if resp.StatusCode == http.StatusTemporaryRedirect {
			u, err := resp.Location()

			if err != nil {
				logger.Warning(err)
			} else {
				// Update cluster leader based on redirect location
				// because it should point to the leader address
				c.cluster.updateLeaderFromURL(u)
				logger.Debug("recv.response.relocate", u.String())
			}
			continue
		}

		if checkErr := checkRetry(c.cluster, reqs, resps,
			errors.New("Unexpected HTTP status code")); checkErr != nil {
			return nil, checkErr
		}
	}

	r := &RawResponse{
		StatusCode: resp.StatusCode,
		Body:       respBody,
		Header:     resp.Header,
	}

	return r, nil
}

// DefaultCheckRetry checks retry cases
// If it has retried 2 * machine number, stop to retry it anymore
// If resp is nil, sleep for 200ms
// If status code is InternalServerError, sleep for 200ms.
func DefaultCheckRetry(cluster *Cluster, reqs []http.Request,
	resps []http.Response, err error) error {

	if len(reqs) >= 2*len(cluster.Machines) {
		return newError(ErrCodeEtcdNotReachable,
			"Tried to connect to each peer twice and failed", 0)
	}

	resp := &resps[len(resps)-1]

	if resp == nil {
		time.Sleep(time.Millisecond * 200)
		return nil
	}

	code := resp.StatusCode
	if code == http.StatusInternalServerError {
		time.Sleep(time.Millisecond * 200)

	}

	logger.Warning("bad response status code", code)
	return nil
}

func (c *Client) getHttpPath(random bool, s ...string) string {
	var machine string
	if random {
		machine = c.cluster.Machines[rand.Intn(len(c.cluster.Machines))]
	} else {
		machine = c.cluster.Leader
	}

	fullPath := machine + "/" + version
	for _, seg := range s {
		fullPath = fullPath + "/" + seg
	}

	return fullPath
}

// buildValues builds a url.Values map according to the given value and ttl
func buildValues(value string, ttl uint64) url.Values {
	v := url.Values{}

	if value != "" {
		v.Set("value", value)
	}

	if ttl > 0 {
		v.Set("ttl", fmt.Sprintf("%v", ttl))
	}

	return v
}

// convert key string to http path exclude version
// for example: key[foo] -> path[keys/foo]
// key[/] -> path[keys/]
func keyToPath(key string) string {
	p := path.Join("keys", key)

	// corner case: if key is "/" or "//" ect
	// path join will clear the tailing "/"
	// we need to add it back
	if p == "keys" {
		p = "keys/"
	}

	return p
}
<|endoftext|>