<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package aliup

import (
	"errors"
	"fmt"
	"os"
	"time"

	"k8s.io/klog"

	"github.com/denverdino/aliyungo/common"
	"github.com/denverdino/aliyungo/ecs"
	"github.com/denverdino/aliyungo/ess"
	"github.com/denverdino/aliyungo/ram"
	"github.com/denverdino/aliyungo/slb"

	prj "k8s.io/kops"
	"k8s.io/kops/dnsprovider/pkg/dnsprovider"
	"k8s.io/kops/pkg/apis/kops"
	"k8s.io/kops/pkg/cloudinstances"
	"k8s.io/kops/upup/pkg/fi"
)

const (
	TagClusterName           = "KubernetesCluster"
	TagNameRolePrefix        = "k8s.io/role/"
	TagNameEtcdClusterPrefix = "k8s.io/etcd/"
	TagRoleMaster            = "master"
)

// This is for statistic purpose.
var KubernetesKopsIdentity = fmt.Sprintf("Kubernetes.Kops/%s", prj.Version)

type ALICloud interface {
	fi.Cloud

	EcsClient() *ecs.Client
	SlbClient() *slb.Client
	RamClient() *ram.RamClient
	EssClient() *ess.Client
	VpcClient() *ecs.Client

	AddClusterTags(tags map[string]string)
	GetTags(resourceId string, resourceType string) (map[string]string, error)
	CreateTags(resourceId string, resourceType string, tags map[string]string) error
	RemoveTags(resourceId string, resourceType string, tags map[string]string) error
	GetClusterTags() map[string]string
	FindClusterStatus(cluster *kops.Cluster) (*kops.ClusterStatus, error)
	GetApiIngressStatus(cluster *kops.Cluster) ([]kops.ApiIngressStatus, error)
}

type aliCloudImplementation struct {
	ecsClient *ecs.Client
	slbClient *slb.Client
	ramClient *ram.RamClient
	essClient *ess.Client
	vpcClient *ecs.Client

	region string
	tags   map[string]string
}

var _ fi.Cloud = &aliCloudImplementation{}

// NewALICloud returns a Cloud, expecting the env vars ALIYUN_ACCESS_KEY_ID && ALIYUN_ACCESS_KEY_SECRET
// NewALICloud will return an err if env vars are not defined
func NewALICloud(region string, tags map[string]string) (ALICloud, error) {

	c := &aliCloudImplementation{region: region}

	accessKeyID := os.Getenv("ALIYUN_ACCESS_KEY_ID")
	if accessKeyID == "" {
		return nil, errors.New("ALIYUN_ACCESS_KEY_ID is required")
	}
	accessKeySecret := os.Getenv("ALIYUN_ACCESS_KEY_SECRET")
	if accessKeySecret == "" {
		return nil, errors.New("ALIYUN_ACCESS_KEY_SECRET is required")
	}

	c.ecsClient = ecs.NewClient(accessKeyID, accessKeySecret)
	c.ecsClient.SetUserAgent(KubernetesKopsIdentity)
	c.slbClient = slb.NewClient(accessKeyID, accessKeySecret)
	ramclient := ram.NewClient(accessKeyID, accessKeySecret)
	c.ramClient = ramclient.(*ram.RamClient)
	c.essClient = ess.NewClient(accessKeyID, accessKeySecret)
	c.vpcClient = ecs.NewVPCClient(accessKeyID, accessKeySecret, common.Region(region))

	c.tags = tags

	return c, nil
}

func (c *aliCloudImplementation) EcsClient() *ecs.Client {
	return c.ecsClient
}

func (c *aliCloudImplementation) SlbClient() *slb.Client {
	return c.slbClient
}

func (c *aliCloudImplementation) RamClient() *ram.RamClient {
	return c.ramClient
}

func (c *aliCloudImplementation) EssClient() *ess.Client {
	return c.essClient
}

func (c *aliCloudImplementation) VpcClient() *ecs.Client {
	return c.vpcClient
}

func (c *aliCloudImplementation) Region() string {
	return c.region
}

func (c *aliCloudImplementation) ProviderID() kops.CloudProviderID {
	return kops.CloudProviderALI
}

func (c *aliCloudImplementation) DNS() (dnsprovider.Interface, error) {
	return nil, errors.New("DNS not implemented on aliCloud")
}

func (c *aliCloudImplementation) DeleteGroup(g *cloudinstances.CloudInstanceGroup) error {
	return errors.New("DeleteGroup not implemented on aliCloud")
}

func (c *aliCloudImplementation) DeleteInstance(i *cloudinstances.CloudInstanceGroupMember) error {
	id := i.ID
	if id == "" {
		return fmt.Errorf("id was not set on CloudInstanceGroupMember: %v", i)
	}

	if err := c.EcsClient().StopInstance(id, false); err != nil {
		return fmt.Errorf("error stopping instance %q: %v", id, err)
	}

	// Wait for 3 min to stop the instance
	for i := 0; i < 36; i++ {
		ins, err := c.EcsClient().DescribeInstanceAttribute(id)
		if err != nil {
			return fmt.Errorf("error describing instance %q: %v", id, err)
		}

		klog.V(8).Infof("stopping Alicloud ecs instance %q, current Status: %q", id, ins.Status)
		time.Sleep(time.Second * 5)

		if ins.Status == ecs.Stopped {
			break
		}

		if i == 35 {
			return fmt.Errorf("fail to stop ecs instance %s in 3 mins", id)
		}
	}

	if err := c.EcsClient().DeleteInstance(id); err != nil {
		return fmt.Errorf("error deleting instance %q: %v", id, err)
	}

	klog.V(8).Infof("deleted Alicloud ecs instance %q", id)

	return nil
}

func (c *aliCloudImplementation) DetachInstance(i *cloudinstances.CloudInstanceGroupMember) error {
	return errors.New("aliCloud cloud provider does not support surging")
}

func (c *aliCloudImplementation) FindVPCInfo(id string) (*fi.VPCInfo, error) {
	request := &ecs.DescribeVpcsArgs{
		RegionId: common.Region(c.Region()),
		VpcId:    id,
	}
	vpcs, _, err := c.EcsClient().DescribeVpcs(request)
	if err != nil {
		return nil, fmt.Errorf("error listing VPCs: %v", err)
	}

	if len(vpcs) != 1 {
		return nil, fmt.Errorf("found multiple VPCs for %q", id)
	}
	vpcInfo := &fi.VPCInfo{
		CIDR: vpcs[0].CidrBlock,
	}

	describeVSwitchesArgs := &ecs.DescribeVSwitchesArgs{
		VpcId:    id,
		RegionId: common.Region(c.Region()),
	}
	vswitchList, _, err := c.EcsClient().DescribeVSwitches(describeVSwitchesArgs)
	if err != nil {
		return nil, fmt.Errorf("error listing VSwitchs: %v", err)
	}

	for _, vswitch := range vswitchList {
		s := &fi.SubnetInfo{
			ID:   vswitch.VSwitchId,
			Zone: vswitch.ZoneId,
			CIDR: vswitch.CidrBlock,
		}
		vpcInfo.Subnets = append(vpcInfo.Subnets, s)
	}

	return vpcInfo, nil

}

// GetTags will get the specified resource's tags.
func (c *aliCloudImplementation) GetTags(resourceId string, resourceType string) (map[string]string, error) {
	if resourceId == "" {
		return nil, errors.New("resourceId not provided to GetTags")
	}
	tags := map[string]string{}

	request := &ecs.DescribeTagsArgs{
		RegionId:     common.Region(c.Region()),
		ResourceType: ecs.TagResourceType(resourceType), //image, instance, snapshot or disk
		ResourceId:   resourceId,
	}
	responseTags, _, err := c.EcsClient().DescribeTags(request)
	if err != nil {
		return tags, fmt.Errorf("error getting tags on %v: %v", resourceId, err)
	}

	for _, tag := range responseTags {
		tags[tag.TagKey] = tag.TagValue
	}
	return tags, nil

}

// AddClusterTags will add ClusterTags to resources (in ALI, only disk, instance, snapshot or image can be tagged )
func (c *aliCloudImplementation) AddClusterTags(tags map[string]string) {

	if c.tags != nil && len(c.tags) != 0 && tags != nil {
		for k, v := range c.tags {
			tags[k] = v
		}
	}
}

// CreateTags will add tags to the specified resource.
func (c *aliCloudImplementation) CreateTags(resourceId string, resourceType string, tags map[string]string) error {
	if len(tags) == 0 {
		return nil
	} else if len(tags) > 10 {
		klog.V(4).Infof("The number of specified resource's tags exceeds 10, resourceId:%q", resourceId)
	}
	if resourceId == "" {
		return errors.New("resourceId not provided to CreateTags")
	}
	if resourceType == "" {
		return errors.New("resourceType not provided to CreateTags")
	}

	request := &ecs.AddTagsArgs{
		ResourceId:   resourceId,
		ResourceType: ecs.TagResourceType(resourceType), //image, instance, snapshot or disk
		RegionId:     common.Region(c.Region()),
		Tag:          tags,
	}
	err := c.EcsClient().AddTags(request)
	if err != nil {
		return fmt.Errorf("error creating tags on %v: %v", resourceId, err)
	}

	return nil
}

// RemoveTags will remove tags from the specified resource.
func (c *aliCloudImplementation) RemoveTags(resourceId string, resourceType string, tags map[string]string) error {
	if len(tags) == 0 {
		return nil
	}
	if resourceId == "" {
		return errors.New("resourceId not provided to RemoveTags")
	}
	if resourceType == "" {
		return errors.New("resourceType not provided to RemoveTags")
	}

	request := &ecs.RemoveTagsArgs{
		ResourceId:   resourceId,
		ResourceType: ecs.TagResourceType(resourceType), //image, instance, snapshot or disk
		RegionId:     common.Region(c.Region()),
		Tag:          tags,
	}
	err := c.EcsClient().RemoveTags(request)
	if err != nil {
		return fmt.Errorf("error removing tags on %v: %v", resourceId, err)
	}

	return nil
}

// GetClusterTags will get the ClusterTags
func (c *aliCloudImplementation) GetClusterTags() map[string]string {
	return c.tags
}

func (c *aliCloudImplementation) GetApiIngressStatus(cluster *kops.Cluster) ([]kops.ApiIngressStatus, error) {
	var ingresses []kops.ApiIngressStatus
	name := "api." + cluster.Name

	describeLoadBalancersArgs := &slb.DescribeLoadBalancersArgs{
		RegionId:         common.Region(c.Region()),
		LoadBalancerName: name,
	}

	responseLoadBalancers, err := c.SlbClient().DescribeLoadBalancers(describeLoadBalancersArgs)
	if err != nil {
		return nil, fmt.Errorf("error finding LoadBalancers: %v", err)
	}
	// Don't exist loadbalancer with specified ClusterTags or Name.
	if len(responseLoadBalancers) == 0 {
		return nil, nil
	}
	if len(responseLoadBalancers) > 1 {
		klog.V(4).Infof("The number of specified loadbalancer with the same name exceeds 1, loadbalancerName:%q", name)
	}

	address := responseLoadBalancers[0].Address
	ingresses = append(ingresses, kops.ApiIngressStatus{IP: address})

	return ingresses, nil
}

func ZoneToVSwitchID(VPCID string, zones []string, vswitchIDs []string) (map[string]string, error) {
	regionId, err := getRegionByZones(zones)
	if err != nil {
		return nil, err
	}

	res := make(map[string]string)
	cloudTags := map[string]string{}
	aliCloud, err := NewALICloud(regionId, cloudTags)
	if err != nil {
		return res, fmt.Errorf("error loading cloud: %v", err)
	}

	describeVpcsArgs := &ecs.DescribeVpcsArgs{
		RegionId: common.Region(regionId),
		VpcId:    VPCID,
	}

	vpc, _, err := aliCloud.EcsClient().DescribeVpcs(describeVpcsArgs)
	if err != nil {
		return res, fmt.Errorf("error describing VPC: %v", err)
	}

	if len(vpc) == 0 {
		return res, fmt.Errorf("VPC %q not found", VPCID)
	}

	if len(vpc) != 1 {
		return nil, fmt.Errorf("found multiple VPCs for %q", VPCID)
	}
	subnetByID := make(map[string]string)
	for _, VSId := range vpc[0].VSwitchIds.VSwitchId {
		subnetByID[VSId] = VSId
	}

	for _, VSwitchId := range vswitchIDs {

		_, ok := subnetByID[VSwitchId]
		if !ok {
			return res, fmt.Errorf("vswitch %s not found in VPC %s", VSwitchId, VPCID)
		}
		describeVSwitchesArgs := &ecs.DescribeVSwitchesArgs{
			VpcId:     vpc[0].VpcId,
			RegionId:  common.Region(regionId),
			VSwitchId: VSwitchId,
		}

		vswitchList, _, err := aliCloud.EcsClient().DescribeVSwitches(describeVSwitchesArgs)
		if err != nil {
			return nil, fmt.Errorf("error listing VSwitchs: %v", err)
		}

		if len(vswitchList) == 0 {
			return nil, fmt.Errorf("VSwitch %q not found", VSwitchId)
		}

		if len(vswitchList) != 1 {
			return nil, fmt.Errorf("found multiple VSwitchs for %q", VSwitchId)
		}

		zone := vswitchList[0].ZoneId
		if res[zone] != "" {
			return res, fmt.Errorf("vswitch %s and %s have the same zone", vswitchList[0].VSwitchId, zone)
		}
		res[zone] = vswitchList[0].VSwitchId

	}
	return res, nil
}
<commit_msg>Add new SLB() for ALICloud with official SDK<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package aliup

import (
	"errors"
	"fmt"
	"os"
	"time"

	"k8s.io/klog"

	alicloud "github.com/aliyun/alibaba-cloud-sdk-go/sdk"
	"github.com/aliyun/alibaba-cloud-sdk-go/sdk/auth/credentials"
	slbnew "github.com/aliyun/alibaba-cloud-sdk-go/services/slb"
	"github.com/denverdino/aliyungo/common"
	"github.com/denverdino/aliyungo/ecs"
	"github.com/denverdino/aliyungo/ess"
	"github.com/denverdino/aliyungo/ram"
	"github.com/denverdino/aliyungo/slb"

	prj "k8s.io/kops"
	"k8s.io/kops/dnsprovider/pkg/dnsprovider"
	"k8s.io/kops/pkg/apis/kops"
	"k8s.io/kops/pkg/cloudinstances"
	"k8s.io/kops/upup/pkg/fi"
)

const (
	TagClusterName           = "KubernetesCluster"
	TagNameRolePrefix        = "k8s.io/role/"
	TagNameEtcdClusterPrefix = "k8s.io/etcd/"
	TagRoleMaster            = "master"
)

// This is for statistic purpose.
var KubernetesKopsIdentity = fmt.Sprintf("Kubernetes.Kops/%s", prj.Version)

type ALICloud interface {
	fi.Cloud

	// Clients that use official Alicloud go sdk
	SLB() *slbnew.Client

	// Since package github.com/denverdino/aliyungo is not actively maintained,
	// clients that use github.com/denverdino/aliyungo, will be deprecated after
	// the above official SDK clients covers all of them
	EcsClient() *ecs.Client
	SlbClient() *slb.Client
	RamClient() *ram.RamClient
	EssClient() *ess.Client
	VpcClient() *ecs.Client

	AddClusterTags(tags map[string]string)
	GetTags(resourceId string, resourceType string) (map[string]string, error)
	CreateTags(resourceId string, resourceType string, tags map[string]string) error
	RemoveTags(resourceId string, resourceType string, tags map[string]string) error
	GetClusterTags() map[string]string
	FindClusterStatus(cluster *kops.Cluster) (*kops.ClusterStatus, error)
	GetApiIngressStatus(cluster *kops.Cluster) ([]kops.ApiIngressStatus, error)
}

type aliCloudImplementation struct {
	slb *slbnew.Client

	ecsClient *ecs.Client
	slbClient *slb.Client
	ramClient *ram.RamClient
	essClient *ess.Client
	vpcClient *ecs.Client

	region string
	tags   map[string]string
}

var _ fi.Cloud = &aliCloudImplementation{}

// NewALICloud returns a Cloud, expecting the env vars ALIYUN_ACCESS_KEY_ID && ALIYUN_ACCESS_KEY_SECRET
// NewALICloud will return an err if env vars are not defined
func NewALICloud(region string, tags map[string]string) (ALICloud, error) {

	c := &aliCloudImplementation{region: region}

	accessKeyID := os.Getenv("ALIYUN_ACCESS_KEY_ID")
	if accessKeyID == "" {
		return nil, errors.New("ALIYUN_ACCESS_KEY_ID is required")
	}
	accessKeySecret := os.Getenv("ALIYUN_ACCESS_KEY_SECRET")
	if accessKeySecret == "" {
		return nil, errors.New("ALIYUN_ACCESS_KEY_SECRET is required")
	}

	c.ecsClient = ecs.NewClient(accessKeyID, accessKeySecret)
	c.ecsClient.SetUserAgent(KubernetesKopsIdentity)
	c.slbClient = slb.NewClient(accessKeyID, accessKeySecret)
	ramclient := ram.NewClient(accessKeyID, accessKeySecret)
	c.ramClient = ramclient.(*ram.RamClient)
	c.essClient = ess.NewClient(accessKeyID, accessKeySecret)
	c.vpcClient = ecs.NewVPCClient(accessKeyID, accessKeySecret, common.Region(region))
	c.tags = tags

	// With Alicloud official go SDK
	var err error

	config := alicloud.NewConfig()
	credential := credentials.NewAccessKeyCredential(accessKeyID, accessKeySecret)

	c.slb, err = slbnew.NewClientWithOptions(region, config, credential)
	if err != nil {
		return nil, fmt.Errorf("error creating slb sdk client: %v", err)
	}

	return c, nil
}

func (c *aliCloudImplementation) SLB() *slbnew.Client {
	return c.slb
}

// Clients as below are with github.com/denverdino/aliyungo
func (c *aliCloudImplementation) EcsClient() *ecs.Client {
	return c.ecsClient
}

func (c *aliCloudImplementation) SlbClient() *slb.Client {
	return c.slbClient
}

func (c *aliCloudImplementation) RamClient() *ram.RamClient {
	return c.ramClient
}

func (c *aliCloudImplementation) EssClient() *ess.Client {
	return c.essClient
}

func (c *aliCloudImplementation) VpcClient() *ecs.Client {
	return c.vpcClient
}

func (c *aliCloudImplementation) Region() string {
	return c.region
}

func (c *aliCloudImplementation) ProviderID() kops.CloudProviderID {
	return kops.CloudProviderALI
}

func (c *aliCloudImplementation) DNS() (dnsprovider.Interface, error) {
	return nil, errors.New("DNS not implemented on aliCloud")
}

func (c *aliCloudImplementation) DeleteGroup(g *cloudinstances.CloudInstanceGroup) error {
	return errors.New("DeleteGroup not implemented on aliCloud")
}

func (c *aliCloudImplementation) DeleteInstance(i *cloudinstances.CloudInstanceGroupMember) error {
	id := i.ID
	if id == "" {
		return fmt.Errorf("id was not set on CloudInstanceGroupMember: %v", i)
	}

	if err := c.EcsClient().StopInstance(id, false); err != nil {
		return fmt.Errorf("error stopping instance %q: %v", id, err)
	}

	// Wait for 3 min to stop the instance
	for i := 0; i < 36; i++ {
		ins, err := c.EcsClient().DescribeInstanceAttribute(id)
		if err != nil {
			return fmt.Errorf("error describing instance %q: %v", id, err)
		}

		klog.V(8).Infof("stopping Alicloud ecs instance %q, current Status: %q", id, ins.Status)
		time.Sleep(time.Second * 5)

		if ins.Status == ecs.Stopped {
			break
		}

		if i == 35 {
			return fmt.Errorf("fail to stop ecs instance %s in 3 mins", id)
		}
	}

	if err := c.EcsClient().DeleteInstance(id); err != nil {
		return fmt.Errorf("error deleting instance %q: %v", id, err)
	}

	klog.V(8).Infof("deleted Alicloud ecs instance %q", id)

	return nil
}

func (c *aliCloudImplementation) DetachInstance(i *cloudinstances.CloudInstanceGroupMember) error {
	return errors.New("aliCloud cloud provider does not support surging")
}

func (c *aliCloudImplementation) FindVPCInfo(id string) (*fi.VPCInfo, error) {
	request := &ecs.DescribeVpcsArgs{
		RegionId: common.Region(c.Region()),
		VpcId:    id,
	}
	vpcs, _, err := c.EcsClient().DescribeVpcs(request)
	if err != nil {
		return nil, fmt.Errorf("error listing VPCs: %v", err)
	}

	if len(vpcs) != 1 {
		return nil, fmt.Errorf("found multiple VPCs for %q", id)
	}
	vpcInfo := &fi.VPCInfo{
		CIDR: vpcs[0].CidrBlock,
	}

	describeVSwitchesArgs := &ecs.DescribeVSwitchesArgs{
		VpcId:    id,
		RegionId: common.Region(c.Region()),
	}
	vswitchList, _, err := c.EcsClient().DescribeVSwitches(describeVSwitchesArgs)
	if err != nil {
		return nil, fmt.Errorf("error listing VSwitchs: %v", err)
	}

	for _, vswitch := range vswitchList {
		s := &fi.SubnetInfo{
			ID:   vswitch.VSwitchId,
			Zone: vswitch.ZoneId,
			CIDR: vswitch.CidrBlock,
		}
		vpcInfo.Subnets = append(vpcInfo.Subnets, s)
	}

	return vpcInfo, nil

}

// GetTags will get the specified resource's tags.
func (c *aliCloudImplementation) GetTags(resourceId string, resourceType string) (map[string]string, error) {
	if resourceId == "" {
		return nil, errors.New("resourceId not provided to GetTags")
	}
	tags := map[string]string{}

	request := &ecs.DescribeTagsArgs{
		RegionId:     common.Region(c.Region()),
		ResourceType: ecs.TagResourceType(resourceType), //image, instance, snapshot or disk
		ResourceId:   resourceId,
	}
	responseTags, _, err := c.EcsClient().DescribeTags(request)
	if err != nil {
		return tags, fmt.Errorf("error getting tags on %v: %v", resourceId, err)
	}

	for _, tag := range responseTags {
		tags[tag.TagKey] = tag.TagValue
	}
	return tags, nil

}

// AddClusterTags will add ClusterTags to resources (in ALI, only disk, instance, snapshot or image can be tagged )
func (c *aliCloudImplementation) AddClusterTags(tags map[string]string) {

	if c.tags != nil && len(c.tags) != 0 && tags != nil {
		for k, v := range c.tags {
			tags[k] = v
		}
	}
}

// CreateTags will add tags to the specified resource.
func (c *aliCloudImplementation) CreateTags(resourceId string, resourceType string, tags map[string]string) error {
	if len(tags) == 0 {
		return nil
	} else if len(tags) > 10 {
		klog.V(4).Infof("The number of specified resource's tags exceeds 10, resourceId:%q", resourceId)
	}
	if resourceId == "" {
		return errors.New("resourceId not provided to CreateTags")
	}
	if resourceType == "" {
		return errors.New("resourceType not provided to CreateTags")
	}

	request := &ecs.AddTagsArgs{
		ResourceId:   resourceId,
		ResourceType: ecs.TagResourceType(resourceType), //image, instance, snapshot or disk
		RegionId:     common.Region(c.Region()),
		Tag:          tags,
	}
	err := c.EcsClient().AddTags(request)
	if err != nil {
		return fmt.Errorf("error creating tags on %v: %v", resourceId, err)
	}

	return nil
}

// RemoveTags will remove tags from the specified resource.
func (c *aliCloudImplementation) RemoveTags(resourceId string, resourceType string, tags map[string]string) error {
	if len(tags) == 0 {
		return nil
	}
	if resourceId == "" {
		return errors.New("resourceId not provided to RemoveTags")
	}
	if resourceType == "" {
		return errors.New("resourceType not provided to RemoveTags")
	}

	request := &ecs.RemoveTagsArgs{
		ResourceId:   resourceId,
		ResourceType: ecs.TagResourceType(resourceType), //image, instance, snapshot or disk
		RegionId:     common.Region(c.Region()),
		Tag:          tags,
	}
	err := c.EcsClient().RemoveTags(request)
	if err != nil {
		return fmt.Errorf("error removing tags on %v: %v", resourceId, err)
	}

	return nil
}

// GetClusterTags will get the ClusterTags
func (c *aliCloudImplementation) GetClusterTags() map[string]string {
	return c.tags
}

func (c *aliCloudImplementation) GetApiIngressStatus(cluster *kops.Cluster) ([]kops.ApiIngressStatus, error) {
	var ingresses []kops.ApiIngressStatus
	name := "api." + cluster.Name

	describeLoadBalancersArgs := &slb.DescribeLoadBalancersArgs{
		RegionId:         common.Region(c.Region()),
		LoadBalancerName: name,
	}

	responseLoadBalancers, err := c.SlbClient().DescribeLoadBalancers(describeLoadBalancersArgs)
	if err != nil {
		return nil, fmt.Errorf("error finding LoadBalancers: %v", err)
	}
	// Don't exist loadbalancer with specified ClusterTags or Name.
	if len(responseLoadBalancers) == 0 {
		return nil, nil
	}
	if len(responseLoadBalancers) > 1 {
		klog.V(4).Infof("The number of specified loadbalancer with the same name exceeds 1, loadbalancerName:%q", name)
	}

	address := responseLoadBalancers[0].Address
	ingresses = append(ingresses, kops.ApiIngressStatus{IP: address})

	return ingresses, nil
}

func ZoneToVSwitchID(VPCID string, zones []string, vswitchIDs []string) (map[string]string, error) {
	regionId, err := getRegionByZones(zones)
	if err != nil {
		return nil, err
	}

	res := make(map[string]string)
	cloudTags := map[string]string{}
	aliCloud, err := NewALICloud(regionId, cloudTags)
	if err != nil {
		return res, fmt.Errorf("error loading cloud: %v", err)
	}

	describeVpcsArgs := &ecs.DescribeVpcsArgs{
		RegionId: common.Region(regionId),
		VpcId:    VPCID,
	}

	vpc, _, err := aliCloud.EcsClient().DescribeVpcs(describeVpcsArgs)
	if err != nil {
		return res, fmt.Errorf("error describing VPC: %v", err)
	}

	if len(vpc) == 0 {
		return res, fmt.Errorf("VPC %q not found", VPCID)
	}

	if len(vpc) != 1 {
		return nil, fmt.Errorf("found multiple VPCs for %q", VPCID)
	}
	subnetByID := make(map[string]string)
	for _, VSId := range vpc[0].VSwitchIds.VSwitchId {
		subnetByID[VSId] = VSId
	}

	for _, VSwitchId := range vswitchIDs {

		_, ok := subnetByID[VSwitchId]
		if !ok {
			return res, fmt.Errorf("vswitch %s not found in VPC %s", VSwitchId, VPCID)
		}
		describeVSwitchesArgs := &ecs.DescribeVSwitchesArgs{
			VpcId:     vpc[0].VpcId,
			RegionId:  common.Region(regionId),
			VSwitchId: VSwitchId,
		}

		vswitchList, _, err := aliCloud.EcsClient().DescribeVSwitches(describeVSwitchesArgs)
		if err != nil {
			return nil, fmt.Errorf("error listing VSwitchs: %v", err)
		}

		if len(vswitchList) == 0 {
			return nil, fmt.Errorf("VSwitch %q not found", VSwitchId)
		}

		if len(vswitchList) != 1 {
			return nil, fmt.Errorf("found multiple VSwitchs for %q", VSwitchId)
		}

		zone := vswitchList[0].ZoneId
		if res[zone] != "" {
			return res, fmt.Errorf("vswitch %s and %s have the same zone", vswitchList[0].VSwitchId, zone)
		}
		res[zone] = vswitchList[0].VSwitchId

	}
	return res, nil
}
<|endoftext|>


<commit_before>package imagick

import (
	"github.com/stretchr/testify/assert"
	"testing"
)

func TestGetVersion(t *testing.T) {
	sversion, nversion := GetVersion()
	assert.Regexp(t, "ImageMagick", sversion)
	assert.True(t, nversion > 0)
}

func TestResourceLimit(t *testing.T) {
	area := GetResourceLimit(RESOURCE_AREA)/2
	assert.True(t, SetResourceLimit(RESOURCE_AREA, area))
	assert.Equal(t, area, GetResourceLimit(RESOURCE_AREA))
}<commit_msg>Assert first that area is non zero<commit_after>package imagick

import (
	"github.com/stretchr/testify/assert"
	"testing"
)

func TestGetVersion(t *testing.T) {
	sversion, nversion := GetVersion()
	assert.Regexp(t, "ImageMagick", sversion)
	assert.True(t, nversion > 0)
}

func TestResourceLimit(t *testing.T) {
	area := GetResourceLimit(RESOURCE_AREA)
	assert.True(t, area > 0)
	assert.True(t, SetResourceLimit(RESOURCE_AREA, area/2))
	assert.Equal(t, area/2, GetResourceLimit(RESOURCE_AREA))
}<|endoftext|>


<commit_before>// Tideland Go Cells
//
// Copyright (C) 2010-2017 Frank Mueller / Tideland / Oldenburg / Germany
//
// All rights reserved. Use of this source code is governed
// by the new BSD license.

// Tideland Cells is a framework for event and behavior based applications.
//
// Cell behaviors are defined based on an interface and can be added
// to an envrionment. Here they are running as concurrent cells that
// can be networked and communicate via events. Several useful behaviors
// are provided with the behaviors package.
//
// New environments are created with
//
//     env := cells.NewEnvironment(identifier)
//
// and cells are added with
//
//    env.StartCell("foo", NewFooBehavior())
//
// Cells then can be subscribed with
//
//    env.Subscribe("foo", "bar")
//
// so that events emitted by the "foo" cell during the processing of
// events will be received by the "bar" cell. Each cell can have
// multiple cells subscibed.
//
// Events from the outside are emitted using
//
//     env.Emit("foo", myEvent)
//
// or
//
//    env.EmitNew("foo", "myTopic", cells.PayloadValues{
//        "KeyA": 12345,
//        "KeyB": true,
//    }, ctx)
//
// Behaviors have to implement the cells.Behavior interface. Here
// the Init() method is called with a cells.Context. This can be
// used inside the ProcessEvent() method to emit events to subscribers
// or directly to other cells of the environment.
//
// Sometimes it's needed to directly communicate with a cell to retrieve
// information. In this case the method
//
//     response, err := env.RequestContext("foo", "myRequest?", myPayload, myTimeout)
//
// is to be used. Inside the ProcessEvent() of the addressed cell the
// event can be used to send the response with
//
//    switch event.Topic() {
//    case "myRequest?":
//        event.Respond(someIncredibleData)
//    case ...:
//        ...
//    }
//
// Instructions without a response are simply done by emitting an event.
package cells

//--------------------
// IMPORTS
//--------------------

import (
	"github.com/tideland/golib/version"
)

//--------------------
// VERSION
//--------------------

// Version returns the version of the cells package.
func Version() version.Version {
	return version.New(6, 0, 0, "beta", "2017-10-23")
}

// EOF
<commit_msg>Made doc comment standard conforming<commit_after>// Tideland Go Cells
//
// Copyright (C) 2010-2017 Frank Mueller / Tideland / Oldenburg / Germany
//
// All rights reserved. Use of this source code is governed
// by the new BSD license.

// Package cells is the main package of the Tideland Cells. They provide
// a framework for event and behavior based applications.
//
// Cell behaviors are defined based on an interface and can be added
// to an envrionment. Here they are running as concurrent cells that
// can be networked and communicate via events. Several useful behaviors
// are provided with the behaviors package.
//
// New environments are created with
//
//     env := cells.NewEnvironment(identifier)
//
// and cells are added with
//
//    env.StartCell("foo", NewFooBehavior())
//
// Cells then can be subscribed with
//
//    env.Subscribe("foo", "bar")
//
// so that events emitted by the "foo" cell during the processing of
// events will be received by the "bar" cell. Each cell can have
// multiple cells subscibed.
//
// Events from the outside are emitted using
//
//     env.Emit("foo", myEvent)
//
// or
//
//    env.EmitNew("foo", "myTopic", cells.PayloadValues{
//        "KeyA": 12345,
//        "KeyB": true,
//    }, ctx)
//
// Behaviors have to implement the cells.Behavior interface. Here
// the Init() method is called with a cells.Context. This can be
// used inside the ProcessEvent() method to emit events to subscribers
// or directly to other cells of the environment.
//
// Sometimes it's needed to directly communicate with a cell to retrieve
// information. In this case the method
//
//     response, err := env.RequestContext("foo", "myRequest?", myPayload, myTimeout)
//
// is to be used. Inside the ProcessEvent() of the addressed cell the
// event can be used to send the response with
//
//    switch event.Topic() {
//    case "myRequest?":
//        event.Respond(someIncredibleData)
//    case ...:
//        ...
//    }
//
// Instructions without a response are simply done by emitting an event.
package cells

//--------------------
// IMPORTS
//--------------------

import (
	"github.com/tideland/golib/version"
)

//--------------------
// VERSION
//--------------------

// Version returns the version of the cells package.
func Version() version.Version {
	return version.New(6, 0, 0, "beta", "2017-10-23")
}

// EOF
<|endoftext|>


<commit_before>package main

import (
	"../nsq"
	"../util"
	"../util/pqueue"
	"bytes"
	"errors"
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	"os"
	"runtime/pprof"
	"strings"
	"time"
)

import _ "net/http/pprof"

func HttpServer(listener net.Listener) {
	log.Printf("HTTP: listening on %s", listener.Addr().String())
	handler := http.NewServeMux()
	handler.HandleFunc("/ping", pingHandler)
	handler.HandleFunc("/put", putHandler)
	handler.HandleFunc("/mput", mputHandler)
	handler.HandleFunc("/stats", statsHandler)
	handler.HandleFunc("/empty", emptyHandler)
	handler.HandleFunc("/mem_profile", memProfileHandler)
	handler.HandleFunc("/dump_inflight", dumpInFlightHandler)
	server := &http.Server{Handler: handler}
	err := server.Serve(listener)
	// theres no direct way to detect this error because it is not exposed
	if err != nil && !strings.Contains(err.Error(), "use of closed network connection") {
		log.Printf("ERROR: http.Serve() - %s", err.Error())
	}
}

func dumpInFlightHandler(w http.ResponseWriter, req *http.Request) {
	reqParams, err := util.NewReqParams(req)
	if err != nil {
		log.Printf("ERROR: failed to parse request params - %s", err.Error())
		w.Write(util.ApiResponse(500, "INVALID_REQUEST", nil))
		return
	}

	topicName, channelName, err := getTopicChannelArgs(reqParams)
	if err != nil {
		w.Write(util.ApiResponse(500, err.Error(), nil))
		return
	}

	log.Printf("NOTICE: dumping inflight for %s:%s", topicName, channelName)

	topic := nsqd.GetTopic(topicName)
	channel := topic.GetChannel(channelName)

	fmt.Fprintf(w, "inFlightMessages:\n")
	channel.Lock()
	for _, item := range channel.inFlightMessages {
		msg := item.(*pqueue.Item).Value.(*inFlightMessage).msg
		fmt.Fprintf(w, "%s %s %d\n", msg.Id, time.Unix(msg.Timestamp, 0).String(), msg.Attempts)
	}
	channel.Unlock()

	fmt.Fprintf(w, "inFlightPQ:\n")
	channel.deferredMutex.Lock()
	for i := 0; i < len(channel.inFlightPQ); i++ {
		item := channel.inFlightPQ[i]
		msg := item.Value.(*inFlightMessage).msg
		fmt.Fprintf(w, "id: %s created: %s attempts: %d priority: %d\n", msg.Id, time.Unix(msg.Timestamp, 0).String(), msg.Attempts, item.Priority)
	}
	channel.deferredMutex.Unlock()
}

func memProfileHandler(w http.ResponseWriter, req *http.Request) {
	log.Printf("MEMORY Profiling Enabled")
	f, err := os.Create("nsqd.mprof")
	if err != nil {
		log.Fatal(err)
	}
	pprof.WriteHeapProfile(f)
	f.Close()

	w.Header().Set("Content-Length", "2")
	io.WriteString(w, "OK")
}

func pingHandler(w http.ResponseWriter, req *http.Request) {
	w.Header().Set("Content-Length", "2")
	io.WriteString(w, "OK")
}

func putHandler(w http.ResponseWriter, req *http.Request) {
	reqParams, err := util.NewReqParams(req)
	if err != nil {
		log.Printf("ERROR: failed to parse request params - %s", err.Error())
		w.Write(util.ApiResponse(500, "INVALID_REQUEST", nil))
		return
	}

	topicName, err := reqParams.Query("topic")
	if err != nil {
		w.Write(util.ApiResponse(500, "MISSING_ARG_TOPIC", nil))
		return
	}

	if len(topicName) > nsq.MaxNameLength {
		w.Write(util.ApiResponse(500, "INVALID_ARG_TOPIC", nil))
		return
	}

	topic := nsqd.GetTopic(topicName)
	msg := nsq.NewMessage(<-nsqd.idChan, reqParams.Body)
	topic.PutMessage(msg)

	w.Header().Set("Content-Length", "2")
	io.WriteString(w, "OK")
}

func mputHandler(w http.ResponseWriter, req *http.Request) {
	reqParams, err := util.NewReqParams(req)
	if err != nil {
		log.Printf("ERROR: failed to parse request params - %s", err.Error())
		w.Write(util.ApiResponse(500, "INVALID_REQUEST", nil))
		return
	}

	topicName, err := reqParams.Query("topic")
	if err != nil {
		w.Write(util.ApiResponse(500, "MISSING_ARG_TOPIC", nil))
		return
	}

	if len(topicName) > nsq.MaxNameLength {
		w.Write(util.ApiResponse(500, "INVALID_ARG_TOPIC", nil))
		return
	}

	topic := nsqd.GetTopic(topicName)
	for _, block := range bytes.Split(reqParams.Body, []byte("\n")) {
		if len(block) != 0 {
			msg := nsq.NewMessage(<-nsqd.idChan, block)
			topic.PutMessage(msg)
		}
	}

	w.Header().Set("Content-Length", "2")
	io.WriteString(w, "OK")
}

func emptyHandler(w http.ResponseWriter, req *http.Request) {
	reqParams, err := util.NewReqParams(req)
	if err != nil {
		log.Printf("ERROR: failed to parse request params - %s", err.Error())
		w.Write(util.ApiResponse(500, "INVALID_REQUEST", nil))
		return
	}

	topicName, channelName, err := getTopicChannelArgs(reqParams)
	if err != nil {
		w.Write(util.ApiResponse(500, err.Error(), nil))
		return
	}

	topic := nsqd.GetTopic(topicName)
	channel := topic.GetChannel(channelName)
	err = EmptyQueue(channel)
	if err != nil {
		w.Write(util.ApiResponse(500, "INTERNAL_ERROR", nil))
		return
	}

	w.Header().Set("Content-Length", "2")
	io.WriteString(w, "OK")
}

func getTopicChannelArgs(rp *util.ReqParams) (string, string, error) {
	topicName, err := rp.Query("topic")
	if err != nil {
		return "", "", errors.New("MISSING_ARG_TOPIC")
	}

	if len(topicName) > nsq.MaxNameLength {
		return "", "", errors.New("INVALID_ARG_TOPIC")
	}

	channelName, err := rp.Query("channel")
	if err != nil {
		return "", "", errors.New("MISSING_ARG_CHANNEL")
	}

	if len(topicName) > nsq.MaxNameLength {
		return "", "", errors.New("INVALID_ARG_CHANNEL")
	}

	return topicName, channelName, nil
}
<commit_msg>add endpoint to cpu profile<commit_after>package main

import (
	"../nsq"
	"../util"
	"../util/pqueue"
	"bytes"
	"errors"
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	"os"
	"runtime/pprof"
	"strings"
	"time"
)

import httpprof "net/http/pprof"

func HttpServer(listener net.Listener) {
	log.Printf("HTTP: listening on %s", listener.Addr().String())
	handler := http.NewServeMux()
	handler.HandleFunc("/ping", pingHandler)
	handler.HandleFunc("/put", putHandler)
	handler.HandleFunc("/mput", mputHandler)
	handler.HandleFunc("/stats", statsHandler)
	handler.HandleFunc("/empty", emptyHandler)
	handler.HandleFunc("/mem_profile", memProfileHandler)
	handler.HandleFunc("/cpu_profile", httpprof.Profile)
	handler.HandleFunc("/dump_inflight", dumpInFlightHandler)
	server := &http.Server{Handler: handler}
	err := server.Serve(listener)
	// theres no direct way to detect this error because it is not exposed
	if err != nil && !strings.Contains(err.Error(), "use of closed network connection") {
		log.Printf("ERROR: http.Serve() - %s", err.Error())
	}
}

func dumpInFlightHandler(w http.ResponseWriter, req *http.Request) {
	reqParams, err := util.NewReqParams(req)
	if err != nil {
		log.Printf("ERROR: failed to parse request params - %s", err.Error())
		w.Write(util.ApiResponse(500, "INVALID_REQUEST", nil))
		return
	}

	topicName, channelName, err := getTopicChannelArgs(reqParams)
	if err != nil {
		w.Write(util.ApiResponse(500, err.Error(), nil))
		return
	}

	log.Printf("NOTICE: dumping inflight for %s:%s", topicName, channelName)

	topic := nsqd.GetTopic(topicName)
	channel := topic.GetChannel(channelName)

	fmt.Fprintf(w, "inFlightMessages:\n")
	channel.Lock()
	for _, item := range channel.inFlightMessages {
		msg := item.(*pqueue.Item).Value.(*inFlightMessage).msg
		fmt.Fprintf(w, "%s %s %d\n", msg.Id, time.Unix(msg.Timestamp, 0).String(), msg.Attempts)
	}
	channel.Unlock()

	fmt.Fprintf(w, "inFlightPQ:\n")
	channel.deferredMutex.Lock()
	for i := 0; i < len(channel.inFlightPQ); i++ {
		item := channel.inFlightPQ[i]
		msg := item.Value.(*inFlightMessage).msg
		fmt.Fprintf(w, "id: %s created: %s attempts: %d priority: %d\n", msg.Id, time.Unix(msg.Timestamp, 0).String(), msg.Attempts, item.Priority)
	}
	channel.deferredMutex.Unlock()
}

func memProfileHandler(w http.ResponseWriter, req *http.Request) {
	log.Printf("MEMORY Profiling Enabled")
	f, err := os.Create("nsqd.mprof")
	if err != nil {
		log.Fatal(err)
	}
	pprof.WriteHeapProfile(f)
	f.Close()

	w.Header().Set("Content-Length", "2")
	io.WriteString(w, "OK")
}

func pingHandler(w http.ResponseWriter, req *http.Request) {
	w.Header().Set("Content-Length", "2")
	io.WriteString(w, "OK")
}

func putHandler(w http.ResponseWriter, req *http.Request) {
	reqParams, err := util.NewReqParams(req)
	if err != nil {
		log.Printf("ERROR: failed to parse request params - %s", err.Error())
		w.Write(util.ApiResponse(500, "INVALID_REQUEST", nil))
		return
	}

	topicName, err := reqParams.Query("topic")
	if err != nil {
		w.Write(util.ApiResponse(500, "MISSING_ARG_TOPIC", nil))
		return
	}

	if len(topicName) > nsq.MaxNameLength {
		w.Write(util.ApiResponse(500, "INVALID_ARG_TOPIC", nil))
		return
	}

	topic := nsqd.GetTopic(topicName)
	msg := nsq.NewMessage(<-nsqd.idChan, reqParams.Body)
	topic.PutMessage(msg)

	w.Header().Set("Content-Length", "2")
	io.WriteString(w, "OK")
}

func mputHandler(w http.ResponseWriter, req *http.Request) {
	reqParams, err := util.NewReqParams(req)
	if err != nil {
		log.Printf("ERROR: failed to parse request params - %s", err.Error())
		w.Write(util.ApiResponse(500, "INVALID_REQUEST", nil))
		return
	}

	topicName, err := reqParams.Query("topic")
	if err != nil {
		w.Write(util.ApiResponse(500, "MISSING_ARG_TOPIC", nil))
		return
	}

	if len(topicName) > nsq.MaxNameLength {
		w.Write(util.ApiResponse(500, "INVALID_ARG_TOPIC", nil))
		return
	}

	topic := nsqd.GetTopic(topicName)
	for _, block := range bytes.Split(reqParams.Body, []byte("\n")) {
		if len(block) != 0 {
			msg := nsq.NewMessage(<-nsqd.idChan, block)
			topic.PutMessage(msg)
		}
	}

	w.Header().Set("Content-Length", "2")
	io.WriteString(w, "OK")
}

func emptyHandler(w http.ResponseWriter, req *http.Request) {
	reqParams, err := util.NewReqParams(req)
	if err != nil {
		log.Printf("ERROR: failed to parse request params - %s", err.Error())
		w.Write(util.ApiResponse(500, "INVALID_REQUEST", nil))
		return
	}

	topicName, channelName, err := getTopicChannelArgs(reqParams)
	if err != nil {
		w.Write(util.ApiResponse(500, err.Error(), nil))
		return
	}

	topic := nsqd.GetTopic(topicName)
	channel := topic.GetChannel(channelName)
	err = EmptyQueue(channel)
	if err != nil {
		w.Write(util.ApiResponse(500, "INTERNAL_ERROR", nil))
		return
	}

	w.Header().Set("Content-Length", "2")
	io.WriteString(w, "OK")
}

func getTopicChannelArgs(rp *util.ReqParams) (string, string, error) {
	topicName, err := rp.Query("topic")
	if err != nil {
		return "", "", errors.New("MISSING_ARG_TOPIC")
	}

	if len(topicName) > nsq.MaxNameLength {
		return "", "", errors.New("INVALID_ARG_TOPIC")
	}

	channelName, err := rp.Query("channel")
	if err != nil {
		return "", "", errors.New("MISSING_ARG_CHANNEL")
	}

	if len(topicName) > nsq.MaxNameLength {
		return "", "", errors.New("INVALID_ARG_CHANNEL")
	}

	return topicName, channelName, nil
}
<|endoftext|>


<commit_before>package uax29

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"unicode/utf8"
)

// BreakFunc defines a func that indicates when to break tokens.
// It is called for every rune in the incoming reader. The current rune is buffer[pos].
// Returing true indicates break at the current rune, i.e. it begins a new token;
// false indicates accept the current rune and continue.
type BreakFunc func(buffer Runes, pos Pos) bool

// NewScanner instantiates a new Scanner
func NewScanner(r io.Reader, breakFunc BreakFunc) *Scanner {
	return &Scanner{
		incoming:  bufio.NewReaderSize(r, 64*1024),
		breakFunc: breakFunc,
	}
}

// Break is the result of a BreakFunc, signaling to break at (before) the current rune
const Break = true

// Accept is the result of a BreakFunc, signaling to accept the current rune and continue
const Accept = false

const lookahead = 8

// Scanner is a structure for scanning an input Reader. Use NewScanner to instantiate.
// Loop over scanner.Scan while true.
type Scanner struct {
	incoming  *bufio.Reader
	breakFunc BreakFunc

	// a buffer of runes to evaluate
	buffer Runes
	// a cursor for runes in the buffer
	pos Pos

	// outputs
	bytes []byte
	err   error
}

// Scan advances to the next token, returning true if successful. Returns false on error or EOF.
// Use Bytes or Text to retrieve the token value, or Err to retrieve the current error.
func (sc *Scanner) Scan() bool {
	sc.reset()

	for {
		// Fill the buffer with enough runes for lookahead
		for len(sc.buffer) < int(sc.pos)+lookahead {
			r, eof, err := sc.readRune()
			if err != nil {
				sc.err = err
				return false
			}
			if eof {
				break
			}
			sc.buffer = append(sc.buffer, r)
		}

		if sc.breakFunc(sc.buffer, sc.pos) == Break {
			// The current rune represents a new token
			break
		}

		// Otherwise, accept the current rune and continue
		sc.pos++
	}

	return sc.token()
}

// Bytes returns the current token as a byte slice, after a successful call to Scan
func (sc *Scanner) Bytes() []byte {
	return sc.bytes
}

// Text returns the current token, after a successful call to Scan
func (sc *Scanner) Text() string {
	return string(sc.bytes)
}

// Err returns the current error, after an unsuccessful call to Scan
func (sc *Scanner) Err() error {
	return sc.err
}

// reset sets the Scanner to evaluate a new token
func (sc *Scanner) reset() {
	// Drop the emitted runes (optimization to avoid growing array)
	copy(sc.buffer, sc.buffer[sc.pos:])
	sc.buffer = sc.buffer[:len(sc.buffer)-int(sc.pos)]

	sc.pos = 0

	sc.bytes = nil
	sc.err = nil
}

func (sc *Scanner) token() bool {
	var bb bytes.Buffer
	for _, r := range sc.buffer[:sc.pos] {
		bb.WriteRune(r)
	}
	sc.bytes = bb.Bytes()
	return len(sc.bytes) > 0
}

// readRune gets the next rune, advancing the reader
func (sc *Scanner) readRune() (r rune, eof bool, err error) {
	r, _, err = sc.incoming.ReadRune()

	if err != nil {
		if err == io.EOF {
			return r, true, nil
		}
		return r, false, err
	}

	return r, false, nil
}

// NewSplitFunc creates a new bufio.SplitFunc, based on a uax29.BreakFunc
func NewSplitFunc(breakFunc BreakFunc) bufio.SplitFunc {
	return func(data []byte, atEOF bool) (advance int, token []byte, err error) {
		if atEOF && len(data) == 0 {
			return 0, nil, nil
		}

		var buffer Runes
		var pos Pos

		i := 0

		for {
			// Fill the buffer with enough runes for lookahead
			for i < len(data) && len(buffer) < lookahead {
				r, w := utf8.DecodeRune(data[i:])
				if r == utf8.RuneError {
					return 0, nil, fmt.Errorf("error decoding rune")
				}
				i += w
				buffer = append(buffer, r)
			}

			if len(buffer) < lookahead {
				if !atEOF {
					// Need to request more data
					return 0, nil, nil
				}
			}

			if breakFunc(buffer, pos) == Break {
				// The current rune represents a new token
				break
			}

			// Otherwise, accept the current rune and continue
			pos++
		}

		// Count the bytes
		n := 0
		for _, r := range buffer[:pos] {
			n += utf8.RuneLen(r)
		}
		return n, data[:n], nil
	}
}
<commit_msg>Pool the Runes buffer for SplitFunc; perf is better<commit_after>package uax29

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"sync"
	"unicode/utf8"
)

// BreakFunc defines a func that indicates when to break tokens.
// It is called for every rune in the incoming reader. The current rune is buffer[pos].
// Returing true indicates break at the current rune, i.e. it begins a new token;
// false indicates accept the current rune and continue.
type BreakFunc func(buffer Runes, pos Pos) bool

// NewScanner instantiates a new Scanner
func NewScanner(r io.Reader, breakFunc BreakFunc) *Scanner {
	return &Scanner{
		incoming:  bufio.NewReaderSize(r, 64*1024),
		breakFunc: breakFunc,
	}
}

// Break is the result of a BreakFunc, signaling to break at (before) the current rune
const Break = true

// Accept is the result of a BreakFunc, signaling to accept the current rune and continue
const Accept = false

const lookahead = 8

// Scanner is a structure for scanning an input Reader. Use NewScanner to instantiate.
// Loop over scanner.Scan while true.
type Scanner struct {
	incoming  *bufio.Reader
	breakFunc BreakFunc

	// a buffer of runes to evaluate
	buffer Runes
	// a cursor for runes in the buffer
	pos Pos

	// outputs
	bytes []byte
	err   error
}

// Scan advances to the next token, returning true if successful. Returns false on error or EOF.
// Use Bytes or Text to retrieve the token value, or Err to retrieve the current error.
func (sc *Scanner) Scan() bool {
	sc.reset()

	for {
		// Fill the buffer with enough runes for lookahead
		for len(sc.buffer) < int(sc.pos)+lookahead {
			r, eof, err := sc.readRune()
			if err != nil {
				sc.err = err
				return false
			}
			if eof {
				break
			}
			sc.buffer = append(sc.buffer, r)
		}

		if sc.breakFunc(sc.buffer, sc.pos) == Break {
			// The current rune represents a new token
			break
		}

		// Otherwise, accept the current rune and continue
		sc.pos++
	}

	return sc.token()
}

// Bytes returns the current token as a byte slice, after a successful call to Scan
func (sc *Scanner) Bytes() []byte {
	return sc.bytes
}

// Text returns the current token, after a successful call to Scan
func (sc *Scanner) Text() string {
	return string(sc.bytes)
}

// Err returns the current error, after an unsuccessful call to Scan
func (sc *Scanner) Err() error {
	return sc.err
}

// reset sets the Scanner to evaluate a new token
func (sc *Scanner) reset() {
	// Drop the emitted runes (optimization to avoid growing array)
	copy(sc.buffer, sc.buffer[sc.pos:])
	sc.buffer = sc.buffer[:len(sc.buffer)-int(sc.pos)]

	sc.pos = 0

	sc.bytes = nil
	sc.err = nil
}

func (sc *Scanner) token() bool {
	var bb bytes.Buffer
	for _, r := range sc.buffer[:sc.pos] {
		bb.WriteRune(r)
	}
	sc.bytes = bb.Bytes()
	return len(sc.bytes) > 0
}

// readRune gets the next rune, advancing the reader
func (sc *Scanner) readRune() (r rune, eof bool, err error) {
	r, _, err = sc.incoming.ReadRune()

	if err != nil {
		if err == io.EOF {
			return r, true, nil
		}
		return r, false, err
	}

	return r, false, nil
}

var runesPool = sync.Pool{
	New: func() interface{} {
		var runes Runes
		return runes
	},
}

// NewSplitFunc creates a new bufio.SplitFunc, based on a uax29.BreakFunc
func NewSplitFunc(breakFunc BreakFunc) bufio.SplitFunc {
	return func(data []byte, atEOF bool) (advance int, token []byte, err error) {
		if atEOF && len(data) == 0 {
			return 0, nil, nil
		}

		buffer := runesPool.Get().(Runes)[:0]
		var pos Pos

		i := 0

		for {
			// Fill the buffer with enough runes for lookahead
			for i < len(data) && len(buffer) < lookahead {
				r, w := utf8.DecodeRune(data[i:])
				if r == utf8.RuneError {
					return 0, nil, fmt.Errorf("error decoding rune")
				}
				i += w
				buffer = append(buffer, r)
			}

			if len(buffer) < lookahead {
				if !atEOF {
					// Need to request more data
					return 0, nil, nil
				}
			}

			if breakFunc(buffer, pos) == Break {
				// The current rune represents a new token
				break
			}

			// Otherwise, accept the current rune and continue
			pos++
		}

		// Count the bytes
		n := 0
		for _, r := range buffer[:pos] {
			n += utf8.RuneLen(r)
		}

		runesPool.Put(buffer)

		return n, data[:n], nil
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"errors"
	"github.com/couchbaselabs/indexing/api"
	"log"
	"net"
	"net/rpc"
	"net/rpc/jsonrpc"
)

type MutationManager struct {
	enginemap   map[string]api.Finder
	sequencemap api.IndexSequenceMap
}

var META_DOC_ID = "."
var mutationMgr MutationManager

type ddlNotification struct {
	indexinfo api.IndexInfo
	engine    api.Finder
	ddltype   api.RequestType
}

type seqNotification struct {
	engine  api.Finder
	indexid string
	seqno   uint64
	vbucket uint16
}

var chseq chan seqNotification

//This function returns a map of <Index, SequenceVector> based on the IndexList received in request
func (m *MutationManager) GetSequenceVectors(indexList api.IndexList, reply *api.IndexSequenceMap) error {

	//if indexList is nil, return the complete map
	if len(indexList) == 0 {
		*reply = m.sequencemap
		log.Printf("Mutation Manager returning complete SequenceMap")
		return nil
	}

	//loop through the list of requested indexes and return the sequenceVector for those indexes
	var replyMap = make(api.IndexSequenceMap)
	for _, idx := range indexList {
		//if the requested index is not found, return an error
		v, ok := m.sequencemap[idx]
		if !ok {
			return errors.New("Requested Index Not Found")
		}

		//add to the reply map
		log.Printf("Mutation Manager returning sequence vector for index %v", idx)
		replyMap[idx] = v
	}
	*reply = replyMap
	return nil

}

func (m *MutationManager) ProcessSingleMutation(mutation *api.Mutation, reply *bool) error {
	log.Printf("Received Mutation Type %s Indexid %v, Docid %v, Vbucket %v, Seqno %v", mutation.Type, mutation.Indexid, mutation.Docid, mutation.Vbucket, mutation.Seqno)

	//FIXME change this to channel based
	*reply = false

	if mutation.Type == api.INSERT {

		var key api.Key
		var value api.Value
		var err error

		if key, err = api.NewKey(mutation.SecondaryKey, mutation.Docid); err != nil {
			log.Printf("Error Generating Key From Mutation %v", err)
			*reply = false
			return err
		}

		if value, err = api.NewValue(mutation.SecondaryKey, mutation.Docid, mutation.Vbucket, mutation.Seqno); err != nil {
			log.Printf("Error Generating Value From Mutation %v", err)
			*reply = false
			return err
		}

		if engine, ok := m.enginemap[mutation.Indexid]; ok {
			if err := engine.InsertMutation(key, value); err != nil {
				log.Printf("Error from Engine during InsertMutation %v", err)
				*reply = false
				return err
			}
			//send notification for this seqno to be recorded in SeqVector
			seqnotify := seqNotification{engine: engine,
				indexid: mutation.Indexid,
				seqno:   mutation.Seqno,
				vbucket: mutation.Vbucket,
			}
			chseq <- seqnotify
		} else {
			log.Printf("Unknown Index %v or Engine not found", mutation.Indexid)
			*reply = false
			return errors.New("Unknown Index or Engine not found")
		}

		*reply = true

	} else if mutation.Type == api.DELETE {

		if engine, ok := m.enginemap[mutation.Indexid]; ok {
			if err := engine.DeleteMutation(mutation.Docid); err != nil {
				log.Printf("Error from Engine during Delete Mutation %v", err)
				*reply = false
				return err
			}
			//send notification for this seqno to be recorded in SeqVector
			seqnotify := seqNotification{engine: engine,
				indexid: mutation.Indexid,
				seqno:   mutation.Seqno,
				vbucket: mutation.Vbucket,
			}
			chseq <- seqnotify
		} else {
			log.Printf("Unknown Index %v or Engine not found", mutation.Indexid)
			*reply = false
			return errors.New("Unknown Index or Engine not found")
		}
		*reply = true

	}
	return nil
}

func StartMutationManager(engineMap map[string]api.Finder) (chan ddlNotification, error) {

	var err error

	//init the mutation manager maps
	//mutationMgr.enginemap= make(map[string]api.Finder)
	mutationMgr.sequencemap = make(api.IndexSequenceMap)
	//copy the inital map from the indexer
	mutationMgr.enginemap = engineMap
	mutationMgr.initSequenceMapFromPersistence()

	//create channel to receive notification for new sequence numbers
	//and start a goroutine to manage it
	chseq = make(chan seqNotification, api.MAX_VBUCKETS)
	go mutationMgr.manageSeqNotification(chseq)

	//create a channel to receive notification from indexer
	//and start a goroutine to listen to it
	chnotify = make(chan ddlNotification)
	go acceptIndexerNotification(chnotify)

	//start the rpc server
	if err = startRPCServer(); err != nil {
		return nil, err
	}

	return chnotify, nil
}

func startRPCServer() error {

	log.Println("Starting Mutation Manager")
	server := rpc.NewServer()
	server.Register(&mutationMgr)

	server.HandleHTTP(rpc.DefaultRPCPath, rpc.DefaultDebugPath)

	l, err := net.Listen("tcp", ":8096")
	if err != nil {
		return err
	}

	go func() {
		for {
			conn, err := l.Accept()
			if err != nil {
				log.Printf("Error in Accept %v. Shutting down")
				//FIXME Add a cleanup function
				return
			}
			go server.ServeCodec(jsonrpc.NewServerCodec(conn))
		}
	}()
	return nil

}

func acceptIndexerNotification(chnotify chan ddlNotification) {

	ok := true
	var ddl ddlNotification
	for ok {
		ddl, ok = <-chnotify
		if ok {
			switch ddl.ddltype {
			case api.CREATE:
				mutationMgr.enginemap[ddl.indexinfo.Uuid] = ddl.engine
			case api.DROP:
				delete(mutationMgr.enginemap, ddl.indexinfo.Uuid)
			default:
				log.Printf("Mutation Manager Received Unsupported Notification %v", ddl.ddltype)
			}
		}
	}
}

func (m *MutationManager) manageSeqNotification(chseq chan seqNotification) {

	ok := true
	var seq seqNotification
	for ok {
		seq, ok = <-chseq
		if ok {
			seqVector := m.sequencemap[seq.indexid]
			seqVector[seq.vbucket] = seq.seqno
			m.sequencemap[seq.indexid] = seqVector
			jsonval, err := json.Marshal(m.sequencemap[seq.indexid])
			if err != nil {
				log.Printf("Error Marshalling SequenceMap %v", err)
			} else {
				m.enginemap[seq.indexid].InsertMeta(META_DOC_ID, string(jsonval))
			}
		}
	}
}

func (m *MutationManager) initSequenceMapFromPersistence() {

	var sequenceVector api.SequenceVector
	for idx, engine := range m.enginemap {
		metaval, err := engine.GetMeta(META_DOC_ID)
		if err != nil {
			log.Printf("Error retreiving Meta from Engine %v", err)
		}
		err = json.Unmarshal([]byte(metaval), &sequenceVector)
		if err != nil {
			log.Printf("Error unmarshalling SequenceVector %v", err)
		}
		m.sequencemap[idx] = sequenceVector
	}
}
<commit_msg>Performance optimization for mutation manager<commit_after>package main

import (
	"encoding/json"
	"errors"
	"github.com/couchbaselabs/indexing/api"
	"log"
	"net"
	"net/rpc"
	"net/rpc/jsonrpc"
)

type MutationManager struct {
	enginemap   map[string]api.Finder
	sequencemap api.IndexSequenceMap
}

var META_DOC_ID = "."
var mutationMgr MutationManager

type ddlNotification struct {
	indexinfo api.IndexInfo
	engine    api.Finder
	ddltype   api.RequestType
}

type seqNotification struct {
	engine  api.Finder
	indexid string
	seqno   uint64
	vbucket uint16
}

//error state flag
var indexerErrorState bool
var indexerErrorString string

//channel for sequence notification
var chseq chan seqNotification

//This function returns a map of <Index, SequenceVector> based on the IndexList received in request
func (m *MutationManager) GetSequenceVectors(indexList api.IndexList, reply *api.IndexSequenceMap) error {

	//reset the error state at each handshake
	indexerErrorState = false
	indexerErrorString = ""

	//if indexList is nil, return the complete map
	if len(indexList) == 0 {
		*reply = m.sequencemap
		log.Printf("Mutation Manager returning complete SequenceMap")
		return nil
	}

	//loop through the list of requested indexes and return the sequenceVector for those indexes
	var replyMap = make(api.IndexSequenceMap)
	for _, idx := range indexList {
		//if the requested index is not found, return an error
		v, ok := m.sequencemap[idx]
		if !ok {
			return errors.New("Requested Index Not Found")
		}

		//add to the reply map
		log.Printf("Mutation Manager returning sequence vector for index %v", idx)
		replyMap[idx] = v
	}
	*reply = replyMap
	return nil

}

func (m *MutationManager) ProcessSingleMutation(mutation *api.Mutation, reply *bool) error {
	log.Printf("Received Mutation Type %s Indexid %v, Docid %v, Vbucket %v, Seqno %v", mutation.Type, mutation.Indexid, mutation.Docid, mutation.Vbucket, mutation.Seqno)

	//if there is any pending error, reply with that. This will force a handshake again.
	if indexerErrorState == true {
		*reply = false
	}

	//copy the mutation data and return
	mutationCopy := *mutation
	go m.handleMutation(mutationCopy)
	*reply = true
	return nil

}

//process the mutation, store it. In case of any error, set indexerInErrorState to TRUE
func (m *MutationManager) handleMutation(mutation api.Mutation) {

	if mutation.Type == api.INSERT {

		var key api.Key
		var value api.Value
		var err error

		if key, err = api.NewKey(mutation.SecondaryKey, mutation.Docid); err != nil {
			log.Printf("Error Generating Key From Mutation %v. Skipped.", err)
			return
		}

		if value, err = api.NewValue(mutation.SecondaryKey, mutation.Docid, mutation.Vbucket, mutation.Seqno); err != nil {
			log.Printf("Error Generating Value From Mutation %v. Skipped.", err)
			return
		}

		if engine, ok := m.enginemap[mutation.Indexid]; ok {
			if err := engine.InsertMutation(key, value); err != nil {
				log.Printf("Error from Engine during InsertMutation. Key %v. Index %v. Error %v", key, mutation.Docid, err)
			}
			//send notification for this seqno to be recorded in SeqVector
			seqnotify := seqNotification{engine: engine,
				indexid: mutation.Indexid,
				seqno:   mutation.Seqno,
				vbucket: mutation.Vbucket,
			}
			chseq <- seqnotify
		} else {
			log.Printf("Unknown Index %v or Engine not found", mutation.Indexid)
			indexerErrorState = true
			indexerErrorString = "Unknown Index or Engine not found"
		}

	} else if mutation.Type == api.DELETE {

		if engine, ok := m.enginemap[mutation.Indexid]; ok {
			if err := engine.DeleteMutation(mutation.Docid); err != nil {
				log.Printf("Error from Engine during Delete Mutation. Key %v. Error %v", mutation.Docid, err)
				return
			}
			//send notification for this seqno to be recorded in SeqVector
			seqnotify := seqNotification{engine: engine,
				indexid: mutation.Indexid,
				seqno:   mutation.Seqno,
				vbucket: mutation.Vbucket,
			}
			chseq <- seqnotify
		} else {
			log.Printf("Unknown Index %v or Engine not found", mutation.Indexid)
			indexerErrorState = true
			indexerErrorString = "Unknown Index or Engine not found"
		}
	}
}

func StartMutationManager(engineMap map[string]api.Finder) (chan ddlNotification, error) {

	var err error

	//init the mutation manager maps
	//mutationMgr.enginemap= make(map[string]api.Finder)
	mutationMgr.sequencemap = make(api.IndexSequenceMap)
	//copy the inital map from the indexer
	mutationMgr.enginemap = engineMap
	mutationMgr.initSequenceMapFromPersistence()

	//create channel to receive notification for new sequence numbers
	//and start a goroutine to manage it
	chseq = make(chan seqNotification, api.MAX_VBUCKETS)
	go mutationMgr.manageSeqNotification(chseq)

	//create a channel to receive notification from indexer
	//and start a goroutine to listen to it
	chnotify = make(chan ddlNotification)
	go acceptIndexerNotification(chnotify)

	//start the rpc server
	if err = startRPCServer(); err != nil {
		return nil, err
	}

	return chnotify, nil
}

func startRPCServer() error {

	log.Println("Starting Mutation Manager")
	server := rpc.NewServer()
	server.Register(&mutationMgr)

	server.HandleHTTP(rpc.DefaultRPCPath, rpc.DefaultDebugPath)

	l, err := net.Listen("tcp", ":8096")
	if err != nil {
		return err
	}

	go func() {
		for {
			conn, err := l.Accept()
			if err != nil {
				log.Printf("Error in Accept %v. Shutting down")
				//FIXME Add a cleanup function
				return
			}
			go server.ServeCodec(jsonrpc.NewServerCodec(conn))
		}
	}()
	return nil

}

func acceptIndexerNotification(chnotify chan ddlNotification) {

	ok := true
	var ddl ddlNotification
	for ok {
		ddl, ok = <-chnotify
		if ok {
			switch ddl.ddltype {
			case api.CREATE:
				mutationMgr.enginemap[ddl.indexinfo.Uuid] = ddl.engine
				//init sequence map of new index
				var seqVec api.SequenceVector
				mutationMgr.sequencemap[ddl.indexinfo.Uuid] = seqVec
			case api.DROP:
				delete(mutationMgr.enginemap, ddl.indexinfo.Uuid)
				//FIXME : Delete index entry from sequence map
			default:
				log.Printf("Mutation Manager Received Unsupported Notification %v", ddl.ddltype)
			}
		}
	}
}

func (m *MutationManager) manageSeqNotification(chseq chan seqNotification) {

	ok := true
	var seq seqNotification
	for ok {
		seq, ok = <-chseq
		if ok {
			seqVector := m.sequencemap[seq.indexid]
			seqVector[seq.vbucket] = seq.seqno
			m.sequencemap[seq.indexid] = seqVector
			jsonval, err := json.Marshal(m.sequencemap[seq.indexid])
			if err != nil {
				log.Printf("Error Marshalling SequenceMap %v", err)
			} else {
				m.enginemap[seq.indexid].InsertMeta(META_DOC_ID, string(jsonval))
			}
		}
	}
}

func (m *MutationManager) initSequenceMapFromPersistence() {

	var sequenceVector api.SequenceVector
	for idx, engine := range m.enginemap {
		metaval, err := engine.GetMeta(META_DOC_ID)
		if err != nil {
			log.Printf("Error retreiving Meta from Engine %v", err)
		}
		err = json.Unmarshal([]byte(metaval), &sequenceVector)
		if err != nil {
			log.Printf("Error unmarshalling SequenceVector %v", err)
		}
		m.sequencemap[idx] = sequenceVector
	}
}
<|endoftext|>


<commit_before>package main

import (
    "os"
    "log"
    "strconv"
    "database/sql"
    "encoding/json"
    _ "github.com/mattn/go-sqlite3"
    "github.com/mediocregopher/radix.v2/redis"
)

func boolToInt(val bool) uint {
    if val {
        return 1
    }
    return 0
}

type Town struct {
	Id             uint32  `json:"id"`
	Name           string  `json:"name"`
	NameTr         string  `json:"name_tr"`
	RegionId       *uint32 `json:"region_id"`
	RegionalCenter bool    `json:"regional_center"`
	Latitude       float32 `json:"latitude"`
	Longitude      float32 `json:"longitude"`
	Zoom           uint32  `json:"zoom"`
}

type Region struct {
	Id        uint32  `json:"id"`
	Name      string  `json:"name"`
	NameTr    string  `json:"name_tr"`
	Latitude  float32 `json:"latitude"`
	Longitude float32 `json:"longitude"`
	Zoom      uint32  `json:"zoom"`
}

type CashPoint struct {
	Id             uint32  `json:"id"`
	Type           string  `json:"type"`
	BankId         uint32  `json:"bank_id"`
	TownId         uint32  `json:"town_id"`
	Longitude      float32 `json:"longitude"`
	Latitude       float32 `json:"latitude"`
	Address        string  `json:"address"`
	AddressComment string  `json:"address_comment"`
	MetroName      string  `json:"metro_name"`
	FreeAccess     bool    `json:"free_access"`
	MainOffice     bool    `json:"main_office"`
	WithoutWeekend bool    `json:"without_weekend"`
	RoundTheClock  bool    `json:"round_the_clock"`
	WorksAsShop    bool    `json:"works_as_shop"`
	Schedule       string  `json:"schedule"`
	Tel            string  `json:"tel"`
	Additional     string  `json:"additional"`
	Rub            bool    `json:"rub"`
	Usd            bool    `json:"usd"`
	Eur            bool    `json:"eur"`
	CashIn         bool    `json:"cash_in"`
}

func migrateTowns(townsDb *sql.DB, redisCli *redis.Client) {
    var townsCount int
    err := townsDb.QueryRow(`SELECT COUNT(*) FROM towns`).Scan(&townsCount)
    if err != nil {
        log.Fatalf("migrate: towns: %v\n", err)
    }

    rows, err := townsDb.Query(`SELECT id, name, name_tr, region_id,
                                       regional_center, latitude,
                                       longitude, zoom FROM towns`)
    if err != nil {
        log.Fatalf("migrate: towns: %v\n", err)
    }

    currentTownIdx := 1
    for rows.Next() {
        town := new(Town)
        var regionId uint32 = 0
        err = rows.Scan(&town.Id, &town.Name, &town.NameTr,
                        &regionId, &town.RegionalCenter,
                        &town.Latitude, &town.Longitude, &town.Zoom)
        if err != nil {
            log.Fatal(err)
        }

        if regionId != 0 {
            town.RegionId = new(uint32)
            *town.RegionId = regionId
        }

        jsonData, err := json.Marshal(town)
        if err != nil {
            log.Fatal(err)
        }

        err = redisCli.Cmd("SET", "town:" + strconv.FormatUint(uint64(town.Id), 10), string(jsonData)).Err
        if err != nil {
            log.Fatal(err)
        }

        err = redisCli.Cmd("GEOADD", "towns", town.Longitude, town.Latitude, town.Id).Err
        if err != nil {
            log.Fatal(err)
        }

        currentTownIdx++

        if currentTownIdx % 500 == 0 {
            log.Printf("[%d/%d] Towns processed\n", currentTownIdx, townsCount)
        }
    }
    log.Printf("[%d/%d] Towns processed\n", townsCount, townsCount)
}

func migrateRegions(townsDb *sql.DB, redisCli *redis.Client) {
    var regionsCount int
    err := townsDb.QueryRow(`SELECT COUNT(*) FROM regions`).Scan(&regionsCount)
    if err != nil {
        log.Fatalf("migrate: regions: %v\n", err)
    }

    rows, err := townsDb.Query(`SELECT id, name, name_tr,
                                       latitude, longitude, zoom FROM regions`)
    if err != nil {
        log.Fatalf("migrate: regions: %v\n", err)
    }

    currentRegionIdx := 1
    for rows.Next() {
        region := new(Region)
        err = rows.Scan(&region.Id, &region.Name, &region.NameTr,
                        &region.Latitude, &region.Longitude, &region.Zoom)
        if err != nil {
            log.Fatal(err)
        }

        jsonData, err := json.Marshal(region)
        if err != nil {
            log.Fatal(err)
        }

        err = redisCli.Cmd("SET", "region:" + strconv.FormatUint(uint64(region.Id), 10), string(jsonData)).Err
        if err != nil {
            log.Fatal(err)
        }

        err = redisCli.Cmd("GEOADD", "regions", region.Longitude, region.Latitude, region.Id).Err
        if err != nil {
            log.Fatal(err)
	}

        currentRegionIdx++

        if currentRegionIdx % 500 == 0 {
            log.Printf("[%d/%d] Regions processed\n", currentRegionIdx, regionsCount)
        }
    }
    log.Printf("[%d/%d] Regions processed\n", regionsCount, regionsCount)
}

func migrateCashpoints(cpDb *sql.DB, redisCli *redis.Client) {
    var cashpointsCount int
    err := cpDb.QueryRow(`SELECT COUNT(*) FROM cashpoints`).Scan(&cashpointsCount)
    if err != nil {
        log.Fatalf("migrate: cashpoints: %v\n", err)
    }

    rows, err := cpDb.Query(`SELECT id, type, bank_id, town_id,
                                    longitude, latitude,
                                    address, address_comment,
                                    metro_name, free_access,
                                    main_office, without_weekend,
                                    round_the_clock, works_as_shop,
                                    schedule_general, tel, additional,
                                    rub, usd, eur, cash_in FROM cashpoints`)
    if err != nil {
        log.Fatalf("migrate: cashpoints: %v\n", err)
    }

    currentCashpointIndex := 1
    for rows.Next() {
        cp := new(CashPoint)
        err = rows.Scan(&cp.Id, &cp.Type, &cp.BankId, &cp.TownId,
                        &cp.Longitude, &cp.Latitude,
                        &cp.Address, &cp.AddressComment,
                        &cp.MetroName, &cp.FreeAccess,
                        &cp.MainOffice, &cp.WithoutWeekend,
                        &cp.RoundTheClock, &cp.WorksAsShop,
                        &cp.Schedule, &cp.Tel, &cp.Additional,
                        &cp.Rub, &cp.Usd, &cp.Eur, &cp.CashIn)
        if err != nil {
            log.Fatal(err)
        }

        cashpointIdStr := strconv.FormatUint(uint64(cp.Id), 10)
        townIdStr := strconv.FormatUint(uint64(cp.TownId), 10)
        bankIdStr := strconv.FormatUint(uint64(cp.BankId), 10)

        jsonData, err := json.Marshal(cp)
        if err != nil {
            log.Fatal(err)
        }

        err = redisCli.Cmd("SET", "cp:" + cashpointIdStr, string(jsonData)).Err

        if err != nil {
            log.Fatal(err)
        }

        err = redisCli.Cmd("GEOADD", "cashpoints", cp.Longitude, cp.Latitude, cp.Id).Err
        if err != nil {
            log.Fatal(err)
        }

        err = redisCli.Cmd("SADD", "town:" + townIdStr + ":cp", cp.Id).Err
        if err != nil {
            log.Fatal(err)
        }

        err = redisCli.Cmd("SADD", "bank:" + bankIdStr + ":cp", cp.Id).Err
        if err != nil {
            log.Fatal(err)
        }

        currentCashpointIndex++

        if currentCashpointIndex % 500 == 0 {
            log.Printf("[%d/%d] Cashpoints processed\n", currentCashpointIndex, cashpointsCount)
        }
    }
    log.Printf("[%d/%d] Cashpoints processed\n", cashpointsCount, cashpointsCount)
}

func migrate(townsDb *sql.DB, cpDb *sql.DB, redisCli *redis.Client) {
    migrateTowns(townsDb, redisCli)
    migrateRegions(townsDb, redisCli)
    migrateCashpoints(cpDb, redisCli)
}

func main() {
    args := os.Args[1:]

    if len(args) == 0 {
        log.Fatal("Towns db file path is not specified")
    }

    if len(args) == 1 {
        log.Fatal("Cashpoints db file path is not specified")
    }

    if len(args) == 2 {
        log.Fatal("Redis database url is not specified")
    }

    townsDbPath := args[0]
    cashpointsDbPath := args[1]
    redisUrl := args[2]

    townsDb, err := sql.Open("sqlite3", townsDbPath)
    if err != nil {
        log.Fatal(err)
    }
    defer townsDb.Close()

    cashpointsDb, err := sql.Open("sqlite3", cashpointsDbPath)
    if err != nil {
        log.Fatal(err)
    }
    defer cashpointsDb.Close()

    redisCli, err := redis.Dial("tcp", redisUrl)
    if err != nil {
        log.Fatal(err)
    }
    defer redisCli.Close()

    migrate(townsDb, cashpointsDb, redisCli)
}
<commit_msg>[C|Server] Mirgation redis: added bank:{id} keys and banks SET<commit_after>package main

import (
    "os"
    "log"
    "strconv"
    "database/sql"
    "encoding/json"
    _ "github.com/mattn/go-sqlite3"
    "github.com/mediocregopher/radix.v2/redis"
)

func boolToInt(val bool) uint {
    if val {
        return 1
    }
    return 0
}

type Town struct {
	Id             uint32  `json:"id"`
	Name           string  `json:"name"`
	NameTr         string  `json:"name_tr"`
	RegionId       *uint32 `json:"region_id"`
	RegionalCenter bool    `json:"regional_center"`
	Latitude       float32 `json:"latitude"`
	Longitude      float32 `json:"longitude"`
	Zoom           uint32  `json:"zoom"`
}

type Region struct {
	Id        uint32  `json:"id"`
	Name      string  `json:"name"`
	NameTr    string  `json:"name_tr"`
	Latitude  float32 `json:"latitude"`
	Longitude float32 `json:"longitude"`
	Zoom      uint32  `json:"zoom"`
}

type Bank struct {
	Id        uint32 `json:"id"`
	Name      string `json:"name"`
	NameTr    string `json:"name_tr"`
	NameTrAlt string `json:"name_tr_alt"`
	Town      string `json:"town"`
	Licence   uint32 `json:"licence"`
	Rating    uint32 `json:"rating"`
	Tel       string `json:"tel"`
}

type CashPoint struct {
	Id             uint32  `json:"id"`
	Type           string  `json:"type"`
	BankId         uint32  `json:"bank_id"`
	TownId         uint32  `json:"town_id"`
	Longitude      float32 `json:"longitude"`
	Latitude       float32 `json:"latitude"`
	Address        string  `json:"address"`
	AddressComment string  `json:"address_comment"`
	MetroName      string  `json:"metro_name"`
	FreeAccess     bool    `json:"free_access"`
	MainOffice     bool    `json:"main_office"`
	WithoutWeekend bool    `json:"without_weekend"`
	RoundTheClock  bool    `json:"round_the_clock"`
	WorksAsShop    bool    `json:"works_as_shop"`
	Schedule       string  `json:"schedule"`
	Tel            string  `json:"tel"`
	Additional     string  `json:"additional"`
	Rub            bool    `json:"rub"`
	Usd            bool    `json:"usd"`
	Eur            bool    `json:"eur"`
	CashIn         bool    `json:"cash_in"`
}

func migrateTowns(townsDb *sql.DB, redisCli *redis.Client) {
    var townsCount int
    err := townsDb.QueryRow(`SELECT COUNT(*) FROM towns`).Scan(&townsCount)
    if err != nil {
        log.Fatalf("migrate: towns: %v\n", err)
    }

    rows, err := townsDb.Query(`SELECT id, name, name_tr, region_id,
                                       regional_center, latitude,
                                       longitude, zoom FROM towns`)
    if err != nil {
        log.Fatalf("migrate: towns: %v\n", err)
    }

    currentTownIdx := 1
    for rows.Next() {
        town := new(Town)
        var regionId uint32 = 0
        err = rows.Scan(&town.Id, &town.Name, &town.NameTr,
                        &regionId, &town.RegionalCenter,
                        &town.Latitude, &town.Longitude, &town.Zoom)
        if err != nil {
            log.Fatal(err)
        }

        if regionId != 0 {
            town.RegionId = new(uint32)
            *town.RegionId = regionId
        }

        jsonData, err := json.Marshal(town)
        if err != nil {
            log.Fatal(err)
        }

        err = redisCli.Cmd("SET", "town:" + strconv.FormatUint(uint64(town.Id), 10), string(jsonData)).Err
        if err != nil {
            log.Fatal(err)
        }

        err = redisCli.Cmd("GEOADD", "towns", town.Longitude, town.Latitude, town.Id).Err
        if err != nil {
            log.Fatal(err)
        }

        currentTownIdx++

        if currentTownIdx % 500 == 0 {
            log.Printf("[%d/%d] Towns processed\n", currentTownIdx, townsCount)
        }
    }
    log.Printf("[%d/%d] Towns processed\n", townsCount, townsCount)
}

func migrateRegions(townsDb *sql.DB, redisCli *redis.Client) {
    var regionsCount int
    err := townsDb.QueryRow(`SELECT COUNT(*) FROM regions`).Scan(&regionsCount)
    if err != nil {
        log.Fatalf("migrate: regions: %v\n", err)
    }

    rows, err := townsDb.Query(`SELECT id, name, name_tr,
                                       latitude, longitude, zoom FROM regions`)
    if err != nil {
        log.Fatalf("migrate: regions: %v\n", err)
    }

    currentRegionIdx := 1
    for rows.Next() {
        region := new(Region)
        err = rows.Scan(&region.Id, &region.Name, &region.NameTr,
                        &region.Latitude, &region.Longitude, &region.Zoom)
        if err != nil {
            log.Fatal(err)
        }

        jsonData, err := json.Marshal(region)
        if err != nil {
            log.Fatal(err)
        }

        err = redisCli.Cmd("SET", "region:" + strconv.FormatUint(uint64(region.Id), 10), string(jsonData)).Err
        if err != nil {
            log.Fatal(err)
        }

        err = redisCli.Cmd("GEOADD", "regions", region.Longitude, region.Latitude, region.Id).Err
        if err != nil {
            log.Fatal(err)
	}

        currentRegionIdx++

        if currentRegionIdx % 500 == 0 {
            log.Printf("[%d/%d] Regions processed\n", currentRegionIdx, regionsCount)
        }
    }
    log.Printf("[%d/%d] Regions processed\n", regionsCount, regionsCount)
}

func migrateBanks(banksDb *sql.DB, redisCli *redis.Client) {
    var banksCount int
    err := banksDb.QueryRow(`SELECT COUNT(*) FROM banks`).Scan(&banksCount)
    if err != nil {
        log.Fatalf("migrate: banks: %v", err)
    }

    rows, err := banksDb.Query(`SELECT id, name, name_tr, name_tr_alt, town,
                                       licence, rating, tel FROM banks`)
    if err != nil {
        log.Fatalf("migrate: banks: %v", err)
    }

    currentBankIdx := 1
    for rows.Next() {
        bank := new(Bank)
        var nameTr sql.NullString
        err = rows.Scan(&bank.Id, &bank.Name, &nameTr, &bank.NameTrAlt,
                        &bank.Town, &bank.Licence, &bank.Rating, &bank.Tel)
        if err != nil {
            log.Fatal(err)
        }

        if nameTr.Valid {
            bank.NameTr = nameTr.String
        } else {
            bank.NameTr = ""
        }

        jsonData, err := json.Marshal(bank)
        if err != nil {
            log.Fatal(err)
        }

        err = redisCli.Cmd("SET", "bank:" + strconv.FormatUint(uint64(bank.Id), 10), string(jsonData)).Err
        if err != nil {
            log.Fatal(err)
        }

        err = redisCli.Cmd("SADD", "banks", bank.Id).Err
        if err != nil {
            log.Fatal(err)
        }

        currentBankIdx++

        if currentBankIdx % 100 == 0 {
            log.Printf("[%d/%d] Banks processed\n", currentBankIdx, banksCount)
        }
    }
    log.Printf("[%d/%d] Banks processed\n", banksCount, banksCount)
}

func migrateCashpoints(cpDb *sql.DB, redisCli *redis.Client) {
    var cashpointsCount int
    err := cpDb.QueryRow(`SELECT COUNT(*) FROM cashpoints`).Scan(&cashpointsCount)
    if err != nil {
        log.Fatalf("migrate: cashpoints: %v\n", err)
    }

    rows, err := cpDb.Query(`SELECT id, type, bank_id, town_id,
                                    longitude, latitude,
                                    address, address_comment,
                                    metro_name, free_access,
                                    main_office, without_weekend,
                                    round_the_clock, works_as_shop,
                                    schedule_general, tel, additional,
                                    rub, usd, eur, cash_in FROM cashpoints`)
    if err != nil {
        log.Fatalf("migrate: cashpoints: %v\n", err)
    }

    currentCashpointIndex := 1
    for rows.Next() {
        cp := new(CashPoint)
        err = rows.Scan(&cp.Id, &cp.Type, &cp.BankId, &cp.TownId,
                        &cp.Longitude, &cp.Latitude,
                        &cp.Address, &cp.AddressComment,
                        &cp.MetroName, &cp.FreeAccess,
                        &cp.MainOffice, &cp.WithoutWeekend,
                        &cp.RoundTheClock, &cp.WorksAsShop,
                        &cp.Schedule, &cp.Tel, &cp.Additional,
                        &cp.Rub, &cp.Usd, &cp.Eur, &cp.CashIn)
        if err != nil {
            log.Fatal(err)
        }

        cashpointIdStr := strconv.FormatUint(uint64(cp.Id), 10)
        townIdStr := strconv.FormatUint(uint64(cp.TownId), 10)
        bankIdStr := strconv.FormatUint(uint64(cp.BankId), 10)

        jsonData, err := json.Marshal(cp)
        if err != nil {
            log.Fatal(err)
        }

        err = redisCli.Cmd("SET", "cp:" + cashpointIdStr, string(jsonData)).Err

        if err != nil {
            log.Fatal(err)
        }

        err = redisCli.Cmd("GEOADD", "cashpoints", cp.Longitude, cp.Latitude, cp.Id).Err
        if err != nil {
            log.Fatal(err)
        }

        err = redisCli.Cmd("SADD", "town:" + townIdStr + ":cp", cp.Id).Err
        if err != nil {
            log.Fatal(err)
        }

        err = redisCli.Cmd("SADD", "bank:" + bankIdStr + ":cp", cp.Id).Err
        if err != nil {
            log.Fatal(err)
        }

        currentCashpointIndex++

        if currentCashpointIndex % 500 == 0 {
            log.Printf("[%d/%d] Cashpoints processed\n", currentCashpointIndex, cashpointsCount)
        }
    }
    log.Printf("[%d/%d] Cashpoints processed\n", cashpointsCount, cashpointsCount)
}

func migrate(townsDb *sql.DB, cpDb *sql.DB, banksDb *sql.DB, redisCli *redis.Client) {
    migrateTowns(townsDb, redisCli)
    migrateRegions(townsDb, redisCli)
    migrateCashpoints(cpDb, redisCli)
    migrateBanks(banksDb, redisCli)
}

func main() {
    args := os.Args[1:]

    if len(args) == 0 {
        log.Fatal("Towns db file path is not specified")
    }

    if len(args) == 1 {
        log.Fatal("Cashpoints db file path is not specified")
    }

    if len(args) == 2 {
        log.Fatal("Banks db file path is not specified")
    }

    if len(args) == 3 {
        log.Fatal("Redis database url is not specified")
    }

    townsDbPath := args[0]
    cashpointsDbPath := args[1]
    banksDbPath := args[2]
    redisUrl := args[3]

    townsDb, err := sql.Open("sqlite3", townsDbPath)
    if err != nil {
        log.Fatal(err)
    }
    defer townsDb.Close()

    cashpointsDb, err := sql.Open("sqlite3", cashpointsDbPath)
    if err != nil {
        log.Fatal(err)
    }
    defer cashpointsDb.Close()

    banksDb, err := sql.Open("sqlite3", banksDbPath)
    if err != nil {
        log.Fatal(err)
    }
    defer banksDb.Close()

    redisCli, err := redis.Dial("tcp", redisUrl)
    if err != nil {
        log.Fatal(err)
    }
    defer redisCli.Close()

    migrate(townsDb, cashpointsDb, banksDb, redisCli)
}
<|endoftext|>


<commit_before>//  Copyright (c) 2018 Rackspace
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
//  implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

package indexdb

import (
	"encoding/json"
	"flag"
	"io/ioutil"
	"net"
	"net/http"
	"net/http/httptest"
	"net/url"
	"strconv"
	"testing"
	"time"

	"go.uber.org/zap"

	"github.com/stretchr/testify/require"
	"github.com/troubling/hummingbird/common/conf"
	"github.com/troubling/hummingbird/common/ring"
	"github.com/troubling/hummingbird/common/test"
	"github.com/troubling/hummingbird/objectserver"
)

func TestEcEngineConstructor(t *testing.T) {
	configString := "[app:object-server]\nmount_check=false\n"
	pol := conf.Policy{Index: 0, Type: "hec", Name: "gold",
		Aliases: []string{}, Default: true, Deprecated: false,
		Config: map[string]string{"policy_type": "hec",
			"default": "yes", "name": "gold", "data_frags": "2",
			"parity_frags": "1"}}
	config, _ := conf.StringConfig(configString)
	_, err := ecEngineConstructor(
		config, &pol, &flag.FlagSet{})
	require.Nil(t, err)
}

func getTestEce() (*ecEngine, error) {
	driveRoot, err := ioutil.TempDir("", "")
	if err != nil {
		return nil, err
	}
	policy := conf.Policy{Index: 0, Type: "hec", Name: "gold",
		Aliases: []string{}, Default: true, Deprecated: false,
		Config: map[string]string{"policy_type": "hec",
			"default": "yes", "name": "gold", "data_frags": "2",
			"parity_frags": "1"}}
	testRing := &test.FakeRing{}
	transport := &http.Transport{
		MaxIdleConnsPerHost: 256,
		MaxIdleConns:        0,
		IdleConnTimeout:     5 * time.Second,
		DisableCompression:  true,
		Dial: (&net.Dialer{
			Timeout:   10 * time.Second,
			KeepAlive: 5 * time.Second,
		}).Dial,
		ExpectContinueTimeout: 10 * time.Minute,
	}
	logger, _ := zap.NewProduction()
	engine := &ecEngine{
		driveRoot:      driveRoot,
		hashPathPrefix: "a",
		hashPathSuffix: "b",
		reserve:        0,
		policy:         0,
		logger:         logger,
		ring:           testRing,
		idbs:           map[string]*IndexDB{},
		client: &http.Client{
			Timeout:   time.Second,
			Transport: transport,
		},
	}
	if engine.dataFrags, err = strconv.Atoi(policy.Config["data_frags"]); err != nil {
		return nil, err
	}
	if engine.parityFrags, err = strconv.Atoi(policy.Config["parity_frags"]); err != nil {
		return nil, err
	}
	if engine.chunkSize, err = strconv.Atoi(policy.Config["chunk_size"]); err != nil {
		engine.chunkSize = 1 << 20
	}
	return engine, nil
}

func TestGetObjectsToReplicate(t *testing.T) {
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		remoteItems := []*IndexDBItem{}
		d, err := json.Marshal(remoteItems)
		require.Nil(t, err)
		w.WriteHeader(200)
		w.Write(d)
	}))
	u, err := url.Parse(ts.URL)
	require.Nil(t, err)
	host, ports, err := net.SplitHostPort(u.Host)
	require.Nil(t, err)
	port, err := strconv.Atoi(ports)
	require.Nil(t, err)

	ece, err := getTestEce()
	require.Nil(t, err)
	idb, err := ece.getDB("sdb1")
	require.Nil(t, err)

	timestamp := time.Now().UnixNano()
	body := "just testing"
	hsh0 := md5hash("object0")
	f, err := idb.TempFile(hsh0, 0, timestamp, int64(len(body)), true)
	require.Nil(t, err)
	f.Write([]byte(body))
	metadata, err := json.Marshal(map[string]string{"moo": "cow"})
	require.Nil(t, err)
	require.Nil(t, idb.Commit(f, hsh0, 0, timestamp, false, "", metadata, false, ""))
	osc := make(chan objectserver.ObjectStabilizer)
	cancel := make(chan struct{})
	defer close(cancel)
	go ece.GetObjectsToReplicate(
		objectserver.PriorityRepJob{FromDevice: &ring.Device{Device: "sdb1"},
			ToDevice: &ring.Device{Device: "sdb2", Scheme: "http", Port: port, Ip: host}}, osc, cancel)
	os := <-osc
	require.Equal(t, "cow", os.Metadata()["moo"])
	os = <-osc
	require.Nil(t, os)
}

func TestGetObjectsToReplicateRemoteHasAll(t *testing.T) {
	ece, err := getTestEce()
	require.Nil(t, err)
	idb, err := ece.getDB("sdb1")
	require.Nil(t, err)

	timestamp := time.Now().UnixNano()
	body := "just testing"
	hsh0 := "00000000000000000000000000000001"
	f, err := idb.TempFile(hsh0, 0, timestamp, int64(len(body)), true)
	require.Nil(t, err)
	f.Write([]byte(body))
	metadata, err := json.Marshal(map[string]string{"name": "o1"})
	require.Nil(t, err)
	require.Nil(t, idb.Commit(f, hsh0, 0, timestamp, false, "", metadata, false, ""))
	hsh1 := "00000000000000000000000000000002"
	metadata, err = json.Marshal(map[string]string{"name": "o2"})
	require.Nil(t, err)
	f, err = idb.TempFile(hsh1, 0, timestamp, int64(len(body)), true)
	require.Nil(t, err)
	f.Write([]byte(body))
	require.Nil(t, idb.Commit(f, hsh1, 0, timestamp, false, "", metadata, false, ""))
	remoteItems, err := idb.List("", "", "", 0)
	require.Nil(t, err)
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		d, err := json.Marshal(remoteItems)
		require.Nil(t, err)
		w.WriteHeader(200)
		w.Write(d)
	}))
	u, err := url.Parse(ts.URL)
	require.Nil(t, err)
	host, ports, err := net.SplitHostPort(u.Host)
	require.Nil(t, err)
	port, err := strconv.Atoi(ports)
	require.Nil(t, err)

	osc := make(chan objectserver.ObjectStabilizer)
	cancel := make(chan struct{})
	defer close(cancel)
	go ece.GetObjectsToReplicate(
		objectserver.PriorityRepJob{FromDevice: &ring.Device{Device: "sdb1"},
			ToDevice: &ring.Device{Device: "sdb2", Scheme: "http", Port: port, Ip: host}}, osc, cancel)
	os := <-osc
	require.Nil(t, os)
}

func TestGetObjectsToReplicateRemoteHasSome(t *testing.T) {
	ece, err := getTestEce()
	require.Nil(t, err)
	idb, err := ece.getDB("sdb1")
	require.Nil(t, err)

	timestamp := time.Now().UnixNano()
	body := "just testing"
	hsh0 := "00000000000000000000000000000001"
	f, err := idb.TempFile(hsh0, 0, timestamp, int64(len(body)), true)
	require.Nil(t, err)
	f.Write([]byte(body))
	metadata, err := json.Marshal(map[string]string{"name": "o1"})
	require.Nil(t, err)
	require.Nil(t, idb.Commit(f, hsh0, 0, timestamp, false, "", metadata, false, ""))
	hsh1 := "00000000000000000000000000000002"
	metadata, err = json.Marshal(map[string]string{"name": "o2"})
	require.Nil(t, err)
	f, err = idb.TempFile(hsh1, 0, timestamp, int64(len(body)), true)
	require.Nil(t, err)
	f.Write([]byte(body))
	require.Nil(t, idb.Commit(f, hsh1, 0, timestamp, false, "", metadata, false, ""))
	remoteItems, err := idb.List("", "", "", 0)
	require.Nil(t, err)
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		newRemoteItems := []*IndexDBItem{remoteItems[0]}
		d, err := json.Marshal(newRemoteItems)
		require.Nil(t, err)
		w.WriteHeader(200)
		w.Write(d)
	}))
	u, err := url.Parse(ts.URL)
	require.Nil(t, err)
	host, ports, err := net.SplitHostPort(u.Host)
	require.Nil(t, err)
	port, err := strconv.Atoi(ports)
	require.Nil(t, err)

	osc := make(chan objectserver.ObjectStabilizer)
	cancel := make(chan struct{})
	defer close(cancel)
	go ece.GetObjectsToReplicate(
		objectserver.PriorityRepJob{FromDevice: &ring.Device{Device: "sdb1"},
			ToDevice: &ring.Device{Device: "sdb2", Scheme: "http", Port: port, Ip: host}}, osc, cancel)
	os := <-osc
	require.Equal(t, "o2", os.Metadata()["name"])
	os = <-osc
	require.Nil(t, os)
}
<commit_msg>fix test<commit_after>//  Copyright (c) 2018 Rackspace
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
//  implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

package indexdb

import (
	"encoding/json"
	"flag"
	"io/ioutil"
	"net"
	"net/http"
	"net/http/httptest"
	"net/url"
	"strconv"
	"testing"
	"time"

	"go.uber.org/zap"

	"github.com/stretchr/testify/require"
	"github.com/troubling/hummingbird/common/conf"
	"github.com/troubling/hummingbird/common/ring"
	"github.com/troubling/hummingbird/common/test"
	"github.com/troubling/hummingbird/objectserver"
)

func TestEcEngineConstructor(t *testing.T) {
	configString := "[app:object-server]\nmount_check=false\n"
	pol := conf.Policy{Index: 0, Type: "hec", Name: "gold",
		Aliases: []string{}, Default: true, Deprecated: false,
		Config: map[string]string{"policy_type": "hec",
			"default": "yes", "name": "gold", "data_frags": "2",
			"parity_frags": "1"}}
	config, _ := conf.StringConfig(configString)
	_, err := ecEngineConstructor(
		config, &pol, &flag.FlagSet{})
	require.Nil(t, err)
}

func getTestEce() (*ecEngine, error) {
	driveRoot, err := ioutil.TempDir("", "")
	if err != nil {
		return nil, err
	}
	policy := conf.Policy{Index: 0, Type: "hec", Name: "gold",
		Aliases: []string{}, Default: true, Deprecated: false,
		Config: map[string]string{"policy_type": "hec",
			"default": "yes", "name": "gold", "data_frags": "2",
			"parity_frags": "1"}}
	testRing := &test.FakeRing{}
	transport := &http.Transport{
		MaxIdleConnsPerHost: 256,
		MaxIdleConns:        0,
		IdleConnTimeout:     5 * time.Second,
		DisableCompression:  true,
		Dial: (&net.Dialer{
			Timeout:   10 * time.Second,
			KeepAlive: 5 * time.Second,
		}).Dial,
		ExpectContinueTimeout: 10 * time.Minute,
	}
	logger, _ := zap.NewProduction()
	engine := &ecEngine{
		driveRoot:      driveRoot,
		hashPathPrefix: "a",
		hashPathSuffix: "b",
		reserve:        0,
		policy:         0,
		logger:         logger,
		ring:           testRing,
		idbs:           map[string]*IndexDB{},
		dbPartPower:    1,
		numSubDirs:     32,
		client: &http.Client{
			Timeout:   time.Second,
			Transport: transport,
		},
	}
	if engine.dataFrags, err = strconv.Atoi(policy.Config["data_frags"]); err != nil {
		return nil, err
	}
	if engine.parityFrags, err = strconv.Atoi(policy.Config["parity_frags"]); err != nil {
		return nil, err
	}
	if engine.chunkSize, err = strconv.Atoi(policy.Config["chunk_size"]); err != nil {
		engine.chunkSize = 1 << 20
	}
	return engine, nil
}

func TestGetObjectsToReplicate(t *testing.T) {
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		remoteItems := []*IndexDBItem{}
		d, err := json.Marshal(remoteItems)
		require.Nil(t, err)
		w.WriteHeader(200)
		w.Write(d)
	}))
	u, err := url.Parse(ts.URL)
	require.Nil(t, err)
	host, ports, err := net.SplitHostPort(u.Host)
	require.Nil(t, err)
	port, err := strconv.Atoi(ports)
	require.Nil(t, err)

	ece, err := getTestEce()
	require.Nil(t, err)
	idb, err := ece.getDB("sdb1")
	require.Nil(t, err)

	timestamp := time.Now().UnixNano()
	body := "just testing"
	hsh0 := md5hash("object0")
	f, err := idb.TempFile(hsh0, 0, timestamp, int64(len(body)), true)
	require.Nil(t, err)
	f.Write([]byte(body))
	metadata, err := json.Marshal(map[string]string{"moo": "cow"})
	require.Nil(t, err)
	require.Nil(t, idb.Commit(f, hsh0, 0, timestamp, false, "", metadata, false, ""))
	osc := make(chan objectserver.ObjectStabilizer)
	cancel := make(chan struct{})
	defer close(cancel)
	go ece.GetObjectsToReplicate(
		objectserver.PriorityRepJob{FromDevice: &ring.Device{Device: "sdb1"},
			ToDevice: &ring.Device{Device: "sdb2", Scheme: "http", Port: port, Ip: host}}, osc, cancel)
	os := <-osc
	require.Equal(t, "cow", os.Metadata()["moo"])
	os = <-osc
	require.Nil(t, os)
}

func TestGetObjectsToReplicateRemoteHasAll(t *testing.T) {
	ece, err := getTestEce()
	require.Nil(t, err)
	idb, err := ece.getDB("sdb1")
	require.Nil(t, err)

	timestamp := time.Now().UnixNano()
	body := "just testing"
	hsh0 := "00000000000000000000000000000001"
	f, err := idb.TempFile(hsh0, 0, timestamp, int64(len(body)), true)
	require.Nil(t, err)
	f.Write([]byte(body))
	metadata, err := json.Marshal(map[string]string{"name": "o1"})
	require.Nil(t, err)
	require.Nil(t, idb.Commit(f, hsh0, 0, timestamp, false, "", metadata, false, ""))
	hsh1 := "00000000000000000000000000000002"
	metadata, err = json.Marshal(map[string]string{"name": "o2"})
	require.Nil(t, err)
	f, err = idb.TempFile(hsh1, 0, timestamp, int64(len(body)), true)
	require.Nil(t, err)
	f.Write([]byte(body))
	require.Nil(t, idb.Commit(f, hsh1, 0, timestamp, false, "", metadata, false, ""))
	remoteItems, err := idb.List("", "", "", 0)
	require.Nil(t, err)
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		d, err := json.Marshal(remoteItems)
		require.Nil(t, err)
		w.WriteHeader(200)
		w.Write(d)
	}))
	u, err := url.Parse(ts.URL)
	require.Nil(t, err)
	host, ports, err := net.SplitHostPort(u.Host)
	require.Nil(t, err)
	port, err := strconv.Atoi(ports)
	require.Nil(t, err)

	osc := make(chan objectserver.ObjectStabilizer)
	cancel := make(chan struct{})
	defer close(cancel)
	go ece.GetObjectsToReplicate(
		objectserver.PriorityRepJob{FromDevice: &ring.Device{Device: "sdb1"},
			ToDevice: &ring.Device{Device: "sdb2", Scheme: "http", Port: port, Ip: host}}, osc, cancel)
	os := <-osc
	require.Nil(t, os)
}

func TestGetObjectsToReplicateRemoteHasSome(t *testing.T) {
	ece, err := getTestEce()
	require.Nil(t, err)
	idb, err := ece.getDB("sdb1")
	require.Nil(t, err)

	timestamp := time.Now().UnixNano()
	body := "just testing"
	hsh0 := "00000000000000000000000000000001"
	f, err := idb.TempFile(hsh0, 0, timestamp, int64(len(body)), true)
	require.Nil(t, err)
	f.Write([]byte(body))
	metadata, err := json.Marshal(map[string]string{"name": "o1"})
	require.Nil(t, err)
	require.Nil(t, idb.Commit(f, hsh0, 0, timestamp, false, "", metadata, false, ""))
	hsh1 := "00000000000000000000000000000002"
	metadata, err = json.Marshal(map[string]string{"name": "o2"})
	require.Nil(t, err)
	f, err = idb.TempFile(hsh1, 0, timestamp, int64(len(body)), true)
	require.Nil(t, err)
	f.Write([]byte(body))
	require.Nil(t, idb.Commit(f, hsh1, 0, timestamp, false, "", metadata, false, ""))
	remoteItems, err := idb.List("", "", "", 0)
	require.Nil(t, err)
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		newRemoteItems := []*IndexDBItem{remoteItems[0]}
		d, err := json.Marshal(newRemoteItems)
		require.Nil(t, err)
		w.WriteHeader(200)
		w.Write(d)
	}))
	u, err := url.Parse(ts.URL)
	require.Nil(t, err)
	host, ports, err := net.SplitHostPort(u.Host)
	require.Nil(t, err)
	port, err := strconv.Atoi(ports)
	require.Nil(t, err)

	osc := make(chan objectserver.ObjectStabilizer)
	cancel := make(chan struct{})
	defer close(cancel)
	go ece.GetObjectsToReplicate(
		objectserver.PriorityRepJob{FromDevice: &ring.Device{Device: "sdb1"},
			ToDevice: &ring.Device{Device: "sdb2", Scheme: "http", Port: port, Ip: host}}, osc, cancel)
	os := <-osc
	require.Equal(t, "o2", os.Metadata()["name"])
	os = <-osc
	require.Nil(t, os)
}
<|endoftext|>