<commit_before>package executor_api

import "github.com/cloudfoundry-incubator/runtime-schema/models"

const (
	StateReserved = "reserved"
	StateCreated  = "created"
)

type Container struct {
	Guid string `json:"guid"`

	ExecutorGuid    string  `json:"executor_guid"`
	MemoryMB        int     `json:"memory_mb"`
	DiskMB          int     `json:"disk_mb"`
	CpuPercent      float64 `json:"cpu_percent"`
	FileDescriptors int     `json:"file_descriptors"`
	State           string  `json:"state"`
	ContainerHandle string  `json:"container_handle"`
}

type ContainerAllocationRequest struct {
	MemoryMB        int     `json:"memory_mb"`
	DiskMB          int     `json:"disk_mb"`
	CpuPercent      float64 `json:"cpu_percent"`
	FileDescriptors int     `json:"file_descriptors"`
}

type ContainerRunRequest struct {
	Actions     []models.ExecutorAction `json:"actions"`
	CompleteURL string                  `json:"complete_url"`
}

type ContainerRunResult struct {
	Failed        bool   `json:"failed"`
	FailureReason string `json:"failure_reason"`
	Result        string `json:"result"`
}
<commit_msg>Add metadata to executor_api resources<commit_after>package executor_api

import "github.com/cloudfoundry-incubator/runtime-schema/models"

const (
	StateReserved = "reserved"
	StateCreated  = "created"
)

type Container struct {
	Guid string `json:"guid"`

	ExecutorGuid    string  `json:"executor_guid"`
	MemoryMB        int     `json:"memory_mb"`
	DiskMB          int     `json:"disk_mb"`
	CpuPercent      float64 `json:"cpu_percent"`
	FileDescriptors int     `json:"file_descriptors"`
	State           string  `json:"state"`
	ContainerHandle string  `json:"container_handle"`
}

type ContainerAllocationRequest struct {
	MemoryMB        int     `json:"memory_mb"`
	DiskMB          int     `json:"disk_mb"`
	CpuPercent      float64 `json:"cpu_percent"`
	FileDescriptors int     `json:"file_descriptors"`
}

type ContainerRunRequest struct {
	Actions     []models.ExecutorAction `json:"actions"`
	Metadata    []byte                  `json:"metadata"`
	CompleteURL string                  `json:"complete_url"`
}

type ContainerRunResult struct {
	Failed        bool   `json:"failed"`
	FailureReason string `json:"failure_reason"`
	Result        string `json:"result"`
	Metadata      []byte `json:"metadata"`
}
<|endoftext|>


<commit_before>/*
Copyright (c) 2017 VMware, Inc. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package simulator

import (
	"errors"
	"fmt"
	"net"
	"strconv"
	"strings"

	"github.com/vmware/govmomi/vim25/methods"
	"github.com/vmware/govmomi/vim25/mo"
	"github.com/vmware/govmomi/vim25/soap"
	"github.com/vmware/govmomi/vim25/types"
)

var ipPool = MustNewIpPool(&types.IpPool{
	Id:   1,
	Name: "ip-pool",
	AvailableIpv4Addresses: 250,
	AvailableIpv6Addresses: 250,
	AllocatedIpv6Addresses: 0,
	AllocatedIpv4Addresses: 0,
	Ipv4Config: &types.IpPoolIpPoolConfigInfo{
		Netmask:       "10.10.10.255",
		Gateway:       "10.10.10.1",
		SubnetAddress: "10.10.10.0",
		Range:         "10.10.10.2#250",
	},
	Ipv6Config: &types.IpPoolIpPoolConfigInfo{
		Netmask:       "2001:4860:0:2001::ff",
		Gateway:       "2001:4860:0:2001::1",
		SubnetAddress: "2001:4860:0:2001::0",
		Range:         "2001:4860:0:2001::2#250",
	},
})

// IpPoolManager implements a simple IP Pool manager in which all pools are shared
// across different datacenters.
type IpPoolManager struct {
	mo.IpPoolManager

	pools      map[int32]*IpPool
	nextPoolId int32
}

func NewIpPoolManager(ref types.ManagedObjectReference) *IpPoolManager {
	m := &IpPoolManager{}
	m.Self = ref

	m.pools = map[int32]*IpPool{
		1: ipPool,
	}
	m.nextPoolId = 2

	return m
}

func (m *IpPoolManager) CreateIpPool(req *types.CreateIpPool) soap.HasFault {
	body := &methods.CreateIpPoolBody{}
	id := m.nextPoolId

	var err error
	m.pools[id], err = NewIpPool(&req.Pool)
	if err != nil {
		body.Fault_ = Fault("", &types.RuntimeFault{})
		return body
	}

	m.nextPoolId++

	body.Res = &types.CreateIpPoolResponse{
		Returnval: id,
	}

	return body
}

func (m *IpPoolManager) DestroyIpPool(req *types.DestroyIpPool) soap.HasFault {
	delete(m.pools, req.Id)

	return &methods.DestroyIpPoolBody{
		Res: &types.DestroyIpPoolResponse{},
	}
}

func (m *IpPoolManager) QueryIpPools(req *types.QueryIpPools) soap.HasFault {
	pools := []types.IpPool{}

	for _, pool := range m.pools {
		pools = append(pools, *pool.config)
	}

	return &methods.QueryIpPoolsBody{
		Res: &types.QueryIpPoolsResponse{
			Returnval: pools,
		},
	}
}

func (m *IpPoolManager) UpdateIpPool(req *types.UpdateIpPool) soap.HasFault {
	body := &methods.UpdateIpPoolBody{}

	var pool *IpPool
	var err error
	var ok bool

	if pool, ok = m.pools[req.Pool.Id]; !ok {
		body.Fault_ = Fault("", &types.NotFoundFault{})
		return body
	}

	if pool.config.AllocatedIpv4Addresses+pool.config.AllocatedIpv6Addresses != 0 {
		body.Fault_ = Fault("update a pool has been used is not supported", &types.RuntimeFault{})
		return body
	}

	m.pools[req.Pool.Id], err = NewIpPool(&req.Pool)
	if err != nil {
		body.Fault_ = Fault(err.Error(), &types.RuntimeFault{})
		return body
	}

	body.Res = &types.UpdateIpPoolResponse{}

	return body
}

func (m *IpPoolManager) AllocateIpv4Address(req *types.AllocateIpv4Address) soap.HasFault {
	body := &methods.AllocateIpv4AddressBody{}

	pool, ok := m.pools[req.PoolId]
	if !ok {
		body.Fault_ = Fault("", &types.InvalidArgument{})
		return body
	}

	ip, err := pool.AllocateIPv4(req.AllocationId)
	if err != nil {
		body.Fault_ = Fault(err.Error(), &types.RuntimeFault{})
		return body
	}

	body.Res = &types.AllocateIpv4AddressResponse{
		Returnval: ip,
	}

	return body
}

func (m *IpPoolManager) AllocateIpv6Address(req *types.AllocateIpv6Address) soap.HasFault {
	body := &methods.AllocateIpv6AddressBody{}

	pool, ok := m.pools[req.PoolId]
	if !ok {
		body.Fault_ = Fault("", &types.InvalidArgument{})
		return body
	}

	ip, err := pool.AllocateIpv6(req.AllocationId)
	if err != nil {
		body.Fault_ = Fault(err.Error(), &types.RuntimeFault{})
		return body
	}

	body.Res = &types.AllocateIpv6AddressResponse{
		Returnval: ip,
	}

	return body
}

func (m *IpPoolManager) ReleaseIpAllocation(req *types.ReleaseIpAllocation) soap.HasFault {
	body := &methods.ReleaseIpAllocationBody{}

	pool, ok := m.pools[req.PoolId]
	if !ok {
		body.Fault_ = Fault("", &types.InvalidArgument{})
		return body
	}

	pool.ReleaseIpv4(req.AllocationId)
	pool.ReleaseIpv6(req.AllocationId)

	body.Res = &types.ReleaseIpAllocationResponse{}

	return body
}

func (m *IpPoolManager) QueryIPAllocations(req *types.QueryIPAllocations) soap.HasFault {
	body := &methods.QueryIPAllocationsBody{}

	pool, ok := m.pools[req.PoolId]
	if !ok {
		body.Fault_ = Fault("", &types.InvalidArgument{})
		return body
	}

	body.Res = &types.QueryIPAllocationsResponse{}

	ipv4, ok := pool.ipv4Allocation[req.ExtensionKey]
	if ok {
		body.Res.Returnval = append(body.Res.Returnval, types.IpPoolManagerIpAllocation{
			IpAddress:    ipv4,
			AllocationId: req.ExtensionKey,
		})
	}

	ipv6, ok := pool.ipv6Allocation[req.ExtensionKey]
	if ok {
		body.Res.Returnval = append(body.Res.Returnval, types.IpPoolManagerIpAllocation{
			IpAddress:    ipv6,
			AllocationId: req.ExtensionKey,
		})
	}

	return body
}

var (
	errNoIpAvailable     = errors.New("no ip address available")
	errInvalidAllocation = errors.New("allocation id not recognized")
)

type IpPool struct {
	config         *types.IpPool
	ipv4Allocation map[string]string
	ipv6Allocation map[string]string
	ipv4Pool       []string
	ipv6Pool       []string
}

func MustNewIpPool(config *types.IpPool) *IpPool {
	pool, err := NewIpPool(config)
	if err != nil {
		panic(err)
	}

	return pool
}

func NewIpPool(config *types.IpPool) (*IpPool, error) {
	pool := &IpPool{
		config:         config,
		ipv4Allocation: make(map[string]string),
		ipv6Allocation: make(map[string]string),
	}

	return pool, pool.init()
}

func (p *IpPool) init() error {
	// IPv4 range
	if p.config.Ipv4Config != nil {
		ranges := strings.Split(p.config.Ipv4Config.Range, ",")
		for _, r := range ranges {
			sp := strings.Split(r, "#")
			if len(sp) != 2 {
				return fmt.Errorf("format of range should be ip#number; got %q", r)
			}

			ip := net.ParseIP(strings.TrimSpace(sp[0])).To4()
			if ip == nil {
				return fmt.Errorf("bad ip format: %q", sp[0])
			}

			length, err := strconv.Atoi(sp[1])
			if err != nil {
				return err
			}

			for i := 0; i < length; i++ {
				p.ipv4Pool = append(p.ipv4Pool, net.IPv4(ip[0], ip[1], ip[2], ip[3]+byte(i)).String())
			}
		}
	}

	// IPv6 range
	if p.config.Ipv6Config != nil {
		ranges := strings.Split(p.config.Ipv6Config.Range, ",")
		for _, r := range ranges {
			sp := strings.Split(r, "#")
			if len(sp) != 2 {
				return fmt.Errorf("format of range should be ip#number; got %q", r)
			}

			ip := net.ParseIP(strings.TrimSpace(sp[0])).To16()
			if ip == nil {
				return fmt.Errorf("bad ip format: %q", sp[0])
			}

			length, err := strconv.Atoi(sp[1])
			if err != nil {
				return err
			}

			for i := 0; i < length; i++ {
				var ipv6 [16]byte
				copy(ipv6[:], ip)
				ipv6[15] += byte(i)
				p.ipv6Pool = append(p.ipv6Pool, net.IP(ipv6[:]).String())
			}
		}
	}

	return nil
}

func (p *IpPool) AllocateIPv4(allocation string) (string, error) {
	if ip, ok := p.ipv4Allocation[allocation]; ok {
		return ip, nil
	}

	l := len(p.ipv4Pool)
	if l == 0 {
		return "", errNoIpAvailable
	}

	ip := p.ipv4Pool[l-1]

	p.config.AvailableIpv4Addresses--
	p.config.AllocatedIpv4Addresses++
	p.ipv4Pool = p.ipv4Pool[:l-1]
	p.ipv4Allocation[allocation] = ip

	return ip, nil
}

func (p *IpPool) ReleaseIpv4(allocation string) error {
	ip, ok := p.ipv4Allocation[allocation]
	if !ok {
		return errInvalidAllocation
	}

	delete(p.ipv4Allocation, allocation)
	p.config.AvailableIpv4Addresses++
	p.config.AllocatedIpv4Addresses--
	p.ipv4Pool = append(p.ipv4Pool, ip)

	return nil
}

func (p *IpPool) AllocateIpv6(allocation string) (string, error) {
	if ip, ok := p.ipv6Allocation[allocation]; ok {
		return ip, nil
	}

	l := len(p.ipv6Pool)
	if l == 0 {
		return "", errNoIpAvailable
	}

	ip := p.ipv6Pool[l-1]

	p.config.AvailableIpv6Addresses--
	p.config.AllocatedIpv6Addresses++
	p.ipv6Pool = p.ipv6Pool[:l-1]
	p.ipv6Allocation[allocation] = ip

	return ip, nil
}

func (p *IpPool) ReleaseIpv6(allocation string) error {
	ip, ok := p.ipv6Allocation[allocation]
	if !ok {
		return errInvalidAllocation
	}

	delete(p.ipv6Allocation, allocation)
	p.config.AvailableIpv6Addresses++
	p.config.AllocatedIpv6Addresses--
	p.ipv6Pool = append(p.ipv6Pool, ip)

	return nil
}
<commit_msg>vcsim: preserve order in QueryIpPools (#914)<commit_after>/*
Copyright (c) 2017 VMware, Inc. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package simulator

import (
	"errors"
	"fmt"
	"net"
	"strconv"
	"strings"

	"github.com/vmware/govmomi/vim25/methods"
	"github.com/vmware/govmomi/vim25/mo"
	"github.com/vmware/govmomi/vim25/soap"
	"github.com/vmware/govmomi/vim25/types"
)

var ipPool = MustNewIpPool(&types.IpPool{
	Id:   1,
	Name: "ip-pool",
	AvailableIpv4Addresses: 250,
	AvailableIpv6Addresses: 250,
	AllocatedIpv6Addresses: 0,
	AllocatedIpv4Addresses: 0,
	Ipv4Config: &types.IpPoolIpPoolConfigInfo{
		Netmask:       "10.10.10.255",
		Gateway:       "10.10.10.1",
		SubnetAddress: "10.10.10.0",
		Range:         "10.10.10.2#250",
	},
	Ipv6Config: &types.IpPoolIpPoolConfigInfo{
		Netmask:       "2001:4860:0:2001::ff",
		Gateway:       "2001:4860:0:2001::1",
		SubnetAddress: "2001:4860:0:2001::0",
		Range:         "2001:4860:0:2001::2#250",
	},
})

// IpPoolManager implements a simple IP Pool manager in which all pools are shared
// across different datacenters.
type IpPoolManager struct {
	mo.IpPoolManager

	pools      map[int32]*IpPool
	nextPoolId int32
}

func NewIpPoolManager(ref types.ManagedObjectReference) *IpPoolManager {
	m := &IpPoolManager{}
	m.Self = ref

	m.pools = map[int32]*IpPool{
		1: ipPool,
	}
	m.nextPoolId = 2

	return m
}

func (m *IpPoolManager) CreateIpPool(req *types.CreateIpPool) soap.HasFault {
	body := &methods.CreateIpPoolBody{}
	id := m.nextPoolId

	var err error
	m.pools[id], err = NewIpPool(&req.Pool)
	if err != nil {
		body.Fault_ = Fault("", &types.RuntimeFault{})
		return body
	}

	m.nextPoolId++

	body.Res = &types.CreateIpPoolResponse{
		Returnval: id,
	}

	return body
}

func (m *IpPoolManager) DestroyIpPool(req *types.DestroyIpPool) soap.HasFault {
	delete(m.pools, req.Id)

	return &methods.DestroyIpPoolBody{
		Res: &types.DestroyIpPoolResponse{},
	}
}

func (m *IpPoolManager) QueryIpPools(req *types.QueryIpPools) soap.HasFault {
	pools := []types.IpPool{}

	for i := int32(1); i < m.nextPoolId; i++ {
		if p, ok := m.pools[i]; ok {
			pools = append(pools, *p.config)
		}
	}

	return &methods.QueryIpPoolsBody{
		Res: &types.QueryIpPoolsResponse{
			Returnval: pools,
		},
	}
}

func (m *IpPoolManager) UpdateIpPool(req *types.UpdateIpPool) soap.HasFault {
	body := &methods.UpdateIpPoolBody{}

	var pool *IpPool
	var err error
	var ok bool

	if pool, ok = m.pools[req.Pool.Id]; !ok {
		body.Fault_ = Fault("", &types.NotFoundFault{})
		return body
	}

	if pool.config.AllocatedIpv4Addresses+pool.config.AllocatedIpv6Addresses != 0 {
		body.Fault_ = Fault("update a pool has been used is not supported", &types.RuntimeFault{})
		return body
	}

	m.pools[req.Pool.Id], err = NewIpPool(&req.Pool)
	if err != nil {
		body.Fault_ = Fault(err.Error(), &types.RuntimeFault{})
		return body
	}

	body.Res = &types.UpdateIpPoolResponse{}

	return body
}

func (m *IpPoolManager) AllocateIpv4Address(req *types.AllocateIpv4Address) soap.HasFault {
	body := &methods.AllocateIpv4AddressBody{}

	pool, ok := m.pools[req.PoolId]
	if !ok {
		body.Fault_ = Fault("", &types.InvalidArgument{})
		return body
	}

	ip, err := pool.AllocateIPv4(req.AllocationId)
	if err != nil {
		body.Fault_ = Fault(err.Error(), &types.RuntimeFault{})
		return body
	}

	body.Res = &types.AllocateIpv4AddressResponse{
		Returnval: ip,
	}

	return body
}

func (m *IpPoolManager) AllocateIpv6Address(req *types.AllocateIpv6Address) soap.HasFault {
	body := &methods.AllocateIpv6AddressBody{}

	pool, ok := m.pools[req.PoolId]
	if !ok {
		body.Fault_ = Fault("", &types.InvalidArgument{})
		return body
	}

	ip, err := pool.AllocateIpv6(req.AllocationId)
	if err != nil {
		body.Fault_ = Fault(err.Error(), &types.RuntimeFault{})
		return body
	}

	body.Res = &types.AllocateIpv6AddressResponse{
		Returnval: ip,
	}

	return body
}

func (m *IpPoolManager) ReleaseIpAllocation(req *types.ReleaseIpAllocation) soap.HasFault {
	body := &methods.ReleaseIpAllocationBody{}

	pool, ok := m.pools[req.PoolId]
	if !ok {
		body.Fault_ = Fault("", &types.InvalidArgument{})
		return body
	}

	pool.ReleaseIpv4(req.AllocationId)
	pool.ReleaseIpv6(req.AllocationId)

	body.Res = &types.ReleaseIpAllocationResponse{}

	return body
}

func (m *IpPoolManager) QueryIPAllocations(req *types.QueryIPAllocations) soap.HasFault {
	body := &methods.QueryIPAllocationsBody{}

	pool, ok := m.pools[req.PoolId]
	if !ok {
		body.Fault_ = Fault("", &types.InvalidArgument{})
		return body
	}

	body.Res = &types.QueryIPAllocationsResponse{}

	ipv4, ok := pool.ipv4Allocation[req.ExtensionKey]
	if ok {
		body.Res.Returnval = append(body.Res.Returnval, types.IpPoolManagerIpAllocation{
			IpAddress:    ipv4,
			AllocationId: req.ExtensionKey,
		})
	}

	ipv6, ok := pool.ipv6Allocation[req.ExtensionKey]
	if ok {
		body.Res.Returnval = append(body.Res.Returnval, types.IpPoolManagerIpAllocation{
			IpAddress:    ipv6,
			AllocationId: req.ExtensionKey,
		})
	}

	return body
}

var (
	errNoIpAvailable     = errors.New("no ip address available")
	errInvalidAllocation = errors.New("allocation id not recognized")
)

type IpPool struct {
	config         *types.IpPool
	ipv4Allocation map[string]string
	ipv6Allocation map[string]string
	ipv4Pool       []string
	ipv6Pool       []string
}

func MustNewIpPool(config *types.IpPool) *IpPool {
	pool, err := NewIpPool(config)
	if err != nil {
		panic(err)
	}

	return pool
}

func NewIpPool(config *types.IpPool) (*IpPool, error) {
	pool := &IpPool{
		config:         config,
		ipv4Allocation: make(map[string]string),
		ipv6Allocation: make(map[string]string),
	}

	return pool, pool.init()
}

func (p *IpPool) init() error {
	// IPv4 range
	if p.config.Ipv4Config != nil {
		ranges := strings.Split(p.config.Ipv4Config.Range, ",")
		for _, r := range ranges {
			sp := strings.Split(r, "#")
			if len(sp) != 2 {
				return fmt.Errorf("format of range should be ip#number; got %q", r)
			}

			ip := net.ParseIP(strings.TrimSpace(sp[0])).To4()
			if ip == nil {
				return fmt.Errorf("bad ip format: %q", sp[0])
			}

			length, err := strconv.Atoi(sp[1])
			if err != nil {
				return err
			}

			for i := 0; i < length; i++ {
				p.ipv4Pool = append(p.ipv4Pool, net.IPv4(ip[0], ip[1], ip[2], ip[3]+byte(i)).String())
			}
		}
	}

	// IPv6 range
	if p.config.Ipv6Config != nil {
		ranges := strings.Split(p.config.Ipv6Config.Range, ",")
		for _, r := range ranges {
			sp := strings.Split(r, "#")
			if len(sp) != 2 {
				return fmt.Errorf("format of range should be ip#number; got %q", r)
			}

			ip := net.ParseIP(strings.TrimSpace(sp[0])).To16()
			if ip == nil {
				return fmt.Errorf("bad ip format: %q", sp[0])
			}

			length, err := strconv.Atoi(sp[1])
			if err != nil {
				return err
			}

			for i := 0; i < length; i++ {
				var ipv6 [16]byte
				copy(ipv6[:], ip)
				ipv6[15] += byte(i)
				p.ipv6Pool = append(p.ipv6Pool, net.IP(ipv6[:]).String())
			}
		}
	}

	return nil
}

func (p *IpPool) AllocateIPv4(allocation string) (string, error) {
	if ip, ok := p.ipv4Allocation[allocation]; ok {
		return ip, nil
	}

	l := len(p.ipv4Pool)
	if l == 0 {
		return "", errNoIpAvailable
	}

	ip := p.ipv4Pool[l-1]

	p.config.AvailableIpv4Addresses--
	p.config.AllocatedIpv4Addresses++
	p.ipv4Pool = p.ipv4Pool[:l-1]
	p.ipv4Allocation[allocation] = ip

	return ip, nil
}

func (p *IpPool) ReleaseIpv4(allocation string) error {
	ip, ok := p.ipv4Allocation[allocation]
	if !ok {
		return errInvalidAllocation
	}

	delete(p.ipv4Allocation, allocation)
	p.config.AvailableIpv4Addresses++
	p.config.AllocatedIpv4Addresses--
	p.ipv4Pool = append(p.ipv4Pool, ip)

	return nil
}

func (p *IpPool) AllocateIpv6(allocation string) (string, error) {
	if ip, ok := p.ipv6Allocation[allocation]; ok {
		return ip, nil
	}

	l := len(p.ipv6Pool)
	if l == 0 {
		return "", errNoIpAvailable
	}

	ip := p.ipv6Pool[l-1]

	p.config.AvailableIpv6Addresses--
	p.config.AllocatedIpv6Addresses++
	p.ipv6Pool = p.ipv6Pool[:l-1]
	p.ipv6Allocation[allocation] = ip

	return ip, nil
}

func (p *IpPool) ReleaseIpv6(allocation string) error {
	ip, ok := p.ipv6Allocation[allocation]
	if !ok {
		return errInvalidAllocation
	}

	delete(p.ipv6Allocation, allocation)
	p.config.AvailableIpv6Addresses++
	p.config.AllocatedIpv6Addresses--
	p.ipv6Pool = append(p.ipv6Pool, ip)

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2014 The Serviced Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package api

import (
	"fmt"
	"path/filepath"
	"strconv"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/control-center/serviced/config"
	"github.com/control-center/serviced/domain/service"
	"github.com/control-center/serviced/isvcs"
	"github.com/control-center/serviced/node"
	"github.com/control-center/serviced/rpc/rpcutils"
	"github.com/control-center/serviced/utils"
	"github.com/control-center/serviced/validation"
	"github.com/control-center/serviced/volume"
)

const (
	DefaultHomeDir       = "/opt/serviced"
	DefaultRPCPort       = 4979
	outboundIPRetryDelay = 1
	outboundIPMaxWait    = 90
)

// Validate options which are common to all CLI commands
func ValidateCommonOptions(opts config.Options) error {
	var err error

	rpcutils.RPCCertVerify, err = strconv.ParseBool(opts.RPCCertVerify)
	if err != nil {
		return fmt.Errorf("error parsing rpc-cert-verify value %v", err)
	}
	rpcutils.RPCDisableTLS, err = strconv.ParseBool(opts.RPCDisableTLS)
	if err != nil {
		return fmt.Errorf("error parsing rpc-disable-tls value %v", err)
	}

	if err := validation.ValidUIAddress(opts.UIPort); err != nil {
		return fmt.Errorf("error validating UI port: %s", err)
	}

	// TODO: move this to ValidateServerOptions if this is really only used by master/agent, and not cli
	if err := validation.IsSubnetCIDR(opts.VirtualAddressSubnet); err != nil {
		return fmt.Errorf("error validating virtual-address-subnet: %s", err)
	}

	return nil
}

// Validate options which are specific to running as a server
func ValidateServerOptions(options *config.Options) error {
	if err := validateStorageArgs(options); err != nil {
		return err
	}

	// Make sure we have an endpoint to work with
	if len(options.Endpoint) == 0 {
		if options.Master {
			outboundIP, err := getOutboundIP()
			if err != nil {
				log.WithError(err).Fatal("Unable to determine outbound IP")
			}
			options.Endpoint = fmt.Sprintf("%s:%s", outboundIP, options.RPCPort)
		} else {
			return fmt.Errorf("No endpoint to master has been configured")
		}
	}

	if options.Master {
		log.WithFields(logrus.Fields{
			"poolid": options.MasterPoolID,
		}).Debug("Using configured default pool ID")
	}
	return nil
}

// GetOptionsRPCEndpoint returns the serviced RPC endpoint from options
func GetOptionsRPCEndpoint() string {
	return config.GetOptions().Endpoint
}

// GetOptionsRPCPort returns the serviced RPC port from options
func GetOptionsRPCPort() string {
	return config.GetOptions().RPCPort
}

// GetOptionsMaster returns the master mode setting from options
func GetOptionsMaster() bool {
	return config.GetOptions().Master
}

// GetOptionsAgent returns the agent mode setting from options
func GetOptionsAgent() bool {
	return config.GetOptions().Agent
}

// GetOptionsMasterPoolID returns the master pool ID from options
func GetOptionsMasterPoolID() string {
	return config.GetOptions().MasterPoolID
}

// GetOptionsMaxRPCClients returns the max RPC clients setting from options
func GetOptionsMaxRPCClients() int {
	return config.GetOptions().MaxRPCClients
}

func GetDefaultOptions(cfg utils.ConfigReader) config.Options {
	masterIP := cfg.StringVal("MASTER_IP", "127.0.0.1")

	options := config.Options{
		UIPort:                     service.ScrubPortString(cfg.StringVal("UI_PORT", ":443")),
		NFSClient:                  cfg.StringVal("NFS_CLIENT", "1"),
		RPCPort:                    cfg.StringVal("RPC_PORT", fmt.Sprintf("%d", DefaultRPCPort)),
		OutboundIP:                 cfg.StringVal("OUTBOUND_IP", ""),
		DockerDNS:                  cfg.StringSlice("DOCKER_DNS", []string{}),
		Master:                     cfg.BoolVal("MASTER", false),
		MuxPort:                    cfg.IntVal("MUX_PORT", 22250),
		MuxDisableTLS:              strconv.FormatBool(cfg.BoolVal("MUX_DISABLE_TLS", false)),
		KeyPEMFile:                 cfg.StringVal("KEY_FILE", ""),
		CertPEMFile:                cfg.StringVal("CERT_FILE", ""),
		Zookeepers:                 cfg.StringSlice("ZK", []string{}),
		HostStats:                  cfg.StringVal("STATS_PORT", fmt.Sprintf("%s:8443", masterIP)),
		StatsPeriod:                cfg.IntVal("STATS_PERIOD", 10),
		SvcStatsCacheTimeout:       cfg.IntVal("SVCSTATS_CACHE_TIMEOUT", 5),
		MCUsername:                 "scott",
		MCPasswd:                   "tiger",
		FSType:                     volume.DriverType(cfg.StringVal("FS_TYPE", "devicemapper")),
		ESStartupTimeout:           getDefaultESStartupTimeout(cfg.IntVal("ES_STARTUP_TIMEOUT", isvcs.DEFAULT_ES_STARTUP_TIMEOUT_SECONDS)),
		HostAliases:                cfg.StringSlice("VHOST_ALIASES", []string{}),
		Verbosity:                  cfg.IntVal("LOG_LEVEL", 0),
		StaticIPs:                  cfg.StringSlice("STATIC_IPS", []string{}),
		DockerRegistry:             cfg.StringVal("DOCKER_REGISTRY", "localhost:5000"),
		MaxContainerAge:            cfg.IntVal("MAX_CONTAINER_AGE", 60*60*24),
		MaxDFSTimeout:              cfg.IntVal("MAX_DFS_TIMEOUT", 60*5),
		VirtualAddressSubnet:       cfg.StringVal("VIRTUAL_ADDRESS_SUBNET", "10.3.0.0/16"),
		MasterPoolID:               cfg.StringVal("MASTER_POOLID", "default"),
		LogstashES:                 cfg.StringVal("LOGSTASH_ES", fmt.Sprintf("%s:9100", masterIP)),
		LogstashURL:                cfg.StringVal("LOG_ADDRESS", fmt.Sprintf("%s:5042", masterIP)),
		LogstashMaxDays:            cfg.IntVal("LOGSTASH_MAX_DAYS", 14),
		LogstashMaxSize:            cfg.IntVal("LOGSTASH_MAX_SIZE", 10),
		LogstashCycleTime:          cfg.IntVal("LOGSTASH_CYCLE_TIME", 6),
		DebugPort:                  cfg.IntVal("DEBUG_PORT", 6006),
		AdminGroup:                 cfg.StringVal("ADMIN_GROUP", getDefaultAdminGroup()),
		MaxRPCClients:              cfg.IntVal("MAX_RPC_CLIENTS", 3),
		MUXTLSCiphers:              cfg.StringSlice("MUX_TLS_CIPHERS", utils.GetDefaultCiphers("mux")),
		MUXTLSMinVersion:           cfg.StringVal("MUX_TLS_MIN_VERSION", utils.DefaultTLSMinVersion),
		RPCDialTimeout:             cfg.IntVal("RPC_DIAL_TIMEOUT", 30),
		RPCCertVerify:              strconv.FormatBool(cfg.BoolVal("RPC_CERT_VERIFY", false)),
		RPCDisableTLS:              strconv.FormatBool(cfg.BoolVal("RPC_DISABLE_TLS", false)),
		RPCTLSCiphers:              cfg.StringSlice("RPC_TLS_CIPHERS", utils.GetDefaultCiphers("rpc")),
		RPCTLSMinVersion:           cfg.StringVal("RPC_TLS_MIN_VERSION", utils.DefaultTLSMinVersion),
		SnapshotTTL:                cfg.IntVal("SNAPSHOT_TTL", 12),
		StartISVCS:                 cfg.StringSlice("ISVCS_START", []string{}),
		IsvcsZKID:                  cfg.IntVal("ISVCS_ZOOKEEPER_ID", 0),
		IsvcsZKQuorum:              cfg.StringSlice("ISVCS_ZOOKEEPER_QUORUM", []string{}),
		TLSCiphers:                 cfg.StringSlice("TLS_CIPHERS", utils.GetDefaultCiphers("http")),
		TLSMinVersion:              cfg.StringVal("TLS_MIN_VERSION", utils.DefaultTLSMinVersion),
		DockerLogDriver:            cfg.StringVal("DOCKER_LOG_DRIVER", "json-file"),
		DockerLogConfigList:        cfg.StringSlice("DOCKER_LOG_CONFIG", []string{"max-file=5", "max-size=10m"}),
		AllowLoopBack:              strconv.FormatBool(cfg.BoolVal("ALLOW_LOOP_BACK", false)),
		UIPollFrequency:            cfg.IntVal("UI_POLL_FREQUENCY", 3),
		StorageStatsUpdateInterval: cfg.IntVal("STORAGE_STATS_UPDATE_INTERVAL", 300),
		SnapshotSpacePercent:       cfg.IntVal("SNAPSHOT_USE_PERCENT", 20),
		ZKSessionTimeout:           cfg.IntVal("ZK_SESSION_TIMEOUT", 15),
		TokenExpiration:            cfg.IntVal("AUTH_TOKEN_EXPIRATION", 60*60),
		ServiceRunLevelTimeout:     cfg.IntVal("RUN_LEVEL_TIMEOUT", 60),
		StorageReportInterval:      cfg.IntVal("STORAGE_REPORT_INTERVAL", 30),
		StorageMetricMonitorWindow: cfg.IntVal("STORAGE_METRIC_MONITOR_WINDOW", 300),
		StorageLookaheadPeriod:     cfg.IntVal("STORAGE_LOOKAHEAD_PERIOD", 360),
		StorageMinimumFreeSpace:    cfg.StringVal("STORAGE_MIN_FREE", "3G"),
	}

	options.Endpoint = cfg.StringVal("ENDPOINT", "")

	// Set the path to the controller binary
	dir, _, err := node.ExecPath()
	if err != nil {
		log.Warn("Unable to find path to serviced binary; assuming /opt/serviced/bin")
		dir = "/opt/serviced/bin"
	}
	defaultControllerBinary := filepath.Join(dir, "serviced-controller")
	options.ControllerBinary = cfg.StringVal("CONTROLLER_BINARY", defaultControllerBinary)

	homepath := cfg.StringVal("HOME", DefaultHomeDir)
	varpath := filepath.Join(homepath, "var")

	options.IsvcsPath = cfg.StringVal("ISVCS_PATH", filepath.Join(varpath, "isvcs"))
	options.VolumesPath = cfg.StringVal("VOLUMES_PATH", filepath.Join(varpath, "volumes"))
	options.BackupsPath = cfg.StringVal("BACKUPS_PATH", filepath.Join(varpath, "backups"))
	options.EtcPath = cfg.StringVal("ETC_PATH", filepath.Join(homepath, "etc"))
	options.StorageArgs = getDefaultStorageOptions(options.FSType, cfg)

	return options
}

func getDefaultESStartupTimeout(timeout int) int {
	minTimeout := isvcs.MIN_ES_STARTUP_TIMEOUT_SECONDS
	if timeout < minTimeout {
		timeout = minTimeout
	}
	return timeout
}

func getDefaultAdminGroup() string {
	if utils.Platform == utils.Rhel {
		return "wheel"
	}
	return "sudo"
}

// getOutboundIP queries the network configuration for an IP address suitable for reaching the outside world.
// Will retry for a while if a path to the outside world is not yet available.
func getOutboundIP() (string, error) {
	var outboundIP string
	var err error
	timeout := time.After(outboundIPMaxWait * time.Second)
	for {
		if outboundIP, err = utils.GetIPAddress(); err == nil {
			// Success
			return outboundIP, nil
		}
		select {
		case <-timeout:
			// Give up
			return "", fmt.Errorf("Gave up waiting for network (to determine our outbound IP address)")
		default:
			// Retry
			log.Debug("Waiting for network initialization")
			time.Sleep(outboundIPRetryDelay * time.Second)
		}
	}
}
<commit_msg>Make the timeout longer<commit_after>// Copyright 2014 The Serviced Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package api

import (
	"fmt"
	"path/filepath"
	"strconv"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/control-center/serviced/config"
	"github.com/control-center/serviced/domain/service"
	"github.com/control-center/serviced/isvcs"
	"github.com/control-center/serviced/node"
	"github.com/control-center/serviced/rpc/rpcutils"
	"github.com/control-center/serviced/utils"
	"github.com/control-center/serviced/validation"
	"github.com/control-center/serviced/volume"
)

const (
	DefaultHomeDir       = "/opt/serviced"
	DefaultRPCPort       = 4979
	outboundIPRetryDelay = 1
	outboundIPMaxWait    = 90
)

// Validate options which are common to all CLI commands
func ValidateCommonOptions(opts config.Options) error {
	var err error

	rpcutils.RPCCertVerify, err = strconv.ParseBool(opts.RPCCertVerify)
	if err != nil {
		return fmt.Errorf("error parsing rpc-cert-verify value %v", err)
	}
	rpcutils.RPCDisableTLS, err = strconv.ParseBool(opts.RPCDisableTLS)
	if err != nil {
		return fmt.Errorf("error parsing rpc-disable-tls value %v", err)
	}

	if err := validation.ValidUIAddress(opts.UIPort); err != nil {
		return fmt.Errorf("error validating UI port: %s", err)
	}

	// TODO: move this to ValidateServerOptions if this is really only used by master/agent, and not cli
	if err := validation.IsSubnetCIDR(opts.VirtualAddressSubnet); err != nil {
		return fmt.Errorf("error validating virtual-address-subnet: %s", err)
	}

	return nil
}

// Validate options which are specific to running as a server
func ValidateServerOptions(options *config.Options) error {
	if err := validateStorageArgs(options); err != nil {
		return err
	}

	// Make sure we have an endpoint to work with
	if len(options.Endpoint) == 0 {
		if options.Master {
			outboundIP, err := getOutboundIP()
			if err != nil {
				log.WithError(err).Fatal("Unable to determine outbound IP")
			}
			options.Endpoint = fmt.Sprintf("%s:%s", outboundIP, options.RPCPort)
		} else {
			return fmt.Errorf("No endpoint to master has been configured")
		}
	}

	if options.Master {
		log.WithFields(logrus.Fields{
			"poolid": options.MasterPoolID,
		}).Debug("Using configured default pool ID")
	}
	return nil
}

// GetOptionsRPCEndpoint returns the serviced RPC endpoint from options
func GetOptionsRPCEndpoint() string {
	return config.GetOptions().Endpoint
}

// GetOptionsRPCPort returns the serviced RPC port from options
func GetOptionsRPCPort() string {
	return config.GetOptions().RPCPort
}

// GetOptionsMaster returns the master mode setting from options
func GetOptionsMaster() bool {
	return config.GetOptions().Master
}

// GetOptionsAgent returns the agent mode setting from options
func GetOptionsAgent() bool {
	return config.GetOptions().Agent
}

// GetOptionsMasterPoolID returns the master pool ID from options
func GetOptionsMasterPoolID() string {
	return config.GetOptions().MasterPoolID
}

// GetOptionsMaxRPCClients returns the max RPC clients setting from options
func GetOptionsMaxRPCClients() int {
	return config.GetOptions().MaxRPCClients
}

func GetDefaultOptions(cfg utils.ConfigReader) config.Options {
	masterIP := cfg.StringVal("MASTER_IP", "127.0.0.1")

	options := config.Options{
		UIPort:                     service.ScrubPortString(cfg.StringVal("UI_PORT", ":443")),
		NFSClient:                  cfg.StringVal("NFS_CLIENT", "1"),
		RPCPort:                    cfg.StringVal("RPC_PORT", fmt.Sprintf("%d", DefaultRPCPort)),
		OutboundIP:                 cfg.StringVal("OUTBOUND_IP", ""),
		DockerDNS:                  cfg.StringSlice("DOCKER_DNS", []string{}),
		Master:                     cfg.BoolVal("MASTER", false),
		MuxPort:                    cfg.IntVal("MUX_PORT", 22250),
		MuxDisableTLS:              strconv.FormatBool(cfg.BoolVal("MUX_DISABLE_TLS", false)),
		KeyPEMFile:                 cfg.StringVal("KEY_FILE", ""),
		CertPEMFile:                cfg.StringVal("CERT_FILE", ""),
		Zookeepers:                 cfg.StringSlice("ZK", []string{}),
		HostStats:                  cfg.StringVal("STATS_PORT", fmt.Sprintf("%s:8443", masterIP)),
		StatsPeriod:                cfg.IntVal("STATS_PERIOD", 10),
		SvcStatsCacheTimeout:       cfg.IntVal("SVCSTATS_CACHE_TIMEOUT", 5),
		MCUsername:                 "scott",
		MCPasswd:                   "tiger",
		FSType:                     volume.DriverType(cfg.StringVal("FS_TYPE", "devicemapper")),
		ESStartupTimeout:           getDefaultESStartupTimeout(cfg.IntVal("ES_STARTUP_TIMEOUT", isvcs.DEFAULT_ES_STARTUP_TIMEOUT_SECONDS)),
		HostAliases:                cfg.StringSlice("VHOST_ALIASES", []string{}),
		Verbosity:                  cfg.IntVal("LOG_LEVEL", 0),
		StaticIPs:                  cfg.StringSlice("STATIC_IPS", []string{}),
		DockerRegistry:             cfg.StringVal("DOCKER_REGISTRY", "localhost:5000"),
		MaxContainerAge:            cfg.IntVal("MAX_CONTAINER_AGE", 60*60*24),
		MaxDFSTimeout:              cfg.IntVal("MAX_DFS_TIMEOUT", 60*5),
		VirtualAddressSubnet:       cfg.StringVal("VIRTUAL_ADDRESS_SUBNET", "10.3.0.0/16"),
		MasterPoolID:               cfg.StringVal("MASTER_POOLID", "default"),
		LogstashES:                 cfg.StringVal("LOGSTASH_ES", fmt.Sprintf("%s:9100", masterIP)),
		LogstashURL:                cfg.StringVal("LOG_ADDRESS", fmt.Sprintf("%s:5042", masterIP)),
		LogstashMaxDays:            cfg.IntVal("LOGSTASH_MAX_DAYS", 14),
		LogstashMaxSize:            cfg.IntVal("LOGSTASH_MAX_SIZE", 10),
		LogstashCycleTime:          cfg.IntVal("LOGSTASH_CYCLE_TIME", 6),
		DebugPort:                  cfg.IntVal("DEBUG_PORT", 6006),
		AdminGroup:                 cfg.StringVal("ADMIN_GROUP", getDefaultAdminGroup()),
		MaxRPCClients:              cfg.IntVal("MAX_RPC_CLIENTS", 3),
		MUXTLSCiphers:              cfg.StringSlice("MUX_TLS_CIPHERS", utils.GetDefaultCiphers("mux")),
		MUXTLSMinVersion:           cfg.StringVal("MUX_TLS_MIN_VERSION", utils.DefaultTLSMinVersion),
		RPCDialTimeout:             cfg.IntVal("RPC_DIAL_TIMEOUT", 30),
		RPCCertVerify:              strconv.FormatBool(cfg.BoolVal("RPC_CERT_VERIFY", false)),
		RPCDisableTLS:              strconv.FormatBool(cfg.BoolVal("RPC_DISABLE_TLS", false)),
		RPCTLSCiphers:              cfg.StringSlice("RPC_TLS_CIPHERS", utils.GetDefaultCiphers("rpc")),
		RPCTLSMinVersion:           cfg.StringVal("RPC_TLS_MIN_VERSION", utils.DefaultTLSMinVersion),
		SnapshotTTL:                cfg.IntVal("SNAPSHOT_TTL", 12),
		StartISVCS:                 cfg.StringSlice("ISVCS_START", []string{}),
		IsvcsZKID:                  cfg.IntVal("ISVCS_ZOOKEEPER_ID", 0),
		IsvcsZKQuorum:              cfg.StringSlice("ISVCS_ZOOKEEPER_QUORUM", []string{}),
		TLSCiphers:                 cfg.StringSlice("TLS_CIPHERS", utils.GetDefaultCiphers("http")),
		TLSMinVersion:              cfg.StringVal("TLS_MIN_VERSION", utils.DefaultTLSMinVersion),
		DockerLogDriver:            cfg.StringVal("DOCKER_LOG_DRIVER", "json-file"),
		DockerLogConfigList:        cfg.StringSlice("DOCKER_LOG_CONFIG", []string{"max-file=5", "max-size=10m"}),
		AllowLoopBack:              strconv.FormatBool(cfg.BoolVal("ALLOW_LOOP_BACK", false)),
		UIPollFrequency:            cfg.IntVal("UI_POLL_FREQUENCY", 3),
		StorageStatsUpdateInterval: cfg.IntVal("STORAGE_STATS_UPDATE_INTERVAL", 300),
		SnapshotSpacePercent:       cfg.IntVal("SNAPSHOT_USE_PERCENT", 20),
		ZKSessionTimeout:           cfg.IntVal("ZK_SESSION_TIMEOUT", 15),
		TokenExpiration:            cfg.IntVal("AUTH_TOKEN_EXPIRATION", 60*60),
		ServiceRunLevelTimeout:     cfg.IntVal("RUN_LEVEL_TIMEOUT", 60*10),
		StorageReportInterval:      cfg.IntVal("STORAGE_REPORT_INTERVAL", 30),
		StorageMetricMonitorWindow: cfg.IntVal("STORAGE_METRIC_MONITOR_WINDOW", 300),
		StorageLookaheadPeriod:     cfg.IntVal("STORAGE_LOOKAHEAD_PERIOD", 360),
		StorageMinimumFreeSpace:    cfg.StringVal("STORAGE_MIN_FREE", "3G"),
	}

	options.Endpoint = cfg.StringVal("ENDPOINT", "")

	// Set the path to the controller binary
	dir, _, err := node.ExecPath()
	if err != nil {
		log.Warn("Unable to find path to serviced binary; assuming /opt/serviced/bin")
		dir = "/opt/serviced/bin"
	}
	defaultControllerBinary := filepath.Join(dir, "serviced-controller")
	options.ControllerBinary = cfg.StringVal("CONTROLLER_BINARY", defaultControllerBinary)

	homepath := cfg.StringVal("HOME", DefaultHomeDir)
	varpath := filepath.Join(homepath, "var")

	options.IsvcsPath = cfg.StringVal("ISVCS_PATH", filepath.Join(varpath, "isvcs"))
	options.VolumesPath = cfg.StringVal("VOLUMES_PATH", filepath.Join(varpath, "volumes"))
	options.BackupsPath = cfg.StringVal("BACKUPS_PATH", filepath.Join(varpath, "backups"))
	options.EtcPath = cfg.StringVal("ETC_PATH", filepath.Join(homepath, "etc"))
	options.StorageArgs = getDefaultStorageOptions(options.FSType, cfg)

	return options
}

func getDefaultESStartupTimeout(timeout int) int {
	minTimeout := isvcs.MIN_ES_STARTUP_TIMEOUT_SECONDS
	if timeout < minTimeout {
		timeout = minTimeout
	}
	return timeout
}

func getDefaultAdminGroup() string {
	if utils.Platform == utils.Rhel {
		return "wheel"
	}
	return "sudo"
}

// getOutboundIP queries the network configuration for an IP address suitable for reaching the outside world.
// Will retry for a while if a path to the outside world is not yet available.
func getOutboundIP() (string, error) {
	var outboundIP string
	var err error
	timeout := time.After(outboundIPMaxWait * time.Second)
	for {
		if outboundIP, err = utils.GetIPAddress(); err == nil {
			// Success
			return outboundIP, nil
		}
		select {
		case <-timeout:
			// Give up
			return "", fmt.Errorf("Gave up waiting for network (to determine our outbound IP address)")
		default:
			// Retry
			log.Debug("Waiting for network initialization")
			time.Sleep(outboundIPRetryDelay * time.Second)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/xml"
	"fmt"
	"regexp"
	"time"

	"github.com/ninjasphere/go-ninja/api"
	"github.com/ninjasphere/go-ninja/channels"
	"github.com/ninjasphere/go-ninja/devices"
	"github.com/ninjasphere/go-ninja/logger"
	"github.com/ninjasphere/go-ninja/model"
	"github.com/ninjasphere/go-sonos"
	"github.com/ninjasphere/go-sonos/didl"
	"github.com/ninjasphere/go-sonos/upnp"
)

const (
	defaultInstanceID = 0
	defaultSpeed      = "1"
)

type sonosPlayer struct {
	*sonos.Sonos
	log    *logger.Logger
	player *devices.MediaPlayerDevice
}

func (sp *sonosPlayer) applyPlayPause(playing bool) error {

	sp.log.Infof("applyPlayPause called, playing: %t", playing)

	if playing {
		err := sp.Play(defaultInstanceID, defaultSpeed)

		if err != nil {
			return err
		}

		return sp.player.UpdateControlState(channels.MediaControlEventPlaying)

	}

	err := sp.Pause(defaultInstanceID)

	if err != nil {
		return err
	}

	return sp.player.UpdateControlState(channels.MediaControlEventPaused)
}

func (sp *sonosPlayer) applyStop() error {
	sp.log.Infof("applyStop called")

	err := sp.Stop(defaultInstanceID)

	if err != nil {
		return err
	}

	return sp.player.UpdateControlState(channels.MediaControlEventStopped)
}

func (sp *sonosPlayer) applyPlaylistJump(delta int) error {
	sp.log.Infof("applyPlaylistJump called, delta : %d", delta)
	if delta < 0 {
		return sp.Previous(defaultInstanceID)
	}
	return sp.Next(defaultInstanceID)
}

func (sp *sonosPlayer) applyVolume(volume float64) error {
	sp.log.Infof("applyVolume called, volume %f", volume)

	vol := uint16(volume * 100)

	// XXX: HALVING THE VOLUME BECAUSE DAN IS AN OLD MAN
	vol = vol / 2

	err := sp.SetVolume(defaultInstanceID, upnp.Channel_Master, vol)

	if err != nil {
		return err
	}

	return sp.player.UpdateVolumeState(volume)
}

func (sp *sonosPlayer) applyMuted(muted bool) error {
	err := sp.SetMute(defaultInstanceID, upnp.Channel_Master, muted)

	if err != nil {
		return err
	}

	return sp.player.UpdateMutedState(muted)
}

func (sp *sonosPlayer) applyPlayURL(url string, queue bool) error {
	return fmt.Errorf("Playing a URL has not been implemented yet.")
}

func (sp *sonosPlayer) bindMethods() error {

	sp.player.ApplyPlayPause = sp.applyPlayPause
	sp.player.ApplyStop = sp.applyStop
	sp.player.ApplyPlaylistJump = sp.applyPlaylistJump
	sp.player.ApplyVolume = sp.applyVolume
	sp.player.ApplyMuted = sp.applyMuted
	sp.player.ApplyPlayURL = sp.applyPlayURL

	err := sp.player.EnableControlChannel([]string{
		"playing",
		"paused",
		"stopped",
		"idle",
	})
	if err != nil {
		return err
	}

	err = sp.player.EnableVolumeChannel()
	if err != nil {
		return err
	}

	err = sp.player.EnableMediaChannel()
	if err != nil {
		return err
	}

	return nil
}

var timeDuration = regexp.MustCompile("([0-9]{1,2})\\:([0-9]{2})\\:([0-9]{2})")

func parseDuration(t string) (*time.Duration, error) {

	found := timeDuration.FindAllStringSubmatch(t, -1)

	if found == nil || len(found) == 0 {
		return nil, fmt.Errorf("Failed to parse duration from '%s'", t)
	}

	duration, err := time.ParseDuration(fmt.Sprintf("%sh%sm%ss", found[0][1], found[0][2], found[0][3]))

	if found == nil || len(found) == 0 {
		return nil, fmt.Errorf("Failed to parse duration from '%s': %s", t, err)
	}

	return &duration, nil
}

func (sp *sonosPlayer) updateMedia() error {
	t := sp.log

	positionInfo, err := sp.GetPositionInfo(0)
	if err != nil {
		return err
	}

	if positionInfo.TrackMetaData == "" {
		t.Infof("No track!")
		err = sp.player.UpdateMusicMediaState(nil, nil)
		return err
	}

	duration, err := parseDuration(positionInfo.TrackDuration)
	if err != nil {
		return err
	}

	durationMs := int(*duration / time.Millisecond)

	position, err := parseDuration(positionInfo.RelTime)
	if err != nil {
		return err
	}

	positionMs := int(*position / time.Millisecond)

	var trackMetadata didl.Lite

	err = xml.Unmarshal([]byte(positionInfo.TrackMetaData), &trackMetadata)
	if err != nil {
		return err
	}

	//spew.Dump("DIDL", trackMetadata)

	track := &channels.MusicTrackMediaItem{
		ID:    &positionInfo.TrackURI,
		Title: &trackMetadata.Item[0].Title[0].Value,
		Album: &channels.MediaItemAlbum{
			Name: trackMetadata.Item[0].Album[0].Value,
		},
		Artists: &[]channels.MediaItemArtist{
			channels.MediaItemArtist{
				Name: trackMetadata.Item[0].Creator[0].Value,
			},
		},
		Duration: &durationMs,
	}

	err = sp.player.UpdateMusicMediaState(track, &positionMs)
	if err != nil {
		return err
	}

	return nil
}

func (sp *sonosPlayer) updateState() error {

	/*mediaInfo, err := sp.GetMediaInfo(defaultInstanceID)

	  if err != nil {
	    return err
	  }*/

	//func (d *MediaPlayerDevice) UpdateMusicMediaState(item *MusicTrackMediaItem, position *int) error {
	err := sp.updateMedia()

	if err != nil {
		return fmt.Errorf("Failed to update current media: %s", err)
	}

	muted, err := sp.GetMute(defaultInstanceID, upnp.Channel_Master)

	if err != nil {
		return err
	}

	sp.log.Infof("UpdateMutedState %t", muted)
	if sp.player.UpdateMutedState(muted); err != nil {
		return err
	}

	vol, err := sp.GetVolume(defaultInstanceID, upnp.Channel_Master)

	if err != nil {
		return err
	}

	var volume float64

	if vol > 0 {
		volume = float64(vol) / 100
	} else {
		volume = float64(0)

	}

	sp.log.Infof("UpdateVolumeState %d  %f", vol, volume)
	return sp.player.UpdateVolumeState(volume)
}

func NewPlayer(driver *sonosDriver, conn *ninja.Connection, sonosUnit *sonos.Sonos) (*sonosPlayer, error) {

	group, _ := sonosUnit.GetZoneGroupAttributes()

	id := group.CurrentZoneGroupID
	name := group.CurrentZoneGroupName

	nlog.Infof("Making media player with ID: %s Label: %s", id, name)

	player, err := devices.CreateMediaPlayerDevice(driver, &model.Device{
		NaturalID:     id,
		NaturalIDType: "sonos",
		Name:          &name,
		Signatures: &map[string]string{
			"ninja:manufacturer": "Sonos",
			"ninja:productName":  "Sonos Player",
			"ninja:productType":  "MediaPlayer",
			"ninja:thingType":    "mediaplayer",
		},
	}, conn)

	if err != nil {
		nlog.FatalError(err, "Failed to create media player device")
	}

	sp := &sonosPlayer{sonosUnit, logger.GetLogger("sonosPlayer"), player}

	err = sp.bindMethods()
	if err != nil {
		sp.log.FatalError(err, "Failed to bind channels to sonos device")
	}

	err = sp.updateState()

	if err != nil {
		sp.log.FatalError(err, "Failed to create media player device bus")
	}

	return sp, nil
}
<commit_msg>You can play something that has no album or artist!?<commit_after>package main

import (
	"encoding/xml"
	"fmt"
	"regexp"
	"time"

	"github.com/ninjasphere/go-ninja/api"
	"github.com/ninjasphere/go-ninja/channels"
	"github.com/ninjasphere/go-ninja/devices"
	"github.com/ninjasphere/go-ninja/logger"
	"github.com/ninjasphere/go-ninja/model"
	"github.com/ninjasphere/go-sonos"
	"github.com/ninjasphere/go-sonos/didl"
	"github.com/ninjasphere/go-sonos/upnp"
)

const (
	defaultInstanceID = 0
	defaultSpeed      = "1"
)

type sonosPlayer struct {
	*sonos.Sonos
	log    *logger.Logger
	player *devices.MediaPlayerDevice
}

func (sp *sonosPlayer) applyPlayPause(playing bool) error {

	sp.log.Infof("applyPlayPause called, playing: %t", playing)

	if playing {
		err := sp.Play(defaultInstanceID, defaultSpeed)

		if err != nil {
			return err
		}

		return sp.player.UpdateControlState(channels.MediaControlEventPlaying)

	}

	err := sp.Pause(defaultInstanceID)

	if err != nil {
		return err
	}

	return sp.player.UpdateControlState(channels.MediaControlEventPaused)
}

func (sp *sonosPlayer) applyStop() error {
	sp.log.Infof("applyStop called")

	err := sp.Stop(defaultInstanceID)

	if err != nil {
		return err
	}

	return sp.player.UpdateControlState(channels.MediaControlEventStopped)
}

func (sp *sonosPlayer) applyPlaylistJump(delta int) error {
	sp.log.Infof("applyPlaylistJump called, delta : %d", delta)
	if delta < 0 {
		return sp.Previous(defaultInstanceID)
	}
	return sp.Next(defaultInstanceID)
}

func (sp *sonosPlayer) applyVolume(volume float64) error {
	sp.log.Infof("applyVolume called, volume %f", volume)

	vol := uint16(volume * 100)

	// XXX: HALVING THE VOLUME BECAUSE DAN IS AN OLD MAN
	vol = vol / 2

	err := sp.SetVolume(defaultInstanceID, upnp.Channel_Master, vol)

	if err != nil {
		return err
	}

	return sp.player.UpdateVolumeState(volume)
}

func (sp *sonosPlayer) applyMuted(muted bool) error {
	err := sp.SetMute(defaultInstanceID, upnp.Channel_Master, muted)

	if err != nil {
		return err
	}

	return sp.player.UpdateMutedState(muted)
}

func (sp *sonosPlayer) applyPlayURL(url string, queue bool) error {
	return fmt.Errorf("Playing a URL has not been implemented yet.")
}

func (sp *sonosPlayer) bindMethods() error {

	sp.player.ApplyPlayPause = sp.applyPlayPause
	sp.player.ApplyStop = sp.applyStop
	sp.player.ApplyPlaylistJump = sp.applyPlaylistJump
	sp.player.ApplyVolume = sp.applyVolume
	sp.player.ApplyMuted = sp.applyMuted
	sp.player.ApplyPlayURL = sp.applyPlayURL

	err := sp.player.EnableControlChannel([]string{
		"playing",
		"paused",
		"stopped",
		"idle",
	})
	if err != nil {
		return err
	}

	err = sp.player.EnableVolumeChannel()
	if err != nil {
		return err
	}

	err = sp.player.EnableMediaChannel()
	if err != nil {
		return err
	}

	return nil
}

var timeDuration = regexp.MustCompile("([0-9]{1,2})\\:([0-9]{2})\\:([0-9]{2})")

func parseDuration(t string) (*time.Duration, error) {

	found := timeDuration.FindAllStringSubmatch(t, -1)

	if found == nil || len(found) == 0 {
		return nil, fmt.Errorf("Failed to parse duration from '%s'", t)
	}

	duration, err := time.ParseDuration(fmt.Sprintf("%sh%sm%ss", found[0][1], found[0][2], found[0][3]))

	if found == nil || len(found) == 0 {
		return nil, fmt.Errorf("Failed to parse duration from '%s': %s", t, err)
	}

	return &duration, nil
}

func (sp *sonosPlayer) updateMedia() error {
	t := sp.log

	positionInfo, err := sp.GetPositionInfo(0)
	if err != nil {
		return err
	}

	if positionInfo.TrackMetaData == "" {
		t.Infof("No track!")
		err = sp.player.UpdateMusicMediaState(nil, nil)
		return err
	}

	duration, err := parseDuration(positionInfo.TrackDuration)
	if err != nil {
		return err
	}

	durationMs := int(*duration / time.Millisecond)

	position, err := parseDuration(positionInfo.RelTime)
	if err != nil {
		return err
	}

	positionMs := int(*position / time.Millisecond)

	var trackMetadata didl.Lite

	err = xml.Unmarshal([]byte(positionInfo.TrackMetaData), &trackMetadata)
	if err != nil {
		return err
	}

	//sp.log.Infof(spew.Sdump("DIDL", trackMetadata))

	track := &channels.MusicTrackMediaItem{
		ID:    &positionInfo.TrackURI,
		Title: &trackMetadata.Item[0].Title[0].Value,
		// Album: &channels.MediaItemAlbum{
		// 	Name: trackMetadata.Item[0].Album[0].Value,
		// },
		// Artists: &[]channels.MediaItemArtist{
		// 	channels.MediaItemArtist{
		// 		Name: trackMetadata.Item[0].Creator[0].Value,
		// 	},
		// },
		Duration: &durationMs,
	}

	if trackMetadata.Item[0].Album != nil {
		track.Album = &channels.MediaItemAlbum{
			Name: trackMetadata.Item[0].Album[0].Value,
		}
	}

	if trackMetadata.Item[0].Creator != nil {
		track.Artists = &[]channels.MediaItemArtist{
			channels.MediaItemArtist{
				Name: trackMetadata.Item[0].Creator[0].Value,
			},
		}
	}

	err = sp.player.UpdateMusicMediaState(track, &positionMs)
	if err != nil {
		return err
	}

	return nil
}

func (sp *sonosPlayer) updateState() error {

	/*mediaInfo, err := sp.GetMediaInfo(defaultInstanceID)

	  if err != nil {
	    return err
	  }*/

	//func (d *MediaPlayerDevice) UpdateMusicMediaState(item *MusicTrackMediaItem, position *int) error {
	err := sp.updateMedia()

	if err != nil {
		return fmt.Errorf("Failed to update current media: %s", err)
	}

	muted, err := sp.GetMute(defaultInstanceID, upnp.Channel_Master)

	if err != nil {
		return err
	}

	sp.log.Infof("UpdateMutedState %t", muted)
	if sp.player.UpdateMutedState(muted); err != nil {
		return err
	}

	vol, err := sp.GetVolume(defaultInstanceID, upnp.Channel_Master)

	if err != nil {
		return err
	}

	var volume float64

	if vol > 0 {
		volume = float64(vol) / 100
	} else {
		volume = float64(0)

	}

	sp.log.Infof("UpdateVolumeState %d  %f", vol, volume)
	return sp.player.UpdateVolumeState(volume)
}

func NewPlayer(driver *sonosDriver, conn *ninja.Connection, sonosUnit *sonos.Sonos) (*sonosPlayer, error) {

	group, _ := sonosUnit.GetZoneGroupAttributes()

	id := group.CurrentZoneGroupID
	name := group.CurrentZoneGroupName

	nlog.Infof("Making media player with ID: %s Label: %s", id, name)

	player, err := devices.CreateMediaPlayerDevice(driver, &model.Device{
		NaturalID:     id,
		NaturalIDType: "sonos",
		Name:          &name,
		Signatures: &map[string]string{
			"ninja:manufacturer": "Sonos",
			"ninja:productName":  "Sonos Player",
			"ninja:productType":  "MediaPlayer",
			"ninja:thingType":    "mediaplayer",
		},
	}, conn)

	if err != nil {
		nlog.FatalError(err, "Failed to create media player device")
	}

	sp := &sonosPlayer{sonosUnit, logger.GetLogger("sonosPlayer"), player}

	err = sp.bindMethods()
	if err != nil {
		sp.log.FatalError(err, "Failed to bind channels to sonos device")
	}

	err = sp.updateState()

	if err != nil {
		sp.log.FatalError(err, "Failed to create media player device bus")
	}

	return sp, nil
}
<|endoftext|>


<commit_before>package scws4go

/*
// 引入头文件有几种方法
// (1) 直接在.go文件中使用CFLAGS指定路径
// (2) shell的环境变量C_INCLUDE_PATH
// (3) shell的环境变量CGO_CFLAGS也可以
// 引入库文件同样的方法,设置LDFLAGS或者LIBRARY_PATH或者CGO_CFLAGS
//
// 这里假设通过C_INCLUDE_PATH和LIBRARY_PATH能找到scws的头文件和库文件

char * CharOff2String(char* text,int off) {
    return text+off;
}

char * CharArray2String(char text[3]) {
    return &text[0];
}

#include <stdlib.h>
#include <string.h>
#include "scws/scws.h"
#cgo LDFLAGS : -lscws
*/
import "C"

import (
    "errors"
    "unsafe"
)

const (
    // SetMulti mode
    SCWS_MULTI_SHORT = C.SCWS_MULTI_SHORT
    SCWS_MULTI_DUALITY = C.SCWS_MULTI_DUALITY
    SCWS_MULTI_ZMAIN = C.SCWS_MULTI_ZMAIN
    SCWS_MULTI_ZALL = C.SCWS_MULTI_ZALL

    // SetDict/AddDict mode
    SCWS_XDICT_TXT = C.SCWS_XDICT_TXT
    SCWS_XDICT_XDB = C.SCWS_XDICT_XDB
    SCWS_XDICT_MEM = C.SCWS_XDICT_MEM
)

// 分词结果
type ScwsRes struct {
    Term   string  //分词的结果
    Attr   string  //词性
    Idf    float64 //idf值
}

// Scws是封装好的切词服务.提供一个简单的切词接口.
type Scws struct {
    root        C.scws_t

    forkScws    chan C.scws_t
}

// 设定当前 scws 所使用的字符集.
// 参数cs 新指定的字符集.若无此调用则系统缺省使用gbk,还支持utf8,指定字符集时参数的大小写不敏感
// 错误若指定的字符集不存在,则会自动使用gbk 字符集替代.
func (this *Scws) SetCharset(cs string) {
    ctext := C.CString(cs)
    defer C.free(unsafe.Pointer(ctext))
    C.scws_set_charset(this.root,ctext)
}

// 添加词典文件到当前scws 对象
// 参数fpath 词典的文件路径,词典格式是XDB或XT 格式。
// 参数mode有3种值,分别为预定义的：
// SCWS_XDICT_TXT 表示要读取的词典文件是文本格式，可以和后2项结合用
// SCWS_XDICT_XDB 表示直接读取 xdb 文件
// SCWS_XDICT_MEM 表示将 xdb 文件全部加载到内存中，以 XTree 结构存放，可用异或结合另外2个使用。
// 具体用哪种方式需要根据自己的实际应用来决定。当使用本库做为守护进程时推荐使用 mem 方式， 当只是嵌入调用时应该使用 xdb 方式，将 xdb 文件加载进内存不仅占用了比较多的内存， 而且也需要一定的时间（35万条数据约需要0.3~0.5秒左右）
func (this *Scws) AddDict(fPath string, mode int) error {
    ctext := C.CString(fPath)
    defer C.free(unsafe.Pointer(ctext))
    ret := int(C.scws_add_dict(this.root,ctext,C.int(mode)))
    if ret != 0 {
        return errors.New("AddDict Fail")
    }
    return nil
}

// 清除并设定当前scws 操作所有的词典文件
func (this *Scws) SetDict(fPath string, mode int) error {
    ctext := C.CString(fPath)
    defer C.free(unsafe.Pointer(ctext))
    ret := int(C.scws_set_dict(this.root,ctext,C.int(mode)))
    if ret != 0 {
        return errors.New("AddDict Fail")
    }
    return nil
}

// 设定规则集文件
func (this *Scws) SetRule(fPath string) error {
    ctext := C.CString(fPath)
    defer C.free(unsafe.Pointer(ctext))
    C.scws_set_rule(this.root,ctext)
    if this.root.r == nil{
        return errors.New("Set Rule Fail")
    }
    return nil
}

// 设定分词结果是否忽略所有的标点等特殊符号(不会忽略\r和\n)
// 参数yes 1 表示忽略,0 表示不忽略,缺省情况为不忽略
func (this *Scws) SetIgnore(yes int) {
    C.scws_set_ignore(this.root,C.int(yes))
}

// 设定分词执行时是否执行针对长词复合切分。（例：“中国人”分为“中国”、“人”、“中国人”）。
// 参数mode复合分词法的级别,缺省不复合分词.取值由下面几个常量异或组合:
// SCWS_MULTI_SHORT 短词
// SCWS_MULTI_DUALITY 二元（将相邻的2个单字组合成一个词）
// SCWS_MULTI_ZMAIN 重要单字
// SCWS_MULTI_ZALL 全部单字
func (this *Scws) SetMulti(mode int) {
    C.scws_set_multi(this.root,C.int(mode))
}

// 设定是否将闲散文字自动以二字分词法聚合。
// 参数yes 如果为1 表示执行二分聚合,0 表示不处理,缺省为0
func (this *Scws) SetDuality(yes int) {
    C.scws_set_duality(this.root,C.int(yes))
}

// 未实现,若有需要加上
// scws_get_tops
// scws_free_tops
// scws_has_word
// scws_get_words

func (this *Scws) Segment(text string)([]ScwsRes,error) {
    if this.forkScws == nil {
        return nil,errors.New("必须在非并发情况下调用一次Scws.Init")
    }
    // 分词结果数组
    scwsResult := make([]ScwsRes,0)

    // 从队列取一个用
    tmpScws := <-this.forkScws

    ctext := C.CString(text)
    defer C.free(unsafe.Pointer(ctext))

    C.scws_send_text(tmpScws,ctext,C.int(len(text)))
    res := C.scws_get_result(tmpScws)
    for res != nil {

        cur := res
        for cur != nil {
            attr := (*C.char)(unsafe.Pointer(&cur.attr[0]))
            scwsResult = append(scwsResult,ScwsRes{
                Term : C.GoStringN(C.CharOff2String(ctext,cur.off),C.int(cur.len)),
                Idf : float64(cur.idf),
                Attr : C.GoStringN(attr,C.int(C.strlen(attr))) })
            cur = cur.next
        }

        // 释放这个结果,获取下个结果
        C.scws_free_result(res)
        res = C.scws_get_result(tmpScws)
    }

    // 用完放入队列
    this.forkScws<-tmpScws

    return scwsResult,nil
}

// 释放Scws的全部资源.
func (this *Scws) Free() (error) {
    if this.forkScws != nil {
        close(this.forkScws)
        for s := range this.forkScws {
            C.scws_free(s)
        }
    }

    if this.root != nil {
        C.scws_free(this.root)
        this.root = nil
    }

    return nil
}

// 内部复制多个scws实例,并发调用Segment的时候可以并发切词.
// 否则Segment也能正常使用,只是所有切词都是串行执行.
func (this *Scws) Init(count int) (error) {
    if this.forkScws != nil {
        return errors.New("Scws.Init只允许调用一次")
    }
    if count < 1 {
        return errors.New("不能少于1个实例")
    }
    this.forkScws = make(chan C.scws_t,count)

    for i:=0;i<count;i++ {
        tmp := C.scws_fork(this.root)
        if tmp != nil {
            this.forkScws<-tmp
        }
    }
    if len(this.forkScws) != count {
        return errors.New("内存不足导致fork数量不符合预期")
    }
    return nil
}

// Scws构造函数
func NewScws() (*Scws) {
    s := &Scws{}
    s.root = C.scws_new()
    s.forkScws = nil
    return s
}
<commit_msg>add debug info<commit_after>package scws4go

/*
// 引入头文件有几种方法
// (1) 直接在.go文件中使用CFLAGS指定路径
// (2) shell的环境变量C_INCLUDE_PATH
// (3) shell的环境变量CGO_CFLAGS也可以
// 引入库文件同样的方法,设置LDFLAGS或者LIBRARY_PATH或者CGO_CFLAGS
//
// 这里假设通过C_INCLUDE_PATH和LIBRARY_PATH能找到scws的头文件和库文件

char * CharOff2String(char* text,int off) {
    return text+off;
}

char * CharArray2String(char text[3]) {
    return &text[0];
}

#include <stdlib.h>
#include <string.h>
#include "scws/scws.h"
#cgo LDFLAGS : -lscws
*/
import "C"

import (
    "errors"
    "unsafe"
    "fmt"
)

const (
    // SetMulti mode
    SCWS_MULTI_SHORT = C.SCWS_MULTI_SHORT
    SCWS_MULTI_DUALITY = C.SCWS_MULTI_DUALITY
    SCWS_MULTI_ZMAIN = C.SCWS_MULTI_ZMAIN
    SCWS_MULTI_ZALL = C.SCWS_MULTI_ZALL

    // SetDict/AddDict mode
    SCWS_XDICT_TXT = C.SCWS_XDICT_TXT
    SCWS_XDICT_XDB = C.SCWS_XDICT_XDB
    SCWS_XDICT_MEM = C.SCWS_XDICT_MEM
)

// 分词结果
type ScwsRes struct {
    Term   string  //分词的结果
    Attr   string  //词性
    Idf    float64 //idf值
}

// Scws是封装好的切词服务.提供一个简单的切词接口.
type Scws struct {
    root        C.scws_t

    forkScws    chan C.scws_t
}

// 设定当前 scws 所使用的字符集.
// 参数cs 新指定的字符集.若无此调用则系统缺省使用gbk,还支持utf8,指定字符集时参数的大小写不敏感
// 错误若指定的字符集不存在,则会自动使用gbk 字符集替代.
func (this *Scws) SetCharset(cs string) {
    ctext := C.CString(cs)
    defer C.free(unsafe.Pointer(ctext))
    C.scws_set_charset(this.root,ctext)
}

// 添加词典文件到当前scws 对象
// 参数fpath 词典的文件路径,词典格式是XDB或XT 格式。
// 参数mode有3种值,分别为预定义的：
// SCWS_XDICT_TXT 表示要读取的词典文件是文本格式，可以和后2项结合用
// SCWS_XDICT_XDB 表示直接读取 xdb 文件
// SCWS_XDICT_MEM 表示将 xdb 文件全部加载到内存中，以 XTree 结构存放，可用异或结合另外2个使用。
// 具体用哪种方式需要根据自己的实际应用来决定。当使用本库做为守护进程时推荐使用 mem 方式， 当只是嵌入调用时应该使用 xdb 方式，将 xdb 文件加载进内存不仅占用了比较多的内存， 而且也需要一定的时间（35万条数据约需要0.3~0.5秒左右）
func (this *Scws) AddDict(fPath string, mode int) error {
    ctext := C.CString(fPath)
    defer C.free(unsafe.Pointer(ctext))
    ret := int(C.scws_add_dict(this.root,ctext,C.int(mode)))
    if ret != 0 {
        return errors.New(fmt.Sprintf("Add Dict [%s] Fail",fPath))
    }
    return nil
}

// 清除并设定当前scws 操作所有的词典文件
func (this *Scws) SetDict(fPath string, mode int) error {
    ctext := C.CString(fPath)
    defer C.free(unsafe.Pointer(ctext))
    ret := int(C.scws_set_dict(this.root,ctext,C.int(mode)))
    if ret != 0 {
        return errors.New(fmt.Sprintf("Set Dict [%s] Fail",fPath))
    }
    return nil
}

// 设定规则集文件
func (this *Scws) SetRule(fPath string) error {
    ctext := C.CString(fPath)
    defer C.free(unsafe.Pointer(ctext))
    C.scws_set_rule(this.root,ctext)
    if this.root.r == nil{
        return errors.New(fmt.Sprintf("Set Rule [%s] Fail",fPath))
    }
    return nil
}

// 设定分词结果是否忽略所有的标点等特殊符号(不会忽略\r和\n)
// 参数yes 1 表示忽略,0 表示不忽略,缺省情况为不忽略
func (this *Scws) SetIgnore(yes int) {
    C.scws_set_ignore(this.root,C.int(yes))
}

// 设定分词执行时是否执行针对长词复合切分。（例：“中国人”分为“中国”、“人”、“中国人”）。
// 参数mode复合分词法的级别,缺省不复合分词.取值由下面几个常量异或组合:
// SCWS_MULTI_SHORT 短词
// SCWS_MULTI_DUALITY 二元（将相邻的2个单字组合成一个词）
// SCWS_MULTI_ZMAIN 重要单字
// SCWS_MULTI_ZALL 全部单字
func (this *Scws) SetMulti(mode int) {
    C.scws_set_multi(this.root,C.int(mode))
}

// 设定是否将闲散文字自动以二字分词法聚合。
// 参数yes 如果为1 表示执行二分聚合,0 表示不处理,缺省为0
func (this *Scws) SetDuality(yes int) {
    C.scws_set_duality(this.root,C.int(yes))
}

// 未实现,若有需要加上
// scws_get_tops
// scws_free_tops
// scws_has_word
// scws_get_words

func (this *Scws) Segment(text string)([]ScwsRes,error) {
    if this.forkScws == nil {
        return nil,errors.New("必须在非并发情况下调用一次Scws.Init")
    }
    // 分词结果数组
    scwsResult := make([]ScwsRes,0)

    // 从队列取一个用
    tmpScws := <-this.forkScws

    ctext := C.CString(text)
    defer C.free(unsafe.Pointer(ctext))

    C.scws_send_text(tmpScws,ctext,C.int(len(text)))
    res := C.scws_get_result(tmpScws)
    for res != nil {

        cur := res
        for cur != nil {
            attr := (*C.char)(unsafe.Pointer(&cur.attr[0]))
            scwsResult = append(scwsResult,ScwsRes{
                Term : C.GoStringN(C.CharOff2String(ctext,cur.off),C.int(cur.len)),
                Idf : float64(cur.idf),
                Attr : C.GoStringN(attr,C.int(C.strlen(attr))) })
            cur = cur.next
        }

        // 释放这个结果,获取下个结果
        C.scws_free_result(res)
        res = C.scws_get_result(tmpScws)
    }

    // 用完放入队列
    this.forkScws<-tmpScws

    return scwsResult,nil
}

// 释放Scws的全部资源.
func (this *Scws) Free() (error) {
    if this.forkScws != nil {
        close(this.forkScws)
        for s := range this.forkScws {
            C.scws_free(s)
        }
    }

    if this.root != nil {
        C.scws_free(this.root)
        this.root = nil
    }

    return nil
}

// 内部复制多个scws实例,并发调用Segment的时候可以并发切词.
// 否则Segment也能正常使用,只是所有切词都是串行执行.
func (this *Scws) Init(count int) (error) {
    if this.forkScws != nil {
        return errors.New("Scws.Init只允许调用一次")
    }
    if count < 1 {
        return errors.New("不能少于1个实例")
    }
    this.forkScws = make(chan C.scws_t,count)

    for i:=0;i<count;i++ {
        tmp := C.scws_fork(this.root)
        if tmp != nil {
            this.forkScws<-tmp
        }
    }
    if len(this.forkScws) != count {
        return errors.New("内存不足导致fork数量不符合预期")
    }
    return nil
}

// Scws构造函数
func NewScws() (*Scws) {
    s := &Scws{}
    s.root = C.scws_new()
    s.forkScws = nil
    return s
}
<|endoftext|>


<commit_before>// rabbitmq provides a concrete client implementation using
// rabbitmq / amqp as a message bus

package client

import (
	"fmt"
	"os"
	"sync"
	"time"

	log "github.com/cihub/seelog"
	"github.com/golang/protobuf/proto"
	uuid "github.com/nu7hatch/gouuid"
	"github.com/streadway/amqp"
	"golang.org/x/net/context"

	"github.com/b2aio/typhon/errors"
	pe "github.com/b2aio/typhon/proto/error"
	"github.com/b2aio/typhon/rabbit"
)

type RabbitClient struct {
	once       sync.Once
	inflight   *inflightRegistry
	replyTo    string
	connection *rabbit.RabbitConnection
}

var NewRabbitClient = func() Client {
	uuidQueue, err := uuid.NewV4()
	if err != nil {
		log.Criticalf("[Client] Failed to create UUID for reply queue")
		os.Exit(1)
	}
	return &RabbitClient{
		inflight:   newInflightRegistry(),
		connection: rabbit.NewRabbitConnection(),
		replyTo:    fmt.Sprintf("replyTo-%s", uuidQueue.String()),
	}
}

func (c *RabbitClient) Init() {
	select {
	case <-c.connection.Init():
		log.Info("[Client] Connected to RabbitMQ")
	case <-time.After(10 * time.Second):
		log.Critical("[Client] Failed to connect to RabbitMQ")
		os.Exit(1)
	}
	c.initConsume()
}

func (c *RabbitClient) initConsume() {
	err := c.connection.Channel.DeclareReplyQueue(c.replyTo)
	if err != nil {
		log.Critical("[Client] Failed to declare reply queue")
		log.Critical(err.Error())
		os.Exit(1)
	}
	deliveries, err := c.connection.Channel.ConsumeQueue(c.replyTo)
	if err != nil {
		log.Critical("[Client] Failed to consume from reply queue")
		log.Critical(err.Error())
		os.Exit(1)
	}
	go func() {
		log.Infof("[Client] Listening for deliveries on %s", c.replyTo)
		for delivery := range deliveries {
			go c.handleDelivery(delivery)
		}
	}()
}

func (c *RabbitClient) handleDelivery(delivery amqp.Delivery) {
	channel := c.inflight.pop(delivery.CorrelationId)
	if channel == nil {
		log.Errorf("[Client] CorrelationID '%s' does not exist in inflight registry", delivery.CorrelationId)
		return
	}
	select {
	case channel <- delivery:
	default:
		log.Errorf("[Client] Error in delivery for correlation %s", delivery.CorrelationId)
	}
}

func (c *RabbitClient) Call(ctx context.Context, serviceName, endpoint string, req proto.Message, resp proto.Message) error {

	// Ensure we're initialised, but only do this once
	//
	// @todo we need a connection loop here where we check if we're connected,
	// and if not, block for a short period of time while attempting to reconnect
	c.once.Do(c.Init)

	routingKey := c.buildRoutingKey(serviceName, endpoint)

	correlation, err := uuid.NewV4()
	if err != nil {
		log.Errorf("[Client] Failed to create unique request id: %v", err)
		return errors.InternalService("request.uuid", err.Error())
	}

	replyChannel := c.inflight.push(correlation.String())
	defer close(replyChannel)

	requestBody, err := proto.Marshal(req)
	if err != nil {
		log.Errorf("[Client] Failed to marshal request: %v", err)
		return errors.BadRequest("request.marshal", err.Error())
	}

	message := amqp.Publishing{
		CorrelationId: correlation.String(),
		Timestamp:     time.Now().UTC(),
		Body:          requestBody,
		ReplyTo:       c.replyTo,
	}

	err = c.connection.Publish(rabbit.Exchange, routingKey, message)
	if err != nil {
		log.Errorf("[Client] Failed to publish to '%s': %v", routingKey, err)
		return errors.InternalService("request.publish", err.Error())
	}

	select {
	case delivery := <-replyChannel:
		return handleResponse(delivery, resp)
	case <-time.After(defaultTimeout):
		e := fmt.Errorf("Timeout caling %v")
		log.Warnf("[Client] %v", e)
		return errors.Timeout(fmt.Sprintf("%s.timeout", routingKey), e.Error())
	}
}

func (c *RabbitClient) buildRoutingKey(serviceName, endpoint string) string {
	return fmt.Sprintf("%s.%s", serviceName, endpoint)
}

// handleResponse returned from a service by marshaling into the response type,
// or converting an error from the remote service
func handleResponse(delivery amqp.Delivery, resp proto.Message) error {
	// deal with error responses, by converting back from wire format
	if deliveryIsError(delivery) {
		p := &pe.Error{}
		if err := proto.Unmarshal(delivery.Body, p); err != nil {
			return errors.BadResponse("response.unmarshal", err.Error())
		}

		return errors.Unmarshal(p)
	}

	// Otherwise try to marshal to the expected response type
	if err := proto.Unmarshal(delivery.Body, resp); err != nil {
		return errors.BadResponse("response.unmarshal", err.Error())
	}

	return nil
}

// deliveryIsError checks if the delivered response contains an error
func deliveryIsError(delivery amqp.Delivery) bool {
	encoding, ok := delivery.Headers["Content-Encoding"].(string)
	if !ok {
		// Can't type assert header to string, assume error
		log.Warnf("Service returned invalid Content-Encoding header %v", encoding)
		return true
	}

	if encoding == "" || encoding == "ERROR" {
		return true
	}

	return false
}
<commit_msg>Don't close reply channels to prevent panics<commit_after>// rabbitmq provides a concrete client implementation using
// rabbitmq / amqp as a message bus

package client

import (
	"fmt"
	"os"
	"sync"
	"time"

	log "github.com/cihub/seelog"
	"github.com/golang/protobuf/proto"
	uuid "github.com/nu7hatch/gouuid"
	"github.com/streadway/amqp"
	"golang.org/x/net/context"

	"github.com/b2aio/typhon/errors"
	pe "github.com/b2aio/typhon/proto/error"
	"github.com/b2aio/typhon/rabbit"
)

type RabbitClient struct {
	once       sync.Once
	inflight   *inflightRegistry
	replyTo    string
	connection *rabbit.RabbitConnection
}

var NewRabbitClient = func() Client {
	uuidQueue, err := uuid.NewV4()
	if err != nil {
		log.Criticalf("[Client] Failed to create UUID for reply queue")
		os.Exit(1)
	}
	return &RabbitClient{
		inflight:   newInflightRegistry(),
		connection: rabbit.NewRabbitConnection(),
		replyTo:    fmt.Sprintf("replyTo-%s", uuidQueue.String()),
	}
}

func (c *RabbitClient) Init() {
	select {
	case <-c.connection.Init():
		log.Info("[Client] Connected to RabbitMQ")
	case <-time.After(10 * time.Second):
		log.Critical("[Client] Failed to connect to RabbitMQ")
		os.Exit(1)
	}
	c.initConsume()
}

func (c *RabbitClient) initConsume() {
	err := c.connection.Channel.DeclareReplyQueue(c.replyTo)
	if err != nil {
		log.Critical("[Client] Failed to declare reply queue")
		log.Critical(err.Error())
		os.Exit(1)
	}
	deliveries, err := c.connection.Channel.ConsumeQueue(c.replyTo)
	if err != nil {
		log.Critical("[Client] Failed to consume from reply queue")
		log.Critical(err.Error())
		os.Exit(1)
	}
	go func() {
		log.Infof("[Client] Listening for deliveries on %s", c.replyTo)
		for delivery := range deliveries {
			go c.handleDelivery(delivery)
		}
	}()
}

func (c *RabbitClient) handleDelivery(delivery amqp.Delivery) {
	channel := c.inflight.pop(delivery.CorrelationId)
	if channel == nil {
		log.Errorf("[Client] CorrelationID '%s' does not exist in inflight registry", delivery.CorrelationId)
		return
	}
	select {
	case channel <- delivery:
	default:
		log.Errorf("[Client] Error in delivery for correlation %s", delivery.CorrelationId)
	}
}

func (c *RabbitClient) Call(ctx context.Context, serviceName, endpoint string, req proto.Message, resp proto.Message) error {

	// Ensure we're initialised, but only do this once
	//
	// @todo we need a connection loop here where we check if we're connected,
	// and if not, block for a short period of time while attempting to reconnect
	c.once.Do(c.Init)

	routingKey := c.buildRoutingKey(serviceName, endpoint)

	correlation, err := uuid.NewV4()
	if err != nil {
		log.Errorf("[Client] Failed to create unique request id: %v", err)
		return errors.InternalService("request.uuid", err.Error())
	}

	replyChannel := c.inflight.push(correlation.String())

	requestBody, err := proto.Marshal(req)
	if err != nil {
		log.Errorf("[Client] Failed to marshal request: %v", err)
		return errors.BadRequest("request.marshal", err.Error())
	}

	message := amqp.Publishing{
		CorrelationId: correlation.String(),
		Timestamp:     time.Now().UTC(),
		Body:          requestBody,
		ReplyTo:       c.replyTo,
	}

	err = c.connection.Publish(rabbit.Exchange, routingKey, message)
	if err != nil {
		log.Errorf("[Client] Failed to publish to '%s': %v", routingKey, err)
		return errors.InternalService("request.publish", err.Error())
	}

	select {
	case delivery := <-replyChannel:
		return handleResponse(delivery, resp)
	case <-time.After(defaultTimeout):
		e := fmt.Errorf("Timeout caling %v")
		log.Warnf("[Client] %v", e)
		return errors.Timeout(fmt.Sprintf("%s.timeout", routingKey), e.Error())
	}
}

func (c *RabbitClient) buildRoutingKey(serviceName, endpoint string) string {
	return fmt.Sprintf("%s.%s", serviceName, endpoint)
}

// handleResponse returned from a service by marshaling into the response type,
// or converting an error from the remote service
func handleResponse(delivery amqp.Delivery, resp proto.Message) error {
	// deal with error responses, by converting back from wire format
	if deliveryIsError(delivery) {
		p := &pe.Error{}
		if err := proto.Unmarshal(delivery.Body, p); err != nil {
			return errors.BadResponse("response.unmarshal", err.Error())
		}

		return errors.Unmarshal(p)
	}

	// Otherwise try to marshal to the expected response type
	if err := proto.Unmarshal(delivery.Body, resp); err != nil {
		return errors.BadResponse("response.unmarshal", err.Error())
	}

	return nil
}

// deliveryIsError checks if the delivered response contains an error
func deliveryIsError(delivery amqp.Delivery) bool {
	encoding, ok := delivery.Headers["Content-Encoding"].(string)
	if !ok {
		// Can't type assert header to string, assume error
		log.Warnf("Service returned invalid Content-Encoding header %v", encoding)
		return true
	}

	if encoding == "" || encoding == "ERROR" {
		return true
	}

	return false
}
<|endoftext|>


<commit_before>package rrdtool

import (
	"errors"
	"fmt"
	"log"
	"net"
	"net/rpc"
	"os"
	"sync/atomic"
	"time"

	pfc "github.com/niean/goperfcounter"
	"stathat.com/c/consistent"

	cmodel "github.com/open-falcon/common/model"
	"github.com/open-falcon/graph/g"
	"github.com/open-falcon/graph/store"
)

const (
	_ = iota
	NET_TASK_M_SEND
	NET_TASK_M_QUERY
	NET_TASK_M_PULL
)

type Net_task_t struct {
	Method int
	Key    string
	Done   chan error
	Args   interface{}
	Reply  interface{}
}

const (
	FETCH_S_SUCCESS = iota
	FETCH_S_ERR
	FETCH_S_ISNOTEXIST
	SEND_S_SUCCESS
	SEND_S_ERR
	QUERY_S_SUCCESS
	QUERY_S_ERR
	CONN_S_ERR
	CONN_S_DIAL
	STAT_SIZE
)

var (
	Consistent       *consistent.Consistent
	Net_task_ch      map[string]chan *Net_task_t
	clients          map[string][]*rpc.Client
	flushrrd_timeout int32
	stat_cnt         [STAT_SIZE]uint64
)

func init() {
	Consistent = consistent.New()
	Net_task_ch = make(map[string]chan *Net_task_t)
	clients = make(map[string][]*rpc.Client)
}

func GetCounter() (ret string) {
	return fmt.Sprintf("FETCH_S_SUCCESS[%d] FETCH_S_ERR[%d] FETCH_S_ISNOTEXIST[%d] SEND_S_SUCCESS[%d] SEND_S_ERR[%d] QUERY_S_SUCCESS[%d] QUERY_S_ERR[%d] CONN_S_ERR[%d] CONN_S_DIAL[%d]",
		atomic.LoadUint64(&stat_cnt[FETCH_S_SUCCESS]),
		atomic.LoadUint64(&stat_cnt[FETCH_S_ERR]),
		atomic.LoadUint64(&stat_cnt[FETCH_S_ISNOTEXIST]),
		atomic.LoadUint64(&stat_cnt[SEND_S_SUCCESS]),
		atomic.LoadUint64(&stat_cnt[SEND_S_ERR]),
		atomic.LoadUint64(&stat_cnt[QUERY_S_SUCCESS]),
		atomic.LoadUint64(&stat_cnt[QUERY_S_ERR]),
		atomic.LoadUint64(&stat_cnt[CONN_S_ERR]),
		atomic.LoadUint64(&stat_cnt[CONN_S_DIAL]))
}

func dial(address string, timeout time.Duration) (*rpc.Client, error) {
	d := net.Dialer{Timeout: timeout}
	conn, err := d.Dial("tcp", address)
	if err != nil {
		return nil, err
	}
	if tc, ok := conn.(*net.TCPConn); ok {
		if err := tc.SetKeepAlive(true); err != nil {
			conn.Close()
			return nil, err
		}
	}
	return rpc.NewClient(conn), err
}

func migrate_start(cfg *g.GlobalConfig) {
	var err error
	var i int
	if cfg.Migrate.Enabled {
		Consistent.NumberOfReplicas = cfg.Migrate.Replicas

		for node, addr := range cfg.Migrate.Cluster {
			Consistent.Add(node)
			Net_task_ch[node] = make(chan *Net_task_t, 16)
			clients[node] = make([]*rpc.Client, cfg.Migrate.Concurrency)

			for i = 0; i < cfg.Migrate.Concurrency; i++ {
				if clients[node][i], err = dial(addr, time.Second); err != nil {
					log.Fatalf("node:%s addr:%s err:%s\n", node, addr, err)
				}
				go net_task_worker(i, Net_task_ch[node], &clients[node][i], addr)
			}
		}
	}
}

func net_task_worker(idx int, ch chan *Net_task_t, client **rpc.Client, addr string) {
	var err error
	for {
		select {
		case task := <-ch:
			if task.Method == NET_TASK_M_SEND {
				if err = send_data(client, task.Key, addr); err != nil {
					pfc.Meter("migrate.send.err", 1)
					atomic.AddUint64(&stat_cnt[SEND_S_ERR], 1)
				} else {
					pfc.Meter("migrate.send.ok", 1)
					atomic.AddUint64(&stat_cnt[SEND_S_SUCCESS], 1)
				}
			} else if task.Method == NET_TASK_M_QUERY {
				if err = query_data(client, addr, task.Args, task.Reply); err != nil {
					pfc.Meter("migrate.query.err", 1)
					atomic.AddUint64(&stat_cnt[QUERY_S_ERR], 1)
				} else {
					pfc.Meter("migrate.query.ok", 1)
					atomic.AddUint64(&stat_cnt[QUERY_S_SUCCESS], 1)
				}
			} else if task.Method == NET_TASK_M_PULL {
				if atomic.LoadInt32(&flushrrd_timeout) != 0 {
					// hope this more faster than fetch_rrd
					if err = send_data(client, task.Key, addr); err != nil {
						pfc.Meter("migrate.sendbusy.err", 1)
						atomic.AddUint64(&stat_cnt[SEND_S_ERR], 1)
					} else {
						pfc.Meter("migrate.sendbusy.ok", 1)
						atomic.AddUint64(&stat_cnt[SEND_S_SUCCESS], 1)
					}
				} else {
					if err = fetch_rrd(client, task.Key, addr); err != nil {
						if os.IsNotExist(err) {
							pfc.Meter("migrate.scprrd.null", 1)
							//文件不存在时，直接将缓存数据刷入本地
							atomic.AddUint64(&stat_cnt[FETCH_S_ISNOTEXIST], 1)
							store.GraphItems.SetFlag(task.Key, 0)
							CommitByKey(task.Key)
						} else {
							pfc.Meter("migrate.scprrd.err", 1)
							//warning:其他异常情况，缓存数据会堆积
							atomic.AddUint64(&stat_cnt[FETCH_S_ERR], 1)
						}
					} else {
						pfc.Meter("migrate.scprrd.ok", 1)
						atomic.AddUint64(&stat_cnt[FETCH_S_SUCCESS], 1)
					}
				}
			} else {
				err = errors.New("error net task method")
			}
			if task.Done != nil {
				task.Done <- err
			}
		}
	}
}

// TODO addr to node
func reconnection(client **rpc.Client, addr string) {
	pfc.Meter("migrate.reconnection."+addr, 1)

	var err error

	atomic.AddUint64(&stat_cnt[CONN_S_ERR], 1)
	if *client != nil {
		(*client).Close()
	}

	*client, err = dial(addr, time.Second)
	atomic.AddUint64(&stat_cnt[CONN_S_DIAL], 1)

	for err != nil {
		//danger!! block routine
		time.Sleep(time.Millisecond * 500)
		*client, err = dial(addr, time.Second)
		atomic.AddUint64(&stat_cnt[CONN_S_DIAL], 1)
	}
}

func query_data(client **rpc.Client, addr string,
	args interface{}, resp interface{}) error {
	var (
		err error
		i   int
	)

	for i = 0; i < 3; i++ {
		err = rpc_call(*client, "Graph.Query", args, resp,
			time.Duration(g.Config().CallTimeout)*time.Millisecond)

		if err == nil {
			break
		}
		if err == rpc.ErrShutdown {
			reconnection(client, addr)
		}
	}
	return err
}

func send_data(client **rpc.Client, key string, addr string) error {
	var (
		err  error
		flag uint32
		resp *cmodel.SimpleRpcResponse
		i    int
	)

	//remote
	if flag, err = store.GraphItems.GetFlag(key); err != nil {
		return err
	}
	cfg := g.Config()

	store.GraphItems.SetFlag(key, flag|g.GRAPH_F_SENDING)

	items := store.GraphItems.PopAll(key)
	items_size := len(items)
	if items_size == 0 {
		goto out
	}
	resp = &cmodel.SimpleRpcResponse{}

	for i = 0; i < 3; i++ {
		err = rpc_call(*client, "Graph.Send", items, resp,
			time.Duration(cfg.CallTimeout)*time.Millisecond)

		if err == nil {
			goto out
		}
		if err == rpc.ErrShutdown {
			reconnection(client, addr)
		}
	}
	// err
	store.GraphItems.PushAll(key, items)
	//flag |= g.GRAPH_F_ERR
out:
	flag &= ^g.GRAPH_F_SENDING
	store.GraphItems.SetFlag(key, flag)
	return err

}

func fetch_rrd(client **rpc.Client, key string, addr string) error {
	var (
		err      error
		flag     uint32
		md5      string
		dsType   string
		filename string
		step, i  int
		rrdfile  g.File
	)

	cfg := g.Config()

	if flag, err = store.GraphItems.GetFlag(key); err != nil {
		return err
	}

	store.GraphItems.SetFlag(key, flag|g.GRAPH_F_FETCHING)

	md5, dsType, step, _ = g.SplitRrdCacheKey(key)
	filename = g.RrdFileName(cfg.RRD.Storage, md5, dsType, step)

	for i = 0; i < 3; i++ {
		err = rpc_call(*client, "Graph.GetRrd", key, &rrdfile,
			time.Duration(cfg.CallTimeout)*time.Millisecond)

		if err == nil {
			done := make(chan error, 1)
			io_task_chan <- &io_task_t{
				method: IO_TASK_M_WRITE,
				args: &g.File{
					Filename: filename,
					Body:     rrdfile.Body[:],
				},
				done: done,
			}
			if err = <-done; err != nil {
				goto out
			} else {
				flag &= ^g.GRAPH_F_MISS
				goto out
			}
		} else {
			log.Println(err)
		}
		if err == rpc.ErrShutdown {
			reconnection(client, addr)
		}
	}
out:
	flag &= ^g.GRAPH_F_FETCHING
	store.GraphItems.SetFlag(key, flag)
	return err
}

func rpc_call(client *rpc.Client, method string, args interface{},
	reply interface{}, timeout time.Duration) error {
	done := make(chan *rpc.Call, 1)
	client.Go(method, args, reply, done)
	select {
	case <-time.After(timeout):
		return errors.New("i/o timeout[rpc]")
	case call := <-done:
		if call.Error == nil {
			return nil
		} else {
			return call.Error
		}
	}
}
<commit_msg>fix consistent package import path<commit_after>package rrdtool

import (
	"errors"
	"fmt"
	"log"
	"net"
	"net/rpc"
	"os"
	"sync/atomic"
	"time"

	pfc "github.com/niean/goperfcounter"
	"github.com/toolkits/consistent"

	cmodel "github.com/open-falcon/common/model"
	"github.com/open-falcon/graph/g"
	"github.com/open-falcon/graph/store"
)

const (
	_ = iota
	NET_TASK_M_SEND
	NET_TASK_M_QUERY
	NET_TASK_M_PULL
)

type Net_task_t struct {
	Method int
	Key    string
	Done   chan error
	Args   interface{}
	Reply  interface{}
}

const (
	FETCH_S_SUCCESS = iota
	FETCH_S_ERR
	FETCH_S_ISNOTEXIST
	SEND_S_SUCCESS
	SEND_S_ERR
	QUERY_S_SUCCESS
	QUERY_S_ERR
	CONN_S_ERR
	CONN_S_DIAL
	STAT_SIZE
)

var (
	Consistent       *consistent.Consistent
	Net_task_ch      map[string]chan *Net_task_t
	clients          map[string][]*rpc.Client
	flushrrd_timeout int32
	stat_cnt         [STAT_SIZE]uint64
)

func init() {
	Consistent = consistent.New()
	Net_task_ch = make(map[string]chan *Net_task_t)
	clients = make(map[string][]*rpc.Client)
}

func GetCounter() (ret string) {
	return fmt.Sprintf("FETCH_S_SUCCESS[%d] FETCH_S_ERR[%d] FETCH_S_ISNOTEXIST[%d] SEND_S_SUCCESS[%d] SEND_S_ERR[%d] QUERY_S_SUCCESS[%d] QUERY_S_ERR[%d] CONN_S_ERR[%d] CONN_S_DIAL[%d]",
		atomic.LoadUint64(&stat_cnt[FETCH_S_SUCCESS]),
		atomic.LoadUint64(&stat_cnt[FETCH_S_ERR]),
		atomic.LoadUint64(&stat_cnt[FETCH_S_ISNOTEXIST]),
		atomic.LoadUint64(&stat_cnt[SEND_S_SUCCESS]),
		atomic.LoadUint64(&stat_cnt[SEND_S_ERR]),
		atomic.LoadUint64(&stat_cnt[QUERY_S_SUCCESS]),
		atomic.LoadUint64(&stat_cnt[QUERY_S_ERR]),
		atomic.LoadUint64(&stat_cnt[CONN_S_ERR]),
		atomic.LoadUint64(&stat_cnt[CONN_S_DIAL]))
}

func dial(address string, timeout time.Duration) (*rpc.Client, error) {
	d := net.Dialer{Timeout: timeout}
	conn, err := d.Dial("tcp", address)
	if err != nil {
		return nil, err
	}
	if tc, ok := conn.(*net.TCPConn); ok {
		if err := tc.SetKeepAlive(true); err != nil {
			conn.Close()
			return nil, err
		}
	}
	return rpc.NewClient(conn), err
}

func migrate_start(cfg *g.GlobalConfig) {
	var err error
	var i int
	if cfg.Migrate.Enabled {
		Consistent.NumberOfReplicas = cfg.Migrate.Replicas

		for node, addr := range cfg.Migrate.Cluster {
			Consistent.Add(node)
			Net_task_ch[node] = make(chan *Net_task_t, 16)
			clients[node] = make([]*rpc.Client, cfg.Migrate.Concurrency)

			for i = 0; i < cfg.Migrate.Concurrency; i++ {
				if clients[node][i], err = dial(addr, time.Second); err != nil {
					log.Fatalf("node:%s addr:%s err:%s\n", node, addr, err)
				}
				go net_task_worker(i, Net_task_ch[node], &clients[node][i], addr)
			}
		}
	}
}

func net_task_worker(idx int, ch chan *Net_task_t, client **rpc.Client, addr string) {
	var err error
	for {
		select {
		case task := <-ch:
			if task.Method == NET_TASK_M_SEND {
				if err = send_data(client, task.Key, addr); err != nil {
					pfc.Meter("migrate.send.err", 1)
					atomic.AddUint64(&stat_cnt[SEND_S_ERR], 1)
				} else {
					pfc.Meter("migrate.send.ok", 1)
					atomic.AddUint64(&stat_cnt[SEND_S_SUCCESS], 1)
				}
			} else if task.Method == NET_TASK_M_QUERY {
				if err = query_data(client, addr, task.Args, task.Reply); err != nil {
					pfc.Meter("migrate.query.err", 1)
					atomic.AddUint64(&stat_cnt[QUERY_S_ERR], 1)
				} else {
					pfc.Meter("migrate.query.ok", 1)
					atomic.AddUint64(&stat_cnt[QUERY_S_SUCCESS], 1)
				}
			} else if task.Method == NET_TASK_M_PULL {
				if atomic.LoadInt32(&flushrrd_timeout) != 0 {
					// hope this more faster than fetch_rrd
					if err = send_data(client, task.Key, addr); err != nil {
						pfc.Meter("migrate.sendbusy.err", 1)
						atomic.AddUint64(&stat_cnt[SEND_S_ERR], 1)
					} else {
						pfc.Meter("migrate.sendbusy.ok", 1)
						atomic.AddUint64(&stat_cnt[SEND_S_SUCCESS], 1)
					}
				} else {
					if err = fetch_rrd(client, task.Key, addr); err != nil {
						if os.IsNotExist(err) {
							pfc.Meter("migrate.scprrd.null", 1)
							//文件不存在时，直接将缓存数据刷入本地
							atomic.AddUint64(&stat_cnt[FETCH_S_ISNOTEXIST], 1)
							store.GraphItems.SetFlag(task.Key, 0)
							CommitByKey(task.Key)
						} else {
							pfc.Meter("migrate.scprrd.err", 1)
							//warning:其他异常情况，缓存数据会堆积
							atomic.AddUint64(&stat_cnt[FETCH_S_ERR], 1)
						}
					} else {
						pfc.Meter("migrate.scprrd.ok", 1)
						atomic.AddUint64(&stat_cnt[FETCH_S_SUCCESS], 1)
					}
				}
			} else {
				err = errors.New("error net task method")
			}
			if task.Done != nil {
				task.Done <- err
			}
		}
	}
}

// TODO addr to node
func reconnection(client **rpc.Client, addr string) {
	pfc.Meter("migrate.reconnection."+addr, 1)

	var err error

	atomic.AddUint64(&stat_cnt[CONN_S_ERR], 1)
	if *client != nil {
		(*client).Close()
	}

	*client, err = dial(addr, time.Second)
	atomic.AddUint64(&stat_cnt[CONN_S_DIAL], 1)

	for err != nil {
		//danger!! block routine
		time.Sleep(time.Millisecond * 500)
		*client, err = dial(addr, time.Second)
		atomic.AddUint64(&stat_cnt[CONN_S_DIAL], 1)
	}
}

func query_data(client **rpc.Client, addr string,
	args interface{}, resp interface{}) error {
	var (
		err error
		i   int
	)

	for i = 0; i < 3; i++ {
		err = rpc_call(*client, "Graph.Query", args, resp,
			time.Duration(g.Config().CallTimeout)*time.Millisecond)

		if err == nil {
			break
		}
		if err == rpc.ErrShutdown {
			reconnection(client, addr)
		}
	}
	return err
}

func send_data(client **rpc.Client, key string, addr string) error {
	var (
		err  error
		flag uint32
		resp *cmodel.SimpleRpcResponse
		i    int
	)

	//remote
	if flag, err = store.GraphItems.GetFlag(key); err != nil {
		return err
	}
	cfg := g.Config()

	store.GraphItems.SetFlag(key, flag|g.GRAPH_F_SENDING)

	items := store.GraphItems.PopAll(key)
	items_size := len(items)
	if items_size == 0 {
		goto out
	}
	resp = &cmodel.SimpleRpcResponse{}

	for i = 0; i < 3; i++ {
		err = rpc_call(*client, "Graph.Send", items, resp,
			time.Duration(cfg.CallTimeout)*time.Millisecond)

		if err == nil {
			goto out
		}
		if err == rpc.ErrShutdown {
			reconnection(client, addr)
		}
	}
	// err
	store.GraphItems.PushAll(key, items)
	//flag |= g.GRAPH_F_ERR
out:
	flag &= ^g.GRAPH_F_SENDING
	store.GraphItems.SetFlag(key, flag)
	return err

}

func fetch_rrd(client **rpc.Client, key string, addr string) error {
	var (
		err      error
		flag     uint32
		md5      string
		dsType   string
		filename string
		step, i  int
		rrdfile  g.File
	)

	cfg := g.Config()

	if flag, err = store.GraphItems.GetFlag(key); err != nil {
		return err
	}

	store.GraphItems.SetFlag(key, flag|g.GRAPH_F_FETCHING)

	md5, dsType, step, _ = g.SplitRrdCacheKey(key)
	filename = g.RrdFileName(cfg.RRD.Storage, md5, dsType, step)

	for i = 0; i < 3; i++ {
		err = rpc_call(*client, "Graph.GetRrd", key, &rrdfile,
			time.Duration(cfg.CallTimeout)*time.Millisecond)

		if err == nil {
			done := make(chan error, 1)
			io_task_chan <- &io_task_t{
				method: IO_TASK_M_WRITE,
				args: &g.File{
					Filename: filename,
					Body:     rrdfile.Body[:],
				},
				done: done,
			}
			if err = <-done; err != nil {
				goto out
			} else {
				flag &= ^g.GRAPH_F_MISS
				goto out
			}
		} else {
			log.Println(err)
		}
		if err == rpc.ErrShutdown {
			reconnection(client, addr)
		}
	}
out:
	flag &= ^g.GRAPH_F_FETCHING
	store.GraphItems.SetFlag(key, flag)
	return err
}

func rpc_call(client *rpc.Client, method string, args interface{},
	reply interface{}, timeout time.Duration) error {
	done := make(chan *rpc.Call, 1)
	client.Go(method, args, reply, done)
	select {
	case <-time.After(timeout):
		return errors.New("i/o timeout[rpc]")
	case call := <-done:
		if call.Error == nil {
			return nil
		} else {
			return call.Error
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	capn "github.com/glycerine/go-capnproto"
	mdb "github.com/msackman/gomdb"
	mdbs "github.com/msackman/gomdb/server"
	"goshawkdb.io/common"
	"goshawkdb.io/server"
	msgs "goshawkdb.io/server/capnp"
	"goshawkdb.io/server/db"
	eng "goshawkdb.io/server/txnengine"
	"log"
	"os"
	"runtime"
	"time"
)

func main() {
	log.SetPrefix(common.ProductName + " ")
	log.SetFlags(log.Ldate | log.Ltime | log.Lmicroseconds)
	log.Println(os.Args)

	procs := runtime.NumCPU()
	if procs < 2 {
		procs = 2
	}
	runtime.GOMAXPROCS(procs)

	var vUUIdStr string
	flag.StringVar(&vUUIdStr, "var", "", "var to interrogate")
	flag.Parse()

	dirs := flag.Args()
	if len(dirs) == 0 {
		log.Fatal("No dirs supplied")
	}

	vars := make(map[common.VarUUId]*varstate)
	for _, d := range dirs {
		dir := d
		log.Printf("...loading from %v\n", dir)
		disk, err := mdbs.NewMDBServer(dir, 0, 0600, server.MDBInitialSize, 1, time.Millisecond, db.DB)
		if err != nil {
			log.Println(err)
			continue
		}
		loadVars(disk, vars)
		disk.Shutdown()
	}

	log.Printf("Found %v unique vars", len(vars))

	if vUUIdStr != "" {
		vUUId := common.MakeVarUUIdFromStr(vUUIdStr)
		if vUUId == nil {
			log.Printf("Unable to parse %v as vUUId\n", vUUIdStr)
		}
		if state, found := vars[*vUUId]; found {
			log.Println(state)
		} else {
			log.Printf("Unable to find %v\n", vUUId)
		}
	}
}

func loadVars(disk *mdbs.MDBServer, vars map[common.VarUUId]*varstate) {
	_, err := disk.ReadonlyTransaction(func(rtxn *mdbs.RTxn) interface{} {
		_, err := rtxn.WithCursor(db.DB.Vars, func(cursor *mdbs.Cursor) interface{} {
			key, data, err := cursor.Get(nil, nil, mdb.FIRST)
			for ; err == nil; key, data, err = cursor.Get(nil, nil, mdb.NEXT) {
				vUUId := common.MakeVarUUId(key)
				seg, _, err := capn.ReadFromMemoryZeroCopy(data)
				if err != nil {
					log.Printf("Error when decoding %v: %v\n", vUUId, err)
					continue
				}

				varCap := msgs.ReadRootVar(seg)
				pos := varCap.Positions()
				positions := (*common.Positions)(&pos)
				writeTxnId := common.MakeTxnId(varCap.WriteTxnId())
				writeTxnClock := eng.VectorClockFromCap(varCap.WriteTxnClock())
				writesClock := eng.VectorClockFromCap(varCap.WritesClock())

				if state, found := vars[*vUUId]; found {
					if err := state.matches(disk, writeTxnId, writeTxnClock, writesClock, positions); err != nil {
						log.Println(err)
					}
				} else {
					state = &varstate{
						vUUId:            vUUId,
						disks:            []*mdbs.MDBServer{disk},
						writeTxnId:       writeTxnId,
						writeTxnClock:    writeTxnClock,
						writeWritesClock: writesClock,
						positions:        positions,
					}
					vars[*vUUId] = state
				}
			}
			return nil
		})
		return err
	}).ResultError()
	if err != nil {
		log.Println(err)
	}
}

type varstate struct {
	vUUId            *common.VarUUId
	disks            []*mdbs.MDBServer
	writeTxnId       *common.TxnId
	writeTxnClock    *eng.VectorClock
	writeWritesClock *eng.VectorClock
	positions        *common.Positions
}

func (vs *varstate) matches(disk *mdbs.MDBServer, writeTxnId *common.TxnId, writeTxnClock, writesClock *eng.VectorClock, positions *common.Positions) error {
	if vs.writeTxnId.Compare(writeTxnId) != common.EQ {
		return fmt.Errorf("%v TxnId divergence: %v vs %v", vs.vUUId, vs.writeTxnId, writeTxnId)
	}
	if !vs.positions.Equal(positions) {
		return fmt.Errorf("%v positions divergence: %v vs %v", vs.vUUId, vs.positions, positions)
	}
	if !vs.writeTxnClock.Equal(writeTxnClock) {
		return fmt.Errorf("%v Txn %v Clock divergence: %v vs %v", vs.vUUId, vs.writeTxnId, vs.writeTxnClock, writeTxnClock)
	}
	if !vs.writeWritesClock.Equal(writesClock) {
		return fmt.Errorf("%v Txn %v WritesClock divergence: %v vs %v", vs.vUUId, vs.writeTxnId, vs.writeWritesClock, writesClock)
	}
	vs.disks = append(vs.disks, disk)
	return nil
}

func (vs *varstate) String() string {
	return fmt.Sprintf("%v found in %v stores:\n positions:\t%v\n writeTxnId:\t%v\n writeTxnClock:\t%v\n writesClock:\t%v\n", vs.vUUId, len(vs.disks), vs.positions, vs.writeTxnId, vs.writeTxnClock, vs.writeWritesClock)
}
<commit_msg>Reworking consistency checker to be way more useful and be able to verify that topology migrations work. Ref T6.<commit_after>package main

import (
	"encoding/binary"
	"flag"
	"fmt"
	capn "github.com/glycerine/go-capnproto"
	mdb "github.com/msackman/gomdb"
	mdbs "github.com/msackman/gomdb/server"
	"goshawkdb.io/common"
	"goshawkdb.io/server"
	msgs "goshawkdb.io/server/capnp"
	"goshawkdb.io/server/configuration"
	"goshawkdb.io/server/db"
	eng "goshawkdb.io/server/txnengine"
	"io/ioutil"
	"log"
	"os"
	"runtime"
	"time"
)

type store struct {
	dir      string
	disk     *mdbs.MDBServer
	rmId     common.RMId
	topology *configuration.Topology
}

type stores []*store

func main() {
	log.SetPrefix(common.ProductName + "ConsistencyChecker ")
	log.SetFlags(log.Ldate | log.Ltime | log.Lmicroseconds)
	log.Println(os.Args)

	procs := runtime.NumCPU()
	if procs < 2 {
		procs = 2
	}
	runtime.GOMAXPROCS(procs)

	dirs := flag.Args()
	if len(dirs) == 0 {
		log.Fatal("No dirs supplied")
	}

	stores := stores(make([]*store, 0, len(dirs)))
	defer stores.Shutdown()
	for _, d := range dirs {
		log.Printf("...loading from %v\n", d)
		store := &store{dir: d}
		var err error
		if err = store.LoadRMId(); err == nil {
			if err = store.StartDisk(); err == nil {
				err = store.LoadTopology()
			}
		}
		if err != nil {
			log.Println(err)
		}
	}

	if err := stores.CheckEqualTopology(); err != nil {
		log.Println(err)
		return
	}

}

func (ss stores) CheckEqualTopology() error {
	var first *store
	for idx, s := range ss {
		if idx == 0 {
			first = s
		} else if !first.topology.Configuration.Equal(s.topology.Configuration) {
			return fmt.Errorf("Unequal topologies: %v has %v; %v has %v",
				first, first.topology, s, s.topology)
		}
	}
	return nil
}

func (ss stores) IterateVars(f func(*varWrapper) error) error {
	is := &iterateState{
		stores:   ss,
		wrappers: make([]*varWrapper, 0, len(ss)),
		f:        f,
	}
	return is.init()
}

func (ss stores) Shutdown() {
	for _, s := range ss {
		s.Shutdown()
	}
}

func (s *store) Shutdown() {
	if s.disk == nil {
		return
	}
	s.disk.Shutdown()
	s.disk = nil
}

func (s *store) String() string {
	return fmt.Sprintf("%v(%v)", s.rmId, s.dir)
}

func (s *store) LoadRMId() error {
	rmIdBytes, err := ioutil.ReadFile(s.dir + "/rmid")
	if err != nil {
		return err
	}
	s.rmId = common.RMId(binary.BigEndian.Uint32(rmIdBytes))
	return nil
}

func (s *store) StartDisk() error {
	disk, err := mdbs.NewMDBServer(s.dir, 0, 0600, server.MDBInitialSize, 1, time.Millisecond, db.DB)
	if err != nil {
		return err
	}
	s.disk = disk
	return nil
}

func (s *store) LoadTopology() error {
	res, err := s.disk.ReadonlyTransaction(func(rtxn *mdbs.RTxn) interface{} {
		bites, err := rtxn.Get(db.DB.Vars, configuration.TopologyVarUUId[:])
		if err != nil {
			rtxn.Error(err)
			return nil
		}
		seg, _, err := capn.ReadFromMemoryZeroCopy(bites)
		if err != nil {
			rtxn.Error(err)
			return nil
		}
		varCap := msgs.ReadRootVar(seg)
		txnId := common.MakeTxnId(varCap.WriteTxnId())
		bites = db.ReadTxnBytesFromDisk(rtxn, txnId)
		if bites == nil {
			rtxn.Error(fmt.Errorf("Unable to find txn for topology: %v", txnId))
			return nil
		}
		seg, _, err = capn.ReadFromMemoryZeroCopy(bites)
		if err != nil {
			rtxn.Error(err)
			return nil
		}
		txnCap := msgs.ReadRootTxn(seg)
		actions := txnCap.Actions()
		if actions.Len() != 1 {
			rtxn.Error(fmt.Errorf("Topology txn has %v actions; expected 1", actions.Len()))
			return nil
		}
		action := actions.At(0)
		var refs msgs.VarIdPos_List
		switch action.Which() {
		case msgs.ACTION_WRITE:
			w := action.Write()
			bites = w.Value()
			refs = w.References()
		case msgs.ACTION_READWRITE:
			rw := action.Readwrite()
			bites = rw.Value()
			refs = rw.References()
		case msgs.ACTION_CREATE:
			c := action.Create()
			bites = c.Value()
			refs = c.References()
		default:
			rtxn.Error(fmt.Errorf("Expected topology txn action to be w, rw, or c; found %v", action.Which()))
			return nil
		}

		if refs.Len() != 1 {
			rtxn.Error(fmt.Errorf("Topology txn action has %v references; expected 1", refs.Len()))
			return nil
		}
		rootRef := refs.At(0)

		seg, _, err = capn.ReadFromMemoryZeroCopy(bites)
		if err != nil {
			rtxn.Error(err)
			return nil
		}
		topology, err := configuration.TopologyFromCap(txnId, &rootRef, bites)
		if err != nil {
			rtxn.Error(err)
			return nil
		}
		return topology
	}).ResultError()
	if err != nil {
		return err
	}
	s.topology = res.(*configuration.Topology)
	return nil
}

type varWrapper struct {
	*iterateState
	vUUId  *common.VarUUId
	rtxn   *mdbs.RTxn
	cursor *mdbs.Cursor
}

type iterateState struct {
	stores   stores
	wrappers []*varWrapper
	f        func(*varWrapper) error
}

func (is *iterateState) init() error {
	if len(is.stores) == len(is.wrappers) {
		return nil // actually do nextState
	}
	s := is.stores[len(is.wrappers)]
	_, err := s.disk.ReadonlyTransaction(func(rtxn *mdbs.RTxn) interface{} {
		rtxn.WithCursor(db.DB.Vars, func(cursor *mdbs.Cursor) interface{} {
			is.wrappers = append(is.wrappers, &varWrapper{
				iterateState: is,
				rtxn:         rtxn,
				cursor:       cursor,
			})
			return is.init()
		})
		return nil
	}).ResultError()
	return err

}
<|endoftext|>