<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package subnet

import (
	"encoding/json"
	"fmt"
	"strings"

	"github.com/r3labs/terraform/helper/schema"

	aes "github.com/ernestio/crypto/aes"
	"github.com/ernestio/ernestprovider/event"
	"github.com/ernestio/ernestprovider/providers/azure"
)

// Event : This is the Ernest representation of an azure subnet
type Event struct {
	event.Base
	ID                     string            `json:"id"`
	Name                   string            `json:"name" validate:"required"`
	ResourceGroupName      string            `json:"resource_group_name" validate:"required"`
	VirtualNetworkName     string            `json:"virtual_network_name" validate:"required"`
	AddressPrefix          string            `json:"address_prefix"  validate:"required"`
	NetworkSecurityGroup   string            `json:"network_security_group"`
	NetworkSecurityGroupID string            `json:"network_security_group_id"`
	RouteTable             string            `json:"route_table_id"`
	IPConfigurations       []string          `json:"ip_configurations"`
	ClientID               string            `json:"azure_client_id"`
	ClientSecret           string            `json:"azure_client_secret"`
	TenantID               string            `json:"azure_tenant_id"`
	SubscriptionID         string            `json:"azure_subscription_id"`
	Environment            string            `json:"environment"`
	ErrorMessage           string            `json:"error,omitempty"`
	Components             []json.RawMessage `json:"components"`
	CryptoKey              string            `json:"-"`
}

// New : Constructor
func New(subject, cryptoKey string, body []byte, val *event.Validator) (event.Event, error) {
	var ev azure.Resource
	ev = &Event{CryptoKey: cryptoKey}
	if err := json.Unmarshal(body, &ev); err != nil {
		err := fmt.Errorf("Error on input message : %s", err)
		return nil, err
	}

	return azure.New(subject, "azurerm_subnet", body, val, ev)
}

// SetComponents : ....
func (ev *Event) SetComponents(components []event.Event) {
	for _, v := range components {
		ev.Components = append(ev.Components, v.GetBody())
	}
}

// ValidateID : determines if the given id is valid for this resource type
func (ev *Event) ValidateID(id string) bool {
	parts := strings.Split(strings.ToLower(id), "/")
	if len(parts) != 11 {
		return false
	}
	if parts[6] != "microsoft.network" {
		return false
	}
	if parts[9] != "subnets" {
		return false
	}
	return true
}

// SetID : id setter
func (ev *Event) SetID(id string) {
	ev.ID = id
}

// GetID : id getter
func (ev *Event) GetID() string {
	return ev.ID
}

// SetState : state setter
func (ev *Event) SetState(state string) {
	ev.State = state
}

// ResourceDataToEvent : Translates a ResourceData on a valid Ernest Event
func (ev *Event) ResourceDataToEvent(d *schema.ResourceData) error {
	ev.Name = d.Get("name").(string)
	ev.ResourceGroupName = d.Get("resource_group_name").(string)
	ev.VirtualNetworkName = d.Get("virtual_network_name").(string)
	ev.AddressPrefix = d.Get("address_prefix").(string)
	ev.NetworkSecurityGroup = d.Get("network_security_group_id").(string)
	ev.RouteTable = d.Get("route_table_id").(string)

	configs := []string{}
	for _, config := range d.Get("ip_configurations").(*schema.Set).List() {
		configs = append(configs, config.(string))
	}
	ev.IPConfigurations = configs

	return nil
}

// EventToResourceData : Translates the current event on a valid ResourceData
func (ev *Event) EventToResourceData(d *schema.ResourceData) error {
	crypto := aes.New()

	encFields := make(map[string]string)
	encFields["subscription_id"] = ev.SubscriptionID
	encFields["client_id"] = ev.ClientID
	encFields["client_secret"] = ev.ClientSecret
	encFields["tenant_id"] = ev.TenantID
	encFields["environment"] = ev.Environment
	for k, v := range encFields {
		dec, err := crypto.Decrypt(v, ev.CryptoKey)
		if err != nil {
			err := fmt.Errorf("Field '%s' not valid : %s", k, err)
			ev.Log("error", err.Error())
			return err
		}
		if err := d.Set(k, dec); err != nil {
			err := fmt.Errorf("Field '%s' not valid : %s", k, err)
			ev.Log("error", err.Error())
			return err
		}
	}

	fields := make(map[string]interface{})
	fields["name"] = ev.Name
	fields["resource_group_name"] = ev.ResourceGroupName
	fields["virtual_network_name"] = ev.VirtualNetworkName
	fields["address_prefix"] = ev.AddressPrefix
	fields["network_security_group_id"] = ev.NetworkSecurityGroup
	fields["route_table_id"] = ev.RouteTable
	fields["ip_configurations"] = ev.IPConfigurations
	for k, v := range fields {
		if err := d.Set(k, v); err != nil {
			err := fmt.Errorf("Field '%s' not valid : %s", k, err)
			ev.Log("error", err.Error())
			return err
		}
	}

	return nil
}

// Error : will mark the event as errored
func (ev *Event) Error(err error) {
	ev.ErrorMessage = err.Error()
	ev.Body, err = json.Marshal(ev)
}
<commit_msg>fixed mapping of subnet security group id<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package subnet

import (
	"encoding/json"
	"fmt"
	"strings"

	"github.com/r3labs/terraform/helper/schema"

	aes "github.com/ernestio/crypto/aes"
	"github.com/ernestio/ernestprovider/event"
	"github.com/ernestio/ernestprovider/providers/azure"
)

// Event : This is the Ernest representation of an azure subnet
type Event struct {
	event.Base
	ID                     string            `json:"id"`
	Name                   string            `json:"name" validate:"required"`
	ResourceGroupName      string            `json:"resource_group_name" validate:"required"`
	VirtualNetworkName     string            `json:"virtual_network_name" validate:"required"`
	AddressPrefix          string            `json:"address_prefix"  validate:"required"`
	NetworkSecurityGroup   string            `json:"network_security_group"`
	NetworkSecurityGroupID string            `json:"network_security_group_id"`
	RouteTable             string            `json:"route_table_id"`
	IPConfigurations       []string          `json:"ip_configurations"`
	ClientID               string            `json:"azure_client_id"`
	ClientSecret           string            `json:"azure_client_secret"`
	TenantID               string            `json:"azure_tenant_id"`
	SubscriptionID         string            `json:"azure_subscription_id"`
	Environment            string            `json:"environment"`
	ErrorMessage           string            `json:"error,omitempty"`
	Components             []json.RawMessage `json:"components"`
	CryptoKey              string            `json:"-"`
}

// New : Constructor
func New(subject, cryptoKey string, body []byte, val *event.Validator) (event.Event, error) {
	var ev azure.Resource
	ev = &Event{CryptoKey: cryptoKey}
	if err := json.Unmarshal(body, &ev); err != nil {
		err := fmt.Errorf("Error on input message : %s", err)
		return nil, err
	}

	return azure.New(subject, "azurerm_subnet", body, val, ev)
}

// SetComponents : ....
func (ev *Event) SetComponents(components []event.Event) {
	for _, v := range components {
		ev.Components = append(ev.Components, v.GetBody())
	}
}

// ValidateID : determines if the given id is valid for this resource type
func (ev *Event) ValidateID(id string) bool {
	parts := strings.Split(strings.ToLower(id), "/")
	if len(parts) != 11 {
		return false
	}
	if parts[6] != "microsoft.network" {
		return false
	}
	if parts[9] != "subnets" {
		return false
	}
	return true
}

// SetID : id setter
func (ev *Event) SetID(id string) {
	ev.ID = id
}

// GetID : id getter
func (ev *Event) GetID() string {
	return ev.ID
}

// SetState : state setter
func (ev *Event) SetState(state string) {
	ev.State = state
}

// ResourceDataToEvent : Translates a ResourceData on a valid Ernest Event
func (ev *Event) ResourceDataToEvent(d *schema.ResourceData) error {
	ev.Name = d.Get("name").(string)
	ev.ResourceGroupName = d.Get("resource_group_name").(string)
	ev.VirtualNetworkName = d.Get("virtual_network_name").(string)
	ev.AddressPrefix = d.Get("address_prefix").(string)
	ev.NetworkSecurityGroup = d.Get("network_security_group_id").(string)
	ev.RouteTable = d.Get("route_table_id").(string)

	configs := []string{}
	for _, config := range d.Get("ip_configurations").(*schema.Set).List() {
		configs = append(configs, config.(string))
	}
	ev.IPConfigurations = configs

	return nil
}

// EventToResourceData : Translates the current event on a valid ResourceData
func (ev *Event) EventToResourceData(d *schema.ResourceData) error {
	crypto := aes.New()

	encFields := make(map[string]string)
	encFields["subscription_id"] = ev.SubscriptionID
	encFields["client_id"] = ev.ClientID
	encFields["client_secret"] = ev.ClientSecret
	encFields["tenant_id"] = ev.TenantID
	encFields["environment"] = ev.Environment
	for k, v := range encFields {
		dec, err := crypto.Decrypt(v, ev.CryptoKey)
		if err != nil {
			err := fmt.Errorf("Field '%s' not valid : %s", k, err)
			ev.Log("error", err.Error())
			return err
		}
		if err := d.Set(k, dec); err != nil {
			err := fmt.Errorf("Field '%s' not valid : %s", k, err)
			ev.Log("error", err.Error())
			return err
		}
	}

	fields := make(map[string]interface{})
	fields["name"] = ev.Name
	fields["resource_group_name"] = ev.ResourceGroupName
	fields["virtual_network_name"] = ev.VirtualNetworkName
	fields["address_prefix"] = ev.AddressPrefix
	fields["network_security_group_id"] = ev.NetworkSecurityGroupID
	fields["route_table_id"] = ev.RouteTable
	fields["ip_configurations"] = ev.IPConfigurations
	for k, v := range fields {
		if err := d.Set(k, v); err != nil {
			err := fmt.Errorf("Field '%s' not valid : %s", k, err)
			ev.Log("error", err.Error())
			return err
		}
	}

	return nil
}

// Error : will mark the event as errored
func (ev *Event) Error(err error) {
	ev.ErrorMessage = err.Error()
	ev.Body, err = json.Marshal(ev)
}
<|endoftext|>


<commit_before>package cluster

import (
	"encoding/json"
	"errors"
	"fmt"
	log "github.com/Sirupsen/logrus"
	"github.com/megamsys/libgo/cmd"
	"github.com/megamsys/opennebula-go/api"
	"github.com/megamsys/opennebula-go/compute"
	"github.com/megamsys/opennebula-go/virtualmachine"
	"net"
	"net/url"
	"strconv"
	"strings"
)

// CreateVM creates a vm in the specified node.
// It returns the vm, or an error, in case of failures.
const (
	START   = "start"
	STOP    = "stop"
	RESTART = "restart"
)

var ErrConnRefused = errors.New("connection refused")

func (c *Cluster) CreateVM(opts compute.VirtualMachine, t string) (string, string, string, error) {
	var (
		addr    string
		machine string
		vmid    string
		err     error
	)
	maxTries := 5
	for ; maxTries > 0; maxTries-- {

		nodlist, err := c.Nodes()

		for _, v := range nodlist {
			if v.Metadata[api.ONEZONE] == opts.Region {
				addr = v.Address
				opts.Vnets, opts.ClusterId = c.getVnets(v, opts.Vnets)
				if v.Metadata[api.VCPU_PERCENTAGE] != "" {
					opts.Cpu = cpuThrottle(v.Metadata[api.VCPU_PERCENTAGE], opts.Cpu)
				} else {
					opts.Cpu = cpuThrottle(t, opts.Cpu)
				}
			}
		}

		if addr == "" {
			return addr, machine, vmid, fmt.Errorf("%s", cmd.Colorfy("Unavailable nodes (hint: start or beat it).\n", "red", "", ""))
		}

		if err == nil {
			machine, vmid, err = c.createVMInNode(opts, addr)
			if err == nil {
				c.handleNodeSuccess(addr)
				break
			}
			log.Errorf("  > Trying... %s", addr)
		}
		shouldIncrementFailures := false
		isCreateMachineErr := false
		baseErr := err
		if nodeErr, ok := baseErr.(OneNodeError); ok {
			isCreateMachineErr = nodeErr.cmd == "createVM"
			baseErr = nodeErr.BaseError()
		}
		if urlErr, ok := baseErr.(*url.Error); ok {
			baseErr = urlErr.Err
		}
		_, isNetErr := baseErr.(*net.OpError)
		if isNetErr || isCreateMachineErr || baseErr == ErrConnRefused {
			shouldIncrementFailures = true
		}
		c.handleNodeError(addr, err, shouldIncrementFailures)
		return addr, machine, vmid, err
	}
	if err != nil {
		return addr, machine, vmid, fmt.Errorf("CreateVM: maximum number of tries exceeded, last error: %s", err.Error())
	}
	return addr, machine, vmid, err
}

//create a vm in a node.
func (c *Cluster) createVMInNode(opts compute.VirtualMachine, nodeAddress string) (string, string, error) {
	var vmid string
	node, err := c.getNodeByAddr(nodeAddress)
	if err != nil {
		return "", "", err
	}
	opts.TemplateName = node.template
	opts.T = node.Client

	res, err := opts.Create()
	b, err := json.Marshal(res)

	if err != nil {
		return "", "", err
	}
	str := string(b)
	spstr := strings.Split(str, ",")

	isSuccess, Err := strconv.ParseBool(spstr[0])
	if Err != nil {
		return "", "", Err
	}

	if !isSuccess {
   return "", "", wrapErrorWithCmd(node, errors.New(spstr[1]), "createVM")
	}

   vmid = spstr[1]
	return opts.Name, vmid, nil
}

func (c *Cluster) GetIpPort(opts virtualmachine.Vnc, region string) (string, string, error) {

	addr, err := c.getRegion(region)
	if err != nil {
		return "", "", err
	}

	node, err := c.getNodeByAddr(addr)
	if err != nil {
		return "", "", err
	}
	//opts.TemplateName = node.template
	opts.T = node.Client

	res, err := opts.GetVm()
	b, err := json.Marshal(res)

	if err != nil {
		return "", "", err
	}
	str := string(b)
	spstr := strings.Split(str, ",")

	isSuccess, Err := strconv.ParseBool(spstr[0])
	if Err != nil {
		return "", "", Err
	}

	if !isSuccess {
   	return "", "", wrapErrorWithCmd(node, errors.New(spstr[1]), "createVM")
	}
	vnchost := res.GetHostIp()
	vncport := res.GetPort()

	if err != nil {
		return "", "", wrapErrorWithCmd(node, err, "createVM")
	}

	return vnchost, vncport, nil
}

// DestroyVM kills a vm, returning an error in case of failure.
func (c *Cluster) DestroyVM(opts compute.VirtualMachine) error {

	addr, err := c.getRegion(opts.Region)
	if err != nil {
		return err
	}

	node, err := c.getNodeByAddr(addr)
	if err != nil {
		return err
	}
	opts.T = node.Client

	res, err := opts.Delete()
	if err != nil {
		return wrapError(node, err)
	}

	b, err := json.Marshal(res)

	if err != nil {
		return err
	}
	str := string(b)
	spstr := strings.Split(str, ",")

	isSuccess, Err := strconv.ParseBool(spstr[0])
	if Err != nil {
		return Err
	}

	if !isSuccess {
   return wrapErrorWithCmd(node, errors.New(spstr[1]), "createVM")
	}

	return nil
}

func (c *Cluster) VM(opts compute.VirtualMachine, action string) error {
	switch action {
	case START:
		return c.StartVM(opts)
	case STOP:
		return c.StopVM(opts)
	case RESTART:
		return c.RestartVM(opts)
	default:
		return nil
	}
}
func (c *Cluster) StartVM(opts compute.VirtualMachine) error {

	addr, err := c.getRegion(opts.Region)
	if err != nil {
		return err
	}

	node, err := c.getNodeByAddr(addr)
	if err != nil {
		return err
	}
	opts.T = node.Client

	res, err := opts.Resume()

	if err != nil {
		return wrapError(node, err)
	}

	b, err := json.Marshal(res)

	if err != nil {
		return err
	}
	str := string(b)
	spstr := strings.Split(str, ",")

	isSuccess, Err := strconv.ParseBool(spstr[0])
	if Err != nil {
		return Err
	}

	if !isSuccess {
   return wrapErrorWithCmd(node, errors.New(spstr[1]), "createVM")
	}

	return nil
}

func (c *Cluster) RestartVM(opts compute.VirtualMachine) error {

	addr, err := c.getRegion(opts.Region)
	if err != nil {
		return err
	}

	node, err := c.getNodeByAddr(addr)
	if err != nil {
		return err
	}
	opts.T = node.Client

	res, err := opts.Reboot()
	if err != nil {
		return wrapError(node, err)
	}

	b, err := json.Marshal(res)

	if err != nil {
		return err
	}
	str := string(b)
	spstr := strings.Split(str, ",")

	isSuccess, Err := strconv.ParseBool(spstr[0])
	if Err != nil {
		return Err
	}

	if !isSuccess {
   return wrapErrorWithCmd(node, errors.New(spstr[1]), "createVM")
	}

	return nil
}

func (c *Cluster) StopVM(opts compute.VirtualMachine) error {

	addr, err := c.getRegion(opts.Region)
	if err != nil {
		return err
	}

	node, err := c.getNodeByAddr(addr)
	if err != nil {
		return err
	}
	opts.T = node.Client

	res, err := opts.Poweroff()
	b, err := json.Marshal(res)

	if err != nil {
		return err
	}
	str := string(b)
	spstr := strings.Split(str, ",")

	isSuccess, Err := strconv.ParseBool(spstr[0])
	if Err != nil {
		return Err
	}

	if !isSuccess {
   return wrapErrorWithCmd(node, errors.New(spstr[1]), "createVM")
	}

	if err != nil {
		return wrapError(node, err)
	}
	return nil
}

func (c *Cluster) getNodeByAddr(addr string) (node, error) {
	return c.getNode(func(s Storage) (Node, error) {
		return s.RetrieveNode(addr)
	})
}

func (c *Cluster) SnapVMDisk(opts compute.VirtualMachine) error {

	addr, err := c.getRegion(opts.Region)
	if err != nil {
		return err
	}

	node, err := c.getNodeByAddr(addr)
	if err != nil {
		return err
	}
	opts.T = node.Client

	res, err := opts.DiskSnap()
	if err != nil {
		return wrapError(node, err)
	}

	b, err := json.Marshal(res)

	if err != nil {
		return err
	}
	str := string(b)
	spstr := strings.Split(str, ",")

	isSuccess, Err := strconv.ParseBool(spstr[0])
	if Err != nil {
		return  Err
	}

	if !isSuccess {
   return wrapErrorWithCmd(node, errors.New(spstr[1]), "createVM")
	}

	return nil
}

func cpuThrottle(vcpu, cpu string) string {
	ThrottleFactor, _ := strconv.Atoi(vcpu)
	cpuThrottleFactor := float64(ThrottleFactor)
	ICpu, _ := strconv.Atoi(cpu)
	throttle := float64(ICpu)
	realCPU := throttle / cpuThrottleFactor
	cpu = strconv.FormatFloat(realCPU, 'f', 6, 64) //ugly, compute has the info.
	return cpu
}

func (c *Cluster) getRegion(region string) (string, error) {
	var (
		addr string
	)
	nodlist, err := c.Nodes()
	if err != nil {
		addr = ""
	}
	for _, v := range nodlist {
		if v.Metadata[api.ONEZONE] == region {
			addr = v.Address
		}
	}

	if addr == "" {
		return addr, fmt.Errorf("%s", cmd.Colorfy("Unavailable nodes (hint: start or beat it).\n", "red", "", ""))
	}

	return addr, nil
}
<commit_msg>reduce the redundant codes<commit_after>package cluster

import (
	"encoding/json"
	"errors"
	"fmt"
	log "github.com/Sirupsen/logrus"
	"github.com/megamsys/libgo/cmd"
	"github.com/megamsys/opennebula-go/api"
	"github.com/megamsys/opennebula-go/compute"
	"github.com/megamsys/opennebula-go/virtualmachine"
	"net"
	"net/url"
	"strconv"
	"strings"
)

// CreateVM creates a vm in the specified node.
// It returns the vm, or an error, in case of failures.
const (
	START   = "start"
	STOP    = "stop"
	RESTART = "restart"
)

var ErrConnRefused = errors.New("connection refused")

func (c *Cluster) CreateVM(opts compute.VirtualMachine, t string) (string, string, string, error) {
	var (
		addr    string
		machine string
		vmid    string
		err     error
	)
	maxTries := 5
	for ; maxTries > 0; maxTries-- {

		nodlist, err := c.Nodes()

		for _, v := range nodlist {
			if v.Metadata[api.ONEZONE] == opts.Region {
				addr = v.Address
				opts.Vnets, opts.ClusterId = c.getVnets(v, opts.Vnets)
				if v.Metadata[api.VCPU_PERCENTAGE] != "" {
					opts.Cpu = cpuThrottle(v.Metadata[api.VCPU_PERCENTAGE], opts.Cpu)
				} else {
					opts.Cpu = cpuThrottle(t, opts.Cpu)
				}
			}
		}

		if addr == "" {
			return addr, machine, vmid, fmt.Errorf("%s", cmd.Colorfy("Unavailable nodes (hint: start or beat it).\n", "red", "", ""))
		}

		if err == nil {
			machine, vmid, err = c.createVMInNode(opts, addr)
			if err == nil {
				c.handleNodeSuccess(addr)
				break
			}
			log.Errorf("  > Trying... %s", addr)
		}
		shouldIncrementFailures := false
		isCreateMachineErr := false
		baseErr := err
		if nodeErr, ok := baseErr.(OneNodeError); ok {
			isCreateMachineErr = nodeErr.cmd == "createVM"
			baseErr = nodeErr.BaseError()
		}
		if urlErr, ok := baseErr.(*url.Error); ok {
			baseErr = urlErr.Err
		}
		_, isNetErr := baseErr.(*net.OpError)
		if isNetErr || isCreateMachineErr || baseErr == ErrConnRefused {
			shouldIncrementFailures = true
		}
		c.handleNodeError(addr, err, shouldIncrementFailures)
		return addr, machine, vmid, err
	}
	if err != nil {
		return addr, machine, vmid, fmt.Errorf("CreateVM: maximum number of tries exceeded, last error: %s", err.Error())
	}
	return addr, machine, vmid, err
}

//create a vm in a node.
func (c *Cluster) createVMInNode(opts compute.VirtualMachine, nodeAddress string) (string, string, error) {

	node, err := c.getNodeByAddr(nodeAddress)
	if err != nil {
		return "", "", err
	}
	opts.TemplateName = node.template
	opts.T = node.Client

	res, err := opts.Create()
	if err != nil {
		return "", "",err
	}

	vmid, err := IsSuccess(node,res,"CreateVM")
	if err != nil {
		return "", "", err
	}

	return opts.Name, vmid, nil
}

func (c *Cluster) GetIpPort(opts virtualmachine.Vnc, region string) (string, string, error) {

	addr, err := c.getRegion(region)
	if err != nil {
		return "", "", err
	}

	node, err := c.getNodeByAddr(addr)
	if err != nil {
		return "", "", err
	}
	//opts.TemplateName = node.template
	opts.T = node.Client

	res, err := opts.GetVm()
	if err != nil {
		return "", "", err
	}
	_, err = IsSuccess(node,res,"HostIP")
	if err != nil {
		return "", "", err
	}

	vnchost := res.GetHostIp()
	vncport := res.GetPort()

	if err != nil {
		return "", "", wrapErrorWithCmd(node, err, "createVM")
	}

	return vnchost, vncport, nil
}

// DestroyVM kills a vm, returning an error in case of failure.
func (c *Cluster) DestroyVM(opts compute.VirtualMachine) error {

	addr, err := c.getRegion(opts.Region)
	if err != nil {
		return err
	}

	node, err := c.getNodeByAddr(addr)
	if err != nil {
		return err
	}
	opts.T = node.Client

	res, err := opts.Delete()
	if err != nil {
		return wrapError(node, err)
	}

	_, err = IsSuccess(node,res,"DestroyVM")
	if err != nil {
		return err
	}

	return nil
}

func (c *Cluster) VM(opts compute.VirtualMachine, action string) error {
	switch action {
	case START:
		return c.StartVM(opts)
	case STOP:
		return c.StopVM(opts)
	case RESTART:
		return c.RestartVM(opts)
	default:
		return nil
	}
}
func (c *Cluster) StartVM(opts compute.VirtualMachine) error {

	addr, err := c.getRegion(opts.Region)
	if err != nil {
		return err
	}

	node, err := c.getNodeByAddr(addr)
	if err != nil {
		return err
	}
	opts.T = node.Client

	res, err := opts.Resume()

	if err != nil {
		return wrapError(node, err)
	}

	_, err = IsSuccess(node,res,"StartVM")
	if err != nil {
		return err
	}

	return nil
}

func (c *Cluster) RestartVM(opts compute.VirtualMachine) error {

	addr, err := c.getRegion(opts.Region)
	if err != nil {
		return err
	}

	node, err := c.getNodeByAddr(addr)
	if err != nil {
		return err
	}
	opts.T = node.Client

	res, err := opts.Reboot()
	if err != nil {
		return wrapError(node, err)
	}

	_, err = IsSuccess(node,res,"RebootVM")
	if err != nil {
		return err
	}

	return nil
}

func (c *Cluster) StopVM(opts compute.VirtualMachine) error {

	addr, err := c.getRegion(opts.Region)
	if err != nil {
		return err
	}

	node, err := c.getNodeByAddr(addr)
	if err != nil {
		return err
	}
	opts.T = node.Client

	res, err := opts.Poweroff()
	if err != nil {
		return wrapError(node, err)
	}

	_, err = IsSuccess(node,res,"StopVM")
	if err != nil {
		return err
	}

	return nil
}

func (c *Cluster) getNodeByAddr(addr string) (node, error) {
	return c.getNode(func(s Storage) (Node, error) {
		return s.RetrieveNode(addr)
	})
}

func (c *Cluster) SnapVMDisk(opts compute.VirtualMachine) error {

	addr, err := c.getRegion(opts.Region)
	if err != nil {
		return err
	}

	node, err := c.getNodeByAddr(addr)
	if err != nil {
		return err
	}
	opts.T = node.Client

	res, err := opts.DiskSnap()
	if err != nil {
		return wrapError(node, err)
	}

	_, err = IsSuccess(node,res,"CreateSnap")
	if err != nil {
		return err
	}

	return nil
}

func cpuThrottle(vcpu, cpu string) string {
	ThrottleFactor, _ := strconv.Atoi(vcpu)
	cpuThrottleFactor := float64(ThrottleFactor)
	ICpu, _ := strconv.Atoi(cpu)
	throttle := float64(ICpu)
	realCPU := throttle / cpuThrottleFactor
	cpu = strconv.FormatFloat(realCPU, 'f', 6, 64) //ugly, compute has the info.
	return cpu
}

func (c *Cluster) getRegion(region string) (string, error) {
	var (
		addr string
	)
	nodlist, err := c.Nodes()
	if err != nil {
		addr = ""
	}
	for _, v := range nodlist {
		if v.Metadata[api.ONEZONE] == region {
			addr = v.Address
		}
	}

	if addr == "" {
		return addr, fmt.Errorf("%s", cmd.Colorfy("Unavailable nodes (hint: start or beat it).\n", "red", "", ""))
	}

	return addr, nil
}

func IsSuccess(n node,result interface{},cmd string)  (string, error) {
	b, err := json.Marshal(result)

	if err != nil {
		return "", err
	}

	spstr := strings.Split(string(b), ",")
	isSuccess, err := strconv.ParseBool(spstr[0])
	if err != nil {
		return "", err
	}
	if !isSuccess {
	return "", wrapErrorWithCmd(n, errors.New(spstr[1]),cmd)
	}
  //spstr[1] is error message or ID of action vm,vnet,cluster and etc.,
  return  spstr[1], nil
}
<|endoftext|>


<commit_before>// This package implements a provisioner for Packer that executes
// shell scripts within the remote machine.
package shell

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strings"
	"time"

	"github.com/mitchellh/packer/common"
	"github.com/mitchellh/packer/helper/config"
	"github.com/mitchellh/packer/packer"
	"github.com/mitchellh/packer/template/interpolate"
)

const DefaultRemotePath = "/tmp/script.sh"

type Config struct {
	common.PackerConfig `mapstructure:",squash"`

	// If true, the script contains binary and line endings will not be
	// converted from Windows to Unix-style.
	Binary bool

	// An inline script to execute. Multiple strings are all executed
	// in the context of a single shell.
	Inline []string

	// The shebang value used when running inline scripts.
	InlineShebang string `mapstructure:"inline_shebang"`

	// The local path of the shell script to upload and execute.
	Script string

	// An array of multiple scripts to run.
	Scripts []string

	// An array of environment variables that will be injected before
	// your command(s) are executed.
	Vars []string `mapstructure:"environment_vars"`

	// The remote path where the local shell script will be uploaded to.
	// This should be set to a writable file that is in a pre-existing directory.
	RemotePath string `mapstructure:"remote_path"`

	// The command used to execute the script. The '{{ .Path }}' variable
	// should be used to specify where the script goes, {{ .Vars }}
	// can be used to inject the environment_vars into the environment.
	ExecuteCommand string `mapstructure:"execute_command"`

	// The timeout for retrying to start the process. Until this timeout
	// is reached, if the provisioner can't start a process, it retries.
	// This can be set high to allow for reboots.
	RawStartRetryTimeout string `mapstructure:"start_retry_timeout"`

	startRetryTimeout time.Duration
	ctx               interpolate.Context
}

type Provisioner struct {
	config Config
}

type ExecuteCommandTemplate struct {
	Vars string
	Path string
}

func (p *Provisioner) Prepare(raws ...interface{}) error {
	err := config.Decode(&p.config, &config.DecodeOpts{
		Interpolate: true,
		InterpolateFilter: &interpolate.RenderFilter{
			Exclude: []string{
				"execute_command",
			},
		},
	}, raws...)
	if err != nil {
		return err
	}

	if p.config.ExecuteCommand == "" {
		p.config.ExecuteCommand = "chmod +x {{.Path}}; {{.Vars}} {{.Path}}"
	}

	if p.config.Inline != nil && len(p.config.Inline) == 0 {
		p.config.Inline = nil
	}

	if p.config.InlineShebang == "" {
		p.config.InlineShebang = "/bin/sh -e"
	}

	if p.config.RawStartRetryTimeout == "" {
		p.config.RawStartRetryTimeout = "5m"
	}

	if p.config.RemotePath == "" {
		p.config.RemotePath = DefaultRemotePath
	}

	if p.config.Scripts == nil {
		p.config.Scripts = make([]string, 0)
	}

	if p.config.Vars == nil {
		p.config.Vars = make([]string, 0)
	}

	var errs *packer.MultiError
	if p.config.Script != "" && len(p.config.Scripts) > 0 {
		errs = packer.MultiErrorAppend(errs,
			errors.New("Only one of script or scripts can be specified."))
	}

	if p.config.Script != "" {
		p.config.Scripts = []string{p.config.Script}
	}

	if len(p.config.Scripts) == 0 && p.config.Inline == nil {
		errs = packer.MultiErrorAppend(errs,
			errors.New("Either a script file or inline script must be specified."))
	} else if len(p.config.Scripts) > 0 && p.config.Inline != nil {
		errs = packer.MultiErrorAppend(errs,
			errors.New("Only a script file or an inline script can be specified, not both."))
	}

	for _, path := range p.config.Scripts {
		if _, err := os.Stat(path); err != nil {
			errs = packer.MultiErrorAppend(errs,
				fmt.Errorf("Bad script '%s': %s", path, err))
		}
	}

	// Do a check for bad environment variables, such as '=foo', 'foobar'
	for idx, kv := range p.config.Vars {
		vs := strings.SplitN(kv, "=", 2)
		if len(vs) != 2 || vs[0] == "" {
			errs = packer.MultiErrorAppend(errs,
				fmt.Errorf("Environment variable not in format 'key=value': %s", kv))
		} else {
			// Replace single quotes so they parse
			vs[1] = strings.Replace(vs[1], "'", `'"'"'`, -1)

			// Single quote env var values
			p.config.Vars[idx] = fmt.Sprintf("%s='%s'", vs[0], vs[1])
		}
	}

	if p.config.RawStartRetryTimeout != "" {
		p.config.startRetryTimeout, err = time.ParseDuration(p.config.RawStartRetryTimeout)
		if err != nil {
			errs = packer.MultiErrorAppend(
				errs, fmt.Errorf("Failed parsing start_retry_timeout: %s", err))
		}
	}

	if errs != nil && len(errs.Errors) > 0 {
		return errs
	}

	return nil
}

func (p *Provisioner) Provision(ui packer.Ui, comm packer.Communicator) error {
	scripts := make([]string, len(p.config.Scripts))
	copy(scripts, p.config.Scripts)

	// If we have an inline script, then turn that into a temporary
	// shell script and use that.
	if p.config.Inline != nil {
		tf, err := ioutil.TempFile("", "packer-shell")
		if err != nil {
			return fmt.Errorf("Error preparing shell script: %s", err)
		}
		defer os.Remove(tf.Name())

		// Set the path to the temporary file
		scripts = append(scripts, tf.Name())

		// Write our contents to it
		writer := bufio.NewWriter(tf)
		writer.WriteString(fmt.Sprintf("#!%s\n", p.config.InlineShebang))
		for _, command := range p.config.Inline {
			if _, err := writer.WriteString(command + "\n"); err != nil {
				return fmt.Errorf("Error preparing shell script: %s", err)
			}
		}

		if err := writer.Flush(); err != nil {
			return fmt.Errorf("Error preparing shell script: %s", err)
		}

		tf.Close()
	}

	// Build our variables up by adding in the build name and builder type
	envVars := make([]string, len(p.config.Vars)+2)
	envVars[0] = fmt.Sprintf("PACKER_BUILD_NAME='%s'", p.config.PackerBuildName)
	envVars[1] = fmt.Sprintf("PACKER_BUILDER_TYPE='%s'", p.config.PackerBuilderType)
	copy(envVars[2:], p.config.Vars)

	for _, path := range scripts {
		ui.Say(fmt.Sprintf("Provisioning with shell script: %s", path))

		log.Printf("Opening %s for reading", path)
		f, err := os.Open(path)
		if err != nil {
			return fmt.Errorf("Error opening shell script: %s", err)
		}
		defer f.Close()

		// Flatten the environment variables
		flattendVars := strings.Join(envVars, " ")

		// Compile the command
		p.config.ctx.Data = &ExecuteCommandTemplate{
			Vars: flattendVars,
			Path: p.config.RemotePath,
		}
		command, err := interpolate.Render(p.config.ExecuteCommand, &p.config.ctx)
		if err != nil {
			return fmt.Errorf("Error processing command: %s", err)
		}

		// Upload the file and run the command. Do this in the context of
		// a single retryable function so that we don't end up with
		// the case that the upload succeeded, a restart is initiated,
		// and then the command is executed but the file doesn't exist
		// any longer.
		var cmd *packer.RemoteCmd
		err = p.retryable(func() error {
			if _, err := f.Seek(0, 0); err != nil {
				return err
			}

			var r io.Reader = f
			if !p.config.Binary {
				r = &UnixReader{Reader: r}
			}

			if err := comm.Upload(p.config.RemotePath, r, nil); err != nil {
				return fmt.Errorf("Error uploading script: %s", err)
			}

			cmd = &packer.RemoteCmd{
				Command: fmt.Sprintf("chmod 0755 %s", p.config.RemotePath),
			}
			if err := comm.Start(cmd); err != nil {
				return fmt.Errorf(
					"Error chmodding script file to 0755 in remote "+
						"machine: %s", err)
			}
			cmd.Wait()

			cmd = &packer.RemoteCmd{Command: command}
			return cmd.StartWithUi(comm, ui)
		})
		if err != nil {
			return err
		}

		// Close the original file since we copied it
		f.Close()

		if cmd.ExitStatus != 0 {
			return fmt.Errorf("Script exited with non-zero exit status: %d", cmd.ExitStatus)
		}
	}

	return nil
}

func (p *Provisioner) Cancel() {
	// Just hard quit. It isn't a big deal if what we're doing keeps
	// running on the other side.
	os.Exit(0)
}

// retryable will retry the given function over and over until a
// non-error is returned.
func (p *Provisioner) retryable(f func() error) error {
	startTimeout := time.After(p.config.startRetryTimeout)
	for {
		var err error
		if err = f(); err == nil {
			return nil
		}

		// Create an error and log it
		err = fmt.Errorf("Retryable error: %s", err)
		log.Printf(err.Error())

		// Check if we timed out, otherwise we retry. It is safe to
		// retry since the only error case above is if the command
		// failed to START.
		select {
		case <-startTimeout:
			return err
		default:
			time.Sleep(2 * time.Second)
		}
	}
}
<commit_msg>provisioner/shell: remove file after exec [GH-1536]<commit_after>// This package implements a provisioner for Packer that executes
// shell scripts within the remote machine.
package shell

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strings"
	"time"

	"github.com/mitchellh/packer/common"
	"github.com/mitchellh/packer/helper/config"
	"github.com/mitchellh/packer/packer"
	"github.com/mitchellh/packer/template/interpolate"
)

const DefaultRemotePath = "/tmp/script.sh"

type Config struct {
	common.PackerConfig `mapstructure:",squash"`

	// If true, the script contains binary and line endings will not be
	// converted from Windows to Unix-style.
	Binary bool

	// An inline script to execute. Multiple strings are all executed
	// in the context of a single shell.
	Inline []string

	// The shebang value used when running inline scripts.
	InlineShebang string `mapstructure:"inline_shebang"`

	// The local path of the shell script to upload and execute.
	Script string

	// An array of multiple scripts to run.
	Scripts []string

	// An array of environment variables that will be injected before
	// your command(s) are executed.
	Vars []string `mapstructure:"environment_vars"`

	// The remote path where the local shell script will be uploaded to.
	// This should be set to a writable file that is in a pre-existing directory.
	RemotePath string `mapstructure:"remote_path"`

	// The command used to execute the script. The '{{ .Path }}' variable
	// should be used to specify where the script goes, {{ .Vars }}
	// can be used to inject the environment_vars into the environment.
	ExecuteCommand string `mapstructure:"execute_command"`

	// The timeout for retrying to start the process. Until this timeout
	// is reached, if the provisioner can't start a process, it retries.
	// This can be set high to allow for reboots.
	RawStartRetryTimeout string `mapstructure:"start_retry_timeout"`

	startRetryTimeout time.Duration
	ctx               interpolate.Context
}

type Provisioner struct {
	config Config
}

type ExecuteCommandTemplate struct {
	Vars string
	Path string
}

func (p *Provisioner) Prepare(raws ...interface{}) error {
	err := config.Decode(&p.config, &config.DecodeOpts{
		Interpolate: true,
		InterpolateFilter: &interpolate.RenderFilter{
			Exclude: []string{
				"execute_command",
			},
		},
	}, raws...)
	if err != nil {
		return err
	}

	if p.config.ExecuteCommand == "" {
		p.config.ExecuteCommand = "chmod +x {{.Path}}; {{.Vars}} {{.Path}}"
	}

	if p.config.Inline != nil && len(p.config.Inline) == 0 {
		p.config.Inline = nil
	}

	if p.config.InlineShebang == "" {
		p.config.InlineShebang = "/bin/sh -e"
	}

	if p.config.RawStartRetryTimeout == "" {
		p.config.RawStartRetryTimeout = "5m"
	}

	if p.config.RemotePath == "" {
		p.config.RemotePath = DefaultRemotePath
	}

	if p.config.Scripts == nil {
		p.config.Scripts = make([]string, 0)
	}

	if p.config.Vars == nil {
		p.config.Vars = make([]string, 0)
	}

	var errs *packer.MultiError
	if p.config.Script != "" && len(p.config.Scripts) > 0 {
		errs = packer.MultiErrorAppend(errs,
			errors.New("Only one of script or scripts can be specified."))
	}

	if p.config.Script != "" {
		p.config.Scripts = []string{p.config.Script}
	}

	if len(p.config.Scripts) == 0 && p.config.Inline == nil {
		errs = packer.MultiErrorAppend(errs,
			errors.New("Either a script file or inline script must be specified."))
	} else if len(p.config.Scripts) > 0 && p.config.Inline != nil {
		errs = packer.MultiErrorAppend(errs,
			errors.New("Only a script file or an inline script can be specified, not both."))
	}

	for _, path := range p.config.Scripts {
		if _, err := os.Stat(path); err != nil {
			errs = packer.MultiErrorAppend(errs,
				fmt.Errorf("Bad script '%s': %s", path, err))
		}
	}

	// Do a check for bad environment variables, such as '=foo', 'foobar'
	for idx, kv := range p.config.Vars {
		vs := strings.SplitN(kv, "=", 2)
		if len(vs) != 2 || vs[0] == "" {
			errs = packer.MultiErrorAppend(errs,
				fmt.Errorf("Environment variable not in format 'key=value': %s", kv))
		} else {
			// Replace single quotes so they parse
			vs[1] = strings.Replace(vs[1], "'", `'"'"'`, -1)

			// Single quote env var values
			p.config.Vars[idx] = fmt.Sprintf("%s='%s'", vs[0], vs[1])
		}
	}

	if p.config.RawStartRetryTimeout != "" {
		p.config.startRetryTimeout, err = time.ParseDuration(p.config.RawStartRetryTimeout)
		if err != nil {
			errs = packer.MultiErrorAppend(
				errs, fmt.Errorf("Failed parsing start_retry_timeout: %s", err))
		}
	}

	if errs != nil && len(errs.Errors) > 0 {
		return errs
	}

	return nil
}

func (p *Provisioner) Provision(ui packer.Ui, comm packer.Communicator) error {
	scripts := make([]string, len(p.config.Scripts))
	copy(scripts, p.config.Scripts)

	// If we have an inline script, then turn that into a temporary
	// shell script and use that.
	if p.config.Inline != nil {
		tf, err := ioutil.TempFile("", "packer-shell")
		if err != nil {
			return fmt.Errorf("Error preparing shell script: %s", err)
		}
		defer os.Remove(tf.Name())

		// Set the path to the temporary file
		scripts = append(scripts, tf.Name())

		// Write our contents to it
		writer := bufio.NewWriter(tf)
		writer.WriteString(fmt.Sprintf("#!%s\n", p.config.InlineShebang))
		for _, command := range p.config.Inline {
			if _, err := writer.WriteString(command + "\n"); err != nil {
				return fmt.Errorf("Error preparing shell script: %s", err)
			}
		}

		if err := writer.Flush(); err != nil {
			return fmt.Errorf("Error preparing shell script: %s", err)
		}

		tf.Close()
	}

	// Build our variables up by adding in the build name and builder type
	envVars := make([]string, len(p.config.Vars)+2)
	envVars[0] = fmt.Sprintf("PACKER_BUILD_NAME='%s'", p.config.PackerBuildName)
	envVars[1] = fmt.Sprintf("PACKER_BUILDER_TYPE='%s'", p.config.PackerBuilderType)
	copy(envVars[2:], p.config.Vars)

	for _, path := range scripts {
		ui.Say(fmt.Sprintf("Provisioning with shell script: %s", path))

		log.Printf("Opening %s for reading", path)
		f, err := os.Open(path)
		if err != nil {
			return fmt.Errorf("Error opening shell script: %s", err)
		}
		defer f.Close()

		// Flatten the environment variables
		flattendVars := strings.Join(envVars, " ")

		// Compile the command
		p.config.ctx.Data = &ExecuteCommandTemplate{
			Vars: flattendVars,
			Path: p.config.RemotePath,
		}
		command, err := interpolate.Render(p.config.ExecuteCommand, &p.config.ctx)
		if err != nil {
			return fmt.Errorf("Error processing command: %s", err)
		}

		// Upload the file and run the command. Do this in the context of
		// a single retryable function so that we don't end up with
		// the case that the upload succeeded, a restart is initiated,
		// and then the command is executed but the file doesn't exist
		// any longer.
		var cmd *packer.RemoteCmd
		err = p.retryable(func() error {
			if _, err := f.Seek(0, 0); err != nil {
				return err
			}

			var r io.Reader = f
			if !p.config.Binary {
				r = &UnixReader{Reader: r}
			}

			if err := comm.Upload(p.config.RemotePath, r, nil); err != nil {
				return fmt.Errorf("Error uploading script: %s", err)
			}

			cmd = &packer.RemoteCmd{
				Command: fmt.Sprintf("chmod 0755 %s", p.config.RemotePath),
			}
			if err := comm.Start(cmd); err != nil {
				return fmt.Errorf(
					"Error chmodding script file to 0755 in remote "+
						"machine: %s", err)
			}
			cmd.Wait()

			cmd = &packer.RemoteCmd{Command: command}
			return cmd.StartWithUi(comm, ui)
		})
		if err != nil {
			return err
		}

		if cmd.ExitStatus != 0 {
			return fmt.Errorf("Script exited with non-zero exit status: %d", cmd.ExitStatus)
		}

		// Delete the temporary file we created
		cmd = &packer.RemoteCmd{
			Command: fmt.Sprintf("rm -f %s", p.config.RemotePath),
		}
		if err := comm.Start(cmd); err != nil {
			return fmt.Errorf(
				"Error removing temporary script at %s: %s",
				p.config.RemotePath, err)
		}
		cmd.Wait()
		if cmd.ExitStatus != 0 {
			return fmt.Errorf(
				"Error removing temporary script at %s!")
		}
	}

	return nil
}

func (p *Provisioner) Cancel() {
	// Just hard quit. It isn't a big deal if what we're doing keeps
	// running on the other side.
	os.Exit(0)
}

// retryable will retry the given function over and over until a
// non-error is returned.
func (p *Provisioner) retryable(f func() error) error {
	startTimeout := time.After(p.config.startRetryTimeout)
	for {
		var err error
		if err = f(); err == nil {
			return nil
		}

		// Create an error and log it
		err = fmt.Errorf("Retryable error: %s", err)
		log.Printf(err.Error())

		// Check if we timed out, otherwise we retry. It is safe to
		// retry since the only error case above is if the command
		// failed to START.
		select {
		case <-startTimeout:
			return err
		default:
			time.Sleep(2 * time.Second)
		}
	}
}
<|endoftext|>


<commit_before><commit_msg>dtdiff: move option parse to Entry<commit_after><|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"math"
	"os"
	"strconv"
)

// Predict The Number
// https://www.codeeval.com/open_challenges/125/
func main() {
	file, _ := os.Open(os.Args[1])
	defer file.Close()

	seq := newSequence()
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		num, _ := strconv.Atoi(scanner.Text())
		fmt.Println(seq.get(num))
	}
}

type Sequence []uint8

func newSequence() *Sequence {
	return &Sequence{0}
}

func (seq *Sequence) get(n int) uint8 {
	if len(*seq) < n {
		seq = seq.expand(n)
	}

	return []uint8(*seq)[n]
}

func (seq Sequence) expand(n int) *Sequence {
	newCap := int(math.Pow(2, math.Ceil(math.Log2(float64(n)))))
	newSlice := make(Sequence, len(seq), newCap)
	copy(newSlice, seq)

	for len(newSlice) < newCap {
		for _, val := range newSlice {
			newSlice = append(newSlice, changeNum(val))
		}
	}

	return &newSlice
}

func changeNum(n uint8) uint8 {
	switch n {
	case 0:
		return 1
	case 1:
		return 2
	default:
		return 0
	}
}
<commit_msg>Much easier when one researches the sequence. :p The sequence is the number of 1's in a binary expansion for the given n. You can read more about it here: https://oeis.org/A071858<commit_after>package main

import (
	"bufio"
	"fmt"
	"os"
	"strconv"
	"strings"
)

// Predict The Number
// https://www.codeeval.com/open_challenges/125/
func main() {
	file, _ := os.Open(os.Args[1])
	defer file.Close()

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		num, _ := strconv.Atoi(scanner.Text())
		fmt.Println(getSeqN(num))
	}
}

func getOnes(n int) int {
	bin := strconv.FormatInt(int64(n), 2)
	return strings.Count(bin, "1")
}

func getSeqN(n int) int {
	return getOnes(n) % 3
}
<|endoftext|>


<commit_before>package errors

import "sync"

// ErrorList is used to chain a list of potential errors
type ErrorList struct {
	mux  sync.RWMutex
	errs []error
}

// Error will return the string-form of the errors
func (e *ErrorList) Error() string {
	if e == nil || len(e.errs) == 0 {
		return ""
	}

	b := []byte("the following errors occured:\n")

	e.mux.RLock()
	for _, err := range e.errs {
		b = append(b, err.Error()...)
		b = append(b, '\n')
	}
	e.mux.RUnlock()

	return string(b)
}

// Err will return an error if the errorlist is not empty
// If the errorlist is empty - nil is returned
func (e *ErrorList) Err() (err error) {
	e.mux.RLock()
	if e != nil && len(e.errs) > 0 {
		err = e
	}
	e.mux.RLock()
	return
}

// Push will push an error to the errorlist
// If the errorlist is nil, it will be created
func (e *ErrorList) Push(err error) {
	if err == nil {
		return
	}

	e.mux.Lock()
	if e == nil {
		*e = ErrorList{}
	}

	e.errs = append(e.errs, err)
	e.mux.Unlock()
}
<commit_msg>Fixing race condition<commit_after>package errors

import "sync"

// ErrorList is used to chain a list of potential errors
type ErrorList struct {
	mux  sync.RWMutex
	errs []error
}

// Error will return the string-form of the errors
func (e *ErrorList) Error() (str string) {
	e.mux.RLock()
	if e == nil || len(e.errs) == 0 {
		goto END
	}

	b := []byte("the following errors occured:\n")

	for _, err := range e.errs {
		b = append(b, err.Error()...)
		b = append(b, '\n')
	}

	str = string(b)

END:
	e.mux.RUnlock()
	return
}

// Err will return an error if the errorlist is not empty
// If the errorlist is empty - nil is returned
func (e *ErrorList) Err() (err error) {
	e.mux.RLock()
	if e != nil && len(e.errs) > 0 {
		err = e
	}
	e.mux.RLock()
	return
}

// Push will push an error to the errorlist
// If the errorlist is nil, it will be created
func (e *ErrorList) Push(err error) {
	if err == nil {
		return
	}

	e.mux.Lock()
	if e == nil {
		*e = ErrorList{}
	}

	e.errs = append(e.errs, err)
	e.mux.Unlock()
}
<|endoftext|>


<commit_before>// Package notifier contains types and methods for sending notifications
// that a job has completed.
package notifier

import (
	"io"
	"io/ioutil"
	"fmt"
	"net/http"
)

// Notifiable objects have a destination in mind and a way to serialize
// themselves into a Reader that yields JSON text.
type Notifiable interface {
	Recipient() string
	Content() io.Reader
}

// SendNotification sends an HTTP message to the recipient with the job status.
func SendNotification(n Notifiable) (response *http.Response, err error) {
	destination := n.Recipient()
	payload := n.Content()
	client := &http.Client{}
	req, _ := http.NewRequest("POST", destination, payload)
	req.SetBasicAuth("admin","password")
	req.Header.Set("Content-Type","application/json")
	response, err = client.Do(req)
	if err != nil {
		fmt.Println("Error posting notification:",err)
		return
	}
	body, err := ioutil.ReadAll(response.Body)
	response.Body.Close()
	fmt.Println("Notification response body:", string(body))
	return
}
<commit_msg>gofmt<commit_after>// Package notifier contains types and methods for sending notifications
// that a job has completed.
package notifier

import (
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
)

// Notifiable objects have a destination in mind and a way to serialize
// themselves into a Reader that yields JSON text.
type Notifiable interface {
	Recipient() string
	Content() io.Reader
}

// SendNotification sends an HTTP message to the recipient with the job status.
func SendNotification(n Notifiable) (response *http.Response, err error) {
	destination := n.Recipient()
	payload := n.Content()
	client := &http.Client{}
	req, _ := http.NewRequest("POST", destination, payload)
	req.SetBasicAuth("admin", "password")
	req.Header.Set("Content-Type", "application/json")
	response, err = client.Do(req)
	if err != nil {
		fmt.Println("Error posting notification:", err)
		return
	}
	body, err := ioutil.ReadAll(response.Body)
	response.Body.Close()
	fmt.Println("Notification response body:", string(body))
	return
}
<|endoftext|>


<commit_before>package cache

import (
	"fmt"
	"log"
	"math"
	"math/rand"
	"net/http"
	"os"
	"runtime"
	"strconv"
	"sync"
	"time"

	"github.com/go-redis/redis"
	"github.com/syou6162/go-active-learning/lib/example"
	"github.com/syou6162/go-active-learning/lib/feature"
	"github.com/syou6162/go-active-learning/lib/fetcher"
	"github.com/syou6162/go-active-learning/lib/model"
	"github.com/syou6162/go-active-learning/lib/util"
)

var redisPrefix = "url"

func (c *cache) ExistMetadata(e model.Example) bool {
	key := redisPrefix + ":" + e.Url
	val, err := c.client.HGet(key, "StatusCode").Result()
	if err != nil {
		return false
	}

	statusCode, err := strconv.Atoi(val)
	if err != nil {
		return false
	}

	if statusCode == http.StatusOK {
		return true
	}
	return false
}

func (c *cache) AttachMetadata(examples model.Examples) error {
	for _, e := range examples {
		key := redisPrefix + ":" + e.Url
		vals, err := c.client.HMGet(key,
			"Fv",              // 0
			"FinalUrl",        // 1
			"Title",           // 2
			"Description",     // 3
			"OgDescription",   // 4
			"OgType",          // 5
			"OgImage",         // 6
			"Body",            // 7
			"Score",           // 8
			"IsNew",           // 9
			"StatusCode",      // 10
			"Favicon",         // 11
			"ReferringTweets", // 12
			"HatenaBookmark",  // 13
		).Result()
		if err != nil {
			return err
		}

		// Fv
		if result, ok := vals[0].(string); ok {
			fv := feature.FeatureVector{}
			if err := fv.UnmarshalBinary([]byte(result)); err == nil {
				e.Fv = fv
			}
		}
		// FinalUrl
		if result, ok := vals[1].(string); ok {
			e.FinalUrl = result
		}
		// Title
		if result, ok := vals[2].(string); ok {
			e.Title = result
		}
		// Description
		if result, ok := vals[3].(string); ok {
			e.Description = result
		}
		// OgDescription
		if result, ok := vals[4].(string); ok {
			e.OgDescription = result
		}
		// OgType
		if result, ok := vals[5].(string); ok {
			e.OgType = result
		}
		// OgImage
		if result, ok := vals[6].(string); ok {
			e.OgImage = result
		}
		// Body
		if result, ok := vals[7].(string); ok {
			e.Body = result
		}
		// Score
		if result, ok := vals[8].(string); ok {
			if score, err := strconv.ParseFloat(result, 64); err == nil {
				e.Score = score
			}
		}
		// IsNew
		if result, ok := vals[9].(string); ok {
			if isNew, err := strconv.ParseBool(result); err == nil {
				e.IsNew = isNew
			}
		}
		// StatusCode
		if result, ok := vals[10].(string); ok {
			if statusCode, err := strconv.Atoi(result); err == nil {
				e.StatusCode = statusCode
			}
		}
		// Favicon
		if result, ok := vals[11].(string); ok {
			e.Favicon = result
		}
		// ReferringTweets
		if result, ok := vals[12].(string); ok {
			tweets := model.ReferringTweets{}
			if err := tweets.UnmarshalBinary([]byte(result)); err == nil {
				e.ReferringTweets = tweets
			}
		}
		// HatenaBookmark
		if result, ok := vals[13].(string); ok {
			bookmarks := model.HatenaBookmark{}
			if err := bookmarks.UnmarshalBinary([]byte(result)); err == nil {
				e.HatenaBookmark = &bookmarks
			}
		}
	}
	return nil
}

func (c *cache) AttachLightMetadata(examples model.Examples) error {
	url2Cmd := make(map[string]*redis.SliceCmd)
	url2Example := make(map[string]*model.Example)
	pipe := c.client.Pipeline()

	for _, e := range examples {
		key := redisPrefix + ":" + e.Url
		url2Cmd[key] = pipe.HMGet(key,
			"FinalUrl",        // 0
			"Title",           // 1
			"Description",     // 2
			"OgDescription",   // 3
			"OgType",          // 4
			"OgImage",         // 5
			"Score",           // 6
			"StatusCode",      // 7
			"Favicon",         // 8
			"ReferringTweets", // 9
			"HatenaBookmark",  // 10
		)
		url2Example[key] = e
	}
	_, err := pipe.Exec()
	if err != nil {
		return err
	}

	for k, cmd := range url2Cmd {
		e := url2Example[k]
		vals, err := cmd.Result()
		if err != nil {
			return err
		}
		// FinalUrl
		if result, ok := vals[0].(string); ok {
			e.FinalUrl = result
		}
		// Title
		if result, ok := vals[1].(string); ok {
			e.Title = result
		}
		// Description
		if result, ok := vals[2].(string); ok {
			e.Description = result
		}
		// OgDescription
		if result, ok := vals[3].(string); ok {
			e.OgDescription = result
		}
		// OgType
		if result, ok := vals[4].(string); ok {
			e.OgType = result
		}
		// OgImage
		if result, ok := vals[5].(string); ok {
			e.OgImage = result
		}
		// Score
		if result, ok := vals[6].(string); ok {
			if score, err := strconv.ParseFloat(result, 64); err == nil {
				e.Score = score
			}
		}
		// StatusCode
		if result, ok := vals[7].(string); ok {
			if statusCode, err := strconv.Atoi(result); err == nil {
				e.StatusCode = statusCode
			}
		}
		// Favicon
		if result, ok := vals[8].(string); ok {
			e.Favicon = result
		}
		// ReferringTweets
		if result, ok := vals[9].(string); ok {
			tweets := model.ReferringTweets{}
			if err := tweets.UnmarshalBinary([]byte(result)); err == nil {
				e.ReferringTweets = tweets
			}
		}
		// HatenaBookmark
		if result, ok := vals[10].(string); ok {
			bookmarks := model.HatenaBookmark{}
			if err := bookmarks.UnmarshalBinary([]byte(result)); err == nil {
				e.HatenaBookmark = &bookmarks
			}
		}
	}
	return nil
}

var errorCountPrefix = "errorCountPrefix:"

func (c *cache) incErrorCount(url string) error {
	key := errorCountPrefix + url
	exist, err := c.client.Exists(key).Result()
	if err != nil {
		return err
	}
	if exist == 0 {
		hour := 24 * 10
		c.client.Set(key, 1, time.Hour*time.Duration(hour))
		return nil
	} else {
		if _, err = c.client.Incr(key).Result(); err != nil {
			return err
		}
	}
	return nil
}

func (c *cache) getErrorCount(url string) (int, error) {
	key := errorCountPrefix + url
	ok, err := c.client.Exists(key).Result()
	if err != nil {
		return 0, err
	}
	if ok == 0 {
		return 0, nil
	}

	cntStr, err := c.client.Get(key).Result()
	if err != nil {
		return 0, err
	}
	cnt, err := strconv.Atoi(cntStr)
	if err != nil {
		return 0, err
	}
	return cnt, nil
}

func fetchMetaData(e *model.Example) error {
	article, err := fetcher.GetArticle(e.Url)
	if err != nil {
		return err
	}

	e.Title = article.Title
	e.FinalUrl = article.Url
	e.Description = article.Description
	e.OgDescription = article.OgDescription
	e.OgType = article.OgType
	e.OgImage = article.OgImage
	e.Body = article.Body
	e.StatusCode = article.StatusCode
	e.Favicon = article.Favicon
	e.Fv = util.RemoveDuplicate(example.ExtractFeatures(*e))

	return nil
}

func (c *cache) UpdateExampleMetadata(e model.Example) error {
	key := redisPrefix + ":" + e.Url

	vals := make(map[string]interface{})
	vals["Label"] = &e.Label
	vals["Fv"] = &e.Fv
	vals["Url"] = e.Url
	vals["FinalUrl"] = e.FinalUrl
	vals["Title"] = e.Title
	vals["Description"] = e.Description
	vals["OgDescription"] = e.OgDescription
	vals["OgType"] = e.OgType
	vals["OgImage"] = e.OgImage
	vals["Body"] = e.Body
	vals["Score"] = e.Score
	vals["IsNew"] = e.IsNew
	vals["StatusCode"] = e.StatusCode
	vals["Favicon"] = e.Favicon
	vals["ReferringTweets"] = &e.ReferringTweets
	vals["HatenaBookmark"] = e.HatenaBookmark

	if err := c.client.HMSet(key, vals).Err(); err != nil {
		return err
	}

	// 一度にexpireされるとクロールも一度に走ってOOMが発生するので、多少ばらしてそれを避ける
	hour := int64(240 * rand.Float64())
	return c.UpdateExampleExpire(e, time.Hour*time.Duration(hour))
}

func (c *cache) UpdateExampleExpire(e model.Example, duration time.Duration) error {
	key := redisPrefix + ":" + e.Url
	if err := c.client.Expire(key, duration).Err(); err != nil {
		return err
	}
	return nil
}

func (c *cache) Fetch(examples model.Examples) {
	batchSize := 100
	examplesList := make([]model.Examples, 0)
	n := len(examples)

	for i := 0; i < n; i += batchSize {
		max := int(math.Min(float64(i+batchSize), float64(n)))
		examplesList = append(examplesList, examples[i:max])
	}
	for _, l := range examplesList {
		examplesWithMetaData := model.Examples{}
		examplesWithEmptyMetaData := model.Examples{}
		for _, e := range l {
			if !c.ExistMetadata(*e) {
				examplesWithEmptyMetaData = append(examplesWithEmptyMetaData, e)
			} else {
				examplesWithMetaData = append(examplesWithMetaData, e)
			}
		}
		c.AttachMetadata(examplesWithMetaData)

		wg := &sync.WaitGroup{}
		cpus := runtime.NumCPU()
		runtime.GOMAXPROCS(cpus)
		sem := make(chan struct{}, batchSize)
		for idx, e := range examplesWithEmptyMetaData {
			wg.Add(1)
			sem <- struct{}{}
			go func(e *model.Example, idx int) {
				defer wg.Done()
				cnt, err := c.getErrorCount(e.Url)
				if err != nil {
					log.Println(err.Error())
				}
				if cnt < 5 {
					fmt.Fprintln(os.Stderr, "Fetching("+strconv.Itoa(idx)+"): "+e.Url)
					if err := fetchMetaData(e); err != nil {
						c.incErrorCount(e.Url)
						log.Println(err.Error())
					}
				}
				<-sem
			}(e, idx)
		}
		wg.Wait()
	}
}

func (c *cache) UpdateExamplesMetadata(examples model.Examples) error {
	for _, e := range examples {
		if !c.ExistMetadata(*e) {
			if err := c.UpdateExampleMetadata(*e); err != nil {
				return err
			}
		}
	}
	return nil
}
<commit_msg>cacheへのブクマの読み書きをやめる<commit_after>package cache

import (
	"fmt"
	"log"
	"math"
	"math/rand"
	"net/http"
	"os"
	"runtime"
	"strconv"
	"sync"
	"time"

	"github.com/go-redis/redis"
	"github.com/syou6162/go-active-learning/lib/example"
	"github.com/syou6162/go-active-learning/lib/feature"
	"github.com/syou6162/go-active-learning/lib/fetcher"
	"github.com/syou6162/go-active-learning/lib/model"
	"github.com/syou6162/go-active-learning/lib/util"
)

var redisPrefix = "url"

func (c *cache) ExistMetadata(e model.Example) bool {
	key := redisPrefix + ":" + e.Url
	val, err := c.client.HGet(key, "StatusCode").Result()
	if err != nil {
		return false
	}

	statusCode, err := strconv.Atoi(val)
	if err != nil {
		return false
	}

	if statusCode == http.StatusOK {
		return true
	}
	return false
}

func (c *cache) AttachMetadata(examples model.Examples) error {
	for _, e := range examples {
		key := redisPrefix + ":" + e.Url
		vals, err := c.client.HMGet(key,
			"Fv",              // 0
			"FinalUrl",        // 1
			"Title",           // 2
			"Description",     // 3
			"OgDescription",   // 4
			"OgType",          // 5
			"OgImage",         // 6
			"Body",            // 7
			"Score",           // 8
			"IsNew",           // 9
			"StatusCode",      // 10
			"Favicon",         // 11
			"ReferringTweets", // 12
		).Result()
		if err != nil {
			return err
		}

		// Fv
		if result, ok := vals[0].(string); ok {
			fv := feature.FeatureVector{}
			if err := fv.UnmarshalBinary([]byte(result)); err == nil {
				e.Fv = fv
			}
		}
		// FinalUrl
		if result, ok := vals[1].(string); ok {
			e.FinalUrl = result
		}
		// Title
		if result, ok := vals[2].(string); ok {
			e.Title = result
		}
		// Description
		if result, ok := vals[3].(string); ok {
			e.Description = result
		}
		// OgDescription
		if result, ok := vals[4].(string); ok {
			e.OgDescription = result
		}
		// OgType
		if result, ok := vals[5].(string); ok {
			e.OgType = result
		}
		// OgImage
		if result, ok := vals[6].(string); ok {
			e.OgImage = result
		}
		// Body
		if result, ok := vals[7].(string); ok {
			e.Body = result
		}
		// Score
		if result, ok := vals[8].(string); ok {
			if score, err := strconv.ParseFloat(result, 64); err == nil {
				e.Score = score
			}
		}
		// IsNew
		if result, ok := vals[9].(string); ok {
			if isNew, err := strconv.ParseBool(result); err == nil {
				e.IsNew = isNew
			}
		}
		// StatusCode
		if result, ok := vals[10].(string); ok {
			if statusCode, err := strconv.Atoi(result); err == nil {
				e.StatusCode = statusCode
			}
		}
		// Favicon
		if result, ok := vals[11].(string); ok {
			e.Favicon = result
		}
		// ReferringTweets
		if result, ok := vals[12].(string); ok {
			tweets := model.ReferringTweets{}
			if err := tweets.UnmarshalBinary([]byte(result)); err == nil {
				e.ReferringTweets = tweets
			}
		}
	}
	return nil
}

func (c *cache) AttachLightMetadata(examples model.Examples) error {
	url2Cmd := make(map[string]*redis.SliceCmd)
	url2Example := make(map[string]*model.Example)
	pipe := c.client.Pipeline()

	for _, e := range examples {
		key := redisPrefix + ":" + e.Url
		url2Cmd[key] = pipe.HMGet(key,
			"FinalUrl",        // 0
			"Title",           // 1
			"Description",     // 2
			"OgDescription",   // 3
			"OgType",          // 4
			"OgImage",         // 5
			"Score",           // 6
			"StatusCode",      // 7
			"Favicon",         // 8
			"ReferringTweets", // 9
		)
		url2Example[key] = e
	}
	_, err := pipe.Exec()
	if err != nil {
		return err
	}

	for k, cmd := range url2Cmd {
		e := url2Example[k]
		vals, err := cmd.Result()
		if err != nil {
			return err
		}
		// FinalUrl
		if result, ok := vals[0].(string); ok {
			e.FinalUrl = result
		}
		// Title
		if result, ok := vals[1].(string); ok {
			e.Title = result
		}
		// Description
		if result, ok := vals[2].(string); ok {
			e.Description = result
		}
		// OgDescription
		if result, ok := vals[3].(string); ok {
			e.OgDescription = result
		}
		// OgType
		if result, ok := vals[4].(string); ok {
			e.OgType = result
		}
		// OgImage
		if result, ok := vals[5].(string); ok {
			e.OgImage = result
		}
		// Score
		if result, ok := vals[6].(string); ok {
			if score, err := strconv.ParseFloat(result, 64); err == nil {
				e.Score = score
			}
		}
		// StatusCode
		if result, ok := vals[7].(string); ok {
			if statusCode, err := strconv.Atoi(result); err == nil {
				e.StatusCode = statusCode
			}
		}
		// Favicon
		if result, ok := vals[8].(string); ok {
			e.Favicon = result
		}
		// ReferringTweets
		if result, ok := vals[9].(string); ok {
			tweets := model.ReferringTweets{}
			if err := tweets.UnmarshalBinary([]byte(result)); err == nil {
				e.ReferringTweets = tweets
			}
		}
	}
	return nil
}

var errorCountPrefix = "errorCountPrefix:"

func (c *cache) incErrorCount(url string) error {
	key := errorCountPrefix + url
	exist, err := c.client.Exists(key).Result()
	if err != nil {
		return err
	}
	if exist == 0 {
		hour := 24 * 10
		c.client.Set(key, 1, time.Hour*time.Duration(hour))
		return nil
	} else {
		if _, err = c.client.Incr(key).Result(); err != nil {
			return err
		}
	}
	return nil
}

func (c *cache) getErrorCount(url string) (int, error) {
	key := errorCountPrefix + url
	ok, err := c.client.Exists(key).Result()
	if err != nil {
		return 0, err
	}
	if ok == 0 {
		return 0, nil
	}

	cntStr, err := c.client.Get(key).Result()
	if err != nil {
		return 0, err
	}
	cnt, err := strconv.Atoi(cntStr)
	if err != nil {
		return 0, err
	}
	return cnt, nil
}

func fetchMetaData(e *model.Example) error {
	article, err := fetcher.GetArticle(e.Url)
	if err != nil {
		return err
	}

	e.Title = article.Title
	e.FinalUrl = article.Url
	e.Description = article.Description
	e.OgDescription = article.OgDescription
	e.OgType = article.OgType
	e.OgImage = article.OgImage
	e.Body = article.Body
	e.StatusCode = article.StatusCode
	e.Favicon = article.Favicon
	e.Fv = util.RemoveDuplicate(example.ExtractFeatures(*e))

	return nil
}

func (c *cache) UpdateExampleMetadata(e model.Example) error {
	key := redisPrefix + ":" + e.Url

	vals := make(map[string]interface{})
	vals["Label"] = &e.Label
	vals["Fv"] = &e.Fv
	vals["Url"] = e.Url
	vals["FinalUrl"] = e.FinalUrl
	vals["Title"] = e.Title
	vals["Description"] = e.Description
	vals["OgDescription"] = e.OgDescription
	vals["OgType"] = e.OgType
	vals["OgImage"] = e.OgImage
	vals["Body"] = e.Body
	vals["Score"] = e.Score
	vals["IsNew"] = e.IsNew
	vals["StatusCode"] = e.StatusCode
	vals["Favicon"] = e.Favicon
	vals["ReferringTweets"] = &e.ReferringTweets

	if err := c.client.HMSet(key, vals).Err(); err != nil {
		return err
	}

	// 一度にexpireされるとクロールも一度に走ってOOMが発生するので、多少ばらしてそれを避ける
	hour := int64(240 * rand.Float64())
	return c.UpdateExampleExpire(e, time.Hour*time.Duration(hour))
}

func (c *cache) UpdateExampleExpire(e model.Example, duration time.Duration) error {
	key := redisPrefix + ":" + e.Url
	if err := c.client.Expire(key, duration).Err(); err != nil {
		return err
	}
	return nil
}

func (c *cache) Fetch(examples model.Examples) {
	batchSize := 100
	examplesList := make([]model.Examples, 0)
	n := len(examples)

	for i := 0; i < n; i += batchSize {
		max := int(math.Min(float64(i+batchSize), float64(n)))
		examplesList = append(examplesList, examples[i:max])
	}
	for _, l := range examplesList {
		examplesWithMetaData := model.Examples{}
		examplesWithEmptyMetaData := model.Examples{}
		for _, e := range l {
			if !c.ExistMetadata(*e) {
				examplesWithEmptyMetaData = append(examplesWithEmptyMetaData, e)
			} else {
				examplesWithMetaData = append(examplesWithMetaData, e)
			}
		}
		c.AttachMetadata(examplesWithMetaData)

		wg := &sync.WaitGroup{}
		cpus := runtime.NumCPU()
		runtime.GOMAXPROCS(cpus)
		sem := make(chan struct{}, batchSize)
		for idx, e := range examplesWithEmptyMetaData {
			wg.Add(1)
			sem <- struct{}{}
			go func(e *model.Example, idx int) {
				defer wg.Done()
				cnt, err := c.getErrorCount(e.Url)
				if err != nil {
					log.Println(err.Error())
				}
				if cnt < 5 {
					fmt.Fprintln(os.Stderr, "Fetching("+strconv.Itoa(idx)+"): "+e.Url)
					if err := fetchMetaData(e); err != nil {
						c.incErrorCount(e.Url)
						log.Println(err.Error())
					}
				}
				<-sem
			}(e, idx)
		}
		wg.Wait()
	}
}

func (c *cache) UpdateExamplesMetadata(examples model.Examples) error {
	for _, e := range examples {
		if !c.ExistMetadata(*e) {
			if err := c.UpdateExampleMetadata(*e); err != nil {
				return err
			}
		}
	}
	return nil
}
<|endoftext|>


<commit_before><commit_msg>Removed a nonsensical parameter.<commit_after><|endoftext|>


<commit_before>package selector

import (
	"math/rand"
	"time"

	"github.com/micro/go-micro/registry"
	"github.com/micro/go-micro/selector/internal/blacklist"
)

type defaultSelector struct {
	so   Options
	exit chan bool
	bl   *blacklist.BlackList
}

func init() {
	rand.Seed(time.Now().Unix())
}

func (r *defaultSelector) run() {
	t := time.NewTicker(time.Second * 30)

	for {
		select {
		case <-t.C:
			// TODO
		case <-r.exit:
			t.Stop()
			return
		}
	}
}

func (r *defaultSelector) Init(opts ...Option) error {
	for _, o := range opts {
		o(&r.so)
	}
	return nil
}

func (r *defaultSelector) Options() Options {
	return r.so
}

func (r *defaultSelector) Select(service string, opts ...SelectOption) (Next, error) {
	sopts := SelectOptions{
		Strategy: r.so.Strategy,
	}

	for _, opt := range opts {
		opt(&sopts)
	}

	// get the service
	services, err := r.so.Registry.GetService(service)
	if err != nil {
		return nil, err
	}

	// apply the filters
	for _, filter := range sopts.Filters {
		services = filter(services)
	}

	// apply the blacklist
	services, err = r.bl.Filter(services)
	if err != nil {
		return nil, err
	}

	// if there's nothing left, return
	if len(services) == 0 {
		return nil, ErrNoneAvailable
	}

	return sopts.Strategy(services), nil
}

func (r *defaultSelector) Mark(service string, node *registry.Node, err error) {
	r.bl.Mark(service, node, err)
}

func (r *defaultSelector) Reset(service string) {
	r.bl.Reset(service)
}

func (r *defaultSelector) Close() error {
	select {
	case <-r.exit:
		return nil
	default:
		close(r.exit)
		r.bl.Close()
	}
	return nil
}

func (r *defaultSelector) String() string {
	return "default"
}

func newDefaultSelector(opts ...Option) Selector {
	sopts := Options{
		Strategy: Random,
	}

	for _, opt := range opts {
		opt(&sopts)
	}

	if sopts.Registry == nil {
		sopts.Registry = registry.DefaultRegistry
	}

	se := &defaultSelector{
		so:   sopts,
		exit: make(chan bool),
		bl:   blacklist.New(),
	}

	go se.run()
	return se
}
<commit_msg>Run method is obsolete<commit_after>package selector

import (
	"math/rand"
	"time"

	"github.com/micro/go-micro/registry"
	"github.com/micro/go-micro/selector/internal/blacklist"
)

type defaultSelector struct {
	so   Options
	exit chan bool
	bl   *blacklist.BlackList
}

func init() {
	rand.Seed(time.Now().Unix())
}

func (r *defaultSelector) Init(opts ...Option) error {
	for _, o := range opts {
		o(&r.so)
	}
	return nil
}

func (r *defaultSelector) Options() Options {
	return r.so
}

func (r *defaultSelector) Select(service string, opts ...SelectOption) (Next, error) {
	sopts := SelectOptions{
		Strategy: r.so.Strategy,
	}

	for _, opt := range opts {
		opt(&sopts)
	}

	// get the service
	services, err := r.so.Registry.GetService(service)
	if err != nil {
		return nil, err
	}

	// apply the filters
	for _, filter := range sopts.Filters {
		services = filter(services)
	}

	// apply the blacklist
	services, err = r.bl.Filter(services)
	if err != nil {
		return nil, err
	}

	// if there's nothing left, return
	if len(services) == 0 {
		return nil, ErrNoneAvailable
	}

	return sopts.Strategy(services), nil
}

func (r *defaultSelector) Mark(service string, node *registry.Node, err error) {
	r.bl.Mark(service, node, err)
}

func (r *defaultSelector) Reset(service string) {
	r.bl.Reset(service)
}

func (r *defaultSelector) Close() error {
	select {
	case <-r.exit:
		return nil
	default:
		close(r.exit)
		r.bl.Close()
	}
	return nil
}

func (r *defaultSelector) String() string {
	return "default"
}

func newDefaultSelector(opts ...Option) Selector {
	sopts := Options{
		Strategy: Random,
	}

	for _, opt := range opts {
		opt(&sopts)
	}

	if sopts.Registry == nil {
		sopts.Registry = registry.DefaultRegistry
	}

	return &defaultSelector{
		so:   sopts,
		exit: make(chan bool),
		bl:   blacklist.New(),
	}
}
<|endoftext|>


<commit_before>package apns

import (
	"fmt"
	"github.com/sideshow/apns2"
	"github.com/smancke/guble/server/connector"
	"github.com/smancke/guble/server/router"
)

const (
	// schema is the default database schema for APNS
	schema = "apns_registration"
)

// Config is used for configuring the APNS module.
type Config struct {
	Enabled             *bool
	Production          *bool
	CertificateFileName *string
	CertificateBytes    *[]byte
	CertificatePassword *string
	AppTopic            *string
	Workers             *int
	Prefix              *string
}

// conn is the private struct for handling the communication with APNS
type conn struct {
	Config
	connector.Connector
}

// New creates a new Connector without starting it
func New(router router.Router, sender connector.Sender, config Config) (connector.ReactiveConnector, error) {
	baseConn, err := connector.NewConnector(
		router,
		sender,
		connector.Config{
			Name:       "apns",
			Schema:     schema,
			Prefix:     *config.Prefix,
			URLPattern: fmt.Sprintf("/{%s}/{%s}/{%s:.*}", deviceIDKey, userIDKey, connector.TopicParam),
			Workers:    *config.Workers,
		},
	)
	if err != nil {
		logger.WithError(err).Error("Base connector error")
		return nil, err
	}
	newConn := &conn{
		Config:    config,
		Connector: baseConn,
	}
	newConn.SetResponseHandler(newConn)
	return newConn, nil
}

func (c *conn) HandleResponse(request connector.Request, responseIface interface{}, errSend error) error {
	logger.Debug("HandleResponse")
	if errSend != nil {
		logger.WithError(errSend).Error("error when trying to send APNS notification")
		return errSend
	}
	if r, ok := responseIface.(*apns2.Response); ok {
		messageID := request.Message().ID
		subscriber := request.Subscriber()
		subscriber.SetLastID(messageID)
		if r.Sent() {
			logger.WithField("id", r.ApnsID).Debug("APNS notification was successfully sent")
			return nil
		}
		logger.WithField("id", r.ApnsID).WithField("reason", r.Reason).Error("APNS notification was not sent")
		switch r.Reason {
		case
			apns2.ReasonMissingDeviceToken,
			apns2.ReasonBadDeviceToken,
			apns2.ReasonDeviceTokenNotForTopic,
			apns2.ReasonUnregistered:

			logger.WithField("id", r.ApnsID).Info("removing subscriber because a relevant error was received from APNS")
			c.Manager().Remove(subscriber)
		}
		//TODO Cosmin Bogdan: extra-APNS-handling
	}
	return nil
}

// Check returns nil if health-check succeeds, or an error if health-check fails
func (c *conn) Check() error {
	//TODO implement
	return nil
}
<commit_msg>apns: invoking manager update<commit_after>package apns

import (
	"fmt"
	"github.com/sideshow/apns2"
	"github.com/smancke/guble/server/connector"
	"github.com/smancke/guble/server/router"
)

const (
	// schema is the default database schema for APNS
	schema = "apns_registration"
)

// Config is used for configuring the APNS module.
type Config struct {
	Enabled             *bool
	Production          *bool
	CertificateFileName *string
	CertificateBytes    *[]byte
	CertificatePassword *string
	AppTopic            *string
	Workers             *int
	Prefix              *string
}

// conn is the private struct for handling the communication with APNS
type conn struct {
	Config
	connector.Connector
}

// New creates a new Connector without starting it
func New(router router.Router, sender connector.Sender, config Config) (connector.ReactiveConnector, error) {
	baseConn, err := connector.NewConnector(
		router,
		sender,
		connector.Config{
			Name:       "apns",
			Schema:     schema,
			Prefix:     *config.Prefix,
			URLPattern: fmt.Sprintf("/{%s}/{%s}/{%s:.*}", deviceIDKey, userIDKey, connector.TopicParam),
			Workers:    *config.Workers,
		},
	)
	if err != nil {
		logger.WithError(err).Error("Base connector error")
		return nil, err
	}
	newConn := &conn{
		Config:    config,
		Connector: baseConn,
	}
	newConn.SetResponseHandler(newConn)
	return newConn, nil
}

func (c *conn) HandleResponse(request connector.Request, responseIface interface{}, errSend error) error {
	logger.Debug("HandleResponse")
	if errSend != nil {
		logger.WithError(errSend).Error("error when trying to send APNS notification")
		return errSend
	}
	if r, ok := responseIface.(*apns2.Response); ok {
		messageID := request.Message().ID
		subscriber := request.Subscriber()
		subscriber.SetLastID(messageID)
		if err := c.Manager().Update(subscriber); err != nil {
			logger.WithField("error", err.Error()).Error("Manager could not update subscription")
			return err
		}
		if r.Sent() {
			logger.WithField("id", r.ApnsID).Debug("APNS notification was successfully sent")
			return nil
		}
		logger.WithField("id", r.ApnsID).WithField("reason", r.Reason).Error("APNS notification was not sent")
		switch r.Reason {
		case
			apns2.ReasonMissingDeviceToken,
			apns2.ReasonBadDeviceToken,
			apns2.ReasonDeviceTokenNotForTopic,
			apns2.ReasonUnregistered:

			logger.WithField("id", r.ApnsID).Info("removing subscriber because a relevant error was received from APNS")
			c.Manager().Remove(subscriber)
		}
		//TODO Cosmin Bogdan: extra-APNS-handling
	}
	return nil
}

// Check returns nil if health-check succeeds, or an error if health-check fails
func (c *conn) Check() error {
	//TODO implement
	return nil
}
<|endoftext|>


<commit_before>// package http implements an http server that serves static content from ipfs
package http

import (
	"fmt"
	"io"
	"net/http"
	"strings"

	"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/gorilla/mux"
	ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr"
	manet "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr/net"
	mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash"

	cmds "github.com/jbenet/go-ipfs/commands"
	core "github.com/jbenet/go-ipfs/core"
	"github.com/jbenet/go-ipfs/core/commands"
)

type objectHandler struct {
	ipfs
}

type apiHandler struct{}

// Serve starts the http server
func Serve(address ma.Multiaddr, node *core.IpfsNode) error {
	r := mux.NewRouter()
	objectHandler := &objectHandler{&ipfsHandler{node}}
	apiHandler := &apiHandler{}

	r.PathPrefix("/api/v0/").Handler(apiHandler).Methods("GET", "POST")

	r.HandleFunc("/ipfs/", objectHandler.postHandler).Methods("POST")
	r.PathPrefix("/ipfs/").Handler(objectHandler).Methods("GET")

	http.Handle("/", r)

	_, host, err := manet.DialArgs(address)
	if err != nil {
		return err
	}

	return http.ListenAndServe(host, nil)
}

func (i *objectHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	path := r.URL.Path[5:]

	nd, err := i.ResolvePath(path)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Println(err)
		return
	}

	dr, err := i.NewDagReader(nd)
	if err != nil {
		// TODO: return json object containing the tree data if it's a directory (err == ErrIsDir)
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Println(err)
		return
	}

	io.Copy(w, dr)
}

func (i *objectHandler) postHandler(w http.ResponseWriter, r *http.Request) {
	nd, err := i.NewDagFromReader(r.Body)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Println(err)
		return
	}

	k, err := i.AddNodeToDAG(nd)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Println(err)
		return
	}

	//TODO: return json representation of list instead
	w.WriteHeader(http.StatusCreated)
	w.Write([]byte(mh.Multihash(k).B58String()))
}

func (i *apiHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	path := strings.Split(r.URL.Path, "/")[3:]
	opts := getOptions(r)

	// TODO: get args

	// ensure the requested command exists, otherwise 404
	_, err := commands.Root.Get(path)
	if err != nil {
		w.WriteHeader(http.StatusNotFound)
		w.Write([]byte("404 page not found"))
		return
	}

	// build the Request and call the command
	req := cmds.NewRequest(path, opts, nil, nil)
	res := commands.Root.Call(req)

	// if response contains an error, write an HTTP error status code
	if err = res.Error(); err != nil {
		e := err.(cmds.Error)

		if e.Code == cmds.ErrClient {
			w.WriteHeader(http.StatusBadRequest)
		} else {
			w.WriteHeader(http.StatusInternalServerError)
		}
	}

	val := res.Value()

	// if the output value is a io.Reader, stream its output in the request body
	if stream, ok := val.(io.Reader); ok {
		io.Copy(w, stream)
		return
	}

	// otherwise, marshall and output the response value or error
	if val != nil || res.Error() != nil {
		output, err := res.Marshal()

		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Println(err)
			return
		}

		if output != nil {
			w.Write(output)
		}
	}
}

// getOptions returns the command options in the given HTTP request
// (from the querystring and request body)
func getOptions(r *http.Request) map[string]interface{} {
	opts := make(map[string]interface{})

	query := r.URL.Query()
	for k, v := range query {
		opts[k] = v[0]
	}

	// TODO: get more options from request body (formdata, json, etc)

	if _, exists := opts[cmds.EncShort]; !exists {
		opts[cmds.EncShort] = cmds.JSON
	}

	return opts
}
<commit_msg>server/http: Don't cast res.Error() to an Error (fixes panix on response errors)<commit_after>// package http implements an http server that serves static content from ipfs
package http

import (
	"fmt"
	"io"
	"net/http"
	"strings"

	"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/gorilla/mux"
	ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr"
	manet "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr/net"
	mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash"

	cmds "github.com/jbenet/go-ipfs/commands"
	core "github.com/jbenet/go-ipfs/core"
	"github.com/jbenet/go-ipfs/core/commands"
)

type objectHandler struct {
	ipfs
}

type apiHandler struct{}

// Serve starts the http server
func Serve(address ma.Multiaddr, node *core.IpfsNode) error {
	r := mux.NewRouter()
	objectHandler := &objectHandler{&ipfsHandler{node}}
	apiHandler := &apiHandler{}

	r.PathPrefix("/api/v0/").Handler(apiHandler).Methods("GET", "POST")

	r.HandleFunc("/ipfs/", objectHandler.postHandler).Methods("POST")
	r.PathPrefix("/ipfs/").Handler(objectHandler).Methods("GET")

	http.Handle("/", r)

	_, host, err := manet.DialArgs(address)
	if err != nil {
		return err
	}

	return http.ListenAndServe(host, nil)
}

func (i *objectHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	path := r.URL.Path[5:]

	nd, err := i.ResolvePath(path)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Println(err)
		return
	}

	dr, err := i.NewDagReader(nd)
	if err != nil {
		// TODO: return json object containing the tree data if it's a directory (err == ErrIsDir)
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Println(err)
		return
	}

	io.Copy(w, dr)
}

func (i *objectHandler) postHandler(w http.ResponseWriter, r *http.Request) {
	nd, err := i.NewDagFromReader(r.Body)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Println(err)
		return
	}

	k, err := i.AddNodeToDAG(nd)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Println(err)
		return
	}

	//TODO: return json representation of list instead
	w.WriteHeader(http.StatusCreated)
	w.Write([]byte(mh.Multihash(k).B58String()))
}

func (i *apiHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	path := strings.Split(r.URL.Path, "/")[3:]
	opts := getOptions(r)

	// TODO: get args

	// ensure the requested command exists, otherwise 404
	_, err := commands.Root.Get(path)
	if err != nil {
		w.WriteHeader(http.StatusNotFound)
		w.Write([]byte("404 page not found"))
		return
	}

	// build the Request and call the command
	req := cmds.NewRequest(path, opts, nil, nil)
	res := commands.Root.Call(req)

	// if response contains an error, write an HTTP error status code
	if e := res.Error(); e != nil {
		if e.Code == cmds.ErrClient {
			w.WriteHeader(http.StatusBadRequest)
		} else {
			w.WriteHeader(http.StatusInternalServerError)
		}
	}

	val := res.Value()

	// if the output value is a io.Reader, stream its output in the request body
	if stream, ok := val.(io.Reader); ok {
		io.Copy(w, stream)
		return
	}

	// otherwise, marshall and output the response value or error
	if val != nil || res.Error() != nil {
		output, err := res.Marshal()

		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Println(err)
			return
		}

		if output != nil {
			w.Write(output)
		}
	}
}

// getOptions returns the command options in the given HTTP request
// (from the querystring and request body)
func getOptions(r *http.Request) map[string]interface{} {
	opts := make(map[string]interface{})

	query := r.URL.Query()
	for k, v := range query {
		opts[k] = v[0]
	}

	// TODO: get more options from request body (formdata, json, etc)

	if _, exists := opts[cmds.EncShort]; !exists {
		opts[cmds.EncShort] = cmds.JSON
	}

	return opts
}
<|endoftext|>