<commit_before>package recovery

import (
	"github.com/rancher/os/config"
	"github.com/rancher/os/pkg/compose"
	"github.com/rancher/os/pkg/init/docker"
	"github.com/rancher/os/pkg/log"
	"github.com/rancher/os/pkg/netconf"
	"github.com/rancher/os/pkg/sysinit"

	composeConfig "github.com/docker/libcompose/config"
	"github.com/docker/libcompose/yaml"
)

var (
	// TODO: move this into the os-config file so it can be customised.
	recoveryDockerService = composeConfig.ServiceConfigV1{
		Image: config.OsBase,
		Command: yaml.Command{
			"ros",
			"recovery-init",
		},
		Labels: map[string]string{
			config.DetachLabel: "false",
			config.ScopeLabel:  "system",
		},
		LogDriver:  "json-file",
		Net:        "host",
		Uts:        "host",
		Pid:        "host",
		Ipc:        "host",
		Privileged: true,
		Volumes: []string{
			"/dev:/host/dev",
			"/etc/ssl/certs/ca-certificates.crt:/etc/ssl/certs/ca-certificates.crt.rancher",
			"/lib/modules:/lib/modules",
			"/lib/firmware:/lib/firmware",
			"/usr/bin/ros:/usr/bin/ros:ro",
			"/usr/bin/ros:/usr/bin/cloud-init-save",
			"/usr/bin/ros:/usr/bin/respawn:ro",
			"/usr/share/ros:/usr/share/ros:ro",
			"/var/lib/rancher:/var/lib/rancher",
			"/var/lib/rancher/conf:/var/lib/rancher/conf",
			"/var/run:/var/run",
		},
	}
)

func LoadRecoveryConsole(cfg *config.CloudConfig) (*config.CloudConfig, error) {
	if cfg.Rancher.Recovery {
		Recovery(nil)
	}
	return cfg, nil
}

func Recovery(initFailure error) {
	if initFailure != nil {
		log.Errorf("RancherOS has failed to boot: %v", initFailure)
	}
	log.Info("Launching recovery console")

	var recoveryConfig config.CloudConfig
	recoveryConfig.Rancher.Defaults = config.Defaults{
		Network: netconf.NetworkConfig{
			DNS: netconf.DNSConfig{
				Nameservers: []string{
					"8.8.8.8",
					"8.8.4.4",
				},
			},
		},
	}
	recoveryConfig.Rancher.BootstrapDocker = config.DockerConfig{
		EngineOpts: config.EngineOpts{
			Bridge:        "none",
			StorageDriver: "overlay",
			Restart:       &[]bool{false}[0],
			Graph:         "/var/lib/recovery-docker",
			Group:         "root",
			Host:          []string{"unix:///var/run/system-docker.sock"},
			UserlandProxy: &[]bool{false}[0],
		},
	}

	_, err := docker.Start(&recoveryConfig)
	if err != nil {
		log.Fatal(err)
	}

	_, err = config.ChainCfgFuncs(&recoveryConfig,
		[]config.CfgFuncData{
			{"loadSystemImages", sysinit.LoadSystemImages},
			{"recovery console", recoveryServices},
		})
	if err != nil {
		log.Fatal(err)
	}
}

func recoveryServices(cfg *config.CloudConfig) (*config.CloudConfig, error) {
	_, err := compose.RunServiceSet("recovery", cfg, map[string]*composeConfig.ServiceConfigV1{
		"recovery": &recoveryDockerService,
	})
	return nil, err
}
<commit_msg>Reduce recovery console memory<commit_after>package recovery

import (
	"github.com/rancher/os/config"
	"github.com/rancher/os/pkg/compose"
	"github.com/rancher/os/pkg/init/docker"
	"github.com/rancher/os/pkg/log"
	"github.com/rancher/os/pkg/netconf"
	"github.com/rancher/os/pkg/sysinit"

	composeConfig "github.com/docker/libcompose/config"
	"github.com/docker/libcompose/yaml"
)

var (
	// TODO: move this into the os-config file so it can be customised.
	recoveryDockerService = composeConfig.ServiceConfigV1{
		Image: config.OsBase,
		Command: yaml.Command{
			"ros",
			"recovery-init",
		},
		Labels: map[string]string{
			config.DetachLabel: "false",
			config.ScopeLabel:  "system",
		},
		LogDriver:  "json-file",
		Net:        "host",
		Uts:        "host",
		Pid:        "host",
		Ipc:        "host",
		Privileged: true,
		Volumes: []string{
			"/dev:/host/dev",
			"/etc/ssl/certs/ca-certificates.crt:/etc/ssl/certs/ca-certificates.crt.rancher",
			"/lib/modules:/lib/modules",
			"/lib/firmware:/lib/firmware",
			"/usr/bin/ros:/usr/bin/ros:ro",
			"/usr/bin/ros:/usr/bin/cloud-init-save",
			"/usr/bin/ros:/usr/bin/respawn:ro",
			"/usr/share/ros:/usr/share/ros:ro",
			"/var/lib/rancher:/var/lib/rancher",
			"/var/lib/rancher/conf:/var/lib/rancher/conf",
			"/var/run:/var/run",
		},
	}
)

func LoadRecoveryConsole(cfg *config.CloudConfig) (*config.CloudConfig, error) {
	if cfg.Rancher.Recovery {
		Recovery(nil)
	}
	return cfg, nil
}

func Recovery(initFailure error) {
	if initFailure != nil {
		log.Errorf("RancherOS has failed to boot: %v", initFailure)
	}
	log.Info("Launching recovery console")

	var recoveryConfig config.CloudConfig
	recoveryConfig.Rancher.Defaults = config.Defaults{
		Network: netconf.NetworkConfig{
			DNS: netconf.DNSConfig{
				Nameservers: []string{
					"8.8.8.8",
					"8.8.4.4",
				},
			},
		},
	}
	recoveryConfig.Rancher.BootstrapDocker = config.DockerConfig{
		EngineOpts: config.EngineOpts{
			Bridge:        "none",
			StorageDriver: "overlay2",
			Restart:       &[]bool{false}[0],
			Graph:         "/var/lib/recovery-docker",
			Group:         "root",
			Host:          []string{"unix:///var/run/system-docker.sock"},
			UserlandProxy: &[]bool{false}[0],
		},
	}

	_, err := docker.Start(&recoveryConfig)
	if err != nil {
		log.Fatal(err)
	}

	_, err = config.ChainCfgFuncs(&recoveryConfig,
		[]config.CfgFuncData{
			{"loadSystemImages", sysinit.LoadBootstrapImages},
			{"recovery console", recoveryServices},
		})
	if err != nil {
		log.Fatal(err)
	}
}

func recoveryServices(cfg *config.CloudConfig) (*config.CloudConfig, error) {
	_, err := compose.RunServiceSet("recovery", cfg, map[string]*composeConfig.ServiceConfigV1{
		"recovery": &recoveryDockerService,
	})
	return nil, err
}
<|endoftext|>


<commit_before>package calcium

import (
	"context"

	"github.com/pkg/errors"
	enginetypes "github.com/projecteru2/core/engine/types"
	"github.com/projecteru2/core/resources"
	resourcetypes "github.com/projecteru2/core/resources/types"
	"github.com/projecteru2/core/types"
	"github.com/projecteru2/core/utils"
)

// ReallocResource updates workload resource dynamically
func (c *Calcium) ReallocResource(ctx context.Context, opts *types.ReallocOptions) (err error) {
	return c.withContainerLocked(ctx, opts.ID, func(container *types.Container) error {
		rrs, err := resources.MakeRequests(
			types.ResourceOptions{
				CPUQuotaRequest: container.CPUQuotaRequest + opts.ResourceOpts.CPUQuotaRequest,
				CPUQuotaLimit:   container.CPUQuotaLimit + opts.ResourceOpts.CPUQuotaLimit,
				CPUBind:         types.ParseTriOption(opts.CPUBindOpts, len(container.CPU) > 0),
				MemoryRequest:   container.MemoryRequest + opts.ResourceOpts.MemoryRequest,
				MemoryLimit:     container.MemoryLimit + opts.ResourceOpts.MemoryLimit,
				StorageRequest:  container.StorageRequest + opts.ResourceOpts.StorageRequest,
				StorageLimit:    container.StorageLimit + opts.ResourceOpts.StorageLimit,
				VolumeRequest:   types.MergeVolumeBindings(container.VolumeRequest, opts.ResourceOpts.VolumeRequest),
				VolumeLimit:     types.MergeVolumeBindings(container.VolumeLimit, opts.ResourceOpts.VolumeLimit),
			},
		)
		if err != nil {
			return errors.WithStack(err)
		}
		return c.doReallocOnNode(ctx, container.Nodename, container, rrs)
	})
}

// transaction: node resource
func (c *Calcium) doReallocOnNode(ctx context.Context, nodename string, container *types.Container, rrs resourcetypes.ResourceRequests) error {
	return c.withNodeLocked(ctx, nodename, func(node *types.Node) error {
		node.RecycleResources(&container.ResourceMeta)
		_, total, plans, err := resources.SelectNodesByResourceRequests(rrs, map[string]*types.Node{node.Name: node})
		if err != nil {
			return errors.WithStack(err)
		}
		if total < 1 {
			return errors.WithStack(types.ErrInsufficientRes)
		}

		return utils.Txn(
			ctx,

			// if update workload resources
			func(ctx context.Context) (err error) {
				return c.doReallocContainersOnInstance(ctx, node, plans, container)
			},
			// then commit changes
			func(ctx context.Context) error {
				for _, plan := range plans {
					plan.ApplyChangesOnNode(node, 1)
				}
				return c.store.UpdateNodes(ctx, node)
			},
			// no need rollback
			nil,

			c.config.GlobalTimeout,
		)
	})
}

func (c *Calcium) doReallocContainersOnInstance(ctx context.Context, node *types.Node, plans []resourcetypes.ResourcePlans, container *types.Container) (err error) {
	r := &types.ResourceMeta{}
	for _, plan := range plans {
		if r, err = plan.Dispense(resourcetypes.DispenseOptions{
			Node:             node,
			ExistingInstance: container,
		}, r); err != nil {
			return
		}
	}

	originalContainer := *container
	return utils.Txn(
		ctx,

		// if: update container resources
		func(ctx context.Context) error {
			r := &enginetypes.VirtualizationResource{
				CPU:           r.CPU,
				Quota:         r.CPUQuotaLimit,
				NUMANode:      r.NUMANode,
				Memory:        r.MemoryLimit,
				Volumes:       r.VolumeLimit.ToStringSlice(false, false),
				VolumePlan:    r.VolumePlanLimit.ToLiteral(),
				VolumeChanged: r.VolumeChanged,
				Storage:       r.StorageLimit,
			}
			return errors.WithStack(node.Engine.VirtualizationUpdateResource(ctx, container.ID, r))
		},

		// then: update container meta
		func(ctx context.Context) error {
			container.CPUQuotaRequest = r.CPUQuotaRequest
			container.CPUQuotaLimit = r.CPUQuotaLimit
			container.CPU = r.CPU
			container.MemoryRequest = r.MemoryRequest
			container.MemoryLimit = r.MemoryLimit
			container.VolumeRequest = r.VolumeRequest
			container.VolumePlanRequest = r.VolumePlanRequest
			container.VolumeLimit = r.VolumeLimit
			container.VolumePlanLimit = r.VolumePlanLimit
			container.StorageRequest = r.StorageRequest
			container.StorageLimit = r.StorageLimit
			return errors.WithStack(c.store.UpdateContainer(ctx, container))
		},

		// rollback: container meta
		func(ctx context.Context, failureByCond bool) error {
			if failureByCond {
				return nil
			}
			r := &enginetypes.VirtualizationResource{
				CPU:           originalContainer.CPU,
				Quota:         originalContainer.CPUQuotaLimit,
				NUMANode:      originalContainer.NUMANode,
				Memory:        originalContainer.MemoryLimit,
				Volumes:       originalContainer.VolumeLimit.ToStringSlice(false, false),
				VolumePlan:    originalContainer.VolumePlanLimit.ToLiteral(),
				VolumeChanged: r.VolumeChanged,
				Storage:       originalContainer.StorageLimit,
			}
			return errors.WithStack(node.Engine.VirtualizationUpdateResource(ctx, container.ID, r))
		},

		c.config.GlobalTimeout,
	)
}
<commit_msg>fix realloc transaction (#265)<commit_after>package calcium

import (
	"context"

	"github.com/pkg/errors"
	"github.com/projecteru2/core/engine"
	enginetypes "github.com/projecteru2/core/engine/types"
	"github.com/projecteru2/core/resources"
	resourcetypes "github.com/projecteru2/core/resources/types"
	"github.com/projecteru2/core/types"
	"github.com/projecteru2/core/utils"
)

// ReallocResource updates workload resource dynamically
func (c *Calcium) ReallocResource(ctx context.Context, opts *types.ReallocOptions) (err error) {
	return c.withContainerLocked(ctx, opts.ID, func(container *types.Container) error {
		rrs, err := resources.MakeRequests(
			types.ResourceOptions{
				CPUQuotaRequest: container.CPUQuotaRequest + opts.ResourceOpts.CPUQuotaRequest,
				CPUQuotaLimit:   container.CPUQuotaLimit + opts.ResourceOpts.CPUQuotaLimit,
				CPUBind:         types.ParseTriOption(opts.CPUBindOpts, len(container.CPU) > 0),
				MemoryRequest:   container.MemoryRequest + opts.ResourceOpts.MemoryRequest,
				MemoryLimit:     container.MemoryLimit + opts.ResourceOpts.MemoryLimit,
				StorageRequest:  container.StorageRequest + opts.ResourceOpts.StorageRequest,
				StorageLimit:    container.StorageLimit + opts.ResourceOpts.StorageLimit,
				VolumeRequest:   types.MergeVolumeBindings(container.VolumeRequest, opts.ResourceOpts.VolumeRequest),
				VolumeLimit:     types.MergeVolumeBindings(container.VolumeLimit, opts.ResourceOpts.VolumeLimit),
			},
		)
		if err != nil {
			return errors.WithStack(err)
		}
		return c.doReallocOnNode(ctx, container.Nodename, container, rrs)
	})
}

// transaction: node resource
func (c *Calcium) doReallocOnNode(ctx context.Context, nodename string, container *types.Container, rrs resourcetypes.ResourceRequests) error {
	return c.withNodeLocked(ctx, nodename, func(node *types.Node) error {
		node.RecycleResources(&container.ResourceMeta)
		_, total, plans, err := resources.SelectNodesByResourceRequests(rrs, map[string]*types.Node{node.Name: node})
		if err != nil {
			return errors.WithStack(err)
		}
		if total < 1 {
			return errors.WithStack(types.ErrInsufficientRes)
		}

		originalContainer := *container
		resourceMeta := &types.ResourceMeta{}
		return utils.Txn(
			ctx,

			// if update workload resources
			func(ctx context.Context) (err error) {
				resourceMeta := &types.ResourceMeta{}
				for _, plan := range plans {
					if resourceMeta, err = plan.Dispense(resourcetypes.DispenseOptions{
						Node:             node,
						ExistingInstance: container,
					}, resourceMeta); err != nil {
						return
					}
				}
				return errors.WithStack(c.doReallocContainersOnInstance(ctx, node.Engine, resourceMeta, container))
			},
			// then commit changes
			func(ctx context.Context) error {
				for _, plan := range plans {
					plan.ApplyChangesOnNode(node, 1)
				}
				return errors.WithStack(c.store.UpdateNodes(ctx, node))
			},
			// no need rollback
			func(ctx context.Context, failureByCond bool) (err error) {
				if failureByCond {
					return
				}
				r := &types.ResourceMeta{
					CPUQuotaRequest:   originalContainer.CPUQuotaRequest,
					CPUQuotaLimit:     originalContainer.CPUQuotaLimit,
					CPU:               originalContainer.CPU,
					NUMANode:          originalContainer.NUMANode,
					MemoryRequest:     originalContainer.MemoryRequest,
					MemoryLimit:       originalContainer.MemoryLimit,
					VolumeRequest:     originalContainer.VolumeRequest,
					VolumeLimit:       originalContainer.VolumeLimit,
					VolumePlanRequest: originalContainer.VolumePlanRequest,
					VolumePlanLimit:   originalContainer.VolumePlanLimit,
					VolumeChanged:     resourceMeta.VolumeChanged,
					StorageRequest:    originalContainer.StorageRequest,
					StorageLimit:      originalContainer.StorageLimit,
				}
				return errors.WithStack(c.doReallocContainersOnInstance(ctx, node.Engine, r, container))
			},

			c.config.GlobalTimeout,
		)
	})
}

func (c *Calcium) doReallocContainersOnInstance(ctx context.Context, engine engine.API, resourceMeta *types.ResourceMeta, container *types.Container) (err error) {

	originalContainer := *container
	return utils.Txn(
		ctx,

		// if: update container resources
		func(ctx context.Context) error {
			r := &enginetypes.VirtualizationResource{
				CPU:           resourceMeta.CPU,
				Quota:         resourceMeta.CPUQuotaLimit,
				NUMANode:      resourceMeta.NUMANode,
				Memory:        resourceMeta.MemoryLimit,
				Volumes:       resourceMeta.VolumeLimit.ToStringSlice(false, false),
				VolumePlan:    resourceMeta.VolumePlanLimit.ToLiteral(),
				VolumeChanged: resourceMeta.VolumeChanged,
				Storage:       resourceMeta.StorageLimit,
			}
			return errors.WithStack(engine.VirtualizationUpdateResource(ctx, container.ID, r))
		},

		// then: update container meta
		func(ctx context.Context) error {
			container.CPUQuotaRequest = resourceMeta.CPUQuotaRequest
			container.CPUQuotaLimit = resourceMeta.CPUQuotaLimit
			container.CPU = resourceMeta.CPU
			container.NUMANode = resourceMeta.NUMANode
			container.MemoryRequest = resourceMeta.MemoryRequest
			container.MemoryLimit = resourceMeta.MemoryLimit
			container.VolumeRequest = resourceMeta.VolumeRequest
			container.VolumePlanRequest = resourceMeta.VolumePlanRequest
			container.VolumeLimit = resourceMeta.VolumeLimit
			container.VolumePlanLimit = resourceMeta.VolumePlanLimit
			container.StorageRequest = resourceMeta.StorageRequest
			container.StorageLimit = resourceMeta.StorageLimit
			return errors.WithStack(c.store.UpdateContainer(ctx, container))
		},

		// rollback: container meta
		func(ctx context.Context, failureByCond bool) error {
			if failureByCond {
				return nil
			}
			r := &enginetypes.VirtualizationResource{
				CPU:           originalContainer.CPU,
				Quota:         originalContainer.CPUQuotaLimit,
				NUMANode:      originalContainer.NUMANode,
				Memory:        originalContainer.MemoryLimit,
				Volumes:       originalContainer.VolumeLimit.ToStringSlice(false, false),
				VolumePlan:    originalContainer.VolumePlanLimit.ToLiteral(),
				VolumeChanged: resourceMeta.VolumeChanged,
				Storage:       originalContainer.StorageLimit,
			}
			return errors.WithStack(engine.VirtualizationUpdateResource(ctx, container.ID, r))
		},

		c.config.GlobalTimeout,
	)
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package openstack

import (
	"github.com/gophercloud/gophercloud"
	"github.com/gophercloud/gophercloud/openstack/networking/v2/extensions"
	"github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/external"
	"github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/layer3/floatingips"
	"github.com/gophercloud/gophercloud/openstack/networking/v2/networks"
	neutronports "github.com/gophercloud/gophercloud/openstack/networking/v2/ports"
	"github.com/gophercloud/gophercloud/pagination"

	"k8s.io/cloud-provider-openstack/pkg/metrics"
	cpoerrors "k8s.io/cloud-provider-openstack/pkg/util/errors"
)

// GetNetworkExtensions returns an extension map.
func GetNetworkExtensions(client *gophercloud.ServiceClient) (map[string]bool, error) {
	seen := make(map[string]bool)

	mc := metrics.NewMetricContext("network_extension", "list")
	pager := extensions.List(client)
	err := pager.EachPage(func(page pagination.Page) (bool, error) {
		exts, err := extensions.ExtractExtensions(page)
		if err != nil {
			return false, err
		}
		for _, ext := range exts {
			seen[ext.Alias] = true
		}
		return true, nil
	})

	return seen, mc.ObserveRequest(err)
}

// GetFloatingIPs returns all the filtered floating IPs
func GetFloatingIPs(client *gophercloud.ServiceClient, opts floatingips.ListOpts) ([]floatingips.FloatingIP, error) {
	var floatingIPList []floatingips.FloatingIP

	mc := metrics.NewMetricContext("floating_ip", "list")
	allPages, err := floatingips.List(client, opts).AllPages()
	if mc.ObserveRequest(err) != nil {
		return floatingIPList, err
	}
	floatingIPList, err = floatingips.ExtractFloatingIPs(allPages)
	if err != nil {
		return floatingIPList, err
	}

	return floatingIPList, nil
}

// GetFloatingIPByPortID get the floating IP of the given port.
func GetFloatingIPByPortID(client *gophercloud.ServiceClient, portID string) (*floatingips.FloatingIP, error) {
	opt := floatingips.ListOpts{
		PortID: portID,
	}
	ips, err := GetFloatingIPs(client, opt)
	if err != nil {
		return nil, err
	}

	if len(ips) == 0 {
		return nil, nil
	}

	return &ips[0], nil
}

// GetFloatingNetworkID returns a floating network ID.
func GetFloatingNetworkID(client *gophercloud.ServiceClient) (string, error) {
	var floatingNetworkIds []string

	type NetworkWithExternalExt struct {
		networks.Network
		external.NetworkExternalExt
	}

	mc := metrics.NewMetricContext("network", "list")
	err := networks.List(client, networks.ListOpts{}).EachPage(func(page pagination.Page) (bool, error) {
		var externalNetwork []NetworkWithExternalExt
		err := networks.ExtractNetworksInto(page, &externalNetwork)
		if err != nil {
			return false, err
		}

		for _, externalNet := range externalNetwork {
			if externalNet.External {
				floatingNetworkIds = append(floatingNetworkIds, externalNet.ID)
			}
		}

		if len(floatingNetworkIds) > 1 {
			return false, cpoerrors.ErrMultipleResults
		}
		return true, nil
	})
	if err != nil {
		if cpoerrors.IsNotFound(err) {
			return "", mc.ObserveRequest(cpoerrors.ErrNotFound)
		}

		if err == cpoerrors.ErrMultipleResults {
			return floatingNetworkIds[0], mc.ObserveRequest(nil)
		}
		return "", mc.ObserveRequest(err)
	}

	if len(floatingNetworkIds) == 0 {
		return "", mc.ObserveRequest(cpoerrors.ErrNotFound)
	}

	return floatingNetworkIds[0], mc.ObserveRequest(nil)
}

// GetPorts gets all the filtered ports.
func GetPorts(client *gophercloud.ServiceClient, listOpts neutronports.ListOpts) ([]neutronports.Port, error) {
	mc := metrics.NewMetricContext("port", "list")
	allPages, err := neutronports.List(client, listOpts).AllPages()
	if mc.ObserveRequest(err) != nil {
		return []neutronports.Port{}, err
	}
	allPorts, err := neutronports.ExtractPorts(allPages)
	if err != nil {
		return []neutronports.Port{}, err
	}

	return allPorts, nil
}
<commit_msg>[occm] Skip External network with IPv6 for FIPs creation (#1992)<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package openstack

import (
	"github.com/gophercloud/gophercloud"
	"github.com/gophercloud/gophercloud/openstack/networking/v2/extensions"
	"github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/external"
	"github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/layer3/floatingips"
	"github.com/gophercloud/gophercloud/openstack/networking/v2/networks"
	neutronports "github.com/gophercloud/gophercloud/openstack/networking/v2/ports"
	"github.com/gophercloud/gophercloud/openstack/networking/v2/subnets"
	"github.com/gophercloud/gophercloud/pagination"

	"k8s.io/cloud-provider-openstack/pkg/metrics"
)

// GetNetworkExtensions returns an extension map.
func GetNetworkExtensions(client *gophercloud.ServiceClient) (map[string]bool, error) {
	seen := make(map[string]bool)

	mc := metrics.NewMetricContext("network_extension", "list")
	pager := extensions.List(client)
	err := pager.EachPage(func(page pagination.Page) (bool, error) {
		exts, err := extensions.ExtractExtensions(page)
		if err != nil {
			return false, err
		}
		for _, ext := range exts {
			seen[ext.Alias] = true
		}
		return true, nil
	})

	return seen, mc.ObserveRequest(err)
}

// GetFloatingIPs returns all the filtered floating IPs
func GetFloatingIPs(client *gophercloud.ServiceClient, opts floatingips.ListOpts) ([]floatingips.FloatingIP, error) {
	var floatingIPList []floatingips.FloatingIP

	mc := metrics.NewMetricContext("floating_ip", "list")
	allPages, err := floatingips.List(client, opts).AllPages()
	if mc.ObserveRequest(err) != nil {
		return floatingIPList, err
	}
	floatingIPList, err = floatingips.ExtractFloatingIPs(allPages)
	if err != nil {
		return floatingIPList, err
	}

	return floatingIPList, nil
}

// GetFloatingIPByPortID get the floating IP of the given port.
func GetFloatingIPByPortID(client *gophercloud.ServiceClient, portID string) (*floatingips.FloatingIP, error) {
	opt := floatingips.ListOpts{
		PortID: portID,
	}
	ips, err := GetFloatingIPs(client, opt)
	if err != nil {
		return nil, err
	}

	if len(ips) == 0 {
		return nil, nil
	}

	return &ips[0], nil
}

// GetFloatingNetworkID returns a floating network ID.
func GetFloatingNetworkID(client *gophercloud.ServiceClient) (string, error) {
	type NetworkWithExternalExt struct {
		networks.Network
		external.NetworkExternalExt
	}
	var externalNetworks []NetworkWithExternalExt

	mc := metrics.NewMetricContext("network", "list")
	page, err := networks.List(client, networks.ListOpts{}).AllPages()
	if err != nil {
		return "", mc.ObserveRequest(err)
	}

	err = networks.ExtractNetworksInto(page, &externalNetworks)
	if err != nil {
		return "", mc.ObserveRequest(err)
	}
	for _, externalNet := range externalNetworks {
		if externalNet.External {
			mc := metrics.NewMetricContext("subnet", "list")
			page, err := subnets.List(client, subnets.ListOpts{NetworkID: externalNet.ID, IPVersion: 4}).AllPages()
			if err != nil {
				return "", mc.ObserveRequest(err)
			}
			subnetList, err := subnets.ExtractSubnets(page)
			if err != nil {
				return "", mc.ObserveRequest(err)
			}
			if len(subnetList) == 0 {
				continue
			}
			return externalNet.ID, nil
		}
	}
	return "", nil
}

// GetPorts gets all the filtered ports.
func GetPorts(client *gophercloud.ServiceClient, listOpts neutronports.ListOpts) ([]neutronports.Port, error) {
	mc := metrics.NewMetricContext("port", "list")
	allPages, err := neutronports.List(client, listOpts).AllPages()
	if mc.ObserveRequest(err) != nil {
		return []neutronports.Port{}, err
	}
	allPorts, err := neutronports.ExtractPorts(allPages)
	if err != nil {
		return []neutronports.Port{}, err
	}

	return allPorts, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cinder

import (
	"fmt"
	"os"
	"path"
	"strings"
	"time"

	"github.com/golang/glog"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/kubernetes/pkg/util/exec"
	"k8s.io/kubernetes/pkg/util/mount"
	"k8s.io/kubernetes/pkg/volume"
	volumeutil "k8s.io/kubernetes/pkg/volume/util"
)

type cinderDiskAttacher struct {
	host           volume.VolumeHost
	cinderProvider CinderProvider
}

var _ volume.Attacher = &cinderDiskAttacher{}

var _ volume.AttachableVolumePlugin = &cinderPlugin{}

const (
	checkSleepDuration = time.Second
)

func (plugin *cinderPlugin) NewAttacher() (volume.Attacher, error) {
	cinder, err := getCloudProvider(plugin.host.GetCloudProvider())
	if err != nil {
		return nil, err
	}
	return &cinderDiskAttacher{
		host:           plugin.host,
		cinderProvider: cinder,
	}, nil
}

func (plugin *cinderPlugin) GetDeviceMountRefs(deviceMountPath string) ([]string, error) {
	mounter := plugin.host.GetMounter()
	return mount.GetMountRefs(mounter, deviceMountPath)
}

func (attacher *cinderDiskAttacher) Attach(spec *volume.Spec, nodeName types.NodeName) (string, error) {
	volumeSource, _, err := getVolumeSource(spec)
	if err != nil {
		return "", err
	}

	volumeID := volumeSource.VolumeID

	instances, res := attacher.cinderProvider.Instances()
	if !res {
		return "", fmt.Errorf("failed to list openstack instances")
	}
	instanceid, err := instances.InstanceID(nodeName)
	if err != nil {
		return "", err
	}
	if ind := strings.LastIndex(instanceid, "/"); ind >= 0 {
		instanceid = instanceid[(ind + 1):]
	}
	attached, err := attacher.cinderProvider.DiskIsAttached(volumeID, instanceid)
	if err != nil {
		// Log error and continue with attach
		glog.Warningf(
			"Error checking if volume (%q) is already attached to current instance (%q). Will continue and try attach anyway. err=%v",
			volumeID, instanceid, err)
	}

	if err == nil && attached {
		// Volume is already attached to instance.
		glog.Infof("Attach operation is successful. volume %q is already attached to instance %q.", volumeID, instanceid)
	} else {
		_, err = attacher.cinderProvider.AttachDisk(instanceid, volumeID)
		if err == nil {
			glog.Infof("Attach operation successful: volume %q attached to instance %q.", volumeID, instanceid)
		} else {
			glog.Infof("Attach volume %q to instance %q failed with %v", volumeID, instanceid, err)
			return "", err
		}
	}

	devicePath, err := attacher.cinderProvider.GetAttachmentDiskPath(instanceid, volumeID)
	if err != nil {
		glog.Infof("Attach volume %q to instance %q failed with %v", volumeID, instanceid, err)
		return "", err
	}

	return devicePath, err
}

func (attacher *cinderDiskAttacher) VolumesAreAttached(specs []*volume.Spec, nodeName types.NodeName) (map[*volume.Spec]bool, error) {
	volumesAttachedCheck := make(map[*volume.Spec]bool)
	volumeSpecMap := make(map[string]*volume.Spec)
	volumeIDList := []string{}
	for _, spec := range specs {
		volumeSource, _, err := getVolumeSource(spec)
		if err != nil {
			glog.Errorf("Error getting volume (%q) source : %v", spec.Name(), err)
			continue
		}

		volumeIDList = append(volumeIDList, volumeSource.VolumeID)
		volumesAttachedCheck[spec] = true
		volumeSpecMap[volumeSource.VolumeID] = spec
	}
	attachedResult, err := attacher.cinderProvider.DisksAreAttached(volumeIDList, string(nodeName))
	if err != nil {
		// Log error and continue with attach
		glog.Errorf(
			"Error checking if Volumes (%v) are already attached to current node (%q). Will continue and try attach anyway. err=%v",
			volumeIDList, nodeName, err)
		return volumesAttachedCheck, err
	}

	for volumeID, attached := range attachedResult {
		if !attached {
			spec := volumeSpecMap[volumeID]
			volumesAttachedCheck[spec] = false
			glog.V(2).Infof("VolumesAreAttached: check volume %q (specName: %q) is no longer attached", volumeID, spec.Name())
		}
	}
	return volumesAttachedCheck, nil
}

func (attacher *cinderDiskAttacher) WaitForAttach(spec *volume.Spec, devicePath string, timeout time.Duration) (string, error) {
	// NOTE: devicePath is is path as reported by Cinder, which may be incorrect and should not be used. See Issue #33128
	volumeSource, _, err := getVolumeSource(spec)
	if err != nil {
		return "", err
	}

	volumeID := volumeSource.VolumeID

	if devicePath == "" {
		return "", fmt.Errorf("WaitForAttach failed for Cinder disk %q: devicePath is empty.", volumeID)
	}

	ticker := time.NewTicker(checkSleepDuration)
	defer ticker.Stop()
	timer := time.NewTimer(timeout)
	defer timer.Stop()

	for {
		probeAttachedVolume()
		select {
		case <-ticker.C:
			glog.V(5).Infof("Checking Cinder disk %q is attached.", volumeID)
			probeAttachedVolume()
			if !attacher.cinderProvider.ShouldTrustDevicePath() {
				// Using the Cinder volume ID, find the real device path (See Issue #33128)
				devicePath = attacher.cinderProvider.GetDevicePath(volumeID)
			}
			exists, err := volumeutil.PathExists(devicePath)
			if exists && err == nil {
				glog.Infof("Successfully found attached Cinder disk %q at %v.", volumeID, devicePath)
				return devicePath, nil
			} else {
				// Log an error, and continue checking periodically
				glog.Errorf("Error: could not find attached Cinder disk %q (path: %q): %v", volumeID, devicePath, err)
			}
		case <-timer.C:
			return "", fmt.Errorf("Could not find attached Cinder disk %q. Timeout waiting for mount paths to be created.", volumeID)
		}
	}
}

func (attacher *cinderDiskAttacher) GetDeviceMountPath(
	spec *volume.Spec) (string, error) {
	volumeSource, _, err := getVolumeSource(spec)
	if err != nil {
		return "", err
	}

	return makeGlobalPDName(attacher.host, volumeSource.VolumeID), nil
}

// FIXME: this method can be further pruned.
func (attacher *cinderDiskAttacher) MountDevice(spec *volume.Spec, devicePath string, deviceMountPath string) error {
	mounter := attacher.host.GetMounter()
	notMnt, err := mounter.IsLikelyNotMountPoint(deviceMountPath)
	if err != nil {
		if os.IsNotExist(err) {
			if err := os.MkdirAll(deviceMountPath, 0750); err != nil {
				return err
			}
			notMnt = true
		} else {
			return err
		}
	}

	volumeSource, readOnly, err := getVolumeSource(spec)
	if err != nil {
		return err
	}

	options := []string{}
	if readOnly {
		options = append(options, "ro")
	}
	if notMnt {
		diskMounter := &mount.SafeFormatAndMount{Interface: mounter, Runner: exec.New()}
		err = diskMounter.FormatAndMount(devicePath, deviceMountPath, volumeSource.FSType, options)
		if err != nil {
			os.Remove(deviceMountPath)
			return err
		}
	}
	return nil
}

type cinderDiskDetacher struct {
	mounter        mount.Interface
	cinderProvider CinderProvider
}

var _ volume.Detacher = &cinderDiskDetacher{}

func (plugin *cinderPlugin) NewDetacher() (volume.Detacher, error) {
	cinder, err := getCloudProvider(plugin.host.GetCloudProvider())
	if err != nil {
		return nil, err
	}
	return &cinderDiskDetacher{
		mounter:        plugin.host.GetMounter(),
		cinderProvider: cinder,
	}, nil
}

func (detacher *cinderDiskDetacher) Detach(deviceMountPath string, nodeName types.NodeName) error {
	volumeID := path.Base(deviceMountPath)
	instances, res := detacher.cinderProvider.Instances()
	if !res {
		return fmt.Errorf("failed to list openstack instances")
	}
	instanceid, err := instances.InstanceID(nodeName)
	if ind := strings.LastIndex(instanceid, "/"); ind >= 0 {
		instanceid = instanceid[(ind + 1):]
	}

	attached, err := detacher.cinderProvider.DiskIsAttached(volumeID, instanceid)
	if err != nil {
		// Log error and continue with detach
		glog.Errorf(
			"Error checking if volume (%q) is already attached to current node (%q). Will continue and try detach anyway. err=%v",
			volumeID, nodeName, err)
	}

	if err == nil && !attached {
		// Volume is already detached from node.
		glog.Infof("detach operation was successful. volume %q is already detached from node %q.", volumeID, nodeName)
		return nil
	}

	if err = detacher.cinderProvider.DetachDisk(instanceid, volumeID); err != nil {
		glog.Errorf("Error detaching volume %q: %v", volumeID, err)
		return err
	}
	glog.Infof("detatached volume %q from instance %q", volumeID, instanceid)
	return nil
}

func (detacher *cinderDiskDetacher) UnmountDevice(deviceMountPath string) error {
	return volumeutil.UnmountPath(deviceMountPath, detacher.mounter)
}
<commit_msg>Cinder volume attacher: use instanceID not NodeID<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cinder

import (
	"fmt"
	"os"
	"path"
	"strings"
	"time"

	"github.com/golang/glog"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/kubernetes/pkg/util/exec"
	"k8s.io/kubernetes/pkg/util/mount"
	"k8s.io/kubernetes/pkg/volume"
	volumeutil "k8s.io/kubernetes/pkg/volume/util"
)

type cinderDiskAttacher struct {
	host           volume.VolumeHost
	cinderProvider CinderProvider
}

var _ volume.Attacher = &cinderDiskAttacher{}

var _ volume.AttachableVolumePlugin = &cinderPlugin{}

const (
	checkSleepDuration = time.Second
)

func (plugin *cinderPlugin) NewAttacher() (volume.Attacher, error) {
	cinder, err := getCloudProvider(plugin.host.GetCloudProvider())
	if err != nil {
		return nil, err
	}
	return &cinderDiskAttacher{
		host:           plugin.host,
		cinderProvider: cinder,
	}, nil
}

func (plugin *cinderPlugin) GetDeviceMountRefs(deviceMountPath string) ([]string, error) {
	mounter := plugin.host.GetMounter()
	return mount.GetMountRefs(mounter, deviceMountPath)
}

func (attacher *cinderDiskAttacher) Attach(spec *volume.Spec, nodeName types.NodeName) (string, error) {
	volumeSource, _, err := getVolumeSource(spec)
	if err != nil {
		return "", err
	}

	volumeID := volumeSource.VolumeID

	instanceid, err := attacher.nodeInstanceID(nodeName)
	if err != nil {
		return "", err
	}

	attached, err := attacher.cinderProvider.DiskIsAttached(volumeID, instanceid)
	if err != nil {
		// Log error and continue with attach
		glog.Warningf(
			"Error checking if volume (%q) is already attached to current instance (%q). Will continue and try attach anyway. err=%v",
			volumeID, instanceid, err)
	}

	if err == nil && attached {
		// Volume is already attached to instance.
		glog.Infof("Attach operation is successful. volume %q is already attached to instance %q.", volumeID, instanceid)
	} else {
		_, err = attacher.cinderProvider.AttachDisk(instanceid, volumeID)
		if err == nil {
			glog.Infof("Attach operation successful: volume %q attached to instance %q.", volumeID, instanceid)
		} else {
			glog.Infof("Attach volume %q to instance %q failed with %v", volumeID, instanceid, err)
			return "", err
		}
	}

	devicePath, err := attacher.cinderProvider.GetAttachmentDiskPath(instanceid, volumeID)
	if err != nil {
		glog.Infof("Attach volume %q to instance %q failed with %v", volumeID, instanceid, err)
		return "", err
	}

	return devicePath, err
}

func (attacher *cinderDiskAttacher) VolumesAreAttached(specs []*volume.Spec, nodeName types.NodeName) (map[*volume.Spec]bool, error) {
	volumesAttachedCheck := make(map[*volume.Spec]bool)
	volumeSpecMap := make(map[string]*volume.Spec)
	volumeIDList := []string{}
	for _, spec := range specs {
		volumeSource, _, err := getVolumeSource(spec)
		if err != nil {
			glog.Errorf("Error getting volume (%q) source : %v", spec.Name(), err)
			continue
		}

		volumeIDList = append(volumeIDList, volumeSource.VolumeID)
		volumesAttachedCheck[spec] = true
		volumeSpecMap[volumeSource.VolumeID] = spec
	}

	instanceid, err := attacher.nodeInstanceID(nodeName)
	if err != nil {
		return volumesAttachedCheck, err
	}

	attachedResult, err := attacher.cinderProvider.DisksAreAttached(volumeIDList, instanceid)
	if err != nil {
		// Log error and continue with attach
		glog.Errorf(
			"Error checking if Volumes (%v) are already attached to current node (%q). Will continue and try attach anyway. err=%v",
			volumeIDList, nodeName, err)
		return volumesAttachedCheck, err
	}

	for volumeID, attached := range attachedResult {
		if !attached {
			spec := volumeSpecMap[volumeID]
			volumesAttachedCheck[spec] = false
			glog.V(2).Infof("VolumesAreAttached: check volume %q (specName: %q) is no longer attached", volumeID, spec.Name())
		}
	}
	return volumesAttachedCheck, nil
}

func (attacher *cinderDiskAttacher) WaitForAttach(spec *volume.Spec, devicePath string, timeout time.Duration) (string, error) {
	// NOTE: devicePath is is path as reported by Cinder, which may be incorrect and should not be used. See Issue #33128
	volumeSource, _, err := getVolumeSource(spec)
	if err != nil {
		return "", err
	}

	volumeID := volumeSource.VolumeID

	if devicePath == "" {
		return "", fmt.Errorf("WaitForAttach failed for Cinder disk %q: devicePath is empty.", volumeID)
	}

	ticker := time.NewTicker(checkSleepDuration)
	defer ticker.Stop()
	timer := time.NewTimer(timeout)
	defer timer.Stop()

	for {
		probeAttachedVolume()
		select {
		case <-ticker.C:
			glog.V(5).Infof("Checking Cinder disk %q is attached.", volumeID)
			probeAttachedVolume()
			if !attacher.cinderProvider.ShouldTrustDevicePath() {
				// Using the Cinder volume ID, find the real device path (See Issue #33128)
				devicePath = attacher.cinderProvider.GetDevicePath(volumeID)
			}
			exists, err := volumeutil.PathExists(devicePath)
			if exists && err == nil {
				glog.Infof("Successfully found attached Cinder disk %q at %v.", volumeID, devicePath)
				return devicePath, nil
			} else {
				// Log an error, and continue checking periodically
				glog.Errorf("Error: could not find attached Cinder disk %q (path: %q): %v", volumeID, devicePath, err)
			}
		case <-timer.C:
			return "", fmt.Errorf("Could not find attached Cinder disk %q. Timeout waiting for mount paths to be created.", volumeID)
		}
	}
}

func (attacher *cinderDiskAttacher) GetDeviceMountPath(
	spec *volume.Spec) (string, error) {
	volumeSource, _, err := getVolumeSource(spec)
	if err != nil {
		return "", err
	}

	return makeGlobalPDName(attacher.host, volumeSource.VolumeID), nil
}

// FIXME: this method can be further pruned.
func (attacher *cinderDiskAttacher) MountDevice(spec *volume.Spec, devicePath string, deviceMountPath string) error {
	mounter := attacher.host.GetMounter()
	notMnt, err := mounter.IsLikelyNotMountPoint(deviceMountPath)
	if err != nil {
		if os.IsNotExist(err) {
			if err := os.MkdirAll(deviceMountPath, 0750); err != nil {
				return err
			}
			notMnt = true
		} else {
			return err
		}
	}

	volumeSource, readOnly, err := getVolumeSource(spec)
	if err != nil {
		return err
	}

	options := []string{}
	if readOnly {
		options = append(options, "ro")
	}
	if notMnt {
		diskMounter := &mount.SafeFormatAndMount{Interface: mounter, Runner: exec.New()}
		err = diskMounter.FormatAndMount(devicePath, deviceMountPath, volumeSource.FSType, options)
		if err != nil {
			os.Remove(deviceMountPath)
			return err
		}
	}
	return nil
}

type cinderDiskDetacher struct {
	mounter        mount.Interface
	cinderProvider CinderProvider
}

var _ volume.Detacher = &cinderDiskDetacher{}

func (plugin *cinderPlugin) NewDetacher() (volume.Detacher, error) {
	cinder, err := getCloudProvider(plugin.host.GetCloudProvider())
	if err != nil {
		return nil, err
	}
	return &cinderDiskDetacher{
		mounter:        plugin.host.GetMounter(),
		cinderProvider: cinder,
	}, nil
}

func (detacher *cinderDiskDetacher) Detach(deviceMountPath string, nodeName types.NodeName) error {
	volumeID := path.Base(deviceMountPath)
	instances, res := detacher.cinderProvider.Instances()
	if !res {
		return fmt.Errorf("failed to list openstack instances")
	}
	instanceid, err := instances.InstanceID(nodeName)
	if ind := strings.LastIndex(instanceid, "/"); ind >= 0 {
		instanceid = instanceid[(ind + 1):]
	}

	attached, err := detacher.cinderProvider.DiskIsAttached(volumeID, instanceid)
	if err != nil {
		// Log error and continue with detach
		glog.Errorf(
			"Error checking if volume (%q) is already attached to current node (%q). Will continue and try detach anyway. err=%v",
			volumeID, nodeName, err)
	}

	if err == nil && !attached {
		// Volume is already detached from node.
		glog.Infof("detach operation was successful. volume %q is already detached from node %q.", volumeID, nodeName)
		return nil
	}

	if err = detacher.cinderProvider.DetachDisk(instanceid, volumeID); err != nil {
		glog.Errorf("Error detaching volume %q: %v", volumeID, err)
		return err
	}
	glog.Infof("detatached volume %q from instance %q", volumeID, instanceid)
	return nil
}

func (detacher *cinderDiskDetacher) UnmountDevice(deviceMountPath string) error {
	return volumeutil.UnmountPath(deviceMountPath, detacher.mounter)
}

func (attacher *cinderDiskAttacher) nodeInstanceID(nodeName types.NodeName) (string, error) {
	instances, res := attacher.cinderProvider.Instances()
	if !res {
		return "", fmt.Errorf("failed to list openstack instances")
	}
	instanceid, err := instances.InstanceID(nodeName)
	if err != nil {
		return "", err
	}
	if ind := strings.LastIndex(instanceid, "/"); ind >= 0 {
		instanceid = instanceid[(ind + 1):]
	}
	return instanceid, nil
}
<|endoftext|>


<commit_before>package main_test

import (
	"fmt"
	"syscall"
	"time"

	"github.com/cloudfoundry/storeadapter"
	"github.com/cloudfoundry/storeadapter/storerunner/etcdstorerunner"
	. "github.com/onsi/ginkgo"
	"github.com/onsi/ginkgo/config"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"
	"github.com/pivotal-golang/clock"
	"github.com/pivotal-golang/lager"
	"github.com/pivotal-golang/lager/lagertest"

	"github.com/cloudfoundry-incubator/consuladapter"
	"github.com/cloudfoundry-incubator/converger/cmd/converger/testrunner"
	Bbs "github.com/cloudfoundry-incubator/runtime-schema/bbs"
	"github.com/cloudfoundry-incubator/runtime-schema/bbs/shared"
	"github.com/cloudfoundry-incubator/runtime-schema/models"
)

var _ = Describe("Converger", func() {
	const (
		exitDuration = 4 * time.Second
	)

	var (
		etcdRunner *etcdstorerunner.ETCDClusterRunner
		bbs        *Bbs.BBS
		runner     *testrunner.ConvergerRunner

		consulRunner  *consuladapter.ClusterRunner
		consulSession *consuladapter.Session

		convergeRepeatInterval      time.Duration
		taskKickInterval            time.Duration
		expireCompletedTaskDuration time.Duration

		etcdClient storeadapter.StoreAdapter

		logger lager.Logger
	)

	SynchronizedBeforeSuite(func() []byte {
		convergerBinPath, err := Build("github.com/cloudfoundry-incubator/converger/cmd/converger", "-race")
		Expect(err).NotTo(HaveOccurred())
		return []byte(convergerBinPath)
	}, func(convergerBinPath []byte) {
		etcdPort := 5001 + config.GinkgoConfig.ParallelNode
		etcdCluster := fmt.Sprintf("http://127.0.0.1:%d", etcdPort)
		etcdRunner = etcdstorerunner.NewETCDClusterRunner(etcdPort, 1, nil)

		etcdClient = etcdRunner.Adapter(nil)

		consulRunner = consuladapter.NewClusterRunner(
			9001+config.GinkgoConfig.ParallelNode*consuladapter.PortOffsetLength,
			1,
			"http",
		)

		logger = lagertest.NewTestLogger("test")

		runner = testrunner.New(
			string(convergerBinPath),
			testrunner.Config{
				EtcdCluster:   etcdCluster,
				ConsulCluster: consulRunner.ConsulCluster(),
				LogLevel:      "info",
			})
	})

	SynchronizedAfterSuite(func() {
	}, func() {
		CleanupBuildArtifacts()
	})

	BeforeEach(func() {
		etcdRunner.Start()
		consulRunner.Start()
		consulRunner.WaitUntilReady()

		consulSession = consulRunner.NewSession("a-session")
		bbs = Bbs.NewBBS(etcdClient, consulSession, "http://receptor.bogus.com", clock.NewClock(), logger)

		capacity := models.NewCellCapacity(512, 1024, 124)
		cellPresence := models.NewCellPresence("the-cell-id", "1.2.3.4", "the-zone", capacity)

		value, err := models.ToJSON(cellPresence)
		Expect(err).NotTo(HaveOccurred())

		_, err = consulSession.SetPresence(shared.CellSchemaPath(cellPresence.CellID), value)
		Expect(err).NotTo(HaveOccurred())

		convergeRepeatInterval = 500 * time.Millisecond
		taskKickInterval = convergeRepeatInterval
		expireCompletedTaskDuration = 3 * convergeRepeatInterval
	})

	AfterEach(func() {
		runner.KillWithFire()
		consulRunner.Stop()
		etcdRunner.Stop()
	})

	startConverger := func() {
		runner.Start(convergeRepeatInterval, taskKickInterval, 30*time.Minute, expireCompletedTaskDuration)
		time.Sleep(convergeRepeatInterval)
	}

	createRunningTaskWithDeadCell := func() {
		task := models.Task{
			Domain: "tests",

			TaskGuid: "task-guid",
			RootFS:   "some:rootfs",
			Action: &models.RunAction{
				Path: "cat",
				Args: []string{"/tmp/file"},
			},
		}

		err := bbs.DesireTask(logger, task)
		Expect(err).NotTo(HaveOccurred())

		_, err = bbs.StartTask(logger, task.TaskGuid, "dead-cell")
		Expect(err).NotTo(HaveOccurred())
	}

	itIsInactive := func() {
		Describe("when a task is desired but its cell is dead", func() {
			JustBeforeEach(createRunningTaskWithDeadCell)

			It("does not converge the task", func() {
				Consistently(func() ([]models.Task, error) {
					return bbs.CompletedTasks(logger)
				}, 10*convergeRepeatInterval).Should(BeEmpty())
			})
		})
	}

	Context("when the converger has the lock", func() {
		Describe("when a task is desired but its cell is dead", func() {
			JustBeforeEach(createRunningTaskWithDeadCell)

			It("marks the task as completed and failed", func() {
				Consistently(func() ([]models.Task, error) {
					return bbs.CompletedTasks(logger)
				}, 0.5).Should(BeEmpty())

				startConverger()

				Eventually(func() ([]models.Task, error) {
					return bbs.CompletedTasks(logger)
				}, 10*convergeRepeatInterval).Should(HaveLen(1))
			})
		})
	})

	Context("when the converger loses the lock", func() {
		BeforeEach(func() {
			startConverger()
			Eventually(runner.Session, 5*time.Second).Should(gbytes.Say("acquire-lock-succeeded"))

			consulRunner.Reset()
		})

		It("exits with an error", func() {
			Eventually(runner.Session, exitDuration).Should(Exit(1))
		})
	})

	Context("when the converger initially does not have the lock", func() {
		var otherSession *consuladapter.Session

		BeforeEach(func() {
			otherSession = consulRunner.NewSession("other-session")
			err := otherSession.AcquireLock(shared.LockSchemaPath("converge_lock"), []byte("something-else"))
			Expect(err).NotTo(HaveOccurred())

			startConverger()
		})

		itIsInactive()

		Describe("when the lock becomes available", func() {
			BeforeEach(func() {
				otherSession.Destroy()
				time.Sleep(convergeRepeatInterval + 10*time.Millisecond)
			})

			Describe("when a running task with a dead cell is present", func() {
				JustBeforeEach(createRunningTaskWithDeadCell)

				It("eventually marks the task as failed", func() {
					Eventually(func() ([]models.Task, error) {
						return bbs.FailedTasks(logger)
					}, 10*convergeRepeatInterval).Should(HaveLen(1))
				})
			})
		})
	})

	Describe("signal handling", func() {
		BeforeEach(func() {
			startConverger()
		})

		Describe("when it receives SIGINT", func() {
			It("exits successfully", func() {
				runner.Session.Command.Process.Signal(syscall.SIGINT)
				Eventually(runner.Session, exitDuration).Should(Exit(0))
			})
		})

		Describe("when it receives SIGTERM", func() {
			It("exits successfully", func() {
				runner.Session.Command.Process.Signal(syscall.SIGTERM)
				Eventually(runner.Session, exitDuration).Should(Exit(0))
			})
		})
	})

	Context("when etcd is down", func() {
		BeforeEach(func() {
			etcdRunner.Stop()
			startConverger()
		})

		It("starts", func() {
			Consistently(runner.Session).ShouldNot(Exit())
		})
	})
})
<commit_msg>Move consulrunner to a separate package<commit_after>package main_test

import (
	"fmt"
	"syscall"
	"time"

	"github.com/cloudfoundry/storeadapter"
	"github.com/cloudfoundry/storeadapter/storerunner/etcdstorerunner"
	. "github.com/onsi/ginkgo"
	"github.com/onsi/ginkgo/config"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"
	"github.com/pivotal-golang/clock"
	"github.com/pivotal-golang/lager"
	"github.com/pivotal-golang/lager/lagertest"

	"github.com/cloudfoundry-incubator/consuladapter"
	"github.com/cloudfoundry-incubator/consuladapter/consulrunner"
	"github.com/cloudfoundry-incubator/converger/cmd/converger/testrunner"
	Bbs "github.com/cloudfoundry-incubator/runtime-schema/bbs"
	"github.com/cloudfoundry-incubator/runtime-schema/bbs/shared"
	"github.com/cloudfoundry-incubator/runtime-schema/models"
)

var _ = Describe("Converger", func() {
	const (
		exitDuration = 4 * time.Second
	)

	var (
		etcdRunner *etcdstorerunner.ETCDClusterRunner
		bbs        *Bbs.BBS
		runner     *testrunner.ConvergerRunner

		consulRunner  *consulrunner.ClusterRunner
		consulSession *consuladapter.Session

		convergeRepeatInterval      time.Duration
		taskKickInterval            time.Duration
		expireCompletedTaskDuration time.Duration

		etcdClient storeadapter.StoreAdapter

		logger lager.Logger
	)

	SynchronizedBeforeSuite(func() []byte {
		convergerBinPath, err := Build("github.com/cloudfoundry-incubator/converger/cmd/converger", "-race")
		Expect(err).NotTo(HaveOccurred())
		return []byte(convergerBinPath)
	}, func(convergerBinPath []byte) {
		etcdPort := 5001 + config.GinkgoConfig.ParallelNode
		etcdCluster := fmt.Sprintf("http://127.0.0.1:%d", etcdPort)
		etcdRunner = etcdstorerunner.NewETCDClusterRunner(etcdPort, 1, nil)

		etcdClient = etcdRunner.Adapter(nil)

		consulRunner = consulrunner.NewClusterRunner(
			9001+config.GinkgoConfig.ParallelNode*consulrunner.PortOffsetLength,
			1,
			"http",
		)

		logger = lagertest.NewTestLogger("test")

		runner = testrunner.New(
			string(convergerBinPath),
			testrunner.Config{
				EtcdCluster:   etcdCluster,
				ConsulCluster: consulRunner.ConsulCluster(),
				LogLevel:      "info",
			})
	})

	SynchronizedAfterSuite(func() {
	}, func() {
		CleanupBuildArtifacts()
	})

	BeforeEach(func() {
		etcdRunner.Start()
		consulRunner.Start()
		consulRunner.WaitUntilReady()

		consulSession = consulRunner.NewSession("a-session")
		bbs = Bbs.NewBBS(etcdClient, consulSession, "http://receptor.bogus.com", clock.NewClock(), logger)

		capacity := models.NewCellCapacity(512, 1024, 124)
		cellPresence := models.NewCellPresence("the-cell-id", "1.2.3.4", "the-zone", capacity)

		value, err := models.ToJSON(cellPresence)
		Expect(err).NotTo(HaveOccurred())

		_, err = consulSession.SetPresence(shared.CellSchemaPath(cellPresence.CellID), value)
		Expect(err).NotTo(HaveOccurred())

		convergeRepeatInterval = 500 * time.Millisecond
		taskKickInterval = convergeRepeatInterval
		expireCompletedTaskDuration = 3 * convergeRepeatInterval
	})

	AfterEach(func() {
		runner.KillWithFire()
		consulRunner.Stop()
		etcdRunner.Stop()
	})

	startConverger := func() {
		runner.Start(convergeRepeatInterval, taskKickInterval, 30*time.Minute, expireCompletedTaskDuration)
		time.Sleep(convergeRepeatInterval)
	}

	createRunningTaskWithDeadCell := func() {
		task := models.Task{
			Domain: "tests",

			TaskGuid: "task-guid",
			RootFS:   "some:rootfs",
			Action: &models.RunAction{
				Path: "cat",
				Args: []string{"/tmp/file"},
			},
		}

		err := bbs.DesireTask(logger, task)
		Expect(err).NotTo(HaveOccurred())

		_, err = bbs.StartTask(logger, task.TaskGuid, "dead-cell")
		Expect(err).NotTo(HaveOccurred())
	}

	itIsInactive := func() {
		Describe("when a task is desired but its cell is dead", func() {
			JustBeforeEach(createRunningTaskWithDeadCell)

			It("does not converge the task", func() {
				Consistently(func() ([]models.Task, error) {
					return bbs.CompletedTasks(logger)
				}, 10*convergeRepeatInterval).Should(BeEmpty())
			})
		})
	}

	Context("when the converger has the lock", func() {
		Describe("when a task is desired but its cell is dead", func() {
			JustBeforeEach(createRunningTaskWithDeadCell)

			It("marks the task as completed and failed", func() {
				Consistently(func() ([]models.Task, error) {
					return bbs.CompletedTasks(logger)
				}, 0.5).Should(BeEmpty())

				startConverger()

				Eventually(func() ([]models.Task, error) {
					return bbs.CompletedTasks(logger)
				}, 10*convergeRepeatInterval).Should(HaveLen(1))
			})
		})
	})

	Context("when the converger loses the lock", func() {
		BeforeEach(func() {
			startConverger()
			Eventually(runner.Session, 5*time.Second).Should(gbytes.Say("acquire-lock-succeeded"))

			consulRunner.Reset()
		})

		It("exits with an error", func() {
			Eventually(runner.Session, exitDuration).Should(Exit(1))
		})
	})

	Context("when the converger initially does not have the lock", func() {
		var otherSession *consuladapter.Session

		BeforeEach(func() {
			otherSession = consulRunner.NewSession("other-session")
			err := otherSession.AcquireLock(shared.LockSchemaPath("converge_lock"), []byte("something-else"))
			Expect(err).NotTo(HaveOccurred())

			startConverger()
		})

		itIsInactive()

		Describe("when the lock becomes available", func() {
			BeforeEach(func() {
				otherSession.Destroy()
				time.Sleep(convergeRepeatInterval + 10*time.Millisecond)
			})

			Describe("when a running task with a dead cell is present", func() {
				JustBeforeEach(createRunningTaskWithDeadCell)

				It("eventually marks the task as failed", func() {
					Eventually(func() ([]models.Task, error) {
						return bbs.FailedTasks(logger)
					}, 10*convergeRepeatInterval).Should(HaveLen(1))
				})
			})
		})
	})

	Describe("signal handling", func() {
		BeforeEach(func() {
			startConverger()
		})

		Describe("when it receives SIGINT", func() {
			It("exits successfully", func() {
				runner.Session.Command.Process.Signal(syscall.SIGINT)
				Eventually(runner.Session, exitDuration).Should(Exit(0))
			})
		})

		Describe("when it receives SIGTERM", func() {
			It("exits successfully", func() {
				runner.Session.Command.Process.Signal(syscall.SIGTERM)
				Eventually(runner.Session, exitDuration).Should(Exit(0))
			})
		})
	})

	Context("when etcd is down", func() {
		BeforeEach(func() {
			etcdRunner.Stop()
			startConverger()
		})

		It("starts", func() {
			Consistently(runner.Session).ShouldNot(Exit())
		})
	})
})
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"os";
	"io";
	"flag";
	"fmt";
	"tabwriter";
)

var (
	tabwidth = flag.Int("tabwidth", 4, "tab width");
	usetabs = flag.Bool("usetabs", false, "align with tabs instead of blanks");
)

func Error(format string, params ...) {
	fmt.printf(format, params);
	sys.exit(1);
}

func Untab(name string, src *os.FD, dst *tabwriter.Writer) {
	n, err := io.Copy(src, dst);
	if err != nil {
		Error("error while processing %s (%v)", name, err);
	}
	//dst.Flush();
}

func main() {
	flag.Parse();
	padchar := byte(' ');
	if *usetabs {
		padchar = '\t';
	}
	dst := tabwriter.New(os.Stdout, *tabwidth, 1, padchar, true, false);
	if flag.NArg() > 0 {
		for i := 0; i < flag.NArg(); i++ {
			name := flag.Arg(i);
			src, err := os.Open(name, os.O_RDONLY, 0);
			if err != nil {
				Error("could not open %s (%v)\n", name, err);
			}
			Untab(name, src, dst);
			src.Close();  // ignore errors
		}
	} else {
		// no files => use stdin
		Untab("/dev/stdin", os.Stdin, dst);
	}
}
<commit_msg>- fixed untab.go to compile again<commit_after>// Copyright 2009 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"os";
	"io";
	"flag";
	"fmt";
	"tabwriter";
)

var (
	tabwidth = flag.Int("tabwidth", 4, "tab width");
	usetabs = flag.Bool("usetabs", false, "align with tabs instead of blanks");
)

func error(format string, params ...) {
	fmt.Printf(format, params);
	sys.exit(1);
}

func untab(name string, src *os.FD, dst *tabwriter.Writer) {
	n, err := io.Copy(src, dst);
	if err != nil {
		error("error while processing %s (%v)", name, err);
	}
	//dst.Flush();
}

func main() {
	flag.Parse();
	padchar := byte(' ');
	if *usetabs {
		padchar = '\t';
	}
	dst := tabwriter.New(os.Stdout, *tabwidth, 1, padchar, true, false);
	if flag.NArg() > 0 {
		for i := 0; i < flag.NArg(); i++ {
			name := flag.Arg(i);
			src, err := os.Open(name, os.O_RDONLY, 0);
			if err != nil {
				error("could not open %s (%v)\n", name, err);
			}
			untab(name, src, dst);
			src.Close();  // ignore errors
		}
	} else {
		// no files => use stdin
		untab("/dev/stdin", os.Stdin, dst);
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 ikawaha
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// 	You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package tokenize

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"runtime"
	"strings"

	"github.com/ikawaha/kagome.ipadic/tokenizer"
)

// subcommand property
const (
	CommandName  = "tokenize"
	Description  = `command line tokenize`
	usageMessage = "%s [-file input_file] [-dic dic_file] [-udic userdic_file] [-sysdic (normal|simple)] [-mode (normal|search|extended)]\n"
)

// ErrorWriter writes to stderr
var (
	ErrorWriter = os.Stderr
)

// options
type option struct {
	file    string
	dic     string
	udic    string
	sysdic  string
	mode    string
	flagSet *flag.FlagSet
}

// ContinueOnError ErrorHandling // Return a descriptive error.
// ExitOnError                   // Call os.Exit(2).
// PanicOnError                  // Call panic with a descriptive error.flag.ContinueOnError
func newOption(w io.Writer, eh flag.ErrorHandling) (o *option) {
	o = &option{
		flagSet: flag.NewFlagSet(CommandName, eh),
	}
	// option settings
	o.flagSet.SetOutput(w)
	o.flagSet.StringVar(&o.file, "file", "", "input file")
	o.flagSet.StringVar(&o.dic, "dic", "", "dic")
	o.flagSet.StringVar(&o.udic, "udic", "", "user dic")
	o.flagSet.StringVar(&o.sysdic, "sysdic", "normal", "system dic type (normal|simple)")
	o.flagSet.StringVar(&o.mode, "mode", "normal", "tokenize mode (normal|search|extended)")

	return
}

func (o *option) parse(args []string) (err error) {
	if err = o.flagSet.Parse(args); err != nil {
		return
	}
	// validations
	if nonFlag := o.flagSet.Args(); len(nonFlag) != 0 {
		return fmt.Errorf("invalid argument: %v", nonFlag)
	}
	if o.mode != "" && o.mode != "normal" && o.mode != "search" && o.mode != "extended" {
		return fmt.Errorf("invalid argument: -mode %v\n", o.mode)
	}
	if o.sysdic != "" && o.sysdic != "normal" && o.sysdic != "simple" {
		return fmt.Errorf("invalid argument: -sysdic %v\n", o.sysdic)
	}
	return
}

//OptionCheck receives a slice of args and returns an error if it was not successfully parsed
func OptionCheck(args []string) (err error) {
	opt := newOption(ioutil.Discard, flag.ContinueOnError)
	if e := opt.parse(args); e != nil {
		return fmt.Errorf("%v, %v", CommandName, e)
	}
	return nil
}

// command main
func command(opt *option) error {
	var dic tokenizer.Dic
	if opt.dic == "" {
		if opt.sysdic == "simple" {
			dic = tokenizer.SysDicIPASimple()
		} else {
			dic = tokenizer.SysDic()
		}
	} else {
		var err error
		dic, err = tokenizer.NewDic(opt.dic)
		if err != nil {
			return err
		}
	}
	var udic tokenizer.UserDic
	if opt.udic != "" {
		var err error
		udic, err = tokenizer.NewUserDic(opt.udic)
		if err != nil {
			return err
		}
	}
	var fp = os.Stdin
	if opt.file != "" {
		var err error
		fp, err = os.Open(opt.file)
		if err != nil {
			return err
		}
		defer fp.Close()
	}

	t := tokenizer.NewWithDic(dic)
	t.SetUserDic(udic)

	mode := tokenizer.Normal
	switch opt.mode {
	case "normal":
		mode = tokenizer.Normal
		break
	case "search":
		mode = tokenizer.Search
	case "extended":
		mode = tokenizer.Extended
	}

	scanner := bufio.NewScanner(fp)
	for scanner.Scan() {
		line := scanner.Text()
		tokens := t.Analyze(line, mode)
		for i, size := 1, len(tokens); i < size; i++ {
			tok := tokens[i]
			c := tok.Features()
			if tok.Class == tokenizer.DUMMY {
				fmt.Printf("%s\n", tok.Surface)
			} else {
				fmt.Printf("%s\t%v\n", tok.Surface, strings.Join(c, ","))
			}
		}
	}
	return scanner.Err()
}

// Run receives the slice of args and executes the tokenize tool
func Run(args []string) error {
	opt := newOption(ErrorWriter, flag.ExitOnError)
	if e := opt.parse(args); e != nil {
		Usage()
		PrintDefaults(flag.ExitOnError)
		return fmt.Errorf("%v, %v", CommandName, e)
	}
	return command(opt)
}

// Usage provides information on the use of the tokenize tool
func Usage() {
	fmt.Fprintf(ErrorWriter, usageMessage, CommandName)
}

// PrintDefaults prints out the default flags
func PrintDefaults(eh flag.ErrorHandling) {
	o := newOption(ErrorWriter, eh)
	o.flagSet.PrintDefaults()
}
<commit_msg>Bugfix: remove an unused pkg<commit_after>// Copyright 2015 ikawaha
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// 	You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package tokenize

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"strings"

	"github.com/ikawaha/kagome.ipadic/tokenizer"
)

// subcommand property
const (
	CommandName  = "tokenize"
	Description  = `command line tokenize`
	usageMessage = "%s [-file input_file] [-dic dic_file] [-udic userdic_file] [-sysdic (normal|simple)] [-mode (normal|search|extended)]\n"
)

// ErrorWriter writes to stderr
var (
	ErrorWriter = os.Stderr
)

// options
type option struct {
	file    string
	dic     string
	udic    string
	sysdic  string
	mode    string
	flagSet *flag.FlagSet
}

// ContinueOnError ErrorHandling // Return a descriptive error.
// ExitOnError                   // Call os.Exit(2).
// PanicOnError                  // Call panic with a descriptive error.flag.ContinueOnError
func newOption(w io.Writer, eh flag.ErrorHandling) (o *option) {
	o = &option{
		flagSet: flag.NewFlagSet(CommandName, eh),
	}
	// option settings
	o.flagSet.SetOutput(w)
	o.flagSet.StringVar(&o.file, "file", "", "input file")
	o.flagSet.StringVar(&o.dic, "dic", "", "dic")
	o.flagSet.StringVar(&o.udic, "udic", "", "user dic")
	o.flagSet.StringVar(&o.sysdic, "sysdic", "normal", "system dic type (normal|simple)")
	o.flagSet.StringVar(&o.mode, "mode", "normal", "tokenize mode (normal|search|extended)")

	return
}

func (o *option) parse(args []string) (err error) {
	if err = o.flagSet.Parse(args); err != nil {
		return
	}
	// validations
	if nonFlag := o.flagSet.Args(); len(nonFlag) != 0 {
		return fmt.Errorf("invalid argument: %v", nonFlag)
	}
	if o.mode != "" && o.mode != "normal" && o.mode != "search" && o.mode != "extended" {
		return fmt.Errorf("invalid argument: -mode %v\n", o.mode)
	}
	if o.sysdic != "" && o.sysdic != "normal" && o.sysdic != "simple" {
		return fmt.Errorf("invalid argument: -sysdic %v\n", o.sysdic)
	}
	return
}

//OptionCheck receives a slice of args and returns an error if it was not successfully parsed
func OptionCheck(args []string) (err error) {
	opt := newOption(ioutil.Discard, flag.ContinueOnError)
	if e := opt.parse(args); e != nil {
		return fmt.Errorf("%v, %v", CommandName, e)
	}
	return nil
}

// command main
func command(opt *option) error {
	var dic tokenizer.Dic
	if opt.dic == "" {
		if opt.sysdic == "simple" {
			dic = tokenizer.SysDicIPASimple()
		} else {
			dic = tokenizer.SysDic()
		}
	} else {
		var err error
		dic, err = tokenizer.NewDic(opt.dic)
		if err != nil {
			return err
		}
	}
	var udic tokenizer.UserDic
	if opt.udic != "" {
		var err error
		udic, err = tokenizer.NewUserDic(opt.udic)
		if err != nil {
			return err
		}
	}
	var fp = os.Stdin
	if opt.file != "" {
		var err error
		fp, err = os.Open(opt.file)
		if err != nil {
			return err
		}
		defer fp.Close()
	}

	t := tokenizer.NewWithDic(dic)
	t.SetUserDic(udic)

	mode := tokenizer.Normal
	switch opt.mode {
	case "normal":
		mode = tokenizer.Normal
		break
	case "search":
		mode = tokenizer.Search
	case "extended":
		mode = tokenizer.Extended
	}

	scanner := bufio.NewScanner(fp)
	for scanner.Scan() {
		line := scanner.Text()
		tokens := t.Analyze(line, mode)
		for i, size := 1, len(tokens); i < size; i++ {
			tok := tokens[i]
			c := tok.Features()
			if tok.Class == tokenizer.DUMMY {
				fmt.Printf("%s\n", tok.Surface)
			} else {
				fmt.Printf("%s\t%v\n", tok.Surface, strings.Join(c, ","))
			}
		}
	}
	return scanner.Err()
}

// Run receives the slice of args and executes the tokenize tool
func Run(args []string) error {
	opt := newOption(ErrorWriter, flag.ExitOnError)
	if e := opt.parse(args); e != nil {
		Usage()
		PrintDefaults(flag.ExitOnError)
		return fmt.Errorf("%v, %v", CommandName, e)
	}
	return command(opt)
}

// Usage provides information on the use of the tokenize tool
func Usage() {
	fmt.Fprintf(ErrorWriter, usageMessage, CommandName)
}

// PrintDefaults prints out the default flags
func PrintDefaults(eh flag.ErrorHandling) {
	o := newOption(ErrorWriter, eh)
	o.flagSet.PrintDefaults()
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/gob"
	"fmt"
	"sync"
	"time"

	"github.com/Symantec/Dominator/lib/constants"
	"github.com/Symantec/Dominator/lib/log"
	"github.com/Symantec/Dominator/lib/mdb"
	"github.com/Symantec/Dominator/lib/srpc"
	proto "github.com/Symantec/Dominator/proto/hypervisor"
)

var emptyTags = make(map[string]string)

type hypervisorGeneratorType struct {
	logger       log.DebugLogger
	eventChannel chan<- struct{}
	mutex        sync.Mutex
	vms          map[string]*proto.VmInfo
}

func newHypervisorGenerator(args []string,
	logger log.DebugLogger) (generator, error) {
	g := &hypervisorGeneratorType{
		logger: logger,
		vms:    make(map[string]*proto.VmInfo),
	}
	go g.daemon()
	return g, nil
}

func (g *hypervisorGeneratorType) daemon() {
	address := fmt.Sprintf(":%d", constants.HypervisorPortNumber)
	for {
		if err := g.getUpdates(address); err != nil {
			g.logger.Println(err)
			time.Sleep(time.Second)
		}
	}
}

func (g *hypervisorGeneratorType) getUpdates(hypervisor string) error {
	client, err := srpc.DialHTTP("tcp", hypervisor, 0)
	if err != nil {
		return err
	}
	defer client.Close()
	conn, err := client.Call("Hypervisor.GetUpdates")
	if err != nil {
		return err
	}
	defer conn.Close()
	decoder := gob.NewDecoder(conn)
	initialUpdate := true
	for {
		var update proto.Update
		if err := decoder.Decode(&update); err != nil {
			return err
		}
		g.updateVMs(update.VMs, initialUpdate)
		initialUpdate = false
		select {
		case g.eventChannel <- struct{}{}:
		default:
		}
	}
}

func (g *hypervisorGeneratorType) Generate(unused_datacentre string,
	logger log.Logger) (*mdb.Mdb, error) {
	var newMdb mdb.Mdb
	g.mutex.Lock()
	defer g.mutex.Unlock()
	for ipAddr, vm := range g.vms {
		if vm.State == proto.StateRunning {
			tags := vm.Tags
			if tags == nil {
				tags = emptyTags
			}
			_, disableUpdates := tags["DisableUpdates"]
			var ownerGroup string
			if len(vm.OwnerGroups) > 0 {
				ownerGroup = vm.OwnerGroups[0]
			}
			newMdb.Machines = append(newMdb.Machines, mdb.Machine{
				Hostname:       ipAddr,
				IpAddress:      ipAddr,
				RequiredImage:  tags["RequiredImage"],
				PlannedImage:   tags["PlannedImage"],
				DisableUpdates: disableUpdates,
				OwnerGroup:     ownerGroup,
				Tags:           vm.Tags,
			})
		}
	}
	return &newMdb, nil
}

func (g *hypervisorGeneratorType) RegisterEventChannel(events chan<- struct{}) {
	g.eventChannel = events
}

func (g *hypervisorGeneratorType) updateVMs(vms map[string]*proto.VmInfo,
	initialUpdate bool) {
	vmsToDelete := make(map[string]struct{}, len(g.vms))
	if initialUpdate {
		for ipAddr := range g.vms {
			vmsToDelete[ipAddr] = struct{}{}
		}
	}
	g.mutex.Lock()
	defer g.mutex.Unlock()
	for ipAddr, vm := range vms {
		if vm == nil {
			delete(g.vms, ipAddr)
		} else {
			g.vms[ipAddr] = vm
		}
	}
	for ipAddr := range vmsToDelete {
		delete(g.vms, ipAddr)
	}
}
<commit_msg>Fix bug in mdbd: fix deleting of all VMs in hypervisor driver.<commit_after>package main

import (
	"encoding/gob"
	"fmt"
	"sync"
	"time"

	"github.com/Symantec/Dominator/lib/constants"
	"github.com/Symantec/Dominator/lib/log"
	"github.com/Symantec/Dominator/lib/mdb"
	"github.com/Symantec/Dominator/lib/srpc"
	proto "github.com/Symantec/Dominator/proto/hypervisor"
)

var emptyTags = make(map[string]string)

type hypervisorGeneratorType struct {
	logger       log.DebugLogger
	eventChannel chan<- struct{}
	mutex        sync.Mutex
	vms          map[string]*proto.VmInfo
}

func newHypervisorGenerator(args []string,
	logger log.DebugLogger) (generator, error) {
	g := &hypervisorGeneratorType{
		logger: logger,
		vms:    make(map[string]*proto.VmInfo),
	}
	go g.daemon()
	return g, nil
}

func (g *hypervisorGeneratorType) daemon() {
	address := fmt.Sprintf(":%d", constants.HypervisorPortNumber)
	for {
		if err := g.getUpdates(address); err != nil {
			g.logger.Println(err)
			time.Sleep(time.Second)
		}
	}
}

func (g *hypervisorGeneratorType) getUpdates(hypervisor string) error {
	client, err := srpc.DialHTTP("tcp", hypervisor, 0)
	if err != nil {
		return err
	}
	defer client.Close()
	conn, err := client.Call("Hypervisor.GetUpdates")
	if err != nil {
		return err
	}
	defer conn.Close()
	decoder := gob.NewDecoder(conn)
	initialUpdate := true
	for {
		var update proto.Update
		if err := decoder.Decode(&update); err != nil {
			return err
		}
		g.updateVMs(update.VMs, initialUpdate)
		initialUpdate = false
		select {
		case g.eventChannel <- struct{}{}:
		default:
		}
	}
}

func (g *hypervisorGeneratorType) Generate(unused_datacentre string,
	logger log.Logger) (*mdb.Mdb, error) {
	var newMdb mdb.Mdb
	g.mutex.Lock()
	defer g.mutex.Unlock()
	for ipAddr, vm := range g.vms {
		if vm.State == proto.StateRunning {
			tags := vm.Tags
			if tags == nil {
				tags = emptyTags
			}
			_, disableUpdates := tags["DisableUpdates"]
			var ownerGroup string
			if len(vm.OwnerGroups) > 0 {
				ownerGroup = vm.OwnerGroups[0]
			}
			newMdb.Machines = append(newMdb.Machines, mdb.Machine{
				Hostname:       ipAddr,
				IpAddress:      ipAddr,
				RequiredImage:  tags["RequiredImage"],
				PlannedImage:   tags["PlannedImage"],
				DisableUpdates: disableUpdates,
				OwnerGroup:     ownerGroup,
				Tags:           vm.Tags,
			})
		}
	}
	return &newMdb, nil
}

func (g *hypervisorGeneratorType) RegisterEventChannel(events chan<- struct{}) {
	g.eventChannel = events
}

func (g *hypervisorGeneratorType) updateVMs(vms map[string]*proto.VmInfo,
	initialUpdate bool) {
	vmsToDelete := make(map[string]struct{}, len(g.vms))
	if initialUpdate {
		for ipAddr := range g.vms {
			vmsToDelete[ipAddr] = struct{}{}
		}
	}
	g.mutex.Lock()
	defer g.mutex.Unlock()
	for ipAddr, vm := range vms {
		if vm == nil {
			delete(g.vms, ipAddr)
		} else {
			g.vms[ipAddr] = vm
			delete(vmsToDelete, ipAddr)
		}
	}
	for ipAddr := range vmsToDelete {
		delete(g.vms, ipAddr)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	notable "github.com/joshualcoffee/notable"
	"log"
	"net/http"
	"net/url"
	"os"
)

type OKResponse struct {
	Text string `json:"text"`
}

type SummaryResponse struct {
	Notes []notable.Note `json:"notes"`
}

func getAndSetHandler(responseWriter http.ResponseWriter, request *http.Request) {
	request.ParseForm()
	token := request.Form.Get("token")

	if validToken(token) {
		var err error

		if request.Method == "POST" {
			recordNote(request.Form)
			responseWriter.Write([]byte("Got it!"))
		} else {
			err = respondWithJSON(responseWriter, SummaryResponse{notable.Notes()})
		}

		if err != nil {
			http.Error(responseWriter, err.Error(), http.StatusInternalServerError)
		}
	} else {
		fmt.Printf("Invalid token received: %s\n", token)
		http.Error(responseWriter, "Invalid token", http.StatusForbidden)
	}
}

func clearHandler(responseWriter http.ResponseWriter, request *http.Request) {
	request.ParseForm()
	token := request.Form.Get("token")

	if validToken(token) && request.Method == "POST" {
		notable.Reset()
	}
}

func emailHandler(responseWriter http.ResponseWriter, request *http.Request) {
	request.ParseForm()
	token := request.Form.Get("token")

	if validToken(token) {
		responseWriter.Header().Set("Content-Type", "text/html")
		responseWriter.Write([]byte(notable.Email()))
	}
}

func recordNote(form url.Values) {
	user_id := form.Get("user_id")
	category := form.Get("trigger_word")
	text := form.Get("text")
	slackToken := os.Getenv("SLACK_API_TOKEN")

	notable.Record(user_id, category, text, slackToken)
}

func slashCommandToken() string {
	return os.Getenv("SLACK_TOKEN")
}

func validToken(token string) bool {
	return token == slashCommandToken()
}

func respondWithJSON(responseWriter http.ResponseWriter, response interface{}) error {
	responseJSON, err := json.Marshal(response)
	if err != nil {
		return err
	}

	responseWriter.Header().Set("Content-Type", "application/json")
	responseWriter.Write(responseJSON)

	return nil
}

func main() {
	port := os.Getenv("PORT")

	if port == "" {
		log.Fatal("$PORT must be set")
	}

	http.HandleFunc("/email", emailHandler)
	http.HandleFunc("/clear", clearHandler)
	http.HandleFunc("/", getAndSetHandler)

	log.Fatal(http.ListenAndServe(":"+port, nil))
}
<commit_msg>use harvesthq notable<commit_after>package main

import (
	"encoding/json"
	"fmt"
	notable "github.com/harvesthq/notable"
	"log"
	"net/http"
	"net/url"
	"os"
)

type OKResponse struct {
	Text string `json:"text"`
}

type SummaryResponse struct {
	Notes []notable.Note `json:"notes"`
}

func getAndSetHandler(responseWriter http.ResponseWriter, request *http.Request) {
	request.ParseForm()
	token := request.Form.Get("token")

	if validToken(token) {
		var err error

		if request.Method == "POST" {
			recordNote(request.Form)
			responseWriter.Write([]byte("Got it!"))
		} else {
			err = respondWithJSON(responseWriter, SummaryResponse{notable.Notes()})
		}

		if err != nil {
			http.Error(responseWriter, err.Error(), http.StatusInternalServerError)
		}
	} else {
		fmt.Printf("Invalid token received: %s\n", token)
		http.Error(responseWriter, "Invalid token", http.StatusForbidden)
	}
}

func clearHandler(responseWriter http.ResponseWriter, request *http.Request) {
	request.ParseForm()
	token := request.Form.Get("token")

	if validToken(token) && request.Method == "POST" {
		notable.Reset()
	}
}

func emailHandler(responseWriter http.ResponseWriter, request *http.Request) {
	request.ParseForm()
	token := request.Form.Get("token")

	if validToken(token) {
		responseWriter.Header().Set("Content-Type", "text/html")
		responseWriter.Write([]byte(notable.Email()))
	}
}

func recordNote(form url.Values) {
	user_id := form.Get("user_id")
	category := form.Get("trigger_word")
	text := form.Get("text")
	slackToken := os.Getenv("SLACK_API_TOKEN")

	notable.Record(user_id, category, text, slackToken)
}

func slashCommandToken() string {
	return os.Getenv("SLACK_TOKEN")
}

func validToken(token string) bool {
	return token == slashCommandToken()
}

func respondWithJSON(responseWriter http.ResponseWriter, response interface{}) error {
	responseJSON, err := json.Marshal(response)
	if err != nil {
		return err
	}

	responseWriter.Header().Set("Content-Type", "application/json")
	responseWriter.Write(responseJSON)

	return nil
}

func main() {
	port := os.Getenv("PORT")

	if port == "" {
		log.Fatal("$PORT must be set")
	}

	http.HandleFunc("/email", emailHandler)
	http.HandleFunc("/clear", clearHandler)
	http.HandleFunc("/", getAndSetHandler)

	log.Fatal(http.ListenAndServe(":"+port, nil))
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"github.com/timeredbull/tsuru/cmd"
	"io"
	"io/ioutil"
	"net/http"
	"strings"
)

type Service struct{}
type ServiceCreate struct{}

func (c *Service) Info() *cmd.Info {
	return &cmd.Info{
		Name:    "service",
		Usage:   "service (init|list|create|remove|update) [args]",
		Desc:    "manage services.",
		MinArgs: 1,
	}
}

func (c *Service) Subcommands() map[string]interface{} {
	return map[string]interface{}{
		"create": &ServiceCreate{},
		"update": &ServiceUpdate{},
	}
}

func (c *ServiceCreate) Info() *cmd.Info {
	desc := "Creates a service based on a passed manifest. The manifest format should be a yaml and follow the standard described in the documentation (should link to it here)"
	return &cmd.Info{
		Name:    "create",
		Usage:   "create path/to/manifesto",
		Desc:    desc,
		MinArgs: 1,
	}
}

func (c *ServiceCreate) Run(context *cmd.Context, client cmd.Doer) error {
	manifest := context.Args[0]
	url := cmd.GetUrl("/services")
	b, err := ioutil.ReadFile(manifest)
	if err != nil {
		return err
	}
	body := strings.NewReader(string(b))
	request, err := http.NewRequest("POST", url, body)
	if err != nil {
		return err
	}
	r, err := client.Do(request)
	if err != nil {
		return err
	}
	b, err = ioutil.ReadAll(r.Body)
	if err != nil {
		return err
	}
	io.WriteString(context.Stdout, string(b)+"\n")
	return nil
}

type ServiceRemove struct{}

func (c *ServiceRemove) Run(context *cmd.Context, client cmd.Doer) error {
	serviceName := context.Args[0]
	url := cmd.GetUrl("/services/" + serviceName)
	request, err := http.NewRequest("DELETE", url, nil)
	if err != nil {
		return err
	}
	_, err = client.Do(request)
	if err != nil {
		return err
	}
	io.WriteString(context.Stdout, "Service successfully removed.\n")
	return nil
}

func (c *ServiceRemove) Info() *cmd.Info {
	return &cmd.Info{
		Name:    "remove",
		Usage:   "remove <servicename>",
		Desc:    "removes a service from catalog",
		MinArgs: 1,
	}
}

type ServiceList struct{}

func (c *ServiceList) Info() *cmd.Info {
	return &cmd.Info{
		Name:  "list",
		Usage: "list",
		Desc:  "list services that belongs to user's team and it's service instances.",
	}
}

func (c *ServiceList) Run(ctx *cmd.Context, client cmd.Doer) error {
	url := cmd.GetUrl("/services")
	request, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return err
	}
	resp, err := client.Do(request)
	if err != nil {
		return err
	}
	b, err := ioutil.ReadAll(resp.Body)
	defer resp.Body.Close()
	if err != nil {
		return err
	}
	rslt, err := cmd.ShowServicesInstancesList(b)
	if err != nil {
		return err
	}
	ctx.Stdout.Write(rslt)
	return nil
}

type ServiceUpdate struct{}

func (c *ServiceUpdate) Info() *cmd.Info {
	return &cmd.Info{
		Name:    "update",
		Usage:   "service update <path/to/manifesto>",
		Desc:    "Update service data, extracting it from the given manifesto file.",
		MinArgs: 1,
	}
}

func (c *ServiceUpdate) Run(ctx *cmd.Context, client cmd.Doer) error {
	manifest := ctx.Args[0]
	b, err := ioutil.ReadFile(manifest)
	if err != nil {
		return err
	}
	request, err := http.NewRequest("PUT", cmd.GetUrl("/services"), bytes.NewReader(b))
	if err != nil {
		return err
	}
	resp, err := client.Do(request)
	if err != nil {
		return err
	}
	if resp.StatusCode == http.StatusNoContent {
		io.WriteString(ctx.Stdout, "Service successfully updated.\n")
	}
	return nil
}
<commit_msg>cmd/servicecli: removed strings package dependency<commit_after>package main

import (
	"bytes"
	"github.com/timeredbull/tsuru/cmd"
	"io"
	"io/ioutil"
	"net/http"
)

type Service struct{}
type ServiceCreate struct{}

func (c *Service) Info() *cmd.Info {
	return &cmd.Info{
		Name:    "service",
		Usage:   "service (init|list|create|remove|update) [args]",
		Desc:    "manage services.",
		MinArgs: 1,
	}
}

func (c *Service) Subcommands() map[string]interface{} {
	return map[string]interface{}{
		"create": &ServiceCreate{},
		"update": &ServiceUpdate{},
	}
}

func (c *ServiceCreate) Info() *cmd.Info {
	desc := "Creates a service based on a passed manifest. The manifest format should be a yaml and follow the standard described in the documentation (should link to it here)"
	return &cmd.Info{
		Name:    "create",
		Usage:   "create path/to/manifesto",
		Desc:    desc,
		MinArgs: 1,
	}
}

func (c *ServiceCreate) Run(context *cmd.Context, client cmd.Doer) error {
	manifest := context.Args[0]
	url := cmd.GetUrl("/services")
	b, err := ioutil.ReadFile(manifest)
	if err != nil {
		return err
	}
	request, err := http.NewRequest("POST", url, bytes.NewReader(b))
	if err != nil {
		return err
	}
	r, err := client.Do(request)
	if err != nil {
		return err
	}
	b, err = ioutil.ReadAll(r.Body)
	if err != nil {
		return err
	}
	io.WriteString(context.Stdout, string(b)+"\n")
	return nil
}

type ServiceRemove struct{}

func (c *ServiceRemove) Run(context *cmd.Context, client cmd.Doer) error {
	serviceName := context.Args[0]
	url := cmd.GetUrl("/services/" + serviceName)
	request, err := http.NewRequest("DELETE", url, nil)
	if err != nil {
		return err
	}
	_, err = client.Do(request)
	if err != nil {
		return err
	}
	io.WriteString(context.Stdout, "Service successfully removed.\n")
	return nil
}

func (c *ServiceRemove) Info() *cmd.Info {
	return &cmd.Info{
		Name:    "remove",
		Usage:   "remove <servicename>",
		Desc:    "removes a service from catalog",
		MinArgs: 1,
	}
}

type ServiceList struct{}

func (c *ServiceList) Info() *cmd.Info {
	return &cmd.Info{
		Name:  "list",
		Usage: "list",
		Desc:  "list services that belongs to user's team and it's service instances.",
	}
}

func (c *ServiceList) Run(ctx *cmd.Context, client cmd.Doer) error {
	url := cmd.GetUrl("/services")
	request, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return err
	}
	resp, err := client.Do(request)
	if err != nil {
		return err
	}
	b, err := ioutil.ReadAll(resp.Body)
	defer resp.Body.Close()
	if err != nil {
		return err
	}
	rslt, err := cmd.ShowServicesInstancesList(b)
	if err != nil {
		return err
	}
	ctx.Stdout.Write(rslt)
	return nil
}

type ServiceUpdate struct{}

func (c *ServiceUpdate) Info() *cmd.Info {
	return &cmd.Info{
		Name:    "update",
		Usage:   "service update <path/to/manifesto>",
		Desc:    "Update service data, extracting it from the given manifesto file.",
		MinArgs: 1,
	}
}

func (c *ServiceUpdate) Run(ctx *cmd.Context, client cmd.Doer) error {
	manifest := ctx.Args[0]
	b, err := ioutil.ReadFile(manifest)
	if err != nil {
		return err
	}
	request, err := http.NewRequest("PUT", cmd.GetUrl("/services"), bytes.NewReader(b))
	if err != nil {
		return err
	}
	resp, err := client.Do(request)
	if err != nil {
		return err
	}
	if resp.StatusCode == http.StatusNoContent {
		io.WriteString(ctx.Stdout, "Service successfully updated.\n")
	}
	return nil
}
<|endoftext|>