<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package model

import (
	"fmt"
	"strings"

	"github.com/golang/glog"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/kops/pkg/apis/kops"
	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/cloudup/awstasks"
)

// NetworkModelBuilder configures network objects
type NetworkModelBuilder struct {
	*KopsModelContext
	Lifecycle *fi.Lifecycle
}

var _ fi.ModelBuilder = &NetworkModelBuilder{}

func (b *NetworkModelBuilder) Build(c *fi.ModelBuilderContext) error {
	kubernetesVersion, err := b.KubernetesVersion()
	if err != nil {
		return err
	}

	sharedVPC := b.Cluster.SharedVPC()
	vpcName := b.ClusterName()

	// VPC that holds everything for the cluster
	{
		tags := b.CloudTags(vpcName, sharedVPC)

		t := &awstasks.VPC{
			Name:             s(vpcName),
			Lifecycle:        b.Lifecycle,
			Shared:           fi.Bool(sharedVPC),
			EnableDNSSupport: fi.Bool(true),
			Tags:             tags,
		}

		if sharedVPC && VersionGTE(kubernetesVersion, 1, 5) {
			// If we're running k8s 1.5, and we have e.g.  --kubelet-preferred-address-types=InternalIP,Hostname,ExternalIP,LegacyHostIP
			// then we don't need EnableDNSHostnames any more
			glog.V(4).Infof("Kubernetes version %q; skipping EnableDNSHostnames requirement on VPC", kubernetesVersion)
		} else {
			// In theory we don't need to enable it for >= 1.5,
			// but seems safer to stick with existing behaviour

			t.EnableDNSHostnames = fi.Bool(true)
		}

		if b.Cluster.Spec.NetworkID != "" {
			t.ID = s(b.Cluster.Spec.NetworkID)
		}
		if b.Cluster.Spec.NetworkCIDR != "" {
			t.CIDR = s(b.Cluster.Spec.NetworkCIDR)
		}
		c.AddTask(t)
	}

	if !sharedVPC {
		dhcp := &awstasks.DHCPOptions{
			Name:              s(b.ClusterName()),
			Lifecycle:         b.Lifecycle,
			DomainNameServers: s("AmazonProvidedDNS"),
		}
		if b.Region == "us-east-1" {
			dhcp.DomainName = s("ec2.internal")
		} else {
			dhcp.DomainName = s(b.Region + ".compute.internal")
		}
		c.AddTask(dhcp)

		c.AddTask(&awstasks.VPCDHCPOptionsAssociation{
			Name:        s(b.ClusterName()),
			Lifecycle:   b.Lifecycle,
			VPC:         b.LinkToVPC(),
			DHCPOptions: dhcp,
		})
	} else {
		// TODO: would be good to create these as shared, to verify them
	}

	allSubnetsShared := true
	for i := range b.Cluster.Spec.Subnets {
		subnetSpec := &b.Cluster.Spec.Subnets[i]
		sharedSubnet := subnetSpec.ProviderID != ""
		if !sharedSubnet {
			allSubnetsShared = false
		}
	}

	// We always have a public route table, though for private networks it is only used for NGWs and ELBs
	var publicRouteTable *awstasks.RouteTable
	{
		// The internet gateway is the main entry point to the cluster.
		igw := &awstasks.InternetGateway{
			Name:      s(b.ClusterName()),
			Lifecycle: b.Lifecycle,
			VPC:       b.LinkToVPC(),
			Shared:    fi.Bool(sharedVPC),
		}
		c.AddTask(igw)

		if !allSubnetsShared {
			publicRouteTable = &awstasks.RouteTable{
				Name:      s(b.ClusterName()),
				Lifecycle: b.Lifecycle,

				VPC: b.LinkToVPC(),
			}
			c.AddTask(publicRouteTable)

			// TODO: Validate when allSubnetsShared
			c.AddTask(&awstasks.Route{
				Name:            s("0.0.0.0/0"),
				Lifecycle:       b.Lifecycle,
				CIDR:            s("0.0.0.0/0"),
				RouteTable:      publicRouteTable,
				InternetGateway: igw,
			})
		}
	}

	privateZones := sets.NewString()

	for i := range b.Cluster.Spec.Subnets {
		subnetSpec := &b.Cluster.Spec.Subnets[i]
		sharedSubnet := subnetSpec.ProviderID != ""
		subnetName := subnetSpec.Name + "." + b.ClusterName()
		tags := b.CloudTags(subnetName, sharedSubnet)

		subnet := &awstasks.Subnet{
			Name:             s(subnetName),
			Lifecycle:        b.Lifecycle,
			VPC:              b.LinkToVPC(),
			AvailabilityZone: s(subnetSpec.Zone),
			CIDR:             s(subnetSpec.CIDR),
			Shared:           fi.Bool(sharedSubnet),
			Tags:             tags,
		}

		if subnetSpec.ProviderID != "" {
			subnet.ID = s(subnetSpec.ProviderID)
		}
		c.AddTask(subnet)

		switch subnetSpec.Type {
		case kops.SubnetTypePublic, kops.SubnetTypeUtility:
			if !sharedSubnet {
				c.AddTask(&awstasks.RouteTableAssociation{
					Name:       s(subnetSpec.Name + "." + b.ClusterName()),
					Lifecycle:  b.Lifecycle,
					RouteTable: publicRouteTable,
					Subnet:     subnet,
				})
			}

		case kops.SubnetTypePrivate:
			// Private subnets get a Network Gateway, and their own route table to associate them with the network gateway

			if !sharedSubnet {
				// Private Subnet Route Table Associations
				//
				// Map the Private subnet to the Private route table
				c.AddTask(&awstasks.RouteTableAssociation{
					Name:       s("private-" + subnetSpec.Name + "." + b.ClusterName()),
					Lifecycle:  b.Lifecycle,
					RouteTable: b.LinkToPrivateRouteTableInZone(subnetSpec.Zone),
					Subnet:     subnet,
				})

				// TODO: validate even if shared?
				privateZones.Insert(subnetSpec.Zone)
			}
		default:
			return fmt.Errorf("subnet %q has unknown type %q", subnetSpec.Name, subnetSpec.Type)
		}
	}

	// Loop over zones
	for i, zone := range privateZones.List() {

		utilitySubnet, err := b.LinkToUtilitySubnetInZone(zone)
		if err != nil {
			return err
		}

		var ngw *awstasks.NatGateway
		if b.Cluster.Spec.Subnets[i].Egress != "" {
			if strings.Contains(b.Cluster.Spec.Subnets[i].Egress, "nat-") {

				ngw = &awstasks.NatGateway{
					Name:                 s(zone + "." + b.ClusterName()),
					Lifecycle:            b.Lifecycle,
					Subnet:               utilitySubnet,
					ID:                   s(b.Cluster.Spec.Subnets[i].Egress),
					AssociatedRouteTable: b.LinkToPrivateRouteTableInZone(zone),
					// If we're here, it means this NatGateway was specified, so we are Shared
					Shared: fi.Bool(true),
				}

				c.AddTask(ngw)

			} else {
				return fmt.Errorf("kops currently only supports re-use of NAT Gateways. We will support more eventually! Please see https://github.com/kubernetes/kops/issues/1530")
			}

		} else {

			// Every NGW needs a public (Elastic) IP address, every private
			// subnet needs a NGW, lets create it. We tie it to a subnet
			// so we can track it in AWS
			var eip = &awstasks.ElasticIP{}

			eip = &awstasks.ElasticIP{
				Name:                           s(zone + "." + b.ClusterName()),
				Lifecycle:                      b.Lifecycle,
				AssociatedNatGatewayRouteTable: b.LinkToPrivateRouteTableInZone(zone),
			}

			c.AddTask(eip)
			// NAT Gateway
			//
			// All private subnets will need a NGW, one per zone
			//
			// The instances in the private subnet can access the Internet by
			// using a network address translation (NAT) gateway that resides
			// in the public subnet.

			//var ngw = &awstasks.NatGateway{}
			ngw = &awstasks.NatGateway{
				Name:                 s(zone + "." + b.ClusterName()),
				Lifecycle:            b.Lifecycle,
				Subnet:               utilitySubnet,
				ElasticIP:            eip,
				AssociatedRouteTable: b.LinkToPrivateRouteTableInZone(zone),
			}
			c.AddTask(ngw)
		}

		// Private Route Table
		//
		// The private route table that will route to the NAT Gateway
		rt := &awstasks.RouteTable{
			Name:      s(b.NamePrivateRouteTableInZone(zone)),
			VPC:       b.LinkToVPC(),
			Lifecycle: b.Lifecycle,
		}
		c.AddTask(rt)

		// Private Routes
		//
		// Routes for the private route table.
		// Will route to the NAT Gateway
		c.AddTask(&awstasks.Route{
			Name:       s("private-" + zone + "-0.0.0.0/0"),
			Lifecycle:  b.Lifecycle,
			CIDR:       s("0.0.0.0/0"),
			RouteTable: rt,
			NatGateway: ngw,
		})

	}

	return nil
}
<commit_msg>First pass<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package model

import (
	"fmt"
	"strings"

	"github.com/golang/glog"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/kops/pkg/apis/kops"
	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/cloudup/awstasks"
	"k8s.io/kubernetes/pkg/cloudprovider/providers/aws"
)

// NetworkModelBuilder configures network objects
type NetworkModelBuilder struct {
	*KopsModelContext
	Lifecycle *fi.Lifecycle
}

var _ fi.ModelBuilder = &NetworkModelBuilder{}

func (b *NetworkModelBuilder) Build(c *fi.ModelBuilderContext) error {
	kubernetesVersion, err := b.KubernetesVersion()
	if err != nil {
		return err
	}

	sharedVPC := b.Cluster.SharedVPC()
	vpcName := b.ClusterName()

	// VPC that holds everything for the cluster
	{
		tags := b.CloudTags(vpcName, sharedVPC)

		t := &awstasks.VPC{
			Name:             s(vpcName),
			Lifecycle:        b.Lifecycle,
			Shared:           fi.Bool(sharedVPC),
			EnableDNSSupport: fi.Bool(true),
			Tags:             tags,
		}

		if sharedVPC && VersionGTE(kubernetesVersion, 1, 5) {
			// If we're running k8s 1.5, and we have e.g.  --kubelet-preferred-address-types=InternalIP,Hostname,ExternalIP,LegacyHostIP
			// then we don't need EnableDNSHostnames any more
			glog.V(4).Infof("Kubernetes version %q; skipping EnableDNSHostnames requirement on VPC", kubernetesVersion)
		} else {
			// In theory we don't need to enable it for >= 1.5,
			// but seems safer to stick with existing behaviour

			t.EnableDNSHostnames = fi.Bool(true)
		}

		if b.Cluster.Spec.NetworkID != "" {
			t.ID = s(b.Cluster.Spec.NetworkID)
		}
		if b.Cluster.Spec.NetworkCIDR != "" {
			t.CIDR = s(b.Cluster.Spec.NetworkCIDR)
		}
		c.AddTask(t)
	}

	if !sharedVPC {
		dhcp := &awstasks.DHCPOptions{
			Name:              s(b.ClusterName()),
			Lifecycle:         b.Lifecycle,
			DomainNameServers: s("AmazonProvidedDNS"),
		}
		if b.Region == "us-east-1" {
			dhcp.DomainName = s("ec2.internal")
		} else {
			dhcp.DomainName = s(b.Region + ".compute.internal")
		}
		c.AddTask(dhcp)

		c.AddTask(&awstasks.VPCDHCPOptionsAssociation{
			Name:        s(b.ClusterName()),
			Lifecycle:   b.Lifecycle,
			VPC:         b.LinkToVPC(),
			DHCPOptions: dhcp,
		})
	} else {
		// TODO: would be good to create these as shared, to verify them
	}

	allSubnetsShared := true
	for i := range b.Cluster.Spec.Subnets {
		subnetSpec := &b.Cluster.Spec.Subnets[i]
		sharedSubnet := subnetSpec.ProviderID != ""
		if !sharedSubnet {
			allSubnetsShared = false
		}
	}

	// We always have a public route table, though for private networks it is only used for NGWs and ELBs
	var publicRouteTable *awstasks.RouteTable
	{
		// The internet gateway is the main entry point to the cluster.
		igw := &awstasks.InternetGateway{
			Name:      s(b.ClusterName()),
			Lifecycle: b.Lifecycle,
			VPC:       b.LinkToVPC(),
			Shared:    fi.Bool(sharedVPC),
		}
		c.AddTask(igw)

		if !allSubnetsShared {
			publicRouteTable = &awstasks.RouteTable{
				Name:      s(b.ClusterName()),
				Lifecycle: b.Lifecycle,

				VPC: b.LinkToVPC(),
			}
			c.AddTask(publicRouteTable)

			// TODO: Validate when allSubnetsShared
			c.AddTask(&awstasks.Route{
				Name:            s("0.0.0.0/0"),
				Lifecycle:       b.Lifecycle,
				CIDR:            s("0.0.0.0/0"),
				RouteTable:      publicRouteTable,
				InternetGateway: igw,
			})
		}
	}

	privateZones := sets.NewString()

	for i := range b.Cluster.Spec.Subnets {
		subnetSpec := &b.Cluster.Spec.Subnets[i]
		sharedSubnet := subnetSpec.ProviderID != ""
		subnetName := subnetSpec.Name + "." + b.ClusterName()
		tags := b.CloudTags(subnetName, sharedSubnet)

		// Apply tags so that Kubernetes knows which subnets should be used for internal/external ELBs
		switch subnetSpec.Type {
		case kops.SubnetTypePublic, kops.SubnetTypeUtility:
			tags[aws.TagNameSubnetPublicELB] = "1"

		case kops.SubnetTypePrivate:
			tags[aws.TagNameSubnetInternalELB] = "1"

		default:
			glog.V(2).Infof("unable to properly tag subnet %q because it has unknown type %q. Load balancers may be created in incorrect subnets", subnetSpec.Name, subnetSpec.Type)
		}

		subnet := &awstasks.Subnet{
			Name:             s(subnetName),
			Lifecycle:        b.Lifecycle,
			VPC:              b.LinkToVPC(),
			AvailabilityZone: s(subnetSpec.Zone),
			CIDR:             s(subnetSpec.CIDR),
			Shared:           fi.Bool(sharedSubnet),
			Tags:             tags,
		}

		if subnetSpec.ProviderID != "" {
			subnet.ID = s(subnetSpec.ProviderID)
		}
		c.AddTask(subnet)

		switch subnetSpec.Type {
		case kops.SubnetTypePublic, kops.SubnetTypeUtility:
			if !sharedSubnet {
				c.AddTask(&awstasks.RouteTableAssociation{
					Name:       s(subnetSpec.Name + "." + b.ClusterName()),
					Lifecycle:  b.Lifecycle,
					RouteTable: publicRouteTable,
					Subnet:     subnet,
				})
			}

		case kops.SubnetTypePrivate:
			// Private subnets get a Network Gateway, and their own route table to associate them with the network gateway

			if !sharedSubnet {
				// Private Subnet Route Table Associations
				//
				// Map the Private subnet to the Private route table
				c.AddTask(&awstasks.RouteTableAssociation{
					Name:       s("private-" + subnetSpec.Name + "." + b.ClusterName()),
					Lifecycle:  b.Lifecycle,
					RouteTable: b.LinkToPrivateRouteTableInZone(subnetSpec.Zone),
					Subnet:     subnet,
				})

				// TODO: validate even if shared?
				privateZones.Insert(subnetSpec.Zone)
			}
		default:
			return fmt.Errorf("subnet %q has unknown type %q", subnetSpec.Name, subnetSpec.Type)
		}
	}

	// Loop over zones
	for i, zone := range privateZones.List() {

		utilitySubnet, err := b.LinkToUtilitySubnetInZone(zone)
		if err != nil {
			return err
		}

		var ngw *awstasks.NatGateway
		if b.Cluster.Spec.Subnets[i].Egress != "" {
			if strings.Contains(b.Cluster.Spec.Subnets[i].Egress, "nat-") {

				ngw = &awstasks.NatGateway{
					Name:                 s(zone + "." + b.ClusterName()),
					Lifecycle:            b.Lifecycle,
					Subnet:               utilitySubnet,
					ID:                   s(b.Cluster.Spec.Subnets[i].Egress),
					AssociatedRouteTable: b.LinkToPrivateRouteTableInZone(zone),
					// If we're here, it means this NatGateway was specified, so we are Shared
					Shared: fi.Bool(true),
				}

				c.AddTask(ngw)

			} else {
				return fmt.Errorf("kops currently only supports re-use of NAT Gateways. We will support more eventually! Please see https://github.com/kubernetes/kops/issues/1530")
			}

		} else {

			// Every NGW needs a public (Elastic) IP address, every private
			// subnet needs a NGW, lets create it. We tie it to a subnet
			// so we can track it in AWS
			var eip = &awstasks.ElasticIP{}

			eip = &awstasks.ElasticIP{
				Name:                           s(zone + "." + b.ClusterName()),
				Lifecycle:                      b.Lifecycle,
				AssociatedNatGatewayRouteTable: b.LinkToPrivateRouteTableInZone(zone),
			}

			c.AddTask(eip)
			// NAT Gateway
			//
			// All private subnets will need a NGW, one per zone
			//
			// The instances in the private subnet can access the Internet by
			// using a network address translation (NAT) gateway that resides
			// in the public subnet.

			//var ngw = &awstasks.NatGateway{}
			ngw = &awstasks.NatGateway{
				Name:                 s(zone + "." + b.ClusterName()),
				Lifecycle:            b.Lifecycle,
				Subnet:               utilitySubnet,
				ElasticIP:            eip,
				AssociatedRouteTable: b.LinkToPrivateRouteTableInZone(zone),
			}
			c.AddTask(ngw)
		}

		// Private Route Table
		//
		// The private route table that will route to the NAT Gateway
		rt := &awstasks.RouteTable{
			Name:      s(b.NamePrivateRouteTableInZone(zone)),
			VPC:       b.LinkToVPC(),
			Lifecycle: b.Lifecycle,
		}
		c.AddTask(rt)

		// Private Routes
		//
		// Routes for the private route table.
		// Will route to the NAT Gateway
		c.AddTask(&awstasks.Route{
			Name:       s("private-" + zone + "-0.0.0.0/0"),
			Lifecycle:  b.Lifecycle,
			CIDR:       s("0.0.0.0/0"),
			RouteTable: rt,
			NatGateway: ngw,
		})

	}

	return nil
}
<|endoftext|>


<commit_before>package paxos

import (
	"log"
	"os"

	"junta/store"
	"junta/util"
	"time"
)

const (
	fillDelay = 5e8 // 500ms
)

type result struct {
	seqn uint64
	v    string
}

type instReq struct {
	seqn uint64
	ch   chan *instance
}

type Manager struct {
	st      *store.Store
	rg      *Registrar
	learned chan result
	seqns   chan uint64
	fillUntil chan uint64
	reqs    chan instReq
	logger  *log.Logger
	Self    string
	alpha   int
	outs    PutterTo
}

// start is the seqn at which this member was defined.
// start+alpha is the first seqn this manager is expected to participate in.
func NewManager(self string, start uint64, alpha int, st *store.Store, outs PutterTo) *Manager {
	m := &Manager{
		st:      st,
		rg:      NewRegistrar(st, start, alpha),
		learned: make(chan result),
		seqns:   make(chan uint64),
		fillUntil:make(chan uint64),
		reqs:    make(chan instReq),
		logger:  util.NewLogger("manager"),
		Self:    self,
		alpha:   alpha,
		outs:    outs,
	}

	go m.gen(start+uint64(alpha))
	go m.fill(start+uint64(alpha))
	go m.process()

	return m
}

func (m *Manager) Alpha() int {
	return m.alpha
}

func (m *Manager) cluster(seqn uint64) *cluster {
	members, cals := m.rg.setsForSeqn(seqn)
	return newCluster(m.Self, members, cals, putToWrapper{seqn, m.outs})
}

func (mg *Manager) gen(next uint64) {
	for {
		cx := mg.cluster(next)
		leader := int(next % uint64(cx.Len()))
		if leader == cx.SelfIndex() {
			mg.seqns <- next
		}
		next++
	}
}

func (mg *Manager) fill(seqn uint64) {
	for next := range mg.fillUntil {
		for seqn < next {
			go mg.fillOne(seqn)
			seqn++
		}
		seqn++ // no need to fill in our own seqn
	}
}

func (m *Manager) process() {
	instances := make(map[uint64]*instance)
	for req := range m.reqs {
		inst, ok := instances[req.seqn]
		if !ok {
			inst = newInstance(req.seqn, m, m.learned)
			instances[req.seqn] = inst
		}
		req.ch <- inst
	}
}

func (m *Manager) getInstance(seqn uint64) *instance {
	ch := make(chan *instance)
	m.reqs <- instReq{seqn, ch}
	return <-ch
}

func (m *Manager) PutFrom(addr string, msg Msg) {
	if !msg.Ok() {
		return
	}
	it := m.getInstance(msg.Seqn())
	it.PutFrom(addr, msg)
}

func (m *Manager) proposeAt(seqn uint64, v string) {
	m.getInstance(seqn).Propose(v)
	m.logger.Logf("paxos propose -> %d %q", seqn, v)
}

func (m *Manager) Propose(v string) (uint64, string, os.Error) {
	seqn := <-m.seqns
	ch := m.st.Wait(seqn)
	m.proposeAt(seqn, v)
	m.fillUntil <- seqn
	ev := <-ch
	return seqn, ev.Mut, ev.Err
}

func (m *Manager) fillOne(seqn uint64) {
	time.Sleep(fillDelay)
	// yes, we'll act as coordinator for a seqn we don't "own"
	// this is intentional, since we want to exersize this code all the time,
	// not just during a failure.
	m.proposeAt(seqn, store.Nop)
}

func (m *Manager) Recv() (uint64, string) {
	result := <-m.learned
	m.logger.Logf("paxos %d learned <- %q", result.seqn, result.v)
	return result.seqn, result.v
}
<commit_msg>the meaning is more clear this way<commit_after>package paxos

import (
	"log"
	"os"

	"junta/store"
	"junta/util"
	"time"
)

const (
	fillDelay = 5e8 // 500ms
)

type result struct {
	seqn uint64
	v    string
}

type instReq struct {
	seqn uint64
	ch   chan *instance
}

type Manager struct {
	st      *store.Store
	rg      *Registrar
	learned chan result
	seqns   chan uint64
	fillUntil chan uint64
	reqs    chan instReq
	logger  *log.Logger
	Self    string
	alpha   int
	outs    PutterTo
}

// start is the seqn at which this member was defined.
// start+alpha is the first seqn this manager is expected to participate in.
func NewManager(self string, start uint64, alpha int, st *store.Store, outs PutterTo) *Manager {
	m := &Manager{
		st:      st,
		rg:      NewRegistrar(st, start, alpha),
		learned: make(chan result),
		seqns:   make(chan uint64),
		fillUntil:make(chan uint64),
		reqs:    make(chan instReq),
		logger:  util.NewLogger("manager"),
		Self:    self,
		alpha:   alpha,
		outs:    outs,
	}

	go m.gen(start+uint64(alpha))
	go m.fill(start+uint64(alpha))
	go m.process()

	return m
}

func (m *Manager) Alpha() int {
	return m.alpha
}

func (m *Manager) cluster(seqn uint64) *cluster {
	members, cals := m.rg.setsForSeqn(seqn)
	return newCluster(m.Self, members, cals, putToWrapper{seqn, m.outs})
}

func (mg *Manager) gen(next uint64) {
	for {
		cx := mg.cluster(next)
		leader := int(next % uint64(cx.Len()))
		if leader == cx.SelfIndex() {
			mg.seqns <- next
		}
		next++
	}
}

func (mg *Manager) fill(seqn uint64) {
	for next := range mg.fillUntil {
		for seqn < next {
			go mg.fillOne(seqn)
			seqn++
		}
		seqn = next + 1 // no need to fill in our own seqn
	}
}

func (m *Manager) process() {
	instances := make(map[uint64]*instance)
	for req := range m.reqs {
		inst, ok := instances[req.seqn]
		if !ok {
			inst = newInstance(req.seqn, m, m.learned)
			instances[req.seqn] = inst
		}
		req.ch <- inst
	}
}

func (m *Manager) getInstance(seqn uint64) *instance {
	ch := make(chan *instance)
	m.reqs <- instReq{seqn, ch}
	return <-ch
}

func (m *Manager) PutFrom(addr string, msg Msg) {
	if !msg.Ok() {
		return
	}
	it := m.getInstance(msg.Seqn())
	it.PutFrom(addr, msg)
}

func (m *Manager) proposeAt(seqn uint64, v string) {
	m.getInstance(seqn).Propose(v)
	m.logger.Logf("paxos propose -> %d %q", seqn, v)
}

func (m *Manager) Propose(v string) (uint64, string, os.Error) {
	seqn := <-m.seqns
	ch := m.st.Wait(seqn)
	m.proposeAt(seqn, v)
	m.fillUntil <- seqn
	ev := <-ch
	return seqn, ev.Mut, ev.Err
}

func (m *Manager) fillOne(seqn uint64) {
	time.Sleep(fillDelay)
	// yes, we'll act as coordinator for a seqn we don't "own"
	// this is intentional, since we want to exersize this code all the time,
	// not just during a failure.
	m.proposeAt(seqn, store.Nop)
}

func (m *Manager) Recv() (uint64, string) {
	result := <-m.learned
	m.logger.Logf("paxos %d learned <- %q", result.seqn, result.v)
	return result.seqn, result.v
}
<|endoftext|>


<commit_before>// package runner provides a simple interface for running docker containers.
package runner

import (
	"bytes"
	"fmt"
	"io"

	"code.google.com/p/go-uuid/uuid"

	"github.com/fsouza/go-dockerclient"
	shellwords "github.com/mattn/go-shellwords"
	"github.com/remind101/empire/pkg/dockerutil"
	"github.com/remind101/empire/pkg/image"
	"golang.org/x/net/context"
)

// DefaultStopTimeout is the number of seconds to wait when stopping a
// container.
const DefaultStopTimeout = 10

// RunOpts is used when running.
type RunOpts struct {
	// Image is the image to run.
	Image image.Image

	// Command is the command to run.
	Command string

	// Environment variables to set.
	Env map[string]string

	// Streams fo Stdout, Stderr and Stdin.
	Input  io.Reader
	Output io.Writer
}

// Runner is a service for running containers.
type Runner struct {
	client *dockerutil.Client
}

// NewRunner returns a new Runner instance using the docker.Client as the docker
// client.
func NewRunner(client *dockerutil.Client) *Runner {
	return &Runner{client: client}
}

func (r *Runner) Run(ctx context.Context, opts RunOpts) error {
	if err := r.pull(ctx, opts.Image, replaceNL(opts.Output)); err != nil {
		return fmt.Errorf("runner: pull: %v", err)
	}

	c, err := r.create(ctx, opts)
	if err != nil {
		return fmt.Errorf("runner: create container: %v", err)
	}
	defer r.remove(ctx, c.ID)

	if err := r.start(ctx, c.ID); err != nil {
		return fmt.Errorf("runner: start container: %v", err)
	}
	defer tryClose(opts.Output)

	if err := r.attach(ctx, c.ID, opts.Input, opts.Output); err != nil {
		return fmt.Errorf("runner: attach: %v", err)
	}

	return nil
}

func (r *Runner) pull(ctx context.Context, img image.Image, out io.Writer) error {
	return r.client.PullImage(ctx, docker.PullImageOptions{
		Registry:     img.Registry,
		Repository:   img.Repository,
		Tag:          img.Tag,
		OutputStream: out,
	})
}

func (r *Runner) create(ctx context.Context, opts RunOpts) (*docker.Container, error) {
	cmd, err := shellwords.Parse(opts.Command)
	if err != nil {
		return nil, err
	}

	return r.client.CreateContainer(ctx, docker.CreateContainerOptions{
		Name: uuid.New(),
		Config: &docker.Config{
			Tty:          true,
			AttachStdin:  true,
			AttachStdout: true,
			AttachStderr: true,
			OpenStdin:    true,
			Image:        opts.Image.String(),
			Cmd:          cmd,
			Env:          envKeys(opts.Env),
		},
		HostConfig: &docker.HostConfig{},
	})
}

func (r *Runner) start(ctx context.Context, id string) error {
	return r.client.StartContainer(ctx, id, nil)
}

func (r *Runner) attach(ctx context.Context, id string, in io.Reader, out io.Writer) error {
	return r.client.AttachToContainer(ctx, docker.AttachToContainerOptions{
		Container:    id,
		InputStream:  in,
		OutputStream: out,
		ErrorStream:  out,
		Logs:         true,
		Stream:       true,
		Stdin:        true,
		Stdout:       true,
		Stderr:       true,
		RawTerminal:  true,
	})
}

func (r *Runner) remove(ctx context.Context, id string) error {
	return r.client.RemoveContainer(ctx, docker.RemoveContainerOptions{
		ID:            id,
		RemoveVolumes: true,
		Force:         true,
	})
}

func envKeys(env map[string]string) []string {
	var s []string

	for k, v := range env {
		s = append(s, fmt.Sprintf("%s=%s", k, v))
	}

	return s
}

func tryClose(w io.Writer) error {
	if w, ok := w.(io.Closer); ok {
		return w.Close()
	}

	return nil
}

// replaceNL returns an io.Writer that will replace "\n" with "\r\n" in the
// stream.
var replaceNL = func(w io.Writer) io.Writer {
	o, n := []byte("\n"), []byte("\r\n")
	return writerFunc(func(p []byte) (int, error) {
		return w.Write(bytes.Replace(p, o, n, -1))
	})
}

// writerFunc is a function that implements io.Writer.
type writerFunc func([]byte) (int, error)

func (f writerFunc) Write(p []byte) (int, error) {
	return f(p)
}
<commit_msg>Force json-file logging driver for attached runs.<commit_after>// package runner provides a simple interface for running docker containers.
package runner

import (
	"bytes"
	"fmt"
	"io"

	"code.google.com/p/go-uuid/uuid"

	"github.com/fsouza/go-dockerclient"
	shellwords "github.com/mattn/go-shellwords"
	"github.com/remind101/empire/pkg/dockerutil"
	"github.com/remind101/empire/pkg/image"
	"golang.org/x/net/context"
)

// DefaultStopTimeout is the number of seconds to wait when stopping a
// container.
const DefaultStopTimeout = 10

// RunOpts is used when running.
type RunOpts struct {
	// Image is the image to run.
	Image image.Image

	// Command is the command to run.
	Command string

	// Environment variables to set.
	Env map[string]string

	// Streams fo Stdout, Stderr and Stdin.
	Input  io.Reader
	Output io.Writer
}

// Runner is a service for running containers.
type Runner struct {
	client *dockerutil.Client
}

// NewRunner returns a new Runner instance using the docker.Client as the docker
// client.
func NewRunner(client *dockerutil.Client) *Runner {
	return &Runner{client: client}
}

func (r *Runner) Run(ctx context.Context, opts RunOpts) error {
	if err := r.pull(ctx, opts.Image, replaceNL(opts.Output)); err != nil {
		return fmt.Errorf("runner: pull: %v", err)
	}

	c, err := r.create(ctx, opts)
	if err != nil {
		return fmt.Errorf("runner: create container: %v", err)
	}
	defer r.remove(ctx, c.ID)

	if err := r.start(ctx, c.ID); err != nil {
		return fmt.Errorf("runner: start container: %v", err)
	}
	defer tryClose(opts.Output)

	if err := r.attach(ctx, c.ID, opts.Input, opts.Output); err != nil {
		return fmt.Errorf("runner: attach: %v", err)
	}

	return nil
}

func (r *Runner) pull(ctx context.Context, img image.Image, out io.Writer) error {
	return r.client.PullImage(ctx, docker.PullImageOptions{
		Registry:     img.Registry,
		Repository:   img.Repository,
		Tag:          img.Tag,
		OutputStream: out,
	})
}

func (r *Runner) create(ctx context.Context, opts RunOpts) (*docker.Container, error) {
	cmd, err := shellwords.Parse(opts.Command)
	if err != nil {
		return nil, err
	}

	return r.client.CreateContainer(ctx, docker.CreateContainerOptions{
		Name: uuid.New(),
		Config: &docker.Config{
			Tty:          true,
			AttachStdin:  true,
			AttachStdout: true,
			AttachStderr: true,
			OpenStdin:    true,
			Image:        opts.Image.String(),
			Cmd:          cmd,
			Env:          envKeys(opts.Env),
		},
		HostConfig: &docker.HostConfig{
			LogConfig: docker.LogConfig{
				Type: "json-file",
			},
		},
	})
}

func (r *Runner) start(ctx context.Context, id string) error {
	return r.client.StartContainer(ctx, id, nil)
}

func (r *Runner) attach(ctx context.Context, id string, in io.Reader, out io.Writer) error {
	return r.client.AttachToContainer(ctx, docker.AttachToContainerOptions{
		Container:    id,
		InputStream:  in,
		OutputStream: out,
		ErrorStream:  out,
		Logs:         true,
		Stream:       true,
		Stdin:        true,
		Stdout:       true,
		Stderr:       true,
		RawTerminal:  true,
	})
}

func (r *Runner) remove(ctx context.Context, id string) error {
	return r.client.RemoveContainer(ctx, docker.RemoveContainerOptions{
		ID:            id,
		RemoveVolumes: true,
		Force:         true,
	})
}

func envKeys(env map[string]string) []string {
	var s []string

	for k, v := range env {
		s = append(s, fmt.Sprintf("%s=%s", k, v))
	}

	return s
}

func tryClose(w io.Writer) error {
	if w, ok := w.(io.Closer); ok {
		return w.Close()
	}

	return nil
}

// replaceNL returns an io.Writer that will replace "\n" with "\r\n" in the
// stream.
var replaceNL = func(w io.Writer) io.Writer {
	o, n := []byte("\n"), []byte("\r\n")
	return writerFunc(func(p []byte) (int, error) {
		return w.Write(bytes.Replace(p, o, n, -1))
	})
}

// writerFunc is a function that implements io.Writer.
type writerFunc func([]byte) (int, error)

func (f writerFunc) Write(p []byte) (int, error) {
	return f(p)
}
<|endoftext|>


<commit_before>// Copyright 2021 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package migrations

import (
	"context"
	"fmt"
	"net/http"
	"net/url"
	"testing"
	"time"

	"code.gitea.io/gitea/modules/migrations/base"

	"github.com/stretchr/testify/assert"
)

func TestOneDevDownloadRepo(t *testing.T) {
	resp, err := http.Get("https://code.onedev.io/projects/go-gitea-test_repo")
	if err != nil || resp.StatusCode != 200 {
		t.Skipf("Can't access test repo, skipping %s", t.Name())
	}

	u, _ := url.Parse("https://code.onedev.io")
	downloader := NewOneDevDownloader(context.Background(), u, "", "", "go-gitea-test_repo")
	if err != nil {
		t.Fatal(fmt.Sprintf("NewOneDevDownloader is nil: %v", err))
	}
	repo, err := downloader.GetRepoInfo()
	assert.NoError(t, err)
	assert.EqualValues(t, &base.Repository{
		Name:        "go-gitea-test_repo",
		Owner:       "",
		Description: "Test repository for testing migration from OneDev to gitea",
		CloneURL:    "https://code.onedev.io/go-gitea-test_repo",
		OriginalURL: "https://code.onedev.io/projects/go-gitea-test_repo",
	}, repo)

	milestones, err := downloader.GetMilestones()
	assert.NoError(t, err)
	assert.Len(t, milestones, 2)
	deadline := time.Unix(1620086400, 0)
	assert.EqualValues(t, []*base.Milestone{
		{
			Title:    "1.0.0",
			Deadline: &deadline,
			Closed:   &deadline,
		},
		{
			Title:       "1.1.0",
			Description: "next things?",
		},
	}, milestones)

	labels, err := downloader.GetLabels()
	assert.NoError(t, err)
	assert.Len(t, labels, 6)

	issues, isEnd, err := downloader.GetIssues(1, 2)
	assert.NoError(t, err)
	assert.Len(t, issues, 2)
	assert.False(t, isEnd)
	assert.EqualValues(t, []*base.Issue{
		{
			Number:     4,
			Title:      "Hi there",
			Content:    "an issue not assigned to a milestone",
			PosterName: "User 336",
			State:      "open",
			Created:    time.Unix(1628549776, 734000000),
			Updated:    time.Unix(1628549776, 734000000),
			Labels: []*base.Label{
				{
					Name: "Improvement",
				},
			},
			Context: onedevIssueContext{
				foreignID:     398,
				localID:       4,
				IsPullRequest: false,
			},
		},
		{
			Number:     3,
			Title:      "Add an awesome feature",
			Content:    "just another issue to test against",
			PosterName: "User 336",
			State:      "open",
			Milestone:  "1.1.0",
			Created:    time.Unix(1628549749, 878000000),
			Updated:    time.Unix(1628549749, 878000000),
			Labels: []*base.Label{
				{
					Name: "New Feature",
				},
			},
			Context: onedevIssueContext{
				foreignID:     397,
				localID:       3,
				IsPullRequest: false,
			},
		},
	}, issues)

	comments, _, err := downloader.GetComments(base.GetCommentOptions{
		Context: onedevIssueContext{
			foreignID:     398,
			localID:       4,
			IsPullRequest: false,
		},
	})
	assert.NoError(t, err)
	assert.Len(t, comments, 1)
	assert.EqualValues(t, []*base.Comment{
		{
			IssueIndex: 4,
			PosterName: "User 336",
			Created:    time.Unix(1628549791, 128000000),
			Updated:    time.Unix(1628549791, 128000000),
			Content:    "it has a comment\r\n\r\nEDIT: that got edited",
		},
	}, comments)

	prs, _, err := downloader.GetPullRequests(1, 1)
	assert.NoError(t, err)
	assert.Len(t, prs, 1)
	assert.EqualValues(t, []*base.PullRequest{
		{
			Number:     5,
			Title:      "Pull to add a new file",
			Content:    "just do some git stuff",
			PosterName: "User 336",
			State:      "open",
			Created:    time.Unix(1628550076, 25000000),
			Updated:    time.Unix(1628550076, 25000000),
			Head: base.PullRequestBranch{
				Ref:      "branch-for-a-pull",
				SHA:      "343deffe3526b9bc84e873743ff7f6e6d8b827c0",
				RepoName: "go-gitea-test_repo",
			},
			Base: base.PullRequestBranch{
				Ref:      "master",
				SHA:      "f32b0a9dfd09a60f616f29158f772cedd89942d2",
				RepoName: "go-gitea-test_repo",
			},
			Context: onedevIssueContext{
				foreignID:     186,
				localID:       5,
				IsPullRequest: true,
			},
		},
	}, prs)

	rvs, err := downloader.GetReviews(onedevIssueContext{
		foreignID: 186,
		localID:   5,
	})
	assert.NoError(t, err)
	assert.Len(t, rvs, 1)
	assert.EqualValues(t, []*base.Review{
		{
			IssueIndex:   5,
			ReviewerName: "User 317",
			State:        "PENDING",
		},
	}, rvs)
}
<commit_msg>Make TestOneDevDownloadRepo work again & more resistant (#16987)<commit_after>// Copyright 2021 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package migrations

import (
	"context"
	"fmt"
	"net/http"
	"net/url"
	"testing"
	"time"

	"code.gitea.io/gitea/modules/migrations/base"

	"github.com/stretchr/testify/assert"
)

func TestOneDevDownloadRepo(t *testing.T) {
	resp, err := http.Get("https://code.onedev.io/projects/go-gitea-test_repo")
	if err != nil || resp.StatusCode != 200 {
		t.Skipf("Can't access test repo, skipping %s", t.Name())
	}

	u, _ := url.Parse("https://code.onedev.io")
	downloader := NewOneDevDownloader(context.Background(), u, "", "", "go-gitea-test_repo")
	if err != nil {
		t.Fatal(fmt.Sprintf("NewOneDevDownloader is nil: %v", err))
	}
	repo, err := downloader.GetRepoInfo()
	assert.NoError(t, err)
	assertRepositoryEqual(t, &base.Repository{
		Name:        "go-gitea-test_repo",
		Owner:       "",
		Description: "Test repository for testing migration from OneDev to gitea",
		CloneURL:    "https://code.onedev.io/go-gitea-test_repo",
		OriginalURL: "https://code.onedev.io/projects/go-gitea-test_repo",
	}, repo)

	milestones, err := downloader.GetMilestones()
	assert.NoError(t, err)
	deadline := time.Unix(1620086400, 0)
	assertMilestonesEqual(t, []*base.Milestone{
		{
			Title:    "1.0.0",
			Deadline: &deadline,
			Closed:   &deadline,
		},
		{
			Title:       "1.1.0",
			Description: "next things?",
		},
	}, milestones)

	labels, err := downloader.GetLabels()
	assert.NoError(t, err)
	assert.Len(t, labels, 6)

	issues, isEnd, err := downloader.GetIssues(1, 2)
	assert.NoError(t, err)
	assert.False(t, isEnd)
	assertIssuesEqual(t, []*base.Issue{
		{
			Number:     4,
			Title:      "Hi there",
			Content:    "an issue not assigned to a milestone",
			PosterName: "User 336",
			State:      "open",
			Created:    time.Unix(1628549776, 734000000),
			Updated:    time.Unix(1628549776, 734000000),
			Labels: []*base.Label{
				{
					Name: "Improvement",
				},
			},
			Context: onedevIssueContext{
				foreignID:     398,
				localID:       4,
				IsPullRequest: false,
			},
		},
		{
			Number:     3,
			Title:      "Add an awesome feature",
			Content:    "just another issue to test against",
			PosterName: "User 336",
			State:      "open",
			Milestone:  "1.1.0",
			Created:    time.Unix(1628549749, 878000000),
			Updated:    time.Unix(1628549749, 878000000),
			Labels: []*base.Label{
				{
					Name: "New Feature",
				},
			},
			Context: onedevIssueContext{
				foreignID:     397,
				localID:       3,
				IsPullRequest: false,
			},
		},
	}, issues)

	comments, _, err := downloader.GetComments(base.GetCommentOptions{
		Context: onedevIssueContext{
			foreignID:     398,
			localID:       4,
			IsPullRequest: false,
		},
	})
	assert.NoError(t, err)
	assertCommentsEqual(t, []*base.Comment{
		{
			IssueIndex: 4,
			PosterName: "User 336",
			Created:    time.Unix(1628549791, 128000000),
			Updated:    time.Unix(1628549791, 128000000),
			Content:    "it has a comment\n\nEDIT: that got edited",
		},
	}, comments)

	prs, _, err := downloader.GetPullRequests(1, 1)
	assert.NoError(t, err)
	assertPullRequestsEqual(t, []*base.PullRequest{
		{
			Number:     5,
			Title:      "Pull to add a new file",
			Content:    "just do some git stuff",
			PosterName: "User 336",
			State:      "open",
			Created:    time.Unix(1628550076, 25000000),
			Updated:    time.Unix(1628550076, 25000000),
			Head: base.PullRequestBranch{
				Ref:      "branch-for-a-pull",
				SHA:      "343deffe3526b9bc84e873743ff7f6e6d8b827c0",
				RepoName: "go-gitea-test_repo",
			},
			Base: base.PullRequestBranch{
				Ref:      "master",
				SHA:      "f32b0a9dfd09a60f616f29158f772cedd89942d2",
				RepoName: "go-gitea-test_repo",
			},
			Context: onedevIssueContext{
				foreignID:     186,
				localID:       5,
				IsPullRequest: true,
			},
		},
	}, prs)

	rvs, err := downloader.GetReviews(onedevIssueContext{
		foreignID: 186,
		localID:   5,
	})
	assert.NoError(t, err)
	assertReviewsEqual(t, []*base.Review{
		{
			IssueIndex:   5,
			ReviewerName: "User 317",
			State:        "PENDING",
		},
	}, rvs)
}
<|endoftext|>


<commit_before>// Copyright 2017 Kumina, https://kumina.nl/
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"path"
	"regexp"
	"strconv"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	client_model "github.com/prometheus/client_model/go"
	"github.com/prometheus/common/expfmt"
	"github.com/prometheus/common/version"
	"github.com/tomasen/fcgi_client"
	"gopkg.in/alecthomas/kingpin.v2"
	"path/filepath"
)

var (
	phpfpmSocketPathLabel = "socket_path"
	phpfpmScriptPathLabel = "script_path"

	phpfpmUpDesc = prometheus.NewDesc(
		prometheus.BuildFQName("php", "fpm", "up"),
		"Whether scraping PHP-FPM's metrics was successful.",
		[]string{phpfpmSocketPathLabel}, nil)

	phpfpmAcceptedConnections = prometheus.NewDesc(
		prometheus.BuildFQName("php", "fpm", "accepted_connections_total"),
		"Number of request accepted by the pool.",
		[]string{phpfpmSocketPathLabel}, nil)

	phpfpmStartTime = prometheus.NewDesc(
		prometheus.BuildFQName("php", "fpm", "start_time_seconds"),
		"Unix time when FPM has started or reloaded.",
		[]string{phpfpmSocketPathLabel}, nil)

	phpfpmGauges = map[string]*prometheus.Desc{
		"listen queue": prometheus.NewDesc(
			prometheus.BuildFQName("php", "fpm", "listen_queue"),
			"Number of request in the queue of pending connections.",
			[]string{phpfpmSocketPathLabel}, nil),
		"max listen queue": prometheus.NewDesc(
			prometheus.BuildFQName("php", "fpm", "max_listen_queue"),
			"Maximum number of requests in the queue of pending connections since FPM has started.",
			[]string{phpfpmSocketPathLabel}, nil),
		"listen queue len": prometheus.NewDesc(
			prometheus.BuildFQName("php", "fpm", "listen_queue_length"),
			"The size of the socket queue of pending connections.",
			[]string{phpfpmSocketPathLabel}, nil),
		"idle processes": prometheus.NewDesc(
			prometheus.BuildFQName("php", "fpm", "idle_processes"),
			"Number of idle processes.",
			[]string{phpfpmSocketPathLabel}, nil),
		"active processes": prometheus.NewDesc(
			prometheus.BuildFQName("php", "fpm", "active_processes"),
			"Number of active processes.",
			[]string{phpfpmSocketPathLabel}, nil),
		"max active processes": prometheus.NewDesc(
			prometheus.BuildFQName("php", "fpm", "max_active_processes"),
			"Maximum number of active processes since FPM has started.",
			[]string{phpfpmSocketPathLabel}, nil),
		"max children reached": prometheus.NewDesc(
			prometheus.BuildFQName("php", "fpm", "max_children_reached"),
			"Number of times, the process limit has been reached.",
			[]string{phpfpmSocketPathLabel}, nil),
		"slow requests": prometheus.NewDesc(
			prometheus.BuildFQName("php", "fpm", "slow_requests"),
			"Enable php-fpm slow-log before you consider this. If this value is non-zero you may have slow php processes.",
			[]string{phpfpmSocketPathLabel}, nil),
	}
)

func CollectStatusFromReader(reader io.Reader, socketPath string, ch chan<- prometheus.Metric) error {
	scanner := bufio.NewScanner(reader)
	re := regexp.MustCompile("^(.*): +(.*)$")

	// Scrape the interesting values:
	for scanner.Scan() {
		fields := re.FindStringSubmatch(scanner.Text())
		if fields == nil {
			return fmt.Errorf("Failed to parse %s", scanner.Text())
		}

		if gauge, ok := phpfpmGauges[fields[1]]; ok {
			f, err := strconv.ParseFloat(fields[2], 64)
			if err != nil {
				return err
			}
			ch <- prometheus.MustNewConstMetric(
				gauge,
				prometheus.GaugeValue,
				f,
				socketPath)
		} else if fields[1] == "accepted conn" {
			f, err := strconv.ParseFloat(fields[2], 64)
			if err != nil {
				return err
			}
			ch <- prometheus.MustNewConstMetric(
				phpfpmAcceptedConnections,
				prometheus.CounterValue,
				f,
				socketPath)
		} else if fields[1] == "start time" {
			location, err := time.LoadLocation("Local")
			if err != nil {
				return err
			}
			since, err := time.ParseInLocation("02/Jan/2006:15:04:05 -0700", fields[2], location)
			if err != nil {
				return err
			}
			f := float64(since.Unix())
			ch <- prometheus.MustNewConstMetric(
				phpfpmStartTime,
				prometheus.GaugeValue,
				f,
				socketPath)
		}
	}
	return nil
}

func CollectStatusFromSocket(path string, statusPath string, ch chan<- prometheus.Metric) error {

	env := make(map[string]string)
	env["SCRIPT_FILENAME"] = statusPath
	env["SCRIPT_NAME"] = statusPath
	env["REQUEST_METHOD"] = "GET"

	fcgi, err := fcgiclient.Dial("unix", path)
	if err != nil {
		return err
	}
	defer fcgi.Close()

	resp, err := fcgi.Get(env)
	if err != nil {
		return err
	}

	return CollectStatusFromReader(resp.Body, path, ch)
}

func CollectMetricsFromScript(socketPaths []string, scriptPaths []string) ([]*client_model.MetricFamily, error) {
	var result []*client_model.MetricFamily

	for _, socketPath := range socketPaths {

		for _, scriptPath := range scriptPaths {
			fcgi, err := fcgiclient.Dial("unix", socketPath)
			if err != nil {
				return result, err
			}
			defer fcgi.Close()

			env := make(map[string]string)
			env["DOCUMENT_ROOT"] = path.Dir(scriptPath)
			env["SCRIPT_FILENAME"] = scriptPath
			env["SCRIPT_NAME"] = path.Base(scriptPath)
			env["REQUEST_METHOD"] = "GET"

			resp, err := fcgi.Get(env)
			if err != nil {
				return result, err
			}

			var parser expfmt.TextParser
			metricFamilies, err := parser.TextToMetricFamilies(resp.Body)
			if err != nil {
				return result, err
			}

			for _, metricFamily := range metricFamilies {
				for _, metric := range metricFamily.Metric {
					socketPathCopy := socketPath
					scriptPathCopy := scriptPath
					metric.Label = append(
						metric.Label,
						&client_model.LabelPair{
							Name:  &phpfpmSocketPathLabel,
							Value: &socketPathCopy,
						},
						&client_model.LabelPair{
							Name:  &phpfpmScriptPathLabel,
							Value: &scriptPathCopy,
						})
				}
				result = append(result, metricFamily)
			}
		}
	}
	return result, nil
}

type PhpfpmExporter struct {
	socketPaths []string
	statusPath  string
}

func NewPhpfpmExporter(socketPaths []string, statusPath string) (*PhpfpmExporter, error) {
	return &PhpfpmExporter{
		socketPaths: socketPaths,
		statusPath:  statusPath,
	}, nil
}

func (e *PhpfpmExporter) Describe(ch chan<- *prometheus.Desc) {
	ch <- phpfpmUpDesc
	ch <- phpfpmAcceptedConnections
	ch <- phpfpmStartTime
	for _, desc := range phpfpmGauges {
		ch <- desc
	}
}

func (e *PhpfpmExporter) Collect(ch chan<- prometheus.Metric) {

	for _, socketPath := range e.socketPaths {
		err := CollectStatusFromSocket(socketPath, e.statusPath, ch)
		if err == nil {
			ch <- prometheus.MustNewConstMetric(
				phpfpmUpDesc,
				prometheus.GaugeValue,
				1.0,
				socketPath)
		} else {
			log.Printf("Failed to scrape socket: %s", err)
			ch <- prometheus.MustNewConstMetric(
				phpfpmUpDesc,
				prometheus.GaugeValue,
				0.0,
				socketPath)
		}
	}
}

func main() {
	var (
		listenAddress        = kingpin.Flag("web.listen-address", "Address to listen on for web interface and telemetry.").Default(":9253").String()
		metricsPath          = kingpin.Flag("web.telemetry-path", "Path under which to expose metrics.").Default("/metrics").String()
		socketPaths          = kingpin.Flag("phpfpm.socket-paths", "Paths of the PHP-FPM sockets.").Strings()
		socketDirectories    = kingpin.Flag("phpfpm.socket-directories", "Path(s) of the directory where PHP-FPM sockets are located.").Strings()
		statusPath           = kingpin.Flag("phpfpm.status-path", "Path which has been configured in PHP-FPM to show status page.").Default("/status").String()
		scriptCollectorPaths = kingpin.Flag("phpfpm.script-collector-paths", "Paths of the PHP file whose output needs to be collected.").Strings()
		showVersion          = kingpin.Flag("version", "Print version information.").Bool()
	)

	kingpin.CommandLine.HelpFlag.Short('h')
	kingpin.Parse()

	var sockets []string
	for _, socketDirectory := range *socketDirectories {
		filepath.Walk(socketDirectory, func(path string, info os.FileInfo, err error) error {
			if err == nil && info.Mode()&os.ModeSocket != 0 {
				sockets = append(sockets, path)
			}
			return nil
		})
	}

	for _, socket := range *socketPaths {
		sockets = append(sockets, socket)
	}

	if *showVersion {
		fmt.Println(version.Print("phpfpm_exporter"))
		os.Exit(0)
	}

	exporter, err := NewPhpfpmExporter(sockets, *statusPath)
	if err != nil {
		panic(err)
	}
	prometheus.MustRegister(exporter)

	if len(*scriptCollectorPaths) != 0 {
		prometheus.DefaultGatherer = prometheus.Gatherers{
			prometheus.DefaultGatherer,
			prometheus.GathererFunc(func() ([]*client_model.MetricFamily, error) {
				return CollectMetricsFromScript(sockets, *scriptCollectorPaths)
			}),
		}
	}

	log.Println("Starting phpfpm_exporter", version.Info())
	log.Println("Build context", version.BuildContext())
	log.Printf("Starting Server: %s", *listenAddress)

	http.Handle(*metricsPath, prometheus.Handler())
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(`

			<title>PHP-FPM Exporter</title>

PHP-FPM Exporter

<a href='` + *metricsPath + `'>Metrics

			`))
	})
	log.Fatal(http.ListenAndServe(*listenAddress, nil))
}
<commit_msg>Export the total processes count<commit_after>// Copyright 2017 Kumina, https://kumina.nl/
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"path"
	"regexp"
	"strconv"
	"time"

	"path/filepath"

	"github.com/prometheus/client_golang/prometheus"
	client_model "github.com/prometheus/client_model/go"
	"github.com/prometheus/common/expfmt"
	"github.com/prometheus/common/version"
	"github.com/tomasen/fcgi_client"
	"gopkg.in/alecthomas/kingpin.v2"
)

var (
	phpfpmSocketPathLabel = "socket_path"
	phpfpmScriptPathLabel = "script_path"

	phpfpmUpDesc = prometheus.NewDesc(
		prometheus.BuildFQName("php", "fpm", "up"),
		"Whether scraping PHP-FPM's metrics was successful.",
		[]string{phpfpmSocketPathLabel}, nil)

	phpfpmAcceptedConnections = prometheus.NewDesc(
		prometheus.BuildFQName("php", "fpm", "accepted_connections_total"),
		"Number of request accepted by the pool.",
		[]string{phpfpmSocketPathLabel}, nil)

	phpfpmStartTime = prometheus.NewDesc(
		prometheus.BuildFQName("php", "fpm", "start_time_seconds"),
		"Unix time when FPM has started or reloaded.",
		[]string{phpfpmSocketPathLabel}, nil)

	phpfpmGauges = map[string]*prometheus.Desc{
		"listen queue": prometheus.NewDesc(
			prometheus.BuildFQName("php", "fpm", "listen_queue"),
			"Number of request in the queue of pending connections.",
			[]string{phpfpmSocketPathLabel}, nil),
		"max listen queue": prometheus.NewDesc(
			prometheus.BuildFQName("php", "fpm", "max_listen_queue"),
			"Maximum number of requests in the queue of pending connections since FPM has started.",
			[]string{phpfpmSocketPathLabel}, nil),
		"listen queue len": prometheus.NewDesc(
			prometheus.BuildFQName("php", "fpm", "listen_queue_length"),
			"The size of the socket queue of pending connections.",
			[]string{phpfpmSocketPathLabel}, nil),
		"idle processes": prometheus.NewDesc(
			prometheus.BuildFQName("php", "fpm", "idle_processes"),
			"Number of idle processes.",
			[]string{phpfpmSocketPathLabel}, nil),
		"active processes": prometheus.NewDesc(
			prometheus.BuildFQName("php", "fpm", "active_processes"),
			"Number of active processes.",
			[]string{phpfpmSocketPathLabel}, nil),
		"total processes": prometheus.NewDesc(
			prometheus.BuildFQName("php", "fpm", "total_processes"),
			"Number of total processes.",
			[]string{phpfpmSocketPathLabel}, nil),
		"max active processes": prometheus.NewDesc(
			prometheus.BuildFQName("php", "fpm", "max_active_processes"),
			"Maximum number of active processes since FPM has started.",
			[]string{phpfpmSocketPathLabel}, nil),
		"max children reached": prometheus.NewDesc(
			prometheus.BuildFQName("php", "fpm", "max_children_reached"),
			"Number of times, the process limit has been reached.",
			[]string{phpfpmSocketPathLabel}, nil),
		"slow requests": prometheus.NewDesc(
			prometheus.BuildFQName("php", "fpm", "slow_requests"),
			"Enable php-fpm slow-log before you consider this. If this value is non-zero you may have slow php processes.",
			[]string{phpfpmSocketPathLabel}, nil),
	}
)

func CollectStatusFromReader(reader io.Reader, socketPath string, ch chan<- prometheus.Metric) error {
	scanner := bufio.NewScanner(reader)
	re := regexp.MustCompile("^(.*): +(.*)$")

	// Scrape the interesting values:
	for scanner.Scan() {
		fields := re.FindStringSubmatch(scanner.Text())
		if fields == nil {
			return fmt.Errorf("Failed to parse %s", scanner.Text())
		}

		if gauge, ok := phpfpmGauges[fields[1]]; ok {
			f, err := strconv.ParseFloat(fields[2], 64)
			if err != nil {
				return err
			}
			ch <- prometheus.MustNewConstMetric(
				gauge,
				prometheus.GaugeValue,
				f,
				socketPath)
		} else if fields[1] == "accepted conn" {
			f, err := strconv.ParseFloat(fields[2], 64)
			if err != nil {
				return err
			}
			ch <- prometheus.MustNewConstMetric(
				phpfpmAcceptedConnections,
				prometheus.CounterValue,
				f,
				socketPath)
		} else if fields[1] == "start time" {
			location, err := time.LoadLocation("Local")
			if err != nil {
				return err
			}
			since, err := time.ParseInLocation("02/Jan/2006:15:04:05 -0700", fields[2], location)
			if err != nil {
				return err
			}
			f := float64(since.Unix())
			ch <- prometheus.MustNewConstMetric(
				phpfpmStartTime,
				prometheus.GaugeValue,
				f,
				socketPath)
		}
	}
	return nil
}

func CollectStatusFromSocket(path string, statusPath string, ch chan<- prometheus.Metric) error {

	env := make(map[string]string)
	env["SCRIPT_FILENAME"] = statusPath
	env["SCRIPT_NAME"] = statusPath
	env["REQUEST_METHOD"] = "GET"

	fcgi, err := fcgiclient.Dial("unix", path)
	if err != nil {
		return err
	}
	defer fcgi.Close()

	resp, err := fcgi.Get(env)
	if err != nil {
		return err
	}

	return CollectStatusFromReader(resp.Body, path, ch)
}

func CollectMetricsFromScript(socketPaths []string, scriptPaths []string) ([]*client_model.MetricFamily, error) {
	var result []*client_model.MetricFamily

	for _, socketPath := range socketPaths {

		for _, scriptPath := range scriptPaths {
			fcgi, err := fcgiclient.Dial("unix", socketPath)
			if err != nil {
				return result, err
			}
			defer fcgi.Close()

			env := make(map[string]string)
			env["DOCUMENT_ROOT"] = path.Dir(scriptPath)
			env["SCRIPT_FILENAME"] = scriptPath
			env["SCRIPT_NAME"] = path.Base(scriptPath)
			env["REQUEST_METHOD"] = "GET"

			resp, err := fcgi.Get(env)
			if err != nil {
				return result, err
			}

			var parser expfmt.TextParser
			metricFamilies, err := parser.TextToMetricFamilies(resp.Body)
			if err != nil {
				return result, err
			}

			for _, metricFamily := range metricFamilies {
				for _, metric := range metricFamily.Metric {
					socketPathCopy := socketPath
					scriptPathCopy := scriptPath
					metric.Label = append(
						metric.Label,
						&client_model.LabelPair{
							Name:  &phpfpmSocketPathLabel,
							Value: &socketPathCopy,
						},
						&client_model.LabelPair{
							Name:  &phpfpmScriptPathLabel,
							Value: &scriptPathCopy,
						})
				}
				result = append(result, metricFamily)
			}
		}
	}
	return result, nil
}

type PhpfpmExporter struct {
	socketPaths []string
	statusPath  string
}

func NewPhpfpmExporter(socketPaths []string, statusPath string) (*PhpfpmExporter, error) {
	return &PhpfpmExporter{
		socketPaths: socketPaths,
		statusPath:  statusPath,
	}, nil
}

func (e *PhpfpmExporter) Describe(ch chan<- *prometheus.Desc) {
	ch <- phpfpmUpDesc
	ch <- phpfpmAcceptedConnections
	ch <- phpfpmStartTime
	for _, desc := range phpfpmGauges {
		ch <- desc
	}
}

func (e *PhpfpmExporter) Collect(ch chan<- prometheus.Metric) {

	for _, socketPath := range e.socketPaths {
		err := CollectStatusFromSocket(socketPath, e.statusPath, ch)
		if err == nil {
			ch <- prometheus.MustNewConstMetric(
				phpfpmUpDesc,
				prometheus.GaugeValue,
				1.0,
				socketPath)
		} else {
			log.Printf("Failed to scrape socket: %s", err)
			ch <- prometheus.MustNewConstMetric(
				phpfpmUpDesc,
				prometheus.GaugeValue,
				0.0,
				socketPath)
		}
	}
}

func main() {
	var (
		listenAddress        = kingpin.Flag("web.listen-address", "Address to listen on for web interface and telemetry.").Default(":9253").String()
		metricsPath          = kingpin.Flag("web.telemetry-path", "Path under which to expose metrics.").Default("/metrics").String()
		socketPaths          = kingpin.Flag("phpfpm.socket-paths", "Paths of the PHP-FPM sockets.").Strings()
		socketDirectories    = kingpin.Flag("phpfpm.socket-directories", "Path(s) of the directory where PHP-FPM sockets are located.").Strings()
		statusPath           = kingpin.Flag("phpfpm.status-path", "Path which has been configured in PHP-FPM to show status page.").Default("/status").String()
		scriptCollectorPaths = kingpin.Flag("phpfpm.script-collector-paths", "Paths of the PHP file whose output needs to be collected.").Strings()
		showVersion          = kingpin.Flag("version", "Print version information.").Bool()
	)

	kingpin.CommandLine.HelpFlag.Short('h')
	kingpin.Parse()

	var sockets []string
	for _, socketDirectory := range *socketDirectories {
		filepath.Walk(socketDirectory, func(path string, info os.FileInfo, err error) error {
			if err == nil && info.Mode()&os.ModeSocket != 0 {
				sockets = append(sockets, path)
			}
			return nil
		})
	}

	for _, socket := range *socketPaths {
		sockets = append(sockets, socket)
	}

	if *showVersion {
		fmt.Println(version.Print("phpfpm_exporter"))
		os.Exit(0)
	}

	exporter, err := NewPhpfpmExporter(sockets, *statusPath)
	if err != nil {
		panic(err)
	}
	prometheus.MustRegister(exporter)

	if len(*scriptCollectorPaths) != 0 {
		prometheus.DefaultGatherer = prometheus.Gatherers{
			prometheus.DefaultGatherer,
			prometheus.GathererFunc(func() ([]*client_model.MetricFamily, error) {
				return CollectMetricsFromScript(sockets, *scriptCollectorPaths)
			}),
		}
	}

	log.Println("Starting phpfpm_exporter", version.Info())
	log.Println("Build context", version.BuildContext())
	log.Printf("Starting Server: %s", *listenAddress)

	http.Handle(*metricsPath, prometheus.Handler())
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(`

			<title>PHP-FPM Exporter</title>

PHP-FPM Exporter

<a href='` + *metricsPath + `'>Metrics

			`))
	})
	log.Fatal(http.ListenAndServe(*listenAddress, nil))
}
<|endoftext|>


<commit_before>package resources

import (
	"fmt"

	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudwatchevents"
)

func init() {
	register("CloudWatchEventsRule", ListCloudWatchEventsRules)
}

func ListCloudWatchEventsRules(sess *session.Session) ([]Resource, error) {
	svc := cloudwatchevents.New(sess)

	resp, err := svc.ListRules(nil)
	if err != nil {
		return nil, err
	}
	resources := make([]Resource, 0)
	for _, rule := range resp.Rules {
		resources = append(resources, &CloudWatchEventsRule{
			svc:  svc,
			name: rule.Name,
		})

	}
	return resources, nil
}

type CloudWatchEventsRule struct {
	svc  *cloudwatchevents.CloudWatchEvents
	name *string
}

func (rule *CloudWatchEventsRule) Remove() error {
	_, err := rule.svc.DeleteRule(&cloudwatchevents.DeleteRuleInput{
		Name: rule.name,
	})
	return err
}

func (rule *CloudWatchEventsRule) String() string {
	return fmt.Sprintf("Rule: %s", *rule.name)
}
<commit_msg>Add Force flag to CW events rule deletion<commit_after>package resources

import (
	"fmt"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudwatchevents"
)

func init() {
	register("CloudWatchEventsRule", ListCloudWatchEventsRules)
}

func ListCloudWatchEventsRules(sess *session.Session) ([]Resource, error) {
	svc := cloudwatchevents.New(sess)

	resp, err := svc.ListRules(nil)
	if err != nil {
		return nil, err
	}
	resources := make([]Resource, 0)
	for _, rule := range resp.Rules {
		resources = append(resources, &CloudWatchEventsRule{
			svc:  svc,
			name: rule.Name,
		})

	}
	return resources, nil
}

type CloudWatchEventsRule struct {
	svc  *cloudwatchevents.CloudWatchEvents
	name *string
}

func (rule *CloudWatchEventsRule) Remove() error {
	_, err := rule.svc.DeleteRule(&cloudwatchevents.DeleteRuleInput{
		Name: rule.name,
		Force: aws.Bool(true),
	})
	return err
}

func (rule *CloudWatchEventsRule) String() string {
	return fmt.Sprintf("Rule: %s", *rule.name)
}
<|endoftext|>


<commit_before>/*
Copyright (c) 2012 krepa098 (krepa098 at gmail dot com)
This software is provided 'as-is', without any express or implied warranty.
In no event will the authors be held liable for any damages arising from the use of this software.
Permission is granted to anyone to use this software for any purpose, including commercial applications,
and to alter it and redistribute it freely, subject to the following restrictions:
	1.	The origin of this software must not be misrepresented; you must not claim that you wrote the original software.
		If you use this software in a product, an acknowledgment in the product documentation would be appreciated but is not required.
	2. 	Altered source versions must be plainly marked as such, and must not be misrepresented as being the original software.
	3. 	This notice may not be removed or altered from any source distribution.
*/

package gosfml2

// #include <SFML/Audio/Music.h>
// #include <stdlib.h>
import "C"

import (
	"errors"
	"runtime"
	"time"
	"unsafe"
)

/////////////////////////////////////
///		STRUCTS
/////////////////////////////////////

type Music struct {
	cptr *C.sfMusic
}

/////////////////////////////////////
///		FUNCS
/////////////////////////////////////

func NewMusicFromFile(file string) (music *Music, err error) {
	cFile := C.CString(file)
	defer C.free(unsafe.Pointer(cFile))
	music = &Music{C.sfMusic_createFromFile(cFile)}
	runtime.SetFinalizer(music, (*Music).Destroy)

	if music.cptr == nil {
		err = errors.New("NewMusicFromFile: Cannot load music " + file)
	}

	return
}

func NewMusicFromMemory(data []byte) (music *Music, err error) {
	if len(data) > 0 {
		music = &Music{C.sfMusic_createFromMemory(unsafe.Pointer(&data[0]), C.size_t(len(data)))}
		runtime.SetFinalizer(music, (*Music).Destroy)

		if music.cptr == nil {
			err = errors.New("NewMusicFromMemory: Cannot load music")
		}
		return
	}
	return
}

func (this *Music) Destroy() {
	C.sfMusic_destroy(this.cptr)
	this.cptr = nil
}

func (this *Music) Play() {
	C.sfMusic_play(this.cptr)
}

func (this *Music) Pause() {
	C.sfMusic_pause(this.cptr)
}

func (this *Music) Stop() {
	C.sfMusic_stop(this.cptr)
}

func (this *Music) SetLoop(loop bool) {
	C.sfMusic_setLoop(this.cptr, goBool2C(loop))
}

func (this *Music) GetStatus() SoundStatus {
	return SoundStatus(C.sfMusic_getStatus(this.cptr))
}

func (this *Music) SetPitch(pitch float32) {
	C.sfMusic_setPitch(this.cptr, C.float(pitch))
}

func (this *Music) SetVolume(volume float32) {
	C.sfMusic_setVolume(this.cptr, C.float(volume))
}

func (this *Music) SetPosition(pos Vector3f) {
	C.sfMusic_setPosition(this.cptr, pos.toC())
}

func (this *Music) SetRelativeToListener(relative bool) {
	C.sfMusic_setRelativeToListener(this.cptr, goBool2C(relative))
}

func (this *Music) SetMinDistance(distance float32) {
	C.sfMusic_setMinDistance(this.cptr, C.float(distance))
}

func (this *Music) SetAttenuation(attenuation float32) {
	C.sfMusic_setAttenuation(this.cptr, C.float(attenuation))
}

func (this *Music) SetPlayingOffset(offset time.Duration) {
	C.sfMusic_setPlayingOffset(this.cptr, C.sfTime{microseconds: (C.sfInt64(offset / time.Microsecond))})
}

func (this *Music) GetPitch() float32 {
	return float32(C.sfMusic_getPitch(this.cptr))
}

func (this *Music) GetVolume() float32 {
	return float32(C.sfMusic_getVolume(this.cptr))
}

func (this *Music) GetPosition() (pos Vector3f) {
	pos.fromC(C.sfMusic_getPosition(this.cptr))
	return
}

func (this *Music) IsRelativeToListner() bool {
	return sfBool2Go(C.sfMusic_isRelativeToListener(this.cptr))
}

func (this *Music) GetMinDistance() float32 {
	return float32(C.sfMusic_getMinDistance(this.cptr))
}

func (this *Music) GetAttenuation() float32 {
	return float32(C.sfMusic_getAttenuation(this.cptr))
}

func (this *Music) GetPlayingOffset() time.Duration {
	return time.Duration(C.sfMusic_getPlayingOffset(this.cptr).microseconds) * time.Microsecond
}

func (this *Music) GetSampleRate() uint {
	return uint(C.sfMusic_getSampleRate(this.cptr))
}

func (this *Music) GetChannelCount() uint {
	return uint(C.sfMusic_getChannelCount(this.cptr))
}

func (this *Music) GetDuration() time.Duration {
	return time.Duration(C.sfMusic_getPlayingOffset(this.cptr).microseconds) * time.Microsecond
}
<commit_msg>documented music.go<commit_after>/*
Copyright (c) 2012 krepa098 (krepa098 at gmail dot com)
This software is provided 'as-is', without any express or implied warranty.
In no event will the authors be held liable for any damages arising from the use of this software.
Permission is granted to anyone to use this software for any purpose, including commercial applications,
and to alter it and redistribute it freely, subject to the following restrictions:
	1.	The origin of this software must not be misrepresented; you must not claim that you wrote the original software.
		If you use this software in a product, an acknowledgment in the product documentation would be appreciated but is not required.
	2. 	Altered source versions must be plainly marked as such, and must not be misrepresented as being the original software.
	3. 	This notice may not be removed or altered from any source distribution.
*/

package gosfml2

// #include <SFML/Audio/Music.h>
// #include <stdlib.h>
import "C"

import (
	"errors"
	"runtime"
	"time"
	"unsafe"
)

/////////////////////////////////////
///		STRUCTS
/////////////////////////////////////

type Music struct {
	cptr *C.sfMusic
}

/////////////////////////////////////
///		FUNCS
/////////////////////////////////////

// Create a new music and load it from a file
//
///This function doesn't start playing the music (call
// Music.Play to do so).
// Here is a complete list of all the supported audio formats:
// ogg, wav, flac, aiff, au, raw, paf, svx, nist, voc, ircam,
// w64, mat4, mat5 pvf, htk, sds, avr, sd2, caf, wve, mpc2k, rf64.
//
// file: Path of the music file to open
func NewMusicFromFile(file string) (music *Music, err error) {
	cFile := C.CString(file)
	defer C.free(unsafe.Pointer(cFile))
	music = &Music{C.sfMusic_createFromFile(cFile)}
	runtime.SetFinalizer(music, (*Music).Destroy)

	if music.cptr == nil {
		err = errors.New("NewMusicFromFile: Cannot load music " + file)
	}

	return
}

// Create a new music and load it from a file in memory
//
// This function doesn't start playing the music (call
// Music.Play to do so).
// Here is a complete list of all the supported audio formats:
// ogg, wav, flac, aiff, au, raw, paf, svx, nist, voc, ircam,
// w64, mat4, mat5 pvf, htk, sds, avr, sd2, caf, wve, mpc2k, rf64.
//
// data: Slice of file data
func NewMusicFromMemory(data []byte) (music *Music, err error) {
	if len(data) > 0 {
		music = &Music{C.sfMusic_createFromMemory(unsafe.Pointer(&data[0]), C.size_t(len(data)))}
		runtime.SetFinalizer(music, (*Music).Destroy)

		if music.cptr == nil {
			err = errors.New("NewMusicFromMemory: Cannot load music")
		}
		return
	}
	return
}

// Destroy a music
func (this *Music) Destroy() {
	C.sfMusic_destroy(this.cptr)
	this.cptr = nil
}

// Start or resume playing a music
//
// This function starts the music if it was stopped, resumes
// it if it was paused, and restarts it from beginning if it
// was it already playing.
// This function uses its own thread so that it doesn't block
// the rest of the program while the music is played.
func (this *Music) Play() {
	C.sfMusic_play(this.cptr)
}

// Pause a music
//
// This function pauses the music if it was playing,
// otherwise (music already paused or stopped) it has no effect.
func (this *Music) Pause() {
	C.sfMusic_pause(this.cptr)
}

// Stop playing a music
//
// This function stops the music if it was playing or paused,
// and does nothing if it was already stopped.
// It also resets the playing position (unlike Music.Pause).
func (this *Music) Stop() {
	C.sfMusic_stop(this.cptr)
}

// Set whether or not a music should loop after reaching the end
//
// If set, the music will restart from beginning after
// reaching the end and so on, until it is stopped or
// Music.SetLoop(false) is called.
// The default looping state for musics is false.
//
// loop:  true to play in loop, false to play once
func (this *Music) SetLoop(loop bool) {
	C.sfMusic_setLoop(this.cptr, goBool2C(loop))
}

// Get the current status of a music (stopped, paused, playing)
func (this *Music) GetStatus() SoundStatus {
	return SoundStatus(C.sfMusic_getStatus(this.cptr))
}

// Set the pitch of a music
//
// The pitch represents the perceived fundamental frequency
// of a sound; thus you can make a music more acute or grave
// by changing its pitch. A side effect of changing the pitch
// is to modify the playing speed of the music as well.
// The default value for the pitch is 1.
//
// pitch: New pitch to apply to the music
func (this *Music) SetPitch(pitch float32) {
	C.sfMusic_setPitch(this.cptr, C.float(pitch))
}

// Set the volume of a music
//
// The volume is a value between 0 (mute) and 100 (full volume).
// The default value for the volume is 100.
//
// volume: Volume of the music
func (this *Music) SetVolume(volume float32) {
	C.sfMusic_setVolume(this.cptr, C.float(volume))
}

// Set the 3D position of a music in the audio scene
//
// Only musics with one channel (mono musics) can be
// spatialized.
// The default position of a music is (0, 0, 0).
//
// position: Position of the music in the scene
func (this *Music) SetPosition(pos Vector3f) {
	C.sfMusic_setPosition(this.cptr, pos.toC())
}

// Make a musics's position relative to the listener or absolute
//
// Making a music relative to the listener will ensure that it will always
// be played the same way regardless the position of the listener.
// This can be useful for non-spatialized musics, musics that are
// produced by the listener, or musics attached to it.
// The default value is false (position is absolute).
//
// relative: true to set the position relative, false to set it absolute
func (this *Music) SetRelativeToListener(relative bool) {
	C.sfMusic_setRelativeToListener(this.cptr, goBool2C(relative))
}

// Set the minimum distance of a music
//
// The "minimum distance" of a music is the maximum
// distance at which it is heard at its maximum volume. Further
// than the minimum distance, it will start to fade out according
// to its attenuation factor. A value of 0 ("inside the head
// of the listener") is an invalid value and is forbidden.
// The default value of the minimum distance is 1.
//
// distance: New minimum distance of the music
func (this *Music) SetMinDistance(distance float32) {
	C.sfMusic_setMinDistance(this.cptr, C.float(distance))
}

// Set the attenuation factor of a music
//
// The attenuation is a multiplicative factor which makes
// the music more or less loud according to its distance
// from the listener. An attenuation of 0 will produce a
// non-attenuated music, i.e. its volume will always be the same
// whether it is heard from near or from far. On the other hand,
// an attenuation value such as 100 will make the music fade out
// very quickly as it gets further from the listener.
// The default value of the attenuation is 1.
//
// attenuation: New attenuation factor of the music
func (this *Music) SetAttenuation(attenuation float32) {
	C.sfMusic_setAttenuation(this.cptr, C.float(attenuation))
}

// Change the current playing position of a music
//
// The playing position can be changed when the music is
// either paused or playing.
//
// timeOffset: New playing position
func (this *Music) SetPlayingOffset(offset time.Duration) {
	C.sfMusic_setPlayingOffset(this.cptr, C.sfTime{microseconds: (C.sfInt64(offset / time.Microsecond))})
}

// Get the pitch of a music
func (this *Music) GetPitch() float32 {
	return float32(C.sfMusic_getPitch(this.cptr))
}

// Get the volume of a music
func (this *Music) GetVolume() float32 {
	return float32(C.sfMusic_getVolume(this.cptr))
}

// Get the 3D position of a music in the audio scene
func (this *Music) GetPosition() (pos Vector3f) {
	pos.fromC(C.sfMusic_getPosition(this.cptr))
	return
}

// Tell whether a music's position is relative to the
// listener or is absolute
func (this *Music) IsRelativeToListner() bool {
	return sfBool2Go(C.sfMusic_isRelativeToListener(this.cptr))
}

// Get the minimum distance of a music
func (this *Music) GetMinDistance() float32 {
	return float32(C.sfMusic_getMinDistance(this.cptr))
}

// Get the attenuation factor of a music
func (this *Music) GetAttenuation() float32 {
	return float32(C.sfMusic_getAttenuation(this.cptr))
}

// Get the current playing position of a music
func (this *Music) GetPlayingOffset() time.Duration {
	return time.Duration(C.sfMusic_getPlayingOffset(this.cptr).microseconds) * time.Microsecond
}

// Get the sample rate of a music
//
// The sample rate is the number of audio samples played per
// second. The higher, the better the quality.
func (this *Music) GetSampleRate() uint {
	return uint(C.sfMusic_getSampleRate(this.cptr))
}

// Return the number of channels of a music
//
// 1 channel means a mono sound, 2 means stereo, etc
func (this *Music) GetChannelCount() uint {
	return uint(C.sfMusic_getChannelCount(this.cptr))
}

// Get the total duration of a music
func (this *Music) GetDuration() time.Duration {
	return time.Duration(C.sfMusic_getPlayingOffset(this.cptr).microseconds) * time.Microsecond
}
<|endoftext|>


<commit_before>// Synchronization built on top of Redis.
// Depends on github.com/garyburd/redigo/redis
package redisync

import (
	"fmt"
	"github.com/garyburd/redigo/redis"
	"io/ioutil"
	"os"
	"sync"
	"time"
)

type Mutex struct {
	// The key used in Redis.
	Name string
	// The amount of time before Redis will expire the lock.
	Ttl time.Duration
	// The time to sleep before retrying a lock attempt.
	Backoff time.Duration
	// A uuid representing the local instantiation of the mutex.
	id string
	// Local conrrency controll.
	l sync.Mutex
	// See lock.lua
	lock *redis.Script
	// See unlock.lua
	unlock *redis.Script
}

// Each lock will have a name which corresponds to a key in the Redis server.
// The mutex will also be initialized with a uuid. The mutex uuid
// can be used to extend the TTL for the lock.
func NewMutex(name string, ttl time.Duration) *Mutex {
	m := new(Mutex)
	m.Name = name
	m.Ttl = ttl
	m.Backoff = time.Second
	m.id = uuid()
	m.lock = redis.NewScript(1, readSource("lock.lua"))
	m.unlock = redis.NewScript(1, readSource("unlock.lua"))
	return m
}

// With similar behaviour to Go's sync pkg,
// this function will sleep until TryLock() returns true.
// The connection will be used once to execute the lock script.
func (m *Mutex) Lock(c redis.Conn) {
	for {
		if m.TryLock(c) {
			return
		}
		time.Sleep(m.Backoff)
	}
}

// Makes a single attempt to acquire the lock.
// Locking a mutex which has already been locked
// using the mutex uuid will result in the TTL of the mutex being extended.
// The connection will be used once to execute the lock script.
func (m *Mutex) TryLock(c redis.Conn) bool {
	m.l.Lock()
	defer m.l.Unlock()

	reply, err := m.lock.Do(c, m.Name, m.id, m.Ttl.Seconds())
	if err != nil {
		return false
	}
	return reply.(int64) == 1
}

// If the local mutex uuid matches the uuid in Redis,
// the lock will be deleted.
// The connection will be used once to execute the unlock script.
func (m *Mutex) Unlock(c redis.Conn) (bool, error) {
	m.l.Lock()
	defer m.l.Unlock()
	reply, err := m.unlock.Do(c, m.Name, m.id)
	if err != nil {
		return false, err
	}
	return reply.(int64) == 1, nil
}

func readSource(name string) string {
	path := os.Getenv("GOPATH")
	prefix := path + "/src/github.com/ryandotsmith/redisync/"
	src, err := ioutil.ReadFile(prefix + name)
	if err != nil {
		panic("redisync: Unable to read unlock.lua")
	}
	return string(src)
}

func uuid() string {
	f, _ := os.Open("/dev/urandom")
	b := make([]byte, 16)
	f.Read(b)
	f.Close()
	return fmt.Sprintf("%x-%x-%x-%x-%x",
		b[0:4], b[4:6], b[6:8], b[8:10], b[10:])
}
<commit_msg>read files at compile time<commit_after>// Synchronization built on top of Redis.
// Depends on github.com/garyburd/redigo/redis
package redisync

import (
	"fmt"
	"github.com/garyburd/redigo/redis"
	"io/ioutil"
	"os"
	"sync"
	"time"
)

var luaLock, luaUnlock string

func init() {
	luaLock = readSource("lock.lua")
	luaUnlock = readSource("unlock.lua")
}

type Mutex struct {
	// The key used in Redis.
	Name string
	// The amount of time before Redis will expire the lock.
	Ttl time.Duration
	// The time to sleep before retrying a lock attempt.
	Backoff time.Duration
	// A uuid representing the local instantiation of the mutex.
	id string
	// Local conrrency controll.
	l sync.Mutex
	// See lock.lua
	lock *redis.Script
	// See unlock.lua
	unlock *redis.Script
}

// Each lock will have a name which corresponds to a key in the Redis server.
// The mutex will also be initialized with a uuid. The mutex uuid
// can be used to extend the TTL for the lock.
func NewMutex(name string, ttl time.Duration) *Mutex {
	m := new(Mutex)
	m.Name = name
	m.Ttl = ttl
	m.Backoff = time.Second
	m.id = uuid()
	m.lock = redis.NewScript(1, luaLock)
	m.unlock = redis.NewScript(1, luaUnlock)
	return m
}

// With similar behaviour to Go's sync pkg,
// this function will sleep until TryLock() returns true.
// The connection will be used once to execute the lock script.
func (m *Mutex) Lock(c redis.Conn) {
	for {
		if m.TryLock(c) {
			return
		}
		time.Sleep(m.Backoff)
	}
}

// Makes a single attempt to acquire the lock.
// Locking a mutex which has already been locked
// using the mutex uuid will result in the TTL of the mutex being extended.
// The connection will be used once to execute the lock script.
func (m *Mutex) TryLock(c redis.Conn) bool {
	m.l.Lock()
	defer m.l.Unlock()

	reply, err := m.lock.Do(c, m.Name, m.id, m.Ttl.Seconds())
	if err != nil {
		return false
	}
	return reply.(int64) == 1
}

// If the local mutex uuid matches the uuid in Redis,
// the lock will be deleted.
// The connection will be used once to execute the unlock script.
func (m *Mutex) Unlock(c redis.Conn) (bool, error) {
	m.l.Lock()
	defer m.l.Unlock()
	reply, err := m.unlock.Do(c, m.Name, m.id)
	if err != nil {
		return false, err
	}
	return reply.(int64) == 1, nil
}

func readSource(name string) string {
	path := os.Getenv("GOPATH")
	prefix := path + "/src/github.com/ryandotsmith/redisync/"
	src, err := ioutil.ReadFile(prefix + name)
	if err != nil {
		panic("redisync: Unable to read unlock.lua")
	}
	return string(src)
}

func uuid() string {
	f, _ := os.Open("/dev/urandom")
	b := make([]byte, 16)
	f.Read(b)
	f.Close()
	return fmt.Sprintf("%x-%x-%x-%x-%x",
		b[0:4], b[4:6], b[6:8], b[8:10], b[10:])
}
<|endoftext|>


<commit_before>package backend

import (
	"bytes"
	"crypto/x509"
	"encoding/json"
	"encoding/pem"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"

	"github.com/pkg/sftp"
	workerctx "github.com/travis-ci/worker/lib/context"
	"github.com/travis-ci/worker/lib/metrics"
	"golang.org/x/crypto/ssh"
	"golang.org/x/net/context"
)

type SauceLabsProvider struct {
	client           *http.Client
	baseURL          *url.URL
	imageAliases     map[string]string
	sshKeyPath       string
	sshKeyPassphrase string
}

type SauceLabsInstance struct {
	payload  sauceLabsInstancePayload
	provider *SauceLabsProvider
}

type sauceLabsInstancePayload struct {
	ID          string                 `json:"instance_id"`
	ImageID     string                 `json:"image_id"`
	RealImageID string                 `json:"real_image_id"`
	State       string                 `json:"state"`
	PrivateIP   string                 `json:"private_ip"`
	ExtraInfo   map[string]interface{} `json:"extra_info"`
}

func NewSauceLabsProvider(config map[string]string) (*SauceLabsProvider, error) {
	endpoint, ok := config["endpoint"]
	if !ok {
		return nil, fmt.Errorf("expected endpoint config key")
	}
	baseURL, err := url.Parse(endpoint)
	if err != nil {
		return nil, err
	}

	aliasNames, ok := config["image_aliases"]
	if !ok {
		return nil, fmt.Errorf("expected image_aliases config key")
	}

	imageAliases := make(map[string]string, len(aliasNames))

	for _, aliasName := range strings.Split(aliasNames, ",") {
		imageName, ok := config[fmt.Sprintf("image_alias_%s", aliasName)]
		if !ok {
			return nil, fmt.Errorf("expected image alias %q", aliasName)
		}

		imageAliases[aliasName] = imageName
	}

	sshKeyPath, ok := config["ssh_key_path"]
	if !ok {
		return nil, fmt.Errorf("expected ssh_key_path config key")
	}

	sshKeyPassphrase, ok := config["ssh_key_passphrase"]
	if !ok {
		return nil, fmt.Errorf("expected ssh_key_passphrase config key")
	}

	return &SauceLabsProvider{
		client:           http.DefaultClient,
		baseURL:          baseURL,
		imageAliases:     imageAliases,
		sshKeyPath:       sshKeyPath,
		sshKeyPassphrase: sshKeyPassphrase,
	}, nil
}

func (p *SauceLabsProvider) Start(ctx context.Context, startAttributes StartAttributes) (Instance, error) {
	startupInfo, err := json.Marshal(map[string]interface{}{"worker_pid": os.Getpid(), "source": "worker"})
	if err != nil {
		return nil, err
	}

	imageName, ok := p.imageAliases[startAttributes.OsxImage]
	if !ok {
		imageName, _ = p.imageAliases["default"]
	}

	if imageName == "" {
		return nil, fmt.Errorf("no image alias for %s", startAttributes.OsxImage)
	}

	u, err := p.baseURL.Parse(fmt.Sprintf("instances?image=%s", url.QueryEscape(imageName)))
	if err != nil {
		return nil, err
	}

	req, err := http.NewRequest("POST", u.String(), bytes.NewReader(startupInfo))
	if err != nil {
		return nil, err
	}

	startBooting := time.Now()

	resp, err := p.client.Do(req)
	if err != nil {
		return nil, err
	}
	defer io.Copy(ioutil.Discard, resp.Body)
	defer resp.Body.Close()

	if c := resp.StatusCode; c < 200 || c >= 300 {
		return nil, fmt.Errorf("expected 2xx from Sauce Labs API, got %d", c)
	}

	var payload sauceLabsInstancePayload
	err = json.NewDecoder(resp.Body).Decode(&payload)

	instanceReady := make(chan sauceLabsInstancePayload, 1)
	errChan := make(chan error, 1)
	go func(id string) {
		u, err := p.baseURL.Parse(fmt.Sprintf("instances/%s", url.QueryEscape(id)))
		if err != nil {
			errChan <- err
			return
		}

		req, err := http.NewRequest("GET", u.String(), nil)
		if err != nil {
			errChan <- err
			return
		}

		for true {
			resp, err := p.client.Do(req)
			if err != nil {
				errChan <- err
				return
			}
			defer io.Copy(ioutil.Discard, resp.Body)
			defer resp.Body.Close()

			var payload sauceLabsInstancePayload
			err = json.NewDecoder(resp.Body).Decode(&payload)
			if err != nil {
				errChan <- err
				return
			}

			conn, err := net.Dial("tcp", fmt.Sprintf("%s:3422", payload.PrivateIP))
			if conn != nil {
				conn.Close()
			}

			if err == nil {
				instanceReady <- payload
				return
			}
		}
	}(payload.ID)

	select {
	case payload := <-instanceReady:
		metrics.TimeSince("worker.vm.provider.sauce_labs.boot", startBooting)
		workerctx.LoggerFromContext(ctx).WithField("instance_id", payload.ID).Info("booted instance")
		return &SauceLabsInstance{
			payload:  payload,
			provider: p,
		}, nil
	case err := <-errChan:
		instance := &SauceLabsInstance{
			payload:  payload,
			provider: p,
		}
		instance.Stop(ctx)

		return nil, err
	case <-ctx.Done():
		if ctx.Err() == context.DeadlineExceeded {
			metrics.Mark("worker.vm.provider.sauce_labs.boot.timeout")
		}

		instance := &SauceLabsInstance{
			payload:  payload,
			provider: p,
		}
		instance.Stop(ctx)

		return nil, ctx.Err()
	}
}

func (i *SauceLabsInstance) UploadScript(ctx context.Context, script []byte) error {
	client, err := i.sshClient()
	if err != nil {
		return err
	}
	defer client.Close()

	sftp, err := sftp.NewClient(client)
	if err != nil {
		return err
	}
	defer sftp.Close()

	f, err := sftp.Create("build.sh")
	if err != nil {
		return err
	}

	_, err = f.Write(script)
	if err != nil {
		return err
	}

	f, err = sftp.Create("wrapper.sh")
	if err != nil {
		return err
	}

	_, err = fmt.Fprintf(f, `#!/bin/bash

[[ -f ~/build.sh.exit ]] && rm ~/build.sh.exit

until nc 127.0.0.1 15782; do sleep 1; done

until [[ -f ~/build.sh.exit ]]; do sleep 1; done
exit $(cat ~/build.sh.exit)
`)

	return err
}

func (i *SauceLabsInstance) RunScript(ctx context.Context, output io.WriteCloser) (RunResult, error) {
	client, err := i.sshClient()
	if err != nil {
		return RunResult{Completed: false}, err
	}
	defer client.Close()

	session, err := client.NewSession()
	if err != nil {
		return RunResult{Completed: false}, err
	}
	defer session.Close()

	err = session.RequestPty("xterm", 80, 40, ssh.TerminalModes{})
	if err != nil {
		return RunResult{Completed: false}, err
	}

	session.Stdout = output
	session.Stderr = output

	err = session.Run("bash ~/wrapper.sh")
	if err == nil {
		return RunResult{Completed: true, ExitCode: 0}, nil
	}

	switch err := err.(type) {
	case *ssh.ExitError:
		return RunResult{Completed: true, ExitCode: uint8(err.ExitStatus())}, nil
	default:
		return RunResult{Completed: false}, err
	}
}

func (i *SauceLabsInstance) Stop(ctx context.Context) error {
	u, err := i.provider.baseURL.Parse(fmt.Sprintf("instances/%s", url.QueryEscape(i.payload.ID)))
	if err != nil {
		return err
	}

	req, err := http.NewRequest("DELETE", u.String(), nil)
	if err != nil {
		return err
	}

	resp, err := i.provider.client.Do(req)
	io.Copy(ioutil.Discard, resp.Body)
	resp.Body.Close()
	return err
}

func (i *SauceLabsInstance) sshClient() (*ssh.Client, error) {
	file, err := ioutil.ReadFile(i.provider.sshKeyPath)
	if err != nil {
		return nil, err
	}

	block, _ := pem.Decode(file)
	if block == nil {
		return nil, fmt.Errorf("ssh key does not contain a valid PEM block")
	}

	der, err := x509.DecryptPEMBlock(block, []byte(i.provider.sshKeyPassphrase))
	if err != nil {
		return nil, err
	}

	key, err := x509.ParsePKCS1PrivateKey(der)
	if err != nil {
		return nil, err
	}

	signer, err := ssh.NewSignerFromKey(key)
	if err != nil {
		return nil, err
	}

	return ssh.Dial("tcp", fmt.Sprintf("%s:3422", i.payload.PrivateIP), &ssh.ClientConfig{
		User: "travis",
		Auth: []ssh.AuthMethod{
			ssh.PublicKeys(signer),
		},
	})
}
<commit_msg>sauce-labs: close log writer when done<commit_after>package backend

import (
	"bytes"
	"crypto/x509"
	"encoding/json"
	"encoding/pem"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"

	"github.com/pkg/sftp"
	workerctx "github.com/travis-ci/worker/lib/context"
	"github.com/travis-ci/worker/lib/metrics"
	"golang.org/x/crypto/ssh"
	"golang.org/x/net/context"
)

type SauceLabsProvider struct {
	client           *http.Client
	baseURL          *url.URL
	imageAliases     map[string]string
	sshKeyPath       string
	sshKeyPassphrase string
}

type SauceLabsInstance struct {
	payload  sauceLabsInstancePayload
	provider *SauceLabsProvider
}

type sauceLabsInstancePayload struct {
	ID          string                 `json:"instance_id"`
	ImageID     string                 `json:"image_id"`
	RealImageID string                 `json:"real_image_id"`
	State       string                 `json:"state"`
	PrivateIP   string                 `json:"private_ip"`
	ExtraInfo   map[string]interface{} `json:"extra_info"`
}

func NewSauceLabsProvider(config map[string]string) (*SauceLabsProvider, error) {
	endpoint, ok := config["endpoint"]
	if !ok {
		return nil, fmt.Errorf("expected endpoint config key")
	}
	baseURL, err := url.Parse(endpoint)
	if err != nil {
		return nil, err
	}

	aliasNames, ok := config["image_aliases"]
	if !ok {
		return nil, fmt.Errorf("expected image_aliases config key")
	}

	imageAliases := make(map[string]string, len(aliasNames))

	for _, aliasName := range strings.Split(aliasNames, ",") {
		imageName, ok := config[fmt.Sprintf("image_alias_%s", aliasName)]
		if !ok {
			return nil, fmt.Errorf("expected image alias %q", aliasName)
		}

		imageAliases[aliasName] = imageName
	}

	sshKeyPath, ok := config["ssh_key_path"]
	if !ok {
		return nil, fmt.Errorf("expected ssh_key_path config key")
	}

	sshKeyPassphrase, ok := config["ssh_key_passphrase"]
	if !ok {
		return nil, fmt.Errorf("expected ssh_key_passphrase config key")
	}

	return &SauceLabsProvider{
		client:           http.DefaultClient,
		baseURL:          baseURL,
		imageAliases:     imageAliases,
		sshKeyPath:       sshKeyPath,
		sshKeyPassphrase: sshKeyPassphrase,
	}, nil
}

func (p *SauceLabsProvider) Start(ctx context.Context, startAttributes StartAttributes) (Instance, error) {
	startupInfo, err := json.Marshal(map[string]interface{}{"worker_pid": os.Getpid(), "source": "worker"})
	if err != nil {
		return nil, err
	}

	imageName, ok := p.imageAliases[startAttributes.OsxImage]
	if !ok {
		imageName, _ = p.imageAliases["default"]
	}

	if imageName == "" {
		return nil, fmt.Errorf("no image alias for %s", startAttributes.OsxImage)
	}

	u, err := p.baseURL.Parse(fmt.Sprintf("instances?image=%s", url.QueryEscape(imageName)))
	if err != nil {
		return nil, err
	}

	req, err := http.NewRequest("POST", u.String(), bytes.NewReader(startupInfo))
	if err != nil {
		return nil, err
	}

	startBooting := time.Now()

	resp, err := p.client.Do(req)
	if err != nil {
		return nil, err
	}
	defer io.Copy(ioutil.Discard, resp.Body)
	defer resp.Body.Close()

	if c := resp.StatusCode; c < 200 || c >= 300 {
		return nil, fmt.Errorf("expected 2xx from Sauce Labs API, got %d", c)
	}

	var payload sauceLabsInstancePayload
	err = json.NewDecoder(resp.Body).Decode(&payload)

	instanceReady := make(chan sauceLabsInstancePayload, 1)
	errChan := make(chan error, 1)
	go func(id string) {
		u, err := p.baseURL.Parse(fmt.Sprintf("instances/%s", url.QueryEscape(id)))
		if err != nil {
			errChan <- err
			return
		}

		req, err := http.NewRequest("GET", u.String(), nil)
		if err != nil {
			errChan <- err
			return
		}

		for true {
			resp, err := p.client.Do(req)
			if err != nil {
				errChan <- err
				return
			}
			defer io.Copy(ioutil.Discard, resp.Body)
			defer resp.Body.Close()

			var payload sauceLabsInstancePayload
			err = json.NewDecoder(resp.Body).Decode(&payload)
			if err != nil {
				errChan <- err
				return
			}

			conn, err := net.Dial("tcp", fmt.Sprintf("%s:3422", payload.PrivateIP))
			if conn != nil {
				conn.Close()
			}

			if err == nil {
				instanceReady <- payload
				return
			}
		}
	}(payload.ID)

	select {
	case payload := <-instanceReady:
		metrics.TimeSince("worker.vm.provider.sauce_labs.boot", startBooting)
		workerctx.LoggerFromContext(ctx).WithField("instance_id", payload.ID).Info("booted instance")
		return &SauceLabsInstance{
			payload:  payload,
			provider: p,
		}, nil
	case err := <-errChan:
		instance := &SauceLabsInstance{
			payload:  payload,
			provider: p,
		}
		instance.Stop(ctx)

		return nil, err
	case <-ctx.Done():
		if ctx.Err() == context.DeadlineExceeded {
			metrics.Mark("worker.vm.provider.sauce_labs.boot.timeout")
		}

		instance := &SauceLabsInstance{
			payload:  payload,
			provider: p,
		}
		instance.Stop(ctx)

		return nil, ctx.Err()
	}
}

func (i *SauceLabsInstance) UploadScript(ctx context.Context, script []byte) error {
	client, err := i.sshClient()
	if err != nil {
		return err
	}
	defer client.Close()

	sftp, err := sftp.NewClient(client)
	if err != nil {
		return err
	}
	defer sftp.Close()

	f, err := sftp.Create("build.sh")
	if err != nil {
		return err
	}

	_, err = f.Write(script)
	if err != nil {
		return err
	}

	f, err = sftp.Create("wrapper.sh")
	if err != nil {
		return err
	}

	_, err = fmt.Fprintf(f, `#!/bin/bash

[[ -f ~/build.sh.exit ]] && rm ~/build.sh.exit

until nc 127.0.0.1 15782; do sleep 1; done

until [[ -f ~/build.sh.exit ]]; do sleep 1; done
exit $(cat ~/build.sh.exit)
`)

	return err
}

func (i *SauceLabsInstance) RunScript(ctx context.Context, output io.WriteCloser) (RunResult, error) {
	client, err := i.sshClient()
	if err != nil {
		return RunResult{Completed: false}, err
	}
	defer client.Close()

	session, err := client.NewSession()
	if err != nil {
		return RunResult{Completed: false}, err
	}
	defer session.Close()

	err = session.RequestPty("xterm", 80, 40, ssh.TerminalModes{})
	if err != nil {
		return RunResult{Completed: false}, err
	}

	session.Stdout = output
	session.Stderr = output

	err = session.Run("bash ~/wrapper.sh")
	defer output.Close()
	if err == nil {
		return RunResult{Completed: true, ExitCode: 0}, nil
	}

	switch err := err.(type) {
	case *ssh.ExitError:
		return RunResult{Completed: true, ExitCode: uint8(err.ExitStatus())}, nil
	default:
		return RunResult{Completed: false}, err
	}
}

func (i *SauceLabsInstance) Stop(ctx context.Context) error {
	u, err := i.provider.baseURL.Parse(fmt.Sprintf("instances/%s", url.QueryEscape(i.payload.ID)))
	if err != nil {
		return err
	}

	req, err := http.NewRequest("DELETE", u.String(), nil)
	if err != nil {
		return err
	}

	resp, err := i.provider.client.Do(req)
	io.Copy(ioutil.Discard, resp.Body)
	resp.Body.Close()
	return err
}

func (i *SauceLabsInstance) sshClient() (*ssh.Client, error) {
	file, err := ioutil.ReadFile(i.provider.sshKeyPath)
	if err != nil {
		return nil, err
	}

	block, _ := pem.Decode(file)
	if block == nil {
		return nil, fmt.Errorf("ssh key does not contain a valid PEM block")
	}

	der, err := x509.DecryptPEMBlock(block, []byte(i.provider.sshKeyPassphrase))
	if err != nil {
		return nil, err
	}

	key, err := x509.ParsePKCS1PrivateKey(der)
	if err != nil {
		return nil, err
	}

	signer, err := ssh.NewSignerFromKey(key)
	if err != nil {
		return nil, err
	}

	return ssh.Dial("tcp", fmt.Sprintf("%s:3422", i.payload.PrivateIP), &ssh.ClientConfig{
		User: "travis",
		Auth: []ssh.AuthMethod{
			ssh.PublicKeys(signer),
		},
	})
}
<|endoftext|>