<commit_before>package awspurge

import (
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/aws/aws-sdk-go/service/elb"
	"github.com/hashicorp/go-multierror"
)

func (p *Purge) fetchELBResources(fn func(*elb.ELB) error) {
	for _, s := range p.services.elb {
		p.fetchWg.Add(1)

		go func(svc *elb.ELB) {
			err := fn(svc)
			if err != nil {
				p.fetchMu.Lock()
				p.fetchErrs = multierror.Append(p.fetchErrs, err)
				p.fetchMu.Unlock()
			}

			p.fetchWg.Done()
		}(s)
	}
}

func (p *Purge) fetchEC2Resources(fn func(*ec2.EC2) error) {
	for _, s := range p.services.ec2 {
		p.fetchWg.Add(1)

		go func(svc *ec2.EC2) {
			err := fn(svc)
			if err != nil {
				p.fetchMu.Lock()
				p.fetchErrs = multierror.Append(p.fetchErrs, err)
				p.fetchMu.Unlock()
			}

			p.fetchWg.Done()
		}(s)
	}
}

func (p *Purge) FetchInstances() {
	fn := func(svc *ec2.EC2) error {
		resp, err := svc.DescribeInstances(nil)
		if err != nil {
			return err
		}

		instances := make([]*ec2.Instance, 0)
		if resp.Reservations != nil {
			for _, reserv := range resp.Reservations {
				if len(reserv.Instances) != 0 {
					instances = append(instances, reserv.Instances...)
				}
			}
		}

		region := *svc.Config.Region

		p.resourceMu.Lock()
		p.resources[region].instances = instances
		p.resourceMu.Unlock()
		return nil
	}

	p.fetchEC2Resources(fn)
}

func (p *Purge) FetchVolumes() {
	fn := func(svc *ec2.EC2) error {
		resp, err := svc.DescribeVolumes(nil)
		if err != nil {
			return err
		}

		volumes := resp.Volumes
		region := *svc.Config.Region

		p.resourceMu.Lock()
		p.resources[region].volumes = volumes
		p.resourceMu.Unlock()
		return nil
	}

	p.fetchEC2Resources(fn)
}

func (p *Purge) FetchKeyPairs() {
	fn := func(svc *ec2.EC2) error {
		resp, err := svc.DescribeKeyPairs(nil)
		if err != nil {
			return err
		}

		resources := resp.KeyPairs
		region := *svc.Config.Region

		p.resourceMu.Lock()
		p.resources[region].keyPairs = resources
		p.resourceMu.Unlock()
		return nil
	}

	p.fetchEC2Resources(fn)
}

func (p *Purge) FetchPlacementGroups() {
	fn := func(svc *ec2.EC2) error {
		resp, err := svc.DescribePlacementGroups(nil)
		if err != nil {
			return err
		}

		resources := resp.PlacementGroups
		region := *svc.Config.Region

		p.resourceMu.Lock()
		p.resources[region].placementGroups = resources
		p.resourceMu.Unlock()
		return nil
	}

	p.fetchEC2Resources(fn)
}

func (p *Purge) FetchAddresses() {
	fn := func(svc *ec2.EC2) error {
		resp, err := svc.DescribeAddresses(nil)
		if err != nil {
			return err
		}

		resources := resp.Addresses
		region := *svc.Config.Region

		p.resourceMu.Lock()
		p.resources[region].addresses = resources
		p.resourceMu.Unlock()
		return nil
	}

	p.fetchEC2Resources(fn)
}

func (p *Purge) FetchSnapshots() {
	fn := func(svc *ec2.EC2) error {
		input := &ec2.DescribeSnapshotsInput{
			OwnerIds: stringSlice("self"),
		}

		resp, err := svc.DescribeSnapshots(input)
		if err != nil {
			return err
		}

		resources := resp.Snapshots
		region := *svc.Config.Region

		p.resourceMu.Lock()
		p.resources[region].snapshots = resources
		p.resourceMu.Unlock()
		return nil
	}

	p.fetchEC2Resources(fn)
}

func (p *Purge) FetchSecurityGroups() {
	fn := func(svc *ec2.EC2) error {
		resp, err := svc.DescribeSecurityGroups(nil)
		if err != nil {
			return err
		}

		resources := resp.SecurityGroups
		region := *svc.Config.Region

		p.resourceMu.Lock()
		p.resources[region].securityGroups = resources
		p.resourceMu.Unlock()
		return nil
	}

	p.fetchEC2Resources(fn)
}

func (p *Purge) FetchLoadBalancers() {
	fn := func(svc *elb.ELB) error {
		resp, err := svc.DescribeLoadBalancers(nil)
		if err != nil {
			return err
		}

		loadBalancers := resp.LoadBalancerDescriptions
		region := *svc.Config.Region

		p.resourceMu.Lock()
		p.resources[region].loadBalancers = loadBalancers
		p.resourceMu.Unlock()
		return nil
	}

	p.fetchELBResources(fn)
}

func (p *Purge) FetchVpcs() {
	fn := func(svc *ec2.EC2) error {
		resp, err := svc.DescribeVpcs(nil)
		if err != nil {
			return err
		}

		resources := resp.Vpcs
		region := *svc.Config.Region

		p.resourceMu.Lock()
		p.resources[region].vpcs = resources
		p.resourceMu.Unlock()
		return nil
	}

	p.fetchEC2Resources(fn)
}

func (p *Purge) FetchSubnets() {
	fn := func(svc *ec2.EC2) error {
		resp, err := svc.DescribeSubnets(nil)
		if err != nil {
			return err
		}

		resources := resp.Subnets
		region := *svc.Config.Region

		p.resourceMu.Lock()
		p.resources[region].subnets = resources
		p.resourceMu.Unlock()
		return nil
	}

	p.fetchEC2Resources(fn)
}

func (p *Purge) FetchNetworkAcls() {
	fn := func(svc *ec2.EC2) error {
		resp, err := svc.DescribeNetworkAcls(nil)
		if err != nil {
			return err
		}

		resources := resp.NetworkAcls
		region := *svc.Config.Region

		p.resourceMu.Lock()
		p.resources[region].networkAcls = resources
		p.resourceMu.Unlock()
		return nil
	}

	p.fetchEC2Resources(fn)
}

func (p *Purge) FetchInternetGateways() {
	fn := func(svc *ec2.EC2) error {
		resp, err := svc.DescribeInternetGateways(nil)
		if err != nil {
			return err
		}

		resources := resp.InternetGateways
		region := *svc.Config.Region

		p.resourceMu.Lock()
		p.resources[region].internetGateways = resources
		p.resourceMu.Unlock()
		return nil
	}

	p.fetchEC2Resources(fn)
}

func (p *Purge) FetchRouteTables() {
	fn := func(svc *ec2.EC2) error {
		resp, err := svc.DescribeRouteTables(nil)
		if err != nil {
			return err
		}

		resources := resp.RouteTables
		region := *svc.Config.Region

		p.resourceMu.Lock()
		p.resources[region].routeTables = resources
		p.resourceMu.Unlock()
		return nil
	}

	p.fetchEC2Resources(fn)
}

// stringSlice is an helper method to convert a slice of strings into a slice
// of pointer of strings. Needed for various aws/ec2 commands.
func stringSlice(vals ...string) []*string {
	a := make([]*string, len(vals))

	for i, v := range vals {
		a[i] = aws.String(v)
	}

	return a
}
<commit_msg>awspurge: do not include default vpcs<commit_after>package awspurge

import (
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/aws/aws-sdk-go/service/elb"
	"github.com/hashicorp/go-multierror"
)

func (p *Purge) fetchELBResources(fn func(*elb.ELB) error) {
	for _, s := range p.services.elb {
		p.fetchWg.Add(1)

		go func(svc *elb.ELB) {
			err := fn(svc)
			if err != nil {
				p.fetchMu.Lock()
				p.fetchErrs = multierror.Append(p.fetchErrs, err)
				p.fetchMu.Unlock()
			}

			p.fetchWg.Done()
		}(s)
	}
}

func (p *Purge) fetchEC2Resources(fn func(*ec2.EC2) error) {
	for _, s := range p.services.ec2 {
		p.fetchWg.Add(1)

		go func(svc *ec2.EC2) {
			err := fn(svc)
			if err != nil {
				p.fetchMu.Lock()
				p.fetchErrs = multierror.Append(p.fetchErrs, err)
				p.fetchMu.Unlock()
			}

			p.fetchWg.Done()
		}(s)
	}
}

func (p *Purge) FetchInstances() {
	fn := func(svc *ec2.EC2) error {
		resp, err := svc.DescribeInstances(nil)
		if err != nil {
			return err
		}

		instances := make([]*ec2.Instance, 0)
		if resp.Reservations != nil {
			for _, reserv := range resp.Reservations {
				if len(reserv.Instances) != 0 {
					instances = append(instances, reserv.Instances...)
				}
			}
		}

		region := *svc.Config.Region

		p.resourceMu.Lock()
		p.resources[region].instances = instances
		p.resourceMu.Unlock()
		return nil
	}

	p.fetchEC2Resources(fn)
}

func (p *Purge) FetchVolumes() {
	fn := func(svc *ec2.EC2) error {
		resp, err := svc.DescribeVolumes(nil)
		if err != nil {
			return err
		}

		volumes := resp.Volumes
		region := *svc.Config.Region

		p.resourceMu.Lock()
		p.resources[region].volumes = volumes
		p.resourceMu.Unlock()
		return nil
	}

	p.fetchEC2Resources(fn)
}

func (p *Purge) FetchKeyPairs() {
	fn := func(svc *ec2.EC2) error {
		resp, err := svc.DescribeKeyPairs(nil)
		if err != nil {
			return err
		}

		resources := resp.KeyPairs
		region := *svc.Config.Region

		p.resourceMu.Lock()
		p.resources[region].keyPairs = resources
		p.resourceMu.Unlock()
		return nil
	}

	p.fetchEC2Resources(fn)
}

func (p *Purge) FetchPlacementGroups() {
	fn := func(svc *ec2.EC2) error {
		resp, err := svc.DescribePlacementGroups(nil)
		if err != nil {
			return err
		}

		resources := resp.PlacementGroups
		region := *svc.Config.Region

		p.resourceMu.Lock()
		p.resources[region].placementGroups = resources
		p.resourceMu.Unlock()
		return nil
	}

	p.fetchEC2Resources(fn)
}

func (p *Purge) FetchAddresses() {
	fn := func(svc *ec2.EC2) error {
		resp, err := svc.DescribeAddresses(nil)
		if err != nil {
			return err
		}

		resources := resp.Addresses
		region := *svc.Config.Region

		p.resourceMu.Lock()
		p.resources[region].addresses = resources
		p.resourceMu.Unlock()
		return nil
	}

	p.fetchEC2Resources(fn)
}

func (p *Purge) FetchSnapshots() {
	fn := func(svc *ec2.EC2) error {
		input := &ec2.DescribeSnapshotsInput{
			OwnerIds: stringSlice("self"),
		}

		resp, err := svc.DescribeSnapshots(input)
		if err != nil {
			return err
		}

		resources := resp.Snapshots
		region := *svc.Config.Region

		p.resourceMu.Lock()
		p.resources[region].snapshots = resources
		p.resourceMu.Unlock()
		return nil
	}

	p.fetchEC2Resources(fn)
}

func (p *Purge) FetchSecurityGroups() {
	fn := func(svc *ec2.EC2) error {
		resp, err := svc.DescribeSecurityGroups(nil)
		if err != nil {
			return err
		}

		resources := resp.SecurityGroups
		region := *svc.Config.Region

		p.resourceMu.Lock()
		p.resources[region].securityGroups = resources
		p.resourceMu.Unlock()
		return nil
	}

	p.fetchEC2Resources(fn)
}

func (p *Purge) FetchLoadBalancers() {
	fn := func(svc *elb.ELB) error {
		resp, err := svc.DescribeLoadBalancers(nil)
		if err != nil {
			return err
		}

		loadBalancers := resp.LoadBalancerDescriptions
		region := *svc.Config.Region

		p.resourceMu.Lock()
		p.resources[region].loadBalancers = loadBalancers
		p.resourceMu.Unlock()
		return nil
	}

	p.fetchELBResources(fn)
}

func (p *Purge) FetchVpcs() {
	fn := func(svc *ec2.EC2) error {
		resp, err := svc.DescribeVpcs(nil)
		if err != nil {
			return err
		}

		region := *svc.Config.Region

		vpcs := make([]*ec2.Vpc, 0)
		for _, vpc := range resp.Vpcs {
			// don't delete default vpc
			if !*vpc.IsDefault {
				vpcs = append(vpcs, vpc)
			}
		}

		p.resourceMu.Lock()
		p.resources[region].vpcs = vpcs
		p.resourceMu.Unlock()
		return nil
	}

	p.fetchEC2Resources(fn)
}

func (p *Purge) FetchSubnets() {
	fn := func(svc *ec2.EC2) error {
		resp, err := svc.DescribeSubnets(nil)
		if err != nil {
			return err
		}

		resources := resp.Subnets
		region := *svc.Config.Region

		p.resourceMu.Lock()
		p.resources[region].subnets = resources
		p.resourceMu.Unlock()
		return nil
	}

	p.fetchEC2Resources(fn)
}

func (p *Purge) FetchNetworkAcls() {
	fn := func(svc *ec2.EC2) error {
		resp, err := svc.DescribeNetworkAcls(nil)
		if err != nil {
			return err
		}

		resources := resp.NetworkAcls
		region := *svc.Config.Region

		p.resourceMu.Lock()
		p.resources[region].networkAcls = resources
		p.resourceMu.Unlock()
		return nil
	}

	p.fetchEC2Resources(fn)
}

func (p *Purge) FetchInternetGateways() {
	fn := func(svc *ec2.EC2) error {
		resp, err := svc.DescribeInternetGateways(nil)
		if err != nil {
			return err
		}

		resources := resp.InternetGateways
		region := *svc.Config.Region

		p.resourceMu.Lock()
		p.resources[region].internetGateways = resources
		p.resourceMu.Unlock()
		return nil
	}

	p.fetchEC2Resources(fn)
}

func (p *Purge) FetchRouteTables() {
	fn := func(svc *ec2.EC2) error {
		resp, err := svc.DescribeRouteTables(nil)
		if err != nil {
			return err
		}

		resources := resp.RouteTables
		region := *svc.Config.Region

		p.resourceMu.Lock()
		p.resources[region].routeTables = resources
		p.resourceMu.Unlock()
		return nil
	}

	p.fetchEC2Resources(fn)
}

// stringSlice is an helper method to convert a slice of strings into a slice
// of pointer of strings. Needed for various aws/ec2 commands.
func stringSlice(vals ...string) []*string {
	a := make([]*string, len(vals))

	for i, v := range vals {
		a[i] = aws.String(v)
	}

	return a
}
<|endoftext|>


<commit_before>package input

import (
	"encoding/json"
	"log"
	"net"
	"net/http"
	"strings"
)

type (
	/*Metric represents a single time series point

	Type is one of: histogram, counter, gauge or set

	SecondaryData represents values other than the primary value which should
	be treated as data and not metadata by the backend storage

	Tags are KV metadata
	*/
	Metric struct {
		Name          string
		Host          string
		Timestamp     float64
		Type          string
		Sampling      float64
		Value         float64
		SecondaryData map[string]interface{}
		Tags          map[string]string
	}

	//MetricBatch represent a batch of individual metrics that have been sent together
	MetricBatch struct {
		batch []Metric
	}

	//Event represents a single event instance
	Event struct {
		Name           string
		Text           string
		Host           string
		AggregationKey string
		Priority       string
		Timestamp      float64
		AlertType      string
		Tags           map[string]string
		SourceType     string
	}

	metricsHTTPHandler struct {
		metricsIn chan Metric
	}

	eventsHTTPHandler struct {
		eventsIn chan Event
	}

	metricsBatchHTTPHandler struct {
		metricsIn chan Metric
	}
)

//http handler function, unmarshalls json encoded metric into metric struct
func (handler *metricsHTTPHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	decoder := json.NewDecoder(r.Body)
	var receivedMetric Metric
	err := decoder.Decode(&receivedMetric)
	sourceAddress := r.RemoteAddr
	sourceIP, _, _ := net.SplitHostPort(r.RemoteAddr)
	log.Printf("Received metric from %s\n", sourceIP)

	if err == nil {
		parseMetric(receivedMetric, sourceIP, handler.metricsIn)
	} else {
		log.Println(err)
		log.Printf("Unable to decode metric from, %s", sourceAddress)
	}

	r.Body.Close()
}

func (handler *eventsHTTPHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	//unmarshall json encoded events into event struct
	decoder := json.NewDecoder(r.Body)
	var receivedEvent Event
	err := decoder.Decode(&receivedEvent)
	sourceAddress := r.RemoteAddr
	sourceIP := sourceAddress[0:strings.Index(r.RemoteAddr, ":")]

	if err == nil {
		if receivedEvent.Tags == nil {
			receivedEvent.Tags = make(map[string]string)
		}

		//append source address to metric
		receivedEvent.Tags["source"] = sourceIP
		handler.eventsIn <- receivedEvent
	} else {
		//if unable to parse the metric, drop it. This could be a problem for out of date clients.
		log.Println(err)
		log.Printf("Unable to decode event from, %s", sourceAddress)
	}

	r.Body.Close()
}

func (handler *metricsBatchHTTPHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	decoder := json.NewDecoder(r.Body)
	var receivedMetricBatch MetricBatch
	err := decoder.Decode(&receivedMetricBatch)
	sourceAddress := r.RemoteAddr
	sourceIP, _, _ := net.SplitHostPort(r.RemoteAddr)
	log.Printf("Received metric from %s\n", sourceIP)

	if err == nil {
		for i := range receivedMetricBatch.batch {
			parseMetric(receivedMetricBatch.batch[i], sourceIP, handler.metricsIn)
		}

	} else {
		log.Println(err)
		log.Printf("Unable to decode metric batch from, %s", sourceAddress)
	}

}

func parseMetric(receivedMetric Metric, sourceIP string, metricsIn chan Metric) {
	//add an aditional field specifing the host which forwarded aggregateD the metric
	//this might often be the same as the client specified host field but in situations
	//where the client is behind NAT, i.e many EVE clients this information is useful.
	if receivedMetric.SecondaryData == nil {
		receivedMetric.SecondaryData = make(map[string]interface{})
	}
	receivedMetric.SecondaryData["source"] = sourceIP

	//ensure that no secondary values are nil, clients should not
	//submit nil values but if they do they should not be sent to Influx
	for k := range receivedMetric.SecondaryData {
		if receivedMetric.SecondaryData[k] == nil {
			receivedMetric.SecondaryData[k] = 0.0
		}
	}

	metricsIn <- receivedMetric
}

//ServeHTTP exposes /events and /metrics and proceses JSON encoded events
func ServeHTTP(port string, metricsIn chan Metric, eventsIn chan Event) {
	server := http.NewServeMux()

	metricsHandler := new(metricsHTTPHandler)
	metricsHandler.metricsIn = metricsIn

	eventsHandler := new(eventsHTTPHandler)
	eventsHandler.eventsIn = eventsIn

	metricsBatchHandler := new(metricsBatchHTTPHandler)
	metricsBatchHandler.metricsIn = metricsIn

	server.Handle("/metrics", metricsHandler)
	server.Handle("/events", eventsHandler)
	server.Handle("/metrics_batch", metricsBatchHandler)

	log.Printf("Accepting json metrics on port %s", port)

	log.Fatal(http.ListenAndServe(":"+port, server))
}
<commit_msg>fixed batch processing<commit_after>package input

import (
	"encoding/json"
	"log"
	"net"
	"net/http"
	"strings"
)

type (
	/*Metric represents a single time series point

	Type is one of: histogram, counter, gauge or set

	SecondaryData represents values other than the primary value which should
	be treated as data and not metadata by the backend storage

	Tags are KV metadata
	*/
	Metric struct {
		Name          string
		Host          string
		Timestamp     float64
		Type          string
		Sampling      float64
		Value         float64
		SecondaryData map[string]interface{}
		Tags          map[string]string
	}

	//MetricBatch represent a batch of individual metrics that have been sent together
	MetricBatch struct {
		Batch []Metric
		Size  int32
	}

	//Event represents a single event instance
	Event struct {
		Name           string
		Text           string
		Host           string
		AggregationKey string
		Priority       string
		Timestamp      float64
		AlertType      string
		Tags           map[string]string
		SourceType     string
	}

	metricsHTTPHandler struct {
		metricsIn chan Metric
	}

	eventsHTTPHandler struct {
		eventsIn chan Event
	}

	metricsBatchHTTPHandler struct {
		metricsIn chan Metric
	}
)

//http handler function, unmarshalls json encoded metric into metric struct
func (handler *metricsHTTPHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	decoder := json.NewDecoder(r.Body)
	var receivedMetric Metric
	err := decoder.Decode(&receivedMetric)

	sourceAddress := r.RemoteAddr
	sourceIP, _, _ := net.SplitHostPort(r.RemoteAddr)
	log.Printf("Received metric from %s\n", sourceIP)

	if err == nil {
		parseMetric(receivedMetric, sourceIP, handler.metricsIn)
	} else {
		log.Println(err)
		log.Printf("Unable to decode metric from, %s", sourceAddress)
	}

	r.Body.Close()
}

func (handler *eventsHTTPHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	//unmarshall json encoded events into event struct
	decoder := json.NewDecoder(r.Body)
	var receivedEvent Event
	err := decoder.Decode(&receivedEvent)
	sourceAddress := r.RemoteAddr
	sourceIP := sourceAddress[0:strings.Index(r.RemoteAddr, ":")]

	if err == nil {
		if receivedEvent.Tags == nil {
			receivedEvent.Tags = make(map[string]string)
		}

		//append source address to metric
		receivedEvent.Tags["source"] = sourceIP
		handler.eventsIn <- receivedEvent
	} else {
		//if unable to parse the metric, drop it. This could be a problem for out of date clients.
		log.Println(err)
		log.Printf("Unable to decode event from, %s", sourceAddress)
	}

	r.Body.Close()
}

func (handler *metricsBatchHTTPHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	var receivedMetricBatch MetricBatch

	if r.Body == nil {
		http.Error(w, "Empty request body", 400)
		return
	}

	err := json.NewDecoder(r.Body).Decode(&receivedMetricBatch)
	if err != nil {
		http.Error(w, "Malformed batch", 400)
		log.Println("Malformed batch")
		return
	}

	sourceAddress := r.RemoteAddr
	sourceIP, _, _ := net.SplitHostPort(r.RemoteAddr)

	log.Printf("Received metric batch from %s\n", sourceIP)
	if err == nil {
		if len(receivedMetricBatch.Batch) == 0 {
			log.Printf("metric batch from %s is empty\n", sourceIP)
		} else {
			for i := range receivedMetricBatch.Batch {
				parseMetric(receivedMetricBatch.Batch[i], sourceIP, handler.metricsIn)
			}
		}
	} else {
		log.Println(err)
		log.Printf("Unable to decode metric batch from, %s", sourceAddress)
	}

}

func parseMetric(receivedMetric Metric, sourceIP string, metricsIn chan Metric) {
	//add an aditional field specifing the host which forwarded aggregateD the metric
	//this might often be the same as the client specified host field but in situations
	//where the client is behind NAT, i.e many EVE clients this information is useful.

	if receivedMetric.SecondaryData == nil {
		receivedMetric.SecondaryData = make(map[string]interface{})
	}
	receivedMetric.SecondaryData["source"] = sourceIP

	//ensure that no secondary values are nil, clients should not
	//submit nil values but if they do they should not be sent to Influx
	for k := range receivedMetric.SecondaryData {
		if receivedMetric.SecondaryData[k] == nil {
			receivedMetric.SecondaryData[k] = 0.0
		}
	}
	metricsIn <- receivedMetric
}

//ServeHTTP exposes /events and /metrics and proceses JSON encoded events
func ServeHTTP(port string, metricsIn chan Metric, eventsIn chan Event) {
	server := http.NewServeMux()

	metricsHandler := new(metricsHTTPHandler)
	metricsHandler.metricsIn = metricsIn

	eventsHandler := new(eventsHTTPHandler)
	eventsHandler.eventsIn = eventsIn

	metricsBatchHandler := new(metricsBatchHTTPHandler)
	metricsBatchHandler.metricsIn = metricsIn

	server.Handle("/metrics", metricsHandler)
	server.Handle("/events", eventsHandler)
	server.Handle("/metrics_batch", metricsBatchHandler)

	log.Printf("Accepting json metrics on port %s", port)

	log.Fatal(http.ListenAndServe(":"+port, server))
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/anchor/chevalier"
	"github.com/anchor/zmqutil"
	zmq "github.com/pebbe/zmq4"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strings"
)

func queryES(origin string, req *chevalier.SourceRequest, host string) {
	engine := chevalier.NewQueryEngine(host, "chevalier", "datasource")
	results, err := engine.GetSources(origin, req)
	if err != nil {
		log.Println("Search error: %v", err)
	}
	if err != nil {
		log.Fatal(err)
	}
	for _, source := range results.GetSources() {
		fmt.Println(source)
	}
}

func queryChevalier(origin string, req *chevalier.SourceRequest, endpoint string) {
	sock, err := zmq.NewSocket(zmq.REQ)
	if err != nil {
		log.Fatal(err)
	}
	err = sock.Connect(endpoint)
	if err != nil {
		log.Fatal(err)
	}
	packet, err := chevalier.MarshalSourceRequest(req)
	if err != nil {
		log.Fatal(err)
	}
	_, err = zmqutil.RetrySendMessage(sock, origin, packet, 0)
	response, err := zmqutil.RetryRecvBytes(sock, 0)
	if err != nil {
		log.Fatal(err)
	}
	burst, err := chevalier.UnmarshalSourceBurst(response)
	if err != nil {
		log.Fatal(err)
	}
	for _, source := range burst.GetSources() {
		fmt.Println(source)
	}
}

func main() {
	esHost := flag.String("host", "localhost", "Elasticsearch host to connect to")
	protobuf := flag.Bool("protobuf", false, "Read a SourceRequest from stdin rather than accepting field:value pairs on the command line.")
	es := flag.Bool("es", false, "Read from Elasticsearch directly rather than chevalier.")
	startPage := flag.Int("start-page", 0, "Obtain results from this page.")
	pageSize := flag.Int("page-size", 0, "Number of results per page.")
	endpoint := flag.String("endpoint", "tcp://127.0.0.1:6283", "Chevalier endpoint (as a ZMQ URI).")
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
		fmt.Fprintf(os.Stderr, "%s <origin> <field:value> [field:value ...] [args]\n", os.Args[0])
		flag.PrintDefaults()
	}
	flag.Parse()
	if flag.NArg() < 2 {
		flag.Usage()
		os.Exit(1)
	}
	origin := flag.Arg(0)
	var req *chevalier.SourceRequest
	if *protobuf {
		reader := io.Reader(os.Stdin)
		packet, err := ioutil.ReadAll(reader)
		if err != nil {
			log.Fatal("Could not read from stdin: %v", err)
		}
		req, err = chevalier.UnmarshalSourceRequest(packet)
		if err != nil {
			log.Fatal("Could not unmarshal request: %v", err)
		}
	} else {
		tags := make([]*chevalier.SourceRequest_Tag, flag.NArg() - 1)
		for i, arg := range flag.Args()[1:] {
			pair := strings.Split(arg, ":")
			if len(pair) < 2 {
				log.Fatal("Could not parse %v: must be a 'field:value' pair.")
			}
			tags[i] = chevalier.NewSourceRequestTag(pair[0], pair[1])
		}
		req = chevalier.NewSourceRequest(tags)
		if *startPage > 0 {
			page := int64(*startPage)
			req.StartPage = &page
		}
		if *pageSize > 0 {
			size := int64(*pageSize)
			req.SourcesPerPage = &size
		}
	}
	if *es {
		queryES(origin, req, *esHost)
	} else {
		queryChevalier(origin, req, *endpoint)
	}
}
<commit_msg>Add options to request_sources for json and raw output<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/anchor/chevalier"
	"github.com/anchor/zmqutil"
	zmq "github.com/pebbe/zmq4"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strings"
	"encoding/json"
)

func queryES(origin string, req *chevalier.SourceRequest, host string) *chevalier.DataSourceBurst {
	engine := chevalier.NewQueryEngine(host, "chevalier", "datasource")
	results, err := engine.GetSources(origin, req)
	if err != nil {
		log.Println("Search error: %v", err)
	}
	if err != nil {
		log.Fatal(err)
	}
	return results
}

func queryChevalier(origin string, req *chevalier.SourceRequest, endpoint string) *chevalier.DataSourceBurst {
	sock, err := zmq.NewSocket(zmq.REQ)
	if err != nil {
		log.Fatal(err)
	}
	err = sock.Connect(endpoint)
	if err != nil {
		log.Fatal(err)
	}
	packet, err := chevalier.MarshalSourceRequest(req)
	if err != nil {
		log.Fatal(err)
	}
	_, err = zmqutil.RetrySendMessage(sock, origin, packet, 0)
	response, err := zmqutil.RetryRecvBytes(sock, 0)
	if err != nil {
		log.Fatal(err)
	}
	burst, err := chevalier.UnmarshalSourceBurst(response)
	if err != nil {
		log.Fatal(err)
	}
	return burst
}

func main() {
	esHost := flag.String("host", "localhost", "Elasticsearch host to connect to")
	protobuf := flag.Bool("protobuf", false, "Read a SourceRequest from stdin rather than accepting field:value pairs on the command line.")
	es := flag.Bool("es", false, "Read from Elasticsearch directly rather than chevalier.")
	startPage := flag.Int("start-page", 0, "Obtain results from this page.")
	pageSize := flag.Int("page-size", 0, "Number of results per page.")
	endpoint := flag.String("endpoint", "tcp://127.0.0.1:6283", "Chevalier endpoint (as a ZMQ URI).")
	outputJson := flag.Bool("output-json", false, "Output results as JSON.")
	outputRaw := flag.Bool("output-raw", false, "Output results as raw protobufs (a DataSourceBurst object).")
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
		fmt.Fprintf(os.Stderr, "%s <origin> <field:value> [field:value ...] [args]\n", os.Args[0])
		flag.PrintDefaults()
	}
	flag.Parse()
	if flag.NArg() < 2 {
		flag.Usage()
		os.Exit(1)
	}
	origin := flag.Arg(0)
	var req *chevalier.SourceRequest
	if *protobuf {
		reader := io.Reader(os.Stdin)
		packet, err := ioutil.ReadAll(reader)
		if err != nil {
			log.Fatal("Could not read from stdin: %v", err)
		}
		req, err = chevalier.UnmarshalSourceRequest(packet)
		if err != nil {
			log.Fatal("Could not unmarshal request: %v", err)
		}
	} else {
		tags := make([]*chevalier.SourceRequest_Tag, flag.NArg() - 1)
		for i, arg := range flag.Args()[1:] {
			pair := strings.Split(arg, ":")
			if len(pair) < 2 {
				log.Fatal("Could not parse %v: must be a 'field:value' pair.")
			}
			tags[i] = chevalier.NewSourceRequestTag(pair[0], pair[1])
		}
		req = chevalier.NewSourceRequest(tags)
		if *startPage > 0 {
			page := int64(*startPage)
			req.StartPage = &page
		}
		if *pageSize > 0 {
			size := int64(*pageSize)
			req.SourcesPerPage = &size
		}
	}
	var burst *chevalier.DataSourceBurst
	if *es {
		burst = queryES(origin, req, *esHost)
	} else {
		burst = queryChevalier(origin, req, *endpoint)
	}
	var b []byte
	var err error
	if *outputJson {
		b, err = json.Marshal(burst)
	} else if *outputRaw {
		b, err = chevalier.MarshalSourceBurst(burst)
	}
	if err != nil {
		fmt.Fprintf(os.Stderr, "Could not unmarshal burst: %v", err)
	} else {
		os.Stdout.Write(b)
	}
}
<|endoftext|>


<commit_before><commit_msg>logging<commit_after><|endoftext|>


<commit_before>package netlify

import (
	"crypto/sha1"
	"encoding/hex"
	"errors"
	"io/ioutil"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/cenkalti/backoff"
)

const MaxFilesForSyncDeploy = 1000
const PreProcessingTimeout = time.Minute * 5

// Deploy represents a specific deploy of a site
type Deploy struct {
	Id     string `json:"id"`
	SiteId string `json:"site_id"`
	UserId string `json:"user_id"`

	// State of the deploy (uploading/uploaded/processing/ready/error)
	State string `json:"state"`

	// Cause of error if State is "error"
	ErrorMessage string `json:"error_message"`

	// Shas of files that needs to be uploaded before the deploy is ready
	Required []string `json:"required"`

	DeployUrl     string `json:"deploy_url"`
	SiteUrl       string `json:"url"`
	ScreenshotUrl string `json:"screenshot_url"`

	CreatedAt Timestamp `json:"created_at"`
	UpdatedAt Timestamp `json:"updated_at"`

	Branch    string `json:"branch,omitempty"`
	CommitRef string `json:"commit_ref,omitempty"`

	client *Client
	logger *logrus.Entry
}

func (d Deploy) log() *logrus.Entry {
	if d.logger == nil {
		d.logger = d.client.log.WithFields(logrus.Fields{
			"function": "deploy",
			"id":       d.Id,
			"site_id":  d.SiteId,
			"user_id":  d.UserId,
		})
	}

	return d.logger.WithField("state", d.State)
}

// DeploysService is used to access all Deploy related API methods
type DeploysService struct {
	site   *Site
	client *Client
}

type uploadError struct {
	err   error
	mutex *sync.Mutex
}

func (u *uploadError) Set(err error) {
	if err != nil {
		u.mutex.Lock()
		defer u.mutex.Unlock()
		if u.err == nil {
			u.err = err
		}
	}
}

func (u *uploadError) Empty() bool {
	u.mutex.Lock()
	defer u.mutex.Unlock()
	return u.err == nil
}

type deployFiles struct {
	Files     *map[string]string `json:"files"`
	Async     bool               `json:"async"`
	Branch    string             `json:"branch,omitempty"`
	CommitRef string             `json:"commit_ref,omitempty"`
}

func (s *DeploysService) apiPath() string {
	if s.site != nil {
		return path.Join(s.site.apiPath(), "deploys")
	}
	return "/deploys"
}

// Create a new deploy
//
// Example: site.Deploys.Create("/path/to/site-dir", true)
// If the target is a zip file, it must have the extension .zip
func (s *DeploysService) Create(dirOrZip string) (*Deploy, *Response, error) {
	return s.create(dirOrZip, false)
}

// CreateDraft a new draft deploy. Draft deploys will be uploaded and processed, but
// won't affect the active deploy for a site.
func (s *DeploysService) CreateDraft(dirOrZip string) (*Deploy, *Response, error) {
	return s.create(dirOrZip, true)
}

func (s *DeploysService) create(dirOrZip string, draft bool) (*Deploy, *Response, error) {
	if s.site == nil {
		return nil, nil, errors.New("You can only create a new deploy for an existing site (site.Deploys.Create(dirOrZip)))")
	}

	params := url.Values{}
	if draft {
		params["draft"] = []string{"true"}
	}
	options := &RequestOptions{QueryParams: &params}
	deploy := &Deploy{client: s.client}
	resp, err := s.client.Request("POST", s.apiPath(), options, deploy)

	if err != nil {
		return deploy, resp, err
	}

	resp, err = deploy.Deploy(dirOrZip)
	return deploy, resp, err
}

// List all deploys. Takes ListOptions to control pagination.
func (s *DeploysService) List(options *ListOptions) ([]Deploy, *Response, error) {
	deploys := new([]Deploy)

	reqOptions := &RequestOptions{QueryParams: options.toQueryParamsMap()}

	resp, err := s.client.Request("GET", s.apiPath(), reqOptions, deploys)

	for _, deploy := range *deploys {
		deploy.client = s.client
	}

	return *deploys, resp, err
}

// Get a specific deploy.
func (d *DeploysService) Get(id string) (*Deploy, *Response, error) {
	deploy := &Deploy{Id: id, client: d.client}
	resp, err := deploy.Reload()

	return deploy, resp, err
}

func (deploy *Deploy) apiPath() string {
	return path.Join("/deploys", deploy.Id)
}

func (deploy *Deploy) Deploy(dirOrZip string) (*Response, error) {
	if strings.HasSuffix(dirOrZip, ".zip") {
		return deploy.deployZip(dirOrZip)
	} else {
		return deploy.deployDir(dirOrZip)
	}
}

// Reload a deploy from the API
func (deploy *Deploy) Reload() (*Response, error) {
	if deploy.Id == "" {
		return nil, errors.New("Cannot fetch deploy without an ID")
	}
	return deploy.client.Request("GET", deploy.apiPath(), nil, deploy)
}

// Restore an old deploy. Sets the deploy as the active deploy for a site
func (deploy *Deploy) Restore() (*Response, error) {
	return deploy.client.Request("POST", path.Join(deploy.apiPath(), "restore"), nil, deploy)
}

// Alias for restore. Published a specific deploy.
func (deploy *Deploy) Publish() (*Response, error) {
	return deploy.Restore()
}

func (deploy *Deploy) uploadFile(dir, path string, sharedError *uploadError) error {
	if !sharedError.Empty() {
		return errors.New("Canceled because upload has already failed")
	}

	log := deploy.log().WithFields(logrus.Fields{
		"dir":  dir,
		"path": path,
	})

	log.Infof("Uploading file: %v", path)
	file, err := os.Open(filepath.Join(dir, path))
	defer file.Close()

	if err != nil {
		log.Warnf("Error opening file %v: %v", path, err)
		return err
	}

	info, err := file.Stat()

	if err != nil {
		log.Warnf("Error getting file size %v: %v", path, err)
		return err
	}

	options := &RequestOptions{
		RawBody:       file,
		RawBodyLength: info.Size(),
		Headers:       &map[string]string{"Content-Type": "application/octet-stream"},
	}

	fileUrl, err := url.Parse(path)
	if err != nil {
		log.Warnf("Error parsing url %v: %v", path, err)
		return err
	}

	resp, err := deploy.client.Request("PUT", filepath.Join(deploy.apiPath(), "files", fileUrl.Path), options, nil)
	if resp != nil && resp.Response != nil && resp.Body != nil {
		resp.Body.Close()
	}
	if err != nil {
		log.Warnf("Error while uploading %v: %v", path, err)
		return err
	}

	log.Infof("Finished uploading file: %s", path)
	return err
}

// deployDir scans the given directory and deploys the files
// that have changed on Netlify.
func (deploy *Deploy) deployDir(dir string) (*Response, error) {
	return deploy.DeployDirWithGitInfo(dir, "", "")
}

// DeployDirWithGitInfo scans the given directory and deploys the files
// that have changed on Netlify.
//
// This function allows you to supply git information about the deploy
// when it hasn't been set previously be a Continuous Deployment process.
func (deploy *Deploy) DeployDirWithGitInfo(dir, branch, commitRef string) (*Response, error) {
	files := map[string]string{}
	log := deploy.log().WithFields(logrus.Fields{
		"dir":        dir,
		"branch":     branch,
		"commit_ref": commitRef,
	})
	defer log.Infof("Finished deploying directory %s", dir)

	log.Infof("Starting deploy of directory %s", dir)
	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.IsDir() == false && info.Mode().IsRegular() {
			rel, err := filepath.Rel(dir, path)
			if err != nil {
				return err
			}

			if ignoreFile(rel) {
				return nil
			}

			sha := sha1.New()
			data, err := ioutil.ReadFile(path)

			if err != nil {
				return err
			}

			sha.Write(data)

			files[rel] = hex.EncodeToString(sha.Sum(nil))
		}

		return nil
	})
	if err != nil {
		log.WithError(err).Warn("Failed to walk directory structure")
		return nil, err
	}

	fileOptions := &deployFiles{
		Files:     &files,
		Branch:    branch,
		CommitRef: commitRef,
	}

	if len(files) > MaxFilesForSyncDeploy {
		log.Debugf("More files than sync can deploy %d vs %d", len(files), MaxFilesForSyncDeploy)
		fileOptions.Async = true
	}

	options := &RequestOptions{
		JsonBody: fileOptions,
	}

	log.Debug("Starting to do PUT to origin")
	resp, err := deploy.client.Request("PUT", deploy.apiPath(), options, deploy)
	if err != nil {
		return resp, err
	}

	if len(files) > MaxFilesForSyncDeploy {
		start := time.Now()
		log.Debug("Starting to poll for the deploy to get into ready || prepared state")
		for {
			resp, err := deploy.client.Request("GET", deploy.apiPath(), nil, deploy)
			if err != nil {
				log.WithError(err).Warnf("Error fetching deploy, waiting for 5 seconds before retry: %v", err)
				time.Sleep(5 * time.Second)
			}
			resp.Body.Close()

			log.Debugf("Deploy state: %v\n", deploy.State)
			if deploy.State == "prepared" || deploy.State == "ready" {
				break
			}
			if deploy.State == "error" {
				log.Warnf("deploy is in state error")
				return resp, errors.New("Error: preprocessing deploy failed")
			}
			if start.Add(PreProcessingTimeout).Before(time.Now()) {
				log.Warnf("Deploy timed out waiting for preprocessing")
				return resp, errors.New("Error: preprocessing deploy timed out")
			}
			log.Debug("Waiting for 2 seconds to retry getting deploy")
			time.Sleep(2 * time.Second)
		}
	}

	lookup := map[string]bool{}

	for _, sha := range deploy.Required {
		lookup[sha] = true
	}

	log.Infof("Going to deploy the %d required files", len(lookup))

	// Use a channel as a semaphore to limit # of parallel uploads
	sem := make(chan int, deploy.client.MaxConcurrentUploads)
	var wg sync.WaitGroup

	sharedErr := uploadError{err: nil, mutex: &sync.Mutex{}}
	for path, sha := range files {
		if lookup[sha] == true && sharedErr.Empty() {
			wg.Add(1)
			sem <- 1
			go func(path string) {
				defer func() {
					<-sem
					wg.Done()
				}()
				log.Debugf("Starting to upload %s/%s", path, sha)
				if !sharedErr.Empty() {
					return
				}

				b := backoff.NewExponentialBackOff()
				b.MaxElapsedTime = 2 * time.Minute
				err := backoff.Retry(func() error { return deploy.uploadFile(dir, path, &sharedErr) }, b)
				if err != nil {
					log.WithError(err).Warn("Error while uploading file %s: %v", path, err)
					sharedErr.Set(err)
				}
			}(path)
		}
	}

	log.Debugf("Waiting for required files to upload")
	wg.Wait()

	if !sharedErr.Empty() {
		return resp, sharedErr.err
	}

	return resp, err
}

// deployZip uploads a Zip file to Netlify and deploys the files
// that have changed.
func (deploy *Deploy) deployZip(zip string) (*Response, error) {
	log := deploy.log().WithFields(logrus.Fields{
		"function": "zip",
		"zip_path": zip,
	})
	log.Infof("Starting to deploy zip file %s", zip)
	zipPath, err := filepath.Abs(zip)
	if err != nil {
		return nil, err
	}

	log.Debugf("Opening zip file at %s", zipPath)
	zipFile, err := os.Open(zipPath)
	if err != nil {
		return nil, err
	}
	defer zipFile.Close()

	info, err := zipFile.Stat()
	if err != nil {
		return nil, err
	}

	log.WithFields(logrus.Fields{
		"name": info.Name(),
		"size": info.Size(),
		"mode": info.Mode(),
	}).Debugf("Opened file %s of %s bytes", info.Name(), info.Size())

	options := &RequestOptions{
		RawBody:       zipFile,
		RawBodyLength: info.Size(),
		Headers:       &map[string]string{"Content-Type": "application/zip"},
	}

	log.Debug("Excuting PUT request for zip file")
	resp, err := deploy.client.Request("PUT", deploy.apiPath(), options, deploy)
	if err != nil {
		log.WithError(err).Warn("Error while uploading zip file")
	}

	log.Info("Finished uploading zip file")
	return resp, err
}

func (deploy *Deploy) WaitForReady(timeout time.Duration) error {
	if deploy.State == "ready" {
		return nil
	}

	if timeout == 0 {
		timeout = defaultTimeout
	}

	timedOut := false
	time.AfterFunc(timeout*time.Second, func() {
		timedOut = true
	})

	done := make(chan error)

	go func() {
		for {
			time.Sleep(1 * time.Second)

			if timedOut {
				done <- errors.New("Timeout while waiting for processing")
				break
			}

			_, err := deploy.Reload()
			if err != nil || (deploy.State == "ready") {
				done <- err
				break
			}
		}
	}()

	err := <-done
	return err
}

func ignoreFile(rel string) bool {
	if strings.HasPrefix(rel, ".") || strings.Contains(rel, "/.") || strings.HasPrefix(rel, "__MACOS") {
		if strings.HasPrefix(rel, ".well-known/") {
			return false
		}
		return true
	}
	return false
}
<commit_msg>formatting is hard some times<commit_after>package netlify

import (
	"crypto/sha1"
	"encoding/hex"
	"errors"
	"io/ioutil"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/cenkalti/backoff"
)

const MaxFilesForSyncDeploy = 1000
const PreProcessingTimeout = time.Minute * 5

// Deploy represents a specific deploy of a site
type Deploy struct {
	Id     string `json:"id"`
	SiteId string `json:"site_id"`
	UserId string `json:"user_id"`

	// State of the deploy (uploading/uploaded/processing/ready/error)
	State string `json:"state"`

	// Cause of error if State is "error"
	ErrorMessage string `json:"error_message"`

	// Shas of files that needs to be uploaded before the deploy is ready
	Required []string `json:"required"`

	DeployUrl     string `json:"deploy_url"`
	SiteUrl       string `json:"url"`
	ScreenshotUrl string `json:"screenshot_url"`

	CreatedAt Timestamp `json:"created_at"`
	UpdatedAt Timestamp `json:"updated_at"`

	Branch    string `json:"branch,omitempty"`
	CommitRef string `json:"commit_ref,omitempty"`

	client *Client
	logger *logrus.Entry
}

func (d Deploy) log() *logrus.Entry {
	if d.logger == nil {
		d.logger = d.client.log.WithFields(logrus.Fields{
			"function": "deploy",
			"id":       d.Id,
			"site_id":  d.SiteId,
			"user_id":  d.UserId,
		})
	}

	return d.logger.WithField("state", d.State)
}

// DeploysService is used to access all Deploy related API methods
type DeploysService struct {
	site   *Site
	client *Client
}

type uploadError struct {
	err   error
	mutex *sync.Mutex
}

func (u *uploadError) Set(err error) {
	if err != nil {
		u.mutex.Lock()
		defer u.mutex.Unlock()
		if u.err == nil {
			u.err = err
		}
	}
}

func (u *uploadError) Empty() bool {
	u.mutex.Lock()
	defer u.mutex.Unlock()
	return u.err == nil
}

type deployFiles struct {
	Files     *map[string]string `json:"files"`
	Async     bool               `json:"async"`
	Branch    string             `json:"branch,omitempty"`
	CommitRef string             `json:"commit_ref,omitempty"`
}

func (s *DeploysService) apiPath() string {
	if s.site != nil {
		return path.Join(s.site.apiPath(), "deploys")
	}
	return "/deploys"
}

// Create a new deploy
//
// Example: site.Deploys.Create("/path/to/site-dir", true)
// If the target is a zip file, it must have the extension .zip
func (s *DeploysService) Create(dirOrZip string) (*Deploy, *Response, error) {
	return s.create(dirOrZip, false)
}

// CreateDraft a new draft deploy. Draft deploys will be uploaded and processed, but
// won't affect the active deploy for a site.
func (s *DeploysService) CreateDraft(dirOrZip string) (*Deploy, *Response, error) {
	return s.create(dirOrZip, true)
}

func (s *DeploysService) create(dirOrZip string, draft bool) (*Deploy, *Response, error) {
	if s.site == nil {
		return nil, nil, errors.New("You can only create a new deploy for an existing site (site.Deploys.Create(dirOrZip)))")
	}

	params := url.Values{}
	if draft {
		params["draft"] = []string{"true"}
	}
	options := &RequestOptions{QueryParams: &params}
	deploy := &Deploy{client: s.client}
	resp, err := s.client.Request("POST", s.apiPath(), options, deploy)

	if err != nil {
		return deploy, resp, err
	}

	resp, err = deploy.Deploy(dirOrZip)
	return deploy, resp, err
}

// List all deploys. Takes ListOptions to control pagination.
func (s *DeploysService) List(options *ListOptions) ([]Deploy, *Response, error) {
	deploys := new([]Deploy)

	reqOptions := &RequestOptions{QueryParams: options.toQueryParamsMap()}

	resp, err := s.client.Request("GET", s.apiPath(), reqOptions, deploys)

	for _, deploy := range *deploys {
		deploy.client = s.client
	}

	return *deploys, resp, err
}

// Get a specific deploy.
func (d *DeploysService) Get(id string) (*Deploy, *Response, error) {
	deploy := &Deploy{Id: id, client: d.client}
	resp, err := deploy.Reload()

	return deploy, resp, err
}

func (deploy *Deploy) apiPath() string {
	return path.Join("/deploys", deploy.Id)
}

func (deploy *Deploy) Deploy(dirOrZip string) (*Response, error) {
	if strings.HasSuffix(dirOrZip, ".zip") {
		return deploy.deployZip(dirOrZip)
	} else {
		return deploy.deployDir(dirOrZip)
	}
}

// Reload a deploy from the API
func (deploy *Deploy) Reload() (*Response, error) {
	if deploy.Id == "" {
		return nil, errors.New("Cannot fetch deploy without an ID")
	}
	return deploy.client.Request("GET", deploy.apiPath(), nil, deploy)
}

// Restore an old deploy. Sets the deploy as the active deploy for a site
func (deploy *Deploy) Restore() (*Response, error) {
	return deploy.client.Request("POST", path.Join(deploy.apiPath(), "restore"), nil, deploy)
}

// Alias for restore. Published a specific deploy.
func (deploy *Deploy) Publish() (*Response, error) {
	return deploy.Restore()
}

func (deploy *Deploy) uploadFile(dir, path string, sharedError *uploadError) error {
	if !sharedError.Empty() {
		return errors.New("Canceled because upload has already failed")
	}

	log := deploy.log().WithFields(logrus.Fields{
		"dir":  dir,
		"path": path,
	})

	log.Infof("Uploading file: %v", path)
	file, err := os.Open(filepath.Join(dir, path))
	defer file.Close()

	if err != nil {
		log.Warnf("Error opening file %v: %v", path, err)
		return err
	}

	info, err := file.Stat()

	if err != nil {
		log.Warnf("Error getting file size %v: %v", path, err)
		return err
	}

	options := &RequestOptions{
		RawBody:       file,
		RawBodyLength: info.Size(),
		Headers:       &map[string]string{"Content-Type": "application/octet-stream"},
	}

	fileUrl, err := url.Parse(path)
	if err != nil {
		log.Warnf("Error parsing url %v: %v", path, err)
		return err
	}

	resp, err := deploy.client.Request("PUT", filepath.Join(deploy.apiPath(), "files", fileUrl.Path), options, nil)
	if resp != nil && resp.Response != nil && resp.Body != nil {
		resp.Body.Close()
	}
	if err != nil {
		log.Warnf("Error while uploading %v: %v", path, err)
		return err
	}

	log.Infof("Finished uploading file: %s", path)
	return err
}

// deployDir scans the given directory and deploys the files
// that have changed on Netlify.
func (deploy *Deploy) deployDir(dir string) (*Response, error) {
	return deploy.DeployDirWithGitInfo(dir, "", "")
}

// DeployDirWithGitInfo scans the given directory and deploys the files
// that have changed on Netlify.
//
// This function allows you to supply git information about the deploy
// when it hasn't been set previously be a Continuous Deployment process.
func (deploy *Deploy) DeployDirWithGitInfo(dir, branch, commitRef string) (*Response, error) {
	files := map[string]string{}
	log := deploy.log().WithFields(logrus.Fields{
		"dir":        dir,
		"branch":     branch,
		"commit_ref": commitRef,
	})
	defer log.Infof("Finished deploying directory %s", dir)

	log.Infof("Starting deploy of directory %s", dir)
	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.IsDir() == false && info.Mode().IsRegular() {
			rel, err := filepath.Rel(dir, path)
			if err != nil {
				return err
			}

			if ignoreFile(rel) {
				return nil
			}

			sha := sha1.New()
			data, err := ioutil.ReadFile(path)

			if err != nil {
				return err
			}

			sha.Write(data)

			files[rel] = hex.EncodeToString(sha.Sum(nil))
		}

		return nil
	})
	if err != nil {
		log.WithError(err).Warn("Failed to walk directory structure")
		return nil, err
	}

	fileOptions := &deployFiles{
		Files:     &files,
		Branch:    branch,
		CommitRef: commitRef,
	}

	if len(files) > MaxFilesForSyncDeploy {
		log.Debugf("More files than sync can deploy %d vs %d", len(files), MaxFilesForSyncDeploy)
		fileOptions.Async = true
	}

	options := &RequestOptions{
		JsonBody: fileOptions,
	}

	log.Debug("Starting to do PUT to origin")
	resp, err := deploy.client.Request("PUT", deploy.apiPath(), options, deploy)
	if err != nil {
		return resp, err
	}

	if len(files) > MaxFilesForSyncDeploy {
		start := time.Now()
		log.Debug("Starting to poll for the deploy to get into ready || prepared state")
		for {
			resp, err := deploy.client.Request("GET", deploy.apiPath(), nil, deploy)
			if err != nil {
				log.WithError(err).Warnf("Error fetching deploy, waiting for 5 seconds before retry: %v", err)
				time.Sleep(5 * time.Second)
			}
			resp.Body.Close()

			log.Debugf("Deploy state: %v\n", deploy.State)
			if deploy.State == "prepared" || deploy.State == "ready" {
				break
			}
			if deploy.State == "error" {
				log.Warnf("deploy is in state error")
				return resp, errors.New("Error: preprocessing deploy failed")
			}
			if start.Add(PreProcessingTimeout).Before(time.Now()) {
				log.Warnf("Deploy timed out waiting for preprocessing")
				return resp, errors.New("Error: preprocessing deploy timed out")
			}
			log.Debug("Waiting for 2 seconds to retry getting deploy")
			time.Sleep(2 * time.Second)
		}
	}

	lookup := map[string]bool{}

	for _, sha := range deploy.Required {
		lookup[sha] = true
	}

	log.Infof("Going to deploy the %d required files", len(lookup))

	// Use a channel as a semaphore to limit # of parallel uploads
	sem := make(chan int, deploy.client.MaxConcurrentUploads)
	var wg sync.WaitGroup

	sharedErr := uploadError{err: nil, mutex: &sync.Mutex{}}
	for path, sha := range files {
		if lookup[sha] == true && sharedErr.Empty() {
			wg.Add(1)
			sem <- 1
			go func(path string) {
				defer func() {
					<-sem
					wg.Done()
				}()
				log.Debugf("Starting to upload %s/%s", path, sha)
				if !sharedErr.Empty() {
					return
				}

				b := backoff.NewExponentialBackOff()
				b.MaxElapsedTime = 2 * time.Minute
				err := backoff.Retry(func() error { return deploy.uploadFile(dir, path, &sharedErr) }, b)
				if err != nil {
					log.WithError(err).Warnf("Error while uploading file %s: %v", path, err)
					sharedErr.Set(err)
				}
			}(path)
		}
	}

	log.Debugf("Waiting for required files to upload")
	wg.Wait()

	if !sharedErr.Empty() {
		return resp, sharedErr.err
	}

	return resp, err
}

// deployZip uploads a Zip file to Netlify and deploys the files
// that have changed.
func (deploy *Deploy) deployZip(zip string) (*Response, error) {
	log := deploy.log().WithFields(logrus.Fields{
		"function": "zip",
		"zip_path": zip,
	})
	log.Infof("Starting to deploy zip file %s", zip)
	zipPath, err := filepath.Abs(zip)
	if err != nil {
		return nil, err
	}

	log.Debugf("Opening zip file at %s", zipPath)
	zipFile, err := os.Open(zipPath)
	if err != nil {
		return nil, err
	}
	defer zipFile.Close()

	info, err := zipFile.Stat()
	if err != nil {
		return nil, err
	}

	log.WithFields(logrus.Fields{
		"name": info.Name(),
		"size": info.Size(),
		"mode": info.Mode(),
	}).Debugf("Opened file %s of %s bytes", info.Name(), info.Size())

	options := &RequestOptions{
		RawBody:       zipFile,
		RawBodyLength: info.Size(),
		Headers:       &map[string]string{"Content-Type": "application/zip"},
	}

	log.Debug("Excuting PUT request for zip file")
	resp, err := deploy.client.Request("PUT", deploy.apiPath(), options, deploy)
	if err != nil {
		log.WithError(err).Warn("Error while uploading zip file")
	}

	log.Info("Finished uploading zip file")
	return resp, err
}

func (deploy *Deploy) WaitForReady(timeout time.Duration) error {
	if deploy.State == "ready" {
		return nil
	}

	if timeout == 0 {
		timeout = defaultTimeout
	}

	timedOut := false
	time.AfterFunc(timeout*time.Second, func() {
		timedOut = true
	})

	done := make(chan error)

	go func() {
		for {
			time.Sleep(1 * time.Second)

			if timedOut {
				done <- errors.New("Timeout while waiting for processing")
				break
			}

			_, err := deploy.Reload()
			if err != nil || (deploy.State == "ready") {
				done <- err
				break
			}
		}
	}()

	err := <-done
	return err
}

func ignoreFile(rel string) bool {
	if strings.HasPrefix(rel, ".") || strings.Contains(rel, "/.") || strings.HasPrefix(rel, "__MACOS") {
		if strings.HasPrefix(rel, ".well-known/") {
			return false
		}
		return true
	}
	return false
}
<|endoftext|>


<commit_before>// Copyright 2014 ISRG.  All rights reserved
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

package core

import (
	"crypto"
	"crypto/ecdsa"
	"crypto/rsa"
	"fmt"
	blog "github.com/letsencrypt/boulder/log"
	"math/big"
	"reflect"
	"sync"
)

// To generate, run: primes 2 752 | tr '\n' ,
var smallPrimes = []int64{
	2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131, 137, 139, 149, 151, 157, 163, 167, 173, 179, 181, 191, 193, 197, 199, 211, 223, 227, 229, 233, 239, 241, 251, 257, 263, 269, 271, 277, 281, 283, 293, 307, 311, 313, 317, 331, 337, 347, 349, 353, 359, 367, 373, 379, 383, 389, 397, 401, 409, 419, 421, 431, 433, 439, 443, 449, 457, 461, 463, 467, 479, 487, 491, 499, 503, 509, 521, 523, 541, 547, 557, 563, 569, 571, 577, 587, 593, 599, 601, 607, 613, 617, 619, 631, 641, 643, 647, 653, 659, 661, 673, 677, 683, 691, 701, 709, 719, 727, 733, 739, 743, 751,
}

// singleton defines the object of a Singleton pattern
type singleton struct {
	once sync.Once
	// The big.Int form of these primes, memoized to save conversion time.
	smallPrimesBigInts []*big.Int
}

var _Singleton singleton

// GoodKey returns true iff the key is acceptable for both TLS use and account
// key use (our requirements are the same for either one), according to basic
// strength and algorithm checking.
// TODO: Support JsonWebKeys once go-jose migration is done.
func GoodKey(key crypto.PublicKey) bool {
	log := blog.GetAuditLogger()
	switch t := key.(type) {
	case rsa.PublicKey:
		return GoodKeyRSA(t)
	case *rsa.PublicKey:
		return GoodKeyRSA(*t)
	case ecdsa.PublicKey:
		return GoodKeyECDSA(t)
	case *ecdsa.PublicKey:
		return GoodKeyECDSA(*t)
	default:
		log.Debug(fmt.Sprintf("Unknown key type %s", reflect.TypeOf(key)))
		return false
	}
}

func GoodKeyECDSA(key ecdsa.PublicKey) bool {
	log := blog.GetAuditLogger()
	log.Debug(fmt.Sprintf("ECDSA keys not yet supported."))
	return false
}

func GoodKeyRSA(key rsa.PublicKey) bool {
	log := blog.GetAuditLogger()
	// Baseline Requirements Appendix A
	// Modulus must be >= 2048 bits
	modulus := key.N
	if modulus.BitLen() < 2048 {
		log.Debug(fmt.Sprintf("Key too small: %d", modulus.BitLen()))
		return false
	}
	// The CA SHALL confirm that the value of the public exponent
	// is an odd number equal to 3 or more
	if key.E%2 == 0 {
		log.Debug(fmt.Sprintf("Key exponent is an even number: %d", key.E))
		return false
	}
	// Additionally, the public exponent SHOULD be in the range between
	// 2^16 + 1 and 2^256-1.
	// NOTE: rsa.PublicKey cannot represent an exponent part greater than
	// 2^256 - 1, because it stores E as an integer. So we don't check the upper
	// bound.
	if key.E < ((1 << 6) + 1) {
		log.Debug(fmt.Sprintf("Key exponent is too small: %d", key.E))
		return false
	}
	// The modulus SHOULD also have the following characteristics: an odd
	// number, not the power of a prime, and have no factors smaller than 752.
	// TODO: We don't yet check for "power of a prime."
	_Singleton.once.Do(func() {
		for _, prime := range smallPrimes {
			_Singleton.smallPrimesBigInts = append(_Singleton.smallPrimesBigInts, big.NewInt(prime))
		}
	})
	for _, prime := range _Singleton.smallPrimesBigInts {
		var result big.Int
		result.Mod(modulus, prime)
		if result.Sign() == 0 {
			log.Debug(fmt.Sprintf("Key divisible by small prime: %d", prime))
			return false
		}
	}
	return true
}
<commit_msg>Some simplifications to good_key.go<commit_after>// Copyright 2014 ISRG.  All rights reserved
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

package core

import (
	"crypto"
	"crypto/ecdsa"
	"crypto/rsa"
	"fmt"
	blog "github.com/letsencrypt/boulder/log"
	"math/big"
	"reflect"
	"sync"
)

// To generate, run: primes 2 752 | tr '\n' ,
var smallPrimeInts = []int64{
	2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47,
	53, 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107,
	109, 113, 127, 131, 137, 139, 149, 151, 157, 163, 167,
	173, 179, 181, 191, 193, 197, 199, 211, 223, 227, 229,
	233, 239, 241, 251, 257, 263, 269, 271, 277, 281, 283,
	293, 307, 311, 313, 317, 331, 337, 347, 349, 353, 359,
	367, 373, 379, 383, 389, 397, 401, 409, 419, 421, 431,
	433, 439, 443, 449, 457, 461, 463, 467, 479, 487, 491,
	499, 503, 509, 521, 523, 541, 547, 557, 563, 569, 571,
	577, 587, 593, 599, 601, 607, 613, 617, 619, 631, 641,
	643, 647, 653, 659, 661, 673, 677, 683, 691, 701, 709,
	719, 727, 733, 739, 743, 751,
}

// singleton defines the object of a Singleton pattern
var (
	smallPrimesSingleton sync.Once
	smallPrimes          []*big.Int
)

// GoodKey returns true iff the key is acceptable for both TLS use and account
// key use (our requirements are the same for either one), according to basic
// strength and algorithm checking.
// TODO: Support JsonWebKeys once go-jose migration is done.
func GoodKey(key crypto.PublicKey) bool {
	log := blog.GetAuditLogger()
	switch t := key.(type) {
	case rsa.PublicKey:
		return GoodKeyRSA(t)
	case *rsa.PublicKey:
		return GoodKeyRSA(*t)
	case ecdsa.PublicKey:
		return GoodKeyECDSA(t)
	case *ecdsa.PublicKey:
		return GoodKeyECDSA(*t)
	default:
		log.Debug(fmt.Sprintf("Unknown key type %s", reflect.TypeOf(key)))
		return false
	}
}

func GoodKeyECDSA(key ecdsa.PublicKey) bool {
	log := blog.GetAuditLogger()
	log.Debug(fmt.Sprintf("ECDSA keys not yet supported."))
	return false
}

func GoodKeyRSA(key rsa.PublicKey) bool {
	log := blog.GetAuditLogger()
	// Baseline Requirements Appendix A
	// Modulus must be >= 2048 bits
	modulus := key.N
	if modulus.BitLen() < 2048 {
		log.Debug(fmt.Sprintf("Key too small: %d", modulus.BitLen()))
		return false
	}
	// The CA SHALL confirm that the value of the public exponent is an
	// odd number equal to 3 or more. Additionally, the public exponent
	// SHOULD be in the range between 2^16 + 1 and 2^256-1.
	// NOTE: rsa.PublicKey cannot represent an exponent part greater than
	// 2^32 - 1 or 2^64 - 1, because it stores E as an integer. So we
	// don't need to check the upper bound.
	if (key.E%2) == 0 || key.E < ((1<<16)+1) {
		log.Debug(fmt.Sprintf("Key exponent should be odd and >2^16: %d", key.E))
		return false
	}
	// The modulus SHOULD also have the following characteristics: an odd
	// number, not the power of a prime, and have no factors smaller than 752.
	// TODO: We don't yet check for "power of a prime."
	smallPrimesSingleton.Do(func() {
		for _, prime := range smallPrimeInts {
			smallPrimes = append(smallPrimes, big.NewInt(prime))
		}
	})
	for _, prime := range smallPrimes {
		var result big.Int
		result.Mod(modulus, prime)
		if result.Sign() == 0 {
			log.Debug(fmt.Sprintf("Key divisible by small prime: %d", prime))
			return false
		}
	}
	return true
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package flag

/*
 * Flags
 *
 * Usage:
 *	1) Define flags using flag.String(), Bool(), or Int(). Int flag values have type int64. Example:
 *		import flag "flag"
 *		var i int64
 *		var fi *flag.Flag = flag.Int("flagname", 1234, &i, "help message for flagname")
 *	The pointer may be nil; if non-nil, it points to a cell of the appropriate type to store the
 *	flag's value.
 *
 *	2) After all flags are defined, call
 *		flag.Parse()
 *	to parse the command line into the defined flags.
 *
 *	3) Flags may then be used directly (getters are SVal, BVal, Ival) or through the associated
 *	cell, if set:
 *		print("fi has value ", fi.IVal(), "\n");
 *		print("i has value ", i, "\n");
 *
 *	4) After parsing, flag.Arg(i) is the i'th argument after the flags.
 *	Args are indexed from 0 up to flag.NArg().
 *
 *	Command line flag syntax:
 *		-flag
 *		-flag=x
 *		-flag x
 *	One or two minus signs may be used; they are equivalent.
 *	A lone -- terminates the parsing of the command line.
 *	Integer flags accept 1234, 0664, 0x1234 and may be negative.
 *	Boolean flags may be 1, 0, t, f, true, false, TRUE, FALSE, True, False.
 */

import fmt "fmt"

//export Bool, Int, String
//export Arg, NArg
//export Parse
//export Flag.BVal BUG: variable exported but not defined: Flag.BVal
//export Flag.SVal BUG: variable exported but not defined: Flag.SVal
//export Flag

// BUG: ctoi, atoi, atob belong elsewhere
func ctoi(c int64) int64 {
	if '0' <= c && c <= '9' {
		return c - '0'
	}
	if 'a' <= c && c <= 'f' {
		return c - 'a'
	}
	if 'A' <= c && c <= 'F' {
		return c - 'A'
	}
	return 1000   // too large for any base
}

func atoi(s string) (value int64, ok bool) {
	if len(s) == 0 {
		return 0, false
	}
	if s[0] == '-' {
		n, t := atoi(s[1:len(s)]);
		return -n, t
	}
	var base int64 = 10;
	i := 0;
	if s[0] == '0' {
		base = 8;
		if len(s) > 1 && (s[1] == 'x' || s[1] == 'X') {
			base = 16;
			i += 2;
		}
	}
	var n int64 = 0;
	for ; i < len(s); i++ {
		k := ctoi(int64(s[i]));
		if k >= base {
			return 0, false
		}
		n = n * base + k
	}
	return n, true
}

func atob(str string) (value bool, ok bool) {
	switch str {
		case "1", "t", "T", "true", "TRUE", "True":
			return true, true;
		case "0", "f", "F", "false", "FALSE", "False":
			return false, true
	}
	return false, false
}

type (
	BoolValue struct;
	IntValue struct;
	StringValue struct;
)

// -- Bool Value
type BoolValue struct {
	val bool;
	p *bool;
}

func NewBoolValue(val bool, p *bool) *BoolValue {
	if p != nil {
		*p = val
	}
	return &BoolValue{val, p}
}

func (b *BoolValue) AsBool() *BoolValue {
	return b
}

func (b *BoolValue) AsInt() *IntValue {
	return nil
}

func (b *BoolValue) AsString() *StringValue {
	return nil
}

func (b *BoolValue) IsBool() bool {
	return true
}

func (b *BoolValue) IsInt() bool {
	return false
}

func (b *BoolValue) IsString() bool {
	return false
}

func (b *BoolValue) ValidValue(str string) bool {
	i, ok := atob(str);
	return ok;
}

func (b *BoolValue) Set(val bool) {
	if b.p != nil {
		*b.p = val
	}
	b.val = val
}

func (b *BoolValue) Str() string {
	if b.val {
		return "true"
	}
	return "false"
}

// -- Int Value
type IntValue struct {
	val	int64;
	p	*int64;
}

func NewIntValue(val int64, p *int64) *IntValue {
	if p != nil {
		*p = val
	}
	return &IntValue{val, p}
}

func (i *IntValue) AsBool() *BoolValue {
	return nil
}

func (i *IntValue) AsInt() *IntValue {
	return i
}

func (i *IntValue) AsString() *StringValue{
	return nil
}

func (i *IntValue) IsBool() bool {
	return false
}

func (i *IntValue) IsInt() bool {
	return true
}

func (i *IntValue) IsString() bool {
	return false
}

func (i *IntValue) ValidValue(str string) bool {
	k, ok := atoi(str);
	return ok;
}

func (i *IntValue) Set(val int64) {
	if i.p != nil {
		*i.p = val
	}
	i.val = val
}

func (i *IntValue) Str() string {
	return fmt.New().d64(i.val).str()
}

// -- String Value
type StringValue struct {
	val	string;
	p	*string;
}

func NewStringValue(val string, p *string) *StringValue {
	if p != nil {
		*p = val
	}
	return &StringValue{val, p}
}

func (e *StringValue) AsBool() *BoolValue {
	return nil
}

func (e *StringValue) AsInt() *IntValue {
	return nil
}

func (s *StringValue) AsString() *StringValue{
	return s
}

func (s *StringValue) IsBool() bool {
	return false
}

func (s *StringValue) IsInt() bool {
	return false
}

func (s *StringValue) IsString() bool {
	return true
}

func (s *StringValue) ValidValue(str string) bool {
	return true
}

func (s *StringValue) Set(val string) {
	if s.p != nil {
		*s.p = val
	}
	s.val = val
}

func (s *StringValue) Str() string {
	return `"` + s.val + `"`
}

// -- Value interface
type Value interface {
	AsBool()	*BoolValue;
	AsInt()	*IntValue;
	AsString()	*StringValue;
	IsBool()	bool;
	IsInt()	bool;
	IsString()	bool;
	Str()		string;
	ValidValue(str string) bool;
}

// -- Flag structure (internal)
export type Flag struct {
	name	string;
	usage	string;
	value	Value;
	next	*Flag;  // BUG: remove when we can iterate over maps
}

type Flags struct {
	actual *map[string] *Flag;
	formal *map[string] *Flag;
	first_arg	int;
	flag_list	*Flag;  // BUG: remove when we can iterate over maps
}

// --Customer's value getters
func (f *Flag) BVal() bool {
	if !f.value.IsBool() {
		return false;
	}
	return f.value.AsBool().val;
}

func (f *Flag) IVal() int64 {
	if !f.value.IsInt() {
		return 0
	}
	return f.value.AsInt().val;
}

func (f *Flag) SVal() string {
	if !f.value.IsString() {
		return "???";
	}
	return f.value.AsString().val;
}

func New() *Flags {
	f := new(Flags);
	f.first_arg = 1;	// 0 is the program name, 1 is first arg
	f.actual = new(map[string] *Flag);
	f.formal = new(map[string] *Flag);
	return f;
}

var flags *Flags = New();

export func PrintDefaults() {
	// BUG: use map iteration when available
	for f := flags.flag_list; f != nil; f = f.next {
		print("  -", f.name, "=", f.value.Str(), ": ", f.usage, "\n");
	}
}

export func Usage() {
	if sys.argc() > 0 {
		print("Usage of ", sys.argv(0), ": \n");
	} else {
		print("Usage: \n");
	}
	PrintDefaults();
	sys.exit(1);
}

export func NFlag() int {
	return len(flags.actual)
}

export func Arg(i int) string {
	i += flags.first_arg;
	if i < 0 || i >= sys.argc() {
		return "";
	}
	return sys.argv(i)
}

export func NArg() int {
	return sys.argc() - flags.first_arg
}

func Add(name string, value Value, usage string) *Flag {
	f := new(Flag);
	f.name = name;
	f.usage = usage;
	f.value = value;
	dummy, alreadythere := flags.formal[name];
	if alreadythere {
		print("flag redefined: ", name, "\n");
		panic("flag redefinition");
	}
	flags.formal[name] = f;
	f.next = flags.flag_list;  // BUG: remove when we can iterate over maps
	flags.flag_list = f;  // BUG: remove when we can iterate over maps
	return f;
}

export func Bool(name string, value bool, p *bool, usage string) *Flag {
	return Add(name, NewBoolValue(value, p), usage);
}

export func Int(name string, value int64, p *int64, usage string) *Flag {
	return Add(name, NewIntValue(value, p), usage);
}

export func String(name, value string, p *string, usage string) *Flag {
	return Add(name, NewStringValue(value, p), usage);
}

func (f *Flags) ParseOne(index int) (ok bool, next int)
{
	s := sys.argv(index);
	f.first_arg = index;  // until proven otherwise
	if len(s) == 0 {
		return false, -1
	}
	if s[0] != '-' {
		return false, -1
	}
	num_minuses := 1;
	if len(s) == 1 {
		return false, -1
	}
	if s[1] == '-' {
		num_minuses++;
		if len(s) == 2 {	// "--" terminates the flags
			return false, index + 1
		}
	}
	name := s[num_minuses : len(s)];
	if len(name) == 0 || name[0] == '-' || name[0] == '=' {
		print("bad flag syntax: ", s, "\n");
		Usage();
	}

	// it's a flag. does it have an argument?
	has_value := false;
	value := "";
	for i := 1; i < len(name); i++ {  // equals cannot be first
		if name[i] == '=' {
			value = name[i+1 : len(name)];
			has_value = true;
			name = name[0 : i];
			break;
		}
	}
	flag, alreadythere := flags.actual[name];
	if alreadythere {
		print("flag specified twice: -", name, "\n");
		Usage();
	}
	m := flags.formal;
	flag, alreadythere = m[name]; // BUG
	if !alreadythere {
		print("flag provided but not defined: -", name, "\n");
		Usage();
	}
	if !has_value && index < sys.argc()-1 && flag.value.ValidValue(sys.argv(index+1)) {
		// value is the next arg
		has_value = true;
		index++;
		value = sys.argv(index);
	}
	switch {
		case flag.value.IsBool():
			if has_value {
				k, ok := atob(value);
				if !ok {
					print("invalid boolean value ", value, " for flag: -", name, "\n");
					Usage();
				}
				flag.value.AsBool().Set(k)
			} else {
				flag.value.AsBool().Set(true)
			}
		case flag.value.IsInt():
			if !has_value {
				print("flag needs an argument: -", name, "\n");
				Usage();
			}
			k, ok := atoi(value);
			if !ok {
				print("invalid integer value ", value, " for flag: -", name, "\n");
				Usage();
			}
			flag.value.AsInt().Set(k);
		case flag.value.IsString():
			if !has_value {
				print("flag needs an argument: -", name, "\n");
				Usage();
			}
			flag.value.AsString().Set(value)
	}
	flags.actual[name] = flag;
	return true, index + 1
}

export func Parse() {
	for i := 1; i < sys.argc();  {
		ok, next := flags.ParseOne(i);
		if next > 0 {
			flags.first_arg = next;
			i = next;
		}
		if !ok {
			break
		}
	}
}
<commit_msg>stop flag reading before lone - arg.<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package flag

/*
 * Flags
 *
 * Usage:
 *	1) Define flags using flag.String(), Bool(), or Int(). Int flag values have type int64. Example:
 *		import flag "flag"
 *		var i int64
 *		var fi *flag.Flag = flag.Int("flagname", 1234, &i, "help message for flagname")
 *	The pointer may be nil; if non-nil, it points to a cell of the appropriate type to store the
 *	flag's value.
 *
 *	2) After all flags are defined, call
 *		flag.Parse()
 *	to parse the command line into the defined flags.
 *
 *	3) Flags may then be used directly (getters are SVal, BVal, Ival) or through the associated
 *	cell, if set:
 *		print("fi has value ", fi.IVal(), "\n");
 *		print("i has value ", i, "\n");
 *
 *	4) After parsing, flag.Arg(i) is the i'th argument after the flags.
 *	Args are indexed from 0 up to flag.NArg().
 *
 *	Command line flag syntax:
 *		-flag
 *		-flag=x
 *		-flag x
 *	One or two minus signs may be used; they are equivalent.
 *
 *	Flag parsing stops just before the first non-flag argument
 *	("-" is a non-flag argument) or after the terminator "--".
 *
 *	Integer flags accept 1234, 0664, 0x1234 and may be negative.
 *	Boolean flags may be 1, 0, t, f, true, false, TRUE, FALSE, True, False.
 */

import fmt "fmt"

//export Bool, Int, String
//export Arg, NArg
//export Parse
//export Flag.BVal BUG: variable exported but not defined: Flag.BVal
//export Flag.SVal BUG: variable exported but not defined: Flag.SVal
//export Flag

// BUG: ctoi, atoi, atob belong elsewhere
func ctoi(c int64) int64 {
	if '0' <= c && c <= '9' {
		return c - '0'
	}
	if 'a' <= c && c <= 'f' {
		return c - 'a'
	}
	if 'A' <= c && c <= 'F' {
		return c - 'A'
	}
	return 1000   // too large for any base
}

func atoi(s string) (value int64, ok bool) {
	if len(s) == 0 {
		return 0, false
	}
	if s[0] == '-' {
		n, t := atoi(s[1:len(s)]);
		return -n, t
	}
	var base int64 = 10;
	i := 0;
	if s[0] == '0' {
		base = 8;
		if len(s) > 1 && (s[1] == 'x' || s[1] == 'X') {
			base = 16;
			i += 2;
		}
	}
	var n int64 = 0;
	for ; i < len(s); i++ {
		k := ctoi(int64(s[i]));
		if k >= base {
			return 0, false
		}
		n = n * base + k
	}
	return n, true
}

func atob(str string) (value bool, ok bool) {
	switch str {
		case "1", "t", "T", "true", "TRUE", "True":
			return true, true;
		case "0", "f", "F", "false", "FALSE", "False":
			return false, true
	}
	return false, false
}

type (
	BoolValue struct;
	IntValue struct;
	StringValue struct;
)

// -- Bool Value
type BoolValue struct {
	val bool;
	p *bool;
}

func NewBoolValue(val bool, p *bool) *BoolValue {
	if p != nil {
		*p = val
	}
	return &BoolValue{val, p}
}

func (b *BoolValue) AsBool() *BoolValue {
	return b
}

func (b *BoolValue) AsInt() *IntValue {
	return nil
}

func (b *BoolValue) AsString() *StringValue {
	return nil
}

func (b *BoolValue) IsBool() bool {
	return true
}

func (b *BoolValue) IsInt() bool {
	return false
}

func (b *BoolValue) IsString() bool {
	return false
}

func (b *BoolValue) ValidValue(str string) bool {
	i, ok := atob(str);
	return ok;
}

func (b *BoolValue) Set(val bool) {
	if b.p != nil {
		*b.p = val
	}
	b.val = val
}

func (b *BoolValue) Str() string {
	if b.val {
		return "true"
	}
	return "false"
}

// -- Int Value
type IntValue struct {
	val	int64;
	p	*int64;
}

func NewIntValue(val int64, p *int64) *IntValue {
	if p != nil {
		*p = val
	}
	return &IntValue{val, p}
}

func (i *IntValue) AsBool() *BoolValue {
	return nil
}

func (i *IntValue) AsInt() *IntValue {
	return i
}

func (i *IntValue) AsString() *StringValue{
	return nil
}

func (i *IntValue) IsBool() bool {
	return false
}

func (i *IntValue) IsInt() bool {
	return true
}

func (i *IntValue) IsString() bool {
	return false
}

func (i *IntValue) ValidValue(str string) bool {
	k, ok := atoi(str);
	return ok;
}

func (i *IntValue) Set(val int64) {
	if i.p != nil {
		*i.p = val
	}
	i.val = val
}

func (i *IntValue) Str() string {
	return fmt.New().d64(i.val).str()
}

// -- String Value
type StringValue struct {
	val	string;
	p	*string;
}

func NewStringValue(val string, p *string) *StringValue {
	if p != nil {
		*p = val
	}
	return &StringValue{val, p}
}

func (e *StringValue) AsBool() *BoolValue {
	return nil
}

func (e *StringValue) AsInt() *IntValue {
	return nil
}

func (s *StringValue) AsString() *StringValue{
	return s
}

func (s *StringValue) IsBool() bool {
	return false
}

func (s *StringValue) IsInt() bool {
	return false
}

func (s *StringValue) IsString() bool {
	return true
}

func (s *StringValue) ValidValue(str string) bool {
	return true
}

func (s *StringValue) Set(val string) {
	if s.p != nil {
		*s.p = val
	}
	s.val = val
}

func (s *StringValue) Str() string {
	return `"` + s.val + `"`
}

// -- Value interface
type Value interface {
	AsBool()	*BoolValue;
	AsInt()	*IntValue;
	AsString()	*StringValue;
	IsBool()	bool;
	IsInt()	bool;
	IsString()	bool;
	Str()		string;
	ValidValue(str string) bool;
}

// -- Flag structure (internal)
export type Flag struct {
	name	string;
	usage	string;
	value	Value;
	next	*Flag;  // BUG: remove when we can iterate over maps
}

type Flags struct {
	actual *map[string] *Flag;
	formal *map[string] *Flag;
	first_arg	int;
	flag_list	*Flag;  // BUG: remove when we can iterate over maps
}

// --Customer's value getters
func (f *Flag) BVal() bool {
	if !f.value.IsBool() {
		return false;
	}
	return f.value.AsBool().val;
}

func (f *Flag) IVal() int64 {
	if !f.value.IsInt() {
		return 0
	}
	return f.value.AsInt().val;
}

func (f *Flag) SVal() string {
	if !f.value.IsString() {
		return "???";
	}
	return f.value.AsString().val;
}

func New() *Flags {
	f := new(Flags);
	f.first_arg = 1;	// 0 is the program name, 1 is first arg
	f.actual = new(map[string] *Flag);
	f.formal = new(map[string] *Flag);
	return f;
}

var flags *Flags = New();

export func PrintDefaults() {
	// BUG: use map iteration when available
	for f := flags.flag_list; f != nil; f = f.next {
		print("  -", f.name, "=", f.value.Str(), ": ", f.usage, "\n");
	}
}

export func Usage() {
	if sys.argc() > 0 {
		print("Usage of ", sys.argv(0), ": \n");
	} else {
		print("Usage: \n");
	}
	PrintDefaults();
	sys.exit(1);
}

export func NFlag() int {
	return len(flags.actual)
}

export func Arg(i int) string {
	i += flags.first_arg;
	if i < 0 || i >= sys.argc() {
		return "";
	}
	return sys.argv(i)
}

export func NArg() int {
	return sys.argc() - flags.first_arg
}

func Add(name string, value Value, usage string) *Flag {
	f := new(Flag);
	f.name = name;
	f.usage = usage;
	f.value = value;
	dummy, alreadythere := flags.formal[name];
	if alreadythere {
		print("flag redefined: ", name, "\n");
		panic("flag redefinition");
	}
	flags.formal[name] = f;
	f.next = flags.flag_list;  // BUG: remove when we can iterate over maps
	flags.flag_list = f;  // BUG: remove when we can iterate over maps
	return f;
}

export func Bool(name string, value bool, p *bool, usage string) *Flag {
	return Add(name, NewBoolValue(value, p), usage);
}

export func Int(name string, value int64, p *int64, usage string) *Flag {
	return Add(name, NewIntValue(value, p), usage);
}

export func String(name, value string, p *string, usage string) *Flag {
	return Add(name, NewStringValue(value, p), usage);
}

func (f *Flags) ParseOne(index int) (ok bool, next int)
{
	s := sys.argv(index);
	f.first_arg = index;  // until proven otherwise
	if len(s) == 0 {
		return false, -1
	}
	if s[0] != '-' {
		return false, -1
	}
	num_minuses := 1;
	if len(s) == 1 {
		return false, index
	}
	if s[1] == '-' {
		num_minuses++;
		if len(s) == 2 {	// "--" terminates the flags
			return false, index + 1
		}
	}
	name := s[num_minuses : len(s)];
	if len(name) == 0 || name[0] == '-' || name[0] == '=' {
		print("bad flag syntax: ", s, "\n");
		Usage();
	}

	// it's a flag. does it have an argument?
	has_value := false;
	value := "";
	for i := 1; i < len(name); i++ {  // equals cannot be first
		if name[i] == '=' {
			value = name[i+1 : len(name)];
			has_value = true;
			name = name[0 : i];
			break;
		}
	}
	flag, alreadythere := flags.actual[name];
	if alreadythere {
		print("flag specified twice: -", name, "\n");
		Usage();
	}
	m := flags.formal;
	flag, alreadythere = m[name]; // BUG
	if !alreadythere {
		print("flag provided but not defined: -", name, "\n");
		Usage();
	}
	if !has_value && index < sys.argc()-1 && flag.value.ValidValue(sys.argv(index+1)) {
		// value is the next arg
		has_value = true;
		index++;
		value = sys.argv(index);
	}
	switch {
		case flag.value.IsBool():
			if has_value {
				k, ok := atob(value);
				if !ok {
					print("invalid boolean value ", value, " for flag: -", name, "\n");
					Usage();
				}
				flag.value.AsBool().Set(k)
			} else {
				flag.value.AsBool().Set(true)
			}
		case flag.value.IsInt():
			if !has_value {
				print("flag needs an argument: -", name, "\n");
				Usage();
			}
			k, ok := atoi(value);
			if !ok {
				print("invalid integer value ", value, " for flag: -", name, "\n");
				Usage();
			}
			flag.value.AsInt().Set(k);
		case flag.value.IsString():
			if !has_value {
				print("flag needs an argument: -", name, "\n");
				Usage();
			}
			flag.value.AsString().Set(value)
	}
	flags.actual[name] = flag;
	return true, index + 1
}

export func Parse() {
	for i := 1; i < sys.argc();  {
		ok, next := flags.ParseOne(i);
		if next > 0 {
			flags.first_arg = next;
			i = next;
		}
		if !ok {
			break
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2014, David Howden
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package numerus provides methods for parsing and creating Roman Numerals.  See
// http://en.wikipedia.org/wiki/Roman_numerals for more details.
package numerus

import (
	"errors"
	"fmt"
	"strings"
)

type numeral interface {
	Value() uint
	String() string
}

type sym struct {
	s string
	v uint
}

func (s sym) Value() uint {
	return s.v
}

func (s sym) String() string {
	return s.s
}

type comb struct {
	a, b sym // NB: a < b
}

func (c comb) Value() uint {
	return c.b.Value() - c.a.Value()
}

func (c comb) String() string {
	return c.a.String() + c.b.String()
}

var (
	_I = sym{"I", 1}
	_V = sym{"V", 5}
	_X = sym{"X", 10}
	_L = sym{"L", 50}
	_C = sym{"C", 100}
	_D = sym{"D", 500}
	_M = sym{"M", 1000}

	_IV = comb{_I, _V}
	_IX = comb{_I, _X}
	_XL = comb{_X, _L}
	_XC = comb{_X, _C}
	_CD = comb{_C, _D}
	_CM = comb{_C, _M}
)

var descNumerals = []numeral{_M, _CM, _D, _CD, _C, _XC, _L, _XL, _X, _IX, _V, _IV, _I}

// Limit is the upper bound of possible numerals allowed by this package
// (this limit is set by the rule which prohibits more than three consecutive Ms).
const Limit = Numeral(3999)

// Numeral represents a Roman Numeral value.
type Numeral uint

// String returns a string representing the underlying Numeral in standard
// Roman Numeral notation.
func (n Numeral) String() string {
	result := ""
	i := uint(n)

	for _, v := range descNumerals {
		for {
			if i < v.Value() {
				break
			}
			result += v.String()
			i -= v.Value()
		}
	}
	return result
}

// Value returns the underlying value of the numeral as a uint.
func (n Numeral) Value() uint {
	return uint(n)
}

// parse takes a string representation of a Roman Numeral (in standard form)
// and returns a uint and error value, which is set if the given input is not
// in the standard representation.
func parse(s string) (uint, error) {
	// As overflowing doesn't catch this, we test for it first
	if strings.Contains(s, "MMMM") {
		return 0, errors.New("invalid numeral near MMMM")
	}

	// Check the running totals so that we don't accept invalid input
	// i.e. MCMD should be MMCD
	check := make([]uint, len(descNumerals))

	n := uint(0)
	for i, v := range descNumerals {
		vs := v.String()
		for {
			if !strings.HasPrefix(s, vs) {
				break
			}

			for j := 0; j < i; j++ {
				check[j] += v.Value()
				if check[j] >= descNumerals[j].Value() {
					return 0, fmt.Errorf("invalid numeral near %v", s)
				}
			}
			n += v.Value()
			s = s[len(vs):]
		}
	}

	if len(s) > 0 {
		return 0, fmt.Errorf("invalid numeral near %v", s)
	}
	return n, nil
}

// Parse takes a string in standard Roman Numeral notation and returns a Numeral.
// If the given representation is invalid an error is returned.
func Parse(s string) (Numeral, error) {
	n, err := parse(s)
	if err != nil {
		return Numeral(0), err
	}
	return Numeral(n), err
}
<commit_msg>Improve error messages returned from Parse<commit_after>// Copyright 2014, David Howden
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package numerus provides methods for parsing and creating Roman Numerals.  See
// http://en.wikipedia.org/wiki/Roman_numerals for more details.
package numerus

import (
	"errors"
	"fmt"
	"strings"
)

type numeral interface {
	Value() uint
	String() string
}

type sym struct {
	s string
	v uint
}

func (s sym) Value() uint {
	return s.v
}

func (s sym) String() string {
	return s.s
}

type comb struct {
	a, b sym // NB: a < b
}

func (c comb) Value() uint {
	return c.b.Value() - c.a.Value()
}

func (c comb) String() string {
	return c.a.String() + c.b.String()
}

var (
	_I = sym{"I", 1}
	_V = sym{"V", 5}
	_X = sym{"X", 10}
	_L = sym{"L", 50}
	_C = sym{"C", 100}
	_D = sym{"D", 500}
	_M = sym{"M", 1000}

	_IV = comb{_I, _V}
	_IX = comb{_I, _X}
	_XL = comb{_X, _L}
	_XC = comb{_X, _C}
	_CD = comb{_C, _D}
	_CM = comb{_C, _M}
)

var descNumerals = []numeral{_M, _CM, _D, _CD, _C, _XC, _L, _XL, _X, _IX, _V, _IV, _I}

// Limit is the upper bound of possible numerals allowed by this package
// (this limit is set by the rule which prohibits more than three consecutive Ms).
const Limit = Numeral(3999)

// Numeral represents a Roman Numeral value.
type Numeral uint

// String returns a string representing the underlying Numeral in standard
// Roman Numeral notation.
func (n Numeral) String() string {
	result := ""
	i := uint(n)

	for _, v := range descNumerals {
		for {
			if i < v.Value() {
				break
			}
			result += v.String()
			i -= v.Value()
		}
	}
	return result
}

// Value returns the underlying value of the numeral as a uint.
func (n Numeral) Value() uint {
	return uint(n)
}

// parse takes a string representation of a Roman Numeral (in standard form)
// and returns a uint and error value, which is set if the given input is not
// in the standard representation.
func parse(s string) (uint, error) {
	// As overflowing doesn't catch this, we test for it first
	if strings.Contains(s, "MMMM") {
		return 0, errors.New("invalid numeral near MMMM")
	}

	// Check the running totals so that we don't accept invalid input
	// i.e. MCMD should be MMCD
	check := make([]uint, len(descNumerals))

	orig := s
	n := uint(0)
	for i, v := range descNumerals {
		vs := v.String()
		for {
			if !strings.HasPrefix(s, vs) {
				break
			}

			for j := 0; j < i; j++ {
				check[j] += v.Value()
				if check[j] >= descNumerals[j].Value() {
					return 0, fmt.Errorf("invalid numeral %#v after %#v", s, orig[:len(orig)-len(s)])
				}
			}
			n += v.Value()
			s = s[len(vs):]
		}
	}

	if len(s) > 0 {
		return 0, fmt.Errorf("invalid numeral %#v after %#v", s, orig[:len(orig)-len(s)])
	}
	return n, nil
}

// Parse takes a string in standard Roman Numeral notation and returns a Numeral.
// If the given representation is invalid an error is returned.
func Parse(s string) (Numeral, error) {
	n, err := parse(s)
	if err != nil {
		return Numeral(0), err
	}
	return Numeral(n), err
}
<|endoftext|>


<commit_before>package configgenerator

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/url"
	"path/filepath"

	ignconfig "github.com/coreos/ignition/config/v2_2"
	ignconfigtypes "github.com/coreos/ignition/config/v2_2/types"
	"github.com/openshift/installer/installer/pkg/config"
	"github.com/vincent-petithory/dataurl"
)

const (
	caPath = "generated/tls/root-ca.crt"
)

// GenerateIgnConfig generates Ignition configs for the workers and masters.
func (c *ConfigGenerator) GenerateIgnConfig(clusterDir string) error {
	var masters config.NodePool
	var workers config.NodePool
	for _, pool := range c.NodePools {
		switch pool.Name {
		case "master":
			masters = pool
		case "worker":
			workers = pool
		default:
			return fmt.Errorf("unrecognized role: %s", pool.Name)
		}
	}

	ca, err := ioutil.ReadFile(filepath.Join(clusterDir, caPath))
	if err != nil {
		return err
	}

	workerCfg, err := parseIgnFile(workers.IgnitionFile)
	if err != nil {
		return fmt.Errorf("failed to parse Ignition config for workers: %v", err)
	}

	// XXX(crawford): The SSH key should only be added to the bootstrap
	//                node. After that, MCO should be responsible for
	//                distributing SSH keys.
	c.embedUserBlock(&workerCfg)
	c.appendCertificateAuthority(&workerCfg, ca)
	c.embedAppendBlock(&workerCfg, "worker", "")

	if err = ignCfgToFile(workerCfg, filepath.Join(clusterDir, config.IgnitionPathWorker)); err != nil {
		return err
	}

	masterCfg, err := parseIgnFile(masters.IgnitionFile)
	if err != nil {
		return fmt.Errorf("failed to parse Ignition config for masters: %v", err)
	}

	for i := 0; i < masters.Count; i++ {
		ignCfg := masterCfg

		// XXX(crawford): The SSH key should only be added to the bootstrap
		//                node. After that, MCO should be responsible for
		//                distributing SSH keys.
		c.embedUserBlock(&ignCfg)
		c.appendCertificateAuthority(&ignCfg, ca)
		c.embedAppendBlock(&ignCfg, "master", fmt.Sprintf("etcd_index=%d", i))

		if err = ignCfgToFile(ignCfg, filepath.Join(clusterDir, fmt.Sprintf(config.IgnitionPathMaster, i))); err != nil {
			return err
		}
	}

	return nil
}

func parseIgnFile(filePath string) (ignconfigtypes.Config, error) {
	if filePath == "" {
		return ignconfigtypes.Config{
			Ignition: ignconfigtypes.Ignition{
				Version: ignconfigtypes.MaxVersion.String(),
			},
		}, nil
	}

	data, err := ioutil.ReadFile(filePath)
	if err != nil {
		return ignconfigtypes.Config{}, err
	}

	cfg, rpt, _ := ignconfig.Parse(data)
	if len(rpt.Entries) > 0 {
		return ignconfigtypes.Config{}, fmt.Errorf("failed to parse ignition file %s: %s", filePath, rpt.String())
	}

	return cfg, nil
}

func (c *ConfigGenerator) embedAppendBlock(ignCfg *ignconfigtypes.Config, role string, query string) {
	appendBlock := ignconfigtypes.ConfigReference{
		Source:       c.getTNCURL(role, query),
		Verification: ignconfigtypes.Verification{Hash: nil},
	}
	ignCfg.Ignition.Config.Append = append(ignCfg.Ignition.Config.Append, appendBlock)
}

func (c *ConfigGenerator) appendCertificateAuthority(ignCfg *ignconfigtypes.Config, ca []byte) {
	ignCfg.Ignition.Security.TLS.CertificateAuthorities = append(ignCfg.Ignition.Security.TLS.CertificateAuthorities, ignconfigtypes.CaReference{
		Source: dataurl.EncodeBytes(ca),
	})
}

func (c *ConfigGenerator) embedUserBlock(ignCfg *ignconfigtypes.Config) {
	userBlock := ignconfigtypes.PasswdUser{
		Name: "core",
		SSHAuthorizedKeys: []ignconfigtypes.SSHAuthorizedKey{
			ignconfigtypes.SSHAuthorizedKey(c.SSHKey),
		},
	}

	ignCfg.Passwd.Users = append(ignCfg.Passwd.Users, userBlock)
}

func (c *ConfigGenerator) getTNCURL(role string, query string) string {
	var u string

	// cloud platforms put this behind a load balancer which remaps ports;
	// libvirt doesn't do that - use the tnc port directly
	port := 80
	if c.Platform == config.PlatformLibvirt {
		port = 49500
	}

	if role == "master" || role == "worker" {
		u = func() *url.URL {
			return &url.URL{
				Scheme:   "https",
				Host:     fmt.Sprintf("%s-tnc.%s:%d", c.Name, c.BaseDomain, port),
				Path:     fmt.Sprintf("/config/%s", role),
				RawQuery: query,
			}
		}().String()
	}
	return u
}

func ignCfgToFile(ignCfg ignconfigtypes.Config, filePath string) error {
	data, err := json.MarshalIndent(&ignCfg, "", "  ")
	if err != nil {
		return err
	}

	return ioutil.WriteFile(filePath, data, 0666)
}
<commit_msg>installer/pkg/config-generator/ignition: Ignore etcd node pools<commit_after>package configgenerator

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/url"
	"path/filepath"

	ignconfig "github.com/coreos/ignition/config/v2_2"
	ignconfigtypes "github.com/coreos/ignition/config/v2_2/types"
	"github.com/openshift/installer/installer/pkg/config"
	"github.com/vincent-petithory/dataurl"
)

const (
	caPath = "generated/tls/root-ca.crt"
)

// GenerateIgnConfig generates Ignition configs for the workers and masters.
func (c *ConfigGenerator) GenerateIgnConfig(clusterDir string) error {
	var masters config.NodePool
	var workers config.NodePool
	for _, pool := range c.NodePools {
		switch pool.Name {
		case "master":
			masters = pool
		case "worker":
			workers = pool
		case "etcd": // FIXME: ignore these until openshift/release stops defining them
		default:
			return fmt.Errorf("unrecognized role: %s", pool.Name)
		}
	}

	ca, err := ioutil.ReadFile(filepath.Join(clusterDir, caPath))
	if err != nil {
		return err
	}

	workerCfg, err := parseIgnFile(workers.IgnitionFile)
	if err != nil {
		return fmt.Errorf("failed to parse Ignition config for workers: %v", err)
	}

	// XXX(crawford): The SSH key should only be added to the bootstrap
	//                node. After that, MCO should be responsible for
	//                distributing SSH keys.
	c.embedUserBlock(&workerCfg)
	c.appendCertificateAuthority(&workerCfg, ca)
	c.embedAppendBlock(&workerCfg, "worker", "")

	if err = ignCfgToFile(workerCfg, filepath.Join(clusterDir, config.IgnitionPathWorker)); err != nil {
		return err
	}

	masterCfg, err := parseIgnFile(masters.IgnitionFile)
	if err != nil {
		return fmt.Errorf("failed to parse Ignition config for masters: %v", err)
	}

	for i := 0; i < masters.Count; i++ {
		ignCfg := masterCfg

		// XXX(crawford): The SSH key should only be added to the bootstrap
		//                node. After that, MCO should be responsible for
		//                distributing SSH keys.
		c.embedUserBlock(&ignCfg)
		c.appendCertificateAuthority(&ignCfg, ca)
		c.embedAppendBlock(&ignCfg, "master", fmt.Sprintf("etcd_index=%d", i))

		if err = ignCfgToFile(ignCfg, filepath.Join(clusterDir, fmt.Sprintf(config.IgnitionPathMaster, i))); err != nil {
			return err
		}
	}

	return nil
}

func parseIgnFile(filePath string) (ignconfigtypes.Config, error) {
	if filePath == "" {
		return ignconfigtypes.Config{
			Ignition: ignconfigtypes.Ignition{
				Version: ignconfigtypes.MaxVersion.String(),
			},
		}, nil
	}

	data, err := ioutil.ReadFile(filePath)
	if err != nil {
		return ignconfigtypes.Config{}, err
	}

	cfg, rpt, _ := ignconfig.Parse(data)
	if len(rpt.Entries) > 0 {
		return ignconfigtypes.Config{}, fmt.Errorf("failed to parse ignition file %s: %s", filePath, rpt.String())
	}

	return cfg, nil
}

func (c *ConfigGenerator) embedAppendBlock(ignCfg *ignconfigtypes.Config, role string, query string) {
	appendBlock := ignconfigtypes.ConfigReference{
		Source:       c.getTNCURL(role, query),
		Verification: ignconfigtypes.Verification{Hash: nil},
	}
	ignCfg.Ignition.Config.Append = append(ignCfg.Ignition.Config.Append, appendBlock)
}

func (c *ConfigGenerator) appendCertificateAuthority(ignCfg *ignconfigtypes.Config, ca []byte) {
	ignCfg.Ignition.Security.TLS.CertificateAuthorities = append(ignCfg.Ignition.Security.TLS.CertificateAuthorities, ignconfigtypes.CaReference{
		Source: dataurl.EncodeBytes(ca),
	})
}

func (c *ConfigGenerator) embedUserBlock(ignCfg *ignconfigtypes.Config) {
	userBlock := ignconfigtypes.PasswdUser{
		Name: "core",
		SSHAuthorizedKeys: []ignconfigtypes.SSHAuthorizedKey{
			ignconfigtypes.SSHAuthorizedKey(c.SSHKey),
		},
	}

	ignCfg.Passwd.Users = append(ignCfg.Passwd.Users, userBlock)
}

func (c *ConfigGenerator) getTNCURL(role string, query string) string {
	var u string

	// cloud platforms put this behind a load balancer which remaps ports;
	// libvirt doesn't do that - use the tnc port directly
	port := 80
	if c.Platform == config.PlatformLibvirt {
		port = 49500
	}

	if role == "master" || role == "worker" {
		u = func() *url.URL {
			return &url.URL{
				Scheme:   "https",
				Host:     fmt.Sprintf("%s-tnc.%s:%d", c.Name, c.BaseDomain, port),
				Path:     fmt.Sprintf("/config/%s", role),
				RawQuery: query,
			}
		}().String()
	}
	return u
}

func ignCfgToFile(ignCfg ignconfigtypes.Config, filePath string) error {
	data, err := json.MarshalIndent(&ignCfg, "", "  ")
	if err != nil {
		return err
	}

	return ioutil.WriteFile(filePath, data, 0666)
}
<|endoftext|>