<commit_before>package main

// TODO: implement JSON parser that loops through the output from api.Get()

import (
	"bytes"
	"crypto/tls"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"regexp"
	"strconv"
	"sync"
	"time"

	"github.com/prometheus/client_golang/prometheus"
)

// Chunk size of metric requests
const ChunkSize = 10

// Namespace for metrics
const NameSpace = "newrelic"

// User-Agent string
const UserAgent = "NewRelic Exporter"

// This is to support skipping verification for testing and
// is deliberately not exposed to the user
var TlsIgnore bool = false

// Regular expression to parse Link headers
var rexp = `<([[:graph:]]+)>; rel="next", <([[:graph:]]+)>; rel="last"`
var LinkRexp *regexp.Regexp

func init() {
	LinkRexp = regexp.MustCompile(rexp)
}

type Metric struct {
	App   string
	Name  string
	Value float64
	Label string
}

type AppList struct {
	Applications []struct {
		Id         int
		Name       string
		Health     string             `json:"health_status"`
		AppSummary map[string]float64 `json:"application_summary"`
		UsrSummary map[string]float64 `json:"end_user_summary"`
	}
}

func (a *AppList) get(api newRelicApi) error {

	body, err := api.req("/v2/applications.json", "")
	if err != nil {
		return err
	}

	err = json.Unmarshal(body, a)
	return err

}

func (a *AppList) sendMetrics(ch chan<- Metric) {
	for _, app := range a.Applications {
		for name, value := range app.AppSummary {
			ch <- Metric{
				App:   app.Name,
				Name:  name,
				Value: value,
				Label: "application_summary",
			}
		}

		for name, value := range app.UsrSummary {
			ch <- Metric{
				App:   app.Name,
				Name:  name,
				Value: value,
				Label: "end_user_summary",
			}
		}
	}
}

type MetricNames struct {
	Metrics []struct {
		Name   string
		Values []string
	}
}

func (m *MetricNames) get(api newRelicApi, appId int) error {

	path := fmt.Sprintf("/v2/applications/%s/metrics.json", strconv.Itoa(appId))

	body, err := api.req(path, "")
	if err != nil {
		return err
	}

	dec := json.NewDecoder(bytes.NewReader(body))

	for {
		var part MetricNames
		if err = dec.Decode(&part); err == io.EOF {
			break
		} else if err != nil {
			return err
		}
		tmpMetrics := append(m.Metrics, part.Metrics...)
		m.Metrics = tmpMetrics
	}

	return nil
}

type MetricData struct {
	Metric_Data struct {
		Metrics []struct {
			Name       string
			Timeslices []struct {
				Values map[string]interface{}
			}
		}
	}
}

func (m *MetricData) get(api newRelicApi, appId int, names MetricNames) error {

	path := fmt.Sprintf("/v2/applications/%s/metrics/data.json", strconv.Itoa(appId))

	var nameList []string

	for i := range names.Metrics {
		// We urlencode the metric names as the API will return
		// unencoded names which it cannot read
		nameList = append(nameList, names.Metrics[i].Name)
	}

	// Because the Go client does not yet support 100-continue
	// ( see issue #3665 ),
	// we have to process this in chunks, to ensure the response
	// fits within a single request.

	chans := make([]chan MetricData, 0)

	for i := 0; i < len(nameList); i += ChunkSize {

		chans = append(chans, make(chan MetricData))

		var thisList []string

		if i+ChunkSize > len(nameList) {
			thisList = nameList[i:]
		} else {
			thisList = nameList[i : i+ChunkSize]
		}

		go func(names []string, ch chan<- MetricData) {

			var data MetricData

			params := url.Values{}

			for _, thisName := range thisList {
				params.Add("names[]", thisName)
			}

			params.Add("raw", "true")
			params.Add("summarize", "true")
			params.Add("period", strconv.Itoa(api.period))
			params.Add("from", api.from.Format(time.RFC3339))
			params.Add("to", api.to.Format(time.RFC3339))

			body, err := api.req(path, params.Encode())
			if err != nil {
				close(ch)
				return
			}

			err = json.Unmarshal(body, &data)
			if err != nil {
				close(ch)
				return
			}

			ch <- data
			close(ch)

		}(thisList, chans[len(chans)-1])

	}

	allData := m.Metric_Data.Metrics

	for _, ch := range chans {
		m := <-ch
		allData = append(allData, m.Metric_Data.Metrics...)
	}
	m.Metric_Data.Metrics = allData

	return nil
}

func (m *MetricData) sendMetrics(ch chan<- Metric, app string) {
	for _, set := range m.Metric_Data.Metrics {

		if len(set.Timeslices) == 0 {
			continue
		}

		// As we set summarise=true there will only be one timeseries.
		for name, value := range set.Timeslices[0].Values {

			if v, ok := value.(float64); ok {

				ch <- Metric{
					App:   app,
					Name:  name,
					Value: v,
					Label: set.Name,
				}

			}
		}

	}
}

type Exporter struct {
	mu              sync.Mutex
	duration, error prometheus.Gauge
	totalScrapes    prometheus.Counter
	metrics         map[string]prometheus.GaugeVec
	api             newRelicApi
}

func NewExporter() *Exporter {
	return &Exporter{
		duration: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: NameSpace,
			Name:      "exporter_last_scrape_duration_seconds",
			Help:      "The last scrape duration.",
		}),
		totalScrapes: prometheus.NewCounter(prometheus.CounterOpts{
			Namespace: NameSpace,
			Name:      "exporter_scrapes_total",
			Help:      "Total scraped metrics",
		}),
		error: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: NameSpace,
			Name:      "exporter_last_scrape_error",
			Help:      "The last scrape error status.",
		}),
		metrics: map[string]prometheus.GaugeVec{},
	}
}

func (e *Exporter) scrape(ch chan<- Metric) {

	e.error.Set(0)
	e.totalScrapes.Inc()

	now := time.Now().UnixNano()

	var apps AppList
	err := apps.get(e.api)
	if err != nil {
		e.error.Set(1)
	}

	apps.sendMetrics(ch)

	for _, app := range apps.Applications {

		var names MetricNames

		err = names.get(e.api, app.Id)
		if err != nil {
			e.error.Set(1)
		}

		var data MetricData

		err = data.get(e.api, app.Id, names)
		if err != nil {
			e.error.Set(1)
		}

		data.sendMetrics(ch, app.Name)

	}

	close(ch)
	e.duration.Set(float64(time.Now().UnixNano()-now) / 1000000000)
}

func (e *Exporter) recieve(ch <-chan Metric) {

	for metric := range ch {
		id := fmt.Sprintf("%s_%s_%s", NameSpace, metric.App, metric.Name)

		if m, ok := e.metrics[id]; ok {
			m.WithLabelValues(metric.Label).Set(metric.Value)
		} else {
			g := prometheus.NewGaugeVec(
				prometheus.GaugeOpts{
					Namespace: NameSpace,
					Subsystem: metric.App,
					Name:      metric.Name,
				},
				[]string{"component"})

			e.metrics[id] = *g
		}
	}
}

func (e *Exporter) Describe(ch chan<- *prometheus.Desc) {
	e.mu.Lock()
	defer e.mu.Unlock()

	for _, m := range e.metrics {
		m.Describe(ch)
	}

	ch <- e.duration.Desc()
	ch <- e.totalScrapes.Desc()
	ch <- e.error.Desc()
}

func (e *Exporter) Collect(ch chan<- prometheus.Metric) {
	e.mu.Lock()
	defer e.mu.Unlock()

	e.api.to = time.Now().UTC()
	e.api.from = e.api.to.Add(-time.Duration(e.api.period) * time.Second)

	metricChan := make(chan Metric)

	go e.scrape(metricChan)

	e.recieve(metricChan)

	ch <- e.duration
	ch <- e.totalScrapes
	ch <- e.error

	for _, m := range e.metrics {
		m.Collect(ch)
	}
}

type newRelicApi struct {
	server string
	apiKey string
	from   time.Time
	to     time.Time
	period int
}

func (a *newRelicApi) req(path string, params string) ([]byte, error) {

	u, err := url.Parse(a.server)
	if err != nil {
		return nil, err
	}
	u.Path = path
	u.RawQuery = params

	req := &http.Request{
		Method: "GET",
		URL:    u,
		Header: http.Header{
			"User-Agent": {UserAgent},
			"X-Api-Key":  {a.apiKey},
		},
	}

	client := &http.Client{
		Transport: &http.Transport{
			TLSClientConfig: &tls.Config{
				InsecureSkipVerify: TlsIgnore,
			},
		},
	}

	var data []byte
	pageCount := 1

	for page := 1; page <= pageCount; page++ {
		resp, err := client.Do(req)
		if err != nil {
			return nil, err
		}

		if resp.StatusCode != 200 {
			return nil, fmt.Errorf("Bad response code: %s", resp.Status)
		}

		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return nil, err
		}
		resp.Body.Close()
		data = append(data, body...)

		link := resp.Header.Get("Link")
		vals := LinkRexp.FindStringSubmatch(link)

		if len(vals) == 3 { // Full string plus two sub-expressions
			u, err := url.Parse(vals[2]) // Parse the second URL
			if err != nil {
				return nil, err
			}
			pageCount, err = strconv.Atoi(u.Query().Get("page"))
			if err != nil {
				return nil, err
			}
		}
		qry := req.URL.Query()
		qry.Set("page", strconv.Itoa(page+1))
		req.URL.RawQuery = qry.Encode()
	}

	return data, nil

}

func main() {

	exporter := NewExporter()

	var listenAddress, metricPath string

	flag.StringVar(&exporter.api.apiKey, "api.key", "", "NewRelic API key")
	flag.StringVar(&exporter.api.server, "api.server", "https://api.newrelic.com", "NewRelic API URL")
	flag.IntVar(&exporter.api.period, "api.period", 60, "Period of data to extract in seconds")

	flag.StringVar(&listenAddress, "web.listen-address", ":9126", "Address to listen on for web interface and telemetry.")
	flag.StringVar(&metricPath, "web.telemetry-path", "/metrics", "Path under which to expose metrics.")

	flag.Parse()

	prometheus.MustRegister(exporter)

	http.Handle(metricPath, prometheus.Handler())
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(`
<title>NewRelic exporter</title>

NewRelic exporter

<a href='` + metricPath + `'>Metrics

`))
	})

	http.ListenAndServe(listenAddress, nil)

}
<commit_msg>Add logging library and some basic logging.<commit_after>package main

// TODO: implement JSON parser that loops through the output from api.Get()

import (
	"bytes"
	"crypto/tls"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"regexp"
	"strconv"
	"sync"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/log"
)

// Chunk size of metric requests
const ChunkSize = 10

// Namespace for metrics
const NameSpace = "newrelic"

// User-Agent string
const UserAgent = "NewRelic Exporter"

// This is to support skipping verification for testing and
// is deliberately not exposed to the user
var TlsIgnore bool = false

// Regular expression to parse Link headers
var rexp = `<([[:graph:]]+)>; rel="next", <([[:graph:]]+)>; rel="last"`
var LinkRexp *regexp.Regexp

func init() {
	LinkRexp = regexp.MustCompile(rexp)
}

type Metric struct {
	App   string
	Name  string
	Value float64
	Label string
}

type AppList struct {
	Applications []struct {
		Id         int
		Name       string
		Health     string             `json:"health_status"`
		AppSummary map[string]float64 `json:"application_summary"`
		UsrSummary map[string]float64 `json:"end_user_summary"`
	}
}

func (a *AppList) get(api newRelicApi) error {
	log.Debugf("Requesting application list from %s.", api.server.String())
	body, err := api.req("/v2/applications.json", "")
	if err != nil {
		log.Print("Error getting application list: ", err)
		return err
	}

	err = json.Unmarshal(body, a)
	return err
}

func (a *AppList) sendMetrics(ch chan<- Metric) {
	for _, app := range a.Applications {
		for name, value := range app.AppSummary {
			ch <- Metric{
				App:   app.Name,
				Name:  name,
				Value: value,
				Label: "application_summary",
			}
		}

		for name, value := range app.UsrSummary {
			ch <- Metric{
				App:   app.Name,
				Name:  name,
				Value: value,
				Label: "end_user_summary",
			}
		}
	}
}

type MetricNames struct {
	Metrics []struct {
		Name   string
		Values []string
	}
}

func (m *MetricNames) get(api newRelicApi, appId int) error {
	log.Debugf("Requesting metrics names for application id %d.", appId)
	path := fmt.Sprintf("/v2/applications/%s/metrics.json", strconv.Itoa(appId))

	body, err := api.req(path, "")
	if err != nil {
		log.Print("Error getting metric names: ", err)
		return err
	}

	dec := json.NewDecoder(bytes.NewReader(body))

	for {
		var part MetricNames
		if err = dec.Decode(&part); err == io.EOF {
			break
		} else if err != nil {
			log.Print("Error decoding metric names: ", err)
			return err
		}
		tmpMetrics := append(m.Metrics, part.Metrics...)
		m.Metrics = tmpMetrics
	}

	return nil
}

type MetricData struct {
	Metric_Data struct {
		Metrics []struct {
			Name       string
			Timeslices []struct {
				Values map[string]interface{}
			}
		}
	}
}

func (m *MetricData) get(api newRelicApi, appId int, names MetricNames) error {
	path := fmt.Sprintf("/v2/applications/%s/metrics/data.json", strconv.Itoa(appId))

	var nameList []string

	for i := range names.Metrics {
		// We urlencode the metric names as the API will return
		// unencoded names which it cannot read
		nameList = append(nameList, names.Metrics[i].Name)
	}
	log.Debugf("Requesting %d metrics for application id %d.", len(nameList), appId)

	// Because the Go client does not yet support 100-continue
	// ( see issue #3665 ),
	// we have to process this in chunks, to ensure the response
	// fits within a single request.

	chans := make([]chan MetricData, 0)

	for i := 0; i < len(nameList); i += ChunkSize {

		chans = append(chans, make(chan MetricData))

		var thisList []string

		if i+ChunkSize > len(nameList) {
			thisList = nameList[i:]
		} else {
			thisList = nameList[i : i+ChunkSize]
		}

		go func(names []string, ch chan<- MetricData) {

			var data MetricData

			params := url.Values{}

			for _, thisName := range thisList {
				params.Add("names[]", thisName)
			}

			params.Add("raw", "true")
			params.Add("summarize", "true")
			params.Add("period", strconv.Itoa(api.period))
			params.Add("from", api.from.Format(time.RFC3339))
			params.Add("to", api.to.Format(time.RFC3339))

			body, err := api.req(path, params.Encode())
			if err != nil {
				log.Print("Error requesting metrics: ", err)
				close(ch)
				return
			}

			err = json.Unmarshal(body, &data)
			if err != nil {
				log.Print("Error decoding metrics data: ", err)
				close(ch)
				return
			}

			ch <- data
			close(ch)

		}(thisList, chans[len(chans)-1])

	}

	allData := m.Metric_Data.Metrics

	for _, ch := range chans {
		m := <-ch
		allData = append(allData, m.Metric_Data.Metrics...)
	}
	m.Metric_Data.Metrics = allData

	return nil
}

func (m *MetricData) sendMetrics(ch chan<- Metric, app string) {
	for _, set := range m.Metric_Data.Metrics {

		if len(set.Timeslices) == 0 {
			continue
		}

		// As we set summarise=true there will only be one timeseries.
		for name, value := range set.Timeslices[0].Values {

			if v, ok := value.(float64); ok {

				ch <- Metric{
					App:   app,
					Name:  name,
					Value: v,
					Label: set.Name,
				}

			}
		}

	}
}

type Exporter struct {
	mu              sync.Mutex
	duration, error prometheus.Gauge
	totalScrapes    prometheus.Counter
	metrics         map[string]prometheus.GaugeVec
	api             newRelicApi
}

func NewExporter() *Exporter {
	return &Exporter{
		duration: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: NameSpace,
			Name:      "exporter_last_scrape_duration_seconds",
			Help:      "The last scrape duration.",
		}),
		totalScrapes: prometheus.NewCounter(prometheus.CounterOpts{
			Namespace: NameSpace,
			Name:      "exporter_scrapes_total",
			Help:      "Total scraped metrics",
		}),
		error: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: NameSpace,
			Name:      "exporter_last_scrape_error",
			Help:      "The last scrape error status.",
		}),
		metrics: map[string]prometheus.GaugeVec{},
	}
}

func (e *Exporter) scrape(ch chan<- Metric) {

	e.error.Set(0)
	e.totalScrapes.Inc()

	now := time.Now().UnixNano()
	log.Debugf("Starting new scrape at %d.", now)

	var apps AppList
	err := apps.get(e.api)
	if err != nil {
		e.error.Set(1)
	}

	apps.sendMetrics(ch)

	for _, app := range apps.Applications {

		var names MetricNames

		err = names.get(e.api, app.Id)
		if err != nil {
			e.error.Set(1)
		}

		var data MetricData

		err = data.get(e.api, app.Id, names)
		if err != nil {
			e.error.Set(1)
		}

		data.sendMetrics(ch, app.Name)

	}

	close(ch)
	e.duration.Set(float64(time.Now().UnixNano()-now) / 1000000000)
}

func (e *Exporter) recieve(ch <-chan Metric) {

	for metric := range ch {
		id := fmt.Sprintf("%s_%s_%s", NameSpace, metric.App, metric.Name)

		if m, ok := e.metrics[id]; ok {
			m.WithLabelValues(metric.Label).Set(metric.Value)
		} else {
			g := prometheus.NewGaugeVec(
				prometheus.GaugeOpts{
					Namespace: NameSpace,
					Subsystem: metric.App,
					Name:      metric.Name,
				},
				[]string{"component"})

			e.metrics[id] = *g
		}
	}
}

func (e *Exporter) Describe(ch chan<- *prometheus.Desc) {
	e.mu.Lock()
	defer e.mu.Unlock()

	for _, m := range e.metrics {
		m.Describe(ch)
	}

	ch <- e.duration.Desc()
	ch <- e.totalScrapes.Desc()
	ch <- e.error.Desc()
}

func (e *Exporter) Collect(ch chan<- prometheus.Metric) {
	e.mu.Lock()
	defer e.mu.Unlock()

	e.api.to = time.Now().UTC()
	e.api.from = e.api.to.Add(-time.Duration(e.api.period) * time.Second)

	metricChan := make(chan Metric)

	go e.scrape(metricChan)

	e.recieve(metricChan)

	ch <- e.duration
	ch <- e.totalScrapes
	ch <- e.error

	for _, m := range e.metrics {
		m.Collect(ch)
	}
}

type newRelicApi struct {
	server string
	apiKey string
	from   time.Time
	to     time.Time
	period int
}

func (a *newRelicApi) req(path string, params string) ([]byte, error) {

	u, err := url.Parse(a.server)
	if err != nil {
		return nil, err
	}
	u.Path = path
	u.RawQuery = params

	log.Debug("Making API call: ", u.String())

	req := &http.Request{
		Method: "GET",
		URL:    u,
		Header: http.Header{
			"User-Agent": {UserAgent},
			"X-Api-Key":  {a.apiKey},
		},
	}

	client := &http.Client{
		Transport: &http.Transport{
			TLSClientConfig: &tls.Config{
				InsecureSkipVerify: TlsIgnore,
			},
		},
	}

	var data []byte
	pageCount := 1

	for page := 1; page <= pageCount; page++ {
		resp, err := client.Do(req)
		if err != nil {
			return nil, err
		}

		if resp.StatusCode != 200 {
			return nil, fmt.Errorf("Bad response code: %s", resp.Status)
		}

		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return nil, err
		}
		resp.Body.Close()
		data = append(data, body...)

		link := resp.Header.Get("Link")
		vals := LinkRexp.FindStringSubmatch(link)

		if len(vals) == 3 { // Full string plus two sub-expressions
			u, err := url.Parse(vals[2]) // Parse the second URL
			if err != nil {
				return nil, err
			}
			pageCount, err = strconv.Atoi(u.Query().Get("page"))
			if err != nil {
				return nil, err
			}
		}
		qry := req.URL.Query()
		qry.Set("page", strconv.Itoa(page+1))
		req.URL.RawQuery = qry.Encode()
	}

	return data, nil

}

func main() {

	exporter := NewExporter()

	var listenAddress, metricPath string

	flag.StringVar(&exporter.api.apiKey, "api.key", "", "NewRelic API key")
	flag.StringVar(&exporter.api.server, "api.server", "https://api.newrelic.com", "NewRelic API URL")
	flag.IntVar(&exporter.api.period, "api.period", 60, "Period of data to extract in seconds")

	flag.StringVar(&listenAddress, "web.listen-address", ":9126", "Address to listen on for web interface and telemetry.")
	flag.StringVar(&metricPath, "web.telemetry-path", "/metrics", "Path under which to expose metrics.")

	flag.Parse()

	prometheus.MustRegister(exporter)

	http.Handle(metricPath, prometheus.Handler())
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(`
<title>NewRelic exporter</title>

NewRelic exporter

<a href='` + metricPath + `'>Metrics

`))
	})

	log.Printf("Listening on %s.", listenAddress)
	err := http.ListenAndServe(listenAddress, nil)
	if err != nil {
		log.Fatal(err)
	}
	log.Print("HTTP server stopped.")
}
<|endoftext|>


<commit_before>//+build !windows

package utils

import (
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/docker/docker/api/types"
	"github.com/patrickmn/go-cache"
	"github.com/pkg/errors"
	"github.com/rancher/agent/utilities/constants"
	"github.com/vishvananda/netlink"
	"github.com/vishvananda/netns"
)

const (
	cniLabels = "io.rancher.cni.network"
	linkName  = "eth0"
)

func getIP(inspect types.ContainerJSON, c *cache.Cache) (string, error) {
	if inspect.Config.Labels[cniLabels] != "" && c != nil {
		cacheIP, ok := c.Get(inspect.Config.Labels[constants.UUIDLabel])
		if ok && InterfaceToString(cacheIP) == "error" {
			c.Delete(inspect.Config.Labels[constants.UUIDLabel])
			return "", errors.New("Timeout getting IP address")
		} else if ok {
			c.Delete(inspect.Config.Labels[constants.UUIDLabel])
			return InterfaceToString(cacheIP), nil
		}
		ip, err := lookUpIP(inspect)
		if err != nil {
			c.Add(inspect.Config.Labels[constants.UUIDLabel], "error", cache.DefaultExpiration)
			return "", err
		}
		c.Add(inspect.Config.Labels[constants.UUIDLabel], ip, cache.DefaultExpiration)
		return ip, nil
	}
	return inspect.NetworkSettings.IPAddress, nil
}

func lookUpIP(inspect types.ContainerJSON) (string, error) {
	// if container is stopped just return empty ip
	if inspect.State.Pid == 0 {
		return "", nil
	}
	endTime := time.Now().Add(30 * time.Second)
	initTime := 250 * time.Millisecond
	maxTime := 2 * time.Second
	for {
		ip, err := getIPForPID(inspect.State.Pid)
		if err != nil || ip != "" {
			return ip, err
		}

		logrus.Debugf("Sleeping %v (%v remaining) waiting for IP on %s", initTime, endTime.Sub(time.Now()), inspect.ID)
		time.Sleep(initTime)
		initTime = initTime * 2
		if initTime.Seconds() > maxTime.Seconds() {
			initTime = maxTime
		}
		if time.Now().After(endTime) {
			return "", errors.New("Timeout getting IP address")
		}
	}
}

func getIPForPID(pid int) (string, error) {
	nsHandler, err := netns.GetFromPid(pid)
	if err != nil {
		return "", err
	}
	defer nsHandler.Close()
	handler, err := netlink.NewHandleAt(nsHandler)
	if err != nil {
		return "", err
	}
	defer handler.Delete()
	link, err := handler.LinkByName(linkName)
	if err != nil {
		// Don't return error, it's expected this may fail until iface is created
		return "", nil
	}
	addrs, err := handler.AddrList(link, netlink.FAMILY_V4)
	if err != nil {
		return "", err
	}
	if len(addrs) > 0 {
		return addrs[0].IP.String(), nil
	}
	return "", nil
}
<commit_msg>Check state file for CNI IP and error<commit_after>//+build !windows

package utils

import (
	"encoding/json"
	"io/ioutil"
	"os"
	"path"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/docker/docker/api/types"
	"github.com/patrickmn/go-cache"
	"github.com/pkg/errors"
	"github.com/rancher/agent/utilities/constants"
	"github.com/vishvananda/netlink"
	"github.com/vishvananda/netns"
)

const (
	cniLabels       = "io.rancher.cni.network"
	linkName        = "eth0"
	cniStateBaseDir = "/var/lib/rancher/state/cni"
)

func getIP(inspect types.ContainerJSON, c *cache.Cache) (string, error) {
	if inspect.Config.Labels[cniLabels] != "" && c != nil {
		cacheIP, ok := c.Get(inspect.Config.Labels[constants.UUIDLabel])
		if ok && InterfaceToString(cacheIP) == "error" {
			c.Delete(inspect.Config.Labels[constants.UUIDLabel])
			return "", errors.New("Timeout getting IP address")
		} else if ok {
			c.Delete(inspect.Config.Labels[constants.UUIDLabel])
			return InterfaceToString(cacheIP), nil
		}
		ip, err := lookUpIP(inspect)
		if err != nil {
			c.Add(inspect.Config.Labels[constants.UUIDLabel], "error", cache.DefaultExpiration)
			return "", err
		}
		c.Add(inspect.Config.Labels[constants.UUIDLabel], ip, cache.DefaultExpiration)
		return ip, nil
	}
	return inspect.NetworkSettings.IPAddress, nil
}

func lookUpIP(inspect types.ContainerJSON) (string, error) {
	// if container is stopped just return empty ip
	if inspect.State.Pid == 0 {
		return "", nil
	}
	endTime := time.Now().Add(30 * time.Second)
	initTime := 250 * time.Millisecond
	maxTime := 2 * time.Second
	for {
		if ip, cniError := getIPFromStateFile(inspect); ip != "" || cniError != "" {
			var err error
			if cniError != "" {
				err = errors.New(cniError)
			}
			return ip, err
		}

		ip, err := getIPForPID(inspect.State.Pid)
		if err != nil || ip != "" {
			return ip, err
		}

		logrus.Debugf("Sleeping %v (%v remaining) waiting for IP on %s", initTime, endTime.Sub(time.Now()), inspect.ID)
		time.Sleep(initTime)
		initTime = initTime * 2
		if initTime.Seconds() > maxTime.Seconds() {
			initTime = maxTime
		}
		if time.Now().After(endTime) {
			return "", errors.New("Timeout getting IP address")
		}
	}
}

func getIPFromStateFile(inspect types.ContainerJSON) (string, string) {
	if inspect.ID == "" || inspect.State == nil || inspect.State.StartedAt == "" {
		return "", ""
	}
	filename := path.Join(cniStateBaseDir, inspect.ID, inspect.State.StartedAt)

	data, err := ioutil.ReadFile(filename)
	if err != nil {
		if !os.IsNotExist(err) {
			logrus.Warnf("Error reading cni state file %v: %v. Falling back to container inspection logic.", filename, err)
		}
		return "", ""
	}

	var state cniState
	if err := json.Unmarshal(data, &state); err != nil {
		logrus.Warnf("Error unmarshalling cni state data %s: %v. Falling back to container inspection logic.", data, err)
		return "", ""
	}

	return state.IP4.IP, state.Error
}

type cniState struct {
	Error string
	IP4   struct {
		IP string
	}
}

func getIPForPID(pid int) (string, error) {
	nsHandler, err := netns.GetFromPid(pid)
	if err != nil {
		return "", err
	}
	defer nsHandler.Close()
	handler, err := netlink.NewHandleAt(nsHandler)
	if err != nil {
		return "", err
	}
	defer handler.Delete()
	link, err := handler.LinkByName(linkName)
	if err != nil {
		// Don't return error, it's expected this may fail until iface is created
		return "", nil
	}
	addrs, err := handler.AddrList(link, netlink.FAMILY_V4)
	if err != nil {
		return "", err
	}
	if len(addrs) > 0 {
		return addrs[0].IP.String(), nil
	}
	return "", nil
}
<|endoftext|>


<commit_before>//
// tun.go -- tun interface with cgo for linux / bsd
//

package samtun

/*

#include <unistd.h>
#include <fcntl.h>
#include <netinet/in.h>
#include <netinet/ip.h>
#include <arpa/inet.h>
#include <sys/ioctl.h>
#include <sys/socket.h>
#include <sys/types.h>
#include <linux/if.h>
#include <linux/if_tun.h>

int tundev_open(char * ifname) {
  int fd = open("/dev/net/tun", O_RDWR);
  if (fd > 0) {
    struct ifreq ifr;
    memset(&ifr, 0, sizeof(ifr));
    ifr.ifr_flags = IFF_TUN | IFF_NO_PI;
    strncpy(ifr.ifr_name, ifname, IFNAMSIZ);
    if ( ioctl(fd , TUNSETIFF, (void*) &ifr) < 0) {
      close(fd);
      return -1;
    }
  }
  return fd;
}

int tundev_up(char * ifname, char * addr, char * dstaddr, int mtu) {

  struct ifreq ifr;
  memset(&ifr, 0, sizeof(ifr));
  strncpy(ifr.ifr_name, ifname, IFNAMSIZ);
  int fd = socket(AF_INET6, SOCK_DGRAM, IPPROTO_IP);
  if ( fd > 0 ) {
    if ( ioctl(fd, SIOCGIFINDEX, &ifr) < 0 ) {
      close(fd);
      return -1;
    }
    struct sockaddr_in dst;
    memset(&dst, 0, sizeof(dst));
    inet_aton(dstaddr, &dst.sin_addr);
    struct sockaddr_in src;
    memset(&src, 0, sizeof(src));
    inet_aton(addr, &src.sin_addr);
    memcpy(&ifr.ifr_addr, &src, sizeof(src));
    memcpy(&ifr.ifr_dstaddr, &dst, sizeof(dst));
    if ( ioctl(fd, SIOCSIFADDR, &ifr) < 0 ) {
      close(fd);
      return -1;
    }
    ifr.ifr_mtu = mtu;
    if ( ioctl(fd, SIOCSIFMTU, &ifr) < 0) {
      close(fd);
      return -1;
    }
    if ( ioctl(fd, SIOCGIFFLAGS, &ifr) < 0 ) {
      close(fd);
      return -1;
    }
    ifr.ifr_flags |= IFF_UP | IFF_RUNNING;
    if ( ioctl(fd, SIOCSIFFLAGS, &ifr) < 0 ) {
      close(fd);
      return -1;
    }
    close(fd);
    return 0;
  }
  return -1;
}

void tundev_close(int fd) {
  close(fd);
}

*/
import "C"

type tunDev C.int

func newTun(ifname, addr, dstaddr string) (t tunDev, err error) {
  t = C.tundev_open(C.CString(ifname))
  if t == -1 {
    err = errors.New("cannot open tun interface")
  } else {
    if C.tundev_up(C.CString(ifname), C.CString(addr), C.CString(dstaddr), C.int(mtu)) < 0 {
      err = errors.New("cannot put up interface")
    }
  }
  return
}

func (d tunDev) Close() {
  C.tundev_close(d)
}
<commit_msg>add string.h include<commit_after>//
// tun.go -- tun interface with cgo for linux / bsd
//

package samtun

/*

#include <string.h>
#include <unistd.h>
#include <fcntl.h>
#include <netinet/in.h>
#include <netinet/ip.h>
#include <arpa/inet.h>
#include <sys/ioctl.h>
#include <sys/socket.h>
#include <sys/types.h>
#include <linux/if.h>
#include <linux/if_tun.h>

int tundev_open(char * ifname) {
  int fd = open("/dev/net/tun", O_RDWR);
  if (fd > 0) {
    struct ifreq ifr;
    memset(&ifr, 0, sizeof(ifr));
    ifr.ifr_flags = IFF_TUN | IFF_NO_PI;
    strncpy(ifr.ifr_name, ifname, IFNAMSIZ);
    if ( ioctl(fd , TUNSETIFF, (void*) &ifr) < 0) {
      close(fd);
      return -1;
    }
  }
  return fd;
}

int tundev_up(char * ifname, char * addr, char * dstaddr, int mtu) {

  struct ifreq ifr;
  memset(&ifr, 0, sizeof(ifr));
  strncpy(ifr.ifr_name, ifname, IFNAMSIZ);
  int fd = socket(AF_INET6, SOCK_DGRAM, IPPROTO_IP);
  if ( fd > 0 ) {
    if ( ioctl(fd, SIOCGIFINDEX, &ifr) < 0 ) {
      close(fd);
      return -1;
    }
    struct sockaddr_in dst;
    memset(&dst, 0, sizeof(dst));
    inet_aton(dstaddr, &dst.sin_addr);
    struct sockaddr_in src;
    memset(&src, 0, sizeof(src));
    inet_aton(addr, &src.sin_addr);
    memcpy(&ifr.ifr_addr, &src, sizeof(src));
    memcpy(&ifr.ifr_dstaddr, &dst, sizeof(dst));
    if ( ioctl(fd, SIOCSIFADDR, &ifr) < 0 ) {
      close(fd);
      return -1;
    }
    ifr.ifr_mtu = mtu;
    if ( ioctl(fd, SIOCSIFMTU, &ifr) < 0) {
      close(fd);
      return -1;
    }
    if ( ioctl(fd, SIOCGIFFLAGS, &ifr) < 0 ) {
      close(fd);
      return -1;
    }
    ifr.ifr_flags |= IFF_UP | IFF_RUNNING;
    if ( ioctl(fd, SIOCSIFFLAGS, &ifr) < 0 ) {
      close(fd);
      return -1;
    }
    close(fd);
    return 0;
  }
  return -1;
}

void tundev_close(int fd) {
  close(fd);
}

*/
import "C"

type tunDev C.int

func newTun(ifname, addr, dstaddr string) (t tunDev, err error) {
  t = C.tundev_open(C.CString(ifname))
  if t == -1 {
    err = errors.New("cannot open tun interface")
  } else {
    if C.tundev_up(C.CString(ifname), C.CString(addr), C.CString(dstaddr), C.int(mtu)) < 0 {
      err = errors.New("cannot put up interface")
    }
  }
  return
}

func (d tunDev) Close() {
  C.tundev_close(d)
}
<|endoftext|>


<commit_before>// Copyright 2015 Unknwon
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package setting

import (
	"github.com/Unknwon/com"
	"github.com/Unknwon/log"
	"gopkg.in/ini.v1"
	"gopkg.in/macaron.v1"

	"github.com/peachdocs/peach/modules/bindata"
)

type NavbarItem struct {
	Icon         string
	Locale, Link string
	Blank        bool
}

const (
	LOCAL  = "local"
	REMOTE = "remote"
)

type DocType string

func (t DocType) IsLocal() bool {
	return t == LOCAL
}

func (t DocType) IsRemote() bool {
	return t == REMOTE
}

var (
	CustomConf = "custom/app.ini"

	AppVer   string
	ProdMode bool
	HTTPPort int

	Site struct {
		Name   string
		Desc   string
		UseCDN bool
		URL    string
	}

	Page struct {
		HasLandingPage bool
		DocsBaseURL    string

		UseCustomTpl   bool
		NavbarTplPath  string
		HomeTplPath    string
		DocsTplPath    string
		FooterTplPath  string
		DisqusTplPath  string
		DuoShuoTplPath string
	}

	Navbar struct {
		Items []*NavbarItem
	}

	Asset struct {
		CustomCSS string
	}

	Docs struct {
		Type   DocType
		Target string
		Secret string
		Langs  []string

		// Only used for languages are not en-US or zh-CN to bypass error panic.
		Locales map[string][]byte
	}

	Extension struct {
		EnableEditPage       bool
		EditPageLinkFormat   string
		EnableDisqus         bool
		DisqusShortName      string
		EnableDuoShuo        bool
		DuoShuoShortName     string
		HighlightJSCustomCSS string
		EnableSearch         bool
		GABlock              string
	}

	Cfg *ini.File
)

func NewContext() {
	log.Prefix = "[Peach]"

	if !com.IsFile(CustomConf) {
		log.Fatal("No custom configuration found: 'custom/app.ini'")
	}
	sources := []interface{}{bindata.MustAsset("conf/app.ini"), CustomConf}

	var err error
	Cfg, err = macaron.SetConfig(sources[0], sources[1:]...)
	if err != nil {
		log.Fatal("Fail to load config: %v", err)
	}

	sec := Cfg.Section("")
	if sec.Key("RUN_MODE").String() == "prod" {
		ProdMode = true
		macaron.Env = macaron.PROD
		macaron.ColorLog = false
	}

	HTTPPort = sec.Key("HTTP_PORT").MustInt(5555)

	sec = Cfg.Section("site")
	Site.Name = sec.Key("NAME").MustString("Peach Server")
	Site.Desc = sec.Key("DESC").String()
	Site.UseCDN = sec.Key("USE_CDN").MustBool()
	Site.URL = sec.Key("URL").String()

	sec = Cfg.Section("page")
	Page.HasLandingPage = sec.Key("HAS_LANDING_PAGE").MustBool()
	Page.DocsBaseURL = sec.Key("DOCS_BASE_URL").Validate(func(in string) string {
		if len(in) == 0 {
			return "/docs"
		} else if in[0] != '/' {
			return "/" + in
		}
		return in
	})

	Page.UseCustomTpl = sec.Key("USE_CUSTOM_TPL").MustBool()
	Page.NavbarTplPath = "navbar.html"
	Page.HomeTplPath = "home.html"
	Page.DocsTplPath = "docs.html"
	Page.FooterTplPath = "footer.html"
	Page.DisqusTplPath = "disqus.html"
	Page.DuoShuoTplPath = "duoshuo.html"

	sec = Cfg.Section("navbar")
	list := sec.KeyStrings()
	Navbar.Items = make([]*NavbarItem, len(list))
	for i, name := range list {
		secName := "navbar." + sec.Key(name).String()
		Navbar.Items[i] = &NavbarItem{
			Icon:   Cfg.Section(secName).Key("ICON").String(),
			Locale: Cfg.Section(secName).Key("LOCALE").MustString(secName),
			Link:   Cfg.Section(secName).Key("LINK").MustString("/"),
			Blank:  Cfg.Section(secName).Key("BLANK").MustBool(),
		}
	}

	sec = Cfg.Section("asset")
	Asset.CustomCSS = sec.Key("CUSTOM_CSS").String()

	sec = Cfg.Section("docs")
	Docs.Type = DocType(sec.Key("TYPE").In("local", []string{LOCAL, REMOTE}))
	Docs.Target = sec.Key("TARGET").String()
	Docs.Secret = sec.Key("SECRET").String()
	Docs.Langs = Cfg.Section("i18n").Key("LANGS").Strings(",")
	Docs.Locales = make(map[string][]byte)
	for _, lang := range Docs.Langs {
		if lang == "en-US" || lang == "zh-CN" {
			Docs.Locales["locale_"+lang+".ini"] = bindata.MustAsset("conf/locale/locale_" + lang + ".ini")
		} else {
			Docs.Locales["locale_"+lang+".ini"] = []byte("")
		}
	}

	sec = Cfg.Section("extension")
	Extension.EnableEditPage = sec.Key("ENABLE_EDIT_PAGE").MustBool()
	Extension.EditPageLinkFormat = sec.Key("EDIT_PAGE_LINK_FORMAT").String()
	Extension.EnableDisqus = sec.Key("ENABLE_DISQUS").MustBool()
	Extension.DisqusShortName = sec.Key("DISQUS_SHORT_NAME").String()
	Extension.EnableDuoShuo = sec.Key("ENABLE_DUOSHUO").MustBool()
	Extension.DuoShuoShortName = sec.Key("DUOSHUO_SHORT_NAME").String()
	Extension.HighlightJSCustomCSS = sec.Key("HIGHLIGHTJS_CUSTOM_CSS").String()
	Extension.EnableSearch = sec.Key("ENABLE_SEARCH").MustBool()
	Extension.GABlock = sec.Key("GA_BLOCK").String()
}
<commit_msg>update - 文档语言类型支持.<commit_after>// Copyright 2015 Unknwon
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package setting

import (
	"github.com/Unknwon/com"
	"github.com/Unknwon/log"
	"gopkg.in/ini.v1"
	"gopkg.in/macaron.v1"

	"github.com/peachdocs/peach/modules/bindata"
)

type NavbarItem struct {
	Icon         string
	Locale, Link string
	Blank        bool
}

const (
	LOCAL  = "local"
	REMOTE = "remote"
)

type DocType string

func (t DocType) IsLocal() bool {
	return t == LOCAL
}

func (t DocType) IsRemote() bool {
	return t == REMOTE
}

var (
	CustomConf = "custom/app.ini"

	AppVer   string
	ProdMode bool
	HTTPPort int

	Site struct {
		Name   string
		Desc   string
		UseCDN bool
		URL    string
	}

	Page struct {
		HasLandingPage bool
		DocsBaseURL    string

		UseCustomTpl   bool
		NavbarTplPath  string
		HomeTplPath    string
		DocsTplPath    string
		FooterTplPath  string
		DisqusTplPath  string
		DuoShuoTplPath string
	}

	Navbar struct {
		Items []*NavbarItem
	}

	Asset struct {
		CustomCSS string
	}

	Docs struct {
		Type   DocType
		Target string
		Secret string
		Langs  []string		// 文档语言类型, eg: 中文

		// Only used for languages are not en-US or zh-CN to bypass error panic.
		Locales map[string][]byte
	}

	Extension struct {
		EnableEditPage       bool
		EditPageLinkFormat   string
		EnableDisqus         bool
		DisqusShortName      string
		EnableDuoShuo        bool
		DuoShuoShortName     string
		HighlightJSCustomCSS string
		EnableSearch         bool
		GABlock              string
	}

	Cfg *ini.File
)

func NewContext() {
	log.Prefix = "[Peach]"

	if !com.IsFile(CustomConf) {
		log.Fatal("No custom configuration found: 'custom/app.ini'")
	}
	sources := []interface{}{bindata.MustAsset("conf/app.ini"), CustomConf}

	var err error
	Cfg, err = macaron.SetConfig(sources[0], sources[1:]...)
	if err != nil {
		log.Fatal("Fail to load config: %v", err)
	}

	sec := Cfg.Section("")
	if sec.Key("RUN_MODE").String() == "prod" {
		ProdMode = true
		macaron.Env = macaron.PROD
		macaron.ColorLog = false
	}

	HTTPPort = sec.Key("HTTP_PORT").MustInt(5555)

	sec = Cfg.Section("site")
	Site.Name = sec.Key("NAME").MustString("Peach Server")
	Site.Desc = sec.Key("DESC").String()
	Site.UseCDN = sec.Key("USE_CDN").MustBool()
	Site.URL = sec.Key("URL").String()

	sec = Cfg.Section("page")
	Page.HasLandingPage = sec.Key("HAS_LANDING_PAGE").MustBool()
	Page.DocsBaseURL = sec.Key("DOCS_BASE_URL").Validate(func(in string) string {
		if len(in) == 0 {
			return "/docs"
		} else if in[0] != '/' {
			return "/" + in
		}
		return in
	})

	Page.UseCustomTpl = sec.Key("USE_CUSTOM_TPL").MustBool()
	Page.NavbarTplPath = "navbar.html"
	Page.HomeTplPath = "home.html"
	Page.DocsTplPath = "docs.html"
	Page.FooterTplPath = "footer.html"
	Page.DisqusTplPath = "disqus.html"
	Page.DuoShuoTplPath = "duoshuo.html"

	sec = Cfg.Section("navbar")
	list := sec.KeyStrings()
	Navbar.Items = make([]*NavbarItem, len(list))
	for i, name := range list {
		secName := "navbar." + sec.Key(name).String()
		Navbar.Items[i] = &NavbarItem{
			Icon:   Cfg.Section(secName).Key("ICON").String(),
			Locale: Cfg.Section(secName).Key("LOCALE").MustString(secName),
			Link:   Cfg.Section(secName).Key("LINK").MustString("/"),
			Blank:  Cfg.Section(secName).Key("BLANK").MustBool(),
		}
	}

	sec = Cfg.Section("asset")
	Asset.CustomCSS = sec.Key("CUSTOM_CSS").String()

	sec = Cfg.Section("docs")
	Docs.Type = DocType(sec.Key("TYPE").In("local", []string{LOCAL, REMOTE}))
	Docs.Target = sec.Key("TARGET").String()
	Docs.Secret = sec.Key("SECRET").String()
	Docs.Langs = Cfg.Section("i18n").Key("LANGS").Strings(",")
	Docs.Locales = make(map[string][]byte)
	for _, lang := range Docs.Langs {
		if lang == "en-US" || lang == "zh-CN" {
			Docs.Locales["locale_"+lang+".ini"] = bindata.MustAsset("conf/locale/locale_" + lang + ".ini")
		} else {
			Docs.Locales["locale_"+lang+".ini"] = []byte("")
		}
	}

	sec = Cfg.Section("extension")
	Extension.EnableEditPage = sec.Key("ENABLE_EDIT_PAGE").MustBool()
	Extension.EditPageLinkFormat = sec.Key("EDIT_PAGE_LINK_FORMAT").String()
	Extension.EnableDisqus = sec.Key("ENABLE_DISQUS").MustBool()
	Extension.DisqusShortName = sec.Key("DISQUS_SHORT_NAME").String()
	Extension.EnableDuoShuo = sec.Key("ENABLE_DUOSHUO").MustBool()
	Extension.DuoShuoShortName = sec.Key("DUOSHUO_SHORT_NAME").String()
	Extension.HighlightJSCustomCSS = sec.Key("HIGHLIGHTJS_CUSTOM_CSS").String()
	Extension.EnableSearch = sec.Key("ENABLE_SEARCH").MustBool()
	Extension.GABlock = sec.Key("GA_BLOCK").String()
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/mozilla/tls-observatory/database"
)

type scan struct {
	ID string `json:"scan_id"`
}

func main() {
	var observatory = flag.String("observatory", "https://tls-observatory.services.mozilla.com", "URL of the observatory")
	flag.Parse()
	db, err := database.RegisterConnection(
		os.Getenv("TLSOBS_POSTGRESDB"),
		os.Getenv("TLSOBS_POSTGRESUSER"),
		os.Getenv("TLSOBS_POSTGRESPASS"),
		os.Getenv("TLSOBS_POSTGRES"),
		"require")
	defer db.Close()
	if err != nil {
		panic(err)
	}
	// batch side: do 100 certs at a time
	limit := 100
	batch := 0
	var donedomains []string
	for {
		fmt.Printf("\nProcessing batch %d to %d\n", batch*limit, batch*limit+limit)
		rows, err := db.Query(`	SELECT domains
					FROM certificates INNER JOIN trust ON (trust.cert_id=certificates.id)
					WHERE is_ca='false' AND trusted_mozilla='true'
					ORDER BY certificates.id ASC LIMIT $1 OFFSET $2`, limit, batch*limit)
		if rows != nil {
			defer rows.Close()
		}
		if err != nil {
			panic(fmt.Errorf("Error while retrieving certs: '%v'", err))
		}
		i := 0
		for rows.Next() {
			i++
			var domains string
			err = rows.Scan(&domains)
			if err != nil {
				fmt.Println("error while retrieving domains:", err)
				continue
			}
			for _, domain := range strings.Split(domains, ",") {
				domain = strings.TrimSpace(domain)
				if domain == "" {
					continue
				}
				if domain[0] == '*' {
					domain = "www" + domain[1:]
				}
				if contains(donedomains, domain) {
					continue
				}
				resp, err := http.Post(*observatory+"/api/v1/scan?target="+domain, "application/json", nil)
				if err != nil {
					panic(err)
				}
				defer resp.Body.Close()
				body, err := ioutil.ReadAll(resp.Body)
				if err != nil {
					panic(err)
				}
				var scan scan
				err = json.Unmarshal(body, &scan)
				if err != nil {
					panic(err)
				}
				fmt.Printf("Started scan %s on %s - %s/api/v1/results?id=%s\n", scan.ID, domain, *observatory, scan.ID)
				donedomains = append(donedomains, domain)
				time.Sleep(500 * time.Millisecond)
			}
		}
		if i == 0 {
			fmt.Println("done!")
			break
		}
		batch++
	}
}

func contains(list []string, test string) bool {
	for _, item := range list {
		if item == test {
			return true
		}
	}
	return false
}
<commit_msg>Fix scan ID in rescanDomains<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/mozilla/tls-observatory/database"
)

type scan struct {
	ID int `json:"scan_id"`
}

func main() {
	var observatory = flag.String("observatory", "https://tls-observatory.services.mozilla.com", "URL of the observatory")
	flag.Parse()
	db, err := database.RegisterConnection(
		os.Getenv("TLSOBS_POSTGRESDB"),
		os.Getenv("TLSOBS_POSTGRESUSER"),
		os.Getenv("TLSOBS_POSTGRESPASS"),
		os.Getenv("TLSOBS_POSTGRES"),
		"require")
	defer db.Close()
	if err != nil {
		panic(err)
	}
	// batch side: do 100 certs at a time
	limit := 100
	batch := 0
	var donedomains []string
	for {
		fmt.Printf("\nProcessing batch %d to %d\n", batch*limit, batch*limit+limit)
		rows, err := db.Query(`	SELECT domains
					FROM certificates INNER JOIN trust ON (trust.cert_id=certificates.id)
					WHERE is_ca='false' AND trusted_mozilla='true'
					ORDER BY certificates.id ASC LIMIT $1 OFFSET $2`, limit, batch*limit)
		if rows != nil {
			defer rows.Close()
		}
		if err != nil {
			panic(fmt.Errorf("Error while retrieving certs: '%v'", err))
		}
		i := 0
		for rows.Next() {
			i++
			var domains string
			err = rows.Scan(&domains)
			if err != nil {
				fmt.Println("error while retrieving domains:", err)
				continue
			}
			for _, domain := range strings.Split(domains, ",") {
				domain = strings.TrimSpace(domain)
				if domain == "" {
					continue
				}
				if domain[0] == '*' {
					domain = "www" + domain[1:]
				}
				if contains(donedomains, domain) {
					continue
				}
				resp, err := http.Post(*observatory+"/api/v1/scan?target="+domain, "application/json", nil)
				if err != nil {
					panic(err)
				}
				defer resp.Body.Close()
				body, err := ioutil.ReadAll(resp.Body)
				if err != nil {
					panic(err)
				}
				var scan scan
				err = json.Unmarshal(body, &scan)
				if err != nil {
					panic(err)
				}
				fmt.Printf("Started scan %d on %s - %s/api/v1/results?id=%d\n", scan.ID, domain, *observatory, scan.ID)
				donedomains = append(donedomains, domain)
				time.Sleep(500 * time.Millisecond)
			}
		}
		if i == 0 {
			fmt.Println("done!")
			break
		}
		batch++
	}
}

func contains(list []string, test string) bool {
	for _, item := range list {
		if item == test {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package transport_test

import (
	"bufio"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"strings"
	"time"

	"code.cloudfoundry.org/garden"
	gconn "code.cloudfoundry.org/garden/client/connection"
	"github.com/concourse/atc/dbng/dbngfakes"
	"github.com/concourse/atc/worker/transport"
	"github.com/concourse/atc/worker/transport/transportfakes"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/tedsuo/rata"

	"github.com/concourse/atc/dbng"
	"github.com/concourse/retryhttp/retryhttpfakes"
)

var _ = Describe("hijackStreamer", func() {
	var (
		savedWorker          *dbngfakes.FakeWorker
		savedWorkerAddress   string
		fakeDB               *transportfakes.FakeTransportDB
		fakeRoundTripper     *transportfakes.FakeRoundTripper
		fakeHijackableClient *retryhttpfakes.FakeHijackableClient
		hijackStreamer       gconn.HijackStreamer
		fakeRequestGenerator *transportfakes.FakeRequestGenerator
		handler              string
		params               rata.Params
		query                url.Values
		contentType          string
	)
	BeforeEach(func() {
		savedWorkerAddress = "some-garden-addr"

		savedWorker = new(dbngfakes.FakeWorker)

		savedWorker.GardenAddrReturns(&savedWorkerAddress)
		savedWorker.ExpiresAtReturns(time.Now().Add(123 * time.Minute))
		savedWorker.StateReturns(dbng.WorkerStateRunning)

		fakeDB = new(transportfakes.FakeTransportDB)
		fakeDB.GetWorkerReturns(savedWorker, true, nil)

		fakeRequestGenerator = new(transportfakes.FakeRequestGenerator)

		fakeRoundTripper = new(transportfakes.FakeRoundTripper)
		fakeHijackableClient = new(retryhttpfakes.FakeHijackableClient)

		hijackStreamer = &transport.WorkerHijackStreamer{
			HttpClient:       &http.Client{Transport: fakeRoundTripper},
			HijackableClient: fakeHijackableClient,
			Req:              fakeRequestGenerator,
		}

		handler = "Ping"
		params = map[string]string{"param1": "value1"}
		contentType = "application/json"
		query = map[string][]string{"key": []string{"some", "values"}}

		request, err := http.NewRequest("POST", "http://example.url", strings.NewReader("some-request-body"))
		Expect(err).NotTo(HaveOccurred())
		fakeRequestGenerator.CreateRequestReturns(request, nil)
	})

	Describe("hijackStreamer #Stream", func() {
		var (
			body             io.Reader
			actualReadCloser io.ReadCloser
			streamErr        error
			httpResp         http.Response
			expectedString   string
		)

		BeforeEach(func() {
			expectedString = "some-example-string"
			body = strings.NewReader(expectedString)

			fakeRoundTripper.RoundTripReturns(&httpResp, nil)
		})

		JustBeforeEach(func() {
			actualReadCloser, streamErr = hijackStreamer.Stream(handler, body, params, query, contentType)
		})

		Context("when httpResponse is success", func() {
			BeforeEach(func() {
				httpResp = http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(body)}
			})

			It("returns response body", func() {
				actualBodyBytes, err := ioutil.ReadAll(actualReadCloser)
				Expect(err).ToNot(HaveOccurred())
				Expect(expectedString).To(Equal(string(actualBodyBytes)))
			})
		})

		Context("when httpResponse is not success", func() {
			var fakeBody *transportfakes.FakeReadCloser

			BeforeEach(func() {
				fakeBody = new(transportfakes.FakeReadCloser)
				httpResp = http.Response{StatusCode: http.StatusTeapot, Body: fakeBody}

				bodyBuf, _ := json.Marshal(garden.Error{Err: errors.New("some-error")})
				realBody := strings.NewReader(string(bodyBuf))
				fakeBody.ReadStub = func(buf []byte) (int, error) {
					Expect(fakeBody.CloseCallCount()).To(BeZero())
					return realBody.Read(buf)
				}
			})

			It("closes httpResp.Body and returns error", func() {
				Expect(actualReadCloser).To(BeNil())
				Expect(streamErr).To(HaveOccurred())
				Expect(fakeBody.CloseCallCount()).To(Equal(1))
				Expect(streamErr).To(MatchError("some-error"))
			})
		})

		Context("when httpResponse is not success with bad response", func() {
			var fakeBody *transportfakes.FakeReadCloser

			BeforeEach(func() {
				fakeBody = new(transportfakes.FakeReadCloser)
				httpResp = http.Response{StatusCode: http.StatusTeapot, Body: fakeBody}

				realBody := strings.NewReader("some-error")
				fakeBody.ReadStub = func(buf []byte) (int, error) {
					Expect(fakeBody.CloseCallCount()).To(BeZero())
					return realBody.Read(buf)
				}
			})

			It("closes httpResp.Body and returns bad response", func() {
				Expect(actualReadCloser).To(BeNil())
				Expect(streamErr).To(HaveOccurred())
				Expect(fakeBody.CloseCallCount()).To(Equal(1))
				Expect(streamErr).To(MatchError(fmt.Errorf("bad response: %s", errors.New("invalid character 's' looking for beginning of value"))))
			})
		})

		Context("when httpResponse fails", func() {
			BeforeEach(func() {
				httpResp = http.Response{StatusCode: http.StatusTeapot, Body: ioutil.NopCloser(body)}
			})

			It("returns error", func() {
				Expect(actualReadCloser).To(BeNil())
				Expect(streamErr).To(HaveOccurred())
			})

			It("creates request with the right arguments", func() {
				Expect(fakeRequestGenerator.CreateRequestCallCount()).To(Equal(1))
				actualHandler, actualParams, actualBody := fakeRequestGenerator.CreateRequestArgsForCall(0)
				Expect(actualHandler).To(Equal(handler))
				Expect(actualParams).To(Equal(params))
				Expect(actualBody).To(Equal(body))
			})

			It("httpClient makes the right request", func() {
				expectedRequest, err := http.NewRequest("POST", "http://example.url", strings.NewReader("some-request-body"))
				expectedRequest.Header.Add("Content-Type", "application/json")
				expectedRequest.URL.RawQuery = "key=some&key=values"
				Expect(err).NotTo(HaveOccurred())
				Expect(fakeRoundTripper.RoundTripCallCount()).To(Equal(1))
				actualRequest := fakeRoundTripper.RoundTripArgsForCall(0)
				Expect(actualRequest).To(Equal(expectedRequest))
			})
		})
	})

	Describe("hijackStreamer #Hijack", func() {
		var (
			body                   io.Reader
			hijackError            error
			httpResp               http.Response
			expectedString         string
			actualHijackedConn     net.Conn
			actualResponseReader   *bufio.Reader
			fakeHijackCloser       *retryhttpfakes.FakeHijackCloser
			expectedResponseReader *bufio.Reader
			fakeHijackedConn       *retryhttpfakes.FakeConn
		)

		BeforeEach(func() {
			expectedResponseReader = new(bufio.Reader)
			fakeHijackedConn = new(retryhttpfakes.FakeConn)

			expectedString = "some-example-string"
			body = strings.NewReader(expectedString)
			fakeHijackCloser = new(retryhttpfakes.FakeHijackCloser)
		})

		JustBeforeEach(func() {
			actualHijackedConn, actualResponseReader, hijackError = hijackStreamer.Hijack(handler, body, params, query, contentType)
		})

		Context("when request is successful", func() {
			BeforeEach(func() {
				fakeHijackableClient.DoReturns(&httpResp, fakeHijackCloser, nil)
				httpResp = http.Response{StatusCode: http.StatusOK}
				fakeHijackCloser.HijackReturns(fakeHijackedConn, expectedResponseReader)
			})

			It("returns success response and hijackCloser", func() {
				Expect(hijackError).ToNot(HaveOccurred())
				Expect(fakeHijackCloser.HijackCallCount()).To(Equal(1))
				Expect(actualHijackedConn).To(Equal(fakeHijackedConn))
				Expect(actualResponseReader).To(Equal(expectedResponseReader))
			})
		})

		Context("when httpResponse is not success", func() {
			var fakeBody *transportfakes.FakeReadCloser

			BeforeEach(func() {
				fakeHijackableClient.DoReturns(&httpResp, fakeHijackCloser, nil)
				fakeBody = new(transportfakes.FakeReadCloser)
				httpResp = http.Response{StatusCode: http.StatusTeapot, Body: fakeBody}

				realBody := strings.NewReader("some-error")
				fakeBody.ReadStub = func(buf []byte) (int, error) {
					Expect(fakeBody.CloseCallCount()).To(BeZero())
					return realBody.Read(buf)
				}
			})

			It("closes httpResp.Body, hijackCloser and returns error", func() {
				Expect(fakeHijackCloser).NotTo(BeNil())
				Expect(hijackError).To(HaveOccurred())
				Expect(fakeBody.CloseCallCount()).To(Equal(1))
				Expect(hijackError).To(MatchError(fmt.Errorf("Backend error: Exit status: %d, message: %s", httpResp.StatusCode, "some-error")))
				Expect(fakeHijackCloser.CloseCallCount()).To(Equal(1))
			})
		})

		Context("when httpResponse is not success with bad response", func() {
			var fakeBody *transportfakes.FakeReadCloser

			BeforeEach(func() {
				fakeHijackableClient.DoReturns(&httpResp, fakeHijackCloser, nil)
				fakeBody = new(transportfakes.FakeReadCloser)
				httpResp = http.Response{StatusCode: http.StatusTeapot, Body: fakeBody}

				fakeBody.ReadStub = func(buf []byte) (int, error) {
					Expect(fakeBody.CloseCallCount()).To(BeZero())
					return 0, errors.New("error reading")
				}
			})

			It("closes httpResp.Body and returns bad response", func() {
				Expect(fakeHijackCloser).NotTo(BeNil())
				Expect(hijackError).To(HaveOccurred())
				Expect(fakeBody.CloseCallCount()).To(Equal(1))
				Expect(hijackError).To(MatchError(fmt.Errorf("Backend error: Exit status: %d, error reading response body: %s", httpResp.StatusCode, "error reading")))
				Expect(fakeHijackCloser.CloseCallCount()).To(Equal(1))
			})
		})

		Context("when httpResponse fails", func() {
			BeforeEach(func() {
				fakeHijackableClient.DoReturns(nil, nil, errors.New("Request failed"))
				httpResp = http.Response{StatusCode: http.StatusTeapot, Body: ioutil.NopCloser(body)}
			})

			It("returns error", func() {
				Expect(hijackError).To(HaveOccurred())
				Expect(actualHijackedConn).To(BeNil())
				Expect(actualResponseReader).To(BeNil())
			})

			It("makes the right request", func() {
				expectedRequest, err := http.NewRequest("POST", "http://example.url", strings.NewReader("some-request-body"))
				expectedRequest.Header.Add("Content-Type", "application/json")
				expectedRequest.URL.RawQuery = "key=some&key=values"
				Expect(err).NotTo(HaveOccurred())
				Expect(fakeHijackableClient.DoCallCount()).To(Equal(1))
				actualRequest := fakeHijackableClient.DoArgsForCall(0)
				Expect(actualRequest).To(Equal(expectedRequest))
			})
		})
	})
})
<commit_msg>fixed tests to work with go 1.8<commit_after>package transport_test

import (
	"bufio"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"strings"
	"time"

	"code.cloudfoundry.org/garden"
	gconn "code.cloudfoundry.org/garden/client/connection"
	"github.com/concourse/atc/dbng/dbngfakes"
	"github.com/concourse/atc/worker/transport"
	"github.com/concourse/atc/worker/transport/transportfakes"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/tedsuo/rata"

	"github.com/concourse/atc/dbng"
	"github.com/concourse/retryhttp/retryhttpfakes"
)

var _ = Describe("hijackStreamer", func() {
	var (
		savedWorker          *dbngfakes.FakeWorker
		savedWorkerAddress   string
		fakeDB               *transportfakes.FakeTransportDB
		fakeRoundTripper     *transportfakes.FakeRoundTripper
		fakeHijackableClient *retryhttpfakes.FakeHijackableClient
		hijackStreamer       gconn.HijackStreamer
		fakeRequestGenerator *transportfakes.FakeRequestGenerator
		handler              string
		params               rata.Params
		query                url.Values
		contentType          string
	)
	BeforeEach(func() {
		savedWorkerAddress = "some-garden-addr"

		savedWorker = new(dbngfakes.FakeWorker)

		savedWorker.GardenAddrReturns(&savedWorkerAddress)
		savedWorker.ExpiresAtReturns(time.Now().Add(123 * time.Minute))
		savedWorker.StateReturns(dbng.WorkerStateRunning)

		fakeDB = new(transportfakes.FakeTransportDB)
		fakeDB.GetWorkerReturns(savedWorker, true, nil)

		fakeRequestGenerator = new(transportfakes.FakeRequestGenerator)

		fakeRoundTripper = new(transportfakes.FakeRoundTripper)
		fakeHijackableClient = new(retryhttpfakes.FakeHijackableClient)

		hijackStreamer = &transport.WorkerHijackStreamer{
			HttpClient:       &http.Client{Transport: fakeRoundTripper},
			HijackableClient: fakeHijackableClient,
			Req:              fakeRequestGenerator,
		}

		handler = "Ping"
		params = map[string]string{"param1": "value1"}
		contentType = "application/json"
		query = map[string][]string{"key": []string{"some", "values"}}

		request, err := http.NewRequest("POST", "http://example.url", strings.NewReader("some-request-body"))
		Expect(err).NotTo(HaveOccurred())
		fakeRequestGenerator.CreateRequestReturns(request, nil)
	})

	Describe("hijackStreamer #Stream", func() {
		var (
			body             io.Reader
			actualReadCloser io.ReadCloser
			streamErr        error
			httpResp         http.Response
			expectedString   string
		)

		BeforeEach(func() {
			expectedString = "some-example-string"
			body = strings.NewReader(expectedString)

			fakeRoundTripper.RoundTripReturns(&httpResp, nil)
		})

		JustBeforeEach(func() {
			actualReadCloser, streamErr = hijackStreamer.Stream(handler, body, params, query, contentType)
		})

		Context("when httpResponse is success", func() {
			BeforeEach(func() {
				httpResp = http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(body)}
			})

			It("returns response body", func() {
				actualBodyBytes, err := ioutil.ReadAll(actualReadCloser)
				Expect(err).ToNot(HaveOccurred())
				Expect(expectedString).To(Equal(string(actualBodyBytes)))
			})
		})

		Context("when httpResponse is not success", func() {
			var fakeBody *transportfakes.FakeReadCloser

			BeforeEach(func() {
				fakeBody = new(transportfakes.FakeReadCloser)
				httpResp = http.Response{StatusCode: http.StatusTeapot, Body: fakeBody}

				bodyBuf, _ := json.Marshal(garden.Error{Err: errors.New("some-error")})
				realBody := strings.NewReader(string(bodyBuf))
				fakeBody.ReadStub = func(buf []byte) (int, error) {
					Expect(fakeBody.CloseCallCount()).To(BeZero())
					return realBody.Read(buf)
				}
			})

			It("closes httpResp.Body and returns error", func() {
				Expect(actualReadCloser).To(BeNil())
				Expect(streamErr).To(HaveOccurred())
				Expect(fakeBody.CloseCallCount()).To(Equal(1))
				Expect(streamErr).To(MatchError("some-error"))
			})
		})

		Context("when httpResponse is not success with bad response", func() {
			var fakeBody *transportfakes.FakeReadCloser

			BeforeEach(func() {
				fakeBody = new(transportfakes.FakeReadCloser)
				httpResp = http.Response{StatusCode: http.StatusTeapot, Body: fakeBody}

				realBody := strings.NewReader("some-error")
				fakeBody.ReadStub = func(buf []byte) (int, error) {
					Expect(fakeBody.CloseCallCount()).To(BeZero())
					return realBody.Read(buf)
				}
			})

			It("closes httpResp.Body and returns bad response", func() {
				Expect(actualReadCloser).To(BeNil())
				Expect(streamErr).To(HaveOccurred())
				Expect(fakeBody.CloseCallCount()).To(Equal(1))
				Expect(streamErr).To(MatchError(fmt.Errorf("bad response: %s", errors.New("invalid character 's' looking for beginning of value"))))
			})
		})

		Context("when httpResponse fails", func() {
			BeforeEach(func() {
				httpResp = http.Response{StatusCode: http.StatusTeapot, Body: ioutil.NopCloser(body)}
			})

			It("returns error", func() {
				Expect(actualReadCloser).To(BeNil())
				Expect(streamErr).To(HaveOccurred())
			})

			It("creates request with the right arguments", func() {
				Expect(fakeRequestGenerator.CreateRequestCallCount()).To(Equal(1))
				actualHandler, actualParams, actualBody := fakeRequestGenerator.CreateRequestArgsForCall(0)
				Expect(actualHandler).To(Equal(handler))
				Expect(actualParams).To(Equal(params))
				Expect(actualBody).To(Equal(body))
			})

			It("httpClient makes the right request", func() {
				Expect(fakeRoundTripper.RoundTripCallCount()).To(Equal(1))
				actualRequest := fakeRoundTripper.RoundTripArgsForCall(0)
				Expect(actualRequest.Method).To(Equal("POST"))
				Expect(actualRequest.Header["Content-Type"]).To(Equal([]string{"application/json"}))
				Expect(actualRequest.URL.RawQuery).To(Equal("key=some&key=values"))
				Expect(actualRequest.URL.Host).To(Equal("example.url"))
				Expect(ioutil.ReadAll(actualRequest.Body)).To(Equal([]byte("some-request-body")))
			})
		})
	})

	Describe("hijackStreamer #Hijack", func() {
		var (
			body                   io.Reader
			hijackError            error
			httpResp               http.Response
			expectedString         string
			actualHijackedConn     net.Conn
			actualResponseReader   *bufio.Reader
			fakeHijackCloser       *retryhttpfakes.FakeHijackCloser
			expectedResponseReader *bufio.Reader
			fakeHijackedConn       *retryhttpfakes.FakeConn
		)

		BeforeEach(func() {
			expectedResponseReader = new(bufio.Reader)
			fakeHijackedConn = new(retryhttpfakes.FakeConn)

			expectedString = "some-example-string"
			body = strings.NewReader(expectedString)
			fakeHijackCloser = new(retryhttpfakes.FakeHijackCloser)
		})

		JustBeforeEach(func() {
			actualHijackedConn, actualResponseReader, hijackError = hijackStreamer.Hijack(handler, body, params, query, contentType)
		})

		Context("when request is successful", func() {
			BeforeEach(func() {
				fakeHijackableClient.DoReturns(&httpResp, fakeHijackCloser, nil)
				httpResp = http.Response{StatusCode: http.StatusOK}
				fakeHijackCloser.HijackReturns(fakeHijackedConn, expectedResponseReader)
			})

			It("returns success response and hijackCloser", func() {
				Expect(hijackError).ToNot(HaveOccurred())
				Expect(fakeHijackCloser.HijackCallCount()).To(Equal(1))
				Expect(actualHijackedConn).To(Equal(fakeHijackedConn))
				Expect(actualResponseReader).To(Equal(expectedResponseReader))
			})
		})

		Context("when httpResponse is not success", func() {
			var fakeBody *transportfakes.FakeReadCloser

			BeforeEach(func() {
				fakeHijackableClient.DoReturns(&httpResp, fakeHijackCloser, nil)
				fakeBody = new(transportfakes.FakeReadCloser)
				httpResp = http.Response{StatusCode: http.StatusTeapot, Body: fakeBody}

				realBody := strings.NewReader("some-error")
				fakeBody.ReadStub = func(buf []byte) (int, error) {
					Expect(fakeBody.CloseCallCount()).To(BeZero())
					return realBody.Read(buf)
				}
			})

			It("closes httpResp.Body, hijackCloser and returns error", func() {
				Expect(fakeHijackCloser).NotTo(BeNil())
				Expect(hijackError).To(HaveOccurred())
				Expect(fakeBody.CloseCallCount()).To(Equal(1))
				Expect(hijackError).To(MatchError(fmt.Errorf("Backend error: Exit status: %d, message: %s", httpResp.StatusCode, "some-error")))
				Expect(fakeHijackCloser.CloseCallCount()).To(Equal(1))
			})
		})

		Context("when httpResponse is not success with bad response", func() {
			var fakeBody *transportfakes.FakeReadCloser

			BeforeEach(func() {
				fakeHijackableClient.DoReturns(&httpResp, fakeHijackCloser, nil)
				fakeBody = new(transportfakes.FakeReadCloser)
				httpResp = http.Response{StatusCode: http.StatusTeapot, Body: fakeBody}

				fakeBody.ReadStub = func(buf []byte) (int, error) {
					Expect(fakeBody.CloseCallCount()).To(BeZero())
					return 0, errors.New("error reading")
				}
			})

			It("closes httpResp.Body and returns bad response", func() {
				Expect(fakeHijackCloser).NotTo(BeNil())
				Expect(hijackError).To(HaveOccurred())
				Expect(fakeBody.CloseCallCount()).To(Equal(1))
				Expect(hijackError).To(MatchError(fmt.Errorf("Backend error: Exit status: %d, error reading response body: %s", httpResp.StatusCode, "error reading")))
				Expect(fakeHijackCloser.CloseCallCount()).To(Equal(1))
			})
		})

		Context("when httpResponse fails", func() {
			BeforeEach(func() {
				fakeHijackableClient.DoReturns(nil, nil, errors.New("Request failed"))
				httpResp = http.Response{StatusCode: http.StatusTeapot, Body: ioutil.NopCloser(body)}
			})

			It("returns error", func() {
				Expect(hijackError).To(HaveOccurred())
				Expect(actualHijackedConn).To(BeNil())
				Expect(actualResponseReader).To(BeNil())
			})

			It("makes the right request", func() {
				Expect(fakeHijackableClient.DoCallCount()).To(Equal(1))
				actualRequest := fakeHijackableClient.DoArgsForCall(0)
				Expect(actualRequest.Method).To(Equal("POST"))
				Expect(actualRequest.Header["Content-Type"]).To(Equal([]string{"application/json"}))
				Expect(actualRequest.URL.RawQuery).To(Equal("key=some&key=values"))
				Expect(actualRequest.URL.Host).To(Equal("example.url"))
				Expect(ioutil.ReadAll(actualRequest.Body)).To(Equal([]byte("some-request-body")))
			})
		})
	})
})
<|endoftext|>


<commit_before>package eventhub

import (
	"errors"
	"sync"
)

type DummyDataSource struct {
	evs []*Event
	m   sync.Mutex
}

func (d *DummyDataSource) GetById(id int) (*Event, error) {

	d.m.Lock()
	defer d.m.Unlock()

	for _, e := range d.evs {
		if e.ID == id {
			return e, nil
		}
	}
	return nil, errors.New("No event found")
}

func (d *DummyDataSource) Save(e *Event) error {

	d.m.Lock()
	defer d.m.Unlock()

	if e.ID != 0 {
		switchIdx := -1
		for idx, x := range d.evs {
			if x.ID == e.ID {
				switchIdx = idx
			}
		}
		if switchIdx == -1 {
			return errors.New("ID provided but no event found with provided id")
		}
		d.evs[switchIdx] = e
	} else {
		e.ID = len(d.evs) + 1
		d.evs = append(d.evs, e)
	}

	return nil
}

func (d *DummyDataSource) FilterBy(m map[string]interface{}) ([]*Event, error) {
	d.m.Lock()
	defer d.m.Unlock()
	var matched []*Event
	matchedIndexes := []int{}

	for idx, event := range d.evs {
		match := false
		for key, value := range m {
			//TODO, get field at reflect and DeepEqual
		}
		if match {
			matchedIndexes = append(matchedIndexes, idx)
		}
	}

	return matched, nil
}

func (d *DummyDataSource) Clear() {
	d.m.Lock()
	defer d.m.Unlock()
	d.evs = nil
}

func NewDummyBackend() *DummyDataSource {
	return &DummyDataSource{}
}
<commit_msg>field comparision<commit_after>package eventhub

import (
	"errors"
	"reflect"
	"sync"
)

type DummyDataSource struct {
	evs []*Event
	m   sync.Mutex
}

func (d *DummyDataSource) GetById(id int) (*Event, error) {

	d.m.Lock()
	defer d.m.Unlock()

	for _, e := range d.evs {
		if e.ID == id {
			return e, nil
		}
	}
	return nil, errors.New("No event found")
}

func (d *DummyDataSource) Save(e *Event) error {

	d.m.Lock()
	defer d.m.Unlock()

	if e.ID != 0 {
		switchIdx := -1
		for idx, x := range d.evs {
			if x.ID == e.ID {
				switchIdx = idx
			}
		}
		if switchIdx == -1 {
			return errors.New("ID provided but no event found with provided id")
		}
		d.evs[switchIdx] = e
	} else {
		e.ID = len(d.evs) + 1
		d.evs = append(d.evs, e)
	}

	return nil
}

func stringInSlice(a string, list []string) bool {
	for _, b := range list {
		if b == a {
			return true
		}
	}
	return false
}

func (d *DummyDataSource) FilterBy(m map[string]interface{}) ([]*Event, error) {
	d.m.Lock()
	defer d.m.Unlock()
	var matched []*Event

	for _, event := range d.evs {
		r := reflect.ValueOf(event)
		if r.Kind() == reflect.Ptr {
			r = r.Elem()
		}
		match := false
		for key, value := range m {
			f := r.FieldByName(key)
			if reflect.DeepEqual(f.Interface(), value) {
				match = true
			}
			if vAsArray, ok := value.([]string); ok {
				eventData := f.Interface().([]string)
				allMatch := true
				for _, s := range vAsArray {
					if !stringInSlice(s, eventData) {
						allMatch = false
					}
				}
				match = allMatch
			}
		}
		if match {
			matched = append(matched, event)
		}
	}

	return matched, nil
}

func (d *DummyDataSource) Clear() {
	d.m.Lock()
	defer d.m.Unlock()
	d.evs = nil
}

func NewDummyBackend() *DummyDataSource {
	return &DummyDataSource{}
}
<|endoftext|>


<commit_before>package vsphere

import (
	"errors"
	"fmt"
	"github.com/hashicorp/go-uuid"
	"github.com/hashicorp/terraform/helper/schema"
	"github.com/vmware/govmomi/find"
	"github.com/vmware/govmomi/object"
	"github.com/vmware/govmomi/vim25/mo"
	"github.com/vmware/govmomi/vim25/types"
	"golang.org/x/net/context"
	"strings"
)

func resourceVirtualDisk() *schema.Resource {
	return &schema.Resource{
		Create: resourceVirtualDiskCreate,
		Read:   resourceVirtualDiskRead,
		Delete: resourceVirtualDiskDelete,

		Schema: map[string]*schema.Schema{
			"datacenter": {
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
				ForceNew: true,
			},
			"datastore": {
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
				ForceNew: true,
			},
			"path": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"fullPath": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"size": {
				Type:        schema.TypeInt,
				Required:    true,
				ForceNew:    true,
				Description: "Disk size in megabytes",
			},
			"thick": {
				Type:     schema.TypeBool,
				Optional: true,
				Default:  false,
				ForceNew: true,
			},
		},
	}
}

func resourceVirtualDiskCreate(resourceData *schema.ResourceData, meta interface{}) error {
	id, err := uuid.GenerateUUID()
	if err != nil {
		return err
	}

	providerMeta := meta.(providerMeta)
	client := providerMeta.client
	finder := find.NewFinder(client, false)
	ctx := providerMeta.context

	datacenter, err := findDatacenter(ctx, finder, resourceData)
	if err != nil {
		return err
	}

	finder.SetDatacenter(datacenter)

	datastore, err := findDatastore(ctx, finder, resourceData)
	if err != nil {
		return err
	}

	diskPath, err := getDiskPath(resourceData, datastore)
	if err != nil {
		return err
	}

	diskSizeMb := resourceData.Get("size").(int)
	if diskSizeMb == 0 {
		return errors.New("Virtual disk size is not specified")
	}

	diskType := string(types.VirtualDiskTypeThin)
	if resourceData.Get("thick").(bool) {
		diskType = string(types.VirtualDiskTypeThick)
	}

	diskManager := object.NewVirtualDiskManager(client)

	diskTask, err := diskManager.CreateVirtualDisk(ctx, diskPath, datacenter, &types.FileBackedVirtualDiskSpec{
		VirtualDiskSpec: types.VirtualDiskSpec{
			DiskType:    diskType,
			AdapterType: string(types.VirtualDiskAdapterTypeLsiLogic),
		},
		CapacityKb: int64(diskSizeMb) * 1024,
	})

	if err != nil {
		return fmt.Errorf("Failed to create virtual disk: %v", err)
	}

	if err := diskTask.Wait(ctx); err != nil {
		return fmt.Errorf("Failed to create virtual disk: %v", err)
	}

	resourceData.Set("fullPath", diskPath)
	resourceData.SetId(id)

	return nil
}

func resourceVirtualDiskRead(_ *schema.ResourceData, _ interface{}) error {
    return nil // todo
}

func resourceVirtualDiskDelete(resourceData *schema.ResourceData, meta interface{}) error {
	providerMeta := meta.(providerMeta)
	client := providerMeta.client
	finder := find.NewFinder(client, false)
	ctx := providerMeta.context

	datacenter, err := findDatacenter(ctx, finder, resourceData)
	if err != nil {
		return err
	}

	finder.SetDatacenter(datacenter)

	datastore, err := findDatastore(ctx, finder, resourceData)
	if err != nil {
		return err
	}

	diskPath, err := getDiskPath(resourceData, datastore)
	if err != nil {
		return err
	}

	diskManager := object.NewVirtualDiskManager(client)

	diskTask, err := diskManager.DeleteVirtualDisk(ctx, diskPath, datacenter)
	if err != nil {
		return fmt.Errorf("Failed to destroy virtual disk: %v", err)
	}

	if err := diskTask.Wait(ctx); err != nil {
		return fmt.Errorf("Failed to destroy virtual disk: %v", err)
	}

	resourceData.SetId("")
	resourceData.Set("fullPath", "")

	return nil
}

func findDatacenter(ctx context.Context, finder *find.Finder, resourceData *schema.ResourceData) (*object.Datacenter, error) {
	datacenterName := resourceData.Get("datacenter").(string)
	if datacenterName == "" {
		datacenter, err := finder.DefaultDatacenter(ctx)
		if err != nil {
			return nil, fmt.Errorf("Failed to read default datacenter: %v", err)
		}

		var moDatacenter *mo.Datacenter
		if err = datacenter.Properties(ctx, datacenter.Reference(), []string{"name"}, moDatacenter); err != nil {
			return nil, fmt.Errorf("Failed to read default datacenter name: %v", err)
		}

		resourceData.Set("datacenter", moDatacenter.Name)

		return datacenter, nil
	}

	datacenter, err := finder.Datacenter(ctx, datacenterName)
	if err != nil {
		return nil, fmt.Errorf("Failed to find datacenter \"%s\": %v", datacenterName, err)
	}

	return datacenter, nil
}

func findDatastore(ctx context.Context, finder *find.Finder, resourceData *schema.ResourceData) (*object.Datastore, error) {
	datastoreName := resourceData.Get("datastore").(string)
	if datastoreName == "" {
		datastore, err := finder.DefaultDatastore(ctx)
		if err != nil {
			return nil, fmt.Errorf("Failed to read default datastore: %v", err)
		}

		var moDatastore *mo.Datastore
		if err = datastore.Properties(ctx, datastore.Reference(), []string{"name"}, moDatastore); err != nil {
			return nil, fmt.Errorf("Failed to read default datastore name: %v", err)
		}

		resourceData.Set("datastore", moDatastore.Name)

		return datastore, nil
	}

	datastore, err := finder.Datastore(ctx, datastoreName)
	if err != nil {
		return nil, fmt.Errorf("Failed to find datastore \"%s\": %v", datastore, err)
	}

	return datastore, nil
}

func getDiskPath(resourceData *schema.ResourceData, datastore *object.Datastore) (string, error) {
	diskPath := resourceData.Get("path").(string)
	if diskPath == "" {
		return "", errors.New("Virtual disk path is not specified")
	}

	diskPath = ensureVmdkSuffix(diskPath)

	return datastore.Path(diskPath), nil
}

func ensureVmdkSuffix(diskPath string) string {
	if strings.HasSuffix(diskPath, ".vmdk") {
		return diskPath
	}
	return fmt.Sprintf("%s.vmdk", diskPath)
}
<commit_msg>fix name<commit_after>package vsphere

import (
	"errors"
	"fmt"
	"github.com/hashicorp/go-uuid"
	"github.com/hashicorp/terraform/helper/schema"
	"github.com/vmware/govmomi/find"
	"github.com/vmware/govmomi/object"
	"github.com/vmware/govmomi/vim25/mo"
	"github.com/vmware/govmomi/vim25/types"
	"golang.org/x/net/context"
	"strings"
)

func resourceVirtualDisk() *schema.Resource {
	return &schema.Resource{
		Create: resourceVirtualDiskCreate,
		Read:   resourceVirtualDiskRead,
		Delete: resourceVirtualDiskDelete,

		Schema: map[string]*schema.Schema{
			"datacenter": {
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
				ForceNew: true,
			},
			"datastore": {
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
				ForceNew: true,
			},
			"path": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"full_path": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"size": {
				Type:        schema.TypeInt,
				Required:    true,
				ForceNew:    true,
				Description: "Disk size in megabytes",
			},
			"thick": {
				Type:     schema.TypeBool,
				Optional: true,
				Default:  false,
				ForceNew: true,
			},
		},
	}
}

func resourceVirtualDiskCreate(resourceData *schema.ResourceData, meta interface{}) error {
	id, err := uuid.GenerateUUID()
	if err != nil {
		return err
	}

	providerMeta := meta.(providerMeta)
	client := providerMeta.client
	finder := find.NewFinder(client, false)
	ctx := providerMeta.context

	datacenter, err := findDatacenter(ctx, finder, resourceData)
	if err != nil {
		return err
	}

	finder.SetDatacenter(datacenter)

	datastore, err := findDatastore(ctx, finder, resourceData)
	if err != nil {
		return err
	}

	diskPath, err := getDiskPath(resourceData, datastore)
	if err != nil {
		return err
	}

	diskSizeMb := resourceData.Get("size").(int)
	if diskSizeMb == 0 {
		return errors.New("Virtual disk size is not specified")
	}

	diskType := string(types.VirtualDiskTypeThin)
	if resourceData.Get("thick").(bool) {
		diskType = string(types.VirtualDiskTypeThick)
	}

	diskManager := object.NewVirtualDiskManager(client)

	diskTask, err := diskManager.CreateVirtualDisk(ctx, diskPath, datacenter, &types.FileBackedVirtualDiskSpec{
		VirtualDiskSpec: types.VirtualDiskSpec{
			DiskType:    diskType,
			AdapterType: string(types.VirtualDiskAdapterTypeLsiLogic),
		},
		CapacityKb: int64(diskSizeMb) * 1024,
	})

	if err != nil {
		return fmt.Errorf("Failed to create virtual disk: %v", err)
	}

	if err := diskTask.Wait(ctx); err != nil {
		return fmt.Errorf("Failed to create virtual disk: %v", err)
	}

	resourceData.Set("fullPath", diskPath)
	resourceData.SetId(id)

	return nil
}

func resourceVirtualDiskRead(_ *schema.ResourceData, _ interface{}) error {
    return nil // todo
}

func resourceVirtualDiskDelete(resourceData *schema.ResourceData, meta interface{}) error {
	providerMeta := meta.(providerMeta)
	client := providerMeta.client
	finder := find.NewFinder(client, false)
	ctx := providerMeta.context

	datacenter, err := findDatacenter(ctx, finder, resourceData)
	if err != nil {
		return err
	}

	finder.SetDatacenter(datacenter)

	datastore, err := findDatastore(ctx, finder, resourceData)
	if err != nil {
		return err
	}

	diskPath, err := getDiskPath(resourceData, datastore)
	if err != nil {
		return err
	}

	diskManager := object.NewVirtualDiskManager(client)

	diskTask, err := diskManager.DeleteVirtualDisk(ctx, diskPath, datacenter)
	if err != nil {
		return fmt.Errorf("Failed to destroy virtual disk: %v", err)
	}

	if err := diskTask.Wait(ctx); err != nil {
		return fmt.Errorf("Failed to destroy virtual disk: %v", err)
	}

	resourceData.SetId("")
	resourceData.Set("fullPath", "")

	return nil
}

func findDatacenter(ctx context.Context, finder *find.Finder, resourceData *schema.ResourceData) (*object.Datacenter, error) {
	datacenterName := resourceData.Get("datacenter").(string)
	if datacenterName == "" {
		datacenter, err := finder.DefaultDatacenter(ctx)
		if err != nil {
			return nil, fmt.Errorf("Failed to read default datacenter: %v", err)
		}

		var moDatacenter *mo.Datacenter
		if err = datacenter.Properties(ctx, datacenter.Reference(), []string{"name"}, moDatacenter); err != nil {
			return nil, fmt.Errorf("Failed to read default datacenter name: %v", err)
		}

		resourceData.Set("datacenter", moDatacenter.Name)

		return datacenter, nil
	}

	datacenter, err := finder.Datacenter(ctx, datacenterName)
	if err != nil {
		return nil, fmt.Errorf("Failed to find datacenter \"%s\": %v", datacenterName, err)
	}

	return datacenter, nil
}

func findDatastore(ctx context.Context, finder *find.Finder, resourceData *schema.ResourceData) (*object.Datastore, error) {
	datastoreName := resourceData.Get("datastore").(string)
	if datastoreName == "" {
		datastore, err := finder.DefaultDatastore(ctx)
		if err != nil {
			return nil, fmt.Errorf("Failed to read default datastore: %v", err)
		}

		var moDatastore *mo.Datastore
		if err = datastore.Properties(ctx, datastore.Reference(), []string{"name"}, moDatastore); err != nil {
			return nil, fmt.Errorf("Failed to read default datastore name: %v", err)
		}

		resourceData.Set("datastore", moDatastore.Name)

		return datastore, nil
	}

	datastore, err := finder.Datastore(ctx, datastoreName)
	if err != nil {
		return nil, fmt.Errorf("Failed to find datastore \"%s\": %v", datastore, err)
	}

	return datastore, nil
}

func getDiskPath(resourceData *schema.ResourceData, datastore *object.Datastore) (string, error) {
	diskPath := resourceData.Get("path").(string)
	if diskPath == "" {
		return "", errors.New("Virtual disk path is not specified")
	}

	diskPath = ensureVmdkSuffix(diskPath)

	return datastore.Path(diskPath), nil
}

func ensureVmdkSuffix(diskPath string) string {
	if strings.HasSuffix(diskPath, ".vmdk") {
		return diskPath
	}
	return fmt.Sprintf("%s.vmdk", diskPath)
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"net/http"
	"net/url"
	"os"
	"time"

	"k8s.io/contrib/cluster-autoscaler/config"
	"k8s.io/contrib/cluster-autoscaler/simulator"
	"k8s.io/contrib/cluster-autoscaler/utils/gce"
	kube_api "k8s.io/kubernetes/pkg/api"
	kube_record "k8s.io/kubernetes/pkg/client/record"
	kube_client "k8s.io/kubernetes/pkg/client/unversioned"

	"github.com/golang/glog"
	"github.com/prometheus/client_golang/prometheus"
)

var (
	migConfigFlag           config.MigConfigFlag
	address                 = flag.String("address", ":8085", "The address to expose prometheus metrics.")
	kubernetes              = flag.String("kubernetes", "", "Kuberentes master location. Leave blank for default")
	cloudConfig             = flag.String("cloud-config", "", "The path to the cloud provider configuration file.  Empty string for no configuration file.")
	verifyUnschedulablePods = flag.Bool("verify-unschedulable-pods", true,
		"If enabled CA will ensure that each pod marked by Scheduler as unschedulable actually can't be scheduled on any node."+
			"This prevents from adding unnecessary nodes in situation when CA and Scheduler have different configuration.")
	scaleDownEnabled = flag.Bool("scale-down-enabled", true, "Should CA scale down the cluster")
	scaleDownDelay   = flag.Duration("scale-down-delay", 10*time.Minute,
		"Duration from the last scale up to the time when CA starts to check scale down options")
	scaleDownUnderutilizedTime = flag.Duration("scale-down-underutilized-time", 10*time.Minute,
		"How long the node should be underutilized before it is eligible for scale down")
	scaleDownUtilizationThreshold = flag.Float64("scale-down-utilization-threshold", 0.5,
		"Node reservation level below which a node can be considered for scale down")
	scaleDownTrialFrequency = flag.Duration("scale-down-trial-frequency", 10*time.Minute,
		"How often scale down possiblity is check")
)

func main() {
	flag.Var(&migConfigFlag, "nodes", "sets min,max size and url of a MIG to be controlled by Cluster Autoscaler. "+
		"Can be used multiple times. Format: <min>:<max>:<migurl>")
	flag.Parse()

	http.Handle("/metrics", prometheus.Handler())
	err := http.ListenAndServe(*address, nil)
	if err != nil {
		glog.Fatalf("Failed to start http server metrics: %v", err)
	}

	url, err := url.Parse(*kubernetes)
	if err != nil {
		glog.Fatalf("Failed to parse Kuberentes url: %v", err)
	}

	// Configuration
	kubeConfig, err := config.GetKubeClientConfig(url)
	if err != nil {
		glog.Fatalf("Failed to build Kuberentes client configuration: %v", err)
	}
	migConfigs := make([]*config.MigConfig, 0, len(migConfigFlag))
	for i := range migConfigFlag {
		migConfigs = append(migConfigs, &migConfigFlag[i])
	}

	// GCE Manager
	var gceManager *gce.GceManager
	if *cloudConfig != "" {
		config, err := os.Open(*cloudConfig)
		if err != nil {
			glog.Fatalf("Couldn't open cloud provider configuration %s: %#v", *cloudConfig, err)
		}
		defer config.Close()
		gceManager, err = gce.CreateGceManager(migConfigs, config)
	} else {
		gceManager, err = gce.CreateGceManager(migConfigs, nil)
	}
	if err != nil {
		glog.Fatalf("Failed to create GCE Manager: %v", err)
	}

	kubeClient := kube_client.NewOrDie(kubeConfig)

	predicateChecker, err := simulator.NewPredicateChecker(kubeClient)
	if err != nil {
		glog.Fatalf("Failed to create predicate checker: %v", err)
	}
	unschedulablePodLister := NewUnschedulablePodLister(kubeClient)
	scheduledPodLister := NewScheduledPodLister(kubeClient)
	nodeLister := NewNodeLister(kubeClient)

	lastScaleUpTime := time.Now()
	lastScaleDownFailedTrial := time.Now()
	underutilizedNodes := make(map[string]time.Time)

	eventBroadcaster := kube_record.NewBroadcaster()
	eventBroadcaster.StartLogging(glog.Infof)
	eventBroadcaster.StartRecordingToSink(kubeClient.Events(""))
	recorder := eventBroadcaster.NewRecorder(kube_api.EventSource{Component: "cluster-autoscaler"})

	for {
		select {
		case <-time.After(time.Minute):
			{
				loopStart := time.Now()
				updateLastTime("main")

				nodes, err := nodeLister.List()
				if err != nil {
					glog.Errorf("Failed to list nodes: %v", err)
					continue
				}
				if len(nodes) == 0 {
					glog.Errorf("No nodes in the cluster")
					continue
				}

				if err := CheckMigsAndNodes(nodes, gceManager); err != nil {
					glog.Warningf("Cluster is not ready for autoscaling: %v", err)
					continue
				}

				allUnschedulablePods, err := unschedulablePodLister.List()
				if err != nil {
					glog.Errorf("Failed to list unscheduled pods: %v", err)
					continue
				}

				allScheduled, err := scheduledPodLister.List()
				if err != nil {
					glog.Errorf("Failed to list scheduled pods: %v", err)
					continue
				}

				// We need to reset all pods that have been marked as unschedulable not after
				// the newest node became available for the scheduler.
				allNodesAvailableTime := GetAllNodesAvailableTime(nodes)
				podsToReset, unschedulablePodsToHelp := SlicePodsByPodScheduledTime(allUnschedulablePods, allNodesAvailableTime)
				ResetPodScheduledCondition(kubeClient, podsToReset)

				// We need to check whether pods marked as unschedulable are actually unschedulable.
				// This should prevent from adding unnecessary nodes. Example of such situation:
				// - CA and Scheduler has slightly different configuration
				// - Scheduler can't schedule a pod and marks it as unschedulable
				// - CA added a node which should help the pod
				// - Scheduler doesn't schedule the pod on the new node
				//   because according to it logic it doesn't fit there
				// - CA see the pod is still unschedulable, so it adds another node to help it
				//
				// With the check enabled the last point won't happen because CA will ignore a pod
				// which is supposed to schedule on an existing node.
				//
				// Without below check cluster might be unnecessary scaled up to the max allowed size
				// in the describe situation.
				schedulablePodsPresent := false
				if *verifyUnschedulablePods {
					newUnschedulablePodsToHelp := FilterOutSchedulable(unschedulablePodsToHelp, nodes, allScheduled, predicateChecker)

					if len(newUnschedulablePodsToHelp) != len(unschedulablePodsToHelp) {
						schedulablePodsPresent = true
					}
					unschedulablePodsToHelp = newUnschedulablePodsToHelp
				}

				if len(unschedulablePodsToHelp) == 0 {
					glog.V(1).Info("No unschedulable pods")
				} else {
					scaleUpStart := time.Now()
					updateLastTime("scaleup")
					scaledUp, err := ScaleUp(unschedulablePodsToHelp, nodes, migConfigs, gceManager, kubeClient, predicateChecker, recorder)

					updateDuration("scaleup", scaleUpStart)

					if err != nil {
						glog.Errorf("Failed to scale up: %v", err)
						continue
					} else {
						if scaledUp {
							lastScaleUpTime = time.Now()
							// No scale down in this iteration.
							continue
						}
					}
				}

				if *scaleDownEnabled {
					utilizationStart := time.Now()

					// In dry run only utilization is updated
					calculateUtilizationOnly := lastScaleUpTime.Add(*scaleDownDelay).After(time.Now()) ||
						lastScaleDownFailedTrial.Add(*scaleDownTrialFrequency).After(time.Now()) ||
						schedulablePodsPresent

					updateLastTime("utilization")

					underutilizedNodes = CalculateUnderutilizedNodes(
						nodes,
						underutilizedNodes,
						*scaleDownUtilizationThreshold,
						allScheduled,
						predicateChecker)

					updateDuration("utilization", utilizationStart)

					if !calculateUtilizationOnly {
						scaleDownStart := time.Now()
						updateLastTime("scaledown")

						result, err := ScaleDown(
							nodes,
							underutilizedNodes,
							*scaleDownUnderutilizedTime,
							allScheduled,
							gceManager, kubeClient, predicateChecker)

						updateDuration("scaledown", scaleDownStart)

						if err != nil {
							glog.Errorf("Failed to scale down: %v", err)
						} else {
							if result == ScaleDownNodeDeleted {
								// Clean the utilization map to be super sure that the simulated
								// deletions are made in the new context.
								underutilizedNodes = make(map[string]time.Time, len(underutilizedNodes))
							} else {
								lastScaleDownFailedTrial = time.Now()
							}
						}
					}
				}
				updateDuration("main", loopStart)
			}
		}
	}
}

func updateDuration(label string, start time.Time) {
	duration.WithLabelValues(label).Observe(durationToMicro(start))
	lastDuration.WithLabelValues(label).Set(durationToMicro(start))
}

func updateLastTime(label string) {
	lastTimestamp.WithLabelValues(label).Set(float64(time.Now().Unix()))
}
<commit_msg>Cluster-autoscaler: start http server in a separate gorouting<commit_after>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"net/http"
	"net/url"
	"os"
	"time"

	"k8s.io/contrib/cluster-autoscaler/config"
	"k8s.io/contrib/cluster-autoscaler/simulator"
	"k8s.io/contrib/cluster-autoscaler/utils/gce"
	kube_api "k8s.io/kubernetes/pkg/api"
	kube_record "k8s.io/kubernetes/pkg/client/record"
	kube_client "k8s.io/kubernetes/pkg/client/unversioned"

	"github.com/golang/glog"
	"github.com/prometheus/client_golang/prometheus"
)

var (
	migConfigFlag           config.MigConfigFlag
	address                 = flag.String("address", ":8085", "The address to expose prometheus metrics.")
	kubernetes              = flag.String("kubernetes", "", "Kuberentes master location. Leave blank for default")
	cloudConfig             = flag.String("cloud-config", "", "The path to the cloud provider configuration file.  Empty string for no configuration file.")
	verifyUnschedulablePods = flag.Bool("verify-unschedulable-pods", true,
		"If enabled CA will ensure that each pod marked by Scheduler as unschedulable actually can't be scheduled on any node."+
			"This prevents from adding unnecessary nodes in situation when CA and Scheduler have different configuration.")
	scaleDownEnabled = flag.Bool("scale-down-enabled", true, "Should CA scale down the cluster")
	scaleDownDelay   = flag.Duration("scale-down-delay", 10*time.Minute,
		"Duration from the last scale up to the time when CA starts to check scale down options")
	scaleDownUnderutilizedTime = flag.Duration("scale-down-underutilized-time", 10*time.Minute,
		"How long the node should be underutilized before it is eligible for scale down")
	scaleDownUtilizationThreshold = flag.Float64("scale-down-utilization-threshold", 0.5,
		"Node reservation level below which a node can be considered for scale down")
	scaleDownTrialFrequency = flag.Duration("scale-down-trial-frequency", 10*time.Minute,
		"How often scale down possiblity is check")
)

func main() {
	flag.Var(&migConfigFlag, "nodes", "sets min,max size and url of a MIG to be controlled by Cluster Autoscaler. "+
		"Can be used multiple times. Format: <min>:<max>:<migurl>")
	flag.Parse()

	go func() {
		http.Handle("/metrics", prometheus.Handler())
		err := http.ListenAndServe(*address, nil)
		glog.Fatalf("Failed to start metrics: %v", err)
	}()

	url, err := url.Parse(*kubernetes)
	if err != nil {
		glog.Fatalf("Failed to parse Kuberentes url: %v", err)
	}

	// Configuration
	kubeConfig, err := config.GetKubeClientConfig(url)
	if err != nil {
		glog.Fatalf("Failed to build Kuberentes client configuration: %v", err)
	}
	migConfigs := make([]*config.MigConfig, 0, len(migConfigFlag))
	for i := range migConfigFlag {
		migConfigs = append(migConfigs, &migConfigFlag[i])
	}

	// GCE Manager
	var gceManager *gce.GceManager
	if *cloudConfig != "" {
		config, err := os.Open(*cloudConfig)
		if err != nil {
			glog.Fatalf("Couldn't open cloud provider configuration %s: %#v", *cloudConfig, err)
		}
		defer config.Close()
		gceManager, err = gce.CreateGceManager(migConfigs, config)
	} else {
		gceManager, err = gce.CreateGceManager(migConfigs, nil)
	}
	if err != nil {
		glog.Fatalf("Failed to create GCE Manager: %v", err)
	}

	kubeClient := kube_client.NewOrDie(kubeConfig)

	predicateChecker, err := simulator.NewPredicateChecker(kubeClient)
	if err != nil {
		glog.Fatalf("Failed to create predicate checker: %v", err)
	}
	unschedulablePodLister := NewUnschedulablePodLister(kubeClient)
	scheduledPodLister := NewScheduledPodLister(kubeClient)
	nodeLister := NewNodeLister(kubeClient)

	lastScaleUpTime := time.Now()
	lastScaleDownFailedTrial := time.Now()
	underutilizedNodes := make(map[string]time.Time)

	eventBroadcaster := kube_record.NewBroadcaster()
	eventBroadcaster.StartLogging(glog.Infof)
	eventBroadcaster.StartRecordingToSink(kubeClient.Events(""))
	recorder := eventBroadcaster.NewRecorder(kube_api.EventSource{Component: "cluster-autoscaler"})

	for {
		select {
		case <-time.After(time.Minute):
			{
				loopStart := time.Now()
				updateLastTime("main")

				nodes, err := nodeLister.List()
				if err != nil {
					glog.Errorf("Failed to list nodes: %v", err)
					continue
				}
				if len(nodes) == 0 {
					glog.Errorf("No nodes in the cluster")
					continue
				}

				if err := CheckMigsAndNodes(nodes, gceManager); err != nil {
					glog.Warningf("Cluster is not ready for autoscaling: %v", err)
					continue
				}

				allUnschedulablePods, err := unschedulablePodLister.List()
				if err != nil {
					glog.Errorf("Failed to list unscheduled pods: %v", err)
					continue
				}

				allScheduled, err := scheduledPodLister.List()
				if err != nil {
					glog.Errorf("Failed to list scheduled pods: %v", err)
					continue
				}

				// We need to reset all pods that have been marked as unschedulable not after
				// the newest node became available for the scheduler.
				allNodesAvailableTime := GetAllNodesAvailableTime(nodes)
				podsToReset, unschedulablePodsToHelp := SlicePodsByPodScheduledTime(allUnschedulablePods, allNodesAvailableTime)
				ResetPodScheduledCondition(kubeClient, podsToReset)

				// We need to check whether pods marked as unschedulable are actually unschedulable.
				// This should prevent from adding unnecessary nodes. Example of such situation:
				// - CA and Scheduler has slightly different configuration
				// - Scheduler can't schedule a pod and marks it as unschedulable
				// - CA added a node which should help the pod
				// - Scheduler doesn't schedule the pod on the new node
				//   because according to it logic it doesn't fit there
				// - CA see the pod is still unschedulable, so it adds another node to help it
				//
				// With the check enabled the last point won't happen because CA will ignore a pod
				// which is supposed to schedule on an existing node.
				//
				// Without below check cluster might be unnecessary scaled up to the max allowed size
				// in the describe situation.
				schedulablePodsPresent := false
				if *verifyUnschedulablePods {
					newUnschedulablePodsToHelp := FilterOutSchedulable(unschedulablePodsToHelp, nodes, allScheduled, predicateChecker)

					if len(newUnschedulablePodsToHelp) != len(unschedulablePodsToHelp) {
						schedulablePodsPresent = true
					}
					unschedulablePodsToHelp = newUnschedulablePodsToHelp
				}

				if len(unschedulablePodsToHelp) == 0 {
					glog.V(1).Info("No unschedulable pods")
				} else {
					scaleUpStart := time.Now()
					updateLastTime("scaleup")
					scaledUp, err := ScaleUp(unschedulablePodsToHelp, nodes, migConfigs, gceManager, kubeClient, predicateChecker, recorder)

					updateDuration("scaleup", scaleUpStart)

					if err != nil {
						glog.Errorf("Failed to scale up: %v", err)
						continue
					} else {
						if scaledUp {
							lastScaleUpTime = time.Now()
							// No scale down in this iteration.
							continue
						}
					}
				}

				if *scaleDownEnabled {
					utilizationStart := time.Now()

					// In dry run only utilization is updated
					calculateUtilizationOnly := lastScaleUpTime.Add(*scaleDownDelay).After(time.Now()) ||
						lastScaleDownFailedTrial.Add(*scaleDownTrialFrequency).After(time.Now()) ||
						schedulablePodsPresent

					updateLastTime("utilization")

					underutilizedNodes = CalculateUnderutilizedNodes(
						nodes,
						underutilizedNodes,
						*scaleDownUtilizationThreshold,
						allScheduled,
						predicateChecker)

					updateDuration("utilization", utilizationStart)

					if !calculateUtilizationOnly {
						scaleDownStart := time.Now()
						updateLastTime("scaledown")

						result, err := ScaleDown(
							nodes,
							underutilizedNodes,
							*scaleDownUnderutilizedTime,
							allScheduled,
							gceManager, kubeClient, predicateChecker)

						updateDuration("scaledown", scaleDownStart)

						if err != nil {
							glog.Errorf("Failed to scale down: %v", err)
						} else {
							if result == ScaleDownNodeDeleted {
								// Clean the utilization map to be super sure that the simulated
								// deletions are made in the new context.
								underutilizedNodes = make(map[string]time.Time, len(underutilizedNodes))
							} else {
								lastScaleDownFailedTrial = time.Now()
							}
						}
					}
				}
				updateDuration("main", loopStart)
			}
		}
	}
}

func updateDuration(label string, start time.Time) {
	duration.WithLabelValues(label).Observe(durationToMicro(start))
	lastDuration.WithLabelValues(label).Set(durationToMicro(start))
}

func updateLastTime(label string) {
	lastTimestamp.WithLabelValues(label).Set(float64(time.Now().Unix()))
}
<|endoftext|>