<commit_before>// Copyright © 2017 Circonus, Inc. <support@circonus.com>
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//

package server

import (
	"encoding/json"
	"fmt"
	"net/http"
	"reflect"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/circonus-labs/circonus-agent/internal/config"
	"github.com/circonus-labs/circonus-agent/internal/plugins"
	"github.com/circonus-labs/circonus-agent/internal/server/receiver"
	circonusgometrics "github.com/circonus-labs/circonus-gometrics"
	"github.com/rs/zerolog/log"
	"github.com/spf13/viper"
)

// run handles requests to execute plugins and return metrics emitted
// handles /, /run, or /run/plugin_name
func (s *Server) run(w http.ResponseWriter, r *http.Request) {
	plugin := ""

	if strings.HasPrefix(r.URL.Path, "/run/") { // run specific plugin
		plugin = strings.Replace(r.URL.Path, "/run/", "", -1)
		if plugin != "" {
			if !s.plugins.IsInternal(plugin) && !s.plugins.IsValid(plugin) {
				s.logger.Warn().
					Str("plugin", plugin).
					Msg("Unknown plugin requested")
				http.NotFound(w, r)
				return
			}
		}
	}

	lastMeticsmu.Lock()
	defer lastMeticsmu.Unlock()

	metrics := &map[string]interface{}{}

	if plugin == "" || !s.plugins.IsInternal(plugin) {
		// NOTE: errors are ignored from plugins.Run
		//       1. errors are already logged by Run
		//       2. do not expose execution state to callers
		s.plugins.Run(plugin)
		metrics = s.plugins.Flush(plugin)
	}

	if plugin == "" || plugin == "write" {
		receiverMetrics := receiver.Flush()
		for metricGroup, value := range *receiverMetrics {
			(*metrics)[metricGroup] = value
		}
	}

	if plugin == "" || plugin == "statsd" {
		if s.statsdSvr != nil {
			statsdMetrics := s.statsdSvr.Flush()
			if statsdMetrics != nil {
				(*metrics)[viper.GetString(config.KeyStatsdHostCategory)] = *statsdMetrics
			}
		}
	}

	lastMetrics.metrics = metrics
	lastMetrics.ts = time.Now()
	// s.logger.Debug().Interface("m", metrics).Msg("metrics")

	w.Header().Set("Content-Type", "application/json")
	if err := json.NewEncoder(w).Encode(metrics); err != nil {
		s.logger.Error().
			Err(err).
			Msg("Writing metrics to response")
	}
}

// promOutput returns the last metrics in prom format
func (s *Server) promOutput(w http.ResponseWriter, r *http.Request) {
	if lastMetrics.metrics == nil {
		w.WriteHeader(http.StatusNoContent)
		return
	}

	ms := lastMetrics.ts.UnixNano() / int64(time.Millisecond)

	for group, data := range *lastMetrics.metrics {
		// t := reflect.TypeOf(data)
		// s.logger.Debug().Str("group", group).Str("type", t.String()).Msg("item")
		walkMetrics(w, group, ms, data)
	}
}

func walkMetrics(w http.ResponseWriter, prefix string, ts int64, val interface{}) {
	t := reflect.TypeOf(val)
	// log.Debug().Str("pfx", prefix).Str("type", t.String()).Interface("val", val).Msg("val")
	switch t.String() {
	case "circonusgometrics.Metrics":
		metrics, ok := val.(circonusgometrics.Metrics)
		if !ok {
			log.Warn().Interface("val", val).Str("target_type", t.String()).Str("pkg", "prom export").Msg("unable to coerce")
			return
		}
		for pfx, metric := range metrics {
			switch t2 := metric.Value.(type) {
			case uint64:
				w.Write([]byte(fmt.Sprintf("%s`%s %d %d\n", prefix, pfx, metric.Value, ts)))
			case float64:
				w.Write([]byte(fmt.Sprintf("%s`%s %f %d\n", prefix, pfx, metric.Value, ts)))
			case string:
				s := fmt.Sprintf("%v", metric.Value)
				ok, err := regexp.MatchString("^[0-9]+$", s)
				if err != nil {
					log.Error().Err(err).Msg("testing string for digits")
					continue
				}
				if ok {
					v, err := strconv.ParseInt(s, 10, 64)
					if err != nil {
						log.Error().Err(err).Msg("conv int64")
						continue
					}
					w.Write([]byte(fmt.Sprintf("%s`%s %d %d\n", prefix, pfx, v, ts)))
					continue
				}
				w.Write([]byte(fmt.Sprintf("#TEXT %s`%s %s %d\n", prefix, pfx, s, ts)))
			case []string:
				s := fmt.Sprintf("%v", metric.Value)
				if strings.Contains(s, "[H[") {
					w.Write([]byte(fmt.Sprintf("#HISTOGRAM %s`%s %s %d\n", prefix, pfx, s, ts)))
					continue
				}
				walkMetrics(w, prefix+"`"+pfx, ts, metric.Value)
			default:
				w.Write([]byte(fmt.Sprintf("?? %s`%s %v - %T\n", prefix, pfx, metric.Value, t2)))
				// walkMetrics(w, prefix+"`"+pfx, ts, metric.Value)
			}
		}
	case "*plugins.Metrics":
		for pfx, v := range *val.(*plugins.Metrics) {
			walkMetrics(w, prefix+"`"+pfx, ts, v.Value)
		}
	case "[]string":
		v, ok := val.([]string)
		if ok {
			if len(v) > 0 && v[0][0:2] == "H[" {
				w.Write([]byte(fmt.Sprintf("#HISTOGRAM %s %s %d\n", prefix, val, ts)))
				return
			}
			for idx, v2 := range v {
				w.Write([]byte(fmt.Sprintf("%s`%d %s %d\n", prefix, idx, v2, ts)))
			}
		}
	case "[]interface {}":
		v, ok := val.([]interface{})
		if !ok {
			log.Warn().Interface("val", val).Str("target_type", t.String()).Str("pkg", "prom export").Msg("unable to coerce")
			return
		}
		if fmt.Sprintf("%v", v)[0:3] == "[H[" {
			w.Write([]byte(fmt.Sprintf("#HISTOGRAM %s %s %d\n", prefix, val, ts)))
			return
		}
		for idx, v2 := range v {
			walkMetrics(w, fmt.Sprintf("%s`%d", prefix, idx), ts, v2)
		}
	case "map[string]interface {}":
		v, ok := val.(map[string]interface{})
		if !ok {
			log.Warn().Interface("val", val).Str("target_type", t.String()).Str("pkg", "prom export").Msg("unable to coerce")
			return
		}
		if _, isMetric := v["_type"]; isMetric {
			walkMetrics(w, prefix, ts, v["_value"])
		} else {
			for pfx, v2 := range v {
				walkMetrics(w, prefix+"`"+pfx, ts, v2)
			}
		}
	case "string":
		w.Write([]byte(fmt.Sprintf("#TEXT %s %s %d\n", prefix, val, ts)))
	case "float32":
		fallthrough
	case "float64":
		w.Write([]byte(fmt.Sprintf("%s %e %d\n", prefix, val, ts)))
	case "int32":
		fallthrough
	case "int64":
		fallthrough
	case "uint32":
		fallthrough
	case "uint64":
		w.Write([]byte(fmt.Sprintf("%s %d %d\n", prefix, val, ts)))
	default:
		w.Write([]byte(fmt.Sprintf("%v(%v) = %#v\n", prefix, t, val)))
	}
}

// inventory returns the current, active plugin inventory
func (s *Server) inventory(w http.ResponseWriter, r *http.Request) {
	inventory := s.plugins.Inventory()
	if inventory == nil {
		inventory = []byte(`{"error": "empty inventory"}`)
		s.logger.Error().Msg("inventory is nil/empty...")
	}

	w.Header().Set("Content-Type", "application/json")
	w.Write(inventory)
}

// write handles PUT/POST requests with a JSON playload containing "freeform"
// metrics. No validation is applied to the "format" of the metrics beyond k/v.
// Where 'key' is the metric name and 'value' is the metric value as either a
// simple value (e.g. {"name": 1, "foo": "bar", ...}) or a structured value
// representation (e.g. {"foo": {_type: "i", _value: 1}, ...}).
func (s *Server) write(w http.ResponseWriter, r *http.Request) {
	id := strings.Replace(r.URL.Path, "/write/", "", -1)

	log.Debug().Str("path", r.URL.Path).Str("id", id).Msg("write request")
	// a write request *MUST* include a metric group id to act as a namespace.
	// in other words, a "plugin name", all metrics for that write will appear
	// _under_ the metric group id (aka plugin name)
	if id == "" {
		http.NotFound(w, r)
		return
	}

	if err := receiver.Parse(id, r.Body); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	w.WriteHeader(http.StatusNoContent)
}
<commit_msg>better type support for cgm metrics via statsd in prom<commit_after>// Copyright © 2017 Circonus, Inc. <support@circonus.com>
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//

package server

import (
	"encoding/json"
	"fmt"
	"net/http"
	"reflect"
	"strconv"
	"strings"
	"time"

	"github.com/circonus-labs/circonus-agent/internal/config"
	"github.com/circonus-labs/circonus-agent/internal/plugins"
	"github.com/circonus-labs/circonus-agent/internal/server/receiver"
	circonusgometrics "github.com/circonus-labs/circonus-gometrics"
	"github.com/rs/zerolog/log"
	"github.com/spf13/viper"
)

// run handles requests to execute plugins and return metrics emitted
// handles /, /run, or /run/plugin_name
func (s *Server) run(w http.ResponseWriter, r *http.Request) {
	plugin := ""

	if strings.HasPrefix(r.URL.Path, "/run/") { // run specific plugin
		plugin = strings.Replace(r.URL.Path, "/run/", "", -1)
		if plugin != "" {
			if !s.plugins.IsInternal(plugin) && !s.plugins.IsValid(plugin) {
				s.logger.Warn().
					Str("plugin", plugin).
					Msg("Unknown plugin requested")
				http.NotFound(w, r)
				return
			}
		}
	}

	lastMeticsmu.Lock()
	defer lastMeticsmu.Unlock()

	metrics := &map[string]interface{}{}

	if plugin == "" || !s.plugins.IsInternal(plugin) {
		// NOTE: errors are ignored from plugins.Run
		//       1. errors are already logged by Run
		//       2. do not expose execution state to callers
		s.plugins.Run(plugin)
		metrics = s.plugins.Flush(plugin)
	}

	if plugin == "" || plugin == "write" {
		receiverMetrics := receiver.Flush()
		for metricGroup, value := range *receiverMetrics {
			(*metrics)[metricGroup] = value
		}
	}

	if plugin == "" || plugin == "statsd" {
		if s.statsdSvr != nil {
			statsdMetrics := s.statsdSvr.Flush()
			if statsdMetrics != nil {
				(*metrics)[viper.GetString(config.KeyStatsdHostCategory)] = *statsdMetrics
			}
		}
	}

	lastMetrics.metrics = metrics
	lastMetrics.ts = time.Now()
	// s.logger.Debug().Interface("m", metrics).Msg("metrics")

	w.Header().Set("Content-Type", "application/json")
	if err := json.NewEncoder(w).Encode(metrics); err != nil {
		s.logger.Error().
			Err(err).
			Msg("Writing metrics to response")
	}
}

// promOutput returns the last metrics in prom format
func (s *Server) promOutput(w http.ResponseWriter, r *http.Request) {
	if lastMetrics.metrics == nil {
		w.WriteHeader(http.StatusNoContent)
		return
	}

	ms := lastMetrics.ts.UnixNano() / int64(time.Millisecond)

	for group, data := range *lastMetrics.metrics {
		// t := reflect.TypeOf(data)
		// s.logger.Debug().Str("group", group).Str("type", t.String()).Msg("item")
		walkMetrics(w, group, ms, data)
	}
}

func walkMetrics(w http.ResponseWriter, prefix string, ts int64, val interface{}) {
	t := reflect.TypeOf(val)
	// log.Debug().Str("pfx", prefix).Str("type", t.String()).Interface("val", val).Msg("val")
	switch t.String() {
	case "circonusgometrics.Metrics":
		metrics, ok := val.(circonusgometrics.Metrics)
		if !ok {
			log.Warn().Interface("val", val).Str("target_type", t.String()).Str("pkg", "prom export").Msg("unable to coerce")
			return
		}
		for pfx, metric := range metrics {
			switch t2 := metric.Value.(type) {
			case uint64:
				w.Write([]byte(fmt.Sprintf("%s`%s %d %d\n", prefix, pfx, metric.Value, ts)))
			case float64:
				w.Write([]byte(fmt.Sprintf("%s`%s %f %d\n", prefix, pfx, metric.Value, ts)))
			case string:
				s := fmt.Sprintf("%v", metric.Value)
				switch metric.Type {
				case "i":
					fallthrough
				case "I":
					fallthrough
				case "l":
					fallthrough
				case "L":
					v, err := strconv.ParseInt(s, 10, 64)
					if err != nil {
						log.Error().Err(err).Msg("conv int64")
						continue
					}
					w.Write([]byte(fmt.Sprintf("%s`%s %d %d\n", prefix, pfx, v, ts)))
					continue
				case "n":
					v, err := strconv.ParseFloat(s, 64)
					if err != nil {
						log.Error().Err(err).Msg("conv float64")
						continue
					}
					w.Write([]byte(fmt.Sprintf("%s`%s %f %d\n", prefix, pfx, v, ts)))
					continue
				}
				w.Write([]byte(fmt.Sprintf("#TEXT %s`%s %s %d\n", prefix, pfx, s, ts)))
			case []string:
				s := fmt.Sprintf("%v", metric.Value)
				if strings.Contains(s, "[H[") {
					w.Write([]byte(fmt.Sprintf("#HISTOGRAM %s`%s %s %d\n", prefix, pfx, s, ts)))
					continue
				}
				walkMetrics(w, prefix+"`"+pfx, ts, metric.Value)
			default:
				w.Write([]byte(fmt.Sprintf("?? %s`%s %v - %T\n", prefix, pfx, metric.Value, t2)))
				// walkMetrics(w, prefix+"`"+pfx, ts, metric.Value)
			}
		}
	case "*plugins.Metrics":
		for pfx, v := range *val.(*plugins.Metrics) {
			walkMetrics(w, prefix+"`"+pfx, ts, v.Value)
		}
	case "[]string":
		v, ok := val.([]string)
		if ok {
			if len(v) > 0 && v[0][0:2] == "H[" {
				w.Write([]byte(fmt.Sprintf("#HISTOGRAM %s %s %d\n", prefix, val, ts)))
				return
			}
			for idx, v2 := range v {
				w.Write([]byte(fmt.Sprintf("%s`%d %s %d\n", prefix, idx, v2, ts)))
			}
		}
	case "[]interface {}":
		v, ok := val.([]interface{})
		if !ok {
			log.Warn().Interface("val", val).Str("target_type", t.String()).Str("pkg", "prom export").Msg("unable to coerce")
			return
		}
		if fmt.Sprintf("%v", v)[0:3] == "[H[" {
			w.Write([]byte(fmt.Sprintf("#HISTOGRAM %s %s %d\n", prefix, val, ts)))
			return
		}
		for idx, v2 := range v {
			walkMetrics(w, fmt.Sprintf("%s`%d", prefix, idx), ts, v2)
		}
	case "map[string]interface {}":
		v, ok := val.(map[string]interface{})
		if !ok {
			log.Warn().Interface("val", val).Str("target_type", t.String()).Str("pkg", "prom export").Msg("unable to coerce")
			return
		}
		if _, isMetric := v["_type"]; isMetric {
			walkMetrics(w, prefix, ts, v["_value"])
		} else {
			for pfx, v2 := range v {
				walkMetrics(w, prefix+"`"+pfx, ts, v2)
			}
		}
	case "string":
		w.Write([]byte(fmt.Sprintf("#TEXT %s %s %d\n", prefix, val, ts)))
	case "float32":
		fallthrough
	case "float64":
		w.Write([]byte(fmt.Sprintf("%s %e %d\n", prefix, val, ts)))
	case "int32":
		fallthrough
	case "int64":
		fallthrough
	case "uint32":
		fallthrough
	case "uint64":
		w.Write([]byte(fmt.Sprintf("%s %d %d\n", prefix, val, ts)))
	default:
		w.Write([]byte(fmt.Sprintf("%v(%v) = %#v\n", prefix, t, val)))
	}
}

// inventory returns the current, active plugin inventory
func (s *Server) inventory(w http.ResponseWriter, r *http.Request) {
	inventory := s.plugins.Inventory()
	if inventory == nil {
		inventory = []byte(`{"error": "empty inventory"}`)
		s.logger.Error().Msg("inventory is nil/empty...")
	}

	w.Header().Set("Content-Type", "application/json")
	w.Write(inventory)
}

// write handles PUT/POST requests with a JSON playload containing "freeform"
// metrics. No validation is applied to the "format" of the metrics beyond k/v.
// Where 'key' is the metric name and 'value' is the metric value as either a
// simple value (e.g. {"name": 1, "foo": "bar", ...}) or a structured value
// representation (e.g. {"foo": {_type: "i", _value: 1}, ...}).
func (s *Server) write(w http.ResponseWriter, r *http.Request) {
	id := strings.Replace(r.URL.Path, "/write/", "", -1)

	log.Debug().Str("path", r.URL.Path).Str("id", id).Msg("write request")
	// a write request *MUST* include a metric group id to act as a namespace.
	// in other words, a "plugin name", all metrics for that write will appear
	// _under_ the metric group id (aka plugin name)
	if id == "" {
		http.NotFound(w, r)
		return
	}

	if err := receiver.Parse(id, r.Body); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	w.WriteHeader(http.StatusNoContent)
}
<|endoftext|>


<commit_before>// Copyright © 2017 Circonus, Inc. <support@circonus.com>
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//

package server

import (
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/circonus-labs/circonus-agent/internal/config"
	"github.com/circonus-labs/circonus-agent/internal/server/receiver"
	cgm "github.com/circonus-labs/circonus-gometrics"
	"github.com/spf13/viper"
)

// run handles requests to execute plugins and return metrics emitted
// handles /, /run, or /run/plugin_name
func (s *Server) run(w http.ResponseWriter, r *http.Request) {
	plugin := ""

	if strings.HasPrefix(r.URL.Path, "/run/") { // run specific plugin
		plugin = strings.Replace(r.URL.Path, "/run/", "", -1)
		if plugin != "" {
			if !s.plugins.IsInternal(plugin) && !s.plugins.IsValid(plugin) {
				s.logger.Warn().
					Str("plugin", plugin).
					Msg("Unknown plugin requested")
				http.NotFound(w, r)
				return
			}
		}
	}

	lastMeticsmu.Lock()
	defer lastMeticsmu.Unlock()

	metrics := map[string]interface{}{}

	if plugin == "" || !s.plugins.IsInternal(plugin) {
		// NOTE: errors are ignored from plugins.Run
		//       1. errors are already logged by Run
		//       2. do not expose execution state to callers
		s.plugins.Run(plugin)
		pluginMetrics := s.plugins.Flush(plugin)
		metrics = *pluginMetrics
	}

	if plugin == "" || plugin == "write" {
		receiverMetrics := receiver.Flush()
		for metricName, metric := range *receiverMetrics {
			metrics[metricName] = metric
		}
	}

	if plugin == "" || plugin == "statsd" {
		if s.statsdSvr != nil {
			statsdMetrics := s.statsdSvr.Flush()
			if statsdMetrics != nil {
				metrics[viper.GetString(config.KeyStatsdHostCategory)] = statsdMetrics
			}
		}
	}

	lastMetrics.metrics = metrics
	lastMetrics.ts = time.Now()

	w.Header().Set("Content-Type", "application/json")
	if err := json.NewEncoder(w).Encode(metrics); err != nil {
		s.logger.Error().
			Err(err).
			Msg("Writing metrics to response")
	}
}

// inventory returns the current, active plugin inventory
func (s *Server) inventory(w http.ResponseWriter, r *http.Request) {
	inventory := s.plugins.Inventory()
	if inventory == nil {
		inventory = []byte(`{"error": "empty inventory"}`)
		s.logger.Error().Msg("inventory is nil/empty...")
	}

	w.Header().Set("Content-Type", "application/json")
	w.Write(inventory)
}

// write handles PUT/POST requests with a JSON playload containing "freeform"
// metrics. No validation is applied to the "format" of the metrics beyond k/v.
// Where 'key' is the metric name and 'value' is the metric value as either a
// simple value (e.g. {"name": 1, "foo": "bar", ...}) or a structured value
// representation (e.g. {"foo": {_type: "i", _value: 1}, ...}).
func (s *Server) write(w http.ResponseWriter, r *http.Request) {
	id := strings.Replace(r.URL.Path, "/write/", "", -1)

	s.logger.Debug().Str("path", r.URL.Path).Str("id", id).Msg("write request")
	// a write request *MUST* include a metric group id to act as a namespace.
	// in other words, a "plugin name", all metrics for that write will appear
	// _under_ the metric group id (aka plugin name)
	if id == "" {
		http.NotFound(w, r)
		return
	}

	if err := receiver.Parse(id, r.Body); err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	w.WriteHeader(http.StatusNoContent)
}

// promOutput returns the last metrics in prom format
func (s *Server) promOutput(w http.ResponseWriter, r *http.Request) {
	if lastMetrics.metrics == nil || len(lastMetrics.metrics) == 0 {
		w.WriteHeader(http.StatusNoContent)
		return
	}

	ms := lastMetrics.ts.UnixNano() / int64(time.Millisecond)

	w.Header().Set("Content-Type", "text/plain")
	w.WriteHeader(http.StatusOK)
	for id, data := range lastMetrics.metrics {
		s.metricsToPromFormat(w, id, ms, data)
	}
}

func (s *Server) metricsToPromFormat(w io.Writer, prefix string, ts int64, val interface{}) {
	l := s.logger.With().Str("op", "prom export").Logger()
	switch t := val.(type) {
	case cgm.Metric:
		metric := val.(cgm.Metric)
		sv := fmt.Sprintf("%v", metric.Value)
		switch metric.Type {
		case "i":
			fallthrough
		case "I":
			fallthrough
		case "l":
			fallthrough
		case "L":
			v, err := strconv.ParseInt(sv, 10, 64)
			if err != nil {
				l.Error().Err(err).Msg("conv int64")
				return
			}
			if _, err := w.Write([]byte(fmt.Sprintf("%s %d %d\n", prefix, v, ts))); err != nil {
				l.Error().Err(err).Msg("writing prom output")
			}
		case "n":
			if strings.Contains(sv, "[H[") {
				l.Warn().
					Str("type", "histogram != [prom]histogram(percentile)").
					Str("metric", fmt.Sprintf("%s = %s", prefix, sv)).
					Msg("unsupported metric type")
			} else {
				v, err := strconv.ParseFloat(sv, 64)
				if err != nil {
					l.Error().Err(err).Msg("conv float64")
					return
				}
				if _, err := w.Write([]byte(fmt.Sprintf("%s %f %d\n", prefix, v, ts))); err != nil {
					l.Error().Err(err).Msg("writing prom output")
				}
			}
		case "s":
			l.Warn().
				Str("type", "text [prom]???").
				Str("metric", fmt.Sprintf("%s = %s", prefix, sv)).
				Msg("unsuported metric type")
		default:
			l.Warn().
				Str("type", metric.Type).
				Str("name", prefix).
				Interface("metric", metric).
				Msg("invalid metric type")
		}
	case cgm.Metrics:
		metrics := val.(cgm.Metrics)
		for pfx, metric := range metrics {
			name := prefix
			if pfx != "" {
				name = strings.Join([]string{name, pfx}, config.MetricNameSeparator)
			}
			s.metricsToPromFormat(w, name, ts, metric)
		}
	case *cgm.Metrics:
		metrics := val.(*cgm.Metrics)
		s.metricsToPromFormat(w, prefix, ts, *metrics)
	default:
		l.Warn().
			Str("metric", fmt.Sprintf("#TYPE(%T) %v = %#v", t, prefix, val)).
			Msg("unhandled export type")
	}
}
<commit_msg>add socketHandler specifically for the socket listener - only allows /write<commit_after>// Copyright © 2017 Circonus, Inc. <support@circonus.com>
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//

package server

import (
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/circonus-labs/circonus-agent/internal/config"
	"github.com/circonus-labs/circonus-agent/internal/server/receiver"
	cgm "github.com/circonus-labs/circonus-gometrics"
	appstats "github.com/maier/go-appstats"
	"github.com/spf13/viper"
)

// run handles requests to execute plugins and return metrics emitted
// handles /, /run, or /run/plugin_name
func (s *Server) run(w http.ResponseWriter, r *http.Request) {
	plugin := ""

	if strings.HasPrefix(r.URL.Path, "/run/") { // run specific plugin
		plugin = strings.Replace(r.URL.Path, "/run/", "", -1)
		if plugin != "" {
			if !s.plugins.IsInternal(plugin) && !s.plugins.IsValid(plugin) {
				s.logger.Warn().
					Str("plugin", plugin).
					Msg("Unknown plugin requested")
				http.NotFound(w, r)
				return
			}
		}
	}

	lastMeticsmu.Lock()
	defer lastMeticsmu.Unlock()

	metrics := map[string]interface{}{}

	if plugin == "" || !s.plugins.IsInternal(plugin) {
		// NOTE: errors are ignored from plugins.Run
		//       1. errors are already logged by Run
		//       2. do not expose execution state to callers
		s.plugins.Run(plugin)
		pluginMetrics := s.plugins.Flush(plugin)
		metrics = *pluginMetrics
	}

	if plugin == "" || plugin == "write" {
		receiverMetrics := receiver.Flush()
		for metricName, metric := range *receiverMetrics {
			metrics[metricName] = metric
		}
	}

	if plugin == "" || plugin == "statsd" {
		if s.statsdSvr != nil {
			statsdMetrics := s.statsdSvr.Flush()
			if statsdMetrics != nil {
				metrics[viper.GetString(config.KeyStatsdHostCategory)] = statsdMetrics
			}
		}
	}

	lastMetrics.metrics = metrics
	lastMetrics.ts = time.Now()

	w.Header().Set("Content-Type", "application/json")
	if err := json.NewEncoder(w).Encode(metrics); err != nil {
		s.logger.Error().
			Err(err).
			Msg("Writing metrics to response")
	}
}

// inventory returns the current, active plugin inventory
func (s *Server) inventory(w http.ResponseWriter, r *http.Request) {
	inventory := s.plugins.Inventory()
	if inventory == nil {
		inventory = []byte(`{"error": "empty inventory"}`)
		s.logger.Error().Msg("inventory is nil/empty...")
	}

	w.Header().Set("Content-Type", "application/json")
	w.Write(inventory)
}

// socketHandler gates /write for the socket server only
func (s *Server) socketHandler(w http.ResponseWriter, r *http.Request) {
	if !writePathRx.MatchString(r.URL.Path) {
		appstats.IncrementInt("requests_bad")
		s.logger.Warn().
			Str("method", r.Method).
			Str("url", r.URL.String()).
			Msg("Not found")
		http.NotFound(w, r)
		return
	}

	if r.Method != "PUT" && r.Method != "POST" {
		appstats.IncrementInt("requests_bad")
		s.logger.Warn().
			Str("method", r.Method).
			Str("url", r.URL.String()).
			Msg("Not found")
		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
		return
	}

	s.write(w, r)
}

// write handles PUT/POST requests with a JSON playload containing "freeform"
// metrics. No validation is applied to the "format" of the metrics beyond k/v.
// Where 'key' is the metric name and 'value' is the metric value as either a
// simple value (e.g. {"name": 1, "foo": "bar", ...}) or a structured value
// representation (e.g. {"foo": {_type: "i", _value: 1}, ...}).
func (s *Server) write(w http.ResponseWriter, r *http.Request) {
	id := strings.Replace(r.URL.Path, "/write/", "", -1)

	s.logger.Debug().Str("path", r.URL.Path).Str("id", id).Msg("write request")
	// a write request *MUST* include a metric group id to act as a namespace.
	// in other words, a "plugin name", all metrics for that write will appear
	// _under_ the metric group id (aka plugin name)
	if id == "" {
		http.NotFound(w, r)
		return
	}

	if err := receiver.Parse(id, r.Body); err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	w.WriteHeader(http.StatusNoContent)
}

// promOutput returns the last metrics in prom format
func (s *Server) promOutput(w http.ResponseWriter, r *http.Request) {
	if lastMetrics.metrics == nil || len(lastMetrics.metrics) == 0 {
		w.WriteHeader(http.StatusNoContent)
		return
	}

	ms := lastMetrics.ts.UnixNano() / int64(time.Millisecond)

	w.Header().Set("Content-Type", "text/plain")
	w.WriteHeader(http.StatusOK)
	for id, data := range lastMetrics.metrics {
		s.metricsToPromFormat(w, id, ms, data)
	}
}

func (s *Server) metricsToPromFormat(w io.Writer, prefix string, ts int64, val interface{}) {
	l := s.logger.With().Str("op", "prom export").Logger()
	switch t := val.(type) {
	case cgm.Metric:
		metric := val.(cgm.Metric)
		sv := fmt.Sprintf("%v", metric.Value)
		switch metric.Type {
		case "i":
			fallthrough
		case "I":
			fallthrough
		case "l":
			fallthrough
		case "L":
			v, err := strconv.ParseInt(sv, 10, 64)
			if err != nil {
				l.Error().Err(err).Msg("conv int64")
				return
			}
			if _, err := w.Write([]byte(fmt.Sprintf("%s %d %d\n", prefix, v, ts))); err != nil {
				l.Error().Err(err).Msg("writing prom output")
			}
		case "n":
			if strings.Contains(sv, "[H[") {
				l.Warn().
					Str("type", "histogram != [prom]histogram(percentile)").
					Str("metric", fmt.Sprintf("%s = %s", prefix, sv)).
					Msg("unsupported metric type")
			} else {
				v, err := strconv.ParseFloat(sv, 64)
				if err != nil {
					l.Error().Err(err).Msg("conv float64")
					return
				}
				if _, err := w.Write([]byte(fmt.Sprintf("%s %f %d\n", prefix, v, ts))); err != nil {
					l.Error().Err(err).Msg("writing prom output")
				}
			}
		case "s":
			l.Warn().
				Str("type", "text [prom]???").
				Str("metric", fmt.Sprintf("%s = %s", prefix, sv)).
				Msg("unsuported metric type")
		default:
			l.Warn().
				Str("type", metric.Type).
				Str("name", prefix).
				Interface("metric", metric).
				Msg("invalid metric type")
		}
	case cgm.Metrics:
		metrics := val.(cgm.Metrics)
		for pfx, metric := range metrics {
			name := prefix
			if pfx != "" {
				name = strings.Join([]string{name, pfx}, config.MetricNameSeparator)
			}
			s.metricsToPromFormat(w, name, ts, metric)
		}
	case *cgm.Metrics:
		metrics := val.(*cgm.Metrics)
		s.metricsToPromFormat(w, prefix, ts, *metrics)
	default:
		l.Warn().
			Str("metric", fmt.Sprintf("#TYPE(%T) %v = %#v", t, prefix, val)).
			Msg("unhandled export type")
	}
}
<|endoftext|>


<commit_before>package task

import (
	"context"
	"fmt"
	"sort"
	"strings"

	"github.com/google/go-github/github"
	"github.com/shurcooL/githubv4"
	"golang.org/x/build/internal/workflow"
	goversion "golang.org/x/build/maintner/maintnerd/maintapi/version"
)

// MilestoneTasks contains the tasks used to check and modify GitHub issues' milestones.
type MilestoneTasks struct {
	Client              GitHubClientInterface
	RepoOwner, RepoName string
}

// ReleaseKind is the type of release being run.
type ReleaseKind int

const (
	KindUnknown ReleaseKind = iota
	KindBeta
	KindRC
	KindMajor
	KindCurrentMinor
	KindPrevMinor
)

type ReleaseMilestones struct {
	Current, Next int
}

// FetchMilestones returns the milestone numbers for the version currently being
// released, and the next version that outstanding issues should be moved to.
// If this is a major release, it also creates its first minor release
// milestone.
func (m *MilestoneTasks) FetchMilestones(ctx *workflow.TaskContext, currentVersion string, kind ReleaseKind) (ReleaseMilestones, error) {
	x, ok := goversion.Go1PointX(currentVersion)
	if !ok {
		return ReleaseMilestones{}, fmt.Errorf("could not parse %q as a Go version", currentVersion)
	}
	majorVersion := fmt.Sprintf("go1.%d", x)

	// RCs and betas use the major version's milestone.
	if kind == KindRC || kind == KindBeta {
		currentVersion = majorVersion
	}

	currentMilestone, err := m.Client.FetchMilestone(ctx, m.RepoOwner, m.RepoName, uppercaseVersion(currentVersion), false)
	if err != nil {
		return ReleaseMilestones{}, err
	}
	nextV, err := nextVersion(currentVersion)
	if err != nil {
		return ReleaseMilestones{}, err
	}
	nextMilestone, err := m.Client.FetchMilestone(ctx, m.RepoOwner, m.RepoName, uppercaseVersion(nextV), true)
	if err != nil {
		return ReleaseMilestones{}, err
	}
	if kind == KindMajor {
		// Create the first minor release milestone too.
		firstMinor := majorVersion + ".1"
		if err != nil {
			return ReleaseMilestones{}, err
		}
		_, err = m.Client.FetchMilestone(ctx, m.RepoOwner, m.RepoName, uppercaseVersion(firstMinor), true)
		if err != nil {
			return ReleaseMilestones{}, err
		}
	}
	return ReleaseMilestones{Current: currentMilestone, Next: nextMilestone}, nil
}

func uppercaseVersion(version string) string {
	return strings.Replace(version, "go", "Go", 1)
}

// CheckBlockers returns an error if there are open release blockers in
// the current milestone.
func (m *MilestoneTasks) CheckBlockers(ctx *workflow.TaskContext, milestones ReleaseMilestones, version string, kind ReleaseKind) error {
	issues, err := m.loadMilestoneIssues(ctx, milestones.Current, kind)
	if err != nil {
		return err
	}
	var blockers []string
	for number, labels := range issues {
		releaseBlocker := labels["release-blocker"]
		if kind == KindBeta && (labels["okay-after-beta1"] || !strings.HasSuffix(version, "beta1")) {
			releaseBlocker = false
		}
		if releaseBlocker {
			blockers = append(blockers, fmt.Sprintf("https://go.dev/issue/%v", number))
		}
	}
	sort.Strings(blockers)
	if len(blockers) != 0 {
		return fmt.Errorf("open release blockers:\n%v", strings.Join(blockers, "\n"))
	}
	return nil
}

// loadMilestoneIssues returns all the open issues in the specified milestone
// and their labels.
func (m *MilestoneTasks) loadMilestoneIssues(ctx *workflow.TaskContext, milestoneID int, kind ReleaseKind) (map[int]map[string]bool, error) {
	issues := map[int]map[string]bool{}
	var query struct {
		Repository struct {
			Issues struct {
				PageInfo struct {
					EndCursor   githubv4.String
					HasNextPage bool
				}

				Nodes []struct {
					Number int
					ID     githubv4.ID
					Title  string
					Labels struct {
						PageInfo struct {
							HasNextPage bool
						}
						Nodes []struct {
							Name string
						}
					} `graphql:"labels(first:10)"`
				}
			} `graphql:"issues(first:100, after:$afterToken, filterBy:{states:OPEN, milestoneNumber:$milestoneNumber})"`
		} `graphql:"repository(owner: $repoOwner, name: $repoName)"`
	}
	var afterToken *githubv4.String
more:
	if err := m.Client.Query(ctx, &query, map[string]interface{}{
		"repoOwner":       githubv4.String(m.RepoOwner),
		"repoName":        githubv4.String(m.RepoName),
		"milestoneNumber": githubv4.String(fmt.Sprint(milestoneID)),
		"afterToken":      afterToken,
	}); err != nil {
		return nil, err
	}
	for _, issue := range query.Repository.Issues.Nodes {
		if issue.Labels.PageInfo.HasNextPage {
			return nil, fmt.Errorf("issue %v (#%v) has more than 10 labels", issue.Title, issue.Number)
		}
		labels := map[string]bool{}
		for _, label := range issue.Labels.Nodes {
			labels[label.Name] = true
		}
		issues[issue.Number] = labels
	}
	if query.Repository.Issues.PageInfo.HasNextPage {
		afterToken = &query.Repository.Issues.PageInfo.EndCursor
		goto more
	}
	return issues, nil
}

// PushIssues updates issues to reflect a finished release. For beta1 releases,
// it removes the okay-after-beta1 label. For major and minor releases,
// it moves them to the next milestone and closes the current one.
func (m *MilestoneTasks) PushIssues(ctx *workflow.TaskContext, milestones ReleaseMilestones, version string, kind ReleaseKind) error {
	// For RCs we don't change issues at all.
	if kind == KindRC {
		return nil
	}

	issues, err := m.loadMilestoneIssues(ctx, milestones.Current, KindUnknown)
	if err != nil {
		return err
	}
	for issueNumber, labels := range issues {
		var newLabels *[]string
		var newMilestone *int
		if kind == KindBeta && strings.HasSuffix(version, "beta1") {
			if labels["okay-after-beta1"] {
				newLabels = &[]string{}
				for label := range labels {
					if label == "okay-after-beta1" {
						continue
					}
					*newLabels = append(*newLabels, label)
				}
			}
		} else if kind == KindMajor || kind == KindCurrentMinor || kind == KindPrevMinor {
			newMilestone = &milestones.Next
		}
		_, _, err := m.Client.EditIssue(ctx, m.RepoOwner, m.RepoName, issueNumber, &github.IssueRequest{
			Milestone: newMilestone,
			Labels:    newLabels,
		})
		if err != nil {
			return err
		}
	}
	if kind == KindMajor || kind == KindCurrentMinor || kind == KindPrevMinor {
		_, _, err := m.Client.EditMilestone(ctx, m.RepoOwner, m.RepoName, milestones.Current, &github.Milestone{
			State: github.String("closed"),
		})
		if err != nil {
			return err
		}
	}
	return nil
}

// GitHubClientInterface is a wrapper around the GitHub v3 and v4 APIs, for
// testing and dry-run support.
type GitHubClientInterface interface {
	// FetchMilestone returns the number of the requested milestone. If create is true,
	// and the milestone doesn't exist, it will be created.
	FetchMilestone(ctx context.Context, owner, repo, name string, create bool) (int, error)

	// See githubv4.Client.Query.
	Query(ctx context.Context, q interface{}, variables map[string]interface{}) error

	// See github.Client.Issues.Edit.
	EditIssue(ctx context.Context, owner string, repo string, number int, issue *github.IssueRequest) (*github.Issue, *github.Response, error)

	// See github.Client.Issues.EditMilestone
	EditMilestone(ctx context.Context, owner string, repo string, number int, milestone *github.Milestone) (*github.Milestone, *github.Response, error)
}

type GitHubClient struct {
	V3 *github.Client
	V4 *githubv4.Client
}

func (c *GitHubClient) Query(ctx context.Context, q interface{}, variables map[string]interface{}) error {
	return c.V4.Query(ctx, q, variables)
}

func (c *GitHubClient) FetchMilestone(ctx context.Context, owner, repo, name string, create bool) (int, error) {
	n, found, err := findMilestone(ctx, c.V4, owner, repo, name)
	if err != nil {
		return 0, err
	}
	if found {
		return n, nil
	} else if !create {
		return 0, fmt.Errorf("no milestone named %q found, and creation was disabled", name)
	}
	m, _, createErr := c.V3.Issues.CreateMilestone(ctx, owner, repo, &github.Milestone{
		Title: github.String(name),
	})
	if createErr != nil {
		return 0, fmt.Errorf("could not find an open milestone named %q and creating it failed: %v", name, createErr)
	}
	return *m.Number, nil
}

func findMilestone(ctx context.Context, client *githubv4.Client, owner, repo, name string) (int, bool, error) {
	var query struct {
		Repository struct {
			Milestones struct {
				Nodes []struct {
					Title  string
					Number int
					State  string
				}
			} `graphql:"milestones(first:10, query: $milestoneName)"`
		} `graphql:"repository(owner: $repoOwner, name: $repoName)"`
	}
	if err := client.Query(ctx, &query, map[string]interface{}{
		"repoOwner":     githubv4.String(owner),
		"repoName":      githubv4.String(repo),
		"milestoneName": githubv4.String(name),
	}); err != nil {
		return 0, false, err
	}
	// The milestone query is case-insensitive and a partial match; we're okay
	// with case variations but it needs to be a full match.
	var open, closed []string
	milestoneNumber := 0
	for _, m := range query.Repository.Milestones.Nodes {
		if strings.ToLower(name) != strings.ToLower(m.Title) {
			continue
		}
		if m.State == "OPEN" {
			open = append(open, m.Title)
			milestoneNumber = m.Number
		} else {
			closed = append(closed, m.Title)
		}
	}
	// GitHub allows "go" and "Go" to exist at the same time.
	// If there's any confusion, fail: we expect either one open milestone,
	// or no matching milestones at all.
	switch {
	case len(open) == 1:
		return milestoneNumber, true, nil
	case len(open) > 1:
		return 0, false, fmt.Errorf("multiple open milestones matching %q: %q", name, open)
	// No open milestones.
	case len(closed) == 0:
		return 0, false, nil
	case len(closed) > 0:
		return 0, false, fmt.Errorf("no open milestones matching %q, but some closed: %q (re-open or delete?)", name, closed)
	}
	// The switch above is exhaustive.
	panic(fmt.Errorf("unhandled case: open: %q closed: %q", open, closed))
}

func (c *GitHubClient) EditIssue(ctx context.Context, owner string, repo string, number int, issue *github.IssueRequest) (*github.Issue, *github.Response, error) {
	return c.V3.Issues.Edit(ctx, owner, repo, number, issue)
}

func (c *GitHubClient) EditMilestone(ctx context.Context, owner string, repo string, number int, milestone *github.Milestone) (*github.Milestone, *github.Response, error) {
	return c.V3.Issues.EditMilestone(ctx, owner, repo, number, milestone)
}
<commit_msg>internal/task: don't check blockers for RC releases<commit_after>package task

import (
	"context"
	"fmt"
	"sort"
	"strings"

	"github.com/google/go-github/github"
	"github.com/shurcooL/githubv4"
	"golang.org/x/build/internal/workflow"
	goversion "golang.org/x/build/maintner/maintnerd/maintapi/version"
)

// MilestoneTasks contains the tasks used to check and modify GitHub issues' milestones.
type MilestoneTasks struct {
	Client              GitHubClientInterface
	RepoOwner, RepoName string
}

// ReleaseKind is the type of release being run.
type ReleaseKind int

const (
	KindUnknown ReleaseKind = iota
	KindBeta
	KindRC
	KindMajor
	KindCurrentMinor
	KindPrevMinor
)

type ReleaseMilestones struct {
	Current, Next int
}

// FetchMilestones returns the milestone numbers for the version currently being
// released, and the next version that outstanding issues should be moved to.
// If this is a major release, it also creates its first minor release
// milestone.
func (m *MilestoneTasks) FetchMilestones(ctx *workflow.TaskContext, currentVersion string, kind ReleaseKind) (ReleaseMilestones, error) {
	x, ok := goversion.Go1PointX(currentVersion)
	if !ok {
		return ReleaseMilestones{}, fmt.Errorf("could not parse %q as a Go version", currentVersion)
	}
	majorVersion := fmt.Sprintf("go1.%d", x)

	// RCs and betas use the major version's milestone.
	if kind == KindRC || kind == KindBeta {
		currentVersion = majorVersion
	}

	currentMilestone, err := m.Client.FetchMilestone(ctx, m.RepoOwner, m.RepoName, uppercaseVersion(currentVersion), false)
	if err != nil {
		return ReleaseMilestones{}, err
	}
	nextV, err := nextVersion(currentVersion)
	if err != nil {
		return ReleaseMilestones{}, err
	}
	nextMilestone, err := m.Client.FetchMilestone(ctx, m.RepoOwner, m.RepoName, uppercaseVersion(nextV), true)
	if err != nil {
		return ReleaseMilestones{}, err
	}
	if kind == KindMajor {
		// Create the first minor release milestone too.
		firstMinor := majorVersion + ".1"
		if err != nil {
			return ReleaseMilestones{}, err
		}
		_, err = m.Client.FetchMilestone(ctx, m.RepoOwner, m.RepoName, uppercaseVersion(firstMinor), true)
		if err != nil {
			return ReleaseMilestones{}, err
		}
	}
	return ReleaseMilestones{Current: currentMilestone, Next: nextMilestone}, nil
}

func uppercaseVersion(version string) string {
	return strings.Replace(version, "go", "Go", 1)
}

// CheckBlockers returns an error if there are open release blockers in
// the current milestone.
func (m *MilestoneTasks) CheckBlockers(ctx *workflow.TaskContext, milestones ReleaseMilestones, version string, kind ReleaseKind) error {
	if kind == KindRC {
		// We don't check blockers for release candidates; they're expected to
		// at least have recurring blockers, and we don't have an okay-after
		// label to suppress them.
		return nil
	}
	issues, err := m.loadMilestoneIssues(ctx, milestones.Current, kind)
	if err != nil {
		return err
	}
	var blockers []string
	for number, labels := range issues {
		releaseBlocker := labels["release-blocker"]
		if kind == KindBeta && (labels["okay-after-beta1"] || !strings.HasSuffix(version, "beta1")) {
			releaseBlocker = false
		}
		if releaseBlocker {
			blockers = append(blockers, fmt.Sprintf("https://go.dev/issue/%v", number))
		}
	}
	sort.Strings(blockers)
	if len(blockers) != 0 {
		return fmt.Errorf("open release blockers:\n%v", strings.Join(blockers, "\n"))
	}
	return nil
}

// loadMilestoneIssues returns all the open issues in the specified milestone
// and their labels.
func (m *MilestoneTasks) loadMilestoneIssues(ctx *workflow.TaskContext, milestoneID int, kind ReleaseKind) (map[int]map[string]bool, error) {
	issues := map[int]map[string]bool{}
	var query struct {
		Repository struct {
			Issues struct {
				PageInfo struct {
					EndCursor   githubv4.String
					HasNextPage bool
				}

				Nodes []struct {
					Number int
					ID     githubv4.ID
					Title  string
					Labels struct {
						PageInfo struct {
							HasNextPage bool
						}
						Nodes []struct {
							Name string
						}
					} `graphql:"labels(first:10)"`
				}
			} `graphql:"issues(first:100, after:$afterToken, filterBy:{states:OPEN, milestoneNumber:$milestoneNumber})"`
		} `graphql:"repository(owner: $repoOwner, name: $repoName)"`
	}
	var afterToken *githubv4.String
more:
	if err := m.Client.Query(ctx, &query, map[string]interface{}{
		"repoOwner":       githubv4.String(m.RepoOwner),
		"repoName":        githubv4.String(m.RepoName),
		"milestoneNumber": githubv4.String(fmt.Sprint(milestoneID)),
		"afterToken":      afterToken,
	}); err != nil {
		return nil, err
	}
	for _, issue := range query.Repository.Issues.Nodes {
		if issue.Labels.PageInfo.HasNextPage {
			return nil, fmt.Errorf("issue %v (#%v) has more than 10 labels", issue.Title, issue.Number)
		}
		labels := map[string]bool{}
		for _, label := range issue.Labels.Nodes {
			labels[label.Name] = true
		}
		issues[issue.Number] = labels
	}
	if query.Repository.Issues.PageInfo.HasNextPage {
		afterToken = &query.Repository.Issues.PageInfo.EndCursor
		goto more
	}
	return issues, nil
}

// PushIssues updates issues to reflect a finished release. For beta1 releases,
// it removes the okay-after-beta1 label. For major and minor releases,
// it moves them to the next milestone and closes the current one.
func (m *MilestoneTasks) PushIssues(ctx *workflow.TaskContext, milestones ReleaseMilestones, version string, kind ReleaseKind) error {
	// For RCs we don't change issues at all.
	if kind == KindRC {
		return nil
	}

	issues, err := m.loadMilestoneIssues(ctx, milestones.Current, KindUnknown)
	if err != nil {
		return err
	}
	for issueNumber, labels := range issues {
		var newLabels *[]string
		var newMilestone *int
		if kind == KindBeta && strings.HasSuffix(version, "beta1") {
			if labels["okay-after-beta1"] {
				newLabels = &[]string{}
				for label := range labels {
					if label == "okay-after-beta1" {
						continue
					}
					*newLabels = append(*newLabels, label)
				}
			}
		} else if kind == KindMajor || kind == KindCurrentMinor || kind == KindPrevMinor {
			newMilestone = &milestones.Next
		}
		_, _, err := m.Client.EditIssue(ctx, m.RepoOwner, m.RepoName, issueNumber, &github.IssueRequest{
			Milestone: newMilestone,
			Labels:    newLabels,
		})
		if err != nil {
			return err
		}
	}
	if kind == KindMajor || kind == KindCurrentMinor || kind == KindPrevMinor {
		_, _, err := m.Client.EditMilestone(ctx, m.RepoOwner, m.RepoName, milestones.Current, &github.Milestone{
			State: github.String("closed"),
		})
		if err != nil {
			return err
		}
	}
	return nil
}

// GitHubClientInterface is a wrapper around the GitHub v3 and v4 APIs, for
// testing and dry-run support.
type GitHubClientInterface interface {
	// FetchMilestone returns the number of the requested milestone. If create is true,
	// and the milestone doesn't exist, it will be created.
	FetchMilestone(ctx context.Context, owner, repo, name string, create bool) (int, error)

	// See githubv4.Client.Query.
	Query(ctx context.Context, q interface{}, variables map[string]interface{}) error

	// See github.Client.Issues.Edit.
	EditIssue(ctx context.Context, owner string, repo string, number int, issue *github.IssueRequest) (*github.Issue, *github.Response, error)

	// See github.Client.Issues.EditMilestone
	EditMilestone(ctx context.Context, owner string, repo string, number int, milestone *github.Milestone) (*github.Milestone, *github.Response, error)
}

type GitHubClient struct {
	V3 *github.Client
	V4 *githubv4.Client
}

func (c *GitHubClient) Query(ctx context.Context, q interface{}, variables map[string]interface{}) error {
	return c.V4.Query(ctx, q, variables)
}

func (c *GitHubClient) FetchMilestone(ctx context.Context, owner, repo, name string, create bool) (int, error) {
	n, found, err := findMilestone(ctx, c.V4, owner, repo, name)
	if err != nil {
		return 0, err
	}
	if found {
		return n, nil
	} else if !create {
		return 0, fmt.Errorf("no milestone named %q found, and creation was disabled", name)
	}
	m, _, createErr := c.V3.Issues.CreateMilestone(ctx, owner, repo, &github.Milestone{
		Title: github.String(name),
	})
	if createErr != nil {
		return 0, fmt.Errorf("could not find an open milestone named %q and creating it failed: %v", name, createErr)
	}
	return *m.Number, nil
}

func findMilestone(ctx context.Context, client *githubv4.Client, owner, repo, name string) (int, bool, error) {
	var query struct {
		Repository struct {
			Milestones struct {
				Nodes []struct {
					Title  string
					Number int
					State  string
				}
			} `graphql:"milestones(first:10, query: $milestoneName)"`
		} `graphql:"repository(owner: $repoOwner, name: $repoName)"`
	}
	if err := client.Query(ctx, &query, map[string]interface{}{
		"repoOwner":     githubv4.String(owner),
		"repoName":      githubv4.String(repo),
		"milestoneName": githubv4.String(name),
	}); err != nil {
		return 0, false, err
	}
	// The milestone query is case-insensitive and a partial match; we're okay
	// with case variations but it needs to be a full match.
	var open, closed []string
	milestoneNumber := 0
	for _, m := range query.Repository.Milestones.Nodes {
		if strings.ToLower(name) != strings.ToLower(m.Title) {
			continue
		}
		if m.State == "OPEN" {
			open = append(open, m.Title)
			milestoneNumber = m.Number
		} else {
			closed = append(closed, m.Title)
		}
	}
	// GitHub allows "go" and "Go" to exist at the same time.
	// If there's any confusion, fail: we expect either one open milestone,
	// or no matching milestones at all.
	switch {
	case len(open) == 1:
		return milestoneNumber, true, nil
	case len(open) > 1:
		return 0, false, fmt.Errorf("multiple open milestones matching %q: %q", name, open)
	// No open milestones.
	case len(closed) == 0:
		return 0, false, nil
	case len(closed) > 0:
		return 0, false, fmt.Errorf("no open milestones matching %q, but some closed: %q (re-open or delete?)", name, closed)
	}
	// The switch above is exhaustive.
	panic(fmt.Errorf("unhandled case: open: %q closed: %q", open, closed))
}

func (c *GitHubClient) EditIssue(ctx context.Context, owner string, repo string, number int, issue *github.IssueRequest) (*github.Issue, *github.Response, error) {
	return c.V3.Issues.Edit(ctx, owner, repo, number, issue)
}

func (c *GitHubClient) EditMilestone(ctx context.Context, owner string, repo string, number int, milestone *github.Milestone) (*github.Milestone, *github.Response, error) {
	return c.V3.Issues.EditMilestone(ctx, owner, repo, number, milestone)
}
<|endoftext|>


<commit_before>package template

var (
	Module = `module {{.Dir}}

go 1.15

require (
	github.com/micro/micro/v3 v3.0.0-beta.4.0.20200922151713-de8b56c2b15d
	github.com/micro/go-micro/v3 v3.0.0-beta.2.0.20200922112322-927d4f8eced6
)

// This can be removed once etcd becomes go gettable, version 3.4 and 3.5 is not,
// see https://github.com/etcd-io/etcd/issues/11154 and https://github.com/etcd-io/etcd/issues/11931.
replace google.golang.org/grpc => google.golang.org/grpc v1.26.0
`
)
<commit_msg>Bump deps of micro newed services (#1561)<commit_after>package template

var (
	Module = `module {{.Dir}}

go 1.15

require (
	github.com/micro/micro/v3 v3.0.0-beta.7
)

// This can be removed once etcd becomes go gettable, version 3.4 and 3.5 is not,
// see https://github.com/etcd-io/etcd/issues/11154 and https://github.com/etcd-io/etcd/issues/11931.
replace google.golang.org/grpc => google.golang.org/grpc v1.26.0
`
)
<|endoftext|>


<commit_before>/*--------------------------------------------------------*\
|                                                          |
|                          hprose                          |
|                                                          |
| Official WebSite: https://hprose.com                     |
|                                                          |
| io/encoding/time_encoder.go                              |
|                                                          |
| LastModified: Mar 20, 2020                               |
| Author: Ma Bingyao <andot@hprose.com>                    |
|                                                          |
\*________________________________________________________*/

package encoding

import (
	"reflect"
	"time"

	"github.com/modern-go/reflect2"
)

// TimeEncoder is the implementation of ValueEncoder for time.Time/*time.Time.
type TimeEncoder struct{}

// Encode writes the hprose encoding of v to stream
// if v is already written to stream, it will writes it as reference
func (valenc TimeEncoder) Encode(enc *Encoder, v interface{}) (err error) {
	if reflect.TypeOf(v).Kind() == reflect.Struct {
		return valenc.Write(enc, v)
	}
	if reflect2.IsNil(v) {
		return WriteNil(enc.Writer)
	}
	var ok bool
	if ok, err = enc.WriteReference(v); !ok && err == nil {
		err = valenc.Write(enc, v)
	}
	return
}

// Write writes the hprose encoding of v to stream
// if v is already written to stream, it will writes it as value
func (TimeEncoder) Write(enc *Encoder, v interface{}) (err error) {
	t := reflect.TypeOf(v)
	if t.Kind() == reflect.Ptr {
		enc.SetReference(v)
	} else {
		enc.AddReferenceCount(1)
	}
	return WriteTime(enc.Writer, *(*time.Time)(reflect2.PtrOf(v)))
}

func init() {
	RegisterEncoder((*time.Time)(nil), TimeEncoder{})
}
<commit_msg>Update time_encoder.go<commit_after>/*--------------------------------------------------------*\
|                                                          |
|                          hprose                          |
|                                                          |
| Official WebSite: https://hprose.com                     |
|                                                          |
| io/encoding/time_encoder.go                              |
|                                                          |
| LastModified: Mar 21, 2020                               |
| Author: Ma Bingyao <andot@hprose.com>                    |
|                                                          |
\*________________________________________________________*/

package encoding

import (
	"time"

	"github.com/modern-go/reflect2"
)

// TimeEncoder is the implementation of ValueEncoder for time.Time/*time.Time.
type TimeEncoder struct{}

// Encode writes the hprose encoding of v to stream
// if v is already written to stream, it will writes it as reference
func (valenc TimeEncoder) Encode(enc *Encoder, v interface{}) (err error) {
	return ReferenceEncode(valenc, enc, v)
}

// Write writes the hprose encoding of v to stream
// if v is already written to stream, it will writes it as value
func (TimeEncoder) Write(enc *Encoder, v interface{}) (err error) {
	SetReference(enc, v)
	return WriteTime(enc.Writer, *(*time.Time)(reflect2.PtrOf(v)))
}

func init() {
	RegisterEncoder((*time.Time)(nil), TimeEncoder{})
}
<|endoftext|>


<commit_before>// +build linux

package hsup

import (
	"bytes"
	"encoding/gob"
	"fmt"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"syscall"

	"code.google.com/p/go-uuid/uuid"

	"github.com/docker/libcontainer"
	"github.com/docker/libcontainer/cgroups"
	"github.com/docker/libcontainer/devices"
	"github.com/docker/libcontainer/mount"
	"github.com/docker/libcontainer/namespaces"
)

type LibContainerDynoDriver struct {
	workDir       string
	stacksDir     string
	containersDir string
	allocator     *Allocator
}

func NewLibContainerDynoDriver(workDir string) (*LibContainerDynoDriver, error) {
	var (
		stacksDir     = filepath.Join(workDir, "stacks")
		containersDir = filepath.Join(workDir, "containers")
	)
	if err := os.MkdirAll(stacksDir, 0755); err != nil {
		return nil, err
	}
	if err := os.MkdirAll(containersDir, 0755); err != nil {
		return nil, err
	}
	allocator, err := NewAllocator(workDir)
	if err != nil {
		return nil, err
	}
	return &LibContainerDynoDriver{
		workDir:       workDir,
		stacksDir:     stacksDir,
		containersDir: containersDir,
		allocator:     allocator,
	}, nil
}

func (dd *LibContainerDynoDriver) Build(release *Release) error {
	stacks, err := HerokuStacksFromManifest(dd.stacksDir)
	if err != nil {
		return err
	}
	for _, stack := range stacks {
		if strings.TrimSpace(stack.Name) != release.stack {
			continue
		}
		if err := stack.mount(); err != nil {
			return err
		}
	}
	return nil
}

func (dd *LibContainerDynoDriver) Start(ex *Executor) error {
	containerUUID := uuid.New()
	uid, gid, err := dd.allocator.ReserveUID()
	if err != nil {
		return err
	}
	sn, err := dd.allocator.privateNetForUID(uid)
	if err != nil {
		return err
	}
	subnet, err := newSmallSubnet(sn)
	if err != nil {
		return err
	}
	stackImagePath, err := CurrentStackImagePath(
		dd.stacksDir, ex.Release.stack,
	)
	if err != nil {
		return err
	}
	dataPath := filepath.Join(dd.containersDir, containerUUID)
	if err := os.MkdirAll(dataPath, 0755); err != nil {
		return err
	}
	writablePaths := []string{
		filepath.Join(dataPath, "app"),
		filepath.Join(dataPath, "tmp"),
		filepath.Join(dataPath, "var", "tmp"),
	}
	for _, path := range writablePaths {
		if err := os.MkdirAll(path, 0755); err != nil {
			return err
		}
		if err := os.Chown(path, uid, gid); err != nil {
			return err
		}
	}
	rootFSPath := filepath.Join(dataPath, "root")
	if err := os.MkdirAll(rootFSPath, 0755); err != nil {
		return err
	}

	// stack image is the rootFS
	if err := syscall.Mount(
		stackImagePath, rootFSPath, "bind",
		syscall.MS_RDONLY|syscall.MS_BIND, "",
	); err != nil {
		return err
	}

	if err := createPasswdWithDynoUser(
		stackImagePath, dataPath, uid, gid,
	); err != nil {
		return err
	}

	// TODO: inject /tmp/slug.tgz if local

	outsideContainer, err := filepath.Abs(linuxAmd64Path())
	if err != nil {
		return err
	}
	insideContainer := filepath.Join(dataPath, "tmp", "hsup")
	if err := copyFile(outsideContainer, insideContainer, 0755); err != nil {
		return err
	}

	// TODO tty
	console := ""

	ex.initExitStatus = make(chan *ExitStatus)

	cfgReader, cfgWriter, err := os.Pipe()
	initCtx := &containerInit{
		hsupBinaryPath: outsideContainer,
		ex:             ex,
		configPipe:     cfgReader,
	}

	container := containerConfig(
		containerUUID,
		uid, gid,
		dataPath,
		subnet,
		ex.Release.ConfigSlice(),
	)

	// send config to the init process inside the container
	go func() {
		defer cfgWriter.Close()
		encoder := gob.NewEncoder(cfgWriter)
		if err := encoder.Encode(container); err != nil {
			log.Fatal(err)
		}
	}()

	go func() {
		// TODO: stop swallowing errors
		code, err := namespaces.Exec(
			container, os.Stdin, os.Stdout, os.Stderr,
			console, dataPath, []string{},
			initCtx.createCommand, nil, initCtx.startCallback,
		)
		log.Println(code, err)

		// GC
		// TODO: gc after sending back the exit status
		// doing so right now terminates the program too early,
		// before everything is removed
		if err := syscall.Unmount(rootFSPath, 0); err != nil {
			log.Printf("unmount error: %#+v", err)
		}
		for _, path := range writablePaths {
			if err := os.RemoveAll(path); err != nil {
				log.Printf("remove all error: %#+v", err)
			}
		}
		if err := os.RemoveAll(dataPath); err != nil {
			log.Printf("remove all error: %#+v", err)
		}

		// it's probably safe to ignore errors here. Worst case
		// scenario, this uid won't be be reused.
		dd.allocator.FreeUID(uid)

		ex.initExitStatus <- &ExitStatus{Code: code, Err: err}
		close(ex.initExitStatus)
	}()

	return nil
}

func (dd *LibContainerDynoDriver) Wait(ex *Executor) (s *ExitStatus) {
	return <-ex.initExitStatus
}

func (dd *LibContainerDynoDriver) Stop(ex *Executor) error {
	if ex.cmd.ProcessState != nil {
		return nil // already exited
	}

	// TODO: fix a race conditition when Stop() is called before the
	// libcontainer driver re-execs itself

	// tell the abspath-driver to stop
	return ex.cmd.Process.Signal(syscall.SIGTERM)
}

func createPasswdWithDynoUser(stackImagePath, dataPath string, uid, gid int) error {
	var contents bytes.Buffer
	original, err := os.Open(filepath.Join(stackImagePath, "etc", "passwd"))
	if err != nil {
		return err
	}
	defer original.Close()

	if _, err := contents.ReadFrom(original); err != nil {
		return err
	}
	// TODO: allocate a free uid. It is currently hardcoded to 1000
	dynoUser := fmt.Sprintf("\ndyno:x:%d:%d::/app:/bin/bash\n", uid, gid)
	if _, err := contents.WriteString(dynoUser); err != nil {
		return err
	}

	dst, err := os.Create(filepath.Join(dataPath, "passwd"))
	if err != nil {
		return err
	}
	defer dst.Close()
	if err := dst.Chmod(0644); err != nil {
		return err
	}

	_, err = contents.WriteTo(dst)
	return err
}

type containerInit struct {
	hsupBinaryPath string
	ex             *Executor
	configPipe     *os.File
}

func (ctx *containerInit) createCommand(container *libcontainer.Config, console,
	dataPath, init string, controlPipe *os.File, args []string) *exec.Cmd {

	hs := Startup{
		App: AppSerializable{
			Version: ctx.ex.Release.version,
			Env:     ctx.ex.Release.config,
			Slug:    ctx.ex.Release.slugURL,
			Stack:   ctx.ex.Release.stack,
			Processes: []FormationSerializable{
				{
					FArgs:     ctx.ex.Args,
					FQuantity: 1,
					FType:     ctx.ex.ProcessType,
				},
			},
			LogplexURL: ctx.ex.logplexURLString(),
		},
		OneShot:     true,
		StartNumber: ctx.ex.ProcessID,
		Action:      Start,
		Driver:      &LibContainerInitDriver{},
		FormName:    ctx.ex.ProcessType,
	}
	cmd := exec.Command(ctx.hsupBinaryPath)
	cmd.Env = []string{"HSUP_CONTROL_GOB=" + hs.ToBase64Gob()}
	if cmd.SysProcAttr == nil {
		cmd.SysProcAttr = &syscall.SysProcAttr{}
	}
	cmd.SysProcAttr.Cloneflags = uintptr(
		namespaces.GetNamespaceFlags(container.Namespaces),
	)
	cmd.SysProcAttr.Pdeathsig = syscall.SIGKILL
	cmd.ExtraFiles = []*os.File{controlPipe, ctx.configPipe}
	ctx.ex.cmd = cmd
	return cmd
}

func (ctx *containerInit) startCallback() {
	//TODO: log("Starting process web.1 with command `...`")

	//child process is already running, it's safe to close the parent's read
	//side of the pipe
	ctx.configPipe.Close()
}

func containerConfig(
	containerUUID string,
	uid, gid int,
	dataPath string,
	subnet *smallSubnet,
	env []string,
) *libcontainer.Config {
	return &libcontainer.Config{
		MountConfig: &libcontainer.MountConfig{
			Mounts: []*mount.Mount{
				{
					Type:        "bind",
					Destination: "/app",
					Writable:    true,
					Source:      filepath.Join(dataPath, "app"),
				},
				{
					Type:        "bind",
					Destination: "/tmp",
					Writable:    true,
					Source:      filepath.Join(dataPath, "tmp"),
				},
				{
					Type:        "bind",
					Destination: "/var/tmp",
					Writable:    true,
					Source: filepath.Join(
						dataPath,
						"var", "tmp",
					),
				},
				{
					Type:        "bind",
					Destination: "/etc/passwd",
					Writable:    false,
					Source: filepath.Join(
						dataPath, "passwd",
					),
				},
				{
					Type:        "bind",
					Writable:    false,
					Destination: "/etc/resolv.conf",
					Source:      "/etc/resolv.conf",
				},
			},
			MountLabel:  containerUUID,
			DeviceNodes: devices.DefaultSimpleDevices,
			PivotDir:    "/tmp",
		},
		RootFs:   filepath.Join(dataPath, "root"),
		Hostname: containerUUID,
		User:     fmt.Sprintf("%d:%d", uid, gid),
		Env:      env,
		Namespaces: []libcontainer.Namespace{
			{Type: "NEWIPC"},
			{Type: "NEWNET"},
			{Type: "NEWNS"},
			{Type: "NEWPID"},
			{Type: "NEWUTS"},
		},
		Capabilities: []string{
			"CHOWN",
			"DAC_OVERRIDE",
			"FOWNER",
			"MKNOD",
			"NET_RAW",
			"SETGID",
			"SETUID",
			"SETFCAP",
			"SETPCAP",
			"NET_BIND_SERVICE",
			"SYS_CHROOT",
			"KILL",
		},
		Networks: []*libcontainer.Network{
			{
				Address: "127.0.0.1/0",
				Gateway: "localhost",
				Mtu:     1500,
				Type:    "loopback",
			},
			// TODO: setup our own network instead of using the docker bridge
			{
				Address:    subnet.Host().String(),
				VethPrefix: fmt.Sprintf("veth%d", uid),
				Gateway:    subnet.Gateway().IP.String(),
				Mtu:        1500,
				Type:       "routed",
			},
		},
		Cgroups: &cgroups.Cgroup{
			Name:           containerUUID,
			AllowedDevices: devices.DefaultAllowedDevices,
		},
	}
}
<commit_msg>support local slugs in the libcontainer driver<commit_after>// +build linux

package hsup

import (
	"bytes"
	"encoding/gob"
	"fmt"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"syscall"

	"code.google.com/p/go-uuid/uuid"

	"github.com/docker/libcontainer"
	"github.com/docker/libcontainer/cgroups"
	"github.com/docker/libcontainer/devices"
	"github.com/docker/libcontainer/mount"
	"github.com/docker/libcontainer/namespaces"
)

type LibContainerDynoDriver struct {
	workDir       string
	stacksDir     string
	containersDir string
	allocator     *Allocator
}

func NewLibContainerDynoDriver(workDir string) (*LibContainerDynoDriver, error) {
	var (
		stacksDir     = filepath.Join(workDir, "stacks")
		containersDir = filepath.Join(workDir, "containers")
	)
	if err := os.MkdirAll(stacksDir, 0755); err != nil {
		return nil, err
	}
	if err := os.MkdirAll(containersDir, 0755); err != nil {
		return nil, err
	}
	allocator, err := NewAllocator(workDir)
	if err != nil {
		return nil, err
	}
	return &LibContainerDynoDriver{
		workDir:       workDir,
		stacksDir:     stacksDir,
		containersDir: containersDir,
		allocator:     allocator,
	}, nil
}

func (dd *LibContainerDynoDriver) Build(release *Release) error {
	stacks, err := HerokuStacksFromManifest(dd.stacksDir)
	if err != nil {
		return err
	}
	for _, stack := range stacks {
		if strings.TrimSpace(stack.Name) != release.stack {
			continue
		}
		if err := stack.mount(); err != nil {
			return err
		}
	}
	return nil
}

func (dd *LibContainerDynoDriver) Start(ex *Executor) error {
	containerUUID := uuid.New()
	uid, gid, err := dd.allocator.ReserveUID()
	if err != nil {
		return err
	}
	sn, err := dd.allocator.privateNetForUID(uid)
	if err != nil {
		return err
	}
	subnet, err := newSmallSubnet(sn)
	if err != nil {
		return err
	}
	stackImagePath, err := CurrentStackImagePath(
		dd.stacksDir, ex.Release.stack,
	)
	if err != nil {
		return err
	}
	dataPath := filepath.Join(dd.containersDir, containerUUID)
	if err := os.MkdirAll(dataPath, 0755); err != nil {
		return err
	}
	writablePaths := []string{
		filepath.Join(dataPath, "app"),
		filepath.Join(dataPath, "tmp"),
		filepath.Join(dataPath, "var", "tmp"),
	}
	for _, path := range writablePaths {
		if err := os.MkdirAll(path, 0755); err != nil {
			return err
		}
		if err := os.Chown(path, uid, gid); err != nil {
			return err
		}
	}
	rootFSPath := filepath.Join(dataPath, "root")
	if err := os.MkdirAll(rootFSPath, 0755); err != nil {
		return err
	}

	// stack image is the rootFS
	if err := syscall.Mount(
		stackImagePath, rootFSPath, "bind",
		syscall.MS_RDONLY|syscall.MS_BIND, "",
	); err != nil {
		return err
	}

	if err := createPasswdWithDynoUser(
		stackImagePath, dataPath, uid, gid,
	); err != nil {
		return err
	}

	if ex.Release.Where() == Local {
		// move into the container
		if err := copyFile(
			ex.Release.slugURL,
			filepath.Join(dataPath, "tmp", "slug.tgz"),
			0644,
		); err != nil {
			return err
		}
		ex.Release.slugURL = "/tmp/slug.tgz"
	}

	outsideContainer, err := filepath.Abs(linuxAmd64Path())
	if err != nil {
		return err
	}
	insideContainer := filepath.Join(dataPath, "tmp", "hsup")
	if err := copyFile(outsideContainer, insideContainer, 0755); err != nil {
		return err
	}

	// TODO tty
	console := ""

	ex.initExitStatus = make(chan *ExitStatus)

	cfgReader, cfgWriter, err := os.Pipe()
	initCtx := &containerInit{
		hsupBinaryPath: outsideContainer,
		ex:             ex,
		configPipe:     cfgReader,
	}

	container := containerConfig(
		containerUUID,
		uid, gid,
		dataPath,
		subnet,
		ex.Release.ConfigSlice(),
	)

	// send config to the init process inside the container
	go func() {
		defer cfgWriter.Close()
		encoder := gob.NewEncoder(cfgWriter)
		if err := encoder.Encode(container); err != nil {
			log.Fatal(err)
		}
	}()

	go func() {
		// TODO: stop swallowing errors
		code, err := namespaces.Exec(
			container, os.Stdin, os.Stdout, os.Stderr,
			console, dataPath, []string{},
			initCtx.createCommand, nil, initCtx.startCallback,
		)
		if err != nil {
			log.Printf("namespaces.Exec fails: %q", err)
		}

		// GC
		// TODO: gc after sending back the exit status
		// doing so right now terminates the program too early,
		// before everything is removed
		if err := syscall.Unmount(rootFSPath, 0); err != nil {
			log.Printf("unmount error: %#+v", err)
		}
		for _, path := range writablePaths {
			if err := os.RemoveAll(path); err != nil {
				log.Printf("remove all error: %#+v", err)
			}
		}
		if err := os.RemoveAll(dataPath); err != nil {
			log.Printf("remove all error: %#+v", err)
		}

		// it's probably safe to ignore errors here. Worst case
		// scenario, this uid won't be be reused.
		dd.allocator.FreeUID(uid)

		ex.initExitStatus <- &ExitStatus{Code: code, Err: err}
		close(ex.initExitStatus)
	}()

	return nil
}

func (dd *LibContainerDynoDriver) Wait(ex *Executor) (s *ExitStatus) {
	return <-ex.initExitStatus
}

func (dd *LibContainerDynoDriver) Stop(ex *Executor) error {
	if ex.cmd.ProcessState != nil {
		return nil // already exited
	}

	// TODO: fix a race conditition when Stop() is called before the
	// libcontainer driver re-execs itself

	// tell the abspath-driver to stop
	return ex.cmd.Process.Signal(syscall.SIGTERM)
}

func createPasswdWithDynoUser(stackImagePath, dataPath string, uid, gid int) error {
	var contents bytes.Buffer
	original, err := os.Open(filepath.Join(stackImagePath, "etc", "passwd"))
	if err != nil {
		return err
	}
	defer original.Close()

	if _, err := contents.ReadFrom(original); err != nil {
		return err
	}
	// TODO: allocate a free uid. It is currently hardcoded to 1000
	dynoUser := fmt.Sprintf("\ndyno:x:%d:%d::/app:/bin/bash\n", uid, gid)
	if _, err := contents.WriteString(dynoUser); err != nil {
		return err
	}

	dst, err := os.Create(filepath.Join(dataPath, "passwd"))
	if err != nil {
		return err
	}
	defer dst.Close()
	if err := dst.Chmod(0644); err != nil {
		return err
	}

	_, err = contents.WriteTo(dst)
	return err
}

type containerInit struct {
	hsupBinaryPath string
	ex             *Executor
	configPipe     *os.File
}

func (ctx *containerInit) createCommand(container *libcontainer.Config, console,
	dataPath, init string, controlPipe *os.File, args []string) *exec.Cmd {

	hs := Startup{
		App: AppSerializable{
			Version: ctx.ex.Release.version,
			Env:     ctx.ex.Release.config,
			Slug:    ctx.ex.Release.slugURL,
			Stack:   ctx.ex.Release.stack,
			Processes: []FormationSerializable{
				{
					FArgs:     ctx.ex.Args,
					FQuantity: 1,
					FType:     ctx.ex.ProcessType,
				},
			},
			LogplexURL: ctx.ex.logplexURLString(),
		},
		OneShot:     true,
		StartNumber: ctx.ex.ProcessID,
		Action:      Start,
		Driver:      &LibContainerInitDriver{},
		FormName:    ctx.ex.ProcessType,
	}
	cmd := exec.Command(ctx.hsupBinaryPath)
	cmd.Env = []string{"HSUP_CONTROL_GOB=" + hs.ToBase64Gob()}
	if cmd.SysProcAttr == nil {
		cmd.SysProcAttr = &syscall.SysProcAttr{}
	}
	cmd.SysProcAttr.Cloneflags = uintptr(
		namespaces.GetNamespaceFlags(container.Namespaces),
	)
	cmd.SysProcAttr.Pdeathsig = syscall.SIGKILL
	cmd.ExtraFiles = []*os.File{controlPipe, ctx.configPipe}
	ctx.ex.cmd = cmd
	return cmd
}

func (ctx *containerInit) startCallback() {
	//TODO: log("Starting process web.1 with command `...`")

	//child process is already running, it's safe to close the parent's read
	//side of the pipe
	ctx.configPipe.Close()
}

func containerConfig(
	containerUUID string,
	uid, gid int,
	dataPath string,
	subnet *smallSubnet,
	env []string,
) *libcontainer.Config {
	return &libcontainer.Config{
		MountConfig: &libcontainer.MountConfig{
			Mounts: []*mount.Mount{
				{
					Type:        "bind",
					Destination: "/app",
					Writable:    true,
					Source:      filepath.Join(dataPath, "app"),
				},
				{
					Type:        "bind",
					Destination: "/tmp",
					Writable:    true,
					Source:      filepath.Join(dataPath, "tmp"),
				},
				{
					Type:        "bind",
					Destination: "/var/tmp",
					Writable:    true,
					Source: filepath.Join(
						dataPath,
						"var", "tmp",
					),
				},
				{
					Type:        "bind",
					Destination: "/etc/passwd",
					Writable:    false,
					Source: filepath.Join(
						dataPath, "passwd",
					),
				},
				{
					Type:        "bind",
					Writable:    false,
					Destination: "/etc/resolv.conf",
					Source:      "/etc/resolv.conf",
				},
			},
			MountLabel:  containerUUID,
			DeviceNodes: devices.DefaultSimpleDevices,
			PivotDir:    "/tmp",
		},
		RootFs:   filepath.Join(dataPath, "root"),
		Hostname: containerUUID,
		User:     fmt.Sprintf("%d:%d", uid, gid),
		Env:      env,
		Namespaces: []libcontainer.Namespace{
			{Type: "NEWIPC"},
			{Type: "NEWNET"},
			{Type: "NEWNS"},
			{Type: "NEWPID"},
			{Type: "NEWUTS"},
		},
		Capabilities: []string{
			"CHOWN",
			"DAC_OVERRIDE",
			"FOWNER",
			"MKNOD",
			"NET_RAW",
			"SETGID",
			"SETUID",
			"SETFCAP",
			"SETPCAP",
			"NET_BIND_SERVICE",
			"SYS_CHROOT",
			"KILL",
		},
		Networks: []*libcontainer.Network{
			{
				Address: "127.0.0.1/0",
				Gateway: "localhost",
				Mtu:     1500,
				Type:    "loopback",
			},
			// TODO: setup our own network instead of using the docker bridge
			{
				Address:    subnet.Host().String(),
				VethPrefix: fmt.Sprintf("veth%d", uid),
				Gateway:    subnet.Gateway().IP.String(),
				Mtu:        1500,
				Type:       "routed",
			},
		},
		Cgroups: &cgroups.Cgroup{
			Name:           containerUUID,
			AllowedDevices: devices.DefaultAllowedDevices,
		},
	}
}
<|endoftext|>


<commit_before>package libnetwork

import (
	"encoding/json"
	"net"

	"github.com/docker/libnetwork/driverapi"
	"github.com/docker/libnetwork/types"
)

// EndpointInfo provides an interface to retrieve network resources bound to the endpoint.
type EndpointInfo interface {
	// InterfaceList returns an interface list which were assigned to the endpoint
	// by the driver. This can be used after the endpoint has been created.
	InterfaceList() []InterfaceInfo

	// Gateway returns the IPv4 gateway assigned by the driver.
	// This will only return a valid value if a container has joined the endpoint.
	Gateway() net.IP

	// GatewayIPv6 returns the IPv6 gateway assigned by the driver.
	// This will only return a valid value if a container has joined the endpoint.
	GatewayIPv6() net.IP

	// SandboxKey returns the sanbox key for the container which has joined
	// the endpoint. If there is no container joined then this will return an
	// empty string.
	SandboxKey() string
}

// InterfaceInfo provides an interface to retrieve interface addresses bound to the endpoint.
type InterfaceInfo interface {
	// MacAddress returns the MAC address assigned to the endpoint.
	MacAddress() net.HardwareAddr

	// Address returns the IPv4 address assigned to the endpoint.
	Address() net.IPNet

	// AddressIPv6 returns the IPv6 address assigned to the endpoint.
	AddressIPv6() net.IPNet
}

// ContainerInfo provides an interface to retrieve the info about the container attached to the endpoint
type ContainerInfo interface {
	// ID returns the ID of the container
	ID() string
	// Labels returns the container's labels
	Labels() map[string]interface{}
}

type endpointInterface struct {
	id        int
	mac       net.HardwareAddr
	addr      net.IPNet
	addrv6    net.IPNet
	srcName   string
	dstPrefix string
	routes    []*net.IPNet
}

func (epi *endpointInterface) MarshalJSON() ([]byte, error) {
	epMap := make(map[string]interface{})
	epMap["id"] = epi.id
	epMap["mac"] = epi.mac.String()
	epMap["addr"] = epi.addr.String()
	epMap["addrv6"] = epi.addrv6.String()
	epMap["srcName"] = epi.srcName
	epMap["dstPrefix"] = epi.dstPrefix
	var routes []string
	for _, route := range epi.routes {
		routes = append(routes, route.String())
	}
	epMap["routes"] = routes
	return json.Marshal(epMap)
}

func (epi *endpointInterface) UnmarshalJSON(b []byte) (err error) {
	var epMap map[string]interface{}
	if err := json.Unmarshal(b, &epMap); err != nil {
		return err
	}
	epi.id = int(epMap["id"].(float64))

	mac, _ := net.ParseMAC(epMap["mac"].(string))
	epi.mac = mac

	_, ipnet, _ := net.ParseCIDR(epMap["addr"].(string))
	if ipnet != nil {
		epi.addr = *ipnet
	}

	_, ipnet, _ = net.ParseCIDR(epMap["addrv6"].(string))
	if ipnet != nil {
		epi.addrv6 = *ipnet
	}

	epi.srcName = epMap["srcName"].(string)
	epi.dstPrefix = epMap["dstPrefix"].(string)

	rb, _ := json.Marshal(epMap["routes"])
	var routes []string
	json.Unmarshal(rb, &routes)
	epi.routes = make([]*net.IPNet, 0)
	for _, route := range routes {
		_, ipr, err := net.ParseCIDR(route)
		if err == nil {
			epi.routes = append(epi.routes, ipr)
		}
	}

	return nil
}

type endpointJoinInfo struct {
	gw             net.IP
	gw6            net.IP
	hostsPath      string
	resolvConfPath string
	StaticRoutes   []*types.StaticRoute
}

func (ep *endpoint) ContainerInfo() ContainerInfo {
	ep.Lock()
	ci := ep.container
	defer ep.Unlock()

	// Need this since we return the interface
	if ci == nil {
		return nil
	}
	return ci
}

func (ep *endpoint) Info() EndpointInfo {
	return ep
}

func (ep *endpoint) DriverInfo() (map[string]interface{}, error) {
	ep.Lock()
	network := ep.network
	epid := ep.id
	ep.Unlock()

	network.Lock()
	driver := network.driver
	nid := network.id
	network.Unlock()

	return driver.EndpointOperInfo(nid, epid)
}

func (ep *endpoint) InterfaceList() []InterfaceInfo {
	ep.Lock()
	defer ep.Unlock()

	iList := make([]InterfaceInfo, len(ep.iFaces))

	for i, iface := range ep.iFaces {
		iList[i] = iface
	}

	return iList
}

func (ep *endpoint) Interfaces() []driverapi.InterfaceInfo {
	ep.Lock()
	defer ep.Unlock()

	iList := make([]driverapi.InterfaceInfo, len(ep.iFaces))

	for i, iface := range ep.iFaces {
		iList[i] = iface
	}

	return iList
}

func (ep *endpoint) AddInterface(id int, mac net.HardwareAddr, ipv4 net.IPNet, ipv6 net.IPNet) error {
	ep.Lock()
	defer ep.Unlock()

	iface := &endpointInterface{
		id:     id,
		addr:   *types.GetIPNetCopy(&ipv4),
		addrv6: *types.GetIPNetCopy(&ipv6),
	}
	iface.mac = types.GetMacCopy(mac)

	ep.iFaces = append(ep.iFaces, iface)
	return nil
}

func (epi *endpointInterface) ID() int {
	return epi.id
}

func (epi *endpointInterface) MacAddress() net.HardwareAddr {
	return types.GetMacCopy(epi.mac)
}

func (epi *endpointInterface) Address() net.IPNet {
	return (*types.GetIPNetCopy(&epi.addr))
}

func (epi *endpointInterface) AddressIPv6() net.IPNet {
	return (*types.GetIPNetCopy(&epi.addrv6))
}

func (epi *endpointInterface) SetNames(srcName string, dstPrefix string) error {
	epi.srcName = srcName
	epi.dstPrefix = dstPrefix
	return nil
}

func (ep *endpoint) InterfaceNames() []driverapi.InterfaceNameInfo {
	ep.Lock()
	defer ep.Unlock()

	iList := make([]driverapi.InterfaceNameInfo, len(ep.iFaces))

	for i, iface := range ep.iFaces {
		iList[i] = iface
	}

	return iList
}

func (ep *endpoint) AddStaticRoute(destination *net.IPNet, routeType int, nextHop net.IP, interfaceID int) error {
	ep.Lock()
	defer ep.Unlock()

	r := types.StaticRoute{Destination: destination, RouteType: routeType, NextHop: nextHop, InterfaceID: interfaceID}

	if routeType == types.NEXTHOP {
		// If the route specifies a next-hop, then it's loosely routed (i.e. not bound to a particular interface).
		ep.joinInfo.StaticRoutes = append(ep.joinInfo.StaticRoutes, &r)
	} else {
		// If the route doesn't specify a next-hop, it must be a connected route, bound to an interface.
		if err := ep.addInterfaceRoute(&r); err != nil {
			return err
		}
	}
	return nil
}

func (ep *endpoint) addInterfaceRoute(route *types.StaticRoute) error {
	for _, iface := range ep.iFaces {
		if iface.id == route.InterfaceID {
			iface.routes = append(iface.routes, route.Destination)
			return nil
		}
	}
	return types.BadRequestErrorf("Interface with ID %d doesn't exist.",
		route.InterfaceID)
}

func (ep *endpoint) SandboxKey() string {
	ep.Lock()
	defer ep.Unlock()

	if ep.container == nil {
		return ""
	}

	return ep.container.data.SandboxKey
}

func (ep *endpoint) Gateway() net.IP {
	ep.Lock()
	defer ep.Unlock()

	if ep.joinInfo == nil {
		return net.IP{}
	}

	return types.GetIPCopy(ep.joinInfo.gw)
}

func (ep *endpoint) GatewayIPv6() net.IP {
	ep.Lock()
	defer ep.Unlock()

	if ep.joinInfo == nil {
		return net.IP{}
	}

	return types.GetIPCopy(ep.joinInfo.gw6)
}

func (ep *endpoint) SetGateway(gw net.IP) error {
	ep.Lock()
	defer ep.Unlock()

	ep.joinInfo.gw = types.GetIPCopy(gw)
	return nil
}

func (ep *endpoint) SetGatewayIPv6(gw6 net.IP) error {
	ep.Lock()
	defer ep.Unlock()

	ep.joinInfo.gw6 = types.GetIPCopy(gw6)
	return nil
}

func (ep *endpoint) SetHostsPath(path string) error {
	ep.Lock()
	defer ep.Unlock()

	ep.joinInfo.hostsPath = path
	return nil
}

func (ep *endpoint) SetResolvConfPath(path string) error {
	ep.Lock()
	defer ep.Unlock()

	ep.joinInfo.resolvConfPath = path
	return nil
}
<commit_msg>Fix endpoint ip data-store sync issue<commit_after>package libnetwork

import (
	"encoding/json"
	"net"

	"github.com/docker/libnetwork/driverapi"
	"github.com/docker/libnetwork/types"
)

// EndpointInfo provides an interface to retrieve network resources bound to the endpoint.
type EndpointInfo interface {
	// InterfaceList returns an interface list which were assigned to the endpoint
	// by the driver. This can be used after the endpoint has been created.
	InterfaceList() []InterfaceInfo

	// Gateway returns the IPv4 gateway assigned by the driver.
	// This will only return a valid value if a container has joined the endpoint.
	Gateway() net.IP

	// GatewayIPv6 returns the IPv6 gateway assigned by the driver.
	// This will only return a valid value if a container has joined the endpoint.
	GatewayIPv6() net.IP

	// SandboxKey returns the sanbox key for the container which has joined
	// the endpoint. If there is no container joined then this will return an
	// empty string.
	SandboxKey() string
}

// InterfaceInfo provides an interface to retrieve interface addresses bound to the endpoint.
type InterfaceInfo interface {
	// MacAddress returns the MAC address assigned to the endpoint.
	MacAddress() net.HardwareAddr

	// Address returns the IPv4 address assigned to the endpoint.
	Address() net.IPNet

	// AddressIPv6 returns the IPv6 address assigned to the endpoint.
	AddressIPv6() net.IPNet
}

// ContainerInfo provides an interface to retrieve the info about the container attached to the endpoint
type ContainerInfo interface {
	// ID returns the ID of the container
	ID() string
	// Labels returns the container's labels
	Labels() map[string]interface{}
}

type endpointInterface struct {
	id        int
	mac       net.HardwareAddr
	addr      net.IPNet
	addrv6    net.IPNet
	srcName   string
	dstPrefix string
	routes    []*net.IPNet
}

func (epi *endpointInterface) MarshalJSON() ([]byte, error) {
	epMap := make(map[string]interface{})
	epMap["id"] = epi.id
	epMap["mac"] = epi.mac.String()
	epMap["addr"] = epi.addr.String()
	epMap["addrv6"] = epi.addrv6.String()
	epMap["srcName"] = epi.srcName
	epMap["dstPrefix"] = epi.dstPrefix
	var routes []string
	for _, route := range epi.routes {
		routes = append(routes, route.String())
	}
	epMap["routes"] = routes
	return json.Marshal(epMap)
}

func (epi *endpointInterface) UnmarshalJSON(b []byte) (err error) {
	var epMap map[string]interface{}
	if err := json.Unmarshal(b, &epMap); err != nil {
		return err
	}
	epi.id = int(epMap["id"].(float64))

	mac, _ := net.ParseMAC(epMap["mac"].(string))
	epi.mac = mac

	ip, ipnet, _ := net.ParseCIDR(epMap["addr"].(string))
	if ipnet != nil {
		ipnet.IP = ip
		epi.addr = *ipnet
	}

	ip, ipnet, _ = net.ParseCIDR(epMap["addrv6"].(string))
	if ipnet != nil {
		ipnet.IP = ip
		epi.addrv6 = *ipnet
	}

	epi.srcName = epMap["srcName"].(string)
	epi.dstPrefix = epMap["dstPrefix"].(string)

	rb, _ := json.Marshal(epMap["routes"])
	var routes []string
	json.Unmarshal(rb, &routes)
	epi.routes = make([]*net.IPNet, 0)
	for _, route := range routes {
		ip, ipr, err := net.ParseCIDR(route)
		if err == nil {
			ipr.IP = ip
			epi.routes = append(epi.routes, ipr)
		}
	}

	return nil
}

type endpointJoinInfo struct {
	gw             net.IP
	gw6            net.IP
	hostsPath      string
	resolvConfPath string
	StaticRoutes   []*types.StaticRoute
}

func (ep *endpoint) ContainerInfo() ContainerInfo {
	ep.Lock()
	ci := ep.container
	defer ep.Unlock()

	// Need this since we return the interface
	if ci == nil {
		return nil
	}
	return ci
}

func (ep *endpoint) Info() EndpointInfo {
	return ep
}

func (ep *endpoint) DriverInfo() (map[string]interface{}, error) {
	ep.Lock()
	network := ep.network
	epid := ep.id
	ep.Unlock()

	network.Lock()
	driver := network.driver
	nid := network.id
	network.Unlock()

	return driver.EndpointOperInfo(nid, epid)
}

func (ep *endpoint) InterfaceList() []InterfaceInfo {
	ep.Lock()
	defer ep.Unlock()

	iList := make([]InterfaceInfo, len(ep.iFaces))

	for i, iface := range ep.iFaces {
		iList[i] = iface
	}

	return iList
}

func (ep *endpoint) Interfaces() []driverapi.InterfaceInfo {
	ep.Lock()
	defer ep.Unlock()

	iList := make([]driverapi.InterfaceInfo, len(ep.iFaces))

	for i, iface := range ep.iFaces {
		iList[i] = iface
	}

	return iList
}

func (ep *endpoint) AddInterface(id int, mac net.HardwareAddr, ipv4 net.IPNet, ipv6 net.IPNet) error {
	ep.Lock()
	defer ep.Unlock()

	iface := &endpointInterface{
		id:     id,
		addr:   *types.GetIPNetCopy(&ipv4),
		addrv6: *types.GetIPNetCopy(&ipv6),
	}
	iface.mac = types.GetMacCopy(mac)

	ep.iFaces = append(ep.iFaces, iface)
	return nil
}

func (epi *endpointInterface) ID() int {
	return epi.id
}

func (epi *endpointInterface) MacAddress() net.HardwareAddr {
	return types.GetMacCopy(epi.mac)
}

func (epi *endpointInterface) Address() net.IPNet {
	return (*types.GetIPNetCopy(&epi.addr))
}

func (epi *endpointInterface) AddressIPv6() net.IPNet {
	return (*types.GetIPNetCopy(&epi.addrv6))
}

func (epi *endpointInterface) SetNames(srcName string, dstPrefix string) error {
	epi.srcName = srcName
	epi.dstPrefix = dstPrefix
	return nil
}

func (ep *endpoint) InterfaceNames() []driverapi.InterfaceNameInfo {
	ep.Lock()
	defer ep.Unlock()

	iList := make([]driverapi.InterfaceNameInfo, len(ep.iFaces))

	for i, iface := range ep.iFaces {
		iList[i] = iface
	}

	return iList
}

func (ep *endpoint) AddStaticRoute(destination *net.IPNet, routeType int, nextHop net.IP, interfaceID int) error {
	ep.Lock()
	defer ep.Unlock()

	r := types.StaticRoute{Destination: destination, RouteType: routeType, NextHop: nextHop, InterfaceID: interfaceID}

	if routeType == types.NEXTHOP {
		// If the route specifies a next-hop, then it's loosely routed (i.e. not bound to a particular interface).
		ep.joinInfo.StaticRoutes = append(ep.joinInfo.StaticRoutes, &r)
	} else {
		// If the route doesn't specify a next-hop, it must be a connected route, bound to an interface.
		if err := ep.addInterfaceRoute(&r); err != nil {
			return err
		}
	}
	return nil
}

func (ep *endpoint) addInterfaceRoute(route *types.StaticRoute) error {
	for _, iface := range ep.iFaces {
		if iface.id == route.InterfaceID {
			iface.routes = append(iface.routes, route.Destination)
			return nil
		}
	}
	return types.BadRequestErrorf("Interface with ID %d doesn't exist.",
		route.InterfaceID)
}

func (ep *endpoint) SandboxKey() string {
	ep.Lock()
	defer ep.Unlock()

	if ep.container == nil {
		return ""
	}

	return ep.container.data.SandboxKey
}

func (ep *endpoint) Gateway() net.IP {
	ep.Lock()
	defer ep.Unlock()

	if ep.joinInfo == nil {
		return net.IP{}
	}

	return types.GetIPCopy(ep.joinInfo.gw)
}

func (ep *endpoint) GatewayIPv6() net.IP {
	ep.Lock()
	defer ep.Unlock()

	if ep.joinInfo == nil {
		return net.IP{}
	}

	return types.GetIPCopy(ep.joinInfo.gw6)
}

func (ep *endpoint) SetGateway(gw net.IP) error {
	ep.Lock()
	defer ep.Unlock()

	ep.joinInfo.gw = types.GetIPCopy(gw)
	return nil
}

func (ep *endpoint) SetGatewayIPv6(gw6 net.IP) error {
	ep.Lock()
	defer ep.Unlock()

	ep.joinInfo.gw6 = types.GetIPCopy(gw6)
	return nil
}

func (ep *endpoint) SetHostsPath(path string) error {
	ep.Lock()
	defer ep.Unlock()

	ep.joinInfo.hostsPath = path
	return nil
}

func (ep *endpoint) SetResolvConfPath(path string) error {
	ep.Lock()
	defer ep.Unlock()

	ep.joinInfo.resolvConfPath = path
	return nil
}
<|endoftext|>