<commit_before>package main

import (
	"fmt"
	"github.com/albrow/forms"
	"github.com/codegangsta/negroni"
	"github.com/gorilla/mux"
	"github.com/martini-contrib/cors"
	"github.com/unrolled/render"
	"net/http"
	"strconv"
)

// NOTE: This is a test server specifically designed for testing the humble framework.
// As such, it is designed to be completely idempotent. That means nothing you do will
// actually change the data on the server, and sending the same request will always
// give you the same response. However, when possible the responses are designed to mimic
// that of a real server that does hold state.

type todo struct {
	Id          int
	Title       string
	IsCompleted bool
}

// Since the server is idempotent, the list of todos will never change, regardless of
// requests to create, update, or delete todos.
var todos = []todo{
	{
		Id:          0,
		Title:       "Todo 0",
		IsCompleted: false,
	},
	{
		Id:          1,
		Title:       "Todo 1",
		IsCompleted: false,
	},
	{
		Id:          2,
		Title:       "Todo 2",
		IsCompleted: true,
	},
}

var (
	// r is used to render responses
	r = render.New(render.Options{
		IndentJSON: true,
	})
)

const (
	statusUnprocessableEntity = 422
)

func main() {
	// Routes
	router := mux.NewRouter()
	router.HandleFunc("/todos", todosController.Index).Methods("GET")
	router.HandleFunc("/todos", todosController.Create).Methods("POST")
	router.HandleFunc("/todos/{id}", todosController.Show).Methods("GET")
	router.HandleFunc("/todos/{id}", todosController.Update).Methods("PUT")
	router.HandleFunc("/todos/{id}", todosController.Delete).Methods("DELETE")

	// Other middleware
	n := negroni.New(negroni.NewLogger())
	n.UseHandler(cors.Allow(&cors.Options{
		AllowOrigins:     []string{"*"},
		AllowMethods:     []string{"GET", "POST", "DELETE", "PUT", "PATCH"},
		AllowHeaders:     []string{"Origin", "Content-Type", "X-Requested-With"},
		ExposeHeaders:    []string{"Content-Length"},
		AllowCredentials: true,
	}))

	// Router must always come last
	n.UseHandler(router)

	// Start the server
	n.Run(":3000")
}

// Todos Controller and its methods
type todosControllerType struct{}

var todosController = todosControllerType{}

// Index returns a list of todos as an array of json objects. It always returns the
// same list of todos and is idempotent.
func (todosControllerType) Index(w http.ResponseWriter, req *http.Request) {
	r.JSON(w, http.StatusOK, todos)
}

// Create accepts form data for creating a new todo. Since this server is designed
// for testing, it does not actually create the todo, as that would make the server
// non-idempotent. Create returns the todo that would be created as a json object.
// It assigns the id of 3 to the todo.
func (todosControllerType) Create(w http.ResponseWriter, req *http.Request) {
	// Parse data and do validations
	todoData, err := forms.Parse(req)
	if err != nil {
		panic(err)
	}
	val := todoData.Validator()
	val.Require("Title")
	val.Require("IsCompleted")
	val.TypeBool("IsCompleted")
	if val.HasErrors() {
		r.JSON(w, statusUnprocessableEntity, val.ErrorMap())
		return
	}

	// Return the todo that would be created
	todo := todo{
		Id:          3,
		Title:       todoData.Get("Title"),
		IsCompleted: todoData.GetBool("IsCompleted"),
	}
	r.JSON(w, http.StatusOK, todo)
}

// Show returns the json data for an existing todo. Since the todos never change
// and there are three of them, Show will only respond with a todo object for id
// parameters between 0 and 2. Any other id will result in a 422 error.
func (todosControllerType) Show(w http.ResponseWriter, req *http.Request) {
	// Get the id from the url parameters
	id, err := parseId(req)
	if err != nil {
		r.JSON(w, statusUnprocessableEntity, map[string]error{
			"error": err,
		})
		return
	}
	r.JSON(w, http.StatusOK, todos[id])
}

func (todosControllerType) Update(w http.ResponseWriter, req *http.Request) {
	// Get the id from the url parameters
	id, err := parseId(req)
	if err != nil {
		r.JSON(w, statusUnprocessableEntity, map[string]error{
			"error": err,
		})
		return
	}
	// Create a copy of the todo corresponding to id
	todoCopy := todos[id]
	// Parse data from the request
	todoData, err := forms.Parse(req)
	if err != nil {
		panic(err)
	}
	// Validate and update the data only if it was provided in the request
	if todoData.KeyExists("IsCompleted") {
		val := todoData.Validator()
		val.TypeBool("IsCompleted")
		if val.HasErrors() {
			r.JSON(w, statusUnprocessableEntity, val.ErrorMap())
			return
		}
		// Update todoCopy with the given data
		todoCopy.IsCompleted = todoData.GetBool("IsCompleted")
	}
	if todoData.KeyExists("Title") {
		todoCopy.Title = todoData.Get("Title")
	}
	r.JSON(w, http.StatusOK, todoCopy)
}

func (todosControllerType) Delete(w http.ResponseWriter, req *http.Request) {
	// TODO: rewrite this method in an idempotent way
}

// parseId gets the id out of the url parameters of req, converts it to an int,
// and then checks that it is in the range of existing todos. It will return an
// an error if there was problem converting the id parameter to an int or the
// id was outside the range of existing todos.
func parseId(req *http.Request) (int, error) {
	idStr := mux.Vars(req)["id"]
	id, err := strconv.Atoi(idStr)
	if err != nil {
		return 0, fmt.Errorf(`Could not convert id paramater "%s" to int`, idStr)
	}
	if id < 0 || id > 2 {
		return 0, fmt.Errorf(`Could not find todo with id = %d`, id)
	}
	return id, nil
}
<commit_msg>Add test for model.Delete<commit_after>package main

import (
	"fmt"
	"github.com/albrow/forms"
	"github.com/codegangsta/negroni"
	"github.com/gorilla/mux"
	"github.com/martini-contrib/cors"
	"github.com/unrolled/render"
	"net/http"
	"strconv"
)

// NOTE: This is a test server specifically designed for testing the humble framework.
// As such, it is designed to be completely idempotent. That means nothing you do will
// actually change the data on the server, and sending the same request will always
// give you the same response. However, when possible the responses are designed to mimic
// that of a real server that does hold state.

type todo struct {
	Id          int
	Title       string
	IsCompleted bool
}

// Since the server is idempotent, the list of todos will never change, regardless of
// requests to create, update, or delete todos.
var todos = []todo{
	{
		Id:          0,
		Title:       "Todo 0",
		IsCompleted: false,
	},
	{
		Id:          1,
		Title:       "Todo 1",
		IsCompleted: false,
	},
	{
		Id:          2,
		Title:       "Todo 2",
		IsCompleted: true,
	},
}

var (
	// r is used to render responses
	r = render.New(render.Options{
		IndentJSON: true,
	})
)

const (
	statusUnprocessableEntity = 422
)

func main() {
	// Routes
	router := mux.NewRouter()
	router.HandleFunc("/todos", todosController.Index).Methods("GET")
	router.HandleFunc("/todos", todosController.Create).Methods("POST")
	router.HandleFunc("/todos/{id}", todosController.Show).Methods("GET")
	router.HandleFunc("/todos/{id}", todosController.Update).Methods("PUT")
	router.HandleFunc("/todos/{id}", todosController.Delete).Methods("DELETE")

	// Other middleware
	n := negroni.New(negroni.NewLogger())
	n.UseHandler(cors.Allow(&cors.Options{
		AllowOrigins:     []string{"*"},
		AllowMethods:     []string{"GET", "POST", "DELETE", "PUT", "PATCH"},
		AllowHeaders:     []string{"Origin", "Content-Type", "X-Requested-With"},
		ExposeHeaders:    []string{"Content-Length"},
		AllowCredentials: true,
	}))

	// Router must always come last
	n.UseHandler(router)

	// Start the server
	n.Run(":3000")
}

// Todos Controller and its methods
type todosControllerType struct{}

var todosController = todosControllerType{}

// Index returns a list of todos as an array of json objects. It always returns the
// same list of todos and is idempotent.
func (todosControllerType) Index(w http.ResponseWriter, req *http.Request) {
	r.JSON(w, http.StatusOK, todos)
}

// Create accepts form data for creating a new todo. Since this server is designed
// for testing, it does not actually create the todo, as that would make the server
// non-idempotent. Create returns the todo that would be created as a json object.
// It assigns the id of 3 to the todo.
func (todosControllerType) Create(w http.ResponseWriter, req *http.Request) {
	// Parse data and do validations
	todoData, err := forms.Parse(req)
	if err != nil {
		panic(err)
	}
	val := todoData.Validator()
	val.Require("Title")
	val.Require("IsCompleted")
	val.TypeBool("IsCompleted")
	if val.HasErrors() {
		r.JSON(w, statusUnprocessableEntity, val.ErrorMap())
		return
	}

	// Return the todo that would be created
	todo := todo{
		Id:          3,
		Title:       todoData.Get("Title"),
		IsCompleted: todoData.GetBool("IsCompleted"),
	}
	r.JSON(w, http.StatusOK, todo)
}

// Show returns the json data for an existing todo. Since the todos never change
// and there are three of them, Show will only respond with a todo object for id
// parameters between 0 and 2. Any other id will result in a 422 error.
func (todosControllerType) Show(w http.ResponseWriter, req *http.Request) {
	// Get the id from the url parameters
	id, err := parseId(req)
	if err != nil {
		r.JSON(w, statusUnprocessableEntity, map[string]error{
			"error": err,
		})
		return
	}
	r.JSON(w, http.StatusOK, todos[id])
}

func (todosControllerType) Update(w http.ResponseWriter, req *http.Request) {
	// Get the id from the url parameters
	id, err := parseId(req)
	if err != nil {
		r.JSON(w, statusUnprocessableEntity, map[string]error{
			"error": err,
		})
		return
	}
	// Create a copy of the todo corresponding to id
	todoCopy := todos[id]
	// Parse data from the request
	todoData, err := forms.Parse(req)
	if err != nil {
		panic(err)
	}
	// Validate and update the data only if it was provided in the request
	if todoData.KeyExists("IsCompleted") {
		val := todoData.Validator()
		val.TypeBool("IsCompleted")
		if val.HasErrors() {
			r.JSON(w, statusUnprocessableEntity, val.ErrorMap())
			return
		}
		// Update todoCopy with the given data
		todoCopy.IsCompleted = todoData.GetBool("IsCompleted")
	}
	if todoData.KeyExists("Title") {
		todoCopy.Title = todoData.Get("Title")
	}
	r.JSON(w, http.StatusOK, todoCopy)
}

func (todosControllerType) Delete(w http.ResponseWriter, req *http.Request) {
	// Get the id from the url parameters
	if _, err := parseId(req); err != nil {
		r.JSON(w, statusUnprocessableEntity, map[string]error{
			"error": err,
		})
		return
	}
	r.JSON(w, http.StatusOK, struct{}{})
}

// parseId gets the id out of the url parameters of req, converts it to an int,
// and then checks that it is in the range of existing todos. It will return an
// an error if there was problem converting the id parameter to an int or the
// id was outside the range of existing todos.
func parseId(req *http.Request) (int, error) {
	idStr := mux.Vars(req)["id"]
	id, err := strconv.Atoi(idStr)
	if err != nil {
		return 0, fmt.Errorf(`Could not convert id paramater "%s" to int`, idStr)
	}
	if id < 0 || id > 2 {
		return 0, fmt.Errorf(`Could not find todo with id = %d`, id)
	}
	return id, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// spoof contains logic to make polling HTTP requests against an endpoint with optional host spoofing.

package spoof

import (
	"context"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"strings"
	"time"

	"github.com/pkg/errors"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/kubernetes"
	"knative.dev/pkg/test/ingress"
	"knative.dev/pkg/test/logging"
	"knative.dev/pkg/test/zipkin"

	"go.opencensus.io/plugin/ochttp"
	"go.opencensus.io/plugin/ochttp/propagation/b3"
	"go.opencensus.io/trace"
)

const (
	requestInterval = 1 * time.Second
	// RequestTimeout is the default timeout for the polling requests.
	RequestTimeout = 5 * time.Minute
	// Name of the temporary HTTP header that is added to http.Request to indicate that
	// it is a SpoofClient.Poll request. This header is removed before making call to backend.
	pollReqHeader = "X-Kn-Poll-Request-Do-Not-Trace"
)

// Response is a stripped down subset of http.Response. The is primarily useful
// for ResponseCheckers to inspect the response body without consuming it.
// Notably, Body is a byte slice instead of an io.ReadCloser.
type Response struct {
	Status     string
	StatusCode int
	Header     http.Header
	Body       []byte
}

func (r *Response) String() string {
	return fmt.Sprintf("status: %d, body: %s, headers: %v", r.StatusCode, string(r.Body), r.Header)
}

// Interface defines the actions that can be performed by the spoofing client.
type Interface interface {
	Do(*http.Request) (*Response, error)
	Poll(*http.Request, ResponseChecker) (*Response, error)
}

// https://medium.com/stupid-gopher-tricks/ensuring-go-interface-satisfaction-at-compile-time-1ed158e8fa17
var (
	_           Interface = (*SpoofingClient)(nil)
	dialContext           = (&net.Dialer{}).DialContext
)

// ResponseChecker is used to determine when SpoofinClient.Poll is done polling.
// This allows you to predicate wait.PollImmediate on the request's http.Response.
//
// See the apimachinery wait package:
// https://github.com/kubernetes/apimachinery/blob/cf7ae2f57dabc02a3d215f15ca61ae1446f3be8f/pkg/util/wait/wait.go#L172
type ResponseChecker func(resp *Response) (done bool, err error)

// SpoofingClient is a minimal HTTP client wrapper that spoofs the domain of requests
// for non-resolvable domains.
type SpoofingClient struct {
	Client          *http.Client
	RequestInterval time.Duration
	RequestTimeout  time.Duration

	logf logging.FormatLogger
}

// TransportOption allows callers to customize the http.Transport used by a SpoofingClient
type TransportOption func(transport *http.Transport) *http.Transport

// New returns a SpoofingClient that rewrites requests if the target domain is not `resolvable`.
// It does this by looking up the ingress at construction time, so reusing a client will not
// follow the ingress if it moves (or if there are multiple ingresses).
//
// If that's a problem, see test/request.go#WaitForEndpointState for oneshot spoofing.
func New(
	kubeClientset *kubernetes.Clientset,
	logf logging.FormatLogger,
	domain string,
	resolvable bool,
	endpointOverride string,
	opts ...TransportOption) (*SpoofingClient, error) {
	endpoint, err := ResolveEndpoint(kubeClientset, domain, resolvable, endpointOverride)
	if err != nil {
		fmt.Errorf("failed get the cluster endpoint: %v", err)
	}

	// Spoof the hostname at the resolver level
	transport := &http.Transport{
		DialContext: func(ctx context.Context, network, addr string) (conn net.Conn, e error) {
			spoofed := addr
			if i := strings.LastIndex(addr, ":"); i != -1 && domain == addr[:i] {
				// The original hostname:port is spoofed by replacing the hostname by the value
				// returned by ResolveEndpoint.
				spoofed = endpoint + ":" + addr[i+1:]
				logf("Spoofing %s -> %s", addr, spoofed)
			}
			return dialContext(ctx, network, spoofed)
		},
	}

	for _, opt := range opts {
		transport = opt(transport)
	}

	// Enable Zipkin tracing
	roundTripper := &ochttp.Transport{
		Base:        transport,
		Propagation: &b3.HTTPFormat{},
	}

	sc := SpoofingClient{
		Client:          &http.Client{Transport: roundTripper},
		RequestInterval: requestInterval,
		RequestTimeout:  RequestTimeout,
		logf:            logf,
	}
	return &sc, nil
}

// ResolveEndpoint resolves the endpoint address considering whether the domain is resolvable and taking into
// account whether the user overrode the endpoint address externally
func ResolveEndpoint(kubeClientset *kubernetes.Clientset, domain string, resolvable bool, endpointOverride string) (string, error) {
	// If the domain is resolvable, it can be used directly
	if resolvable {
		return domain, nil
	}
	// If an override is provided, use it
	if endpointOverride != "" {
		return endpointOverride, nil
	}
	// Otherwise, use the actual cluster endpoint
	return ingress.GetIngressEndpoint(kubeClientset)
}

// Do dispatches to the underlying http.Client.Do, spoofing domains as needed
// and transforming the http.Response into a spoof.Response.
// Each response is augmented with "ZipkinTraceID" header that identifies the zipkin trace corresponding to the request.
func (sc *SpoofingClient) Do(req *http.Request) (*Response, error) {
	// Starting span to capture zipkin trace.
	traceContext, span := trace.StartSpan(req.Context(), "SpoofingClient-Trace")
	defer span.End()

	// Check to see if the call to this method is coming from a Poll call.
	logZipkinTrace := true
	if req.Header.Get(pollReqHeader) != "" {
		req.Header.Del(pollReqHeader)
		logZipkinTrace = false
	}
	resp, err := sc.Client.Do(req.WithContext(traceContext))
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()

	resp.Header.Add(zipkin.ZipkinTraceIDHeader, span.SpanContext().TraceID.String())
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	spoofResp := &Response{
		Status:     resp.Status,
		StatusCode: resp.StatusCode,
		Header:     resp.Header,
		Body:       body,
	}

	if logZipkinTrace {
		sc.logZipkinTrace(spoofResp)
	}

	return spoofResp, nil
}

// Poll executes an http request until it satisfies the inState condition or encounters an error.
func (sc *SpoofingClient) Poll(req *http.Request, inState ResponseChecker) (*Response, error) {
	var (
		resp *Response
		err  error
	)

	err = wait.PollImmediate(sc.RequestInterval, sc.RequestTimeout, func() (bool, error) {
		// As we may do multiple Do calls as part of a single Poll we add this temporary header
		// to the request to indicate to Do method not to log Zipkin trace, instead it is
		// handled by this method itself.
		req.Header.Add(pollReqHeader, "True")
		resp, err = sc.Do(req)
		if err != nil {
			if isTCPTimeout(err) {
				sc.logf("Retrying %s for TCP timeout %v", req.URL.String(), err)
				return false, nil
			}
			// Retrying on DNS error, since we may be using xip.io or nip.io in tests.
			if isDNSError(err) {
				sc.logf("Retrying %s for DNS error %v", req.URL.String(), err)
				return false, nil
			}
			// Repeat the poll on `connection refused` errors, which are usually transient Istio errors.
			if isTCPConnectRefuse(err) {
				sc.logf("Retrying %s for connection refused %v", req.URL.String(), err)
				return false, nil
			}
			return true, err
		}

		return inState(resp)
	})

	if resp != nil {
		sc.logZipkinTrace(resp)
	}

	if err != nil {
		return resp, errors.Wrapf(err, "response: %s did not pass checks", resp)
	}
	return resp, nil
}

// logZipkinTrace provides support to log Zipkin Trace for param: spoofResponse
// We only log Zipkin trace for HTTP server errors i.e for HTTP status codes between 500 to 600
func (sc *SpoofingClient) logZipkinTrace(spoofResp *Response) {
	if !zipkin.ZipkinTracingEnabled || spoofResp.StatusCode < http.StatusInternalServerError || spoofResp.StatusCode >= 600 {
		return
	}

	traceID := spoofResp.Header.Get(zipkin.ZipkinTraceIDHeader)
	sc.logf("Logging Zipkin Trace for: %s", traceID)

	json, err := zipkin.JSONTrace(traceID /* We don't know the expected number of spans */, -1, 5*time.Second)
	if err != nil {
		if _, ok := err.(*zipkin.TimeoutError); !ok {
			sc.logf("Error getting zipkin trace: %v", err)
		}
	}

	sc.logf("%s", json)
}
<commit_msg>Log spoofing details only once. (#710)<commit_after>/*
Copyright 2019 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// spoof contains logic to make polling HTTP requests against an endpoint with optional host spoofing.

package spoof

import (
	"context"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"strings"
	"time"

	"github.com/pkg/errors"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/kubernetes"
	"knative.dev/pkg/test/ingress"
	"knative.dev/pkg/test/logging"
	"knative.dev/pkg/test/zipkin"

	"go.opencensus.io/plugin/ochttp"
	"go.opencensus.io/plugin/ochttp/propagation/b3"
	"go.opencensus.io/trace"
)

const (
	requestInterval = 1 * time.Second
	// RequestTimeout is the default timeout for the polling requests.
	RequestTimeout = 5 * time.Minute
	// Name of the temporary HTTP header that is added to http.Request to indicate that
	// it is a SpoofClient.Poll request. This header is removed before making call to backend.
	pollReqHeader = "X-Kn-Poll-Request-Do-Not-Trace"
)

// Response is a stripped down subset of http.Response. The is primarily useful
// for ResponseCheckers to inspect the response body without consuming it.
// Notably, Body is a byte slice instead of an io.ReadCloser.
type Response struct {
	Status     string
	StatusCode int
	Header     http.Header
	Body       []byte
}

func (r *Response) String() string {
	return fmt.Sprintf("status: %d, body: %s, headers: %v", r.StatusCode, string(r.Body), r.Header)
}

// Interface defines the actions that can be performed by the spoofing client.
type Interface interface {
	Do(*http.Request) (*Response, error)
	Poll(*http.Request, ResponseChecker) (*Response, error)
}

// https://medium.com/stupid-gopher-tricks/ensuring-go-interface-satisfaction-at-compile-time-1ed158e8fa17
var (
	_           Interface = (*SpoofingClient)(nil)
	dialContext           = (&net.Dialer{}).DialContext
)

// ResponseChecker is used to determine when SpoofinClient.Poll is done polling.
// This allows you to predicate wait.PollImmediate on the request's http.Response.
//
// See the apimachinery wait package:
// https://github.com/kubernetes/apimachinery/blob/cf7ae2f57dabc02a3d215f15ca61ae1446f3be8f/pkg/util/wait/wait.go#L172
type ResponseChecker func(resp *Response) (done bool, err error)

// SpoofingClient is a minimal HTTP client wrapper that spoofs the domain of requests
// for non-resolvable domains.
type SpoofingClient struct {
	Client          *http.Client
	RequestInterval time.Duration
	RequestTimeout  time.Duration

	logf logging.FormatLogger
}

// TransportOption allows callers to customize the http.Transport used by a SpoofingClient
type TransportOption func(transport *http.Transport) *http.Transport

// New returns a SpoofingClient that rewrites requests if the target domain is not `resolvable`.
// It does this by looking up the ingress at construction time, so reusing a client will not
// follow the ingress if it moves (or if there are multiple ingresses).
//
// If that's a problem, see test/request.go#WaitForEndpointState for oneshot spoofing.
func New(
	kubeClientset *kubernetes.Clientset,
	logf logging.FormatLogger,
	domain string,
	resolvable bool,
	endpointOverride string,
	opts ...TransportOption) (*SpoofingClient, error) {
	endpoint, err := ResolveEndpoint(kubeClientset, domain, resolvable, endpointOverride)
	if err != nil {
		fmt.Errorf("failed get the cluster endpoint: %v", err)
	}

	// Spoof the hostname at the resolver level
	logf("Spoofing %s -> %s", domain, endpoint)
	transport := &http.Transport{
		DialContext: func(ctx context.Context, network, addr string) (conn net.Conn, e error) {
			spoofed := addr
			if i := strings.LastIndex(addr, ":"); i != -1 && domain == addr[:i] {
				// The original hostname:port is spoofed by replacing the hostname by the value
				// returned by ResolveEndpoint.
				spoofed = endpoint + ":" + addr[i+1:]
			}
			return dialContext(ctx, network, spoofed)
		},
	}

	for _, opt := range opts {
		transport = opt(transport)
	}

	// Enable Zipkin tracing
	roundTripper := &ochttp.Transport{
		Base:        transport,
		Propagation: &b3.HTTPFormat{},
	}

	sc := SpoofingClient{
		Client:          &http.Client{Transport: roundTripper},
		RequestInterval: requestInterval,
		RequestTimeout:  RequestTimeout,
		logf:            logf,
	}
	return &sc, nil
}

// ResolveEndpoint resolves the endpoint address considering whether the domain is resolvable and taking into
// account whether the user overrode the endpoint address externally
func ResolveEndpoint(kubeClientset *kubernetes.Clientset, domain string, resolvable bool, endpointOverride string) (string, error) {
	// If the domain is resolvable, it can be used directly
	if resolvable {
		return domain, nil
	}
	// If an override is provided, use it
	if endpointOverride != "" {
		return endpointOverride, nil
	}
	// Otherwise, use the actual cluster endpoint
	return ingress.GetIngressEndpoint(kubeClientset)
}

// Do dispatches to the underlying http.Client.Do, spoofing domains as needed
// and transforming the http.Response into a spoof.Response.
// Each response is augmented with "ZipkinTraceID" header that identifies the zipkin trace corresponding to the request.
func (sc *SpoofingClient) Do(req *http.Request) (*Response, error) {
	// Starting span to capture zipkin trace.
	traceContext, span := trace.StartSpan(req.Context(), "SpoofingClient-Trace")
	defer span.End()

	// Check to see if the call to this method is coming from a Poll call.
	logZipkinTrace := true
	if req.Header.Get(pollReqHeader) != "" {
		req.Header.Del(pollReqHeader)
		logZipkinTrace = false
	}
	resp, err := sc.Client.Do(req.WithContext(traceContext))
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()

	resp.Header.Add(zipkin.ZipkinTraceIDHeader, span.SpanContext().TraceID.String())
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	spoofResp := &Response{
		Status:     resp.Status,
		StatusCode: resp.StatusCode,
		Header:     resp.Header,
		Body:       body,
	}

	if logZipkinTrace {
		sc.logZipkinTrace(spoofResp)
	}

	return spoofResp, nil
}

// Poll executes an http request until it satisfies the inState condition or encounters an error.
func (sc *SpoofingClient) Poll(req *http.Request, inState ResponseChecker) (*Response, error) {
	var (
		resp *Response
		err  error
	)

	err = wait.PollImmediate(sc.RequestInterval, sc.RequestTimeout, func() (bool, error) {
		// As we may do multiple Do calls as part of a single Poll we add this temporary header
		// to the request to indicate to Do method not to log Zipkin trace, instead it is
		// handled by this method itself.
		req.Header.Add(pollReqHeader, "True")
		resp, err = sc.Do(req)
		if err != nil {
			if isTCPTimeout(err) {
				sc.logf("Retrying %s for TCP timeout %v", req.URL.String(), err)
				return false, nil
			}
			// Retrying on DNS error, since we may be using xip.io or nip.io in tests.
			if isDNSError(err) {
				sc.logf("Retrying %s for DNS error %v", req.URL.String(), err)
				return false, nil
			}
			// Repeat the poll on `connection refused` errors, which are usually transient Istio errors.
			if isTCPConnectRefuse(err) {
				sc.logf("Retrying %s for connection refused %v", req.URL.String(), err)
				return false, nil
			}
			return true, err
		}

		return inState(resp)
	})

	if resp != nil {
		sc.logZipkinTrace(resp)
	}

	if err != nil {
		return resp, errors.Wrapf(err, "response: %s did not pass checks", resp)
	}
	return resp, nil
}

// logZipkinTrace provides support to log Zipkin Trace for param: spoofResponse
// We only log Zipkin trace for HTTP server errors i.e for HTTP status codes between 500 to 600
func (sc *SpoofingClient) logZipkinTrace(spoofResp *Response) {
	if !zipkin.ZipkinTracingEnabled || spoofResp.StatusCode < http.StatusInternalServerError || spoofResp.StatusCode >= 600 {
		return
	}

	traceID := spoofResp.Header.Get(zipkin.ZipkinTraceIDHeader)
	sc.logf("Logging Zipkin Trace for: %s", traceID)

	json, err := zipkin.JSONTrace(traceID /* We don't know the expected number of spans */, -1, 5*time.Second)
	if err != nil {
		if _, ok := err.(*zipkin.TimeoutError); !ok {
			sc.logf("Error getting zipkin trace: %v", err)
		}
	}

	sc.logf("%s", json)
}
<|endoftext|>


<commit_before><commit_msg>Fix debug paths for MultiFunctions.<commit_after><|endoftext|>


<commit_before><commit_msg>testing godoc example<commit_after><|endoftext|>


<commit_before>// Copyright 2016 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"time"

	"go.chromium.org/luci/common/clock"
	"go.chromium.org/luci/common/data/rand/mathrand"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/gcloud/gs"
	gcps "go.chromium.org/luci/common/gcloud/pubsub"
	log "go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/retry"
	"go.chromium.org/luci/common/retry/transient"
	"go.chromium.org/luci/common/tsmon/distribution"
	"go.chromium.org/luci/common/tsmon/field"
	"go.chromium.org/luci/common/tsmon/metric"
	montypes "go.chromium.org/luci/common/tsmon/types"
	"go.chromium.org/luci/grpc/grpcutil"
	"go.chromium.org/luci/logdog/api/config/svcconfig"
	"go.chromium.org/luci/logdog/common/types"
	"go.chromium.org/luci/logdog/server/archivist"
	"go.chromium.org/luci/logdog/server/bundleServicesClient"
	"go.chromium.org/luci/logdog/server/service"

	"cloud.google.com/go/pubsub"
	"golang.org/x/net/context"

	"go.chromium.org/luci/hardcoded/chromeinfra"

	"net/http"
	_ "net/http/pprof"
)

var (
	errInvalidConfig = errors.New("invalid configuration")

	// tsTaskProcessingTime measures the amount of time spent processing a single
	// task.
	//
	// The "consumed" field is true if the underlying task was consumed and
	// false if it was not.
	tsTaskProcessingTime = metric.NewCumulativeDistribution("logdog/archivist/task_processing_time_ms",
		"The amount of time (in milliseconds) that a single task takes to process.",
		&montypes.MetricMetadata{Units: montypes.Milliseconds},
		distribution.DefaultBucketer,
		field.Bool("consumed"))
)

// application is the Archivist application state.
type application struct {
	service.Service
}

// run is the main execution function.
func (a *application) runArchivist(c context.Context) error {
	cfg := a.ServiceConfig()

	// Starting a webserver for pprof.
	// TODO(hinoka): Checking for memory leaks, Remove me when 795156 is fixed.
	go func() {
		log.WithError(http.ListenAndServe("localhost:6060", nil)).Errorf(c, "failed to start webserver")
	}()

	coordCfg, acfg := cfg.GetCoordinator(), cfg.GetArchivist()
	switch {
	case coordCfg == nil:
		fallthrough

	case acfg == nil:
		return errors.New("missing required config: archivist")
	case acfg.GsStagingBucket == "":
		return errors.New("missing required config: archivist.gs_staging_bucket")
	}

	// Initialize Pub/Sub client.
	//
	// We will initialize both an authenticated Client instance and an
	// authenticated Context, since we need the latter for raw ACK deadline
	// updates.
	taskSub := gcps.Subscription(acfg.Subscription)
	if err := taskSub.Validate(); err != nil {
		log.Fields{
			log.ErrorKey: err,
			"value":      taskSub,
		}.Errorf(c, "Task subscription did not validate.")
		return errors.New("invalid task subscription name")
	}
	psProject, psSubscriptionName := taskSub.Split()

	// New PubSub instance with the authenticated client.
	psClient, err := a.Service.PubSubSubscriberClient(c, psProject)
	if err != nil {
		log.WithError(err).Errorf(c, "Failed to create Pub/Sub client.")
		return err
	}
	sub := psClient.Subscription(psSubscriptionName)
	sub.ReceiveSettings = pubsub.ReceiveSettings{
		// These must be -1 (unlimited), otherwise the flow controller will saturate
		// since we do not Nack messages.  PubSub performs poorly as a Task Queue otherwise.
		// https://github.com/GoogleCloudPlatform/google-cloud-go/issues/919#issuecomment-372403175
		MaxExtension:           -1,
		MaxOutstandingMessages: -1,
		MaxOutstandingBytes:    -1,
		NumGoroutines:          8,
	}

	// Initialize our Storage.
	//
	// NOTE: We're requesting read/write access even though we only need read-only
	// access because GKE doesn't understand the read-only scope:
	// https://www.googleapis.com/auth/bigtable.readonly
	st, err := a.IntermediateStorage(c, true)
	if err != nil {
		log.WithError(err).Errorf(c, "Failed to get storage instance.")
		return err
	}
	defer st.Close()

	// Initialize our Google Storage client.
	gsClient, err := a.GSClient(c)
	if err != nil {
		log.WithError(err).Errorf(c, "Failed to get Google Storage client.")
		return err
	}
	defer gsClient.Close()

	// Initialize a Coordinator client that bundles requests together.
	coordClient := &bundleServicesClient.Client{
		ServicesClient:       a.Coordinator(),
		DelayThreshold:       time.Second,
		BundleCountThreshold: 100,
	}
	defer coordClient.Flush()

	ar := archivist.Archivist{
		Service:        coordClient,
		SettingsLoader: a.GetSettingsLoader(acfg),
		Storage:        st,
		GSClient:       gsClient,
	}

	// Application shutdown will now operate by stopping the Iterator.
	c, cancelFunc := context.WithCancel(c)
	defer cancelFunc()

	// Application shutdown will now operate by cancelling the Collector's
	// shutdown Context.
	a.SetShutdownFunc(cancelFunc)

	// Execute our main subscription pull loop. It will run until the supplied
	// Context is cancelled.
	log.Fields{
		"subscription": taskSub,
	}.Infof(c, "Pulling tasks from Pub/Sub subscription.")

	retryForever := func() retry.Iterator {
		return &retry.ExponentialBackoff{
			Limited: retry.Limited{
				Delay:   200 * time.Millisecond,
				Retries: -1, // Unlimited.
			},
			MaxDelay:   10 * time.Second,
			Multiplier: 2,
		}
	}

	err = retry.Retry(c, transient.Only(retryForever), func() error {
		return grpcutil.WrapIfTransient(sub.Receive(c, func(c context.Context, msg *pubsub.Message) {
			c = log.SetFields(c, log.Fields{
				"messageID": msg.ID,
			})

			// ACK or NACK the message based on whether our task was consumed.
			deleteTask := false
			defer func() {
				// ACK the message if it is completed. If not, NACK it.
				if deleteTask {
					msg.Ack()
				} else {
					msg.Nack()
				}
			}()

			// Time how long task processing takes for metrics.
			startTime := clock.Now(c)
			defer func() {
				duration := clock.Now(c).Sub(startTime)

				if deleteTask {
					log.Fields{
						"duration": duration,
					}.Infof(c, "Task successfully processed; deleting.")
				} else {
					log.Fields{
						"duration": duration,
					}.Infof(c, "Task processing incomplete. Not deleting.")
				}

				// Add to our processing time metric.
				tsTaskProcessingTime.Add(c, duration.Seconds()*1000, deleteTask)
			}()

			task, err := makePubSubArchivistTask(c, psSubscriptionName, msg)
			c = log.SetFields(c, log.Fields{
				"consumed":         task.consumed,
				"subscriptionName": task.subscriptionName,
				"taskTimestamp":    task.timestamp.Format(time.RFC3339Nano),
				"archiveTask":      task.at,
			})
			if task.msg != nil {
				// Log all fields except data.
				c = log.SetFields(c, log.Fields{
					"message": map[string]interface{}{
						"id":          task.msg.ID,
						"attributes":  task.msg.Attributes,
						"publishTime": task.msg.PublishTime.Format(time.RFC3339Nano),
					},
				})
			}
			if err != nil {
				log.WithError(err).Errorf(c, "Failed to unmarshal archive task from message.")
				deleteTask = true
				return
			}

			ar.ArchiveTask(c, task)
			deleteTask = task.consumed
		}))
	}, func(err error, d time.Duration) {
		log.Fields{
			log.ErrorKey: err,
			"delay":      d,
		}.Warningf(c, "Transient error during subscription Receive loop; retrying...")
	})

	if err := errors.Unwrap(err); err != nil && err != context.Canceled {
		log.WithError(err).Errorf(c, "Failed during Pub/Sub Receive.")
		return err
	}

	log.Debugf(c, "Archivist finished.")
	return nil
}

// GetSettingsLoader is an archivist.SettingsLoader implementation that merges
// global and project-specific settings.
//
// The resulting settings object will be verified by the Archivist.
func (a *application) GetSettingsLoader(acfg *svcconfig.Archivist) archivist.SettingsLoader {
	serviceID := a.ServiceID()

	return func(c context.Context, proj types.ProjectName) (*archivist.Settings, error) {
		// Fold in our project-specific configuration, if valid.
		pcfg, err := a.ProjectConfig(c, proj)
		if err != nil {
			log.Fields{
				log.ErrorKey: err,
				"project":    proj,
			}.Errorf(c, "Failed to fetch project configuration.")
			return nil, err
		}

		indexParam := func(get func(ic *svcconfig.ArchiveIndexConfig) int32) int {
			if ic := pcfg.ArchiveIndexConfig; ic != nil {
				if v := get(ic); v > 0 {
					return int(v)
				}
			}

			if ic := acfg.ArchiveIndexConfig; ic != nil {
				if v := get(ic); v > 0 {
					return int(v)
				}
			}

			return 0
		}

		// Load our base settings.
		//
		// Archival bases are:
		// Staging: gs://<services:gs_staging_bucket>/<project-id>/...
		// Archive: gs://<project:archive_gs_bucket>/<project-id>/...
		st := archivist.Settings{
			GSBase:        gs.MakePath(pcfg.ArchiveGsBucket, "").Concat(serviceID),
			GSStagingBase: gs.MakePath(acfg.GsStagingBucket, "").Concat(serviceID),

			IndexStreamRange: indexParam(func(ic *svcconfig.ArchiveIndexConfig) int32 { return ic.StreamRange }),
			IndexPrefixRange: indexParam(func(ic *svcconfig.ArchiveIndexConfig) int32 { return ic.PrefixRange }),
			IndexByteRange:   indexParam(func(ic *svcconfig.ArchiveIndexConfig) int32 { return ic.ByteRange }),
			AlwaysRender:     (acfg.RenderAllStreams || pcfg.RenderAllStreams),
		}

		// Fold project settings into loaded ones.
		return &st, nil
	}
}

// Entry point.
func main() {
	mathrand.SeedRandomly()
	a := application{
		Service: service.Service{
			Name:               "archivist",
			DefaultAuthOptions: chromeinfra.DefaultAuthOptions(),
		},
	}
	a.Run(context.Background(), a.runArchivist)
}
<commit_msg>[logdog] Set MaxOutstandingMessages to 16<commit_after>// Copyright 2016 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"time"

	"go.chromium.org/luci/common/clock"
	"go.chromium.org/luci/common/data/rand/mathrand"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/gcloud/gs"
	gcps "go.chromium.org/luci/common/gcloud/pubsub"
	log "go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/retry"
	"go.chromium.org/luci/common/retry/transient"
	"go.chromium.org/luci/common/tsmon/distribution"
	"go.chromium.org/luci/common/tsmon/field"
	"go.chromium.org/luci/common/tsmon/metric"
	montypes "go.chromium.org/luci/common/tsmon/types"
	"go.chromium.org/luci/grpc/grpcutil"
	"go.chromium.org/luci/logdog/api/config/svcconfig"
	"go.chromium.org/luci/logdog/common/types"
	"go.chromium.org/luci/logdog/server/archivist"
	"go.chromium.org/luci/logdog/server/bundleServicesClient"
	"go.chromium.org/luci/logdog/server/service"

	"cloud.google.com/go/pubsub"
	"golang.org/x/net/context"

	"go.chromium.org/luci/hardcoded/chromeinfra"

	"net/http"
	_ "net/http/pprof"
)

var (
	errInvalidConfig = errors.New("invalid configuration")

	// tsTaskProcessingTime measures the amount of time spent processing a single
	// task.
	//
	// The "consumed" field is true if the underlying task was consumed and
	// false if it was not.
	tsTaskProcessingTime = metric.NewCumulativeDistribution("logdog/archivist/task_processing_time_ms",
		"The amount of time (in milliseconds) that a single task takes to process.",
		&montypes.MetricMetadata{Units: montypes.Milliseconds},
		distribution.DefaultBucketer,
		field.Bool("consumed"))
)

// application is the Archivist application state.
type application struct {
	service.Service
}

// run is the main execution function.
func (a *application) runArchivist(c context.Context) error {
	cfg := a.ServiceConfig()

	// Starting a webserver for pprof.
	// TODO(hinoka): Checking for memory leaks, Remove me when 795156 is fixed.
	go func() {
		log.WithError(http.ListenAndServe("localhost:6060", nil)).Errorf(c, "failed to start webserver")
	}()

	coordCfg, acfg := cfg.GetCoordinator(), cfg.GetArchivist()
	switch {
	case coordCfg == nil:
		fallthrough

	case acfg == nil:
		return errors.New("missing required config: archivist")
	case acfg.GsStagingBucket == "":
		return errors.New("missing required config: archivist.gs_staging_bucket")
	}

	// Initialize Pub/Sub client.
	//
	// We will initialize both an authenticated Client instance and an
	// authenticated Context, since we need the latter for raw ACK deadline
	// updates.
	taskSub := gcps.Subscription(acfg.Subscription)
	if err := taskSub.Validate(); err != nil {
		log.Fields{
			log.ErrorKey: err,
			"value":      taskSub,
		}.Errorf(c, "Task subscription did not validate.")
		return errors.New("invalid task subscription name")
	}
	psProject, psSubscriptionName := taskSub.Split()

	// New PubSub instance with the authenticated client.
	psClient, err := a.Service.PubSubSubscriberClient(c, psProject)
	if err != nil {
		log.WithError(err).Errorf(c, "Failed to create Pub/Sub client.")
		return err
	}
	sub := psClient.Subscription(psSubscriptionName)
	sub.ReceiveSettings = pubsub.ReceiveSettings{
		// These must be -1 (unlimited), otherwise the flow controller will saturate.
		// PubSub performs poorly as a Task Queue otherwise.
		// https://github.com/GoogleCloudPlatform/google-cloud-go/issues/919#issuecomment-372403175
		MaxExtension:        -1,
		MaxOutstandingBytes: -1,

		MaxOutstandingMessages: 16,
		NumGoroutines:          8,
	}

	// Initialize our Storage.
	//
	// NOTE: We're requesting read/write access even though we only need read-only
	// access because GKE doesn't understand the read-only scope:
	// https://www.googleapis.com/auth/bigtable.readonly
	st, err := a.IntermediateStorage(c, true)
	if err != nil {
		log.WithError(err).Errorf(c, "Failed to get storage instance.")
		return err
	}
	defer st.Close()

	// Initialize our Google Storage client.
	gsClient, err := a.GSClient(c)
	if err != nil {
		log.WithError(err).Errorf(c, "Failed to get Google Storage client.")
		return err
	}
	defer gsClient.Close()

	// Initialize a Coordinator client that bundles requests together.
	coordClient := &bundleServicesClient.Client{
		ServicesClient:       a.Coordinator(),
		DelayThreshold:       time.Second,
		BundleCountThreshold: 100,
	}
	defer coordClient.Flush()

	ar := archivist.Archivist{
		Service:        coordClient,
		SettingsLoader: a.GetSettingsLoader(acfg),
		Storage:        st,
		GSClient:       gsClient,
	}

	// Application shutdown will now operate by stopping the Iterator.
	c, cancelFunc := context.WithCancel(c)
	defer cancelFunc()

	// Application shutdown will now operate by cancelling the Collector's
	// shutdown Context.
	a.SetShutdownFunc(cancelFunc)

	// Execute our main subscription pull loop. It will run until the supplied
	// Context is cancelled.
	log.Fields{
		"subscription": taskSub,
	}.Infof(c, "Pulling tasks from Pub/Sub subscription.")

	retryForever := func() retry.Iterator {
		return &retry.ExponentialBackoff{
			Limited: retry.Limited{
				Delay:   200 * time.Millisecond,
				Retries: -1, // Unlimited.
			},
			MaxDelay:   10 * time.Second,
			Multiplier: 2,
		}
	}

	err = retry.Retry(c, transient.Only(retryForever), func() error {
		return grpcutil.WrapIfTransient(sub.Receive(c, func(c context.Context, msg *pubsub.Message) {
			c = log.SetFields(c, log.Fields{
				"messageID": msg.ID,
			})

			// ACK or NACK the message based on whether our task was consumed.
			deleteTask := false
			defer func() {
				// ACK the message if it is completed. If not, NACK it.
				if deleteTask {
					msg.Ack()
				} else {
					msg.Nack()
				}
			}()

			// Time how long task processing takes for metrics.
			startTime := clock.Now(c)
			defer func() {
				duration := clock.Now(c).Sub(startTime)

				if deleteTask {
					log.Fields{
						"duration": duration,
					}.Infof(c, "Task successfully processed; deleting.")
				} else {
					log.Fields{
						"duration": duration,
					}.Infof(c, "Task processing incomplete. Not deleting.")
				}

				// Add to our processing time metric.
				tsTaskProcessingTime.Add(c, duration.Seconds()*1000, deleteTask)
			}()

			task, err := makePubSubArchivistTask(c, psSubscriptionName, msg)
			c = log.SetFields(c, log.Fields{
				"consumed":         task.consumed,
				"subscriptionName": task.subscriptionName,
				"taskTimestamp":    task.timestamp.Format(time.RFC3339Nano),
				"archiveTask":      task.at,
			})
			if task.msg != nil {
				// Log all fields except data.
				c = log.SetFields(c, log.Fields{
					"message": map[string]interface{}{
						"id":          task.msg.ID,
						"attributes":  task.msg.Attributes,
						"publishTime": task.msg.PublishTime.Format(time.RFC3339Nano),
					},
				})
			}
			if err != nil {
				log.WithError(err).Errorf(c, "Failed to unmarshal archive task from message.")
				deleteTask = true
				return
			}

			ar.ArchiveTask(c, task)
			deleteTask = task.consumed
		}))
	}, func(err error, d time.Duration) {
		log.Fields{
			log.ErrorKey: err,
			"delay":      d,
		}.Warningf(c, "Transient error during subscription Receive loop; retrying...")
	})

	if err := errors.Unwrap(err); err != nil && err != context.Canceled {
		log.WithError(err).Errorf(c, "Failed during Pub/Sub Receive.")
		return err
	}

	log.Debugf(c, "Archivist finished.")
	return nil
}

// GetSettingsLoader is an archivist.SettingsLoader implementation that merges
// global and project-specific settings.
//
// The resulting settings object will be verified by the Archivist.
func (a *application) GetSettingsLoader(acfg *svcconfig.Archivist) archivist.SettingsLoader {
	serviceID := a.ServiceID()

	return func(c context.Context, proj types.ProjectName) (*archivist.Settings, error) {
		// Fold in our project-specific configuration, if valid.
		pcfg, err := a.ProjectConfig(c, proj)
		if err != nil {
			log.Fields{
				log.ErrorKey: err,
				"project":    proj,
			}.Errorf(c, "Failed to fetch project configuration.")
			return nil, err
		}

		indexParam := func(get func(ic *svcconfig.ArchiveIndexConfig) int32) int {
			if ic := pcfg.ArchiveIndexConfig; ic != nil {
				if v := get(ic); v > 0 {
					return int(v)
				}
			}

			if ic := acfg.ArchiveIndexConfig; ic != nil {
				if v := get(ic); v > 0 {
					return int(v)
				}
			}

			return 0
		}

		// Load our base settings.
		//
		// Archival bases are:
		// Staging: gs://<services:gs_staging_bucket>/<project-id>/...
		// Archive: gs://<project:archive_gs_bucket>/<project-id>/...
		st := archivist.Settings{
			GSBase:        gs.MakePath(pcfg.ArchiveGsBucket, "").Concat(serviceID),
			GSStagingBase: gs.MakePath(acfg.GsStagingBucket, "").Concat(serviceID),

			IndexStreamRange: indexParam(func(ic *svcconfig.ArchiveIndexConfig) int32 { return ic.StreamRange }),
			IndexPrefixRange: indexParam(func(ic *svcconfig.ArchiveIndexConfig) int32 { return ic.PrefixRange }),
			IndexByteRange:   indexParam(func(ic *svcconfig.ArchiveIndexConfig) int32 { return ic.ByteRange }),
			AlwaysRender:     (acfg.RenderAllStreams || pcfg.RenderAllStreams),
		}

		// Fold project settings into loaded ones.
		return &st, nil
	}
}

// Entry point.
func main() {
	mathrand.SeedRandomly()
	a := application{
		Service: service.Service{
			Name:               "archivist",
			DefaultAuthOptions: chromeinfra.DefaultAuthOptions(),
		},
	}
	a.Run(context.Background(), a.runArchivist)
}
<|endoftext|>


<commit_before>/*
Copyright The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Code generated by client-gen. DO NOT EDIT.

package fake

import (
	admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
	admissionregistrationv1beta1 "k8s.io/api/admissionregistration/v1beta1"
	appsv1 "k8s.io/api/apps/v1"
	appsv1beta1 "k8s.io/api/apps/v1beta1"
	appsv1beta2 "k8s.io/api/apps/v1beta2"
	auditregistrationv1alpha1 "k8s.io/api/auditregistration/v1alpha1"
	authenticationv1 "k8s.io/api/authentication/v1"
	authenticationv1beta1 "k8s.io/api/authentication/v1beta1"
	authorizationv1 "k8s.io/api/authorization/v1"
	authorizationv1beta1 "k8s.io/api/authorization/v1beta1"
	autoscalingv1 "k8s.io/api/autoscaling/v1"
	autoscalingv2beta1 "k8s.io/api/autoscaling/v2beta1"
	autoscalingv2beta2 "k8s.io/api/autoscaling/v2beta2"
	batchv1 "k8s.io/api/batch/v1"
	batchv1beta1 "k8s.io/api/batch/v1beta1"
	batchv2alpha1 "k8s.io/api/batch/v2alpha1"
	certificatesv1beta1 "k8s.io/api/certificates/v1beta1"
	coordinationv1 "k8s.io/api/coordination/v1"
	coordinationv1beta1 "k8s.io/api/coordination/v1beta1"
	corev1 "k8s.io/api/core/v1"
	discoveryv1alpha1 "k8s.io/api/discovery/v1alpha1"
	discoveryv1beta1 "k8s.io/api/discovery/v1beta1"
	eventsv1beta1 "k8s.io/api/events/v1beta1"
	extensionsv1beta1 "k8s.io/api/extensions/v1beta1"
	flowcontrolv1alpha1 "k8s.io/api/flowcontrol/v1alpha1"
	networkingv1 "k8s.io/api/networking/v1"
	networkingv1beta1 "k8s.io/api/networking/v1beta1"
	nodev1alpha1 "k8s.io/api/node/v1alpha1"
	nodev1beta1 "k8s.io/api/node/v1beta1"
	policyv1beta1 "k8s.io/api/policy/v1beta1"
	rbacv1 "k8s.io/api/rbac/v1"
	rbacv1alpha1 "k8s.io/api/rbac/v1alpha1"
	rbacv1beta1 "k8s.io/api/rbac/v1beta1"
	schedulingv1 "k8s.io/api/scheduling/v1"
	schedulingv1alpha1 "k8s.io/api/scheduling/v1alpha1"
	schedulingv1beta1 "k8s.io/api/scheduling/v1beta1"
	settingsv1alpha1 "k8s.io/api/settings/v1alpha1"
	storagev1 "k8s.io/api/storage/v1"
	storagev1alpha1 "k8s.io/api/storage/v1alpha1"
	storagev1beta1 "k8s.io/api/storage/v1beta1"
	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	runtime "k8s.io/apimachinery/pkg/runtime"
	schema "k8s.io/apimachinery/pkg/runtime/schema"
	serializer "k8s.io/apimachinery/pkg/runtime/serializer"
	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
)

var scheme = runtime.NewScheme()
var codecs = serializer.NewCodecFactory(scheme)
var parameterCodec = runtime.NewParameterCodec(scheme)
var localSchemeBuilder = runtime.SchemeBuilder{
	admissionregistrationv1.AddToScheme,
	admissionregistrationv1beta1.AddToScheme,
	appsv1.AddToScheme,
	appsv1beta1.AddToScheme,
	appsv1beta2.AddToScheme,
	auditregistrationv1alpha1.AddToScheme,
	authenticationv1.AddToScheme,
	authenticationv1beta1.AddToScheme,
	authorizationv1.AddToScheme,
	authorizationv1beta1.AddToScheme,
	autoscalingv1.AddToScheme,
	autoscalingv2beta1.AddToScheme,
	autoscalingv2beta2.AddToScheme,
	batchv1.AddToScheme,
	batchv1beta1.AddToScheme,
	batchv2alpha1.AddToScheme,
	certificatesv1beta1.AddToScheme,
	coordinationv1beta1.AddToScheme,
	coordinationv1.AddToScheme,
	corev1.AddToScheme,
	discoveryv1alpha1.AddToScheme,
	discoveryv1beta1.AddToScheme,
	eventsv1beta1.AddToScheme,
	extensionsv1beta1.AddToScheme,
	flowcontrolv1alpha1.AddToScheme,
	networkingv1.AddToScheme,
	networkingv1beta1.AddToScheme,
	nodev1alpha1.AddToScheme,
	nodev1beta1.AddToScheme,
	policyv1beta1.AddToScheme,
	rbacv1.AddToScheme,
	rbacv1beta1.AddToScheme,
	rbacv1alpha1.AddToScheme,
	schedulingv1alpha1.AddToScheme,
	schedulingv1beta1.AddToScheme,
	schedulingv1.AddToScheme,
	settingsv1alpha1.AddToScheme,
	storagev1beta1.AddToScheme,
	storagev1.AddToScheme,
	storagev1alpha1.AddToScheme,
}

// AddToScheme adds all types of this clientset into the given scheme. This allows composition
// of clientsets, like in:
//
//   import (
//     "k8s.io/client-go/kubernetes"
//     clientsetscheme "k8s.io/client-go/kubernetes/scheme"
//     aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme"
//   )
//
//   kclientset, _ := kubernetes.NewForConfig(c)
//   _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme)
//
// After this, RawExtensions in Kubernetes types will serialize kube-aggregator types
// correctly.
var AddToScheme = localSchemeBuilder.AddToScheme

func init() {
	v1.AddToGroupVersion(scheme, schema.GroupVersion{Version: "v1"})
	utilruntime.Must(AddToScheme(scheme))
}
<commit_msg>Run ./hack/update-codegen.sh.<commit_after>/*
Copyright The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Code generated by client-gen. DO NOT EDIT.

package fake

import (
	admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
	admissionregistrationv1beta1 "k8s.io/api/admissionregistration/v1beta1"
	appsv1 "k8s.io/api/apps/v1"
	appsv1beta1 "k8s.io/api/apps/v1beta1"
	appsv1beta2 "k8s.io/api/apps/v1beta2"
	auditregistrationv1alpha1 "k8s.io/api/auditregistration/v1alpha1"
	authenticationv1 "k8s.io/api/authentication/v1"
	authenticationv1beta1 "k8s.io/api/authentication/v1beta1"
	authorizationv1 "k8s.io/api/authorization/v1"
	authorizationv1beta1 "k8s.io/api/authorization/v1beta1"
	autoscalingv1 "k8s.io/api/autoscaling/v1"
	autoscalingv2beta1 "k8s.io/api/autoscaling/v2beta1"
	autoscalingv2beta2 "k8s.io/api/autoscaling/v2beta2"
	batchv1 "k8s.io/api/batch/v1"
	batchv1beta1 "k8s.io/api/batch/v1beta1"
	batchv2alpha1 "k8s.io/api/batch/v2alpha1"
	certificatesv1beta1 "k8s.io/api/certificates/v1beta1"
	coordinationv1 "k8s.io/api/coordination/v1"
	coordinationv1beta1 "k8s.io/api/coordination/v1beta1"
	corev1 "k8s.io/api/core/v1"
	discoveryv1alpha1 "k8s.io/api/discovery/v1alpha1"
	discoveryv1beta1 "k8s.io/api/discovery/v1beta1"
	eventsv1beta1 "k8s.io/api/events/v1beta1"
	extensionsv1beta1 "k8s.io/api/extensions/v1beta1"
	flowcontrolv1alpha1 "k8s.io/api/flowcontrol/v1alpha1"
	networkingv1 "k8s.io/api/networking/v1"
	networkingv1beta1 "k8s.io/api/networking/v1beta1"
	nodev1alpha1 "k8s.io/api/node/v1alpha1"
	nodev1beta1 "k8s.io/api/node/v1beta1"
	policyv1beta1 "k8s.io/api/policy/v1beta1"
	rbacv1 "k8s.io/api/rbac/v1"
	rbacv1alpha1 "k8s.io/api/rbac/v1alpha1"
	rbacv1beta1 "k8s.io/api/rbac/v1beta1"
	schedulingv1 "k8s.io/api/scheduling/v1"
	schedulingv1alpha1 "k8s.io/api/scheduling/v1alpha1"
	schedulingv1beta1 "k8s.io/api/scheduling/v1beta1"
	settingsv1alpha1 "k8s.io/api/settings/v1alpha1"
	storagev1 "k8s.io/api/storage/v1"
	storagev1alpha1 "k8s.io/api/storage/v1alpha1"
	storagev1beta1 "k8s.io/api/storage/v1beta1"
	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	runtime "k8s.io/apimachinery/pkg/runtime"
	schema "k8s.io/apimachinery/pkg/runtime/schema"
	serializer "k8s.io/apimachinery/pkg/runtime/serializer"
	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
)

var scheme = runtime.NewScheme()
var codecs = serializer.NewCodecFactory(scheme)

var localSchemeBuilder = runtime.SchemeBuilder{
	admissionregistrationv1.AddToScheme,
	admissionregistrationv1beta1.AddToScheme,
	appsv1.AddToScheme,
	appsv1beta1.AddToScheme,
	appsv1beta2.AddToScheme,
	auditregistrationv1alpha1.AddToScheme,
	authenticationv1.AddToScheme,
	authenticationv1beta1.AddToScheme,
	authorizationv1.AddToScheme,
	authorizationv1beta1.AddToScheme,
	autoscalingv1.AddToScheme,
	autoscalingv2beta1.AddToScheme,
	autoscalingv2beta2.AddToScheme,
	batchv1.AddToScheme,
	batchv1beta1.AddToScheme,
	batchv2alpha1.AddToScheme,
	certificatesv1beta1.AddToScheme,
	coordinationv1beta1.AddToScheme,
	coordinationv1.AddToScheme,
	corev1.AddToScheme,
	discoveryv1alpha1.AddToScheme,
	discoveryv1beta1.AddToScheme,
	eventsv1beta1.AddToScheme,
	extensionsv1beta1.AddToScheme,
	flowcontrolv1alpha1.AddToScheme,
	networkingv1.AddToScheme,
	networkingv1beta1.AddToScheme,
	nodev1alpha1.AddToScheme,
	nodev1beta1.AddToScheme,
	policyv1beta1.AddToScheme,
	rbacv1.AddToScheme,
	rbacv1beta1.AddToScheme,
	rbacv1alpha1.AddToScheme,
	schedulingv1alpha1.AddToScheme,
	schedulingv1beta1.AddToScheme,
	schedulingv1.AddToScheme,
	settingsv1alpha1.AddToScheme,
	storagev1beta1.AddToScheme,
	storagev1.AddToScheme,
	storagev1alpha1.AddToScheme,
}

// AddToScheme adds all types of this clientset into the given scheme. This allows composition
// of clientsets, like in:
//
//   import (
//     "k8s.io/client-go/kubernetes"
//     clientsetscheme "k8s.io/client-go/kubernetes/scheme"
//     aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme"
//   )
//
//   kclientset, _ := kubernetes.NewForConfig(c)
//   _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme)
//
// After this, RawExtensions in Kubernetes types will serialize kube-aggregator types
// correctly.
var AddToScheme = localSchemeBuilder.AddToScheme

func init() {
	v1.AddToGroupVersion(scheme, schema.GroupVersion{Version: "v1"})
	utilruntime.Must(AddToScheme(scheme))
}
<|endoftext|>


<commit_before>package drivers

import (
	"bufio"
	"fmt"
	"os"
	"strings"

	"github.com/pkg/errors"

	"github.com/lxc/lxd/shared"
)

// fsExists checks that the Ceph FS instance indeed exists.
func (d *cephfs) fsExists(clusterName string, userName string, fsName string) bool {
	_, err := shared.RunCommand("ceph", "--name", fmt.Sprintf("client.%s", userName), "--cluster", clusterName, "fs", "get", fsName)
	if err != nil {
		return false
	}

	return true
}

// getConfig parses the Ceph configuration file and returns the list of monitors and secret key.
func (d *cephfs) getConfig(clusterName string, userName string) ([]string, string, error) {
	// Parse the CEPH configuration.
	cephConf, err := os.Open(fmt.Sprintf("/etc/ceph/%s.conf", clusterName))
	if err != nil {
		return nil, "", errors.Wrapf(err, "Failed to open '%s", fmt.Sprintf("/etc/ceph/%s.conf", clusterName))
	}

	cephMon := []string{}

	scan := bufio.NewScanner(cephConf)
	for scan.Scan() {
		line := scan.Text()
		line = strings.TrimSpace(line)

		if line == "" {
			continue
		}

		if strings.HasPrefix(line, "mon_host") || strings.HasPrefix(line, "mon-host") || strings.HasPrefix(line, "mon host") {
			fields := strings.SplitN(line, "=", 2)
			if len(fields) < 2 {
				continue
			}

			servers := strings.Split(fields[1], ",")
			for _, server := range servers {
				cephMon = append(cephMon, strings.TrimSpace(server))
			}
			break
		}
	}

	if len(cephMon) == 0 {
		return nil, "", fmt.Errorf("Couldn't find a CPEH mon")
	}

	// Parse the CEPH keyring.
	cephKeyring, err := os.Open(fmt.Sprintf("/etc/ceph/%v.client.%v.keyring", clusterName, userName))
	if err != nil {
		return nil, "", errors.Wrapf(err, "Failed to open '%s", fmt.Sprintf("/etc/ceph/%v.client.%v.keyring", clusterName, userName))
	}

	var cephSecret string

	scan = bufio.NewScanner(cephKeyring)
	for scan.Scan() {
		line := scan.Text()
		line = strings.TrimSpace(line)

		if line == "" {
			continue
		}

		if strings.HasPrefix(line, "key") {
			fields := strings.SplitN(line, "=", 2)
			if len(fields) < 2 {
				continue
			}

			cephSecret = strings.TrimSpace(fields[1])
			break
		}
	}

	if cephSecret == "" {
		return nil, "", fmt.Errorf("Couldn't find a keyring entry")
	}

	return cephMon, cephSecret, nil
}
<commit_msg>lxd/storage/cephfs: Use new ceph parsing funtions<commit_after>package drivers

import (
	"fmt"

	"github.com/lxc/lxd/shared"
)

// fsExists checks that the Ceph FS instance indeed exists.
func (d *cephfs) fsExists(clusterName string, userName string, fsName string) bool {
	_, err := shared.RunCommand("ceph", "--name", fmt.Sprintf("client.%s", userName), "--cluster", clusterName, "fs", "get", fsName)
	if err != nil {
		return false
	}

	return true
}

// getConfig parses the Ceph configuration file and returns the list of monitors and secret key.
func (d *cephfs) getConfig(clusterName string, userName string) ([]string, string, error) {
	// Get the monitor list.
	monitors, err := CephMonitors(clusterName)
	if err != nil {
		return nil, "", err
	}

	// Get the keyring entry.
	secret, err := CephKeyring(clusterName, userName)
	if err != nil {
		return nil, "", err
	}

	return monitors, secret, nil
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"flag"
	"fmt"
	"log"
	"os"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/ec2metadata"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudwatch"
	"github.com/aws/aws-sdk-go/service/ec2"
	mp "github.com/mackerelio/go-mackerel-plugin"
)

var metricPeriodDefault = 300
var metricPeriodByVolumeType = map[string]int{
	"io1": 60,
}

var volumeTypesHavingExtraMetrics = []string{
	"io1",
}

var defaultGraphs = []string{
	"ec2.ebs.bandwidth",
	"ec2.ebs.throughput",
	"ec2.ebs.size_per_op",
	"ec2.ebs.latency",
	"ec2.ebs.queue_length",
	"ec2.ebs.idle_time",
}

var extraGraphs = []string{
	"ec2.ebs.throughput_delivered",
	"ec2.ebs.consumed_ops",
}

type cloudWatchSetting struct {
	MetricName string
	Statistics string
	CalcFunc   func(float64, float64) float64
}

// http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/monitoring-volume-status.html
var cloudwatchdef = map[string](cloudWatchSetting){
	"bw_%s_read": cloudWatchSetting{
		MetricName: "VolumeReadBytes", Statistics: "Sum",
		CalcFunc: func(val float64, period float64) float64 { return val / period },
	},
	"bw_%s_write": cloudWatchSetting{
		MetricName: "VolumeWriteBytes", Statistics: "Sum",
		CalcFunc: func(val float64, period float64) float64 { return val / period },
	},
	"throughput_%s_read": cloudWatchSetting{
		MetricName: "VolumeReadOps", Statistics: "Sum",
		CalcFunc: func(val float64, period float64) float64 { return val / period },
	},
	"throughput_%s_write": cloudWatchSetting{
		MetricName: "VolumeWriteOps", Statistics: "Sum",
		CalcFunc: func(val float64, period float64) float64 { return val / period },
	},
	"size_per_op_%s_read": cloudWatchSetting{
		MetricName: "VolumeReadBytes", Statistics: "Average",
		CalcFunc: func(val float64, period float64) float64 { return val },
	},
	"size_per_op_%s_write": cloudWatchSetting{
		MetricName: "VolumeWriteBytes", Statistics: "Average",
		CalcFunc: func(val float64, period float64) float64 { return val },
	},
	"latency_%s_read": cloudWatchSetting{
		MetricName: "VolumeTotalReadTime", Statistics: "Average",
		CalcFunc: func(val float64, period float64) float64 { return val * 1000 },
	},
	"latency_%s_write": cloudWatchSetting{
		MetricName: "VolumeTotalWriteTime", Statistics: "Average",
		CalcFunc: func(val float64, period float64) float64 { return val * 1000 },
	},
	"queue_length_%s": cloudWatchSetting{
		MetricName: "VolumeQueueLength", Statistics: "Average",
		CalcFunc: func(val float64, period float64) float64 { return val },
	},
	"idle_time_%s": cloudWatchSetting{
		MetricName: "VolumeIdleTime", Statistics: "Sum",
		CalcFunc: func(val float64, period float64) float64 { return val / period * 100 },
	},
	"throughput_delivered_%s": cloudWatchSetting{
		MetricName: "VolumeThroughputPercentage", Statistics: "Average",
		CalcFunc: func(val float64, period float64) float64 { return val },
	},
	"consumed_ops_%s": cloudWatchSetting{
		MetricName: "VolumeConsumedReadWriteOps", Statistics: "Sum",
		CalcFunc: func(val float64, period float64) float64 { return val },
	},
}

var graphdef = map[string](mp.Graphs){
	"ec2.ebs.bandwidth": mp.Graphs{
		Label: "EBS Bandwidth",
		Unit:  "bytes/sec",
		Metrics: [](mp.Metrics){
			mp.Metrics{Name: "bw_%s_read", Label: "%s Read", Diff: false},
			mp.Metrics{Name: "bw_%s_write", Label: "%s Write", Diff: false},
		},
	},
	"ec2.ebs.throughput": mp.Graphs{
		Label: "EBS Throughput (op/s)",
		Unit:  "iops",
		Metrics: [](mp.Metrics){
			mp.Metrics{Name: "throughput_%s_read", Label: "%s Read", Diff: false},
			mp.Metrics{Name: "throughput_%s_write", Label: "%s Write", Diff: false},
		},
	},
	"ec2.ebs.size_per_op": mp.Graphs{
		Label: "EBS Avg Op Size (Bytes/op)",
		Unit:  "bytes",
		Metrics: [](mp.Metrics){
			mp.Metrics{Name: "size_per_op_%s_read", Label: "%s Read", Diff: false},
			mp.Metrics{Name: "size_per_op_%s_write", Label: "%s Write", Diff: false},
		},
	},
	"ec2.ebs.latency": mp.Graphs{
		Label: "EBS Avg Latency (ms/op)",
		Unit:  "float",
		Metrics: [](mp.Metrics){
			mp.Metrics{Name: "latency_%s_read", Label: "%s Read", Diff: false},
			mp.Metrics{Name: "latency_%s_write", Label: "%s Write", Diff: false},
		},
	},
	"ec2.ebs.queue_length": mp.Graphs{
		Label: "EBS Avg Queue Length (ops)",
		Unit:  "float",
		Metrics: [](mp.Metrics){
			mp.Metrics{Name: "queue_length_%s", Label: "%s", Diff: false},
		},
	},
	"ec2.ebs.idle_time": mp.Graphs{
		Label: "EBS Time Spent Idle (%)",
		Unit:  "percentage",
		Metrics: [](mp.Metrics){
			mp.Metrics{Name: "idle_time_%s", Label: "%s", Diff: false},
		},
	},
	"ec2.ebs.throughput_delivered": mp.Graphs{
		Label: "EBS Throughput of Provisioned IOPS (%)",
		Unit:  "percentage",
		Metrics: [](mp.Metrics){
			mp.Metrics{Name: "throughput_delivered_%s", Label: "%s", Diff: false},
		},
	},
	"ec2.ebs.consumed_ops": mp.Graphs{
		Label: "EBS Consumed Ops (Provisioned IOPS)",
		Unit:  "float",
		Metrics: [](mp.Metrics){
			mp.Metrics{Name: "consumed_ops_%s", Label: "%s", Diff: false},
		},
	},
}

var stderrLogger *log.Logger

// EBSPlugin mackerel plugin for ebs
type EBSPlugin struct {
	Region          string
	AccessKeyID     string
	SecretAccessKey string
	InstanceID      string
	Credentials     *credentials.Credentials
	EC2             *ec2.EC2
	CloudWatch      *cloudwatch.CloudWatch
	Volumes         *[]*ec2.Volume
}

func (p *EBSPlugin) prepare() error {
	if p.AccessKeyID != "" && p.SecretAccessKey != "" {
		p.Credentials = credentials.NewStaticCredentials(p.AccessKeyID, p.SecretAccessKey, "")
	}

	p.EC2 = ec2.New(session.New(&aws.Config{Credentials: p.Credentials, Region: &p.Region}))
	resp, err := p.EC2.DescribeVolumes(&ec2.DescribeVolumesInput{
		Filters: []*ec2.Filter{
			&ec2.Filter{
				Name: aws.String("attachment.instance-id"),
				Values: []*string{
					&p.InstanceID,
				},
			},
		},
	})
	if err != nil {
		return err
	}
	if resp.NextToken != nil {
		return errors.New("DescribeVolumes response has NextToken")
	}

	p.Volumes = &resp.Volumes
	if len(*p.Volumes) == 0 {
		return errors.New("DescribeVolumes response has no volumes")
	}

	return nil
}

func (p EBSPlugin) getLastPoint(vol *ec2.Volume, metricName string, statType string) (float64, int, error) {
	now := time.Now()

	period := metricPeriodDefault
	if tmp, ok := metricPeriodByVolumeType[*vol.VolumeType]; ok {
		period = tmp
	}
	start := now.Add(time.Duration(period) * 3 * time.Second * -1)

	resp, err := p.CloudWatch.GetMetricStatistics(&cloudwatch.GetMetricStatisticsInput{
		Dimensions: []*cloudwatch.Dimension{
			&cloudwatch.Dimension{
				Name:  aws.String("VolumeId"),
				Value: vol.VolumeId,
			},
		},
		StartTime:  &start,
		EndTime:    &now,
		MetricName: &metricName,
		Period:     aws.Int64(60),
		Statistics: []*string{&statType},
		Namespace:  aws.String("AWS/EBS"),
	})
	if err != nil {
		return 0, 0, err
	}

	datapoints := resp.Datapoints
	if len(datapoints) == 0 {
		return 0, 0, errors.New("fetched no datapoints")
	}

	latest := time.Unix(0, 0)
	var latestVal float64
	for _, dp := range datapoints {
		if dp.Timestamp.Before(latest) {
			continue
		}

		latest = *dp.Timestamp
		switch statType {
		case "Average":
			latestVal = *dp.Average
		case "Sum":
			latestVal = *dp.Sum
		}
	}

	return latestVal, period, nil
}

// FetchMetrics fetch the metrics
func (p EBSPlugin) FetchMetrics() (map[string]float64, error) {
	stat := make(map[string]float64)
	p.CloudWatch = cloudwatch.New(session.New(&aws.Config{Credentials: p.Credentials, Region: &p.Region}))

	for _, vol := range *p.Volumes {
		graphs := graphsToProcess(vol.VolumeType)
		for _, graph := range *graphs {
			for _, met := range graphdef[graph].Metrics {
				cwdef := cloudwatchdef[met.Name]
				val, period, err := p.getLastPoint(vol, cwdef.MetricName, cwdef.Statistics)
				skey := fmt.Sprintf(met.Name, *vol.VolumeId)

				if err != nil {
					retErr := errors.New(*vol.VolumeId + " " + err.Error() + ":" + cwdef.MetricName)
					if err.Error() == "fetched no datapoints" {
						getStderrLogger().Println(retErr)
					} else {
						return nil, retErr
					}
				} else {
					stat[skey] = cwdef.CalcFunc(val, float64(period))
				}
			}
		}
	}

	return stat, nil
}

// GraphDefinition for plugin
func (p EBSPlugin) GraphDefinition() map[string](mp.Graphs) {
	graphMetrics := map[string]([]mp.Metrics){}
	for _, vol := range *p.Volumes {
		graphs := graphsToProcess(vol.VolumeType)

		for _, graph := range *graphs {
			if _, ok := graphMetrics[graph]; !ok {
				graphMetrics[graph] = []mp.Metrics{}
			}

			for _, metric := range graphdef[graph].Metrics {
				m := mp.Metrics{
					Name:  fmt.Sprintf(metric.Name, *vol.VolumeId),
					Label: fmt.Sprintf(metric.Label, *vol.VolumeId+":"+*vol.Attachments[0].Device),
					Diff:  metric.Diff,
				}
				graphMetrics[graph] = append(graphMetrics[graph], m)
			}
		}
	}

	for k := range graphdef {
		graphdef[k] = mp.Graphs{
			Label:   graphdef[k].Label,
			Unit:    graphdef[k].Unit,
			Metrics: graphMetrics[k],
		}
	}

	return graphdef
}

func stringInSlice(a string, list []string) bool {
	for _, b := range list {
		if b == a {
			return true
		}
	}
	return false
}

func graphsToProcess(volumeType *string) *[]string {
	if stringInSlice(*volumeType, volumeTypesHavingExtraMetrics) {
		var graphsWithExtra = append(defaultGraphs, extraGraphs...)
		return &graphsWithExtra
	}
	return &defaultGraphs
}

func getStderrLogger() *log.Logger {
	if stderrLogger == nil {
		stderrLogger = log.New(os.Stderr, "", log.LstdFlags)
	}
	return stderrLogger
}

func main() {
	optRegion := flag.String("region", "", "AWS Region")
	optInstanceID := flag.String("instance-id", "", "Instance ID")
	optAccessKeyID := flag.String("access-key-id", "", "AWS Access Key ID")
	optSecretAccessKey := flag.String("secret-access-key", "", "AWS Secret Access Key")
	optTempfile := flag.String("tempfile", "", "Temp file name")
	flag.Parse()

	var ebs EBSPlugin

	ebs.Region = *optRegion
	ebs.InstanceID = *optInstanceID

	// get metadata in ec2 instance
	ec2MC := ec2metadata.New(session.New())
	if *optRegion == "" {
		ebs.Region, _ = ec2MC.Region()
	}
	if *optInstanceID == "" {
		ebs.InstanceID, _ = ec2MC.GetMetadata("instance-id")
	}

	ebs.AccessKeyID = *optAccessKeyID
	ebs.SecretAccessKey = *optSecretAccessKey

	if err := ebs.prepare(); err != nil {
		log.Fatalln(err)
	}

	helper := mp.NewMackerelPlugin(ebs)
	if *optTempfile != "" {
		helper.Tempfile = *optTempfile
	} else {
		helper.Tempfile = "/tmp/mackerel-plugin-ebs"
	}

	if os.Getenv("MACKEREL_AGENT_PLUGIN_META") != "" {
		helper.OutputDefinitions()
	} else {
		helper.OutputValues()
	}
}
<commit_msg>aws-ec2-ebs: use wildcard in the graph definitions<commit_after>package main

import (
	"errors"
	"flag"
	"log"
	"os"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/ec2metadata"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudwatch"
	"github.com/aws/aws-sdk-go/service/ec2"
	mp "github.com/mackerelio/go-mackerel-plugin-helper"
)

var metricPeriodDefault = 300
var metricPeriodByVolumeType = map[string]int{
	"io1": 60,
}

var defaultGraphs = []string{
	"ec2.ebs.bandwidth.#",
	"ec2.ebs.throughput.#",
	"ec2.ebs.size_per_op.#",
	"ec2.ebs.latency.#",
	"ec2.ebs.queue_length.#",
	"ec2.ebs.idle_time.#",
}

var allGraphs = append([]string{
	"ec2.ebs.throughput_delivered.#",
	"ec2.ebs.consumed_ops.#",
}, defaultGraphs...)

type cloudWatchSetting struct {
	MetricName string
	Statistics string
	CalcFunc   func(float64, float64) float64
}

// http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/monitoring-volume-status.html
var cloudwatchdefs = map[string](cloudWatchSetting){
	"ec2.ebs.bandwidth.#.read": cloudWatchSetting{
		MetricName: "VolumeReadBytes", Statistics: "Sum",
		CalcFunc: func(val float64, period float64) float64 { return val / period },
	},
	"ec2.ebs.bandwidth.#.write": cloudWatchSetting{
		MetricName: "VolumeWriteBytes", Statistics: "Sum",
		CalcFunc: func(val float64, period float64) float64 { return val / period },
	},
	"ec2.ebs.throughput.#.read": cloudWatchSetting{
		MetricName: "VolumeReadOps", Statistics: "Sum",
		CalcFunc: func(val float64, period float64) float64 { return val / period },
	},
	"ec2.ebs.throughput.#.write": cloudWatchSetting{
		MetricName: "VolumeWriteOps", Statistics: "Sum",
		CalcFunc: func(val float64, period float64) float64 { return val / period },
	},
	"ec2.ebs.size_per_op.#.read": cloudWatchSetting{
		MetricName: "VolumeReadBytes", Statistics: "Average",
		CalcFunc: func(val float64, period float64) float64 { return val },
	},
	"ec2.ebs.size_per_op.#.write": cloudWatchSetting{
		MetricName: "VolumeWriteBytes", Statistics: "Average",
		CalcFunc: func(val float64, period float64) float64 { return val },
	},
	"ec2.ebs.latency.#.read": cloudWatchSetting{
		MetricName: "VolumeTotalReadTime", Statistics: "Average",
		CalcFunc: func(val float64, period float64) float64 { return val * 1000 },
	},
	"ec2.ebs.latency.#.write": cloudWatchSetting{
		MetricName: "VolumeTotalWriteTime", Statistics: "Average",
		CalcFunc: func(val float64, period float64) float64 { return val * 1000 },
	},
	"ec2.ebs.queue_length.#.queue_length": cloudWatchSetting{
		MetricName: "VolumeQueueLength", Statistics: "Average",
		CalcFunc: func(val float64, period float64) float64 { return val },
	},
	"ec2.ebs.idle_time.#.idle_time": cloudWatchSetting{
		MetricName: "VolumeIdleTime", Statistics: "Sum",
		CalcFunc: func(val float64, period float64) float64 { return val / period * 100 },
	},
	"ec2.ebs.throughput_delivered.#.throughput_delivered": cloudWatchSetting{
		MetricName: "VolumeThroughputPercentage", Statistics: "Average",
		CalcFunc: func(val float64, period float64) float64 { return val },
	},
	"ec2.ebs.consumed_ops.#.consumed_ops": cloudWatchSetting{
		MetricName: "VolumeConsumedReadWriteOps", Statistics: "Sum",
		CalcFunc: func(val float64, period float64) float64 { return val },
	},
}

var graphdef = map[string](mp.Graphs){
	"ec2.ebs.bandwidth.#": mp.Graphs{
		Label: "EBS Bandwidth",
		Unit:  "bytes/sec",
		Metrics: [](mp.Metrics){
			mp.Metrics{Name: "read", Label: "Read", Diff: false},
			mp.Metrics{Name: "write", Label: "Write", Diff: false},
		},
	},
	"ec2.ebs.throughput.#": mp.Graphs{
		Label: "EBS Throughput (op/s)",
		Unit:  "iops",
		Metrics: [](mp.Metrics){
			mp.Metrics{Name: "read", Label: "Read", Diff: false},
			mp.Metrics{Name: "write", Label: "Write", Diff: false},
		},
	},
	"ec2.ebs.size_per_op.#": mp.Graphs{
		Label: "EBS Avg Op Size (Bytes/op)",
		Unit:  "bytes",
		Metrics: [](mp.Metrics){
			mp.Metrics{Name: "read", Label: "Read", Diff: false},
			mp.Metrics{Name: "write", Label: "Write", Diff: false},
		},
	},
	"ec2.ebs.latency.#": mp.Graphs{
		Label: "EBS Avg Latency (ms/op)",
		Unit:  "float",
		Metrics: [](mp.Metrics){
			mp.Metrics{Name: "read", Label: "Read", Diff: false},
			mp.Metrics{Name: "write", Label: "Write", Diff: false},
		},
	},
	"ec2.ebs.queue_length.#": mp.Graphs{
		Label: "EBS Avg Queue Length (ops)",
		Unit:  "float",
		Metrics: [](mp.Metrics){
			mp.Metrics{Name: "queue_length", Label: "Queue Length", Diff: false},
		},
	},
	"ec2.ebs.idle_time.#": mp.Graphs{
		Label: "EBS Time Spent Idle",
		Unit:  "percentage",
		Metrics: [](mp.Metrics){
			mp.Metrics{Name: "idle_time", Label: "Idle Time", Diff: false},
		},
	},
	"ec2.ebs.throughput_delivered.#": mp.Graphs{
		Label: "EBS Throughput of Provisioned IOPS",
		Unit:  "percentage",
		Metrics: [](mp.Metrics){
			mp.Metrics{Name: "throughput_delivered", Label: "Throughput", Diff: false},
		},
	},
	"ec2.ebs.consumed_ops.#": mp.Graphs{
		Label: "EBS Consumed Ops of Provisioned IOPS",
		Unit:  "float",
		Metrics: [](mp.Metrics){
			mp.Metrics{Name: "consumed_ops", Label: "Consumed Ops", Diff: false},
		},
	},
}

var stderrLogger *log.Logger

// EBSPlugin mackerel plugin for ebs
type EBSPlugin struct {
	Region          string
	AccessKeyID     string
	SecretAccessKey string
	InstanceID      string
	Credentials     *credentials.Credentials
	EC2             *ec2.EC2
	CloudWatch      *cloudwatch.CloudWatch
	Volumes         []*ec2.Volume
}

func (p *EBSPlugin) prepare() error {
	if p.AccessKeyID != "" && p.SecretAccessKey != "" {
		p.Credentials = credentials.NewStaticCredentials(p.AccessKeyID, p.SecretAccessKey, "")
	}

	p.EC2 = ec2.New(session.New(&aws.Config{Credentials: p.Credentials, Region: &p.Region}))
	resp, err := p.EC2.DescribeVolumes(&ec2.DescribeVolumesInput{
		Filters: []*ec2.Filter{
			&ec2.Filter{
				Name: aws.String("attachment.instance-id"),
				Values: []*string{
					&p.InstanceID,
				},
			},
		},
	})
	if err != nil {
		return err
	}
	if resp.NextToken != nil {
		return errors.New("DescribeVolumes response has NextToken")
	}

	p.Volumes = resp.Volumes
	if len(p.Volumes) == 0 {
		return errors.New("DescribeVolumes response has no volumes")
	}

	return nil
}

func (p EBSPlugin) getLastPoint(vol *ec2.Volume, metricName string, statType string) (float64, int, error) {
	now := time.Now()

	period := metricPeriodDefault
	if tmp, ok := metricPeriodByVolumeType[*vol.VolumeType]; ok {
		period = tmp
	}
	start := now.Add(time.Duration(period) * 3 * time.Second * -1)

	resp, err := p.CloudWatch.GetMetricStatistics(&cloudwatch.GetMetricStatisticsInput{
		Dimensions: []*cloudwatch.Dimension{
			&cloudwatch.Dimension{
				Name:  aws.String("VolumeId"),
				Value: vol.VolumeId,
			},
		},
		StartTime:  &start,
		EndTime:    &now,
		MetricName: &metricName,
		Period:     aws.Int64(60),
		Statistics: []*string{&statType},
		Namespace:  aws.String("AWS/EBS"),
	})
	if err != nil {
		return 0, 0, err
	}

	datapoints := resp.Datapoints
	if len(datapoints) == 0 {
		return 0, 0, errors.New("fetched no datapoints")
	}

	latest := time.Unix(0, 0)
	var latestVal float64
	for _, dp := range datapoints {
		if dp.Timestamp.Before(latest) {
			continue
		}

		latest = *dp.Timestamp
		switch statType {
		case "Average":
			latestVal = *dp.Average
		case "Sum":
			latestVal = *dp.Sum
		}
	}

	return latestVal, period, nil
}

// FetchMetrics fetch the metrics
func (p EBSPlugin) FetchMetrics() (map[string]interface{}, error) {
	stat := make(map[string]interface{})
	p.CloudWatch = cloudwatch.New(session.New(&aws.Config{Credentials: p.Credentials, Region: &p.Region}))
	for _, vol := range p.Volumes {
		volumeID := normalizeVolumeID(*vol.VolumeId)
		graphs := defaultGraphs
		if *vol.VolumeType == "io1" {
			graphs = allGraphs
		}
		for _, graphName := range graphs {
			for _, metric := range graphdef[graphName].Metrics {
				metricKey := graphName + "." + metric.Name
				cloudwatchdef := cloudwatchdefs[metricKey]
				val, period, err := p.getLastPoint(vol, cloudwatchdef.MetricName, cloudwatchdef.Statistics)
				if err != nil {
					retErr := errors.New(volumeID + " " + err.Error() + ":" + cloudwatchdef.MetricName)
					if err.Error() == "fetched no datapoints" {
						getStderrLogger().Println(retErr)
					} else {
						return nil, retErr
					}
				} else {
					stat[strings.Replace(metricKey, "#", volumeID, -1)] = cloudwatchdef.CalcFunc(val, float64(period))
				}
			}
		}
	}
	return stat, nil
}

// GraphDefinition for plugin
func (p EBSPlugin) GraphDefinition() map[string](mp.Graphs) {
	return graphdef
}

func getStderrLogger() *log.Logger {
	if stderrLogger == nil {
		stderrLogger = log.New(os.Stderr, "", log.LstdFlags)
	}
	return stderrLogger
}

func normalizeVolumeID(volumeID string) string {
	return strings.Replace(volumeID, ".", "_", -1)
}

func main() {
	optRegion := flag.String("region", "", "AWS Region")
	optInstanceID := flag.String("instance-id", "", "Instance ID")
	optAccessKeyID := flag.String("access-key-id", "", "AWS Access Key ID")
	optSecretAccessKey := flag.String("secret-access-key", "", "AWS Secret Access Key")
	optTempfile := flag.String("tempfile", "", "Temp file name")
	flag.Parse()

	var ebs EBSPlugin

	ebs.Region = *optRegion
	ebs.InstanceID = *optInstanceID

	// get metadata in ec2 instance
	ec2MC := ec2metadata.New(session.New())
	if *optRegion == "" {
		ebs.Region, _ = ec2MC.Region()
	}
	if *optInstanceID == "" {
		ebs.InstanceID, _ = ec2MC.GetMetadata("instance-id")
	}

	ebs.AccessKeyID = *optAccessKeyID
	ebs.SecretAccessKey = *optSecretAccessKey

	if err := ebs.prepare(); err != nil {
		log.Fatalln(err)
	}

	helper := mp.NewMackerelPlugin(ebs)
	if *optTempfile != "" {
		helper.Tempfile = *optTempfile
	} else {
		helper.Tempfile = "/tmp/mackerel-plugin-ebs"
	}

	helper.Run()
}
<|endoftext|>


<commit_before>package alerts

import (
	log "github.com/Sirupsen/logrus"
	constants "github.com/megamsys/libgo/utils"
	mailgun "github.com/mailgun/mailgun-go"
	"strings"
)

const (
	LAUNCHED EventAction = iota
	DESTROYED
	STATUS
	DEDUCT
	ONBOARD
	RESET
	INVITE
	BALANCE
	INVOICE
	TRANSACTION
	DESCRIPTION
)

type Notifier interface {
	Notify(eva EventAction, edata EventData) error
	satisfied(eva EventAction) bool
}

// Extra information about an event.
type EventData struct {
	M map[string]string
	D []string
}

type EventAction int

func (v *EventAction) String() string {
	switch *v {
	case LAUNCHED:
		return "launched"
	case DESTROYED:
		return "destroyed"
	case STATUS:
		return "status"
	case DEDUCT:
		return "deduct"
	case ONBOARD:
		return "onboard"
	case RESET:
		return "reset"
	case INVITE:
		return "invite"
	case BALANCE:
		return "balance"
	case DESCRIPTION:
		return "description"
	default:
		return "arrgh"
	}
}

type mailgunner struct {
	api_key string
	domain  string
	nilavu  string
	logo    string
	home    string
	dir     string
}

func NewMailgun(m map[string]string, n map[string]string) Notifier {
	return &mailgunner{
		api_key: m[constants.API_KEY],
		domain:  m[constants.DOMAIN],
		nilavu:  m[constants.NILAVU],
		logo:    m[constants.LOGO],
		home:    n[constants.HOME],
		dir:     n[constants.DIR],
	}
}

func (m *mailgunner) satisfied(eva EventAction) bool {
	if eva == STATUS {
		return false
	}
	return true
}

/*{
		"email":     "nkishore@megam.io",
		"logo":      "vertice.png",
		"nilavu":    "console.megam.io",
		"appname": "vertice.megambox.com"
		"type": "torpedo"
		"token": "9090909090",
		"days":      "20",
		"cost":      "$12",
}*/
func (m *mailgunner) Notify(eva EventAction, edata EventData) error {
	if !m.satisfied(eva) {
		return nil
	}
	edata.M[constants.NILAVU] = m.nilavu
	edata.M[constants.LOGO] = m.logo

	bdy, err := body(eva.String(), edata.M, m.dir)
	if err != nil {
		return err
	}
	m.Send(bdy, "", subject(eva), edata.M[constants.EMAIL])
	return nil
}

func (m *mailgunner) Send(msg string, sender string, subject string, to string) error {
	if len(strings.TrimSpace(sender)) <= 0 {
		sender = "Kishore CEO <nkishore@megam.io>"
	}
	mg := mailgun.NewMailgun(m.domain, m.api_key, "")
	g := mailgun.NewMessage(
		sender,
		subject,
		"You are in !",
		to,
	)
	g.SetHtml(msg)
	g.SetTracking(false)
	//g.SetTrackingClicks(false)
	//g.SetTrackingOpens(false)
	_, id, err := mg.Send(g)
	if err != nil {
		return err
	}
	log.Infof("Mailgun sent %s", id)
	return nil
}

func subject(eva EventAction) string {
	var sub string
	switch eva {
	case ONBOARD:
		sub = "Ahoy. Welcome aboard!"
	case RESET:
		sub = "You have fat finger.!"
	case INVITE:
		sub = "Lets party!"
	case BALANCE:
		sub = "Piggy bank!"
	case LAUNCHED:
		sub = "Up!"
	case DESTROYED:
		sub = "Nuked"
	default:
		break
	}
	return sub
}
<commit_msg>commit (#59)<commit_after>package alerts

import (
	log "github.com/Sirupsen/logrus"
	constants "github.com/megamsys/libgo/utils"
	mailgun "github.com/mailgun/mailgun-go"
	"strings"
)

const (
	LAUNCHED EventAction = iota
	DESTROYED
	STATUS
	DEDUCT
	ONBOARD
	RESET
	INVITE
	BALANCE
	INVOICE
	TRANSACTION
	DESCRIPTION
)

type Notifier interface {
	Notify(eva EventAction, edata EventData) error
	satisfied(eva EventAction) bool
}

// Extra information about an event.
type EventData struct {
	M map[string]string
	D []string
}

type EventAction int

func (v *EventAction) String() string {
	switch *v {
	case LAUNCHED:
		return "launched"
	case DESTROYED:
		return "destroyed"
	case STATUS:
		return "status"
	case DEDUCT:
		return "deduct"
	case ONBOARD:
		return "onboard"
	case RESET:
		return "reset"
	case INVITE:
		return "invite"
	case BALANCE:
		return "balance"
	case DESCRIPTION:
		return "description"
	default:
		return "arrgh"
	}
}

type mailgunner struct {
	api_key string
	domain  string
	nilavu  string
	logo    string
	home    string
	dir     string
}

func NewMailgun(m map[string]string, n map[string]string) Notifier {
	return &mailgunner{
		api_key: m[constants.API_KEY],
		domain:  m[constants.DOMAIN],
		nilavu:  m[constants.NILAVU],
		logo:    m[constants.LOGO],
		home:    n[constants.HOME],
		dir:     n[constants.DIR],
	}
}

func (m *mailgunner) satisfied(eva EventAction) bool {
	if eva == STATUS {
		return false
	}
	return true
}

/*{
		"email":     "nkishore@megam.io",
		"logo":      "vertice.png",
		"nilavu":    "console.megam.io",
		"appname": "vertice.megambox.com"
		"type": "torpedo"
		"token": "9090909090",
		"days":      "20",
		"cost":      "$12",
}*/
func (m *mailgunner) Notify(eva EventAction, edata EventData) error {
	if !m.satisfied(eva) {
		return nil
	}
	edata.M[constants.NILAVU] = m.nilavu
	edata.M[constants.LOGO] = m.logo

	bdy, err := body(eva.String(), edata.M, m.dir)
	if err != nil {
		return err
	}
	m.Send(bdy, "", subject(eva), edata.M[constants.EMAIL])
	return nil
}

func (m *mailgunner) Send(msg string, sender string, subject string, to string) error {
	if len(strings.TrimSpace(sender)) <= 0 {
		sender = "OYA <support@megamafrica.com>"
	}
	mg := mailgun.NewMailgun(m.domain, m.api_key, "")
	g := mailgun.NewMessage(
		sender,
		subject,
		"You are in !",
		to,
	)
	g.SetHtml(msg)
	g.SetTracking(false)
	//g.SetTrackingClicks(false)
	//g.SetTrackingOpens(false)
	_, id, err := mg.Send(g)
	if err != nil {
		return err
	}
	log.Infof("Mailgun sent %s", id)
	return nil
}

func subject(eva EventAction) string {
	var sub string
	switch eva {
	case ONBOARD:
		sub = "Ahoy. Welcome aboard!"
	case RESET:
		sub = "You have fat finger.!"
	case INVITE:
		sub = "Lets party!"
	case BALANCE:
		sub = "Piggy bank!"
	case LAUNCHED:
		sub = "Up!"
	case DESTROYED:
		sub = "Nuked"
	default:
		break
	}
	return sub
}
<|endoftext|>


<commit_before>package api

// ServerEnvironment represents the read-only environment fields of a LXD server
type ServerEnvironment struct {
	Addresses              []string `json:"addresses" yaml:"addresses"`
	Architectures          []string `json:"architectures" yaml:"architectures"`
	Certificate            string   `json:"certificate" yaml:"certificate"`
	CertificateFingerprint string   `json:"certificate_fingerprint" yaml:"certificate_fingerprint"`
	Driver                 string   `json:"driver" yaml:"driver"`
	DriverVersion          string   `json:"driver_version" yaml:"driver_version"`
	Kernel                 string   `json:"kernel" yaml:"kernel"`
	KernelArchitecture     string   `json:"kernel_architecture" yaml:"kernel_architecture"`
	KernelVersion          string   `json:"kernel_version" yaml:"kernel_version"`
	Server                 string   `json:"server" yaml:"server"`
	ServerPid              int      `json:"server_pid" yaml:"server_pid"`
	ServerVersion          string   `json:"server_version" yaml:"server_version"`
	Storage                string   `json:"storage" yaml:"storage"`
	StorageVersion         string   `json:"storage_version" yaml:"storage_version"`

	// API extension: clustering
	ServerClustered bool   `json:"server_clustered" yaml:"server_clustered"`
	ServerName      string `json:"server_name" yaml:"server_name"`

	// API extension: projects
	Project string `json:"project" yaml:"project"`

	// API extension: kernel_features
	KernelFeatures map[string]string `json:"kernel_features" yaml:"kernel_features"`
}

// ServerPut represents the modifiable fields of a LXD server configuration
type ServerPut struct {
	Config map[string]interface{} `json:"config" yaml:"config"`
}

// ServerUntrusted represents a LXD server for an untrusted client
type ServerUntrusted struct {
	APIExtensions []string `json:"api_extensions" yaml:"api_extensions"`
	APIStatus     string   `json:"api_status" yaml:"api_status"`
	APIVersion    string   `json:"api_version" yaml:"api_version"`
	Auth          string   `json:"auth" yaml:"auth"`
	Public        bool     `json:"public" yaml:"public"`

	// API extension: macaroon_authentication
	AuthMethods []string `json:"auth_methods" yaml:"auth_methods"`
}

// Server represents a LXD server
type Server struct {
	ServerPut       `yaml:",inline"`
	ServerUntrusted `yaml:",inline"`

	Environment ServerEnvironment `json:"environment" yaml:"environment"`
}

// Writable converts a full Server struct into a ServerPut struct (filters read-only fields)
func (srv *Server) Writable() ServerPut {
	return srv.ServerPut
}
<commit_msg>shared/api: Sort ServerEnvironment struct<commit_after>package api

// ServerEnvironment represents the read-only environment fields of a LXD server
type ServerEnvironment struct {
	Addresses              []string `json:"addresses" yaml:"addresses"`
	Architectures          []string `json:"architectures" yaml:"architectures"`
	Certificate            string   `json:"certificate" yaml:"certificate"`
	CertificateFingerprint string   `json:"certificate_fingerprint" yaml:"certificate_fingerprint"`
	Driver                 string   `json:"driver" yaml:"driver"`
	DriverVersion          string   `json:"driver_version" yaml:"driver_version"`
	Kernel                 string   `json:"kernel" yaml:"kernel"`
	KernelArchitecture     string   `json:"kernel_architecture" yaml:"kernel_architecture"`

	// API extension: kernel_features
	KernelFeatures map[string]string `json:"kernel_features" yaml:"kernel_features"`

	KernelVersion string `json:"kernel_version" yaml:"kernel_version"`

	// API extension: projects
	Project string `json:"project" yaml:"project"`

	Server string `json:"server" yaml:"server"`

	// API extension: clustering
	ServerClustered bool   `json:"server_clustered" yaml:"server_clustered"`
	ServerName      string `json:"server_name" yaml:"server_name"`

	ServerPid      int    `json:"server_pid" yaml:"server_pid"`
	ServerVersion  string `json:"server_version" yaml:"server_version"`
	Storage        string `json:"storage" yaml:"storage"`
	StorageVersion string `json:"storage_version" yaml:"storage_version"`
}

// ServerPut represents the modifiable fields of a LXD server configuration
type ServerPut struct {
	Config map[string]interface{} `json:"config" yaml:"config"`
}

// ServerUntrusted represents a LXD server for an untrusted client
type ServerUntrusted struct {
	APIExtensions []string `json:"api_extensions" yaml:"api_extensions"`
	APIStatus     string   `json:"api_status" yaml:"api_status"`
	APIVersion    string   `json:"api_version" yaml:"api_version"`
	Auth          string   `json:"auth" yaml:"auth"`
	Public        bool     `json:"public" yaml:"public"`

	// API extension: macaroon_authentication
	AuthMethods []string `json:"auth_methods" yaml:"auth_methods"`
}

// Server represents a LXD server
type Server struct {
	ServerPut       `yaml:",inline"`
	ServerUntrusted `yaml:",inline"`

	Environment ServerEnvironment `json:"environment" yaml:"environment"`
}

// Writable converts a full Server struct into a ServerPut struct (filters read-only fields)
func (srv *Server) Writable() ServerPut {
	return srv.ServerPut
}
<|endoftext|>