<commit_before>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package filters

import (
	"crypto/tls"
	"io"
	"math/rand"
	"net"
	"net/http"
	"net/http/httptest"
	"testing"
	"time"

	"golang.org/x/net/http2"
)

func TestProbabilisticGoawayDecider(t *testing.T) {
	cases := []struct {
		name         string
		chance       float64
		nextFn       func(chance float64) func() float64
		expectGOAWAY bool
	}{
		{
			name:   "always not GOAWAY",
			chance: 0,
			nextFn: func(chance float64) func() float64 {
				return rand.Float64
			},
			expectGOAWAY: false,
		},
		{
			name:   "always GOAWAY",
			chance: 1,
			nextFn: func(chance float64) func() float64 {
				return rand.Float64
			},
			expectGOAWAY: true,
		},
		{
			name:   "hit GOAWAY",
			chance: rand.Float64() + 0.01,
			nextFn: func(chance float64) func() float64 {
				return func() float64 {
					return chance - 0.001
				}
			},
			expectGOAWAY: true,
		},
		{
			name:   "does not hit GOAWAY",
			chance: rand.Float64() + 0.01,
			nextFn: func(chance float64) func() float64 {
				return func() float64 {
					return chance + 0.001
				}
			},
			expectGOAWAY: false,
		},
	}

	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			d := probabilisticGoawayDecider{chance: tc.chance, next: tc.nextFn(tc.chance)}
			result := d.Goaway(nil)
			if result != tc.expectGOAWAY {
				t.Errorf("expect GOAWAY: %v, got: %v", tc.expectGOAWAY, result)
			}
		})
	}
}

// TestClientReceivedGOAWAY tests the in-flight watch requests will not be affected and new requests use a
// connection after client received GOAWAY, and server response watch request with GOAWAY will not break client
// watching body read.
func TestClientReceivedGOAWAY(t *testing.T) {
	const (
		urlNormal          = "/normal"
		urlWatch           = "/watch"
		urlGoaway          = "/goaway"
		urlWatchWithGoaway = "/watch-with-goaway"
	)

	const (
		// indicate the bytes watch request will be sent
		// used to check if watch request was broke by GOAWAY
		watchExpectSendBytes = 5
	)

	watchHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		timer := time.NewTicker(time.Second)

		w.Header().Set("Transfer-Encoding", "chunked")
		w.WriteHeader(200)

		flusher, _ := w.(http.Flusher)
		flusher.Flush()

		count := 0
		for {
			select {
			case <-timer.C:
				n, err := w.Write([]byte("w"))
				if err != nil {
					return
				}
				flusher.Flush()
				count += n
				if count == watchExpectSendBytes {
					return
				}
			}
		}
	})

	mux := http.NewServeMux()
	mux.Handle(urlNormal, WithProbabilisticGoaway(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(http.StatusOK)
		w.Write([]byte("hello"))
		return
	}), 0))
	mux.Handle(urlWatch, WithProbabilisticGoaway(watchHandler, 0))
	mux.Handle(urlGoaway, WithProbabilisticGoaway(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(http.StatusOK)
		w.Write([]byte("hello"))
		return
	}), 1))
	mux.Handle(urlWatchWithGoaway, WithProbabilisticGoaway(watchHandler, 1))

	s := httptest.NewUnstartedServer(mux)

	http2Options := &http2.Server{}

	if err := http2.ConfigureServer(s.Config, http2Options); err != nil {
		t.Fatalf("failed to configure test server to be HTTP2 server, err: %v", err)
	}

	s.TLS = s.Config.TLSConfig
	s.StartTLS()
	defer s.Close()

	tlsConfig := &tls.Config{
		InsecureSkipVerify: true,
		NextProtos:         []string{http2.NextProtoTLS},
	}

	cases := []struct {
		name string
		reqs []string
		// expectConnections always equals to GOAWAY requests(urlGoaway or urlWatchWithGoaway) + 1
		expectConnections int
	}{
		{
			name:              "all normal requests use only one connection",
			reqs:              []string{urlNormal, urlNormal, urlNormal},
			expectConnections: 1,
		},
		{
			name:              "got GOAWAY after set-up watch",
			reqs:              []string{urlNormal, urlWatch, urlGoaway, urlNormal, urlNormal},
			expectConnections: 2,
		},
		{
			name:              "got GOAWAY after set-up watch, and set-up a new watch",
			reqs:              []string{urlNormal, urlWatch, urlGoaway, urlWatch, urlNormal, urlNormal},
			expectConnections: 2,
		},
		{
			name:              "got 2 GOAWAY after set-up watch",
			reqs:              []string{urlNormal, urlWatch, urlGoaway, urlGoaway, urlNormal, urlNormal},
			expectConnections: 3,
		},
		{
			name:              "combine with watch-with-goaway",
			reqs:              []string{urlNormal, urlWatchWithGoaway, urlNormal, urlWatch, urlGoaway, urlNormal, urlNormal},
			expectConnections: 3,
		},
	}

	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			// localAddr indicates how many TCP connection set up
			localAddr := make([]string, 0)

			// init HTTP2 client
			client := http.Client{
				Transport: &http2.Transport{
					TLSClientConfig: tlsConfig,
					DialTLS: func(network, addr string, cfg *tls.Config) (conn net.Conn, err error) {
						conn, err = tls.Dial(network, addr, cfg)
						if err != nil {
							t.Fatalf("unexpect connection err: %v", err)
						}
						localAddr = append(localAddr, conn.LocalAddr().String())
						return
					},
				},
			}

			watchChs := make([]chan int, 0)
			for _, url := range tc.reqs {
				req, err := http.NewRequest(http.MethodGet, s.URL+url, nil)
				if err != nil {
					t.Fatalf("unexpect new request error: %v", err)
				}
				resp, err := client.Do(req)
				if err != nil {
					t.Fatalf("failed request test server, err: %v", err)
				}

				// encounter watch bytes received, does not expect to be broken
				if url == urlWatch || url == urlWatchWithGoaway {
					ch := make(chan int)
					watchChs = append(watchChs, ch)
					go func() {
						count := 0
						for {
							buffer := make([]byte, 1)
							n, err := resp.Body.Read(buffer)
							if err != nil {
								// urlWatch will receive io.EOF,
								// urlWatchWithGoaway will receive http2.GoAwayError
								if err != io.EOF {
									if _, ok := err.(http2.GoAwayError); !ok {
										t.Errorf("watch received not EOF err: %v", err)
									}
								}
								ch <- count
								return
							}
							count += n
						}
					}()
				}
			}

			// check TCP connection count
			if tc.expectConnections != len(localAddr) {
				t.Fatalf("expect TCP connection: %d, actual: %d", tc.expectConnections, len(localAddr))
			}

			// check if watch request is broken by GOAWAY response
			watchTimeout := time.NewTimer(time.Second * 10)
			for _, watchCh := range watchChs {
				select {
				case n := <-watchCh:
					if n != watchExpectSendBytes {
						t.Fatalf("in-flight watch was broken by GOAWAY response, expect go bytes: %d, actual got: %d", watchExpectSendBytes, n)
					}
				case <-watchTimeout.C:
					t.Error("watch receive timeout")
				}
			}
		})
	}
}

func TestHTTP1Requests(t *testing.T) {
	s := httptest.NewUnstartedServer(WithProbabilisticGoaway(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(http.StatusOK)
		w.Write([]byte("hello"))
		return
	}), 1))

	http2Options := &http2.Server{}

	if err := http2.ConfigureServer(s.Config, http2Options); err != nil {
		t.Fatalf("failed to configure test server to be HTTP2 server, err: %v", err)
	}

	s.TLS = s.Config.TLSConfig
	s.StartTLS()
	defer s.Close()

	tlsConfig := &tls.Config{
		InsecureSkipVerify: true,
		NextProtos:         []string{"http/1.1"},
	}

	client := http.Client{
		Transport: &http.Transport{
			TLSClientConfig: tlsConfig,
		},
	}

	resp, err := client.Get(s.URL)
	if err != nil {
		t.Fatalf("failed to request the server, err: %v", err)
	}

	if v := resp.Header.Get("Connection"); v != "" {
		t.Errorf("expect response HTTP header Connection to be empty, but got: %s", v)
	}
}
<commit_msg>fix S1000 simplify ch switch cases<commit_after>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package filters

import (
	"crypto/tls"
	"io"
	"math/rand"
	"net"
	"net/http"
	"net/http/httptest"
	"testing"
	"time"

	"golang.org/x/net/http2"
)

func TestProbabilisticGoawayDecider(t *testing.T) {
	cases := []struct {
		name         string
		chance       float64
		nextFn       func(chance float64) func() float64
		expectGOAWAY bool
	}{
		{
			name:   "always not GOAWAY",
			chance: 0,
			nextFn: func(chance float64) func() float64 {
				return rand.Float64
			},
			expectGOAWAY: false,
		},
		{
			name:   "always GOAWAY",
			chance: 1,
			nextFn: func(chance float64) func() float64 {
				return rand.Float64
			},
			expectGOAWAY: true,
		},
		{
			name:   "hit GOAWAY",
			chance: rand.Float64() + 0.01,
			nextFn: func(chance float64) func() float64 {
				return func() float64 {
					return chance - 0.001
				}
			},
			expectGOAWAY: true,
		},
		{
			name:   "does not hit GOAWAY",
			chance: rand.Float64() + 0.01,
			nextFn: func(chance float64) func() float64 {
				return func() float64 {
					return chance + 0.001
				}
			},
			expectGOAWAY: false,
		},
	}

	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			d := probabilisticGoawayDecider{chance: tc.chance, next: tc.nextFn(tc.chance)}
			result := d.Goaway(nil)
			if result != tc.expectGOAWAY {
				t.Errorf("expect GOAWAY: %v, got: %v", tc.expectGOAWAY, result)
			}
		})
	}
}

// TestClientReceivedGOAWAY tests the in-flight watch requests will not be affected and new requests use a
// connection after client received GOAWAY, and server response watch request with GOAWAY will not break client
// watching body read.
func TestClientReceivedGOAWAY(t *testing.T) {
	const (
		urlNormal          = "/normal"
		urlWatch           = "/watch"
		urlGoaway          = "/goaway"
		urlWatchWithGoaway = "/watch-with-goaway"
	)

	const (
		// indicate the bytes watch request will be sent
		// used to check if watch request was broke by GOAWAY
		watchExpectSendBytes = 5
	)

	watchHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		timer := time.NewTicker(time.Second)

		w.Header().Set("Transfer-Encoding", "chunked")
		w.WriteHeader(200)

		flusher, _ := w.(http.Flusher)
		flusher.Flush()

		count := 0
		for {
			<-timer.C
			n, err := w.Write([]byte("w"))
			if err != nil {
				return
			}
			flusher.Flush()
			count += n
			if count == watchExpectSendBytes {
				return
			}
		}
	})

	mux := http.NewServeMux()
	mux.Handle(urlNormal, WithProbabilisticGoaway(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(http.StatusOK)
		w.Write([]byte("hello"))
		return
	}), 0))
	mux.Handle(urlWatch, WithProbabilisticGoaway(watchHandler, 0))
	mux.Handle(urlGoaway, WithProbabilisticGoaway(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(http.StatusOK)
		w.Write([]byte("hello"))
		return
	}), 1))
	mux.Handle(urlWatchWithGoaway, WithProbabilisticGoaway(watchHandler, 1))

	s := httptest.NewUnstartedServer(mux)

	http2Options := &http2.Server{}

	if err := http2.ConfigureServer(s.Config, http2Options); err != nil {
		t.Fatalf("failed to configure test server to be HTTP2 server, err: %v", err)
	}

	s.TLS = s.Config.TLSConfig
	s.StartTLS()
	defer s.Close()

	tlsConfig := &tls.Config{
		InsecureSkipVerify: true,
		NextProtos:         []string{http2.NextProtoTLS},
	}

	cases := []struct {
		name string
		reqs []string
		// expectConnections always equals to GOAWAY requests(urlGoaway or urlWatchWithGoaway) + 1
		expectConnections int
	}{
		{
			name:              "all normal requests use only one connection",
			reqs:              []string{urlNormal, urlNormal, urlNormal},
			expectConnections: 1,
		},
		{
			name:              "got GOAWAY after set-up watch",
			reqs:              []string{urlNormal, urlWatch, urlGoaway, urlNormal, urlNormal},
			expectConnections: 2,
		},
		{
			name:              "got GOAWAY after set-up watch, and set-up a new watch",
			reqs:              []string{urlNormal, urlWatch, urlGoaway, urlWatch, urlNormal, urlNormal},
			expectConnections: 2,
		},
		{
			name:              "got 2 GOAWAY after set-up watch",
			reqs:              []string{urlNormal, urlWatch, urlGoaway, urlGoaway, urlNormal, urlNormal},
			expectConnections: 3,
		},
		{
			name:              "combine with watch-with-goaway",
			reqs:              []string{urlNormal, urlWatchWithGoaway, urlNormal, urlWatch, urlGoaway, urlNormal, urlNormal},
			expectConnections: 3,
		},
	}

	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			// localAddr indicates how many TCP connection set up
			localAddr := make([]string, 0)

			// init HTTP2 client
			client := http.Client{
				Transport: &http2.Transport{
					TLSClientConfig: tlsConfig,
					DialTLS: func(network, addr string, cfg *tls.Config) (conn net.Conn, err error) {
						conn, err = tls.Dial(network, addr, cfg)
						if err != nil {
							t.Fatalf("unexpect connection err: %v", err)
						}
						localAddr = append(localAddr, conn.LocalAddr().String())
						return
					},
				},
			}

			watchChs := make([]chan int, 0)
			for _, url := range tc.reqs {
				req, err := http.NewRequest(http.MethodGet, s.URL+url, nil)
				if err != nil {
					t.Fatalf("unexpect new request error: %v", err)
				}
				resp, err := client.Do(req)
				if err != nil {
					t.Fatalf("failed request test server, err: %v", err)
				}

				// encounter watch bytes received, does not expect to be broken
				if url == urlWatch || url == urlWatchWithGoaway {
					ch := make(chan int)
					watchChs = append(watchChs, ch)
					go func() {
						count := 0
						for {
							buffer := make([]byte, 1)
							n, err := resp.Body.Read(buffer)
							if err != nil {
								// urlWatch will receive io.EOF,
								// urlWatchWithGoaway will receive http2.GoAwayError
								if err != io.EOF {
									if _, ok := err.(http2.GoAwayError); !ok {
										t.Errorf("watch received not EOF err: %v", err)
									}
								}
								ch <- count
								return
							}
							count += n
						}
					}()
				}
			}

			// check TCP connection count
			if tc.expectConnections != len(localAddr) {
				t.Fatalf("expect TCP connection: %d, actual: %d", tc.expectConnections, len(localAddr))
			}

			// check if watch request is broken by GOAWAY response
			watchTimeout := time.NewTimer(time.Second * 10)
			for _, watchCh := range watchChs {
				select {
				case n := <-watchCh:
					if n != watchExpectSendBytes {
						t.Fatalf("in-flight watch was broken by GOAWAY response, expect go bytes: %d, actual got: %d", watchExpectSendBytes, n)
					}
				case <-watchTimeout.C:
					t.Error("watch receive timeout")
				}
			}
		})
	}
}

func TestHTTP1Requests(t *testing.T) {
	s := httptest.NewUnstartedServer(WithProbabilisticGoaway(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(http.StatusOK)
		w.Write([]byte("hello"))
		return
	}), 1))

	http2Options := &http2.Server{}

	if err := http2.ConfigureServer(s.Config, http2Options); err != nil {
		t.Fatalf("failed to configure test server to be HTTP2 server, err: %v", err)
	}

	s.TLS = s.Config.TLSConfig
	s.StartTLS()
	defer s.Close()

	tlsConfig := &tls.Config{
		InsecureSkipVerify: true,
		NextProtos:         []string{"http/1.1"},
	}

	client := http.Client{
		Transport: &http.Transport{
			TLSClientConfig: tlsConfig,
		},
	}

	resp, err := client.Get(s.URL)
	if err != nil {
		t.Fatalf("failed to request the server, err: %v", err)
	}

	if v := resp.Header.Get("Connection"); v != "" {
		t.Errorf("expect response HTTP header Connection to be empty, but got: %s", v)
	}
}
<|endoftext|>


<commit_before>// Copyright © 2017 The Kubicorn Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"errors"
	"fmt"
	"os"
	"strings"

	"github.com/kris-nova/kubicorn/cutil"
	"github.com/kris-nova/kubicorn/cutil/initapi"
	"github.com/kris-nova/kubicorn/cutil/kubeconfig"
	"github.com/kris-nova/kubicorn/cutil/logger"
	"github.com/kris-nova/kubicorn/state"
	"github.com/kris-nova/kubicorn/state/fs"
	"github.com/spf13/cobra"
)

type ApplyOptions struct {
	Options
}

var ao = &ApplyOptions{}

// applyCmd represents the apply command
var applyCmd = &cobra.Command{
	Use:   "apply <NAME>",
	Short: "Apply a cluster resource to a cloud",
	Long: `Use this command to apply an API model in a cloud.

This command will attempt to find an API model in a defined state store, and then apply any changes needed directly to a cloud.
The apply will run once, and ultimately time out if something goes wrong.`,
	Run: func(cmd *cobra.Command, args []string) {
		if len(args) == 0 {
			ao.Name = strEnvDef("KUBICORN_NAME", "")
		} else if len(args) > 1 {
			logger.Critical("Too many arguments.")
			os.Exit(1)
		} else {
			ao.Name = args[0]
		}

		err := RunApply(ao)
		if err != nil {
			logger.Critical(err.Error())
			os.Exit(1)
		}

	},
}

func init() {
	applyCmd.Flags().StringVarP(&ao.StateStore, "state-store", "s", strEnvDef("KUBICORN_STATE_STORE", "fs"), "The state store type to use for the cluster")
	applyCmd.Flags().StringVarP(&ao.StateStorePath, "state-store-path", "S", strEnvDef("KUBICORN_STATE_STORE_PATH", "./_state"), "The state store path to use")
	applyCmd.Flags().StringVarP(&ao.Set, "set", "e", strEnvDef("KUBICORN_SET", ""), "set cluster setting")
	RootCmd.AddCommand(applyCmd)
}

func RunApply(options *ApplyOptions) error {

	// Ensure we have a name
	name := options.Name
	if name == "" {
		return errors.New("Empty name. Must specify the name of the cluster to apply")
	}

	// Expand state store path
	options.StateStorePath = expandPath(options.StateStorePath)

	// Register state store
	var stateStore state.ClusterStorer
	switch options.StateStore {
	case "fs":
		logger.Info("Selected [fs] state store")
		stateStore = fs.NewFileSystemStore(&fs.FileSystemStoreOptions{
			BasePath:    options.StateStorePath,
			ClusterName: name,
		})
	}

	cluster, err := stateStore.GetCluster()
	if err != nil {
		return fmt.Errorf("Unable to get cluster [%s]: %v", name, err)
	}
	logger.Info("Loaded cluster: %s", cluster.Name)

	cluster, err = initapi.InitCluster(cluster)
	if err != nil {
		return err
	}

	reconciler, err := cutil.GetReconciler(cluster)
	if err != nil {
		return fmt.Errorf("Unable to get reconciler: %v", err)
	}

	logger.Info("Query existing resources")
	actual, err := reconciler.Actual(cluster)
	if err != nil {
		return fmt.Errorf("Unable to get actual cluster: %v", err)
	}
	logger.Info("Resolving expected resources")
	expected, err := reconciler.Expected(cluster)
	if err != nil {
		return fmt.Errorf("Unable to get expected cluster: %v", err)
	}

	logger.Info("Reconciling")
	newCluster, err := reconciler.Reconcile(actual, expected)
	if err != nil {
		return fmt.Errorf("Unable to reconcile cluster: %v", err)
	}

	err = stateStore.Commit(newCluster)
	if err != nil {
		return fmt.Errorf("Unable to commit state store: %v", err)
	}

	logger.Info("Updating state store for cluster [%s]", options.Name)

	err = kubeconfig.RetryGetConfig(newCluster)
	if err != nil {
		return fmt.Errorf("Unable to write kubeconfig: %v", err)
	}

	logger.Always("The [%s] cluster has applied successfully!", newCluster.Name)
	logger.Always("You can now `kubectl get nodes`")
	privKeyPath := strings.Replace(cluster.SSH.PublicKeyPath, ".pub", "", 1)
	logger.Always("You can SSH into your cluster ssh -i %s %s@%s", privKeyPath, newCluster.SSH.User, newCluster.KubernetesAPI.Endpoint)

	return nil
}
<commit_msg>cmd: implement set flag for apply<commit_after>// Copyright © 2017 The Kubicorn Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"errors"
	"fmt"
	"os"
	"strings"

	"github.com/kris-nova/kubicorn/cutil"
	"github.com/kris-nova/kubicorn/cutil/initapi"
	"github.com/kris-nova/kubicorn/cutil/kubeconfig"
	"github.com/kris-nova/kubicorn/cutil/logger"
	"github.com/kris-nova/kubicorn/state"
	"github.com/kris-nova/kubicorn/state/fs"
	"github.com/spf13/cobra"
	"github.com/yuroyoro/swalker"
)

type ApplyOptions struct {
	Options
}

var ao = &ApplyOptions{}

// applyCmd represents the apply command
var applyCmd = &cobra.Command{
	Use:   "apply <NAME>",
	Short: "Apply a cluster resource to a cloud",
	Long: `Use this command to apply an API model in a cloud.

This command will attempt to find an API model in a defined state store, and then apply any changes needed directly to a cloud.
The apply will run once, and ultimately time out if something goes wrong.`,
	Run: func(cmd *cobra.Command, args []string) {
		if len(args) == 0 {
			ao.Name = strEnvDef("KUBICORN_NAME", "")
		} else if len(args) > 1 {
			logger.Critical("Too many arguments.")
			os.Exit(1)
		} else {
			ao.Name = args[0]
		}

		err := RunApply(ao)
		if err != nil {
			logger.Critical(err.Error())
			os.Exit(1)
		}

	},
}

func init() {
	applyCmd.Flags().StringVarP(&ao.StateStore, "state-store", "s", strEnvDef("KUBICORN_STATE_STORE", "fs"), "The state store type to use for the cluster")
	applyCmd.Flags().StringVarP(&ao.StateStorePath, "state-store-path", "S", strEnvDef("KUBICORN_STATE_STORE_PATH", "./_state"), "The state store path to use")
	applyCmd.Flags().StringVarP(&ao.Set, "set", "e", strEnvDef("KUBICORN_SET", ""), "set cluster setting")
	RootCmd.AddCommand(applyCmd)
}

func RunApply(options *ApplyOptions) error {

	// Ensure we have a name
	name := options.Name
	if name == "" {
		return errors.New("Empty name. Must specify the name of the cluster to apply")
	}

	// Expand state store path
	options.StateStorePath = expandPath(options.StateStorePath)

	// Register state store
	var stateStore state.ClusterStorer
	switch options.StateStore {
	case "fs":
		logger.Info("Selected [fs] state store")
		stateStore = fs.NewFileSystemStore(&fs.FileSystemStoreOptions{
			BasePath:    options.StateStorePath,
			ClusterName: name,
		})
	}

	cluster, err := stateStore.GetCluster()
	if err != nil {
		return fmt.Errorf("Unable to get cluster [%s]: %v", name, err)
	}
	logger.Info("Loaded cluster: %s", cluster.Name)

	if options.Set != "" {
		sets := strings.Split(options.Set, ",")
		for _, set := range sets {
			parts := strings.SplitN(set, "=", 2)
			err := swalker.Write(strings.Title(parts[0]), cluster, parts[1])
			if err != nil {
				println(err)
			}
		}
	}

	cluster, err = initapi.InitCluster(cluster)
	if err != nil {
		return err
	}

	reconciler, err := cutil.GetReconciler(cluster)
	if err != nil {
		return fmt.Errorf("Unable to get reconciler: %v", err)
	}

	logger.Info("Query existing resources")
	actual, err := reconciler.Actual(cluster)
	if err != nil {
		return fmt.Errorf("Unable to get actual cluster: %v", err)
	}
	logger.Info("Resolving expected resources")
	expected, err := reconciler.Expected(cluster)
	if err != nil {
		return fmt.Errorf("Unable to get expected cluster: %v", err)
	}

	logger.Info("Reconciling")
	newCluster, err := reconciler.Reconcile(actual, expected)
	if err != nil {
		return fmt.Errorf("Unable to reconcile cluster: %v", err)
	}

	err = stateStore.Commit(newCluster)
	if err != nil {
		return fmt.Errorf("Unable to commit state store: %v", err)
	}

	logger.Info("Updating state store for cluster [%s]", options.Name)

	err = kubeconfig.RetryGetConfig(newCluster)
	if err != nil {
		return fmt.Errorf("Unable to write kubeconfig: %v", err)
	}

	logger.Always("The [%s] cluster has applied successfully!", newCluster.Name)
	logger.Always("You can now `kubectl get nodes`")
	privKeyPath := strings.Replace(cluster.SSH.PublicKeyPath, ".pub", "", 1)
	logger.Always("You can SSH into your cluster ssh -i %s %s@%s", privKeyPath, newCluster.SSH.User, newCluster.KubernetesAPI.Endpoint)

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cadvisor

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/http/httptest"
	"reflect"
	"testing"
	"time"

	"github.com/google/cadvisor/info"
	itest "github.com/google/cadvisor/info/test"
)

func testGetJsonData(
	expected interface{},
	f func() (interface{}, error),
) error {
	reply, err := f()
	if err != nil {
		return fmt.Errorf("unable to retrieve data: %v", err)
	}
	if !reflect.DeepEqual(reply, expected) {
		return fmt.Errorf("retrieved wrong data: %+v != %+v", reply, expected)
	}
	return nil
}

func cadvisorTestClient(path string, expectedPostObj, expectedPostObjEmpty, replyObj interface{}, t *testing.T) (*Client, *httptest.Server, error) {
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path == path {
			if expectedPostObj != nil {
				decoder := json.NewDecoder(r.Body)
				err := decoder.Decode(expectedPostObjEmpty)
				if err != nil {
					t.Errorf("Recived invalid object: %v", err)
				}
				if !reflect.DeepEqual(expectedPostObj, expectedPostObjEmpty) {
					t.Errorf("Recived unexpected object: %+v", expectedPostObjEmpty)
				}
			}
			encoder := json.NewEncoder(w)
			encoder.Encode(replyObj)
		} else if r.URL.Path == "/api/v1.0/machine" {
			fmt.Fprint(w, `{"num_cores":8,"memory_capacity":31625871360}`)
		} else {
			w.WriteHeader(http.StatusNotFound)
			fmt.Fprintf(w, "Page not found.")
		}
	}))
	client, err := NewClient(ts.URL)
	if err != nil {
		ts.Close()
		return nil, nil, err
	}
	return client, ts, err
}

func TestGetMachineinfo(t *testing.T) {
	minfo := &info.MachineInfo{
		NumCores:       8,
		MemoryCapacity: 31625871360,
	}
	client, server, err := cadvisorTestClient("/api/v1.0/machine", nil, nil, minfo, t)
	if err != nil {
		t.Fatalf("unable to get a client %v", err)
	}
	defer server.Close()
	err = testGetJsonData(minfo, func() (interface{}, error) {
		return client.MachineInfo()
	})
	if err != nil {
		t.Fatal(err)
	}
}

func TestGetContainerInfo(t *testing.T) {
	query := &info.ContainerInfoQuery{
		NumStats:               3,
		NumSamples:             2,
		CpuUsagePercentages:    []int{10, 50, 90},
		MemoryUsagePercentages: []int{10, 80, 90},
	}
	containerName := "/some/container"
	cinfo := itest.GenerateRandomContainerInfo(containerName, 4, query, 1*time.Second)
	client, server, err := cadvisorTestClient(fmt.Sprintf("/api/v1.0/containers%v", containerName), query, &info.ContainerInfoQuery{}, cinfo, t)
	if err != nil {
		t.Fatalf("unable to get a client %v", err)
	}
	defer server.Close()
	err = testGetJsonData(cinfo, func() (interface{}, error) {
		return client.ContainerInfo(containerName, query)
	})
	if err != nil {
		t.Fatal(err)
	}
}
<commit_msg>better error message for unit test<commit_after>// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cadvisor

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/http/httptest"
	"reflect"
	"testing"
	"time"

	"github.com/google/cadvisor/info"
	itest "github.com/google/cadvisor/info/test"
	"github.com/kr/pretty"
)

func testGetJsonData(
	expected interface{},
	f func() (interface{}, error),
) error {
	reply, err := f()
	if err != nil {
		return fmt.Errorf("unable to retrieve data: %v", err)
	}
	if !reflect.DeepEqual(reply, expected) {
		return pretty.Errorf("retrieved wrong data: %# v != %# v", reply, expected)
	}
	return nil
}

func cadvisorTestClient(path string, expectedPostObj, expectedPostObjEmpty, replyObj interface{}, t *testing.T) (*Client, *httptest.Server, error) {
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path == path {
			if expectedPostObj != nil {
				decoder := json.NewDecoder(r.Body)
				err := decoder.Decode(expectedPostObjEmpty)
				if err != nil {
					t.Errorf("Recived invalid object: %v", err)
				}
				if !reflect.DeepEqual(expectedPostObj, expectedPostObjEmpty) {
					t.Errorf("Recived unexpected object: %+v", expectedPostObjEmpty)
				}
			}
			encoder := json.NewEncoder(w)
			encoder.Encode(replyObj)
		} else if r.URL.Path == "/api/v1.0/machine" {
			fmt.Fprint(w, `{"num_cores":8,"memory_capacity":31625871360}`)
		} else {
			w.WriteHeader(http.StatusNotFound)
			fmt.Fprintf(w, "Page not found.")
		}
	}))
	client, err := NewClient(ts.URL)
	if err != nil {
		ts.Close()
		return nil, nil, err
	}
	return client, ts, err
}

func TestGetMachineinfo(t *testing.T) {
	minfo := &info.MachineInfo{
		NumCores:       8,
		MemoryCapacity: 31625871360,
	}
	client, server, err := cadvisorTestClient("/api/v1.0/machine", nil, nil, minfo, t)
	if err != nil {
		t.Fatalf("unable to get a client %v", err)
	}
	defer server.Close()
	err = testGetJsonData(minfo, func() (interface{}, error) {
		return client.MachineInfo()
	})
	if err != nil {
		t.Fatal(err)
	}
}

func TestGetContainerInfo(t *testing.T) {
	query := &info.ContainerInfoQuery{
		NumStats:               3,
		NumSamples:             2,
		CpuUsagePercentages:    []int{10, 50, 90},
		MemoryUsagePercentages: []int{10, 80, 90},
	}
	containerName := "/some/container"
	cinfo := itest.GenerateRandomContainerInfo(containerName, 4, query, 1*time.Second)
	client, server, err := cadvisorTestClient(fmt.Sprintf("/api/v1.0/containers%v", containerName), query, &info.ContainerInfoQuery{}, cinfo, t)
	if err != nil {
		t.Fatalf("unable to get a client %v", err)
	}
	defer server.Close()
	err = testGetJsonData(cinfo, func() (interface{}, error) {
		return client.ContainerInfo(containerName, query)
	})
	if err != nil {
		t.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package client

import (
	"bufio"
	"strings"
	"io"
	"testing"
	"time"
	"os"
)

// Thanks rsc (Russ Cox, Google Inc.) for this nice fake code
type fakePipes struct {
	io.ReadCloser
	io.WriteCloser
}

func (p *fakePipes) Close() error {
	p.ReadCloser.Close()
	p.WriteCloser.Close()
	return nil
}

func fakeConnect() (io.ReadWriteCloser, error) {
	r1, w1 := io.Pipe()
	r2, w2 := io.Pipe()
	go fakeServer(&fakePipes{r1, w2})
	return &fakePipes{r2, w1}, nil
}

func fakeServer(rw io.ReadWriteCloser) {
	b := bufio.NewReader(rw)
	rw.Write([]byte(fakeReply[""]))
	for {
		line, err := b.ReadString('\n')
		if err != nil {
			break
		}
		reply := fakeReply[strings.TrimSpace(line)]
		if reply == "" {
			rw.Write([]byte("* BYE\r\n"))
			break
		}
		rw.Write([]byte(reply))
	}
	rw.Close()
}

// End of rsc (Russ Cox, Google Inc.) code

var fakeReply = map[string]string{
	"list" : "cpu cpuspeed\n",
	"quit" : "",
	"fetch cpu" : "user.value 234600\n" +
			"nice.value 1931\n" +
			"system.value 80354\n" +
			"idle.value 11153645\n" +
			"iowait.value 98142\n" +
			"irq.value 1\n" +
			"softirq.value 706\n" +
			"steal.value 0\n" +
			".\n",
}

var expectedReply = map[string]string {
	"cpu.user" : "234600",
	"cpu.nice" : "1931",
	"cpu.system" : "80354",
	"cpu.idle" : "11153645",
	"cpu.iowait" : "98142",
	"cpu.irq" : "1",
	"cpu.softirq" : "706",
	"cpu.steal" : "0",
}

func TestConnect(t *testing.T) {
	t.Logf("Seting up fake server\n")
	conn, _ := fakeConnect()
	t.Logf("Seting up fake server done\n")
	interval := time.Millisecond * 200
	done := make(chan os.Signal, 32)
	go func(die chan<- os.Signal){
		time.Sleep(interval * 2)
		die <- os.Interrupt
	}(done)
	valChan := NewMuninClient(conn, interval, done)
	reply := make(map[string]string)
	for values := range valChan {
		for key, value := range values {
			reply[key] = value
		}
	}

	for key, value := range expectedReply {
		if _, exists := reply[key]; !exists {
			t.Errorf("missing key %s in reply", key)
		}
		if reply[key] != value {
			t.Errorf("bad value for key %s in reply: got = %s, want = %s", key, reply[key], value)
		}
	}
	for key, value := range reply {
		if _, exists := expectedReply[key]; !exists {
			t.Errorf("extra key %s in reply, value %s", key, value)
		}
	}
}
<commit_msg>implement banner command<commit_after>package client

import (
	"bufio"
	"strings"
	"io"
	"testing"
	"time"
	"os"
)

// Thanks rsc (Russ Cox, Google Inc.) for this nice fake code
type fakePipes struct {
	io.ReadCloser
	io.WriteCloser
}

func (p *fakePipes) Close() error {
	p.ReadCloser.Close()
	p.WriteCloser.Close()
	return nil
}

func fakeConnect() (io.ReadWriteCloser, error) {
	r1, w1 := io.Pipe()
	r2, w2 := io.Pipe()
	go fakeServer(&fakePipes{r1, w2})
	return &fakePipes{r2, w1}, nil
}

func fakeServer(rw io.ReadWriteCloser) {
	b := bufio.NewReader(rw)
	rw.Write([]byte(fakeReply[""]))
	for {
		line, err := b.ReadString('\n')
		if err != nil {
			break
		}
		reply := fakeReply[strings.TrimSpace(line)]
		if reply == "" {
			rw.Write([]byte("* BYE\r\n"))
			break
		}
		rw.Write([]byte(reply))
	}
	rw.Close()
}

// End of rsc (Russ Cox, Google Inc.) code

var fakeReply = map[string]string{
	"" : "# munin node at localhost\n",
	"list" : "cpu cpuspeed\n",
	"quit" : "",
	"fetch cpu" : "user.value 234600\n" +
			"nice.value 1931\n" +
			"system.value 80354\n" +
			"idle.value 11153645\n" +
			"iowait.value 98142\n" +
			"irq.value 1\n" +
			"softirq.value 706\n" +
			"steal.value 0\n" +
			".\n",
}

var expectedReply = map[string]string {
	"cpu.user" : "234600",
	"cpu.nice" : "1931",
	"cpu.system" : "80354",
	"cpu.idle" : "11153645",
	"cpu.iowait" : "98142",
	"cpu.irq" : "1",
	"cpu.softirq" : "706",
	"cpu.steal" : "0",
}

func TestConnect(t *testing.T) {
	t.Logf("Seting up fake server\n")
	conn, _ := fakeConnect()
	t.Logf("Seting up fake server done\n")
	interval := time.Millisecond * 200
	done := make(chan os.Signal, 32)
	go func(die chan<- os.Signal){
		time.Sleep(interval * 2)
		die <- os.Interrupt
	}(done)
	valChan := NewMuninClient(conn, interval, done)
	reply := make(map[string]string)
	for values := range valChan {
		for key, value := range values {
			reply[key] = value
		}
	}

	for key, value := range expectedReply {
		if _, exists := reply[key]; !exists {
			t.Errorf("missing key %s in reply", key)
		}
		if reply[key] != value {
			t.Errorf("bad value for key %s in reply: got = %s, want = %s", key, reply[key], value)
		}
	}
	for key, value := range reply {
		if _, exists := expectedReply[key]; !exists {
			t.Errorf("extra key %s in reply, value %s", key, value)
		}
	}
}
<|endoftext|>


<commit_before>// This package provides utilities that underlie the specific commands.
// The idea is to make the specific command files very small, e.g.:
//
//    func main() {
//      app := cmd.NewAppShell("command-name")
//      app.Action = func(c cmd.Config) {
//        // command logic
//      }
//      app.Run()
//    }
//
// All commands share the same invocation pattern.  They take a single
// parameter "-config", which is the name of a JSON file containing
// the configuration for the app.  This JSON file is unmarshalled into
// a Config object, which is provided to the app.

package cmd

import (
	"encoding/json"
	"encoding/pem"
	"errors"
	"expvar"
	"fmt"
	"io/ioutil"
	"log"
	"log/syslog"
	"net/http"
	"net/http/pprof"
	"os"
	"os/signal"
	"path"
	"runtime"
	"strings"
	"syscall"

	"google.golang.org/grpc/grpclog"

	cfsslLog "github.com/cloudflare/cfssl/log"
	"github.com/go-sql-driver/mysql"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"

	"github.com/letsencrypt/boulder/core"
	blog "github.com/letsencrypt/boulder/log"
	"github.com/letsencrypt/boulder/metrics"
)

// Because we don't know when this init will be called with respect to
// flag.Parse() and other flag definitions, we can't rely on the regular
// flag mechanism. But this one is fine.
func init() {
	for _, v := range os.Args {
		if v == "--version" || v == "-version" {
			fmt.Println(VersionString())
			os.Exit(0)
		}
	}
}

// mysqlLogger proxies blog.AuditLogger to provide a Print(...) method.
type mysqlLogger struct {
	blog.Logger
}

func (m mysqlLogger) Print(v ...interface{}) {
	m.AuditErrf("[mysql] %s", fmt.Sprint(v...))
}

// cfsslLogger provides two additional methods that are expected by CFSSL's
// logger but not supported by Boulder's Logger.
type cfsslLogger struct {
	blog.Logger
}

func (cl cfsslLogger) Crit(msg string) {
	cl.AuditErr(msg)
}

func (cl cfsslLogger) Emerg(msg string) {
	cl.AuditErr(msg)
}

type grpcLogger struct {
	blog.Logger
}

// V returns true if the verbosity level l is less than the verbosity we want to
// log at.
func (log grpcLogger) V(l int) bool {
	return l < 0
}

func (log grpcLogger) Fatal(args ...interface{}) {
	log.Error(args...)
	os.Exit(1)
}
func (log grpcLogger) Fatalf(format string, args ...interface{}) {
	log.Error(args...)
	os.Exit(1)
}
func (log grpcLogger) Fatalln(args ...interface{}) {
	log.Error(args...)
	os.Exit(1)
}

func (log grpcLogger) Error(args ...interface{}) {
	log.Logger.AuditErr(fmt.Sprintln(args...))
}
func (log grpcLogger) Errorf(format string, args ...interface{}) {
	log.Logger.AuditErrf(format, args...)
}
func (log grpcLogger) Errorln(args ...interface{}) {
	log.Logger.AuditErr(fmt.Sprintln(args...))
}

func (log grpcLogger) Warning(args ...interface{}) {
	log.Error(args...)
}
func (log grpcLogger) Warningf(format string, args ...interface{}) {
	log.Errorf(format, args...)
}
func (log grpcLogger) Warningln(args ...interface{}) {
	log.Errorln(args...)
}

func (log grpcLogger) Info(args ...interface{}) {
	log.Logger.Info(fmt.Sprintln(args...))
}
func (log grpcLogger) Infof(format string, args ...interface{}) {
	log.Logger.Infof(format, args...)
}
func (log grpcLogger) Infoln(args ...interface{}) {
	log.Logger.Info(fmt.Sprintln(args...))
}

type promLogger struct {
	blog.Logger
}

func (log promLogger) Println(args ...interface{}) {
	log.AuditErr(fmt.Sprintln(args...))
}

// StatsAndLogging constructs a metrics.Scope and an AuditLogger based on its config
// parameters, and return them both. It also spawns off an HTTP server on the
// provided port to report the stats and provide pprof profiling handlers.
// Crashes if any setup fails.
// Also sets the constructed AuditLogger as the default logger, and configures
// the cfssl, mysql, and grpc packages to use our logger.
// This must be called before any gRPC code is called, because gRPC's SetLogger
// doesn't use any locking.
func StatsAndLogging(logConf SyslogConfig, addr string) (metrics.Scope, blog.Logger) {
	logger := NewLogger(logConf)
	scope := newScope(addr, logger)
	return scope, logger
}

func NewLogger(logConf SyslogConfig) blog.Logger {
	tag := path.Base(os.Args[0])
	syslogger, err := syslog.Dial(
		"",
		"",
		syslog.LOG_INFO, // default, not actually used
		tag)
	FailOnError(err, "Could not connect to Syslog")
	syslogLevel := int(syslog.LOG_INFO)
	if logConf.SyslogLevel != 0 {
		syslogLevel = logConf.SyslogLevel
	}
	logger, err := blog.New(syslogger, logConf.StdoutLevel, syslogLevel)
	FailOnError(err, "Could not connect to Syslog")

	_ = blog.Set(logger)
	cfsslLog.SetLogger(cfsslLogger{logger})
	_ = mysql.SetLogger(mysqlLogger{logger})
	grpclog.SetLoggerV2(grpcLogger{logger})
	return logger
}

func newScope(addr string, logger blog.Logger) metrics.Scope {
	registry := prometheus.NewRegistry()
	registry.MustRegister(prometheus.NewGoCollector())
	registry.MustRegister(prometheus.NewProcessCollector(os.Getpid(), ""))

	mux := http.NewServeMux()
	// Register the available pprof handlers. These are all registered on
	// DefaultServeMux just by importing pprof, but since we eschew
	// DefaultServeMux, we need to explicitly register them on our own mux.
	mux.Handle("/debug/pprof/", http.HandlerFunc(pprof.Index))
	mux.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile))
	mux.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol))
	mux.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace))
	// These handlers are defined in runtime/pprof instead of net/http/pprof, and
	// have to be accessed through net/http/pprof's Handler func.
	mux.Handle("/debug/pprof/goroutine", pprof.Handler("goroutine"))
	mux.Handle("/debug/pprof/block", pprof.Handler("block"))
	mux.Handle("/debug/pprof/heap", pprof.Handler("heap"))
	mux.Handle("/debug/pprof/mutex", pprof.Handler("mutex"))
	mux.Handle("/debug/pprof/threadcreate", pprof.Handler("threadcreate"))

	mux.Handle("/debug/vars", expvar.Handler())
	mux.Handle("/metrics", promhttp.HandlerFor(registry, promhttp.HandlerOpts{
		ErrorLog: promLogger{logger},
	}))

	server := http.Server{
		Addr:    addr,
		Handler: mux,
	}
	go func() {
		err := server.ListenAndServe()
		if err != nil {
			log.Fatalf("unable to boot debug server on %s: %v", addr, err)
		}
	}()
	return metrics.NewPromScope(registry)
}

// Fail exits and prints an error message to stderr and the logger audit log.
func Fail(msg string) {
	logger := blog.Get()
	logger.AuditErr(msg)
	fmt.Fprintf(os.Stderr, msg)
	os.Exit(1)
}

// FailOnError exits and prints an error message, but only if we encountered
// a problem and err != nil
func FailOnError(err error, msg string) {
	if err != nil {
		msg := fmt.Sprintf("%s: %s", msg, err)
		Fail(msg)
	}
}

// LoadCert loads a PEM-formatted certificate from the provided path, returning
// it as a byte array, or an error if it couldn't be decoded.
func LoadCert(path string) (cert []byte, err error) {
	if path == "" {
		err = errors.New("Issuer certificate was not provided in config.")
		return
	}
	pemBytes, err := ioutil.ReadFile(path)
	if err != nil {
		return
	}

	block, _ := pem.Decode(pemBytes)
	if block == nil || block.Type != "CERTIFICATE" {
		err = errors.New("Invalid certificate value returned")
		return
	}

	cert = block.Bytes
	return
}

// ReadConfigFile takes a file path as an argument and attempts to
// unmarshal the content of the file into a struct containing a
// configuration of a boulder component.
func ReadConfigFile(filename string, out interface{}) error {
	configData, err := ioutil.ReadFile(filename)
	if err != nil {
		return err
	}
	return json.Unmarshal(configData, out)
}

// VersionString produces a friendly Application version string.
func VersionString() string {
	name := path.Base(os.Args[0])
	return fmt.Sprintf("Versions: %s=(%s %s) Golang=(%s) BuildHost=(%s)", name, core.GetBuildID(), core.GetBuildTime(), runtime.Version(), core.GetBuildHost())
}

var signalToName = map[os.Signal]string{
	syscall.SIGTERM: "SIGTERM",
	syscall.SIGINT:  "SIGINT",
	syscall.SIGHUP:  "SIGHUP",
}

// CatchSignals catches SIGTERM, SIGINT, SIGHUP and executes a callback
// method before exiting
func CatchSignals(logger blog.Logger, callback func()) {
	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan, syscall.SIGTERM)
	signal.Notify(sigChan, syscall.SIGINT)
	signal.Notify(sigChan, syscall.SIGHUP)

	sig := <-sigChan
	if logger != nil {
		logger.Infof("Caught %s", signalToName[sig])
	}

	if callback != nil {
		callback()
	}

	if logger != nil {
		logger.Info("Exiting")
	}
	os.Exit(0)
}

// FilterShutdownErrors returns the input error, with the exception of "use of
// closed network connection," on which it returns nil
// Per https://github.com/grpc/grpc-go/issues/1017, a gRPC server's `Serve()`
// will always return an error, even when GracefulStop() is called. We don't
// want to log graceful stops as errors, so we filter out the meaningless
// error we get in that situation.
func FilterShutdownErrors(err error) error {
	if err == nil {
		return nil
	}
	if strings.Contains(err.Error(), "use of closed network connection") {
		return nil
	}
	return err
}
<commit_msg>cmd: Log less from gRPC, no INFO level. (#4367)<commit_after>// This package provides utilities that underlie the specific commands.
// The idea is to make the specific command files very small, e.g.:
//
//    func main() {
//      app := cmd.NewAppShell("command-name")
//      app.Action = func(c cmd.Config) {
//        // command logic
//      }
//      app.Run()
//    }
//
// All commands share the same invocation pattern.  They take a single
// parameter "-config", which is the name of a JSON file containing
// the configuration for the app.  This JSON file is unmarshalled into
// a Config object, which is provided to the app.

package cmd

import (
	"encoding/json"
	"encoding/pem"
	"errors"
	"expvar"
	"fmt"
	"io/ioutil"
	"log"
	"log/syslog"
	"net/http"
	"net/http/pprof"
	"os"
	"os/signal"
	"path"
	"runtime"
	"strings"
	"syscall"

	"google.golang.org/grpc/grpclog"

	cfsslLog "github.com/cloudflare/cfssl/log"
	"github.com/go-sql-driver/mysql"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"

	"github.com/letsencrypt/boulder/core"
	blog "github.com/letsencrypt/boulder/log"
	"github.com/letsencrypt/boulder/metrics"
)

// Because we don't know when this init will be called with respect to
// flag.Parse() and other flag definitions, we can't rely on the regular
// flag mechanism. But this one is fine.
func init() {
	for _, v := range os.Args {
		if v == "--version" || v == "-version" {
			fmt.Println(VersionString())
			os.Exit(0)
		}
	}
}

// mysqlLogger proxies blog.AuditLogger to provide a Print(...) method.
type mysqlLogger struct {
	blog.Logger
}

func (m mysqlLogger) Print(v ...interface{}) {
	m.AuditErrf("[mysql] %s", fmt.Sprint(v...))
}

// cfsslLogger provides two additional methods that are expected by CFSSL's
// logger but not supported by Boulder's Logger.
type cfsslLogger struct {
	blog.Logger
}

func (cl cfsslLogger) Crit(msg string) {
	cl.AuditErr(msg)
}

func (cl cfsslLogger) Emerg(msg string) {
	cl.AuditErr(msg)
}

type grpcLogger struct {
	blog.Logger
}

// V returns true if the verbosity level l is less than the verbosity we want to
// log at.
func (log grpcLogger) V(l int) bool {
	return l < 0
}

func (log grpcLogger) Fatal(args ...interface{}) {
	log.Error(args...)
	os.Exit(1)
}
func (log grpcLogger) Fatalf(format string, args ...interface{}) {
	log.Error(args...)
	os.Exit(1)
}
func (log grpcLogger) Fatalln(args ...interface{}) {
	log.Error(args...)
	os.Exit(1)
}

func (log grpcLogger) Error(args ...interface{}) {
	log.Logger.AuditErr(fmt.Sprintln(args...))
}
func (log grpcLogger) Errorf(format string, args ...interface{}) {
	log.Logger.AuditErrf(format, args...)
}
func (log grpcLogger) Errorln(args ...interface{}) {
	log.Logger.AuditErr(fmt.Sprintln(args...))
}

func (log grpcLogger) Warning(args ...interface{}) {
	log.Error(args...)
}
func (log grpcLogger) Warningf(format string, args ...interface{}) {
	log.Errorf(format, args...)
}
func (log grpcLogger) Warningln(args ...interface{}) {
	log.Errorln(args...)
}

// Don't log any INFO-level gRPC stuff. In practice this is all noise, like
// failed TXT lookups for service discovery (we only use A records).
func (log grpcLogger) Info(args ...interface{}) {
}
func (log grpcLogger) Infof(format string, args ...interface{}) {
}
func (log grpcLogger) Infoln(args ...interface{}) {
}

type promLogger struct {
	blog.Logger
}

func (log promLogger) Println(args ...interface{}) {
	log.AuditErr(fmt.Sprintln(args...))
}

// StatsAndLogging constructs a metrics.Scope and an AuditLogger based on its config
// parameters, and return them both. It also spawns off an HTTP server on the
// provided port to report the stats and provide pprof profiling handlers.
// Crashes if any setup fails.
// Also sets the constructed AuditLogger as the default logger, and configures
// the cfssl, mysql, and grpc packages to use our logger.
// This must be called before any gRPC code is called, because gRPC's SetLogger
// doesn't use any locking.
func StatsAndLogging(logConf SyslogConfig, addr string) (metrics.Scope, blog.Logger) {
	logger := NewLogger(logConf)
	scope := newScope(addr, logger)
	return scope, logger
}

func NewLogger(logConf SyslogConfig) blog.Logger {
	tag := path.Base(os.Args[0])
	syslogger, err := syslog.Dial(
		"",
		"",
		syslog.LOG_INFO, // default, not actually used
		tag)
	FailOnError(err, "Could not connect to Syslog")
	syslogLevel := int(syslog.LOG_INFO)
	if logConf.SyslogLevel != 0 {
		syslogLevel = logConf.SyslogLevel
	}
	logger, err := blog.New(syslogger, logConf.StdoutLevel, syslogLevel)
	FailOnError(err, "Could not connect to Syslog")

	_ = blog.Set(logger)
	cfsslLog.SetLogger(cfsslLogger{logger})
	_ = mysql.SetLogger(mysqlLogger{logger})
	grpclog.SetLoggerV2(grpcLogger{logger})
	return logger
}

func newScope(addr string, logger blog.Logger) metrics.Scope {
	registry := prometheus.NewRegistry()
	registry.MustRegister(prometheus.NewGoCollector())
	registry.MustRegister(prometheus.NewProcessCollector(os.Getpid(), ""))

	mux := http.NewServeMux()
	// Register the available pprof handlers. These are all registered on
	// DefaultServeMux just by importing pprof, but since we eschew
	// DefaultServeMux, we need to explicitly register them on our own mux.
	mux.Handle("/debug/pprof/", http.HandlerFunc(pprof.Index))
	mux.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile))
	mux.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol))
	mux.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace))
	// These handlers are defined in runtime/pprof instead of net/http/pprof, and
	// have to be accessed through net/http/pprof's Handler func.
	mux.Handle("/debug/pprof/goroutine", pprof.Handler("goroutine"))
	mux.Handle("/debug/pprof/block", pprof.Handler("block"))
	mux.Handle("/debug/pprof/heap", pprof.Handler("heap"))
	mux.Handle("/debug/pprof/mutex", pprof.Handler("mutex"))
	mux.Handle("/debug/pprof/threadcreate", pprof.Handler("threadcreate"))

	mux.Handle("/debug/vars", expvar.Handler())
	mux.Handle("/metrics", promhttp.HandlerFor(registry, promhttp.HandlerOpts{
		ErrorLog: promLogger{logger},
	}))

	server := http.Server{
		Addr:    addr,
		Handler: mux,
	}
	go func() {
		err := server.ListenAndServe()
		if err != nil {
			log.Fatalf("unable to boot debug server on %s: %v", addr, err)
		}
	}()
	return metrics.NewPromScope(registry)
}

// Fail exits and prints an error message to stderr and the logger audit log.
func Fail(msg string) {
	logger := blog.Get()
	logger.AuditErr(msg)
	fmt.Fprintf(os.Stderr, msg)
	os.Exit(1)
}

// FailOnError exits and prints an error message, but only if we encountered
// a problem and err != nil
func FailOnError(err error, msg string) {
	if err != nil {
		msg := fmt.Sprintf("%s: %s", msg, err)
		Fail(msg)
	}
}

// LoadCert loads a PEM-formatted certificate from the provided path, returning
// it as a byte array, or an error if it couldn't be decoded.
func LoadCert(path string) (cert []byte, err error) {
	if path == "" {
		err = errors.New("Issuer certificate was not provided in config.")
		return
	}
	pemBytes, err := ioutil.ReadFile(path)
	if err != nil {
		return
	}

	block, _ := pem.Decode(pemBytes)
	if block == nil || block.Type != "CERTIFICATE" {
		err = errors.New("Invalid certificate value returned")
		return
	}

	cert = block.Bytes
	return
}

// ReadConfigFile takes a file path as an argument and attempts to
// unmarshal the content of the file into a struct containing a
// configuration of a boulder component.
func ReadConfigFile(filename string, out interface{}) error {
	configData, err := ioutil.ReadFile(filename)
	if err != nil {
		return err
	}
	return json.Unmarshal(configData, out)
}

// VersionString produces a friendly Application version string.
func VersionString() string {
	name := path.Base(os.Args[0])
	return fmt.Sprintf("Versions: %s=(%s %s) Golang=(%s) BuildHost=(%s)", name, core.GetBuildID(), core.GetBuildTime(), runtime.Version(), core.GetBuildHost())
}

var signalToName = map[os.Signal]string{
	syscall.SIGTERM: "SIGTERM",
	syscall.SIGINT:  "SIGINT",
	syscall.SIGHUP:  "SIGHUP",
}

// CatchSignals catches SIGTERM, SIGINT, SIGHUP and executes a callback
// method before exiting
func CatchSignals(logger blog.Logger, callback func()) {
	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan, syscall.SIGTERM)
	signal.Notify(sigChan, syscall.SIGINT)
	signal.Notify(sigChan, syscall.SIGHUP)

	sig := <-sigChan
	if logger != nil {
		logger.Infof("Caught %s", signalToName[sig])
	}

	if callback != nil {
		callback()
	}

	if logger != nil {
		logger.Info("Exiting")
	}
	os.Exit(0)
}

// FilterShutdownErrors returns the input error, with the exception of "use of
// closed network connection," on which it returns nil
// Per https://github.com/grpc/grpc-go/issues/1017, a gRPC server's `Serve()`
// will always return an error, even when GracefulStop() is called. We don't
// want to log graceful stops as errors, so we filter out the meaningless
// error we get in that situation.
func FilterShutdownErrors(err error) error {
	if err == nil {
		return nil
	}
	if strings.Contains(err.Error(), "use of closed network connection") {
		return nil
	}
	return err
}
<|endoftext|>


<commit_before>package mpawskinesisstreams

import (
	"errors"
	"flag"
	"log"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudwatch"
	mp "github.com/mackerelio/go-mackerel-plugin-helper"
)

const (
	namespace          = "AWS/Kinesis"
	metricsTypeAverage = "Average"
	metricsTypeMaximum = "Maximum"
	metricsTypeMinimum = "Minimum"
)

type metrics struct {
	CloudWatchName string
	MackerelName   string
	Type           string
}

// KinesisStreamsPlugin mackerel plugin for aws kinesis
type KinesisStreamsPlugin struct {
	Name   string
	Prefix string

	AccessKeyID     string
	SecretAccessKey string
	Region          string
	CloudWatch      *cloudwatch.CloudWatch
}

// MetricKeyPrefix interface for PluginWithPrefix
func (p KinesisStreamsPlugin) MetricKeyPrefix() string {
	if p.Prefix == "" {
		p.Prefix = "kinesis-streams"
	}
	return p.Prefix
}

// prepare creates CloudWatch instance
func (p *KinesisStreamsPlugin) prepare() error {
	sess, err := session.NewSession()
	if err != nil {
		return err
	}

	config := aws.NewConfig()
	if p.AccessKeyID != "" && p.SecretAccessKey != "" {
		config = config.WithCredentials(credentials.NewStaticCredentials(p.AccessKeyID, p.SecretAccessKey, ""))
	}
	if p.Region != "" {
		config = config.WithRegion(p.Region)
	}

	p.CloudWatch = cloudwatch.New(sess, config)

	return nil
}

// getLastPoint fetches a CloudWatch metric and parse
func (p KinesisStreamsPlugin) getLastPoint(metric metrics) (float64, error) {
	now := time.Now()

	dimensions := []*cloudwatch.Dimension{
		{
			Name:  aws.String("StreamName"),
			Value: aws.String(p.Name),
		},
	}

	response, err := p.CloudWatch.GetMetricStatistics(&cloudwatch.GetMetricStatisticsInput{
		Dimensions: dimensions,
		StartTime:  aws.Time(now.Add(time.Duration(180) * time.Second * -1)), // 3 min
		EndTime:    aws.Time(now),
		MetricName: aws.String(metric.CloudWatchName),
		Period:     aws.Int64(60),
		Statistics: []*string{aws.String(metric.Type)},
		Namespace:  aws.String(namespace),
	})
	if err != nil {
		return 0, err
	}

	datapoints := response.Datapoints
	if len(datapoints) == 0 {
		return 0, errors.New("fetched no datapoints")
	}

	latest := new(time.Time)
	var latestVal float64
	for _, dp := range datapoints {
		if dp.Timestamp.Before(*latest) {
			continue
		}

		latest = dp.Timestamp
		switch metric.Type {
		case metricsTypeAverage:
			latestVal = *dp.Average
		case metricsTypeMaximum:
			latestVal = *dp.Maximum
		case metricsTypeMinimum:
			latestVal = *dp.Minimum
		}
	}

	return latestVal, nil
}

// FetchMetrics fetch the metrics
func (p KinesisStreamsPlugin) FetchMetrics() (map[string]interface{}, error) {
	stat := make(map[string]interface{})

	for _, met := range [...]metrics{
		{CloudWatchName: "GetRecords.Bytes", MackerelName: "GetRecordsBytes", Type: metricsTypeAverage},
		{CloudWatchName: "GetRecords.IteratorAgeMilliseconds", MackerelName: "GetRecordsDelayMaxMilliseconds", Type: metricsTypeMaximum},
		{CloudWatchName: "GetRecords.IteratorAgeMilliseconds", MackerelName: "GetRecordsDelayMinMilliseconds", Type: metricsTypeMinimum},
		{CloudWatchName: "GetRecords.Latency", MackerelName: "GetRecordsLatency", Type: metricsTypeAverage},
		{CloudWatchName: "GetRecords.Records", MackerelName: "GetRecordsRecords", Type: metricsTypeAverage},
		{CloudWatchName: "GetRecords.Success", MackerelName: "GetRecordsSuccess", Type: metricsTypeAverage},
		{CloudWatchName: "IncomingBytes", MackerelName: "IncomingBytes", Type: metricsTypeAverage},
		{CloudWatchName: "IncomingRecords", MackerelName: "IncomingRecords", Type: metricsTypeAverage},
		{CloudWatchName: "PutRecord.Bytes", MackerelName: "PutRecordBytes", Type: metricsTypeAverage},
		{CloudWatchName: "PutRecord.Latency", MackerelName: "PutRecordLatency", Type: metricsTypeAverage},
		{CloudWatchName: "PutRecord.Success", MackerelName: "PutRecordSuccess", Type: metricsTypeAverage},
		{CloudWatchName: "PutRecords.Bytes", MackerelName: "PutRecordsBytes", Type: metricsTypeAverage},
		{CloudWatchName: "PutRecords.Latency", MackerelName: "PutRecordsLatency", Type: metricsTypeAverage},
		{CloudWatchName: "PutRecords.Records", MackerelName: "PutRecordsRecords", Type: metricsTypeAverage},
		{CloudWatchName: "PutRecords.Success", MackerelName: "PutRecordsSuccess", Type: metricsTypeAverage},
		{CloudWatchName: "ReadProvidionedThroughputExceeded", MackerelName: "ReadThroughputExceeded", Type: metricsTypeAverage},
		{CloudWatchName: "WriteProvidionedThroughputExceeded", MackerelName: "WriteThroughputExceeded", Type: metricsTypeAverage},
	} {
		v, err := p.getLastPoint(met)
		if err == nil {
			stat[met.MackerelName] = v
		} else {
			log.Printf("%s: %s", met, err)
		}
	}
	return stat, nil
}

// GraphDefinition of KinesisStreamsPlugin
func (p KinesisStreamsPlugin) GraphDefinition() map[string]mp.Graphs {
	labelPrefix := strings.Title(p.Prefix)
	labelPrefix = strings.Replace(labelPrefix, "-", " ", -1)

	var graphdef = map[string]mp.Graphs{
		"bytes": {
			Label: (labelPrefix + " Bytes"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "GetRecordsBytes", Label: "GetRecords"},
				{Name: "IncomingBytes", Label: "Total Incoming"},
				{Name: "PutRecordBytes", Label: "PutRecord"},
				{Name: "PutRecordsBytes", Label: "PutRecords"},
			},
		},
		"iteratorage": {
			Label: (labelPrefix + " Read Delay"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "GetRecordsDelayMaxMilliseconds", Label: "Max"},
				{Name: "GetRecordsDelayMinMilliseconds", Label: "min"},
			},
		},
		"latency": {
			Label: (labelPrefix + " Operation Latency"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "GetRecordsLatency", Label: "GetRecords"},
				{Name: "PutRecordLatency", Label: "PutRecord"},
				{Name: "PutRecordsLatency", Label: "PutRecords"},
			},
		},
		"records": {
			Label: (labelPrefix + " Records"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "GetRecordsRecords", Label: "GetRecords"},
				{Name: "IncomingRecords", Label: "Total Incoming"},
				{Name: "PutRecordsRecords", Label: "PutRecords"},
			},
		},
		"success": {
			Label: (labelPrefix + " Operation Success"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "GetRecordsSuccess", Label: "GetRecords"},
				{Name: "PutRecordSuccess", Label: "PutRecord"},
				{Name: "PutRecordsSuccess", Label: "PutRecords"},
			},
		},
		"pending": {
			Label: (labelPrefix + " Pending Operations"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "ReadThroughputExceeded", Label: "Read"},
				{Name: "WriteThroughputExceeded", Label: "Write"},
			},
		},
	}
	return graphdef
}

// Do the plugin
func Do() {
	optAccessKeyID := flag.String("access-key-id", "", "AWS Access Key ID")
	optSecretAccessKey := flag.String("secret-access-key", "", "AWS Secret Access Key")
	optRegion := flag.String("region", "", "AWS Region")
	optIdentifier := flag.String("identifier", "", "Stream Name")
	optTempfile := flag.String("tempfile", "", "Temp file name")
	optPrefix := flag.String("metric-key-prefix", "kinesis-streams", "Metric key prefix")
	flag.Parse()

	var plugin KinesisStreamsPlugin

	plugin.AccessKeyID = *optAccessKeyID
	plugin.SecretAccessKey = *optSecretAccessKey
	plugin.Region = *optRegion
	plugin.Name = *optIdentifier
	plugin.Prefix = *optPrefix

	err := plugin.prepare()
	if err != nil {
		log.Fatalln(err)
	}

	helper := mp.NewMackerelPlugin(plugin)
	helper.Tempfile = *optTempfile

	helper.Run()
}
<commit_msg>s/min/Min/<commit_after>package mpawskinesisstreams

import (
	"errors"
	"flag"
	"log"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudwatch"
	mp "github.com/mackerelio/go-mackerel-plugin-helper"
)

const (
	namespace          = "AWS/Kinesis"
	metricsTypeAverage = "Average"
	metricsTypeMaximum = "Maximum"
	metricsTypeMinimum = "Minimum"
)

type metrics struct {
	CloudWatchName string
	MackerelName   string
	Type           string
}

// KinesisStreamsPlugin mackerel plugin for aws kinesis
type KinesisStreamsPlugin struct {
	Name   string
	Prefix string

	AccessKeyID     string
	SecretAccessKey string
	Region          string
	CloudWatch      *cloudwatch.CloudWatch
}

// MetricKeyPrefix interface for PluginWithPrefix
func (p KinesisStreamsPlugin) MetricKeyPrefix() string {
	if p.Prefix == "" {
		p.Prefix = "kinesis-streams"
	}
	return p.Prefix
}

// prepare creates CloudWatch instance
func (p *KinesisStreamsPlugin) prepare() error {
	sess, err := session.NewSession()
	if err != nil {
		return err
	}

	config := aws.NewConfig()
	if p.AccessKeyID != "" && p.SecretAccessKey != "" {
		config = config.WithCredentials(credentials.NewStaticCredentials(p.AccessKeyID, p.SecretAccessKey, ""))
	}
	if p.Region != "" {
		config = config.WithRegion(p.Region)
	}

	p.CloudWatch = cloudwatch.New(sess, config)

	return nil
}

// getLastPoint fetches a CloudWatch metric and parse
func (p KinesisStreamsPlugin) getLastPoint(metric metrics) (float64, error) {
	now := time.Now()

	dimensions := []*cloudwatch.Dimension{
		{
			Name:  aws.String("StreamName"),
			Value: aws.String(p.Name),
		},
	}

	response, err := p.CloudWatch.GetMetricStatistics(&cloudwatch.GetMetricStatisticsInput{
		Dimensions: dimensions,
		StartTime:  aws.Time(now.Add(time.Duration(180) * time.Second * -1)), // 3 min
		EndTime:    aws.Time(now),
		MetricName: aws.String(metric.CloudWatchName),
		Period:     aws.Int64(60),
		Statistics: []*string{aws.String(metric.Type)},
		Namespace:  aws.String(namespace),
	})
	if err != nil {
		return 0, err
	}

	datapoints := response.Datapoints
	if len(datapoints) == 0 {
		return 0, errors.New("fetched no datapoints")
	}

	latest := new(time.Time)
	var latestVal float64
	for _, dp := range datapoints {
		if dp.Timestamp.Before(*latest) {
			continue
		}

		latest = dp.Timestamp
		switch metric.Type {
		case metricsTypeAverage:
			latestVal = *dp.Average
		case metricsTypeMaximum:
			latestVal = *dp.Maximum
		case metricsTypeMinimum:
			latestVal = *dp.Minimum
		}
	}

	return latestVal, nil
}

// FetchMetrics fetch the metrics
func (p KinesisStreamsPlugin) FetchMetrics() (map[string]interface{}, error) {
	stat := make(map[string]interface{})

	for _, met := range [...]metrics{
		{CloudWatchName: "GetRecords.Bytes", MackerelName: "GetRecordsBytes", Type: metricsTypeAverage},
		{CloudWatchName: "GetRecords.IteratorAgeMilliseconds", MackerelName: "GetRecordsDelayMaxMilliseconds", Type: metricsTypeMaximum},
		{CloudWatchName: "GetRecords.IteratorAgeMilliseconds", MackerelName: "GetRecordsDelayMinMilliseconds", Type: metricsTypeMinimum},
		{CloudWatchName: "GetRecords.Latency", MackerelName: "GetRecordsLatency", Type: metricsTypeAverage},
		{CloudWatchName: "GetRecords.Records", MackerelName: "GetRecordsRecords", Type: metricsTypeAverage},
		{CloudWatchName: "GetRecords.Success", MackerelName: "GetRecordsSuccess", Type: metricsTypeAverage},
		{CloudWatchName: "IncomingBytes", MackerelName: "IncomingBytes", Type: metricsTypeAverage},
		{CloudWatchName: "IncomingRecords", MackerelName: "IncomingRecords", Type: metricsTypeAverage},
		{CloudWatchName: "PutRecord.Bytes", MackerelName: "PutRecordBytes", Type: metricsTypeAverage},
		{CloudWatchName: "PutRecord.Latency", MackerelName: "PutRecordLatency", Type: metricsTypeAverage},
		{CloudWatchName: "PutRecord.Success", MackerelName: "PutRecordSuccess", Type: metricsTypeAverage},
		{CloudWatchName: "PutRecords.Bytes", MackerelName: "PutRecordsBytes", Type: metricsTypeAverage},
		{CloudWatchName: "PutRecords.Latency", MackerelName: "PutRecordsLatency", Type: metricsTypeAverage},
		{CloudWatchName: "PutRecords.Records", MackerelName: "PutRecordsRecords", Type: metricsTypeAverage},
		{CloudWatchName: "PutRecords.Success", MackerelName: "PutRecordsSuccess", Type: metricsTypeAverage},
		{CloudWatchName: "ReadProvidionedThroughputExceeded", MackerelName: "ReadThroughputExceeded", Type: metricsTypeAverage},
		{CloudWatchName: "WriteProvidionedThroughputExceeded", MackerelName: "WriteThroughputExceeded", Type: metricsTypeAverage},
	} {
		v, err := p.getLastPoint(met)
		if err == nil {
			stat[met.MackerelName] = v
		} else {
			log.Printf("%s: %s", met, err)
		}
	}
	return stat, nil
}

// GraphDefinition of KinesisStreamsPlugin
func (p KinesisStreamsPlugin) GraphDefinition() map[string]mp.Graphs {
	labelPrefix := strings.Title(p.Prefix)
	labelPrefix = strings.Replace(labelPrefix, "-", " ", -1)

	var graphdef = map[string]mp.Graphs{
		"bytes": {
			Label: (labelPrefix + " Bytes"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "GetRecordsBytes", Label: "GetRecords"},
				{Name: "IncomingBytes", Label: "Total Incoming"},
				{Name: "PutRecordBytes", Label: "PutRecord"},
				{Name: "PutRecordsBytes", Label: "PutRecords"},
			},
		},
		"iteratorage": {
			Label: (labelPrefix + " Read Delay"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "GetRecordsDelayMaxMilliseconds", Label: "Max"},
				{Name: "GetRecordsDelayMinMilliseconds", Label: "Min"},
			},
		},
		"latency": {
			Label: (labelPrefix + " Operation Latency"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "GetRecordsLatency", Label: "GetRecords"},
				{Name: "PutRecordLatency", Label: "PutRecord"},
				{Name: "PutRecordsLatency", Label: "PutRecords"},
			},
		},
		"records": {
			Label: (labelPrefix + " Records"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "GetRecordsRecords", Label: "GetRecords"},
				{Name: "IncomingRecords", Label: "Total Incoming"},
				{Name: "PutRecordsRecords", Label: "PutRecords"},
			},
		},
		"success": {
			Label: (labelPrefix + " Operation Success"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "GetRecordsSuccess", Label: "GetRecords"},
				{Name: "PutRecordSuccess", Label: "PutRecord"},
				{Name: "PutRecordsSuccess", Label: "PutRecords"},
			},
		},
		"pending": {
			Label: (labelPrefix + " Pending Operations"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "ReadThroughputExceeded", Label: "Read"},
				{Name: "WriteThroughputExceeded", Label: "Write"},
			},
		},
	}
	return graphdef
}

// Do the plugin
func Do() {
	optAccessKeyID := flag.String("access-key-id", "", "AWS Access Key ID")
	optSecretAccessKey := flag.String("secret-access-key", "", "AWS Secret Access Key")
	optRegion := flag.String("region", "", "AWS Region")
	optIdentifier := flag.String("identifier", "", "Stream Name")
	optTempfile := flag.String("tempfile", "", "Temp file name")
	optPrefix := flag.String("metric-key-prefix", "kinesis-streams", "Metric key prefix")
	flag.Parse()

	var plugin KinesisStreamsPlugin

	plugin.AccessKeyID = *optAccessKeyID
	plugin.SecretAccessKey = *optSecretAccessKey
	plugin.Region = *optRegion
	plugin.Name = *optIdentifier
	plugin.Prefix = *optPrefix

	err := plugin.prepare()
	if err != nil {
		log.Fatalln(err)
	}

	helper := mp.NewMackerelPlugin(plugin)
	helper.Tempfile = *optTempfile

	helper.Run()
}
<|endoftext|>


<commit_before>// dockerstatus provides a few functions for getting very simple data out
// of Docker, mostly for use in simple status checks.
package dockerstatus

import (
	"github.com/CiscoCloud/distributive/tabular"
	"os/exec"
	"strings"
)

// DockerImageRepositories returns a slice of the names of the Docker images
// present on the host (what's under the REPOSITORIES column of `docker images`)
func DockerImageRepositories() (images []string, err error) {
	cmd := exec.Command("docker", "images")
	out, err := cmd.CombinedOutput()
	if err != nil {
		// try escalating to sudo, the error might have been one of permissions
		cmd = exec.Command("sudo", "docker", "images")
		out, err = cmd.CombinedOutput()
		if err != nil {
			return images, err
		}
	}
	table := tabular.ProbabalisticSplit(string(out))
	return tabular.GetColumnByHeader("REPOSITORIES", table), nil
}

// RunningContainers returns a list of names of running docker containers
// (what's under the IMAGE column of `docker ps -a` if it has status "Up".
func RunningContainers() (containers []string, err error) {
	cmd := exec.Command("docker", "ps", "-a")
	out, err := cmd.CombinedOutput()
	if err != nil {
		cmd = exec.Command("sudo", "docker", "ps", "-a")
		out, err = cmd.CombinedOutput()
		if err != nil {
			return containers, err
		}
	}
	// the output of `docker ps -a` has spaces in columns, but each column
	// is separated by 2 or more spaces. Just what Probabalistic was made for!
	lines := tabular.ProbabalisticSplit(string(out))
	names := tabular.GetColumnByHeader("IMAGE", lines)
	statuses := tabular.GetColumnByHeader("STATUS", lines)
	for i, status := range statuses {
		// index error caught by second condition in if clause
		if strings.Contains(status, "Up") && len(names) > i {
			containers = append(containers, names[i])
		}
	}
	return containers, nil
}
<commit_msg>fix typo in dockerimages - repositor(ies -> y)<commit_after>// dockerstatus provides a few functions for getting very simple data out
// of Docker, mostly for use in simple status checks.
package dockerstatus

import (
	"os/exec"
	"strings"

	"github.com/CiscoCloud/distributive/tabular"
)

// DockerImageRepositories returns a slice of the names of the Docker images
// present on the host (what's under the REPOSITORIES column of `docker images`)
func DockerImageRepositories() (images []string, err error) {
	cmd := exec.Command("docker", "images")
	out, err := cmd.CombinedOutput()
	if err != nil {
		// try escalating to sudo, the error might have been one of permissions
		cmd = exec.Command("sudo", "docker", "images")
		out, err = cmd.CombinedOutput()
		if err != nil {
			return images, err
		}
	}
	table := tabular.ProbabalisticSplit(string(out))
	return tabular.GetColumnByHeader("REPOSITORY", table), nil
}

// RunningContainers returns a list of names of running docker containers
// (what's under the IMAGE column of `docker ps -a` if it has status "Up".
func RunningContainers() (containers []string, err error) {
	cmd := exec.Command("docker", "ps", "-a")
	out, err := cmd.CombinedOutput()
	if err != nil {
		cmd = exec.Command("sudo", "docker", "ps", "-a")
		out, err = cmd.CombinedOutput()
		if err != nil {
			return containers, err
		}
	}
	// the output of `docker ps -a` has spaces in columns, but each column
	// is separated by 2 or more spaces. Just what Probabalistic was made for!
	lines := tabular.ProbabalisticSplit(string(out))
	names := tabular.GetColumnByHeader("IMAGE", lines)
	statuses := tabular.GetColumnByHeader("STATUS", lines)
	for i, status := range statuses {
		// index error caught by second condition in if clause
		if strings.Contains(status, "Up") && len(names) > i {
			containers = append(containers, names[i])
		}
	}
	return containers, nil
}
<|endoftext|>


<commit_before>package hashmap

import (
	"sync/atomic"
	"unsafe"
)

// List is a sorted doubly linked list.
type List struct {
	count uintptr
	head  *ListElement
}

// NewList returns an initialized list.
func NewList() *List {
	return &List{head: &ListElement{}}
}

// Len returns the number of elements within the list.
func (l *List) Len() int {
	if l == nil { // not initialized yet?
		return 0
	}

	return int(atomic.LoadUintptr(&l.count))
}

// First returns the first item of the list.
func (l *List) First() *ListElement {
	return l.head.Next()
}

// Add adds an item to the list and returns false if an item for the hash existed.
func (l *List) Add(element *ListElement, searchStart *ListElement) (existed bool, inserted bool) {
	if searchStart == nil || element.keyHash < searchStart.keyHash { // key needs to be inserted on the left? {
		searchStart = nil // start search at root
	}

	left, found, right := l.search(searchStart, element)
	if found != nil { // existing item found
		return true, false
	}

	return false, l.insertAt(element, left, right)
}

// AddOrUpdate adds or updates an item to the list.
func (l *List) AddOrUpdate(element *ListElement, searchStart *ListElement) bool {
	if searchStart == nil || element.keyHash < searchStart.keyHash { // key needs to be inserted on the left? {
		searchStart = nil // start search at root
	}

	left, found, right := l.search(searchStart, element)
	if found != nil { // existing item found
		found.SetValue(element.value) // update the value
		return true
	}

	return l.insertAt(element, left, right)
}

// Cas compares and swaps the value of an item in the list.
func (l *List) Cas(element *ListElement, oldValue unsafe.Pointer, searchStart *ListElement) bool {
	if searchStart == nil || element.keyHash < searchStart.keyHash { // key needs to be inserted on the left? {
		searchStart = nil // start search at root
	}

	_, found, _ := l.search(searchStart, element)
	if found == nil { // no existing item found
		return false
	}

	if found.CasValue(oldValue, element.value) {
		atomic.AddUintptr(&l.count, 1)
		return true
	}
	return false
}

func (l *List) search(searchStart *ListElement, item *ListElement) (left *ListElement, found *ListElement, right *ListElement) {
	if searchStart == nil { // start search at head?
		left = l.head
		found = left.Next()
		if found == nil { // no items beside head?
			return nil, nil, nil
		}
	} else {
		found = searchStart
	}

	for {
		if item.keyHash == found.keyHash { // key already exists
			return nil, found, nil
		}

		if item.keyHash < found.keyHash { // new item needs to be inserted before the found value
			return left, nil, found
		}

		// go to next element in sorted linked list
		left = found
		found = left.Next()
		if found == nil { // no more items on the right
			return left, nil, nil
		}
	}
}

func (l *List) insertAt(element *ListElement, left *ListElement, right *ListElement) bool {
	if left == nil { // insert at head
		if !atomic.CompareAndSwapPointer(&l.head.nextElement, unsafe.Pointer(nil), unsafe.Pointer(element)) {
			return false // item was modified concurrently
		}
	} else {
		element.previousElement = unsafe.Pointer(left)
		element.nextElement = unsafe.Pointer(right)
		if !atomic.CompareAndSwapPointer(&left.nextElement, unsafe.Pointer(right), unsafe.Pointer(element)) {
			return false // item was modified concurrently
		}
	}

	atomic.AddUintptr(&l.count, 1)
	return true
}

// Delete marks the list element as deleted.
func (l *List) Delete(element *ListElement) {
	for {
		left := element.Previous()
		right := element.Next()
		if left != nil {
			if !atomic.CompareAndSwapPointer(&left.nextElement, unsafe.Pointer(element), unsafe.Pointer(right)) {
				continue // item was modified concurrently
			}
		}
		break
	}

	atomic.AddUintptr(&l.count, ^uintptr(0)) // decrease counter
}
<commit_msg>Moved searchStart check into search subfunction<commit_after>package hashmap

import (
	"sync/atomic"
	"unsafe"
)

// List is a sorted doubly linked list.
type List struct {
	count uintptr
	head  *ListElement
}

// NewList returns an initialized list.
func NewList() *List {
	return &List{head: &ListElement{}}
}

// Len returns the number of elements within the list.
func (l *List) Len() int {
	if l == nil { // not initialized yet?
		return 0
	}

	return int(atomic.LoadUintptr(&l.count))
}

// First returns the first item of the list.
func (l *List) First() *ListElement {
	if l == nil { // not initialized yet?
		return nil
	}

	return l.head.Next()
}

// Add adds an item to the list and returns false if an item for the hash existed.
// searchStart = nil will start to search at the head item
func (l *List) Add(element *ListElement, searchStart *ListElement) (existed bool, inserted bool) {
	left, found, right := l.search(searchStart, element)
	if found != nil { // existing item found
		return true, false
	}

	return false, l.insertAt(element, left, right)
}

// AddOrUpdate adds or updates an item to the list.
func (l *List) AddOrUpdate(element *ListElement, searchStart *ListElement) bool {
	left, found, right := l.search(searchStart, element)
	if found != nil { // existing item found
		found.SetValue(element.value) // update the value
		return true
	}

	return l.insertAt(element, left, right)
}

// Cas compares and swaps the value of an item in the list.
func (l *List) Cas(element *ListElement, oldValue unsafe.Pointer, searchStart *ListElement) bool {
	_, found, _ := l.search(searchStart, element)
	if found == nil { // no existing item found
		return false
	}

	if found.CasValue(oldValue, element.value) {
		atomic.AddUintptr(&l.count, 1)
		return true
	}
	return false
}

func (l *List) search(searchStart *ListElement, item *ListElement) (left *ListElement, found *ListElement, right *ListElement) {
	if searchStart != nil && item.keyHash < searchStart.keyHash { // key would remain left from item? {
		searchStart = nil // start search at head
	}

	if searchStart == nil { // start search at head?
		left = l.head
		found = left.Next()
		if found == nil { // no items beside head?
			return nil, nil, nil
		}
	} else {
		found = searchStart
	}

	for {
		if item.keyHash == found.keyHash { // key already exists
			return nil, found, nil
		}

		if item.keyHash < found.keyHash { // new item needs to be inserted before the found value
			return left, nil, found
		}

		// go to next element in sorted linked list
		left = found
		found = left.Next()
		if found == nil { // no more items on the right
			return left, nil, nil
		}
	}
}

func (l *List) insertAt(element *ListElement, left *ListElement, right *ListElement) bool {
	if left == nil { // insert at head
		if !atomic.CompareAndSwapPointer(&l.head.nextElement, unsafe.Pointer(nil), unsafe.Pointer(element)) {
			return false // item was modified concurrently
		}
	} else {
		element.previousElement = unsafe.Pointer(left)
		element.nextElement = unsafe.Pointer(right)
		if !atomic.CompareAndSwapPointer(&left.nextElement, unsafe.Pointer(right), unsafe.Pointer(element)) {
			return false // item was modified concurrently
		}
	}

	atomic.AddUintptr(&l.count, 1)
	return true
}

// Delete marks the list element as deleted.
func (l *List) Delete(element *ListElement) {
	for {
		left := element.Previous()
		right := element.Next()
		if left != nil {
			if !atomic.CompareAndSwapPointer(&left.nextElement, unsafe.Pointer(element), unsafe.Pointer(right)) {
				continue // item was modified concurrently
			}
		}
		break
	}

	atomic.AddUintptr(&l.count, ^uintptr(0)) // decrease counter
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"
	"text/tabwriter"

	"github.com/koding/kite"
	"github.com/mitchellh/cli"
)

func ListCommandFactory(k *kite.Client) cli.CommandFactory {
	return func() (cli.Command, error) {
		return &ListCommand{
			k: k,
		}, nil
	}
}

type ListCommand struct {
	k *kite.Client
}

func (c *ListCommand) Run(_ []string) int {
	err := c.k.Dial()
	if err != nil {
		log.Fatal(err)
		return 1
	}

	res, err := c.k.Tell("remote.list")
	if err != nil {
		log.Fatal(err)
		return 1
	}

	type kiteInfo struct {
		// The Ip of the running machine
		Ip       string
		Hostname string
	}

	var infos []kiteInfo
	res.Unmarshal(&infos)

	w := tabwriter.NewWriter(os.Stdout, 2, 0, 1, ' ', 0)
	for i, info := range infos {
		// TODO: UX: Decide how this should be presented to the user
		fmt.Fprintf(w, "  %d.\t%s\t[%s]\n", i+1, info.Ip, info.Hostname)
	}
	w.Flush()

	return 1
}

func (*ListCommand) Help() string {
	helpText := `
Usage: %s list

	List the available machines.
`
	return fmt.Sprintf(helpText, Name)
}

func (*ListCommand) Synopsis() string {
	return fmt.Sprintf("List the available machines")
}
<commit_msg>klientctl: Tweaked the display of list<commit_after>package main

import (
	"fmt"
	"log"
	"os"
	"text/tabwriter"

	"github.com/koding/kite"
	"github.com/mitchellh/cli"
)

func ListCommandFactory(k *kite.Client) cli.CommandFactory {
	return func() (cli.Command, error) {
		return &ListCommand{
			k: k,
		}, nil
	}
}

type ListCommand struct {
	k *kite.Client
}

func (c *ListCommand) Run(_ []string) int {
	err := c.k.Dial()
	if err != nil {
		log.Fatal(err)
		return 1
	}

	res, err := c.k.Tell("remote.list")
	if err != nil {
		log.Fatal(err)
		return 1
	}

	type kiteInfo struct {
		// The Ip of the running machine
		Ip       string
		Hostname string
	}

	var infos []kiteInfo
	res.Unmarshal(&infos)

	w := tabwriter.NewWriter(os.Stdout, 2, 0, 2, ' ', 0)
	fmt.Fprintf(w, "\tMACHINE IP\tHOSTNAME\n")
	for i, info := range infos {
		// TODO: UX: Decide how this should be presented to the user
		fmt.Fprintf(w, "  %d.\t%s\t%s\n", i+1, info.Ip, info.Hostname)
	}
	w.Flush()

	return 1
}

func (*ListCommand) Help() string {
	helpText := `
Usage: %s list

	List the available machines.
`
	return fmt.Sprintf(helpText, Name)
}

func (*ListCommand) Synopsis() string {
	return fmt.Sprintf("List the available machines")
}
<|endoftext|>