<commit_before>package kloud

import (
	"fmt"
	"koding/db/mongodb"
	"koding/kites/kloud/eventer"
	"koding/kites/kloud/idlock"
	"koding/kites/kloud/kloud/protocol"
	"koding/kites/kloud/provider/digitalocean"
	"koding/kodingkite"
	"koding/tools/config"
	"log"
	"os"

	"github.com/koding/kite"
	"github.com/koding/logging"
	"github.com/koding/redis"
)

const (
	VERSION = "0.0.1"
	NAME    = "kloud"
)

var (
	providers = make(map[string]protocol.Provider)
)

type Kloud struct {
	Config *config.Config
	Log    logging.Logger
	Kite   *kite.Kite

	Storage  Storage
	Eventers map[string]eventer.Eventer

	idlock *idlock.IdLock

	Name    string
	Version string
	Region  string
	Port    int

	// Used to uniquely identifiy kloud instances
	UniqueId string

	// needed for signing/generating kite tokens
	KontrolPublicKey  string
	KontrolPrivateKey string
	KontrolURL        string

	Debug bool
}

func (k *Kloud) NewKloud() *kodingkite.KodingKite {
	if k.Config == nil {
		panic("config is not initialized")
	}

	k.Name = NAME
	k.Version = VERSION

	k.idlock = idlock.New()

	kt, err := kodingkite.New(k.Config, k.Name, k.Version)
	if err != nil {
		log.Fatalln(err)
	}
	k.Kite = kt.Kite

	if k.Log == nil {
		k.Log = createLogger(NAME, k.Debug)
	}

	if k.UniqueId == "" {
		k.UniqueId = uniqueId()
	}

	if k.Storage == nil {
		mongodbSession := &MongoDB{
			session:  mongodb.NewMongoDB(k.Config.Mongo),
			assignee: k.UniqueId,
			log:      k.Log,
		}

		if err := mongodbSession.CleanupOldData(); err != nil {
			k.Log.Notice("Cleaning up mongodb err: %s", err.Error())
		}

		k.Storage = mongodbSession
	}

	if k.Eventers == nil {
		k.Eventers = make(map[string]eventer.Eventer)
	}

	kt.Config.Region = k.Region
	kt.Config.Port = k.Port

	k.ControlFunc("build", k.build)
	k.ControlFunc("start", k.start)
	k.ControlFunc("stop", k.stop)
	k.ControlFunc("restart", k.restart)
	k.ControlFunc("destroy", k.destroy)
	k.ControlFunc("info", k.info)
	kt.HandleFunc("event", k.event)

	k.InitializeProviders()

	return kt
}

func (k *Kloud) SignFunc(username string) (string, string, error) {
	k.Log.Debug("Signing a key for user: '%s' kontrolURL: %s ", username, k.KontrolURL)
	return createKey(username, k.KontrolURL, k.KontrolPrivateKey, k.KontrolPublicKey)
}

func (k *Kloud) GetProvider(providerName string) (protocol.Provider, error) {
	provider, ok := providers[providerName]
	if !ok {
		return nil, NewError(ErrProviderNotFound)
	}

	return provider, nil
}

func (k *Kloud) InitializeProviders() {
	r := redis.MustRedisSession(&redis.RedisConf{Server: k.Config.Redis})
	prefix := fmt.Sprintf("%s:%s", NAME, k.Kite.Config.Environment)
	r.SetPrefix(prefix)

	providers = map[string]protocol.Provider{
		"digitalocean": &digitalocean.Provider{
			Log:         createLogger("digitalocean", k.Debug),
			SignFunc:    k.SignFunc,
			Redis:       r,
			Region:      k.Region,
			Environment: k.Config.Environment,
		},
	}
}

func uniqueId() string {
	// TODO: add a unique identifier, for letting multiple version of the same
	// worker work on the same hostname.
	hostname, err := os.Hostname()
	if err != nil {
		panic(err) // we should not let it start
	}

	return fmt.Sprintf("%s-%s", NAME, hostname)
}

func createLogger(name string, debug bool) logging.Logger {
	log := logging.NewLogger(name)
	logHandler := logging.NewWriterHandler(os.Stderr)
	logHandler.Colorize = true
	log.SetHandler(logHandler)

	if debug {
		fmt.Println("DEBUG mode is enabled.")
		log.SetLevel(logging.DEBUG)
		logHandler.SetLevel(logging.DEBUG)
	}

	return log
}
<commit_msg>kloud/digitalocean: disable api logging<commit_after>package kloud

import (
	"fmt"
	"io/ioutil"
	"koding/db/mongodb"
	"koding/kites/kloud/eventer"
	"koding/kites/kloud/idlock"
	"koding/kites/kloud/kloud/protocol"
	"koding/kites/kloud/provider/digitalocean"
	"koding/kodingkite"
	"koding/tools/config"
	"log"
	"os"

	"github.com/koding/kite"
	"github.com/koding/logging"
	"github.com/koding/redis"
)

const (
	VERSION = "0.0.1"
	NAME    = "kloud"
)

var (
	providers = make(map[string]protocol.Provider)
)

type Kloud struct {
	Config *config.Config
	Log    logging.Logger
	Kite   *kite.Kite

	Storage  Storage
	Eventers map[string]eventer.Eventer

	idlock *idlock.IdLock

	Name    string
	Version string
	Region  string
	Port    int

	// Used to uniquely identifiy kloud instances
	UniqueId string

	// needed for signing/generating kite tokens
	KontrolPublicKey  string
	KontrolPrivateKey string
	KontrolURL        string

	Debug bool
}

func (k *Kloud) NewKloud() *kodingkite.KodingKite {
	if k.Config == nil {
		panic("config is not initialized")
	}

	k.Name = NAME
	k.Version = VERSION

	k.idlock = idlock.New()

	kt, err := kodingkite.New(k.Config, k.Name, k.Version)
	if err != nil {
		log.Fatalln(err)
	}
	k.Kite = kt.Kite

	if k.Log == nil {
		k.Log = createLogger(NAME, k.Debug)
	}

	if k.UniqueId == "" {
		k.UniqueId = uniqueId()
	}

	if k.Storage == nil {
		mongodbSession := &MongoDB{
			session:  mongodb.NewMongoDB(k.Config.Mongo),
			assignee: k.UniqueId,
			log:      k.Log,
		}

		if err := mongodbSession.CleanupOldData(); err != nil {
			k.Log.Notice("Cleaning up mongodb err: %s", err.Error())
		}

		k.Storage = mongodbSession
	}

	if k.Eventers == nil {
		k.Eventers = make(map[string]eventer.Eventer)
	}

	kt.Config.Region = k.Region
	kt.Config.Port = k.Port

	k.ControlFunc("build", k.build)
	k.ControlFunc("start", k.start)
	k.ControlFunc("stop", k.stop)
	k.ControlFunc("restart", k.restart)
	k.ControlFunc("destroy", k.destroy)
	k.ControlFunc("info", k.info)
	kt.HandleFunc("event", k.event)

	k.InitializeProviders()

	return kt
}

func (k *Kloud) SignFunc(username string) (string, string, error) {
	k.Log.Debug("Signing a key for user: '%s' kontrolURL: %s ", username, k.KontrolURL)
	return createKey(username, k.KontrolURL, k.KontrolPrivateKey, k.KontrolPublicKey)
}

func (k *Kloud) GetProvider(providerName string) (protocol.Provider, error) {
	provider, ok := providers[providerName]
	if !ok {
		return nil, NewError(ErrProviderNotFound)
	}

	return provider, nil
}

func (k *Kloud) InitializeProviders() {
	r := redis.MustRedisSession(&redis.RedisConf{Server: k.Config.Redis})
	prefix := fmt.Sprintf("%s:%s", NAME, k.Kite.Config.Environment)
	r.SetPrefix(prefix)

	// Our digitalocean api uses lots of logs, the only way to supress them is
	// to disable std log package.
	log.SetOutput(ioutil.Discard)

	providers = map[string]protocol.Provider{
		"digitalocean": &digitalocean.Provider{
			Log:         createLogger("digitalocean", k.Debug),
			SignFunc:    k.SignFunc,
			Redis:       r,
			Region:      k.Region,
			Environment: k.Config.Environment,
		},
	}
}

func uniqueId() string {
	// TODO: add a unique identifier, for letting multiple version of the same
	// worker work on the same hostname.
	hostname, err := os.Hostname()
	if err != nil {
		panic(err) // we should not let it start
	}

	return fmt.Sprintf("%s-%s", NAME, hostname)
}

func createLogger(name string, debug bool) logging.Logger {
	log := logging.NewLogger(name)
	logHandler := logging.NewWriterHandler(os.Stderr)
	logHandler.Colorize = true
	log.SetHandler(logHandler)

	if debug {
		fmt.Println("DEBUG mode is enabled.")
		log.SetLevel(logging.DEBUG)
		logHandler.SetLevel(logging.DEBUG)
	}

	return log
}
<|endoftext|>


<commit_before>package intfns

import (
	"encoding/xml"
	"fmt"

	"github.com/ChrisTrenkamp/goxpath/goxpath/xpfn"
	"github.com/ChrisTrenkamp/goxpath/goxpath/xpfn/noarg"
	"github.com/ChrisTrenkamp/goxpath/goxpath/xpfn/optarg"
	"github.com/ChrisTrenkamp/goxpath/tree"
	"github.com/ChrisTrenkamp/goxpath/tree/literals/numlit"
	"github.com/ChrisTrenkamp/goxpath/tree/literals/strlit"
)

//BuiltIn contains the list of built-in XPath functions
var BuiltIn = map[string]xpfn.XPFn{
	"last":       noarg.Wrap{Fn: last},
	"count":      noarg.Wrap{Fn: count},
	"local-name": optarg.Wrap{Fn: name},
}

func last(arg []tree.Res) ([]tree.Res, error) {
	return []tree.Res{numlit.NumLit(len(arg))}, nil
}

func count(arg []tree.Res) ([]tree.Res, error) {
	return []tree.Res{numlit.NumLit(len(arg))}, nil
}

func name(arg []tree.Res) ([]tree.Res, error) {
	if len(arg) == 0 {
		return nil, fmt.Errorf("No node in argument.")
	}

	node, ok := arg[0].(tree.Node)

	if !ok {
		return nil, fmt.Errorf("Argument is not a node")
	}

	tok := node.GetToken()

	switch t := tok.(type) {
	case xml.StartElement:
		if t.Name.Local == "" {
			return nil, fmt.Errorf("Cannot run local-name on root node.")
		}
		ret := []tree.Res{strlit.StrLit(t.Name.Local)}
		return ret, nil
	case xml.Attr:
		ret := []tree.Res{strlit.StrLit(t.Name.Local)}
		return ret, nil
	case xml.ProcInst:
		ret := []tree.Res{strlit.StrLit(t.Target)}
		return ret, nil
	}

	return []tree.Res{strlit.StrLit("")}, nil
}
<commit_msg>Fixing last() function.<commit_after>package intfns

import (
	"encoding/xml"
	"fmt"

	"github.com/ChrisTrenkamp/goxpath/goxpath/xpfn"
	"github.com/ChrisTrenkamp/goxpath/goxpath/xpfn/noarg"
	"github.com/ChrisTrenkamp/goxpath/goxpath/xpfn/optarg"
	"github.com/ChrisTrenkamp/goxpath/tree"
	"github.com/ChrisTrenkamp/goxpath/tree/literals/numlit"
	"github.com/ChrisTrenkamp/goxpath/tree/literals/strlit"
)

//BuiltIn contains the list of built-in XPath functions
var BuiltIn = map[string]xpfn.XPFn{
	"last":       noarg.Wrap{Fn: last},
	"count":      noarg.Wrap{Fn: count},
	"local-name": optarg.Wrap{Fn: name},
}

func last(arg []tree.Res) ([]tree.Res, error) {
	if len(arg) == 0 {
		return nil, nil
	}

	return []tree.Res{arg[len(arg)-1]}, nil
}

func count(arg []tree.Res) ([]tree.Res, error) {
	return []tree.Res{numlit.NumLit(len(arg))}, nil
}

func name(arg []tree.Res) ([]tree.Res, error) {
	if len(arg) == 0 {
		return nil, fmt.Errorf("No node in argument.")
	}

	node, ok := arg[0].(tree.Node)

	if !ok {
		return nil, fmt.Errorf("Argument is not a node")
	}

	tok := node.GetToken()

	switch t := tok.(type) {
	case xml.StartElement:
		if t.Name.Local == "" {
			return nil, fmt.Errorf("Cannot run local-name on root node.")
		}
		ret := []tree.Res{strlit.StrLit(t.Name.Local)}
		return ret, nil
	case xml.Attr:
		ret := []tree.Res{strlit.StrLit(t.Name.Local)}
		return ret, nil
	case xml.ProcInst:
		ret := []tree.Res{strlit.StrLit(t.Target)}
		return ret, nil
	}

	return []tree.Res{strlit.StrLit("")}, nil
}
<|endoftext|>


<commit_before>// Copyright 2015-2018 Kuzzle
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// 		http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package realtime

import (
	"encoding/json"

	"github.com/kuzzleio/sdk-go/types"
)

// Count returns the number of other subscriptions on that room.
func (r *Realtime) Count(roomID string, options types.QueryOptions) (int, error) {
	if roomID == "" {
		return -1, types.NewError("Realtime.Count: roomID required", 400)
	}

	query := &types.KuzzleRequest{
		Controller: "realtime",
		Action:     "count",
		Body: struct {
			RoomID string `json:"roomID"`
		}{roomID},
	}

	result := make(chan *types.KuzzleResponse)

	go r.k.Query(query, options, result)

	res := <-result

	if res.Error.Error() != "" {
		return -1, res.Error
	}

	var countRes struct {
		Count int `json:"count"`
	}

	json.Unmarshal(res.Result, &countRes)

	return countRes.Count, nil
}
<commit_msg>Fix body argument name for realtime:count<commit_after>// Copyright 2015-2018 Kuzzle
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// 		http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package realtime

import (
	"encoding/json"

	"github.com/kuzzleio/sdk-go/types"
)

// Count returns the number of other subscriptions on that room.
func (r *Realtime) Count(roomID string, options types.QueryOptions) (int, error) {
	if roomID == "" {
		return -1, types.NewError("Realtime.Count: roomID required", 400)
	}

	query := &types.KuzzleRequest{
		Controller: "realtime",
		Action:     "count",
		Body: struct {
			RoomID string `json:"roomId"`
		}{roomID},
	}

	result := make(chan *types.KuzzleResponse)

	go r.k.Query(query, options, result)

	res := <-result

	if res.Error.Error() != "" {
		return -1, res.Error
	}

	var countRes struct {
		Count int `json:"count"`
	}

	json.Unmarshal(res.Result, &countRes)

	return countRes.Count, nil
}
<|endoftext|>


<commit_before>package stacks

import "fmt"

// cleanup functions in create_failed or delete_failed states
func (s *Stack) cleanup() error {
	Log.Info(fmt.Sprintf("Running stack cleanup on [%s]", s.Name))
	resp, err := s.State()
	if err != nil {
		return err
	}

	if resp == state.failed {
		if err := s.terminate(); err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>updated info log print to debug for cleanup function<commit_after>package stacks

import "fmt"

// cleanup functions in create_failed or delete_failed states
func (s *Stack) cleanup() error {
	Log.Debug(fmt.Sprintf("Running stack cleanup on [%s]", s.Name))
	resp, err := s.State()
	if err != nil {
		return err
	}

	if resp == state.failed {
		if err := s.terminate(); err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package stacks

import (
	"bytes"
	"text/template"
)

// DeployTimeParser - Parses templates during deployment to resolve specfic Dependency functions like stackout...
func (s *Stack) DeployTimeParser() error {

	// define Delims
	left, right := s.delims("deploy")

	// Create template
	t, err := template.New("deploy-template").Delims(left, right).Funcs(*s.DeployTimeFunc).Parse(s.Template)
	if err != nil {
		return err
	}

	// so that we can write to string
	var doc bytes.Buffer

	// Add metadata specific to the stack we're working with to the parser
	s.TemplateValues["stack"] = s.TemplateValues[s.Name]
	s.TemplateValues["parameters"] = s.Parameters
	s.TemplateValues["name"] = s.Name

	t.Execute(&doc, s.TemplateValues)
	s.Template = doc.String()
	Log.Debug("Deploy Time Template Generate:\n%s", s.Template)

	return nil
}

// GenTimeParser - Parses templates before deploying them...
func (s *Stack) GenTimeParser() error {

	if err := s.FetchContent(); err != nil {
		return err
	}

	// define Delims
	left, right := s.delims("gen")

	// create template
	t, err := template.New("gen-template").Delims(left, right).Funcs(*s.GenTimeFunc).Parse(s.Template)
	if err != nil {
		return err
	}

	// so that we can write to string
	var doc bytes.Buffer

	// Add metadata specific to the stack we're working with to the parser
	s.TemplateValues["stack"] = s.TemplateValues[s.Name]
	s.TemplateValues["parameters"] = s.Parameters
	s.TemplateValues["name"] = s.Name

	t.Execute(&doc, s.TemplateValues)
	s.Template = doc.String()
	return nil
}
<commit_msg>added comments outlining potential sprig template funcMap integration<commit_after>package stacks

import (
	"bytes"
	"text/template"
)

// DeployTimeParser - Parses templates during deployment to resolve specfic Dependency functions like stackout...
func (s *Stack) DeployTimeParser() error {

	// define Delims
	left, right := s.delims("deploy")

	// Create template
	// t, err := template.New("deploy-template").Delims(left, right).Funcs(*s.DeployTimeFunc).Funcs(sprig.FuncMap()).Parse(s.Template)
	t, err := template.New("deploy-template").Delims(left, right).Funcs(*s.DeployTimeFunc).Parse(s.Template)
	if err != nil {
		return err
	}

	// so that we can write to string
	var doc bytes.Buffer

	// Add metadata specific to the stack we're working with to the parser
	s.TemplateValues["stack"] = s.TemplateValues[s.Name]
	s.TemplateValues["parameters"] = s.Parameters
	s.TemplateValues["name"] = s.Name

	t.Execute(&doc, s.TemplateValues)
	s.Template = doc.String()
	Log.Debug("Deploy Time Template Generate:\n%s", s.Template)

	return nil
}

// GenTimeParser - Parses templates before deploying them...
func (s *Stack) GenTimeParser() error {

	if err := s.FetchContent(); err != nil {
		return err
	}

	// define Delims
	left, right := s.delims("gen")

	// create template
	t, err := template.New("gen-template").Delims(left, right).Funcs(*s.GenTimeFunc).Parse(s.Template)
	// t, err := template.New("gen-template").Delims(left, right).Funcs(*s.GenTimeFunc).Funcs(sprig.FuncMap()).Parse(s.Template)

	if err != nil {
		return err
	}

	// so that we can write to string
	var doc bytes.Buffer

	// Add metadata specific to the stack we're working with to the parser
	s.TemplateValues["stack"] = s.TemplateValues[s.Name]
	s.TemplateValues["parameters"] = s.Parameters
	s.TemplateValues["name"] = s.Name

	t.Execute(&doc, s.TemplateValues)
	s.Template = doc.String()
	return nil
}
<|endoftext|>


<commit_before>// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements;
// and to You under the Apache License, Version 2.0.  See LICENSE in project root for full license + copyright.

package keynuker

import (
	"fmt"
	"testing"

	"log"
	"os"

	"time"

	"context"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/iam"
	"github.com/google/go-github/github"
	"github.com/tleyden/keynuker/keynuker-go-common"
)

// - Create a test AWS user w/ minimal permissions
//     - Note: this was simplified and so it just re-uses the KeyNuker user and leaks a key under that account
// - Loop over leaked key scenarios
//    - Create AWS key #1
//    - Scenario 1: Commit and push text file to github private repo w/ leaked key
//    - Invoke keynuker
//    - Verify that the AWS key #1 was nuked
//    - Create AWS key #2
//    - Scenario 1: Create a secret gist w/ leaked key
//    - Verify that the AWS key #2 was nuked
// - Cleanup test user
// - Cleanup other residue (gists, etc)
func TestEndToEndIntegration(t *testing.T) {

	SkipIfIntegrationsTestsNotEnabled(t)

	endToEndIntegrationTest := NewEndToEndIntegrationTest()

	// Setup
	if err := endToEndIntegrationTest.InitAwsIamSession(); err != nil {
		t.Fatalf("Error setting up test: %v", err)
	}
	if err := endToEndIntegrationTest.InitGithubAccess(); err != nil {
		t.Fatalf("Error setting up test: %v", err)
	}

	// Run the full end-to-end integration test
	if err := endToEndIntegrationTest.Run(); err != nil {
		t.Fatalf("Error running test: %v", err)
	}

}

type KeyLeakScenario interface {
	Leak(accessKey *iam.AccessKey) error
	Cleanup() error
}

type LeakKeyViaNewGithubIssue struct {
	GithubAccessToken        string
	GithubRepoLeakTargetRepo string
}

func NewLeakKeyViaCommit(githubAccessToken, targetGithubRepo string) *LeakKeyViaNewGithubIssue {
	return &LeakKeyViaNewGithubIssue{
		GithubAccessToken:        githubAccessToken,
		GithubRepoLeakTargetRepo: targetGithubRepo,
	}
}

func (lkvc LeakKeyViaNewGithubIssue) Leak(accessKey *iam.AccessKey) error {

	githubApiClient := NewGithubClientWrapper(lkvc.GithubAccessToken)

	ctx := context.Background()

	user, _, err := githubApiClient.ApiClient.Users.Get(ctx, "")
	if err != nil {
		return err
	}
	username := *user.Name

	log.Printf("github login: %v, name: %v", *user.Login, username)

	issueRequest := &github.IssueRequest{
		Title: aws.String("KeyNuker Leaked Key 🔐 End-to-End Test"),
		Body:  aws.String(fmt.Sprintf("Nukable 🔐💥 Key: %v.  Keynuker Project url: github.com/tleyden/keynuker", *accessKey.AccessKeyId)),
	}
	_, _, err = githubApiClient.ApiClient.Issues.Create(ctx, *user.Login, lkvc.GithubRepoLeakTargetRepo, issueRequest)
	if err != nil {
		return err
	}

	return nil

}

func (lkvc LeakKeyViaNewGithubIssue) Cleanup() error {
	return nil
}

func (e EndToEndIntegrationTest) GetEndToEndKeyLeakScenarios() []KeyLeakScenario {
	return []KeyLeakScenario{
		NewLeakKeyViaCommit(e.GithubAccessToken, e.GithubRepoLeakTargetRepo),
	}
}

type EndToEndIntegrationTest struct {
	IamUsername              string
	IamService               *iam.IAM
	AwsSession               *session.Session
	TargetAwsAccount         TargetAwsAccount
	GithubAccessToken        string
	GithubOrgs               []string
	GithubRepoLeakTargetRepo string
}

func NewEndToEndIntegrationTest() *EndToEndIntegrationTest {
	return &EndToEndIntegrationTest{}
}

func (e *EndToEndIntegrationTest) InitGithubAccess() error {

	githubRepoLeakTargetRepo, ok := os.LookupEnv(keynuker_go_common.EnvVarKeyNukerTestGithubLeakTargetRepo)
	if !ok {
		return fmt.Errorf("You must define environment variable %v to run this test", keynuker_go_common.EnvVarKeyNukerTestGithubLeakTargetRepo)
	}
	e.GithubRepoLeakTargetRepo = githubRepoLeakTargetRepo

	githubAccessToken, ok := os.LookupEnv(keynuker_go_common.EnvVarKeyNukerTestGithubAccessToken)
	if !ok {
		return fmt.Errorf("You must define environment variable %v to run this test", keynuker_go_common.EnvVarKeyNukerTestGithubAccessToken)
	}
	e.GithubAccessToken = githubAccessToken

	githubOrgs, err := GetGithubOrgsFromEnv()
	if err != nil {
		return err
	}
	e.GithubOrgs = githubOrgs

	return nil

}

func (e *EndToEndIntegrationTest) InitAwsIamSession() error {

	targetAwsAccounts, err := GetTargetAwsAccountsFromEnv()
	if err != nil {
		return err
	}

	// Just use the first aws account
	e.TargetAwsAccount = targetAwsAccounts[0]

	// Create AWS session
	sess, err := session.NewSession(&aws.Config{
		Credentials: credentials.NewCredentials(
			&credentials.StaticProvider{Value: credentials.Value{
				AccessKeyID:     e.TargetAwsAccount.AwsAccessKeyId,
				SecretAccessKey: e.TargetAwsAccount.AwsSecretAccessKey,
			}},
		),
	})
	if err != nil {
		return fmt.Errorf("Error creating aws session: %v", err)
	}

	// Create IAM client with the session.
	svc := iam.New(sess)

	e.AwsSession = sess
	e.IamService = svc

	// Discover IAM username based on aws key
	username, err := e.DiscoverIAMUsernameForKey(e.TargetAwsAccount.AwsAccessKeyId)
	if err != nil {
		return err
	}

	e.IamUsername = username

	return nil

}

// List all users
// For each user
// List all keys
// If you find the AwsAccessKeyId param, return the current user
func (e EndToEndIntegrationTest) DiscoverIAMUsernameForKey(AwsAccessKeyId string) (username string, err error) {

	// Fetch list of IAM users
	iamUsers, err := FetchIAMUsers(e.IamService)
	if err != nil {
		return "", err
	}

	for _, user := range iamUsers {

		listAccessKeysInput := &iam.ListAccessKeysInput{
			UserName: user.UserName,
			MaxItems: aws.Int64(1000),
		}

		listAccessKeysOutput, err := e.IamService.ListAccessKeys(listAccessKeysInput)
		if err != nil {
			return "", fmt.Errorf("Error listing access keys for user: %v.  Err: %v", user, err)
		}

		// Panic if more than 1K results, which is not handled
		if *listAccessKeysOutput.IsTruncated {
			// TODO: remove panic and put in a paginated loop.  Move to tleyden/awsutils + unit tests against mocks
			return "", fmt.Errorf("Output is truncated and this code does not handle it")
		}

		for _, accessKeyMetadata := range listAccessKeysOutput.AccessKeyMetadata {

			if *accessKeyMetadata.AccessKeyId == AwsAccessKeyId {
				return *user.UserName, nil
			}

		}

	}

	return "", fmt.Errorf("Unable to lookup username for key")

}

func (e EndToEndIntegrationTest) Run() error {

	// Set this to true to verify that the end-to-end integration test catches a real bug
	SetArtificialErrorInjection(false)

	keyLeakScenarios := e.GetEndToEndKeyLeakScenarios()
	for _, keyLeakScenario := range keyLeakScenarios {

		awsAccessKey, err := e.CreateKeyToLeak()
		if err != nil {
			return err
		}

		if err := keyLeakScenario.Leak(awsAccessKey); err != nil {
			return fmt.Errorf("Error running testScenario: %v", err)
		}

		if err := e.RunKeyNuker(awsAccessKey); err != nil {
			return fmt.Errorf("Error running keynuker: %v", err)
		}

		nuked, err := e.VerifyKeyNuked(awsAccessKey)
		if err != nil {
			return fmt.Errorf("Error verifying key was nuked: %v", err)
		}

		if !nuked {
			e.CleanupUnNuked(awsAccessKey)
			return fmt.Errorf("Key %v should have been nuked, but it wasn't", *awsAccessKey.AccessKeyId)
		}

		if err := keyLeakScenario.Cleanup(); err != nil {
			return fmt.Errorf("Error cleaning up keyleak scenario: %v", err)
		}

	}

	return nil

}

// NOTE: the aws key will need more permissions than usual, will need to be able to create AWS keys.
// Also, the aws key must be owned by a user named "KeyNuker"
func (e EndToEndIntegrationTest) CreateKeyToLeak() (accessKey *iam.AccessKey, err error) {

	createAccessKeyInput := &iam.CreateAccessKeyInput{
		UserName: aws.String(e.IamUsername),
	}
	createAccessKeyOutput, err := e.IamService.CreateAccessKey(createAccessKeyInput)
	if err != nil {
		return nil, fmt.Errorf("Error creating access key: %v", err)
	}

	return createAccessKeyOutput.AccessKey, nil

}

// If the initial attempt to nuke the key failed for some reason, maybe a bug, nuke it here
func (e EndToEndIntegrationTest) CleanupUnNuked(accessKeyNukeFailed *iam.AccessKey) (err error) {

	// Nuke the key from AWS
	deleteAccessKeyInput := &iam.DeleteAccessKeyInput{
		AccessKeyId: accessKeyNukeFailed.AccessKeyId,
		UserName:    accessKeyNukeFailed.UserName,
	}
	_, errDelKey := e.IamService.DeleteAccessKey(deleteAccessKeyInput)

	// Only consider it an error if it's not a "KeyNotFound error", which means the key was already nuked
	if errDelKey != nil && !IsKeyNotFoundError(errDelKey) {
		return nil
	}

	return err

}

func (e EndToEndIntegrationTest) VerifyKeyNuked(nukedAccessKey *iam.AccessKey) (nuked bool, err error) {

	listAccessKeysInput := &iam.ListAccessKeysInput{
		UserName: aws.String(e.IamUsername),
		MaxItems: aws.Int64(1000),
	}

	listAccessKeysOutput, err := e.IamService.ListAccessKeys(listAccessKeysInput)
	if err != nil {
		return false, fmt.Errorf("Error listing access keys for user: %v. Err: %v", e.IamUsername, err)
	}

	// Panic if more than 1K results, which is not handled
	if *listAccessKeysOutput.IsTruncated {
		// TODO: remove panic and put in a paginated loop.  Move to tleyden/awsutils + unit tests against mocks
		return false, fmt.Errorf("Output is truncated and this code does not handle it")
	}

	for _, accessKeyMetadata := range listAccessKeysOutput.AccessKeyMetadata {

		if *accessKeyMetadata.AccessKeyId == *nukedAccessKey.AccessKeyId {
			// Ugh, found the key that was supposed to be nuked.  Something is not working.
			return false, nil
		}
	}

	return true, nil

}

func (e EndToEndIntegrationTest) RunKeyNuker(accessKeyToNuke *iam.AccessKey) (err error) {

	keyNukerOrg := keynuker_go_common.DefaultKeyNukerOrg

	// ------------------------ Fetch Aws Keys -------------------------

	targetAwsAccounts, err := GetTargetAwsAccountsFromEnv()
	if err != nil {
		return err
	}

	paramsFetchAwsKeys := ParamsFetchAwsKeys{
		KeyNukerOrg:       keyNukerOrg,
		TargetAwsAccounts: targetAwsAccounts,
	}

	fetchedAwsKeys, err := FetchAwsKeys(paramsFetchAwsKeys)
	if err != nil {
		return err
	}
	log.Printf("fetchedAwsKeys: %v", fetchedAwsKeys)

	// ------------------------ Github User Aggregator -------------------------

	paramsAggregateGithubUsers := ParamsGithubUserAggregator{
		KeyNukerOrg:       keyNukerOrg,
		GithubAccessToken: e.GithubAccessToken,
		GithubOrgs:        e.GithubOrgs,
	}

	resultAggregateGithubUsers, err := AggregateGithubUsers(paramsAggregateGithubUsers)
	if err != nil {
		return err
	}

	// ------------------------ Github User Events Scanner -------------------------

	paramsScanGithubUserEventsForAwsKeys := ParamsScanGithubUserEventsForAwsKeys{
		KeyNukerOrg:       keyNukerOrg,
		GithubAccessToken: e.GithubAccessToken,
		GithubUsers:       resultAggregateGithubUsers.Doc.GithubUsers,
		AccessKeyMetadata: fetchedAwsKeys.Doc.AccessKeyMetadata,
	}

	recentEventTimeWindow := time.Minute * -10 // Last 5 seconds would probably work too, but give it some margin of error

	paramsScanGithubUserEventsForAwsKeys = paramsScanGithubUserEventsForAwsKeys.WithDefaultCheckpoints(recentEventTimeWindow)

	fetcher := NewGoGithubUserEventFetcher(e.GithubAccessToken)

	scanner := NewGithubUserEventsScanner(fetcher)

	scanAwsKeysResults, err := scanner.ScanAwsKeys(paramsScanGithubUserEventsForAwsKeys)
	if err != nil {
		return err
	}

	// ------------------------ Nuke Leaked Aws Keys -------------------------

	log.Printf("LeakedKeyEvents: %+v", scanAwsKeysResults.LeakedKeyEvents)

	params := ParamsNukeLeakedAwsKeys{
		KeyNukerOrg:            keyNukerOrg,
		TargetAwsAccounts:      targetAwsAccounts,
		LeakedKeyEvents:        scanAwsKeysResults.LeakedKeyEvents,
		GithubEventCheckpoints: scanAwsKeysResults.GithubEventCheckpoints,
	}

	resultNukeLeakedAwsKeys, err := NukeLeakedAwsKeys(params)
	if err != nil {
		return fmt.Errorf("Error nuking leaked aws keys: %v", err)
	}

	if len(resultNukeLeakedAwsKeys.NukedKeyEvents) <= 0 {
		return fmt.Errorf("Expected a key to be nuked, but none were nuked.  result: %+v", resultNukeLeakedAwsKeys)
	}

	for _, nukedKeyEvent := range resultNukeLeakedAwsKeys.NukedKeyEvents {
		log.Printf("NukedKeyEvent: %+v", nukedKeyEvent)
		if *nukedKeyEvent.LeakedKeyEvent.AccessKeyMetadata.AccessKeyId != *accessKeyToNuke.AccessKeyId {
			return fmt.Errorf(
				"Expected to nuke: %v, but nuked: %v",
				*accessKeyToNuke.AccessKeyId,
				*nukedKeyEvent.LeakedKeyEvent.AccessKeyMetadata.AccessKeyId,
			)
		}
	}

	return nil

}
<commit_msg>Re-arrange code<commit_after>// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements;
// and to You under the Apache License, Version 2.0.  See LICENSE in project root for full license + copyright.

package keynuker

import (
	"fmt"
	"testing"

	"log"
	"os"

	"time"

	"context"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/iam"
	"github.com/google/go-github/github"
	"github.com/tleyden/keynuker/keynuker-go-common"
)

// - Create a test AWS user w/ minimal permissions
//     - Note: this was simplified and so it just re-uses the KeyNuker user and leaks a key under that account
// - Loop over leaked key scenarios
//    - Create AWS key #1
//    - Scenario 1: Commit and push text file to github private repo w/ leaked key
//    - Invoke keynuker
//    - Verify that the AWS key #1 was nuked
//    - Create AWS key #2
//    - Scenario 1: Create a secret gist w/ leaked key
//    - Verify that the AWS key #2 was nuked
// - Cleanup test user
// - Cleanup other residue (gists, etc)
func TestEndToEndIntegration(t *testing.T) {

	SkipIfIntegrationsTestsNotEnabled(t)

	endToEndIntegrationTest := NewEndToEndIntegrationTest()

	// Setup
	if err := endToEndIntegrationTest.InitAwsIamSession(); err != nil {
		t.Fatalf("Error setting up test: %v", err)
	}
	if err := endToEndIntegrationTest.InitGithubAccess(); err != nil {
		t.Fatalf("Error setting up test: %v", err)
	}

	// Run the full end-to-end integration test
	if err := endToEndIntegrationTest.Run(); err != nil {
		t.Fatalf("Error running test: %v", err)
	}

}

type EndToEndIntegrationTest struct {
	IamUsername              string
	IamService               *iam.IAM
	AwsSession               *session.Session
	TargetAwsAccount         TargetAwsAccount
	GithubAccessToken        string
	GithubOrgs               []string
	GithubRepoLeakTargetRepo string
}

func NewEndToEndIntegrationTest() *EndToEndIntegrationTest {
	return &EndToEndIntegrationTest{}
}

func (e *EndToEndIntegrationTest) InitGithubAccess() error {

	githubRepoLeakTargetRepo, ok := os.LookupEnv(keynuker_go_common.EnvVarKeyNukerTestGithubLeakTargetRepo)
	if !ok {
		return fmt.Errorf("You must define environment variable %v to run this test", keynuker_go_common.EnvVarKeyNukerTestGithubLeakTargetRepo)
	}
	e.GithubRepoLeakTargetRepo = githubRepoLeakTargetRepo

	githubAccessToken, ok := os.LookupEnv(keynuker_go_common.EnvVarKeyNukerTestGithubAccessToken)
	if !ok {
		return fmt.Errorf("You must define environment variable %v to run this test", keynuker_go_common.EnvVarKeyNukerTestGithubAccessToken)
	}
	e.GithubAccessToken = githubAccessToken

	githubOrgs, err := GetGithubOrgsFromEnv()
	if err != nil {
		return err
	}
	e.GithubOrgs = githubOrgs

	return nil

}

func (e *EndToEndIntegrationTest) InitAwsIamSession() error {

	targetAwsAccounts, err := GetTargetAwsAccountsFromEnv()
	if err != nil {
		return err
	}

	// Just use the first aws account
	e.TargetAwsAccount = targetAwsAccounts[0]

	// Create AWS session
	sess, err := session.NewSession(&aws.Config{
		Credentials: credentials.NewCredentials(
			&credentials.StaticProvider{Value: credentials.Value{
				AccessKeyID:     e.TargetAwsAccount.AwsAccessKeyId,
				SecretAccessKey: e.TargetAwsAccount.AwsSecretAccessKey,
			}},
		),
	})
	if err != nil {
		return fmt.Errorf("Error creating aws session: %v", err)
	}

	// Create IAM client with the session.
	svc := iam.New(sess)

	e.AwsSession = sess
	e.IamService = svc

	// Discover IAM username based on aws key
	username, err := e.DiscoverIAMUsernameForKey(e.TargetAwsAccount.AwsAccessKeyId)
	if err != nil {
		return err
	}

	e.IamUsername = username

	return nil

}

// List all users
// For each user
// List all keys
// If you find the AwsAccessKeyId param, return the current user
func (e EndToEndIntegrationTest) DiscoverIAMUsernameForKey(AwsAccessKeyId string) (username string, err error) {

	// Fetch list of IAM users
	iamUsers, err := FetchIAMUsers(e.IamService)
	if err != nil {
		return "", err
	}

	for _, user := range iamUsers {

		listAccessKeysInput := &iam.ListAccessKeysInput{
			UserName: user.UserName,
			MaxItems: aws.Int64(1000),
		}

		listAccessKeysOutput, err := e.IamService.ListAccessKeys(listAccessKeysInput)
		if err != nil {
			return "", fmt.Errorf("Error listing access keys for user: %v.  Err: %v", user, err)
		}

		// Panic if more than 1K results, which is not handled
		if *listAccessKeysOutput.IsTruncated {
			// TODO: remove panic and put in a paginated loop.  Move to tleyden/awsutils + unit tests against mocks
			return "", fmt.Errorf("Output is truncated and this code does not handle it")
		}

		for _, accessKeyMetadata := range listAccessKeysOutput.AccessKeyMetadata {

			if *accessKeyMetadata.AccessKeyId == AwsAccessKeyId {
				return *user.UserName, nil
			}

		}

	}

	return "", fmt.Errorf("Unable to lookup username for key")

}

func (e EndToEndIntegrationTest) Run() error {

	// Set this to true to verify that the end-to-end integration test catches a real bug
	SetArtificialErrorInjection(false)

	keyLeakScenarios := e.GetEndToEndKeyLeakScenarios()
	for _, keyLeakScenario := range keyLeakScenarios {

		awsAccessKey, err := e.CreateKeyToLeak()
		if err != nil {
			return err
		}

		if err := keyLeakScenario.Leak(awsAccessKey); err != nil {
			return fmt.Errorf("Error running testScenario: %v", err)
		}

		if err := e.RunKeyNuker(awsAccessKey); err != nil {
			return fmt.Errorf("Error running keynuker: %v", err)
		}

		nuked, err := e.VerifyKeyNuked(awsAccessKey)
		if err != nil {
			return fmt.Errorf("Error verifying key was nuked: %v", err)
		}

		if !nuked {
			e.CleanupUnNuked(awsAccessKey)
			return fmt.Errorf("Key %v should have been nuked, but it wasn't", *awsAccessKey.AccessKeyId)
		}

		if err := keyLeakScenario.Cleanup(); err != nil {
			return fmt.Errorf("Error cleaning up keyleak scenario: %v", err)
		}

	}

	return nil

}

func (e EndToEndIntegrationTest) GetEndToEndKeyLeakScenarios() []KeyLeakScenario {
	return []KeyLeakScenario{
		NewLeakKeyViaCommit(e.GithubAccessToken, e.GithubRepoLeakTargetRepo),
	}
}

// NOTE: the aws key will need more permissions than usual, will need to be able to create AWS keys.
// Also, the aws key must be owned by a user named "KeyNuker"
func (e EndToEndIntegrationTest) CreateKeyToLeak() (accessKey *iam.AccessKey, err error) {

	createAccessKeyInput := &iam.CreateAccessKeyInput{
		UserName: aws.String(e.IamUsername),
	}
	createAccessKeyOutput, err := e.IamService.CreateAccessKey(createAccessKeyInput)
	if err != nil {
		return nil, fmt.Errorf("Error creating access key: %v", err)
	}

	return createAccessKeyOutput.AccessKey, nil

}

// If the initial attempt to nuke the key failed for some reason, maybe a bug, nuke it here
func (e EndToEndIntegrationTest) CleanupUnNuked(accessKeyNukeFailed *iam.AccessKey) (err error) {

	// Nuke the key from AWS
	deleteAccessKeyInput := &iam.DeleteAccessKeyInput{
		AccessKeyId: accessKeyNukeFailed.AccessKeyId,
		UserName:    accessKeyNukeFailed.UserName,
	}
	_, errDelKey := e.IamService.DeleteAccessKey(deleteAccessKeyInput)

	// Only consider it an error if it's not a "KeyNotFound error", which means the key was already nuked
	if errDelKey != nil && !IsKeyNotFoundError(errDelKey) {
		return nil
	}

	return err

}

func (e EndToEndIntegrationTest) VerifyKeyNuked(nukedAccessKey *iam.AccessKey) (nuked bool, err error) {

	listAccessKeysInput := &iam.ListAccessKeysInput{
		UserName: aws.String(e.IamUsername),
		MaxItems: aws.Int64(1000),
	}

	listAccessKeysOutput, err := e.IamService.ListAccessKeys(listAccessKeysInput)
	if err != nil {
		return false, fmt.Errorf("Error listing access keys for user: %v. Err: %v", e.IamUsername, err)
	}

	// Panic if more than 1K results, which is not handled
	if *listAccessKeysOutput.IsTruncated {
		// TODO: remove panic and put in a paginated loop.  Move to tleyden/awsutils + unit tests against mocks
		return false, fmt.Errorf("Output is truncated and this code does not handle it")
	}

	for _, accessKeyMetadata := range listAccessKeysOutput.AccessKeyMetadata {

		if *accessKeyMetadata.AccessKeyId == *nukedAccessKey.AccessKeyId {
			// Ugh, found the key that was supposed to be nuked.  Something is not working.
			return false, nil
		}
	}

	return true, nil

}

func (e EndToEndIntegrationTest) RunKeyNuker(accessKeyToNuke *iam.AccessKey) (err error) {

	keyNukerOrg := keynuker_go_common.DefaultKeyNukerOrg

	// ------------------------ Fetch Aws Keys -------------------------

	targetAwsAccounts, err := GetTargetAwsAccountsFromEnv()
	if err != nil {
		return err
	}

	paramsFetchAwsKeys := ParamsFetchAwsKeys{
		KeyNukerOrg:       keyNukerOrg,
		TargetAwsAccounts: targetAwsAccounts,
	}

	fetchedAwsKeys, err := FetchAwsKeys(paramsFetchAwsKeys)
	if err != nil {
		return err
	}
	log.Printf("fetchedAwsKeys: %v", fetchedAwsKeys)

	// ------------------------ Github User Aggregator -------------------------

	paramsAggregateGithubUsers := ParamsGithubUserAggregator{
		KeyNukerOrg:       keyNukerOrg,
		GithubAccessToken: e.GithubAccessToken,
		GithubOrgs:        e.GithubOrgs,
	}

	resultAggregateGithubUsers, err := AggregateGithubUsers(paramsAggregateGithubUsers)
	if err != nil {
		return err
	}

	// ------------------------ Github User Events Scanner -------------------------

	paramsScanGithubUserEventsForAwsKeys := ParamsScanGithubUserEventsForAwsKeys{
		KeyNukerOrg:       keyNukerOrg,
		GithubAccessToken: e.GithubAccessToken,
		GithubUsers:       resultAggregateGithubUsers.Doc.GithubUsers,
		AccessKeyMetadata: fetchedAwsKeys.Doc.AccessKeyMetadata,
	}

	recentEventTimeWindow := time.Minute * -10 // Last 5 seconds would probably work too, but give it some margin of error

	paramsScanGithubUserEventsForAwsKeys = paramsScanGithubUserEventsForAwsKeys.WithDefaultCheckpoints(recentEventTimeWindow)

	fetcher := NewGoGithubUserEventFetcher(e.GithubAccessToken)

	scanner := NewGithubUserEventsScanner(fetcher)

	scanAwsKeysResults, err := scanner.ScanAwsKeys(paramsScanGithubUserEventsForAwsKeys)
	if err != nil {
		return err
	}

	// ------------------------ Nuke Leaked Aws Keys -------------------------

	log.Printf("LeakedKeyEvents: %+v", scanAwsKeysResults.LeakedKeyEvents)

	params := ParamsNukeLeakedAwsKeys{
		KeyNukerOrg:            keyNukerOrg,
		TargetAwsAccounts:      targetAwsAccounts,
		LeakedKeyEvents:        scanAwsKeysResults.LeakedKeyEvents,
		GithubEventCheckpoints: scanAwsKeysResults.GithubEventCheckpoints,
	}

	resultNukeLeakedAwsKeys, err := NukeLeakedAwsKeys(params)
	if err != nil {
		return fmt.Errorf("Error nuking leaked aws keys: %v", err)
	}

	if len(resultNukeLeakedAwsKeys.NukedKeyEvents) <= 0 {
		return fmt.Errorf("Expected a key to be nuked, but none were nuked.  result: %+v", resultNukeLeakedAwsKeys)
	}

	for _, nukedKeyEvent := range resultNukeLeakedAwsKeys.NukedKeyEvents {
		log.Printf("NukedKeyEvent: %+v", nukedKeyEvent)
		if *nukedKeyEvent.LeakedKeyEvent.AccessKeyMetadata.AccessKeyId != *accessKeyToNuke.AccessKeyId {
			return fmt.Errorf(
				"Expected to nuke: %v, but nuked: %v",
				*accessKeyToNuke.AccessKeyId,
				*nukedKeyEvent.LeakedKeyEvent.AccessKeyMetadata.AccessKeyId,
			)
		}
	}

	return nil

}

type KeyLeakScenario interface {
	Leak(accessKey *iam.AccessKey) error
	Cleanup() error
}

type LeakKeyViaNewGithubIssue struct {
	GithubAccessToken        string
	GithubRepoLeakTargetRepo string
}

func NewLeakKeyViaCommit(githubAccessToken, targetGithubRepo string) *LeakKeyViaNewGithubIssue {
	return &LeakKeyViaNewGithubIssue{
		GithubAccessToken:        githubAccessToken,
		GithubRepoLeakTargetRepo: targetGithubRepo,
	}
}

func (lkvc LeakKeyViaNewGithubIssue) Leak(accessKey *iam.AccessKey) error {

	githubApiClient := NewGithubClientWrapper(lkvc.GithubAccessToken)

	ctx := context.Background()

	user, _, err := githubApiClient.ApiClient.Users.Get(ctx, "")
	if err != nil {
		return err
	}
	username := *user.Name

	log.Printf("github login: %v, name: %v", *user.Login, username)

	issueRequest := &github.IssueRequest{
		Title: aws.String("KeyNuker Leaked Key 🔐 End-to-End Test"),
		Body:  aws.String(fmt.Sprintf("Nukable 🔐💥 Key: %v.  Keynuker Project url: github.com/tleyden/keynuker", *accessKey.AccessKeyId)),
	}
	_, _, err = githubApiClient.ApiClient.Issues.Create(ctx, *user.Login, lkvc.GithubRepoLeakTargetRepo, issueRequest)
	if err != nil {
		return err
	}

	return nil

}

func (lkvc LeakKeyViaNewGithubIssue) Cleanup() error {
	return nil
}

<|endoftext|>


<commit_before>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package azure

import (
	. "launchpad.net/gocheck"
	"launchpad.net/juju-core/environs/config"
	"sync"
)

type EnvironSuite struct {
	ProviderSuite
}

var _ = Suite(new(EnvironSuite))

func makeEnviron(c *C) *azureEnviron {
	attrs := makeAzureConfigMap(c)
	cfg, err := config.New(attrs)
	c.Assert(err, IsNil)
	ecfg, err := azureEnvironProvider{}.newConfig(cfg)
	c.Assert(err, IsNil)
	return &azureEnviron{
		name: "env",
		ecfg: ecfg,
	}
}

// A note on locking tests.  Proper locking is hard to test for.  Tests here
// use a fixed pattern to verify that a function obeys a particular lock:
//
// 1. Create a channel for the function's result.
// 2. Grab the lock.
// 3. Launch goroutine 1: invoke the function and report result to the channel.
// 4. Launch goroutine 2: modify the object and then release the lock.
// 5. Retrieve result from the channel.
// 6. Test that the result reflects goroutine 2's modification.
// 7. Test that the lock was released in the end.
//
// If the function obeys the lock, it can't complete until goroutine 2 has
// completed.  If it doesn't, it can.  The pattern aims for this scenario:
//
//  The mainline code blocks on the channel.
//  Goroutine 1 starts.  It invokes the function you want to test.
//  The function tries to grab the lock, and blocks.
//  Goroutine 2 starts.  It releases the lock and exits.
//  The function in goroutine 1 is now unblocked.
//
// It would be simpler to have just one goroutine (and skip the channel), and
// release the lock inline.  But then the ordering depends on a more
// fundamental choice within the language implementation: it may choose to
// start running a goroutine immediately at the "go" statement, or it may
// continue executing the inline code and postpone execution of the goroutine
// until the inline code blocks.
//
// The pattern is still not a full guarantee that the lock is obeyed.  The
// language implementation might choose to run the goroutines in LIFO order,
// and then the locking would not be exercised.  The lock would simply be
// available by the time goroutine 1 ran, and the test would never fail unless
// the function you're testing neglected to release the lock.  But as long as
// there is a reasonable chance of the first goroutine starting before the
// second, there is a chance of exposing a function that disobeys the lock.

func (EnvironSuite) TestGetSnapshot(c *C) {
	original := azureEnviron{name: "this-env", ecfg: new(azureEnvironConfig)}
	snapshot := original.getSnapshot()

	// The snapshot is identical to the original.
	c.Check(*snapshot, DeepEquals, original)

	// However, they are distinct objects.
	c.Check(snapshot, Not(Equals), &original)

	// It's a shallow copy; they still share pointers.
	c.Check(snapshot.ecfg, Equals, original.ecfg)

	// Neither object is locked at the end of the copy.
	c.Check(original.Mutex, Equals, sync.Mutex{})
	c.Check(snapshot.Mutex, Equals, sync.Mutex{})
}

func (EnvironSuite) TestGetSnapshotLocksEnviron(c *C) {
	// This tests follows the locking-test pattern.  See comment above.
	// If you want to change how this works, you probably want to update
	// any other tests with the same pattern as well.
	original := azureEnviron{name: "old-name"}
	// 1. Result comes out of this channel.
	snaps := make(chan *azureEnviron)
	// 2. Stop a well-behaved getSnapshot from running (for now).
	original.Lock()
	// 3. Goroutine 1: ask for a snapshot.  The point of the test is that
	// this blocks until we release our lock.
	go func() {
		snaps <- original.getSnapshot()
	}()
	// 4. Goroutine 2: release the lock.  The getSnapshot call can't
	// complete until we've done this.
	go func() {
		original.name = "new-name"
		original.Unlock()
	}()
	// 5. Let the goroutines do their work.
	snapshot := <-snaps
	// 6. Test: the snapshot was made only after the lock was released.
	c.Check(snapshot.name, Equals, "new-name")
	// 7. Test: getSnapshot released the lock.
	c.Check(original.Mutex, Equals, sync.Mutex{})
}

func (EnvironSuite) TestName(c *C) {
	env := azureEnviron{name: "foo"}
	c.Check(env.Name(), Equals, env.name)
}

func (EnvironSuite) TestConfigReturnsConfig(c *C) {
	cfg := new(config.Config)
	ecfg := azureEnvironConfig{Config: cfg}
	env := azureEnviron{ecfg: &ecfg}
	c.Check(env.Config(), Equals, cfg)
}

func (EnvironSuite) TestConfigLocksEnviron(c *C) {
	// This tests follows the locking-test pattern.  See comment above.
	// If you want to change how this works, you probably want to update
	// any other tests with the same pattern as well.
	env := azureEnviron{name: "env", ecfg: new(azureEnvironConfig)}
	newConfig := new(config.Config)
	// 1. Create results channel.
	configs := make(chan *config.Config)
	// 2. Stop a well-behaved Config() from running, for now.
	env.Lock()
	// 3. Goroutine 1: call Config().  We want to test that this locks.
	go func() {
		configs <- env.Config()
	}()
	// 4. Goroutine 2: change the Environ object, and release the lock.
	go func() {
		env.ecfg = &azureEnvironConfig{Config: newConfig}
		env.Unlock()
	}()
	// 5. Let the goroutines do their work.
	config := <-configs
	// 6. Test that goroutine 2 completed before Config did.
	c.Check(config, Equals, newConfig)
	// 7. Test: Config() released the lock.
	c.Check(env.Mutex, Equals, sync.Mutex{})
}

// TODO: Temporarily deactivating this code.  Passing certificate in-memory
// may require gwacl change.
/*
func (EnvironSuite) TestGetManagementAPI(c *C) {
	env := makeEnviron(c)
	management, err := env.getManagementAPI()
	c.Assert(err, IsNil)
	c.Check(management, NotNil)
}
*/

func (EnvironSuite) TestGetStorageContext(c *C) {
	env := makeEnviron(c)
	storage, err := env.getStorageContext()
	c.Assert(err, IsNil)
	c.Assert(storage, NotNil)
	c.Check(storage.Account, Equals, env.ecfg.StorageAccountName())
	c.Check(storage.Key, Equals, env.ecfg.StorageAccountKey())
}
<commit_msg>Generalize locking-test pattern, as suggested by John Meinel.<commit_after>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package azure

import (
	. "launchpad.net/gocheck"
	"launchpad.net/juju-core/environs/config"
	"sync"
)

type EnvironSuite struct {
	ProviderSuite
}

var _ = Suite(new(EnvironSuite))

func makeEnviron(c *C) *azureEnviron {
	attrs := makeAzureConfigMap(c)
	cfg, err := config.New(attrs)
	c.Assert(err, IsNil)
	ecfg, err := azureEnvironProvider{}.newConfig(cfg)
	c.Assert(err, IsNil)
	return &azureEnviron{
		name: "env",
		ecfg: ecfg,
	}
}

// exerciseLockingFunction verifies that a function obeys a given lock.
//
// Use this as a building block in your own tests for proper locking.
// Parameters are the lock that you expect your function to block on; the
// function that you want to test for proper locking on that lock; and a
// marker callback whose effect you can observe.
//
// There needs to be a testable interaction between your "function" and your
// "marker," such that you can see from the result of "function" which of the
// two executed first.  So the "marker" must make a change that your "function"
// can observe, and the "function" must return a result that you can look at
// later to see whether it ran before or after the "marker."  You will get that
// result back as the return value.
//
// If the function that you're actually testing does not return anything that
// you can check for the marker effect, then pass a wrapper function that first
// invokes the function whose locking you want to test, and then records a
// result that shows whether the marker has run.
//
// After calling this function, you should test that:
// 1. The lock has been released.
// 2. Your "marker" function was completed before "function" executed.
//
// The "marker" will be executed while holding "lock."
//
// The return value of "function" must be suitable for transmission through a
// channel.
func exerciseLockingFunction(lock sync.Locker, function func() interface{}, marker func()) interface{} {
	// Here's the scenario this test aims for:
	// 1. We grab the lock.
	// 2. Goroutine invokes "function."
	// 3. Since the lock is not available, goroutine should block there.
	// 4. Execute "marker," while "function" is still blocked.
	// 5. Release the lock.
	// 6. Now execution of "function" can complete.
	//
	// If "function" blocks on "lock," then you should see the effect of
	// "marker" already present in the result.  If it doesn't, then
	// "function" will execute straight through.  In single-thread
	// execution you'll see a result that predates execution of the marker,
	// or in multi-thread execution the result will be indeterminate.
	result := make(chan interface{})
	proceed := make(chan bool)

	lock.Lock()
	go func() {
		proceed <- true
		result <- function()
	}()

	// Wait for the goroutine to start.  It should get stuck on "function."
	<-proceed

	// TODO: In Go 1.1, call runtime.GoSched a few times to give a
	// misbehaved "function" plenty of rope to hang itself.

	// Run the marker, while still holding the lock.  If "function" is
	// well-behaved, it's still waiting for us to do this.
	marker()
	// And now, finally, allow "function" to complete.
	lock.Unlock()

	return <-result
}

func (EnvironSuite) TestGetSnapshot(c *C) {
	original := azureEnviron{name: "this-env", ecfg: new(azureEnvironConfig)}
	snapshot := original.getSnapshot()

	// The snapshot is identical to the original.
	c.Check(*snapshot, DeepEquals, original)

	// However, they are distinct objects.
	c.Check(snapshot, Not(Equals), &original)

	// It's a shallow copy; they still share pointers.
	c.Check(snapshot.ecfg, Equals, original.ecfg)

	// Neither object is locked at the end of the copy.
	c.Check(original.Mutex, Equals, sync.Mutex{})
	c.Check(snapshot.Mutex, Equals, sync.Mutex{})
}

func (EnvironSuite) TestGetSnapshotLocksEnviron(c *C) {
	original := azureEnviron{name: "old-name"}
	snapshot := exerciseLockingFunction(
		&original,
		func() interface{} { return original.getSnapshot() },
		func() { original.name = "new-name" })

	c.Check(original.Mutex, Equals, sync.Mutex{})
	c.Check(snapshot.(*azureEnviron).name, Equals, "new-name")
}

func (EnvironSuite) TestName(c *C) {
	env := azureEnviron{name: "foo"}
	c.Check(env.Name(), Equals, env.name)
}

func (EnvironSuite) TestConfigReturnsConfig(c *C) {
	cfg := new(config.Config)
	ecfg := azureEnvironConfig{Config: cfg}
	env := azureEnviron{ecfg: &ecfg}
	c.Check(env.Config(), Equals, cfg)
}

func (EnvironSuite) TestConfigLocksEnviron(c *C) {
	env := azureEnviron{name: "env", ecfg: new(azureEnvironConfig)}
	newConfig := new(config.Config)
	config := exerciseLockingFunction(
		&env,
		func() interface{} { return env.Config() },
		func() { env.ecfg = &azureEnvironConfig{Config: newConfig} })
	c.Check(env.Mutex, Equals, sync.Mutex{})
	c.Check(config, Equals, newConfig)
}

// TODO: Temporarily deactivating this code.  Passing certificate in-memory
// may require gwacl change.
/*
func (EnvironSuite) TestGetManagementAPI(c *C) {
	env := makeEnviron(c)
	management, err := env.getManagementAPI()
	c.Assert(err, IsNil)
	c.Check(management, NotNil)
}
*/

func (EnvironSuite) TestGetStorageContext(c *C) {
	env := makeEnviron(c)
	storage, err := env.getStorageContext()
	c.Assert(err, IsNil)
	c.Assert(storage, NotNil)
	c.Check(storage.Account, Equals, env.ecfg.StorageAccountName())
	c.Check(storage.Key, Equals, env.ecfg.StorageAccountKey())
}
<|endoftext|>


<commit_before>// Copyright 2017 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package command

import (
	"context"
	"encoding/binary"
	"fmt"
	"math"
	"math/rand"
	"os"
	"strconv"
	"sync"
	"time"

	v3 "github.com/coreos/etcd/clientv3"
	"github.com/coreos/etcd/pkg/report"

	"github.com/spf13/cobra"
	"golang.org/x/time/rate"
	"gopkg.in/cheggaaa/pb.v1"
)

var (
	checkPerfLoad        string
	checkPerfPrefix      string
	checkDatascaleLoad   string
	checkDatascalePrefix string
	autoCompact          bool
	autoDefrag           bool
)

type checkPerfCfg struct {
	limit    int
	clients  int
	duration int
}

var checkPerfCfgMap = map[string]checkPerfCfg{
	// TODO: support read limit
	"s": {
		limit:    150,
		clients:  50,
		duration: 60,
	},
	"m": {
		limit:    1000,
		clients:  200,
		duration: 60,
	},
	"l": {
		limit:    8000,
		clients:  500,
		duration: 60,
	},
	"xl": {
		limit:    15000,
		clients:  1000,
		duration: 60,
	},
}

type checkDatascaleCfg struct {
	limit   int
	kvSize  int
	clients int
}

var checkDatascaleCfgMap = map[string]checkDatascaleCfg{
	"s": {
		limit:   10000,
		kvSize:  1024,
		clients: 50,
	},
	"m": {
		limit:   100000,
		kvSize:  1024,
		clients: 200,
	},
	"l": {
		limit:   1000000,
		kvSize:  1024,
		clients: 500,
	},
	"xl": {
		// xl tries to hit the upper bound aggressively which is 3 versions of 1M objects (3M in total)
		limit:   30000000,
		kvSize:  1024,
		clients: 1000,
	},
}

// NewCheckCommand returns the cobra command for "check".
func NewCheckCommand() *cobra.Command {
	cc := &cobra.Command{
		Use:   "check <subcommand>",
		Short: "commands for checking properties of the etcd cluster",
	}

	cc.AddCommand(NewCheckPerfCommand())
	cc.AddCommand(NewCheckDatascaleCommand())

	return cc
}

// NewCheckPerfCommand returns the cobra command for "check perf".
func NewCheckPerfCommand() *cobra.Command {
	cmd := &cobra.Command{
		Use:   "perf [options]",
		Short: "Check the performance of the etcd cluster",
		Run:   newCheckPerfCommand,
	}

	// TODO: support customized configuration
	cmd.Flags().StringVar(&checkPerfLoad, "load", "s", "The performance check's workload model. Accepted workloads: s(small), m(medium), l(large), xl(xLarge)")
	cmd.Flags().StringVar(&checkPerfPrefix, "prefix", "/etcdctl-check-perf/", "The prefix for writing the performance check's keys.")
	cmd.Flags().BoolVar(&autoCompact, "auto-compact", false, "Compact storage with last revision after test is finished.")
	cmd.Flags().BoolVar(&autoDefrag, "auto-defrag", false, "Defragment storage after test is finished.")

	return cmd
}

// newCheckPerfCommand executes the "check perf" command.
func newCheckPerfCommand(cmd *cobra.Command, args []string) {
	var checkPerfAlias = map[string]string{
		"s": "s", "small": "s",
		"m": "m", "medium": "m",
		"l": "l", "large": "l",
		"xl": "xl", "xLarge": "xl",
	}

	model, ok := checkPerfAlias[checkPerfLoad]
	if !ok {
		ExitWithError(ExitBadFeature, fmt.Errorf("unknown load option %v", checkPerfLoad))
	}
	cfg := checkPerfCfgMap[model]

	requests := make(chan v3.Op, cfg.clients)
	limit := rate.NewLimiter(rate.Limit(cfg.limit), 1)

	cc := clientConfigFromCmd(cmd)
	clients := make([]*v3.Client, cfg.clients)
	for i := 0; i < cfg.clients; i++ {
		clients[i] = cc.mustClient()
	}

	ctx, cancel := context.WithTimeout(context.Background(), time.Duration(cfg.duration)*time.Second)
	resp, err := clients[0].Get(ctx, checkPerfPrefix, v3.WithPrefix(), v3.WithLimit(1))
	cancel()
	if err != nil {
		ExitWithError(ExitError, err)
	}
	if len(resp.Kvs) > 0 {
		ExitWithError(ExitInvalidInput, fmt.Errorf("prefix %q has keys. Delete with etcdctl del --prefix %s first.", checkPerfPrefix, checkPerfPrefix))
	}

	ksize, vsize := 256, 1024
	k, v := make([]byte, ksize), string(make([]byte, vsize))

	bar := pb.New(cfg.duration)
	bar.Format("Bom !")
	bar.Start()

	r := report.NewReport("%4.4f")
	var wg sync.WaitGroup

	wg.Add(len(clients))
	for i := range clients {
		go func(c *v3.Client) {
			defer wg.Done()
			for op := range requests {
				st := time.Now()
				_, derr := c.Do(context.Background(), op)
				r.Results() <- report.Result{Err: derr, Start: st, End: time.Now()}
			}
		}(clients[i])
	}

	go func() {
		cctx, ccancel := context.WithTimeout(context.Background(), time.Duration(cfg.duration)*time.Second)
		defer ccancel()
		for limit.Wait(cctx) == nil {
			binary.PutVarint(k, int64(rand.Int63n(math.MaxInt64)))
			requests <- v3.OpPut(checkPerfPrefix+string(k), v)
		}
		close(requests)
	}()

	go func() {
		for i := 0; i < cfg.duration; i++ {
			time.Sleep(time.Second)
			bar.Add(1)
		}
		bar.Finish()
	}()

	sc := r.Stats()
	wg.Wait()
	close(r.Results())

	s := <-sc

	ctx, cancel = context.WithTimeout(context.Background(), 30*time.Second)
	dresp, err := clients[0].Delete(ctx, checkPerfPrefix, v3.WithPrefix())
	cancel()
	if err != nil {
		ExitWithError(ExitError, err)
	}

	if autoCompact {
		compact(clients[0], dresp.Header.Revision)
	}

	if autoDefrag {
		for _, ep := range clients[0].Endpoints() {
			defrag(clients[0], ep)
		}
	}

	ok = true
	if len(s.ErrorDist) != 0 {
		fmt.Println("FAIL: too many errors")
		for k, v := range s.ErrorDist {
			fmt.Printf("FAIL: ERROR(%v) -> %d\n", k, v)
		}
		ok = false
	}

	if s.RPS/float64(cfg.limit) <= 0.9 {
		fmt.Printf("FAIL: Throughput too low: %d writes/s\n", int(s.RPS)+1)
		ok = false
	} else {
		fmt.Printf("PASS: Throughput is %d writes/s\n", int(s.RPS)+1)
	}
	if s.Slowest > 0.5 { // slowest request > 500ms
		fmt.Printf("Slowest request took too long: %fs\n", s.Slowest)
		ok = false
	} else {
		fmt.Printf("PASS: Slowest request took %fs\n", s.Slowest)
	}
	if s.Stddev > 0.1 { // stddev > 100ms
		fmt.Printf("Stddev too high: %fs\n", s.Stddev)
		ok = false
	} else {
		fmt.Printf("PASS: Stddev is %fs\n", s.Stddev)
	}

	if ok {
		fmt.Println("PASS")
	} else {
		fmt.Println("FAIL")
		os.Exit(ExitError)
	}
}

// NewCheckDatascaleCommand returns the cobra command for "check datascale".
func NewCheckDatascaleCommand() *cobra.Command {
	cmd := &cobra.Command{
		Use:   "datascale [options]",
		Short: "Check the memory usage of holding data for different workloads on a given server endpoint.",
		Long:  "If no endpoint is provided, localhost will be used. If multiple endpoints are provided, first endpoint will be used.",
		Run:   newCheckDatascaleCommand,
	}

	cmd.Flags().StringVar(&checkDatascaleLoad, "load", "s", "The datascale check's workload model. Accepted workloads: s(small), m(medium), l(large), xl(xLarge)")
	cmd.Flags().StringVar(&checkDatascalePrefix, "prefix", "/etcdctl-check-datascale/", "The prefix for writing the datascale check's keys.")
	cmd.Flags().BoolVar(&autoCompact, "auto-compact", false, "Compact storage with last revision after test is finished.")
	cmd.Flags().BoolVar(&autoDefrag, "auto-defrag", false, "Defragment storage after test is finished.")

	return cmd
}

// newCheckDatascaleCommand executes the "check datascale" command.
func newCheckDatascaleCommand(cmd *cobra.Command, args []string) {
	var checkDatascaleAlias = map[string]string{
		"s": "s", "small": "s",
		"m": "m", "medium": "m",
		"l": "l", "large": "l",
		"xl": "xl", "xLarge": "xl",
	}

	model, ok := checkDatascaleAlias[checkDatascaleLoad]
	if !ok {
		ExitWithError(ExitBadFeature, fmt.Errorf("unknown load option %v", checkDatascaleLoad))
	}
	cfg := checkDatascaleCfgMap[model]

	requests := make(chan v3.Op, cfg.clients)

	cc := clientConfigFromCmd(cmd)
	clients := make([]*v3.Client, cfg.clients)
	for i := 0; i < cfg.clients; i++ {
		clients[i] = cc.mustClient()
	}

	// get endpoints
	eps, errEndpoints := endpointsFromCmd(cmd)
	if errEndpoints != nil {
		ExitWithError(ExitError, errEndpoints)
	}

	ctx, cancel := context.WithCancel(context.Background())
	resp, err := clients[0].Get(ctx, checkDatascalePrefix, v3.WithPrefix(), v3.WithLimit(1))
	cancel()
	if err != nil {
		ExitWithError(ExitError, err)
	}
	if len(resp.Kvs) > 0 {
		ExitWithError(ExitInvalidInput, fmt.Errorf("prefix %q has keys. Delete with etcdctl del --prefix %s first.", checkDatascalePrefix, checkDatascalePrefix))
	}

	ksize, vsize := 512, 512
	k, v := make([]byte, ksize), string(make([]byte, vsize))

	r := report.NewReport("%4.4f")
	var wg sync.WaitGroup
	wg.Add(len(clients))

	// get the process_resident_memory_bytes and process_virtual_memory_bytes before the put operations
	bytesBefore := endpointMemoryMetrics(eps[0])
	if bytesBefore == 0 {
		fmt.Println("FAIL: Could not read process_resident_memory_bytes before the put operations.")
		os.Exit(ExitError)
	}

	fmt.Println(fmt.Sprintf("Start data scale check for work load [%v key-value pairs, %v bytes per key-value, %v concurrent clients].", cfg.limit, cfg.kvSize, cfg.clients))
	for i := range clients {
		go func(c *v3.Client) {
			defer wg.Done()
			for op := range requests {
				st := time.Now()
				_, derr := c.Do(context.Background(), op)
				r.Results() <- report.Result{Err: derr, Start: st, End: time.Now()}
			}
		}(clients[i])
	}

	go func() {
		for i := 0; i < cfg.limit; i++ {
			binary.PutVarint(k, int64(rand.Int63n(math.MaxInt64)))
			requests <- v3.OpPut(checkDatascalePrefix+string(k), v)
		}
		close(requests)
	}()

	sc := r.Stats()
	wg.Wait()
	close(r.Results())
	s := <-sc

	// get the process_resident_memory_bytes after the put operations
	bytesAfter := endpointMemoryMetrics(eps[0])
	if bytesAfter == 0 {
		fmt.Println("FAIL: Could not read process_resident_memory_bytes after the put operations.")
		os.Exit(ExitError)
	}

	// delete the created kv pairs
	ctx, cancel = context.WithCancel(context.Background())
	dresp, derr := clients[0].Delete(ctx, checkDatascalePrefix, v3.WithPrefix())
	defer cancel()
	if derr != nil {
		ExitWithError(ExitError, derr)
	}

	if autoCompact {
		compact(clients[0], dresp.Header.Revision)
	}

	if autoDefrag {
		for _, ep := range clients[0].Endpoints() {
			defrag(clients[0], ep)
		}
	}

	if bytesAfter == 0 {
		fmt.Println("FAIL: Could not read process_resident_memory_bytes after the put operations.")
		os.Exit(ExitError)
	}

	bytesUsed := bytesAfter - bytesBefore
	mbUsed := bytesUsed / (1024 * 1024)

	if len(s.ErrorDist) != 0 {
		fmt.Println("FAIL: too many errors")
		for k, v := range s.ErrorDist {
			fmt.Printf("FAIL: ERROR(%v) -> %d\n", k, v)
		}
		os.Exit(ExitError)
	} else {
		fmt.Println(fmt.Sprintf("PASS: Approximate system memory used : %v MB.", strconv.FormatFloat(float64(mbUsed), 'f', 2, 64)))
	}
}
<commit_msg>etcdctl/ctlv3: fix "unconvert" warnings<commit_after>// Copyright 2017 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package command

import (
	"context"
	"encoding/binary"
	"fmt"
	"math"
	"math/rand"
	"os"
	"strconv"
	"sync"
	"time"

	v3 "github.com/coreos/etcd/clientv3"
	"github.com/coreos/etcd/pkg/report"

	"github.com/spf13/cobra"
	"golang.org/x/time/rate"
	"gopkg.in/cheggaaa/pb.v1"
)

var (
	checkPerfLoad        string
	checkPerfPrefix      string
	checkDatascaleLoad   string
	checkDatascalePrefix string
	autoCompact          bool
	autoDefrag           bool
)

type checkPerfCfg struct {
	limit    int
	clients  int
	duration int
}

var checkPerfCfgMap = map[string]checkPerfCfg{
	// TODO: support read limit
	"s": {
		limit:    150,
		clients:  50,
		duration: 60,
	},
	"m": {
		limit:    1000,
		clients:  200,
		duration: 60,
	},
	"l": {
		limit:    8000,
		clients:  500,
		duration: 60,
	},
	"xl": {
		limit:    15000,
		clients:  1000,
		duration: 60,
	},
}

type checkDatascaleCfg struct {
	limit   int
	kvSize  int
	clients int
}

var checkDatascaleCfgMap = map[string]checkDatascaleCfg{
	"s": {
		limit:   10000,
		kvSize:  1024,
		clients: 50,
	},
	"m": {
		limit:   100000,
		kvSize:  1024,
		clients: 200,
	},
	"l": {
		limit:   1000000,
		kvSize:  1024,
		clients: 500,
	},
	"xl": {
		// xl tries to hit the upper bound aggressively which is 3 versions of 1M objects (3M in total)
		limit:   30000000,
		kvSize:  1024,
		clients: 1000,
	},
}

// NewCheckCommand returns the cobra command for "check".
func NewCheckCommand() *cobra.Command {
	cc := &cobra.Command{
		Use:   "check <subcommand>",
		Short: "commands for checking properties of the etcd cluster",
	}

	cc.AddCommand(NewCheckPerfCommand())
	cc.AddCommand(NewCheckDatascaleCommand())

	return cc
}

// NewCheckPerfCommand returns the cobra command for "check perf".
func NewCheckPerfCommand() *cobra.Command {
	cmd := &cobra.Command{
		Use:   "perf [options]",
		Short: "Check the performance of the etcd cluster",
		Run:   newCheckPerfCommand,
	}

	// TODO: support customized configuration
	cmd.Flags().StringVar(&checkPerfLoad, "load", "s", "The performance check's workload model. Accepted workloads: s(small), m(medium), l(large), xl(xLarge)")
	cmd.Flags().StringVar(&checkPerfPrefix, "prefix", "/etcdctl-check-perf/", "The prefix for writing the performance check's keys.")
	cmd.Flags().BoolVar(&autoCompact, "auto-compact", false, "Compact storage with last revision after test is finished.")
	cmd.Flags().BoolVar(&autoDefrag, "auto-defrag", false, "Defragment storage after test is finished.")

	return cmd
}

// newCheckPerfCommand executes the "check perf" command.
func newCheckPerfCommand(cmd *cobra.Command, args []string) {
	var checkPerfAlias = map[string]string{
		"s": "s", "small": "s",
		"m": "m", "medium": "m",
		"l": "l", "large": "l",
		"xl": "xl", "xLarge": "xl",
	}

	model, ok := checkPerfAlias[checkPerfLoad]
	if !ok {
		ExitWithError(ExitBadFeature, fmt.Errorf("unknown load option %v", checkPerfLoad))
	}
	cfg := checkPerfCfgMap[model]

	requests := make(chan v3.Op, cfg.clients)
	limit := rate.NewLimiter(rate.Limit(cfg.limit), 1)

	cc := clientConfigFromCmd(cmd)
	clients := make([]*v3.Client, cfg.clients)
	for i := 0; i < cfg.clients; i++ {
		clients[i] = cc.mustClient()
	}

	ctx, cancel := context.WithTimeout(context.Background(), time.Duration(cfg.duration)*time.Second)
	resp, err := clients[0].Get(ctx, checkPerfPrefix, v3.WithPrefix(), v3.WithLimit(1))
	cancel()
	if err != nil {
		ExitWithError(ExitError, err)
	}
	if len(resp.Kvs) > 0 {
		ExitWithError(ExitInvalidInput, fmt.Errorf("prefix %q has keys. Delete with etcdctl del --prefix %s first.", checkPerfPrefix, checkPerfPrefix))
	}

	ksize, vsize := 256, 1024
	k, v := make([]byte, ksize), string(make([]byte, vsize))

	bar := pb.New(cfg.duration)
	bar.Format("Bom !")
	bar.Start()

	r := report.NewReport("%4.4f")
	var wg sync.WaitGroup

	wg.Add(len(clients))
	for i := range clients {
		go func(c *v3.Client) {
			defer wg.Done()
			for op := range requests {
				st := time.Now()
				_, derr := c.Do(context.Background(), op)
				r.Results() <- report.Result{Err: derr, Start: st, End: time.Now()}
			}
		}(clients[i])
	}

	go func() {
		cctx, ccancel := context.WithTimeout(context.Background(), time.Duration(cfg.duration)*time.Second)
		defer ccancel()
		for limit.Wait(cctx) == nil {
			binary.PutVarint(k, rand.Int63n(math.MaxInt64))
			requests <- v3.OpPut(checkPerfPrefix+string(k), v)
		}
		close(requests)
	}()

	go func() {
		for i := 0; i < cfg.duration; i++ {
			time.Sleep(time.Second)
			bar.Add(1)
		}
		bar.Finish()
	}()

	sc := r.Stats()
	wg.Wait()
	close(r.Results())

	s := <-sc

	ctx, cancel = context.WithTimeout(context.Background(), 30*time.Second)
	dresp, err := clients[0].Delete(ctx, checkPerfPrefix, v3.WithPrefix())
	cancel()
	if err != nil {
		ExitWithError(ExitError, err)
	}

	if autoCompact {
		compact(clients[0], dresp.Header.Revision)
	}

	if autoDefrag {
		for _, ep := range clients[0].Endpoints() {
			defrag(clients[0], ep)
		}
	}

	ok = true
	if len(s.ErrorDist) != 0 {
		fmt.Println("FAIL: too many errors")
		for k, v := range s.ErrorDist {
			fmt.Printf("FAIL: ERROR(%v) -> %d\n", k, v)
		}
		ok = false
	}

	if s.RPS/float64(cfg.limit) <= 0.9 {
		fmt.Printf("FAIL: Throughput too low: %d writes/s\n", int(s.RPS)+1)
		ok = false
	} else {
		fmt.Printf("PASS: Throughput is %d writes/s\n", int(s.RPS)+1)
	}
	if s.Slowest > 0.5 { // slowest request > 500ms
		fmt.Printf("Slowest request took too long: %fs\n", s.Slowest)
		ok = false
	} else {
		fmt.Printf("PASS: Slowest request took %fs\n", s.Slowest)
	}
	if s.Stddev > 0.1 { // stddev > 100ms
		fmt.Printf("Stddev too high: %fs\n", s.Stddev)
		ok = false
	} else {
		fmt.Printf("PASS: Stddev is %fs\n", s.Stddev)
	}

	if ok {
		fmt.Println("PASS")
	} else {
		fmt.Println("FAIL")
		os.Exit(ExitError)
	}
}

// NewCheckDatascaleCommand returns the cobra command for "check datascale".
func NewCheckDatascaleCommand() *cobra.Command {
	cmd := &cobra.Command{
		Use:   "datascale [options]",
		Short: "Check the memory usage of holding data for different workloads on a given server endpoint.",
		Long:  "If no endpoint is provided, localhost will be used. If multiple endpoints are provided, first endpoint will be used.",
		Run:   newCheckDatascaleCommand,
	}

	cmd.Flags().StringVar(&checkDatascaleLoad, "load", "s", "The datascale check's workload model. Accepted workloads: s(small), m(medium), l(large), xl(xLarge)")
	cmd.Flags().StringVar(&checkDatascalePrefix, "prefix", "/etcdctl-check-datascale/", "The prefix for writing the datascale check's keys.")
	cmd.Flags().BoolVar(&autoCompact, "auto-compact", false, "Compact storage with last revision after test is finished.")
	cmd.Flags().BoolVar(&autoDefrag, "auto-defrag", false, "Defragment storage after test is finished.")

	return cmd
}

// newCheckDatascaleCommand executes the "check datascale" command.
func newCheckDatascaleCommand(cmd *cobra.Command, args []string) {
	var checkDatascaleAlias = map[string]string{
		"s": "s", "small": "s",
		"m": "m", "medium": "m",
		"l": "l", "large": "l",
		"xl": "xl", "xLarge": "xl",
	}

	model, ok := checkDatascaleAlias[checkDatascaleLoad]
	if !ok {
		ExitWithError(ExitBadFeature, fmt.Errorf("unknown load option %v", checkDatascaleLoad))
	}
	cfg := checkDatascaleCfgMap[model]

	requests := make(chan v3.Op, cfg.clients)

	cc := clientConfigFromCmd(cmd)
	clients := make([]*v3.Client, cfg.clients)
	for i := 0; i < cfg.clients; i++ {
		clients[i] = cc.mustClient()
	}

	// get endpoints
	eps, errEndpoints := endpointsFromCmd(cmd)
	if errEndpoints != nil {
		ExitWithError(ExitError, errEndpoints)
	}

	ctx, cancel := context.WithCancel(context.Background())
	resp, err := clients[0].Get(ctx, checkDatascalePrefix, v3.WithPrefix(), v3.WithLimit(1))
	cancel()
	if err != nil {
		ExitWithError(ExitError, err)
	}
	if len(resp.Kvs) > 0 {
		ExitWithError(ExitInvalidInput, fmt.Errorf("prefix %q has keys. Delete with etcdctl del --prefix %s first.", checkDatascalePrefix, checkDatascalePrefix))
	}

	ksize, vsize := 512, 512
	k, v := make([]byte, ksize), string(make([]byte, vsize))

	r := report.NewReport("%4.4f")
	var wg sync.WaitGroup
	wg.Add(len(clients))

	// get the process_resident_memory_bytes and process_virtual_memory_bytes before the put operations
	bytesBefore := endpointMemoryMetrics(eps[0])
	if bytesBefore == 0 {
		fmt.Println("FAIL: Could not read process_resident_memory_bytes before the put operations.")
		os.Exit(ExitError)
	}

	fmt.Println(fmt.Sprintf("Start data scale check for work load [%v key-value pairs, %v bytes per key-value, %v concurrent clients].", cfg.limit, cfg.kvSize, cfg.clients))
	for i := range clients {
		go func(c *v3.Client) {
			defer wg.Done()
			for op := range requests {
				st := time.Now()
				_, derr := c.Do(context.Background(), op)
				r.Results() <- report.Result{Err: derr, Start: st, End: time.Now()}
			}
		}(clients[i])
	}

	go func() {
		for i := 0; i < cfg.limit; i++ {
			binary.PutVarint(k, rand.Int63n(math.MaxInt64))
			requests <- v3.OpPut(checkDatascalePrefix+string(k), v)
		}
		close(requests)
	}()

	sc := r.Stats()
	wg.Wait()
	close(r.Results())
	s := <-sc

	// get the process_resident_memory_bytes after the put operations
	bytesAfter := endpointMemoryMetrics(eps[0])
	if bytesAfter == 0 {
		fmt.Println("FAIL: Could not read process_resident_memory_bytes after the put operations.")
		os.Exit(ExitError)
	}

	// delete the created kv pairs
	ctx, cancel = context.WithCancel(context.Background())
	dresp, derr := clients[0].Delete(ctx, checkDatascalePrefix, v3.WithPrefix())
	defer cancel()
	if derr != nil {
		ExitWithError(ExitError, derr)
	}

	if autoCompact {
		compact(clients[0], dresp.Header.Revision)
	}

	if autoDefrag {
		for _, ep := range clients[0].Endpoints() {
			defrag(clients[0], ep)
		}
	}

	if bytesAfter == 0 {
		fmt.Println("FAIL: Could not read process_resident_memory_bytes after the put operations.")
		os.Exit(ExitError)
	}

	bytesUsed := bytesAfter - bytesBefore
	mbUsed := bytesUsed / (1024 * 1024)

	if len(s.ErrorDist) != 0 {
		fmt.Println("FAIL: too many errors")
		for k, v := range s.ErrorDist {
			fmt.Printf("FAIL: ERROR(%v) -> %d\n", k, v)
		}
		os.Exit(ExitError)
	} else {
		fmt.Println(fmt.Sprintf("PASS: Approximate system memory used : %v MB.", strconv.FormatFloat(mbUsed, 'f', 2, 64)))
	}
}
<|endoftext|>


<commit_before>package main

import "regexp"

var (
  regexp_nowplaying *regexp.Regexp = regexp.MustCompile(
    `(?i)what(.?s)? (is |song is )?(this|the|playing)( song| now)? ?\??$`,
  )
  regexp_playsong *regexp.Regexp = regexp.MustCompile(
    `(?i)listen to (.+)$`,
  )
  regexp_playalbum *regexp.Regexp = regexp.MustCompile(
    `(?i)put on (?:album|playlist) (.+)$`,
  )
  regexp_searchsong *regexp.Regexp = regexp.MustCompile(
    `(?i)search(?: for)? (song|title|playlist|album) (.+)$`,
  )
  regexp_playeraction *regexp.Regexp = regexp.MustCompile(
    `(?i)(play|pause|next|previous|stop) (?:song|track|music)$`,
  )
  regexp_boombox *regexp.Regexp = regexp.MustCompile(
    `^(?i)(start|stop) boombox$`,
  )
)
<commit_msg>Removed 'stop' action; doesn't work like you think<commit_after>package main

import "regexp"

var (
  regexp_nowplaying *regexp.Regexp = regexp.MustCompile(
    `(?i)what(.?s)? (is |song is )?(this|the|playing)( song| now)? ?\??$`,
  )
  regexp_playsong *regexp.Regexp = regexp.MustCompile(
    `(?i)listen to (.+)$`,
  )
  regexp_playalbum *regexp.Regexp = regexp.MustCompile(
    `(?i)put on (?:album|playlist) (.+)$`,
  )
  regexp_searchsong *regexp.Regexp = regexp.MustCompile(
    `(?i)search(?: for)? (song|title|playlist|album) (.+)$`,
  )
  regexp_playeraction *regexp.Regexp = regexp.MustCompile(
    `(?i)(play|pause|next|previous) (?:song|track|music)$`,
  )
  regexp_boombox *regexp.Regexp = regexp.MustCompile(
    `^(?i)(start|stop) boombox$`,
  )
)
<|endoftext|>


<commit_before>package main

import (
	"github.com/codegangsta/cli"
)

func registerFlags(app cli.App) *cli.App {
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "user, u",
			Usage:  "username for authentication",
			EnvVar: "SOMA_ADM_USER,USER",
		},
		cli.BoolFlag{
			Name:  "password, p",
			Usage: "prompt for password",
		},
		cli.IntFlag{
			Name:   "timeout, t",
			Usage:  "connect timeout in seconds",
			EnvVar: "SOMA_ADM_TIMEOUT",
		},
		cli.StringFlag{
			Name:   "host, h",
			Usage:  "API URI to connect to",
			EnvVar: "SOMA_ADM_API",
		},
		cli.StringFlag{
			Name:   "dbdir, d",
			Usage:  "name of the db subdirectory",
			EnvVar: "SOMA_ADM_DBDIR",
		},
		cli.StringFlag{
			Name:   "logdir, l",
			Usage:  "name of the log subdirectory",
			EnvVar: "SOMA_ADM_LOGDIR",
		},
		cli.StringFlag{
			Name:   "config, c",
			Usage:  "configuration file location",
			EnvVar: "SOMA_ADM_CONFIG",
		},
		cli.BoolFlag{
			Name:  "json, J",
			Usage: "output reply as JSON",
		},
	}
	return &app
}

// vim: ts=4 sw=4 sts=4 noet fenc=utf-8 ffs=unix
<commit_msg>Fix redefined flag (help, h)<commit_after>package main

import (
	"github.com/codegangsta/cli"
)

func registerFlags(app cli.App) *cli.App {
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "user, u",
			Usage:  "username for authentication",
			EnvVar: "SOMA_ADM_USER,USER",
		},
		cli.BoolFlag{
			Name:  "password, p",
			Usage: "prompt for password",
		},
		cli.IntFlag{
			Name:   "timeout, t",
			Usage:  "connect timeout in seconds",
			EnvVar: "SOMA_ADM_TIMEOUT",
		},
		cli.StringFlag{
			Name:   "host, H",
			Usage:  "API URI to connect to",
			EnvVar: "SOMA_ADM_API",
		},
		cli.StringFlag{
			Name:   "dbdir, d",
			Usage:  "name of the db subdirectory",
			EnvVar: "SOMA_ADM_DBDIR",
		},
		cli.StringFlag{
			Name:   "logdir, l",
			Usage:  "name of the log subdirectory",
			EnvVar: "SOMA_ADM_LOGDIR",
		},
		cli.StringFlag{
			Name:   "config, c",
			Usage:  "configuration file location",
			EnvVar: "SOMA_ADM_CONFIG",
		},
		cli.BoolFlag{
			Name:  "json, J",
			Usage: "output reply as JSON",
		},
	}
	return &app
}

// vim: ts=4 sw=4 sts=4 noet fenc=utf-8 ffs=unix
<|endoftext|>


<commit_before>package registry

import (
	"path"
	"strings"
	"time"

	"github.com/coreos/go-etcd/etcd"
	log "github.com/golang/glog"

	"github.com/coreos/coreinit/event"
)

type EventStream struct {
	etcd *etcd.Client
	stop chan bool
}

func NewEventStream(client *etcd.Client) *EventStream {
	return &EventStream{client, make(chan bool)}
}

func (self *EventStream) Stream(eventChannel chan *event.Event) {
	watchMap := map[string][]func(*etcd.Response) *event.Event{
		path.Join(keyPrefix, statePrefix):   []func(*etcd.Response) *event.Event{filterEventJobStatePublished, filterEventJobStateExpired},
		path.Join(keyPrefix, jobPrefix):     []func(*etcd.Response) *event.Event{filterEventJobCreated, filterEventJobScheduled, filterEventJobCancelled},
		path.Join(keyPrefix, machinePrefix): []func(*etcd.Response) *event.Event{self.filterEventMachineUpdated, self.filterEventMachineRemoved},
		path.Join(keyPrefix, requestPrefix): []func(*etcd.Response) *event.Event{filterEventRequestCreated},
		path.Join(keyPrefix, offerPrefix):   []func(*etcd.Response) *event.Event{self.filterEventJobOffered, filterEventJobBidSubmitted},
	}

	for key, funcs := range watchMap {
		for _, f := range funcs {
			etcdchan := make(chan *etcd.Response)
			go watch(self.etcd, etcdchan, key, self.stop)
			go pipe(etcdchan, f, eventChannel, self.stop)
		}
	}
}

func pipe(etcdchan chan *etcd.Response, translate func(resp *etcd.Response) *event.Event, eventchan chan *event.Event, stopchan chan bool) {
	for true {
		select {
		case <-stopchan:
			return
		case resp := <-etcdchan:
			log.V(2).Infof("Received response from etcd watcher: Action=%s ModifiedIndex=%d Key=%s", resp.Action, resp.Node.ModifiedIndex, resp.Node.Key)
			ev := translate(resp)
			if ev != nil {
				log.V(2).Infof("Translated response(ModifiedIndex=%d) to event(Type=%s)", resp.Node.ModifiedIndex, ev.Type)
				eventchan <- ev
			} else {
				log.V(2).Infof("Discarding response(ModifiedIndex=%d) from etcd watcher", resp.Node.ModifiedIndex)
			}
		}
	}
}

func watch(client *etcd.Client, etcdchan chan *etcd.Response, key string, stopchan chan bool) {
	for true {
		select {
		case <-stopchan:
			log.V(2).Infof("Gracefully closing etcd watcher: key=%s", key)
			return
		default:
			log.V(2).Infof("Creating etcd watcher: key=%s, machines=%s", key, strings.Join(client.GetCluster(), ","))
			_, err := client.Watch(key, 0, true, etcdchan, stopchan)

			if err != nil {
				log.V(2).Infof("etcd watch closed exited: key=%s, err=\"%s\"", key, err.Error())

				// Let's not slam the etcd server in the event that we know
				// an unexpected error occurred.
				time.Sleep(time.Second)
			}
		}
	}
}
<commit_msg>refactor(registry): s/eventChannel/eventchan/g<commit_after>package registry

import (
	"path"
	"strings"
	"time"

	"github.com/coreos/go-etcd/etcd"
	log "github.com/golang/glog"

	"github.com/coreos/coreinit/event"
)

type EventStream struct {
	etcd *etcd.Client
	stop chan bool
}

func NewEventStream(client *etcd.Client) *EventStream {
	return &EventStream{client, make(chan bool)}
}

func (self *EventStream) Stream(eventchan chan *event.Event) {
	watchMap := map[string][]func(*etcd.Response) *event.Event{
		path.Join(keyPrefix, statePrefix):   []func(*etcd.Response) *event.Event{filterEventJobStatePublished, filterEventJobStateExpired},
		path.Join(keyPrefix, jobPrefix):     []func(*etcd.Response) *event.Event{filterEventJobCreated, filterEventJobScheduled, filterEventJobCancelled},
		path.Join(keyPrefix, machinePrefix): []func(*etcd.Response) *event.Event{self.filterEventMachineUpdated, self.filterEventMachineRemoved},
		path.Join(keyPrefix, requestPrefix): []func(*etcd.Response) *event.Event{filterEventRequestCreated},
		path.Join(keyPrefix, offerPrefix):   []func(*etcd.Response) *event.Event{self.filterEventJobOffered, filterEventJobBidSubmitted},
	}

	for key, funcs := range watchMap {
		for _, f := range funcs {
			etcdchan := make(chan *etcd.Response)
			go watch(self.etcd, etcdchan, key, self.stop)
			go pipe(etcdchan, f, eventchan, self.stop)
		}
	}
}

func pipe(etcdchan chan *etcd.Response, translate func(resp *etcd.Response) *event.Event, eventchan chan *event.Event, stopchan chan bool) {
	for true {
		select {
		case <-stopchan:
			return
		case resp := <-etcdchan:
			log.V(2).Infof("Received response from etcd watcher: Action=%s ModifiedIndex=%d Key=%s", resp.Action, resp.Node.ModifiedIndex, resp.Node.Key)
			ev := translate(resp)
			if ev != nil {
				log.V(2).Infof("Translated response(ModifiedIndex=%d) to event(Type=%s)", resp.Node.ModifiedIndex, ev.Type)
				eventchan <- ev
			} else {
				log.V(2).Infof("Discarding response(ModifiedIndex=%d) from etcd watcher", resp.Node.ModifiedIndex)
			}
		}
	}
}

func watch(client *etcd.Client, etcdchan chan *etcd.Response, key string, stopchan chan bool) {
	for true {
		select {
		case <-stopchan:
			log.V(2).Infof("Gracefully closing etcd watcher: key=%s", key)
			return
		default:
			log.V(2).Infof("Creating etcd watcher: key=%s, machines=%s", key, strings.Join(client.GetCluster(), ","))
			_, err := client.Watch(key, 0, true, etcdchan, stopchan)

			if err != nil {
				log.V(2).Infof("etcd watch closed exited: key=%s, err=\"%s\"", key, err.Error())

				// Let's not slam the etcd server in the event that we know
				// an unexpected error occurred.
				time.Sleep(time.Second)
			}
		}
	}
}
<|endoftext|>


<commit_before>package stripe

import (
	"net/url"
	"strconv"
)

type Coupon struct {
	Id               string `json:"id"`
	Object           string `json:"object"`
	Livemode         bool   `json:"livemode"`
	Duration         string `json:"duration"`
	AmountOff        int64  `json:"amount_off"`
	Currency         string `json:"currency"`
	DurationInMonths int64  `json:"duration_in_months"`
	MaxRedemptions   int64  `json:"max_redemptions"`
	PercentOff       int64  `json:"percent_off"`
	RedeemBy         int64  `json:"redeem_by"`
	TimesRedeemed    int64  `json:"times_redeemed"`
	Valid            bool   `json:"valid"`
}

// Delete deletes a coupon.
//
// For more information: https://stripe.com/docs/api#delete_coupon
func (c *Coupon) Delete() (*DeleteResponse, error) {
	response := DeleteResponse{}
	err := delete("/coupons/"+c.Id, nil, &response)
	return &response, err
}

// The CouponClient is the receiver for most standard coupon related endpoints.
type CouponClient struct{}

// Create creates a coupon.
//
// For more information: https://stripe.com/docs/api#create_coupon
func (c *CouponClient) Create(params *CouponParams) (*Coupon, error) {
	coupon := Coupon{}
	values := url.Values{}
	parseCouponParams(params, &values)
	err := post("/coupons", values, &coupon)
	return &coupon, err
}

// Retrieve loads a coupon.
//
// For more information: https://stripe.com/docs/api#retrieve_coupon
func (c *CouponClient) Retrieve(id string) (*Coupon, error) {
	coupon := Coupon{}
	err := get("/coupons/"+id, nil, &coupon)
	return &coupon, err
}

// Delete deletes a coupon.
//
// For more information: https://stripe.com/docs/api#delete_coupon
func (c *CouponClient) Delete(id string) (*DeleteResponse, error) {
	response := DeleteResponse{}
	err := delete("/coupons/"+id, nil, &response)
	return &response, err
}

// List lists the first 10 coupons. It calls ListCount with 10 as
// the count and 0 as the offset, which are the defaults in the Stripe API.
//
// For more information: https://stripe.com/docs/api#list_coupons
func (c *CouponClient) List() ([]*Coupon, error) {
	return c.ListCount(10, 0)
}

// ListCount lists `count` coupons starting at `offset`.
//
// For more information: https://stripe.com/docs/api#list_coupons
func (c *CouponClient) ListCount(count, offset int) ([]*Coupon, error) {
	type coupons struct{ Data []*Coupon }
	list := coupons{}

	params := url.Values{
		"count":  {strconv.Itoa(count)},
		"offset": {strconv.Itoa(offset)},
	}

	err := get("/coupons", params, &list)
	return list.Data, err
}

// parseCouponParams takes a pointer to a CouponParams and a pointer to a url.Values,
// it iterates over everything in the CouponParams struct and Adds what is there
// to the url.Values.
func parseCouponParams(params *CouponParams, values *url.Values) {
	if params.Id != "" {
		values.Add("id", params.Id)
	}

	if params.Duration != "" {
		values.Add("duration", params.Duration)
	}

	if params.AmountOff != 0 {
		values.Add("amount_off", strconv.Itoa(params.AmountOff))
	}

	if params.Currency != "" {
		values.Add("currency", params.Currency)
	}

	if params.DurationInMonths != 0 {
		values.Add("duration_in_months", strconv.Itoa(params.DurationInMonths))
	}

	if params.MaxRedemptions != 0 {
		values.Add("max_redemptions", strconv.Itoa(params.MaxRedemptions))
	}

	if params.PercentOff != 0 {
		values.Add("percent_off", strconv.Itoa(params.PercentOff))
	}

	if params.RedeemBy != 0 {
		values.Add("redeem_by", strconv.Itoa(params.RedeemBy))
	}
}
<commit_msg>updates to coupons list response<commit_after>package stripe

import (
	"net/url"
	"strconv"
)

type Coupon struct {
	Id               string `json:"id"`
	Object           string `json:"object"`
	Livemode         bool   `json:"livemode"`
	Duration         string `json:"duration"`
	AmountOff        int64  `json:"amount_off"`
	Currency         string `json:"currency"`
	DurationInMonths int64  `json:"duration_in_months"`
	MaxRedemptions   int64  `json:"max_redemptions"`
	PercentOff       int64  `json:"percent_off"`
	RedeemBy         int64  `json:"redeem_by"`
	TimesRedeemed    int64  `json:"times_redeemed"`
	Valid            bool   `json:"valid"`
}

// CouponListResponse is what is returned with a List request.
type CouponListResponse struct {
	Object string    `json:"object"`
	Url    string    `json:"url"`
	Count  int       `json:"count"`
	Data   []*Coupon `json:"data"`
}

// Delete deletes a coupon.
//
// For more information: https://stripe.com/docs/api#delete_coupon
func (c *Coupon) Delete() (*DeleteResponse, error) {
	response := DeleteResponse{}
	err := delete("/coupons/"+c.Id, nil, &response)
	return &response, err
}

// The CouponClient is the receiver for most standard coupon related endpoints.
type CouponClient struct{}

// Create creates a coupon.
//
// For more information: https://stripe.com/docs/api#create_coupon
func (c *CouponClient) Create(params *CouponParams) (*Coupon, error) {
	coupon := Coupon{}
	values := url.Values{}
	parseCouponParams(params, &values)
	err := post("/coupons", values, &coupon)
	return &coupon, err
}

// Retrieve loads a coupon.
//
// For more information: https://stripe.com/docs/api#retrieve_coupon
func (c *CouponClient) Retrieve(id string) (*Coupon, error) {
	coupon := Coupon{}
	err := get("/coupons/"+id, nil, &coupon)
	return &coupon, err
}

// Delete deletes a coupon.
//
// For more information: https://stripe.com/docs/api#delete_coupon
func (c *CouponClient) Delete(id string) (*DeleteResponse, error) {
	response := DeleteResponse{}
	err := delete("/coupons/"+id, nil, &response)
	return &response, err
}

// List lists the first 10 coupons. It calls ListCount with 10 as
// the count and 0 as the offset, which are the defaults in the Stripe API.
//
// For more information: https://stripe.com/docs/api#list_coupons
func (c *CouponClient) List() (*CouponListResponse, error) {
	return c.ListCount(10, 0)
}

// ListCount lists `count` coupons starting at `offset`.
//
// For more information: https://stripe.com/docs/api#list_coupons
func (c *CouponClient) ListCount(count, offset int) (*CouponListResponse, error) {
	response := CouponListResponse{}

	params := url.Values{
		"count":  {strconv.Itoa(count)},
		"offset": {strconv.Itoa(offset)},
	}

	err := get("/coupons", params, &response)
	return &response, err
}

// parseCouponParams takes a pointer to a CouponParams and a pointer to a url.Values,
// it iterates over everything in the CouponParams struct and Adds what is there
// to the url.Values.
func parseCouponParams(params *CouponParams, values *url.Values) {
	if params.Id != "" {
		values.Add("id", params.Id)
	}

	if params.Duration != "" {
		values.Add("duration", params.Duration)
	}

	if params.AmountOff != 0 {
		values.Add("amount_off", strconv.Itoa(params.AmountOff))
	}

	if params.Currency != "" {
		values.Add("currency", params.Currency)
	}

	if params.DurationInMonths != 0 {
		values.Add("duration_in_months", strconv.Itoa(params.DurationInMonths))
	}

	if params.MaxRedemptions != 0 {
		values.Add("max_redemptions", strconv.Itoa(params.MaxRedemptions))
	}

	if params.PercentOff != 0 {
		values.Add("percent_off", strconv.Itoa(params.PercentOff))
	}

	if params.RedeemBy != 0 {
		values.Add("redeem_by", strconv.Itoa(params.RedeemBy))
	}
}
<|endoftext|>