<commit_before>// A command-line utility to lock old issues.
package main

import (
	"flag"
	"log"
	"sync"
	"time"

	"github.com/parkr/auto-reply/ctx"
	"github.com/parkr/auto-reply/freeze"
)

type repository struct {
	Owner, Name string
}

var (
	defaultRepos = []repository{
		repository{"jekyll", "jekyll"},
		repository{"jekyll", "jekyll-admin"},
		repository{"jekyll", "jekyll-import"},
		repository{"jekyll", "github-metadata"},
		repository{"jekyll", "jekyll-redirect-from"},
		repository{"jekyll", "jekyll-feed"},
		repository{"jekyll", "jekyll-compose"},
		repository{"jekyll", "jekyll-watch"},
		repository{"jekyll", "jekyll-seo-tag"},
		repository{"jekyll", "jekyll-sitemap"},
		repository{"jekyll", "jekyll-sass-converter"},
		repository{"jekyll", "jemoji"},
		repository{"jekyll", "jekyll-gist"},
		repository{"jekyll", "jekyll-coffeescript"},
		repository{"jekyll", "plugins"},
	}

	sleepBetweenFreezes = 150 * time.Millisecond
)

func main() {
	var actuallyDoIt bool
	flag.BoolVar(&actuallyDoIt, "f", false, "Whether to actually mark the issues or close them.")
	var inputRepos string
	flag.StringVar(&inputRepos, "repos", "", "Specify a list of comma-separated repo name/owner pairs, e.g. 'jekyll/jekyll-admin'.")
	flag.Parse()

	var repos []repository
	if inputRepos == "" {
		repos = defaultRepos
	}

	context := ctx.NewDefaultContext()
	if context.GitHub == nil {
		log.Fatalln("cannot proceed without github client")
	}

	var wg sync.WaitGroup
	for _, repo := range repos {
		wg.Add(1)
		go func(context *ctx.Context, repo repository, actuallyDoIt bool) {
			defer wg.Done()
			if err := processRepo(context, repo.Owner, repo.Name, actuallyDoIt); err != nil {
				log.Printf("%s/%s: error: %#v", repo.Owner, repo.Name, err)
			}
		}(context, repo, actuallyDoIt)
	}

	wg.Wait()
}

func processRepo(context *ctx.Context, owner, repo string, actuallyDoIt bool) error {
	start := time.Now()

	issues, err := freeze.AllTooOldIssues(context, owner, repo)
	if err != nil {
		return err
	}

	log.Printf("%s/%s: Freezing %d closed issues before %v", owner, repo, len(issues), freeze.TooOld)
	for _, issue := range issues {
		if actuallyDoIt {
			log.Printf("%s/%s: freezing %s", owner, repo, *issue.HTMLURL)
			if err = freeze.Freeze(context, owner, repo, *issue.Number); err != nil {
				return err
			}
			time.Sleep(sleepBetweenFreezes)
		} else {
			log.Printf("%s/%s: would have frozen %s", owner, repo, *issue.HTMLURL)
			time.Sleep(sleepBetweenFreezes)
		}
	}

	log.Printf("%s/%s: finished in %s", owner, repo, time.Since(start))

	return nil
}
<commit_msg>Give us the ability to pass a list of issues (either as URL or owner/repo#number) to freeze-ancient-issues<commit_after>// A command-line utility to lock old issues.
package main

import (
	"flag"
	"fmt"
	"log"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/google/go-github/github"
	"github.com/parkr/auto-reply/ctx"
	"github.com/parkr/auto-reply/freeze"
)

type repository struct {
	Owner, Name string
}

var (
	defaultRepos = []repository{
		repository{"jekyll", "jekyll"},
		repository{"jekyll", "jekyll-admin"},
		repository{"jekyll", "jekyll-import"},
		repository{"jekyll", "github-metadata"},
		repository{"jekyll", "jekyll-redirect-from"},
		repository{"jekyll", "jekyll-feed"},
		repository{"jekyll", "jekyll-compose"},
		repository{"jekyll", "jekyll-watch"},
		repository{"jekyll", "jekyll-seo-tag"},
		repository{"jekyll", "jekyll-sitemap"},
		repository{"jekyll", "jekyll-sass-converter"},
		repository{"jekyll", "jemoji"},
		repository{"jekyll", "jekyll-gist"},
		repository{"jekyll", "jekyll-coffeescript"},
		repository{"jekyll", "plugins"},
	}

	sleepBetweenFreezes = 150 * time.Millisecond
)

func main() {
	var actuallyDoIt bool
	flag.BoolVar(&actuallyDoIt, "f", false, "Whether to actually mark the issues or close them.")
	var inputRepos string
	flag.StringVar(&inputRepos, "repos", "", "Specify a list of comma-separated repo name/owner pairs, e.g. 'jekyll/jekyll-admin'.")
	flag.Parse()

	var repos []repository
	if inputRepos == "" {
		repos = defaultRepos
	}

	context := ctx.NewDefaultContext()
	if context.GitHub == nil {
		log.Fatalln("cannot proceed without github client")
	}

	// Support running on just a list of issues. Either a URL or a `owner/name#number` syntax.
	if flag.NArg() > 0 {
		if err := processSingleIssues(context, actuallyDoIt, flag.Args()...); err != nil {
			log.Fatalf("error: %#v", err)
		}
		return
	}

	var wg sync.WaitGroup
	for _, repo := range repos {
		wg.Add(1)
		go func(context *ctx.Context, repo repository, actuallyDoIt bool) {
			defer wg.Done()
			if err := processRepo(context, repo.Owner, repo.Name, actuallyDoIt); err != nil {
				log.Printf("%s/%s: error: %#v", repo.Owner, repo.Name, err)
			}
		}(context, repo, actuallyDoIt)
	}

	wg.Wait()
}

func extractIssueInfo(issueName string) (owner, repo string, number int) {
	issueName = strings.TrimPrefix(issueName, "https://github.com/")

	var err error
	pieces := strings.Split(issueName, "/")

	// Ex: `owner/repo#number`
	if len(pieces) == 2 {
		owner = pieces[0]
		morePieces := strings.Split(pieces[1], "#")
		if len(morePieces) == 2 {
			repo = morePieces[0]
			number, err = strconv.Atoi(morePieces[1])
			if err != nil {
				log.Printf("huh? %#v for %s", err, morePieces[1])
			}
		}
		return
	}

	// Ex: `owner/repo/issues/number`
	if len(pieces) == 4 {
		owner = pieces[0]
		repo = pieces[1]
		number, err = strconv.Atoi(pieces[3])
		if err != nil {
			log.Printf("huh? %#v for %s", err, pieces[3])
		}
		return
	}

	return "", "", 0
}

func processSingleIssues(context *ctx.Context, actuallyDoIt bool, issueNames ...string) error {
	issues := []github.Issue{}
	for _, issueName := range issueNames {
		owner, repo, number := extractIssueInfo(issueName)
		if owner == "" || repo == "" || number <= 0 {
			return fmt.Errorf("couldn't extract issue info from '%s': owner=%s repo=%s number=%d",
				issueName, owner, repo, number)
		}

		issues = append(issues, github.Issue{
			Number: github.Int(number),
			Repository: &github.Repository{
				Owner: &github.User{Login: github.String(owner)},
				Name:  github.String(repo),
			},
		})
	}
	return processIssues(context, actuallyDoIt, issues)
}

func processRepo(context *ctx.Context, owner, repo string, actuallyDoIt bool) error {
	start := time.Now()

	issues, err := freeze.AllTooOldIssues(context, owner, repo)
	if err != nil {
		return err
	}

	log.Printf("%s/%s: freezing %d closed issues before %v", owner, repo, len(issues), freeze.TooOld)
	err = processIssues(context, actuallyDoIt, issues)
	log.Printf("%s/%s: finished in %s", owner, repo, time.Since(start))

	return err
}

func processIssues(context *ctx.Context, actuallyDoIt bool, issues []github.Issue) error {
	for _, issue := range issues {
		owner, repo := *issue.Repository.Owner.Login, *issue.Repository.Name
		if actuallyDoIt {
			log.Printf("%s/%s: freezing %s", owner, repo, *issue.HTMLURL)
			if err := freeze.Freeze(context, owner, repo, *issue.Number); err != nil {
				return err
			}
			time.Sleep(sleepBetweenFreezes)
		} else {
			log.Printf("%s/%s: would have frozen %s", owner, repo, *issue.HTMLURL)
			time.Sleep(sleepBetweenFreezes)
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package redis

import (
	"net"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/funkygao/Go-Redis"
	"github.com/funkygao/gafka/cmd/kguard/monitor"
	"github.com/funkygao/gafka/telemetry"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/go-metrics"
	log "github.com/funkygao/log4go"
)

func init() {
	monitor.RegisterWatcher("redis.info", func() monitor.Watcher {
		return &WatchRedisInfo{
			Tick: time.Minute,
		}
	})
}

// WatchRedisInfo watches registered redis instances with redis 'info' command.
type WatchRedisInfo struct {
	Zkzone *zk.ZkZone
	Stop   <-chan struct{}
	Tick   time.Duration
	Wg     *sync.WaitGroup

	mu                        sync.Mutex
	deadInstance, syncPartial metrics.Counter
	instances                 metrics.Gauge
	conns                     map[string]metrics.Gauge
	blocked                   map[string]metrics.Gauge
	usedMem                   map[string]metrics.Gauge
	ops                       map[string]metrics.Gauge
	rejected                  map[string]metrics.Gauge
	rxKbps                    map[string]metrics.Gauge
	txKbps                    map[string]metrics.Gauge
	expiredKeys               map[string]metrics.Gauge
	keys                      map[string]metrics.Gauge
}

func (this *WatchRedisInfo) Init(ctx monitor.Context) {
	this.Zkzone = ctx.ZkZone()
	this.Stop = ctx.StopChan()
	this.Wg = ctx.Inflight()
}

func (this *WatchRedisInfo) Run() {
	defer this.Wg.Done()

	ticker := time.NewTicker(this.Tick)
	defer ticker.Stop()

	this.instances = metrics.NewRegisteredGauge("redis.n", nil)
	this.deadInstance = metrics.NewRegisteredCounter("redis.dead", nil)
	this.syncPartial = metrics.NewRegisteredCounter("redis.sync.partial", nil)

	this.conns = make(map[string]metrics.Gauge, 10)
	this.blocked = make(map[string]metrics.Gauge, 10)
	this.usedMem = make(map[string]metrics.Gauge, 10)
	this.ops = make(map[string]metrics.Gauge, 10)
	this.rejected = make(map[string]metrics.Gauge, 10)
	this.rxKbps = make(map[string]metrics.Gauge, 10)
	this.txKbps = make(map[string]metrics.Gauge, 10)
	this.expiredKeys = make(map[string]metrics.Gauge, 10)
	this.keys = make(map[string]metrics.Gauge, 10)

	for {
		select {
		case <-this.Stop:
			log.Info("redis.info stopped")
			return

		case <-ticker.C:
			var (
				wg     sync.WaitGroup
				redisN int64
			)

			for _, hostPort := range this.Zkzone.AllRedis() {
				host, port, err := net.SplitHostPort(hostPort)
				if err != nil {
					log.Error("invalid redis instance: %s", hostPort)
					continue
				}

				nport, err := strconv.Atoi(port)
				if err != nil || nport < 0 {
					log.Error("invalid redis instance: %s", hostPort)
					continue
				}

				redisN++

				// TODO ver=role(master|slave)
				tag := telemetry.Tag(strings.Replace(host, ".", "_", -1), port, "v1")
				if _, present := this.conns[tag]; !present {
					this.mu.Lock()
					this.conns[tag] = metrics.NewRegisteredGauge(tag+"redis.conns", nil)              // connected_clients
					this.blocked[tag] = metrics.NewRegisteredGauge(tag+"redis.blocked", nil)          // blocked_clients
					this.usedMem[tag] = metrics.NewRegisteredGauge(tag+"redis.mem.used", nil)         // used_memory
					this.ops[tag] = metrics.NewRegisteredGauge(tag+"redis.ops", nil)                  // instantaneous_ops_per_sec
					this.rejected[tag] = metrics.NewRegisteredGauge(tag+"redis.rejected", nil)        // rejected_connections
					this.rxKbps[tag] = metrics.NewRegisteredGauge(tag+"redis.rx.kbps", nil)           // instantaneous_input_kbps
					this.txKbps[tag] = metrics.NewRegisteredGauge(tag+"redis.tx.kbps", nil)           // instantaneous_output_kbps
					this.expiredKeys[tag] = metrics.NewRegisteredGauge(tag+"redis.expired.keys", nil) // expired_keys
					this.keys[tag] = metrics.NewRegisteredGauge(tag+"redis.keys", nil)                // db0:keys=15500,expires=15500,avg_ttl=27438570
					this.mu.Unlock()
				}

				wg.Add(1)
				go this.updateRedisInfo(&wg, host, nport, tag)
			}

			this.instances.Update(redisN)
			wg.Wait()

		}
	}
}

func (this *WatchRedisInfo) updateRedisInfo(wg *sync.WaitGroup, host string, port int, tag string) {
	defer wg.Done()

	spec := redis.DefaultSpec().Host(host).Port(port)
	client, err := redis.NewSynchClientWithSpec(spec)
	if err != nil {
		log.Error("redis[%s:%d]: %v", host, port, err)
		this.deadInstance.Inc(1)
		return
	}
	defer client.Quit()

	infoMap, err := client.Info()
	if err != nil {
		log.Error("redis[%s:%d] info: %v", host, port, err)
		this.deadInstance.Inc(1)
		return
	}

	var keysN int64
	// db0:keys=15500,expires=15500,avg_ttl=27438570
	for key, value := range infoMap {
		if strings.HasPrefix(key, "db") && strings.Contains(value, "keys=") {
			keysN += extractKeysCount(value)
		}
	}

	conns, _ := strconv.ParseInt(infoMap["connected_clients"], 10, 64)
	blocked, _ := strconv.ParseInt(infoMap["blocked_clients"], 10, 64)
	mem, _ := strconv.ParseInt(infoMap["used_memory"], 10, 64)
	ops, _ := strconv.ParseInt(infoMap["instantaneous_ops_per_sec"], 10, 64)
	rejected, _ := strconv.ParseInt(infoMap["rejected_connections"], 10, 64)
	syncPartial, _ := strconv.ParseInt(infoMap["sync_partial_err"], 10, 64)
	rxKbps, _ := strconv.ParseFloat(infoMap["instantaneous_input_kbps"], 64)
	txKbps, _ := strconv.ParseFloat(infoMap["instantaneous_output_kbps"], 64)
	expiredKeys, _ := strconv.ParseInt(infoMap["expired_keys"], 10, 64)

	this.syncPartial.Inc(syncPartial)

	this.mu.Lock()
	this.keys[tag].Update(keysN)
	this.conns[tag].Update(conns)
	this.blocked[tag].Update(blocked)
	this.usedMem[tag].Update(mem)
	this.ops[tag].Update(ops)
	this.rejected[tag].Update(rejected)
	this.rxKbps[tag].Update(int64(rxKbps))
	this.txKbps[tag].Update(int64(txKbps))
	this.expiredKeys[tag].Update(expiredKeys)
	this.mu.Unlock()
}

func extractKeysCount(info string) (n int64) {
	parts := strings.SplitN(info, "keys=", 2)
	if len(parts) != 2 {
		return
	}

	p := strings.SplitN(parts[1], ",", 2)
	if len(p) != 2 {
		return
	}

	n, _ = strconv.ParseInt(p[0], 10, 64)
	return

}
<commit_msg>should use gauge instead of counter<commit_after>package redis

import (
	"net"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/funkygao/Go-Redis"
	"github.com/funkygao/gafka/cmd/kguard/monitor"
	"github.com/funkygao/gafka/telemetry"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/go-metrics"
	log "github.com/funkygao/log4go"
)

func init() {
	monitor.RegisterWatcher("redis.info", func() monitor.Watcher {
		return &WatchRedisInfo{
			Tick: time.Minute,
		}
	})
}

// WatchRedisInfo watches registered redis instances with redis 'info' command.
type WatchRedisInfo struct {
	Zkzone *zk.ZkZone
	Stop   <-chan struct{}
	Tick   time.Duration
	Wg     *sync.WaitGroup

	deadN, syncPartialN int64

	mu                        sync.Mutex
	deadInstance, syncPartial metrics.Gauge
	instances                 metrics.Gauge
	conns                     map[string]metrics.Gauge
	blocked                   map[string]metrics.Gauge
	usedMem                   map[string]metrics.Gauge
	ops                       map[string]metrics.Gauge
	rejected                  map[string]metrics.Gauge
	rxKbps                    map[string]metrics.Gauge
	txKbps                    map[string]metrics.Gauge
	expiredKeys               map[string]metrics.Gauge
	keys                      map[string]metrics.Gauge
}

func (this *WatchRedisInfo) Init(ctx monitor.Context) {
	this.Zkzone = ctx.ZkZone()
	this.Stop = ctx.StopChan()
	this.Wg = ctx.Inflight()
}

func (this *WatchRedisInfo) Run() {
	defer this.Wg.Done()

	ticker := time.NewTicker(this.Tick)
	defer ticker.Stop()

	this.instances = metrics.NewRegisteredGauge("redis.n", nil)
	this.deadInstance = metrics.NewRegisteredGauge("redis.dead", nil)
	this.syncPartial = metrics.NewRegisteredGauge("redis.sync.partial", nil)

	this.conns = make(map[string]metrics.Gauge, 10)
	this.blocked = make(map[string]metrics.Gauge, 10)
	this.usedMem = make(map[string]metrics.Gauge, 10)
	this.ops = make(map[string]metrics.Gauge, 10)
	this.rejected = make(map[string]metrics.Gauge, 10)
	this.rxKbps = make(map[string]metrics.Gauge, 10)
	this.txKbps = make(map[string]metrics.Gauge, 10)
	this.expiredKeys = make(map[string]metrics.Gauge, 10)
	this.keys = make(map[string]metrics.Gauge, 10)

	for {
		select {
		case <-this.Stop:
			log.Info("redis.info stopped")
			return

		case <-ticker.C:
			var (
				wg     sync.WaitGroup
				redisN int64
			)

			for _, hostPort := range this.Zkzone.AllRedis() {
				host, port, err := net.SplitHostPort(hostPort)
				if err != nil {
					log.Error("invalid redis instance: %s", hostPort)
					continue
				}

				nport, err := strconv.Atoi(port)
				if err != nil || nport < 0 {
					log.Error("invalid redis instance: %s", hostPort)
					continue
				}

				redisN++

				// TODO ver=role(master|slave)
				tag := telemetry.Tag(strings.Replace(host, ".", "_", -1), port, "v1")
				if _, present := this.conns[tag]; !present {
					this.mu.Lock()
					this.conns[tag] = metrics.NewRegisteredGauge(tag+"redis.conns", nil)              // connected_clients
					this.blocked[tag] = metrics.NewRegisteredGauge(tag+"redis.blocked", nil)          // blocked_clients
					this.usedMem[tag] = metrics.NewRegisteredGauge(tag+"redis.mem.used", nil)         // used_memory
					this.ops[tag] = metrics.NewRegisteredGauge(tag+"redis.ops", nil)                  // instantaneous_ops_per_sec
					this.rejected[tag] = metrics.NewRegisteredGauge(tag+"redis.rejected", nil)        // rejected_connections
					this.rxKbps[tag] = metrics.NewRegisteredGauge(tag+"redis.rx.kbps", nil)           // instantaneous_input_kbps
					this.txKbps[tag] = metrics.NewRegisteredGauge(tag+"redis.tx.kbps", nil)           // instantaneous_output_kbps
					this.expiredKeys[tag] = metrics.NewRegisteredGauge(tag+"redis.expired.keys", nil) // expired_keys
					this.keys[tag] = metrics.NewRegisteredGauge(tag+"redis.keys", nil)                // db0:keys=15500,expires=15500,avg_ttl=27438570
					this.mu.Unlock()
				}

				wg.Add(1)
				go this.updateRedisInfo(&wg, host, nport, tag)
			}

			this.instances.Update(redisN)
			this.syncPartial.Update(atomic.LoadInt64(&this.syncPartialN))
			this.deadInstance.Update(atomic.LoadInt64(&this.deadN))
			// reset for next round
			atomic.StoreInt64(&this.syncPartialN, 0)
			atomic.StoreInt64(&this.deadN, 0)
			wg.Wait()

		}
	}
}

func (this *WatchRedisInfo) updateRedisInfo(wg *sync.WaitGroup, host string, port int, tag string) {
	defer wg.Done()

	spec := redis.DefaultSpec().Host(host).Port(port)
	client, err := redis.NewSynchClientWithSpec(spec)
	if err != nil {
		log.Error("redis[%s:%d]: %v", host, port, err)
		atomic.AddInt64(&this.deadN, 1)
		return
	}
	defer client.Quit()

	infoMap, err := client.Info()
	if err != nil {
		log.Error("redis[%s:%d] info: %v", host, port, err)
		atomic.AddInt64(&this.deadN, 1)
		return
	}

	var keysN int64
	// db0:keys=15500,expires=15500,avg_ttl=27438570
	for key, value := range infoMap {
		if strings.HasPrefix(key, "db") && strings.Contains(value, "keys=") {
			keysN += extractKeysCount(value)
		}
	}

	conns, _ := strconv.ParseInt(infoMap["connected_clients"], 10, 64)
	blocked, _ := strconv.ParseInt(infoMap["blocked_clients"], 10, 64)
	mem, _ := strconv.ParseInt(infoMap["used_memory"], 10, 64)
	ops, _ := strconv.ParseInt(infoMap["instantaneous_ops_per_sec"], 10, 64)
	rejected, _ := strconv.ParseInt(infoMap["rejected_connections"], 10, 64)
	syncPartial, _ := strconv.ParseInt(infoMap["sync_partial_err"], 10, 64)
	rxKbps, _ := strconv.ParseFloat(infoMap["instantaneous_input_kbps"], 64)
	txKbps, _ := strconv.ParseFloat(infoMap["instantaneous_output_kbps"], 64)
	expiredKeys, _ := strconv.ParseInt(infoMap["expired_keys"], 10, 64)

	atomic.AddInt64(&this.syncPartialN, syncPartial)

	this.mu.Lock()
	this.keys[tag].Update(keysN)
	this.conns[tag].Update(conns)
	this.blocked[tag].Update(blocked)
	this.usedMem[tag].Update(mem)
	this.ops[tag].Update(ops)
	this.rejected[tag].Update(rejected)
	this.rxKbps[tag].Update(int64(rxKbps))
	this.txKbps[tag].Update(int64(txKbps))
	this.expiredKeys[tag].Update(expiredKeys)
	this.mu.Unlock()
}

func extractKeysCount(info string) (n int64) {
	parts := strings.SplitN(info, "keys=", 2)
	if len(parts) != 2 {
		return
	}

	p := strings.SplitN(parts[1], ",", 2)
	if len(p) != 2 {
		return
	}

	n, _ = strconv.ParseInt(p[0], 10, 64)
	return

}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package config

import (
	"encoding/json"
	"fmt"
	"io"
	"os"
	"strings"

	"github.com/golang/glog"
	"github.com/spf13/cobra"
	"k8s.io/minikube/pkg/minikube/config"
	"k8s.io/minikube/pkg/minikube/constants"
)

// Bootstrapper is the name for bootstrapper
const Bootstrapper = "bootstrapper"

type setFn func(string, string) error

// Setting represents a setting
type Setting struct {
	name        string
	set         func(config.MinikubeConfig, string, string) error
	setMap      func(config.MinikubeConfig, string, map[string]interface{}) error
	validations []setFn
	callbacks   []setFn
}

// These are all the settings that are configurable
// and their validation and callback fn run on Set
var settings = []Setting{
	{
		name:        "vm-driver",
		set:         SetString,
		validations: []setFn{IsValidDriver},
		callbacks:   []setFn{RequiresRestartMsg},
	},
	{
		name:      "feature-gates",
		set:       SetString,
		callbacks: []setFn{RequiresRestartMsg},
	},
	{
		name:        "v",
		set:         SetInt,
		validations: []setFn{IsPositive},
	},
	{
		name:        "cpus",
		set:         SetInt,
		validations: []setFn{IsPositive},
		callbacks:   []setFn{RequiresRestartMsg},
	},
	{
		name:        "disk-size",
		set:         SetString,
		validations: []setFn{IsValidDiskSize},
		callbacks:   []setFn{RequiresRestartMsg},
	},
	{
		name:        "host-only-cidr",
		set:         SetString,
		validations: []setFn{IsValidCIDR},
	},
	{
		name:        "memory",
		set:         SetInt,
		validations: []setFn{IsPositive},
		callbacks:   []setFn{RequiresRestartMsg},
	},
	{
		name:        "log_dir",
		set:         SetString,
		validations: []setFn{IsValidPath},
	},
	{
		name: "kubernetes-version",
		set:  SetString,
	},
	{
		name:        "iso-url",
		set:         SetString,
		validations: []setFn{IsValidURL, IsURLExists},
	},
	{
		name: config.WantUpdateNotification,
		set:  SetBool,
	},
	{
		name: config.ReminderWaitPeriodInHours,
		set:  SetInt,
	},
	{
		name: config.WantReportError,
		set:  SetBool,
	},
	{
		name: config.WantReportErrorPrompt,
		set:  SetBool,
	},
	{
		name: config.WantKubectlDownloadMsg,
		set:  SetBool,
	},
	{
		name: config.WantNoneDriverWarning,
		set:  SetBool,
	},
	{
		name: config.MachineProfile,
		set:  SetString,
	},
	{
		name: Bootstrapper,
		set:  SetString, //TODO(r2d4): more validation here?
	},
	{
		name: config.ShowDriverDeprecationNotification,
		set:  SetBool,
	},
	{
		name: config.ShowBootstrapperDeprecationNotification,
		set:  SetBool,
	},
	{
		name:        "dashboard",
		set:         SetBool,
		validations: []setFn{IsValidAddon},
		callbacks:   []setFn{EnableOrDisableAddon},
	},
	{
		name:        "addon-manager",
		set:         SetBool,
		validations: []setFn{IsValidAddon},
		callbacks:   []setFn{EnableOrDisableAddon},
	},
	{
		name:        "default-storageclass",
		set:         SetBool,
		validations: []setFn{IsValidAddon},
		callbacks:   []setFn{EnableOrDisableStorageClasses},
	},
	{
		name:        "heapster",
		set:         SetBool,
		validations: []setFn{IsValidAddon},
		callbacks:   []setFn{EnableOrDisableAddon},
	},
	{
		name:        "efk",
		set:         SetBool,
		validations: []setFn{IsValidAddon},
		callbacks:   []setFn{EnableOrDisableAddon},
	},
	{
		name:        "ingress",
		set:         SetBool,
		validations: []setFn{IsValidAddon},
		callbacks:   []setFn{EnableOrDisableAddon},
	},
	{
		name:        "registry",
		set:         SetBool,
		validations: []setFn{IsValidAddon},
		callbacks:   []setFn{EnableOrDisableAddon},
	},
	{
		name:        "registry-creds",
		set:         SetBool,
		validations: []setFn{IsValidAddon},
		callbacks:   []setFn{EnableOrDisableAddon},
	},
	{
		name:        "freshpod",
		set:         SetBool,
		validations: []setFn{IsValidAddon},
		callbacks:   []setFn{EnableOrDisableAddon},
	},
	{
		name:        "default-storageclass",
		set:         SetBool,
		validations: []setFn{IsValidAddon},
		callbacks:   []setFn{EnableOrDisableStorageClasses},
	},
	{
		name:        "storage-provisioner",
		set:         SetBool,
		validations: []setFn{IsValidAddon},
		callbacks:   []setFn{EnableOrDisableAddon},
	},
	{
		name:        "storage-provisioner-gluster",
		set:         SetBool,
		validations: []setFn{IsValidAddon},
		callbacks:   []setFn{EnableOrDisableStorageClasses},
	},
	{
		name:        "metrics-server",
		set:         SetBool,
		validations: []setFn{IsValidAddon},
		callbacks:   []setFn{EnableOrDisableAddon},
	},
	{
		name:        "nvidia-driver-installer",
		set:         SetBool,
		validations: []setFn{IsValidAddon},
		callbacks:   []setFn{EnableOrDisableAddon},
	},
	{
		name:        "nvidia-gpu-device-plugin",
		set:         SetBool,
		validations: []setFn{IsValidAddon},
		callbacks:   []setFn{EnableOrDisableAddon},
	},
	{
		name:        "logviewer",
		set:         SetBool,
		validations: []setFn{IsValidAddon},
	},
	{
		name:        "gvisor",
		set:         SetBool,
		validations: []setFn{IsValidAddon, IsContainerdRuntime},
		callbacks:   []setFn{EnableOrDisableAddon},
	},
	{
		name: "hyperv-virtual-switch",
		set:  SetString,
	},
	{
		name: "disable-driver-mounts",
		set:  SetBool,
	},
	{
		name:   "cache",
		set:    SetConfigMap,
		setMap: SetMap,
	},
	{
		name: "embed-certs",
		set:  SetBool,
	},
}

// ConfigCmd represents the config command
var ConfigCmd = &cobra.Command{
	Use:   "config SUBCOMMAND [flags]",
	Short: "Modify minikube config",
	Long: `config modifies minikube config files using subcommands like "minikube config set vm-driver kvm"
Configurable fields: ` + "\n\n" + configurableFields(),
	Run: func(cmd *cobra.Command, args []string) {
		if err := cmd.Help(); err != nil {
			glog.Errorf("help: %v", err)
		}
	},
}

func configurableFields() string {
	fields := []string{}
	for _, s := range settings {
		fields = append(fields, " * "+s.name)
	}
	return strings.Join(fields, "\n")
}

// ListConfigMap list entries from config file
func ListConfigMap(name string) ([]string, error) {
	configFile, err := config.ReadConfig()
	if err != nil {
		return nil, err
	}
	var images []string
	if values, ok := configFile[name].(map[string]interface{}); ok {
		for key := range values {
			images = append(images, key)
		}
	}
	return images, nil
}

// AddToConfigMap adds entries to a map in the config file
func AddToConfigMap(name string, images []string) error {
	s, err := findSetting(name)
	if err != nil {
		return err
	}
	// Set the values
	configFile, err := config.ReadConfig()
	if err != nil {
		return err
	}
	newImages := make(map[string]interface{})
	for _, image := range images {
		newImages[image] = nil
	}
	if values, ok := configFile[name].(map[string]interface{}); ok {
		for key := range values {
			newImages[key] = nil
		}
	}
	if err = s.setMap(configFile, name, newImages); err != nil {
		return err
	}
	// Write the values
	return WriteConfig(configFile)
}

// DeleteFromConfigMap deletes entries from a map in the config file
func DeleteFromConfigMap(name string, images []string) error {
	s, err := findSetting(name)
	if err != nil {
		return err
	}
	// Set the values
	configFile, err := config.ReadConfig()
	if err != nil {
		return err
	}
	values, ok := configFile[name]
	if !ok {
		return nil
	}
	for _, image := range images {
		delete(values.(map[string]interface{}), image)
	}
	if err = s.setMap(configFile, name, values.(map[string]interface{})); err != nil {
		return err
	}
	// Write the values
	return WriteConfig(configFile)
}

// WriteConfig writes a minikube config to the JSON file
func WriteConfig(m config.MinikubeConfig) error {
	f, err := os.Create(constants.ConfigFile)
	if err != nil {
		return fmt.Errorf("create %s: %s", constants.ConfigFile, err)
	}
	defer f.Close()
	err = encode(f, m)
	if err != nil {
		return fmt.Errorf("encode %s: %s", constants.ConfigFile, err)
	}
	return nil
}

func encode(w io.Writer, m config.MinikubeConfig) error {
	b, err := json.MarshalIndent(m, "", "    ")
	if err != nil {
		return err
	}

	_, err = w.Write(b)

	return err
}
<commit_msg>[FEATURE] allow to set container-runtime via config<commit_after>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package config

import (
	"encoding/json"
	"fmt"
	"io"
	"os"
	"strings"

	"github.com/golang/glog"
	"github.com/spf13/cobra"
	"k8s.io/minikube/pkg/minikube/config"
	"k8s.io/minikube/pkg/minikube/constants"
)

// Bootstrapper is the name for bootstrapper
const Bootstrapper = "bootstrapper"

type setFn func(string, string) error

// Setting represents a setting
type Setting struct {
	name        string
	set         func(config.MinikubeConfig, string, string) error
	setMap      func(config.MinikubeConfig, string, map[string]interface{}) error
	validations []setFn
	callbacks   []setFn
}

// These are all the settings that are configurable
// and their validation and callback fn run on Set
var settings = []Setting{
	{
		name:        "vm-driver",
		set:         SetString,
		validations: []setFn{IsValidDriver},
		callbacks:   []setFn{RequiresRestartMsg},
	},
	{
		name:        "container-runtime",
		set:         SetString,
		validations: []setFn{IsContainerdRuntime},
		callbacks:   []setFn{RequiresRestartMsg},
	},
	{
		name:      "feature-gates",
		set:       SetString,
		callbacks: []setFn{RequiresRestartMsg},
	},
	{
		name:        "v",
		set:         SetInt,
		validations: []setFn{IsPositive},
	},
	{
		name:        "cpus",
		set:         SetInt,
		validations: []setFn{IsPositive},
		callbacks:   []setFn{RequiresRestartMsg},
	},
	{
		name:        "disk-size",
		set:         SetString,
		validations: []setFn{IsValidDiskSize},
		callbacks:   []setFn{RequiresRestartMsg},
	},
	{
		name:        "host-only-cidr",
		set:         SetString,
		validations: []setFn{IsValidCIDR},
	},
	{
		name:        "memory",
		set:         SetInt,
		validations: []setFn{IsPositive},
		callbacks:   []setFn{RequiresRestartMsg},
	},
	{
		name:        "log_dir",
		set:         SetString,
		validations: []setFn{IsValidPath},
	},
	{
		name: "kubernetes-version",
		set:  SetString,
	},
	{
		name:        "iso-url",
		set:         SetString,
		validations: []setFn{IsValidURL, IsURLExists},
	},
	{
		name: config.WantUpdateNotification,
		set:  SetBool,
	},
	{
		name: config.ReminderWaitPeriodInHours,
		set:  SetInt,
	},
	{
		name: config.WantReportError,
		set:  SetBool,
	},
	{
		name: config.WantReportErrorPrompt,
		set:  SetBool,
	},
	{
		name: config.WantKubectlDownloadMsg,
		set:  SetBool,
	},
	{
		name: config.WantNoneDriverWarning,
		set:  SetBool,
	},
	{
		name: config.MachineProfile,
		set:  SetString,
	},
	{
		name: Bootstrapper,
		set:  SetString, //TODO(r2d4): more validation here?
	},
	{
		name: config.ShowDriverDeprecationNotification,
		set:  SetBool,
	},
	{
		name: config.ShowBootstrapperDeprecationNotification,
		set:  SetBool,
	},
	{
		name:        "dashboard",
		set:         SetBool,
		validations: []setFn{IsValidAddon},
		callbacks:   []setFn{EnableOrDisableAddon},
	},
	{
		name:        "addon-manager",
		set:         SetBool,
		validations: []setFn{IsValidAddon},
		callbacks:   []setFn{EnableOrDisableAddon},
	},
	{
		name:        "default-storageclass",
		set:         SetBool,
		validations: []setFn{IsValidAddon},
		callbacks:   []setFn{EnableOrDisableStorageClasses},
	},
	{
		name:        "heapster",
		set:         SetBool,
		validations: []setFn{IsValidAddon},
		callbacks:   []setFn{EnableOrDisableAddon},
	},
	{
		name:        "efk",
		set:         SetBool,
		validations: []setFn{IsValidAddon},
		callbacks:   []setFn{EnableOrDisableAddon},
	},
	{
		name:        "ingress",
		set:         SetBool,
		validations: []setFn{IsValidAddon},
		callbacks:   []setFn{EnableOrDisableAddon},
	},
	{
		name:        "registry",
		set:         SetBool,
		validations: []setFn{IsValidAddon},
		callbacks:   []setFn{EnableOrDisableAddon},
	},
	{
		name:        "registry-creds",
		set:         SetBool,
		validations: []setFn{IsValidAddon},
		callbacks:   []setFn{EnableOrDisableAddon},
	},
	{
		name:        "freshpod",
		set:         SetBool,
		validations: []setFn{IsValidAddon},
		callbacks:   []setFn{EnableOrDisableAddon},
	},
	{
		name:        "default-storageclass",
		set:         SetBool,
		validations: []setFn{IsValidAddon},
		callbacks:   []setFn{EnableOrDisableStorageClasses},
	},
	{
		name:        "storage-provisioner",
		set:         SetBool,
		validations: []setFn{IsValidAddon},
		callbacks:   []setFn{EnableOrDisableAddon},
	},
	{
		name:        "storage-provisioner-gluster",
		set:         SetBool,
		validations: []setFn{IsValidAddon},
		callbacks:   []setFn{EnableOrDisableStorageClasses},
	},
	{
		name:        "metrics-server",
		set:         SetBool,
		validations: []setFn{IsValidAddon},
		callbacks:   []setFn{EnableOrDisableAddon},
	},
	{
		name:        "nvidia-driver-installer",
		set:         SetBool,
		validations: []setFn{IsValidAddon},
		callbacks:   []setFn{EnableOrDisableAddon},
	},
	{
		name:        "nvidia-gpu-device-plugin",
		set:         SetBool,
		validations: []setFn{IsValidAddon},
		callbacks:   []setFn{EnableOrDisableAddon},
	},
	{
		name:        "logviewer",
		set:         SetBool,
		validations: []setFn{IsValidAddon},
	},
	{
		name:        "gvisor",
		set:         SetBool,
		validations: []setFn{IsValidAddon, IsContainerdRuntime},
		callbacks:   []setFn{EnableOrDisableAddon},
	},
	{
		name: "hyperv-virtual-switch",
		set:  SetString,
	},
	{
		name: "disable-driver-mounts",
		set:  SetBool,
	},
	{
		name:   "cache",
		set:    SetConfigMap,
		setMap: SetMap,
	},
	{
		name: "embed-certs",
		set:  SetBool,
	},
}

// ConfigCmd represents the config command
var ConfigCmd = &cobra.Command{
	Use:   "config SUBCOMMAND [flags]",
	Short: "Modify minikube config",
	Long: `config modifies minikube config files using subcommands like "minikube config set vm-driver kvm"
Configurable fields: ` + "\n\n" + configurableFields(),
	Run: func(cmd *cobra.Command, args []string) {
		if err := cmd.Help(); err != nil {
			glog.Errorf("help: %v", err)
		}
	},
}

func configurableFields() string {
	fields := []string{}
	for _, s := range settings {
		fields = append(fields, " * "+s.name)
	}
	return strings.Join(fields, "\n")
}

// ListConfigMap list entries from config file
func ListConfigMap(name string) ([]string, error) {
	configFile, err := config.ReadConfig()
	if err != nil {
		return nil, err
	}
	var images []string
	if values, ok := configFile[name].(map[string]interface{}); ok {
		for key := range values {
			images = append(images, key)
		}
	}
	return images, nil
}

// AddToConfigMap adds entries to a map in the config file
func AddToConfigMap(name string, images []string) error {
	s, err := findSetting(name)
	if err != nil {
		return err
	}
	// Set the values
	configFile, err := config.ReadConfig()
	if err != nil {
		return err
	}
	newImages := make(map[string]interface{})
	for _, image := range images {
		newImages[image] = nil
	}
	if values, ok := configFile[name].(map[string]interface{}); ok {
		for key := range values {
			newImages[key] = nil
		}
	}
	if err = s.setMap(configFile, name, newImages); err != nil {
		return err
	}
	// Write the values
	return WriteConfig(configFile)
}

// DeleteFromConfigMap deletes entries from a map in the config file
func DeleteFromConfigMap(name string, images []string) error {
	s, err := findSetting(name)
	if err != nil {
		return err
	}
	// Set the values
	configFile, err := config.ReadConfig()
	if err != nil {
		return err
	}
	values, ok := configFile[name]
	if !ok {
		return nil
	}
	for _, image := range images {
		delete(values.(map[string]interface{}), image)
	}
	if err = s.setMap(configFile, name, values.(map[string]interface{})); err != nil {
		return err
	}
	// Write the values
	return WriteConfig(configFile)
}

// WriteConfig writes a minikube config to the JSON file
func WriteConfig(m config.MinikubeConfig) error {
	f, err := os.Create(constants.ConfigFile)
	if err != nil {
		return fmt.Errorf("create %s: %s", constants.ConfigFile, err)
	}
	defer f.Close()
	err = encode(f, m)
	if err != nil {
		return fmt.Errorf("encode %s: %s", constants.ConfigFile, err)
	}
	return nil
}

func encode(w io.Writer, m config.MinikubeConfig) error {
	b, err := json.MarshalIndent(m, "", "    ")
	if err != nil {
		return err
	}

	_, err = w.Write(b)

	return err
}
<|endoftext|>


<commit_before>// Package collect provides functions for sending data to OpenTSDB.
//
// The "collect" namespace is used (i.e., <metric_root>.collect) to collect
// program and queue metrics.
package collect

import (
	"fmt"
	"net/http"
	"net/url"
	"os"
	"runtime"
	"strings"
	"sync"
	"time"

	"github.com/StackExchange/scollector/opentsdb"
)

var (
	// Freq is how often metrics are sent to OpenTSDB.
	Freq = time.Second * 15

	// MaxQueueLen is the maximum size of the queue, above which incoming data will
	// be discarded. Defaults to about 150MB.
	MaxQueueLen = 200000

	// BatchSize is the maximum length of data points sent at once to OpenTSDB.
	BatchSize = 50

	// Debug enables debug logging.
	Debug = false

	// Dropped is the number of dropped data points due to a full queue.
	dropped int64

	// Sent is the number of sent data points.
	sent int64

	tchan               chan *opentsdb.DataPoint
	tsdbURL             string
	osHostname          string
	metricRoot          string
	queue               opentsdb.MultiDataPoint
	qlock, mlock, slock sync.Mutex   // Locks for queues, maps, stats.
	counters                         = make(map[string]*addMetric)
	sets                             = make(map[string]*setMetric)
	puts                             = make(map[string]*putMetric)
	client              *http.Client = &http.Client{
		Transport: &timeoutTransport{Transport: new(http.Transport)},
		Timeout:   time.Minute,
	}
)

type timeoutTransport struct {
	*http.Transport
	Timeout time.Time
}

func (t *timeoutTransport) RoundTrip(r *http.Request) (*http.Response, error) {
	if time.Now().After(t.Timeout) {
		t.Transport.CloseIdleConnections()
		t.Timeout = time.Now().Add(time.Minute * 5)
	}
	return t.Transport.RoundTrip(r)
}

// InitChan is similar to Init, but uses the given channel instead of creating a
// new one.
func InitChan(tsdbhost, metric_root string, ch chan *opentsdb.DataPoint) error {
	if tchan != nil {
		return fmt.Errorf("cannot init twice")
	}
	if err := checkClean(metric_root, "metric root"); err != nil {
		return err
	}
	if tsdbhost == "" {
		return fmt.Errorf("must specify non-empty tsdb host")
	}
	u := url.URL{
		Scheme: "http",
		Path:   "/api/put",
	}
	if !strings.Contains(tsdbhost, ":") {
		tsdbhost += ":4242"
	}
	u.Host = tsdbhost
	tsdbURL = u.String()
	metricRoot = metric_root + "."
	tchan = ch
	go func() {
		for dp := range tchan {
			qlock.Lock()
			for {
				if len(queue) > MaxQueueLen {
					slock.Lock()
					dropped++
					slock.Unlock()
					break
				}
				queue = append(queue, dp)
				select {
				case dp = <-tchan:
					continue
				default:
				}
				break
			}
			qlock.Unlock()
		}
	}()
	go send()

	go collect()
	Set("collect.dropped", nil, func() (i interface{}) {
		slock.Lock()
		i = dropped
		slock.Unlock()
		return
	})
	Set("collect.sent", nil, func() (i interface{}) {
		slock.Lock()
		i = sent
		slock.Unlock()
		return
	})
	Set("collect.alloc", nil, func() interface{} {
		var ms runtime.MemStats
		runtime.ReadMemStats(&ms)
		return ms.Alloc
	})
	Set("collect.goroutines", nil, func() interface{} {
		return runtime.NumGoroutine()
	})
	return nil
}

// Init sets up the channels and the queue for sending data to OpenTSDB. It also
// sets up the basename for all metrics.
func Init(tsdbhost, metric_root string) error {
	return InitChan(tsdbhost, metric_root, make(chan *opentsdb.DataPoint))
}

func setHostName() error {
	h, err := os.Hostname()
	if err != nil {
		return err
	}
	osHostname = strings.ToLower(strings.SplitN(h, ".", 2)[0])
	if err := checkClean(osHostname, "host tag"); err != nil {
		return err
	}
	return nil
}

type setMetric struct {
	metric string
	ts     opentsdb.TagSet
	f      func() interface{}
}

func Set(metric string, ts opentsdb.TagSet, f func() interface{}) error {
	if err := check(metric, &ts); err != nil {
		return err
	}
	tss := metric + ts.String()
	mlock.Lock()
	sets[tss] = &setMetric{metric, ts.Copy(), f}
	mlock.Unlock()
	return nil
}

type addMetric struct {
	metric string
	ts     opentsdb.TagSet
	value  int64
}

// Add takes a metric and increments a counter for that metric. The metric name
// is appended to the basename specified in the Init function.
func Add(metric string, ts opentsdb.TagSet, inc int64) error {
	if err := check(metric, &ts); err != nil {
		return err
	}
	tss := metric + ts.String()
	mlock.Lock()
	if counters[tss] == nil {
		counters[tss] = &addMetric{
			metric: metric,
			ts:     ts.Copy(),
		}
	}
	counters[tss].value += inc
	mlock.Unlock()
	return nil
}

type putMetric struct {
	metric string
	ts     opentsdb.TagSet
	value  interface{}
}

// Put is useful for capturing "events" that have a gauge value. Subsequent
// calls between the sending interval will overwrite previous calls.
func Put(metric string, ts opentsdb.TagSet, v interface{}) error {
	if err := check(metric, &ts); err != nil {
		return err
	}
	tss := metric + ts.String()
	mlock.Lock()
	puts[tss] = &putMetric{metric, ts.Copy(), v}
	mlock.Unlock()
	return nil
}

func check(metric string, ts *opentsdb.TagSet) error {
	if err := checkClean(metric, "metric"); err != nil {
		return err
	}
	for k, v := range *ts {
		if err := checkClean(k, "tagk"); err != nil {
			return err
		}
		if err := checkClean(v, "tagv"); err != nil {
			return err
		}
	}
	if osHostname == "" {
		if err := setHostName(); err != nil {
			return err
		}
	}
	if *ts == nil {
		*ts = make(opentsdb.TagSet)
	}
	if (*ts)["host"] == "" {
		(*ts)["host"] = osHostname
	}
	return nil
}

func checkClean(s, t string) error {
	if sc, err := opentsdb.Clean(s); s != sc || err != nil {
		if err != nil {
			return err
		}
		return fmt.Errorf("%s %s may only contain a to z, A to Z, 0 to 9, -, _, ., / or Unicode letters and may not be empty", t, s)
	}
	return nil
}

func collect() {
	for {
		mlock.Lock()
		now := time.Now().Unix()
		for _, c := range counters {
			dp := &opentsdb.DataPoint{
				Metric:    metricRoot + c.metric,
				Timestamp: now,
				Value:     c.value,
				Tags:      c.ts,
			}
			tchan <- dp
		}
		for _, s := range sets {
			dp := &opentsdb.DataPoint{
				Metric:    metricRoot + s.metric,
				Timestamp: now,
				Value:     s.f(),
				Tags:      s.ts,
			}
			tchan <- dp
		}
		for _, s := range puts {
			dp := &opentsdb.DataPoint{
				Metric:    metricRoot + s.metric,
				Timestamp: now,
				Value:     s.value,
				Tags:      s.ts,
			}
			tchan <- dp
		}
		puts = make(map[string]*putMetric)
		mlock.Unlock()
		time.Sleep(Freq)
	}
}
<commit_msg>cmd/scollector: Append localhost on queue<commit_after>// Package collect provides functions for sending data to OpenTSDB.
//
// The "collect" namespace is used (i.e., <metric_root>.collect) to collect
// program and queue metrics.
package collect

import (
	"fmt"
	"net/http"
	"net/url"
	"os"
	"runtime"
	"strings"
	"sync"
	"time"

	"github.com/StackExchange/scollector/opentsdb"
)

var (
	// Freq is how often metrics are sent to OpenTSDB.
	Freq = time.Second * 15

	// MaxQueueLen is the maximum size of the queue, above which incoming data will
	// be discarded. Defaults to about 150MB.
	MaxQueueLen = 200000

	// BatchSize is the maximum length of data points sent at once to OpenTSDB.
	BatchSize = 50

	// Debug enables debug logging.
	Debug = false

	// Dropped is the number of dropped data points due to a full queue.
	dropped int64

	// Sent is the number of sent data points.
	sent int64

	tchan               chan *opentsdb.DataPoint
	tsdbURL             string
	osHostname          string
	metricRoot          string
	queue               opentsdb.MultiDataPoint
	qlock, mlock, slock sync.Mutex   // Locks for queues, maps, stats.
	counters                         = make(map[string]*addMetric)
	sets                             = make(map[string]*setMetric)
	puts                             = make(map[string]*putMetric)
	client              *http.Client = &http.Client{
		Transport: &timeoutTransport{Transport: new(http.Transport)},
		Timeout:   time.Minute,
	}
)

type timeoutTransport struct {
	*http.Transport
	Timeout time.Time
}

func (t *timeoutTransport) RoundTrip(r *http.Request) (*http.Response, error) {
	if time.Now().After(t.Timeout) {
		t.Transport.CloseIdleConnections()
		t.Timeout = time.Now().Add(time.Minute * 5)
	}
	return t.Transport.RoundTrip(r)
}

// InitChan is similar to Init, but uses the given channel instead of creating a
// new one.
func InitChan(tsdbhost, metric_root string, ch chan *opentsdb.DataPoint) error {
	if tchan != nil {
		return fmt.Errorf("cannot init twice")
	}
	if err := checkClean(metric_root, "metric root"); err != nil {
		return err
	}
	if tsdbhost == "" {
		return fmt.Errorf("must specify non-empty tsdb host")
	}
	u := url.URL{
		Scheme: "http",
		Path:   "/api/put",
	}
	if strings.HasPrefix(tsdbhost, ":") {
		tsdbhost = "localhost" + tsdbhost
	} else if !strings.Contains(tsdbhost, ":") {
		tsdbhost += ":4242"
	}
	u.Host = tsdbhost
	tsdbURL = u.String()
	metricRoot = metric_root + "."
	tchan = ch
	go func() {
		for dp := range tchan {
			qlock.Lock()
			for {
				if len(queue) > MaxQueueLen {
					slock.Lock()
					dropped++
					slock.Unlock()
					break
				}
				queue = append(queue, dp)
				select {
				case dp = <-tchan:
					continue
				default:
				}
				break
			}
			qlock.Unlock()
		}
	}()
	go send()

	go collect()
	Set("collect.dropped", nil, func() (i interface{}) {
		slock.Lock()
		i = dropped
		slock.Unlock()
		return
	})
	Set("collect.sent", nil, func() (i interface{}) {
		slock.Lock()
		i = sent
		slock.Unlock()
		return
	})
	Set("collect.alloc", nil, func() interface{} {
		var ms runtime.MemStats
		runtime.ReadMemStats(&ms)
		return ms.Alloc
	})
	Set("collect.goroutines", nil, func() interface{} {
		return runtime.NumGoroutine()
	})
	return nil
}

// Init sets up the channels and the queue for sending data to OpenTSDB. It also
// sets up the basename for all metrics.
func Init(tsdbhost, metric_root string) error {
	return InitChan(tsdbhost, metric_root, make(chan *opentsdb.DataPoint))
}

func setHostName() error {
	h, err := os.Hostname()
	if err != nil {
		return err
	}
	osHostname = strings.ToLower(strings.SplitN(h, ".", 2)[0])
	if err := checkClean(osHostname, "host tag"); err != nil {
		return err
	}
	return nil
}

type setMetric struct {
	metric string
	ts     opentsdb.TagSet
	f      func() interface{}
}

func Set(metric string, ts opentsdb.TagSet, f func() interface{}) error {
	if err := check(metric, &ts); err != nil {
		return err
	}
	tss := metric + ts.String()
	mlock.Lock()
	sets[tss] = &setMetric{metric, ts.Copy(), f}
	mlock.Unlock()
	return nil
}

type addMetric struct {
	metric string
	ts     opentsdb.TagSet
	value  int64
}

// Add takes a metric and increments a counter for that metric. The metric name
// is appended to the basename specified in the Init function.
func Add(metric string, ts opentsdb.TagSet, inc int64) error {
	if err := check(metric, &ts); err != nil {
		return err
	}
	tss := metric + ts.String()
	mlock.Lock()
	if counters[tss] == nil {
		counters[tss] = &addMetric{
			metric: metric,
			ts:     ts.Copy(),
		}
	}
	counters[tss].value += inc
	mlock.Unlock()
	return nil
}

type putMetric struct {
	metric string
	ts     opentsdb.TagSet
	value  interface{}
}

// Put is useful for capturing "events" that have a gauge value. Subsequent
// calls between the sending interval will overwrite previous calls.
func Put(metric string, ts opentsdb.TagSet, v interface{}) error {
	if err := check(metric, &ts); err != nil {
		return err
	}
	tss := metric + ts.String()
	mlock.Lock()
	puts[tss] = &putMetric{metric, ts.Copy(), v}
	mlock.Unlock()
	return nil
}

func check(metric string, ts *opentsdb.TagSet) error {
	if err := checkClean(metric, "metric"); err != nil {
		return err
	}
	for k, v := range *ts {
		if err := checkClean(k, "tagk"); err != nil {
			return err
		}
		if err := checkClean(v, "tagv"); err != nil {
			return err
		}
	}
	if osHostname == "" {
		if err := setHostName(); err != nil {
			return err
		}
	}
	if *ts == nil {
		*ts = make(opentsdb.TagSet)
	}
	if (*ts)["host"] == "" {
		(*ts)["host"] = osHostname
	}
	return nil
}

func checkClean(s, t string) error {
	if sc, err := opentsdb.Clean(s); s != sc || err != nil {
		if err != nil {
			return err
		}
		return fmt.Errorf("%s %s may only contain a to z, A to Z, 0 to 9, -, _, ., / or Unicode letters and may not be empty", t, s)
	}
	return nil
}

func collect() {
	for {
		mlock.Lock()
		now := time.Now().Unix()
		for _, c := range counters {
			dp := &opentsdb.DataPoint{
				Metric:    metricRoot + c.metric,
				Timestamp: now,
				Value:     c.value,
				Tags:      c.ts,
			}
			tchan <- dp
		}
		for _, s := range sets {
			dp := &opentsdb.DataPoint{
				Metric:    metricRoot + s.metric,
				Timestamp: now,
				Value:     s.f(),
				Tags:      s.ts,
			}
			tchan <- dp
		}
		for _, s := range puts {
			dp := &opentsdb.DataPoint{
				Metric:    metricRoot + s.metric,
				Timestamp: now,
				Value:     s.value,
				Tags:      s.ts,
			}
			tchan <- dp
		}
		puts = make(map[string]*putMetric)
		mlock.Unlock()
		time.Sleep(Freq)
	}
}
<|endoftext|>


<commit_before>package whisper

import (
	"bytes"
	"io/ioutil"
	"net/http"
	"testing"

	"github.com/status-im/status-go/e2e"
	"github.com/status-im/status-go/geth/node"
	"github.com/status-im/status-go/geth/params"
	"github.com/stretchr/testify/suite"
)

func TestMailServiceSuite(t *testing.T) {
	suite.Run(t, new(MailServiceSuite))
}

type MailServiceSuite struct {
	e2e.NodeManagerTestSuite
}

func (s *MailServiceSuite) SetupTest() {
	s.NodeManager = node.NewNodeManager()
}

// TestShhRequestMessagesRPCMethodAvailability tests if `shh_requestMessages` is available
// through inproc and HTTP interfaces.
func (s *MailServiceSuite) TestShhRequestMessagesRPCMethodAvailability() {
	r := s.Require()

	s.StartTestNode(func(config *params.NodeConfig) {
		config.RPCEnabled = true
	})
	defer s.StopTestNode()

	client := s.NodeManager.RPCClient()
	r.NotNil(client)

	// This error means that the method is available through inproc communication
	// as the validation of params occurred.
	err := client.Call(nil, "shh_requestMessages", map[string]interface{}{})
	r.EqualError(err, `invalid mailServerPeer value: invalid URL scheme, want "enode"`)

	// Do the same but using HTTP interface.
	req, err := http.NewRequest("POST", "http://localhost:8645", bytes.NewBuffer([]byte(`{
		"jsonrpc": "2.0",
		"id": 1,
		"method": "shh_requestMessages",
		"params": [{}]
	}`)))
	r.NoError(err)
	resp, err := http.DefaultClient.Do(req)
	r.NoError(err)
	defer resp.Body.Close()
	r.Equal(200, resp.StatusCode)
	data, err := ioutil.ReadAll(resp.Body)
	r.NoError(err)
	r.Contains(string(data), `invalid mailServerPeer value`)
}
<commit_msg>Add `Content-Type` to the whisper tests as required in geth 1.7.3. (#553)<commit_after>package whisper

import (
	"bytes"
	"io/ioutil"
	"net/http"
	"testing"

	"github.com/status-im/status-go/e2e"
	"github.com/status-im/status-go/geth/node"
	"github.com/status-im/status-go/geth/params"
	"github.com/stretchr/testify/suite"
)

func TestMailServiceSuite(t *testing.T) {
	suite.Run(t, new(MailServiceSuite))
}

type MailServiceSuite struct {
	e2e.NodeManagerTestSuite
}

func (s *MailServiceSuite) SetupTest() {
	s.NodeManager = node.NewNodeManager()
}

// TestShhRequestMessagesRPCMethodAvailability tests if `shh_requestMessages` is available
// through inproc and HTTP interfaces.
func (s *MailServiceSuite) TestShhRequestMessagesRPCMethodAvailability() {
	r := s.Require()

	s.StartTestNode(func(config *params.NodeConfig) {
		config.RPCEnabled = true
	})
	defer s.StopTestNode()

	client := s.NodeManager.RPCClient()
	r.NotNil(client)

	// This error means that the method is available through inproc communication
	// as the validation of params occurred.
	err := client.Call(nil, "shh_requestMessages", map[string]interface{}{})
	r.EqualError(err, `invalid mailServerPeer value: invalid URL scheme, want "enode"`)

	// Do the same but using HTTP interface.
	req, err := http.NewRequest("POST", "http://localhost:8645", bytes.NewBuffer([]byte(`{
		"jsonrpc": "2.0",
		"id": 1,
		"method": "shh_requestMessages",
		"params": [{}]
	}`)))
	req.Header.Set("Content-Type", "application/json")
	r.NoError(err)
	resp, err := http.DefaultClient.Do(req)
	r.NoError(err)
	defer resp.Body.Close()
	r.Equal(200, resp.StatusCode)
	data, err := ioutil.ReadAll(resp.Body)
	r.NoError(err)
	r.Contains(string(data), `invalid mailServerPeer value`)
}
<|endoftext|>


<commit_before>/*
 * Funkensturm
 * Miek Gieben <miek@miek.nl> (c) 2011
 * GPLv2
 */

package main

import (
	"os"
	"flag"
	"net"
	"fmt"
	"dns"
	"dns/resolver"
	"dns/responder"
	"os/signal"
)

// Setup a responder takes takes care of the incoming queries.
type server responder.Server

// Setup a initial resolver for sending the queries somewhere else.
var qr chan resolver.Msg

// The configuration of Funkensturm
var f *Funkensturm

// Where does the packet come from?
// IN: initial packet received by the Responder
// any modifications here will reflect what kind of
// pkt is sent through. Normally there is no modification here.
// OUT: pkt as received back from a server. Modifications here will reflect
// how the packet is send back to the original requester.
const (
	IN  = iota // set when receiving a packet
	OUT        // set when sending a packet

	OR  // chain match functions with logical or
	AND // chain match functions with logical and
)

// A Match function is used on a DNS packet and
// returns (a possibly modified) DNS packet. It should
// return true when the packets matches the criteria set in
// the function.
// Op is used in chaining Match-functions together
type Match struct {
	Op   int // boolean op: OR, AND
	Func func(*dns.Msg, int) (*dns.Msg, bool)
}

// An action is something that is done with a packet. Funkensturm
// does not impose any restriction on what this can be, except that
// is must remain a valid DNS packet.
type Action struct {
	Func func(*dns.Msg, bool) *dns.Msg
}

// A complete config for Funkensturm. All matches in the Matches slice are
// chained together: incoming dns.Msg -> Match[0] -> dns.Msg -> Match[1] -> dns.Msg -> ...
// The dns.Msg output of Match[n] is the input for Match[n+1].
//
// The final outcome (does a packet match or not?) is calculated as follows:
// true Match[0].Op Match[0].Func() Match[1].Op Match[1].Func() ...
// The result of this macthing is given to the action function(s). They can then
// decide what to do with a packet in the 'true' and in the 'false' case.
type Funkensturm struct {
	Setup   func() bool // Inital setup (for extra resolvers, or loading keys, or ...)
	Matches []Match     // Match- and modify functions
	Actions []Action    // What to do with the packets
}

// No matter what, we refuse to answer request with the response bit set.
func doFunkensturm(i []byte) ([]byte, os.Error) {
	pkt := new(dns.Msg)
	if !pkt.Unpack(i) {
		return nil, &dns.Error{Error: "Unpacking packet failed"}
	}
	if *verbose {
		fmt.Printf(">>>>>> ORIGINAL INCOMING\n")
		fmt.Printf("%v\n", pkt)
	}
	if pkt.MsgHdr.Response == true {
		return nil, &dns.Error{Error: "Response bit set, not replying"}
	}

	// Loop through the Match* functions and decide what to do
	// Note the packet can be changed by these function, this
	// change is cumulative.
	ok, ok1 := true, true
	pkt1 := pkt
	for _, m := range f.Matches {
		pkt1, ok1 = m.Func(pkt1, IN)
		switch m.Op {
		case AND:
			ok = ok && ok1
		case OR:
			ok = ok || ok1
		}
	}
	if *verbose {
		fmt.Printf(">>>>>> MODIFIED INCOMING\n")
		fmt.Printf("%v\n", pkt1)
	}

	// Loop through the Actions.Func* and do something with the
	// packet. Note there can only be one returned packet.
	// We use 'ok' to signal what the above match did: true or false
	var resultpkt *dns.Msg
	for _, a := range f.Actions {
		resultpkt = a.Func(pkt1, ok)
	}

	if resultpkt == nil {
		return nil, &dns.Error{Error: "Action function returned nil packet"}
	}

	if *verbose {
		fmt.Printf(">>>>>> ORIGINAL OUTGOING\n")
		fmt.Printf("%v\n", resultpkt)
	}

	// loop again for matching, but now with OUT, this is done
	// for some last minute packet changing. Note the boolean return
	// code isn't used any more, i.e No more actions are allowed
	// anymore
	pkt1 = resultpkt
	for _, m := range f.Matches {
		pkt1, _ = m.Func(pkt1, OUT)
	}

	if *verbose {
		fmt.Printf(">>>>>> MODIFIED OUTGOING\n")
		fmt.Printf("%v\n", pkt1)
	}

	out, ok1 := pkt1.Pack()
	if !ok1 {
		return nil, &dns.Error{Error: "Packing packet failed"}
	}
	// Some final byte changing function here?
	return out, nil

}

func (s *server) ResponderUDP(c *net.UDPConn, a net.Addr, i []byte) {
	out, err := doFunkensturm(i)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error: %s\n", err.String())
		return
	}

	if out != nil {
		responder.SendUDP(out, c, a)
	}
	// nothing is send back
}

func (s *server) ResponderTCP(c *net.TCPConn, i []byte) {
	out, err := doFunkensturm(i)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error: %s\n", err.String())
		return
	}

	if out != nil {
		responder.SendTCP(out, c)
	}
	// nothing is send back
}

var verbose *bool

func main() {
	var sserver *string = flag.String("sserver", "127.0.0.1", "Set the listener address")
	var sport *string = flag.String("sport", "8053", "Set the listener port")
	var rserver *string = flag.String("rserver", "127.0.0.1", "Remote server address")
	// multiple rservers??
	var rport *string = flag.String("rpost", "53", "Remote server port to forward queries to")
	verbose = flag.Bool("verbose", false, "Print packet as the flow through")       // needs to be global
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: %s\n", os.Args[0])
		flag.PrintDefaults()
	}
	flag.Parse()

	f = funkensturm()
	ok := f.Setup()
	if !ok {
		fmt.Fprintf(os.Stderr, "Setup failed")
		return
	}
	// The resolver
	r := new(resolver.Resolver)
	r.Servers = []string{*rserver}
	r.Port = *rport
	qr = r.NewQuerier() // connect to global qr

	// The responder
	s := new(responder.Server)
	s.Address = *sserver
	s.Port = *sport
	var srv *server
	rs := make(chan bool)
	go s.NewResponder(srv, rs)

forever:
	for {
		// Wait for a signal to stop
		select {
		case <-signal.Incoming:
			println("Signal received, stopping")
			break forever
		}
	}
	rs <- true // shutdown responder and resolver
	qr <- resolver.Msg{}
	<-rs // wait for confirmation
	<-qr
}
<commit_msg>nicer verbose output<commit_after>/*
 * Funkensturm
 * Miek Gieben <miek@miek.nl> (c) 2011
 * GPLv2
 */

package main

import (
	"os"
	"flag"
	"net"
	"fmt"
	"dns"
	"dns/resolver"
	"dns/responder"
	"os/signal"
)

// Setup a responder takes takes care of the incoming queries.
type server responder.Server

// Setup a initial resolver for sending the queries somewhere else.
var qr chan resolver.Msg

// The configuration of Funkensturm
var f *Funkensturm

// Where does the packet come from?
// IN: initial packet received by the Responder
// any modifications here will reflect what kind of
// pkt is sent through. Normally there is no modification here.
// OUT: pkt as received back from a server. Modifications here will reflect
// how the packet is send back to the original requester.
const (
	IN  = iota // set when receiving a packet
	OUT        // set when sending a packet

	OR  // chain match functions with logical or
	AND // chain match functions with logical and
)

// A Match function is used on a DNS packet and
// returns (a possibly modified) DNS packet. It should
// return true when the packets matches the criteria set in
// the function.
// Op is used in chaining Match-functions together
type Match struct {
	Op   int // boolean op: OR, AND
	Func func(*dns.Msg, int) (*dns.Msg, bool)
}

// An action is something that is done with a packet. Funkensturm
// does not impose any restriction on what this can be, except that
// is must remain a valid DNS packet.
type Action struct {
	Func func(*dns.Msg, bool) *dns.Msg
}

// A complete config for Funkensturm. All matches in the Matches slice are
// chained together: incoming dns.Msg -> Match[0] -> dns.Msg -> Match[1] -> dns.Msg -> ...
// The dns.Msg output of Match[n] is the input for Match[n+1].
//
// The final outcome (does a packet match or not?) is calculated as follows:
// true Match[0].Op Match[0].Func() Match[1].Op Match[1].Func() ...
// The result of this macthing is given to the action function(s). They can then
// decide what to do with a packet in the 'true' and in the 'false' case.
type Funkensturm struct {
	Setup   func() bool // Inital setup (for extra resolvers, or loading keys, or ...)
	Matches []Match     // Match- and modify functions
	Actions []Action    // What to do with the packets
}

// No matter what, we refuse to answer request with the response bit set.
func doFunkensturm(i []byte) ([]byte, os.Error) {
	pkt := new(dns.Msg)
	if !pkt.Unpack(i) {
		return nil, &dns.Error{Error: "Unpacking packet failed"}
	}
	if *verbose {
		fmt.Printf(">>>>>> ORIGINAL INCOMING\n")
		fmt.Printf("%v", pkt)
		fmt.Printf("<<<<<< ORIGINAL INCOMING\n")
	}
	if pkt.MsgHdr.Response == true {
		return nil, &dns.Error{Error: "Response bit set, not replying"}
	}

	// Loop through the Match* functions and decide what to do
	// Note the packet can be changed by these function, this
	// change is cumulative.
	ok, ok1 := true, true
	pkt1 := pkt
	for _, m := range f.Matches {
		pkt1, ok1 = m.Func(pkt1, IN)
		switch m.Op {
		case AND:
			ok = ok && ok1
		case OR:
			ok = ok || ok1
		}
	}
	if *verbose {
		fmt.Printf(">>>>>> MODIFIED INCOMING\n")
		fmt.Printf("%v", pkt1)
		fmt.Printf("<<<<<< MODIFIED INCOMING\n")
	}

	// Loop through the Actions.Func* and do something with the
	// packet. Note there can only be one returned packet.
	// We use 'ok' to signal what the above match did: true or false
	var resultpkt *dns.Msg
	for _, a := range f.Actions {
		resultpkt = a.Func(pkt1, ok)
	}

	if resultpkt == nil {
		return nil, &dns.Error{Error: "Action function returned nil packet"}
	}

	if *verbose {
		fmt.Printf(">>>>>> ORIGINAL OUTGOING\n")
		fmt.Printf("%v", resultpkt)
		fmt.Printf("<<<<<< ORIGINAL OUTGOING\n")
	}

	// loop again for matching, but now with OUT, this is done
	// for some last minute packet changing. Note the boolean return
	// code isn't used any more, i.e No more actions are allowed
	// anymore
	pkt1 = resultpkt
	for _, m := range f.Matches {
		pkt1, _ = m.Func(pkt1, OUT)
	}

	if *verbose {
		fmt.Printf(">>>>>> MODIFIED OUTGOING\n")
		fmt.Printf("%v", pkt1)
		fmt.Printf("<<<<<< MODIFIED OUTGOING\n")
	}

	out, ok1 := pkt1.Pack()
	if !ok1 {
		return nil, &dns.Error{Error: "Packing packet failed"}
	}
	// Some final byte changing function here?
	return out, nil

}

func (s *server) ResponderUDP(c *net.UDPConn, a net.Addr, i []byte) {
	out, err := doFunkensturm(i)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error: %s\n", err.String())
		return
	}

	if out != nil {
		responder.SendUDP(out, c, a)
	}
	// nothing is send back
}

func (s *server) ResponderTCP(c *net.TCPConn, i []byte) {
	out, err := doFunkensturm(i)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error: %s\n", err.String())
		return
	}

	if out != nil {
		responder.SendTCP(out, c)
	}
	// nothing is send back
}

var verbose *bool

func main() {
	var sserver *string = flag.String("sserver", "127.0.0.1", "Set the listener address")
	var sport *string = flag.String("sport", "8053", "Set the listener port")
	var rserver *string = flag.String("rserver", "127.0.0.1", "Remote server address")
	// multiple rservers??
	var rport *string = flag.String("rpost", "53", "Remote server port to forward queries to")
	verbose = flag.Bool("verbose", false, "Print packet as the flow through")       // needs to be global
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: %s\n", os.Args[0])
		flag.PrintDefaults()
	}
	flag.Parse()

	f = funkensturm()
	ok := f.Setup()
	if !ok {
		fmt.Fprintf(os.Stderr, "Setup failed")
		return
	}
	// The resolver
	r := new(resolver.Resolver)
	r.Servers = []string{*rserver}
	r.Port = *rport
	qr = r.NewQuerier() // connect to global qr

	// The responder
	s := new(responder.Server)
	s.Address = *sserver
	s.Port = *sport
	var srv *server
	rs := make(chan bool)
	go s.NewResponder(srv, rs)

forever:
	for {
		// Wait for a signal to stop
		select {
		case <-signal.Incoming:
			println("Signal received, stopping")
			break forever
		}
	}
	rs <- true // shutdown responder and resolver
	qr <- resolver.Msg{}
	<-rs // wait for confirmation
	<-qr
}
<|endoftext|>


<commit_before>package groups_test

import (
	"fmt"
	"testing"

	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"

	"github.com/hashicorp/terraform-provider-azuread/internal/acceptance"
	"github.com/hashicorp/terraform-provider-azuread/internal/acceptance/check"
)

type GroupsDataSource struct{}

func TestAccGroupsDataSource_byDisplayNames(t *testing.T) {
	data := acceptance.BuildTestData(t, "data.azuread_groups", "test")
	r := GroupsDataSource{}

	data.DataSourceTest(t, []resource.TestStep{
		{
			Config: r.byDisplayNames(data),
			Check: resource.ComposeTestCheckFunc(
				check.That(data.ResourceName).Key("display_names.#").HasValue("2"),
				check.That(data.ResourceName).Key("object_ids.#").HasValue("2"),
			),
		},
	})
}

func TestAccGroupsDataSource_byObjectIds(t *testing.T) {
	data := acceptance.BuildTestData(t, "data.azuread_groups", "test")
	r := GroupsDataSource{}

	data.DataSourceTest(t, []resource.TestStep{
		{
			Config: r.byObjectIds(data),
			Check: resource.ComposeTestCheckFunc(
				check.That(data.ResourceName).Key("display_names.#").HasValue("2"),
				check.That(data.ResourceName).Key("object_ids.#").HasValue("2"),
			),
		},
	})
}

func TestAccGroupsDataSource_noNames(t *testing.T) {
	data := acceptance.BuildTestData(t, "data.azuread_groups", "test")

	data.DataSourceTest(t, []resource.TestStep{
		{
			Config: GroupsDataSource{}.noNames(),
			Check: resource.ComposeTestCheckFunc(
				check.That(data.ResourceName).Key("display_names.#").HasValue("0"),
				check.That(data.ResourceName).Key("object_ids.#").HasValue("0"),
			),
		},
	})
}

func (GroupsDataSource) template(data acceptance.TestData) string {
	return fmt.Sprintf(`
resource "azuread_group" "testA" {
  display_name     = "acctestGroupA-%[1]d"
  security_enabled = true
}

resource "azuread_group" "testB" {
  display_name = "acctestGroupB-%[1]d"
  mail_enabled = true
  types        = ["Unified"]
}
`, data.RandomInteger)
}

func (r GroupsDataSource) byDisplayNames(data acceptance.TestData) string {
	return fmt.Sprintf(`
%[1]s

data "azuread_groups" "test" {
  display_names = [azuread_group.testA.display_name, azuread_group.testB.display_name]
}
`, r.template(data))
}

func (r GroupsDataSource) byObjectIds(data acceptance.TestData) string {
	return fmt.Sprintf(`
%[1]s

data "azuread_groups" "test" {
  object_ids = [azuread_group.testA.object_id, azuread_group.testB.object_id]
}
`, r.template(data))
}

func (GroupsDataSource) noNames() string {
	return `
data "azuread_groups" "test" {
  display_names = []
}
`
}
<commit_msg>Fix test config for groups data source<commit_after>package groups_test

import (
	"fmt"
	"testing"

	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"

	"github.com/hashicorp/terraform-provider-azuread/internal/acceptance"
	"github.com/hashicorp/terraform-provider-azuread/internal/acceptance/check"
)

type GroupsDataSource struct{}

func TestAccGroupsDataSource_byDisplayNames(t *testing.T) {
	data := acceptance.BuildTestData(t, "data.azuread_groups", "test")
	r := GroupsDataSource{}

	data.DataSourceTest(t, []resource.TestStep{
		{
			Config: r.byDisplayNames(data),
			Check: resource.ComposeTestCheckFunc(
				check.That(data.ResourceName).Key("display_names.#").HasValue("2"),
				check.That(data.ResourceName).Key("object_ids.#").HasValue("2"),
			),
		},
	})
}

func TestAccGroupsDataSource_byObjectIds(t *testing.T) {
	data := acceptance.BuildTestData(t, "data.azuread_groups", "test")
	r := GroupsDataSource{}

	data.DataSourceTest(t, []resource.TestStep{
		{
			Config: r.byObjectIds(data),
			Check: resource.ComposeTestCheckFunc(
				check.That(data.ResourceName).Key("display_names.#").HasValue("2"),
				check.That(data.ResourceName).Key("object_ids.#").HasValue("2"),
			),
		},
	})
}

func TestAccGroupsDataSource_noNames(t *testing.T) {
	data := acceptance.BuildTestData(t, "data.azuread_groups", "test")

	data.DataSourceTest(t, []resource.TestStep{
		{
			Config: GroupsDataSource{}.noNames(),
			Check: resource.ComposeTestCheckFunc(
				check.That(data.ResourceName).Key("display_names.#").HasValue("0"),
				check.That(data.ResourceName).Key("object_ids.#").HasValue("0"),
			),
		},
	})
}

func (GroupsDataSource) template(data acceptance.TestData) string {
	return fmt.Sprintf(`
resource "azuread_group" "testA" {
  display_name     = "acctestGroupA-%[1]d"
  security_enabled = true
}

resource "azuread_group" "testB" {
  display_name  = "acctestGroupB-%[1]d"
  mail_enabled  = true
  mail_nickname = "acctestGroupB-%[1]d"
  types         = ["Unified"]
}
`, data.RandomInteger)
}

func (r GroupsDataSource) byDisplayNames(data acceptance.TestData) string {
	return fmt.Sprintf(`
%[1]s

data "azuread_groups" "test" {
  display_names = [azuread_group.testA.display_name, azuread_group.testB.display_name]
}
`, r.template(data))
}

func (r GroupsDataSource) byObjectIds(data acceptance.TestData) string {
	return fmt.Sprintf(`
%[1]s

data "azuread_groups" "test" {
  object_ids = [azuread_group.testA.object_id, azuread_group.testB.object_id]
}
`, r.template(data))
}

func (GroupsDataSource) noNames() string {
	return `
data "azuread_groups" "test" {
  display_names = []
}
`
}
<|endoftext|>


<commit_before>package isitcandle

import (
	"fmt"
	"time"

	"github.com/igungor/ilberbot"
)

func init() {
	ilberbot.RegisterCommand("/bugunkandilmi", isitcandle)
}

const timeformat = "2 Jan 2006"

var dasCandles = map[string]string{
	"2 Jan 2015":  "Mevlid Kandili",
	"23 Apr 2015": "Regaib Kandili",
	"15 May 2015": "Mirac Kandili",
	"1 Jun 2015":  "Berat Kandili",
	"13 Jul 2015": "Kadir Gecesi",
	"22 Dec 2015": "Mevlid Kandili",

	"7 Apr 2016":  "Regaib Kandili",
	"3 Apr 2016":  "Mirac Kandili",
	"21 May 2016": "Berat Kandili",
	"1 Jul 2016":  "Kadir Gecesi",
	"11 Dec 2016": "Mevlid Kandili",
}

// i know it's a lame name but funny nonetheless
func isitcandle(args ...string) string {
	now := time.Now().UTC().Format(timeformat)

	v, ok := dasCandles[now]
	if !ok {
		return "bugun kandil degilmis"
	}

	return fmt.Sprintf("Evet, bugun %v\n", v)

}
<commit_msg>just say no to the "is it candle or what?" question<commit_after>package isitcandle

import (
	"fmt"
	"time"

	"github.com/igungor/ilberbot"
)

func init() {
	ilberbot.RegisterCommand("/bugunkandilmi", isitcandle)
}

const timeformat = "2 Jan 2006"

var dasCandles = map[string]string{
	"2 Jan 2015":  "Mevlid Kandili",
	"23 Apr 2015": "Regaib Kandili",
	"15 May 2015": "Mirac Kandili",
	"1 Jun 2015":  "Berat Kandili",
	"13 Jul 2015": "Kadir Gecesi",
	"22 Dec 2015": "Mevlid Kandili",

	"7 Apr 2016":  "Regaib Kandili",
	"3 Apr 2016":  "Mirac Kandili",
	"21 May 2016": "Berat Kandili",
	"1 Jul 2016":  "Kadir Gecesi",
	"11 Dec 2016": "Mevlid Kandili",
}

// i know it's a lame name but funny nonetheless
func isitcandle(args ...string) string {
	now := time.Now().UTC().Format(timeformat)

	v, ok := dasCandles[now]
	if !ok {
		return "hayır"
	}

	return fmt.Sprintf("Evet, bugun %v\n", v)

}
<|endoftext|>


<commit_before>// call somatic variants with multiple tumor samples associated with 1 normal. This uses
// the genotype likelihood approach from bcbio and speedseq but allows for multiple tumor
// samples. Anything that meets the criteria will have a "PASS" or "." FILTER and a list
// of tumor samples with the somatic variant in INFO["SOMATIC"]
// Copied from bcbio and speedseq.
package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"strings"

	"github.com/brentp/vcfgo"
	"github.com/brentp/xopen"
)

func getTumorLOD(gls []float64) float64 {
	lod := -1.0
	if len(gls) < 3 {
		return lod
	}
	for _, gl := range gls[1:] {
		t := gl - gls[0]
		if t > lod {
			lod = t
		}
	}
	return lod
}

func getNormalLOD(gls []float64, thresh float64) float64 {
	lod := 1e7
	if len(gls) < 3 {
		return thresh
	}
	for _, gl := range gls[1:] {
		t := gls[0] - gl
		if t < lod {
			lod = t
		}
	}
	return lod
}

func somaticLOD(normalGLs []float64, tumorGLs []float64, thresh float64) bool {
	normalLOD := getNormalLOD(normalGLs, thresh)
	tumorLOD := getTumorLOD(tumorGLs)
	return tumorLOD >= thresh && normalLOD >= thresh
}

func getFreq(refCount int, altCounts []int) float64 {
	ac := 0
	for _, a := range altCounts {
		ac += a
	}
	return float64(ac) / float64(refCount+ac)
}

func somaticFreqs(normal *vcfgo.SampleGenotype, tumor *vcfgo.SampleGenotype, freqRatio float64) bool {
	var nFreq, tFreq float64
	nrd, err := normal.RefDepth()
	if err != nil {
		nFreq = 0.0
	} else {
		nads, err := normal.AltDepths()
		if err != nil {
			nFreq = 0.0
		} else {
			nFreq = getFreq(nrd, nads)
		}
	}
	trd, err := tumor.RefDepth()
	if err != nil {
		tFreq = 0.0
	} else {
		tads, err := tumor.AltDepths()
		if err != nil {
			tFreq = 0.0
		} else {
			tFreq = getFreq(trd, tads)
		}
	}
	return nFreq <= 0.001 || nFreq <= tFreq/freqRatio

}

func Somatics(v *vcfgo.Variant, normalIdx int, thresh float64, freqRatio float64, skipMissing bool) []string {

	normal := v.Samples[normalIdx]
	somatics := make([]string, 0)
	for i, tumor := range v.Samples {
		if i == normalIdx {
			continue
		}
		// if we don't have GLs for normal, dont call a somatic.
		if skipMissing && len(normal.GL) == 0 {
			return somatics
		}
		if !somaticFreqs(normal, tumor, freqRatio) {
			continue
		}
		if somaticLOD(normal.GL, tumor.GL, thresh) {
			somatics = append(somatics, v.Header.SampleNames[i])
		}
	}
	return somatics
}

func check(e error) {
	if e != nil {
		log.Fatal(e)
	}
}

func sampleIndex(samples []string, sample string) int {
	for p, s := range samples {
		if s == sample {
			return p
		}
	}
	return -1
}

func main() {
	normalStr := flag.String("normal", "", "sample name of the normal sample. Default is the first sample")
	threshold := flag.Float64("threshold", 3.5, "threshold for difference in GLs to call a somatic variant")
	freqRatio := flag.Float64("freq-ratio", 2.7, "frequency in the tumor must be at least this many times the frequency in the normal")
	onlySomatic := flag.Bool("only-somatic", false, "print only the PASSing somatic variants (default is to set a flag and print all variants")
	skipMissing := flag.Bool("skip-missing-normals", false, "skip variants with missing normal. default is not to flag these as somatic.")
	flag.Parse()
	vcfs := flag.Args()
	if len(vcfs) != 1 {
		fmt.Printf("-----------------------------------------------------------------------------------\n")
		fmt.Printf("---------------- call somatic variants present in any tumor sample ----------------\n")
		fmt.Printf("----- uses the method from bcbio and speedseq but for multiple tumor samples ------\n")
		fmt.Printf("-----------------------------------------------------------------------------------\n")
		fmt.Printf("%s normal-and-tumors.vcf.gz\n", os.Args[0])
		flag.PrintDefaults()
		os.Exit(1)
	}

	fhr, err := xopen.Ropen(vcfs[0])
	check(err)

	rdr, err := vcfgo.NewReader(fhr, false)
	check(err)

	normal := 0
	if *normalStr != "" {
		normal = sampleIndex(rdr.Header.SampleNames, *normalStr)
		if normal == -1 {
			log.Fatalf("sample %s: not found in vcf (%v)\n", *normalStr, rdr.Header.SampleNames)
		}
	}

	fhw, err := xopen.Wopen("-")
	check(err)

	hdr := rdr.Header
	hdr.Filters["NOT_SOMATIC"] = "not somatic between normal and any tumor"

	hdr.Infos["SOMATIC"] = &vcfgo.Info{Id: "SOMATIC", Number: "1", Type: "String", Description: "Tumor samples with somatic event"}

	wtr, err := vcfgo.NewWriter(fhw, hdr)
	check(err)

	log.Printf("using %s as the normal sample\n", hdr.SampleNames[normal])

	j := 0
	for v := rdr.Read(); v != nil; v = rdr.Read() {
		j++
		if *onlySomatic && !(v.Filter == "." || v.Filter == "PASS") {
			continue
		}
		somatics := Somatics(v, normal, *threshold, *freqRatio, *skipMissing)
		if len(somatics) > 0 {
			v.Info.Add("SOMATIC", strings.Join(somatics, "|"))
		} else {
			if *onlySomatic {
				continue
			}
			if v.Filter == "." || v.Filter == "PASS" {
				v.Filter = ""
			}
			if v.Filter != "" {
				v.Filter += ";"
			}
			v.Filter += "NOT_SOMATIC"
		}
		wtr.WriteVariant(v)
	}
	log.Println("VCF warnings:", rdr.Error())
	log.Printf("evaluated %d variants\n", j)
	fhw.Close()
}
<commit_msg>fix docs<commit_after>// call somatic variants with multiple tumor samples associated with 1 normal. This uses
// the genotype likelihood approach from bcbio and speedseq but allows for multiple tumor
// samples. Anything that meets the criteria will have a "PASS" or "." FILTER and a list
// of tumor samples with the somatic variant in INFO["SOMATIC"]
// Copied from bcbio and speedseq.
package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"strings"

	"github.com/brentp/vcfgo"
	"github.com/brentp/xopen"
)

func getTumorLOD(gls []float64) float64 {
	lod := -1.0
	if len(gls) < 3 {
		return lod
	}
	for _, gl := range gls[1:] {
		t := gl - gls[0]
		if t > lod {
			lod = t
		}
	}
	return lod
}

func getNormalLOD(gls []float64, thresh float64) float64 {
	lod := 1e7
	if len(gls) < 3 {
		return thresh
	}
	for _, gl := range gls[1:] {
		t := gls[0] - gl
		if t < lod {
			lod = t
		}
	}
	return lod
}

func somaticLOD(normalGLs []float64, tumorGLs []float64, thresh float64) bool {
	normalLOD := getNormalLOD(normalGLs, thresh)
	tumorLOD := getTumorLOD(tumorGLs)
	return tumorLOD >= thresh && normalLOD >= thresh
}

func getFreq(refCount int, altCounts []int) float64 {
	ac := 0
	for _, a := range altCounts {
		ac += a
	}
	return float64(ac) / float64(refCount+ac)
}

func somaticFreqs(normal *vcfgo.SampleGenotype, tumor *vcfgo.SampleGenotype, freqRatio float64) bool {
	var nFreq, tFreq float64
	nrd, err := normal.RefDepth()
	if err != nil {
		nFreq = 0.0
	} else {
		nads, err := normal.AltDepths()
		if err != nil {
			nFreq = 0.0
		} else {
			nFreq = getFreq(nrd, nads)
		}
	}
	trd, err := tumor.RefDepth()
	if err != nil {
		tFreq = 0.0
	} else {
		tads, err := tumor.AltDepths()
		if err != nil {
			tFreq = 0.0
		} else {
			tFreq = getFreq(trd, tads)
		}
	}
	return nFreq <= 0.001 || nFreq <= tFreq/freqRatio

}

func Somatics(v *vcfgo.Variant, normalIdx int, thresh float64, freqRatio float64, skipMissing bool) []string {

	normal := v.Samples[normalIdx]
	somatics := make([]string, 0)
	for i, tumor := range v.Samples {
		if i == normalIdx {
			continue
		}
		// if we don't have GLs for normal, dont call a somatic.
		if skipMissing && len(normal.GL) == 0 {
			return somatics
		}
		if !somaticFreqs(normal, tumor, freqRatio) {
			continue
		}
		if somaticLOD(normal.GL, tumor.GL, thresh) {
			somatics = append(somatics, v.Header.SampleNames[i])
		}
	}
	return somatics
}

func check(e error) {
	if e != nil {
		log.Fatal(e)
	}
}

func sampleIndex(samples []string, sample string) int {
	for p, s := range samples {
		if s == sample {
			return p
		}
	}
	return -1
}

func main() {
	normalStr := flag.String("normal", "", "sample name of the normal sample. Default is the first sample")
	threshold := flag.Float64("threshold", 3.5, "threshold for difference in GLs to call a somatic variant")
	freqRatio := flag.Float64("freq-ratio", 2.7, "frequency in the tumor must be at least this many times the frequency in the normal")
	onlySomatic := flag.Bool("only-somatic", false, "print only the PASSing somatic variants (default is to set a flag and print all variants")
	skipMissing := flag.Bool("skip-missing-normals", false, "skip variants with missing normal. default is flag these as somatic.")
	flag.Parse()
	vcfs := flag.Args()
	if len(vcfs) != 1 {
		fmt.Printf("-----------------------------------------------------------------------------------\n")
		fmt.Printf("---------------- call somatic variants present in any tumor sample ----------------\n")
		fmt.Printf("----- uses the method from bcbio and speedseq but for multiple tumor samples ------\n")
		fmt.Printf("-----------------------------------------------------------------------------------\n")
		fmt.Printf("%s normal-and-tumors.vcf.gz\n", os.Args[0])
		flag.PrintDefaults()
		os.Exit(1)
	}

	fhr, err := xopen.Ropen(vcfs[0])
	check(err)

	rdr, err := vcfgo.NewReader(fhr, false)
	check(err)

	normal := 0
	if *normalStr != "" {
		normal = sampleIndex(rdr.Header.SampleNames, *normalStr)
		if normal == -1 {
			log.Fatalf("sample %s: not found in vcf (%v)\n", *normalStr, rdr.Header.SampleNames)
		}
	}

	fhw, err := xopen.Wopen("-")
	check(err)

	hdr := rdr.Header
	hdr.Filters["NOT_SOMATIC"] = "not somatic between normal and any tumor"

	hdr.Infos["SOMATIC"] = &vcfgo.Info{Id: "SOMATIC", Number: "1", Type: "String", Description: "Tumor samples with somatic event"}

	wtr, err := vcfgo.NewWriter(fhw, hdr)
	check(err)

	log.Printf("using %s as the normal sample\n", hdr.SampleNames[normal])

	j := 0
	for v := rdr.Read(); v != nil; v = rdr.Read() {
		j++
		if *onlySomatic && !(v.Filter == "." || v.Filter == "PASS") {
			continue
		}
		somatics := Somatics(v, normal, *threshold, *freqRatio, *skipMissing)
		if len(somatics) > 0 {
			v.Info.Add("SOMATIC", strings.Join(somatics, "|"))
		} else {
			if *onlySomatic {
				continue
			}
			if v.Filter == "." || v.Filter == "PASS" {
				v.Filter = ""
			}
			if v.Filter != "" {
				v.Filter += ";"
			}
			v.Filter += "NOT_SOMATIC"
		}
		wtr.WriteVariant(v)
	}
	log.Println("VCF warnings:", rdr.Error())
	log.Printf("evaluated %d variants\n", j)
	fhw.Close()
}
<|endoftext|>


<commit_before>package source

import (
	"errors"
	"io"
	"os"
	"path"
	"path/filepath"
)

type Input interface {
	Files() []*File
}

type File struct {
	name        string
	LogicalName string
	Contents    io.Reader
	Section     string
	Dir         string
}

type Filesystem struct {
	files      []*File
	Base       string
	AvoidPaths []string
}

func (f *Filesystem) Files() []*File {
	f.captureFiles()
	return f.files
}

var errMissingBaseDir = errors.New("source: missing base directory")

func (f *Filesystem) add(name string, reader io.Reader) (err error) {

	if name, err = f.getRelativePath(name); err != nil {
		return err
	}

	dir, logical := path.Split(name)
	_, section := path.Split(path.Dir(name))
	if section == "." {
		section = ""
	}

	f.files = append(f.files, &File{
		name:        name,
		LogicalName: logical,
		Contents:    reader,
		Section:     section,
		Dir:         dir,
	})
	return
}

func (f *Filesystem) getRelativePath(name string) (final string, err error) {
	if filepath.IsAbs(name) && f.Base == "" {
		return "", errMissingBaseDir
	}
	name = filepath.Clean(name)
	base := filepath.Clean(f.Base)

	name, err = filepath.Rel(base, name)
	if err != nil {
		return "", err
	}
	name = filepath.ToSlash(name)
	return name, nil
}

func (f *Filesystem) captureFiles() {

	walker := func(filePath string, fi os.FileInfo, err error) error {
		if err != nil {
			return nil
		}

		if fi.IsDir() {
			if f.avoid(filePath) {
				return filepath.SkipDir
			}
			return nil
		} else {
			if ignoreDotFile(filePath) {
				return nil
			}
			file, err := os.Open(filePath)
			if err != nil {
				return err
			}
			f.add(filePath, file)
			return nil
		}
	}

	filepath.Walk(f.Base, walker)
}

func (f *Filesystem) avoid(filePath string) bool {
	for _, avoid := range f.AvoidPaths {
		if avoid == filePath {
			return true
		}
	}
	return false
}

func ignoreDotFile(filePath string) bool {
	return filepath.Base(filePath)[0] == '.'
}
<commit_msg>Update filesystem.go<commit_after>package source

import (
	"errors"
	"io"
	"os"
	"path"
	"path/filepath"
)

type Input interface {
	Files() []*File
}

type File struct {
	name        string
	LogicalName string
	Contents    io.Reader
	Section     string
	Dir         string
}

type Filesystem struct {
	files      []*File
	Base       string
	AvoidPaths []string
}

func (f *Filesystem) Files() []*File {
	if len(f.files)<1 {f.captureFiles()}
	return f.files
}

var errMissingBaseDir = errors.New("source: missing base directory")

func (f *Filesystem) add(name string, reader io.Reader) (err error) {

	if name, err = f.getRelativePath(name); err != nil {
		return err
	}

	dir, logical := path.Split(name)
	_, section := path.Split(path.Dir(name))
	if section == "." {
		section = ""
	}

	f.files = append(f.files, &File{
		name:        name,
		LogicalName: logical,
		Contents:    reader,
		Section:     section,
		Dir:         dir,
	})
	return
}

func (f *Filesystem) getRelativePath(name string) (final string, err error) {
	if filepath.IsAbs(name) && f.Base == "" {
		return "", errMissingBaseDir
	}
	name = filepath.Clean(name)
	base := filepath.Clean(f.Base)

	name, err = filepath.Rel(base, name)
	if err != nil {
		return "", err
	}
	name = filepath.ToSlash(name)
	return name, nil
}

func (f *Filesystem) captureFiles() {

	walker := func(filePath string, fi os.FileInfo, err error) error {
		if err != nil {
			return nil
		}

		if fi.IsDir() {
			if f.avoid(filePath) {
				return filepath.SkipDir
			}
			return nil
		} else {
			if ignoreDotFile(filePath) {
				return nil
			}
			file, err := os.Open(filePath)
			if err != nil {
				return err
			}
			f.add(filePath, file)
			return nil
		}
	}

	filepath.Walk(f.Base, walker)
}

func (f *Filesystem) avoid(filePath string) bool {
	for _, avoid := range f.AvoidPaths {
		if avoid == filePath {
			return true
		}
	}
	return false
}

func ignoreDotFile(filePath string) bool {
	return filepath.Base(filePath)[0] == '.'
}
<|endoftext|>