<commit_before>package main

import (
	"flag"
	"log"
	"os"
	"os/signal"
	"time"

	"github.com/stathat/go"
)

var StaticRoot *string

func postStat(name string, value float64) {
	if err := stathat.PostEZValue(name, "eikeon@eikeon.com", value); err != nil {
		log.Printf("error posting value %v: %d", err, value)
	}
}
func main() {
	log.Println("starting marvin")

	Address := flag.String("address", ":9999", "http service address")
	StaticRoot = flag.String("root", "static", "...")
	config := flag.String("config", "/etc/marvin.json", "file path to configuration file")
	verbose := flag.Bool("verbose", false, "verbose output")
	flag.Parse()

	if err, s := NewSchedulerFromJSONPath(*config); err == nil {
		go ListenAndServe(*Address, s)
		go listen(s)

		if flag.NArg() == 0 {
			s.Hue.Do("chime") // visual display of scheduler starting
			go s.run()
		} else {
			transition := flag.Arg(0)
			go s.Hue.Do(transition)
		}
	} else {
		log.Fatal(err)
	}

	t, err := NewTSL2561(1, ADDRESS_FLOAT)
	if err != nil {
		log.Println("WARNING: could not create TSL2561:", err)
	}

	notifyChannel := make(chan os.Signal, 1)
	signal.Notify(notifyChannel, os.Interrupt)
	ticker := time.NewTicker(1 * time.Second)
	for {
		select {
		case <-ticker.C:
			if t != nil {
				if err = t.On(); err != nil {
					log.Fatal("could not turn on:", err)
				}
				time.Sleep(t.IntegrationDuration())

				if value, err := t.GetBroadband(); err == nil {
					if *verbose {
						log.Println("broadband:", value)
					}
					go postStat("light broadband", float64(value))
				} else {
					log.Println("error getting broadband value:", err)
				}
				if value, err := t.GetInfrared(); err == nil {
					if *verbose {
						log.Println("infrared:", value)
					}
					go postStat("light infrared", float64(value))
				} else {
					log.Println("error getting infrared value:", err)
				}

				if err := t.Off(); err != nil {
					log.Fatal("Could not turn off:", err)
				}
			}
		case sig := <-notifyChannel:
			switch sig {
			case os.Interrupt:
				log.Println("handling:", sig)
				goto Done
			default:
				log.Fatal("Unexpected Signal:", sig)
			}
		}
	}
Done:
	log.Println("stopping Marvin")

}
<commit_msg>Added rough first pass at ambient light interaction.<commit_after>package main

import (
	"flag"
	"log"
	"os"
	"os/signal"
	"time"

	"github.com/stathat/go"
)

var StaticRoot *string

func postStat(name string, value float64) {
	if err := stathat.PostEZValue(name, "eikeon@eikeon.com", value); err != nil {
		log.Printf("error posting value %v: %d", err, value)
	}
}
func main() {
	log.Println("starting marvin")

	Address := flag.String("address", ":9999", "http service address")
	StaticRoot = flag.String("root", "static", "...")
	config := flag.String("config", "/etc/marvin.json", "file path to configuration file")
	verbose := flag.Bool("verbose", false, "verbose output")
	flag.Parse()

	err, s := NewSchedulerFromJSONPath(*config)
	if err == nil {
		go ListenAndServe(*Address, s)
		go listen(s)

		if flag.NArg() == 0 {
			s.Hue.Do("chime") // visual display of scheduler starting
			go s.run()
		} else {
			transition := flag.Arg(0)
			go s.Hue.Do(transition)
		}
	} else {
		log.Fatal(err)
	}

	t, err := NewTSL2561(1, ADDRESS_FLOAT)
	if err != nil {
		log.Println("WARNING: could not create TSL2561:", err)
	}

	notifyChannel := make(chan os.Signal, 1)
	signal.Notify(notifyChannel, os.Interrupt)
	ticker := time.NewTicker(1 * time.Second)

	dayLight := make(chan bool, 10)
	var lastDayLight interface{}
	lastDayLightTime := time.Now()
	for {
		select {
		case value := <-dayLight:
			if value {
				s.Hue.Do("daylight")
				log.Println("daylight")
			} else {
				s.Hue.Do("daylight off")
				log.Println("daylight off")
			}
		case <-ticker.C:
			if t != nil {
				if err = t.On(); err != nil {
					log.Fatal("could not turn on:", err)
				}
				time.Sleep(t.IntegrationDuration())

				if value, err := t.GetBroadband(); err == nil {
					dl := value > 5000
					if lastDayLight == nil {
						lastDayLight = dl
						lastDayLightTime = time.Now()
					} else if time.Since(lastDayLightTime) > time.Duration(60*time.Second) {
						if value > 5000 && lastDayLight == false {
							lastDayLight = true
							lastDayLightTime = time.Now()
							dayLight <- true
						} else if value < 4900 && lastDayLight == true {
							lastDayLight = false
							lastDayLightTime = time.Now()
							dayLight <- false
						}
					}
					if *verbose {
						log.Println("broadband:", value)
					}
					go postStat("light broadband", float64(value))
				} else {
					log.Println("error getting broadband value:", err)
				}
				if value, err := t.GetInfrared(); err == nil {
					if *verbose {
						log.Println("infrared:", value)
					}
					go postStat("light infrared", float64(value))
				} else {
					log.Println("error getting infrared value:", err)
				}

				if err := t.Off(); err != nil {
					log.Fatal("Could not turn off:", err)
				}
			}
		case sig := <-notifyChannel:
			switch sig {
			case os.Interrupt:
				log.Println("handling:", sig)
				goto Done
			default:
				log.Fatal("Unexpected Signal:", sig)
			}
		}
	}
Done:
	log.Println("stopping Marvin")

}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"html/template"
	"io/ioutil"
	"os"
	"os/exec"
	"regexp"
	"strings"
	"sync"
	"time"

	"github.com/fatih/color"

	"golang.org/x/crypto/ssh/terminal"
	yaml "gopkg.in/yaml.v2"
)

type marvin struct {
	Config struct {
		Relative  bool   `yaml:"relative"`
		Delemiter string `yaml:"delemiter"`
	}
	Tasks     map[string]string `yaml:"tasks"`
	Inventory struct {
		static    []map[string]string
		StaticRaw string            `yaml:"static"`
		Dynamic   map[string]string `yaml:"dynamic"`
	} `yaml:"inventory"`
	filtered []map[string]string
	lock     *sync.Mutex
}

func newMarvin(config, currentDirectory, query, task, args string) *marvin {
	m := &marvin{
		lock: &sync.Mutex{},
	}

	// set configuration defaults
	m.setConfigDefaults()

	// set task defaults
	m.setTaskDefaults()

	// set some default dynamic inventory
	m.setDynamicInventoryDefaults()

	err := yaml.Unmarshal([]byte(config), &m)
	if err != nil {
		speak(err.Error(), false)
		speak("marvin.yml> invalid", true)
	}

	if m.Config.Relative {
		os.Chdir(currentDirectory)
	}

	// stdin to static inventory
	if !terminal.IsTerminal(0) {
		stdin, _ := ioutil.ReadAll(os.Stdin)
		m.Inventory.StaticRaw += "\n" + string(stdin)
	}

	m.filter(query)

	if len(m.filtered) == 0 {
		speak("inventory> none matched your criteria", true)
	}
	m.task(task, args)
	return m
}

func (m *marvin) rawToInventory(raw string) []map[string]string {
	inventory := make([]map[string]string, 0)
	// convert strings to actual inventory
	for _, iRecord := range strings.Split(strings.TrimSpace(m.Inventory.StaticRaw), "\n") {
		for _, row := range strings.Split(iRecord, "\n") {
			i := make(map[string]string)
			if row == "" {
				continue
			}

			for id, kvs := range strings.Split(row, m.Config.Delemiter) {
				kv := strings.SplitN(kvs, ":", 2)
				if len(kv) == 1 {
					i["id"] = kv[0]
				} else {
					i[kv[0]] = strings.Join(kv[1:], m.Config.Delemiter)
					// always set an id
					if id == 0 {
						i["id"] = i[kv[0]]
					}
				}

				// regardless, lets add a raw
				i["raw"] = row
			}

			// add the inventory
			inventory = append(inventory, i)
		}
	}
	m.Inventory.static = inventory
	return m.Inventory.static
}

func (m *marvin) filter(queryString string) []map[string]string {
	var sKey, sValue string
	// kiss for now, replace * with .* and then regex match
	filtered := make([]map[string]string, 0)
	// figure out what we are searching for.
	q := strings.Split(queryString, ":")
	if len(q) == 2 {
		sKey, sValue = q[0], q[1]
		if sKey == "" {
			sKey = "id"
		}
		if sValue == "" {
			sValue = "*"
		}
	} else {
		sKey, sValue = "id", q[0]
	}

	if dynamicCmd, dynamicCmdExists := m.Inventory.Dynamic[sKey]; dynamicCmdExists {
		dInventoryOutput, dInventoryError := m.exec(dynamicCmd)
		if dInventoryError == nil && strings.TrimSpace(dInventoryOutput) != "" {
			// if no errors, add it
			m.lock.Lock()
			m.Inventory.StaticRaw += "\n"
			m.lock.Unlock()
			for _, row := range strings.Split(dInventoryOutput, "\n") {
				// add each record, with the inventory name
				m.lock.Lock()
				m.Inventory.StaticRaw += sKey + ":" + row + "\n"
				m.lock.Unlock()
			}
		}
	}

	var wg sync.WaitGroup
	if sKey == "id" || sKey == "raw" {
		// sigh, we need to generate ALL inventory because we just don't know
		for dInventoryName, dInventoryCmd := range m.Inventory.Dynamic {
			wg.Add(1)
			go func(dInventoryCmd, dInventoryName string) {
				m.lock.Lock()
				m.Inventory.StaticRaw += "\n"
				m.lock.Unlock()
				if dInventoryOutput, dInventoryError := m.exec(dInventoryCmd); dInventoryError == nil && strings.TrimSpace(dInventoryOutput) != "" {
					// no errors, sweet
					for _, row := range strings.Split(dInventoryOutput, "\n") {
						if row == "" {
							continue
						}
						// add each record, with the inventory name
						m.lock.Lock()
						m.Inventory.StaticRaw += dInventoryName + ":" + row + "\n"
						m.lock.Unlock()
					}
				}
				wg.Done()
			}(dInventoryCmd, dInventoryName)
		}
		wg.Wait()
	}

	// inventory should now be set ...
	m.rawToInventory(m.Inventory.StaticRaw)

	// replace
	sKey = strings.Replace(sKey, "*", ".*", -1)
	sValue = strings.Replace(sValue, "*", ".*", -1)

	for _, i := range m.Inventory.static {
		for k, v := range i {
			keyMatch, _ := regexp.MatchString(sKey, k)
			valueMatch, _ := regexp.MatchString(sValue, v)
			if keyMatch && valueMatch {
				filtered = append(filtered, i)
			}
		}
	}
	m.filtered = filtered
	return filtered
}

func (m *marvin) task(taskName, args string) {
	rawTask, exists := m.Tasks[taskName]
	var wg sync.WaitGroup
	if !exists {
		rawTask = "sh -c '" + taskName + " " + args + "'"
	}

	// first lets do an inventory check
	rawCommands := m.inventoryCheck(rawTask, m.filtered, args)

	// sweet, moving along
	// TODO: we are just going to spin them all up(we will figure out batching later)
	for cmdID, command := range rawCommands {
		wg.Add(1)
		go func(cmdID, command string) {

			output, execError := m.exec(command)

			// hopefully only one line :fingers_crossed
			if len(strings.Split(output, "\n")) > 1 {
				output = "\n" + output + "\n\n"
			}

			if execError != nil {
				color.Red(cmdID + "> " + output)
			} else {
				color.Green(cmdID + "> " + output)
			}
			wg.Done()
		}(cmdID, command)
	}
	wg.Wait()
}

func (m *marvin) inventoryCheck(task string, filtered []map[string]string, args string) map[string]string {
	task = strings.Replace(task, "<", "<", -1)
	rawCommands := make(map[string]string, 0)
	for _, inventory := range filtered {
		// add defaults to filtered inventory
		inventory["args"] = args
		inventory["time"] = time.Now().String()
		cmd, cmdParseError := m.template(task, inventory)
		if cmdParseError != nil {
			speak(inventory["id"]+"> missing/invalid arguments", true)
		}
		rawCommands[inventory["id"]] = cmd
	}
	return rawCommands
}

func (m *marvin) template(task string, inventory map[string]string) (string, error) {
	template := template.Must(template.New("translate").Parse(task))
	b := new(bytes.Buffer)
	err := template.Execute(b, inventory)
	if err == nil {
		return b.String(), nil
	}
	return "", fmt.Errorf("id: " + inventory["id"])
}

func (m *marvin) exec(command string) (string, error) {
	cmd := exec.Command("sh", "-c", command)
	execOutput, execError := cmd.CombinedOutput()
	output := strings.TrimSpace(string(execOutput))
	output = strings.Trim(output, "\r\n")
	return output, execError
}

func (m *marvin) setDynamicInventoryDefaults() {
	m.Inventory.Dynamic = make(map[string]string)

	m.Inventory.Dynamic["file"] = "ls -R -1"
	m.Inventory.Dynamic["dir"] = "ls -d */ | cut -f1 -d'/'"
	m.Inventory.Dynamic["branch"] = "git branch | cut -c 3-"
	m.Inventory.Dynamic["docker"] = "docker ps --format \"{{ .Names }}\""
	m.Inventory.Dynamic["bookmark"] = `cat ~/Library/Application\ Support/Google/Chrome/Default/Bookmarks | grep -i http | sed "s/ //g" | sed "s/\"//g" | sed "s/url://g"`
	m.Inventory.Dynamic["ec2"] = `aws ec2 describe-instances --output=text | grep "TAGS$(printf '\t')Name" | grep -v " " | sed "s/TAGS$(printf '\t')Name$(printf '\t')//g"`
}

func (m *marvin) setTaskDefaults() {
	m.Tasks = make(map[string]string)
	// set default tasks
	m.Tasks["ssh"] = "ssh {{ .host }} {{ .args }}"
	m.Tasks["ls"] = "echo {{ .raw }}"
	m.Tasks["open"] = "open {{ .id }} && echo opening"
}

func (m *marvin) setConfigDefaults() {
	// delemiter
	m.Config.Delemiter = "\t"

	// relative?
	m.Config.Relative = true
}
<commit_msg>Updated del to be a space<commit_after>package main

import (
	"bytes"
	"fmt"
	"html/template"
	"io/ioutil"
	"os"
	"os/exec"
	"regexp"
	"strings"
	"sync"
	"time"

	"github.com/fatih/color"

	"golang.org/x/crypto/ssh/terminal"
	yaml "gopkg.in/yaml.v2"
)

type marvin struct {
	Config struct {
		Relative  bool   `yaml:"relative"`
		Delemiter string `yaml:"delemiter"`
	}
	Tasks     map[string]string `yaml:"tasks"`
	Inventory struct {
		static    []map[string]string
		StaticRaw string            `yaml:"static"`
		Dynamic   map[string]string `yaml:"dynamic"`
	} `yaml:"inventory"`
	filtered []map[string]string
	lock     *sync.Mutex
}

func newMarvin(config, currentDirectory, query, task, args string) *marvin {
	m := &marvin{
		lock: &sync.Mutex{},
	}

	// set configuration defaults
	m.setConfigDefaults()

	// set task defaults
	m.setTaskDefaults()

	// set some default dynamic inventory
	m.setDynamicInventoryDefaults()

	err := yaml.Unmarshal([]byte(config), &m)
	if err != nil {
		speak(err.Error(), false)
		speak("marvin.yml> invalid", true)
	}

	if m.Config.Relative {
		os.Chdir(currentDirectory)
	}

	// stdin to static inventory
	if !terminal.IsTerminal(0) {
		stdin, _ := ioutil.ReadAll(os.Stdin)
		m.Inventory.StaticRaw += "\n" + string(stdin)
	}

	m.filter(query)

	if len(m.filtered) == 0 {
		speak("inventory> none matched your criteria", true)
	}
	m.task(task, args)
	return m
}

func (m *marvin) rawToInventory(raw string) []map[string]string {
	inventory := make([]map[string]string, 0)
	// convert strings to actual inventory
	for _, iRecord := range strings.Split(strings.TrimSpace(m.Inventory.StaticRaw), "\n") {
		for _, row := range strings.Split(iRecord, "\n") {
			i := make(map[string]string)
			if row == "" {
				continue
			}

			for id, kvs := range strings.Split(row, m.Config.Delemiter) {
				kv := strings.SplitN(kvs, ":", 2)
				if len(kv) == 1 {
					i["id"] = kv[0]
				} else {
					i[kv[0]] = strings.Join(kv[1:], m.Config.Delemiter)
					// always set an id
					if id == 0 {
						i["id"] = i[kv[0]]
					}
				}

				// regardless, lets add a raw
				i["raw"] = row
			}

			// add the inventory
			inventory = append(inventory, i)
		}
	}
	m.Inventory.static = inventory
	return m.Inventory.static
}

func (m *marvin) filter(queryString string) []map[string]string {
	var sKey, sValue string
	// kiss for now, replace * with .* and then regex match
	filtered := make([]map[string]string, 0)
	// figure out what we are searching for.
	q := strings.Split(queryString, ":")
	if len(q) == 2 {
		sKey, sValue = q[0], q[1]
		if sKey == "" {
			sKey = "id"
		}
		if sValue == "" {
			sValue = "*"
		}
	} else {
		sKey, sValue = "id", q[0]
	}

	if dynamicCmd, dynamicCmdExists := m.Inventory.Dynamic[sKey]; dynamicCmdExists {
		dInventoryOutput, dInventoryError := m.exec(dynamicCmd)
		if dInventoryError == nil && strings.TrimSpace(dInventoryOutput) != "" {
			// if no errors, add it
			m.lock.Lock()
			m.Inventory.StaticRaw += "\n"
			m.lock.Unlock()
			for _, row := range strings.Split(dInventoryOutput, "\n") {
				// add each record, with the inventory name
				m.lock.Lock()
				m.Inventory.StaticRaw += sKey + ":" + row + "\n"
				m.lock.Unlock()
			}
		}
	}

	var wg sync.WaitGroup
	if sKey == "id" || sKey == "raw" {
		// sigh, we need to generate ALL inventory because we just don't know
		for dInventoryName, dInventoryCmd := range m.Inventory.Dynamic {
			wg.Add(1)
			go func(dInventoryCmd, dInventoryName string) {
				m.lock.Lock()
				m.Inventory.StaticRaw += "\n"
				m.lock.Unlock()
				if dInventoryOutput, dInventoryError := m.exec(dInventoryCmd); dInventoryError == nil && strings.TrimSpace(dInventoryOutput) != "" {
					// no errors, sweet
					for _, row := range strings.Split(dInventoryOutput, "\n") {
						if row == "" {
							continue
						}
						// add each record, with the inventory name
						m.lock.Lock()
						m.Inventory.StaticRaw += dInventoryName + ":" + row + "\n"
						m.lock.Unlock()
					}
				}
				wg.Done()
			}(dInventoryCmd, dInventoryName)
		}
		wg.Wait()
	}

	// inventory should now be set ...
	m.rawToInventory(m.Inventory.StaticRaw)

	// replace
	sKey = strings.Replace(sKey, "*", ".*", -1)
	sValue = strings.Replace(sValue, "*", ".*", -1)

	for _, i := range m.Inventory.static {
		for k, v := range i {
			keyMatch, _ := regexp.MatchString(sKey, k)
			valueMatch, _ := regexp.MatchString(sValue, v)
			if keyMatch && valueMatch {
				filtered = append(filtered, i)
			}
		}
	}
	m.filtered = filtered
	return filtered
}

func (m *marvin) task(taskName, args string) {
	rawTask, exists := m.Tasks[taskName]
	var wg sync.WaitGroup
	if !exists {
		rawTask = "sh -c '" + taskName + " " + args + "'"
	}

	// first lets do an inventory check
	rawCommands := m.inventoryCheck(rawTask, m.filtered, args)

	// sweet, moving along
	// TODO: we are just going to spin them all up(we will figure out batching later)
	for cmdID, command := range rawCommands {
		wg.Add(1)
		go func(cmdID, command string) {

			output, execError := m.exec(command)

			// hopefully only one line :fingers_crossed
			if len(strings.Split(output, "\n")) > 1 {
				output = "\n" + output + "\n\n"
			}

			if execError != nil {
				color.Red(cmdID + "> " + output)
			} else {
				color.Green(cmdID + "> " + output)
			}
			wg.Done()
		}(cmdID, command)
	}
	wg.Wait()
}

func (m *marvin) inventoryCheck(task string, filtered []map[string]string, args string) map[string]string {
	task = strings.Replace(task, "<", "<", -1)
	rawCommands := make(map[string]string, 0)
	for _, inventory := range filtered {
		// add defaults to filtered inventory
		inventory["args"] = args
		inventory["time"] = time.Now().String()
		cmd, cmdParseError := m.template(task, inventory)
		if cmdParseError != nil {
			speak(inventory["id"]+"> missing/invalid arguments", true)
		}
		rawCommands[inventory["id"]] = cmd
	}
	return rawCommands
}

func (m *marvin) template(task string, inventory map[string]string) (string, error) {
	template := template.Must(template.New("translate").Parse(task))
	b := new(bytes.Buffer)
	err := template.Execute(b, inventory)
	if err == nil {
		return b.String(), nil
	}
	return "", fmt.Errorf("id: " + inventory["id"])
}

func (m *marvin) exec(command string) (string, error) {
	cmd := exec.Command("sh", "-c", command)
	execOutput, execError := cmd.CombinedOutput()
	output := strings.TrimSpace(string(execOutput))
	output = strings.Trim(output, "\r\n")
	return output, execError
}

func (m *marvin) setDynamicInventoryDefaults() {
	m.Inventory.Dynamic = make(map[string]string)

	m.Inventory.Dynamic["file"] = "ls -R -1"
	m.Inventory.Dynamic["dir"] = "ls -d */ | cut -f1 -d'/'"
	m.Inventory.Dynamic["branch"] = "git branch | cut -c 3-"
	m.Inventory.Dynamic["docker"] = "docker ps --format \"{{ .Names }}\""
	m.Inventory.Dynamic["bookmark"] = `cat ~/Library/Application\ Support/Google/Chrome/Default/Bookmarks | grep -i http | sed "s/ //g" | sed "s/\"//g" | sed "s/url://g"`
	m.Inventory.Dynamic["ec2"] = `aws ec2 describe-instances --output=text | grep "TAGS$(printf '\t')Name" | grep -v " " | sed "s/TAGS$(printf '\t')Name$(printf '\t')//g"`
}

func (m *marvin) setTaskDefaults() {
	m.Tasks = make(map[string]string)
	// set default tasks
	m.Tasks["ssh"] = "ssh {{ .host }} {{ .args }}"
	m.Tasks["ls"] = "echo {{ .raw }}"
	m.Tasks["open"] = "open {{ .id }} && echo opening"
}

func (m *marvin) setConfigDefaults() {
	// delemiter
	m.Config.Delemiter = " "

	// relative?
	m.Config.Relative = true
}
<|endoftext|>


<commit_before>package ratelimit

import (
	"errors"
	"time"
)

var (
	ErrLimitReached = errors.New("Limit reached")
)

type TokenBucket struct {
	Used           int64
	LastAccessTime time.Time
	Limit          int64
	Duration       time.Duration
}

func NewTokenBucket(limit int64, duration time.Duration) *TokenBucket {
	return &TokenBucket{0, time.Now(), limit, duration}
}

func (bucket *TokenBucket) Consume(count int64, limit int64, duration time.Duration) error {
	now := time.Now()

	if duration.Seconds() == 0 {
		return nil
	}

	if bucket.Limit != limit || bucket.Duration != duration {
		bucket.Used = 0
		bucket.LastAccessTime = now
		bucket.Limit = limit
		bucket.Duration = duration
	}

	used := bucket.Used

	if bucket.LastAccessTime.Unix() > 0 {
		elapsed := now.Sub(bucket.LastAccessTime)
		back := limit * int64(elapsed.Seconds()) / int64(duration.Seconds())
		used -= back
		if used < 0 {
			used = 0
		}
	}

	if used+count <= limit {
		bucket.Used = used + count
		bucket.LastAccessTime = now

		return nil
	}

	return ErrLimitReached
}
<commit_msg>In TokenBucket Moved the Usage Calculation to separate function<commit_after>package ratelimit

import (
	"errors"
	"time"
)

var (
	ErrLimitReached = errors.New("Limit reached")
)

type TokenBucket struct {
	Used           int64
	LastAccessTime time.Time
	Limit          int64
	Duration       time.Duration
}

func NewTokenBucket(limit int64, duration time.Duration) *TokenBucket {
	return &TokenBucket{0, time.Now(), limit, duration}
}

func (bucket *TokenBucket) Consume(count int64, limit int64, duration time.Duration) error {
	now := time.Now()

	if duration.Seconds() == 0 {
		return nil
	}

	if bucket.Limit != limit || bucket.Duration != duration {
		bucket.Used = 0
		bucket.LastAccessTime = now
		bucket.Limit = limit
		bucket.Duration = duration
	}

	used := bucket.GetAdjustedUsage()

	if used+count <= limit {
		bucket.Used = used + count
		bucket.LastAccessTime = now
		return nil
	}

	return ErrLimitReached
}

func (bucket *TokenBucket) GetAdjustedUsage() int64 {
	now := time.Now()
	used := bucket.Used
	if bucket.LastAccessTime.Unix() > 0 {
		elapsed := now.Sub(bucket.LastAccessTime)
		back := bucket.Limit * int64(elapsed.Seconds()) / int64(bucket.Duration.Seconds())
		used -= back
		if used < 0 {
			used = 0
		}
	}
	return used
}
<|endoftext|>


<commit_before>package mathgl

import (
	"errors"
)

type MatrixMultiplyable interface {
	Mul(m MatrixMultiplyable) Matrix
}

type Matrix struct {
	m, n int // an m x n matrix
	typ  VecType
	dat  []Scalar
}

func NewMatrix(m, n int, typ VecType) *Matrix {
	return &Matrix{m: m, n: n, typ: typ, dat: make([]Scalar, 0, 2)}
}

func Identity(size int, typ VecType) Matrix {
	dat := make([]Scalar, size*size)

	for i := 0; i < size; i++ {
		for j := 0; j < size; j++ {
			if i == j {
				dat[i*size+j] = MakeScalar(1, typ)
			} else {
				dat[i*size+j] = vecNumZero(typ)
			}
		}
	}
	return *unsafeMatrixFromSlice(typ, dat, size, size)
}

// [1, 1]
// [0, 1] Would be entered as a 2D array [[1,1],[0,1]] -- but converted to RMO
//
// This may seem confusing, but it's because it's easier to type out and visualize things in CMO
// So it's easier to type write your matrix as a slice in CMO, and pass it into this method
func MatrixFromCols(typ VecType, el [][]Scalar) (mat *Matrix, err error) {
	mat.typ = typ

	mat.m = len(el)
	mat.n = len(el[0])
	mat.dat = make([]Scalar, 0, mat.m*mat.n)

	// Row Major Order, like in OpenGL
	for i := 0; i < mat.n; i++ {
		for j := 0; j < mat.m; j++ {
			if !checkType(mat.typ, el[j][i]) {
				return nil, errors.New("Element type does not match matrix")
			}
			mat.dat = append(mat.dat, el[j][i])
		}
	}

	return mat, nil
}

// This function is MatrixFromCols, except each slice is a row of the vector (from top to bottom), rather than a column
func MatrixFromRows(typ VecType, el [][]Scalar) (mat *Matrix, err error) {
	mat.typ = typ

	mat.m = len(el)
	mat.n = len(el[0])
	mat.dat = make([]Scalar, 0, mat.m*mat.n)

	// Row Major Order, like in OpenGL
	for i := 0; i < mat.m; i++ {
		for j := 0; j < mat.n; j++ {
			if !checkType(mat.typ, el[j][i]) {
				return nil, errors.New("Element type does not match matrix")
			}
			mat.dat = append(mat.dat, el[j][i])
		}
	}

	return mat, nil
}

// Slice-format data should be in Row Major Order
func MatrixFromSlice(typ VecType, el []Scalar, m, n int) (mat *Matrix, err error) {
	mat = &Matrix{}
	mat.typ = typ
	mat.m = m
	mat.n = n

	if mat.m*mat.n != len(el) {
		return nil, errors.New("Matrix dimensions do not match data passed in")
	}

	for _, e := range el {
		if !checkType(mat.typ, e) {
			return nil, errors.New("Type of at least one element does not match declared type")
		}
	}

	mat.dat = el

	return mat, nil
}

// Quick and dirty internal function to make a matrix without spending time checking types
func unsafeMatrixFromSlice(typ VecType, el []Scalar, m, n int) (mat *Matrix) {
	mat = &Matrix{}
	mat.typ = typ
	mat.m = m
	mat.n = n

	mat.dat = el

	return mat
}

func (m Matrix) Type() VecType {
	return m.typ
}

func (m1 Matrix) Equal(m2 Matrix) (eq bool) {
	if m1.typ != m2.typ || m1.n != m2.n || m1.m != m2.m {
		return false
	}

	for i := 0; i < len(m1.dat); i++ {
		eq = m1.dat[i].Equal(m2.dat[i])
		if !eq {
			break
		}
	}

	return eq
}

func (m Matrix) AsSlice() []Scalar {
	return m.dat
}

// TODO: "Append" data method (expand the matrix)

func (mat *Matrix) SetElement(i, j int, el Scalar) error {
	if i < mat.m || j < mat.n {
		return errors.New("Dimensions out of bounds")
	}

	if !checkType(mat.typ, el) {
		return errors.New("Type of element does not match matrix's type")
	}

	mat.dat[mat.m*j+i] = el

	return nil
}

func (mat Matrix) AsVector() (v Vector, err error) {
	if mat.m != 1 && mat.n != 1 {
		return v, errors.New("Matrix is not 1-dimensional in either direction.")
	}

	vPoint, err := VectorOf(mat.typ, mat.dat)
	if err != nil {
		return v, err
	}

	return *vPoint, nil
}

func (mat Matrix) ToScalar() Scalar {
	if mat.m != 1 || mat.n != 1 {
		return nil
	}

	return mat.dat[0]
}

// TODO: AsArray (and possibly As2dSliceRow/Col)

func (m1 Matrix) Add(m2 Matrix) (m3 Matrix) {
	if m1.typ != m2.typ || len(m1.dat) != len(m2.dat) {
		return
	}

	m3.typ = m1.typ
	m3.dat = make([]Scalar, len(m1.dat))

	for i := range m1.dat {
		m3.dat[i] = m1.dat[i].Add(m2.dat[i])
	}

	return m3
}

func (m1 Matrix) Sub(m2 Matrix) (m3 Matrix) {
	if m1.typ != m2.typ || len(m1.dat) != len(m2.dat) {
		return
	}

	m3.typ = m1.typ
	m3.dat = make([]Scalar, len(m1.dat))

	for i := range m1.dat {
		m3.dat[i] = m1.dat[i].Sub(m2.dat[i])
	}

	return m3
}

func (m1 Matrix) ScalarMul(c Scalar) (mat Matrix) {
	if !checkType(m1.typ, c) {
		return
	}

	dat := make([]Scalar, len(m1.dat))
	for i := range m1.dat {
		dat[i] = m1.dat[i].Mul(c)
	}

	return *unsafeMatrixFromSlice(m1.typ, dat, m1.m, m1.n)

}

func (m1 Matrix) Mul(m2 MatrixMultiplyable) (m3 Matrix) {
	var indat []Scalar
	m, n, o := m1.m, m1.n, 0

	if vec, ok := m2.(Vector); ok {
		if vec.typ != m1.typ || n != len(vec.dat) {
			return
		}
		if m1.m == 1 && m1.n == 1 {
			m3, _ = vec.ScalarMul(m1.ToScalar()).AsMatrix(false)
			return m3
		}
		indat = vec.dat
		o = 1
	} else {
		mat := m2.(Matrix)
		if m1.typ != mat.typ || n != mat.m {
			return
		}
		if m1.m == 1 && m1.n == 1 {
			return mat.ScalarMul(m1.ToScalar())
		}
		indat = mat.dat
		o = mat.n
	}

	dat := make([]Scalar, m*o)

	for j := 0; j < o; j++ { // Columns of m2 and m3
		for i := 0; i < m; i++ { // Rows of m1 and m3
			for k := 0; k < n; k++ { // Columns of m1, rows of m2
				dat[j*o+i] = dat[j*o+i].Add(m1.dat[k*n+i].Mul(indat[j*o+k])) // I think, needs testing
			}
		}
	}

	return *unsafeMatrixFromSlice(m1.typ, dat, m, o)
}

/*
Batch Multiply, as its name implies, is supposed to Multiply a huge amount of matrices at once
Since matrix Multiplication is associative, it can do pieces of the problem at the same time.
Since starting a goroutine has some overhead, I'd wager it's probably not worth it to use this function unless you have
6-8+ larger (i.e. 3x3 or 4x4) matrices, but I haven't benchmarked it so until then who knows?

Make sure the matrices are in order, and that they can indeed be Multiplied. If not you'll end up with an untyped 0x0 matrix (a.k.a the "zero type" for a Matrix struct)

If the only input is a single vector, it will return it as a vector as if you called vector.AsMatrix(true), meaning as a *ROW* vector
*/
func BatchMultiply(args []MatrixMultiplyable) Matrix {
	// Fun fact: Since in (Go's) integer division 3/2=1, in the case where you have an odd number
	// of matrices to Multiply, the only way a vector can end up alone is when it's on the left, meaning it has to be a row vector.
	// this allows us to not need a special case for a slice length of 3!
	if len(args) == 1 {
		// We're expected to return a Matrix, so if it's suddenly a vector Go will panic from bad typing
		if vec, ok := args[0].(Vector); ok {
			ret, _ := vec.AsMatrix(true)
			return ret
		}
		return args[0].(Matrix)
	}

	var m1, m2 MatrixMultiplyable
	if len(args) > 2 {
		ch1 := make(chan Matrix)
		ch2 := make(chan Matrix)

		// Split up the work, matrix Mult is associative
		go batchMultHelper(ch1, args[0:len(args)/2])
		go batchMultHelper(ch2, args[len(args)/2:len(args)])

		m1 = <-ch1
		m2 = <-ch2
	} else {
		m1 = args[0]
		m2 = args[1]
	}

	return m1.Mul(m2)
}

func batchMultHelper(ch chan<- Matrix, args []MatrixMultiplyable) {
	ch <- BatchMultiply(args)
	close(ch)
}

// INCOMPLETE DO NOT USE
// Need better function to make matrices for recursion
func (m1 Matrix) Det() Scalar {
	if m1.m != m1.n { // Determinants are only for square matrices
		return nil
	}

	return nil
}

func (m Matrix) Transpose() Matrix {
	dat := make([]Scalar, len(m.dat))

	for i := 0; i < m.n; i++ {
		for j := 0; j < m.m; j++ {
			dat[j+i*m.m] = m.dat[j*m.n+i] // Basically convert to CMO
		}
	}

	return *unsafeMatrixFromSlice(m.typ, dat, m.n, m.m)
}
<commit_msg>Fixed pointer return methods<commit_after>package mathgl

import (
	"errors"
)

type MatrixMultiplyable interface {
	Mul(m MatrixMultiplyable) Matrix
}

type Matrix struct {
	m, n int // an m x n matrix
	typ  VecType
	dat  []Scalar
}

func NewMatrix(m, n int, typ VecType) *Matrix {
	return &Matrix{m: m, n: n, typ: typ, dat: make([]Scalar, 0, 2)}
}

func Identity(size int, typ VecType) Matrix {
	dat := make([]Scalar, size*size)

	for i := 0; i < size; i++ {
		for j := 0; j < size; j++ {
			if i == j {
				dat[i*size+j] = MakeScalar(1, typ)
			} else {
				dat[i*size+j] = vecNumZero(typ)
			}
		}
	}
	return *unsafeMatrixFromSlice(typ, dat, size, size)
}

// [1, 1]
// [0, 1] Would be entered as a 2D array [[1,1],[0,1]] -- but converted to RMO
//
// This may seem confusing, but it's because it's easier to type out and visualize things in CMO
// So it's easier to type write your matrix as a slice in CMO, and pass it into this method
func MatrixFromCols(typ VecType, el [][]Scalar) (mat *Matrix, err error) {
	mat = &Matrix{}
	mat.typ = typ

	mat.m = len(el)
	mat.n = len(el[0])
	mat.dat = make([]Scalar, 0, mat.m*mat.n)

	// Row Major Order, like in OpenGL
	for i := 0; i < mat.n; i++ {
		for j := 0; j < mat.m; j++ {
			if !checkType(mat.typ, el[j][i]) {
				return nil, errors.New("Element type does not match matrix")
			}
			mat.dat = append(mat.dat, el[j][i])
		}
	}

	return mat, nil
}

// This function is MatrixFromCols, except each slice is a row of the vector (from top to bottom), rather than a column
func MatrixFromRows(typ VecType, el [][]Scalar) (mat *Matrix, err error) {
	mat = &Matrix{}
	mat.typ = typ

	mat.m = len(el)
	mat.n = len(el[0])
	mat.dat = make([]Scalar, 0, mat.m*mat.n)

	// Row Major Order, like in OpenGL
	for i := 0; i < mat.m; i++ {
		for j := 0; j < mat.n; j++ {
			if !checkType(mat.typ, el[j][i]) {
				return nil, errors.New("Element type does not match matrix")
			}
			mat.dat = append(mat.dat, el[j][i])
		}
	}

	return mat, nil
}

// Slice-format data should be in Row Major Order
func MatrixFromSlice(typ VecType, el []Scalar, m, n int) (mat *Matrix, err error) {
	mat = &Matrix{}
	mat.typ = typ
	mat.m = m
	mat.n = n

	if mat.m*mat.n != len(el) {
		return nil, errors.New("Matrix dimensions do not match data passed in")
	}

	for _, e := range el {
		if !checkType(mat.typ, e) {
			return nil, errors.New("Type of at least one element does not match declared type")
		}
	}

	mat.dat = el

	return mat, nil
}

// Quick and dirty internal function to make a matrix without spending time checking types
func unsafeMatrixFromSlice(typ VecType, el []Scalar, m, n int) (mat *Matrix) {
	mat = &Matrix{}
	mat.typ = typ
	mat.m = m
	mat.n = n

	mat.dat = el

	return mat
}

func (m Matrix) Type() VecType {
	return m.typ
}

func (m1 Matrix) Equal(m2 Matrix) (eq bool) {
	if m1.typ != m2.typ || m1.n != m2.n || m1.m != m2.m {
		return false
	}

	for i := 0; i < len(m1.dat); i++ {
		eq = m1.dat[i].Equal(m2.dat[i])
		if !eq {
			break
		}
	}

	return eq
}

func (m Matrix) AsSlice() []Scalar {
	return m.dat
}

// TODO: "Append" data method (expand the matrix)

func (mat *Matrix) SetElement(i, j int, el Scalar) error {
	if i < mat.m || j < mat.n {
		return errors.New("Dimensions out of bounds")
	}

	if !checkType(mat.typ, el) {
		return errors.New("Type of element does not match matrix's type")
	}

	mat.dat[mat.m*j+i] = el

	return nil
}

func (mat Matrix) AsVector() (v Vector, err error) {
	if mat.m != 1 && mat.n != 1 {
		return v, errors.New("Matrix is not 1-dimensional in either direction.")
	}

	vPoint, err := VectorOf(mat.typ, mat.dat)
	if err != nil {
		return v, err
	}

	return *vPoint, nil
}

func (mat Matrix) ToScalar() Scalar {
	if mat.m != 1 || mat.n != 1 {
		return nil
	}

	return mat.dat[0]
}

// TODO: AsArray (and possibly As2dSliceRow/Col)

func (m1 Matrix) Add(m2 Matrix) (m3 Matrix) {
	if m1.typ != m2.typ || len(m1.dat) != len(m2.dat) {
		return
	}

	m3.typ = m1.typ
	m3.dat = make([]Scalar, len(m1.dat))

	for i := range m1.dat {
		m3.dat[i] = m1.dat[i].Add(m2.dat[i])
	}

	return m3
}

func (m1 Matrix) Sub(m2 Matrix) (m3 Matrix) {
	if m1.typ != m2.typ || len(m1.dat) != len(m2.dat) {
		return
	}

	m3.typ = m1.typ
	m3.dat = make([]Scalar, len(m1.dat))

	for i := range m1.dat {
		m3.dat[i] = m1.dat[i].Sub(m2.dat[i])
	}

	return m3
}

func (m1 Matrix) ScalarMul(c Scalar) (mat Matrix) {
	if !checkType(m1.typ, c) {
		return
	}

	dat := make([]Scalar, len(m1.dat))
	for i := range m1.dat {
		dat[i] = m1.dat[i].Mul(c)
	}

	return *unsafeMatrixFromSlice(m1.typ, dat, m1.m, m1.n)

}

func (m1 Matrix) Mul(m2 MatrixMultiplyable) (m3 Matrix) {
	var indat []Scalar
	m, n, o := m1.m, m1.n, 0

	if vec, ok := m2.(Vector); ok {
		if vec.typ != m1.typ || n != len(vec.dat) {
			return
		}
		if m1.m == 1 && m1.n == 1 {
			m3, _ = vec.ScalarMul(m1.ToScalar()).AsMatrix(false)
			return m3
		}
		indat = vec.dat
		o = 1
	} else {
		mat := m2.(Matrix)
		if m1.typ != mat.typ || n != mat.m {
			return
		}
		if m1.m == 1 && m1.n == 1 {
			return mat.ScalarMul(m1.ToScalar())
		}
		indat = mat.dat
		o = mat.n
	}

	dat := make([]Scalar, m*o)

	for j := 0; j < o; j++ { // Columns of m2 and m3
		for i := 0; i < m; i++ { // Rows of m1 and m3
			for k := 0; k < n; k++ { // Columns of m1, rows of m2
				dat[j*o+i] = dat[j*o+i].Add(m1.dat[k*n+i].Mul(indat[j*o+k])) // I think, needs testing
			}
		}
	}

	return *unsafeMatrixFromSlice(m1.typ, dat, m, o)
}

/*
Batch Multiply, as its name implies, is supposed to Multiply a huge amount of matrices at once
Since matrix Multiplication is associative, it can do pieces of the problem at the same time.
Since starting a goroutine has some overhead, I'd wager it's probably not worth it to use this function unless you have
6-8+ larger (i.e. 3x3 or 4x4) matrices, but I haven't benchmarked it so until then who knows?

Make sure the matrices are in order, and that they can indeed be Multiplied. If not you'll end up with an untyped 0x0 matrix (a.k.a the "zero type" for a Matrix struct)

If the only input is a single vector, it will return it as a vector as if you called vector.AsMatrix(true), meaning as a *ROW* vector
*/
func BatchMultiply(args []MatrixMultiplyable) Matrix {
	// Fun fact: Since in (Go's) integer division 3/2=1, in the case where you have an odd number
	// of matrices to Multiply, the only way a vector can end up alone is when it's on the left, meaning it has to be a row vector.
	// this allows us to not need a special case for a slice length of 3!
	if len(args) == 1 {
		// We're expected to return a Matrix, so if it's suddenly a vector Go will panic from bad typing
		if vec, ok := args[0].(Vector); ok {
			ret, _ := vec.AsMatrix(true)
			return ret
		}
		return args[0].(Matrix)
	}

	var m1, m2 MatrixMultiplyable
	if len(args) > 2 {
		ch1 := make(chan Matrix)
		ch2 := make(chan Matrix)

		// Split up the work, matrix Mult is associative
		go batchMultHelper(ch1, args[0:len(args)/2])
		go batchMultHelper(ch2, args[len(args)/2:len(args)])

		m1 = <-ch1
		m2 = <-ch2
	} else {
		m1 = args[0]
		m2 = args[1]
	}

	return m1.Mul(m2)
}

func batchMultHelper(ch chan<- Matrix, args []MatrixMultiplyable) {
	ch <- BatchMultiply(args)
	close(ch)
}

// INCOMPLETE DO NOT USE
// Need better function to make matrices for recursion
func (m1 Matrix) Det() Scalar {
	if m1.m != m1.n { // Determinants are only for square matrices
		return nil
	}

	return nil
}

func (m Matrix) Transpose() Matrix {
	dat := make([]Scalar, len(m.dat))

	for i := 0; i < m.n; i++ {
		for j := 0; j < m.m; j++ {
			dat[j+i*m.m] = m.dat[j*m.n+i] // Basically convert to CMO
		}
	}

	return *unsafeMatrixFromSlice(m.typ, dat, m.n, m.m)
}
<|endoftext|>


<commit_before>// Get the number of open files for a process and compare that against /proc/<pid>/limits and alert if
// over the given threshold.
//
//
// LICENSE:
//   Copyright 2015 Yieldbot. <devops@yieldbot.com>
//   Released under the MIT License; see LICENSE
//   for details.

package main

import (
	"flag"
	"fmt"
	"github.com/yieldbot/ybsensupluginfile/cmd/check-file-handles/Godeps/_workspace/src/github.com/yieldbot/ybsensuplugin/ybsensupluginutil"
	"github.com/yieldbot/ybsensupluginfile/cmd/check-file-handles/Godeps/_workspace/src/github.com/yieldbot/ybsensupluginfile/ybfilesys"
)

// Calculate if the value is over a threshold
func determineThreshold(limit float64, threshold float64, numFD float64) bool {
	alarm := true
	tLimit := threshold / float64(100) * limit

	if numFD > float64(tLimit) {
		alarm = true
	} else {
		alarm = false
	}
	return alarm
}

func main() {

	// set commandline flags
	AppPtr := flag.String("app", "sbin/init", "the process name")
	WarnPtr := flag.Int("warn", 75, "the alert warning threshold percentage")
	CritPtr := flag.Int("crit", 75, "the alert critical threshold percentage")
	DebugPtr := flag.Bool("debug", false, "print debugging info instead of an alert")
	JavaAppPtr := flag.Bool("java", false, "java apps process detection is different")

	flag.Parse()
	app := *AppPtr
	warnThreshold := *WarnPtr
	critThreshold := *CritPtr
	ybsensupluginutil.Debug = *DebugPtr
	ybfilesys.JavaApp = *JavaAppPtr

	var appPid string
	var sLimit, hLimit, openFd float64

	// need to find a way to get the printf stuff into the sensu exit function
	if app != "" {
		appPid = ybfilesys.GetPid(app)
		sLimit, hLimit, openFd = ybfilesys.GetFileHandles(appPid)
		if ybsensupluginutil.Debug {
			fmt.Printf("warning threshold: %v percent, critical threshold: %v percent\n", warnThreshold, critThreshold)
			fmt.Printf("this is the number of open files at the specific point in time: %v\n", openFd)
			fmt.Printf("app pid is: %v\n", appPid)
			fmt.Printf("This is the soft limit: %v\n", sLimit)
			fmt.Printf("This is the hard limit: %v\n", hLimit)
			ybsensupluginutil.Exit("ok")
		}
		if determineThreshold(hLimit, float64(critThreshold), openFd) {
			fmt.Printf("%v is over %v percent of the the open file handles hard limit of %v\n", app, critThreshold, hLimit)
			ybsensupluginutil.Exit("critical")
		} else if determineThreshold(sLimit, float64(warnThreshold), openFd) {
			fmt.Printf("%v is over %v percent of the open file handles soft limit of %v\n", app, warnThreshold, sLimit)
			ybsensupluginutil.Exit("warning")
		} else {
			fmt.Printf("There was an error calculating the thresholds. Check to make sure everything got convert to a float64.\n")
			fmt.Printf("If unsure of the use, consult the documentation for examples and requirements\n")
			ybsensupluginutil.Exit(42)
		}
	} else {
		fmt.Printf("Please enter a process name to check. \n")
		fmt.Printf("If unsure consult the documentation for examples and requirements\n")
		ybsensupluginutil.Exit(127)
	}
}
<commit_msg>debugging<commit_after>// Get the number of open files for a process and compare that against /proc/<pid>/limits and alert if
// over the given threshold.
//
//
// LICENSE:
//   Copyright 2015 Yieldbot. <devops@yieldbot.com>
//   Released under the MIT License; see LICENSE
//   for details.

package main

import (
	"flag"
	"fmt"
	"github.com/yieldbot/ybsensupluginfile/cmd/check-file-handles/Godeps/_workspace/src/github.com/yieldbot/ybsensuplugin/ybsensupluginutil"
	"github.com/yieldbot/ybsensupluginfile/cmd/check-file-handles/Godeps/_workspace/src/github.com/yieldbot/ybsensupluginfile/ybfilesys"
m)

// Calculate if the value is over a threshold
//
func determineThreshold(limit float64, threshold float64, numFD float64) bool {
	alarm := true
	tLimit := threshold / float64(100) * limit

	if numFD > float64(tLimit) {
		alarm = true
	} else {
		alarm = false
	}
	return alarm
}

func main() {

	// set commandline flags
	AppPtr := flag.String("app", "sbin/init", "the process name")
	WarnPtr := flag.Int("warn", 75, "the alert warning threshold percentage")
	CritPtr := flag.Int("crit", 75, "the alert critical threshold percentage")
	DebugPtr := flag.Bool("debug", false, "print debugging info instead of an alert")
	JavaAppPtr := flag.Bool("java", false, "java apps process detection is different")

	flag.Parse()
	app := *AppPtr
	warnThreshold := *WarnPtr
	critThreshold := *CritPtr
	ybsensupluginutil.Debug = *DebugPtr
	ybfilesys.JavaApp = *JavaAppPtr

	var appPid string
	var sLimit, hLimit, openFd float64

	// need to find a way to get the printf stuff into the sensu exit function
	if app != "" {
		appPid = ybfilesys.GetPid(app)
		sLimit, hLimit, openFd = ybfilesys.GetFileHandles(appPid)
		if ybsensupluginutil.Debug {
			fmt.Printf("warning threshold: %v percent, critical threshold: %v percent\n", warnThreshold, critThreshold)
			fmt.Printf("this is the number of open files at the specific point in time: %v\n", openFd)
			fmt.Printf("app pid is: %v\n", appPid)
			fmt.Printf("This is the soft limit: %v\n", sLimit)
			fmt.Printf("This is the hard limit: %v\n", hLimit)
			ybsensupluginutil.Exit("ok")
		}
		if determineThreshold(hLimit, float64(critThreshold), openFd) {
			fmt.Printf("%v is over %v percent of the the open file handles hard limit of %v\n", app, critThreshold, hLimit)
			ybsensupluginutil.Exit("critical")
		} else if determineThreshold(sLimit, float64(warnThreshold), openFd) {
			fmt.Printf("%v is over %v percent of the open file handles soft limit of %v\n", app, warnThreshold, sLimit)
			ybsensupluginutil.Exit("warning")
		} else {
			fmt.Printf("There was an error calculating the thresholds. Check to make sure everything got convert to a float64.\n")
			fmt.Printf("If unsure of the use, consult the documentation for examples and requirements\n")
			ybsensupluginutil.Exit(42)
		}
	} else {
		fmt.Printf("Please enter a process name to check. \n")
		fmt.Printf("If unsure consult the documentation for examples and requirements\n")
		ybsensupluginutil.Exit(127)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package app does all of the work necessary to create a Kubernetes
// APIServer by binding together the API, master and APIServer infrastructure.
// It can be configured and called directly or via the hyperkube cache.
package app

import (
	"strings"
	"time"

	"github.com/golang/glog"
	"github.com/pborman/uuid"
	"github.com/spf13/cobra"
	"github.com/spf13/pflag"

	"k8s.io/kubernetes/federation/cmd/federation-apiserver/app/options"
	"k8s.io/kubernetes/pkg/admission"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/unversioned"
	"k8s.io/kubernetes/pkg/apiserver/authenticator"
	apiserveropenapi "k8s.io/kubernetes/pkg/apiserver/openapi"
	authorizerunion "k8s.io/kubernetes/pkg/auth/authorizer/union"
	"k8s.io/kubernetes/pkg/auth/user"
	"k8s.io/kubernetes/pkg/controller/informers"
	"k8s.io/kubernetes/pkg/generated/openapi"
	"k8s.io/kubernetes/pkg/genericapiserver"
	"k8s.io/kubernetes/pkg/genericapiserver/authorizer"
	genericvalidation "k8s.io/kubernetes/pkg/genericapiserver/validation"
	"k8s.io/kubernetes/pkg/registry/cachesize"
	"k8s.io/kubernetes/pkg/registry/generic"
	"k8s.io/kubernetes/pkg/registry/generic/registry"
	"k8s.io/kubernetes/pkg/routes"
	"k8s.io/kubernetes/pkg/util/wait"
	"k8s.io/kubernetes/pkg/version"
	authenticatorunion "k8s.io/kubernetes/plugin/pkg/auth/authenticator/request/union"
)

// NewAPIServerCommand creates a *cobra.Command object with default parameters
func NewAPIServerCommand() *cobra.Command {
	s := options.NewServerRunOptions()
	s.AddFlags(pflag.CommandLine)
	cmd := &cobra.Command{
		Use: "federation-apiserver",
		Long: `The Kubernetes federation API server validates and configures data
for the api objects which include pods, services, replicationcontrollers, and
others. The API Server services REST operations and provides the frontend to the
cluster's shared state through which all other components interact.`,
		Run: func(cmd *cobra.Command, args []string) {
		},
	}
	return cmd
}

// Run runs the specified APIServer.  This should never exit.
func Run(s *options.ServerRunOptions) error {
	genericvalidation.VerifyEtcdServersList(s.ServerRunOptions)
	genericapiserver.DefaultAndValidateRunOptions(s.ServerRunOptions)
	genericConfig := genericapiserver.NewConfig(). // create the new config
							ApplyOptions(s.ServerRunOptions). // apply the options selected
							Complete()                        // set default values based on the known values

	if err := genericConfig.MaybeGenerateServingCerts(); err != nil {
		glog.Fatalf("Failed to generate service certificate: %v", err)
	}

	// TODO: register cluster federation resources here.
	resourceConfig := genericapiserver.NewResourceConfig()

	if s.StorageConfig.DeserializationCacheSize == 0 {
		// When size of cache is not explicitly set, set it to 50000
		s.StorageConfig.DeserializationCacheSize = 50000
	}
	storageGroupsToEncodingVersion, err := s.StorageGroupsToEncodingVersion()
	if err != nil {
		glog.Fatalf("error generating storage version map: %s", err)
	}
	storageFactory, err := genericapiserver.BuildDefaultStorageFactory(
		s.StorageConfig, s.DefaultStorageMediaType, api.Codecs,
		genericapiserver.NewDefaultResourceEncodingConfig(), storageGroupsToEncodingVersion,
		[]unversioned.GroupVersionResource{}, resourceConfig, s.RuntimeConfig)
	if err != nil {
		glog.Fatalf("error in initializing storage factory: %s", err)
	}

	for _, override := range s.EtcdServersOverrides {
		tokens := strings.Split(override, "#")
		if len(tokens) != 2 {
			glog.Errorf("invalid value of etcd server overrides: %s", override)
			continue
		}

		apiresource := strings.Split(tokens[0], "/")
		if len(apiresource) != 2 {
			glog.Errorf("invalid resource definition: %s", tokens[0])
			continue
		}
		group := apiresource[0]
		resource := apiresource[1]
		groupResource := unversioned.GroupResource{Group: group, Resource: resource}

		servers := strings.Split(tokens[1], ";")
		storageFactory.SetEtcdLocation(groupResource, servers)
	}

	apiAuthenticator, securityDefinitions, err := authenticator.New(authenticator.AuthenticatorConfig{
		Anonymous:         s.AnonymousAuth,
		AnyToken:          s.EnableAnyToken,
		BasicAuthFile:     s.BasicAuthFile,
		ClientCAFile:      s.ClientCAFile,
		TokenAuthFile:     s.TokenAuthFile,
		OIDCIssuerURL:     s.OIDCIssuerURL,
		OIDCClientID:      s.OIDCClientID,
		OIDCCAFile:        s.OIDCCAFile,
		OIDCUsernameClaim: s.OIDCUsernameClaim,
		OIDCGroupsClaim:   s.OIDCGroupsClaim,
		KeystoneURL:       s.KeystoneURL,
	})
	if err != nil {
		glog.Fatalf("Invalid Authentication Config: %v", err)
	}

	privilegedLoopbackToken := uuid.NewRandom().String()
	selfClientConfig, err := s.NewSelfClientConfig(privilegedLoopbackToken)
	if err != nil {
		glog.Fatalf("Failed to create clientset: %v", err)
	}
	client, err := s.NewSelfClient(privilegedLoopbackToken)
	if err != nil {
		glog.Errorf("Failed to create clientset: %v", err)
	}
	sharedInformers := informers.NewSharedInformerFactory(client, 10*time.Minute)

	authorizationConfig := authorizer.AuthorizationConfig{
		PolicyFile:                  s.AuthorizationPolicyFile,
		WebhookConfigFile:           s.AuthorizationWebhookConfigFile,
		WebhookCacheAuthorizedTTL:   s.AuthorizationWebhookCacheAuthorizedTTL,
		WebhookCacheUnauthorizedTTL: s.AuthorizationWebhookCacheUnauthorizedTTL,
		RBACSuperUser:               s.AuthorizationRBACSuperUser,
		InformerFactory:             sharedInformers,
	}
	authorizationModeNames := strings.Split(s.AuthorizationMode, ",")
	apiAuthorizer, err := authorizer.NewAuthorizerFromAuthorizationConfig(authorizationModeNames, authorizationConfig)
	if err != nil {
		glog.Fatalf("Invalid Authorization Config: %v", err)
	}

	admissionControlPluginNames := strings.Split(s.AdmissionControl, ",")

	// TODO(dims): We probably need to add an option "EnableLoopbackToken"
	if apiAuthenticator != nil {
		var uid = uuid.NewRandom().String()
		tokens := make(map[string]*user.DefaultInfo)
		tokens[privilegedLoopbackToken] = &user.DefaultInfo{
			Name:   user.APIServerUser,
			UID:    uid,
			Groups: []string{user.SystemPrivilegedGroup},
		}

		tokenAuthenticator := authenticator.NewAuthenticatorFromTokens(tokens)
		apiAuthenticator = authenticatorunion.New(tokenAuthenticator, apiAuthenticator)

		tokenAuthorizer := authorizer.NewPrivilegedGroups(user.SystemPrivilegedGroup)
		apiAuthorizer = authorizerunion.New(tokenAuthorizer, apiAuthorizer)
	}

	pluginInitializer := admission.NewPluginInitializer(sharedInformers, apiAuthorizer)

	admissionController, err := admission.NewFromPlugins(client, admissionControlPluginNames, s.AdmissionControlConfigFile, pluginInitializer)
	if err != nil {
		glog.Fatalf("Failed to initialize plugins: %v", err)
	}

	kubeVersion := version.Get()
	genericConfig.Version = &kubeVersion
	genericConfig.LoopbackClientConfig = selfClientConfig
	genericConfig.Authenticator = apiAuthenticator
	genericConfig.SupportsBasicAuth = len(s.BasicAuthFile) > 0
	genericConfig.Authorizer = apiAuthorizer
	genericConfig.AuthorizerRBACSuperUser = s.AuthorizationRBACSuperUser
	genericConfig.AdmissionControl = admissionController
	genericConfig.APIResourceConfigSource = storageFactory.APIResourceConfigSource
	genericConfig.MasterServiceNamespace = s.MasterServiceNamespace
	genericConfig.OpenAPIConfig.Definitions = openapi.OpenAPIDefinitions
	// Reusing api-server's GetOperationID function. if federation and api-server spec diverge and
	// this method does not provide good operation IDs for federation, we should create federation's own GetOperationID.
	genericConfig.OpenAPIConfig.GetOperationID = apiserveropenapi.GetOperationID
	genericConfig.EnableOpenAPISupport = true
	genericConfig.OpenAPIConfig.SecurityDefinitions = securityDefinitions

	// TODO: Move this to generic api server (Need to move the command line flag).
	if s.EnableWatchCache {
		cachesize.InitializeWatchCacheSizes(s.TargetRAMMB)
		cachesize.SetWatchCacheSizes(s.WatchCacheSizes)
	}

	m, err := genericConfig.New()
	if err != nil {
		return err
	}

	routes.UIRedirect{}.Install(m.HandlerContainer)
	routes.Logs{}.Install(m.HandlerContainer)

	restOptionsFactory := restOptionsFactory{
		storageFactory:          storageFactory,
		deleteCollectionWorkers: s.DeleteCollectionWorkers,
	}
	if s.EnableWatchCache {
		restOptionsFactory.storageDecorator = registry.StorageWithCacher
	} else {
		restOptionsFactory.storageDecorator = generic.UndecoratedStorage
	}

	installFederationAPIs(m, restOptionsFactory)
	installCoreAPIs(s, m, restOptionsFactory)
	installExtensionsAPIs(m, restOptionsFactory)

	sharedInformers.Start(wait.NeverStop)
	m.Run()
	return nil
}

type restOptionsFactory struct {
	storageFactory          genericapiserver.StorageFactory
	storageDecorator        generic.StorageDecorator
	deleteCollectionWorkers int
}

func (f restOptionsFactory) NewFor(resource unversioned.GroupResource) generic.RESTOptions {
	config, err := f.storageFactory.NewConfig(resource)
	if err != nil {
		glog.Fatalf("Unable to find storage config for %v, due to %v", resource, err.Error())
	}
	return generic.RESTOptions{
		StorageConfig:           config,
		Decorator:               f.storageDecorator,
		DeleteCollectionWorkers: f.deleteCollectionWorkers,
		ResourcePrefix:          f.storageFactory.ResourcePrefix(resource),
	}
}
<commit_msg>Move swagger+openapi setup to routes and decouple from run<commit_after>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package app does all of the work necessary to create a Kubernetes
// APIServer by binding together the API, master and APIServer infrastructure.
// It can be configured and called directly or via the hyperkube cache.
package app

import (
	"strings"
	"time"

	"github.com/golang/glog"
	"github.com/pborman/uuid"
	"github.com/spf13/cobra"
	"github.com/spf13/pflag"

	"k8s.io/kubernetes/federation/cmd/federation-apiserver/app/options"
	"k8s.io/kubernetes/pkg/admission"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/unversioned"
	"k8s.io/kubernetes/pkg/apiserver/authenticator"
	authorizerunion "k8s.io/kubernetes/pkg/auth/authorizer/union"
	"k8s.io/kubernetes/pkg/auth/user"
	"k8s.io/kubernetes/pkg/controller/informers"
	"k8s.io/kubernetes/pkg/generated/openapi"
	"k8s.io/kubernetes/pkg/genericapiserver"
	"k8s.io/kubernetes/pkg/genericapiserver/authorizer"
	genericvalidation "k8s.io/kubernetes/pkg/genericapiserver/validation"
	"k8s.io/kubernetes/pkg/registry/cachesize"
	"k8s.io/kubernetes/pkg/registry/generic"
	"k8s.io/kubernetes/pkg/registry/generic/registry"
	"k8s.io/kubernetes/pkg/routes"
	"k8s.io/kubernetes/pkg/util/wait"
	"k8s.io/kubernetes/pkg/version"
	authenticatorunion "k8s.io/kubernetes/plugin/pkg/auth/authenticator/request/union"
)

// NewAPIServerCommand creates a *cobra.Command object with default parameters
func NewAPIServerCommand() *cobra.Command {
	s := options.NewServerRunOptions()
	s.AddFlags(pflag.CommandLine)
	cmd := &cobra.Command{
		Use: "federation-apiserver",
		Long: `The Kubernetes federation API server validates and configures data
for the api objects which include pods, services, replicationcontrollers, and
others. The API Server services REST operations and provides the frontend to the
cluster's shared state through which all other components interact.`,
		Run: func(cmd *cobra.Command, args []string) {
		},
	}
	return cmd
}

// Run runs the specified APIServer.  This should never exit.
func Run(s *options.ServerRunOptions) error {
	genericvalidation.VerifyEtcdServersList(s.ServerRunOptions)
	genericapiserver.DefaultAndValidateRunOptions(s.ServerRunOptions)
	genericConfig := genericapiserver.NewConfig(). // create the new config
							ApplyOptions(s.ServerRunOptions). // apply the options selected
							Complete()                        // set default values based on the known values

	if err := genericConfig.MaybeGenerateServingCerts(); err != nil {
		glog.Fatalf("Failed to generate service certificate: %v", err)
	}

	// TODO: register cluster federation resources here.
	resourceConfig := genericapiserver.NewResourceConfig()

	if s.StorageConfig.DeserializationCacheSize == 0 {
		// When size of cache is not explicitly set, set it to 50000
		s.StorageConfig.DeserializationCacheSize = 50000
	}
	storageGroupsToEncodingVersion, err := s.StorageGroupsToEncodingVersion()
	if err != nil {
		glog.Fatalf("error generating storage version map: %s", err)
	}
	storageFactory, err := genericapiserver.BuildDefaultStorageFactory(
		s.StorageConfig, s.DefaultStorageMediaType, api.Codecs,
		genericapiserver.NewDefaultResourceEncodingConfig(), storageGroupsToEncodingVersion,
		[]unversioned.GroupVersionResource{}, resourceConfig, s.RuntimeConfig)
	if err != nil {
		glog.Fatalf("error in initializing storage factory: %s", err)
	}

	for _, override := range s.EtcdServersOverrides {
		tokens := strings.Split(override, "#")
		if len(tokens) != 2 {
			glog.Errorf("invalid value of etcd server overrides: %s", override)
			continue
		}

		apiresource := strings.Split(tokens[0], "/")
		if len(apiresource) != 2 {
			glog.Errorf("invalid resource definition: %s", tokens[0])
			continue
		}
		group := apiresource[0]
		resource := apiresource[1]
		groupResource := unversioned.GroupResource{Group: group, Resource: resource}

		servers := strings.Split(tokens[1], ";")
		storageFactory.SetEtcdLocation(groupResource, servers)
	}

	apiAuthenticator, securityDefinitions, err := authenticator.New(authenticator.AuthenticatorConfig{
		Anonymous:         s.AnonymousAuth,
		AnyToken:          s.EnableAnyToken,
		BasicAuthFile:     s.BasicAuthFile,
		ClientCAFile:      s.ClientCAFile,
		TokenAuthFile:     s.TokenAuthFile,
		OIDCIssuerURL:     s.OIDCIssuerURL,
		OIDCClientID:      s.OIDCClientID,
		OIDCCAFile:        s.OIDCCAFile,
		OIDCUsernameClaim: s.OIDCUsernameClaim,
		OIDCGroupsClaim:   s.OIDCGroupsClaim,
		KeystoneURL:       s.KeystoneURL,
	})
	if err != nil {
		glog.Fatalf("Invalid Authentication Config: %v", err)
	}

	privilegedLoopbackToken := uuid.NewRandom().String()
	selfClientConfig, err := s.NewSelfClientConfig(privilegedLoopbackToken)
	if err != nil {
		glog.Fatalf("Failed to create clientset: %v", err)
	}
	client, err := s.NewSelfClient(privilegedLoopbackToken)
	if err != nil {
		glog.Errorf("Failed to create clientset: %v", err)
	}
	sharedInformers := informers.NewSharedInformerFactory(client, 10*time.Minute)

	authorizationConfig := authorizer.AuthorizationConfig{
		PolicyFile:                  s.AuthorizationPolicyFile,
		WebhookConfigFile:           s.AuthorizationWebhookConfigFile,
		WebhookCacheAuthorizedTTL:   s.AuthorizationWebhookCacheAuthorizedTTL,
		WebhookCacheUnauthorizedTTL: s.AuthorizationWebhookCacheUnauthorizedTTL,
		RBACSuperUser:               s.AuthorizationRBACSuperUser,
		InformerFactory:             sharedInformers,
	}
	authorizationModeNames := strings.Split(s.AuthorizationMode, ",")
	apiAuthorizer, err := authorizer.NewAuthorizerFromAuthorizationConfig(authorizationModeNames, authorizationConfig)
	if err != nil {
		glog.Fatalf("Invalid Authorization Config: %v", err)
	}

	admissionControlPluginNames := strings.Split(s.AdmissionControl, ",")

	// TODO(dims): We probably need to add an option "EnableLoopbackToken"
	if apiAuthenticator != nil {
		var uid = uuid.NewRandom().String()
		tokens := make(map[string]*user.DefaultInfo)
		tokens[privilegedLoopbackToken] = &user.DefaultInfo{
			Name:   user.APIServerUser,
			UID:    uid,
			Groups: []string{user.SystemPrivilegedGroup},
		}

		tokenAuthenticator := authenticator.NewAuthenticatorFromTokens(tokens)
		apiAuthenticator = authenticatorunion.New(tokenAuthenticator, apiAuthenticator)

		tokenAuthorizer := authorizer.NewPrivilegedGroups(user.SystemPrivilegedGroup)
		apiAuthorizer = authorizerunion.New(tokenAuthorizer, apiAuthorizer)
	}

	pluginInitializer := admission.NewPluginInitializer(sharedInformers, apiAuthorizer)

	admissionController, err := admission.NewFromPlugins(client, admissionControlPluginNames, s.AdmissionControlConfigFile, pluginInitializer)
	if err != nil {
		glog.Fatalf("Failed to initialize plugins: %v", err)
	}

	kubeVersion := version.Get()
	genericConfig.Version = &kubeVersion
	genericConfig.LoopbackClientConfig = selfClientConfig
	genericConfig.Authenticator = apiAuthenticator
	genericConfig.SupportsBasicAuth = len(s.BasicAuthFile) > 0
	genericConfig.Authorizer = apiAuthorizer
	genericConfig.AuthorizerRBACSuperUser = s.AuthorizationRBACSuperUser
	genericConfig.AdmissionControl = admissionController
	genericConfig.APIResourceConfigSource = storageFactory.APIResourceConfigSource
	genericConfig.MasterServiceNamespace = s.MasterServiceNamespace
	genericConfig.OpenAPIConfig.Definitions = openapi.OpenAPIDefinitions
	genericConfig.EnableOpenAPISupport = true
	genericConfig.OpenAPIConfig.SecurityDefinitions = securityDefinitions

	// TODO: Move this to generic api server (Need to move the command line flag).
	if s.EnableWatchCache {
		cachesize.InitializeWatchCacheSizes(s.TargetRAMMB)
		cachesize.SetWatchCacheSizes(s.WatchCacheSizes)
	}

	m, err := genericConfig.New()
	if err != nil {
		return err
	}

	routes.UIRedirect{}.Install(m.HandlerContainer)
	routes.Logs{}.Install(m.HandlerContainer)

	restOptionsFactory := restOptionsFactory{
		storageFactory:          storageFactory,
		deleteCollectionWorkers: s.DeleteCollectionWorkers,
	}
	if s.EnableWatchCache {
		restOptionsFactory.storageDecorator = registry.StorageWithCacher
	} else {
		restOptionsFactory.storageDecorator = generic.UndecoratedStorage
	}

	installFederationAPIs(m, restOptionsFactory)
	installCoreAPIs(s, m, restOptionsFactory)
	installExtensionsAPIs(m, restOptionsFactory)

	sharedInformers.Start(wait.NeverStop)
	m.PrepareRun().Run()
	return nil
}

type restOptionsFactory struct {
	storageFactory          genericapiserver.StorageFactory
	storageDecorator        generic.StorageDecorator
	deleteCollectionWorkers int
}

func (f restOptionsFactory) NewFor(resource unversioned.GroupResource) generic.RESTOptions {
	config, err := f.storageFactory.NewConfig(resource)
	if err != nil {
		glog.Fatalf("Unable to find storage config for %v, due to %v", resource, err.Error())
	}
	return generic.RESTOptions{
		StorageConfig:           config,
		Decorator:               f.storageDecorator,
		DeleteCollectionWorkers: f.deleteCollectionWorkers,
		ResourcePrefix:          f.storageFactory.ResourcePrefix(resource),
	}
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/tls"
	"encoding/base64"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"net/http"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/grafana/metrictank/api"
	"github.com/grafana/metrictank/conf"
	"github.com/grafana/metrictank/mdata/chunk"
	"github.com/grafana/metrictank/mdata/chunk/archive"
	"github.com/kisielk/whisper-go/whisper"
	"gopkg.in/raintank/schema.v1"
)

var (
	httpEndpoint = flag.String(
		"http-endpoint",
		"http://127.0.0.1:8080/chunks",
		"The http endpoint to send the data to",
	)
	namePrefix = flag.String(
		"name-prefix",
		"",
		"Prefix to prepend before every metric name, should include the '.' if necessary",
	)
	threads = flag.Int(
		"threads",
		10,
		"Number of workers threads to process and convert .wsp files",
	)
	writeUnfinishedChunks = flag.Bool(
		"write-unfinished-chunks",
		false,
		"Defines if chunks that have not completed their chunk span should be written",
	)
	orgId = flag.Int(
		"orgid",
		1,
		"Organization ID the data belongs to ",
	)
	insecureSSL = flag.Bool(
		"insecure-ssl",
		false,
		"Disables ssl certificate verification",
	)
	whisperDirectory = flag.String(
		"whisper-directory",
		"/opt/graphite/storage/whisper",
		"The directory that contains the whisper file structure",
	)
	httpAuth = flag.String(
		"http-auth",
		"",
		"The credentials used to authenticate in the format \"user:password\"",
	)
	dstSchemas = flag.String(
		"dst-schemas",
		"",
		"The filename of the output schemas definition file",
	)
	nameFilterPattern = flag.String(
		"name-filter",
		"",
		"A regex pattern to be applied to all metric names, only matching ones will be imported",
	)
	importUpTo = flag.Uint(
		"import-up-to",
		math.MaxUint32,
		"Only import up to the specified timestamp",
	)
	verbose = flag.Bool(
		"verbose",
		false,
		"More detailed logging",
	)
	schemas        conf.Schemas
	nameFilter     *regexp.Regexp
	processedCount uint32
	skippedCount   uint32
)

func main() {
	var err error
	flag.Parse()
	if *verbose {
		log.SetLevel(log.DebugLevel)
	} else {
		log.SetLevel(log.InfoLevel)
	}

	nameFilter = regexp.MustCompile(*nameFilterPattern)
	schemas, err = conf.ReadSchemas(*dstSchemas)
	if err != nil {
		panic(fmt.Sprintf("Error when parsing schemas file: %q", err))
	}

	fileChan := make(chan string)

	wg := &sync.WaitGroup{}
	wg.Add(*threads)
	for i := 0; i < *threads; i++ {
		go processFromChan(fileChan, wg)
	}

	getFileListIntoChan(fileChan)
	wg.Wait()
}

func processFromChan(files chan string, wg *sync.WaitGroup) {
	tr := &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: *insecureSSL},
	}
	client := &http.Client{Transport: tr}

	for file := range files {
		fd, err := os.Open(file)
		if err != nil {
			log.Errorf("Failed to open whisper file %q: %q\n", file, err)
			continue
		}
		w, err := whisper.OpenWhisper(fd)
		if err != nil {
			log.Errorf("Failed to open whisper file %q: %q\n", file, err)
			continue
		}

		name := getMetricName(file)
		log.Debugf("Processing file %s (%s)", file, name)
		met, err := getMetric(w, file, name)
		if err != nil {
			log.Errorf("Failed to get metric: %q", err)
			continue
		}

		success := false
		attempts := 0
		for !success {
			b, err := met.MarshalCompressed()
			if err != nil {
				log.Errorf("Failed to encode metric: %q", err)
				continue
			}
			size := b.Len()

			req, err := http.NewRequest("POST", *httpEndpoint, io.Reader(b))
			if err != nil {
				log.Fatal(fmt.Sprintf("Cannot construct request to http endpoint %q: %q", *httpEndpoint, err))
			}

			req.Header.Set("Content-Type", "application/json")
			req.Header.Set("Content-Encoding", "gzip")

			if len(*httpAuth) > 0 {
				req.Header.Add("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(*httpAuth)))
			}

			pre := time.Now()
			resp, err := client.Do(req)
			passed := time.Now().Sub(pre).Seconds()
			if err != nil || resp.StatusCode >= 300 {
				if err != nil {
					log.Warningf("Error posting %s (%d bytes), to endpoint %q (attempt %d/%fs, retrying): %s", name, size, *httpEndpoint, attempts, passed, err)
					attempts++
					continue
				} else {
					log.Warningf("Error posting %s (%d bytes) to endpoint %q status %d (attempt %d/%fs, retrying)", name, size, *httpEndpoint, resp.StatusCode, attempts, passed)
				}
				attempts++
			} else {
				log.Debugf("Posted %s (%d bytes) to endpoint %q in %f seconds", name, size, *httpEndpoint, passed)
				success = true
			}
			io.Copy(ioutil.Discard, resp.Body)
			resp.Body.Close()
		}

		processed := atomic.AddUint32(&processedCount, 1)
		if processed%100 == 0 {
			skipped := atomic.LoadUint32(&skippedCount)
			log.Infof("Processed %d files, %d skipped", processed, skipped)
		}
	}
	wg.Done()
}

// generate the metric name based on the file name and given prefix
func getMetricName(file string) string {
	// remove all leading '/' from file name
	for file[0] == '/' {
		file = file[1:]
	}

	return *namePrefix + strings.Replace(strings.TrimSuffix(strings.TrimPrefix(file, *whisperDirectory), ".wsp"), "/", ".", -1)
}

// pointSorter sorts points by timestamp
type pointSorter []whisper.Point

func (a pointSorter) Len() int           { return len(a) }
func (a pointSorter) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a pointSorter) Less(i, j int) bool { return a[i].Timestamp < a[j].Timestamp }

// the whisper archives are organized like a ringbuffer. since we need to
// insert the points into the chunks in order we first need to sort them
func sortPoints(points pointSorter) pointSorter {
	sort.Sort(points)
	return points
}

func shortAggMethodString(aggMethod whisper.AggregationMethod) (string, error) {
	switch aggMethod {
	case whisper.AggregationAverage:
		return "avg", nil
	case whisper.AggregationSum:
		return "sum", nil
	case whisper.AggregationMin:
		return "min", nil
	case whisper.AggregationMax:
		return "max", nil
	case whisper.AggregationLast:
		return "lst", nil
	default:
		return "", fmt.Errorf("Unknown aggregation method %d", aggMethod)
	}
}

func getMetric(w *whisper.Whisper, file, name string) (archive.Metric, error) {
	res := archive.Metric{
		AggregationMethod: uint32(w.Header.Metadata.AggregationMethod),
	}
	if len(w.Header.Archives) == 0 {
		return res, fmt.Errorf("Whisper file contains no archives: %q", file)
	}

	method, err := shortAggMethodString(w.Header.Metadata.AggregationMethod)
	if err != nil {
		return res, err
	}

	md := schema.MetricData{
		Name:     name,
		Metric:   name,
		Interval: int(w.Header.Archives[0].SecondsPerPoint),
		Value:    0,
		Unit:     "unknown",
		Time:     0,
		Mtype:    "gauge",
		Tags:     []string{},
		OrgId:    *orgId,
	}
	md.SetId()
	_, schema := schemas.Match(md.Name, 0)

	points := make(map[int][]whisper.Point)
	for i := range w.Header.Archives {
		p, err := w.DumpArchive(i)
		if err != nil {
			return res, fmt.Errorf("Failed to dump archive %d from whisper file %s", i, file)
		}
		points[i] = p
	}

	conversion := newConversion(w.Header.Archives, points, method)
	for retIdx, retention := range schema.Retentions {
		convertedPoints := conversion.getPoints(retIdx, uint32(retention.SecondsPerPoint), uint32(retention.NumberOfPoints))
		for m, p := range convertedPoints {
			if len(p) == 0 {
				continue
			}
			rowKey := getRowKey(retIdx, md.Id, m, retention.SecondsPerPoint)
			encodedChunks := encodedChunksFromPoints(p, uint32(retention.SecondsPerPoint), retention.ChunkSpan)
			log.Debugf("Archive %d Method %s got %d points = %d chunks at a span of %d", retIdx, m, len(p), len(encodedChunks), retention.ChunkSpan)
			res.Archives = append(res.Archives, archive.Archive{
				SecondsPerPoint: uint32(retention.SecondsPerPoint),
				Points:          uint32(retention.NumberOfPoints),
				Chunks:          encodedChunks,
				RowKey:          rowKey,
			})
			if int64(p[len(p)-1].Timestamp) > md.Time {
				md.Time = int64(p[len(p)-1].Timestamp)
			}
		}
	}
	res.MetricData = md

	return res, nil
}

func getRowKey(retIdx int, id, meth string, secondsPerPoint int) string {
	if retIdx == 0 {
		return id
	} else {
		return api.AggMetricKey(
			id,
			meth,
			uint32(secondsPerPoint),
		)
	}
}

func encodedChunksFromPoints(points []whisper.Point, intervalIn, chunkSpan uint32) []chunk.IterGen {
	var point whisper.Point
	var t0, prevT0 uint32
	var c *chunk.Chunk
	var encodedChunks []chunk.IterGen

	for _, point = range points {
		// this shouldn't happen, but if it would we better catch it here because Metrictank wouldn't handle it well:
		// https://github.com/grafana/metrictank/blob/f1868cccfb92fc82cd853914af958f6d187c5f74/mdata/aggmetric.go#L378
		if point.Timestamp == 0 {
			continue
		}

		t0 = point.Timestamp - (point.Timestamp % chunkSpan)
		if prevT0 == 0 {
			c = chunk.New(t0)
			prevT0 = t0
		} else if prevT0 != t0 {
			c.Finish()

			encodedChunks = append(encodedChunks, *chunk.NewBareIterGen(c.Bytes(), c.T0, chunkSpan))

			c = chunk.New(t0)
			prevT0 = t0
		}

		err := c.Push(point.Timestamp, point.Value)
		if err != nil {
			panic(fmt.Sprintf("ERROR: Failed to push value into chunk at t0 %d: %q", t0, err))
		}
	}

	// if the last written point was also the last one of the current chunk,
	// or if writeUnfinishedChunks is on, we close the chunk and push it
	if point.Timestamp == t0+chunkSpan-intervalIn || *writeUnfinishedChunks {
		c.Finish()
		encodedChunks = append(encodedChunks, *chunk.NewBareIterGen(c.Bytes(), c.T0, chunkSpan))
	}

	return encodedChunks
}

// scan a directory and feed the list of whisper files relative to base into the given channel
func getFileListIntoChan(fileChan chan string) {
	filepath.Walk(
		*whisperDirectory,
		func(path string, info os.FileInfo, err error) error {
			name := getMetricName(path)
			if !nameFilter.Match([]byte(getMetricName(name))) {
				log.Debugf("Skipping file %s with name %s", path, name)
				atomic.AddUint32(&skippedCount, 1)
				return nil
			}
			if len(path) >= 4 && path[len(path)-4:] == ".wsp" {
				fileChan <- path
			}
			return nil
		},
	)

	close(fileChan)
}
<commit_msg>fix leading / bug<commit_after>package main

import (
	"crypto/tls"
	"encoding/base64"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"net/http"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/grafana/metrictank/api"
	"github.com/grafana/metrictank/conf"
	"github.com/grafana/metrictank/mdata/chunk"
	"github.com/grafana/metrictank/mdata/chunk/archive"
	"github.com/kisielk/whisper-go/whisper"
	"gopkg.in/raintank/schema.v1"
)

var (
	httpEndpoint = flag.String(
		"http-endpoint",
		"http://127.0.0.1:8080/chunks",
		"The http endpoint to send the data to",
	)
	namePrefix = flag.String(
		"name-prefix",
		"",
		"Prefix to prepend before every metric name, should include the '.' if necessary",
	)
	threads = flag.Int(
		"threads",
		10,
		"Number of workers threads to process and convert .wsp files",
	)
	writeUnfinishedChunks = flag.Bool(
		"write-unfinished-chunks",
		false,
		"Defines if chunks that have not completed their chunk span should be written",
	)
	orgId = flag.Int(
		"orgid",
		1,
		"Organization ID the data belongs to ",
	)
	insecureSSL = flag.Bool(
		"insecure-ssl",
		false,
		"Disables ssl certificate verification",
	)
	whisperDirectory = flag.String(
		"whisper-directory",
		"/opt/graphite/storage/whisper",
		"The directory that contains the whisper file structure",
	)
	httpAuth = flag.String(
		"http-auth",
		"",
		"The credentials used to authenticate in the format \"user:password\"",
	)
	dstSchemas = flag.String(
		"dst-schemas",
		"",
		"The filename of the output schemas definition file",
	)
	nameFilterPattern = flag.String(
		"name-filter",
		"",
		"A regex pattern to be applied to all metric names, only matching ones will be imported",
	)
	importUpTo = flag.Uint(
		"import-up-to",
		math.MaxUint32,
		"Only import up to the specified timestamp",
	)
	verbose = flag.Bool(
		"verbose",
		false,
		"More detailed logging",
	)
	schemas        conf.Schemas
	nameFilter     *regexp.Regexp
	processedCount uint32
	skippedCount   uint32
)

func main() {
	var err error
	flag.Parse()
	if *verbose {
		log.SetLevel(log.DebugLevel)
	} else {
		log.SetLevel(log.InfoLevel)
	}

	nameFilter = regexp.MustCompile(*nameFilterPattern)
	schemas, err = conf.ReadSchemas(*dstSchemas)
	if err != nil {
		panic(fmt.Sprintf("Error when parsing schemas file: %q", err))
	}

	fileChan := make(chan string)

	wg := &sync.WaitGroup{}
	wg.Add(*threads)
	for i := 0; i < *threads; i++ {
		go processFromChan(fileChan, wg)
	}

	getFileListIntoChan(fileChan)
	wg.Wait()
}

func processFromChan(files chan string, wg *sync.WaitGroup) {
	tr := &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: *insecureSSL},
	}
	client := &http.Client{Transport: tr}

	for file := range files {
		fd, err := os.Open(file)
		if err != nil {
			log.Errorf("Failed to open whisper file %q: %q\n", file, err)
			continue
		}
		w, err := whisper.OpenWhisper(fd)
		if err != nil {
			log.Errorf("Failed to open whisper file %q: %q\n", file, err)
			continue
		}

		name := getMetricName(file)
		log.Debugf("Processing file %s (%s)", file, name)
		met, err := getMetric(w, file, name)
		if err != nil {
			log.Errorf("Failed to get metric: %q", err)
			continue
		}

		success := false
		attempts := 0
		for !success {
			b, err := met.MarshalCompressed()
			if err != nil {
				log.Errorf("Failed to encode metric: %q", err)
				continue
			}
			size := b.Len()

			req, err := http.NewRequest("POST", *httpEndpoint, io.Reader(b))
			if err != nil {
				log.Fatal(fmt.Sprintf("Cannot construct request to http endpoint %q: %q", *httpEndpoint, err))
			}

			req.Header.Set("Content-Type", "application/json")
			req.Header.Set("Content-Encoding", "gzip")

			if len(*httpAuth) > 0 {
				req.Header.Add("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(*httpAuth)))
			}

			pre := time.Now()
			resp, err := client.Do(req)
			passed := time.Now().Sub(pre).Seconds()
			if err != nil || resp.StatusCode >= 300 {
				if err != nil {
					log.Warningf("Error posting %s (%d bytes), to endpoint %q (attempt %d/%fs, retrying): %s", name, size, *httpEndpoint, attempts, passed, err)
					attempts++
					continue
				} else {
					log.Warningf("Error posting %s (%d bytes) to endpoint %q status %d (attempt %d/%fs, retrying)", name, size, *httpEndpoint, resp.StatusCode, attempts, passed)
				}
				attempts++
			} else {
				log.Debugf("Posted %s (%d bytes) to endpoint %q in %f seconds", name, size, *httpEndpoint, passed)
				success = true
			}
			io.Copy(ioutil.Discard, resp.Body)
			resp.Body.Close()
		}

		processed := atomic.AddUint32(&processedCount, 1)
		if processed%100 == 0 {
			skipped := atomic.LoadUint32(&skippedCount)
			log.Infof("Processed %d files, %d skipped", processed, skipped)
		}
	}
	wg.Done()
}

// generate the metric name based on the file name and given prefix
func getMetricName(file string) string {
	// remove all leading '/' from file name
	file = strings.TrimPrefix(file, *whisperDirectory)
	for file[0] == '/' {
		file = file[1:]
	}

	return *namePrefix + strings.Replace(strings.TrimSuffix(file, ".wsp"), "/", ".", -1)
}

// pointSorter sorts points by timestamp
type pointSorter []whisper.Point

func (a pointSorter) Len() int           { return len(a) }
func (a pointSorter) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a pointSorter) Less(i, j int) bool { return a[i].Timestamp < a[j].Timestamp }

// the whisper archives are organized like a ringbuffer. since we need to
// insert the points into the chunks in order we first need to sort them
func sortPoints(points pointSorter) pointSorter {
	sort.Sort(points)
	return points
}

func shortAggMethodString(aggMethod whisper.AggregationMethod) (string, error) {
	switch aggMethod {
	case whisper.AggregationAverage:
		return "avg", nil
	case whisper.AggregationSum:
		return "sum", nil
	case whisper.AggregationMin:
		return "min", nil
	case whisper.AggregationMax:
		return "max", nil
	case whisper.AggregationLast:
		return "lst", nil
	default:
		return "", fmt.Errorf("Unknown aggregation method %d", aggMethod)
	}
}

func getMetric(w *whisper.Whisper, file, name string) (archive.Metric, error) {
	res := archive.Metric{
		AggregationMethod: uint32(w.Header.Metadata.AggregationMethod),
	}
	if len(w.Header.Archives) == 0 {
		return res, fmt.Errorf("Whisper file contains no archives: %q", file)
	}

	method, err := shortAggMethodString(w.Header.Metadata.AggregationMethod)
	if err != nil {
		return res, err
	}

	md := schema.MetricData{
		Name:     name,
		Metric:   name,
		Interval: int(w.Header.Archives[0].SecondsPerPoint),
		Value:    0,
		Unit:     "unknown",
		Time:     0,
		Mtype:    "gauge",
		Tags:     []string{},
		OrgId:    *orgId,
	}
	md.SetId()
	_, schema := schemas.Match(md.Name, 0)

	points := make(map[int][]whisper.Point)
	for i := range w.Header.Archives {
		p, err := w.DumpArchive(i)
		if err != nil {
			return res, fmt.Errorf("Failed to dump archive %d from whisper file %s", i, file)
		}
		points[i] = p
	}

	conversion := newConversion(w.Header.Archives, points, method)
	for retIdx, retention := range schema.Retentions {
		convertedPoints := conversion.getPoints(retIdx, uint32(retention.SecondsPerPoint), uint32(retention.NumberOfPoints))
		for m, p := range convertedPoints {
			if len(p) == 0 {
				continue
			}
			rowKey := getRowKey(retIdx, md.Id, m, retention.SecondsPerPoint)
			encodedChunks := encodedChunksFromPoints(p, uint32(retention.SecondsPerPoint), retention.ChunkSpan)
			log.Debugf("Archive %d Method %s got %d points = %d chunks at a span of %d", retIdx, m, len(p), len(encodedChunks), retention.ChunkSpan)
			res.Archives = append(res.Archives, archive.Archive{
				SecondsPerPoint: uint32(retention.SecondsPerPoint),
				Points:          uint32(retention.NumberOfPoints),
				Chunks:          encodedChunks,
				RowKey:          rowKey,
			})
			if int64(p[len(p)-1].Timestamp) > md.Time {
				md.Time = int64(p[len(p)-1].Timestamp)
			}
		}
	}
	res.MetricData = md

	return res, nil
}

func getRowKey(retIdx int, id, meth string, secondsPerPoint int) string {
	if retIdx == 0 {
		return id
	} else {
		return api.AggMetricKey(
			id,
			meth,
			uint32(secondsPerPoint),
		)
	}
}

func encodedChunksFromPoints(points []whisper.Point, intervalIn, chunkSpan uint32) []chunk.IterGen {
	var point whisper.Point
	var t0, prevT0 uint32
	var c *chunk.Chunk
	var encodedChunks []chunk.IterGen

	for _, point = range points {
		// this shouldn't happen, but if it would we better catch it here because Metrictank wouldn't handle it well:
		// https://github.com/grafana/metrictank/blob/f1868cccfb92fc82cd853914af958f6d187c5f74/mdata/aggmetric.go#L378
		if point.Timestamp == 0 {
			continue
		}

		t0 = point.Timestamp - (point.Timestamp % chunkSpan)
		if prevT0 == 0 {
			c = chunk.New(t0)
			prevT0 = t0
		} else if prevT0 != t0 {
			c.Finish()

			encodedChunks = append(encodedChunks, *chunk.NewBareIterGen(c.Bytes(), c.T0, chunkSpan))

			c = chunk.New(t0)
			prevT0 = t0
		}

		err := c.Push(point.Timestamp, point.Value)
		if err != nil {
			panic(fmt.Sprintf("ERROR: Failed to push value into chunk at t0 %d: %q", t0, err))
		}
	}

	// if the last written point was also the last one of the current chunk,
	// or if writeUnfinishedChunks is on, we close the chunk and push it
	if point.Timestamp == t0+chunkSpan-intervalIn || *writeUnfinishedChunks {
		c.Finish()
		encodedChunks = append(encodedChunks, *chunk.NewBareIterGen(c.Bytes(), c.T0, chunkSpan))
	}

	return encodedChunks
}

// scan a directory and feed the list of whisper files relative to base into the given channel
func getFileListIntoChan(fileChan chan string) {
	filepath.Walk(
		*whisperDirectory,
		func(path string, info os.FileInfo, err error) error {
			name := getMetricName(path)
			if !nameFilter.Match([]byte(getMetricName(name))) {
				log.Debugf("Skipping file %s with name %s", path, name)
				atomic.AddUint32(&skippedCount, 1)
				return nil
			}
			if len(path) >= 4 && path[len(path)-4:] == ".wsp" {
				fileChan <- path
			}
			return nil
		},
	)

	close(fileChan)
}
<|endoftext|>


<commit_before>package cmdline

import "fmt"

// java [ options ] class [ arguments ]
type Command struct {
    options []*Option
    class   string
    args    []string
}

type Option struct {

}

func ParseCommand(cmdLineArgs []string) {
    // todo
    for idx, arg := range cmdLineArgs {
        fmt.Printf("idx: %v arg:%v \n", idx, arg)
    }
}<commit_msg>parse command...<commit_after>package cmdline

//import "fmt"

// java [ options ] class [ arguments ]
type Command struct {
    options []*Option
    class   string
    args    []string
}

type Option struct {
    name    string
    value   string
}

func (self *Command) parseOptions() {

}

func (self *Command) parseClass() {

}

func (self *Command) parseArgs() {

}

func ParseCommand(cmdLineArgs []string) {
    // todo
    cmd := &Command{}
    cmd.options = []*Option{} // len == 0
    cmd.parseOptions()
    cmd.parseClass()
    cmd.parseArgs()
}
<|endoftext|>


<commit_before><commit_msg>"Jeff: Give yourself to the Dark Side. It is the only way you can save your friends."<commit_after><|endoftext|>


<commit_before><commit_msg>"Jeff: Come to the dark side... together we will rule the galaxy."<commit_after><|endoftext|>


<commit_before>// +build linux
// Network utility functions.

package netutils

import (
	"fmt"
	"net"
	"strings"

	"github.com/docker/libnetwork/ipamutils"
	"github.com/docker/libnetwork/ns"
	"github.com/docker/libnetwork/osl"
	"github.com/docker/libnetwork/resolvconf"
	"github.com/docker/libnetwork/types"
	"github.com/vishvananda/netlink"
)

var (
	networkGetRoutesFct func(netlink.Link, int) ([]netlink.Route, error)
)

// CheckRouteOverlaps checks whether the passed network overlaps with any existing routes
func CheckRouteOverlaps(toCheck *net.IPNet) error {
	if networkGetRoutesFct == nil {
		networkGetRoutesFct = ns.NlHandle().RouteList
	}
	networks, err := networkGetRoutesFct(nil, netlink.FAMILY_V4)
	if err != nil {
		return err
	}
	for _, network := range networks {
		if network.Dst != nil && NetworkOverlaps(toCheck, network.Dst) {
			return ErrNetworkOverlaps
		}
	}
	return nil
}

// GenerateIfaceName returns an interface name using the passed in
// prefix and the length of random bytes. The api ensures that the
// there are is no interface which exists with that name.
func GenerateIfaceName(nlh *netlink.Handle, prefix string, len int) (string, error) {
	linkByName := netlink.LinkByName
	if nlh != nil {
		linkByName = nlh.LinkByName
	}
	for i := 0; i < 3; i++ {
		name, err := GenerateRandomName(prefix, len)
		if err != nil {
			continue
		}
		_, err = linkByName(name)
		if err != nil {
			if strings.Contains(err.Error(), "not found") {
				return name, nil
			}
			return "", err
		}
	}
	return "", types.InternalErrorf("could not generate interface name")
}

// ElectInterfaceAddresses looks for an interface on the OS with the
// specified name and returns returns all its IPv4 and IPv6 addresses in CIDR notation.
// If a failure in retrieving the addresses or no IPv4 address is found, an error is returned.
// If the interface does not exist, it chooses from a predefined
// list the first IPv4 address which does not conflict with other
// interfaces on the system.
func ElectInterfaceAddresses(name string) ([]*net.IPNet, []*net.IPNet, error) {
	var (
		v4Nets []*net.IPNet
		v6Nets []*net.IPNet
	)

	defer osl.InitOSContext()()

	link, _ := ns.NlHandle().LinkByName(name)
	if link != nil {
		v4addr, err := ns.NlHandle().AddrList(link, netlink.FAMILY_V4)
		if err != nil {
			return nil, nil, err
		}
		v6addr, err := ns.NlHandle().AddrList(link, netlink.FAMILY_V6)
		if err != nil {
			return nil, nil, err
		}
		for _, nlAddr := range v4addr {
			v4Nets = append(v4Nets, nlAddr.IPNet)
		}
		for _, nlAddr := range v6addr {
			v6Nets = append(v6Nets, nlAddr.IPNet)
		}
	}

	if link == nil || len(v4Nets) == 0 {
		// Choose from predefined local scope  networks
		v4Net, err := FindAvailableNetwork(ipamutils.PredefinedLocalScopeDefaultNetworks)
		if err != nil {
			return nil, nil, err
		}
		v4Nets = append(v4Nets, v4Net)
	}

	return v4Nets, v6Nets, nil
}

// FindAvailableNetwork returns a network from the passed list which does not
// overlap with existing interfaces in the system
func FindAvailableNetwork(list []*net.IPNet) (*net.IPNet, error) {
	// We don't check for an error here, because we don't really care if we
	// can't read /etc/resolv.conf. So instead we skip the append if resolvConf
	// is nil. It either doesn't exist, or we can't read it for some reason.
	var nameservers []string
	if rc, err := resolvconf.Get(); err == nil {
		nameservers = resolvconf.GetNameserversAsCIDR(rc.Content)
	}
	for _, nw := range list {
		if err := CheckNameserverOverlaps(nameservers, nw); err == nil {
			if err := CheckRouteOverlaps(nw); err == nil {
				return nw, nil
			}
		}
	}
	return nil, fmt.Errorf("no available network")
}
<commit_msg>Improve error if auto-selecting IP-range failed.<commit_after>// +build linux
// Network utility functions.

package netutils

import (
	"fmt"
	"net"
	"strings"

	"github.com/docker/libnetwork/ipamutils"
	"github.com/docker/libnetwork/ns"
	"github.com/docker/libnetwork/osl"
	"github.com/docker/libnetwork/resolvconf"
	"github.com/docker/libnetwork/types"
	"github.com/vishvananda/netlink"
)

var (
	networkGetRoutesFct func(netlink.Link, int) ([]netlink.Route, error)
)

// CheckRouteOverlaps checks whether the passed network overlaps with any existing routes
func CheckRouteOverlaps(toCheck *net.IPNet) error {
	if networkGetRoutesFct == nil {
		networkGetRoutesFct = ns.NlHandle().RouteList
	}
	networks, err := networkGetRoutesFct(nil, netlink.FAMILY_V4)
	if err != nil {
		return err
	}
	for _, network := range networks {
		if network.Dst != nil && NetworkOverlaps(toCheck, network.Dst) {
			return ErrNetworkOverlaps
		}
	}
	return nil
}

// GenerateIfaceName returns an interface name using the passed in
// prefix and the length of random bytes. The api ensures that the
// there are is no interface which exists with that name.
func GenerateIfaceName(nlh *netlink.Handle, prefix string, len int) (string, error) {
	linkByName := netlink.LinkByName
	if nlh != nil {
		linkByName = nlh.LinkByName
	}
	for i := 0; i < 3; i++ {
		name, err := GenerateRandomName(prefix, len)
		if err != nil {
			continue
		}
		_, err = linkByName(name)
		if err != nil {
			if strings.Contains(err.Error(), "not found") {
				return name, nil
			}
			return "", err
		}
	}
	return "", types.InternalErrorf("could not generate interface name")
}

// ElectInterfaceAddresses looks for an interface on the OS with the
// specified name and returns returns all its IPv4 and IPv6 addresses in CIDR notation.
// If a failure in retrieving the addresses or no IPv4 address is found, an error is returned.
// If the interface does not exist, it chooses from a predefined
// list the first IPv4 address which does not conflict with other
// interfaces on the system.
func ElectInterfaceAddresses(name string) ([]*net.IPNet, []*net.IPNet, error) {
	var (
		v4Nets []*net.IPNet
		v6Nets []*net.IPNet
	)

	defer osl.InitOSContext()()

	link, _ := ns.NlHandle().LinkByName(name)
	if link != nil {
		v4addr, err := ns.NlHandle().AddrList(link, netlink.FAMILY_V4)
		if err != nil {
			return nil, nil, err
		}
		v6addr, err := ns.NlHandle().AddrList(link, netlink.FAMILY_V6)
		if err != nil {
			return nil, nil, err
		}
		for _, nlAddr := range v4addr {
			v4Nets = append(v4Nets, nlAddr.IPNet)
		}
		for _, nlAddr := range v6addr {
			v6Nets = append(v6Nets, nlAddr.IPNet)
		}
	}

	if link == nil || len(v4Nets) == 0 {
		// Choose from predefined local scope networks
		v4Net, err := FindAvailableNetwork(ipamutils.PredefinedLocalScopeDefaultNetworks)
		if err != nil {
			return nil, nil, fmt.Errorf("%s, PredefinedLocalScopeDefaultNetworks List: %+v",
				err.Error(),
				ipamutils.PredefinedLocalScopeDefaultNetworks)
		}
		v4Nets = append(v4Nets, v4Net)
	}

	return v4Nets, v6Nets, nil
}

// FindAvailableNetwork returns a network from the passed list which does not
// overlap with existing interfaces in the system
func FindAvailableNetwork(list []*net.IPNet) (*net.IPNet, error) {
	// We don't check for an error here, because we don't really care if we
	// can't read /etc/resolv.conf. So instead we skip the append if resolvConf
	// is nil. It either doesn't exist, or we can't read it for some reason.
	var nameservers []string
	if rc, err := resolvconf.Get(); err == nil {
		nameservers = resolvconf.GetNameserversAsCIDR(rc.Content)
	}
	for _, nw := range list {
		if err := CheckNameserverOverlaps(nameservers, nw); err == nil {
			if err := CheckRouteOverlaps(nw); err == nil {
				return nw, nil
			}
		}
	}
	return nil, fmt.Errorf("no available network")
}
<|endoftext|>


<commit_before><commit_msg>Implement Store and Retrieve default methods<commit_after><|endoftext|>