<commit_before>package main

import (
	"bufio"
	"fmt"
	"math/rand"
	"os"
	"regexp"
	"strings"
)

var thesaurus map[string][]string
var cmudict map[string]int
var skipwords []string
var err error

func loadCmudict(path string) (map[string]int, error) {

	m := make(map[string]int, 140000)

	f, err := os.Open(path)
	if err != nil {
		return nil, err
	}

	scan := bufio.NewScanner(f)

	for scan.Scan() {
		s := scan.Text()
		s = strings.ToLower(s)
		if s[0] == ';' {
			// skip comments
			continue
		}

		// find first word
		idx := strings.Index(s, " ")
		w := s[0:idx]

		if w[idx-1] == ')' {
			w = w[:idx-3]
		}

		c := 0
		// count syllables == digits in remaining string
		for _, r := range s[idx:] {
			if r >= '0' && r <= '9' {
				c++
			}
		}

		m[w] = c
	}

	if err := scan.Err(); err != nil {
		return nil, err
	}

	return m, nil
}

func loadThesaurus(path string) (map[string][]string, error) {

	m := make(map[string][]string)

	f, err := os.Open(path)
	if err != nil {
		return nil, err
	}

	scan := bufio.NewScanner(f)

	for scan.Scan() {
		s := scan.Text()
		words := strings.Split(s, "|")
		if len(words) <= 2 {
			// skip if doesn't have syllables
			continue
		}

		for i, w := range words {
			if i > 1 {
				m[words[1]] = append(m[words[1]], w)
			}
		}
	}

	if err := scan.Err(); err != nil {
		return nil, err
	}

	return m, nil

}

func getSynonyms(w string) (possibilities []string) {
	// returns a list of synonyms that have a unique number of syllables
	possibilities = append(possibilities, w)

	for _, skipword := range skipwords {
		if w == skipword {
			return
		}
	}

	syllablesAccountedFor := make(map[int]string)
	for _, synonym := range thesaurus[w] {
		sylbls := cmudict[synonym]
		if sylbls > 0 {
			if _, ok := syllablesAccountedFor[sylbls]; ok {
				// pass
			} else {
				syllablesAccountedFor[sylbls] = synonym
			}
		}
	}

	for _, word := range syllablesAccountedFor {
		possibilities = append(possibilities, word)
	}
	return
}

type haikuCluster struct {
	haikus    []string
	isHaikus  []bool
	numHaikus int
}

func getHaikus(words []string) (haikus []string, isHaikus []bool, numHaikus int) {
	numHaikus = 0
	for {
		haikuString, i, gotHaiku := isHaiku(words)
		haikus = append(haikus, haikuString)
		isHaikus = append(isHaikus, gotHaiku)
		if gotHaiku == false {
			return
		} else {
			numHaikus = numHaikus + 1
		}

		words = words[i+1:]
	}
}

func isHaiku(words []string) (string, int, bool) {
	checks := []int{5, 7, 5}
	curCheck := 0
	runningTotal := 0
	currentHaiku := ""
	for i, word := range words {
		slbles := cmudict[word]
		runningTotal = runningTotal + slbles
		currentHaiku = currentHaiku + word + " "
		if runningTotal == checks[curCheck] {
			curCheck += 1
			if curCheck == 3 {
				return currentHaiku, i, true
			} else {
				currentHaiku = currentHaiku + "\n"

			}
			runningTotal = 0
		}
	}
	return currentHaiku, -1, false
}

func randChoices(limits []int) (choices []int) {
	for _, l := range limits {
		choices = append(choices, rand.Intn(l))
	}
	return
}

func listAlternates(input []string) (output [10000][]string) {
	totals := make([]int, len(input))
	for i, w := range input {
		totals[i] = len(getSynonyms(w))
	}
	for i := 0; i < len(output); i++ {
		choices := randChoices(totals)
		for j, w := range input {
			output[i] = append(output[i], getSynonyms(w)[choices[j]])
		}
	}
	return
}

func init() {
	// initialize the thesaurs and the syllable dictionary

	thesaurus, err = loadThesaurus("./resources/th_en_US_new.dat")
	if err != nil {
		panic(err)
	}
	cmudict, err = loadCmudict("./resources/cmudict.0.7a")
	if err != nil {
		panic(err)
	}
	skipwords = []string{"a", "about", "above", "above", "across", "after", "afterwards", "again", "against", "all", "almost", "alone", "along", "already", "also", "although", "always", "am", "among", "amongst", "amoungst", "amount", "an", "and", "another", "any", "anyhow", "anyone", "anything", "anyway", "anywhere", "are", "around", "as", "at", "back", "be", "became", "because", "become", "becomes", "becoming", "been", "before", "beforehand", "behind", "being", "below", "beside", "besides", "between", "beyond", "bill", "both", "bottom", "but", "by", "call", "can", "cannot", "cant", "co", "con", "could", "couldnt", "cry", "de", "describe", "detail", "do", "done", "down", "due", "during", "each", "eg", "eight", "either", "eleven", "else", "elsewhere", "empty", "enough", "etc", "even", "ever", "every", "everyone", "everything", "everywhere", "except", "few", "fifteen", "fify", "fill", "find", "fire", "first", "five", "for", "former", "formerly", "forty", "found", "four", "from", "front", "full", "further", "get", "give", "go", "had", "has", "hasnt", "have", "he", "hence", "her", "here", "hereafter", "hereby", "herein", "hereupon", "hers", "herself", "him", "himself", "his", "how", "however", "hundred", "ie", "if", "in", "inc", "indeed", "interest", "into", "is", "it", "its", "itself", "keep", "last", "latter", "latterly", "least", "less", "ltd", "made", "many", "may", "me", "meanwhile", "might", "mill", "mine", "more", "moreover", "most", "mostly", "move", "much", "must", "my", "myself", "name", "namely", "neither", "never", "nevertheless", "next", "nine", "no", "nobody", "none", "noone", "nor", "not", "nothing", "now", "nowhere", "of", "off", "often", "on", "once", "one", "only", "onto", "or", "other", "others", "otherwise", "our", "ours", "ourselves", "out", "over", "own", "part", "per", "perhaps", "please", "put", "rather", "re", "same", "see", "seem", "seemed", "seeming", "seems", "serious", "several", "she", "should", "show", "side", "since", "sincere", "six", "sixty", "so", "some", "somehow", "someone", "something", "sometime", "sometimes", "somewhere", "still", "such", "system", "take", "ten", "than", "that", "the", "their", "them", "themselves", "then", "thence", "there", "thereafter", "thereby", "therefore", "therein", "thereupon", "these", "they", "thickv", "thin", "third", "this", "those", "though", "three", "through", "throughout", "thru", "thus", "to", "together", "too", "top", "toward", "towards", "twelve", "twenty", "two", "un", "under", "until", "up", "upon", "us", "very", "via", "was", "we", "well", "were", "what", "whatever", "when", "whence", "whenever", "where", "whereafter", "whereas", "whereby", "wherein", "whereupon", "wherever", "whether", "which", "while", "whither", "who", "whoever", "whole", "whom", "whose", "why", "will", "with", "within", "without", "would", "yet", "you", "your", "yours", "yourself", "yourselves", "the"}

}

func main() {
	// words := strings.Split(`want to play a game with seventeen syllables we write some poem want to play a game with seventeen syllables we write some poem something else`, " ")
	// fmt.Println(getHaikus(words))
	//input := []string{"cat", "is", "nice"}
	sentence := `This is the sixth time we have had the pleasure of writing a birthday blog post for Go, and we would not be doing so if not for the wonderful and passionate people in our community. The Go team would like to thank everyone who has contributed code, written an open source library, authored a blog post, helped a new gopher, or just given Go a try. `
	fmt.Println("ORIGINAL:")
	fmt.Println(sentence)
	sentence = strings.ToLower(sentence)
	sentence = strings.Replace(sentence, "don't", "do not", -1)
	sentence = strings.Replace(sentence, "'", "", -1)
	words := regexp.MustCompile(`(\w+)`).FindAllString(sentence, -1)
	alternatives := listAlternates(words)
	bestNum := 0
	var bestHaikus []string
	var bestIsHaikus []bool

	for i := 0; i < len(alternatives); i++ {
		for j := 0; j < 3; j++ {
			haikuString, isHaikus, numHaikus := getHaikus(alternatives[i][j:])
			if numHaikus > bestNum {
				if j > 0 {
					haikuString = append([]string{strings.Join(alternatives[i][:j], " ")}, haikuString...)
					isHaikus = append([]bool{false}, isHaikus...)
				}
				bestHaikus = haikuString
				bestIsHaikus = isHaikus
			}
		}
	}
	fmt.Println("\n\nBEST HAIKU:")
	for i, bestHaiku := range bestHaikus {
		if bestIsHaikus[i] == true {
			fmt.Printf("\n")
		}
		fmt.Println(bestHaiku)
		if bestIsHaikus[i] == true {
			fmt.Printf("\n")
		}
	}

	// Todo: replace each word with the puncuation near the word in the original

}
<commit_msg>Better thesaurus<commit_after>package main

import (
	"bufio"
	"fmt"
	"math/rand"
	"os"
	"regexp"
	"strings"
)

var thesaurus map[string][]string
var cmudict map[string]int
var skipwords []string
var err error

func loadCmudict(path string) (map[string]int, error) {

	m := make(map[string]int, 140000)

	f, err := os.Open(path)
	if err != nil {
		return nil, err
	}

	scan := bufio.NewScanner(f)

	for scan.Scan() {
		s := scan.Text()
		s = strings.ToLower(s)
		if s[0] == ';' {
			// skip comments
			continue
		}

		// find first word
		idx := strings.Index(s, " ")
		w := s[0:idx]

		if w[idx-1] == ')' {
			w = w[:idx-3]
		}

		c := 0
		// count syllables == digits in remaining string
		for _, r := range s[idx:] {
			if r >= '0' && r <= '9' {
				c++
			}
		}

		m[w] = c
	}

	if err := scan.Err(); err != nil {
		return nil, err
	}

	return m, nil
}

func stringInSlice(a string, list []string) bool {
	for _, b := range list {
		if b == a {
			return true
		}
	}
	return false
}

func loadThesaurus(path string) (map[string][]string, error) {

	m := make(map[string][]string)

	f, err := os.Open(path)
	if err != nil {
		return nil, err
	}

	scan := bufio.NewScanner(f)

	for scan.Scan() {
		s := scan.Text()
		words := strings.Split(s, "|")
		if len(words) <= 2 {
			// skip if doesn't have syllables
			continue
		}
		for j, _ := range words {
			for i, w := range words {
				if j > 0 && i > 0 && i != j {
					if stringInSlice(w, m[words[j]]) == false {
						m[words[j]] = append(m[words[j]], w)
					}
				}
			}
		}
	}

	if err := scan.Err(); err != nil {
		return nil, err
	}

	return m, nil

}

func getSynonyms(w string) (possibilities []string) {
	// returns a list of synonyms that have a unique number of syllables
	possibilities = append(possibilities, w)

	for _, skipword := range skipwords {
		if w == skipword {
			return
		}
	}

	syllablesAccountedFor := make(map[int]string)
	for _, synonym := range thesaurus[w] {
		sylbls := cmudict[synonym]
		if sylbls > 0 {
			if _, ok := syllablesAccountedFor[sylbls]; ok {
				// pass
			} else {
				syllablesAccountedFor[sylbls] = synonym
			}
		}
	}

	for _, word := range syllablesAccountedFor {
		possibilities = append(possibilities, word)
	}
	return
}

type haikuCluster struct {
	haikus    []string
	isHaikus  []bool
	numHaikus int
}

func getHaikus(words []string) (haikus []string, isHaikus []bool, numHaikus int) {
	numHaikus = 0
	for {
		haikuString, i, gotHaiku := isHaiku(words)
		haikus = append(haikus, haikuString)
		isHaikus = append(isHaikus, gotHaiku)
		if gotHaiku == false {
			return
		} else {
			numHaikus = numHaikus + 1
		}

		words = words[i+1:]
	}
}

func isHaiku(words []string) (string, int, bool) {
	checks := []int{5, 7, 5}
	curCheck := 0
	runningTotal := 0
	currentHaiku := ""
	for i, word := range words {
		slbles := cmudict[word]
		runningTotal = runningTotal + slbles
		currentHaiku = currentHaiku + word + " "
		if runningTotal == checks[curCheck] {
			curCheck += 1
			if curCheck == 3 {
				return currentHaiku, i, true
			} else {
				currentHaiku = currentHaiku + "\n"

			}
			runningTotal = 0
		}
	}
	return currentHaiku, -1, false
}

func randChoices(limits []int) (choices []int) {
	for _, l := range limits {
		choices = append(choices, rand.Intn(l))
	}
	return
}

func listAlternates(input []string) (output [100][]string) {
	totals := make([]int, len(input))
	for i, w := range input {
		totals[i] = len(getSynonyms(w))
	}
	for i := 0; i < len(output); i++ {
		choices := randChoices(totals)
		for j, w := range input {
			output[i] = append(output[i], getSynonyms(w)[choices[j]])
		}
	}
	return
}

func init() {
	// initialize the thesaurs and the syllable dictionary

	thesaurus, err = loadThesaurus("./resources/th_en_US_new.dat")
	if err != nil {
		panic(err)
	}
	fmt.Println(thesaurus["wonderful"])
	cmudict, err = loadCmudict("./resources/cmudict.0.7a")
	if err != nil {
		panic(err)
	}
	skipwords = []string{"a", "about", "above", "above", "across", "after", "afterwards", "again", "against", "all", "almost", "alone", "along", "already", "also", "although", "always", "am", "among", "amongst", "amoungst", "amount", "an", "and", "another", "any", "anyhow", "anyone", "anything", "anyway", "anywhere", "are", "around", "as", "at", "back", "be", "became", "because", "become", "becomes", "becoming", "been", "before", "beforehand", "behind", "being", "below", "beside", "besides", "between", "beyond", "bill", "both", "bottom", "but", "by", "call", "can", "cannot", "cant", "co", "con", "could", "couldnt", "cry", "de", "describe", "detail", "do", "done", "down", "due", "during", "each", "eg", "eight", "either", "eleven", "else", "elsewhere", "empty", "enough", "etc", "even", "ever", "every", "everyone", "everything", "everywhere", "except", "few", "fifteen", "fify", "fill", "find", "fire", "first", "five", "for", "former", "formerly", "forty", "found", "four", "from", "front", "full", "further", "get", "give", "go", "had", "has", "hasnt", "have", "he", "hence", "her", "here", "hereafter", "hereby", "herein", "hereupon", "hers", "herself", "him", "himself", "his", "how", "however", "hundred", "ie", "if", "in", "inc", "indeed", "interest", "into", "is", "it", "its", "itself", "keep", "last", "latter", "latterly", "least", "less", "ltd", "made", "many", "may", "me", "meanwhile", "might", "mill", "mine", "more", "moreover", "most", "mostly", "move", "much", "must", "my", "myself", "name", "namely", "neither", "never", "nevertheless", "next", "nine", "no", "nobody", "none", "noone", "nor", "not", "nothing", "now", "nowhere", "of", "off", "often", "on", "once", "one", "only", "onto", "or", "other", "others", "otherwise", "our", "ours", "ourselves", "out", "over", "own", "part", "per", "perhaps", "please", "put", "rather", "re", "same", "see", "seem", "seemed", "seeming", "seems", "serious", "several", "she", "should", "show", "side", "since", "sincere", "six", "sixty", "so", "some", "somehow", "someone", "something", "sometime", "sometimes", "somewhere", "still", "such", "system", "take", "ten", "than", "that", "the", "their", "them", "themselves", "then", "thence", "there", "thereafter", "thereby", "therefore", "therein", "thereupon", "these", "they", "thickv", "thin", "third", "this", "those", "though", "three", "through", "throughout", "thru", "thus", "to", "together", "too", "top", "toward", "towards", "twelve", "twenty", "two", "un", "under", "until", "up", "upon", "us", "very", "via", "was", "we", "well", "were", "what", "whatever", "when", "whence", "whenever", "where", "whereafter", "whereas", "whereby", "wherein", "whereupon", "wherever", "whether", "which", "while", "whither", "who", "whoever", "whole", "whom", "whose", "why", "will", "with", "within", "without", "would", "yet", "you", "your", "yours", "yourself", "yourselves", "the"}

}

func main() {
	// words := strings.Split(`want to play a game with seventeen syllables we write some poem want to play a game with seventeen syllables we write some poem something else`, " ")
	// fmt.Println(getHaikus(words))
	//input := []string{"cat", "is", "nice"}
	sentence := `This is the sixth time we have had the pleasure of writing a birthday blog post for Go, and we would not be doing so if not for the wonderful and passionate people in our community. The Go team would like to thank everyone who has contributed code, written an open source library, authored a blog post, helped a new gopher, or just given Go a try. `
	fmt.Println("ORIGINAL:")
	fmt.Println(sentence)
	sentence = strings.ToLower(sentence)
	sentence = strings.Replace(sentence, "don't", "do not", -1)
	sentence = strings.Replace(sentence, "'", "", -1)
	words := regexp.MustCompile(`(\w+)`).FindAllString(sentence, -1)
	alternatives := listAlternates(words)
	bestNum := 0
	var bestHaikus []string
	var bestIsHaikus []bool

	for i := 0; i < len(alternatives); i++ {
		for j := 0; j < 3; j++ {
			haikuString, isHaikus, numHaikus := getHaikus(alternatives[i][j:])
			if numHaikus > bestNum {
				if j > 0 {
					haikuString = append([]string{strings.Join(alternatives[i][:j], " ")}, haikuString...)
					isHaikus = append([]bool{false}, isHaikus...)
				}
				bestHaikus = haikuString
				bestIsHaikus = isHaikus
			}
		}
	}
	fmt.Println("\n\nBEST HAIKU:")
	for i, bestHaiku := range bestHaikus {
		if bestIsHaikus[i] == true {
			fmt.Printf("\n")
		}
		fmt.Println(bestHaiku)
		if bestIsHaikus[i] == true {
			fmt.Printf("\n")
		}
	}

	// Todo: replace each word with the puncuation near the word in the original

}
<|endoftext|>


<commit_before>package lazyexp

import (
	"context"
	"sync"
	"sync/atomic"
)

// A Node represents a blocking calculation. Create one using NewNode().
//
// You'll probably want to embed a Node in your struct that will contain the result.
type Node interface {
	Fetch(context.Context) error
	noUserImplementations()
}

// A Dependency is a Node whose result another Node depends on.
//
// Created using ContinueOnError(), CancelOnError(), CancelOnCompletion() and AbortOnError().
type Dependency struct {
	node         Node
	onCompletion completionStrategy
}

// ContinueOnError returns a Dependency where any Node Fetch() errors are simply passed on without affecting any other Nodes.
func ContinueOnError(node Node) Dependency {
	return Dependency{
		node:         node,
		onCompletion: onErrorContinue,
	}
}

// CancelOnError returns a Dependency where any Node Fetch() error causes sibling Dependencies' Fetches to be canceled.
func CancelOnError(node Node) Dependency {
	return Dependency{
		node:         node,
		onCompletion: onErrorCancel,
	}
}

// CancelOnCompletion returns a Dependency that upon Node Fetch() completion causes sibling Dependencies' Fetches to be canceled.
func CancelOnCompletion(node Node) Dependency {
	return Dependency{
		node:         node,
		onCompletion: onCompletionCancel,
	}
}

// AbortOnError returns a Dependency where any Node Fetch() error causes sibling Dependencies' Fetches to be canceled and propagates the error.
//
// TODO: enrich the error with context before passing it on once Nodes have a description
func AbortOnError(node Node) Dependency {
	return Dependency{
		node:         node,
		onCompletion: onErrorAbort,
	}
}

// Dependencies are Nodes that must be Fetched before the given one can be.
type Dependencies []Dependency

// NewNode returns a Node backed by the given function.
//
// The fetch function must not be nil, unless you never intend to Fetch() this Node. It will be called with the errors from the optional dependencies, unless they are fatal, or context.Canceled if they returned CancelFetchSuccess().
//
// The dependencies will be fetched in parallel before fetch() is called and must not contain zero values.
func NewNode(dependencies Dependencies, fetch func(context.Context, []error) error) Node {
	return &node{
		fetcher:      fetch,
		dependencies: dependencies,
	}
}

type node struct {
	fetcher      func(context.Context, []error) error
	dependencies Dependencies
	once         sync.Once
	err          error
}

func (n *node) Fetch(ctx context.Context) error {
	n.once.Do(func() {
		var errs []error
		// fetch dependencies in parallel
		if l := len(n.dependencies); l > 0 {
			errs = make([]error, l)
			if l == 1 {
				// no need to fetch single dependency in parallel
				err := n.dependencies[0].node.Fetch(ctx)
				switch n.dependencies[0].onCompletion {
				case onErrorAbort:
					if err != nil {
						n.err = err
						return
					}
				case onErrorCancel:
					fallthrough
				case onCompletionCancel:
					// no siblings to cancel
					errs[0] = err
				case onErrorContinue:
					errs[0] = err
				}
			} else {
				// we can save one goroutine by fetching that dependency on the current one
				var wg sync.WaitGroup
				var fatalErr atomic.Value
				subCtx, cancel := context.WithCancel(ctx)
				wg.Add(l - 1)
				for i := 1; i < l; i++ {
					go func(i int) {
						errs[i] = fetchDependency(subCtx, cancel, n.dependencies[i], &fatalErr)
						wg.Done()
					}(i)
				}
				errs[0] = fetchDependency(subCtx, cancel, n.dependencies[0], &fatalErr)
				wg.Wait()
				cancel()
				if iErr := fatalErr.Load(); iErr != nil {
					n.err = iErr.(error)
					return
				}
			}

		}
		n.err = n.fetcher(ctx, errs)
	})
	return n.err
}

func (n *node) noUserImplementations() {}

func fetchDependency(ctx context.Context, cancel func(), dependency Dependency, outFatalErr *atomic.Value) error {
	err := dependency.node.Fetch(ctx)
	switch dependency.onCompletion {
	case onErrorCancel:
		if err == nil {
			break
		}
		fallthrough
	case onCompletionCancel:
		cancel()
	case onErrorAbort:
		if err != nil {
			cancel()
			outFatalErr.Store(err)
		}
	case onErrorContinue:
	}
	return err
}

type completionStrategy int

const (
	onErrorContinue completionStrategy = iota
	onErrorCancel
	onCompletionCancel
	onErrorAbort
)
<commit_msg>note on circular dependencies<commit_after>package lazyexp

import (
	"context"
	"sync"
	"sync/atomic"
)

// A Node represents a blocking calculation. Create one using NewNode().
//
// You'll probably want to embed a Node in your struct that will contain the result.
type Node interface {
	Fetch(context.Context) error
	noUserImplementations()
}

// A Dependency is a Node whose result another Node depends on.
//
// Created using ContinueOnError(), CancelOnError(), CancelOnCompletion() and AbortOnError().
type Dependency struct {
	node         Node
	onCompletion completionStrategy
}

// ContinueOnError returns a Dependency where any Node Fetch() errors are simply passed on without affecting any other Nodes.
func ContinueOnError(node Node) Dependency {
	return Dependency{
		node:         node,
		onCompletion: onErrorContinue,
	}
}

// CancelOnError returns a Dependency where any Node Fetch() error causes sibling Dependencies' Fetches to be canceled.
func CancelOnError(node Node) Dependency {
	return Dependency{
		node:         node,
		onCompletion: onErrorCancel,
	}
}

// CancelOnCompletion returns a Dependency that upon Node Fetch() completion causes sibling Dependencies' Fetches to be canceled.
func CancelOnCompletion(node Node) Dependency {
	return Dependency{
		node:         node,
		onCompletion: onCompletionCancel,
	}
}

// AbortOnError returns a Dependency where any Node Fetch() error causes sibling Dependencies' Fetches to be canceled and propagates the error.
//
// TODO: enrich the error with context before passing it on once Nodes have a description
func AbortOnError(node Node) Dependency {
	return Dependency{
		node:         node,
		onCompletion: onErrorAbort,
	}
}

// Dependencies are Nodes that must be Fetched before the given one can be.
type Dependencies []Dependency

// NewNode returns a Node backed by the given function.
//
// The fetch function must not be nil, unless you never intend to Fetch() this Node. It will be called with the errors from the optional dependencies, unless they are fatal, or context.Canceled if they returned CancelFetchSuccess().
//
// The dependencies will be fetched in parallel before fetch() is called and must not contain zero values. Don't introduce circular dependencies or the Fetch will deadlock waiting for itself to finish.
func NewNode(dependencies Dependencies, fetch func(context.Context, []error) error) Node {
	return &node{
		fetcher:      fetch,
		dependencies: dependencies,
	}
}

type node struct {
	fetcher      func(context.Context, []error) error
	dependencies Dependencies
	once         sync.Once
	err          error
}

func (n *node) Fetch(ctx context.Context) error {
	n.once.Do(func() {
		var errs []error
		// fetch dependencies in parallel
		if l := len(n.dependencies); l > 0 {
			errs = make([]error, l)
			if l == 1 {
				// no need to fetch single dependency in parallel
				err := n.dependencies[0].node.Fetch(ctx)
				switch n.dependencies[0].onCompletion {
				case onErrorAbort:
					if err != nil {
						n.err = err
						return
					}
				case onErrorCancel:
					fallthrough
				case onCompletionCancel:
					// no siblings to cancel
					errs[0] = err
				case onErrorContinue:
					errs[0] = err
				}
			} else {
				// we can save one goroutine by fetching that dependency on the current one
				var wg sync.WaitGroup
				var fatalErr atomic.Value
				subCtx, cancel := context.WithCancel(ctx)
				wg.Add(l - 1)
				for i := 1; i < l; i++ {
					go func(i int) {
						errs[i] = fetchDependency(subCtx, cancel, n.dependencies[i], &fatalErr)
						wg.Done()
					}(i)
				}
				errs[0] = fetchDependency(subCtx, cancel, n.dependencies[0], &fatalErr)
				wg.Wait()
				cancel()
				if iErr := fatalErr.Load(); iErr != nil {
					n.err = iErr.(error)
					return
				}
			}

		}
		n.err = n.fetcher(ctx, errs)
	})
	return n.err
}

func (n *node) noUserImplementations() {}

func fetchDependency(ctx context.Context, cancel func(), dependency Dependency, outFatalErr *atomic.Value) error {
	err := dependency.node.Fetch(ctx)
	switch dependency.onCompletion {
	case onErrorCancel:
		if err == nil {
			break
		}
		fallthrough
	case onCompletionCancel:
		cancel()
	case onErrorAbort:
		if err != nil {
			cancel()
			outFatalErr.Store(err)
		}
	case onErrorContinue:
	}
	return err
}

type completionStrategy int

const (
	onErrorContinue completionStrategy = iota
	onErrorCancel
	onCompletionCancel
	onErrorAbort
)
<|endoftext|>


<commit_before>package clock

import (
	"fmt"
	"log"
	"sync"
	"time"
)

//ExampleClock_AddJobRepeat 基于函数回调，一个对重复任务的使用演示。
func ExampleClock_AddJobRepeat() {
	var (
		myClock   = NewClock()
		counter   = 0
		mut       sync.Mutex
		sigalChan = make(chan struct{}, 0)
	)
	fn := func() {
		fmt.Println("schedule repeat")
		mut.Lock()
		defer mut.Unlock()
		counter++
		if counter == 3 {
			sigalChan <- struct{}{}
		}

	}
	//create a task that executes three times,interval 50 millisecond
	event, inserted := myClock.AddJobRepeat(time.Duration(time.Millisecond*50), 0, fn)
	if !inserted {
		log.Println("failure")
	}

	//等待阻塞信号
	<-sigalChan
	myClock.DelJob(event)

	//wait a second,watching
	time.Sleep(time.Second)
	//Output:
	//
	//schedule repeat
	//schedule repeat
	//schedule repeat
}

//ExampleClock_AddJobRepeat2 ，基于函数回调，演示添加有次数限制的重复任务
//  执行3次之后，撤销定时事件
func ExampleClock_AddJobRepeat2() {
	var (
		myClock = NewClock()
	)
	//define a repeat task
	fn := func() {
		fmt.Println("schedule repeat")
	}
	//add in clock,execute three times,interval 200 millisecond
	_, inserted := myClock.AddJobRepeat(time.Duration(time.Millisecond*200), 3, fn)
	if !inserted {
		log.Println("failure")
	}
	//wait a second,watching
	time.Sleep(time.Second)
	//Output:
	//
	//schedule repeat
	//schedule repeat
	//schedule repeat
}

//ExampleClock_AddJobWithTimeout 基于函数回调，对一次性任务正常使用的演示。
func ExampleClock_AddJobWithTimeout() {
	var (
		jobClock = NewClock()
		jobFunc  = func() {
			fmt.Println("schedule once")
		}
	)
	//add a task that executes once,interval 100 millisecond
	jobClock.AddJobWithInterval(time.Duration(100*time.Millisecond), jobFunc)

	//wait a second,watching
	time.Sleep(1 * time.Second)

	//Output:
	//
	//schedule once
}

//ExampleClock_AddJobWithDeadtime 基于事件提醒，对一次性任务中途放弃的使用演示。
func ExampleClock_AddJobWithDeadtime() {
	var (
		myClock = NewClock()
		jobFunc = func() {
			fmt.Println("schedule once")
		}
		actionTime = time.Now().Add(time.Millisecond * 500)
	)
	//创建一次性任务，定时500ms
	job, _ := myClock.AddJobWithDeadtime(actionTime, jobFunc)

	//任务执行前，撤销任务
	time.Sleep(time.Millisecond * 300)
	myClock.DelJob(job)

	//等待2秒，正常情况下，事件不会再执行
	time.Sleep(2 * time.Second)

	//Output:
	//
	//
}
<commit_msg>fix renew test function name<commit_after>package clock

import (
	"fmt"
	"log"
	"sync"
	"time"
)

//ExampleClock_AddJobRepeat 基于函数回调，一个对重复任务的使用演示。
func ExampleClock_AddJobRepeat() {
	var (
		myClock   = NewClock()
		counter   = 0
		mut       sync.Mutex
		sigalChan = make(chan struct{}, 0)
	)
	fn := func() {
		fmt.Println("schedule repeat")
		mut.Lock()
		defer mut.Unlock()
		counter++
		if counter == 3 {
			sigalChan <- struct{}{}
		}

	}
	//create a task that executes three times,interval 50 millisecond
	event, inserted := myClock.AddJobRepeat(time.Duration(time.Millisecond*50), 0, fn)
	if !inserted {
		log.Println("failure")
	}

	//等待阻塞信号
	<-sigalChan
	myClock.DelJob(event)

	//wait a second,watching
	time.Sleep(time.Second)
	//Output:
	//
	//schedule repeat
	//schedule repeat
	//schedule repeat
}

//ExampleClock_AddJobRepeat2 ，基于函数回调，演示添加有次数限制的重复任务
//  执行3次之后，撤销定时事件
func ExampleClock_AddJobRepeat2() {
	var (
		myClock = NewClock()
	)
	//define a repeat task
	fn := func() {
		fmt.Println("schedule repeat")
	}
	//add in clock,execute three times,interval 200 millisecond
	_, inserted := myClock.AddJobRepeat(time.Duration(time.Millisecond*200), 3, fn)
	if !inserted {
		log.Println("failure")
	}
	//wait a second,watching
	time.Sleep(time.Second)
	//Output:
	//
	//schedule repeat
	//schedule repeat
	//schedule repeat
}

//ExampleClock_AddJobWithInterval 基于函数回调，对一次性任务正常使用的演示。
func ExampleClock_AddJobWithInterval() {
	var (
		jobClock = NewClock()
		jobFunc  = func() {
			fmt.Println("schedule once")
		}
	)
	//add a task that executes once,interval 100 millisecond
	jobClock.AddJobWithInterval(time.Duration(100*time.Millisecond), jobFunc)

	//wait a second,watching
	time.Sleep(1 * time.Second)

	//Output:
	//
	//schedule once
}

//ExampleClock_AddJobWithDeadtime 基于事件提醒，对一次性任务中途放弃的使用演示。
func ExampleClock_AddJobWithDeadtime() {
	var (
		myClock = NewClock()
		jobFunc = func() {
			fmt.Println("schedule once")
		}
		actionTime = time.Now().Add(time.Millisecond * 500)
	)
	//创建一次性任务，定时500ms
	job, _ := myClock.AddJobWithDeadtime(actionTime, jobFunc)

	//任务执行前，撤销任务
	time.Sleep(time.Millisecond * 300)
	myClock.DelJob(job)

	//等待2秒，正常情况下，事件不会再执行
	time.Sleep(2 * time.Second)

	//Output:
	//
	//
}
<|endoftext|>


<commit_before>package cli

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/user"

	"github.com/opentable/sous/ext/git"
	"github.com/opentable/sous/sous"
	"github.com/opentable/sous/util/cmdr"
	"github.com/opentable/sous/util/shell"
	"github.com/samsalisbury/psyringe"
	"github.com/samsalisbury/semv"
)

type (
	// Out is an output used for real data a Command returns. This should only
	// be used when a command needs to write directly to stdout, using the
	// formatting options that come with an output. Usually, you should use a
	// SuccessResult with Data to return data.
	Out struct{ *cmdr.Output }
	// ErrOut is an output used for logging from a Command. This should only be
	// used when a Command needs to write a lot of data to stderr, using the
	// formatting options that come with and Output. Usually you should use and
	// ErrorResult to return error messages.
	ErrOut struct{ *cmdr.Output }
	// SousCLIGraph is a dependency injector used to flesh out Sous commands
	// with their dependencies.
	SousCLIGraph struct{ *psyringe.Psyringe }
	// SousVersion represents a version of Sous.
	Version struct{ semv.Version }
	// LocalUser is the currently logged in user.
	LocalUser struct{ *User }
	// LocalSousConfig is the configuration for Sous.
	LocalSousConfig struct{ *sous.Config }
	// WorkDir is the user's current working directory when they invoke Sous.
	LocalWorkDir string
	// WorkdirShell is a shell for working in the user's current working
	// directory.
	LocalWorkDirShell struct{ *shell.Sh }
	// LocalGitClient is a git client rooted in WorkdirShell.Dir.
	LocalGitClient struct{ *git.Client }
	// LocalGitRepo is the git repository containing WorkDir.
	LocalGitRepo struct{ *git.Repo }
	// LocalGitContext is the git context snapshot of the user when they invok
	// Sous.
	LocalGitContext struct{ *git.Context }
	// ScratchDirShell is a shell for working in the scratch area where things
	// like artefacts, and build metadata are stored. It is a new, empty
	// directory, and should be cleaned up eventually.
	ScratchDirShell struct{ *shell.Sh }
)

// buildGraph builds the dependency injection graph, used to populate commands
// invoked by the user.
func BuildGraph(s *Sous, c *cmdr.CLI) (*SousCLIGraph, error) {
	g := &SousCLIGraph{psyringe.New()}
	return g, g.Fill(
		s, c,
		newOut,
		newErrOut,
		newLocalUser,
		newLocalSousConfig,
		newLocalWorkDir,
		newLocalWorkDirShell,
		newScratchDirShell,
		newLocalGitClient,
		newLocalGitRepo,
		newSourceContext,
	)
}

func newOut(c *cmdr.CLI) Out {
	return Out{c.Out}
}

func newErrOut(c *cmdr.CLI) ErrOut {
	return ErrOut{c.Err}
}

func newSourceContext(g LocalGitRepo) (c *sous.SourceContext, err error) {
	c, err = g.SourceContext()
	return c, initErr(err, "getting local git context")
}

func newLocalWorkDir() (LocalWorkDir, error) {
	s, err := os.Getwd()
	return LocalWorkDir(s), initErr(err, "determining working directory")
}

func newLocalUser() (v LocalUser, err error) {
	u, err := user.Current()
	v.User = &User{u}
	return v, initErr(err, "getting current user")
}

func newLocalSousConfig(u LocalUser) (v LocalSousConfig, err error) {
	v.Config, err = newDefaultConfig(u.User)
	return v, initErr(err, "getting default config")
}

func newLocalWorkDirShell(l LocalWorkDir) (v LocalWorkDirShell, err error) {
	v.Sh, err = shell.DefaultInDir(string(l))
	return v, initErr(err, "getting current working directory")
}

// TODO: This should register a cleanup task with the cli, to delete the temp
// dir.
func newScratchDirShell() (v ScratchDirShell, err error) {
	what := "getting scratch directory"
	dir, err := ioutil.TempDir("", "sous")
	if err != nil {
		return v, initErr(err, what)
	}
	v.Sh, err = shell.DefaultInDir(dir)
	return v, initErr(err, what)
}

func newLocalGitClient(sh LocalWorkDirShell) (v LocalGitClient, err error) {
	v.Client, err = git.NewClient(sh.Sh)
	return v, initErr(err, "initialising git client")
}

func newLocalGitRepo(c LocalGitClient) (v LocalGitRepo, err error) {
	v.Repo, err = c.OpenRepo(".")
	return v, initErr(err, "opening local git repository")
}

// initErr returns nil if error is nil, otherwise an initialisation error.
func initErr(err error, what string) error {
	if err == nil {
		return nil
	}
	message := fmt.Sprintf("error %s:", what)
	if shellErr, ok := err.(shell.Error); ok {
		message += fmt.Sprintf("\ncommand failed:\nshell> %s\n%s",
			shellErr.Command.String(), shellErr.Result.Combined.String())
	} else {
		message += " " + err.Error()
	}
	return fmt.Errorf(message)
}
<commit_msg>Remove unused type<commit_after>package cli

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/user"

	"github.com/opentable/sous/ext/git"
	"github.com/opentable/sous/sous"
	"github.com/opentable/sous/util/cmdr"
	"github.com/opentable/sous/util/shell"
	"github.com/samsalisbury/psyringe"
	"github.com/samsalisbury/semv"
)

type (
	// Out is an output used for real data a Command returns. This should only
	// be used when a command needs to write directly to stdout, using the
	// formatting options that come with an output. Usually, you should use a
	// SuccessResult with Data to return data.
	Out struct{ *cmdr.Output }
	// ErrOut is an output used for logging from a Command. This should only be
	// used when a Command needs to write a lot of data to stderr, using the
	// formatting options that come with and Output. Usually you should use and
	// ErrorResult to return error messages.
	ErrOut struct{ *cmdr.Output }
	// SousCLIGraph is a dependency injector used to flesh out Sous commands
	// with their dependencies.
	SousCLIGraph struct{ *psyringe.Psyringe }
	// SousVersion represents a version of Sous.
	Version struct{ semv.Version }
	// LocalUser is the currently logged in user.
	LocalUser struct{ *User }
	// LocalSousConfig is the configuration for Sous.
	LocalSousConfig struct{ *sous.Config }
	// WorkDir is the user's current working directory when they invoke Sous.
	LocalWorkDir string
	// WorkdirShell is a shell for working in the user's current working
	// directory.
	LocalWorkDirShell struct{ *shell.Sh }
	// LocalGitClient is a git client rooted in WorkdirShell.Dir.
	LocalGitClient struct{ *git.Client }
	// LocalGitRepo is the git repository containing WorkDir.
	LocalGitRepo struct{ *git.Repo }
	// ScratchDirShell is a shell for working in the scratch area where things
	// like artefacts, and build metadata are stored. It is a new, empty
	// directory, and should be cleaned up eventually.
	ScratchDirShell struct{ *shell.Sh }
)

// buildGraph builds the dependency injection graph, used to populate commands
// invoked by the user.
func BuildGraph(s *Sous, c *cmdr.CLI) (*SousCLIGraph, error) {
	g := &SousCLIGraph{psyringe.New()}
	return g, g.Fill(
		s, c,
		newOut,
		newErrOut,
		newLocalUser,
		newLocalSousConfig,
		newLocalWorkDir,
		newLocalWorkDirShell,
		newScratchDirShell,
		newLocalGitClient,
		newLocalGitRepo,
		newSourceContext,
	)
}

func newOut(c *cmdr.CLI) Out {
	return Out{c.Out}
}

func newErrOut(c *cmdr.CLI) ErrOut {
	return ErrOut{c.Err}
}

func newSourceContext(g LocalGitRepo) (c *sous.SourceContext, err error) {
	c, err = g.SourceContext()
	return c, initErr(err, "getting local git context")
}

func newLocalWorkDir() (LocalWorkDir, error) {
	s, err := os.Getwd()
	return LocalWorkDir(s), initErr(err, "determining working directory")
}

func newLocalUser() (v LocalUser, err error) {
	u, err := user.Current()
	v.User = &User{u}
	return v, initErr(err, "getting current user")
}

func newLocalSousConfig(u LocalUser) (v LocalSousConfig, err error) {
	v.Config, err = newDefaultConfig(u.User)
	return v, initErr(err, "getting default config")
}

func newLocalWorkDirShell(l LocalWorkDir) (v LocalWorkDirShell, err error) {
	v.Sh, err = shell.DefaultInDir(string(l))
	return v, initErr(err, "getting current working directory")
}

// TODO: This should register a cleanup task with the cli, to delete the temp
// dir.
func newScratchDirShell() (v ScratchDirShell, err error) {
	what := "getting scratch directory"
	dir, err := ioutil.TempDir("", "sous")
	if err != nil {
		return v, initErr(err, what)
	}
	v.Sh, err = shell.DefaultInDir(dir)
	return v, initErr(err, what)
}

func newLocalGitClient(sh LocalWorkDirShell) (v LocalGitClient, err error) {
	v.Client, err = git.NewClient(sh.Sh)
	return v, initErr(err, "initialising git client")
}

func newLocalGitRepo(c LocalGitClient) (v LocalGitRepo, err error) {
	v.Repo, err = c.OpenRepo(".")
	return v, initErr(err, "opening local git repository")
}

// initErr returns nil if error is nil, otherwise an initialisation error.
func initErr(err error, what string) error {
	if err == nil {
		return nil
	}
	message := fmt.Sprintf("error %s:", what)
	if shellErr, ok := err.(shell.Error); ok {
		message += fmt.Sprintf("\ncommand failed:\nshell> %s\n%s",
			shellErr.Command.String(), shellErr.Result.Combined.String())
	} else {
		message += " " + err.Error()
	}
	return fmt.Errorf(message)
}
<|endoftext|>


<commit_before>package expr

import (
	"github.com/grafana/metrictank/api/models"
	"github.com/grafana/metrictank/consolidation"
)

// Context describes a series timeframe and consolidator
type Context struct {
	from          uint32
	to            uint32
	consol        consolidation.Consolidator // can be 0 to mean undefined
	PNGroup       models.PNGroup             // pre-normalization group. if the data can be safely pre-normalized
	MDP           uint32                     // if we can MDP-optimize, reflects runtime consolidation MaxDataPoints. 0 otherwise
	optimizations Optimizations
}

// GraphiteFunc defines a graphite processing function
type GraphiteFunc interface {
	// Signature declares input and output arguments (return values)
	// input args can be optional in which case they can be specified positionally or via keys if you want to specify params that come after un-specified optional params
	// the val pointers of each input Arg should point to a location accessible to the function,
	// so that the planner can set up the inputs for your function based on user input.
	// NewPlan() will only create the plan if the expressions it parsed correspond to the signatures provided by the function
	Signature() ([]Arg, []Arg)

	// Context allows a func to alter the context that will be passed down the expression tree.
	// this function will be called after validating and setting up all non-series and non-serieslist parameters.
	// (as typically, context alterations require integer/string/bool/etc parameters, and shall affect series[list] parameters)
	// examples:
	// * movingAverage(foo,5min) -> the 5min arg will be parsed, so we can request 5min of earlier data, which will affect the request for foo.
	// * consolidateBy(bar, "sum") -> the "sum" arg will be parsed, so we can pass on the fact that bar needs to be sum-consolidated
	Context(c Context) Context
	// Exec executes the function. the function should call any input functions, do its processing, and return output.
	// IMPORTANT: for performance and correctness, functions should
	// * not modify slices of points that they get from their inputs
	// * use the pool to get new slices in which to store any new/modified dat
	// * add the newly created slices into the dataMap so they can be reclaimed after the output is consumed
	// * not modify other properties on its input series, such as Tags map or Meta
	Exec(dataMap DataMap) ([]models.Series, error)
}

type funcConstructor func() GraphiteFunc

type funcDef struct {
	constr funcConstructor
	stable bool
}

var funcs map[string]funcDef

func init() {
	// keys must be sorted alphabetically. but functions with aliases can go together, in which case they are sorted by the first of their aliases
	funcs = map[string]funcDef{
		"absolute":              {NewAbsolute, true},
		"aggregate":             {NewAggregate, true},
		"alias":                 {NewAlias, true},
		"aliasByMetric":         {NewAliasByMetric, true},
		"aliasByTags":           {NewAliasByNode, true},
		"aliasByNode":           {NewAliasByNode, true},
		"aliasSub":              {NewAliasSub, true},
		"asPercent":             {NewAsPercent, true},
		"avg":                   {NewAggregateConstructor("average"), true},
		"averageAbove":          {NewFilterSeriesConstructor("average", ">"), true},
		"averageBelow":          {NewFilterSeriesConstructor("average", "<="), true},
		"averageSeries":         {NewAggregateConstructor("average"), true},
		"consolidateBy":         {NewConsolidateBy, true},
		"constantLine":          {NewConstantLine, false},
		"countSeries":           {NewCountSeries, true},
		"cumulative":            {NewConsolidateByConstructor("sum"), true},
		"currentAbove":          {NewFilterSeriesConstructor("last", ">"), true},
		"currentBelow":          {NewFilterSeriesConstructor("last", "<="), true},
		"derivative":            {NewDerivative, true},
		"diffSeries":            {NewAggregateConstructor("diff"), true},
		"divideSeries":          {NewDivideSeries, true},
		"divideSeriesLists":     {NewDivideSeriesLists, true},
		"exclude":               {NewExclude, true},
		"fallbackSeries":        {NewFallbackSeries, true},
		"filterSeries":          {NewFilterSeries, true},
		"grep":                  {NewGrep, true},
		"group":                 {NewGroup, true},
		"groupByNode":           {NewGroupByNodesConstructor(true), true},
		"groupByNodes":          {NewGroupByNodesConstructor(false), true},
		"groupByTags":           {NewGroupByTags, true},
		"highest":               {NewHighestLowestConstructor("", true), true},
		"highestAverage":        {NewHighestLowestConstructor("average", true), true},
		"highestCurrent":        {NewHighestLowestConstructor("current", true), true},
		"highestMax":            {NewHighestLowestConstructor("max", true), true},
		"integral":              {NewIntegral, true},
		"invert":                {NewInvert, true},
		"isNonNull":             {NewIsNonNull, true},
		"keepLastValue":         {NewKeepLastValue, true},
		"lowest":                {NewHighestLowestConstructor("", false), true},
		"lowestAverage":         {NewHighestLowestConstructor("average", false), true},
		"lowestCurrent":         {NewHighestLowestConstructor("current", false), true},
		"max":                   {NewAggregateConstructor("max"), true},
		"maximumAbove":          {NewFilterSeriesConstructor("max", ">"), true},
		"maximumBelow":          {NewFilterSeriesConstructor("max", "<="), true},
		"maxSeries":             {NewAggregateConstructor("max"), true},
		"min":                   {NewAggregateConstructor("min"), true},
		"minimumAbove":          {NewFilterSeriesConstructor("min", ">"), true},
		"minimumBelow":          {NewFilterSeriesConstructor("min", "<="), true},
		"minMax":                {NewMinMax, true},
		"minSeries":             {NewAggregateConstructor("min"), true},
		"multiplySeries":        {NewAggregateConstructor("multiply"), true},
		"movingAverage":         {NewMovingAverage, false},
		"nonNegativeDerivative": {NewNonNegativeDerivative, true},
		"offset":                {NewOffset, true},
		"perSecond":             {NewPerSecond, true},
		"rangeOfSeries":         {NewAggregateConstructor("rangeOf"), true},
		"removeAbovePercentile": {NewRemoveAboveBelowPercentileConstructor(true), true},
		"removeAboveValue":      {NewRemoveAboveBelowValueConstructor(true), true},
		"removeBelowPercentile": {NewRemoveAboveBelowPercentileConstructor(false), true},
		"removeBelowValue":      {NewRemoveAboveBelowValueConstructor(false), true},
		"removeEmptySeries":     {NewRemoveEmptySeries, true},
		"round":                 {NewRound, true},
		"scale":                 {NewScale, true},
		"scaleToSeconds":        {NewScaleToSeconds, true},
		"smartSummarize":        {NewSmartSummarize, false},
		"sortBy":                {NewSortByConstructor("", false), true},
		"sortByMaxima":          {NewSortByConstructor("max", true), true},
		"sortByName":            {NewSortByName, true},
		"sortByTotal":           {NewSortByConstructor("sum", true), true},
		"stddevSeries":          {NewAggregateConstructor("stddev"), true},
		"sum":                   {NewAggregateConstructor("sum"), true},
		"sumSeries":             {NewAggregateConstructor("sum"), true},
		"summarize":             {NewSummarize, true},
		"transformNull":         {NewTransformNull, true},
		"unique":                {NewUnique, true},
	}
}

// summarizeCons returns the first explicitly specified Consolidator, QueryCons for the given set of input series,
// or the first one, otherwise.
func summarizeCons(series []models.Series) (consolidation.Consolidator, consolidation.Consolidator) {
	for _, serie := range series {
		if serie.QueryCons != 0 {
			return serie.Consolidator, serie.QueryCons
		}
	}
	return series[0].Consolidator, series[0].QueryCons
}

func consumeFuncs(dataMap DataMap, fns []GraphiteFunc) ([]models.Series, []string, error) {
	var series []models.Series
	var queryPatts []string
	for i := range fns {
		in, err := fns[i].Exec(dataMap)
		if err != nil {
			return nil, nil, err
		}
		if len(in) != 0 {
			series = append(series, in...)
			queryPatts = append(queryPatts, in[0].QueryPatt)
		}
	}
	return series, queryPatts, nil
}
<commit_msg>timeShift stable<commit_after>package expr

import (
	"github.com/grafana/metrictank/api/models"
	"github.com/grafana/metrictank/consolidation"
)

// Context describes a series timeframe and consolidator
type Context struct {
	from          uint32
	to            uint32
	consol        consolidation.Consolidator // can be 0 to mean undefined
	PNGroup       models.PNGroup             // pre-normalization group. if the data can be safely pre-normalized
	MDP           uint32                     // if we can MDP-optimize, reflects runtime consolidation MaxDataPoints. 0 otherwise
	optimizations Optimizations
}

// GraphiteFunc defines a graphite processing function
type GraphiteFunc interface {
	// Signature declares input and output arguments (return values)
	// input args can be optional in which case they can be specified positionally or via keys if you want to specify params that come after un-specified optional params
	// the val pointers of each input Arg should point to a location accessible to the function,
	// so that the planner can set up the inputs for your function based on user input.
	// NewPlan() will only create the plan if the expressions it parsed correspond to the signatures provided by the function
	Signature() ([]Arg, []Arg)

	// Context allows a func to alter the context that will be passed down the expression tree.
	// this function will be called after validating and setting up all non-series and non-serieslist parameters.
	// (as typically, context alterations require integer/string/bool/etc parameters, and shall affect series[list] parameters)
	// examples:
	// * movingAverage(foo,5min) -> the 5min arg will be parsed, so we can request 5min of earlier data, which will affect the request for foo.
	// * consolidateBy(bar, "sum") -> the "sum" arg will be parsed, so we can pass on the fact that bar needs to be sum-consolidated
	Context(c Context) Context
	// Exec executes the function. the function should call any input functions, do its processing, and return output.
	// IMPORTANT: for performance and correctness, functions should
	// * not modify slices of points that they get from their inputs
	// * use the pool to get new slices in which to store any new/modified dat
	// * add the newly created slices into the dataMap so they can be reclaimed after the output is consumed
	// * not modify other properties on its input series, such as Tags map or Meta
	Exec(dataMap DataMap) ([]models.Series, error)
}

type funcConstructor func() GraphiteFunc

type funcDef struct {
	constr funcConstructor
	stable bool
}

var funcs map[string]funcDef

func init() {
	// keys must be sorted alphabetically. but functions with aliases can go together, in which case they are sorted by the first of their aliases
	funcs = map[string]funcDef{
		"absolute":              {NewAbsolute, true},
		"aggregate":             {NewAggregate, true},
		"alias":                 {NewAlias, true},
		"aliasByMetric":         {NewAliasByMetric, true},
		"aliasByTags":           {NewAliasByNode, true},
		"aliasByNode":           {NewAliasByNode, true},
		"aliasSub":              {NewAliasSub, true},
		"asPercent":             {NewAsPercent, true},
		"avg":                   {NewAggregateConstructor("average"), true},
		"averageAbove":          {NewFilterSeriesConstructor("average", ">"), true},
		"averageBelow":          {NewFilterSeriesConstructor("average", "<="), true},
		"averageSeries":         {NewAggregateConstructor("average"), true},
		"consolidateBy":         {NewConsolidateBy, true},
		"constantLine":          {NewConstantLine, false},
		"countSeries":           {NewCountSeries, true},
		"cumulative":            {NewConsolidateByConstructor("sum"), true},
		"currentAbove":          {NewFilterSeriesConstructor("last", ">"), true},
		"currentBelow":          {NewFilterSeriesConstructor("last", "<="), true},
		"derivative":            {NewDerivative, true},
		"diffSeries":            {NewAggregateConstructor("diff"), true},
		"divideSeries":          {NewDivideSeries, true},
		"divideSeriesLists":     {NewDivideSeriesLists, true},
		"exclude":               {NewExclude, true},
		"fallbackSeries":        {NewFallbackSeries, true},
		"filterSeries":          {NewFilterSeries, true},
		"grep":                  {NewGrep, true},
		"group":                 {NewGroup, true},
		"groupByNode":           {NewGroupByNodesConstructor(true), true},
		"groupByNodes":          {NewGroupByNodesConstructor(false), true},
		"groupByTags":           {NewGroupByTags, true},
		"highest":               {NewHighestLowestConstructor("", true), true},
		"highestAverage":        {NewHighestLowestConstructor("average", true), true},
		"highestCurrent":        {NewHighestLowestConstructor("current", true), true},
		"highestMax":            {NewHighestLowestConstructor("max", true), true},
		"integral":              {NewIntegral, true},
		"invert":                {NewInvert, true},
		"isNonNull":             {NewIsNonNull, true},
		"keepLastValue":         {NewKeepLastValue, true},
		"lowest":                {NewHighestLowestConstructor("", false), true},
		"lowestAverage":         {NewHighestLowestConstructor("average", false), true},
		"lowestCurrent":         {NewHighestLowestConstructor("current", false), true},
		"max":                   {NewAggregateConstructor("max"), true},
		"maximumAbove":          {NewFilterSeriesConstructor("max", ">"), true},
		"maximumBelow":          {NewFilterSeriesConstructor("max", "<="), true},
		"maxSeries":             {NewAggregateConstructor("max"), true},
		"min":                   {NewAggregateConstructor("min"), true},
		"minimumAbove":          {NewFilterSeriesConstructor("min", ">"), true},
		"minimumBelow":          {NewFilterSeriesConstructor("min", "<="), true},
		"minMax":                {NewMinMax, true},
		"minSeries":             {NewAggregateConstructor("min"), true},
		"multiplySeries":        {NewAggregateConstructor("multiply"), true},
		"movingAverage":         {NewMovingAverage, false},
		"nonNegativeDerivative": {NewNonNegativeDerivative, true},
		"offset":                {NewOffset, true},
		"perSecond":             {NewPerSecond, true},
		"rangeOfSeries":         {NewAggregateConstructor("rangeOf"), true},
		"removeAbovePercentile": {NewRemoveAboveBelowPercentileConstructor(true), true},
		"removeAboveValue":      {NewRemoveAboveBelowValueConstructor(true), true},
		"removeBelowPercentile": {NewRemoveAboveBelowPercentileConstructor(false), true},
		"removeBelowValue":      {NewRemoveAboveBelowValueConstructor(false), true},
		"removeEmptySeries":     {NewRemoveEmptySeries, true},
		"round":                 {NewRound, true},
		"scale":                 {NewScale, true},
		"scaleToSeconds":        {NewScaleToSeconds, true},
		"smartSummarize":        {NewSmartSummarize, false},
		"sortBy":                {NewSortByConstructor("", false), true},
		"sortByMaxima":          {NewSortByConstructor("max", true), true},
		"sortByName":            {NewSortByName, true},
		"sortByTotal":           {NewSortByConstructor("sum", true), true},
		"stddevSeries":          {NewAggregateConstructor("stddev"), true},
		"sum":                   {NewAggregateConstructor("sum"), true},
		"sumSeries":             {NewAggregateConstructor("sum"), true},
		"summarize":             {NewSummarize, true},
		"timeShift":             {NewTimeShift, true},
		"transformNull":         {NewTransformNull, true},
		"unique":                {NewUnique, true},
	}
}

// summarizeCons returns the first explicitly specified Consolidator, QueryCons for the given set of input series,
// or the first one, otherwise.
func summarizeCons(series []models.Series) (consolidation.Consolidator, consolidation.Consolidator) {
	for _, serie := range series {
		if serie.QueryCons != 0 {
			return serie.Consolidator, serie.QueryCons
		}
	}
	return series[0].Consolidator, series[0].QueryCons
}

func consumeFuncs(dataMap DataMap, fns []GraphiteFunc) ([]models.Series, []string, error) {
	var series []models.Series
	var queryPatts []string
	for i := range fns {
		in, err := fns[i].Exec(dataMap)
		if err != nil {
			return nil, nil, err
		}
		if len(in) != 0 {
			series = append(series, in...)
			queryPatts = append(queryPatts, in[0].QueryPatt)
		}
	}
	return series, queryPatts, nil
}
<|endoftext|>


<commit_before><commit_msg>container_registry/container_analysis: skip TestUpdateOccurrence (#786)<commit_after><|endoftext|>


<commit_before>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gccgoimporter

import (
	"bufio"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"code.google.com/p/go.tools/go/types"
)

// Information about a specific installation of gccgo.
type GccgoInstallation struct {
	// Version of gcc (e.g. 4.8.0).
	GccVersion string

	// Target triple (e.g. x86_64-unknown-linux-gnu).
	TargetTriple string

	// Built-in library paths used by this installation.
	LibPaths []string
}

// Ask the driver at the given path for information for this GccgoInstallation.
func (inst *GccgoInstallation) InitFromDriver(gccgoPath string) (err error) {
	cmd := exec.Command(gccgoPath, "-###", "-S", "-x", "go", "-")
	stderr, err := cmd.StderrPipe()
	if err != nil {
		return
	}

	err = cmd.Start()
	if err != nil {
		return
	}

	scanner := bufio.NewScanner(stderr)
	for scanner.Scan() {
		line := scanner.Text()
		switch {
		case strings.HasPrefix(line, "Target: "):
			inst.TargetTriple = line[8:]

		case strings.HasPrefix(line, "gcc version "):
			inst.GccVersion = strings.SplitN(line[12:], " ", 2)[0]

		case line[0] == ' ':
			args := strings.Fields(line)
			for _, arg := range args[1:] {
				if strings.HasPrefix(arg, "-L") {
					inst.LibPaths = append(inst.LibPaths, arg[2:])
				}
			}
		}
	}

	return
}

// Return the list of export search paths for this GccgoInstallation.
func (inst *GccgoInstallation) SearchPaths() (paths []string) {
	for _, lpath := range inst.LibPaths {
		spath := filepath.Join(lpath, "go", inst.GccVersion)
		fi, err := os.Stat(spath)
		if err != nil || !fi.IsDir() {
			continue
		}
		paths = append(paths, spath)

		spath = filepath.Join(spath, inst.TargetTriple)
		fi, err = os.Stat(spath)
		if err != nil || !fi.IsDir() {
			continue
		}
		paths = append(paths, spath)
	}

	paths = append(paths, inst.LibPaths...)

	return
}

// Return an importer that searches incpaths followed by the gcc installation's
// built-in search paths and the current directory.
func (inst *GccgoInstallation) GetImporter(incpaths []string) types.Importer {
	return GetImporter(append(append(incpaths, inst.SearchPaths()...), "."))
}
<commit_msg>go.tools/go/gccgoimporter: use gccgo -dumpversion<commit_after>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gccgoimporter

import (
	"bufio"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"code.google.com/p/go.tools/go/types"
)

// Information about a specific installation of gccgo.
type GccgoInstallation struct {
	// Version of gcc (e.g. 4.8.0).
	GccVersion string

	// Target triple (e.g. x86_64-unknown-linux-gnu).
	TargetTriple string

	// Built-in library paths used by this installation.
	LibPaths []string
}

// Ask the driver at the given path for information for this GccgoInstallation.
func (inst *GccgoInstallation) InitFromDriver(gccgoPath string) (err error) {
	cmd := exec.Command(gccgoPath, "-###", "-S", "-x", "go", "-")
	stderr, err := cmd.StderrPipe()
	if err != nil {
		return
	}

	err = cmd.Start()
	if err != nil {
		return
	}

	scanner := bufio.NewScanner(stderr)
	for scanner.Scan() {
		line := scanner.Text()
		switch {
		case strings.HasPrefix(line, "Target: "):
			inst.TargetTriple = line[8:]

		case line[0] == ' ':
			args := strings.Fields(line)
			for _, arg := range args[1:] {
				if strings.HasPrefix(arg, "-L") {
					inst.LibPaths = append(inst.LibPaths, arg[2:])
				}
			}
		}
	}

	stdout, err := exec.Command(gccgoPath, "-dumpversion").Output()
	if err != nil {
		return
	}
	inst.GccVersion = strings.TrimSpace(string(stdout))

	return
}

// Return the list of export search paths for this GccgoInstallation.
func (inst *GccgoInstallation) SearchPaths() (paths []string) {
	for _, lpath := range inst.LibPaths {
		spath := filepath.Join(lpath, "go", inst.GccVersion)
		fi, err := os.Stat(spath)
		if err != nil || !fi.IsDir() {
			continue
		}
		paths = append(paths, spath)

		spath = filepath.Join(spath, inst.TargetTriple)
		fi, err = os.Stat(spath)
		if err != nil || !fi.IsDir() {
			continue
		}
		paths = append(paths, spath)
	}

	paths = append(paths, inst.LibPaths...)

	return
}

// Return an importer that searches incpaths followed by the gcc installation's
// built-in search paths and the current directory.
func (inst *GccgoInstallation) GetImporter(incpaths []string) types.Importer {
	return GetImporter(append(append(incpaths, inst.SearchPaths()...), "."))
}
<|endoftext|>


<commit_before><commit_msg>add IsMissingVersionError<commit_after><|endoftext|>


<commit_before>package vsphere

import (
	"bytes"
	"log"
	"strings"

	"github.com/cblomart/vsphere-graphite/utils"

	"github.com/vmware/govmomi/vim25/types"
)

// Cache will hold some informations in memory
type Cache map[string]interface{}

// Gets an index
func index(vcenter, section, i string) string {
	var buffer bytes.Buffer
	_, err := buffer.WriteString(vcenter)
	if err != nil {
		return ""
	}
	_, err = buffer.WriteString("|")
	if err != nil {
		return ""
	}
	_, err = buffer.WriteString(section)
	if err != nil {
		return ""
	}
	_, err = buffer.WriteString("|")
	if err != nil {
		return ""
	}
	_, err = buffer.WriteString(i)
	if err != nil {
		return ""
	}
	return buffer.String()
}

// Add a value to the cache
func (c *Cache) Add(vcenter, section, i string, v interface{}) {
	if len(vcenter) == 0 || len(section) == 0 || len(i) == 0 || v == nil {
		return
	}
	switch typed := v.(type) {
	case string:
		c.add(vcenter, section, i, &typed)
	case []string:
		if len(typed) > 0 {
			c.add(vcenter, section, i, &typed)
		}
	case int32:
		c.add(vcenter, section, i, v)
	case types.ManagedObjectReference:
		c.add(vcenter, section, i, &(typed.Value))
	case types.ArrayOfManagedObjectReference:
		if len(typed.ManagedObjectReference) > 0 {
			c.add(vcenter, section, i, &(typed.ManagedObjectReference))
		}
	case types.ArrayOfTag:
		if len(typed.Tag) > 0 {
			c.add(vcenter, section, i, &(typed.Tag))
		}
	case types.ArrayOfGuestDiskInfo:
		if len(typed.GuestDiskInfo) > 0 {
			c.add(vcenter, section, i, &(typed.GuestDiskInfo))
		}
	default:
		log.Printf("Adding a unhandled type %T to cache for %s section %s and ref %s\n", v, vcenter, section, i)
	}
}

// add to the cache without type check
func (c *Cache) add(vcenter, section, i string, v interface{}) {
	if v != nil {
		(*c)[index(vcenter, section, i)] = v
	}
}

// get a value from the cache
func (c *Cache) get(vcenter, section, i string) interface{} {
	if v, ok := (*c)[index(vcenter, section, i)]; ok {
		return v
	}
	return nil
}

// GetString gets a string from cache
func (c *Cache) GetString(vcenter, section, i string) *string {
	if v, ok := c.get(vcenter, section, i).(*string); ok {
		return v
	}
	return nil
}

// GetStrings gets an array of strings from cache
func (c *Cache) GetStrings(vcenter, section, i string) *[]string {
	if v, ok := c.get(vcenter, section, i).(*[]string); ok {
		return v
	}
	return nil
}

// GetInt32 get an int32 from cache
func (c *Cache) GetInt32(vcenter, section, i string) *int32 {
	if v, ok := c.get(vcenter, section, i).(*int32); ok {
		return v
	}
	return nil
}

// GetMoref gets a managed object reference from cache
func (c *Cache) GetMoref(vcenter, section, i string) *types.ManagedObjectReference {
	if v, ok := c.get(vcenter, section, i).(*types.ManagedObjectReference); ok {
		return v
	}
	return nil
}

// GetMorefs gets an array of managed references from cache
func (c *Cache) GetMorefs(vcenter, section, i string) *[]types.ManagedObjectReference {
	if v, ok := c.get(vcenter, section, i).(*[]types.ManagedObjectReference); ok {
		return v
	}
	return nil
}

// GetTags gets an array of vsphere tags from cache
func (c *Cache) GetTags(vcenter, section, i string) *[]types.Tag {
	if v, ok := c.get(vcenter, section, i).(*[]types.Tag); ok {
		return v
	}
	return nil
}

// GetDiskInfos gets an array of diskinfos from cache
func (c *Cache) GetDiskInfos(vcenter, section, i string) *[]types.GuestDiskInfo {
	if v, ok := c.get(vcenter, section, i).(*[]types.GuestDiskInfo); ok {
		return v
	}
	return nil
}

// Clean cache of unknows references
func (c *Cache) Clean(vcenter string, section string, refs []string) {
	for e := range *c {
		// get back index parts
		m := strings.Split(e, "|")
		// check that we have tree parts or delete
		if len(m) != 3 {
			delete(*c, e)
		}
		// check vcenter
		if m[0] != vcenter || m[1] != section {
			continue
		}
		// find the value in the range
		found := false
		for _, ref := range refs {
			if m[2] == ref {
				found = true
				break
			}
		}
		// remove if not found
		if !found {
			log.Printf("removing %s from cache\n", e)
			delete(*c, e)
		}
	}
}

// CleanAll cleans all sections of unknown references
// poolpaths and metrics are ignored as they will be cleaned real time
func (c *Cache) CleanAll(vcenter string, refs []string) {
	for e := range *c {
		// get back index parts
		m := strings.Split(e, "|")
		// check that we have tree parts or delete
		if len(m) != 3 {
			delete(*c, e)
		}
		// check vcenter and ignored sections
		if m[0] != vcenter || m[1] == "metrics" || m[1] == "poolpaths" {
			continue
		}
		// find the value in the range
		found := false
		for _, ref := range refs {
			if m[2] == ref {
				found = true
				break
			}
		}
		// remove if not found
		if !found {
			log.Printf("removing %s from cache\n", e)
			delete(*c, e)
		}
	}
}

// Purge purges a section of the cache
func (c *Cache) Purge(vcenter, section string) {
	for e := range *c {
		// get back index parts
		m := strings.Split(e, "|")
		// check that we have tree parts
		if len(m) != 3 {
			continue
		}
		// check vcenter and ignored sections
		if m[0] != vcenter || m[1] != section {
			continue
		}
		delete(*c, e)
	}
}

// lookup items in the cache
func (c *Cache) lookup(vcenter, section string) *map[string]interface{} {
	result := make(map[string]interface{})
	for e := range *c {
		// get back index parts
		m := strings.Split(e, "|")
		// check that we have tree parts
		if len(m) != 3 {
			continue
		}
		// check vcenter and ignored sections
		if m[0] != vcenter || m[1] != section {
			continue
		}
		result[m[2]] = (*c)[e]
	}
	return &result
}

// LookupString looks for items in the cache of type string
func (c *Cache) LookupString(vcenter, section string) *map[string]*string {
	result := make(map[string]*string)
	for key, val := range *c.lookup(vcenter, section) {
		if typed, ok := val.(*string); ok {
			result[key] = typed
		}
	}
	return &result
}

// LookupMorefs looks for items in the cache of type Morefs
func (c *Cache) LookupMorefs(vcenter, section string) *map[string]*[]types.ManagedObjectReference {
	result := make(map[string]*[]types.ManagedObjectReference)
	for key, val := range *c.lookup(vcenter, section) {
		if typed, ok := val.(*[]types.ManagedObjectReference); ok {
			result[key] = typed
		}
	}
	return &result
}

// FindHostAndCluster finds host and cluster of a host or a vm
func (c *Cache) FindHostAndCluster(vcenter, moref string) (string, string) {
	// get host
	if strings.HasPrefix(moref, "vm-") {
		// find host of the vm
		host := c.GetString(vcenter, "hosts", moref)
		if host == nil {
			log.Printf("VM %s has no host.\n", moref)
			return "", ""
		}
		moref = *host
	}
	// find hostname
	hostnameptr := cache.GetString(vcenter, "names", moref)
	hostname := ""
	if hostnameptr != nil {
		hostname = *hostnameptr
	}
	// find cluster
	cluster := cache.GetString(vcenter, "parents", moref)
	if cluster == nil {
		log.Printf("Host %s has no parents.\n", moref)
		return hostname, ""
	}
	if strings.HasPrefix(*cluster, "domain-s") {
		//ignore standalone hosts
		return hostname, ""
	}
	if !strings.HasPrefix(*cluster, "domain-c") {
		log.Printf("Host %s has no suitable parent %s.", moref, *cluster)
		return hostname, ""
	}
	clusternameptr := cache.GetString(vcenter, "names", *cluster)
	if clusternameptr == nil {
		return hostname, ""
	}
	return hostname, *clusternameptr
}

// FindString finds and return a string
func (c *Cache) FindString(vcenter, section, moref string) string {
	ptr := cache.GetString(vcenter, section, moref)
	if ptr == nil {
		return ""
	}
	return *ptr
}

// FindName finds an object in cache and resolves its name
func (c *Cache) FindName(vcenter, section, moref string) string {
	ptr := cache.GetString(vcenter, section, moref)
	if ptr == nil {
		return ""
	}
	return cache.FindString(vcenter, "names", *ptr)
}

// FindNames finds objects in cache and resolves their names
func (c *Cache) FindNames(vcenter, section, moref string) []string {
	ptr := cache.GetMorefs(vcenter, section, moref)
	if ptr == nil {
		return []string{}
	}
	if len(*ptr) == 0 {
		return []string{}
	}
	names := make([]string, len(*ptr))
	for _, mor := range *ptr {
		nptr := cache.GetString(vcenter, "names", mor.Value)
		if nptr == nil {
			continue
		}
		if len(*nptr) == 0 {
			continue
		}
		names = append(names, *nptr)
	}
	return names
}

// FindTags finds objects in cachee and create a tag array
func (c *Cache) FindTags(vcenter, moref string) []string {
	ptr := cache.GetTags(vcenter, "tags", moref)
	if ptr == nil {
		return []string{}
	}
	if len(*ptr) == 0 {
		return []string{}
	}
	tags := make([]string, len(*ptr))
	for _, tag := range *ptr {
		tags = append(tags, tag.Key)
	}
	return tags
}

// FindMetricName find metricname from cache
func (c *Cache) FindMetricName(vcenter string, id int32) string {
	ptr := cache.GetString(vcenter, "metrics", utils.ValToString(id, "", true))
	if ptr == nil {
		return ""
	}
	return *ptr
}
<commit_msg>use locks<commit_after>package vsphere

import (
	"bytes"
	"log"
	"strings"
	"sync"

	"github.com/cblomart/vsphere-graphite/utils"

	"github.com/vmware/govmomi/vim25/types"
)

// Cache will hold some informations in memory
type Cache map[string]interface{}

var lock = sync.RWMutex{}

// Gets an index
func index(vcenter, section, i string) string {
	var buffer bytes.Buffer
	_, err := buffer.WriteString(vcenter)
	if err != nil {
		return ""
	}
	_, err = buffer.WriteString("|")
	if err != nil {
		return ""
	}
	_, err = buffer.WriteString(section)
	if err != nil {
		return ""
	}
	_, err = buffer.WriteString("|")
	if err != nil {
		return ""
	}
	_, err = buffer.WriteString(i)
	if err != nil {
		return ""
	}
	return buffer.String()
}

// Add a value to the cache
func (c *Cache) Add(vcenter, section, i string, v interface{}) {
	if len(vcenter) == 0 || len(section) == 0 || len(i) == 0 || v == nil {
		return
	}
	switch typed := v.(type) {
	case string:
		c.add(vcenter, section, i, &typed)
	case []string:
		if len(typed) > 0 {
			c.add(vcenter, section, i, &typed)
		}
	case int32:
		c.add(vcenter, section, i, v)
	case types.ManagedObjectReference:
		c.add(vcenter, section, i, &(typed.Value))
	case types.ArrayOfManagedObjectReference:
		if len(typed.ManagedObjectReference) > 0 {
			c.add(vcenter, section, i, &(typed.ManagedObjectReference))
		}
	case types.ArrayOfTag:
		if len(typed.Tag) > 0 {
			c.add(vcenter, section, i, &(typed.Tag))
		}
	case types.ArrayOfGuestDiskInfo:
		if len(typed.GuestDiskInfo) > 0 {
			c.add(vcenter, section, i, &(typed.GuestDiskInfo))
		}
	default:
		log.Printf("Adding a unhandled type %T to cache for %s section %s and ref %s\n", v, vcenter, section, i)
	}
}

// add to the cache without type check
func (c *Cache) add(vcenter, section, i string, v interface{}) {
	lock.Lock()
	defer lock.Unlock()
	if v != nil {
		(*c)[index(vcenter, section, i)] = v
	}
}

// get a value from the cache
func (c *Cache) get(vcenter, section, i string) interface{} {
	lock.RLock()
	defer lock.RUnlock()
	if v, ok := (*c)[index(vcenter, section, i)]; ok {
		return v
	}
	return nil
}

// GetString gets a string from cache
func (c *Cache) GetString(vcenter, section, i string) *string {
	if v, ok := c.get(vcenter, section, i).(*string); ok {
		return v
	}
	return nil
}

// GetStrings gets an array of strings from cache
func (c *Cache) GetStrings(vcenter, section, i string) *[]string {
	if v, ok := c.get(vcenter, section, i).(*[]string); ok {
		return v
	}
	return nil
}

// GetInt32 get an int32 from cache
func (c *Cache) GetInt32(vcenter, section, i string) *int32 {
	if v, ok := c.get(vcenter, section, i).(*int32); ok {
		return v
	}
	return nil
}

// GetMoref gets a managed object reference from cache
func (c *Cache) GetMoref(vcenter, section, i string) *types.ManagedObjectReference {
	if v, ok := c.get(vcenter, section, i).(*types.ManagedObjectReference); ok {
		return v
	}
	return nil
}

// GetMorefs gets an array of managed references from cache
func (c *Cache) GetMorefs(vcenter, section, i string) *[]types.ManagedObjectReference {
	if v, ok := c.get(vcenter, section, i).(*[]types.ManagedObjectReference); ok {
		return v
	}
	return nil
}

// GetTags gets an array of vsphere tags from cache
func (c *Cache) GetTags(vcenter, section, i string) *[]types.Tag {
	if v, ok := c.get(vcenter, section, i).(*[]types.Tag); ok {
		return v
	}
	return nil
}

// GetDiskInfos gets an array of diskinfos from cache
func (c *Cache) GetDiskInfos(vcenter, section, i string) *[]types.GuestDiskInfo {
	if v, ok := c.get(vcenter, section, i).(*[]types.GuestDiskInfo); ok {
		return v
	}
	return nil
}

// Clean cache of unknows references
func (c *Cache) Clean(vcenter string, section string, refs []string) {
	lock.Lock()
	defer lock.Unlock()
	for e := range *c {
		// get back index parts
		m := strings.Split(e, "|")
		// check that we have tree parts or delete
		if len(m) != 3 {
			delete(*c, e)
		}
		// check vcenter
		if m[0] != vcenter || m[1] != section {
			continue
		}
		// find the value in the range
		found := false
		for _, ref := range refs {
			if m[2] == ref {
				found = true
				break
			}
		}
		// remove if not found
		if !found {
			log.Printf("removing %s from cache\n", e)
			delete(*c, e)
		}
	}
}

// CleanAll cleans all sections of unknown references
// poolpaths and metrics are ignored as they will be cleaned real time
func (c *Cache) CleanAll(vcenter string, refs []string) {
	lock.Lock()
	defer lock.Unlock()
	for e := range *c {
		// get back index parts
		m := strings.Split(e, "|")
		// check that we have tree parts or delete
		if len(m) != 3 {
			delete(*c, e)
		}
		// check vcenter and ignored sections
		if m[0] != vcenter || m[1] == "metrics" || m[1] == "poolpaths" {
			continue
		}
		// find the value in the range
		found := false
		for _, ref := range refs {
			if m[2] == ref {
				found = true
				break
			}
		}
		// remove if not found
		if !found {
			log.Printf("removing %s from cache\n", e)
			delete(*c, e)
		}
	}
}

// Purge purges a section of the cache
func (c *Cache) Purge(vcenter, section string) {
	lock.Lock()
	defer lock.Unlock()
	for e := range *c {
		// get back index parts
		m := strings.Split(e, "|")
		// check that we have tree parts
		if len(m) != 3 {
			continue
		}
		// check vcenter and ignored sections
		if m[0] != vcenter || m[1] != section {
			continue
		}
		delete(*c, e)
	}
}

// lookup items in the cache
func (c *Cache) lookup(vcenter, section string) *map[string]interface{} {
	lock.RLock()
	defer lock.RUnlock()
	result := make(map[string]interface{})
	for e := range *c {
		// get back index parts
		m := strings.Split(e, "|")
		// check that we have tree parts
		if len(m) != 3 {
			continue
		}
		// check vcenter and ignored sections
		if m[0] != vcenter || m[1] != section {
			continue
		}
		result[m[2]] = (*c)[e]
	}
	return &result
}

// LookupString looks for items in the cache of type string
func (c *Cache) LookupString(vcenter, section string) *map[string]*string {
	result := make(map[string]*string)
	for key, val := range *c.lookup(vcenter, section) {
		if typed, ok := val.(*string); ok {
			result[key] = typed
		}
	}
	return &result
}

// LookupMorefs looks for items in the cache of type Morefs
func (c *Cache) LookupMorefs(vcenter, section string) *map[string]*[]types.ManagedObjectReference {
	result := make(map[string]*[]types.ManagedObjectReference)
	for key, val := range *c.lookup(vcenter, section) {
		if typed, ok := val.(*[]types.ManagedObjectReference); ok {
			result[key] = typed
		}
	}
	return &result
}

// FindHostAndCluster finds host and cluster of a host or a vm
func (c *Cache) FindHostAndCluster(vcenter, moref string) (string, string) {
	// get host
	if strings.HasPrefix(moref, "vm-") {
		// find host of the vm
		host := c.GetString(vcenter, "hosts", moref)
		if host == nil {
			log.Printf("VM %s has no host.\n", moref)
			return "", ""
		}
		moref = *host
	}
	// find hostname
	hostnameptr := cache.GetString(vcenter, "names", moref)
	hostname := ""
	if hostnameptr != nil {
		hostname = *hostnameptr
	}
	// find cluster
	cluster := cache.GetString(vcenter, "parents", moref)
	if cluster == nil {
		log.Printf("Host %s has no parents.\n", moref)
		return hostname, ""
	}
	if strings.HasPrefix(*cluster, "domain-s") {
		//ignore standalone hosts
		return hostname, ""
	}
	if !strings.HasPrefix(*cluster, "domain-c") {
		log.Printf("Host %s has no suitable parent %s.", moref, *cluster)
		return hostname, ""
	}
	clusternameptr := cache.GetString(vcenter, "names", *cluster)
	if clusternameptr == nil {
		return hostname, ""
	}
	return hostname, *clusternameptr
}

// FindString finds and return a string
func (c *Cache) FindString(vcenter, section, moref string) string {
	ptr := cache.GetString(vcenter, section, moref)
	if ptr == nil {
		return ""
	}
	return *ptr
}

// FindName finds an object in cache and resolves its name
func (c *Cache) FindName(vcenter, section, moref string) string {
	ptr := cache.GetString(vcenter, section, moref)
	if ptr == nil {
		return ""
	}
	return cache.FindString(vcenter, "names", *ptr)
}

// FindNames finds objects in cache and resolves their names
func (c *Cache) FindNames(vcenter, section, moref string) []string {
	ptr := cache.GetMorefs(vcenter, section, moref)
	if ptr == nil {
		return []string{}
	}
	if len(*ptr) == 0 {
		return []string{}
	}
	names := make([]string, len(*ptr))
	for _, mor := range *ptr {
		nptr := cache.GetString(vcenter, "names", mor.Value)
		if nptr == nil {
			continue
		}
		if len(*nptr) == 0 {
			continue
		}
		names = append(names, *nptr)
	}
	return names
}

// FindTags finds objects in cachee and create a tag array
func (c *Cache) FindTags(vcenter, moref string) []string {
	ptr := cache.GetTags(vcenter, "tags", moref)
	if ptr == nil {
		return []string{}
	}
	if len(*ptr) == 0 {
		return []string{}
	}
	tags := make([]string, len(*ptr))
	for _, tag := range *ptr {
		tags = append(tags, tag.Key)
	}
	return tags
}

// FindMetricName find metricname from cache
func (c *Cache) FindMetricName(vcenter string, id int32) string {
	ptr := cache.GetString(vcenter, "metrics", utils.ValToString(id, "", true))
	if ptr == nil {
		return ""
	}
	return *ptr
}
<|endoftext|>